From 2215e12b5ff581e0bc5c6cbb89a55ce37c313da5 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Mon, 22 Jul 2024 10:32:56 +0800 Subject: [PATCH 001/791] =?UTF-8?q?[accuracy=5Ftools]atat=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E6=94=B9=E5=90=8D=E4=B8=BAmaprobe=E8=B5=84=E6=96=99?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 14 ++++++++++++-- debug/accuracy_tools/README.md | 3 ++- ...237\272\347\272\277\346\212\245\345\221\212.md" | 0 3 files changed, 14 insertions(+), 3 deletions(-) rename "debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" => "debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" (100%) diff --git a/README.md b/README.md index 014a4d59f..1b097fdfd 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,7 @@ # 变更通知 + + 原Ascend Training Tools工具更名为MindStudio Training Tools，MindStudio训练工具链。变更计划如下： 1. 2024.06.25本代码仓名称变更为mstt。 @@ -34,11 +36,19 @@ MindStudio Training Tools，MindStudio训练工具链。针对训练&大模型 ### [精度工具](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools) -1. [api_accuracy_checker（Ascend模型精度预检工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/api_accuracy_checker) +1. [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/cai-weiwei1989/att_all/tree/master/debug/accuracy_tools/msprobe)。 + + MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 + +2. [api_accuracy_checker（Ascend模型精度预检工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/api_accuracy_checker) + + 2024.09.30下线 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 -2. [ptdbg_ascend（PyTorch精度工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend) +3. [ptdbg_ascend（PyTorch精度工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend) + + 2024.09.30下线 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位PyTorch训练场景下的精度问题。 diff --git a/debug/accuracy_tools/README.md b/debug/accuracy_tools/README.md index 962736908..0445a0006 100644 --- a/debug/accuracy_tools/README.md +++ b/debug/accuracy_tools/README.md @@ -4,10 +4,11 @@ MindStudio Training Tools工具针对模型训练精度问题设计推出了一 ### 子功能介绍 -NPU上训练的网络存在精度问题，精度指标（loss或者具体的评价指标）与标杆相差较多。对于该场景的问题，可以使用**Ascend模型精度预检工具**或者**PyTorch精度工具**进行定位。 +NPU上训练的网络存在精度问题，精度指标（loss或者具体的评价指标）与标杆相差较多。对于该场景的问题，可以使用**MindStudio Probe工具**的精度预检工具和精度比对工具或**Ascend模型精度预检工具**、**PyTorch精度工具**进行定位。 | 工具名称 | 说明 | | ------------------------------------------------------------ | ------------------------------------------------------------ | +| [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/cai-weiwei1989/att_all/tree/master/debug/accuracy_tools/msprobe) | MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 | | [api_accuracy_checker（Ascend模型精度预检工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/api_accuracy_checker) | 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 | | [ptdbg_ascend（PyTorch精度工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend) | 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位PyTorch训练场景下的精度问题。 | diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" similarity index 100% rename from "debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" rename to "debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" -- Gitee From 0c7fa5f0cb6a04cff95e2564882abb7c86cc335e Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Mon, 22 Jul 2024 10:33:47 +0800 Subject: [PATCH 002/791] =?UTF-8?q?[accuracy=5Ftools]atat=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E6=94=B9=E5=90=8D=E4=B8=BAmaprobe=E8=B5=84=E6=96=99?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...50\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename "debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" => "debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" (100%) diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" similarity index 100% rename from "debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" rename to "debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" -- Gitee From 32a892fba6709cd9bd98a2a520bfab649fc8e12f Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Mon, 22 Jul 2024 10:40:58 +0800 Subject: [PATCH 003/791] =?UTF-8?q?[accuracy=5Ftools]atat=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E6=94=B9=E5=90=8D=E4=B8=BAmaprobe=E8=B5=84=E6=96=99?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- debug/accuracy_tools/README.md | 10 +++++----- ...345\237\272\347\272\277\346\212\245\345\221\212.md" | 0 3 files changed, 6 insertions(+), 6 deletions(-) rename "debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" => "debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" (100%) diff --git a/README.md b/README.md index 1b097fdfd..964ac19ee 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ MindStudio Training Tools，MindStudio训练工具链。针对训练&大模型 ### [精度工具](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools) -1. [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/cai-weiwei1989/att_all/tree/master/debug/accuracy_tools/msprobe)。 +1. [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe)。 MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 diff --git a/debug/accuracy_tools/README.md b/debug/accuracy_tools/README.md index 0445a0006..e28c1d34a 100644 --- a/debug/accuracy_tools/README.md +++ b/debug/accuracy_tools/README.md @@ -4,18 +4,18 @@ MindStudio Training Tools工具针对模型训练精度问题设计推出了一 ### 子功能介绍 -NPU上训练的网络存在精度问题，精度指标（loss或者具体的评价指标）与标杆相差较多。对于该场景的问题，可以使用**MindStudio Probe工具**的精度预检工具和精度比对工具或**Ascend模型精度预检工具**、**PyTorch精度工具**进行定位。 +NPU上训练的网络存在精度问题，精度指标（loss或者具体的评价指标）与标杆相差较多。对于该场景的问题，可以使用**MindStudio Probe工具**的**精度预检工具**和**精度比对工具**进行定位。 | 工具名称 | 说明 | | ------------------------------------------------------------ | ------------------------------------------------------------ | -| [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/cai-weiwei1989/att_all/tree/master/debug/accuracy_tools/msprobe) | MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 | +| [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe) | MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 | | [api_accuracy_checker（Ascend模型精度预检工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/api_accuracy_checker) | 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 | | [ptdbg_ascend（PyTorch精度工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend) | 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位PyTorch训练场景下的精度问题。 | ### 场景介绍 -**Ascend模型精度预检工具**会对全网每一个API根据其实际训练中的shape、dtype和数值范围生成随机的输入，对比它与标杆的输出差异，并指出输出差异过大不符合精度标准的API。该工具检查单API精度问题准确率超过80%，对比一般dump比对方法减少落盘数据量99%以上。具体使用请参见《[Ascend模型精度预检工具](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/api_accuracy_checker/README.md)》 +**精度预检工具**会对全网每一个API根据其实际训练中的shape、dtype和数值范围生成随机的输入，对比它与标杆的输出差异，并指出输出差异过大不符合精度标准的API。该工具检查单API精度问题准确率超过80%，对比一般dump比对方法减少落盘数据量99%以上。 -**PyTorch精度工具精度比对功能**可以对NPU整网API数据进行与CPU或GPU标杆数据的精度比对，从而检测精度问题。具体来说，dump统计量、分段dump、模块化dump，通讯算子dump等功能可以用较轻的数据量实现不同侧重的精度比对，从而定位精度问题。具体使用请参见《[ptdbg_ascend精度工具功能说明](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend/doc)》。 +**精度比对功能**可以对NPU整网API数据进行与CPU或GPU标杆数据的精度比对，从而检测精度问题。具体来说，dump统计量、分段dump、模块化dump，通讯算子dump等功能可以用较轻的数据量实现不同侧重的精度比对，从而定位精度问题。 -**PyTorch精度工具溢出检测功能**是在判断训练网络可能存在溢出现象时，例如某个step的loss突然变成inf nan，或者混精场景下loss_scale不断减小，可以通过ptdbg_ascend的精度检测工具检测API的溢出情况。具体使用请参见《[ptdbg_ascend精度工具功能说明](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend/doc)》。 \ No newline at end of file +**溢出检测功能**是在判断训练网络可能存在溢出现象时，例如某个step的loss突然变成inf nan，或者混精场景下loss_scale不断减小，可以通过ptdbg_ascend的精度检测工具检测API的溢出情况。 \ No newline at end of file diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" similarity index 100% rename from "debug/accuracy_tools/msprobe/pytorch/doc/atat\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" rename to "debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" -- Gitee From 43c3df4bb77e376dcd4e4ea2ad2ea101b99b139a Mon Sep 17 00:00:00 2001 From: l30036321 Date: Thu, 18 Jul 2024 17:16:07 +0800 Subject: [PATCH 004/791] mindspore pynative dump --- .../core/data_dump/data_processor/base.py | 4 +- .../core/data_dump/data_processor/factory.py | 6 +- .../data_processor/mindspore_processor.py | 108 ++ .../data_processor/pytorch_processor.py | 7 +- .../atat/mindspore/common/log.py | 34 + .../atat/mindspore/common/utils.py | 9 + .../mindspore/debugger/debugger_config.py | 8 +- .../mindspore/debugger/precision_debugger.py | 23 +- .../mindspore/dump/hook_cell/api_registry.py | 93 ++ .../mindspore/dump/hook_cell/hook_cell.py | 55 ++ .../dump/hook_cell/support_wrap_ops.yaml | 925 ++++++++++++++++++ .../dump/hook_cell/wrap_functional.py | 90 ++ .../mindspore/dump/hook_cell/wrap_tensor.py | 65 ++ .../accuracy_tools/atat/mindspore/service.py | 132 +++ debug/accuracy_tools/atat/pytorch/service.py | 2 +- 15 files changed, 1547 insertions(+), 14 deletions(-) create mode 100644 debug/accuracy_tools/atat/core/data_dump/data_processor/mindspore_processor.py create mode 100644 debug/accuracy_tools/atat/mindspore/common/log.py create mode 100644 debug/accuracy_tools/atat/mindspore/common/utils.py create mode 100644 debug/accuracy_tools/atat/mindspore/dump/hook_cell/api_registry.py create mode 100644 debug/accuracy_tools/atat/mindspore/dump/hook_cell/hook_cell.py create mode 100644 debug/accuracy_tools/atat/mindspore/dump/hook_cell/support_wrap_ops.yaml create mode 100644 debug/accuracy_tools/atat/mindspore/dump/hook_cell/wrap_functional.py create mode 100644 debug/accuracy_tools/atat/mindspore/dump/hook_cell/wrap_tensor.py create mode 100644 debug/accuracy_tools/atat/mindspore/service.py diff --git a/debug/accuracy_tools/atat/core/data_dump/data_processor/base.py b/debug/accuracy_tools/atat/core/data_dump/data_processor/base.py index 1ee3314b3..a85afb60f 100644 --- a/debug/accuracy_tools/atat/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/atat/core/data_dump/data_processor/base.py @@ -141,9 +141,11 @@ class BaseDataProcessor: resutl_dict[k] = cls.recursive_apply_transform(arg, transform) cls._recursive_key_stack.pop() return resutl_dict - else: + elif args is not None: logger.warning(f"Data type {type(args)} is not supported.") return None + else: + return None def if_return_forward_new_output(self): return self._return_forward_new_output diff --git a/debug/accuracy_tools/atat/core/data_dump/data_processor/factory.py b/debug/accuracy_tools/atat/core/data_dump/data_processor/factory.py index 00f2f72e7..22529f560 100644 --- a/debug/accuracy_tools/atat/core/data_dump/data_processor/factory.py +++ b/debug/accuracy_tools/atat/core/data_dump/data_processor/factory.py @@ -51,11 +51,7 @@ class DataProcessorFactory: elif framework == Const.MS_FRAMEWORK: from .mindspore_processor import ( StatisticsDataProcessor as MindsporeStatisticsDataProcessor, - TensorDataProcessor as MindsporeTensorDataProcessor, - OverflowCheckDataProcessor as MindsporeOverflowCheckDataProcessor, - FreeBenchmarkDataProcessor as MindsporeFreeBenchmarkDataProcessor + TensorDataProcessor as MindsporeTensorDataProcessor ) cls.register_processor(Const.MS_FRAMEWORK, Const.STATISTICS, MindsporeStatisticsDataProcessor) cls.register_processor(Const.MS_FRAMEWORK, Const.TENSOR, MindsporeTensorDataProcessor) - cls.register_processor(Const.MS_FRAMEWORK, Const.OVERFLOW_CHECK, MindsporeOverflowCheckDataProcessor) - cls.register_processor(Const.MS_FRAMEWORK, Const.FREE_BENCHMARK, MindsporeFreeBenchmarkDataProcessor) \ No newline at end of file diff --git a/debug/accuracy_tools/atat/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/atat/core/data_dump/data_processor/mindspore_processor.py new file mode 100644 index 000000000..02d95d1d4 --- /dev/null +++ b/debug/accuracy_tools/atat/core/data_dump/data_processor/mindspore_processor.py @@ -0,0 +1,108 @@ +import os +import zlib +import mindspore as ms +import numpy as np + +from atat.core.common.utils import Const +from atat.core.data_dump.data_processor.base import BaseDataProcessor, TensorStatInfo +from atat.core.common.log import logger +from atat.core.common.file_check import path_len_exceeds_limit, change_mode, FileCheckConst +from atat.mindspore.dump.hook_cell.wrap_functional import ops_func, mint_ops_func + + +class MindsporeDataProcessor(BaseDataProcessor): + mindspore_special_type = tuple([ms.Tensor]) + + def __init__(self, config, data_writer): + super().__init__(config, data_writer) + self.mindspore_object_key = { + "dtype": self.analyze_dtype_in_kwargs + } + + @staticmethod + def get_md5_for_tensor(x): + if x.dtype == ms.bfloat16: + x = x.to(ms.float32) + tensor_bytes = x.asnumpy().tobytes() + crc32_hash = zlib.crc32(tensor_bytes) + return f"{crc32_hash:08x}" + + @staticmethod + def analyze_dtype_in_kwargs(element): + single_arg = {} + single_arg.update({"type": "mindspore.dtype"}) + single_arg.update({"value": str(element)}) + return single_arg + + @staticmethod + def get_stat_info(data): + tensor_stat = TensorStatInfo() + if data.numel() == 0: + return tensor_stat + elif data.dtype == ms.bool_: + tensor_stat.max = mint_ops_func["max"](data).item() + tensor_stat.min = mint_ops_func["min"](data).item() + elif not data.shape: + tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data.item() + else: + tensor_stat.max = mint_ops_func["max"](data).item() + tensor_stat.min = mint_ops_func["min"](data).item() + tensor_stat.mean = mint_ops_func["mean"](data).item() + tensor_stat.norm = ops_func["norm"](data).item() + return tensor_stat + + @classmethod + def get_special_types(cls): + return super().get_special_types() + cls.mindspore_special_type + + def _analyze_tensor(self, tensor, suffix): + tensor_stat = self.get_stat_info(tensor) + tensor_json = {} + tensor_json.update({'type': 'mindspore.Tensor'}) + tensor_json.update({'dtype': str(tensor.dtype)}) + tensor_json.update({"shape": tensor.shape}) + tensor_json.update({"Max": tensor_stat.max}) + tensor_json.update({"Min": tensor_stat.min}) + tensor_json.update({"Mean": tensor_stat.mean}) + tensor_json.update({"Norm": tensor_stat.norm}) + if self.config.summary_mode == "md5": + tensor_md5 = self.get_md5_for_tensor(tensor) + tensor_json.update({"md5": tensor_md5}) + return tensor_json + + def analyze_single_element(self, element, suffix_stack): + if suffix_stack and suffix_stack[-1] in self.mindspore_object_key: + return self.mindspore_object_key[suffix_stack[-1]](element) + + converted_numpy, numpy_type = self._convert_numpy_to_builtin(element) + if converted_numpy is not element: + return self._analyze_numpy(converted_numpy, numpy_type) + if isinstance(element, ms.Tensor): + return self._analyze_tensor(element, Const.SEP.join(suffix_stack)) + + if isinstance(element, (bool, int, float, str, slice)): + return self._analyze_builtin(element) + return None + + def analyze_element(self, element): + return self.recursive_apply_transform(element, self.analyze_single_element) + + +class StatisticsDataProcessor(MindsporeDataProcessor): + pass + + +class TensorDataProcessor(MindsporeDataProcessor): + def _analyze_tensor(self, tensor, suffix): + dump_data_name, file_path = self.get_save_file_path(suffix) + single_arg = super()._analyze_tensor(tensor, suffix) + single_arg.update({"data_name": dump_data_name}) + if not path_len_exceeds_limit(file_path): + if tensor.dtype == ms.bfloat16: + tensor = tensor.to(ms.float32) + saved_tensor = tensor.asnumpy() + np.save(file_path, saved_tensor) + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + else: + logger.warning(f'The file path {file_path} length exceeds limit.') + return single_arg diff --git a/debug/accuracy_tools/atat/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/atat/core/data_dump/data_processor/pytorch_processor.py index 9f96635e9..66e16b580 100644 --- a/debug/accuracy_tools/atat/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/atat/core/data_dump/data_processor/pytorch_processor.py @@ -77,6 +77,10 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_stat.norm = torch._C._VariableFunctionsClass.norm(data_clone).item() return tensor_stat + @staticmethod + def _analyze_torch_size(arg): + return {"type": "torch.Size", "value": list(arg)} + @classmethod def get_special_types(cls): return super().get_special_types() + cls.pytorch_special_type @@ -98,9 +102,6 @@ class PytorchDataProcessor(BaseDataProcessor): def analyze_element(self, element): return self.recursive_apply_transform(element, self.analyze_single_element) - def _analyze_torch_size(arg): - return {"type": "torch.Size", "value": list(arg)} - def _analyze_tensor(self, tensor, suffix): tensor_stat = self.get_stat_info(tensor) tensor_json = {} diff --git a/debug/accuracy_tools/atat/mindspore/common/log.py b/debug/accuracy_tools/atat/mindspore/common/log.py new file mode 100644 index 000000000..0bcd1e5b8 --- /dev/null +++ b/debug/accuracy_tools/atat/mindspore/common/log.py @@ -0,0 +1,34 @@ +import os +import time +import sys + +from atat.mindspore.common.utils import get_rank_if_initialized +from atat.core.common.log import BaseLogger +from atat.core.common.exceptions import DistributedNotInitializedError + + +class MindsporeLogger(BaseLogger): + def __init__(self): + super().__init__() + + def get_rank(self): + try: + current_rank = get_rank_if_initialized() + except DistributedNotInitializedError: + current_rank = None + + return current_rank + + def _print_log(self, level, msg, end='\n'): + current_rank = self.get_rank() + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + pid = os.getpid() + if current_rank is not None: + full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}" + else: + full_msg = f"{current_time} ({pid}) [{level}] {msg}" + print(full_msg, end=end) + sys.stdout.flush() + + +logger = MindsporeLogger() \ No newline at end of file diff --git a/debug/accuracy_tools/atat/mindspore/common/utils.py b/debug/accuracy_tools/atat/mindspore/common/utils.py new file mode 100644 index 000000000..c326a2abf --- /dev/null +++ b/debug/accuracy_tools/atat/mindspore/common/utils.py @@ -0,0 +1,9 @@ +import mindspore +from atat.core.common.exceptions import DistributedNotInitializedError + + +def get_rank_if_initialized(): + if mindspore.communication.GlobalComm.INITED: + return mindspore.communication.get_rank() + else: + raise DistributedNotInitializedError("mindspore distributed environment is not initialized") diff --git a/debug/accuracy_tools/atat/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/atat/mindspore/debugger/debugger_config.py index 56a4b9bf7..fed9d0a1c 100644 --- a/debug/accuracy_tools/atat/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/atat/mindspore/debugger/debugger_config.py @@ -1,4 +1,5 @@ import os +from atat.core.common.utils import Const class DebuggerConfig: @@ -16,17 +17,20 @@ class DebuggerConfig: if not common_config.level: common_config.level = "L1" self.level = DebuggerConfig.convert_map[common_config.level] + self.level_ori = common_config.level self.list = [] if not task_config.list else task_config.list + self.scope =[] if not task_config.scope else task_config.scope self.data_mode = [] if not task_config.data_mode else task_config.data_mode self.file_format = task_config.file_format self.check_mode = task_config.check_mode - + self.framework = Const.MS_FRAMEWORK + self.summary_mode = task_config.summary_mode self.check() def check(self): if not self.dump_path: raise Exception("Dump path is empty.") - if not os.path.isabs(self.dump_path): + if self.level_ori != "L1" and not os.path.isabs(self.dump_path): raise Exception("Dump path must be absolute path.") if not self.task: self.task = "statistics" diff --git a/debug/accuracy_tools/atat/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/atat/mindspore/debugger/precision_debugger.py index 009907476..31b6f4f94 100644 --- a/debug/accuracy_tools/atat/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/atat/mindspore/debugger/precision_debugger.py @@ -1,7 +1,9 @@ import os +import mindspore as ms from atat.mindspore.ms_config import parse_json_config from atat.mindspore.debugger.debugger_config import DebuggerConfig from atat.mindspore.task_handler_factory import TaskHandlerFactory +from atat.mindspore.service import Service class PrecisionDebugger: @@ -22,11 +24,28 @@ class PrecisionDebugger: common_config, task_config = parse_json_config(config_path) self.config = DebuggerConfig(common_config, task_config) self.initialized = True + self.service = Service(self.config) @classmethod def start(cls, target=None): instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") - handler = TaskHandlerFactory.create(instance.config) - handler.handle() + if ms.get_context("mode") == 1 and instance.config.level_ori == "L1": + instance.service.start(target) + else: + handler = TaskHandlerFactory.create(instance.config) + handler.handle() + + @classmethod + def stop(cls): + instance = cls._instance + if not instance: + raise Exception("PrecisionDebugger instance is not created.") + instance.service.stop() + + @classmethod + def step(cls): + if not cls._instance: + raise Exception("PrecisionDebugger instance is not created.") + cls._instance.service.step() \ No newline at end of file diff --git a/debug/accuracy_tools/atat/mindspore/dump/hook_cell/api_registry.py b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/api_registry.py new file mode 100644 index 000000000..71c51d60f --- /dev/null +++ b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/api_registry.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2023-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import mindspore as ms +from atat.mindspore.dump.hook_cell import wrap_tensor, wrap_functional +from atat.mindspore.dump.hook_cell.wrap_functional import get_functional_ops +from atat.mindspore.dump.hook_cell.wrap_tensor import get_tensor_ops +from atat.core.common.utils import Const + + +class ApiRegistry: + def __init__(self): + self.tensor_ori_attr = {} + self.functional_ori_attr = {} + self.mint_ops_ori_attr = {} + self.mint_func_ops_ori_attr = {} + + self.tensor_hook_attr = {} + self.functional_hook_attr = {} + self.mint_ops_hook_attr = {} + self.mint_func_ops_hook_attr = {} + + @staticmethod + def store_ori_attr(ori_api_group, api_list, api_ori_attr): + for api in api_list: + if Const.SEP in api: + sub_module_name, sub_op = api.rsplit(Const.SEP, 1) + sub_module = getattr(ori_api_group, sub_module_name) + api_ori_attr[api] = getattr(sub_module, sub_op) + else: + api_ori_attr[api] = getattr(ori_api_group, api) + + @staticmethod + def set_api_attr(api_group, attr_dict): + for api, api_attr in attr_dict.items(): + if Const.SEP in api: + sub_module_name, sub_op = api.rsplit(Const.SEP, 1) + sub_module = getattr(api_group, sub_module_name, None) + if sub_module is not None: + setattr(sub_module, sub_op, api_attr) + else: + setattr(api_group, api, api_attr) + + def api_modularity(self): + self.set_api_attr(ms.Tensor, self.tensor_hook_attr) + self.set_api_attr(ms.ops, self.functional_hook_attr) + self.set_api_attr(ms.mint, self.mint_ops_hook_attr) + self.set_api_attr(ms.mint.nn.functional, self.mint_func_ops_hook_attr) + + def api_originality(self): + self.set_api_attr(ms.Tensor, self.tensor_ori_attr) + self.set_api_attr(ms.ops, self.functional_ori_attr) + self.set_api_attr(ms.mint, self.mint_ops_ori_attr) + self.set_api_attr(ms.mint.nn.functional, self.mint_func_ops_ori_attr) + + def initialize_hook(self, hook): + self.store_ori_attr(ms.Tensor, get_tensor_ops(), self.tensor_ori_attr) + wrap_tensor.wrap_tensor_ops_and_bind(hook) + for attr_name in dir(wrap_tensor.HOOKTensor): + if attr_name.startswith("wrap_"): + self.tensor_hook_attr[attr_name[5:]] = getattr(wrap_tensor.HOOKTensor, attr_name) + + functional_ops, mint_ops, mint_func_ops = get_functional_ops() + self.store_ori_attr(ms.ops, functional_ops, self.functional_ori_attr) + self.store_ori_attr(ms.mint, mint_ops, self.mint_ops_ori_attr) + self.store_ori_attr(ms.mint.nn.functional, mint_func_ops, self.mint_func_ops_ori_attr) + wrap_functional.setup_hooks(hook) + for attr_name in dir(wrap_functional.HOOKFunctionalOP): + if attr_name.startswith("wrap_"): + self.functional_hook_attr[attr_name[5:]] = getattr(wrap_functional.HOOKFunctionalOP, attr_name) + for attr_name in dir(wrap_functional.HOOKMintOP): + if attr_name.startswith("wrap_"): + self.mint_ops_hook_attr[attr_name[5:]] = getattr(wrap_functional.HOOKMintOP, attr_name) + for attr_name in dir(wrap_functional.HOOKMintNNFunctionalOP): + if attr_name.startswith("wrap_"): + self.mint_func_ops_hook_attr[attr_name[5:]] = getattr(wrap_functional.HOOKMintNNFunctionalOP, attr_name) + + +api_register = ApiRegistry() diff --git a/debug/accuracy_tools/atat/mindspore/dump/hook_cell/hook_cell.py b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/hook_cell.py new file mode 100644 index 000000000..3d72b352f --- /dev/null +++ b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/hook_cell.py @@ -0,0 +1,55 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from collections import defaultdict + +from mindspore import nn + +cell_count = defaultdict(int) +g_stop_hook = False + + +class HOOKCell(nn.Cell): + + def __init__(self, build_hook) -> None: + super(HOOKCell, self).__init__() + self.changed_status = False + self.input_kwargs = {} + self.prefix = "" + global g_stop_hook + if not g_stop_hook: + g_stop_hook = True + self.changed_status = True + if hasattr(self, "prefix_op_name_"): + self.prefix = self.prefix_op_name_ + + cell_count[self.prefix] += 1 + self.prefix = self.prefix + str(cell_count[self.prefix] - 1) + '.' + forward_hook, backward_hook = build_hook(self.prefix) + self.register_forward_hook(forward_hook) + self.register_backward_hook(backward_hook) + + # 重载call，加全局标志。 + def __call__(self, *args, **kwargs): + try: + self.input_kwargs = kwargs + out = super(HOOKCell, self).__call__(*args, **kwargs) + except Exception as e: + raise e + finally: + if self.changed_status: + self.changed_status = False + global g_stop_hook + g_stop_hook = False + return out diff --git a/debug/accuracy_tools/atat/mindspore/dump/hook_cell/support_wrap_ops.yaml b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/support_wrap_ops.yaml new file mode 100644 index 000000000..e4cc5cf32 --- /dev/null +++ b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/support_wrap_ops.yaml @@ -0,0 +1,925 @@ +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +# List of ops that register hooks + + +ops: + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d + - adaptive_max_pool1d + - adaptive_max_pool2d + - avg_pool1d + - avg_pool2d + - avg_pool3d + - batch_norm + - bias_add + - ctc_greedy_decoder + - conv1d + - conv2d + - conv3d + - deformable_conv2d + - dense + - dropout + - dropout1d + - dropout2d + - dropout3d + - flatten + - fold + - fractional_max_pool3d + - lp_pool1d + - lp_pool2d + - lrn + - max_pool2d + - max_pool3d + - max_unpool1d + - max_unpool2d + - max_unpool3d + - unfold + - binary_cross_entropy + - binary_cross_entropy_with_logits + - cosine_embedding_loss + - cross_entropy + - ctc_loss + - gaussian_nll_loss + - hinge_embedding_loss + - huber_loss + - kl_div + - l1_loss + - margin_ranking_loss + - mse_loss + - multi_margin_loss + - multilabel_margin_loss + - multilabel_soft_margin_loss + - nll_loss + - smooth_l1_loss + - triplet_margin_loss + - elu + - fast_gelu + - gelu + - glu + - gumbel_softmax + - hardshrink + - hardsigmoid + - hardswish + - hardtanh + - leaky_relu + - log_softmax + - logsigmoid + - mish + - prelu + - relu + - relu6 + - celu + - rrelu + - selu + - sigmoid + - silu + - softmax + - softmin + - softshrink + - softsign + - tanh + - threshold + - cdist + - dist + - pdist + - choice_with_mask + - random_categorical + - log_uniform_candidate_sampler + - uniform_candidate_sampler + - affine_grid + - bounding_box_decode + - bounding_box_encode + - col2im + - check_valid + - crop_and_resize + - grid_sample + - interpolate + - iou + - pad + - padding + - pixel_shuffle + - pixel_unshuffle + - upsample + - abs + - absolute + - accumulate_n + - acos + - arccos + - acosh + - add + - addcdiv + - addcmul + - addmv + - addn + - angle + - arccosh + - arcsin + - arcsinh + - arctan + - arctanh + - arctan2 + - asin + - asinh + - atan + - atan2 + - atanh + - atleast_1d + - atleast_2d + - atleast_3d + - bessel_i0 + - bessel_i0e + - bessel_i1 + - bessel_i1e + - bessel_j0 + - bessel_j1 + - bessel_k0 + - bessel_k0e + - bessel_k1 + - bessel_k1e + - bessel_y0 + - bessel_y1 + - bitwise_and + - bitwise_left_shift + - bitwise_or + - bitwise_right_shift + - bitwise_xor + - ceil + - clamp + - clip + - combinations + - copysign + - cos + - cosh + - cosine_similarity + - cov + - diag_embed + - diff + - deg2rad + - digamma + - div + - divide + - erf + - erfc + - erfinv + - exp + - exp2 + - expm1 + - floor + - floor_div + - floor_mod + - float_power + - fmod + - frac + - gcd + - hypot + - igamma + - igammac + - imag + - i0 + - inv + - invert + - lcm + - ldexp + - lerp + - log + - log2 + - log10 + - log1p + - logaddexp + - logaddexp2 + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - mul + - multiply + - mvlgamma + - neg + - negative + - nextafter + - polar + - polygamma + - positive + - pow + - rad2deg + - ravel + - real + - reciprocal + - remainder + - rot90 + - round + - rsqrt + - sgn + - sign + - signbit + - sin + - sinc + - sinh + - sqrt + - square + - sub + - subtract + - t + - tan + - tanhshrink + - trapz + - tril_indices + - triu_indices + - true_divide + - trunc + - truncate_div + - truncate_mod + - xdivy + - xlogy + - zeta + - all + - amax + - amin + - aminmax + - any + - argmax + - argmin + - cummax + - cummin + - cumprod + - cumsum + - fmax + - histc + - logsumexp + - max + - mean + - median + - min + - norm + - prod + - std + - std_mean + - var + - var_mean + - argsort + - approximate_equal + - equal + - ge + - greater + - greater_equal + - gt + - intopk + - isclose + - isfinite + - isinf + - isnan + - isneginf + - isposinf + - isreal + - is_complex + - le + - less + - less_equal + - lt + - maximum + - minimum + - msort + - ne + - not_equal + - searchsorted + - topk + - bmm + - addbmm + - addmm + - baddbmm + - addr + - adjoint + - cholesky + - cholesky_solve + - batch_dot + - dot + - eig + - inner + - inverse + - geqrf + - ger + - kron + - lu_solve + - lu_unpack + - matmul + - matrix_solve + - matrix_band_part + - matrix_diag + - matrix_diag_part + - matrix_set_diag + - mm + - mv + - outer + - orgqr + - ormqr + - pinv + - svd + - tensor_dot + - logdet + - slogdet + - qr + - trace + - bartlett_window + - blackman_window + - hamming_window + - hann_window + - kaiser_window + - eye + - fill + - full + - full_like + - linspace + - logspace + - one_hot + - arange + - range + - heaviside + - bernoulli + - gamma + - laplace + - multinomial + - multinomial_with_replacement + - rand + - rand_like + - randint + - randint_like + - randn + - randn_like + - random_gamma + - random_poisson + - randperm + - standard_laplace + - standard_normal + - uniform + - argwhere + - batch_to_space_nd + - bincount + - block_diag + - broadcast_to + - cat + - channel_shuffle + - chunk + - column_stack + - concat + - conj + - count_nonzero + - deepcopy + - diag + - diagflat + - diagonal + - dyn_shape + - dsplit + - dstack + - einsum + - expand + - expand_dims + - flip + - fliplr + - flipud + - gather_d + - gather_elements + - gather_nd + - hsplit + - hstack + - index_add + - index_fill + - index_select + - inplace_add + - inplace_index_add + - inplace_sub + - inplace_update + - masked_fill + - masked_select + - meshgrid + - moveaxis + - movedim + - narrow + - nan_to_num + - nansum + - normal + - nonzero + - population_count + - rank + - repeat_elements + - repeat_interleave + - reshape + - reverse + - reverse_sequence + - roll + - scatter + - scatter_nd + - select + - sequence_mask + - shuffle + - size + - slice + - sort + - space_to_batch_nd + - sparse_segment_mean + - split + - squeeze + - stack + - strided_slice + - sum + - swapaxes + - swapdims + - tensor_scatter_add + - tensor_scatter_div + - tensor_scatter_max + - tensor_scatter_min + - tensor_scatter_mul + - tensor_scatter_sub + - tensor_scatter_elements + - tensor_split + - tile + - tril + - triu + - transpose + - unbind + - unique + - unique_consecutive + - unique_with_pad + - unsorted_segment_max + - unsorted_segment_min + - unsorted_segment_prod + - unsorted_segment_sum + - unsqueeze + - unstack + - view_as_real + - vsplit + - vstack + - where + - cross + - renorm + - is_tensor + - scalar_cast + - scalar_to_tensor + - tuple_to_array + - clip_by_global_norm + - clip_by_value + - assign + - assign_add + - assign_sub + - scatter_add + - scatter_div + - scatter_max + - scatter_min + - scatter_mul + - scatter_nd_add + - scatter_nd_div + - scatter_nd_max + - scatter_nd_min + - scatter_nd_mul + - scatter_nd_sub + - scatter_update + - derivative + - jet + +tensor: + - __abs__ + - __add__ + - __and__ + - __bool__ + - __eq__ + - __ge__ + - __gt__ + - __iadd__ + - __ifloordiv__ + - __imatmul__ + - __imod__ + - __imul__ + - __isub__ + - __le__ + - __lt__ + - __matmul__ + - __mod__ + - __mul__ + - __ne__ + - __neg__ + - __or__ + - __pow__ + - __radd__ + - __rmatmul__ + - __rmod__ + - __rmul__ + - __rpow__ + - __rsub__ + - __sub__ + - __truediv__ + - __xor__ + - abs + - absolute + - acos + - acosh + - add + - addbmm + - addcdiv + - addcmul + - addmm + - addmv + - addr + - all + - amax + - amin + - any + - arccos + - arccosh + - argmax + - angle + - arcsin + - arcsinh + - arctan + - arctanh + - argmin + - argsort + - asin + - asinh + - atan + - atan2 + - atanh + - baddbmm + - bernoulli + - bincount + - bitwise_and + - bitwise_or + - bitwise_xor + - bmm + - bool + - broadcast_to + - ceil + - cholesky_solve + - cholesky + - clamp + - clip + - conj + - copysign + - cos + - cosh + - cross + - cummax + - cummin + - cumprod + - cumsum + - deg2rad + - diag + - diagflat + - diff + - digamma + - div + - divide + - equal + - erf + - erfc + - erfinv + - exp + - expand_as + - expm1 + - flip + - fliplr + - flipud + - float_power + - floor + - fmod + - frac + - gather_elements + - ge + - geqrf + - ger + - greater + - greater_equal + - gt + - half + - hardshrink + - heaviside + - histc + - hypot + - i0 + - igamma + - igammac + - imag + - index_add + - index_fill + - index_put + - index_select + - inner + - int + - inverse + - isclose + - isfinite + - isinf + - isnan + - is_complex + - is_signed + - isneginf + - isposinf + - isreal + - lcm + - ldexp + - le + - lerp + - less + - less_equal + - log + - log10 + - log1p + - log2 + - logaddexp + - logaddexp2 + - logdet + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - logsumexp + - long + - lt + - masked_fill + - masked_scatter + - masked_select + - matmul + - max + - maximum + - mean + - median + - min + - minimum + - moveaxis + - movedim + - msort + - multinomial + - multiply + - mvlgamma + - nan_to_num + - nansum + - narrow + - ne + - neg + - negative + - nelement + - new_ones + - new_zeros + - nextafter + - norm + - nonzero + - not_equal + - ormqr + - permute + - pow + - prod + - qr + - ravel + - real + - reciprocal + - remainder + - renorm + - rad2deg + - tile + - repeat_interleave + - reshape + - reshape + - round + - rot90 + - rsqrt + - sum_to_size + - scatter + - sgn + - short + - sigmoid + - sign + - signbit + - sin + - sinc + - sinh + - slogdet + - sort + - split + - sqrt + - square + - squeeze + - std + - subtract + - subtract + - svd + - swapaxes + - swapdims + - t + - take + - tan + - tanh + - trace + - swapaxes + - tile + - to + - topk + - tril + - tensor_split + - transpose + - true_divide + - trunc + - unbind + - unique_consecutive + - unsqueeze + - var + - view + - where + - xlogy + - from_numpy + - std + - take + - var + - all + - any + - copy + - diagonal + - flatten + - resize + - sum + +mint.ops: + - abs + - absolute_import + - add + - add_ex + - all + - any + - any_ex + - arange + - argmax + - avg_pool2d + - baddbmm + - baddbmm_ex + - batch_norm + - binary_cross_entropy_with_logits + - bitwise_and + - bitwise_or + - bitwise_xor + - bmm + - broadcast_to + - cat + - cat_ex + - ceil + - chunk + - clamp + - conv2d + - conv_transpose2d + - cos + - cross + - cummax + - cummin + - cumsum + - div + - divide + - dropout + - embedding + - eq + - erf + - erfinv + - exp + - flatten + - flip + - flip_ex + - fold + - full + - functional + - gather + - gelu + - greater + - grid_sample + - group_norm + - gt + - index_select + - interpolate + - isclose + - isfinite + - layer_norm + - le + - leaky_relu + - less + - less_equal + - linear + - linspace + - log + - logical_and + - logical_not + - logical_or + - lt + - masked_select + - matmul + - max + - max_pool2d + - maximum + - mean + - mean_ex + - min + - minimum + - mul + - ne + - neg + - negative + - nn + - nonzero + - normal + - one_hot + - ones + - ones_ex + - ones_like + - pad + - permute + - permute_ex + - pow + - prod + - reciprocal + - relu + - remainder + - repeat_interleave + - rsqrt + - scatter + - scatter_add + - searchsorted + - sigmoid + - silu + - sin + - softmax + - softplus + - sort + - split + - sqrt + - sqrt_ex + - square + - stack + - sub + - sub_ex + - sum + - tanh + - tile + - topk + - tril + - triu + - unfold + - unique + - where + - xlogy + - zeros + - zeros_ex + - zeros_like + +mint.nn: + - Dropout + - Embedding + - Fold + - LayerNorm + - Linear + - MaxPool2d + - Unfold + - Upsample + +mint.nn.functional: + - absolute_import + - avg_pool2d + - batch_norm + - batch_norm_ex + - bce_with_logits + - binary_cross_entropy_with_logits + - conv_transpose2d + - dense + - dropout + - embedding + - fold + - gelu + - grid_sample + - group_norm + - interpolate + - layer_norm + - leaky_relu + - linear + - max_pool2d + - max_pool2d_ex + - normal + - one_hot + - one_hot_ext + - pad + - relu + - sigmoid + - silu + - softmax + - softmax_ex + - softplus + - tanh + - unfold diff --git a/debug/accuracy_tools/atat/mindspore/dump/hook_cell/wrap_functional.py b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/wrap_functional.py new file mode 100644 index 000000000..4eef7deef --- /dev/null +++ b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/wrap_functional.py @@ -0,0 +1,90 @@ +# Copyright 2023 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import yaml +import mindspore as ms +from atat.mindspore.dump.hook_cell.hook_cell import HOOKCell +from atat.core.common.utils import Const +from atat.core.common.file_check import FileOpen + + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") + +ops_func = {f: getattr(ms.ops, f) for f in dir(ms.ops)} +mint_ops_func = {f: getattr(ms.mint, f) for f in dir(ms.mint)} +mint_func_ops_func = {f: getattr(ms.mint.nn.functional, f) for f in dir(ms.mint.nn.functional)} + + +def get_functional_ops(): + global ops_func, mint_ops_func, mint_func_ops_func + with FileOpen(yaml_path, 'r') as f: + config = yaml.safe_load(f) + WrapFunctionalOps = config.get("ops") + WrapMintOps = config.get("mint.ops") + WrapMintFunctionalOps = config.get("mint.nn.functional") + return ( + set(WrapFunctionalOps) & set(ops_func.keys()), + set(WrapMintOps) & set(mint_ops_func.keys()), + set(WrapMintFunctionalOps) & set(mint_func_ops_func.keys()) + ) + + +class HOOKFunctionalOP(object): + pass + + +class HOOKMintOP(object): + pass + + +class HOOKMintNNFunctionalOP(object): + pass + + +class FunctionalOPTemplate(HOOKCell): + def __init__(self, op_name, op_dict, prefix, hook): + self.op_name = op_name + self.op_func = op_dict[op_name] + self.prefix_op_name_ = prefix + str(op_name.split(Const.SEP)[-1]) + Const.SEP + super().__init__(hook) + + def construct(self, *args, **kwargs): + if self.op_name.startswith('dropout'): + return args[0] if args else kwargs.get('input') + return self.op_func(*args, **kwargs) + +def wrap_functional_op(op_name, op_dict, prefix, hook): + def op_template(*args, **kwargs): + return FunctionalOPTemplate(op_name, op_dict, prefix, hook)(*args, **kwargs) + return op_template + + +def wrap_functional_ops_and_bind(ops, op_dict, prefix, hook, hook_class): + for op_name in ops: + if callable(op_dict[op_name]): + setattr(hook_class, "wrap_" + op_name, wrap_functional_op(op_name, op_dict, prefix, hook)) + + +def setup_hooks(hook): + functional_ops, mint_ops, mint_func_ops = get_functional_ops() + wrap_functional_ops_and_bind( + functional_ops, {f: getattr(ms.ops, f) for f in dir(ms.ops)}, "Functional.", hook, HOOKFunctionalOP) + wrap_functional_ops_and_bind( + mint_ops, {f: getattr(ms.mint, f) for f in dir(ms.mint)}, "Mint.", hook, HOOKMintOP) + wrap_functional_ops_and_bind( + mint_func_ops, {f: getattr(ms.mint.nn.functional, f) for f in dir(ms.mint.nn.functional)}, "MintNNFunctional.", hook, HOOKMintNNFunctionalOP) + diff --git a/debug/accuracy_tools/atat/mindspore/dump/hook_cell/wrap_tensor.py b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/wrap_tensor.py new file mode 100644 index 000000000..379d210a4 --- /dev/null +++ b/debug/accuracy_tools/atat/mindspore/dump/hook_cell/wrap_tensor.py @@ -0,0 +1,65 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import yaml +import mindspore as ms + +from atat.mindspore.dump.hook_cell.hook_cell import HOOKCell +from atat.core.common.utils import Const + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with open(yaml_path, 'r') as f: + WrapTensorOps = yaml.safe_load(f).get('tensor') + +TensorFunc = {} +for f in dir(ms.Tensor): + TensorFunc[f] = getattr(ms.Tensor, f) + + +def get_tensor_ops(): + global WrapTensorOps + _tensor_ops = dir(ms.Tensor) + return set(WrapTensorOps) & set(_tensor_ops) + + +class HOOKTensor(object): + pass + + +class TensorOPTemplate(HOOKCell): + + def __init__(self, op_name, hook): + self.op_name_ = op_name + self.prefix_op_name_ = "Tensor." + str(op_name) + Const.SEP + super().__init__(hook) + + def construct(self, *args, **kwargs): + return TensorFunc[str(self.op_name_)](*args, **kwargs) + + +def wrap_tensor_op(op_name, hook): + def tensor_op_template(*args, **kwargs): + return TensorOPTemplate(op_name, hook)(*args, **kwargs) + + return tensor_op_template + + +def wrap_tensor_ops_and_bind(hook): + _tensor_ops = get_tensor_ops() + for op_name in _tensor_ops: + if callable(TensorFunc[op_name]): + setattr(HOOKTensor, "wrap_" + str(op_name), wrap_tensor_op(op_name, hook)) diff --git a/debug/accuracy_tools/atat/mindspore/service.py b/debug/accuracy_tools/atat/mindspore/service.py new file mode 100644 index 000000000..6e9db8526 --- /dev/null +++ b/debug/accuracy_tools/atat/mindspore/service.py @@ -0,0 +1,132 @@ +import os +from pathlib import Path +import functools +import mindspore + +from atat.core.data_dump.data_collector import build_data_collector +from atat.core.data_dump.scope import BaseScope +from atat.mindspore.common.utils import get_rank_if_initialized +from atat.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create +from atat.mindspore.common.log import logger +from atat.core.common.utils import Const +from atat.core.common.exceptions import DistributedNotInitializedError +from atat.mindspore.dump.hook_cell.api_registry import api_register +from atat.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs + + +class Service: + def __init__(self, config): + self.model = None + self.config = config + self.config.level = self.config.level_ori + self.data_collector = build_data_collector(config) + self.switch = False + self.current_iter = 0 + self.first_start = True + self.current_rank = None + self.dump_iter_dir = None + + def build_hook(self, module_type, name): + def forward_hook(api_or_module_name, module, input, output): + self.data_collector.visit_and_clear_overflow_status(api_or_module_name) + if not self.switch: + return + if self.data_collector: + module_input_output = ModuleForwardInputsOutputs(args=input, kwargs=module.input_kwargs, output=output) + self.data_collector.forward_data_collect(api_or_module_name, module, pid, module_input_output) + if self.data_collector.if_return_forward_new_output(): + return self.data_collector.get_forward_new_output() + return output + + def backward_hook(api_or_module_name, module, grad_input, grad_output): + self.data_collector.visit_and_clear_overflow_status(api_or_module_name) + if not self.switch: + return + if self.data_collector: + module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_input, grad_output=grad_output) + self.data_collector.backward_data_collect(api_or_module_name, module, pid, module_input_output) + + pid = os.getpid() + forward_name_template = name + Const.FORWARD + backward_name_template = name + Const.BACKWARD + forward_hook = functools.partial(forward_hook, forward_name_template) + backward_hook = functools.partial(backward_hook, backward_name_template) + + def wrap_forward_hook(*args, **kwargs): + return forward_hook(*args, **kwargs) + + def wrap_backward_hook(*args, **kwargs): + return backward_hook(*args, **kwargs) + + return wrap_forward_hook, wrap_backward_hook + + def step(self): + self.current_iter += 1 + self.data_collector.update_iter(self.current_iter) + + def start(self, model): + self.model = model + if self.config.step and self.current_iter > max(self.config.step): + self.stop() + raise Exception("atat: exit after iteration {}".format(max(self.config.step))) + if self.config.step and self.current_iter not in self.config.step: + return + if self.first_start: + try: + self.current_rank = get_rank_if_initialized() + except DistributedNotInitializedError: + self.current_rank = None + + if self.config.rank and self.current_rank not in self.config.rank: + return + self.register_hook_new() + self.first_start = False + self.switch = True + logger.info_on_rank_0(f"Dump switch is turned on at step {self.current_iter}. ") + if self.config.level != "L2": + self.create_dirs() + logger.info_on_rank_0(f"Dump data will be saved in {self.dump_iter_dir}.") + + def stop(self): + if self.config.level == "L2": + return + if self.config.step and self.current_iter not in self.config.step: + return + if self.config.rank and self.current_rank not in self.config.rank: + return + self.switch = False + self.data_collector.write_json() + + def create_dirs(self): + check_path_before_create(self.config.dump_path) + if not os.path.exists(self.config.dump_path): + Path(self.config.dump_path).mkdir(mode=0o750, exist_ok=True) + file_check = FileChecker(self.config.dump_path, FileCheckConst.DIR) + file_check.common_check() + self.dump_iter_dir = os.path.join(self.config.dump_path, f"step{self.current_iter}") + cur_rank = self.current_rank if self.current_rank is not None else '' + dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}") + if not os.path.exists(dump_dir): + Path(dump_dir).mkdir(mode=0o750, parents=True, exist_ok=True) + if self.config.task in self.data_collector.tasks_need_tensor_data: + dump_data_dir = os.path.join(dump_dir, "dump_tensor_data") + Path(dump_data_dir).mkdir(mode=0o750, exist_ok=True) + else: + dump_data_dir = None + + dump_file_path = os.path.join(dump_dir, "dump.json") + stack_file_path = os.path.join(dump_dir, "stack.json") + construct_file_path = os.path.join(dump_dir, "construct.json") + free_benchmark_file_path = os.path.join(self.config.dump_path, "free_benchmark.csv") + self.data_collector.update_dump_paths( + dump_file_path, stack_file_path, construct_file_path, dump_data_dir, free_benchmark_file_path) + + + + def register_hook_new(self): + logger.info_on_rank_0("The {} hook function is successfully mounted to the model.".format(self.config.task)) + if self.config.level == "L1": + api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) + api_register.api_modularity() + + diff --git a/debug/accuracy_tools/atat/pytorch/service.py b/debug/accuracy_tools/atat/pytorch/service.py index cd80d0852..9b9d7da74 100644 --- a/debug/accuracy_tools/atat/pytorch/service.py +++ b/debug/accuracy_tools/atat/pytorch/service.py @@ -164,4 +164,4 @@ class Service: api_register.api_modularity() if Const.STATISTICS == self.config.task or Const.TENSOR == self.config.task: - remove_dropout() + remove_dropout() \ No newline at end of file -- Gitee From 9c6e4c602344dba1f4e4ca07dddaf9734b7f70a2 Mon Sep 17 00:00:00 2001 From: lcw Date: Tue, 23 Jul 2024 09:55:41 +0800 Subject: [PATCH 005/791] =?UTF-8?q?[Bugfix]=20dump=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E6=94=AF=E6=8C=81GPU?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/data_dump/data_processor/pytorch_processor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 95be091b2..588d137f9 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -15,8 +15,9 @@ from msprobe.pytorch.free_benchmark import FreeBenchmarkCheck, UnequalRow try: import torch_npu + is_gpu = False except ImportError: - pass + is_gpu = True class PytorchDataProcessor(BaseDataProcessor): @@ -213,7 +214,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): def _analyze_maybe_overflow_tensor(self, tensor_json, tensor): data_clone = tensor.detach() - if hasattr(torch_npu._C, '_npu_is_support_inf_nan') and torch_npu._C._npu_is_support_inf_nan(): + if is_gpu or (hasattr(torch_npu._C, '_npu_is_support_inf_nan') and torch_npu._C._npu_is_support_inf_nan()): if tensor_json['Max'] is None: return if np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']): -- Gitee From 7366695cdc54a7159425a03429ca18efa2849e0b Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Tue, 23 Jul 2024 15:42:19 +0800 Subject: [PATCH 006/791] =?UTF-8?q?[advisor]=E7=AE=97=E5=AD=90=E9=99=8D?= =?UTF-8?q?=E9=A2=91=E9=97=AE=E9=A2=98=E8=B5=84=E6=96=99=E9=80=82=E9=85=8D?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E7=AE=97=E5=AD=90=E9=99=8D=E9=A2=91=E6=8F=8F?= =?UTF-8?q?=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/profiler/advisor/README.md b/profiler/advisor/README.md index c650f40b3..b6d157f5a 100644 --- a/profiler/advisor/README.md +++ b/profiler/advisor/README.md @@ -72,6 +72,7 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 | | block_dim_analysis | block dim算子调优 | | | operator_no_bound_analysis | operator no bound | | | graph | 融合算子图调优 | +| | freq_analysis | AI Core算子降频分析 | | scheduling | timeline_fusion_ops | 亲和API替换调优 | | | timeline_op_dispatch | 识别算子下发问题(路径3/路径5) | @@ -152,7 +153,7 @@ torch_npu.npu.config.allow_internal_format = False ![schedule_3](./img/schedule_3.png) -computation模块从device计算性能维度进行分析，能够识别AI CPU、计算bound、动态Shape等问题并给出相应建议。此处不再详细展开，按照报告进行调优即可。 +computation模块从device计算性能维度进行分析，能够识别AI CPU、计算bound、动态Shape、AI Core算子降频分析等问题并给出相应建议。此处不再详细展开，按照报告进行调优即可。 ![computation_1](./img/computation_1.png) -- Gitee From e7e741cad26b5a8265b1f8f371d9a725e97b8660 Mon Sep 17 00:00:00 2001 From: fanhong <2532845962@qq.com> Date: Sat, 20 Jul 2024 14:53:36 +0800 Subject: [PATCH 007/791] =?UTF-8?q?=E9=80=82=E9=85=8Ddisaggregate=5Fperf?= =?UTF-8?q?=E6=96=B0=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../analyzer/overall/overall_analyzer.py | 45 -------- .../overall/overall_summary_analyzer.py | 107 ++++++++---------- profiler/cli/analyze_cli.py | 3 - 3 files changed, 47 insertions(+), 108 deletions(-) delete mode 100644 profiler/advisor/analyzer/overall/overall_analyzer.py diff --git a/profiler/advisor/analyzer/overall/overall_analyzer.py b/profiler/advisor/analyzer/overall/overall_analyzer.py deleted file mode 100644 index 916a396b3..000000000 --- a/profiler/advisor/analyzer/overall/overall_analyzer.py +++ /dev/null @@ -1,45 +0,0 @@ -import logging -from typing import Dict, List - -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.result.result import OptimizeResult -from profiler.compare_tools.compare_backend.utils.constant import Constant -from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface - -logger = logging.getLogger() - - -class OverallSummaryAnalyzer(BaseAnalyzer): - - def __init__(self, profiling_path, benchmark_profiling_path=None, **kwargs): - self.benchmark_profiling_path = benchmark_profiling_path or profiling_path - self.profiling_path = profiling_path - self.html_render = HTMLRender() - self.result = OptimizeResult() - - def optimize(self, **kwargs): - compare_result = ComparisonInterface(self.benchmark_profiling_path, self.profiling_path).compare( - Constant.OVERALL_COMPARE) - - headers = compare_result.get('Model Profiling Time Distribution').get("headers", []) - rows = compare_result.get('Model Profiling Time Distribution').get("rows", []) - - self.make_record() - self.make_render(headers=headers, rows=rows) - return compare_result - - def make_record(self): - pass - - def make_render(self, **kwargs): - headers = kwargs.get("headers") - rows = kwargs.get("rows") - - if not headers or not rows: - logger.info("Empty headers or rows, skip render overall analysis html") - self.html_render.render_template(key="overall", - template_dir="templates", - template_name="overall_analysis.html", - headers=kwargs.get("headers"), - rows=kwargs.get("rows")) diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py index c74ae0510..563a55abb 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py @@ -12,20 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os import copy +import os -import logging -from typing import Dict, List - +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.common import constant as const from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult from profiler.compare_tools.compare_backend.utils.constant import Constant -from profiler.advisor.common import constant as const from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface -from profiler.advisor.utils.utils import get_file_path_from_directory, load_parameter class OverallSummaryAnalyzer(BaseAnalyzer): @@ -52,16 +48,28 @@ class OverallSummaryAnalyzer(BaseAnalyzer): "Uncovered Communication Time(Wait Time)": [], "Free Time": ['SDMA Time(Num)'] } + time_field_map = { + "Computing Time": ['compute_time', None], + "Uncovered Communication Time(Wait Time)": ['communication_not_overlapped', 'wait_time'], + "Free Time": ['scheduling_time', None], + "Cube Time(Num)": ['cube_time', 'cube_num'], + "Vector Time(Num)": ['vec_time', 'vec_num'], + "Flash Attention Time(Forward)(Num)": ['fa_time_fwd', 'fa_num_fwd'], + "Flash Attention Time(Backward)(Num)": ['fa_time_bwd', 'fa_num_bwd'], + "Other Time": ['other_time', None], + "SDMA Time(Num)": ['sdma_time', 'sdma_num'], + } def __init__(self, collection_path: str, n_processes: int = 1, **kwargs): profile_path = get_profile_path(collection_path) super().__init__(profile_path, n_processes, **kwargs) - self.base_collection_path = kwargs.get("base_collection_path", "") + self.base_collection_path = kwargs.get("benchmark_profiling_path", "") self._has_base_collection = False self._is_minimal_profiling = False self.cur_data = {} self.cur_data_table = {} self.cur_bottleneck = {} + self._disaggregate_perf = {} self.cur_advices = "" self._headers = [] self._base_data = [] @@ -71,22 +79,6 @@ class OverallSummaryAnalyzer(BaseAnalyzer): self.bottleneck_str = "" self.bottleneck_table = {} - @staticmethod - def split_duration_and_num(time_value: str) -> tuple: - split_data = time_value.split("s") # time value example: 0.229s(1756) - duration, num = 0.0, None - if len(split_data) >= 2: - try: - num = int(split_data[1].strip("()")) - except ValueError: - pass - if len(split_data) >= 1: - try: - duration = float(split_data[0]) - except ValueError: - print(f"[WARNING] Invalid time value: {time_value}.") - return duration, num - @staticmethod def calculate_ratio(dividend, divisor): if not divisor: @@ -102,49 +94,43 @@ class OverallSummaryAnalyzer(BaseAnalyzer): return os.path.exists(self.collection_path) def process(self): - base_collection_path = self.base_collection_path if self._has_base_collection else self.collection_path - result_data = ComparisonInterface(base_collection_path, self.collection_path).compare(Constant.OVERALL_COMPARE) - for data in result_data.values(): - self._headers = data.get("headers", []) - rows = data.get("rows", []) - if len(rows) == 2: - self._base_data = rows[0] - self._comparison_data = rows[1] - if not self._headers or not self._comparison_data: + self._disaggregate_perf = ComparisonInterface(self.collection_path).disaggregate_perf(Constant.OVERALL_COMPARE) + if not self._disaggregate_perf: return - self._is_minimal_profiling = 'E2E Time(Not minimal profiling)' not in self._headers + self._is_minimal_profiling = self._disaggregate_perf.get("minimal_profiling", False) if self._has_base_collection: - self.cur_data["comparison_result"] = result_data + base_collection_path = self.base_collection_path if self._has_base_collection else self.collection_path + comparison_result = ComparisonInterface(base_collection_path, self.collection_path).compare( + Constant.OVERALL_COMPARE) + self.cur_data["comparison_result"] = comparison_result + time_category_dict = {} for time_category, time_list in self.performance_time_dict.items(): - time_value = self.get_time_value(time_category, self._comparison_data) - if time_value == Constant.INVALID_VALUE: + duration, _ = self.get_duration_and_num(time_category) + if duration == Constant.INVALID_VALUE: continue - duration, _ = self.split_duration_and_num(time_value) time_category = time_category.split("(")[0] time_category_dict[time_category] = duration self.get_sub_category_time(time_category, time_list, duration) self.cur_data["overall_data"] = time_category_dict - def get_time_value(self, header_name: str, data_list: list): - try: - data_index = self._headers.index(header_name) - except ValueError: - return Constant.INVALID_VALUE - try: - time_value = data_list[data_index] - except IndexError: - return Constant.INVALID_VALUE - return time_value + def get_duration_and_num(self, time_category: str): + field_list = self.time_field_map.get(time_category) + if not field_list: + return Constant.INVALID_VALUE, Constant.INVALID_VALUE + duration = round(self._disaggregate_perf.get(field_list[0], 0.0), 3) + num = self._disaggregate_perf.get(field_list[1], None) + if isinstance(num, float): + num = round(num, 3) + return duration, num def get_sub_category_time(self, category: str, time_list: list, total_duration: float): sub_time_dict = {} for time_name in time_list: - time_value = self.get_time_value(time_name, self._comparison_data) - if time_value == Constant.INVALID_VALUE: - continue sub_time_dict.setdefault(f"{category} Subtype", []).append(self.time_name_map.get(time_name, "")) - duration, num = self.split_duration_and_num(time_value) + duration, num = self.get_duration_and_num(time_name) + if duration == Constant.INVALID_VALUE or num == Constant.INVALID_VALUE: + continue sub_time_dict.setdefault(f"Duration(s)", []).append(duration) sub_time_dict.setdefault(f"Duration Ratio", []).append( "{:.2%}".format(self.calculate_ratio(duration, total_duration))) @@ -171,13 +157,14 @@ class OverallSummaryAnalyzer(BaseAnalyzer): # add comparison bottleneck time_type_origin = "Uncovered Communication Time(Wait Time)" \ if time_type == "Uncovered Communication Time" else time_type - base_duration, _ = self.split_duration_and_num(self.get_time_value(time_type_origin, self._base_data)) + base_duration, _ = self.get_duration_and_num(time_type_origin) if time_value > base_duration: ratio = "{:.2%}".format(self.calculate_ratio(time_value - base_duration, base_duration)) comparison_bottleneck += f"{time_type} exceeds the benchmark by {ratio}\n" self.cur_bottleneck["overall_data"] = overall_bottleneck if comparison_bottleneck: self.cur_bottleneck["comparison_result"] = comparison_bottleneck + def optimize(self, **kwargs): if self.path_check(): self.process() @@ -218,7 +205,6 @@ class OverallSummaryAnalyzer(BaseAnalyzer): data_table = {"headers": headers, "data": [data_list]} self.cur_data_table[data_type] = copy.deepcopy(data_table) - def make_record(self): """ make record for what and how to optimize @@ -232,7 +218,7 @@ class OverallSummaryAnalyzer(BaseAnalyzer): ) self.result.add(OptimizeRecord(optimization_item)) - self.result.add_detail(const.BOTTLENECK, self.bottleneck_table["headers"], self.bottleneck_table["data"][0]) + self.result.add_detail(const.BOTTLENECK, self.bottleneck_table["headers"], self.bottleneck_table["data"][0]) for data_type, data_dict in self.cur_data_table.items(): if data_dict: self.result.add_detail(const.DATA + data_type, data_dict["headers"], data_dict["data"][0]) @@ -241,9 +227,9 @@ class OverallSummaryAnalyzer(BaseAnalyzer): if not self.bottleneck_str and not self.cur_advices: return result_for_html = { - "Description" : self.bottleneck_str, - "suggestion" : self.cur_advices, - "details" : [self.bottleneck_table] + "Description": self.bottleneck_str, + "suggestion": self.cur_advices, + "details": [self.bottleneck_table] } self.html_render.render_template(key="overall", @@ -254,9 +240,10 @@ class OverallSummaryAnalyzer(BaseAnalyzer): torch_version=self.torch_version, result=result_for_html) + def get_profile_path(collection_path): for root, dirs, files in os.walk(collection_path): for file in files: if file.startswith("profiler_info"): return root - return "" \ No newline at end of file + return "" diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index 2e173dc87..f400a265b 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -83,9 +83,6 @@ def analyze_cli(**kwargs): help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @debug_option def analyze_all(**kwargs) -> None: - # 当前compare_tools必须输入两个profiling路径，att-advisor有等价功能支持输入一个Profiling路径，后续替换成对应实现 - if not kwargs.get("benchmark_profiling_path"): - kwargs["benchmark_profiling_path"] = kwargs.get("profiling_path") try: _analyze(Interface.all_dimension, **kwargs) except RuntimeError as e: -- Gitee From c8838df21f97f5abb1c0f85a94d8bbb6c2b5ac83 Mon Sep 17 00:00:00 2001 From: fanhong <2532845962@qq.com> Date: Tue, 23 Jul 2024 15:38:38 +0800 Subject: [PATCH 008/791] =?UTF-8?q?=E9=80=82=E9=85=8Ddisaggregate=5Fperf?= =?UTF-8?q?=E6=96=B0=E6=8E=A5=E5=8F=A3=EF=BC=8C=E4=BB=85=E4=BF=9D=E7=95=99?= =?UTF-8?q?overall=5Fdata?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../overall/overall_summary_analyzer.py | 100 +++++------------- .../disaggregate/overall_perf_interface.py | 28 ++++- 2 files changed, 56 insertions(+), 72 deletions(-) diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py index 563a55abb..2cb0164eb 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py @@ -42,34 +42,18 @@ class OverallSummaryAnalyzer(BaseAnalyzer): 'Other Time': "Other Computing Time", 'SDMA Time(Num)': 'SDMA Time' } - performance_time_dict = { - "Computing Time": ['Cube Time(Num)', 'Vector Time(Num)', 'Flash Attention Time(Forward)(Num)', - 'Flash Attention Time(Backward)(Num)', 'Other Time'], - "Uncovered Communication Time(Wait Time)": [], - "Free Time": ['SDMA Time(Num)'] - } - time_field_map = { - "Computing Time": ['compute_time', None], - "Uncovered Communication Time(Wait Time)": ['communication_not_overlapped', 'wait_time'], - "Free Time": ['scheduling_time', None], - "Cube Time(Num)": ['cube_time', 'cube_num'], - "Vector Time(Num)": ['vec_time', 'vec_num'], - "Flash Attention Time(Forward)(Num)": ['fa_time_fwd', 'fa_num_fwd'], - "Flash Attention Time(Backward)(Num)": ['fa_time_bwd', 'fa_num_bwd'], - "Other Time": ['other_time', None], - "SDMA Time(Num)": ['sdma_time', 'sdma_num'], - } def __init__(self, collection_path: str, n_processes: int = 1, **kwargs): profile_path = get_profile_path(collection_path) super().__init__(profile_path, n_processes, **kwargs) - self.base_collection_path = kwargs.get("benchmark_profiling_path", "") - self._has_base_collection = False + self.benchmark_profiling_path = kwargs.get("benchmark_profiling_path", "") + self._has_benchmark_profiling = False self._is_minimal_profiling = False self.cur_data = {} self.cur_data_table = {} self.cur_bottleneck = {} self._disaggregate_perf = {} + self._disaggregate_benchmark_perf = {} self.cur_advices = "" self._headers = [] self._base_data = [] @@ -85,79 +69,51 @@ class OverallSummaryAnalyzer(BaseAnalyzer): return float("inf") return dividend / divisor + @staticmethod + def get_time_category_dict(overall_dict: dict): + time_category_dict = { + "Computing Time": round(overall_dict.get('computing_time_ms', 0.0), 3), + "Uncovered Communication Time": round(overall_dict.get('uncovered_communication_time_ms', 0.0), 3), + "Free Time": round(overall_dict.get('free_time_ms', 0.0), 3) + } + return time_category_dict + def path_check(self): - if self.base_collection_path: - if os.path.exists(self.base_collection_path): - self._has_base_collection = True + if self.benchmark_profiling_path: + if os.path.exists(self.benchmark_profiling_path): + self._has_benchmark_profiling = True else: - print(f"[WARNING] Invalid path which not exists: {self.base_collection_path}.") + print(f"[WARNING] Invalid path which not exists: {self.benchmark_profiling_path}.") return os.path.exists(self.collection_path) def process(self): self._disaggregate_perf = ComparisonInterface(self.collection_path).disaggregate_perf(Constant.OVERALL_COMPARE) + if self._has_benchmark_profiling: + self._disaggregate_benchmark_perf = (ComparisonInterface(self.benchmark_profiling_path) + .disaggregate_perf(Constant.OVERALL_COMPARE)) if not self._disaggregate_perf: return self._is_minimal_profiling = self._disaggregate_perf.get("minimal_profiling", False) - if self._has_base_collection: - base_collection_path = self.base_collection_path if self._has_base_collection else self.collection_path - comparison_result = ComparisonInterface(base_collection_path, self.collection_path).compare( - Constant.OVERALL_COMPARE) - self.cur_data["comparison_result"] = comparison_result - - time_category_dict = {} - for time_category, time_list in self.performance_time_dict.items(): - duration, _ = self.get_duration_and_num(time_category) - if duration == Constant.INVALID_VALUE: - continue - time_category = time_category.split("(")[0] - time_category_dict[time_category] = duration - self.get_sub_category_time(time_category, time_list, duration) - self.cur_data["overall_data"] = time_category_dict - - def get_duration_and_num(self, time_category: str): - field_list = self.time_field_map.get(time_category) - if not field_list: - return Constant.INVALID_VALUE, Constant.INVALID_VALUE - duration = round(self._disaggregate_perf.get(field_list[0], 0.0), 3) - num = self._disaggregate_perf.get(field_list[1], None) - if isinstance(num, float): - num = round(num, 3) - return duration, num - - def get_sub_category_time(self, category: str, time_list: list, total_duration: float): - sub_time_dict = {} - for time_name in time_list: - sub_time_dict.setdefault(f"{category} Subtype", []).append(self.time_name_map.get(time_name, "")) - duration, num = self.get_duration_and_num(time_name) - if duration == Constant.INVALID_VALUE or num == Constant.INVALID_VALUE: - continue - sub_time_dict.setdefault(f"Duration(s)", []).append(duration) - sub_time_dict.setdefault(f"Duration Ratio", []).append( - "{:.2%}".format(self.calculate_ratio(duration, total_duration))) - sub_time_dict.setdefault(f"Kernel Number", []).append(num) - self.cur_data[self.time_name_map.get(category)] = sub_time_dict + self.cur_data["overall_data"] = self.get_time_category_dict(self._disaggregate_perf.get('overall', {})) def identify_bottleneck(self): overall_data = self.cur_data.get("overall_data") if not overall_data: return e2e_time = '%.3f' % sum([data for data in overall_data.values()]) - overall_bottleneck = f"The Model E2E Time is {e2e_time}s.\n" + overall_bottleneck = f"The Model E2E Time is {e2e_time}ms.\n" comparison_bottleneck = "" for time_type, time_value in overall_data.items(): - # add subtype time bottleneck - self.cur_bottleneck[self.time_name_map.get(time_type)] = f"{time_type} is {time_value}s.\n" # add overall bottleneck - overall_bottleneck += f" -- {time_type} is {time_value}s\n" + overall_bottleneck += f" -- {time_type} is {time_value}ms\n" if time_type == "Free Time" and self._is_minimal_profiling and self.calculate_ratio(time_value, e2e_time) > 0.1: overall_bottleneck += "percentage of free time exceed the threshold 10%." - if not self._has_base_collection: + if not self._has_benchmark_profiling: continue # add comparison bottleneck - time_type_origin = "Uncovered Communication Time(Wait Time)" \ - if time_type == "Uncovered Communication Time" else time_type - base_duration, _ = self.get_duration_and_num(time_type_origin) + base_duration = self.get_time_category_dict(self._disaggregate_benchmark_perf.get('overall', {})).get( + time_type) if time_value > base_duration: ratio = "{:.2%}".format(self.calculate_ratio(time_value - base_duration, base_duration)) comparison_bottleneck += f"{time_type} exceeds the benchmark by {ratio}\n" @@ -183,7 +139,7 @@ class OverallSummaryAnalyzer(BaseAnalyzer): for key, value in self.cur_bottleneck.items(): if not value: continue - result += f'{key}: {value} \n' + result += f'{value} \n' headers.append(key) data.append(value) data_list.append(data) @@ -226,8 +182,10 @@ class OverallSummaryAnalyzer(BaseAnalyzer): def make_render(self): if not self.bottleneck_str and not self.cur_advices: return + # 将\n替换为html换行 + bottleneck_str = self.bottleneck_str.replace('\n', '
') result_for_html = { - "Description": self.bottleneck_str, + "Description": bottleneck_str, "suggestion": self.cur_advices, "details": [self.bottleneck_table] } diff --git a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index c89e84519..7bac2b033 100644 --- a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -31,4 +31,30 @@ class OverallPerfInterface: def _generate_result(self): overall_data = self._profiling_data.overall_metrics - self._result_data = getattr(overall_data, "__dict__", {}) + + self._result_data = { + "profiling_type": overall_data.profiling_type, + "minimal_profiling": overall_data.minimal_profiling, + "overall": {"e2e_time_ms": overall_data.e2e_time_ms, + "computing_time_ms": overall_data.compute_time_ms, + "uncovered_communication_time_ms": overall_data.communication_not_overlapped_ms, + "free_time_ms": overall_data.free_time_ms}, + "computing_time_disaggregate": {"fa_time_ms": overall_data.fa_total_time, + "conv_time_ms": overall_data.conv_total_time, + "matmul_time_ms": overall_data.mm_total_time, + "page_attention_time_ms": overall_data.page_attention_time, + "vector_time_ms": overall_data.vector_total_time, + "tensor_move_time_ms": overall_data.sdma_time_tensor_move, + "other_cube_time_ms": overall_data.other_cube_time}, + "computing_num_disaggregate": {"fa_num": overall_data.fa_total_num, + "conv_num": overall_data.conv_total_num, + "matmul_num": overall_data.mm_total_num, + "page_attention_num": overall_data.page_attention_num, + "vector_num": overall_data.vector_total_num, + "tensor_move_num": overall_data.sdma_num_tensor_move, + "other_cube_num": overall_data.other_cube_num}, + "communication_time_disaggregate": {"wait_time_ms": overall_data.wait_time_ms, + "transmit_time_ms": overall_data.transmit_time_ms}, + "free_time_disaggregate": {"sdma_time_ms": overall_data.sdma_time_stream, + "free_ms": overall_data.free_time_ms - overall_data.sdma_time_stream} + } -- Gitee From b3230c665fb3c62f1f69f21befc025ff4ebf3b0f Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Wed, 24 Jul 2024 14:53:10 +0800 Subject: [PATCH 009/791] =?UTF-8?q?[msprobe\pytorch\dump]=E8=B5=84?= =?UTF-8?q?=E6=96=99=E7=A4=BA=E4=BE=8B=E4=BB=A3=E7=A0=81=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/pytorch/doc/dump.md | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md index 7d0763b68..0554736c3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md @@ -12,7 +12,7 @@ msprobe工具主要通过在训练脚本内添加dump接口并启动训练的方 通过加载dump配置文件的方式来确定dump操作的详细配置。 -可以在from msprobe.pytorch import PrecisionDebugger和模型初始化之间的任意位置添加该接口。 +PrecisionDebugger接口可以在from msprobe.pytorch import PrecisionDebugger之后的位置添加。详细使用可参考“**示例代码**”或“**model配置代码示例**”。 **原型** @@ -125,22 +125,25 @@ debugger.step() ```Python from msprobe.pytorch import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") -# 请勿将以上初始化流程插入到循环代码中 -# 模型初始化 -# 下面代码也可以用PrecisionDebugger.start()和PrecisionDebugger.stop() -debugger.start() -# 需要dump的代码片段1 +# 模型、损失函数的定义及初始化等操作 +# ... -debugger.stop() -debugger.start() +# 数据集迭代的位置一般为模型训练开始的位置 +for data, label in data_loader: + debugger.start() # 开启数据dump -# 需要dump的代码片段2 + # 如下是模型每个step执行的逻辑 + output = model(data) + #... + loss.backward() -debugger.stop() -debugger.step() + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump ``` ## dump结果文件介绍 -- Gitee From b2ec87875ea9c327ca447739fc64c6076c2beca5 Mon Sep 17 00:00:00 2001 From: hanqing Date: Wed, 24 Jul 2024 15:23:49 +0800 Subject: [PATCH 010/791] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8C=87=E5=AE=9A?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E6=AE=B5=E8=87=AA=E5=8A=A8dump=E5=89=8D?= =?UTF-8?q?=E5=8F=8D=E5=90=91=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/pytorch/debugger/precision_debugger.py | 6 ++++++ debug/accuracy_tools/msprobe/pytorch/service.py | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py index 1fce5a303..2c8692d93 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py @@ -64,6 +64,12 @@ class PrecisionDebugger: else: instance.service.start(instance.model) + # 指定代码段dump前反向结束符，之后的计算过程数据将被忽略，无法被dump + @classmethod + def forward_backward_dump_end(cls): + instance = cls._instance + instance.service.forward_backward_dump_end() + @classmethod def stop(cls): instance = cls._instance diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index e5da44484..adfbddc5c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -100,6 +100,10 @@ class Service: self.create_dirs() logger.info_on_rank_0(f"Dump data will be saved in {self.dump_iter_dir}.") + def forward_backward_dump_end(self): + logger.info_on_rank_0("Data needed ends here.") + api_register.api_originality() + def stop(self): if self.config.level == "L2": return -- Gitee From 698018c5faa754ca26a700531893c6f05b539212 Mon Sep 17 00:00:00 2001 From: fanhong <2532845962@qq.com> Date: Tue, 23 Jul 2024 22:46:00 +0800 Subject: [PATCH 011/791] =?UTF-8?q?=E9=80=82=E9=85=8Ddisaggregate=5Fperf?= =?UTF-8?q?=E6=96=B0=E6=8E=A5=E5=8F=A3=EF=BC=8C=E6=96=B0=E5=A2=9Eover=5Fsu?= =?UTF-8?q?mmary=5Fanalysis?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../overall/overall_summary_analyzer.py | 131 ++++++++++-------- 1 file changed, 75 insertions(+), 56 deletions(-) diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py index 2cb0164eb..e9f39c60a 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py @@ -12,11 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import copy import os from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common import constant as const from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.result.result import OptimizeResult @@ -26,21 +24,21 @@ from profiler.compare_tools.compare_interface.comparison_interface import Compar class OverallSummaryAnalyzer(BaseAnalyzer): OVERALL_SUMMARY_ANALYZER = "overall_summary_analysis" - advice_map = { - "Computing Time": "if you want more detailed advice please go to att_advisor_*.html", - "Uncovered Communication Time": "if you want more detailed advice please go to att_advisor_*.html", - "Free Time": "if you want more detailed advice please go to att_advisor_*.html" - } - time_name_map = { - "Computing Time": "computing", - "Uncovered Communication Time": "communication", - "Free Time": "free", - 'Cube Time(Num)': 'Cube Time', - 'Vector Time(Num)': 'Vector Time', - 'Flash Attention Time(Forward)(Num)': 'Flash Attention Time(Forward)', - 'Flash Attention Time(Backward)(Num)': 'Flash Attention Time(Backward)', - 'Other Time': "Other Computing Time", - 'SDMA Time(Num)': 'SDMA Time' + performance_time_dict = { + "Computing Time": "computing_time_ms", + " -- Flash Attention": "fa_time_ms", + " -- Conv": "conv_time_ms", + " -- Matmul": "matmul_time_ms", + " -- Vector": "vector_time_ms", + " -- SDMA(Tensor Move)": "tensor_move_time_ms", + " -- Other Cube": "other_cube_time_ms", + "Uncovered Communication Time": "uncovered_communication_time_ms", + " -- Wait": "wait_time_ms", + " -- Transmit": "transmit_time_ms", + "Free Time": "free_time_ms", + " -- SDMA": "sdma_time_ms", + " -- Free": "free_ms", + "E2E Time": "e2e_time_ms" } def __init__(self, collection_path: str, n_processes: int = 1, **kwargs): @@ -50,18 +48,14 @@ class OverallSummaryAnalyzer(BaseAnalyzer): self._has_benchmark_profiling = False self._is_minimal_profiling = False self.cur_data = {} - self.cur_data_table = {} self.cur_bottleneck = {} self._disaggregate_perf = {} self._disaggregate_benchmark_perf = {} self.cur_advices = "" - self._headers = [] - self._base_data = [] - self._comparison_data = [] self.html_render = HTMLRender() self.result = OptimizeResult() self.bottleneck_str = "" - self.bottleneck_table = {} + self.over_summary_analysis = {} @staticmethod def calculate_ratio(dividend, divisor): @@ -88,13 +82,13 @@ class OverallSummaryAnalyzer(BaseAnalyzer): def process(self): self._disaggregate_perf = ComparisonInterface(self.collection_path).disaggregate_perf(Constant.OVERALL_COMPARE) - if self._has_benchmark_profiling: - self._disaggregate_benchmark_perf = (ComparisonInterface(self.benchmark_profiling_path) - .disaggregate_perf(Constant.OVERALL_COMPARE)) if not self._disaggregate_perf: return self._is_minimal_profiling = self._disaggregate_perf.get("minimal_profiling", False) self.cur_data["overall_data"] = self.get_time_category_dict(self._disaggregate_perf.get('overall', {})) + if self._has_benchmark_profiling: + self._disaggregate_benchmark_perf = ComparisonInterface( + self.benchmark_profiling_path).disaggregate_perf(Constant.OVERALL_COMPARE) def identify_bottleneck(self): overall_data = self.cur_data.get("overall_data") @@ -112,8 +106,9 @@ class OverallSummaryAnalyzer(BaseAnalyzer): if not self._has_benchmark_profiling: continue # add comparison bottleneck - base_duration = self.get_time_category_dict(self._disaggregate_benchmark_perf.get('overall', {})).get( - time_type) + base_duration = self.get_time_category_dict( + self._disaggregate_benchmark_perf.get('overall', {}) + ).get(time_type) if time_value > base_duration: ratio = "{:.2%}".format(self.calculate_ratio(time_value - base_duration, base_duration)) comparison_bottleneck += f"{time_type} exceeds the benchmark by {ratio}\n" @@ -126,40 +121,63 @@ class OverallSummaryAnalyzer(BaseAnalyzer): self.process() self.identify_bottleneck() self.format_bottleneck() - self.format_cur_data() + self.format_over_summary_analysis() self.make_record() self.make_render() return self.result def format_bottleneck(self): result = '' - headers = [] - data_list = [] - data = [] - for key, value in self.cur_bottleneck.items(): + for _, value in self.cur_bottleneck.items(): if not value: continue result += f'{value} \n' - headers.append(key) - data.append(value) - data_list.append(data) self.bottleneck_str = result - self.bottleneck_table["headers"] = headers - self.bottleneck_table["data"] = data_list - def format_cur_data(self): - if not self.cur_data: - return - for data_type, data in self.cur_data.items(): - if not data: - continue - if data_type not in list(self.time_name_map.values()): - data_list = list(data.values()) - else: - data_list = [','.join(map(str, value)) for value in data.values()] - headers = list(data.keys()) - data_table = {"headers": headers, "data": [data_list]} - self.cur_data_table[data_type] = copy.deepcopy(data_table) + def format_over_summary_analysis(self): + headers = ['Performance Index', 'Duration(ms)', 'Duration Ratio'] + performance_data = self.get_analysis_data(self._disaggregate_perf) + benchmark_data = self.get_analysis_data(self._disaggregate_benchmark_perf) + if self._has_benchmark_profiling: + headers.append('Diff Duration(ms)') + self.format_analysis_with_benchmark(performance_data, benchmark_data, headers) + else: + self.format_analysis_only(performance_data, headers) + + def get_analysis_data(self, data_dict: dict): + if not data_dict: + return {} + return { + **data_dict.get("overall"), + **data_dict.get("computing_time_disaggregate"), + **data_dict.get("communication_time_disaggregate"), + **data_dict.get("free_time_disaggregate"), + } + + def format_analysis_only(self, performance_data: dict, headers: list): + res = [] + total_duration = performance_data.get('e2e_time_ms', 0.0) + for time_name, time_key in self.performance_time_dict.items(): + row = [time_name] + duration = performance_data.get(time_key, 0.0) + row.append("{:.3f}".format(duration)) + row.append("{:.2%}".format(self.calculate_ratio(duration, total_duration))) + res.append(row) + self.over_summary_analysis["headers"] = headers + self.over_summary_analysis["data"] = res + + def format_analysis_with_benchmark(self, performance_data: dict, benchmark_data: dict, headers: list): + res = [] + total_duration = performance_data.get('e2e_time_ms', 0.0) + for time_name, time_key in self.performance_time_dict.items(): + row = [time_name] + duration = performance_data.get(time_key, 0.0) + row.append("{:.3f}".format(duration)) + row.append("{:.2%}".format(self.calculate_ratio(duration, total_duration))) + row.append("{:.3f}".format(duration - benchmark_data.get(time_key, 0.0))) + res.append(row) + self.over_summary_analysis["headers"] = headers + self.over_summary_analysis["data"] = res def make_record(self): """ @@ -174,10 +192,12 @@ class OverallSummaryAnalyzer(BaseAnalyzer): ) self.result.add(OptimizeRecord(optimization_item)) - self.result.add_detail(const.BOTTLENECK, self.bottleneck_table["headers"], self.bottleneck_table["data"][0]) - for data_type, data_dict in self.cur_data_table.items(): - if data_dict: - self.result.add_detail(const.DATA + data_type, data_dict["headers"], data_dict["data"][0]) + self.result.add_detail( + OverallSummaryAnalyzer.OVERALL_SUMMARY_ANALYZER, + headers=self.over_summary_analysis["headers"] + ) + for data in self.over_summary_analysis["data"]: + self.result.add_detail(OverallSummaryAnalyzer.OVERALL_SUMMARY_ANALYZER, detail=data) def make_render(self): if not self.bottleneck_str and not self.cur_advices: @@ -187,9 +207,8 @@ class OverallSummaryAnalyzer(BaseAnalyzer): result_for_html = { "Description": bottleneck_str, "suggestion": self.cur_advices, - "details": [self.bottleneck_table] + "details": [self.over_summary_analysis] } - self.html_render.render_template(key="overall", title=OverallSummaryAnalyzer.OVERALL_SUMMARY_ANALYZER, template_dir="templates", -- Gitee From 4872fb2d7e4230a07aa1922f2b79773da49c1a21 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Wed, 24 Jul 2024 16:22:29 +0800 Subject: [PATCH 012/791] Add threshold for dtype bfloat16. --- .../atat/pytorch/free_benchmark/common/constant.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/atat/pytorch/free_benchmark/common/constant.py b/debug/accuracy_tools/atat/pytorch/free_benchmark/common/constant.py index 9b72437f2..47ede61ee 100644 --- a/debug/accuracy_tools/atat/pytorch/free_benchmark/common/constant.py +++ b/debug/accuracy_tools/atat/pytorch/free_benchmark/common/constant.py @@ -51,6 +51,7 @@ class ThresholdConfig: DTYPE_PER_THD = { torch.float16: 1.002, + torch.bfloat16: 1.004, torch.float32: 1.0002, } BENCHMARK_THD_DICT = { -- Gitee From 03c49724e842dfec0c7251343d1154c483bd626b Mon Sep 17 00:00:00 2001 From: l30036321 Date: Wed, 24 Jul 2024 16:05:22 +0800 Subject: [PATCH 013/791] fix norm inner ops bug --- debug/accuracy_tools/msprobe/mindspore/common/log.py | 11 ----------- .../msprobe/mindspore/debugger/precision_debugger.py | 2 +- .../msprobe/mindspore/dump/hook_cell/api_registry.py | 2 +- 3 files changed, 2 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/common/log.py b/debug/accuracy_tools/msprobe/mindspore/common/log.py index de7625675..ec027c750 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/log.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/log.py @@ -34,16 +34,5 @@ class MindsporeLogger(BaseLogger): return current_rank - def _print_log(self, level, msg, end='\n'): - current_rank = self.get_rank() - current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) - pid = os.getpid() - if current_rank is not None: - full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}" - else: - full_msg = f"{current_time} ({pid}) [{level}] {msg}" - print(full_msg, end=end) - sys.stdout.flush() - logger = MindsporeLogger() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index d112fe8d2..30f7162ff 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -31,7 +31,7 @@ class PrecisionDebugger: instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") - if ms.get_context("mode") == 1 and instance.config.level_ori == "L1": + if ms.get_context("mode") == ms.PYNATIVE_MODE and instance.config.level_ori == "L1": instance.service.start() else: handler = TaskHandlerFactory.create(instance.config) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py index 9c1e5f1ca..5508416fd 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py @@ -83,7 +83,7 @@ class ApiRegistry: self.tensor_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKTensor, attr_name) functional_ops, mint_ops, mint_func_ops = get_functional_ops() - self.store_ori_attr(ms.ops, self.norm_inner_ops, self.functional_ori_attr) + self.store_ori_attr(ms.ops, self.norm_inner_ops, self.norm_inner_ops_ori_attr) self.store_ori_attr(ms.ops, functional_ops, self.functional_ori_attr) self.store_ori_attr(ms.mint, mint_ops, self.mint_ops_ori_attr) self.store_ori_attr(ms.mint.nn.functional, mint_func_ops, self.mint_func_ops_ori_attr) -- Gitee From 71d1db3b8181647875d6f8ea4d369e586c1dbf8b Mon Sep 17 00:00:00 2001 From: zhaolei Date: Tue, 23 Jul 2024 14:19:17 +0800 Subject: [PATCH 014/791] =?UTF-8?q?1.ai=20core=E9=99=8D=E9=A2=91=E5=88=86?= =?UTF-8?q?=E6=9E=90=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../computation/ai_core_freq/__init__.py | 0 .../ai_core_freq/ai_core_freq_analyzer.py | 42 +++++ .../ai_core_freq/ai_core_freq_checker.py | 100 ++++++++++++ .../computation/profiling_analyzer.py | 9 +- profiler/advisor/common/analyzer_scopes.py | 1 + profiler/advisor/common/constant.py | 5 +- profiler/advisor/config/config.ini | 1 + profiler/advisor/config/config.py | 7 + .../advisor/dataset/ai_core_freq/__init__.py | 0 .../ai_core_freq/ai_core_freq_dataset.py | 148 ++++++++++++++++++ .../advisor/dataset/profiling/device_info.py | 2 + .../html/templates/ai_core_frequency.html | 27 ++++ profiler/advisor/interface/interface.py | 4 +- profiler/advisor/result/result.py | 14 +- profiler/advisor/utils/utils.py | 50 +++++- .../compute_advice/test_frequency_advice.py | 145 +++++++++++++++++ 16 files changed, 542 insertions(+), 13 deletions(-) create mode 100644 profiler/advisor/analyzer/computation/ai_core_freq/__init__.py create mode 100644 profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py create mode 100644 profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py create mode 100644 profiler/advisor/dataset/ai_core_freq/__init__.py create mode 100644 profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py create mode 100644 profiler/advisor/display/html/templates/ai_core_frequency.html create mode 100644 profiler/test/ut/advisor/compute_advice/test_frequency_advice.py diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/__init__.py b/profiler/advisor/analyzer/computation/ai_core_freq/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py new file mode 100644 index 000000000..cee16cce5 --- /dev/null +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py @@ -0,0 +1,42 @@ +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_checker import AICoreFreqChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.ai_core_freq.ai_core_freq_dataset import AICoreFreqDataset +from profiler.advisor.config.config import Config + +logger = logging.getLogger() + + +class AICoreFreqAnalyzer(BaseAnalyzer): + dataset_cls_list = [AICoreFreqDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = AICoreFreqDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseAnalyzer.check_data((AICoreFreqDataset.get_key(),)) + def optimize(self, **kwargs): + if not Config().get_config("aic_frequency"): + logger.warning("Can not find ai core frequency in info.json*, please check data integrity.") + return self.result + add_render_list = kwargs.get("add_render_list", True) + ai_core_freq_checker = AICoreFreqChecker() + ai_core_freq_checker.check_ai_core_freq(self.dataset) + if not ai_core_freq_checker.ai_core_freq_issues: + return self.result + ai_core_freq_checker.make_record(self.result) + self.html = ai_core_freq_checker.make_render(self.html_render, add_render_list) + return self.result + + def make_record(self): + pass + + def make_render(self): + pass diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py new file mode 100644 index 000000000..e2d4a3c26 --- /dev/null +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -0,0 +1,100 @@ +import logging + +from profiler.advisor.dataset.ai_core_freq.ai_core_freq_dataset import AICoreFreqDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.config.config import Config +from profiler.advisor.utils.utils import convert_to_float + +logger = logging.getLogger() + + +class AICoreFreqChecker: + DEFAULT_FREQ = 1800 + DECREASE_FREQ_RATIO = 0.05 + SHOW_TOPK_OPS = 10 + TOTAL_DURATION_INDEX = 2 + DECREASE_FREQ_RATIO_INDEX = 3 + + def __init__(self): + + self.ai_core_freq_issues = False + self.desc = "" + self.suggestions = "" + self.decrease_freq_ops = [] + self.headers = [] + self.op_freq = None + self.rank_id = None + self.stage = None + + def check_ai_core_freq(self, event_dataset: AICoreFreqDataset, rank_id=None, stage=None): + """ + :Param event_dataset: dataset of timeline event + """ + if not hasattr(event_dataset, "op_freq") or not getattr(event_dataset, "op_freq"): + logger.debug("Skip slow ai core frequency checker, " + "because no ai core frequency were recorded in trace_view.json") + return + + self.rank_id = rank_id + self.stage = stage + self.op_freq = event_dataset.op_freq + for op_name, op_info in self.op_freq.items(): + freq_list = op_info.get("freq_list", []) + if not freq_list: + continue + + op_count = op_info.get("count", 0) + op_total_duration = round(op_info.get("dur", 0), 2) + max_freq = max(self.DEFAULT_FREQ, convert_to_float(Config().get_config("aic_frequency"))) + + decrease_freq_ratio = sum(max_freq - freq for freq in freq_list) / (max_freq * len(freq_list)) + if decrease_freq_ratio >= self.DECREASE_FREQ_RATIO: + self.ai_core_freq_issues = True + self.decrease_freq_ops.append([op_name, op_count, op_total_duration, + f"{round(decrease_freq_ratio, 4):.2%}", + round(sum(freq_list) / len(freq_list), 2), + max(freq_list), min(freq_list)]) + + if self.decrease_freq_ops: + # 按算子总耗时和降频比率 降序排列 + self.decrease_freq_ops.sort(key= + lambda x: (x[self.TOTAL_DURATION_INDEX], x[self.DECREASE_FREQ_RATIO_INDEX]), + reverse=True) + + self.desc = (f"{len(self.decrease_freq_ops)} operators are found during frequency reduction, and the reduction " + f"ratio is larger than {self.DECREASE_FREQ_RATIO}.") + if self.rank_id: + self.desc = f"For rank {self.rank_id}, " + self.desc.lower() + self.suggestions = "Please check the temperature or max power of your machine." + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("AI Core Frequency", self.desc, [self.suggestions]) + result.add(OptimizeRecord(optimization_item)) + + self.headers = ["Operator name", "Count", "Total duration(us)", "AI CORE frequency decreased ratio", + "Average frequency", "Max frequency", "Min frequency"] + if self.rank_id: + self.headers = ["Rank id"] + self.headers + sub_table_name = "AI Core Frequency" if not self.stage else f"Stage-{self.stage}: AI Core Frequency" + result.add_detail(sub_table_name, headers=self.headers) + + for row in self.decrease_freq_ops: + if self.rank_id: + row = [self.rank_id] + row + result.add_detail(sub_table_name, detail=row) + + def make_render(self, html_render, add_render_list=True): + if self.SHOW_TOPK_OPS: + self.desc += f" Only show {self.SHOW_TOPK_OPS} operators here, see latest att_advisor.xlsx for details." + return html_render.render_template(key="computation", + template_dir="templates", + template_name="ai_core_frequency.html", + desc=self.desc, + suggestion=self.suggestions, + headers=self.headers, + data=self.decrease_freq_ops[:self.SHOW_TOPK_OPS], + add_render_list=add_render_list) diff --git a/profiler/advisor/analyzer/computation/profiling_analyzer.py b/profiler/advisor/analyzer/computation/profiling_analyzer.py index 868261770..2021bcd57 100644 --- a/profiler/advisor/analyzer/computation/profiling_analyzer.py +++ b/profiler/advisor/analyzer/computation/profiling_analyzer.py @@ -1,19 +1,15 @@ import logging from abc import ABC -from typing import Dict, List from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common import constant from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.computation.aicpu.aicpu_checker import AicpuChecker from profiler.advisor.analyzer.computation.bound.block_dim_checker import BlockDimChecker from profiler.advisor.analyzer.computation.bound.operator_bound_checker import OperatorBoundChecker -from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker from profiler.advisor.analyzer.computation.op_compile.dynamic_shape_checker import DynamicShapeChecker from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.utils.utils import get_supported_subclass logger = logging.getLogger() @@ -76,14 +72,15 @@ class BlockDimAnalyzer(ProfilingAnalyzer): def __init__(self, collection_path, **kwargs) -> None: super().__init__(collection_path, **kwargs) self.checker = BlockDimChecker(self.cann_version) - + class OperatorBoundAnalyzer(ProfilingAnalyzer): def __init__(self, collection_path, **kwargs) -> None: super().__init__(collection_path, **kwargs) self.checker = OperatorBoundChecker(self.cann_version) + class AicpuAnalyzer(ProfilingAnalyzer): def __init__(self, collection_path, **kwargs) -> None: super().__init__(collection_path, **kwargs) - self.checker = AicpuChecker(self.cann_version) \ No newline at end of file + self.checker = AicpuChecker(self.cann_version) diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 592f9d421..de31b2160 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -12,3 +12,4 @@ class SupportedScopes: BLOCK_DIM_ANALYSIS = "block_dim_analysis" OPERATOR_NO_BOUND_ANALYSIS = "operator_no_bound_analysis" TIMELINE_OP_DISPATCH = "timeline_op_dispatch" + FREQ_ANALYSIS = "freq_analysis" diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index 697430ee6..e5318dc0f 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -138,4 +138,7 @@ CLUSTER_STEP_TIME_CSV = "cluster_step_trace_time.csv" CLUSTER_COMM_JSON = "cluster_communication.json" BOTTLENECK = "bottleneck" -DATA = "data" \ No newline at end of file +DATA = "data" + +DISABLE_STREAMING_READER = "DISABLE_STREAMING_READER" +MAX_FILE_SIZE = 10**10 diff --git a/profiler/advisor/config/config.ini b/profiler/advisor/config/config.ini index c56c1dad9..06e993160 100644 --- a/profiler/advisor/config/config.ini +++ b/profiler/advisor/config/config.ini @@ -9,6 +9,7 @@ tune_ops_file = operator_tuning_file.cfg [THRESHOLD] # operator_bound_ratio: (mte, cube, vector, scalar) ratio greater than this value will be checked in operator_bound_checker operator_bound_ratio = 0.8 +frequency_threshold = 0.05 [RULE-BUCKET] # region : URL of different regions where can download rule yaml file cn-north-9 = cnnorth9-modelarts-sdk diff --git a/profiler/advisor/config/config.py b/profiler/advisor/config/config.py index 12f4526f8..4f36dfedf 100644 --- a/profiler/advisor/config/config.py +++ b/profiler/advisor/config/config.py @@ -97,6 +97,13 @@ class Config: """ return float(self.config.get("THRESHOLD", "operator_bound_ratio")) + @property + def frequency_threshold(self) -> float: + """ + frequency_threshold + """ + return float(self.config.get("THRESHOLD", "frequency_threshold")) + def set_log_path(self, result_file: str, log_path: str = None): self.log_path = log_path if log_path is not None else os.path.join(self._work_path, "log") os.makedirs(self.log_path, exist_ok=True) diff --git a/profiler/advisor/dataset/ai_core_freq/__init__.py b/profiler/advisor/dataset/ai_core_freq/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py b/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py new file mode 100644 index 000000000..c99baea65 --- /dev/null +++ b/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py @@ -0,0 +1,148 @@ +import json +import logging +import math +import os +import traceback + +import ijson +from tqdm import tqdm + +from profiler.advisor.common import constant as const +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.advisor.utils.utils import get_file_path_from_directory +from profiler.advisor.utils.utils import convert_to_float, parse_json_with_generator +from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser +from profiler.advisor.config.config import Config + +logger = logging.getLogger() + + +class AICoreFreqDataset: + + def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: + + self._profiler_step = [] + self._ai_core_ops = [] + self._ai_core_freq: [TimelineEvent] = [] + self._previous_freq_index = -1 + + self.timeline_dir = collection_path + self.timeline_data_list = get_file_path_from_directory(collection_path, + lambda file: file.endswith("trace_view.json")) + + self.step = kwargs.get("step") + self.op_freq = {} + info = DeviceInfoParser(collection_path) + info.parse_data() + if not Config().get_config("aic_frequency"): + return + if self.parse(): + key = self.get_key() + if key not in data: + data[key] = [] + data[key].append(self) + + @property + def profiler_step(self): + return self._profiler_step + + @property + def ai_core_freq(self): + return self._ai_core_freq + + @property + def ai_core_ops(self): + return self._ai_core_ops + + @classmethod + def get_key(cls): + """ + get key of dataset + :return: key + """ + return cls.__module__.rsplit('.', maxsplit=1)[-1] + + def parse(self): + + if len(self.timeline_data_list) == 0: + logger.warning("Please ensure trace_view.json in %s, skip timeline analysis.", self.timeline_dir) + return False + + if len(self.timeline_data_list) > 1: + logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis .", + self.timeline_dir) + + _ = parse_json_with_generator(sorted(self.timeline_data_list)[0], self._add_event) + + target_ai_core_ops = self._get_target_ai_core_ops() + self._get_op_frequency(target_ai_core_ops) + return True + + def _add_profiler_step(self, event): + if event.name.startswith("ProfilerStep"): + self._profiler_step.append(event) + + def _add_ai_core_ops(self, event): + if event.args.get("Task Type") in ["MIX_AIC", "AI_CORE"]: + self._ai_core_ops.append(event) + + def _add_ai_core_freq(self, event): + if event.name == "AI Core Freq": + if self._previous_freq_index != -1: + self._ai_core_freq[self._previous_freq_index]["end"] = event.get("ts", float(math.inf)) + self._previous_freq_index += 1 + event.setdefault("end", float(math.inf)) + self._ai_core_freq.append(event) + + def _add_event(self, index, event): + event["dataset_index"] = index + if not isinstance(event, TimelineEvent): + event = TimelineEvent(event) + + self._add_profiler_step(event) + self._add_ai_core_ops(event) + self._add_ai_core_freq(event) + + return True + + def _get_target_ai_core_ops(self): + target_ai_core_ops = [] + if not self.step or f"ProfilerStep#{self.step}" not in [event.name for event in self._profiler_step]: + target_ai_core_ops = self._ai_core_ops + else: + for step_event in self._profiler_step: + if step_event.name != f"ProfilerStep#{self.step}": + continue + + for ai_core_op_event in self._ai_core_ops: + if step_event.ts_include(ai_core_op_event): + target_ai_core_ops.append(ai_core_op_event) + target_ai_core_ops = sorted(target_ai_core_ops, key=lambda x: float(x.ts)) + return target_ai_core_ops + + def _get_op_frequency(self, ai_core_ops): + ai_core_freq = sorted(self._ai_core_freq, key=lambda x: float(x.ts)) + + op_index, freq_index = 0, 0 + while op_index < len(ai_core_ops) and freq_index < len(ai_core_freq): + op_event = ai_core_ops[op_index] + op_end_time = convert_to_float(op_event.ts) + convert_to_float(op_event.dur) + op_freq_list = [] + while freq_index < len(ai_core_freq): + freq_event = ai_core_freq[freq_index] + if convert_to_float(freq_event.end) < op_end_time: + op_freq_list.append(convert_to_float(freq_event.args.MHz)) + freq_index += 1 + continue + elif convert_to_float(freq_event.ts) < op_end_time: + if op_event.name not in self.op_freq: + self.op_freq[op_event.name] = {"count": 0, "dur": 0, "freq_list": []} + self.op_freq[op_event.name]["count"] += 1 + self.op_freq[op_event.name]["dur"] += convert_to_float(op_event.dur) + op_freq_list.append(convert_to_float(freq_event.args.MHz)) + self.op_freq[op_event.name]["freq_list"].append(min(op_freq_list)) + break + else: + break + + op_index += 1 diff --git a/profiler/advisor/dataset/profiling/device_info.py b/profiler/advisor/dataset/profiling/device_info.py index b58930777..110cd0794 100644 --- a/profiler/advisor/dataset/profiling/device_info.py +++ b/profiler/advisor/dataset/profiling/device_info.py @@ -54,6 +54,8 @@ class DeviceInfoParser: config.set_config("device_id", device_info["id"]) if "aiv_num" in device_info: config.set_config("aiv_num", device_info["aiv_num"]) + if "aic_frequency" in device_info: + config.set_config("aic_frequency", device_info["aic_frequency"]) if "ai_core_num" in device_info: config.set_config("ai_core_num", device_info["ai_core_num"]) return True diff --git a/profiler/advisor/display/html/templates/ai_core_frequency.html b/profiler/advisor/display/html/templates/ai_core_frequency.html new file mode 100644 index 000000000..d04514203 --- /dev/null +++ b/profiler/advisor/display/html/templates/ai_core_frequency.html @@ -0,0 +1,27 @@ +{% if data|length > 0 %} +
+

AI CORE Frequency Issues

+
+ Issue: {{ desc }} +
+ Suggestion: {{ suggestion }} +

+ + + {% for header in headers %} + + {% endfor %} + + + {% for row in data %} + + {% for element in row %} + + {% endfor %} + + {% endfor %} +
{{ header }}
{{ element|safe }}
+ +
+
+{% endif %} \ No newline at end of file diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index 59bfee77f..20b7a5f0c 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -13,6 +13,7 @@ from profiler.advisor.analyzer.cluster.slow_rank_analyser import SlowRankAnalyze from profiler.advisor.analyzer.cluster.slow_link_analyser import SlowLinkAnalyzer from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer +from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_analyzer import AICoreFreqAnalyzer class Interface: supported_analyzer = { @@ -25,7 +26,8 @@ class Interface: SupportedScopes.AICPU_ANALYSIS: AicpuAnalyzer, SupportedScopes.OPERATOR_NO_BOUND_ANALYSIS: OperatorBoundAnalyzer, SupportedScopes.BLOCK_DIM_ANALYSIS: BlockDimAnalyzer, - SupportedScopes.GRAPH: FusionOPAnalyzer + SupportedScopes.GRAPH: FusionOPAnalyzer, + SupportedScopes.FREQ_ANALYSIS: AICoreFreqAnalyzer }), "communication": OrderedDict(), "overall": OrderedDict({SupportedScopes.OVER_ALL: OverallSummaryAnalyzer}), diff --git a/profiler/advisor/result/result.py b/profiler/advisor/result/result.py index c7d7da866..42b617ca5 100644 --- a/profiler/advisor/result/result.py +++ b/profiler/advisor/result/result.py @@ -93,6 +93,9 @@ class SheetRecoder: if data not in self._sheet_data[sheet_name]["data"]: self._sheet_data[sheet_name]["data"].append(data) + def clear(self): + self._sheet_data.clear() + @singleton class OptimizeResult: @@ -110,12 +113,12 @@ class OptimizeResult: def add_tune_op_list(self, tune_op_list) -> None: """ add tune op name to tune op list - :param tune_op_list: tune op name list to be added + :param tune_op_list: list of operators to be optimized :return: None """ - for op_name in tune_op_list: - if op_name not in self._tune_op_list: - self._tune_op_list.append(op_name) + for operator_name in tune_op_list: + if operator_name not in self._tune_op_list: + self._tune_op_list.append(operator_name) def add(self, overview_item): sheet_name = "problems" @@ -148,6 +151,9 @@ class OptimizeResult: logger.info("Save problems details file to %s", Config().analysis_result_file) self._save_op_file_list() + def clear(self) -> None: + self.data.clear() + def _save_op_file_list(self) -> None: if not self._tune_op_list: return diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index 84419b670..dd8342320 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -1,5 +1,6 @@ import inspect import json + import logging import multiprocessing as mp import os @@ -11,7 +12,7 @@ import traceback import types from functools import wraps from typing import Any, Set - +import ijson import click import requests from requests.adapters import HTTPAdapter @@ -550,3 +551,50 @@ def get_file_path_by_walk(root, filename): file_path = os.path.join(root, name) return file_path return file_path + + +def check_path_valid(path): + if os.path.islink(os.path.abspath(path)): + logger.error("fThe path is detected as a soft connection. path:%ss", path) + return False + elif not os.access(path, os.R_OK): + logger.error(f"The file is not readable. path:%ss", path) + return False + elif os.path.getsize(path) > const.MAX_FILE_SIZE: + logger.error(f"The file size exceeds the limit. path:%ss, MAX_FILE_SIZE:%ss B",path, const.MAX_FILE_SIZE) + return False + return True + + +def parse_json_with_generator(timeline_data_path, func): + result = [] + if not check_path_valid(timeline_data_path): + return result + try: + with open(timeline_data_path, "r") as f: + if os.getenv(const.DISABLE_STREAMING_READER) == "1": + logger.debug("Disable streaming reader.") + file_parser = json.loads(f.read()) + else: + logger.debug("Enable streaming reader.") + file_parser = ijson.items(f, "item") + + for i, event in tqdm(enumerate(file_parser), + leave=False, ncols=100, desc="Building dataset for timeline analysis"): + func_res = func(index=i, event=event) + if func_res is not None: + result.append(func_res) + + except Exception: + logger.warning("Error %s while parsing file %s, continue to timeline analysis", traceback.format_exc(), + timeline_data_path) + return result + + +def convert_to_float(num): + try: + return float(num) + except (ValueError, FloatingPointError): + logger.error(f"Can not convert %ss to float", num) + pass + return 0 diff --git a/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py b/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py new file mode 100644 index 000000000..51acf3b8e --- /dev/null +++ b/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py @@ -0,0 +1,145 @@ +import os +import shutil +import stat +import json + +import unittest +from profiler.advisor.interface.interface import Interface +from profiler.advisor.common.analyzer_scopes import SupportedScopes + + +class TestFrequencyAdvice(unittest.TestCase): + TMP_DIR = "./ascend_pt" + OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" + DEVICE_DIR = "./ascend_pt/PROF_000001_20240415174447255_OAANHDOMMJMHGIFC/device_0" + interface = None + err_interface = None + + def tearDown(self): + if os.path.exists(TestFrequencyAdvice.TMP_DIR): + shutil.rmtree(TestFrequencyAdvice.TMP_DIR) + self.clear_htmls() + + def setUp(self): + if os.path.exists(TestFrequencyAdvice.TMP_DIR): + shutil.rmtree(TestFrequencyAdvice.TMP_DIR) + if not os.path.exists(TestFrequencyAdvice.TMP_DIR): + os.makedirs(TestFrequencyAdvice.TMP_DIR) + if not os.path.exists(TestFrequencyAdvice.OUTPUT_DIR): + os.makedirs(TestFrequencyAdvice.OUTPUT_DIR) + if not os.path.exists(TestFrequencyAdvice.DEVICE_DIR): + os.makedirs(TestFrequencyAdvice.DEVICE_DIR) + self.clear_htmls() + + @classmethod + def clear_htmls(cls): + current_path = os.path.dirname(os.path.abspath(__file__)) + for filename in os.listdir(current_path): + # 检查文件是否以“att”开头 + if filename.startswith("att"): + # 构建文件的完整路径 + file_path = os.path.join(current_path, filename) + # 删除文件 + os.remove(file_path) + + @classmethod + def get_basic_trace_view(cls): + # Python pid + py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} + # ascend pid + ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} + # ascend pid + cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} + # ascend hardware ops + ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, + "args": {"Task Type": "AI_CORE"}} + ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, + "args": {"Task Type": "AI_CORE"}} + # flow event + flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} + flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} + return [py_pid_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, flow_event_s, flow_event_e] + + @classmethod + def create_info_json(cls): + info = { + "DeviceInfo": [ + { + "id": 7, + "env_type": 3, + "ctrl_cpu_id": "ARMv8_Cortex_A55", + "ctrl_cpu_core_num": 1, + "ctrl_cpu_endian_little": 1, + "ts_cpu_core_num": 0, + "ai_cpu_core_num": 6, + "ai_core_num": 25, + "ai_cpu_core_id": 2, + "ai_core_id": 0, + "aicpu_occupy_bitmap": 252, + "ctrl_cpu": "0", + "ai_cpu": "2,3,4,5,6", + "aiv_num": 50, + "hwts_frequency": "49.999001", + "aic_frequency": "1850", + "aiv_frequency": "1850" + } + ] + } + with os.fdopen(os.open(f"{TestFrequencyAdvice.DEVICE_DIR}/info.json.0", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(info)) + + @classmethod + def create_non_910B_trace_view(cls): + basic_info = cls.get_basic_trace_view() + + # python ops + py_event1 = {"ph": "X", "cat": "python_function", "name": "aten::slice", "ts": "200", "dur": 100, "tid": 2, + "pid": 1, + "args": {"Call stack": "/root/test/slice.py(116);\r\n/root/torch/module.py"}} + py_event2 = {"ph": "X", "cat": "python_function", "name": "slice", "ts": "199", "dur": 200, "tid": 2, "pid": 1, + "args": {"Call stack": "/root/test/slice.py(116);\r\n/root/torch/module.py"}} + raw_data = [ + *basic_info, py_event1, py_event2 + ] + with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/trace_view.json", + # with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/msprof_20240415174455.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + @classmethod + def create_910B_trace_view(cls): + basic_info = cls.get_basic_trace_view() + + # python ops + py_event1 = {"name": "AI Core Freq", "ts": "1699529623106000.061", "pid": 682820896, "tid": 0, + "args": {"MHz": 1850}, "ph": "C"} + py_event2 = {"name": "AI Core Freq", "ts": "1699529623106770.541", "pid": 682820896, "tid": 0, + "args": {"MHz": 800}, "ph": "C"} + raw_data = [ + *basic_info, py_event1, py_event2 + ] + + with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/trace_view.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + def test_run_should_run_success_when_msprof_not_contain_frequency_data(self): + self.create_info_json() + self.create_non_910B_trace_view() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "computation" + scope = SupportedScopes.FREQ_ANALYSIS + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(0, len(result.data.get("AI Core Frequency", []))) + result.clear() + + def test_run_should_run_success_when_trace_view_contain_frequency_data(self): + self.create_info_json() + self.create_910B_trace_view() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "computation" + scope = SupportedScopes.FREQ_ANALYSIS + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(2, len(result.data.get("AI Core Frequency", dict).get("data", []))) + result.clear() -- Gitee From 96632320df063cf3340d6bc60baec2082f4df3eb Mon Sep 17 00:00:00 2001 From: wuyuhan Date: Mon, 22 Jul 2024 15:57:02 +0800 Subject: [PATCH 015/791] =?UTF-8?q?dataloader,=20syncBatchNorm,=20synchron?= =?UTF-8?q?izeStream=20=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- debug/__init__.py | 0 debug/accuracy_tools/LICENSE | 201 ++++ debug/accuracy_tools/MANIFEST.in | 7 +- debug/accuracy_tools/grad_tool/README.md | 4 +- .../grad_tool/common/constant.py | 8 + .../grad_tool/grad_ms/global_context.py | 13 +- .../grad_tool/grad_ms/grad_stat_csv.py | 130 +++ .../accuracy_tools/grad_tool/grad_ms/hook.py | 84 +- .../accuracy_tools/grad_tool/grad_ms/utils.py | 42 + debug/accuracy_tools/msprobe/README.md | 33 + .../accuracy_tools/msprobe/config/config.json | 5 + .../msprobe/core/common/const.py | 15 +- .../msprobe/core/common/exceptions.py | 36 +- .../msprobe/core/common_config.py | 22 +- .../core/data_dump/data_processor/base.py | 4 +- .../core/data_dump/data_processor/factory.py | 6 +- .../data_processor/mindspore_processor.py | 131 +++ .../data_processor/pytorch_processor.py | 4 +- .../msprobe/mindspore/common/log.py | 38 + .../msprobe/mindspore/common/utils.py | 31 + .../mindspore/debugger/debugger_config.py | 8 +- .../mindspore/debugger/precision_debugger.py | 25 +- .../mindspore/dump/hook_cell/api_registry.py | 104 ++ .../mindspore/dump/hook_cell/hook_cell.py | 57 ++ .../dump/hook_cell/support_wrap_ops.yaml | 925 ++++++++++++++++++ .../dump/hook_cell/wrap_functional.py | 94 ++ .../mindspore/dump/hook_cell/wrap_tensor.py | 66 ++ .../msprobe/mindspore/ms_config.py | 11 +- .../msprobe/mindspore/service.py | 138 +++ .../api_accuracy_checker/common/config.py | 18 +- .../api_accuracy_checker/common/utils.py | 1 + .../pytorch/api_accuracy_checker/config.yaml | 1 + .../run_ut/run_overflow_check.py | 16 +- .../api_accuracy_checker/run_ut/run_ut.py | 53 +- .../msprobe/pytorch/common/utils.py | 1 - .../msprobe/pytorch/compare/acc_compare.py | 5 + .../pytorch/debugger/precision_debugger.py | 6 +- .../msprobe/pytorch/functional/dump_module.py | 6 +- .../pytorch/online_dispatch/compare.py | 2 +- .../msprobe/pytorch/pt_config.py | 43 +- .../accuracy_tools/msprobe/pytorch/service.py | 6 +- .../test/core_ut/test_common_config.py | 22 +- .../test/mindspore_ut/test_debugger_config.py | 2 +- .../mindspore_ut/test_precision_debugger.py | 2 +- .../common/test_config.py | 2 +- .../pytorch_ut/compare/test_acc_compare.py | 260 ++++- .../test/pytorch_ut/compare/test_match.py | 20 + .../msprobe/test/pytorch_ut/test_pt_config.py | 15 + debug/accuracy_tools/setup.py | 70 +- profiler/advisor/README.md | 6 +- profiler/advisor/analyzer/base_analyzer.py | 22 +- .../analyzer/cluster/slow_link_analyser.py | 18 +- .../analyzer/cluster/slow_rank_analyser.py | 6 +- .../computation/ai_core_freq/__init__.py | 0 .../ai_core_freq/ai_core_freq_analyzer.py | 42 + .../ai_core_freq/ai_core_freq_checker.py | 100 ++ .../computation/aicpu/aicpu_checker.py | 6 +- .../computation/bound/block_dim_checker.py | 1 - .../computation/profiling_analyzer.py | 9 +- .../dataloader/dataloader_analyzer.py | 30 + .../analyzer/dataloader/dataloader_checker.py | 84 ++ .../graph_fusion/graph_fusion_checker.py | 2 +- .../analyzer/overall/overall_analyzer.py | 45 - .../overall/overall_summary_analyzer.py | 240 +++-- .../analyzer/schedule/syncbn/__init__.py | 0 .../schedule/syncbn/syncbn_analyzer.py | 30 + .../schedule/syncbn/syncbn_checker.py | 70 ++ .../schedule/synchronize_stream/__init__.py | 0 .../synchronize_stream_analyzer.py | 32 + .../synchronize_stream_checker.py | 89 ++ .../schedule/timeline_base_checker.py | 91 ++ profiler/advisor/common/analyzer_scopes.py | 4 + profiler/advisor/common/constant.py | 9 +- profiler/advisor/common/graph/graph_parser.py | 9 +- profiler/advisor/common/timeline/event.py | 5 +- .../advisor/common/timeline/fusion_ops_db.py | 6 +- profiler/advisor/config/config.ini | 1 + profiler/advisor/config/config.py | 7 + .../advisor/dataset/ai_core_freq/__init__.py | 0 .../ai_core_freq/ai_core_freq_dataset.py | 148 +++ .../dataset/cluster/cluster_dataset.py | 14 +- .../advisor/dataset/profiling/device_info.py | 2 + .../dataset/profiling/profiling_dataset.py | 4 +- .../advisor/dataset/timeline_event_dataset.py | 165 +++- profiler/advisor/display/html/render.py | 5 +- .../html/templates/ai_core_frequency.html | 27 + .../html/templates/slow_dataloader.html | 18 + .../html/templates/sync_batchnorm.html | 30 + .../html/templates/synchronize_stream.html | 57 ++ profiler/advisor/interface/interface.py | 18 +- profiler/advisor/result/item.py | 2 +- profiler/advisor/result/result.py | 18 +- profiler/advisor/rules/dataloader.yaml | 9 + profiler/advisor/rules/sync_batchnorm.yaml | 41 + profiler/advisor/rules/synchronize.yaml | 8 + profiler/advisor/utils/utils.py | 52 +- profiler/cli/__init__.py | 2 +- profiler/cli/analyze_cli.py | 3 - profiler/cluster_analyse/README.md | 12 +- .../common_func/file_manager.py | 19 + profiler/compare_tools/README.md | 2 +- .../comparator/base_comparator.py | 2 +- .../comparator/overall_metrics_comparator.py | 50 + .../origin_data_bean/kernel_details_bean.py | 33 +- .../origin_data_bean/trace_event_bean.py | 50 +- .../compare_bean/overall_metrics_bean.py | 255 +++++ .../compare_bean/profiling_info.py | 184 +++- .../disaggregate/overall_perf_interface.py | 28 +- .../generator/detail_performance_generator.py | 15 +- .../profiling_parser/base_profiling_parser.py | 97 ++ .../profiling_parser/gpu_profiling_parser.py | 27 +- .../profiling_parser/npu_profiling_parser.py | 36 +- .../compare_backend/utils/constant.py | 8 + .../compare_backend/utils/excel_config.py | 79 +- .../view/work_sheet_creator.py | 29 +- profiler/test/run_ut.py | 2 + .../test_dataloader_checker.py | 65 ++ .../timeline_advice/test_syncbn_checker.py | 62 ++ .../test_synchronize_stream.py | 55 ++ .../compute_advice/test_frequency_advice.py | 145 +++ .../test_kernel_details_bean.py | 4 +- .../test_gpu_profiling_parser.py | 1 + 123 files changed, 5150 insertions(+), 492 deletions(-) create mode 100644 debug/__init__.py create mode 100644 debug/accuracy_tools/LICENSE create mode 100644 debug/accuracy_tools/grad_tool/grad_ms/grad_stat_csv.py create mode 100644 debug/accuracy_tools/grad_tool/grad_ms/utils.py create mode 100644 debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/common/log.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/common/utils.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml create mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/service.py create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py create mode 100644 profiler/advisor/analyzer/computation/ai_core_freq/__init__.py create mode 100644 profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py create mode 100644 profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py create mode 100644 profiler/advisor/analyzer/dataloader/dataloader_analyzer.py create mode 100644 profiler/advisor/analyzer/dataloader/dataloader_checker.py delete mode 100644 profiler/advisor/analyzer/overall/overall_analyzer.py create mode 100644 profiler/advisor/analyzer/schedule/syncbn/__init__.py create mode 100644 profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py create mode 100644 profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py create mode 100644 profiler/advisor/analyzer/schedule/synchronize_stream/__init__.py create mode 100644 profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py create mode 100644 profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py create mode 100644 profiler/advisor/analyzer/schedule/timeline_base_checker.py create mode 100644 profiler/advisor/dataset/ai_core_freq/__init__.py create mode 100644 profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py create mode 100644 profiler/advisor/display/html/templates/ai_core_frequency.html create mode 100644 profiler/advisor/display/html/templates/slow_dataloader.html create mode 100644 profiler/advisor/display/html/templates/sync_batchnorm.html create mode 100644 profiler/advisor/display/html/templates/synchronize_stream.html create mode 100644 profiler/advisor/rules/dataloader.yaml create mode 100644 profiler/advisor/rules/sync_batchnorm.yaml create mode 100644 profiler/advisor/rules/synchronize.yaml create mode 100644 profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py create mode 100644 profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py create mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py create mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py create mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py create mode 100644 profiler/test/ut/advisor/compute_advice/test_frequency_advice.py diff --git a/README.md b/README.md index 014a4d59f..dd25d2015 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ MindStudio Training Tools，MindStudio训练工具链。针对训练&大模型 2. [cluster_analyse（集群分析工具）](https://gitee.com/ascend/mstt/tree/master/profiler/cluster_analyse) - 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合Ascend Insight的集群分析功能使用。 + 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合MindStudio Insight的集群分析功能使用。 3. [affinity_cpu_bind (亲和性cpu绑核工具) ](https://gitee.com/ascend/mstt/tree/master/profiler/affinity_cpu_bind) diff --git a/debug/__init__.py b/debug/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/LICENSE b/debug/accuracy_tools/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/debug/accuracy_tools/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/debug/accuracy_tools/MANIFEST.in b/debug/accuracy_tools/MANIFEST.in index 547864a6c..7997215ff 100644 --- a/debug/accuracy_tools/MANIFEST.in +++ b/debug/accuracy_tools/MANIFEST.in @@ -1,2 +1,5 @@ -recursive-include msprobe/ * -recursive-exclude msprobe/test * \ No newline at end of file +include README.md +include LICENSE +recursive-include msprobe * +recursive-exclude msprobe/test * + diff --git a/debug/accuracy_tools/grad_tool/README.md b/debug/accuracy_tools/grad_tool/README.md index a3f683b41..a7929ca81 100644 --- a/debug/accuracy_tools/grad_tool/README.md +++ b/debug/accuracy_tools/grad_tool/README.md @@ -54,7 +54,7 @@ **不同级别的level的导出数据** -- PyTorch不同level数据 +- PyTorch/MindSpore动态图不同level数据 | 级别 | 特征数据表头 | 是否有方向数据 | | ---- | ------------------------------------------------------------ | -------------- | @@ -62,7 +62,7 @@ | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | -- MindSpore不同level数据 +- MindSpore静态图不同level数据 | 级别 | 特征数据表头 | 是否有方向数据 | | ---- | ------------------------------------------------------------ | -------------- | diff --git a/debug/accuracy_tools/grad_tool/common/constant.py b/debug/accuracy_tools/grad_tool/common/constant.py index 902f54f5e..d569d47c1 100644 --- a/debug/accuracy_tools/grad_tool/common/constant.py +++ b/debug/accuracy_tools/grad_tool/common/constant.py @@ -46,3 +46,11 @@ class GradConst: STEP_FINISH = "step_finish" SUMMARY = "summary" + + # csv header entry + MD5 = "MD5" + DISTRIBUTION = "distribution" + SHAPE = "shape" + MAX = "max" + MIN = "min" + NORM = "norm" \ No newline at end of file diff --git a/debug/accuracy_tools/grad_tool/grad_ms/global_context.py b/debug/accuracy_tools/grad_tool/grad_ms/global_context.py index 02d1f7445..d44bea52c 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/global_context.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/global_context.py @@ -14,6 +14,8 @@ class GlobalContext: _setting = { GradConst.LEVEL: GradConst.LEVEL0, GradConst.PARAM_LIST: None, + GradConst.STEP: None, + GradConst.RANK: None, GradConst.CURRENT_STEP: 0, GradConst.BOUNDS: [-1., 0., 1.], GradConst.OUTPUT_PATH: "./grad_stat" @@ -33,6 +35,8 @@ class GlobalContext: print_warn_log("Invalid level set in config yaml file, use L0 instead.") self._set_input_list(config_dict, GradConst.PARAM_LIST, str) self._set_input_list(config_dict, GradConst.BOUNDS, float) + self._set_input_list(config_dict, GradConst.STEP, int) + self._set_input_list(config_dict, GradConst.RANK, int) output_path = config_dict.get(GradConst.OUTPUT_PATH) if output_path: try: @@ -55,6 +59,14 @@ class GlobalContext: def update_step(self): self._setting[GradConst.CURRENT_STEP] += 1 + def step_need_dump(self, step): + dump_step_list = self.get_context(GradConst.STEP) + return (not dump_step_list) or (step in dump_step_list) + + def rank_need_dump(self, rank): + dump_rank_list = self.get_context(GradConst.RANK) + return (not dump_rank_list) or (rank in dump_rank_list) + def _set_input_list(self, config_dict: Dict, name: str, dtype: Union[int, str, float]): value = config_dict.get(name) if dtype == int: @@ -72,5 +84,4 @@ class GlobalContext: else: print_warn_log(f"{name} is None or not a list with valid items, use default value.") - grad_context = GlobalContext() diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_stat_csv.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_stat_csv.py new file mode 100644 index 000000000..11c2fc820 --- /dev/null +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_stat_csv.py @@ -0,0 +1,130 @@ +from abc import ABC, abstractmethod +import hashlib + +import mindspore +from mindspore import ops, Tensor +from grad_tool.common.constant import GradConst + + +class CsvInput: + def __init__(self, param_name, grad, bounds): + self.param_name = param_name + self.grad = grad + self.bounds = bounds + +class GradStatCsv: + csv = {} + + @staticmethod + def get_csv_header(level, csv_input): + header = ["param_name"] + for key in level["header"]: + header.extend(GradStatCsv.csv[key].generate_csv_header(csv_input)) + return header + + @staticmethod + def get_csv_line(level, csv_input): + line = [csv_input.param_name] + for key in level["header"]: + line.extend(GradStatCsv.csv[key].generate_csv_content(csv_input)) + return line + + +def register_csv_item(key, cls=None): + if cls is None: + # 无参数时，返回装饰器函数 + return lambda cls: register_csv_item(key, cls) + GradStatCsv.csv[key] = cls + return cls + + +class CsvItem(ABC): + @staticmethod + @abstractmethod + def generate_csv_header(csv_input): + pass + + @staticmethod + @abstractmethod + def generate_csv_content(csv_input): + pass + + +@register_csv_item(GradConst.MD5) +class CsvMd5(CsvItem): + def generate_csv_header(csv_input): + return ["MD5"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + tensor_bytes = grad.float().numpy().tobytes() + md5_hash = hashlib.md5(tensor_bytes) + return [md5_hash.hexdigest()] + + +@register_csv_item(GradConst.DISTRIBUTION) +class CsvDistribution(CsvItem): + def generate_csv_header(csv_input): + bounds = csv_input.bounds + intervals = [] + for i, _ in enumerate(bounds): + if i == 0: + intervals.append(f"(-inf, {bounds[i]}]") + else: + intervals.append(f"({bounds[i-1]}, {bounds[i]}]") + intervals.extend([f"({bounds[-1]}, inf)", "=0"]) + return intervals + + def generate_csv_content(csv_input): + grad = csv_input.grad + bounds = csv_input.bounds + if grad.dtype == mindspore.bfloat16: + grad = grad.to(mindspore.float32) + element_num = grad.numel() + grad_equal_0_num = (grad == 0).sum().item() + bucketsize_result = ops.bucketize(grad.float(), bounds) + bucketsize_result = bucketsize_result.astype(mindspore.int8) + interval_nums = [(bucketsize_result == i).sum().item() for i in range(len(bounds) + 1)] + interval_nums.append(grad_equal_0_num) + return_list = [x / element_num if element_num != 0 else 0 for x in interval_nums] + return return_list + + +@register_csv_item(GradConst.MAX) +class CsvMax(CsvItem): + def generate_csv_header(csv_input): + return ["max"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.amax(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.MIN) +class CsvMin(CsvItem): + def generate_csv_header(csv_input): + return ["min"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.amin(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.NORM) +class CsvNorm(CsvItem): + def generate_csv_header(csv_input): + return ["norm"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.norm(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.SHAPE) +class CsvShape(CsvItem): + def generate_csv_header(csv_input): + return ["shape"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [list(grad.shape)] \ No newline at end of file diff --git a/debug/accuracy_tools/grad_tool/grad_ms/hook.py b/debug/accuracy_tools/grad_tool/grad_ms/hook.py index ceadfee61..f0d479818 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/hook.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/hook.py @@ -1,4 +1,4 @@ -from functools import wraps + import os import shutil @@ -10,38 +10,82 @@ from mindspore.common.parameter import Parameter from mindspore.common.initializer import initializer from grad_tool.common.constant import GradConst -from grad_tool.common.utils import print_warn_log +from grad_tool.common.utils import print_warn_log, write_csv from grad_tool.grad_ms.global_context import grad_context from grad_tool.grad_ms.grad_analyzer import grad_dump, get_rank_id from grad_tool.grad_ms.grad_analyzer import csv_generator +from grad_tool.grad_ms.grad_stat_csv import GradStatCsv, CsvInput +from grad_tool.grad_ms.utils import save_grad_direction, get_adapted_level +class HookInput: -def hook_optimizer(opt: Optimizer): - func = opt.construct - g_names = [param.name for param in opt._parameters] - param_list = grad_context.get_context(GradConst.PARAM_LIST) - rank_id = get_rank_id() - output_path = grad_context.get_context(GradConst.OUTPUT_PATH) - dump_dir = f"{output_path}/rank_{rank_id}/Dump/" - save_dir = f"{output_path}/rank_{rank_id}/" - step_finish_flag = f"{output_path}/rank_{rank_id}/Dump/{GradConst.STEP_FINISH}" - if os.path.exists(save_dir): - print_warn_log(f"Delete existing path {save_dir}.") - shutil.rmtree(save_dir) - level = grad_context.get_context(GradConst.LEVEL) - bounds = grad_context.get_context(GradConst.BOUNDS) + ''' + HookInput is a class wrapping all the variables used for hooking optimizer + ''' + + def __init__(self, opt) -> None: + self.func = opt.construct + self.g_names = [param.name for param in opt._parameters] + self.param_list = grad_context.get_context(GradConst.PARAM_LIST) + self.rank_id = get_rank_id() + output_path = grad_context.get_context(GradConst.OUTPUT_PATH) + self.dump_dir = os.path.join(output_path, f"rank_{self.rank_id}", "Dump") + self.save_dir = os.path.join(output_path, f"rank_{self.rank_id}") + self.step_finish_flag = os.path.join(self.dump_dir, GradConst.STEP_FINISH) + if os.path.exists(self.save_dir): + print_warn_log(f"Delete existing path {self.save_dir}.") + shutil.rmtree(self.save_dir) + self.level = grad_context.get_context(GradConst.LEVEL) + self.bounds = grad_context.get_context(GradConst.BOUNDS) + self.mode = mindspore.get_context("mode") +def hook_graph_mode_optimizer(opt, hook_input): @jit def new_construct(self, gradients): for index, grad_value in enumerate(gradients): - if param_list and g_names[index] not in param_list: + if hook_input.param_list and hook_input.g_names[index] not in hook_input.param_list: continue - grad_dump(dump_dir, g_names[index], self.dump_step, grad_value, level, bounds) - ms.ops.TensorDump()(step_finish_flag, self.dump_step) + grad_dump(hook_input.dump_dir, hook_input.g_names[index], self.dump_step, + grad_value, hook_input.level, hook_input.bounds) + ms.ops.TensorDump()(hook_input.step_finish_flag, self.dump_step) self.assignadd(self.dump_step, self.global_step_increase_tensor) - out = func(gradients) + out = hook_input.func(gradients) return out opt.dump_step = Parameter(initializer(0, [1], ms.int32), name="dump_step") opt.construct = new_construct.__get__(opt, type(opt)) csv_generator.start() + +def hook_pynative_optimizer(opt, hook_input): + level_adapted = get_adapted_level(hook_input.level) + + def hook_fn(cell, input): + gradients, = input + cur_step = grad_context.get_context(GradConst.CURRENT_STEP) + if grad_context.step_need_dump(cur_step) and grad_context.rank_need_dump(hook_input.rank_id): + output_lines = [] + for index, grad_value in enumerate(gradients): + param_name = hook_input.g_names[index] + if hook_input.param_list and param_name not in hook_input.param_list: + continue + csv_input = CsvInput(param_name, grad_value, hook_input.bounds) + grad_info = GradStatCsv.get_csv_line(level_adapted, csv_input) + output_lines.append(grad_info) + if level_adapted["have_grad_direction"]: + save_grad_direction(param_name, grad_value, os.path.join(hook_input.save_dir, f'step_{cur_step}')) + output_csv_path = os.path.join(hook_input.save_dir, f"grad_summary_{cur_step}.csv") + dummy_csv_input = CsvInput(None, None, hook_input.bounds) + write_csv(output_csv_path, output_lines, + GradStatCsv.get_csv_header(level_adapted, dummy_csv_input)) + grad_context.update_step() + + opt.register_forward_pre_hook(hook_fn) + + +def hook_optimizer(opt: Optimizer): + hook_input = HookInput(opt) + + if hook_input.mode == mindspore.GRAPH_MODE: + hook_graph_mode_optimizer(opt, hook_input) + else: + hook_pynative_optimizer(opt, hook_input) \ No newline at end of file diff --git a/debug/accuracy_tools/grad_tool/grad_ms/utils.py b/debug/accuracy_tools/grad_tool/grad_ms/utils.py new file mode 100644 index 000000000..23703f282 --- /dev/null +++ b/debug/accuracy_tools/grad_tool/grad_ms/utils.py @@ -0,0 +1,42 @@ +import os + +import numpy as np +import mindspore +from grad_tool.common.constant import GradConst +from grad_tool.common.utils import print_warn_log, create_directory, change_mode, check_file_or_directory_path + +level_adp = { + "L0": { + "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": False + }, + "L1": { + "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": True + }, + "L2": { + "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": True + }, + } + +def save_grad_direction(param_name, grad, save_path): + if not os.path.exists(save_path): + create_directory(save_path) + save_filepath = os.path.join(save_path, f"{param_name}.npy") + check_file_or_directory_path(save_filepath) + + if grad.dtype == mindspore.bfloat16: + grad = grad.to(mindspore.float32) + grad_direction_tensor = grad > 0 + grad_direction_ndarray = grad_direction_tensor.numpy() + + np.save(save_filepath, grad_direction_ndarray) + change_mode(save_filepath, 0o640) + +def get_adapted_level(level: str): + if level == GradConst.LEVEL3: + print_warn_log(f"In mindpsore pynative mode, only 'L0', 'L1' and 'L2' are supported, use L0 instead") + level = GradConst.LEVEL0 + level_adapted = level_adp.get(level) + return level_adapted \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 84c531995..1e8c1a1f0 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -6,6 +6,16 @@ MindStudio精度调试工具（MindStudio Probe），简称msprobe，是MindStud 精度工具合一软件包名称：`mindstudio_probe-{version}-py3-none-any.whl` +### pip安装 + ```shell + pip install mindstudio-probe + ``` + 说明 + 1. 使用`pip install mindstudio-probe==版本号`可安装指定版本的包 + 2. pip命令会自动安装包及其依赖 + 3. 安装成功后，日志会显示`Successfully installed mindstudio-probe-版本号` + +### 下载whl包安装 1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、torch、tqdm依赖。 若环境中已安装部分依赖，不需要重复安装。 @@ -58,6 +68,29 @@ MindStudio精度调试工具（MindStudio Probe），简称msprobe，是MindStud Successfully installed mindstudio_probe-{version} ``` +### 从源码安装 +1. 克隆或者下载项目源代码 + + ```shell + git clone https://gitee.com/ascend/mstt.git + cd debug/accuracy_tools + ``` + +2. 安装setuptools和wheel + + ```shell + pip install setuptools wheel + ``` + +3. 安装msprobe + + ```shell + python setup.py install + ``` + 提示出现如下信息则表示源码安装成功。 + ```shell + Finished processing dependencies for mindstudio-probe=={version} + ``` ## 工具使用 diff --git a/debug/accuracy_tools/msprobe/config/config.json b/debug/accuracy_tools/msprobe/config/config.json index 70a630a40..c6077b75a 100644 --- a/debug/accuracy_tools/msprobe/config/config.json +++ b/debug/accuracy_tools/msprobe/config/config.json @@ -24,5 +24,10 @@ "overflow_check": { "overflow_nums": 1, "check_mode":"all" + }, + "run_ut": { + "white_list": [], + "black_list": [], + "error_data_path": "./" } } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index dea829c3f..df82455a6 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -2,6 +2,7 @@ import os import stat import numpy as np + class Const: """ Class for const @@ -15,6 +16,10 @@ class Const: OFF = 'OFF' BACKWARD = 'backward' FORWARD = 'forward' + DEFAULT_LIST = [] + DEFAULT_PATH = './' + WHITE_LIST = 'white_list' + BLACK_LIST = 'black_list' # dump mode ALL = "all" @@ -25,6 +30,8 @@ class Const: API_LIST = "api_list" API_STACK = "api_stack" DUMP_MODE = [ALL, LIST, RANGE, STACK, ACL, API_LIST, API_STACK] + AUTO = "auto" + ONLINE_DUMP_MODE = [ALL, LIST, AUTO, OFF] SUMMARY = "summary" MD5 = "md5" SUMMARY_MODE = [ALL, SUMMARY, MD5] @@ -35,6 +42,7 @@ class Const: PKL_SUFFIX = ".pkl" NUMPY_SUFFIX = ".npy" + PT_SUFFIX = ".pt" ONE_GB = 1073741824 # 1 * 1024 * 1024 * 1024 TEN_GB = 10737418240 # 10 * 1024 * 1024 * 1024 FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$' @@ -52,13 +60,15 @@ class Const: ENV_ENABLE = "1" ENV_DISABLE = "0" MAX_SEED_VALUE = 4294967295 # 2**32 - 1 - TASK_LIST = ["tensor", "statistics", "overflow_check", "free_benchmark"] + TASK_LIST = ["tensor", "statistics", "overflow_check", "free_benchmark", "run_ut"] LEVEL_LIST = ["L0", "L1", "L2", "mix"] STATISTICS = "statistics" TENSOR = "tensor" OVERFLOW_CHECK = "overflow_check" FREE_BENCHMARK = "free_benchmark" + RUN_UT = "run_ut" ATTR_NAME_PREFIX = "wrap_" + ATTR_NAME_PREFIX_LEN = len(ATTR_NAME_PREFIX) KERNEL_DUMP = "kernel_dump" DATA = "data" PT_FRAMEWORK = "pytorch" @@ -84,6 +94,7 @@ class Const: "int32_to_int64": ["cross_entropy"] } + class CompareConst: """ Class for compare module const @@ -196,6 +207,7 @@ class CompareConst: MAX_RELATIVE_OUT_YELLOW = 0.1 MAX_RELATIVE_IN_YELLOW = 0.01 + class FileCheckConst: """ Class for file check const @@ -232,6 +244,7 @@ class FileCheckConst: YAML_SUFFIX: MAX_YAML_SIZE } + class OverflowConst: """ Class for Overflow diff --git a/debug/accuracy_tools/msprobe/core/common/exceptions.py b/debug/accuracy_tools/msprobe/core/common/exceptions.py index df89699ce..ea61f8cd5 100644 --- a/debug/accuracy_tools/msprobe/core/common/exceptions.py +++ b/debug/accuracy_tools/msprobe/core/common/exceptions.py @@ -8,13 +8,13 @@ class CodedException(Exception): return self.error_info -class MsaccException(CodedException): +class MsprobeException(CodedException): INVALID_PARAM_ERROR = 0 OVERFLOW_NUMS_ERROR = 1 err_strs = { - INVALID_PARAM_ERROR: "[msacc] 无效参数： ", - OVERFLOW_NUMS_ERROR: "[msacc] 超过预设溢出次数 当前溢出次数:" + INVALID_PARAM_ERROR: "[msprobe] 无效参数： ", + OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数:" } @@ -27,12 +27,12 @@ class FileCheckException(CodedException): FILE_TOO_LARGE_ERROR = 5 err_strs = { - SOFT_LINK_ERROR: "[msacc] 检测到软链接： ", - FILE_PERMISSION_ERROR: "[msacc] 文件权限错误： ", - INVALID_FILE_ERROR: "[msacc] 无效文件： ", - ILLEGAL_PATH_ERROR: "[msacc] 非法文件路径： ", - ILLEGAL_PARAM_ERROR: "[msacc] 非法打开方式： ", - FILE_TOO_LARGE_ERROR: "[msacc] 文件过大： " + SOFT_LINK_ERROR: "[msprobe] 检测到软链接： ", + FILE_PERMISSION_ERROR: "[msprobe] 文件权限错误： ", + INVALID_FILE_ERROR: "[msprobe] 无效文件： ", + ILLEGAL_PATH_ERROR: "[msprobe] 非法文件路径： ", + ILLEGAL_PARAM_ERROR: "[msprobe] 非法打开方式： ", + FILE_TOO_LARGE_ERROR: "[msprobe] 文件过大： " } @@ -40,8 +40,8 @@ class ParseJsonException(CodedException): UnexpectedNameStruct = 0 InvalidDumpJson = 1 err_strs = { - UnexpectedNameStruct: "[msacc] Unexpected name in json: ", - InvalidDumpJson: "[msacc] json格式不正确: ", + UnexpectedNameStruct: "[msprobe] Unexpected name in json: ", + InvalidDumpJson: "[msprobe] json格式不正确: ", } @@ -50,23 +50,23 @@ class ScopeException(CodedException): InvalidScope = 1 ArgConflict = 2 err_strs = { - InvalidApiStr: "[msacc] Invalid api_list: ", - InvalidScope: "[msacc] Invalid scope: ", - ArgConflict: "[msacc] Scope and api_list conflict: ", + InvalidApiStr: "[msprobe] Invalid api_list: ", + InvalidScope: "[msprobe] Invalid scope: ", + ArgConflict: "[msprobe] Scope and api_list conflict: ", } class RepairException(CodedException): InvalidRepairType = 0 err_strs = { - InvalidRepairType: "[msacc] Invalid repair_type: " + InvalidRepairType: "[msprobe] Invalid repair_type: " } class StepException(CodedException): InvalidPostProcess = 0 err_strs = { - InvalidPostProcess: "[msacc] 错误的step后处理配置: ", + InvalidPostProcess: "[msprobe] 错误的step后处理配置: ", } @@ -74,8 +74,8 @@ class FreeBenchmarkException(CodedException): UnsupportedType = 0 InvalidGrad = 1 err_strs = { - UnsupportedType: "[msacc] Free benchmark get unsupported type: ", - InvalidGrad: "[msacc] Free benchmark gradient invalid: ", + UnsupportedType: "[msprobe] Free benchmark get unsupported type: ", + InvalidGrad: "[msprobe] Free benchmark gradient invalid: ", } diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index b7d446ce8..ed38eba00 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -1,6 +1,6 @@ from msprobe.core.common.const import Const from msprobe.core.common.log import logger -from msprobe.core.common.exceptions import MsaccException +from msprobe.core.common.exceptions import MsprobeException class CommonConfig: @@ -19,22 +19,22 @@ class CommonConfig: def _check_config(self): if self.task and self.task not in Const.TASK_LIST: logger.error_log_with_exp( - "task is invalid, it should be one of {}".format(Const.TASK_LIST), MsaccException(MsaccException.INVALID_PARAM_ERROR)) + "task is invalid, it should be one of {}".format(Const.TASK_LIST), MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.rank is not None and not isinstance(self.rank, list): - logger.error_log_with_exp("rank is invalid, it should be a list", MsaccException(MsaccException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("rank is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.step is not None and not isinstance(self.step, list): - logger.error_log_with_exp("step is invalid, it should be a list", MsaccException(MsaccException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("step is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.level and self.level not in Const.LEVEL_LIST: logger.error_log_with_exp( - "level is invalid, it should be one of {}".format(Const.LEVEL_LIST), MsaccException(MsaccException.INVALID_PARAM_ERROR)) + "level is invalid, it should be one of {}".format(Const.LEVEL_LIST), MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.seed is not None and not isinstance(self.seed, int): - logger.error_log_with_exp("seed is invalid, it should be an integer", MsaccException(MsaccException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("seed is invalid, it should be an integer", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if not isinstance(self.is_deterministic, bool): logger.error_log_with_exp( - "is_deterministic is invalid, it should be a boolean", MsaccException(MsaccException.INVALID_PARAM_ERROR)) + "is_deterministic is invalid, it should be a boolean", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if not isinstance(self.enable_dataloader, bool): logger.error_log_with_exp( - "enable_dataloader is invalid, it should be a boolean", MsaccException(MsaccException.INVALID_PARAM_ERROR)) + "enable_dataloader is invalid, it should be a boolean", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) class BaseConfig: @@ -50,9 +50,9 @@ class BaseConfig: def check_config(self): if self.scope is not None and not isinstance(self.scope, list): - logger.error_log_with_exp("scope is invalid, it should be a list", MsaccException(MsaccException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("scope is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.list is not None and not isinstance(self.list, list): - logger.error_log_with_exp("list is invalid, it should be a list", MsaccException(MsaccException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("list is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.data_mode is not None and not isinstance(self.data_mode, list): - logger.error_log_with_exp("data_mode is invalid, it should be a list", MsaccException(MsaccException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("data_mode is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 430d13634..5d9012919 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -238,8 +238,8 @@ class BaseDataProcessor: return api_info_struct def get_save_file_path(self, suffix): - file_format = "pt" if self.config.framework == Const.PT_FRAMEWORK else "npy" + file_format = Const.PT_SUFFIX if self.config.framework == Const.PT_FRAMEWORK else Const.NUMPY_SUFFIX dump_data_name = (self.current_api_or_module_name + Const.SEP + self.api_data_category + Const.SEP + - suffix + Const.SEP + file_format) + suffix + file_format) file_path = os.path.join(self.data_writer.dump_tensor_data_dir, dump_data_name) return dump_data_name, file_path \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/factory.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/factory.py index 2c536ba57..86ef2115f 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/factory.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/factory.py @@ -51,11 +51,7 @@ class DataProcessorFactory: elif framework == Const.MS_FRAMEWORK: from .mindspore_processor import ( StatisticsDataProcessor as MindsporeStatisticsDataProcessor, - TensorDataProcessor as MindsporeTensorDataProcessor, - OverflowCheckDataProcessor as MindsporeOverflowCheckDataProcessor, - FreeBenchmarkDataProcessor as MindsporeFreeBenchmarkDataProcessor + TensorDataProcessor as MindsporeTensorDataProcessor ) cls.register_processor(Const.MS_FRAMEWORK, Const.STATISTICS, MindsporeStatisticsDataProcessor) cls.register_processor(Const.MS_FRAMEWORK, Const.TENSOR, MindsporeTensorDataProcessor) - cls.register_processor(Const.MS_FRAMEWORK, Const.OVERFLOW_CHECK, MindsporeOverflowCheckDataProcessor) - cls.register_processor(Const.MS_FRAMEWORK, Const.FREE_BENCHMARK, MindsporeFreeBenchmarkDataProcessor) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py new file mode 100644 index 000000000..7533e2ee0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -0,0 +1,131 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import zlib +import mindspore as ms +from mindspore import ops +import numpy as np + +from msprobe.core.common.const import Const +from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, TensorStatInfo +from msprobe.core.common.file_check import path_len_exceeds_limit, change_mode, FileCheckConst +from msprobe.mindspore.dump.hook_cell.wrap_functional import load_ops_functions +from msprobe.mindspore.common.utils import convert_bf16_to_fp32 +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.dump.hook_cell.api_registry import api_register + + +class MindsporeDataProcessor(BaseDataProcessor): + mindspore_special_type = tuple([ms.Tensor]) + ops_func, mint_ops_func, _ = load_ops_functions() + + def __init__(self, config, data_writer): + super().__init__(config, data_writer) + self.mindspore_object_key = { + "dtype": self.analyze_dtype_in_kwargs + } + + @staticmethod + def get_md5_for_tensor(x): + x = convert_bf16_to_fp32(x) + tensor_bytes = x.asnumpy().tobytes() + crc32_hash = zlib.crc32(tensor_bytes) + return f"{crc32_hash:08x}" + + @staticmethod + def analyze_dtype_in_kwargs(element): + return {"type": "mindspore.dtype", "value": str(element)} + + @classmethod + def get_special_types(cls): + return super().get_special_types() + cls.mindspore_special_type + + def get_stat_info(self, data): + tensor_stat = TensorStatInfo() + if data.numel() == 0: + return tensor_stat + elif data.dtype == ms.bool_: + tensor_stat.max = self.mint_ops_func["max"](data).item() + tensor_stat.min = self.mint_ops_func["min"](data).item() + elif not data.shape: + tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data.item() + elif data.dtype == ms.complex64 or data.dtype == ms.complex128: + data_abs = np.abs(data.asnumpy()) + tensor_stat.max = np.max(data_abs) + tensor_stat.min = np.min(data_abs) + tensor_stat.mean = np.mean(data_abs) + tensor_stat.norm = np.linalg.norm(data_abs) + else: + if data.dtype == ms.bfloat16 or not ops.is_floating_point(data): + data = data.to(ms.float32) + api_register.norm_inner_op_set_ori_func() + tensor_stat.max = self.mint_ops_func["max"](data).item() + tensor_stat.min = self.mint_ops_func["min"](data).item() + tensor_stat.mean = self.mint_ops_func["mean"](data).item() + tensor_stat.norm = self.ops_func["norm"](data).item() + api_register.norm_inner_op_set_hook_func() + return tensor_stat + + def analyze_single_element(self, element, suffix_stack): + if suffix_stack and suffix_stack[-1] in self.mindspore_object_key: + return self.mindspore_object_key[suffix_stack[-1]](element) + + converted_numpy, numpy_type = self._convert_numpy_to_builtin(element) + if converted_numpy is not element: + return self._analyze_numpy(converted_numpy, numpy_type) + if isinstance(element, ms.Tensor): + return self._analyze_tensor(element, Const.SEP.join(suffix_stack)) + + if isinstance(element, (bool, int, float, str, slice)): + return self._analyze_builtin(element) + return None + + def analyze_element(self, element): + return self.recursive_apply_transform(element, self.analyze_single_element) + + def _analyze_tensor(self, tensor, suffix): + tensor_stat = self.get_stat_info(tensor) + tensor_json = { + 'type': 'mindspore.Tensor', + 'dtype': str(tensor.dtype), + 'shape': tensor.shape, + 'Max': tensor_stat.max, + 'Min': tensor_stat.min, + 'Mean': tensor_stat.mean, + 'Norm': tensor_stat.norm + } + if self.config.summary_mode == Const.MD5: + tensor_md5 = self.get_md5_for_tensor(tensor) + tensor_json.update({Const.MD5: tensor_md5}) + return tensor_json + + +class StatisticsDataProcessor(MindsporeDataProcessor): + pass + + +class TensorDataProcessor(MindsporeDataProcessor): + def _analyze_tensor(self, tensor, suffix): + dump_data_name, file_path = self.get_save_file_path(suffix) + single_arg = super()._analyze_tensor(tensor, suffix) + single_arg.update({"data_name": dump_data_name}) + if not path_len_exceeds_limit(file_path): + tensor = convert_bf16_to_fp32(tensor) + saved_tensor = tensor.asnumpy() + np.save(file_path, saved_tensor) + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + else: + logger.warning(f'The file path {file_path} length exceeds limit.') + return single_arg diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 95be091b2..f307909a4 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -5,7 +5,7 @@ from typing import List import numpy as np import torch -from msprobe.core.common.exceptions import MsaccException +from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common.file_check import path_len_exceeds_limit, change_mode from msprobe.core.common.log import logger from msprobe.core.common.const import Const, OverflowConst, FileCheckConst @@ -191,7 +191,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): if self.overflow_nums == -1: return if self.real_overflow_dump_times >= self.overflow_nums: - raise MsaccException(MsaccException.OVERFLOW_NUMS_ERROR, str(self.real_overflow_dump_times)) + raise MsprobeException(MsprobeException.OVERFLOW_NUMS_ERROR, str(self.real_overflow_dump_times)) def check_overflow_npu(self): if self.overflow_debug_mode_enalbe(): diff --git a/debug/accuracy_tools/msprobe/mindspore/common/log.py b/debug/accuracy_tools/msprobe/mindspore/common/log.py new file mode 100644 index 000000000..ec027c750 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/common/log.py @@ -0,0 +1,38 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import time +import sys + +from msprobe.mindspore.common.utils import get_rank_if_initialized +from msprobe.core.common.log import BaseLogger +from msprobe.core.common.exceptions import DistributedNotInitializedError + + +class MindsporeLogger(BaseLogger): + def __init__(self): + super().__init__() + + def get_rank(self): + try: + current_rank = get_rank_if_initialized() + except DistributedNotInitializedError: + current_rank = None + + return current_rank + + +logger = MindsporeLogger() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py new file mode 100644 index 000000000..d02f38195 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -0,0 +1,31 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import mindspore as ms +from msprobe.core.common.exceptions import DistributedNotInitializedError + + +def get_rank_if_initialized(): + if ms.communication.GlobalComm.INITED: + return ms.communication.get_rank() + else: + raise DistributedNotInitializedError("mindspore distributed environment is not initialized") + + +def convert_bf16_to_fp32(tensor): + if tensor.dtype == ms.bfloat16: + tensor = tensor.to(ms.float32) + return tensor + diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 56a4b9bf7..04d66d6a2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -1,4 +1,5 @@ import os +from msprobe.core.common.utils import Const class DebuggerConfig: @@ -16,17 +17,20 @@ class DebuggerConfig: if not common_config.level: common_config.level = "L1" self.level = DebuggerConfig.convert_map[common_config.level] + self.level_ori = common_config.level self.list = [] if not task_config.list else task_config.list + self.scope =[] if not task_config.scope else task_config.scope self.data_mode = [] if not task_config.data_mode else task_config.data_mode self.file_format = task_config.file_format self.check_mode = task_config.check_mode - + self.framework = Const.MS_FRAMEWORK + self.summary_mode = task_config.summary_mode self.check() def check(self): if not self.dump_path: raise Exception("Dump path is empty.") - if not os.path.isabs(self.dump_path): + if self.level_ori != "L1" and not os.path.isabs(self.dump_path): raise Exception("Dump path must be absolute path.") if not self.task: self.task = "statistics" diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 358d0d6f7..30f7162ff 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -1,4 +1,6 @@ import os +import mindspore as ms +from msprobe.mindspore.service import Service from msprobe.mindspore.ms_config import parse_json_config from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.task_handler_factory import TaskHandlerFactory @@ -22,11 +24,28 @@ class PrecisionDebugger: common_config, task_config = parse_json_config(config_path) self.config = DebuggerConfig(common_config, task_config) self.initialized = True + self.service = Service(self.config) @classmethod - def start(cls, target=None): + def start(cls): instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") - handler = TaskHandlerFactory.create(instance.config) - handler.handle() + if ms.get_context("mode") == ms.PYNATIVE_MODE and instance.config.level_ori == "L1": + instance.service.start() + else: + handler = TaskHandlerFactory.create(instance.config) + handler.handle() + + @classmethod + def stop(cls): + instance = cls._instance + if not instance: + raise Exception("PrecisionDebugger instance is not created.") + instance.service.stop() + + @classmethod + def step(cls): + if not cls._instance: + raise Exception("PrecisionDebugger instance is not created.") + cls._instance.service.step() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py new file mode 100644 index 000000000..5508416fd --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py @@ -0,0 +1,104 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import mindspore as ms +from msprobe.mindspore.dump.hook_cell.wrap_functional import get_functional_ops, setup_hooks, \ + HOOKFunctionalOP, HOOKMintOP, HOOKMintNNFunctionalOP +from msprobe.mindspore.dump.hook_cell.wrap_tensor import get_tensor_ops, wrap_tensor_ops_and_bind, HOOKTensor +from msprobe.core.common.utils import Const + + +class ApiRegistry: + def __init__(self): + self.tensor_ori_attr = {} + self.functional_ori_attr = {} + self.mint_ops_ori_attr = {} + self.mint_func_ops_ori_attr = {} + self.norm_inner_ops_ori_attr = {} + + self.tensor_hook_attr = {} + self.functional_hook_attr = {} + self.mint_ops_hook_attr = {} + self.mint_func_ops_hook_attr = {} + self.norm_inner_ops_hook_attr = {} + + self.norm_inner_ops = ["norm", "square", "sqrt", "is_complex"] + + @staticmethod + def store_ori_attr(ori_api_group, api_list, api_ori_attr): + for api in api_list: + if Const.SEP in api: + sub_module_name, sub_op = api.rsplit(Const.SEP, 1) + sub_module = getattr(ori_api_group, sub_module_name) + api_ori_attr[api] = getattr(sub_module, sub_op) + else: + api_ori_attr[api] = getattr(ori_api_group, api) + + @staticmethod + def set_api_attr(api_group, attr_dict): + for api, api_attr in attr_dict.items(): + if Const.SEP in api: + sub_module_name, sub_op = api.rsplit(Const.SEP, 1) + sub_module = getattr(api_group, sub_module_name, None) + if sub_module is not None: + setattr(sub_module, sub_op, api_attr) + else: + setattr(api_group, api, api_attr) + + def norm_inner_op_set_hook_func(self): + self.set_api_attr(ms.ops, self.norm_inner_ops_hook_attr) + + def norm_inner_op_set_ori_func(self): + self.set_api_attr(ms.ops, self.norm_inner_ops_ori_attr) + + def api_set_hook_func(self): + self.set_api_attr(ms.Tensor, self.tensor_hook_attr) + self.set_api_attr(ms.ops, self.functional_hook_attr) + self.set_api_attr(ms.mint, self.mint_ops_hook_attr) + self.set_api_attr(ms.mint.nn.functional, self.mint_func_ops_hook_attr) + + def api_set_ori_func(self): + self.set_api_attr(ms.Tensor, self.tensor_ori_attr) + self.set_api_attr(ms.ops, self.functional_ori_attr) + self.set_api_attr(ms.mint, self.mint_ops_ori_attr) + self.set_api_attr(ms.mint.nn.functional, self.mint_func_ops_ori_attr) + + def initialize_hook(self, hook): + self.store_ori_attr(ms.Tensor, get_tensor_ops(), self.tensor_ori_attr) + wrap_tensor_ops_and_bind(hook) + for attr_name in dir(HOOKTensor): + if attr_name.startswith(Const.ATTR_NAME_PREFIX): + self.tensor_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKTensor, attr_name) + + functional_ops, mint_ops, mint_func_ops = get_functional_ops() + self.store_ori_attr(ms.ops, self.norm_inner_ops, self.norm_inner_ops_ori_attr) + self.store_ori_attr(ms.ops, functional_ops, self.functional_ori_attr) + self.store_ori_attr(ms.mint, mint_ops, self.mint_ops_ori_attr) + self.store_ori_attr(ms.mint.nn.functional, mint_func_ops, self.mint_func_ops_ori_attr) + setup_hooks(hook) + for attr_name in dir(HOOKFunctionalOP): + if attr_name.startswith(Const.ATTR_NAME_PREFIX): + self.functional_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKFunctionalOP, attr_name) + if attr_name[Const.ATTR_NAME_PREFIX_LEN:] in self.norm_inner_ops: + self.norm_inner_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKFunctionalOP, attr_name) + for attr_name in dir(HOOKMintOP): + if attr_name.startswith(Const.ATTR_NAME_PREFIX): + self.mint_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKMintOP, attr_name) + for attr_name in dir(HOOKMintNNFunctionalOP): + if attr_name.startswith(Const.ATTR_NAME_PREFIX): + self.mint_func_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKMintNNFunctionalOP, attr_name) + + +api_register = ApiRegistry() diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py new file mode 100644 index 000000000..bcb80dd22 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py @@ -0,0 +1,57 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +from collections import defaultdict + +from mindspore import nn +from msprobe.core.common.const import Const + + +cell_count = defaultdict(int) +g_stop_hook = False + + +class HOOKCell(nn.Cell): + + def __init__(self, build_hook) -> None: + super(HOOKCell, self).__init__() + self.changed_status = False + self.input_kwargs = {} + self.prefix = "" + global g_stop_hook + if not g_stop_hook: + g_stop_hook = True + self.changed_status = True + if hasattr(self, "prefix_op_name_"): + self.prefix = self.prefix_op_name_ + + cell_count[self.prefix] += 1 + self.prefix = self.prefix + str(cell_count[self.prefix] - 1) + Const.SEP + forward_hook, backward_hook = build_hook(self.prefix) + self.register_forward_hook(forward_hook) + self.register_backward_hook(backward_hook) + + # 重载call，加全局标志。 + def __call__(self, *args, **kwargs): + try: + self.input_kwargs = kwargs + out = super(HOOKCell, self).__call__(*args, **kwargs) + except Exception as e: + raise e + finally: + if self.changed_status: + self.changed_status = False + global g_stop_hook + g_stop_hook = False + return out diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml new file mode 100644 index 000000000..089f444b6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml @@ -0,0 +1,925 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +# List of ops that register hooks + + +ops: + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d + - adaptive_max_pool1d + - adaptive_max_pool2d + - avg_pool1d + - avg_pool2d + - avg_pool3d + - batch_norm + - bias_add + - ctc_greedy_decoder + - conv1d + - conv2d + - conv3d + - deformable_conv2d + - dense + - dropout + - dropout1d + - dropout2d + - dropout3d + - flatten + - fold + - fractional_max_pool3d + - lp_pool1d + - lp_pool2d + - lrn + - max_pool2d + - max_pool3d + - max_unpool1d + - max_unpool2d + - max_unpool3d + - unfold + - binary_cross_entropy + - binary_cross_entropy_with_logits + - cosine_embedding_loss + - cross_entropy + - ctc_loss + - gaussian_nll_loss + - hinge_embedding_loss + - huber_loss + - kl_div + - l1_loss + - margin_ranking_loss + - mse_loss + - multi_margin_loss + - multilabel_margin_loss + - multilabel_soft_margin_loss + - nll_loss + - smooth_l1_loss + - triplet_margin_loss + - elu + - fast_gelu + - gelu + - glu + - gumbel_softmax + - hardshrink + - hardsigmoid + - hardswish + - hardtanh + - leaky_relu + - log_softmax + - logsigmoid + - mish + - prelu + - relu + - relu6 + - celu + - rrelu + - selu + - sigmoid + - silu + - softmax + - softmin + - softshrink + - softsign + - tanh + - threshold + - cdist + - dist + - pdist + - choice_with_mask + - random_categorical + - log_uniform_candidate_sampler + - uniform_candidate_sampler + - affine_grid + - bounding_box_decode + - bounding_box_encode + - col2im + - check_valid + - crop_and_resize + - grid_sample + - interpolate + - iou + - pad + - padding + - pixel_shuffle + - pixel_unshuffle + - upsample + - abs + - absolute + - accumulate_n + - acos + - arccos + - acosh + - add + - addcdiv + - addcmul + - addmv + - addn + - angle + - arccosh + - arcsin + - arcsinh + - arctan + - arctanh + - arctan2 + - asin + - asinh + - atan + - atan2 + - atanh + - atleast_1d + - atleast_2d + - atleast_3d + - bessel_i0 + - bessel_i0e + - bessel_i1 + - bessel_i1e + - bessel_j0 + - bessel_j1 + - bessel_k0 + - bessel_k0e + - bessel_k1 + - bessel_k1e + - bessel_y0 + - bessel_y1 + - bitwise_and + - bitwise_left_shift + - bitwise_or + - bitwise_right_shift + - bitwise_xor + - ceil + - clamp + - clip + - combinations + - copysign + - cos + - cosh + - cosine_similarity + - cov + - diag_embed + - diff + - deg2rad + - digamma + - div + - divide + - erf + - erfc + - erfinv + - exp + - exp2 + - expm1 + - floor + - floor_div + - floor_mod + - float_power + - fmod + - frac + - gcd + - hypot + - igamma + - igammac + - imag + - i0 + - inv + - invert + - lcm + - ldexp + - lerp + - log + - log2 + - log10 + - log1p + - logaddexp + - logaddexp2 + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - mul + - multiply + - mvlgamma + - neg + - negative + - nextafter + - polar + - polygamma + - positive + - pow + - rad2deg + - ravel + - real + - reciprocal + - remainder + - rot90 + - round + - rsqrt + - sgn + - sign + - signbit + - sin + - sinc + - sinh + - sqrt + - square + - sub + - subtract + - t + - tan + - tanhshrink + - trapz + - tril_indices + - triu_indices + - true_divide + - trunc + - truncate_div + - truncate_mod + - xdivy + - xlogy + - zeta + - all + - amax + - amin + - aminmax + - any + - argmax + - argmin + - cummax + - cummin + - cumprod + - cumsum + - fmax + - histc + - logsumexp + - max + - mean + - median + - min + - norm + - prod + - std + - std_mean + - var + - var_mean + - argsort + - approximate_equal + - equal + - ge + - greater + - greater_equal + - gt + - intopk + - isclose + - isfinite + - isinf + - isnan + - isneginf + - isposinf + - isreal + - is_complex + - le + - less + - less_equal + - lt + - maximum + - minimum + - msort + - ne + - not_equal + - searchsorted + - topk + - bmm + - addbmm + - addmm + - baddbmm + - addr + - adjoint + - cholesky + - cholesky_solve + - batch_dot + - dot + - eig + - inner + - inverse + - geqrf + - ger + - kron + - lu_solve + - lu_unpack + - matmul + - matrix_solve + - matrix_band_part + - matrix_diag + - matrix_diag_part + - matrix_set_diag + - mm + - mv + - outer + - orgqr + - ormqr + - pinv + - svd + - tensor_dot + - logdet + - slogdet + - qr + - trace + - bartlett_window + - blackman_window + - hamming_window + - hann_window + - kaiser_window + - eye + - fill + - full + - full_like + - linspace + - logspace + - one_hot + - arange + - range + - heaviside + - bernoulli + - gamma + - laplace + - multinomial + - multinomial_with_replacement + - rand + - rand_like + - randint + - randint_like + - randn + - randn_like + - random_gamma + - random_poisson + - randperm + - standard_laplace + - standard_normal + - uniform + - argwhere + - batch_to_space_nd + - bincount + - block_diag + - broadcast_to + - cat + - channel_shuffle + - chunk + - column_stack + - concat + - conj + - count_nonzero + - deepcopy + - diag + - diagflat + - diagonal + - dyn_shape + - dsplit + - dstack + - einsum + - expand + - expand_dims + - flip + - fliplr + - flipud + - gather_d + - gather_elements + - gather_nd + - hsplit + - hstack + - index_add + - index_fill + - index_select + - inplace_add + - inplace_index_add + - inplace_sub + - inplace_update + - masked_fill + - masked_select + - meshgrid + - moveaxis + - movedim + - narrow + - nan_to_num + - nansum + - normal + - nonzero + - population_count + - rank + - repeat_elements + - repeat_interleave + - reshape + - reverse + - reverse_sequence + - roll + - scatter + - scatter_nd + - select + - sequence_mask + - shuffle + - size + - slice + - sort + - space_to_batch_nd + - sparse_segment_mean + - split + - squeeze + - stack + - strided_slice + - sum + - swapaxes + - swapdims + - tensor_scatter_add + - tensor_scatter_div + - tensor_scatter_max + - tensor_scatter_min + - tensor_scatter_mul + - tensor_scatter_sub + - tensor_scatter_elements + - tensor_split + - tile + - tril + - triu + - transpose + - unbind + - unique + - unique_consecutive + - unique_with_pad + - unsorted_segment_max + - unsorted_segment_min + - unsorted_segment_prod + - unsorted_segment_sum + - unsqueeze + - unstack + - view_as_real + - vsplit + - vstack + - where + - cross + - renorm + - is_tensor + - scalar_cast + - scalar_to_tensor + - tuple_to_array + - clip_by_global_norm + - clip_by_value + - assign + - assign_add + - assign_sub + - scatter_add + - scatter_div + - scatter_max + - scatter_min + - scatter_mul + - scatter_nd_add + - scatter_nd_div + - scatter_nd_max + - scatter_nd_min + - scatter_nd_mul + - scatter_nd_sub + - scatter_update + - derivative + - jet + +tensor: + - __abs__ + - __add__ + - __and__ + - __bool__ + - __eq__ + - __ge__ + - __gt__ + - __iadd__ + - __ifloordiv__ + - __imatmul__ + - __imod__ + - __imul__ + - __isub__ + - __le__ + - __lt__ + - __matmul__ + - __mod__ + - __mul__ + - __ne__ + - __neg__ + - __or__ + - __pow__ + - __radd__ + - __rmatmul__ + - __rmod__ + - __rmul__ + - __rpow__ + - __rsub__ + - __sub__ + - __truediv__ + - __xor__ + - abs + - absolute + - acos + - acosh + - add + - addbmm + - addcdiv + - addcmul + - addmm + - addmv + - addr + - all + - amax + - amin + - any + - arccos + - arccosh + - argmax + - angle + - arcsin + - arcsinh + - arctan + - arctanh + - argmin + - argsort + - asin + - asinh + - atan + - atan2 + - atanh + - baddbmm + - bernoulli + - bincount + - bitwise_and + - bitwise_or + - bitwise_xor + - bmm + - bool + - broadcast_to + - ceil + - cholesky_solve + - cholesky + - clamp + - clip + - conj + - copysign + - cos + - cosh + - cross + - cummax + - cummin + - cumprod + - cumsum + - deg2rad + - diag + - diagflat + - diff + - digamma + - div + - divide + - equal + - erf + - erfc + - erfinv + - exp + - expand_as + - expm1 + - flip + - fliplr + - flipud + - float_power + - floor + - fmod + - frac + - gather_elements + - ge + - geqrf + - ger + - greater + - greater_equal + - gt + - half + - hardshrink + - heaviside + - histc + - hypot + - i0 + - igamma + - igammac + - imag + - index_add + - index_fill + - index_put + - index_select + - inner + - int + - inverse + - isclose + - isfinite + - isinf + - isnan + - is_complex + - is_signed + - isneginf + - isposinf + - isreal + - lcm + - ldexp + - le + - lerp + - less + - less_equal + - log + - log10 + - log1p + - log2 + - logaddexp + - logaddexp2 + - logdet + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - logsumexp + - long + - lt + - masked_fill + - masked_scatter + - masked_select + - matmul + - max + - maximum + - mean + - median + - min + - minimum + - moveaxis + - movedim + - msort + - multinomial + - multiply + - mvlgamma + - nan_to_num + - nansum + - narrow + - ne + - neg + - negative + - nelement + - new_ones + - new_zeros + - nextafter + - norm + - nonzero + - not_equal + - ormqr + - permute + - pow + - prod + - qr + - ravel + - real + - reciprocal + - remainder + - renorm + - rad2deg + - tile + - repeat_interleave + - reshape + - reshape + - round + - rot90 + - rsqrt + - sum_to_size + - scatter + - sgn + - short + - sigmoid + - sign + - signbit + - sin + - sinc + - sinh + - slogdet + - sort + - split + - sqrt + - square + - squeeze + - std + - subtract + - subtract + - svd + - swapaxes + - swapdims + - t + - take + - tan + - tanh + - trace + - swapaxes + - tile + - to + - topk + - tril + - tensor_split + - transpose + - true_divide + - trunc + - unbind + - unique_consecutive + - unsqueeze + - var + - view + - where + - xlogy + - from_numpy + - std + - take + - var + - all + - any + - copy + - diagonal + - flatten + - resize + - sum + +mint.ops: + - abs + - absolute_import + - add + - add_ex + - all + - any + - any_ex + - arange + - argmax + - avg_pool2d + - baddbmm + - baddbmm_ex + - batch_norm + - binary_cross_entropy_with_logits + - bitwise_and + - bitwise_or + - bitwise_xor + - bmm + - broadcast_to + - cat + - cat_ex + - ceil + - chunk + - clamp + - conv2d + - conv_transpose2d + - cos + - cross + - cummax + - cummin + - cumsum + - div + - divide + - dropout + - embedding + - eq + - erf + - erfinv + - exp + - flatten + - flip + - flip_ex + - fold + - full + - functional + - gather + - gelu + - greater + - grid_sample + - group_norm + - gt + - index_select + - interpolate + - isclose + - isfinite + - layer_norm + - le + - leaky_relu + - less + - less_equal + - linear + - linspace + - log + - logical_and + - logical_not + - logical_or + - lt + - masked_select + - matmul + - max + - max_pool2d + - maximum + - mean + - mean_ex + - min + - minimum + - mul + - ne + - neg + - negative + - nn + - nonzero + - normal + - one_hot + - ones + - ones_ex + - ones_like + - pad + - permute + - permute_ex + - pow + - prod + - reciprocal + - relu + - remainder + - repeat_interleave + - rsqrt + - scatter + - scatter_add + - searchsorted + - sigmoid + - silu + - sin + - softmax + - softplus + - sort + - split + - sqrt + - sqrt_ex + - square + - stack + - sub + - sub_ex + - sum + - tanh + - tile + - topk + - tril + - triu + - unfold + - unique + - where + - xlogy + - zeros + - zeros_ex + - zeros_like + +mint.nn: + - Dropout + - Embedding + - Fold + - LayerNorm + - Linear + - MaxPool2d + - Unfold + - Upsample + +mint.nn.functional: + - absolute_import + - avg_pool2d + - batch_norm + - batch_norm_ex + - bce_with_logits + - binary_cross_entropy_with_logits + - conv_transpose2d + - dense + - dropout + - embedding + - fold + - gelu + - grid_sample + - group_norm + - interpolate + - layer_norm + - leaky_relu + - linear + - max_pool2d + - max_pool2d_ex + - normal + - one_hot + - one_hot_ext + - pad + - relu + - sigmoid + - silu + - softmax + - softmax_ex + - softplus + - tanh + - unfold diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py new file mode 100644 index 000000000..be3d1bd25 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py @@ -0,0 +1,94 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import yaml +import mindspore as ms +from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell +from msprobe.core.common.utils import Const +from msprobe.core.common.file_check import FileOpen + + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") + + +def load_ops_functions(): + ops_func = {f: getattr(ms.ops, f) for f in dir(ms.ops)} + mint_ops_func = {f: getattr(ms.mint, f) for f in dir(ms.mint)} + mint_func_ops_func = {f: getattr(ms.mint.nn.functional, f) for f in dir(ms.mint.nn.functional)} + return ops_func, mint_ops_func, mint_func_ops_func + + +def get_functional_ops(): + ops_func, mint_ops_func, mint_func_ops_func = load_ops_functions() + with FileOpen(yaml_path, 'r') as f: + config = yaml.safe_load(f) + WrapFunctionalOps = config.get("ops") + WrapMintOps = config.get("mint.ops") + WrapMintFunctionalOps = config.get("mint.nn.functional") + return ( + set(WrapFunctionalOps) & set(ops_func.keys()), + set(WrapMintOps) & set(mint_ops_func.keys()), + set(WrapMintFunctionalOps) & set(mint_func_ops_func.keys()) + ) + + +class HOOKFunctionalOP(object): + pass + + +class HOOKMintOP(object): + pass + + +class HOOKMintNNFunctionalOP(object): + pass + + +class FunctionalOPTemplate(HOOKCell): + def __init__(self, op_name, op_dict, prefix, hook): + self.op_name = op_name + self.op_func = op_dict[op_name] + self.prefix_op_name_ = prefix + str(op_name.split(Const.SEP)[-1]) + Const.SEP + super().__init__(hook) + + def construct(self, *args, **kwargs): + if self.op_name.startswith('dropout'): + return args[0] if args else kwargs.get('input') + return self.op_func(*args, **kwargs) + + +def wrap_functional_op(op_name, op_dict, prefix, hook): + def op_template(*args, **kwargs): + return FunctionalOPTemplate(op_name, op_dict, prefix, hook)(*args, **kwargs) + return op_template + + +def wrap_functional_ops_and_bind(ops, op_dict, prefix, hook, hook_class): + for op_name in ops: + if callable(op_dict[op_name]): + setattr(hook_class, Const.ATTR_NAME_PREFIX + op_name, wrap_functional_op(op_name, op_dict, prefix, hook)) + + +def setup_hooks(hook): + functional_ops, mint_ops, mint_func_ops = get_functional_ops() + wrap_functional_ops_and_bind( + functional_ops, {f: getattr(ms.ops, f) for f in dir(ms.ops)}, "Functional.", hook, HOOKFunctionalOP) + wrap_functional_ops_and_bind( + mint_ops, {f: getattr(ms.mint, f) for f in dir(ms.mint)}, "Mint.", hook, HOOKMintOP) + wrap_functional_ops_and_bind( + mint_func_ops, {f: getattr(ms.mint.nn.functional, f) for f in dir(ms.mint.nn.functional)}, "MintFunctional.", hook, HOOKMintNNFunctionalOP) + diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py new file mode 100644 index 000000000..ae6a9a979 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py @@ -0,0 +1,66 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import yaml +import mindspore as ms + +from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell +from msprobe.core.common.utils import Const +from msprobe.core.common.file_check import FileOpen + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") +with FileOpen(yaml_path, 'r') as f: + WrapTensorOps = yaml.safe_load(f).get('tensor') + +TensorFunc = {} +for f in dir(ms.Tensor): + TensorFunc[f] = getattr(ms.Tensor, f) + + +def get_tensor_ops(): + global WrapTensorOps + _tensor_ops = dir(ms.Tensor) + return set(WrapTensorOps) & set(_tensor_ops) + + +class HOOKTensor(object): + pass + + +class TensorOPTemplate(HOOKCell): + + def __init__(self, op_name, hook): + self.op_name_ = op_name + self.prefix_op_name_ = "Tensor." + str(op_name) + Const.SEP + super().__init__(hook) + + def construct(self, *args, **kwargs): + return TensorFunc[str(self.op_name_)](*args, **kwargs) + + +def wrap_tensor_op(op_name, hook): + def tensor_op_template(*args, **kwargs): + return TensorOPTemplate(op_name, hook)(*args, **kwargs) + + return tensor_op_template + + +def wrap_tensor_ops_and_bind(hook): + _tensor_ops = get_tensor_ops() + for op_name in _tensor_ops: + if callable(TensorFunc[op_name]): + setattr(HOOKTensor, Const.ATTR_NAME_PREFIX + str(op_name), wrap_tensor_op(op_name, hook)) diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index 2b390ae9e..49ce4cf2c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -1,6 +1,7 @@ import json from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.const import Const class TensorConfig(BaseConfig): @@ -31,6 +32,8 @@ class StatisticsConfig(BaseConfig): if self.data_mode is not None and len(self.data_mode) > 0: if len(self.data_mode) > 1 or self.data_mode[0] not in ["all", "input", "output"]: raise Exception("data_mode must be all, input or output") + if self.summary_mode and self.summary_mode not in ["statistics", "md5"]: + raise Exception("summary_mode is invalid") class OverflowCheck(BaseConfig): @@ -56,11 +59,11 @@ def parse_task_config(task, json_config): task_map = json_config[task] if not task_map: task_map = dict() - if task == "tensor": + if task == Const.TENSOR: return TensorConfig(task_map) - elif task == "statistics": + elif task == Const.STATISTICS: return StatisticsConfig(task_map) - elif task == "overflow_check": + elif task == Const.OVERFLOW_CHECK: return OverflowCheck(task_map) else: raise Exception("task is invalid.") @@ -73,6 +76,6 @@ def parse_json_config(json_file_path): json_config = json.load(file) common_config = parse_common_config(json_config) if not common_config.task: - common_config.task = "statistics" + common_config.task = Const.STATISTICS task_config = parse_task_config(common_config.task, json_config) return common_config, task_config diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py new file mode 100644 index 000000000..e8aa34dc4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -0,0 +1,138 @@ +# Copyright 2024 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +from pathlib import Path +import functools + +from msprobe.core.data_dump.data_collector import build_data_collector +from msprobe.core.data_dump.scope import BaseScope +from msprobe.mindspore.common.utils import get_rank_if_initialized +from msprobe.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create +from msprobe.mindspore.common.log import logger +from msprobe.core.common.utils import Const +from msprobe.core.common.exceptions import DistributedNotInitializedError +from msprobe.mindspore.dump.hook_cell.api_registry import api_register +from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs + + +class Service: + def __init__(self, config): + self.model = None + self.config = config + self.config.level = self.config.level_ori + self.data_collector = build_data_collector(config) + self.switch = False + self.current_iter = 0 + self.first_start = True + self.current_rank = None + self.dump_iter_dir = None + + def build_hook(self, module_type, name): + def forward_hook(api_or_module_name, module, input, output): + self.data_collector.visit_and_clear_overflow_status(api_or_module_name) + if not self.switch: + return None + if self.data_collector: + module_input_output = ModuleForwardInputsOutputs(args=input, kwargs=module.input_kwargs, output=output) + self.data_collector.forward_data_collect(api_or_module_name, module, pid, module_input_output) + if self.data_collector.if_return_forward_new_output(): + return self.data_collector.get_forward_new_output() + return output + + def backward_hook(api_or_module_name, module, grad_input, grad_output): + self.data_collector.visit_and_clear_overflow_status(api_or_module_name) + if not self.switch: + return + if self.data_collector: + module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_input, grad_output=grad_output) + self.data_collector.backward_data_collect(api_or_module_name, module, pid, module_input_output) + + pid = os.getpid() + forward_name_template = name + Const.FORWARD + backward_name_template = name + Const.BACKWARD + forward_hook = functools.partial(forward_hook, forward_name_template) + backward_hook = functools.partial(backward_hook, backward_name_template) + + def wrap_forward_hook(*args, **kwargs): + return forward_hook(*args, **kwargs) + + def wrap_backward_hook(*args, **kwargs): + return backward_hook(*args, **kwargs) + + return wrap_forward_hook, wrap_backward_hook + + def step(self): + self.current_iter += 1 + self.data_collector.update_iter(self.current_iter) + + def start(self, model=None): + self.model = model + if self.config.step and self.current_iter > max(self.config.step): + self.stop() + raise Exception("msprobe: exit after iteration {}".format(max(self.config.step))) + if self.config.step and self.current_iter not in self.config.step: + return + if self.first_start: + try: + self.current_rank = get_rank_if_initialized() + except DistributedNotInitializedError: + self.current_rank = None + + if self.config.rank and self.current_rank not in self.config.rank: + return + self.register_hook_new() + self.first_start = False + self.switch = True + logger.info_on_rank_0(f"Dump switch is turned on at step {self.current_iter}. ") + self.create_dirs() + logger.info_on_rank_0(f"Dump data will be saved in {self.dump_iter_dir}.") + + def stop(self): + if self.config.step and self.current_iter not in self.config.step: + return + if self.config.rank and self.current_rank not in self.config.rank: + return + self.switch = False + self.data_collector.write_json() + + def create_dirs(self): + check_path_before_create(self.config.dump_path) + if not os.path.exists(self.config.dump_path): + Path(self.config.dump_path).mkdir(mode=0o750, exist_ok=True) + file_check = FileChecker(self.config.dump_path, FileCheckConst.DIR) + file_check.common_check() + self.dump_iter_dir = os.path.join(self.config.dump_path, f"step{self.current_iter}") + cur_rank = self.current_rank if self.current_rank is not None else '' + dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}") + if not os.path.exists(dump_dir): + Path(dump_dir).mkdir(mode=0o750, parents=True, exist_ok=True) + if self.config.task in self.data_collector.tasks_need_tensor_data: + dump_data_dir = os.path.join(dump_dir, "dump_tensor_data") + Path(dump_data_dir).mkdir(mode=0o750, exist_ok=True) + else: + dump_data_dir = None + + dump_file_path = os.path.join(dump_dir, "dump.json") + stack_file_path = os.path.join(dump_dir, "stack.json") + construct_file_path = os.path.join(dump_dir, "construct.json") + self.data_collector.update_dump_paths( + dump_file_path, stack_file_path, construct_file_path, dump_data_dir, None) + + def register_hook_new(self): + logger.info_on_rank_0("The {} hook function is successfully mounted to the model.".format(self.config.task)) + if self.config.level == "L1": + api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) + api_register.api_set_hook_func() diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index 8e8ceda94..760e7c862 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -1,10 +1,8 @@ import os import yaml from msprobe.pytorch.api_accuracy_checker.common.utils import check_file_or_directory_path -from msprobe.pytorch.hook_module.utils import WrapFunctionalOps, WrapTensorOps, WrapTorchOps from msprobe.core.common.file_check import FileOpen - -WrapApi = set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) +from msprobe.pytorch.pt_config import RunUTConfig class Config: @@ -24,6 +22,7 @@ class Config: def validate(key, value): validators = { 'white_list': list, + 'black_list': list, 'error_data_path': str, 'precision': int } @@ -34,14 +33,11 @@ class Config: if key == 'precision' and value < 0: raise ValueError("precision must be greater than 0") if key == 'white_list': - if not isinstance(value, list): - raise ValueError("white_list must be a list type") - if not all(isinstance(i, str) for i in value): - raise ValueError("All elements in white_list must be of str type") - invalid_api = [i for i in value if i not in WrapApi] - if invalid_api: - raise ValueError( - f"{', '.join(invalid_api)} is not in support_wrap_ops.yaml, please check the white_list") + RunUTConfig.check_filter_list_config(key, value) + if key == 'black_list': + RunUTConfig.check_filter_list_config(key, value) + if key == 'error_data_path': + RunUTConfig.check_error_data_path_config(value) return value diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index d5d08818a..b6e893296 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -166,6 +166,7 @@ def initialize_save_path(save_path, dir_name): os.mkdir(data_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) data_path_checker = FileChecker(data_path, FileCheckConst.DIR) data_path_checker.common_check() + return data_path def write_pt(file_path, tensor): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml index 7f26c72aa..2dac535dc 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml @@ -1,4 +1,5 @@ white_list: [] +black_list: [] error_data_path: './' precision: 14 \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index e38b4e6b2..732745ee8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -14,6 +14,9 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import exec_api, generat from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents from msprobe.core.common.file_check import check_link from msprobe.pytorch.common.log import logger +from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward +from msprobe.core.common.const import Const + def check_tensor_overflow(x): if isinstance(x, torch.Tensor) and x.numel() != 0 and x.dtype != torch.bool: @@ -52,12 +55,12 @@ def check_data_overflow(x): def run_overflow_check(forward_file): logger.info("start UT test") - forward_content = get_json_contents(forward_file) + forward_content, _, real_data_path = parse_json_info_forward_backward(forward_file) for api_full_name, api_info_dict in tqdm(forward_content.items()): try: - run_torch_api(api_full_name, api_info_dict) + run_torch_api(api_full_name, api_info_dict, real_data_path) except Exception as err: - api_name = api_full_name.split("_", 1)[1].rsplit("_", 2)[0] + _, api_name, _ = api_full_name.split(Const.SEP) if "not implemented for 'Half'" in str(err): logger.warning(f"API {api_name} not support half tensor in CPU, please add {api_name} to CONVERT_API " f"'fp16_to_fp32' list in accuracy_tools/api_accuracy_check/common/utils.py file.") @@ -68,11 +71,10 @@ def run_overflow_check(forward_file): logger.error(f"Run {api_full_name} UT Error: %s" % str(err)) -def run_torch_api(api_full_name, api_info_dict): +def run_torch_api(api_full_name, api_info_dict, real_data_path): torch.npu.clear_npu_overflow_flag() - api_type = api_full_name.split(".")[0] - api_name = api_full_name.split(".", 1)[1].rsplit(".", 2)[0] - args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path='') + api_type, api_name, _ = api_full_name.split(Const.SEP) + args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path) if not need_grad: logger.warning("%s function with out=... arguments don't support automatic differentiation, skip backward." % api_full_name) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 6295245a2..30994f709 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -32,6 +32,7 @@ from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward from msprobe.core.common.file_check import FileOpen, FileChecker, \ change_mode, check_file_suffix, check_link, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger +from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.const import Const, FileCheckConst, CompareConst current_time = time.strftime("%Y%m%d%H%M%S") @@ -39,7 +40,8 @@ UT_ERROR_DATA_DIR = 'ut_error_data' + current_time RESULT_FILE_NAME = "accuracy_checking_result_" + current_time + ".csv" DETAILS_FILE_NAME = "accuracy_checking_details_" + current_time + ".csv" RunUTConfig = namedtuple('RunUTConfig', ['forward_content', 'backward_content', 'result_csv_path', 'details_csv_path', - 'save_error_data', 'is_continue_run_ut', 'real_data_path']) + 'save_error_data', 'is_continue_run_ut', 'real_data_path', 'white_list', + 'black_list', 'error_data_path']) not_backward_list = ['repeat_interleave'] not_detach_set = {'resize_', 'resize_as_', 'set_', 'transpose_', 't_', 'squeeze_', 'unsqueeze_'} not_raise_dtype_set = {'type_as'} @@ -176,8 +178,7 @@ def run_ut(config): logger.info(f"UT task result will be saved in {config.result_csv_path}") logger.info(f"UT task details will be saved in {config.details_csv_path}") if config.save_error_data: - error_data_path = os.path.abspath(os.path.join(msCheckerConfig.error_data_path, UT_ERROR_DATA_DIR)) - logger.info(f"UT task error_datas will be saved in {error_data_path}") + logger.info(f"UT task error_datas will be saved in {config.error_data_path}") compare = Comparator(config.result_csv_path, config.details_csv_path, config.is_continue_run_ut) with FileOpen(config.result_csv_path, 'r') as file: csv_reader = csv.reader(file) @@ -188,17 +189,17 @@ def run_ut(config): continue if is_unsupported_api(api_full_name): # TODO run_ut does not support to the npu fusion api and distributed api continue + [_, api_name, _] = api_full_name.split(Const.SEP) try: - if msCheckerConfig.white_list: - [_, api_name, _] = api_full_name.split(Const.SEP) - if api_name not in set(msCheckerConfig.white_list): - continue + if config.black_list and api_name in config.black_list: + continue + if config.white_list and api_name not in config.white_list: + continue data_info = run_torch_api(api_full_name, config.real_data_path, config.backward_content, api_info_dict) is_fwd_success, is_bwd_success = compare.compare_output(api_full_name, data_info) if config.save_error_data: - do_save_error_data(api_full_name, data_info, is_fwd_success, is_bwd_success) + do_save_error_data(api_full_name, data_info, config.error_data_path, is_fwd_success, is_bwd_success) except Exception as err: - [_, api_name, _] = api_full_name.split(Const.SEP) if "expected scalar type Long" in str(err): logger.warning(f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API " f"'int32_to_int64' list in accuracy_tools/api_accuracy_check/common/utils.py file.") @@ -227,16 +228,16 @@ def is_unsupported_api(api_name): return flag -def do_save_error_data(api_full_name, data_info, is_fwd_success, is_bwd_success): +def do_save_error_data(api_full_name, data_info, error_data_path, is_fwd_success, is_bwd_success): if not is_fwd_success or not is_bwd_success: - processor = UtDataProcessor(os.path.join(msCheckerConfig.error_data_path, UT_ERROR_DATA_DIR)) + processor = UtDataProcessor(error_data_path) for element in data_info.in_fwd_data_list: processor.save_tensors_in_element(api_full_name + '.forward.input', element) - processor.save_tensors_in_element(api_full_name + '.forward.output.bench', data_info.bench_out) - processor.save_tensors_in_element(api_full_name + '.forward.output.device', data_info.device_out) + processor.save_tensors_in_element(api_full_name + '.forward.output.bench', data_info.bench_output) + processor.save_tensors_in_element(api_full_name + '.forward.output.device', data_info.device_output) processor.save_tensors_in_element(api_full_name + '.backward.input', data_info.grad_in) - processor.save_tensors_in_element(api_full_name + '.backward.output.bench', data_info.bench_grad_out) - processor.save_tensors_in_element(api_full_name + '.backward.output.device', data_info.device_grad_out) + processor.save_tensors_in_element(api_full_name + '.backward.output.bench', data_info.bench_grad) + processor.save_tensors_in_element(api_full_name + '.backward.output.device', data_info.device_grad) def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict): @@ -314,14 +315,14 @@ def run_backward(args, grad, grad_index, out): return grad_out -def initialize_save_error_data(): - error_data_path = msCheckerConfig.error_data_path +def initialize_save_error_data(error_data_path): check_path_before_create(error_data_path) create_directory(error_data_path) - error_data_path_checker = FileChecker(msCheckerConfig.error_data_path, FileCheckConst.DIR, + error_data_path_checker = FileChecker(error_data_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE) error_data_path = error_data_path_checker.common_check() - initialize_save_path(error_data_path, UT_ERROR_DATA_DIR) + error_data_path =initialize_save_path(error_data_path, UT_ERROR_DATA_DIR) + return error_data_path def get_validated_result_csv_path(result_csv_path, mode): @@ -384,6 +385,8 @@ def _run_ut_parser(parser): required=False) parser.add_argument("-f", "--filter_api", dest="filter_api", action="store_true", help=" Whether to filter the api in the api_info_file.", required=False) + parser.add_argument("-config", "--config_path", dest="config_path", default="", type=str, + help=" The path of config.json", required=False) def preprocess_forward_content(forward_content): @@ -464,14 +467,22 @@ def run_ut_command(args): if args.result_csv_path: result_csv_path = get_validated_result_csv_path(args.result_csv_path, 'result') details_csv_path = get_validated_details_csv_path(result_csv_path) + white_list = msCheckerConfig.white_list + black_list = msCheckerConfig.black_list + error_data_path = msCheckerConfig.error_data_path + if args.config_path: + _, task_config = parse_json_config(args.config_path, Const.RUN_UT) + white_list = task_config.white_list + black_list = task_config.black_list + error_data_path = task_config.error_data_path if save_error_data: if args.result_csv_path: time_info = result_csv_path.split('.')[0].split('_')[-1] global UT_ERROR_DATA_DIR UT_ERROR_DATA_DIR = 'ut_error_data' + time_info - initialize_save_error_data() + error_data_path = initialize_save_error_data(error_data_path) run_ut_config = RunUTConfig(forward_content, backward_content, result_csv_path, details_csv_path, save_error_data, - args.result_csv_path, real_data_path) + args.result_csv_path, real_data_path, set(white_list), set(black_list), error_data_path) run_ut(run_ut_config) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index a3118e21c..acc1de105 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -29,7 +29,6 @@ except ImportError: else: is_gpu = False - torch_without_guard_version_list = ['2.1', '2.2'] for version in torch_without_guard_version_list: if torch.__version__.startswith(version): diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index a4b688434..e21491056 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -37,6 +37,7 @@ from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, C format_value, check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, create_directory from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.core.common.exceptions import FileCheckException def check_graph_mode(a_op_name, b_op_name): @@ -491,6 +492,10 @@ def compare_by_op(op_name, op_name_mapping_dict, input_parma): error_file = error.filename n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE error_flag = True + except FileCheckerException: + error_file = data_name + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True n_value, b_value, error_flag = get_error_type(n_value, b_value, error_flag) if not error_flag: diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py index 1fce5a303..e28e588c5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py @@ -4,7 +4,7 @@ from msprobe.pytorch.debugger.debugger_config import DebuggerConfig from msprobe.pytorch.service import Service from msprobe.pytorch.common.log import logger from msprobe.pytorch.pt_config import parse_json_config -from msprobe.core.common.exceptions import MsaccException +from msprobe.core.common.exceptions import MsprobeException class PrecisionDebugger: @@ -50,8 +50,8 @@ class PrecisionDebugger: def check_model_valid(model): if not model or isinstance(model, torch.nn.Module): return model - raise MsaccException( - MsaccException.INVALID_PARAM_ERROR, "model 参数必须是torch.nn.Module类型。" + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, "model 参数必须是torch.nn.Module类型。" ) @classmethod diff --git a/debug/accuracy_tools/msprobe/pytorch/functional/dump_module.py b/debug/accuracy_tools/msprobe/pytorch/functional/dump_module.py index 7e72aab8a..efb95c336 100644 --- a/debug/accuracy_tools/msprobe/pytorch/functional/dump_module.py +++ b/debug/accuracy_tools/msprobe/pytorch/functional/dump_module.py @@ -3,7 +3,7 @@ from msprobe.pytorch.common.log import logger from msprobe.core.common.const import Const from msprobe.pytorch.hook_module.api_registry import api_register from msprobe.pytorch.debugger.precision_debugger import PrecisionDebugger -from msprobe.core.common.exceptions import MsaccException +from msprobe.core.common.exceptions import MsprobeException from msprobe.core.data_dump.scope import BaseScope module_count = {} @@ -12,10 +12,10 @@ module_count = {} def module_dump(module, dump_name): if not isinstance(module, nn.Module): logger.error("The parameter:module in module_dump is not a Module subclass.") - raise MsaccException(MsaccException.INVALID_PARAM_ERROR) + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) if not isinstance(dump_name, str): logger.error("The parameter:dump_name in module_dump is not a str type.") - raise MsaccException(MsaccException.INVALID_PARAM_ERROR) + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR) api_register.api_originality() if dump_name not in module_count: module_count[dump_name] = 0 diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py index 19c18c124..048ab3f90 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py @@ -228,7 +228,7 @@ class Comparator: else: is_bwd_success, bwd_compare_alg_results = True, None if is_bwd_success and bwd_compare_alg_results is None: - self.saver.record_results(ResultInfo(api_name, is_fwd_success, CompareConst.NA, fwd_compare_alg_results, + self.saver.record_results(ResultInfo(api_name, is_fwd_success, CompareConst.NAN, fwd_compare_alg_results, bwd_compare_alg_results)) else: self.saver.record_results(ResultInfo(api_name, is_fwd_success, is_bwd_success, fwd_compare_alg_results, diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index ff09bfd8e..a3d765f3a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -4,6 +4,7 @@ import os from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.file_check import FileOpen from msprobe.core.common.const import Const +from msprobe.pytorch.hook_module.utils import WrapFunctionalOps, WrapTensorOps, WrapTorchOps class TensorConfig(BaseConfig): @@ -61,20 +62,54 @@ class FreeBenchmarkCheckConfig(BaseConfig): if self.preheat_step and self.preheat_step == 0: raise Exception("preheat_step cannot be 0") + +class RunUTConfig(BaseConfig): + WrapApi = set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) + def __init__(self, json_config): + super().__init__(json_config) + self.white_list = json_config.get("white_list", Const.DEFAULT_LIST) + self.black_list = json_config.get("black_list", Const.DEFAULT_LIST) + self.error_data_path = json_config.get("error_data_path", Const.DEFAULT_PATH) + self.check_run_ut_config() + + @classmethod + def check_filter_list_config(cls, key, filter_list): + if not isinstance(filter_list, list): + raise Exception("%s must be a list type" % key) + if not all(isinstance(item, str) for item in filter_list): + raise Exception("All elements in %s must be string type" % key) + invalid_api = [item for item in filter_list if item not in cls.WrapApi] + if invalid_api: + raise Exception("Invalid api in %s: %s" % (key, invalid_api)) + + @classmethod + def check_error_data_path_config(cls, error_data_path): + if not os.path.exists(error_data_path): + raise Exception("error_data_path: %s does not exist" % error_data_path) + + def check_run_ut_config(self): + RunUTConfig.check_filter_list_config(Const.WHITE_LIST, self.white_list) + RunUTConfig.check_filter_list_config(Const.BLACK_LIST, self.black_list) + RunUTConfig.check_error_data_path_config(self.error_data_path) + + def parse_task_config(task, json_config): default_dic = {} if task == Const.TENSOR: - config_dic = json_config.get(Const.TENSOR) if json_config.get(Const.TENSOR) else default_dic + config_dic = json_config.get(Const.TENSOR, default_dic) return TensorConfig(config_dic) elif task == Const.STATISTICS: - config_dic = json_config.get(Const.STATISTICS) if json_config.get(Const.STATISTICS) else default_dic + config_dic = json_config.get(Const.STATISTICS, default_dic) return StatisticsConfig(config_dic) elif task == Const.OVERFLOW_CHECK: - config_dic = json_config.get(Const.OVERFLOW_CHECK) if json_config.get(Const.OVERFLOW_CHECK) else default_dic + config_dic = json_config.get(Const.OVERFLOW_CHECK, default_dic) return OverflowCheckConfig(config_dic) elif task == Const.FREE_BENCHMARK: - config_dic = json_config.get(Const.FREE_BENCHMARK) if json_config.get(Const.FREE_BENCHMARK) else default_dic + config_dic = json_config.get(Const.FREE_BENCHMARK, default_dic) return FreeBenchmarkCheckConfig(config_dic) + elif task == Const.RUN_UT: + config_dic = json_config.get(Const.RUN_UT, default_dic) + return RunUTConfig(config_dic) else: return StatisticsConfig(default_dic) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index e5da44484..0ab7d0c58 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -5,7 +5,7 @@ from pathlib import Path from msprobe.pytorch.common.log import logger from msprobe.core.common.file_check import FileChecker, check_path_before_create from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.exceptions import DistributedNotInitializedError, MsaccException +from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs @@ -138,7 +138,7 @@ class Service: logger.info_on_rank_0("The {} hook function is successfully mounted to the model.".format(self.config.task)) if self.config.level in ["L0", "mix"]: if self.model is None: - logger.error_log_with_exp("The model is None.", MsaccException.INVALID_PARAM_ERROR) + logger.error_log_with_exp("The model is None.", MsprobeException.INVALID_PARAM_ERROR) logger.info_on_rank_0("The init dump mode is enabled, and the module dump function will not be available") for name, module in self.model.named_modules(): if module == self.model: @@ -164,4 +164,4 @@ class Service: api_register.api_modularity() if Const.STATISTICS == self.config.task or Const.TENSOR == self.config.task: - remove_dropout() + remove_dropout() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index 15957af21..06c7378ed 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -19,7 +19,7 @@ from unittest.mock import patch from msprobe.core.common.log import logger from msprobe.core.common.const import Const -from msprobe.core.common.exceptions import MsaccException +from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common_config import CommonConfig, BaseConfig @@ -44,7 +44,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "task is invalid, it should be one of {}".format(Const.TASK_LIST)) self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) json_config.update({"task": Const.TENSOR}) json_config.update({"rank": 0}) @@ -52,7 +52,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "rank is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) @@ -61,7 +61,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "step is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) @@ -71,7 +71,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "level is invalid, it should be one of {}".format(Const.LEVEL_LIST)) self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) @@ -82,7 +82,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "seed is invalid, it should be an integer") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) @@ -94,7 +94,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "is_deterministic is invalid, it should be a boolean") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) @@ -107,7 +107,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "enable_dataloader is invalid, it should be a boolean") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) @patch.object(logger, "error_log_with_exp") def test_base_config(self, mock_error_log_with_exp): @@ -130,7 +130,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "scope is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) json_config.update({"scope": ["Tensor_Add"]}) json_config.update({"list": "Tensor_Add"}) @@ -139,7 +139,7 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "list is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) json_config.update({"scope": ["Tensor_Add"]}) json_config.update({"list": ["Tensor_Add"]}) @@ -149,4 +149,4 @@ class TestCommonConfig(TestCase): self.assertEqual(mock_error_log_with_exp.call_args[0][0], "data_mode is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsaccException.err_strs.get(MsaccException.INVALID_PARAM_ERROR)) + MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_debugger_config.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_debugger_config.py index 54bc1393a..5187d3951 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_debugger_config.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_debugger_config.py @@ -27,7 +27,7 @@ class TestDebuggerConfig(TestCase): "dump_path": "/absolute_path", "rank": [], "step": [], - "level": "L1" + "level": "L0" } common_config = CommonConfig(json_config) task_config = BaseConfig(json_config) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_precision_debugger.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_precision_debugger.py index b33167dc7..425ed3040 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_precision_debugger.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_precision_debugger.py @@ -35,7 +35,7 @@ class TestPrecisionDebugger(TestCase): "dump_path": "/absolute_path", "rank": [], "step": [], - "level": "L1" + "level": "L0" } common_config = CommonConfig(json_config) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_config.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_config.py index ec606d9aa..35fc61647 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_config.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_config.py @@ -35,5 +35,5 @@ class TestConfig(unittest.TestCase): validate_white_list = ['conv1d', 'max_pool1d', 'dropout', '__add__'] self.assertEqual(self.cfg.validate('white_list', validate_white_list), validate_white_list) - with self.assertRaises(ValueError): + with self.assertRaises(Exception): self.cfg.validate('white_list', ['invalid_api1', 'invalid_api2']) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py index fb442941b..288e259c0 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py @@ -1,17 +1,267 @@ # coding=utf-8 import unittest -from msprobe.pytorch.compare.acc_compare import rename_api +import pandas as pd +from msprobe.pytorch.compare import acc_compare as compare + +npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', + 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], + 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), + ('torch.float32', [16])], + 'output_struct': [('torch.float32', [1, 16, 28, 28])], + 'summary': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], + [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], + [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], + [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} + +bench_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', + 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], + 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), + ('torch.float32', [16])], + 'output_struct': [('torch.float32', [1, 16, 28, 28])], + 'summary': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], + [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], + [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], + [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} + +tensor_list = [ + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, + 'Min': -0.331031858921051,'Mean': -0.030964046716690063, 'Norm': 2.2533628940582275, 'requires_grad': True, + 'full_op_name': 'Tensor.add_.0.forward_input.0'}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_.0.forward_input.1'}, + {'full_op_name': 'Tensor.add_.0.forward_input.alpha.0', 'dtype': "", "shape": '[]', 'md5': None, + 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_.0.forward_output.0'} +] + +result_op_dict = {'op_name': ['Tensor.add_.0.forward_input.0', 'Tensor.add_.0.forward_input.1', + 'Tensor.add_.0.forward_input.alpha.0', 'Tensor.add_.0.forward_output.0'], + 'input_struct': [('torch.float32', [16, 1, 3, 3]), ('torch.float32', [16, 1, 3, 3]), + ("", '[]')], + 'output_struct': [('torch.float32', [16, 1, 3, 3])], + 'summary': [[0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275], + [0.003992878366261721, -0.008102823048830032, -0.0002002553956117481, 0.02844562754034996], + [-0.1, -0.1, -0.1, -0.1], + [0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275]], + 'stack_info': []} + +o_result = [ + ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.0', 'torch.float32', 'torch.float32', + [1, 1, 28, 28], [1, 1, 28, 28], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 3.029174327850342, -2.926689624786377, + -0.06619918346405029, 3.029174327850342, -2.926689624786377, -0.06619918346405029, '', '', 'None'], + ['Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.1', 'torch.float32', 'torch.float32', + [16, 1, 5, 5], [16, 1, 5, 5], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 0.19919930398464203, -0.19974489510059357, + 0.006269412115216255, 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, '', '', 'None'], + ['Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_input.2', 'torch.float32', 'torch.float32', + [16], [16], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, + 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, '', '', 'None'], + ['Functional_conv2d_0_forward_output', 'Functional_conv2d_0_forward_output', 'torch.float32', 'torch.float32', + [1, 16, 28, 28], [1, 16, 28, 28], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 2.1166646480560303, -2.190781354904175, + -0.003579073818400502, 2.1166646480560303, -2.190781354904175, -0.003579073818400502, '', '', 'None']] + +npu_dict_aten = {'op_name': ['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', + 'Aten__native_batch_norm_legit_functional.default_0_forward_input.1', + 'Aten__native_batch_norm_legit_functional.default_0_forward_input.2', + 'Aten__native_batch_norm_legit_functional.default_0_forward_input.3', + 'Aten__native_batch_norm_legit_functional.default_0_forward_input.4', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.0', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.1', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.2', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.3', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.4'], + 'input_struct': [('torch.float16', [256, 256, 14, 14]), ('torch.float32', [256]), + ('torch.float32', [256]), ('torch.float32', [256]), ('torch.float32', [256])], + 'output_struct': [('torch.float16', [256, 256, 14, 14]), ('torch.float32', [256]), + ('torch.float32', [256]), ('torch.float32', [256]), ('torch.float32', [256])], + 'summary': [[139.625, -127.5625, -0.0103607177734375], + [2.5276029109954834, -2.1788690090179443, -0.0008259844034910202], + [2.472219944000244, -2.845968723297119, -0.008756577968597412], + [2.763145923614502, -3.398397922515869, -0.052132632583379745], + [2.673110008239746, -3.149275064468384, 0.01613386906683445], + [13.5546875, -10.640625, -0.008758544921875], + [0.30550330877304077, -0.24485322833061218, -0.010361209511756897], + [623.9192504882812, 432.96826171875, 520.2276611328125], + [2.4797861576080322, -3.055997371673584, -0.04795549064874649], + [61.7945556640625, 42.59713363647461, 52.03831481933594]]} + +bench_dict_functional = { + 'op_name': ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.1', + 'Functional_batch_norm_0_forward_input.2', 'Functional_batch_norm_0_forward_input.3', + 'Functional_batch_norm_0_forward_input.4', 'Functional_batch_norm_0_forward_output'], + 'input_struct': [('torch.float32', [256, 256, 14, 14]), ('torch.float32', [256]), ('torch.float32', [256]), + ('torch.float32', [256]), ('torch.float32', [256])], + 'output_struct': [('torch.float32', [256, 256, 14, 14])], + 'summary': [[3.061628818511963, -3.22507381439209, 3.634914173744619e-05], + [0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06], + [0.9338104128837585, 0.9277191162109375, 0.930335283279419], + [1.0, 1.0, 1.0], [0.0, 0.0, 0.0], + [5.397906303405762, -5.796811580657959, 2.5283952709287405e-10]] +} + +aten_result = [ + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', + 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 136.56337118148804, -124.33742618560791, + -0.010397066915174946, ' ', '4460.480981749501%', '3855.335826136584%', '28603.33536971545%', ' ', 139.625, + -127.5625, -0.0103607177734375, 3.061628818511963, -3.22507381439209, 3.634914173744619e-05, 'Warning', + 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.1', 'Functional_batch_norm_0_forward_input.1', + 'torch.float32', 'torch.float32', [256], [256], 2.527024927258026, -2.1782388387364335, -0.0008296193100250093, + ' ', '437213.84590749856%', '345658.76916858414%', '22823.676544842117%', ' ', 2.5276029109954834, + -2.1788690090179443, -0.0008259844034910202, 0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06, + 'Warning', 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.2', 'Functional_batch_norm_0_forward_input.2', + 'torch.float32', 'torch.float32', [256], [256], 1.5384095311164856, -3.7736878395080566, -0.9390918612480164, ' ', + '164.74538192025793%', '406.7705163736246%', '100.94122819224167%', ' ', 2.472219944000244, -2.845968723297119, + -0.008756577968597412, 0.9338104128837585, 0.9277191162109375, 0.930335283279419, 'Warning', + 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.3', 'Functional_batch_norm_0_forward_input.3', + 'torch.float32', 'torch.float32', [256], [256], 1.763145923614502, -4.398397922515869, -1.0521326325833797, ' ', + '176.3145923614502%', '439.8397922515869%', '105.21326325833797%', ' ', 2.763145923614502, -3.398397922515869, + -0.052132632583379745, 1.0, 1.0, 1.0, 'Warning', 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.4', 'Functional_batch_norm_0_forward_input.4', + 'torch.float32', 'torch.float32', [256], [256], 2.673110008239746, -3.149275064468384, 0.01613386906683445, ' ', + 'N/A', 'N/A', 'N/A', ' ', 2.673110008239746, -3.149275064468384, 0.01613386906683445, 0.0, 0.0, 0.0, 'Warning', + 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.0', 'Functional_batch_norm_0_forward_output', + 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 8.156781196594238, -4.843813419342041, + -0.008758545174714527, ' ', '151.11009228611078%', '83.55995967687207%', '3464072756.115108%', ' ', 13.5546875, + -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Warning', + 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', + ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', + 'Yes', '', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', + ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', + 'Yes', '', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.3', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', + ' ', ' ', ' ', ' ', ' ', 2.4797861576080322, -3.055997371673584, -0.04795549064874649, 'Nan', 'Nan', 'Nan', + 'Yes', '', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.4', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', + ' ', ' ', ' ', ' ', ' ', 61.7945556640625, 42.59713363647461, 52.03831481933594, 'Nan', 'Nan', 'Nan', + 'Yes', '', 'None']] + +highlight_dict = {'red_rows': [], 'yellow_rows': []} + +num_0, num_1, num_2, num_3 = 0, 1, 2, 3 +summary_line_input = ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.01, 0, 0, 0, 1, 1, 1, 1, 1.01, 1, 1, 1, + 'Yes', ''] +summary_line_1 = ['Functional_batch_norm_0_forward_output.0', 'Functional_batch_norm_0_forward_output.0', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 10, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, + 'Warning', ''] +summary_line_2 = ['Functional_batch_norm_0_forward_output.1', 'Functional_batch_norm_0_forward_output.1', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.02, 0, 0, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, + 'Warning', ''] +summary_line_3 = ['Functional_batch_norm_0_forward_output.2', 'Functional_batch_norm_0_forward_output.2', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, + 'Warning', ''] +line_input = ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 1, 1, 1, 0.95, 1, 1, 1, 1, 1, 1.01, 1, 1, 1, + 'Yes', ''] +line_1 = ['Functional_batch_norm_0_forward_output.0', 'Functional_batch_norm_0_forward_output.0', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 1, 1, 0.59, 1, 'nan', 0, 1, 1, 19, 1, 1, 1, + 'Warning', ''] +line_2 = ['Functional_batch_norm_0_forward_output.1', 'Functional_batch_norm_0_forward_output.1', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.9, 1, 1, 0.8, 1, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, + 'Warning', ''] +line_3 = ['Functional_batch_norm_0_forward_output.2', 'Functional_batch_norm_0_forward_output.2', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 1.1e+10, 1, 0.85, 1, 9, 0.12, 0, 1, 1, 0.1, 1, + 1, 1, 'Warning', ''] + +op_data = { + 'input_args': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051,'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, + 'Norm': 0.02844562754034996, 'requires_grad': False}], + 'input_kwargs': {'alpha': {'type': 'float', 'value': -0.1}}, + 'output': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051,'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True}]} + +op_name = "Tensor.add_0.0.forward" + +op_result = [ + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward_input.0'}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.forward_input.1'}, + {'full_op_name': 'Tensor.add_0.0.forward_input.alpha.0', 'dtype': "", 'shape': '[]', 'md5': None, + 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward_output.0'}] + class TestUtilsMethods(unittest.TestCase): + def test_check_graph_mode(self): + op1 = "Aten" + op2 = "torch" + self.assertTrue(compare.check_graph_mode(op1, op2)) + self.assertTrue(compare.check_graph_mode(op2, op1)) + self.assertFalse(compare.check_graph_mode(op1, op1)) + self.assertFalse(compare.check_graph_mode(op2, op2)) + + def test_check_op(self): + fuzzy_match = False + result = compare.check_op(npu_dict, bench_dict, fuzzy_match) + self.assertEqual(result, True) + + def test_merge_tensor(self): + op_dict = compare.merge_tensor(tensor_list, True, False) + self.assertEqual(op_dict, result_op_dict) + + def test_read_op(self): + result = compare.read_op(op_data, op_name) + self.assertEqual(result, op_result) + + def test_match_op(self): + fuzzy_match = False + a, b = compare.match_op([npu_dict], [bench_dict], fuzzy_match) + self.assertEqual(a, 0) + self.assertEqual(b, 0) + + def test_get_accuracy(self): + result = [] + compare.get_accuracy(result, npu_dict, bench_dict, highlight_dict) + self.assertEqual(result, o_result) + + def test_get_accuracy_graph_mode(self): + result = [] + compare.get_accuracy(result, npu_dict_aten, bench_dict_functional, highlight_dict) + self.assertEqual(result, aten_result) + + def test_find_error_rows(self): + summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] + highlight_dict = {'red_rows': [], 'yellow_rows': []} + compare.find_error_rows(summary_result, 0, 1, highlight_dict, summary_compare=True) + self.assertEqual(highlight_dict, {'red_rows': [], 'yellow_rows': []}) + + def test_find_compare_result_error_rows(self): + result = [line_input, line_1, line_2, line_3] + result_df = pd.DataFrame(result) + highlight_dict = {'red_rows': [], 'yellow_rows': []} + compare.find_compare_result_error_rows(result_df, highlight_dict, False, False) + self.assertEqual(highlight_dict, {'red_rows': [num_1, num_3], 'yellow_rows': [num_2]}) + def test_rename_api(self): test_name_1 = "Distributed.broadcast.0.forward.input.0" expect_name_1 = "Distributed.broadcast.input.0" - actual_name_1 = rename_api(test_name_1, "forward") + actual_name_1 = compare.rename_api(test_name_1, "forward") self.assertEqual(actual_name_1, expect_name_1) - + test_name_2 = "Torch.sum.0.backward.output.0" expect_name_2 = "Torch.sum.output.0" - actual_name_2 = rename_api(test_name_2, "backward") + actual_name_2 = compare.rename_api(test_name_2, "backward") self.assertEqual(actual_name_2, expect_name_2) - \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py new file mode 100644 index 000000000..ac28e994e --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py @@ -0,0 +1,20 @@ +# coding=utf-8 +import unittest +from msprobe.pytorch.compare import match + + +class TestMatch(unittest.TestCase): + def test_graph_mapping(self): + op1 = "Aten_convolution_1_forward_0.input.0" + op2 = "Torch_conv2d_0_forward_0.input.0" + op3 = "Torch_batch_norm_0_forward_0.input.0" + op4 = "Aten_convolution.default_1_forward_0.input.0" + op5 = "Aten_foo_1_forward_0.input.0" + self.assertTrue(match.graph_mapping.match(op1, op2)) + self.assertTrue(match.graph_mapping.match(op2, op1)) + self.assertTrue(match.graph_mapping.match(op4, op2)) + self.assertTrue(match.graph_mapping.match(op2, op4)) + self.assertFalse(match.graph_mapping.match(op1, op3)) + self.assertFalse(match.graph_mapping.match(op3, op1)) + self.assertFalse(match.graph_mapping.match(op5, op2)) + self.assertFalse(match.graph_mapping.match(op2, op5)) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py index 53b4e66c1..c344f0b66 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py @@ -67,3 +67,18 @@ class TestPtConfig(TestCase): self.assertEqual(result.handler_type, "check") self.assertEqual(result.preheat_step, 15) self.assertEqual(result.max_sample, 20) + + run_ut_config = { + "run_ut": { + "white_list": ["conv2d"], + "black_list": ["matmul"], + "error_data_path": '/home/dump_path' + + } + } + with patch('os.path.exists', return_value=True) as mocked_exists: + result = parse_task_config(Const.RUN_UT, run_ut_config) + self.assertEqual(result.white_list, ["conv2d"]) + self.assertEqual(result.black_list, ["matmul"]) + self.assertEqual(result.error_data_path, '/home/dump_path') + mocked_exists.assert_called_once_with('/home/dump_path') diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index 3d2c3bb87..4e0eaa1f3 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" # Copyright (C) 2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -13,28 +10,59 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -from setuptools import setup, find_packages +import setuptools -setup( - name='mindstudio_probe', - version='1.0', - description='This is a pytorch precision comparison tools', - long_description='This is a pytorch precision comparison tools, include ptdbg and api accuracy checker', - packages=find_packages(), - install_requires=[ - "wheel", - "numpy", - "pandas >= 1.3.5", - "pyyaml", - "rich", - "tqdm", - "openpyxl" - ], + +__version__ = '1.0.0' + +INSTALL_REQUIRED = [ + "wheel", + "numpy", + "pandas >= 1.3.5", + "pyyaml", + "rich", + "tqdm", + "openpyxl" +] + +EXCLUDE_PKGS = [ + "api_accuracy_checker*", + "grad_tool*", + "kj600*", + "ptdbg_ascend*", + "msprobe.test*", +] + +setuptools.setup( + name="mindstudio-probe", + version=__version__, + description="Pytorch Ascend Probe Utils", + long_description="MindStudio-Probe is a set of tools for diagnosing and improving model accuracy on Ascend NPU, " + "including API acc checker, ptdbg, grad tool etc.", + url="https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe", + author="Ascend Team", + author_email="pmail_mindstudio@huawei.com", + packages=setuptools.find_namespace_packages(exclude=EXCLUDE_PKGS, include=["msprobe", "msprobe*"]), include_package_data=True, + python_requires=">=3.6.2", + install_requires=INSTALL_REQUIRED, + classifiers=[ + 'Intended Audience :: Developers', + 'Intended Audience :: Education', + 'Intended Audience :: Science/Research', + 'Programming Language :: Python :: 3', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Mathematics', + 'Topic :: Scientific/Engineering :: Artificial Intelligence', + 'Topic :: Software Development', + 'Topic :: Software Development :: Libraries', + 'Topic :: Software Development :: Libraries :: Python Modules', + ], + license='Apache License 2.0', + keywords='pytorch msprobe ascend', ext_modules=[], zip_safe=False, entry_points={ 'console_scripts': ['msprobe=msprobe.msprobe:main'], - },) \ No newline at end of file + },) diff --git a/profiler/advisor/README.md b/profiler/advisor/README.md index c650f40b3..47e64a90b 100644 --- a/profiler/advisor/README.md +++ b/profiler/advisor/README.md @@ -36,11 +36,11 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 3. 查看结果。 - 分析结果输出相关简略建议到执行终端中，并生成`att_advisor_{timestamp}.html`和`att_advisor_{timestamp}.xlsx`文件供用户预览。 + 分析结果输出相关简略建议到执行终端中，并生成`mstt_advisor_{timestamp}.html`和`mstt_advisor_{timestamp}.xlsx`文件供用户预览。 - `att_advisor_{timestamp}.xlsx`文件内容与执行终端输出一致。 + `mstt_advisor_{timestamp}.xlsx`文件内容与执行终端输出一致。 - `att_advisor_{timestamp}.html`文件分析详见“**报告解析**”。 + `mstt_advisor_{timestamp}.html`文件分析详见“**报告解析**”。 执行终端输出示例如下： diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index 5f4bd3202..ada1b0bf4 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import logging from functools import wraps from typing import Dict, List, Union @@ -59,14 +73,6 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): def optimize(self, **kwargs): pass - @abstractmethod - def make_record(self): - pass - - @abstractmethod - def make_render(self): - pass - def init_dataset_list(self)->None: dataset_cls_list = self.dataset_cls_list if len(dataset_cls_list) == 0: diff --git a/profiler/advisor/analyzer/cluster/slow_link_analyser.py b/profiler/advisor/analyzer/cluster/slow_link_analyser.py index 846b79a50..0b585cbc7 100644 --- a/profiler/advisor/analyzer/cluster/slow_link_analyser.py +++ b/profiler/advisor/analyzer/cluster/slow_link_analyser.py @@ -19,7 +19,7 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.common import constant from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataSet +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset class SlowLinkAnalyzer(BaseAnalyzer): @@ -35,11 +35,11 @@ class SlowLinkAnalyzer(BaseAnalyzer): SDMA = "SDMA" RDMA = "RDMA" SLOW_LINK_ANALYSIS = "slow_link_analysis" - dataset_cls_list = [ClusterCommunicationDataSet] + dataset_cls_list = [ClusterCommunicationDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs): super().__init__(collection_path, n_processes, **kwargs) - key = ClusterCommunicationDataSet.get_key() + key = ClusterCommunicationDataset.get_key() self.communication_data_class = self.get_first_data_by_key(self.dataset_list, key) self.rank_bw_dict = self.communication_data_class.get_data() self.result = OptimizeResult() @@ -49,8 +49,9 @@ class SlowLinkAnalyzer(BaseAnalyzer): def optimize(self, **kwargs): if self.rank_bw_dict is None: - print("slow_link 分析失败，原因是数据加载失败，请检查你的cluster_analysis_outpu文件夹, \ - 如不关心这类数据请忽略") + print("Slow link analysis failed due to data loading failure. \ + Please check your cluster_analysis_output folder. \ + If you are not concerned about this type of data, please ignore this message.") return self.result self.process() self.format_datas = self.format_details() @@ -65,8 +66,11 @@ class SlowLinkAnalyzer(BaseAnalyzer): def produce_bottleneck(self, link_type: str): data_list = [rank_dict.get(link_type, 0) for rank_id, rank_dict in self.rank_bw_dict.items()] - avg_bw = round(sum(data_list) / len(data_list), 3) - if avg_bw == 0: + if len(data_list) > 0: + avg_bw = round(sum(data_list) / len(data_list), 3) + else: + print("The slow link (identified bottleneck) cannot provide a bottleneck \ + because the analysis data is missing bandwidth information.") return self.bottelneck += f'{link_type}: \n' \ f' The average is {avg_bw}, \n' \ diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyser.py b/profiler/advisor/analyzer/cluster/slow_rank_analyser.py index aa0ddad50..f439b31f7 100644 --- a/profiler/advisor/analyzer/cluster/slow_rank_analyser.py +++ b/profiler/advisor/analyzer/cluster/slow_rank_analyser.py @@ -19,7 +19,7 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.common import constant from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataSet +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset class SlowRankAnalyzer(BaseAnalyzer): @@ -27,11 +27,11 @@ class SlowRankAnalyzer(BaseAnalyzer): RANK = "rank" RATIO_THRESHOLD = 0.05 BOTTLENECK_LIST = ['Computing', 'Communication', "Free"] - dataset_cls_list = [ClusterStepTraceTimeDataSet] + dataset_cls_list = [ClusterStepTraceTimeDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs): super().__init__(collection_path, n_processes, **kwargs) - key = ClusterStepTraceTimeDataSet.get_key() + key = ClusterStepTraceTimeDataset.get_key() self.step_trace_class = self.get_first_data_by_key(self.dataset_list, key) self.step_trace_dict = self.step_trace_class.get_data() self.result = OptimizeResult() diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/__init__.py b/profiler/advisor/analyzer/computation/ai_core_freq/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py new file mode 100644 index 000000000..cee16cce5 --- /dev/null +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py @@ -0,0 +1,42 @@ +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_checker import AICoreFreqChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.ai_core_freq.ai_core_freq_dataset import AICoreFreqDataset +from profiler.advisor.config.config import Config + +logger = logging.getLogger() + + +class AICoreFreqAnalyzer(BaseAnalyzer): + dataset_cls_list = [AICoreFreqDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = AICoreFreqDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseAnalyzer.check_data((AICoreFreqDataset.get_key(),)) + def optimize(self, **kwargs): + if not Config().get_config("aic_frequency"): + logger.warning("Can not find ai core frequency in info.json*, please check data integrity.") + return self.result + add_render_list = kwargs.get("add_render_list", True) + ai_core_freq_checker = AICoreFreqChecker() + ai_core_freq_checker.check_ai_core_freq(self.dataset) + if not ai_core_freq_checker.ai_core_freq_issues: + return self.result + ai_core_freq_checker.make_record(self.result) + self.html = ai_core_freq_checker.make_render(self.html_render, add_render_list) + return self.result + + def make_record(self): + pass + + def make_render(self): + pass diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py new file mode 100644 index 000000000..5ea4dbd75 --- /dev/null +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -0,0 +1,100 @@ +import logging + +from profiler.advisor.dataset.ai_core_freq.ai_core_freq_dataset import AICoreFreqDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.config.config import Config +from profiler.advisor.utils.utils import convert_to_float + +logger = logging.getLogger() + + +class AICoreFreqChecker: + DEFAULT_FREQ = 1800 + DECREASE_FREQ_RATIO = 0.05 + SHOW_TOPK_OPS = 10 + TOTAL_DURATION_INDEX = 2 + DECREASE_FREQ_RATIO_INDEX = 3 + + def __init__(self): + + self.ai_core_freq_issues = False + self.desc = "" + self.suggestions = "" + self.decrease_freq_ops = [] + self.headers = [] + self.op_freq = None + self.rank_id = None + self.stage = None + + def check_ai_core_freq(self, event_dataset: AICoreFreqDataset, rank_id=None, stage=None): + """ + :Param event_dataset: dataset of timeline event + """ + if not hasattr(event_dataset, "op_freq") or not getattr(event_dataset, "op_freq"): + logger.debug("Skip slow ai core frequency checker, " + "because no ai core frequency were recorded in trace_view.json") + return + + self.rank_id = rank_id + self.stage = stage + self.op_freq = event_dataset.op_freq + for op_name, op_info in self.op_freq.items(): + freq_list = op_info.get("freq_list", []) + if not freq_list: + continue + + op_count = op_info.get("count", 0) + op_total_duration = round(op_info.get("dur", 0), 2) + max_freq = max(self.DEFAULT_FREQ, convert_to_float(Config().get_config("aic_frequency"))) + + decrease_freq_ratio = sum(max_freq - freq for freq in freq_list) / (max_freq * len(freq_list)) + if decrease_freq_ratio >= self.DECREASE_FREQ_RATIO: + self.ai_core_freq_issues = True + self.decrease_freq_ops.append([op_name, op_count, op_total_duration, + f"{round(decrease_freq_ratio, 4):.2%}", + round(sum(freq_list) / len(freq_list), 2), + max(freq_list), min(freq_list)]) + + if self.decrease_freq_ops: + # 按算子总耗时和降频比率 降序排列 + self.decrease_freq_ops.sort(key= + lambda x: (x[self.TOTAL_DURATION_INDEX], x[self.DECREASE_FREQ_RATIO_INDEX]), + reverse=True) + + self.desc = (f"{len(self.decrease_freq_ops)} operators are found during frequency reduction, and the reduction " + f"ratio is larger than {self.DECREASE_FREQ_RATIO}.") + if self.rank_id: + self.desc = f"For rank {self.rank_id}, " + self.desc.lower() + self.suggestions = "Please check the temperature or max power of your machine." + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("AI Core Frequency", self.desc, [self.suggestions]) + result.add(OptimizeRecord(optimization_item)) + + self.headers = ["Operator name", "Count", "Total duration(us)", "AI CORE frequency decreased ratio", + "Average frequency", "Max frequency", "Min frequency"] + if self.rank_id: + self.headers = ["Rank id"] + self.headers + sub_table_name = "AI Core Frequency" if not self.stage else f"Stage-{self.stage}: AI Core Frequency" + result.add_detail(sub_table_name, headers=self.headers) + + for row in self.decrease_freq_ops: + if self.rank_id: + row = [self.rank_id] + row + result.add_detail(sub_table_name, detail=row) + + def make_render(self, html_render, add_render_list=True): + if self.SHOW_TOPK_OPS: + self.desc += f" Only show {self.SHOW_TOPK_OPS} operators here, see latest mstt_advisor.xlsx for details." + return html_render.render_template(key="computation", + template_dir="templates", + template_name="ai_core_frequency.html", + desc=self.desc, + suggestion=self.suggestions, + headers=self.headers, + data=self.decrease_freq_ops[:self.SHOW_TOPK_OPS], + add_render_list=add_render_list) diff --git a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py b/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py index 4eca1c6c0..0caede4b8 100644 --- a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py +++ b/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py @@ -3,13 +3,13 @@ import os from functools import partial from typing import List, Dict, Optional -import yaml from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker, logger from profiler.advisor.analyzer.schedule.fusion_ops.timeline_api_stack_checker import OpStackFinder from profiler.advisor.common import constant from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.cluster_analyse.common_func.file_manager import FileManager class AicpuChecker(OperatorChecker): @@ -47,8 +47,8 @@ class AicpuChecker(OperatorChecker): if not os.path.exists(rule_path): logger.warning("Skip analyze aicpu issues, because %s does not exist.", rule_path) return {} - with open(rule_path, 'r') as f: - self.aicpu_rules = yaml.safe_load(f) + + self.aicpu_rules = FileManager.read_yaml_file(rule_path) self.filter_aicpu_rules(self.aicpu_rules) for checker_name, check_rule in self.aicpu_rules.items(): if not isinstance(check_rule, (list, dict,)): diff --git a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py index a7d7ddd93..7a873c656 100644 --- a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py +++ b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py @@ -1,5 +1,4 @@ import logging - from typing import List from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker diff --git a/profiler/advisor/analyzer/computation/profiling_analyzer.py b/profiler/advisor/analyzer/computation/profiling_analyzer.py index 868261770..2021bcd57 100644 --- a/profiler/advisor/analyzer/computation/profiling_analyzer.py +++ b/profiler/advisor/analyzer/computation/profiling_analyzer.py @@ -1,19 +1,15 @@ import logging from abc import ABC -from typing import Dict, List from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common import constant from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.computation.aicpu.aicpu_checker import AicpuChecker from profiler.advisor.analyzer.computation.bound.block_dim_checker import BlockDimChecker from profiler.advisor.analyzer.computation.bound.operator_bound_checker import OperatorBoundChecker -from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker from profiler.advisor.analyzer.computation.op_compile.dynamic_shape_checker import DynamicShapeChecker from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.utils.utils import get_supported_subclass logger = logging.getLogger() @@ -76,14 +72,15 @@ class BlockDimAnalyzer(ProfilingAnalyzer): def __init__(self, collection_path, **kwargs) -> None: super().__init__(collection_path, **kwargs) self.checker = BlockDimChecker(self.cann_version) - + class OperatorBoundAnalyzer(ProfilingAnalyzer): def __init__(self, collection_path, **kwargs) -> None: super().__init__(collection_path, **kwargs) self.checker = OperatorBoundChecker(self.cann_version) + class AicpuAnalyzer(ProfilingAnalyzer): def __init__(self, collection_path, **kwargs) -> None: super().__init__(collection_path, **kwargs) - self.checker = AicpuChecker(self.cann_version) \ No newline at end of file + self.checker = AicpuChecker(self.cann_version) diff --git a/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py b/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py new file mode 100644 index 000000000..291c3a1f9 --- /dev/null +++ b/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py @@ -0,0 +1,30 @@ +import logging + +from typing import List, Dict, Any + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset + +logger = logging.getLogger() + + +class DataloaderAnalyzer(BaseAnalyzer): + dataset_cls_list = [TimelineEventDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = TimelineEventDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + + @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + def optimize(self, **kwargs): + dataloader_checker = DataloaderChecker() + dataloader_checker.check_slow_dataloader(self.dataset) + dataloader_checker.make_record(self.result) + dataloader_checker.make_render(self.html_render) + return self.result diff --git a/profiler/advisor/analyzer/dataloader/dataloader_checker.py b/profiler/advisor/analyzer/dataloader/dataloader_checker.py new file mode 100644 index 000000000..eb1886284 --- /dev/null +++ b/profiler/advisor/analyzer/dataloader/dataloader_checker.py @@ -0,0 +1,84 @@ +import os +import re +import logging +import yaml + +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager + +logger = logging.getLogger() + + +class DataloaderChecker: + + def __init__(self): + + self.dataloader_issues = False + self.optimization_item = [] + self.desc = "" + self.suggestions = [] + self.dataloader_duration_threshold = None + self._init_rule() + + def check_slow_dataloader(self, event_dataset: TimelineEventDataset): + """ + :Param event_dataset: dataset of timeline event + """ + if not hasattr(event_dataset, "dataloader") or not getattr(event_dataset, "dataloader"): + logger.debug("Skip slow dataloader checker, because no dataloader duration larger than %s", + self.dataloader_duration_threshold) + return + for event in event_dataset.dataloader: + + dataloader_duration = float(event.dur) / 1000 + if dataloader_duration < self.dataloader_duration_threshold: + continue + self.desc = self.desc.format(dataloader_duration=dataloader_duration, + dataloader_duration_threshold=self.dataloader_duration_threshold) + self.dataloader_issues = True + + if re.search("singleprocess", event.name.lower()): + self.suggestions = self._reset_suggestions(["I/O", "num_workers"]) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + if not self.dataloader_issues: + return + + self.optimization_item.append(OptimizeItem("Slow dataloader", self.desc, self.suggestions)) + for optimization in self.optimization_item: + result.add(OptimizeRecord(optimization)) + + def make_render(self, html_render): + if not self.dataloader_issues: + return + html_render.render_template(key="dataloader", + template_dir="templates", + template_name="slow_dataloader.html", + desc=self.desc, + suggestions=self.suggestions) + + def _init_rule(self): + dataloader_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "dataloader.yaml" + ) + dataloader_rule = FileManager.read_yaml_file(dataloader_rule_path) + + self.dataloader_duration_threshold = dataloader_rule.get("dataloader_duration_threshold") + self.desc = dataloader_rule.get("problem") + self.suggestions = dataloader_rule.get("solutions") + + def _reset_suggestions(self, suggestion_pattern_list): + + suggestions = [] + for solution in self.suggestions: + for suggestion_pattern in suggestion_pattern_list: + if re.search(suggestion_pattern, solution): + suggestions.append(solution) + return suggestions diff --git a/profiler/advisor/analyzer/graph_fusion/graph_fusion_checker.py b/profiler/advisor/analyzer/graph_fusion/graph_fusion_checker.py index e64020fdf..30bd43237 100644 --- a/profiler/advisor/analyzer/graph_fusion/graph_fusion_checker.py +++ b/profiler/advisor/analyzer/graph_fusion/graph_fusion_checker.py @@ -149,7 +149,7 @@ class GraphFusionRules: optimization_item = OptimizeItem( "fusion issue", f"Found {len(self.candidates)} fusion issues", - ["Check fusion issues detail in att_advisor*.html"] + ["Check fusion issues detail in mstt_advisor*.html"] ) total_time = 0.0 for candidate in self.task_duration_list: diff --git a/profiler/advisor/analyzer/overall/overall_analyzer.py b/profiler/advisor/analyzer/overall/overall_analyzer.py deleted file mode 100644 index 916a396b3..000000000 --- a/profiler/advisor/analyzer/overall/overall_analyzer.py +++ /dev/null @@ -1,45 +0,0 @@ -import logging -from typing import Dict, List - -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.result.result import OptimizeResult -from profiler.compare_tools.compare_backend.utils.constant import Constant -from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface - -logger = logging.getLogger() - - -class OverallSummaryAnalyzer(BaseAnalyzer): - - def __init__(self, profiling_path, benchmark_profiling_path=None, **kwargs): - self.benchmark_profiling_path = benchmark_profiling_path or profiling_path - self.profiling_path = profiling_path - self.html_render = HTMLRender() - self.result = OptimizeResult() - - def optimize(self, **kwargs): - compare_result = ComparisonInterface(self.benchmark_profiling_path, self.profiling_path).compare( - Constant.OVERALL_COMPARE) - - headers = compare_result.get('Model Profiling Time Distribution').get("headers", []) - rows = compare_result.get('Model Profiling Time Distribution').get("rows", []) - - self.make_record() - self.make_render(headers=headers, rows=rows) - return compare_result - - def make_record(self): - pass - - def make_render(self, **kwargs): - headers = kwargs.get("headers") - rows = kwargs.get("rows") - - if not headers or not rows: - logger.info("Empty headers or rows, skip render overall analysis html") - self.html_render.render_template(key="overall", - template_dir="templates", - template_name="overall_analysis.html", - headers=kwargs.get("headers"), - rows=kwargs.get("rows")) diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py index c74ae0510..8e93dbda7 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py @@ -13,27 +13,21 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import copy - -import logging -from typing import Dict, List +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult from profiler.compare_tools.compare_backend.utils.constant import Constant -from profiler.advisor.common import constant as const from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface -from profiler.advisor.utils.utils import get_file_path_from_directory, load_parameter class OverallSummaryAnalyzer(BaseAnalyzer): OVERALL_SUMMARY_ANALYZER = "overall_summary_analysis" advice_map = { - "Computing Time": "if you want more detailed advice please go to att_advisor_*.html", - "Uncovered Communication Time": "if you want more detailed advice please go to att_advisor_*.html", - "Free Time": "if you want more detailed advice please go to att_advisor_*.html" + "Computing Time": "if you want more detailed advice please go to mstt_advisor_*.html", + "Uncovered Communication Time": "if you want more detailed advice please go to mstt_advisor_*.html", + "Free Time": "if you want more detailed advice please go to mstt_advisor_*.html" } time_name_map = { "Computing Time": "computing", @@ -47,45 +41,37 @@ class OverallSummaryAnalyzer(BaseAnalyzer): 'SDMA Time(Num)': 'SDMA Time' } performance_time_dict = { - "Computing Time": ['Cube Time(Num)', 'Vector Time(Num)', 'Flash Attention Time(Forward)(Num)', - 'Flash Attention Time(Backward)(Num)', 'Other Time'], - "Uncovered Communication Time(Wait Time)": [], - "Free Time": ['SDMA Time(Num)'] + "Computing Time": "computing_time_ms", + " -- Flash Attention": "fa_time_ms", + " -- Conv": "conv_time_ms", + " -- Matmul": "matmul_time_ms", + " -- Vector": "vector_time_ms", + " -- SDMA(Tensor Move)": "tensor_move_time_ms", + " -- Other Cube": "other_cube_time_ms", + "Uncovered Communication Time": "uncovered_communication_time_ms", + " -- Wait": "wait_time_ms", + " -- Transmit": "transmit_time_ms", + "Free Time": "free_time_ms", + " -- SDMA": "sdma_time_ms", + " -- Free": "free_ms", + "E2E Time": "e2e_time_ms" } def __init__(self, collection_path: str, n_processes: int = 1, **kwargs): profile_path = get_profile_path(collection_path) super().__init__(profile_path, n_processes, **kwargs) - self.base_collection_path = kwargs.get("base_collection_path", "") - self._has_base_collection = False + self.benchmark_profiling_path = kwargs.get("benchmark_profiling_path", "") + self._has_benchmark_profiling = False self._is_minimal_profiling = False self.cur_data = {} - self.cur_data_table = {} self.cur_bottleneck = {} + self._disaggregate_perf = {} + self._disaggregate_benchmark_perf = {} self.cur_advices = "" - self._headers = [] - self._base_data = [] - self._comparison_data = [] self.html_render = HTMLRender() self.result = OptimizeResult() self.bottleneck_str = "" - self.bottleneck_table = {} - - @staticmethod - def split_duration_and_num(time_value: str) -> tuple: - split_data = time_value.split("s") # time value example: 0.229s(1756) - duration, num = 0.0, None - if len(split_data) >= 2: - try: - num = int(split_data[1].strip("()")) - except ValueError: - pass - if len(split_data) >= 1: - try: - duration = float(split_data[0]) - except ValueError: - print(f"[WARNING] Invalid time value: {time_value}.") - return duration, num + self.over_summary_analysis = {} @staticmethod def calculate_ratio(dividend, divisor): @@ -93,131 +79,121 @@ class OverallSummaryAnalyzer(BaseAnalyzer): return float("inf") return dividend / divisor + @staticmethod + def get_time_category_dict(overall_dict: dict): + time_category_dict = { + "Computing Time": round(overall_dict.get('computing_time_ms', 0.0), 3), + "Uncovered Communication Time": round(overall_dict.get('uncovered_communication_time_ms', 0.0), 3), + "Free Time": round(overall_dict.get('free_time_ms', 0.0), 3) + } + return time_category_dict + def path_check(self): - if self.base_collection_path: - if os.path.exists(self.base_collection_path): - self._has_base_collection = True + if self.benchmark_profiling_path: + if os.path.exists(self.benchmark_profiling_path): + self._has_benchmark_profiling = True else: - print(f"[WARNING] Invalid path which not exists: {self.base_collection_path}.") + print(f"[WARNING] Invalid path which not exists: {self.benchmark_profiling_path}.") return os.path.exists(self.collection_path) def process(self): - base_collection_path = self.base_collection_path if self._has_base_collection else self.collection_path - result_data = ComparisonInterface(base_collection_path, self.collection_path).compare(Constant.OVERALL_COMPARE) - for data in result_data.values(): - self._headers = data.get("headers", []) - rows = data.get("rows", []) - if len(rows) == 2: - self._base_data = rows[0] - self._comparison_data = rows[1] - if not self._headers or not self._comparison_data: + self._disaggregate_perf = ComparisonInterface(self.collection_path).disaggregate_perf(Constant.OVERALL_COMPARE) + if not self._disaggregate_perf: return - self._is_minimal_profiling = 'E2E Time(Not minimal profiling)' not in self._headers - if self._has_base_collection: - self.cur_data["comparison_result"] = result_data - time_category_dict = {} - for time_category, time_list in self.performance_time_dict.items(): - time_value = self.get_time_value(time_category, self._comparison_data) - if time_value == Constant.INVALID_VALUE: - continue - duration, _ = self.split_duration_and_num(time_value) - time_category = time_category.split("(")[0] - time_category_dict[time_category] = duration - self.get_sub_category_time(time_category, time_list, duration) - self.cur_data["overall_data"] = time_category_dict - - def get_time_value(self, header_name: str, data_list: list): - try: - data_index = self._headers.index(header_name) - except ValueError: - return Constant.INVALID_VALUE - try: - time_value = data_list[data_index] - except IndexError: - return Constant.INVALID_VALUE - return time_value - - def get_sub_category_time(self, category: str, time_list: list, total_duration: float): - sub_time_dict = {} - for time_name in time_list: - time_value = self.get_time_value(time_name, self._comparison_data) - if time_value == Constant.INVALID_VALUE: - continue - sub_time_dict.setdefault(f"{category} Subtype", []).append(self.time_name_map.get(time_name, "")) - duration, num = self.split_duration_and_num(time_value) - sub_time_dict.setdefault(f"Duration(s)", []).append(duration) - sub_time_dict.setdefault(f"Duration Ratio", []).append( - "{:.2%}".format(self.calculate_ratio(duration, total_duration))) - sub_time_dict.setdefault(f"Kernel Number", []).append(num) - self.cur_data[self.time_name_map.get(category)] = sub_time_dict + self._is_minimal_profiling = self._disaggregate_perf.get("minimal_profiling", False) + self.cur_data["overall_data"] = self.get_time_category_dict(self._disaggregate_perf.get('overall', {})) + if self._has_benchmark_profiling: + self._disaggregate_benchmark_perf = ComparisonInterface( + self.benchmark_profiling_path).disaggregate_perf(Constant.OVERALL_COMPARE) def identify_bottleneck(self): overall_data = self.cur_data.get("overall_data") if not overall_data: return e2e_time = '%.3f' % sum([data for data in overall_data.values()]) - overall_bottleneck = f"The Model E2E Time is {e2e_time}s.\n" + overall_bottleneck = f"The Model E2E Time is {e2e_time}ms.\n" comparison_bottleneck = "" for time_type, time_value in overall_data.items(): - # add subtype time bottleneck - self.cur_bottleneck[self.time_name_map.get(time_type)] = f"{time_type} is {time_value}s.\n" # add overall bottleneck - overall_bottleneck += f" -- {time_type} is {time_value}s\n" + overall_bottleneck += f" -- {time_type} is {time_value}ms\n" if time_type == "Free Time" and self._is_minimal_profiling and self.calculate_ratio(time_value, e2e_time) > 0.1: overall_bottleneck += "percentage of free time exceed the threshold 10%." - if not self._has_base_collection: + if not self._has_benchmark_profiling: continue # add comparison bottleneck - time_type_origin = "Uncovered Communication Time(Wait Time)" \ - if time_type == "Uncovered Communication Time" else time_type - base_duration, _ = self.split_duration_and_num(self.get_time_value(time_type_origin, self._base_data)) + base_duration = self.get_time_category_dict( + self._disaggregate_benchmark_perf.get('overall', {}) + ).get(time_type) if time_value > base_duration: ratio = "{:.2%}".format(self.calculate_ratio(time_value - base_duration, base_duration)) comparison_bottleneck += f"{time_type} exceeds the benchmark by {ratio}\n" self.cur_bottleneck["overall_data"] = overall_bottleneck if comparison_bottleneck: self.cur_bottleneck["comparison_result"] = comparison_bottleneck + def optimize(self, **kwargs): if self.path_check(): self.process() self.identify_bottleneck() self.format_bottleneck() - self.format_cur_data() + self.format_over_summary_analysis() self.make_record() self.make_render() return self.result def format_bottleneck(self): result = '' - headers = [] - data_list = [] - data = [] - for key, value in self.cur_bottleneck.items(): + for _, value in self.cur_bottleneck.items(): if not value: continue - result += f'{key}: {value} \n' - headers.append(key) - data.append(value) - data_list.append(data) + result += f'{value} \n' self.bottleneck_str = result - self.bottleneck_table["headers"] = headers - self.bottleneck_table["data"] = data_list - def format_cur_data(self): - if not self.cur_data: - return - for data_type, data in self.cur_data.items(): - if not data: - continue - if data_type not in list(self.time_name_map.values()): - data_list = list(data.values()) - else: - data_list = [','.join(map(str, value)) for value in data.values()] - headers = list(data.keys()) - data_table = {"headers": headers, "data": [data_list]} - self.cur_data_table[data_type] = copy.deepcopy(data_table) + def format_over_summary_analysis(self): + headers = ['Performance Index', 'Duration(ms)', 'Duration Ratio'] + performance_data = self.get_analysis_data(self._disaggregate_perf) + benchmark_data = self.get_analysis_data(self._disaggregate_benchmark_perf) + if self._has_benchmark_profiling: + headers.append('Diff Duration(ms)') + self.format_analysis_with_benchmark(performance_data, benchmark_data, headers) + else: + self.format_analysis_only(performance_data, headers) + + def get_analysis_data(self, data_dict: dict): + if not data_dict: + return {} + return { + **data_dict.get("overall"), + **data_dict.get("computing_time_disaggregate"), + **data_dict.get("communication_time_disaggregate"), + **data_dict.get("free_time_disaggregate"), + } + def format_analysis_only(self, performance_data: dict, headers: list): + res = [] + total_duration = performance_data.get('e2e_time_ms', 0.0) + for time_name, time_key in self.performance_time_dict.items(): + row = [time_name] + duration = performance_data.get(time_key, 0.0) + row.append("{:.3f}".format(duration)) + row.append("{:.2%}".format(self.calculate_ratio(duration, total_duration))) + res.append(row) + self.over_summary_analysis["headers"] = headers + self.over_summary_analysis["data"] = res + + def format_analysis_with_benchmark(self, performance_data: dict, benchmark_data: dict, headers: list): + res = [] + total_duration = performance_data.get('e2e_time_ms', 0.0) + for time_name, time_key in self.performance_time_dict.items(): + row = [time_name] + duration = performance_data.get(time_key, 0.0) + row.append("{:.3f}".format(duration)) + row.append("{:.2%}".format(self.calculate_ratio(duration, total_duration))) + row.append("{:.3f}".format(duration - benchmark_data.get(time_key, 0.0))) + res.append(row) + self.over_summary_analysis["headers"] = headers + self.over_summary_analysis["data"] = res def make_record(self): """ @@ -232,20 +208,23 @@ class OverallSummaryAnalyzer(BaseAnalyzer): ) self.result.add(OptimizeRecord(optimization_item)) - self.result.add_detail(const.BOTTLENECK, self.bottleneck_table["headers"], self.bottleneck_table["data"][0]) - for data_type, data_dict in self.cur_data_table.items(): - if data_dict: - self.result.add_detail(const.DATA + data_type, data_dict["headers"], data_dict["data"][0]) + self.result.add_detail( + OverallSummaryAnalyzer.OVERALL_SUMMARY_ANALYZER, + headers=self.over_summary_analysis["headers"] + ) + for data in self.over_summary_analysis["data"]: + self.result.add_detail(OverallSummaryAnalyzer.OVERALL_SUMMARY_ANALYZER, detail=data) def make_render(self): if not self.bottleneck_str and not self.cur_advices: return + # 将\n替换为html换行 + bottleneck_str = self.bottleneck_str.replace('\n', '
') result_for_html = { - "Description" : self.bottleneck_str, - "suggestion" : self.cur_advices, - "details" : [self.bottleneck_table] + "Description": bottleneck_str, + "suggestion": self.cur_advices, + "details": [self.over_summary_analysis] } - self.html_render.render_template(key="overall", title=OverallSummaryAnalyzer.OVERALL_SUMMARY_ANALYZER, template_dir="templates", @@ -254,9 +233,10 @@ class OverallSummaryAnalyzer(BaseAnalyzer): torch_version=self.torch_version, result=result_for_html) + def get_profile_path(collection_path): for root, dirs, files in os.walk(collection_path): for file in files: if file.startswith("profiler_info"): return root - return "" \ No newline at end of file + return "" diff --git a/profiler/advisor/analyzer/schedule/syncbn/__init__.py b/profiler/advisor/analyzer/schedule/syncbn/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py new file mode 100644 index 000000000..2786a7840 --- /dev/null +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py @@ -0,0 +1,30 @@ +import logging + +from typing import List, Dict, Any + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset + +logger = logging.getLogger() + + +class SyncBNAnalyzer(BaseAnalyzer): + dataset_cls_list = [TimelineEventDataset] + + def __init__(self, collection_path, **kwargs): + super().__init__(collection_path, **kwargs) + self.result = OptimizeResult() + self.html_render = HTMLRender() + key = TimelineEventDataset.get_key() + self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) + + @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + def optimize(self, **kwargs): + syncbn_checker = SyncBNChecker() + syncbn_checker.check_syncbn(self.timeline_event_dataset) + syncbn_checker.make_record(self.result) + syncbn_checker.make_render(self.html_render) + return self.result diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py new file mode 100644 index 000000000..c0e10448f --- /dev/null +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py @@ -0,0 +1,70 @@ +import logging +import os + +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager + +logger = logging.getLogger() + + +class SyncBNChecker: + + def __init__(self): + self.optimization_item = [] + self.syncbn_issues = False + self.desc = "" + self.suggestions = [] + self.solutions = None + self.max_syncbn_num = None + self._init_rule() + + def check_syncbn(self, event_dataset: TimelineEventDataset): + """ + :Param event_dataset: dataset of timeline event + """ + if not hasattr(event_dataset, "sync_batchnorm") or not getattr(event_dataset, "sync_batchnorm"): + logger.debug("Skip syncbn checker, because no syncbn found") + return + + syncbn_num = len(event_dataset.sync_batchnorm) + self.syncbn_issues = syncbn_num >= self.max_syncbn_num + self.desc = self.desc.format(syncbn_num=syncbn_num) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + if not self.syncbn_issues: + return + + self.optimization_item.append(OptimizeItem("SyncBatchNorm", self.desc, self.suggestions)) + for optimization in self.optimization_item: + result.add(OptimizeRecord(optimization)) + + def make_render(self, html_render): + if not self.syncbn_issues: + return + html_render.render_template(key="schedule", + template_dir="templates", + template_name="sync_batchnorm.html", + desc=self.desc, + solutions=self.solutions) + + def _init_rule(self): + syncbn_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))), + "rules", + "sync_batchnorm.yaml" + ) + + syncbn_rule = FileManager.read_yaml_file(syncbn_rule_path) + + self.max_syncbn_num = syncbn_rule.get("max_syncbn_num") + self.desc = syncbn_rule.get("problem") + + self.solutions = syncbn_rule.get("solutions") + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/__init__.py b/profiler/advisor/analyzer/schedule/synchronize_stream/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py new file mode 100644 index 000000000..d8906504c --- /dev/null +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py @@ -0,0 +1,32 @@ +import logging + +from typing import List, Dict, Any + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset + +logger = logging.getLogger() + + +class SynchronizeStreamAnalyzer(BaseAnalyzer): + dataset_cls_list = [TimelineEventDataset] + + def __init__(self, collection_path, **kwargs): + super().__init__(collection_path, **kwargs) + self.result = OptimizeResult() + self.html_render = HTMLRender() + + key = TimelineEventDataset.get_key() + self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) + + @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + def optimize(self, **kwargs): + + synchronize_stream_checker = SynchronizeStreamChecker() + synchronize_stream_checker.check_synchronize(self.timeline_event_dataset, kwargs.get("profiling_with_stack")) + synchronize_stream_checker.make_record(self.result) + synchronize_stream_checker.make_render(self.html_render) + return self.result diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py new file mode 100644 index 000000000..83ddd80a0 --- /dev/null +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py @@ -0,0 +1,89 @@ +import logging + +from profiler.advisor.common import constant as const +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.analyzer.schedule.timeline_base_checker import TimelineBaseChecker +from profiler.advisor.utils.utils import format_timeline_result + +logger = logging.getLogger() + + +class SynchronizeStreamChecker(TimelineBaseChecker): + + def __init__(self): + super().__init__(n_processes=1) + self.optimization_item = [] + self.synchronize_issues = False + self.desc = "" + self.suggestions = [] + self.solutions = [] + self.max_synchronize_num = None + + def check_synchronize(self, event_dataset: TimelineEventDataset, profiling_with_stack=None): + """ + :Param event_dataset: dataset of timeline event + """ + if not hasattr(event_dataset, "synchronize_stream") or not getattr(event_dataset, "synchronize_stream"): + logger.debug("Skip synchronize stream checker, because no synchronize stream found") + return + + synchronize_num = event_dataset.synchronize_stream.total_count + slow_synchronize_stream = event_dataset.synchronize_stream.slow_synchronize_stream + total_slow_synchronize_time = sum((float(sync_stream.dur) for sync_stream in slow_synchronize_stream)) + + synchronize_stream_rule = event_dataset.synchronize_stream.rule + self.max_synchronize_num = synchronize_stream_rule.get("max_synchronize_num") + self.synchronize_issues = synchronize_num >= self.max_synchronize_num and len(slow_synchronize_stream) > 0 + if not self.synchronize_issues: + return + + for sync_stream in slow_synchronize_stream: + if sync_stream.name not in self._matched_op_index: + self._matched_op_index[sync_stream.name] = [] + self._matched_op_index[sync_stream.name].append(sync_stream.dataset_index) + self.query_stack(event_dataset, profiling_with_stack) + + self.desc = synchronize_stream_rule.get("problem") + self.desc = self.desc.format(synchronize_num=synchronize_num, + slow_synchronize_num=len(slow_synchronize_stream), + total_synchronize_stream_time=total_slow_synchronize_time) + + solutions = synchronize_stream_rule.get("solutions") + for solution in solutions: + renderer_solution = {} + for key, val in solution.items(): + if self.empty_stacks and self.framework_black_list: + # 如果堆栈源于torch, torch_npu等框架，则不提示修改的代码 + if "modify code" in key.lower(): + continue + self.suggestions.append(f"{key}, {val.get('desc')}") + renderer_solution.update({key: val}) + self.solutions.append(renderer_solution) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + if not self.synchronize_issues: + return + + self.optimization_item.append(OptimizeItem("SynchronizeStream", self.desc, self.suggestions)) + for optimization in self.optimization_item: + result.add(OptimizeRecord(optimization)) + + def make_render(self, html_render): + if not self.synchronize_issues: + return + + format_result_for_html = format_timeline_result(dict(self.matched_op_stacks), dump_html=True) + html_render.render_template(key="schedule", + template_dir="templates", + template_name="synchronize_stream.html", + desc=self.desc, + solutions=self.solutions, + result=format_result_for_html, + with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, + empty_stacks=self.empty_stacks, + framework_black_list=self.framework_black_list) diff --git a/profiler/advisor/analyzer/schedule/timeline_base_checker.py b/profiler/advisor/analyzer/schedule/timeline_base_checker.py new file mode 100644 index 000000000..8bc691502 --- /dev/null +++ b/profiler/advisor/analyzer/schedule/timeline_base_checker.py @@ -0,0 +1,91 @@ +from abc import ABC, abstractmethod +import multiprocessing +import logging + +from profiler.advisor.common import constant as const +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.result.result import OptimizeResult + +logger = logging.getLogger() + + +class TimelineBaseChecker(ABC): + + def __init__(self, n_processes: int = 1): + self.n_processes = n_processes + self._matched_op_index = {} if self.n_processes <= 1 else multiprocessing.Manager().dict() + self.matched_op_stacks = {} + self.empty_stacks = True + self.framework_black_list = False + + @abstractmethod + def make_record(self, result: OptimizeResult): + pass + + @abstractmethod + def make_render(self, html_render): + pass + + def query_stack(self, event_dataset: TimelineEventDataset = None, profiling_with_stack: str = None): + if all([len(matched_index) == 0 for matched_index in self._matched_op_index.values()]): + return + + event_dataset = event_dataset if not profiling_with_stack else TimelineEventDataset( + collection_path=profiling_with_stack, data={}, _datasets={}, analysis_mode="fusion_ops", + build_dataset=False) + + op_stack_list = event_dataset.parse_data_with_generator(self._query_stack_by_matched_index) + for op_stack in op_stack_list: + for op, stack in op_stack.items(): + if op not in self.matched_op_stacks: + self.matched_op_stacks[op] = {} + if stack == const.TIMELINE_FUSION_OPS_NO_STACK_FLAG: + continue + if stack not in self.matched_op_stacks[op]: + self.matched_op_stacks[op][stack] = 0 + self.matched_op_stacks[op][stack] += 1 + + def _query_stack_by_matched_index(self, index, event): + stack_record = {} + event = TimelineEvent(event) + + matched_ops = [] + for op, matched_index in self._matched_op_index.items(): + if index not in matched_index: + continue + + matched_ops.append(op) + stack = event.args.get(const.CALL_STACKS) + + if not stack: + logger.debug("Got empty '%s' for event %s", const.CALL_STACKS, event) + continue + + if not self._is_keep_stack(stack): + self.framework_black_list = True + logger.debug("Drop stack from framework %s", const.FRAMEWORK_STACK_BLACK_LIST) + continue + + if self.empty_stacks and stack: + self.empty_stacks = False + + stack_record[op] = stack + + if matched_ops and not stack_record: + for op in matched_ops: + stack_record[op] = const.TIMELINE_FUSION_OPS_NO_STACK_FLAG + + return stack_record + + def _is_keep_stack(self, stack): + # 过滤掉torch, torch_npu, megatron, deepspeed等框架下的堆栈，这些源码基本是不能被修改的 + stack_list = stack.replace("\\r\\n", ";").split(";") + if not stack_list: + return False + + final_called_stack = stack_list[0] + for framework in const.FRAMEWORK_STACK_BLACK_LIST: + if framework in final_called_stack.split("/"): + return False + return True diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 592f9d421..52e3e0755 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -12,3 +12,7 @@ class SupportedScopes: BLOCK_DIM_ANALYSIS = "block_dim_analysis" OPERATOR_NO_BOUND_ANALYSIS = "operator_no_bound_analysis" TIMELINE_OP_DISPATCH = "timeline_op_dispatch" + DATALOADER = "dataloader" + SYNCBN = "syncbn" + SYNCHRONIZE_STREAM = "synchronize_stream" + FREQ_ANALYSIS = "freq_analysis" diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index 697430ee6..87245a43e 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -26,6 +26,7 @@ ENQUEUE = "enqueue" TORCH_TO_NPU = "torch_to_npu" OP_COMPILE_NAME = "AscendCL@aclopCompileAndExecute" OP_COMPILE_ID = "aclopCompileAndExecute" +SYNC_STREAM = "AscendCL@aclrtSynchronizeStream" MAX_OP_COMPILE_NUM = 20 ACL_TO_NPU = "acl_to_npu" TASK_TYPE = "Task Type" @@ -111,7 +112,7 @@ HTTP_PREFIXES = "http://" HTTPS_PREFIXES = "https://" COMMON_YAML_DIR = "modelarts/solution/ma_advisor_rules/" COMMON_ENDPOINT_SUFFIX = "obs.{}.myhuaweicloud.com" -INNER_ENDPOINT_SUFFIX= "obs.{}.ulanqab.huawei.com" +INNER_ENDPOINT_SUFFIX = "obs.{}.ulanqab.huawei.com" AICPU_RULES_YAML_NAME = "aicpu_rules.yaml" FUSION_PASS_YAML_NAME = "op_fusion_pass.yaml" @@ -138,4 +139,8 @@ CLUSTER_STEP_TIME_CSV = "cluster_step_trace_time.csv" CLUSTER_COMM_JSON = "cluster_communication.json" BOTTLENECK = "bottleneck" -DATA = "data" \ No newline at end of file +DATA = "data" + +FRAMEWORK_STACK_BLACK_LIST = ["torch", "torch_npu", "megatron", "deepspeed"] +DISABLE_STREAMING_READER = "DISABLE_STREAMING_READER" +MAX_FILE_SIZE = 10**10 diff --git a/profiler/advisor/common/graph/graph_parser.py b/profiler/advisor/common/graph/graph_parser.py index d4c67fc19..ef4dc4d68 100644 --- a/profiler/advisor/common/graph/graph_parser.py +++ b/profiler/advisor/common/graph/graph_parser.py @@ -1,11 +1,12 @@ import os import logging -import yaml import itertools from collections import deque from dataclasses import dataclass from typing import List, Tuple, Dict +from profiler.cluster_analyse.common_func.file_manager import FileManager + logger = logging.getLogger() @@ -344,9 +345,9 @@ class QueryGraphParser: if not os.path.exists(rule_database): raise FileNotFoundError(f"Path {rule_database} does not exist.") - with open(rule_database, 'r') as f: - database = yaml.safe_load(f) - self.parse_yaml(database) + + database = FileManager.read_yaml_file(rule_database) + self.parse_yaml(database) def parse_yaml(self, yaml_database): fusion_strategy_list = yaml_database.get("GraphFusion", []) diff --git a/profiler/advisor/common/timeline/event.py b/profiler/advisor/common/timeline/event.py index 6001ac887..e24d983a0 100644 --- a/profiler/advisor/common/timeline/event.py +++ b/profiler/advisor/common/timeline/event.py @@ -1,3 +1,4 @@ +from decimal import Decimal class AdvisorDict(dict): def __getstate__(self): return self.__dict__ @@ -18,6 +19,6 @@ class AdvisorDict(dict): class TimelineEvent(AdvisorDict): def ts_include(self, event): - - return float(self.ts) <= float(event.ts) and float(self.ts) + float(self.dur) >= float(event.ts) + float( + return Decimal(self.ts) <= Decimal(event.ts) and Decimal(self.ts) + Decimal(self.dur) >= Decimal( + event.ts) + Decimal( event.dur) \ No newline at end of file diff --git a/profiler/advisor/common/timeline/fusion_ops_db.py b/profiler/advisor/common/timeline/fusion_ops_db.py index 8637befd1..64cc84929 100644 --- a/profiler/advisor/common/timeline/fusion_ops_db.py +++ b/profiler/advisor/common/timeline/fusion_ops_db.py @@ -1,13 +1,12 @@ import logging import os -import yaml - from profiler.advisor.common import constant from profiler.advisor.common.timeline.fusion_ops_rule import OpRule from profiler.advisor.common.timeline.fusion_ops_rule_handler import TimelineOpRuleHandler from profiler.advisor.utils.log import get_log_level from profiler.advisor.utils.utils import get_file_path_by_walk +from profiler.cluster_analyse.common_func.file_manager import FileManager logger = logging.getLogger() logger.setLevel(get_log_level()) @@ -241,8 +240,7 @@ class FusionOperatorDB: logger.debug("The rule yaml file is successfully found in path: %s", os.path.abspath(file_path)) - with open(file_path, "rb") as file: - db_content = yaml.safe_load(file) + db_content = FileManager.read_yaml_file(file_path) if not self._is_version_supported(db_content): self.is_empty = True diff --git a/profiler/advisor/config/config.ini b/profiler/advisor/config/config.ini index c56c1dad9..06e993160 100644 --- a/profiler/advisor/config/config.ini +++ b/profiler/advisor/config/config.ini @@ -9,6 +9,7 @@ tune_ops_file = operator_tuning_file.cfg [THRESHOLD] # operator_bound_ratio: (mte, cube, vector, scalar) ratio greater than this value will be checked in operator_bound_checker operator_bound_ratio = 0.8 +frequency_threshold = 0.05 [RULE-BUCKET] # region : URL of different regions where can download rule yaml file cn-north-9 = cnnorth9-modelarts-sdk diff --git a/profiler/advisor/config/config.py b/profiler/advisor/config/config.py index 12f4526f8..4f36dfedf 100644 --- a/profiler/advisor/config/config.py +++ b/profiler/advisor/config/config.py @@ -97,6 +97,13 @@ class Config: """ return float(self.config.get("THRESHOLD", "operator_bound_ratio")) + @property + def frequency_threshold(self) -> float: + """ + frequency_threshold + """ + return float(self.config.get("THRESHOLD", "frequency_threshold")) + def set_log_path(self, result_file: str, log_path: str = None): self.log_path = log_path if log_path is not None else os.path.join(self._work_path, "log") os.makedirs(self.log_path, exist_ok=True) diff --git a/profiler/advisor/dataset/ai_core_freq/__init__.py b/profiler/advisor/dataset/ai_core_freq/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py b/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py new file mode 100644 index 000000000..c99baea65 --- /dev/null +++ b/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py @@ -0,0 +1,148 @@ +import json +import logging +import math +import os +import traceback + +import ijson +from tqdm import tqdm + +from profiler.advisor.common import constant as const +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.advisor.utils.utils import get_file_path_from_directory +from profiler.advisor.utils.utils import convert_to_float, parse_json_with_generator +from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser +from profiler.advisor.config.config import Config + +logger = logging.getLogger() + + +class AICoreFreqDataset: + + def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: + + self._profiler_step = [] + self._ai_core_ops = [] + self._ai_core_freq: [TimelineEvent] = [] + self._previous_freq_index = -1 + + self.timeline_dir = collection_path + self.timeline_data_list = get_file_path_from_directory(collection_path, + lambda file: file.endswith("trace_view.json")) + + self.step = kwargs.get("step") + self.op_freq = {} + info = DeviceInfoParser(collection_path) + info.parse_data() + if not Config().get_config("aic_frequency"): + return + if self.parse(): + key = self.get_key() + if key not in data: + data[key] = [] + data[key].append(self) + + @property + def profiler_step(self): + return self._profiler_step + + @property + def ai_core_freq(self): + return self._ai_core_freq + + @property + def ai_core_ops(self): + return self._ai_core_ops + + @classmethod + def get_key(cls): + """ + get key of dataset + :return: key + """ + return cls.__module__.rsplit('.', maxsplit=1)[-1] + + def parse(self): + + if len(self.timeline_data_list) == 0: + logger.warning("Please ensure trace_view.json in %s, skip timeline analysis.", self.timeline_dir) + return False + + if len(self.timeline_data_list) > 1: + logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis .", + self.timeline_dir) + + _ = parse_json_with_generator(sorted(self.timeline_data_list)[0], self._add_event) + + target_ai_core_ops = self._get_target_ai_core_ops() + self._get_op_frequency(target_ai_core_ops) + return True + + def _add_profiler_step(self, event): + if event.name.startswith("ProfilerStep"): + self._profiler_step.append(event) + + def _add_ai_core_ops(self, event): + if event.args.get("Task Type") in ["MIX_AIC", "AI_CORE"]: + self._ai_core_ops.append(event) + + def _add_ai_core_freq(self, event): + if event.name == "AI Core Freq": + if self._previous_freq_index != -1: + self._ai_core_freq[self._previous_freq_index]["end"] = event.get("ts", float(math.inf)) + self._previous_freq_index += 1 + event.setdefault("end", float(math.inf)) + self._ai_core_freq.append(event) + + def _add_event(self, index, event): + event["dataset_index"] = index + if not isinstance(event, TimelineEvent): + event = TimelineEvent(event) + + self._add_profiler_step(event) + self._add_ai_core_ops(event) + self._add_ai_core_freq(event) + + return True + + def _get_target_ai_core_ops(self): + target_ai_core_ops = [] + if not self.step or f"ProfilerStep#{self.step}" not in [event.name for event in self._profiler_step]: + target_ai_core_ops = self._ai_core_ops + else: + for step_event in self._profiler_step: + if step_event.name != f"ProfilerStep#{self.step}": + continue + + for ai_core_op_event in self._ai_core_ops: + if step_event.ts_include(ai_core_op_event): + target_ai_core_ops.append(ai_core_op_event) + target_ai_core_ops = sorted(target_ai_core_ops, key=lambda x: float(x.ts)) + return target_ai_core_ops + + def _get_op_frequency(self, ai_core_ops): + ai_core_freq = sorted(self._ai_core_freq, key=lambda x: float(x.ts)) + + op_index, freq_index = 0, 0 + while op_index < len(ai_core_ops) and freq_index < len(ai_core_freq): + op_event = ai_core_ops[op_index] + op_end_time = convert_to_float(op_event.ts) + convert_to_float(op_event.dur) + op_freq_list = [] + while freq_index < len(ai_core_freq): + freq_event = ai_core_freq[freq_index] + if convert_to_float(freq_event.end) < op_end_time: + op_freq_list.append(convert_to_float(freq_event.args.MHz)) + freq_index += 1 + continue + elif convert_to_float(freq_event.ts) < op_end_time: + if op_event.name not in self.op_freq: + self.op_freq[op_event.name] = {"count": 0, "dur": 0, "freq_list": []} + self.op_freq[op_event.name]["count"] += 1 + self.op_freq[op_event.name]["dur"] += convert_to_float(op_event.dur) + op_freq_list.append(convert_to_float(freq_event.args.MHz)) + self.op_freq[op_event.name]["freq_list"].append(min(op_freq_list)) + break + else: + break + + op_index += 1 diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index 09fda2d4d..e1163f1cd 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -25,9 +25,9 @@ class ClusterDataset(Dataset): """ for file in os.listdir(self.collection_path): if file == 'cluster_analysis_output': - print("[INFO]Cluster has been analyzed " - "because of the existence of cluster analysis output directory.") - print("[INFO]Skip Cluster analyze backend.") + logger.info("[INFO]Cluster has been analyzed " + "because of the existence of cluster analysis output directory.") + logger.info("[INFO]Skip Cluster analyze backend.") return True return False @@ -62,7 +62,7 @@ class ClusterDataset(Dataset): @singleton -class ClusterStepTraceTimeDataSet(ClusterDataset): +class ClusterStepTraceTimeDataset(ClusterDataset): RANK = "rank" def __init__(self, collection_path: str, data: dict, **kwargs): @@ -77,10 +77,10 @@ class ClusterStepTraceTimeDataSet(ClusterDataset): print("捕获到异常：", e) self._step_dict = None return False - self._step_dict = self.formate_data(step_data) + self._step_dict = self.format_data(step_data) return True - def formate_data(self, step_data: list): + def format_data(self, step_data: list): step_dict = defaultdict(lambda: [0, 0, 0]) for step_bean in step_data: if step_bean.type == self.RANK: @@ -94,7 +94,7 @@ class ClusterStepTraceTimeDataSet(ClusterDataset): @singleton -class ClusterCommunicationDataSet(ClusterDataset): +class ClusterCommunicationDataset(ClusterDataset): RDMA_TIME_MS = "RDMA time(ms)" RDMA_SIZE_MB = "RDMA size(mb)" SDMA_TIME_MS = "SDMA time(ms)" diff --git a/profiler/advisor/dataset/profiling/device_info.py b/profiler/advisor/dataset/profiling/device_info.py index b58930777..110cd0794 100644 --- a/profiler/advisor/dataset/profiling/device_info.py +++ b/profiler/advisor/dataset/profiling/device_info.py @@ -54,6 +54,8 @@ class DeviceInfoParser: config.set_config("device_id", device_info["id"]) if "aiv_num" in device_info: config.set_config("aiv_num", device_info["aiv_num"]) + if "aic_frequency" in device_info: + config.set_config("aic_frequency", device_info["aic_frequency"]) if "ai_core_num" in device_info: config.set_config("ai_core_num", device_info["ai_core_num"]) return True diff --git a/profiler/advisor/dataset/profiling/profiling_dataset.py b/profiler/advisor/dataset/profiling/profiling_dataset.py index 46d4a4fe8..99a19d3b6 100644 --- a/profiler/advisor/dataset/profiling/profiling_dataset.py +++ b/profiler/advisor/dataset/profiling/profiling_dataset.py @@ -10,6 +10,7 @@ from profiler.advisor.common.profiling.tasktime import TaskTime from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser from profiler.advisor.utils.utils import join_prof_path +from profiler.cluster_analyse.common_func.file_manager import FileManager logger = logging.getLogger() @@ -69,8 +70,7 @@ class ProfilingDataset(Dataset): logger.warning("Skip parse profiling dataset, because %s does not exist.", config_path) return [] - with open(config_path, 'r') as f: - patterns = yaml.safe_load(f) + patterns = FileManager.read_yaml_file(config_path) return patterns diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/advisor/dataset/timeline_event_dataset.py index 94b6fdfef..1504e65f5 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/advisor/dataset/timeline_event_dataset.py @@ -1,14 +1,17 @@ +import json import logging -from typing import List +import os +from typing import List, Any +import traceback import ijson -from profiler.advisor.dataset.dataset import Dataset from tqdm import tqdm +import yaml from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.utils.utils import get_file_path_from_directory -from profiler.advisor.utils.utils import singleton +from profiler.advisor.utils.utils import get_file_path_from_directory, check_path_valid, singleton +from profiler.cluster_analyse.common_func.file_manager import FileManager logger = logging.getLogger() @@ -38,37 +41,76 @@ class OpCompileCollector: self._total_op_compile_time = 0.0 +class SynchronizeStreamCollector: + + def __init__(self): + self._synchronize_stream_count = 0 + self._slow_synchronize_stream = [] + self.rule = SynchronizeStreamCollector._load_rule() + + @property + def total_count(self): + return self._synchronize_stream_count + + @property + def slow_synchronize_stream(self): + return self._slow_synchronize_stream + + @staticmethod + def _load_rule(): + sync_stream_rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "rules", + "synchronize.yaml") + + sync_stream_rule = FileManager.read_yaml_file(sync_stream_rule_path) + return sync_stream_rule + + def update_sync_stream_count(self): + self._synchronize_stream_count += 1 + + def append_slow_sync_stream(self, event): + if float(event.dur) / 1000 >= self.rule.get("slow_synchronize_threshold", 10): + self._slow_synchronize_stream.append(event) + + def unset(self): + self._synchronize_stream_count = 0 + self._slow_synchronize_stream = [] + + @singleton -class TimelineEventDataset(Dataset): +class TimelineEventDataset: - def __init__(self, collection_path, data: dict, **kwargs) -> None: + def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: self._ops_with_task_type = {} self._ops_with_stack = {} self._ops_compile = OpCompileCollector() self._torch_to_npu = {} self._acl_to_npu = set() - self._aten: List[str] = [] - self._optimizer: List[str] = [] + self._aten: List[Any] = [] + self._optimizer: List[Any] = [] + self._dataloader: List[Any] = [] + self._sync_batchnorm: List[Any] = [] + self._synchronize_stream = SynchronizeStreamCollector() self.timeline_dir = collection_path - self.timeline_data_list = get_file_path_from_directory(collection_path, lambda file: file.endswith("trace_view.json")) + self.timeline_data_list = get_file_path_from_directory(collection_path, + lambda file: file.endswith("trace_view.json")) self.dataset_len = None self.analysis_mode = kwargs.get("analysis_mode") self.task_type = kwargs.get("task_type") - self.cann_version = kwargs.get("cann_version") - self.torch_version = kwargs.get("torch_version") - if self.analysis_mode in ["fusion_ops", "all"]: - logger.info("Load fusion operators database for cann version '%s' and torch version '%s'", - self.cann_version, self.torch_version) + if not build_dataset: + return - super().__init__(collection_path, data) + if self.parse(): + key = self.get_key() + if key not in data: + data[key] = [] + data[key].append(self) if self.analysis_mode in ["op_stack", "all"]: self._task_op_names = list(set([event_key.split("-")[0] for event_key in self._ops_with_task_type.keys()])) self._post_process() - @property def ops_with_stack(self): return self._ops_with_stack @@ -101,36 +143,60 @@ class TimelineEventDataset(Dataset): def aten(self): return self._aten - def _parse(self): + @property + def dataloader(self): + return self._dataloader + + @property + def sync_batchnorm(self): + return self._sync_batchnorm + + @property + def synchronize_stream(self): + return self._synchronize_stream + + @classmethod + def get_key(cls): + """ + get key of dataset + :return: key + """ + return cls.__module__.rsplit('.', maxsplit=1)[-1] + + def parse(self): if len(self.timeline_data_list) == 0: logger.warning("Please ensure trace_view.json in %s, skip timeline analysis.", self.timeline_dir) return False if len(self.timeline_data_list) > 1: - logger.warning("Please ensure only one trace_view.json in %s, there will analyze first timeline profiling data.", self.timeline_dir) - self.timeline_data_list = [self.timeline_data_list[0]] + logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis .", + self.timeline_dir) result = self.parse_data_with_generator(self._add_event) if not self.dataset_len: self.dataset_len = len(result) - return True def parse_data_with_generator(self, func): result = [] + timeline_data_path = sorted(self.timeline_data_list)[0] + if not check_path_valid(timeline_data_path): + return result + try: - with open(self.timeline_data_list[0], "r") as f: + with open(timeline_data_path, "r") as f: for i, event in tqdm(enumerate(ijson.items(f, "item")), leave=False, ncols=100, desc="Building dataset for timeline analysis", total=self.dataset_len): func_res = func(index=i, event=event) if func_res is not None: result.append(func_res) - except Exception as e: - logger.warning("Error %s while parsing file %s, continue to timeline analysis", e, - self.timeline_data_list[0]) + + except Exception: + logger.warning("Error %s while parsing file %s, continue to timeline analysis", traceback.format_exc(), + timeline_data_path) return result def _add_ops_with_task_type(self, event): @@ -168,12 +234,40 @@ class TimelineEventDataset(Dataset): "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur })) + def _add_dataloader(self, event: TimelineEvent): + if "dataloader" in event.name.lower(): + self._dataloader.append(TimelineEvent({ + "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur, + "stack": event.args.get("Call stack") + })) + + def _add_sync_batchnorm(self, event: TimelineEvent): + if event.name.lower() == "syncbatchnorm": + self._sync_batchnorm.append(TimelineEvent({ + "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur + })) + + def _add_synchronize(self, event: TimelineEvent): + if event.name.startswith(const.SYNC_STREAM): + self._synchronize.append(TimelineEvent({ + "name": event.name, "ts": event.ts, "dur": event.dur + })) + + def _add_specific_operator(self, event): + # for analysis of operator aclOpCompile, enable jit_compILE=False + self._add_op_compile(event) + # for analysis of slow dataloader.__next__ + self._add_dataloader(event) + # for analysis of syncBatchNorm operator, prompt users to replace source code of torch_npu's syncbn + self._add_sync_batchnorm(event) + def _add_event(self, index, event): event["dataset_index"] = index if not isinstance(event, TimelineEvent): event = TimelineEvent(event) - self._add_op_compile(event) + self._add_specific_operator(event) + if self.analysis_mode == "fusion_ops": self._add_event_for_fusion_ops(event) elif self.analysis_mode == "op_stack": @@ -189,6 +283,10 @@ class TimelineEventDataset(Dataset): self._add_aten(event) return + # 检查cann层同步操作，根据时间窗口索引到host侧的aten算子并给出堆栈 + if event.name.startswith(const.SYNC_STREAM): + self._add_aten(event) + if event.name.startswith(f"{const.OPTIMIZER}.{const.OPTIMIZER_STEP}{const.OPTIMIZER_SEP}"): self._add_optimizer(event) return @@ -214,7 +312,18 @@ class TimelineEventDataset(Dataset): # eliminate sub aten operator of the first level aten operator by 'ts' and 'dur', # keep the first level aten operator contiguous formated_atens = [] - for aten_event in sorted(self._aten, key=lambda x: x.get("ts", -1)): - if not formated_atens or not formated_atens[-1].ts_include(aten_event): - formated_atens.append(aten_event) + for event in sorted(self._aten, key=lambda x: x.get("ts", -1)): + if event.name.startswith(const.ATEN): + if not formated_atens or not formated_atens[-1].ts_include(event): + formated_atens.append(event) + + elif event.name.startswith(const.SYNC_STREAM): + self._synchronize_stream.update_sync_stream_count() + if formated_atens[-1].ts_include(event): + # 使用aten算子的索引，用于查询堆栈 + event["dataset_index"] = formated_atens[-1].get("dataset_index") + self._synchronize_stream.append_slow_sync_stream(event) + + else: + continue self._aten = formated_atens diff --git a/profiler/advisor/display/html/render.py b/profiler/advisor/display/html/render.py index 8ea7c9e0f..3984fa8f3 100644 --- a/profiler/advisor/display/html/render.py +++ b/profiler/advisor/display/html/render.py @@ -1,6 +1,7 @@ import os import logging from typing import List, Dict +from collections import defaultdict from jinja2 import Environment, FileSystemLoader from profiler.advisor.common import constant @@ -15,7 +16,7 @@ logger = logging.getLogger() class HTMLRender: def __init__(self): self.html = "" - self.render_list: Dict[str, List] = {} + self.render_list = defaultdict(list) def render_html(self, template_dir: str = "templates", template_name: str = "main.html", template_header=constant.DEFAULT_TEMPLATE_HEADER): @@ -30,8 +31,6 @@ class HTMLRender: autoescape=True) template = env.get_template(template_name) rendered_html = template.render(**kwargs) - if key not in self.render_list: - self.render_list[key] = [] self.render_list[key].append(rendered_html) return rendered_html diff --git a/profiler/advisor/display/html/templates/ai_core_frequency.html b/profiler/advisor/display/html/templates/ai_core_frequency.html new file mode 100644 index 000000000..d04514203 --- /dev/null +++ b/profiler/advisor/display/html/templates/ai_core_frequency.html @@ -0,0 +1,27 @@ +{% if data|length > 0 %} +
+

AI CORE Frequency Issues

+
+ Issue: {{ desc }} +
+ Suggestion: {{ suggestion }} +

+ + + {% for header in headers %} + + {% endfor %} + + + {% for row in data %} + + {% for element in row %} + + {% endfor %} + + {% endfor %} +
{{ header }}
{{ element|safe }}
+ +
+
+{% endif %} \ No newline at end of file diff --git a/profiler/advisor/display/html/templates/slow_dataloader.html b/profiler/advisor/display/html/templates/slow_dataloader.html new file mode 100644 index 000000000..bf71a7085 --- /dev/null +++ b/profiler/advisor/display/html/templates/slow_dataloader.html @@ -0,0 +1,18 @@ +
+

Slow Dataloader Issues

+
+ {{ desc }} + + + + + + {% for suggestion in suggestions %} + + + + {% endfor %} +
Suggestions
{{ loop.index }}. {{ suggestion|safe }}
+ +
+
diff --git a/profiler/advisor/display/html/templates/sync_batchnorm.html b/profiler/advisor/display/html/templates/sync_batchnorm.html new file mode 100644 index 000000000..bb46c1f06 --- /dev/null +++ b/profiler/advisor/display/html/templates/sync_batchnorm.html @@ -0,0 +1,30 @@ + +
+

SyncBatchNorm Issues

+
+ {{ desc }} + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+ + More efficient code of syncbn forward as follows: + {% for item in solutions %} + {% for key, value in item.items() %} + {% if 'efficient_code' in value %} + 
{{ value.efficient_code|safe }}
+ {% endif %} + {% endfor %} + {% endfor %} + +
+
diff --git a/profiler/advisor/display/html/templates/synchronize_stream.html b/profiler/advisor/display/html/templates/synchronize_stream.html new file mode 100644 index 000000000..1832f9406 --- /dev/null +++ b/profiler/advisor/display/html/templates/synchronize_stream.html @@ -0,0 +1,57 @@ +
+

Synchronize Stream Issues

+
+ {{ desc }} + + + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+ +
+ {% if not empty_stacks %} + Please click on the collapsible box below to view the detailed code stack that triggers synchronizeStream + {% elif not framework_black_list %} + Suggestion: + These operators have no code stack. If parameter 'with_stack=False' was set while profiling, please refer to + Ascend PyTorch Profiler to set + 'with_stack=True'. Otherwise, ignore following affinity APIs due to backward broadcast lack of stack. + {% endif %} + + {% for api_name, stacks in result.items() %} + + {% if empty_stacks %} +
{{api_name|safe}}
+ + {% elif stacks | length > 0 %} + +
{{api_name|safe}}
+
+
+ {% for stack in stacks %} +
No.{{loop.index|safe}} code stack, called {{stack[1]|safe}} times
+
+ {{stack[0]|safe}} +
+ {% endfor %} +
+
+ {% endif %} + + {% endfor %} + +
+ +
+
diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index 59bfee77f..1d3872a17 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -13,23 +13,31 @@ from profiler.advisor.analyzer.cluster.slow_rank_analyser import SlowRankAnalyze from profiler.advisor.analyzer.cluster.slow_link_analyser import SlowLinkAnalyzer from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer +from profiler.advisor.analyzer.schedule.syncbn.syncbn_analyzer import SyncBNAnalyzer +from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_analyzer import SynchronizeStreamAnalyzer +from profiler.advisor.analyzer.dataloader.dataloader_analyzer import DataloaderAnalyzer +from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_analyzer import AICoreFreqAnalyzer + class Interface: supported_analyzer = { "schedule": OrderedDict({ - SupportedScopes.TIMELINE_FUSION_OPS: TimelineFusionOpsAnalyzer, - SupportedScopes.TIMELINE_OP_DISPATCH: OpDispatchAnalyzer + SupportedScopes.SYNCBN: SyncBNAnalyzer, + SupportedScopes.TIMELINE_OP_DISPATCH: OpDispatchAnalyzer, + SupportedScopes.SYNCHRONIZE_STREAM: SynchronizeStreamAnalyzer, + SupportedScopes.TIMELINE_FUSION_OPS: TimelineFusionOpsAnalyzer }), "computation": OrderedDict({ SupportedScopes.DYNAMIC_SHAPE_ANALYSIS: DynamicShapeAnalyzer, SupportedScopes.AICPU_ANALYSIS: AicpuAnalyzer, SupportedScopes.OPERATOR_NO_BOUND_ANALYSIS: OperatorBoundAnalyzer, SupportedScopes.BLOCK_DIM_ANALYSIS: BlockDimAnalyzer, - SupportedScopes.GRAPH: FusionOPAnalyzer + SupportedScopes.GRAPH: FusionOPAnalyzer, + SupportedScopes.FREQ_ANALYSIS: AICoreFreqAnalyzer }), "communication": OrderedDict(), "overall": OrderedDict({SupportedScopes.OVER_ALL: OverallSummaryAnalyzer}), - "dataloader": OrderedDict(), + "dataloader": OrderedDict({SupportedScopes.DATALOADER: DataloaderAnalyzer}), "cluster": OrderedDict({ SupportedScopes.SLOW_RANK: SlowRankAnalyzer, SupportedScopes.SLOW_LINK: SlowLinkAnalyzer @@ -66,7 +74,7 @@ class Interface: if render_html and result.data: if hasattr(analyzer, "html_render"): analyzer.html_render.render_html() - analyzer.html_render.save_to_file(f'att_advisor_{Timer().strftime}.html') + analyzer.html_render.save_to_file(f'mstt_advisor_{Timer().strftime}.html') return result if not output_dict else dict(result.data) diff --git a/profiler/advisor/result/item.py b/profiler/advisor/result/item.py index fa0ffb5b1..02db7fdd0 100644 --- a/profiler/advisor/result/item.py +++ b/profiler/advisor/result/item.py @@ -15,7 +15,7 @@ class OptimizeItem: @property def headers(self): - return ["problem", "description", "suggestion"] + return ["category", "description", "suggestion"] class StatisticsItem: diff --git a/profiler/advisor/result/result.py b/profiler/advisor/result/result.py index c7d7da866..0d0602ee5 100644 --- a/profiler/advisor/result/result.py +++ b/profiler/advisor/result/result.py @@ -93,6 +93,9 @@ class SheetRecoder: if data not in self._sheet_data[sheet_name]["data"]: self._sheet_data[sheet_name]["data"].append(data) + def clear(self): + self._sheet_data.clear() + @singleton class OptimizeResult: @@ -110,12 +113,12 @@ class OptimizeResult: def add_tune_op_list(self, tune_op_list) -> None: """ add tune op name to tune op list - :param tune_op_list: tune op name list to be added + :param tune_op_list: list of operators to be optimized :return: None """ - for op_name in tune_op_list: - if op_name not in self._tune_op_list: - self._tune_op_list.append(op_name) + for operator_name in tune_op_list: + if operator_name not in self._tune_op_list: + self._tune_op_list.append(operator_name) def add(self, overview_item): sheet_name = "problems" @@ -148,6 +151,9 @@ class OptimizeResult: logger.info("Save problems details file to %s", Config().analysis_result_file) self._save_op_file_list() + def clear(self) -> None: + self.data.clear() + def _save_op_file_list(self) -> None: if not self._tune_op_list: return @@ -173,9 +179,9 @@ class TerminalResult: def __init__(self): self.width, _ = self.get_terminal_size() if self.width is None: - self.table = PrettyTable(["No.", "Problem", "Description", "Suggestion"]) + self.table = PrettyTable(["No.", "Category", "Description", "Suggestion"]) else: - self.table = PrettyTable(["No.", "Problem", "Description", "Suggestion"], + self.table = PrettyTable(["No.", "Category", "Description", "Suggestion"], max_table_width=max(self.width - 20, 180)) self.table.hrules = ALL self.result_list = [] diff --git a/profiler/advisor/rules/dataloader.yaml b/profiler/advisor/rules/dataloader.yaml new file mode 100644 index 000000000..a84abcfdf --- /dev/null +++ b/profiler/advisor/rules/dataloader.yaml @@ -0,0 +1,9 @@ +# unit is milliseconds +dataloader_duration_threshold: 10 +problem: "Found slow dataloader, cost {dataloader_duration} milliseconds for one step while profiling, normally less than {dataloader_duration_threshold} milliseconds." +solutions: + - "Please check the disk I/O of your data directory. If you are training model in ModelArts, please move data to '/cache' or mount a more efficient cloud disk for better I/O." + - "Please check if there are any other multiprocess operations in runtime that may have affected the dataloader, such as training process core binding command 'taskset ...' used for launching the training job." + - "Please check the format of your data, avoid file format like tar, tar.gz, zip." + - "Please set 'pin_memory=True' for your dataloader." + - "Try to adjust dataloader parameter 'num_workers'." \ No newline at end of file diff --git a/profiler/advisor/rules/sync_batchnorm.yaml b/profiler/advisor/rules/sync_batchnorm.yaml new file mode 100644 index 000000000..0f702af6e --- /dev/null +++ b/profiler/advisor/rules/sync_batchnorm.yaml @@ -0,0 +1,41 @@ +problem: "Found {syncbn_num} SyncBatchNorm, which can lead to slow python task dispatch and frequent communication between devices and finally reducing training efficiency." +max_syncbn_num: 20 +solutions: + - enable batchnorm: + desc: "disable SyncBatchNorm by remove the code like 'torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)' if possible." + - enable efficient SyncBatchNorm: + desc: "replace the 'forward' method of python script 'torch_npu/utils/syncbatchnorm.py' in your runtime environment." + efficient_code: | + @staticmethod + def forward(self, input_tensor, weight, bias, running_mean, running_var, eps, momentum, process_group, world_size): + input_tensor = input_tensor.contiguous() + input_shape = input_tensor.shape + input_tensor_ = input_tensor.reshape(input_shape[0], input_shape[1], 1, -1) + sum_val, sum_square_val = torch.batch_norm_reduce(input_tensor_, eps) + + count = torch.full((1,), + input_tensor.numel() // input_tensor.size(1), + dtype=sum_val.dtype, + device=sum_val.device) + + num_channels = input_tensor.shape[1] + combined = torch.cat([sum_val, sum_square_val, count], dim=0) + combined_list = torch.empty((world_size,) + combined.shape, dtype=combined.dtype, device=combined.device) + dist.all_gather_togather(combined_list, combined, process_group, async_op=False) + sum_all, square_sum_all, count_all = torch.split(combined_list, num_channels, dim=1) + size = count_all.view(-1).sum() + if size == 1: + raise ValueError('Expected more than 1 value per channel when training, got input size {}'.format(size)) + + mean, invstd = torch.batch_norm_gather_stats_update(input_tensor, + sum_all, + square_sum_all, + running_mean, + running_var, + momentum, + eps, + count_all.view(-1)) + self.save_for_backward(input_tensor, weight, mean, invstd, count_all.to(torch.int32)) + self.process_group = process_group + out = torch.batch_norm_elemt(input_tensor, weight, bias, mean, invstd, eps) + return out \ No newline at end of file diff --git a/profiler/advisor/rules/synchronize.yaml b/profiler/advisor/rules/synchronize.yaml new file mode 100644 index 000000000..3bd518d00 --- /dev/null +++ b/profiler/advisor/rules/synchronize.yaml @@ -0,0 +1,8 @@ +problem: "SynchronizeStream will reduce training efficiency. Found {synchronize_num} SynchronizeStream, {slow_synchronize_num} slow SynchronizeStream cost {total_synchronize_stream_time} us." +max_synchronize_num: 20 +slow_synchronize_threshold: 10 #ms +solutions: + - disable ascend launch blocking: + desc: "please check your env 'ASCEND_LAUNCH_BLOCKING', if ASCEND_LAUNCH_BLOCKING=1, please execute 'unset ASCEND_LAUNCH_BLOCKING' and then start your training job." + - modify code to avoid synchronize stream: + desc: "please try to modify your training code to avoid synchronize stream between cpu and npu." \ No newline at end of file diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index 84419b670..b373d7bad 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -1,5 +1,6 @@ import inspect import json + import logging import multiprocessing as mp import os @@ -11,7 +12,7 @@ import traceback import types from functools import wraps from typing import Any, Set - +import ijson import click import requests from requests.adapters import HTTPAdapter @@ -43,7 +44,7 @@ class ContextObject(object): def debug_option(f): - return click.option('--debug', '-D', + return click.option('--debug', is_flag=True, expose_value=False, is_eager=True, @@ -550,3 +551,50 @@ def get_file_path_by_walk(root, filename): file_path = os.path.join(root, name) return file_path return file_path + + +def check_path_valid(path): + if os.path.islink(os.path.abspath(path)): + logger.error("fThe path is detected as a soft connection. path:%ss", path) + return False + elif not os.access(path, os.R_OK): + logger.error(f"The file is not readable. path:%ss", path) + return False + elif os.path.getsize(path) > const.MAX_FILE_SIZE: + logger.error(f"The file size exceeds the limit. path:%ss, MAX_FILE_SIZE:%ss B",path, const.MAX_FILE_SIZE) + return False + return True + + +def parse_json_with_generator(timeline_data_path, func): + result = [] + if not check_path_valid(timeline_data_path): + return result + try: + with open(timeline_data_path, "r") as f: + if os.getenv(const.DISABLE_STREAMING_READER) == "1": + logger.debug("Disable streaming reader.") + file_parser = json.loads(f.read()) + else: + logger.debug("Enable streaming reader.") + file_parser = ijson.items(f, "item") + + for i, event in tqdm(enumerate(file_parser), + leave=False, ncols=100, desc="Building dataset for timeline analysis"): + func_res = func(index=i, event=event) + if func_res is not None: + result.append(func_res) + + except Exception: + logger.warning("Error %s while parsing file %s, continue to timeline analysis", traceback.format_exc(), + timeline_data_path) + return result + + +def convert_to_float(num): + try: + return float(num) + except (ValueError, FloatingPointError): + logger.error(f"Can not convert %ss to float", num) + pass + return 0 diff --git a/profiler/cli/__init__.py b/profiler/cli/__init__.py index eab13571c..e768e4cb8 100644 --- a/profiler/cli/__init__.py +++ b/profiler/cli/__init__.py @@ -1,4 +1,4 @@ from profiler.advisor.config.config import Config from profiler.advisor.utils.utils import Timer -Config().set_log_path(f"att_advisor_{Timer().strftime}.xlsx") +Config().set_log_path(f"mstt_advisor_{Timer().strftime}.xlsx") diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index 2e173dc87..f400a265b 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -83,9 +83,6 @@ def analyze_cli(**kwargs): help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @debug_option def analyze_all(**kwargs) -> None: - # 当前compare_tools必须输入两个profiling路径，att-advisor有等价功能支持输入一个Profiling路径，后续替换成对应实现 - if not kwargs.get("benchmark_profiling_path"): - kwargs["benchmark_profiling_path"] = kwargs.get("profiling_path") try: _analyze(Interface.all_dimension, **kwargs) except RuntimeError as e: diff --git a/profiler/cluster_analyse/README.md b/profiler/cluster_analyse/README.md index deaebb6cd..fdd43ca96 100644 --- a/profiler/cluster_analyse/README.md +++ b/profiler/cluster_analyse/README.md @@ -86,7 +86,7 @@ experimental_config = torch_npu.profiler._ExperimentalConfig( ### 交付件 -集群分析工具的交付件通过Ascend Insight工具展示，详见《[MindStudio Ascend Insight用户指南](https://www.hiascend.com/document/detail/zh/mindstudio/70RC1/GUI-baseddevelopmenttool/msascendinsightug/AscendInsight_0002.html)》。 +集群分析工具的交付件通过MindStudio Insight工具展示，详见《[MindStudio Insight用户指南](https://www.hiascend.com/document/detail/zh/mindstudio/70RC2/GUI-baseddevelopmenttool/msascendinsightug/AscendInsight_0002.html)》。 #### cluster_step_trace_time.csv @@ -156,25 +156,25 @@ L列：Preparing，指迭代开始到首个计算或通信算子运行的时间 #### cluster_analysis.db -解析analysis.db或ascend_pytorch_profiler_{rank_id}.db生成的交付件，根据数据解析模式不同而解析不同的数据，可以使用Ascend Insight工具展示。 +解析analysis.db或ascend_pytorch_profiler_{rank_id}.db生成的交付件，根据数据解析模式不同而解析不同的数据，可以使用MindStudio Insight工具展示。 #### stats.ipynb - 数据解析模式为cann_api_sum时生成，保存在cluster_analysis_output/CannApiSum目录下。 - 可使用jupyter notebook工具或Ascend Insight工具打开，主要展示集群API耗时信息。 + 可使用jupyter notebook工具或MindStudio Insight工具打开，主要展示集群API耗时信息。 - 数据解析模式为compute_op_sum时生成，保存在cluster_analysis_output/ComputeOpSum目录下。 - 可使用jupyter notebook工具或Ascend Insight工具打开，主要展示集群计算算子耗时分析（将集群所有计算算子进行汇总并以图表展示），集群Rank计算算子耗时分析（将每个Rank的计算算子进行各自汇总）。 + 可使用jupyter notebook工具或MindStudio Insight工具打开，主要展示集群计算算子耗时分析（将集群所有计算算子进行汇总并以图表展示），集群Rank计算算子耗时分析（将每个Rank的计算算子进行各自汇总）。 - 数据解析模式为hccl_sum时生成，保存在cluster_analysis_output/HcclSum目录下。 - 可使用jupyter notebook工具或Ascend Insight工具打开，主要展示集群通信算子耗时分析（将集群所有通信算子进行汇总并以图表展示），集群Rank通信算子耗时分析（将每个Rank的通信算子进行各自汇总）、Top通信算子信息展示。 + 可使用jupyter notebook工具或MindStudio Insight工具打开，主要展示集群通信算子耗时分析（将集群所有通信算子进行汇总并以图表展示），集群Rank通信算子耗时分析（将每个Rank的通信算子进行各自汇总）、Top通信算子信息展示。 - 数据解析模式为mstx_sum时生成，保存在cluster_analysis_output/MstxSum目录下。 - 可使用jupyter notebook工具或Ascend Insight工具打开，主要展示集群场景mstx打点信息，分为框架侧、CANN侧和Device侧三部分的打点信息。 + 可使用jupyter notebook工具或MindStudio Insight工具打开，主要展示集群场景mstx打点信息，分为框架侧、CANN侧和Device侧三部分的打点信息。 diff --git a/profiler/cluster_analyse/common_func/file_manager.py b/profiler/cluster_analyse/common_func/file_manager.py index e7e2d5adc..380192f87 100644 --- a/profiler/cluster_analyse/common_func/file_manager.py +++ b/profiler/cluster_analyse/common_func/file_manager.py @@ -17,6 +17,8 @@ import os import csv import json +import yaml + from common_func.constant import Constant from common_func.path_manager import PathManager @@ -60,6 +62,23 @@ class FileManager: raise RuntimeError(f"Failed to read the file: {base_name}") from e return result_data + @classmethod + def read_yaml_file(cls, file_path: str) -> dict: + PathManager.check_path_readable(file_path) + base_name = os.path.basename(file_path) + file_size = os.path.getsize(file_path) + if file_size <= 0: + return {} + if file_size > Constant.MAX_JSON_SIZE: + raise RuntimeError(f"The file({base_name}) size exceeds the preset max value.") + + try: + with open(file_path, "r") as yaml_file: + result_data = yaml.safe_load(yaml_file) + except Exception as e: + raise RuntimeError(f"Failed to read the file: {base_name}") from e + return result_data + @classmethod def create_csv_file(cls, profiler_path: str, data: list, file_name: str, headers: list = None) -> None: if not data: diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index d81ce05f4..78ea5d897 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -213,7 +213,7 @@ activities配置仅采集NPU数据，不配置experimental_config参数以及其 - 当Computing Time耗时增大，分析**算子性能**。 - 当Uncovered Communication Time耗时增大，分析**通信性能**，若通信性能分析没有劣化的通信算子，代表通信与计算的并行度较差，继续进行NPU的集群性能分析。 -- 当Mem Usage增大，分析**算子内存**，若没有明显占用较大的算子，则代表算子内存申请量没有差异，问题在于内存的释放（持有时间过久），可以使用tensorboard或ascend insight继续进行NPU内存的分析。 +- 当Mem Usage增大，分析**算子内存**，若没有明显占用较大的算子，则代表算子内存申请量没有差异，问题在于内存的释放（持有时间过久），可以使用TensorBoard或MindStudio insight继续进行NPU内存的分析。 ### 算子性能 diff --git a/profiler/compare_tools/compare_backend/comparator/base_comparator.py b/profiler/compare_tools/compare_backend/comparator/base_comparator.py index 330fb871e..8012dfae9 100644 --- a/profiler/compare_tools/compare_backend/comparator/base_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/base_comparator.py @@ -21,4 +21,4 @@ class BaseComparator(ABC): @abstractmethod def _compare(self): - raise NotImplementedError("Function _compare need to be implemented.") + raise NotImplementedError("Function _compare need to be implemented.") \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py b/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py new file mode 100644 index 000000000..d438dc41d --- /dev/null +++ b/profiler/compare_tools/compare_backend/comparator/overall_metrics_comparator.py @@ -0,0 +1,50 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from math import isclose + +from compare_backend.comparator.base_comparator import BaseComparator +from compare_backend.utils.constant import Constant +from compare_backend.utils.excel_config import ExcelConfig + + +class OverallMetricsComparator(BaseComparator): + + def __init__(self, origin_data: dict, bean: any): + super().__init__(origin_data, bean) + self._row_style = [] + + @property + def base_info(self): + return self._origin_data.get(Constant.BASE_DATA) + + @property + def comp_info(self): + return self._origin_data.get(Constant.COMPARISON_DATA) + + def generate_data(self) -> dict: + self._compare() + return {self._sheet_name: { + "headers": self._headers, + "rows": self._rows, + "overhead": self._overhead, + "row_style": self._row_style + }} + + def _compare(self): + if isclose(self.base_info.e2e_time_ms, 0) or isclose(self.comp_info.e2e_time_ms, 0): + return + self._rows.extend(self._bean(self.base_info, self.comp_info).rows) + for row in self._rows: + self._row_style.append(ExcelConfig.ROW_STYLE_MAP.get(row[0], {})) # index 0 for metric index name diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py index 122009b90..9c4825c0e 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py @@ -1,8 +1,9 @@ import math +from decimal import Decimal import pandas as pd -from compare_backend.utils.common_func import convert_to_float +from compare_backend.utils.common_func import convert_to_float, convert_to_decimal from compare_backend.utils.constant import Constant @@ -12,8 +13,10 @@ class KernelDetailsBean: self._op_type = "" self._name = "" self._aiv_vec_time = 0.0 + self._aicore_time = 0.0 self._mac_time = 0.0 self._duration = 0.0 + self._start_time = Decimal("0") self.init() @property @@ -30,6 +33,12 @@ class KernelDetailsBean: return float("nan") return convert_to_float(self._aiv_vec_time) + @property + def aicore_time(self) -> float: + if self._aicore_time == "" or self._aicore_time == "N/A": + return float("nan") + return convert_to_float(self._aicore_time) + @property def mac_time(self) -> float: if self._mac_time == "" or self._mac_time == "N/A": @@ -40,6 +49,18 @@ class KernelDetailsBean: def duration(self) -> float: return convert_to_float(self._duration) + @property + def dur(self) -> float: + return convert_to_float(self._duration) + + @property + def start_time(self) -> Decimal: + return convert_to_decimal(self._start_time) + + @property + def end_time(self) -> Decimal: + return self.start_time + convert_to_decimal(self._duration) + def is_hide_op_pmu(self): if "mac_time(us)" in self._data.keys() or "aiv_vec_time(us)" in self._data.keys(): return False @@ -66,7 +87,7 @@ class KernelDetailsBean: def is_flash_attention(self): return "flashattention" in self.op_type.lower() - def is_cube(self): + def is_matmul(self): return "matmul" in self.op_type.lower() def is_conv(self): @@ -79,9 +100,17 @@ class KernelDetailsBean: def is_page_attention(self): return "pagedattention" in self.op_type.lower() + def is_trans(self): + return any(trans_mask in self.name.lower() for trans_mask in Constant.KERNEL_TRANS_MASK) + + def is_cube_kernel_cat(self): + return self.mac_time > 0 or self.aicore_time > 0 + def init(self): self._op_type = self._data.get('Type', "") self._name = self._data.get('Name', "") self._aiv_vec_time = self._data.get('aiv_vec_time(us)', "") + self._aicore_time = self._data.get("aicore_time(us)", "") self._mac_time = self._data.get('mac_time(us)', "") self._duration = self._data.get('Duration(us)', 0) + self._start_time = Decimal(self._data.get("Start Time(us)", "0")) diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py index cef6bb071..245b51d10 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py @@ -114,6 +114,21 @@ class TraceEventBean: def is_torch_op(self, value: bool): self._is_torch_op = value + @classmethod + def is_sdma(cls): + return False + + @classmethod + def is_page_attention(cls): + return False + + @classmethod + def is_trans(cls) -> bool: + """ + 暂时没找到GPU判断trans的方法，暂时都是notrans + """ + return False + def is_m_mode(self) -> bool: return self._ph == "M" @@ -199,11 +214,44 @@ class TraceEventBean: self._name = name def is_conv(self): - return self.name.lower().startswith("aten::conv") + return self.lower_name.startswith("aten::conv") def is_lccl(self): return self.lower_name == "kernel_aivec" + def is_fa_for_cpu_op(self) -> bool: + """ + 这个类在cpu op和gpu中均有用到，这里是在cpu op阶段判断 + """ + return any(cube_mask in self.lower_name for cube_mask in Constant.CPU_OP_FA_MASK) + + def is_conv_for_cpu_op(self) -> bool: + """ + 这个类在cpu op和gpu中均有用到，这里是在cpu op阶段判断 + """ + return self.lower_name.startswith(Constant.CPU_OP_CONV) + + def is_matmul_for_cpu_op(self) -> bool: + """ + 这个类在cpu op和gpu中均有用到，这里是在cpu op阶段判断 + """ + return any(bwd_mask in self.lower_name for bwd_mask in Constant.CPU_OP_MATMUL_MASK) + + def is_bwd_for_cpu_op(self) -> bool: + """ + 这个类在cpu op和gpu中均有用到，这里是在cpu op阶段判断 + """ + return any(bwd_mask in self.lower_name for bwd_mask in Constant.BWD_LIST) + + def is_cpu_cube_op(self) -> bool: + return self.is_matmul_for_cpu_op() or self.is_fa_for_cpu_op() or self.is_conv_for_cpu_op() + + def is_vector(self): + return not any(cube_mask in self.lower_name for cube_mask in Constant.KERNEL_CUBE_MASK) + + def is_cube_kernel_cat(self): + return any(cube_mask in self.lower_name for cube_mask in Constant.KERNEL_CUBE_MASK) + def init(self): if isinstance(self._event, dict): self._pid = self._event.get("pid", 0) diff --git a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py b/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py new file mode 100644 index 000000000..544f8f523 --- /dev/null +++ b/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py @@ -0,0 +1,255 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from math import isclose + +from compare_backend.compare_bean.profiling_info import ProfilingInfo +from compare_backend.utils.common_func import calculate_diff_ratio +from compare_backend.utils.constant import Constant +from compare_backend.utils.excel_config import ExcelConfig + + +class OverallMetricsBean: + TABLE_NAME = Constant.OVERALL_METRICS_TABLE + HEADERS = ExcelConfig.HEADERS.get(TABLE_NAME) + OVERHEAD = ExcelConfig.OVERHEAD.get(TABLE_NAME) + + def __init__(self, base_info: ProfilingInfo, comparison_info: ProfilingInfo): + self._base_data = OverallMetricsInfo(base_info).overall_metrics + self._comparison_data = OverallMetricsInfo(comparison_info).overall_metrics + + @property + def rows(self): + rows_data = [] + for index, base_data in self._base_data.items(): + comparison_data = self._comparison_data.get(index) + row = self.get_row_data(index, base_data, comparison_data) + if row: + rows_data.append(row) + return rows_data + + @staticmethod + def get_row_data(index, base_data, comparison_data): + if isclose(base_data[0], 0) and isclose(comparison_data[0], 0): + return [] + row_data = [index] + row_data.extend(base_data) + row_data.extend(comparison_data) + row_data.extend(calculate_diff_ratio(base_data[0], comparison_data[0])) + return row_data + + +class OverallMetricsInfo: + def __init__(self, profiling_info: ProfilingInfo): + self._profiling_info = profiling_info + self._overall_metrics_data_map = { + ExcelConfig.COMPUTING: self.computing_data, + ExcelConfig.FA: self.fa_data, + ExcelConfig.FA_FWD_CUBE: self.fa_fwd_cube_data, + ExcelConfig.FA_FWD_VECTOR: self.fa_fwd_vector_data, + ExcelConfig.FA_BWD_CUBE: self.fa_bwd_cube_data, + ExcelConfig.FA_BWD_VECTOR: self.fa_bwd_vector_data, + ExcelConfig.CONV: self.conv_data, + ExcelConfig.CONV_FWD_CUBE: self.conv_fwd_cube_data, + ExcelConfig.CONV_FWD_VECTOR: self.conv_fwd_vector_data, + ExcelConfig.CONV_BWD_CUBE: self.conv_bwd_cube_data, + ExcelConfig.CONV_BWD_VECTOR: self.conv_bwd_vector_data, + ExcelConfig.MM: self.mm_data, + ExcelConfig.MM_CUBE: self.mm_cube_data, + ExcelConfig.MM_VECTOR: self.mm_vector_data, + ExcelConfig.PA: self.pa_data, + ExcelConfig.VECTOR: self.vector_data, + ExcelConfig.VECTOR_TRANS: self.vector_trans_data, + ExcelConfig.VECTOR_NO_TRANS: self.vector_no_trans_data, + ExcelConfig.CUBE: self.cube_data, + ExcelConfig.SDMA_TM: self.sdma_tm_data, + ExcelConfig.OTHER: self.other_data, + ExcelConfig.COMMUNICATION_TIME: self.communication_data, + ExcelConfig.WAIT: self.wait_data, + ExcelConfig.TRANSMIT: self.transmit_data, + ExcelConfig.FREE_TIME: self.free_time_data, + ExcelConfig.SDMA: self.sdma_data, + ExcelConfig.FREE: self.free_data, + ExcelConfig.E2E_TIME: self.e2e_time_data + } + + @property + def overall_metrics(self): + return self._overall_metrics_data_map + + @property + def computing_data(self): + return [self._profiling_info.compute_time_ms, + self._profiling_info.compute_time_ms / self._profiling_info.e2e_time_ms, + sum((self._profiling_info.fa_total_num, self._profiling_info.conv_total_num, + self._profiling_info.mm_total_num, self._profiling_info.vector_total_num, + self._profiling_info.sdma_num_tensor_move, self._profiling_info.other_cube_num, + self._profiling_info.page_attention_num))] + + @property + def fa_data(self): + return [self._profiling_info.fa_total_time, + self._profiling_info.fa_total_time / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_total_num] + + @property + def fa_fwd_cube_data(self): + return [self._profiling_info.fa_time_fwd_cube, + self._profiling_info.fa_time_fwd_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_num_fwd_cube] + + @property + def fa_fwd_vector_data(self): + return [self._profiling_info.fa_time_fwd_vector, + self._profiling_info.fa_time_fwd_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_num_fwd_vector] + + @property + def fa_bwd_cube_data(self): + return [self._profiling_info.fa_time_bwd_cube, + self._profiling_info.fa_time_bwd_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_num_bwd_cube] + + @property + def fa_bwd_vector_data(self): + return [self._profiling_info.fa_time_bwd_vector, + self._profiling_info.fa_time_bwd_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_num_bwd_vector] + + @property + def conv_data(self): + return [self._profiling_info.conv_total_time, + self._profiling_info.conv_total_time / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_total_num] + + @property + def conv_fwd_cube_data(self): + return [self._profiling_info.conv_time_fwd_cube, + self._profiling_info.conv_time_fwd_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_num_fwd_cube] + + @property + def conv_fwd_vector_data(self): + return [self._profiling_info.conv_time_fwd_vector, + self._profiling_info.conv_time_fwd_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_num_fwd_vector] + + @property + def conv_bwd_cube_data(self): + return [self._profiling_info.conv_time_bwd_cube, + self._profiling_info.conv_time_bwd_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_num_bwd_cube] + + @property + def conv_bwd_vector_data(self): + return [self._profiling_info.conv_time_bwd_vector, + self._profiling_info.conv_time_bwd_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_num_bwd_vector] + + @property + def mm_data(self): + return [self._profiling_info.mm_total_time, + self._profiling_info.mm_total_time / self._profiling_info.e2e_time_ms, + self._profiling_info.mm_total_num] + + @property + def mm_cube_data(self): + return [self._profiling_info.matmul_time_cube, + self._profiling_info.matmul_time_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.matmul_num_cube] + + @property + def mm_vector_data(self): + return [self._profiling_info.matmul_time_vector, + self._profiling_info.matmul_time_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.matmul_num_vector] + + @property + def pa_data(self): + return [self._profiling_info.page_attention_time, + self._profiling_info.page_attention_time / self._profiling_info.e2e_time_ms, + self._profiling_info.page_attention_num] + + @property + def vector_data(self): + return [self._profiling_info.vector_total_time, + self._profiling_info.vector_total_time / self._profiling_info.e2e_time_ms, + self._profiling_info.vector_total_num] + + @property + def vector_trans_data(self): + return [self._profiling_info.vector_time_trans, + self._profiling_info.vector_time_trans / self._profiling_info.e2e_time_ms, + self._profiling_info.vector_num_trans] + + @property + def vector_no_trans_data(self): + return [self._profiling_info.vector_time_notrans, + self._profiling_info.vector_time_notrans / self._profiling_info.e2e_time_ms, + self._profiling_info.vector_num_notrans] + + @property + def cube_data(self): + return [self._profiling_info.other_cube_time, + self._profiling_info.other_cube_time / self._profiling_info.e2e_time_ms, + self._profiling_info.other_cube_num] + + @property + def sdma_tm_data(self): + return [self._profiling_info.sdma_time_tensor_move, + self._profiling_info.sdma_time_tensor_move / self._profiling_info.e2e_time_ms, + self._profiling_info.sdma_num_tensor_move] + + @property + def other_data(self): + other_time = max((0, + self._profiling_info.compute_time_ms - self._profiling_info.fa_total_time - + self._profiling_info.conv_total_time - self._profiling_info.mm_total_time - + self._profiling_info.vector_total_time - self._profiling_info.sdma_time_tensor_move - + self._profiling_info.other_cube_time - self._profiling_info.page_attention_time)) + return [other_time, other_time / self._profiling_info.e2e_time_ms, "/"] + + @property + def communication_data(self): + return [self._profiling_info.communication_not_overlapped_ms, + self._profiling_info.communication_not_overlapped_ms / self._profiling_info.e2e_time_ms, "/"] + + @property + def wait_data(self): + return [self._profiling_info.wait_time_ms, + self._profiling_info.wait_time_ms / self._profiling_info.e2e_time_ms, "/"] + + @property + def transmit_data(self): + return [self._profiling_info.transmit_time_ms, + self._profiling_info.transmit_time_ms / self._profiling_info.e2e_time_ms, "/"] + + @property + def free_time_data(self): + return [self._profiling_info.free_time_ms, + self._profiling_info.free_time_ms / self._profiling_info.e2e_time_ms, "/"] + + @property + def sdma_data(self): + return [self._profiling_info.sdma_time_stream, + self._profiling_info.sdma_time_stream / self._profiling_info.e2e_time_ms, "/"] + + @property + def free_data(self): + free = self._profiling_info.free_time_ms - self._profiling_info.sdma_time_stream + return [free, free / self._profiling_info.e2e_time_ms, "/"] + + @property + def e2e_time_data(self): + return [self._profiling_info.e2e_time_ms, 1, "/"] diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index e5d9bf26e..e0a80a4d3 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -37,6 +37,105 @@ class ProfilingInfo: self.hide_op_details = False self.is_level0 = False + # 性能拆解新指标 + self.fa_time_fwd_cube = 0.0 + self.fa_num_fwd_cube = 0 + self.fa_time_bwd_cube = 0.0 + self.fa_num_bwd_cube = 0 + self.fa_time_fwd_vector = 0.0 + self.fa_num_fwd_vector = 0 + self.fa_time_bwd_vector = 0.0 + self.fa_num_bwd_vector = 0 + + self.conv_time_fwd_cube = 0.0 + self.conv_num_fwd_cube = 0 + self.conv_time_bwd_cube = 0.0 + self.conv_num_bwd_cube = 0 + self.conv_time_fwd_vector = 0.0 + self.conv_num_fwd_vector = 0 + self.conv_time_bwd_vector = 0.0 + self.conv_num_bwd_vector = 0 + + self.matmul_time_cube = 0.0 + self.matmul_num_cube = 0 + self.matmul_time_vector = 0.0 + self.matmul_num_vector = 0 + + self.page_attention_time = 0.0 + self.page_attention_num = 0 + + self.vector_time_trans = 0.0 + self.vector_num_trans = 0 + self.vector_time_notrans = 0.0 + self.vector_num_notrans = 0 + + self.sdma_time_tensor_move = 0.0 + self.sdma_num_tensor_move = 0 + self.sdma_time_stream = 0.0 + self.sdma_num_stream = 0 + + self.other_cube_time = 0.0 + self.other_cube_num = 0 + + @property + def e2e_time_ms(self): + return self.e2e_time * 10 ** 3 + + @property + def compute_time_ms(self): + return self.compute_time * 10 ** 3 + + @property + def free_time_ms(self): + return self.scheduling_time * 10 ** 3 + + @property + def communication_not_overlapped_ms(self): + return self.communication_not_overlapped * 10 ** 3 + + @property + def wait_time_ms(self): + return self.wait_time * 10 ** 3 + + @property + def transmit_time_ms(self): + return (self.communication_not_overlapped - self.wait_time) * 10 ** 3 + + @property + def fa_total_time(self): + return sum((self.fa_time_fwd_cube, self.fa_time_fwd_vector, self.fa_time_bwd_cube, self.fa_time_bwd_vector)) + + @property + def fa_total_num(self): + return sum((self.fa_num_fwd_cube, self.fa_num_fwd_vector, self.fa_num_bwd_cube, self.fa_num_bwd_vector)) + + @property + def conv_total_time(self): + return sum( + (self.conv_time_fwd_cube, self.conv_time_fwd_vector, self.conv_time_bwd_cube, + self.conv_time_bwd_vector)) + + @property + def conv_total_num(self): + return sum((self.conv_num_fwd_cube, self.conv_num_fwd_vector, self.conv_num_bwd_cube, + self.conv_num_bwd_vector)) + + @property + def mm_total_time(self): + return sum((self.matmul_time_cube, self.matmul_time_vector)) + + @property + def mm_total_num(self): + return sum((self.matmul_num_cube, self.matmul_num_vector)) + + @property + def vector_total_time(self): + return sum((self.vector_time_trans, self.vector_time_notrans)) + + @property + def vector_total_num(self): + return sum((self.vector_num_trans, self.vector_num_notrans)) + def trans_time_to_s(self): self.cube_time = self.cube_time / 10 ** 6 self.other_time = self.other_time / 10 ** 6 @@ -54,6 +153,24 @@ class ProfilingInfo: self.conv_time_fwd = self.conv_time_fwd / 10 ** 6 self.conv_time_bwd = self.conv_time_bwd / 10 ** 6 + # 新指标单位为ms + self.fa_time_fwd_cube /= 10 ** 3 + self.fa_time_bwd_cube /= 10 ** 3 + self.fa_time_fwd_vector /= 10 ** 3 + self.fa_time_bwd_vector /= 10 ** 3 + self.conv_time_fwd_cube /= 10 ** 3 + self.conv_time_bwd_cube /= 10 ** 3 + self.conv_time_fwd_vector /= 10 ** 3 + self.conv_time_bwd_vector /= 10 ** 3 + self.matmul_time_cube /= 10 ** 3 + self.matmul_time_vector /= 10 ** 3 + self.vector_time_trans /= 10 ** 3 + self.vector_time_notrans /= 10 ** 3 + self.sdma_time_tensor_move /= 10 ** 3 + self.sdma_time_stream /= 10 ** 3 + self.page_attention_time /= 10 ** 3 + self.other_cube_time /= 10 ** 3 + def calculate_other_time(self): self.other_time = max( [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - @@ -64,8 +181,7 @@ class ProfilingInfo: - self.conv_time_fwd - self.conv_time_bwd def calculate_schedule_time(self): - self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time \ - - self.communication_not_overlapped) + self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time - self.communication_not_overlapped) def update_fa_fwd_info(self, time: float): self.fa_time_fwd += time @@ -75,6 +191,30 @@ class ProfilingInfo: self.fa_time_bwd += time self.fa_num_bwd += 1 + def update_fa_fwd_cube_info(self, time: float): + self.fa_time_fwd_cube += time + self.fa_num_fwd_cube += 1 + + def update_fa_bwd_cube_info(self, time: float): + self.fa_time_bwd_cube += time + self.fa_num_bwd_cube += 1 + + def update_fa_fwd_vector_info(self, time: float): + self.fa_time_fwd_vector += time + self.fa_num_fwd_vector += 1 + + def update_fa_bwd_vector_info(self, time: float): + self.fa_time_bwd_vector += time + self.fa_num_bwd_vector += 1 + + def update_sdma_tensor_move_info(self, time: float): + self.sdma_time_tensor_move += time + self.sdma_num_tensor_move += 1 + + def update_sdma_stream_info(self, time: float, num: int = 1): + self.sdma_time_stream += time + self.sdma_num_stream += num + def update_pa_info(self, time: float): self.pa_time += time self.pa_num += 1 @@ -91,6 +231,42 @@ class ProfilingInfo: self.conv_time_bwd += time self.conv_num_bwd += 1 + def update_conv_bwd_cube_info(self, time: float): + self.conv_time_bwd_cube += time + self.conv_num_bwd_cube += 1 + + def update_conv_fwd_cube_info(self, time: float): + self.conv_time_fwd_cube += time + self.conv_num_fwd_cube += 1 + + def update_conv_bwd_vector_info(self, time: float): + self.conv_time_bwd_vector += time + self.conv_num_bwd_vector += 1 + + def update_conv_fwd_vector_info(self, time: float): + self.conv_time_fwd_vector += time + self.conv_num_fwd_vector += 1 + + def update_matmul_cube_info(self, time: float): + self.matmul_time_cube += time + self.matmul_num_cube += 1 + + def update_matmul_vector_info(self, time: float): + self.matmul_time_vector += time + self.matmul_num_vector += 1 + + def update_page_attention_info(self, time: float): + self.page_attention_time += time + self.page_attention_num += 1 + + def update_vector_trans_info(self, time: float): + self.vector_time_trans += time + self.vector_num_trans += 1 + + def update_vector_notrans_info(self, time: float): + self.vector_time_notrans += time + self.vector_num_notrans += 1 + def update_sdma_info(self, time: float, num: int = 1): self.sdma_time += time self.sdma_num += num @@ -103,6 +279,10 @@ class ProfilingInfo: self.vec_time += time self.vec_num += 1 + def update_other_cube_info(self, time: float): + self.other_cube_time += time + self.other_cube_num += 1 + def set_compute_time(self, time: float): self.compute_time = time diff --git a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index c89e84519..7bac2b033 100644 --- a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -31,4 +31,30 @@ class OverallPerfInterface: def _generate_result(self): overall_data = self._profiling_data.overall_metrics - self._result_data = getattr(overall_data, "__dict__", {}) + + self._result_data = { + "profiling_type": overall_data.profiling_type, + "minimal_profiling": overall_data.minimal_profiling, + "overall": {"e2e_time_ms": overall_data.e2e_time_ms, + "computing_time_ms": overall_data.compute_time_ms, + "uncovered_communication_time_ms": overall_data.communication_not_overlapped_ms, + "free_time_ms": overall_data.free_time_ms}, + "computing_time_disaggregate": {"fa_time_ms": overall_data.fa_total_time, + "conv_time_ms": overall_data.conv_total_time, + "matmul_time_ms": overall_data.mm_total_time, + "page_attention_time_ms": overall_data.page_attention_time, + "vector_time_ms": overall_data.vector_total_time, + "tensor_move_time_ms": overall_data.sdma_time_tensor_move, + "other_cube_time_ms": overall_data.other_cube_time}, + "computing_num_disaggregate": {"fa_num": overall_data.fa_total_num, + "conv_num": overall_data.conv_total_num, + "matmul_num": overall_data.mm_total_num, + "page_attention_num": overall_data.page_attention_num, + "vector_num": overall_data.vector_total_num, + "tensor_move_num": overall_data.sdma_num_tensor_move, + "other_cube_num": overall_data.other_cube_num}, + "communication_time_disaggregate": {"wait_time_ms": overall_data.wait_time_ms, + "transmit_time_ms": overall_data.transmit_time_ms}, + "free_time_disaggregate": {"sdma_time_ms": overall_data.sdma_time_stream, + "free_ms": overall_data.free_time_ms - overall_data.sdma_time_stream} + } diff --git a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py index 5b93d888a..292e31281 100644 --- a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -8,6 +8,7 @@ from compare_backend.comparator.module_comparetor import ModuleComparator from compare_backend.comparator.module_statistic_comparator import ModuleStatisticComparator from compare_backend.comparator.operator_comparator import OperatorComparator from compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator +from compare_backend.comparator.overall_metrics_comparator import OverallMetricsComparator from compare_backend.compare_bean.communication_bean import CommunicationBean from compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean from compare_backend.compare_bean.memory_statistic_bean import MemoryStatisticBean @@ -15,6 +16,7 @@ from compare_backend.compare_bean.module_compare_bean import ModuleCompareBean from compare_backend.compare_bean.module_statistic_bean import ModuleStatisticBean from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean from compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean +from compare_backend.compare_bean.overall_metrics_bean import OverallMetricsBean from compare_backend.data_prepare.module_data_prepare import ModuleDataPrepare from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare from compare_backend.generator.base_generator import BaseGenerator @@ -41,8 +43,16 @@ class DetailPerformanceGenerator(BaseGenerator): self._args.enable_communication_compare: print("[INFO] Start to compare performance detail data, please wait.") comparator_list = self._create_comparator() - for comparator in comparator_list: - self._result_data.update(comparator.generate_data()) + else: + comparator_list = [] + if self._args.enable_profiling_compare: + overall_data = {Constant.BASE_DATA: self._profiling_data_dict.get(Constant.BASE_DATA).overall_metrics, + Constant.COMPARISON_DATA: self._profiling_data_dict.get( + Constant.COMPARISON_DATA).overall_metrics} + # overall 数据在最前面 + comparator_list.insert(0, OverallMetricsComparator(overall_data, OverallMetricsBean)) + for comparator in comparator_list: + self._result_data.update(comparator.generate_data()) def generate_view(self): if not self._result_data: @@ -57,6 +67,7 @@ class DetailPerformanceGenerator(BaseGenerator): comparator_list = [] op_compare_result = [] + if self._args.enable_operator_compare: module_compare_result = self.match_nn_module() if self._profiling_data_dict.get( Constant.BASE_DATA).python_function_data and self._profiling_data_dict.get( diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 2127ff5e7..6ee07a656 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -2,6 +2,7 @@ from abc import abstractmethod, ABC from decimal import Decimal from compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent +from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean from compare_backend.compare_bean.origin_data_bean.trace_event_bean import TraceEventBean from compare_backend.compare_bean.profiling_info import ProfilingInfo from compare_backend.utils.constant import Constant @@ -66,6 +67,18 @@ class BaseProfilingParser(ABC): self._comm_list = [] self._read_trace_event() self._cur_func_index = 0 + self._categorize_performance_index = 0 + self._cpu_cube_op = None + self._bwd_tid = None + + @property + def cpu_cube_op(self): + if self._cpu_cube_op is not None: + return self._cpu_cube_op + cpu_cube_op = [op for op in self._result_data.torch_op_data if op.is_cpu_cube_op()] + cpu_cube_op.sort(key=lambda x: x.start_time) + self._cpu_cube_op = cpu_cube_op + return self._cpu_cube_op @abstractmethod def _update_memory_list(self): @@ -102,6 +115,90 @@ class BaseProfilingParser(ABC): self._check_result_data() return self._result_data + def categorize_computing_performance_data(self, tk: (TraceEventBean, KernelDetailsBean), flow_dict_new: dict): + if tk.is_page_attention(): + self._result_data.overall_metrics.update_page_attention_info(tk.dur) + return + if tk.is_sdma(): + self._result_data.overall_metrics.update_sdma_tensor_move_info(tk.dur) + return + flow_start_time = flow_dict_new.get(tk.start_time) + if flow_start_time: + while self._categorize_performance_index < len(self.cpu_cube_op): + cur_op = self.cpu_cube_op[self._categorize_performance_index] + if cur_op.end_time < flow_start_time: + self._categorize_performance_index += 1 + continue + if cur_op.start_time <= flow_start_time: + self._categorize_cube_performance_data(cur_op, tk) + return + break + if self._profiling_type == Constant.NPU: + # 缺失torch至npu连线的算子，判断fa/conv/matmul使用kernel_details.csv的op_type字段 + if tk.is_flash_attention(): + if tk.is_fa_bwd(): + self._result_data.overall_metrics.update_fa_bwd_cube_info(tk.dur) + else: + self._result_data.overall_metrics.update_fa_fwd_cube_info(tk.dur) + return + elif tk.is_conv(): + if tk.is_conv_bwd(): + self._result_data.overall_metrics.update_conv_bwd_cube_info(tk.dur) + else: + self._result_data.overall_metrics.update_conv_fwd_cube_info(tk.dur) + return + elif tk.is_matmul(): + self._result_data.overall_metrics.update_matmul_cube_info(tk.dur) + return + if tk.is_cube_kernel_cat(): + self._result_data.overall_metrics.update_other_cube_info(tk.dur) + elif tk.is_trans(): + self._result_data.overall_metrics.update_vector_trans_info(tk.dur) + else: + self._result_data.overall_metrics.update_vector_notrans_info(tk.dur) + + def _categorize_cube_performance_data(self, cpu_op: TraceEventBean, tk: (TraceEventBean, KernelDetailsBean)): + """ + 判断fa/conv/matmul/vector使用cpu_op + """ + if cpu_op.is_fa_for_cpu_op(): + if self._is_backward(cpu_op): + if tk.is_cube_kernel_cat(): + self._result_data.overall_metrics.update_fa_bwd_cube_info(tk.dur) + else: + self._result_data.overall_metrics.update_fa_bwd_vector_info(tk.dur) + else: + if tk.is_cube_kernel_cat(): + self._result_data.overall_metrics.update_fa_fwd_cube_info(tk.dur) + else: + self._result_data.overall_metrics.update_fa_fwd_vector_info(tk.dur) + elif cpu_op.is_conv_for_cpu_op(): + if self._is_backward(cpu_op): + if tk.is_cube_kernel_cat(): + self._result_data.overall_metrics.update_conv_bwd_cube_info(tk.dur) + else: + self._result_data.overall_metrics.update_conv_bwd_vector_info(tk.dur) + else: + if tk.is_cube_kernel_cat(): + self._result_data.overall_metrics.update_conv_fwd_cube_info(tk.dur) + else: + self._result_data.overall_metrics.update_conv_fwd_vector_info(tk.dur) + elif cpu_op.is_matmul_for_cpu_op(): # matmul + if tk.is_cube_kernel_cat(): + self._result_data.overall_metrics.update_matmul_cube_info(tk.dur) + else: + self._result_data.overall_metrics.update_matmul_vector_info(tk.dur) + + def _is_backward(self, event: TraceEventBean): + return event.tid == self._bwd_tid or event.is_bwd_for_cpu_op() + + def _get_flow_time_dict(self): + return { + flow_event["end"].start_time: flow_event["start"].start_time + for flow_event in self._flow_dict.values() + if flow_event.get("end") and flow_event.get("start") + } + def _dispatch_events(self): if not self._dispatch_func: return diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index c4089aec9..7b1ae1a5a 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -20,6 +20,7 @@ class GPUProfilingParser(BaseProfilingParser): self._compute_stream_id = self._infer_compute_stream_id() self._marks = defaultdict(int) self._aten_index = 0 + self._find_bwd_tid() @classmethod def __is_flash_attention(cls, name: str): @@ -30,10 +31,7 @@ class GPUProfilingParser(BaseProfilingParser): @classmethod def __is_sdma_time(cls, name: str): - for mark in cls.SDMA_MARK_LIST: - if mark in name.lower(): - return True - return False + return any(mask in name.lower() for mask in cls.SDMA_MARK_LIST) def _update_memory_list(self): if not self._enable_memory_compare: @@ -68,19 +66,15 @@ class GPUProfilingParser(BaseProfilingParser): min_ts = sys.float_info.max max_ts = sys.float_info.min self._trace_events.sort(key=lambda x: x.start_time) - aten_events = list(filter(lambda x: x.name.startswith("aten::"), self._trace_events)) - flow_dict_new = {} - for flow_event in self._flow_dict.values(): - start_event = flow_event.get("start") - end_event = flow_event.get("end") - if start_event and end_event: - flow_dict_new[end_event.start_time] = start_event.start_time + aten_events = [event for event in self._trace_events if event.name.startswith("aten::")] + flow_dict_new = self._get_flow_time_dict() for event in self._trace_events: if event.stream: min_ts = min(event.start_time, min_ts) max_ts = max(event.end_time, max_ts) if event.stream == self._compute_stream_id and self.__is_sdma_time(event.name): self._result_data.overall_metrics.update_sdma_info(event.dur) + self._result_data.overall_metrics.update_sdma_stream_info(event.dur) continue if not event.is_kernel_cat(): continue @@ -88,6 +82,7 @@ class GPUProfilingParser(BaseProfilingParser): if event.is_nccl_name(): continue self.__add_compute_time(event, aten_events, flow_dict_new) + self.categorize_computing_performance_data(event, flow_dict_new) self._aten_events = None self._result_data.overall_metrics.set_e2e_time(float(max_ts - min_ts)) self.__add_compute_and_overlap_time() @@ -162,7 +157,7 @@ class GPUProfilingParser(BaseProfilingParser): def _get_dispatch_func(self): func_set = set() - if self._enable_memory_compare or self._enable_operator_compare: + if self._enable_memory_compare or self._enable_operator_compare or self._enable_profiling_compare: func_set.add(self._picking_torch_op_event) if self._enable_communication_compare: func_set.add(self._picking_kernel_event) @@ -174,6 +169,8 @@ class GPUProfilingParser(BaseProfilingParser): func_set.add(self._picking_flow_event) if self._enable_memory_compare or self._enable_profiling_compare: func_set.add(self._picking_memory_event) + if self._enable_profiling_compare: + func_set.add(self._picking_flow_event) return list(func_set) def _infer_compute_stream_id(self): @@ -187,3 +184,9 @@ class GPUProfilingParser(BaseProfilingParser): raise RuntimeError('[ERROR] The profiling data does not contain kernel running data.') counter = Counter(kernel_stream_ids) return counter.most_common(1)[0][0] + + def _find_bwd_tid(self): + for event in self._trace_events: + if event.is_fwdbwd() and event.is_flow_end(): + self._bwd_tid = event.tid + break diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 70ce44b44..457a3b6be 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -36,7 +36,7 @@ class NPUProfilingParser(BaseProfilingParser): def _get_dispatch_func(self): func_list = set() - if self._enable_memory_compare or self._enable_operator_compare: + if self._enable_memory_compare or self._enable_operator_compare or self._enable_profiling_compare: func_list.add(self._picking_torch_op_event) if self._enable_operator_compare or self._args.max_kernel_num: func_list.add(self._picking_kernel_event) @@ -52,6 +52,7 @@ class NPUProfilingParser(BaseProfilingParser): func_list.add(self._picking_overlap_analysis_data) func_list.add(self._picking_kernel_event) func_list.add(self._picking_hccl_event) + func_list.add(self._picking_flow_event) return list(func_list) def _update_memory_list(self): @@ -205,6 +206,8 @@ class NPUProfilingParser(BaseProfilingParser): def _filter_meta_id(self): for event in self._trace_events: + if event.is_fwdbwd() and event.is_flow_end(): + self._bwd_tid = event.tid if not event.is_process_meta(): continue if event.is_hccl_process_name(): @@ -244,17 +247,7 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.update_lccl_info(event.dur) def __parse_kernel_csv(self): - try: - kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) - except Exception: - print('[WARNING] Npu kernel details csv file is not available.') - return - if not kernel_details or kernel_details[0].is_hide_op_pmu(): - self._result_data.overall_metrics.hide_op_details = True - return - for kernel in kernel_details: - if kernel.is_invalid(): - continue + def __screen_data(kernel: KernelDetailsBean): if kernel.is_flash_attention(): if kernel.is_fa_bwd(): self._result_data.overall_metrics.update_fa_bwd_info(kernel.duration) @@ -265,7 +258,7 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.update_conv_bwd_info(kernel.duration) else: self._result_data.overall_metrics.update_conv_fwd_info(kernel.duration) - elif kernel.is_cube(): + elif kernel.is_matmul(): self._result_data.overall_metrics.update_cube_info(kernel.duration) elif kernel.is_sdma(): self._result_data.overall_metrics.update_sdma_info(kernel.duration) @@ -276,6 +269,22 @@ class NPUProfilingParser(BaseProfilingParser): else: self._result_data.overall_metrics.update_cube_info(kernel.duration) + try: + kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) + except Exception: + print('[WARNING] Npu kernel details csv file is not available.') + return + if not kernel_details or kernel_details[0].is_hide_op_pmu(): + self._result_data.overall_metrics.hide_op_details = True + return + flow_dict_new = self._get_flow_time_dict() + kernel_details.sort(key=lambda x: x.start_time) + for kernel in kernel_details: + if kernel.is_invalid(): + continue + __screen_data(kernel) + self.categorize_computing_performance_data(kernel, flow_dict_new) + def __parse_mem_csv(self): try: memory_record = FileReader.read_csv_file(self._memory_record_path, MemoryRecordBean) @@ -321,3 +330,4 @@ class NPUProfilingParser(BaseProfilingParser): for stream in compute_stream: dur_list = sdma_dict.get(stream, []) self._result_data.overall_metrics.update_sdma_info(sum(dur_list), len(dur_list)) + self._result_data.overall_metrics.update_sdma_stream_info(sum(dur_list), len(dur_list)) diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index e2854692a..e20025880 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -11,6 +11,7 @@ class Constant(object): GREEN_COLOR = "00FF00" RED_COLOR = "FF0000" BLUE_COLOR = "00BFFF" + LIGHT_BLUE_COLOR = "87CEFA" US_TO_MS = 1000 KB_TO_MB = 1024 INVALID_VALUE = -1 @@ -55,6 +56,7 @@ class Constant(object): PERFORMANCE_TABLE = "Model Profiling Time Distribution" MODULE_TABLE = "ModuleCompare" MODULE_TOP_TABLE = "ModuleCompareStatistic" + OVERALL_METRICS_TABLE = "OverallMetrics" # memory SIZE = "Size(KB)" @@ -78,3 +80,9 @@ class Constant(object): OVERALL_COMPARE = "overall" BWD_LIST = ["bwd", "backward", "back"] + + CPU_OP_FA_MASK = ("flash_attention", "fusion_attention", "flashattn", "xformers_flash", "efficient_attention") + CPU_OP_CONV = "aten::conv" + CPU_OP_MATMUL_MASK = ("aten::addmm", "aten::bmm", "aten::mm", "aten::matmul") + KERNEL_CUBE_MASK = ("gemm", "conv", "cutlass", "wgrad") + KERNEL_TRANS_MASK = ("cast", "transdata", "transpose") diff --git a/profiler/compare_tools/compare_backend/utils/excel_config.py b/profiler/compare_tools/compare_backend/utils/excel_config.py index 306abcdfe..ae808863e 100644 --- a/profiler/compare_tools/compare_backend/utils/excel_config.py +++ b/profiler/compare_tools/compare_backend/utils/excel_config.py @@ -18,6 +18,8 @@ class CellFormatType: 'valign': 'vcenter', 'bold': True, 'border': True} # 绿色背景，加粗 YELLOW_BOLD = {"font_name": "Arial", 'font_size': 11, 'fg_color': Constant.YELLOW_COLOR, 'align': 'left', 'valign': 'vcenter', 'bold': True, 'border': True} # 黄色背景，加粗 + BLUE_NORMAL = {'fg_color': Constant.BLUE_COLOR} # 蓝色背景，主要用于行样式 + LIGHT_BLUE_NORMAL = {'fg_color': Constant.LIGHT_BLUE_COLOR} # 淡蓝色背景，主要用于行样式 class ExcelConfig(object): @@ -65,6 +67,10 @@ class ExcelConfig(object): MODULE_LEVEL = "Module Level" BASE_CALL_STACK = "Base Call Stack" COMPARISON_CALL_STACK = "Comparison Call Stack" + INDEX = "Index" + DURATION = "Duration(ms)" + DURATION_RATIO = "Duration Ratio" + DIFF_DUR_MS = "Diff Duration(ms)" HEADERS = { Constant.OPERATOR_TABLE: [ @@ -176,10 +182,81 @@ class ExcelConfig(object): {"name": DIFF_TOTAL_RATIO, "type": CellFormatType.DEFAULT_RATIO, "width": 15}, {"name": BASE_CALL_STACK, "type": CellFormatType.DEFAULT, "width": 30}, {"name": COMPARISON_CALL_STACK, "type": CellFormatType.DEFAULT, "width": 30} + ], + Constant.OVERALL_METRICS_TABLE: [ + {"name": INDEX, "type": CellFormatType.DEFAULT, "width": 40}, + {"name": DURATION, "type": CellFormatType.DEFAULT_FLOAT, "width": 20}, + {"name": DURATION_RATIO, "type": CellFormatType.DEFAULT_RATIO, "width": 20}, + {"name": NUMBER, "type": CellFormatType.DEFAULT, "width": 10}, + {"name": DURATION, "type": CellFormatType.DEFAULT_FLOAT, "width": 20}, + {"name": DURATION_RATIO, "type": CellFormatType.DEFAULT_RATIO, "width": 20}, + {"name": NUMBER, "type": CellFormatType.DEFAULT, "width": 10}, + {"name": DIFF_DUR_MS, "type": CellFormatType.DEFAULT_FLOAT, "width": 20}, + {"name": DIFF_RATIO, "type": CellFormatType.DEFAULT_RATIO, "width": 10}, + ] } OVERHEAD = {Constant.OPERATOR_TABLE: ["B1:F1", "G1:K1"], Constant.MEMORY_TABLE: ["B1:F1", "G1:K1"], Constant.COMMUNICATION_TABLE: ["B1:H1", "I1:O1"], Constant.OPERATOR_TOP_TABLE: ["C1:D1", "E1:F1"], Constant.MEMORY_TOP_TABLE: ["C1:E1", "F1:H1"], Constant.MODULE_TOP_TABLE: ["F1:I1", "J1:M1"], - Constant.MODULE_TABLE: ["E1:H1", "I1:L1"]} + Constant.MODULE_TABLE: ["E1:H1", "I1:L1"], + Constant.OVERALL_METRICS_TABLE: ["B1:D1", "E1:G1"]} + + # overall metrics index + # computing time + COMPUTING = "Computing Time" + + FA = "\tFlash Attention" + FA_FWD_CUBE = "\t\tFlash Attention (Forward) (Cube)" + FA_FWD_VECTOR = "\t\tFlash Attention (Forward) (Vector)" + FA_BWD_CUBE = "\t\tFlash Attention (Backward) (Cube)" + FA_BWD_VECTOR = "\t\tFlash Attention (Backward) (Vector)" + + CONV = "\tConv" + CONV_FWD_CUBE = "\t\tConv (Forward) (Cube)" + CONV_FWD_VECTOR = "\t\tConv (Forward) (Vector)" + CONV_BWD_CUBE = "\t\tConv (Backward) (Cube)" + CONV_BWD_VECTOR = "\t\tConv (Backward) (Vector)" + + MM = "\tMatmul" + MM_CUBE = "\t\tMatmul (Cube)" + MM_VECTOR = "\t\tMatmul (Vector)" + + PA = "\tPage Attention" + + VECTOR = "\tVector" + VECTOR_TRANS = "\t\tVector (Trans)" + VECTOR_NO_TRANS = "\t\tVector (No Trans)" + + CUBE = "\tCube" + SDMA_TM = "\tSDMA (Tensor Move)" + OTHER = "\tOther" + + # communication time + COMMUNICATION_TIME = "Uncovered Communication Time" + WAIT = "\tWait" + TRANSMIT = "\tTransmit" + + # free time + FREE_TIME = "Free Time" + SDMA = "\tSDMA" + FREE = "\tFree" + + # e2e time + E2E_TIME = "E2E Time" + + ROW_STYLE_MAP = { + COMPUTING: CellFormatType.BLUE_NORMAL, + COMMUNICATION_TIME: CellFormatType.BLUE_NORMAL, + FREE_TIME: CellFormatType.BLUE_NORMAL, + E2E_TIME: CellFormatType.BLUE_NORMAL, + FA: CellFormatType.LIGHT_BLUE_NORMAL, + CONV: CellFormatType.LIGHT_BLUE_NORMAL, + MM: CellFormatType.LIGHT_BLUE_NORMAL, + PA: CellFormatType.LIGHT_BLUE_NORMAL, + VECTOR: CellFormatType.LIGHT_BLUE_NORMAL, + CUBE: CellFormatType.LIGHT_BLUE_NORMAL, + SDMA_TM: CellFormatType.LIGHT_BLUE_NORMAL, + OTHER: CellFormatType.LIGHT_BLUE_NORMAL + } diff --git a/profiler/compare_tools/compare_backend/view/work_sheet_creator.py b/profiler/compare_tools/compare_backend/view/work_sheet_creator.py index 7a33168da..dffb7549f 100644 --- a/profiler/compare_tools/compare_backend/view/work_sheet_creator.py +++ b/profiler/compare_tools/compare_backend/view/work_sheet_creator.py @@ -20,7 +20,10 @@ class WorkSheetCreator: return self._work_sheet = self._work_book.add_worksheet(self._sheet_name) self._write_headers() - self._write_data() + if "row_style" in self._data: + self._write_data_with_row_style() + else: + self._write_data() def _write_headers(self): base_header_format = self._work_book.add_format(CellFormatType.GREEN_BOLD) @@ -43,7 +46,7 @@ class WorkSheetCreator: col_id = self._col_ids[index] self._work_sheet.set_column(f"{col_id}:{col_id}", header.get("width")) self._work_sheet.write(f"{col_id}{self._row_id}", header.get("name"), header_format) - self._field_format[index] = self._work_book.add_format(header.get("type")) + self._field_format[index] = header.get("type") if header.get("name") in (ExcelConfig.DIFF_RATIO, ExcelConfig.DIFF_TOTAL_RATIO): self._diff_ratio_index = index self._row_id += 1 @@ -52,7 +55,27 @@ class WorkSheetCreator: red_ratio_format = self._work_book.add_format(CellFormatType.RED_RATIO) for data in self._data.get("rows"): for index, cell_data in enumerate(data): - cell_format = self._field_format.get(index) + cell_format = self._work_book.add_format(self._field_format.get(index)) + if index == self._diff_ratio_index and cell_data and cell_data > 1: + cell_format = red_ratio_format + cell_data = "INF" if cell_data == float('inf') else cell_data + self._work_sheet.write(f"{self._col_ids[index]}{self._row_id}", cell_data, cell_format) + self._row_id += 1 + + def _write_data_with_row_style(self): + """ + 带行样式及缩进的sheet + """ + red_ratio_format = self._work_book.add_format(CellFormatType.RED_RATIO) + rows = self._data.get("rows") + row_style = self._data.get("row_style") # 行样式 + + for data, row_style in zip(rows, row_style): + for index, cell_data in enumerate(data): + cell_style = {**self._field_format.get(index), **row_style} + if index == 0: # 0 for Index field + cell_style["indent"] = cell_data.count("\t") + cell_format = self._work_book.add_format(cell_style) if index == self._diff_ratio_index and cell_data and cell_data > 1: cell_format = red_ratio_format cell_data = "INF" if cell_data == float('inf') else cell_data diff --git a/profiler/test/run_ut.py b/profiler/test/run_ut.py index ee27abaac..6ab208dc2 100644 --- a/profiler/test/run_ut.py +++ b/profiler/test/run_ut.py @@ -13,6 +13,7 @@ def set_python_path(): os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "advisor") advisor_backend_root = os.path.join( os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "advisor", "advisor_backend") + profiler_parent_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # Update PYTHONPATH python_path = os.environ.get("PYTHONPATH", "") if not python_path: @@ -22,6 +23,7 @@ def set_python_path(): python_path += f":{compare_tools_root}" python_path += f":{advisor_root}" python_path += f":{advisor_backend_root}" + python_path += f":{profiler_parent_dir}" os.environ["PYTHONPATH"] = python_path diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py new file mode 100644 index 000000000..3d8e22b7c --- /dev/null +++ b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py @@ -0,0 +1,65 @@ +import unittest +import os +import sys +import yaml + +from profiler.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +class TestDataloaderChecker(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), + "advisor", "rules", "dataloader.yaml") + + with open(rule_path, "rb") as file: + self.rule = yaml.safe_load(file) + + def test_no_dataloader(self): + dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) * 1000 + dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=True) + + checker = DataloaderChecker() + checker.check_slow_dataloader(dataset) + self.assertFalse(checker.dataloader_issues) + + def test_no_slow_dataloader(self): + dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) * 1000 + dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=False) + checker = DataloaderChecker() + checker.check_slow_dataloader(dataset) + self.assertFalse(checker.dataloader_issues) + + def test_found_slow_dataloader(self): + dataloader_duration = (self.rule.get("dataloader_duration_threshold") + 1) * 1000 + dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=False) + checker = DataloaderChecker() + checker.check_slow_dataloader(dataset) + self.assertTrue(checker.dataloader_issues) + + desc = self.rule.get("problem").format(dataloader_duration=dataloader_duration / 1000, + dataloader_duration_threshold=self.rule.get( + "dataloader_duration_threshold")) + + self.assertEqual(desc, checker.desc) + + def _get_mock_dataset(self, dur, is_empty_dataset=False): + dataset = TimelineEvent() + if is_empty_dataset: + return dataset + + dataset["dataloader"] = [TimelineEvent({"dur": dur, "name": "dataloader"})] + return dataset + + +if __name__ == '__main__': + tester = TestDataloaderChecker() + tester.test_no_dataloader() + tester.test_no_slow_dataloader() + tester.test_found_slow_dataloader() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py new file mode 100644 index 000000000..d1df810a0 --- /dev/null +++ b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py @@ -0,0 +1,62 @@ +import unittest +import os +import sys +import yaml + +from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +class TestSyncBNChecker(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), + "advisor", "rules", "sync_batchnorm.yaml") + + with open(rule_path, "rb") as file: + self.rule = yaml.safe_load(file) + + def test_no_syncbn(self): + dataset = self._get_mock_dataset(1, is_empty_dataset=True) + + checker = SyncBNChecker() + checker.check_syncbn(dataset) + self.assertFalse(checker.syncbn_issues) + + def test_syncbn_not_reach_threshold(self): + dataset = self._get_mock_dataset(self.rule.get("max_syncbn_num") - 1, is_empty_dataset=False) + checker = SyncBNChecker() + checker.check_syncbn(dataset) + self.assertFalse(checker.syncbn_issues) + + def test_found_slow_dataloader(self): + dataset = self._get_mock_dataset(self.rule.get("max_syncbn_num") + 1, is_empty_dataset=False) + checker = SyncBNChecker() + checker.check_syncbn(dataset) + self.assertTrue(checker.syncbn_issues) + + desc = self.rule.get("problem").format(syncbn_num=self.rule.get("max_syncbn_num") + 1) + + self.assertEqual(desc, checker.desc) + + def _get_mock_dataset(self, syncbn_num, is_empty_dataset=False): + dataset = TimelineEvent() + if is_empty_dataset: + return dataset + + dataset["sync_batchnorm"] = [] + for _ in range(syncbn_num): + dataset["sync_batchnorm"].append(TimelineEvent({"name": "SyncBatchNorm"})) + return dataset + + +if __name__ == '__main__': + tester = TestSyncBNChecker() + tester.test_no_syncbn() + tester.test_syncbn_not_reach_threshold() + tester.test_found_slow_dataloader() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py new file mode 100644 index 000000000..e87efcc21 --- /dev/null +++ b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py @@ -0,0 +1,55 @@ +import unittest +import os +import sys +import yaml + +from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +class TestSynchronizeChecker(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), + "advisor", "rules", "synchronize.yaml") + + with open(rule_path, "rb") as file: + self.rule = yaml.safe_load(file) + + def test_no_synchronize_stream(self): + dataset = self._get_mock_dataset(1, [], is_empty_dataset=True) + + checker = SynchronizeStreamChecker() + checker.check_synchronize(dataset) + self.assertFalse(checker.synchronize_issues) + + def test_max_synchronize_stream(self): + dataset = self._get_mock_dataset(100, [], is_empty_dataset=False) + checker = SynchronizeStreamChecker() + checker.check_synchronize(dataset) + self.assertTrue(checker.synchronize_issues) + + def _get_mock_dataset(self, total_count, slow_synchronize_stream, is_empty_dataset=False): + dataset = TimelineEvent() + if is_empty_dataset: + return dataset + + dataset["synchronize_stream"] = TimelineEvent( + dict( + total_count=total_count, + slow_synchronize_stream=slow_synchronize_stream, + rule=dict(max_synchronize_num=10, problem="", solutions=[]), + ) + ) + return dataset + + +if __name__ == '__main__': + tester = TestSynchronizeChecker() + tester.test_no_synchronize_stream() + tester.test_max_synchronize_stream() diff --git a/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py b/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py new file mode 100644 index 000000000..51acf3b8e --- /dev/null +++ b/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py @@ -0,0 +1,145 @@ +import os +import shutil +import stat +import json + +import unittest +from profiler.advisor.interface.interface import Interface +from profiler.advisor.common.analyzer_scopes import SupportedScopes + + +class TestFrequencyAdvice(unittest.TestCase): + TMP_DIR = "./ascend_pt" + OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" + DEVICE_DIR = "./ascend_pt/PROF_000001_20240415174447255_OAANHDOMMJMHGIFC/device_0" + interface = None + err_interface = None + + def tearDown(self): + if os.path.exists(TestFrequencyAdvice.TMP_DIR): + shutil.rmtree(TestFrequencyAdvice.TMP_DIR) + self.clear_htmls() + + def setUp(self): + if os.path.exists(TestFrequencyAdvice.TMP_DIR): + shutil.rmtree(TestFrequencyAdvice.TMP_DIR) + if not os.path.exists(TestFrequencyAdvice.TMP_DIR): + os.makedirs(TestFrequencyAdvice.TMP_DIR) + if not os.path.exists(TestFrequencyAdvice.OUTPUT_DIR): + os.makedirs(TestFrequencyAdvice.OUTPUT_DIR) + if not os.path.exists(TestFrequencyAdvice.DEVICE_DIR): + os.makedirs(TestFrequencyAdvice.DEVICE_DIR) + self.clear_htmls() + + @classmethod + def clear_htmls(cls): + current_path = os.path.dirname(os.path.abspath(__file__)) + for filename in os.listdir(current_path): + # 检查文件是否以“att”开头 + if filename.startswith("att"): + # 构建文件的完整路径 + file_path = os.path.join(current_path, filename) + # 删除文件 + os.remove(file_path) + + @classmethod + def get_basic_trace_view(cls): + # Python pid + py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} + # ascend pid + ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} + # ascend pid + cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} + # ascend hardware ops + ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, + "args": {"Task Type": "AI_CORE"}} + ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, + "args": {"Task Type": "AI_CORE"}} + # flow event + flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} + flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} + return [py_pid_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, flow_event_s, flow_event_e] + + @classmethod + def create_info_json(cls): + info = { + "DeviceInfo": [ + { + "id": 7, + "env_type": 3, + "ctrl_cpu_id": "ARMv8_Cortex_A55", + "ctrl_cpu_core_num": 1, + "ctrl_cpu_endian_little": 1, + "ts_cpu_core_num": 0, + "ai_cpu_core_num": 6, + "ai_core_num": 25, + "ai_cpu_core_id": 2, + "ai_core_id": 0, + "aicpu_occupy_bitmap": 252, + "ctrl_cpu": "0", + "ai_cpu": "2,3,4,5,6", + "aiv_num": 50, + "hwts_frequency": "49.999001", + "aic_frequency": "1850", + "aiv_frequency": "1850" + } + ] + } + with os.fdopen(os.open(f"{TestFrequencyAdvice.DEVICE_DIR}/info.json.0", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(info)) + + @classmethod + def create_non_910B_trace_view(cls): + basic_info = cls.get_basic_trace_view() + + # python ops + py_event1 = {"ph": "X", "cat": "python_function", "name": "aten::slice", "ts": "200", "dur": 100, "tid": 2, + "pid": 1, + "args": {"Call stack": "/root/test/slice.py(116);\r\n/root/torch/module.py"}} + py_event2 = {"ph": "X", "cat": "python_function", "name": "slice", "ts": "199", "dur": 200, "tid": 2, "pid": 1, + "args": {"Call stack": "/root/test/slice.py(116);\r\n/root/torch/module.py"}} + raw_data = [ + *basic_info, py_event1, py_event2 + ] + with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/trace_view.json", + # with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/msprof_20240415174455.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + @classmethod + def create_910B_trace_view(cls): + basic_info = cls.get_basic_trace_view() + + # python ops + py_event1 = {"name": "AI Core Freq", "ts": "1699529623106000.061", "pid": 682820896, "tid": 0, + "args": {"MHz": 1850}, "ph": "C"} + py_event2 = {"name": "AI Core Freq", "ts": "1699529623106770.541", "pid": 682820896, "tid": 0, + "args": {"MHz": 800}, "ph": "C"} + raw_data = [ + *basic_info, py_event1, py_event2 + ] + + with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/trace_view.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + def test_run_should_run_success_when_msprof_not_contain_frequency_data(self): + self.create_info_json() + self.create_non_910B_trace_view() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "computation" + scope = SupportedScopes.FREQ_ANALYSIS + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(0, len(result.data.get("AI Core Frequency", []))) + result.clear() + + def test_run_should_run_success_when_trace_view_contain_frequency_data(self): + self.create_info_json() + self.create_910B_trace_view() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "computation" + scope = SupportedScopes.FREQ_ANALYSIS + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(2, len(result.data.get("AI Core Frequency", dict).get("data", []))) + result.clear() diff --git a/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py b/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py index 7abf8da64..869ee8557 100644 --- a/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py +++ b/profiler/test/ut/compare_tools/compare_bean/origin_data_bean/test_kernel_details_bean.py @@ -47,5 +47,5 @@ class TestKernelDetailsBean(unittest.TestCase): self.assertFalse(self.kernel_bean2.is_flash_attention()) def test_is_cube(self): - self.assertTrue(self.kernel_bean2.is_cube()) - self.assertFalse(self.kernel_bean3.is_cube()) + self.assertTrue(self.kernel_bean2.is_matmul()) + self.assertFalse(self.kernel_bean3.is_matmul()) diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index 044687215..d7cb3d058 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -68,6 +68,7 @@ class TestGpuProfilingParser(unittest.TestCase): patch("compare_backend.profiling_parser.gpu_profiling_parser.GPUProfilingParser.__init__", return_value=None): res = GPUProfilingParser({}, {}) + res._profiling_type = "GPU" res._trace_events = [TraceEventBean(event) for event in self.trace_events] res._result_data = ProfilingResult("GPU") res._compute_stream_id = 3 -- Gitee From b9a14f9a4bc7f3940fdd7082ca219c34f1cf51e1 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 25 Jul 2024 15:35:01 +0800 Subject: [PATCH 016/791] input,output kwargs bugfix --- .../msprobe/pytorch/compare/acc_compare.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index e21491056..cf7ad912e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -764,9 +764,14 @@ def op_item_parse(item, op_name, index, item_list=None, top_bool=True): else: full_op_name = op_name else: - full_op_name = op_name + '.' + str(index) + full_op_name = op_name + Const.SEP + str(index) if isinstance(item, dict): - if 'dtype' in item: + if 'type' not in item: + for kwarg in item: + kwarg_parsed_list = op_item_parse(item[kwarg], op_name + Const.SEP + kwarg, None) + item_list += kwarg_parsed_list + kwarg_parsed_list.clear() + elif 'dtype' in item: parsed_item = item parsed_item['full_op_name'] = full_op_name item_list.append(parsed_item) -- Gitee From 1ad93cf00a318b5b8daf3e99db1dbbac2c3fa26b Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Wed, 24 Jul 2024 16:46:12 +0800 Subject: [PATCH 017/791] feature: api compare and kernel compare --- .../comparator/api_compare_comparator.py | 32 ++++++++ .../comparator/kernel_compare_comparator.py | 35 +++++++++ .../compare_bean/api_compare_bean.py | 47 ++++++++++++ .../compare_bean/kernel_compare_bean.py | 75 +++++++++++++++++++ .../origin_data_bean/kernel_details_bean.py | 6 ++ .../data_prepare/operator_data_prepare.py | 17 +++++ .../generator/detail_performance_generator.py | 22 +++++- .../profiling_parser/base_profiling_parser.py | 16 ++++ .../profiling_parser/gpu_profiling_parser.py | 3 + .../profiling_parser/npu_profiling_parser.py | 24 ++++++ .../compare_backend/utils/args_manager.py | 13 +++- .../compare_backend/utils/compare_args.py | 4 + .../compare_backend/utils/constant.py | 7 +- .../compare_backend/utils/excel_config.py | 48 +++++++++++- .../compare_backend/utils/torch_op_node.py | 8 ++ .../compare_backend/utils/tree_builder.py | 3 +- .../view/work_sheet_creator.py | 12 +-- profiler/compare_tools/performance_compare.py | 2 + .../test_base_profiling_parser.py | 5 ++ 19 files changed, 366 insertions(+), 13 deletions(-) create mode 100644 profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py create mode 100644 profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py create mode 100644 profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py create mode 100644 profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py diff --git a/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py b/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py new file mode 100644 index 000000000..bc5810068 --- /dev/null +++ b/profiler/compare_tools/compare_backend/comparator/api_compare_comparator.py @@ -0,0 +1,32 @@ +from compare_backend.comparator.base_comparator import BaseComparator +from compare_backend.utils.constant import Constant +from compare_backend.utils.common_func import update_order_id + + +class ApiCompareComparator(BaseComparator): + def __init__(self, origin_data: list, bean: any): + super().__init__(origin_data, bean) + + @classmethod + def _aggregated_api_by_name(cls, ops: list): + ops_dict = {} + for op in ops: + ops_dict.setdefault(op.name, []).append(op) + return ops_dict + + def _compare(self): + if not self._origin_data: + return + base_ops = self._origin_data.get(Constant.BASE_DATA, {}) + comparison_ops = self._origin_data.get(Constant.COMPARISON_DATA, {}) + if not base_ops or not comparison_ops: + return + base_aggregated_ops = self._aggregated_api_by_name(base_ops) + comparison_aggregated_ops = self._aggregated_api_by_name(comparison_ops) + for op_name, base_data in base_aggregated_ops.items(): + comparsion_data = comparison_aggregated_ops.pop(op_name, []) + self._rows.append(self._bean(op_name, base_data, comparsion_data).row) + if comparison_aggregated_ops: + for op_name, comparison_data in comparison_aggregated_ops.items(): + self._rows.append(self._bean(op_name, [], comparison_data).row) + update_order_id(self._rows) diff --git a/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py b/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py new file mode 100644 index 000000000..13c0f776a --- /dev/null +++ b/profiler/compare_tools/compare_backend/comparator/kernel_compare_comparator.py @@ -0,0 +1,35 @@ +from compare_backend.comparator.base_comparator import BaseComparator +from compare_backend.utils.constant import Constant +from compare_backend.utils.common_func import update_order_id + + +class KernelCompareComparator(BaseComparator): + def __init__(self, origin_data: list, bean: any): + super().__init__(origin_data, bean) + + @classmethod + def _aggregated_kernel_by_type_and_shape(cls, kernels: dict): + result_dict = {} + for type_shape, shape_values in kernels.items(): + for shape, kernel_data in shape_values.items(): + kernel = [single[1] for single in kernel_data] + result_list = [type_shape, shape, sum(kernel), len(kernel), max(kernel), min(kernel)] + result_dict.setdefault(f"{type_shape}{shape}", []).extend(result_list) + return result_dict + + def _compare(self): + if not self._origin_data: + return + base_kernels = self._origin_data.get(Constant.BASE_DATA, {}) + comparison_kernels = self._origin_data.get(Constant.COMPARISON_DATA, {}) + if not base_kernels or not comparison_kernels: + return + base_aggregated_kernels = self._aggregated_kernel_by_type_and_shape(base_kernels) + comparison_aggregated_kernels = self._aggregated_kernel_by_type_and_shape(comparison_kernels) + for type_shape, base_data in base_aggregated_kernels.items(): + comparsion_data = comparison_aggregated_kernels.pop(type_shape, []) + self._rows.append(self._bean(base_data, comparsion_data).row) + if comparison_aggregated_kernels: + for _, comparison_data in comparison_aggregated_kernels.items(): + self._rows.append(self._bean([], comparison_data).row) + update_order_id(self._rows) \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py new file mode 100644 index 000000000..55e08a86b --- /dev/null +++ b/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py @@ -0,0 +1,47 @@ +from compare_backend.utils.common_func import calculate_diff_ratio +from compare_backend.utils.constant import Constant +from compare_backend.utils.excel_config import ExcelConfig + + +class ApiInfo: + def __init__(self, op_name: str, data_list: list): + self._data_list = data_list + self.name = op_name + self.total_dur = 0.0 + self.self_time = 0.0 + self.avg_dur = 0.0 + self.number = len(data_list) + self._get_info() + + def _get_info(self): + for data in self._data_list: + self.total_dur += data.api_dur + self.self_time += data.api_self_time + self.total_dur /= 1000.0 + self.self_time /= 1000.0 + self.avg_dur = self.total_dur / self.number if self.number else 0.0 + + +class ApiCompareBean: + TABLE_NAME = Constant.API_TABLE + HEADERS = ExcelConfig.HEADERS.get(TABLE_NAME) + OVERHEAD = ExcelConfig.OVERHEAD.get(TABLE_NAME) + + def __init__(self, op_name: str, base_api: list, comparison_api: list): + self._name = op_name + self._base_api = ApiInfo(op_name, base_api) + self._comparison_api = ApiInfo(op_name, comparison_api) + + @property + def row(self): + row = [None, self._name, + self._base_api.total_dur, self._base_api.self_time, self._base_api.avg_dur, self._base_api.number, + self._comparison_api.total_dur, self._comparison_api.self_time, + self._comparison_api.avg_dur, self._comparison_api.number] + diff_fields = [calculate_diff_ratio(self._base_api.total_dur, self._comparison_api.total_dur)[1], + calculate_diff_ratio(self._base_api.self_time, self._comparison_api.self_time)[1], + calculate_diff_ratio(self._base_api.avg_dur, self._comparison_api.avg_dur)[1], + calculate_diff_ratio(self._base_api.number, self._comparison_api.number)[1]] + row.extend(diff_fields) + return row + diff --git a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py new file mode 100644 index 000000000..df96addc4 --- /dev/null +++ b/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py @@ -0,0 +1,75 @@ +from compare_backend.utils.common_func import calculate_diff_ratio +from compare_backend.utils.constant import Constant +from compare_backend.utils.excel_config import ExcelConfig + + +class KernelCompareInfo: + def __init__(self, data_list: list): + self._kernel_type = None + self._input_shapes = None + self._total_dur = None + self._number = None + self._max_dur = None + self._min_dur = None + if not data_list: + return + self._kernel_type = data_list[0] + self._input_shapes = data_list[1] + self._total_dur = data_list[2] + self._number = data_list[3] + self._max_dur = data_list[4] + self._min_dur = data_list[5] + + @property + def kernel_type(self): + return self._kernel_type + + @property + def input_shapes(self): + return self._input_shapes + + @property + def total_dur(self): + return self._total_dur if self._total_dur else 0.0 + + @property + def number(self): + return self._number + + @property + def max_dur(self): + return self._max_dur + + @property + def min_dur(self): + return self._min_dur + + @property + def avg_dur(self): + return self._total_dur / self._number if self._total_dur and self._number else 0.0 + + +class KernelCompareBean: + TABLE_NAME = Constant.KERNEL_TABLE + HEADERS = ExcelConfig.HEADERS.get(TABLE_NAME) + OVERHEAD = ExcelConfig.OVERHEAD.get(TABLE_NAME) + + def __init__(self, base_kernel: list, comparison_kernel: list): + self._base_kernel = KernelCompareInfo(base_kernel) + self._comparison_kernel = KernelCompareInfo(comparison_kernel) + self._kernel_type = self._base_kernel.kernel_type \ + if self._base_kernel.kernel_type else self._comparison_kernel.kernel_type + self._input_shapes = self._base_kernel.input_shapes \ + if self._base_kernel.input_shapes else self._comparison_kernel.input_shapes + + @property + def row(self): + row = [None, self._kernel_type, self._input_shapes, + self._base_kernel.total_dur, self._base_kernel.avg_dur, + self._base_kernel.max_dur, self._base_kernel.min_dur, self._base_kernel.number, + self._comparison_kernel.total_dur, self._comparison_kernel.avg_dur, + self._comparison_kernel.max_dur, self._comparison_kernel.min_dur, self._comparison_kernel.number] + diff_fields = [calculate_diff_ratio(self._base_kernel.total_dur, self._comparison_kernel.total_dur)[1], + calculate_diff_ratio(self._base_kernel.avg_dur, self._comparison_kernel.avg_dur)[1]] + row.extend(diff_fields) + return row \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py index 9c4825c0e..c15396e9c 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py @@ -12,6 +12,7 @@ class KernelDetailsBean: self._data = data self._op_type = "" self._name = "" + self._input_shapes = "" self._aiv_vec_time = 0.0 self._aicore_time = 0.0 self._mac_time = 0.0 @@ -27,6 +28,10 @@ class KernelDetailsBean: def name(self) -> str: return self._name + @property + def input_shapes(self) -> str: + return self._input_shapes + @property def aiv_vec_time(self) -> float: if self._aiv_vec_time == "" or self._aiv_vec_time == "N/A": @@ -109,6 +114,7 @@ class KernelDetailsBean: def init(self): self._op_type = self._data.get('Type', "") self._name = self._data.get('Name', "") + self._input_shapes = self._data.get('Input Shapes', "") self._aiv_vec_time = self._data.get('aiv_vec_time(us)', "") self._aicore_time = self._data.get("aicore_time(us)", "") self._mac_time = self._data.get('mac_time(us)', "") diff --git a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py index fdce23c6a..3106527c4 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py +++ b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py @@ -17,3 +17,20 @@ class OperatorDataPrepare: else: result_data.append(level1_node) return result_data + + def get_all_layer_ops(self) -> any: + root_node = TreeBuilder.build_tree(self.profiling_data.torch_op_data, [], []) + level1_child_nodes = root_node.child_nodes + node_queue = [] + result_data = [] + for level1_node in level1_child_nodes: + if level1_node.is_step_profiler(): + node_queue.extend(level1_node.child_nodes) + else: + node_queue.append(level1_node) + while len(node_queue) > 0: + node = node_queue.pop(0) + result_data.append(node) + if node.child_nodes: + node_queue.extend(node.child_nodes) + return result_data \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py index 292e31281..6fe693fb0 100644 --- a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -8,6 +8,8 @@ from compare_backend.comparator.module_comparetor import ModuleComparator from compare_backend.comparator.module_statistic_comparator import ModuleStatisticComparator from compare_backend.comparator.operator_comparator import OperatorComparator from compare_backend.comparator.operator_statistic_comparator import OperatorStatisticComparator +from compare_backend.comparator.api_compare_comparator import ApiCompareComparator +from compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator from compare_backend.comparator.overall_metrics_comparator import OverallMetricsComparator from compare_backend.compare_bean.communication_bean import CommunicationBean from compare_backend.compare_bean.memory_compare_bean import MemoryCompareBean @@ -16,6 +18,8 @@ from compare_backend.compare_bean.module_compare_bean import ModuleCompareBean from compare_backend.compare_bean.module_statistic_bean import ModuleStatisticBean from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean from compare_backend.compare_bean.operator_statistic_bean import OperatorStatisticBean +from compare_backend.compare_bean.api_compare_bean import ApiCompareBean +from compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean from compare_backend.compare_bean.overall_metrics_bean import OverallMetricsBean from compare_backend.data_prepare.module_data_prepare import ModuleDataPrepare from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare @@ -39,8 +43,10 @@ class DetailPerformanceGenerator(BaseGenerator): return op_compare_result def compare(self): - if self._args.enable_operator_compare or self._args.enable_memory_compare or \ - self._args.enable_communication_compare: + enable_compare = [self._args.enable_operator_compare, self._args.enable_memory_compare, + self._args.enable_communication_compare, self._args.enable_api_compare, + self._args.enable_kernel_compare] + if any(enable_compare): print("[INFO] Start to compare performance detail data, please wait.") comparator_list = self._create_comparator() else: @@ -97,6 +103,18 @@ class DetailPerformanceGenerator(BaseGenerator): comparator_list.append(OperatorStatisticComparator(op_compare_result, MemoryStatisticBean)) if not self._args.disable_details: comparator_list.append(OperatorComparator(op_compare_result, MemoryCompareBean)) + if self._args.enable_api_compare: + api_compare_result = { + Constant.BASE_DATA: OperatorDataPrepare( + self._profiling_data_dict.get(Constant.BASE_DATA)).get_all_layer_ops(), + Constant.COMPARISON_DATA: OperatorDataPrepare( + self._profiling_data_dict.get(Constant.COMPARISON_DATA)).get_all_layer_ops()} + comparator_list.append(ApiCompareComparator(api_compare_result, ApiCompareBean)) + if self._args.enable_kernel_compare: + kernel_compare_result = { + Constant.BASE_DATA: self._profiling_data_dict.get(Constant.BASE_DATA).kernel_details, + Constant.COMPARISON_DATA: self._profiling_data_dict.get(Constant.COMPARISON_DATA).kernel_details} + comparator_list.append(KernelCompareComparator(kernel_compare_result, KernelCompareBean)) return comparator_list def match_torch_op(self) -> list: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 6ee07a656..625eee7c6 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -20,6 +20,7 @@ class ProfilingResult: self.overall_metrics = ProfilingInfo(profiling_type) self.python_function_data = [] self.fwdbwd_dict = {} + self.kernel_details = {} def update_torch_op_data(self, event: TraceEventBean): event.is_torch_op = True @@ -43,6 +44,9 @@ class ProfilingResult: def update_comm_task_data(self, comm_name: str, task_event: TraceEventBean): self.communication_dict.setdefault(comm_name, {}).setdefault("comm_task", {}).setdefault( task_event.name, []).append(task_event.dur) + + def update_kernel_details(self, kernels: dict): + self.kernel_details = kernels class BaseProfilingParser(ABC): @@ -57,6 +61,8 @@ class BaseProfilingParser(ABC): self._enable_operator_compare = args.enable_operator_compare self._enable_memory_compare = args.enable_memory_compare self._enable_communication_compare = args.enable_communication_compare + self._enable_api_compare = args.enable_api_compare + self._enable_kernel_compare = args.enable_kernel_compare self._dispatch_func = self._get_dispatch_func() self._result_data = ProfilingResult(self._profiling_type) self._memory_events = [] @@ -80,6 +86,10 @@ class BaseProfilingParser(ABC): self._cpu_cube_op = cpu_cube_op return self._cpu_cube_op + @abstractmethod + def _update_kernel_details(self): + raise NotImplementedError("Function _update_kernel_details need to be implemented.") + @abstractmethod def _update_memory_list(self): raise NotImplementedError("Function _update_memory_list need to be implemented.") @@ -112,6 +122,8 @@ class BaseProfilingParser(ABC): self._update_memory_list() if self._enable_profiling_compare: self._update_overall_metrics() + if self._enable_kernel_compare: + self._update_kernel_details() self._check_result_data() return self._result_data @@ -300,6 +312,10 @@ class BaseProfilingParser(ABC): print(f"[WARNING] Can't find any memory event in the file: {self._profiling_path}") if self._enable_communication_compare and not self._result_data.communication_dict: print(f"[WARNING] Can't find any communication op in the file: {self._profiling_path}") + if self._enable_api_compare and not self._result_data.torch_op_data: + print(f"[WARNING] Can't find any torch op in the file: {self._profiling_path}") + if self._enable_kernel_compare and not self._result_data.kernel_details: + print(f"[WARNING] Can't find any kernel details in the file: {self._profiling_path}") def _read_trace_event(self): try: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 7b1ae1a5a..ea732a60e 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -33,6 +33,9 @@ class GPUProfilingParser(BaseProfilingParser): def __is_sdma_time(cls, name: str): return any(mask in name.lower() for mask in cls.SDMA_MARK_LIST) + def _update_kernel_details(self): + pass + def _update_memory_list(self): if not self._enable_memory_compare: return diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 457a3b6be..cb25c252c 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -53,8 +53,32 @@ class NPUProfilingParser(BaseProfilingParser): func_list.add(self._picking_kernel_event) func_list.add(self._picking_hccl_event) func_list.add(self._picking_flow_event) + if self._enable_api_compare: + func_list.add(self._picking_torch_op_event) return list(func_list) + def _update_kernel_details(self): + try: + kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) + except FileNotFoundError: + print("[WARNING] The file kernel_details.csv does not exist.") + except Exception: + print("[ERROR] Failed to read kernel_details.csv.") + return + if not kernel_details: + return + kernels_dict = {} + for kernel in kernel_details: + if kernel.is_invalid(): + continue + input_shapes = kernel.input_shapes if kernel.input_shapes else 'N/A' + kernels_dict.setdefault(kernel.op_type, {}).setdefault(input_shapes, []).append( + [kernel.name, kernel.duration]) + if len(kernels_dict) == 1: + print("[ERROR] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") + return + self._result_data.update_kernel_details(kernels_dict) + def _update_memory_list(self): try: memory_data = FileReader.read_csv_file(self._operator_memory_path, OperatorMemoryBean) diff --git a/profiler/compare_tools/compare_backend/utils/args_manager.py b/profiler/compare_tools/compare_backend/utils/args_manager.py index 4b5947fa7..ab9fb43a9 100644 --- a/profiler/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/compare_tools/compare_backend/utils/args_manager.py @@ -69,6 +69,14 @@ class ArgsManager: def enable_communication_compare(self): return self._args.enable_communication_compare + @property + def enable_api_compare(self): + return self._args.enable_api_compare + + @property + def enable_kernel_compare(self): + return self._args.enable_kernel_compare + @classmethod def check_profiling_path(cls, file_path: str): PathManager.input_path_common_check(file_path) @@ -119,11 +127,14 @@ class ArgsManager: raise RuntimeError(msg) if not any([self._args.enable_profiling_compare, self._args.enable_operator_compare, - self._args.enable_memory_compare, self._args.enable_communication_compare]): + self._args.enable_memory_compare, self._args.enable_communication_compare, + self._args.enable_api_compare, self._args.enable_kernel_compare]): self._args.enable_profiling_compare = True self._args.enable_operator_compare = True self._args.enable_memory_compare = True self._args.enable_communication_compare = True + self._args.enable_api_compare = True + self._args.enable_kernel_compare = True base_profiling_path = PathManager.get_realpath(self._args.base_profiling_path) self.check_profiling_path(base_profiling_path) diff --git a/profiler/compare_tools/compare_backend/utils/compare_args.py b/profiler/compare_tools/compare_backend/utils/compare_args.py index ab9bc364f..9e6291e89 100644 --- a/profiler/compare_tools/compare_backend/utils/compare_args.py +++ b/profiler/compare_tools/compare_backend/utils/compare_args.py @@ -6,6 +6,8 @@ class Args: enable_operator_compare: bool = False, enable_memory_compare: bool = False, enable_communication_compare: bool = False, + enable_api_compare: bool = False, + enable_kernel_compare: bool = False, output_path: str = "", max_kernel_num: int = None, op_name_map: dict = {}, @@ -17,6 +19,8 @@ class Args: self.enable_operator_compare = enable_operator_compare self.enable_memory_compare = enable_memory_compare self.enable_communication_compare = enable_communication_compare + self.enable_api_compare = enable_api_compare + self.enable_kernel_compare = enable_kernel_compare self.output_path = output_path self.max_kernel_num = max_kernel_num self.op_name_map = op_name_map diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index e20025880..252aa536e 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -39,13 +39,16 @@ class Constant(object): # compare type OPERATOR_COMPARE = "OperatorCompare" MEMORY_COMPARE = "MemoryCompare" - + API_COMPARE = "ApiCompare" + KERNEL_COMPARE = "KernelCompare" # sheet name OPERATOR_SHEET = "OperatorCompare" MEMORY_SHEET = "MemoryCompare" OPERATOR_TOP_SHEET = "OperatorCompareStatistic" MEMORY_TOP_SHEET = "MemoryCompareStatistic" COMMUNICATION_SHEET = "CommunicationCompare" + API_SHEET = "ApiCompare" + KERNEL_SHEET = "KernelCompare" # table name OPERATOR_TABLE = "OperatorCompare" @@ -57,6 +60,8 @@ class Constant(object): MODULE_TABLE = "ModuleCompare" MODULE_TOP_TABLE = "ModuleCompareStatistic" OVERALL_METRICS_TABLE = "OverallMetrics" + API_TABLE = "ApiCompare" + KERNEL_TABLE = "KernelCompare" # memory SIZE = "Size(KB)" diff --git a/profiler/compare_tools/compare_backend/utils/excel_config.py b/profiler/compare_tools/compare_backend/utils/excel_config.py index ae808863e..b6be0ae2e 100644 --- a/profiler/compare_tools/compare_backend/utils/excel_config.py +++ b/profiler/compare_tools/compare_backend/utils/excel_config.py @@ -57,7 +57,7 @@ class ExcelConfig(object): DEVICE_SELF_TIME = "Device Self Time(ms)" DEVICE_TOTAL_TIME = "Device Total Time(ms)" DIFF_SELF_TIME = "Device Self Time Diff(ms)" - DIFF_TOTAL_RATIO = "Total Diff Ratio" + DIFF_TOTAL_RATIO = "Diff Total Ratio" DIFF_TOTAL_TIME = "Device Total Time Diff(ms)" DEVICE_SELF_TIME_US = "Device Self Time(us)" DEVICE_TOTAL_TIME_US = "Device Total Time(us)" @@ -71,6 +71,14 @@ class ExcelConfig(object): DURATION = "Duration(ms)" DURATION_RATIO = "Duration Ratio" DIFF_DUR_MS = "Diff Duration(ms)" + API_NAME = "api name" + TOTAL_DURATION_MS = "Total Duration(ms)" + AVG_DURATION_MS = "Avg Duration(ms)" + SELF_TIME_MS = "Self Time(ms)" + DIFF_SELF_RATIO = "Diff Self Ratio" + DIFF_AVG_RATIO = "Diff Avg Ratio" + DIFF_CALLS_RATIO = "Diff Calls Ratio" + KERNEL = "Kernel" HEADERS = { Constant.OPERATOR_TABLE: [ @@ -193,7 +201,39 @@ class ExcelConfig(object): {"name": NUMBER, "type": CellFormatType.DEFAULT, "width": 10}, {"name": DIFF_DUR_MS, "type": CellFormatType.DEFAULT_FLOAT, "width": 20}, {"name": DIFF_RATIO, "type": CellFormatType.DEFAULT_RATIO, "width": 10}, - + ], + Constant.API_TABLE: [ + {"name": ORDER, "type": CellFormatType.DEFAULT, "width": 10}, + {"name": API_NAME, "type": CellFormatType.BOLD_STR, "width": 30}, + {"name": TOTAL_DURATION_MS, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": SELF_TIME_MS, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": AVG_DURATION_MS, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": CALLS, "type": CellFormatType.DEFAULT,"width": 20}, + {"name": TOTAL_DURATION_MS, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": SELF_TIME_MS, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": AVG_DURATION_MS, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": CALLS, "type": CellFormatType.DEFAULT,"width": 20}, + {"name": DIFF_TOTAL_RATIO, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": DIFF_SELF_RATIO, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": DIFF_AVG_RATIO, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": DIFF_CALLS_RATIO, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + ], + Constant.KERNEL_COMPARE: [ + {"name": ORDER, "type": CellFormatType.DEFAULT, "width": 10}, + {"name": KERNEL, "type": CellFormatType.BOLD_STR, "width": 30}, + {"name": INPUT_SHAPE, "type": CellFormatType.DEFAULT,"width": 20}, + {"name": TOTAL_DURATION, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": AVG_DURATION, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": MAX_DURATION, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": MIN_DURATION, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": CALLS, "type": CellFormatType.DEFAULT,"width": 20}, + {"name": TOTAL_DURATION, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": AVG_DURATION, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": MAX_DURATION, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": MIN_DURATION, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": CALLS, "type": CellFormatType.DEFAULT,"width": 20}, + {"name": DIFF_TOTAL_RATIO, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, + {"name": DIFF_AVG_RATIO, "type": CellFormatType.DEFAULT_FLOAT,"width": 20}, ] } @@ -201,7 +241,9 @@ class ExcelConfig(object): Constant.COMMUNICATION_TABLE: ["B1:H1", "I1:O1"], Constant.OPERATOR_TOP_TABLE: ["C1:D1", "E1:F1"], Constant.MEMORY_TOP_TABLE: ["C1:E1", "F1:H1"], Constant.MODULE_TOP_TABLE: ["F1:I1", "J1:M1"], Constant.MODULE_TABLE: ["E1:H1", "I1:L1"], - Constant.OVERALL_METRICS_TABLE: ["B1:D1", "E1:G1"]} + Constant.OVERALL_METRICS_TABLE: ["B1:D1", "E1:G1"], + Constant.API_TABLE: ["C1:F1", "G1:J1"], + Constant.KERNEL_TABLE: ["D1:H1", "I1:M1"]} # overall metrics index # computing time diff --git a/profiler/compare_tools/compare_backend/utils/torch_op_node.py b/profiler/compare_tools/compare_backend/utils/torch_op_node.py index 690c46cd5..69ee92d12 100644 --- a/profiler/compare_tools/compare_backend/utils/torch_op_node.py +++ b/profiler/compare_tools/compare_backend/utils/torch_op_node.py @@ -64,6 +64,14 @@ class TorchOpNode: def device_dur(self): return sum([kernel.device_dur for kernel in self._kernel_list]) + @property + def api_dur(self): + return self._event.dur + + @property + def api_self_time(self): + return self.api_dur - sum(child.api_dur for child in self._child_nodes) + def add_child_node(self, child_node): self._child_nodes.append(child_node) diff --git a/profiler/compare_tools/compare_backend/utils/tree_builder.py b/profiler/compare_tools/compare_backend/utils/tree_builder.py index 34c1fe1a1..d5aa787ac 100644 --- a/profiler/compare_tools/compare_backend/utils/tree_builder.py +++ b/profiler/compare_tools/compare_backend/utils/tree_builder.py @@ -23,7 +23,8 @@ class TreeBuilder: tree_node = TorchOpNode(event, last_node) last_node.add_child_node(tree_node) last_node = tree_node - tree_node.set_kernel_list(kernel_dict.get(event.start_time, [])) + if kernel_dict: + tree_node.set_kernel_list(kernel_dict.get(event.start_time, [])) else: event.set_name(last_node.name) last_node.set_memory_allocated(event) diff --git a/profiler/compare_tools/compare_backend/view/work_sheet_creator.py b/profiler/compare_tools/compare_backend/view/work_sheet_creator.py index dffb7549f..58bad621b 100644 --- a/profiler/compare_tools/compare_backend/view/work_sheet_creator.py +++ b/profiler/compare_tools/compare_backend/view/work_sheet_creator.py @@ -12,7 +12,7 @@ class WorkSheetCreator: self._work_sheet = None self._row_id = 1 self._field_format = {} - self._diff_ratio_index = None + self._diff_ratio_index = [] self._col_ids = "ABCDEFGHIJKLMNOPQRSTUVW" def create_sheet(self): @@ -47,8 +47,10 @@ class WorkSheetCreator: self._work_sheet.set_column(f"{col_id}:{col_id}", header.get("width")) self._work_sheet.write(f"{col_id}{self._row_id}", header.get("name"), header_format) self._field_format[index] = header.get("type") - if header.get("name") in (ExcelConfig.DIFF_RATIO, ExcelConfig.DIFF_TOTAL_RATIO): - self._diff_ratio_index = index + ratio_white_list = [ExcelConfig.DIFF_RATIO, ExcelConfig.DIFF_TOTAL_RATIO, + ExcelConfig.DIFF_AVG_RATIO, ExcelConfig.DIFF_CALLS_RATIO, ExcelConfig.DIFF_SELF_RATIO] + if header.get("name") in ratio_white_list: + self._diff_ratio_index.append(index) self._row_id += 1 def _write_data(self): @@ -56,7 +58,7 @@ class WorkSheetCreator: for data in self._data.get("rows"): for index, cell_data in enumerate(data): cell_format = self._work_book.add_format(self._field_format.get(index)) - if index == self._diff_ratio_index and cell_data and cell_data > 1: + if index in self._diff_ratio_index and cell_data and cell_data > 1: cell_format = red_ratio_format cell_data = "INF" if cell_data == float('inf') else cell_data self._work_sheet.write(f"{self._col_ids[index]}{self._row_id}", cell_data, cell_format) @@ -76,7 +78,7 @@ class WorkSheetCreator: if index == 0: # 0 for Index field cell_style["indent"] = cell_data.count("\t") cell_format = self._work_book.add_format(cell_style) - if index == self._diff_ratio_index and cell_data and cell_data > 1: + if index in self._diff_ratio_index and cell_data and cell_data > 1: cell_format = red_ratio_format cell_data = "INF" if cell_data == float('inf') else cell_data self._work_sheet.write(f"{self._col_ids[index]}{self._row_id}", cell_data, cell_format) diff --git a/profiler/compare_tools/performance_compare.py b/profiler/compare_tools/performance_compare.py index 8de0a72cb..7c9d60aac 100644 --- a/profiler/compare_tools/performance_compare.py +++ b/profiler/compare_tools/performance_compare.py @@ -18,6 +18,8 @@ def main(): parser.add_argument("--enable_operator_compare", default=False, action='store_true', help="开启算子性能比较") parser.add_argument("--enable_memory_compare", default=False, action='store_true', help="开启算子内存比较") parser.add_argument("--enable_communication_compare", default=False, action='store_true', help="开启通信性能比较") + parser.add_argument("--enable_api_compare", default=False, action='store_true', help="开启host api性能比较") + parser.add_argument("--enable_kernel_compare", default=False, action='store_true', help="开启kernel性能比较") parser.add_argument("--disable_details", default=False, action='store_true', help="不展示比对明细") parser.add_argument("--output_path", type=str, default='', help="性能数据比对结果的存放路径") parser.add_argument("--max_kernel_num", type=int, help="每个torch op的kernel数量限制") diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py index 44d97b248..807346359 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py @@ -24,6 +24,11 @@ class ProfilingParser(BaseProfilingParser): self._enable_operator_compare = True self._enable_memory_compare = True self._enable_communication_compare = True + self._enable_kernel_compare = True + self._enable_api_compare = True + + def _update_kernel_details(self): + pass def _update_memory_list(self): pass -- Gitee From 2612e37388fa6c84c059af81e38d80bc8e931702 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 25 Jul 2024 16:43:45 +0800 Subject: [PATCH 018/791] bugfix --- debug/accuracy_tools/grad_tool/common/constant.py | 1 + debug/accuracy_tools/grad_tool/common/utils.py | 4 ++++ debug/accuracy_tools/grad_tool/grad_ms/utils.py | 7 +++++-- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/common/constant.py b/debug/accuracy_tools/grad_tool/common/constant.py index d569d47c1..7f2a708ab 100644 --- a/debug/accuracy_tools/grad_tool/common/constant.py +++ b/debug/accuracy_tools/grad_tool/common/constant.py @@ -40,6 +40,7 @@ class GradConst: DIRECTORY_LENGTH = 4096 FILE_NAME_LENGTH = 255 FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" + PARAM_VALID_PATTERN = r"^[a-zA-Z0-9.]+$" DIR = "dir" FILE = "file" diff --git a/debug/accuracy_tools/grad_tool/common/utils.py b/debug/accuracy_tools/grad_tool/common/utils.py index cdce3fda7..a1e639558 100644 --- a/debug/accuracy_tools/grad_tool/common/utils.py +++ b/debug/accuracy_tools/grad_tool/common/utils.py @@ -220,3 +220,7 @@ def change_mode(path, mode): except PermissionError as ex: print_error_log(f'Failed to change {path} authority. {str(ex)}') raise ex + +def check_param(param_name): + if not re.match(GradConst.PARAM_VALID_PATTERN, param_name): + raise RuntimeError("The parameter name contains special characters.") \ No newline at end of file diff --git a/debug/accuracy_tools/grad_tool/grad_ms/utils.py b/debug/accuracy_tools/grad_tool/grad_ms/utils.py index 23703f282..64829bdeb 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/utils.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/utils.py @@ -3,7 +3,8 @@ import os import numpy as np import mindspore from grad_tool.common.constant import GradConst -from grad_tool.common.utils import print_warn_log, create_directory, change_mode, check_file_or_directory_path +from grad_tool.common.utils import (print_warn_log, create_directory, change_mode, check_file_or_directory_path, + path_valid_check, check_param) level_adp = { "L0": { @@ -23,8 +24,10 @@ level_adp = { def save_grad_direction(param_name, grad, save_path): if not os.path.exists(save_path): create_directory(save_path) + check_file_or_directory_path(save_path, file_type=GradConst.DIR) + check_param(param_name) save_filepath = os.path.join(save_path, f"{param_name}.npy") - check_file_or_directory_path(save_filepath) + path_valid_check(save_filepath) if grad.dtype == mindspore.bfloat16: grad = grad.to(mindspore.float32) -- Gitee From b9d3f8636f11c60b44cd0798860e69be6cdfd4a9 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Wed, 24 Jul 2024 15:45:26 +0800 Subject: [PATCH 019/791] =?UTF-8?q?[maprobe/dump]=E6=96=B0=E5=A2=9Edump?= =?UTF-8?q?=E6=8C=87=E5=AE=9A=E4=BB=A3=E7=A0=81=E7=9A=84=E5=89=8D=E5=8F=8D?= =?UTF-8?q?=E5=90=91=E6=95=B0=E6=8D=AE=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/pytorch/doc/dump.md | 83 +++++++++++++++---- 1 file changed, 68 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md index 7d0763b68..9bfa66ae7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md @@ -77,9 +77,9 @@ if __name__ == "__main__" **功能说明** -启动函数。 +dump启动函数。 -在模型初始化之后的任意位置添加。 +在模型初始化之后的位置添加。需要与stop函数一起添加在for循环内。 **原型** @@ -93,9 +93,9 @@ debugger.start() **功能说明** -停止函数。 +dump停止函数。 -在**start**函数之后的任意位置添加。 +在**start**函数之后的任意位置添加。若需要dump反向数据，则需要添加在反向计算代码（如loss.backward）之后。 **原型** @@ -105,13 +105,33 @@ debugger.stop() 该函数为类函数，可以使用debugger.stop()也可以使用PrecisionDebugger.stop()。 +### forward_backward_dump_end函数 + +**功能说明** + +dump停止函数。用于dump指定代码的前反向数据。 + +在**start**函数之后，反向计算代码（如loss.backward）之前的任意位置添加，可以dump **start**函数和该函数之间的前反向数据，可以通过调整**start**函数与该函数的位置，来指定需要dump的代码块。 + +要求**stop**函数添加在反向计算代码（如loss.backward）之后，此时该函数与**stop**函数之间的代码不会被dump。 + +使用示例参见“**示例代码 > 扩展示例**”。 + +**原型** + +```Python +forward_backward_dump_end() +``` + +该函数为类函数，可以使用debugger.forward_backward_dump_end()也可以使用PrecisionDebugger.forward_backward_dump_end()。 + ### step函数 **功能说明** 结束标识。 -在最后一个**stop**函数后或一个step结束的位置添加。 +在最后一个**stop**函数后或一个step结束的位置添加。需要与start函数一起添加在for循环内。 **原型** @@ -123,24 +143,57 @@ debugger.step() ## 示例代码 +### 基础操作 + +如下示例可dump完整代码的前反向数据。 + ```Python from msprobe.pytorch import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") -# 请勿将以上初始化流程插入到循环代码中 -# 模型初始化 -# 下面代码也可以用PrecisionDebugger.start()和PrecisionDebugger.stop() -debugger.start() +# 模型、损失函数的定义及初始化等操作 +# ... -# 需要dump的代码片段1 +# 数据集迭代的位置一般为模型训练开始的位置 +for data, label in data_loader: + debugger.start() # 开启数据dump -debugger.stop() -debugger.start() + # 如下是模型每个step执行的逻辑 + output = model(data) + #... + loss.backward() + + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` -# 需要dump的代码片段2 +### 扩展示例 -debugger.stop() -debugger.step() +如下示例dump指定代码块前反向数据。 + +```Python +from msprobe.pytorch import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") + +# 模型、损失函数的定义及初始化等操作 +# ... + +# 数据集迭代的位置一般为模型训练开始的位置 +for data, label in data_loader: + debugger.start() # 开启数据dump + + # 如下是模型每个step执行的逻辑 + output = model(data) + debugger.forward_backward_dump_end() # 插入该函数到start函数之后，只dump start函数到该函数之间代码的前反向数据，本函数到stop函数之间的数据则不dump + #... + loss.backward() + + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump ``` ## dump结果文件介绍 -- Gitee From abddd175a7b559a95b236949fe9762fe554989d6 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 25 Jul 2024 17:11:55 +0800 Subject: [PATCH 020/791] update branch number --- debug/accuracy_tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index 4e0eaa1f3..1cc36f2a8 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -14,7 +14,7 @@ import setuptools -__version__ = '1.0.0' +__version__ = '1.1.0' INSTALL_REQUIRED = [ "wheel", -- Gitee From 5d2e493f771e1fc240157c3e661a158bb6afb0bf Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 25 Jul 2024 17:31:39 +0800 Subject: [PATCH 021/791] change branch number --- debug/accuracy_tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index 1cc36f2a8..afbf8feb3 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -14,7 +14,7 @@ import setuptools -__version__ = '1.1.0' +__version__ = '1.0.1' INSTALL_REQUIRED = [ "wheel", -- Gitee From b7ffc4b1675b4797be51f3fe6d5f3a7baf5431a3 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Thu, 25 Jul 2024 10:42:39 +0800 Subject: [PATCH 022/791] check dump start --- debug/accuracy_tools/msprobe/mindspore/service.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index e8aa34dc4..cb7a6f2a7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -39,6 +39,7 @@ class Service: self.first_start = True self.current_rank = None self.dump_iter_dir = None + self.start_call = False def build_hook(self, module_type, name): def forward_hook(api_or_module_name, module, input, output): @@ -80,6 +81,8 @@ class Service: def start(self, model=None): self.model = model + self.start_call = True + logger.info_on_rank_0("msprobe: debugger.start() is set successfully") if self.config.step and self.current_iter > max(self.config.step): self.stop() raise Exception("msprobe: exit after iteration {}".format(max(self.config.step))) @@ -101,11 +104,16 @@ class Service: logger.info_on_rank_0(f"Dump data will be saved in {self.dump_iter_dir}.") def stop(self): + if not self.start_call: + logger.error_on_rank_0("msprobe: debugger.start() is not set in the current scope.") + raise Exception("debugger.start() is not set in the current scope.") if self.config.step and self.current_iter not in self.config.step: return if self.config.rank and self.current_rank not in self.config.rank: return self.switch = False + self.start_call = False + logger.info_on_rank_0(f"msprobe: debugger.stop() is set successfully. Please set debugger.start() to turn on the dump switch again. ") self.data_collector.write_json() def create_dirs(self): -- Gitee From bf629891ec4bd60610dd2f8b0a1e08994f5de79a Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Thu, 25 Jul 2024 18:46:23 +0800 Subject: [PATCH 023/791] =?UTF-8?q?=E3=80=90feature=E3=80=91=E6=97=A0?= =?UTF-8?q?=E6=A0=87=E6=9D=86=E6=98=BE=E5=AD=98=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/const.py | 1 + .../pytorch/free_benchmark/common/constant.py | 2 + .../pytorch/free_benchmark/common/utils.py | 4 ++ .../result_handlers/base_handler.py | 72 +++++++++++-------- 4 files changed, 50 insertions(+), 29 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index df82455a6..85d5c65e5 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -45,6 +45,7 @@ class Const: PT_SUFFIX = ".pt" ONE_GB = 1073741824 # 1 * 1024 * 1024 * 1024 TEN_GB = 10737418240 # 10 * 1024 * 1024 * 1024 + ONE_MB = 1048576 # 1 * 1024 * 1024 FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$' DISTRIBUTED_PREFIX_LENGTH = 60 # env dump path diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/constant.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/constant.py index e737e7b21..08ac3dc66 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/constant.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/constant.py @@ -60,6 +60,8 @@ class ThresholdConfig: torch.bfloat16: BenchmarkThd(2**-8, 1.0, 2**-8, 1e-4), } + TENSOR_SPLIT_MAX_CHUNK = 128 + class PreheatConfig: IF_PREHEAT = "if_preheat" diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py index ddcbd9d0f..1aa099864 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py @@ -96,3 +96,7 @@ class TorchC: add = torch._C._VariableFunctionsClass.add bitwise_xor = torch._C._VariableFunctionsClass.bitwise_xor clone = torch._C._VariableFunctionsClass.clone + clamp = torch._C._VariableFunctionsClass.clamp + tensor_split = torch._C._VariableFunctionsClass.tensor_split + tensor = torch._C._VariableFunctionsClass.tensor + reshape = torch._C._VariableFunctionsClass.reshape diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py index 1728b096f..945a9c43a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py @@ -1,6 +1,7 @@ import math from abc import ABC, abstractmethod from typing import Any, Optional, Tuple +import numpy as np import torch from msprobe.core.common.const import Const @@ -34,15 +35,36 @@ class FuzzHandler(ABC): origin_ouput = origin_ouput.values perturbed_output = perturbed_output.values if hasattr(perturbed_output, "dtype"): - abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(perturbed_output.dtype) + abs_tol = ThresholdConfig.ABS_TOL_VALUE_DICT.get(perturbed_output.dtype, FuzzThreshold.F32_THD) else: - abs_tol = FuzzThreshold.F32_THD.value + abs_tol = FuzzThreshold.F32_THD return ( origin_ouput.to(perturbed_output.dtype).to(perturbed_output.device), perturbed_output, abs_tol, ) + @staticmethod + def tensor_split_for_error_calculate(origin_output, perturbed_output): + """ + 对将投入误差值计算的扰动前后输出张量进行分块 + :param origin_output: 原始输出 + :param perturbed_output: 扰动后输出 + :return origin_output_chunks: 切块后原始输出列表 + :return perturbed_output_chunks: 切块后扰动后输出列表 + """ + single_output_mem = origin_output.element_size() * origin_output.nelement() / Const.ONE_MB + if single_output_mem == 0 or origin_output.ndim == 0: + return [origin_output], [perturbed_output] + # 张量大小和批数之间的关系：chunks_exp=math.log(M,2)-4, chunks=2**chunks_exp (M为对比张量数据大小[Mb]) + chunks_exp = int(math.log(single_output_mem, 2)) - 4 + chunks = 2 ** chunks_exp + chunks = max(chunks, 1) + chunks = min(chunks, ThresholdConfig.TENSOR_SPLIT_MAX_CHUNK) + origin_output_chunks = TorchC.tensor_split(TorchC.reshape(origin_output, (-1,)), chunks) + perturbed_output_chunks = TorchC.tensor_split(TorchC.reshape(perturbed_output, (-1,)), chunks) + return origin_output_chunks, perturbed_output_chunks + @staticmethod def convert_overflow_ratio_to_consistent(ratio): if math.isnan(ratio) or math.isinf(ratio): @@ -61,36 +83,28 @@ class FuzzHandler(ABC): self, origin_output, perturbed_output, norm_type, abs_tol ): if norm_type == NormType.ENDLESS_NORM: - return self.get_endless_norm(origin_output, perturbed_output, abs_tol) + return self.calculate_error(origin_output, perturbed_output, abs_tol) return ThresholdConfig.COMP_CONSISTENT - def get_endless_norm(self, origin_output, perturbed_output, abs_tol): - ratio_tensor1 = TorchC.where( - TorchC.gt(TorchC.abs(perturbed_output), abs_tol), - TorchC.div( - TorchC.abs(origin_output), - TorchC.add(TorchC.abs(perturbed_output), abs_tol), - ), - 1, - ) - ratio_tensor2 = TorchC.where( - TorchC.gt(TorchC.abs(origin_output), abs_tol), - TorchC.div( - TorchC.abs(perturbed_output), - TorchC.add(TorchC.abs(origin_output), abs_tol), - ), - 1, - ) + def calculate_error(self, origin_output, perturbed_output, abs_tol): + origin_output_chunks, perturbed_output_chunks = self.tensor_split_for_error_calculate(origin_output, perturbed_output) + norm1 = -np.inf + norm2 = -np.inf + norm3 = np.inf + for i, chunk_origin in enumerate(origin_output_chunks): + if chunk_origin.nelement() == 0: + break + chunk_perturbed = perturbed_output_chunks[i] + ratio_tensor1 = TorchC.where(TorchC.abs(chunk_perturbed) > abs_tol, + TorchC.div(TorchC.clamp(chunk_origin, min=abs_tol), TorchC.clamp(chunk_perturbed, min=abs_tol)), 1) + ratio_tensor2 = TorchC.where(TorchC.abs(chunk_origin) > abs_tol, + TorchC.div(TorchC.clamp(chunk_perturbed, min=abs_tol), TorchC.clamp(chunk_origin, min=abs_tol)), 1) + norm_values = TorchC.tensor([TorchC.max(ratio_tensor1), TorchC.max(ratio_tensor2)]) + max_ratio1, max_ratio2 = norm_values.tolist() + norm1 = max(norm1, self.convert_overflow_ratio_to_consistent(max_ratio1)) + norm2 = max(norm2, self.convert_overflow_ratio_to_consistent(max_ratio2)) + norm3 = min(norm3, self.convert_overflow_ratio_to_consistent(max_ratio1)) - norm1 = self.convert_overflow_ratio_to_consistent( - TorchC.max(ratio_tensor1).item() - ) - norm2 = self.convert_overflow_ratio_to_consistent( - TorchC.max(ratio_tensor2).item() - ) - norm3 = self.convert_overflow_ratio_to_consistent( - TorchC.min(ratio_tensor1).item() - ) if norm3 < 0: ratio = ThresholdConfig.SYMBOL_FLIPPING else: -- Gitee From ed264708ccd0f66ea194d3266e10c8d347caadb6 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Thu, 25 Jul 2024 20:32:05 +0800 Subject: [PATCH 024/791] =?UTF-8?q?[msprobe]=E8=BF=AD=E4=BB=A3=E4=B8=80?= =?UTF-8?q?=E5=8F=91=E5=8C=85=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 1e8c1a1f0..fb47846c3 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -26,6 +26,7 @@ MindStudio精度调试工具（MindStudio Probe），简称msprobe，是MindStud | 版本 | 发布日期 | 支持PyTorch版本 | 下载链接 | 校验码 | | ----- | ---------- | --------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | + | 1.0.1 | 2024-07-25 | 2.0/2.1/2.2 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | | 1.0 | 2024-07-09 | 2.0/2.1/2.2 | [ascend_training_accuracy_tools-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/1.0/ascend_training_accuracy_tools-1.0-py3-none-any.whl) | 5016dfe886c5d340ec6f60a959673355855f313c91f100680da814efb49f8e81 | | 0.0.3 | 2024-06-11 | 2.0/2.1/2.2 | [ascend_training_accuracy_tools-0.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.3-py3-none-any.whl) | f46d9714704859e2d67861a65bbb3c76b0a250cf6e238b978b5b959ab1fe125a | | 0.0.2 | 2024-05-23 | 2.0/2.1/2.2 | [ascend_training_accuracy_tools-0.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.2-py3-none-any.whl) | 2e35809bde559e9c4d2f16a02ccde779ed9e436bb65fded0b7ebaf6ac2c88d93 | -- Gitee From b2d607d1fb9ad9c494f8d1a9dae7ac808c828abf Mon Sep 17 00:00:00 2001 From: wuyuhan Date: Thu, 25 Jul 2024 20:41:08 +0800 Subject: [PATCH 025/791] =?UTF-8?q?=E8=AF=86=E5=88=ABAICPU=E4=BC=98?= =?UTF-8?q?=E5=85=88=E8=AF=BB=E5=8F=96kernel=5Fdetails.csv,=20=E5=85=B6?= =?UTF-8?q?=E6=AC=A1=E8=AF=BB=E5=8F=96op=5Fsummary=5Fxxx.csv?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../config/profiling_data_version_config.yaml | 17 ++++++------- .../dataset/profiling/profiling_dataset.py | 9 +++++-- .../dataset/profiling/profiling_parser.py | 24 ++++++++++++------- profiler/advisor/utils/utils.py | 12 +++++++++- 4 files changed, 42 insertions(+), 20 deletions(-) diff --git a/profiler/advisor/config/profiling_data_version_config.yaml b/profiler/advisor/config/profiling_data_version_config.yaml index 4ef76105a..b8c92fe07 100644 --- a/profiler/advisor/config/profiling_data_version_config.yaml +++ b/profiler/advisor/config/profiling_data_version_config.yaml @@ -1,18 +1,19 @@ versions: - version: 8.0.RC1 dirs_pattern: + ASCEND_PROFILER_OUTPUT: [ op_summary ] ^PROF_\d{6}_\d{17}_\w+$: - mindstudio_profiler_output: - [ op_summary, msprof ] + mindstudio_profiler_output: [ op_summary, msprof ] class_attr: op_summary: OpSummary msprof: Msprof file_attr: - op_summary: ^op_summary_\d{14}\.csv$ msprof: ^msprof_\d{14}\.json$ + op_summary: [ kernel_details.csv, '^op_summary_\d{14}\.csv$' ] - version: 7.0.0 dirs_pattern: + ASCEND_PROFILER_OUTPUT: [ op_summary ] ^PROF_\d{6}_\d{17}_\w+$: ^device_\d+$: summary: @@ -28,13 +29,14 @@ versions: msprof: Msprof ge_info: GeInfo file_attr: - op_summary: ^op_summary_\d+_\d+_\d{14}\.csv$ + op_summary: [ kernel_details.csv, '^op_summary_\d+_\d+_\d{14}\.csv$'] task_time: ^task_time_\d+_\d+_\d{14}\.json$ msprof: ^msprof_\d+_\d+_\d{14}\.json$ ge_info: ge_info.db - version: 7.0.RC1 dirs_pattern: + ASCEND_PROFILER_OUTPUT: [ op_summary ] ^PROF_\d{6}_\d{17}_\w+$: ^device_\d+$: summary: @@ -50,13 +52,14 @@ versions: msprof: Msprof ge_info: GeInfo file_attr: - op_summary: ^op_summary_\d+_\d+_\d+_\d{14}\.csv$ + op_summary: [ kernel_details.csv, '^op_summary_\d+_\d+_\d+_\d{14}\.csv$'] task_time: ^task_time_\d+_\d+_\d+_\d{14}\.json$ msprof: ^msprof_\d+_\d+_\d+_\d{14}\.json$ ge_info: ge_info.db - version: 6.3.RC2 dirs_pattern: + ASCEND_PROFILER_OUTPUT: [ op_summary ] ^PROF_\d{6}_\d{17}_\w+$: ^device_\d+$: summary: @@ -72,9 +75,7 @@ versions: msprof: Msprof ge_info: GeInfo file_attr: - op_summary: ^op_summary_\d+_\d+\.csv$ + op_summary: [ kernel_details.csv, '^op_summary_\d+_\d+\.csv$'] task_time: ^task_time_\d+_\d+\.json$ msprof: ^msprof_\d+_\d+\.json$ ge_info: ge_info.db - - diff --git a/profiler/advisor/dataset/profiling/profiling_dataset.py b/profiler/advisor/dataset/profiling/profiling_dataset.py index 99a19d3b6..4f7eb305b 100644 --- a/profiler/advisor/dataset/profiling/profiling_dataset.py +++ b/profiler/advisor/dataset/profiling/profiling_dataset.py @@ -43,14 +43,19 @@ class ProfilingDataset(Dataset): self.build_from_pattern(value, join_prof_path(current_path, key)) elif isinstance(dirs_pattern, list): for item in dirs_pattern: + if hasattr(self, item) and getattr(self, item): + # 避免重复构建kernel_details.csv, op_summary.csv的数据对象 + continue + file_pattern = self.current_version_pattern.get('file_attr').get(item) data_class = globals()[self.current_version_pattern.get('class_attr').get(item)] - data_class.FILE_PATTERN = self.current_version_pattern.get('file_attr').get(item) + data_class.FILE_PATTERN = file_pattern data_object = data_class(current_path) is_success = data_object.parse_data() if is_success: setattr(self, item, data_object) else: - logger.warning("Skip parse %s from local path %s", self.current_version_pattern.get('class_attr').get(item), current_path) + logger.warning("Skip parse %s with file pattern %s from local path %s", + self.current_version_pattern.get('class_attr').get(item), file_pattern, current_path) else: logger.warning(f"Unsupported arguments : %s to build %s", dirs_pattern, self.__class__.__name__) diff --git a/profiler/advisor/dataset/profiling/profiling_parser.py b/profiler/advisor/dataset/profiling/profiling_parser.py index bb4caeb29..3fe7bcd67 100644 --- a/profiler/advisor/dataset/profiling/profiling_parser.py +++ b/profiler/advisor/dataset/profiling/profiling_parser.py @@ -37,15 +37,21 @@ class ProfilingParser: return False def _parse_from_file(self): - file_list = get_file_path_from_directory(self._path, self.file_match_func(self.FILE_PATTERN)) - if not file_list: - return False - ## get last file - file = file_list[-1] - self.FILE_PATH = file - if len(file_list) > 1: - logger.warning("Multiple copies of %s were found, use %s", self.FILE_INFO, file) - return self.parse_from_file(file) + + if not isinstance(self.FILE_PATTERN, list): + self.FILE_PATTERN = [self.FILE_PATTERN] + + for file_pattern in self.FILE_PATTERN: + file_list = get_file_path_from_directory(self._path, self.file_match_func(file_pattern)) + if not file_list: + continue + ## get last file + file = file_list[-1] + self.FILE_PATH = file + if len(file_list) > 1: + logger.warning("Multiple copies of %s were found, use %s", self.FILE_INFO, file) + return self.parse_from_file(file) + return False @staticmethod def get_float(data) -> float: diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index b373d7bad..3488e7dff 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -414,7 +414,17 @@ def format_excel_title(title: str) -> str: title = title.replace("(ns)", '') title = title.replace("(%)", '') title = title.replace(" ", "_") - return title + + # 将kernel_details中的列名转为与op_summary_x.csv中一致 + kernel_details_col_name_map = dict( + name="op_name", + type="op_type", + accelerator_core="task_type", + start_time="task_start_time", + duration="task_duration", + wait_time="wait_time" + ) + return kernel_details_col_name_map.get(title, title) def format_float(num: float) -> float: -- Gitee From 69c391d1067bbb0c988927a7494fd44e59416341 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 26 Jul 2024 09:54:37 +0800 Subject: [PATCH 026/791] graph mode grad_tool incorrect output bugfix --- debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py index 75280b319..eb6a28aa9 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py @@ -35,7 +35,7 @@ def grad_dump(dump_dir: str, g_name: str, dump_step: Parameter, grad: ms.Tensor, level2: [step, max, min, norm, shape_dim, shape] + grad_bool_data level3: [step, max, min, norm, shape_dim, shape, dist_dim, dist] + grad_bool_data ''' - dump_path = dump_dir + g_name + dump_path = os.path.join(dump_dir, g_name) dump_dir_path = dump_path + "_dir" save_op = ms.ops.TensorDump() -- Gitee From d49b5e1c7cb3015fb55f277a7fb03ff830a22051 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 26 Jul 2024 10:28:21 +0800 Subject: [PATCH 027/791] fix bug --- .../pytorch/api_accuracy_checker/run_ut/multi_run_ut.py | 8 ++++---- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 8 ++------ 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index 9c96a52d8..8f1aa5b73 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -68,7 +68,7 @@ signal.signal(signal.SIGTERM, signal_handler) ParallelUTConfig = namedtuple('ParallelUTConfig', ['api_files', 'out_path', 'num_splits', 'save_error_data_flag', 'jit_compile_flag', 'device_id', - 'result_csv_path', 'total_items', 'real_data_path']) + 'result_csv_path', 'total_items', 'config_path']) def run_parallel_ut(config): @@ -90,7 +90,7 @@ def run_parallel_ut(config): *(['-j'] if config.jit_compile_flag else []), *(['-save_error_data'] if config.save_error_data_flag else []), '-csv_path', config.result_csv_path, - *(['-real_data_path', config.real_data_path] if config.real_data_path else []) + *(['-config', config.config_path] if config.config_path else []) ] return cmd @@ -175,7 +175,7 @@ def prepare_config(args): out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE) out_path = out_path_checker.common_check() split_files, total_items = split_json_file(api_info, args.num_splits, args.filter_api) - + config_path = os.path.realpath(args.config_path) if args.config_path else None result_csv_path = args.result_csv_path or os.path.join(out_path, f"accuracy_checking_result_{time.strftime('%Y%m%d%H%M%S')}.csv") if not args.result_csv_path: details_csv_path = os.path.join(out_path, f"accuracy_checking_details_{time.strftime('%Y%m%d%H%M%S')}.csv") @@ -187,7 +187,7 @@ def prepare_config(args): logger.info(f"UT task details will be saved in {details_csv_path}") return ParallelUTConfig(split_files, out_path, args.num_splits, args.save_error_data, args.jit_compile, args.device_id, result_csv_path, - total_items, args.real_data_path) + total_items, config_path) def main(): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 30994f709..a8ff9b599 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -379,10 +379,6 @@ def _run_ut_parser(parser): help=" The path of accuracy_checking_result_{timestamp}.csv, " "when run ut is interrupted, enter the file path to continue run ut.", required=False) - parser.add_argument("-real_data_path", dest="real_data_path", nargs="?", const="", default="", type=str, - help=" In real data mode, the root directory for storing real data " - "must be configured.", - required=False) parser.add_argument("-f", "--filter_api", dest="filter_api", action="store_true", help=" Whether to filter the api in the api_info_file.", required=False) parser.add_argument("-config", "--config_path", dest="config_path", default="", type=str, @@ -400,9 +396,9 @@ def preprocess_forward_content(forward_content): if key not in arg_cache: filtered_new_args = [ {k: v for k, v in arg.items() if k not in ['Max', 'Min']} - for arg in value['args'] if isinstance(arg, dict) + for arg in value['input_args'] if isinstance(arg, dict) ] - arg_cache[key] = (filtered_new_args, value['kwargs']) + arg_cache[key] = (filtered_new_args, value['input_kwargs']) filtered_new_args, new_kwargs = arg_cache[key] -- Gitee From 5fd40e4cfc0b545b9e649243a32d64d03c335ac5 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Wed, 24 Jul 2024 17:24:53 +0800 Subject: [PATCH 028/791] =?UTF-8?q?[mstt/msprobe/pytorch/api=5Faccuracy=5F?= =?UTF-8?q?checker]=E6=96=B0=E5=A2=9Eblack=5Flist=E9=A2=84=E6=A3=80?= =?UTF-8?q?=E9=BB=91=E5=90=8D=E5=8D=95=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/README.md | 44 ++++++++++-- debug/accuracy_tools/msprobe/config/README.md | 60 +++++++++++++++- .../pytorch/doc/api_accuracy_checker.md | 68 +++++++++++++++---- .../msprobe/pytorch/doc/dump.md | 2 +- 4 files changed, 156 insertions(+), 18 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 1e8c1a1f0..d4eb1849b 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -10,10 +10,15 @@ MindStudio精度调试工具（MindStudio Probe），简称msprobe，是MindStud ```shell pip install mindstudio-probe ``` - 说明 - 1. 使用`pip install mindstudio-probe==版本号`可安装指定版本的包 - 2. pip命令会自动安装包及其依赖 - 3. 安装成功后，日志会显示`Successfully installed mindstudio-probe-版本号` +使用`pip install mindstudio-probe==版本号`可安装指定版本的包。 + +pip命令会自动安装最新的包及其配套依赖。 + +提示如下信息则表示安装成功。 + +```bash +Successfully installed mindstudio_probe-{version} +``` ### 下载whl包安装 1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、torch、tqdm依赖。 @@ -92,6 +97,37 @@ MindStudio精度调试工具（MindStudio Probe），简称msprobe，是MindStud Finished processing dependencies for mindstudio-probe=={version} ``` +### 查看msprobe工具信息 + +执行如下命令查看msprobe工具信息。 + +```bash +pip show mindstudio-probe +``` + +输出结果如下示例： + +```bash +Name: mindstudio-probe +Version: 1.0 +Summary: This is a pytorch precision comparison tools +Home-page: +Author: +Author-email: +License: +Location: /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages +Requires: numpy, openpyxl, pandas, pyyaml, rich, tqdm, wheel +Required-by: +``` + +关键字段含义： + +- Name：工具名称。 +- Version：工具版本号。 +- Summary：工具概述。 +- Location：工具安装路径。 +- Requires：工具依赖。 + ## 工具使用 安装msprobe工具后，可以按照如下思路选择合适的子工具进行精度调试： diff --git a/debug/accuracy_tools/msprobe/config/README.md b/debug/accuracy_tools/msprobe/config/README.md index 7b91bd26f..7d11a3652 100644 --- a/debug/accuracy_tools/msprobe/config/README.md +++ b/debug/accuracy_tools/msprobe/config/README.md @@ -2,13 +2,38 @@ 当前配置文件主要为PrecisionDebugger接口执行dump或无标杆比对操作时调用的配置，当PrecisionDebugger接口未指定该配置文件时，使用该文件的默认配置。配置文件详见[config.json](./config.json)。 +当在环境上安装msprobe工具后，config.json文件位置可通过如下方式查找： + +查找msprobe工具安装路径。 + +``` +pip show mindstudio-probe +``` + +输出结果如下示例： + +``` +Name: mindstudio-probe +Version: 1.0 +Summary: This is a pytorch precision comparison tools +Home-page: +Author: +Author-email: +License: +Location: /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages +Requires: numpy, openpyxl, pandas, pyyaml, rich, tqdm, wheel +Required-by: +``` + +Location字段为msprobe工具的安装路径，那么config.json文件位置为/home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages/msprobe/config + ## 参数说明 ### **通用配置参数** | 参数名 | 说明 | 是否必选 | | ----------------- | ------------------------------------------------------------ | -------- | -| task | dump的任务类型，str类型。可取值"free_benchmark"（无标杆比对，仅PyTorch场景支持）、"statistics"（仅dump API统计信息，默认值）、"tensor"（dump API统计信息和完全复刻整网的API运行情况的真实数据）、"overflow_check"（溢出检测）。配置示例："task": "tensor"。根据task参数取值的不同，可以配置不同场景参数，详见：“**task配置为free_benchmark**”，“**task配置为statistics**”，“**task配置为tensor**”，“**task配置为overflow_check**”。 | 否 | +| task | dump的任务类型，str类型。可取值：
"free_benchmark"（无标杆比对，仅PyTorch场景支持）。
"statistics"（仅dump API统计信息，默认值）。
"tensor"（dump API统计信息和完全复刻整网的API运行情况的真实数据）。
"overflow_check"（溢出检测，仅PyTorch和MindSpore静态图场景支持）。
"run_ut"（精度预检配置，仅PyTorch场景支持）。
配置示例："task": "tensor"。
根据task参数取值的不同，可以配置不同场景参数，详见：“**task配置为free_benchmark**”，“**task配置为statistics**”，“**task配置为tensor**”，“**task配置为overflow_check**”，“**task配置为run_ut**”。 | 否 | | dump_path | 设置dump数据目录路径，str类型。配置示例："dump_path": "./dump_path"。MindSpore场景仅支持绝对路径。 | 是 | | rank | 指定对某张卡上的数据进行dump，list[int]类型，默认未配置（表示dump所有卡的数据），应配置为大于等于0的整数，且须配置实际可用的Rank ID。配置示例："rank": [1]。
对于PyTorch场景，Rank ID从0开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的Rank ID，则dump数据为空，比如当前环境Rank ID为0到7，实际训练运行0到3卡，此时若配置Rank ID为4或不存在的10等其他值，此时dump数据为空。
对于MindSpore场景，所有节点的Rank ID均从0开始计数，最大取值为每个节点可用卡总数-1，config.json配置一次rank参数对所有节点同时生效。 | 否 | | step | 指定dump某个step的数据，list[int]类型。默认未配置，表示dump所有step数据。dump特定step时，须指定为训练脚本中存在的step。step为list格式，可配置逐个step，例如："step": [0,1,2]。 | 否 | @@ -85,6 +110,18 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 | overflow_nums | 控制溢出次数，int类型，仅PyTorch场景支持，表示第N次溢出时，停止训练，过程中检测到溢出API对应kernel数据均dump。配置示例："overflow_nums": 3。默认为1，即检测到1次溢出，训练停止，配置为-1时，表示持续检测溢出直到训练结束。 | 否 | | check_mode | MindSpore场景kernel级别的溢出检测，str类型，可取值"aicore"（开启AI Core的溢出检测）、"atomic"（开启Atomic的溢出检测）、"all"（开启AI Core和Atomic的溢出检测，默认值）。配置示例"check_mode": "aicore"。 | 否 | +### task配置为run_ut + +仅PyTorch场景支持。 + +| 参数名称 | 说明 | 是否必选 | +| --------------- | ------------------------------------------------------------ | -------- | +| white_list | API dump白名单，仅对指定的API进行dump。配置示例："white_list": ["conv1d", "conv2d"]。默认未配置白名单，即dump全量API数据。 | 否 | +| black_list | API dump黑名单，被指定的API不进行dump。配置示例："black_list": ["conv1d", "conv2d"]。默认未配置黑名单，即dump全量API数据。 | 否 | +| error_data_path | 配置保存精度未达标的API输入输出数据路径，默认为当前路径。配置示例"error_data_path": "./"。 | 否 | + +说明：white_list和black_list同时配置时，二者配置的API名单若无交集，则白名单生效，若API名单存在交集，则白名单排除的部分以及交集的API不进行dump。 + ## 配置示例 以下示例包含当前支持的所有场景可配置的完整参数。 @@ -180,6 +217,27 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 } ``` +### PyTorch场景task配置为run_ut + +```json +{ + "task": "run_ut", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + "seed": 1234, + "is_deterministic": false, + "enable_dataloader": false, + + "run_ut": { + "white_list": [], + "black_list": [], + "error_data_path": "./" + } +} +``` + ### MindSpore场景task配置为statistics ```json diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md index b3ed4a9e2..41b97098a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md @@ -21,7 +21,7 @@ 精度预检操作流程如下： 1. 在NPU和GPU环境下分别安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 -2. 在NPU训练脚本内添加msprobe工具dump接口PrecisionDebugger采集待预检数据。详见《[精度数据采集](./dump.md)》。 +2. 在NPU训练脚本内添加msprobe工具dump接口PrecisionDebugger，采集待预检数据。详见《[精度数据采集](./dump.md)》，注意需要配置level="L1"。 3. 将NPU环境下dump的预检数据拷贝至GPU环境。 4. 在NPU和GPU环境下分别执行run_ut，生成结果用于最终api_precision_compare操作的输入。详见“**run_ut预检操作**”。 5. 将NPU和GPU执行run_ut生成的`accuracy_checking_details_{timestamp}.csv`结果文件拷贝至同一环境下。 @@ -51,10 +51,12 @@ run_ut预检操作包括如下场景： | -api_info或--api_info_file | 指定API信息文件dump.json。 | 是 | | -save_error_data | 保存精度未达标的API输入输出数据。 | 否 | | -o或--out_path | 指定run_ut执行结果存盘路径，默认“./”（相对于run_ut的路径）。 | 否 | + | | | | | -j或--jit_compile | 开启jit编译。 | 否 | | -d或--device | 指定Device ID，选择UT代码运行所在的卡，默认值为0。 | 否 | | -csv_path或--result_csv_path | 指定本次运行中断时生成的`accuracy_checking_result_{timestamp}.csv`文件路径，执行run_ut中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的`accuracy_checking_result_{timestamp}.csv`文件。详见“**断点续检**”。 | run_ut操作中断后继续执行场景下必选 | | -f或--filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的API。适用于模型较大且重复API较多的场景。 | 否 | + | -config或--config_path | 指定预检操作过程中的额外配置（包括黑名单、白名单等）的[config.json](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe/config)文件，默认未配置。config.json文件的配置可参考《[配置文件说明](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/config/README.md#pytorch场景task配置为run_ut)》。 | 否 | run_ut执行结果包括`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`两个文件。`accuracy_checking_result_{timestamp}.csv`是API粒度的，标明每个API是否通过测试。建议用户先查看`accuracy_checking_result_{timestamp}.csv`文件，对于其中没有通过测试的或者特定感兴趣的API，根据其API name字段在`accuracy_checking_details_{timestamp}.csv`中查询其各个输出的达标情况以及比较指标。详细介绍请参见“**预检结果**”。 @@ -64,7 +66,7 @@ run_ut预检操作包括如下场景： msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data ``` - 数据默认会存盘到'./ut_error_data{timestamp}'路径下（相对于启动run_ut的路径），有需要的话，用户可以通过修改mstt/debug/accuracy_tools/api_accuracy_checker目录下，config.yaml文件的error_data_path参数来配置保存路径，详见“config.yaml文件说明”。 + 数据默认会存盘到'./ut_error_data{timestamp}'路径下（相对于启动run_ut的路径），有需要的话，用户可以通过error_data_path参数来配置保存路径，error_data_path参数在[config.json](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe/config)文件或config.yaml文件配置，config.json文件需要在run_ut操作时通过-config参数指定，config.yaml文件详见“**config.yaml文件说明**”。 #### 使用multi_run_ut.py执行多线程预检 @@ -99,23 +101,65 @@ msprobe -f pytorch multi_run_ut -api_info ./dump.json -n 32 -d 0 1 2 3 msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_checking_result_{timestamp}.csv ``` -#### API预检白名单 +#### API预检黑名单和白名单 -run_ut过程支持API预检白名单，操作方式如下： +run_ut过程支持API预检黑名单和白名单，通过如下文件配置black_list（黑名单）或white_list（白名单）参数来指定不需要或需要预检的API名称： -修改mstt/debug/accuracy_tools/api_accuracy_checker目录下config.yaml文件的white_list参数，配置需要预检的API名称，详见“config.yaml文件说明”。 +- 配置[config.json](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe/config)文件，config.json文件需要在run_ut操作时通过-config参数指定。 +- 配置config.yaml文件，详见“**config.yaml文件说明**”。 + +config.json文件的优先级高于config.yaml文件，即执行config.json文件时，config.yaml文件的配置不生效。 ### config.yaml文件说明 -config.yaml文件可以通过配置参数来控制dump和run_ut操作的白名单等功能。 +config.yaml文件可以通过配置参数来控制dump和run_ut操作的白名单、黑名单等功能。操作步骤如下： + +1. 查找msprobe工具安装路径。 + + ```bash + pip show mindstudio-probe + ``` + + 输出结果如下示例： + + ```bash + Name: mindstudio-probe + Version: 1.0 + Summary: This is a pytorch precision comparison tools + Home-page: + Author: + Author-email: + License: + Location: /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages + Requires: numpy, openpyxl, pandas, pyyaml, rich, tqdm, wheel + Required-by: + ``` + + Location字段为msprobe工具的安装路径，那么config.yaml文件位置为/home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml + +2. 进入config.yaml文件 + + ```bash + vi /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml + ``` + +3. 修改config.yaml文件参数。 + + ```yaml + white_list: [] + black_list: [] + error_data_path: './' + precision: 14 + ``` -文件路径为：mstt/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml + | 参数名称 | 说明 | 是否必选 | + | --------------- | ------------------------------------------------------------ | -------- | + | white_list | API dump白名单，仅对指定的API进行dump。参数示例：white_list=["conv1d", "conv2d"]。默认未配置白名单，即dump全量API数据。 | 否 | + | black_list | API dump黑名单，被指定的API不进行dump。参数示例：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即dump全量API数据。 | 否 | + | error_data_path | 配置保存精度未达标的API输入输出数据路径。参数示例"error_data_path": "./"。默认为当前路径。 | 否 | + | precision | 浮点数表示位数，默认取小数点后14位。 | 否 | -| 参数名称 | 说明 | 是否必选 | -| --------------- | ------------------------------------------------------------ | -------- | -| white_list | API dump白名单，指定dump具体API数据，也可以直接配置预检的API白名单，详细请参见“**API预检白名单**”。参数示例：white_list=["conv1d", "conv2d"]。默认未配置白名单，即dump全量API数据。 | 否 | -| error_data_path | 配置保存精度未达标的API输入输出数据路径。 | 否 | -| precision | 浮点数表示位数，默认取小数点后14位。 | 否 | + 说明：white_list和black_list同时配置时，二者配置的API名单若无交集，则白名单生效，若API名单存在交集，则白名单排除的部分以及交集的API不进行dump。 ## 预检结果 diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md index 7d0763b68..a9eddaeda 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md @@ -20,7 +20,7 @@ msprobe工具主要通过在训练脚本内添加dump接口并启动训练的方 PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, model=None, step=None) ``` -说明：上述参数除config_path和model外，其他参数均在[config.json](../../config)文件中可配，此处的参数优先级高于[config.json](../../config)文件中的配置，而config.json文件可以配置更多参数，若需要进行更多场景的精度数据dump，建议配置[config.json](../../config)文件。 +说明：上述参数除config_path和model外，其他参数均在[config.json](../../config)文件中可配，此处的参数优先级高于[config.json](../../config)文件中的配置，而config.json文件可以配置更多参数，若需要进行更多场景的精度数据dump，建议配置[config.json](../../config)文件。config.json文件的配置可参考《[配置文件说明](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/config/README.md)》。 **参数说明** -- Gitee From 29a07c6876ca53216f220c7ee99b880c86ef8580 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 26 Jul 2024 10:52:26 +0800 Subject: [PATCH 029/791] fix bug --- .../api_accuracy_checker/run_ut/test_multi_run_ut.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_multi_run_ut.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_multi_run_ut.py index 771e04238..27126cddd 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_multi_run_ut.py @@ -48,7 +48,7 @@ class TestMultiRunUT(unittest.TestCase): device_id=[0, 1], result_csv_path='result.csv', total_items=2, - real_data_path=None + config_path=None ) mock_file.side_effect = [ @@ -81,7 +81,7 @@ class TestMultiRunUT(unittest.TestCase): args.jit_compile = False args.device_id = [0, 1] args.result_csv_path = None - args.real_data_path = None + args.config_path = None config = prepare_config(args) -- Gitee From 6a57435a1de01683f50eb51a095f52fc30d64602 Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Fri, 26 Jul 2024 11:04:05 +0800 Subject: [PATCH 030/791] add api and kernel cli switch --- profiler/cli/compare_cli.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/profiler/cli/compare_cli.py b/profiler/cli/compare_cli.py index e794578da..f9add948e 100644 --- a/profiler/cli/compare_cli.py +++ b/profiler/cli/compare_cli.py @@ -32,6 +32,8 @@ from profiler.compare_tools.compare_backend.comparison_generator import Comparis @click.option('--enable_operator_compare', is_flag=True) @click.option('--enable_memory_compare', is_flag=True) @click.option('--enable_communication_compare', is_flag=True) +@click.option('--enable_api_compare', is_flag=True) +@click.option('--enable_kernel_compare', is_flag=True) @click.option('--disable_details', is_flag=True) @click.option('--output_path', '-o', 'output_path', type=click.Path()) @click.option('--max_kernel_num', 'max_kernel_num', type=int, help="The number of kernels per torch op is limited.") -- Gitee From cdb0c06fd4d3e0bba150ff078214e0514ad5a339 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Fri, 26 Jul 2024 11:38:31 +0800 Subject: [PATCH 031/791] bugfix --- .../msprobe/pytorch/free_benchmark/common/utils.py | 2 +- .../pytorch/free_benchmark/result_handlers/base_handler.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py index 1aa099864..631beeb85 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/utils.py @@ -98,5 +98,5 @@ class TorchC: clone = torch._C._VariableFunctionsClass.clone clamp = torch._C._VariableFunctionsClass.clamp tensor_split = torch._C._VariableFunctionsClass.tensor_split - tensor = torch._C._VariableFunctionsClass.tensor + stack = torch._C._VariableFunctionsClass.stack reshape = torch._C._VariableFunctionsClass.reshape diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py index 945a9c43a..e36f58673 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/base_handler.py @@ -99,7 +99,7 @@ class FuzzHandler(ABC): TorchC.div(TorchC.clamp(chunk_origin, min=abs_tol), TorchC.clamp(chunk_perturbed, min=abs_tol)), 1) ratio_tensor2 = TorchC.where(TorchC.abs(chunk_origin) > abs_tol, TorchC.div(TorchC.clamp(chunk_perturbed, min=abs_tol), TorchC.clamp(chunk_origin, min=abs_tol)), 1) - norm_values = TorchC.tensor([TorchC.max(ratio_tensor1), TorchC.max(ratio_tensor2)]) + norm_values = TorchC.stack([TorchC.max(ratio_tensor1), TorchC.max(ratio_tensor2)]) max_ratio1, max_ratio2 = norm_values.tolist() norm1 = max(norm1, self.convert_overflow_ratio_to_consistent(max_ratio1)) norm2 = max(norm2, self.convert_overflow_ratio_to_consistent(max_ratio2)) -- Gitee From 039408adbfb9c89bf2ef92dcaa8b9eb5085aec35 Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Fri, 26 Jul 2024 11:52:48 +0800 Subject: [PATCH 032/791] add gpu adapt --- .../profiling_parser/base_profiling_parser.py | 9 +++++---- .../profiling_parser/gpu_profiling_parser.py | 2 ++ 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 625eee7c6..9daaa55ef 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -303,7 +303,7 @@ class BaseProfilingParser(ABC): task_index += 1 def _check_result_data(self): - if self._enable_operator_compare or self._enable_memory_compare: + if self._enable_operator_compare or self._enable_memory_compare or self._enable_api_compare: if not self._result_data.torch_op_data: print(f"[WARNING] Can't find any torch op in the file: {self._profiling_path}") if self._enable_operator_compare and not self._result_data.kernel_dict: @@ -312,10 +312,11 @@ class BaseProfilingParser(ABC): print(f"[WARNING] Can't find any memory event in the file: {self._profiling_path}") if self._enable_communication_compare and not self._result_data.communication_dict: print(f"[WARNING] Can't find any communication op in the file: {self._profiling_path}") - if self._enable_api_compare and not self._result_data.torch_op_data: - print(f"[WARNING] Can't find any torch op in the file: {self._profiling_path}") if self._enable_kernel_compare and not self._result_data.kernel_details: - print(f"[WARNING] Can't find any kernel details in the file: {self._profiling_path}") + if self._profiling_type == Constant.GPU: + print(f"[WARNING] kernel compare between GPU data and NPU data is not supported.") + else: + print(f"[WARNING] Can't find any kernel details in the file: {self._profiling_path}") def _read_trace_event(self): try: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index ea732a60e..0aeeba83e 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -174,6 +174,8 @@ class GPUProfilingParser(BaseProfilingParser): func_set.add(self._picking_memory_event) if self._enable_profiling_compare: func_set.add(self._picking_flow_event) + if self._enable_api_compare: + func_set.add(self._picking_torch_op_event) return list(func_set) def _infer_compute_stream_id(self): -- Gitee From 9071f6d4a4ecf58da74853e2c8f9502a2d3c5318 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 26 Jul 2024 15:45:36 +0800 Subject: [PATCH 033/791] Security issues fixed --- .../accuracy_tools/api_accuracy_checker/common/utils.py | 6 +++++- .../api_accuracy_checker/run_ut/multi_run_ut.py | 2 +- .../api_accuracy_checker/tensor_transport_layer/attl.py | 9 ++++++--- .../pytorch/api_accuracy_checker/run_ut/multi_run_ut.py | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/api_accuracy_checker/common/utils.py index 76d117afb..83b73e90f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/api_accuracy_checker/common/utils.py @@ -634,7 +634,11 @@ def initialize_save_path(save_path, dir_name): def write_pt(file_path, tensor): if os.path.exists(file_path): raise ValueError(f"File {file_path} already exists") - torch.save(tensor, file_path) + try: + torch.save(tensor, file_path) + except Exception as e: + error_message = "An unexpected error occurred: %s when saving tensor to %s" % (str(e), file_path) + print_error_log(error_message) full_path = os.path.realpath(file_path) file_check_util.change_mode(full_path, FileCheckConst.DATA_FILE_AUTHORITY) return full_path diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py index df6c99a56..f2fdec494 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py @@ -89,7 +89,7 @@ def run_parallel_ut(config): def update_progress_bar(progress_bar, result_csv_path): while any(process.poll() is None for process in processes): try: - with open(result_csv_path, 'r') as result_file: + with FileOpen(result_csv_path, 'r') as result_file: completed_items = len(result_file.readlines()) - 1 progress_bar.update(completed_items - progress_bar.n) except FileNotFoundError: diff --git a/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py index 0b91d2bbc..34a3cbd12 100644 --- a/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py @@ -12,7 +12,7 @@ import torch from api_accuracy_checker.tensor_transport_layer.client import TCPClient from api_accuracy_checker.tensor_transport_layer.server import TCPServer -from api_accuracy_checker.common.utils import logger +from api_accuracy_checker.common.utils import logger, check_file_or_directory_path from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import remove_path @@ -138,8 +138,10 @@ class ATTL: file_path = os.path.join(self.session_config.nfs_path, buffer.name + ".pt") else: file_path = os.path.join(self.session_config.nfs_path, buffer + f"_{int(time.time())}") - - torch.save(buffer, file_path) + try: + torch.save(buffer, file_path) + except Exception as e: + self.logger.error("there is something error. please check it. %s", e) def download(self): for file_type in ("start*", "*.pt", "end*"): @@ -150,6 +152,7 @@ class ATTL: if cur_file is None: return None else: + check_file_or_directory_path(cur_file) buffer = torch.load(cur_file) remove_path(cur_file) return buffer diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index 9c96a52d8..931ce5641 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -111,7 +111,7 @@ def run_parallel_ut(config): def update_progress_bar(progress_bar, result_csv_path): while any(process.poll() is None for process in processes): try: - with open(result_csv_path, 'r') as result_file: + with FileOpen(result_csv_path, 'r') as result_file: completed_items = len(result_file.readlines()) - 1 progress_bar.update(completed_items - progress_bar.n) except FileNotFoundError: -- Gitee From 50ac71e0aa55c1a390286ada80f4ccac03610d07 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 26 Jul 2024 16:10:50 +0800 Subject: [PATCH 034/791] safe issue fix --- .../api_accuracy_checker/run_ut/data_generate.py | 8 +++++--- .../tensor_transport_layer/attl.py | 5 +++-- .../api_accuracy_checker/run_ut/data_generate.py | 10 ++++++---- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py index 67dc5ad25..578116483 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/data_generate.py @@ -20,9 +20,10 @@ import math import torch import numpy -from api_accuracy_checker.common.utils import Const, check_file_or_directory_path, check_object_type, print_warn_log, \ - print_error_log, get_full_data_path, CompareException +from api_accuracy_checker.common.utils import Const, check_object_type, print_warn_log, print_error_log, \ + get_full_data_path, CompareException from api_accuracy_checker.run_ut.run_ut_utils import hf_32_standard_api +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker TORCH_TYPE = ["torch.device", "torch.dtype"] TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"] @@ -83,7 +84,8 @@ def gen_real_tensor(data_path, convert_type): convert_type: convert ori_type to dist_type flag. """ data_path = os.path.realpath(data_path) - check_file_or_directory_path(data_path) + data_path_checker = FileChecker(data_path, FileCheckConst.FILE, ability=FileCheckConst.READ_ABLE) + data_path = data_path_checker.common_check() if not data_path.endswith('.pt') and not data_path.endswith('.npy'): error_info = f"The file: {data_path} is not a pt or numpy file." raise CompareException(CompareException.INVALID_FILE_ERROR, error_info) diff --git a/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py index 34a3cbd12..995d20288 100644 --- a/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py @@ -12,7 +12,7 @@ import torch from api_accuracy_checker.tensor_transport_layer.client import TCPClient from api_accuracy_checker.tensor_transport_layer.server import TCPServer -from api_accuracy_checker.common.utils import logger, check_file_or_directory_path +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import remove_path @@ -152,7 +152,8 @@ class ATTL: if cur_file is None: return None else: - check_file_or_directory_path(cur_file) + cur_file_checker = FileChecker(cur_file, FileCheckConst.FILE, ability=FileCheckConst.READ_ABLE) + cur_file = cur_file_checker.common_check() buffer = torch.load(cur_file) remove_path(cur_file) return buffer diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index f495cd673..b103643c0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -21,10 +21,11 @@ import torch import numpy from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import hf_32_standard_api -from msprobe.pytorch.api_accuracy_checker.common.utils import check_file_or_directory_path, check_object_type, \ - get_full_data_path, CompareException +from msprobe.pytorch.api_accuracy_checker.common.utils import check_object_type, get_full_data_path, \ + CompareException +from msprobe.core.common.file_check import FileChecker from msprobe.pytorch.common.log import logger -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, FileCheckConst TORCH_TYPE = ["torch.device", "torch.dtype"] TENSOR_DATA_LIST = ["torch.Tensor", "torch.nn.parameter.Parameter"] @@ -87,7 +88,8 @@ def gen_real_tensor(data_path, convert_type): convert_type: convert ori_type to dist_type flag. """ data_path = os.path.realpath(data_path) - check_file_or_directory_path(data_path) + data_path_checker = FileChecker(data_path, FileCheckConst.FILE, ability=FileCheckConst.READ_ABLE) + data_path = data_path_checker.common_check() if not data_path.endswith('.pt') and not data_path.endswith('.npy'): error_info = f"The file: {data_path} is not a pt or numpy file." raise CompareException(CompareException.INVALID_FILE_ERROR, error_info) -- Gitee From 4703e86edadfe694b3ca6909c38043a80114d462 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 26 Jul 2024 16:54:09 +0800 Subject: [PATCH 035/791] bugfix --- .../api_accuracy_checker/run_ut/multi_run_ut.py | 11 +++-------- .../tensor_transport_layer/attl.py | 1 + .../api_accuracy_checker/run_ut/multi_run_ut.py | 11 +++-------- 3 files changed, 7 insertions(+), 16 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py index f2fdec494..0ab807393 100644 --- a/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/api_accuracy_checker/run_ut/multi_run_ut.py @@ -88,14 +88,9 @@ def run_parallel_ut(config): def update_progress_bar(progress_bar, result_csv_path): while any(process.poll() is None for process in processes): - try: - with FileOpen(result_csv_path, 'r') as result_file: - completed_items = len(result_file.readlines()) - 1 - progress_bar.update(completed_items - progress_bar.n) - except FileNotFoundError: - print_warn_log(f"Result CSV file not found: {result_csv_path}.") - except Exception as e: - print_error_log(f"An unexpected error occurred while reading result CSV: {e}") + with FileOpen(result_csv_path, 'r') as result_file: + completed_items = len(result_file.readlines()) - 1 + progress_bar.update(completed_items - progress_bar.n) time.sleep(1) for fwd, bwd in zip(config.forward_files, config.backward_files): diff --git a/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py index 995d20288..5fb63779f 100644 --- a/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/api_accuracy_checker/tensor_transport_layer/attl.py @@ -12,6 +12,7 @@ import torch from api_accuracy_checker.tensor_transport_layer.client import TCPClient from api_accuracy_checker.tensor_transport_layer.server import TCPServer +from api_accuracy_checker.common.utils import logger from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker from ptdbg_ascend.src.python.ptdbg_ascend.common.utils import remove_path diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index 931ce5641..879f11630 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -110,14 +110,9 @@ def run_parallel_ut(config): def update_progress_bar(progress_bar, result_csv_path): while any(process.poll() is None for process in processes): - try: - with FileOpen(result_csv_path, 'r') as result_file: - completed_items = len(result_file.readlines()) - 1 - progress_bar.update(completed_items - progress_bar.n) - except FileNotFoundError: - logger.warning(f"Result CSV file not found: {result_csv_path}.") - except Exception as e: - logger.error(f"An unexpected error occurred while reading result CSV: {e}") + with FileOpen(result_csv_path, 'r') as result_file: + completed_items = len(result_file.readlines()) - 1 + progress_bar.update(completed_items - progress_bar.n) time.sleep(1) for api_info in config.api_files: -- Gitee From e959f71dec4a9bab60c3fc95aac8183733c1bb8b Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 27 Jul 2024 10:25:14 +0800 Subject: [PATCH 036/791] align pt and mindspore defalut setting --- debug/accuracy_tools/grad_tool/README.md | 34 +++++++------------ .../grad_tool/common/constant.py | 4 +-- .../accuracy_tools/grad_tool/common/utils.py | 6 +++- .../grad_tool/grad_ms/global_context.py | 30 ++++++++-------- .../grad_tool/grad_ms/grad_analyzer.py | 13 ++++--- 5 files changed, 41 insertions(+), 46 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/README.md b/debug/accuracy_tools/grad_tool/README.md index a7929ca81..ed84abd46 100644 --- a/debug/accuracy_tools/grad_tool/README.md +++ b/debug/accuracy_tools/grad_tool/README.md @@ -28,7 +28,7 @@ ### 梯度数据导出 -1. 创建配置文件config.yaml，PyTorch框架样例代码如下： +1. 创建配置文件config.yaml，样例如下： ```python level: L1 @@ -38,40 +38,30 @@ bounds: output_path: your_output_dir ``` - > 在MindSpore框架下，当前不支持rank和step配置，默认所有rank和所有step都进行采集， - > MindSpore中step指的是优化器被调用的次数（并非模型跑的step，某些step，例如loss为nan时，不会调用优化器） + > step指的是优化器被调用的次数（并非模型跑的step，某些step，例如loss为nan时，不会调用优化器） **参数说明** - | 参数 | 说明 | 是否必选 | - |--------------------------------|----------------------------------------------------|----------| - | level | Level级别，PyTorch可取值：L0、L1、L2，MindSpore可取值：L0, L1, L2, L3。决定导出数据的详细程度，级别越大导出数据越详细。数据类型：str。 | PyTorch是（MindSpore否，默认为L0） | - | param_list | 填写需要监控的权重名称。不指定或列表为空就表示监控所有权重。数据类型：List[str]。 | 否 | - | rank | 在多卡场景下，填写需要导出梯度数据的卡的Rank ID，不指定或列表为空就表示导出所有Rank的数据。单卡场景无需关注该参数。数据类型：List[int]。（MindSpore当前不支持指定rank） | 否 | - | step | 指定需要导出数据的step。对于PyTorch不指定或列表为空就表示导出所有step的数据，对于MindSpore不指定表示导出所有step，指定时要求传入range列表，例如[1, 2]，否则无效。数据类型：List[int]。（MindSpore当前不支持指定step） | 否 | - | bounds | 用来划分区间以统计值分布。需要保证由数据小到大排列。不传则使用默认值[-10, -1, -0.1, -0.01, -0.001, 0, 0.001, 0.01, 0.1, 1, 10]（mindspore为[-0.1, 0., 1.0]），数据类型：List。 | 否 | - | output_path | 输出目录。如果不存在就会创建一个新目录。数据类型：str。 | PyTorch是（MindSpore否，默认为./grad_stat | + | 参数 | 说明 | 输入类型 | 是否必选 | + |--------------------------------|-----------------------------------|-----------------|----------| + | level | 输出级别。决定导出数据的详细程度，级别越大导出数据越详细。可取值：L0, L1, L2|str | 是 | + | param_list | 权重名称列表，表示需要监控的权重。不指定或列表为空就表示监控所有权重。 | List[str] | 否 | + | rank | rank id列表，在多卡场景下，表示需要导出梯度数据的进程的rank id。不指定或列表为空就表示导出所有rank的数据。单卡场景无需关注该参数。 （MindSpore Pynative模式下，当前暂不支持指定rank功能） | List[int] | 否 | + | step | step列表，表示需要导出数据的step列表。不指定或列表为空就表示导出所有step的数据。（MindSpore Pynative模式下，当前暂不支持指定step功能） | List[int] | 否 | + | bounds | 区间列表，用来划分区间以统计数值的分布。需要保证由数据小到大排列。不指定则使用默认值[-10, -1, -0.1, -0.01, -0.001, 0, 0.001, 0.01, 0.1, 1, 10] | List[float] | 否 | + | output_path | 输出目录。如果不存在就会创建一个新目录。 | str | 是 | **不同级别的level的导出数据** -- PyTorch/MindSpore动态图不同level数据 | 级别 | 特征数据表头 | 是否有方向数据 | | ---- | ------------------------------------------------------------ | -------------- | | L0 | ("param_name", "MD5", "max", "min", "norm", "shape") | 否 | | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | - -- MindSpore静态图不同level数据 - - | 级别 | 特征数据表头 | 是否有方向数据 | - | ---- | ------------------------------------------------------------ | -------------- | - | L0 | ("param_name", "max", "min", "norm", "shape") | 否 | - | L1 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 否 | - | L2 | ("param_name", "max", "min", "norm", "shape") | 是 | - | L3 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | intervals就是根据值分布bounds划分出的区间。 + MindSpore Pynative模式下，L0级别中暂不支持"MD5" **方向数据解释** @@ -98,7 +88,7 @@ gm = GradientMonitor("config_path", framework="MindSpore") gm.monitor(optimizer) ``` -3. 结束监控（MindSpore需要） +3. 结束监控（MindSpore Pynative模式下需要） 在训练结束之后，调用stop接口 diff --git a/debug/accuracy_tools/grad_tool/common/constant.py b/debug/accuracy_tools/grad_tool/common/constant.py index 7f2a708ab..f37389d12 100644 --- a/debug/accuracy_tools/grad_tool/common/constant.py +++ b/debug/accuracy_tools/grad_tool/common/constant.py @@ -23,8 +23,8 @@ class GradConst: LEVEL0 = "L0" LEVEL1 = "L1" LEVEL2 = "L2" - LEVEL3 = "L3" - SUPPORTED_LEVEL = {"L0", "L1", "L2", "L3"} + # LEVEL3 = "L3" + SUPPORTED_LEVEL = {"L0", "L1", "L2"} # numpy coding STEP_IDX = 0 diff --git a/debug/accuracy_tools/grad_tool/common/utils.py b/debug/accuracy_tools/grad_tool/common/utils.py index a1e639558..f40f8688c 100644 --- a/debug/accuracy_tools/grad_tool/common/utils.py +++ b/debug/accuracy_tools/grad_tool/common/utils.py @@ -223,4 +223,8 @@ def change_mode(path, mode): def check_param(param_name): if not re.match(GradConst.PARAM_VALID_PATTERN, param_name): - raise RuntimeError("The parameter name contains special characters.") \ No newline at end of file + raise RuntimeError("The parameter name contains special characters.") + +def check_str(string, variable_name): + if not isinstance(string, str): + raise ValueError(f'The variable: "{variable_name}" is not a string.') \ No newline at end of file diff --git a/debug/accuracy_tools/grad_tool/grad_ms/global_context.py b/debug/accuracy_tools/grad_tool/grad_ms/global_context.py index d44bea52c..3bb1459ff 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/global_context.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/global_context.py @@ -4,7 +4,7 @@ from typing import Dict, List, Union from grad_tool.common.utils import print_warn_log from grad_tool.common.constant import GradConst -from grad_tool.common.utils import path_valid_check, create_directory +from grad_tool.common.utils import path_valid_check, create_directory, check_str class GlobalContext: @@ -12,13 +12,13 @@ class GlobalContext: _instance = None _instance_lock = threading.Lock() _setting = { - GradConst.LEVEL: GradConst.LEVEL0, + GradConst.LEVEL: None, GradConst.PARAM_LIST: None, GradConst.STEP: None, GradConst.RANK: None, GradConst.CURRENT_STEP: 0, - GradConst.BOUNDS: [-1., 0., 1.], - GradConst.OUTPUT_PATH: "./grad_stat" + GradConst.BOUNDS: [-10, -1, -0.1, -0.01, -0.001, 0, 0.001, 0.01, 0.1, 1, 10], + GradConst.OUTPUT_PATH: None } def __new__(cls, *args, **kwargs): @@ -29,23 +29,25 @@ class GlobalContext: return cls._instance def init_context(self, config_dict: Dict): - if config_dict.get(GradConst.LEVEL, None) in GradConst.SUPPORTED_LEVEL: + level = config_dict.get(GradConst.LEVEL) + check_str(level, variable_name = "level in yaml") + if level in GradConst.SUPPORTED_LEVEL: self._setting[GradConst.LEVEL] = config_dict.get(GradConst.LEVEL) else: - print_warn_log("Invalid level set in config yaml file, use L0 instead.") + raise ValueError("Invalid level set in config yaml file, level option: L0, L1, L2") + self._set_input_list(config_dict, GradConst.PARAM_LIST, str) self._set_input_list(config_dict, GradConst.BOUNDS, float) self._set_input_list(config_dict, GradConst.STEP, int) self._set_input_list(config_dict, GradConst.RANK, int) + output_path = config_dict.get(GradConst.OUTPUT_PATH) - if output_path: - try: - path_valid_check(output_path) - except RuntimeError as err: - print_warn_log(f"Invalid output_path, use default output_path. The error message is {err}.") - output_path = None - if output_path: - self._setting[GradConst.OUTPUT_PATH] = output_path + check_str(output_path, variable_name = "output_path in yaml") + try: + path_valid_check(output_path) + except RuntimeError as err: + raise ValueError(f"Invalid output_path: {output_path}. The error message is {err}.") + self._setting[GradConst.OUTPUT_PATH] = output_path if not os.path.isdir(self._setting.get(GradConst.OUTPUT_PATH)): create_directory(self._setting.get(GradConst.OUTPUT_PATH)) else: diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py index eb6a28aa9..9a67f2b3d 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py @@ -31,9 +31,8 @@ def grad_dump(dump_dir: str, g_name: str, dump_step: Parameter, grad: ms.Tensor, ''' Dump gradient statistic data. level0: [step, max, min, norm, shape_dim, shape] - level1: [step, max, min, norm, shape_dim, shape, dist_dim, dist] - level2: [step, max, min, norm, shape_dim, shape] + grad_bool_data - level3: [step, max, min, norm, shape_dim, shape, dist_dim, dist] + grad_bool_data + level1: [step, max, min, norm, shape_dim, shape] + grad_bool_data + level2: [step, max, min, norm, shape_dim, shape, dist_dim, dist] + grad_bool_data ''' dump_path = os.path.join(dump_dir, g_name) dump_dir_path = dump_path + "_dir" @@ -51,7 +50,7 @@ def grad_dump(dump_dir: str, g_name: str, dump_step: Parameter, grad: ms.Tensor, level0_stat = ms.ops.concat((extrem_stat, shape_stat), axis=0) level_stat = level0_stat - if level == "L1" or level == "L3": + if level == GradConst.LEVEL2: zero_grad = (grad == 0).sum() dist_dim = ms.Tensor([len(bounds) + 2]).float() bucket_result = ms.ops.bucketize(grad.float(), bounds) @@ -60,11 +59,11 @@ def grad_dump(dump_dir: str, g_name: str, dump_step: Parameter, grad: ms.Tensor, dist_stat.append(zero_grad) dist_stat.append(ms.Tensor(1, dtype=ms.int64)) # make sure dist_stat is not empty dist_stat = ms.ops.stack(dist_stat, axis=0).float() - level1_stat = ms.ops.concat((level0_stat, dist_dim, dist_stat), axis=0) - level_stat = level1_stat + level2_stat = ms.ops.concat((level0_stat, dist_dim, dist_stat), axis=0) + level_stat = level2_stat save_op(dump_path, level_stat) - if level == "L2" or level == "L3": + if level == GradConst.LEVEL1 or level == GradConst.LEVEL2: grad_direction = grad > 0 save_op(dump_dir_path, grad_direction) -- Gitee From 81699a631eedc256d82c4fec76a3f3134107dc89 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Sat, 27 Jul 2024 11:25:25 +0800 Subject: [PATCH 037/791] add divide zero protection --- .../bench_functions/npu_fusion_attention.py | 56 +++++++++++-------- 1 file changed, 34 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/api_accuracy_checker/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/api_accuracy_checker/bench_functions/npu_fusion_attention.py index 4c230c17c..d5a91ce3b 100644 --- a/debug/accuracy_tools/api_accuracy_checker/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/api_accuracy_checker/bench_functions/npu_fusion_attention.py @@ -8,7 +8,6 @@ from api_accuracy_checker.common.utils import logger gtype = torch.float64 # arm host必须选择float64，x86环境选择float32即可，64也行。arm计算很慢，s=8k的场景建议使用x86 softmax_build_mode = "QKV" # "MAX_SUM" - """ # 前向函数声明对比 标杆实现:fusion_attention_forward: q, k, v, drop_mask, atten_mask, pse, scale, keep_prob @@ -45,6 +44,9 @@ def softmax_grad(dp, softmax_res): def broadcast_kv(num_heads, num_kv_heads, kv_tensor, dtype): + if num_kv_heads == 0 or num_kv_heads < num_heads: + raise ValueError(f"num_kv_heads must be non-zero and less than num_heads.") + factor = num_heads // num_kv_heads kv_shape = kv_tensor.shape B = kv_shape[0] @@ -102,28 +104,34 @@ def parse_bsnd_args(query, key, head_num, input_layout): if not isinstance(input_layout, str) or input_layout not in supported_input_layout: raise ValueError(f"Invalid input_layout arg which must be one of {supported_input_layout}.") - if input_layout == "BSH": - B, S1, H1 = query.shape - _, S2, H2 = key.shape - D = H1 // N1 - N2 = H2 // D - elif input_layout == "SBH": - S1, B, H1 = query.shape - S2, _, H2 = key.shape - D = H1 // N1 - N2 = H2 // D - elif input_layout == "BSND": - B, S1, N1, D = query.shape - _, S2, N2, _ = key.shape - H1 = N1 * D - H2 = N2 * D - elif input_layout == "BNSD": - B, N1, S1, D = query.shape - _, N2, S2, _ = key.shape - H1 = N1 * D - H2 = N2 * D - elif input_layout == "TND": + if input_layout == "TND": raise ValueError(f"input_layout {input_layout} does not supported for now.") + try: + if input_layout == "BSH": + B, S1, H1 = query.shape + _, S2, H2 = key.shape + D = H1 // N1 + N2 = H2 // D + elif input_layout == "SBH": + S1, B, H1 = query.shape + S2, _, H2 = key.shape + D = H1 // N1 + N2 = H2 // D + elif input_layout == "BSND": + B, S1, N1, D = query.shape + _, S2, N2, _ = key.shape + H1 = N1 * D + H2 = N2 * D + elif input_layout == "BNSD": + B, N1, S1, D = query.shape + _, N2, S2, _ = key.shape + H1 = N1 * D + H2 = N2 * D + except Exception as e: + raise ValueError(f"query.shape: {query.shape}, key.shape: {key.shape}, parse_bsnd_args error: {e}") from e + + if D == 0: + raise ValueError(f"Value D must be non-zero.") DTYPE = query.dtype return B, S1, S2, N1, N2, D, H1, H2, DTYPE @@ -251,6 +259,8 @@ def rebuild_softmax_by_max_sum(q, k, atten_mask, pse, scale, softmax_max, softma """ print(f"Using softmax_max and softmax_sum to rebuild original softmax") qk = calculate_qk(q, k, atten_mask, pse, scale) + if softmax_max.shape[-1] == 0: + raise ValueError(f"softmax_max.shape[-1] must be non-zero, softmax_max.shape: {softmax_max.shape}") repeat_dim = qk.shape[-1] // softmax_max.shape[-1] softmax_res = torch.exp(qk.sub(softmax_max.repeat(1, 1, 1, repeat_dim))).div( softmax_sum.repeat(1, 1, 1, repeat_dim)) @@ -394,6 +404,8 @@ def npu_fusion_attention_grad(*args, **kwargs): # N不等长适配by cdy if not (N1 == N2): + if N2 == 0: + raise ValueError("dims_kwargs.N2 must be non-zero.") G = int(N1 / N2) dk = torch.sum(dk.reshape(B, N2, G, S2, D), dim=2, keepdim=True).reshape(B, N2, S2, D) dv = torch.sum(dv.reshape(B, N2, G, S2, D), dim=2, keepdim=True).reshape(B, N2, S2, D) -- Gitee From a8e11073f6a4c42560e7ea1dbf498fb4c6f85d4d Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 27 Jul 2024 14:39:32 +0800 Subject: [PATCH 038/791] bug fix --- debug/accuracy_tools/grad_tool/README.md | 8 ++++---- debug/accuracy_tools/grad_tool/common/constant.py | 1 - debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py | 6 +++--- debug/accuracy_tools/grad_tool/grad_ms/utils.py | 3 --- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/README.md b/debug/accuracy_tools/grad_tool/README.md index ed84abd46..1d35f03e4 100644 --- a/debug/accuracy_tools/grad_tool/README.md +++ b/debug/accuracy_tools/grad_tool/README.md @@ -46,8 +46,8 @@ |--------------------------------|-----------------------------------|-----------------|----------| | level | 输出级别。决定导出数据的详细程度，级别越大导出数据越详细。可取值：L0, L1, L2|str | 是 | | param_list | 权重名称列表，表示需要监控的权重。不指定或列表为空就表示监控所有权重。 | List[str] | 否 | - | rank | rank id列表，在多卡场景下，表示需要导出梯度数据的进程的rank id。不指定或列表为空就表示导出所有rank的数据。单卡场景无需关注该参数。 （MindSpore Pynative模式下，当前暂不支持指定rank功能） | List[int] | 否 | - | step | step列表，表示需要导出数据的step列表。不指定或列表为空就表示导出所有step的数据。（MindSpore Pynative模式下，当前暂不支持指定step功能） | List[int] | 否 | + | rank | rank id列表，在多卡场景下，表示需要导出梯度数据的进程的rank id。不指定或列表为空就表示导出所有rank的数据。单卡场景无需关注该参数。 （MindSpore静态图模式下，当前暂不支持指定rank功能） | List[int] | 否 | + | step | step列表，表示需要导出数据的step列表。不指定或列表为空就表示导出所有step的数据。（MindSpore静态图模式下，当前暂不支持指定step功能） | List[int] | 否 | | bounds | 区间列表，用来划分区间以统计数值的分布。需要保证由数据小到大排列。不指定则使用默认值[-10, -1, -0.1, -0.01, -0.001, 0, 0.001, 0.01, 0.1, 1, 10] | List[float] | 否 | | output_path | 输出目录。如果不存在就会创建一个新目录。 | str | 是 | @@ -61,7 +61,7 @@ | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | intervals就是根据值分布bounds划分出的区间。 - MindSpore Pynative模式下，L0级别中暂不支持"MD5" + MindSpore静态图模式下，L0级别中暂不支持"MD5" **方向数据解释** @@ -88,7 +88,7 @@ gm = GradientMonitor("config_path", framework="MindSpore") gm.monitor(optimizer) ``` -3. 结束监控（MindSpore Pynative模式下需要） +3. 结束监控（MindSpore静态图模式下需要） 在训练结束之后，调用stop接口 diff --git a/debug/accuracy_tools/grad_tool/common/constant.py b/debug/accuracy_tools/grad_tool/common/constant.py index f37389d12..38d33e988 100644 --- a/debug/accuracy_tools/grad_tool/common/constant.py +++ b/debug/accuracy_tools/grad_tool/common/constant.py @@ -23,7 +23,6 @@ class GradConst: LEVEL0 = "L0" LEVEL1 = "L1" LEVEL2 = "L2" - # LEVEL3 = "L3" SUPPORTED_LEVEL = {"L0", "L1", "L2"} # numpy coding diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py index 9a67f2b3d..895b8f2ae 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py @@ -154,7 +154,7 @@ class CSVGenerator(Process): level = grad_context.get_context(GradConst.LEVEL) try: shape_dim = int(stat_data[GradConst.SHAPE_DIM_IDX]) - if level in [GradConst.LEVEL1, GradConst.LEVEL3]: + if level == GradConst.LEVEL2: dist_dim = int(stat_data[shape_dim + GradConst.SHAPE_DIM_IDX + 1]) length = shape_dim + dist_dim + 7 else: @@ -186,7 +186,7 @@ class CSVGenerator(Process): if not param_name: raise RuntimeError("Invalid gradient statistic file name.") csv_line = [param_name] - if self.level == GradConst.LEVEL1 or self.level == GradConst.LEVEL3: + if self.level == GradConst.LEVEL2: csv_line.extend(self.get_dist_data(shape_dim, stat_data)) csv_line.extend(self.get_extrem_data(shape_dim, stat_data)) self.cache_list.append(csv_line) @@ -207,7 +207,7 @@ class CSVGenerator(Process): def create_csv_file(self): headers = ["Param_name"] - if self.level == GradConst.LEVEL1 or self.level == GradConst.LEVEL3: + if self.level == GradConst.LEVEL2: headers.extend(self.get_dist_header()) headers.extend(self.get_extrem_headers()) output_path = f"{self.save_dir}/grad_summary_{self.current_step}.csv" diff --git a/debug/accuracy_tools/grad_tool/grad_ms/utils.py b/debug/accuracy_tools/grad_tool/grad_ms/utils.py index 64829bdeb..c0efbdc7b 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/utils.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/utils.py @@ -38,8 +38,5 @@ def save_grad_direction(param_name, grad, save_path): change_mode(save_filepath, 0o640) def get_adapted_level(level: str): - if level == GradConst.LEVEL3: - print_warn_log(f"In mindpsore pynative mode, only 'L0', 'L1' and 'L2' are supported, use L0 instead") - level = GradConst.LEVEL0 level_adapted = level_adp.get(level) return level_adapted \ No newline at end of file -- Gitee From 970003a983f89515ffca433887de4865c65037e5 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Sat, 27 Jul 2024 14:57:52 +0800 Subject: [PATCH 039/791] fix slice bug --- .../core/data_dump/data_processor/base.py | 11 ----------- .../data_processor/mindspore_processor.py | 18 +++++++++++++++++- .../data_processor/pytorch_processor.py | 18 +++++++++++++++++- 3 files changed, 34 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 5d9012919..a6858e8cb 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -108,17 +108,6 @@ class BaseDataProcessor: def _analyze_numpy(value, numpy_type): return {"type": numpy_type, "value": value} - @staticmethod - def _analyze_builtin(arg): - single_arg = {} - if isinstance(arg, slice): - single_arg.update({"type": "slice"}) - single_arg.update({"value": [arg.start, arg.stop, arg.step]}) - else: - single_arg.update({"type": type(arg).__name__}) - single_arg.update({"value": arg}) - return single_arg - @classmethod def get_special_types(cls): return cls.special_type diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 7533e2ee0..a66cb9459 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -48,6 +48,22 @@ class MindsporeDataProcessor(BaseDataProcessor): def analyze_dtype_in_kwargs(element): return {"type": "mindspore.dtype", "value": str(element)} + @staticmethod + def _analyze_builtin(arg): + single_arg = {} + if isinstance(arg, slice): + single_arg.update({"type": "slice"}) + # slice参数中可能存在tensor类型，json序列化，需要转换为python数值类型 + values = [ + value if not isinstance(value, ms.Tensor) else value.item() + for value in [arg.start, arg.stop, arg.step] + ] + single_arg.update({"value": values}) + else: + single_arg.update({"type": type(arg).__name__}) + single_arg.update({"value": arg}) + return single_arg + @classmethod def get_special_types(cls): return super().get_special_types() + cls.mindspore_special_type @@ -90,7 +106,7 @@ class MindsporeDataProcessor(BaseDataProcessor): if isinstance(element, (bool, int, float, str, slice)): return self._analyze_builtin(element) - return None + return {} def analyze_element(self, element): return self.recursive_apply_transform(element, self.analyze_single_element) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 1c599573c..9441aa79f 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -78,6 +78,22 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_stat.norm = torch._C._VariableFunctionsClass.norm(data_clone).item() return tensor_stat + @staticmethod + def _analyze_builtin(arg): + single_arg = {} + if isinstance(arg, slice): + single_arg.update({"type": "slice"}) + # slice参数中可能存在tensor类型，json序列化，需要转换为python数值类型 + values = [ + value if not isinstance(value, torch.Tensor) else value.item() + for value in [arg.start, arg.stop, arg.step] + ] + single_arg.update({"value": values}) + else: + single_arg.update({"type": type(arg).__name__}) + single_arg.update({"value": arg}) + return single_arg + @staticmethod def _analyze_torch_size(arg): return {"type": "torch.Size", "value": list(arg)} @@ -98,7 +114,7 @@ class PytorchDataProcessor(BaseDataProcessor): return self._analyze_tensor(element, Const.SEP.join(suffix_stack)) if isinstance(element, (bool, int, float, str, slice)): return self._analyze_builtin(element) - return None + return {} def analyze_element(self, element): return self.recursive_apply_transform(element, self.analyze_single_element) -- Gitee From 55a077363ab17b537481d689890a8e2150f1789e Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 27 Jul 2024 15:52:00 +0800 Subject: [PATCH 040/791] cleancode --- debug/accuracy_tools/grad_tool/grad_ms/global_context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/grad_tool/grad_ms/global_context.py b/debug/accuracy_tools/grad_tool/grad_ms/global_context.py index 3bb1459ff..424f16aed 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/global_context.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/global_context.py @@ -46,7 +46,7 @@ class GlobalContext: try: path_valid_check(output_path) except RuntimeError as err: - raise ValueError(f"Invalid output_path: {output_path}. The error message is {err}.") + raise ValueError(f"Invalid output_path: {output_path}. The error message is {err}.") from err self._setting[GradConst.OUTPUT_PATH] = output_path if not os.path.isdir(self._setting.get(GradConst.OUTPUT_PATH)): create_directory(self._setting.get(GradConst.OUTPUT_PATH)) -- Gitee From 5713546e107058283c5b25cbb9dbf9414b57d886 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Sat, 27 Jul 2024 16:30:10 +0800 Subject: [PATCH 041/791] fix handle_tensor_extremum_nan_inf call position --- .../data_processor/pytorch_processor.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 9441aa79f..4cdd3ea04 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -78,6 +78,22 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_stat.norm = torch._C._VariableFunctionsClass.norm(data_clone).item() return tensor_stat + @staticmethod + def handle_tensor_extremum_nan_inf(tensor, operator): + data_clone = tensor.detach() + data_nan = torch._C._VariableFunctionsClass.isnan(data_clone) + if int(torch._C._VariableFunctionsClass.sum(data_nan)) == data_clone.numel(): + return float('nan') + finite_mask = torch._C._VariableFunctionsClass.isfinite(data_clone) + if int(torch._C._VariableFunctionsClass.sum(finite_mask)) > 0: + finite_values = data_clone[finite_mask] + return torch._C._VariableFunctionsClass.max(finite_values).item() if operator == 'max' else \ + torch._C._VariableFunctionsClass.min(finite_values).item() + else: + data_no_nan = data_clone[~data_nan] + return torch._C._VariableFunctionsClass.max(data_no_nan).item() if operator == 'max' else \ + torch._C._VariableFunctionsClass.min(data_no_nan).item() + @staticmethod def _analyze_builtin(arg): single_arg = {} @@ -130,9 +146,15 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_json.update({"Mean": tensor_stat.mean}) tensor_json.update({"Norm": tensor_stat.norm}) tensor_json.update({"requires_grad": tensor.requires_grad}) - if self.config.summary_mode == "md5": + + if np.isinf(tensor_stat.max) or np.isnan(tensor_stat.max): + tensor_json['Max_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(tensor, "max") + if np.isinf(tensor_stat.min) or np.isnan(tensor_stat.min): + tensor_json['Min_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(tensor, "min") + + if self.config.summary_mode == Const.MD5: tensor_md5 = self.get_md5_for_tensor(tensor) - tensor_json.update({"md5": tensor_md5}) + tensor_json.update({Const.MD5: tensor_md5}) return tensor_json @@ -168,21 +190,6 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): overflow_mode = os.getenv(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) return overflow_mode == Const.ENV_ENABLE - @staticmethod - def handle_tensor_extremum_nan_inf(data_clone, operator): - data_nan = torch._C._VariableFunctionsClass.isnan(data_clone) - if int(torch._C._VariableFunctionsClass.sum(data_nan)) == data_clone.numel(): - return float('nan') - finite_mask = torch._C._VariableFunctionsClass.isfinite(data_clone) - if int(torch._C._VariableFunctionsClass.sum(finite_mask)) > 0: - finite_values = data_clone[finite_mask] - return torch._C._VariableFunctionsClass.max(finite_values).item() if operator == 'max' else \ - torch._C._VariableFunctionsClass.min(finite_values).item() - else: - data_no_nan = data_clone[~data_nan] - return torch._C._VariableFunctionsClass.max(data_no_nan).item() if operator == 'max' else \ - torch._C._VariableFunctionsClass.min(data_no_nan).item() - def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False api_info_struct = super().analyze_forward(name, module, module_input_output) @@ -228,16 +235,13 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): else: torch_npu._C._clear_overflow_npu() - def _analyze_maybe_overflow_tensor(self, tensor_json, tensor): - data_clone = tensor.detach() + def _analyze_maybe_overflow_tensor(self, tensor_json): if is_gpu or (hasattr(torch_npu._C, '_npu_is_support_inf_nan') and torch_npu._C._npu_is_support_inf_nan()): if tensor_json['Max'] is None: return if np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']): - tensor_json['Max_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(data_clone, "max") self.has_overflow = True if np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']): - tensor_json['Min_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(data_clone, "min") self.has_overflow = True else: self.has_overflow = self.check_overflow_npu() -- Gitee From f18e6e63af3d206e59b7607859fbc6cda8e9cbab Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 27 Jul 2024 16:37:13 +0800 Subject: [PATCH 042/791] FileCheckerException spelling bugfix --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index cf7ad912e..ea9323ae0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -492,7 +492,7 @@ def compare_by_op(op_name, op_name_mapping_dict, input_parma): error_file = error.filename n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE error_flag = True - except FileCheckerException: + except FileCheckException: error_file = data_name n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE error_flag = True -- Gitee From 9fc522c75cdbb7b18dad72dfaad7ca2674b7f686 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Sat, 27 Jul 2024 17:38:40 +0800 Subject: [PATCH 043/791] clear count each step --- .../mindspore/dump/hook_cell/hook_cell.py | 18 +++++++----------- .../msprobe/mindspore/service.py | 4 ++++ 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py index bcb80dd22..57ed44111 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py @@ -18,26 +18,23 @@ from mindspore import nn from msprobe.core.common.const import Const -cell_count = defaultdict(int) -g_stop_hook = False - - class HOOKCell(nn.Cell): + cell_count = defaultdict(int) + g_stop_hook = False def __init__(self, build_hook) -> None: super(HOOKCell, self).__init__() self.changed_status = False self.input_kwargs = {} self.prefix = "" - global g_stop_hook - if not g_stop_hook: - g_stop_hook = True + if not HOOKCell.g_stop_hook: + HOOKCell.g_stop_hook = True self.changed_status = True if hasattr(self, "prefix_op_name_"): self.prefix = self.prefix_op_name_ - cell_count[self.prefix] += 1 - self.prefix = self.prefix + str(cell_count[self.prefix] - 1) + Const.SEP + HOOKCell.cell_count[self.prefix] += 1 + self.prefix = self.prefix + str(HOOKCell.cell_count[self.prefix] - 1) + Const.SEP forward_hook, backward_hook = build_hook(self.prefix) self.register_forward_hook(forward_hook) self.register_backward_hook(backward_hook) @@ -52,6 +49,5 @@ class HOOKCell(nn.Cell): finally: if self.changed_status: self.changed_status = False - global g_stop_hook - g_stop_hook = False + HOOKCell.g_stop_hook = False return out diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index cb7a6f2a7..fb593aec9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -16,6 +16,7 @@ import os from pathlib import Path import functools +from collections import defaultdict from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope @@ -26,6 +27,7 @@ from msprobe.core.common.utils import Const from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.mindspore.dump.hook_cell.api_registry import api_register from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs +from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell class Service: @@ -51,6 +53,7 @@ class Service: self.data_collector.forward_data_collect(api_or_module_name, module, pid, module_input_output) if self.data_collector.if_return_forward_new_output(): return self.data_collector.get_forward_new_output() + del module.input_kwargs return output def backward_hook(api_or_module_name, module, grad_input, grad_output): @@ -78,6 +81,7 @@ class Service: def step(self): self.current_iter += 1 self.data_collector.update_iter(self.current_iter) + HOOKCell.cell_count = defaultdict(int) def start(self, model=None): self.model = model -- Gitee From cd9f0bd61b929cfd5c46f66261aed42aa06e479b Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 27 Jul 2024 19:11:20 +0800 Subject: [PATCH 044/791] md5 distributed compare bugfix --- .../msprobe/pytorch/compare/distributed_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 0298eca9e..316ca2a1a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -88,7 +88,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): for nr, br in zip(npu_ranks, bench_ranks): n_dir = os.path.join(npu_dump_dir, nr) b_dir = os.path.join(bench_dump_dir, br) - s_dir = b_dir + s_dir = n_dir npu_json_path = extract_json(n_dir, stack_json=False) bench_json_path = extract_json(b_dir, stack_json=False) stack_json_path = extract_json(s_dir, stack_json=True) @@ -103,7 +103,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): summary_compare, md5_compare = task_dumppath_get(dump_result_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(dump_result_param, output_path, stack_mode=stack_mode, summary_compare=summary_compare) + check_compare_param(dump_result_param, output_path, stack_mode=stack_mode, summary_compare=summary_compare, md5_compare=md5_compare) except CompareException as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) -- Gitee From f95a776d221cf886fdac343e57e78853884822b8 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Wed, 24 Jul 2024 11:52:37 +0800 Subject: [PATCH 045/791] =?UTF-8?q?[profiler\compare=5Ftools]=E6=AF=94?= =?UTF-8?q?=E5=AF=B9=E7=BB=93=E6=9E=9C=E6=80=BB=E4=BD=93=E6=80=A7=E8=83=BD?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E6=96=B0=E5=A2=9Esheet=E9=A1=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 1 + profiler/compare_tools/README.md | 54 +++++++++++++++++- profiler/compare_tools/img/OverallMetrics.png | Bin 0 -> 66941 bytes 3 files changed, 54 insertions(+), 1 deletion(-) create mode 100644 profiler/compare_tools/img/OverallMetrics.png diff --git a/profiler/README.md b/profiler/README.md index 1669e3524..549ffefc1 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -91,6 +91,7 @@ ascend pytorch profiler数据目录结构如下： | profiler版本 | 发布日期 | 下载链接 | 校验码 | | ------------ | ---------- | ------------------------------------------------------------ | ------------------------------------------------------------ | + | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | | 1.1.2 | 2024-07-12 | [msprof_analyze-1.1.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.2/msprof_analyze-1.1.2-py3-none-any.whl) | af62125b1f9348bf491364e03af712fc6d0282ccee3fb07458bc9bbef82dacc6 | | 1.1.1 | 2024-06-20 | [msprof_analyze-1.1.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.1/msprof_analyze-1.1.1-py3-none-any.whl) | 76aad967a3823151421153d368d4d2f8e5cfbcb356033575e0b8ec5acea8e5e4 | | 1.1.0 | 2024-05-28 | [msprof_analyze-1.1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.0/msprof_analyze-1.1.0-py3-none-any.whl) | b339f70e7d1e45e81f289332ca64990a744d0e7ce6fdd84a8d82e814fa400698 | diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index 78ea5d897..2772ef998 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -174,9 +174,13 @@ python performance_compare.py [基准性能数据文件] [比对性能数据文 MindSpore场景仅支持**总体性能**和**通信性能**的对比。 +比对结果分为打屏和performance_comparison_result_{timestamp}.csv两种形式输出，其中打屏输出为概要信息，csv文件保存详细结果。 + ### 总体性能 -总体性能比对结果以打屏的形式呈现。 +#### 打屏结果 + +总体性能比对结果以打屏的形式呈现时，字段如下： | 字段 | 说明 | | --------------------------------------- | ------------------------------------------------------------ | @@ -196,6 +200,54 @@ MindSpore场景仅支持**总体性能**和**通信性能**的对比。 | E2E Time(Not minimal profiling) | E2E总耗时，计算流端到端耗时。当存在Not minimal profiling时，表示该时间存在性能膨胀，会影响通信和调度耗时。 | | Other Time | AI CPU、DSA、TensorMove等其他算子耗时。 | +#### csv文件结果 + +总体性能比对结果在performance_comparison_result_*.xlsx中OverallMetrics的sheet页呈现时，示例如下： + +![OverallMetrics](./img/OverallMetrics.png) + +表头字段说明： + +| 字段 | 说明 | +| -------------- | --------------------------- | +| Index | 指标。 | +| Duration(ms) | 执行耗时，单位ms。 | +| Duration Ratio | 执行耗时占E2E总耗时的比例。 | +| Number | 计算算子的数量。 | + +Index列字段说明： + +| 字段 | | | 说明 | +| ---------------------------- | ------------------ | ----------------------------------- | ------------------------------------------------------------ | +| Computing Time | | | 计算流耗时，计算流所有event耗时总和。如果有多条并发计算，计算流耗时对重叠部分只会计算一次。 | +| | Flash Attention | | Flash Attention算子。 | +| | | Flash Attention (Forward) (Cube) | Flash Attention前向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Flash Attention (Forward) (Vector) | Flash Attention前向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | | Flash Attention (Backward) (Cube) | Flash Attention反向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Flash Attention (Backward) (Vector) | Flash Attention反向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | Conv | | Conv算子。 | +| | | Conv (Forward) (Cube) | Conv前向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Conv (Forward) (Vector) | Conv前向Vector算子。Conv前向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | | Conv (Backward) (Cube) | Conv反向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Conv (Backward) (Vector) | Conv反向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | Matmul | | Matmul算子。 | +| | | Matmul (Cube) | Matmul算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Matmul (Vector) | Matmul算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | Paged Attention | | Paged Attention算子。 | +| | Vector | | Vector算子。 | +| | | Vector (Trans) | 转换类Vector算子，主要包含Cast、TransPose、TransData算子。（仅针对NPU数据） | +| | | Vector ( No Trans) | 非转换类Vector算子。 | +| | Cube | | 未识别出Flash Attention、Conv和Matmul的Cube算子。 | +| | SDMA (Tensor Move) | | 拷贝类任务。 | +| | Other | | AI CPU、DSA等其他算子。 | +| Uncovered Communication Time | | | 通信未掩盖耗时，包含卡间等待时间。 | +| | Wait | | 卡间同步等待耗时。（仅针对NPU数据） | +| | Transmit | | 通信传输耗时。 | +| Free Time | | | 调度耗时 = E2E耗时 - 算子耗时 - 通信不可掩盖耗时。Free的定义为Device侧既不在通信又不在计算的时间，因此包含拷贝时间（SDMA Time）。 | +| | SDMA | | NPU为除Tensor Move外的拷贝类任务，GPU为所有拷贝类任务。 | +| | Free | | 排除SDMA的空闲耗时。 | +| E2E Time | | | E2E总耗时，计算流端到端耗时。当存在Not minimal profiling时，表示该时间存在性能膨胀，会影响通信和调度耗时。 | + 可以采取最简性能数据采集的方式来减少E2E耗时的性能膨胀，示例代码如下： ```python diff --git a/profiler/compare_tools/img/OverallMetrics.png b/profiler/compare_tools/img/OverallMetrics.png new file mode 100644 index 0000000000000000000000000000000000000000..b130d3607344c983a9304440e38a45fe96a4bb56 GIT binary patch literal 66941 zcmdqIXH=72w>GK|3i^maK`9z~Rhk6pT~NT#M5P8I9So6<5C}-ofb`x$q$tv)geFxW z^j-p?_YyjU651D@x9s=X-`?Yl^W*$EW3aeqa>KgUTC=QcUTfZgS{lkX|Gf9-rAwD? zs;VeyU%GUq;?kwd``5@x-#GP%tdlO6owSvoTq@`Wu8=-lu~JZ1xOAx~?8b@7Rnq6{ z_A2^Lmo5PsFWTi6yYJ?gE}dtoDk;2jGhVBa30+X>=e#GCbCt5!^fDuiCttMp32U@w zzKmza<6AuNaJL6FH=S>!*D?l2TXPchn|QWQ#3xjooOb0fr>vVYC~*hXbD{T%NeNN3 zV#3B}J?BSqY18Ms=O=R8Q5>o})aGieR;o!CZ6!*Q?eF5sz(8Ne!RC})9rAepsO}7V z&ZZj4@$a9u0f$p^I)Q7xECL_n;^H)!q}{fr9hTUeW^YB1{-{mLF|hfHP$;vUSEle+ za!qA5``cF>?mlX0_M@O;q7aXtCiPY=l`zd_*>FXd`LDaej~ZMe=h$z-2(bk)iq*ai ztEyN<;@nH-LZ4xBB}_h50yxO}rIZ#op#l|K@P)axgy=@4RQbO3uwT>pPko)bs%|Fj zZ}^%cmHXaFv624a|qXoBi4P?G&)N)okBgNVW?W8jjnvmfID#}7M#9BR+Ap<>t*aGog0?+ zKo)Uu7K?!%bEZ5qGCW zGOfV56GNE2nd#XTAu_HC;v3%}mVl(o_m2?%s8Ba}AS%v~F^)-2#x37*D-iX}-M(}w zPF6Z)Hl1o=zuqsslU0TTa2vkx6YJ=9z~iLoA%O}tw?%$YJGkrqr6x^>vHQj1&?uQ1 z;y_!9g{q15fThfAN$`NzqF=|hi3w2)^0wh6eakLcj~+&8VPf_{P$NO7CUHlB(>>{R zY56|CR_uDs(Y2?t0Ei*7-N!K}N)Z=8nViwd03@mr1Z?mO(9IK zaPaQdM%Hh(hP#+>bFk34bDcJMO*)+a_8r2Xqi@elHJ|J$AlGQ-F`^Y3ONpR@(zqi_ zf=%c)Q~wc8v+HHIFqcfg$y#ml(KU4Fum@lXB-a-dHyD)^(&!r|)Sau5ZdN2}j`*+O?W$!1S2 zORCbo_4yBgJ!yx07Y=2_BO^K39i)m>=ZkKk&}9hocM69Qk0%}x4_rWU9)(pJ2T=ri zR!odU=q)(m1b!Zy7g3Z4$L%}9$J$sA3a|=$*G^R!0u-Ge-Z|P2uPsBunTX*MffGZR zaK#>703?GMFbHN`yn;}+OEzRCGCd2s(8Z%zueL}!Mb&F6e`FKBo~xA#Ht+oYr7K6| zl*t9xYI*e4(l=IPIi5ZY61~;F29}PzRh81qb>*u@bf|x1ll&8V^+)b%#PI#zb(~o6 z94A9@WrD}&^`|ZA8inaKx<+vXFka|MFs~EP)H-*GxkoW!StZRdG}r<@Q05KBzP_(P z8u{gI*6s((%G@^T%6&On^M)Qzq&la{n0K-sa~DZfa!X;Ji$J&0opCSHRzZfKwb0$G zPXrV?KU^=6C53})vzGPGYMEvl*o@~`RRQktP22#&rPrlsmwpNg^NvOuS%-DT?=5~o zKv#`^<}{X7Sh0(g))aUhf@uV1*1Zq=9)$oJe3D%|aab+iv>%T7H#NdBj&H%z+f%Q?ClN$ zr5#xxayJ9CFA)uEY)M(P=6FG#Djc!Z0bdy0V+TiovR=MoeEOkFFw2;AdY&)48@>> z2Sxik<+RJuOT1Ru*~wCmc5%Ij;v)Hi@X8nj=xvd^Tr}a--wg69_xUf@AKg zDzO82LYHvsUGJaKN3HZ3f$ZCi$NXsD=ie%ZVI{=kM5qba2luVm+^FA>z~l=Nn^dXm zHC}ZH)%1{z$nUM z>DyR`rxI$CXzi{OTN6uW!9tw1CPX>0Vtz~eS842-ECtFE4K{A1A~{m zBoYl`1>!0LJ@?QD*@0(l(8WI(;;CdU-honqlhQC$svJPdN}WNsVBwaL$K2|%VhvE6 zt4Z2^`sIf5BG+jqKOlnF1w_9Bxd)D%xQH-AZQ!{>^(|cOtFqwZ8YvTZ;Iwam~b}y@#pzocyh?N_ZWQGvP;IsGV&>hODh&tAE}w>zp6kOHA46)7tFPqJ`pAL@I(0JQx@%pNuCn8}aB{ zUkGX$|UU(nd9c1hjuoG;Rmv_XIaNaOl%LMuOA~jViUk72%Ow>V7sDi1 zU@hUtB|Fu}jaG>i#+Yil#~o{&H!WtIkVjz0^&6!eH>&BX(fP4{c=>9j!YQwCuXG5v zBpX3oRU?`amv=s;^~Py~C_RYYd(#4i=0!mar+~UR)N!!j0MYpEQb;AP2!Us?nZJHR zh8r^vUN*zNJWx(a;a&92^chF(L628FD6yaeaq@X7m;GAvcc6k!C=Tqc&JSKJExVD1 z_G0%x4fN~;Q;ThWh~JIgCgXg-VMccv#)OWta(2bQK#|U-SEbo7%u;HFs9whdyqf?&UrwH-X8h*0DKA>JP ze835ENm>7fY}g+lvIa0bY8axRfm~4-r*sgGW4ArozDIS%vGr3JD5r&tvVFni>VkN` zU(@qJkA){Pp+I9PlIuJb07URR)u)e519zKQ4}5!Yl|w$`>?Cu5lmmDRb+{-njnaEQ zM;#UYx*Mth4`y-mjb$^u`G$5O6|#re>qO)Jr8G_wx$How%%@>sS29EfXz=BG5l*tG z3Sg%Sty`3xo`u>kf~}E9 zU&cdOcf(~TWv1&T+O^$1+y3M;Zl|mvirkJ|G=<{kv!w8_RggrwY^@sV_^GL)@jZ)9 zwrx}Mwi^YHxT>DGJ8jsgZ9aXCvqVx(9IIfv92TU zNUu31$P*YU{%-ZxWYnX*wzKP&JRQTsgKgtj1ZQrU?CAJ!t7K&s+e=HXll|`)b<>h0 z8Jq&r;i8|Gn|fEplDt=G3VBa;8H{E2(iZIqxtpB1;Dzm%h0w?}q#rM#<(`teWlaeI zyfiBLDH!Q!tplxWw*l+bIgsv-pDRe7>YtH`3+-jCIF+BjLoE zsyOtk?%=4`^y)!&8pQ8EQ8eqjMMr;%4{jk8e;~h5Qc?O{sBSm=p#1m3@ASay zhFQ&q0ETuJV;=Pqi@u6X##{PQ<);Lh(8)U4JZYm(+iO0qRr~;(G^R(`<04comBc(Q zH|jW|=bnYM-8*rmvgN1D2#6Fa!pk^?gcoR>VqA)lgp;S=Ky(sHbi|)W4eXE<}Gz2%ZMZEEX`yd1cM979;T?I`%OAF$AfTbX&LwKA94_Ft( zS(F1D8R*s>LKHV{z-?33x#u#*_oHB*v%N?|NBe2WMnf54@kh(0eCY{gC^xe0M!eSw zq~lIdT-jD-?c?m){Va8J3nbi-vn4bgdd~q)qk@*0TktxS`|+EkdQu!16%q;Bz6nP< zL2Q^8-9xVE;$?O2xmR_r-f5!OH++O{K`Nzmoa(I}t5{T=&oaH7K*+6>v_DRj5_{BU zjIGacT6)ymlm9XppUbd#?_4!)fTZdEoMBX{!j!GL`dGzAuWDk(BV0gm&SKrFev0yE z#`_ZbDNKT-C{Gi-yGP_U?N*6ju`^ydQt<_^KD9ESas=pCK^Ey4h)?g|B~REkm_Y46 zZEY}EuKc3D_;RJzyvv}LXRf6zJ8hWiKXGC}QYFx2E)46&60Ruf8J$J(QI89*kc@W~ zuP!SCEbPlmFloF+<2RK_qBMfZ!TuccVHOG0u-1<}jmOnjcn zSq9J-D4Utjg^BP$6C2aZi-fs~#~3q3fd$Yb#C>LbzfmHP|?=T8DfZkTPL4Vkpn{a;8u&B1?DWP zNa$EPKWHdsnd8^NperL)llY2p{u$fr6jF43ucl)vYt6g!+bZ~aI%JKP9e0&sRJz7J z6I3ZTA-nou#bijoN?BiQ!g#Dwpaq%d3Mcv@@`GTVR7=@uOt4GGbb3sUZxDn#ZJ8z^_#45Jiw*pC? zPRH0fzQDAgLZ$p7yI=scH?#7prw@;UAzN!qq!|YG8!l`hp?o+*_fz9!iEJi>=xbd_ zWn_gVTXK{u17O*15GtrDxq;sm?*n<_oFsZRdaf436IvrBb9obfk#Il)s#86#L$3;u zh5+|-*ZO*md_n8*+_K2t{ItAb!nf9^HKaiAjz$34Ue2pZ z04n!aS!fot!(hO#R1Dw(8Kdd%Z4In&1z6XvC{D3t@u&p49c~9s`ftJ4e(jQkJ3Ju(Hh^G>M|Y4ZC{63WTwaO zcn9P3uBuqaRK1aB;fzb^?K%_Lq_yf(b=}NfmkfJco(I7^ciA~lT9Gh6NIy>3#OqZS zCoUb||F#iGV*u`l4T`j(d*Sb66q$zQgJD$ey<~l~s7}rDIFw{>_iJ;1RLxp=ksb1l zm!qH~v!kXXGikSvz9D1ePO+hngE8uV)hLf~EV?{KxA0YWB}f9sICsy*QH7ZwI)%a- z&(DQ>At)cvNc1ZMTAM@yBt29-_OOz4M%SL!M#J>k6r+?+uMqZ{d{uFBRIkPeX|i(*RHI;8kcYOkTjTYD!{7< z2U9qegu#+3UH|(OtB2U)VfTlVxa`M5vVU_UDjI1F72zieh2hJozhLD&!Va}v%(CMQ zH=AQl?zxy=^7yC@h~RKn6DW#j!Tc9Z*MzeB2irHrUi&J7Lf zdqmPGj#B6bPK|l+HtF&oq8f8*=wF1v!#7%7mI_I%q_*NG$@X_~rE%c)1x$oorEG8F zvlhN+NG;|6CL-M=O)ogO+z$!~@mJxWNG+RcHQ?`JHr2oROCFk%FI~4tZBK#m_TNQP zKnb{@jtc%Dw=ktGWgxW#+3)Os7ZYTq8HFyA+t>A9)H;1d6-jM?!bR!t;sEL!`(NqK zpY+jz`n{F`q(=M_dU5#=k+|jDbcIggmhNRg(BD_+|3`~e!sY=XLZQY_M+diRlnboIPcYLMkMJijNZfX zFB&=w52S3Dwxq&QhK>4(4#>?mGOLD0i7IwSiVkoe@G=chz_Y!ZMxsc3AgO!E(i^SC zi+Y^2 z*2^{NYmu((ut!y12eQ8**toneDBVsY;blB~bRk@zHBCX+PC;{9a%G+06X#mu`71m* z#^xQf@dOQZm0lrplOpt`3(74&C(WV1Cf@_u59Pp(z$6fkf~3G0ACzn+ov^F>kI}l8DfU#Bm+72Cix{+q z(eK|>!$#_tZ(~YGEH){8U?rUS!l!fmIDL1}vsdTYn2dJ9LuUF>Lmk4*26bU?gh`w= z+i{jx>vu+M^4KU9oQlY*o(gTUQ{IzCv#)CYa@V)fMYatWCSP`~(T zRQOJC;ptKTZZo83=c^}NHnA(ud1HaSc{ZY{5ykyipKmk#&Ac`f4Mbi`WKBvKl1*bZzQ(JrhriQ;51ul>Q_ zEjO6#JY`w{ij(&!>`_}bsqwm$q(AyA&K|j4DRbRn3h&at`y*E?ma`&q1CkUWiH?1p zrUTCsFdV}qy17>F|Dv9gp1(HN3|oF0>MXbYGOa`A+B#bH*}`*V|0!Y`(t(mLje%(q zS6#)ibJOxr8$V45Sfc$=*&zPtIU^t=q8JR@9(_RXy5)u~{BYP-(I`aC%j2?au6S8I?s5Jd*$Vh% zX$&2TM5 zd5=+gg+Q^Ynot>0ffe^riB*Y_LyILCo*>Rf=dZ?K2pBs^pjuV{{9Z zzUH%E{*!@p7NXwSP;kWgmka0ukmtSZ-(5JWXik?2#V!Ut;s%a;6;+G?UW%nuBjep` zW(9)!CdT_$nIX{e0na4CkJfPo9}iudiRyv!?nQ;1S3Y0)kQ$eeaqxs!^{N!qFfAl% z9<#5osL5bxZ6PO=jw0alP+N~hGZJuZ@ew9~P0KDp$#Ra}zQAK>;m~P|xfzRd)hf?Q zw~^q>Z-ijH8n!I|Ub!9(`!~xtEC&V}_NdS;#C?^$t7Agy3s_eY}=b6}fPOTwZV zfv!55^at9a_HFE~r4_cm-~vKrR_%ib&gU}?el2srEx1JWm+TIXMU|RRVR8F{Isd(w zOML5`v?O$xstSp870=91$w>9v^P>d1)*?v1fR(hX9!15eM<1Bxf8tF#@+S8XGxe3j zC3`h7UCWJSjg{}-hg8aR_#DS5;`R49sLZ}cf$VB*Q-*~B`7lEi88)1Io$skYw_S~_ zNhj&(p|j3yd)Q)SGP5=*PX(E7#c@6J)cJ{gxro!CQ@Gboq(Sxc4iTOG6&7n|e=d+;akB!!*Xv5&4AVJ&vix&yta(=c^agktSGlp+pCGCCExN;@g23^_n56mo-5qo;2r{s=0OH))$NbhG8wbU3PqF7S_7$jH_PJ-i+-D6@ToSo ztZXurono~}!L4|n_)m+jqO78>amRloTRJFZK#sVwK{3BDJM*?b1yrkVe`epyCP zSb1~dJu+Qn7fmppWsOB4jy{FPppE}!#R5T9&k<62UDk> z_yBx&nauy18@Uk$TQB zmVsH=bpb=vu0cwTs*t43(u3rSZBmAVcKYY`r50K+&@ zn3i5x8W-1Y@I)S?V6^Q`n?QQ>K}_6??Zywg^^b=M7cp69$>&=!gZcmpdW75~d}!G<|B589a=(?<3gW^91r@F)U z_bnc`s~{pIJDX2$SlNW`n7)m=YaD5^Y(bArhfLL-fjx#YTWP2fdW8dsfNnNTOJQF1 zU#*Wnb#Bi(c~H;RVqwZXk`V#;A6*Og$v7V&FRi{sU1E}(JN%@O0lJJ?hn#<7K4Sd3RC?$ZzCN_7gT5z~aw*JL)fW=NluHFn8l!Kob5u#f5eDsRp?IL++^$pe4h!pu}Cz+uaY1Ut<)`du(bX~5MJR_7; zQNgCVBdOH8-L6{v?4P3>yiFS9KL$z?JKhM-P>jAAj3Ax-a;O@ku1Ok8s1B?Ak9A25 zwp=GH_k$%t*4!0mONMKuJKoiC@QvnkvCaj~&Yo271i41!50A0~@4h0H9iQuIZSAFY zHRRBK7p9lT=Q-_P&jW)$Gw*xu)%_mHrDW$0a^KrcqCv#%nx;@UGd;7tY@2`Zyk*!6 zyk3Iru~D9k(_=@bj6Q|uOX5e0^KT#3&^yCSQ}su!`QDYnh(jMvz(->4j88U+#9Tb% zQ4yhhlZL;}luwqT=ozsJovnCW{FUxy66ju;tEQ=}apfH`FB5L9X93ooQ*I$ML$ol~Lo+Yi_bN7h&_x1IgKb`D+COa+!-T3Y^7#C| zNlv%Ok6~$3=JXA%eRdMwbvH26m@lI1{Y3ZxSr8;+vR#kn)y=x0-QLnOz){^tA(#6* zRiz{}G)ig9ninR$0ai749Z-s4C?++-sm9?g`Wvjkk6me+p?eVRE(&lPWQKQs6JC=Y=0q$RdHQ`vCa=s zYv)V0HpW}~IXx)sI*$xCqklyW!937O{BnDY-`A-{=~wT0U6T!2(Sv-A#MW-NLk3aW{aIJS6cp2b@tNvyi3ApaJD`SOIwY;r)SqQe5-N6hu-65 z@ElE<;$$e!a>?C@e{ckRV#HS?e%*%~96WoVB9*|}GnJ>!ZPg;S%qH(QTEBWj05+O-cp&PS6f z=^{0W23QZrYV*7Eh`>VQlvd?PFf}B9Kh7LDR}n`%)n`T5 zXX5il)iR5saENl^EBKAl9hxMeWx$~1y^!WE*PC_sVhStA>e_N|!^(}JZ)&n840N^` z4Zj@=(K09Lc^67+7rt*yee2I?E`(+^Yaf26);YjiPO5x)bl*GaWcFV8fr(zyR%4dB z{fxo+uKaO6f0qGcJKAWOLoZhYgYldP!ZXhjgXic6X&@6_WZfoyA>yIZO3zbjT$62G z_=bj^+$xzQ1Pj)a*Q+V>^S;{d%feG4)FZ=`4$;ZVbuYX^@oSfrZc`Ns-KO;JFKTG| zv$VtWVc+gpp2mTL>4(mtD0SmFoTUzKB&VZD!$-t$HUIt4@E;2ep!Qg|d`vV{C;WhO z_&0By$Pj+eU^AGkY@v;=%(PdAl)-H0(*+r>n;JEfe53Hh*E9{Fani5jiUpIYq_(L_7AEQ6B0NnQm zT$vieqX;zwe1#k|R6@#EAB9<$_{5W8V(kLc^F${(>0QjHxhqQ9hv-ujCCnCq!!Op* z08x$U221j9WoFCx2&Y1UUxlHHhWbxO*O2dm>V67|s-~v=bdV7AEZnJ&K>NvEWhia{ z*BthEUu8%=M|h2WZA$uD08-vT42rVzG3k)a-OEJ>PGJ}!PBnF zqI33$nimD0zlp5@P#cEe=9Ua}Lb`}L1Y7buBK`-w*aCK7-c|TnB9!>rRNea@;T})t zRZ)6ffFr{8cZV@H^0ml@s(qWZbIn;9HjBZknyxVYXBovJL$43=nMm%5cvaN3ymidm4! z?K~Q{YiKg_o}L}ezuM_GPq2Y!2jm1gguHeSUg%CNAC}5Mi##1kXj&&ElQjD_DbV$W zeoJr&;oa!)_Y)n;DvATltii8T=*YGECM~e^Ci>IhKDcLHN;$_P+?>jwz^bQq*CKbDgJqqH2KN!{v$h|7oqRy z0_r);PUw^ri^E2P& zV#X$wgsMK3e!kXxiSebu_9s$+d$OsY5R(3R^ehwCnRl_%%eS_#sg3@WmX0EM`IGJv zv<@HBRu{rBDTlNYp1hlrO~8`kK0SyHh3M(qXSM4|dWK{>qW+TvLi4Z`dceV@sgNb8 z=y0UTpw-<@9k%}}D8=~byZUhI-A#qg7T))pzh-r(1J3pfF1z!6B3!>%2)Y2J$h=&R zjIGLi!ZTGj8;P=fjpyi9|8UMZUl6mZvyor8U7v>igunf?l!Oj{cqGS>3)`~W0+V+J zbEJ>3sAA-FmhEDM*)2062{6XgF*djv*KL6j7XwsuaB4|?pTYWc4h3fa+1*`7fvyb$?O8$TC?HNVMILhB-d0t<&<(7(TY{O3c35YLnL+ z;T52uE){@@ty6Mj>|ej5VLP%kvOD$ww}G^8eVs_#?osDQD8%O2^_`tigm6;LUPt)~oSCEG;i(SW8=kczPrWM90V6Rp8lz)Xs?pVj>r`GH;Fz`B{i)KD z!bvY+5A$mI+=WatH~c%D2X_MV)SH(KTq5%6)z2kw;E&7A1b=74MTS$04q3&Sf(&8D zBly(g*StE3RkoUM39v^|MS036`&g*Eggf7F%im$r2^j(YksK+HYm{O%tVaL%jgRP$*XGJGw)P9+dhH)%mifEo>%ZAI^SxeUqiwdbMrKxN* z1GMuVQ;w6VrvBww%V%6|`5k6Y^6qa1Y1c|Wo+;TkeKwht)kT?i-_**IrrnYp&mmC0 z7AehT=#cG}tpyBIShC8_h}CSu$M%FA&+T_e^4~X8J$izfH3O9}G-umAhUXUT2RBnS zBzExuOFvSC-L6O@RNdt@tQ`z4a?wHlxijeT(4EgTq%rIvX*Q>>cf}K@iwD;YK$#k~ z4GUc}1|;M*zIi@P%?*>J1sbe6+dnP^Jp|2+>;KV~q&SQX%~>3{SMorU(6?i_qq5`g zpGLRPS8$sQ2rU?H(9|Ap74<~jWk}i-Gwy8tHGu2vt-4!B+3Vm@#JEDoycG?fws>2@ z8`zUXVg;_c>s2{$ITj_D-hAKu#Yq_ZVvO5BR%|{@6$N7~XqGce8RnmtjxAvRSPNYl zSuHX#O!>?4P-YGtv`o^^%q2e`7zD=TFD2^8h2! z|Ct1Ngzj4}sU2IadO;E29?!lMA^Kf+K4Sj*b?@zVx*uItujx+Z(T&-1^K{QJRY!sj z^++N1&F)UiXIOq1#l5l5qNPon(6f{G1QUJZ$?~|v*pNu&bY`oct~GJ9!$jF(9382W zPN`!_U?J?2+oYMaER#{)y`#9Re%rKp`7&nP5|wha<#YAy%>io4`FfS*@A!B1Z0L}& zn@P%?q#(%?(DZ`e#$Q(QHm8!u;0#S*j*f3fR2bgC7fb`s%y0dUuDL>jVSq!;JFi(_ zgn|3{@Fjv`!e~79OU{@TQYfwcFkc9@kn?6BQaO^_yevWri1=dTCG|}`qUeOy{rZQ* z8VZ4({idqcqr<9%3S4jsI=(i>&NavD@{3aLuqPu*I<(k2Cd0=l)9{z?MQJ75D@MIK zmaOCQxBCZsGCT57)Rns3Jx z{@!Z)Lv=Cw9XO?F=#Z`UOpd0Kao*+ri-Kdpr8%mQ`P2AOwP3otQlt0%hKqMW2f0FN z^xKsNz%JvjKe@h+J?RN=TuNpy4sO=VP1SsC3$rIh%bt$YBE8YyeiyNFNRMIj*FI!m z^_zHAVKdqkD0^XvE$upvH}W_mdzIU_yB_lDjzor?MM~$k^-ket#VTGIInO>GrQ3}| zO!vZ4VyzbF=7#>r6hEd`p-mt{aINtVA3bh)1^QfR?T$X{7Ew(N{^@{BIVu}9H*|4` z+FuP0du)*h@f<)dvu$eRrRW;x8S)n3C}iMrlZq_ncd2~99_|$!VPLqhJLL$EYl#$j zN^-@wyu2lvhPTuRZeh9_*>3MXXe&OM~?I@c;F~(qLjp%w0E+ae7zuTGp%st(_`qAcXm~IPuQR{UADQWXKhWj zm+A7|s`IQRr|a$juiCnYr>~tocur0{y^%+rer%7Hum@V&EeLPlu<{cosZ$Zz$^Hsq z`uMqw%Ltd5QTsBc;o2E9cKqPm@{taIRZ0K=2fEAblKz$m(!{P@~`>PuzmAuQ)H}~ z1%1zOu+g#!kpLlnjpXSc;&0#YGS&wN-;$o3oZOiY-0;R9tzDk)P;5rGRMGEHFkd-( zORX^lt#0}hK4s=vanEyPGG_Wu>+bE#Ggp)0C9jEMqI6iR*Y=g&hAGc}8I@ValR@(U zfhI{p^!utc`ws=aoUbNX3v&dE|KXUVAIYr`)I(03fF5E}mQ-BBj$eblmqD{um8nh) z8QaMfVgl4bF4ZA%caKf@yskwTRnSu0E8Cj>>4olq8uWq^dxo3zZ^1D$)Iq!RG~KW^ zjuh)ij@SaaGV^O$b?;E-_uKoJm$Z%VFI});(q6v2#)4wZbc*i{#9+Pq3bQ}@?XEHn z->O$+f%^6De5h!<``OdT4nql=3=&d5o}IK>h-vk;W}W&fV%)2LqTU1(F9!_;Dc$~% zB4N2>4is_yF}88t<*a&F{g3tPxO;?YxXYV@jOy|(SJ6@Vr*gLm60&Sx$Zjt4pM6?w zT*k5ul=meVCEd+;?a4Y2HcRVL0cym(t_P?ApLYjQ)GcJaeMReV$6S=G4tiiJpI>QU zr%)>q;A2@qRn)vSbIhIJ%_m_c3Gdtd7gH zt2N6Die#TLX$7wj;_+7(&we1Zeo8*V(zh5TU6N_BMbex|$Z}PGpfMr9v<{z^8DlK3 z+w9jeIHxP0F$0X9iI3Lik8n@ePu>vy~@216q@UJNrPNv3JgwCE>jsCpMhgM>ox3 z)gQkt|HuNs337}|DRG`}+b(h1Tt(~%F-s_W=hSh{iIa!zdfNvMl!;E!KgtxKs%6Lv z{;YN-h;L@Lixm%Rd9`a3G8+Aj*o5b(WGAHhZWy!Y8Jk{Ia%`^mXwJrh>>MmOXhX*)T(!d+<_(C>M(J7j(Yl}e^oqv zNTAIKONrGtoaKoxV_+G!N!l_EwiDZ$pr89H6Hqtn^uU`IDkDttjbRN>8QETsZ8lu% zZr)}ERAf(V-paQNRa_x`uP5-A*|_B$QzMJQ^${ww?V}esHBNw9b7aet>}3Uo*zQ+S)P8sWOY)73;Bym>(!au555)3IAcPuOL4wQQZ_@<#83ZeKMbPS`@Z(j zQ1JD)zPDlz{VJsP$J7A#?|&AbmiKhpeSvFZ&>B%Ijp012t@RccysD==DAV&>^LXdVW#R4G-a*3!r{4Lz7#h5x6 zHg!qN-F#E9efy8=9SSlF5?Z@02+|Q!JU|?`*S-fwxmSzqG{6$(m}m?geY+(Fp z-GuEc?p?!{+oe^%KGI8!l-S(Kafxh#!~9({nEqw2Y0(3^pPd=eHKU5PZ6o_d`IP31 z`a_z&axE?XboOj(I9fzM5lS79aI^AH+wzXR(Cx0OTP^iJyR7f*lNt8Hj|G)~(!r%V z+>cG7#K|3#98BNS=B9C|qe2Mz$mOf}UQ;6gA?vy&uUll|~m|8n|>HjOT! zt8}*Bn$KNcua5Ymj!!;ij`<@cv5(<4Jo`yf<`ctHetwSVOeRvuk2H@d9k6ZdRjTAD zd`u`W+mpb78io{Mej+a0mmf}4pLTd$;s`fiiz2%2`@{(=68H1Oa(mxO!`U0CriTB3d zwycnh|7FHcQXpK${r3CX+sEYXfb7Q3l&l^_d_dp0Ajjf33j9UWOGgb-oBACQAb-*U zxVg#4}n;fJk101Bo54~Q!u;3YSdoLcXF zT^)rATet6hiT|Z!Hpp7oJSr9X&m!HBaoYEPkYt@rua(fU|<{PYvD zzZIuSgS<};BO<0Q-fVs-Ur%~*?Qg}q|DV5HUWwQ6kPL0-#f@)JoghO`*s4tf!#QF;ydMRUPVsV_lgf8?u3?G12(!#4YEn@0*`2E zO5X5sSvKg6Qe`RExm?51GVa0oRvZmf*Ssa+ti392T07BCV%fARgWhc}vxZE2ZFtA&wWKi~Sn&fG`Q3!Zl%gDuw>|KEdUX z@`>*Kn^|sY?~Kfl!O~yo$8xaP8=a>gM=b#;hrxT6PR1Y8!y@?);LajH?)%71wJ6Zq zN2k)oqv=XWv(uA`k%K1K2IS4m4uLNv1UL4o?_q8UEu^36>s zI;sZrYC^`e;#)?`kz9pRU{OKrmj+W6BMXPiycujx;K3^_=`(*vD=-UN5%+sMSn~U0 zubri&Cf}~5cl}!a7Sjh`R>VGy&wMFg#`K4W_vNYy*MbesuOe~&dE?!hZMl2zmyYE> ztmpM!ue+AJ>96za1w)e!$p9Ok0nH$`ev*d|WaD;}_j{BZ1d`lA|J^t`Eg!G_THsup zzX94{Xp~`;X+Cd2Ka|lPYA(|*pv7Ha2x`86Q#McR)aJ@CK#4G|wv1Tmo3Ni~*ZNMk z8C&6vt;%8W5&3&$p0t(mGFILGE`Z^ZevumYm?X^1;u1om`NW_~f9uutf+win@@3=n zp?9yy9Sp_?ESs3|s&=yhBNoD%!FG@c(ED!FMI z{v)o&a?utyZr--xn~=o^9lgTXg-GyeGH8KQ$=eCYLEKXk<8X_x&D2g1D<8IVyu1w6 zPkWNecz=hS8nS=J)+^T9zMAKGb?N`G_ntvfb?eqB8IdR`IVn*xG&C6$5hMx*az>J5 zu#qMQB?vS@B-lhz0f|k{Ip-XjoDpb{oZ+ta4QKCj&)MfxeYXmi@D268&&m)BToZZu*M`91Wov0N|CPLQnA@?{7O0 znEA&Jpqs6F#!c#c+)G;MHD%6GfiBCp?dm5%iSu$Uava+Cn6ckSh?^kBzqYL>#K*;( zj+jspMdGK)P$!oJ_c_&W^d_52-+vLFXMt-6`)az)e>Xq%bTfX`lTfz!)#skx9%p4y z%quS@d}xF2n}o>#&;2~N455(XMI&&a%Z#PC;?}&%=pdrsGKKJu<$`{~<3=6To_pmq z?-ytG$ldlJG@Rr1ar>KXPEQp{ub|-(nFb(Vo`;%@P)V zP5r7D6OIv3J}CdSg(XA`(qzHx56L?B^n15>h>+9?d6*k)y1(Wq9C%bW*ri=VcJ~gO;$p-p< zoOPZ+M4o}{uZ#~V#XiJSj-5=O9!}rUNhMN)Ph!k{0uzqXioR-iY7Vt@<#Hek#vki{ zY;gR!Ky<8Pawlm(g3H^&G@DRSF~b>HNYT|JGfS0z)R%;SIP zYHIdc7EdGv|0&7zM}_@Ki}!2z<`T5N;Jr3B7zWS>iPkLmsmO)x`@Kl7)`xNoXb{ol zNq8yKAk&GwNZ;jk9sbRGnCCd$1_?D;UL769aVWgG2F|0T%*I$>MYVX&y8<}__LC_L zDsKAA=yMR~1-gf--5r1Ag+VuM_m-^!>$9sA&7Pk*d1Ip5r_F;RF8&>=&1n(q--D)G z0)Ncdb|j|hS{w8|l&ncZIU6nV{bICRq-_gk^qn(UtTqQF`&hltVZq#<3=RGUJ8CtX z;@cnXnrOK+wYRjMG4q=7;>f>5W+VDRl(IO}Vj%M8AkWv*q$x~l^*@+5n8R9^Bz|;&h{m%6lq_P|&M2~YJG~+ssRd+5+mLt2Ew_(okL4jSR47Nw33kwd=?AIOlNJ4uo zu?p*)cTNQa^~jd|XXxBtBfJdfW9(<;JI)SmqT)0S@ebav@wM^(Vt@U8$u z7>fy&M>3t;9!3X5*knmC#ckH&hs5iSNS9LsFFbj;4cSh6%&(^MZx5#MgjM3|dR`~;HF=3bo*0_fyoua&$9yl7zXlt{?>(2i3ktK0wN0XT*()c|!o=UzkNsWzPby6|& zuRE!u`RC?~T&w?POvb@jKNkm{X%pVS)KYlNjwKhTy1N>6SZe;`w4}|h>$&$LBxWuq z)8B<*#7r_nzmWEX_PEfDV5s2kgMu@xkqIV8)7hObj|esppB~yu<&d8H!T4VH;%MP} z@Y)ud!+(<}qqL7F#I;x5Sz2nfwAG~+3kCllmj=Bp2m6UV4rj~^fRqf%Y42Jp|m_e1RN&uK6` zEq7go`FPAa>~#!3GP+kRvFai6B8R=^I# zpi|Ejeb=Y_G_~l_V&N})_J*)Q+ob!)C3ni^up34-cVli33=LY-+n07PV;>+g02*ax z^E*iVN&=mw z%TlbGFE=x7EW$_bHog~Jrm)C^`g}?;mTr{QBvKxIm_GItUvH{!GgycsVZR|7`425F z=5h4r;?4BQ4g7FvqXeu#)Yn~Ox5wV)O?pY$%8Nq7F3^&$cPzi4+r^Cz8YEAwlw?`u z>;djNbV5ANY0X{kHYUdgr*R~%0*79FE4GHcJ=W_zt5;-iTPu1?qEF|d!h7+`>-4AZ z!3cL6+2z`!-A`$!^s3Guawwz{S3D)0sH{Ey4pUgNE;bYcbOX4B+jK_-QTgL0{x2BeD8z#e$k3mg`rg z3mfO>cSA@m*`x&gJ3Zpm-=_m{4xcFmF!qn(-aZ>B!Wx@TXJ$OPhx}KfmY+u-X0R;Lsd@5XS;Zb2`NnN0ji7WCd z&Q<1DGG=10bxUc%Y2{>&@aN0E&}WL4Om3#=%~TRqbV+RAd!y=}sa{ub+=bm4|Uu^LWylIfddaAgg|Cumsn1iNJ|rh1oH5v4wt{ zN{{%&ZP#f%N}w6J=HQv%UN5{?sY&yjlBo_d#p-5Q2hIZKtttW`qxECS`23^x`HVsbh8 zeba{bKU`lt>xAu%)qly_=Ko@b4N&kY#1*m+%Yr`S&rIO2*)e!YZgdrNGLzuuMy6GM zZ#&*En>sr}@nIBwh%fq%3VK6*1XL-=< z#^ny;;NnTc*>YCylnIZ~z{DCRfIzV~mzyH8^QcD27kIzj(g0i3lsN6z+@3KUVAQ^YweR0Nc{iK5FFgEZvM^3#4SfXKnsPAF!oxRYY}FX+T<_rpExZj$G9`blGM2M*716oqTC{(WXqVD z24Pg9i=}9qpzi}f6YN3ldU6WhC#Y9TkRDC;xc+CVU>W?;Z-TnZxahcBWujA^<}Lbk z%P8A;r{KAkpuq6{Yv#5mG2x<}3P>OB!igu-Km$`Lfb)rNF`VZAh3*=t$E)OIn2+zr zTR;Dpv&Q26=DPKvkx`(}rwkqd-s#W{+Su2)M~2aB*Sm%5Snb$9Ob>)eV@h9tsAJix z^|a@{%sF1{KPGCBb`UrwA0MeM|4kz3IzNYN((!{gEFOUoFfpnBdKiIDp~o#`bh2CM zldxM?g)33H1Qw%IPE_ zyf{t4)@{&~FfbW=Ie>0Ti19kS2K`XK5IVL2xTOD%RWPcrMo)d02IM#d{o>aEO%7X| zL~`yx!#}VzO*qWD|2<1XGZT=gX)C;K+8QI^;r+6c;X%aZF6HVapM)^MccMHXMNxI= zKC?c^fApcBLwPdyr2^5krV0Vf6H)FrLi|k|LVuAVNXx=oj=RK=Uk|R8$Uoti^9Dm`JssCVXLeqaUHV)N7n#&uvy+KP&1bIq$1o(c*A_tNtB|1%~pqCnPyu$Kd5pT|O z`B$IQ)QhYZN?APMMe|_Cnl|NZA7~}=B5r+Exn~7B!qwdNTrcNzuqpTXNSHv=zhSPF zo9B(ab#g1pe9Yx1-75=-R-GJbw%BEgY5dcnveTL;`7&nvw5OVJ?;oUz54gB=$}Cy= z0Yz}@f6bKM{!m)?F4;hSOkXocKoc^GN6iFMQ4S{D;uY(ve!+9Cq> zy<7Za2U!ajCn?Ylkezd7CKU_wDT25;=Kkiyw5Gc)HS(u;*0{EB{r69K+y?9#;}&TK z<+^#Q)g1%s=OMFgoeraiwI%ICnuB)82#QJ+3JqAWhL!@^AAP&1;<5JbY0 zim+aw?L*rP299Q9x6B2jHIS-urYsF4l2AWtlE2EPX;dUZ@=HFa2Moe%o>=aHW<>8_ z2aDWW-0F8~455xmJl`h2FY1@4mvS%kr<4wm=jF zYiCXro%lZ*y#B%KrSg|>7MQQKgU8LvA^<@F@|GmBTHQ5>ofiaaN0hAtO`KKD~ks}o7{>=e2V?cr=cRGvRYT+;HZ5TG6)dtQATf(C(~5^ zz$SS2YmEq4=9}swVBgQ0ga_V$Jz|V@ zNFp#u5g+MBKu-#$hL@bBC?n4dnhQP)CP@(%P=s;2HR@cD(MN*1CBk^6C|ntcWrkr} zhBH>Sd$}!DK}_7yPd3=XDfK%(>a4R|vTA;#(3DG#ig9~3PD|GbDp90MQmIV77NKqz zsL%}PmLkgQ5q&g?AvoBhG`xBYzX;+GIMReKviH*kvv%rJ`1~k{pBuotzTlqA1yRc6 znFZ^&_g+LNOoi(j??;J_^g$o$G3CBwV5E+Z=i0t&l@S4o7Sy`fJ+(dZGSw-~Y+rKc zVLYgeGk!dyZC3I6DI=UJvX!^m1klgG+7s6dr&Ja5pSt^4SYJ4Qz+Wj#H7IY31J%4Uc1iTvnTdcP(4fVa0zbfpj(;x;m6wc%U({Xb)2(8 z3gIu^r%uwtdZg`(_l<`RE`MZopG7IIwwmvnFqrQ`#jkpTOO1_`-;Wy0OHl*tk&p11 zs{UbwT>ml)8K70qmB&&(SKp+2=?NIR-0~2B$T!xtB>K!&2xjh7jidcGBOqf6BMU5h zP8OHHupV2vtfg`{Z#-A-OQ2NiTGbs$7 zC@9G($2R07xhq=a{Emrn|Y zQj8Td7EIgdtjaXk7;B8*_iCyIb7jdrVb$_VqsPuN0`;uGIPP{Zl9x0(FP>E?+>Rio z%Y8Xrr#w+Pz!un;cbs?_4=ZzLcXK$$Ll>%uq1Hxhl%z;pK-4NuZ$*EvQ8#ilvbm|C z#E?{W7<>`g-$JSgY%-eP0H;l*nv)XEveFl2j+{4ey@2i_F!(Ht-s?*)+gc9bAr%|A zZhQ|$3ch68ESZET+bzs7TG4R>-^Q{wqZ{}~Md9WY?s6jVB~McPa2vLvD)1!jhlisc z!s~^_u}gb-=5FdTx(SEjs4hOQdtpNN}8&=IE`ww%+?`xJ_f&kdUrkz zS?OFW6Y{#fT-`Q*1t1H_RAQDF<^H-B`p-3`%Pxg?@`&fatVyLF{>p*+j3a zRJVl$t@n3^nd%Sj>M~epDQ#TO;9Q~HP>0Hc$(GyXYB45qzTDa&;JrNVDTh~fmwCM2 zzYogv#13)uAHw^pRau2xb;#7Y54)7|hO5S>FP`i@dB^0rV(2^O+|(Ar6Ev{K6!EP$ zOZwqdKxv|K^$9uK%7@7?aX?%esUis-ycemG`%&A7wF1l4(>>)xb5wId(PVC|%4hV$ z!t(u)((dj$$}kpBV2AU1ZI)c)!rv!A$uX8<6VVrRu(F;gC5u=Ewg)8%0$U}P9k_N? zs`?9Pr$E6>$3)b(^AMgtg``uzg`|)EevEg-2`eMPx&8Oie52rr7jdFnFfn?d)#YT~Vn^gb6e~eH^61FR98!7G`Rw=% zTA|3es4`&>VnNZ*y#jOuz;Dj)BR^dFuyJ~0?bsr&l{e0!0p+B|8iR>!l|yM}Lu$5a z*17jrp~yTpkyC55^#v(z^qG2iv4kMMH9eWwh4X>ets7%PeKuHc8E=Q-#0AyWyPt{} z?kGFq3a>q(XM{G~N9}wwza2IsURKPmUl}S(do-zM9ye~!6UY9ZYjn!RT(<7}`D9{zCaqu{!KVRwN>iZtm@9|gQ!rwT6~ zl2Q?1c6@Gxw=b@>%G4`|2u{jS4L@aM00xQ@AyR;d6OBCq)=JX!wa(1 zPlx4TMJXINU=C+Yl3VWb=j+byFXuxA%>N zfM0y4_wpHFZAlVs96dAuq zo9?t_eP|Wm7gb84nYobsT2IY!uu`cZx3@gHeefg+OSj=Z`8PL|sGV&e&UmqDd~XR7 z@`{LWjOAqcZHR=-0CB=%)?(1@0 zwV+S~X#VPYT|@Y0Lg90)J)HPDDgnoCoN=iCp@HUvrUgqsGBLuS8A-oOc2d=g@na?{ z93xajzbCt#4HgJIr(JkL;d{kqhx(xvcQBc^41zBe7d=u4@;#sKF(=# zbq??zXX%Tu29fMu*?m-~$Y)E_$oyy+m2+L0UO1Z1W(;^$cxCkM`5*EhN&P~Xc8Rnf zGT;fz@%seteL*R>DL8adxw%9Xt~Wv+{*;bwr+c!mQ{Rc3RzGd*7pK`#Y##q`P>pf& zJj_L+s>~d|DK$|xmi10jaq)HsHcWT3$%x9NcCde_bQp#rTf-jnXin%Ynd64|t58gDF8Z%Eb`oI(Kz7xoZ3=|4T;M1b(qREkoT|LA1&m#Z@tg+U~4 zF61QomJ-V{ww-v_o1vSiWm*@7nCIwK^LdD*MZAkS5NNgVkfh>ciIw^=Hy_es95jC> zzPG9u5XvN_AbN(|?257I_@w+uC{jeDa$g-@$5KG8-+l8}U|MCpUtRv7mFLM{)ccQA z9A8)qO%#3-8zOPi?X4+9Xry7EY4HG%ALY533;jNVLk(D9p@35N12uvJPa4nd^pj5= z#v(|*?yaK(CGvAETuhXYDeMmI`7x#Mp}6@jjR0d^H`-n>Z93HOnt@41t?Sd~H=yH{~b4(X5#|A#t?jm65qTjD>~wm1|wx z>Fj1U&n)$%CPc80=jkyd^Py=tW*E1jX9YGD`#M;rgJ7)vy)_j3CyA=DeGhNn{#T07 zO=HaxkI}Q;jSF$|cKk8nU_AiQzGM=jt5g`@2?3ea_W~luk_{|oZtgi`cOZwe0a};V z76Rjy;#6iJK6_cVQ-0*owg`NJ@a)oICvT7+;$(!=o;|&6_Gt*Z0HuZ%A2+pCb33cc z!Cb!0{OaePNKj7E?`4!cUq$sn?Zj)s z2edlRd_11S;_Ut}&iQ|vJ|vwtmb)KdZEhbQV<@#);r~y1h(Gg7|0hA?KLlz1&R+cw ze*6#n3QPU(SW-Vf;JSr>%|!gWB2F`0IVxmk7uE7=7K~@~ue+G9kf|w;7I&|j7W?~; z$K(6UNt0k#cUmL!BHL!`ov0y)M>egH#`-LGXa8Yz|^_G-A5L9on2xlK4(5+`Z@oF;_qyUe`rk8zr+Z*5I*PAj|SkTSghzykrxnH*eS%JY_Jvkcs()OJJ= zP*AwQC&|VoMx5_&O6Uo!Tdf*)m2JvAqR5o91of6Gy1Zb^G1LE8qugP~I#o|6L_(hl zXk=^xx_KHtNQv>FewnP+Pt*kC>TNt@Ds?AhE}-WCg;$*9;jfI*0~*XGuV`uSKfna? z#4I_nJhtEur${$sslU3S{)al(F=hJZz^twQPvcrD-?|n-f5?o<7(WtGo>JFOU;-aZ zk$HU?infQ8OuYnjja}|fbikWsm$%vtZ6mseY z4Wbc4v4)_)l=CWptZ|5=au;v&Mm5RhuraGE1(WE0*7a410Pu@o+Qn2+xq(6|S&s|UvQmMp!JRChGq zvzeR_Uo6SY*k`Y(y4m_=9bc&U$o9h~e+*-i{z467YQy;07FIkO=`ZRTVANuKP+bgvB9-)|oKaDAh<8FQ}U%`ZL zZ(GQ7^Y~uXNi7G^#8>kk*d-hLg6n;&8gUsR_@H%b2uBzb`uTqp_}8`Eciz>Rjs{Lm zJ`*}M(#4=gwnB+60Xn$3L6C127@>M>Q~QYA%Fh-wi?q`5>WhG9?`%X?^lhnYbcwxA zI(qY_4@Vm!czY3R17B!mxlG--posfnBVL*r{rg1mcg4LLO>27mD|A}eVX6?oQ^9|* z$zJgRgAPOYy*fB>Hfy_ApGi8?%!hY|$0T6Wz)vlg(+=xs=_DT8w*1x|y1RW5=qboI z1(6$C7S5&ORfu><}0JLF$`#=u>pEy8`4! zU>h%_TqFSCIVB!={e0H&#(jx=-c)I$2l$Y!kY0jqR~I>eO_o7|&BsKMs*bgR$))55 z8A2qtGo=}lg&Wx#L2tb{b+5s<#%%>g>upLNHXEpz9AWPoV9=&u`+11u_ppc>NV=gl zmG(q#AZ21#UxI}0sVV*MA2(_rv%FokUlscX#{gEYW;A|ME4$Ft5hbXY1>;g#KWp-a zv%^+V;-DKb{2?Hb5*LNbI6qkiXAFF}^^{Wf==4)S4u40wuM`(h^xI$A?P{ljC{r;< zb&|&Fs4P=3Aq<#-K3Y;7+}G6J=q1ntWPvap{}hz%p3NNO_q^Wj7Sl^Qy4V1YWH)%; zRAhge{DQ4fHo}cxPHMXi@s7UPY%Rk{+J{-V)OOxI)#AsocVNu!yoSK({_@+cFr~&? zDKU5qe#WsM?ac9pV(M(44Mvb_OHw+E)b91L0vqi=8dnG@HZSOXF=aDihqt1esupKA z-{K7Ti{!W!q;xMPcC-l`#9zHw1UY+bl_zTPIE>;xpfumL(L{#(5%4`UR16T5wUy=| zl?HZH%rVW3vO4Igf~;cNR_<$UI?$Z7NmAPc*+A1bGqfe`boMa@=pnuerFIi%w9p6=?3o5mnyl(+$m-> z53?{<(qLe2e9sMU7#ffWi8t1dG$hFPTcr=JzDFwd&A#>pWsL=)C+eLTKDx$NA1-LL za^7h+(YB?3s82g-GcP?+u_=8kkde3&7UegQ!vBSF+S^0v#j7*DsTH9w{N6tkgT-ip zWk%=L4zTysk*)nZ1;;V2NYvW zt#=NGW5FIgyx8(ZD*4VZ8ET5PAv~ew3+`|iBK6PnLZsTCo5#FtG$9~JM!~Q#BmO&Ew>Xk&3w^hIRC+QLUc&}z9spWp0mPvG#Dw1ql3~q4e z@iE%^&`POxozX8v3!Esr%G0BynyTy6Ui`G})si*Xe5a*C5^`jXt(++OG&^1*+`mq& zUFwb87I*j_G5QSKqrp;PSx*onBPiw$v3$fEKq^o;XmzRD(1|p!F~Jy`Obo6IwqG*t zL!=G8&9M!q>K_FpkFY-^k0!6Ab5nXC(>ERmJ&&I$PmGdPT%7H|A{i4djz%hk@3H9+ zl$Y9%-ipM?{rN!?o@irbuCc)(*cQg$f`|r6mw7GKh}po>-EmEyxxrOXEvxF!P3W6b z9-t05OeAl#>v$OvwJ&nC=Oe3mYLSbx>^`Qp|DvDDE_xQ&I# zlWgeAz^j;f4{ze2w03QC-jFM!7DBPd(=(*X3*c6#*TMd!%amP%RBL%SR2yFzLJ%c(2agiNv!}N2zS05Q zU$`7hH*qA=kW>~ge{&P@R@MGgOBmhQscI7)i}}|*4h-FLjMQ_ob44DtKZF0F$6v=5 zM_JLy@)UpDoAWQ5XYkD{?)@@VCk6_goICF$!X4Yj*hTapEvJRumX^pw_sqM_ufjN% zz{=C4zejr|M z`B@n81}(WtLb(h!mv0|gol<`KKK``4IRT7`_xiLHN85{Cz~XZZ>u_fgQcCe>YYP6A zY{h^A{>|#aA5~itY-PV3+iFex%c^R~E9?U-y$Qxq+}`8j?;idsI{eZw;&91MB>+kp zjUJoBlJvn@4wu@ph63}N%)xaawMDN|;mW!VlbTO}@Yo@MG|&|Mb34mI!{Jph4P4FM zIYErC1|J?`R(sr@R?IT2xqA<%HhiKoQ?_kRu@&fX70%cT0SS-q(fnqGD$iIQ0%P2Z z)Z-7cYN5LP!lm!i07CPFvvfuj51SHhn-vxWb?cCtF1Cz1j36I}RmAQ)9Z7!R=leHOMRzZ7r=o^P`(0g!%RyX+*VA7ISH?;!hC8=VD4p8nxT~=^8AVz43Slz=9S5EvR+%_v=0h&)36o8u z@xaG%8lm>1^yG)LOrp?i*Xw8L2OkC13WxTKU0t!vAX1*Gx4N;FydBCeu|%^+bV;QP zPWt&jZE-qYa(~b|PJ!VZijyDXj&TR;DcaMu@qx}oHfMgY^pWP;!UW;jWy4NO#u#12 zGH@QCVFC*qvLv5AWGn<0VtR&MftK^ghHClqv#zCIZHtM1V!VYZ;(TObwr(7NN>-#$ zoo`y_B8l^jFm>2ef88$%8B$rUFTF29E&|o$^L9lGfG2wPLGv`|1yn4ZD#~xN5PWm7{ z11rNPZeE6v-=}!}l5r=dg$=;S=H80l4r{YMe={3A59&BIcUdqOa{Mp>+~X8#VNAd! z5GTn|;dF;W=aUjAgFEW8xBjmK@sJsFa_xtVPRdgr+47H0oIx7-{?OE9vUWgcL}R@+ z8UDevuu{Y=&3+gol%)~fMZqzmAE`TA@jjE9SkboRFGE#1e-;x#Lmmjae==rix-~16?M2G_;)!tZ$a%J@{r9@MQ;_=T+R-)&A2ys~3>=Lt7++{FH=5eu zuN@oP%{F84Lb4c(8{3(;>7}o60`kC>6eFWKnYn}7$BD0r?3YV- z7cY5Tz}{KAC~8FC>{Ngp{>$2zW6Vvn)#GJD^&b3JUZGRiwq=dEY9*SEd0wd$4clMx zyhhF_tl9u(raT83j*LY#^1~rhS0PJx%M%fYl&{VwumP%r<`rt4BNFx()4^O!lk0e7 zY8S`PLMh?bQqAgl^}`7fci}p9|DSmsVY)>z6p})bL~CCCE2*x`gPG|JU4A*M%262 z+5T%be{0{`Sz=hT2})sL_}(n~pgB*^ouB!uEl-?!M{egGf4%vT2~p9Yjvt!Y&`D^n z4!J@>0aqn<7D9HmskSDJQ8nQHBfwY;~z1)|^fL;OAaf&N%*jk53Cx)#GI zT>9js@WQfavhkPk+IH2JvMt$&Zp-g>+~qWOL;5vw?t%T^k`T)$9d-&CG%Rto+qz?K zm4*+k^(3 zac93nBh6eSibSYFZ)m{4U`9*zn*#9|EB$;zL_e zZVrEr=MPTggATp0_ElDM9m<9AHI;9B+q6qQ!xZiyx(RgOnqBONQR- z(%JSbrL~t&eoq(nHIspL2Gtm?Kf#Sni1L^DIT0&`Sd)nEF69X#tR=3cfjA%#FS5>z zTg8cuZh)ummEBQUtLb6|;48aVa|p9gmx4MOh){BrY=rDD>>=EZ@PNPwjfmD;mY1ow zpG(k&_CAI)XT)LLRS&&KIX?EbUlfC4_1LXLMjcHrd?nyJ5w6?*EGZM4OGPzqTl9#( zhWN&+N3`KN@lILREeDKe3M()+JQH$dTYY=hHWM=iH#oefG&VY|y%+*u{*qk36jNIh zK1WB##MvLV8r5$>?F%BRzt&Wyu>l+Em}IfOW4aI>=0JhL8uE-{t&I;(V?SOHi7c=)(Ez%aGji;rm#I3s7%cIUVb_c}-6 zZ+!!hwM%+Hns0N++3!6q?or;iqnE{TiIBP3te*=?&d@wH^c=iW4_&En1&;v_LX zp#5&$_9>!O<_O4iSUvIJVf--MV&f0#bq>{;4jl#(y%bDAXAHgiI~)M$_+44zxM_@- z7GH1z=es&xBY9MJ{PYwlb*5lhYbAxju^gfzl7JdIszaI;xat)OZ~pHQ1NsIw^nGin zmS&FUf9M@QvNH7J{5x({%ZT5ZtM~OeKjLMt81Va#76_1VAKCMNTUseD#mVKI?y0O# z`&V>cDVHnFIcW)eLF82oYh}?y2OA0N)mi)xm5crU`hvsHwo|G>yK`E}s|`}rOxurg z6}67WnKmWv*dl+ZY)wAbt?$%5%4|TVmG-|l0s>iOQ|mAD%s|VAAtGH}cw1O@d6#W2 zwe4pNVF^}RZb}Ry<`(V7T>I;%yijksIbB`bIr?c&_|Zrdz+#e^MJ#)D1?wLTXGwbU zY;R;OxDFpPT1=TJdCOOHw@6*X5N5BufU&4J#nH{pNpanAAJFGGW(fQb!v95 zb}-a^g}S|OX{aTEQdHIH@^U|Z?r`;Br{DORvP%IgW;4Nf`S(!CF|DdeuJT0(Zc|jz zaT)oI1tSdRP?*dG!}JO5XG5(gPPUd{bF()F&rjKua`IbyQsSsGH?ZHn4hGj> zoTh%Gzf^3y>(~aQdgbQVnr#&<-V__!v99?HlRyj&&vnr-fqhhrQWJZHk9!Ae%PpQp z)FvZNgnR7U3OAa0&uu0i&;K+>SO7_x{c5wGZ0c4*)aIlrKyIC;lkPAe{FjeWH@G{= zHV(+-)Yff3YNx3`NjHj|`_Zh!*(@d&?=rKh2jqE83p5_KuNBm<^_^|*_`V_)Q>nK2 zGQa4q@=L#KfRbj7dF`q5M9D%PNiMiS*nv;ha7rSEm0$T}(S!I}m^9;(7&$p#OiX>) zl^P~7sy&g2-?!vgbY{_?d-5u=jq1eV09$0X=XBi#EDtPWWOk07lZE4Jv z-5omX*j&`N+l-MC`mMb@{eC+}?TF66NMGs6!t?UH!LniVl8sk(td*>d@5Pk^ zP@lui>86Dn4@N7WB=Q6KL@S(P!j&DFFfEfMWrP&c{nI>6O$$l)bJ^`-u}(ncer2V1z`viDZeuSk@8O|uvpx3En**(n2Fp6zgWqol%_QJ2H5;xYRSgJpD~nU|s={H?t#wsA zb1HFjULTUvWw%U=>&q@<$`?y|a}N(o6)$wg-iF@V?0%FvrwJuySV8US=#HkguS(I^ zvpLF*Ti%NonOU5hpPeere9=0o-5 ztmOqv3J;i1AlfwMHIS&3qQO5drYNUuy!q-A+0tl?PM$;$%b5)`GL6LIL_RU48}0Zi zS95A4Sh{Y$lSC8Bx9Q}+sX&>yQ3P~aP^C9-Yt_}cwXrMhxx6EdU+HdEKK8$D++PN_ zKhRy-w0?eaOfKnjx<8jpTn{2`3$7k0r!n2UTlkV{ee&a@E!o-bikd=4{_W)54!CoD z@3O%-V_iM6YtL+9@?D`NNO7?h^0nJOH)&)}!HO{uSga|}&yPoDs|EMPYmeLIAMkQfGQZvvrNA`j`=-8#c^*xt@hihU5dTEn@v-C)ded^2SBE80|hl zKG9E9%tN7b^_`Y*sFDQ422y=_`aX+KFSg_aZdBzm0GE&AN zlkZcrE2By$Ek)hR&`-r@!zFvp(nFCoQV2`k$zNA96Q0E5hm5mRG+~iFKpNbdx6Wk$ z2*{X57S?f}K}`lEI(4VN7h?VRh9;yadii!&sj9R>j{0+? zznCITo5woNc#Qv3^{?G|`>nQ_GpYgIVq1IjFKHrKJHv#gIx^}bWV~K5>OfQ*AmMC_ z&9DN`>(s_|bmeFQnmDB8&q?dOXcd9y>(0n`xm)R^MR7;ljs~4@3fuGo zMbww-qv-D9U>K?SCG2s}OJm~~ioH5E2cm5C2+}BRjxqVZeF(B5f3sm$ze~_|3OV(# z1Gw%*7(ofuZ4I{X$(oGJ0~G}=4mXBWu3FT=JY=+MkM*S7x<#4x$h(#WL(mcM_n@o< zd`e=v8(%5=zD={Y9h5rYS3aA~!TztqPN$b-eb!c$`zDE$0>VMw&lHz=Xu>*mh58PZ z>hD?oryx`MxsL1o)2Y-xSsUh57;wvvRj1f62RghRk&wi;q?&rb?GpEww*qK@#jqC0 zIV>;p*nrUge*f`*_z{?(-#7C#b|3G|$<}+HQ&JdMKlrDdE5+4FCRrg%bk0>k#ceXD z8)kU_bx-V%^p5|fTIGKkzXkC8XQx-EALu;JpWqV8%sPE^Ooe6#PWYzH5J?F6*46ve z)m^J3fSvIvR$=(@K8x9alheZ02LRtXM|!yC}W2jRo?J@;jK-rHIxPWu|S9sg&+uJ;us?_Vc&Jt%1` z4lpnd1az(~1o`9rrnZ^@RGP z+p}PoXw26%>%p!ijg<~8YvX40&GWzj%lpgAy_&r}J-e}*h8!xLot&3jv*U$IS_Ac> z_W2q`?9*%uU()>`5SuK`d)|Sq!x$@?h2J(NhN)>(p`AF#PFHxG=`Z06!*RB0z|$5Z z%@~dFe%z~#@m!RPsV>%ju~bmcoMIdh(ojZF>ldi(&KR~UBo}c=o$z`-P@#0@-zueP z2|j`EkWPI^JgwTz7&>||fzqbcm2{n}>KoPUceI1Mt{+At9IqX6A;WKhfKyCByO^gl zRFb)LZ+!3_gVouTYeAc+eia+Zdm!cX33EEpT=-CDmQgZm+A?h=)+KQ+Z7Kc1ppb|~ z3N^O36Uixyx!0#kV_q>xz$S~GS5f1YmoKPs(AhG(8w(dpRilrclTXzLbx3GEnkV3Y z0}i2lL0{5F5CZq_aN1zKYX!RUyVj5+iw~x90w0Jz@AtvYtHQl=`YYaFt(HHv0#6TI z+|O88*S*Ry{aIuqHu;I_H#K?+opRcsv7RZ%W@LSPJfB8;u#}PfY3=0&#z_$jtH>=+ zOoG~6*D34|g8QKc;!suR44m(G0 zW|^&3GQ2m@a!sJNvRci8MAQZ_(>exX;}jj?@ZD19<2zDP2M_d!7XI7;q$-(}!S& z3y4n{@Yj zmy&zsS;YoJ_qv)$6etXwYlkaYQ}qCq`gRv+3#ZgX_#+Koo_7ZZMD-Dk%g)72d#@(c~$iKH9VR* zA1f+b`*(LkHUX_9tNXeA>b0_hUC#1dE%)=Q#$Of;kEx|c-ryXV`nSAPIDFgJReh3G3)=t z01&pwwRqVC`@{?sK|Wiv*zZoh_9XD|f$ZT>U4X@w&!>t8v-esTUZxE|bgQ{JDl}wn zhX^=)cC=0MlfwAnqge*R$3+h?{f5{etQ0kdNmnpwQ|cgh6*OdpUdivvm25_su8w?4tU}4FHD)bhn!k-cd;rpmc;R7;$)KU#sT4QK7#HwDaUuQfP7JO2WpMd@{B>f)nL1f-HUb{A=*Z3jEj!-q+QFzYV+a;s(#(dyRY z;H2^DhyttDo(=1>bMb@KSEzCfC9?6ylulogVMwD*|I%loJafBwxfD!sJ3yq69%*3) zY58NxT*nXbc@nsnD*mV%Ax39cAZ~}aZ#EhAF^eh@krxxZt2zg`l7SjD&plPFz^Ld7 zwFn>0YMyA<~V_vT8>J-QFz#_FG zH;|xt&-zl0ugg}6C%F(yFvx*$!_oeyEAQG+&jr`Q1wlR%T0=wvcer4~i6}pWt2?>J zX5>M9JW)9Qtow(T(j%wlwiQa>sKbY+7cchaUodbm|Be|YHb^MwI7CZzTfSk^-F{V< zMSc0@*Z)J^dxk~1WbMBa1Q8Sg1(7IGi2~XL2_gz2K~W?J$wGG%1<42~L6U$Zn;@c~ z7l zx`tZ5M0q+383-kUeh$xpRtof%Vx({I$eq$|kem8xU{DaSPY6vGvZCW1T!SYB#iYWP zonTY1tMzUPjNXn5g)wF|^xef~olNK4>tCl!m|%m6P~Q0$*s{^v!41=-n7iX_$l}fN zttO<{bFm|}8ffez0p9g$aU^ZA!(3^p7^|Ceb^ zfczk~yJXAGZ7%yi5biU;t6HHWu#>adH5}b>u2VUGFx%gETkR4g23a6Gh>r}n^f2@alJ1;E})vG=3 z3;u+()|avB1IklCVOv)-Y!pY(2Td!GOwH7;XS7r(6wKXpTJWz&zIFuH^I8(=_&i6c zr*SJ%kLQ3=2;(8MV!;EoGlvzdo0^<=3UxbEf@hglfXn4<$PLL46@GcqDdgpM)}9rZ zeine^&hX?+S$6U=)Y2`PY{wD{oe>;?A>h_U&c*to8U=QSnJZ&o4Wgeg=9=>{h>Fgw zgYwZqvRHOT**E^O7fp}t=(mc{a(yfz(>I%7=!HN>1lc!q+-r>?vT!^#U2UeJ&stJJ zduG^Oh<)iDV4a#{`E~dGfY3sQSPwxq(-1^)4iRW?WhXxWN)Paf=YNrI+J8TMU(EZt zT}%(g^+@=t_7@t>2V%{)aI>b=dk^yzxj#PEV0bEa8`5!=ZqZHv?m8YU+#)1q=QF>! zzQ{k!2T-lXyK|Mup~4G6hU_;U(BsCu1)5x9PDMhnw?@7|tBHdaaG2|+lx{C#PKV$0 z^dqMLxAWUcGaIiRsF-VD(cHvaC-CK9afm#)kVm$8C<$nV-ugLW(F2IibBE>k; z?YKS_CW77c9>a^-7k!3oB*qIbBo%IN#|4TJrb9%EtvzIn`8;oafQW1>Yh=4Hdl`Gs zCeg+>YoHI)9hXg9k>6%#mG|86?LN8D0n}_YU#jlET{koAh*bL4ef;wAc7~l3?C@N6 zxJ_XDkM#^t(O;oYpso?o*Az7%exn|Piaft;+;hK_ zXWuLRS!kJ93gEzR;(7UtLIxTBEPAg*=O?`t2bRq{O(>t%uL1$D>Q9Bf%d<6o_Z3JC zQg2NClsCR69b=Df9Ip!zxt_Yy92LzJ($}{O>O7H0!WT^x#$8~V(lHkr@ATqXPSXmL zN9{nctc8ek8eJLhF`GG*PLH2kh zESvNDTOonCkvx+W{es>*!9=^)Xc#c?Z9Gh1kW7HMedpNrH0k|`mxT4Dmh!0}#Y33m zWbKv}Kiq=FcJxs-5Sg69p@pTQ&uZ}Y78{&`hUS=RPN9kE>{P1#^tIgh7Rri71u7%@ zt+|K_huvb?LIc{&1+3}r==;^z@e6FB*+XAj#RuAAehfr{+c!-ew(Cn(FuG4IN$&PC zm_%Z~Z5v7z8T*9}&}kC?ZF!ie(-ifJvtsFm34Ga9$gM9&XIuI+9bG6(g0fd<0Jt}T z!3m*13hQA(TCVhp1sYQ?CY)HnK%aBJGe5D)v7Q3Dw{W8zG0aGvM4>J~too*+dUS+3wqtZ_@eRYSu- z12!$63dsCH1?y&Q_3LXTZ^TH|i9X?PQ&z%NK;HbQVXj*Gg zK$CPSv>J?;@eIf{<%(JO%)_oh7Wau{!zd|0_;H%i=Y{t_-ZG5QK-r=!Jgs=Aip~6Q zHaXUB5-)=*~*CPMwMf4Y2B4AWR#-#m+U*RWUx%hV{#eev- zoBO3HusOn39qsNaU1p^DaWL6=yt9~7>*aN{x3=9meDt+ttQ=nrlp1bTY6V_}G}552 zfHd;>crhu-%iQrJ9+uoBvW9`I1Ln!vvHQ_hQ4A=l;c(%Nlb#3{<|S;GbQN1?r|tj+ zxD1sHn*56HTn>0gW{ z@qU#h?Hp_nzqJ^?Y5cF@@$8JHxy2?vyqY~fxy+6Uv1<KB?UK3@HrsDZ63bu+lkSi{}V{|BfQ~|S`Vpg&G_)? z!yAi1uuoPQ^J~f}Y!lzxj1$$B#&4qe@;=Ch_q%ggLI7y}>31*%BIGT(>uuQ~oxfi~ zG@`eq(eQY0nrzPUq_X=s7=5}1%e5z!xLaR;l$*flz&;{zwn*^SW zw;Rkiuti={K%Ses;I>-wA!nb&MQbT9j+^~29-;W>XoJajPmv24aT~t3>%`NDULA11 zoI@V;(b2j&%i`eDRaJO^vS z(&Z5%-0%&o>O!(df_E8gs`;PauLz>69yf4Mqq#V7x6o<+@_8n0Mo=PT$T{e)(MsC~ z@&;~tb-t#E2SFyYcEG>_=k~q%MN1t@IQAoOUK3jX1atpnHT}ng;z3rr+y1bet6?qw zdQDxU#e?!oGJ|y+Y-gIrE|Ur9vgC-~zay(eazk&N*FH-kvhD7ctx7yu*ZE^*wJ_+^ zjXPrk8SetkpA};lNF$!q_o{WROO@KGX+p_bP@RHnFNip zLGCpGOg8_!dlnlq3RaNOQJ}H9dhB!41Es(@27NW5?@QZWB&VUA-%IbhXi?bcscqzG zwaRD%dL%l0K>5LS`UWZJABJonayQGBcVDaCD{^n($Q*E-=y#z*cxPt1Lj@5xUG&Zm zw9zgy) z{>A;kwz=r@8|?l&2E~XNc3}R^pvkuQ&aNoow~L05aksP=F7k8gc8V zd6V`}^Sv*j`2YOn&%HOHe6b^JifcTkkGcMho0}qHmPR7)5BQX9+&`GW{uNl2`UtEH z60+>c99E#6xPFY4k@4CckdZm=F=M}16(b{kJlYF<<1W+6?&ioC_t*MSiVne$w1~p9Kh;86c0}kurZ{lqpn(DGxQ;0UyfV2EW7T8+nc}-I{}+c1!)>08 zG!@2oCG8aT-kYQ4TYrpdvLKX=d&$CrFZ3mLytvezb_x>Nx0z7gM(d8=q_8*jK!72o zjRv=?C)G)IpaU*D6c#GHW62KXEAzy>Lw*x+)jqV7&MrN2emo@2lhxjnWcf_H+Y_A> zBr+gj%ZEPH8JtSG#`BfW%r_PU zq1?;Nblv8cTIreKj1?>ePd$bBs!+_0y_5Mr&|`MPv-J-$&8Rx@TGh1C;`ZR2;rqt* z-edmz!6{-B7Fm}WJes7j0omN+>C<5b2;Z44wK#yllB-)4)Bg_b$9S6vfulX;*thlC z9dXu++}hQWG{Ms5k#a2K%B9CRl4+!V-F3x?avoiV-5DE?Jg4i{2ed@aXUBR!rHeZ1 z3LMg5jO%9@n_H0u^!>NnJU?x{|Eh0Rfq+DA2hL}0hrJYVqgWh`<%7%T8_}Ui%cJ3o zS-P!Dbwd(Yijyi0DQ0T28@(NO`7&|A`n(O5{OV&a8LIt+CaXwsYmt>{J1=kjaG6Vt18lbiz2pw4m zKI94YhVXrq4pYwy0%9B+#hrZ3%G^ril=|Yk*pX%};}+%j_p1dU%S~Xh#Tc85^t&Sb zGq38NfuVxj-pu*>SG6QwrF0vRV}Xxf+Zb2oJ`Xs?k}MlA^G7Fx!Kf{JzN6!64|P{5 zV(a?NGrgn0_TI)BlkVA38GsJWEDaqad~ zz7Jhmr*Z^%iw>jgb%Ov=syFVmng0;!$NDV&m1T{`q0%y3PxEhu;YXBQNDup1TsR1M za%nhYwbDqWuBv?yjawkt@ArV%3v%Gc4bqs^4^mj##pD9F3B{b}XyZv;% z(%oMyBOv+CnkDMTp%W9vcA^AjDZ@lBSBUEWNo9=xit325V>hdgjufUi9=EwBRM^I5 zBKe}V3#^W(@GB2)F6(kL?uTUz{L9K}+b}7l|E}l5@-oLL@d!3>>AvG?;t;@C+ZefOE%3k(Vu=L!PzQJdR)CwD) zThrDGX~d?NUw!Xd-Nf(=cfML3*=>qykbiqs9Jo5%e8yL_3Vy8`R5KVHK6eW z(Z_va6z#4}A_~dT7`VT8l(^l0*>~g3!B*b{iVPe*4RRV^@=jOO&>^ex6kZE`=XC6F zmD5 zeoj2KLz_RKF8|OjY5!LT`p%r00OYQCv(_L8foe|RkfY9zk2u`dZn>It#CEdPQx5pf+1wA-90SCP>lQ>zBRyIz%+5#i76C4rV zOO;kHUF8FZ3-*=3K{{K)9-&x$#c$2kn6oJE1^K*VPDfuPO=IYZZ4zKkRZK+bVR3zX z3MNaMl<4$yYlz`{@gL#l6M3IpScQN?Rb6i+%Jg}OgkU_ zh(qr7nY*a28iocZiu~|rCMY{-LGfE+fjai!LKB>%_1&w;TtAjeZUj9za?JL3jiCpO zw^{M5;Cl$QjR;M4_{+nTE;RGVmt)cxJDp1$7Y-(hWQ67`xaij~(Qz9G{KQQtdn!smI2-C>MDO>}A%N?pO z)aBCr;#48x7T7HVr58+8^;zMk!Ysc{3uzIvYzyX+uyZtJDo-Zq7ij1KvM z-*D|EOA%_pqGH8;i4zC3tKC$mq#CA4%5RKOjyoUybou`Ahe)E0o_M_x@;2?fy(Gi@ zv^MBqmZ!2H?h?%sRBOVOe3ZU0X1+p&~}b4 zN-rEdc}m_T3lZJ2MS{o2Yj4DO@Z9nmx1E99S&B}I7f@|!g*Vdf-(6ok9}=b5d$z8t z<14HVx~v_p0>>-^#?Z~E%U$WM&ll!DeVe;xTt3~Xa`3HG`e3Wp;l*oN2uo*BS3I zG6QyrOci17+fDjtdc#P06y(NXX$|K25`u>Zf?SpNM)TwodL+m9jYTAmd4%60-RvTEjLp-)Ztx8J?x zbAn#)6QLfLD9i3%7#P0YX6$HwGqO!MGZsp~kt?u8KbG@p$IU|>v<|T_wCf~Lk3LQr> zwZbdp-_Vvo`+|&f4Q?0unVk!)Q@VlP&o2pe#jT*>o750Jh}9(%EN^aNG5uP4@I^Bk z=v_TxjQJc?tQ11DXyu8(G4a!acBKwLx&Huas@H?%vPtd_pR!R!CN$l2I4j{sxz0EDx;Iw#HeFl3h>XvT*Zj{$`}6%X z_3O?joSL^Gl+bv21B}?%7S4sv=Ins~C7hT3j1T9hs$df|g>r)waqTb1PIZj`I__Fq z{5|SS!*jDWo0OeL(8)U)0#`##4iS}3Ho#^Exz9s_b1AIWa=)_hPUH^1MyXI!{Iqi9@0|uKekvB7=%wBaI-9a**zE*NtXR7bxg%|+u z%DTmyoj}4F<^e7=N2No(YJQtgMfFJ zI5nL=Dvp0C*1u3w5X<$X{bQaw+kI8X>0XU4vz4mxT-oo85P?{teS0uhw1oJtbXF`9_W3XqmF2)} zc-k}GvcFqOtvCF>ip7;@htFQp4p7$jE*W5SX~rU&V5wz(AZzra16h-T9#d8zOH;AE z-=J`MzTp{pp>S{3;<*h%+Mw^Ix5+DNNr<|g&K0DXxU8Ua+AMEXH?=q~p}OK~3$-uQ zUw*FSSgpyhRe2ZX8)7cK0FnjO!CAG22mVQ5yfHv?+V9FPU1;!`2ij{zXOcMQE4Ebs z^v>8YEvY4gf;WqWgdJ;kMHNOL(f{<7g}RBI66Kvt!$fj}VVM?-4`cTqBtIiH-4i}q z{KT9V2nC<4{9vCbswIn09-f?MKQJueC|7u%2wZ@>DyParnL$cX zfCw%UAii4DX*=*hVNAQu{GFJ^f@K2)V@`QJl@1=LO9Tlw;y7QhC4>4!r9A*LrT3HB zYkC;l4^LOKSaqHTYR<)B&dWhIZQ z8N|nh@D~wJ3B74PH0d2)acC!$$+q^SO`aCM4uyr+qz@)rkJ9FfE+C8*^^gUFosA4U|?69NB(*^44RoN~3q^Szho44Hwd*Dz&^A zOo&)@!}8<1A)!8P=JMlrV4B&Rzdu{SbkE)%IqUJs!x2xY^pEuQ+!^aj!=T+#<%Wq! z?`7x>Oh=zj?Qhzq8Zb#q>C{0c@d$3sgo`~bN^|>k6l^lG)Bi5|NJoMpG$W{Eup+aJ ztNv?vzcTck0@ge|p{h^;4Aw9~qUIUj6XJ_}^!wqZ4dF!%z3`q|2S*)A07lQ`2wG(<5Szx&9sqf+bR`aMHGH#B3r;t>ul&m*#bTd- zcM-4FbKM@5`og38gTa%Ej=>IMz3^l@Qxp)rBb40w2Lpj-@9GOQ!8s^MLo{Pp`gC8( zJeL-5b^UQP`U}c0uQ7L4iu~cRIbWrqADruMzej7L)uOO7Quqq9*-YZ-59xu_p|=4u5a z`8yIuB&gq=_F}A#+c_oM1~3)3+{?vfKR8RY^-;*4iph&p2$lVdi|pzpd_sX6STL`= ztS82vUVR_*~?s{Og)^;Ue($#C&-{97O&b}2Sg*gtqDDPsc{0riaOzm}ark?MG}v(wN3)UXva zJpWQ`=BwERt%8&^O>}HLDc>hTKuLX~B;a+v{)t|9Rwl zI2j{zod02*PN*QG+ZOLue9{$jk{()$1l=xX)*^xPbbc=lCaInBo}v>>#9kWNr4Z8z zdwLxoEE*Q-U2ipKYj*^_rw#UFUl``lEfYV)%yQs39zw^k znRnIX(b^Wz+BeWs*?Avd$i`QZLG}q>+>IM(i=cFoRh<+p!^Gr9CI~Mv)+MtC0Rsx# z@d_I&Z8OAt@h7&r?~yR0L^y}ynn9b7usbA(;LHcrkrOAc#NTyew=>&e>vni!p?@BY zxyVN?H-zYJdcs0f^4t>D$v@9o>?%q&`I1@>NJ<7!`W$+^;=nuNt@dMqh{32496mOb zBp0#o>&NB`I`ziTIpEi}Tixo1E4QJz%`YZC%$cIV$cl_@2GwO@n_9bKy6I5rs+FCy(Sq%&Ny!@(P6e@Xp`R}GdVXb)6 zp!0asAm#EC)1Z&|aFY>l8Wg96Q#LkYANkWX2y|YCbL`%{SH0Szm?w{;0QFhrq#mk& zEYsp^v!_A3qN2fDik5%Z(gm8l{%gTXx%=_a;rc+4jZn?MWd=t~JPUX|9u;XC@<03d z{MWr^fT@We0m%6NV;%T!CE;50TFSf{kdt8GDn2^gg?_BB1%=zIj>mh`(}LPVwT8z} z4!^c9AOC70JYKzxnT{2G z`{!X~8ppvk!KXr>EDisrq{**;tb3GHENHq<2}Bh|annZMQ|fnMYA$}Zb1Q=vH%s%@ z4@R3~rsMAI^$P`}np;T@d31;2i%J99*XBEcg3uXjz`fQx^T#C?Ev8(sk6ULKy)DTi;`X9X6z130~>+xqodu{W!H6b z_z`UjY`6++pqtw=(q6mkGP-eGrv2w|33ak!4$E|Gn6z}62u4=I4yOc}Z}P{YWCNb~ z&s5t5nA`22GRVd_(``}WdR>Y^1}3N^l-g)IdC)^By{G2Uy(0Zm;T63!&)wKWxK33t z$8w#AXOv7)qZ7*X@`79y5fJqycH5y-3WHLm5=5ev=f6Dxv_i>x+*=mjzb9iswh&IO z*JYPE&NFR4|Fq=SDLI6pS0|G&S6*IaUqBaElOd6js^|^%>EdewB%_2aYE@cxIKjE<1|kWANC}~%7Kx_D?^>WS(&TGQ`4!8Qd6tmXqjT09=LEz|V()N{s(YPzR-=p~ zPP<477Lm-uv}XieuQZxoqV1<%AFUz2EQo3#vv&ErU2RGZG7UdP$89PJ4~+;rFIU27 z&V+al?kjCpQHCqAaKI3FM(q){*ZO;1Qhfo+ zh}xxfStU;n?DV(6Nlo9ycVZcPfW{M3HWqB$DiEEgQ7)#y*Z(!;^ZD%-tpmRLJ>cyG z(qZ3@zq8TkCGSAUvhQ)||GZ}`ug&;<&Y4G?9lkL_n3@QHutTm{w)ngE-2N-V7E;#5?n_N+9RH?{tPs;d7( z0t7|Ww5kt=88h@;B^d+k;e*@Kulm~}g}TG1nj93IvK%cvU8aznGFxe3Oi}v>R)Glv zg2uhb5Zjt{m>J?OaG>3%lxL+0v@&kvwWk-~=CxQ$iZ>?ie}>)DPW zZp+P~+nFso+cBmEWkvMqLi}ywQL7s+xW0^ksjIT1S|74*+V=7b znTKQ2?fBCArK0i-;amatWC`I{`_J^&C*$mQGm^*FvFYRZ?CUh#0nssCZ_=zxIM&0?n$UD~ zLluFNL=b`G2#I$gyh~Gv9H)_rE6^2I*Wdi5o5zWK>-AGBK}nITxGCqxa8pRdv&9O0 zI^9L%SEqTQ(?DXf*VQ}BQFw7%?EC}d*Cc|WMc41|%o02&hFs{Ygb#s-&*Jn$^jN;V zX1wCU!0DO((Gp`a{0Rpvl8|{(<*tuReRwYpPci77d58&o91Y>ax~Q-WktnjK2~rEc zO`&bRMZ@~G1GxX{Di&Vy$bX~ZBE|Q#yCr8Pi|6)*Qg4Epb6s~^u?Vy zYq8pezVS}VW_;3uuNvZ-KI-3GnEir$gYQ~~aUE#&D;XqHc*KsBD*BM$`=rB#e0(~8 z;)U)|ZQE*rL+SJ_QbAF9QM(kg>>h{0^=I(*TN5_hT@l(b;cD344tCx=Y@rj;_rYNu z%{WTM^B0_=Vy?S&BOF>oReqxM6%rHvlJsSc!Joo>jvusk)XD@ZpTD!#?YwvQY2x;9 zQ%vSK)qF(EIrP|$l2)2NhiRXM%6)wQH@^}>u>kUo`0 zUn^$C#7wZ}tyg-0w6g6GUyUPcZP)I|0EZ6?%eAJQf2}UHfK9PfX-0@ss_tDnnJifd zsJpGilV#uGvb|zT_`dv9N%;RwZOT{_hgW<~I~;Y}{louPaMI|l^nakydyV1bbOr@o zaYf7n#8n{3nvIlT`>QVNpE}YfKT-jy0c&|4CW; zkb9L}h`fL&-!vL7xhZ-D4sRy1EG+~3SnM5_v% z|679kTZ4R#H)k|gwfRuwJ<>)=(*`?jTaW8MPMHC19pWo+aMX>KFy`YLsQ(ByRYM|~ zo?TrD8&vb6$#uD*6nrOvIa&71>$r6R)k7e(($0<>VQcnh`+er3oma}XEk6GtBUFKt z>?`9USL{(s_q`5f%Ot2A5M=7}AERW1ws$WjcJ;Jcx(f_skBYGQx=a367@mYLC(C}8 zGiRa{gT}~Q`F$twagZR(nE7%|lOCp*wq`gfD&bwS{&7p^*YMq{3pZkGxSIE=k4p3r zmvi?jd&vLx1oFV%r^eppdo2;S*SI^^6ojTJX3PiFEW5_H*zU`?y!;oxE03E2h1GH| zyougOyfv+%Sq{pi(P3SevDfg}FKvha@l2yKMDbTrzx?^(HsezeCyS@ncu(R#BbkSX zqTeRefOd%03!!fxV|b}&(N>_!RPRB84DG9aAU}R8k{@>mC>_cU(mz@IJAhI&7v|=H z78vuL!$+6mM$R^fD%n1Rq~AJK+m%nO$)5oyI+e`};)QYsDqGAvTeE(D)5#FiCbPgx zrbO!jTFE}Cbj;dj4Bv!NTt9^uViEkXnVu(o$)21$=BLNEZH3>G*fD8*d_^u$zW??30zI2SOS?LMn~xyGdf=J{6&R`iJ>#YdO4#Ab_SP|n@2%;T6>dB zNdHkF#o{~#Y=DAnX)PnOGU@q9(Jkhnzl4SOZ$Se?t@prsw6u3hjULp_-yi#C*cs?) zntB(qC`P<8DGb-G%G$H~<3^Lz4qGhrmpkIxplE}o?zI+7Pf1rrlpxFRz21SmEJEv-BcW>+I2TsLbJ|P7PaD5V5t1t{p&o zg_v(v_RbcaNVT`~=s(OuTE(}nRjM;3;LLZPFFV8-q71&h^U@)TbKi31_~$k8w=Eb1 zThyVkT!d2{rDCN=yzfxNF&vGw;#S=z!2Od_ZpXqP^+3a>iT}6*vSh&I;`FP*{7?54 z7d>l;uf>^4Vzi6DT79Up8h>MGkN z6a2N=Qjs}M#`B;l*28yx@^~L`#mXtDUY6TOWQPAsp}+7yxS(RM`uq<9SQ4;;nn=Lc zf1-hvxcB4VM}V!*|G;&4CynfR4aD6Sdb+1lOw;rE4yPIU7KXOd59ea^>SEkm( zZ85YXCp%`zs|)pL4=aHw^}KxB&E8n5Vr^;9b7E@*wQ5?Q*OS9pb#mcgy1Q)P@~VQ! zf6N80k{~r&)We-$)A398SE5!*CIxik%7R>slz3L?5$}w}CRy$jU5J(>TiyCV@Sb*c zX8-$Ppm^9=igFXOy-B-_q6ekt3(yZyLbP;zU={Q`u+xUsw`lMQ90;pT36~qb*EIU! zvNO6SwbK5ff}}+UxV(V|%D1t6+>w4Q&|t;VEknVzgKr z<`a?ovza}<71g7ZM@IH7{2mRpRzwdq{S*JA2U?&Rv*lJt7qbUwzqDGf;uyb7vQMK& zUHp7U(Rt}q-^LeJ8>iC2CY}A-xUXT|o{L1>J(E#|h2Va0J_zEPeV0Zref*2dvF5c% zR~Q;ke7KEcOv@uohtlDDJJ%Rcy4%%=NrN$V7YKtsmGfu3DX= zM;7gx#)s+&sIZR|$lIVU{~<(lfKxn6VhzHGqTi|V2#_1l$Rx;;Qj8m-DR7s(fxaAR ztcH|%|LQA>z7vv|5Em-H*jHcG_DVu{tA3AGs){p5XJazma<*Q9w41Y1XcDfqhUOnUps_ z1FH?*eR3(QB)fH?p)buk%`A8;e=;{5Wtb6KPPW~rYznV97Zm$?$H((}#{>7CEe@f# zs}r#W5Z}!_xBYyPZYp<9J*exR6}pFvzCtT(^PTXOBcOjj_T#4ENP}3lEi$C z7Bp!UEq3(~Y!(Zxp`+0=r%CV4dXq9^Kl#O$Cm}XqI{nv{Ao@i?dq>SfR$1ZHGD{t8 zS3NTICl)!c3KO~_osWJ=RJORGe< z*98d>S>U5NZw`5?J!~<}Ty@1oV5#9@XUrD}`AyIuJ!k~USymB|r+RgHAWy2=5-j-}bVW4_w-d0yf3M4lw z-Gd973jef?c#A8)rMvxDtg-@AWAZtR>8W8DX0+VVX6+P}_Uv+kLVC$S>F2Cu&7`RI znF1psHbgm7vR6*I8O^tO_{7f+6ry11_|l%4)8~P2_L5=&@fu7n45|alYrCU#jAoDC zm*B39J$yf2S#Y{VZf5U>4pM1ts|OO9b-g5SO0zb3Iytl*YwCUr^J=~+BX>>{r4@MV z?4(F@sy2cEuqKqi$B}l(vqFu`o7Q|D{vhMWI({YK5BYna&yaD%vp0XC(R&gmy#wP! zZpaA2sNk6tA5hd95pTMi#U6Eh1>wO6cd^cSHndH`eM3zqGJD4oX3`F(j@9Kinxy%Q zmx6?@z=KM4Si9D{dKRhWv(6q`0P)uB(BM7uINTg+ct zP2c|&mCKt`Zb@T)4hna3dY1ndV^b^dsU&X@BLbmmEf@QhEEzv~n(UB2i0-DD(!6g% zh}^_bz^zD!og2rqPZYD*!(_rq(tXdg0SrSf^FtDV)7D z@UYM$>WBAz&YDYSh&p^|AmNXr;V*9aE5REGUM4uP?%%!Mky+u^;D9^K;C7$B5O2n)qD5^C^25 z%R@N&BZIL3+i?!FR6}ycEyUAasvB~FKm~mu=(wEacv+u@|8{rC4Bv-aX5ZH^?vk#X z+WlV=t+pS41D`0T9LjEWj(k{YV?lmDYsq53{0I*F0y;#KfR@-A`?!%$r9R7i0QWIf z%VL|jiHDgvlCOd#Vw)YqmN-!1tyeO44LL<~Uk_eCMpc!!rn8p6R(%zm7xFVhyhX^h zpUC1|s%LmwqcQ)OxQSgCS^Ce}~n$KYazze7iO-Up#+ z_eodYpLeI<*GB&{5(X|cNlB|RZ^8GPHkRA|-jofV`UsyS#1qsC;!fg+|0kETzp>N` zIf|dK|5btpVuZXS`>%Rp9V3B*&p_++fI{ghV9RIv zE-$mXrgOU!8&lNT_b`cu*iP&N!Swh#|LM;$*zL}w?*+egvUuU~a~qykFJKP*@i0(~ z?;)aQ53n&r#4>?$E-5Kukvi#xs{o*?>&( zPS|7UZIKH#vEAhf0&cIzAq++PJtw6vFVPjB@Hzc0qZyjhdMrJ>r?jLP+BZAmF=nOslrJa zcQV&{dt>-@Ky%>*iv|@qJ|-WvcKI&sMrG$b0imgm86%7fWjurl%AVWzUp5uIgD%qO zX(7_1-zZ`?9yGWABn1Iddqf@xik7}cU$HH7&}EL`8wYr%T)$T8CH`kt%W9KDJSC)xPAAStbBd8qAST^YU~b@xNE z0ot~rP=8NJ-(ARrLWaW+o=UP!UttXN@eHp&Ut6qmx7(!NO%OSqS|vkv;aJk9=4|efUO98F*i}_x3B}(+U^T8l>Rz zTgh;l&mIW`aKzQu$_l*Chv(j=`z+9f<5=EHZY9>ZO4Cj9bvorNv$&Hwts1i#&7paOB}FEGF!2k};0 zzgv|M|B{`*c*aNU;@WG_?R}*xZj;6HkCpmI8|*LF%N(JEaFGFWq2%!0XY+5tSn2{eg$%e29c&ws9!*?>DV!@WzzUQ7D=)ec#42hzgsInx+7K($~ zO({L9F?bBrxN$zcB+UwEUtKDDY?*82lMju#<-cY`Dj$eD@_Q9@zR)2t*)j6oq`C=D z)!HY{DL0oPM2|fsnlE^bEcR1TV5;H)e6OTST1o$qT;Rhypo-M|do($Cm3fMtIbb3G zA7mx{CnWiV=!u}@|7EhK^?N|p1e^9DLvwJ>a`M*m9l+V#Tie<->Ll}tC3GLWkIey~ zOIM-VNUBzj%9|5K_9(Af>e+6d#ypsd~6|bBmV=qcv zSqaqOTSCu{is9Bq8?dzWdY!sNu9Px*m88Z)u((Wdz!v`%Bl9G#Q3vqFm7mV7UN?K; zFZ=^B{xwjD-+KR?z%u7#lh}Ye+1Q;G!j8iJP^m$5`+#&!sc>|=`24i=ihX`?+lShk8Y+usgB-6eS>^B;v~ zM`1f6LBadV4chI|y{P9+uddkTY12K98eTJcsxZh1?CaPbeU4OP?U-bE79H_A0$P9? zULn+7T$$Y@FiV`_CCk>>SNgf0VZu>dTff%+MEu)QB+k65Mw{imJ$IYP4<#qp3RU|x zsFa_B>V^7?=?pHwpk8$CrwNGDMJ9z*O@+Q_C=3E8D6Y?izr@!(2;S3uH4E!23~oTTo&oz?l0{Cp_}|nZx5`C~LG|w@Ul1N2Tf0@(^q? zj(c!;i@s4v?14|l_E^Lmd|;sz^}PihClCZ3Z$qt~@ww~=cu+c?QJh)c?+odLe+Uo{ zs9z9Wvy#-pp-&}*0J2m76d1eKJ2OMny^wVEM_SlHuIJnXqJ)rrf)(4!eu4$Uaht6! z#;{^~dj!($HUXgO^m~x?mG{n$ksdk^thE!+Ujpke%m_h0lz$prUC6=JCDZW!!SP~3 zyW@B$oF~Y{EVSGFS-cGvYq{P$Fk;mk zxwUa?O@gTv9o=)O+3~0R9(RbnK*cmW$D*!kSq)0%g&MRZYo0j``jSmI1=R?N6xO1)I35lqYv^j^gY5C$E>Jjw-0-_)WqzMQ?V?aU=3Q8|ZCqOs?0qG@xgeoGS^iF6ZT?j>b3n0=7M5&=k4TL7W zNKxL#?|$ET-@W(UH^v*|<&Qlw##(Fdx!0a+{^tCxwdPtF=WLKbi6S8Oe~oT$-R$>@ zKIxV?LKX`^N;GPkdAqXs#7rDkJiqgnSyY1%dn32d-+ig>skw1GF2s{(A4I%B4^+h7 zWxe- zi4!_CksadED3g+M?F(`gP4D2}`Y4lKWO9Vm@ic(N4`R^++b~NkjATQ=Nn5jeN$rkJ z@hur)Zkr}=q2(&DuozjJCH3rK>coV&pEJsi?L0Bn*)Fan|cz)R-{2e6unIngA9 z+xHf0_a17NPeq%w{QW)Pu3n$a2pUtzD#%zT|F=Ci!TmP=GVhYOX7%e4Ngix`#L4Lq zOmk}&@yZL|&TPj&7fC`}#8(kX!!6C4xykzBHKreJa~`!gDx%AW+MOmpKMm{XnNAI7 z@$=f4g&QK(4~%<5164*Xh(j*oAl1$NLUr-v>mt{>f}$t?Dg$WWmx14+D#vUPznWDNT>WRZcyQ>ycf&nejzAvEW*j`4JN$F@( zQTWpAZrdy4F+t`xsz%n(i~GR(zGmV|)W}h%*=*y+Q2ROkHb*O$^5s$gQhZ=ml<)o~ zzD(F^u1z*X#&To!h8M>j9kf|{C;BZlrM z0LkZ<>MA|A5Q!L&o~O2Y#X*JS3w!0GgmDL_GoK=xMN)69e^E=m$09;?7CLYN5@gs3 z#s*@S0$5svpq5Xxk;u~K^Hjn2;L(1*4fuQEEowbfLJT548)n!*{42vZNpKHn&lPiO z(fRpnAY&@0i$d;yF`H3aa#BduK1Z=K!C@D1yIuvsqT@M3QW zb_u)_9Y}Zg^;hWUxW0cY`MY{jqa26f$h6!=b{;iWpv~VU|LI0;Q`3j*sHG>ksdP+$ z{?q5%928`x-{+$@>+_?vm$_VRlrQIzn$ou>G!C`F`VuGn_&uUgoRbH2;#O|`n z6hDuqv^g|hw=QAZGmAyT9F5@N#4J^FJ^`8AOQprVEgqZuQQ@rknz%%LMAokl+g9#H zadDLy#P9|2y6`eAblWyHku*0WF=nTvyK-J8L^Bv(*5l~ZX8OeUSsInCORaLBpvcx* zN?Pp0OXoJF*z#aG0oAXKBdb^V*TNaVd@8%q7lSl zSR3kMnSR;FqSh`zRnq81uB*eCL79|)kuZ3@aa3Zd^DEMZry+6Gz(f}9`1-4xZc1mU zF>Ogsae~!wrp-;UF$UA9E?JGq%&+YYz1I(7HkC^TYV7BQ+tW(R(GR$Wf*fRfuk|oR z^0SL@vl5d1j&(-iDmL?~5SCXpSYy0EaSp$+c(u}(z5x&Ea6I^ti-a5iy9o+BMClTKkHpkw%6Rjm`%9gI7 z;yc#!I9`6tcCPYJvVEl@IU-4ScA!rTWhW2voQ#Lr{#N&Hv!Dt0_}%EfC|S=*{6g0T zqRUo3KXGocs-V4RxpWv5OB!xBXnY$lb8g72yY^QhoPEu{b^~Nwr}onzy zOB}VR?(lrTxBgpiD66%f4ZsDyTX|@F)?ko_5~-(8z`~sGu*u`a*Y$f24d=_nf*%!a zKeXLar?(tGcf)xL&@64TWJnn%UWbs#m0nC%w9`&YteQwQ*4_1V)$eUr6C9sa7^c|R zg-+Q003(mFpF@xOed1s=k18FT4g3l*=pI64(Ig_DvR=fdSe+P=q(1vrcz8ClN+^Fe zg$F5JMlBwz8Ql2M*g620aIr{Oanh$8opCM?R>SvO+oiP=y6UVuaR_>uEc!=A4b$gk zL=-7eQqlZ$9-eNOm)gQ?yh1@zP4Bf;I>Lmyi5qSQ<@pKv$i_Fn+VesKojV*Af3khk zD~IyLTwt#y3d$VQww*^Fe~c;BDA_bKt&3E!`03o4qI(&$T-n#{?CLd&Wkn1WyHNC* zI3_(_%&l6$*$-P>0_lo5zcIQy4+xfTjS-2X#PW{*nPrcKQynS5cG@=<3%Y187fG1L zC9!GRRC0~ciYrN+=mQH)^uI2V0+^FhW2*r0FRhI7%vA^puqCGgYgBhsq5o+>%FuM% zOaX=SHbp&$k8$BWG=h^sYY@OxvvEX_oCla**hkp3zm7q&b&9rS{#BI7=j4j)5^p29 z%e;iy`&f_{`p(1c$NMl+%aldqn9WV%c&WR&(yfE*W1Tl!B~KN2WDKy^#WpKhA$2Pw zDovM5;&0gB(na6+_E^%=({`*+0^w!5P@7-K$YZs$Mu4le?@55Rvez0V(~xQ~a| z%}|q$NNA69TYSti!6M*vh^eOGXeOmdj&rC>BSg?zQ&!M2%m7emA)USk#Ld_)ihKV zfB&g|ulsvCagNDb*E)n#A+nuH;-TW#t_vGF220+x&@ePrO|x%|k{tT1Ec;JcE(6%+(5CYq&gq$v!Up{Emyi=31l+itlX7q^$iSAne*TPogl!td zYS;RKIEz?WnJsEMJHpO4x`t|>TZAo*z}V&I*utGYZDMBBOcVEzLR)bruT?{fz_0vF zXbOye)Y{rFsV~$~DNgg@k`dnz6u0KKqBI@-gVK*P88w{ir@h0sRWL>^Ehq+H_z$C( zEc*h7N<0$HlB~2I&K!^uUn&{ zth?(UwrkR|$#%k(os<>oJwo&=r>IlomuE{P75|8%K(H8|xQ5p7GPiOe-TUZ3HF7I) z?wNTIrs>wosRYzp1>@AxLVyIiK`gR&f#;4ZN??$s8q{YXYjg8M_9vbARoWo2QYE@K zt{;vzVq$cV#ct%L(ySai89h?>10ku1GjJKI8c@#6o43UW>uO_3U3w$i^)ToC@t51P z>RQpuY>W))j_Si1y>~Ro21D_eydxEBHP%WymSBmWth>w=idg^R`7O+EWeJ}Dhq3@4 z1|KQChyAk68%+tnutHvc%q-$j8k(qI{cXCx^+IH&JdPexFiURJcf2t(H@kX7{ z+PNj{Ah9J>Lr#}s^6N5*aHvS$t!#Ur??v@Vy^A&v`$RQ1&e()@P?6T%i1zl9t*8B` z+UQN!{9Z=Jx4gOuLv$#4ri00LxK-!uhBuQ0*#fF~1hqAg)LfZ1F!BKZPFYki^ffSU3UyJTPu8ewj6P z23;uz*^;;UD(@P_?Nh zHMRINMR|!|D$i1Ri>b669D8C^4o-IB^G8a@WxLkU(I9+UbfPc>#QnJ?6 zd`)WgU`1k0JXcbcJCPky(M>fi6qjpEAgL1W=f2xxLv_<=ky4IlCM}?I5h*M*;iV-= zrYEswYO@53JF!xwn@U${KsQ5Z>^Qn+@B7vr6{FkwH@k$sT5@C{8cy=cy0=;6|)X=n`o37=zvOEt~RFK$n!-Ru|cML0sVV*PNm8js=1^K(~! z!a*JJ{-us>32eETK#BRyPIF6EVDDSUUSAOT0m^=u5a@{kJLU+@&vLgfry5@op;_`* z{Do_0f&AJV0s2n008`mVOpg;Dwn8INH;hVL9BaVeu}{rMwnlCYe=%})2acybT=ew| z0K2{UKEXXh7LTt6I!(~G$S){XV)f@Or(!I7g9fE2&Y?ygBXx<+7ti#--FkLJ? zhqy9;~ih7X!t8Bn9MD< z`I|~Sg$6^;m`B$0kqqdP%7}Pr)=>+m<=r*97d!-9_~i9J&ou(aZyp>x0$FWYmqcpR zHR~GDR-X17If%)n+9o+M!J3SRT6>dcI?ZyomHRbpSLH`7Ku71Qg|_f@PJAKQXS&E~ zNYuG&>2Yj}vbUCupVCskp9L9HEMS&@V7^AL6jlXu0Ima%d4oSoy@ij(b4n!YORQ24 z;S8i`Fr{8>;tFeU*s?*SXJO_B8d5CXB%7$o;&W9Ta0dGgjkJ>5v~aXk!~oIM)rSgO zljL1qloWwqqYE@^(O?pn;*F%X;tov`7yHnqbv&7-h%lFBmURhl^u@@06n|NtWzGk( z9ob;^i^iJ6@&rQ%`@(U0lg##=4T84vHDEM!uJlJ4Q4#;cD}ZG`e?iy3H@#gFwm^R1 z7deb}VOdjT7DISOVt zxart8EcBbGSn`1ZVf;ZQo%l6j_+9tHZ`ZHjgDvnsq{=&Jk;Ylc`sX!%=?s_F=BO5#;DOapO*7m> z0Y3cS81%Pae(2%TO3gyWb~cw5@Xh~)LI3XZ`!+N*_%t*&w$t6sfB7$Z?)BHC-ObHt zfg@^C$%zjaz<1orol8^vz-LB%O2>u5J{B85nsl$X@v1U`Wh!a1 zOe6&#-=BvaIF=pF=R8&0h08pzZ>q!)4#l^F6dXu+UX`Pu-K*OC^&0sHaD_3&VCBPa zQtQME>nq~tyHcUXxo*!uC!Q23=irFwFnM>4cK?Z9ot?gdVM&}WpUuIdli}AW&2i5k zJ2NqjyTp2Df0isx*5Fk(q;cn7UwV|PK$VfN)GP)j<`?W~G_C^76d%LDwhfA|Z{(VS zIf~!Cn(VF3PJC~LcHmGvsr$ld!mm7L9UsV2+Tg0O+$|Bk4==t`o0n7CJoKkolAX@0 zw!;}ejzC}J;MXYKH78F436+_jT86K)G^C9FTuM-q-)*8?uchqKDSpZ7T->Ea#- zikBQ7JWAT#S5E`pZSzJ1EB<;>84xQa_WNA*QJ>cmm3`&;jq-l@8Ne(h@EsGr?*!Z0VEnLK|4P=?^=CD90fZE3 zt*zbSU9w4RS}d?lZD`#I|BKt1d;E&0%w(}Cld#QJ1Lx~vp01`H#Fj+LRbIuk2|JAH z_ZvkX=}#0PZ$e#;;5&~ZOeb_DEguQ*3B&z9V^QYj?Sd%~1PZ}Ym0_;K?B%>>Ncm9l z;8AsKIO`XA-@2gf0__Cy;Uc11cX;PTPE*>2uTwM-g>rzjDf7t|P3F+!dB;Bff&7@@ zI4`41>RIIb6UyXkr!o~6v3yQ_(Kbsz?d@`_%9NLY+TDRaTc_J7jHDa>HOOvh!%bpjWb_^S0Bl*z?vH5ao$f01!^El#Lcei~G+K5Pt4xFTS zQBQ8i%E*Ysou(J?6#7?WrVcy1xb^DOH}V@$Ks;}k)Z$xL|5g7^6B-(!&nyyiL^L>Y z4PO{~ZWtG`FQo5@QpRb%K(`IH&znXgMD!)&_E6t|b`A~& z)KN$Yn&S1h`*`-iZ1QOpZdiQNz=-_&JBWSC_j%f2oF_km`sWq4bo9te`e@J`*nK%ri*5jhzF+SJZzBMZ~X zJm775UeS~$AFR^eBW2pd=n0H5oZ3paz1<003sH0N$r6?^eI}hh{ss}(T$Yf&kB3*K zxa4HmG<>}Y_rY(N-3Q}qN)Sk1`%>JsL+93Qn_4dWz2VQBi1kT5K90W4Kj*Ia*7Spn z%aX2ke7$Xd&oIYPf2bQl8tRV8Hp$Oh=Snf6OikPH%B>?bCq+jkq?AX^V^<}ULfe?- z7WuA3DwUYSacuGU9q-KfJKQn{swQI9Z5%fC*Gr>`L!Z%#yq_<=Vp zqDZ(1T){#RP?-r#{Lk6QBEwDoDPqmm()Nm8#`iD?G!0h zX`QjH9D}06raMQsnJ@7Gqy5&;wY2?(i!ype&V#7OQY;nGx9m|A-Jd(}1A*c5!dvEJ z)Jgu>VP|JAP9;gE_yB_|tampaZ|atQ`Aen0dJ-dMd3U2PGmU0UDW+;;_DJ(* zvtNP4_zx6ikXv-*1wed%#}V~eK~_}+8W8mcyR?Gvd98WE7OnpqqeXmPPjNr}uP9C9 zGT)rCq yhx@@o^{J8};0)IP|J}p+zmS^RdwnF*$(aZJv9lZL_bF7sPx-ONqhh(gUi~k Date: Mon, 29 Jul 2024 11:38:58 +0800 Subject: [PATCH 046/791] =?UTF-8?q?[advisor]=E6=9B=B4=E6=96=B0=E5=88=86?= =?UTF-8?q?=E6=9E=90=E7=BB=93=E6=9E=9C=E6=88=AA=E5=9B=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/README.md | 2 ++ profiler/advisor/img/overall.png | Bin 64492 -> 49616 bytes profiler/advisor/img/overall_0.png | Bin 0 -> 56377 bytes 3 files changed, 2 insertions(+) create mode 100644 profiler/advisor/img/overall_0.png diff --git a/profiler/advisor/README.md b/profiler/advisor/README.md index cf9a9a967..770271105 100644 --- a/profiler/advisor/README.md +++ b/profiler/advisor/README.md @@ -133,6 +133,8 @@ cluster模块的分析包含快慢卡和快慢链路分析，仅识别问题， overall模块的分析包含当前训练任务慢卡的性能拆解，按照计算、通信和下发三个维度进行耗时的统计，可以基于该分析识别到训练性能瓶颈是计算、通信还是下发问题，同样不提供调优建议。 +![输入图片说明](./img/overall_0.png) + ![输入图片说明](./img/overall.png) schedule模块包含亲和API、aclOpCompile、syncBatchNorm、SynchronizeStream等多项检测。 diff --git a/profiler/advisor/img/overall.png b/profiler/advisor/img/overall.png index 6d5da107a3f7f6c8c655922bd80d193708fe71aa..1883d4c97388b1cfb774d05fc9e0d368d0c66901 100644 GIT binary patch literal 49616 zcmd42Rajlk(lrVz$ijjINpN=w1PSi$?yztI1b26Wy9ak)!QFxc2<`+6?j*Q#X7;!D z`(^*n|D2n1b8hGC*;2E*yK0P4;YtdUXvp};FfcG^(o$k7FfcDjVPHV)U=Z-cihKzH z28I+yT1;5Y9rmc>QI$Zmm6_*xx?Dwgwq=SkzE}nOjcO15q26oN-|5{p1ovujcG^l7 zjHy9oqmx(e&WT?`;?Q}`k08}B*wvM(y5pqJQq0BANQQs$uh)te|vg z=6NKktE+o{I=y)T5`qE4z>>ni`TI{}-o!300xvHrYOWlq<@eOoRptG7xzXD@UDJ7YHo~&Ivom$`KOMqgz<`9h zhrr)QM}2O$Qx&`JtQ?GtwhwD^o{#O;{GJ~L4cc!$xTQYdTD&MS4d`AP+HGoT?EJBC zGv3gUT+`*-SYO`^efvD$HTHa2SUU%;%gybVflo8kZI1_l<7Olej%c*6^- z7fhT1`s)P|T=^X_<8OF0o`!Cw{~r0EV<-zS6N#Iv@4~7prl%$HpC9&bcW}ng($l)R z%CzHjY$0iE@>^cy?t4~`p@lX!H9A`b=>Df2uo8Pr5X$-aT4m+d)6?~F6%&{d?+vaz z>_1&ug~7mS9783C{`tmV7BlVn&``7U2_KU&H3M%H^uLV*fW}NplIj23xB}C^+p)o8 z^8G*4&&3757lmZK@whwAd4}3~8*qDc?sEBxe1kn-bc6pz`p-Jn>EQIppikR%c3thh z#g!{hhhDLoY&qFs-Sm+dzW=*gM)7VGkSr`JCL=tG9xUvC1_}%HkA~N{d8pbiveVS) z^vTHZtnW(!XKqrI-EJoXk9J6W+~*XlAJPp_{1l&^Gva>T3i5l>d+TT1*LP{doDAn^ z?nqh&%VKJI4!uM(HARK{&!2Zkf&GD8X=r$RuCg;3_KZ#bG~a4hq$w6~`2)SUOaJa@ z*zSSyLeVe7AiZl=ezb@Y^XYCyk|AC2Id4?)q03`;Ek&dFDa5ve9!INVQ#QS;or3cI z>&(<}*JF*L*T|72(?3SAPXjM#KrR)l7=4x~II_gEpY|2^ll|`f(q1RHBzp@#bUg{i za)tGkJZL_V!&l3|Ym8TwtQFH#KNybIV4TU^-1zxKAH4Pc&(vXHNg+s}MjS1_G#$*8 zhhPo10GAy_!(o)bjed(~)mJ^_Piv1qpSn;`B70l&n%T7Ip`V`JPbG6knHYWAYT4aR z(VyXb%#bqUkp7t?DJeuZKwTj4auAE7Pz>5?$FG08vH7kGH{n~WOeaJ{9?5y~a*#!H zN8pL(7wli#OM?vfu&4|1@cZDtU5>k4By>z{_3YHA%mejoPN zp630$LT>yX*TSAYmMvqEt(0^4)pU(2O3@+*nBk}Q=p z#QAVOv6zti`Udu&-wqStLjf@fXgKBkDf}NV13Y~O8x~xd(Hq+v@b7~G29}E)CSY5l zB47TWjUXfsY=m;U2U@~^)^AS%!qBp$Q^IXU%PTV1+yu-M(r@umOe-O^>4bl>yc=LtBtmlqd-S2>T5{v`jb6$WH0 zzW#7~K8vd*Xk%x$<(SH>@3~);x{S=Q^uI@;`>O!Mr?b?qXOid55tc5!zYj5Vu0Zwm zTi`bH{{0mlxZ8jmZagI=+xJqPC+i8g`WF43-5&mT_X~CJgC7QuYODM`Yo+I0;ltz+w8pS@gs1#;o;!|JQ*&LA;9tJq{2~DOCD`I)h^zaFW4}0r9yi-Uw*~_vs!n`zr&k*4F7KWdp}Ia zoG+8)|BRo@0~Ue^L;n8j#&D!iMud8v*4ITau>Wc3DggsR0ipcmjkpvr0cNbZt{T&& z1pmxXi3s?Z6#AFXLs&tx7+OsdlK=I;GASTN_7yUqkzRZX20RLTYv-B%D$(gmaTuE~ zxQckY2<6QS&2yZiI^{wTZ9=U2xdnrfQ5{DwH2#_fJe%Jc{nsl7N@i;l9>>DgWuO4~ z05b}>N~)%Ly`cFelt%Z_jeA>`(ptN{_N&|mwSey0`1W7;$+ADJnObDe2Z^}IU>ci; zGQyT{m9~|Bx~T5VslWrP8<7BO$=Y(cc-0qS!$OO&UoB6H!>CP$z;nrBLR085v^W#+ ziUzKhh^&W>IWF4g450nixNB}+!pT0OyA6m{MaXcta4FFk(3E8ZCo}_VmW2(D$kVot z+~Ll=^QE9agN#U=^;)Yjf2?WPVjd-NTMmX<*r2v1w$c?!77dw~PuG>j0W*Rb3CJK! zEh{s}lPM~2-6<+^_L6w8O5%99%fLnb$0UnV21)ootlj=BdE#8hs|UA6+k5R8Ui}I! zYo541gBYA%WgbN9jW)l^cBJ!iwH!-IYbWoe?gD{sB$~ zT3KeZ+plBi=uNI2mV1xgRGS0LR{beic2kjhvsJDz5jug51@9I~)Mr8sIvpZja^_pC zdMCWcB@ON?E*A}YSX%CVBJ7{r6uT#xzOU8aDM*(y2w9x&|)2U854~XPU10E6b3Np5vx4NtY2RBJEuDYU-4Lf-H6DQVNzA{ zg6Z&2EHhjmQ}#UNO8aPA)t7vx@KhEmH2Vx@f9OjQevo56NtSDJ9$#a(51HYXGFpF_ ztJBpdoARY-B6-0Yrw%%V4CI3%9n879_&n*?5AoNSQ+EUCh7bJW8r#;jUzCGEvbEpF z%*~IgFg%=03&31f4O$T^GgLv3w5bFh?@S%vGFfc&k3>hGGA7;6CSOOF_JSduKd-QW zpg*;xje~1J+N8Fy^Y_fRcL)9|{Drs6MPr)#v=zwapL|T5Y^A_I6Yjcyi^-sh%k_Vo z)LI+DAp3%SOjd$rjBiLfm7~*65<5X_=d+oPNeB9pPK-52h$R2ZG%}y=eRBZj7wQ0$ z@DC20LYg$0F9gU7<)l^9XTu!GY~?`leVh5~5gW80UffIef$Ns*druDBV<(n`C|9Qk z1`s-52kU1jZuuBVJD=N@u*_U9B)#TP{IpcePTfQfh0C|ov!k$6(xi0?)yN)F<!ME2oRlV#+LK07Na0rMj~UIH?{~L)gD=;QZ74T{OVWywS6OOz)E=J#F0(Hyd?~_! zyY>N<^pkqaF8!}|SY5S(tXzF!Lu9z%mS*!p(+(B;mp+f9mw^}YyhMlJ%F+nAal|rv zn^tC}CpS8U&eJhjGpg5mwh^Xk)4KDyR9UYsA=pebllD2}5Fwm~?);ag*V2Zh6W3`1 zRBt?GzWAB$JrnRy*9CZ>_B5N(4e}+d>!i2Ln!KvHxn)Z-QJg$TFJ3-_P}+acq0H|7 z#UCoTc&dEe@4CWBO|UIz`m*h=_Y-|G*Rlb~+TvEj!rwW17iC;>$tQ2&aTJte*tVn) zLx~69ny56tP`5llt&Wp4!5Ao>iX!^u_wH@)$)LB+UAXjFPVp&9`(19 zu`m(46`g`u7Sx8g#zXEUCQ(W@p~1~6$yEGEi=ywzBULw{8ZHldLNbJwDIITz zbZnh0&TUh-Jm7fmjOmQR#YS0gJzH7CbP({GZbf`uj;fS$!>%NUr% zI8Q1t*=PwDi>0~3BPu=yv5M8m5`whiJ%ki+sHOP~&;yoxsXR6K%p3~T?+V{NApH($lGw>s6)eL?)+SMIP$R1&u)|j_I=fmDhJ3mG zTz-{Qxup1@%vzPctUM}KMEII!<~y@-n$<|lRHO#UKrG^3dEqb2mo&AN!Jmt;%4nF% z>gb~w`JXyH4XYAjRX&qLlMLzHv>RN6bCJ&6+Q1*$%E#8ewY)DCe4VJP2$}3rL037J z&#orrn#M8uP4V&z{eqe2+h0@@g*D|#5#3YENmbN*w7g(_x*JQen4x%vO{~R`-F#m~;VbOGo6%9gH?tEi)77e- z4OK}}>UN5YpQ5frsAXy&4&42`&PQhW!;M}};xf*y;{?7@7B{m^6l*B!&hH`MZXePq zQW9=1z|vX*$2ETvsG`M*=wC(El_DlP)_ljCfdh+%ryZUF)-du)WY!jlV3=gnFiJ*C z@@B7!;|Xefstnr;RKm#b%iH`I9k8&~85_1YGlBa!n6@kqvL5HxI=km}9ccR&FH2|S zSZo|!>W$i0+8q6$MB{M)jY#fzwFNm&cemvYLwy5&1j zY`HRDy6IPf)cL$E<-dD>&>m*>WA^WUUDVmawY{L~4WINb0~GcRU^=w zp`1fC!Jm5TCVkC0Ab=yIS3DRqECF>lLf43ehA?!!ol#MA%d#uk$Wyv^X+186cD!{w znKD&4K1;&VcId6C3>DdBTns8g6fwF}k&4rp_Q_LHlvxO`Vg9x_Kd_|%H)reX!MFNd z0aKb7Bf36pC2k*hIyn{i3l{qU~Ye0@CIkVb6tg4&OJLbwOHw zLr<}Iw~^B9-<{k zG3OAM3eQ8mg?*4k7o>}gGkZVCGi=#PjEAyET2Py$ZjM$i9lVgohVv}XA$q^2&=1O6 zP@8lRx473faa0YU@~J=VSu%L-sjBsiB8Fv`1E5y$R|+~RYp=a_Rm2t;M8~D|>`KC( z#xi-uzZuY%V$|$mlhtM1S*B%(jF_;-LoTC_rCrCIJ!k_}gUhp3U;}{7|FKU%9O(9S ztaM4@xL!nfrZW4OkyESTd*lO@`mV&dm$bhg#tLLF%2LqhXEv={DD=-NVz*y1x0yRM zo3RvQq7ORzwt7G!H2i-&O}JjDBB)`)rF_S^1Uynl`zyw*&tj!9M0ZT${Gj1s!++gk zFJMD9s|XaimgpGKP45)}O>*on0c%HT@an4#6j(GFZ_wS`4O|IEgzbeSlYAyu({7nE zP_hni@h{h0QY&l{1WEW#gsSCvrZQb0)p?>ww=>MYBH*Z75mjivsVbbY*R?e+J=}VZ!Ln zSgtR@z$*2_5Iz)2{3{s~hWx9ysuRv;R}v5;^4`W%iV39t5=raY>|etHy(~-4$>^Fg zwJ};Hbbto9p~?2*!Z9P z)5o-`UVJX0ZE7lhRU7QjG*+vLW{pxlgW+v~^YPEBfprJ_1H)(xN_=DhGV(HP*70IQ zbw2Am83J1-^>Rw6ZFI~A`d?Upt3M$?R*2{en)`&P6{Jvi9l6|30SimE)nccPaPn_24=MHZUA?qx6ajvO}vF}V+gre zq~u2M9(#hT0=s$Ol;Vr{G5ksD!-zgBqd@!LxYi0Y3F8pF?s=mQz~iA=N+Ui%zs=|q zeD~mm4ee>oC3~eJl|H^qV$eukv76u5pNK1VCWkuUMy;@-5hswko-O1s12MpiDCOIb zIWX|jchbh{mkY+$=L=3`Vy6l4axxe73A6wMRW+?xG+iISox~dShjBAiwcHT zm{IcOt>nAK&kfup64`Te6Rs^i=u2aioYfc$#wME!6;`2j=| zW3et_kL=3&qLOErZ55+fYcFwNMH|%=U}82M_kSj$&EVWV$kT*PZ$w4%ltR}e<&W+5 zv#{l|d?tRV(JVY!-6{R}^qon#JC1(&PR8@B<0%H$&sxP~^s>g*wF&M~fKzUwT+UgQ z8al3g{DQEgIV4rbi^Ot8Bqrv4#O82u0{P+rKF?~Vc2i;2x2J6f&TU&H#K$WSz%>ayUcKJASzA$?pfzqb2Q4Dw%GLm4o94(}YU!cqWRZ2>9J>^hIGmlv>kfQBA{b zP_~2;R%lO3_$p2CxX{>@e-k-uc^cZE$|S#e5%^m@I(^?yUecuu3p%na$0D1q5SlDa z^Uh&M!S7m~=MI^~-13U(SpyO9Hl+eJDVV&|*e|WI;?ky&>`W3{yFH(7TN$_sisD^bA9R9tAr%HikkM#oxX2*k%E>~}hCxd{zsCT`Jr@R^Z=@E(N^am^Ed z8oHog42_xQ|Ckr!X#FK^_<{V3V~fm(Uy3dm_Nm>*6dJ6qEko{c!8PA2t>FPvio{$M z2iy0KG7FOZd>Ria+n#+nH6bg>*9yTqc|dbVP7+u-RP6-~{iSUuFkX&{bgY-{b z5(<7BlZ#PbNZMD#b=nF;xV7&OR=3!`(nvvtm4=3~R&8qO7BI~=+BRQ|I{Ie>uv=Yb z8abS`_}1{2MnK@jFIh^)s!D2H;V531ohPVn^jg71EdZ`+Ilxyn9ak?0_O!PIZIkN@7qomR6E|>xk8LD^(9AOr}Ro^5?=s zjtqgf#X1Qnd7n4_qXj64e`_=tB_Uh2#wV@d9XC0pSx{8SuQhBVTGje~j)c0^=jp~G zA$nv^TeaQ_fnq+|JWiprx^k~>N6z^^=^HXby=PLbqI9TlT!zF{=de}QiF}N1=9|v6 z&ssXWj)(6wd&Ts63pRM&?@W~UsYh5O#xJ^dVn>8gY<+4a!3J0=c+$mcQ>nu~BH#SD zS@r6(Mg;s@Td5^%7SUl%$4PTFmb3aVxA(-9l}tF_Fo&17;=4$4j(wu96#h&K-r_@aiRf<#TWH4cgetcUiMY=?BOb7pe@@8-o=+tgymyqEMHB;ol9Qj zXY0qaJHxvbdwoi2{qo$|gVi{4-O8v-)D;_t)}uz$l$XysG5%z_Gezl?{>pF+_oS;^ zc0>?s9+i`p^i`>gD;ZC%1wo;L@@?1M(u$zNatcT&eMXNsu~KmP0n5)x+Wl~HH=#re znoY@vs?*xst7V}Z%>-smB*u(N&4m}L^AnTL_Zns-BC|ch0akF!RM5^ny9pq1!tcRC z;FKp(icNL&Q(D`qjDW}wJT8F~ZD-_Foq?n+yIZAZup;`n!CF*Ms%r6$_y-3)65~qd zXV+M40z<*mk4nc9EEG_-uU$rU>Jb9`QN@)D%xFysujVx(&T@dmBwWUG)Re=f23Ge< z9LDO9_D4{IZ-MASkob?LCvi4Cnj@?EM%Q8%LZT7J>(>4ou><25(W#pzf&gGZuBq{^ zxpL(QqbU_EI0CZXv2uGRGB?%_5Ra9dDdQSZ?5j_-xRd9eVKh}N(1yg=d|EPSmW4R< z()Es6f7r&(gR<&%AX@Ji6Xwlcz&7M6qcG-Jp{FP=c+2?VJQP4p23UwDOL}5sV*6>U z2@9sg#I}$SS14R^`L!Hlr?$p3>I5)NBQT5u~jZx}zU<;ye_(-(C+RBW-jyUAs)jUv8yvUTIArV zHrE*z-Ey8bE4_6CUwfY&{@>mucgbq4TxiHp;6~rr9tT{95w)1(rNsP6qCmq4m=WIH3>% zF6~VxEpk((g-b(|@=pzO$c%U8FNb!gP1w%vPiyIZ^SxJY?|Eo5nqrfa%$RZ`FE^Yo zSqu^%DV#Y&)NKphUOtY<2|L*42{5ZVj!=&gZ>d;y6tf`HZ(Fyhc_{`mt=8QN=%SV% zy?TGjj}TyHdHxks>q5L>>OS$Iir5#kl%@?7lsL4t5Hjh zEe;pTZ85YO*%0YXsDRfR`TJMx-MCG<3$<4P^eMvoS5c(UWm_ZkCUoD4uLCvB`w-dw zIFCyp!mmiFXIDjdC-z(=@N(q-jRO!z{hz#@` z2C9&K2d}GVrFEDIU#=BfChHX3?O0|s$ruwv0*CUQ0;Vx-aCnnW7$^2sxA!?2^t(w& z(o0bo-w7jxiVrS5DH*+|g}uT!G};x#E-r=LtsFcNfAgvfZvsH$x|i1|6>MzZ26dz@ zl`woAc|f@l(@a{UcR@_EVm~I$0_bBn4LKO>{-iyO)8(_20lFi#Da*FvNXJ6nVMJ(3>7q`{b{+k^$5tWQ851__k@R~nfTy%i#j(EL6f zNCH`vEb*@(0sl1%8sti~D6!BK222T6R!1BA#n$V!)5O5#-es;^9|7I-Va+1YTmpeo z8v!h|f90wH>XI7~0%_%Cur4!p3P_H?JPtc3{EDSTF#khped?lGrnM}2@eC5&r;>bt zI;-lGD!R2*RVCx}OXZILTSq!E&$N~zxD23bI-v#P<{qRXGN|<~g1EF4g8(o}AV`dS zBV{Ag=uUs4`dgl17_HW^Y4FR(!{NH3KD#7)h_x+LHGirsUfg4;K-9l#Fnq@|z_H-X z50QX~Hxx?Q^eSs>8AL~X_@$cXTZ^B*VPILFwCAz;__{oZ=+X*FDVEnyG~EA|^1q9_ z8&DN>$}|ql8l7#q5BklO`C?Z+_S@=rJWGLhwUxwf_1RVR*vs5}a94#g;^MqQkg@^6 zFYgEwx^<+8KiJq%>TlLIZKI?hS4N<2j`oFajm36}i|_-i>ur{utxE{#3te(iHkw2! z=M??=XxJAFGOH3guWE~+_`97OcXI}o3WiST{aXozXuFD&Z~JJq3DfKOt>D+S;%0sO zvUwCk;w~$Vr>?gmU+y#E1-dvT%9606t3~~1V|BZsDK4i1-5!iIa(PLe& z9#y@xYG74wEH(yU#R+FyNo8)7Aqs%Jv2fsx48k-taa4xP*qWCXb>2R#Y|AH~$Cq(r zg5edSC(R-r21>s8v+FDNzZgBnLkJt=C}-EU-IB-JSF}4m3pozS!^JvKv#3t zJGYT45gR$G%>tClSZqYl%u`plpfr0nzIHP5>{DS?I2sSWbvzBdxfHNH?CZN#MDXEH zY#B+bFTwAE-0I)Omc1s+tNt=el)Z)@3-A2}`;eQ>rFiWnduN?euBz@VLJb>cWi-Jt zC3fZ(z|371a3T4`m4G7aDR=fpL$+A4-eBbqms5!p;Q6SxF=hOS zvqD{6TGZZv{_^VLAA@0vL&}08M?~W%Zf9bU4?yG7+8;*3?ofu=l_qg{S-`dpDOsJ^ z;9z`s2jcg*Ii@G$v#XzvN=|DemUVwQPNSy_vwCC3`km%&zE6AvM-6gG!UDaL%(isQ4#YDDxtr z=Px1CqYKcX*|+_f-ztL&njVDjq5P%$n9Jr{<2xh5BWOluOc#yOW8CI}%~Zr!F6heK8q`8osl7z=5T!1GO;97l^V4Oant-iXQWVOB+L7&v>iwac2&7u&dK z3YJE#iEx>OO!@7%0(zB-(h_OI$~i!)TELq za;<<)nhcmIjcWBA(+2uU_LdK(fe5oC{j(yP?J+BNN{jaczUH2?(50&tg(>RO!g%<( z4NE@m{(PeR?mMfRO8QgBhTb`!?UNzg4=Za666_hX0tp3u6J-AOIZw{qB3%7~5!3ya z**S-HeHk%9uK|{~1nH8zmJ+HV7!~*WM^G|}AjL^fKAwzFy%^-fHDvOqG!c;!0=&ex zs{1cWvmvpn<% zXk797%fhKz6g%}!ADXTjeHQ+!V*HU)$pgc#r2LAw+vqYe8%rZehVZ1$LSLEHt?H$q zhVvvMWo2nrF=*85_AKekrrEH1mYMYi*7f=08QXz;t7ef^)*{fW8o2}{rG()Jpx|jk9Z-_Gu)md zOQ%ryBI`GE>%6A>Vji@m(I*P&LS%@T@#h}-jQZs=ywj#vlBHBKKOp7rodiW%IQU|@ zze|5Fx~D+x8FQx*l|^pzTwq_XMD&SX`(1`FFUHj>q0Fy(RQQWZnX_5kBI1aQrnfY#IGa`%6PKO+L`i2{ZB47$ORJSj z9TkVRwxLaKFTG7Qn9Xy%id?}*!pX=C z=1u|IX)hTshma=+RmB0p+e!ZS-?L@_55W9x?u;$CIb59q!PDa*ug6RPqEx_o9QLF~9{ z++fda&$+*2Z@RAao`FCnV)nbI3?Q{a=Rz0w581=_Z6}{v_y5f#-L?N;O(K&FH&->O z{9i)bF|ZabZ3QLty|=$V5*)y;WJrZj3i$(}xj!N`CXBiqD8w`-3esaE)Qt+HPUcD? z_?WOzc)7RXFr~&=w3I>JAVj@LAm_4{_X4B6(0vK3=4Osm-aK6$HbD=r6otXjUU~t0 zBkZ-%^QHkvON?j;X0X)01Ymwvh6giREFd*3BLe;2!;IF&`ZHy7_!OZbs?&>quiGbs z@>zRca}oMZJaRTxhd?4-v|9OSl=`<38mm1pwC0S$Kih-?1COE_A0GHKHFCyg(Ib{G z4B=e_N`a`NHBWXIGH+fMqb9|O7?pt#@TwEOb`4lsks*~s>b6$t;`L)+l z{aVHPG$^4mDY&_o4z!A;wQt&dBIRuy{xaauX7Ea|BpdmrjIWN6`Um)eFa}& zq8+1~^Lztv30Yd3@0p#Bn<%|mlSdtvm*k$6r|t|S5^*{gP+?y*R{A++z!fxUzGqJD z@Zpr=v9szJkbAbK^cIJV;tfG-@}5y-;{Kv1qNg^-cm_w&#d?RdV@3=Pe+|-b$7IT9 z@1;-dn7`9g%EJ01**Qbdb7jQ8$m*2t2so40N4--t$SfhIcB&gWedae4=2<+j+u!r6 zX-W>p2`GAx`&gMI=U}D@*G>s_e5GtCdP>ocy>FV<&MG&^@1a4~$5c49lSRwD#LGCQ zcZeTsFK&LrRk#_th5qieA*ES|%1LjQ5SBuaR@0XGjRu?sk4KW&;DVirGS-_ki?zKj zbWeh%utg%W-$XHO^YR9Ywf15>w>am1k zhVQ*~L)1C+TSE??h4Owdig>g|2Crr9Hiz39zM56(rST~W@1u@@*I>@F;bv_&#e-8q zPXxbK5~qA|c<@TO0g~ThL5;OCU;2bA;QT3GA92uO>B24FGbV&T(l5*3KNT*;oObrL z#hFOl0Mnc$>V}TRgr7#5>S{O^7T+UlmW`=k_?^#WIAQATIvh9MnBlvxcBC-H0h6|d zQ4hI#nrIB{LFAf8Pg@qjkw>F3JVwqlH^R0Xj7Swe)Pq$l@>j%Yu4|&7MNo1_JCiL& z9)!fG+BkwSmvWUC*kGyi#W1zlQVXd>456D=JXtpUJ}(XhI5z{r50Xld2dm_r5&PX6 zFuTO}Y(~jh!Zc=etadW1Yw^${(_Y%1y`yti-EdHe4ikzOZ)T_WUwPfJg^6RP& z)=9;Zd+8mxSsEf;Nk!@X`=f&%MTSed$~f)Ms=Nla=!qM7?$JMN({xT{&D!3Cse#XQ zUSG!_=5cBoUn4CNvOV>+3k|{K)!E*+tNb~R&quM(p<<_&){VtYS+bdMYgREr9*svV zf&C7PL#qYOk&eKz@O-MvaYD{DO%T3nAdTKM{bj+g-c6-VwPeRK7lZ)*WAIV5;KSab z@v`dtAME|4O|}My>6!iaw_D1g0{-9>W}C6red>S)yfgNx+&07t_CS}wlkZb_T7US> zL!PiO#k)=PD|*bXdF+2}0yzpOyC~i4gt!-U&y~M*l-pfHS<_MBBG6HeG}aXzcZI5I z(I(FWYGDyIQ{{q_23!4#TSE%tvd49wGh#z!#=VRnazhU6^ zR>F;JXedOu6dI9r(?)xF{PYLZ9fVHPUHioy#UAWkUgrpSKG6*cgtdSviiS3{J4D%y+gGEKe2z_Jv1aft@kIX z&{iLX0n3R1#86)BPUmI3?=F%`Ev*4K|A6%M?DBqiWR=T7$9tz2M8-Gg25!(pS6tAD zuj$+Bcv$u3t_uVo8iY<`Ovc~2tf}&A>4ljd{dGlI041Qu>B1`K>m@(Avc#{|kBC*Y zIW(aDwdP8sTNe~rP*WVg8uaH(zkbb&p>c(s*RHSqsy*wWV~r)c;EIjLszBn8!|&Gv zZBKiw^6mJ}ZZpSLu?-ExEsB_We+p-4-N$dJID0HTSIq5sO?iE^ykdCO!Q}fpS;rkc zC93Ho3+iG*R{H||Lhab|58t7I^A|>j70$)ZuFadDe)W?3sa_zQ(-W@zQh4Z@t2V-6 zQiIuoij|6d6eDa`f8=K)z+ck#7E-nZJ`1XOU}5C(tdR)*R<+z z&hNTN))^E2UdM5n(S25#*Rf!4BDg0mHnDRCgpk``dRXw`gmBOZZ&ZkZlCHCbb-XYN z7{NXJ7a9$B5Lxw~u8u~z5s6irv#p{KbsSr44SBeA#ES67=@F3bXWO3gSrWA}{22un zFlb~SSL|^F2mrKj@gI}|VHQ+CNVaIE?S<`*4@%Y$V9))`LMFR^T zJ2>Za3{2^UGm1vt6GWC{($!fzH=ne-1i2?ZU+a5{) z_Lk0#`~FuLjArrE7I^3OW4?DTfF3t+|D^Jdg6I$Q+ZJwJ>;lC7a^X^&dO(}qrRrf0 z06#`;c8aX~8$WJJEZ8x}fYJS)8_W)$ji@g4A`wU^fC?6ZA(TW^dj~ckEilfVV2{dJ z_xFHX*ZOX<{X6MzwNtDY@;n!bjYCIxl@)2JF-}7D_~5_mTbi@myf`Ox98ZDrpZ9>+ zsef{(!hBme*((2dul8@PQ_RV3=|eW)d|o2KaO9v|zwj$QULGKW2V0Gzi{<$cyc`A>f;DFseT`^ z+fS}*qjvmWY-fY2A6c5vm{Nb4o*OX=k&C8pSRL+;;{h3Sr~*fq9!nzqYymc{!Ty#j z%pHy}<1w_T zKOW_p0{C_GV~p6{v~HWXkr$aqRK5QAWbv^;RJ`iu5Bktw4`)ic>o;Xvmtq9rN#M%S zA*V|x6W#x40mwEOaJ0*jXz zf5p6)zSO(ORK*XBQ$9Oh%WPyyqd@@d|Uw_`L1!6RcBE0yvsP_9UMwfB9Iq zh~X#5Y@ri=f|t>e2wQTZW$tTBkGC6)0&QldR{g=WRwBsqW@rlt$9vTBYmrM7Ujwg9)u zg@aTfA7m``7@)52LRNREPqJjyLAi!ZhZ!v_*6tW{p^k&=(@UcOctHX2P6 z+4aJ7>~#M+H0F-?AVKHM(PffAPkp!#6TordUKm}^`V2r-qD$3+Mp5Jc!)AO?3u+@k zf!cQKSp}5%1dW<~SO|hM-W)#TBA%4c zl66$2nsawr79<5xFV?!Zu+Cf;lEygIc4P60I$-O?%JRl;kf{Uplrn(yC4@)v-H7N% z{hTQ!#3!3bf9~?6{gF&J-oteNITNW56&xnW08j=Gs^Of1x1L=M9dar>{#*-TsxeO8 z3FQSqXF~PwuCDhej98iZ^fEf=o&r-lKm}HDMR2~;@=r%ykFtiuT`k6leM)7>kgc;v zuD15E`3Lr_fR*6xa_W-p+AHeoGlCb;UW7;6j0ObBYoY8g z;=!rwy)Ev?>jI=wK2L8tQX3aT?-JHuSBaqUlXGvQOQVNo=nHlG02M_DPni~^Y#gDM zJ!7ofRs@p18y7-1n&9PSR2uS!w9xxgbypt?H4_V<875}LkFb2Bhl_j9F2O>CrFLtr zf?LC&%(BQ4!-PHS&LJ}i1r>PUtW29bfu9*_OtNWqx&a?a<2V%3o{RuR zl4~I%yQ+5tFt0^rswy4um*ze$Thb(nl$%K!p&L3a`n+t|*DH!X`$qIit*o0IfY-e= z)NoZqeqAY-vyh%pYpl`yscjH1pq`e-&>B0y3Ei4IiqD^x|Jl9%2NR|`a@_?Oxe--K z0Nb^~{EAWjY!#D{GW1+qx({ECi^N!bp9NoQZDsup*Vn?~tqO~c_lU0e!cR*$}YvnLZ2uiLA`du@Rc=2zKF1~X`&OS$LM3O*Es5ntqAtJoPBBZ_4-9V z+V7_^4kk+(_(S)ijCBHv92n&w;a>W$b+ge#AdA8JUaCb2)yLQY{eNoaE-!7kC2+MA z??3?yB`pBEq}@4-qr!yANxf5gFeQi0?m$P~yEkeg0x{uz=Sofb@<&EeGK|%$l=t=c zp<4z(Wk>u75c)nGETjvS3hJ-lMo+pr%0<+VUrN3f6{{)1)xsAnJ@d~TQB~Y?cxNE! z*J`;J`Twx@mO*j7+q!R%;I6^l-3jh)!JWo~y9a{11b2cv1b3IJ%*Uv*b80DI8SX-A1}F12lHxcHfOR#F3;l&IpAp$cvpUeqq!E1xW^QBfE)Tjh2l zY|EI4k{qDsB@WN$EuDDPlVdR5AFo?Ihv?20mdg=-|O5qhaGKt{LO0Fhi-|b@mh=`W;=+wTB z6%G>zBwozs`SjMR=B`qGWO@-C11o=k{7b zy+D}1sp$JiQQZ&QTKie_WVO08{o_dApxlc8ACv!ofXTbo!5x~w0_t>9*Y8-lF07aj4;y2UYDm#J zl8%4Ab)`MEe+TgPWtV3O=<@+8r$evcdAHo1nkb+Cle*8}$&P2Zd(s{*kccyy# zxu|mUf1@k>npFQwx?dHwyaDJY5qKS0nO~iG7oBp4)+=)L{;Irr$ZsO=qUa)YyMz{| zdDPh)8Bmk2c)GeeHg)D*7pPBO;Sb1XFc`guO+H6FpfeQ7rA3+;HXq3KAw{Km%x)}+ z{?-y4UG|4@PG{}{KdLyFQ7(p4*Y?FVCK~!%X4vcJvp%>HQ`aU@`%iuAGv~ruw#Ch= zb4+7MZuqEeiGnNqoa1m^ogopvA&^+NS}SL~-cIsR>}*d^7E+^w>$e@gT}yamcm>Q? z0W%Mq5hd2q+jC}dsAz2|TimvklR~6JZ@_U8m&@;SExtQ#L5eZ+^X$&;B@K3qz4*@) zZd=+C(>Gx$od;2;JBgnIEbaAPci3Si?0(qmGI~8)e{){b-RHJF*|RWU=8>3H2h`e` zD%E(Z4tzpzC~Ijdoo5Du=d{FOau z(#;+E0n;BYciZtjWK^D%)GU9hDZrsrKh0|-bH-!)zr@e)y>E25wEp&_ZjWbh#dX&+ z#5C8e%Q=f}Hhq`+GB0%(z7f?S73^&aQ@E-8vD1W+&gI0x<~_JRnkGKQ;(=nywl=OU zMMbYHc=%;wYmCgHK9V>f#RH-{1pZ$szr2jHe5;KuT_(I19yk2k9zJlRTz_uk*F09N zV8*;C#sCu)T)!JTTD=wJnU7ly;6xWW%C}1xpfZOKtb4soaKA6S*XEX4lQn z%lVTUnuO=MA7?nb!Q+s`u4l+Lxo8rv2xI$cX>c;e`NCz&JH|BfYgrHg-uB5P)H~it zNUjd0)8m(vIsgL-kge+{YEHh}HR$F6n{M%uY}$T_vQ#<>Qa;j&=!_=;zObdmbY+EO zPxfA-XAn1Z>DA+%c$9jL1+C4?pU?_@B<%?r7r{sVLcm8+zahkRY4?+7Tu|Cp2-KmU zyi#D?Wk3$j`1~||2;7X9`gHeW%!oV_i$(082G=6ELeB%5{p|IcZke=b1V}2$YhJn> zboT<#$v?C==Vv;t{FRp-c;bgx3EWcVMAw30Qf>lNIWzBwdmR9Yh!R&8MUd7>*6oe+ z)l+$CYb4P$tI#5=0xmTz*nrPelLpdh%%;KWSNSS&ewh+FuIbk_LB^Q0QiG`>duGn+ zUkD5@aqZ}c5>Qhaap`qpzu{eV<%{Agir&BfF$|Rq$YzjgGnncK6&jqs#D?IrN6yoUCZ-qVgZJ8U<#ibD+NUy_h}!rW z!^8kE9#dgHo?wHq`7l3shi&|5q2Myd?@p5qHvKiIHmvY7=1ynbw9Mzi``*8VdJ~FN zla;ZdmAM;hH8$?X+jQG+s{BeQaL6_Y;t0K(I6^dGCn^0PO3dy@e#ikK#FpR^Z;AB> zwTphWF;r?rOjCq7WYOQW2gd>K@trP0NJ&n*4-kgEVdF#{3wYP?tx7kcARNb%L8dnk zAz)IG)%j?CYV0%fhu88wYR_D7TQto=1WE-iDm28G^;$%BZNiA=&t#fMi*l~~Kv`LU zN-tPI51B+mrb_e4Ly%g{_mn!BD)uT%;@zZZAs|%8civ*3c@*Jo9xb?Q6@D7tkU^L0 z1Ey^S&TLR_K^7mlrYJ5azb)jR*&Z7mI9e3%OI@x0O!mKTqejll5=N+MchYW`OvIg z1Ltrh!l&O`HHxhcey0YEY8s1InbMwOQZJw1s{^X|tJY?fCh+rXLM7d%iXLR-BI!(l zd;U(-=pVaKb*My?Z1+@6+eFUBX20kZ6sh*VM0ikFoI}le(A%d^1;smj^KcCeA*@Ui%`y^~QBMlu*ah4py++T@T6K_vF7O>)X26|s z_-fNB#t|Jqq<%XU_96*j7HSCZ2#enVgG8bC#lbA+P~K{ilbgY4;3*iIj`9HMolsqM z@b((5Ds!R^*;{see!J3zk-NOiD_)fvC>OM~mC|^M@`!MwlDY6(-;2{qss(l!Y}Ft? zD2D!+QI2vr)ZThvM){N5#e;w!TZD*OJb}Da^}bm7`mN9osRzD>)bMS)4%LXj%RDvn z@@WAb9Jt*G^zaw{5f-y${))FKn$es@w041jB#g>)Cm-={A-5l5eV7#BEP|i}KLeR( zQ(Jy^j3R*0E&h}5s?e7q1gTmF?w#6-(U7PeG~`^4Xe790(7Pv4#Wy#jW>5rrg?c(oUU_9|1ZB5-96>w7(z-9^7z}>6BSxixE1VP4~2M2VzRp^$@$iQ)L!P3l$z;ymj zhXPD{c#8S7D9B^(>eP<=Df`jJS`WS(>!`}vLm@kS1kqaWlDMk87wzLB0`SD?IDK=o zU^><|guH4Yq73fR3`P5*a7`5%Vbb}P*yvwnwDg>;G= z8&IE{8_FEVgcE5YPFJ@6v(5N*vQylQ1GE`z)FkqL`@)zLUF6{Z=32-hbTdY^9a}$8 zvm@qFa)SuLxUnuZJ@^X#(l!4=Nw%|kuE(HtbnT4FaA8U{gYu6qtN(@8RPLWN7_y*aL`B=3W;CnyJB$46tRB?MWvYd zM#KY5IyOX5p&>HX|8^EraOGoEVO;S}bL}+it|oc_iS^W5p=VQOGp%nmWW)Q%CZLx0 z8<@_D;Q(zNxM~Ub;3;*FcIkpXsNxMPoIw});q@Tkr6?8!EnT@J9FQ=`CvG)eLImROizzVDm*lRiR@7c4f?3NGa*N7eX;rgX znv2IgSftY@<9R?F0Nt3=K?Oa!^NnylVC8N2Tc6=N)DMerO2~}6Z`v_#x|8nA%ProJ)J$ywE4YJ1OzR^A* zC8u^|9rF=>R{7}Rjon?eHG^3-UfpfVaN<%MHzWZ*mdW)YLUoPOadR_GkE&gNJCX&! zXnNPct$^qs>WPnOi;KJPt?5q5WV!CS;Ic-pK1Yicl_(aSvFO^2Fy*AXb&vT>LXQF3 zAcl_Vg@pyFwU*YrLqa^RB~gBw>r7STlETWkC@R{vu4P;t_UDF)NC^1yo>P%6sW4BK zO=69_2kf6a3Yp04oj63ep&X@<@uxr*5Dw2$io3P~Lcx~b-VJCGX(H^-B`eOT}n>|Q98v;Wy{3gPiw?zaR#p7S&#q;OnX6Y|< zEbqeipW0#FJuPW!P@~A&V5^vJOL*~LBbO1?iv?$$2aV}u`pHi7y%C!}!rK%(4;wuV z0P;{!#kX+dsIyNbiDp;a-d-880pE&jyffnttCJsmWOn@}e-&GL)E3L#Xa4}yhb=P- z=$>3Ry#34MF|iFtTPW(IqRS$ObKy8ZO3SCIHSmaEG-}ps^4A;yM~z9Ud05iBE$as` z?mQ4u{Zjcy3{j4`l{Zov^DK#7xp8GI=XbVL7I+?v=saxB^b=eRUkkAN!L0z%G9}v) z2%s`vGK2%>8R%I^eS!t0P3UX+;lLPmjyd{k7}XLw|7&RxP;yG-bAoI;I#Il6*vv}5)7+G`LlF;f1t?}k z)1WnrHx@y)ikKHW_dRbU`Z5cfb3z}HMG2{#2&7*nCXo1+vwcEal;d6M_^LyYtrQNX zW?~=S-VT!AgHFjwh9(o&onmXDEc=(vUo-lR9gQJUPpZ&SxuL({_b$ymI(A67FHUL7 z58w9pSbDW&*OJ@DP9JgmjOy~7XCj@P*$K3hAm=8wV>!h??hcFtX-|qDDuU#IkgN&Q zzNb5#QW{x6)$OBHD*)n;a23d6{el2V(bHL4SrGU@5YaO(yV`T7--sx1z3;gy5F+NR z%;dKN7q@QJmWB~aA0hzECxwq@39OP^yCryN&VsgJ^;K-K!HnjlA>n;Lrp=s@A_T} zFA8E(O{76g>Z{Ecb3Z6+3;PJ0g(*490{x-`s+cV?R0?a-Tv!X{%*GV!EAfXyRvl+O z3#Wh)>_7%#NB0uq;E60aFy_R(u^{vEDb!J^|2WE*WymAb*0Qva7#OS-Cjhlp>3g15DZg~M@CBHpyYSw#cCxZ+Xt;PUgC zurwVZWl@xsBg2cV44tXssoMe(4sKi^({z&AA(2C0Jf-uYjaXg{!-!j=Ryk3z(t3 z6As}+wPjE>3(`cF-dR!ltKr&AQIk_vHE?lp2DzWv`kZINSlrvoQ~6UM+@}p-vZs7Q zdv8R0?J9KabL?XQDT{`}yRCO^7ZsTit4bCH%JeD@sHl`M6cDHKuJ>UTw_P2!va@}g zd|{WOc>?C_8cqj9a#CdU-mvz74A;mfpxnW@=ple;$p2M#1LBuw$30HZnWVpFrV)>b zcn3HhIq6$(X!rrrZS%{|EzBLu>)71z%`%*~30}dd^&Jx>>!8d93~%CO~s1B?J9rwK#K z&0~JP7NwHD3M)=>;V|bnN)c0-2iDC>Kz(L-h{(x2_HWy&lb~3qWJ;$k%G)iNq^b*d z>4l$+@bvoiRx+A(XvvhfK$c=lpO(Q{Jm)r>pBg1s7SnN|(Y2Tss?H0u^7;x_;&Kj2 zm+IEahLQg7mdf>je}<~On+5op zU+Yz{{}N89Va8DN4Gr4;{#)~5hrX@1JYY*yN1>-EnkjyEJ z=K5~h5*Z4v`O|?Q4r1kXcbTP~jt9fOdCuIW>+f;-m0GHYMF*S{)IKW`6dl@+Kc6cU6)}M{EoPZ&;VgJ*^A6|{sELn`&JJ|8b1fo`xq+^muceSkDUw~< zKfwJ82w4x(R~`vlpPG&@l=AOu+61LnpOy@O_@T>8IGW5*fQ|p_%w)^RgWQaA-uevL zGTJ|{q~!;va6q@ycN)c$4mdP_$3^{V38{SV_x@5r6jnnN zTgze9{tsK^f!4%P0F0H=$$U-*Ud~&i@=lcn79&pUjT`z`^6e)5N)+%?Z6CE4c$E4@ zm{&6s|CQov&*d;y_e*U2a1y>j zc~y0RUBYM{{m;=wU2V9_Y^I5jSRu$L%zj+}n~z@d4H8BPXzKm|&D=laR4gie3L;F( z8*YXmBSrQ`=EMG*%tryhA}as}bFQ#J&M^dykoX?{q8G+=Te?Qke(&c3lT9k?^+u{A z^K$25fg;$BO#-z2X*rqw*-C+oa|GU;q(`B5KK4Y}9yh2}|0F(T%M^rqaPmVYnaiKO z1otAhtpA*=N3pI(l1<)+ENxopn&UtQAK{^ojURgpMM@R6)+4CupWCZe*%iX%)#{yd z0YZ=KJ(QXQfG2iGxoW!sMlb#|j2J~zzqTo43109wi7`=A324NRx0u!9> zk$TZVjQZz3vYTGIVgGMUXmD9*3@h7{O=zdLq#>6;6hH&ovFFjy{d^<*LoRU*TI*nc zaT95iWXtaG&KK5rHNF3Sv$f)XQTchcz3WrFLrvLrlF9EpAx8v$+mw>!17toeSxg*! zWrm+Gi5jy&H(ehe*G~&OZUsbwKFGeG zvu4$u$LNgDsj+16u3!Vu_oPH%h&p7BE%JFoF4s>iPW7>JoSZD5ge9+JaA|D11LPbT zT>yl`Ot?Ml;R00b^N-j+n439FafSw>uh_e`maM(lk7S^(q-#Zmc+k}i+`w*4<9L#* zxt_YcCuBNNXkdITescTVW{jdEK36*5B-^LCj|)T`igOdbv>6@iN}#FTxYIHIT4mP$ zL_O3$ab3gmVnBKdL?R6!W4_#xBWhKTFTyH>f+z%0wlIK<9@6w5zt&F$Xdw%p3~91} zi>4p*b2&h)HJ=+WE_CH+`X>++5zM(2@(L(($ulz0Z9CR7k5TsqTo7fH^6R)MBz03o z06x{xKcCR}KJsrMk9Xdy^7gQ7S*3< z9e<)3_n-CY1i8bg?Kw4KQa#>)yQTA*D@ac{DE~7+`8TN-(;D$lwo@@!*sK@s%pnss zCnYuJ^1LOq9j8IQu|zvSzKlG360r|^VNT=TI%0g~^0B$uwAU)_ zY`=U4#imKxbo)?NU#EgFfRO_x205+OxiVL-;yZ7DY1zOI(7>!S&!Xpp#DQa;%DYQj z{t=ae$aSUc|F{q~Zx_OILazvs!vPyFDB9=nprZPb|Rbq)YKCpyW@=Y9I) z00q8VQe(a#{m*-A7)R5iEeVkFy;5sd2QU)@p3TgIys8qi zLuvgBtlt^oY_G$izCB%$gXIwyPdFgd(+f;56CeMJIJfDIEbyxBDJAt7Lj`UwF}Zr^ zqsz76Tz<_5NM_OnIfIm}?@jXrk&^_cB@u|nb@F3|Lho-Rx_oJu5FtkGifd;6wJ5vc zvgxE76n^~Qgr>?@qSlk7yyLRRTBH z#uEVby0%A{B(2syn7T2mV`=mCb=m86o-?}M=IQ=BRv_XTnM1fLk_1Ig^q{}DrIEt% z7Wcgp9m=A8={KG|j@&C$hM$#>42t5q1A&yTQ%xxvBvh`jtVIiju@=>csBrMEKHpf% zeV&;gy*0{amk2A&+?b5xW-VR!vW_}#vaHNcwXCF1b@`srY1)c*L5rN`pTR0AAB4=~g90;f3NAI*u}=i@H+CQFTQ-cAdF&#ZPe!!VS}Q(4Q)|+~ z_><|}sGnU)QZS0Weq>mEGC^X;Jouh`rlAAAfI154f+-tF$@g^89ZTEbU~I}a#ZhSu zYTS+kc}+FYcKSl{!h##3YyR;JMX`?#AOQ#0bMXUe3~+J`5+EI8!%r3f!*#)fj9x_F zIZS*NILhb;$?jL(YK>%vkK@mM1Y%WqLin|9*}y{sY@oQ6W&;@f9L;%oAxi7eSA}K~ z{V=KBa@iyxQJ|wfZ!F*dsxl7Imoi|W^0U%L7Y-P}jeq%55rZm(e^q_S$3y)*O=(qcE=ko*|@XzfcJTND|48nhMsl4B*@%ep(i4p?6?~eE#zyLgPL)At; z15o!c&~q;WO7K)D?5_o2D-~0Ccm*ghu)c0$_sKE=Uj}`WlSfw}%=OlZG9)38M-cnc zIX+mQ@Hok>LjWiuCUd{yAUnL#{jrGyhMtHQxsaF4m-}}lat8-c`Fxn{s)hjSA_(rg z;m~)GNvMeU&~U(^g_>)}GQo6~5bW(sq1F7iM>uc$lEII>`tPx;r6 z26eQK#UYjC&Yxin?g@bWm@~qwDW9r-**fo4fW5Z5^|0X610qvCO_%D&5f|G`Qn)F7 z66rJn$};9|3gGEwvy~zl|Hf5*5bG(;lKq!kt0NU`3I4Tgz zqDw0D&oSVW5n$|DT6U0_m44$&$lG`EgTv6DR>OHnx4aF?b{WBd0e|$!hxu~boz&Hu zq7E&LsVxQ7LHPD(BPo5ztX2kIspJ)gbWtz^Alc?_$y*;6xzs(Z;m30BKb3DVO17-6 z!TLUP2L(7tRyI^gNaCF;82bNw3XVWUdkaHc)igl6Lvir7u|#_p?YQ0^tJUdF8}+0)wF)j9VJ69iyv6u}J`8{I;q#>kURRDdt9M;I^-_uMek z>RVWLt^3IPU>FI^0itc&@h_CPXls;!Y<;peBna_Ki1b!JTH%~t99KFIX)$UnkqU9V zn9TJhxlBC){qKhdO8Q&RK?P8I)^4j;(Ol_3uI5(_{Ri@H?~4@t$E$hb zs?W~dNGy^jzvI#+u)iG1NM;21AuOOF2zpMyEA>Ns3z&q|E3Klfay9p0WOIAfuF*-u zN%JWg?aN4j=khd0y*D}k9T@r51Su?GyJa13kYBJoV(Y=YdS`z*ywhMMN^kHi)acpd z>ow;OeL6KM?PgAn)#b5sr&ibZQoHvgtb((dZLtLM+Q~2x=(>(A`REa<+^WS(I%>&j zBV!hsw5NPy7c#p+dQ;fBQW8apr+tk3;k@HD$-8Lta6hE$`(dNM)6p@MdWL3XFz$>} zxb3^!=29o}>3$~+m4>0P!$|np&^xt)2wN`|nY_WE@VQn25^;Msy=F|;?Kn^O(>Hc# z3m%P>q(8?B(Q6v=tWihmex}7LrG2U?89B1Ht4xT9n`Q5a-!(vXg{?V*sh&gazpE2T zrlp6ckA7`)b|xh}+wstP7goPj6xK(DjAUu_yn3Tc#P!Fx{-ux^L)s*12!Cp_44)`A zf9hdXlT}hIk-Ac`Kh6NHx?Ui0k3kE*@`IV@Q78x>c_z|-Ak@7tONb5 zBX*Y8@k@Koajn98Kht388Ce>xG%9=?o4(Ld9?{B=9sBgU-R^R}uhuVPR3tb0-d-M2 zzpL2P*TDZL-b`Ta#R|o4ee_~|Dfm%^^Ukk1GOk5nb17AsL z>hsxS)2NDCUmHz@026vUwQ9_abM>swVR?#n4pOWXV{+ej)n3@{_;*&+G@HZ++DBrs zCA`5rYR2WeECI6^sUS$EmiBBsxPztOr4*5Os)oY^^mwAlv>j-gc5o2Gs!0M!C9EPJ z^SoE?S*0l}ea0yg%WX)5s9N9%wkaqTwKdlm5x1>BwpGx8Igjd%2^eUoUFr= z2;5oJ@bOq|j6m|8>Om9ylhHf9QfF<@bC&F(lhjmh9N!P)IDw@Pc4^(EoxESXG_5k| z!sXk^J649Mn0Da5Qw_0_;@phR4@2aXV77uYm{kpmq}w1RwW}5G)hVEsHkM zobusrVcg;F9!~kvrcRDJcH<>>Qx4Ja0I+X@PqQ#54incnsK0Q|rJKq)-2$a!%0tBs zAy>X;S|MVBqR~Wjy5Ud7I*_^bbSgZ445SqYES(`v7RVI6pBow}fgJKDEs>(CUoEH> zGqJwUq&Ai0U__PqsEMw;rflO?`6KXaoYr+MsB;^T|L^rdxha zCJ~#8V%eVhF^4i+oCy!jowbvkVWJ#O4HxDV_Y@fCw7nSnT;(nJHhk*Jy|jEB zX@o{&Q837uy^)am_;M7|V#Mpo!Rtv=c>JwuCLrcSvP+(t&oeCyXXe$frb289<FGB5q?mT?HGy=daYCR-R?si*<= zPP>Y3xyE{*@d}>v#rY+DBF6>(~n&6602mDwI0 zMEG8*)Haq@kI%6V!Fquvij+(&Lb);I^S1vi|clO z?fg+#f_jCNeoU&mma?1K#p98*S&8~waj%Nu)*(mhg&n%Lw}XgiLUS3HlH<@Qnd{WdPYkf}p|_Ei9+`x|9$8 zGdt|S8u0`V2BE+RIky73=8+qEwUr4|sHRtIhBT0(mXFxkQtt}6oQW6e?ac6M{OjBI zs#k9!#W-PPqR5}8l@U9_R=*prOiT?i1Zw<*tV%;e${Nvh#Uc<)qQ8G%q>Rbtt^FeA zWBY^Ad-QzTBTuA;IK@<4993 zr4lFX}*QSBdk{E0EvWnblt(1w78j zO)J0n4x`ve$ZUoocB{_IMkDi-k zD;Q>iH^KGj;^1}Q+EEg{aREJmu#=96tTXe_- ziS>H~XL#h_slkitlG4aWx>EfZvq{4WNP{h()miCcxF0jo zJ$C%ngiF1y%KM!P-7$FmXmPEtKC+E{@rnb40OnPiy^-AW%Q`k>R4<;H)qUqym({^d zkMOYW(?l?6i9)QV3|v+<35G0evj5_ArE6iF(*OQ_*Pn$0xx@!UmP@ynM?D5?^>V>+ z$K*O3&t8v2+5tQLJw~dU@#~V)J@k9rAgN!KY3THG1cOk11zX4b;yOyZ3Bkf3i{k;| zebmpwr(FF{R}gV>l!n#P~4Ktp4s?g)g z%{*v2>nKxbhCgZs&->!2qwpYS-<}r-Fz{6G>aoxvrrz|f8M4Gnnatq=RzNrV_cX|) zwWImkHkUb-R!Hb_;tes(gX?heOpSGG6RDwZP^v0@#D^hIS8M79Un>w=j)hGZgx!K63*0U8-M zxV<-7h3F?#KG^IKXAme&5$}-1f_=HUW3*I>G4h)#6YG(M0*Z5~0aexs zz<6z#fW`8+PoVm=?zna53bQZ6X3H#Mw~iIuIc*n9!I)Sz!k~ku`Uo^m{cpG~*)p}V z_`d*V=|m9Pe1p#1@r>T=(&|E%tFHwuPL3&u`{h0@BjDPV;k%ydt9o3W=~PfxDZ@Im z+y)*bR%+g1Qe0a7^2=?;xgiF48QF}@aMJS`E~jnD(8$H%;2~1%>bo!>#zLUV^ zRn-e0-N`5>S5ds^t(uReYBv}J40}PiOKUYdmy3sMvXYb+G)EXh)JM&M5#!b#yNp6t83ce616P*4mlb``BbLiRCA+(kLdF*?9 z@0f+YHfh}{sp=ks{%+l~5p=}7(a%(mZ714h3uL!>{W!r6^6&wx)RYHFzbgun19|+V zbz~)ekva4sO0tWhj5>LVjkqEh?ppEd1jB`R*RWDHowO1jFl$&dl^FaZw@$q0I;jRR zzHv%heqJ<^Dt4(Zi2|{&DK4bjAbE;Sqk)k>>GvG7u`z}uo~CwSJ>oe+IUr zkbSH|)Evu}>B3*?IGrefg8-L(iEteLjx?daTl6xSg7wuFXIjJWt%F{6L(etC0_teZ zKmJVC@6^6*Qes8s19dcHk_C1!Z58f?a=}TBpE4$WRX8dh(6YrUbVqUm+f3BKE^Mg5 zmqLyg*&L$kl`=C`jWT_)LN6>%LWTT4K5PxMHmq+VhMcAB#kSS{=(O~qk#jkasq};x z=*L&Pqq8l5J!y6L-hKQa?@QbLCl9(EDC_s$Cs7}^!nDTcIZ<4g2|ht#0KGSq;{VY7 zXJToBVZwD6f*w63lUGh^(K%{;-!1g`V}CJ(DBgnSss@q2DGY`JktA>D8MKVaS%(Zv zCV4MFm76J1fRJnq+MaH_C5)8lidEvn~I zAbw5VKBoV*J?J|~O^FTb;n5q~7|0}FBH|5jKj?@EMwuKK6k=fD>rG)3!b}#*NntXt z+TbMKfC3~bF6x}f$Tz3332+J>U@^jlfizk_@plf)Rh2fP16@g+zPO{b!ik`Vdc=1B zE=!2w;y6S=CR_XyAXpBx1DjM(<&bJ0?EMAhe7i0}zJqpgP=xULAU@}`Lt4XD94Rh4 zDDBpT1~u?Os(;oadWt4Pf?6NA@`P!a=w;pD@S_EDV{Shc22J*&aj*ox+afkbGF-y1 z3bMGZGQltIiQ`IX0PYZr+peZdVqb1|!mA!vh6OavnD5P<*~6(h@XJ9cnueF|p$`$< zA4tu=x0ZoMnn@63-)GBc@_C>7h=4NjL{z(I0qL&XGg!Q}3k$2gCtp4rF5cE~k{YHw zNx#IJe|@2>fj$^~l*E^NFhlkOCyce>!bj^=sM4xk#m6bseaR&*SuVJ#g zvcp9ePsHt+CY_8RyQB~o z{Shh2{VeHlyLOoQLQS(Y%NECUy69YU@jd9g!=1gSfT(2M^4jeh;cI&i%8!<7ds0ja zd1vuQN{y3JAiw=Ji*%zeNnO9}*akECVq1ezf{m_aa(3!zP<-o1ik;)!UY2xWjVT_x z6(%P5t5QIW4_Ne*pD(%^q@k4s-#(b4z`t1lObU|xBfIDL&SJN_cnAJc`@;$20Tg&h zuU96)C&Ri3*IMO$HL95?^C{`f-CzWw&q>+Y1ocxYIeWxRKYjVH%!3H?vFL z#k31!md{-gvFa`1TBK86nw&;N`*2YUWAv-)l1gtTu_~5C^W9j8ms-xncdz;F$t$vp z)x#5a=bbfmaTMC!pmc}R@3$CGr5u~D3M3m{z@q5j#OUK^4DJtjHl??8Ee&4Vf61ag zf+NvXv=v2ZFuPNINhCgA#Y-}tEPLAM3;&g-;6iwk1qB9O3e#z}J-O-i8hiTZ`_%bh zwqpoW0bM%VPB{H9-vW-d6^VI*NpVprcrKPYXp9TAf$ftzkG=b@|-X64o!j z-+>#VY0f4(?~sPs`YSaHj;`V>N+#qu8<$df>cr^>@B7XWc=#AnF!1x2ugqi#U4?`p zxg*H8NOaL4!Ij1H_g?DA=$W{x5aM!OmJButNXe`pt;6|)$ZF`|^YnUeXC>7Nh7%nU zt^J7;I+_M!o1-f3bFzjEW{?(C3w(lJ`6aVI>a2K<^`Q%EHkT-b?>mGa z!<@!jW1v>aCNgMy<{EqX-4cr2g_V9Hp@Ch{(154lRvU*_cm2GA=8u2n%t8&M+Fl7+ zqLra}CE-D*AM3QB zH^+mVRrOCN?a*4gXT6eh#2aC_`-Gh^z2M38SXlxMY9~LpNv3Va=;MB+rl}f^KW{RG zqGPmjsDNWVnEn1s3jS+sv2Ze-Wfn%=v@)Iw|6FMukHf}CaK^>04GuMNn!@A`8J7b# zO}Op^iXN#icOl9&>t9z(NBGY;aryYc8B0Xw&eF9}{M&+(k1tw4i7@?0fp|!H_klU*s?8C$%9+G6O zy*Ag<+4bP26!aAVxrArz+svEd)9(%m74Lwat69DG*M9o71l}SCVV9vhc!1YNjAJv< z@uc5l3=|aR&;Ar7D%@VV{w`hq8)W-kw+FY^=Q75C_3{_L4 zp8abO`Kbdw_UkL5_PoErt_m6apVvi#xn{amCGiCv>4EJ7$?%Yom_n+r3&pBI!$j~0 z-1D#XV$kNnd@@eDPY+naLW-`Z>hjU0M{GIRC}bN2jipvG^d9 z4D7JD9ICEPKu^yOUOA#@*P$2cgzU(WIogYNq_YJDr$b#Reu;DHn6tA&!V5Z?ZDNHz zEuO(eud{ZezZM@UyjYe5$tu(zF;}`4SK}BM+Rd;u6BFh5e4KLVO}FV+BGb+2%JAa9 zJZye1xF;c9$xEix7_<25C(5NnU@47hXvg72vh~+uzd*PRl`KW z`z%?SCaj~(+=S;5xwjHm*go9M@50xP2c6t+7u=cEEwM#lDeshsewRs4Zhc05fU8h& z1Ik?1Q6i^B2nl$ICS^ojWPq3K4#{+JC8y7a?>_mzIHPuUXNCl#Lox>|nsuaOMjm2c{_TBVtaas=emJdKg24{xod1b1!|> z@a@ac^^;YQwMzZ*ht&+6Us9znL+)RWHwhuJWZCVFBhb}K3;66lX_zFhw|_15N_wSN z(2q-oNoO`GYr05ptD zSP@p&QesuU@K=Z9B;9m8neO;Zume@7eN`L<6Y=DT+o;{=h4%!0Z_{Gm#c7%OiO+=={g7g!x4e0%qfLTY-XG_jI~KyM$?U%%D=PVLTU5p* zUTZoyr$JMxxVrwdAD&^>C9bk9dq^$LN#!_s`#5c2xBeCwG6pn%Eq_`r>{`8}z_Xx1i zA+7>`i{wQHYL$IqChK{B2E)s{)JH%;!fC0}gYm=Uj0q|33=JT0ylHNQYskRssZxDc z8rVO*e@X>8*GwYzzMK@OU`ow3>)@>oC+XytAWNAKu&|;418tJ7Dhv)duqp^angS#% zh41}h>rcBsz)a2|1Pbb#G(4u<0)1%rnG*|e)dG7<8P z0IXGCKau&j%_dG!fkrb%ZBTaVzZwa*K{D};R^$fAwf;IDn9>Om)<4%kCkcrqdO`YU z#b!~sWyf|AAMWlgrEC85a>`xl9~1YjU-?%<^Y|bTa?Sv#^V%YT;H+;UUH|hbd_9gC-U6*QI+*<2lAVi(6@B;{)u@T+Q{&-3pA@ zU*C^^tnC@2Mo0AB9c)d8^`r(AX$eR${`RpWzTKLD8#J^ym8ePol8OK9po2Q6t)}aC z@$-1kP*iQm$|)UDZE}~yvQ`xM#*EG~Gr|{aEOS5kE&dA=jFjc&rWPWRlT|TNjo?jJ zm*vwElT)gyUz2xbJW|CmFa2#O+@9>VkGnN2R_#*95(3j+3kVU)x|>GrdAA%r7`3RP ze_Hrj>B6t$9XBO@pf%xsDApK6YXGYqPNgk6p&tR4ZM6%Vx=__voWuq{s$1a^_0ZVy z;U9_bB8R1^Em~V6I(*qVuckizlDhL?=4NT6N4@)?YTD+*h1&Y&$uZ8YF`A>r<`k6v zWT|GHOA_2fLm!V;=-cijjHED1t=HDW;3`6|#(Q%W+tu|tFeAKg;;xR=6n-4;%h9{K zt@3^No^#f;QwLLLHnLeO=fBPFcYS)UNgnGVGxKM1T8@;p;pvx@@8Q${zMvkdw7;VMbuH~lX?E-5U z)K3G3r3GMp{&>`oeKBDn!YwPbl5=fk*}J2Zirn zl_rC4b}|#c-C!w$&45Bp4I={QL|WA67dQU|l0cURE_lU|w{q5ztV!q4c@+4O39lPT z>Z4wH#N+y{Lu%0>{~z|=Yi~E=aOm3~6h#ysVk;V>pSLTw#sJYuRzm(@`&5qU)}oqG zJ;J~&dB>MJ{JI{23U%W|TcEYLzvE}=OksDtE9pcyLr8xZ1W4I*-Oa2QowFx zd|dfma#FB-pIh2Ig#T0QlWAcAZEY71&A2zas=IMH+<-O7hEU?%?ZsL1PS8s9F04!p z=M9O7X*AzRmX;cmL3{uC`f$(Pl26qka?h@Psg9UdB?(P~S!GlAP zpuyc8g1ftGf&}*raGHEsYkh0g-c{$t0n!$p%q2KKURaSR6S^U6zrpHd?zjRY;ElOpz z$RjDGaEDu*U%V537nNf9$gd)q9sJNHM_n)PoQg4sJlQwlxQuotVEyST)Y0U@jxp(t zEIwit&@1p&9zW*sX%M(lFsp3-a7?(T)YjHCv)nV$;()giPG6pVTGXjb0R0{!y@nuGChi}Cdd_@;-YxszQHq41`WNAL67)5a?q=XVNVY{(s)O>2&_y?te`AKe)Dqo^!2DawX6ycM9!EBzG zueLr#5+4?UgrXi6YNi?7nhIVKrB2GrAM?adSG?|ZJaeP2C8?2S2Zwcf+7*c0KUDXbPOQsy#Umwz8t&In6<3x0Yh7aXIVr z$L5s>C$&g;zlyP-(rv!qvzLf9v5G0T87Dg*e$6ZAzFzGVshuJS+*4)KU-n>`567(P zlc2S(o!rT_{b<1*l&?n>?poc+u3CsuO+`)Ik;oxLgAqo@l;34R6TSQ~fWNqrE2{=- zI6tv?B^KY32zW3I4ouv%oXfQn$V9<#7OC?4Fbwos+53<6?DX1C*MxZSSxVYYDq%w> z!LhA|t&G>42W6ZYxE_kvIfPh`h$SMg-Ld0OZzXV^m7}zwTvF6%O@Ukax{PVQBsg`9 zcoI^EsnzvlEBHv_2Y$6#d{Xe$ogNfRw<9>Hca|w2VyuRB@pNm@Qd4GJx=W{_yBM!A zS^RwT;f=9;e-N!>DD`J$Nvw?HU7irgLk0LMX6>9P-;Wbflvv8ix;;zh$W(CF=4@#O zDjE>jfodaGbf2>GL?D+EgaJy? z_7+jSSI6aTlxfg)-g|mYz*lUUDs35|ANfym0lt;=y`~tFR%}W3d+NSC=U$_~Y|xAj(mZh&SEH1*qV4AOBFx?vT8 z={{B6ie{o#bGam^F~@T0;7QO}(>EgxP#W>YqCkLiJLzKo>YOSf)P+HIsE8GOThhMNs_PyO>{0-n4V~WZPV50Qk*ZXjpw4KHsn*fuqt<1e6K0hV4L49yT6*B zY0`d~H+4JNSS=>xbBFJHJJ8}g#{Tg`1z_dN1*Dxe$pErt4isZkf+2DU1Edh3+H3ob zuUlGkl|~AH-wL^=-gcjpT0E2fY=xjbi7gz4% z!j}Ergo8cNPmutU`yu2hUF85$uJ@uZkvr!StQ6dq%nv2U`kF3IM9&E46 zD|`A@Ezi|l4>u*~2{w7lOUx+^R;*dUv*6PZ+H5IKuJ&5fyXH&Da0e;AmIki^;icJD z`r+Q&AJ|X|Hbv^!`ekE^s5eh#!X1k%Sylk72POm*RM)XilEfU)-1w!oE=YQCg2jtG zC2vEYpV`LjKGvZ;(bD_aMpU)}qR}esx%EkXDo*fCoK5EiML5}7x)^})OzTLu#M4CI zo1d5#g&bgMCLjpL5(vycO=J%Z-de(wX|hwDr-n}7-jJpCIvetMjBruJ98c$F;wu8U1dP!ZvQo|qA*wm zt=JpYzK^fSPu!Bo4G9viKH>afvVC*Eg67xk=ER(AP98;b5(a=?)?L|ZfZfLncN_jm zx}(hyqV#v+Xw+*fW{q#ZSOVpzaeYg|Q>FR&`kr&2B0#-K ztI!=QuZdz>Iu^OI7L}WA?>U&*yz2icO$rTl`1bQJ){Ix-3_&Y%-DjX7lkOdsA>E#} z3$R0sKnbhB(-4YlNvF09Lw1rt4a)jk<{16Yx_9zL9VOCbvT08rB~;vZ`jJ|w9iq9R z%D1v+Yl_ilqXtoTHZ=7Iqt>T6r*mDzZ>I-H;h{Fg?_D)Ky!k<$$1k(n1A-xkE zypdTc$CCE?HI7ts=n3X~8{fpOK4N+URg%|w`TbNzyVUMyh-zVNLk5wB9A{RD-Zviz z@e2JuJPC@D5MGF?*gQGaRoXR9stv7??yNmty zzTFjXPK*g(X?8yp;_yip3YyJFP0jbA&(tcy`&=l1A46zX zxz)fFZI>`^MTPfLcNLWf9uXc7ugPj}T$J=)mvwtXbo|{-Lfft7Qq(r~_s{*ub zE$+(O#R6s885Bs*Jz2f5ylOV_AEssvlGIl{>hh z@vmlphQQ@pS&b($@wNoFB~;03!czZA7<5bG`6+v*OcG~(i@*(B4t z$YTSyX(C3Gf&fxJcgJL1cRND;e0unPm2Cbr9pr-c_CB#&s(e(w8d`X}_Z{l*LfR!X z=^jp8tCw;{-=~qX_EAkwyhCB%Aqc|aPzyxWCr1+qoc$DJs{7H?M4nB~w@%VH=gu4~ z6NFgWU5R5Q&&U(3A*!@wJ7BewHjdB38G-p-v{Q|!)+>oq(=#D@Nx?*Y2r(*51&+Z) z{h)w-uo)SHB{WF6SkcPT4OJfMyUT82&;_MyU<7a@39`p6nZz0rp9B%IqS)?6wLEx9fjIP3`nzFCRB6 zN=cjRn2n!%LVlNbnFySi(F~ftYakbK)0-*nvu!@8(e11*tP*n=QQGmRZoIS+KFtn! zo%pRUk%Y;K8c(y>9V%Fvk!h0rcL{zH#yP}Sk0~eLVNbN&knrWYvy=av$&vQpLZg>t zA!8)oSK1LM>v!~$P8gYy@z^HS5jp0kIZ>9+`0d{9rJNGq41VCgp6bFLL1X#E!WtZ`8d*J zsarE6`}>K3?lwNeWp6fO@tcQR|M;^Xr)^K4ihh=ngyt?`Y?53&pbM%@PwdLx-GQ%e zT2$-WG`wnMjYGiiqCivRS!CH9UMkDcvUw@mVP1TXC(ux2n3^Wtq0JGMCFYvG`LXp8 z6W4-LJuIPY%1-e(Ryr>kaqZt81aEgwe0@G!8_&mZqbcWmdwF?3#UEoUHW1f<&E*y_}ybGHx5^W92>y#e)@*XHdZJyU-*y5aM%)R3H_6 zVpPTK>R^3^VhON;n)sL^9CKlgi?nwOHUg4W+%KR}+g+l+L-v*7esd%5k-RPO?9F^* zT)rJ{@&FJ+OFoc%;w$U;ax!nDi|Oq5jVquY7GwxBA^Kx#5#50VZhQ^49O)$>)hG!p zUi$^m+kW7sMFzq*z41S!UjXnL`O}7k3IJ=rPHZ4@01*{@R3EEw2xxT`i_?(x=U`6@ zTw5)P%^8NRK?6+jUm=jdU8*IEVHLnw5DZ!V9u>$W!j>@-Bcy(KWVlJg(k_<9}cFLOqE76~yWnc`?JgY5U407~r^>Ma0lyfar!QP{;pH zhCFX51_e-aLqomg(C9!WTL`WcwfFR0m)3+{FoCnMN^hwMfr_en6V6`~vi`PP5lIuF zm~{i%KLtoMqY8HoaXh;auoGh52n!h6^Iz(>vxY3u5p6vUUX}vJ$1Mz$2n0L??u>Z(3 z@8xxQ+Hyg5JWM%}{?GRKA%6@LV2{JDbc_N>85#$m+|PEGzrB;@F)RSMk!5LBa3z&s zIR7kp03k=0owT#}M;w2rNVYW|SJ%G>a}WW02v{)j@5GXyuEOa+ofvDsf>yI2!@u?V zqZXli9M>O_>Nz#opuqS%=iYxup9;6AmfGzi{$>{W`hMxYO#)U`Ok`q^_>f94a{u)q z40J)ktxyDXZV{J!0%l|ay3b!K&+#Ye@J@FU8-&&fOII}2>=2WULqO!&gPD7*Ig+lj z16+`*HNu)SsJAa#oCOLVdxVIF1XnfM^hX?s+{rrLJ?e<@8&LB_50~O6^rU@*z)kZq z;(gaHQHBO1AejZz1TZ5)nM97wX%;7+NY>n6ZGkHh;=N%2*`AWrEp@;2TimDQ z-C%$wT6Ag@8to?!My!f)I{ctsJY%b6Bwd+{+q7mEeW(!4T)cCJEemg2kb3kSruo#D zcDxg{C@)56DfjjR;l970T|=M(29=w+NAxh3WK}zV3F_OSx+%{pY$nk%1&R$kwWEh) zhlnbsN(FrdD&yRU-EGV&AcbncqLUMdq* z{?_oPPpTYs{0dD>4F69Bxb@&uzzN07Ug2p`!91+{*(Ah#w_A^p;~vvb9yX$qQv#rW zjM%DOcVoSkF@ufU)SvI(3aq8>QVfC>YT^spJ>|-R-T>ucXVUgA8*{7{_L~*Wj5nMs z`}L)0KjG^E(_8pPyOcx-FKSf}xA*>VF?(6E2iZS%J&?XHx;7_D)1V{G!XrS#%0qA% zQ9$^FkH@ZEW;d%SI z&R0yCFI2DQr;+AwRs-3H%S;x$biy%pPaUIfNI%8M{7fPAX z+Zzcf6v=>~5wD*@)HkbLZ3f;`Vn}Hx+?rT@_GD7;Vlwb6pKj-d%pz`3(C%kV|4gnz zW{i!G{kDqlA^gMvgXOy8gu}v7Mj;hCNf7b~;VUD$6bA6AT5SqsTRxk$pD1bw^WuS7 zeFShKnFClLjaANv`m6rufTcm2L>kX&HK#dQfPXtuOiRQHY#PwDQo%Rmm`=1ALdDOH zHRpv7x3e-=_S_HpIWmi5STpiz_Te+KJI-cMA*h*zvDQc>On1gi*H0tF?CK=2c1+GUX;%yqO{YpBCEy2H>4z3 z$u}sUYw~ARyDZHru~-%Boqbc)vUTyuUN{^%*=3mAZqAdya~6aP!H)h6Z&^PDj+FnT z=U&FvJoKcUeq2DtcO_#tu~q?;JiLBG#cn;-Y!s9rtUvswgzB8C%mU#xTL7SjYl4pt z-kHOwBVw*I>w}`PE(Ox!&Civy)C^OdDtA)p_nS679qFm9={(IMfbGNUz@)PoklMrV z5-FPGH}`M9Th9?OP8T+1fPmo8uXX%t+W{}O!LDk-K_qy*dOdRB7T+oeB`RHamI0vv zACq+W`U^hAYej+axJ1&oE9nZwgx1>7Y{|Z&RrTZ>BG}?B!4U6;xpt8}u}HMzuFK{n z9Up2^Ts5MI*fPzTw!@72z3Lo#m9lB;v#7+-UcfizhA+2dS_Z25tKfxG_9gA;1p|R5 zHc#smV0(hK@^O?@cl04ppuIoQMsv$-uf4os?5gksK zI^3Rvch1rk5r_UVY?IXytVv|?*K>NoPUV@Vm|D}m^QT;Ow8Ood8K;VQu?L$03dOTX z?ecQup=N5bkYAXdvNyC^Mc4@h6*nI7F9X@2m?y8&R=9}PacY$XN{*^Nt-`(=L%Jq) zlg8OhV%0#&z2+T9jTVLh@lYA3z~c;Y;Bhl(9qu(a90feH!usK++COX+=-${;;a>fa zwA&Co&n!K@2|dS98?ZfXX1&k8%!$^t-!s2Tw5+G$pLa}dU3dozkk5gAonoad=@w+x;efQovsAW*WQP^@NN)80<-QlLt1|e+J7X14+6^2?Y|5`8+vjjL0K=E zC|?+$>J@uXRuVHZZ{hxW10*C5P{lTjBAEVLtQ-jnkqDjx!Mg(3W*I*n<|(A56$t{dF5 z(fi{zJaXJj$dnqsKYvhTw(J9Dkwx%!V|iw*k7EVosK!M%ZqCk>g_V!x@VV=LM1nHv zow5Cuf`4s^r&?0P*(6|`Ehv{AwP|O3AwGW6@%O7~5}j3k#OPmDU62@3tH+_f-+ueB z^G|D7M&<;N}I7?-iP3* zHUP@VnlE$HGgMmna9;-|`t2ZBTI65A^Y(Wo%3W?Ho`G@qiM1rT z7IYrnDy?0z$>DqX&wdQFm4%b44k5mq{L!I4YWnQzDI3^)I`JPaAQznOz;E|YR75wW zM&J@H1iNHJ6p%d9!OymR;Mq=B?a1at;SU9>BH6n~i}lmX`S^z=R!ej0w^QhPAcR@v z%#-F6E!v-IGg~?wKJ-trXxGQc(lxi3`_G?Y~XQDC}1vfmG_C!>oG@qXKp7! zqAxUETjN`l<8%k#=WPUD`m)I;3^~z?)li9C;wl*5{UP*7X zZ*+4&lOcLJM&t(!PHHXQ*jm#9)jGry2c&1KrdlQi;8q$a@6p&~GA*4@);a*tpjT83 zLh>O+yU0$?Z-V2_UKDMJq3O)~vl*a~mtuZlu;lW336=I0^pVkZI0TXb&Zj5q3?4}$wejU&ahCSBvCdmOFd08>ea%tc_bJIyW}!uR zrBkHN5`2OHg{EnuM$1*#thQF78Zx`=cy(q>^$XJdp&~;qPUcLKIT0;d}sfpp)VShAWKb?r;@5fB^ zyd=O$P4S)=$q&ii%SgTg58Qotr~8yreT>=0Nhe%R2MUh8Jw?q|ZhyM>t$|QE?0#Z= zX1(Ee-Z4{-eqSjqMq-IOIRi~UB6;~g2`*xemluOR+sE1Gr@^dxPj4!(~g_{D9md=gVZ1pmG$Z0G=5i%Sh7Mkt-o=C*#BtP9~^7L_lI;*RW{tXmp*hq2p zPEKB(n>t-qcP821q~J5Ga2c#8^v(eICxx|{&jEvLId@|%ywo`Gp~Hks@Y5j9&n1y7 zDPDq*%5R-W;dQMPGt2j*kB1Hy#+f?vxYl5AA3L?m;^cgLAKud+^+L2zoxA;wtDa>F zB}Z+jI19Q%7mjZVC%#mIiw?2YtS zg6XD7KC4I)iYniY4#`rNQ>PH7s4$HV?;lLg?5MuB8sS;c8P)%ItwS^5N-MjL(zi~M zoci?(l{X#1HbypyR$!N+G&+1-rxgOFQAbTDL5waXU18ssXIXtA)JUR-ryqcq*@&=L zxtUCV!OJF;Kk(8SmaG20f;sN}y@@XN{{7RwH02BN7%%^k8|R>#N7c06r&p4lFgi%x zd0jJ!k;leYv~A=-{Gjo%Vs>AyrgJGpHb40q(Be&CQcW0cG;w{AuC_mMLF+0>0Dqt+#~v=CI#@t0E}^A>rWMM z?s?*?kYn}97-LR^vGg-OJ&X#k4SFV-N>QmlZ&0*(ae)?zVQ%L=6(QHI$nPvfVQoZZ zvqo$D)+3$mS&Aq>kSf4p)>vuJSkKEV)FweKqq?{D*-kc z$unYzU_#89sehYmy!scNgdGAT<*E&!2PUBNrea zSXEqFB=)=%C2i8Bc9kryi~;5GdP)SFG~8xUkdReILX`{>#?<~EMdW5Ldhqf4gazpN z+b!rmmY$M7=PC@YDQc(>h^ZWUSJpoZx}MhKYb?`2Qy{B>Ur_}3#x^4En9J7=!pmv5IPXS z705un3Crs6BN!lh!6@w6B*0=H>s6_Eg~T4V=!W7B`Q|JDcNcC3{Al8C^?n<=?|~ z&|zI^>__Z~>7jYOVhKGIg}iO;E|iXQnHDdP|0uo6YDc*%&DyhFJk)z;b3L$rPT+Jg zy*BPq8=`bVNSgX|fd6$cg2(FuZ(uKJ$-;jl%iz%C8(1e6ET|a+T!P%FF4GT((*VHY z_q=6yFVe|b30if&88d|VHt&hoDSUvSJX;iF{Z$Y-kn`C2+N z#_c#q!#E2=5+f9lGf$pqIX~b1_H~sU-cJ+CzI8vhfFj886AJU<^y_jHp#S0$7rB@A zO(}T1jePUu_69Iq2inNN)GtyfJJMgvB&bV~_zJqm#M`od#fBQ&VfG_9Z2O#%Hb{xg zTrcR^D$qy8Uh-a3m=BmU;_yX)`s{FZyXp=3MZ)R94@+uvd>ucUxwKF67u)x$3xOA8oGX(${(|pMF%)ye9yN-EIambm(Jc7#s$VKxi zLLO(RmC8 z$qDekMhBPxYtl#4IV;zy=1aDI#pHpw;g+9ZQ@f4+;v%Pab-Z?{PztJiz7cXSOL3ca z-fW8O%XuDGpF=oO0|px{deRKX>oU2>-ajLo2D@$C&$Tz}h@en-zL(RDWAVIf@$ljm z{4$1*q8q!cE}o`%Ja*QMV=(NHY(5zIz9>>QlG;RtzuEv#W>3G6B3M0<^5)L|de+zq z61yt*ONHyYhfkiLjXKiz@#ij7p?=bX+LU?wwx^V>|4@ILW_R>pSi z%D~q*#zz~Q=TV_0M-*~wg!>sPeXktdzb5~x)x>Q`w)-BTmM^A)R)ZH1V=E3|Rr4&8 zc&OoztDD=C$JyTqzcpPTGP=^g7a($|8aYqXe#oom%h<8|gdg2-^C7`9X6f-pqiH93 zNzc4B#9>7W8n_VL&8dpS!sFCP9GYQ8$vbNv8kFXB_@%-3g~ey1BHYW~mftV(loK_X zbaydiTtofKP3DxB?+~l#RZvjfxrJ{QLp57QZ%oY!($(vW6c>KUr z7FhN#vY_r*JmH5am!P!VD7$e5p(6Q!MK`S&)?TCn#M0)?LHY8Zx1+-=yAs!)@)axf zV(CCHm7%!NE&(RcRJV(p1vm3~TuYsY`KV)+9&@d-K}4mMKw=h|BGS_C%O;r#q6aN1 zmlxZjE~sb@l6EB#`MlPGS~?vWpTcHWd0L7T&Ph53=*^$fzAUTyGOaV2RdcyrTQC80 zG{R+3)NQ}}+1ZuJ&Ls}e*UKoxUga{tcv4kW5vON|JohIXgvr7@WkrzT_xRgH4U2kB z;%GT*1TFDS5R%K8O7^{K?<2g%B1J&Ov_ZrEsMtHL`db#!PrBM=n3<*+D+|?Rx|+FB zM_$zj-)jra!&(1f39}EE%hkqD%z7`E9_XSTm`fqxDzB%FyzgX>ppTNG+5OfZ)>9z) zb8-{Rp{ZS-LpMW@o&+sF9aD<~G3FU2GR*rXvMS1MFa94qbwFi*3~Ak0Esh@$mRG?m zCzMJUYI$Rkajyvn%=WJ)BU;+30#9b?e1{tjRjN`1WTX~AA5C8N6XZLJU_;{CupyMz zj~f=XE#IY&3ka^X`7@z!HsXEJ;fWlz!bc1_ttx|W%nEQ`|JpI~MZw1c?#b9rSC=qP_;{HX4&s_`9Y$?qt3|JOh%8WlQ+?r!mhe=9EOHY}Y z#ZIIWlMXb^yta?ISR#;sQfgvw;)$^H+T(Z3XLek}f^E#E64fSarZa89{H~6u(U>hs z*Z(crMv7JD*qNt-O72rdaWD8MoHl)ljA2_NVU0=Cd(cl&pxTy$DzVc>UKyjxCn|k7 z@nFu~Gn`Gb=iD*$wHoNU7feoYt~^qpPrnoFU-xpO#Hm4`>-ObHPPp>*7GcG}?jJ>S z$O`9!i#YNi?e1ym&s_;f|17pAqz7MmHEt|lBdaZe$DT%&e+Urg0g1+RftPz89|K;f zeXbhb6i9*b_VH-Am0D7$P8uF5(lHmjEDtLAcACiG)PSS8wdeA_ z<^I*;Jq=J^@5sL*Uu4F@zt6P6Wj|J^qgviPUIIBa(asH;k=Hbn`_S%UR898!R!iK@ z2Ti8bOVI9f=W%;G4FiIQmZ6p#dLMhQyYXssgi@x`Jc8jOD_$BcMnPIb(GX2Oe=Ydo zjk>lpvl(XUtaud=Pei~^@6LUpMyU}k(*oa{tHCG4UMn#*V%;02X}oo|{8`hzM&kA> zE81J95NyyjsP`40^cwfKLKV`Jq2B#KJ1P>@r3v~9(A;&hT@S`I)px&5ode)Gv5RC| z1*BgOog`}F(PzLE2%gp`IOZOLVQ#28(gsZ5K2K2}CBgSL7ycyQF;1>2`E?+er4}8; zO|cQaqK*r)t4)Ky7q~*NJ+Cy6#S@ zASa-y<}l`D%MoA>5Bb>E1EW*BY|xZ^g`3?5=(GmGQfon1*_Upm&(PS@yig&ac{9VR zD(2#VfA$i$3}4|hIcyUAVs;XRniapW{$b*Jw3!d8x@}8VTY+p#*Y2-*3|ooV){$Kf zOPVOJp!lmFF?nsL*XW^2Aw5Z&#vAIQREe6n;Y~4lH#eJtn)GqRH3PBqUkO*DjDVRH zub?K8Icc{ApJ!6GJy=YF0H*q%oo`nw`GfqL?l~Mirw_<&l~=en2x%UiiB!9J(X9P`I*S0Hn7{NzC5^YYhxa`}#VTELojRWaZtzk=s{! z%f!Bx&*ZiT+S~?+sM#2G4NCok-A?{Es?5j+9am=K)<6DxB=F5MA^ro9jS!KI4q5(v zGQ#tF^nPlECrl*#1xj9ty4sfe^gPFJXSDtrM%7YxGL{_z0#Z>%LR6KZRkS$$Z{6@9 zm*rfRC}52fm9i1Z1h^J|4LtqR)+Yc4;&8YOzT&?8HfK;tWkW3N>@S+gA&y&*`0OG7 zqbYBbO=j}|qw{U=jN=AZm5;;4sO1y0ngGUsk?qSWq-#Ud#2S@U`lR!U zv_&yR$$6@Tl5<^AKJ>k398Moj3LUGv1(v(JoPeMxu{DyeJMU281AI~6$Fe3KU?1Y+ z{u&MQ{dwGv-S*2Kx%L&#-ER{1ISnY9PS4`^J>W+^zg@e26Z!7Lr+@tOL*H-L`3?Wz z`apu{fu~>Uy-2vOA^LaU-RlA&-iG=SWdGUr_itP+QjfhE5p@6bkEhdU&@a!})&u@I zvY+kA9$y!@7jyT2j9|G6<5=Mxf<`219)j-qUp4=4y`VpO2zu?SpU6sJeH}qJSx(oI zePulnw13wS`C785AglGXj}wjPe=PF%sZe0w^+y)EQVPHWZr*qA`m;V~k--b%bVJb1 zQ0*VG+uw2#c{`G;)Uqu9!=CSGhr;s`@U7V;#yS{DFe2qGg3;ibe-qPNEXT>pQ3 z_^S1kiUjoNAK(6*7M{RUs)4)R8pm^qsF+tM+~rioZh4Xj+(V=i?;qXtBth@*3iH;L zzPf>H*WWPy8-eDZTgco(xmxU>(g3YWMn)j+hkxqzvRKn&%0C;BzgzK{EI^s3 zTz`-_?p%0fC$(97)Q_E=?)*ivs8m1{BYz*4k6mxo=;HOJQAXzqc+o)#HIB*B?PQj}Y(QrU?!b}+05 z-!rhs9@m^&g2%DXf`kwAKgro%Xy9^#|ESv+nQ?j3?VlOvFLjWbPU~6<>s!mesB~t@ zM)%J?Q+~aLWZmEE!N6YsIXv=*Iz6O*2->au@h6jzo8cJy{g({qbvB(z1v0xv!L}2I zj=ck+)dlAkk=_$w#Kf6g7UN%f@oF8G1D6^&?8GF}<+3xd;k&zs4+sv4YUB8-UNwhh zD;SA*^}{Rg(*U8m zj`=YtZGXUmh~NiDi@-f)6}~IQmlwBEUe4}a>soE=@E9dN6qY}69ypU``f{;A2%hP7 z*~7<+onPV4?5_H>T8c4>w25{2c0fdG9m5t~Iyo=1FJZYk+;?2t+=WFp59Hs~LTIoR zYX^sR=q2c_apwuL3<~P!j|CU=>{nJ2XR5oD)T#YY5`EdHRPzhef2svAwii^PCz>by z$?Ge}xt*|3*SX%{D)aSZ(y(GT}R(#H20cwh2Qa(DjpwuXn!cHcc8!C zPlzAM$N<{NPB&)TJmYXF-}6aYbvM_#zr|5OU5_R_IGDo*qb7Vi;WOMj{a7jCvJE&> zLQ-C6H!(P=egW7FP$L=ZL+skKx~cltruWDq#Cc0?PQBT4K#XcI>T))PixDqPE-N~u zy4*UC=GE!^!E^RJ^sKW=%ii-_cPjT}?5#>4&t{cm4-F7@FMaXQ zL~^H)PLGTfGF<9;_A1JJOg_12?zGuV{v>kMvydcWgBml4d8H>rt;BYu0gAk@?{TiD7}!hMF>l(+>d>e&&x5vUB2=S20|Lpikh zG?CqW(LKgwWn6=-=P%Rq2Ow7RvKyP-Wl*?|Uc#N%888PdhI-vBG%~kw| zt!E+y%oVZl4`U_|%3|J7zhC7a*h>Dpmo-13__ijtFh&2nnXpc~cxvMJ!2%&~)<>HY zOTWt6)x{U$3q^^i%e6o5o(uB8_jme8*(0u4_u9nd06f~PUz?@`TT|OfX34$SlPZW% zHdqVVw<}1RDxz~J9|~LN%l+8iA-f(|SP9zARN&$NuonsVm$(r^bhHP?9SsMGE4r@z znz2e7Rd>mF`%jyqj&l7o&OHs-0R`W$3w^wff<#*Yr;h&Zoza9VMMvTF7AWDMlRy?R zoG;zt|SgTMyq z(@hoc@z4mDs#A90LdhRzC3N#uxRTAl+>5PdqiSPJN>$S5Gn*Oj2c9US3T2$*`2}{$ zWD+Z_(O+(rX04;R-U1nim|T_gP&dQdWI?hUnxp&-5J8orxMiCPT$ep`vg6!-W6F+l z9@VPa(}56e_}QMUp4#@C@pTc}gPrRVgZoYcyUj~FrD)UQ%pX;5L>%oSjc>%W+8x@J zDHx3#C-p3{2`HW^?sbznLagI|Sh=q0cGbUW>dP<&!KbyBfMJ!>h=tMvD}Dxz)&{7@$v76ODb8aPF|1 za)lHutWGae@#n-|`&>X!ubrzLg+41YHebof_o^V|B-#lY)W3iVV{$$X=dFP4dauS* zR?6jQO+zx)mo0|*mbnqY5Dmzb;t2c%OuO_M9R1z?xHeti-z zH@sFm>V@VD8$99%_tujJ&Pc(P56mh}%;^~49?q$#hR%s>#LKyHzTR3**aRPEqHV{g zviCwM(ioj4#cFms9etC2Mk-ygf5?8$l=;*4egfV4aQpnubnV3wcj0W2%oa8CPz`T1 zr0)QlI06+Nw-{}mGoOum^}C;Jnu*%e!YoB3Ve2G3pI8W1&pVZ>^pD5`tFp7{9xvpI zY-G4NkMOFsUHdbs&`>N>zGlivtzXdAm&vUK%}bOg)kWeD*G|`v5}-KmC5(JF7_Wx7 zrgJ>3hILZKy>=zN|CMg`F*n;tjQFi5cwhZZ^=A&uj8%H3JPd7^hf9o;E}&KIuJ zVrT`6V`Dj44JYn<1+G_1*We7j#PgWOok5Rvn%87VIm;+Z5+mcXTv6cgdHCX_Xzz1u zflWLYI|b1Vx(9SYJ|ut>#SykDAP)LH!&;sdb^ku*>2qrP;iY1dMS&aSkO;TW0V&-&3eEh7&v+*LVR(Src{Du$KvcY&qc2lsD z>Dj`HCbrfe%|t}T>)GBUFvtvM9VwmD=0d(RWWC;o%o(@iT9hY=9se}u(PFx~w9OLu zL-`=FH>3ZQb?Apd=Yjy5(2B!u40eX;kB49U_8X*m3K>`}ty}S_#ub;dDx2oXts45# zwiz@_Uyp{cMD1+{7JikS;)RC)*o4|gKxF9$;qN6Xp~d1wG4pG|l5cnGpNhPl<*on@Mp-M^j{EM1ebS}f^^*5*i5;Tnh~7?jZy zoxAU7Md^n$p3I`|5!dQ7T0EZ5pQpN;v*{z9R?yly@56_o%sgR1!sbE7)~4P}W)$+2 zE+hQ@S9)F+rP)xNx0)}P8uC?au_wzjSi^h|eZ(i^QH6A1)~q90a8PG&)t!7~jWCa% z;bV{(o1IQ&n!iLkd2era3u@ARXc5>w`suk zjO3UdpYhccLLd_dbLnI2`860fMJUX+>~E0YT>em*I-fVyuMvHY8D5jSrkmFDGk1u# z29m1Y)IYNkH)_<+!Z}SZ8WueU-s}{QQ{hvol2hYl(mwbStr7fcs|1J5WNle2=6ryncAlEf^->ThvQ*I zFZ$K}_*W~f!cJJUBF#aREl-PIJb@5^-6$(wul@Flb)=rBGFFu%n_RI`(a~$QBSdbk zU`AwRF+62=Z~*6R{>K!(@q86~c}xt+6isXzrMFK+bmR?EnA2tqgJ{%YT=KEhn3H1Z zduIxkDaq9?nz+)?*XQ0At%sq3a(k2_ZzHtm6jh17KEWimPEzB4t92*jWlfM*D?5Yp z$+EX0cuF5&6dl^QlTXH(ccRVvxDNyDzdg z+H2jM&xQluv@cP1u&UoRN?O%X^o=6jl*YzEX?-C zk5up`a+lj5vScT>HDR-q)_u3lo_&wgx{%yZT5xh)Q+$PYF+KGB>d8cCa_-2t_~|q` z1J>j8uD=G{P_RU=FL>`9Nu$b^VoE-ZO)$jY<0CfT(-t;0#%VgT8L@>M580;&8gTN^n5qwlNv^AVf*T9PRieJVNlvXW7MkKX%LZzt02ZV5tpWsJ8+8SD*hJdp7;|} zF4b3=o?}{fq;|9KSJH~~q%{|o5>SkV_2K>HL;gIPJCFDQ&}8droe}Md zbad9F=FWsSvn)_L+pt_5T!hUfK5LgvJ4|4b^{>Yy zmOQYEZeJ?SfNSBh?yky$mG^zB*)<<@^#{Aj zdH6~^gk@rN#+2n8sOw8V0k%&BXvjkDz4KVm)aKwHs0yc>VOqt(aYD|y!lZM>nL*u$ z`f5UmIGvT~owpRYagJGmBUtq?!s^!%UuD=qemyc|cd^jzl<;v1Ys)ArKr4rY zam^Fm(fBjtw6D=l8%FJ{DQT}3Ac2auK;Y1D61*48eI(*E(OUMk{Y}j(cKFzJO8;=V zY`w=H%u4vCYDn8m_2eJ`)39$n~*L8GI9bB z^{OR9uJXb5Z*kpv;_VTu7{gR|PBbAM!K&2$f|X41s4z!gUZJ8w=)=+t_9Sqx2wKx9 z|144739%i%(S_8*qs0$)^)+nqxYe$X=O5DX%iz1B`v+y>L`Kut(-eome`V|rv5Nu$ z%mKVLd-r@Uc-D|ey3B+}vyJ{y$*%U9?bf~#pAqO~$puNbz=?y2ya3KOy*PCspxLdyu@=BrB*kGQ=n zb6jh-qjvVWfv_N7t&~0C1R@fCmdkC)a*e~%TG{lykE2AcW9e5wn8i7#s~4N~YE?_m zb^KL^Fgk7XWE5%+@*cjhGx#e~e-A|J22y`T>NFVV(r4|t>-8xyH|BQC4&JjS`J$uq z>rsWeSinTXAUfw5}9GAqhI%Z~bu zHtt>&g^VmtMSU*#q)eRP{H|8@6~+olWAJRv-8k=OE*Vps zK9;2rivG^%=T5RX7pB-5EQr^wO!B&@+R3zI>9hFIENbqA;U%~H4JjLL;JnS5@WHGKy4ZhfBu_H7l_@ch2mc2w!1m+@jh zaR+MvokM{?kfcrQ-Fw1(u}A6IA^hPCi6J6-+4xpr}|P12KOe8Zx(VT>`Z z)$D9>&u+h|o4osR|Bhv$yW$7_X<4#3soYEuC~r+f=2kriaZ*!AYc^X3;ER$!d93mF zB1PTak~_H=YVpu~_4@g2%w{1c`Yt8iv{jK!(+XxSDx`a}yoO>zhR?NjYT*)xv|Q?L z(M@jaogKsKzJrqgF_)}|TETWzz~n_LG+SQXjRuuoXYRR9nYeSVGs#ZJG-;Kq?8U(L z?l}bx4w{Vf*KF;H=i?W&eyY#CD4|e`Ar-9SuWNO}AA%}nKFjTWmA-iMalNnQs|(q1 ziH@?hS*-Fnw`8+&pFIBTcwvUG5}yB}oLg|`%v}Ke;IjGxzOiC2h`Yt%xcOXlH#V|< zf6nRUT}3>9p6C%L%|m6wE$kv_0&0$bb5!baC*`4K(5&VeejG?Xa>YmG>ccl@Q8i&IOE(Q{@kwzM$U3HP*upSRh%9*K;bWy0R z^scUhZj4w?W|SQksSdpbI;OBp8&>QLE(MU<=O#ZDrhVT|1EEojR&NO+vaL853jix0 zd2)J|5u3x(^|KaIls$yB(h-n;AHg=$<15#W&i zu}`0B3tZ+>D=c(<$Nd>o1ld@)Rc4>ru<)v4E(+c>T(_3mE~7AaTv(9m{GJ-j-Px)1 zBNcQh+#JU`9}#{2Q|FMohygBZZx9ZBU5Nsx&y)K(@O_1+RLnB?`L+SP(50h?P%70L zx|8+Gh#?*79b1k&dWP0R4Xd+}sj|)4m=*u|-c1UC+B%_<`8n42wm=?UI*9W|J~VlR zJu1>0Z*De8!!C;!$ahcZ3+MK^BfVC_`3%*LGxH4xW4};iL5l}}217TIhk^j2%j_aM zmE>n1Zyr7bWqSzcHq(S}uFe2D5i13O)~uTy4$M^S%;ih*Ll`D*)vc{9@k8FgWAdv@ z_c7GSe3Z;NrKl7Tq3-VP78Z7NYg_-!6aTqcw2y9?9UdOe)~W-qRZ|+zsX`qzraN5_8Nf|sy?l+z-Dx{Wy>;ftwJJa? z65Kt(!GV`klWw9#`RADaoRuM~n>iO(zGkgsPQmuf7wwk@A*!gih!$QE!zDu+y-7%$ zngT~^hkgbYzb*#gIzKISMmbegpPS7P5Zu52fQX0~Kc~JO@NAt12&Eij!!@1IKDDt) zxVnw&-!(F!s_#D}eLIWnAJ*qjThRCM`r6UamSl!96?0Cf6N(Vt(XEX6tDlwSzI?VT zg0^-RVxG}vB8BDft{YXzD6HHGan*#oc6%v-(f}^}p!}aZ<%eBmp|~8sWC2Slq|9z%fKo1dNF$9A zp)xZ8>E6BMwi1Dm%7jp6YeJZ%RI?TMY7d-6cxj}cQ!&$DqrBR~esTujmo6XKwBn@` z^>t;w;`bj-+Zqj&@*(Rx<7M_qIjn9ApVu4sNDGqQ%2a0ALUUw}OjWDtDmt+OqVk|i zlOh)r_bBWsn_MK^*O%+ePsw8EaA}Lg>SF|(Df$)=_I3M^bf1IiKHuJ@B8ei;bt&NN5+Zs6 zTH`5vGtM7A^?qlRTwNMu9pWb#GSUOsH#xyeQbi5dG_6{2Y zNa_9eJcatBYXPSuZ&-;Po0Kc)?SvYBldUcW!Iov2yoIytidE?LwG)!@$$S5%nfPJ`M{HXJ@X z3KgeU+3n0b=i_*jPd_0=R`sSC{|ynEExu|cqgp@hAQmAQrX>LsGRTs>AY~M~S|J}Y zodaNn+8rUjah#3)!lhJ!lNX!y!8rbU6L)Gsn>+WGNoDA?YG2J#<~2{HJDy<%a5b(( z@=)~{`Xz9x27VYGvP97E&O4yt*gHb*5AJ7c$K)DyLw%D)Gbmk*R(f&5$sRPfxT&Qg;i{1Nc|(@A3hRan_;vSjhpMW+_c&AE+ANzzn%{p-(ccF)=u zHPU*PfnA_puVXo$8v+@h)#5202u#V+g(VwGgnl$BMCy;pId;M3PNWkyDP>ssXX}4T z!|eeO+YVV#4eZUt>bQnXxuq_&3s82}3EdH0NEQ zap*H^GxfusAZub1EldI!aXqccCoSq6!qJ$nd-=-|V)$Yw>1&6c1L9akDPZCS5 z;H&+rwi!@b_=M2^z{a^~Gv=W=75oAtXO@;h0Q(hozZPLak_pPuYetCCg(wr8w5k(} zvBv2G$2r-w^Ww11Nm1JUT0KYR{+aD&GqRMNqpoIc&W22=4IbJDtZJEbv}4;K(+xsN=DFJcxU~Y6wJ{jUZ40Xp{2o5P_QNavvS?5L z_yh%bA!H#P7S3gePvjCJl{UKcP5;J?s?n@+{N4Ee)h#Q496bLo<*S#K!1sVMToLp5 zczLO-b?iM*BJ!`=D2PPmLJ*q-ev3{1MOcAPxuPI{wN=Xc=Gy;78#(FjL03EIrwc%B z;eSCjE~{Yg0fo&r2QBTK!$0dNe)T*bzNPXALyEuJFv@~!PIvSJAN{71LX1NeE)7iFBNsD_UUm1?JRWD99&+eWnxI|HKsGbxeQ^n!Y`T8rc*=;=P#W$C}A zbdmcj>Wmr{2Zy7xHZ~g7j)zL%+1iE%ph2M72QhdryLV3A|v#m0)ImEc8!_{xBSlxyL%K@zXnb}OucpQ8mc3fNc zmsR=IV7Wa2t!~ziPfptHF9S(o{cI&m-z<36Pc)A%>F{A(fO_L8Nt2);u*TKw92^`> zOntN4|4n!eZf=0sfyeQQ@IEYXl6aA&P#l4%LF8RREy5zL!XhFf!ouD@E(o)?Vj3nc zX1V(@#6YIksaV{2Kc!3{M1UcbkeIOx?*TFXKH_CBd-_tZt_~|?LXD61w$VIBqy7#e z5#5cpbUTT5p61W5j3FVPXPK#DKy(rU+4DMi)7~L$bOKdqVsgTDHvkOX@!^*|6-3l{ zSJ5X7$Vn487Ytb&NdLWE)QAf7+T0k+r_1Wt*5PXV_)wN+r-}GtXJ8Z@92hiV$jfcO z^6Ij$@G_73tL-7MJ&PH41|%5K|AH7;go+jxGW@z+3v7`Pb@c3gcJ}M{^$mD!GJYUG`#79P)j)PRSb`gLXOQ`R!B-Q{oe)#^@zn)Axirn%0Bx7Y#2bb(E%(2 z7+qkHerxNB2_%rSIz%>2po#lAg0;hwdI;>{!0o!r9Svv^7GO?-))#bjQ%Onx2U5UJ zNI06WRXaRtXKwNFV{u!&)R*XDeBO%~5-)TiK;?`x=OPdtiD8Ys5@^6K@gxZx$r(n3 zPbl0K%^aML>FS=Dal3F$ZZn}zdX*vSmA``M{7@3iFdAUKp#v1IxiA6}?IJV8hM2Si+d7Tr{>Mbh)oJWAFD91UF=$6+`Oe2uzQy`P{ z1>{g}fe5)DO1caiz7YLm-Ef4NqAYJ~c+lxRX3D9bii@2hCuZ_gb$D$|gIVx%-j zDgNmR50`hxlkHF3a16+;VesWu}dHij;K<&Xj zojd)G2Zz|tjveM;ksZ}h^|rai(Wdd^u&~B)G%aXT%olCY>jORJP0Cqh#`|bO6KZIF zqH|o}4Yg45betMzGS`px0aL?V{7&3ezOrP74-v@WOCcZzBOZ_{Buo5A*7vCzMn2uMmdoI+feg`ITVNVAD zK&Qew)gfd1DLw&SB=@{UuGVH*kME$v3?Cr{F{I6O@!mOBrv#r%wY_r#{#?zwg@UCc z*vWJ|uoykTT8xQU0P4U=MULUW&!c0I=)zv-;Hcp5)tfuLH!?#?*%M!VUy`*QcT6&* ztqlS7mGuB9;e;X!r9?kk<>KZ?qX==v@g&I}T23`>&f}9dlM;F$W)c;mGV~(U%M(J6 zA=I&5x07j30H?X<=_#|_uEIxaNQPkH-bY?ivjqO0-0pfkQh@2DiEvy}ae8Sbb~YAi z+F+!Hi-A;Km?;MUQ_z`SiVeU1n8H9o)LxaHvt2{^Fi9Ep{qfXuySg&|u{BL9=0~~2 zKMQt#iIP6)Vt&IXiuP)kwYl0rN1%4Q_YatMIRHn04N}HU~^8n%aB~niwg4do5`a0-=qbb zqL>pUm}iLk&i^dc`_;(|z55Qk4Y@}Jm)r~!a+exyicX^p;95kh(L!Ul zj9V>UW(dB|kn?2sR_eP{lhfQ)R6>K}JaD);y&CMOohs9h{s_B;G6t}@X7v{y^DUIT zyKidUlg<$2k~j6mzGI7ksEz{dXYR^8SK}We$k)VPCq_%*9b<_pu9jd9QJ^W+UTDzP z5@p6+gC1;R&v{WG3(4iAzNwv8UZq=RPav+fUS^oQ(TsZw>E=+k=xuZ+=BUHS_2Ls_u*|L}X2A>2$T1cqohUElK4)(0%hG zUHC=S05N?(^%`c2`Bwq&uaCGne&Z{@CKKn}xEeXQ8x0`1K+$MP7ze`RxgOs(?ut9Wc1>=ZGmb$q=57Jp_s&?)eW1@+%>V zSeddfhsyk#(h}i;XQXcd4bfR4*RM+7AFiy&)@tnDiXqpF64UBL_8u`@B-no%-&42pGQe< zi}xm3AIXa|Pk8QtAk+co2IcSNPck##?$L8}azZ*g7xDIKZEXRv1&ku=5Kz~+rYnf$4FyO`T7l6qBET{g5B-|Y6c0Oat@W6|cTt99iCavmUk0bY>K99%oHMV$S2p`{H1k6mP>k^=9P zj~2{+UB_SDCMLqe1Tv*PRWDN^VG&{Pt=a8*r(*y&0Wzs-+f6k9o&l1Mgm!?%-9?-~ zzkmAjW?g3W`9!JlB_5E(VTfz%&&q=#8vFUTqcpo}kw0z>li{)2VCDfwpzY{_@ii3nHx zaS4ONfJGNV8^86h?mu`#yp{d%=NkO_;kz@snvwyY{PPk9p4S}92uRGY-@G}J*-O**;ldAA1AZUPM=wD=`ET9!5jug&lg#UeRu2mdtn2A06X11pM|?R&GnXXStxhTr>o(%r&iHL z3SBe+M)FvG@zVv28iH`gsp6b05W_jxxVZS(lte|uQ-G>~TpyB^o`kq13WSK;)Av+6 zOXI~r4a!Sa%-%~R0N-D)wY9wFqAC$Vk_;TB3j9=GfA$r?qX8?l2NcdQuP3*~J~3X- zh?(|6w7B*crz_i2b-Wx*vTc_D-JBjUKsxX5#XXOceU%aJ92Pgl93>aXLQBGA`8N8U zpxYR@N4S8^5(oh*;sAd#TC>v_Us{5X08VxL9rt8S2I5$nnOR<54&Ry%1!`w5Uj>|m z7I1^h$-2zp)ggoSom&^DN$|>?;{9s#`daJrBfx9uyO2w#9(JEIy2I1YR8YMr#z8+r6&`F7+6GnD+4jtjSaitiork zXRksv{{)eMmnFYc!Vl{QHa7tv9w;e86C$Fx2ao~McG7{edOc_52$veTi(~KIGv64J zm&AJrk`m-Dv~R(mmW_fJqC0Pbzuo@BV??}%s4$m_@mtjG(EkEE6oJO^e7KC5_p)5t z5Y@DPs`-{fEDMP!iXxEUPS&N-ZA(3R)QhM(U>~?Y2S_t;2^Ak9kmRRS)Ju>pc)}+G z22IFxWR8K3W+ya+jY9gAc<;pv5iodN%ml0Bqjb*YZ zId*FX6%)H<^VHNUe?{BBYR&&2R*#S;xQni^DcV$FrrsJ^rr$V;qOtxgBv6Ze@xswIx=$ue=1+^F2vdR&9_2D7tO zR?l)>MhQ&lwgze5tcBHi0Cx}((Qb-N^A&3&EywNT>>Ic|R1nL?OH6gv^e z;7knMSbaYP%&%F|ml9>0U}CmTN6@SHGiennoJ^A|zUxm1U0ZgcjGY{8o?hWU9O29*LNoF zV4^$mGMgG7ttQV6)X+AE$NSIL@^SZZRZa%;MCz!_G6o!WyF$PPbkr6!5wWPRBWyX3 z;jmeq7^k|iQZljK#&i1MBEQbYBs1QFs7QOka`q%H1+|NE8C)kd0Cc0c?KD82d2Ff` zLN@@SRUc4!Av6(WDy?NPXU@NN+pYiwOo*mrHgnuw8Q>;K0pC0Q5-1ShiAz-}Vd30= zR(>bRbj_D0(Tv+pXM5SeBkBAqFyW~w5FIUR0!ZqD+p=~Rd)sfQr1fho=cpcsm3-v@ zj@q1526;DD>D(#OU}0hM-g$1K*1*FzokK~hLVuF7J^=m1+haR{p1zQ|gb+e**NM$7 z(|o6;bHqwdkb{C3fI=J5k<^>+XK)~E)i|vqF}CV@(~wSMH-!Rh4xFY9qcOlPe&Td6 zYS#D;na4sxQXSzd3;<~w!7C9moID+$q-EdWS2milJj+|XE>D>^!LUt?rncC%mGpVm zKV|C@GdjV@*t4JWZhE$eW4M$&5?~INGy%bfj6VOxP1hqkA4@fFRVgb)V&ToV=MgTm%R3jZLcf=(!jY30ugE z`?1>`=RTzpRtpmWT4RJX+K?*8HMZ^S&@)Ayju^PTTdVb1n}Li?xw~g&V4FUqdj(IZ;qeIiQdlnY4IR+dP#| zY6MEF6}S?Sh1B>b&KlJ?Ufj) z-Z^%OxBbK0JvHPDOdNED2st9*LdFKocf4gcMc7-<#AQ!{WZVzZ1f+3h%J&(oQ_{bF zpciIGNg=)pTi>wen#&Jpy~g*>EDoFE_F4&IoeJaL`g+e8e$!i%~$R#|E@ML7h?!2yvhtr#`6 zOW;T^!I@_-t!A4nHxun0Xo!QoNWg@*S2}We`UoTm);LZlKcb}c!!5%3vAvOEBu)H2 zL^QAlNXpPfeK5pibl9n2UVFZYDTRun+~+^iBQ8mdvbB467T0Sq^n)i!;YFml5So@y zvS@moQof0o@LLKm&udyk!yY|J)lFsE&+;@#ZOh_$s~{`J#!<~hwIA;<+4((j$;d3; zoXYe$Hg$3ka;*|Op}R6m%(A@12~mZ#c6h4C``p@a)-HOhbUNZShVa^fU0-s(QL;#__;Er_r+Ha2wK{6-$=tdX#%c;JIXa7k zc$-FB>VT(>(PW7oH`(L%j&Dhf)YVzf&V@ljeS8uo^aEkC5V`ZWaVI3aRxFIM9~Ajpn~9 zS{teZYNPqQNU(o5YGT0YmN4Yr>)bJ{m{Ky+;Oh|8K|=*L{wjnCiG*`GuKhp(xHLpv z1Y}ljxL?PzsmR8Y)GbREp|)dXa+)35NLhA~|GUbX{pZlJ>^iv=d57?NtAMnW-$-S< zcNOb46vR@EDGWY=@F-&E-+}39HMnUXJCSZMhw-a?+_P&{^GJBzaV7~=20n0A=y)e5 zpJ$TZBA(t@LONTXnoktB6jm_q9*C4u+9QrSd%8AB>;6@_rqh4cRj$+r@`kBS#>wKr zl^jJDz#d+TI;DRzyBj2yE;n~w3Baz2da^{{?zIBrygw+~xf7S@+CT(HgUnDOMX@gw4QpO*y{swL8EWr!eD z%x?xzCdiR+HW{n|qM*@_baz}}Shdr&Xs1+nv+y;}X7}k~$DNy#Rtl?gpbbkNw9; zUSn^qUCC~j0x)ijOtR>~`c6#t26gDR_MDoSDg86;cPg?}5o(vz*83gsv3ET+xX@0C z*ct4Um2Mw$!Zs8k5@|pQLP4uph2#UY(#0UIhxn_MY?c=en7YR=rSF25*9(0}1|R0l zFx-j&9BVQ0P3~GseKaroF#fUA*S|UKjxAfT2Z@`iIb4_8y0gS0l<{Ib>)wKxIvv~? zzPi({!-iv)DwnPm5LJ6JF}dx6M^5lkOJ$Rx-wTIMb=O?|m@u%Ln_2=jHH`NvQ13n4 zcS|PL%!a*f$X8_g#*EVAl5)8l*ws=V(Q{ll)-&TZYh*q{JUjC!LR<(-mN{l5xUB9S z{+iZ7W0`JV^Q?QYt4rP47bno8&#vr{XEv_R(5ivo&K7n1hP9aHs>EUzk?l(y75AO^ zY55%Hs^>htCD-S=C2$rG`A}_*#5PmY`JWOX!xx+ zbn3)(j;CaH(ZSQaW9Mlb4E^g_i{vPfdXc8E}H3YK8j>229K9MWg~- z1%yc^0k&`k+e-54RI>M!VPbcXR(e(MqT?u6=vXiN!n30 zylojzb=zwN`Ad3~I9A^@>u*``@{j7mL6wecJhr%3mKelxuB8^<**kMaR`y}w?Y#6O83XVq$xy5HSY6+?{|;+E}{27(bZjIApqX( zt#hfGsGBA9+sl*#0yw3*u;&?v#VpM)VZb>fIeE$i9>8{_Fp-9>ovzP_tD8PY=j~5@ za^5S{jc2^BQv;g)K=#!>47l*?up_b+`{nBe>qJaW4SQju(;gXGn?Ia!#w;CI_I5st z+o^th+@&C1R3e#i=n;kMEpv2Ls_Le8c@&~rmTWWv2<;qK2bYhu<#&4mx-)?Ur%3K$ z=(c@--QX)U)Ml>b`NL`%)^eXU0g=f{tm+Y;IN?Q^c>5if=17nJ(c;x3*kdYH%xi1G5o{gUks{oZ6E06I}X!Yi{m|IR5eDui}29 zdMj6udTw|)z&dowrg=MsrwnI(Y8-8pC96CY4~#rc_CMU1v2NxhFl_}pWYOAV{w?|Z z>FeImf1|eEAeo#hT|At?Prd8!TxcwGO1|ts z6-q-52SVfytE4?~?yC~#r#HTh?aA1t7YWM<6(f-$Lr>Y736Q#t24D}mhsr*Bj7!O> znnq#ZpXPcjX1TK86Esp!FtxYwcCGSCPx9tlsqm0$loH`pxi^3U%Bp^2*``6{Bs(*e25PVUCYl&N10C)ARqM1jMVb3DD zZMCfpfPK|nT)^?1#V(U&f%xooH7}dI!T+TaH0{%RLA~n+9GZkiPajoxQMF(WskS}* z3Y6Lcq!&O(MfS$pGB8ieEIYDFF7?pC3);_w*bcT@Htyo_?kGf3TY| z_c$PR>FG-V_vqy0q-?NaH|y%Oa2exHl*m zHzfopj0b)~=3XMEW4;xQH*t_nfqCPlcKh~;8)=6oJOF}n4_<7?C}Bll;GT6TT^e%0 z$|9oc;l(<=e=`sLEqKa z9@)R*hGO$8Vzl5LG-gv1XaY8v&st#v}_LpwD7j?i5`YAetkBt%Z( zrua3n5KZ=$;3lo3*87&Mg4C}J?fG%@@W!%@oP^%O}IAn`hMKk}NX~Icf{0`V z$qYkAa?UgG_PDouKj$1c-{0?D>;2=Jg?IPeU0q#uT~*b6`)sdN78=#(adFo5tpyqI zr)3}ev&#Fa&FtiA_tHETsG~UJNK*}#v9QnnW=qIcr2~6+4?Fw)OWiwbR$Q_X)gnxr zzu;>%rrp=i8ZfKBL&NbQc!dM9R=~^Xk(!y7Y(z&0UicgGr93x-)0_Pw_1Iq@?#lwh z75?2u6w@XoWNYjA{Cvk*UyW$oz^)e)@;;mnf7}xy1MOsHaz5V8hn<@70sDX~v2oBC zvV$HTF3nOlZt!wF-d!oKH+ZD>NNwi1?R?<%6km{UuY$lkEE0BjS zFDB0TRph?Zn|&(=dG6$UyxWx+pUjwW#sLm4!u<}*RLDUKDKa&=&f~ZiWf3tzhkD{m zaiVcjaScxq8f|=Ub8ctpR??k>Gwc}GjVkxG@p%HO6cJw(YPgiI(AYRBm1_10k!(b~ zk54S@TPIe)+^I7#QNjXEAs$|b`#&-)l~d)QwnzNm5>`Z*ri3KUu!N0O*sH#qZOwgs zk0WX0B&qs!qHtujWub3vKJ+j}WwV`bRvdZ!;i*2(hZCPO*6O!lG6}s^$2PJ=vS-~i z!Eqnh&!u@WFZp-Gw+6)<^t!(ro!&DCBY1K=`s)s<98~74Pnq~T;mSY_#7=(bzl43O zx^$TIAb#XDh5_uU_KkPM3yM|-erd4avlS-p1?=0-7F8X+4AhqI`^iovWAe%Is?d8}(mT_{@PubuVxzSy2vNCi73Q@HfrGuZ-uySf9w8-OM*$v+h{h;mu1A&V0<5)(61wDg=C;U$3cE5<>AZvBvg8u=r}R0 zNZ-Q&zr!p)d4|F$L#3&CCv*og8MR1tlFgcXFYACcFFgr)9G+R`5xqVn=Q^X}TPp9A zr_y8J0T;*lB|xnxap4X3>{M1X+pW}y@Izb}#lf_#NHaxqf zq9-g@As-{Rz^175@-Kh+Z%E~@Ep_caHB)aQ&`1f{h-5k`D7j?3!!xzp{jzHF@)DHk zZI0ZnY*asG8nR&RFU=N0`?f;{`V8cYof4k5?dC+$CTB3h%a)R2zW?N!vtT{F_Q)R% zeD5Cxbi2UkJ(AIeKOh^Sw3>$c8u1PoUaUp5kzq7#%sxC9Pj5T zEq>fdrj4Y9I6V$K{RWfAMDK{JeUCFLv1KFP`;+l9jw_l3I9!oU&Aqot$hfgp(loMp zw@A`ed*sb?ZGjS5BWm)F92(dg7 zO3iq5`EbD1t?=h7iV15xGYa8dg@>o~zTZl&DM;poA^l=derKakH)$8i4 zOHs59hpm1&EKwvR;cBc3k+fM^sX@!~t|&87V6tD!cb%p~CW`=QM-Sqd9KmJwt|J{!MS^lp)-ybK&YH*&81Nh(n zYZm@@Gyi9J1{csKR??SG3vvNm_y30XKji$|N&c3exSip|IzJVq{WfL|0osWwil&LnUW&vwfzfVItQbMdVaBn+Nxj4 zv&f|;zmwsfc0X^Cb{afmFSVIMcEEG!u&=LweieSQodtJa6BQPLurfJ& zR*y!vfwA9^OMGD(sEgKPa_K zAs&8WWR@lyk$RsU;9O+A=hoE0)>=<@J!&eX!Dn~4)$gz`Ib8Ixl0Xc#PT;qGR)~mP zX96jTTJL^L1wmzrp=V2bFg<{djD-C#*bv}Y3WiI8q_fkdfuf%t_+|ra3$V3uy~oBN zl`jTb0)&adcd25>`{Cf!_`$EcXWb3g=zsCpV;+>IAMb-Y=PUwBpQA25KlEf!9J^6c z1H?O;o%mp3(;20XZoVNr8`}X0~1d@2%6TH8lmj+7tXfTTr4y5UDJm=th&|#dif95jFPQ*3rw>bEsmk3VQgGhT) z8^ENymIg`Z=Ucs`VLlI<-Jw^zLKLJHqr^^h1!rU!tD4X7i29%_)GSagudPGZgU`C~ z!=X>PF^Hb z+39V55$(6I`%Vf;RmizsMS4`rMr`U2cn2m*ow0!?K4Ja69vOavHQ_x8J?>BgBLj?{ z{Zw69+bjmJl&NWPM`Pq5fcr zK&;khLqg4LKg-l^%()%7ouMG>Ydb(SI}<6xBfe{8(aKJ?KL@OE6DYA}>U1mI0; zX3v?=I=8*7G68lKP)Cp5{f#Zlp7B~=h?B2xwDR{z;9rL)KWUv6p7}*o<-NBG2vch~ zrHJ>ThMHLr!DrU(=~7{)UYAJS(~>0sr%n@A9knqMC@WLRLJh7#G)zsV0AnV+rvr+hrD<*Cq+&KcTT0b;3kWU zbW#;vKp;kHGkTUCMX+zk+LPA4p38JnkvTF@Ix?eEH>1fDfT1o#k&oIlFgfRQ$%4i{ zZl_}8bN-?LMA?;6T&RpAUbtg#6G!sqUElgz_*z}6pO4nYf%a_9Tyr7RQgo{6L>EFm zAcpFxn%Um!6Kgmz&YjK&CTIPFEbA9b>--gRmQxpu<1%=IzigNtUlj51>Nd`;2vL!M^cYip#jt^;ieIm5Tfttots#^W9yhK1SWA8Q)&QknE8&GKVR|OeJ@%ibpjD zD^B0#9J2p0&^e!b*U2{S7B%@w1!gEf>8e9>zR6D5kIms*^DZTYBOc_4?9d_f)> z9Ul25!am!a5SCm5v?UiL9c^$tD>0aK`fJrJ7eE3+m%on~>! z|GVPDb3VFT$+Rn)jEstx=oJIZ!e1n57?;(rPM7NK*!a=Q+LRlgx~SqN7?}RIij^SR zi0W}*UOKQtjoihMfewP|oZdT(lUs#TcPXsY=xW2@0bgV{s&!`LD)%5hTE|jLh@Ty9 z{gphYJ81>+f45o%CAy0gcCt~ox5TbT&;m0)CgP%fBDL}J+roVRUID3R*#1SjlPxNV zl#H<|b|z=FIqj+4EZt)LLBtuy3pvjJyUg*8zqHS|d`>4C0QZXFF#l0`DC?qdug>ZI z=lu)tSR-9)MWQTJ!DKu4jPLHU%KbZ~o34~rtpCf*|F#|s9EX~JzAyLRLTESX0hh9? zFrCrd%d+IL*B36HPWqSAB(9<&w{B} zuOA@7i~P%VdpZ@{VND#4o*eb~Y8}IURqrxA0mMfPg*-U|7wcpdw9FRdr*RK6$2 zM}D;~^Zx~62s4#5*M5_Tq7^eVGD=EqfNagRY|Ue!ItUp19W0k(*pCnuWZMCN-bt$C zi8H}^QDXySe-cAj_Lcn(|4YajwUWpOP^yr^)vN67t(N!@6?2qY94YC%1G3&{W2zAF z8-UuAL_Hh=BZ0rzDz-S1Vw7rE{_eH2Ion6&hXU9W10**)*v0(z*8v3qz^v~w0qWEx0yyk4 z;l25fm`6L<8&H6c{he0;`3P1qp$@P%E5Ggx#wB8?)#m0@dUuGh_u)21YJgGU%*Y#&^-3TNy#(;xm5@5SBg53A(qq;5tK3#?MRoN@0BHOHm%AZHMfhGj}tc;+<#1+2hY zbJKH(WwQ|67CPO~ode(0>vYILlI9aAZYem`=5U=cw5k%|mcA1=pI`&?p@ckt4+g<085+)5E z=>WgypMitZ{rHKy3UKy$M1=8!6FT55xW6f`0t>%9)K-Zm`mU9dTHo!AR>3F=%#~M{ zZxSXETwQepyM&CEnx8eVf?5Sqv zl3=N4`h4PzB^P*^f}yx$iWfnS7`U*Em@vt5IRCcPi$>VRM{~07sqHtYIkf$)E?UY) zbTI`u!@6ayLCb;&2{?Gc_v?g7`qaL_%{G!@W6yJLzr@_{KutmzuZ~{`9301W&~jct zaVPEq;EaiV!eI1w1&TE>R~H_E(eJV4VNTH*JQR1V$4W2+Jr-%7kMJz4y#h|OHTGj? zk|fbf@n#XWJ8>{(TZR6}O%B=xe7A81ysXVN{>lyNBJA^@J#p*uLcuiT#m!Q*{Ic23 zW8o3K*K$nh)9@2${4H^=e9oG8jcM^fQiMsN7`1(?BJN1}B4d6Dus*tzIt!Z(PFn0gYuZ zw~{gsJ0>T=`dFm^8ObK?GJ`KJfv7W1_aL7J=1LB?`_%kvNgDoO`c;`PzWFGM!@-#l z6Dq8}E8UT_iRw7B_u`{|mVNyt+O1{|Hor*#*2^s%A*mQJMQZ*#7cdHz_~M&z1*Iw# z5CDiB3FJ=*XJsEd@Bh&y4IL1FZ}Xk2+;NlK#b5aLW4PbRQdE?2gI|U7 zE4vMMqlpTVB9Zf0Tf}5!Nj=90e2kM~l^OOG;59`K{{9i3$bKDtUpE`C`L&P*sE|Hl z&-duqK}pGXt0#Pr(mX}mNRdI@Bppi~r(1C-n%IhUfS^i1R?+5d!D8H00d2A(V3>=S zI=%wkQZ}Ld?5X*Em9=O5p4-5+D#0K=Xxq$mIUYIkUDcHkE$~Jg|A(K4Lw^+vU&N?X z9uXOFd^mYriGE)lgOr4pZ^zq?2&YS9i@7ua`x3vGaI~N5!7MXNCZJp6hOPvhK%>)1 zhAF;6?ReFPFi?G(=)IJqsL2G+MS;)onT;fSYYCI8#&;9864q5&Y#3oe{AnT%4j)$= z;SNW)7)8-)GmY8N0rl9W=6F3Zrl2bW<-WPZ?(vrb{$T&zd7xVX0<)1X`RiiOU$|r> z8zC~*5;plr(VQusnfgubHvOw0=afsl%8NPw>P?6l zekSW2*cwKgAbhjEnv?kAvnjVhONa7fC9zM7n;BC}LQ{UjmIhkt8f{^ICqI*l=^bX|}%*uyZ`Ch9}- zPv!!}5hKOX7 z8fjG)y0^1OK;6S3e7SLAEukH2J-G#V$$@3#x9Q5!h-61A8J76GP~)rwBhe|Id(rVm ziBsm!`hMugY_NcpsvsewOsZt_qyoH80X$u9RX97s{iY^ zKesO0#(O$V@+ta7F^1NSUb%Xun`VG#Elkv>p`AX9lOynpSLq^!2FYH#$(?cDor}A6 zysH6I?}CC`LoY+)B2}D|QQdoXhKh?J-9Zf@A7`CJD>%O%bi2|J4=s-}i-FTjU)swC zyjlS$@PA|?-T{+&a5RweVhQ&Jr_pR~q_wTLVlmjK*cPlDbPwDa@Z0tKoSV{a z*ft|fTutK<+Cgmr&n2d3=wzda`95*JsI=EAl?)i2;2-=!HQKjmIN&#?PU^ec(~!O9 z-7Lj8-Z~J!pE=apaN-Kd%e&1*c|AcImLbYeTT1;4Uw`G`S@vu$ zdyOUJ=N`*d)>Fp43H8{pcQ7#A&&~C9yX*WBCX*Bxf*Z3~rTe-WBB^uz_~`ycD3 z(<1mF$rZUfg0@<%ySwJJY;+JqlE{lE-52l8CUgb&5(2|b%R$XwdF+;@0R18~J~du) zBN}>7!@kHuljYK33|Jv-X8+INE~1uxQ_tCCYd6dHXk3e-BAbA>F=mHMp}a`N<8X<|gMR8oY*(M3J>iK)EUM^x0S)4gsG}R8zNc$6t1(fX#Y{A8Bz&i{pm2&VE_HGSzOBZ#fzRU$JBNGyNiT1ExHv z$X2hzp9r9yDAsuI?0W8WSywn6U4q{j>Vt3OR4mll?A+2#GipJ&dZ{B0j$`B43@L9G?2nFihsrpKbg%L* zhPPaOEL*W=P~D7>M#~xuQ%Dy()Icgzx<LyRzE?E~0H@H_V^K&h@c^ z+LxBFbN-rGiLbVbU^2dacbYwJC+vE@(d`%Pz&I}{|ML+7)m*meHaa?%a?m?XPXp7c z2=Y$C7^`2eetX_-i*(7n11p4khIo(!yV@ZATeFgKj}~|9mTF)!@K3w-E0XRBR+ucU zfhySI^-tT5f;BLB9sea>n1fYSis=o2XL{i;}Y}im~ZLsYI z!cb^+=h6_11GzivV$^G-(ueNjPi%&X^-2=kpRaK%8K~m&On*%LP zwh8HP8k2=OcUPWyl_uDY)!%Vw&+{rhx-9raRx@gM0oPi< zHn|+Ge!7Qh-t#-LBZ9ZjDw5-ImL8X*d07KKd)dX@2~%~-4eiLHl>9%E8EkZ-av&i|IGUbuve@kW9VTc0?|Ig_f7_9MIpbWwLANU#?Y0!x5#CAer|MEZ?=4g77uAO5yA*8@;{a;lln$(1mNIBG@?O z5+*v1qK_0Kf7NU{VsvSANj4%T*^nSO&6L|HFeE%(fgQE5sdb+wH88h+w<2X`(OV4w z-LEWOOZQp~B}x|8#ID5c&LJ#HEAsTcel3U5ynxwEBOhp|-Z}E@(GR1Dg)E?EzZfth68k^)L2-l@38*o#_Q>a zIqoqFEg$QD(S@>|gt6CA#yIGjYQbHq${O*tS(Kqwy+qVSYGC>85t$Vk_MYG-gEH=D zl1Zm7=E_W3*#3Tlw@c(6&5MGV&ztagRcFze*c= zsX({pB-qyn(ycCb_pPs|8e1Xbu2RszMYv*sd&j!?^?K6lhoBYM41jPMMPMjY1YU6G z-8Jg|yW%GC80v$OdTSye6UtNJ=5NConAA}H(rE3rHtz}ZK#qI{@d+I)Q{3(*YRtBw z1C*%ven=RGsA~cO#(WE7S{xt!ON}|YWUX0Ew{>JA9FQl9kyuQr5<46VM6WU>Z&0gT zc}vgFsP11Ah;iE3mZq36x9JVS_|}2D06m}c2Q?E}f6>QlV!+iXZM`*rOUc3%>9?9i z(0#oMmyqmDqt46Hzltj{W05I{%mG;+e?60$2mzmJ88kg(f5`r8pF~C%>+_i83x~vC zLrc?6{!LTD*6FnAF}qe0(5s-F2bj?-p8tH1@WKs4wz~n{PmW@K2gWr>`Zs$Xb_X>b zNptZ7WOc^^*_g|}6eK{X;1ORopbheXvBeEckI27h+y`wp_98$HeUbRT@*wQ_9V+mD z_$UgOP$%=nL~W<ZSk0k)(2c(GM*y}#trqRp4t@;CJNPH#{lnWO&3^NCAu%Zu49Il40%-FxX*sR9 z$s~|k)8!X)fO<;+>K%}rj`^zSpFqLxpFja~CJ14e(13>}1ogN8$Bun5A)J4#rVp?O zHT47bw7Bw!bN+4rMr9-&f4?yb?>w^tcw;UuPPev8Dds%NE;AT-jP@jWP1Q@lyDxIX zF?}QdOx1hfPf+AY)HDWsbIv~kgn;fY;Phj-1ZbRf+XzF{uY#G9c|F8j1p58@KMY#P zZ*&%7=8+4AIhfLaAKdB5M$NBWbEJ`S&R-V@xO7=76ik2d&s!gd{Sj~$CDG4R zu;3R!P{Lp3x1ps0KP9b&Zi3MV;skd}M%!Yv^TNOKz3@y)ZLMK1<*X>C2_x2JrDlWq z-!_fnpLmzZ#_u@$TLlny(GB`^4fKEI{O4hQ45Apqel%CS6O1Vs|4?gJuM|7gcm%4d zLBPxK0V5M1p#%wdS{j5x!i&^3-hlDGEak+kdNFQrr|A;W4u}qbfTMquAfowqIHDCl zKm`*BAOdTDcbQldV2l?NwpTHtw*`_M@_Gcp1U8s@yVS@%-hhhbU~ypJ{bjFsxbwBy zt6O1V&jA(fsS7FxD!Kq((znFL161@pR{b+{qwc$bNjfPIb|tjn28tknY1L}f&j94k z4&+{s#{>Er2CYh$)bN4auYrV4K5L7=Uks+CP?G$|B}`mr0)fC1G6WlgtO2asv8b9P zP%j3cUiq!%abPSULq*Py*DII=hWCvQ>$MyR%Y0y)Kp5(OSp%z0<_}uER8qMJ_M{mW zt;^R*Qt%6t|A)nVZGV4#u{|SO~U@weZpFopYcWf53|g9~|>mjMRC7D0uZM zFtIy-kioi^bkT}J2~)f;o>kL~kyRzuIAUc$-3qiDdIPrI4e8gIKL>$!*EoMiS`0zD zt*nIj^06jpXQn-0;tb^KsB(U7^;-Li31*lKAS~%pf?2<7j36x8JraSjOgiFfGyll( zQfeMA25`T}u7%%Gcld+{9%crrq4M8UW1=mJ@joO<6pZOf7QBmL*p6B8MdKLI-0Hoe zT_S1@5wN|zAejJFv4U*wzpwiL(UJ{@cBYQkI5|5{BaU>_rjKTKg%}wb`5YYbhA#7k ztcVU(As+IP=^ZR-SV!YGD80H_w?@?6endk!HeyN5-$?V7$C^$`PZxUOgc;NUJtznU z*B}6S-*ZH`yI&wM#e{VUhopg}m+rt+9PkG;&V3`#>`?YPhC40wF)F)pqrFZLcP4RI zHg8VedsT9qr897jWMJk}jv$Qi;RpVw?+fw=Yoi#9?Mba-Q4RLJ^cw8QPpB1X&JN+S zO_mYSXB!EDg+pR)x`r!yrY+Z~#)NUZ+O!!(MQl1Mw1fEdR@v4L?e}Kc1e+Yv*dtvm zA6uFaM3Sei?!12_EdF)scF;ZM)O|DjT)(#Nsj|4fv=o9Sv+Y>g;cDYJpB9h+x!nD- z>>4AOn{$i$pgm>^1&wp=?w~w617i{+fPJQg8)kqFRNlxNA)s)(!U7E@bvu*7VXW%M zNwjoQ^ca7Bb)?rbqbG{?Ca&@!i#)7PQp)0y1P#14+5<&;L}8Ii-|R;wpDp3_X2#p( z65ZBRxLsM%BP#^?8EN1hT$g{fyghJs#l6IM&uxb7%IbH!;x!UpMEX*p%3+v$_SQ|C zjK>0xj=K9E>}zD1i9YR!UO(;z$8)rR0c*eL8rothMl+3EtR1G&{=vq^P9Ml{e4}s86^+4B#pKzIRdl>)0joJmB<6F%qteN zf2BspCgC{QAwJAKm-*`X>CrEA5l3d+%l=UPJ?%X!x9{q5~Axp?fhCD>5D{azKZgQh68_SRtfZEs=hA& z8vU0(Mth?6H+Yhe#V@jX;>okt$#TPmx&p=Po0c`qwFMViGC99bwTIZ5Rt&^ehh7oW zC|?NX$>26FpQ5v-%GrU#zP!OqzI7 zB}U~Ay+HcnA=mONE`41v%WC(7lai19#-(vwt&riDDiT;~T@M(K==0?}l*4*+#Cyx! z>a)^=6+`-Ft=eHJ#|2|v=)QR5vLoWQmb_eVjqZZ#;T{t-G-|H@=I5LO~<2RpWC(B3I^OXUE1Yh zEqCF2rFIEU@3mQnKMoaArYpC1_Kk3f2#KGtN75e2dt7DG1r8T>(piQ%a(I^%V_m)_ zrh`lp;rnm^3{vZVf&+zfv_*<$*ZShxz8F9Go}6qu$1T*CoZOS1O%er!SjeBM-^mtR z7=@p+FsC&~tuPnJ)L|b}BG6u5ex)>NC9*?rmqdTC!H0->IN@-u4OzC^?u-A}N|QQq z(|)k1k4bxzJ*%l>S}XsV>vOG+0rs?O=o`abu@J5WO>M307oUwiFJw zpSvU_*X6L3rhi!#hZ4Y}pX+aHti4n6sxZ_znhdlJDC=pr7r|@XEVEu48n0|vfIJvh z9cgisc2$?yD_gWBXNjPE3g>($6|Q4|=K9GiYZY+i=dG|>w#llxEa_hvUWF^l={15~ z;8ZeAb_m+;E~Yp(Dbxt?1QQ9Y(OP(-N~s1WVA=<=j#&slYvmV>-rXt1-U%6)#!CD2e4xa`&|aO`s0)&Hhx;JB=Jp1`R)`h4Ubf%N!o@B>p*>3~ zXb=%=kr-WC*7no7+---|y22Y}E;$P?oljO`N>EA9x3dy9+2L~V;88U22~$J%apXfn zjQRLV@3}}?De6nOTUv$N%BnD5)?o4fsb2noWRTLgxS@R((l^%jeu1T2teG@Hg_nyl zpYpj3!Ov}tN_|t`j}4zF=%m^gj#89{Il3h+_nw>guCrdg^-D0LM!>!|#+5XGA62kS zxN~`rBsHXRVXLp?*N=rx`^S9pf;?;Z?B)@}g!zOLiM&^jlwMC^$wr{*g#)8mfv~^@ zyO@$!4knB%LGe#x~j#L_|Z^5cRaG=)i%i;0^{x?%c%X~oFQnH<4(^iovqfv>au5+ zK)K{e9vUjS$0Phc3)xcg6#3LC>vpnTqUU;dPSuCSy{W-;(Ss}ko)u^!!4cM5ucdtB z?RdJ^0SuM%@Ebw9t8FEx&j_m9#u(4XIT6sFX2rH|)2Tm&c9_QmF zCdNNk(NaUpYmGM1$Uk^pJ{(zOnswmnB;@f!2oD)Os{XO;Xats=lUea(+BQdojTu4_ zmO3~6-0x2<0R9^rI;k2(9CqqV45MnCSGyHrq*e&j+M5^pG} z9bhD-W~*|y(B8`J=kodJ(KYVAR<>QUtWmyFdqry<-fCU_mFUh_txrYkTuYCom)zi+ zT5;29J`)TG7V_LL1ahv;(=-J~T<7L`^2U6#C_N0vXYMwQu<4DeKbbD4h{MsoQ(41g z<;|{Plc(`q)}*z1Y#xG(NqQ{tm>kCOgQF{f*S?>9{?iDybKmmqrBZ&*iFT%Y#7 zBQZ76bZ`%Hq7;veoW8u#au}f5b?%!N>KM+KAU&H;=5{?l$D!On%}i$Tso$s!z>T*2 zQXu*SYRxZV{A2gcPZ$Da0SItUkvh+tbN+AefPZX8nJh(pC|MXZA!4|kSl#3dtv#%H zVK&a|zSOCAvlY4(*^-%zpW}(z7r_SBT(-w~rHf$m)^2*O^ZhD=)aB6vAGHpCWAbnv z)YUG95IHR^6%m_0H~Pys?9Hmz>r@7F8Ktx;%Umb&CasbaC=ztJ=NzVV+~T z6D{(Rf>xai+(qzbKMp!3Ip&F`mfpRbY-_bm0NP3+OBNRFhcyp`V3kLQeOu=_4-wIjQO|;nzM?eQ37Wj448ZLdK5T#9~Mho4dFsN zW~qdRrb@dCXPQ%hdXRqEAP!r`o{fQWKv8(|qn*SCuU`NhJ$x%^J ziIwhNS`PP?&mr%-zL2zi2I~h88iIjG@8f(H7D1&MBO6?&XJ)?U9|;MqcH@m1lwNo~ zeEwQ-b#jGH3Z{_eyi(K!@~55|3L&_JLrNxk5uWf03g;#BF0N(NuE+dE`_-ch1}_O5 zgS%GBr0zSuP#W&1p_2-P-QTSYg?$T)qMiK0pM5+!>!aukHAYvC@+m9X*N%4HU0GC6 zvWcd7^mfV;*ZGCent< zP$BPbO{BUi+|A8<=Xz)9#F%-!v}A{#(UFLz_f-RUBW0b6#qGVN!8Kw(Ck|eUuxMxO@IylTrHQ-iUu=j=#I}gGH|5t3>(} z0qUDy^ObLu%=o(Q*jfGxu`(u#Wbl@K(Rc-9&OBREF5D#D$hspvvZr_9%Iy;d$+=sR zL;6*)C!Q$$PHT^q(dk{XDmQ=KopxgeuNY;AfPrz`tY)UM$})WE=MBlc)**M^yMn@#*!EUFV&P|Ha1{2ZSzC?WV#ud((L+CY!=jPf?TvQ*<~>58 zW%3~I#ZjgGPwj%uE@`yQ*NCvnqqV?Rw~{^7PGb5o!c@1#A_&g-=JAdT66?ZC7c!c< zwUs@+>b9gey1iv<#7i8n-^Y$y&w1x#6U;j!Z7t?;Zi4M=1tfB?xT4>U-i^ZNdpBFB zepqng%Sj)NxNC^;^zyc!7;$~Y;1WaS)sP%onUY;wjBqHd9UgL3o}X3BL~dEPBL72W zB56g4n8P+y(ZY@*&6B&kS$^)hLBG@=PT=Ss_3}w918R0w_l#~XSs6K z(|yoSTA7g>KMVJUJ`2MK0`h|(?ZaUGo16l;KaL989?=C49rx=E~tbYyV7SYAAywy71SIw_dZoEb8seEt~ zgwZ^IPu0$#8|}s`i@1bWDQS9?G2}*1aOCZ@;M7ppR=bB~xEzf%skF~;^=MdGwO2RZ z%a0=-Hf}PNX^W(V%R;x2m&>GmJc88yJfyh{jr; zemBrg!f@12=Z9_e++hMGEoHH-C;UOOHR)`nUb3xvp4}m>t||q6z8wvcyx>!BPIX%U z!~qdX8pPE(W)Oxaw~}9HlJO=MMI}|wZfNCQQLk}T$yZ+2F;N?|-Q>fES(RmDX;&rY zWwhpmrV4vT=0R1<3M8GEH?K!g-qXM3Co5-D=Z&knH@9N;s$cul{$STD-c<=6(!52x zM&QkkHsD8r^yUt9p^8WMx(OKc16LB~=!8T))m$HGnr#+GZ7|11N3D>__N7X(^x{1# zE_lhtm+K|Q#ARR;c~~jPH(a9b44GLb?AOnJ#yt~$^C-u#R*HoZziqKkP({~&)xx!< zDWnYE!Sm)IN8ZjT0Qr>()QSYvuP)76vuZN>l3U2iSRrBgy8ZnNcY23^QMASniMW@@ zxs7Pr8cTJMSz^h%km@a~y`Ptda;O=KmQc?rEEDQu&emL@-Ri%i`{fhs+ko^EapF~d zt?|K_AR{85UVh}4Vj!Uy} z+y4#s70p7J4+C3Rw$D)O$F3eQ#g8mXWfpNP4Oi$HFO=W^bh~(R*=GC^T}B;+_&{C` z9v}wiwH~up;8V z3nL3Rx*3ZSYgNz+h`Z&Ro76v;Q=}b=TFBSmI!t-R8;QhSRxS z#}SZB_l)@Ag9Dzg?}~J9oa?J=Iv8J$M8_-N3@H7es#KoU%G^h(yIz>lck{d_Wa0{4 zzGDI@WrEh_uirEqB0BWS(6f2=s;UuXIUC|=bY$OaLGD{ig717CxfWf=rOFZvPCh>r zc(f_4d61 zTTuo&rOZsFo&phE#o}a+P|cMqI4x+$ks5`}4jtC)p6@8f?k^U#9#Y_P9gOxRCI&_5 zC{ZLsFD|VuM|U2|;L zn3U}^Z~-w5!opSmsT%CsaYP5&4RT^IHRwLgWQ(pLv`2>BM631a⪙~lKwWjjh{Hj zUM6ucd@m=Q%m@~Qli0sTSm@f2Y&g(y*TS?s=aKK1T8p-F(So5R&2fZbn=lM%8ib*R ze@Ks81Tk)Lwx3)5?M0CvH2BGruGi~3Oc0My+am@>)Jy#OaW@gqehDURSD;-_2o=iBsZ3Z2*7zx4V>=MMFR!-c zrMh`c^XP|8+xsDpZbTQ!?<<9m0+>3r#P0`380^%1gs9_et~) zYCfb1d>r7W+#c)&Ag{^ihFmaft$EoHSbDn)M$?dIGZ6`KB&F7;|2U#>A7AMR`k z+Gw_!TG?TB$~zvFVutqQ8->GbwDs8k_3{tFog!&jIy#2C3G&{azR%~1B9GoGz5P}& zhA8(Ww?zn=S%ohWa8HHGtHxMJXOx;MoTGE+&5wdZ7>Tt(P~>S;Y~acbXoZ^Y!S*!< z%CyeaXnZCIgUF*dUKz)vD_3CpuiJmt-MlzeCqEwBov)*=qWjT4y1t#)}2xBDs!N~K%vL_2m4|*8{*4%QV+P60f zma@0?;wiZx$sPkCrK&&mQ@8U$3{R~;C$ESO!B2Ttt#{P)RK#P1*pJe^ceTEkWBZUR>7P+-SR8>4j(eJWW6rjYX6mF zpor@&eB*M;(*sw=4wvY2v%1FJN@FN${FsF$VWues10D}nMh>Un2!?I>jUMNV*6P>x zoeEg06hr!&_ay8YFBH>Wq?)CIHTq=HGyCsx?H z;??zkMQG7&ifH8yWOnG=<@&KzFdgS+JsZhaV(yD^2#w#ZZyVEqG!BG(DK+A1%Jng< zT&I1%jc_m)V{%?UTH!c1hM1+O-QfF&j+KK;j~2OoODSt{H+~`JM(b8t9dUe#QP)uTK%A+-t8-LayUUO?4=7Q zMn{X(o57zx z)rdX3f2?O`t)BL7V#;8f$TOnYT)aolC0VL@dWn|{9@N4TV+un;OAob07k+S+6m6sx zsCfHD-Q*f;Ybj99AjYXvxi~nQ?+#&Gd2Z+6&t&S2OcA5&tf4*{eQlvQV7~G|@N%@ttmdL_(iX5oy9%?q5E5eyM{uaw6OUml!|I3P zI3(LaSau~=Xi;i@z9$v^Wj^0QVl8EZ^6rz99IdpS!Y)q@_V$bc{_ZaHY&29Q-Ud!z-=dVs|7ggo`XcZLUYL=amDJ>4R{JqUM zrE9Qnpky;&$IE@NVF2y6bFJ9YjC|Es=xR8sRt6dVWilq|qLb=Qsl&NePIA3i((z>q zx4coK(8clBU80jQaQFw_ZHU%K#(zSKz=y~(s_Q*TLzVB1=(6TDecvgtUPukxG44-T zC{iNpx`xQM6y5%*bf-sG|iMUz4*DJ3KgF|UwG+;Ag?*`H-qMBsO9Re9DZwH^^kdOwZMm6xCohh430bhO^<|R zp))toR(aV*dyj%_4Er}_Yur>fp6ZedW5X?0^LA{OZfFa}Ezb0g+ezp|4#E)IjY#8iFL6!dYnPPG>H5wdqm@gWns`c%VqIHi zIY#^0v2ir8UX3GoDl0d@4wHJi_p5vthD8BPugmo89$IRy<(a_%%A;4 zSkJ?t?v7^DqFWojlt(`s+gMM_i-9ohw(gG&Vt5$uFCXcCFxReg&ZrPS(J^$5*6fJ7 zlE-gf@;c7+CK+Lpcazd^diRGFD#It!Jh@uy#FCW|s`nI5(?d#fWE_#X)@DI6ITa-V zq5U#ftc)03ZEk`u)ajD+=bD;UWN~p$ujBd4;sLKe_{`WC)h1~WETQ{JmK}RxsqD5$ zhdxd2emSI}Zdb`F@!V+W_!HSBHgUtrl#zj@90j+n%nv$I6(1q2d!s8UdgExfLvBqn zS+I=w1>>)mj`_21MYja6RQEXW4(;~0tA7xw?WlA(SRf>*SPi4Mk2+Zn5gR5NHO+Pj zh6QM*9MFUyOC;%XgW^Hgj2(Zt!JLPFt8pzec+9X0ty=6>pd)v##tfFV{hZjw*9 zyF(jindY5V<6N;!8H-<=TmprfZ(;rL)=hC&)wKW9-gkvXl`Y|##*7FeIf+P6k|LmF zktPaAk_-xnk_E|8L?nk65y?W6bC4Vr5y?sx1Oz1KoZ;4X&diy6ru!Tp?%N&a`{tqD zyY>pTs@AIY*I&&jqYWF&-9jB3as`th>I)GEF5Mxn5qmqdmi`QK3JJ_BgM$)lWlvdb zMHy_J8%)u1TU1a|VP?KAs7Tg;pJo(urxL>2_Gh9qHkUh_Hl@3&bLt6mO~WQt{NER*Ffb*tN_k)01ZSVpgdkc(sqMfa>G0-Xf3Lk9!yKiBMis`0TOhG#<8_ zqJ2L8`-}6Pch{pd_TxpuSI6z zoefCyN$lHvV_lwh+&q(gCQVMxjmtbbqb^GqRdZ^GA;UqZ@B26Q=kkz^xFWSRyPIJf zKDN`wcBwFZckf+OVn@gKz^)=%VzhLNkus@}MP7W-*@nlH4qNzU+=0*Os~P-~ft0$` z;W9VGIPBMZM<@M^#S92b7fD}uMtV45kD# z%0i?@6$Fcv#j;n{k{5(|de(X~3g>IS=*^qgZpbRPrQIU4oHFus?%J_%Hr|SJX^QPC zOrS~*c8tx{^TE@VDd>SQ%1ttwoYhiuaBJ>*)s&UxJzLliP{&Yf>sYB3MbbfK`9R`j zZHpP9Zpdc;;B)Is5X^GvatJvUo&ILkYv#Gva+xtI&`SA^(RM<&VDxtjHJcB&?=Fd9 zPKvKc1eo^@Yqjmp&gzaV#{7A4tPh!-9NEN^pVYNwada|&; zDvJ7oNzk1vCQj{XUy+^*N-7JvGa1i$lqK-x`w5j*1@gF?v8!GFW zrYX-9MRxK}?I`6ZGLdAO+)ZnBxsWa4HaIsclI@dZFjpKF)iASFY_+zWek=5WTiRs6 zqR;1Jd=vS!0%pWTx`CNJ*G{ZdAFw1oxgzAKClJVFoW6l(iFC(Y!KbTB?$)-Py9pZ2N$th-kegOcf?(9fRf$6ryi z&!gs=JiK;aG_|;S)11V?TlcrA7%pjf8+A9!H3P}yd&+M5cJ03@NA+rT6tjAl#xp84 zQ4kkQ3{krTC?N?za%1UdzuL=ZNj8tzWNYQ6_LsX{j^fM-Zyt}9nR8x$zuWbri1jixAQ#{*6o{h8n66->lj@AUVgIIFSThRHrP%zUB*17zgUZbw5L=hK}o3ddi;_4xs%n}by|wo%pBf; z*DB25*5Bodf1c!HyeSXjOwEYdJ9h*vuGK z6*=GuKb8b_2N(t$&ZB}?Gnt_^T>ByWW}llg+|+K}R%SqcK2Li^N4%v1u3)-@pCC6Dqi;W0 z_>0*bv}g=$krHyv-W6>S@Hkw97^Kyvlng+d zg8+SM!MH5|x!o6r59{98FR2Ky;Q#<)L{%s#0@{-vvUrp8xI*CO4luQ6CYU-vfEwUJ zA^EnAu$Ig6U{3n$c%a?8B?`AX65=V)$OcHQ4(%9#&sGruK9+g;1uwv>5thy+A z6_)yl_hIP$i^Fgu^&42bMchZFb z;~$50z0r~p0Glg^_HD8$+(1&QpnW-A`2dl(0EqnXEKNSZzha?%wNdjxYCihXkHX92 zFk=&7V7~$Gu4qeq1`;BZ0TcQ;pvwfXxh%9-0*f9i`R{WZIsWqkQ&yU?>l5Aw%i&;1 zE#uII_I-%L7(x3~pnYZo#@7M;ewLInY;K^$f^ZLkB>K-3J1;>hFWk=5JzpCz^(or8 zkxS>iVR)^Xudwa@PaL%2Uz*hTox&?2stn-gkRlJDwc2)DXvxuZNbpvvQbmB+m}4h> z8_%65>uNziOXbvKa2Ws;6-LL%VPMr!Qj94niZ3|D;K1GV~rM z$18Ost(k}?1pQEjX)KGZ=u`lBFHECcG6ExU^Ei6ar;+9x%OXWn+qoJ=T4!eehj)JP zn|&)n)1yR)>79A$`MFx_(w`kX%O~X{Dh<#I0KorSOE||5>xLy|-GR&U?{+Yv3)55x z!SN-4u!dZP^kCud5=gSiLXf!AEqWzR$4Bvs3%WD@wsQ|0^V=@UMFXSJi--1$KP1t{ z#Tz%6_@#ZG$qii;D!HW5e?9j%1P0{$Z?%5@O}KHysr?j;4$wC}ioT_i!UXVr!c$N8 zdH#U~09|YE?U8y?bro@TX0L`MSJhPQZ%&B|^Gc$FmnG4Vc1*89SZGeg=6Z3&UBjOG zf>6N#vLZ?TqLBtwcpo%n#b*%Yt`%I|O)Q6mgVNH%^xufeNI~=)cH(CDnSGvjgaz7M z3T711eS!H#Pt@vkMsj6dJc>c`cq4vSUDVsaafGI?5opfQF!Lv7>=4cT0FlGT(FsRQB`rw}dvlX`7vm++lEv1T?UU)jLv2NZ zOfG3K`8)wXJWr%?yJR`4ANxsBaPU7maI#rc2I;^D=zzEN>q+RqN$3FCPI?G*KoUC8 zoJSOEfqIXzpT?M&JLbtM&_6JYX0B6UR-UEbnGKrHmXJt_*IA7Y9pynfA^m@M!dd}A z2_8xWxSL#TXgfg2Yv69wcghkEI{Gi5+^0Rl24}qCrDy!Sf?CgR#LT zOPiq-bNzu4p*oG)=_x`MX=)=su>p*<%$DDdxgwb(btjA6UiW#Y+yFHoErmtCO057? zFvQHT$a8zSAahXu#7XbQbLMElgf9*Sm`XPR?_h&!t_yEtt{K`m@pN>K~!#yJp)iOGKfNFijcp}6CmvWEettyX~9ZVs$?a3iiZ|kMc~KONZ9au2U$Lyy(m> zlKS$(;$Y4?$8XS{%b;j_(`T&#(7yl+b_iAu*ge3s2WCnud4X8D=)V}gXA=KA|6m@V z_KTm$_QRab?)%jw2MZdv^@&9VSQVnQ$n`iqVEZHV3sbHF&IU*(#`>#!#Gh0xgTt6u z2_px3jD7oo?eSX~|t2Pd@ukuTB)EljW_fd=2y_sKNA|BE>SwjQ-GBT1yYEt-F9*qY`VW_V3FHv$?PvD0fKk|Fe_<7 z`Og8<2@}dWA<+bLE(WgCk5^IgCtC4O?+aFnvHv09H8{cZJ!lUL7kL6pQ^KW$q zZPsZ2LJh434Q`2};DVdgV(8%GDfyXm$PF0X((=~r6VSfczZh+B?a$rBgW=Z-SOPnO z?Faq?lmo7PPPpSISne;b{c=k;Mj0%JCbZA)NU9*hKmjxLHA@{d@FiqUF|S3JfCg&( z#SVdMH>nPM4tKEk!SvLZC(S|!?|=Z)E+bN)(Ex%K#$M}-kW?J`Z`ZyIv@#pw*!j~S zhw9Ncvbl_a-QRazY2_cwau`t`qhF9Mjmslu3Y^Y=zWO=`djFHh$ACYb%O8{++hbP0 zjq-HN0l|=A^z6fITuh*ikn^Gc@c2C!1y1Tt$iT-gtmMx!Dsr}0RdEy>0Ac`ytjGh% zipcPdWpHs&X}N94tiVR;VR#ZBISz(-P&Q&fpoELOj%oMv%ufW1gso+6nMkb|0r+TB z8xE-c&rg6db;g-HEkcI*9*#_Vs{LC6M1H+LvJ7 z;i^Z%p$A$T%lvFZvdK=>7;9;FXvO$Xo~U5Dn{E7kU+y9hENt!nN4@?vEWfMQ+HSmr z%;;}81`A1IH>=Q(1HLa8?wW<9W~@UB8^^l`&d9h~rLltR!?k{nrk@!%AYmxc*O5ef zTPAbk{GxFKl|CdLHr9I2+3% z0+xfIuZ)4!0NLtbmEeW!c5xM|q0K%D{XkhQi6}UQ0~QUdkWNUr!%309|9~&>PpXOX zWJ&s}foI1mffzjCFLWb_Csm!2bWD5^k`SU}I;)Ne6jol|Fm4*k>byT;*rEtUw@l^!3CS5)xR! zMGk+uJFSWYpGI(VwHLVSJIO*fE{L=e`!g{wZgyOn2r55i<^hLiY)cWWDZGf#aBk1^ z`YtU2?c!Bmj=Zk(Qa(WPU1}zWb6;_ETH|{!l!F9b&0Ze!(I^r!Y1pCgRv#%c{EZSRGSh(~L;8kei+6|rVuB=*n@IcOW+oIndokNdm z9;gT*!dAiL=Qjf`{NJSlqAH#F8rVsI_fep8-V#RU{|N-0V=d9;_B)$H+fFX{P*!YxodvrQ2j8^}%%uoL(24=bI){*G#TRiB)oJ=EfoP|(k!nY+N8+kNI%HNdQ~Ox9ig13-)VWBHxC z_<-{mNEcuLQq_d`Pj+Hu;aJ~zQl$&qu+`TmQep%l@o)NA;vjaL$3cx8E(mloATbc@ zm=K@)c^36@6O87M$Mvxs@Ri7a*v;ziz7)La<b%Xoiq?J-?*Kj>a(ALA-cnx zc8ftqIGk=RkTR>?QTWj;v@;pMJqIPV#;jTW4;l#E#(7#hIX2I@9Czt5^$3BhTUk)Y2>Exx_`4Y6x$+%Fl%yW85G-e#oPzL{jb zW;dAj3O~Ze)0lemN7_>5shTcL&LoL@JMJ~CbB4Uj8xs|=)^7BipO@fm=Te`DZyn3` zq-(=2Rfk!p{q}CYV|EpjpfzX2+hemj`PnWc)uLD+(jb0E!RdU*E0qnqjq^3mTyv%@ zCTW9Fqk1u}y^lF7xaP2{{uAM|`{m;cFcPk7IRf{@R99@P(V4q;Go&rcF*`*s+3sY7 zO3m=DYTa*ACEA$W=J9NZy|DSAqM#q$t###5rnSy;UVX%KcgC$DXqkVebp-OX#kCa2 z`*ABq3mA@gOwZz^JiW8?&OMS-854PPmy(L5J_g|`RCk*QM((;#ykkIUZcXXT*x@TR z3uM(KJ>m8Cx~^k;V*Ahc^%AgUe()Xd+iliqFuoHfv0U9@9$r~|hwXUSZYGmax^@zW z;P9N#N%2Y%6*;rqxgqghhlWckzBC>*${O0NKKi>ItNkVJ$4IV5m5{sH;aZoS$GzzC zDN%7skFNC#y;peQFxMVpSyj~M%yRKyZA+g73ER?q%K$aA;N8^E96juzoqpnI7zc;V zHjfb=z4Ic1lhiDS{Te+w7q7keSRExq67w!Za$r4$Ynn#>g?GKo#g3xNVG-AJ{psqQ zAw30Xx}7MRmo0hB{7nnJD!f7$X$-3ue8ZSJa5nJHiJuM3jC9~Jp-OpPPLppFUF@`D za9JtFr4-ci zp%d6s)P6J`lSVClmQ`p`%TPPGf$E;a_11i&#NZ0{ILtsIv@fX|V>0C(HEQ>^Y(exI zeS)RYh}m29MiFf51=w}oJYiC3kf$H#JA10pYe?c^NKannNWWi~=75)LWz+v}y%LBh;yFFoA;MK7N8T+?L>?aIKhV?@&lu(7`cd zw^8uv5hg;vK+Rb@SNG%DSs8^dC8hMvU)gye6hNJpfAQK?Dj^=O8HFQVy_etGttXM)!^<|Yx((mnC+G(%qP2` z*_mYCrR_|;feZ(`u3n0ZRB(BQO@*d|U@i8_(=!SBOg4HWt|v6@U1nk>mFZjwrFFo1 z)MfBU_2G`;;eiF#TI8zvA}ZImJ`J|H=P=5c5=npLJSd=3KQ;HzvK zwUJP|^KgQe;`o`c$l_PW*gV^kYU3mZ7j#G}clG!Bb;-TCyD2hDl!=&MmJg=&&4@=t zNWDw6Fkkvokf*{Eg(ql}Xn}{rE78h>x;2Rt=#1mo%ENu~Ia(htI_2opSIHdX`q-Tt z?YRSSUQBd2%b1ED^`eWDpfg*3CMuhRja+e7Db*B#pO1Gj_#KiC6fH$Um;1ld>*?939lzG z6kc@L6a@A=v;@dlz3`$CIEQXC#2$yJV)(`JlzIP#274evSmY24g(%JwjrVC7E41jh z@nhbRvN)<4asTh}w9k#fu2}n!#o^A^91@G zX62x7ca7rEnKNhF6cv3>oM_nCu3uc7yOf0?Dk~`8?lSIe6XR1?)7EYtBOxKyhO!zF zn)pI+6NA;29I_Fi<>R(lYmS2Y?wZxG)zU^nP0iP^um{E8+@WH9LYwW}H?lHo zD6^iLjpAEtoj9^KC)6-v@W9{|F-hpshAA1jyds7D zQeexb^Zonx?XOq9y~@M41;;?fK$;_ZdKe7mCKT|Xrlc&hZuoB>>2-I92Ll2E4(sX3 zQ46_1g@td^CN734y9`7d&i50Z$9;N_)ZK2+)13WM?fI^|yWGkFx~UI;cHOuoySSmz zU%q^CoUr2HUL7B-hW>+)_uFMmx$c227iBKxzAd^Y#KQx{D?oDHa4nSgB5g6qyAg!< z^sY%GZoy)1tvvI?U#H!V;79YjY_o($IBZzDm}M?cvz$k1E0TmohdV$g@-*3a08k*m zy|g^>4B>gh>Q1x$51cs@eCAAXT^%&gcdR8TDRHvYtG#|W&qas%$E$~BNbkQv*}cZb z#^vSZ_LsOt-8{hA)|SV~`8Y4{#L|*AzgWByosq6Ax;0i*i@x&hOC5D}bxlo8)p^{S zsC3uRB4@d2+fq;`RIbVAil`|Htw5rtrp@M0uGTEM)@r+NABB#j=nbW@a;*IU9e0Z& zW^(4#89>DsP{*A02w#sFzk+k;deo#;c$;sd#8v0t}Pp#5*%`R>BaxA3EG`YXi~?7jX6m=u2YE1n z8wnTAWMq{Ups$>fi^;4c#C4qmdC08uKfYi*HjrhuIZKy+MM3|*{Z|fjR7H|0LYs$e z-jzr`8Nfu3j*U*1`nHP_!2pA;NYgnS?EAOl<{!h%$3Fj4)p9FJB{D?O+pRBfI-;Xx z%x>ZkZ)zAxOiCmk9b1x?zSPi=W$NPJRd8OO-cGSFbO*at-{;mtaP;FxM6Y2bx5iJ! zeUg@QClzZbs@+ zV3W`uP&Gb2ZpR%F5g8d4t*xo5+B`O8p6$gnv;Z0S*oQ=z7G1!20?VeNDMX zSDBR#C#=V{d^oPWB+{RNSz)Cocc$~9UnObu#=ch(p82_hNU_G%CpyF<(i8AF%ved_ z>WI12ulfJo#8eE9Y5rF`a4OKBI~n7oo4h!>Y^R8902K7(k>CWX_iKdfYVwtv&RQ`p zeH2^k2?a_UavJ8RHD(2{PdWW9q!&iXXHIHgvwtXeF@*QqN>bdX`Hr@j@_`_!gdX-9 zSt%mP?!ESYrF#WfH`hiwhzQ-C?@ca_E=$Yk#h?Ho5)@mZ=S>k&bkCkn;CZYCz^bN5 zkbQ+iAf0)^ayK9L=C%9%e$HwXZV6@eT%Eay_pG~o?sh}ySSG%*H(A)OZ&VD8sY}xQ z(G;cO{!x65iVBWwhWwT1!^{Czczy$A>nn^~Yut$na2h|NQR*TYdWX8~(W0t_E#BN! znysOsy9^as4~4}v%ng#~ZfO`cfs}(WP91rpIVuzaB}OxgZkg4COo8149_LwhQeWCE z@AeOQCn%(7$ zYGPI4v2dwcrL;GYJG7=>$YC;R_g2&Pm8{~=yEyi^#)m}F;em8JeYMBiOwzaV#>T(y zcCT&SD4A>D%_d7%w_mFr(56XACtGnJ4GiEWv6PsW8y?Kuja}(ATu*k)UhHrsIwojp zZtmk;v|f-?JgiWhPGpwv|MWx8NQ4;rmTQ-|atljGyGwC9eO$1*HGhOSdY4A9x6o*1 zQHVUEx5eXf;-k(l8{CE47@tRq;v1x$+eRA*QU}YLX%GMXFymZ|nIu_Dn41$?Uv;>q zq&r_)-BEhOqRFl!xgS4{H#^HvYg{5hVJ^Ca{C)@%p=gZSPGZhTZSP<`@P&&t@c7ap zK@!=S@)lPnKef8}tUXTNuZz>?JLHUlnR2h2q^4kar1&(V%u`2d^WCN2x23SXkkz!2XjEUO!8;(HzZHxnc25XrAa|Ngr6NDIGC|*pth*PKaz6g`InRnSanO zD7oD?Bkogvb7e@NRTHz{-sX(kqFmg&%OOpfCC4X@3b>r?22XkubCIOlBBLnFso`yYcQ7`zbaPWEfP)~+0us=vfUc;9`#d$)u5 zc}PKP#qNA0T@9;Jz&UOv`b_~ze9cO$%&CTghuX`n6}meZ1$NyNY5qiH*E}EhiTvgsRx>2*4^c%>lvDxw)cPo;3CaYut8qcM zO;eX9FR9=Y!m_ZQ(A_BtT9KBjy<=^^qIdM8C!6hV`SnJ@dp5`R0#*X0%Dq_AiBa4= z@`K%-F^h*I`~}2JI%bE~`*C<5)?+`v7>B0I7>l#7OcO;{C=NbO(>9EkD0`zccWCOw z4X0!^j3()y6z;B{x^#Hnq4}>nd+E*HJ`_ei?G3|pfk>e$gOgXgH zkGrgI%}ej9ge(5Y5<|C7VtksI7vZ>;#~?`sA77=s@TGlI(e)uhcYf&fs`U3KN@;YZ z&iw3Rzdc#ER=;Otw3BjeX=A1#>C~NOuGPCa^<~Q=a-08%NrXR= z$gIfJk8URPX(M~z49yJdbe6@On_R_Yij9&xNrlmMRZYhHw2A6hStmS+&U)Ap9s78M z&1I){e?G%m(Ubk4CynX*rT`A*M+z(F68I!FqiMlt9?2=sjb;^2R}omiLi zKDY3n!5Niu0u);1Y#!hhu?cH`hQyi5CjFUw%2Hpi;Mi?5Ae-KJ2D1;W4=QtzN@Raq z0PxGY?=QxiDoOF3Be9<|CWU`(=d68Xe;YG)m9Nr)wFu4qp8>qR8oTjftUaNHs#q|< zzYvP#J^EVK4SQF1<9;oIx5NJNCJl}$%J90|NtcG$kR1@Kngi$bgErIJ2Fp<3?6jL6X1C z+uX7;IC8SFl&!b8PJXYjIF3W&_Zh-Z0sejEk00pk(^DJ@j{77gpr`RlTwdS6!C`2k zt#C@r;w{d1&m!+`tyRvQJcY!msHmt~3?2suLl8yGXZriTun{L0LkkNVgHbiMwePz- zjuXp$e71+p`^eX43J1*kmvM(tB<@0V4W2A<9BVGO+I+XO)nZu;F*pbjfKZL+V(sl> z{ZLBWMMn*e{>|*HO-;=ecDvgUg@X7N9ECskl@=a3a>TrE@xOfJ`|9`a@E}Cm;HBW& zKplQpK_0#c9re`8V<-@v63dj!0q_~F6_!xP<@tLY+&1h{)X+^b_@hQjBx2XC0-d}1 z`c_uU+PqYL{sIo`^J;870$lkVLhLXs!+BC|J}j(k%`Htb{>?JpwjKA@_jm3PC4_KdqIN}OOvEJtz6J?&S33%HLYv=pmnF18Mpjmq zXpi5ScMunn?x+c->cfX0A4JV87h!@RTrrW7vdKo5zqp=0pmMzU0y#UGw7uVUq6#En4)F?qtr%~`Bu`Su&N>qM3ZkA6%PqrZnh+R? z8`W7=bxjs$&dSJ^t`Wt&xRdf4^6~AcJFh~eF=DTl?2(#QZi~E zpkf(z8A?v#@V78sy==;t=|ziUI=T#~+tDVh_E8Bx`VrOmw*3 z7lud#T2yniB+yY7srG^GITL2&+6CLcARYZf0_*VWv)Ft7M7R&mN^v+5naE8BJwa&k z>~QoM?4BJd9QF$m>>I8bATTg x!$0cqm$m(mI{c#!|ER;ChWr18v-rNZ?_IJVUXWL=`#$)SM9GR~irjtre*n3g-NXO@ diff --git a/profiler/advisor/img/overall_0.png b/profiler/advisor/img/overall_0.png new file mode 100644 index 0000000000000000000000000000000000000000..f74cf2dcf131f36df9901e20ea327d509c6fee67 GIT binary patch literal 56377 zcmeFZc{tSVA3yp$tx8nZvR9T$sD!e&ptA2Wwk#3GIt&K$Xdwxe5JE_{5Ms>O%9?#& z$5^wDnXwPXa&GGR{+?=n-{)M{Ip_La=bT^H{4tr&=llNllc1O|boH-4 z{@k^P<_M4cPhX-KR!}OPidMhdnB`{Mk%Krn9lc$g$D0nJGfIq$BqdGy zU02Fg^xjNV%OaAJMyqhfrY4QI!wPdS_a+Bzws#c!jnagpw?ygtq)&5N{#=IqSnqwJW7fE_M!70s#(cQ1s ztW;g=t5e7vTu~)b*UhKHtn0`2U0rBG4)}>Tacn{_4@8?1(Ne*fJ9Xm4a36z|f-Fa3 zS28@$FnrcpmR2crTX(%zuXAr zjh9`S8P4fb!ZrQnW^cNlSsrH0I2_Q5)vxR+3RTFoC9#z)2pK9eVoaZ;cdXx5=TFz! zQ?}NOlT$`!Gw6!a7@AI%B1~t`Rm?62KgSLuYXas7sg@OK>F2*Il7{I|Gm{8)c*tXe zUT|edk?rMpd{y1uxS^WqtG$){l3YA)(<{ygasBeB*mOikSOfv5k6e4)QX(ULKl6B( zKEfS(T5SdDfF#eTi)2(E=P&CiNkFg2T&0*22iA#|%%~ECb9@}3uekIKO>|dTj|_UH zlezbz#3#z|$N1@W+yr9n3#?6c!YTM!b}&56O%`fB)ea#a-qB^WP=y9Og~pZ z^ZB7)G{pyK>a=vMjR;I+u^`X*F2@XHO&BJ!KER%5bf21ZcXL-7d78nDyXUCV#N{hK zy%;eFqf^&ivs&)=vmPCOobU`p7zU*xq9+Fsj!R)&dL+GK&o%FL=w zsd47<2ZMB+{hb3|NGlhIg@;hARFO#2xizHK8>+gja6}b{^2R8}O!x63-@5h6&MQd690V#c#uXvj^M-(h2TyN=bTl+?=U}X`w zlXgv@Y^JaDLh)_`S362p(o#g=X?>Am86E9~=03QHGo>&phuD`h5S6ROm2pK=->8O9OhXa#3)Qi!K4oEgjiWQ-TkbvWUy`OcPg z^P`O=OQX~X1-)0b5sxh(i3BHp{9ryc_xMO$+9G+m63!f$$QrIC-5FZ4Cqkx&MUZam z%=GM1(SEH{Th$#m$uE-P!@1HZ;Z;JaJ||#dUE!GAHfz0=lD$LJL9}9d=Qup)oJ*1VH|8!ONsaCMj`!rJvJWow70uY9-Sme;!yu`7Ae>Q{Cg zw;3}HZ(@LDzsBm8@f+yg&Z}+TSVd4>M)dhsh$U#_NabB0Z1tr6hl-Jif!v#XhFMC zZuC|}e&mf~eVhuOQV(Wa#g2lac-j7g>U0g-8Li(M`YJ=p|Yd#r5g|D z(rLw=<2o3U1A|V=O3$715qg8uPYHN1^db`RQgBxd>$MTftIrP|XH-a0KEvd&-lJ@` z`Y^LDWqy=ZM5*23VZ^dbjncgg5tPF2GY4$PM)!XFyA`QS&Ay6h*7 zKJmh0r}tjPFo%aYG$3QsrGGe8J|2wmVsA$Z$4+!=`MN6*i4_?2Z`&e`dyPJK*dPvD ziZ&C17Fu|0Fi4-v0-u;9m=|2fd7(wY&wZUz{Qwrq(;j{?XHFBpieK9^nDItR%!7Q=esg zM7#irmDQC`ZtEFb=6Hb!=VZ+-o7JWF;~l9q4;lj!(T_%P2q{Su`_OAc<27+Y(*#1? zlMEe+1<0`pkuM+e$n-F*T4PHBHoc0H z!?S0L4VQF8th`jGo5EyflrkD>yIK_>F12Y*pfwBGn(0IcZANaf?|09cLmZ}rGA70* zF?MTrRxu8Cfu+wqJk%GvE;)OvgFPj*Li?j7q#-lyGPqIA_%6O-C9BYQ&K978@&b>u5dc13r=>-Dcb0hhrgHIgT^C!IdPwlv8vAqm;Z=dCeDv@-V zMQM7!P}cy>$L$|8+a@nax~=mrW);-CT-lP)bHhY}YY{zz~}Bmv7ILpD{LLTA}sJr@{}a z_@brMR@)Cu1(!3V>D>*~XJo#~1oJ?=3f$L7AnDyZE?`m9L1TInlX(x_S0aO9;u?_g zZoiW9xbo@ep!tzXW_^ouojyw`M!V){OwB(&PLh}zV=C|O8UDsOojBK5oG|K?$*er0 zENf+w?N&P$4)I02&7HhGy-PiILeac4WGDyc6OLcB9gH8rOKga{*g4*DmSHr#!9 ziUs%u{}ACRnkNpC>qb}_<4KF-JEnW{VC7bE=*a6Tb;AO<0Foqi;eu%FzUo%X>193V zkai9}uUbi_cmakA*tpiZ9I2}8Mxcrpm)(e>?UbATXfzXYf0<%eM9+!i8vGf~yqod9 z*UlIv!qc{loEqK4QwYcpg0!)ev6btp$&^?0B>C z?MrB=L;F$aVAgqU`gEeJ?&0)vrsU4c7*%E_(%@QIVDj0KVOH3iClnaE6?(BSdr4}l z^Awsl>e%_t)fom6+|7+R-|5fsumikG9wlE`=>A$$=MOjxD^SySvr_0v3kkf&baug)$MGJ!WT3t3}Y$r?;kWt za^HVBajUKpc3{CP&mrUBQ+BOn)_}~k28@;-PJeXn`PtN_P3NgJU8_h|9Dn2)n_gdO z!`Ty=`aUtayoj@^kdtqO;5O#(lIfqttZR$-N#1tedLL8ZLFu^Jyon=(R%*Wq%EZmz znihxUc&69jKM6bMzt0-^Cf(lQX&$^N-HvG`WR_a_v@Axy7 z4MJ>l79q9@ned0x5q$yQ_3UdYsQmFc*f28^Ic$+g}WhM}dEKzkSh zbdD}vjMRF{BydU&A?3FF(>3#bHfOCrg!xoP>BMqQ>m+4qndjGavC3YN@Xh~p@@fg} zgIdB8Ljxn{U9(Tm?x04E3Bv>U_(A<5ZMA3p-d`S-tBmZ^NXv%_ZiMsPRy|;3P8@zg zxZ9atZ6K36&!FWGu?+x%LK^2bX(vilX@Eg;4d z$yFVwCx^U5$By)QiK(2n>Cg;bb4F+0>^>pc&-fioc>86~4HRA1UD-!RcZ@c9FT>W1 z`r~uToZ&YQWJKr+afyj)``E`;p2&xkG$g-Ds0PFo-*O3%>`YBz z%5IH%`g}Us!tG!tE3ONIcP~b$&j~NjFF zz>=VkC}yn{+T1i5iOs#$hDmkej;fx?G{j%I4@n7S*gd{{ z1BD`i-FjuOGw*2*?H@|pou+&AAlRwLG+K-eK--FL%S5b5Vy@fa2Kpdf4c}H)ZcJUc zQ+mpL4hk(6DBa(;hTENnJlb*mxdeMP$}SVzWTQIs&8%ol@7=)jFpJNwKlUX7>`U5V zTiKBv#KljM71(0hbE{f+zH!VxAp#}fUIk)C#dp;mQJ=_n760vW+R16l_u)RJQ?|4z z;^J~>$HF%$4aOVxN^i1C+QQb6u#)GO`cj(c-O@GH&xF`VqmV2Ewb1#S>yYI%X}n%H zMwK|2yOBq#3y(j(Ie0ql7GHu)_6@eHctITF-RVn+;jj>;=r*xH4qMqaGMsN8r+_)JhbE2)=dybkXV&-4G0^M`pcJS7$*r!!jS2(VM zxRH?6G&LQJrE4u|z2`5b>>dHDc zRp^Vq+=eKdzC<%)Q8^FG zg8gA@xHHbDvWIeJ`R}{xQ$}F<>Di7`LV_j?yi1)5Z?yFf2UdPXu9>aK9tjCs8v+3c z4|fm9R67(WMlc@aRlF(vQk%`Kx0pW^(KTN7nE17q@*LmRL{ye+TtZy zi5y&UG#_tEiJ?9q&kM`aM5DN~qhH7@JG|KlZ;nw}oW5t`d=<`*QH8;+>;vrfz1s4{ z2$~QsUFf|+6U}X8%B%jYfv0El&PtV{)p-H)MwghdtFDQYyb8mL!|ukr+;l7KGBFTG zozRStYdy8j2KH4J6fMCh6`y%~h6Hr-G=?Uffq+iHIj2L!v(F|>HfQ=l3pne@ zKgX0mG_l_=x!{_SnJV>1Ic-1v54l_iX%(x_o2C2yJKp+IuKY;+hVJlJ)y+tWUQ~)u z-Nmz>3S-$3tQhU9wm>dRtto2v*hr!hZA&pW2Tjt8-v?8glEYrDow(nTDtX2>*0w`8 zn*CNo_946c_i1P9GPS-q$6muLp{j00Bhwm`8);lY*6ptPxjp; z7;!YaZmFn*3UlQYPZl_hE*pWZ9kqkbr5=;&o6Ad8&=jl?2rcf{mg&5Ne&<*SIr2$1 zS9bkGZ)oUc*wpH*8JUBe`MPO zoUkM7(pYl_8gr1!=3+grdWtH-dED1s;?b&!Xq<$8F|05v{cs)`GE8q1`%9FHK9I ziL-4U4<1gd8`>Xmwyc*EmEq#iKOJ8t;|;vm?B{W%K6p3r{fK4T@ZswvBbSD82eL3x z`nhi_Et|K)X0~KghHKmniRAWAvnkVpR?FoAMe+7R>IG0t8I-a)P@Ctdv>=X^yV;+c z{cPEo9Otz47R~2L$*#Pa@pQQEtK0?%4WCjaGHP_Ertu!RVjKZgXjF27uH)$@BkAIz zjTd#F@O8(YP!jd+f)l(KT&X?1Hh6&jrM^xRoY8 zVl_gy59zWZp987*idp?q(CElep-(vB7`zD6hOots`vWsds+-_rp+5BfT5zBlN5Qp4jUv3Hs`L00OhLB!{o zL@5rnnNXG$)yLuCnk6<{?17yYc1kEsfBb>(AcMSFIH*7)ms%GbHbrmU`EGlE_?-jB zm)AlYY#5M%!*bhuE}Kcqc)NR;B0lNNS0MZ|Hl1~iGTxz~j*im%r9Ig{i8XTx;~?3Z zV(DynrSDpvs@^UKFv@U&K;A3ObH3O`tvivm2DMx)3N_7T~zzZdq;3v+pRFw8iHUpSiY z$-FDT=^ET_u`u|!QTyG@>ZgtW=cqK6YVxocwD6tYbEZ>2g#GjHSLI=Xm~BY>r$2A6 zSAHDiF;V!J=Xlqqsr~-B4rr1BirT10zP|ZuQry)zH}_6y+uxh_&kMgk_-j%RrcC9j zfA}>Vc;X=1&3EVKVyEBlv--m?ocRdusROl9d7fVZmE<_a(kjn8d8d;6Wl_)f{`qg* zRPOW1jNQN7wzGOc!G9A_ZQZ@q+X7U!iih@J^}_N*tNwm!dZDK`1Tt5P%^w>0&rJj} z`rOk)IOy%D4nq{#j!OUX{Bu{-|Ia04@PZnuA`aBpM#?4ObuXyUV zcFWH?&hwu8y?ey9rJ^?j1BA(fj~~tIp!`a2Z|_uuzgUsR^|xth_ibI~1E!|7ukYNx z{^NN$)1)PFYS+v2WbBF*;Lq~i+AvmgB)P++kxPF^iJvS&p3V6y$eD<(LlaKjzDpq>d>d|qBJ>3|q775o27p7*@;rAuDJ>f@?$m&J2DY$ROgcN%uP{ z?o>ta8OH=*C{?HxRbiUl0YS{_LT@0|y@hMb!w?er;osTNf5ucWbw4W*5hANB1*orC zb$iVndMrOr^@oN>Y?CT1+NWj~`{ulTKP_I~v`sFBVD{BP6e(oK`;Wv^Pf@$i3XGY4 z{6HdBw>_L~hc^A$xAGjpWkp{FLyH-wo%s2u9nsxDWmq42;-If?YD-O|c@FxG$F_&a z>NrAWQTK4dSN3(ulrEGsBa~divB=;>GRO3~F&0yI0dFFshdyB7JG5av!YH#iL3Bcp z5QO1nH+DG8*E?NK-}8)w)3nA!NKJ+7-m6DqGm%@ap(IHd=94vPQ#F}UdbY!k#7#KP z@sz%KDRyNxGlh7#nQM>Vq`2SkncugJMgUeg26iqi#)qf>{axP&a(V`sv;#zZ_(?N|7b#vQ-G)R5>Ow;L0ZCxSS_g=p7_bYrZ(>$4_~5LcLa~e zX+!oC4c-!^-HgHx&y5b8SY2P}eri?OO33Nfwqikg3`I%}Di6Fr=j>6Kn>p$JBASkh zHL&T)!yp-WhE@Wl`JMZdB0I;y8A{*C6lU1=z?TRz1!)U|kp1r`|Dg|ynLZ)+ zlXL$*a|33e3x>GYAcGgji=*#dBTHrb8s3={H(|fqPARzyL#&OC%;yJjZy6hgB@BGP zdZ_xZe+_IB+A=9k>m9jL5QH)~pP)CF59p{hqLK#z!} zZrEjV6>3*KFdapgVa)#3b(`i=Q@&6{;eutctM4VFeG>i2Vi@?H3&S${l30z)%o6cZ z^KB2SWJaHe1CC?a$62+B?v1q9jAgU@#WS+26Kxx9yYd{aJ0hth*f!oE|HjFFK)8M! z2$kZ>l$=hmye!p|pFBPG)(T5WVXSpLIl*~l&Csgxkyk79BN3f7V5tXL@}7MNO&+;{ z>YVna;ssg?f#8$Z9B`vjgOp;4Xh*5GLD$p;cE_}}R%-<)yXdaqCBmW$2k zubDA&_&CvnC>uN`9It`wg3d9K9laJtwTG`^PNm5C;L`$3wxk;#AeQr)$wzVgA7!YN zKP^Mxnfplcoz(JqGci#4=v!plGXCy5{@g_;3@Y?ldL17so_&vA5{RH;%jr~;lK=hC z9p~}CYJ2+yTcO(8?Y1+rP-+Q|8S}vp#zo5}fesRm1H}iWG{SxgRX?1HLGZ4!^)Zov zefveeEsPHON{Ls$Kqqf?SbVNxivD9Ktqu6QT+Mf$ieFQ#ep4`fwe_@GL~ zb?U`@?#s86TboQm@L@{C#g?jS!^$?7vb(d4i~78@5LBnYyvwdTYom%$okQB`ZCM%8 z!^UQ@{BhvioSn9IhEo?I_}bmOzP=!BpR(&NE%Nr9!Dly*yX>}ZT3lY<9E>#e^<4Y3 zpPFj#$vriTH1+MpVx^qM8+wf#z@24afVIASG`^d4d?E}_*Pk!}2X2Y|F?h&bkcNY7 zUr;~G+jUB7bA5FXa-?ubaIi_C(~0=9=gkw%@#J3$ zr|%L2NXVj0NaJ7%?RNGi!AQ)k*ViD$<|48>TzM%NtwO0dbs%j=^m$9PyNP+4a zkji&G{5>Lrzw{n`eT;jHI6nqAynjLpVWfa-$aY710%T>d?X*Dn9T8h_H&ACOnA&Zq2)KR@`piGFzU zuh^3y!?55hD$?;&0Wti)LtZb_Y)q-PR2v-qSL{iTw_N}9Piy%hpD572`rjchUhUS> zI%?AVc=BJdrw=zDWou9~mW!Zx`WH;{eWFh|H8VC-00R0SVDL|i`WuscL1UUgO+c2n z{}q!|QJ|dt8{_?_f_#1owC2AHDEH*i(reUe->=kqrPAuR$O3Uh2RckB}4i5eo2=p@yi_UpJ2qPt|d?J#Qrp>4PGw{d%i9wP)zi z_QMB#fP0*n7-t!u08>wrR@l`nz0fez(9pkxr;8jA zH`zv#&X2jb(tM(Z`T=BKRaI3xEDZP(2_rA8St}H{Y7n6m5BJ=l`T)ZpK#WgCGAfuA z*VNqDIgbjaVjzF+65F%sHa7x1U|MZ$Z&Va;HHvo@^B% zml47U^-mMpBG6mq>+4`S8TVRjXnxY3A4(249I_XdSYGNLG%l6)AsBjAVXfZ35G;b6 zYQzYMzz`s?s*#fys3fU8A8q&e2ly+B7y(CHCNAdoJv^W2QrWJ^kET+q=V>~Q9Ao1s ze%>64q!2p3y_;MPIrrME3mWWfO*IEp1_>hRrcsQM+BI0wH=0GNfQ;ClePwHBk(HQ1F(5OT#pVC`aYTBE6V%t``j&{)6Q3P^0UOTgM-|%Z+!Ar=NymFEXbASksfTy3apuc3HJNAGG=jFd!v z|5{GSKW%qMoi6P-X9Rn|v|Y`@YHQo*N=A6Q!xY^WHtSW(Y6E}V-%5>|*r4rnE`)JPFE%^H8LH1x{0 z*MVSHLQgJ>n~N`}5iMctpGw!JhS2MiXur{|h4|pccgw#TuHV|WgN1IiI@f1|(lp9G zpUX)p3ETPfz433C0zvA*VmJ=V&4FhK?8=I(P(Wp*h~)oTE*hKwbyQap7N7~gsaVd*3K?LezJBtEh*g%AE`jaVdSI{NA{pZFvh64WO(RO zr--21zJ(f-P`e}94XC81at7$RzMSrQ-E`OuhuO-_5Tzg`N3YRQ?cK~_FWPGc1Y<>|Woe#{ zbH_=?vR`2(56!{hiZTZMdE%6m_&9ofm?~}W_$D1~UGN*C z?Ckja{R{|?+rRoZ8y>LL?~|}y`k{}n=)Beq536kFj&P~qF%U+0)6&{mAJYAqHd=cTx-&Z9@O6*aAmqDHgzrnQ)d zAB!2yla%0`$5VAXy@}uACBpG+@$S8?1hXyqW7bLnB%iI|4WDo)@xIq2{6U40Nq4jJ z7%{z1NYZ9^k(b>YQE5pJkBNjny4ujVfi0s{E+y3cWgL=lKt1m37&lS+xSrYUXHo?0 zP|*+z#ew&BL&XEdh1~(}k2;M_l1Av$zcopP-Z(YA&|U7jS4#F^*@3U{vL&Yr2a&{V zH$_8IP51@wJ#(`mVD)lbCjDhqvYc01?;O8W|0VbEuNK843JT&wN~mUD?W+E-8p6>C zh{~cGEP|muU_kQdy^yUZ3^|wiX@3RbocSVy60w)_>RFSpH{_-J27T5ib4HblDE{Rx zt~Dki{thU+E&@hwQ|8%%OG&^0j?xuF38~DvukOj<{N|9b6hxs+ zRg3J%imJ;PAu;ht_l(m*lGevm`Oik-DK$BX*ucvshbHz4+{5YR$mNi93i)Pg-VR9P zF^*?Y1MvI=wHcx2K;wH!Be?Q26(XD=n5Fi}vz@^cH$o#?`^XBn)`sMk%NE85;PjQZ zh+l_e#~ki(;P~$?jjz!Y5ctLt(uT}?*(H6rXoi%0@vktyCBV31dMCwS)B<$JW#CIFXu#myc@EjCmmY0uQc)3;S)DL<8?kf(-B_f6=KNRq zs6)A~HG@3{!ta-Yi7i6#)MAFKCou$y8DvbK#2q|Q5>jp4zP2V;ZsRgJx`sc-`(Vn6 z@2N)@R8}t99C@0)UmX_Vb;8->SfHL;eXg%jE=iG5!#OFjRX6Fc8=2+6+?YB=xgt(D z=CUR0GIf<4r61os98XL8)Oy_iV^9)yLYC%X-wN%sDzD+=8>+wA*B)SBdP&$y%i2)ZMJD7;xdNd znNs*1--cw_pC5$cz1VfS9J2P|DhDp)&1C;P>8@9d>{+#ak({{dJfkpxE}cBYkY^-F zD_hwLHLrcZ6DKIS@}(fv1Uk32uXYTc)Geq!60O9u)XQkc64}l8WI{VdNx|)m1jW-? zy$bSD!#N__B};Z=L{VsfVb5HPkjT=JpfLdWrNuK=Epd5G$>D{x*@LmIPG25f#x%N> z+iJ&jXVcy@^H&00+WspJ#i z@vl|2W@oij9;F94Xh*2k1>T6()aY_4`bVx7LD10s(_aVSy5(H~ zaL&Wl5A?gBmvKJt>>O)E!r&FkM!W99*x%`~)`pI|Je0xhAHdxaSt~pKy8E}BL=f~! zKtUJ4Q8pGO6%L$#uPLa4n@bBkFb=PXRhkxW*!drHk5tDfpxhi^>q)8~>kxh_tGxgr zJKiDwhSRGQq@LE`GKjq*R>OBZY4)>N{;tWXaClRv0Vy*)hyr314kX>VtO~4GUD}E?1S4& zR7qt2qdilh0OfZKu0ck2S*AvwsGevRV7c%l5w%VTaZv8QS&wutMsIx$9O!85oyU!Z zbBtKd3MishEyCTrf~iVl|VDYilK3WYkv!qu+%$ zBYUq4(srza4MVoTLy==irCsN_+#K`1= zQUeBhABr%yn48;5+{-}R>Qv??KhD#uXLG)|5|)JRPTD7wwWSK2!iQ!BJKD-XPb=LK zt*g$BpvQh+;EN;6*P0RPmEonec@R<&=q4KzPgfMph~{>;F+Y}#nA{4-Yz78rhO^i0 z#c%r5O9_m$QnaRcL7bsL>Mh0tx1Q{4++Ei1d3von(b^QN`nveHvT3J3C8HI3GMKZG z6R|Nj536vC86X^3DC)?L5+GXZ#&y0f&eZ!zJUpz^@9q0-p}Vi#R2?~#))aPqwKNMF zPPdeZ^+0^0xjy?D9K|~J-Vx>6MIa@m^E-yK3>JMLImSGV)HF)=yDoLM6FF2S`;1}W zJn|W95DkhcG~{8G&V147XPdWLp+>Oc2s|d(apTmXlaa@9`|u<0au4|`U$s1x-(RYH zjqgpp9Zndfnd8*aFkfVK2;Vvz%r^3r9@wHguac(nUq_{G6HQKY%;FmxMLL6>Ml8xf0zBhyg9-{fjdcU6;RP`aLfm(hBNz z&^2NCRO))tNc2QJH=qR|>&j#3Qz9}3(swu(_=vYaPDv>(mFJmQ+{uwYlHMAdl`*D8+_$vxJm}XgdJmO+>GkHVk z4G+hMAbr0l@F(pA4IcV~6+%_f)n7(lAvTx)?gij{=J0T+wL3(_;VYWvBV}p6p$#*0 zmo*#f#w)At()iRVAbGQN3@8LxO62ykB{6iqOAgWzFO_e-VryNisnAs6z8W5Ph!Y{h zu>gWHkT1FP$;0^$1DTR7w9P(}mWIjqBWM3GcuQpvqrfU?9(*q}R`iTdZO8;jB3Uy% zo^z-MiZN&^;E*i=fLGt^9`ZeAwF;9BoQfsD!kC&YfYg@P)B*P*6 z0@^Yu;)w%eSfOjcS;o*XRfFzj?}>yszZd66xu!^0Hi=E9k241MT|KfiZ{w0V%p+60 zfPT>qqE)|^n9WN*PqkmOl&`rUQsG;n5lYRWsSplKm9|w_m|(^|%H|UWy|EK>ucn=# zPj8E|x4G8Tgo_+bu7+Ze!hrzAK@ z@1T)>gxam3*v`!OReswz!cOC$0xF9A)mdsN)^_QQKl>Zt0N^AZ&|e8E`0;(ozpn#q z>-KCg2Y=gNU`N|g1l05Bt7+H%VO9S@VYio_V2Cbmt zJ=Ss3zfe{8h$l6uJ^47JL6JK_iQ9)4Q!;wSjR|KG6TLR_t=4pFtsC%BjXh=>oYW?) zXJSU^%P;d734FMpevwQ*e3%YUF6S@E`a*5%K47JoN5QO`jmu;KqXAm4LO}->V4eW3 zIxsM>MVYS`QIB7yCf2{??+k5~uIHjqaBy)Ry1%E}6bWCQU(B}y^oB@7^I&8C9$(m< z;7?}d!PgSDThS!Q6^o1Eg8HRs*cs&J`fN+f=P1#QjSU|kpAzao?^_fqJ3AY2Y~T@q z+*)5<7&OAb#_Gbk-=_c+m(4r@3z)F$irdr;KwJUW1mIC~z=VPMfZ@URgAV|OJ^!OM z|M=o0Li;MI91NR>HUhW&v%yUfhDc#iIV}R0f^Vz=o=3lIYZEYG5|P5j9PI2RPQ%w^ zktgDtz)=4OStV}dfc2o|??27NIBY{C07M!L*$%kMnCR%}H;ex>DfM-90HFXh zy1A+0QkOT@>}{?U(B$^`Em_)ro}Z;h&y!|$P%fWx$)M6I z1YA@EXvr5EAU9BFh*I_PiZ(Y8(gXjlH<+|7<|{a@p5wmtEG?I(|Vx~%*CXwg3` zY+Jz0NB;}C=7*5P2%S6U@$)_3KvtPRHU9O`LtCh@T)$wU-Y8}4N~ z23iM7obH-?uLEzi)Y}WV?@r9U2b1D4r!;=NMd-WKfy(TBNj-7?(q#`&AZ;yU(`rE} zw_9X~5o(1}VTAnr!J!3lCqM{AU4vmicE5c}g_r^({@tGoT>*V?5DOTVN+$F6E~rrG zxced|2A(>wFSF@5Q;Di#+w?1zt)iT2qp%(TD+Mvey+>22o<-R{C8K~6^M6dOBL#F& zY};B9qb$;bkBCfZR`pHvMhur%8ouX+oMTZ{xo^KF zs%*LG=I^Q8P{>aHx!SrJDizn$vs_s{VbTYGCnx8XMc#nW=^z_;a4Su$tF0oESO@o+ zh%CR>%?-%eA|Y~WOl&8fO15)RXgi4;XZ&Jw)6;>*q98oZ#v=1|g%n|cG zVdLXGI5%MA%<1onE-fo7NGs+q&!|ycz>&6Ad$viWiR``Gt&a8T4|qHX98f2``I6PWWs-H*VRoVEwpgN} z7nMlGQ?Z@I5)vofadrX=tEFYovmw}d-hMv|$Bpjlf~{%ksQS>j$#aj>50|)|l#9mu zPLzok$Sjp31YD9<0Zdk}G()PN*dI#tAqED=nDimfMsi=l;1hUgp0+}LVQ(koCk9a9 zt#$1u?9U|1bR#A^p-^>u1b*_f4OmHY=2a&oo{?QjmhH62)GYT_sj!dEY)0AgvvJoH zjJt!E`KYTS7kk451yg4y^GbBkXfUBW4q>icQ-nLU>OEMr6*VlV*-6D8yxY-ag;wZ& zdi(oN?OM8eMWabV*11(Y&#`M+paS-C5q4jfhgJ>szG?v%Sqs4>dRC6Y+CImAdtynACUsddGq zd%3=CrVmP?_Qpo8{(JA+=uDi%JLJ6y9ipe{BMkGGqH!`GninsAXn`6 zKFw|21+8i}{oVZOcFcKPi6Tclt(K1Km>b_5-B@kiA@NA0rP-`eV4I(FP@8rZSVpXg zT#M*S(wTvLdmNA#Zvm?{j{a3xzZ>?cw&~y2v56S01;Fz{x8cJWB~%hRZg;kWV1$-5 zWb<2m(EhzwH4Mq_L!Xh3>z$*All8yS`RW=mNWjzWCf*B(%aX@)zIgl2%CBZwX}&T- zA5lOt(;$BjXy2Le7=uRfNIb=RP&qUJ4fQ-Cw^mLO*rzyOg;VqjEg_rpY0|yKj^;ZA z)qE-~-1;V;RVYqG`)LQ5V0B~%8@n(vtEldfMZFNoRbs*ZMCHlh*nqX5|1axjK&H@Iq#D@IWTI-CTz`$hp~R50njFy4r|xNnA9Mh(t|`mWZS(*NT~ z()`-Y%=he6@oxCHs9OJZ$b3L2`8VJ|OZxfH>tZP=h3B>7EE4WJ3s zB<}SJ=u`zqw?Ln6#r5mgA3g*ha29ZkiWUAPFTS_#!~j4SdX0nuit*3n`xFK`vzM{1 z9@k`Cwh6F~PEuK!^o2P90kGCogN~@e=gp>HC%|J6R0|$eKbI#G;22v~PR@3IE`7my zi9$lYjuXy~FzX^r-`YWujL?ALs%IP>1>Mhhs`Fdk%U@js`i58Q!UX{+LcZpJP(Qcb z;@S|Kgx*F<{TVJ~Ci2gz05>HF$b9Z|0Q#jYZy zxIsvs1L=iYrwTP4U^51%Vu}N!sCQa#6OQ57-+0LFr)}ev+vmamKh=Br-|P2L;i-Y6 z9jkc>3r5366aG9s1IACtWmSiK-z`jBJ#wjRULja?amJqhf!XzmuR?Ko8Xp!yh{>Vp zkcCmK-6!&dW4vJg+{g>?LM|pd<`4HBaP#hioge#@xpcE?+g?gE`Q%cnP>4jvO^>#Z zx95&4(vzkq*GHz?c*IO%kK+d@i)*ZQx4L|%Y)vQHGx_4)qTO7rNtQ+dYa)%oJ_Q*i zO`3rP=kg(*Onj8gOZ;~3w~yG%#a)IPU1z6~`pAPfbHLkXI_CoC=0`v0U2R!YbX*zj z>_uZiQGDAlMb^_@^};hq;ADYYbEIa5vU+I@TJy9D*Wdudr%xn;tc0v@&IDl@oX*O% zOMJ7ztXScZCOr9 zisuaEDyi~~LbHW*-gb;bWXl*V%nYYV`OUofJ>!CdenN;g`q*>by=iU}S;Fv(7S5%X z0LmMYhc36Ye~Dhgck4&KezIVzms>a|(2NmMJ8h!V0$xtM_lu)k6kUwohr1t#aXD>q zRg(qdnnBIlg_@}+(5Z#d`v;PPaZ(T7FM(HzaN`#8rgcg-mHFZ)LKW#&XI6PU><6n( z>Tg6=FGFb_SRZB$n`^TepN*JIy&@B}dM7$>Dn8 zZtjJ4uAZ6x<6Vv1yeuyY`>^Qb{Ql)GA>#_sAY~PJKOBdMJGZ3Q(ovivtxOIu|a=&UBqOrw)O7LE-p+#NN2eYW<~HA#hoM$Cc~x3%l@gS z+UNqCHBR>*8_1Kj!LPkH(z`!I*mx)S4m&&28YHS;1kcy4##iI!Mwmsf?~fM=&H^u_ zeb>X9)-J4>NM%39iaaVT3+8m%-?=e+@!jni&tyJ>_CDv! zYmI9Wgqr#jeJW&Hp)b+k&2z{p&>2FTS>?u3{`R9mMV;oU{YWgs2hTO9`!c-{@FF`^ z{$&(1JA})-c4%q5;zDQE(_8b0swS?sSL1NjHP<^MBKFs390X@7b7X0{NXLactsUMi z zWMX$$IeEJe5$FOh`vyjqGe%?>KAmxty&zr!X|8X)>WXhx_JldpjYO|5H4G2(3#dj; zm+|KpUsqOE#{VDez4upBUAqNpM@7VfAR?fGAVhlaf&x;c_fV87y@p-{M5U_`dO%9( zp+g`LkS?LOAcWA78hYsEZr<;FcieO2jPv~g_ZP=-jO}t~?WfK;pS4Iy_Cp$pgzZ1? z=lcACQ$a;V#2r(Sd4eBD8V`}4uQPB{ntvr1f86OgC6)<_=RhLS7kb_^nM}%Xy~@j{ zQArY~ZmtZH;+oelMOS-H*~XpoU3YS(kfprd=ru$_zDljc{(S9?Hu#o}IEAwN^Gx*Y zj7v?P7_o0KP&ufVuBy{B0itr zcb1q3zCMRN7R_+4$Rd|PJ_qGqNhA--yB>`tdB$;7Wc{7jt!_rP(P!MBn1j=QzjAC= z6m?7~m_qleMH%)_n*0`7&q?!F-Gs;28o)hzt-yH8v(@X&y6uRPb_`n+L^b2 z^WM}?ymy)-0%u1>M}@*E1^$u%Oq^J(uK3}8t%#1FsFFNxpy%gX`*H4K?A?pT7tITb zDIb(FiGC|!&tC|-#=selGnve-N)FcwtTpZ_3cTI=LHg0BxO*A8@8)@5nUbwH(2#x; zM1;At4o|tqIB{!4vmbk|ixFLNci-;iBvymJZ-#EM~uKugyC^nfrHCb76&cO+BeY_ zHXTZ~NEU^l_B7(%u}|I<^(z=D{3w+S+ZUCe-s>fu5xHuCf?WLp92XnzemhPKN|f$< zHt<%F+a{=&X{wU)EZ4xiOT@Ejix-#<`+lqkGrT>J(i-G%daWn+eh;k@-N<;@8sl> zq7|JJCel=tkP;m<^Ost%9XH3?Mc3zR(kA+B1RP~vrsI@|R^!7g(1V8T3(sM}0z<&GlML1j8u%9WY33c8&1oh3rAiS`48pw z+r-Q`d3lyY%69c`LiXpe1lU8Hj=9npl`NjUAlcr&)fWt7ZCAhU-2NZG_4Kb}CFgnx zFM8ZBWN+q}o_ipTyc2G=VpC4rdMG9@{;K`;xg+q1h|C+cbZK3Ph$atJh&bzhDCp^Y z7~8M#LKG}lJd~Jvq>oi%@}g_8HUIdamKuFO8s}@>ivU>&ml|J3(q`EUA?okLzIYvW z6prfg4RO=Z+|bvf`3P7PDA7@|qX!cmMuuFD785x9~R^>As$0y4dp!1%Xo%n&Da+RzLcE zv3_+`@#hvr`U_{6p$P~N@B_A5Lj;(XgbEvG_Pra%fQOrQ2fEvB?duMMprF<5y&A@~ zH#YW3LqA<--)FJL{h)_x{pQ4;(24OVVI|xLM%QJyqFC%cUUN6XP&1Yqe(vm*OOhIw zWhMsbYW*f{RhMQt;G9a&-v211dB=9aZ(o}AiVf2qkzBiB7S&ol>|TE<=k_}v)S#ag zuzHb~8#j+`L1=-{kHl&qZ}+k;lm8?_~WR+lvfZY=<}g0n&8`kJ2mwGUbMiCC|jK_W z;J_qS53aras`m7v3J(f`AFKCVFM)1beKcgTeYi!>Pj_=MAtPR7tP%4aIKz{`+pg~0 z*6SSA%$J_F)K=euNP?=dn$6lF#;fUMkOft-B~C%Jw)P(8C3E%E3z|ru%cjJO`KAzy zlStqB;VJiL6YoEk1(P}aCk zG)fXiy}7FVs@x7ozG3x3#qK6=;s5H);48EgMtzt5-kElHl;(ve4g!zSwi0e!4 zKkp>)FaLY&{V%ikw}0_rK4zZs3@<-@{=UfhG~5vF_b%7j{S2tqhKX5LJ5H9l(CgWb z0l|H=5}agYiS}sKQxNh=yLdb&gPud#BT{`suvQCLwe-!9SIn?KPVY8q;2H&^A*&+G z>77zj%TBZC`Sz%HUYBX@pd{BXBT^pPjBZ#c?2no{5*`VcC}t!<`!?S{sawmw>qGG{ z$?bopjnh}}`f;w;R&+?jOb44oWdmmlfwmaphOM7dm9m(PXVb%+cHEHhGm$}_4u{); zHS)80UDhxYhg(&wN+{Wger#PQy#)}SuAJDop~QxKtjXfTJqzh$ZGfOxhha%thH|i& zDaXVh^_1p*D;9kAV;r{5bg;MeVc10=X9l(w$R*zXo}{AfW&#FKb6*17b{FlpP zY;wA1qf6!)12+&_O~JEFtdXY2f$Z!Pzl4Q-)4K}GmemwI{)4uuF6 zE(A-l3BFy;P*2&hn^U$hkICJv`n?|5<)t0q(XlqdITmdtwZsYV(CXfG zMEldyeaWj&=fsP&J|r2axc-Dh(y`5meJkVhus-V1S6w2iM|Rn;>2O-R`ZmcHfW#lL ziL#tD|3ZJKA?c(YGKWH{C*73O!0CqpbSxYwg6l{zr2Y2sSSF=ouSlH7eGys@Rr_c!(=0jz&UJ zDkHtB6JO`CV!kV7d8NL57Y$l}zSe;7#;ajx)qDir?@!BvQ#6*GtAhMoJq~(=-bcf8 z1j6hzF%o*t!xGKQo%e)hThzwX7DpC~?RMUmzCBPO2^SH%2ljigme5P+l9I_8r6sHO z5Q$Z*NO8JCbFiettkCW&1?y6TCQ1@@bf<)87>086RTrhO?d2vy8`zH_%|Yk2;LHlD@t4~x2#3b#r<=K7 zRo~t1OG0T_EsGHF`$-g`SJh3XcZ=yZ#LozrKP!PyR=a;SJi?9$+xR24N|=>{Z`?C* z&?*^F%%RS|H~6^FvF`9L`j48nB(Y)uss(! z1mVR8*v36B%R%|(W>l;=NoQ57T9p*@92xABQMo3okp~klayNU9PNP5%E8C12(Mm5a zaTvVmJoC+qEMC+|US^qZ(QUaY=eWl!1Ljgt^rx_ETOvI9LM)ru%^^|+7ZC$8Ckt&> z&TMK4;z|{MBuq`)Wj8tQ>&9DT(0(G7Ak(7s83wki1?1Sz8TJe^PrpS!tr#`Hu!1Tw z`*LOi2Q(XGOMm2kQ;-j_cqmi_S6ZnVtS+R-GbLiL<=d6*K69G47T%wa?zOGNRLz5LcN<&79{03%L)`+!z2s1a=R~H`IrqrEV{PRy>1lTZ z$>T=bS7nf{Y5zx*RsnTy!kD4{@zF-YQm4#g`@3dxVXDl zJS>daNBA6B0;xFnShR0~aj}M=5hY;=6w3G z->w$YhGuX1OXT-RQl5?V+rVie zxhE{vpE;g?y!<4C+$ZZ%gW>BVDT&Mu!3eI(!aMjYvVy)9231UaPjV>~uPPo2HA?(l z@A&V$0g^+AoihD8uXuU1K|YN@7QI~XU`Im)&dU-3=DGLgqi2h!q1;Z`ono&NmPE#%}{6|bdmGQu@OGFmg$AuKCe4(4Q9>BjCQYxc?$i18u9}oR4qEu~v5An^* zRlv@STVMz+n?we}_w+01XVG7IYK+#xprE(uJ%pTil5U_b`D?@8HW?miAA}ALMoFMou&~R`1Wh4d3Ov6Z7{e+mD(KBLY!)3A@w~KW|4CM~Saprh z{PG~xrdsRIWmE4fiLE!HB&R#osT3Uy3Ju5?i>lO;RJOCg@3@!fM_(f$;wz2&OhXN= zuF{P|aiyB90?4W%AaAsrU^1;}Px>e67g}>*DTw*=2zdnhIK-*&f}uIIRl9%aZM5ElOgVGFg4>N{VWHDDokMs+CDS#|?cw z@zRq1SzC@ipV|3s<9}5IRXVv6``W^pGV(a#_dI-V<7* zf>-e}z5`;}`0JQvB_NU2qwu#$qMgvLTdz@3S#VO!zKiwG%k~dNfw$BV`Etj~NKUNwT4rhP#Nj%Xi1@yUoDH#dGreRuKi(&()|5Q1BIXTJ?VcKe2qNiUi&s2e)NieKC=L+9O!Hcww2I%>6HIM&`L*wqCX~ z|F5f$I9-7#Y$CvDUeaa#_b)$n7s15;vq`Dl|5_H~B!7>8TEvbr8UqEd3*dJaQ=AN6 zDo;8fqEm{Z4>J($%_F`$V*3Ttm5xZkx;|kKg~nbzHoTsd%UDD2qCnaFhm+om>)Q_& zY4Yu+LydD#Lp#S`qDDYQiY{d;I~MM479Olioi{<+?rU3u3409NeYOzt#ZdKlFfn#3 z!%HQx+oBr|y-owgr(4FCl<+8YHvPCUy!6yo8b+U5e@0aPDFT3QK9H$WDtrY-2)=wc z<>@>wWK+PQ4`h4oF&_vpPE27f@heZGcCp6w^U`IWd_VmXsDbCw!VM zChNg8VIl=KeHXa{ygVrgwx|g&C@3>}_v+HSjjsI`z1?a8!B;;~6O(K(CJqFM_R8e7 zz0_pGd67>Z+!UAkJB+o+12c<*fr+J*c=0?%Oo*{b(4W(09$tX>^ms(4bHS}A-reQk zy5rn}MJ(^_m%hx1q*@6daIFrEe{(6@OKCPlO=g=_A-M4>WZuy3=##v~27F|{LADrw z@JX5eq)%-;Z6VMr);MZycQRV7%8SIX`!iwC&9FCk?e^cc#c2o_O`$j)N3b2_%8rN6 zv{siq3$&y}K#M#bOcd#yl~NklOBNT<)ff2^kQ+#xG6NyD*e4O=ooX^bv(%u3AmgSaqz~_Y6GI0%d?~l^sR8{tNKsg^Nw~AbPokQSOLO>|_A8WZC){#a3C^QF zYi0Si{t|k|HFx{Ta$wDSwmBXuc`}aeMH?VkIhCMd8a}1Ks>HOJlgSsKuGaL$FKle9 z?pQYCbXFV+FL{jSAQh)bzr?yP`cmdjQp4Igczy1!rcbQ+?qut-=x??BOsFZR3-3C` zF66|nuzu`2_!M+u3%?&cb^oQ3-z&c~$w;S0$zH_pS32xs`ge(`_#CHF=Nk3+?OYw5 z;e`X43OZLghE*36lPvHMoLz zpZiej%iFol`TLc#f^XgRvVx#!uOtr->l!Xu*Y)0BVb2m^r0o8{o~*2CNn<$_yU7d$o~u2OQ?(+Z#V z9B^(p^F9URbksA~R8(zmv}?G`PVSHmG$GgHsU+CVnZ?W~?=|(MRo_j{J#MC?=V>*S z1KeZRUJ83SmhdqPlu`kJmzQ6j_n?+*>PqaxL5hX8zCS>RpLg0@+u-UN(V8IBXv}l* zL0J)EjNIWXm%P6CVfoD;XmCn(Ml_^4^`$e1G%+2AhwICm*%y3r8^#GNdG0x6@iXik zY0ti@0A1h%`4dBxaNn%1lxk)X@CbXR+(u<10pUsst{3!Lm%iuOb$+UOht|(s6|Ns1 z-rhS5s6gOvDM(C741Yzp5C$BR)rkO7SGJXv4e9$M;_xfE?yv`TF~HjniRc7DNAPkq&#a1ccI=cm||Sg;tLFE zR)vCV4Ky;4uc9xi4O2Q%lLW09o^Qh@&B}fZuX0Z>X>j{lg6Zu=%$U8STz*;=Q}U+< zQZL12KsLlF-*3ePlsM4rfx6P;0Wj10-Cl&_)SU|)V{)pNoRsJqVs9jd>q)Fw>^1xQ(+wvtxbRtj);QvfEVLOUAr zKrDFdhD<%g>^P4z7&~rtLor4quRrJg`sx?7sXjN{+ySw`*HAx1SAg?U!(1AVXtJmk z{f*fDd64;dFkF_C#SKRDy-<<8CEWe-tW8|B2Z_Uyh4_Q2pv?Zg5Rp>Kl9MfPu&f^ z-%I$?FYsWHTgLCotWTRI1MWUq^G1*9&fy4!;Ra}v(1&katJ&9X#>kq#&HXpIGiAng zh1gF&o1RM%!-l-0vftly;|6eRRIg!gqPg;6U?Kstpc53%Sqkjf{Yc6FYIo9b4m{b= zr^w$5EzSDUc28R0p*4|Fo1o64`Qi8G{EB{)o z!tq<|`iJk-Ne{8@pmf^+^x-*BRcPI#A!VVbx>1**-|P1V?|4Qkxb~puPclQ)?mnVR zqHTN7QA;1Zoz~Nz^6q7c^g7x|u*Gxs=nqaJf{$EpD38C_XumN@rbjq-EAaUHaqhRZ+cE6_-#BS$o}Mz@vp zXI}|W>a_c$YA?F3*c3Fbf|ncVND@uZOS`HDiU?X(hb*#zBVzBDW%YzhFh-n{9~el|vw$0i6I zyCu15r$`xhrjNe0yN7KE02EnO(`>sD#Vo>Hol8$p)D5bfEV6DZm4u-+Y>o@bn5W6#XFo5PDV8a@OS zJ55b(H$vAOEl_bEv<+WU?es)RrYHcwN^UBbci}nb*IynMRf^>-S`VZRmU1O&1SDa| zo=I_ga}58>>9p`G_ya+JyQ)>;%C!FNVff#X{(oZCstQm8LL4IbNh{Qb6QL~y{bTst zQA{S(;P=*r64NWp!OAKVn#^y8joG}-eTs%k<;DNaY5{kD@)q9`EAJS5S4{o3%5Et% zNzJvk1~M=Bxf43|u|xtdyaN2he|W8p2N1nPQV$UwhG7(l^je{2!ol%@FR|}^P9WM#^8<|?AplC3co9Pf;DOX9DMo3t=pK3 z*!-ou&j%4UmY;kYl}`&VeeN_RC7R(Do7aZSUF^%rx}5a{bT(DvZO84!P!ADkBprB& z3ctN|vG%YExr#u+5J7TwzZ+X@Pw6n?!}kCgcFNUx9cI5jP+$L1C29@IE20*au-!F0 zFXg1QHEL(Vl~R|^Tv}F;{?4nSPjrnVe8S&%pIcFRd_0tmYdiJ|IbNYVb?lgWQ;0iM5bIY{^+8yTA^QPTH)6^@9^#6&g*g}2@4 z3bXUF%hCk^cTc2ZZJD{oBQYHgLoA7+?r=QZd{>_d29xXj_}g~6kI+~?H6v6=8h?1W zpQ_#e=-YbRifTFrb1w$jSRMsb$O~Z)AH(8VwG}SXt7kIguDoM=4>B_=jJ7(E0O`kpNQRqQCnZuKqNy_YJ9y)c@6-#SV56+Ad{Ccp7QW@_RRofxY} za@x8({A&o^d#op?Syr?W$xlU%&ndoXz4kVPh%jy5Cm57YHg97a_02b;oMdy!^5EFd zSQJ!fwOTg{8L*c*XOXA(Yk}+AH7!2M_th(3&|3o`JFhIt)e3gxfjxMRb4k%EoXd|+ z&T4@_*48>vUgK*kLMzeZTR4)Qm)jBDgVeuJ{gtTKx`2j#T!_nqo!^tNtqP%q3M+WqreTRbXBuurwa4N_U(an*{&cuuNX%EG z3<^7KFS_290$~ASORTKx#p3nlhlf}@kKUGopr+%(dHpPwVgO26s_h3ptK)eD+@o_9 zpc$UWKIGH@PS%Kts101>9ft3d!P(i%uWT_^%?k9ypMPURsUaMdVuasy)MDX5y>Tr> zi{>oj$l5Aln0s*ZwEXSU9$1}E4sKHS*=Q!3)#hXIuAXGhDEgvvVpj{K9ijWsG?-i26yeeupVF!~e|+uN!eBzI z`dEi?m5eE8TgCn<67Y_STp(Of*|4Ep`aF1_9gk7X{DJvhYu0ShNv9b#`p`$hjVbLc*5<{8!Kp@u?8)c;C5QgJlHpvY8WFBiCP}#5B zmIYkBfxbJ;t%ITwj_XBKRSXf(zfo>^I{0fqQn*iz#|S;hT3!WZW%~6==v&Gv1e(Z- z`|4w^bS+LvwYw+iE^Q_O{Ngv->0ZRrO`L*oJPTi7#nxtL*yy1gOL~6QCznTE?8^2a z@wK~aZzqh8X5O5M29Bkj+HHot@MxGsrlN#!2UMr$nLKEC9A5o-vSgGh&sPkKRa43V z6gdkp5o+hwrgN=4uk1N^G~Q&ym~XSv$W}AwkPF_Xc~NBxG@gm zk`c32v%74P`nknxG#f{Tr8mBILDELt8tRm^^qqUotzSsqJbcG5Y4Z^B6#_u%* zB{K}cWvX{ZU%B9N49OyIqks80_IT-+qzeRS=NN1B@`?AUZa+Yu2=@}c3ZWCn#HQtDHy>A^U_&;2Lm1lU0$KNtYTGrE7rSHsghD zt!7s%9t^6ME(2XW@49Ok%(Z9@C`kX+HYm>8HyT$=3e0R{P z|9`6|&IKP!-VvBa#xg@Se{&{ZblpNOiV1-FlCet0ZfD!mc}%D&(U}_tJ1^FM126c2!55WIE@$$6T(5X9?TR`48O%8vO9B z&Z`39PXY1K-O;g7&)!o~2nRV0L0tA(8)NG2dO*xC*_=hZWSK<$i?M0Eo_q5;Sto_2 zq1UV~k@2-__ir?~&N5E}(0!4DI{vHzI#-ir-;FYt`uP)mC9+&?niX+D)SwW=oD1^; zO%U@Rp0#CDi7(gOAIa4|ny4)PTBl}J)AO>w(iT*Qwe@l<7fg+&6}!rABfJ+^GU@en zig-KE4t~zj^l6^mL3hLaPisX+*E2bSx7mQtsw7P2B;B?HBB23tXqdg*6cIhzW^eFZ zBcV5B=(KLED)au@H8VFv4usvNUxtmgh8!r!KZWOp)etL4TW9R1Rr4p=h!oiHN$^k( z#OqX~BF9k_5#>s5o%(^#19ay-fR#U*Ua4B?orYT$PS{{DUC;SaJ{#~EF>WlSdcnj| z;b5~I2o1gX-IO!aqP-E&zz`C@JDgoIox5QF?35e%UwDqI|0LDlWNkE`)|VsxhC5F? zCi)Nm2^m8Flj=g!g|lnNn$-Vf1^fFgr6Ol9;C=2tsXe{V8C`X_erE0PXzkN~vO>+f z``s)5%L3gGip9w7bljc9FUO&APt^QxCSCU8n$)_Y)3p$r|X8HU7 z8^gc8%zt1E+hlD3UEwWm&9Wy4jzVgYSNq^9$G1fggMA3N%v@)6v36~Cx698HibVhm zwI)M*Xphg{21I>O5YiBNZP?%*zmc1pw#;MgF|WqecZRhGiQAj^lpRWcL?=t97sR4$ zwnkk33|jpo4tzOc1$7PWd(2o%`F4osOQo6F^p1$XY;r9XFvph7 z=qU8YQ-FqR7}-vB?$HPT!pR%x0T)0E98i9eNOlloAI7kY`I4fAfs&Q>=)nDP-hd6? zZdSjLD}D@T{XU-|OB)#ef47tQ5Li41O$ZSkL377;`;oyfg;9UaL9`)Yi*>J5#?ixx-C+> zG`~Gz$L?zDYRw~WfbtUjp|GD$LzJXiE{fd7Pz474PP`0p%mIYUks%enT=Rm(b&NJ= zSbt5`xn2Zy<^1A#p0N$o+7TKvsQ9NV#@?sMu*RnC%>WBXq;%kh0_A*k?tV6}hmyFW zMJA@RYhB6AQ%c*gsHA@<5I0GT*};Yg9R(xdZVo=v6^?-(UzuC%=dm=uBoFo5aCsy8 z30MKQtnOK=1ek;Q!+3+-7#Zm0`}gro4^XE9z>Qov9jZ_o68!w`9Fu~P3}IU;yNplB zNa8__O;!JM64^wc4Vm|@sphi)E$yTH=Dbc-)J`d0o`5E_-c96``@5>_?Cu+DHScO7*BVqa@`jga~7Nd5|8_aCPqdL z;;YM5+yeYicFtql+}QWVyId<@Af1iki(rTskkTKkd*0z28oEw@M%uDh2W~hXh^DaA zXzR6rddm-M$38evI!)*A2Gn{S4fmaLYM08~mYib&UhAO74YXrOkdes&bTxLM3b^=d ztK3Y6w;W-c>FhZ?G!$=rdM!mj|B?p2xM?hdWQD~ec*z4Y?FDKk?6s|mMu2A6XRfH2 zaVLtzi|jQ_=`Nnt%r7kxo8K`aKU^WEn$J%@asZwM#Q<%!;k@&J5R3%!``N4-tMiU7zSns3y3g$*&5{JVE^D!a4uHsA9T zYk@*SUw1JnEWCD zKS@uUPbJYe-sH_#ZU>%C>Yi_{|C4gl@QieJTDD+$edJtly93uvUS0d-k~A~t`CaXe zW}|dvI4-8je64xWCGOlJ+tC+wEe82pe`cG7!)S^YwLYtOIvVftG7Ej7#w%$BG{D3x zM^0XcxL>KtdYGY$xkTR_$QJI9Fwd`UbtO*{x3l>!?3rv7=1xo(m^dX1`VwiN9h>@W zAdvPb69yR98+HK`mClg%UF%GcqD{I&33hOE=&lW>T5Sfh+x)_2XX2s3&irJw{A@bu zSU&;k z+#L-4Q|uOy!{j|K^FF0j$S7d-{)moM-m5Y-RuB;WMe_<7oQa;-Dx9sc9N3Be*sBa7 zX;RJ#wo6y+AFNMuEPKCG>-uz%uVPDz9VR|5a`kBXsVpQ7p~k8=_l%^i1#|1*-SFC< zPyD^e(8tpNoD68;SC%e!1u_T;N!{W-_y(MK+naUgQsSUCT_&?&i-Wb!8ms^r1J_n!92JiXpFB@bUZQw! zDZRz#=PTl85z+iYi?W03y|z0=l)JmkbAHa8&rq5R?RzzY+AdEX)b2J#kd+zwi612ElodK=VY=*V8ndXQ`$-`!vQd1g(gvCy7LQHtS#{$9EH@NO-wmJ!Q=#kMR6h)YZ|m!jdM3Ws z2wYaY`F^I9ffaT0OVanfbURRQPfi+G`v-uprcG7Awo4yHkvR+GP}tnV3act7i^v|c zqrNmjF{zAk_542Nb~J7!@k<-O3R64D?2Ris#?X;GQM1{Vwl!O}6NSJ5!p|hXvxgX) z$>;`=rFSg#@=QmkJM=t%Z#{q_%AZRdBo|f!f1YRrtU3ah55YY%;}Ld#IM)4x-PR$X zjV&EobM0%omCE=}g{UZ=rs`+H<Kb1o!2h$>o<7>;t1{~_pqQ3Ft zf0(c8q8t;IRbCQafwJJ++-i#ai_m;j=W`i3yRS5h4trBl{a~rYhbzxerxcqlFo08T zkVQC^wr+orwOo+?f$8GFg1aP3^*S=shhAJa)HQMUrf?}>vOtDx&Vf!pdz!FHExP(Jf} zzc!E3T)<8qW}QC60tRMoN9W3bS-0D`d2J4o6@JRiJ&_A=h6#ia7|szplZFFEEy~kd zqhU8rUT=$K>&I`<`?TbO{eE4ExNQFyX#Eoh$N~vG1iAscH+q%^&Hb1^bK2yc9qRxS zrt25?kFwB#Z#E~2!`@>WU2fdgxKR=r_UCze9u~}hbz9o*>eP#jC0oV%O@G8779t01ccQloB7WTA`$toF<)U9gRsL1Gp z5d+}MRcv)$!W*Q=!`G0?fEtf4-FUg@Pntm*lgkANK+?oCz>m29un#{j2QEyNVF7u~ z&B$5H2{WLeJR<{=~Zr$BI9n4|>>SAu5qsx{(vF_LhH?5?9g)G3u%t6iMJKt9< z^$%UXOE;h}W(Mt2FIoU3- z&G3{BYqwwkYiYG9!1sDaUl1w8jKgs)L_LLU5&=dcq1P7u zL!+RE%EXp!7T~QC64CaBvlLFV*a;0Bu%iDIjAW!0NNM z7nw`66FFk)mTUGSd}zOZ?j<|!y}L}{;~l%GQI|RpzHjY?S)q#6EA)vN?_oGka)fs$ zh}U_yqBojrCDhx{N~LecGx%;g4dhd>FG`RC=4zIDJorSV#>-O;?3wN_4Vt|Q5o2MvR-g|eWTqkBE zI{7cU4cJ$#{83}|axR}01ZJE@pTgOnFYo49qu+uK7S&9~|0~@i@42cSC!1d1!D0mh z#y#^+@_wMbVxtqDVO?Pr|BPFE7hR_Kw-x}Z`@5HrybLAE0<;sLdBk@B4Hn;)EWnQB zsd%kyCl^xYzdc|k-J@BOo%Xr*QSdL7{sbRDadaL|g-IiUDVAW(7G`Ays%TV&r?*z$ zR{@1erL3*Yz3Bx#kMPhFn%_b5+q92T%=?nfX*QLOmLULR5&u195M{RtP(wg|C<*^q ziY3St^=$+7x!PXH^lGg$^X7(TUpUUm#fLY1QzOE6E~my+$jnBe*1SPeoXR?&wZzThJBOQpK`~_g2##W{EUU9fcn_4iKl;-kiOF2 z!!M$4d09@kI~+v;d+nEgv^h8P!%3Qoe(feP&9Dp#5z{Fa?67_1eDua0f6K~;qbB*@ zPJyg*`A$vkM0f;y=cLh9)j6}=R}!d>hF_pATK>J4d&#w?xsme6eBa|)1x$AS98hI; z#9)+$s;_VLcpH*;Kk+gla&Aco{f_HiqRz|@2EYH)|)Wfv8H_~xBHNy1|3d+!I4WCZ9G4apy-yN%P1*aNcG`2>?iJ1FB&Q5;v07 zQ=t)tec-9uGC-VaG(fyg#jmrgA{*nAu=$voMj4ja>oFM%*<7%Az3c*eUbFV}H&t#z zr}I4k>pW9y;>VU)J3d^i^^N;#IJ^d7yl2j7jn;oo$CB{C&ZJ8zN#pCSg=jXjZgzWm zVSoevR%9#U<=kxFdp&j~*;fJPciCT~8BGSu9<;S}q30Lq-m8Q0of$PWiWNxBIpmWKTy49w~$n-_0*X&MUvQ{=*WO)cI+q z{2vND;GmTaSFsriOazo!1%;`_OQV^3!JG5SrRdzGUm9#-S&f-wpMM|rmNq1sln8T= z4*eEi$&A&xY18%lDfS}{F&>>j* zN;O+8-DWnZ9h=EuhpB-}i}P9q;#KGUkOfQrGobFRj%zEOvAAtFZPP0ULOZ1Qc9;q%y+ z&+?x-fhACbfsr(O<{KEx_Be^dzQ3& zo)c31hgaGUx!Y=$ z4Pax}LkD^Ib%;GNe)QGyxZ7wF(ggAA_Llh_b*26Ks1DJbSSz0glnE}Z^aot=qpRMj zWp@j#tz)>~)mkbJOMWwUz!u>8XDKr2Wpch|h+=KgkF+o3C~Dl*lI3rq#om!ZZ^X!X zFC_1L1c_a`Qg})Dcsep{p5GDVro7#!a5<%rk~_XiwmyfzZk7%SaC*`b6&)rXCU|uF zh|9gv+v;m+@7J4knR%(DNwDG~RoO#Nc488M21(x({>?4CnljO|H;qDcJ7ZV}+(3J4 zUIDe>2?pe3Cz-8OVLk7kuwtve%7|Ao$s9-(a=S8eL=B|G>CJ*sDvf(HS;}E=>ft=L z($p}&EAw#DPU2T<0n;_!bm&lV`(lU?dEn}e3@>hvt@7vTLhe-P&V_<#LQrV@q0`#A zf+)!Jw_vBk_oI#Tu5K(;NlMH4jkVH#4@Y6mifcnMOmRj(ehwA}a$LDX(!1nA! zslm9;vP9G~ie~GV=SPM>%Mmgr(g7Q5?HO-hq%&=K_sDj~(9wXr!`WuLUmMk~ z(!QHhViF9RK=HeV=}rD6TNrabp9bEnJM4VH<~|Ozq|`|>pM+6Ry|2Um$~z;KD;AaS z5*}48Hb2!R+c$>j?a6(uS!i|^j$LmT@pM+WS@05GB(&;-FE);Qjq)5!eyGc_wR!!O z4ETzIdiY*~DD5=E0zt1n-=6+bpIfy1@8dbmuis35Kk%D7m;?Nv z7g1ohKo-*LNr7Mdvbpa$FS-@P_2bxQwpYz4ot!~Q!u8|CpRM^v^&Qks7?g?3OW+&| zV9Xt>e)C@NLKK>6aQ+L&=yu`~&5^_LF98x%j3cXlwHuZoRObE4_n& zroy~E(Q9AQR_>`^&CJ!Yj^LDPH`mFmm&|^LMU&g&N{mdoz^vpIlMD`-!hl>MUP<~b zmS1jcxka~q8SbnsAw>C@UhqPU-W|7k(TQZbuyDO}O40t)@tsQ5qGtO3(3hQWTx5)Q zh_&yz#vVGLYre`7X!*6bE^I%ovufpOI|^)#W}E3&*w@oha4|gDGzGH`?x&9<>*aQS zebZbUOdR%)l-Z)BIM`ul>?$aaE)69Qfe~F1H@b74vuyX~ZVU!5qfwd!>*ujVy6f&rkom)Yae= z4KCH|Jc2>z@e0XG-gjtHtzIxvn9$%CMIB!&x%{INhth1L10Fs9<{*9d?Z5(!x>T!7 zaNs3<|G7tV*^gIU4}M70BUW01KUUMXN%hv!^Y}$mnz6Z_DBoZABH;>oP7a|N0}$0a z>I3MiZfPB1g%HhWhiNiX>$GPM~m-(5<>DEZ|nF#;@^JRoa(VTRqeoL35 z-*UJScgJ_`^Ge?i{=@C2B_ESkZ@N8q7^zk%W<83B(QK~p&nGr{g8k5;2o|TiMkm5V zU9cV=8kuze+wS;_oPJ|$udj=Ct)ErXa3BNYXAXCXv7R5)f>x5n>Q&!kpm-RfPdcX& z))mLsO0vT^_5}Ao3Ej0ZfPG7NS+0Hjt6eQzfOteS+<+=RX?XjZvXIOOLIWaOz}hcn zNiR@LSMq~)0u{dB5r-QDgmu6W71MEbviRD0tqVxXXSJmFy;`&~bzo3B0f*Q@TqQJp zDCr$H9o4wMG|v4$?R|GtQ&|_UA_|TL1QkcBqJjlb5KvmghEf%j&_P8|Ne!cxyfWa3Q(5XP>?I z*=L{M*(H9@?ETz?o>K4jwoPAlw1v5Vz~XJk*U5VlR2uXfCnn~S;l2l)_03)hYuK;G zz06F2X;mjs1A~`8VM<&W?QXac5>p~;q$Z*@bfr@HwN-FUKH}opqW0W~p|fIp-G{(J zlydumwvxNG$D-sqE`BL6JCDiA#x|tYyj+|FH>%>?r-j#Cc+U@qlILOZNqZ>9$}-sYvlqGwg2hv z28%6jKfOr@s!o&)DiZfJS!7*1j^}K-aD1OJ-d8E5Ow|G=e|p?nefh}^8f_W}vr6-J zZ@;Lv0^}WtyX0}@jclF)n}U=*SXi2 zU34dN-Ral^3&`DsL{-~DcT4@CIL;TUybi|oe0P!g%={S2nD$$-+?ZAc2xqRw1^ks@ zZT974^+)yg1id}Rw;!mSUauQIUGrGyVDD`^eEs>db9#gHkz_cc`@u5K;Z<4oq+g!1 z!}aFh=QX+d-6Sh+9An`k%u-M92f*N)-m2?o@q8nR_r0&Gmn}OongZSx@Ct0lwdRWN zTkex#;BN4pcg{=~-Wy}>3!uKfO1|?vqCF)b>&x)-BHJhJt69^=uA9bd7G!BA>n@x^ z!Luax^-E5!7;fRbx>(BId;FjwX1kx~PWK`1Ief@JA@VA&8i;wWSi`yMGpE$*hw~h4Ucm2G-osdeMWx0GP0r@>?*$H~ZLVL=wf82=O*gEsGtIUW zpnp;v2gg54YzU5@qrwi+&sinqv6QHx-q$bSX;!eHg5fcdXO0e+*4p_VoSikAt$Dsi zvS9GV9q)$lQ%*|AXl)_23v9%K+kvh8_bURlZe7;%;-6^9t#H3yE6D22Z7kk76SfZ6@_S); zr;n~6PPneIHe)$p`OxONT$u7j5BBDcoSv0nZ5@4%=M&tMnc~DJbM1o?tn=0(yBt5& zo#{wHrt<_T@k%ZYs(LL0`Nb{)neEqAR2VWMR0|mxAuWY1*YN(Sy^7l{yE&2Qe+Hb( z-vzH#reRm&+3=f5-X{06caHO13j@Xb$+}xljFqLih3wj$BXw2>%x;i9TAW?%m_GFj z_6e8}9Xqnms`W*?sVL*XT0oZv0*pO(Cy(c7z>WE;#}OOkcs|$zAx+f1VEX$eSDC`l zdmWaiva~{q^zmUftKx1K6~T>5(27U3EXMj=_vz$WiCDTdJZ)Azw+XA$UUI5lp{hPUIZHgMV?lydtv|EZdWN53Kwmk6UlqVoU}R#C$9Dg*n{O74En4 zLcsH?WZa{vt-f`k6Ajl^2>TmaD-Ud->W|$OROq4`YS)(&KWs^bafUb}m43*O62MjO z0{3Og!Tf|qruL__>BZt+4hAuXs95VWy#&4NPHZ`RAyc(8KciR*#&&JIeJ%l3p;Ms- zfBJ3&-XK)oay9SS&8x;U7xNLcgGdo+{9`|2ZZ~Zfej^Obo=yLNF?{AwXszaKS%(Sl zyYj09h{rMNGm{hNIG&U*;HjNkpHSpC!<<_I;*b@m6%Mxz1kVQ7GE4k z{+qx5S=q~F0vx&qaiX$>7JOlzBDDFNs2f3CNvhe;yrXF-NHq0Zpmx!bcK&xuxnNZ$`}(Kn`@kOk~O9=L-68b2t2Rw#Y9cwSb> z9U{|Fz=Qw$^%cNel+;Fs_<`?KXxIz76fp>B_AH4k1b0xUrh3|na%&eY5()zrk`S=`;4w|(|6ur`ZIdm;F<;*b}FjiBPe9} zLlz!Pp>FZ%gj`4pEp}5hAu6a5VxQqd`ozZXAzP)!5Sf>g#A>ajx z!DHbmRKVcYgXY7bVb+hyFjV>PrHuNV6A*QZK!qBSn#)~Sz;G!Hez4y(Qyv(JSYV1| z^rt6|bb@1YMN%Lu3WBysUt*3917>{jVbjsD)T^^B)1&w=UCzaWEPaB|=q^YitA(9p zWEaQ`3}$01SqvtaKW7}q_w7D016AcrFe@V=pcN6s?*eqder?Iu5dcW0oS6Xu;_gUb zUaE!pr0QzSWxTVDtZYy@M(^PKm7Dz%ln6~PiOKJbaVw$R_?bEmYM1@7Ucby zOP_EA6UMI*u;dv5%~I;p8WO*269rtL5sWN_bPlY*4`0R-`Po}Xm61D6>=Q*F~R=y-5pAB@dq}d*4l}g zvb8pA06yn;r{ETjz3Hze@du4)&MyWp`MGKqPv4K@^*Ww6f)rT)DuMMR+ZcxtMoRfT zzt=@rV@B5_`LLvpAXUIMWkl;5&r)0KKc?z`rgCxg`@1SBBb^mhHUXXCNFg0BR-vB7kXM&;AQ_(w4;C75!7-kn+$=J+vKPqWyWnXaHD8Zgv_0TKX3E|D=u#;}~3ck*)nRNh4m{`#LtF!{kY9)X$4 zC^Amt-UOy+oDte`5rOj-iNAPK_+$KrjHmty84IQcgv7IB;l2y`R37J~KX(`tV2B0) zPKZCZHod(NJ>6CzGoS=Kr|_o1kR)a*r^%T&d%5VuZ|f+0PA*)M?95LA4xNKE#pqt4uxdGM+_WGa=@&&*gZ?+r8p1?|B6Awa`voCOed z{rSAuM1`=0=y}F&O@GW$6xZeWuV4@@e*y;7ka)G2Re@raXGMEyXh_L63DUKC;t=wU z?undjrj8_zapya8ftKJR+Sn~a*u5RGM);WLXBh|72~2AW|B(%UUM*uTXM?B*v_J8u zyR~MlV@wz<2j{7s6MXpdur8)-Ys&U4GEt4O5m;aodrSJ|6v3O>h&B%%C0-Q+gcZ?) znw|N(aW@cKg@!U9x6ssci(1H_1aDF~5D$hp>4jC!`IMJ=^9_0u^0&)T42WDQ>ggV7 zv4~Gn9?%L1L<%R( zSrm;h)%f+zm@-G_`TTmzZLjhJxQRf^pgW99W3ki#&-nzH6()4SJ+y<)qVmFWb{II4 z`}rMhxQH356y{NThSS1$H1NTOOSS-_R4$4i#}d33&{G(zv1a~sE~t8aENeM_Ag<`nKGO=4K2Wp4Qk53>v+

1s4)RPwK<#{pTPb$MA zrj6=JUh`#R{naVdBz!SYwuQm-pgqC&1T_Qm?j_R=K|IYe%Lv29k?EJ5rI%ifQ}yQO z-9t!PU)cOxcKGZLNJ(y6bfyygw>B`pfmr^tG^zI&vuM6omQ0YzqtZg}HmgI9hpp~~ zTuv_Gg2UB!y|%>zPz-nkvW#oimY#(uHUXZ>kS0H2TmvU|(C|$C74&ZcO$3wKYw(jc zzGgUOP3Ui5o2I{*SB2j$@Sp!qY@$6rmA+(&^vjb+4bKRP@&G!8>S;gxs%mP%V#mkb zR$fBaT2^mfG4-hN*l`=@Kf>-*R=l|9cH609-+;<~%CQyWhYt&>sI7W7er)r@0P*|n zurs?8C1vN0Cf?jl9E*z)z(6qSaGLUN0pfzPuoD8do((7!YqNc&Xn!&m=k{%zkEQSrHTLSjQhcZ-XYmF>{Hmp000 zw6w=d?6RCNy$2NZi@AmoEekLaMnIMmuFE5_u~Es>6ZSdlBOK~oqDK`H%$3~SvdPOV z^T?y6XZ9F4+1kE$RUbtu$ZO&azna=#bF7*Xb!{ExQX_OVG&S}0(Q3)5&8|iH`8nrX zo|2_>2Rc1VJaQ~m6FdFN-d??0c4pT;4UK~b=b4o9AV!}rRLd^$%Brp)hPk=<4mY zdOFy;dp7>RV)%>JC6UjUKPl}ZgD?KFJ>j$tv8V$4ZdX0ct|au7HUZsxN#;JPysYr_ zp7nk`6V*96%4z8)IpCwUJw1aH6BF9h1CU&^di3iQ_>_O%nS16i2fMn9#cvOv^S^L} zfG*TFy@Uv|$DZk|r!ms?YA+V=uCF(_d2=S&bjJ<#)9Z8E)80gx9mAdYWb4nLQgE_U5Op`%&wFaNaCH=M|$M6-U?&d(U&k1K72`zR;Fl;duKTz3#zbP9?;)W zh;-}IT}@$$dq4Pq=%A?eYS5SeEw-^lIe^{42AQr!G?)p$PsU1=36SH+ox>en*~yQlsA%EfdryqQXiF zloi_B4Rg@iN1!<2{6l)3B`OV}SRrcsBPIzavM8&+lIh>9Mz0fC(to2DeR6IE6e|{m z`map-|3bZT=>IvPMxJT~?M4vPvN)t56Uo2d$Im9$y?Zy7Wd!Ewgs(pcWeOFIbPOhUaPNKhCYE$j9t^B^+N117ph_FN88A&rf$SDPtWEv?XahD?4o|w7D|CZ z=>G+MX;$w^hd0cDTtc$#>lb_Wz0Rjan`O)^$d-Sdp*2W}l@1N64z z7fA8_pl6Rj+4uB?M?QIjh3x(K(MkPaMy)FxZu0j0MgL-_e5OmY@M!0?m4C1MuYZ}_ z5;-{Gwd01hJWO@tFRgU3leN6BKmXgm8cDu_+T8^?qQZg59%N-q>*ahzRLX<&w_(e^ z?NuzUApQt^LGO27uoUzccdpz!S-u_09EI*tpcvyJ$r3d`L|B6`4wYf7F@j+Fp(_G+ z5~@e-5`nWw{`h2DRTTqLX+{b`dMV$fH;|M&m@pODNdoKlvQ!$V4})n&anl`draF<8 ze*FAPBu$452(5`61~-X}o1;qG;G==0tBw%QR4rY98^I2<%posq3`9flL*&#U0)=rh z5y7_aW1XCwaTDr_>so? zHI=A|!E!02Iar%4bRU++tZAk^W?GVcW8g66nlms%H($9h{kry3 z8Vt#&)+3!Q_T&0a{&B=8+8&KxkT#0LOoA=UPlN}uy671J#7#2XkM&KLi@V+#5h7kP zr{7PWaCi`J(bro^Y*oVM>MitDlLh!OZRumQJh*)mc%!_9v?a(bXErQQYKOwqx5 zeu#{WJkTEaE|xkk)=F=r#+M>5PUe8GZs}D2h&2r#v&=VUm`lOpGN2T8asU*#BOPJJ zdzz?PqgJD|4aWCQ)xdCvvEGtj02;Fs-`8;5#3)SBLwp@J)jMFE_Fskt{(p7ufN%$lyxGuGUj zPB+o8ZSD0_w#%^k?z*cSG5+VCdcL%vdc~^GHaQPx{h=|EPsycleq|Ggz_8WSZKN~9 zt5-VU>1n>rAqf zSd3>)s=oVS;=(K3fF&n|UEow%R6oV8{c;vx!)IG}J??Tx4jrtas8G^l7_KN@lp|xX z&7MrHDjlB*O@Pw;3`1}dYxX5Rc$nI!^8{ZR=fyo?=7?bgSBSC)<&}cGbjidnZ`6PV zfF8nP2l{do!WaSEg*B3oo3xf6(G7e+*{px(L9D-zJC58YAH;0S(4YjOC40M#f&w|) zB#YBj6a)FnmDM&477&vapPLeDr6U5nsB!R~bdJ?y^x@(1`Z(YdD||f#s2|7D@<2Tf z@6qvQO$YPJap>8MZ1OyS6w}!|J(L)@g$Oj?#L3HgA>%)EoQZ7 zH>IJqsr;8`BjmRgR34O3%kxI-+YQ&(^pX|&%Jy!|*grWwqit1KHS5mE*d!A&lMrZR zeHn_zZY6Lt`*K%zV{CsgkmOsTr}H24kUEFEFlC>`z!Waniy>0IGqm;|*sr4#y1%Iu z3|K-&6cKl^LbO#M#7iWZ*C?)Dy&u;V@aj`g0a9(Lc|hQrsE9>YY2m6~RhHMNKhE z+UUHrRZ0W0a=p3nGTyKbu^jTeNKC$auX`ybZ2axV7=T-szEbxvASb_N7{Z&JTM3G> zpP~}LZHt*IWIgQ6{8ZRBth$gz7(e!Q;EJ_E3^5RuD_mX8BjDV8_%ER~V zq=Gbg*gn1XX9ZD5YMk_ETBvr?iZ6g(A0R`X6!5s@df*7nN)lrdE{98?XOop5lF+n? z{F;YJo5L@2-&N|#Be9cFqk&vSP43&bSZAa`&z;OWlDYjrJ-uK};DyMH$$3xM9VsYZ z1GVY|o&(utUidf3+o0=n_e;?iO{k?GZS|uwnoJXEORxvTUN1~o{-cv@kS+Tm)LO0H z`6I7s?J!kb50YZakKCs0@@HXY5Dr(>C(wEM_d{j$3{o^5l48UUkKT@*L@0Opqt7oO zqNoDJ`@dvUJ+Ccn0Y&j4{J;8npkF$wI_@O-wK^oE?#(PVtnc*ni$G{E6ofgTIYKq3~cSg*15meha9b27{mq7yA3_hCR-NH1z)_l zC80LEE!z8uz8ciT)qX#;Bdrpk<29_T+S=OKcWdWOca#(Cc5M{<_>`O*dmJ3mdKb;| zvPzP@gUthz90?7LgoFfVDlv@RBnt^s3BO)9$N563t*w32(BQDEPtLIM5HMO-upSX5!%{X zH?AI5O+mkW0YZ8HdF@13^@F-8scC5oL(C= zA<<6-@wh{IHhYqc$7$0;?~bc=k+pO^EI_PqZBGFnWR^A7dUs;o&=cqhWUd4;eYbbd zx;%(IN|~~!xe16AX6b31_82o1JM>@G{>l$O^gyWbRu4#I#u~}qo|oKX+<7AOgQ7x| zrNwLenfA! z+DhspD9sbz{v#&+X3RxggV6oRO2M7YzgpPJZQ)T+q7w&&`Y#7*!NLFk3l$V9u)=Gi W%r8Mxf7cT5=j1W-qXmaue)}I{>dlJ) literal 0 HcmV?d00001 -- Gitee From e0720ba0bf0e9e63840d7552d6e631fa26ffc805 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 29 Jul 2024 14:28:03 +0800 Subject: [PATCH 047/791] add api --- .../msprobe/pytorch/hook_module/support_wrap_ops.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml index d64c577ff..f68708e94 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml @@ -1873,4 +1873,5 @@ distributed: - reduce_scatter - _reduce_scatter_base - _all_gather_base - - all_to_all_single \ No newline at end of file + - all_to_all_single + - all_to_all \ No newline at end of file -- Gitee From e16dfdd9b06226a1babfce3c6432fbf5b055fb49 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 29 Jul 2024 14:40:57 +0800 Subject: [PATCH 048/791] relocation bench_function and fix hook_module --- .../api_accuracy_checker/run_ut/run_ut.py | 8 + .../pytorch/bench_functions/__init__.py | 15 + .../pytorch/bench_functions/apply_adam_w.py | 30 ++ .../bench_functions/confusion_transpose.py | 25 ++ .../pytorch/bench_functions/fast_gelu.py | 58 +++ .../bench_functions/layer_norm_eval.py | 8 + .../msprobe/pytorch/bench_functions/linear.py | 15 + .../bench_functions/matmul_backward.py | 51 +++ .../bench_functions/npu_fusion_attention.py | 424 ++++++++++++++++++ .../pytorch/bench_functions/rms_norm.py | 18 + .../pytorch/bench_functions/rotary_mul.py | 55 +++ .../bench_functions/scaled_mask_softmax.py | 29 ++ .../msprobe/pytorch/bench_functions/swiglu.py | 58 +++ .../msprobe/pytorch/function_factory.py | 47 ++ .../msprobe/pytorch/hook_module/wrap_aten.py | 21 +- .../pytorch/hook_module/wrap_npu_custom.py | 18 +- 16 files changed, 871 insertions(+), 9 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/__init__.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/function_factory.py diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 30994f709..bca971116 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -27,6 +27,8 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareC from msprobe.pytorch.hook_module.wrap_tensor import TensorOPTemplate from msprobe.pytorch.hook_module.wrap_functional import FunctionalOPTemplate from msprobe.pytorch.hook_module.wrap_torch import TorchOPTemplate +from msprobe.pytorch.hook_module.wrap_npu_custom import NpuOPTemplate +from msprobe.pytorch.hook_module.wrap_aten import AtenOPTemplate from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward from msprobe.core.common.file_check import FileOpen, FileChecker, \ @@ -78,6 +80,12 @@ def exec_api(api_type, api_name, args, kwargs): if api_type == "Torch": torch_api = TorchOPTemplate(api_name, str, False) out = torch_api.forward(*args, **kwargs) + if api_type == "Aten": + torch_api = AtenOPTemplate(api_name, None, False) + out = torch_api.forward(*args, **kwargs) + if api_type == "NPU": + torch_api = NpuOPTemplate(api_name, None, False) + out = torch_api.forward(*args, **kwargs) return out diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/__init__.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/__init__.py new file mode 100644 index 000000000..eb0686737 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/__init__.py @@ -0,0 +1,15 @@ +import os +from pkgutil import iter_modules +from importlib import import_module + +""" +gpu and cpu not implement benchmark function, supplementary benchmarking function implementation +""" + +package_path = os.path.dirname(os.path.realpath(__file__)) +for _, module_name, _ in iter_modules([package_path]): + module = import_module(f"{__name__}.{module_name}") + for attr_name in dir(module): + attr = getattr(module, attr_name) + if callable(attr) and "npu_custom" not in attr_name: + globals()[attr_name] = attr diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py new file mode 100644 index 000000000..3cebd3050 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py @@ -0,0 +1,30 @@ +import torch + +from msprobe.pytorch.function_factory import npu_custom_functions + + +@npu_custom_functions +def npu_apply_adam_w(beta1_power, beta2_power, lr, weight_decay, + beta1, beta2, eps, grad, max_grad_norm, amsgrad, maximize, out): + var, m, v = out + if amsgrad: + max_grad_norm = (torch.rand(var.shape) * 10.0 - 5.0).to(var.dtype) + gt = -grad if maximize else grad + m_out = m * beta1 - (beta1 + (-1)) * gt + v_out = v * beta2 - (beta2 + (-1)) * gt * gt + var_t = var * (1 + (-lr * weight_decay)) + beta1_power_out = beta1_power * beta1 + beta2_power_out = beta2_power * beta2 + if amsgrad: + max_grad_norm_out = torch.max(max_grad_norm, v_out) + if (1 - beta2_power_out) == 0: + beta2_power_out -= eps + denom = torch.sqrt(torch.div(max_grad_norm_out, (1 - beta2_power_out))) + eps + else: + vraintain = torch.div(v_out, (1 - beta2_power_out)) + denom = torch.sqrt(vraintain) + eps + + if (1 - beta1_power_out) == 0: + beta1_power_out -= eps + var_out = var_t + torch.div(-lr * m_out, (1 - beta1_power_out)).div(denom) + return var_out.cpu(), m_out.cpu(), v_out.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py new file mode 100644 index 000000000..dd30bb18a --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py @@ -0,0 +1,25 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions + + +@npu_custom_functions +def npu_confusion_transpose(data, perm, shape, transpose_first): + if transpose_first: + output = data.permute(*perm).contiguous().view(shape) + else: + output = data.view(shape).permute(*perm) + return output.cpu() + + +@npu_custom_grad_functions +def npu_confusion_transpose_backward(grad, perm, shape, transpose_first): + shape_cal = shape if transpose_first else [shape[perm_dim] for perm_dim in perm] + perm_cal = [0] * len(perm) + for i, perm_dim in enumerate(perm): + perm_cal[perm_dim] = i + + if transpose_first: + result = grad.permute(*perm_cal).reshape(shape_cal) + else: + result = grad.reshape(shape_cal).permute(*perm_cal) + return result.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py new file mode 100644 index 000000000..5442eff73 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py @@ -0,0 +1,58 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions + + +@npu_custom_functions +def fast_gelu(input0): + attr = 1.702 + const_0 = 0 - attr + const_1 = 1 + const_2 = attr / 2 + + abs_x = torch.abs(input0) + mul_abs_x = abs_x * const_0 + exp_abs_x = torch.exp(mul_abs_x) + div_down = exp_abs_x + const_1 + + pn_x = input0 - abs_x + mul_pn_x = pn_x * const_2 + exp_pn_x = torch.exp(mul_pn_x) + div_up = input0 * exp_pn_x + div_down_rec = torch.reciprocal(div_down) + result = div_up * div_down_rec + + return result.cpu() + + +@npu_custom_grad_functions +def npu_fast_gelu_backward(grad, input_x): + const_2 = 1.702 + const_3 = 1.0 + const_1 = 0.0 - const_2 + + # e^(-1.702x) + abs_x = torch.abs(input_x) + mul_abs_x = abs_x * const_1 + exp_x = torch.exp(mul_abs_x) + + # 1.702xe^(-1.702x) + add_2 = input_x * exp_x + add_2 = add_2 * const_2 + + # e^(1.702(x-|x|)) + pn_x = input_x - abs_x + mul_pn_x = pn_x * const_2 + exp_pn_x = torch.exp(mul_pn_x) + + # e^(-1.702x) + 1.702xe^(-1.702x) + e^(1.702(x-|x|)) + div_up = exp_x + add_2 + div_up = div_up + exp_pn_x + + # (e^(-1.702x)+1)^2 + div_down_i = exp_x + const_3 + div_down = div_down_i * div_down_i + div_down_rec = torch.reciprocal(div_down) + result_temp = div_up * div_down_rec + result = grad * result_temp + + return result.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py new file mode 100644 index 000000000..885b5c460 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py @@ -0,0 +1,8 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_functions + + +@npu_custom_functions +def npu_layer_norm_eval(data, normalized_shape): + result = torch.nn.functional.layer_norm(data, normalized_shape) + return result.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py new file mode 100644 index 000000000..33b18d759 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py @@ -0,0 +1,15 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions + + +@npu_custom_functions +def npu_linear(x, weight, bias): + output = torch.nn.functional.linear(x, weight, bias) + return output.cpu() + + +@npu_custom_grad_functions +def npu_linear_backward(grad, input_data, weight): + input_grad = torch.matmul(grad, weight) + weight_grad = torch.matmul(grad.t(), input_data) + return input_grad.cpu(), weight_grad.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py new file mode 100644 index 000000000..dae274552 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py @@ -0,0 +1,51 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_grad_functions + + +@npu_custom_grad_functions +def matmul_backward(grad, self, other, mask): + grad_self, grad_other = None, None + dim_self = self.dim() + dim_other = other.dim() + + size_grad = list(grad.size()) + size_self = list(self.size()) + size_other = list(other.size()) + if dim_self == 1 and dim_other == 1: + grad_self = other.mul(grad) if mask[0] else grad_self + grad_other = self.mul(grad) if mask[1] else grad_other + elif dim_self == 2 and dim_other == 1: + grad_self = grad.unsqueeze(1).mm(other.unsqueeze(0)) if mask[0] else grad_self + grad_other = self.transpose(-1, -2).mm(grad.unsqueeze(1)).squeeze_(1) if mask[1] else grad_other + elif dim_self == 1 and dim_other == 2: + grad_self = grad.unsqueeze(0).mm(other.transpose(-1, -2)).squeeze_(0) if mask[0] else grad_self + grad_other = self.unsqueeze(1).mm(grad.unsqueeze(0)) if mask[1] else grad_other + elif dim_self >= 3 and (dim_other == 1 or dim_other == 2): + view_size = 1 if dim_other == 1 else size_grad[-1] + unfolded_grad = (grad.unsqueeze(-1) if dim_other == 1 else grad).contiguous().view(-1, view_size) + if mask[0]: + grad_self = unfolded_grad.mm(other.unsqueeze(0) if dim_other == 1 else other.transpose(-1, -2)) \ + .view(size_self) + print(f'size_self: {size_self}') + if mask[1]: + unfolded_self = self.contiguous().view([-1, size_self[-1]]) + grad_other = unfolded_self.transpose(-1, -2).mm(unfolded_grad).view(size_other) + elif (dim_self == 1 or dim_self == 2) and dim_other >= 3: + view_size = 1 if dim_self == 1 else size_grad[-2] + unfolded_grad_T = grad.view([-1, view_size]) \ + if dim_self == 1 else grad.transpose(-1, -2).contiguous().view([-1, view_size]) + if mask[0]: + # create a 2D-matrix from other + unfolded_other_T = \ + other.transpose(-1, -2).contiguous().view([-1, size_other[-2]]).transpose(-1, -2) + grad_self = unfolded_other_T.mm(unfolded_grad_T).transpose(-1, -2).view(size_self) + if mask[1]: + size_other_T = size_other[:-2] + size_other_T.extend(size_other[::-1][:2]) + grad_other = \ + unfolded_grad_T.mm(self.unsqueeze(0) if dim_self == 1 else self).view(size_other_T).transpose(-1, -2) + else: + grad_self = torch.matmul(grad, other.transpose(-1, -2)) if mask[0] else grad_self + grad_other = torch.matmul(self.transpose(-1, -2), grad) if mask[1] else grad_other + + return grad_self.cpu(), grad_other.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py new file mode 100644 index 000000000..2a46d0200 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -0,0 +1,424 @@ +import torch +import numpy as np +from einops import rearrange + +from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions +from api_accuracy_checker.common.utils import logger + +gtype = torch.float64 # arm host必须选择float64，x86环境选择float32即可，64也行。arm计算很慢，s=8k的场景建议使用x86 +softmax_build_mode = "QKV" # "MAX_SUM" + +""" +# 前向函数声明对比 +标杆实现:fusion_attention_forward: q, k, v, drop_mask, atten_mask, pse, scale, keep_prob +融合算子:npu_fusion_attention_forward: query, key, value, head_num, input_layout, *, pse=None, padding_mask=None, + atten_mask=None, scale=1.0, keep_prob=1.0, pre_tockens=2147483647, + next_tockens=2147483647, inner_precise=0, prefix=None, sparse_mode=0, + gen_mask_parallel=True, sync=False + +# 反向函数声明对比 +标杆实现:fusion_attention_backward: dx, q, k, v, softmax_res, drop_mask, pse, scale, keep_prob +融合算子:npu_fusion_attention_backward: query, key, value, dy, head_num, input_layout, *, pse=None, padding_mask=None, + atten_mask=None, softmax_max=None, softmax_sum=None, softmax_in=None, + attention_in=None, scale_value=1.0, keep_prob=1.0, pre_tockens=2147483647, + next_tockens=2147483647, inner_precise=0, seed=0, offset=0, + numels=0, prefix=None, sparse_mode=0, gen_mask_parallel=True, sync=False +""" + + +def softmax_forward(x): + x_max = torch.max(x, dim=-1, keepdims=True)[0] + x_sub = x.sub(x_max) + y = torch.exp(x_sub) + x_sum = y.sum(dim=-1, keepdims=True) + res = y.div(x_sum) + return res, x_max, x_sum + + +def softmax_grad(dp, softmax_res): + muls = dp * softmax_res + muls_r = muls.sum(dim=-1, keepdims=True) + sub_r = dp - muls_r + res = sub_r * softmax_res + return res + + +def broadcast_kv(num_heads, num_kv_heads, kv_tensor, dtype): + if num_kv_heads == 0 or num_kv_heads < num_heads: + raise ValueError(f"num_kv_heads must be non-zero and less than num_heads.") + + factor = num_heads // num_kv_heads + kv_shape = kv_tensor.shape + B = kv_shape[0] + S = kv_shape[2] + D = kv_shape[3] + kv_res = torch.zeros([B, num_heads, S, D]).to(dtype) + for i in range(num_heads): + j = i // factor + kv_res[:, i:i + 1, :, :] = kv_tensor[:, j:j + 1, :, :] + return kv_res + + +def calculate_qk(q, k, atten_mask, pse, scale): + if pse is None or len(pse.shape) == 0: + qk = torch.matmul(q, k.permute(0, 1, 3, 2)).mul(scale) + else: + qk = (torch.matmul(q, k.permute(0, 1, 3, 2)) + pse).mul(scale) + if atten_mask is None or len(atten_mask.shape) == 0: + return qk + else: + qk = qk + atten_mask.bool() * (-40000.0) # -10000 + return qk + + +def fusion_attention_forward(q, k, v, drop_mask, atten_mask, pse, scale, keep_prob): + qk = calculate_qk(q, k, atten_mask, pse, scale) + softmax_res, softmax_max, softmax_sum = softmax_forward(qk) + if drop_mask is None or len(drop_mask.shape) == 0: + drop_res = softmax_res + else: + drop_res = softmax_res * drop_mask * (1.0 / keep_prob) + y = torch.matmul(drop_res, v) + return y, softmax_max, softmax_sum + + +def fusion_attention_backward(dx, q, k, v, softmax_res, drop_mask, pse, scale, keep_prob): + dp = torch.matmul(dx, v.permute(0, 1, 3, 2)) + if drop_mask is None or len(drop_mask.shape) == 0: + drop_res = softmax_res.permute(0, 1, 3, 2) + dp_drop = dp + else: + drop_res = softmax_res.mul(drop_mask).mul(1.0 / keep_prob).permute(0, 1, 3, 2) + dp_drop = dp * drop_mask * (1.0 / keep_prob) + dv = torch.matmul(drop_res, dx) + softmax_grad_res = (softmax_grad(dp_drop, softmax_res) * scale) + dq = torch.matmul(softmax_grad_res, k) + dk = torch.matmul(softmax_grad_res.permute(0, 1, 3, 2), q) + return dq, dk, dv + + +def parse_bsnd_args(query, key, head_num, input_layout): + supported_input_layout = ["BSH", "SBH", "BSND", "BNSD", "TND"] + B, S1, S2, N1, N2, D, H1, H2 = None, None, None, head_num, None, None, None, None + + if not isinstance(input_layout, str) or input_layout not in supported_input_layout: + raise ValueError(f"Invalid input_layout arg which must be one of {supported_input_layout}.") + + if input_layout == "TND": + raise ValueError(f"input_layout {input_layout} does not supported for now.") + try: + if input_layout == "BSH": + B, S1, H1 = query.shape + _, S2, H2 = key.shape + D = H1 // N1 + N2 = H2 // D + elif input_layout == "SBH": + S1, B, H1 = query.shape + S2, _, H2 = key.shape + D = H1 // N1 + N2 = H2 // D + elif input_layout == "BSND": + B, S1, N1, D = query.shape + _, S2, N2, _ = key.shape + H1 = N1 * D + H2 = N2 * D + elif input_layout == "BNSD": + B, N1, S1, D = query.shape + _, N2, S2, _ = key.shape + H1 = N1 * D + H2 = N2 * D + except Exception as e: + raise ValueError(f"query.shape: {query.shape}, key.shape: {key.shape}, parse_bsnd_args error: {e}") from e + + if D == 0: + raise ValueError(f"Value D must be non-zero.") + DTYPE = query.dtype + return B, S1, S2, N1, N2, D, H1, H2, DTYPE + + +def convert_from_bnsd(_input, input_layout): + if input_layout == "BSH": + # (B,N,S,D)=>(B,S,N*D) + out = rearrange(_input, 'b n s d -> b s (n d)').contiguous() + elif input_layout == "SBH": + # (B,N,S,D)=>(S,B,N*D) + out = rearrange(_input, 'b n s d -> s b (n d)').contiguous() + elif input_layout == "BSND": + # (B,N,S,D)=>(B,S,N,D) + out = rearrange(_input, 'b n s d -> b s n d').contiguous() + elif input_layout == "TND": + raise ValueError(f"input_layout {input_layout} does not supported for now.") + else: + out = _input + return out + + +def convert_to_bnsd(_input, n, input_layout): + # 默认"BNSD"无需处理 + if input_layout == "BSH": + # (B,S,N*D)=>(B,N,S,D) + out = rearrange(_input, 'b s (n d) -> b n s d', n=n) + elif input_layout == "SBH": + # (S,B,N*D)=>(B,N,S,D) + out = rearrange(_input, 's b (n d) -> b n s d', n=n) + elif input_layout == "BSND": + # (B,S,N,D)=>(B,N,S,D) + out = rearrange(_input, 'b s n d -> b n s d', n=n) + elif input_layout == "TND": + raise ValueError(f"input_layout {input_layout} does not supported for now.") + else: + out = _input + if out.dim() != 4: + raise ValueError(f"convert qkv format failed with input_layout {input_layout}.") + return out.to(gtype) + + +def generate_atten_mask(sparse_mode, atten_mask, B, N1, S1, S2, pre_tocken, next_tocken, dtype): + """ + # 当sparse_mode=2、3、4时小算子到融合算子会走这个优化，反过来看就要拆解回原来的基本实现 + ===> atten_mask = torch.from_numpy(np.triu(np.ones([2048, 2048]), k=1)).to(dtype) + """ + shape = [S1, S2] + + if atten_mask is not None: + # 当FA的输入已经包含atten_mask时，可以认为已经是转换之后的mask矩阵了，有三种特殊场景，即稀疏矩阵场景，需要进行逆向还原 + if sparse_mode == 2 or sparse_mode == 3 or sparse_mode == 4: + print(S1, S2, atten_mask.shape, atten_mask.dtype) + + if atten_mask.dim() == 2 and atten_mask.shape[0] == 2048 and atten_mask.shape[1] == 2048: + if atten_mask.equal(torch.from_numpy(np.triu(np.ones([2048, 2048]), k=1)).to(atten_mask.dtype)): + if sparse_mode == 2: + atten_mask = torch.from_numpy(np.triu(np.ones(shape), k=1)) + elif sparse_mode == 3: + atten_mask = torch.from_numpy(np.triu(np.ones(shape), k=S2 - S1 + 1)) + elif sparse_mode == 4: + atten_mask_u = torch.from_numpy(np.triu(np.ones(shape), k=next_tocken + 1)) + atten_mask_l = torch.from_numpy(np.tril(np.ones(shape), k=-pre_tocken - 1)) + atten_mask = atten_mask_u + atten_mask_l + logger.debug(f"反向转换atten_mask {atten_mask.shape}") + return atten_mask.to(dtype) + + return atten_mask.to(dtype) + + if atten_mask is not None: + if atten_mask.dim() == 2: + if atten_mask.shape[0] != S1 or atten_mask.shape[1] != S2: + raise ValueError(f"Invalid atten_mask shape `SS` {atten_mask.shape}") + shape = [S1, S2] + elif atten_mask.dim() == 4: + if atten_mask.shape[1] == 1: + shape = [B, 1, S1, S2] if B != 1 else [1, 1, S1, S2] + else: + shape = [B, N1, S1, S2] if B != 1 else [1, N1, S1, S2] + + if sparse_mode == 0: + atten_mask_u = torch.from_numpy(np.triu(np.ones(shape), k=next_tocken + 1)) + atten_mask_l = torch.from_numpy(np.tril(np.ones(shape), k=-pre_tocken - 1)) + atten_mask = atten_mask_u + atten_mask_l + elif sparse_mode == 1: # no sparse + atten_mask = torch.from_numpy(np.zeros(shape)) + elif sparse_mode == 2: + atten_mask = torch.from_numpy(np.triu(np.ones(shape), k=1)) + elif sparse_mode == 3: + atten_mask = torch.from_numpy(np.triu(np.ones(shape), k=S2 - S1 + 1)) + elif sparse_mode == 4: + atten_mask_u = torch.from_numpy(np.triu(np.ones(shape), k=next_tocken + 1)) + atten_mask_l = torch.from_numpy(np.tril(np.ones(shape), k=-pre_tocken - 1)) + atten_mask = atten_mask_u + atten_mask_l + # 注:不会出现sparse_mode=5的情况，该情况要求必须要传入atten_mask，且atten_mask矩阵数据格式须为BNSS或B1SS， + # 因此可以认为FA的输入已经是正确的atten_mask了 + return atten_mask.to(dtype) + + +def generate_kv(key, value, N1, N2): + # N不等长适配by cdy + if not (N1 == N2): + k_new = broadcast_kv(N1, N2, key, key.dtype) + v_new = broadcast_kv(N1, N2, value, value.dtype) + else: + k_new = key + v_new = value + return k_new, v_new + + +def rebuid_softmax_by_qkv(q, k, atten_mask, pse, scale): + """ + attention = softmax(QK^T/sqrt(d))V + softmax(x_i) = e^(x_i - x_max) / sum(e^(x_i - x_max)) + """ + print(f"Using QKV to rebuild original softmax") + qk = calculate_qk(q, k, atten_mask, pse, scale) + softmax_res, x_max, x_sum = softmax_forward(qk) + return softmax_res + + +def rebuild_softmax_by_max_sum(q, k, atten_mask, pse, scale, softmax_max, softmax_sum): + """ + attention = softmax(QK^T/sqrt(d))V + softmax(x_i) = e^(x_i - x_max_i) / x_sum_i) + """ + print(f"Using softmax_max and softmax_sum to rebuild original softmax") + qk = calculate_qk(q, k, atten_mask, pse, scale) + if softmax_max.shape[-1] == 0: + raise ValueError(f"softmax_max.shape[-1] must be non-zero, softmax_max.shape: {softmax_max.shape}") + repeat_dim = qk.shape[-1] // softmax_max.shape[-1] + softmax_res = torch.exp(qk.sub(softmax_max.repeat(1, 1, 1, repeat_dim))).div( + softmax_sum.repeat(1, 1, 1, repeat_dim)) + return softmax_res + + +def npu_fusion_attention_forward_patch(*args, **kwargs): + # query, key, value, head_num, input_layout + if len(args) != 5: + raise ValueError(f"Unsupported npu_fusion_attention args {args}.") + + B, S1, S2, N1, N2, D, H1, H2, DTYPE = parse_bsnd_args(args[0], args[1], args[3], args[4]) + if N1 == N2 and S1 == S2: + logger.debug(f"running case : BNSD = {B}_{N1}_{S1}_{D}, sparse = {kwargs.get('sparse_mode', 0)}") + else: + logger.debug(f"running case: BNSD = {B}_{N1}({N2})_{S1}({S2})_{D}, sparse = {kwargs.get('sparse_mode', 0)}") + if not (N1 % N2 == 0 and N1 >= N2): + raise ValueError(f"N1与N2不匹配,请检查: N1 = {N1}, N2 = {N2}.") + + dims_kwargs = {"B": B, "S1": S1, "S2": S2, "N1": N1, "N2": N2, + "D": D, "H1": H1, "H2": H2, "DTYPE": DTYPE} + + new_kwargs = {"keep_prob": 1, + "scale": kwargs.get("scale", 1 / (D ** 0.5)), + "sparse_mode": kwargs.get("sparse_mode", 0), + "prefix": kwargs.get("prefix"), + "pre_tockens": kwargs.get("pre_tockens", 2147483647), + "next_tockens": kwargs.get("next_tockens", 2147483647), + "pse": kwargs.get("pse"), + "padding_mask": kwargs.get("padding_mask"), + "atten_mask": kwargs.get("atten_mask")} + + return args, dims_kwargs, new_kwargs + + +def npu_fusion_attention_backward_patch(*args, **kwargs): + if len(args) != 6: + raise ValueError(f"Unsupported npu_fusion_attention_grad args {args}.") + + B, S1, S2, N1, N2, D, H1, H2, DTYPE = parse_bsnd_args(args[0], args[1], args[4], args[5]) + if N1 == N2 and S1 == S2: + print(f"running case : BNSD = {B}_{N1}_{S1}_{D}, sparse = {kwargs.get('sparse_mode', 0)}") + else: + print(f"running case: BNSD = {B}_{N1}({N2})_{S1}({S2})_{D}, sparse = {kwargs.get('sparse_mode', 0)}") + if not (N1 % N2 == 0 and N1 >= N2): + raise ValueError(f"N1与N2不匹配,请检查: N1 = {N1}, N2 = {N2}.") + + dims_kwargs = {"B": B, "S1": S1, "S2": S2, "N1": N1, "N2": N2, + "D": D, "H1": H1, "H2": H2, "DTYPE": DTYPE} + + new_kwargs = {"keep_prob": 1, + "scale_value": kwargs.get("scale_value", 1 / (D ** 0.5)), + "sparse_mode": kwargs.get("sparse_mode", 0), + "prefix": kwargs.get("prefix"), + "pre_tockens": kwargs.get("pre_tockens", 2147483647), + "next_tockens": kwargs.get("next_tockens", 2147483647), + "pse": kwargs.get("pse"), + "padding_mask": kwargs.get("padding_mask"), + "softmax_max": kwargs.get("softmax_max"), + "softmax_sum": kwargs.get("softmax_sum"), + "softmax_in": kwargs.get("softmax_in"), + "attention_in": kwargs.get("attention_in"), + "seed": kwargs.get("seed", 0), + "offset": kwargs.get("offset", 0), + "numels": kwargs.get("numels", 0), + "atten_mask": kwargs.get("atten_mask")} + + return args, dims_kwargs, new_kwargs + + +@npu_custom_functions +def npu_fusion_attention(*args, **kwargs): + new_args, dims_kwargs, new_kwargs = npu_fusion_attention_forward_patch(*args, **kwargs) + query, key, value, input_layout = new_args[0], new_args[1], new_args[2], new_args[4] + N1 = dims_kwargs.get("N1") + N2 = dims_kwargs.get("N2") + S1 = dims_kwargs.get("S1") + S2 = dims_kwargs.get("S2") + B = dims_kwargs.get("B") + DTYPE = dims_kwargs.get("DTYPE") + atten_mask = new_kwargs.get("atten_mask") + keep_prob = new_kwargs.get("keep_prob") + sparse_mode = new_kwargs.get("sparse_mode") + pre_tockens = new_kwargs.get("pre_tockens") + next_tockens = new_kwargs.get("next_tockens") + pse = new_kwargs.get("pse") + scale = new_kwargs.get("scale") + + atten_mask = generate_atten_mask(sparse_mode, atten_mask, B, N1, S1, S2, pre_tockens, next_tockens, DTYPE) + query = convert_to_bnsd(query, N1, input_layout) + key = convert_to_bnsd(key, N2, input_layout) + value = convert_to_bnsd(value, N2, input_layout) + k_new, v_new = generate_kv(key, value, N1, N2) + out_golden, softmax_max, softmax_sum = fusion_attention_forward(q=query, k=k_new, v=v_new, + drop_mask=None, atten_mask=atten_mask, + pse=pse, scale=scale, + keep_prob=keep_prob) + if out_golden.dim() == 5: + out_golden = out_golden.reshape(out_golden.size(0), out_golden.size(1) * out_golden.size(2), out_golden.size(3), + out_golden.size(4)) + out_golden = convert_from_bnsd(out_golden, input_layout) + + return out_golden.cpu(), softmax_max.repeat(1, 1, 1, 8).cpu(), softmax_sum.repeat(1, 1, 1, 8).cpu() + + +@npu_custom_grad_functions +def npu_fusion_attention_grad(*args, **kwargs): + # dx, q, k, v, softmax_res, drop_mask, pse, scale, keep_prob + new_args, dims_kwargs, new_kwargs = npu_fusion_attention_backward_patch(*args, **kwargs) + query, key, value, dx, input_layout = new_args[0], new_args[1], new_args[2], new_args[3], new_args[5] + N1 = dims_kwargs.get("N1") + N2 = dims_kwargs.get("N2") + S1 = dims_kwargs.get("S1") + S2 = dims_kwargs.get("S2") + B = dims_kwargs.get("B") + D = dims_kwargs.get("D") + DTYPE = dims_kwargs.get("DTYPE") + atten_mask = new_kwargs.get("atten_mask") + keep_prob = new_kwargs.get("keep_prob") + sparse_mode = new_kwargs.get("sparse_mode") + pre_tockens = new_kwargs.get("pre_tockens") + next_tockens = new_kwargs.get("next_tockens") + pse = new_kwargs.get("pse") + softmax_max = new_kwargs.get("softmax_max") + softmax_sum = new_kwargs.get("softmax_sum") + scale_value = new_kwargs.get("scale_value") + + atten_mask = generate_atten_mask(sparse_mode, atten_mask, B, N1, S1, S2, pre_tockens, next_tockens, DTYPE) + query = convert_to_bnsd(query, N1, input_layout) + dx = convert_to_bnsd(dx, N1, input_layout) + key = convert_to_bnsd(key, N2, input_layout) + value = convert_to_bnsd(value, N2, input_layout) + k_new, v_new = generate_kv(key, value, N1, N2) + + if softmax_build_mode == "QKV": + softmax_res = rebuid_softmax_by_qkv(query, k_new, atten_mask, pse, scale_value) + else: + softmax_res = rebuild_softmax_by_max_sum(query, k_new, atten_mask, pse, scale_value, softmax_max, softmax_sum) + + dq, dk, dv = fusion_attention_backward(dx, query, k_new, v_new, softmax_res, None, pse, scale_value, keep_prob) + + # N不等长适配by cdy + if not (N1 == N2): + if N2 == 0: + raise ValueError("dims_kwargs.N2 must be non-zero.") + G = int(N1 / N2) + dk = torch.sum(dk.reshape(B, N2, G, S2, D), dim=2, keepdim=True).reshape(B, N2, S2, D) + dv = torch.sum(dv.reshape(B, N2, G, S2, D), dim=2, keepdim=True).reshape(B, N2, S2, D) + + if dq.dim() == 5: + dq = dq.reshape(dq.size(0), dq.size(1) * dq.size(2), dq.size(3), dq.size(4)) + if dk.dim() == 5: + dk = dk.reshape(dk.size(0), dk.size(1) * dk.size(2), dk.size(3), dk.size(4)) + if dv.dim() == 5: + dv = dv.reshape(dv.size(0), dv.size(1) * dv.size(2), dv.size(3), dv.size(4)) + + dq = convert_from_bnsd(dq, input_layout) + dk = convert_from_bnsd(dk, input_layout) + dv = convert_from_bnsd(dv, input_layout) + + return dq.cpu(), dk.cpu(), dv.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py new file mode 100644 index 000000000..0fe6c834a --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py @@ -0,0 +1,18 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions + + +@npu_custom_functions +def npu_rms_norm(x, gamma, epsilon=1e-5): + rstd = torch.rsqrt(torch.mean(torch.pow(x, 2), axis=-1, keepdim=True) + epsilon) + res = x * rstd * gamma + return res.cpu(), rstd.float().cpu() + + +@npu_custom_grad_functions +def npu_rms_norm_backward(grad, x, gamma, rstd): + mean_gy = (grad * x * gamma * rstd).mean(dim=-1, keepdim=True) + grad_x = (grad * gamma - x * rstd * mean_gy) * rstd + grad_gamma = x * grad * rstd + return grad_x.cpu(), grad_gamma.cpu() + diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py new file mode 100644 index 000000000..76b3828da --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py @@ -0,0 +1,55 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions + + +@npu_custom_functions +def npu_rotary_mul(x, r1, r2): + x1, x2 = torch.chunk(x, 2, -1) + x_new = torch.cat((-x2, x1), dim=-1) + output = r1 * x + r2 * x_new + return output.cpu() + + +@npu_custom_grad_functions +def npu_rotary_mul_backward(dy_tensor, x, r1, r2): + x.requires_grad = True + r1.requires_grad = True + r2.requires_grad = True + # golden + x1, x2 = torch.chunk(x, 2, -1) + x_new = torch.cat((-x2, x1), dim=-1) + golden_tensor = r1 * x + r2 * x_new + golden_tensor.backward(dy_tensor) + r1_shape = r1.shape + r1_grad = torch.zeros(r1_shape).type(torch.float32) + r2_grad = torch.zeros(r1_shape).type(torch.float32) + x1, x2 = torch.chunk(x.float(), 2, -1) + x_new2 = torch.cat((-x2, x1), dim=-1) + x_shape = x.shape + h = x.float() + grad = dy_tensor.float() + condition_1 = (((r1_shape[0] == 1 and x_shape[0] != 1) or (r1_shape[0] == 1 and x_shape[0] == 1)) and + ((r1_shape[2] == 1 and x_shape[2] != 1) or (r1_shape[2] == 1 and x_shape[2] == 1)) and + (r1_shape[1] == x_shape[1]) and (r1_shape[3] == x_shape[3])) + condition_2 = (((r1_shape[0] == 1 and x_shape[0] != 1) or (r1_shape[0] == 1 and x_shape[0] == 1)) and + ((r1_shape[1] == 1 and x_shape[1] != 1) or (r1_shape[1] == 1 and x_shape[1] == 1)) and + (r1_shape[2] == x_shape[2]) and (r1_shape[3] == x_shape[3])) + condition_3 = (((r1_shape[2] == 1 and x_shape[2] != 1) or (r1_shape[2] == 1 and x_shape[2] == 1)) and + ((r1_shape[1] == 1 and x_shape[1] != 1) or (r1_shape[1] == 1 and x_shape[1] == 1)) and + (r1_shape[0] == x_shape[0]) and (r1_shape[3] == x_shape[3])) + if condition_1: + for i in range(x_shape[0]): + for j in range(x_shape[2]): + r2_grad[0, :, 0, :] += (x_new2[i, :, j, :] * grad[i, :, j, :]) + r1_grad[0, :, 0, :] += (h[i, :, j, :] * grad[i, :, j, :]) + elif condition_2: + for i in range(x_shape[0]): + for j in range(x_shape[1]): + r2_grad[0, 0, :, :] += (x_new2[i, j, :, :] * grad[i, j, :, :]) + r1_grad[0, 0, :, :] += (h[i, j, :, :] * grad[i, j, :, :]) + elif condition_3: + for i in range(x_shape[1]): + for j in range(x_shape[2]): + r2_grad[:, 0, 0, :] += (x_new2[:, i, j, :] * grad[:, i, j, :]) + r1_grad[:, 0, 0, :] += (h[:, i, j, :] * grad[:, i, j, :]) + return x.grad.cpu(), r1_grad.cpu(), r2_grad.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py new file mode 100644 index 000000000..bcc523ee4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py @@ -0,0 +1,29 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions + + +@npu_custom_functions +def npu_scaled_masked_softmax(x, mask, scale, fixed_triu_mask): + if fixed_triu_mask: + mask = (torch.triu(torch.ones(mask.shape), k=1)).bool().to(mask.device) + dtype = x.dtype + x = (x * scale).masked_fill(mask, value=-10000) + x = x - torch.max(x, dim=-1, keepdims=True)[0] + x = torch.exp(x.float()) + y = torch.div(x, torch.sum(x, dim=-1, keepdims=True)) + return y.to(dtype).cpu() + + +@npu_custom_grad_functions +def npu_scaled_masked_softmax_backward(y_grad, y, mask, scale, fixed_triu_mask): + if fixed_triu_mask: + mask = (torch.triu(torch.ones(mask.shape), k=1)).bool().to(mask.device) + dtype = y_grad.dtype + y_grad = y_grad.float() + y = y.float() + x_grad = y_grad * y + x_grad = y_grad - torch.sum(x_grad, dim=-1, keepdims=True) + x_grad = x_grad * y + x_grad = x_grad * scale + x_grad = x_grad.masked_fill(mask, value=0) + return x_grad.to(dtype).cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py new file mode 100644 index 000000000..973be454d --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py @@ -0,0 +1,58 @@ +import torch +from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions + + +@npu_custom_functions +def npu_swiglu(x, dim=-1): + tensor_dtype = x.dtype + + inTensors = torch.chunk(x, 2, dim=dim) + if tensor_dtype == torch.float32: + tensor_scalar = torch.sigmoid(torch.mul(inTensors[0], 1.0)) + output_data = torch.mul(torch.mul(tensor_scalar, inTensors[0]), inTensors[1]) + else: + tensor_self_float = inTensors[0].type(torch.float) + tensor_other_float = inTensors[1].type(torch.float) + tensor_out_float = torch.nn.functional.silu(tensor_self_float).type(tensor_dtype).type( + torch.float32) * tensor_other_float + output_data = tensor_out_float.type(tensor_dtype) + return output_data.cpu() + + +@npu_custom_grad_functions +def npu_swiglu_backward(grad, x, dim=-1): + tensor_dtype = grad.dtype + in_tensors = torch.chunk(x, 2, dim=dim) + tensor_grad_out = grad + + if tensor_dtype == torch.float16: + tensor_out1 = torch.mul( + torch.mul(in_tensors[1].type(torch.float32), swish_grad(1, in_tensors[0].type(torch.float32))), + tensor_grad_out.type(torch.float32)).type(torch.float16) + tensor_out2 = torch.mul(tensor_grad_out.type(torch.float32), + swish(1, in_tensors[0].type(torch.float32))).type(torch.float16) + output = torch.cat((tensor_out1, tensor_out2), dim) + elif tensor_dtype == torch.bfloat16: + tensor_self_float = in_tensors[0].type(torch.float) + tensor_other_float = in_tensors[1].type(torch.float) + tensor_gradout_float = tensor_grad_out.type(torch.float) + + tensor_out1 = torch.mul(tensor_gradout_float, swish_grad(1.0, tensor_self_float)).type(torch.bfloat16).type( + torch.float32) * tensor_other_float + tensor_out2 = swish(1.0, tensor_self_float).type(torch.bfloat16).type(torch.float32) * tensor_gradout_float + tensor_out_float = torch.cat((tensor_out1, tensor_out2), dim=dim) + output = tensor_out_float.type(torch.bfloat16) + else: + tensor_out1 = torch.mul(torch.mul(in_tensors[1], swish_grad(1.0, in_tensors[0])), tensor_grad_out) + tensor_out2 = torch.mul(tensor_grad_out, swish(1.0, in_tensors[0])) + output = torch.cat((tensor_out1, tensor_out2), dim) + return output.cpu() + + +def swish_grad(beta, x): + return torch.sigmoid(beta * x) + x * (1 - torch.sigmoid(beta * x)) * torch.sigmoid(beta * x) * beta + + +def swish(beta, x): + return x * torch.sigmoid(beta * x) + diff --git a/debug/accuracy_tools/msprobe/pytorch/function_factory.py b/debug/accuracy_tools/msprobe/pytorch/function_factory.py new file mode 100644 index 000000000..6934cc069 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/function_factory.py @@ -0,0 +1,47 @@ +class Register(dict): + def __init__(self, *args, **kwargs): + super(Register, self).__init__(*args, **kwargs) + self._dict = {} + + def register(self, target): + + def add_register_item(key, value): + if key in self._dict: + print(f"warning: {value.__name__} has been registered before, so we will overriden it.") + self[key] = value + return value + + if callable(target): + return add_register_item(target.__name__, target) + else: + raise Exception(f"The func {target} is not callable.") + + def __call__(self, target): + return self.register(target) + + def __setitem__(self, key, value): + self._dict[key] = value + + def __getitem__(self, key): + return self._dict[key] + + def __contains__(self, key): + return key in self._dict + + def __str__(self): + return str(self._dict) + + def keys(self): + return self._dict.keys() + + def values(self): + return self._dict.values() + + def items(self): + return self._dict.items() + + +npu_custom_functions = Register() +npu_custom_grad_functions = Register() + +from msprobe.pytorch.bench_functions import * diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py index 4617e4854..2c1805ab8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py @@ -24,12 +24,14 @@ from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const from msprobe.core.common.file_check import FileOpen - +from msprobe.pytorch.function_factory import npu_custom_grad_functions cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") with FileOpen(yaml_path, 'r') as f: - WrapAtenOps = yaml.safe_load(f).get('aten') + Ops = yaml.safe_load(f) + WrapAtenOps = Ops.get('aten') + WhiteAtenOps = Ops.get('white_aten_ops', []) aten_func = {} @@ -48,7 +50,7 @@ class HOOKAtenOP(object): class AtenOPTemplate(HOOKModule): - def __init__(self, op, hook): + def __init__(self, op, hook, need_hook=True): if isinstance(op, torch._ops.OpOverloadPacket): op_name_ = op._qualified_op_name.split("::")[-1] else: @@ -58,11 +60,20 @@ class AtenOPTemplate(HOOKModule): op_name_ = op_name_ + '.' + overload_name self.op = op self.prefix_op_name_ = "Aten" + Const.SEP + str(op_name_) + Const.SEP - super().__init__(hook) + self.need_hook = need_hook + if self.need_hook: + super().__init__(hook) @torch_device_guard def forward(self, *args, **kwargs): - return self.op(*args, **kwargs) + if self.op in npu_custom_grad_functions: + return npu_custom_grad_functions[self.op](*args, **kwargs) + if self.op in WhiteAtenOps: + return eval(f"torch.ops.aten.{self.op}")(*args, **kwargs) + if self.op not in aten_func: + raise Exception(f"Skip op[{self.op}] accuracy check, because the op is not " + f"in dir(torch.ops.aten) and support yaml.") + return aten_func[self.op](*args, **kwargs) class AtenOPPacketTemplate(): diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index 992713bce..db9f99683 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -21,9 +21,11 @@ import torch_npu import yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule +from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.common.utils import torch_device_guard, torch_without_guard_version from msprobe.core.common.const import Const from msprobe.core.common.file_check import FileOpen +from msprobe.pytorch.function_factory import npu_custom_functions cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") @@ -37,7 +39,10 @@ def get_npu_ops(): _npu_ops = dir(torch.ops.npu) else: _npu_ops = dir(torch_npu._C._VariableFunctionsClass) - return set(WrapNpuOps) & set(_npu_ops) + if msCheckerConfig.white_list: + return set(WrapNpuOps) & set(_npu_ops) & set(msCheckerConfig.white_list) + else: + return set(WrapNpuOps) & set(_npu_ops) class HOOKNpuOP(object): @@ -46,13 +51,19 @@ class HOOKNpuOP(object): class NpuOPTemplate(HOOKModule): - def __init__(self, op_name, hook): + def __init__(self, op_name, hook, need_hook=True): self.op_name_ = op_name self.prefix_op_name_ = "NPU" + Const.SEP + str(op_name) + Const.SEP - super().__init__(hook) + self.need_hook = need_hook + if need_hook: + super().__init__(hook) @torch_device_guard def forward(self, *args, **kwargs): + if not self.need_hook: + if self.op_name_ not in npu_custom_functions: + raise Exception(f'There is not bench function {self.op_name_}') + return npu_custom_functions[self.op_name_](*args, **kwargs) if torch_without_guard_version: return getattr(torch.ops.npu, str(self.op_name_))(*args, **kwargs) else: @@ -60,7 +71,6 @@ class NpuOPTemplate(HOOKModule): def wrap_npu_op(op_name, hook): - def npu_op_template(*args, **kwargs): return NpuOPTemplate(op_name, hook)(*args, **kwargs) -- Gitee From 4e26436f8fef859d698b637bb380c1ab58fae723 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 29 Jul 2024 16:16:46 +0800 Subject: [PATCH 049/791] add codecheck --- .../pytorch/bench_functions/apply_adam_w.py | 7 ++-- .../bench_functions/matmul_backward.py | 1 - .../bench_functions/npu_fusion_attention.py | 10 ++--- .../msprobe/pytorch/common/utils.py | 41 +++++++++++++++++++ .../msprobe/pytorch/function_factory.py | 29 +++++++------ 5 files changed, 66 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py index 3cebd3050..dc0954911 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py @@ -9,12 +9,13 @@ def npu_apply_adam_w(beta1_power, beta2_power, lr, weight_decay, var, m, v = out if amsgrad: max_grad_norm = (torch.rand(var.shape) * 10.0 - 5.0).to(var.dtype) + beta1_power_out = beta1_power * beta1 + beta2_power_out = beta2_power * beta2 + var_t = var * (1 + (-lr * weight_decay)) gt = -grad if maximize else grad m_out = m * beta1 - (beta1 + (-1)) * gt v_out = v * beta2 - (beta2 + (-1)) * gt * gt - var_t = var * (1 + (-lr * weight_decay)) - beta1_power_out = beta1_power * beta1 - beta2_power_out = beta2_power * beta2 + if amsgrad: max_grad_norm_out = torch.max(max_grad_norm, v_out) if (1 - beta2_power_out) == 0: diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py index dae274552..3c4f7dc04 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py @@ -26,7 +26,6 @@ def matmul_backward(grad, self, other, mask): if mask[0]: grad_self = unfolded_grad.mm(other.unsqueeze(0) if dim_other == 1 else other.transpose(-1, -2)) \ .view(size_self) - print(f'size_self: {size_self}') if mask[1]: unfolded_self = self.contiguous().view([-1, size_self[-1]]) grad_other = unfolded_self.transpose(-1, -2).mm(unfolded_grad).view(size_other) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index 2a46d0200..6a49ce740 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -183,7 +183,7 @@ def generate_atten_mask(sparse_mode, atten_mask, B, N1, S1, S2, pre_tocken, next if atten_mask is not None: # 当FA的输入已经包含atten_mask时，可以认为已经是转换之后的mask矩阵了，有三种特殊场景，即稀疏矩阵场景，需要进行逆向还原 if sparse_mode == 2 or sparse_mode == 3 or sparse_mode == 4: - print(S1, S2, atten_mask.shape, atten_mask.dtype) + logger.info(f"S1: {S1}, S2:{S2}, atten_mask.shape:{atten_mask.shape}, atten_mask.dtype:{atten_mask.dtype}") if atten_mask.dim() == 2 and atten_mask.shape[0] == 2048 and atten_mask.shape[1] == 2048: if atten_mask.equal(torch.from_numpy(np.triu(np.ones([2048, 2048]), k=1)).to(atten_mask.dtype)): @@ -246,7 +246,7 @@ def rebuid_softmax_by_qkv(q, k, atten_mask, pse, scale): attention = softmax(QK^T/sqrt(d))V softmax(x_i) = e^(x_i - x_max) / sum(e^(x_i - x_max)) """ - print(f"Using QKV to rebuild original softmax") + logger.info("Using QKV to rebuild original softmax") qk = calculate_qk(q, k, atten_mask, pse, scale) softmax_res, x_max, x_sum = softmax_forward(qk) return softmax_res @@ -257,7 +257,7 @@ def rebuild_softmax_by_max_sum(q, k, atten_mask, pse, scale, softmax_max, softma attention = softmax(QK^T/sqrt(d))V softmax(x_i) = e^(x_i - x_max_i) / x_sum_i) """ - print(f"Using softmax_max and softmax_sum to rebuild original softmax") + logger.info("Using softmax_max and softmax_sum to rebuild original softmax") qk = calculate_qk(q, k, atten_mask, pse, scale) if softmax_max.shape[-1] == 0: raise ValueError(f"softmax_max.shape[-1] must be non-zero, softmax_max.shape: {softmax_max.shape}") @@ -302,9 +302,9 @@ def npu_fusion_attention_backward_patch(*args, **kwargs): B, S1, S2, N1, N2, D, H1, H2, DTYPE = parse_bsnd_args(args[0], args[1], args[4], args[5]) if N1 == N2 and S1 == S2: - print(f"running case : BNSD = {B}_{N1}_{S1}_{D}, sparse = {kwargs.get('sparse_mode', 0)}") + logger.info(f"running case : BNSD = {B}_{N1}_{S1}_{D}, sparse = {kwargs.get('sparse_mode', 0)}") else: - print(f"running case: BNSD = {B}_{N1}({N2})_{S1}({S2})_{D}, sparse = {kwargs.get('sparse_mode', 0)}") + logger.info(f"running case: BNSD = {B}_{N1}({N2})_{S1}({S2})_{D}, sparse = {kwargs.get('sparse_mode', 0)}") if not (N1 % N2 == 0 and N1 >= N2): raise ValueError(f"N1与N2不匹配,请检查: N1 = {N1}, N2 = {N2}.") diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index acc1de105..9028d7918 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -14,10 +14,12 @@ # See the License for the specific language governing permissions and # limitations under the License. """ +import logging import os import random import stat import torch +import torch.distributed as dist import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError @@ -221,3 +223,42 @@ class Const: CONVERT_API = { "int32_to_int64": ["cross_entropy"] } + + +def get_tensor_rank(in_feat, out_feat): + if dist.is_initialized(): + return dist.get_rank() + + def get_tensor_rank_single(x): + if isinstance(x, (list, tuple)): + if len(x) > 0: + return get_tensor_rank_single(x[0]) + return None + elif isinstance(x, torch.Tensor): + device = x.device + if device.type == 'cpu': + return None + else: + return device.index + return None + + in_rank = get_tensor_rank_single(in_feat) + if in_rank is not None: + return in_rank + out_rank = get_tensor_rank_single(out_feat) + if out_rank is not None: + return out_rank + return None + + +def _create_logger(level=logging.INFO): + logger_ = logging.getLogger() + logger_.setLevel(level) + ch = logging.StreamHandler() + ch.setLevel(level) + logger_.addHandler(ch) + return logger_ + + +log_level = logging.DEBUG if os.environ.get("API_ACCURACY_CHECK_LOG_LEVEL") == "1" else logging.INFO +logger = _create_logger(log_level) diff --git a/debug/accuracy_tools/msprobe/pytorch/function_factory.py b/debug/accuracy_tools/msprobe/pytorch/function_factory.py index 6934cc069..4e725de4f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/function_factory.py +++ b/debug/accuracy_tools/msprobe/pytorch/function_factory.py @@ -1,21 +1,11 @@ +from msprobe.pytorch.common.utils import logger + + class Register(dict): def __init__(self, *args, **kwargs): super(Register, self).__init__(*args, **kwargs) self._dict = {} - def register(self, target): - - def add_register_item(key, value): - if key in self._dict: - print(f"warning: {value.__name__} has been registered before, so we will overriden it.") - self[key] = value - return value - - if callable(target): - return add_register_item(target.__name__, target) - else: - raise Exception(f"The func {target} is not callable.") - def __call__(self, target): return self.register(target) @@ -40,6 +30,19 @@ class Register(dict): def items(self): return self._dict.items() + def register(self, target): + + def add_register_item(key, value): + if key in self._dict: + logger.warning(f"{value.__name__} has been registered before, so we will overriden it.") + self[key] = value + return value + + if callable(target): + return add_register_item(target.__name__, target) + else: + raise Exception(f"The func {target} is not callable.") + npu_custom_functions = Register() npu_custom_grad_functions = Register() -- Gitee From 93d34b067504fddc86b868e98360e0e4e20ea5f2 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 30 Jul 2024 09:06:48 +0800 Subject: [PATCH 050/791] fix --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 85d5c65e5..b59536aa5 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -84,7 +84,7 @@ class Const: INPLACE_LIST = [ "broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter", - "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single" + "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single", "all_to_all" ] CONVERT = { -- Gitee From ad47e040b9643a6eb205b1b87c6f92f3d0b95aa0 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Tue, 30 Jul 2024 09:58:12 +0800 Subject: [PATCH 051/791] fix overflow_nums bug --- debug/accuracy_tools/msprobe/core/common_config.py | 4 ++-- .../core/data_dump/data_processor/pytorch_processor.py | 2 +- .../msprobe/pytorch/debugger/debugger_config.py | 2 +- debug/accuracy_tools/msprobe/pytorch/pt_config.py | 4 ++-- .../accuracy_tools/msprobe/test/core_ut/test_common_config.py | 2 +- .../accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index ed38eba00..b4bf5cf28 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -44,8 +44,8 @@ class BaseConfig: self.data_mode = json_config.get('data_mode') self.backward_input = json_config.get("backward_input") self.file_format = json_config.get("file_format") - self.summary_mode = json_config.get("summary_mode") - self.overflow_num = json_config.get("overflow_num") + self.summary_mode = json_config.get("summary_mode") + self.overflow_nums = json_config.get("overflow_nums") self.check_mode = json_config.get("check_mode") def check_config(self): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 4cdd3ea04..00cab5e54 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -182,7 +182,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): super().__init__(config, data_writer) self.cached_tensors_and_file_paths = {} self.real_overflow_dump_times = 0 - self.overflow_nums = config.overflow_num + self.overflow_nums = config.overflow_nums self.bits_for_overflow = 8 @staticmethod diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index cfc588e1e..f1289e9b0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -21,7 +21,7 @@ class DebuggerConfig: self.acl_config = common_config.acl_config if common_config.acl_config else "" self.is_forward_acl_dump = True self.summary_mode = task_config.summary_mode if task_config.summary_mode else Const.STATISTICS - self.overflow_num = task_config.overflow_num if task_config.overflow_num else 1 + self.overflow_nums = task_config.overflow_nums if task_config.overflow_nums else 1 self.framework = Const.PT_FRAMEWORK if self.task == Const.FREE_BENCHMARK: diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index a3d765f3a..ceec92a63 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -32,12 +32,12 @@ class StatisticsConfig(BaseConfig): class OverflowCheckConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) - self.overflow_num = json_config.get("overflow_nums") + self.overflow_nums = json_config.get("overflow_nums") self.check_mode = json_config.get("check_mode") self.check_overflow_config() def check_overflow_config(self): - if self.overflow_num is not None and not isinstance(self.overflow_num, int): + if self.overflow_nums is not None and not isinstance(self.overflow_nums, int): raise Exception("overflow_num is invalid") if self.check_mode is not None and self.check_mode not in ["all", "aicore", "atomic"]: raise Exception("check_mode is invalid") diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index 06c7378ed..8b2138a48 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -121,7 +121,7 @@ class TestCommonConfig(TestCase): self.assertIsNone(base_config.backward_input) self.assertIsNone(base_config.file_format) self.assertIsNone(base_config.summary_mode) - self.assertIsNone(base_config.overflow_num) + self.assertIsNone(base_config.overflow_nums) self.assertIsNone(base_config.check_mode) json_config.update({"scope": "Tensor_Add"}) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py index c344f0b66..470390d77 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py @@ -45,7 +45,7 @@ class TestPtConfig(TestCase): } } result = parse_task_config(Const.OVERFLOW_CHECK, overflow_check_config) - self.assertEqual(result.overflow_num, 1) + self.assertEqual(result.overflow_nums, 1) self.assertEqual(result.check_mode, "all") free_benchmark_config = { -- Gitee From 718154b7a9fdede0544dc93b4cbeacea6faa6b63 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Tue, 30 Jul 2024 10:49:24 +0800 Subject: [PATCH 052/791] fix overflowcheck bug --- .../msprobe/core/data_dump/data_processor/pytorch_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 4cdd3ea04..8dac54fd2 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -255,7 +255,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): else: logger.warning(f'The file path {file_path} length exceeds limit.') single_arg = super()._analyze_tensor(tensor, suffix) - self._analyze_maybe_overflow_tensor(single_arg, tensor) + self._analyze_maybe_overflow_tensor(single_arg) single_arg.update({"data_name": dump_data_name}) return single_arg -- Gitee From 92f57760df818537865272c2c9deee80b79dd962 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Mon, 29 Jul 2024 14:51:06 +0800 Subject: [PATCH 053/791] =?UTF-8?q?[compare]=E6=96=B0=E5=A2=9E--enable=5Fa?= =?UTF-8?q?pi=5Fcompare=E5=92=8C--enable=5Fkernel=5Fcompare?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/compare_tools/README.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index 78ea5d897..31b0935ae 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -145,6 +145,8 @@ python performance_compare.py [基准性能数据文件所在路径] [比对性 | --enable_operator_compare | 开启算子性能比对。MindSpore场景暂不支持。该开关较耗时，建议只采集一个step的性能数据。 | 否 | | --enable_communication_compare | 开启通信性能比对。 | 否 | | --enable_memory_compare | 开启算子内存比对。MindSpore场景暂不支持。该开关较耗时，建议只采集一个step的性能数据。 | 否 | +| --enable_kernel_compare | 开启kernel性能比对。仅针对NPU与NPU比对的场景。需要使用性能数据中的kernel_details.csv文件。 | 否 | +| --enable_api_compare | 开启API性能比对。需要使用性能数据中的trace_view.csv文件。 | 否 | | --disable_details | 隐藏明细比对，只进行统计级比对。 | 否 | 说明：以上开关均不设置的情况下，**工具默认开启所有的性能比对**，当用户设置了以上开关，则按照用户设置的开关进行性能比对，示例如下： @@ -300,3 +302,29 @@ MindSpore场景暂不支持。 步骤1：查看MemoryCompareStatistic页，找出内存占用差距TOP的算子。 步骤2：查看MemoryCompare页，搜索内存占用差距TOP的算子，查看具体占用的子算子。 + +### kernel性能 + +仅针对NPU与NPU比对的场景。 + +kernel比对结果在performance_comparison_result_*.xlsx中KernelCompare页呈现。 + +按照Kernel（Kernel类型）和Input Shapes（输入Shape）分组统计，统计信息包括： + +- Total Duration(us)：总耗时，单位us。 +- Avg Duration(us)：平均耗时，单位us。 +- Max Duration(us)：最大耗时，单位us。 +- Min Duration(us)：最小耗时，单位us。 +- Calls：调用次数。 + +### API性能 + +API比对结果在performance_comparison_result_*.xlsx中ApiCompare页呈现。 + +按照api name（API名称）组统计，统计信息包括： + +- Total Duration(ms)：总耗时，单位ms。 +- Self Time(ms)：Self耗时（排除掉子event），单位ms。 +- Avg Duration(ms)：平均耗时，单位ms。 +- Calls：调用次数。 + -- Gitee From c52ed966597c2e333594b6bd2b0fad226877843c Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 29 Jul 2024 21:01:32 +0800 Subject: [PATCH 054/791] fix aten --- .../msprobe/pytorch/common/utils.py | 12 +++--------- .../msprobe/pytorch/function_factory.py | 18 +++++++++++++++++- .../msprobe/pytorch/hook_module/wrap_aten.py | 18 ++++++++++-------- .../pytorch/hook_module/wrap_npu_custom.py | 14 ++++++++------ 4 files changed, 38 insertions(+), 24 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 9028d7918..181491488 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -233,22 +233,16 @@ def get_tensor_rank(in_feat, out_feat): if isinstance(x, (list, tuple)): if len(x) > 0: return get_tensor_rank_single(x[0]) - return None elif isinstance(x, torch.Tensor): device = x.device - if device.type == 'cpu': - return None - else: + if device.type != 'cpu': return device.index return None in_rank = get_tensor_rank_single(in_feat) - if in_rank is not None: - return in_rank out_rank = get_tensor_rank_single(out_feat) - if out_rank is not None: - return out_rank - return None + tensor_rank = in_rank if in_rank else out_rank + return tensor_rank def _create_logger(level=logging.INFO): diff --git a/debug/accuracy_tools/msprobe/pytorch/function_factory.py b/debug/accuracy_tools/msprobe/pytorch/function_factory.py index 4e725de4f..6d840e561 100644 --- a/debug/accuracy_tools/msprobe/pytorch/function_factory.py +++ b/debug/accuracy_tools/msprobe/pytorch/function_factory.py @@ -47,4 +47,20 @@ class Register(dict): npu_custom_functions = Register() npu_custom_grad_functions = Register() -from msprobe.pytorch.bench_functions import * +from msprobe.pytorch.bench_functions.apply_adam_w import npu_apply_adam_w +from msprobe.pytorch.bench_functions.confusion_transpose import npu_confusion_transpose, \ + npu_confusion_transpose_backward +from msprobe.pytorch.bench_functions.fast_gelu import fast_gelu, npu_fast_gelu_backward +from msprobe.pytorch.bench_functions.layer_norm_eval import npu_layer_norm_eval +from msprobe.pytorch.bench_functions.linear import npu_linear, npu_linear_backward +from msprobe.pytorch.bench_functions.matmul_backward import matmul_backward +from msprobe.pytorch.bench_functions.npu_fusion_attention import softmax_forward, softmax_grad, broadcast_kv, \ + calculate_qk, fusion_attention_forward, fusion_attention_backward, parse_bsnd_args, convert_from_bnsd, \ + convert_to_bnsd, generate_atten_mask, generate_kv, rebuid_softmax_by_qkv, rebuild_softmax_by_max_sum, \ + npu_fusion_attention_forward_patch, npu_fusion_attention_backward_patch, npu_fusion_attention, \ + npu_fusion_attention_grad +from msprobe.pytorch.bench_functions.rms_norm import npu_rms_norm, npu_rms_norm_backward +from msprobe.pytorch.bench_functions.rotary_mul import npu_rotary_mul, npu_rotary_mul_backward +from msprobe.pytorch.bench_functions.scaled_mask_softmax import npu_scaled_masked_softmax, \ + npu_scaled_masked_softmax_backward +from msprobe.pytorch.bench_functions.swiglu import npu_swiglu, npu_swiglu_backward, swish_grad, swish diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py index 2c1805ab8..a02abbe5f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py @@ -66,14 +66,16 @@ class AtenOPTemplate(HOOKModule): @torch_device_guard def forward(self, *args, **kwargs): - if self.op in npu_custom_grad_functions: - return npu_custom_grad_functions[self.op](*args, **kwargs) - if self.op in WhiteAtenOps: - return eval(f"torch.ops.aten.{self.op}")(*args, **kwargs) - if self.op not in aten_func: - raise Exception(f"Skip op[{self.op}] accuracy check, because the op is not " - f"in dir(torch.ops.aten) and support yaml.") - return aten_func[self.op](*args, **kwargs) + if isinstance(self.op, str): + if self.op in npu_custom_grad_functions: + return npu_custom_grad_functions[self.op](*args, **kwargs) + if self.op in WhiteAtenOps: + return eval(f"torch.ops.aten.{self.op}")(*args, **kwargs) + if self.op not in aten_func: + raise Exception(f"Skip op[{self.op}] accuracy check, because the op is not " + f"in dir(torch.ops.aten) and support yaml.") + return aten_func[self.op](*args, **kwargs) + return self.op(*args, **kwargs) class AtenOPPacketTemplate(): diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index db9f99683..8a67ed942 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -17,11 +17,9 @@ import os import torch -import torch_npu import yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule -from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.common.utils import torch_device_guard, torch_without_guard_version from msprobe.core.common.const import Const from msprobe.core.common.file_check import FileOpen @@ -32,6 +30,13 @@ yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") with FileOpen(yaml_path, 'r') as f: WrapNpuOps = yaml.safe_load(f).get('torch_npu') +try: + import torch_npu +except ImportError: + is_gpu = True +else: + is_gpu = False + def get_npu_ops(): global WrapNpuOps @@ -39,10 +44,7 @@ def get_npu_ops(): _npu_ops = dir(torch.ops.npu) else: _npu_ops = dir(torch_npu._C._VariableFunctionsClass) - if msCheckerConfig.white_list: - return set(WrapNpuOps) & set(_npu_ops) & set(msCheckerConfig.white_list) - else: - return set(WrapNpuOps) & set(_npu_ops) + return set(WrapNpuOps) & set(_npu_ops) class HOOKNpuOP(object): -- Gitee From f62843acf20646831235171048043ece98d5a2a8 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 30 Jul 2024 14:53:19 +0800 Subject: [PATCH 055/791] =?UTF-8?q?wb.save=E7=BC=BA=E5=B0=91=E5=BC=82?= =?UTF-8?q?=E5=B8=B8=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index ea9323ae0..0bdf8a4c7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -645,8 +645,11 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): elif (i - 2) in highlight_dict['yellow_rows']: ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.YELLOW, end_color=CompareConst.YELLOW, fill_type="solid") - wb.save(file_path) - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + try: + wb.save(file_path) + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + except Exception: + logger.error('Save result file failed') def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, -- Gitee From 2dd4f7f1e81951632d7d33ecba24beb076a54ef0 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 30 Jul 2024 15:03:11 +0800 Subject: [PATCH 056/791] review fix --- debug/accuracy_tools/msprobe/core/common/utils.py | 2 +- .../msprobe/pytorch/compare/distributed_compare.py | 13 ++++++------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 32aba8d8a..56f4d1e61 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -148,7 +148,7 @@ def check_summary_only_valid(summary_only): return summary_only -def check_compare_param(input_parma, output_path, stack_mode=False, summary_compare=False, md5_compare=False): +def check_compare_param(input_parma, output_path, summary_compare=False, md5_compare=False): if not (isinstance(input_parma, dict) and isinstance(output_path, str)): logger.error("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 316ca2a1a..47a2864e5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -86,12 +86,11 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): 'or use compare() api and manually match the ranks.') raise CompareException(CompareException.INVALID_PATH_ERROR) for nr, br in zip(npu_ranks, bench_ranks): - n_dir = os.path.join(npu_dump_dir, nr) - b_dir = os.path.join(bench_dump_dir, br) - s_dir = n_dir - npu_json_path = extract_json(n_dir, stack_json=False) - bench_json_path = extract_json(b_dir, stack_json=False) - stack_json_path = extract_json(s_dir, stack_json=True) + npu_data_dir = os.path.join(npu_dump_dir, nr) + bench_data_dir = os.path.join(bench_dump_dir, br) + npu_json_path = extract_json(npu_data_dir, stack_json=False) + bench_json_path = extract_json(bench_data_dir, stack_json=False) + stack_json_path = extract_json(npu_data_dir, stack_json=True) dump_result_param = { 'npu_json_path': npu_json_path, @@ -103,7 +102,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): summary_compare, md5_compare = task_dumppath_get(dump_result_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(dump_result_param, output_path, stack_mode=stack_mode, summary_compare=summary_compare, md5_compare=md5_compare) + check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) except CompareException as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) -- Gitee From e2d715b292be818b0d3b61f98dc7cc6383f47b42 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Mon, 29 Jul 2024 19:24:00 +0800 Subject: [PATCH 057/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dapi=E4=BB=A5=E5=8F=8Amodule=E5=9C=A8=E4=B8=8A=E4=B8=80?= =?UTF-8?q?=E4=B8=AAstep=E7=BB=93=E6=9D=9F=E5=90=8E=E8=AE=A1=E6=95=B0?= =?UTF-8?q?=E6=9C=AA=E6=B8=85=E9=9B=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pytorch/hook_module/hook_module.py | 6 ++++ .../msprobe/pytorch/module_processer.py | 29 ++++++++++++------- .../accuracy_tools/msprobe/pytorch/service.py | 12 +++++--- 3 files changed, 33 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py index 6693a09d0..ff6427e51 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py @@ -17,9 +17,11 @@ import functools import threading + import torch import torch.nn as nn import torch.utils.hooks as full_hooks + from msprobe.core.common.const import Const @@ -61,6 +63,10 @@ class HOOKModule(nn.Module): HOOKModule.inner_stop_hook[self.current_thread] = False return result + @classmethod + def reset_module_stats(cls): + cls.module_count = {} + def _call_func(self, *input, **kwargs): full_backward_hooks, non_full_backward_hooks = [], [] if len(self._backward_hooks) > 0: diff --git a/debug/accuracy_tools/msprobe/pytorch/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/module_processer.py index 422d36d6a..f9368a087 100644 --- a/debug/accuracy_tools/msprobe/pytorch/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/module_processer.py @@ -1,15 +1,17 @@ from functools import wraps + import torch from torch.utils.hooks import BackwardHook + from msprobe.core.common.const import Const from msprobe.core.data_dump.scope import ModuleRangeScope class ModuleProcesser: + module_count = {} module_stack = [] api_parent_node = "" module_node = {} - current_module_name = "" def __init__(self, scope): if isinstance(scope, ModuleRangeScope): @@ -19,7 +21,6 @@ class ModuleProcesser: BackwardHook.setup_input_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_input_hook) BackwardHook.setup_output_hook = ModuleProcesser.clone_return_value(BackwardHook.setup_output_hook) BackwardHook.setup_output_hook = ModuleProcesser.filter_tensor_and_tuple(BackwardHook.setup_output_hook) - self.module_count = {} @staticmethod def filter_tensor_and_tuple(func): @@ -55,11 +56,26 @@ class ModuleProcesser: else: return result + @staticmethod + def module_count_func(module_name): + if module_name not in ModuleProcesser.module_count: + ModuleProcesser.module_count[module_name] = 0 + else: + ModuleProcesser.module_count[module_name] += 1 + return ModuleProcesser.module_count[module_name] + + @classmethod + def reset_module_stats(cls): + cls.module_count = {} + cls.module_stack = [] + cls.api_parent_node = "" + cls.module_node = {} + def node_hook(self, name_prefix, start_or_stop, **kwargs): def pre_hook(module, input, output=None): try: - index = self.module_count_func(name_prefix) + index = ModuleProcesser.module_count_func(name_prefix) except IndexError as e: index = None pass @@ -89,10 +105,3 @@ class ModuleProcesser: return pre_hook else: return end_hook - - def module_count_func(self, module_name): - if module_name not in self.module_count: - self.module_count[module_name] = 0 - else: - self.module_count[module_name] += 1 - return self.module_count[module_name] diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index daeda8898..7ff1ab657 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -2,17 +2,18 @@ import functools import os from pathlib import Path -from msprobe.pytorch.common.log import logger -from msprobe.core.common.file_check import FileChecker, check_path_before_create from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException +from msprobe.core.common.file_check import FileChecker, check_path_before_create from msprobe.core.data_dump.data_collector import build_data_collector -from msprobe.core.data_dump.scope import BaseScope from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs +from msprobe.core.data_dump.scope import BaseScope +from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.utils import get_rank_if_initialized -from msprobe.pytorch.module_processer import ModuleProcesser from msprobe.pytorch.hook_module import remove_dropout from msprobe.pytorch.hook_module.api_registry import api_register +from msprobe.pytorch.hook_module.hook_module import HOOKModule +from msprobe.pytorch.module_processer import ModuleProcesser class Service: @@ -82,6 +83,9 @@ class Service: self.current_iter += 1 self.data_collector.update_iter(self.current_iter) + ModuleProcesser.reset_module_stats() + HOOKModule.reset_module_stats() + def start(self, model, api_origin=False): self.model = model if self.config.step and self.current_iter > max(self.config.step): -- Gitee From 8401c942e6b64b6215d3ca4846927df775c41ea6 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 30 Jul 2024 16:13:13 +0800 Subject: [PATCH 058/791] resolve class registry loop import issue --- .../pytorch/bench_functions/apply_adam_w.py | 3 -- .../bench_functions/confusion_transpose.py | 6 --- .../pytorch/bench_functions/fast_gelu.py | 3 -- .../bench_functions/layer_norm_eval.py | 2 - .../msprobe/pytorch/bench_functions/linear.py | 3 -- .../bench_functions/matmul_backward.py | 2 - .../bench_functions/npu_fusion_attention.py | 3 -- .../pytorch/bench_functions/rms_norm.py | 3 -- .../pytorch/bench_functions/rotary_mul.py | 3 -- .../bench_functions/scaled_mask_softmax.py | 3 -- .../msprobe/pytorch/bench_functions/swiglu.py | 3 -- .../msprobe/pytorch/function_factory.py | 49 +++++++++++-------- 12 files changed, 29 insertions(+), 54 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py index dc0954911..caf21a604 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py @@ -1,9 +1,6 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_functions - -@npu_custom_functions def npu_apply_adam_w(beta1_power, beta2_power, lr, weight_decay, beta1, beta2, eps, grad, max_grad_norm, amsgrad, maximize, out): var, m, v = out diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py index dd30bb18a..627bf11b6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py @@ -1,8 +1,3 @@ -import torch -from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions - - -@npu_custom_functions def npu_confusion_transpose(data, perm, shape, transpose_first): if transpose_first: output = data.permute(*perm).contiguous().view(shape) @@ -11,7 +6,6 @@ def npu_confusion_transpose(data, perm, shape, transpose_first): return output.cpu() -@npu_custom_grad_functions def npu_confusion_transpose_backward(grad, perm, shape, transpose_first): shape_cal = shape if transpose_first else [shape[perm_dim] for perm_dim in perm] perm_cal = [0] * len(perm) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py index 5442eff73..a1a9ca080 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py @@ -1,8 +1,6 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions -@npu_custom_functions def fast_gelu(input0): attr = 1.702 const_0 = 0 - attr @@ -24,7 +22,6 @@ def fast_gelu(input0): return result.cpu() -@npu_custom_grad_functions def npu_fast_gelu_backward(grad, input_x): const_2 = 1.702 const_3 = 1.0 diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py index 885b5c460..f6949c079 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py @@ -1,8 +1,6 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_functions -@npu_custom_functions def npu_layer_norm_eval(data, normalized_shape): result = torch.nn.functional.layer_norm(data, normalized_shape) return result.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py index 33b18d759..95db875ed 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py @@ -1,14 +1,11 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions -@npu_custom_functions def npu_linear(x, weight, bias): output = torch.nn.functional.linear(x, weight, bias) return output.cpu() -@npu_custom_grad_functions def npu_linear_backward(grad, input_data, weight): input_grad = torch.matmul(grad, weight) weight_grad = torch.matmul(grad.t(), input_data) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py index 3c4f7dc04..ed1c746ec 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/matmul_backward.py @@ -1,8 +1,6 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_grad_functions -@npu_custom_grad_functions def matmul_backward(grad, self, other, mask): grad_self, grad_other = None, None dim_self = self.dim() diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index 6a49ce740..f4b639e2f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -2,7 +2,6 @@ import torch import numpy as np from einops import rearrange -from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions from api_accuracy_checker.common.utils import logger gtype = torch.float64 # arm host必须选择float64，x86环境选择float32即可，64也行。arm计算很慢，s=8k的场景建议使用x86 @@ -331,7 +330,6 @@ def npu_fusion_attention_backward_patch(*args, **kwargs): return args, dims_kwargs, new_kwargs -@npu_custom_functions def npu_fusion_attention(*args, **kwargs): new_args, dims_kwargs, new_kwargs = npu_fusion_attention_forward_patch(*args, **kwargs) query, key, value, input_layout = new_args[0], new_args[1], new_args[2], new_args[4] @@ -366,7 +364,6 @@ def npu_fusion_attention(*args, **kwargs): return out_golden.cpu(), softmax_max.repeat(1, 1, 1, 8).cpu(), softmax_sum.repeat(1, 1, 1, 8).cpu() -@npu_custom_grad_functions def npu_fusion_attention_grad(*args, **kwargs): # dx, q, k, v, softmax_res, drop_mask, pse, scale, keep_prob new_args, dims_kwargs, new_kwargs = npu_fusion_attention_backward_patch(*args, **kwargs) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py index 0fe6c834a..e647312fd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py @@ -1,15 +1,12 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions -@npu_custom_functions def npu_rms_norm(x, gamma, epsilon=1e-5): rstd = torch.rsqrt(torch.mean(torch.pow(x, 2), axis=-1, keepdim=True) + epsilon) res = x * rstd * gamma return res.cpu(), rstd.float().cpu() -@npu_custom_grad_functions def npu_rms_norm_backward(grad, x, gamma, rstd): mean_gy = (grad * x * gamma * rstd).mean(dim=-1, keepdim=True) grad_x = (grad * gamma - x * rstd * mean_gy) * rstd diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py index 76b3828da..0e0fda5f7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py @@ -1,8 +1,6 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions -@npu_custom_functions def npu_rotary_mul(x, r1, r2): x1, x2 = torch.chunk(x, 2, -1) x_new = torch.cat((-x2, x1), dim=-1) @@ -10,7 +8,6 @@ def npu_rotary_mul(x, r1, r2): return output.cpu() -@npu_custom_grad_functions def npu_rotary_mul_backward(dy_tensor, x, r1, r2): x.requires_grad = True r1.requires_grad = True diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py index bcc523ee4..8717aebaf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py @@ -1,8 +1,6 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions -@npu_custom_functions def npu_scaled_masked_softmax(x, mask, scale, fixed_triu_mask): if fixed_triu_mask: mask = (torch.triu(torch.ones(mask.shape), k=1)).bool().to(mask.device) @@ -14,7 +12,6 @@ def npu_scaled_masked_softmax(x, mask, scale, fixed_triu_mask): return y.to(dtype).cpu() -@npu_custom_grad_functions def npu_scaled_masked_softmax_backward(y_grad, y, mask, scale, fixed_triu_mask): if fixed_triu_mask: mask = (torch.triu(torch.ones(mask.shape), k=1)).bool().to(mask.device) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py index 973be454d..e03c975a5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py @@ -1,8 +1,6 @@ import torch -from msprobe.pytorch.function_factory import npu_custom_functions, npu_custom_grad_functions -@npu_custom_functions def npu_swiglu(x, dim=-1): tensor_dtype = x.dtype @@ -19,7 +17,6 @@ def npu_swiglu(x, dim=-1): return output_data.cpu() -@npu_custom_grad_functions def npu_swiglu_backward(grad, x, dim=-1): tensor_dtype = grad.dtype in_tensors = torch.chunk(x, 2, dim=dim) diff --git a/debug/accuracy_tools/msprobe/pytorch/function_factory.py b/debug/accuracy_tools/msprobe/pytorch/function_factory.py index 6d840e561..c2fd8bfd0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/function_factory.py +++ b/debug/accuracy_tools/msprobe/pytorch/function_factory.py @@ -1,4 +1,17 @@ from msprobe.pytorch.common.utils import logger +from msprobe.pytorch.bench_functions.apply_adam_w import npu_apply_adam_w +from msprobe.pytorch.bench_functions.confusion_transpose import npu_confusion_transpose, \ + npu_confusion_transpose_backward +from msprobe.pytorch.bench_functions.fast_gelu import fast_gelu, npu_fast_gelu_backward +from msprobe.pytorch.bench_functions.layer_norm_eval import npu_layer_norm_eval +from msprobe.pytorch.bench_functions.linear import npu_linear, npu_linear_backward +from msprobe.pytorch.bench_functions.matmul_backward import matmul_backward +from msprobe.pytorch.bench_functions.npu_fusion_attention import npu_fusion_attention, npu_fusion_attention_grad +from msprobe.pytorch.bench_functions.rms_norm import npu_rms_norm, npu_rms_norm_backward +from msprobe.pytorch.bench_functions.rotary_mul import npu_rotary_mul, npu_rotary_mul_backward +from msprobe.pytorch.bench_functions.scaled_mask_softmax import npu_scaled_masked_softmax, \ + npu_scaled_masked_softmax_backward +from msprobe.pytorch.bench_functions.swiglu import npu_swiglu, npu_swiglu_backward, swish_grad, swish class Register(dict): @@ -6,8 +19,10 @@ class Register(dict): super(Register, self).__init__(*args, **kwargs) self._dict = {} - def __call__(self, target): - return self.register(target) + def __call__(self, target_func_list): + for target in target_func_list: + self.register(target) + return def __setitem__(self, key, value): self._dict[key] = value @@ -44,23 +59,17 @@ class Register(dict): raise Exception(f"The func {target} is not callable.") +# register for npu custom bench functions npu_custom_functions = Register() -npu_custom_grad_functions = Register() +npu_custom_functions([ + npu_apply_adam_w, npu_confusion_transpose, fast_gelu, npu_layer_norm_eval, npu_linear, npu_fusion_attention, + npu_rms_norm, npu_rotary_mul, npu_scaled_masked_softmax, npu_swiglu +]) -from msprobe.pytorch.bench_functions.apply_adam_w import npu_apply_adam_w -from msprobe.pytorch.bench_functions.confusion_transpose import npu_confusion_transpose, \ - npu_confusion_transpose_backward -from msprobe.pytorch.bench_functions.fast_gelu import fast_gelu, npu_fast_gelu_backward -from msprobe.pytorch.bench_functions.layer_norm_eval import npu_layer_norm_eval -from msprobe.pytorch.bench_functions.linear import npu_linear, npu_linear_backward -from msprobe.pytorch.bench_functions.matmul_backward import matmul_backward -from msprobe.pytorch.bench_functions.npu_fusion_attention import softmax_forward, softmax_grad, broadcast_kv, \ - calculate_qk, fusion_attention_forward, fusion_attention_backward, parse_bsnd_args, convert_from_bnsd, \ - convert_to_bnsd, generate_atten_mask, generate_kv, rebuid_softmax_by_qkv, rebuild_softmax_by_max_sum, \ - npu_fusion_attention_forward_patch, npu_fusion_attention_backward_patch, npu_fusion_attention, \ - npu_fusion_attention_grad -from msprobe.pytorch.bench_functions.rms_norm import npu_rms_norm, npu_rms_norm_backward -from msprobe.pytorch.bench_functions.rotary_mul import npu_rotary_mul, npu_rotary_mul_backward -from msprobe.pytorch.bench_functions.scaled_mask_softmax import npu_scaled_masked_softmax, \ - npu_scaled_masked_softmax_backward -from msprobe.pytorch.bench_functions.swiglu import npu_swiglu, npu_swiglu_backward, swish_grad, swish +# register for npu custom backward bench functions +npu_custom_grad_functions = Register() +npu_custom_grad_functions([ + npu_confusion_transpose_backward, npu_fast_gelu_backward, npu_linear_backward, matmul_backward, + npu_fusion_attention_grad, npu_rms_norm_backward, npu_rotary_mul_backward, npu_scaled_masked_softmax_backward, + npu_swiglu_backward +]) -- Gitee From 23894c0f0e5ee6b149e4a06a7c6a65ba1da49a52 Mon Sep 17 00:00:00 2001 From: wuyulong11 <2284273586@qq.com> Date: Tue, 30 Jul 2024 17:26:39 +0800 Subject: [PATCH 059/791] =?UTF-8?q?=E3=80=90=E4=BF=AE=E6=94=B9=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E3=80=91=E3=80=90tbplugin=E3=80=91=E5=88=A0=E9=99=A4l?= =?UTF-8?q?ibkineto=E7=AD=89=E6=9C=AA=E4=BD=BF=E7=94=A8=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=20=E3=80=90=E4=BF=AE=E6=94=B9=E4=BA=BA=E3=80=91=20wuyulong=203?= =?UTF-8?q?0031080?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../.github/workflows/libkineto_ci.yml | 56 -- .../workflows/tb_plugin_build_pip_package.yml | 19 - .../.github/workflows/tb_plugin_ci.yml | 57 -- plugins/tensorboard-plugins/.gitignore | 3 - plugins/tensorboard-plugins/.gitmodules | 6 - .../tensorboard-plugins/CODE_OF_CONDUCT.md | 77 -- plugins/tensorboard-plugins/CONTRIBUTING.md | 34 - plugins/tensorboard-plugins/LICENSE | 33 - plugins/tensorboard-plugins/README.md | 38 - .../libkineto/CMakeLists.txt | 198 ----- .../tensorboard-plugins/libkineto/README.md | 65 -- .../libkineto/include/AbstractConfig.h | 113 --- .../include/ActivityProfilerInterface.h | 91 -- .../include/ActivityTraceInterface.h | 21 - .../libkineto/include/ActivityType.h | 34 - .../libkineto/include/ClientInterface.h | 16 - .../libkineto/include/Config.h | 433 --------- .../libkineto/include/GenericTraceActivity.h | 125 --- .../libkineto/include/IActivityProfiler.h | 104 --- .../libkineto/include/ILoggerObserver.h | 50 -- .../libkineto/include/ITraceActivity.h | 53 -- .../libkineto/include/ThreadUtil.h | 22 - .../libkineto/include/TraceSpan.h | 36 - .../libkineto/include/libkineto.h | 138 --- .../libkineto/include/time_since_epoch.h | 16 - .../libkineto/libkineto_defs.bzl | 77 -- .../sample_programs/kineto_playground.cpp | 38 - .../sample_programs/kineto_playground.cu | 60 -- .../sample_programs/kineto_playground.cuh | 18 - .../libkineto/src/AbstractConfig.cpp | 188 ---- .../libkineto/src/ActivityBuffers.h | 29 - .../libkineto/src/ActivityLoggerFactory.h | 60 -- .../src/ActivityProfilerController.cpp | 246 ----- .../src/ActivityProfilerController.h | 84 -- .../libkineto/src/ActivityProfilerProxy.cpp | 119 --- .../libkineto/src/ActivityProfilerProxy.h | 73 -- .../libkineto/src/ActivityTrace.h | 45 - .../libkineto/src/ActivityType.cpp | 58 -- .../libkineto/src/Config.cpp | 473 ---------- .../libkineto/src/ConfigLoader.cpp | 300 ------- .../libkineto/src/ConfigLoader.h | 147 --- .../libkineto/src/CudaDeviceProperties.cpp | 130 --- .../libkineto/src/CudaDeviceProperties.h | 31 - .../libkineto/src/CuptiActivity.h | 114 --- .../libkineto/src/CuptiActivity.tpp | 111 --- .../libkineto/src/CuptiActivityApi.cpp | 343 ------- .../libkineto/src/CuptiActivityApi.h | 100 --- .../libkineto/src/CuptiActivityBuffer.h | 51 -- .../libkineto/src/CuptiActivityPlatform.cpp | 31 - .../libkineto/src/CuptiActivityPlatform.h | 12 - .../libkineto/src/CuptiActivityProfiler.cpp | 841 ------------------ .../libkineto/src/CuptiActivityProfiler.h | 364 -------- .../libkineto/src/CuptiCallbackApi.cpp | 260 ------ .../libkineto/src/CuptiCallbackApi.h | 130 --- .../libkineto/src/CuptiCallbackApiMock.h | 32 - .../libkineto/src/CuptiEventApi.cpp | 112 --- .../libkineto/src/CuptiEventApi.h | 49 - .../libkineto/src/CuptiMetricApi.cpp | 107 --- .../libkineto/src/CuptiMetricApi.h | 38 - .../libkineto/src/CuptiNvPerfMetric.cpp | 504 ----------- .../libkineto/src/CuptiNvPerfMetric.h | 71 -- .../libkineto/src/CuptiRangeProfilerApi.cpp | 751 ---------------- .../libkineto/src/CuptiRangeProfilerApi.h | 220 ----- .../src/CuptiRangeProfilerConfig.cpp | 68 -- .../libkineto/src/CuptiRangeProfilerConfig.h | 86 -- .../libkineto/src/DaemonConfigLoader.h | 27 - .../libkineto/src/Demangle.cpp | 49 - .../libkineto/src/Demangle.h | 12 - .../libkineto/src/EventProfiler.cpp | 635 ------------- .../libkineto/src/EventProfiler.h | 341 ------- .../libkineto/src/EventProfilerController.cpp | 423 --------- .../libkineto/src/EventProfilerController.h | 63 -- .../libkineto/src/GenericTraceActivity.cpp | 10 - .../libkineto/src/ILoggerObserver.cpp | 54 -- .../libkineto/src/Logger.cpp | 136 --- .../libkineto/src/Logger.h | 244 ----- .../libkineto/src/LoggerCollector.h | 70 -- .../libkineto/src/RoctracerActivityApi.cpp | 569 ------------ .../libkineto/src/RoctracerActivityApi.h | 171 ---- .../libkineto/src/RoctracerActivityBuffer.h | 30 - .../libkineto/src/SampleListener.h | 146 --- .../libkineto/src/ScopeExit.h | 29 - .../libkineto/src/ThreadUtil.cpp | 203 ----- .../libkineto/src/WeakSymbols.cpp | 12 - .../libkineto/src/cupti_call.h | 33 - .../libkineto/src/cupti_strings.cpp | 502 ----------- .../libkineto/src/cupti_strings.h | 14 - .../libkineto/src/init.cpp | 139 --- .../libkineto/src/libkineto_api.cpp | 41 - .../libkineto/src/output_base.h | 104 --- .../libkineto/src/output_csv.cpp | 88 -- .../libkineto/src/output_csv.h | 39 - .../libkineto/src/output_json.cpp | 583 ------------ .../libkineto/src/output_json.h | 91 -- .../libkineto/src/output_membuf.h | 130 --- .../libkineto/test/CMakeLists.txt | 3 - .../libkineto/test/ConfigTest.cpp | 315 ------- .../test/CuptiActivityProfilerTest.cpp | 629 ------------- .../libkineto/test/CuptiCallbackApiTest.cpp | 239 ----- .../libkineto/test/CuptiProfilerApiTest.cu | 353 -------- .../test/CuptiRangeProfilerApiTest.cpp | 113 --- .../test/CuptiRangeProfilerConfigTest.cpp | 67 -- .../test/CuptiRangeProfilerTestUtil.h | 96 -- .../libkineto/test/CuptiStringsTest.cpp | 29 - .../libkineto/test/EventProfilerTest.cpp | 578 ------------ .../libkineto/test/LoggerObserverTest.cpp | 96 -- .../test/MockActivitySubProfiler.cpp | 49 - .../libkineto/test/MockActivitySubProfiler.h | 72 -- .../libkineto/test/PidInfoTest.cpp | 27 - 109 files changed, 16063 deletions(-) delete mode 100644 plugins/tensorboard-plugins/.github/workflows/libkineto_ci.yml delete mode 100644 plugins/tensorboard-plugins/.github/workflows/tb_plugin_build_pip_package.yml delete mode 100644 plugins/tensorboard-plugins/.github/workflows/tb_plugin_ci.yml delete mode 100644 plugins/tensorboard-plugins/.gitignore delete mode 100644 plugins/tensorboard-plugins/.gitmodules delete mode 100644 plugins/tensorboard-plugins/CODE_OF_CONDUCT.md delete mode 100644 plugins/tensorboard-plugins/CONTRIBUTING.md delete mode 100644 plugins/tensorboard-plugins/LICENSE delete mode 100644 plugins/tensorboard-plugins/README.md delete mode 100644 plugins/tensorboard-plugins/libkineto/CMakeLists.txt delete mode 100644 plugins/tensorboard-plugins/libkineto/README.md delete mode 100644 plugins/tensorboard-plugins/libkineto/include/AbstractConfig.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/ActivityProfilerInterface.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/ActivityTraceInterface.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/ActivityType.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/ClientInterface.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/Config.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/GenericTraceActivity.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/IActivityProfiler.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/ILoggerObserver.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/ITraceActivity.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/ThreadUtil.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/TraceSpan.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/libkineto.h delete mode 100644 plugins/tensorboard-plugins/libkineto/include/time_since_epoch.h delete mode 100644 plugins/tensorboard-plugins/libkineto/libkineto_defs.bzl delete mode 100644 plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cu delete mode 100644 plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cuh delete mode 100644 plugins/tensorboard-plugins/libkineto/src/AbstractConfig.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ActivityBuffers.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ActivityLoggerFactory.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ActivityProfilerController.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ActivityProfilerController.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ActivityProfilerProxy.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ActivityProfilerProxy.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ActivityTrace.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ActivityType.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/Config.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ConfigLoader.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ConfigLoader.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CudaDeviceProperties.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CudaDeviceProperties.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivity.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivity.tpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivityApi.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivityApi.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivityBuffer.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivityPlatform.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivityPlatform.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivityProfiler.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiActivityProfiler.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApi.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApi.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApiMock.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiEventApi.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiEventApi.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiMetricApi.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiMetricApi.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiNvPerfMetric.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiNvPerfMetric.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerApi.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerApi.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerConfig.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerConfig.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/DaemonConfigLoader.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/Demangle.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/Demangle.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/EventProfiler.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/EventProfiler.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/EventProfilerController.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/EventProfilerController.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/GenericTraceActivity.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ILoggerObserver.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/Logger.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/Logger.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/LoggerCollector.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/RoctracerActivityApi.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/RoctracerActivityApi.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/RoctracerActivityBuffer.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/SampleListener.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ScopeExit.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/ThreadUtil.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/WeakSymbols.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/cupti_call.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/cupti_strings.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/cupti_strings.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/init.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/libkineto_api.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/output_base.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/output_csv.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/output_csv.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/output_json.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/src/output_json.h delete mode 100644 plugins/tensorboard-plugins/libkineto/src/output_membuf.h delete mode 100644 plugins/tensorboard-plugins/libkineto/test/CMakeLists.txt delete mode 100644 plugins/tensorboard-plugins/libkineto/test/ConfigTest.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/CuptiActivityProfilerTest.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/CuptiCallbackApiTest.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/CuptiProfilerApiTest.cu delete mode 100644 plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerApiTest.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerConfigTest.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerTestUtil.h delete mode 100644 plugins/tensorboard-plugins/libkineto/test/CuptiStringsTest.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/EventProfilerTest.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/LoggerObserverTest.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/MockActivitySubProfiler.cpp delete mode 100644 plugins/tensorboard-plugins/libkineto/test/MockActivitySubProfiler.h delete mode 100644 plugins/tensorboard-plugins/libkineto/test/PidInfoTest.cpp diff --git a/plugins/tensorboard-plugins/.github/workflows/libkineto_ci.yml b/plugins/tensorboard-plugins/.github/workflows/libkineto_ci.yml deleted file mode 100644 index 3133d6400..000000000 --- a/plugins/tensorboard-plugins/.github/workflows/libkineto_ci.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: LIBKINETOCI - -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - build: - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-latest] - - steps: - - uses: actions/checkout@v2 - - name: Checkout submodules - shell: bash - run: | - auth_header="$(git config --local --get http.https://github.com/.extraheader)" - git submodule sync --recursive - git -c "http.extraheader=$auth_header" -c protocol.version=2 submodule update --init --force --recursive --depth=1 - - - name: Get env vars - run: | - echo GITHUB_WORKFLOW = $GITHUB_WORKFLOW - echo HOME = $HOME - echo GITHUB_ACTION = $GITHUB_ACTION - echo GITHUB_ACTIONS = $GITHUB_ACTIONS - echo GITHUB_REPOSITORY = $GITHUB_REPOSITORY - echo GITHUB_EVENT_NAME = $GITHUB_EVENT_NAME - echo GITHUB_EVENT_PATH = $GITHUB_EVENT_PATH - echo GITHUB_WORKSPACE = $GITHUB_WORKSPACE - echo GITHUB_SHA = $GITHUB_SHA - echo GITHUB_REF = $GITHUB_REF - c++ --verbose - - # TODO: Figure out how to install cupti headers T84637671 - - name: Build static lib - run: | - set -e - mkdir build_static - cd build_static - cmake -DKINETO_LIBRARY_TYPE=static ../libkineto/ - make -j - - - name: Build shared lib - run: | - set -e - mkdir build_shared - cd build_shared - cmake -DKINETO_LIBRARY_TYPE=shared ../libkineto/ - make -j diff --git a/plugins/tensorboard-plugins/.github/workflows/tb_plugin_build_pip_package.yml b/plugins/tensorboard-plugins/.github/workflows/tb_plugin_build_pip_package.yml deleted file mode 100644 index 9bdafcc44..000000000 --- a/plugins/tensorboard-plugins/.github/workflows/tb_plugin_build_pip_package.yml +++ /dev/null @@ -1,19 +0,0 @@ -name: Build torch-tb-profiler Pip Package - -on: - # TODO: Add an on_release trigger to build on tags - workflow_dispatch: - -jobs: - build-package: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: build pip package - run: | - set -e - cd tb_plugin - python setup.py sdist bdist_wheel - cd dist/ - pip install *.whl - python -c "import torch_tb_profiler;print(torch_tb_profiler.__version__)" diff --git a/plugins/tensorboard-plugins/.github/workflows/tb_plugin_ci.yml b/plugins/tensorboard-plugins/.github/workflows/tb_plugin_ci.yml deleted file mode 100644 index 1b59a7bf9..000000000 --- a/plugins/tensorboard-plugins/.github/workflows/tb_plugin_ci.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: TB_Plugin_CI - -on: - push: - branches: - - main - - release/** - - plugin/** - - pull_request: - branches: - - main - - release/** - - plugin/** - -jobs: - generate-matrix: - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - id: set-matrix - run: | - echo $GITHUB_BASE_REF - if [ $GITHUB_BASE_REF == "plugin/vnext" ] - then - echo "::set-output name=matrix::{\"python-version\":[3.7, 3.8, 3.9], \"cuda-version\":[\"cpu\"], \"pytorch-version\":[\"nightly\"]}" - else - echo "::set-output name=matrix::{\"python-version\":[3.7, 3.8, 3.9], \"cuda-version\":[\"cpu\"], \"pytorch-version\":[\"nightly\", \"1.11rc\", \"stable\"]}" - fi - - build: - needs: generate-matrix - runs-on: ubuntu-latest - strategy: - matrix: ${{fromJSON(needs.generate-matrix.outputs.matrix)}} - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - architecture: 'x64' - - name: Test - env: - CUDA_VERSION: ${{ matrix.cuda-version }} - PYTORCH_VERSION: ${{ matrix.pytorch-version }} - TORCH_PROFILER_LOG_LEVEL: DEBUG - GRPC_VERBOSITY: DEBUG - GRPC_ENABLE_FORK_SUPPORT: 'False' - run: | - set -e - cd tb_plugin - sh ./ci_scripts/install_env.sh - pip install .[gs] - cd test - pytest diff --git a/plugins/tensorboard-plugins/.gitignore b/plugins/tensorboard-plugins/.gitignore deleted file mode 100644 index ce186381c..000000000 --- a/plugins/tensorboard-plugins/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -# ignore common items -.idea -.vscode diff --git a/plugins/tensorboard-plugins/.gitmodules b/plugins/tensorboard-plugins/.gitmodules deleted file mode 100644 index 4660ee8bc..000000000 --- a/plugins/tensorboard-plugins/.gitmodules +++ /dev/null @@ -1,6 +0,0 @@ -[submodule "libkineto/third_party/googletest"] - path = libkineto/third_party/googletest - url = https://github.com/google/googletest.git -[submodule "libkineto/third_party/fmt"] - path = libkineto/third_party/fmt - url = https://github.com/fmtlib/fmt.git diff --git a/plugins/tensorboard-plugins/CODE_OF_CONDUCT.md b/plugins/tensorboard-plugins/CODE_OF_CONDUCT.md deleted file mode 100644 index a0cbeaab7..000000000 --- a/plugins/tensorboard-plugins/CODE_OF_CONDUCT.md +++ /dev/null @@ -1,77 +0,0 @@ -# Code of Conduct - -## Our Pledge - -In the interest of fostering an open and welcoming environment, we as -contributors and maintainers pledge to make participation in our project and -our community a harassment-free experience for everyone, regardless of age, body -size, disability, ethnicity, sex characteristics, gender identity and expression, -level of experience, education, socio-economic status, nationality, personal -appearance, race, religion, or sexual identity and orientation. - -## Our Standards - -Examples of behavior that contributes to creating a positive environment -include: - -* Using welcoming and inclusive language -* Being respectful of differing viewpoints and experiences -* Gracefully accepting constructive criticism -* Focusing on what is best for the community -* Showing empathy towards other community members - -Examples of unacceptable behavior by participants include: - -* The use of sexualized language or imagery and unwelcome sexual attention or - advances -* Trolling, insulting/derogatory comments, and personal or political attacks -* Public or private harassment -* Publishing others' private information, such as a physical or electronic - address, without explicit permission -* Other conduct which could reasonably be considered inappropriate in a - professional setting - -## Our Responsibilities - -Project maintainers are responsible for clarifying the standards of acceptable -behavior and are expected to take appropriate and fair corrective action in -response to any instances of unacceptable behavior. - -Project maintainers have the right and responsibility to remove, edit, or -reject comments, commits, code, wiki edits, issues, and other contributions -that are not aligned to this Code of Conduct, or to ban temporarily or -permanently any contributor for other behaviors that they deem inappropriate, -threatening, offensive, or harmful. - -## Scope - -This Code of Conduct applies within all project spaces, and it also applies when -an individual is representing the project or its community in public spaces. -Examples of representing a project or community include using an official -project e-mail address, posting via an official social media account, or acting -as an appointed representative at an online or offline event. Representation of -a project may be further defined and clarified by project maintainers. - -## Enforcement - -Instances of abusive, harassing, or otherwise unacceptable behavior may be -reported by contacting the project team at . All -complaints will be reviewed and investigated and will result in a response that -is deemed necessary and appropriate to the circumstances. The project team is -obligated to maintain confidentiality with regard to the reporter of an incident. -Further details of specific enforcement policies may be posted separately. - -Project maintainers who do not follow or enforce the Code of Conduct in good -faith may face temporary or permanent repercussions as determined by other -members of the project's leadership. - -## Attribution - -This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, -available at https://www.contributor-covenant.org/version/1/4/code-of-conduct.html - -[homepage]: https://www.contributor-covenant.org - -For answers to common questions about this code of conduct, see -https://www.contributor-covenant.org/faq - diff --git a/plugins/tensorboard-plugins/CONTRIBUTING.md b/plugins/tensorboard-plugins/CONTRIBUTING.md deleted file mode 100644 index a2e931bb6..000000000 --- a/plugins/tensorboard-plugins/CONTRIBUTING.md +++ /dev/null @@ -1,34 +0,0 @@ -# Contributing to Kineto -We want to make contributing to this project as easy and transparent as -possible. - -## Code of Conduct -The code of conduct is described in [`CODE_OF_CONDUCT.md`](CODE_OF_CONDUCT.md). - -## Pull Requests -We actively welcome your pull requests. - -1. Fork the repo and create your branch from `main`. -2. If you've added code that should be tested, add tests. -3. If you've changed APIs, update the documentation. -4. Ensure the test suite passes. -5. Make sure your code lints. -6. If you haven't already, complete the Contributor License Agreement ("CLA"). - -## Contributor License Agreement ("CLA") -In order to accept your pull request, we need you to submit a CLA. You only need -to do this once to work on any of Facebook's open source projects. - -Complete your CLA here: - -## Issues -We use GitHub issues to track public bugs. Please ensure your description is -clear and has sufficient instructions to be able to reproduce the issue. - -Facebook has a [bounty program](https://www.facebook.com/whitehat/) for the safe -disclosure of security bugs. In those cases, please go through the process -outlined on that page and do not file a public issue. - -## License -By contributing to Kineto, you agree that your contributions will be licensed -under the LICENSE file in the root directory of this source tree. diff --git a/plugins/tensorboard-plugins/LICENSE b/plugins/tensorboard-plugins/LICENSE deleted file mode 100644 index edb179715..000000000 --- a/plugins/tensorboard-plugins/LICENSE +++ /dev/null @@ -1,33 +0,0 @@ -BSD License - -For Kineto software - -Copyright (c) Facebook, Inc. and its affiliates. All rights reserved. - -All contributions by Microsoft: -Copyright (c) Microsoft Corporation. (The Azure AI Platform team) - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/plugins/tensorboard-plugins/README.md b/plugins/tensorboard-plugins/README.md deleted file mode 100644 index 3a18f4c62..000000000 --- a/plugins/tensorboard-plugins/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# Kineto - -Kineto is part of the PyTorch Profiler. - -The Kineto project was started to help enable -- **performance observability and diagnostics** across common ML bottleneck components -- **actionable recommendations** for common issues -- integration of external system-level profiling tools -- integration with popular visualization platforms and analysis pipelines - -A central component is libkineto, a profiling library with special focus on low-overhead GPU timeline tracing. - -The PyTorch Profiler TensorBoard plugin provides powerful and intuitive visualizations of profiling results, as well as actionable recommendations, and is the best way to experience the new PyTorch Profiler. - -## Libkineto -Libkineto is an in-process profiling library integrated with the PyTorch Profiler. Please refer to the [README](libkineto/README.md) file in the `libkineto` folder as well as documentation on the [new PyTorch Profiler API](https://pytorch.org/docs/master/profiler.html). - -## PyTorch TensorBoard Profiler NPU Plugin -The goal of the PyTorch TensorBoard Profiler is to provide a seamless and intuitive end-to-end profiling experience, including straightforward collection from PyTorch and insightful visualizations and recommendations in the TensorBoard UI. -Please refer to the [README](tb_plugin/README.md) file in the `tb_plugin` folder. - -## Future Development Direction: -Some areas we're currently working on: -- Support for tracing distributed workloads -- Trace processing, analysis and recommendation engine -- System-level activities, multiple tracing sources -- Profiling and monitoring daemon for larger scale deployments - -## Releases and Contributing -We will follow the PyTorch release schedule which roughly happens on a 3 month basis. - -We appreciate all contributions. If you are planning to contribute back bug-fixes, please do so without any further discussion. - -If you plan to contribute new features, please first open an issue and discuss the feature with us. Sending a PR without discussion might end up resulting in a rejected PR because we might be taking the infrastructure in a different direction than you might be aware of. We expect the architecture to keep evolving. - -## License -Kineto has a BSD-style license, as found in the [LICENSE](LICENSE) file. - diff --git a/plugins/tensorboard-plugins/libkineto/CMakeLists.txt b/plugins/tensorboard-plugins/libkineto/CMakeLists.txt deleted file mode 100644 index 63966de80..000000000 --- a/plugins/tensorboard-plugins/libkineto/CMakeLists.txt +++ /dev/null @@ -1,198 +0,0 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - -list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake/modules") - -#install libraries into correct locations on all platforms -include(GNUInstallDirs) - -# function to extract filelists from libkineto_defs.bzl file -find_package(PythonInterp) -function(get_filelist name outputvar) - execute_process( - COMMAND "${PYTHON_EXECUTABLE}" -c - "exec(open('libkineto_defs.bzl').read());print(';'.join(${name}))" - WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}" - OUTPUT_VARIABLE _tempvar) - string(REPLACE "\n" "" _tempvar "${_tempvar}") - set(${outputvar} ${_tempvar} PARENT_SCOPE) -endfunction() - -project(kineto VERSION 0.1 LANGUAGES CXX C) - -set(KINETO_LIBRARY_TYPE "default" CACHE STRING - "Type of library (default, static or shared) to build") -set_property(CACHE KINETO_LIBRARY_TYPE PROPERTY STRINGS default shared) -option(KINETO_BUILD_TESTS "Build kineto unit tests" ON) - -set(LIBKINETO_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/src") -set(LIBKINETO_INCLUDE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/include") -set(LIBKINETO_BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}) -set(LIBKINETO_THIRDPARTY_DIR "${CMAKE_CURRENT_SOURCE_DIR}/third_party") -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -#We should default to a Release build -if (NOT CMAKE_BUILD_TYPE OR CMAKE_BUILD_TYPE STREQUAL "") - set(CMAKE_BUILD_TYPE "Release" CACHE STRING "" FORCE) -endif() - -if (NOT CUDA_SOURCE_DIR) - set(CUDA_SOURCE_DIR "$ENV{CUDA_SOURCE_DIR}") - message(INFO " CUDA_SOURCE_DIR = ${CUDA_SOURCE_DIR}") -endif() - -if (NOT ROCM_SOURCE_DIR) - set(ROCM_SOURCE_DIR "$ENV{ROCM_SOURCE_DIR}") - message(INFO " ROCM_SOURCE_DIR = ${ROCM_SOURCE_DIR}") -endif() - -# Set LIBKINETO_NOCUPTI to explicitly disable CUPTI -# Otherwise, CUPTI is disabled if not found -IF (NOT CUDA_SOURCE_DIR OR NOT CUPTI_INCLUDE_DIR OR NOT CUDA_cupti_LIBRARY) - set(LIBKINETO_NOCUPTI ON CACHE BOOL "" FORCE) -endif() - -IF (NOT ROCM_SOURCE_DIR AND NOT ROCTRACER_INCLUDE_DIR) - set(LIBKINETO_NOROCTRACER ON CACHE BOOL "" FORCE) -endif() - -# Define file lists -if (LIBKINETO_NOCUPTI AND LIBKINETO_NOROCTRACER) - get_filelist("get_libkineto_cpu_only_srcs(with_api=False)" LIBKINETO_SRCS) - message(INFO " CUPTI unavailable or disabled - not building GPU profilers") -elseif(NOT LIBKINETO_NOROCTRACER) - get_filelist("get_libkineto_roctracer_srcs()" LIBKINETO_SRCS) - message(INFO " Building with roctracer") -else() - get_filelist("get_libkineto_cupti_srcs(with_api=False)" LIBKINETO_SRCS) -endif() -get_filelist("get_libkineto_public_headers()" LIBKINETO_PUBLIC_HEADERS) -get_filelist("get_libkineto_api_srcs()" LIBKINETO_API_SRCS) - -add_library(kineto_base OBJECT ${LIBKINETO_SRCS}) -add_library(kineto_api OBJECT ${LIBKINETO_API_SRCS}) - -# Make libraries depend on libkineto_defs.bzl -add_custom_target(libkineto_defs.bzl DEPENDS libkineto_defs.bzl) -add_dependencies(kineto_base libkineto_defs.bzl) - -set_target_properties(kineto_base kineto_api PROPERTIES - CXX_STANDARD 14 - CXX_STANDARD_REQUIRED YES - CXX_EXTENSIONS NO - CXX_VISIBILITY_PRESET hidden) - -set(KINETO_COMPILE_OPTIONS "-DKINETO_NAMESPACE=libkineto") -list(APPEND KINETO_COMPILE_OPTIONS "-DFMT_HEADER_ONLY") -if(NOT MSVC) - list(APPEND KINETO_COMPILE_OPTIONS "-std=c++14") -else() - list(APPEND KINETO_COMPILE_OPTIONS "/std:c++14") - list(APPEND KINETO_COMPILE_OPTIONS "-DWIN32_LEAN_AND_MEAN") - list(APPEND KINETO_COMPILE_OPTIONS "-DNOGDI") -endif() -if (NOT LIBKINETO_NOCUPTI) - list(APPEND KINETO_COMPILE_OPTIONS "-DHAS_CUPTI") -endif() -if (NOT LIBKINETO_NOROCTRACER) - target_compile_options(kineto_base PRIVATE "-DHAS_ROCTRACER") - target_compile_options(kineto_base PRIVATE "-D__HIP_PLATFORM_HCC__") - target_compile_options(kineto_base PRIVATE "-D__HIP_PLATFORM_AMD__") -endif() - -target_compile_options(kineto_base PRIVATE "${KINETO_COMPILE_OPTIONS}") -target_compile_options(kineto_api PRIVATE "${KINETO_COMPILE_OPTIONS}") - -if(NOT TARGET fmt) - if(NOT FMT_SOURCE_DIR) - set(FMT_SOURCE_DIR "${LIBKINETO_THIRDPARTY_DIR}/fmt" - CACHE STRING "fmt source directory from submodules") - endif() - - # Build FMT. - # FMT and some other libraries use BUILD_SHARED_LIBS to control - # the library type. - # Save and restore the value after configuring FMT - set(TEMP_BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS}) - set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build shared libs" FORCE) - set(FMT_LIBRARY_TYPE static CACHE STRING "Set lib type to static") - add_subdirectory("${FMT_SOURCE_DIR}" "${LIBKINETO_BINARY_DIR}/fmt") - set_property(TARGET fmt PROPERTY POSITION_INDEPENDENT_CODE ON) - set(BUILD_SHARED_LIBS ${TEMP_BUILD_SHARED_LIBS} CACHE BOOL "Build shared libs" FORCE) -endif() - -set(FMT_INCLUDE_DIR "${FMT_SOURCE_DIR}/include") -message(STATUS "Kineto: FMT_SOURCE_DIR = ${FMT_SOURCE_DIR}") -message(STATUS "Kineto: FMT_INCLUDE_DIR = ${FMT_INCLUDE_DIR}") -if (NOT CUPTI_INCLUDE_DIR) - set(CUPTI_INCLUDE_DIR "${CUDA_SOURCE_DIR}/extras/CUPTI/include") -endif() -if (NOT CUDA_INCLUDE_DIRS) - set(CUDA_INCLUDE_DIRS "${CUDA_SOURCE_DIR}/include") -endif() -if (NOT ROCTRACER_INCLUDE_DIR) - set(ROCTRACER_INCLUDE_DIR "${ROCM_SOURCE_DIR}/roctracer/include") -endif() -if (NOT ROCM_INCLUDE_DIRS) - set(ROCM_INCLUDE_DIRS "${ROCM_SOURCE_DIR}/include") -endif() - -message(INFO " CUPTI_INCLUDE_DIR = ${CUPTI_INCLUDE_DIR}") -message(INFO " ROCTRACER_INCLUDE_DIR = ${ROCTRACER_INCLUDE_DIR}") - -target_include_directories(kineto_base PUBLIC - $ - $ - $ - $ - $ - $ - $) - -target_include_directories(kineto_api PUBLIC - $ - $) - -if(KINETO_LIBRARY_TYPE STREQUAL "default") - add_library(kineto - $ - $) -elseif(KINETO_LIBRARY_TYPE STREQUAL "static") - add_library(kineto STATIC - $ - $) -elseif(KINETO_LIBRARY_TYPE STREQUAL "shared") - add_library(kineto SHARED - $) - set_property(TARGET kineto_base PROPERTY POSITION_INDEPENDENT_CODE ON) - set_target_properties(kineto PROPERTIES - CXX_VISIBILITY_PRESET hidden) -else() - message(FATAL_ERROR "Unsupported library type ${KINETO_LIBRARY_TYPE}") -endif() - -if(NOT LIBKINETO_NOROCTRACER) - find_library(ROCTRACER_LIBRARY NAMES libroctracer64.so HINTS /opt/rocm/roctracer/lib) - target_link_libraries(kineto "${ROCTRACER_LIBRARY}") - find_library(KINETO_HIP_LIBRARY NAMES libamdhip64.so HINTS /opt/rocm/lib) - target_link_libraries(kineto "${KINETO_HIP_LIBRARY}") -endif() - -if(NOT LIBKINETO_NOCUPTI) - target_link_libraries(kineto "${CUDA_cupti_LIBRARY}") -endif() -target_link_libraries(kineto $) -add_dependencies(kineto fmt::fmt-header-only) - -install(TARGETS kineto EXPORT kinetoLibraryConfig - ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} - LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR}) - -install(FILES ${LIBKINETO_PUBLIC_HEADERS} - DESTINATION "${CMAKE_INSTALL_INCLUDEDIR}/kineto") - -install(EXPORT kinetoLibraryConfig DESTINATION share/cmake/kineto - FILE kinetoLibraryConfig.cmake) - -if(KINETO_BUILD_TESTS) - add_subdirectory(test) -endif() diff --git a/plugins/tensorboard-plugins/libkineto/README.md b/plugins/tensorboard-plugins/libkineto/README.md deleted file mode 100644 index 37127ca5a..000000000 --- a/plugins/tensorboard-plugins/libkineto/README.md +++ /dev/null @@ -1,65 +0,0 @@ -# Libkineto - -Libkineto is an in-process profiling library, part of the Kineto performance -tools project. - -The library provides a way to collect GPU traces and metrics from the host -process, either via the library public API or by sending a signal, if enabled. - -Currently only NVIDIA GPUs are supported. - -## Build Notes -Libkineto uses the standard CMAKE-based build flow. - -### Dependencies -Libkineto requires gcc 5+ and: - -- NVIDIA CUPTI: used to collect traces and metrics from NVIDIA GPUs. -- fmt: used for its convenient and lightweight string formatting functionality. -- googletest: required to build and run Kineto's tests. - - **googletest is not required** if you don't want to run Kineto tests. -By default, building of tests is **on**. Turn it off by setting `KINETO_BUILD_TESTS` to **off**. - -You can download [NVIDIA CUPTI][1], [fmt][2], [googletest][3] and set -`CUDA_SOURCE_DIR`, `FMT_SOURCE_DIR`, `GOOGLETEST_SOURCE_DIR` respectively for -cmake to find these libraries. If the fmt and googletest variables are not set, cmake will -build the git submodules found in the `third_party` directory. -If `CUDA_SOURCE_DIR` is not set, libkineto will fail to build. - -### Building Libkineto - -``` -# Check out repo and sub modules -git clone --recursive https://github.com/pytorch/kineto.git -# Build libkineto with cmake -cd kineto/libkineto -mkdir build && cd build -cmake .. -make -``` - -To run the tests after building libkineto (if tests are built), use the following -command: -``` -make test -``` - -### Installing Libkineto -``` -make install -``` - -## How Libkineto works -We will provide a high-level overview, design philosophy and brief descriptions of various -parts of Libkineto in upcoming blogs. - -## Full documentation -We strive to keep our source files readable. The best and up-to-date -documentation is available in the source files. - -## License -Libkineto is BSD licensed, as detailed in the [LICENSE](../LICENSE) file. - -[1]:https://developer.nvidia.com/CUPTI-CTK10_2 -[2]:https://github.com/fmt -[3]:https://github.com/google/googletest diff --git a/plugins/tensorboard-plugins/libkineto/include/AbstractConfig.h b/plugins/tensorboard-plugins/libkineto/include/AbstractConfig.h deleted file mode 100644 index 1cadf4906..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/AbstractConfig.h +++ /dev/null @@ -1,113 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include - -namespace KINETO_NAMESPACE { - -class AbstractConfig { - public: - AbstractConfig& operator=(const AbstractConfig&) = delete; - AbstractConfig(AbstractConfig&&) = delete; - AbstractConfig& operator=(AbstractConfig&&) = delete; - - virtual ~AbstractConfig() { - for (const auto& p : featureConfigs_) { - delete p.second; - } - } - - // Return a copy of the full derived class - virtual AbstractConfig* cloneDerived(AbstractConfig& parent) const = 0; - - // Returns true if successfully parsed the config string - bool parse(const std::string& conf); - - // Default setup for signal-triggered profiling - virtual void setSignalDefaults() { - for (auto& p : featureConfigs_) { - p.second->setSignalDefaults(); - } - } - - // Default setup for client-triggered profiling - virtual void setClientDefaults() { - for (auto& p : featureConfigs_) { - p.second->setClientDefaults(); - } - } - - // Time config was created / updated - std::chrono::time_point timestamp() const { - return timestamp_; - } - - // Source config string that this was parsed from - const std::string& source() const { - return source_; - } - - AbstractConfig& feature(std::string name) const { - const auto& pos = featureConfigs_.find(name); - return *pos->second; - } - - // Transfers ownership of cfg arg - void addFeature(const std::string& name, AbstractConfig* cfg) { - featureConfigs_[name] = cfg; - } - - protected: - AbstractConfig() {} - AbstractConfig(const AbstractConfig& other) = default; - - // Return true if the option was recognized and successfully parsed. - // Throw std::invalid_argument if val is invalid. - virtual bool handleOption(const std::string& name, std::string& val); - - // Perform post-validation checks, typically conditons involving - // multiple options. - // Throw std::invalid_argument if automatic correction can not be made. - // - // @param fallbackProfileStartTime Specify a fallback profile start timestamp in case it was never specified by the client - virtual void validate(const std::chrono::time_point& fallbackProfileStartTime) = 0; - - // TODO: Separate out each profiler type into features? - virtual void printActivityProfilerConfig(std::ostream& s) const; - - // Helpers for use in handleOption - // Split a string by delimiter and remove external white space - std::vector splitAndTrim(const std::string& s, char delim) const; - // Lowercase for case-insensitive comparisons - std::string toLower(std::string& s) const; - // Does string end with suffix - bool endsWith(const std::string& s, const std::string& suffix) const; - // Conversions - int64_t toIntRange(const std::string& val, int64_t min, int64_t max) const; - int32_t toInt32(const std::string& val) const; - int64_t toInt64(const std::string& val) const; - bool toBool(std::string& val) const; - - void cloneFeaturesInto(AbstractConfig& cfg) const { - for (const auto& feature : featureConfigs_) { - cfg.featureConfigs_[feature.first] = feature.second->cloneDerived(cfg); - } - } - - private: - // Time config was created / updated - std::chrono::time_point timestamp_{}; - - // Original configuration string, used for comparison - std::string source_{""}; - - // Configuration objects for optional features - std::map featureConfigs_{}; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/include/ActivityProfilerInterface.h b/plugins/tensorboard-plugins/libkineto/include/ActivityProfilerInterface.h deleted file mode 100644 index 29871e47a..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/ActivityProfilerInterface.h +++ /dev/null @@ -1,91 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include - -#include "ActivityType.h" -#include "ActivityTraceInterface.h" -#include "IActivityProfiler.h" - -namespace libkineto { - -class ActivityProfilerController; -struct CpuTraceBuffer; -class Config; - -class ActivityProfilerInterface { - - public: - virtual ~ActivityProfilerInterface() {}; - - virtual void init() {} - virtual bool isInitialized() { - return false; - } - virtual bool isActive(){ - return false; - } - - // *** Asynchronous API *** - // Instead of starting and stopping the trace manually, provide a start time - // and duration and / or iteration stop criterion. - // Tracing terminates when either condition is met. - virtual void scheduleTrace(const std::string& configStr) {} - - // *** Synchronous API *** - // These must be called in order: - // prepareTrace -> startTrace -> stopTrace. - - // Many tracing structures are lazily initialized during trace collection, - // with potentially high overhead. - // Call prepareTrace to enable tracing, then run the region to trace - // at least once (and ideally run the same code that is to be traced) to - // allow tracing structures to be initialized. - virtual void prepareTrace( - const std::set& activityTypes, - const std::string& configStr = "") {} - - // Start recording, potentially reusing any buffers allocated since - // prepareTrace was called. - virtual void startTrace() {} - - // Stop and process trace, producing an in-memory list of trace records. - // The processing will be done synchronously (using the calling thread.) - virtual std::unique_ptr stopTrace() { - return nullptr; - } - - // Re-evaluate internal state to allow for triggering operations based - // on number of iteration. each implicitly increments the iteration count - virtual void step() {} - - // *** TraceActivity API *** - // FIXME: Pass activityProfiler interface into clientInterface? - virtual void pushCorrelationId(uint64_t id){} - virtual void popCorrelationId(){} - virtual void transferCpuTrace( - std::unique_ptr traceBuffer){} - - // Correlation ids for user defined spans - virtual void pushUserCorrelationId(uint64_t){} - virtual void popUserCorrelationId(){} - - // Saves information for the current thread to be used in profiler output - // Client must record any new kernel thread where the activity has occured. - virtual void recordThreadInfo() {} - - // Record trace metadata, currently supporting only string key and values, - // values with the same key are overwritten - virtual void addMetadata(const std::string& key, const std::string& value) = 0; - - // Add a child activity profiler, this enables frameworks in the application - // to enable custom framework events. - virtual void addChildActivityProfiler( - std::unique_ptr profiler) {} -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/ActivityTraceInterface.h b/plugins/tensorboard-plugins/libkineto/include/ActivityTraceInterface.h deleted file mode 100644 index 23d4edab0..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/ActivityTraceInterface.h +++ /dev/null @@ -1,21 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include - -namespace libkineto { - -struct ITraceActivity; - -class ActivityTraceInterface { - public: - virtual ~ActivityTraceInterface() {} - virtual const std::vector* activities() { - return nullptr; - } - virtual void save(const std::string& path) {} -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/ActivityType.h b/plugins/tensorboard-plugins/libkineto/include/ActivityType.h deleted file mode 100644 index 74c6a2531..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/ActivityType.h +++ /dev/null @@ -1,34 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include - -namespace libkineto { - -enum class ActivityType { - CPU_OP = 0, // cpu side ops - USER_ANNOTATION, - GPU_USER_ANNOTATION, - GPU_MEMCPY, - GPU_MEMSET, - CONCURRENT_KERNEL, // on-device kernels - EXTERNAL_CORRELATION, - CUDA_RUNTIME, // host side cuda runtime events - CUDA_PROFILER_RANGE, // CUPTI Profiler range for performance metrics - GLOW_RUNTIME, // host side glow runtime events - CPU_INSTANT_EVENT, // host side point-like events - PYTHON_FUNCTION, - OVERHEAD, // CUPTI induced overhead events sampled from its overhead API. - ENUM_COUNT // This is to add buffer and not used for any profiling logic. Add your new type before it. -}; - -const char* toString(ActivityType t); -ActivityType toActivityType(const std::string& str); - -// Return an array of all activity types except COUNT -constexpr int activityTypeCount = (int)ActivityType::ENUM_COUNT; -const std::array activityTypes(); - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/ClientInterface.h b/plugins/tensorboard-plugins/libkineto/include/ClientInterface.h deleted file mode 100644 index 06dc07583..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/ClientInterface.h +++ /dev/null @@ -1,16 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -namespace libkineto { - -class ClientInterface { - public: - virtual ~ClientInterface() {} - virtual void init() = 0; - virtual void warmup(bool setupOpInputsCollection) = 0; - virtual void start() = 0; - virtual void stop() = 0; -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/Config.h b/plugins/tensorboard-plugins/libkineto/include/Config.h deleted file mode 100644 index 040e96c9f..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/Config.h +++ /dev/null @@ -1,433 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include "AbstractConfig.h" -#include "ActivityType.h" - -#include -#include -#include -#include -#include -#include - -namespace KINETO_NAMESPACE { - -using namespace libkineto; - -class Config : public AbstractConfig { - public: - Config(); - Config& operator=(const Config&) = delete; - Config(Config&&) = delete; - Config& operator=(Config&&) = delete; - - // Return a full copy including feature config object - std::unique_ptr clone() const { - auto cfg = std::unique_ptr(new Config(*this)); - cloneFeaturesInto(*cfg); - return cfg; - } - - bool handleOption(const std::string& name, std::string& val) override; - - void setClientDefaults() override; - - // Log events to this file - const std::string& eventLogFile() const { - return eventLogFile_; - } - - bool activityProfilerEnabled() const { - return activityProfilerEnabled_ || - activitiesOnDemandTimestamp_.time_since_epoch().count() > 0; - } - - // Log activitiy trace to this file - const std::string& activitiesLogFile() const { - return activitiesLogFile_; - } - - // Log activitiy trace to this url - const std::string& activitiesLogUrl() const { - return activitiesLogUrl_; - } - - void setActivitiesLogUrl(const std::string& url) { - activitiesLogUrl_ = url; - } - - bool activitiesLogToMemory() const { - return activitiesLogToMemory_; - } - - // Is profiling enabled for the given device? - bool eventProfilerEnabledForDevice(uint32_t dev) const { - return 0 != (eventProfilerDeviceMask_ & (1 << dev)); - } - - // Take a sample (read hardware counters) at this frequency. - // This controls how often counters are read - if all counters cannot - // be collected simultaneously then multiple samples are needed to - // collect all requested counters - see multiplex period. - std::chrono::milliseconds samplePeriod() const { - return samplePeriod_; - } - - void setSamplePeriod(std::chrono::milliseconds period) { - samplePeriod_ = period; - } - - // When all requested counters cannot be collected simultaneously, - // counters will be multiplexed at this frequency. - // Multiplexing can have a large performance impact if done frequently. - // To avoid a perf impact, keep this at 1s or above. - std::chrono::milliseconds multiplexPeriod() const { - return multiplexPeriod_; - } - - void setMultiplexPeriod(std::chrono::milliseconds period) { - multiplexPeriod_ = period; - } - - // Report counters at this frequency. Note that several samples can - // be reported each time, see samplesPerReport. - std::chrono::milliseconds reportPeriod() const { - return reportPeriod_; - } - - void setReportPeriod(std::chrono::milliseconds msecs); - - // Number of samples dispatched each report period. - // Must be in the range [1, report period / sample period]. - // In other words, aggregation is supported but not interpolation. - int samplesPerReport() const { - return samplesPerReport_; - } - - void setSamplesPerReport(int count) { - samplesPerReport_ = count; - } - - // The names of events to collect - const std::set& eventNames() const { - return eventNames_; - } - - // Add additional events to be profiled - void addEvents(const std::set& names) { - eventNames_.insert(names.begin(), names.end()); - } - - // The names of metrics to collect - const std::set& metricNames() const { - return metricNames_; - } - - // Add additional metrics to be profiled - void addMetrics(const std::set& names) { - metricNames_.insert(names.begin(), names.end()); - } - - const std::vector& percentiles() const { - return eventReportPercentiles_; - } - - // Profile for this long, then revert to base config - std::chrono::seconds eventProfilerOnDemandDuration() const { - return eventProfilerOnDemandDuration_; - } - - void setEventProfilerOnDemandDuration(std::chrono::seconds duration) { - eventProfilerOnDemandDuration_ = duration; - } - - // Too many event profilers on a single system can overload the driver. - // At some point, latencies shoot through the roof and collection of samples - // becomes impossible. To avoid this situation we have a limit of profilers - // per GPU. - // NOTE: Communication with a daemon is needed for this feature. - // Library must be built with an active DaemonConfigLoader. - int maxEventProfilersPerGpu() const { - return eventProfilerMaxInstancesPerGpu_; - } - - // On Cuda11 we've seen occasional hangs when reprogramming counters - // Monitor profiling threads and report when a thread is not responding - // for a given number of seconds. - // A period of 0 means disable. - std::chrono::seconds eventProfilerHeartbeatMonitorPeriod() const { - return eventProfilerHeartbeatMonitorPeriod_; - } - - // The types of activities selected in the configuration file - const std::set& selectedActivityTypes() const { - return selectedActivityTypes_; - } - - void setSelectedActivityTypes(const std::set& types) { - selectedActivityTypes_ = types; - } - - bool isOpInputsCollectionEnabled() const { - return enableOpInputsCollection_; - } - - // Trace for this long - std::chrono::milliseconds activitiesDuration() const { - return activitiesDuration_; - } - - // Trace for this many iterations, determined by external API - int activitiesRunIterations() const { - return activitiesRunIterations_; - } - - std::chrono::milliseconds activitiesDurationDefault() const; - - void setActivitiesDuration(std::chrono::milliseconds duration) { - activitiesDuration_ = duration; - } - - int activitiesMaxGpuBufferSize() const { - return activitiesMaxGpuBufferSize_; - } - - std::chrono::seconds activitiesWarmupDuration() const { - return activitiesWarmupDuration_; - } - - int activitiesWarmupIterations() const { - return activitiesWarmupIterations_; - } - - // Timestamp at which the profiling to start, requested by the user. - const std::chrono::time_point requestTimestamp() - const { - if (profileStartTime_.time_since_epoch().count()) { - return profileStartTime_; - } - - // TODO(T94634890): Deperecate requestTimestamp - return requestTimestamp_ + maxRequestAge() + activitiesWarmupDuration(); - } - - bool hasProfileStartTime() const { - return requestTimestamp_.time_since_epoch().count() > 0 || - profileStartTime_.time_since_epoch().count() > 0; - } - - int profileStartIteration() const { - return profileStartIteration_; - } - - bool hasProfileStartIteration() const { - return profileStartIteration_ >= 0 && activitiesRunIterations_ > 0; - } - - void setProfileStartIteration(int iter) { - profileStartIteration_ = iter; - } - - int profileStartIterationRoundUp() const { - return profileStartIterationRoundUp_; - } - - // calculate the start iteration accounting for warmup - int startIterationIncludingWarmup() const { - if (!hasProfileStartIteration()) { - return -1; - } - return profileStartIteration_ - activitiesWarmupIterations_; - } - - const std::chrono::seconds maxRequestAge() const; - - // All VLOG* macros will log if the verbose log level is >= - // the verbosity specified for the verbose log message. - // Default value is -1, so messages with log level 0 will log by default. - int verboseLogLevel() const { - return verboseLogLevel_; - } - - // Modules for which verbose logging is enabled. - // If empty, logging is enabled for all modules. - const std::vector& verboseLogModules() const { - return verboseLogModules_; - } - - bool sigUsr2Enabled() const { - return enableSigUsr2_; - } - - bool ipcFabricEnabled() const { - return enableIpcFabric_; - } - - static std::chrono::milliseconds alignUp( - std::chrono::milliseconds duration, - std::chrono::milliseconds alignment) { - duration += alignment; - return duration - (duration % alignment); - } - - std::chrono::time_point - eventProfilerOnDemandStartTime() const { - return eventProfilerOnDemandTimestamp_; - } - - std::chrono::time_point - eventProfilerOnDemandEndTime() const { - return eventProfilerOnDemandTimestamp_ + eventProfilerOnDemandDuration_; - } - - std::chrono::time_point - activityProfilerRequestReceivedTime() const { - return activitiesOnDemandTimestamp_; - } - - // Users may request and set trace id and group trace id. - const std::string& requestTraceID() const { - return requestTraceID_; - } - - void setRequestTraceID(const std::string& tid) { - requestTraceID_ = tid; - } - - const std::string& requestGroupTraceID() const { - return requestGroupTraceID_; - } - - void setRequestGroupTraceID(const std::string& gtid) { - requestGroupTraceID_ = gtid; - } - - void updateActivityProfilerRequestReceivedTime(); - - void printActivityProfilerConfig(std::ostream& s) const override; - - void validate( - const std::chrono::time_point& fallbackProfileStartTime) override; - - static void addConfigFactory( - std::string name, - std::function factory); - - void print(std::ostream& s) const; - - private: - explicit Config(const Config& other) = default; - - AbstractConfig* cloneDerived(AbstractConfig& parent) const override { - // Clone from AbstractConfig not supported - assert(false); - return nullptr; - } - - uint8_t createDeviceMask(const std::string& val); - - // Adds valid activity types from the user defined string list in the - // configuration file - void setActivityTypes(const std::vector& selected_activities); - - // Sets the default activity types to be traced - void selectDefaultActivityTypes() { - // If the user has not specified an activity list, add all types - for (ActivityType t : activityTypes()) { - // Do no enable this by default - // TODO: introduce optional types - if (t != ActivityType::OVERHEAD) { - selectedActivityTypes_.insert(t); - } - } - } - - int verboseLogLevel_; - std::vector verboseLogModules_; - - // Event profiler - // These settings are also supported in on-demand mode - std::chrono::milliseconds samplePeriod_; - std::chrono::milliseconds reportPeriod_; - int samplesPerReport_; - std::set eventNames_; - std::set metricNames_; - - // On-demand duration - std::chrono::seconds eventProfilerOnDemandDuration_; - // Last on-demand request - std::chrono::time_point - eventProfilerOnDemandTimestamp_; - - int eventProfilerMaxInstancesPerGpu_; - - // Monitor whether event profiler threads are stuck - // at this frequency - std::chrono::seconds eventProfilerHeartbeatMonitorPeriod_; - - // These settings can not be changed on-demand - std::string eventLogFile_; - std::vector eventReportPercentiles_ = {5, 25, 50, 75, 95}; - uint8_t eventProfilerDeviceMask_ = ~0; - std::chrono::milliseconds multiplexPeriod_; - - // Activity profiler - bool activityProfilerEnabled_; - std::set selectedActivityTypes_; - - // The activity profiler settings are all on-demand - std::string activitiesLogFile_; - - std::string activitiesLogUrl_; - - // Log activities to memory buffer - bool activitiesLogToMemory_{false}; - - int activitiesMaxGpuBufferSize_; - std::chrono::seconds activitiesWarmupDuration_; - int activitiesWarmupIterations_; - - // Client Interface - // Enable inputs collection when tracing ops - bool enableOpInputsCollection_{true}; - - // Profile for specified iterations and duration - std::chrono::milliseconds activitiesDuration_; - int activitiesRunIterations_; - - // Below are not used - // Use this net name for iteration count - std::string activitiesExternalAPIIterationsTarget_; - // Only profile nets that includes this in the name - std::vector activitiesExternalAPIFilter_; - // Only profile nets with at least this many operators - int activitiesExternalAPINetSizeThreshold_; - // Only profile nets with at least this many GPU operators - int activitiesExternalAPIGpuOpCountThreshold_; - // Last activity profiler request - std::chrono::time_point - activitiesOnDemandTimestamp_; - - // Synchronized start timestamp - std::chrono::time_point profileStartTime_; - // or start iteration - int profileStartIteration_; - int profileStartIterationRoundUp_; - - // DEPRECATED - std::chrono::time_point requestTimestamp_; - - // Enable profiling via SIGUSR2 - bool enableSigUsr2_; - - // Enable IPC Fabric instead of thrift communication - bool enableIpcFabric_; - - // Logger Metadata - std::string requestTraceID_; - std::string requestGroupTraceID_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/include/GenericTraceActivity.h b/plugins/tensorboard-plugins/libkineto/include/GenericTraceActivity.h deleted file mode 100644 index 4272cf1ef..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/GenericTraceActivity.h +++ /dev/null @@ -1,125 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include - -#include "ThreadUtil.h" -#include "ITraceActivity.h" -#include "TraceSpan.h" - -namespace libkineto { - -// Link type, used in GenericTraceActivity.flow.type -constexpr unsigned int kLinkFwdBwd = 1; -constexpr unsigned int kLinkAsyncCpuGpu = 2; - -// @lint-ignore-every CLANGTIDY cppcoreguidelines-non-private-member-variables-in-classes -// @lint-ignore-every CLANGTIDY cppcoreguidelines-pro-type-member-init -class GenericTraceActivity : public ITraceActivity { - - public: - GenericTraceActivity() : activityType(ActivityType::ENUM_COUNT), traceSpan_(NULL) {} - - GenericTraceActivity( - const TraceSpan& trace, ActivityType type, const std::string& name) - : activityType(type), activityName(name), traceSpan_(&trace) { - } - - int64_t deviceId() const override { - return device; - } - - int64_t resourceId() const override { - return resource; - } - - int32_t getThreadId() const override { - return threadId; - } - - int64_t timestamp() const override { - return startTime; - } - - int64_t duration() const override { - return endTime - startTime; - } - - int64_t correlationId() const override { - return id; - } - - ActivityType type() const override { - return activityType; - } - - const ITraceActivity* linkedActivity() const override { - return nullptr; - } - - int flowType() const override { - return flow.type; - } - - int flowId() const override { - return flow.id; - } - - bool flowStart() const override { - return flow.start; - } - - const std::string name() const override { - return activityName; - } - - const TraceSpan* traceSpan() const override { - return traceSpan_; - } - - void log(ActivityLogger& logger) const override; - - //Encode client side metadata as a key/value - template - void addMetadata(const std::string& key, const ValType& value) { - metadata_.push_back(fmt::format("\"{}\": {}", key, value)); - } - - void addMetadataQuoted(const std::string& key, const std::string& value) { - metadata_.push_back(fmt::format("\"{}\": \"{}\"", key, value)); - } - - const std::string metadataJson() const override { - return fmt::format("{}", fmt::join(metadata_, ", ")); - } - - virtual ~GenericTraceActivity() {}; - - int64_t startTime{0}; - int64_t endTime{0}; - int32_t id{0}; - int32_t device{0}; - int32_t resource{0}; - int32_t threadId{0}; - ActivityType activityType; - std::string activityName; - struct Flow { - Flow(): id(0), type(0), start(0) {} - // Ids must be unique within each type - uint32_t id : 27; - // Type will be used to connect flows between profilers, as - // well as look up flow information (name etc) - uint32_t type : 4; - uint32_t start : 1; - } flow; - - private: - const TraceSpan* traceSpan_; - std::vector metadata_; -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/IActivityProfiler.h b/plugins/tensorboard-plugins/libkineto/include/IActivityProfiler.h deleted file mode 100644 index f5d4b3fb8..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/IActivityProfiler.h +++ /dev/null @@ -1,104 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include - -#include "Config.h" -#include "GenericTraceActivity.h" - -/* This file includes an abstract base class for an activity profiler - * that can be implemented by multiple tracing agents in the application. - * The high level Kineto profiler can co-ordinate start and end of tracing - * and combine together events from multiple such activity profilers. - */ - -namespace libkineto { - -using namespace KINETO_NAMESPACE; - -#ifdef _MSC_VER -// workaround for the predefined ERROR macro on Windows -#undef ERROR -#endif // _MSC_VER - -enum class TraceStatus { - READY, // Accepting trace requests - WARMUP, // Performing trace warmup - RECORDING, // Actively collecting activities - PROCESSING, // Recording is complete, preparing results - ERROR, // One or more errors (and possibly also warnings) occurred. - WARNING, // One or more warnings occurred. -}; - -/* IActivityProfilerSession: - * an opaque object that can be used by a high level profiler to - * start/stop and return trace events. - */ -class IActivityProfilerSession { - - public: - virtual ~IActivityProfilerSession() {} - - // start the trace collection synchronously - virtual void start() = 0; - - // stop the trace collection synchronously - virtual void stop() = 0; - - TraceStatus status() { - return status_; - } - - // returns list of Trace Activities - virtual std::vector& activities() = 0; - - // returns errors with this trace - virtual std::vector errors() = 0; - - // processes trace activities using logger - virtual void processTrace(ActivityLogger& logger) = 0; - - // XXX define trace formats - // virtual save(string name, TraceFormat format) - - protected: - TraceStatus status_ = TraceStatus::READY; -}; - - -/* Activity Profiler Plugins: - * These allow other frameworks to integrate into Kineto's primariy - * activity profiler. While the primary activity profiler handles - * timing the trace collections and correlating events the plugins - * can become source of new trace activity types. - */ -class IActivityProfiler { - - public: - - virtual ~IActivityProfiler() {} - - // name of profiler - virtual const std::string& name() const = 0; - - // returns activity types this profiler supports - virtual const std::set& availableActivities() const = 0; - - // Calls prepare() on registered tracer providers passing in the relevant - // activity types. Returns a profiler session handle - virtual std::unique_ptr configure( - const std::set& activity_types, - const Config& config) = 0; - - // asynchronous version of the above with future timestamp and duration. - virtual std::unique_ptr configure( - int64_t ts_ms, - int64_t duration_ms, - const std::set& activity_types, - const Config& config) = 0; -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/ILoggerObserver.h b/plugins/tensorboard-plugins/libkineto/include/ILoggerObserver.h deleted file mode 100644 index 4fce7851b..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/ILoggerObserver.h +++ /dev/null @@ -1,50 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -// Stages in libkineto used when pushing logs to UST Logger. -constexpr char kWarmUpStage[] = "Warm Up"; -constexpr char kCollectionStage[] = "Collection"; -constexpr char kPostProcessingStage[] = "Post Processing"; - -#if !USE_GOOGLE_LOG - -#include -#include - -namespace libkineto { - -enum LoggerOutputType { - VERBOSE = 0, - INFO = 1, - WARNING = 2, - ERROR = 3, - STAGE = 4, - ENUM_COUNT = 5 -}; - -const char* toString(LoggerOutputType t); -LoggerOutputType toLoggerOutputType(const std::string& str); - -constexpr int LoggerTypeCount = (int) LoggerOutputType::ENUM_COUNT; - -class ILoggerObserver { - public: - virtual ~ILoggerObserver() = default; - virtual void write(const std::string& message, LoggerOutputType ot) = 0; - virtual const std::map> extractCollectorMetadata() = 0; - virtual void reset() = 0; - virtual void addDevice(const int64_t device) = 0; - virtual void setTraceDurationMS(const int64_t duration) = 0; - virtual void addEventCount(const int64_t count) = 0; - virtual void setTraceID(const std::string&) {} - virtual void setGroupTraceID(const std::string&) {} - virtual void addDestination(const std::string& dest) = 0; - -}; - -} // namespace libkineto - -#endif // !USE_GOOGLE_LOG diff --git a/plugins/tensorboard-plugins/libkineto/include/ITraceActivity.h b/plugins/tensorboard-plugins/libkineto/include/ITraceActivity.h deleted file mode 100644 index a477ed814..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/ITraceActivity.h +++ /dev/null @@ -1,53 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -#include "ActivityType.h" - -namespace libkineto { - -class ActivityLogger; -struct TraceSpan; - -// Generic activity interface is borrowed from tensorboard protobuf format. -struct ITraceActivity { - virtual ~ITraceActivity() {} - // Device is a physical or logical entity, e.g. CPU, GPU or process - virtual int64_t deviceId() const = 0; - // A resource is something on the device, h/w thread, - // functional units etc. - virtual int64_t resourceId() const = 0; - // s/w thread - virtual int32_t getThreadId() const = 0; - // Start timestamp in mucrosecond - virtual int64_t timestamp() const = 0; - // Duration in microseconds - virtual int64_t duration() const = 0; - // Used to link up async activities - virtual int64_t correlationId() const = 0; - // Part of a flow, identified by flow id and type - virtual int flowType() const = 0; - virtual int flowId() const = 0; - virtual bool flowStart() const = 0; - virtual ActivityType type() const = 0; - virtual const std::string name() const = 0; - // Optional linked activity - virtual const ITraceActivity* linkedActivity() const = 0; - // Optional containing trace object - virtual const TraceSpan* traceSpan() const = 0; - // Log activity - virtual void log(ActivityLogger& logger) const = 0; - // Return json formatted metadata - // FIXME: Return iterator to dynamic type map here instead - virtual const std::string metadataJson() const = 0; - - static int64_t nsToUs(int64_t ns) { - // It's important that this conversion is the same everywhere. - // No rounding! - return ns / 1000; - } -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/ThreadUtil.h b/plugins/tensorboard-plugins/libkineto/include/ThreadUtil.h deleted file mode 100644 index d1dc80ad2..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/ThreadUtil.h +++ /dev/null @@ -1,22 +0,0 @@ -#pragma once - -#include -#include -#include -#include - -namespace libkineto { - -int32_t systemThreadId(); -int32_t threadId(); -bool setThreadName(const std::string& name); -std::string getThreadName(); - -int32_t processId(); -std::string processName(int32_t pid); - -// Return a list of pids and process names for the current process -// and its parents. -std::vector> pidCommandPairsOfAncestors(); - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/TraceSpan.h b/plugins/tensorboard-plugins/libkineto/include/TraceSpan.h deleted file mode 100644 index af9a9d5ee..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/TraceSpan.h +++ /dev/null @@ -1,36 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include - -namespace libkineto { - -struct TraceSpan { - TraceSpan() = delete; - TraceSpan( - int64_t startTime, int64_t endTime, std::string name) - : startTime(startTime), endTime(endTime), name(std::move(name)) { - } - TraceSpan( - int opCount, int it, std::string name, std::string prefix) - : opCount(opCount), - iteration(it), - name(std::move(name)), - prefix(std::move(prefix)) { - } - - // FIXME: change to duration? - int64_t startTime{0}; - int64_t endTime{0}; - int opCount{0}; - int iteration{-1}; - // Name is used to identify timeline - std::string name; - // Prefix used to distinguish trace spans on the same timeline - std::string prefix; -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/libkineto.h b/plugins/tensorboard-plugins/libkineto/include/libkineto.h deleted file mode 100644 index 87c3d64f6..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/libkineto.h +++ /dev/null @@ -1,138 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -// Mediator for initialization and profiler control - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ActivityProfilerInterface.h" -#include "ActivityType.h" -#include "ClientInterface.h" -#include "GenericTraceActivity.h" -#include "TraceSpan.h" -#include "IActivityProfiler.h" -#include "ActivityTraceInterface.h" - -#include "ThreadUtil.h" - -extern "C" { - void suppressLibkinetoLogMessages(); - int InitializeInjection(void); - bool libkineto_init(bool cpuOnly, bool logOnError); -} - -namespace libkineto { - -class Config; -class ConfigLoader; - -struct CpuTraceBuffer { - TraceSpan span{0, 0, "none"}; - int gpuOpCount; - std::deque activities; -}; - -using ChildActivityProfilerFactory = - std::function()>; - -class LibkinetoApi { - public: - - explicit LibkinetoApi(ConfigLoader& configLoader) - : configLoader_(configLoader) { - } - - // Called by client that supports tracing API. - // libkineto can still function without this. - void registerClient(ClientInterface* client); - - // Called by libkineto on init - void registerProfiler(std::unique_ptr profiler) { - activityProfiler_ = std::move(profiler); - initClientIfRegistered(); - } - - ActivityProfilerInterface& activityProfiler() { - return *activityProfiler_; - } - - ClientInterface* client() { - return client_; - } - - void initProfilerIfRegistered() { - static std::once_flag once; - if (activityProfiler_) { - std::call_once(once, [this] { - if (!activityProfiler_->isInitialized()) { - activityProfiler_->init(); - initChildActivityProfilers(); - } - }); - } - } - - bool isProfilerInitialized() const { - return activityProfiler_ && activityProfiler_->isInitialized(); - } - - bool isProfilerRegistered() const { - return activityProfiler_ != nullptr; - } - - void suppressLogMessages() { - suppressLibkinetoLogMessages(); - } - - // Provides access to profier configuration manaegement - ConfigLoader& configLoader() { - return configLoader_; - } - - void registerProfilerFactory( - ChildActivityProfilerFactory factory) { - if (isProfilerInitialized()) { - activityProfiler_->addChildActivityProfiler(factory()); - } else { - childProfilerFactories_.push_back(factory); - } - } - - private: - - void initChildActivityProfilers() { - if (!isProfilerInitialized()) { - return; - } - for (const auto& factory : childProfilerFactories_) { - activityProfiler_->addChildActivityProfiler(factory()); - } - childProfilerFactories_.clear(); - } - - // Client is initialized once both it and libkineto has registered - void initClientIfRegistered(); - - ConfigLoader& configLoader_; - std::unique_ptr activityProfiler_{}; - ClientInterface* client_{}; - int32_t clientRegisterThread_{0}; - - bool isLoaded_{false}; - std::vector childProfilerFactories_; -}; - -// Singleton -LibkinetoApi& api(); - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/include/time_since_epoch.h b/plugins/tensorboard-plugins/libkineto/include/time_since_epoch.h deleted file mode 100644 index caa6b4d92..000000000 --- a/plugins/tensorboard-plugins/libkineto/include/time_since_epoch.h +++ /dev/null @@ -1,16 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -namespace libkineto { - -inline int64_t timeSinceEpoch( - const std::chrono::time_point& t) { - return std::chrono::duration_cast( - t.time_since_epoch()) - .count(); -} - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/libkineto_defs.bzl b/plugins/tensorboard-plugins/libkineto/libkineto_defs.bzl deleted file mode 100644 index 330c54a22..000000000 --- a/plugins/tensorboard-plugins/libkineto/libkineto_defs.bzl +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Facebook, Inc. and its affiliates. -# All rights reserved. -# This source code is licensed under the BSD-style license found in the -# LICENSE file in the root directory of this source tree. - -def get_libkineto_api_srcs(): - return [ - "src/ThreadUtil.cpp", - "src/libkineto_api.cpp", - ] - -def get_libkineto_cupti_srcs(with_api = True): - return [ - "src/CudaDeviceProperties.cpp", - "src/CuptiActivityApi.cpp", - "src/CuptiActivityPlatform.cpp", - "src/CuptiCallbackApi.cpp", - "src/CuptiEventApi.cpp", - "src/CuptiMetricApi.cpp", - "src/CuptiRangeProfilerApi.cpp", - "src/Demangle.cpp", - "src/EventProfiler.cpp", - "src/EventProfilerController.cpp", - "src/WeakSymbols.cpp", - "src/cupti_strings.cpp", - ] + (get_libkineto_cpu_only_srcs(with_api)) - -def get_libkineto_roctracer_srcs(with_api = True): - return [ - "src/RoctracerActivityApi.cpp", - ] + (get_libkineto_cpu_only_srcs(with_api)) - -def get_libkineto_cpu_only_srcs(with_api = True): - return [ - "src/AbstractConfig.cpp", - "src/CuptiActivityProfiler.cpp", - "src/ActivityProfilerController.cpp", - "src/ActivityProfilerProxy.cpp", - "src/ActivityType.cpp", - "src/Config.cpp", - "src/ConfigLoader.cpp", - "src/CuptiActivityApi.cpp", - "src/Demangle.cpp", - "src/GenericTraceActivity.cpp", - "src/ILoggerObserver.cpp", - "src/Logger.cpp", - "src/init.cpp", - "src/output_csv.cpp", - "src/output_json.cpp", - ] + (get_libkineto_api_srcs() if with_api else []) - -def get_libkineto_public_headers(): - return [ - "include/AbstractConfig.h", - "include/ActivityProfilerInterface.h", - "include/ActivityType.h", - "include/Config.h", - "include/ClientInterface.h", - "include/GenericTraceActivity.h", - "include/GenericTraceActivity.h", - "include/IActivityProfiler.h", - "include/ILoggerObserver.h", - "include/ITraceActivity.h", - "include/TraceSpan.h", - "include/ThreadUtil.h", - "include/libkineto.h", - "include/time_since_epoch.h", - ] - -# kineto code should be updated to not have to -# suppress these warnings. -KINETO_COMPILER_FLAGS = [ - "-fexceptions", - "-Wno-deprecated-declarations", - "-Wno-unused-function", - "-Wno-unused-private-field", -] diff --git a/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cpp b/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cpp deleted file mode 100644 index 780047912..000000000 --- a/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cpp +++ /dev/null @@ -1,38 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include -#include - -#include -#include - -#include "kineto/libkineto/sample_programs/kineto_playground.cuh" - -using namespace kineto; - -static const std::string kFileName = "/tmp/kineto_playground_trace.json"; - -int main() { - warmup(); - - // Kineto config - - // Empty types set defaults to all types - std::set types; - - auto& profiler = libkineto::api().activityProfiler(); - libkineto::api().initProfilerIfRegistered(); - profiler.prepareTrace(types); - - // Good to warm up after prepareTrace to get cupti initialization to settle - warmup(); - profiler.startTrace(); - playground(); - - auto trace = profiler.stopTrace(); - LOG(INFO) << "Stopped and processed trace. Got " << trace->activities()->size() << " activities."; - trace->save(kFileName); - return 0; -} - diff --git a/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cu b/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cu deleted file mode 100644 index 54c6f82ff..000000000 --- a/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cu +++ /dev/null @@ -1,60 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include - -#include "kineto_playground.cuh" - - -namespace kineto { - -void warmup(void) { - // Inititalizing CUDA can take a while which we normally do not want to see in Kineto traces. - // This is done in various ways that take Kineto as dependency. This is our way of doing warmup - // for kineto_playground - size_t bytes = 1000; - float* mem = NULL; - auto error = cudaMalloc(&mem, bytes); - if (error != cudaSuccess) { - printf("cudaMalloc failed during kineto_playground warmup. error code: %d", error); - return; - } - - cudaFree(mem); -} - -void basicMemcpyMemset(void) { - size_t size = (1 << 8) * sizeof(float); - float *hostMemSrc, *deviceMem, *hostMemDst; - cudaError_t err; - - hostMemSrc = (float*)malloc(size); - hostMemDst = (float*)malloc(size); - err = cudaMalloc(&deviceMem, size); - if (err != cudaSuccess) { - printf("cudaMalloc failed during %s", __func__); - return; - } - - memset(hostMemSrc, 1, size); - cudaMemcpy(deviceMem, hostMemSrc, size, cudaMemcpyHostToDevice); - if (err != cudaSuccess) { - printf("cudaMemcpy failed during %s", __func__); - return; - } - - cudaMemcpy(hostMemDst, deviceMem, size, cudaMemcpyDeviceToHost); - if (err != cudaSuccess) { - printf("cudaMemcpy failed during %s", __func__); - return; - } - - free(hostMemSrc); - free(hostMemDst); - cudaFree(deviceMem); -} - -void playground(void) { - // Add your experimental CUDA implementation here. -} - -} diff --git a/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cuh b/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cuh deleted file mode 100644 index 54e1ee59a..000000000 --- a/plugins/tensorboard-plugins/libkineto/sample_programs/kineto_playground.cuh +++ /dev/null @@ -1,18 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -namespace kineto { - -// Warms up CUDA before the tracing starts -void warmup(void); - -// Basic usage of cudaMemcpy and cudaMemset -void basicMemcpyMemset(void); - -// Your experimental code goes in here! -void playground(void); - -} diff --git a/plugins/tensorboard-plugins/libkineto/src/AbstractConfig.cpp b/plugins/tensorboard-plugins/libkineto/src/AbstractConfig.cpp deleted file mode 100644 index d60ab43c9..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/AbstractConfig.cpp +++ /dev/null @@ -1,188 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "AbstractConfig.h" - -#include -#include -#include - -#include "Logger.h" - -using namespace std::chrono; - -using std::string; -using std::vector; - -namespace KINETO_NAMESPACE { - -constexpr char kWhitespace[] = "\t\n "; - -static bool isWhitespace(string& s) { - return s.find_first_not_of(kWhitespace) == string::npos; -} - -// Remove whitespace from both end of string -static inline string trim(string& s) { - if (s.empty()) { - return s; - } else if (isWhitespace(s)) { - return ""; - } - auto start = s.find_first_not_of(kWhitespace); - auto end = s.find_last_not_of(kWhitespace); - return s.substr(start, end - start + 1); -} - -// Helper function for split. -// Return the index of char d in string s. -// If not found, returns the length of the string. -static int find(const char* s, char delim) { - int i; - for (i = 0; s[i]; i++) { - if (s[i] == delim) { - break; - } - } - return i; -} - -// Split a string by delimiter -static vector split(const string& s, char delim) { - vector res; - const char* cs = s.c_str(); - for (int i = find(cs, delim); cs[i]; cs += i + 1, i = find(cs, delim)) { - res.emplace_back(cs, i); - } - res.emplace_back(cs); - return res; -} - -// Remove a trailing comment. -static inline string stripComment(const string& s) { - std::size_t pos = s.find("#"); - return s.substr(0, pos); -} - -string AbstractConfig::toLower(string& s) const { - string res = s; - for (int i = 0; i < res.size(); i++) { - if (res[i] >= 'A' && res[i] <= 'Z') { - res[i] += ('a' - 'A'); - } - } - return res; -} - -bool AbstractConfig::endsWith(const string& s, const string& suffix) const { - if (suffix.size() > s.size()) { - return false; - } - return s.compare(s.size() - suffix.size(), suffix.size(), suffix) == 0; -} - -vector AbstractConfig::splitAndTrim(const string& s, char delim) const { - auto res = split(s, delim); - for (string& x : res) { - x = trim(x); - } - return res; -} - -int64_t AbstractConfig::toIntRange(const string& val, int64_t min, int64_t max) - const { - char* invalid; - int64_t res = strtoll(val.c_str(), &invalid, 10); - if (val.empty() || *invalid) { - throw std::invalid_argument(fmt::format("Invalid integer: {}", val)); - } else if (res < min || res > max) { - throw std::invalid_argument(fmt::format( - "Invalid argument: {} - expected range [{}, {}]", res, min, max)); - } - return res; -} - -int32_t AbstractConfig::toInt32(const string& val) const { - return toIntRange(val, 0, ~0u / 2); -} - -int64_t AbstractConfig::toInt64(const string& val) const { - return toIntRange(val, 0, ~0ul / 2); -} - -bool AbstractConfig::toBool(string& val) const { - const std::array bool_vals{ - "n", "y", "no", "yes", "f", "t", "false", "true"}; - const string lower_val = toLower(val); - for (int i = 0; i < bool_vals.size(); i++) { - if (lower_val == bool_vals[i]) { - return i % 2; - } - } - throw std::invalid_argument(fmt::format("Invalid bool argument: {}", val)); - return false; -} - -bool AbstractConfig::parse(const string& conf) { - std::istringstream iss(conf); - string line; - - timestamp_ = system_clock::now(); - - // Read the string stream 1 line at a time to parse. - while (std::getline(iss, line)) { - line = stripComment(line); - if (isWhitespace(line)) { - continue; - } - vector key_val = splitAndTrim(line, '='); - if (key_val.size() != 2) { - LOG(ERROR) << "Invalid config line: " << line; - return false; - } else { - bool handled = false; - try { - handled = handleOption(key_val[0], key_val[1]); - if (!handled) { - for (auto& feature_cfg : featureConfigs_) { - if (feature_cfg.second->handleOption(key_val[0], key_val[1])) { - handled = true; - break; - } - } - } - } catch (const std::exception& e) { - LOG(ERROR) << "Failed to parse config line: " << line; - LOG(ERROR) << e.what(); - return false; - } - if (!handled) { - // This might be due to using a newer config option on an - // older binary where it is not supported. In this case, - // print a warning message - but it is expected to work! - LOG(WARNING) << "Unrecognized config line: " << line; - } - } - } - - validate(timestamp_); - - // Store original text, used to detect updates - source_ = conf; - timestamp_ = system_clock::now(); - return true; -} - -bool AbstractConfig::handleOption( - const std::string& /* unused */, - std::string& /* unused */) { - LOG(ERROR) << "handleOption unimplemented"; - return false; -} - -void AbstractConfig::printActivityProfilerConfig(std::ostream& s) const { - for (const auto& feature_cfg : featureConfigs_) { - feature_cfg.second->printActivityProfilerConfig(s); - } -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/ActivityBuffers.h b/plugins/tensorboard-plugins/libkineto/src/ActivityBuffers.h deleted file mode 100644 index 157af8793..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ActivityBuffers.h +++ /dev/null @@ -1,29 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - - -#include -#include - -#include "libkineto.h" -#include "CuptiActivityBuffer.h" - -namespace KINETO_NAMESPACE { - -struct ActivityBuffers { - std::list> cpu; - std::unique_ptr gpu; - - // Add a wrapper object to the underlying struct stored in the buffer - template - const ITraceActivity& addActivityWrapper(const T& act) { - wrappers_.push_back(std::make_unique(act)); - return *wrappers_.back().get(); - } - - private: - std::vector> wrappers_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/ActivityLoggerFactory.h b/plugins/tensorboard-plugins/libkineto/src/ActivityLoggerFactory.h deleted file mode 100644 index 0d1bf642c..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ActivityLoggerFactory.h +++ /dev/null @@ -1,60 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include -#include - -namespace KINETO_NAMESPACE { - -class ActivityLogger; - -class ActivityLoggerFactory { - - public: - using FactoryFunc = - std::function(const std::string& url)>; - - // Add logger factory for a protocol prefix - void addProtocol(const std::string& protocol, FactoryFunc f) { - factories_[tolower(protocol)] = f; - } - - // Create a logger, invoking the factory for the protocol specified in url - std::unique_ptr makeLogger(const std::string& url) const { - std::string protocol = extractProtocol(url); - auto it = factories_.find(tolower(protocol)); - if (it != factories_.end()) { - return it->second(stripProtocol(url)); - } - throw std::invalid_argument(fmt::format( - "No logger registered for the {} protocol prefix", - protocol)); - return nullptr; - } - - private: - static std::string tolower(std::string s) { - std::transform(s.begin(), s.end(), s.begin(), - [](unsigned char c) { return std::tolower(c); } - ); - return s; - } - - static std::string extractProtocol(std::string url) { - return url.substr(0, url.find("://")); - } - - static std::string stripProtocol(std::string url) { - size_t pos = url.find("://"); - return pos == url.npos ? url : url.substr(pos + 3); - } - - std::map factories_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerController.cpp b/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerController.cpp deleted file mode 100644 index c85d41ed7..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerController.cpp +++ /dev/null @@ -1,246 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "ActivityProfilerController.h" - -#include -#include - -#include "ActivityLoggerFactory.h" -#include "ActivityTrace.h" -#include "CuptiActivityApi.h" -#ifdef HAS_ROCTRACER -#include "RoctracerActivityApi.h" -#endif -#include "ThreadUtil.h" -#include "output_json.h" -#include "output_membuf.h" - -#include "Logger.h" - -using namespace std::chrono; - -namespace KINETO_NAMESPACE { - -constexpr milliseconds kProfilerIntervalMsecs(1000); - -ActivityProfilerController::ActivityProfilerController( - ConfigLoader& configLoader, bool cpuOnly) - : configLoader_(configLoader) { -#ifdef HAS_ROCTRACER - profiler_ = std::make_unique( - RoctracerActivityApi::singleton(), cpuOnly); -#else - profiler_ = std::make_unique( - CuptiActivityApi::singleton(), cpuOnly); -#endif - configLoader_.addHandler(ConfigLoader::ConfigKind::ActivityProfiler, this); -} - -ActivityProfilerController::~ActivityProfilerController() { - configLoader_.removeHandler( - ConfigLoader::ConfigKind::ActivityProfiler, this); - if (profilerThread_) { - // signaling termination of the profiler loop - stopRunloop_ = true; - profilerThread_->join(); - delete profilerThread_; - profilerThread_ = nullptr; - } -} - -static ActivityLoggerFactory initLoggerFactory() { - ActivityLoggerFactory factory; - factory.addProtocol("file", [](const std::string& url) { - return std::unique_ptr(new ChromeTraceLogger(url)); - }); - return factory; -} - -static ActivityLoggerFactory& loggerFactory() { - static ActivityLoggerFactory factory = initLoggerFactory(); - return factory; -} - -void ActivityProfilerController::addLoggerFactory( - const std::string& protocol, ActivityLoggerFactory::FactoryFunc factory) { - loggerFactory().addProtocol(protocol, factory); -} - -static std::unique_ptr makeLogger(const Config& config) { - if (config.activitiesLogToMemory()) { - return std::make_unique(config); - } - return loggerFactory().makeLogger(config.activitiesLogUrl()); -} - -bool ActivityProfilerController::canAcceptConfig() { - return !profiler_->isActive(); -} - -void ActivityProfilerController::acceptConfig(const Config& config) { - VLOG(1) << "acceptConfig"; - if (config.activityProfilerEnabled()) { - scheduleTrace(config); - } -} - -void ActivityProfilerController::profilerLoop() { - setThreadName("Kineto Activity Profiler"); - VLOG(0) << "Entering activity profiler loop"; - - auto now = system_clock::now(); - auto next_wakeup_time = now + kProfilerIntervalMsecs; - - while (!stopRunloop_) { - now = system_clock::now(); - - while (now < next_wakeup_time) { - /* sleep override */ - std::this_thread::sleep_for(next_wakeup_time - now); - now = system_clock::now(); - } - - if (!profiler_->isActive()) { - std::lock_guard lock(asyncConfigLock_); - if (asyncRequestConfig_ - && !asyncRequestConfig_->hasProfileStartIteration()) { - // Note on now + kProfilerIntervalMsecs - // Profiler interval does not align perfectly upto startTime - warmup. Waiting until the next tick - // won't allow sufficient time for the profiler to warm up. So check if we are very close to the warmup time and trigger warmup - if (now + kProfilerIntervalMsecs - >= (asyncRequestConfig_->requestTimestamp() - asyncRequestConfig_->activitiesWarmupDuration())) { - LOG(INFO) << "Received on-demand activity trace request by " - << " profile timestamp = " - << asyncRequestConfig_-> - requestTimestamp().time_since_epoch().count(); - activateConfig(now); - } - } - } - - while (next_wakeup_time < now) { - next_wakeup_time += kProfilerIntervalMsecs; - } - - if (profiler_->isActive()) { - next_wakeup_time = profiler_->performRunLoopStep(now, next_wakeup_time); - VLOG(1) << "Profiler loop: " - << duration_cast(system_clock::now() - now).count() - << "ms"; - } - } - - VLOG(0) << "Exited activity profiling loop"; -} - -void ActivityProfilerController::step() { - int64_t currentIter = ++iterationCount_; - VLOG(0) << "Step called , iteration = " << currentIter; - - // optimization to not take the lock unless necessary - if (asyncRequestConfig_ && !profiler_->isActive()) { - std::lock_guard lock(asyncConfigLock_); - auto startIter = asyncRequestConfig_->startIterationIncludingWarmup(); - - if (asyncRequestConfig_->hasProfileStartIteration() - && currentIter >= startIter) { - LOG(INFO) << "Received on-demand activity trace request by profile" - << " start iteration = " - << asyncRequestConfig_->profileStartIteration() - << " current iteration = " << currentIter; - - if (currentIter > startIter) { - // adjust the start iteration if it is in the past - auto newProfileStart = currentIter + - asyncRequestConfig_->activitiesWarmupIterations(); - LOG(INFO) << "Start iteration updated to " << newProfileStart; - asyncRequestConfig_->setProfileStartIteration(newProfileStart); - } - activateConfig(system_clock::now()); - } - } - - if (profiler_->isActive()) { - auto now = system_clock::now(); - auto next_wakeup_time = now + kProfilerIntervalMsecs; - profiler_->performRunLoopStep(now, next_wakeup_time, currentIter); - } -} - -void ActivityProfilerController::activateConfig( - std::chrono::time_point now) { - logger_ = makeLogger(*asyncRequestConfig_); - profiler_->setLogger(logger_.get()); - profiler_->configure(*asyncRequestConfig_, now); - asyncRequestConfig_ = nullptr; -} - -void ActivityProfilerController::scheduleTrace(const Config& config) { - VLOG(1) << "scheduleTrace"; - if (profiler_->isActive()) { - LOG(ERROR) << "Ignored request - profiler busy"; - return; - } - int64_t currentIter = iterationCount_; - if (config.hasProfileStartIteration() && currentIter < 0) { - LOG(ERROR) << "Ignored profile iteration count based request as " - << "application is not updating iteration count"; - return; - } - std::lock_guard lock(asyncConfigLock_); - asyncRequestConfig_ = config.clone(); - - auto startIter = asyncRequestConfig_->startIterationIncludingWarmup(); - - if (asyncRequestConfig_->hasProfileStartIteration() - && (currentIter > startIter) - && asyncRequestConfig_->profileStartIterationRoundUp() > 0) { - auto newProfileStart - = currentIter + asyncRequestConfig_->activitiesWarmupIterations(); - // round up to nearest multiple - auto divisor = asyncRequestConfig_->profileStartIterationRoundUp(); - auto rem = newProfileStart % divisor; - newProfileStart += ((rem == 0) ? 0 : divisor - rem); - LOG(INFO) << "Rounding up profiler start iteration to : " << newProfileStart; - asyncRequestConfig_->setProfileStartIteration(newProfileStart); - } - - // start a profilerLoop() thread to handle request - if (!profilerThread_) { - profilerThread_ = - new std::thread(&ActivityProfilerController::profilerLoop, this); - } -} - -void ActivityProfilerController::prepareTrace(const Config& config) { - // Requests from ActivityProfilerApi have higher priority than - // requests from other sources (signal, daemon). - // Cancel any ongoing request and refuse new ones. - auto now = system_clock::now(); - if (profiler_->isActive()) { - LOG(WARNING) << "Cancelling current trace request in order to start " - << "higher priority synchronous request"; - if (libkineto::api().client()) { - libkineto::api().client()->stop(); - } - profiler_->stopTrace(now); - profiler_->reset(); - } - - profiler_->configure(config, now); -} - -std::unique_ptr ActivityProfilerController::stopTrace() { - profiler_->stopTrace(std::chrono::system_clock::now()); - auto logger = std::make_unique(profiler_->config()); - profiler_->processTrace(*logger); - profiler_->reset(); - return std::make_unique(std::move(logger), loggerFactory()); -} - -void ActivityProfilerController::addMetadata( - const std::string& key, const std::string& value) { - profiler_->addMetadata(key, value); -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerController.h b/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerController.h deleted file mode 100644 index 415f107cb..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerController.h +++ /dev/null @@ -1,84 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include - -#include "ActivityLoggerFactory.h" -#include "CuptiActivityProfiler.h" -#include "ActivityProfilerInterface.h" -#include "ActivityTraceInterface.h" -#include "ConfigLoader.h" -#include "CuptiActivityApi.h" - -namespace KINETO_NAMESPACE { - -class Config; - -class ActivityProfilerController : public ConfigLoader::ConfigHandler { - public: - explicit ActivityProfilerController(ConfigLoader& configLoader, bool cpuOnly); - ActivityProfilerController(const ActivityProfilerController&) = delete; - ActivityProfilerController& operator=(const ActivityProfilerController&) = - delete; - - ~ActivityProfilerController(); - - static void addLoggerFactory( - const std::string& protocol, - ActivityLoggerFactory::FactoryFunc factory); - - bool canAcceptConfig() override; - void acceptConfig(const Config& config) override; - - void scheduleTrace(const Config& config); - - void prepareTrace(const Config& config); - - void startTrace() { - profiler_->startTrace(std::chrono::system_clock::now()); - } - - void step(); - - std::unique_ptr stopTrace(); - - bool isActive() { - return profiler_->isActive(); - } - - void transferCpuTrace( - std::unique_ptr cpuTrace) { - return profiler_->transferCpuTrace(std::move(cpuTrace)); - } - - void recordThreadInfo() { - profiler_->recordThreadInfo(); - } - - void addChildActivityProfiler( - std::unique_ptr profiler) { - profiler_->addChildActivityProfiler(std::move(profiler)); - } - - void addMetadata(const std::string& key, const std::string& value); - - private: - void profilerLoop(); - void activateConfig(std::chrono::time_point now); - - std::unique_ptr asyncRequestConfig_; - std::mutex asyncConfigLock_; - std::unique_ptr profiler_; - std::unique_ptr logger_; - std::thread* profilerThread_{nullptr}; - std::atomic_bool stopRunloop_{false}; - std::atomic iterationCount_{-1}; - ConfigLoader& configLoader_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerProxy.cpp b/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerProxy.cpp deleted file mode 100644 index b2d36b7b3..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerProxy.cpp +++ /dev/null @@ -1,119 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "ActivityProfilerProxy.h" - -#include "ActivityProfilerController.h" -#include "Config.h" -#include "CuptiActivityApi.h" -#include "Logger.h" -#include - -namespace KINETO_NAMESPACE { - -ActivityProfilerProxy::ActivityProfilerProxy( - bool cpuOnly, ConfigLoader& configLoader) - : cpuOnly_(cpuOnly), configLoader_(configLoader) { -} - -ActivityProfilerProxy::~ActivityProfilerProxy() { - delete controller_; -}; - -void ActivityProfilerProxy::init() { - if (!controller_) { - controller_ = new ActivityProfilerController(configLoader_, cpuOnly_); - } -} - -void ActivityProfilerProxy::scheduleTrace(const std::string& configStr) { - Config config; - config.parse(configStr); - controller_->scheduleTrace(config); -} - -void ActivityProfilerProxy::scheduleTrace(const Config& config) { - controller_->scheduleTrace(config); -} - -void ActivityProfilerProxy::prepareTrace( - const std::set& activityTypes, - const std::string& configStr) { - Config config; - bool validate_required = true; - - // allow user provided config to override default options - if (!configStr.empty()) { - if (!config.parse(configStr)) { - LOG(WARNING) << "Failed to parse config : " << configStr; - } - // parse also runs validate - validate_required = false; - } - - config.setClientDefaults(); - config.setSelectedActivityTypes(activityTypes); - - if (validate_required) { - config.validate(std::chrono::system_clock::now()); - } - - controller_->prepareTrace(config); -} - -void ActivityProfilerProxy::startTrace() { - controller_->startTrace(); -} - -std::unique_ptr -ActivityProfilerProxy::stopTrace() { - return controller_->stopTrace(); -} - -void ActivityProfilerProxy::step() { - controller_->step(); -} - -bool ActivityProfilerProxy::isActive() { - return controller_->isActive(); -} - -void ActivityProfilerProxy::pushCorrelationId(uint64_t id) { - CuptiActivityApi::pushCorrelationID(id, - CuptiActivityApi::CorrelationFlowType::Default); -} - -void ActivityProfilerProxy::popCorrelationId() { - CuptiActivityApi::popCorrelationID( - CuptiActivityApi::CorrelationFlowType::Default); -} - -void ActivityProfilerProxy::pushUserCorrelationId(uint64_t id) { - CuptiActivityApi::pushCorrelationID(id, - CuptiActivityApi::CorrelationFlowType::User); -} - -void ActivityProfilerProxy::popUserCorrelationId() { - CuptiActivityApi::popCorrelationID( - CuptiActivityApi::CorrelationFlowType::User); -} - -void ActivityProfilerProxy::transferCpuTrace( - std::unique_ptr traceBuffer) { - controller_->transferCpuTrace(std::move(traceBuffer)); -} - -void ActivityProfilerProxy::addMetadata( - const std::string& key, const std::string& value) { - controller_->addMetadata(key, value); -} - -void ActivityProfilerProxy::recordThreadInfo() { - controller_->recordThreadInfo(); -} - -void ActivityProfilerProxy::addChildActivityProfiler( - std::unique_ptr profiler) { - controller_->addChildActivityProfiler(std::move(profiler)); -} - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerProxy.h b/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerProxy.h deleted file mode 100644 index b5cf84b2f..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ActivityProfilerProxy.h +++ /dev/null @@ -1,73 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include "ActivityProfilerInterface.h" - -#include -#include -#include - -#include "ActivityType.h" -#include "ITraceActivity.h" - -namespace libkineto { - // previous declaration is struct so this one must be too. - struct CpuTraceBuffer; -} - -namespace KINETO_NAMESPACE { - -using namespace libkineto; - -class ActivityProfilerController; -class Config; -class ConfigLoader; - -class ActivityProfilerProxy : public ActivityProfilerInterface { - - public: - ActivityProfilerProxy(bool cpuOnly, ConfigLoader& configLoader); - ~ActivityProfilerProxy() override; - - void init() override; - bool isInitialized() override { - return controller_ != nullptr; - } - - bool isActive() override; - - void recordThreadInfo() override; - - void scheduleTrace(const std::string& configStr) override; - void scheduleTrace(const Config& config); - - void prepareTrace( - const std::set& activityTypes, - const std::string& configStr = "") override; - - void startTrace() override; - void step() override; - std::unique_ptr stopTrace() override; - - void pushCorrelationId(uint64_t id) override; - void popCorrelationId() override; - - void pushUserCorrelationId(uint64_t id) override; - void popUserCorrelationId() override; - - void transferCpuTrace( - std::unique_ptr traceBuffer) override; - - void addMetadata(const std::string& key, const std::string& value) override; - - virtual void addChildActivityProfiler( - std::unique_ptr profiler) override; - - private: - bool cpuOnly_{true}; - ConfigLoader& configLoader_; - ActivityProfilerController* controller_{nullptr}; -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/ActivityTrace.h b/plugins/tensorboard-plugins/libkineto/src/ActivityTrace.h deleted file mode 100644 index 0be76af08..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ActivityTrace.h +++ /dev/null @@ -1,45 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include - -#include "ActivityLoggerFactory.h" -#include "ActivityTraceInterface.h" -#include "output_json.h" -#include "output_membuf.h" - -namespace libkineto { - -class ActivityTrace : public ActivityTraceInterface { - public: - ActivityTrace( - std::unique_ptr tmpLogger, - const ActivityLoggerFactory& factory) - : memLogger_(std::move(tmpLogger)), - loggerFactory_(factory) { - } - - const std::vector* activities() override { - return memLogger_->traceActivities(); - }; - - void save(const std::string& url) override { - std::string prefix; - // if no protocol is specified, default to file - if (url.find("://") == url.npos) { - prefix = "file://"; - } - memLogger_->log(*loggerFactory_.makeLogger(prefix + url)); - }; - - private: - // Activities are logged into a buffer - std::unique_ptr memLogger_; - - // Alternative logger used by save() if protocol prefix is specified - const ActivityLoggerFactory& loggerFactory_; -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/ActivityType.cpp b/plugins/tensorboard-plugins/libkineto/src/ActivityType.cpp deleted file mode 100644 index 18856b723..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ActivityType.cpp +++ /dev/null @@ -1,58 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "ActivityType.h" - -#include - -namespace libkineto { - -struct ActivityTypeName { - const char* name; - ActivityType type; -}; - -static constexpr std::array map{{ - {"cpu_op", ActivityType::CPU_OP}, - {"user_annotation", ActivityType::USER_ANNOTATION}, - {"gpu_user_Annotation", ActivityType::GPU_USER_ANNOTATION}, - {"gpu_memcpy", ActivityType::GPU_MEMCPY}, - {"gpu_memset", ActivityType::GPU_MEMSET}, - {"kernel", ActivityType::CONCURRENT_KERNEL}, - {"external_correlation", ActivityType::EXTERNAL_CORRELATION}, - {"cuda_runtime", ActivityType::CUDA_RUNTIME}, - {"cuda_profiler_range", ActivityType::CUDA_PROFILER_RANGE}, - {"glow_runtime", ActivityType::GLOW_RUNTIME}, - {"cpu_instant_event", ActivityType::CPU_INSTANT_EVENT}, - {"python_function", ActivityType::PYTHON_FUNCTION}, - {"overhead", ActivityType::OVERHEAD}, - {"ENUM_COUNT", ActivityType::ENUM_COUNT} -}}; - -static constexpr bool matchingOrder(int idx = 0) { - return map[idx].type == ActivityType::ENUM_COUNT || - ((idx == (int) map[idx].type) && matchingOrder(idx + 1)); -} -static_assert(matchingOrder(), "ActivityTypeName map is out of order"); - -const char* toString(ActivityType t) { - return map[(int)t].name; -} - -ActivityType toActivityType(const std::string& str) { - for (int i = 0; i < activityTypeCount; i++) { - if (str == map[i].name) { - return map[i].type; - } - } - throw std::invalid_argument(fmt::format("Invalid activity type: {}", str)); -} - -const std::array activityTypes() { - std::array res; - for (int i = 0; i < activityTypeCount; i++) { - res[i] = map[i].type; - } - return res; -} - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/Config.cpp b/plugins/tensorboard-plugins/libkineto/src/Config.cpp deleted file mode 100644 index 95538840f..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/Config.cpp +++ /dev/null @@ -1,473 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "Config.h" - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "Logger.h" -#include "ThreadUtil.h" - -using namespace std::chrono; - -using std::string; -using std::vector; - -namespace KINETO_NAMESPACE { - -constexpr milliseconds kDefaultSamplePeriodMsecs(1000); -constexpr milliseconds kDefaultMultiplexPeriodMsecs(1000); -constexpr milliseconds kDefaultActivitiesProfileDurationMSecs(500); -constexpr int kDefaultActivitiesMaxGpuBufferSize(128 * 1024 * 1024); -constexpr seconds kDefaultActivitiesWarmupDurationSecs(5); -constexpr seconds kDefaultBufferUntilWarmup(10); -constexpr seconds kDefaultReportPeriodSecs(1); -constexpr int kDefaultSamplesPerReport(1); -constexpr int kDefaultMaxEventProfilersPerGpu(1); -constexpr int kDefaultEventProfilerHearbeatMonitorPeriod(0); -constexpr seconds kMaxRequestAge(10); - -// Event Profiler -constexpr char kEventsKey[] = "EVENTS"; -constexpr char kMetricsKey[] = "METRICS"; -constexpr char kSamplePeriodKey[] = "SAMPLE_PERIOD_MSECS"; -constexpr char kMultiplexPeriodKey[] = "MULTIPLEX_PERIOD_MSECS"; -constexpr char kReportPeriodKey[] = "REPORT_PERIOD_SECS"; -constexpr char kSamplesPerReportKey[] = "SAMPLES_PER_REPORT"; -constexpr char kEventsLogFileKey[] = "EVENTS_LOG_FILE"; -constexpr char kEventsEnabledDevicesKey[] = "EVENTS_ENABLED_DEVICES"; -constexpr char kOnDemandDurationKey[] = "EVENTS_DURATION_SECS"; -constexpr char kMaxEventProfilersPerGpuKey[] = "MAX_EVENT_PROFILERS_PER_GPU"; -constexpr char kHeartbeatMonitorPeriodKey[] = - "EVENTS_HEARTBEAT_MONITOR_PERIOD_SECS"; - -// Activity Profiler -constexpr char kActivitiesEnabledKey[] = "ACTIVITIES_ENABLED"; -constexpr char kActivityTypesKey[] = "ACTIVITY_TYPES"; -constexpr char kActivitiesLogFileKey[] = "ACTIVITIES_LOG_FILE"; -constexpr char kActivitiesDurationKey[] = "ACTIVITIES_DURATION_SECS"; -constexpr char kActivitiesDurationMsecsKey[] = "ACTIVITIES_DURATION_MSECS"; -constexpr char kActivitiesWarmupDurationSecsKey[] = "ACTIVITIES_WARMUP_PERIOD_SECS"; -constexpr char kActivitiesMaxGpuBufferSizeKey[] = - "ACTIVITIES_MAX_GPU_BUFFER_SIZE_MB"; - -// Client Interface -constexpr char kClientInterfaceEnableOpInputsCollection[] = "CLIENT_INTERFACE_ENABLE_OP_INPUTS_COLLECTION"; - -constexpr char kActivitiesWarmupIterationsKey[] = "ACTIVITIES_WARMUP_ITERATIONS"; -constexpr char kActivitiesIterationsKey[] = "ACTIVITIES_ITERATIONS"; -// Common - -// Client-side timestamp used for synchronized start across hosts for -// distributed workloads. -// Specified in milliseconds Unix time (milliseconds since epoch). -// To use, compute a future timestamp as follows: -// * C++: + duration_cast( -// system_clock::now().time_since_epoch()).count() -// * Python: + int(time.time() * 1000) -// * Bash: $(( + $(date +%s%3N))) -// If used for a tracing request, timestamp must be far enough in the future -// to accommodate ACTIVITIES_WARMUP_PERIOD_SECS as well as any delays in -// propagating the request to the profiler. -// If the request can not be honored, it is up to the profilers to report -// an error somehow - no checks are done at config parse time. -// Note PROFILE_START_ITERATION has higher precedence -constexpr char kProfileStartTimeKey[] = "PROFILE_START_TIME"; -// DEPRECATED - USE PROFILE_START_TIME instead -constexpr char kRequestTimestampKey[] = "REQUEST_TIMESTAMP"; - -// Alternatively if the application supports reporting iterations -// start the profile at specific iteration. If the iteration count -// is >= this value the profile is started immediately. -// A value >= 0 is valid for this config option to take effect. -// Note PROFILE_START_ITERATION will take precedence over PROFILE_START_TIME. -constexpr char kProfileStartIterationKey[] = "PROFILE_START_ITERATION"; - -// Users can also start the profile on an integer multiple of the config -// value PROFILE_START_ITERATION_ROUNDUP. This knob behaves similar to -// PROFILE_START_ITERATION but instead of saying : "start collection trace on -// iteration 500", one can configure it to "start collecting trace on the next -// 100th iteration". -// -// For example, -// PROFILE_START_ITERATION_ROUNDUP = 1000, and the current iteration is 2010 -// The profile will then be collected on the next multiple of 1000 ie. 3000 -// Note PROFILE_START_ITERATION_ROUNDUP will also take precedence over -// PROFILE_START_TIME. -constexpr char kProfileStartIterationRoundUpKey[] - = "PROFILE_START_ITERATION_ROUNDUP"; - -// Enable on-demand trigger via kill -USR2 -// When triggered in this way, /tmp/libkineto.conf will be used as config. -constexpr char kEnableSigUsr2Key[] = "ENABLE_SIGUSR2"; - -// Enable communication through IPC Fabric -// and disable thrift communication with dynolog daemon -constexpr char kEnableIpcFabricKey[] = "ENABLE_IPC_FABRIC"; - -// Verbose log level -// The actual glog is not used and --v and --vmodule has no effect. -// Instead set the verbose level and modules in the config file. -constexpr char kLogVerboseLevelKey[] = "VERBOSE_LOG_LEVEL"; -// By default, all modules will log verbose messages >= verboseLogLevel. -// But to reduce noise we can specify one or more modules of interest. -// A module is a C/C++ object file (source file name), -// Example argument: ActivityProfiler.cpp,output_json.cpp -constexpr char kLogVerboseModulesKey[] = "VERBOSE_LOG_MODULES"; - -// Max devices supported on any system -constexpr uint8_t kMaxDevices = 8; - -namespace { - -struct FactoryMap { - - void addFactory( - std::string name, - std::function factory) { - std::lock_guard lock(lock_); - factories_[name] = factory; - } - - void addFeatureConfigs(Config& cfg) { - std::lock_guard lock(lock_); - for (const auto& p : factories_) { - cfg.addFeature(p.first, p.second(cfg)); - } - } - -// Config factories are shared between objects and since -// config objects can be created by multiple threads, we need a lock. - std::mutex lock_; - std::map> factories_; -}; - -std::shared_ptr configFactories() { - // Ensure this is safe to call during shutdown, even as static - // destructors are invoked. Once factories destructor has been - // invoked, weak_ptr.lock() will return nullptr. - // But calls before that point will have a valid shared_ptr, - // delaying destruction of the underlying FactoryMap. - static auto factories = std::make_shared(); - static std::weak_ptr weak_ptr = factories; - return weak_ptr.lock(); -} - -} // namespace - -void Config::addConfigFactory( - std::string name, - std::function factory) { - auto factories = configFactories(); - if (factories) { - factories->addFactory(name, factory); - } -} - -static string defaultTraceFileName() { - return fmt::format("/tmp/libkineto_activities_{}.json", processId()); -} - -Config::Config() - : verboseLogLevel_(-1), - samplePeriod_(kDefaultSamplePeriodMsecs), - reportPeriod_(duration_cast(kDefaultReportPeriodSecs)), - samplesPerReport_(kDefaultSamplesPerReport), - eventProfilerOnDemandDuration_(seconds(0)), - eventProfilerMaxInstancesPerGpu_(kDefaultMaxEventProfilersPerGpu), - eventProfilerHeartbeatMonitorPeriod_( - kDefaultEventProfilerHearbeatMonitorPeriod), - multiplexPeriod_(kDefaultMultiplexPeriodMsecs), - activityProfilerEnabled_(true), - activitiesLogFile_(defaultTraceFileName()), - activitiesLogUrl_(fmt::format("file://{}", activitiesLogFile_)), - activitiesMaxGpuBufferSize_(kDefaultActivitiesMaxGpuBufferSize), - activitiesWarmupDuration_(kDefaultActivitiesWarmupDurationSecs), - activitiesWarmupIterations_(0), - activitiesDuration_(kDefaultActivitiesProfileDurationMSecs), - activitiesRunIterations_(0), - activitiesOnDemandTimestamp_(milliseconds(0)), - profileStartTime_(milliseconds(0)), - profileStartIteration_(-1), - profileStartIterationRoundUp_(-1), - requestTimestamp_(milliseconds(0)), - enableSigUsr2_(false), - enableIpcFabric_(false) { - auto factories = configFactories(); - if (factories) { - factories->addFeatureConfigs(*this); - } -} - -uint8_t Config::createDeviceMask(const string& val) { - uint8_t res = 0; - for (const auto& d : splitAndTrim(val, ',')) { - res |= 1 << toIntRange(d, 0, kMaxDevices - 1); - } - return res; -} - -const seconds Config::maxRequestAge() const { - return kMaxRequestAge; -} - -static std::string getTimeStr(time_point t) { - std::time_t t_c = system_clock::to_time_t(t); - return fmt::format("{:%H:%M:%S}", fmt::localtime(t_c)); -} - -static time_point handleRequestTimestamp(int64_t ms) { - auto t = time_point(milliseconds(ms)); - auto now = system_clock::now(); - if (t > now) { - throw std::invalid_argument(fmt::format( - "Invalid {}: {} - time is in future", - kRequestTimestampKey, - getTimeStr(t))); - } else if ((now - t) > kMaxRequestAge) { - throw std::invalid_argument(fmt::format( - "Invalid {}: {} - time is more than {}s in the past", - kRequestTimestampKey, - getTimeStr(t), - kMaxRequestAge.count())); - } - return t; -} - -void Config::setActivityTypes( - const std::vector& selected_activities) { - selectedActivityTypes_.clear(); - if (selected_activities.size() > 0) { - for (const auto& activity : selected_activities) { - if (activity == "") { - continue; - } - selectedActivityTypes_.insert(toActivityType(activity)); - } - } -} - -bool Config::handleOption(const std::string& name, std::string& val) { - // Event Profiler - if (!name.compare(kEventsKey)) { - vector event_names = splitAndTrim(val, ','); - eventNames_.insert(event_names.begin(), event_names.end()); - } else if (!name.compare(kMetricsKey)) { - vector metric_names = splitAndTrim(val, ','); - metricNames_.insert(metric_names.begin(), metric_names.end()); - } else if (!name.compare(kSamplePeriodKey)) { - samplePeriod_ = milliseconds(toInt32(val)); - } else if (!name.compare(kMultiplexPeriodKey)) { - multiplexPeriod_ = milliseconds(toInt32(val)); - } else if (!name.compare(kReportPeriodKey)) { - setReportPeriod(seconds(toInt32(val))); - } else if (!name.compare(kSamplesPerReportKey)) { - samplesPerReport_ = toInt32(val); - } else if (!name.compare(kEventsLogFileKey)) { - eventLogFile_ = val; - } else if (!name.compare(kEventsEnabledDevicesKey)) { - eventProfilerDeviceMask_ = createDeviceMask(val); - } else if (!name.compare(kOnDemandDurationKey)) { - eventProfilerOnDemandDuration_ = seconds(toInt32(val)); - eventProfilerOnDemandTimestamp_ = timestamp(); - } else if (!name.compare(kMaxEventProfilersPerGpuKey)) { - eventProfilerMaxInstancesPerGpu_ = toInt32(val); - } else if (!name.compare(kHeartbeatMonitorPeriodKey)) { - eventProfilerHeartbeatMonitorPeriod_ = seconds(toInt32(val)); - } - - // Activity Profiler - else if (!name.compare(kActivitiesDurationKey)) { - activitiesDuration_ = - duration_cast(seconds(toInt32(val))); - activitiesOnDemandTimestamp_ = timestamp(); - } else if (!name.compare(kActivityTypesKey)) { - vector activity_types = splitAndTrim(toLower(val), ','); - setActivityTypes(activity_types); - } else if (!name.compare(kActivitiesDurationMsecsKey)) { - activitiesDuration_ = milliseconds(toInt32(val)); - activitiesOnDemandTimestamp_ = timestamp(); - } else if (!name.compare(kActivitiesIterationsKey)) { - activitiesRunIterations_ = toInt32(val); - activitiesOnDemandTimestamp_ = timestamp(); - } else if (!name.compare(kLogVerboseLevelKey)) { - verboseLogLevel_ = toInt32(val); - } else if (!name.compare(kLogVerboseModulesKey)) { - verboseLogModules_ = splitAndTrim(val, ','); - } else if (!name.compare(kActivitiesEnabledKey)) { - activityProfilerEnabled_ = toBool(val); - } else if (!name.compare(kActivitiesLogFileKey)) { - activitiesLogFile_ = val; - activitiesLogUrl_ = fmt::format("file://{}", val); - activitiesOnDemandTimestamp_ = timestamp(); - } else if (!name.compare(kActivitiesMaxGpuBufferSizeKey)) { - activitiesMaxGpuBufferSize_ = toInt32(val) * 1024 * 1024; - } else if (!name.compare(kActivitiesWarmupDurationSecsKey)) { - activitiesWarmupDuration_ = seconds(toInt32(val)); - } else if (!name.compare(kActivitiesWarmupIterationsKey)) { - activitiesWarmupIterations_ = toInt32(val); - } - - // Client Interface - else if (!name.compare(kClientInterfaceEnableOpInputsCollection)) { - enableOpInputsCollection_ = toBool(val); - } - - // Common - else if (!name.compare(kRequestTimestampKey)) { - VLOG(0) << kRequestTimestampKey - << " has been deprecated - please use " - << kProfileStartTimeKey; - requestTimestamp_ = handleRequestTimestamp(toInt64(val)); - } else if (!name.compare(kProfileStartTimeKey)) { - profileStartTime_ = - time_point(milliseconds(toInt64(val))); - } else if (!name.compare(kProfileStartIterationKey)) { - profileStartIteration_ = toInt32(val); - } else if (!name.compare(kProfileStartIterationRoundUpKey)) { - profileStartIterationRoundUp_ = toInt32(val); - } else if (!name.compare(kEnableSigUsr2Key)) { - enableSigUsr2_ = toBool(val); - } else if (!name.compare(kEnableIpcFabricKey)) { - enableIpcFabric_ = toBool(val); - } else { - return false; - } - return true; -} - -std::chrono::milliseconds Config::activitiesDurationDefault() const { - return kDefaultActivitiesProfileDurationMSecs; -}; - -void Config::updateActivityProfilerRequestReceivedTime() { - activitiesOnDemandTimestamp_ = system_clock::now(); -} - -void Config::setClientDefaults() { - AbstractConfig::setClientDefaults(); - activitiesLogToMemory_ = true; -} - -void Config::validate( - const time_point& fallbackProfileStartTime) { - if (samplePeriod_.count() == 0) { - LOG(WARNING) << "Sample period must be greater than 0, setting to 1ms"; - samplePeriod_ = milliseconds(1); - } - - if (multiplexPeriod_ < samplePeriod_) { - LOG(WARNING) << "Multiplex period can not be smaller " - << "than sample period"; - LOG(WARNING) << "Setting multiplex period to " << samplePeriod_.count() - << "ms"; - multiplexPeriod_ = samplePeriod_; - } - - if ((multiplexPeriod_ % samplePeriod_).count() != 0) { - LOG(WARNING) << "Multiplex period must be a " - << "multiple of sample period"; - multiplexPeriod_ = alignUp(multiplexPeriod_, samplePeriod_); - LOG(WARNING) << "Setting multiplex period to " << multiplexPeriod_.count() - << "ms"; - } - - if ((reportPeriod_ % multiplexPeriod_).count() != 0 || - reportPeriod_.count() == 0) { - LOG(WARNING) << "Report period must be a " - << "multiple of multiplex period"; - reportPeriod_ = alignUp(reportPeriod_, multiplexPeriod_); - LOG(WARNING) << "Setting report period to " << reportPeriod_.count() - << "ms"; - } - - if (samplesPerReport_ < 1) { - LOG(WARNING) << "Samples per report must be in the range " - << "[1, report period / sample period]"; - LOG(WARNING) << "Setting samples per report to 1"; - samplesPerReport_ = 1; - } - - int max_samples_per_report = reportPeriod_ / samplePeriod_; - if (samplesPerReport_ > max_samples_per_report) { - LOG(WARNING) << "Samples per report must be in the range " - << "[1, report period / sample period] ([1, " - << reportPeriod_.count() << "ms / " << samplePeriod_.count() - << "ms = " << max_samples_per_report << "])"; - LOG(WARNING) << "Setting samples per report to " << max_samples_per_report; - samplesPerReport_ = max_samples_per_report; - } - - if (!hasProfileStartTime()) { - VLOG(0) - << "No explicit timestamp has been set. " - << "Defaulting it to now + activitiesWarmupDuration with buffer."; - profileStartTime_ = fallbackProfileStartTime + - activitiesWarmupDuration() + kDefaultBufferUntilWarmup; - } - - if (profileStartIterationRoundUp_ == 0) { - // setting to 0 will mess up modulo arithmetic, set it to -1 so it has no effect - LOG(WARNING) << "Profiler start iteration round up should be >= 1."; - profileStartIterationRoundUp_ = -1; - } - - if (profileStartIterationRoundUp_ > 0 && !hasProfileStartIteration()) { - VLOG(0) << "Setting profiler start iteration to 0 so this config is " - << "triggered via iteration count."; - profileStartIteration_ = 0; - } - - if (selectedActivityTypes_.size() == 0) { - selectDefaultActivityTypes(); - } -} - -void Config::setReportPeriod(milliseconds msecs) { - reportPeriod_ = msecs; -} - -void Config::printActivityProfilerConfig(std::ostream& s) const { - s << "Log file: " << activitiesLogFile() << std::endl; - if (hasProfileStartIteration()) { - s << "Trace start Iteration: " << profileStartIteration() << std::endl; - s << "Trace warmup Iterations: " << activitiesWarmupIterations() << std::endl; - s << "Trace profile Iterations: " << activitiesRunIterations() << std::endl; - if (profileStartIterationRoundUp() > 0) { - s << "Trace start iteration roundup : " << profileStartIterationRoundUp() - << std::endl; - } - } else if (hasProfileStartTime()) { - std::time_t t_c = system_clock::to_time_t(requestTimestamp()); - LOG(INFO) << "Trace start time: " - << fmt::format("{:%Y-%m-%d %H:%M:%S}", fmt::localtime(t_c)); - s << "Trace duration: " << activitiesDuration().count() << "ms" - << std::endl; - s << "Warmup duration: " << activitiesWarmupDuration().count() << "s" - << std::endl; - } - - s << "Max GPU buffer size: " << activitiesMaxGpuBufferSize() / 1024 / 1024 - << "MB" << std::endl; - - std::vector activities; - for (const auto& activity : selectedActivityTypes_) { - activities.push_back(toString(activity)); - } - s << "Enabled activities: " - << fmt::format("{}", fmt::join(activities, ",")) << std::endl; - - AbstractConfig::printActivityProfilerConfig(s); -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/ConfigLoader.cpp b/plugins/tensorboard-plugins/libkineto/src/ConfigLoader.cpp deleted file mode 100644 index 4080b678d..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ConfigLoader.cpp +++ /dev/null @@ -1,300 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "ConfigLoader.h" - -#ifdef __linux__ -#include -#endif - -#include -#include -#include -#include -#include - -#include "DaemonConfigLoader.h" - -#include "Logger.h" - -using namespace std::chrono; -using std::string; - -namespace KINETO_NAMESPACE { - -using namespace libkineto; - -constexpr char kConfigFileEnvVar[] = "KINETO_CONFIG"; -#ifdef __linux__ -constexpr char kConfigFile[] = "/etc/libkineto.conf"; -constexpr char kOnDemandConfigFile[] = "/tmp/libkineto.conf"; -#else -constexpr char kConfigFile[] = "libkineto.conf"; -constexpr char kOnDemandConfigFile[] = "libkineto.conf"; -#endif - -constexpr std::chrono::seconds kConfigUpdateIntervalSecs(300); -constexpr std::chrono::seconds kOnDemandConfigUpdateIntervalSecs(5); - -#ifdef __linux__ -static struct sigaction originalUsr2Handler = {}; -#endif - -// Use SIGUSR2 to initiate profiling. -// Look for an on-demand config file. -// If none is found, default to base config. -// Try to not affect existing handlers -static bool hasOriginalSignalHandler() { -#ifdef __linux__ - return originalUsr2Handler.sa_handler != nullptr || - originalUsr2Handler.sa_sigaction != nullptr; -#else - return false; -#endif -} - -static void handle_signal(int signal) { -#ifdef __linux__ - if (signal == SIGUSR2) { - ConfigLoader::instance().handleOnDemandSignal(); - if (hasOriginalSignalHandler()) { - // Invoke original handler and reinstate ours - struct sigaction act; - sigaction(SIGUSR2, &originalUsr2Handler, &act); - raise(SIGUSR2); - sigaction(SIGUSR2, &act, &originalUsr2Handler); - } - } -#endif -} - -static void setupSignalHandler(bool enableSigUsr2) { -#ifdef __linux__ - if (enableSigUsr2) { - struct sigaction act = {}; - act.sa_handler = &handle_signal; - act.sa_flags = SA_NODEFER; - if (sigaction(SIGUSR2, &act, &originalUsr2Handler) < 0) { - PLOG(ERROR) << "Failed to register SIGUSR2 handler"; - } - if (originalUsr2Handler.sa_handler == &handle_signal) { - originalUsr2Handler = {}; - } - } else if (hasOriginalSignalHandler()) { - sigaction(SIGUSR2, &originalUsr2Handler, nullptr); - originalUsr2Handler = {}; - } -#endif -} - -// return an empty string if reading gets any errors. Otherwise a config string. -static std::string readConfigFromConfigFile(const char* filename) { - // Read whole file into a string. - std::ifstream file(filename); - std::string conf; - try { - conf.assign( - std::istreambuf_iterator(file), std::istreambuf_iterator()); - } catch (std::exception& e) { - VLOG(0) << "Error reading " << filename << ": " - << e.what(); - conf = ""; - } - return conf; -} - -static std::function()>& -daemonConfigLoaderFactory() { - static std::function()> factory = nullptr; - return factory; -} - -void ConfigLoader::setDaemonConfigLoaderFactory( - std::function()> factory) { - daemonConfigLoaderFactory() = factory; -} - -ConfigLoader& ConfigLoader::instance() { - static ConfigLoader config_loader; - return config_loader; -} - -// return an empty string if polling gets any errors. Otherwise a config string. -std::string ConfigLoader::readOnDemandConfigFromDaemon( - time_point now) { - if (!daemonConfigLoader_) { - return ""; - } - bool events = canHandlerAcceptConfig(ConfigKind::EventProfiler); - bool activities = canHandlerAcceptConfig(ConfigKind::ActivityProfiler); - return daemonConfigLoader_->readOnDemandConfig(events, activities); -} - -int ConfigLoader::contextCountForGpu(uint32_t device) { - if (!daemonConfigLoader_) { - // FIXME: Throw error? - return 0; - } - return daemonConfigLoader_->gpuContextCount(device); -} - -ConfigLoader::ConfigLoader() - : configUpdateIntervalSecs_(kConfigUpdateIntervalSecs), - onDemandConfigUpdateIntervalSecs_(kOnDemandConfigUpdateIntervalSecs), - stopFlag_(false), - onDemandSignal_(false) { -} - -void ConfigLoader::startThread() { - if (!updateThread_) { - // Create default base config here - at this point static initializers - // of extensions should have run and registered all config feature factories - std::lock_guard lock(configLock_); - if (!config_) { - config_ = std::make_unique(); - } - updateThread_ = - std::make_unique(&ConfigLoader::updateConfigThread, this); - } -} - -ConfigLoader::~ConfigLoader() { - if (updateThread_) { - stopFlag_ = true; - { - std::lock_guard lock(updateThreadMutex_); - updateThreadCondVar_.notify_one(); - } - updateThread_->join(); - } -#if !USE_GOOGLE_LOG - Logger::clearLoggerObservers(); -#endif // !USE_GOOGLE_LOG -} - -void ConfigLoader::handleOnDemandSignal() { - onDemandSignal_ = true; - { - std::lock_guard lock(updateThreadMutex_); - updateThreadCondVar_.notify_one(); - } -} - -const char* ConfigLoader::configFileName() { - if (!configFileName_) { - configFileName_ = getenv(kConfigFileEnvVar); - if (configFileName_ == nullptr) { - configFileName_ = kConfigFile; - } - } - return configFileName_; -} - -DaemonConfigLoader* ConfigLoader::daemonConfigLoader() { - if (!daemonConfigLoader_ && daemonConfigLoaderFactory()) { - daemonConfigLoader_ = daemonConfigLoaderFactory()(); - daemonConfigLoader_->setCommunicationFabric(config_->ipcFabricEnabled()); - } - return daemonConfigLoader_.get(); -} - -void ConfigLoader::updateBaseConfig() { - // First try reading local config file - // If that fails, read from daemon - // TODO: Invert these once daemon path fully rolled out - std::string config_str = readConfigFromConfigFile(configFileName()); - if (config_str.empty() && daemonConfigLoader()) { - // If local config file was not successfully loaded (e.g. not found) - // then try the daemon - config_str = daemonConfigLoader()->readBaseConfig(); - } - if (config_str != config_->source()) { - std::lock_guard lock(configLock_); - config_ = std::make_unique(); - config_->parse(config_str); - if (daemonConfigLoader()) { - daemonConfigLoader()->setCommunicationFabric(config_->ipcFabricEnabled()); - } - setupSignalHandler(config_->sigUsr2Enabled()); - SET_LOG_VERBOSITY_LEVEL( - config_->verboseLogLevel(), - config_->verboseLogModules()); - VLOG(0) << "Detected base config change"; - } -} - -void ConfigLoader::configureFromSignal( - time_point now, - Config& config) { - LOG(INFO) << "Received on-demand profiling signal, " - << "reading config from " << kOnDemandConfigFile; - // Reset start time to 0 in order to compute new default start time - const std::string config_str = "PROFILE_START_TIME=0\n" - + readConfigFromConfigFile(kOnDemandConfigFile); - config.parse(config_str); - config.setSignalDefaults(); - notifyHandlers(config); -} - -void ConfigLoader::configureFromDaemon( - time_point now, - Config& config) { - const std::string config_str = readOnDemandConfigFromDaemon(now); - if (config_str.empty()) { - return; - } - - LOG(INFO) << "Received config from dyno:\n" << config_str; - config.parse(config_str); - notifyHandlers(config); -} - -void ConfigLoader::updateConfigThread() { - auto now = system_clock::now(); - auto next_config_load_time = now; - auto next_on_demand_load_time = now + onDemandConfigUpdateIntervalSecs_; - seconds interval = configUpdateIntervalSecs_; - if (interval > onDemandConfigUpdateIntervalSecs_) { - interval = onDemandConfigUpdateIntervalSecs_; - } - auto onDemandConfig = std::make_unique(); - - // This can potentially sleep for long periods of time, so allow - // the desctructor to wake it to avoid a 5-minute long destruct period. - for (;;) { - { - std::unique_lock lock(updateThreadMutex_); - updateThreadCondVar_.wait_for(lock, interval); - } - if (stopFlag_) { - break; - } - now = system_clock::now(); - if (now > next_config_load_time) { - updateBaseConfig(); - next_config_load_time = now + configUpdateIntervalSecs_; - } - if (onDemandSignal_.exchange(false)) { - onDemandConfig = config_->clone(); - configureFromSignal(now, *onDemandConfig); - } else if (now > next_on_demand_load_time) { - onDemandConfig = std::make_unique(); - configureFromDaemon(now, *onDemandConfig); - next_on_demand_load_time = now + onDemandConfigUpdateIntervalSecs_; - } - if (onDemandConfig->verboseLogLevel() >= 0) { - LOG(INFO) << "Setting verbose level to " - << onDemandConfig->verboseLogLevel() - << " from on-demand config"; - SET_LOG_VERBOSITY_LEVEL( - onDemandConfig->verboseLogLevel(), - onDemandConfig->verboseLogModules()); - } - } -} - -bool ConfigLoader::hasNewConfig(const Config& oldConfig) { - std::lock_guard lock(configLock_); - return config_->timestamp() > oldConfig.timestamp(); -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/ConfigLoader.h b/plugins/tensorboard-plugins/libkineto/src/ConfigLoader.h deleted file mode 100644 index 4ce3468e4..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ConfigLoader.h +++ /dev/null @@ -1,147 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "Config.h" - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "ILoggerObserver.h" - -namespace libkineto { - class LibkinetoApi; -} - -namespace KINETO_NAMESPACE { - -using namespace libkineto; -class DaemonConfigLoader; - -class ConfigLoader { - public: - - static ConfigLoader& instance(); - - enum ConfigKind { - ActivityProfiler = 0, - EventProfiler, - NumConfigKinds - }; - - struct ConfigHandler { - virtual ~ConfigHandler() {} - virtual bool canAcceptConfig() = 0; - virtual void acceptConfig(const Config& cfg) = 0; - }; - - void addHandler(ConfigKind kind, ConfigHandler* handler) { - std::lock_guard lock(updateThreadMutex_); - handlers_[kind].push_back(handler); - startThread(); - } - - void removeHandler(ConfigKind kind, ConfigHandler* handler) { - std::lock_guard lock(updateThreadMutex_); - auto it = std::find( - handlers_[kind].begin(), handlers_[kind].end(), handler); - if (it != handlers_[kind].end()) { - handlers_[kind].erase(it); - } - } - - void notifyHandlers(const Config& cfg) { - std::lock_guard lock(updateThreadMutex_); - for (auto& key_val : handlers_) { - for (ConfigHandler* handler : key_val.second) { - handler->acceptConfig(cfg); - } - } - } - - bool canHandlerAcceptConfig(ConfigKind kind) { - std::lock_guard lock(updateThreadMutex_); - for (ConfigHandler* handler : handlers_[kind]) { - if (!handler->canAcceptConfig()) { - return false; - } - } - return true; - } - - void initBaseConfig() { - bool init = false; - { - std::lock_guard lock(configLock_); - init = !config_ || config_->source().empty(); - } - if (init) { - updateBaseConfig(); - } - } - - inline std::unique_ptr getConfigCopy() { - std::lock_guard lock(configLock_); - return config_->clone(); - } - - bool hasNewConfig(const Config& oldConfig); - int contextCountForGpu(uint32_t gpu); - - void handleOnDemandSignal(); - - static void setDaemonConfigLoaderFactory( - std::function()> factory); - - private: - ConfigLoader(); - ~ConfigLoader(); - - const char* configFileName(); - DaemonConfigLoader* daemonConfigLoader(); - - void startThread(); - void updateConfigThread(); - void updateBaseConfig(); - - // Create configuration when receiving SIGUSR2 - void configureFromSignal( - std::chrono::time_point now, - Config& config); - - // Create configuration when receiving request from a daemon - void configureFromDaemon( - std::chrono::time_point now, - Config& config); - - std::string readOnDemandConfigFromDaemon( - std::chrono::time_point now); - - std::mutex configLock_; - std::atomic configFileName_{nullptr}; - std::unique_ptr config_; - std::unique_ptr daemonConfigLoader_; - std::map> handlers_; - - std::chrono::seconds configUpdateIntervalSecs_; - std::chrono::seconds onDemandConfigUpdateIntervalSecs_; - std::unique_ptr updateThread_; - std::condition_variable updateThreadCondVar_; - std::mutex updateThreadMutex_; - std::atomic_bool stopFlag_{false}; - std::atomic_bool onDemandSignal_{false}; - -#if !USE_GOOGLE_LOG - std::unique_ptr> loggerObservers_; - std::mutex loggerObserversMutex_; -#endif // !USE_GOOGLE_LOG -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CudaDeviceProperties.cpp b/plugins/tensorboard-plugins/libkineto/src/CudaDeviceProperties.cpp deleted file mode 100644 index 1e909d5f9..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CudaDeviceProperties.cpp +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Copyright (c) Kineto Contributors - * All rights reserved. - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "CudaDeviceProperties.h" - -#include -#include - -#include -#include - -#include "Logger.h" - -namespace KINETO_NAMESPACE { - -static const std::vector createDeviceProps() { - std::vector props; - int device_count; - cudaError_t error_id = cudaGetDeviceCount(&device_count); - // Return empty vector if error. - if (error_id != cudaSuccess) { - LOG(ERROR) << "cudaGetDeviceCount failed with code " << error_id; - return {}; - } - VLOG(0) << "Device count is " << device_count; - for (size_t i = 0; i < device_count; ++i) { - cudaDeviceProp prop; - error_id = cudaGetDeviceProperties(&prop, i); - // Return empty vector if any device property fail to get. - if (error_id != cudaSuccess) { - LOG(ERROR) << "cudaGetDeviceProperties failed with " << error_id; - return {}; - } - props.push_back(prop); - LOGGER_OBSERVER_ADD_DEVICE(i); - } - return props; -} - -static const std::vector& deviceProps() { - static const std::vector props = createDeviceProps(); - return props; -} - -static const std::string createDevicePropertiesJson( - size_t id, const cudaDeviceProp& props) { - return fmt::format(R"JSON( - {{ - "id": {}, "name": "{}", "totalGlobalMem": {}, - "computeMajor": {}, "computeMinor": {}, - "maxThreadsPerBlock": {}, "maxThreadsPerMultiprocessor": {}, - "regsPerBlock": {}, "regsPerMultiprocessor": {}, "warpSize": {}, - "sharedMemPerBlock": {}, "sharedMemPerMultiprocessor": {}, - "numSms": {}, "sharedMemPerBlockOptin": {} - }})JSON", - id, props.name, props.totalGlobalMem, - props.major, props.minor, - props.maxThreadsPerBlock, props.maxThreadsPerMultiProcessor, - props.regsPerBlock, props.regsPerMultiprocessor, props.warpSize, - props.sharedMemPerBlock, props.sharedMemPerMultiprocessor, - props.multiProcessorCount, props.sharedMemPerBlockOptin); -} - -static const std::string createDevicePropertiesJson() { - std::vector jsonProps; - const auto& props = deviceProps(); - for (size_t i = 0; i < props.size(); i++) { - jsonProps.push_back(createDevicePropertiesJson(i, props[i])); - } - return fmt::format("{}", fmt::join(jsonProps, ",")); -} - -const std::string& devicePropertiesJson() { - static std::string devicePropsJson = createDevicePropertiesJson(); - return devicePropsJson; -} - -int smCount(uint32_t deviceId) { - const std::vector &props = deviceProps(); - return deviceId >= props.size() ? 0 : - props[deviceId].multiProcessorCount; -} - -float kernelOccupancy( - uint32_t deviceId, - uint16_t registersPerThread, - int32_t staticSharedMemory, - int32_t dynamicSharedMemory, - int32_t blockX, - int32_t blockY, - int32_t blockZ, - float blocksPerSm) { - // Calculate occupancy - float occupancy = -1.0; - const std::vector &props = deviceProps(); - if (deviceId < props.size()) { - cudaOccFuncAttributes occFuncAttr; - occFuncAttr.maxThreadsPerBlock = INT_MAX; - occFuncAttr.numRegs = registersPerThread; - occFuncAttr.sharedSizeBytes = staticSharedMemory; - occFuncAttr.partitionedGCConfig = PARTITIONED_GC_OFF; - occFuncAttr.shmemLimitConfig = FUNC_SHMEM_LIMIT_DEFAULT; - occFuncAttr.maxDynamicSharedSizeBytes = 0; - const cudaOccDeviceState occDeviceState = {}; - int blockSize = blockX * blockY * blockZ; - size_t dynamicSmemSize = dynamicSharedMemory; - cudaOccResult occ_result; - cudaOccDeviceProp prop(props[deviceId]); - cudaOccError status = cudaOccMaxActiveBlocksPerMultiprocessor( - &occ_result, &prop, &occFuncAttr, &occDeviceState, - blockSize, dynamicSmemSize); - if (status == CUDA_OCC_SUCCESS) { - if (occ_result.activeBlocksPerMultiprocessor < blocksPerSm) { - blocksPerSm = occ_result.activeBlocksPerMultiprocessor; - } - occupancy = blocksPerSm * blockSize / - (float) props[deviceId].maxThreadsPerMultiProcessor; - } else { - LOG_EVERY_N(ERROR, 1000) << "Failed to calculate occupancy, status = " - << status; - } - } - return occupancy; -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CudaDeviceProperties.h b/plugins/tensorboard-plugins/libkineto/src/CudaDeviceProperties.h deleted file mode 100644 index b731fde0c..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CudaDeviceProperties.h +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (c) Kineto Contributors - * All rights reserved. - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#pragma once - -#include -#include - -namespace KINETO_NAMESPACE { - -int smCount(uint32_t deviceId); - -// Return estimated achieved occupancy for a kernel -float kernelOccupancy( - uint32_t deviceId, - uint16_t registersPerThread, - int32_t staticSharedMemory, - int32_t dynamicSharedMemory, - int32_t blockX, - int32_t blockY, - int32_t blockZ, - float blocks_per_sm); - -// Return compute properties for each device as a json string -const std::string& devicePropertiesJson(); - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivity.h b/plugins/tensorboard-plugins/libkineto/src/CuptiActivity.h deleted file mode 100644 index 09c295040..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivity.h +++ /dev/null @@ -1,114 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -#include "ITraceActivity.h" -#include "CuptiActivityPlatform.h" -#include "ThreadUtil.h" -#include "cupti_strings.h" - -namespace libkineto { - class ActivityLogger; -} - -namespace KINETO_NAMESPACE { - -using namespace libkineto; -struct TraceSpan; - -// These classes wrap the various CUPTI activity types -// into subclasses of ITraceActivity so that they can all be accessed -// using the ITraceActivity interface and logged via ActivityLogger. - -// Abstract base class, templated on Cupti activity type -template -struct CuptiActivity : public ITraceActivity { - explicit CuptiActivity(const T* activity, const ITraceActivity* linked) - : activity_(*activity), linked_(linked) {} - int64_t timestamp() const override { - return nsToUs(unixEpochTimestamp(activity_.start)); - } - int64_t duration() const override { - return nsToUs(activity_.end - activity_.start); - } - // TODO(T107507796): Deprecate ITraceActivity - int64_t correlationId() const override {return 0;} - int32_t getThreadId() const override {return 0;} - const ITraceActivity* linkedActivity() const override {return linked_;} - int flowType() const override {return kLinkAsyncCpuGpu;} - int flowId() const override {return correlationId();} - const T& raw() const {return activity_;} - const TraceSpan* traceSpan() const override {return nullptr;} - - protected: - const T& activity_; - const ITraceActivity* linked_{nullptr}; -}; - -// CUpti_ActivityAPI - CUDA runtime activities -struct RuntimeActivity : public CuptiActivity { - explicit RuntimeActivity( - const CUpti_ActivityAPI* activity, - const ITraceActivity* linked, - int32_t threadId) - : CuptiActivity(activity, linked), threadId_(threadId) {} - int64_t correlationId() const override {return activity_.correlationId;} - int64_t deviceId() const override {return processId();} - int64_t resourceId() const override {return threadId_;} - ActivityType type() const override {return ActivityType::CUDA_RUNTIME;} - bool flowStart() const override; - const std::string name() const override {return runtimeCbidName(activity_.cbid);} - void log(ActivityLogger& logger) const override; - const std::string metadataJson() const override; - - private: - const int32_t threadId_; -}; - -// CUpti_ActivityAPI - CUDA runtime activities -struct OverheadActivity : public CuptiActivity { - explicit OverheadActivity( - const CUpti_ActivityOverhead* activity, - const ITraceActivity* linked, - int32_t threadId=0) - : CuptiActivity(activity, linked), threadId_(threadId) {} - - int64_t timestamp() const override { - return nsToUs(unixEpochTimestamp(activity_.start)); - } - int64_t duration() const override { - return nsToUs(activity_.end - activity_.start); - } - // TODO: Update this with PID ordering - int64_t deviceId() const override {return -1;} - int64_t resourceId() const override {return threadId_;} - ActivityType type() const override {return ActivityType::OVERHEAD;} - bool flowStart() const override; - const std::string name() const override {return overheadKindString(activity_.overheadKind);} - void log(ActivityLogger& logger) const override; - const std::string metadataJson() const override; - - private: - const int32_t threadId_; -}; - -// Base class for GPU activities. -// Can also be instantiated directly. -template -struct GpuActivity : public CuptiActivity { - explicit GpuActivity(const T* activity, const ITraceActivity* linked) - : CuptiActivity(activity, linked) {} - int64_t correlationId() const override {return raw().correlationId;} - int64_t deviceId() const override {return raw().deviceId;} - int64_t resourceId() const override {return raw().streamId;} - ActivityType type() const override; - bool flowStart() const override {return false;} - const std::string name() const override; - void log(ActivityLogger& logger) const override; - const std::string metadataJson() const override; - const T& raw() const {return CuptiActivity::raw();} -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivity.tpp b/plugins/tensorboard-plugins/libkineto/src/CuptiActivity.tpp deleted file mode 100644 index 1ff2dafe0..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivity.tpp +++ /dev/null @@ -1,111 +0,0 @@ - /* - * Copyright (c) Facebook, Inc. and its affiliates. - * All rights reserved. - * This source code is licensed under the BSD-style license found in the - * LICENSE file in the root directory of this source tree. - */ - -#include "CuptiActivity.h" - -#include - -#include "Demangle.h" -#include "output_base.h" - -namespace KINETO_NAMESPACE { - -using namespace libkineto; - -template<> -inline const std::string GpuActivity::name() const { - return demangle(raw().name); -} - -template<> -inline ActivityType GpuActivity::type() const { - return ActivityType::CONCURRENT_KERNEL; -} - -static inline std::string memcpyName(uint8_t kind, uint8_t src, uint8_t dst) { - return fmt::format( - "Memcpy {} ({} -> {})", - memcpyKindString((CUpti_ActivityMemcpyKind)kind), - memoryKindString((CUpti_ActivityMemoryKind)src), - memoryKindString((CUpti_ActivityMemoryKind)dst)); -} - -template<> -inline ActivityType GpuActivity::type() const { - return ActivityType::GPU_MEMCPY; -} - -template<> -inline const std::string GpuActivity::name() const { - return memcpyName(raw().copyKind, raw().srcKind, raw().dstKind); -} - -template<> -inline ActivityType GpuActivity::type() const { - return ActivityType::GPU_MEMCPY; -} - -template<> -inline const std::string GpuActivity::name() const { - return memcpyName(raw().copyKind, raw().srcKind, raw().dstKind); -} - -template<> -inline const std::string GpuActivity::name() const { - const char* memory_kind = - memoryKindString((CUpti_ActivityMemoryKind)raw().memoryKind); - return fmt::format("Memset ({})", memory_kind); -} - -template<> -inline ActivityType GpuActivity::type() const { - return ActivityType::GPU_MEMSET; -} - -inline void RuntimeActivity::log(ActivityLogger& logger) const { - logger.handleActivity(*this); -} - -inline void OverheadActivity::log(ActivityLogger& logger) const { - logger.handleActivity(*this); -} - -inline bool OverheadActivity::flowStart() const { - return false; -} - -inline const std::string OverheadActivity::metadataJson() const { - return ""; -} - -template -inline void GpuActivity::log(ActivityLogger& logger) const { - logger.handleGpuActivity(*this); -} - -inline bool RuntimeActivity::flowStart() const { - return activity_.cbid == CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000 || - (activity_.cbid >= CUPTI_RUNTIME_TRACE_CBID_cudaMemcpy_v3020 && - activity_.cbid <= CUPTI_RUNTIME_TRACE_CBID_cudaMemset2DAsync_v3020) || - activity_.cbid == - CUPTI_RUNTIME_TRACE_CBID_cudaLaunchCooperativeKernel_v9000 || - activity_.cbid == - CUPTI_RUNTIME_TRACE_CBID_cudaLaunchCooperativeKernelMultiDevice_v9000; -} - -inline const std::string RuntimeActivity::metadataJson() const { - return fmt::format(R"JSON( - "cbid": {}, "correlation": {})JSON", - activity_.cbid, activity_.correlationId); -} - -template -inline const std::string GpuActivity::metadataJson() const { - return ""; -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityApi.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiActivityApi.cpp deleted file mode 100644 index 5718bed2f..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityApi.cpp +++ /dev/null @@ -1,343 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "CuptiActivityApi.h" - -#include -#include - -#include "cupti_call.h" -#include "Logger.h" - -using namespace std::chrono; - -namespace KINETO_NAMESPACE { - -// TODO: do we want this to be configurable? -// Set to 2MB to avoid constantly creating buffers (espeically for networks -// that has many small memcpy such as sparseNN) -// Consider putting this on huge pages? -constexpr size_t kBufSize(2 * 1024 * 1024); - -CuptiActivityApi& CuptiActivityApi::singleton() { - static CuptiActivityApi instance; - return instance; -} - -void CuptiActivityApi::pushCorrelationID(int id, CorrelationFlowType type) { -#ifdef HAS_CUPTI - if (!singleton().externalCorrelationEnabled_) { - return; - } - VLOG(2) << "pushCorrelationID(" << id << ")"; - switch(type) { - case Default: - CUPTI_CALL(cuptiActivityPushExternalCorrelationId( - CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM0, id)); - break; - case User: - CUPTI_CALL(cuptiActivityPushExternalCorrelationId( - CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1, id)); - } -#endif -} - -void CuptiActivityApi::popCorrelationID(CorrelationFlowType type) { -#ifdef HAS_CUPTI - if (!singleton().externalCorrelationEnabled_) { - return; - } - switch(type) { - case Default: - CUPTI_CALL(cuptiActivityPopExternalCorrelationId( - CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM0, nullptr)); - break; - case User: - CUPTI_CALL(cuptiActivityPopExternalCorrelationId( - CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1, nullptr)); - } -#endif -} - -static int getSMCount() { -#ifdef HAS_CUPTI - // There may be a simpler way to get the number of SMs.... - // Look for domain_d - this has 80 instances on Volta and - // 56 instances on Pascal, corresponding to the number of SMs - // FIXME: This does not work on Turing and later - uint32_t domainCount{0}; - CUPTI_CALL(cuptiDeviceGetNumEventDomains(0, &domainCount)); - std::vector ids(domainCount); - size_t sz = sizeof(CUpti_EventDomainID) * domainCount; - CUPTI_CALL(cuptiDeviceEnumEventDomains(0, &sz, ids.data())); - for (CUpti_EventDomainID id : ids) { - char name[16]; - name[0] = '\0'; - sz = sizeof(name); - CUPTI_CALL(cuptiEventDomainGetAttribute( - id, CUPTI_EVENT_DOMAIN_ATTR_NAME, &sz, name)); - if (strncmp(name, "domain_d", sz) == 0) { - uint32_t count{0}; - sz = sizeof(count); - CUPTI_CALL(cuptiDeviceGetEventDomainAttribute( - 0, id, CUPTI_EVENT_DOMAIN_ATTR_TOTAL_INSTANCE_COUNT, &sz, &count)); - return count; - } - } -#endif - - return -1; -} - -int CuptiActivityApi::smCount() { - static int sm_count = getSMCount(); - return sm_count; -} - -static bool nextActivityRecord( - uint8_t* buffer, - size_t valid_size, - CUpti_Activity*& record) { -#ifdef HAS_CUPTI - CUptiResult status = CUPTI_CALL_NOWARN( - cuptiActivityGetNextRecord(buffer, valid_size, &record)); - if (status != CUPTI_SUCCESS) { - if (status != CUPTI_ERROR_MAX_LIMIT_REACHED) { - CUPTI_CALL(status); - } - record = nullptr; - } -#endif - return record != nullptr; -} - -void CuptiActivityApi::setMaxBufferSize(int size) { - maxGpuBufferCount_ = 1 + size / kBufSize; -} - -void CuptiActivityApi::forceLoadCupti() { -#ifdef HAS_CUPTI - CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL)); -#endif -} - -#ifdef HAS_CUPTI -void CUPTIAPI CuptiActivityApi::bufferRequestedTrampoline( - uint8_t** buffer, - size_t* size, - size_t* maxNumRecords) { - singleton().bufferRequested(buffer, size, maxNumRecords); -} - -void CuptiActivityApi::bufferRequested( - uint8_t** buffer, size_t* size, size_t* maxNumRecords) { - std::lock_guard guard(mutex_); - if (allocatedGpuTraceBuffers_.size() >= maxGpuBufferCount_) { - stopCollection = true; - LOG(WARNING) << "Exceeded max GPU buffer count (" - << allocatedGpuTraceBuffers_.size() - << " > " << maxGpuBufferCount_ - << ") - terminating tracing"; - } - - auto buf = std::make_unique(kBufSize); - *buffer = buf->data(); - *size = kBufSize; - - allocatedGpuTraceBuffers_[*buffer] = std::move(buf); - - *maxNumRecords = 0; -} -#endif - -std::unique_ptr -CuptiActivityApi::activityBuffers() { - { - std::lock_guard guard(mutex_); - if (allocatedGpuTraceBuffers_.empty()) { - return nullptr; - } - } - -#ifdef HAS_CUPTI - VLOG(1) << "Flushing GPU activity buffers"; - time_point t1; - if (VLOG_IS_ON(1)) { - t1 = system_clock::now(); - } - // Can't hold mutex_ during this call, since bufferCompleted - // will be called by libcupti and mutex_ is acquired there. - CUPTI_CALL(cuptiActivityFlushAll(CUPTI_ACTIVITY_FLAG_FLUSH_FORCED)); - if (VLOG_IS_ON(1)) { - flushOverhead = - duration_cast(system_clock::now() - t1).count(); - } -#endif - std::lock_guard guard(mutex_); - // Transfer ownership of buffers to caller. A new map is created on-demand. - return std::move(readyGpuTraceBuffers_); -} - -#ifdef HAS_CUPTI -int CuptiActivityApi::processActivitiesForBuffer( - uint8_t* buf, - size_t validSize, - std::function handler) { - int count = 0; - if (buf && validSize) { - CUpti_Activity* record{nullptr}; - while ((nextActivityRecord(buf, validSize, record))) { - handler(record); - ++count; - } - } - return count; -} -#endif - -const std::pair CuptiActivityApi::processActivities( - CuptiActivityBufferMap& buffers, - std::function handler) { - std::pair res{0, 0}; -#ifdef HAS_CUPTI - for (auto& pair : buffers) { - // No lock needed - only accessed from this thread - auto& buf = pair.second; - res.first += processActivitiesForBuffer(buf->data(), buf->size(), handler); - res.second += buf->size(); - } -#endif - return res; -} - -void CuptiActivityApi::clearActivities() { - { - std::lock_guard guard(mutex_); - if (allocatedGpuTraceBuffers_.empty()) { - return; - } - } - // Can't hold mutex_ during this call, since bufferCompleted - // will be called by libcupti and mutex_ is acquired there. -#ifdef HAS_CUPTI - CUPTI_CALL(cuptiActivityFlushAll(0)); -#endif - // FIXME: We might want to make sure we reuse - // the same memory during warmup and tracing. - // Also, try to use the amount of memory required - // for active tracing during warmup. - std::lock_guard guard(mutex_); - // Throw away ready buffers as a result of above flush - readyGpuTraceBuffers_ = nullptr; -} - -#ifdef HAS_CUPTI -void CUPTIAPI CuptiActivityApi::bufferCompletedTrampoline( - CUcontext ctx, - uint32_t streamId, - uint8_t* buffer, - size_t /* unused */, - size_t validSize) { - singleton().bufferCompleted(ctx, streamId, buffer, 0, validSize); -} - -void CuptiActivityApi::bufferCompleted( - CUcontext ctx, - uint32_t streamId, - uint8_t* buffer, - size_t /* unused */, - size_t validSize) { - - std::lock_guard guard(mutex_); - auto it = allocatedGpuTraceBuffers_.find(buffer); - if (it == allocatedGpuTraceBuffers_.end()) { - LOG(ERROR) << "bufferCompleted called with unknown buffer: " - << (void*) buffer; - return; - } - - if (!readyGpuTraceBuffers_) { - readyGpuTraceBuffers_ = std::make_unique(); - } - // Set valid size of buffer before moving to ready map - it->second->setSize(validSize); - (*readyGpuTraceBuffers_)[it->first] = std::move(it->second); - allocatedGpuTraceBuffers_.erase(it); - - // report any records dropped from the queue; to avoid unnecessary cupti - // API calls, we make it report only in verbose mode (it doesn't happen - // often in our testing anyways) - if (VLOG_IS_ON(1)) { - size_t dropped = 0; - CUPTI_CALL(cuptiActivityGetNumDroppedRecords(ctx, streamId, &dropped)); - if (dropped != 0) { - LOG(WARNING) << "Dropped " << dropped << " activity records"; - } - } -} -#endif - -void CuptiActivityApi::enableCuptiActivities( - const std::set& selected_activities) { -#ifdef HAS_CUPTI - static bool registered = false; - if (!registered) { - CUPTI_CALL( - cuptiActivityRegisterCallbacks(bufferRequestedTrampoline, bufferCompletedTrampoline)); - } - - externalCorrelationEnabled_ = false; - for (const auto& activity : selected_activities) { - if (activity == ActivityType::GPU_MEMCPY) { - CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMCPY)); - } - if (activity == ActivityType::GPU_MEMSET) { - CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_MEMSET)); - } - if (activity == ActivityType::CONCURRENT_KERNEL) { - CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL)); - } - if (activity == ActivityType::EXTERNAL_CORRELATION) { - CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION)); - externalCorrelationEnabled_ = true; - } - if (activity == ActivityType::CUDA_RUNTIME) { - CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_RUNTIME)); - } - if (activity == ActivityType::OVERHEAD) { - CUPTI_CALL(cuptiActivityEnable(CUPTI_ACTIVITY_KIND_OVERHEAD)); - } - } -#endif - - // Explicitly enabled, so reset this flag if set - stopCollection = false; -} - -void CuptiActivityApi::disableCuptiActivities( - const std::set& selected_activities) { -#ifdef HAS_CUPTI - for (const auto& activity : selected_activities) { - if (activity == ActivityType::GPU_MEMCPY) { - CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMCPY)); - } - if (activity == ActivityType::GPU_MEMSET) { - CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_MEMSET)); - } - if (activity == ActivityType::CONCURRENT_KERNEL) { - CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL)); - } - if (activity == ActivityType::EXTERNAL_CORRELATION) { - CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION)); - } - if (activity == ActivityType::CUDA_RUNTIME) { - CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_RUNTIME)); - } - if (activity == ActivityType::OVERHEAD) { - CUPTI_CALL(cuptiActivityDisable(CUPTI_ACTIVITY_KIND_OVERHEAD)); - } - } - externalCorrelationEnabled_ = false; -#endif -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityApi.h b/plugins/tensorboard-plugins/libkineto/src/CuptiActivityApi.h deleted file mode 100644 index 92af51eca..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityApi.h +++ /dev/null @@ -1,100 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include -#include - -#ifdef HAS_CUPTI -#include -#endif - -#include "ActivityType.h" -#include "CuptiActivityBuffer.h" - - -namespace KINETO_NAMESPACE { - -using namespace libkineto; - -#ifndef HAS_CUPTI -using CUpti_Activity = void; -#endif - -class CuptiActivityApi { - public: - enum CorrelationFlowType { - Default, - User - }; - - CuptiActivityApi() = default; - CuptiActivityApi(const CuptiActivityApi&) = delete; - CuptiActivityApi& operator=(const CuptiActivityApi&) = delete; - - virtual ~CuptiActivityApi() {} - - static CuptiActivityApi& singleton(); - - virtual int smCount(); - static void pushCorrelationID(int id, CorrelationFlowType type); - static void popCorrelationID(CorrelationFlowType type); - - void enableCuptiActivities( - const std::set& selected_activities); - void disableCuptiActivities( - const std::set& selected_activities); - void clearActivities(); - - virtual std::unique_ptr activityBuffers(); - - virtual const std::pair processActivities( - CuptiActivityBufferMap&, - std::function handler); - - void setMaxBufferSize(int size); - - std::atomic_bool stopCollection{false}; - int64_t flushOverhead{0}; - - static void forceLoadCupti(); - - private: -#ifdef HAS_CUPTI - int processActivitiesForBuffer( - uint8_t* buf, - size_t validSize, - std::function handler); - static void CUPTIAPI - bufferRequestedTrampoline(uint8_t** buffer, size_t* size, size_t* maxNumRecords); - static void CUPTIAPI bufferCompletedTrampoline( - CUcontext ctx, - uint32_t streamId, - uint8_t* buffer, - size_t /* unused */, - size_t validSize); -#endif // HAS_CUPTI - - int maxGpuBufferCount_{0}; - CuptiActivityBufferMap allocatedGpuTraceBuffers_; - std::unique_ptr readyGpuTraceBuffers_; - std::mutex mutex_; - bool externalCorrelationEnabled_{false}; - - protected: -#ifdef HAS_CUPTI - void bufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords); - void bufferCompleted( - CUcontext ctx, - uint32_t streamId, - uint8_t* buffer, - size_t /* unused */, - size_t validSize); -#endif -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityBuffer.h b/plugins/tensorboard-plugins/libkineto/src/CuptiActivityBuffer.h deleted file mode 100644 index 1c3fbef62..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityBuffer.h +++ /dev/null @@ -1,51 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -#include "ITraceActivity.h" - -namespace KINETO_NAMESPACE { - -class CuptiActivityBuffer { - public: - explicit CuptiActivityBuffer(size_t size) : size_(size) { - buf_.reserve(size); - } - CuptiActivityBuffer() = delete; - CuptiActivityBuffer& operator=(const CuptiActivityBuffer&) = delete; - CuptiActivityBuffer(CuptiActivityBuffer&&) = default; - CuptiActivityBuffer& operator=(CuptiActivityBuffer&&) = default; - - size_t size() const { - return size_; - } - - void setSize(size_t size) { - assert(size <= buf_.capacity()); - size_ = size; - } - - uint8_t* data() { - return buf_.data(); - } - - private: - - std::vector buf_; - size_t size_; - - std::vector> wrappers_; -}; - -using CuptiActivityBufferMap = - std::map>; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityPlatform.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiActivityPlatform.cpp deleted file mode 100644 index fa2ef2f3a..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityPlatform.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include - -namespace chrono = std::chrono; - -namespace KINETO_NAMESPACE { - -#ifdef _WIN32 -uint64_t epochs_diff() { - // On Windows, steady_clock wraps the QueryPerformanceCounter function. - // https://docs.microsoft.com/en-us/cpp/standard-library/steady-clock-struct?view=msvc-160 - auto steady = - chrono::time_point_cast(chrono::steady_clock::now()); - auto system = - chrono::time_point_cast(chrono::system_clock::now()); - - auto time_since_unix = system.time_since_epoch().count(); - auto time_since_boot = steady.time_since_epoch().count(); - return time_since_unix - time_since_boot; -} - -uint64_t unixEpochTimestamp(uint64_t ts) { - static uint64_t diff = epochs_diff(); - return ts + diff; -} -#else -uint64_t unixEpochTimestamp(uint64_t ts) { - return ts; -} -#endif // _WIN32 - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityPlatform.h b/plugins/tensorboard-plugins/libkineto/src/CuptiActivityPlatform.h deleted file mode 100644 index 78de8373d..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityPlatform.h +++ /dev/null @@ -1,12 +0,0 @@ -#pragma once - -#include - -namespace KINETO_NAMESPACE { - -// cupti's timestamps are platform specific. This function convert the raw -// cupti timestamp to time since unix epoch. So that on different platform, -// correction can work correctly. -uint64_t unixEpochTimestamp(uint64_t ts); - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityProfiler.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiActivityProfiler.cpp deleted file mode 100644 index 97c23ef04..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityProfiler.cpp +++ /dev/null @@ -1,841 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "CuptiActivityProfiler.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAS_CUPTI -#include -#endif - -#include "Config.h" -#include "time_since_epoch.h" -#ifdef HAS_CUPTI -#include "CuptiActivity.h" -#include "CuptiActivity.tpp" -#include "CuptiActivityApi.h" -#endif // HAS_CUPTI -#ifdef HAS_ROCTRACER -#include "RoctracerActivityApi.h" -#endif -#include "output_base.h" - -#include "Logger.h" -#include "ThreadUtil.h" - -using namespace std::chrono; -using namespace libkineto; -using std::string; - -namespace KINETO_NAMESPACE { - -void CuptiActivityProfiler::transferCpuTrace( - std::unique_ptr cpuTrace) { - std::lock_guard guard(mutex_); - const string& trace_name = cpuTrace->span.name; - if (currentRunloopState_ != RunloopState::CollectTrace && - currentRunloopState_ != RunloopState::ProcessTrace) { - VLOG(0) << "Trace collection not in progress - discarding span " - << trace_name; - return; - } - - cpuTrace->span.iteration = iterationCountMap_[trace_name]++; - - VLOG(0) << "Received iteration " << cpuTrace->span.iteration << " of span " - << trace_name << " (" << cpuTrace->activities.size() << " activities / " - << cpuTrace->gpuOpCount << " gpu activities)"; - traceBuffers_->cpu.push_back(std::move(cpuTrace)); -} - -#ifdef HAS_ROCTRACER -CuptiActivityProfiler::CuptiActivityProfiler(RoctracerActivityApi& cupti, bool cpuOnly) -#else -CuptiActivityProfiler::CuptiActivityProfiler(CuptiActivityApi& cupti, bool cpuOnly) -#endif - : cupti_(cupti), - flushOverhead_{0, 0}, - setupOverhead_{0, 0}, - cpuOnly_{cpuOnly}, - currentRunloopState_{RunloopState::WaitForRequest}, - stopCollection_{false} {} - -void CuptiActivityProfiler::processTraceInternal(ActivityLogger& logger) { - LOG(INFO) << "Processing " << traceBuffers_->cpu.size() - << " CPU buffers"; - VLOG(0) << "Profile time range: " << captureWindowStartTime_ << " - " - << captureWindowEndTime_; - logger.handleTraceStart(metadata_); - for (auto& cpu_trace : traceBuffers_->cpu) { - string trace_name = cpu_trace->span.name; - VLOG(0) << "Processing CPU buffer for " << trace_name << " (" - << cpu_trace->span.iteration << ") - " - << cpu_trace->activities.size() << " records"; - VLOG(0) << "Span time range: " << cpu_trace->span.startTime << " - " - << cpu_trace->span.endTime; - processCpuTrace(*cpu_trace, logger); - LOGGER_OBSERVER_ADD_EVENT_COUNT(cpu_trace->activities.size()); - } - -#ifdef HAS_CUPTI - if (!cpuOnly_) { - VLOG(0) << "Retrieving GPU activity buffers"; - traceBuffers_->gpu = cupti_.activityBuffers(); - if (VLOG_IS_ON(1)) { - addOverheadSample(flushOverhead_, cupti_.flushOverhead); - } - if (traceBuffers_->gpu) { - const auto count_and_size = cupti_.processActivities( - *traceBuffers_->gpu, - std::bind(&CuptiActivityProfiler::handleCuptiActivity, this, std::placeholders::_1, &logger)); - LOG(INFO) << "Processed " << count_and_size.first - << " GPU records (" << count_and_size.second << " bytes)"; - LOGGER_OBSERVER_ADD_EVENT_COUNT(count_and_size.first); - } - } -#endif // HAS_CUPTI -#ifdef HAS_ROCTRACER - if (!cpuOnly_) { - VLOG(0) << "Retrieving GPU activity buffers"; - const int count = cupti_.processActivities(logger); - LOG(INFO) << "Processed " << count - << " GPU records"; - LOGGER_OBSERVER_ADD_EVENT_COUNT(count); - } -#endif // HAS_ROCTRACER - - for (const auto& session : sessions_){ - LOG(INFO) << "Processing child profiler trace"; - session->processTrace(logger); - } - - finalizeTrace(*config_, logger); -} - -CuptiActivityProfiler::CpuGpuSpanPair& CuptiActivityProfiler::recordTraceSpan( - TraceSpan& span, int gpuOpCount) { - TraceSpan gpu_span(gpuOpCount, span.iteration, span.name, "GPU: "); - auto& iterations = traceSpans_[span.name]; - iterations.push_back({span, gpu_span}); - return iterations.back(); -} - -void CuptiActivityProfiler::processCpuTrace( - libkineto::CpuTraceBuffer& cpuTrace, - ActivityLogger& logger) { - if (cpuTrace.activities.size() == 0) { - LOG(WARNING) << "CPU trace is empty!"; - return; - } - - CpuGpuSpanPair& span_pair = recordTraceSpan(cpuTrace.span, cpuTrace.gpuOpCount); - TraceSpan& cpu_span = span_pair.first; - for (auto const& act : cpuTrace.activities) { - VLOG(2) << act.correlationId() << ": OP " << act.activityName; - if (config_->selectedActivityTypes().count(act.type())) { - act.log(logger); - } - clientActivityTraceMap_[act.correlationId()] = &span_pair; - activityMap_[act.correlationId()] = &act; - - recordThreadInfo(act.resourceId(), act.getThreadId(), act.deviceId()); - } - logger.handleTraceSpan(cpu_span); -} - -#ifdef HAS_CUPTI -inline void CuptiActivityProfiler::handleCorrelationActivity( - const CUpti_ActivityExternalCorrelation* correlation) { - if (correlation->externalKind == CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM0) { - cpuCorrelationMap_[correlation->correlationId] = correlation->externalId; - } else if (correlation->externalKind == CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1){ - userCorrelationMap_[correlation->correlationId] = correlation->externalId; - } else { - LOG(ERROR) << "Invalid CUpti_ActivityExternalCorrelation sent to handleCuptiActivity"; - } -} -#endif // HAS_CUPTI - -static GenericTraceActivity createUserGpuSpan( - const libkineto::ITraceActivity& cpuTraceActivity, - const libkineto::ITraceActivity& gpuTraceActivity) { - GenericTraceActivity res( - *cpuTraceActivity.traceSpan(), - ActivityType::GPU_USER_ANNOTATION, - cpuTraceActivity.name()); - res.startTime = gpuTraceActivity.timestamp(); - res.device = gpuTraceActivity.deviceId(); - res.resource = gpuTraceActivity.resourceId(); - res.endTime = - gpuTraceActivity.timestamp() + gpuTraceActivity.duration(); - res.id = cpuTraceActivity.correlationId(); - return res; -} - -void CuptiActivityProfiler::GpuUserEventMap::insertOrExtendEvent( - const ITraceActivity& userActivity, - const ITraceActivity& gpuActivity) { - StreamKey key(gpuActivity.deviceId(), gpuActivity.resourceId()); - CorrelationSpanMap& correlationSpanMap = streamSpanMap_[key]; - auto it = correlationSpanMap.find(userActivity.correlationId()); - if (it == correlationSpanMap.end()) { - auto it_success = correlationSpanMap.insert({ - userActivity.correlationId(), createUserGpuSpan(userActivity, gpuActivity) - }); - it = it_success.first; - } - GenericTraceActivity& span = it->second; - if (gpuActivity.timestamp() < span.startTime || span.startTime == 0) { - span.startTime = gpuActivity.timestamp(); - } - int64_t gpu_activity_end = gpuActivity.timestamp() + gpuActivity.duration(); - if (gpu_activity_end > span.endTime) { - span.endTime = gpu_activity_end; - } -} - -const CuptiActivityProfiler::CpuGpuSpanPair& CuptiActivityProfiler::defaultTraceSpan() { - static TraceSpan span(0, 0, "Unknown", ""); - static CpuGpuSpanPair span_pair(span, span); - return span_pair; -} - -void CuptiActivityProfiler::GpuUserEventMap::logEvents(ActivityLogger *logger) { - for (auto const& streamMapPair : streamSpanMap_) { - for (auto const& correlationSpanPair : streamMapPair.second) { - correlationSpanPair.second.log(*logger); - } - } -} - -#ifdef HAS_CUPTI -inline bool CuptiActivityProfiler::outOfRange(const ITraceActivity& act) { - bool out_of_range = act.timestamp() < captureWindowStartTime_ || - (act.timestamp() + act.duration()) > captureWindowEndTime_; - if (out_of_range) { - VLOG(2) << "TraceActivity outside of profiling window: " << act.name() - << " (" << act.timestamp() << " < " << captureWindowStartTime_ << " or " - << (act.timestamp() + act.duration()) << " > " << captureWindowEndTime_; - } - return out_of_range; -} - -inline static bool isBlockListedRuntimeCbid(CUpti_CallbackId cbid) { - // Some CUDA calls that are very frequent and also not very interesting. - // Filter these out to reduce trace size. - if (cbid == CUPTI_RUNTIME_TRACE_CBID_cudaGetDevice_v3020 || - cbid == CUPTI_RUNTIME_TRACE_CBID_cudaSetDevice_v3020 || - cbid == CUPTI_RUNTIME_TRACE_CBID_cudaGetLastError_v3020 || - // Don't care about cudaEvents - cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventCreate_v3020 || - cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventCreateWithFlags_v3020 || - cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventRecord_v3020 || - cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventDestroy_v3020 || - cbid == CUPTI_RUNTIME_TRACE_CBID_cudaEventSynchronize_v3020) { - return true; - } - - return false; -} - -void CuptiActivityProfiler::handleRuntimeActivity( - const CUpti_ActivityAPI* activity, - ActivityLogger* logger) { - if (isBlockListedRuntimeCbid(activity->cbid)) { - return; - } - VLOG(2) << activity->correlationId - << ": CUPTI_ACTIVITY_KIND_RUNTIME, cbid=" << activity->cbid - << " tid=" << activity->threadId; - int32_t tid = activity->threadId; - const auto& it = resourceInfo_.find({processId(), tid}); - if (it != resourceInfo_.end()) { - tid = it->second.id; - } - const ITraceActivity* linked = linkedActivity( - activity->correlationId, cpuCorrelationMap_); - const auto& runtime_activity = - traceBuffers_->addActivityWrapper(RuntimeActivity(activity, linked, tid)); - checkTimestampOrder(&runtime_activity); - if (outOfRange(runtime_activity)) { - return; - } - runtime_activity.log(*logger); -} - -void CuptiActivityProfiler::handleOverheadActivity( - const CUpti_ActivityOverhead* activity, - ActivityLogger* logger) { - VLOG(2) << ": CUPTI_ACTIVITY_KIND_OVERHEAD" << " overheadKind=" << activity->overheadKind; - - const auto& overhead_activity = - traceBuffers_->addActivityWrapper(OverheadActivity(activity, nullptr)); - overhead_activity.log(*logger); -} - - -inline void CuptiActivityProfiler::updateGpuNetSpan( - const ITraceActivity& gpuOp) { - if (!gpuOp.linkedActivity()) { - VLOG(0) << "Missing linked activity"; - return; - } - const auto& it = clientActivityTraceMap_.find( - gpuOp.linkedActivity()->correlationId()); - if (it == clientActivityTraceMap_.end()) { - // No correlation id mapping? - return; - } - TraceSpan& gpu_span = it->second->second; - if (gpuOp.timestamp() < gpu_span.startTime || gpu_span.startTime == 0) { - gpu_span.startTime = gpuOp.timestamp(); - } - if ((gpuOp.timestamp() + gpuOp.duration()) > gpu_span.endTime) { - gpu_span.endTime = gpuOp.timestamp() + gpuOp.duration(); - } -} - -// I've observed occasional broken timestamps attached to GPU events... -void CuptiActivityProfiler::checkTimestampOrder(const ITraceActivity* act1) { - // Correlated GPU runtime activity cannot - // have timestamp greater than the GPU activity's - const auto& it = correlatedCudaActivities_.find(act1->correlationId()); - if (it == correlatedCudaActivities_.end()) { - correlatedCudaActivities_.insert({act1->correlationId(), act1}); - return; - } - - // Activities may be appear in the buffers out of order. - // If we have a runtime activity in the map, it should mean that we - // have a GPU activity passed in, and vice versa. - const ITraceActivity* act2 = it->second; - if (act2->type() == ActivityType::CUDA_RUNTIME) { - // Buffer is out-of-order. - // Swap so that runtime activity is first for the comparison below. - std::swap(act1, act2); - } - if (act1->timestamp() > act2->timestamp()) { - LOG(WARNING) << "GPU op timestamp (" << act2->timestamp() - << ") < runtime timestamp (" << act1->timestamp() << ") by " - << act1->timestamp() - act2->timestamp() << "us"; - LOG(WARNING) << "Name: " << act2->name() - << " Device: " << act2->deviceId() - << " Stream: " << act2->resourceId(); - } -} - -inline void CuptiActivityProfiler::handleGpuActivity( - const ITraceActivity& act, - ActivityLogger* logger) { - if (outOfRange(act)) { - return; - } - checkTimestampOrder(&act); - VLOG(2) << act.correlationId() << ": " - << act.name(); - recordStream(act.deviceId(), act.resourceId(), ""); - act.log(*logger); - updateGpuNetSpan(act); - if (config_->selectedActivityTypes().count(ActivityType::GPU_USER_ANNOTATION)) { - const auto& it = userCorrelationMap_.find(act.correlationId()); - if (it != userCorrelationMap_.end()) { - const auto& it2 = activityMap_.find(it->second); - if (it2 != activityMap_.end()) { - recordStream(act.deviceId(), act.resourceId(), "context"); - gpuUserEventMap_.insertOrExtendEvent(*it2->second, act); - } - } - } -} - -const ITraceActivity* CuptiActivityProfiler::linkedActivity( - int32_t correlationId, - const std::unordered_map& correlationMap) { - const auto& it = correlationMap.find(correlationId); - if (it != correlationMap.end()) { - const auto& it2 = activityMap_.find(it->second); - if (it2 != activityMap_.end()) { - return it2->second; - } - } - return nullptr; -} - -template -inline void CuptiActivityProfiler::handleGpuActivity( - const T* act, ActivityLogger* logger) { - const ITraceActivity* linked = linkedActivity( - act->correlationId, cpuCorrelationMap_); - const auto& gpu_activity = - traceBuffers_->addActivityWrapper(GpuActivity(act, linked)); - handleGpuActivity(gpu_activity, logger); -} - -void CuptiActivityProfiler::handleCuptiActivity(const CUpti_Activity* record, ActivityLogger* logger) { - switch (record->kind) { - case CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION: - handleCorrelationActivity( - reinterpret_cast( - record)); - break; - case CUPTI_ACTIVITY_KIND_RUNTIME: - handleRuntimeActivity( - reinterpret_cast(record), logger); - break; - case CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL: - handleGpuActivity( - reinterpret_cast(record), logger); - break; - case CUPTI_ACTIVITY_KIND_MEMCPY: - handleGpuActivity( - reinterpret_cast(record), logger); - break; - case CUPTI_ACTIVITY_KIND_MEMCPY2: - handleGpuActivity( - reinterpret_cast(record), logger); - break; - case CUPTI_ACTIVITY_KIND_MEMSET: - handleGpuActivity( - reinterpret_cast(record), logger); - break; - case CUPTI_ACTIVITY_KIND_OVERHEAD: - handleOverheadActivity (reinterpret_cast(record), logger); - break; - default: - LOG(WARNING) << "Unexpected activity type: " << record->kind; - break; - } -} -#endif // HAS_CUPTI - -void CuptiActivityProfiler::configureChildProfilers() { - // If child profilers are enabled create profiler sessions - for (auto& profiler: profilers_) { - int64_t start_time_ms = duration_cast( - profileStartTime_.time_since_epoch()).count(); - LOG(INFO) << "Running child profiler " << profiler->name() << " for " - << config_->activitiesDuration().count() << " ms"; - auto session = profiler->configure( - start_time_ms, - config_->activitiesDuration().count(), - config_->selectedActivityTypes(), - *config_ - ); - if (session) { - sessions_.push_back(std::move(session)); - } - } -} - -void CuptiActivityProfiler::configure( - const Config& config, - const time_point& now) { - std::lock_guard guard(mutex_); - if (isActive()) { - LOG(ERROR) << "CuptiActivityProfiler already busy, terminating"; - return; - } - - config_ = config.clone(); - - if (config_->activitiesDuration().count() == 0) { - // Use default if not specified - config_->setActivitiesDuration( - config_->activitiesDurationDefault()); - } - - // Ensure we're starting in a clean state - resetTraceData(); - -#if !USE_GOOGLE_LOG - // Add a LoggerObserverCollector to collect all logs during the trace. - loggerCollectorMetadata_ = std::make_unique(); - Logger::addLoggerObserver(loggerCollectorMetadata_.get()); -#endif // !USE_GOOGLE_LOG - - profileStartTime_ = config_->requestTimestamp(); - - if (config_->hasProfileStartIteration()) { - profileStartIter_ = config_->profileStartIteration(); - profileEndIter_ = profileStartIter_ + config_->activitiesRunIterations(); - } else { - - profileStartIter_ = -1; - profileEndIter_ = (std::numeric_limits::max)(); - - if (profileStartTime_ < now) { - LOG(ERROR) << "Not starting tracing - start timestamp is in the past. Time difference (ms): " << duration_cast(now - profileStartTime_).count(); - return; - } else if ((profileStartTime_ - now) < config_->activitiesWarmupDuration()) { - LOG(ERROR) << "Not starting tracing - insufficient time for warmup. Time to warmup (ms): " << duration_cast(profileStartTime_ - now).count() ; - return; - } - } - - if (LOG_IS_ON(INFO)) { - config_->printActivityProfilerConfig(LIBKINETO_DBG_STREAM); - } - if (!cpuOnly_ && !libkineto::api().client()) { - if (profileStartIter_ < 0) { - LOG(INFO) << "GPU-only tracing for " - << config_->activitiesDuration().count() << "ms"; - } else { - LOG(INFO) << "GPU-only tracing for " - << config_->activitiesRunIterations() << " iterations"; - } - } - - // Set useful metadata into the logger. - LOGGER_OBSERVER_SET_TRACE_DURATION_MS(config_->activitiesDuration().count()); - if (!config_->requestTraceID().empty()) { - LOGGER_OBSERVER_SET_TRACE_ID(config_->requestTraceID()); - } - if (!config_->requestGroupTraceID().empty()) { - LOGGER_OBSERVER_SET_GROUP_TRACE_ID(config_->requestGroupTraceID()); - } - LOGGER_OBSERVER_ADD_DESTINATION(config_->activitiesLogUrl()); - -#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) - if (!cpuOnly_) { - // Enabling CUPTI activity tracing incurs a larger perf hit at first, - // presumably because structures are allocated and initialized, callbacks - // are activated etc. After a while the overhead decreases and stabilizes. - // It's therefore useful to perform some warmup before starting recording. - LOG(INFO) << "Enabling GPU tracing"; - cupti_.setMaxBufferSize(config_->activitiesMaxGpuBufferSize()); - - time_point timestamp; - if (VLOG_IS_ON(1)) { - timestamp = system_clock::now(); - } -#ifdef HAS_CUPTI - cupti_.enableCuptiActivities(config_->selectedActivityTypes()); -#else - cupti_.enableActivities(config_->selectedActivityTypes()); -#endif - if (VLOG_IS_ON(1)) { - auto t2 = system_clock::now(); - addOverheadSample( - setupOverhead_, duration_cast(t2 - timestamp).count()); - } - } -#endif // HAS_CUPTI || HAS_ROCTRACER - - if (profilers_.size() > 0) { - configureChildProfilers(); - } - - if (libkineto::api().client()) { - libkineto::api().client()->warmup(config_->isOpInputsCollectionEnabled()); - } - if (profileStartIter_ >= 0) { - LOG(INFO) << "Tracing starting on iteration = " << profileStartIter_; - } else { - LOG(INFO) << "Tracing starting in " - << duration_cast(profileStartTime_ - now).count() << "s"; - } - - traceBuffers_ = std::make_unique(); - captureWindowStartTime_ = captureWindowEndTime_ = 0; - currentRunloopState_ = RunloopState::Warmup; -} - -void CuptiActivityProfiler::startTraceInternal(const time_point& now) { - captureWindowStartTime_ = libkineto::timeSinceEpoch(now); - VLOG(0) << "Warmup -> CollectTrace"; - for (auto& session: sessions_){ - LOG(INFO) << "Starting child profiler session"; - session->start(); - } - currentRunloopState_ = RunloopState::CollectTrace; -} - -void CuptiActivityProfiler::stopTraceInternal(const time_point& now) { - if (captureWindowEndTime_ == 0) { - captureWindowEndTime_ = libkineto::timeSinceEpoch(now); - } -#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) - if (!cpuOnly_) { - time_point timestamp; - if (VLOG_IS_ON(1)) { - timestamp = system_clock::now(); - } -#ifdef HAS_CUPTI - cupti_.disableCuptiActivities(config_->selectedActivityTypes()); -#else - cupti_.disableActivities(config_->selectedActivityTypes()); -#endif - if (VLOG_IS_ON(1)) { - auto t2 = system_clock::now(); - addOverheadSample( - setupOverhead_, duration_cast(t2 - timestamp).count()); - } - } -#endif // HAS_CUPTI || HAS_ROCTRACER - - if (currentRunloopState_ == RunloopState::CollectTrace) { - VLOG(0) << "CollectTrace -> ProcessTrace"; - } else { - LOG(WARNING) << "Called stopTrace with state == " << - static_cast::type>( - currentRunloopState_.load()); - } - for (auto& session: sessions_){ - LOG(INFO) << "Stopping child profiler session"; - session->stop(); - } - currentRunloopState_ = RunloopState::ProcessTrace; -} - -void CuptiActivityProfiler::resetInternal() { - resetTraceData(); - currentRunloopState_ = RunloopState::WaitForRequest; -} - -bool CuptiActivityProfiler::isWarmupDone( - const time_point& now, - int64_t currentIter) const { - // is it a time based config - if (profileStartIter_ < 0) { - // qualify that this check is not being called from application step() API - // this avoids races between the step() API and periodically invoked - // profiler run loop step() method - return (currentIter < 0) && (now >= profileStartTime_); - } - // this is an iteration based config - if (currentIter < 0) { - return false; - } - return currentIter >= profileStartIter_; -} - -bool CuptiActivityProfiler::isCollectionDone( - const time_point& now, - int64_t currentIter) const { - // is it a time based config - if (profileStartIter_ < 0) { - // qualify that this check is not being called from application step() API - return (currentIter < 0) && (now >= profileEndTime_); - } - // this is an iteration based config - if (currentIter < 0) { - return false; - } - return currentIter >= profileEndIter_; -} - -const time_point CuptiActivityProfiler::performRunLoopStep( - const time_point& now, - const time_point& nextWakeupTime, - int64_t currentIter) { - auto new_wakeup_time = nextWakeupTime; - bool warmup_done = false, collection_done = false; - - VLOG_IF(1, currentIter >= 0) << "Run loop on application step(), iteration = " - << currentIter; - - switch (currentRunloopState_) { - case RunloopState::WaitForRequest: - VLOG(1) << "State: WaitForRequest"; - // Nothing to do - break; - - case RunloopState::Warmup: - VLOG(1) << "State: Warmup"; - warmup_done = isWarmupDone(now, currentIter); -#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) - // Flushing can take a while so avoid doing it close to the start time - if (!cpuOnly_ && currentIter < 0 && - (profileStartIter_ >= 0 || nextWakeupTime < profileStartTime_)) { - cupti_.clearActivities(); - } - - if (cupti_.stopCollection) { - // Go to process trace to clear any outstanding buffers etc - LOG(WARNING) << "Trace terminated during warmup"; - std::lock_guard guard(mutex_); - stopTraceInternal(now); - resetInternal(); - VLOG(0) << "Warmup -> WaitForRequest"; - break; - } -#endif // HAS_CUPTI || HAS_ROCTRACER - - if (warmup_done) { - UST_LOGGER_MARK_COMPLETED(kWarmUpStage); - if (profileStartIter_ < 0 && - (now > profileStartTime_ + milliseconds(10))) { - LOG(WARNING) - << "Tracing started " - << duration_cast(now - profileStartTime_).count() - << "ms late!"; - } else { - LOG(INFO) << "Tracing started"; - } - startTrace(now); - if (libkineto::api().client()) { - libkineto::api().client()->start(); - } - if (nextWakeupTime > profileEndTime_) { - new_wakeup_time = profileEndTime_; - } - } else if (nextWakeupTime > profileStartTime_) { - new_wakeup_time = profileStartTime_; - } - - break; - - case RunloopState::CollectTrace: - VLOG(1) << "State: CollectTrace"; - // captureWindowStartTime_ can be set by external threads, - // so recompute end time. - // FIXME: Is this a good idea for synced start? - if (profileStartIter_ < 0) { - std::lock_guard guard(mutex_); - profileEndTime_ = time_point( - microseconds(captureWindowStartTime_)) + - config_->activitiesDuration(); - } - - collection_done = isCollectionDone(now, currentIter); - - // TODO revisit stopCollection_ is not used right now - if (collection_done || stopCollection_.exchange(false) -#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) - || cupti_.stopCollection -#endif // HAS_CUPTI || HAS_ROCTRACER - ){ - // Update runloop state first to prevent further updates to shared state - LOG(INFO) << "Tracing complete."; - if (currentIter > 0) { - LOG(INFO) << "This state change was invoked by application's step() call"; - } - // FIXME: Need to communicate reason for stopping on errors - if (libkineto::api().client()) { - libkineto::api().client()->stop(); - } - std::lock_guard guard(mutex_); - stopTraceInternal(now); - VLOG_IF(0, collection_done) << "Reached profile end time"; - - UST_LOGGER_MARK_COMPLETED(kCollectionStage); - } else if (profileStartIter_ >= 0) { - // nothing to do here - } else if (now < profileEndTime_ && profileEndTime_ < nextWakeupTime) { - new_wakeup_time = profileEndTime_; - } - - break; - - case RunloopState::ProcessTrace: - VLOG(1) << "State: ProcessTrace"; - // skip this state transition if it called from the step() api - // of the profiler. - // else it could lead to a race between the profiler thread and an - // application thread calling step() - if (currentIter >= 0) { - return new_wakeup_time; - } - // FIXME: Probably want to allow interruption here - // for quickly handling trace request via synchronous API - std::lock_guard guard(mutex_); - processTraceInternal(*logger_); - UST_LOGGER_MARK_COMPLETED(kPostProcessingStage); - resetInternal(); - VLOG(0) << "ProcessTrace -> WaitForRequest"; - break; - } - - return new_wakeup_time; -} - -void CuptiActivityProfiler::finalizeTrace(const Config& config, ActivityLogger& logger) { - LOG(INFO) << "Recorded nets:"; - { - for (const auto& it : iterationCountMap_) { - LOG(INFO) << it.first << ": " << it.second << " iterations"; - } - iterationCountMap_.clear(); - } - - // Process names - int32_t pid = processId(); - string process_name = processName(pid); - if (!process_name.empty()) { - logger.handleDeviceInfo( - {pid, process_name, "CPU"}, captureWindowStartTime_); - if (!cpuOnly_) { - // GPU events use device id as pid (0-7). - constexpr int kMaxGpuCount = 8; - for (int gpu = 0; gpu < kMaxGpuCount; gpu++) { - logger.handleDeviceInfo( - {gpu, process_name, fmt::format("GPU {}", gpu)}, - captureWindowStartTime_); - } - } - } - - // Thread & stream info - for (auto pair : resourceInfo_) { - const auto& resource = pair.second; - logger.handleResourceInfo(resource, captureWindowStartTime_); - } - - for (const auto& iterations : traceSpans_) { - for (const auto& span_pair : iterations.second) { - const TraceSpan& gpu_span = span_pair.second; - if (gpu_span.opCount > 0) { - logger.handleTraceSpan(gpu_span); - } - } - } - - // Overhead info - overheadInfo_.push_back(ActivityLogger::OverheadInfo("CUPTI Overhead")); - for(const auto& info : overheadInfo_) { - logger.handleOverheadInfo(info, captureWindowStartTime_); - } - - gpuUserEventMap_.logEvents(&logger); - -#if !USE_GOOGLE_LOG - // Save logs from LoggerCollector objects into Trace metadata. - auto LoggerMD = loggerCollectorMetadata_->extractCollectorMetadata(); - std::unordered_map> LoggerMDString; - for (auto& md : LoggerMD) { - LoggerMDString[toString(md.first)] = md.second; - } -#endif // !USE_GOOGLE_LOG - - logger.finalizeTrace(config, std::move(traceBuffers_), captureWindowEndTime_, LoggerMDString); -} - -void CuptiActivityProfiler::resetTraceData() { -#if defined(HAS_CUPTI) || defined(HAS_ROCTRACER) - if (!cpuOnly_) { - cupti_.clearActivities(); - } -#endif // HAS_CUPTI || HAS_ROCTRACER - activityMap_.clear(); - cpuCorrelationMap_.clear(); - correlatedCudaActivities_.clear(); - gpuUserEventMap_.clear(); - traceSpans_.clear(); - clientActivityTraceMap_.clear(); - traceBuffers_ = nullptr; - metadata_.clear(); - sessions_.clear(); -#if !USE_GOOGLE_LOG - Logger::removeLoggerObserver(loggerCollectorMetadata_.get()); -#endif // !USE_GOOGLE_LOG -} - - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityProfiler.h b/plugins/tensorboard-plugins/libkineto/src/CuptiActivityProfiler.h deleted file mode 100644 index 208833a4d..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiActivityProfiler.h +++ /dev/null @@ -1,364 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "ThreadUtil.h" -#include "TraceSpan.h" -#include "libkineto.h" -#include "output_base.h" -#include "GenericTraceActivity.h" -#include "IActivityProfiler.h" -#include "LoggerCollector.h" - -namespace KINETO_NAMESPACE { - -class Config; -class CuptiActivityApi; -class RoctracerActivityApi; - -class CuptiActivityProfiler { - public: - CuptiActivityProfiler(CuptiActivityApi& cupti, bool cpuOnly); - CuptiActivityProfiler(RoctracerActivityApi& rai, bool cpuOnly); - CuptiActivityProfiler(const CuptiActivityProfiler&) = delete; - CuptiActivityProfiler& operator=(const CuptiActivityProfiler&) = delete; - - bool isActive() const { - return currentRunloopState_ != RunloopState::WaitForRequest; - } - - // Invoke at a regular interval to perform profiling activities. - // When not active, an interval of 1-5 seconds is probably fine, - // depending on required warm-up time and delayed start time. - // When active, it's a good idea to invoke more frequently to stay below - // memory usage limit (ACTIVITIES_MAX_GPU_BUFFER_SIZE_MB) during warmup. - const std::chrono::time_point performRunLoopStep( - const std::chrono::time_point& now, - const std::chrono::time_point& nextWakeupTime, - int64_t currentIter = -1); - - // Used for async requests - void setLogger(ActivityLogger* logger) { - logger_ = logger; - } - - // Synchronous control API - void startTrace( - const std::chrono::time_point& now) { - std::lock_guard guard(mutex_); - startTraceInternal(now); - } - - void stopTrace(const std::chrono::time_point& now) { - std::lock_guard guard(mutex_); - stopTraceInternal(now); - } - - // Process CPU and GPU traces - void processTrace(ActivityLogger& logger) { - std::lock_guard guard(mutex_); - processTraceInternal(logger); - } - - void reset() { - std::lock_guard guard(mutex_); - resetInternal(); - } - - // Set up profiler as specified in config. - void configure( - const Config& config, - const std::chrono::time_point& now); - - // Registered with client API to pass CPU trace events over - void transferCpuTrace( - std::unique_ptr cpuTrace); - - Config& config() { - return *config_; - } - - inline void recordThreadInfo() { - int32_t sysTid = systemThreadId(); - // Note we're using the lower 32 bits of the (opaque) pthread id - // as key, because that's what CUPTI records. - int32_t tid = threadId(); - int32_t pid = processId(); - std::lock_guard guard(mutex_); - recordThreadInfo(sysTid, tid, pid); - } - - // T107508020: We can deprecate the recordThreadInfo(void) once we optimized profiler_kineto - void recordThreadInfo(int32_t sysTid, int32_t tid, int32_t pid) { - if (resourceInfo_.find({pid, tid}) == resourceInfo_.end()) { - resourceInfo_.emplace( - std::make_pair(pid, tid), - ActivityLogger::ResourceInfo( - pid, - sysTid, - sysTid, // sortindex - fmt::format("thread {} ({})", sysTid, getThreadName()))); - } - } - - void addMetadata(const std::string& key, const std::string& value) { - std::lock_guard guard(mutex_); - metadata_[key] = value; - } - - void addChildActivityProfiler( - std::unique_ptr profiler) { - std::lock_guard guard(mutex_); - profilers_.push_back(std::move(profiler)); - } - - protected: - - using CpuGpuSpanPair = std::pair; - static const CpuGpuSpanPair& defaultTraceSpan(); - - private: - - // Map of gpu activities to user defined events - class GpuUserEventMap { - public: - // Insert a user defined event which maps to the gpu trace activity. - // If the user defined event mapping already exists this will update the - // gpu side span to include the span of gpuTraceActivity. - void insertOrExtendEvent(const ITraceActivity& cpuTraceActivity, - const ITraceActivity& gpuTraceActivity); - // Log out the events to the logger - void logEvents(ActivityLogger *logger); - - void clear() { - streamSpanMap_.clear(); - } - - private: - // device id and stream name - using StreamKey = std::pair; - - // map of correlation id to TraceSpan - using CorrelationSpanMap = - std::unordered_map; - std::map streamSpanMap_; - }; - - GpuUserEventMap gpuUserEventMap_; - // id -> activity* - std::unordered_map activityMap_; - // cuda runtime id -> pytorch op id - // CUPTI provides a mechanism for correlating Cuda events to arbitrary - // external events, e.g.operator activities from PyTorch. - std::unordered_map cpuCorrelationMap_; - // CUDA runtime <-> GPU Activity - std::unordered_map - correlatedCudaActivities_; - std::unordered_map userCorrelationMap_; - - // data structure to collect cuptiActivityFlushAll() latency overhead - struct profilerOverhead { - int64_t overhead; - int cntr; - }; - - bool isWarmupDone( - const std::chrono::time_point& now, - int64_t currentIter) const; - - bool isCollectionDone( - const std::chrono::time_point& now, - int64_t currentIter) const; - - void startTraceInternal( - const std::chrono::time_point& now); - - void stopTraceInternal( - const std::chrono::time_point& now); - - void processTraceInternal(ActivityLogger& logger); - - void resetInternal(); - - void finalizeTrace(const Config& config, ActivityLogger& logger); - - void configureChildProfilers(); - - // Process a single CPU trace - void processCpuTrace( - libkineto::CpuTraceBuffer& cpuTrace, - ActivityLogger& logger); - - // Create resource names for streams - inline void recordStream(int device, int id, const char* postfix) { - if (resourceInfo_.find({device, id}) == resourceInfo_.end()) { - resourceInfo_.emplace( - std::make_pair(device, id), - ActivityLogger::ResourceInfo( - device, id, id, fmt::format( - "stream {} {}", id, postfix))); - } - } - - // Record client trace span for subsequent lookups from activities - // Also creates a corresponding GPU-side span. - CpuGpuSpanPair& recordTraceSpan(TraceSpan& span, int gpuOpCount); - - // Returns true if net name is to be tracked for a specified number of - // iterations. - bool iterationTargetMatch(libkineto::CpuTraceBuffer& trace); - - // net name to id - int netId(const std::string& netName); - - const ITraceActivity* linkedActivity( - int32_t correlationId, - const std::unordered_map& correlationMap); - -#ifdef HAS_CUPTI - // Process generic CUPTI activity - void handleCuptiActivity(const CUpti_Activity* record, ActivityLogger* logger); - - // Process specific GPU activity types - void updateGpuNetSpan(const ITraceActivity& gpuOp); - bool outOfRange(const ITraceActivity& act); - void handleCorrelationActivity( - const CUpti_ActivityExternalCorrelation* correlation); - void handleRuntimeActivity( - const CUpti_ActivityAPI* activity, ActivityLogger* logger); - void handleOverheadActivity( - const CUpti_ActivityOverhead* activity, ActivityLogger* logger); - void handleGpuActivity(const ITraceActivity& act, - ActivityLogger* logger); - template - void handleGpuActivity(const T* act, ActivityLogger* logger); -#endif // HAS_CUPTI - - void resetTraceData(); - - void addOverheadSample(profilerOverhead& counter, int64_t overhead) { - counter.overhead += overhead; - counter.cntr++; - } - int64_t getOverhead(const profilerOverhead& counter) { - if (counter.cntr == 0) { - return 0; - } - return counter.overhead / counter.cntr; - } - - void checkTimestampOrder(const ITraceActivity* act1); - - // On-demand request configuration - std::unique_ptr config_; - - // Logger used during trace processing - ActivityLogger* logger_; - - // Calls to CUPTI is encapsulated behind this interface -#ifdef HAS_ROCTRACER - RoctracerActivityApi& cupti_; // Design failure here -#else - CuptiActivityApi& cupti_; -#endif - - enum class RunloopState { - WaitForRequest, - Warmup, - CollectTrace, - ProcessTrace - }; - - // Start and end time used for triggering and stopping profiling - std::chrono::time_point profileStartTime_; - std::chrono::time_point profileEndTime_; - int64_t profileStartIter_ = -1, profileEndIter_ = -1; - - - // All recorded trace spans, both CPU and GPU - // Trace Id -> list of iterations. - // Using map of lists for the iterator semantics, since we are recording - // pointers to the elements in this structure. - std::map> traceSpans_; - - // Maintain a map of client trace activity to trace span. - // Maps correlation id -> TraceSpan* held by traceSpans_. - using ActivityTraceMap = std::unordered_map; - ActivityTraceMap clientActivityTraceMap_; - - // Cache thread names and system thread ids for pthread ids, - // and stream ids for GPU streams - std::map< - std::pair, - ActivityLogger::ResourceInfo> resourceInfo_; - - std::vector overheadInfo_; - - // the overhead to flush the activity buffer - profilerOverhead flushOverhead_; - // the overhead to enable/disable activity tracking - profilerOverhead setupOverhead_; - - bool cpuOnly_{false}; - - // *************************************************************************** - // Below state is shared with external threads. - // These need to either be atomic, accessed under lock or only used - // by external threads in separate runloop phases from the profiler thread. - // *************************************************************************** - - // Mutex to protect non-atomic access to below state - std::mutex mutex_; - - // Runloop phase - std::atomic currentRunloopState_{RunloopState::WaitForRequest}; - - // Keep track of the start time of the first net in the current trace. - // This is only relevant to Caffe2 as PyTorch does not have nets. - // All CUDA events before this time will be removed - // Can be written by external threads during collection. - int64_t captureWindowStartTime_{0}; - // Similarly, all CUDA API events after the last net event will be removed - int64_t captureWindowEndTime_{0}; - - // span name -> iteration count - std::map iterationCountMap_; - // Flag used to stop tracing from external api callback. - // Needs to be atomic since it's set from a different thread. - std::atomic_bool stopCollection_{false}; - - // Buffers where trace data is stored - std::unique_ptr traceBuffers_; - - // Trace metadata - std::unordered_map metadata_; - - // child activity profilers - std::vector> profilers_; - - // a vector of active profiler plugin sessions - std::vector> sessions_; - - // LoggerCollector to collect all LOGs during the trace -#if !USE_GOOGLE_LOG - std::unique_ptr loggerCollectorMetadata_; -#endif // !USE_GOOGLE_LOG -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApi.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApi.cpp deleted file mode 100644 index 187600399..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApi.cpp +++ /dev/null @@ -1,260 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "CuptiCallbackApi.h" - -#include -#include -#include -#include -#include - -#ifdef HAS_CUPTI -#include "cupti_call.h" -#endif -#include "Logger.h" - - -namespace KINETO_NAMESPACE { - -// limit on number of handles per callback type -constexpr size_t MAX_CB_FNS_PER_CB = 8; - -// Reader Writer lock types -using ReaderWriterLock = std::shared_timed_mutex; -using ReaderLockGuard = std::shared_lock; -using WriteLockGuard = std::unique_lock; - -static ReaderWriterLock callbackLock_; - -/* Callback Table : - * Overall goal of the design is to optimize the lookup of function - * pointers. The table is structured at two levels and the leaf - * elements in the table are std::list to enable fast access/inserts/deletes - * - * | - * -> cb id 0 -> std::list of callbacks - * ... - * -> cb id n -> std::list of callbacks - * | - * ... - * CallbackTable is the finaly table type above - * See type declrartions in header file. - */ - - -/* callback_switchboard : is the global callback handler we register - * with CUPTI. The goal is to make it as efficient as possible - * to re-direct to the registered callback(s). - * - * Few things to care about : - * a) use if/then switches rather than map/hash structures - * b) avoid dynamic memory allocations - * c) be aware of locking overheads - */ -#ifdef HAS_CUPTI -static void CUPTIAPI callback_switchboard( -#else -static void callback_switchboard( -#endif - void* /* unused */, - CUpti_CallbackDomain domain, - CUpti_CallbackId cbid, - const CUpti_CallbackData* cbInfo) { - - // below statement is likey going to call a mutex - // on the singleton access - CuptiCallbackApi::singleton().__callback_switchboard( - domain, cbid, cbInfo); -} - - -void CuptiCallbackApi::__callback_switchboard( - CUpti_CallbackDomain domain, - CUpti_CallbackId cbid, - const CUpti_CallbackData* cbInfo) { - VLOG(0) << "Callback: domain = " << domain << ", cbid = " << cbid; - CallbackList *cblist = nullptr; - - switch (domain) { - - // add the fastest path for kernel launch callbacks - // as these are the most frequent ones - case CUPTI_CB_DOMAIN_RUNTIME_API: - switch (cbid) { - case CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000: - cblist = &callbacks_.runtime[ - CUDA_LAUNCH_KERNEL - __RUNTIME_CB_DOMAIN_START]; - break; - default: - break; - } - break; - - case CUPTI_CB_DOMAIN_RESOURCE: - switch (cbid) { - case CUPTI_CBID_RESOURCE_CONTEXT_CREATED: - cblist = &callbacks_.resource[ - RESOURCE_CONTEXT_CREATED - __RESOURCE_CB_DOMAIN_START]; - break; - case CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING: - cblist = &callbacks_.resource[ - RESOURCE_CONTEXT_DESTROYED - __RESOURCE_CB_DOMAIN_START]; - break; - default: - break; - } - break; - - default: - return; - } - - // ignore callbacks that are not handled - if (cblist == nullptr) { - return; - } - - // make a copy of the callback list so we avoid holding lock - // in common case this should be just one func pointer copy - std::array callbacks; - int num_cbs = 0; - { - ReaderLockGuard rl(callbackLock_); - int i = 0; - for (auto it = cblist->begin(); - it != cblist->end() && i < MAX_CB_FNS_PER_CB; - it++, i++) { - callbacks[i] = *it; - } - num_cbs = i; - } - - for (int i = 0; i < num_cbs; i++) { - auto fn = callbacks[i]; - fn(domain, cbid, cbInfo); - } -} - -CuptiCallbackApi& CuptiCallbackApi::singleton() { - static CuptiCallbackApi instance; - return instance; -} - -CuptiCallbackApi::CuptiCallbackApi() { -#ifdef HAS_CUPTI - lastCuptiStatus_ = CUPTI_ERROR_UNKNOWN; - lastCuptiStatus_ = CUPTI_CALL_NOWARN( - cuptiSubscribe(&subscriber_, - (CUpti_CallbackFunc)callback_switchboard, - nullptr)); - - initSuccess_ = (lastCuptiStatus_ == CUPTI_SUCCESS); -#endif -} - -CuptiCallbackApi::CallbackList* CuptiCallbackApi::CallbackTable::lookup( - CUpti_CallbackDomain domain, CuptiCallBackID cbid) { - size_t idx; - - switch (domain) { - - case CUPTI_CB_DOMAIN_RESOURCE: - assert(cbid >= __RESOURCE_CB_DOMAIN_START); - assert(cbid < __RESOURCE_CB_DOMAIN_END); - idx = cbid - __RESOURCE_CB_DOMAIN_START; - return &resource.at(idx); - - case CUPTI_CB_DOMAIN_RUNTIME_API: - assert(cbid >= __RUNTIME_CB_DOMAIN_START); - assert(cbid < __RUNTIME_CB_DOMAIN_END); - idx = cbid - __RUNTIME_CB_DOMAIN_START; - return &runtime.at(idx); - - default: - LOG(WARNING) << " Unsupported callback domain : " << domain; - return nullptr; - } -} - -bool CuptiCallbackApi::registerCallback( - CUpti_CallbackDomain domain, - CuptiCallBackID cbid, - CuptiCallbackFn cbfn) { - CallbackList* cblist = callbacks_.lookup(domain, cbid); - - if (!cblist) { - LOG(WARNING) << "Could not register callback -- domain = " << domain - << " callback id = " << cbid; - return false; - } - - // avoid duplicates - auto it = std::find(cblist->begin(), cblist->end(), cbfn); - if (it != cblist->end()) { - LOG(WARNING) << "Adding duplicate callback -- domain = " << domain - << " callback id = " << cbid; - return true; - } - - if (cblist->size() == MAX_CB_FNS_PER_CB) { - LOG(WARNING) << "Already registered max callback -- domain = " << domain - << " callback id = " << cbid; - } - - WriteLockGuard wl(callbackLock_); - cblist->push_back(cbfn); - return true; -} - -bool CuptiCallbackApi::deleteCallback( - CUpti_CallbackDomain domain, - CuptiCallBackID cbid, - CuptiCallbackFn cbfn) { - CallbackList* cblist = callbacks_.lookup(domain, cbid); - if (!cblist) { - LOG(WARNING) << "Attempting to remove unsupported callback -- domain = " << domain - << " callback id = " << cbid; - return false; - } - - // Locks are not required here as - // https://en.cppreference.com/w/cpp/container/list/erase - // "References and iterators to the erased elements are invalidated. - // Other references and iterators are not affected." - auto it = std::find(cblist->begin(), cblist->end(), cbfn); - if (it == cblist->end()) { - LOG(WARNING) << "Could not find callback to remove -- domain = " << domain - << " callback id = " << cbid; - return false; - } - - WriteLockGuard wl(callbackLock_); - cblist->erase(it); - return true; -} - -bool CuptiCallbackApi::enableCallback( - CUpti_CallbackDomain domain, CUpti_CallbackId cbid) { -#ifdef HAS_CUPTI - if (initSuccess_) { - lastCuptiStatus_ = CUPTI_CALL_NOWARN( - cuptiEnableCallback(1, subscriber_, domain, cbid)); - return (lastCuptiStatus_ == CUPTI_SUCCESS); - } -#endif - return false; -} - -bool CuptiCallbackApi::disableCallback( - CUpti_CallbackDomain domain, CUpti_CallbackId cbid) { -#ifdef HAS_CUPTI - if (initSuccess_) { - lastCuptiStatus_ = CUPTI_CALL_NOWARN( - cuptiEnableCallback(0, subscriber_, domain, cbid)); - return (lastCuptiStatus_ == CUPTI_SUCCESS); - } -#endif - return false; -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApi.h b/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApi.h deleted file mode 100644 index 4526f3750..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApi.h +++ /dev/null @@ -1,130 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#ifdef HAS_CUPTI -#include -#endif -#include -#include -#include -#include -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "CuptiCallbackApiMock.h" - -namespace KINETO_NAMESPACE { - -using namespace libkineto; - - -/* CuptiCallbackApi : Provides an abstraction over CUPTI callback - * interface. This enables various callback functions to be registered - * with this class. The class registers a global callback handler that - * redirects to the respective callbacks. - * - * Note: one design choice we made is to only support simple function pointers - * in order to speed up the implementation for fast path. - */ - -using CuptiCallbackFn = void(*)( - CUpti_CallbackDomain domain, - CUpti_CallbackId cbid, - const CUpti_CallbackData* cbInfo); - - -class CuptiCallbackApi { - - public: - - /* Global list of supported callback ids - * use the class namespace to avoid confusing with CUPTI enums*/ - enum CuptiCallBackID { - CUDA_LAUNCH_KERNEL = 0, - // can possibly support more callback ids per domain - // - __RUNTIME_CB_DOMAIN_START = CUDA_LAUNCH_KERNEL, - - // Callbacks under Resource CB domain - RESOURCE_CONTEXT_CREATED, - RESOURCE_CONTEXT_DESTROYED, - - __RUNTIME_CB_DOMAIN_END = RESOURCE_CONTEXT_CREATED, - __RESOURCE_CB_DOMAIN_START = RESOURCE_CONTEXT_CREATED, - - __RESOURCE_CB_DOMAIN_END = RESOURCE_CONTEXT_DESTROYED + 1, - }; - - - CuptiCallbackApi(const CuptiCallbackApi&) = delete; - CuptiCallbackApi& operator=(const CuptiCallbackApi&) = delete; - - static CuptiCallbackApi& singleton(); - - bool initSuccess() const { - return initSuccess_; - } - -#ifdef HAS_CUPTI - CUptiResult getCuptiStatus() const { - return lastCuptiStatus_; - } -#endif - - bool registerCallback( - CUpti_CallbackDomain domain, - CuptiCallBackID cbid, - CuptiCallbackFn cbfn); - - // returns false if callback was not found - bool deleteCallback( - CUpti_CallbackDomain domain, - CuptiCallBackID cbid, - CuptiCallbackFn cbfn); - - bool enableCallback(CUpti_CallbackDomain domain, CUpti_CallbackId cbid); - bool disableCallback(CUpti_CallbackDomain domain, CUpti_CallbackId cbid); - - - // Please do not use this method. This has to be exposed as public - // so it is accessible from the callback handler - void __callback_switchboard( - CUpti_CallbackDomain domain, - CUpti_CallbackId cbid, - const CUpti_CallbackData* cbInfo); - - private: - - explicit CuptiCallbackApi(); - - // For callback table design overview see the .cpp file - using CallbackList = std::list; - - // level 2 tables sizes are known at compile time - constexpr static size_t RUNTIME_CB_DOMAIN_SIZE - = (__RUNTIME_CB_DOMAIN_END - __RUNTIME_CB_DOMAIN_START); - - constexpr static size_t RESOURCE_CB_DOMAIN_SIZE - = (__RESOURCE_CB_DOMAIN_END - __RESOURCE_CB_DOMAIN_START); - - // level 1 table is a struct - struct CallbackTable { - std::array runtime; - std::array resource; - - CallbackList* lookup(CUpti_CallbackDomain domain, CuptiCallBackID cbid); - }; - - CallbackTable callbacks_; - bool initSuccess_ = false; - -#ifdef HAS_CUPTI - CUptiResult lastCuptiStatus_; - CUpti_SubscriberHandle subscriber_; -#endif -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApiMock.h b/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApiMock.h deleted file mode 100644 index fd5126727..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiCallbackApiMock.h +++ /dev/null @@ -1,32 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -// Provides data structures to mock CUPTI Callback API -#ifndef HAS_CUPTI - -enum CUpti_CallbackDomain { - CUPTI_CB_DOMAIN_RESOURCE, - CUPTI_CB_DOMAIN_RUNTIME_API, -}; -enum CUpti_CallbackId { - CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000, - CUPTI_CBID_RESOURCE_CONTEXT_CREATED, - CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING, -}; - -using CUcontext = void*; - -struct CUpti_ResourceData { - CUcontext context; -}; - -constexpr int CUPTI_API_ENTER = 0; -constexpr int CUPTI_API_EXIT = 0; - -struct CUpti_CallbackData { - CUcontext context; - const char* symbolName; - int callbackSite; -}; -#endif // HAS_CUPTI diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiEventApi.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiEventApi.cpp deleted file mode 100644 index 7f1d48c1d..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiEventApi.cpp +++ /dev/null @@ -1,112 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "CuptiEventApi.h" - -#include - -#include "Logger.h" -#include "cupti_call.h" - -using namespace std::chrono; -using std::vector; - -namespace KINETO_NAMESPACE { - -CuptiEventApi::CuptiEventApi(CUcontext context) - : context_(context) { - CUPTI_CALL(cuptiGetDeviceId(context_, (uint32_t*)&device_)); -} - -CUpti_EventGroupSets* CuptiEventApi::createGroupSets( - vector& ids) { - CUpti_EventGroupSets* group_sets = nullptr; - CUptiResult res = CUPTI_CALL(cuptiEventGroupSetsCreate( - context_, sizeof(CUpti_EventID) * ids.size(), ids.data(), &group_sets)); - - if (res != CUPTI_SUCCESS || group_sets == nullptr) { - const char* errstr = nullptr; - CUPTI_CALL(cuptiGetResultString(res, &errstr)); - throw std::system_error(EINVAL, std::generic_category(), errstr); - } - - return group_sets; -} - -void CuptiEventApi::destroyGroupSets(CUpti_EventGroupSets* sets) { - CUPTI_CALL(cuptiEventGroupSetsDestroy(sets)); -} - -bool CuptiEventApi::setContinuousMode() { - // Avoid logging noise for CUPTI_ERROR_LEGACY_PROFILER_NOT_SUPPORTED - CUptiResult res = CUPTI_CALL_NOWARN(cuptiSetEventCollectionMode( - context_, CUPTI_EVENT_COLLECTION_MODE_CONTINUOUS)); - if (res == CUPTI_ERROR_LEGACY_PROFILER_NOT_SUPPORTED) { - return false; - } - // Log warning on other errors - CUPTI_CALL(res); - return (res == CUPTI_SUCCESS); -} - -void CuptiEventApi::enablePerInstance(CUpti_EventGroup eventGroup) { - uint32_t profile_all = 1; - CUPTI_CALL(cuptiEventGroupSetAttribute( - eventGroup, - CUPTI_EVENT_GROUP_ATTR_PROFILE_ALL_DOMAIN_INSTANCES, - sizeof(profile_all), - &profile_all)); -} - -uint32_t CuptiEventApi::instanceCount(CUpti_EventGroup eventGroup) { - uint32_t instance_count = 0; - size_t s = sizeof(instance_count); - CUPTI_CALL(cuptiEventGroupGetAttribute( - eventGroup, CUPTI_EVENT_GROUP_ATTR_INSTANCE_COUNT, &s, &instance_count)); - return instance_count; -} - -void CuptiEventApi::enableGroupSet(CUpti_EventGroupSet& set) { - CUptiResult res = CUPTI_CALL_NOWARN(cuptiEventGroupSetEnable(&set)); - if (res != CUPTI_SUCCESS) { - const char* errstr = nullptr; - CUPTI_CALL(cuptiGetResultString(res, &errstr)); - throw std::system_error(EIO, std::generic_category(), errstr); - } -} - -void CuptiEventApi::disableGroupSet(CUpti_EventGroupSet& set) { - CUPTI_CALL(cuptiEventGroupSetDisable(&set)); -} - -void CuptiEventApi::readEvent( - CUpti_EventGroup grp, - CUpti_EventID id, - vector& vals) { - size_t s = sizeof(int64_t) * vals.size(); - CUPTI_CALL(cuptiEventGroupReadEvent( - grp, - CUPTI_EVENT_READ_FLAG_NONE, - id, - &s, - reinterpret_cast(vals.data()))); -} - -vector CuptiEventApi::eventsInGroup(CUpti_EventGroup grp) { - uint32_t group_size = 0; - size_t s = sizeof(group_size); - CUPTI_CALL(cuptiEventGroupGetAttribute( - grp, CUPTI_EVENT_GROUP_ATTR_NUM_EVENTS, &s, &group_size)); - size_t events_size = group_size * sizeof(CUpti_EventID); - vector res(group_size); - CUPTI_CALL(cuptiEventGroupGetAttribute( - grp, CUPTI_EVENT_GROUP_ATTR_EVENTS, &events_size, res.data())); - return res; -} - -CUpti_EventID CuptiEventApi::eventId(const std::string& name) { - CUpti_EventID id{0}; - CUPTI_CALL(cuptiEventGetIdFromName(device_, name.c_str(), &id)); - return id; -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiEventApi.h b/plugins/tensorboard-plugins/libkineto/src/CuptiEventApi.h deleted file mode 100644 index 79610f93f..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiEventApi.h +++ /dev/null @@ -1,49 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include - -namespace KINETO_NAMESPACE { - -// C++ interface to CUPTI Events C API. -// Virtual methods are here mainly to allow easier testing. -class CuptiEventApi { - public: - explicit CuptiEventApi(CUcontext context_); - virtual ~CuptiEventApi() {} - - CUdevice device() { - return device_; - } - - virtual CUpti_EventGroupSets* createGroupSets( - std::vector& ids); - virtual void destroyGroupSets(CUpti_EventGroupSets* sets); - - virtual bool setContinuousMode(); - - virtual void enablePerInstance(CUpti_EventGroup eventGroup); - virtual uint32_t instanceCount(CUpti_EventGroup eventGroup); - - virtual void enableGroupSet(CUpti_EventGroupSet& set); - virtual void disableGroupSet(CUpti_EventGroupSet& set); - - virtual void - readEvent(CUpti_EventGroup g, CUpti_EventID id, std::vector& vals); - virtual std::vector eventsInGroup(CUpti_EventGroup g); - - virtual CUpti_EventID eventId(const std::string& name); - - protected: - // Unit testing - CuptiEventApi() : context_(nullptr), device_(0) {} - - private: - CUcontext context_; - CUdevice device_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiMetricApi.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiMetricApi.cpp deleted file mode 100644 index 36401e743..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiMetricApi.cpp +++ /dev/null @@ -1,107 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "CuptiMetricApi.h" - -#include - -#include "Logger.h" -#include "cupti_call.h" - -using namespace std::chrono; -using std::vector; - -namespace KINETO_NAMESPACE { - -CUpti_MetricID CuptiMetricApi::idFromName(const std::string& name) { - CUpti_MetricID metric_id{~0u}; - CUptiResult res = - CUPTI_CALL(cuptiMetricGetIdFromName(device_, name.c_str(), &metric_id)); - if (res == CUPTI_ERROR_INVALID_METRIC_NAME) { - LOG(WARNING) << "Invalid metric name: " << name; - } - return metric_id; -} - -// Return a map of event IDs and names for a given metric id. -// Note that many events don't have a name. In that case the name will -// be set to the empty string. -std::map CuptiMetricApi::events( - CUpti_MetricID metric_id) { - uint32_t num_events = 0; - CUPTI_CALL(cuptiMetricGetNumEvents(metric_id, &num_events)); - vector ids(num_events); - size_t array_size = num_events * sizeof(CUpti_EventID); - CUPTI_CALL(cuptiMetricEnumEvents(metric_id, &array_size, ids.data())); - std::map res; - for (CUpti_EventID id : ids) { - // Attempt to lookup name from CUPTI - constexpr size_t kMaxEventNameLength = 64; - char cupti_name[kMaxEventNameLength]; - size_t size = kMaxEventNameLength; - CUPTI_CALL( - cuptiEventGetAttribute(id, CUPTI_EVENT_ATTR_NAME, &size, cupti_name)); - cupti_name[kMaxEventNameLength - 1] = 0; - - // CUPTI "helpfully" returns "event_name" when the event is unnamed. - if (size > 0 && strcmp(cupti_name, "event_name") != 0) { - res.emplace(id, cupti_name); - } else { - res.emplace(id, ""); - } - } - return res; -} - -CUpti_MetricValueKind CuptiMetricApi::valueKind(CUpti_MetricID metric) { - CUpti_MetricValueKind res{CUPTI_METRIC_VALUE_KIND_FORCE_INT}; - size_t value_kind_size = sizeof(res); - CUPTI_CALL(cuptiMetricGetAttribute( - metric, CUPTI_METRIC_ATTR_VALUE_KIND, &value_kind_size, &res)); - return res; -} - -CUpti_MetricEvaluationMode CuptiMetricApi::evaluationMode( - CUpti_MetricID metric) { - CUpti_MetricEvaluationMode eval_mode{ - CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE}; - size_t eval_mode_size = sizeof(eval_mode); - CUPTI_CALL(cuptiMetricGetAttribute( - metric, CUPTI_METRIC_ATTR_EVALUATION_MODE, &eval_mode_size, &eval_mode)); - return eval_mode; -} - -// FIXME: Consider caching value kind here -SampleValue CuptiMetricApi::calculate( - CUpti_MetricID metric, - CUpti_MetricValueKind kind, - vector& events, - vector& values, - int64_t duration) { - CUpti_MetricValue metric_value; - CUPTI_CALL(cuptiMetricGetValue( - device_, - metric, - events.size() * sizeof(CUpti_EventID), - events.data(), - values.size() * sizeof(int64_t), - reinterpret_cast(values.data()), - duration, - &metric_value)); - - switch (kind) { - case CUPTI_METRIC_VALUE_KIND_DOUBLE: - case CUPTI_METRIC_VALUE_KIND_PERCENT: - return SampleValue(metric_value.metricValueDouble); - case CUPTI_METRIC_VALUE_KIND_UINT64: - case CUPTI_METRIC_VALUE_KIND_INT64: - case CUPTI_METRIC_VALUE_KIND_THROUGHPUT: - return SampleValue(metric_value.metricValueUint64); - case CUPTI_METRIC_VALUE_KIND_UTILIZATION_LEVEL: - return SampleValue((int)metric_value.metricValueUtilizationLevel); - default: - assert(false); - } - return SampleValue(-1); -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiMetricApi.h b/plugins/tensorboard-plugins/libkineto/src/CuptiMetricApi.h deleted file mode 100644 index f45d38cd6..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiMetricApi.h +++ /dev/null @@ -1,38 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -#include -#include - -#include "SampleListener.h" - -namespace KINETO_NAMESPACE { - -// C++ interface to CUPTI Metrics C API. -// Virtual methods are here mainly to allow easier testing. -class CuptiMetricApi { - public: - explicit CuptiMetricApi(CUdevice device) : device_(device) {} - virtual ~CuptiMetricApi() {} - - virtual CUpti_MetricID idFromName(const std::string& name); - virtual std::map events(CUpti_MetricID metric_id); - - virtual CUpti_MetricValueKind valueKind(CUpti_MetricID metric); - virtual CUpti_MetricEvaluationMode evaluationMode(CUpti_MetricID metric); - - virtual SampleValue calculate( - CUpti_MetricID metric, - CUpti_MetricValueKind kind, - std::vector& events, - std::vector& values, - int64_t duration); - - private: - CUdevice device_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiNvPerfMetric.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiNvPerfMetric.cpp deleted file mode 100644 index d1b08ab2c..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiNvPerfMetric.cpp +++ /dev/null @@ -1,504 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#ifdef HAS_CUPTI -#include -#if defined(CUDART_VERSION) && CUDART_VERSION > 10000 && CUDART_VERSION < 11040 -#include -#include -#include -#endif // cuda version > 10.00 and < 11.04 -#endif // HAS_CUPTI - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "ScopeExit.h" -#include "CuptiNvPerfMetric.h" -#include "Logger.h" - -namespace KINETO_NAMESPACE { - -// Add a namespace to isolate these utility functions that are only -// going to be used by the CuptiRangeProfiler. These included calls -// to NVIDIA PerfWorks APIs. -namespace nvperf { - - -// Largely based on NVIDIA sample code provided with CUDA release -// files Metric.cpp and Eval.cpp - -// ------------------------------------------------- -// Metric and Counter Data Configuration -// ------------------------------------------------- - - -// Note: Be carful before modifying the code below. There is a specific -// sequence one needs to follow to program the metrics else things may -// stop working. We tried to keep the flow consistent with the example -// code from NVIDIA. Since most of the programmability comes from -// the CUPTI profiler metric names this should be okay. - -// Only supported on CUDA RT Version between 10.0 and 11.04. -// After CUDA RT 11.04, the structure has changed. -// TODO update the structure NVPA_RawMetricsConfig to support 11.04 -#if defined(CUDART_VERSION) && CUDART_VERSION > 10000 && CUDART_VERSION < 11040 - -bool getRawMetricRequests( - NVPA_MetricsContext* metricsContext, - std::vector metricNames, - std::vector& rawMetricsDeps, - std::vector& rawMetricRequests) { - bool isolated = true; - /* Bug in collection with collection of metrics without instances, keep it - * to true*/ - bool keepInstances = true; - - for (const auto& metricName : metricNames) { - - NVPW_MetricsContext_GetMetricProperties_Begin_Params - getMetricPropertiesBeginParams = { - NVPW_MetricsContext_GetMetricProperties_Begin_Params_STRUCT_SIZE, nullptr}; - getMetricPropertiesBeginParams.pMetricsContext = metricsContext; - getMetricPropertiesBeginParams.pMetricName = metricName.c_str(); - - if (!NVPW_CALL( - NVPW_MetricsContext_GetMetricProperties_Begin( - &getMetricPropertiesBeginParams))) { - return false; - } - - for (const char** metricDepsIt = - getMetricPropertiesBeginParams.ppRawMetricDependencies; - *metricDepsIt; - ++metricDepsIt) { - rawMetricsDeps.push_back(*metricDepsIt); - } - - NVPW_MetricsContext_GetMetricProperties_End_Params - getMetricPropertiesEndParams = { - NVPW_MetricsContext_GetMetricProperties_End_Params_STRUCT_SIZE, nullptr}; - getMetricPropertiesEndParams.pMetricsContext = metricsContext; - - if (!NVPW_CALL(NVPW_MetricsContext_GetMetricProperties_End( - &getMetricPropertiesEndParams))) { - return false; - } - } - - for (const auto& rawMetricName : rawMetricsDeps) { - NVPA_RawMetricRequest metricRequest = {NVPA_RAW_METRIC_REQUEST_STRUCT_SIZE, nullptr}; - metricRequest.pMetricName = rawMetricName.c_str(); - metricRequest.isolated = isolated; - metricRequest.keepInstances = keepInstances; - rawMetricRequests.push_back(metricRequest); - VLOG(1) << "Adding raw metric struct : raw metric = " << rawMetricName - << " isolated = " << isolated << " keepinst = " << keepInstances; - } - - if (rawMetricRequests.size() == 0) { - LOG(WARNING) << "CUPTI Profiler was unable to configure any metrics"; - return false; - } - return true; -} - -// Setup CUPTI Profiler Config Image -bool getProfilerConfigImage( - const std::string& chipName, - const std::vector& metricNames, - std::vector& configImage, - const uint8_t* counterAvailabilityImage) { - - NVPW_CUDA_MetricsContext_Create_Params metricsContextCreateParams = { - NVPW_CUDA_MetricsContext_Create_Params_STRUCT_SIZE, nullptr}; - metricsContextCreateParams.pChipName = chipName.c_str(); - - if (!NVPW_CALL( - NVPW_CUDA_MetricsContext_Create(&metricsContextCreateParams))) { - return false; - } - - NVPW_MetricsContext_Destroy_Params metricsContextDestroyParams = { - NVPW_MetricsContext_Destroy_Params_STRUCT_SIZE, nullptr}; - metricsContextDestroyParams.pMetricsContext = - metricsContextCreateParams.pMetricsContext; - - SCOPE_EXIT([&]() { - NVPW_MetricsContext_Destroy( - (NVPW_MetricsContext_Destroy_Params*)&metricsContextDestroyParams); - }); - - // Get all raw metrics required for given metricNames list - std::vector rawMetricRequests; - - // note: we need a variable at this functions scope to hold the string - // pointers for underlying C char arrays. - std::vector rawMetricDeps; - - if (!getRawMetricRequests( - metricsContextCreateParams.pMetricsContext, - metricNames, - rawMetricDeps, - rawMetricRequests)) { - return false; - } - - NVPA_RawMetricsConfigOptions metricsConfigOptions = { - NVPA_RAW_METRICS_CONFIG_OPTIONS_STRUCT_SIZE, nullptr}; - metricsConfigOptions.activityKind = NVPA_ACTIVITY_KIND_PROFILER; - metricsConfigOptions.pChipName = chipName.c_str(); - NVPA_RawMetricsConfig* rawMetricsConfig; - if (!NVPW_CALL( - NVPA_RawMetricsConfig_Create( - &metricsConfigOptions, &rawMetricsConfig))) { - return false; - } - - // TODO check if this is required - if (counterAvailabilityImage) { - NVPW_RawMetricsConfig_SetCounterAvailability_Params - setCounterAvailabilityParams = { - NVPW_RawMetricsConfig_SetCounterAvailability_Params_STRUCT_SIZE, nullptr}; - setCounterAvailabilityParams.pRawMetricsConfig = rawMetricsConfig; - setCounterAvailabilityParams.pCounterAvailabilityImage = - counterAvailabilityImage; - if (!NVPW_CALL( - NVPW_RawMetricsConfig_SetCounterAvailability( - &setCounterAvailabilityParams))) { - return false; - } - } - - NVPW_RawMetricsConfig_Destroy_Params rawMetricsConfigDestroyParams = { - NVPW_RawMetricsConfig_Destroy_Params_STRUCT_SIZE, nullptr}; - rawMetricsConfigDestroyParams.pRawMetricsConfig = rawMetricsConfig; - SCOPE_EXIT([&]() { - NVPW_RawMetricsConfig_Destroy( - (NVPW_RawMetricsConfig_Destroy_Params*)&rawMetricsConfigDestroyParams); - }); - - // Start a Raw Metric Pass group - NVPW_RawMetricsConfig_BeginPassGroup_Params beginPassGroupParams = { - NVPW_RawMetricsConfig_BeginPassGroup_Params_STRUCT_SIZE, nullptr}; - beginPassGroupParams.pRawMetricsConfig = rawMetricsConfig; - if (!NVPW_CALL( - NVPW_RawMetricsConfig_BeginPassGroup(&beginPassGroupParams))) { - return false; - } - - // Add all raw metrics - NVPW_RawMetricsConfig_AddMetrics_Params addMetricsParams = { - NVPW_RawMetricsConfig_AddMetrics_Params_STRUCT_SIZE, nullptr}; - addMetricsParams.pRawMetricsConfig = rawMetricsConfig; - addMetricsParams.pRawMetricRequests = rawMetricRequests.data(); - addMetricsParams.numMetricRequests = rawMetricRequests.size(); - if (!NVPW_CALL( - NVPW_RawMetricsConfig_AddMetrics(&addMetricsParams))) { - return false; - } - - // End pass group - NVPW_RawMetricsConfig_EndPassGroup_Params endPassGroupParams = { - NVPW_RawMetricsConfig_EndPassGroup_Params_STRUCT_SIZE, nullptr}; - endPassGroupParams.pRawMetricsConfig = rawMetricsConfig; - if (!NVPW_CALL( - NVPW_RawMetricsConfig_EndPassGroup(&endPassGroupParams))) { - return false; - } - - // Setup Config Image generation - NVPW_RawMetricsConfig_GenerateConfigImage_Params generateConfigImageParams = { - NVPW_RawMetricsConfig_GenerateConfigImage_Params_STRUCT_SIZE, nullptr}; - generateConfigImageParams.pRawMetricsConfig = rawMetricsConfig; - if (!NVPW_CALL( - NVPW_RawMetricsConfig_GenerateConfigImage(&generateConfigImageParams))) { - return false; - } - - // Get the Config Image size... nearly there - NVPW_RawMetricsConfig_GetConfigImage_Params getConfigImageParams = { - NVPW_RawMetricsConfig_GetConfigImage_Params_STRUCT_SIZE, nullptr}; - getConfigImageParams.pRawMetricsConfig = rawMetricsConfig; - getConfigImageParams.bytesAllocated = 0; - getConfigImageParams.pBuffer = nullptr; - if (!NVPW_CALL( - NVPW_RawMetricsConfig_GetConfigImage(&getConfigImageParams))) { - return false; - } - - configImage.resize(getConfigImageParams.bytesCopied); - - // Write the Config image binary - getConfigImageParams.bytesAllocated = configImage.size(); - getConfigImageParams.pBuffer = configImage.data(); - if (!NVPW_CALL( - NVPW_RawMetricsConfig_GetConfigImage(&getConfigImageParams))) { - return false; - } - - return true; -} - -bool getCounterDataPrefixImage( - const std::string& chipName, - const std::vector& metricNames, - std::vector& counterDataImagePrefix) { - - NVPW_CUDA_MetricsContext_Create_Params metricsContextCreateParams = { - NVPW_CUDA_MetricsContext_Create_Params_STRUCT_SIZE, nullptr}; - metricsContextCreateParams.pChipName = chipName.c_str(); - - if (!NVPW_CALL( - NVPW_CUDA_MetricsContext_Create(&metricsContextCreateParams))) { - return false; - } - - NVPW_MetricsContext_Destroy_Params metricsContextDestroyParams = { - NVPW_MetricsContext_Destroy_Params_STRUCT_SIZE, nullptr}; - metricsContextDestroyParams.pMetricsContext = - metricsContextCreateParams.pMetricsContext; - - - SCOPE_EXIT([&]() { - NVPW_MetricsContext_Destroy( - (NVPW_MetricsContext_Destroy_Params*)&metricsContextDestroyParams); - }); - - // Get all raw metrics required for given metricNames list - std::vector rawMetricRequests; - - // note: we need a variable at this functions scope to hold the string - // pointers for underlying C char arrays. - std::vector rawMetricDeps; - - if (!getRawMetricRequests( - metricsContextCreateParams.pMetricsContext, - metricNames, - rawMetricDeps, - rawMetricRequests)) { - return false; - } - - // Setup Counter Data builder - NVPW_CounterDataBuilder_Create_Params counterDataBuilderCreateParams = { - NVPW_CounterDataBuilder_Create_Params_STRUCT_SIZE, nullptr}; - counterDataBuilderCreateParams.pChipName = chipName.c_str(); - if (!NVPW_CALL( - NVPW_CounterDataBuilder_Create(&counterDataBuilderCreateParams))) { - return false; - } - - NVPW_CounterDataBuilder_Destroy_Params counterDataBuilderDestroyParams = { - NVPW_CounterDataBuilder_Destroy_Params_STRUCT_SIZE, nullptr}; - counterDataBuilderDestroyParams.pCounterDataBuilder = - counterDataBuilderCreateParams.pCounterDataBuilder; - SCOPE_EXIT([&]() { - NVPW_CounterDataBuilder_Destroy(( - NVPW_CounterDataBuilder_Destroy_Params*)&counterDataBuilderDestroyParams); - }); - - // Add metrics to counter data image prefix - NVPW_CounterDataBuilder_AddMetrics_Params addMetricsParams = { - NVPW_CounterDataBuilder_AddMetrics_Params_STRUCT_SIZE, nullptr}; - addMetricsParams.pCounterDataBuilder = - counterDataBuilderCreateParams.pCounterDataBuilder; - addMetricsParams.pRawMetricRequests = rawMetricRequests.data(); - addMetricsParams.numMetricRequests = rawMetricRequests.size(); - if (!NVPW_CALL( - NVPW_CounterDataBuilder_AddMetrics(&addMetricsParams))) { - return false; - } - - // Get image prefix size - NVPW_CounterDataBuilder_GetCounterDataPrefix_Params - getCounterDataPrefixParams = { - NVPW_CounterDataBuilder_GetCounterDataPrefix_Params_STRUCT_SIZE, nullptr}; - getCounterDataPrefixParams.pCounterDataBuilder = - counterDataBuilderCreateParams.pCounterDataBuilder; - getCounterDataPrefixParams.bytesAllocated = 0; - getCounterDataPrefixParams.pBuffer = nullptr; - if (!NVPW_CALL( - NVPW_CounterDataBuilder_GetCounterDataPrefix( - &getCounterDataPrefixParams))) { - return false; - } - - counterDataImagePrefix.resize(getCounterDataPrefixParams.bytesCopied); - - // Now write counter data image prefix - getCounterDataPrefixParams.bytesAllocated = counterDataImagePrefix.size(); - getCounterDataPrefixParams.pBuffer = counterDataImagePrefix.data(); - if (!NVPW_CALL( - NVPW_CounterDataBuilder_GetCounterDataPrefix( - &getCounterDataPrefixParams))) { - return false; - } - - return true; -} - -// ------------------------------------------------- -// Metric and Counter Evaluation Utilities -// ------------------------------------------------- - -std::string getRangeDescription( - const std::vector& counterDataImage, - int rangeIndex) { - std::vector descriptionPtrs; - - NVPW_Profiler_CounterData_GetRangeDescriptions_Params getRangeDescParams = { - NVPW_Profiler_CounterData_GetRangeDescriptions_Params_STRUCT_SIZE, nullptr}; - getRangeDescParams.pCounterDataImage = counterDataImage.data(); - getRangeDescParams.rangeIndex = rangeIndex; - - if (!NVPW_CALL( - NVPW_Profiler_CounterData_GetRangeDescriptions(&getRangeDescParams))) { - return ""; - } - - descriptionPtrs.resize(getRangeDescParams.numDescriptions); - getRangeDescParams.ppDescriptions = descriptionPtrs.data(); - - if (!NVPW_CALL( - NVPW_Profiler_CounterData_GetRangeDescriptions(&getRangeDescParams))) { - return ""; - } - - std::string rangeName; - - for (size_t i = 0; i < getRangeDescParams.numDescriptions; i++) { - if (i > 0) { - rangeName.append("/"); - } - rangeName.append(descriptionPtrs[i]); - } - return rangeName; -} - -CuptiProfilerResult evalMetricValues( - const std::string& chipName, - const std::vector& counterDataImage, - const std::vector& metricNames, - bool verbose) { - - if (!counterDataImage.size()) { - LOG(ERROR) << "Counter Data Image is empty!"; - return {}; - } - - NVPW_CUDA_MetricsContext_Create_Params metricsContextCreateParams = { - NVPW_CUDA_MetricsContext_Create_Params_STRUCT_SIZE, nullptr}; - metricsContextCreateParams.pChipName = chipName.c_str(); - if (!NVPW_CALL( - NVPW_CUDA_MetricsContext_Create(&metricsContextCreateParams))) { - return {}; - } - - NVPW_MetricsContext_Destroy_Params metricsContextDestroyParams = { - NVPW_MetricsContext_Destroy_Params_STRUCT_SIZE, nullptr}; - metricsContextDestroyParams.pMetricsContext = - metricsContextCreateParams.pMetricsContext; - SCOPE_EXIT([&]() { - NVPW_MetricsContext_Destroy( - (NVPW_MetricsContext_Destroy_Params*)&metricsContextDestroyParams); - }); - - NVPW_CounterData_GetNumRanges_Params getNumRangesParams = { - NVPW_CounterData_GetNumRanges_Params_STRUCT_SIZE, nullptr}; - getNumRangesParams.pCounterDataImage = counterDataImage.data(); - if (!NVPW_CALL( - NVPW_CounterData_GetNumRanges(&getNumRangesParams))) { - return {}; - } - - // TBD in the future support special chars in metric name - // for now these are default - const bool isolated = true; - - // API takes a 2D array of chars - std::vector metricNamePtrs; - - for (const auto& metric : metricNames) { - metricNamePtrs.push_back(metric.c_str()); - } - - CuptiProfilerResult result{ - .metricNames = metricNames}; - - for (size_t rangeIndex = 0; rangeIndex < getNumRangesParams.numRanges; - ++rangeIndex) { - - CuptiRangeMeasurement rangeData { - .rangeName = getRangeDescription(counterDataImage, rangeIndex)}; - rangeData.values.resize(metricNames.size()); - - // First set Counter data image with current range - NVPW_MetricsContext_SetCounterData_Params setCounterDataParams = { - NVPW_MetricsContext_SetCounterData_Params_STRUCT_SIZE, nullptr}; - - setCounterDataParams.pMetricsContext = - metricsContextCreateParams.pMetricsContext; - setCounterDataParams.pCounterDataImage = counterDataImage.data(); - setCounterDataParams.isolated = isolated; - setCounterDataParams.rangeIndex = rangeIndex; - - NVPW_CALL(NVPW_MetricsContext_SetCounterData(&setCounterDataParams)); - - - // Now we can evaluate GPU metrics - NVPW_MetricsContext_EvaluateToGpuValues_Params evalToGpuParams = { - NVPW_MetricsContext_EvaluateToGpuValues_Params_STRUCT_SIZE, nullptr}; - evalToGpuParams.pMetricsContext = - metricsContextCreateParams.pMetricsContext; - evalToGpuParams.numMetrics = metricNamePtrs.size(); - evalToGpuParams.ppMetricNames = metricNamePtrs.data(); - evalToGpuParams.pMetricValues = rangeData.values.data(); - - if (!NVPW_CALL(NVPW_MetricsContext_EvaluateToGpuValues(&evalToGpuParams))) { - LOG(WARNING) << "Failed to evaluate metris for range : " - << rangeData.rangeName; - continue; - } - - if (verbose) { - for (size_t i = 0; i < metricNames.size(); i++) { - LOG(INFO) << "rangeName: " << rangeData.rangeName - << "\tmetricName: " << metricNames[i] - << "\tgpuValue: " << rangeData.values[i]; - } - } - - result.rangeVals.emplace_back(std::move(rangeData)); - } - - return result; -} - -#else - -bool getProfilerConfigImage( - const std::string& /*chipName*/, - const std::vector& /*metricNames*/, - std::vector& /*configImage*/, - const uint8_t* /*counterAvailabilityImage*/) { - return false; -} - -bool getCounterDataPrefixImage( - const std::string& /*chipName*/, - const std::vector& /*metricNames*/, - std::vector& /*counterDataImagePrefix*/) { - return false; -} - -CuptiProfilerResult evalMetricValues( - const std::string& /*chipName*/, - const std::vector& /*counterDataImage*/, - const std::vector& /*metricNames*/, - bool /*verbose*/) { - return {}; -} - -#endif // cuda version > 10.00 and < 11.04 - -} // namespace nvperf -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiNvPerfMetric.h b/plugins/tensorboard-plugins/libkineto/src/CuptiNvPerfMetric.h deleted file mode 100644 index d5dd1b1c1..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiNvPerfMetric.h +++ /dev/null @@ -1,71 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "Logger.h" - -namespace KINETO_NAMESPACE { - -struct CuptiRangeMeasurement { - std::string rangeName; - std::vector values; -}; - -struct CuptiProfilerResult { - std::vector metricNames; - // rangeName, list values - std::vector rangeVals; -}; - -/* Utilities for CUPTI and NVIDIA PerfWorks Metric API - */ - -#define NVPW_CALL(call) \ - [&]() -> bool { \ - NVPA_Status _status_ = call; \ - if (_status_ != NVPA_STATUS_SUCCESS) { \ - LOG(WARNING) << fmt::format( \ - "function {} failed with error ({})", \ - #call, \ - (int)_status_); \ - return false; \ - } \ - return true; \ - }() - -// fixme - add a results string -// nvpperfGetResultString(_status_, &_errstr_); - -namespace nvperf { - -// Setup CUPTI profiler configuration blob and counter data image prefix -bool getProfilerConfigImage( - const std::string& chipName, - const std::vector& metricNames, - std::vector& configImage, - const uint8_t* counterAvailabilityImage = nullptr); - -// Setup CUPTI profiler configuration blob and counter data image prefix -bool getCounterDataPrefixImage( - const std::string& chipName, - const std::vector& metricNames, - std::vector& counterDataImagePrefix); - -/* NV Perf Metric Evaluation helpers - * - utilities to read binary data and obtain metrics for ranges - */ -CuptiProfilerResult evalMetricValues( - const std::string& chipName, - const std::vector& counterDataImage, - const std::vector& metricNames, - bool verbose = false); - - -} // namespace nvperf -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerApi.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerApi.cpp deleted file mode 100644 index e5f18ed7b..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerApi.cpp +++ /dev/null @@ -1,751 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include -#ifdef HAS_CUPTI -#include -#include -#endif // HAS_CUPTI -#include -#include - -#ifdef HAS_CUPTI -#include "cupti_call.h" -#endif - -#include "time_since_epoch.h" -#include "Logger.h" -#include "Demangle.h" - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "CuptiCallbackApiMock.h" -#include "CuptiRangeProfilerApi.h" - -#if HAS_CUPTI_RANGE_PROFILER -#include -#include -#include "cupti_call.h" -#endif // HAS_CUPTI_RANGE_PROFILER - -namespace KINETO_NAMESPACE { - -#if HAS_CUPTI_RANGE_PROFILER -constexpr char kRootUserRangeName[] = "__profile__"; -constexpr int kCallbacksCountToFlush = 500; - -// Should we set Counter availability image ourselves? -// Disabled this right now as this call conflicts with DCGM -// It is not clear why it should conflict except it being a profiler API call -// TODO Revisit -constexpr bool kSetCounterAvail = false; - -// Shared state to track one Cupti Profiler API per Device -namespace { -// per device profiler maps -std::unordered_map profiler_map; -std::unordered_map enable_flag; -std::unordered_map disable_flag; - -std::mutex contextMutex_; -std::unordered_map ctx_to_dev; -std::set active_devices; -} - -// forward declarations -void __trackCudaCtx(CUcontext ctx, uint32_t device_id, CUpti_CallbackId cbid); -void __trackCudaKernelLaunch(CUcontext ctx, const char* kernelName); - -/// Helper functions - -// Available raw counters -std::vector getCounterAvailiability(CUcontext cuContext) { - std::vector counterAvailabilityImage; - CUpti_Profiler_GetCounterAvailability_Params getCounterAvailabilityParams = { - CUpti_Profiler_GetCounterAvailability_Params_STRUCT_SIZE, nullptr}; - getCounterAvailabilityParams.ctx = cuContext; - CUPTI_CALL( - cuptiProfilerGetCounterAvailability(&getCounterAvailabilityParams)); - - counterAvailabilityImage.clear(); - counterAvailabilityImage.resize( - getCounterAvailabilityParams.counterAvailabilityImageSize); - - getCounterAvailabilityParams.pCounterAvailabilityImage = - counterAvailabilityImage.data(); - CUPTI_CALL( - cuptiProfilerGetCounterAvailability(&getCounterAvailabilityParams)); - - return counterAvailabilityImage; -} - -std::string getChipName(int deviceId) { - // Get chip name for the cuda device - CUpti_Device_GetChipName_Params getChipNameParams = { - CUpti_Device_GetChipName_Params_STRUCT_SIZE, nullptr}; - - getChipNameParams.deviceIndex = deviceId; - CUPTI_CALL(cuptiDeviceGetChipName(&getChipNameParams)); - - return getChipNameParams.pChipName; -} - -inline uint32_t getDevID(CUcontext ctx) { - uint32_t device_id = UINT32_MAX; - CUPTI_CALL(cuptiGetDeviceId(ctx, &device_id)); - if (device_id == UINT32_MAX) { - LOG(ERROR) << "Could not determine dev id for = " << ctx; - } - return device_id; -} - -// We use CUPTI Callback functions in three ways : -// 1. Track cuda contexts and maintain a list of active GPUs to profile -// 2. Callbacks on kernel launches to track the name of automatic -// ranges that correspond to names of kernels -// 3. Lastly CUPTI profiler has to be enabled on the same thread executing -// the CUDA kernels. We use Callbacks to enable the profiler -// asynchronously from another thread. - -void disableKernelCallbacks(); - -void trackCudaCtx( - CUpti_CallbackDomain /*domain*/, - CUpti_CallbackId cbid, - const CUpti_CallbackData* cbInfo) { - auto *d = reinterpret_cast(cbInfo); - auto ctx = d->context; - uint32_t device_id = getDevID(ctx); - - if (device_id == UINT32_MAX) { - return; - } - - __trackCudaCtx(ctx, device_id, cbid); -} - -void __trackCudaCtx(CUcontext ctx, uint32_t device_id, CUpti_CallbackId cbid) { - std::lock_guard g(contextMutex_); - if (cbid == CUPTI_CBID_RESOURCE_CONTEXT_CREATED) { - VLOG(0) << "CUPTI Profiler observed CUDA Context created = " - << ctx << " device id = " << device_id; - active_devices.insert(device_id); - if constexpr (kSetCounterAvail) { - if (active_devices.size() == 1) { - CuptiRBProfilerSession::setCounterAvailabilityImage( - getCounterAvailiability(ctx)); - } - } - ctx_to_dev[ctx] = device_id; - - } else if (cbid == CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING) { - VLOG(0) << "CUPTI Profiler observed CUDA Context destroyed = " - << ctx << " device id = " << device_id; - auto it = active_devices.find(device_id); - if (it != active_devices.end()) { - active_devices.erase(it); - ctx_to_dev.erase(ctx); - } - } -} - -void trackCudaKernelLaunch( - CUpti_CallbackDomain /*domain*/, - CUpti_CallbackId /*cbid*/, - const CUpti_CallbackData* cbInfo) { - VLOG(1) << " Trace : Callback name = " - << (cbInfo->symbolName ? cbInfo->symbolName: "") - << " context ptr = " << cbInfo->context; - auto ctx = cbInfo->context; - // should be in CUPTI_API_ENTER call site - if (cbInfo->callbackSite != CUPTI_API_ENTER) { - return; - } - __trackCudaKernelLaunch(ctx, cbInfo->symbolName); -} - -void __trackCudaKernelLaunch( - CUcontext ctx, - const char* kernelName) { - VLOG(0) << " Tracking kernel name = " << (kernelName ? kernelName : "") - << " context ptr = " << ctx; - - uint32_t device_id = 0; - auto it = ctx_to_dev.find(ctx); - if (it == ctx_to_dev.end()) { - // Warning here could be too noisy - VLOG(0) << " Could not find corresponding device to ctx = " << ctx; - return; - } else { - device_id = it->second; - } - - auto pit = profiler_map.find(device_id); - if (pit == profiler_map.end() || pit->second == nullptr) { - return; - } - auto profiler = pit->second; - - if (enable_flag[device_id]) { - LOG(INFO) << "Callback handler is enabling cupti profiler"; - profiler->startAndEnable(); - enable_flag[device_id] = false; - - } else if (disable_flag[device_id]) { - LOG(INFO) << "Callback handler is disabling cupti profiler"; - profiler->disableAndStop(); - return; - } - - if (profiler->curRange_ == CUPTI_AutoRange) { - profiler->logKernelName(kernelName ? kernelName : "__missing__"); - } - - /* TODO add per kernel time logging - if (measure_per_kernel) { - profiler->kernelStartTs_.push_back( - std::chrono::high_resolution_clock::now()); - } - */ - - // periodically flush profiler data from GPU - if (profiler->numCallbacks_ % kCallbacksCountToFlush == 0) { - profiler->flushCounterData(); - } - profiler->numCallbacks_++; -} - -void enableKernelCallbacks() { - auto& cbapi = CuptiCallbackApi::singleton(); - bool status = cbapi.enableCallback( - CUPTI_CB_DOMAIN_RUNTIME_API, - CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000); - if (!status) { - LOG(WARNING) << "CUPTI Range Profiler unable to " - << "enable cuda kernel launch callback"; - return; - } - LOG(INFO) << "CUPTI Profiler kernel callbacks enabled"; -} - -void disableKernelCallbacks() { - auto& cbapi = CuptiCallbackApi::singleton(); - bool status = cbapi.disableCallback( - CUPTI_CB_DOMAIN_RUNTIME_API, - CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000); - if (!status) { - LOG(WARNING) << "CUPTI Range Profiler unable to " - << "disable cuda kernel launch callback"; - return; - } - LOG(INFO) << "CUPTI Profiler kernel callbacks disabled"; -} - -// static -std::set CuptiRBProfilerSession::getActiveDevices() { - std::lock_guard g(contextMutex_); - return active_devices; -} - -// static -void CuptiRBProfilerSession::initCupti() { - CUpti_Profiler_Initialize_Params profilerInitializeParams = { - CUpti_Profiler_Initialize_Params_STRUCT_SIZE, nullptr}; - CUPTI_CALL(cuptiProfilerInitialize(&profilerInitializeParams)); -} - -// static -void CuptiRBProfilerSession::deInitCupti() { - CUpti_Profiler_DeInitialize_Params profilerDeInitializeParams = { - CUpti_Profiler_DeInitialize_Params_STRUCT_SIZE, nullptr}; - CUPTI_CALL(cuptiProfilerDeInitialize(&profilerDeInitializeParams)); -} - -// static -void CuptiRBProfilerSession::staticInit() { - CuptiRBProfilerSession::initCupti(); - - // Register CUPTI callbacks - auto& cbapi = CuptiCallbackApi::singleton(); - CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RESOURCE; - bool status = cbapi.registerCallback( - domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED, trackCudaCtx); - status = status && cbapi.registerCallback( - domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED, trackCudaCtx); - status = status && cbapi.enableCallback( - domain, CUPTI_CBID_RESOURCE_CONTEXT_CREATED); - status = status && cbapi.enableCallback( - domain, CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING); - - if (!status) { - LOG(WARNING) << "CUPTI Range Profiler unable to attach cuda context " - << "create and destroy callbacks"; - CUPTI_CALL(cbapi.getCuptiStatus()); - return; - } - - domain = CUPTI_CB_DOMAIN_RUNTIME_API; - status = cbapi.registerCallback( - domain, CuptiCallbackApi::CUDA_LAUNCH_KERNEL, trackCudaKernelLaunch); - - if (!status) { - LOG(WARNING) << "CUPTI Range Profiler unable to attach cuda kernel " - << "launch callback"; - return; - } -} - -// static -std::vector& CuptiRBProfilerSession::counterAvailabilityImage() { - static std::vector counterAvailabilityImage_; - return counterAvailabilityImage_; -} - - -// Setup the profiler sessions -CuptiRBProfilerSession::CuptiRBProfilerSession( - const std::vector& metricNames, - int deviceId, - int maxRanges, - int numNestingLevels, - CUcontext cuContext) - : metricNames_(metricNames), - chipName_(getChipName(deviceId)), - deviceId_(deviceId), - maxRanges_(maxRanges), - numNestingLevels_(numNestingLevels), - cuContext_(cuContext) { - CuptiRBProfilerSession::initCupti(); - - LOG(INFO) << "Initializing CUPTI profiler session : device = " << deviceId - << " chip = " << chipName_; - /* Generate configuration for metrics, this can also be done offline*/ - NVPW_InitializeHost_Params initializeHostParams = { - NVPW_InitializeHost_Params_STRUCT_SIZE, nullptr}; - NVPW_CALL(NVPW_InitializeHost(&initializeHostParams)); - - if (metricNames.size()) { - if (!nvperf::getProfilerConfigImage( - chipName_, - metricNames, - configImage, - CuptiRBProfilerSession::counterAvailabilityImage().data())) { - LOG(ERROR) << "Failed to create configImage or counterDataImagePrefix"; - return; - } - if (!nvperf::getCounterDataPrefixImage( - chipName_, - metricNames, - counterDataImagePrefix)) { - LOG(ERROR) << "Failed to create counterDataImagePrefix"; - return; - } - } else { - LOG(ERROR) << "No metrics provided to profile"; - return; - } - - if (!createCounterDataImage()) { - LOG(ERROR) << "Failed to create counterDataImage"; - return; - } - - LOG(INFO) << "Size of structs\n" - << " config image size = " << configImage.size() << " B" - << " counter data image prefix = " - << counterDataImagePrefix.size() << " B" - << " counter data image size = " << counterDataImage.size() / 1024 - << " KB" - << " counter sb image size = " - << counterDataScratchBuffer.size() << " B"; - - beginPassParams_ = {CUpti_Profiler_BeginPass_Params_STRUCT_SIZE, nullptr}; - endPassParams_ = {CUpti_Profiler_EndPass_Params_STRUCT_SIZE, nullptr}; - - initSuccess_ = true; - profiler_map[deviceId] = this; -} - -// used in unittests only -CuptiRBProfilerSession::CuptiRBProfilerSession(int deviceId, CUcontext ctx) - : deviceId_(deviceId), cuContext_(ctx) { - initSuccess_ = true; - profiler_map[deviceId] = this; -} - -void CuptiRBProfilerSession::startInternal( - CUpti_ProfilerRange profilerRange, - CUpti_ProfilerReplayMode profilerReplayMode) { - LOG(INFO) << "Starting profiler session: profiler range = " - << ((profilerRange == CUPTI_AutoRange) ? "autorange" : "userrange") - << " replay mode = " - << ((profilerReplayMode == CUPTI_KernelReplay) ? "kernel" : "user"); - if (!initSuccess_) { - LOG(WARNING) << __func__ << "() bailing out since initialization failed"; - return; - } - - if (cuContext_ == nullptr) { - for (const auto& it : ctx_to_dev) { - if (it.second == deviceId_) { - cuContext_ = it.first; - break; - } - } - LOG(INFO) << " Cupti Profiler using CUDA context = " << cuContext_; - } - - profilerStartTs_ = std::chrono::high_resolution_clock::now(); - curRange_ = profilerRange; - curReplay_ = profilerReplayMode; - - CUpti_Profiler_BeginSession_Params beginSessionParams = { - CUpti_Profiler_BeginSession_Params_STRUCT_SIZE, nullptr}; - - beginSessionParams.ctx = cuContext_; - beginSessionParams.counterDataImageSize = counterDataImage.size(); - beginSessionParams.pCounterDataImage = counterDataImage.data(); - beginSessionParams.counterDataScratchBufferSize = - counterDataScratchBuffer.size(); - beginSessionParams.pCounterDataScratchBuffer = counterDataScratchBuffer.data(); - beginSessionParams.range = profilerRange; - beginSessionParams.replayMode = profilerReplayMode; - beginSessionParams.maxRangesPerPass = maxRanges_; - beginSessionParams.maxLaunchesPerPass = maxRanges_; - - auto status = CUPTI_CALL(cuptiProfilerBeginSession(&beginSessionParams)); - if (status != CUPTI_SUCCESS) { - LOG(WARNING) << "Failed to start CUPTI profiler"; - initSuccess_ = false; - return; - } - - // Set counter configuration - CUpti_Profiler_SetConfig_Params setConfigParams = { - CUpti_Profiler_SetConfig_Params_STRUCT_SIZE, nullptr}; - - setConfigParams.ctx = cuContext_; - setConfigParams.pConfig = configImage.data(); - setConfigParams.configSize = configImage.size(); - setConfigParams.passIndex = 0; - setConfigParams.minNestingLevel = 1; - setConfigParams.numNestingLevels = numNestingLevels_; - status = CUPTI_CALL(cuptiProfilerSetConfig(&setConfigParams)); - - if (status != CUPTI_SUCCESS) { - LOG(WARNING) << "Failed to configure CUPTI profiler"; - initSuccess_ = false; - return; - } - profilerInitDoneTs_ = std::chrono::high_resolution_clock::now(); - - if (curRange_ == CUPTI_AutoRange) { - enableKernelCallbacks(); - } - profilingActive_ = true; -} - -void CuptiRBProfilerSession::stop() { - if (!initSuccess_) { - LOG(WARNING) << __func__ << "() bailing out since initialization failed"; - return; - } - LOG(INFO) << "Stop profiler session on device = " << deviceId_; - - CUpti_Profiler_UnsetConfig_Params unsetConfigParams = { - CUpti_Profiler_UnsetConfig_Params_STRUCT_SIZE, nullptr}; - CUPTI_CALL(cuptiProfilerUnsetConfig(&unsetConfigParams)); - - CUpti_Profiler_EndSession_Params endSessionParams = { - CUpti_Profiler_EndSession_Params_STRUCT_SIZE, nullptr}; - CUPTI_CALL(cuptiProfilerEndSession(&endSessionParams)); - - disableKernelCallbacks(); - - profilerStopTs_ = std::chrono::high_resolution_clock::now(); - profilingActive_ = false; -} - -void CuptiRBProfilerSession::beginPass() { - if (!initSuccess_) { - LOG(WARNING) << __func__ << "() bailing out since initialization failed"; - return; - } - CUPTI_CALL(cuptiProfilerBeginPass(&beginPassParams_)); -} - -bool CuptiRBProfilerSession::endPass() { - if (!initSuccess_) { - LOG(WARNING) << __func__ << "() bailing out since initialization failed"; - return true; - } - CUPTI_CALL(cuptiProfilerEndPass(&endPassParams_)); - return endPassParams_.allPassesSubmitted; -} - -void CuptiRBProfilerSession::flushCounterData() { - LOG(INFO) << "Flushing counter data on device = " << deviceId_; - CUpti_Profiler_FlushCounterData_Params flushCounterDataParams = { - CUpti_Profiler_FlushCounterData_Params_STRUCT_SIZE, nullptr}; - CUPTI_CALL(cuptiProfilerFlushCounterData(&flushCounterDataParams)); -} - -/// Enable and disable the profiler -void CuptiRBProfilerSession::enable() { - if (!initSuccess_) { - LOG(WARNING) << __func__ << "() bailing out since initialization failed"; - return; - } - CUpti_Profiler_EnableProfiling_Params enableProfilingParams = { - CUpti_Profiler_EnableProfiling_Params_STRUCT_SIZE, nullptr}; - CUPTI_CALL(cuptiProfilerEnableProfiling(&enableProfilingParams)); -} - -void CuptiRBProfilerSession::disable() { - if (!initSuccess_) { - LOG(WARNING) << __func__ << "() bailing out since initialization failed"; - return; - } - CUpti_Profiler_DisableProfiling_Params disableProfilingParams = { - CUpti_Profiler_DisableProfiling_Params_STRUCT_SIZE, nullptr}; - CUPTI_CALL(cuptiProfilerDisableProfiling(&disableProfilingParams)); -} - -/// User range based profiling -void CuptiRBProfilerSession::pushRange(const std::string& rangeName) { - LOG(INFO) << " CUPTI pushrange ( " << rangeName << " )"; - CUpti_Profiler_PushRange_Params pushRangeParams = { - CUpti_Profiler_PushRange_Params_STRUCT_SIZE, nullptr}; - pushRangeParams.pRangeName = rangeName.c_str(); - CUPTI_CALL(cuptiProfilerPushRange(&pushRangeParams)); -} - -void CuptiRBProfilerSession::popRange() { - LOG(INFO) << " CUPTI pop range"; - CUpti_Profiler_PopRange_Params popRangeParams = { - CUpti_Profiler_PopRange_Params_STRUCT_SIZE, nullptr}; - CUPTI_CALL(cuptiProfilerPopRange(&popRangeParams)); -} - -void CuptiRBProfilerSession::startAndEnable() { - startInternal(curRange_, curReplay_); - if (curReplay_ == CUPTI_UserReplay) { - beginPass(); - } - enable(); - if (curRange_ == CUPTI_UserRange) { - pushRange(kRootUserRangeName); - } - enable_flag[deviceId_] = false; -} - -void CuptiRBProfilerSession::disableAndStop() { - if (curRange_ == CUPTI_UserRange) { - popRange(); - } - disable(); - if (curReplay_ == CUPTI_UserReplay) { - endPass(); - flushCounterData(); - } - stop(); - disable_flag[deviceId_] = false; -} - -void CuptiRBProfilerSession::asyncStartAndEnable( - CUpti_ProfilerRange profilerRange, - CUpti_ProfilerReplayMode profilerReplayMode) { - LOG(INFO) << "Starting CUPTI profiler asynchronously on device = " - << deviceId_ << " profiler range = " - << ((profilerRange == CUPTI_AutoRange) ? "autorange" : "userrange") - << " replay mode = " - << ((profilerReplayMode == CUPTI_KernelReplay) ? "kernel" : "user"); - curReplay_ = profilerReplayMode; - curRange_ = profilerRange; - enable_flag[deviceId_] = true; - enableKernelCallbacks(); -} - -void CuptiRBProfilerSession::asyncDisableAndStop() { - LOG(INFO) << "Stopping CUPTI profiler asynchronously on device = " - << deviceId_ << " cu context = " << cuContext_; - disable_flag[deviceId_] = true; -} - - -CuptiProfilerResult CuptiRBProfilerSession::evaluateMetrics( - bool verbose) { - if (!initSuccess_) { - LOG(WARNING) << "Profiling failed, no results to return"; - return {}; - } - if (profilingActive_) { - disableAndStop(); - } - - LOG(INFO) << "Total kernels logged = " << kernelNames_.size(); - if (verbose) { - for (const auto& kernel : kernelNames_) { - std::cout << demangle(kernel) << std::endl; - } - LOG(INFO) << "Profiler Range data : "; - } - - auto results = nvperf::evalMetricValues( - chipName_, counterDataImage, metricNames_, verbose /*verbose*/); - - // profiler end-end duration - auto duration_ms = std::chrono::duration_cast( - profilerStopTs_ - profilerStartTs_); - - auto init_dur_ms = std::chrono::duration_cast( - profilerInitDoneTs_ - profilerStartTs_); - LOG(INFO) << "Total profiler time = " << duration_ms.count() << " ms"; - LOG(INFO) << "Total profiler init time = " << init_dur_ms.count() << " ms"; - - return results; -} - -std::unique_ptr CuptiRBProfilerSession::getProfilerTraceSpan() { - return std::make_unique( - timeSinceEpoch(profilerStartTs_), - timeSinceEpoch(profilerStopTs_), - "__cupti_profiler__" - ); -} - -void CuptiRBProfilerSession::saveCounterData( - const std::string& /*CounterDataFileName*/, - const std::string& /*CounterDataSBFileName*/) { - /* TBD write binary files for counter data and counter scratch buffer */ -} - -/// Setup counter data -bool CuptiRBProfilerSession::createCounterDataImage() { - CUpti_Profiler_CounterDataImageOptions counterDataImageOptions; - counterDataImageOptions.pCounterDataPrefix = counterDataImagePrefix.data(); - counterDataImageOptions.counterDataPrefixSize = counterDataImagePrefix.size(); - counterDataImageOptions.maxNumRanges = maxRanges_; - counterDataImageOptions.maxNumRangeTreeNodes = maxRanges_; - counterDataImageOptions.maxRangeNameLength = 64; - - // Calculate size of counter data image - CUpti_Profiler_CounterDataImage_CalculateSize_Params calculateSizeParams = { - CUpti_Profiler_CounterDataImage_CalculateSize_Params_STRUCT_SIZE, nullptr}; - calculateSizeParams.pOptions = &counterDataImageOptions; - calculateSizeParams.sizeofCounterDataImageOptions = - CUpti_Profiler_CounterDataImageOptions_STRUCT_SIZE; - - CUPTI_CALL( - cuptiProfilerCounterDataImageCalculateSize(&calculateSizeParams)); - counterDataImage.resize(calculateSizeParams.counterDataImageSize); - - // Initialize counter data image - CUpti_Profiler_CounterDataImage_Initialize_Params initializeParams = { - CUpti_Profiler_CounterDataImage_Initialize_Params_STRUCT_SIZE, nullptr}; - initializeParams.sizeofCounterDataImageOptions = - CUpti_Profiler_CounterDataImageOptions_STRUCT_SIZE; - initializeParams.pOptions = &counterDataImageOptions; - initializeParams.counterDataImageSize = - calculateSizeParams.counterDataImageSize; - initializeParams.pCounterDataImage = counterDataImage.data(); - CUPTI_CALL(cuptiProfilerCounterDataImageInitialize(&initializeParams)); - - // Calculate counter Scratch Buffer size - CUpti_Profiler_CounterDataImage_CalculateScratchBufferSize_Params - scratchBufferSizeParams = { - CUpti_Profiler_CounterDataImage_CalculateScratchBufferSize_Params_STRUCT_SIZE, nullptr}; - - scratchBufferSizeParams.counterDataImageSize = - calculateSizeParams.counterDataImageSize; - scratchBufferSizeParams.pCounterDataImage = - initializeParams.pCounterDataImage; - CUPTI_CALL(cuptiProfilerCounterDataImageCalculateScratchBufferSize( - &scratchBufferSizeParams)); - - counterDataScratchBuffer.resize( - scratchBufferSizeParams.counterDataScratchBufferSize); - - // Initialize scratch buffer - CUpti_Profiler_CounterDataImage_InitializeScratchBuffer_Params - initScratchBufferParams = { - CUpti_Profiler_CounterDataImage_InitializeScratchBuffer_Params_STRUCT_SIZE, nullptr}; - - initScratchBufferParams.counterDataImageSize = - calculateSizeParams.counterDataImageSize; - - initScratchBufferParams.pCounterDataImage = - initializeParams.pCounterDataImage; - initScratchBufferParams.counterDataScratchBufferSize = - scratchBufferSizeParams.counterDataScratchBufferSize; - initScratchBufferParams.pCounterDataScratchBuffer = - counterDataScratchBuffer.data(); - - CUPTI_CALL(cuptiProfilerCounterDataImageInitializeScratchBuffer( - &initScratchBufferParams)); - - return true; -} - -#elif defined(HAS_CUPTI) - -// Create empty stubs for the API when CUPTI is not present. -CuptiRBProfilerSession::CuptiRBProfilerSession( - const std::vector& metricNames, - int deviceId, - int maxRanges, - int numNestingLevels, - CUcontext cuContext) - : metricNames_(metricNames), - deviceId_(deviceId), - maxRanges_(maxRanges), - numNestingLevels_(numNestingLevels), - cuContext_(cuContext) {} -void CuptiRBProfilerSession::stop() {} -void CuptiRBProfilerSession::enable() {} -void CuptiRBProfilerSession::disable() {} -void CuptiRBProfilerSession::beginPass() {} -bool CuptiRBProfilerSession::endPass() { return true; } -void CuptiRBProfilerSession::flushCounterData() {} -void CuptiRBProfilerSession::pushRange(const std::string& /*rangeName*/) {} -void CuptiRBProfilerSession::popRange() {} -void CuptiRBProfilerSession::asyncStartAndEnable( - CUpti_ProfilerRange /*profilerRange*/, - CUpti_ProfilerReplayMode /*profilerReplayMode*/) {} -void CuptiRBProfilerSession::asyncDisableAndStop() {} -CuptiProfilerResult CuptiRBProfilerSession::evaluateMetrics(bool verbose) { - static CuptiProfilerResult res; - return res; -}; -void CuptiRBProfilerSession::saveCounterData( - const std::string& /*CounterDataFileName*/, - const std::string& /*CounterDataSBFileName*/) {} -void CuptiRBProfilerSession::initCupti() {} -void CuptiRBProfilerSession::deInitCupti() {} -void CuptiRBProfilerSession::staticInit() {} -bool CuptiRBProfilerSession::createCounterDataImage() { return true; } -void CuptiRBProfilerSession::startInternal( - CUpti_ProfilerRange /*profilerRange*/, - CUpti_ProfilerReplayMode /*profilerReplayMode*/) {} -std::vector& CuptiRBProfilerSession::counterAvailabilityImage() { - static std::vector _vec; - return _vec; -} -#endif // HAS_CUPTI_RANGE_PROFILER - -namespace testing { - -void trackCudaCtx(CUcontext ctx, uint32_t device_id, CUpti_CallbackId cbid) { -#if HAS_CUPTI_RANGE_PROFILER - __trackCudaCtx(ctx, device_id, cbid); -#endif // HAS_CUPTI_RANGE_PROFILER -} - -void trackCudaKernelLaunch(CUcontext ctx, const char* kernelName) { -#if HAS_CUPTI_RANGE_PROFILER - __trackCudaKernelLaunch(ctx, kernelName); -#endif // HAS_CUPTI_RANGE_PROFILER -} - -} // namespace testing -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerApi.h b/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerApi.h deleted file mode 100644 index 98a0b3ea5..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerApi.h +++ /dev/null @@ -1,220 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#ifdef HAS_CUPTI -#include -#include -// Using CUDA 11 and above due to usage of API: cuptiProfilerGetCounterAvailability. -#if defined(CUDART_VERSION) && CUDART_VERSION >= 10000 && CUDART_VERSION < 11040 && CUDA_VERSION >= 11000 -#define HAS_CUPTI_RANGE_PROFILER 1 -#endif // CUDART_VERSION > 10.00 and < 11.04 && CUDA_VERSION >= 11.00 -#endif // HAS_CUPTI - -#if HAS_CUPTI_RANGE_PROFILER -#include -#include -#include -#else -using CUpti_ProfilerRange = enum -{ - CUPTI_AutoRange, - CUPTI_UserRange, -}; - -using CUpti_ProfilerReplayMode = enum -{ - CUPTI_KernelReplay, - CUPTI_UserReplay, -}; -#endif // HAS_CUPTI_RANGE_PROFILER - -#include -#include -#include -#include -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "TraceSpan.h" -#include "CuptiCallbackApi.h" -#include "CuptiNvPerfMetric.h" - -/* Cupti Range based profiler session - * See : https://docs.nvidia.com/cupti/Cupti/r_main.html#r_profiler - */ - -namespace KINETO_NAMESPACE { - -class CuptiRBProfilerSession { - public: - // Initialize and configure CUPTI Profiler counters. - // - Metric names must be provided as string vector. - // - Supported values by CUPTI can be found at - - // https://docs.nvidia.com/cupti/Cupti/r_main.html#r_host_metrics_api - explicit CuptiRBProfilerSession( - const std::vector& metricNames, - int deviceId, - int maxRanges, - int numNestingLevels = 1, - CUcontext cuContext = 0); - - virtual ~CuptiRBProfilerSession() = default; - - // Start profiling session - // This function has to be called from the CPU thread running - // the CUDA context. If this is not the case asyncStartAndEnable() - // can be used - void start( - CUpti_ProfilerRange profilerRange = CUPTI_AutoRange, - CUpti_ProfilerReplayMode profilerReplayMode = CUPTI_KernelReplay) { - startInternal(profilerRange, profilerReplayMode); - } - - // Stop profiling session - virtual void stop(); - - virtual void enable(); - virtual void disable(); - - // Profiler passes - // GPU hardware has limited performance monitoring resources - // the CUPTI profiler may need to run multiple passes to collect - // data for a given range - // If we use kernel replay model the kernels are automatically replayed - // else, you can use the beginPass() and endPass() functions below - // for user to manage the replays - - // starts a profiler pass with given kernels in between - virtual void beginPass(); - - // end a profiler pass with given kernels in between - // returns true if no more passes are required - virtual bool endPass(); - - // flushes the counter data - required if you use user replay - virtual void flushCounterData(); - - // Each pass can contain multiple of ranges - // metrics configured in a pass are collected per each range-stack. - virtual void pushRange(const std::string& rangeName); - virtual void popRange(); - - // utilities for common operations - void startAndEnable(); - void disableAndStop(); - - // Async APIs : these will can be called from another thread - // outside the CUDA context being profiled - void asyncStartAndEnable( - CUpti_ProfilerRange profilerRange = CUPTI_AutoRange, - CUpti_ProfilerReplayMode profilerReplayMode = CUPTI_KernelReplay); - void asyncDisableAndStop(); - - void printMetrics() { - evaluateMetrics(true); - } - - std::unique_ptr getProfilerTraceSpan(); - - virtual CuptiProfilerResult evaluateMetrics(bool verbose = false); - - void saveCounterData( - const std::string& CounterDataFileName, - const std::string& CounterDataSBFileName); - - // This is not thread safe so please only call after - // profiling has stopped - const std::vector& getKernelNames() const { - return kernelNames_; - } - - int deviceId() const { - return deviceId_; - } - - bool profilingActive() const { - return profilingActive_; - } - - static std::set getActiveDevices(); - - static void initCupti(); - - static void deInitCupti(); - - static void staticInit(); - - static void setCounterAvailabilityImage(std::vector img) { - counterAvailabilityImage() = img; - } - protected: - CuptiRBProfilerSession(int deviceId, CUcontext ctx); - - virtual void startInternal( - CUpti_ProfilerRange profilerRange, - CUpti_ProfilerReplayMode profilerReplayMode); - - CUpti_ProfilerRange curRange_ = CUPTI_AutoRange; - CUpti_ProfilerReplayMode curReplay_ = CUPTI_KernelReplay; - - private: - - bool createCounterDataImage(); - - - // log kernel name that used with callbacks - void logKernelName(const char* kernel) { - std::lock_guard lg(kernelNamesMutex_); - kernelNames_.emplace_back(kernel); - } - - std::vector metricNames_; - std::string chipName_; - - uint32_t deviceId_ = 0; - int maxRanges_; - int numNestingLevels_; - CUcontext cuContext_; - - - // data buffers for configuration and counter data collection - std::vector counterDataImagePrefix; - std::vector configImage; - std::vector counterDataImage; - std::vector counterDataScratchBuffer; - - std::chrono::time_point profilerStartTs_; - std::chrono::time_point - profilerInitDoneTs_; - std::chrono::time_point profilerStopTs_; - - std::mutex kernelNamesMutex_; - // raw kernel names (not demangled) - std::vector kernelNames_; - - uint32_t numCallbacks_ = 0; - - static std::vector& counterAvailabilityImage(); - -#if HAS_CUPTI_RANGE_PROFILER - CUpti_Profiler_BeginPass_Params beginPassParams_; - CUpti_Profiler_EndPass_Params endPassParams_; -#endif - - bool initSuccess_ = false; - bool profilingActive_ = false; - - friend void __trackCudaKernelLaunch(CUcontext ctx, const char* kernelName); -}; - -// called directly only in unit tests -namespace testing { - -void trackCudaCtx(CUcontext ctx, uint32_t device_id, CUpti_CallbackId cbid); -void trackCudaKernelLaunch(CUcontext ctx, const char* kernelName); - -} // namespace testing - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerConfig.cpp b/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerConfig.cpp deleted file mode 100644 index 04b1ad0cb..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerConfig.cpp +++ /dev/null @@ -1,68 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include - -#include -#include - -#include -#include - -using namespace std::chrono; - -namespace KINETO_NAMESPACE { - -// number of ranges affect the size of counter data binary used by -// the CUPTI Profiler. these defaults can be tuned -constexpr int KMaxAutoRanges = 1500; // supports 1500 kernels -constexpr int KMaxUserRanges = 10; // enable upto 10 sub regions marked by user - -constexpr char kCuptiProfilerMetricsKey[] = "CUPTI_PROFILER_METRICS"; -constexpr char kCuptiProfilerPerKernelKey[] = "CUPTI_PROFILER_ENABLE_PER_KERNEL"; -constexpr char kCuptiProfilerMaxRangesKey[] = "CUPTI_PROFILER_MAX_RANGES"; - -CuptiRangeProfilerConfig::CuptiRangeProfilerConfig(Config& cfg) - : parent_(&cfg), - cuptiProfilerPerKernel_(false), - cuptiProfilerMaxRanges_(0) {} - -bool CuptiRangeProfilerConfig::handleOption(const std::string& name, std::string& val) { - VLOG(0) << " handling : " << name << " = " << val; - // Cupti Range based Profiler configuration - if (!name.compare(kCuptiProfilerMetricsKey)) { - activitiesCuptiMetrics_ = splitAndTrim(val, ','); - } else if (!name.compare(kCuptiProfilerPerKernelKey)) { - cuptiProfilerPerKernel_ = toBool(val); - } else if (!name.compare(kCuptiProfilerMaxRangesKey)) { - cuptiProfilerMaxRanges_ = toInt64(val); - } else { - return false; - } - return true; -} - -void CuptiRangeProfilerConfig::setDefaults() { - if (activitiesCuptiMetrics_.size() > 0 && cuptiProfilerMaxRanges_ == 0) { - cuptiProfilerMaxRanges_ = - cuptiProfilerPerKernel_ ? KMaxAutoRanges : KMaxUserRanges; - } -} - -void CuptiRangeProfilerConfig::printActivityProfilerConfig(std::ostream& s) const { - if (activitiesCuptiMetrics_.size() > 0) { - s << "Cupti Profiler metrics : " - << fmt::format("{}", fmt::join(activitiesCuptiMetrics_, ", ")) << std::endl; - s << "Cupti Profiler measure per kernel : " - << cuptiProfilerPerKernel_ << std::endl; - s << "Cupti Profiler max ranges : " << cuptiProfilerMaxRanges_ << std::endl; - } -} - -void CuptiRangeProfilerConfig::registerFactory() { - Config::addConfigFactory( - kCuptiProfilerConfigName, - [](Config& cfg) { return new CuptiRangeProfilerConfig(cfg); }); -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerConfig.h b/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerConfig.h deleted file mode 100644 index 549b8a4e8..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/CuptiRangeProfilerConfig.h +++ /dev/null @@ -1,86 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include "Config.h" - -#include -#include -#include -#include - -namespace KINETO_NAMESPACE { - -constexpr char kCuptiProfilerConfigName[] = "cupti_rb_profiler"; - -class CuptiRangeProfilerConfig : public AbstractConfig { - public: - bool handleOption(const std::string& name, std::string& val) override; - - void validate( - const std::chrono::time_point& - fallbackProfileStartTime) override {} - - static CuptiRangeProfilerConfig& get(const Config& cfg) { - return dynamic_cast(cfg.feature( - kCuptiProfilerConfigName)); - } - - Config& parent() const { - return *parent_; - } - - std::vector activitiesCuptiMetrics() const { - return activitiesCuptiMetrics_; - } - - bool cuptiProfilerPerKernel() const { - return cuptiProfilerPerKernel_; - } - - int64_t cuptiProfilerMaxRanges() const { - return cuptiProfilerMaxRanges_; - } - - void setSignalDefaults() override { - setDefaults(); - } - - void setClientDefaults() override { - setDefaults(); - } - - void printActivityProfilerConfig(std::ostream& s) const override; - - static void registerFactory(); - protected: - AbstractConfig* cloneDerived(AbstractConfig& parent) const override { - CuptiRangeProfilerConfig* clone = new CuptiRangeProfilerConfig(*this); - clone->parent_ = dynamic_cast(&parent); - return clone; - } - - private: - CuptiRangeProfilerConfig() = delete; - explicit CuptiRangeProfilerConfig(Config& parent); - explicit CuptiRangeProfilerConfig( - const CuptiRangeProfilerConfig& other) = default; - - // some defaults will depend on other configuration - void setDefaults(); - - // Associated Config object - Config* parent_; - - // Counter metrics exposed via CUPTI Profiler API - std::vector activitiesCuptiMetrics_; - - // Collect profiler metrics per kernel - autorange made - bool cuptiProfilerPerKernel_{false}; - - // max number of ranges to configure the profiler for. - // this has to be set before hand to reserve space for the output - int64_t cuptiProfilerMaxRanges_ = 0; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/DaemonConfigLoader.h b/plugins/tensorboard-plugins/libkineto/src/DaemonConfigLoader.h deleted file mode 100644 index 9b0ed9286..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/DaemonConfigLoader.h +++ /dev/null @@ -1,27 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include - -namespace KINETO_NAMESPACE { - -class DaemonConfigLoader { - public: - virtual ~DaemonConfigLoader() {} - - // Return the base config from the daemon - virtual std::string readBaseConfig() = 0; - - // Return a configuration string from the daemon, if one has been posted. - virtual std::string readOnDemandConfig(bool events, bool activities) = 0; - - // Returns the number of tracked contexts for this device. The daemon has a - // global view. If an unexpedted error occurs, return -1. - virtual int gpuContextCount(uint32_t device) = 0; - - virtual void setCommunicationFabric(bool enabled) = 0; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/Demangle.cpp b/plugins/tensorboard-plugins/libkineto/src/Demangle.cpp deleted file mode 100644 index f84f0b8ec..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/Demangle.cpp +++ /dev/null @@ -1,49 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "Demangle.h" - -#ifndef _MSC_VER -#include -#endif -#include -#include - -namespace KINETO_NAMESPACE { - -static constexpr int kMaxSymbolSize = 1024; - -std::string demangle(const char* name) { -#ifndef _MSC_VER - if (!name) { - return ""; - } - - if (strlen(name) > kMaxSymbolSize) { - return name; - } - - int status; - size_t len = 0; - char* demangled = abi::__cxa_demangle(name, nullptr, &len, &status); - if (status != 0) { - return name; - } - std::string res(demangled); - // The returned buffer must be freed! - free(demangled); - return res; -#else - // TODO: demangling on Windows - if (!name) { - return ""; - } else { - return name; - } -#endif -} - -std::string demangle(const std::string& name) { - return demangle(name.c_str()); -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/Demangle.h b/plugins/tensorboard-plugins/libkineto/src/Demangle.h deleted file mode 100644 index 6dcf0776f..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/Demangle.h +++ /dev/null @@ -1,12 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -namespace KINETO_NAMESPACE { - -std::string demangle(const char* name); -std::string demangle(const std::string& name); - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/EventProfiler.cpp b/plugins/tensorboard-plugins/libkineto/src/EventProfiler.cpp deleted file mode 100644 index dbf275523..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/EventProfiler.cpp +++ /dev/null @@ -1,635 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "EventProfiler.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "CuptiEventApi.h" -#include "Logger.h" - -using namespace std::chrono; -using std::accumulate; -using std::endl; -using std::map; -using std::ostream; -using std::string; -using std::unique_ptr; -using std::vector; - -namespace KINETO_NAMESPACE { - -static std::mutex& logMutex() { - static std::mutex instance; - return instance; -} - -// --------------------------------------------------------------------- -// class Event -// --------------------------------------------------------------------- - -// Compute domain instance percentiles -PercentileList& Event::percentiles( - PercentileList& pcs, - const SampleSlice& slice) const { - vector instance_values; - instance_values.reserve(instanceCount); - for (int i = 0; i < instanceCount; i++) { - instance_values.push_back(sumInstance(i, slice)); - } - return KINETO_NAMESPACE::percentiles(instance_values, pcs); -} - -// Add up all samples for a given domain instance -int64_t Event::sumInstance(int i, const SampleSlice& slice) const { - auto r = toIdxRange(slice); - auto start = samples_.cbegin(); - std::advance(start, r.first); - auto end = start; - std::advance(end, r.second); - return accumulate(start, end, 0ul, [i](int64_t a, const Sample& b) { - return a + b.second[i]; - }); -} - -// Add up all samples across all domain instances -int64_t Event::sumAll(const SampleSlice& slice) const { - int64_t res = 0; - for (int i = 0; i < instanceCount; i++) { - res += sumInstance(i, slice); - } - return res; -} - -// Print raw sample values for all domains -void Event::printSamples(ostream& s, CUdevice device) const { - // Don't mess up output with interleaved lines - // Probably OK to reuse logMutex() here since this is - // used for debugging, but need to keep an eye on it. - std::lock_guard lock(logMutex()); - s << "Device " << device << " " << name << ":" << endl; - for (const auto& sample : samples_) { - const auto& vals = sample.second; - for (int64_t val : vals) { - s << val << " "; - } - s << endl; - } -} - -// --------------------------------------------------------------------- -// class Metric -// --------------------------------------------------------------------- -Metric::Metric( - string name, - CUpti_MetricID id, - vector events, - CUpti_MetricEvaluationMode eval_mode, - CuptiMetricApi& cupti_metrics) - : name(std::move(name)), - id_(id), - events_(std::move(events)), - evalMode_(eval_mode), - cuptiMetrics_(cupti_metrics), - valueKind_(cuptiMetrics_.valueKind(id)) {} - -// Return per-SM vector as well as total -struct Metric::CalculatedValues Metric::calculate( - map& event_map, - nanoseconds sample_duration, - const SampleSlice& slice) { - vector metric_values; - vector ev_values; - ev_values.reserve(events_.size()); - if (evalMode_ & CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE) { - int instance_count = instanceCount(event_map); - metric_values.reserve(instance_count); - for (int i = 0; i < instance_count; i++) { - ev_values.clear(); - for (CUpti_EventID event_id : events_) { - ev_values.push_back(event_map[event_id].sumInstance(i, slice)); - } - metric_values.push_back(cuptiMetrics_.calculate( - id_, valueKind_, events_, ev_values, sample_duration.count())); - } - } - - // FIXME: Check assumption that all instances are profiled - ev_values.clear(); - for (CUpti_EventID event_id : events_) { - ev_values.push_back(event_map[event_id].sumAll(slice)); - } - SampleValue total = cuptiMetrics_.calculate( - id_, valueKind_, events_, ev_values, sample_duration.count()); - if (evalMode_ & CUPTI_METRIC_EVALUATION_MODE_AGGREGATE) { - metric_values.push_back(total); - } - return {metric_values, std::move(total)}; -} - -void Metric::printDescription(ostream& s) const { - s << fmt::format("{} ({})", name, fmt::join(events_, ",")) << endl; -} - -// --------------------------------------------------------------------- -// class EventGroupSet -// --------------------------------------------------------------------- - -// Each domain has a set of counters. -// Some counters in a domain can be collected simultaneously in a "group" -// Counters from different domains can also be collected at the same time -// Therefore we have a "set of groups", or group set, with counters that -// can all be collected at once. -EventGroupSet::EventGroupSet( - CUpti_EventGroupSet& set, - map& events, - CuptiEventApi& cupti) - : set_(set), events_(events), cuptiEvents_(cupti), enabled_(false) { - for (int g = 0; g < set.numEventGroups; g++) { - CUpti_EventGroup grp = set.eventGroups[g]; - // Profile all domain instances - cuptiEvents_.enablePerInstance(grp); - uint32_t instance_count = cuptiEvents_.instanceCount(grp); - for (const auto& id : cuptiEvents_.eventsInGroup(grp)) { - VLOG(0) << "Instance count for " << id << ":" << instance_count; - events_[id].instanceCount = instance_count; - } - } -} - -EventGroupSet::~EventGroupSet() { - // Disable EventGroupSet in Cupti. - if (enabled_) { - setEnabled(false); - } -} - -// Enable or disable this group set -void EventGroupSet::setEnabled(bool enabled) { - if (enabled && !enabled_) { - cuptiEvents_.enableGroupSet(set_); - } else if (!enabled && enabled_) { - cuptiEvents_.disableGroupSet(set_); - } - enabled_ = enabled; -} - -// Collect counter values for each counter in group set -void EventGroupSet::collectSample() { - auto timestamp = system_clock::now(); - for (int g = 0; g < set_.numEventGroups; g++) { - CUpti_EventGroup grp = set_.eventGroups[g]; - for (const auto& id : cuptiEvents_.eventsInGroup(grp)) { - Event& ev = events_[id]; - vector vals(ev.instanceCount); - // FIXME: Use cuptiEventGroupReadAllEvents - cuptiEvents_.readEvent(grp, id, vals); - - if (VLOG_IS_ON(0)) { - for (int64_t v : vals) { - if (v == CUPTI_EVENT_OVERFLOW) { - LOG(WARNING) << "Counter overflow detected " - << "- decrease sample period!" << endl; - } - } - } - - ev.addSample(timestamp, vals); - } - } - - if (VLOG_IS_ON(1)) { - auto t2 = system_clock::now(); - VLOG(1) << "Device " << cuptiEvents_.device() << " Sample (us): " - << duration_cast(t2 - timestamp).count(); - } -} - -// Print names of events in this group set, ordered by group -void EventGroupSet::printDescription(ostream& s) const { - for (int g = 0; g < set_.numEventGroups; g++) { - s << " Events in group " << g << ": "; - for (const auto& id : cuptiEvents_.eventsInGroup(set_.eventGroups[g])) { - s << id << " (" << events_[id].name << ") "; - } - s << endl; - } -} - -// --------------------------------------------------------------------- -// class EventProfiler -// --------------------------------------------------------------------- - -// Find nearest factor of a number by linear search, -// starting at hi and lo - hi searches up and lo searches down -static int nearestFactor(int hi, int lo, int number) { - return number % hi == 0 - ? hi - : number % lo == 0 ? lo : nearestFactor(hi + 1, lo - 1, number); -} - -static int nearestFactor(int count, int max) { - return nearestFactor(count, count, max); -} - -void EventProfiler::initEvents(const std::set& eventNames) { - events_.clear(); - // Build event map - for (const auto& name : eventNames) { - events_.emplace(cuptiEvents_->eventId(name), name); - } -} - -void EventProfiler::initMetrics(const std::set& metricNames) { - metrics_.clear(); - // Add events from metrics - metrics_.reserve(metricNames.size()); - for (const auto& metric_name : metricNames) { - CUpti_MetricID metric_id = cuptiMetrics_->idFromName(metric_name); - if (metric_id == ~0) { - continue; - } - - const auto& events = cuptiMetrics_->events(metric_id); - vector event_ids; - event_ids.reserve(events.size()); - for (const auto& pair : events) { - CUpti_EventID id = pair.first; - const string& event_name = pair.second; - if (event_name.empty()) { - // For unnamed events, use metric name and event id - // FIXME: For subsequent metrics using the same event, - // this will be confusing - events_.emplace(id, metric_name + "_" + event_name); - } else { - events_.emplace(id, event_name); - } - event_ids.push_back(id); - } - metrics_.emplace_back( - metric_name, - metric_id, - event_ids, - cuptiMetrics_->evaluationMode(metric_id), - *cuptiMetrics_); - } -} - -bool EventProfiler::initEventGroups() { - sets_.clear(); - if (eventGroupSets_) { - cuptiEvents_->destroyGroupSets(eventGroupSets_); - eventGroupSets_ = nullptr; - } - if (events_.empty()) { - return true; - } - - // Determine sets of groups to be collected - vector ids; - ids.reserve(events_.size()); - for (const auto& ev : events_) { - ids.push_back(ev.first); - } - eventGroupSets_ = cuptiEvents_->createGroupSets(ids); - VLOG(0) << "Number of group sets: " << eventGroupSets_->numSets; - for (int i = 0; i < eventGroupSets_->numSets; i++) { - sets_.push_back( - EventGroupSet(eventGroupSets_->sets[i], events_, *cuptiEvents_)); - } - return !sets_.empty(); -} - -static unique_ptr alignAndValidateConfigs( - Config& base, - Config* onDemand) { - auto now = system_clock::now(); - if (!onDemand || - now > - (onDemand->eventProfilerOnDemandStartTime() + - onDemand->eventProfilerOnDemandDuration())) { - base.validate(now); - return base.clone(); - } - - auto res = base.clone(); - res->addEvents(onDemand->eventNames()); - res->addMetrics(onDemand->metricNames()); - - int sample_period = - std::min(base.samplePeriod().count(), onDemand->samplePeriod().count()); - if (sample_period < base.samplePeriod().count() && - (base.samplePeriod().count() % sample_period) != 0) { - sample_period = nearestFactor(sample_period, base.samplePeriod().count()); - LOG(WARNING) - << "On-demand sample period must be a factor of base sample period. " - << "Adjusting from " << onDemand->samplePeriod().count() << "ms to " - << sample_period << "ms."; - } - base.setSamplePeriod(milliseconds(sample_period)); - base.validate(now); - res->setSamplePeriod(base.samplePeriod()); - res->setMultiplexPeriod(base.multiplexPeriod()); - res->validate(now); - onDemand->setSamplePeriod(base.samplePeriod()); - onDemand->setMultiplexPeriod(base.multiplexPeriod()); - onDemand->validate(now); - - return res; -} - -static milliseconds minReportPeriod(const Config& config, int num_sets) { - return config.multiplexPeriod() * num_sets; -} - -static bool canSupportReportPeriod(const Config& config, int num_sets) { - // Can we get through the groups an even number per report period? - milliseconds min_report_period = minReportPeriod(config, num_sets); - return (config.reportPeriod().count() % min_report_period.count()) == 0; -} - -static int completeSamplesPerReport(const Config& config, int num_sets) { - if (num_sets <= 1) { - return config.reportPeriod() / config.samplePeriod(); - } - // Numnber of complete sample collections in the report period - // E.g. if report period is 10000ms, sample period 500ms, - // multiplex period 2000ms and num_sets is 5 then # of complete samples is - // (2000ms / 500ms) * (10000ms / 2000ms / 5) = 4 * 1 = 4 - int samples_per_multiplex_period = - config.multiplexPeriod() / config.samplePeriod(); - int multiplex_periods_per_report = - config.reportPeriod() / config.multiplexPeriod(); - return (multiplex_periods_per_report / num_sets) * - samples_per_multiplex_period; -} - -static bool canSupportSamplesPerReport(const Config& config, int num_sets) { - // Can samples per report can be honored with an exact *full* set of samples? - // We don't support partial samples at this point. - int full_samples_per_report = completeSamplesPerReport(config, num_sets); - return (full_samples_per_report % config.samplesPerReport()) == 0; -} - -static void adjustConfig(Config& config, int num_sets) { - // Don't change sample period and multiplex period here, since that can - // cause overflows and perf degradation. Report period and samples per - // report is OK to change (with warning). - if (!canSupportReportPeriod(config, num_sets)) { - milliseconds min_report_period = minReportPeriod(config, num_sets); - LOG(WARNING) << "Report period must be a multiple of " - << min_report_period.count() << "ms (" << num_sets - << " event sets * " << config.multiplexPeriod().count() - << "ms multiplex period), in order to get complete samples."; - auto new_report_period = - Config::alignUp(config.reportPeriod(), min_report_period); - double sf = - ((double)new_report_period.count()) / config.reportPeriod().count(); - int new_samples_per_report = std::round(config.samplesPerReport() * sf); - LOG(WARNING) << "Adjusting report period from " - << config.reportPeriod().count() << "ms to " - << new_report_period.count() << "ms"; - if (new_samples_per_report != config.samplesPerReport()) { - LOG(WARNING) << "Adjusting samples per report from " - << config.samplesPerReport() << " to " - << new_samples_per_report; - } - config.setReportPeriod(new_report_period); - config.setSamplesPerReport(new_samples_per_report); - } - // Ensure that samples per report can be honored with - // an exact *full* set of samples. Don't support partial - // samples at this point. - if (!canSupportSamplesPerReport(config, num_sets)) { - int full_samples_per_report = completeSamplesPerReport(config, num_sets); - int adjusted_count = - nearestFactor(config.samplesPerReport(), full_samples_per_report); - LOG(WARNING) - << "Samples per report must be such that an even number of " - << "complete samples can be aggregated in each report period. Adjusting" - << " from " << config.samplesPerReport() << " to " << adjusted_count - << " (complete sample count is " << full_samples_per_report << ")"; - config.setSamplesPerReport(adjusted_count); - } -} - -// Prepare profiler -EventProfiler::EventProfiler( - std::unique_ptr cupti_events, - std::unique_ptr cupti_metrics, - vector>& loggers, - vector>& onDemandLoggers) - : cuptiEvents_(std::move(cupti_events)), - cuptiMetrics_(std::move(cupti_metrics)), - loggers_(loggers), - onDemandLoggers_(onDemandLoggers) {} - -void EventProfiler::reportSamples() { - dispatchSamples(*config_, loggers_, baseSamples_); - baseSamples_ += completeSamplesPerReport(*config_, sets_.size()); -} - -void EventProfiler::reportOnDemandSamples() { - dispatchSamples(*onDemandConfig_, onDemandLoggers_, onDemandSamples_); - onDemandSamples_ += completeSamplesPerReport(*onDemandConfig_, sets_.size()); -} - -EventProfiler::~EventProfiler() { - if (eventGroupSets_) { - for (auto& set : sets_) { - set.setEnabled(false); - } - cuptiEvents_->destroyGroupSets(eventGroupSets_); - } - VLOG(0) << "Stopped event profiler for device " << device(); -} - -void EventProfiler::updateLoggers(Config& config, Config* on_demand_config) { - // Update loggers. - for (auto& logger : loggers_) { - std::lock_guard lock(logMutex()); - logger->update(config); - } - - if (on_demand_config) { - // Update onDemand loggers. - for (auto& logger : onDemandLoggers_) { - std::lock_guard lock(logMutex()); - logger->update(*on_demand_config); - } - } -} - -bool EventProfiler::applyConfig(const Config& config) { - // Initialize events, metrics, and event group sets. - // TODO: Send warnings / errors back to dyno for onDemand config - try { - if (!initEventsAndMetrics(config)) { - return false; - } - } catch (const std::exception& ex) { - LOG(WARNING) << "Failed to apply config (" << ex.what() << ")"; - return false; - } - - return true; -} - -bool EventProfiler::initEventsAndMetrics(const Config& config) { - initEvents(config.eventNames()); - initMetrics(config.metricNames()); - // We now have the total list of events to collect - // They need to be organized into groups for multiplexing - if (!initEventGroups()) { - LOG(WARNING) << "No events/metrics initialized successfully"; - return false; - } - - if (VLOG_IS_ON(1)) { - printMetrics(LIBKINETO_DBG_STREAM); - printSets(LIBKINETO_DBG_STREAM); - } - return true; -} - -void EventProfiler::printSets(ostream& s) const { - for (int i = 0; i < sets_.size(); i++) { - s << "Set " << i << endl; - sets_[i].printDescription(s); - } -} - -void EventProfiler::printMetrics(ostream& s) const { - s << "Metrics:" << endl; - for (const Metric& m : metrics_) { - m.printDescription(s); - } -} - -void EventProfiler::printAllSamples(ostream& s, CUdevice device) const { - for (const auto& pair : events_) { - const Event& ev = pair.second; - ev.printSamples(s, device); - } -} - -void EventProfiler::enableNextCounterSet() { - if (sets_.size() > 1) { - auto t1 = system_clock::now(); - - VLOG(1) << "Disabling set " << curEnabledSet_; - sets_[curEnabledSet_].setEnabled(false); - curEnabledSet_ = (curEnabledSet_ + 1) % sets_.size(); - VLOG(1) << "Enabling set " << curEnabledSet_; - sets_[curEnabledSet_].setEnabled(true); - - if (VLOG_IS_ON(1)) { - auto t2 = system_clock::now(); - VLOG(1) << "Switch (us): " - << duration_cast(t2 - t1).count(); - } - } -} - -// Notify listeners of collected samples -void EventProfiler::dispatchSamples( - const Config& config, - const vector>& loggers, - int sample_offset) { - Sample sample(events_.size() + metrics_.size()); - // Normalize values to per second - auto delta = config.reportPeriod() / config.samplesPerReport(); - double sf = 1000.0 * sets_.size() / delta.count(); - for (int i = 0; i < config.samplesPerReport(); i++) { - sample.stats.clear(); - sample.deltaMsec = (delta * i).count(); - SampleSlice slice = {sample_offset, i, config.samplesPerReport()}; - VLOG(1) << "Slice: " << sample_offset << ", " << i << ", " - << config.samplesPerReport(); - for (const auto& pair : events_) { - const Event& ev = pair.second; - int64_t total = std::round(sf * ev.sumAll(slice)); - PercentileList pcs = initPercentiles(config.percentiles()); - normalize(ev.percentiles(pcs, slice), sf); - sample.stats.push_back({ev.name, std::move(pcs), SampleValue(total)}); - } - - for (auto& m : metrics_) { - // calculate returns a pair of per-SM vector and a total - auto vals = m.calculate(events_, delta, slice); - PercentileList pcs = initPercentiles(config.percentiles()); - sample.stats.push_back( - {m.name, std::move(percentiles(vals.perInstance, pcs)), vals.total}); - } - - for (auto& logger : loggers) { - std::lock_guard lock(logMutex()); - logger->handleSample(device(), sample, config.ipcFabricEnabled()); - } - } - - if (VLOG_IS_ON(2)) { - printAllSamples(LIBKINETO_DBG_STREAM, device()); - } -} - -void EventProfiler::configure(Config& config, Config* onDemandConfig) { - if (!sets_.empty()) { - sets_[curEnabledSet_].setEnabled(false); - clearSamples(); - } - - config_ = config.clone(); - onDemandConfig_ = onDemandConfig ? onDemandConfig->clone() : nullptr; - mergedConfig_ = alignAndValidateConfigs(*config_, onDemandConfig_.get()); - if (!applyConfig(*mergedConfig_)) { - LOG(WARNING) << "Failed to apply config!"; - mergedConfig_ = config_->clone(); - applyConfig(*config_); - } - if (!sets_.empty()) { - // Make timing adjustments based on multiplexing requirements. - adjustConfig(*config_, sets_.size()); - if (onDemandConfig_) { - int duration = onDemandConfig_->eventProfilerOnDemandDuration().count(); - LOG(INFO) << "On demand profiler activated for " << duration << " secs"; - adjustConfig(*onDemandConfig_, sets_.size()); - } - // If events or metrics were added or removed, need to tell loggers - updateLoggers(*config_, onDemandConfig_.get()); - } - - curEnabledSet_ = 0; - if (!sets_.empty()) { - sets_[0].setEnabled(true); - } else { - VLOG(0) << "No counters profiled!"; - } - - baseSamples_ = 0; - onDemandSamples_ = 0; -} - -void EventProfiler::collectSample() { - if (sets_.empty()) { - return; - } - sets_[curEnabledSet_].collectSample(); - if (VLOG_IS_ON(1)) { - printAllSamples(LIBKINETO_DBG_STREAM, device()); - } -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/EventProfiler.h b/plugins/tensorboard-plugins/libkineto/src/EventProfiler.h deleted file mode 100644 index fafd5b9bb..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/EventProfiler.h +++ /dev/null @@ -1,341 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "Config.h" -#include "CuptiEventApi.h" -#include "CuptiMetricApi.h" -#include "SampleListener.h" - -namespace KINETO_NAMESPACE { - -// Helper function for computing percentiles (nearest-rank). -// Modifies the input. -template -inline PercentileList& percentiles(std::vector values, PercentileList& pcs) { - auto size = values.size(); - for (auto& x : pcs) { - int idx = std::min(size - 1, (x.first * size) / 100); - std::nth_element(values.begin(), values.begin() + idx, values.end()); - x.second = SampleValue(values[idx]); - } - return pcs; -} - -// Helper function for normalizing a percentile list -// Modifies the input -inline PercentileList& normalize(PercentileList& pcs, double sf) { - for (auto& pc : pcs) { - pc.second *= sf; - } - return pcs; -} - -// A slice of the sample buffer -struct SampleSlice { - // Start offset (samples) - int offset; - // Slice number - int index; - // Out of this many - int count; -}; - -// A sampled event -class Event { - public: - /* implicit */ Event(std::string name) : name(std::move(name)) {} - /* implicit */ Event(const char* name) : name(name) {} - Event() : name("INVALID") {} - - Event(const Event&) = delete; - Event& operator=(const Event&) = delete; - Event(Event&&) = default; - Event& operator=(Event&&) = default; - - void addSample( - std::chrono::time_point timestamp, - const std::vector& values) { - assert(values.size() == instanceCount); - samples_.emplace_back(timestamp, values); - } - - // Sum samples for a single domain instance - int64_t sumInstance(int i, const SampleSlice& slice) const; - - // Sum all samples across all domain instances - int64_t sumAll(const SampleSlice& slice) const; - - // Create list of percentiles - PercentileList& percentiles(PercentileList& pcs, const SampleSlice& slice) - const; - - void eraseSamples(int count) { - auto end = samples_.begin(); - std::advance(end, count); - samples_.erase(samples_.begin(), end); - } - - void clearSamples() { - samples_.clear(); - } - - int sampleCount() { - return samples_.size(); - } - - void printSamples(std::ostream& s, CUdevice device) const; - - // Event name (see nvprof --query-events) - std::string name; - - // Number of domain instances for this event, e.g. number of SMs - int instanceCount = 0; - - private: - std::pair toIdxRange(const SampleSlice& slice) const { - int size = (samples_.size() - slice.offset) / slice.count; - return std::make_pair(slice.offset + (slice.index * size), size); - } - - // List of collected samples, where each sample has values for - // one or more domain instances - using Sample = std::pair< - std::chrono::time_point, - std::vector>; - std::list samples_; -}; - -class Metric { - public: - Metric( - std::string name, - CUpti_MetricID id, - std::vector events, - CUpti_MetricEvaluationMode eval_mode, - CuptiMetricApi& cupti_metrics); - - struct CalculatedValues { - std::vector perInstance; - SampleValue total; - }; - - struct CalculatedValues calculate( - std::map& events, - std::chrono::nanoseconds sample_duration, - const SampleSlice& slice); - - int instanceCount(std::map& events) { - return events[events_[0]].instanceCount; - } - - void printDescription(std::ostream& s) const; - - std::string name; - - private: - CUpti_MetricID id_; - std::vector events_; - CUpti_MetricEvaluationMode evalMode_; - // Calls to CUPTI is encapsulated behind this interface - CuptiMetricApi& cuptiMetrics_; - CUpti_MetricValueKind valueKind_; -}; - -/** - * A set of event groups. - * Holds all the events that may be collected in a single pass. - * A group contains one or more counters for a single domain. - * A group set contains zero or one groups per domain. - */ -class EventGroupSet { - public: - EventGroupSet( - CUpti_EventGroupSet& set, - std::map& events, - CuptiEventApi& cupti); - ~EventGroupSet(); - - EventGroupSet(const EventGroupSet&) = delete; - EventGroupSet& operator=(const EventGroupSet&) = delete; - EventGroupSet(EventGroupSet&&) = default; - EventGroupSet& operator=(EventGroupSet&&) = delete; - - // Number of groups = number of domains profiled - int groupCount() const { - return set_.numEventGroups; - } - - void setEnabled(bool enabled); - // Take a sample of counters in this group set - void collectSample(); - void printDescription(std::ostream& s) const; - - private: - CUpti_EventGroupSet& set_; - std::map& events_; - // Calls to CUPTI is encapsulated behind this interface - CuptiEventApi& cuptiEvents_; - bool enabled_; -}; - -// The sampler -class EventProfiler { - public: - explicit EventProfiler( - std::unique_ptr cupti_events, - std::unique_ptr cupti_metrics, - std::vector>& loggers, - std::vector>& onDemandLoggers); - EventProfiler(const EventProfiler&) = delete; - EventProfiler& operator=(const EventProfiler&) = delete; - ~EventProfiler(); - - void configure(Config& config, Config* onDemandConfig); - - bool isOnDemandActive() { - return !!onDemandConfig_; - } - - // Print the counter sets. Multiple sets will be multiplexed. - void printSets(std::ostream& s) const; - - // Print metrics descriptions - void printMetrics(std::ostream& s) const; - - bool enableForDevice(Config& cfg); - - CUdevice device() { - return cuptiEvents_->device(); - } - - bool setContinuousMode() { - return cuptiEvents_->setContinuousMode(); - } - - std::chrono::milliseconds samplePeriod() { - return mergedConfig_->samplePeriod(); - } - - std::chrono::milliseconds multiplexPeriod() { - return mergedConfig_->multiplexPeriod(); - } - - std::chrono::milliseconds reportPeriod() { - return config_->reportPeriod(); - } - - std::chrono::milliseconds onDemandReportPeriod() { - return onDemandConfig_->reportPeriod(); - } - - // Read values of currently running counters. - void collectSample(); - - void reportSamples(); - void reportOnDemandSamples(); - - bool enabled() { - return sets_.size() > 0; - } - - bool multiplexEnabled() { - return sets_.size() > 1; - } - - // Multiplex counters. - void enableNextCounterSet(); - - void eraseReportedSamples() { - int erase_count = baseSamples_; - if (onDemandConfig_ && - onDemandConfig_->eventProfilerOnDemandDuration().count() > 0) { - erase_count = std::min(baseSamples_, onDemandSamples_); - } - eraseSamples(erase_count); - baseSamples_ -= erase_count; - onDemandSamples_ -= erase_count; - } - - void clearSamples() { - for (auto& pair : events_) { - pair.second.clearSamples(); - } - baseSamples_ = 0; - onDemandSamples_ = 0; - } - - private: - // Functions to initialize profiler based on Config settings. - bool applyConfig(const Config& config); - bool initEventsAndMetrics(const Config& config); - void initEvents(const std::set& eventNames); - void initMetrics(const std::set& metricNames); - bool initEventGroups(); - - PercentileList initPercentiles(const std::vector& percentiles) { - PercentileList res; - res.reserve(percentiles.size()); - for (int p : percentiles) { - res.emplace_back(p, SampleValue(0)); - } - return res; - } - - // Notify listeners of collected samples - void dispatchSamples( - const Config& config, - const std::vector>& loggers, - int report_nr); - - void eraseSamples(int count) { - for (auto& pair : events_) { - pair.second.eraseSamples(count); - } - } - - void updateLoggers(Config& config, Config* on_demand_config); - - // Print all collected samples since last clear. - void printAllSamples(std::ostream& s, CUdevice device) const; - - // Calls to CUPTI is encapsulated behind these interfaces - std::unique_ptr cuptiEvents_; - std::unique_ptr cuptiMetrics_; - // The CUpti API reports event IDs, we must map them to our event objects - std::map events_; - // List of metrics - std::vector metrics_; - // The countert sets needed to collect all counters - std::vector sets_; - // The event group set object returned by Cupti. - // Saved s.t. we can call cuptiEventGroupSetsDestroy to free memory when - // the object is no longer needed. - CUpti_EventGroupSets* eventGroupSets_ = nullptr; - // Current multiplexed counter set - int curEnabledSet_{0}; - - std::unique_ptr config_; - std::unique_ptr onDemandConfig_; - std::unique_ptr mergedConfig_; - int baseSamples_{0}; - int onDemandSamples_{0}; - - // Shared between profiler threads - // Vectors are read-only but calling loggers require lock - const std::vector>& loggers_; - const std::vector>& onDemandLoggers_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/EventProfilerController.cpp b/plugins/tensorboard-plugins/libkineto/src/EventProfilerController.cpp deleted file mode 100644 index 0427cc7a9..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/EventProfilerController.cpp +++ /dev/null @@ -1,423 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "EventProfilerController.h" - -#include -#include -#include - -#include "ConfigLoader.h" -#include "CuptiEventApi.h" -#include "CuptiMetricApi.h" -#include "EventProfiler.h" -#include "output_csv.h" - -#include "Logger.h" -#include "ThreadUtil.h" - -using namespace std::chrono; -using std::unique_ptr; -using std::vector; - -namespace KINETO_NAMESPACE { - -namespace { - -vector(const Config&)>>& -loggerFactories() { - static vector(const Config&)>> - factories; - return factories; -} - -vector(const Config&)>>& -onDemandLoggerFactories() { - static vector(const Config&)>> - factories; - return factories; -} - -vector> makeLoggers(const Config& config) { - vector> loggers; - for (const auto& factory : loggerFactories()) { - loggers.push_back(factory(config)); - } - loggers.push_back(std::make_unique()); - loggers.push_back(std::make_unique()); - return loggers; -} - -vector> makeOnDemandLoggers( - const Config& config) { - vector> loggers; - for (const auto& factory : onDemandLoggerFactories()) { - loggers.push_back(factory(config)); - } - loggers.push_back(std::make_unique()); - return loggers; -} - -vector>& loggers(const Config& config) { - static auto res = makeLoggers(config); - return res; -} - -vector>& onDemandLoggers( - const Config& config) { - static auto res = makeOnDemandLoggers(config); - return res; -} - -} // anon namespace - -// Keep an eye on profiling threads. -// We've observed deadlocks in Cuda11 in libcuda / libcupti.. -namespace detail { - -class HeartbeatMonitor { - - public: - ~HeartbeatMonitor() { - stopMonitoring(); - } - - static HeartbeatMonitor& instance() { - static HeartbeatMonitor monitor; - return monitor; - } - - void profilerHeartbeat() { - int32_t tid = systemThreadId(); - std::lock_guard lock(mutex_); - profilerAliveMap_[tid]++; - } - - void setPeriod(seconds period) { - { - std::lock_guard lock(mutex_); - if (period_ == period) { - return; - } - period_ = period; - } - if (period == seconds(0)) { - stopMonitoring(); - } else { - startMonitoring(); - } - } - - private: - HeartbeatMonitor() = default; - - void monitorLoop() { - std::unique_lock lock(mutex_); - while(!stopMonitor_) { - auto cv_status = condVar_.wait_for(lock, seconds(period_)); - // Don't perform check on spurious wakeup or on notify - if (cv_status == std::cv_status::timeout) { - for (auto& pair : profilerAliveMap_) { - int32_t tid = pair.first; - int& i = pair.second; - if (i == 0) { - LOG(ERROR) << "Thread " << tid << " appears stuck!"; - } - i = 0; - } - } - } - } - - void startMonitoring() { - if (!monitorThread_) { - VLOG(0) << "Starting monitoring thread"; - stopMonitor_ = false; - monitorThread_ = std::make_unique( - &HeartbeatMonitor::monitorLoop, this); - } - } - - void stopMonitoring() { - if (monitorThread_) { - VLOG(0) << "Stopping monitoring thread"; - stopMonitor_ = true; - condVar_.notify_one(); - monitorThread_->join(); - monitorThread_ = nullptr; - VLOG(0) << "Monitoring thread terminated"; - } - } - - std::map profilerAliveMap_; - std::unique_ptr monitorThread_; - std::mutex mutex_; - std::condition_variable condVar_; - std::atomic_bool stopMonitor_{false}; - seconds period_{0}; -}; - -} // namespace detail - -namespace { -// Profiler map singleton -std::map>& profilerMap() { - static std::map> instance; - return instance; -} - -void reportLateSample( - int sleepMs, - int sampleMs, - int reportMs, - int reprogramMs) { - LOG_EVERY_N(WARNING, 10) << "Lost sample due to delays (ms): " << sleepMs - << ", " << sampleMs << ", " << reportMs << ", " - << reprogramMs; -} - -void configureHeartbeatMonitor( - detail::HeartbeatMonitor& monitor, const Config& base, const Config* onDemand) { - seconds base_period = - base.eventProfilerHeartbeatMonitorPeriod(); - seconds on_demand_period = !onDemand ? seconds(0) : - onDemand->eventProfilerHeartbeatMonitorPeriod(); - monitor.setPeriod( - on_demand_period > seconds(0) ? on_demand_period : base_period); -} - -} // anon namespace - -void EventProfilerController::addLoggerFactory( - std::function(const Config&)> factory) { - loggerFactories().push_back(factory); -} - -void EventProfilerController::addOnDemandLoggerFactory( - std::function(const Config&)> factory) { - onDemandLoggerFactories().push_back(factory); -} - -EventProfilerController::EventProfilerController( - CUcontext context, - ConfigLoader& configLoader, - detail::HeartbeatMonitor& heartbeatMonitor) - : configLoader_(configLoader), heartbeatMonitor_(heartbeatMonitor) { - auto cupti_events = std::make_unique(context); - auto cupti_metrics = - std::make_unique(cupti_events->device()); - configLoader_.addHandler( - ConfigLoader::ConfigKind::EventProfiler, this); - auto config = configLoader.getConfigCopy(); - profiler_ = std::make_unique( - std::move(cupti_events), - std::move(cupti_metrics), - loggers(*config), - onDemandLoggers(*config)); - profilerThread_ = std::make_unique( - &EventProfilerController::profilerLoop, this); -} - -EventProfilerController::~EventProfilerController() { - if (profilerThread_) { - // signaling termination of the profiler loop - stopRunloop_ = true; - profilerThread_->join(); - } - configLoader_.removeHandler( - ConfigLoader::ConfigKind::EventProfiler, this); - VLOG(0) << "Stopped event profiler"; -} - -// Must be called under lock -void EventProfilerController::start(CUcontext ctx, ConfigLoader& configLoader) { - profilerMap()[ctx] = unique_ptr( - new EventProfilerController( - ctx, configLoader, detail::HeartbeatMonitor::instance())); -} - -// Must be called under lock -void EventProfilerController::stop(CUcontext ctx) { - profilerMap()[ctx] = nullptr; -} - -bool EventProfilerController::canAcceptConfig() { - std::lock_guard guard(mutex_); - return !newOnDemandConfig_; -} - -void EventProfilerController::acceptConfig(const Config& config) { - if (config.eventProfilerOnDemandDuration().count() == 0) { - // Ignore - not for this profiler - return; - } - std::lock_guard guard(mutex_); - if (newOnDemandConfig_) { - LOG(ERROR) << "On demand request already queued - ignoring new request"; - return; - } - newOnDemandConfig_ = config.clone(); - LOG(INFO) << "Received new on-demand config"; -} - -bool EventProfilerController::enableForDevice(Config& cfg) { - // FIXME: Use device unique id! - if (!cfg.eventProfilerEnabledForDevice(profiler_->device())) { - return false; - } - // context count includes the new context - int instances = configLoader_.contextCountForGpu(profiler_->device()); - VLOG(0) << "Device context count: " << instances; - return instances >= 0 && instances <= cfg.maxEventProfilersPerGpu(); -} - -void EventProfilerController::profilerLoop() { - // We limit the number of profilers that can exist per GPU - auto config = configLoader_.getConfigCopy(); - if (!enableForDevice(*config)) { - VLOG(0) << "Not starting EventProfiler - profilers for GPU " - << profiler_->device() << " exceeds profilers per GPU limit (" - << config->maxEventProfilersPerGpu() << ")"; - return; - } - - if (!profiler_->setContinuousMode()) { - VLOG(0) << "Continuous mode not supported for GPU " - << profiler_->device() << ". Not starting Event Profiler."; - return; - } - - VLOG(0) << "Starting Event Profiler for GPU " << profiler_->device(); - setThreadName("CUPTI Event Profiler"); - - time_point next_sample_time; - time_point next_report_time; - time_point next_on_demand_report_time; - time_point next_multiplex_time; - std::unique_ptr on_demand_config = nullptr; - bool reconfigure = true; - bool restart = true; - int report_count = 0; - int on_demand_report_count = 0; - while (!stopRunloop_) { - heartbeatMonitor_.profilerHeartbeat(); - if (configLoader_.hasNewConfig(*config)) { - config = configLoader_.getConfigCopy(); - VLOG(0) << "Base config changed"; - report_count = 0; - reconfigure = true; - } - - auto now = system_clock::now(); - if (on_demand_config && - now > (on_demand_config->eventProfilerOnDemandStartTime() + - on_demand_config->eventProfilerOnDemandDuration())) { - on_demand_config = nullptr; - LOG(INFO) << "On-demand profiling complete"; - reconfigure = true; - } - - if (!profiler_->isOnDemandActive()) { - std::lock_guard lock(mutex_); - if (newOnDemandConfig_) { - VLOG(0) << "Received on-demand config, reconfiguring"; - on_demand_config = std::move(newOnDemandConfig_); - reconfigure = true; - on_demand_report_count = 0; - } - } - - if (reconfigure) { - try { - profiler_->configure(*config, on_demand_config.get()); - } catch (const std::exception& ex) { - LOG(ERROR) << "Encountered error while configuring event profiler: " - << ex.what(); - // Exit profiling entirely when encountering an error here - // as it indicates a serious problem or bug. - break; - } - configureHeartbeatMonitor( - heartbeatMonitor_, *config, on_demand_config.get()); - reconfigure = false; - restart = true; - } - - if (restart) { - now = system_clock::now(); - next_sample_time = now + profiler_->samplePeriod(); - next_report_time = now + profiler_->reportPeriod(); - if (profiler_->isOnDemandActive()) { - next_on_demand_report_time = now + profiler_->onDemandReportPeriod(); - } - next_multiplex_time = now + profiler_->multiplexPeriod(); - // Collect an initial sample and throw it away - // The next sample is the first valid one - profiler_->collectSample(); - profiler_->clearSamples(); - restart = false; - } - - auto start_sleep = now; - while (now < next_sample_time) { - /* sleep override */ - std::this_thread::sleep_for(next_sample_time - now); - now = system_clock::now(); - } - int sleep_time = duration_cast(now - start_sleep).count(); - - auto start_sample = now; - profiler_->collectSample(); - now = system_clock::now(); - int sample_time = duration_cast(now - start_sample).count(); - - next_sample_time += profiler_->samplePeriod(); - if (now > next_sample_time) { - reportLateSample(sleep_time, sample_time, 0, 0); - restart = true; - continue; - } - - auto start_report = now; - if (now > next_report_time) { - VLOG(1) << "Report #" << report_count++; - profiler_->reportSamples(); - next_report_time += profiler_->reportPeriod(); - } - if (profiler_->isOnDemandActive() && now > next_on_demand_report_time) { - VLOG(1) << "OnDemand Report #" << on_demand_report_count++; - profiler_->reportOnDemandSamples(); - next_on_demand_report_time += profiler_->onDemandReportPeriod(); - } - profiler_->eraseReportedSamples(); - now = system_clock::now(); - int report_time = duration_cast(now - start_report).count(); - - if (now > next_sample_time) { - reportLateSample(sleep_time, sample_time, report_time, 0); - restart = true; - continue; - } - - auto start_multiplex = now; - if (profiler_->multiplexEnabled() && now > next_multiplex_time) { - profiler_->enableNextCounterSet(); - next_multiplex_time += profiler_->multiplexPeriod(); - } - now = system_clock::now(); - int multiplex_time = - duration_cast(now - start_multiplex).count(); - - if (now > next_sample_time) { - reportLateSample(sleep_time, sample_time, report_time, multiplex_time); - restart = true; - } - - VLOG(0) << "Runloop execution time: " - << duration_cast(now - start_sample).count() << "ms"; - } - - VLOG(0) << "Device " << profiler_->device() - << ": Exited event profiling loop"; -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/EventProfilerController.h b/plugins/tensorboard-plugins/libkineto/src/EventProfilerController.h deleted file mode 100644 index 007a82faa..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/EventProfilerController.h +++ /dev/null @@ -1,63 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include - -#include - -#include "ConfigLoader.h" - -namespace KINETO_NAMESPACE { - -class Config; -class ConfigLoader; -class EventProfiler; -class SampleListener; - -namespace detail { -class HeartbeatMonitor; -} - -class EventProfilerController : public ConfigLoader::ConfigHandler { - public: - EventProfilerController(const EventProfilerController&) = delete; - EventProfilerController& operator=(const EventProfilerController&) = delete; - - ~EventProfilerController(); - - static void start(CUcontext ctx, ConfigLoader& configLoader); - static void stop(CUcontext ctx); - - static void addLoggerFactory( - std::function(const Config&)> factory); - - static void addOnDemandLoggerFactory( - std::function(const Config&)> factory); - - bool canAcceptConfig() override; - - void acceptConfig(const Config& config) override; - - private: - explicit EventProfilerController( - CUcontext context, - ConfigLoader& configLoader, - detail::HeartbeatMonitor& heartbeatMonitor); - bool enableForDevice(Config& cfg); - void profilerLoop(); - - ConfigLoader& configLoader_; - std::unique_ptr newOnDemandConfig_; - detail::HeartbeatMonitor& heartbeatMonitor_; - std::unique_ptr profiler_; - std::unique_ptr profilerThread_; - std::atomic_bool stopRunloop_{false}; - std::mutex mutex_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/GenericTraceActivity.cpp b/plugins/tensorboard-plugins/libkineto/src/GenericTraceActivity.cpp deleted file mode 100644 index 4e00b1256..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/GenericTraceActivity.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "GenericTraceActivity.h" -#include "output_base.h" - -namespace libkineto { - void GenericTraceActivity::log(ActivityLogger& logger) const { - logger.handleGenericActivity(*this); - } -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/ILoggerObserver.cpp b/plugins/tensorboard-plugins/libkineto/src/ILoggerObserver.cpp deleted file mode 100644 index f01065788..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ILoggerObserver.cpp +++ /dev/null @@ -1,54 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "ILoggerObserver.h" - -#if !USE_GOOGLE_LOG - -#include -#include - -namespace libkineto { - -struct LoggerTypeName { - constexpr LoggerTypeName(const char* n, LoggerOutputType t) : name(n), type(t) {}; - const char* name; - LoggerOutputType type; -}; - -static constexpr std::array LoggerMap{{ - {"VERBOSE", LoggerOutputType::VERBOSE}, - {"INFO", LoggerOutputType::INFO}, - {"WARNING", LoggerOutputType::WARNING}, - {"ERROR", LoggerOutputType::ERROR}, - {"STAGE", LoggerOutputType::STAGE}, - {"???", LoggerOutputType::ENUM_COUNT} -}}; - -static constexpr bool matchingOrder(int idx = 0) { - return LoggerMap[idx].type == LoggerOutputType::ENUM_COUNT || - ((idx == (int) LoggerMap[idx].type) && matchingOrder(idx + 1)); -} -static_assert(matchingOrder(), "LoggerTypeName map is out of order"); - -const char* toString(LoggerOutputType t) { - if(t < VERBOSE || t >= ENUM_COUNT) { - return LoggerMap[ENUM_COUNT].name; - } - return LoggerMap[(int)t].name; -} - -LoggerOutputType toLoggerOutputType(const std::string& str) { - for (int i = 0; i < LoggerTypeCount; i++) { - if (str == LoggerMap[i].name) { - return LoggerMap[i].type; - } - } - throw std::invalid_argument(fmt::format("Invalid activity type: {}", str)); -} - -} // namespace libkineto - - -#endif // !USE_GOOGLE_LOG diff --git a/plugins/tensorboard-plugins/libkineto/src/Logger.cpp b/plugins/tensorboard-plugins/libkineto/src/Logger.cpp deleted file mode 100644 index dbde765f5..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/Logger.cpp +++ /dev/null @@ -1,136 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "Logger.h" -#include "ILoggerObserver.h" - -#ifndef USE_GOOGLE_LOG - -#include -#include -#include -#include -#include - -#include -#include - -#include "ThreadUtil.h" - -namespace KINETO_NAMESPACE { - -std::atomic_int Logger::severityLevel_{VERBOSE}; -std::atomic_int Logger::verboseLogLevel_{-1}; -std::atomic Logger::verboseLogModules_{~0ull}; - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wglobal-constructors" -std::mutex Logger::loggerObserversMutex_; -#pragma GCC diagnostic pop - - -Logger::Logger(int severity, int line, const char* filePath, int errnum) - : buf_(), out_(LIBKINETO_DBG_STREAM), errnum_(errnum), messageSeverity_(severity) { - buf_ << toString((LoggerOutputType) severity) << ":"; - - const auto tt = - std::chrono::system_clock::to_time_t(std::chrono::system_clock::now()); - const char* file = strrchr(filePath, '/'); - buf_ << fmt::format("{:%Y-%m-%d %H:%M:%S}", fmt::localtime(tt)) << " " - << processId() << ":" << systemThreadId() << " " - << (file ? file + 1 : filePath) << ":" << line << "] "; -} - -Logger::~Logger() { -#ifdef __linux__ - if (errnum_ != 0) { - thread_local char buf[1024]; - buf_ << " : " << strerror_r(errnum_, buf, sizeof(buf)); - } -#endif - - { - std::lock_guard guard(loggerObserversMutex_); - for (auto* observer : loggerObservers()) { - // Output to observers. Current Severity helps keep track of which bucket the output goes. - if (observer) { - observer->write(buf_.str(), (LoggerOutputType) messageSeverity_); - } - } - } - - // Finally, print to terminal or console. - out_ << buf_.str() << std::endl; -} - -void Logger::setVerboseLogModules(const std::vector& modules) { - uint64_t mask = 0; - if (modules.empty()) { - mask = ~0ull; - } else { - for (const std::string& name : modules) { - mask |= hash(name.c_str()); - } - } - verboseLogModules_ = mask; -} - -void Logger::addLoggerObserver(ILoggerObserver* observer) { - if (observer == nullptr) { - return; - } - std::lock_guard guard(loggerObserversMutex_); - loggerObservers().insert(observer); -} - -void Logger::removeLoggerObserver(ILoggerObserver* observer) { - std::lock_guard guard(loggerObserversMutex_); - loggerObservers().erase(observer); -} - -void Logger::addLoggerObserverDevice(int64_t device) { - std::lock_guard guard(loggerObserversMutex_); - for (auto observer : loggerObservers()) { - observer->addDevice(device); - } -} - -void Logger::addLoggerObserverEventCount(int64_t count) { - std::lock_guard guard(loggerObserversMutex_); - for (auto observer : loggerObservers()) { - observer->addEventCount(count); - } -} - -void Logger::setLoggerObserverTraceDurationMS(int64_t duration) { - std::lock_guard guard(loggerObserversMutex_); - for (auto observer : loggerObservers()) { - observer->setTraceDurationMS(duration); - } -} - -void Logger::setLoggerObserverTraceID(const std::string& tid) { - std::lock_guard guard(loggerObserversMutex_); - for (auto observer : loggerObservers()) { - observer->setTraceID(tid); - } -} - -void Logger::setLoggerObserverGroupTraceID(const std::string& gtid) { - std::lock_guard guard(loggerObserversMutex_); - for (auto observer : loggerObservers()) { - observer->setGroupTraceID(gtid); - } -} - -void Logger::addLoggerObserverDestination(const std::string& dest) { - std::lock_guard guard(loggerObserversMutex_); - for (auto observer : loggerObservers()) { - observer->addDestination(dest); - } -} - -} // namespace KINETO_NAMESPACE - -#endif // USE_GOOGLE_LOG diff --git a/plugins/tensorboard-plugins/libkineto/src/Logger.h b/plugins/tensorboard-plugins/libkineto/src/Logger.h deleted file mode 100644 index 868fc84b9..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/Logger.h +++ /dev/null @@ -1,244 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -#define LIBKINETO_DBG_STREAM std::cerr - -#if USE_GOOGLE_LOG - -#include - -#define SET_LOG_SEVERITY_LEVEL(level) -#define SET_LOG_VERBOSITY_LEVEL(level, modules) -#define LOGGER_OBSERVER_ADD_DEVICE(device) -#define LOGGER_OBSERVER_ADD_EVENT_COUNT(count) -#define LOGGER_OBSERVER_SET_TRACE_DURATION_MS(duration) -#define LOGGER_OBSERVER_SET_TRACE_ID(tid) -#define LOGGER_OBSERVER_SET_GROUP_TRACE_ID(gtid) -#define LOGGER_OBSERVER_ADD_DESTINATION(dest) -#define UST_LOGGER_MARK_COMPLETED(stage) - -#else // !USE_GOOGLE_LOG -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "ILoggerObserver.h" - -#ifdef _MSC_VER -// unset a predefined ERROR (windows) -#undef ERROR -#endif // _MSC_VER - -namespace KINETO_NAMESPACE { - -class Logger { - public: - Logger(int severity, int line, const char* filePath, int errnum = 0); - ~Logger(); - - inline std::ostream& stream() { - return buf_; - } - - static inline void setSeverityLevel(int level) { - severityLevel_ = level; - } - - static inline int severityLevel() { - return severityLevel_; - } - - static inline void setVerboseLogLevel(int level) { - verboseLogLevel_ = level; - } - - static inline int verboseLogLevel() { - return verboseLogLevel_; - } - - // This is constexpr so that the hash for a file name is computed at compile - // time when used in the VLOG macros. - // This way, there is no string comparison for matching VLOG modules, - // only a comparison of pre-computed hashes. - // No fancy hashing needed here. It's pretty inefficient (one character - // at a time) but the strings are not large and it's not in the critical path. - static constexpr uint64_t rol(uint64_t val, int amount) { - return val << amount | val >> (63 - amount); - } - static constexpr uint64_t hash(const char* s) { - uint64_t hash = hash_rec(s, 0); - return hash & rol(0x41a0240682483014ull, hash & 63); - } - static constexpr uint64_t hash_rec(const char* s, int off) { - // Random constants! - return (!s[off] ? 57ull : (hash_rec(s, off + 1) * 293) ^ s[off]); - } - static constexpr const char* basename(const char* s, int off = 0) { - return !s[off] - ? s - : s[off] == '/' ? basename(&s[off + 1]) : basename(s, off + 1); - } - - static void setVerboseLogModules(const std::vector& modules); - - static inline uint64_t verboseLogModules() { - return verboseLogModules_; - } - - static void clearLoggerObservers() { - std::lock_guard g(loggerObserversMutex_); - loggerObservers().clear(); - } - - static void addLoggerObserver(ILoggerObserver* observer); - - static void removeLoggerObserver(ILoggerObserver* observer); - - static void addLoggerObserverDevice(int64_t device); - - static void addLoggerObserverEventCount(int64_t count); - - static void setLoggerObserverTraceDurationMS(int64_t duration); - - static void setLoggerObserverTraceID(const std::string& tid); - - static void setLoggerObserverGroupTraceID(const std::string& gtid); - - static void addLoggerObserverDestination(const std::string& dest); - - private: - std::stringstream buf_; - std::ostream& out_; - int errnum_; - int messageSeverity_; - static std::atomic_int severityLevel_; - static std::atomic_int verboseLogLevel_; - static std::atomic verboseLogModules_; - static std::set& loggerObservers() { - static auto* inst = new std::set(); - return *inst; - } - static std::mutex loggerObserversMutex_; -}; - -class VoidLogger { - public: - VoidLogger() {} - void operator&(std::ostream&) {} -}; - -} // namespace KINETO_NAMESPACE - -#ifdef LOG // Undefine in case these are already defined (quite likely) -#undef LOG -#undef LOG_IS_ON -#undef LOG_IF -#undef LOG_EVERY_N -#undef LOG_IF_EVERY_N -#undef DLOG -#undef DLOG_IF -#undef VLOG -#undef VLOG_IF -#undef VLOG_EVERY_N -#undef VLOG_IS_ON -#undef DVLOG -#undef LOG_FIRST_N -#undef CHECK -#undef DCHECK -#undef DCHECK_EQ -#undef PLOG -#undef PCHECK -#undef LOG_OCCURRENCES -#endif - -#define LOG_IS_ON(severity) \ - (severity >= libkineto::Logger::severityLevel()) - -#define LOG_IF(severity, condition) \ - !(LOG_IS_ON(severity) && (condition)) ? (void)0 : libkineto::VoidLogger() & \ - libkineto::Logger(severity, __LINE__, __FILE__).stream() - -#define LOG(severity) LOG_IF(severity, true) - -#define LOCAL_VARNAME_CONCAT(name, suffix) _##name##suffix##_ - -#define LOCAL_VARNAME(name) LOCAL_VARNAME_CONCAT(name, __LINE__) - -#define LOG_OCCURRENCES LOCAL_VARNAME(log_count) - -#define LOG_EVERY_N(severity, rate) \ - static int LOG_OCCURRENCES = 0; \ - LOG_IF(severity, LOG_OCCURRENCES++ % rate == 0) \ - << "(x" << LOG_OCCURRENCES << ") " - -template -struct __to_constant__ { - static const uint64_t val = n; -}; -#define FILENAME_HASH \ - __to_constant__::val -#define VLOG_IS_ON(verbosity) \ - (libkineto::Logger::verboseLogLevel() >= verbosity && \ - (libkineto::Logger::verboseLogModules() & FILENAME_HASH) == FILENAME_HASH) - -#define VLOG_IF(verbosity, condition) \ - LOG_IF(VERBOSE, VLOG_IS_ON(verbosity) && (condition)) - -#define VLOG(verbosity) VLOG_IF(verbosity, true) - -#define VLOG_EVERY_N(verbosity, rate) \ - static int LOG_OCCURRENCES = 0; \ - VLOG_IF(verbosity, LOG_OCCURRENCES++ % rate == 0) \ - << "(x" << LOG_OCCURRENCES << ") " - -#define PLOG(severity) \ - libkineto::Logger(severity, __LINE__, __FILE__, errno).stream() - -#define SET_LOG_SEVERITY_LEVEL(level) \ - libkineto::Logger::setSeverityLevel(level) - -#define SET_LOG_VERBOSITY_LEVEL(level, modules) \ - libkineto::Logger::setVerboseLogLevel(level); \ - libkineto::Logger::setVerboseLogModules(modules) - -// Logging the set of devices the trace is collect on. -#define LOGGER_OBSERVER_ADD_DEVICE(device_count) \ - libkineto::Logger::addLoggerObserverDevice(device_count) - -// Incrementing the number of events collected by this trace. -#define LOGGER_OBSERVER_ADD_EVENT_COUNT(count) \ - libkineto::Logger::addLoggerObserverEventCount(count) - -// Record duration of trace in milliseconds. -#define LOGGER_OBSERVER_SET_TRACE_DURATION_MS(duration) \ - libkineto::Logger::setLoggerObserverTraceDurationMS(duration) - -// Record the trace id when given. -#define LOGGER_OBSERVER_SET_TRACE_ID(tid) \ - libkineto::Logger::setLoggerObserverTraceID(tid) - -// Record the group trace id when given. -#define LOGGER_OBSERVER_SET_GROUP_TRACE_ID(gtid) \ - libkineto::Logger::setLoggerObserverGroupTraceID(gtid) - -// Log the set of destinations the trace is sent to. -#define LOGGER_OBSERVER_ADD_DESTINATION(dest) \ - libkineto::Logger::addLoggerObserverDestination(dest) - -// UST Logger Semantics to describe when a stage is complete. -#define UST_LOGGER_MARK_COMPLETED(stage) \ - LOG(libkineto::LoggerOutputType::STAGE) << "Completed Stage: " << stage - -#endif // USE_GOOGLE_LOG diff --git a/plugins/tensorboard-plugins/libkineto/src/LoggerCollector.h b/plugins/tensorboard-plugins/libkineto/src/LoggerCollector.h deleted file mode 100644 index bb05aab21..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/LoggerCollector.h +++ /dev/null @@ -1,70 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#if !USE_GOOGLE_LOG - -#include -#include -#include -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "ILoggerObserver.h" - -namespace KINETO_NAMESPACE { - -using namespace libkineto; - -class LoggerCollector : public ILoggerObserver { - public: - LoggerCollector() : buckets_() {} - - void write(const std::string& message, LoggerOutputType ot = ERROR) override { - // Skip STAGE output type which is only used by USTLoggerCollector. - if (ot != STAGE) { - buckets_[ot].push_back(message); - } - } - - const std::map> extractCollectorMetadata() override { - return buckets_; - } - - void reset() override { - trace_duration_ms = 0; - event_count = 0; - destinations.clear(); - } - - void addDevice(const int64_t device) override { - devices.insert(device); - } - - void setTraceDurationMS(const int64_t duration) override { - trace_duration_ms = duration; - } - - void addEventCount(const int64_t count) override { - event_count += count; - } - - void addDestination(const std::string& dest) override { - destinations.insert(dest); - } - - protected: - std::map> buckets_; - - // These are useful metadata to collect from CUPTIActivityProfiler for internal tracking. - std::set devices; - int64_t trace_duration_ms{0}; - std::atomic event_count{0}; - std::set destinations; - -}; - -} // namespace KINETO_NAMESPACE - -#endif // !USE_GOOGLE_LOG diff --git a/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityApi.cpp b/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityApi.cpp deleted file mode 100644 index 73eff13e2..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityApi.cpp +++ /dev/null @@ -1,569 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "RoctracerActivityApi.h" - -#include -#include -#include - -#include "Demangle.h" -#include "output_base.h" -#include "ThreadUtil.h" - -typedef uint64_t timestamp_t; - -static timestamp_t timespec_to_ns(const timespec& time) { - return ((timestamp_t)time.tv_sec * 1000000000) + time.tv_nsec; - } - -using namespace std::chrono; - -namespace KINETO_NAMESPACE { - -constexpr size_t kBufSize(2 * 1024 * 1024); - -RoctracerActivityApi& RoctracerActivityApi::singleton() { - static RoctracerActivityApi instance; - return instance; -} - -RoctracerActivityApi::RoctracerActivityApi() { - gpuTraceBuffers_ = std::make_unique>(); -} - -RoctracerActivityApi::~RoctracerActivityApi() { - disableActivities(std::set()); - endTracing(); -} - -void RoctracerActivityApi::pushCorrelationID(int id, CorrelationFlowType type) { -#ifdef HAS_ROCTRACER - if (!singleton().externalCorrelationEnabled_) { - return; - } - // placeholder -#endif -} - -void RoctracerActivityApi::popCorrelationID(CorrelationFlowType type) { -#ifdef HAS_ROCTRACER - if (!singleton().externalCorrelationEnabled_) { - return; - } - // placeholder -#endif -} - -void RoctracerActivityApi::setMaxBufferSize(int size) { - maxGpuBufferCount_ = 1 + size / kBufSize; -} - -int RoctracerActivityApi::processActivities( - ActivityLogger& logger) { - // Find offset to map from monotonic clock to system clock. - // This will break time-ordering of events but is status quo. - - timespec t0, t1, t00; - clock_gettime(CLOCK_REALTIME, &t0); - clock_gettime(CLOCK_MONOTONIC, &t1); - clock_gettime(CLOCK_REALTIME, &t00); - - const timestamp_t toffset = (timespec_to_ns(t0) >> 1) + (timespec_to_ns(t00) >> 1) - timespec_to_ns(t1); - - int count = 0; - - // Basic Api calls - - for (auto &item : rows_) { - GenericTraceActivity a; - a.startTime = (item.begin + toffset) / 1000; - a.endTime = (item.end + toffset) / 1000; - a.id = item.id; - a.device = item.pid; - a.resource = item.tid; - a.activityType = ActivityType::CUDA_RUNTIME; - a.activityName = std::string(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, item.cid, 0)); - a.flow.id = item.id; - a.flow.type = kLinkAsyncCpuGpu; - a.flow.start = true; - - logger.handleGenericActivity(a); - ++count; - } - - // Malloc/Free calls - for (auto &item : mallocRows_) { - GenericTraceActivity a; - a.startTime = (item.begin + toffset) / 1000; - a.endTime = (item.end + toffset) / 1000; - a.id = item.id; - a.device = item.pid; - a.resource = item.tid; - a.activityType = ActivityType::CUDA_RUNTIME; - a.activityName = std::string(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, item.cid, 0)); - a.flow.id = item.id; - a.flow.type = kLinkAsyncCpuGpu; - a.flow.start = true; - - a.addMetadata("ptr", item.ptr); - if (item.cid == HIP_API_ID_hipMalloc) { - a.addMetadata("size", item.size); - } - - logger.handleGenericActivity(a); - ++count; - } - - // HipMemcpy calls - for (auto &item : copyRows_) { - GenericTraceActivity a; - a.startTime = (item.begin + toffset) / 1000; - a.endTime = (item.end + toffset) / 1000; - a.id = item.id; - a.device = item.pid; - a.resource = item.tid; - a.activityType = ActivityType::CUDA_RUNTIME; - a.activityName = std::string(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, item.cid, 0)); - a.flow.id = item.id; - a.flow.type = kLinkAsyncCpuGpu; - a.flow.start = true; - - a.addMetadata("src", item.src); - a.addMetadata("dst", item.dst); - a.addMetadata("size", item.size); - a.addMetadata("kind", item.kind); - if ((item.cid == HIP_API_ID_hipMemcpyAsync) || (item.cid == HIP_API_ID_hipMemcpyWithStream)) { - a.addMetadata("stream", fmt::format("{}", reinterpret_cast(item.stream))); - } - - logger.handleGenericActivity(a); - ++count; - } - - // Kernel Launch Api calls - - for (auto &item : kernelRows_) { - GenericTraceActivity a; - a.startTime = (item.begin + toffset) / 1000; - a.endTime = (item.end + toffset) / 1000; - a.id = item.id; - a.device = item.pid; - a.resource = item.tid; - a.activityType = ActivityType::CUDA_RUNTIME; - a.activityName = std::string(roctracer_op_string(ACTIVITY_DOMAIN_HIP_API, item.cid, 0)); - a.flow.id = item.id; - a.flow.type = kLinkAsyncCpuGpu; - a.flow.start = true; - - if (item.functionAddr != nullptr) { - a.addMetadataQuoted( - "kernel", demangle(hipKernelNameRefByPtr(item.functionAddr, item.stream))); - } - else if (item.function != nullptr) { - a.addMetadataQuoted( - "kernel", demangle(hipKernelNameRef(item.function))); - } - a.addMetadata("grid dim", fmt::format("[{}, {}, {}]", item.gridX, item.gridY, item.gridZ)); - a.addMetadata("block dim", fmt::format("[{}, {}, {}]", item.workgroupX, item.workgroupY, item.workgroupZ)); - a.addMetadata("shared size", item.groupSegmentSize); - a.addMetadata("stream", fmt::format("{}", reinterpret_cast(item.stream))); - - // Stash launches to tie to the async ops - kernelLaunches_[a.id] = a; - - // Stash kernel names to tie to the async ops - std::string name; - if (item.functionAddr != nullptr) { - name = demangle(hipKernelNameRefByPtr(item.functionAddr, item.stream)); - } - else if (item.function != nullptr) { - name = demangle(hipKernelNameRef(item.function)); - } - if (!name.empty()) { - uint32_t string_id = reverseStrings_[name]; - if (string_id == 0) { - string_id = nextStringId_++; - reverseStrings_[name] = string_id; - strings_[string_id] = name; - } - kernelNames_[item.id] = string_id; - } - - logger.handleGenericActivity(a); - ++count; - } - - // Async Ops - - for (auto& buffer : *gpuTraceBuffers_) { - const roctracer_record_t* record = (const roctracer_record_t*)(buffer.data); - const roctracer_record_t* end_record = (const roctracer_record_t*)(buffer.data + buffer.validSize); - GenericTraceActivity a; - - while (record < end_record) { - if ((record->domain == ACTIVITY_DOMAIN_HIP_API) && (loggedIds_.contains(record->op))) { - const char *name = roctracer_op_string(record->domain, record->op, record->kind); - a.device = record->process_id; - a.resource = record->thread_id; - - a.startTime = (record->begin_ns + toffset) / 1000; - a.endTime = (record->end_ns + toffset) / 1000; - a.id = record->correlation_id; - - a.activityType = ActivityType::CUDA_RUNTIME; - a.activityName = std::string(name); - a.flow.id = record->correlation_id; - a.flow.type = kLinkAsyncCpuGpu; - a.flow.start = true; - - logger.handleGenericActivity(a); - ++count; - } - else if (record->domain == ACTIVITY_DOMAIN_HCC_OPS) { - // Overlay launch metadata for kernels - auto kit = kernelLaunches_.find(record->correlation_id); - if (kit != kernelLaunches_.end()) { - a = (*kit).second; - } - - const char *name = roctracer_op_string(record->domain, record->op, record->kind); - a.device = record->device_id; - a.resource = record->queue_id; - - a.startTime = (record->begin_ns + toffset) / 1000; - a.endTime = (record->end_ns + toffset) / 1000; - a.id = record->correlation_id; - - a.activityType = ActivityType::CONCURRENT_KERNEL; - a.activityName = std::string(name); - a.flow.id = record->correlation_id; - a.flow.type = kLinkAsyncCpuGpu; - - auto it = kernelNames_.find(record->correlation_id); - if (it != kernelNames_.end()) { - a.activityName = strings_[it->second]; - } - - logger.handleGenericActivity(a); - ++count; - } - - roctracer_next_record(record, &record); - } - } - return count; -} - -void RoctracerActivityApi::clearActivities() { - gpuTraceBuffers_->clear(); - rows_.clear(); - kernelRows_.clear(); - copyRows_.clear(); - mallocRows_.clear(); - kernelLaunches_.clear(); -} - -void RoctracerActivityApi::api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg) -{ - RoctracerActivityApi *dis = &singleton(); - - if (domain == ACTIVITY_DOMAIN_HIP_API && dis->loggedIds_.contains(cid)) { - const hip_api_data_t* data = (const hip_api_data_t*)(callback_data); - - // Pack callbacks into row structures - - static timespec timestamp; // FIXME verify thread safety - - if (data->phase == ACTIVITY_API_PHASE_ENTER) { - clock_gettime(CLOCK_MONOTONIC, ×tamp); // record proper clock - } - else { // (data->phase == ACTIVITY_API_PHASE_EXIT) - timespec endTime; - timespec startTime { timestamp }; - clock_gettime(CLOCK_MONOTONIC, &endTime); // record proper clock - - switch (cid) { - case HIP_API_ID_hipLaunchKernel: - case HIP_API_ID_hipExtLaunchKernel: - case HIP_API_ID_hipLaunchCooperativeKernel: // Should work here - { - auto &args = data->args.hipLaunchKernel; - dis->kernelRows_.emplace_back(data->correlation_id, - domain, - cid, - processId(), - systemThreadId(), - timespec_to_ns(startTime), - timespec_to_ns(endTime), - args.function_address, - nullptr, - args.numBlocks.x, - args.numBlocks.y, - args.numBlocks.z, - args.dimBlocks.x, - args.dimBlocks.y, - args.dimBlocks.z, - args.sharedMemBytes, - args.stream - ); - } - break; - case HIP_API_ID_hipHccModuleLaunchKernel: - case HIP_API_ID_hipModuleLaunchKernel: - case HIP_API_ID_hipExtModuleLaunchKernel: - { - auto &args = data->args.hipModuleLaunchKernel; - dis->kernelRows_.emplace_back(data->correlation_id, - domain, - cid, - processId(), - systemThreadId(), - timespec_to_ns(startTime), - timespec_to_ns(endTime), - nullptr, - args.f, - args.gridDimX, - args.gridDimY, - args.gridDimZ, - args.blockDimX, - args.blockDimY, - args.blockDimZ, - args.sharedMemBytes, - args.stream - ); - } - break; - case HIP_API_ID_hipLaunchCooperativeKernelMultiDevice: - case HIP_API_ID_hipExtLaunchMultiKernelMultiDevice: -#if 0 - { - auto &args = data->args.hipLaunchCooperativeKernelMultiDevice.launchParamsList__val; - dis->kernelRows_.emplace_back(data->correlation_id, - domain, - cid, - processId(), - systemThreadId(), - timespec_to_ns(startTime), - timespec_to_ns(endTime), - args.function_address, - nullptr, - args.numBlocks.x, - args.numBlocks.y, - args.numBlocks.z, - args.dimBlocks.x, - args.dimBlocks.y, - args.dimBlocks.z, - args.sharedMemBytes, - args.stream - ); - } -#endif - break; - case HIP_API_ID_hipMalloc: - dis->mallocRows_.emplace_back(data->correlation_id, - domain, - cid, - processId(), - systemThreadId(), - timespec_to_ns(startTime), - timespec_to_ns(endTime), - data->args.hipMalloc.ptr__val, - data->args.hipMalloc.size - ); - break; - case HIP_API_ID_hipFree: - dis->mallocRows_.emplace_back(data->correlation_id, - domain, - cid, - processId(), - systemThreadId(), - timespec_to_ns(startTime), - timespec_to_ns(endTime), - data->args.hipFree.ptr, - 0 - ); - break; - case HIP_API_ID_hipMemcpy: - { - auto &args = data->args.hipMemcpy; - dis->copyRows_.emplace_back(data->correlation_id, - domain, - cid, - processId(), - systemThreadId(), - timespec_to_ns(startTime), - timespec_to_ns(endTime), - args.src, - args.dst, - args.sizeBytes, - args.kind, - static_cast(0) // use placeholder? - ); - } - break; - case HIP_API_ID_hipMemcpyAsync: - case HIP_API_ID_hipMemcpyWithStream: - { - auto &args = data->args.hipMemcpyAsync; - dis->copyRows_.emplace_back(data->correlation_id, - domain, - cid, - processId(), - systemThreadId(), - timespec_to_ns(startTime), - timespec_to_ns(endTime), - args.src, - args.dst, - args.sizeBytes, - args.kind, - args.stream - ); - } - break; - default: - dis->rows_.emplace_back(data->correlation_id, - domain, - cid, - processId(), - systemThreadId(), - timespec_to_ns(startTime), - timespec_to_ns(endTime) - ); - break; - } - } - } -} - -void RoctracerActivityApi::activity_callback(const char* begin, const char* end, void* arg) -{ - size_t size = end - begin; - uint8_t *buffer = (uint8_t*) malloc(size); - auto &gpuTraceBuffers = singleton().gpuTraceBuffers_; - memcpy(buffer, begin, size); - gpuTraceBuffers->emplace_back(buffer, size); -} - -void RoctracerActivityApi::enableActivities( - const std::set& selected_activities) { -#ifdef HAS_ROCTRACER - if (!registered_) { - roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr); // Magic encantation - - // Set some api calls to ignore - loggedIds_.setInvertMode(true); // Omit the specified api - loggedIds_.add("hipGetDevice"); - loggedIds_.add("hipSetDevice"); - loggedIds_.add("hipGetLastError"); - loggedIds_.add("__hipPushCallConfiguration"); - loggedIds_.add("__hipPopCallConfiguration"); - loggedIds_.add("hipCtxSetCurrent"); - loggedIds_.add("hipEventRecord"); - loggedIds_.add("hipEventQuery"); - loggedIds_.add("hipGetDeviceProperties"); - loggedIds_.add("hipPeekAtLastError"); - loggedIds_.add("hipModuleGetFunction"); - loggedIds_.add("hipEventCreateWithFlags"); - - // Enable API callbacks - if (loggedIds_.invertMode() == true) { - // exclusion list - enable entire domain and turn off things in list - roctracer_enable_domain_callback(ACTIVITY_DOMAIN_HIP_API, api_callback, nullptr); - const std::unordered_map &filter = loggedIds_.filterList(); - for (auto it = filter.begin(); it != filter.end(); ++it) { - roctracer_disable_op_callback(ACTIVITY_DOMAIN_HIP_API, it->first); - } - } - else { - // inclusion list - only enable things in the list - const std::unordered_map &filter = loggedIds_.filterList(); - roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API); - for (auto it = filter.begin(); it != filter.end(); ++it) { - roctracer_enable_op_callback(ACTIVITY_DOMAIN_HIP_API, it->first, api_callback, nullptr); - } - } - //roctracer_enable_domain_callback(ACTIVITY_DOMAIN_ROCTX, api_callback, nullptr); - - // Allocate default tracing pool - roctracer_properties_t properties; - memset(&properties, 0, sizeof(roctracer_properties_t)); - properties.buffer_size = 0x1000; - roctracer_open_pool(&properties); - - // Enable async op collection - roctracer_properties_t hcc_cb_properties; - memset(&hcc_cb_properties, 0, sizeof(roctracer_properties_t)); - hcc_cb_properties.buffer_size = 0x4000; - hcc_cb_properties.buffer_callback_fun = activity_callback; - roctracer_open_pool_expl(&hcc_cb_properties, &hccPool_); - roctracer_enable_domain_activity_expl(ACTIVITY_DOMAIN_HCC_OPS, hccPool_); - - registered_ = true; - } - - for (const auto& activity : selected_activities) { - if (activity == ActivityType::EXTERNAL_CORRELATION) { - externalCorrelationEnabled_ = true; - } - } - - roctracer_start(); -#endif -} - -void RoctracerActivityApi::disableActivities( - const std::set& selected_activities) { -#ifdef HAS_ROCTRACER - roctracer_stop(); - roctracer_flush_activity_expl(hccPool_); - - for (const auto& activity : selected_activities) { - if (activity == ActivityType::EXTERNAL_CORRELATION) { - externalCorrelationEnabled_ = false; - } - } -#endif -} - -void RoctracerActivityApi::endTracing() { - if (registered_ == true) { - roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API); - //roctracer_disable_domain_callback(ACTIVITY_DOMAIN_ROCTX); - - roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HCC_OPS); - roctracer_close_pool_expl(hccPool_); - } -} - - -ApiIdList::ApiIdList() -: invert_(true) -{ -} - -void ApiIdList::add(std::string apiName) -{ - uint32_t cid = 0; - if (roctracer_op_code(ACTIVITY_DOMAIN_HIP_API, apiName.c_str(), &cid, nullptr) == ROCTRACER_STATUS_SUCCESS) { - filter_[cid] = 1; - } -} -void ApiIdList::remove(std::string apiName) -{ - uint32_t cid = 0; - if (roctracer_op_code(ACTIVITY_DOMAIN_HIP_API, apiName.c_str(), &cid, nullptr) == ROCTRACER_STATUS_SUCCESS) { - filter_.erase(cid); - } -} - -bool ApiIdList::loadUserPrefs() -{ - // placeholder - return false; -} -bool ApiIdList::contains(uint32_t apiId) -{ - return (filter_.find(apiId) != filter_.end()) ? !invert_ : invert_; // XOR -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityApi.h b/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityApi.h deleted file mode 100644 index 28280253e..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityApi.h +++ /dev/null @@ -1,171 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef HAS_ROCTRACER -#include -#include -#include -#include -#include -#endif - -#include "ActivityType.h" -#include "GenericTraceActivity.h" -#include "RoctracerActivityBuffer.h" - - -namespace KINETO_NAMESPACE { - -using namespace libkineto; - -class ApiIdList -{ -public: - ApiIdList(); - bool invertMode() { return invert_; } - void setInvertMode(bool invert) { invert_ = invert; } - void add(std::string apiName); - void remove(std::string apiName); - bool loadUserPrefs(); - bool contains(uint32_t apiId); - const std::unordered_map &filterList() { return filter_; } - -private: - std::unordered_map filter_; - bool invert_; -}; - -struct roctracerRow { - roctracerRow(uint64_t id, uint32_t domain, uint32_t cid, uint32_t pid - , uint32_t tid, uint64_t begin, uint64_t end) - : id(id), domain(domain), cid(cid), pid(pid), tid(tid), begin(begin), end(end) {} - uint64_t id; // correlation_id - uint32_t domain; - uint32_t cid; - uint32_t pid; - uint32_t tid; - uint64_t begin; - uint64_t end; -}; - -struct kernelRow : public roctracerRow { - kernelRow(uint64_t id, uint32_t domain, uint32_t cid, uint32_t pid - , uint32_t tid, uint64_t begin, uint64_t end - , const void *faddr, hipFunction_t function - , unsigned int gx, unsigned int gy, unsigned int gz - , unsigned int wx, unsigned int wy, unsigned int wz - , size_t gss, hipStream_t stream) - : roctracerRow(id, domain, cid, pid, tid, begin, end), functionAddr(faddr) - , function(function), gridX(gx), gridY(gy), gridZ(gz) - , workgroupX(wx), workgroupY(wy), workgroupZ(wz), groupSegmentSize(gss) - , stream(stream) {} - const void* functionAddr; - hipFunction_t function; - unsigned int gridX; - unsigned int gridY; - unsigned int gridZ; - unsigned int workgroupX; - unsigned int workgroupY; - unsigned int workgroupZ; - size_t groupSegmentSize; - hipStream_t stream; -}; - -struct copyRow : public roctracerRow { - copyRow(uint64_t id, uint32_t domain, uint32_t cid, uint32_t pid - , uint32_t tid, uint64_t begin, uint64_t end - , const void* src, const void *dst, size_t size, hipMemcpyKind kind - , hipStream_t stream) - : roctracerRow(id, domain, cid, pid, tid, begin, end) - , src(src), dst(dst), size(size), kind(kind), stream(stream) {} - const void *src; - const void *dst; - size_t size; - hipMemcpyKind kind; - hipStream_t stream; -}; - -struct mallocRow : public roctracerRow { - mallocRow(uint64_t id, uint32_t domain, uint32_t cid, uint32_t pid - , uint32_t tid, uint64_t begin, uint64_t end - , const void* ptr, size_t size) - : roctracerRow(id, domain, cid, pid, tid, begin, end) - , ptr(ptr), size(size) {} - const void *ptr; - size_t size; -}; - - -class RoctracerActivityApi { - public: - enum CorrelationFlowType { - Default, - User - }; - - RoctracerActivityApi(); - RoctracerActivityApi(const RoctracerActivityApi&) = delete; - RoctracerActivityApi& operator=(const RoctracerActivityApi&) = delete; - - virtual ~RoctracerActivityApi(); - - static RoctracerActivityApi& singleton(); - - static void pushCorrelationID(int id, CorrelationFlowType type); - static void popCorrelationID(CorrelationFlowType type); - - void enableActivities( - const std::set& selected_activities); - void disableActivities( - const std::set& selected_activities); - void clearActivities(); - - int processActivities(ActivityLogger& logger); - - void setMaxBufferSize(int size); - - std::atomic_bool stopCollection{false}; - - private: - bool registered_{false}; - void endTracing(); - -#ifdef HAS_ROCTRACER - roctracer_pool_t *hccPool_{NULL}; - static void api_callback(uint32_t domain, uint32_t cid, const void* callback_data, void* arg); - static void activity_callback(const char* begin, const char* end, void* arg); - - //Name cache - uint32_t nextStringId_{2}; - std::map strings_; - std::map reverseStrings_; - std::map kernelNames_; - - ApiIdList loggedIds_; - - // Api callback data - std::deque rows_; - std::deque kernelRows_; - std::deque copyRows_; - std::deque mallocRows_; - std::map kernelLaunches_; -#endif - - int maxGpuBufferCount_{0}; - std::unique_ptr> gpuTraceBuffers_; - bool externalCorrelationEnabled_{true}; -}; - -} // namespace KINETO_NAMESPACE - diff --git a/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityBuffer.h b/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityBuffer.h deleted file mode 100644 index cd8a5709a..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/RoctracerActivityBuffer.h +++ /dev/null @@ -1,30 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include - -namespace KINETO_NAMESPACE { - -class RoctracerActivityBuffer { - public: - // data must be allocated using malloc. - // Ownership is transferred to this object. - RoctracerActivityBuffer(uint8_t* data, size_t validSize) - : data(data), validSize(validSize) {} - - ~RoctracerActivityBuffer() { - free(data); - } - - // Allocated by malloc - uint8_t* data{nullptr}; - - // Number of bytes used - size_t validSize; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/SampleListener.h b/plugins/tensorboard-plugins/libkineto/src/SampleListener.h deleted file mode 100644 index bff86ad12..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/SampleListener.h +++ /dev/null @@ -1,146 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include - -namespace KINETO_NAMESPACE { - -class Config; - -class SampleValue { - public: - template - explicit SampleValue(T v) { - init(v); - } - - SampleValue(const SampleValue&) = default; - SampleValue& operator=(const SampleValue&) = delete; - SampleValue(SampleValue&&) = default; - SampleValue& operator=(SampleValue&&) = default; - - bool isInt() const { - return type_ == INT64; - } - - int64_t getInt() const { - assert(isInt()); - return int_; - } - - bool isDouble() const { - return type_ == DOUBLE; - } - - double getDouble() const { - assert(isDouble()); - return dbl_; - } - - inline void operator*=(double x) { - assert(isDouble() || isInt()); - if (isDouble()) { - dbl_ *= x; - } else { - int_ = std::round(int_ * x); - } - } - - inline bool operator<(const SampleValue& o) const { - if (type_ != o.type_) { - return type_ < o.type_; - } else if (type_ == INT64) { - return int_ < o.int_; - } else if (type_ == DOUBLE) { - return dbl_ < o.dbl_; - } - assert(false); - return true; - } - - void print(std::ostream& s) const { - if (type_ == INT64) { - s << int_; - } else if (type_ == DOUBLE) { - s << dbl_; - } else { - assert(false); - } - } - - private: - enum Type { INT64, DOUBLE }; - - template - void init(T v); - - Type type_{INT64}; - union { - int64_t int_{0}; - double dbl_; - }; -}; - -template <> -inline void SampleValue::init(uint64_t v) { - int_ = v, type_ = INT64; -} -template <> -inline void SampleValue::init(int64_t v) { - int_ = v, type_ = INT64; -} -template <> -inline void SampleValue::init(int v) { - int_ = v, type_ = INT64; -} -template <> -inline void SampleValue::init(double v) { - dbl_ = v, type_ = DOUBLE; -} - -inline std::ostream& operator<<(std::ostream& out, const SampleValue& s) { - s.print(out); - return out; -} - -using PercentileList = std::vector>; - -struct Stat { - const std::string& name; - const PercentileList percentileValues; - SampleValue total; -}; - -struct Sample { - Sample(int stats_count) { - stats.reserve(stats_count); - } - - // Offset in milliseconds from first sample in report - int deltaMsec; - std::vector stats; -}; - -// Inherit from this to be notified of samples -class SampleListener { - public: - SampleListener(const SampleListener&) = delete; - SampleListener& operator=(const SampleListener&) = delete; - - virtual ~SampleListener(){}; - - // Report bucketed & aggregated values for event - virtual void handleSample(int device, const Sample& sample, bool from_new_version) = 0; - - virtual void update(const Config& config) = 0; - - protected: - SampleListener() = default; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/ScopeExit.h b/plugins/tensorboard-plugins/libkineto/src/ScopeExit.h deleted file mode 100644 index b9a6bc83e..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ScopeExit.h +++ /dev/null @@ -1,29 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -// Implement a simple scope handler allowing a function to release -// resources when an error or exception occurs - -template -class ScopeExit { - public: - explicit ScopeExit(T t) : t(t) {} - ~ScopeExit() { - t(); - } - T t; -}; - -template -ScopeExit makeScopeExit(T t) { - return ScopeExit(t); -}; - -// Add a level of indirection so __LINE__ is expanded -#define __kINETO_CONCAT(name, line) name##line -#define ANON_VAR(name, line) __kINETO_CONCAT(name, line) - -#define SCOPE_EXIT(func) \ - const auto ANON_VAR(SCOPE_BLOCK, __LINE__) = \ - makeScopeExit([=]() { func; }) diff --git a/plugins/tensorboard-plugins/libkineto/src/ThreadUtil.cpp b/plugins/tensorboard-plugins/libkineto/src/ThreadUtil.cpp deleted file mode 100644 index 0f67d54d5..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/ThreadUtil.cpp +++ /dev/null @@ -1,203 +0,0 @@ -#include "ThreadUtil.h" - -#ifndef _MSC_VER -#include -#include -#include -#include -#else // _MSC_VER -#include -#include -#define WIN32_LEAN_AND_MEAN -#define NOGDI -#include -#include -#undef ERROR -#endif // _MSC_VER - -#ifdef __ANDROID__ -#include -#endif - -#include -#include -#include - -namespace libkineto { - -namespace { -thread_local int32_t _pid = 0; -thread_local int32_t _tid = 0; -thread_local int32_t _sysTid = 0; -} - -int32_t processId() { - if (!_pid) { -#ifndef _MSC_VER - _pid = (int32_t)getpid(); -#else - _pid = (int32_t)GetCurrentProcessId(); -#endif - } - return _pid; -} - -int32_t systemThreadId() { - if (!_sysTid) { -#ifdef __APPLE__ - _sysTid = (int32_t)syscall(SYS_thread_selfid); -#elif defined _MSC_VER - _sysTid = (int32_t)GetCurrentThreadId(); -#else - _sysTid = (int32_t)syscall(SYS_gettid); -#endif - } - return _sysTid; -} - -int32_t threadId() { - if (!_tid) { -#ifdef __APPLE__ - uint64_t tid; - pthread_threadid_np(nullptr, &tid); - _tid = tid; -#elif defined _MSC_VER - _tid = (int32_t)GetCurrentThreadId(); -#else - pthread_t pth = pthread_self(); - int32_t* ptr = reinterpret_cast(&pth); - _tid = *ptr; -#endif - } - return _tid; -} - -namespace { -static constexpr size_t kMaxThreadNameLength = 16; - -static constexpr const char* basename(const char* s, int off = 0) { - return !s[off] - ? s - : s[off] == '/' ? basename(&s[off + 1]) : basename(s, off + 1); -} -#if defined(_MSC_VER) -void *getKernel32Func(const char* procName) { - return GetProcAddress(GetModuleHandleA("KERNEL32.DLL"), procName); -} -#endif -} - -bool setThreadName(const std::string& name) { -#ifdef __APPLE__ - return 0 == pthread_setname_np(name.c_str()); -#elif defined _MSC_VER - // Per https://docs.microsoft.com/en-us/windows/win32/api/processthreadsapi/nf-processthreadsapi-setthreaddescription - // Use runtime linking to set thread description - static auto _SetThreadDescription = reinterpret_cast(getKernel32Func("SetThreadDescription")); - if (!_SetThreadDescription) { - return false; - } - std::wstring_convert> conv; - std::wstring wname = conv.from_bytes(name); - HRESULT hr = _SetThreadDescription(GetCurrentThread(), wname.c_str()); - return SUCCEEDED(hr); -#else - return 0 == pthread_setname_np(pthread_self(), name.c_str()); -#endif -} - -std::string getThreadName() { -#ifndef _MSC_VER - char buf[kMaxThreadNameLength] = ""; - if ( -#ifndef __ANDROID__ - pthread_getname_np(pthread_self(), buf, kMaxThreadNameLength) != 0 -#else - prctl(PR_GET_NAME, buf, kMaxThreadNameLength) != 0 -#endif - ) { - return "Unknown"; - } - return buf; -#else // _MSC_VER - static auto _GetThreadDescription = reinterpret_cast(getKernel32Func("GetThreadDescription")); - if (!_GetThreadDescription) { - return "Unknown"; - } - PWSTR data; - HRESULT hr = _GetThreadDescription(GetCurrentThread(), &data); - if (!SUCCEEDED(hr)) { - return ""; - } - std::wstring_convert> conv; - std::string name = conv.to_bytes(data); - LocalFree(data); - return name; -#endif -} - -// Linux: -// Extract process name from /proc/pid/cmdline. This does not have -// the 16 character limit that /proc/pid/status and /prod/pid/comm has. -std::string processName(int32_t pid) { -#ifdef __linux__ - FILE* cmdfile = fopen(fmt::format("/proc/{}/cmdline", pid).c_str(), "r"); - if (cmdfile != nullptr) { - char* command = nullptr; - int scanned = fscanf(cmdfile, "%ms", &command); - fclose(cmdfile); - if (scanned > 0 && command) { - std::string ret(basename(command)); - free(command); - return ret; - } - } - std::cerr << "Failed to read process name for pid " << pid << std::endl; -#endif - return ""; -} - -// Max number of parent pids to collect, just for extra safeguarding. -constexpr int kMaxParentPids = 10; - -// Return a pair of -static std::pair parentPidAndCommand(int32_t pid) { -#ifdef __linux__ - FILE* statfile = fopen(fmt::format("/proc/{}/stat", pid).c_str(), "r"); - if (statfile == nullptr) { - return std::make_pair(0, ""); - } - int32_t parent_pid; - char* command = nullptr; - int scanned = fscanf(statfile, "%*d (%m[^)]) %*c %d", &command, &parent_pid); - fclose(statfile); - std::pair ret; - if (scanned == 2) { - ret = std::make_pair(parent_pid, std::string(command)); - } else { - std::cerr << "Failed to parse /proc/" << pid << "/stat" << std::endl; - ret = std::make_pair(0, ""); - } - - // The 'm' character in the format tells fscanf to allocate memory - // for the parsed string, which we need to free here. - free(command); - return ret; -#else - return std::make_pair(0, ""); -#endif -} - -std::vector> pidCommandPairsOfAncestors() { - std::vector> pairs; - pairs.reserve(kMaxParentPids + 1); - int32_t curr_pid = processId(); - for (int i = 0; i <= kMaxParentPids && curr_pid > 1; i++) { - std::pair ppid_and_comm = parentPidAndCommand(curr_pid); - pairs.push_back(std::make_pair(curr_pid, ppid_and_comm.second)); - curr_pid = ppid_and_comm.first; - } - return pairs; -} - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/WeakSymbols.cpp b/plugins/tensorboard-plugins/libkineto/src/WeakSymbols.cpp deleted file mode 100644 index 540a5ac8f..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/WeakSymbols.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include - -#ifndef _MSC_VER -extern "C" { -// This function is needed to avoid superfluous dependency on GNU OpenMP library when cuPTI is linked statically -// For more details see https://github.com/pytorch/pytorch/issues/51026 -__attribute__((weak)) int acc_get_device_type() { - throw std::runtime_error("Dummy implementation of acc_get_device_type is not supposed to be called!"); -} - -} // extern "C" -#endif diff --git a/plugins/tensorboard-plugins/libkineto/src/cupti_call.h b/plugins/tensorboard-plugins/libkineto/src/cupti_call.h deleted file mode 100644 index fd6ebae76..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/cupti_call.h +++ /dev/null @@ -1,33 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -#ifdef HAS_CUPTI - -#include - -#define CUPTI_CALL(call) \ - [&]() -> CUptiResult { \ - CUptiResult _status_ = call; \ - if (_status_ != CUPTI_SUCCESS) { \ - const char* _errstr_ = nullptr; \ - cuptiGetResultString(_status_, &_errstr_); \ - LOG(WARNING) << fmt::format( \ - "function {} failed with error {} ({})", \ - #call, \ - _errstr_, \ - (int)_status_); \ - } \ - return _status_; \ - }() - -#define CUPTI_CALL_NOWARN(call) call - -#else - -#define CUPTI_CALL(call) call -#define CUPTI_CALL_NOWARN(call) call - -#endif // HAS_CUPTI diff --git a/plugins/tensorboard-plugins/libkineto/src/cupti_strings.cpp b/plugins/tensorboard-plugins/libkineto/src/cupti_strings.cpp deleted file mode 100644 index 4535273a2..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/cupti_strings.cpp +++ /dev/null @@ -1,502 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "cupti_strings.h" - -namespace libkineto { - -const char* memcpyKindString( - CUpti_ActivityMemcpyKind kind) { - switch (kind) { - case CUPTI_ACTIVITY_MEMCPY_KIND_HTOD: - return "HtoD"; - case CUPTI_ACTIVITY_MEMCPY_KIND_DTOH: - return "DtoH"; - case CUPTI_ACTIVITY_MEMCPY_KIND_HTOA: - return "HtoA"; - case CUPTI_ACTIVITY_MEMCPY_KIND_ATOH: - return "AtoH"; - case CUPTI_ACTIVITY_MEMCPY_KIND_ATOA: - return "AtoA"; - case CUPTI_ACTIVITY_MEMCPY_KIND_ATOD: - return "AtoD"; - case CUPTI_ACTIVITY_MEMCPY_KIND_DTOA: - return "DtoA"; - case CUPTI_ACTIVITY_MEMCPY_KIND_DTOD: - return "DtoD"; - case CUPTI_ACTIVITY_MEMCPY_KIND_HTOH: - return "HtoH"; - case CUPTI_ACTIVITY_MEMCPY_KIND_PTOP: - return "PtoP"; - default: - break; - } - return ""; -} - -const char* memoryKindString( - CUpti_ActivityMemoryKind kind) { - switch (kind) { - case CUPTI_ACTIVITY_MEMORY_KIND_UNKNOWN: - return "Unknown"; - case CUPTI_ACTIVITY_MEMORY_KIND_PAGEABLE: - return "Pageable"; - case CUPTI_ACTIVITY_MEMORY_KIND_PINNED: - return "Pinned"; - case CUPTI_ACTIVITY_MEMORY_KIND_DEVICE: - return "Device"; - case CUPTI_ACTIVITY_MEMORY_KIND_ARRAY: - return "Array"; - case CUPTI_ACTIVITY_MEMORY_KIND_MANAGED: - return "Managed"; - case CUPTI_ACTIVITY_MEMORY_KIND_DEVICE_STATIC: - return "Device Static"; - case CUPTI_ACTIVITY_MEMORY_KIND_MANAGED_STATIC: - return "Managed Static"; - case CUPTI_ACTIVITY_MEMORY_KIND_FORCE_INT: - return "Force Int"; - default: - return "Unrecognized"; - } -} - -const char* overheadKindString( - CUpti_ActivityOverheadKind kind) { - switch (kind) { - case CUPTI_ACTIVITY_OVERHEAD_UNKNOWN: - return "Unknown"; - case CUPTI_ACTIVITY_OVERHEAD_DRIVER_COMPILER: - return "Driver Compiler"; - case CUPTI_ACTIVITY_OVERHEAD_CUPTI_BUFFER_FLUSH: - return "Buffer Flush"; - case CUPTI_ACTIVITY_OVERHEAD_CUPTI_INSTRUMENTATION: - return "Instrumentation"; - case CUPTI_ACTIVITY_OVERHEAD_CUPTI_RESOURCE: - return "Resource"; - case CUPTI_ACTIVITY_OVERHEAD_FORCE_INT: - return "Force Int"; - default: - return "Unrecognized"; - } -} - - - -static const char* runtimeCbidNames[] = { - "INVALID", - "cudaDriverGetVersion", - "cudaRuntimeGetVersion", - "cudaGetDeviceCount", - "cudaGetDeviceProperties", - "cudaChooseDevice", - "cudaGetChannelDesc", - "cudaCreateChannelDesc", - "cudaConfigureCall", - "cudaSetupArgument", - "cudaGetLastError", - "cudaPeekAtLastError", - "cudaGetErrorString", - "cudaLaunch", - "cudaFuncSetCacheConfig", - "cudaFuncGetAttributes", - "cudaSetDevice", - "cudaGetDevice", - "cudaSetValidDevices", - "cudaSetDeviceFlags", - "cudaMalloc", - "cudaMallocPitch", - "cudaFree", - "cudaMallocArray", - "cudaFreeArray", - "cudaMallocHost", - "cudaFreeHost", - "cudaHostAlloc", - "cudaHostGetDevicePointer", - "cudaHostGetFlags", - "cudaMemGetInfo", - "cudaMemcpy", - "cudaMemcpy2D", - "cudaMemcpyToArray", - "cudaMemcpy2DToArray", - "cudaMemcpyFromArray", - "cudaMemcpy2DFromArray", - "cudaMemcpyArrayToArray", - "cudaMemcpy2DArrayToArray", - "cudaMemcpyToSymbol", - "cudaMemcpyFromSymbol", - "cudaMemcpyAsync", - "cudaMemcpyToArrayAsync", - "cudaMemcpyFromArrayAsync", - "cudaMemcpy2DAsync", - "cudaMemcpy2DToArrayAsync", - "cudaMemcpy2DFromArrayAsync", - "cudaMemcpyToSymbolAsync", - "cudaMemcpyFromSymbolAsync", - "cudaMemset", - "cudaMemset2D", - "cudaMemsetAsync", - "cudaMemset2DAsync", - "cudaGetSymbolAddress", - "cudaGetSymbolSize", - "cudaBindTexture", - "cudaBindTexture2D", - "cudaBindTextureToArray", - "cudaUnbindTexture", - "cudaGetTextureAlignmentOffset", - "cudaGetTextureReference", - "cudaBindSurfaceToArray", - "cudaGetSurfaceReference", - "cudaGLSetGLDevice", - "cudaGLRegisterBufferObject", - "cudaGLMapBufferObject", - "cudaGLUnmapBufferObject", - "cudaGLUnregisterBufferObject", - "cudaGLSetBufferObjectMapFlags", - "cudaGLMapBufferObjectAsync", - "cudaGLUnmapBufferObjectAsync", - "cudaWGLGetDevice", - "cudaGraphicsGLRegisterImage", - "cudaGraphicsGLRegisterBuffer", - "cudaGraphicsUnregisterResource", - "cudaGraphicsResourceSetMapFlags", - "cudaGraphicsMapResources", - "cudaGraphicsUnmapResources", - "cudaGraphicsResourceGetMappedPointer", - "cudaGraphicsSubResourceGetMappedArray", - "cudaVDPAUGetDevice", - "cudaVDPAUSetVDPAUDevice", - "cudaGraphicsVDPAURegisterVideoSurface", - "cudaGraphicsVDPAURegisterOutputSurface", - "cudaD3D11GetDevice", - "cudaD3D11GetDevices", - "cudaD3D11SetDirect3DDevice", - "cudaGraphicsD3D11RegisterResource", - "cudaD3D10GetDevice", - "cudaD3D10GetDevices", - "cudaD3D10SetDirect3DDevice", - "cudaGraphicsD3D10RegisterResource", - "cudaD3D10RegisterResource", - "cudaD3D10UnregisterResource", - "cudaD3D10MapResources", - "cudaD3D10UnmapResources", - "cudaD3D10ResourceSetMapFlags", - "cudaD3D10ResourceGetSurfaceDimensions", - "cudaD3D10ResourceGetMappedArray", - "cudaD3D10ResourceGetMappedPointer", - "cudaD3D10ResourceGetMappedSize", - "cudaD3D10ResourceGetMappedPitch", - "cudaD3D9GetDevice", - "cudaD3D9GetDevices", - "cudaD3D9SetDirect3DDevice", - "cudaD3D9GetDirect3DDevice", - "cudaGraphicsD3D9RegisterResource", - "cudaD3D9RegisterResource", - "cudaD3D9UnregisterResource", - "cudaD3D9MapResources", - "cudaD3D9UnmapResources", - "cudaD3D9ResourceSetMapFlags", - "cudaD3D9ResourceGetSurfaceDimensions", - "cudaD3D9ResourceGetMappedArray", - "cudaD3D9ResourceGetMappedPointer", - "cudaD3D9ResourceGetMappedSize", - "cudaD3D9ResourceGetMappedPitch", - "cudaD3D9Begin", - "cudaD3D9End", - "cudaD3D9RegisterVertexBuffer", - "cudaD3D9UnregisterVertexBuffer", - "cudaD3D9MapVertexBuffer", - "cudaD3D9UnmapVertexBuffer", - "cudaThreadExit", - "cudaSetDoubleForDevice", - "cudaSetDoubleForHost", - "cudaThreadSynchronize", - "cudaThreadGetLimit", - "cudaThreadSetLimit", - "cudaStreamCreate", - "cudaStreamDestroy", - "cudaStreamSynchronize", - "cudaStreamQuery", - "cudaEventCreate", - "cudaEventCreateWithFlags", - "cudaEventRecord", - "cudaEventDestroy", - "cudaEventSynchronize", - "cudaEventQuery", - "cudaEventElapsedTime", - "cudaMalloc3D", - "cudaMalloc3DArray", - "cudaMemset3D", - "cudaMemset3DAsync", - "cudaMemcpy3D", - "cudaMemcpy3DAsync", - "cudaThreadSetCacheConfig", - "cudaStreamWaitEvent", - "cudaD3D11GetDirect3DDevice", - "cudaD3D10GetDirect3DDevice", - "cudaThreadGetCacheConfig", - "cudaPointerGetAttributes", - "cudaHostRegister", - "cudaHostUnregister", - "cudaDeviceCanAccessPeer", - "cudaDeviceEnablePeerAccess", - "cudaDeviceDisablePeerAccess", - "cudaPeerRegister", - "cudaPeerUnregister", - "cudaPeerGetDevicePointer", - "cudaMemcpyPeer", - "cudaMemcpyPeerAsync", - "cudaMemcpy3DPeer", - "cudaMemcpy3DPeerAsync", - "cudaDeviceReset", - "cudaDeviceSynchronize", - "cudaDeviceGetLimit", - "cudaDeviceSetLimit", - "cudaDeviceGetCacheConfig", - "cudaDeviceSetCacheConfig", - "cudaProfilerInitialize", - "cudaProfilerStart", - "cudaProfilerStop", - "cudaDeviceGetByPCIBusId", - "cudaDeviceGetPCIBusId", - "cudaGLGetDevices", - "cudaIpcGetEventHandle", - "cudaIpcOpenEventHandle", - "cudaIpcGetMemHandle", - "cudaIpcOpenMemHandle", - "cudaIpcCloseMemHandle", - "cudaArrayGetInfo", - "cudaFuncSetSharedMemConfig", - "cudaDeviceGetSharedMemConfig", - "cudaDeviceSetSharedMemConfig", - "cudaCreateTextureObject", - "cudaDestroyTextureObject", - "cudaGetTextureObjectResourceDesc", - "cudaGetTextureObjectTextureDesc", - "cudaCreateSurfaceObject", - "cudaDestroySurfaceObject", - "cudaGetSurfaceObjectResourceDesc", - "cudaMallocMipmappedArray", - "cudaGetMipmappedArrayLevel", - "cudaFreeMipmappedArray", - "cudaBindTextureToMipmappedArray", - "cudaGraphicsResourceGetMappedMipmappedArray", - "cudaStreamAddCallback", - "cudaStreamCreateWithFlags", - "cudaGetTextureObjectResourceViewDesc", - "cudaDeviceGetAttribute", - "cudaStreamDestroy", - "cudaStreamCreateWithPriority", - "cudaStreamGetPriority", - "cudaStreamGetFlags", - "cudaDeviceGetStreamPriorityRange", - "cudaMallocManaged", - "cudaOccupancyMaxActiveBlocksPerMultiprocessor", - "cudaStreamAttachMemAsync", - "cudaGetErrorName", - "cudaOccupancyMaxActiveBlocksPerMultiprocessor", - "cudaLaunchKernel", - "cudaGetDeviceFlags", - "cudaLaunch_ptsz", - "cudaLaunchKernel_ptsz", - "cudaMemcpy_ptds", - "cudaMemcpy2D_ptds", - "cudaMemcpyToArray_ptds", - "cudaMemcpy2DToArray_ptds", - "cudaMemcpyFromArray_ptds", - "cudaMemcpy2DFromArray_ptds", - "cudaMemcpyArrayToArray_ptds", - "cudaMemcpy2DArrayToArray_ptds", - "cudaMemcpyToSymbol_ptds", - "cudaMemcpyFromSymbol_ptds", - "cudaMemcpyAsync_ptsz", - "cudaMemcpyToArrayAsync_ptsz", - "cudaMemcpyFromArrayAsync_ptsz", - "cudaMemcpy2DAsync_ptsz", - "cudaMemcpy2DToArrayAsync_ptsz", - "cudaMemcpy2DFromArrayAsync_ptsz", - "cudaMemcpyToSymbolAsync_ptsz", - "cudaMemcpyFromSymbolAsync_ptsz", - "cudaMemset_ptds", - "cudaMemset2D_ptds", - "cudaMemsetAsync_ptsz", - "cudaMemset2DAsync_ptsz", - "cudaStreamGetPriority_ptsz", - "cudaStreamGetFlags_ptsz", - "cudaStreamSynchronize_ptsz", - "cudaStreamQuery_ptsz", - "cudaStreamAttachMemAsync_ptsz", - "cudaEventRecord_ptsz", - "cudaMemset3D_ptds", - "cudaMemset3DAsync_ptsz", - "cudaMemcpy3D_ptds", - "cudaMemcpy3DAsync_ptsz", - "cudaStreamWaitEvent_ptsz", - "cudaStreamAddCallback_ptsz", - "cudaMemcpy3DPeer_ptds", - "cudaMemcpy3DPeerAsync_ptsz", - "cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags", - "cudaMemPrefetchAsync", - "cudaMemPrefetchAsync_ptsz", - "cudaMemAdvise", - "cudaDeviceGetP2PAttribute", - "cudaGraphicsEGLRegisterImage", - "cudaEGLStreamConsumerConnect", - "cudaEGLStreamConsumerDisconnect", - "cudaEGLStreamConsumerAcquireFrame", - "cudaEGLStreamConsumerReleaseFrame", - "cudaEGLStreamProducerConnect", - "cudaEGLStreamProducerDisconnect", - "cudaEGLStreamProducerPresentFrame", - "cudaEGLStreamProducerReturnFrame", - "cudaGraphicsResourceGetMappedEglFrame", - "cudaMemRangeGetAttribute", - "cudaMemRangeGetAttributes", - "cudaEGLStreamConsumerConnectWithFlags", - "cudaLaunchCooperativeKernel", - "cudaLaunchCooperativeKernel_ptsz", - "cudaEventCreateFromEGLSync", - "cudaLaunchCooperativeKernelMultiDevice", - "cudaFuncSetAttribute", - "cudaImportExternalMemory", - "cudaExternalMemoryGetMappedBuffer", - "cudaExternalMemoryGetMappedMipmappedArray", - "cudaDestroyExternalMemory", - "cudaImportExternalSemaphore", - "cudaSignalExternalSemaphoresAsync", - "cudaSignalExternalSemaphoresAsync_ptsz", - "cudaWaitExternalSemaphoresAsync", - "cudaWaitExternalSemaphoresAsync_ptsz", - "cudaDestroyExternalSemaphore", - "cudaLaunchHostFunc", - "cudaLaunchHostFunc_ptsz", - "cudaGraphCreate", - "cudaGraphKernelNodeGetParams", - "cudaGraphKernelNodeSetParams", - "cudaGraphAddKernelNode", - "cudaGraphAddMemcpyNode", - "cudaGraphMemcpyNodeGetParams", - "cudaGraphMemcpyNodeSetParams", - "cudaGraphAddMemsetNode", - "cudaGraphMemsetNodeGetParams", - "cudaGraphMemsetNodeSetParams", - "cudaGraphAddHostNode", - "cudaGraphHostNodeGetParams", - "cudaGraphAddChildGraphNode", - "cudaGraphChildGraphNodeGetGraph", - "cudaGraphAddEmptyNode", - "cudaGraphClone", - "cudaGraphNodeFindInClone", - "cudaGraphNodeGetType", - "cudaGraphGetRootNodes", - "cudaGraphNodeGetDependencies", - "cudaGraphNodeGetDependentNodes", - "cudaGraphAddDependencies", - "cudaGraphRemoveDependencies", - "cudaGraphDestroyNode", - "cudaGraphInstantiate", - "cudaGraphLaunch", - "cudaGraphLaunch_ptsz", - "cudaGraphExecDestroy", - "cudaGraphDestroy", - "cudaStreamBeginCapture", - "cudaStreamBeginCapture_ptsz", - "cudaStreamIsCapturing", - "cudaStreamIsCapturing_ptsz", - "cudaStreamEndCapture", - "cudaStreamEndCapture_ptsz", - "cudaGraphHostNodeSetParams", - "cudaGraphGetNodes", - "cudaGraphGetEdges", - "cudaStreamGetCaptureInfo", - "cudaStreamGetCaptureInfo_ptsz", - "cudaGraphExecKernelNodeSetParams", - "cudaThreadExchangeStreamCaptureMode", - "cudaDeviceGetNvSciSyncAttributes", - "cudaOccupancyAvailableDynamicSMemPerBlock", - "cudaStreamSetFlags", - "cudaStreamSetFlags_ptsz", - "cudaGraphExecMemcpyNodeSetParams", - "cudaGraphExecMemsetNodeSetParams", - "cudaGraphExecHostNodeSetParams", - "cudaGraphExecUpdate", - "cudaGetFuncBySymbol", - "cudaCtxResetPersistingL2Cache", - "cudaGraphKernelNodeCopyAttributes", - "cudaGraphKernelNodeGetAttribute", - "cudaGraphKernelNodeSetAttribute", - "cudaStreamCopyAttributes", - "cudaStreamCopyAttributes_ptsz", - "cudaStreamGetAttribute", - "cudaStreamGetAttribute_ptsz", - "cudaStreamSetAttribute", - "cudaStreamSetAttribute_ptsz", - "cudaDeviceGetTexture1DLinearMaxWidth", - "cudaGraphUpload", - "cudaGraphUpload_ptsz", - "cudaGraphAddMemcpyNodeToSymbol", - "cudaGraphAddMemcpyNodeFromSymbol", - "cudaGraphAddMemcpyNode1D", - "cudaGraphMemcpyNodeSetParamsToSymbol", - "cudaGraphMemcpyNodeSetParamsFromSymbol", - "cudaGraphMemcpyNodeSetParams1D", - "cudaGraphExecMemcpyNodeSetParamsToSymbol", - "cudaGraphExecMemcpyNodeSetParamsFromSymbol", - "cudaGraphExecMemcpyNodeSetParams1D", - "cudaArrayGetSparseProperties", - "cudaMipmappedArrayGetSparseProperties", - "cudaGraphExecChildGraphNodeSetParams", - "cudaGraphAddEventRecordNode", - "cudaGraphEventRecordNodeGetEvent", - "cudaGraphEventRecordNodeSetEvent", - "cudaGraphAddEventWaitNode", - "cudaGraphEventWaitNodeGetEvent", - "cudaGraphEventWaitNodeSetEvent", - "cudaGraphExecEventRecordNodeSetEvent", - "cudaGraphExecEventWaitNodeSetEvent", - "cudaEventRecordWithFlags", - "cudaEventRecordWithFlags_ptsz", - "cudaDeviceGetDefaultMemPool", - "cudaMallocAsync", - "cudaMallocAsync_ptsz", - "cudaFreeAsync", - "cudaFreeAsync_ptsz", - "cudaMemPoolTrimTo", - "cudaMemPoolSetAttribute", - "cudaMemPoolGetAttribute", - "cudaMemPoolSetAccess", - "cudaArrayGetPlane", - "cudaMemPoolGetAccess", - "cudaMemPoolCreate", - "cudaMemPoolDestroy", - "cudaDeviceSetMemPool", - "cudaDeviceGetMemPool", - "cudaMemPoolExportToShareableHandle", - "cudaMemPoolImportFromShareableHandle", - "cudaMemPoolExportPointer", - "cudaMemPoolImportPointer", - "cudaMallocFromPoolAsync", - "cudaMallocFromPoolAsync_ptsz", - "cudaSignalExternalSemaphoresAsync", - "cudaSignalExternalSemaphoresAsync", - "cudaWaitExternalSemaphoresAsync", - "cudaWaitExternalSemaphoresAsync", - "cudaGraphAddExternalSemaphoresSignalNode", - "cudaGraphExternalSemaphoresSignalNodeGetParams", - "cudaGraphExternalSemaphoresSignalNodeSetParams", - "cudaGraphAddExternalSemaphoresWaitNode", - "cudaGraphExternalSemaphoresWaitNodeGetParams", - "cudaGraphExternalSemaphoresWaitNodeSetParams", - "cudaGraphExecExternalSemaphoresSignalNodeSetParams", - "cudaGraphExecExternalSemaphoresWaitNodeSetParams", - "SIZE" -}; - -const char* runtimeCbidName(CUpti_CallbackId cbid) { - constexpr int names_size = - sizeof(runtimeCbidNames) / sizeof(runtimeCbidNames[0]); - if (cbid < 0 || cbid >= names_size) { - return runtimeCbidNames[CUPTI_RUNTIME_TRACE_CBID_INVALID]; - } - return runtimeCbidNames[cbid]; -} - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/cupti_strings.h b/plugins/tensorboard-plugins/libkineto/src/cupti_strings.h deleted file mode 100644 index bbfebb983..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/cupti_strings.h +++ /dev/null @@ -1,14 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include - -namespace libkineto { - -const char* memoryKindString(CUpti_ActivityMemoryKind kind); -const char* memcpyKindString(CUpti_ActivityMemcpyKind kind); -const char* runtimeCbidName(CUpti_CallbackId cbid); -const char* overheadKindString(CUpti_ActivityOverheadKind kind); - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/init.cpp b/plugins/tensorboard-plugins/libkineto/src/init.cpp deleted file mode 100644 index 4e1022485..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/init.cpp +++ /dev/null @@ -1,139 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include - -#include "ActivityProfilerProxy.h" -#include "Config.h" -#ifdef HAS_CUPTI -#include "CuptiCallbackApi.h" -#include "CuptiActivityApi.h" -#include "EventProfilerController.h" -#endif -#include "cupti_call.h" -#include "libkineto.h" - -#include "Logger.h" - -namespace KINETO_NAMESPACE { - -#ifdef HAS_CUPTI -static bool initialized = false; -static std::mutex initMutex; - -static void initProfilers( - CUpti_CallbackDomain /*domain*/, - CUpti_CallbackId /*cbid*/, - const CUpti_CallbackData* cbInfo) { - CUpti_ResourceData* d = (CUpti_ResourceData*)cbInfo; - CUcontext ctx = d->context; - - VLOG(0) << "CUDA Context created"; - std::lock_guard lock(initMutex); - - if (!initialized) { - libkineto::api().initProfilerIfRegistered(); - initialized = true; - VLOG(0) << "libkineto profilers activated"; - } - if (getenv("KINETO_DISABLE_EVENT_PROFILER") != nullptr) { - VLOG(0) << "Event profiler disabled via env var"; - } else { - ConfigLoader& config_loader = libkineto::api().configLoader(); - config_loader.initBaseConfig(); - EventProfilerController::start(ctx, config_loader); - } -} - -// Some models suffer from excessive instrumentation code gen -// on dynamic attach which can hang for more than 5+ seconds. -// If the workload was meant to be traced, preload the CUPTI -// to take the performance hit early on. -// https://docs.nvidia.com/cupti/r_main.html#r_overhead -static bool shouldPreloadCuptiInstrumentation() { - return getenv("PRELOAD_CUPTI_INSTRUMENTATION"); -} - -static void stopProfiler( - CUpti_CallbackDomain /*domain*/, - CUpti_CallbackId /*cbid*/, - const CUpti_CallbackData* cbInfo) { - CUpti_ResourceData* d = (CUpti_ResourceData*)cbInfo; - CUcontext ctx = d->context; - - LOG(INFO) << "CUDA Context destroyed"; - std::lock_guard lock(initMutex); - EventProfilerController::stop(ctx); -} -#endif // HAS_CUPTI - -} // namespace KINETO_NAMESPACE - -// Callback interface with CUPTI and library constructors -using namespace KINETO_NAMESPACE; -extern "C" { - -// Return true if no CUPTI errors occurred during init -bool libkineto_init(bool cpuOnly, bool logOnError) { - bool success = true; -#ifdef HAS_CUPTI - if (!cpuOnly) { - // libcupti will be lazily loaded on this call. - // If it is not available (e.g. CUDA is not installed), - // then this call will return an error and we just abort init. - auto& cbapi = CuptiCallbackApi::singleton(); - bool status = false; - - if (cbapi.initSuccess()){ - const CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RESOURCE; - status = cbapi.registerCallback( - domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED, initProfilers); - status = status && cbapi.registerCallback( - domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED, stopProfiler); - - if (status) { - status = cbapi.enableCallback( - domain, CuptiCallbackApi::RESOURCE_CONTEXT_CREATED); - status = status && cbapi.enableCallback( - domain, CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED); - } - } - - if (!cbapi.initSuccess() || !status) { - success = false; - cpuOnly = true; - if (logOnError) { - CUPTI_CALL(cbapi.getCuptiStatus()); - LOG(WARNING) << "CUPTI initialization failed - " - << "CUDA profiler activities will be missing"; - LOG(INFO) << "If you see CUPTI_ERROR_INSUFFICIENT_PRIVILEGES, refer to " - << "https://developer.nvidia.com/nvidia-development-tools-solutions-err-nvgpuctrperm-cupti"; - } - } - } - - if (shouldPreloadCuptiInstrumentation()) { - CuptiActivityApi::forceLoadCupti(); - } -#endif // HAS_CUPTI - - ConfigLoader& config_loader = libkineto::api().configLoader(); - libkineto::api().registerProfiler( - std::make_unique(cpuOnly, config_loader)); - - return success; -} - -// The cuda driver calls this function if the CUDA_INJECTION64_PATH environment -// variable is set -int InitializeInjection(void) { - LOG(INFO) << "Injection mode: Initializing libkineto"; - libkineto_init(false /*cpuOnly*/, true /*logOnError*/); - return 1; -} - -void suppressLibkinetoLogMessages() { - SET_LOG_SEVERITY_LEVEL(ERROR); -} - -} // extern C diff --git a/plugins/tensorboard-plugins/libkineto/src/libkineto_api.cpp b/plugins/tensorboard-plugins/libkineto/src/libkineto_api.cpp deleted file mode 100644 index 9a622e4f5..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/libkineto_api.cpp +++ /dev/null @@ -1,41 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "libkineto.h" - -#include "ConfigLoader.h" -#include "ThreadUtil.h" - -namespace libkineto { - -LibkinetoApi& api() { - static LibkinetoApi instance(ConfigLoader::instance()); - return instance; -} - -void LibkinetoApi::initClientIfRegistered() { - if (client_) { - if (clientRegisterThread_ != threadId()) { - fprintf( - stderr, - "ERROR: External init callback must run in same thread as registerClient " - "(%d != %d)\n", - threadId(), - (int)clientRegisterThread_); - } else { - client_->init(); - } - } -} - -void LibkinetoApi::registerClient(ClientInterface* client) { - client_ = client; - if (client && activityProfiler_) { - // Can initialize straight away - client->init(); - } - // Assume here that the external init callback is *not* threadsafe - // and only call it if it's the same thread that called registerClient - clientRegisterThread_ = threadId(); -} - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/src/output_base.h b/plugins/tensorboard-plugins/libkineto/src/output_base.h deleted file mode 100644 index 29d0d5776..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/output_base.h +++ /dev/null @@ -1,104 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include - -#ifdef HAS_CUPTI -#include -#include "CuptiActivity.h" -#endif // HAS_CUPTI -#include "ActivityBuffers.h" -#include "GenericTraceActivity.h" -#include "ThreadUtil.h" -#include "TraceSpan.h" - -namespace KINETO_NAMESPACE { - class Config; - class GpuKernelActivity; - struct RuntimeActivity; -} - -namespace libkineto { - -using namespace KINETO_NAMESPACE; - -class ActivityLogger { - public: - - virtual ~ActivityLogger() = default; - - struct DeviceInfo { - DeviceInfo(int64_t id, const std::string& name, const std::string& label) : - id(id), name(name), label(label) {} - int64_t id; - const std::string name; - const std::string label; - }; - - struct ResourceInfo { - ResourceInfo( - int64_t deviceId, - int64_t id, - int64_t sortIndex, - const std::string& name) : - id(id), sortIndex(sortIndex), deviceId(deviceId), name(name) {} - int64_t id; - int64_t sortIndex; - int64_t deviceId; - const std::string name; - }; - - struct OverheadInfo { - explicit OverheadInfo(const std::string& name) : name(name) {} - const std::string name; - }; - - virtual void handleDeviceInfo( - const DeviceInfo& info, - uint64_t time) = 0; - - virtual void handleResourceInfo(const ResourceInfo& info, int64_t time) = 0; - - virtual void handleOverheadInfo(const OverheadInfo& info, int64_t time) = 0; - - virtual void handleTraceSpan(const TraceSpan& span) = 0; - - virtual void handleActivity( - const libkineto::ITraceActivity& activity) = 0; - virtual void handleGenericActivity( - const libkineto::GenericTraceActivity& activity) = 0; - -#ifdef HAS_CUPTI - virtual void handleGpuActivity( - const GpuActivity& activity) = 0; - virtual void handleGpuActivity( - const GpuActivity& activity) = 0; - virtual void handleGpuActivity( - const GpuActivity& activity) = 0; - virtual void handleGpuActivity( - const GpuActivity& activity) = 0; -#endif // HAS_CUPTI - - virtual void handleTraceStart( - const std::unordered_map& metadata) = 0; - - void handleTraceStart() { - handleTraceStart(std::unordered_map()); - } - - virtual void finalizeTrace( - const KINETO_NAMESPACE::Config& config, - std::unique_ptr buffers, - int64_t endTime, - std::unordered_map>& metadata) = 0; - - protected: - ActivityLogger() = default; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/output_csv.cpp b/plugins/tensorboard-plugins/libkineto/src/output_csv.cpp deleted file mode 100644 index e56c02293..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/output_csv.cpp +++ /dev/null @@ -1,88 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "output_csv.h" - -#include -#include -#include - -#include -#include - -#include "Config.h" -#include "Logger.h" - -namespace KINETO_NAMESPACE { - -static void write_header( - std::ostream& out, - const std::vector& percentiles) { - out << "timestamp,delta_ms,device,event_name"; - for (int p : percentiles) { - out << ",p" << p; - } - out << ",total" << std::endl; -} - -void EventCSVLogger::update(const Config& config) { - eventNames_.clear(); - eventNames_.insert(config.eventNames().begin(), config.eventNames().end()); - eventNames_.insert(config.metricNames().begin(), config.metricNames().end()); - if (config.percentiles() != percentiles_) { - percentiles_ = config.percentiles(); - if (out_) { - write_header(*out_, percentiles_); - } - } -} - -void EventCSVLogger::handleSample(int device, const Sample& sample, bool from_new_version) { - using namespace std::chrono; - if (out_) { - auto now = system_clock::now(); - auto time = system_clock::to_time_t(now); - for (const Stat& s : sample.stats) { - if (eventNames_.find(s.name) == eventNames_.end()) { - continue; - } - *out_ << fmt::format("{:%Y-%m-%d %H:%M:%S}", fmt::localtime(time)) << ","; - *out_ << sample.deltaMsec << ","; - *out_ << device << ","; - *out_ << s.name; - for (const auto& p : s.percentileValues) { - *out_ << "," << p.second; - } - *out_ << "," << s.total << std::endl; - } - } -} - -void EventCSVFileLogger::update(const Config& config) { - if (config.eventLogFile() != filename_) { - if (of_.is_open()) { - of_.close(); - out_ = nullptr; - percentiles_.clear(); - } - filename_ = config.eventLogFile(); - if (!filename_.empty()) { - of_.open(filename_, std::ios::out | std::ios::trunc); - out_ = &of_; - } - } - EventCSVLogger::update(config); -} - -void EventCSVDbgLogger::update(const Config& config) { - if (out_ && config.verboseLogLevel() < 0) { - out_ = nullptr; - } else if (!out_ && config.verboseLogLevel() >= 0) { - out_ = &LIBKINETO_DBG_STREAM; - } - if (config.verboseLogLevel() >= 0) { - percentiles_.clear(); - EventCSVLogger::update(config); - } -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/output_csv.h b/plugins/tensorboard-plugins/libkineto/src/output_csv.h deleted file mode 100644 index bca29f4db..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/output_csv.h +++ /dev/null @@ -1,39 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once -#include "SampleListener.h" - -#include -#include -#include - -namespace KINETO_NAMESPACE { - -class EventCSVLogger : public SampleListener { - public: - void update(const Config& config) override; - void handleSample(int device, const Sample& sample, bool from_new_version) override; - - protected: - EventCSVLogger() : out_(nullptr) {} - - std::ostream* out_; - std::set eventNames_; - std::vector percentiles_; -}; - -class EventCSVFileLogger : public EventCSVLogger { - public: - void update(const Config& config) override; - - private: - std::ofstream of_; - std::string filename_; -}; - -class EventCSVDbgLogger : public EventCSVLogger { - public: - void update(const Config& config) override; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/output_json.cpp b/plugins/tensorboard-plugins/libkineto/src/output_json.cpp deleted file mode 100644 index 0ef22339f..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/output_json.cpp +++ /dev/null @@ -1,583 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "output_json.h" - -#include -#include -#include -#include - -#include "Config.h" -#ifdef HAS_CUPTI -#include "CuptiActivity.h" -#include "CuptiActivity.tpp" -#include "CuptiActivityApi.h" -#include "CudaDeviceProperties.h" -#endif // HAS_CUPTI -#include "Demangle.h" -#include "TraceSpan.h" - -#include "Logger.h" - -using std::endl; -using namespace libkineto; - -namespace KINETO_NAMESPACE { - -static constexpr int kSchemaVersion = 1; -static constexpr char kFlowStart = 's'; -static constexpr char kFlowEnd = 'f'; - -#ifdef __linux__ -static constexpr char kDefaultLogFileFmt[] = - "/tmp/libkineto_activities_{}.json"; -#else -static constexpr char kDefaultLogFileFmt[] = "libkineto_activities_{}.json"; -#endif - -std::string& ChromeTraceLogger::sanitizeStrForJSON(std::string& value) { -// Replace all backslashes with forward slash because Windows paths causing JSONDecodeError. -#ifdef _WIN32 - std::replace(value.begin(), value.end(), '\\', '/'); -#endif - return value; -} - -void ChromeTraceLogger::metadataToJSON( - const std::unordered_map& metadata) { - for (const auto& kv : metadata) { - traceOf_ << fmt::format(R"JSON( - "{}": {},)JSON", kv.first, kv.second); - } -} - -void ChromeTraceLogger::handleTraceStart( - const std::unordered_map& metadata) { - traceOf_ << fmt::format(R"JSON( -{{ - "schemaVersion": {},)JSON", kSchemaVersion); - -#ifdef HAS_CUPTI - traceOf_ << fmt::format(R"JSON( - "deviceProperties": [{} - ],)JSON", devicePropertiesJson()); -#endif - - metadataToJSON(metadata); - traceOf_ << R"JSON( - "traceEvents": [)JSON"; -} - -static std::string defaultFileName() { - return fmt::format(kDefaultLogFileFmt, processId()); -} - -void ChromeTraceLogger::openTraceFile() { - traceOf_.open(fileName_, std::ofstream::out | std::ofstream::trunc); - if (!traceOf_) { - PLOG(ERROR) << "Failed to open '" << fileName_ << "'"; - } else { - LOG(INFO) << "Tracing to " << fileName_; - } -} - -ChromeTraceLogger::ChromeTraceLogger(const std::string& traceFileName) { - fileName_ = traceFileName.empty() ? defaultFileName() : traceFileName; - traceOf_.clear(std::ios_base::badbit); - openTraceFile(); -} - -static int64_t us(int64_t timestamp) { - // It's important that this conversion is the same here and in the CPU trace. - // No rounding! - return timestamp / 1000; -} - -void ChromeTraceLogger::handleDeviceInfo( - const DeviceInfo& info, - uint64_t time) { - if (!traceOf_) { - return; - } - - // M is for metadata - // process_name needs a pid and a name arg - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "name": "process_name", "ph": "M", "ts": {}, "pid": {}, "tid": 0, - "args": {{ - "name": "{}" - }} - }}, - {{ - "name": "process_labels", "ph": "M", "ts": {}, "pid": {}, "tid": 0, - "args": {{ - "labels": "{}" - }} - }}, - {{ - "name": "process_sort_index", "ph": "M", "ts": {}, "pid": {}, "tid": 0, - "args": {{ - "sort_index": {} - }} - }},)JSON", - time, info.id, - info.name, - time, info.id, - info.label, - time, info.id, - info.id < 8 ? info.id + 0x1000000ll : info.id); - // clang-format on -} - -void ChromeTraceLogger::handleResourceInfo( - const ResourceInfo& info, - int64_t time) { - if (!traceOf_) { - return; - } - - // M is for metadata - // thread_name needs a pid and a name arg - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "name": "thread_name", "ph": "M", "ts": {}, "pid": {}, "tid": {}, - "args": {{ - "name": "{}" - }} - }}, - {{ - "name": "thread_sort_index", "ph": "M", "ts": {}, "pid": {}, "tid": {}, - "args": {{ - "sort_index": {} - }} - }},)JSON", - time, info.deviceId, info.id, - info.name, - time, info.deviceId, info.id, - info.sortIndex); - // clang-format on -} - -void ChromeTraceLogger::handleOverheadInfo( - const OverheadInfo& info, - int64_t time) { - if (!traceOf_) { - return; - } - - // TOOD: reserve pid = -1 for overhead but we need to rethink how to scale this for - // other metadata - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "name": "process_name", "ph": "M", "ts": {}, "pid": -1, "tid": 0, - "args": {{ - "name": "{}" - }} - }}, - {{ - "name": "process_sort_index", "ph": "M", "ts": {}, "pid": -1, "tid": 0, - "args": {{ - "sort_index": {} - }} - }},)JSON", - time, - info.name, - time, - 0x100000All); - // clang-format on -} - -void ChromeTraceLogger::handleTraceSpan(const TraceSpan& span) { - if (!traceOf_) { - return; - } - - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "ph": "X", "cat": "Trace", "ts": {}, "dur": {}, - "pid": "Spans", "tid": "{}", - "name": "{}{} ({})", - "args": {{ - "Op count": {} - }} - }}, - {{ - "name": "process_sort_index", "ph": "M", "ts": {}, - "pid": "Spans", "tid": 0, - "args": {{ - "sort_index": {} - }} - }},)JSON", - span.startTime, span.endTime - span.startTime, - span.name, - span.prefix, span.name, span.iteration, - span.opCount, - span.startTime, - // Large sort index to appear at the bottom - 0x20000000ll); - // clang-format on - - addIterationMarker(span); -} - -void ChromeTraceLogger::addIterationMarker(const TraceSpan& span) { - if (!traceOf_) { - return; - } - - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "name": "Iteration Start: {}", "ph": "i", "s": "g", - "pid": "Traces", "tid": "Trace {}", "ts": {} - }},)JSON", - span.name, - span.name, span.startTime); - // clang-format on -} - -static std::string traceActivityJson(const ITraceActivity& activity) { - // clang-format off - int64_t ts = activity.timestamp(); - int64_t duration = activity.duration(); - if (activity.type() == ActivityType::GPU_USER_ANNOTATION) { - // The GPU user annotations start at the same time as the - // first associated GPU activity. Since they appear later - // in the trace file, this causes a visualization issue in Chrome. - // Make it start one us earlier. - ts--; - duration++; // Still need it to end at the orginal point - } - return fmt::format(R"JSON( - "name": "{}", "pid": {}, "tid": {}, - "ts": {}, "dur": {})JSON", - activity.name(), activity.deviceId(), activity.resourceId(), - ts, duration); - // clang-format on -} - -void ChromeTraceLogger::handleGenericInstantEvent( - const libkineto::ITraceActivity& op) { - if (!traceOf_) { - return; - } - - traceOf_ << fmt::format(R"JSON( - {{ - "ph": "i", "s": "t", "name": "{}", - "pid": {}, "tid": {}, - "ts": {}, - "args": {{ - {} - }} - }},)JSON", - op.name(), op.deviceId(), op.resourceId(), - op.timestamp(), op.metadataJson()); -} - -void ChromeTraceLogger::handleActivity( - const libkineto::ITraceActivity& op) { - if (!traceOf_) { - return; - } - - if (op.type() == ActivityType::CPU_INSTANT_EVENT) { - handleGenericInstantEvent(op); - return; - } - - const std::string op_metadata = op.metadataJson(); - std::string separator = ""; - if (op_metadata.find_first_not_of(" \t\n") != std::string::npos) { - separator = ",\n "; - } - std::string span = ""; - if (op.traceSpan()) { - span = fmt::format(R"JSON( - "Trace name": "{}", "Trace iteration": {},)JSON", - op.traceSpan()->name, - op.traceSpan()->iteration); - } - - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "ph": "X", "cat": "{}", {}, - "args": {{{} - "External id": {}{}{} - }} - }},)JSON", - toString(op.type()), traceActivityJson(op), - // args - span, - op.correlationId(), separator, op_metadata); - // clang-format on - if (op.flowId() > 0) { - handleGenericLink(op); - } -} - -void ChromeTraceLogger::handleGenericActivity( - const libkineto::GenericTraceActivity& op) { - handleActivity(op); -} - -void ChromeTraceLogger::handleGenericLink(const ITraceActivity& act) { - static struct { - int type; - char longName[24]; - char shortName[16]; - } flow_names[] = { - {kLinkFwdBwd, "forward_backward", "fwd_bwd"}, - {kLinkAsyncCpuGpu, "async_cpu_to_gpu", "async_gpu"} - }; - for (auto& flow : flow_names) { - if (act.flowType() == flow.type) { - // Link the activities via flow ID in source and destination. - // The source node must return true from flowStart() - // and the destination node false. - if (act.flowStart()) { - handleLink(kFlowStart, act, act.flowId(), flow.longName, flow.shortName); - } else { - handleLink(kFlowEnd, act, act.flowId(), flow.longName, flow.shortName); - } - return; - } - } - LOG(ERROR) << "Unknown flow type: " << act.flowType(); -} - -void ChromeTraceLogger::handleLink( - char type, - const ITraceActivity& e, - int64_t id, - const std::string& cat, - const std::string& name) { - if (!traceOf_) { - return; - } - - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "ph": "{}", "id": {}, "pid": {}, "tid": {}, "ts": {}, - "cat": "{}", "name": "{}", "bp": "e" - }},)JSON", - type, id, e.deviceId(), e.resourceId(), e.timestamp(), cat, name); - // clang-format on -} - -#ifdef HAS_CUPTI -// GPU side kernel activity -void ChromeTraceLogger::handleGpuActivity( - const GpuActivity& activity) { - if (!traceOf_) { - return; - } - const CUpti_ActivityKernel4* kernel = &activity.raw(); - constexpr int threads_per_warp = 32; - float blocks_per_sm = -1.0; - float warps_per_sm = -1.0; - int sm_count = smCount(kernel->deviceId); - if (sm_count) { - blocks_per_sm = - (kernel->gridX * kernel->gridY * kernel->gridZ) / (float) sm_count; - warps_per_sm = - blocks_per_sm * (kernel->blockX * kernel->blockY * kernel->blockZ) - / threads_per_warp; - } - - // Calculate occupancy - float occupancy = KINETO_NAMESPACE::kernelOccupancy( - kernel->deviceId, - kernel->registersPerThread, - kernel->staticSharedMemory, - kernel->dynamicSharedMemory, - kernel->blockX, - kernel->blockY, - kernel->blockZ, - blocks_per_sm); - - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "ph": "X", "cat": "Kernel", {}, - "args": {{ - "queued": {}, "device": {}, "context": {}, - "stream": {}, "correlation": {}, - "registers per thread": {}, - "shared memory": {}, - "blocks per SM": {}, - "warps per SM": {}, - "grid": [{}, {}, {}], - "block": [{}, {}, {}], - "est. achieved occupancy %": {} - }} - }},)JSON", - traceActivityJson(activity), - // args - us(kernel->queued), kernel->deviceId, kernel->contextId, - kernel->streamId, kernel->correlationId, - kernel->registersPerThread, - kernel->staticSharedMemory + kernel->dynamicSharedMemory, - blocks_per_sm, - warps_per_sm, - kernel->gridX, kernel->gridY, kernel->gridZ, - kernel->blockX, kernel->blockY, kernel->blockZ, - (int) (0.5 + occupancy * 100.0)); - // clang-format on - - auto to_id = activity.correlationId(); - handleLink(kFlowEnd, activity, to_id, "async_cpu_to_gpu", "async_gpu"); -} - -static std::string bandwidth(uint64_t bytes, uint64_t duration) { - return duration == 0 ? "\"N/A\"" : fmt::format("{}", bytes * 1.0 / duration); -} - -// GPU side memcpy activity -void ChromeTraceLogger::handleGpuActivity( - const GpuActivity& activity) { - if (!traceOf_) { - return; - } - const CUpti_ActivityMemcpy& memcpy = activity.raw(); - VLOG(2) << memcpy.correlationId << ": MEMCPY"; - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "ph": "X", "cat": "Memcpy", {}, - "args": {{ - "device": {}, "context": {}, - "stream": {}, "correlation": {}, - "bytes": {}, "memory bandwidth (GB/s)": {} - }} - }},)JSON", - traceActivityJson(activity), - // args - memcpy.deviceId, memcpy.contextId, - memcpy.streamId, memcpy.correlationId, - memcpy.bytes, bandwidth(memcpy.bytes, memcpy.end - memcpy.start)); - // clang-format on - - int64_t to_id = activity.correlationId(); - handleLink(kFlowEnd, activity, to_id, "async_cpu_to_gpu", "async_gpu"); -} - -// GPU side memcpy activity -void ChromeTraceLogger::handleGpuActivity( - const GpuActivity& activity) { - if (!traceOf_) { - return; - } - const CUpti_ActivityMemcpy2& memcpy = activity.raw(); - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "ph": "X", "cat": "Memcpy", {}, - "args": {{ - "fromDevice": {}, "inDevice": {}, "toDevice": {}, - "fromContext": {}, "inContext": {}, "toContext": {}, - "stream": {}, "correlation": {}, - "bytes": {}, "memory bandwidth (GB/s)": {} - }} - }},)JSON", - traceActivityJson(activity), - // args - memcpy.srcDeviceId, memcpy.deviceId, memcpy.dstDeviceId, - memcpy.srcContextId, memcpy.contextId, memcpy.dstContextId, - memcpy.streamId, memcpy.correlationId, - memcpy.bytes, bandwidth(memcpy.bytes, memcpy.end - memcpy.start)); - // clang-format on - - int64_t to_id = activity.correlationId(); - handleLink(kFlowEnd, activity, to_id, "async_cpu_to_gpu", "async_gpu"); -} - -void ChromeTraceLogger::handleGpuActivity( - const GpuActivity& activity) { - if (!traceOf_) { - return; - } - const CUpti_ActivityMemset& memset = activity.raw(); - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "ph": "X", "cat": "Memset", {}, - "args": {{ - "device": {}, "context": {}, - "stream": {}, "correlation": {}, - "bytes": {}, "memory bandwidth (GB/s)": {} - }} - }},)JSON", - traceActivityJson(activity), - // args - memset.deviceId, memset.contextId, - memset.streamId, memset.correlationId, - memset.bytes, bandwidth(memset.bytes, memset.end - memset.start)); - // clang-format on - - int64_t to_id = activity.correlationId(); - handleLink(kFlowEnd, activity, to_id, "async_cpu_to_gpu", "async_gpu"); -} -#endif // HAS_CUPTI - -void ChromeTraceLogger::finalizeTrace( - const Config& /*unused*/, - std::unique_ptr /*unused*/, - int64_t endTime, - std::unordered_map>& metadata) { - if (!traceOf_) { - LOG(ERROR) << "Failed to write to log file!"; - return; - } - LOG(INFO) << "Chrome Trace written to " << fileName_; - // clang-format off - traceOf_ << fmt::format(R"JSON( - {{ - "name": "Record Window End", "ph": "i", "s": "g", - "pid": "", "tid": "", "ts": {} - }} - ],)JSON", - endTime); - -#if !USE_GOOGLE_LOG - std::unordered_map PreparedMetadata; - for (const auto& kv : metadata) { - // Skip empty log buckets, ex. skip ERROR if its empty. - if (!kv.second.empty()) { - std::string value = "["; - // Ex. Each metadata from logger is a list of strings, expressed in JSON as - // "ERROR": ["Error 1", "Error 2"], - // "WARNING": ["Warning 1", "Warning 2", "Warning 3"], - // ... - int mdv_count = kv.second.size(); - for (const auto& v : kv.second) { - value.append("\"" + v + "\""); - if(mdv_count > 1) { - value.append(","); - mdv_count--; - } - } - value.append("]"); - PreparedMetadata[kv.first] = sanitizeStrForJSON(value); - } - } - metadataToJSON(PreparedMetadata); -#endif // !USE_GOOGLE_LOG - - // Putting this here because the last entry MUST not end with a comma. - traceOf_ << fmt::format(R"JSON( - "traceName": "{}" -}})JSON", sanitizeStrForJSON(fileName_)); - // clang-format on - - traceOf_.close(); -} - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/output_json.h b/plugins/tensorboard-plugins/libkineto/src/output_json.h deleted file mode 100644 index 5a8a81e4a..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/output_json.h +++ /dev/null @@ -1,91 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include -#include - -#ifdef HAS_CUPTI -#include -#endif -#include "GenericTraceActivity.h" -#include "output_base.h" - -namespace KINETO_NAMESPACE { - // Previous declaration of TraceSpan is struct. Must match the same here. - struct TraceSpan; -} - -namespace KINETO_NAMESPACE { - -class Config; - -class ChromeTraceLogger : public libkineto::ActivityLogger { - public: - explicit ChromeTraceLogger(const std::string& traceFileName); - - // Note: the caller of these functions should handle concurrency - // i.e., we these functions are not thread-safe - void handleDeviceInfo( - const DeviceInfo& info, - uint64_t time) override; - - void handleOverheadInfo(const OverheadInfo& info, int64_t time) override; - - void handleResourceInfo(const ResourceInfo& info, int64_t time) override; - - void handleTraceSpan(const TraceSpan& span) override; - - void handleActivity(const ITraceActivity& activity) override; - void handleGenericActivity(const GenericTraceActivity& activity) override; - -#ifdef HAS_CUPTI - void handleGpuActivity(const GpuActivity& activity) override; - void handleGpuActivity(const GpuActivity& activity) override; - void handleGpuActivity(const GpuActivity& activity) override; - void handleGpuActivity(const GpuActivity& activity) override; -#endif // HAS_CUPTI - - void handleTraceStart( - const std::unordered_map& metadata) override; - - void finalizeTrace( - const Config& config, - std::unique_ptr buffers, - int64_t endTime, - std::unordered_map>& metadata) override; - - std::string traceFileName() const { - return fileName_; - } - - private: - - // Create a flow event (arrow) - void handleLink( - char type, - const ITraceActivity& e, - int64_t id, - const std::string& cat, - const std::string& name); - - void addIterationMarker(const TraceSpan& span); - - void openTraceFile(); - - void handleGenericInstantEvent(const ITraceActivity& op); - - void handleGenericLink(const ITraceActivity& activity); - - void metadataToJSON(const std::unordered_map& metadata); - - std::string& sanitizeStrForJSON(std::string& value); - - std::string fileName_; - std::ofstream traceOf_; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/src/output_membuf.h b/plugins/tensorboard-plugins/libkineto/src/output_membuf.h deleted file mode 100644 index ef6aadeb6..000000000 --- a/plugins/tensorboard-plugins/libkineto/src/output_membuf.h +++ /dev/null @@ -1,130 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include -#include - -#ifdef HAS_CUPTI -#include -#endif - -#include "Config.h" -#include "GenericTraceActivity.h" -#ifdef HAS_CUPTI -#include "CuptiActivity.h" -#include "CuptiActivity.tpp" -#endif // HAS_CUPTI -#include "output_base.h" - -namespace KINETO_NAMESPACE { - -class Config; - -class MemoryTraceLogger : public ActivityLogger { - public: - MemoryTraceLogger(const Config& config) : config_(config.clone()) { - activities_.reserve(100000); - } - - // Note: the caller of these functions should handle concurrency - // i.e., these functions are not thread-safe - void handleDeviceInfo( - const DeviceInfo& info, - uint64_t time) override { - deviceInfoList_.emplace_back(info, time); - } - - void handleResourceInfo(const ResourceInfo& info, int64_t time) override { - resourceInfoList_.emplace_back(info, time); - } - - void handleOverheadInfo(const OverheadInfo& info, int64_t time) override {} - - void handleTraceSpan(const TraceSpan& span) override { - // Handled separately - } - - template - void addActivityWrapper(const T& act) { - wrappers_.push_back(std::make_unique(act)); - activities_.push_back(wrappers_.back().get()); - } - - // Just add the pointer to the list - ownership of the underlying - // objects must be transferred in ActivityBuffers via finalizeTrace - void handleActivity(const ITraceActivity& activity) override { - activities_.push_back(&activity); - } - void handleGenericActivity(const GenericTraceActivity& activity) override { - addActivityWrapper(activity); - } - -#ifdef HAS_CUPTI - void handleGpuActivity(const GpuActivity& activity) override { - addActivityWrapper(activity); - } - void handleGpuActivity(const GpuActivity& activity) override { - addActivityWrapper(activity); - } - void handleGpuActivity(const GpuActivity& activity) override { - addActivityWrapper(activity); - } - void handleGpuActivity(const GpuActivity& activity) override { - addActivityWrapper(activity); - } -#endif // HAS_CUPTI - - void handleTraceStart( - const std::unordered_map& metadata) override { - metadata_ = metadata; - } - - void finalizeTrace( - const Config& config, - std::unique_ptr buffers, - int64_t endTime, - std::unordered_map>& metadata) override { - buffers_ = std::move(buffers); - endTime_ = endTime; - } - - const std::vector* traceActivities() { - return &activities_; - } - - void log(ActivityLogger& logger) { - logger.handleTraceStart(metadata_); - for (auto& activity : activities_) { - activity->log(logger); - } - for (auto& p : deviceInfoList_) { - logger.handleDeviceInfo(p.first, p.second); - } - for (auto& p : resourceInfoList_) { - logger.handleResourceInfo(p.first, p.second); - } - for (auto& cpu_trace_buffer : buffers_->cpu) { - logger.handleTraceSpan(cpu_trace_buffer->span); - } - // Hold on to the buffers - logger.finalizeTrace(*config_, nullptr, endTime_, loggerMetadata_); - } - - private: - - std::unique_ptr config_; - // Optimization: Remove unique_ptr by keeping separate vector per type - std::vector activities_; - std::vector> wrappers_; - std::vector> deviceInfoList_; - std::vector> resourceInfoList_; - std::unique_ptr buffers_; - std::unordered_map metadata_; - std::unordered_map> loggerMetadata_; - int64_t endTime_{0}; -}; - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/test/CMakeLists.txt b/plugins/tensorboard-plugins/libkineto/test/CMakeLists.txt deleted file mode 100644 index ca54460b3..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/CMakeLists.txt +++ /dev/null @@ -1,3 +0,0 @@ -cmake_minimum_required(VERSION 3.5 FATAL_ERROR) - -# TODO diff --git a/plugins/tensorboard-plugins/libkineto/test/ConfigTest.cpp b/plugins/tensorboard-plugins/libkineto/test/ConfigTest.cpp deleted file mode 100644 index 16bc86e75..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/ConfigTest.cpp +++ /dev/null @@ -1,315 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "include/Config.h" - -#include -#include -#include -#include - -using namespace std::chrono; -using namespace KINETO_NAMESPACE; - -TEST(ParseTest, Whitespace) { - Config cfg; - // Check that various types of whitespace is ignored - EXPECT_TRUE(cfg.parse("")); - EXPECT_TRUE(cfg.parse(" ")); - EXPECT_TRUE(cfg.parse("\t")); - EXPECT_TRUE(cfg.parse("\n")); - EXPECT_TRUE(cfg.parse(" ")); - EXPECT_TRUE(cfg.parse("\t \n \t\t\n\n")); - // Only the above characters are supported - EXPECT_FALSE(cfg.parse("\r\n")); -} - -TEST(ParseTest, Comment) { - Config cfg; - // Anything following a '#' should be ignored, up to a newline - EXPECT_TRUE(cfg.parse("# comment")); - EXPECT_TRUE(cfg.parse(" # ~!@#$")); - EXPECT_TRUE(cfg.parse("\t#abc")); - EXPECT_TRUE(cfg.parse("###\n##")); - EXPECT_TRUE(cfg.parse("EVENTS=util ##ok")); - EXPECT_TRUE(cfg.parse("EVENTS=util ## EVENTS=instruction")); - // Whatever appears before the comment must be valid format - EXPECT_FALSE(cfg.parse("util ## not ok")); - EXPECT_FALSE(cfg.parse("## ok \n blah # not OK")); - // Check that a comment does not affect config parsing - EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS = 1 # Sample every millisecond")); - EXPECT_EQ(cfg.samplePeriod(), milliseconds(1)); -} - -TEST(ParseTest, Format) { - Config cfg; - // The basic format is just "name = value". - // Where both value and name can be almost anything. - // Leading and trailing whitespace should be removed - // for both 'name' and 'value', but internal whitespace is not. - EXPECT_FALSE(cfg.parse("events")); - EXPECT_TRUE(cfg.parse("events=")); - EXPECT_FALSE(cfg.parse("=events=")); - EXPECT_TRUE(cfg.parse("events=1,2,3")); - // Only one setting per line - EXPECT_FALSE(cfg.parse("events = 1,2,3 ; metrics = 4,5,6")); - // Names are case sensitive - EXPECT_TRUE(cfg.parse("EVENTS = 1,2,3 \n metrics = 4,5,6")); - EXPECT_EQ(cfg.eventNames(), std::set({"1", "2", "3"})); - EXPECT_EQ(cfg.metricNames().size(), 0); - // Leading and trailing whitespace removed for event and metric names, - // but not internal. - EXPECT_TRUE( - cfg.parse("EVENTS = 1, 2, 3 \n \tMETRICS\t = \t4,\t5\t,\ts i x ")); - EXPECT_EQ(cfg.eventNames(), std::set({"1", "2", "3"})); - EXPECT_EQ(cfg.metricNames(), std::set({"4", "5", "s i x"})); -} - -TEST(ParseTest, DefaultActivityTypes) { - Config cfg; - cfg.validate(std::chrono::system_clock::now()); - auto all_activities = activityTypes(); - // TODO: introduce optional activities - EXPECT_EQ(cfg.selectedActivityTypes(), - std::set(all_activities.begin(), all_activities.end() - 1)); -} - -TEST(ParseTest, ActivityTypes) { - Config cfg; - EXPECT_FALSE(cfg.parse("ACTIVITY_TYPES")); - EXPECT_TRUE(cfg.parse("ACTIVITY_TYPES=")); - EXPECT_FALSE(cfg.parse("=ACTIVITY_TYPES=")); - - EXPECT_EQ(cfg.selectedActivityTypes(), - std::set({ActivityType::CPU_OP, - ActivityType::CPU_INSTANT_EVENT, - ActivityType::PYTHON_FUNCTION, - ActivityType::USER_ANNOTATION, - ActivityType::GPU_USER_ANNOTATION, - ActivityType::GPU_MEMCPY, - ActivityType::GPU_MEMSET, - ActivityType::CONCURRENT_KERNEL, - ActivityType::EXTERNAL_CORRELATION, - ActivityType::GLOW_RUNTIME, - ActivityType::CUDA_RUNTIME, - ActivityType::CUDA_PROFILER_RANGE})); - - Config cfg2; - EXPECT_TRUE(cfg2.parse("ACTIVITY_TYPES=gpu_memcpy,gpu_MeMsEt,kernel")); - EXPECT_EQ(cfg2.selectedActivityTypes(), - std::set({ActivityType::GPU_MEMCPY, - ActivityType::GPU_MEMSET, - ActivityType::CONCURRENT_KERNEL})); - - EXPECT_TRUE(cfg2.parse("ACTIVITY_TYPES = cuda_Runtime,")); - EXPECT_EQ(cfg2.selectedActivityTypes(), - std::set({ActivityType::CUDA_RUNTIME})); - - // Should throw an exception because incorrect activity name - EXPECT_FALSE(cfg2.parse("ACTIVITY_TYPES = memcopy,cuda_runtime")); - - EXPECT_TRUE(cfg2.parse("ACTIVITY_TYPES = cpu_op")); - EXPECT_EQ(cfg2.selectedActivityTypes(), - std::set({ActivityType::CPU_OP})); -} - -TEST(ParseTest, SamplePeriod) { - Config cfg; - EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS=10")); - EXPECT_EQ(cfg.samplePeriod(), milliseconds(10)); - EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS=0")); - cfg.validate(std::chrono::system_clock::now()); - // 0 should be adjustd up to 1 - EXPECT_EQ(cfg.samplePeriod(), milliseconds(1)); - // Negative and non-int values should fail - EXPECT_FALSE(cfg.parse("SAMPLE_PERIOD_MSECS=-10")); - EXPECT_FALSE(cfg.parse("SAMPLE_PERIOD_MSECS=1.5")); - EXPECT_FALSE(cfg.parse("SAMPLE_PERIOD_MSECS=")); - EXPECT_FALSE(cfg.parse("SAMPLE_PERIOD_MSECS=string")); - EXPECT_EQ(cfg.samplePeriod(), milliseconds(1)); -} - -TEST(ParseTest, MultiplexPeriod) { - Config cfg; - auto now = std::chrono::system_clock::now(); - - EXPECT_TRUE(cfg.parse("SAMPLE_PERIOD_MSECS=100\nMULTIPLEX_PERIOD_MSECS=100")); - EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(100)); - EXPECT_TRUE(cfg.parse("MULTIPLEX_PERIOD_MSECS = 0")); - cfg.validate(now); - // Adjusted to match sample period - EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(100)); - EXPECT_TRUE(cfg.parse("MULTIPLEX_PERIOD_MSECS \t= \t 750 \n")); - cfg.validate(now); - // Adjusted to match multiple of sample period - EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(800)); - EXPECT_FALSE(cfg.parse("MULTIPLEX_PERIOD_MSECS=-10")); - EXPECT_FALSE(cfg.parse("MULTIPLEX_PERIOD_MSECS=1.5")); - EXPECT_FALSE(cfg.parse("MULTIPLEX_PERIOD_MSECS=")); - EXPECT_FALSE(cfg.parse("MULTIPLEX_PERIOD_MSECS=string")); - // Previous value not affected - EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(800)); -} - -TEST(ParseTest, ReportPeriod) { - Config cfg; - EXPECT_TRUE(cfg.parse("REPORT_PERIOD_SECS=1")); - EXPECT_EQ(cfg.reportPeriod(), seconds(1)); - // Whitespace - EXPECT_TRUE(cfg.parse("REPORT_PERIOD_SECS = \t100")); - EXPECT_EQ(cfg.reportPeriod(), seconds(100)); - // Invalid types - EXPECT_FALSE(cfg.parse("REPORT_PERIOD_SECS=-1")); - EXPECT_EQ(cfg.reportPeriod(), seconds(100)); -} - -TEST(ParseTest, SamplesPerReport) { - Config cfg; - auto now = std::chrono::system_clock::now(); - - EXPECT_TRUE(cfg.parse(R"( - SAMPLE_PERIOD_MSECS = 1000 - REPORT_PERIOD_SECS = 1 - SAMPLES_PER_REPORT = 10)")); - cfg.validate(now); - // Adjusted down to one sample per report - EXPECT_EQ(cfg.samplesPerReport(), 1); - EXPECT_TRUE(cfg.parse(R"( - SAMPLE_PERIOD_MSECS = 1000 - REPORT_PERIOD_SECS = 10 - SAMPLES_PER_REPORT = 10)")); - cfg.validate(now); - // No adjustment needed - EXPECT_EQ(cfg.samplesPerReport(), 10); - EXPECT_TRUE(cfg.parse(R"( - SAMPLE_PERIOD_MSECS = 1000 - REPORT_PERIOD_SECS = 2 - SAMPLES_PER_REPORT = 10)")); - cfg.validate(now); - // Adjusted to 2 samples per report - EXPECT_EQ(cfg.samplesPerReport(), 2); - EXPECT_TRUE(cfg.parse(R"( - SAMPLE_PERIOD_MSECS = 200 - REPORT_PERIOD_SECS = 2 - SAMPLES_PER_REPORT = 10)")); - cfg.validate(now); - // No adjustment needed - EXPECT_EQ(cfg.samplesPerReport(), 10); - EXPECT_TRUE(cfg.parse("SAMPLES_PER_REPORT=0")); - cfg.validate(now); - // Adjusted up to 1 - EXPECT_EQ(cfg.samplesPerReport(), 1); - // Invalid value types - EXPECT_FALSE(cfg.parse("SAMPLES_PER_REPORT=-10")); - EXPECT_FALSE(cfg.parse("SAMPLES_PER_REPORT=1.5")); - EXPECT_EQ(cfg.samplesPerReport(), 1); - - EXPECT_TRUE(cfg.parse(R"( - SAMPLE_PERIOD_MSECS=1000 - MULTIPLEX_PERIOD_MSECS=500 # Must be a multiple of sample period - REPORT_PERIOD_SECS=0 # Must be non-zero multiple of multiplex period - SAMPLES_PER_REPORT=5 # Max report period / multiplex period)")); - cfg.validate(now); - // Multiple adjustments - EXPECT_EQ(cfg.samplePeriod(), milliseconds(1000)); - EXPECT_EQ(cfg.multiplexPeriod(), milliseconds(1000)); - EXPECT_EQ(cfg.reportPeriod(), seconds(1)); - EXPECT_EQ(cfg.samplesPerReport(), 1); -} - -TEST(ParseTest, EnableSigUsr2) { - Config cfg; - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=yes")); - EXPECT_TRUE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=no")); - EXPECT_FALSE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=YES")); - EXPECT_TRUE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=NO")); - EXPECT_FALSE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=Y")); - EXPECT_TRUE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=N")); - EXPECT_FALSE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=T")); - EXPECT_TRUE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=F")); - EXPECT_FALSE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=true")); - EXPECT_TRUE(cfg.sigUsr2Enabled()); - EXPECT_TRUE(cfg.parse("ENABLE_SIGUSR2=false")); - EXPECT_FALSE(cfg.sigUsr2Enabled()); - EXPECT_FALSE(cfg.parse("ENABLE_SIGUSR2= ")); - EXPECT_FALSE(cfg.parse("ENABLE_SIGUSR2=2")); - EXPECT_FALSE(cfg.parse("ENABLE_SIGUSR2=-1")); - EXPECT_FALSE(cfg.parse("ENABLE_SIGUSR2=yep")); -} - -TEST(ParseTest, DeviceMask) { - Config cfg; - // Single device - EXPECT_TRUE(cfg.parse("EVENTS_ENABLED_DEVICES = 0")); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(0)); - EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(1)); - - // Two devices, internal whitespace - EXPECT_TRUE(cfg.parse("EVENTS_ENABLED_DEVICES = 1, 2")); - EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(0)); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(1)); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(2)); - EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(3)); - - // Three devices, check that previous devices are ignored - EXPECT_TRUE(cfg.parse("EVENTS_ENABLED_DEVICES = 0, 2,4")); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(0)); - EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(1)); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(2)); - EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(3)); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(4)); - EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(5)); - - // Repeated numbers have no effect - EXPECT_TRUE(cfg.parse("EVENTS_ENABLED_DEVICES = 0,1,1,1,2,3,2,1,3,7,7,3")); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(0)); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(1)); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(2)); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(3)); - EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(4)); - EXPECT_FALSE(cfg.eventProfilerEnabledForDevice(6)); - EXPECT_TRUE(cfg.eventProfilerEnabledForDevice(7)); - - // 8 is larger than the max allowed - EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 3,8")); - - // 300 cannot be held in an uint8_t - EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 300")); - - // Various illegal cases - EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 0,1,two,three")); - EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 0,1,,2")); - EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = -1")); - EXPECT_FALSE(cfg.parse("EVENTS_ENABLED_DEVICES = 1.0")); -} - -TEST(ParseTest, RequestTime) { - Config cfg; - system_clock::time_point now = system_clock::now(); - int64_t tgood_ms = - duration_cast(now.time_since_epoch()).count(); - EXPECT_TRUE(cfg.parse(fmt::format("REQUEST_TIMESTAMP = {}", tgood_ms))); - - tgood_ms = duration_cast((now - seconds(5)).time_since_epoch()) - .count(); - EXPECT_TRUE(cfg.parse(fmt::format("REQUEST_TIMESTAMP = {}", tgood_ms))); - - int64_t tbad_ms = - duration_cast((now - seconds(20)).time_since_epoch()) - .count(); - EXPECT_FALSE(cfg.parse(fmt::format("REQUEST_TIMESTAMP = {}", tbad_ms))); - - EXPECT_FALSE(cfg.parse("REQUEST_TIMESTAMP = 0")); - EXPECT_FALSE(cfg.parse("REQUEST_TIMESTAMP = -1")); - - tbad_ms = duration_cast((now + seconds(10)).time_since_epoch()) - .count(); - EXPECT_FALSE(cfg.parse(fmt::format("REQUEST_TIMESTAMP = {}", tbad_ms))); -} diff --git a/plugins/tensorboard-plugins/libkineto/test/CuptiActivityProfilerTest.cpp b/plugins/tensorboard-plugins/libkineto/test/CuptiActivityProfilerTest.cpp deleted file mode 100644 index 6e67980ee..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/CuptiActivityProfilerTest.cpp +++ /dev/null @@ -1,629 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include -#include -#include -#include -#include - -#ifdef __linux__ -#include -#include -#include -#endif - -#include "include/libkineto.h" -#include "include/Config.h" -#include "src/CuptiActivityProfiler.h" -#include "src/ActivityTrace.h" -#include "src/CuptiActivityApi.h" -#include "src/output_base.h" -#include "src/output_json.h" -#include "src/output_membuf.h" - -#include "src/Logger.h" -#include "test/MockActivitySubProfiler.h" - -using namespace std::chrono; -using namespace KINETO_NAMESPACE; - -#define CUDA_LAUNCH_KERNEL CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000 -#define CUDA_MEMCPY CUPTI_RUNTIME_TRACE_CBID_cudaMemcpy_v3020 - -namespace { -const TraceSpan& defaultTraceSpan() { - static TraceSpan span(0, 0, "Unknown", ""); - return span; -} -} - -// Provides ability to easily create a few test CPU-side ops -struct MockCpuActivityBuffer : public CpuTraceBuffer { - MockCpuActivityBuffer(int64_t startTime, int64_t endTime) { - span = TraceSpan(startTime, endTime,"Test trace"); - gpuOpCount = 0; - } - - void addOp(std::string name, int64_t startTime, int64_t endTime, int64_t correlation) { - GenericTraceActivity op(span, ActivityType::CPU_OP, name); - op.startTime = startTime; - op.endTime = endTime; - op.resource = systemThreadId(); - op.id = correlation; - activities.push_back(std::move(op)); - span.opCount++; - } -}; - -// Provides ability to easily create a few test CUPTI ops -struct MockCuptiActivityBuffer { - void addCorrelationActivity(int64_t correlation, CUpti_ExternalCorrelationKind externalKind, int64_t externalId) { - auto& act = *(CUpti_ActivityExternalCorrelation*) malloc(sizeof(CUpti_ActivityExternalCorrelation)); - act.kind = CUPTI_ACTIVITY_KIND_EXTERNAL_CORRELATION; - act.externalId = externalId; - act.externalKind = externalKind; - act.correlationId = correlation; - activities.push_back(reinterpret_cast(&act)); - } - - void addRuntimeActivity( - CUpti_runtime_api_trace_cbid_enum cbid, - int64_t start_us, int64_t end_us, int64_t correlation) { - auto& act = createActivity( - start_us, end_us, correlation); - act.kind = CUPTI_ACTIVITY_KIND_RUNTIME; - act.cbid = cbid; - act.threadId = threadId(); - activities.push_back(reinterpret_cast(&act)); - } - - void addKernelActivity( - int64_t start_us, int64_t end_us, int64_t correlation) { - auto& act = createActivity( - start_us, end_us, correlation); - act.kind = CUPTI_ACTIVITY_KIND_CONCURRENT_KERNEL; - act.deviceId = 0; - act.streamId = 1; - act.name = "kernel"; - act.gridX = act.gridY = act.gridZ = 1; - act.blockX = act.blockY = act.blockZ = 1; - activities.push_back(reinterpret_cast(&act)); - } - - void addMemcpyActivity( - int64_t start_us, int64_t end_us, int64_t correlation) { - auto& act = createActivity( - start_us, end_us, correlation); - act.kind = CUPTI_ACTIVITY_KIND_MEMCPY; - act.deviceId = 0; - act.streamId = 2; - act.copyKind = CUPTI_ACTIVITY_MEMCPY_KIND_HTOD; - act.srcKind = CUPTI_ACTIVITY_MEMORY_KIND_PINNED; - act.dstKind = CUPTI_ACTIVITY_MEMORY_KIND_DEVICE; - activities.push_back(reinterpret_cast(&act)); - } - - template - T& createActivity( - int64_t start_us, int64_t end_us, int64_t correlation) { - T& act = *static_cast(malloc(sizeof(T))); - bzero(&act, sizeof(act)); - act.start = start_us * 1000; - act.end = end_us * 1000; - act.correlationId = correlation; - return act; - } - - ~MockCuptiActivityBuffer() { - for (CUpti_Activity* act : activities) { - free(act); - } - } - - std::vector activities; -}; - -// Mock parts of the CuptiActivityApi -class MockCuptiActivities : public CuptiActivityApi { - public: - virtual int smCount() override { - return 10; - } - - virtual const std::pair processActivities( - CuptiActivityBufferMap&, /*unused*/ - std::function handler) override { - for (CUpti_Activity* act : activityBuffer->activities) { - handler(act); - } - return {activityBuffer->activities.size(), 100}; - } - - virtual std::unique_ptr - activityBuffers() override { - auto map = std::make_unique(); - auto buf = std::make_unique(100); - uint8_t* addr = buf->data(); - (*map)[addr] = std::move(buf); - return map; - } - - void bufferRequestedOverride(uint8_t** buffer, size_t* size, size_t* maxNumRecords) { - this->bufferRequested(buffer, size, maxNumRecords); - } - - std::unique_ptr activityBuffer; -}; - - -// Common setup / teardown and helper functions -class CuptiActivityProfilerTest : public ::testing::Test { - protected: - void SetUp() override { - profiler_ = std::make_unique( - cuptiActivities_, /*cpu only*/ false); - cfg_ = std::make_unique(); - cfg_->validate(std::chrono::system_clock::now()); - loggerFactory.addProtocol("file", [](const std::string& url) { - return std::unique_ptr(new ChromeTraceLogger(url)); - }); - } - - std::unique_ptr cfg_; - MockCuptiActivities cuptiActivities_; - std::unique_ptr profiler_; - ActivityLoggerFactory loggerFactory; -}; - -void checkTracefile(const char* filename) { -#ifdef __linux__ - // Check that the expected file was written and that it has some content - int fd = open(filename, O_RDONLY); - if (!fd) { - perror(filename); - } - EXPECT_TRUE(fd); - // Should expect at least 100 bytes - struct stat buf{}; - fstat(fd, &buf); - EXPECT_GT(buf.st_size, 100); - close(fd); -#endif -} - -TEST(CuptiActivityProfiler, AsyncTrace) { - std::vector log_modules( - {"CuptiActivityProfiler.cpp", "output_json.cpp"}); - SET_LOG_VERBOSITY_LEVEL(1, log_modules); - - MockCuptiActivities activities; - CuptiActivityProfiler profiler(activities, /*cpu only*/ true); - - char filename[] = "/tmp/libkineto_testXXXXXX.json"; - mkstemps(filename, 5); - - Config cfg; - - int iter = 0; - int warmup = 5; - auto now = system_clock::now(); - auto startTime = now + seconds(10); - - bool success = cfg.parse(fmt::format(R"CFG( - ACTIVITIES_WARMUP_PERIOD_SECS = {} - ACTIVITIES_DURATION_SECS = 1 - ACTIVITIES_LOG_FILE = {} - PROFILE_START_TIME = {} - )CFG", warmup, filename, duration_cast(startTime.time_since_epoch()).count())); - - EXPECT_TRUE(success); - EXPECT_FALSE(profiler.isActive()); - - auto logger = std::make_unique(cfg.activitiesLogFile()); - - // Usually configuration is done when now is startTime - warmup to kick off warmup - // but start right away in the test - profiler.configure(cfg, now); - profiler.setLogger(logger.get()); - - EXPECT_TRUE(profiler.isActive()); - - // fast forward in time and we have reached the startTime - now = startTime; - - // Run the profiler - // Warmup - // performRunLoopStep is usually called by the controller loop and takes - // the current time and the controller's next wakeup time. - profiler.performRunLoopStep( - /* Current time */ now, /* Next wakeup time */ now); - - auto next = now + milliseconds(1000); - - // performRunLoopStep can also be called by an application thread to update iteration count - // since this config does not use iteration this should have no effect on the state - while (++iter < 20) { - profiler.performRunLoopStep(now, now, iter); - } - - // Runloop should now be in collect state, so start workload - // Perform another runloop step, passing in the end profile time as current. - // This should terminate collection - profiler.performRunLoopStep( - /* Current time */ next, /* Next wakeup time */ next); - // One step needed for each of the Process and Finalize phases - // Doesn't really matter what times we pass in here. - - EXPECT_TRUE(profiler.isActive()); - - auto nextnext = next + milliseconds(1000); - - while (++iter < 40) { - profiler.performRunLoopStep(next, next, iter); - } - - EXPECT_TRUE(profiler.isActive()); - - profiler.performRunLoopStep(nextnext,nextnext); - profiler.performRunLoopStep(nextnext,nextnext); - - // Assert that tracing has completed - EXPECT_FALSE(profiler.isActive()); - - checkTracefile(filename); -} - -TEST(CuptiActivityProfiler, AsyncTraceUsingIter) { - std::vector log_modules( - {"CuptiActivityProfiler.cpp", "output_json.cpp"}); - SET_LOG_VERBOSITY_LEVEL(1, log_modules); - - auto runIterTest = [&]( - int start_iter, int warmup_iters, int trace_iters) { - - LOG(INFO ) << "Async Trace Test: start_iteration = " << start_iter - << " warmup iterations = " << warmup_iters - << " trace iterations = " << trace_iters; - - MockCuptiActivities activities; - CuptiActivityProfiler profiler(activities, /*cpu only*/ true); - - char filename[] = "/tmp/libkineto_testXXXXXX.json"; - mkstemps(filename, 5); - - Config cfg; - - int iter = 0; - auto now = system_clock::now(); - - bool success = cfg.parse(fmt::format(R"CFG( - PROFILE_START_ITERATION = {} - ACTIVITIES_WARMUP_ITERATIONS={} - ACTIVITIES_ITERATIONS={} - ACTIVITIES_DURATION_SECS = 1 - ACTIVITIES_LOG_FILE = {} - )CFG", start_iter, warmup_iters, trace_iters, filename)); - - EXPECT_TRUE(success); - EXPECT_FALSE(profiler.isActive()); - - auto logger = std::make_unique(cfg.activitiesLogFile()); - - // Usually configuration is done when now is startIter - warmup iter to kick off warmup - // but start right away in the test - while (iter < (start_iter - warmup_iters)) { - profiler.performRunLoopStep(now, now, iter++); - } - - profiler.configure(cfg, now); - profiler.setLogger(logger.get()); - - EXPECT_TRUE(profiler.isActive()); - - // fast forward in time, mimicking what will happen in reality - now += seconds(10); - auto next = now + milliseconds(1000); - - // this call to runloop step should not be effecting the state - profiler.performRunLoopStep(now, next); - EXPECT_TRUE(profiler.isActive()); - - // start trace collection - while (iter < start_iter) { - profiler.performRunLoopStep(now, next, iter++); - } - - // Runloop should now be in collect state, so start workload - - while (iter < (start_iter + trace_iters)) { - profiler.performRunLoopStep(now, next, iter++); - } - - // One step is required for each of the Process and Finalize phases - // Doesn't really matter what times we pass in here. - if (iter >= (start_iter + trace_iters)) { - profiler.performRunLoopStep(now, next, iter++); - } - EXPECT_TRUE(profiler.isActive()); - - auto nextnext = next + milliseconds(1000); - - profiler.performRunLoopStep(nextnext, nextnext); - profiler.performRunLoopStep(nextnext, nextnext); - - // Assert that tracing has completed - EXPECT_FALSE(profiler.isActive()); - - checkTracefile(filename); - }; - - // start iter = 50, warmup iters = 5, trace iters = 10 - runIterTest(50, 5, 10); - // should be able to start at 0 iteration - runIterTest(0, 0, 2); - runIterTest(0, 5, 5); -} - -TEST_F(CuptiActivityProfilerTest, SyncTrace) { - using ::testing::Return; - using ::testing::ByMove; - - // Verbose logging is useful for debugging - std::vector log_modules( - {"CuptiActivityProfiler.cpp"}); - SET_LOG_VERBOSITY_LEVEL(2, log_modules); - - // Start and stop profiling - CuptiActivityProfiler profiler(cuptiActivities_, /*cpu only*/ false); - int64_t start_time_us = 100; - int64_t duration_us = 300; - auto start_time = time_point(microseconds(start_time_us)); - profiler.configure(*cfg_, start_time); - profiler.startTrace(start_time); - profiler.stopTrace(start_time + microseconds(duration_us)); - - profiler.recordThreadInfo(); - - // Log some cpu ops - auto cpuOps = std::make_unique( - start_time_us, start_time_us + duration_us); - cpuOps->addOp("op1", 120, 150, 1); - cpuOps->addOp("op2", 130, 140, 2); - cpuOps->addOp("op3", 200, 250, 3); - profiler.transferCpuTrace(std::move(cpuOps)); - - // And some GPU ops - auto gpuOps = std::make_unique(); - gpuOps->addRuntimeActivity(CUDA_LAUNCH_KERNEL, 133, 138, 1); - gpuOps->addRuntimeActivity(CUDA_MEMCPY, 210, 220, 2); - gpuOps->addRuntimeActivity(CUDA_LAUNCH_KERNEL, 230, 245, 3); - gpuOps->addKernelActivity(150, 170, 1); - gpuOps->addMemcpyActivity(240, 250, 2); - gpuOps->addKernelActivity(260, 320, 3); - cuptiActivities_.activityBuffer = std::move(gpuOps); - - // Have the profiler process them - auto logger = std::make_unique(*cfg_); - profiler.processTrace(*logger); - - // Profiler can be reset at this point - logger owns the activities - profiler_->reset(); - - // Wrapper that allows iterating over the activities - ActivityTrace trace(std::move(logger), loggerFactory); - EXPECT_EQ(trace.activities()->size(), 9); - std::map activityCounts; - std::map resourceIds; - for (auto& activity : *trace.activities()) { - activityCounts[activity->name()]++; - resourceIds[activity->resourceId()]++; - } - for (const auto& p : activityCounts) { - LOG(INFO) << p.first << ": " << p.second; - } - EXPECT_EQ(activityCounts["op1"], 1); - EXPECT_EQ(activityCounts["op2"], 1); - EXPECT_EQ(activityCounts["op3"], 1); - EXPECT_EQ(activityCounts["cudaLaunchKernel"], 2); - EXPECT_EQ(activityCounts["cudaMemcpy"], 1); - EXPECT_EQ(activityCounts["kernel"], 2); - EXPECT_EQ(activityCounts["Memcpy HtoD (Pinned -> Device)"], 1); - - auto sysTid = systemThreadId(); - // Ops and runtime events are on thread sysTid - EXPECT_EQ(resourceIds[sysTid], 6); - // Kernels are on stream 1, memcpy on stream 2 - EXPECT_EQ(resourceIds[1], 2); - EXPECT_EQ(resourceIds[2], 1); - -#ifdef __linux__ - char filename[] = "/tmp/libkineto_testXXXXXX.json"; - mkstemps(filename, 5); - trace.save(filename); - // Check that the expected file was written and that it has some content - int fd = open(filename, O_RDONLY); - if (!fd) { - perror(filename); - } - EXPECT_TRUE(fd); - // Should expect at least 100 bytes - struct stat buf{}; - fstat(fd, &buf); - EXPECT_GT(buf.st_size, 100); -#endif -} - -TEST_F(CuptiActivityProfilerTest, GpuUserAnnotationTest) { - // Verbose logging is useful for debugging - std::vector log_modules( - {"CuptiActivityProfiler.cpp"}); - SET_LOG_VERBOSITY_LEVEL(2, log_modules); - - // Start and stop profiling - CuptiActivityProfiler profiler(cuptiActivities_, /*cpu only*/ false); - int64_t start_time_us = 100; - int64_t duration_us = 300; - auto start_time = time_point(microseconds(start_time_us)); - profiler.configure(*cfg_, start_time); - profiler.startTrace(start_time); - profiler.stopTrace(start_time + microseconds(duration_us)); - - int64_t kernelLaunchTime = 120; - profiler.recordThreadInfo(); - - // set up CPU event - auto cpuOps = std::make_unique( - start_time_us, start_time_us + duration_us); - cpuOps->addOp("annotation", kernelLaunchTime, kernelLaunchTime + 10, 1); - profiler.transferCpuTrace(std::move(cpuOps)); - - // set up a couple of GPU events and correlate with above CPU event. - // CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1 is used for user annotations. - auto gpuOps = std::make_unique(); - gpuOps->addCorrelationActivity(1, CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1, 1); - gpuOps->addKernelActivity(kernelLaunchTime + 5, kernelLaunchTime + 10, 1); - gpuOps->addCorrelationActivity(1, CUPTI_EXTERNAL_CORRELATION_KIND_CUSTOM1, 1); - gpuOps->addKernelActivity(kernelLaunchTime + 15, kernelLaunchTime + 25, 1); - cuptiActivities_.activityBuffer = std::move(gpuOps); - - // process trace - auto logger = std::make_unique(*cfg_); - profiler.processTrace(*logger); - - ActivityTrace trace(std::move(logger), loggerFactory); - std::map counts; - for (auto& activity : *trace.activities()) { - counts[activity->name()]++; - } - - // We should now have an additional annotation activity created - // on the GPU timeline. - EXPECT_EQ(counts["annotation"], 2); - EXPECT_EQ(counts["kernel"], 2); - - auto& annotation = trace.activities()->at(0); - auto& kernel1 = trace.activities()->at(1); - auto& kernel2 = trace.activities()->at(2); - auto& gpu_annotation = trace.activities()->at(3); - EXPECT_EQ(gpu_annotation->type(), ActivityType::GPU_USER_ANNOTATION); - EXPECT_EQ(gpu_annotation->timestamp(), kernel1->timestamp()); - EXPECT_EQ( - gpu_annotation->duration(), - kernel2->timestamp() + kernel2->duration() - kernel1->timestamp()); - EXPECT_EQ(gpu_annotation->deviceId(), kernel1->deviceId()); - EXPECT_EQ(gpu_annotation->resourceId(), kernel1->resourceId()); - EXPECT_EQ(gpu_annotation->correlationId(), annotation->correlationId()); - EXPECT_EQ(gpu_annotation->name(), annotation->name()); -} - -TEST_F(CuptiActivityProfilerTest, SubActivityProfilers) { - using ::testing::Return; - using ::testing::ByMove; - - // Verbose logging is useful for debugging - std::vector log_modules( - {"CuptiActivityProfiler.cpp"}); - SET_LOG_VERBOSITY_LEVEL(2, log_modules); - - // Setup example events to test - GenericTraceActivity ev{defaultTraceSpan(), ActivityType::GLOW_RUNTIME, ""}; - ev.device = 1; - ev.resource = 0; - - int64_t start_time_us = 100; - int64_t duration_us = 1000; - auto start_time = time_point(microseconds(start_time_us)); - - std::vector test_activities{3, ev}; - test_activities[0].startTime = start_time_us; - test_activities[0].endTime = start_time_us + 5000; - test_activities[0].activityName = "SubGraph A execution"; - test_activities[1].startTime = start_time_us; - test_activities[1].endTime = start_time_us + 2000; - test_activities[1].activityName = "Operator foo"; - test_activities[2].startTime = start_time_us + 2500; - test_activities[2].endTime = start_time_us + 2900; - test_activities[2].activityName = "Operator bar"; - - auto mock_activity_profiler = - std::make_unique(test_activities); - - MockCuptiActivities activities; - CuptiActivityProfiler profiler(activities, /*cpu only*/ true); - profiler.addChildActivityProfiler( - std::move(mock_activity_profiler)); - - profiler.configure(*cfg_, start_time); - profiler.startTrace(start_time); - EXPECT_TRUE(profiler.isActive()); - - profiler.stopTrace(start_time + microseconds(duration_us)); - EXPECT_TRUE(profiler.isActive()); - - char filename[] = "/tmp/libkineto_testXXXXXX.json"; - mkstemps(filename, 5); - LOG(INFO) << "Logging to tmp file " << filename; - - // process trace - auto logger = std::make_unique(*cfg_); - profiler.processTrace(*logger); - profiler.setLogger(logger.get()); - - ActivityTrace trace(std::move(logger), loggerFactory); - trace.save(filename); - const auto& traced_activites = trace.activities(); - - // Test we have all the events - EXPECT_EQ(traced_activites->size(), test_activities.size()); - - // Check that the expected file was written and that it has some content - int fd = open(filename, O_RDONLY); - if (!fd) { - perror(filename); - } - EXPECT_TRUE(fd); - - // Should expect at least 100 bytes - struct stat buf{}; - fstat(fd, &buf); - EXPECT_GT(buf.st_size, 100); -} - -TEST_F(CuptiActivityProfilerTest, BufferSizeLimitTestWarmup) { - CuptiActivityProfiler profiler(cuptiActivities_, /*cpu only*/ false); - - auto now = system_clock::now(); - auto startTime = now + seconds(10); - - int maxBufferSizeMB = 3; - - auto startTimeEpoch = std::to_string(duration_cast(startTime.time_since_epoch()).count()); - std::string maxBufferSizeMBStr = std::to_string(maxBufferSizeMB); - cfg_->handleOption("ACTIVITIES_MAX_GPU_BUFFER_SIZE_MB", maxBufferSizeMBStr); - cfg_->handleOption("PROFILE_START_TIME", startTimeEpoch); - - - EXPECT_FALSE(profiler.isActive()); - profiler.configure(*cfg_, now); - EXPECT_TRUE(profiler.isActive()); - - for (size_t i = 0; i < maxBufferSizeMB; i++) { - uint8_t* buf; - size_t gpuBufferSize; - size_t maxNumRecords; - cuptiActivities_.bufferRequestedOverride(&buf, &gpuBufferSize, &maxNumRecords); - } - - // fast forward to startTime and profiler is now running - now = startTime; - - profiler.performRunLoopStep(now, now); - - auto next = now + milliseconds(1000); - profiler.performRunLoopStep(next, next); - profiler.performRunLoopStep(next, next); - profiler.performRunLoopStep(next, next); - - EXPECT_FALSE(profiler.isActive()); -} diff --git a/plugins/tensorboard-plugins/libkineto/test/CuptiCallbackApiTest.cpp b/plugins/tensorboard-plugins/libkineto/test/CuptiCallbackApiTest.cpp deleted file mode 100644 index 253b696da..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/CuptiCallbackApiTest.cpp +++ /dev/null @@ -1,239 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "src/Logger.h" -#include "src/CuptiCallbackApi.h" - -#include -#include -#include -#include - -using namespace std::chrono; -using namespace KINETO_NAMESPACE; -using namespace libkineto; - -const size_t some_data = 42; - -std::atomic simple_cb_calls = 0; - -void simple_cb( - CUpti_CallbackDomain domain, - CUpti_CallbackId cbid, - const CUpti_CallbackData* cbInfo) { - - // simple arg check - EXPECT_EQ(domain, CUPTI_CB_DOMAIN_RUNTIME_API); - EXPECT_EQ(cbid, CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000); - EXPECT_EQ(*reinterpret_cast(cbInfo), some_data); - - simple_cb_calls++; -} - -void atomic_cb( - CUpti_CallbackDomain /*domain*/, - CUpti_CallbackId /*cbid*/, - const CUpti_CallbackData* /*cbInfo)*/) { - // do some atomics in a loop - for (int i = 0; i < 1000; i++) { - // would have used release consistency but this is fine - simple_cb_calls++; - } -} - -void empty_cb( - CUpti_CallbackDomain /*domain*/, - CUpti_CallbackId /*cbid*/, - const CUpti_CallbackData* /*cbInfo*/) { -} - -TEST(CuptiCallbackApiTest, SimpleTest) { - auto& api = CuptiCallbackApi::singleton(); - - auto addSimpleCallback = [&]() -> bool { - bool ret = api.registerCallback( - CUPTI_CB_DOMAIN_RUNTIME_API, - CuptiCallbackApi::CUDA_LAUNCH_KERNEL, - &simple_cb - ); - return ret; - }; - EXPECT_TRUE(addSimpleCallback()) << "Failed to add callback"; - - // duplicate add should be okay - EXPECT_TRUE(addSimpleCallback()) << "Failed to re-add callback"; - - simple_cb_calls = 0; - - // simulate callback - api.__callback_switchboard( - CUPTI_CB_DOMAIN_RUNTIME_API, - CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000, - reinterpret_cast(&some_data)); - - EXPECT_EQ(simple_cb_calls, 1); - - bool ret = api.deleteCallback( - CUPTI_CB_DOMAIN_RUNTIME_API, - CuptiCallbackApi::CUDA_LAUNCH_KERNEL, - &simple_cb - ); - - EXPECT_TRUE(ret) << "Failed to remove callback"; - - ret = api.deleteCallback( - CUPTI_CB_DOMAIN_RUNTIME_API, - CuptiCallbackApi::CUDA_LAUNCH_KERNEL, - &atomic_cb - ); - - EXPECT_FALSE(ret) << "oops! deleted a callback that was never added"; -} - -TEST(CuptiCallbackApiTest, AllCallbacks) { - auto& api = CuptiCallbackApi::singleton(); - - auto testCallback = [&]( - CUpti_CallbackDomain domain, - CUpti_CallbackId cbid, - CuptiCallbackApi::CuptiCallBackID kineto_cbid) -> bool { - - bool ret = api.registerCallback(domain, kineto_cbid, atomic_cb); - EXPECT_TRUE(ret) << "Failed to add callback"; - - if (!ret) { - return false; - } - - simple_cb_calls = 0; - api.__callback_switchboard(domain, cbid, nullptr); - EXPECT_EQ(simple_cb_calls, 1000); - ret = simple_cb_calls == 1000; - - EXPECT_TRUE(api.deleteCallback(domain, kineto_cbid, atomic_cb)); - - return ret; - }; - - EXPECT_TRUE( - testCallback( - CUPTI_CB_DOMAIN_RESOURCE, - CUPTI_CBID_RESOURCE_CONTEXT_CREATED, - CuptiCallbackApi::RESOURCE_CONTEXT_CREATED)) - << "Failed to run callback for RESOURCE_CONTEXT_CREATED"; - - EXPECT_TRUE( - testCallback( - CUPTI_CB_DOMAIN_RESOURCE, - CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING, - CuptiCallbackApi::RESOURCE_CONTEXT_DESTROYED)) - << "Failed to run callback for RESOURCE_CONTEXT_DESTROYED"; - - EXPECT_TRUE( - testCallback( - CUPTI_CB_DOMAIN_RUNTIME_API, - CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000, - CuptiCallbackApi::CUDA_LAUNCH_KERNEL)) - << "Failed to run callback for CUDA_LAUNCH_KERNEL"; - -} - -TEST(CuptiCallbackApiTest, ContentionTest) { - auto& api = CuptiCallbackApi::singleton(); - const CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RUNTIME_API; - const CUpti_CallbackId cbid = CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000; - const CuptiCallbackApi::CuptiCallBackID kineto_cbid = - CuptiCallbackApi::CUDA_LAUNCH_KERNEL; - - bool ret = api.registerCallback(domain, kineto_cbid, empty_cb); - EXPECT_TRUE(ret) << "Failed to add callback"; - - const int iters = 10000; - const int num_readers = 8; - - simple_cb_calls = 0; - - // simulate callbacks being executed on multiple threads in parallel - // during this interval add a new atomic_callback. - // this test ensured mutual exclusion is working fine - auto read_fn = [&](int tid){ - auto start_ts = high_resolution_clock::now(); - for (int i = 0; i < iters; i++) { - api.__callback_switchboard(domain, cbid, nullptr); - } - auto runtime_ms = duration_cast( - high_resolution_clock::now() - start_ts); - LOG(INFO) << "th " << tid << " done in " << runtime_ms.count() << " ms"; - }; - - - std::vector read_ths; - for (int i = 0; i< num_readers; i++) { - read_ths.emplace_back(read_fn, i); - } - - ret = api.registerCallback(domain, kineto_cbid, atomic_cb); - EXPECT_TRUE(ret) << "Failed to add callback"; - - for (auto& t : read_ths) { - t.join(); - } - - //EXPECT_GT(simple_cb_calls, 0) - // << "Atomic callback should have been called at least once."; - - api.deleteCallback(domain, kineto_cbid, empty_cb); - api.deleteCallback(domain, kineto_cbid, atomic_cb); -} - -TEST(CuptiCallbackApiTest, Bechmark) { - - constexpr int iters = 1000; - // atomic bench a number of times to get a baseline - - const CUpti_CallbackDomain domain = CUPTI_CB_DOMAIN_RUNTIME_API; - const CUpti_CallbackId cbid = CUPTI_RUNTIME_TRACE_CBID_cudaLaunchKernel_v7000; - const CuptiCallbackApi::CuptiCallBackID kineto_cbid = - CuptiCallbackApi::CUDA_LAUNCH_KERNEL; - - LOG(INFO) << "Iteration count = " << iters; - - const bool use_empty = true; - auto cbfn = use_empty ? &empty_cb : &atomic_cb; - - // warmup - for (int i = 0; i < 50; i++) { - (*cbfn)(domain, cbid, nullptr); - } - - auto start_ts = high_resolution_clock::now(); - for (int i = 0; i < iters; i++) { - (*cbfn)(domain, cbid, nullptr); - } - auto delta_baseline_ns = duration_cast( - high_resolution_clock::now() - start_ts); - LOG(INFO) << "Baseline runtime = " << delta_baseline_ns.count() << " ns"; - - - auto& api = CuptiCallbackApi::singleton(); - bool ret = api.registerCallback(domain, kineto_cbid, cbfn); - EXPECT_TRUE(ret) << "Failed to add callback"; - - // warmup - for (int i = 0; i < 50; i++) { - api.__callback_switchboard(domain, cbid, nullptr); - } - - start_ts = high_resolution_clock::now(); - for (int i = 0; i < iters; i++) { - api.__callback_switchboard(domain, cbid, nullptr); - } - - auto delta_callback_ns = duration_cast( - high_resolution_clock::now() - start_ts); - LOG(INFO) << "Callback runtime = " << delta_callback_ns.count() << " ns"; - - LOG(INFO) << "Callback runtime per iteration = " << - (delta_callback_ns.count() - delta_baseline_ns.count()) / (double) iters - << " ns"; - -} diff --git a/plugins/tensorboard-plugins/libkineto/test/CuptiProfilerApiTest.cu b/plugins/tensorboard-plugins/libkineto/test/CuptiProfilerApiTest.cu deleted file mode 100644 index 54ad51b0a..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/CuptiProfilerApiTest.cu +++ /dev/null @@ -1,353 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include -#include - -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "src/Logger.h" -#include "src/CuptiRangeProfilerApi.h" - -#define DRIVER_API_CALL(apiFuncCall) \ - do { \ - CUresult _status = apiFuncCall; \ - if (_status != CUDA_SUCCESS) { \ - LOG(ERROR) << "Failed invoking CUDA driver function " \ - << #apiFuncCall << " status = " \ - << _status; \ - exit(-1); \ - } \ - } while (0) - -#define EXPECT(expr)\ - if (!(expr)) {\ - }; - -using namespace KINETO_NAMESPACE; - -static int numRanges = 1; - -using Type = double; - -// Device code -__global__ void VecAdd(const Type* A, const Type* B, Type* C, int N) { - int i = blockDim.x * blockIdx.x + threadIdx.x; - if (i < N) { - C[i] = A[i] + B[i]; - } -} - -// Device code -__global__ void VecSub(const Type* A, const Type* B, Type* C, int N) { - int i = blockDim.x * blockIdx.x + threadIdx.x; - if (i < N) { - C[i] = A[i] - B[i]; - } -} - -static void initVec(Type* vec, int n) { - for (int i = 0; i < n; i++) { - vec[i] = i; - } -} - -static void cleanUp( - Type* h_A, - Type* h_B, - Type* h_C, - Type* h_D, - Type* d_A, - Type* d_B, - Type* d_C, - Type* d_D) { - if (d_A) - cudaFree(d_A); - if (d_B) - cudaFree(d_B); - if (d_C) - cudaFree(d_C); - if (d_D) - cudaFree(d_D); - - // Free host memory - if (h_A) - free(h_A); - if (h_B) - free(h_B); - if (h_C) - free(h_C); - if (h_D) - free(h_D); -} - -/* Benchmark application used to test profiler measurements - * This simply runs two kernels vector Add and Vector Subtract - */ - -void VectorAddSubtract() { - int N = 50000; - size_t size = N * sizeof(Type); - int threadsPerBlock = 0; - int blocksPerGrid = 0; - Type *h_A, *h_B, *h_C, *h_D; - Type *d_A, *d_B, *d_C, *d_D; - int i; - Type sum, diff; - - // Allocate input vectors h_A and h_B in host memory - h_A = (Type*)malloc(size); - h_B = (Type*)malloc(size); - h_C = (Type*)malloc(size); - h_D = (Type*)malloc(size); - - // Initialize input vectors - initVec(h_A, N); - initVec(h_B, N); - memset(h_C, 0, size); - memset(h_D, 0, size); - - // Allocate vectors in device memory - cudaMalloc((void**)&d_A, size); - cudaMalloc((void**)&d_B, size); - cudaMalloc((void**)&d_C, size); - cudaMalloc((void**)&d_D, size); - - // Copy vectors from host memory to device memory - cudaMemcpy(d_A, h_A, size, cudaMemcpyHostToDevice); - cudaMemcpy(d_B, h_B, size, cudaMemcpyHostToDevice); - - // Invoke kernel - threadsPerBlock = 256; - blocksPerGrid = (N + threadsPerBlock - 1) / threadsPerBlock; - LOG(INFO) << fmt::format( - "Launching kernel: blocks {}, thread/block {}", - blocksPerGrid, - threadsPerBlock); - - VecAdd<<>>(d_A, d_B, d_C, N); - - VecSub<<>>(d_A, d_B, d_D, N); - - // Copy result from device memory to host memory - // h_C contains the result in host memory - cudaMemcpy(h_C, d_C, size, cudaMemcpyDeviceToHost); - cudaMemcpy(h_D, d_D, size, cudaMemcpyDeviceToHost); - - // Verify result - for (i = 0; i < N; ++i) { - sum = h_A[i] + h_B[i]; - diff = h_A[i] - h_B[i]; - if (h_C[i] != sum || h_D[i] != diff) { - LOG(ERROR) << "Result verification failed"; - break; - } - } - - cleanUp(h_A, h_B, h_C, h_D, d_A, d_B, d_C, d_D); -} - -#if HAS_CUPTI_RANGE_PROFILER -bool runTestWithAutoRange( - int deviceNum, - const std::vector& metricNames, - CUcontext cuContext, - bool async) { - - // create a CUPTI range based profiling profiler - // this configures the counter data as well - CuptiRBProfilerSession profiler( - metricNames, deviceNum, 2, 1, async ? nullptr : cuContext); - - CUpti_ProfilerRange profilerRange = CUPTI_AutoRange; - CUpti_ProfilerReplayMode profilerReplayMode = CUPTI_KernelReplay; - - if (async) { - profiler.asyncStartAndEnable(profilerRange, profilerReplayMode); - } else { - profiler.start(profilerRange, profilerReplayMode); - profiler.enable(); - } - - VectorAddSubtract(); - - if (!async) { - profiler.disable(); - // stop profiler - profiler.stop(); - } else { - profiler.asyncDisableAndStop(); - } - - auto result = profiler.evaluateMetrics(true); - - // check results - EXPECT_EQ(result.metricNames.size(), 3); - EXPECT_EQ(result.rangeVals.size(), 2); - - for (const auto& measurement : result.rangeVals) { - EXPECT_EQ(measurement.values.size(), 3); - - if (measurement.values.size() == 3) { - // smsp__warps_launched.avg - EXPECT_NE(measurement.values[0], 0); - // smsp__sass_thread_inst_executed_op_dadd_pred_on.sum - // each kernel has 50000 dadd ops - EXPECT_EQ(measurement.values[1], 50000); - // sm__inst_executed_pipe_tensor.sum - //EXPECT_EQ(measurement.values[2], 0); - } - } - return true; -} - -bool runTestWithUserRange( - int deviceNum, - const std::vector& metricNames, - CUcontext cuContext, - bool async = false) { - - // create a CUPTI range based profiling profiler - // this configures the counter data as well - CuptiRBProfilerSession profiler( - metricNames, deviceNum, numRanges, 1, async ? nullptr : cuContext); - - CUpti_ProfilerRange profilerRange = CUPTI_UserRange; - CUpti_ProfilerReplayMode profilerReplayMode = CUPTI_UserReplay; - - if (async) { - profiler.asyncStartAndEnable(profilerRange, profilerReplayMode); - { VectorAddSubtract(); } - profiler.disableAndStop(); - } else { - profiler.start(profilerRange, profilerReplayMode); - - /* User takes the resposiblity of replaying the kernel launches */ - bool replay = true; - do { - profiler.beginPass(); - { - profiler.enable(); - - std::string rangeName = "vecAddSub"; - profiler.pushRange(rangeName); - - { VectorAddSubtract(); } - - profiler.popRange(); - profiler.disable(); - } - LOG(INFO) << "Replay starting."; - replay = profiler.endPass(); - - } while (!replay); - - // stop profiler - profiler.stop(); - } - VectorAddSubtract(); - auto result = profiler.evaluateMetrics(true); - - // check results - EXPECT_EQ(result.metricNames.size(), 3); - EXPECT_EQ(result.rangeVals.size(), 1); - - if (result.rangeVals.size() > 0) { - const auto& measurement = result.rangeVals[0]; - EXPECT_EQ(measurement.values.size(), 3); - - if (measurement.values.size() == 3) { - // smsp__warps_launched.avg - EXPECT_NE(measurement.values[0], 0); - // smsp__sass_thread_inst_executed_op_dadd_pred_on.sum - // in async mode multiple passes are not supported yet - if (!async) { - EXPECT_EQ(measurement.values[1], 100000); - } - // sm__inst_executed_pipe_tensor.sum - //EXPECT_EQ(measurement.values[2], 0); - } - } - return true; -} -#endif // HAS_CUPTI_RANGE_PROFILER - -int main(int argc, char* argv[]) { - - CUdevice cuDevice; - - int deviceCount, deviceNum; - int computeCapabilityMajor = 0, computeCapabilityMinor = 0; - - printf("Usage: %s [device_num]\n", argv[0]); - - DRIVER_API_CALL(cuInit(0)); - DRIVER_API_CALL(cuDeviceGetCount(&deviceCount)); - - if (deviceCount == 0) { - LOG(ERROR) << "There is no device supporting CUDA."; - return -2; - } - - if (argc > 1) - deviceNum = atoi(argv[1]); - else - deviceNum = 0; - LOG(INFO) << "CUDA Device Number: " << deviceNum; - - DRIVER_API_CALL(cuDeviceGet(&cuDevice, deviceNum)); - DRIVER_API_CALL(cuDeviceGetAttribute( - &computeCapabilityMajor, - CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, - cuDevice)); - DRIVER_API_CALL(cuDeviceGetAttribute( - &computeCapabilityMinor, - CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, - cuDevice)); - - LOG(INFO) << "Compute Cabapbility = " - << fmt::format("{},{}",computeCapabilityMajor, computeCapabilityMinor); - - if (computeCapabilityMajor < 7) { - LOG(ERROR) << "CUPTI Profiler is not supported with compute capability < 7.0"; - return -2; - } - - CuptiRBProfilerSession::staticInit(); - - // metrics to profile - std::vector metricNames = { - "smsp__warps_launched.avg", - "smsp__sass_thread_inst_executed_op_dadd_pred_on.sum", - "sm__inst_executed_pipe_tensor.sum", - }; - - CUcontext cuContext; - DRIVER_API_CALL(cuCtxCreate(&cuContext, 0, cuDevice)); - - VectorAddSubtract(); - -#if HAS_CUPTI_RANGE_PROFILER - CuptiRBProfilerSession::staticInit(); - - if (!runTestWithUserRange(deviceNum, metricNames, cuContext, false)) { - LOG(ERROR) << "Failed to profiler test benchmark in user range"; - } else if (!runTestWithAutoRange(deviceNum, metricNames, cuContext, false)) { - LOG(ERROR) << "Failed to profiler test benchmark in auto range"; - } else if (!runTestWithUserRange(deviceNum, metricNames, cuContext, true)) { - LOG(ERROR) << "Failed to profiler test benchmark in user range async"; - } else if (!runTestWithAutoRange(deviceNum, metricNames, cuContext, true)) { - LOG(ERROR) << "Failed to profiler test benchmark in auto range async"; - } - - CuptiRBProfilerSession::deInitCupti(); -#else - LOG(WARNING) << "CuptiRBProfilerSession is not supported."; -#endif // HAS_CUPTI_RANGE_PROFILER - DRIVER_API_CALL(cuCtxDestroy(cuContext)); - - - return 0; -} diff --git a/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerApiTest.cpp b/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerApiTest.cpp deleted file mode 100644 index 28cad722c..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerApiTest.cpp +++ /dev/null @@ -1,113 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include -#include - -#include "include/libkineto.h" -#include "include/Config.h" -#include "src/CuptiRangeProfilerApi.h" - -#include "src/Logger.h" -#include "test/CuptiRangeProfilerTestUtil.h" - -using namespace KINETO_NAMESPACE; - -#if HAS_CUPTI_PROFILER - -TEST(CuptiRangeProfilerApiTest, contextTracking) { - std::vector log_modules( - {"CuptiRangeProfilerApi.cpp"}); - SET_LOG_VERBOSITY_LEVEL(1, log_modules); - - std::array data; - std::array contexts; - for (int i = 0; i < data.size(); i++) { - contexts[i] = reinterpret_cast(&data[i]); - } - - // simulate creating contexts, this calls the trackCudaContexts - // function that would otherwise be called via a callback - uint32_t dev = 0; - for (auto ctx : contexts) { - simulateCudaContextCreate(ctx, dev++); - } - - EXPECT_EQ( - CuptiRBProfilerSession::getActiveDevices(), - std::set({0, 1, 2})); - - simulateCudaContextDestroy(contexts[1], 1); - - EXPECT_EQ( - CuptiRBProfilerSession::getActiveDevices(), - std::set({0, 2})); - - simulateCudaContextDestroy(contexts[0], 0); - simulateCudaContextDestroy(contexts[2], 2); - - EXPECT_TRUE( - CuptiRBProfilerSession::getActiveDevices().empty()); -} - -TEST(CuptiRangeProfilerApiTest, asyncLaunchUserRange) { - std::vector log_modules( - {"CuptiRangeProfilerApi.cpp"}); - SET_LOG_VERBOSITY_LEVEL(1, log_modules); - - // this is bad but the pointer is never accessed - CUcontext ctx0 = reinterpret_cast(10); - simulateCudaContextCreate(ctx0, 0 /*device_id*/); - - auto session = std::make_unique(0, ctx0); - session->asyncStartAndEnable(CUPTI_UserRange, CUPTI_UserReplay); - - simulateKernelLaunch(ctx0, "hello"); - simulateKernelLaunch(ctx0, "foo"); - simulateKernelLaunch(ctx0, "bar"); - - session->asyncDisableAndStop(); - // stop happens after next kernel is run - simulateKernelLaunch(ctx0, "bar"); - simulateCudaContextDestroy(ctx0, 0 /*device_id*/); - - EXPECT_EQ(session->passes_ended, 1); - EXPECT_EQ(session->ranges_ended, 1); - EXPECT_TRUE(session->enabled); -} - -TEST(CuptiRangeProfilerApiTest, asyncLaunchAutoRange) { - std::vector log_modules( - {"CuptiRangeProfilerApi.cpp"}); - SET_LOG_VERBOSITY_LEVEL(1, log_modules); - - // this is bad but the pointer is never accessed - CUcontext ctx0 = reinterpret_cast(10); - CUcontext ctx1 = reinterpret_cast(11); - - simulateCudaContextCreate(ctx0, 0 /*device_id*/); - - auto session = std::make_unique(0, ctx0); - session->asyncStartAndEnable(CUPTI_AutoRange, CUPTI_KernelReplay); - - simulateKernelLaunch(ctx0, "hello"); - simulateKernelLaunch(ctx0, "foo"); - simulateKernelLaunch(ctx1, "kernel_on_different_device"); - simulateKernelLaunch(ctx0, "bar"); - - session->asyncDisableAndStop(); - // stop happens after next kernel is run - simulateKernelLaunch(ctx0, "bar"); - simulateCudaContextDestroy(ctx0, 0 /*device_id*/); - - EXPECT_EQ(session->passes_ended, 0); - EXPECT_EQ(session->ranges_ended, 0); - EXPECT_TRUE(session->enabled); - - EXPECT_EQ( - session->getKernelNames(), - std::vector({"hello", "foo", "bar"})) - << "Kernel names were not tracked"; -} - -#endif // HAS_CUPTI_PROFILER diff --git a/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerConfigTest.cpp b/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerConfigTest.cpp deleted file mode 100644 index 3f5689682..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerConfigTest.cpp +++ /dev/null @@ -1,67 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "include/Config.h" -#include "src/CuptiRangeProfilerConfig.h" - -#include -#include -#include -#include - -using namespace std::chrono; -using namespace KINETO_NAMESPACE; - -class CuptiRangeProfilerConfigTest : public ::testing::Test { - protected: - void SetUp() override { - CuptiRangeProfilerConfig::registerFactory(); - } -}; - -TEST_F(CuptiRangeProfilerConfigTest, ConfigureProfiler) { - Config cfg; - std::vector metrics = { - "kineto__cuda_core_flops", - "sm__inst_executed.sum", - "l1tex__data_bank_conflicts_pipe_lsu.sum", - }; - auto metricsConfigStr = - fmt::format("CUPTI_PROFILER_METRICS = {}", fmt::join(metrics, ",")); - - EXPECT_TRUE(cfg.parse(metricsConfigStr)); - EXPECT_TRUE(cfg.parse("CUPTI_PROFILER_ENABLE_PER_KERNEL = true")); - EXPECT_TRUE(cfg.parse("CUPTI_PROFILER_MAX_RANGES = 42")); - - const CuptiRangeProfilerConfig& cupti_cfg = - CuptiRangeProfilerConfig::get(cfg); - - EXPECT_EQ(cupti_cfg.activitiesCuptiMetrics(), metrics); - EXPECT_EQ(cupti_cfg.cuptiProfilerPerKernel(), true); - EXPECT_EQ(cupti_cfg.cuptiProfilerMaxRanges(), 42); - -} - -TEST_F(CuptiRangeProfilerConfigTest, RangesDefaults) { - Config cfg, cfg_auto; - - // do not set max ranges in config, check defaults are sane - EXPECT_TRUE(cfg.parse("CUPTI_PROFILER_METRICS = kineto__cuda_core_flops")); - EXPECT_TRUE(cfg.parse("CUPTI_PROFILER_ENABLE_PER_KERNEL = false")); - - cfg.setSignalDefaults(); - - EXPECT_TRUE(cfg_auto.parse("CUPTI_PROFILER_METRICS = kineto__cuda_core_flops")); - EXPECT_TRUE(cfg_auto.parse("CUPTI_PROFILER_ENABLE_PER_KERNEL = true")); - - cfg_auto.setClientDefaults(); - - int user_ranges, auto_ranges; - - user_ranges = CuptiRangeProfilerConfig::get(cfg).cuptiProfilerMaxRanges(); - auto_ranges = CuptiRangeProfilerConfig::get(cfg_auto).cuptiProfilerMaxRanges(); - - EXPECT_GE(user_ranges, 1) << " in user range mode default to at least 1 ranges"; - EXPECT_GE(auto_ranges, 1000) << " in auto range mode default to at least 1000 ranges"; - - EXPECT_GT(auto_ranges, user_ranges); -} diff --git a/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerTestUtil.h b/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerTestUtil.h deleted file mode 100644 index 861b65fd7..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/CuptiRangeProfilerTestUtil.h +++ /dev/null @@ -1,96 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "CuptiRangeProfilerApi.h" - -namespace KINETO_NAMESPACE { - -#if HAS_CUPTI_PROFILER - -class MockCuptiRBProfilerSession : public CuptiRBProfilerSession { - public: - MockCuptiRBProfilerSession(int deviceId, CUcontext ctx) - : CuptiRBProfilerSession(deviceId, ctx) {} - - void beginPass() override { - LOG(INFO) << " Mock CUPTI begin pass"; - passes_started++; - } - - bool endPass() override { - passes_ended++; - return true; - } - - void flushCounterData() override {} - - void pushRange(const std::string& rangeName) override { - LOG(INFO) << " Mock CUPTI pushrange ( " << rangeName << " )"; - ranges_started++; - } - - void popRange() override { - LOG(INFO) << " Mock CUPTI poprange"; - ranges_ended++; - } - - void stop() override { - runChecks(); - } - - void enable() override { - enabled = true; - } - void disable() override {} - - CuptiProfilerResult evaluateMetrics(bool /*verbose*/) override { - return result; - } - -protected: - void startInternal( - CUpti_ProfilerRange profilerRange, - CUpti_ProfilerReplayMode profilerReplayMode) override { - curRange_ = profilerRange; - curReplay_ = profilerReplayMode; - } - -private: - void runChecks() { - EXPECT_EQ(passes_started, passes_ended); - EXPECT_EQ(ranges_started, ranges_ended); - } - - public: - int passes_started = 0; - int passes_ended = 0; - int ranges_started = 0; - int ranges_ended = 0; - bool enabled = false; - - CuptiProfilerResult result; - -}; - -inline void simulateCudaContextCreate(CUcontext context, uint32_t dev) { - testing::trackCudaCtx( - context, dev, CUPTI_CBID_RESOURCE_CONTEXT_CREATED); -} - -inline void simulateCudaContextDestroy(CUcontext context, uint32_t dev) { - testing::trackCudaCtx( - context, dev, CUPTI_CBID_RESOURCE_CONTEXT_DESTROY_STARTING); -} - -inline void simulateKernelLaunch( - CUcontext context, const std::string& kernelName) { - testing::trackCudaKernelLaunch(context, kernelName.c_str()); -} - -#endif // HAS_CUPTI_PROFILER - -} // namespace KINETO_NAMESPACE diff --git a/plugins/tensorboard-plugins/libkineto/test/CuptiStringsTest.cpp b/plugins/tensorboard-plugins/libkineto/test/CuptiStringsTest.cpp deleted file mode 100644 index 405f9404a..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/CuptiStringsTest.cpp +++ /dev/null @@ -1,29 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include - -#include "src/cupti_strings.h" - -using namespace KINETO_NAMESPACE; - -TEST(CuptiStringsTest, Valid) { - ASSERT_STREQ( - runtimeCbidName(CUPTI_RUNTIME_TRACE_CBID_INVALID), "INVALID"); - ASSERT_STREQ( - runtimeCbidName(CUPTI_RUNTIME_TRACE_CBID_cudaDriverGetVersion_v3020), - "cudaDriverGetVersion"); - ASSERT_STREQ(runtimeCbidName - (CUPTI_RUNTIME_TRACE_CBID_cudaDeviceSynchronize_v3020), - "cudaDeviceSynchronize"); - ASSERT_STREQ( - runtimeCbidName(CUPTI_RUNTIME_TRACE_CBID_cudaStreamSetAttribute_ptsz_v11000), - "cudaStreamSetAttribute_ptsz"); -} - -TEST(CuptiStringsTest, Invalid) { - ASSERT_STREQ(runtimeCbidName(-1), "INVALID"); - // We can't actually use CUPTI_RUNTIME_TRACE_CBID_SIZE here until we - // auto-generate the string table, since it may have more entries than - // the enum in the version used to compile. - ASSERT_STREQ(runtimeCbidName(1000), "INVALID"); -} diff --git a/plugins/tensorboard-plugins/libkineto/test/EventProfilerTest.cpp b/plugins/tensorboard-plugins/libkineto/test/EventProfilerTest.cpp deleted file mode 100644 index cb36c826a..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/EventProfilerTest.cpp +++ /dev/null @@ -1,578 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "src/EventProfiler.h" - -#include -#include -#include - -using namespace std::chrono; -using namespace KINETO_NAMESPACE; - -TEST(PercentileTest, Create) { - PercentileList pct = {{10, SampleValue(0)}, - {49, SampleValue(0)}, - {50, SampleValue(0)}, - {90, SampleValue(0)}}; - - percentiles({0, 10, 20, 30, 40, 50, 60, 70, 80, 90, 100}, pct); - EXPECT_EQ(pct[0].second.getInt(), 10); - EXPECT_EQ(pct[1].second.getInt(), 50); - EXPECT_EQ(pct[2].second.getInt(), 50); - EXPECT_EQ(pct[3].second.getInt(), 90); - - percentiles({80, 10, 20, 70, 60, 40, 90, 30, 50, 0, 100}, pct); - EXPECT_EQ(pct[0].second.getInt(), 10); - EXPECT_EQ(pct[1].second.getInt(), 50); - EXPECT_EQ(pct[2].second.getInt(), 50); - EXPECT_EQ(pct[3].second.getInt(), 90); - - percentiles({80}, pct); - EXPECT_EQ(pct[0].second.getInt(), 80); - EXPECT_EQ(pct[1].second.getInt(), 80); - EXPECT_EQ(pct[2].second.getInt(), 80); - EXPECT_EQ(pct[3].second.getInt(), 80); - - percentiles({80, 50}, pct); - EXPECT_EQ(pct[0].second.getInt(), 50); - EXPECT_EQ(pct[1].second.getInt(), 50); - EXPECT_EQ(pct[2].second.getInt(), 80); - EXPECT_EQ(pct[3].second.getInt(), 80); -} - -TEST(PercentileTest, Normalize) { - PercentileList pct = { - {10, SampleValue(10)}, {50, SampleValue(100.0)}, {90, SampleValue(2000)}}; - - normalize(pct, 2.5); - - EXPECT_EQ(pct[0].second.getInt(), 25); - EXPECT_EQ((int)pct[1].second.getDouble(), 250); - EXPECT_EQ(pct[2].second.getInt(), 5000); -} - -TEST(EventTest, SumSamples) { - Event ev; - ev.instanceCount = 4; - auto t = system_clock::now(); - ev.addSample(t, {1, 2, 3, 4}); - ev.addSample(t, {10, 20, 30, 40}); - ev.addSample(t, {100, 200, 300, 400}); - - EXPECT_EQ(ev.sumInstance(0, {0, 0, 3}), 1); - EXPECT_EQ(ev.sumInstance(0, {0, 1, 3}), 10); - EXPECT_EQ(ev.sumInstance(0, {0, 2, 3}), 100); - - EXPECT_EQ(ev.sumInstance(0, {0, 0, 1}), 111); - - EXPECT_EQ(ev.sumInstance(3, {0, 0, 1}), 444); - - // Non-zero offset - EXPECT_EQ(ev.sumInstance(0, {1, 0, 2}), 10); - EXPECT_EQ(ev.sumInstance(0, {1, 1, 2}), 100); - EXPECT_EQ(ev.sumInstance(0, {1, 0, 1}), 110); - - ev.addSample(t, {1000, 2000, 3000, 4000}); - - EXPECT_EQ(ev.sumInstance(0, {1, 0, 3}), 10); - EXPECT_EQ(ev.sumInstance(0, {1, 1, 3}), 100); - EXPECT_EQ(ev.sumInstance(0, {2, 1, 2}), 1000); - EXPECT_EQ(ev.sumInstance(0, {2, 0, 1}), 1100); - - EXPECT_EQ(ev.sumAll({0, 0, 4}), 10); - EXPECT_EQ(ev.sumAll({1, 0, 3}), 100); - EXPECT_EQ(ev.sumAll({2, 1, 2}), 10000); - EXPECT_EQ(ev.sumAll({0, 1, 2}), 11000); - EXPECT_EQ(ev.sumAll({0, 0, 1}), 11110); -} - -TEST(EventTest, Percentiles) { - Event ev; - ev.instanceCount = 4; - auto t = system_clock::now(); - ev.addSample(t, {3, 2, 1, 4}); - ev.addSample(t, {30, 20, 10, 40}); - ev.addSample(t, {300, 200, 100, 400}); - - PercentileList pct = { - {10, SampleValue(0)}, {50, SampleValue(0)}, {90, SampleValue(0)}}; - - ev.percentiles(pct, {0, 0, 3}); - EXPECT_EQ(pct[0].second.getInt(), 1); - EXPECT_EQ(pct[1].second.getInt(), 3); - EXPECT_EQ(pct[2].second.getInt(), 4); - - ev.percentiles(pct, {0, 0, 1}); - EXPECT_EQ(pct[0].second.getInt(), 111); - EXPECT_EQ(pct[1].second.getInt(), 333); - EXPECT_EQ(pct[2].second.getInt(), 444); -} - -class MockCuptiMetrics : public CuptiMetricApi { - public: - MockCuptiMetrics() : CuptiMetricApi(0) {} - MOCK_METHOD1(idFromName, CUpti_MetricID(const std::string& name)); - MOCK_METHOD1( - events, - std::map(CUpti_MetricID metric_id)); - MOCK_METHOD1(valueKind, CUpti_MetricValueKind(CUpti_MetricID metric)); - MOCK_METHOD1( - evaluationMode, - CUpti_MetricEvaluationMode(CUpti_MetricID metric)); - MOCK_METHOD5( - calculate, - SampleValue( - CUpti_MetricID metric, - CUpti_MetricValueKind kind, - std::vector& events, - std::vector& values, - int64_t duration)); -}; - -TEST(MetricTest, Calculate) { - using ::testing::Return; - MockCuptiMetrics metrics; - - // The events used for the ipc metrics: instructions and cycles - // Pretend we have 2 SMs and 2 samples of each event - Event instr("instructions"); - instr.instanceCount = 2; - auto t = system_clock::now(); - instr.addSample(t, {100, 200}); - instr.addSample(t, {300, 400}); - - Event cycles("cycles"); - cycles.instanceCount = 2; - cycles.addSample(t, {1000, 1200}); - cycles.addSample(t, {1300, 1300}); - - // 2 & 3 are the event ids we specified in the metric - std::map events; - events[2] = std::move(instr); - events[3] = std::move(cycles); - - // Define an ipc metric - EXPECT_CALL(metrics, valueKind(1)) - .Times(1) - .WillOnce(Return(CUPTI_METRIC_VALUE_KIND_DOUBLE)); - Metric m( - "ipc", 1, {2, 3}, CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE, metrics); - - // Calculate metric for first sample - // Since evaluation mode is CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE, - // Cupti API will be called three times: once for each SM (2) and once - // to get the total across SMs. - std::vector ids = {2, 3}; - std::vector vals = {100, 1000}; - EXPECT_CALL( - metrics, calculate(1, CUPTI_METRIC_VALUE_KIND_DOUBLE, ids, vals, 1000)) - .Times(1) - .WillOnce(Return(SampleValue(0.1))); - vals = {200, 1200}; - EXPECT_CALL( - metrics, calculate(1, CUPTI_METRIC_VALUE_KIND_DOUBLE, ids, vals, 1000)) - .Times(1) - .WillOnce(Return(SampleValue(0.17))); - vals = {300, 2200}; - EXPECT_CALL( - metrics, calculate(1, CUPTI_METRIC_VALUE_KIND_DOUBLE, ids, vals, 1000)) - .Times(1) - .WillOnce(Return(SampleValue(0.14))); - auto v = m.calculate(events, nanoseconds(1000), {0, 0, 2}); - - EXPECT_EQ(v.perInstance.size(), 2); - EXPECT_EQ(v.perInstance[0].getDouble(), 0.1); - EXPECT_EQ(v.perInstance[1].getDouble(), 0.17); - EXPECT_EQ(v.total.getDouble(), 0.14); - - // Calculate second sample. - // Change evaluation mode to CUPTI_METRIC_EVALUATION_MODE_AGGREGATE. - // Now we should get only one call to the Cupti API for the total. - EXPECT_CALL(metrics, valueKind(1)) - .Times(1) - .WillOnce(Return(CUPTI_METRIC_VALUE_KIND_DOUBLE)); - Metric m2("ipc", 1, {2, 3}, CUPTI_METRIC_EVALUATION_MODE_AGGREGATE, metrics); - vals = {700, 2600}; - EXPECT_CALL( - metrics, calculate(1, CUPTI_METRIC_VALUE_KIND_DOUBLE, ids, vals, 1000)) - .Times(1) - .WillOnce(Return(SampleValue(0.27))); - v = m2.calculate(events, nanoseconds(1000), {0, 1, 2}); - - EXPECT_EQ(v.perInstance.size(), 1); - EXPECT_EQ(v.perInstance[0].getDouble(), 0.27); - EXPECT_EQ(v.total.getDouble(), 0.27); -} - -class MockCuptiEvents : public CuptiEventApi { - public: - MOCK_METHOD1( - createGroupSets, - CUpti_EventGroupSets*(std::vector& ids)); - MOCK_METHOD1(destroyGroupSets, void(CUpti_EventGroupSets* sets)); - MOCK_METHOD0(setContinuousMode, bool()); - MOCK_METHOD1(enablePerInstance, void(CUpti_EventGroup eventGroup)); - MOCK_METHOD1(instanceCount, uint32_t(CUpti_EventGroup eventGroup)); - MOCK_METHOD1(enableGroupSet, void(CUpti_EventGroupSet& set)); - MOCK_METHOD1(disableGroupSet, void(CUpti_EventGroupSet& set)); - MOCK_METHOD3( - readEvent, - void(CUpti_EventGroup g, CUpti_EventID id, std::vector& vals)); - MOCK_METHOD1(eventsInGroup, std::vector(CUpti_EventGroup g)); - MOCK_METHOD1(eventId, CUpti_EventID(const std::string& name)); -}; - -TEST(EventGroupSetTest, CollectSample) { - using ::testing::_; - using ::testing::Return; - using ::testing::SetArgPointee; - const CUpti_EventGroup g1{nullptr}; - const CUpti_EventGroup g2{reinterpret_cast(0x1000)}; - CUpti_EventGroup groups[] = {g1, g2}; - CUpti_EventGroupSet set; - set.eventGroups = groups; - set.numEventGroups = 2; - - std::map events; - Event instr("instructions"); - events[4] = std::move(instr); - Event cycles("cycles"); - events[5] = std::move(cycles); - Event branches("branches"); - events[10] = std::move(branches); - - MockCuptiEvents cupti_events; - EXPECT_CALL(cupti_events, enablePerInstance(g1)).Times(1); - EXPECT_CALL(cupti_events, enablePerInstance(g2)).Times(1); - EXPECT_CALL(cupti_events, instanceCount(g1)).Times(1).WillOnce(Return(80)); - EXPECT_CALL(cupti_events, instanceCount(g2)).Times(1).WillOnce(Return(40)); - std::vector events_in_group1 = {4, 5}; - EXPECT_CALL(cupti_events, eventsInGroup(g1)) - .Times(1) - .WillOnce(Return(events_in_group1)); - std::vector events_in_group2 = {10}; - EXPECT_CALL(cupti_events, eventsInGroup(g2)) - .Times(1) - .WillOnce(Return(events_in_group2)); - EventGroupSet group_set(set, events, cupti_events); - - EXPECT_EQ(group_set.groupCount(), 2); - EXPECT_EQ(events[4].instanceCount, 80); - EXPECT_EQ(events[5].instanceCount, 80); - EXPECT_EQ(events[10].instanceCount, 40); - - // This should not cause any Cupti API action as the group - // set is already disabled - group_set.setEnabled(false); - - // Activate group set - if activated twice, only the first - // should cause cupti API to be called - EXPECT_CALL(cupti_events, enableGroupSet(_)).Times(1); - group_set.setEnabled(false); - group_set.setEnabled(true); - - EXPECT_CALL(cupti_events, eventsInGroup(g1)) - .Times(1) - .WillOnce(Return(events_in_group1)); - EXPECT_CALL(cupti_events, eventsInGroup(g2)) - .Times(1) - .WillOnce(Return(events_in_group2)); - EXPECT_CALL(cupti_events, readEvent(g1, 4, _)).Times(1); - EXPECT_CALL(cupti_events, readEvent(g1, 5, _)).Times(1); - EXPECT_CALL(cupti_events, readEvent(g2, 10, _)).Times(1); - group_set.collectSample(); - - EXPECT_EQ(events[4].sampleCount(), 1); - EXPECT_EQ(events[5].sampleCount(), 1); - EXPECT_EQ(events[10].sampleCount(), 1); -} - -class MockLogger : public SampleListener { - public: - MOCK_METHOD3(handleSample, void(int device, const Sample& sample, bool from_new_version)); - MOCK_METHOD1(update, void(const Config& config)); -}; - -class EventProfilerTest : public ::testing::Test { - protected: - void SetUp() override { - auto cupti_events_ptr = std::make_unique(); - auto cupti_metrics_ptr = std::make_unique(); - cuptiEvents_ = cupti_events_ptr.get(); - cuptiMetrics_ = cupti_metrics_ptr.get(); - loggers_.push_back(std::make_unique()); - onDemandLoggers_.push_back(std::make_unique()); - profiler_ = std::make_unique( - std::move(cupti_events_ptr), - std::move(cupti_metrics_ptr), - loggers_, - onDemandLoggers_); - - for (int i = 0; i < kEventGroupCount; i++) { - eventGroups_[i] = &eventGroups_[i]; - } - for (int i = 0; i < kGroupSetCount; i++) { - // Default size to 1 but can be changed by test - groupSet_[i].numEventGroups = 1; - // Two groups per set - groupSet_[i].eventGroups = &eventGroups_[i * 2]; - } - groupSets_.numSets = 1; - groupSets_.sets = groupSet_; - } - - MockCuptiEvents* cuptiEvents_; - MockCuptiMetrics* cuptiMetrics_; - std::vector> loggers_; - std::vector> onDemandLoggers_; - constexpr static int kEventGroupCount = 4; - constexpr static int kGroupSetCount = 2; - CUpti_EventGroup eventGroups_[kEventGroupCount]; - CUpti_EventGroupSet groupSet_[kGroupSetCount]; - CUpti_EventGroupSets groupSets_; - std::unique_ptr profiler_; -}; - -TEST_F(EventProfilerTest, ConfigureFailure) { - using namespace testing; - - // Default config has no counters enabled. - // Check that profiler remains disabled. - Config cfg; - profiler_->configure(cfg, nullptr); - - EXPECT_FALSE(profiler_->enabled()); - - // There is no event named "cycles" - // In this case the profiler should print a warning and remain disabled - bool parsed = cfg.parse("EVENTS = cycles"); - EXPECT_TRUE(parsed); - - // EventProfiler should handle exception thrown from createGroupSets - // Configuration will be applied twice - once for combined base + on-demand - // and then again falling back to base - EXPECT_CALL(*cuptiEvents_, eventId("cycles")) - .Times(2) - .WillRepeatedly(Return(0)); - std::vector ids = {0}; - EXPECT_CALL(*cuptiEvents_, createGroupSets(ids)) - .Times(2) - .WillRepeatedly(Throw( - std::system_error(EINVAL, std::generic_category(), "Event ID"))); - profiler_->configure(cfg, nullptr); - - EXPECT_FALSE(profiler_->enabled()); -} - -TEST_F(EventProfilerTest, ConfigureBase) { - using namespace testing; - - // Test normal path, simple base config - Config cfg; - bool parsed = cfg.parse("EVENTS = elapsed_cycles_sm"); - EXPECT_TRUE(parsed); - - // One valid event - expect one call to eventId and createGroupSets - EXPECT_CALL(*cuptiEvents_, eventId("elapsed_cycles_sm")) - .Times(1) - .WillOnce(Return(5)); - std::vector ids = {5}; - EXPECT_CALL(*cuptiEvents_, createGroupSets(ids)) - .Times(1) - .WillOnce(Return(&groupSets_)); - EXPECT_CALL(*cuptiEvents_, enablePerInstance(eventGroups_[0])).Times(1); - EXPECT_CALL(*cuptiEvents_, instanceCount(eventGroups_[0])) - .Times(1) - .WillOnce(Return(80)); - EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[0])) - .Times(1) - .WillOnce(Return(ids)); - EXPECT_CALL(*cuptiEvents_, enableGroupSet(_)).Times(1); - - profiler_->configure(cfg, nullptr); - - EXPECT_TRUE(profiler_->enabled()); -} - -TEST_F(EventProfilerTest, ConfigureOnDemand) { - using namespace testing; - - // Test base + on-demand config, one event and one metric - Config cfg, on_demand_cfg; - bool parsed = cfg.parse(R"( - EVENTS = active_cycles - SAMPLE_PERIOD_MSECS=500 - REPORT_PERIOD_SECS=10 - SAMPLES_PER_REPORT=5 - )"); - EXPECT_TRUE(parsed); - - parsed = on_demand_cfg.parse(R"( - METRICS = ipc - EVENTS_DURATION_SECS=60 - SAMPLE_PERIOD_MSECS=200 - MULTIPLEX_PERIOD_MSECS=2000 - REPORT_PERIOD_SECS=3 - SAMPLES_PER_REPORT=10 - )"); - EXPECT_TRUE(parsed); - - // One event - EXPECT_CALL(*cuptiEvents_, eventId("active_cycles")) - .Times(1) - .WillOnce(Return(3)); - // One metric - EXPECT_CALL(*cuptiMetrics_, idFromName("ipc")).Times(1).WillOnce(Return(10)); - std::map ipc_events; - ipc_events[4] = "instructions"; - ipc_events[5] = "elapsed_cycles_sm"; - EXPECT_CALL(*cuptiMetrics_, events(10)).Times(1).WillOnce(Return(ipc_events)); - EXPECT_CALL(*cuptiMetrics_, evaluationMode(10)) - .Times(1) - .WillOnce(Return(CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE)); - EXPECT_CALL(*cuptiMetrics_, valueKind(10)) - .Times(1) - .WillOnce(Return(CUPTI_METRIC_VALUE_KIND_DOUBLE)); - std::vector ids = {3, 4, 5}; - groupSet_[0].numEventGroups = 2; - groupSets_.numSets = 2; - EXPECT_CALL(*cuptiEvents_, createGroupSets(ids)) - .Times(1) - .WillOnce(Return(&groupSets_)); - // Specified CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE per instance above - // So check that it's enabled - EXPECT_CALL(*cuptiEvents_, enablePerInstance(eventGroups_[0])).Times(1); - EXPECT_CALL(*cuptiEvents_, enablePerInstance(eventGroups_[1])).Times(1); - EXPECT_CALL(*cuptiEvents_, enablePerInstance(eventGroups_[2])).Times(1); - std::vector ids_g1{3}, ids_g2{4}, ids_g3{5}; - EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[0])) - .Times(1) - .WillOnce(Return(ids_g1)); - EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[1])) - .Times(1) - .WillOnce(Return(ids_g2)); - EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[2])) - .Times(1) - .WillOnce(Return(ids_g3)); - EXPECT_CALL(*cuptiEvents_, enableGroupSet(_)).Times(1); - - profiler_->configure(cfg, &on_demand_cfg); - - EXPECT_TRUE(profiler_->enabled()); - EXPECT_EQ(profiler_->samplePeriod().count(), 250); - EXPECT_EQ(profiler_->multiplexPeriod().count(), 1000); - EXPECT_EQ(profiler_->reportPeriod().count(), 10000); - EXPECT_EQ(profiler_->onDemandReportPeriod().count(), 4000); -} - -TEST_F(EventProfilerTest, ReportSample) { - using namespace testing; - - // Test base + on-demand config, one event and one metric - Config cfg, on_demand_cfg; - bool parsed = cfg.parse("EVENTS = active_cycles"); - EXPECT_TRUE(parsed); - - parsed = on_demand_cfg.parse(R"( - METRICS = ipc - EVENTS_DURATION_SECS=60 - )"); - EXPECT_TRUE(parsed); - - // One event - EXPECT_CALL(*cuptiEvents_, eventId("active_cycles")) - .Times(1) - .WillOnce(Return(3)); - // One metric - EXPECT_CALL(*cuptiMetrics_, idFromName("ipc")).Times(1).WillOnce(Return(10)); - std::map ipc_events; - ipc_events[4] = "instructions"; - ipc_events[5] = "elapsed_cycles_sm"; - EXPECT_CALL(*cuptiMetrics_, events(10)).Times(1).WillOnce(Return(ipc_events)); - EXPECT_CALL(*cuptiMetrics_, evaluationMode(10)) - .Times(1) - .WillOnce(Return(CUPTI_METRIC_EVALUATION_MODE_PER_INSTANCE)); - EXPECT_CALL(*cuptiMetrics_, valueKind(10)) - .Times(1) - .WillOnce(Return(CUPTI_METRIC_VALUE_KIND_DOUBLE)); - std::vector ids = {3, 4, 5}; - groupSet_[0].numEventGroups = 2; - groupSets_.numSets = 2; - EXPECT_CALL(*cuptiEvents_, createGroupSets(ids)) - .Times(1) - .WillOnce(Return(&groupSets_)); - EXPECT_CALL(*cuptiEvents_, instanceCount(_)) - .Times(3) - .WillRepeatedly(Return(4)); - std::vector ids_g1{3}, ids_g2{4}, ids_g3{5}; - // These will be called by collectSample() as well, which is called twice - // per group set - EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[0])) - .Times(3) - .WillRepeatedly(Return(ids_g1)); - EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[1])) - .Times(3) - .WillRepeatedly(Return(ids_g2)); - EXPECT_CALL(*cuptiEvents_, eventsInGroup(eventGroups_[2])) - .Times(3) - .WillRepeatedly(Return(ids_g3)); - EXPECT_CALL(*cuptiEvents_, enableGroupSet(_)).Times(1); - - profiler_->configure(cfg, &on_demand_cfg); - - EXPECT_TRUE(profiler_->enabled()); - - EXPECT_CALL(*cuptiEvents_, readEvent(_, _, _)) - .Times(6) - .WillRepeatedly(Invoke( - [](CUpti_EventGroup g, CUpti_EventID id, std::vector& vals) { - vals = {1, 2, 3, 4}; - })); - - // Need to collect four times - twice for each group set - profiler_->collectSample(); - profiler_->collectSample(); - EXPECT_CALL(*cuptiEvents_, disableGroupSet(_)).Times(1); - EXPECT_CALL(*cuptiEvents_, enableGroupSet(_)).Times(1); - profiler_->enableNextCounterSet(); - profiler_->collectSample(); - profiler_->collectSample(); - - std::vector ipc_ids = {4, 5}; - // Called once for each instance (4) and once for the total. - // x2 since we recompute per logger. - EXPECT_CALL( - *cuptiMetrics_, - calculate(10, CUPTI_METRIC_VALUE_KIND_DOUBLE, ipc_ids, _, 2000000000)) - .Times(10) - .WillRepeatedly(Return(SampleValue(0.3))); - auto& logger = dynamic_cast(*loggers_[0]); - EXPECT_CALL(logger, handleSample(0, _, _)) - .Times(1) - .WillOnce(Invoke([](int device, const Sample& sample, bool from_new_version) { - // Sample will include all stats - logger must pick the - // ones it wants. - EXPECT_EQ(sample.stats.size(), 4); - EXPECT_EQ(sample.stats[0].name, "active_cycles"); - EXPECT_EQ(sample.stats[1].name, "instructions"); - EXPECT_EQ(sample.stats[2].name, "elapsed_cycles_sm"); - EXPECT_EQ(sample.stats[3].name, "ipc"); - // 2 samples, each with values {1, 2, 3, 4} - // i.e. {2, 4, 6, 8} total - EXPECT_EQ(sample.stats[0].total.getInt(), 20); - EXPECT_EQ(sample.stats[0].percentileValues[0].second.getInt(), 2); - EXPECT_EQ(sample.stats[0].percentileValues.back().second.getInt(), 8); - // ipc is always 0.3 from mocked calculate function above - EXPECT_EQ(sample.stats[3].total.getDouble(), 0.3); - EXPECT_EQ(sample.stats[3].percentileValues[0].second.getDouble(), 0.3); - EXPECT_EQ( - sample.stats[3].percentileValues.back().second.getDouble(), 0.3); - })); - profiler_->reportSamples(); - - auto& on_demand_logger = dynamic_cast(*onDemandLoggers_[0]); - EXPECT_CALL(on_demand_logger, handleSample(0, _, _)).Times(1); - profiler_->reportOnDemandSamples(); - - EXPECT_CALL(*cuptiEvents_, disableGroupSet(_)).Times(1); -} diff --git a/plugins/tensorboard-plugins/libkineto/test/LoggerObserverTest.cpp b/plugins/tensorboard-plugins/libkineto/test/LoggerObserverTest.cpp deleted file mode 100644 index 30ba4a824..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/LoggerObserverTest.cpp +++ /dev/null @@ -1,96 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include - -// TODO(T90238193) -// @lint-ignore-every CLANGTIDY facebook-hte-RelativeInclude -#include "include/libkineto.h" -#include "src/Logger.h" -#include "LoggerCollector.h" - -using namespace KINETO_NAMESPACE; - -#if !USE_GOOGLE_LOG - -constexpr char InfoTestStr[] = "Checking LOG(INFO)"; -constexpr char WarningTestStr[] = "Checking LOG(WARNING)"; -constexpr char ErrorTestStr[] = "Checking LOG(ERROR)"; - -TEST(LoggerObserverTest, SingleCollectorObserver) { - // Add a LoggerObserverCollector to collect all logs during the trace. - std::unique_ptr lCollector = std::make_unique(); - Logger::addLoggerObserver(lCollector.get()); - - LOG(INFO) << InfoTestStr; - LOG(WARNING) << WarningTestStr; - LOG(ERROR) << ErrorTestStr; - - auto LoggerMD = lCollector->extractCollectorMetadata(); - EXPECT_TRUE(LoggerMD[LoggerOutputType::INFO][0].find(InfoTestStr) != std::string::npos); - EXPECT_TRUE(LoggerMD[LoggerOutputType::WARNING][0].find(WarningTestStr) != std::string::npos); - EXPECT_TRUE(LoggerMD[LoggerOutputType::ERROR][0].find(ErrorTestStr) != std::string::npos); - - Logger::removeLoggerObserver(lCollector.get()); -} - -#define NUM_OF_MESSAGES_FOR_EACH_TYPE 10 -#define NUM_OF_WRITE_THREADS 200 - -// Writes NUM_OF_MESSAGES_FOR_EACH_TYPE messages for each INFO, WARNING, and ERROR. -// NOLINTNEXTLINE(clang-diagnostic-unused-parameter) -void* writeSeveralMessages(void* ptr) { - for(int i=0; i lc1 = std::make_unique(); - std::unique_ptr lc2 = std::make_unique(); - std::unique_ptr lc3 = std::make_unique(); - std::unique_ptr lc4 = std::make_unique(); - Logger::addLoggerObserver(lc1.get()); - Logger::addLoggerObserver(lc2.get()); - Logger::addLoggerObserver(lc3.get()); - Logger::addLoggerObserver(lc4.get()); - - // Launch NUM_OF_WRITE_THREADS threads writing several messages. - pthread_t ListOfThreads[NUM_OF_WRITE_THREADS]; - for (int i=0; iextractCollectorMetadata(); - int InfoCount = 0, WarnCount = 0, ErrorCount = 0; - for (auto& md : lc1MD) { - InfoCount += md.first == LoggerOutputType::INFO ? md.second.size() : 0; - WarnCount += md.first == LoggerOutputType::WARNING ? md.second.size() : 0; - ErrorCount += md.first == LoggerOutputType::ERROR ? md.second.size() : 0; - } - - EXPECT_EQ(InfoCount, NUM_OF_WRITE_THREADS * NUM_OF_MESSAGES_FOR_EACH_TYPE); - EXPECT_EQ(WarnCount, NUM_OF_WRITE_THREADS * NUM_OF_MESSAGES_FOR_EACH_TYPE); - EXPECT_EQ(ErrorCount, NUM_OF_WRITE_THREADS * NUM_OF_MESSAGES_FOR_EACH_TYPE); - - Logger::removeLoggerObserver(lc1.get()); - Logger::removeLoggerObserver(lc2.get()); - Logger::removeLoggerObserver(lc3.get()); - Logger::removeLoggerObserver(lc4.get()); -} - -#endif // !USE_GOOGLE_LOG - -int main(int argc, char **argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/plugins/tensorboard-plugins/libkineto/test/MockActivitySubProfiler.cpp b/plugins/tensorboard-plugins/libkineto/test/MockActivitySubProfiler.cpp deleted file mode 100644 index 89f1d536c..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/MockActivitySubProfiler.cpp +++ /dev/null @@ -1,49 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include -#include -#include - -#include "test/MockActivitySubProfiler.h" - -namespace libkineto { - -const std::set supported_activities {ActivityType::CPU_OP}; -const std::string profile_name{"MockProfiler"}; - -void MockProfilerSession::processTrace(ActivityLogger& logger) { - for (const auto& activity: activities()) { - activity.log(logger); - } -} - -const std::string& MockActivityProfiler::name() const { - return profile_name; -} - -const std::set& MockActivityProfiler::availableActivities() const { - return supported_activities; -} - -MockActivityProfiler::MockActivityProfiler( - std::vector& activities) : - test_activities_(activities) {}; - -std::unique_ptr MockActivityProfiler::configure( - const std::set& /*activity_types*/, - const Config& /*config*/) { - auto session = std::make_unique(); - session->set_test_activities(std::move(test_activities_)); - return session; -}; - -std::unique_ptr MockActivityProfiler::configure( - int64_t /*ts_ms*/, - int64_t /*duration_ms*/, - const std::set& activity_types, - const Config& config) { - return configure(activity_types, config); -}; - -} // namespace libkineto - diff --git a/plugins/tensorboard-plugins/libkineto/test/MockActivitySubProfiler.h b/plugins/tensorboard-plugins/libkineto/test/MockActivitySubProfiler.h deleted file mode 100644 index 36eaa13d1..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/MockActivitySubProfiler.h +++ /dev/null @@ -1,72 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#pragma once - -#include -#include -#include - -#include "include/IActivityProfiler.h" - -namespace libkineto { - -class MockProfilerSession: public IActivityProfilerSession { - - public: - explicit MockProfilerSession() {} - - void start() override { - start_count++; - status_ = TraceStatus::RECORDING; - } - - void stop() override { - stop_count++; - status_ = TraceStatus::PROCESSING; - } - - std::vector& activities() override { - return test_activities_; - } - - std::vector errors() override { - return {}; - } - - void processTrace(ActivityLogger& logger) override; - - void set_test_activities(std::vector&& acs) { - test_activities_ = std::move(acs); - } - - int start_count = 0; - int stop_count = 0; - private: - std::vector test_activities_; -}; - - -class MockActivityProfiler: public IActivityProfiler { - - public: - explicit MockActivityProfiler(std::vector& activities); - - const std::string& name() const override; - - const std::set& availableActivities() const override; - - std::unique_ptr configure( - const std::set& activity_types, - const Config& config) override; - - std::unique_ptr configure( - int64_t ts_ms, - int64_t duration_ms, - const std::set& activity_types, - const Config& config) override; - - private: - std::vector test_activities_; -}; - -} // namespace libkineto diff --git a/plugins/tensorboard-plugins/libkineto/test/PidInfoTest.cpp b/plugins/tensorboard-plugins/libkineto/test/PidInfoTest.cpp deleted file mode 100644 index b86cfb36d..000000000 --- a/plugins/tensorboard-plugins/libkineto/test/PidInfoTest.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary. - -#include "include/ThreadUtil.h" - -#include -#include - -#include -#include - -using namespace KINETO_NAMESPACE; - -TEST(ThreadNameTest, setAndGet) { - setThreadName("ThreadNameTest"); - EXPECT_EQ(getThreadName(), "ThreadNameTest"); - - setThreadName(""); - EXPECT_EQ(getThreadName(), ""); - - // Spaces etc are ok - setThreadName("Name w/ spaces"); - EXPECT_EQ(getThreadName(), "Name w/ spaces"); - - // More than 16 chars is not OK - setThreadName("More than 16 characters"); - EXPECT_EQ(getThreadName(), "Name w/ spaces"); -} -- Gitee From fe47e2100cbd2a5c66a84fbf5422f86c3dc974a8 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 31 Jul 2024 09:36:41 +0800 Subject: [PATCH 060/791] review fix --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index ea9323ae0..0070b58e5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -655,7 +655,7 @@ def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, summary_compare, md5_compare = task_dumppath_get(input_parma) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(input_parma, output_path, stack_mode, summary_compare, md5_compare) + check_compare_param(input_parma, output_path, summary_compare, md5_compare) except CompareException as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) -- Gitee From 627102bcd400d6ae0ee8f87177ae3c30b6070aaa Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 31 Jul 2024 09:47:28 +0800 Subject: [PATCH 061/791] compare command --- .../msprobe/pytorch/compare/acc_compare.py | 42 +++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index ea9323ae0..59e5faca5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -649,18 +649,21 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) -def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, - fuzzy_match=False): +# def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, +# fuzzy_match=False): +def compare(args): + #### 需要增加文件路径路径校验 + input_param = json.load(args.input_path) try: summary_compare, md5_compare = task_dumppath_get(input_parma) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) - create_directory(output_path) - check_compare_param(input_parma, output_path, stack_mode, summary_compare, md5_compare) + check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) + create_directory(args.output_path) + check_compare_param(input_param, args.output_path, args.stack_mode, summary_compare, md5_compare) except CompareException as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) - compare_core(input_parma, output_path, stack_mode=stack_mode, - auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, + compare_core(input_param, args.output_path, stack_mode=args.stack_mode, + auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) @@ -1032,3 +1035,28 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): else: result_item.extend([CompareConst.NONE, "-1"]) result.append(result_item) + + +def _compare_parser(parser): + parser.add_argument("-i", "--input_path", dest="input_path", type=str, + help=" The compare input path, a dict json.", required=True) + parser.add_argument("-o", "--out_path", dest="out_path", type=str, + help=" The compare task result out path.", required=True) + parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", + help=" Whether to save stack info.", required=False) + parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_true", + help=" Whether to give advisor.", required=False) + parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", + help=" Whether to perform a fuzzy match on the api name.", required=False) + + +def _compare(parser=None): + if not parser: + parser = argparse.ArgumentParser() + _compare_parser(parser) + args = parser.parse_args(sys.argv[1:]) + compare(args) + + +if __name__ == '__main__': + _compare() -- Gitee From a9e4f432835b01f4e01ab394774e0592a14355c3 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 31 Jul 2024 10:29:21 +0800 Subject: [PATCH 062/791] compare command --- debug/accuracy_tools/msprobe/msprobe.py | 5 +++++ .../accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 7 ++++--- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 698165b61..12b04920a 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -22,6 +22,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _ _api_precision_compare_command from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command +from msprobe.pytorch.compare.acc_compare import _compare_parser, compare def main(): @@ -36,10 +37,12 @@ def main(): help='Deep learning framework.') subparsers = parser.add_subparsers() subparsers.add_parser('parse') + compare_cmd_parser = subparsers.add_parser('run_ut') run_ut_cmd_parser = subparsers.add_parser('run_ut') multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut') api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare') run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check') + _compare_parser(compare_cmd_parser) _run_ut_parser(run_ut_cmd_parser) _run_ut_parser(multi_run_ut_cmd_parser) multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, @@ -61,6 +64,8 @@ def main(): _api_precision_compare_command(args) elif sys.argv[3] == "run_overflow_check": _run_overflow_check_command(args) + elif sys.argv[3] == "compare": + compare(args) if __name__ == "__main__": diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 59e5faca5..88c839511 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -653,9 +653,10 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): # fuzzy_match=False): def compare(args): #### 需要增加文件路径路径校验 - input_param = json.load(args.input_path) + with FileOpen(args.input_path, "r") as file: + input_param = json.load(file) try: - summary_compare, md5_compare = task_dumppath_get(input_parma) + summary_compare, md5_compare = task_dumppath_get(input_param) check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) create_directory(args.output_path) check_compare_param(input_param, args.output_path, args.stack_mode, summary_compare, md5_compare) @@ -1040,7 +1041,7 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): def _compare_parser(parser): parser.add_argument("-i", "--input_path", dest="input_path", type=str, help=" The compare input path, a dict json.", required=True) - parser.add_argument("-o", "--out_path", dest="out_path", type=str, + parser.add_argument("-o", "--output_path", dest="output_path", type=str, help=" The compare task result out path.", required=True) parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", help=" Whether to save stack info.", required=False) -- Gitee From b1e3888cad4621632a4cfd981891a68fa222872f Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 31 Jul 2024 22:33:59 +0800 Subject: [PATCH 063/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BA=86mindspore?= =?UTF-8?q?=E7=9A=84=E5=90=8C=E6=A1=86=E6=9E=B6api=E7=B2=BE=E5=BA=A6?= =?UTF-8?q?=E5=AF=B9=E6=AF=94=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/advisor/advisor.py | 124 ++ .../mindspore/advisor/advisor_const.py | 59 + .../mindspore/advisor/advisor_result.py | 58 + .../msprobe/mindspore/compare/acc_compare.py | 1033 +++++++++++++++++ .../mindspore/compare/distributed_compare.py | 112 ++ .../msprobe/mindspore/compare/highlight.py | 100 ++ .../msprobe/mindspore/compare/mapping.yaml | 607 ++++++++++ .../msprobe/mindspore/compare/match.py | 36 + .../msprobe/mindspore/compare/npy_compare.py | 244 ++++ 9 files changed, 2373 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/acc_compare.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/highlight.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/mapping.yaml create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/match.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/npy_compare.py diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py new file mode 100644 index 000000000..ec2773e6d --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +from msprobe.mindspore.advisor.advisor_result import AdvisorResult +from msprobe.mindspore.advisor.advisor_const import AdvisorConst +from msprobe.mindspore.common.log import logger +from msprobe.core.common.utils import CompareException +from msprobe.core.common.file_check import FileChecker +from msprobe.core.common.const import Const, CompareConst, FileCheckConst + +class Advisor: + """ + Class for generate advisor + """ + + def __init__(self, input_data, out_path=""): + self.input_data = input_data + self.out_path = os.path.realpath(out_path) + self.file_type = None + + @staticmethod + def deterministic_advisor(message, node_name): + for api_name in AdvisorConst.NEED_DETERMINISTIC_API: + if api_name in node_name: + return AdvisorConst.DETERMINISTIC_SUGGEST + return message + + @staticmethod + def batch_norm_advisor(message, node_name): + if AdvisorConst.FUNC_BATCH_NORM in node_name and AdvisorConst.FORWARD_INPUT_1 in node_name: + message = AdvisorConst.BATCH_NORM_SUGGEST + return message + + def analyze_unmatched(self, analyze_data): + if self.file_type == Const.ALL: + accuracy_unmatched = analyze_data[ + analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] + else: + accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) | + (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)] + num_unmatch = len(accuracy_unmatched) + if num_unmatch != 0: + for i in range(len(accuracy_unmatched)): + item = accuracy_unmatched.iloc[i] + logger.warning("The tensor name matches but the shape or dtype does not match: {}" + .format(item[CompareConst.NPU_NAME])) + + def gen_advisor_result(self, pd_data): + first_failing_data = pd_data.iloc[0] + node_name = first_failing_data[CompareConst.NPU_NAME] + index = first_failing_data['index'] + message = self.gen_advisor_message(node_name) + logger.warning("Find %s accuracy not reached, the line is %s" % (node_name, index)) + result = AdvisorResult(node_name, index, message) + return result + + def gen_advisor_message(self, node_name): + if AdvisorConst.FORWARD in node_name: + if AdvisorConst.INPUT in node_name: + message = AdvisorConst.FORWARD_INPUT_SUGGEST + else: + message = AdvisorConst.FORWARD_OUTPUT_SUGGEST + message = self.deterministic_advisor(message, node_name) + else: + if AdvisorConst.INPUT in node_name: + message = AdvisorConst.BACKWARD_INPUT_SUGGEST + else: + message = AdvisorConst.BACKWARD_OUTPUT_SUGGEST + message = self.deterministic_advisor(message, node_name) + message = self.batch_norm_advisor(message, node_name) + return message + + def analysis(self): + self._check_path_vaild() + analyze_data = self._parse_input_data() + logger.info("Start analyzing the comparison result: %s" % self.file_type) + self.analyze_unmatched(analyze_data) + if self.file_type == Const.ALL: + failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] + elif self.file_type == Const.MD5: + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF] + elif self.file_type == Const.SUMMARY: + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING] + if failing_data.empty: + logger.info("All data from api input/output accuracy reached") + result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) + else: + result = self.gen_advisor_result(failing_data) + message_list = result.print_advisor_log() + result.gen_summary_file(self.out_path, message_list) + + def _parse_input_data(self): + data_columns = self.input_data.columns.values + if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): + self.file_type = Const.ALL + elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): + self.file_type = Const.MD5 + elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): + self.file_type = Const.SUMMARY + else: + logger.error('Compare result does not meet the required conditions.') + raise CompareException(CompareException.INVALID_DATA_ERROR) + df = self.input_data.reset_index() + return df + + def _check_path_vaild(self): + out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) + out_path_checker.common_check() diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py new file mode 100644 index 000000000..737c67591 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + + +class AdvisorConst: + """ + Class for advisor const + """ + + # text symbol + NEW_LINE = "\n" + COLON = ": " + + # advisor summary key + SUSPECT_NODES = "Suspect Nodes" + LINE = "Line" + ADVISOR_SUGGEST = "Expert Advice" + + NO_ERROR_API = "NA" + + # advisor message + NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements." + FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \ + "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \ + "3. The fault may be caused by memory corruption and further analysis is required." + FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation." + BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected." + BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation." + BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \ + "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \ + "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \ + "3. Use seed_all(mode=True) to enable deterministic computing." + DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \ + "can seed_all(mode=True) to enable deterministic computing." + + FUNC_BATCH_NORM = "Functional_batch_norm" + FORWARD_INPUT_1 = "forward_input.1" + NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"] + BATCH_NORM = "batch_norm" + + # name keyword + INPUT = "input" + OUTPUT = "output" + FORWARD = "forward" + BACKWARD = "backward" diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py new file mode 100644 index 000000000..5d59068fc --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import os +import time + +from msprobe.mindspore.advisor.advisor_const import AdvisorConst +from msprobe.mindspore.common.log import logger +from msprobe.core.common.const import Const, FileCheckConst +from msprobe.core.common.file_check import change_mode + + +class AdvisorResult: + """ + Class for generate advisor result + """ + + def __init__(self, node, line, message): + self.suspect_node = node + self.line = line + self.advisor_message = message + + @staticmethod + def gen_summary_file(out_path, message_list): + file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) + result_file = os.path.join(out_path, file_name) + try: + with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file: + output_file.truncate(0) + message_list = [message + AdvisorConst.NEW_LINE for message in message_list] + output_file.writelines(message_list) + change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY) + except IOError as io_error: + logger.error("Failed to save %s, the reason is %s." % (result_file, io_error)) + else: + logger.info("The advisor summary is saved in: %s" % result_file) + + def print_advisor_log(self): + logger.info("The summary of the expert advice is as follows: ") + message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line), + AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node, + AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message] + for message in message_list: + logger.info(message) + return message_list diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/acc_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/acc_compare.py new file mode 100644 index 000000000..0464995d5 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/acc_compare.py @@ -0,0 +1,1033 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import json +import multiprocessing +import os.path +import sys + +import numpy as np +import pandas as pd +import openpyxl +from openpyxl.styles import PatternFill +from collections import namedtuple +from dataclasses import dataclass + +from msprobe.mindspore.compare.match import graph_mapping +from msprobe.mindspore.compare.highlight import HighlightRules, get_header_index +from msprobe.mindspore.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ + get_error_message +from msprobe.mindspore.advisor.advisor import Advisor +from msprobe.mindspore.common.log import logger +from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ + format_value, check_file_not_exists, check_configuration_param, task_dumppath_get +from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, create_directory +from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.core.common.exceptions import FileCheckException + + +def check_graph_mode(a_op_name, b_op_name): + if "Aten" in a_op_name and "Aten" not in b_op_name: + return True + if "Aten" not in a_op_name and "Aten" in b_op_name: + return True + return False + + +def check_op(npu_dict, bench_dict, fuzzy_match): + a_op_name = npu_dict["op_name"] + b_op_name = bench_dict["op_name"] + graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) + if graph_mode: + return graph_mapping.match(a_op_name[0], b_op_name[0]) + struct_match = check_struct_match(npu_dict, bench_dict) + if not fuzzy_match: + return a_op_name == b_op_name and struct_match + is_match = True + try: + is_match = fuzzy_check_op(a_op_name, b_op_name) + except Exception as err: + logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) + is_match = False + return is_match and struct_match + + +def check_struct_match(npu_dict, bench_dict): + npu_struct_in = npu_dict.get("input_struct") + bench_struct_in = bench_dict.get("input_struct") + npu_struct_out = npu_dict.get("output_struct") + bench_struct_out = bench_dict.get("output_struct") + is_match = npu_struct_in == bench_struct_in and npu_struct_out == bench_struct_out + if not is_match: + if len(npu_struct_in) == 0 or len(bench_struct_in) == 0 or len(npu_struct_in) != len(bench_struct_in): + return False + struct_in_is_match = check_type_shape_match(npu_struct_in, bench_struct_in) + struct_out_is_match = check_type_shape_match(npu_struct_out, bench_struct_out) + is_match = struct_in_is_match and struct_out_is_match + return is_match + + +def check_type_shape_match(npu_struct, bench_struct): + shape_type_match = False + for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct): + npu_type = npu_type_shape[0] + npu_shape = npu_type_shape[1] + bench_type = bench_type_shape[0] + bench_shape = bench_type_shape[1] + shape_match = npu_shape == bench_shape + type_match = npu_type == bench_type + if not type_match: + if [npu_type, bench_type] in [["Float16", "Float32"], ["Float32", "Float16"]]: + type_match = True + else: + type_match = False + shape_type_match = shape_match and type_match + if not shape_type_match: + return False + return shape_type_match + + +def fuzzy_check_op(npu_name_list, bench_name_list): + if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list): + return False + is_match = True + for npu_name, bench_name in zip(npu_name_list, bench_name_list): + is_match = fuzzy_check_name(npu_name, bench_name) + if not is_match: + break + return is_match + + +def fuzzy_check_name(npu_name, bench_name): + if "forward" in npu_name and "forward" in bench_name: + is_match = rename_api(npu_name, "forward") == rename_api(bench_name, "forward") + elif "backward" in npu_name and "backward" in bench_name: + is_match = rename_api(npu_name, "backward") == rename_api(bench_name, "backward") + else: + is_match = npu_name == bench_name + return is_match + + +def rename_api(npu_name, process): + npu_split = npu_name.split(process) + torch_func_index, in_out = npu_split[0], npu_split[1] + torch_func_split = torch_func_index.rsplit(Const.SEP, 2) + torch_func = str(torch_func_split[0]) + str(in_out) + return torch_func + + +def merge_tensor(tensor_list, summary_compare, md5_compare): + op_dict = {} + op_dict["op_name"] = [] + op_dict["input_struct"] = [] + op_dict["kwargs_struct"] = [] + op_dict["output_struct"] = [] + op_dict["summary"] = [] + op_dict["stack_info"] = [] + + all_mode_bool = not (summary_compare or md5_compare) + if all_mode_bool: + op_dict["data_name"] = [] + + for tensor in tensor_list: + if len(tensor) == 2: + op_dict['stack_info'].append(tensor['full_info']) + break + op_dict["op_name"].append(tensor['full_op_name']) + if not md5_compare: + if tensor['full_op_name'].find("input") != -1: + op_dict["input_struct"].append((tensor['dtype'], tensor['shape'])) + elif tensor['full_op_name'].find("kwarg") != -1: + op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'])) + elif tensor['full_op_name'].find("output") != -1: + op_dict["output_struct"].append((tensor['dtype'], tensor['shape'])) + else: + if tensor['full_op_name'].find("input") != -1: + op_dict["input_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) + elif tensor['full_op_name'].find("kwarg") != -1: + op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) + elif tensor['full_op_name'].find("output") != -1: + op_dict["output_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) + + op_dict["summary"].append([tensor['Max'], tensor['Min'], tensor['Mean'], tensor['Norm']]) + + if all_mode_bool: + op_dict["data_name"].append(tensor['data_name']) + + if not op_dict["kwargs_struct"]: + del op_dict["kwargs_struct"] + return op_dict if op_dict["op_name"] else {} + + +def match_op(npu_queue, bench_queue, fuzzy_match): + for b_index, b_op in enumerate(bench_queue[0: -1]): + if check_op(npu_queue[-1], b_op, fuzzy_match): + return len(npu_queue) - 1, b_index + if check_op(npu_queue[-1], bench_queue[-1], fuzzy_match): + return len(npu_queue) - 1, len(bench_queue) - 1 + for n_index, n_op in enumerate(npu_queue[0: -1]): + if check_op(n_op, bench_queue[-1], fuzzy_match): + return n_index, len(bench_queue) - 1 + return -1, -1 + + +def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=False): + def get_accuracy_core(n_start, n_len, b_start, b_len, key): + min_len = min(n_len, b_len) + npu_stack_info = n_dict.get("stack_info", None) + bench_stack_info = b_dict.get("stack_info", None) + has_stack = npu_stack_info and bench_stack_info + + all_mode_bool = not (summary_compare or md5_compare) + if all_mode_bool: + npu_data_name = n_dict.get("data_name", None) + bench_data_name = b_dict.get("data_name", None) + + for index in range(min_len): + + n_name = n_dict['op_name'][n_start + index] + b_name = b_dict['op_name'][b_start + index] + n_struct = n_dict[key][index] + b_struct = b_dict[key][index] + err_msg = "" + if md5_compare: + result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], + n_struct[2], b_struct[2], + CompareConst.PASS if n_struct[2] == b_struct[2] else CompareConst.DIFF] + if has_stack and index == 0 and key == "input_struct": + result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + result.append(result_item) + continue + + if summary_compare: + result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], + " ", " ", " ", " ", " ", " ", " ", " "] + else: + result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], + " ", " ", " ", " ", " "] + + npu_summary_data = n_dict.get("summary")[n_start + index] + result_item.extend(npu_summary_data) + bench_summary_data = b_dict.get("summary")[b_start + index] + result_item.extend(bench_summary_data) + + if summary_compare: + start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) + warning_flag = False + for i, (npu_val, bench_val) in enumerate(zip(npu_summary_data, bench_summary_data)): + if isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)): + diff = npu_val - bench_val + if bench_val != 0: + relative = str(abs((diff / bench_val) * 100)) + '%' + else: + relative = "N/A" + result_item[start_idx + i] = diff + result_item[start_idx + i + 4] = relative + magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) + if magnitude_diff > 0.5: + warning_flag = True + else: + result_item[start_idx + i] = CompareConst.NONE + accuracy_check = CompareConst.WARNING if warning_flag else "" + err_msg += "Need double check api accuracy." if warning_flag else "" + for i in range(start_idx, len(result_item)): + if str(result_item[i]) in ('inf', '-inf', 'nan'): + result_item[i] = f'{result_item[i]}\t' + + result_item.append(accuracy_check if summary_compare else CompareConst.ACCURACY_CHECK_YES) + result_item.append(err_msg) + if has_stack and index == 0 and key == "input_struct": + result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + if all_mode_bool: + result_item.append(npu_data_name[n_start + index]) + + result.append(result_item) + + if n_len > b_len: + for index in range(b_len, n_len): + n_name = n_dict['op_name'][n_start + index] + n_struct = n_dict[key][index] + if md5_compare: + result_item = [n_name, CompareConst.NAN, n_struct[0], CompareConst.NAN, + n_struct[1], CompareConst.NAN, n_struct[2], CompareConst.NAN, CompareConst.NAN] + result.append(result_item) + continue + result_item = [n_name, CompareConst.NAN, n_struct[0], CompareConst.NAN, + n_struct[1], CompareConst.NAN, " ", " ", " ", " ", " "] + summary_data = n_dict.get("summary")[n_start + index] + result_item.extend(summary_data) + summary_data = [CompareConst.NAN for _ in range(len(n_dict.get("summary")[0]))] + result_item.extend(summary_data) + + err_msg = "" + result_item.append(CompareConst.ACCURACY_CHECK_YES) + result_item.append(err_msg) + + if has_stack and index == 0 and key == "input_struct": + result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + if all_mode_bool: + result_item.append(npu_data_name[n_start + index]) + + result.append(result_item) + + n_num = len(n_dict['op_name']) + b_num = len(b_dict['op_name']) + n_num_input = len([name for name in n_dict['op_name'] if 'input' in name]) + b_num_input = len([name for name in b_dict['op_name'] if 'input' in name]) + n_num_kwarg = len([name for name in n_dict['op_name'] if 'kwarg' in name]) + b_num_kwarg = len([name for name in b_dict['op_name'] if 'kwarg' in name]) + n_num_output = n_num - n_num_input - n_num_kwarg + b_num_output = b_num - b_num_input - b_num_kwarg + get_accuracy_core(0, n_num_input, 0, b_num_input, 'input_struct') + get_accuracy_core(n_num_input, n_num_kwarg, b_num_input, b_num_kwarg, "kwargs_struct") + get_accuracy_core(n_num_input + n_num_kwarg, n_num_output, b_num_input + b_num_kwarg, b_num_output, 'output_struct') + + +def _do_multi_process(input_parma, result_df): + try: + result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + +def read_dump_data(result_df): + try: + npu_dump_name_list = result_df.iloc[0:, 0].tolist() + npu_dump_tensor_list = result_df.iloc[0:, -1].tolist() + op_name_mapping_dict = {} + for index, _ in enumerate(npu_dump_name_list): + npu_dump_name = npu_dump_name_list[index] + npu_dump_tensor = npu_dump_tensor_list[index] + op_name_mapping_dict[npu_dump_name] = [npu_dump_tensor, npu_dump_tensor] + return op_name_mapping_dict + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + + +def _handle_multi_process(func, input_parma, result_df, lock): + process_num = int((multiprocessing.cpu_count() + 1) / 2) + op_name_mapping_dict = read_dump_data(result_df) + + df_chunk_size = len(result_df) // process_num + if df_chunk_size > 0: + df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] + else: + df_chunks = [result_df] + + results = [] + pool = multiprocessing.Pool(process_num) + + def err_call(args): + logger.error('multiprocess compare failed! Reason: {}'.format(args)) + try: + pool.terminate() + except OSError as e: + logger.error("pool terminate failed") + + for process_idx, df_chunk in enumerate(df_chunks): + idx = df_chunk_size * process_idx + result = pool.apply_async(func, + args=(idx, op_name_mapping_dict, df_chunk, lock, input_parma), + error_callback=err_call) + results.append(result) + final_results = [r.get() for r in results] + pool.close() + pool.join() + return pd.concat(final_results, ignore_index=True) + + +def compare_ops(idx, dump_path_dict, result_df, lock, input_parma): + cos_result = [] + max_err_result = [] + max_relative_err_result = [] + err_mess = [] + one_thousand_err_ratio_result = [] + five_thousand_err_ratio_result = [] + is_print_compare_log = input_parma.get("is_print_compare_log") + for i in range(len(result_df)): + op_name = result_df.iloc[i, 0] + if is_print_compare_log: + logger.info("start compare: {}".format(op_name)) + cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = compare_by_op( + op_name, dump_path_dict, input_parma) + if is_print_compare_log: + logger.info( + "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " + "five_thousand_err_ratio {}".format(op_name, cos_sim, max_abs_err, max_relative_err, err_msg, + one_thousand_err_ratio, five_thousand_err_ratio)) + cos_result.append(cos_sim) + max_err_result.append(max_abs_err) + max_relative_err_result.append(max_relative_err) + err_mess.append(err_msg) + one_thousand_err_ratio_result.append(one_thousand_err_ratio) + five_thousand_err_ratio_result.append(five_thousand_err_ratio) + + cr = ComparisonResult( + cos_result=cos_result, + max_err_result=max_err_result, + max_relative_err_result=max_relative_err_result, + err_msgs=err_mess, + one_thousand_err_ratio_result=one_thousand_err_ratio_result, + five_thousand_err_ratio_result=five_thousand_err_ratio_result + ) + + return _save_cmp_result(idx, cr, result_df, lock) + + +@dataclass +class ComparisonResult: + cos_result: list + max_err_result: list + max_relative_err_result: list + err_msgs: list + one_thousand_err_ratio_result: list + five_thousand_err_ratio_result: list + + +def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): + """ + Save comparison results into the result DataFrame with thread safety. + Args: + offset: offset for index + result: data struct of ComparisonResult + result_df: result of DataFrame + lock: thread lock + + Returns: + comparison results in DataFrame + """ + + lock.acquire() + try: + for i, _ in enumerate(result.cos_result): + process_index = i + offset + result_df.loc[process_index, CompareConst.COSINE] = result.cos_result[i] + result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] + result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] + result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] + result_df.loc[process_index, CompareConst.ACCURACY] = check_accuracy(result.cos_result[i], result.max_err_result[i]) + result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = result.one_thousand_err_ratio_result[i] + result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = result.five_thousand_err_ratio_result[i] + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + finally: + lock.release() + + +def check_accuracy(cos, max_abs_err): + if cos == CompareConst.SHAPE_UNMATCH: + return CompareConst.ACCURACY_CHECK_UNMATCH + if cos == CompareConst.NONE or max_abs_err == CompareConst.NONE: + return CompareConst.NONE + if cos == "N/A" or max_abs_err == "N/A": + return CompareConst.ACCURACY_CHECK_NO + try: + cos, max_abs_err = float(cos), float(max_abs_err) + except ValueError: + logger.warning("Cosine or MaxAbsErr can not get float value.") + return CompareConst.NONE + if cos < CompareConst.COS_THRESHOLD and max_abs_err > CompareConst.MAX_ABS_ERR_THRESHOLD: + return CompareConst.ACCURACY_CHECK_NO + if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: + return CompareConst.ACCURACY_CHECK_NO + return CompareConst.ACCURACY_CHECK_YES + + +def read_npy_data(dir_path, file_name): + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.NUMPY_SUFFIX, False) + data_path = path_checker.common_check() + data_value = np.load(data_path) # detach for less memory + if data_value.dtype == np.float16: + data_value=data_value.astype(np.float32) + + return data_value + + +def compare_by_op(op_name, op_name_mapping_dict, input_parma): + npu_bench_name_list = op_name_mapping_dict[op_name] + data_name = npu_bench_name_list[1] + error_file, relative_err, error_flag = None, None, False + if data_name == '-1' or data_name == -1: # 没有真实数据路径 + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + else: + try: + n_value = read_npy_data(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) + b_value = read_npy_data(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) + except IOError as error: + error_file = error.filename + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + except FileCheckException: + error_file = data_name + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + + n_value, b_value, error_flag = get_error_type(n_value, b_value, error_flag) + if not error_flag: + relative_err = get_relative_err(n_value, b_value) + n_value, b_value = reshape_value(n_value, b_value) + + err_msg = get_error_message(n_value, b_value, op_name, error_flag, error_file=error_file) + result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=relative_err) + + if npu_bench_name_list[0] != npu_bench_name_list[1]: + err_msg += " Fuzzy matching data, the comparison accuracy may be affected." + result_list.append(err_msg) + return result_list + + +def handle_inf_nan(n_value, b_value): + n_inf = np.isinf(n_value) + b_inf = np.isinf(b_value) + n_nan = np.isnan(n_value) + b_nan = np.isnan(b_value) + + # merge boolean expressions + any_inf = np.any(n_inf) or np.any(b_inf) + any_nan = np.any(n_nan) or np.any(b_nan) + if any_inf or any_nan: + if np.array_equal(n_inf, b_inf) and np.array_equal(n_nan, b_nan): + n_value[n_inf] = 0 + b_value[b_inf] = 0 + n_value[n_nan] = 0 + b_value[b_nan] = 0 + else: + return CompareConst.NAN, CompareConst.NAN + return n_value, b_value + + +def find_error_rows(result, last_len, n_num_input, highlight_dict, summary_compare=False, md5_compare=False): + """找到单个API中需要高亮的行""" + if md5_compare: + return + npu_max_index = get_header_index('NPU max', summary_compare) + bench_max_index = get_header_index('Bench max', summary_compare) + max_diff_index = get_header_index('Max diff' if summary_compare else 'MaxAbsErr', summary_compare) + + red_lines, yellow_lines = [], [] + LineInfo = namedtuple('LineInfo', ['line_data', 'num_pointer']) + ApiInfo = namedtuple('ApiInfo', ['api_input', 'api_output', 'num_pointer']) + ColorColumns = namedtuple('ColorColumns', ['red', 'yellow']) + color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) + + # 对单行API的输入或输出进行误差判断 + for i, line in enumerate(result): + num = last_len + i + line_info = LineInfo(line_data=line, num_pointer=num) + for rule in HighlightRules.basic_rules.values(): + rule.apply(line_info, color_columns, summary_compare) + + # 对API的输出与输入比较，进行误差判断 + for n, api_out in enumerate(result[n_num_input:len(result)]): + num = last_len + n_num_input + n + if num in red_lines: + continue + if not isinstance(api_out[npu_max_index], (float, int)) \ + or not isinstance(api_out[bench_max_index], (float, int)) \ + or not isinstance(api_out[max_diff_index], (float, int)): + continue + for _, api_in in enumerate(result[0:n_num_input]): + if not isinstance(api_in[npu_max_index], (float, int)) \ + or not isinstance(api_in[bench_max_index], (float, int)) \ + or not isinstance(api_in[max_diff_index], (float, int)): + continue + + api_info = ApiInfo(api_input=api_in, api_output=api_out, num_pointer=num) + if summary_compare: + for rule in HighlightRules.summary_compare_rules.values(): + rule.apply(api_info, color_columns, summary_compare) + else: + for rule in HighlightRules.compare_rules.values(): + rule.apply(api_info, color_columns, summary_compare) + + highlight_dict.get('red_rows', []).extend(list(set(red_lines))) + highlight_dict.get('yellow_rows', []).extend(list(set(yellow_lines) - set(red_lines))) + + +def get_name_and_state(name): + """Get api/module name and state""" + if "input" in name: + api_name = name.split("input")[0] + state = "input" + else: + api_name = name.split("output")[0] + state = "output" + return api_name, state + + +def find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare): + """将dataframe根据API分组，并找到有误差的算子用于高亮""" + result = result_df.values + start, input_num, output_num, end = 0, 0, 0, len(result_df) + last_api_name, last_state = None, None + num, last_len = 0, 0 + for res_i in result: + api_name, state = get_name_and_state(res_i[0]) + if last_api_name: + if api_name == last_api_name: + if state == last_state: + num += 1 + else: + input_num = num + num, last_state = 1, state + else: + output_num = num + find_error_rows(result[start:start + input_num + output_num], start, input_num, highlight_dict, + summary_compare, md5_compare) + num, last_api_name, last_state = 1, api_name, state + start += input_num + output_num + input_num, output_num = 1, 0 + else: + num, last_api_name, last_state = 1, api_name, state + if state: + if state == "input": + input_num = num + else: + output_num = num + find_error_rows(result[start:start + input_num + output_num], start, input_num, highlight_dict, summary_compare, md5_compare) + + +def highlight_rows_xlsx(result_df, highlight_dict, file_path): + """Write and highlight results in Excel""" + logger.info('Compare result is %s' % file_path) + + wb = openpyxl.Workbook() + ws = wb.active + + # write header + for j, col_name in enumerate(result_df.columns, start=1): + ws.cell(row=1, column=j, value=col_name) + + for i, row in enumerate(result_df.iterrows(), start=2): + for j, value in enumerate(row[1], start=1): + if not isinstance(value, (float, int)): + value = f'{str(value)}\t' if str(value) in ('inf', '-inf', 'nan') else str(value) + ws.cell(row=i, column=j, value=f'{str(value)}\t' if str(value) in ('inf', '-inf', 'nan') else value) + + if (i - 2) in highlight_dict['red_rows']: + ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.RED, + end_color=CompareConst.RED, fill_type="solid") + elif (i - 2) in highlight_dict['yellow_rows']: + ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.YELLOW, + end_color=CompareConst.YELLOW, fill_type="solid") + wb.save(file_path) + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + + +def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, + fuzzy_match=False): + try: + summary_compare, md5_compare = task_dumppath_get(input_parma) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + create_directory(output_path) + check_compare_param(input_parma, output_path, stack_mode, summary_compare, md5_compare) + except CompareException as error: + logger.error('Compare failed. Please check the arguments and do it again!') + sys.exit(error.code) + compare_core(input_parma, output_path, stack_mode=stack_mode, + auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) + + +def compare_core(input_parma, output_path, **kwargs): + """ + Compares data from multiple JSON files and generates a comparison report. + + Args: + input_parma (dict): A dictionary containing paths to JSON files ("npu_json_path", "bench_json_path", + "stack_json_path"). + output_path (str): The path where the output Excel report will be saved. + **kwargs: Additional keyword arguments including: + - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. + - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. + - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. + - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. + - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. + - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. + + Returns: + """ + # get kwargs or set default value + stack_mode = kwargs.get('stack_mode', False) + auto_analyze = kwargs.get('auto_analyze', True) + suffix = kwargs.get('suffix', '') + fuzzy_match = kwargs.get('fuzzy_match', False) + summary_compare = kwargs.get('summary_compare', False) + md5_compare = kwargs.get('md5_compare', False) + + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") + file_name = add_time_with_xlsx("compare_result" + suffix) + file_path = os.path.join(os.path.realpath(output_path), file_name) + check_file_not_exists(file_path) + highlight_dict = {'red_rows': [], 'yellow_rows': []} + + with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_json_path"), "r") as stack_json: + result_df = compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, + summary_compare, md5_compare) + + if not md5_compare and not summary_compare: + result_df = _do_multi_process(input_parma, result_df) + find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) + highlight_rows_xlsx(result_df, highlight_dict, file_path) + if auto_analyze: + advisor = Advisor(result_df, output_path) + advisor.analysis() + + +def parse(pkl_file, module_name_prefix): + if not isinstance(module_name_prefix, str): + logger.error("The parameter:module_name_prefix is not a string.") + raise CompareException(CompareException.INVALID_PARAM_ERROR) + with FileOpen(pkl_file, "r") as f: + done = False + title_printed = False + while not done: + pkl_line = f.readline() + if pkl_line == '\n': + continue + if len(pkl_line) == 0: + done = True + break + + msg = json.loads(pkl_line) + info_prefix = msg[0] + if not info_prefix.startswith(module_name_prefix): + continue + + if info_prefix.find("stack_info") != -1: + logger.info("\nTrace back({}):".format(msg[0])) + for item in reversed(msg[1]): + logger.info(" File \"{}\", line {}, in {}".format(item[0], item[1], item[2])) + logger.info(" {}".format(item[3])) + continue + if len(msg) > 5: + summary_info = " [{}][dtype: {}][shape: {}][max: {}][min: {}][mean: {}]" \ + .format(msg[0], msg[3], msg[4], msg[5][0], msg[5][1], msg[5][2]) + if not title_printed: + logger.info("\nStatistic Info:") + title_printed = True + logger.info(summary_info) + + +def op_item_parse(item, op_name, index, item_list=None, top_bool=True): + if item_list is None: + item_list = [] + if item is None or (isinstance(item, dict) and not item): + if not top_bool: + tmp = {'full_op_name': op_name + '.' + str(index), 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, + 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'} + else: + tmp = {'full_op_name': op_name + '.0', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, + 'shape': None, 'md5': None, 'data_name': '-1'} + item_list.append(tmp) + return item_list + if index is None: + if isinstance(item, dict): + full_op_name = op_name + '.0' + else: + full_op_name = op_name + else: + full_op_name = op_name + Const.SEP + str(index) + if isinstance(item, dict): + if 'type' not in item: + for kwarg in item: + kwarg_parsed_list = op_item_parse(item[kwarg], op_name + Const.SEP + kwarg, None) + item_list += kwarg_parsed_list + kwarg_parsed_list.clear() + elif 'dtype' in item: + parsed_item = item + parsed_item['full_op_name'] = full_op_name + item_list.append(parsed_item) + elif 'type' in item: + parsed_item = {} + if item['type'] == 'torch.Size': + parsed_item['full_op_name'] = full_op_name + parsed_item['dtype'] = 'torch.Size' + parsed_item['shape'] = str(item['value']) + parsed_item['md5'] = None + parsed_item['Max'] = None + parsed_item['Min'] = None + parsed_item['Mean'] = None + parsed_item['Norm'] = None + parsed_item['data_name'] = '-1' + item_list.append(parsed_item) + elif item['type'] == 'slice': + parsed_item['full_op_name'] = full_op_name + parsed_item['dtype'] = 'slice' + parsed_item['shape'] = str(np.shape(np.array(item['value']))) + parsed_item['md5'] = None + parsed_item['Max'] = None + parsed_item['Min'] = None + parsed_item['Mean'] = None + parsed_item['Norm'] = None + parsed_item['data_name'] = '-1' + item_list.append(parsed_item) + else: + parsed_item['full_op_name'] = full_op_name + parsed_item['dtype'] = str(type(item['value'])) + parsed_item['shape'] = '[]' + parsed_item['md5'] = None + parsed_item['Max'] = item['value'] + parsed_item['Min'] = item['value'] + parsed_item['Mean'] = item['value'] + parsed_item['Norm'] = item['value'] + parsed_item['data_name'] = '-1' + item_list.append(parsed_item) + else: + resolve_api_special_parameters(item, full_op_name, item_list) + else: + for j, item_spec in enumerate(item): + op_item_parse(item_spec, full_op_name, j, item_list=item_list, top_bool=False) + return item_list + + +def resolve_api_special_parameters(data_dict, full_op_name, item_list): + """ + Function Description: + 解析下面格式的数据, 是api参数的一种特殊格式 + { + "last_hidden_state": { + "type": "torch.Tensor", + "dtype": "torch.bfloat16", + ... + }, + "loss": { + "type": "torch.Tensor", + "dtype": "torch.float32", + ... + } + } + Parameter: + data_dict: 字典格式的数据 + full_op_name: 参数的全名字符串 + item_list: 参数信息集合 + """ + for key, value in data_dict.items(): + if isinstance(value, dict): + parsed_item = value + parts = full_op_name.split(".") + parts.insert(-1, key) + full_op_name_new = ".".join(parts) + parsed_item['full_op_name'] = full_op_name_new + item_list.append(parsed_item) + + +def read_op(op_data, op_name): + op_parsed_list = [] + if 'forward' in op_name: + if 'input_args' in op_data: + input_item = op_data['input_args'] + input_parsed_list = op_item_parse(input_item, op_name + '_input', None) + op_parsed_list = input_parsed_list.copy() + input_parsed_list.clear() + if 'input_kwargs' in op_data: + kwargs_item = op_data['input_kwargs'] + if isinstance(kwargs_item, dict) and "type" in kwargs_item or isinstance(kwargs_item, list): + kwarg_parsed_list = op_item_parse(kwargs_item, op_name + '_input', None) + op_parsed_list += kwarg_parsed_list + kwarg_parsed_list.clear() + elif kwargs_item: + for kwarg in kwargs_item: + kwarg_parsed_list = op_item_parse(kwargs_item[kwarg], op_name + '_input.' + kwarg, None) + op_parsed_list += kwarg_parsed_list + kwarg_parsed_list.clear() + if 'output' in op_data: + output_item = op_data['output'] + output_parsed_list = op_item_parse(output_item, op_name + '_output', None) + op_parsed_list += output_parsed_list + output_parsed_list.clear() + if 'backward' in op_name: + if 'grad_input' in op_data: + input_item = op_data['grad_input'] + input_parsed_list = op_item_parse(input_item, op_name + '_input', None) + op_parsed_list = input_parsed_list.copy() + input_parsed_list.clear() + if 'grad_output' in op_data: + output_item = op_data['grad_output'] + output_parsed_list = op_item_parse(output_item, op_name + '_output', None) + op_parsed_list += output_parsed_list + output_parsed_list.clear() + return op_parsed_list + + +def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): + npu_json_handle, bench_json_handle, stack_json_handle = file_handles + npu_json_data = json.load(npu_json_handle) + bench_json_data = json.load(bench_json_handle) + stack_json_data = json.load(stack_json_handle) + + if fuzzy_match: + logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") + + npu_ops_queue = [] + bench_ops_queue = [] + result = [] + + ops_npu_iter = iter(npu_json_data['data']) + ops_bench_iter = iter(bench_json_data['data']) + read_err_npu = True + read_err_bench = True + last_npu_ops_len = 0 + last_bench_ops_len = 0 + + while True: + if not read_err_npu and not read_err_bench: + break + try: + last_npu_ops_len = len(npu_ops_queue) + op_name_npu = next(ops_npu_iter) + read_err_npu = True + + npu_op_data = npu_json_data['data'][op_name_npu] + npu_op_parsed_list = read_op(npu_op_data, op_name_npu) + if op_name_npu in stack_json_data: + npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': stack_json_data[op_name_npu]}) + else: + npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': None}) + + npu_merge_list = merge_tensor(npu_op_parsed_list, summary_compare, md5_compare) + if npu_merge_list: + npu_ops_queue.append(npu_merge_list) + except StopIteration: + read_err_npu = False + try: + last_bench_ops_len = len(bench_ops_queue) + op_name_bench = next(ops_bench_iter) + + bench_op_data = bench_json_data['data'][op_name_bench] + bench_op_parsed_list = read_op(bench_op_data, op_name_bench) + if op_name_bench in stack_json_data: + bench_op_parsed_list.append( + {'full_op_name': op_name_bench, 'full_info': stack_json_data[op_name_bench]}) + else: + bench_op_parsed_list.append({'full_op_name': op_name_bench, 'full_info': None}) + + bench_merge_list = merge_tensor(bench_op_parsed_list, summary_compare, md5_compare) + if bench_merge_list: + bench_ops_queue.append(bench_merge_list) + except StopIteration: + read_err_bench = False + + # merge all boolean expressions + both_empty = not npu_ops_queue and not bench_ops_queue + no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) + if both_empty or no_change: + continue + + n_match_point, b_match_point = match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) + if n_match_point == -1 and b_match_point == -1: + continue + n_match_data = npu_ops_queue[n_match_point] + b_match_data = bench_ops_queue[b_match_point] + un_match_data = npu_ops_queue[0: n_match_point] + for npu_data in un_match_data: + get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) + get_accuracy(result, n_match_data, b_match_data, summary_compare, md5_compare) + del npu_ops_queue[0: n_match_point + 1] + del bench_ops_queue[0: b_match_point + 1] + if npu_ops_queue: + for npu_data in npu_ops_queue: + get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) + + header = [] + if md5_compare: + header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] + elif summary_compare: + header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] + else: + header = CompareConst.COMPARE_RESULT_HEADER[:] + + all_mode_bool = not (summary_compare or md5_compare) + if stack_mode: + if all_mode_bool: + header.append(CompareConst.STACK) + header.append(CompareConst.DATA_NAME) + else: + header.append(CompareConst.STACK) + else: + if all_mode_bool: + for row in result: + del row[-2] + header.append(CompareConst.DATA_NAME) + else: + for row in result: + del row[-1] + + result_df = pd.DataFrame(result, columns=header) + return result_df + + +def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): + index_out = 0 + npu_stack_info = n_dict.get("stack_info", None) + bench_name, bench_type, bench_shape = CompareConst.NAN, CompareConst.NAN, CompareConst.NAN + err_msg = CompareConst.NO_BENCH + accuracy_check_res = CompareConst.NAN + for index, n_name in enumerate(n_dict["op_name"]): + if n_name.find("input") != -1: + n_struct = n_dict["input_struct"][index] + else: + n_struct = n_dict["output_struct"][index_out] + index_out += 1 + + result_item = [n_name, bench_name, n_struct[0], bench_type, n_struct[1], bench_shape] + if md5_compare: + result_item.extend([CompareConst.NAN] * 3) + if npu_stack_info and index == 0: + result_item.extend(npu_stack_info) + result.append(result_item) + continue + if summary_compare: + result_item.extend([CompareConst.NAN] * 8) + else: + result_item.extend([CompareConst.NAN] * 5) + summary_data = n_dict.get("summary")[index] + result_item.extend(summary_data) + summary_data = [CompareConst.NAN] * 4 + result_item.extend(summary_data) + result_item.append(accuracy_check_res) + result_item.append(err_msg) + if npu_stack_info and index == 0: + result_item.extend(npu_stack_info) + if not md5_compare and not summary_compare and result_item[1] == CompareConst.NAN: + if index == 0: + result_item.extend(["-1"]) + else: + result_item.extend([CompareConst.NONE, "-1"]) + result.append(result_item) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py new file mode 100644 index 000000000..4246bdd2b --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2019-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import os +import sys +import re +from msprobe.core.common.utils import CompareException, check_compare_param, \ + check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid +from msprobe.mindspore.compare.acc_compare import compare_core +from msprobe.core.common.file_check import create_directory +from msprobe.mindspore.common.log import logger + + +def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): + def check_and_return_dir_contents(dump_dir, prefix): + """ + check the given dump dir and validate files in dump dir by using the given prefix patterns to build a + pattern: ^{prefix}(?:0|[0-9][1-9]*)?$ + + Args: + dump_dir (str): dump dir + prefix (str): prefix for the patterns, prefix should be less than 20 characters and alphanumeric/-/_ only + + Returns: + content [list]: dir contents + Raises: + CompareException: invalid path + ValueError: prefix not match the patterns + + """ + check_regex_prefix_format_valid(prefix) + check_file_or_directory_path(dump_dir, True) + contents = os.listdir(dump_dir) + pattern = re.compile(rf'^{prefix}(?:0|[0-9][1-9]*)?$') + for name in contents: + if not pattern.match(name): + logger.error( + f"dump_dir contains '{name}'. Expected '{prefix}'. This name is not in the format of dump " + f"output. Please check and delete irrelevant files in {dump_dir} and try again." + ) + raise CompareException(CompareException.INVALID_PATH_ERROR) + return contents + + def extract_json(dirname, stack_json=False): + json_path = '' + for fname in os.listdir(dirname): + full_path = os.path.join(dirname, fname) + if fname=="construct.json": continue + if full_path.endswith('.json'): + json_path = full_path + if not stack_json and 'stack' not in json_path: + break + if stack_json and 'stack' in json_path: + break + + # Provide robustness on invalid directory inputs + if not json_path: + logger.error(f'No file is found in dump dir {dirname}. ') + raise CompareException(CompareException.NO_DUMP_FILE_ERROR) + return json_path + + if kwargs.get('suffix'): + logger.error("Argument 'suffix' is not supported for compare_distributed.") + raise CompareException(CompareException.INVALID_PARAM_ERROR) + stack_mode = kwargs.get('stack_mode', False) + auto_analyze = kwargs.get('auto_analyze', True) + fuzzy_match = kwargs.get('fuzzy_match', False) + # get the ranks and match by order + npu_ranks = sorted(check_and_return_dir_contents(npu_dump_dir, 'rank')) + bench_ranks = sorted(check_and_return_dir_contents(bench_dump_dir, 'rank')) + if len(npu_ranks) != len(bench_ranks): + logger.error('The number of ranks in the two runs are different. ' + 'Unable to match the ranks. Please use another folder to compare ' + 'or use compare() api and manually match the ranks.') + raise CompareException(CompareException.INVALID_PATH_ERROR) + for nr, br in zip(npu_ranks, bench_ranks): + n_dir = os.path.join(npu_dump_dir, nr) + b_dir = os.path.join(bench_dump_dir, br) + s_dir = b_dir + npu_json_path = extract_json(n_dir, stack_json=False) + bench_json_path = extract_json(b_dir, stack_json=False) + stack_json_path = extract_json(s_dir, stack_json=True) + + dump_result_param = { + 'npu_json_path': npu_json_path, + 'bench_json_path': bench_json_path, + 'stack_json_path': stack_json_path, + 'is_print_compare_log': True + } + try: + summary_compare, md5_compare = task_dumppath_get(dump_result_param) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + create_directory(output_path) + check_compare_param(dump_result_param, output_path, stack_mode=stack_mode, summary_compare=summary_compare) + except CompareException as error: + logger.error('Compare failed. Please check the arguments and do it again!') + sys.exit(error.code) + compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, + md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/highlight.py b/debug/accuracy_tools/msprobe/mindspore/compare/highlight.py new file mode 100644 index 000000000..82f0022f8 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/highlight.py @@ -0,0 +1,100 @@ +import math +import abc +import numpy as np +from msprobe.core.common.utils import get_header_index +from msprobe.core.common.const import CompareConst + + +class HighlightCheck(abc.ABC): + @abc.abstractmethod + def apply(self, info, color_columns, summary_compare): + raise NotImplementedError + + +class CheckOrderMagnitude(HighlightCheck): + """检查Max diff的数量级差异""" + def apply(self, info, color_columns, summary_compare=True): + api_in, api_out, num = info + max_diff_index = get_header_index('Max diff' if summary_compare else 'MaxAbsErr', summary_compare) + if abs(api_in[max_diff_index]) > abs(api_out[max_diff_index]): + return + in_order = 0 if abs(api_in[max_diff_index]) < 1 else math.log10(abs(api_in[max_diff_index])) + out_order = 0 if abs(api_out[max_diff_index]) < 1 else math.log10(abs(api_out[max_diff_index])) + if out_order - in_order >= CompareConst.ORDER_MAGNITUDE_DIFF_YELLOW: + color_columns.yellow.append(num) + + +class CheckOneThousandErrorRatio(HighlightCheck): + """检查千分误差比率""" + def apply(self, info, color_columns, summary_compare=True): + api_in, api_out, num = info + one_thousand_index = get_header_index('One Thousandth Err Ratio', summary_compare) + if not isinstance(api_in[one_thousand_index], (float, int)) or not isinstance(api_out[one_thousand_index], (float, int)): + return + if api_in[one_thousand_index] > CompareConst.ONE_THOUSAND_ERROR_IN_RED and api_out[one_thousand_index] < CompareConst.ONE_THOUSAND_ERROR_OUT_RED: + color_columns.red.append(num) + elif api_in[one_thousand_index] - api_out[one_thousand_index] > CompareConst.ONE_THOUSAND_ERROR_DIFF_YELLOW: + color_columns.yellow.append(num) + + +class CheckCosineSimilarity(HighlightCheck): + """检查余弦相似度""" + def apply(self, info, color_columns, summary_compare=True): + api_in, api_out, num = info + cosine_index = get_header_index('Cosine', summary_compare) + if not isinstance(api_in[cosine_index], (float, int)) or not isinstance(api_out[cosine_index], (float, int)): + return + if api_in[cosine_index] - api_out[cosine_index] > CompareConst.COSINE_DIFF_YELLOW: + color_columns.yellow.append(num) + + +class CheckMaxRelativeDiff(HighlightCheck): + """检查最大相对差异""" + def apply(self, info, color_columns, summary_compare=True): + api_in, api_out, num = info + max_diff_index = get_header_index('Max diff', summary_compare) + bench_max_index = get_header_index('Bench max', summary_compare) + input_max_relative_diff = np.abs(np.divide(api_in[max_diff_index], max(0.01, api_in[bench_max_index]))) + output_max_relative_diff = np.abs(np.divide(api_out[max_diff_index], max(0.01, api_out[bench_max_index]))) + if not isinstance(input_max_relative_diff, (float, int)) or not isinstance(output_max_relative_diff, + (float, int)): + return + if output_max_relative_diff > CompareConst.MAX_RELATIVE_OUT_RED: + color_columns.red.append(num) + elif output_max_relative_diff > CompareConst.MAX_RELATIVE_OUT_YELLOW and input_max_relative_diff < CompareConst.MAX_RELATIVE_IN_YELLOW: + color_columns.yellow.append(num) + + +class CheckOverflow(HighlightCheck): + """检查是否存在溢出""" + def apply(self, info, color_columns, summary_compare=True): + line, num = info + npu_max_index = get_header_index('NPU max', summary_compare) + npu_min_index = get_header_index('NPU min', summary_compare) + max_diff_index = get_header_index('Max diff' if summary_compare else 'MaxAbsErr', summary_compare) + if str(line[npu_max_index]) in CompareConst.OVERFLOW_LIST or str( + line[npu_min_index]) in CompareConst.OVERFLOW_LIST: + color_columns.red.append(num) + return + # check if Max_Diff > 1e+10 + if isinstance(line[max_diff_index], (float, int)) and line[max_diff_index] > CompareConst.MAX_DIFF_RED: + color_columns.red.append(num) + + +class HighlightRules: + """高亮规则集合，用于检查API的误差""" + # 适用于每行的规则 + basic_rules = { + "check_overflow": CheckOverflow() + } + + # 用于比较输入和输出的规则 + compare_rules = { + "check_order_magnitude": CheckOrderMagnitude(), + "check_one_thousand_error": CheckOneThousandErrorRatio(), + "check_cosine_similarity": CheckCosineSimilarity() + } + summary_compare_rules = { + "check_order_magnitude": CheckOrderMagnitude(), + "check_max_relative_diff": CheckMaxRelativeDiff(), + } diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/mapping.yaml b/debug/accuracy_tools/msprobe/mindspore/compare/mapping.yaml new file mode 100644 index 000000000..eaffbe7a1 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/mapping.yaml @@ -0,0 +1,607 @@ +__and__: __and__ +__iand__: __iand__ +__ilshift__: __ilshift__ +__ior__: __ior__ +__irshift__: __irshift__ +__ixor__: __ixor__ +__lshift__: __lshift__ +__or__: __or__ +__rshift__: __rshift__ +__xor__: __xor__ +_adaptive_avg_pool2d: adaptive_avg_pool2d +_adaptive_avg_pool3d: adaptive_avg_pool3d +_cdist_forward: cdist +_cudnn_rnn: rnn +_embedding_bag: embedding_bag +_fft_c2c: fft +_fft_c2r: rfft +_foreach_add_: _foreach_add_ +_foreach_addcdiv: _foreach_addcdiv +_foreach_copy_: _foreach_copy_ +_foreach_lerp_: _foreach_lerp_ +_foreach_maximum: _foreach_maximum +_foreach_mul: _foreach_mul +_foreach_neg_: _foreach_neg_ +_foreach_pow: _foreach_pow +_foreach_reciprocal_: _foreach_reciprocal_ +_foreach_sign: _foreach_sign +_foreach_sqrt: _foreach_sqrt +_foreach_sqrt_: _foreach_sqrt_ +_foreach_sub: _foreach_sub +_fused_adam: FusedAdam +_linalg_det: det +_linalg_eigh: eigh +_linalg_slogdet: slogdet +_linalg_svd: svd +_list_to_tensor: as_tensor +_log_softmax: log_softmax +_native_batch_norm_legit: batch_norm +_nested_tensor_from_tensor_list: _nested_tensor_from_tensor_list +_pdist_forward: pdist +_pin_memory: pin_memory +_reshape_alias: reshape +_resize_output_: resize_ +_softmax: softmax +_to_copy: to +abs: abs +abs_: abs_ +absolute: abs +absolute_: abs_ +acos: acos +acos_: acos_ +acosh: acosh +acosh_: acosh_ +adaptive_max_pool2d: adaptive_max_pool2d +adaptive_max_pool3d: adaptive_max_pool3d +add: add +add_: add_ +addbmm: addbmm +addbmm_: addbmm_ +addcdiv: addcdiv +addcdiv_: addcdiv_ +addcmul: addcmul +addcmul_: addcmul_ +addmm: addmm +addmm_: addmm_ +addmv: addmv +addmv_: addmv_ +addr: addr +affine_grid_generator: affine_grid +alias: alias +all: all +alpha_dropout: AlphaDropout +amax: amax +amin: amin +aminmax: aminmax +angle: angle +any: any +arange: arange +arccos: acos +arccos_: arccos_ +arccosh: arccosh +arccosh_: arccosh_ +arcsin: asin +arcsin_: arcsin_ +arcsinh: asinh +arcsinh_: arcsinh_ +arctan: atan +arctan2: atan2 +arctan2_: arctan2_ +arctan_: arctan_ +arctanh: arctanh +arctanh_: arctanh_ +argmax: argmax +argmin: argmin +argsort: argsort +as_strided: as_strided +asin: asin +asin_: asin_ +asinh: asinh +asinh_: asinh_ +atan: atan +atan2: atan2 +atan2_: atan2_ +atan_: atan_ +atanh: atanh +atanh_: atanh_ +avg_pool2d: avg_pool2d +avg_pool3d: avg_pool3d +baddbmm: baddbmm +baddbmm_: baddbmm_ +bernoulli: bernoulli +bernoulli_: bernoulli_ +binary_cross_entropy: BCELoss +binary_cross_entropy_with_logits: binary_cross_entropy_with_logits +bitwise_and: bitwise_and +bitwise_and_: bitwise_and_ +bitwise_left_shift: __lshift__ +bitwise_left_shift_: bitwise_left_shift_ +bitwise_not: bitwise_not +bitwise_not_: bitwise_not_ +bitwise_or: bitwise_or +bitwise_or_: bitwise_or_ +bitwise_right_shift: __rshift__ +bitwise_right_shift_: bitwise_right_shift_ +bitwise_xor: bitwise_xor +bitwise_xor_: bitwise_xor_ +bmm: bmm +broadcast_tensors: broadcast_tensors +bucketize: bucketize +cat: cat +cauchy: Cauchy +cauchy_: cauchy_ +ceil: ceil +ceil_: ceil_ +celu: celu +celu_: celu_ +cholesky: cholesky +cholesky_inverse: cholesky_inverse +cholesky_solve: cholesky_solve +clamp: clamp +clamp_: clamp_ +clamp_max: clamp_max +clamp_max_: clamp_max_ +clamp_min: clamp_min +clamp_min_: clamp_min_ +clip: clip +clip_: clip_ +clone: clone +col2im: col2im +complex: complex +conj_physical: conj +conj_physical_: conj_ +constant_pad_nd: pad +convolution: Conv2d +copy: copy_ +copy_: copy_ +copysign: copysign +copysign_: copysign_ +cos: cos +cos_: cos_ +cosh: cosh +cosh_: cosh_ +count_nonzero: count_nonzero +cudnn_batch_norm: BatchNorm2d +cummax: cummax +cummin: cummin +cumprod: cumprod +cumprod_: cumprod_ +cumsum: cumsum +cumsum_: cumsum_ +deg2rad: deg2rad +deg2rad_: deg2rad_ +detach: detach +diag: diag +diag_embed: diag_embed +diagonal: diagonal +diagonal_copy: diagonal +diagonal_scatter: diagonal +digamma: digamma +digamma_: digamma_ +dist: dist +div: div +div_: div_ +divide: div +divide_: divide_ +dot: dot +dropout: dropout +elu: ELU +elu_: elu_ +embedding: embedding +empty_like: empty_like +empty_strided: empty_strided +eq: eq +eq_: eq_ +erf: erf +erf_: erf_ +erfc: erfc +erfc_: erfc_ +erfinv: erfinv +erfinv_: erfinv_ +exp: exp +exp2: exp2 +exp2_: exp2_ +exp_: exp_ +expand: expand +expm1: expm1 +expm1_: expm1_ +exponential: Exponential +exponential_: exponential_ +eye: eye +fft_fft: fft +fft_fft2: fft2 +fft_fftn: fftn +fft_fftshift: fftshift +fft_hfft: hfft +fft_hfft2: hfft2 +fft_hfftn: hfftn +fft_ifft: ifft +fft_ifft2: ifft2 +fft_ifftn: ifftn +fft_ifftshift: ifftshift +fft_ihfft: ihfft +fft_ihfft2: ihfft2 +fft_ihfftn: ifftn +fft_irfft: irfft +fft_irfft2: irfft2 +fft_irfftn: irfftn +fft_rfft: rfft +fft_rfft2: rfft2 +fft_rfftn: rfftn +fill: fill_ +fill_: fill_ +fix: fix +fix_: fix_ +flip: flip +float_power_: float_power_ +floor: floor +floor_: floor_ +floor_divide: floor_divide +floor_divide_: floor_divide_ +fmax: fmax +fmin: fmin +fmod: fmod +fmod_: fmod_ +frac: frac +frac_: frac_ +full: full +full_like: full_like +gather: gather +gcd: gcd +gcd_: gcd_ +ge: ge +ge_: ge_ +gelu: GELU +gelu_: gelu_ +geometric: Geometric +geometric_: geometric_ +glu: glu +greater: gt +greater_: ge_ +greater_equal: ge +greater_equal_: ge_ +grid_sampler_2d: grid_sample +grid_sampler_3d: grid_sample +gru: GRU +gt: gt +gt_: gt_ +hardshrink: Hardshrink +hardsigmoid: hardsigmoid +hardsigmoid_: hardsigmoid_ +hardswish: hardswish +hardswish_: hardswish_ +hardtanh: hardtanh +hardtanh_: hardtanh_ +heaviside: heaviside +heaviside_: heaviside_ +hinge_embedding_loss: HingeEmbeddingLoss +huber_loss: huber_loss +hypot: hypot +hypot_: hypot_ +i0: i0 +i0_: i0_ +igamma: igamma +igamma_: igamma_ +igammac: igammac +igammac_: igammac_ +index: __getitem__ +index_add: index_add +index_add_: index_add_ +index_copy: index_copy_ +index_copy_: index_copy_ +index_fill: index_fill_ +index_fill_: index_fill_ +index_put: index_put_ +index_put_: index_put_ +index_reduce: index_select +index_select: index_select +is_pinned: is_pinned +is_same_size: is_same_size +isinf: isinf +isnan: isnan +isneginf: isneginf +isposinf: isposinf +istft: istft +item: item +lcm: lcm +lcm_: lcm_ +le: le +le_: le_ +leaky_relu: LeakyReLU +leaky_relu_: leaky_relu_ +lerp: lerp +lerp_: lerp_ +less: less +less_: less_ +less_equal: le +less_equal_: less_equal_ +lgamma: lgamma +lgamma_: lgamma_ +linalg_cholesky_ex: cholesky +linalg_cross: cross +linalg_householder_product: householder_product +linalg_inv_ex: inv +linalg_ldl_factor_ex: ldl +linalg_ldl_solve: ldl_solve +linalg_lu: lu +linalg_lu_factor_ex: lu_factor +linalg_lu_solve: lu_solve +linalg_matrix_exp: matrix_exp +linalg_qr: qr +linalg_solve_triangular: solve +linalg_vector_norm: norm +linspace: linspace +log: log +log10: log10 +log10_: log10_ +log1p: log1p +log1p_: log1p_ +log2: log2 +log2_: log2_ +log_: log_ +log_normal: LogNormal +log_sigmoid_forward: log_sigmoid +logaddexp: logaddexp +logaddexp2: logaddexp2 +_native_batch_norm_legit_functional: batch_norm +logcumsumexp: logcumsumexp +logical_and: logical_and +logical_and_: logical_and_ +logical_not: logical_not +logical_not_: logical_not_ +logical_or: logical_or +logical_or_: logical_or_ +logical_xor: logical_xor +logical_xor_: logical_xor_ +logit: logit +logit_: logit_ +logspace: logspace +logsumexp: logsumexp +lstm: LSTM +lt: lt +lt_: lt_ +lu_unpack: lu_unpack +margin_ranking_loss: margin_ranking_loss +masked_fill: masked_fill +masked_fill_: masked_fill_ +matmul: matmul +max: max +max_pool2d_with_indices: MaxPool2d +max_pool3d_with_indices: MaxPool3d +max_unpool2d: MaxUnpool2d +max_unpool3d: max_unpool3d +maximum: maximum +mean: mean +median: median +meshgrid: meshgrid +min: min +minimum: minimum +mish: Mish +mish_: mish_ +mm: mm +mode: mode +mse_loss: mse_loss +mul: mul +mul_: mul_ +multi_margin_loss: MultiMarginLoss +multilabel_margin_loss_forward: multilabel_margin_loss +multinomial: multinomial +multiply: multiply +multiply_: mul_ +mv: mv +mvlgamma: mvlgamma +mvlgamma_: mvlgamma_ +name: name +nan_to_num: nan_to_num +nan_to_num_: nan_to_num_ +nanmedian: nanmedian +nansum: nansum +narrow_copy: narrow +native_batch_norm: BatchNorm2d +native_dropout: dropout +native_group_norm: group_norm +native_layer_norm: LayerNorm +ne: ne +ne_: ne_ +neg: neg +neg_: neg_ +negative: neg +negative_: neg_ +new_empty: new_empty +new_empty_strided: new_empty_strided +new_full: new_full +new_ones: new_ones +new_zeros: new_zeros +nextafter: nextafter +nextafter_: nextafter_ +nll_loss: nll_loss +nll_loss2d_forward: NLLLoss2d +nll_loss_forward: NLLLoss +nonzero_static: nonzero +norm: norm +normal: normal +normal_: normal_ +not_equal: ne +not_equal_: ne_ +ones: ones +ones_like: ones_like +ormqr: ormqr +pairwise_distance: pairwise_distance +pdist: pdist +permute: permute +pin_memory: pin_memory +pixel_shuffle: PixelShuffle +polar: polar +polygamma: polygamma +positive: positive +pow: pow +pow_: pow_ +prelu: prelu +prod: prod +quantized_gru: GRU +quantized_lstm: LSTM +rad2deg: rad2deg +rad2deg_: rad2deg_ +rand: rand +rand_like: rand_like +randint: randint +randint_like: randint_like +randn: randn +randn_like: randn_like +randperm: randperm +reciprocal: reciprocal +reciprocal_: reciprocal_ +reflection_pad1d: reflection_pad1d +reflection_pad2d: reflection_pad2d +reflection_pad3d: ReflectionPad3d +relu: relu +relu6: relu6 +relu_: relu_ +remainder: remainder +remainder_: remainder_ +renorm: renorm +renorm_: renorm_ +repeat: repeat +repeat_interleave: repeat_interleave +replication_pad1d: ReplicationPad1d +replication_pad2d: replication_pad2d +replication_pad3d: replication_pad3d +resize_as_: resize_as_ +rnn_relu: RNN +rnn_tanh: RNN +roll: roll +rot90: rot90 +round: round +round_: round_ +rrelu_with_noise: RReLU +rrelu_with_noise_: rrelu_with_noise +rsqrt: rsqrt +rsqrt_: rsqrt_ +rsub: rsub +scalar_tensor: scalar_tensor +scatter: scatter_ +scatter_: scatter_ +scatter_add: scatter_add +scatter_add_: scatter_add_ +searchsorted: searchsorted +select: select +selu: selu +selu_: selu_ +sgn: sgn +sgn_: sgn_ +sigmoid: sigmoid +sigmoid_: sigmoid_ +sign: sign +sign_: sign_ +signbit: signbit +silu: silu +silu_: silu_ +sin: sin +sin_: sin_ +sinc: sinc +sinc_: sinc_ +sinh: sinh +sinh_: sinh_ +slice: slice +smooth_l1_loss: smooth_l1_loss +soft_margin_loss: soft_margin_loss +softplus: softplus +softshrink: softshrink +sort: sort +special_airy_ai: airy_ai +special_bessel_j0: j0 +special_bessel_j1: j1 +special_bessel_y0: y0 +special_bessel_y1: y1 +special_chebyshev_polynomial_t: chebyshev_t +special_chebyshev_polynomial_u: chebyshev_u +special_entr: entr +special_erfcx: erfcx +special_hermite_polynomial_h: hermite +special_hermite_polynomial_he: he +special_i0: i0 +special_i0e: i0e +special_i1: i1 +special_i1e: i1e +special_laguerre_polynomial_l: laguerre_l +special_log_ndtr: log_ndtr +special_modified_bessel_i0: i0 +special_modified_bessel_i1: i1 +special_modified_bessel_k0: k0 +special_modified_bessel_k1: i1 +special_ndtr: ndtr +special_ndtri: ndtri +special_scaled_modified_bessel_k0: i0e +special_scaled_modified_bessel_k1: scaled_modified_bessel_k1 +special_spherical_bessel_j0: spherical_jn +special_xlog1py: xlog1py +special_zeta: zeta +split: split +split_with_sizes: split +sqrt: sqrt +sqrt_: sqrt_ +square: square +square_: square_ +squeeze: squeeze +stack: stack +std: std +std_mean: std_mean +stft: stft +sub: sub +sub_: sub_ +subtract: sub +subtract_: subtract_ +sum: sum +t: t +t_: t_ +take: take +tan: tan +tan_: tan_ +tanh: tanh +tanh_: tanh_ +threshold: threshold +threshold_: threshold_ +to: to +topk: topk +trace: trace +transpose: transpose +transpose_: transpose_ +triangular_solve: triangular_solve +tril: tril +tril_: tril_ +tril_indices: tril_indices +triu: triu +triu_: triu_ +triu_indices: triu_indices +true_divide: true_divide +true_divide_: true_divide_ +trunc: trunc +trunc_: trunc_ +unbind: unbind +unfold: unfold +uniform: Uniform +uniform_: uniform_ +unsafe_chunk: unsafe_chunk +unsafe_split: split +unsafe_split_with_sizes: split_with_sizes +unsqueeze: unsqueeze +unsqueeze_: unsqueeze_ +upsample_bicubic2d: interpolate +upsample_bilinear2d: upsample_bilinear +upsample_nearest1d: interpolate +upsample_nearest2d: interpolate +upsample_nearest3d: interpolate +var: var +var_mean: var_mean +vdot: vdot +view: view +where: where +xlogy: xlogy +xlogy_: xlogy_ +zero: zeros +zero_: zero_ +zeros: zeros +zeros_like: zeros_like + + + diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/match.py b/debug/accuracy_tools/msprobe/mindspore/compare/match.py new file mode 100644 index 000000000..6347d8887 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/match.py @@ -0,0 +1,36 @@ +import os +import yaml +from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import CompareException + + +class AtenIrMapping(): + def __init__(self): + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(cur_path, "mapping.yaml") + with FileOpen(yaml_path, 'r') as f: + self.aten_mapping = yaml.safe_load(f) + + def match(self, op1, op2): + if "Aten" in op1 and "Aten" not in op2: + return self.match_op(op1, op2) + else: + return self.match_op(op2, op1) + + def match_op(self, aten_op, torch_op): + try: + aten_op_raw_name_overload = '_'.join(aten_op.split("_")[1:-3]) + aten_op_raw_name = aten_op_raw_name_overload.split('.')[0] + torch_op_raw_name = '_'.join(torch_op.split("_")[1:-3]).lower() + except IndexError as e: + err_msg = f"Dump op name format error: {aten_op}, {torch_op}. Your dump data may be corrupted." + raise CompareException.INVALID_DATA_ERROR(err_msg) from e + matching_op = self.aten_mapping.get(aten_op_raw_name) + if matching_op is None: + return False + if matching_op.lower() == torch_op_raw_name: + return True + return False + + +graph_mapping = AtenIrMapping() diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/npy_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/npy_compare.py new file mode 100644 index 000000000..4ebe6296b --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/npy_compare.py @@ -0,0 +1,244 @@ +import abc +import numpy as np +from msprobe.core.common.utils import format_value +from msprobe.core.common.const import Const, CompareConst +from msprobe.mindspore.common.log import logger + + +def handle_inf_nan(n_value, b_value): + """处理inf和nan的数据""" + n_inf = np.isinf(n_value) + b_inf = np.isinf(b_value) + n_nan = np.isnan(n_value) + b_nan = np.isnan(b_value) + n_invalid = np.any(n_inf) or np.any(n_nan) + b_invalid = np.any(b_inf) or np.any(b_nan) + if n_invalid or b_invalid: + if np.array_equal(n_inf, b_inf) and np.array_equal(n_nan, b_nan): + n_value[n_inf] = 0 + b_value[b_inf] = 0 + n_value[n_nan] = 0 + b_value[b_nan] = 0 + else: + return CompareConst.NAN, CompareConst.NAN + return n_value, b_value + + +def get_error_type(n_value, b_value, error_flag): + """判断数据是否有异常并返回异常的n_value, b_value，同时返回error_flag""" + if error_flag: + return CompareConst.READ_NONE, CompareConst.READ_NONE, True + if n_value.size == 0: # 判断读取到的数据是否为空 + return CompareConst.NONE, CompareConst.NONE, True + if n_value.shape != b_value.shape: # 判断NPU和bench的数据结构是否一致 + return CompareConst.SHAPE_UNMATCH, CompareConst.SHAPE_UNMATCH, True + if not n_value.shape: # 判断数据是否为标量 + return n_value, b_value, False + + n_value, b_value = handle_inf_nan(n_value, b_value) # 判断是否有nan/inf数据 + if n_value is CompareConst.NAN or b_value is CompareConst.NAN: + return CompareConst.NAN, CompareConst.NAN, True + return n_value, b_value, False + + +def reshape_value(n_value, b_value): + """返回reshape后的数据""" + if not n_value.shape: # 判断数据是否为标量 + if n_value.dtype == bool: + n_value = n_value.astype(float) + b_value = b_value.astype(float) + return n_value, b_value + + n_value = n_value.reshape(-1).astype(float) + b_value = b_value.reshape(-1).astype(float) + return n_value, b_value + + +def get_error_message(n_value, b_value, op_name, error_flag, error_file=None): + """获取异常情况的错误信息""" + if error_flag: + if n_value == CompareConst.READ_NONE: + if error_file: + return "Dump file: {} not found.".format(error_file) + return CompareConst.NO_BENCH + if n_value == CompareConst.NONE: + return "This is empty data, can not compare." + if n_value == CompareConst.SHAPE_UNMATCH: + return "Shape of NPU and bench Tensor do not match. Skipped." + if n_value == CompareConst.NAN: + return "The position of inf or nan in NPU and bench Tensor do not match." + else: + if not n_value.shape: + return "This is type of scalar data, can not compare." + if n_value.dtype != b_value.dtype: + logger.warning("Dtype of NPU and bench Tensor do not match: {}".format(op_name)) + return "Dtype of NPU and bench Tensor do not match." + return "" + + +class TensorComparisonBasic(abc.ABC): + """NPU和bench中npy数据的比较模板""" + @abc.abstractmethod + def apply(self, n_value, b_value, error_flag, relative_err=None): + raise NotImplementedError + + +class GetCosineSimilarity(TensorComparisonBasic): + """计算cosine相似度""" + @staticmethod + def correct_data(result): + if result == CompareConst.NAN: + return result + if float(result) > CompareConst.COSINE_THRESHOLD: + return 1.0 + return result + + def apply(self, n_value, b_value, error_flag, relative_err=None): + if error_flag: + if n_value == CompareConst.READ_NONE: + return CompareConst.NONE, '' + if n_value == CompareConst.NONE: + return CompareConst.UNSUPPORTED, '' + if n_value == CompareConst.SHAPE_UNMATCH: + return CompareConst.SHAPE_UNMATCH, '' + if n_value == CompareConst.NAN: + return "N/A", '' + + if not n_value.shape: + return CompareConst.UNSUPPORTED, '' + + with np.errstate(divide='ignore', invalid='ignore'): + if len(n_value) == 1: + return CompareConst.UNSUPPORTED, "This tensor is scalar." + num = n_value.dot(b_value) + a_norm = np.linalg.norm(n_value) + b_norm = np.linalg.norm(b_value) + + if a_norm <= Const.FLOAT_EPSILON and b_norm <= Const.FLOAT_EPSILON: + return 1.0, '' + if a_norm <= Const.FLOAT_EPSILON: + return CompareConst.NAN, 'Cannot compare by Cosine Similarity, All the data is Zero in npu dump data.' + if b_norm <= Const.FLOAT_EPSILON: + return CompareConst.NAN, 'Cannot compare by Cosine Similarity, All the data is Zero in Bench dump data.' + + cos = num / (a_norm * b_norm) + if np.isnan(cos): + return CompareConst.NAN, 'Cannot compare by Cosine Similarity, the dump data has NaN.' + result = format_value(cos) + result = self.correct_data(result) + return 1.0 if float(result) > 0.99999 else result, '' + + +class GetMaxAbsErr(TensorComparisonBasic): + """计算最大绝对误差""" + def apply(self, n_value, b_value, error_flag, relative_err=None): + if error_flag: + if n_value == CompareConst.READ_NONE: + return CompareConst.NONE, "" + if n_value == CompareConst.NONE: + return 0, "" + if n_value == CompareConst.SHAPE_UNMATCH: + return CompareConst.SHAPE_UNMATCH, "" + if n_value == CompareConst.NAN: + return "N/A", "" + + temp_res = n_value - b_value + max_value = np.max(np.abs(temp_res)) + return format_value(max_value), "" + + +def get_relative_err(n_value, b_value): + """计算相对误差""" + with np.errstate(divide='ignore', invalid='ignore'): + if b_value.dtype not in CompareConst.FLOAT_TYPE: + n_value, b_value = n_value.astype(float), b_value.astype(float) + zero_mask = (b_value == 0) + b_value[zero_mask] += np.finfo(b_value.dtype).eps + n_value[zero_mask] += np.finfo(b_value.dtype).eps + relative_err = np.divide((n_value - b_value), b_value) + return np.abs(relative_err) + + +class GetMaxRelativeErr(TensorComparisonBasic): + """计算最大相对误差""" + def apply(self, n_value, b_value, error_flag, relative_err=None): + if error_flag: + if n_value == CompareConst.READ_NONE: + return CompareConst.NONE, '' + if n_value == CompareConst.NONE: + return 0, '' + if n_value == CompareConst.SHAPE_UNMATCH: + return CompareConst.SHAPE_UNMATCH, '' + if n_value == CompareConst.NAN: + return "N/A", '' + + if relative_err is None: + relative_err = get_relative_err(n_value, b_value) + max_relative_err = np.max(np.abs(relative_err)) + if np.isnan(max_relative_err): + message = 'Cannot compare by MaxRelativeError, the data contains nan in dump data.' + return CompareConst.NAN, message + return format_value(max_relative_err), '' + + +class GetThousandErrRatio(TensorComparisonBasic): + """计算相对误差小于千分之一的比例""" + def apply(self, n_value, b_value, error_flag, relative_err=None): + if error_flag: + if n_value == CompareConst.READ_NONE: + return CompareConst.NONE, "" + if n_value == CompareConst.NONE: + return 0, "" + if n_value == CompareConst.SHAPE_UNMATCH: + return CompareConst.SHAPE_UNMATCH, "" + if n_value == CompareConst.NAN: + return "N/A", "" + + if not n_value.shape: + return CompareConst.NAN, "" + if relative_err is None: + relative_err = get_relative_err(n_value, b_value) + if not np.size(relative_err): + return CompareConst.NAN, "" + return format_value(np.sum(relative_err < CompareConst.THOUSAND_RATIO_THRESHOLD) / np.size(relative_err)), "" + + +class GetFiveThousandErrRatio(TensorComparisonBasic): + """计算相对误差小于千分之五的比例""" + def apply(self, n_value, b_value, error_flag, relative_err=None): + if error_flag: + if n_value == CompareConst.READ_NONE: + return CompareConst.NONE, "" + if n_value == CompareConst.NONE: + return 0, "" + if n_value == CompareConst.SHAPE_UNMATCH: + return CompareConst.SHAPE_UNMATCH, "" + if n_value == CompareConst.NAN: + return "N/A", "" + + if not n_value.shape: + return CompareConst.NAN, "" + if relative_err is None: + relative_err = get_relative_err(n_value, b_value) + if not np.size(relative_err): + return CompareConst.NAN, "" + return format_value(np.sum(relative_err < CompareConst.FIVE_THOUSAND_RATIO_THRESHOLD) / np.size(relative_err)), "" + + +class CompareOps: + compare_ops = { + "cosine_similarity": GetCosineSimilarity(), + "max_abs_error": GetMaxAbsErr(), + "max_relative_error": GetMaxRelativeErr(), + "one_thousand_err_ratio": GetThousandErrRatio(), + "five_thousand_err_ratio": GetFiveThousandErrRatio() + } + + +def compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=None): + result_list = [] + for op in CompareOps.compare_ops.values(): + result, msg = op.apply(n_value, b_value, error_flag, relative_err=relative_err) + err_msg += msg + result_list.append(result) + return result_list, err_msg -- Gitee From f96024eb2515d459f643c256f7f03bfbbca314af Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 31 Jul 2024 23:28:06 +0800 Subject: [PATCH 064/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0mindspore=E5=AF=B9msp?= =?UTF-8?q?robe=E5=85=AC=E5=85=B1=E7=BB=84=E4=BB=B6=E7=9A=84=E4=BE=9D?= =?UTF-8?q?=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/mindspore/__init__.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/msprobe/mindspore/__init__.py b/debug/accuracy_tools/msprobe/mindspore/__init__.py index 3bf42d1e3..077144429 100644 --- a/debug/accuracy_tools/msprobe/mindspore/__init__.py +++ b/debug/accuracy_tools/msprobe/mindspore/__init__.py @@ -1 +1,4 @@ from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger +from .common.utils import seed_all +from .compare.acc_compare import compare +from .compare.distributed_compare import compare_distributed -- Gitee From 5087a7c1a16755dfd8c3b1fc637ecda731cd4b60 Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Thu, 1 Aug 2024 10:28:11 +0800 Subject: [PATCH 065/791] del_poc_and_preresearch_code_from_master --- profiler/affinity_cpu_bind/README.md | 40 --- profiler/affinity_cpu_bind/bind_core.py | 213 ---------------- profiler/merge_profiling_timeline/README.md | 115 --------- profiler/merge_profiling_timeline/main.py | 233 ------------------ ...\345\257\274\346\210\252\345\233\2761.png" | Bin 53047 -> 0 bytes ...\345\257\274\346\210\252\345\233\2762.png" | Bin 64432 -> 0 bytes profiler/module_visualization/__init__.py | 0 .../module_visualization/graph/__init__.py | 0 .../module_visualization/graph/prof_node.py | 90 ------- .../graph_build/__init__.py | 0 .../graph_build/fwd_module_node.py | 29 --- .../graph_build/prof_graph_builder.py | 115 --------- .../module_visualization/prof_graph_export.py | 39 --- .../prof_parse/__init__.py | 0 .../prof_parse/prof_data_pre_process.py | 102 -------- 15 files changed, 976 deletions(-) delete mode 100644 profiler/affinity_cpu_bind/README.md delete mode 100644 profiler/affinity_cpu_bind/bind_core.py delete mode 100644 profiler/merge_profiling_timeline/README.md delete mode 100644 profiler/merge_profiling_timeline/main.py delete mode 100644 "profiler/merge_profiling_timeline/perfetto\344\275\277\347\224\250\346\214\207\345\257\274\346\210\252\345\233\2761.png" delete mode 100644 "profiler/merge_profiling_timeline/perfetto\344\275\277\347\224\250\346\214\207\345\257\274\346\210\252\345\233\2762.png" delete mode 100644 profiler/module_visualization/__init__.py delete mode 100644 profiler/module_visualization/graph/__init__.py delete mode 100644 profiler/module_visualization/graph/prof_node.py delete mode 100644 profiler/module_visualization/graph_build/__init__.py delete mode 100644 profiler/module_visualization/graph_build/fwd_module_node.py delete mode 100644 profiler/module_visualization/graph_build/prof_graph_builder.py delete mode 100644 profiler/module_visualization/prof_graph_export.py delete mode 100644 profiler/module_visualization/prof_parse/__init__.py delete mode 100644 profiler/module_visualization/prof_parse/prof_data_pre_process.py diff --git a/profiler/affinity_cpu_bind/README.md b/profiler/affinity_cpu_bind/README.md deleted file mode 100644 index 8c3b47ed5..000000000 --- a/profiler/affinity_cpu_bind/README.md +++ /dev/null @@ -1,40 +0,0 @@ -# 昇腾亲和性CPU绑核工具 - -昇腾亲和性CPU绑核工具支持用户无需修改代码，直接运行工具即可实现按CPU亲和性策略绑核，提升推理或训练性能。 - -绑核工具用户arm服务器环境，对于训练或推理任务因为CPU资源调度等出现host_bound问题时使用，可改善该问题；对于非host_bound的场景无明显改善效果。 - -## 使用须知 - -使用绑核工具前手动执行npu-smi info -t topo，出现以下类似信息，说明环境支持绑核，否则请将环境HDK包升级到Ascend HDK 23.0.RC2及以上版本。 - - NPU0 NPU1 NPU2 NPU3 NPU4 NPU5 NPU6 NPU7 NPUx CPU Affinity - NPU0 X HCCS HCCS HCCS HCCS HCCS HCCS HCCS ... xx-xx - NPU1 HCCS X HCCS HCCS HCCS HCCS HCCS HCCS ... xx-xx - NPU2 HCCS HCCS X HCCS HCCS HCCS HCCS HCCS ... xx-xx - NPU3 HCCS HCCS HCCS X HCCS HCCS HCCS HCCS ... xx-xx - NPU4 HCCS HCCS HCCS HCCS X HCCS HCCS HCCS ... xx-xx - NPU5 HCCS HCCS HCCS HCCS HCCS X HCCS HCCS ... xx-xx - NPU6 HCCS HCCS HCCS HCCS HCCS HCCS X HCCS ... xx-xx - NPU7 HCCS HCCS HCCS HCCS HCCS HCCS HCCS X ... xx-xx - NPUx ... ... ... ... ... ... ... ... ... ... - -## 使用方式 - -1.执行以下命令实施绑核： - - - 直接执行绑核命令 -```bash -python3 bind_core.py -app/--application="inferenec/train cmd" -``` -该方式会自动拉起训练或推理任务，检测任务进程，并实施绑核。 - - - 手动拉起训练或推理任务后再执行绑核 -```bash -python3 bind_core.py -``` -该方式会循环查找(循环5次，每次10s，若找不到进程，则直接退出)使用到NPU的任务进程，并实施绑核。 - -2.绑核运行过程的日志会保存到当前路径的bind_core_时间戳.log。 - -3.如果推理或训练进程拉起后需要一定时间预处理，才会真正执行任务，可在执行绑核命令时设置-t/--time参数(单位秒)，绑核工具会在延迟配置的时间后，再实施绑核动作。例如：python3 bind_core.py -app="cmd" -t=10，配置后工具会在10秒后执行绑核操作。 \ No newline at end of file diff --git a/profiler/affinity_cpu_bind/bind_core.py b/profiler/affinity_cpu_bind/bind_core.py deleted file mode 100644 index 7f27e9242..000000000 --- a/profiler/affinity_cpu_bind/bind_core.py +++ /dev/null @@ -1,213 +0,0 @@ -import subprocess -import argparse -import os -import time -import logging -from datetime import datetime -from datetime import timezone - - -class PathManager: - DATA_FILE_AUTHORITY = 0o640 - - @classmethod - def create_file_safety(cls, path: str): - base_name = os.path.basename(path) - msg = f"Failed to create file: {base_name}" - if os.path.islink(path): - raise RuntimeError(msg) - if os.path.exists(path): - return - try: - os.close(os.open(path, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY)) - except Exception as err: - raise RuntimeError(msg) from err - - -class BindCoreManager(): - DEFAULT_FIND_RUNNING_PID_TIMES = 5 - - def __init__(self): - self.npu_id_list = [] - self.running_pid_on_npu = {} - self.find_running_pid_times = self.DEFAULT_FIND_RUNNING_PID_TIMES - self.npu_affinity_cpu_dict = {} - self.log_file = '' - self._init_log_file() - - - def _init_log_file(self): - now_time = datetime.now(tz=timezone.utc) - time_stamp = str(now_time.year) + '_' + \ - str(now_time.month) + '_' + \ - str(now_time.day) + '_' + \ - str(now_time.hour) + '_' + \ - str(now_time.minute) + '_' + \ - str(now_time.second) - log_file_name = 'bind_core_' + time_stamp + '.log' - msg = f"Failed to create file: {log_file_name}" - try: - PathManager.create_file_safety(os.path.join(os.getcwd(), log_file_name)) - except RuntimeError as err: - raise RuntimeError(msg) from err - self.log_file = log_file_name - logging.basicConfig(filename=self.log_file, - level=logging.INFO, - filemode='w', - format='%(asctime)s-%(name)s-%(levelname)s-%(message)s') - - def _get_all_npu_id(self) -> None: - get_npu_info_cmd = 'npu-smi info -l' - get_npu_info_process = subprocess.run(get_npu_info_cmd.split(), shell=False, capture_output=True) - get_npu_id_cmd = 'grep ID' - get_npu_id_process = subprocess.run(get_npu_id_cmd.split(), shell=False, input=get_npu_info_process.stdout, capture_output=True) - res = get_npu_id_process.stdout.decode('utf-8').split() - for i in res: - if i.isdigit(): - self.npu_id_list.append(int(i)) - logging.info(f'NPU total id list: {self.npu_id_list}') - - def _get_npu_affinity(self) -> bool: - cpu_num = os.cpu_count() - cpu_num_for_each_npu = cpu_num // len(self.npu_id_list) - get_npu_topo_cmd = 'npu-smi info -t topo' - p = subprocess.run(get_npu_topo_cmd.split(), shell=False, capture_output=True) - res = p.stdout.decode('utf-8').split() - if not res: - print('[ERROR] Failed to run get npu affinity info, please check if driver version support cmd npu-smi info -t topo') - return False - - index = 0 - for v in res: - if '-' in v: - affinity_cpus = [] - cpu_lists = v.split(',') - for cpu_list in cpu_lists: - cpus = cpu_list.split('-') - if len(cpus) != 2: - continue - if int(cpus[1]) - int(cpus[0]) == cpu_num_for_each_npu - 1: - cpus[1] = str(int(cpus[1]) + cpu_num_for_each_npu) - affinity_cpus.append(cpus[0] + '-' + cpus[1]) - if index < len(self.npu_id_list): - self.npu_affinity_cpu_dict[self.npu_id_list[index]] = ','.join(affinity_cpu for affinity_cpu in affinity_cpus) - index += 1 - else: - print('[ERROR] Get affinity_cpu_list for {} npus, more than real npu num: {}'.format(index + 1, len(self.npu_id_list))) - return False - - for k in self.npu_affinity_cpu_dict.keys(): - logging.info(f'Affinity CPU list {self.npu_affinity_cpu_dict[k]} for NPU {k}') - return True - - def get_running_pid_on_npu(self) -> bool: - no_running_pids_on_npu_msg = '[INFO] Now there is no running process on all NPUs, stop bind cores' - logging.info('Begin to find running process on all NPUs') - # get running process on NPUs - for times in range(self.find_running_pid_times): - running_pid_on_npu = {} - for npu_id in self.npu_id_list: - get_npu_pids_cmd = 'npu-smi info -t proc-mem -i {} -c 0'.format(npu_id) - get_npu_pids_process = subprocess.run(get_npu_pids_cmd.split(), shell=False, capture_output=True) - res = get_npu_pids_process.stdout.decode('utf-8').split() - pid_list = [] - for value in res: - if value.startswith('id:'): - pid = value.split(':')[1] - pid_list.append(pid) - if pid_list: - running_pid_on_npu[npu_id] = list(set(pid_list)) - - if len(self.running_pid_on_npu.keys()) == len(running_pid_on_npu.keys()) and running_pid_on_npu: - self.running_pid_on_npu = running_pid_on_npu - break - - self.running_pid_on_npu = running_pid_on_npu - time.sleep(5) - - # delete repeat pid - for npu_id in self.npu_id_list: - if npu_id not in self.running_pid_on_npu: - continue - pids_on_npu = self.running_pid_on_npu[npu_id] - for npu_id_with_pids, pids in self.running_pid_on_npu.items(): - if npu_id == npu_id_with_pids: - continue - pids_on_npu = list(set(pids_on_npu) - set(pids)) - self.running_pid_on_npu[npu_id] = pids_on_npu - - if_running_process = False - for npu_id, pids in self.running_pid_on_npu.items(): - if not pids: - logging.info(f'There is no running process on NPU {npu_id}') - else: - logging.info(f'Succeed to find running process {pids} on NPU {npu_id}') - if_running_process = True - if not if_running_process: - print(no_running_pids_on_npu_msg) - return if_running_process - - def get_npu_info(self) -> bool: - try: - self._get_all_npu_id() - if not self._get_npu_affinity(): - return False - except subprocess.CalledProcessError: - return False - return True - - def run_bind_core(self): - if not self.running_pid_on_npu: - return - for npu, pid_list in self.running_pid_on_npu.items(): - if npu not in self.npu_affinity_cpu_dict.keys(): - logging.warning(f'Cannot find affinity cpu for npu: {npu}') - continue - affinity_cpu = self.npu_affinity_cpu_dict.get(npu) - for pid in pid_list: - try: - logging.info(f'Begin to bind cores for process {pid} on NPU {npu}') - set_affinity_cpu_cmd = 'taskset -pc {} {}'.format(affinity_cpu, pid) - p = subprocess.run(set_affinity_cpu_cmd.split(), shell=False, capture_output=True) - logging.info(p.stdout.decode('utf-8')) - except subprocess.CalledProcessError: - print('[ERROR] Failed to bind process {} on NPU {} with cpu cores list {}'.format(pid, npu, affinity_cpu)) - - logging.info(f'Succeed to bind process {pid} on NPU {npu} with cpu cores list {affinity_cpu}') - - def args_parse(self): - parser = argparse.ArgumentParser(description='This is a affinity cpu core bind script.') - parser.add_argument('-t', '--time', type=int, metavar='', help='Wait time before bind cores that you want to set. The unit is \'s\'.') - parser.add_argument('-app', '--application', metavar='', nargs='+', help='Training or inference command that you want to run.') - args = parser.parse_args() - if args.application: - application_cmd = ' '.join(args.application) - self.launch_process(application_cmd) - time.sleep(2) - # if time is set, wait for setting time before bind cores - if args.time: - time.sleep(args.time) - - def launch_process(self, cmd: list): - logging.info(f'Start to execute cmd: {cmd}') - try: - subprocess.Popen(cmd.split(), shell=False) - except subprocess.CalledProcessError as e: - raise RuntimeError(f'Failed to run cmd: {cmd}') from e - - -if __name__ == '__main__': - print('[INFO] Begin to run bind-cores script...') - bind_core_manager = BindCoreManager() - bind_core_manager.args_parse() - - if not bind_core_manager.get_npu_info(): - print('[ERROR] Failed to get current npus info') - exit() - - if not bind_core_manager.get_running_pid_on_npu(): - exit() - bind_core_manager.run_bind_core() - print('[INFO] End to run bind-cores script, the log is saved in {}'.format(bind_core_manager.log_file)) - - diff --git a/profiler/merge_profiling_timeline/README.md b/profiler/merge_profiling_timeline/README.md deleted file mode 100644 index 907a39a6e..000000000 --- a/profiler/merge_profiling_timeline/README.md +++ /dev/null @@ -1,115 +0,0 @@ -# 合并大json工具 - -merge_profiling_timeline（合并大json工具）支持合并Profiling的timeline数据，支持合并指定rank的timline、合并指定timeline中的item。 - - -## 多timeline融合 - -### 性能数据采集 - -使用Ascend PyTorch Profiler或者E2E性能采集工具采集性能数据，E2E profiling将被废弃，不建议使用。Ascend PyTorch Profiler采集方式参考：[Profiling数据采集](https://gitee.com/ascend/mstt/tree/master/profiler)。将采集到的所有节点的性能数据拷贝到当前环境同一目录下，以下假设数据在/home/test/cann_profiling下。 - -E2E Profiling数据目录结构示例如下： - -```bash -|- cann_profiling - |- PROF_*** - |- timeline - |- msprof.json - |- device_* - |- info.json.* - ... - |- PROF_*** - ... -``` - -Ascend PyTorch Profiler数据目录结构示例如下： - -```bash -|- ascend_pytorch_profiling - |- **_ascend_pt - |- ASCEND_PROFILER_OUTPUT - |- trace_view.json - |- FRAMEWORK - |- PROF_*** - |- **_ascend_pt -``` - -### 参数说明 - -| 参数名称 | 说明 | 是否必选 | -| -------- | ------------------------------------------------------------ | -------- | -| -i | 指定Profiling数据目录路径。 | 是 | -| --type | 指定需要合并timeline场景，可选取值：`pytorch`（通过Ascend PyTorch Profiler方式采集profiling数据，合并所有卡的trace_view.json）、`e2e`（通过E2E Profiling方式采集Profiling数据，优先合并总timeline，没有生成则选择合并device目录下的msprof_*.json）、`custom` （自定义需要合并的timeline数据，具体参考**使用示例**）。 | 是 | -| -o | 指定合并后的timeline文件输出的路径（路径末尾可以设置文件名，具体用法参考**使用示例**），不设置该参数的情况下默认文件输出的路径为当前目录（默认文件名为merged.json）。 | 否 | -| --rank | 指定需要合并timeline的Rank ID，默认全部合并。 | 否 | -| --items | 指定需要合并的Profiling数据项，包括：python、Ascend Hardware、CANN、HCCL、PTA、Overlap Analysis，默认全部合并。 | 否 | - -### 使用示例 - -1. 合并单机多卡timeline，默认合并所有卡、所有数据项，生成first.json在path/to/cann_profiling/output/目录下 - - ```bash - python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/first --type pytorch - ``` - -2. 合并单机多卡timeline，默认合并所有卡、所有数据项，不设置-o参数时默认生成merge.json在当前目录下 - - ```bash - python3 main.py -i path/to/cann_profiling/ --type pytorch - ``` - -3. 合并单机多卡timeline，只合并0卡和1卡 - - ```bash - python3 main.py -i path/to/cann_profiling/ -o path/to/cann_profiling/output/2p --type pytorch --rank 0,1 - ``` - -4. 合并单机多卡timeline，合并所有卡的CANN层和Ascend_Hardware层数据 - - ```bash - python3 main.py -i path/to/cann_profiling/ --type pytorch --items "CANN,Ascend Hardware" - ``` - -5. 合并多timeline（自定义） - - 以上场景不支持的情况下，可以使用自定义的合并方式，将需要合并的timeline文件放在同一目录下（附：该场景比较特殊，与正常合并不同，无法直接读取info.json中的rank_id，因此该场景下的rank_id为默认分配的序号，用于区分不同文件的相同层，不代表实际rank_id） - 数据目录结构示意如下： - - ```bash - |- timeline - |- msprof_0.json - |- msprof_1.json - |- msprof_2.json - |- hccl_3.json - |- hccl_4.json - ... - ``` - - 通过下面的命令合并所有timeline，同样支持-o、--rank、--items等参数。 - - ```bash - python3 main.py -i path/to/timeline/ -o path/to/timeline/xxx --type custom - ``` - - 合并timeline查看：在 -o 指定的目录（不设置-o时默认在当前目录下的merged.json）的xxx.json为合并后的文件。 - - -## 超大timeline文件查看 - -[下载whl](https://gitee.com/aerfaliang/trace_processor/releases/download/trace_processor_37.0/trace_processor-37.0-py3-none-any.whl)包并执行如下命令安装（windows）： - -```bash -pip3 install trace_processor-37.0-py3-none-any.whl -``` - -安装完成后直接执行如下命令： - -```bash -python -m trace_processor --httpd path/to/xxx_merged.json -``` - -等待加载完毕，刷新[perfetto](https://ui.perfetto.dev/)界面，单击Use old version regardless，再单击`YES, use loaded trace`即可展示timeline（通过W放大、S缩小、A左移、D右移来查看timeline文件）。 - -![输入图片说明](perfetto%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC%E6%88%AA%E5%9B%BE1.png) -![输入图片说明](perfetto%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC%E6%88%AA%E5%9B%BE2.png) \ No newline at end of file diff --git a/profiler/merge_profiling_timeline/main.py b/profiler/merge_profiling_timeline/main.py deleted file mode 100644 index 678f5d5a8..000000000 --- a/profiler/merge_profiling_timeline/main.py +++ /dev/null @@ -1,233 +0,0 @@ -#! /usr/bin/python3 -# Copyright 2023 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import json -import os -import re - -from functools import partial -from argparse import ArgumentParser -from decimal import Decimal - - -FILTER_DIRS = [".profiler", "HCCL_PROF", "timeline", "query", 'sqlite', 'log'] -RANK_ID_POS = 1000 - -def get_path_dir(path: str) -> list: - """ - check result path exist JOB dir - path : result path - """ - path_dir_filter = filter(partial(_path_dir_filter_func, root_dir=path), os.listdir(path)) - sub_dirs = list(path_dir_filter) - return sub_dirs - - -def _path_dir_filter_func(sub_path, root_dir): - return sub_path not in FILTER_DIRS and os.path.isdir(os.path.realpath(os.path.join(root_dir, sub_path))) - - -def natural_sort(files): - convert = lambda text: int(text) if text.isdigit() else text.lower() - alphanum_key = lambda key: [convert(c) for c in re.split('([0-9]+)', key)] - return sorted(files, key=alphanum_key) - - -def get_timeline_info(args, prof_dirs): - timeline_info = {} - - for prof in prof_dirs: - pro_path = os.path.join(args.input, prof) - - # 从info.json读取rank_id - rank_id = get_rank_id_from_info_json(pro_path) - if rank_id is None: - print(f"WARN, There is not rank id info in {pro_path}") - continue - - timeline_path = get_timeline_path(pro_path, args.type) - - if os.path.exists(timeline_path): - timeline_info[rank_id] = timeline_path - else: - print(f"WARN, The file \"{timeline_path}\" does not exist.") - return timeline_info - - -def get_timeline_path(pro_path, type): - for root, dirs, files in os.walk(pro_path): - for dir_ in dirs: - if 'ASCEND_PROFILER_OUTPUT' == dir_ and type == 'pytorch': - timeline_path = os.path.realpath(os.path.join(root, dir_, 'trace_view.json')) - return timeline_path - - for file_ in sorted(files, reverse=True): - if 'msprof' in file_: - timeline_path = os.path.join(root, file_) - return timeline_path - return - -def get_rank_id_from_info_json(pro_path): - info_json = "" - rank_id = None - for root, dirs, files in os.walk(pro_path): - for file in files: - if "info.json." in file and ".done" not in file: - info_json = os.path.join(root, file) - break - - if info_json: - if os.path.islink(info_json): - print(f"The file: \"{info_json}\" is link. Please check the path.") - return - try: - with open(info_json, "r+") as f: - info = json.load(f) - rank_id = info.get("rank_id") - except Exception as err: - print("[ERROR] %s" % err) - return - return rank_id - - -def merge_timeline_general(args): - """合并e2e profiling生成的msprof*.json""" - if not os.path.isdir(args.input): - print(f"No such file or directory: \"{args.input}\". Please check the path.") - return - prof_dir = get_path_dir(args.input) - if not prof_dir: - message = f"The path \"{args.input}\" does not have PROF dir. Please check the path." - print(message) - return - timeline_info = get_timeline_info(args, prof_dir) - timeline_files_dict = {} - - # 合并部分profiling items - process_list = args.items.split(",") if args.items else None - - # 合并部分rank - if args.rank: - rank_ids = [int(rank_id) for rank_id in args.rank.split(",")] - else: - rank_ids = list(timeline_info.keys()) - - for rank_id in rank_ids: - if not timeline_info.get(rank_id): - print(f"main.py: error rank_id '{rank_id}' ") - return - timeline_files_dict[rank_id] = timeline_info.get(rank_id) - merge_timeline_events(timeline_files_dict, process_list) - - -def merge_timeline_custom(args): - """合并指定目录里所有timeline文件""" - timeline_files = natural_sort(os.listdir(args.input)) - timeline_files_dict = {} - for idx, timeline_file in enumerate(timeline_files): - timeline_files_dict[idx] = os.path.join(args.input, timeline_file) - # 合并部分profiling items - process_list = args.items.split(",") if args.items else None - merge_timeline_events(timeline_files_dict, process_list) - - -def merge_timeline_events(timeline_file_dict, process_list): - """ - 输入需要合并的timeline文件路径及对应的rank_id/id、需要合并的process_list - 输出合并timeline - """ - new_events = [] - for rank_id, timeline_path in timeline_file_dict.items(): - node = rank_id // 8 - print("rank id: ", rank_id, "timeline file: ", timeline_path) - if os.path.islink(timeline_path): - print(f"The file: \"{timeline_path}\" is link. Please check the path.") - return - try: - with open(timeline_path, 'r+') as f: - cur_events = json.load(f) - except Exception as err: - print("[ERROR] %s" % err) - return - - proc_pid_dict = {} - for event in cur_events: - if event.get("name") == "process_name" and event.get("ph") == "M": - if event.get("args"): - proc_pid_dict[event["args"].get("name")] = event.get("pid") - process_list_tmp = process_list if process_list else list(proc_pid_dict.keys()) - # 提取待合并的items的pid - merged_pids = set() - for pro in process_list_tmp: - if pro not in proc_pid_dict.keys(): - print(f"main.py: error argument --items: invalid choice: '{pro}' (choose from {list(proc_pid_dict.keys())})") - return - merged_pids.add(proc_pid_dict.get(pro)) - - for event in cur_events: - - # 只合并特定数据项 - if merged_pids and event.get('pid') not in merged_pids: - continue - - # convert tid to int - if not isinstance(event['tid'], int): - print(f"[WARNNING] {event['tid']} is not int type") - - # 进程名加上rank_id区分不同rank - if event.get("name") == "process_name" and event.get("ph") == "M": - if event.get("args") is not None and event["args"].get("name") is not None: - event["args"]["name"] = event["args"]["name"] + f"_{rank_id}" - - #modify connect id - if event.get('id') and (event.get('ph') == 's' or event.get('ph') == 'f'): - event['id'] = float(event.get('id')) * RANK_ID_POS + rank_id - - new_events.append(event) - out_path = f"{args.output}.json" - if os.path.islink(out_path): - print(f"The file: \"{out_path}\" is link. Please check the path.") - return - if os.path.exists(out_path): - print(f"File {out_path} existed before and is now overwritten.") - os.remove(out_path) - try: - # 设置文件权限为640，安全考虑 - with os.fdopen(os.open(out_path, os.O_WRONLY | os.O_CREAT, 0o640), 'w') as f: - json.dump(new_events, f) - except FileNotFoundError: - print(f"Param -o (output path) is not exists, please check it.") - return - print(f"timeline merged output path: {out_path}") - - -def parse_args(): - parser = ArgumentParser(description="Merge timeline for multi card") - parser.add_argument("-i", "--input", default=None, help="root dir of PROF_* data") - parser.add_argument("-o", "--output", default="./merged", help="save path of merged.json ") - parser.add_argument("--rank", default=None, help="List of ranks to be merged. By default, all ranks are merged") - parser.add_argument("--items", default=None, help="Specify the data items (python，CANN，Ascend Hardware，HCCL，..)to be merged. in the timeline.") - parser.add_argument("--type", choices=('pytorch', 'e2e', 'custom'), help="Customize the timeline file to be merged.") - arg = parser.parse_args() - return arg - - -if __name__ == "__main__": - args = parse_args() - print("========================== start merge timeline ====================") - if args.type == "custom": - merge_timeline_custom(args) - else: - merge_timeline_general(args) \ No newline at end of file diff --git "a/profiler/merge_profiling_timeline/perfetto\344\275\277\347\224\250\346\214\207\345\257\274\346\210\252\345\233\2761.png" "b/profiler/merge_profiling_timeline/perfetto\344\275\277\347\224\250\346\214\207\345\257\274\346\210\252\345\233\2761.png" deleted file mode 100644 index beef396ce2996c25ecd74298285ccab5011ddea1..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 53047 zcmd?QWmr^S`1h-l(%m5-(jbxw!w4uKt+WV3hjce1T>_$%bce*yNDfj%NOw0#4nxD# z;rBn!bI$8?UY_$}Uwf~0t-AKT_gd?-zV{uYt*J~(#6bkIMs$kg1lOGRRfCkwwE43Zz3wl%#dwX4?W%71FVr6J{l@9bY&vRs4pv*p zXElhmBqnn*%$L?S%>TML9rZ*#%&s-8sJ??4pAYA|QM%O;6p<*|-0lvVdpS3#^W);- zcxvEUC+EwTT5~$1w>gKeN*NJ<#l}!kV|z$BLXL0>h0TFzN-m77MPDo}Y(6up*Y-Y- zki+fI-gyVR1cm?2MwAV1vS^=SmQn}0&f2A((qBJSQ+xhaXyk6UwhkOH`~ii!gx1E$B%N={ zIleVU>K#fE=#%+5wif^%z*YEt_HgJdt_Gr$6KK})7&?fQSLs@>b5xx5fb{h7 zD9J@@5DeAcn5KxsGdKE5HH<~!pjG+An<5dN;7%NfFp@(lZa=?zZ~4&onJu$s%zpHD zwzqBwBEjJGfF7PM=4T3z@^1wx_+!`zFF*McaK{au%8=*Yam0d_$hMMQ&XA(6JwoKw zS6jZ=MCU8CE&WSiR>Dgi3j#7J7D5v&OKwAZ;*>%AzuQzM9k%oc`^8LMxs8x?&J%>-gO9z1j zw71;zZn5mI2c@{>(E#;{H9oB(48`@+A(yBh8z}@L)r(h!+iOkvM{E3Mb%-zj=#WEU zQ}IjNL8%?F%_32k(^Lxhj)W^*Mcn{hobX}kBWpF`^=f|82>r{SQp4*wK^U3l! zT_`~DE{UL=)wAv_{xo}ksS2R^rgsS_pxg-!PT_J$%RRdwYm(fAd8Ac<<;bN(%o05Fx}3?4iCMGDzbvnJgFdEvGY?oWpz#dh5sQup!o&sMqFW zF_V6cIL{(t*(0cW5A4Dj%-x#ahpL2h5feym#2`fd3qG;uS-DY?AyzAs$SsER-M4Ws zKV5ouQU)wpVnVm#k>;;X@ik4p&ux^TD_M+F$GQQ3R{jC*|2ipf9BZ4=DtdS|#%Rp3 zy97+s(Oj@MNic8FMob$Irpm49^61|h_z7fxS>Gv=W{t~n|2YB@){X@F1>?olCRaMy z#{{W%hR#nHJ#J3&C2ow+xy83Whv2UhSTq6O>x?=_0#lQbgZPx&r2wD&11T6HrXksJ z2xZ}Dbng4F!@T)PBA^<$>Eno{g<#~F+`pqAyv)q$X@wbnsq$1i3z%G;PmGP7n_%;N zlzCOJQ7OzuS``=E)U%#1yo|Jzbw!g)_eF(Wj%}V^l|JV_3QqsFGqVO;VWa!~&Lh3Z z$nf%<_Bp?w2l}D4+N=fp8Wal1emirfH@~qi5XkPatJM#xg0DABF1^~7r*T|)u!S=e z%g5m&*=WGe zLXOTBE&UJKedZqsGAob?b1%N%?!V| zw%Y)5_=(9Vevi4Kq~03dxWLz`Cx*v8_N9mP>GALgv*_=q)`ZwSqW4}Xk+73VErxTz zauu#H;3!t=s05+qeC24;gIUJJu8Wq#Auml*&%$xjZE6jZVUwsnDjP!gT|8 z=hx{Oe(+++J;U=enWova#nLh1x|M>gd4?t0TvcDTkv6gBP|ItB@d)HU?YSM9`Bl_h zG-Kx9on7WVi9Wo0pa-4rn-nUAEFti7?q{=v^(WCDondvw4h2VXcoiQgdV`1;*)_kf zBE}MPTOXwHLZKK0PS%IT?yp9^d64zK#O#K(>i+LwGSblBCtGc&f4e`VAKm}(_{Ko< zOSvbN*_o1%x9_y?+xdkn&#{fF;^ zJxo-ir2yJZwt5sUkUbw6A~Fh%VucI=_}Zg3W7=tX4g1fy{Mr0Jnp!{PAkVr9aiqsw z>g^PPdDzmSxJurAr6g@FtMd^-lHS_Kx5TT}L6lL+uQ#jNE%cdW9&^XoqwtWXemJEB zmt}%#u%9|NuZ;|lfItJ*!|f|<*w%;%Z#^%;CEQYTm8vp>UBxHSn)%o2I!~PO515vd zOZQVRkmvzBr!6D+g+1K~C411>lEK2@hR4Mx*_a@x-od6!dP*BE-q>GQWm~Uu>2;UY zZN2A*HooI;crinpwWq`pdKI}uOy5z&WE~B_i{Ms6s^Dmz;Qk@R1U!yBU4X~!KuhqO zS;$^F4kVV1`Sp3%mV2U3M%__vEgWcgT8{gRGQP{F1cf_9cwJg5*Y&bfmRNAf3A;mH zZ8L&;ht0*et9fbT^Efg{~1erVw>?h?cXB7{KTpJ^jQZjEC%Z}7iQSFZ^U35 zBmTV5RyJZ297Lv7^mP44iAkAsb%RICF&*mYy?Tu9VEf)bU^1%wd3)gj|xPY_W<9%S6E0)c8*QFO*xtMrt zBoE8suu#FUa*QM{{E-uKRk&o&{hn(Sdf)6D?40n(bOFg7~onsKMn^6IS|Wn1&n!o+Sv46&opSZH`I zq$f2O$eW5~DV4#5o`@_RuF zkh(t+bfFms>Q~DGIavq^wLgQo27Z~MN4mMdQkvgif>Sp4#s(p_`V694jQv(6H8^l! zps_k(M$6&p0X)MD3t8JbEx_U(X0k`W1a|#0kwF|@3Q{cePNAifDVw)6*w?AAs|L2# z#?OCFfnVf&M+(L+JT{xTBb~nx+Zi6LjvZ^+K%hv$dj7Z<*yzwuTEPcKQD6bGqVopy zBiU7EyQp{Pi0&Nxc{!^~=sDl=l!4i4SN+YU$XxB0Bcpch4%UifvwxEW(wNfmobZ88 z0?Qq`_L_v0t=Y8G%um+yTP`=-u>=59sa9olcq7!v*8)R?LZ0^tQh+$Km+1v~Kro|* z>sH{q2qPe**BIVO0E5Nzj=k&2(iJ^i25;!{5DNm^zUau>?NG7xvY;R2JU>FLLsYq# zCE`+q1VD6M&K4m9g0)E}Vz!8V`ZFX&^FL_>v~&2;(Cs;*H<2e+%kWD{ZdhMSVx8GS zm<~^}H=A8s#jI0z^>JlRAIsY@dchBojYn&;#ZK6VrI>&HI|;yb zr()aV)zZ_0QFj_(&(L`qv^zC#OPB-a63XsN7H^DUn~9stVuwM{Op0^L5IeZ=NWGrU+xC!-{I=J@CO?|_lWvMBD8#JWdU+qncid2H^iP&L( zNZZx7pI6A;2Ih6I92Ir!+!#uBdQ@bIr&J5<6OBU!Px5B=E|ISHP!W7(iT zu|Z`}bjRz&h$<$nowWRd6H=7C1Uo^nh={E~F5XDvpJv6827=RdU?6!Rl9AHxSs=S* zig-}K3!9O_G@RzL1%u@$CGr=4LYXh{F z*xK6q=62V_PTzm&=F}F#Q8?&p*01H7sI;vIpsZ)Z~s^&ChFR5vjy7?y9?P`FPFnBHuEq2D1{bbEXBn9!rtT`r!Y{?V-z@QwkJ^ z*|ax7Cj`DFu_Ajh!jut0A@=IhjiEl&d0Gvv?Cna%qU}HbXIE#3|9y5eJb~y{O|aSs z+1ZdsbM6)-Cnq@TY2*VraYW{aqMgdVJ8(sRf&mD|;ho|$1b&g`w?OOO{DP$qTV{Uh zn!HPp{zD?yw;cqdPELzi-z1{btB|2K}dj`AtRzR6<F*8DaUs4xs1H6)`b2oVOJNO@>CNTC|R%8;7 z7)w@TaEjQ%cFsnOh2}g(p7_j6z9r(2(&?VIMSPyYMFq^N2DY3?`D#ji6G6?n4B#uM zsPrZ$zizI7wnMYqQ5eQ>8A^}5w3(&tZ9`>tF9UksX;3o||6 z=I#=GR@c!KQ!MiDT?K>gW5Jcpps~gp&vWHzAT``b`yw0K_STQ(67biknRQli3DG@< za}9d;>X!5m_gQ>17rZyQ(t^cVauEFN$xY&&$(~f;USUy^i9LA@>y?)(OLR_N5O?6= zvj&6%xF2p<p{R9WvxI8kK`z}#p}Xf@)xRK{VD2$Wxtq< z%8NP4nM#j;L5oxqx7x;>Z|a)xrM6StvgL-b^5{wx2m6{))@jWKTW(0*8K5nosBLaF z0;qB{IhIx=a|hnRk`6aptR!3uQD%R8N8Z1QB_RrIh1ACN+O1V#g7*j?PfhhdLWaiO zV8uH_p-WdJrj8wqY4**d=T&1JZGmdysQ)LlE&YAaCowo;U;yU z1a3fdHI@_`bb#ZgSK6HgUJ0d-H89g#4U{!cXHby=O2<#GJ7Z^RsdMqs9Z75$7^&Lr zxHjWZZ~z?^=F=DwWk&3u=PFKm$}s*i zqSz30i7)r_RNBT%QBaTl?ls>^qJ7*IM6`i!K8byX1H960GFuf81923z&(3DHhG zhn_ZF*gj3)=Jxe-5RJ|CBkI4ce=hV7e|HNi$Kt@RxB{akb6exg`O{ zMJ7Mm9*Y3SgoA^xXz0e!q9-Xp8<-%%g$A`h5|E7l{4+lty(7^b*{OLmE$?c><4g>K z{j>v0j$6=9ik4VxDld&DQP4V!GyHuZcxDR5dnx8#gKo4%0J)9k&oEe_R0jV@mnY%} z$n?uH_dGQ5RmWYfFLHTp{P<2R0lOD`q|+K!8UyoO__Nx z|3HRW6UWk~(;tIxL>gqU&j6T?Ld}-PAIx}u64>zaYy-`dM-C@R%U9AcUlgH!j3rXO zX*V)A+y0NJN@c%9C3BSO%iSrh0T5g_t;Xn{5)6B=v_a}B;n>rJMd8SKp-GQZ(eo2eH8%@` zLyds#gW%BHN5q8KfljV8O!OKWCY+8+G`eoiAGi+UlM)*?`~p1WxjFg5)imbT`mWk# z%E3hqD3X+SD_4R#B?uy~CH*Ju#)Hx9Nub^=)H2pw$APV_A6Pk6U+LYt5JtSy#yhxx zfW6l}k-3O{3DT0~(uPu?<(iA%lJ$}s3oL=&`*x}2tHe-V4*eWE_6xA)B4~)+58@}W zskL+ypt{jtx@VZ@buxDPH7C|+2(3AnP*}97*FG}28g%Rt=>8cvD>B;C4qix{5~l5x z*^k1Q53|iC+^og9w_rkbur0*2lVc7PFQl>8<@8BPq8!m_|B+?yQ* znx8n=6{OrfQWO>hdYH7b6dqfD$AHuGEWv;5gFRwUx*oL~xvyAp<-0KJ$1y)d9U|S| zVtCSXhQmS<2VDYqE?nvZb(B_P0xp1vTq~ecWtYx*-E(+u1zM5zlN9@!?Xc$X$I!k_ z)n=rxL~tsA{*Gt1pzL7w=8v*GlNUwQbKO8>xT zI#1We9&sZGo`{~o($BXZr(diW`YPa#3S*S7f`?L(6Fun$zht?6cx1}8YxjL!dbXZ0sg3Irs*{Yb1Ss19VBSIQg3or0o+0QuD(@6VE!j?E z5_-jxE+)ona$uD11={=)P_4uR7^oKP>uo{%+N}0e3I`LR3!J2dsl0s_)lrJQXRL;3 zyBi!5Rr1MVshZ&TbR-LvBOPq0>v9@?6uj+GM^bL5X-KZnkmAoq0_=02PH6Tu%Ua)} z@X~1ahohQ^G*&FfF8kKQV(rE2tv%xU)6;?vx<;X)REf%L-J91E_9^*)#*COv2YLUk6#kXs;7d9SOc346F&ZWf>& zqT&e=R)%-=Eri5i1fHsCMQ@=hMUe`MPt|noD|8J(Gka!C6q#<$LMWQS-eai$m1U=k zR$Yb|Z>c%&HRIU^&qk?%nI^l`#g#kPO3zTxqls|GOcVLis;=R_#5-){chL9`mK9dp z9q6$8To~^>l_!pUjlI@})KYRTj~P!WEVriu%VgfhvZlH7YC9Yov1V-Lc@XI6CyI)B zz3`79r<-_Y3=i|>kI+D&O+OK;6~(6>=Q%c<467}_q96F_&O_A?j3pJR_2T6AoF+FH z1V7H)4F<`TlNYh#A65i~Gt-*eIW1B--ty4BUi)fJA~8Hy;?8in;UvEUo{Y_vUcyd~ zG1pU5A-XEN-~_7mQ*!yRhJl&S*T>fM%^7zQ0wWI>`>xz;<$Oi!`ZGg+|JVpec_Hdf!qI*X_O` zyi+QbivWXv2Jv(^bKT127-ESXRgi2{^R=#{2O7Lrd(_`4A968?GfmY5IOqwv%{94s&)%X@BhWd{@7vZ9%#a+0*^3EZ68@+XWx@~(ULTv zD3^2RCJijtoGcJ*cI#IEW6IztyjY_@43&f;*bU8CdTfiZHz23#w{sgKf40SjN$*H! z-_+kmFfHx!HEDE$G2)aWd)tJ5mY26NXsB!Jk4AdhHYRr1q(7Y8)u(7RD!4&$->2>d z{;Kv`bg6;D&{TLBZYmSEB{?f;81$8GcxVGA4_@S`Jbrh)ZEX5`-qyfRMAI-}+v~L4 zShzI&O)A|E;+iehmHlM|8&utb6)dzff)%+law(w7NY*7lX&8;wQ^9BIu9X(O3eTV}`Ub3f>G zf@LW!A+=k$V=gz>d52@ttCNvhkQnPrU@y)4)D^)mFTEncX$$Qx4x&zrR!?iL{+`br zBFC_T{os0cA4EPdwJ90C0he|D)X`6@$ddV{ZA-DYoz?T)`_JMZ@mVqy_=MvK$$4ym zwOpnSrsPc_U=Q63sbcb3Ln^9U(zAk75?b0jf`F$6TaQ~b-rU8+bFkj4w~-te+$X?? zpdDbq2Yzgy%gENvWsX<62X*8rA%|5>3k126Qb#{S~1b0gw zii8{>uK$GHX8i*ynmtU-7>Aw$;q5c+CvON_%BjS45WRIUL@(<8U=rosuGjI1{7rjE z8ihiqAI*Kw&cc`1h+eTiU-PMY!<5VWQ6p--6$lxb{{N>T|G$*VBMqK-&U`KS*p~ie z`)b`;IJ8_to#Mszy!oP8q_KKL{anU8RX7Pqk)NV&?jUdKZ7{8bGLnhx0eKOpqj7K$ z4ro_W$Bt=|KF>3el}!}sz5TL@F*9++knFhe?Q;+6zZQi0drs?`H5D77G81+%n|)HT1PCky6`VB>jEkv&AdfnwjdpP|s9h z3_y0>j8E?(Y1h!Ygiv_vNdiD{A6(-JvdwVV?Y^`S zatzG>22Ytx9j^6nV%u2z=wb}`CCqqQp%~Z_#LKW2{%bu?eWS4uQ#|~<-BovSwX^Y^ z-Qp|*hn>a#qEVFC<8nhu7bU}tFkh##6@Run(7=+6ZaUE+@ueW_g^vdw6E(5f=Uw}? zn3ih(>DVrA?+nY7pPawHDsqd6a=65%KM4=yid+LIfLhGfbofCn*7SJQsx89ar!)qY zKONL-ykf?XaV85HWJ_kN-OU^zFTX|RnwN`uLbw9G_r zCE#lU=5i(XtIt-SPLj!PPHd=|fZ~?t-Jbv6KwjVIIhp0EPha3d(h~5DnI`-+{{Ehg z>910uax7qnZT4pEJ5Nj%gvH2ugd3P(Gu$qyq; zzuzU|beM=2vvWNINE#|1BnfIf<#LImWYnQfdj8?=$+{zTCH7lX z17g!E$F6HxSuRS(qQ%!7JQY)39{_6iY|4Qx|>GcJ_xg zI=^2k2c{LrkgctzaCer(0VZ|F`qurDvJf-0Qn#NhRJ)r%XI4MK%$+B6puT>b&V;e; zclMdgeXjtlj_A*zVcOctiavu5XAw0#<73|l5%8RH=LuxK*I=bH(R&B^%gLg}?CKRL z|K(G>zt(F>L(lkDggs*5F?$OE)-z*<{`z%6v3ma1H9^&tr;Vo*tFJbccr%6O-iNIMvr>UBC4E&3t;F8C} zTtiYLjwtX>&i~$_LIKn`!<`Za7f6HbSX;BBoz%|Ul}BTOz_S}Nb)rDt?1^*wug#Y) zE2~GN?^;kcF_4<~NAb*eaiBPJo{XDI=;QCH00AEJD0v$QUf>#)3%s`&fisA+q|^HwyN; z2cpayz<5>?n@=m$#;@Vo)3?@aJdxBv#<&@)Cn1MZQ3_I0hG)`PoXMr`tl>}N)1>Th z8hrv(^dE}$9QInvxsAr2*Duf>&UewCon7MFwL8zR&Y8`)1Ex_#ZbeLY+)CbFYkVT~ z!(QYpQ4VHoX}Est-ElniVtKS}uFW6w{Jg!^e|ZZo?VZV(H`Y6}m+ilw6}t;~INd7C zFoB=LdA!0jueD?bU60&*rRIR~-qzS9w$HnqpO`%OGPPyxH>R+IkJH9RcN?JJqM*;= zcIVD8qkn4U%3;kgMI!jTBnDNUAAc@!+$QE1-S0(QFL%c`p_u@|O zmHDxHPxpR|AOQ&+&+sEj4}-qh{TE=-QW?J&{kd-(`zp?tGH?DcOK280JK!1`HwM;l!YN%WEs0I(T?qL7_)h|zL@EnzIt;4H78ZE5-fYD$ zb8+tbWFU?D-$Px3q8IR4YrL(@v!SJG7t%iSD8Vx6 zPLX^Zb7ZyA@yy2rIrSolRz1f#_~@vQDr?tIY5G$}Tznj7wrTp|R$hl%JH0OTGx_s~ zz|iPaHa4>vK~mL3=U4j2CrVPr8i0DjzE>hgNP!~f+pbfM7iX$xd2SN<)vltGVUZji zABFng!D;x_TMcl==q>8j`sE8rCc%p>iHvQDOZ zC;RQl#W+@bmmxR&-CVMc)dea7*|u&@+sLF^&qgv+HsOkpq#O;vcPb9hS(TrQmQk1u-}IFFFZ$>M=3G(y>K{* zVvh(#Y$WTG_!*uZW(hQeb>LuZ##ua#>Af50t>98vwIn+UpiTqhl zbE>mk^sxg;ffR3OQd*e*E34JGy^U5yIr%qgo{XO#b zNv3$?ak$)?Bd=E@Rny6&3hflQ{toE~eikD6i8ufI{7J^O)t?sV+g1^U9hwe{VACw% zUg#Z}sNat_8U}gO!lk` zExgg$&Nq{AtG0>5&HKrKH)@r%BY#XT>Io-dtB=H2U~GHNRvBygc>(U3diTJ&Yvl4? z&*DY%vnw(bXY%om+T(Nv?>aWZne9)>?CWdIg_GBnd&}7>IlQ|O&HELaSz1gyc$8>O z4_qAnE#9@}bc?1xDo)EWqjA4h~kxn6TwF|-6d;_#l4xN{Y%H0Vo9 zCKqdbRj|r1FTy6(&O|yrIa5V{(sVuU{xJh*xuU0N7bgUbFpW9$_w=RC&pmj(V&3H9 z-PrF3Su~Gx-`aTn^k8!RS2Y6u!v{V$8%(CiFEg&BK@Nh;Nqb9m=Pw8Dd-bdixsY6S zwKfvQr}e(7{QJoSpp7tXuD>*Fo1z;~q|fwGenh-$h+%U#S8k+rwwKTs=5a7TW9a(f z=FzJ{|D)@maA{3uD+EbQDsHhHnRXd4D>jLo|DKmQ9*)Cf#Cra&i5!{pVp?Wi7raDe zEWC3nDV{^?(aK`CY^@Y4?EmDsnNL~%?O(3a_XQR_Tn2XV=|7Qq;`7?jDI62XkZbqk z(1GUT-FC_LIRNeTWkOnknf&Cvd2PB_Iq|>#uQT# zQ)lvC`3Lh)xsQzf5@bE=1|P+Xl9wM16z3RyX@9T5JF?Nk|C3kT%c8*@J{r$p`T9@f zJ511S^3rpA#fK!QUTQADq`w-~c*8qc%DPUn>_Qkqk)Pz&@AN2nSUpXL=q z>ca(e%Z$6HzQ>@lz3o2OuB*8z#ArrL7x$^y3n?bdN9_m)V;MziFiqS|%d?++O(lkg ztCoq!P(!@eFG#pZvejd?gu|RS9`!W(34O~%=EO4 z;my0l@1)M&b>~2F7nh!Y4Fes?lqQX4%0_@+shVkv->|e7QVA?BM&B-`R^{`ME0bMo z37g0KKmRkYkkU+X3-16;xx2CUM;pqAwlCasEst;gT1JFdS7TZZ>)zb4J{Ka9y<;=c z#=P8zv(`4X=d)$&#MO!d#;i2A@DrB8e-F(cDpcX|4X1R4E(hxN zhm*V&FsP=1Tgm0Z(Mx5@(G*k~XPn%Go6{#^6lZ7qZ)lA;@Xk$z#1hvx1w0bS&>TIhaqQ_3Nu zV7xQ10OE{%!R6|su7ppw!0uXVyQFw~X6&&`V}YN2-Hq#d=YH2#$uV?fIuU+FA(}PG z^i<-v^o*@HN_8fk)#;3saukoaaQUR|XA!9d(%e znz{Fjv7|>by;<;e>txuZwQ|to+*uBf_{UAc_U~f$w)|;srz|7$MlqmhSLDl>0Pp6Y6ZrCnyu|nE7gg znX!<}C|FWtBsq2DlMY_R5AYDA-|P8FL!Kv_vGP$^n%=-_sK1CsswH$-kH4ME zui4X!63dx5eU-r-Su*itMx(Zrb3-qM9Ii`3;i{10@HwMxwn#fC;SVnZL~yU5W@?t( za9Ni*mzTZQ(WN{$3{>r;Eq_PoxBY^OQ%XxWtO~PoPSNE53@66C=_mDXYX1z){)qbI zwO{duQ5hltoYhP=bz9Y}q<`c`Sgr_c5|M=bhuB7oXn@uG#9{TXTbvw<-LbR+34rsz zVx8-A+Uqaq?+%VE6x@mj4Ww=3T_7FCIi1Vf0!H9oAEOM{+YE1sTC7+?M`yRe*i9?+ zyUk6jmL78U=i*7o)+z^%Z!ERd^#WHs zM0uTZ8*ljaXK&^1)H8Mtz5nB!_g8NH2F+>Xu$ zj=+BBSq|jrmaCJD8@y+T3BJu~fSzp8I<_Qkx8D3(x}K;G1T|``s`Q@0ZwH#4qIiFr zcm&=)s}<&QI61B?mgG&9Jct;aVp88yC!<*y7dqoTy)hSs;#_neD=5cb!N)^VPJ&K zgga--*6U#|bvPQ^+uU#lNP;^{vNntiQLR0UnvYIaz5=|TB~`T*5ncQQaVNtjAgn9>w>7|9uDKD5n3>&)9TmsidKYZEZ(NEl>?a3(E>& z5xD#8le8EsE$LVv?YH8XkrFY0OHoyxN#!`t`+OkUNq3R4!oVBl-qFN5tc_;GJD zQ?S8L8SHK@39UsGS9K>+DkRLD3z&ud_{|)t#E+|L8hyHt5eh-r&yFC^bAt5tL~9Pq zcGVoa<^ryOZ1Bhb#iImmW@qzN+Cn2?!C%FR4+f0}OrAj5zu6lNO!BSFlNrj?)i~$S zO|t2Iw5oi*UKe((T@^BGsjOCkqnQexxe?TLer}e>y3TRi#zVg}$MB8!+04;_CU}(Z zQoqFr({F9dVKle~Fyb}+*1Okr)rnt`dYHefD9<%BE+Sf2q1^&2rH3%jnmqF@J@slq ze!EOJG+3Ce7bKl*Ac~nv>nUsM9u@vdkcKZIPCc)8#mCH|N#4%YII#E;7e}P5%t<9V z!2A2B9LIpcg^Hw!KTUs17iiHx78u84-D8FHS(&tTQxC@yKew}qwQGJ#i($>SwD)0jB~iELLvG zq^PNzk8w9?PJa)H-sD`C3Nl;Tpu!;1hElddK!s z`_%MD+c?@jtCQ%NRx_Bl#KwNSJtCsUj!65t*xApyiW!S6UlASi$qpvpPBRyRpOM>d znUhN0^2?hDL(DU;Y|g?cg7bE@{Uk5ntyQ&SIvqA+~w zeA4n~>Y=-F!D+9*;a-7~B`LQIDeG7I>;%R?7c~W2_RDiU8hjY%{V5^EKp`@EgP!ox zz=HopDowuo^?Pxb@q*fx>w1G4Lu<%4>mzAN15#I9Of|cEv*{9SG|Bm6x=> zL3*J$-yaCBg#6$@I`kztR>>MyMR3$VVD@2qbDkq8ecc=_u>S|qG+Qt#M!yq~+1IIV zD70(7y|Zd?!wLO`V0IH1jIVOII7<v&7IzB4wjH`nvkA2`$Ib3qP!lO?`gx@ z;FHBWZ@qo#PhtJ`uZjk8FV<*ql-O;STA3QE^rJkftJ-t@u^OQ{T1*qJ+QDPhT1oTH zk#Dmc+tdEapIf|(+*l%bA27)}uxG@~63A|QSnFL%%H}Q6@7E@KWR^oA(e;m9D~-AE z-iI&4C48sq+>rR=w$ah=D%fpU{8=X>oST>*R99ohL3vkBHqC0YlUKC!i&fGM=b)>` z39#Lr@L6Da^1$@B)@=8-=#vx^B649;vS~>9TM&o!e$Qr6TbNQ5=X5D zMvvHF3a#tDGDUKvde2Y*4K{hF@fgsMwF|d?d~|-2SMgb+0B6@HMQJEK13*pGlln)<&MN5^}fKQ+YKTg64~SW5V=l zMgjw`x&B4l*qGbX#mqp@RrQ(xxr)D7axUb}@{_7qqgPtL*U@KUQI-7;WPmOX_5WG{ z?sz47raGYQW{j_J@U!CEoL*QT>M_2_GgYn6?@z756(<>Rh$PiRk0bir(>z8$iu=0P z2KgO435~Scc1jN!Jw9uiq}S{6HxcqNI7iF+@Gu#i30tJ*NJ*q8303vliC*XObk}8! zslhh0S5#vBcG$b3L~XK`p6!VRJ>=J{X+N(%9&F)NsVfkV+RJsBidd$_fhzMFZgLHq z)eM`Q@5iZ5{L98=mFZGtJ+~CoM(L7YsoriOJvh05icE4J#spulmOV>KRj0{#CF+0F zrIxm}#NRp>Sr-v#{6F|tAi)U=Q&qf&r%c?#XMQ()=zXT&yO8fEF)=fd-~TrNJsrIV zpgKvNmjeOg1+yE;vVzP;QGM+uHhYD;4fy4euOV75kjt$JBh*@t?57MYH){i1f(_+b z;X#IXvfuwCEB`)4E5kbJ|HDd7xB$46HDj9b<^!&IdS%o}??*|<^$$bQrBBirR^#tq zo+VxVz4ZFW0&q54v)u93&f#+6DqW6z$4q3V9T{g)l(NQ^EV}ce2{5v>Mr|1IF){w- z#NwL^%eP0~^u<*d`B<+R5Q^bsL2*ST{*zXLbY-azx4+aJ!SGnH&7rHx1@P9|>-bEHo z)-Z51$M37BmETZm|o23{*f(P8*D{K5Ktrll%|{GVx*P-%xHH*=aPasEz8{Ug6#oFAjh`; zeFNGDXI&yR=b$n&jP==&C#K>(sCnjj4ElU7{{vvur7EdX>6mtE7|Ku6r0NRi6siW}_o@$c|<$IJiGvo6%8 zO}wYm1iuC2KKF>{J9StJ(vFufyZt~r`e|LqJN>gz&CFR4uK|i}{yG_dvL42EAu!A) zP4*^t50qFub74N7MQ#*KKLmW*npec;&0RJ3YU20Iye(#uKD7X|)pg~cVqQ~WlfLR6 zEsHp&%inO!BlvrdA0ujgi65)YALjo6^IvB%`h3(L0*C9OYPhh++@{a6qxdI$-`%O0 z8MKOge6w1(Cu6vcCekGSHH*88cRi1|ROL!x6(}Fw8WEB_vVH)=8J=aD52YTOe9zys-(i;Ufvi0{@(TRsI6oE^9kRx}&lFia7F7X=& z>%h(HqwR?o^tB;6(+#fe&XxaUTaA*(_;s_jnWUDKs_Ni|!=joFd2FgoW-p(DuL(KO zy z^@z^T6)(ni9Q^$5aU)F2^X95r5FufbynEv>EP1Os%Z@NfEeUN2)adi-6thsB@c5hw zxf2Kb^czS;9gocl1&w+g4PU3U2e0x3bx%$q{9{DGpeXUNOU<81d5|I(lPrORC+=^bhxA2{(i?tC2X&Ch4 zZoQZroDv+Njh8PzoIhNeHRufMTXfNif>i%roO6Fm^X>nX4h3}9?IqsK9m@Ii z8k`MZ7u6lgnQjcPaSy$2$o!ny9{*a-qJg?RYY26YuP+$}$z?Ncgxn?1^K88^7&r{- zzk+^G37a<`qtz!95C|0d;BP3iLc18-bEry*M)qFoZ3b-KK3lW4zMhmj3}olqak!@< zC@QJl`J3S}=5KWkJgsBYz%Es6qJpJ+xP9y{W-?&`s|cN|42ywwMQU4-ks=w(z(s?-hnw3zP!1NkR(-Qocqw||#6x5eQ z{I7FrC%IALem)zY7PruHLg>#JZA47c=YR0I!k$4${{K>QhT7jSq|)^~k2HyC{67V) zt>Ty3@nc;F`)L!D!3*>u|E+^<9lu=)Zp=Q?)p>pv-T2bTMSNKIpK6Mnxv^MMoBbDx2^=|MLd-kN+DkxK;iyxJXPZmLN`HCYiiy^l>l)I)$M_ znR5;W;kQdkQa?v1ZiicyiyYKn)ahI0g*hqGeT>{tvi*;PYOD1V8BKYF&#yn@#16S9 z$tq>^Oj3!9&^3K>ZzCPLtjIPQJSct&iu3xS8^nNSf@sLS>#JP=j ztF9h(SPmalsMTHnUkcoN{Pq<=(ML|%lkUMm4b*T;Ba9yoe)1-zaV>Q z<9PimpZ}5JBuFeS3=dj91@ly14FDV`-eBe2Ol5y&Kdc(4YuDnsP`y1kGke2~+{KHV zIYCagu_z3+C*H*@CRlDbT{c8Cs{ceQ(tA$zN-FeX$Rgr?()BzN%oOUxIne#*$?B?f zFcQwOJS2bysTH(&t7bxLfMQu|okn1BOQbj4||K;(=@MS%5)wrj@ z-R)%QYB#PihEW|anj8L5*BXOP6bw>_`D_TVdqjq>xox7NYqV;DG*8c7o_=m*-~sd2 z{OEoY>}ut)#-Gx9jkX_J{VGWg5@7V>_8b45p>6P+VPAgb;Na(5-dR8MH4q1Bc9s7G zKLN8K_(y~JwNAWNcacM2K+x3k|H0i`MpgB<-Tn#^l1d5)2uMhGcZZa8cc=6Q1nDkG z>6Gpg*mQTdfOL0BpS8i?ec#XXoH5QB|5s=H-^w12&0_7nu6fNlKi{0!hpwmYnumn< z%L_+2hru`uW>HR_WFF{2iqr_n-s3p&@hT$)4#QU4|^`pS3dXIp{RJsskLjKQuXWc_g2EKg}?-}QS)rp z*<90p$MPP!&-5TtUkd-v@7DaRobl_K`pQeyevN?LsuSX$X*vWSJNjX3dqf(meD2zb zzQ;>npGGkZi;qXZbgeuaUBHXtQfoTdp&8>%*#hUS*r(fFG!dUm!dbC8Z=)n5&7pek z_e@Fl^Zz%w@#iv%odp_%N2$m8D>Uw99V)ya zA=h8k%h8Ncd48s+6S1|Qdh_X)3m*EK;4)1;H-3(CPW^cO8^a1B;cwYjD862jOk}r#SsNKCvC>G*@?Ijz_9x;)O-->QaJes9MU3n}e7$ltpNKeID9YhU zb=k(${He)vOY~!JdqtF^>cD0EU-t`2<*awlewUM#Zto@OcZAD3cvJbHjJHJQkOzAy zC1%(8hqBns9wCw>>XA36k$r_AOW5GFoz}%zyBk&Tcu?pqlG)ZLp2}yStVH2DN{S;?@6nIgJ^X;v8jpzY#IX%>s3AKY}okCD~`iE`EA#ZmnCXxaqwr2k*$wHxG(~>{@q?(26O2|zcPjp=e=MbmU>o%(%fW^;Dpx9 z1jUzt?5JUa?Ve14bsc6ssoC8dC}QY!k6MiSWJ(w8HGiaK6f4RtPfnCL#w%2*joSif zd$v)XY%;@lrlz5fDNHZ_E{Gfw^oE330XEW4k^iu!fL8=*K(edwMDMNT|Gt1_4VnnZ z08Sb2rq6eHk_Z1nIM^UgnCs|>Gzc*B;V9Wz)Yf=$s>C4RUoC*&T#_h{ahz)oWMsAAXH?B|DT$M=f&K?X^qoW;HXZF;VD=`W8;I zod1^5b(_F#WHJ%Xspdia^w%I+!n_rI@$2Tk#*F9T7VaogX|(td<*+r3+K^a;X=Z%I z(zyP$*olU5)?Lf#HSUn>)m#5!Jf~QEyByq@r2!VQnT|sgk&ctXBo%~y84@WST|6D_ zmvjDngPKQ8k111ifdMOsKCl1Vm~LiP}bsUX{_ZG2&-fqHl<>&(h*P5^<@_`9BD!9y=xKxYa@d5nX13S!%0-;;t5w1hFg2p#=1DNn|3vmv{VH( zoO+RS5==x zVoXmSHdtB)8diB$jX@c5*{jpKr?1bX;d2mRlSZz3luv4xfifEmy*=ttUg1+-?%NCO_I7V8Dxxz8dUGem`)Iqo-`_7j zw-CdcG9vF?m?4=IAJPsBMM)CxH(kR$jpP(f{Hn|by(ksG9>g0ur0oyRCtgtJF?LDW zvX_op3Ozn4H7e{f(LF!w-`e8xMu@6@Pcg`sf;gxY&;8o+f4D;!7P=zHgD+6`a#d&9 zd5*#-h%GE$BQ4)OJSiyZi&A|@JeI=TryO(@t z4SH@mK1#zCzf}u_Fwt`90_3gyQquh3lCky5cFSLF_U<7?$;8zwBQg);W?+MqcOmkJ zb6r9J@eTuzTJqJf&)ve_M3c5uHP}PB*|rItiQKG`l*yrG>IV?_q;!Ncizo#Pz?=h_jq&5^9NC< z>>`gG&8($rTI#gd5EyLagp;&N65Z262w+Dh^*W1TT5d2n0m-#cU#_DsoM1adH=C~iiGUC`^hUYXdMyu zK+vE6o{rpbEFY5nQkyqml%GV#8ak&=iewS_S%0Y-Zr8Hb;>lxp5PI+8o@-l}G*9gr z$)f+8cBnW(d~hymroJ@F*l8BB9?W~Xxj^nPMCob~Rc<SI$r3(){<8cO$;A_In>Kx?hh1Xy@ZU0!W#C?z@?7 z$~PfPf;~KkW%+D4>s)^oHN)v{0k?P;y1Yl%1-?4nX#Ta0(6O7AY6F%6^y^9#Z;?NX z1yvHnrmxyn$0E__s7TDSNv~gJm4K($h|E2&WOZPobGh{A!64Z05%5dPy$i{FR5>~L9C4m2HTPgyS;QJ`B5)f zrvTo}vZ@CV1k73XZGjjzAy;e~|IOnmO=V@N8n2WqGlN56JrtUXg{|_{fztV_km3St zfXj46qQ8i|r4{f`gpcDl7+!vQS7-iP=%aXq1Xtms*PSQuJfEw3=~LBj(#E(+IgJry-e0rsV0;aCS?dk&FLKL!j`F2dVc=ySJjj?cmnb{I;Y zUrrsN%j-AbV3E$o516u3G;dzvKZ>uFk3Hk#ePd+OW#|&w=xg}qro5-(6L~9R>8CdgHr#yYJr--CC}GZZ90=AC%$b9{igiw1-B^@a*x!pH%*t z^P+M;wa7^24mP_&m)Ka_1&Sw1*0FpF@&_Q@Iq)%XCV6{b@wt5w7oa1uoQ*f=Ei8D` z&2mPy#bcx({NQ07Rp!~^{`JggIH;caXMx4-e!JUb@EjnQe*2BssArRD$e+gbTspE9 z!Qr_sk#zV~70)=nTG7TQVUg8?T+y?pF{P#z;dXA9Y_w^eX1uM3l;7`u(p&7@l^8HMH(aQO9-4KniS-Kl z9zii-05)~iaZj7&tjR(y6SpT0{7n7kk8^zF9|kZqk(2re-dB?8^Pj_>{^FvGPjNO` z%EvsLe(%5)xTGaqD9XkucmNeRBnony7~29m91Au8w}Pu zRXJ}GGyNMatO=#%B4o*B5obY4cP2$RAArJGun?LROkI2%Q!OJnitD4u%5ci4hPn_` zy?44O8`h~_+H57o2*@MSm!@b&(NEGx1j#|M9;YP<{>6=+V*NUW99!Q$Ljy)w5K|=S zhQ!S*{mYt@w`P1C>#QH=!~?6H>E$5);~U3GY{3tm^nmFcx9; z8pD&5iA^ni`EstDnw+-+3AW+(OzE09j_>gOmWymyRVqOp2kiv4{EiUJf~8pq!)|ac zm5n00r2QJm?}tpN@_{S@D+g-qDUYctc`nu6Hsj2}Lf{EJD%TmQF>j{aLRyF>c0aU+ z7$oyZfj`y(Ud$v!)Q|N8l-5M8l+DW^DH7!()z6my8O=lfl4=k;Q%Kul?xggwt?Ikn z)hrao;GW|gQM*JuOzh1Bg$od`l@=zTqD0M;??{trP#T)bs<5XGhJF1C;J&gM{mUfc z1H1VG$s>aC6+HxR;-j|Dep*DIUNH}Ve!KKT?PoD~rz09CgnwvPCYV8)fR4@)duxMx z7=UbQioPoAK2y4gV1))F0AOZqW?%&moTW1Q^Szw~toaTGgTyG`^XY$8{urg#Vt4Woz?FG9Fbaj zz9)hC0*7(0nr!OZOTb>k)%S`7ntGgD4qhrizb=8!rSq2EF$6kPuXqq+Oj}TUiGO6k zr#h0^uvA3Jo_wVm#_ezLPXwMwX!zCMYjtD=3o7-_UAcUM&06KwIZ|VEG|Pg~C1v#` zN-_1O5A=;J&_Vj;rQsvSXL+nnF_UI8~gq9pF`pJc|GyTYr=B#b25#cn{M zEf{9|gnK+7N=CaepmGmVLsVLvZaK+#X;BG^;3u1!nwwF^)MiI{QKRG77DU(imR3Iz z6wwX-r0Rq_yXkf3^4KI)&~}m{;f%e#0_o1GLu=TobSN=Nip)Xp23*#k$MT_57u($j zCmNjBx{S^aoMgqcOKprWbJ9dzdM>=|m$F!Al+^h;1|5#KE3F4&#jj}v`_-a@;K|MB zb-DMEDb9xr0~VHkc%QV7RbQF%xq%de=!@tDCl^0RU(hzstTX4GIn3(1%G_&bKWb!2 zM8n;;#l8$HP5Rj+f;j%yXeAN5BAo4W+!oD!wq(N%=Yk}xA-6@YqX@a&RK|@BNz$gb zBh;ncb3c5(h~d!%eMg%zX8XQ;P@#KjALeC+CMeE_w3?KIpSLvfO!}UexsvKo$WNWQ z;wwKVyvNwrU{)U1eJf*?^+9?g&+vljwf5T89uybGnpKKsbv{KDArnm8A<4?$F2(T3 ziIX$a1Ep&r8DH$}XP(#c;p%kdbKlw49SwJB?53)R3rE$zbeL?UHrz)#M}8ew@wtalowB>;4k$F% z!c#ezJ-)g3)2$k0hM)pDPoGPelqCb$xFaWhVe`-$uR>~eu9&a6I?J0sq*S^lB`0bws4x)F)jb&c&E&TKYRD_MQLr=1#Eo-Hp9<&vS9~p;8QT3_ioQd> zhyoJu#7!O@SvFFq$@8K?=Nrc*8Ur8EX1#a!`L zk9I)lF{Rn3fAVh4Jx!3LS{ZZ}H=?G zPU(-s($D5dZkU7`tQ6hq?in}NA_z70`L+2fU2+ndJf_f|4KcWCq!Qe>Tk0>+!SJbc z9#Ym4;{^FN08U&Gw{YjhIS`XeHM)w5Q}|oK5^rEP>Aadn)O$n4nACI8y!Z^6D}wyV zud~BHI%&{}K-uKqeHR0W7f-GrrJ*MiE3znN_iyyB=SqzrkLwPNgzI@gyvqx1wxWiQ zM>5Ov%ijHTP8`UzJm#ikF1mb1KJC5g3*s~tHCU}JV!2&p0^^Ga8gAsj2*a zdcF2V3L$?pw_b!QP|d?y?ul>`DjN`LGaHdik|$xwvo@G+z?L;O2|Q0Yc72~#Bksjy z@-PCkz>Mob;)^u&3dz@8*N~ii-FQ4%-)YRFZe&!UIMtb_C; zfP}GBIpp(7a&{8Ds>%&cXx zMaKRow#R-~P4%MSj_7;i9_(LrGys{M|b_bKuHRB7-grpI`4Eqq;3Lcn2)Wz820^t?KbV-j=#`_I8U?M zR8EFsTc!_^xATOH1P&4MmE*yAA3=c2Ai|0g=5&z{2OWxr)E@-b53)=b(?tv_f0d#!;X|+6oZhh< zxvK1**{0~5*;sVOGo6336^AGCVQ@cuov7ck)m~kujTzMmbR~;X^0^kr*SoeDaPzLB z622^giePBQ1PN4Yu`-T7>Ef%Xpein{p!d?d-@G%It)JCX+u*S9$(sJcg!{fGN$)i| z)*&eqE=u2_5!{GvLe46uTYrtsd?j@1jl!&#x?c<-NN>&{db7OBfxuchU zPd`8C$J*-`5Nx8t_D#C=diTXs+n(NjbWE4YSK&P@z^}o(uS6;DH?!RaGCtDMzgwSc z|0sEpPd@e(3G}+yzkbU@eScWlbd52wq1NEER~q2V3Aw_7z!lc}Y3$c~O?oz-T@8#K za{kAIW4%4|^w25u*!{BI)CFJ9>(x2`kUh@g+yk*qc^F}f5+h+NK04-aPELQ%ekjfA z|KR87{@gCd&3BE+4`;JRuiV8hXXORg;>C{LwEh8~tgjIZp}NFlYz-tl@Xb(9R38}G zXIKw(LS2gly^5=r5a@5#grkerKycrGqG z=3nqR#UeuNU`jee-MDOxGkI;%Kj?V2zn|%|&px;Bvr4EE!136l^1N$5Rd*;wHbPX> zcKjP(gS}eIll1%iCD`-f`;%2vASpa&2fa5IX~KyW!_toeMVV~r zqYRMaVfi0N8sOtl+y~OfiMdv6B()Q=9VBScBIBXiH5|Py?9g_6uDlDW0*&$gyBnnU zi$wUk(@G$V3E2$eMbvx{#OhJ!{wVN@Xbo2WodcfzzkQvzEAI@q?3*Bh<59_ z?_L5g8X0->SR;Z$2B0N?d-EdW{};zl)LTIlxmmp)Q0eTLZduTLp^%jD&S%Zns2?hO zcTCS5oAAs29{uE8ryKA)*EX;pzP4)zw|2;Uk$0Gt*D)6UAp@s%4HYw`J_67i@B#~s zG5nG>Z)OgW?ZC#~gH&X{y>e=z{NAvo5BdUqqlBN1U_#awZbz;}jU>HfV&2d^=c_LZ zUN)XDc9y56*Z@zgH7lds6v%F{GV|X}IK8#964}+bDRY&CqpVe*F|&d{ozwMu?i!oW zWZ3HGHkOVVE%ZINKPV=9IdaScz7!64mqKGBh`z~Kip9OYGPy#Jf+esd3`oTZx{`XA zR9u09%1@47Y?A%@K5q^7f)pE(MTPpkZZ^W)cAIdyi~w6>J<@)5D_Ct~G1MvLOj8)%N_@n=y-xMqR9P9L|W+=|352+NDz{ zZZuK~K`<;*v-h!gM+6)>+|voV(I@!Y9-jL9{BR^x&0`=-t(;97Q?1l#fPArz(}&pY z^@&~O^M>y{get+^YKnT6Rhzq_lT?#VY z4xx170{AbwW=2+FR_i=+%kG&oFqVD8br7a8K(2PukIZ+_sCh2|%b10)%dWz|vEwll&g*M8n zQm;OyHa(ISjw`rZ$5eh&xzH`&@T?XlDXeg$M$=L));5vRMGG~rduCyr*mC6$v6BT#)eBp8^_N4L;T<$l$` zd*bNXN!GM;@uiAtNyQ4?LbR8u`?T?F9V@Dd;KP!XPON$8x=+p*BLGn$Pqt9QsdL|N z=c>7ZwuC;+c124SWbmg3s5g1%elZGamT{x4;M6W%D|T-5o2msLMA=J%oFG}h%YDb5 z=+?H=1t3dk6P{PNWn?$&oXuF2{BYNp&iS%5pzO@QRzP0RhX6`MMUCR~2HE?k^|3$P+-WVU>2%E8R2ENTc|1GPHV*8?+)0~b`qK?` zY`GftEap_Vxa^|nFtTX4F0K)Dc+h7q^UaqMzu0kotGNUUMjY~DSNhu7PJoX+>f6o5 z4U4PJBe+?_&gVviWYFm%nEAYa8ZSNQ%Xg|{-Z;O|V>bJ@`e@8SD^`3)G`&JQZ>}0I za_uwfcD|8c8wgmZUz$B}5awnbOVy!^yTjv@;gSjH6jv+d>RO-WM38roB64;OAEEky zG?si+asxCeWoBgYj?TD?xr-%zz4nA3P)Hy_w01tLF*Jjw_65f!8T-3{ylbOuj*iKQpP{9!%60B*!V=%jlqF z%^@`XoVV`$xkv4pIC*K9;Ua+Mw)q$;yxsp85{e0{=~NaRG8xzXFO6cEdVN1R(-e4_ z7qk!fX8jV5RY2^}4sW*-|Fxhvp}+DwhZH};xsffQ9BDN2*`%+i3dcmx?{_Nk?zlDY zIJRN0xVC+X@+LQ_YD(u(X@;Kd5i=);y~W|wpd(rL$hifx zpB|gO`4KGfb0%LJoke=l;$$SO+5Iu7HdwFH5L z#AsZ1nQqy?HHf>BOJp&dK*$K%2(g*Bse$h+J0IXmbwCB34wY@S@17IRG=n{S0-Ewg zt4{FJt}H+Wf9NYduRg!2TO<#Z5rg{_4eRs`4+#Q0ICy1ZT8-sD0A=|L^lH$p;`gtm zHt$4R6xI5(1fr6W(Mj)`apZoSFj^!*bVcg={B>eDqB*iF|}i(aah=9(m$`1S3Y#h>3^t*3bJ$ zmLsdYo~dm+8GN)XnJdC*aY}@kH)f)M1Ed{#*fE$_8R>*23h*30{ljy(a9T5BFYQUY z)>_TiHv&7r$@u43h}CQn0yh=9LR5J+d~8ERCddaXY><1zaK2#+$>LDPK^wyg$gWtA zj3D?PK!RY;RZD}J(3_X5!36m3b{~ns#)xk-v6q0v73_M!6_)s|8Lj9cE#Tr9&9oq1 z_TnneG0_9?yu9WEIib_<%jdT<222OInc;(#kNu*uFLL)uwe>kpPt1%_F2MgDcCw(7s_fc9ED2hJHL(wSU_7X z*TaP?$MKqFI$x27LeZrgc|YvENsajC%8p^KVpEtj zI6SzfH&-hyODx!7=He4>xpXhN4IDtjk%^iI2GU=78JO_@;vh`qq}Dxw5N^fu$4J<^ z{SryR4LpGPp)@iWr_#Wn*N(VH%EQ1EoMU7`%YD;{&I&j%Q=BhF@W$Hew{HYe5ZZev zxrumK$i94?_)e*Kl{TbTVZv`f8+%z@MbJaXk9(+;kR0Ouo=Vc*Wp1q8B^R-ds^bgh z{yF_YRDj7)RajR$=la>M?|uP_nO-Bgx)Ukyr6!Es4DT4WRr~q}oD~A1xXkY`J$RGv zFQS74705-YE^|Ik2|4T2+-Q2V_1;;3U?o(5>cBAkSQly~d#YDNUUpGMfM?(F!94Ui z%oDKTxau)v<_Gg~6hLmZBRN>>Q1e|?=Rr;FnsY3@?xXuoW1!VNMvDxpa0D$hn#W9Q z)nTVz#rB{-FM`+_A$&7Er&{UOzEOOimKCI5!gVd5rw7U{ zwf|8(pzDS> zb4Ep_5q)9%hHcdmYCm75%Dt}i+1sixqNXUd4gsXaa>6-wsS)B$zU~wQ&O@t9ZM4H5 zWYRAs$BrGl69T@MRz;7veu(|YId1q@t}}WCFrih0CO zPIpU0(WuOFV2f<~5bmr-(9QqAiDFQ8e>L2;yNuWNfZ_EyH{nPFS)mZ6m~CtbP__vg zb#^%}QKtxgp4DOR}@>1~s z7c=fL97Kj$vm*FfCqVdt zwcS0S6)3Wy+l>c72H;0_4S23zk>lP4EGuH1Kn3XSt;HOKXGe0+Xa|Xa1%p%u=^Kp# z&PlJ}K8<6?Re%VvC zq{e5{j&M6_wrg_VJO)t|)JWENQ=8azIoMubGjO-8G~*Zve0V6AV4&^^*rqNf`=B$1 z>Z0x3)^@T#-2Xh_0UI0(8bh@-h8QIybj_za40LMm?kLPQaM3OM=sLt&Jr#-|Q@+&t zw?`!22M_6#VGCekKa9S8rcEza+Wp4ytty=Z;Ug%oD-d+--qR-}p(4SvCjbgcMY+GR z#+oq)UCEW=0T-XGniaji>Jbl@;1VGl5zHgpXgNtGW0&EWZ&~(N{ zKzuydK7EV`K((h97`W4<)N{I>uLjYU##GiB?Y~R8%W^NRH59$qpcGP$paoTIA{2JV zVm4uE|0v&K2k%)qHr>YaeH&XUFCD49tjb&ZeZ6PA8^adla14G60mX@w%&TtevM93! z#KuZnMgzX36&?5DGQ-Ho&TGzp7StZ}%%fk|fyfO7_VH|gB%2aV^E56!+bD88H_YHS zqn>)P@44Nf*|R-bF$m}4tURBP8<%+2byyp3;I~`rA8P7`tmi+V&rQK0Iac5* zbwY5N8rYJn+=Jzw*O2E&{AuU=r-gas)bA@WurMc=6(BE4F;ivN`W_G8KBoVstE|lu zQjW)G?CQUf|2y&$k#Z+uD5C&e!os}tlv-T1eY@|+4t$U*hj`87mcLN%Rr6*6ZvW0_ zpb9?6!?3HWJ^Y5k4YKw(;LuZYt%bc9XPu$@xm-7N^1WH>K83j&Ry_irrWl2tM>{Sa z(x4Z2#ACw$RYh&MEt~{=V$tFuvnqZb@|gm;mq{NH0Y73ioAlyepHpOQY9~x-|k8y_Q2?BdZ56UMU`P$VH51-pmco zkZ_qb8;hI_8LIjm&Ih#`SP8A{88S=~YR@=LXK-b`y-5^#R|7OMus@^~QG*0kIq(W) zgZX7Z`&%ZZv6%sPkX$X`@Cv;LoD6ZAs^GlXqe4=ERLmrzv+6F0Et03-A2kbSPvmcw8UCeb0z)}^WhtM+`mW|Lhe}lD7JLCjO8AxG=#H1IOf8mUnr~XO(Cy#Z%kvsa8VygK8SXN#eLwzEQG7*Uj3@z>&SuV2JTeF@b zwn^oF|05+nprie~EC7z#8d0*J+U}WMFDx6^=K$19(n`Cs5`*o~Ki%)qqiTLVYhWIT z4TmubJ0jaoZyJItQWV!x4~>~(TszYHk^D*ZVC7Qvxb36pqCd{ouO?MjbLubhFh$do zCp-%BIqwYmj4a&}4|iV{@f3G=B~vu2fRWffo=h9a)FDT+Vzjgz-U0Wqdm|86nR}ml zqOKD#G`gxPGeN($*;=U}LpK}(Sz2H52u9n>M;eNlp%6+!n$@&xC6R}(ga2Ni*24q> z2K^2peRbe$+0sDgwcV8`Hk0R@ExgtT*Gq0J`XC9Bxj18ttk*SWqK2WEglNE(Z6Ek| z@ZR(szO#HyWXbkb2P9V9bH{N~$jEcHVGO)(&FMdYP1!nV<7TRW0_oWg`2PZIV)MfP zfxJ;h=+fv2#2ISDApd_Cas+=~Kh(H52i)FpzCRlyhx)UJ54ag6{SjYXS2(UBxev0?pn)(5!{tH9b8!TGNt$bl8c< z6ta!Nw4OqR&)65PB$< zavzF&Vj(|FoJGJ7#6o zNvf;FHz6pIuMvQ?9jA=F!yn3aWk`}}y|2s(je1>%S<@90yHCyhPN^7tAh{R$+v&}x zS;9j6^WOR7tZT;$17PWvsHNYw`LBm9H5Ny+XPC|W=Xlw~+rK!Q6s^Q{`$mNgD!0b= z={wA{or@ID#ts#|?YNDno-7DhU~X2Fni*sq_K27%3j^R;o9%uY-gvP0;Yku{Q%`+&(zk*HT;IvXfbiKdeMqiVC}GYDD23UEQU}ye=q~Ow?+Mi=Pez zow)Wnxzbkb1ILjz43MkVVKljluKBHRC|27H@-{tK>-5vjHkAf`sRY0MPy(=IwLGU* z3RTtiEnas{KOt-eZ$75-R$7G9EuBSqCfxxI-^=KnB5IK<$4axBw1}9!fJq#>v^Fi1 z4q-y04>@&1mr>#dwX1=phBEl`B*3GMk$(9YCBnLras@~Sn|2aQ zAl7E-8WU&t{?k8oWBc{*V6;g=RE$!iBDy_ot6rHe&gGWe3Ajp(!47kn2-hG%J>F;iZKME|!eSSn|_z-{o%i z!~8!F{MKj|)|ZJF$}@9Z%NH&gO`a?@bdttQ6C)pwT0DYtI=8)?v$51PLUeT-oc?w_ zSGw1f{sxryB_l?^^tgVc>ofUkyh&_RL%|0)R7`vU^Gj}%@mogQkMj)qIlLW!+Obpv z3Ru8^p5^cKZVmEOhRqJ{^{AVASqacP0}|t=0&y(u0m^64B_kG+^jfO~>XC9jbIkkV zcAeRz(G9({OAV--;r`l*c}LxE%EE-f#}Ww+VRVDqz*O*^OEM(8oSIU1_j?Xe4u3tQ z^VuE{*7ZhC@!xD256q$!kCXojb`i%R07Di?*&?@(Xi^bf7ivbf0lXMyjMiV)qEX+- z*3B5OxJkwP`i6M<6vS?D@}hJd?VgqN_ls*$$ZJFy6MkbTc_ z&g^a`)842(9hjR{Zx^Rf);~MH+7ZYE_e&8mVB$yx%hlabB(j-~&J>-A<^MPL*zZvH zamyn`@f6|J@e(Cwtf9Oq74C{bQl#9TO?-{pv4G)Kp$Hi|)OEaYAvYxOb0ZJRWL3YJdciE#XZ zQsTZu?JxT_m!Y}efgDxiUj9Bn8WwY=X0bqE6@zE~(+dS8@y8}&=PSi{=&*(X@zLX) z=tsI`UsLI6QcXXk8EM%F>}2#BDL-{?aHy+Y&uR<@Nn+qT!jA0%cI_vO4gR~t#1`pE zZDlgR%$-4VG_zHj{!_czXf9&4ze%4ROw^!33LbjgGR^){k5!)h+>8H&gI7-%xgTPrfSY zyE3F+9Qb`sMX5jQ%=>Jtt5p9htu-JTAdUEUt^`#Cql4K56EQ^)7=LJX)#P3^rMAqC zyIKweO}WK}f{W{pz~(V&YwAxsh@^qd{K)#MHy?_PdX|NxIdaN=RU?5EKos3me?8`1 z8i>V=%WWw?tmy|X!R2VI{}_`BoQfyW^kBHg%l2|g`XpL!TXwpI#WU6+;uWC`)yz|> zRQvCP{uNUin}AEov0I?1^76~$3Dtp$H8YO`8Rn9hG+ZhObupr+67rzKWBKoBk0y@r z>5p4O41#oy`|b*R^xL8Ezl`3mC5>=5VLm<0)1m}cBZd{!fQNeBq;iSohs9oZUf>`I zCsS(BC>#EZ&-wJ%hl4L&X$VL5=OK?yoJ{`<1z^((JbY_k!3g@qB*+MHup!2h0&l<>fTc!NaoM_`Pl63 znO}}6`z|IEY@g4N0hy(C;!7=6Eda_aw%z6s=MVH*&AzTCCXB{gq|i54VSVSmdz?aA zFERsDADWOjd4M;w0(%aBuEBWXv|0Y(*_bTGmL*V$dy}X!I_CN; zcC`B*(Fq}aH&&c&fR2^;SPeaK)^3o+e%ZAgtLAyPpf;9!tbTV+zN+#UjA9L@NQ#Z$ zpGa7nOS&rNjN`(M6H-qyH%Kky_|e0#YZhW{W$3!)Qv-mL5?t4`_t-y_;!r&>AV0mM zUI6>5Wy-%UpPLZLaiQ2c*0}_8PH9-`w=J~p&=X)u4Pa3g1Bnw#DgnGDR|8OBRw zwER{;>Ur6+O5(}i{clQs3MaN{R;RBIF=zK7!u=d>VI{&=yUbhP7KTpM-V)WQI-tpg zQL&AK8D6x&mmL;EEU3%s+hqtCp6D4Nhk^;H^5o=}N zF@J{rhR^!i))r+(Gk+vP?@*fx;f(Svh5v=LQycg_7MIH_E54P+*Refs`r~*ipi@Kl z?G8IK?A~w|0;o&s-A(>Gf7raR%z*X31-$!pl=Rd~(Y^J?*8yeb4m<;6wz0ToNi`%= zR5Tc`NcMMvBrItlEAaL(+dB%87qA&`4>>c2;g{I$7eYR-b_c!uuFizASt|Uqxoh|9 zb`Hvx6l!@@?aCJ28T-R0ALjR4w>IRTM61u-eo8Pc%eXYXLJy$Xa!-E)2g2V|L!RU% z6*p=f?C9Y409&h~s;5gtosO#9b1ia1pDQ$f+bIMkJrBoTuM%C6jrd!*6U*Ty02+7W zL-0Zsz`bhiRy#V9fOTz%#ZU2ECJ9p}B{fC(31mK2m_9~q;P*Tpf2E3z22oAY=0#Sc z2K0>?-X8X^BTW^dv}SxzbmvEr22A|fUEtm@`1a`G759B`fyH_=o*wIzC%SXz>cDQ0 zVw{zur56x%DYw9f$C9d9l!%kCI&I{29*(p{A|fvDLb70C{|`IElHzIK zYMBheQKDYo{adBU6I+Fq7{i%eywUu#pUGCj3nl5+aeJDE(pcNdHFy=K{`oPPgA^;6Y#{O=hoQW3^Cc+`p{N=pNHI!nqq&LwQ<_Tx<5U3@z2ZRFo ztmSKB3Tz-T)O-48nRhLNAodUzE`0R#!I=cTgG;TALfIRPdo7OI9HowfyV|IcbJ&-L z4tx&igs5Lhehr49T_y^?RIhcJ4q$N2zK=i-w=mK5A-I$vw{*(X1cs;_bhF>pBwVf| zk^dDUQ5%ATfW@DTdbnH%)UL7wW&5Df81@NfWM_P=kCX&>9BI!=m@*o%wXm4KuweU} zwoZ#<(&`}WAKG#O8`@z-gLKWfT+yMVX(|{3?w+le`N;&zY%4TbDQaK$17@8PQ8V2o zppu6SA9y&IT~f>set&mpb0yx|Fo~PW7Gq53SZKYN9lnLz*4vg3G7n5ll?uyak9xm! zM6=$AvmVj23;C=+{*L~o-1beM0;rG;)gXC_&B#}{%IfdLH0U1VIQ$= zK@_5lkVN|ps->lmSf98m!P$(CR>e6>c?iD~FmqIwd$>{DYWHV!%js^f$sOgVSvcB% zf=_H5z62K4NknYSagQblY>^fPsv7m8d3&}_j1(K=Wq|pdVoL!62M!s~4}>J?7(DfZ zP@|J|^~5?B(ODje-I~;>IqNZPoK4T_mM`It#kOlV+D;4PJ?v zjXy`*+BA-rKu8b2rfJ7Y`h~X{kP+f2z0k&uL=0-5Gr6vJKOYxdKP1SHsv$$wjut=M zAu~3Nyhl|<;gkV`64xswG@cOz7AYzGs$G@*RA zD9~+6zrDuZ6fv=nJ+rur+7$a;ok7MkHXL@Q1jo%oRx`nt-S0hU6qiou-B=B2L*R?o zEh?f+&?$NR@B-2Q3apuAUwa=VUI~ZwK|$ZdH1v`4qDB z+O0eo;d@VsEE7REN_1*IG8ct8biLC8FQ0yDEF29ok>J632 zasJT{J8mz3fk}ej*hfS5*gpOogLy%*K=7|}jCdXWo zuhzu5O*t5FQR$Ms)|~J%X>#E6ecrfkDx4`ER~YUaoQ|fSG*7)e4Jo``pR_G{PA1O^ zoY*v=l2cm|xt!G!iKSC=EX2fmN`QC#{n8r;n2Xj^i)g~U>J8kS-crQQQ`#U>KRbIE zuyEtSQk;h}Q_!?a=i)+iJ@%6z(Y!Be##drlNHn!@P1WjNgmMl$Aog}^GyBV=2{O6AbpX^-=f z>B*%;BT4|m?8~H-vTIHps-Z&$XsJK-0BPJRKa9*wW3ZGP`FvL-FaCn5hdJ4*CAYu_*G3@9gu@{YKB4T;3= zgq)*tQ@$~6LZ22ditz)&*?c7Tw3z-^b8i(L$F^(>Zdqh8*kWd8D2thyS+Z4Pw8boo znOU-!nJs2!i%QJQ%=F6J`<#1U_v<^(82!<$zpSiEWv;a{R>X{mIm=3PpAVTQ)@Z}M zPTpIQX4tj%`_heBJh;1MVJ`vzxfij#D6l9wClWiK7PUjboGNVKIl)D$72N-wcM5XL z<4vgh$^&mgIZPG(FJ!CIfai}+&cB(}2vU~*=~cNj4>B;W%<#e?h^eT-nOG2?VYiIQ zODJrMGJcp-`hLvt2ZVJuq@7%W6OBs|xk0tPxT3!dQ1UM_yA6@l z`{ekw{X;f@T+)R=6zhiq!0ODh%AzTG_XHS%W_D!s0eSwoY;@0jr}>LH2mPQJyte@H zABrI7p(`vhN{yKjE{}qX2Mb}RFGq!f%+~AOE#sWj-Fvw;2bf0; z^L>orWwV!mINU(rn?h_Fw~N2aIMt1NaQ`wHz^ytj&O%R14{Ke>;himJ9(~5Qq4{p# zxfm1R_*w({$MHq@?f5oRBSnC94`IM9eegX&fcR_{r<`BEb6j9Lw8dW^`l+2r2}Y3n z<{<1tJg$l_hnJ^dqFDVeY>L@ii6>-&zL`?(+JxF=LM?&Ft>?|nJ zFEwB2XQrs6`>IHQC`>eZ&4;185T_= zZNJ+r7^d)v;hN#(K7E}U6GjQtD_**b&tB~+D8Od^qpPD8G6v=^^Ua9H{p)9We9zJ^ z7xl80e_Hoj&kVns|I8A*8H@oxYx(ObiJc?1>Q){-~YSu$jz%6@lOQEW8o z*&8_wravZ~)c4w|7z$}(>rrsh|M77CPCW<~;mk&&TPM+Lt#DQ#_B@^)Y_u8P zs`E1!2oFcKGM`k3)Vcg}ZG?c$3&>HSqU?T@&(h6z<8y-u+2H#IsC~_yEFgU_LK;mH zHC$H5pI^&tDT)1Ze=9UYy9Nm+V#A~+0~^3Vq%A?}Kj5K=?0#sbLL52X5EBKNEG_=c`{kr~am`c&4RVj)d> zq&wrB>pVwsFSuG0gRC zvLe)3hPt7oJJzo!Qz4j3Pb3+JLujd3SJ41|t&;N+fl0s=Q^}Hyvx`6EKKvr@3BuXS zD2+a(^ql0>&4}ITkdSorX0+2QLl$k*k~oN_-0kbVRH`C}kjsf=otw+Hzy3s{xMxnI z2xnNZ$JX+IJYb`Scvo|*!TtgRq>KY~kRne08dqjwr{njN6Mk6CcpN`eg2QyTV zk;|V8M10jo*_yk#E3tu$oTHt5kVNA<5qpor`Cpi1&y$I{@ZenU-)vq6_qY12z%w|n zs}5c{a6S?Pj66nrDGI$!f2)cg@~&gq(;)D3i;Y4R4#fRawIr#V^{0#}n)p8e{{DC7 zuK$xm4R+K2i`wh|qc3dk{kPtZ7%+j)EYSKVa`x<;bm{g3O9 z1xv!7X)W5-g62LuB@6%K;X8*c7^T3V) z*JWi>8UNE(BDeX_NSKc>o!@F)QS94CW8(j{rzQm2#@et33&pD5vYI$OK>^%DLin|l zZuffAvFBSEnwqirjKt!qBftulwmPw}nDf)!f8#PEQSvx^=zaHLNxguv&Z-q_v`9OS zW{2*#MC_ui+EcXW$`iQvlIibgVH`QH{Ib+l23P$ISyrQfE@)2YIL6M2RP}ZVae89L zY+ey1C|M3h{2$TQVx4tLP}1*?67uro$3Q)c-rYG4&9(-u;U#eNaLmM>r827CxVD0z zA)T0fIJTne)8G3pIaTYSwf=*>o?4N0#7Fq*?-r&6Vm{;sWe4 z3U-P&?Z^8-8U9j2v&YtK*$zn`RFckmCqdFi;c6!HiY>EnA`4112ESd<<28t*@=&?< z(-*K1Qu-GT_WqWBhH}bc1pPyj=2QQjd}@srLw(9KnnC?Aa7wtd^*021kNUo2in9wd z7gmUm3!!v6eSceH>+jH_`olW|jqCSN)?o(LAa8sJf3VC<>+jSt8Ij4aKLls`A8LOK z&tMrFuuM&sE;L8AU8BHh^ll)yvF*2#s_6)2aFnl${7f#{iw3o%LZEh@RQF-Iq^amwOTM!H&h3t1 zPnYON+c7`=q3>(tqYS{zD6}|W>_wFjbkdYQua#agR!9THZ8{g4?vX+F&KU*pq!1J+ zSPWCo;6y*~q54p9Q!OC;UhBWQTyWuacZCho^<>}wQ|1Ql8lh0g{8Q*QGS@@(r-8(v zZ2F%Z_AoD4OmqiamRtx`Ih%ci_Lh$Mr`bB+t>iYntw z`m4%PtE0gt8`sM}X>mgVKUqIuW##!S+bgeAX31^&!d%zdgSOYut*6*=%ibT|7_mji zGKXu%hR=LIEx_V4LnupO#yXki4*#y8D+8@;mQtJU6p){6{>@6Q8H(ec-a*(BrSbf> z)Pi?TUE=P&z7}jlTa(O5V}VK`370STD^-<9xC!0-Lmh=7$ybUTsOThpl1HrbVHvv5 z2=kXY6yKw>P+MCu!rNlELm_Bn66C_@L*c}HNp-TN`pSH@`d%i^Mn_F*7OXgZap%v1 zOo^PFoJ=OT>{<5!dp}k#BHp%a3?if@+A902X8@}nnY2qZ?e&agC^qL{Q1GmnDZ*(*UqAmw>EbY z{bk$vCFxpTi_Ptd#}$)DT;eh3urL%p?8*IvfyU<2`8?{{giZUxUG4EfJCa(W@6RHE z>ZJyU)D+Y5NT%pJ+@_gBmm%2`Z0sG{8<)pK>0H9O2inZE?KHiL4vpFJ1aShuI5f!3 z+Tz(&^rQK^ErJjq#;T#IwiriK+T)Y!eE%;p(Q{0ghpDyc1jP+`Ia3T8d9z1ChIMnScF@3H7zC-(ybN`p_$|=;+sV^c{nJk0#JnO8CM?A=c5kJU*rT41WH_hL4 z4dBymTxQkd$xbzzURVEemqxbx)`_hTKPZx=4vG`k<|U-Wb1VF7+*2GdAq7iYAs>Km z9mi!`@n(loGr$>N+;R*(W9fJygS71$F@cZo@dcJL--@liB+~A{`*Sx{8WoJvQ{qYvj)0BKwiqnd5v;3lN}0Ys~)Bon?SPFl+td~Lb1kzR(zS| zy~?1o1Ps#>g*GmZ2&teWQiGjonKb=cYH#Fz#NnR>+>iXM8QV5AFY zS<&5)5|aiQB7Q`3_`H(4jj1^5lQU|bg*MA$ogH)A~B$DQ(?ma1v? zLxkgGr~B#mxH^pvRFM33lg!=z*_}Wl2Rki9oGpN1R7Bf8X=SlJvWSp63^@B~N}p2e zhX}4=Tnv{TEoPY)nsHQrZsX{&R25mf0_LR&Lr+1C;~;l6TFVZka6ztdNJD>@)`pL} z0S3A)E9;UnF66>i3@0@TTavBtLMnaZ+?makmoDVfPXA^S+M!GRQD}-z z`9YEdp>o76R~~40twe2r46T?&{*yw}M%h#b-qRiaSN6x-GzLl>NZaDBSKxQcCB$uo zxen-j!bRxs@b|t@>~h1;d3X}&++;shC^3^CQj-=weYWj}ahD}H94ivOU?=)0ZV8Q# zdOe$lM`Yjm1upm(Kg2=Gx1vwyd9T9IV zG#EdlGbKbFgkXgiLmGdQZ5rNuAdy$g^_a7%hmjNw7kI~S72ncs)Uv{Bkz6@ zs^m#KI+tg?1$#WiQZ+;;LQ>f`$&E^{Q)uKRB$6l=6TKwk$q^@f2i)msi2TjQfJrm$u`9Bl3u>{b7d z^Ap1fMMy)r1=*&3R){flyCiIiUJ0WDF>wZ@c7!KkhZJPjw%5ySM3J>lCq|FPW>2X} zNE7x=kL~^C0eh#=gE3s3VEcaS^xmWXWT>l*Rs{D-L4mON-F(c7Mt^z&~wvAc-jn)=gb{9!|@OeiWXJS^_X zt+ww?G#q+nq~zA-{5>qKoLtX|>En~xw7pmqj z8Cpe8_#%VqJc?~2>Mva!q{K|;zB#lNXOmk=H6=T#U|_D4qD&yAszo+R0lW9dN`yD6 z0y9yn?j}UW58iD{do}}kr8h5`Df4fH1apB&EbksT7LvJ&nkuIHABfmTeC{P8>YRsr z>ox?d^9ftn3UTw8a|X_`q)x=U>+(umE?rT-d2GzTvaGW1`vv?nT&Q zl?{&t45AKb7Jfd_SQTT)y>FyrjoG#WwKgt8_d)~5h zYUbW9D;DfKc6b{j;X8}Z{p!RBlP_yMljRnbXm~-sa*)yA`zk1+IT}(6*-i{CoT1^PZ9oIgbd3q@8!+kU~PwNe5H>t}4e5 zV=>M1bNu)QN|)+g?{g#i)g4ZC|E`nsy>Gwr`o3y%-n1(D<0va^uR2Tva!Q(73SrIW z<=0GI>0)!Qax5Gx-r9}0PwJ7x7jOpVz1ik#ZQ^yJI@Hg~C5G*1LZ&#h)4g*NAiRQ` zC$r48Mdt7T(uM4+g3*kDF(yUJ=+Vf3|rv(4<5uIk!n#-)i zvc}~-`C87Dw1HmRMeLfFL028RK}0qAp&oR*)HLYvmJr3o{zZRn>Kk->8xigZk@_R& z=#R6dmFRGj69cf1s%$y?bo=_PwX)OPqjS%(eOPYV^NOGb-$D=d=E>7??qu%g)NK6t zU3obxX4yDN)EJjOZy(?#f2-eg?#V)5Z#%8-HCX%df!5QKi0(Y0(ZXZEVCq%iOY3rY zs`3?|+iMbcIGKj`^4;(m<-^s?c{}kTohNiu<;@A;Qq%b!dhJ?5|CdF`ZX1?|ib?xA zkKuO1cs&Sk;N0TLU}9jcvB_2{`x3I!E{0-#p3;OU@uFktDKl4ogAva;)1R8O_q5w# z6~D5_(Y_LETKGyL0gAGv);PI8)8iTjys44pFSZv{(qEhiWY2n@cw4^MNYy&_$V(57py7kg`8vt757X4%K*~gh=5&NquaIbHA}b5JOcwKAPKYxMU}Z){6zs-dh=ovkx5oP9cM? zq*y6Z)xkD}`evhR_PKZC$9JZc=&-z8jSW#Ta5}tG63C*CVm+vmD;w(v7p>~y3a~rG z3xl~Yfj-#t#h4t6SBrjXlH)_dzDi}cdHCUx-#U%GPHC`03s7dy68Vc!0%x*T)`!LIDrVV)9sDZp2IV5UVSm2G)-6m3MJ9h2HVE{lBnqHh* z3xws+=)-Z|L1Cz>;Iym3Z{f)!(<(>2NP&fqu^T{)XPQtmu5gPMo>>uZW&GF@7ri>9 z2|6^h_yLKGT#qhR$BFsAsb0JsuNj(81?Dp65=`;J`{0(99cY5TNXVYfTAh`H`T=%S zOiK(J)5y&94VdutF1+6+w_0W8Bq)43p8*pt9hZN`eaZ)kXY*HW7`^MdW;-T!I~0?} zQ^`Ct5?{lfPT*{)l74!q`9xB7jv)c%VrZ31&}y@r6GOi*&^)G+L7FrdP zFSXm>uITd!^~(~XejdhSR*l+|S>J|fzYZ%VD}_Oqrk7+_JIk4w{WamHSYMCL0k}n6 z{l%dikh4Is&t+V@xYsG=z-n=~aKGQAFp5mRj4pc9``jc#CotbQY>DzUk$wqRCt@Nv-87rvZ&H3_s#VkK#M zx4Y7=Q@wgOAmdK+NPtZPdlEvJGpFIJkLsv4oud+ux}QM*<{n;PSXw!rG@Ch+mMyz! zohQDgT7`rt2j49=gg}01-41c>~f~$>ka5G-#R*~TG_ZjgU zwrUMnRdraTv|wuCx@y59n1LnBh>YrwssBW5H-fo_Yz2-?T|ba!`yc-*w5Yz=A=~HG zvOOMFH`>?=Sj1?}XJFp@c%7mdnGt7Kw<|p#!)eS|?!fu-uM zaN|4v6R@!s=3D{_RHK!!bz zkU!RUkRR&Wr%4OvT}60GqB?3}@K$`dpNBvZ23p4Br@} zFV(;7ZGQh5FqDj7C*!^CUc-`Yfb9vXt*D*Of(8!E<7Jpesy6s$?X#$2kHs`w2HsPV zqw2QHA^_y{>M!L*%3==gA-E(3_GePHBF2!@I_`!bO}}t8BhTU8&g+MK#lXm2{IrVx z>th+an-(&+BG-L(;I+e0Z!iCV(gNz_{a)R*!Ku)#CYfec;g`c>D{@?8;8AR7yFMWD zD9ob9ORN$C*T5w(LKl$}4#NsTvNw{c4pX?;(v#3mj)RRk`q!Oi4LD z8`owoNM1QMRMUR!h?`+Y4LgI;DZj=+SIV={@%beMh{RRsVH-vFpGWl$0ySn7)#%0m z68Sx_r$*_`6bh&drg)ifLA@#9k^MeLU&)B1Ss^PbO#YYVx6N zvauw|(P_n3bq*^$-l&^SM+k@_%;=KR%UZ(isG4n(d=$#E3Xyby4?}|qFKWkK%V|F} zTNham8o0C-56y`@%p@+OxoJ_pF-6P;6%%S9DA;)~pL%zwuO{ItB$%CKYS?|wh87w@ zP~@Sm{DPe)gLw&~3cQeHK*14VdE+RDnAm^+w6oa!8wBuVt$V<}?uBrn=ffKUl1E|8 zv(e#<7s4wU8!1yNr-=Yr@i~Akug!>T*1XZaKXiOKU%b$vcFrU__hUlDqlx+Op}!X5 z4HO#^5QfqTd1oE5#ma$axhy?~kaF85{W1Cyw+KMbGXvZ%26?=+jwnRIQ;!=lw zcB~hg<-|gN0-Rh}-Lm1qDLFIn@<@O8>VmA!wE=spF=f`!!s(m!M=xI&K1UUcc;>3l zO&48QkZbcmXV@(3g8qb=5uJ-v#C&YX7(vEuczp;XG0Y}S{OkpM!x@ruYgpk8gpMpu zj@|W+DFsf|HCC_X@YPB49*4Kk2r9Ax2>?%3EvYD8F1 z45ky>>)Z~|Q2!l0A6l$E4o+9B@_jw2@k*28A(VoQ(XL%vBbzEq)|`io6eBzN5cA@{ zYJah*;wo7x5Q@x~iicY=_qsq18yHADh;Po)cQ?|5J|we1IZA+o)WD;Z$(9E_1qKM9 zw!h)j)CDftcS0m_5s3rCr20E`EUFCLlp`^KuZ#@AS|unELtkf>As(MB-jx0O5bSUC z30#+1ARlPwE$#y|aiHYz6Rfx*6Tafhh&L}l7VpUUhWYnYm_T~OH5g*R=LM-EMM$~9 zvrKAYmDu>pS`ND9CQPuLboXep_`TQGkSiQkx_l7K(+VUKnVS_PIjqRU3_g(&SC7_L zLnE?G{~#tBQxJg^A(orFR6a3g7DF)P*-EW@z%1XZ{l40oLANc|52gG@Qp`?^KNJNg zqKk%#lWz?|YHE!T9FcxuDH(;CI1+!5k%%q|l;$ns`w;XANZsQ;6`+CtG;6;CW3{Xe z9rd`_2&;A!wthl}q(iS9SE9IXP-uI^OB)gqXzDH`qD_9XB=e1&MTp~ItdeaA!zucM zdyAVYV|vsSP@FDnC)Z&_DUZ^E0-ZGqPO_;23h$PEROVZGN<!~!0 zy+KGs+9GcfP)^0?){%HMY@5koRf?AcvMj_2b>87Tg^)w@FUdlav z>Zv_dr#@kb1;sI2yA>E4e$z0bBpk?bzp)h``$j=_#+XaOR|WSW5$nq#v6Mnj&s*B3 z%Lp>AK!pUd$J3O-gl8IUx4(f9@uro(VGy;54I|G<`1Ym5RT} zw(Vh}BwpJ;kr#TF++u4p3@1$pO{yg7W8XxtlD=TOfzmq2*x%LCHGi?d-ofSh%s!?2 z2%um)P4O|b1>~6trLNv#=Gge7*Q}cD#IeGtw0wGY^tk+q9C0Xk<)~&q?6cX-eqDbH z6N1TKTaD_dG5M}K9Zu4g64jolLc~ttw`HZ5ZWkfKz;H~fE(2%1yIpRPKB@Zm4b(=@ zWl~X*KGLdXn{9kE;#-!rMMKkb=#RiN<(+=;Xiv8;laQPy!!tj#@-1y@;ujU&^JzK8 zm=9uoPa5iA&4lWURD5?G^fa;ORmnn{lZA|t%e%Xp7z6{3r{-WFN#vfOjF8#RIK)LA zGHwQi#~F?hNOOMT{)Q{iJdyqr*unzn6YXUg5U7J}^m+eU_!9$wsFWeiLaUMwMxlR` zwdqa`@@!I>-pi$^IYF@MM$UCZH`RxWXY`3D<=u3guTUIjuIdVt?4C9UM!wKlD8afF z!w)>nePtOJJOfnc{*P)u7UCCjK$?0PgT-Ds;^#c3l)br6GGMFy)H zl^%PoJ{i_r^Y}%@EZLx=p-lqH$50{`+>O2%1|p1$^dtCKRZHTfAuM%S)N9t);txg1 zi>5X=6^rX+a+{EDHb5iTIS>Xx5=_@l0;=K1f&F1)RgTg<$x58Z0v4^G!I1#&{Z@hunc*G*$746zb&1&f#fwZ zWRO>>FKrk+;GR8;w?+qg0oYD*yqoWQBgNzL8!!!bt3(-EQL7~-D@2Xs9(ZP{wazSX z-@B%}80*pO7Lm}sOnMKCf3#V$cD3CanqsQR2stJ!PuU0h@f9H7M7X8!p8w#?_d$ZPU_u~SH&T;Oz*_=BC2YkW!a8Aeq{|D2K;=kC?v{bo`;r+T zY~sAM(rS0!!TCbW@<3R30`g1=x$DO2Ps|Gaq1G1u-|M98L?Q7s#;=uehngGJgnEXC zpqB6iX0H|P(Al5$G~MUY@{+lJ=vNACOM`cK*{jb+41u{bRgc!>Y{dUSo3N&tHZwQ(=aWI=uBf@A8v&z~KX573 z0TR3i^S{0<0BRCO^al?LHEA+L^i6~R|It?Y7Zp+8pgtR$gKsL7`ed=bhayRLiuzx7 zwNW#|s1hZ^)TRn8M$60!`fc^}W~?~V;QdG6cDe+A4?3wY$NSi~+}vC=4yWUJ0wpD- z2Ane`ur$UOUa}n9x8_^;Z6}1jQTuT$V2S4_HYYmL>)C%uC{|gtIl1}vvmnXIXcx-a6hH>B5 zZ+t*R6RSel=Ea!w`PKdcyZ*f7<Z&de0hDC@HU*x5kvcS1VLj;V|xM8vCVF6)#ZER zR#R8+K5)R5DX%r3raWG4Thg@1nBge4w|?pG%bt4Zf+hMqk-_)L{kH4^J|N6TF}cPo z!zbzI#|;qCnpJB~+??~vn);2mPE3NQ0O5UQ0RRbUv_~d?DTebiYGS2!ZekC|?kgJY z{GppC))ofwi@;P>twR|6W?3RbMv&MQ5g@Us+=B5wdQPKUd&d?Lu5mZj^8A+wO14N4 zazEX*8rQc?s7QP+U!s?NiGI3rHiv!1>yxE9O>O`vtBRrd@vu}a{dsBa)udH;1N@?v z-#YD(=sGRr-z%Mwga6+F?Am~Fl746gUo&2%Ahh(HQ#(vh6h4PO2`T&pmVt-QoW46! z!?GR1V!fqlt8})<+tb=WHE`hT;S9|d20Sce_SJDUP2Bpjnr{ifX)9vx=hj_EdH`L8 zStV%2Ye_-zvHPlM`3|e~VA;o5X7_0hx>@z9$DqT63;ESE$*oN9hP~POs00L!cN&|` zUJIa{chk~mb}m>Pu%Dn;8ku4v=M$Y$F}lT+a-&#@6_>K?Xm7xEH(PfZgNxex;8E!g z*uuyfxNF`RYoMzyjg_aRMo0gsm$M`Cu8^p3*AvN_ zlb;BgkT+O@&=txL7NtLyLgpO&wiG-1Y>P*ViHTVd`8>gqgX`0V_)RA=BD|lsi276e zOG-;GI^LGuO#}OH)_Yr zO)BMa>>shPNeEcCKcQ*lMb#5Q==9wN09YvQvR zhjLT;t(Sflb2pic;i~<-Y~B{)mym2Xl2njP8F@11aW&A=O5wHv=_^`hWVoMlC3tHI zcs@GtUX8Mqlx@$$UR8R($Php-sNa%2#5+LISxVI&8%&WNBswG;6UgI0%M>;q*NlAa zK?2+#aRvnW{Bn(dn103n^+x65v?5NvI*)~b$=C&DBTE4j_4`xBe^=GM1g{1q;{i`KO5ba=^=lYce-5@-V&W%xB1g z>z6aI!>{*jC5zwo-gE0>!8CJ2AZXdR%)0Dx?`ZQ`j(IC~ix5so3!awupL2@T=<$qYY=T=KHFqs4 zF26~dP+N=W(d)o2T2(MeC*X0)0?+w;*ys&TOO8+3E_BDAb>YWYYu>=TXsitBrw} zl1ESRB9p|}7m7irnngG9d&!q_(+K&sO8Bk{y5F7lb-caUV*DE9TzdpHe7!&a0xTz# z)KF6^_8e?CYTE`Ikw}-#5UTS|NR*;2_X)l`Y#sLK>q>(TLV>r32{l*A7q^$2xrkk^(-0o+j@C}+N;6Ji^k34tcD}Jmdz%x98Ig< zjo?=O_^R-zx5MImdo%E|xQsh|{6eGjZdyaaSFuSrsb9*M*!(Zs_6L*MeHk)Zq>@pU z=o|F!gHqb^$Li=FgFns9%`FB+GDZ??%Q5GrMT{vbhuU}8w%cmXARu9iUSMM9Ha9od zomysVpk$^>iIM!+3HjWv6(PbFMu?x-D}U_p#S%8|D<4F%F zMI|Dj?4=Ac(W0cHDqYvTohw$MQd{-mi<#n~>w~RI8HSkevcrOh$$28fijhq?W;!E}u`-NGUGbytu?~YGQ?@YN3<*jy2X#v)huE`o zb9j}CGT86aC3afyU0N=Qk^%1Ax9!bH&?A`Ls~hmM_s)+tKMr^9TejY2&ZBrK!<(4) z;q+Vto|cKQgLBUaR1#M4ogQf=vu;iM)LoWz)}*6E~Hc(z&IfKq(7mi!D^MyCruceF!$#wtr?Oa`_npRUaLUe%U<2}(D& zu1?@4OmqnF2G|bhn$u5eh~W|*q@=JKhvF7Q3zBhjYwMjzGj+Y(tyR8*a7D!bq^LIp zqn{#8MxoI;^;h!tnIJC0K~r@sjI8{z36LT0n^`2g=_oI+pTn%0MzQ!k*k|b_dOe4o9U(?a<`^UW(JGm1w$gB1$OUq!Q!R)stk7%W zq(wPuH+IK)9U2)?Su!y-E!{GPAFYiKXf=s_fswJ4Ie=Gl+DI&o7UVqFgatt_aGvAS zLUfy-=$aTKt4Vt@L+>(CQi@w!!mCZZ8>)`}-Y4ygcTkM(W6a@zBm)A$P0NlrF{?k?- ziSm|TgS^>M0?UiQY|qe*UY3msdOP>U)VQaMHg9 zT>rxmv%z8xWlSx-t0sx!D}{?;xwuWzhTA0xByFSlZSpQg3|^IQ@G&qkAMs`qe^i!U zBo4EyE(|gkAYy=DD-CO^rl8)RHbPb~aU&J%h=0BF z-rYRsnLS)mBg%M`WKj_>|nNKPM)3KP6+Q{v-l6?%8x&frVB{0nYDu zGFK-rU8f@iM1znuKEFYZT8(*fM^?p8iv+B>3(N{m4ZaJEr zj@}VBASyQKs>sWai7!=rSUcW{cKyIw7oC;&@FhhXBWpZK#M@BZqhe(- zhpTyc;;^b)x>1Ev6|dAIMfp_L@OeV?9S6Wx)D2_1)_1HNZsuQxgq->PolAW$B5U{1 zS`tayC)RH((urf)8gw}C;_WA6t`Ba}h1s<1EeIM)#w%XAaEMB6!1T+kYl87aDA*F~pOhN27NVIUl zUT-Z1z{i=P!!{rEX8j;&d()!3q34_yub7Zwks!6Rp_a3@%D~iZ9kUuQ=~XCjst_G+ zm+?)Vy<*b0;Y#SzbEQ7LK5m2t#wY2AA<+Y5 zjeQYd_HaZyA-dpT)s*-N?b?qeQlS_Rd&%YlV;3p;an|aqF~FKCl?|MH9zYUTyJ;%% zf&B6W%lt2+wquF*4KgYxI&*gNEW1zmhj|P~`FFw<(G;J-hC=YR&328yUO_5O_8n&s zn(qOqJPo%X<|U;ocoh0uldi?P(ub7G6HYJ5GlE#u@*x?^ zdewrdCyKal#?-%bsU9$=o=66+Pa4*4lhR~OkTKq0pRVX6lq22^aZUy6P;I1YOKTL> zAT;O(#2c(X*D%TW!l&?%?~pl4M9!|gz22F0_+W=aF@bE8we-ASZVr6~cS%xqNaCLT zpnK4tC2{lUNAHG3y=vtf3c`J3fDD=6DtuhXzYu?OgTf$Qr_m$mRqmoK!%s;|eDm@je4cRD_psm4{uIlYqV_y@{MYfQLzM!x0IQY}*nZ`acb zrCelF^-LlZpe8ioTQ;kC=GovVTn_26sOW6+1&;drS3*G9r*}4;AV{-z1DJs#=wZL0|8%0-C ze1pf8eER{lF|OpWfy_#LKr3ker zx>QF$P>UJ_LK$0kPe)lst?f?`&AfV_&pp~l=+dC4bsBwCKIPq2i;;}0ScgD$scUDo#gUMqO+rh$QRsU; z7<1H-l2a#c&^-K{>JxNh&%+XTgEXc-(j8*f6cYeE!dY!Ll%Z)aG^>7w2}$nr6krh-HzG;Qdl`xV>cX ziC8=r^m-5&wQ&K8ffEs8-CVmlDE@?lV_jf;s2^QvB(GhvtqFCIFSfgljss1}aDgd7|6oQSkO2 zZ_Lr~pHKS!K!lj!%Yp$$%|C$T|8GxXu%p^F>4MQU@WUV&*jLNV%vS@Q!IA%p3H{TG z*V9Sk^W{hc#owo=Q`k(;!9=gvGDyGSW|P04FefYpO8Cd$!1I4r#9H)itH1npGmNPj kcjWKVR}yg5?8+O84`YOGDY53=U*JC}F?rDn;m^MR3qr^fJpcdz diff --git "a/profiler/merge_profiling_timeline/perfetto\344\275\277\347\224\250\346\214\207\345\257\274\346\210\252\345\233\2762.png" "b/profiler/merge_profiling_timeline/perfetto\344\275\277\347\224\250\346\214\207\345\257\274\346\210\252\345\233\2762.png" deleted file mode 100644 index 48793f136e48f21f618ff3cb13bdcc3388f76930..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 64432 zcmd?RRa9GD6z|)X;$9pI6!?l13GP;)Knul-6!%gbic5++l%mC@NQ(s5K!9Myi(3gU zMH4g-Ieg!_XPj}zeYy|#<=lsioxS$jd+m{|J@@?o)|@j=M@x;Ah>_^Qg9oIq)m8K! zJa`m&|IsJFyKhNN#9zGMJoMI6Q+iN4@$}$+=dt6rx*J zRNor^m>h0PlZa`&2=&wDx~NTcID!3a`HPp?s=q&(5{jTjN%Zhs9H3f*1^p4?DIxRT z=Fa8;u3yW+>TFn0;TyyGmCF`G%wfm;n`nO1mLu5n*S9PLY>^a7>eCG3E{m2i{N^1O z+(Gcq{wWkn_Xqx=eEoF|?Y=4YZy$o!dejt8{%dZpW83lf3s zg~I`7P#0d+9HYFQX798AuI#lg#F~O`tNZ*V{N=wVf7#whLGQCD{rCMJwTEo~_4-=+ z`OCjU{_{&$QT^Xr`nt{i?4SQe_4;3Lbh#MA(w;cGPM{|rmF{q{_p|19SMl99I&_EmMnPrPl z*5)0VJ#>j7<>JGJX@ljTJtx&q6cK#Qv1_hYAmEE|$1=%U#z}1IZODAn7gD}<{T}NV zEo*qOLDdL zrps{J&NrQM__SpQWCV65TMxNg^qFRHm3|n|8=Ltfup2ynD?h|lb6SFCcaGucPT&za z88ZJUAdAQeTLK})gWvBQtl~47WscBSw;BE=R{(@f&g$C_%YryL&eSes zs-Sb%bU72WkPksm|J2KUb~HPEP^i?#&SI(1Oy4#A_j4k7==0EK3z#aH@%J|;eK$ucKKjQ z!%kx0emvq~kXSdOq|1qS1na~5g4eRo^pDLQ^xe%wu)yv&x-N4o!NF253^6)Rx`@MV1Zjq zG8Mo*g@we6&O&Vt0@3zSAAY+#Pe5=Nd}fAXl5aor^T+FGjp0A^!jKBcqn^^;^?-pq zWG521*w8x?2i!o%lR<>#F~K+3XG;a)ddpKQ_et2s-vbE9;uHpVVn47C48lZf=esP0 zY1L&ZIF6!S*LO-7gm9ew;=y1r4&-h}Pd0~bcLT7fi=~w=DQiCUZWF*TcZJw2Yo|Bn zoiL>iS!4?AS?;)A4VwB2y2juFSm-RpQ?Hsm%V8%+TAIzrHn&T68xT~HHESg?32E*~e$5P*$u z4K)grQV69Ol)@FnMAkGGJ$eVha_^FE@^C;BI)%GaUSl9*r)+aT_ZbhdlRQBoNx)W6 z+d0D2_-d>0*;dSv`$x*qmbM2S_+Cqx7jS&c!=Y)9>S7{=pbM@k^gnduc+)}Ez`!7E z^3_6%EraD%*dtlV82+GB7rrWcqczHd4n6Ly!I0g-HWhpkCqA6M4>Jb?%pAcin3W*tNU3g1X=eFX|C(P<3; zi;3o5ugsZoM79x9TKZ9!VxP~jE$C3&-{QSZxV6SvgB*j^+YZ^xo@tCnDY>Z z*ZnrE;^0`$xKWahfe%EE5S8n>YW+x_EEVu&Ls<>_LLBs9hZZ;mH_FbOa5ylH3gP=x zDB6J3H@UM5_;pu9s@8&gD1Pc!BHz9S*{HFsLl2*`^iSquetthCO8%ij70YRnIwx?-T7whis#<-DcRZz8n=`F&BdznaLr#{nHD-Z z@)a!YOg)OVW%Vzq|J~Q@?w4+BpT89LeAIw90z*idDY*)+vR()|?g-?a6-q7S|`^;755nL}Ud+Y1$6^80z8P*s`L(u^CrS}H?3#0VnpRe2LG&i7JHK5|ZOM0_U zHJ-J_hT-@7B~w#%$zQ0-nw|eRF10Fx#xQjOd*x}H&?vu+oiQJ<;Q#{q_ix~C{hqd9 zhJYGUgSD3fuysvmh#jr3RK4gXEkCa7IqZjIah%fe<)bk56kW$4t8tM@HP&6Zb4smG zaSIU$iK9DG;`ajazbcLE&gXsk`2R#HG{unb1SKE$HM}xC`}>XCDSi*~Cwk0D()D9N0fGM>z3oB%a9P}0kK~*vhC|7XnU0cd_tes`!tPh*%HX?9(8=-exp1` zd$hhIg>d6D>wE}7&T*n{Eem?6r3ma?5tCfuL!_N;Prby%O!$5 zX4kwjcm?Qn$8s{76ULw)E>Y8pJ$e4r*X)2nLdY^tE}R*FoVg2;ecwj?T)UoJ(R2F9 zHH7d^>x`Vx*QWWX4+7pM4|N*~NdYM7o}@fd4y;}9vu0HgIxooW*n|JXDsLxu3PtQ3 zcYhKZk4T`iR14b1cUCf%)+*?^!}v7A2K;8>ZhBa39~_T&Ptkq$TCmE4GE;#?(+_0N z`_TfMV8*q;ky5z4_W2j9dFAOKxZ-9eQ^X_7|1b>(iBzZy)B+#Zk^9OJO9RhS5%LDv z7sFKCNQNp2Y#J*aO!fI)U6Rb$X!y=cxi2%9j2|tMsbbG8rhq`^5l0Y_zJHFPY?vDQ zciy3EHd+4QSrE(6ZgaxfK0RDsWQVn|H`-EuLcafQq3l+G_6XVf8`F)L1#JDb>8&A% z)_1IHaOA3S_R7DYy?kxRsF_Z%Hwd~?jvv^XmEExp)VUN zutv0{jL)0apZrVx4N?Hm#_AXE=TCkfmy)L&gfTk?6*nl!qWWV}k%0?$JdR3kk6@!e z1~Hav(c{D;XIBC6^1{26Mbu3oY7b#l$A2BPInK1VOG?lM90DS>HKTzluYc+BS=KXR zJFX~i)cBW=_tuY*&ckwV{7UVvwZIoBPE>6&+H@cE;418z%P&1`b8h{bKF2ELG{pnl zN@(&^e)Zn!j0N6UQ@h^cii51VOhbjSNVE-d^7k52!oS;NtAE8ESN_P+bukX+Y8>AT zLD7)OU)M>dVo8Tw$Rriew3hu&Kuf~ z7#I->WwPVGG|fJSbOcU4)U;v!q;Qj&O(EK7v{j}c3dOJj`+IS*%<~d!bl%-5JDt9p z63&bGD|iJbR{2kzLs<*kKx!-|&(0X`>Ylj2ec_MiY*Z&~(MZM=x(nKa6rmHD)>KxK zPftGp4}CtxUZ0pkm&X8wy@)wy_;mMNHk>3{kgQi!j)d~9HaUz34m46Ib4@USU5c+DOkna1(cTef$Z^?gme(y^>@Y|B< zcJ|h6KN|YY{wRdh*;0Ry=V8VZSu9Nqe?|C9Pxlhn-N~Dj>ovpf`p+BM+7N>-#^=uB z>lt#@6DoFVo|_!OUI#DEH@c=TLF=m?%MquVwm#RxA-&cnZ}A*t`@;8QPF2%ySoWoo zS}|YNg&2NoNZJp8&T_q4jv#f2yE2HDac4)*gzjcDD|t_Ay4#S@@6uqSjgQrW)2&Y^ zHm?;^X(H$SB8-Y!j{Qs8p1ivR zZE?(PEn?MtOVBqdb*?!Nn-8hAZi13L$lT}LdnYwarW#KIjtExiNrM9m@|$cght1*$ z75Ye^*Ts}hvzn%nqu*g_oZB_@sUdR9ZJ26+GlthUHjZ zF8_`SjuqIQ@2;r)fjp2Iv6z7M9BS){F;hc($?A5{$Y2)k_Cu^1pJdmI!#jr{_&(vT zYTiq6zPa?vwgq@VH8>QS(5*}NJBD)z*RN(lepmQ}7uo9L$Vrfu0uP;MggJ{ ztF#q9+Q_{3K(a-D&%e9bWxhA{+t&=CE9(0-99_TxR*Thmg;?Gp8SjNn32m4*%>0dd zBsIEm5n{{E$O2>pwRFe7BW~GZ6_+SRg<;t;+u-+3iq5Fki8Cu93I5eGEc!s=-3?NK zW}_EBUT5ioF5pAy!XX(6Go05|YCpgfDMLxX{TUmwB(qO-Qgne~=DK&9s{`&`7t2=L zAq~*R-0qpEN4Et)7GwP$volo;ZOS>dH9XleHa;U@$-gpJXYo7xn4h1meP)%sH;xD$ z4tZHPs!e@s`?FU)|3WZRp%rI*PTT5D6H6Jz+T`jD04}{_3RC^~A>~y(BMc6_sq@0* z#*`gMZ|l1HKZ4wRgTxTu^dlHQVw#_f($e*vsXV`G#+nPnM6OKO&-6Y=0NO8eV9z2mx?)6e^t)TJ~Q2UvF#@xj*xRe-BMX=wF! z&bin0Y2c!`Nr8X<D0M;GrYr-7DYuBEBYBO@Aw&;~&yXN8>HWTE7Y;)Ml zd^|}SA(m~6fNZbHVNKX&n~ST80VvJsL-}z5`hJ9;Yeg3?&c71J$cV-CmtMfSX~|tiMJ<@F(a=5Oa?bVUax&uD>Q~2_>dvND_7R`4 zdfoBa>0b?NSN*}2GC-49z3k-;$Mj07H2cnSQz0b`TK#hlS5J46w6wkRZhjO2-%jToJ%N@r~^Px|BmR|vaBH+$7R zEfaF2&Gs^r++7SCgYwXLQ!<2WsYSEuyd9xaZ+_gqF|we2u_|aBA;k#|ALw20{VqwO z7kY-{`_$qfnxOsivcQH8@q7=Cbqb_}t#*)1wvuAnx(lLJ6tlgUIybMc6$7-dnL-bJ zur!4?g&o^nI2^%g!Yjq$jY~hhiG_zEC`uo1sq7l6Lsz%(Wj1*?+egp%rC{bW)EJvksLGdiGi+ z71pVJ@Tf5S7C9QHtsLzYh|E$kI%Au8V}QYkVpjZ`dLKOVrhOC2Ft*@+LoplgSQjw! zp0DuAwSMX{t%QLNtX+Gqy-)DEbtr)G7!#v37)@5{>L4QnLxfM?6od&He>WGjp56z*evqr``&qPggB0Sa;V)J)KWK&-C z7$^G@r;PcR<iEtpp@dRlCLKUN9d6Cs#hn%4Yy1_%!pz6!$1I- zmwW-eAvPbHjHY1;1s~lRF82uQ(Rv{!Z#W+WP$I@HxPH(o5jpU-bcl?p8 zZ?DfT$RiH_LCNdi82UzsSi+47hU*Rd5|11A7nb8fI|p_?|3nuHZ4iQ~e~TR7q;R56 zSkXRLTwtF>^To1%79GB)4%_V!?EErj?NnckGD4oFo=ZEg4hRSvfBy9slbzdKBLfm( zH8Lf87+1g~*^X}j#*D>j`xh+?TrmWuln7SZBHqLVKzg6dgb=wiOAREKTJ^jpAiRZf z6{xu(ft%m5S~pVbH^=MAOK*3)*4vTXBKW6L-KqB`HXGqO=8q^mtHjQCbTs^)*;Upy6M|_vseaO^hV{IO0Ti9lF(6N3U%@F#OKwRGSXCIZfm(X5e%hueN z)1T*OR6n1R@bxME7TJ+%VOAK;C54t&=`8UDI&;bB%14?nn9+O6zZ za+-eT3Wax|>^=lRUpD34F>1{o;RLDnPCk;;V1|HHEPPcu#K>)t;Lg;a9b^HY$~Q21 zCcTP9KMC+KuX?TH-BunkX2PS8hnBnZb=NiG4VHbR~an&AwV@5;$ajlPQnR~Se3oNV(83$ z3tuGMl5Q40FVqzD#9gG6G^KQ#(lhK}yOoGqWp?Qg_oiinSMy$U6%Rh7H;+kjO<($b zKwNiLlhWwEx7@iOrv1Gt7<1X2f}ej zENcg$7^^A3F&8(`l>=ivL}fhSuHVsnKMRJxf1m8>u&fyXV+udV6Q)=}&nGv36OQb7 zj%3HwpG3|5&SLG=y2-WZiC%)GDa`8mXs^Tm1+tMo*qadYlv*V2ygezx`3dSebKr?@ zxcx(yc|DZxF_aB<@H~-(Oe|1Zdy9mX2)q@V?qZRohE}#KE|(31xn5#LX^4$k0%!qf zjr3!$V41n6Hk{=OdR)1l3YI+)y<`^ArMp0bofPWys7MFyqrsLKtYrC z;}d5EABffQ&eh9i<`1EfC2!^ha9~07N-{XlSHw&IS^k1jJmz@F2*pNx?ODirT&9^F zMT%`{0tRVW2T%b#H?;THwf7OkiAyQA^F@yg=PiJu9B0kaRzthhV`s7!2SNjdV;%DSP`Jf~JE;wslef?8~ zxXZguY}UicZWKFbwKn`6B%LPCk0|u|IyR9jp2DUuzYEXeRua}g2I|O6 zT743QK6A_+9e&@{GqkgO#saHSN9f#K%5Q!xTQ~s)dyN31uw2x?s2e zrz+)CyWbpj=+uuAyoWkjOXAvft_bvbMMQvK2!M_EcDT@V}+a z?ZKk|KQD;>f4<5~xyo-ZnE!-I;^94g+10SIQE!AAIMRVYUAZ-D!#aJ-yV4@7Ht}EV zME`lD6qL^r7;*t=$OPR{_&~t|vZUTd^F6^6T)VLN#b1J*j~ptpNv!O3vld@ZTNu&O zrb}K?7Z)pQn(@-*)O~!3^V9}W?ltlWXW6T5Sh|BfYmB=y`Q;y<8NeTuq(prtkVZ?Q zXO>nLAZa1qyIXj45MEREaBk8q@_EA;KxWFgAjQSk?N@iwlt7+>=i&%QRWopaDQ^43!MMxKmAnt}D~axdCG zh)Q(eZmMsBpTD_rBd}YG?4Oy2DxAHiSj*8}n&F&j?9+Xjxz9Pos;Iy3x>x1X6P;AY zH@MY6lgZI^9oDEz>wus;!?Mxjv)Mc+N++^jgh}fT9|=UK<@)0f zvUy8aKkutoxB~YC^3{}+EA0mN zAY>GB&0jlmK)?@uf$sr+#ZDT(KO?u~$Vpl8TNgki96uawk*kd#{6&1#_m4mH4~EK- zzDfD`TZ{O`Zk=eHMRt$iZo8_1TCk4P)oc&@9+xlX&b!A;HDxIPZ#^PygVh-Rch!mT zaGc=yfb{B9o$%UfWbLSpYPPXc;xrkUOw=%XcHHwh%xiLT^S$v%K*(*-bDLHM3*Wb_ z+4ZRlz@|0H3bE`HglZlMZ_9^+BnvfoGQPzEP3rM!I^nGp4W9?M%_)8mlWkThVDaiz zKrqj)O8k)z(L5Evk+*})o!Y)xR;TuNaDdQ1+J}rA>rMw*ZY^&!=i~i8>h@>qC*xnS z@UoW4D~~20)LU&eQoc;j!u<>-4O{OE>LbcnQy6%ZGX84#TV8bf#wI^xX1oqlL}6m9 zqLb#(e-tAhBeUEdi|hS%;@2?a^K#`+&G`dI!5q`u$5p8X*Z+KC_BEWLoM_r`xeBI# zrTL?qmrh&Dw})lPIZo8BhEKV7_EV*Pzv;sxn|d!A4dS2!$*4&>S%$@ekD9c^ft^+| z;}xrt$X0%I|ETUzoo3gc-+^-izJK9$}c%W{{Dp}}fZP&XP6uvz5t zmju->YuzlS=5i+2M zbas4>WX>#@SQshW49Q2xF05qe?KtPNAF~|r-R&SxwHm72bGc5j^pcN<85D(Jdu zdh;$f=XuswgUoqly#wzW#Ffl-#+#s(f$o4czsSAQDF&g_My-RXTN@z|kRjSO;Qi>; zke-*9QQ22K2Hpg7R7iA%6FT(Axkx~?Ug+U~?Cy50-VbC+ww2j#pwJ(F=AUHzj+qlc z8Fs`I4y^9Qatr>*@pkQtgIIO^T&m8AAQ_YH7L6LZS1&u>mmODKH&6=})}v(20U_Ki zTlI4q0}!;%n(7Cp#Qecw-}Th@RikBSG9iAz=yEPZ^XO9V$CBlHo6h>>#CEg5i>z_I zR(0RDSl{5(odY1=sJz9u=mp)pt&L5}ABe1&M?S|=-s&%aT0cY%Y#I$Zc(?as^^*&1 z-|4{xe($QVI7ZbTa=Ay$rE#-JX=@w3u0e=rc&&Y;3_#R5wLrK{Qx9Eft6*vD{D0ij zHOk(W?7S-Pgn3R}Lz7v7=#Z;o1FQnC^0J9!N}2rkn!xWIohA7i{yC_ju(NldZQXb? ztAk2fS7D_a`M&&B8WTJhnZv_7_n79+;d35$%S#|F)o&FOuFvCD(!1{GN_W2X!NW-Q z&l!ie9EhN}GO1;8YN64HIoAwH-|h497Ox-E+c2t?HF4mdNVnXskG^UTtg+z9FcdYG>jnz4ZEY(%@OeK~1s zRGF~QC;8Q!-`Du}w^l}}%@H#(PJG^S-_l0db01*0hT-h-)kcBmW)e^P@Z9J^Pa@9z zgjqG28BJAcYU?AZd~xv@b0LTXV|xGt`!VXLKR;py>0H0|wC1Z}2CbAv(!Ouq;~!#E zfyWzj5<0`qm(y05fIkB-F`C)+oUZ36&n(S30)(ZiM}=IrJiB40X1@$+4n z;?IN=wQ&E0o3;6=G3eoR-BR71bVpO{6z> zAgrufp=VNwC1XA_&2^7!*S2Pn47?K)dE{H%WfKAJV_(HoKP>Y3<>KZz##>P!bq|Sk zBlTUJi}BQLjb(J8#vO#XR0E=>^HvwV0lMEv=s@Eh^0Jv(I5q0N*~Iz#BMzgtD{yX( zU4>L02#a7>@#a%t3))Mb&({dOiVo6#Wgo-T z*o`|+XaHNb#8+`y5g9@Ztq?Gs;qUJ^?e)HnX~L8VDaU}!$eE|QyTz&pUw;27%WD)^ zT*t6EFNFB+7$ba3^wYi>{~6;`v{si9(MtL4tuf61cQ<(C#*1zY!)7ZFZfr9gv#Q9f ze;@hBQ$#7@6AMKhRgnox1Jz+2q&-)=m&qqvhvCf`K*F8>t|zc15|_coH=Fy;uA=aw zULL8kH}g^GeSp5z%MD3PF?6@vgpJ%4cRIb5vH~1jLqeTNr?mA^HRM3aZZ8AZP1Z;o z0Da#@er5D?n{8)jGJ&bl1T*Li)ejRopa*_)cc)O@*O8n|%E>~9*?qa*ithO$#L80b zuQo>i*|&qm9Jsx*b^RbD?%OqKI+j4LMl$}JVC>{aP0?qK=E{ zMA+V|g83KDt4V5`8 zEC}af4cbsn1Ows8C$9$;(NZfNsR8c&^%BomNai;`-0E}Z%|gC_)pElQ@SaP)rZ$zF z`0Py3Vzo*Pl{CQP3owk>r2!w0y*l8vSUsPL&>2{%f1xGt+bkor50f^3I<;O*MZcUi9UJ>E!tci6II)3 z2_s7ox^XW2VwaOmW`BU40?J*jpe^&Avvro=uLc%=#|K0VX6 zJx!H#3H7m79(_dAXu31p|E%rYxx+WYyIhI`98N0Qkhf_p;ouwjij(N^0%JM0mw0pYa+d0d%da<)Hs?=WlY5d-Mglm4W1KE>!z}|b);GROzt-}8xraI!{k$%C zRSIO$$T-jcI9r#N3tnp_E1D=Y91P0AeH$FR`3cAHxGBzO@x7eI)kbz7;xNibgCR%Y z@%j4SgQ&kuM`ELvyS+6Zq>mw#1tFD3!W<$VrFZLgeEvi~<(fZSUKa(LNQ*I6s+|$C zu2v)$HJcZ3Xjq>Hm0Y!Webf}W_1NCgS06`Q2loF0(sQ{0HHVV7C|?eZHd=?Q49+L% z?@?XQ!x|Kf_rUn>`Z&@{b~zXpNnZ)v>pgGi8ZAOFp6|D)mqi<7N|9p_CnW1g(ngL4 zk_oRB2B~TGJB#4Zr^Q{+wp2q%+iDiE@m@0+#RJl+Qpq9ZoQcnk_~G_jomMX|i8reh z-Sm~uze&`g7k4U^cF98okKwJh0OD}QdP@s#vmANi+T-T1rs38Pvms=q0>49hAFF)N zjUv%W`4p8x@$>_URH(~`#e*n$VN!XfWPi?t1V-^C-^T^Nu4BdX2{3cKd>~o+yr&`j zr*Mb$)atgb5{5xdQHV{H%y$ zy(8qwmVpP=?)ZK7F>5t>YA-sYnd8`Q$2a2MuJe?Pm5yYSJazxr-TLe#+oPR{@T-EBI6T( zq9{b11Wwdpd9m|~Qh+*t&`X-?kiU^JC7ZvDb&?*PT2RBeB=@MTL9Gwwh|^Q9nvRM`r!-N9|l;JMJ0}!2lW_WYGQcL5dOYHJt@zOP3?b*{%Cru0d=? z^nxuL%-urhv)e@K?%v#|AwsP>6TUT5#4x4LYfz*32#C3d<*ITEp4rVF#Bygr{G5}@hthK+S?y}H0xSwH zf|h6<)OPo09*|;D-LX2R>RP;O@=D}iJy6l|)LM_2mjvUb+!hKa>nHy`c+RaW!4oi6 zr8q;t%(A@_+uvI67F<-Y>)K$A>M(BoU{*Z*sWI;?QF=J=!Y?ykuTUyyrQ

L(jehNidt}CYuhO zpw|6@5Z)@$q!|b)8@7)EW2(Q*ep4N_6e)erTJY{|JfgoT(^DXB^|8MCYv!q)k5i6} z3kGokM|?|f93DgiD>&Tg4jHG!0{G`0@q>!yL^O2#+gVZEQmilTGY~-e zj3&ilzlQmU@Ialh!d#JN692axgU1~YqZ94}@aI+k-(@GneARHs#zW1cOewHC5%drb zFR@jAPIwiyoqGE!?s=$lzq=G{d(}|1n*li!6M6qKrB`}bOm5CEW-L~SHOplA4=V|b z9Aw~CwxMegC$1F$jQ*z6?4#Kl;1z-xe%wT=$mYy_lag%W6rR5-uSg zg)Kz~QAym(RNs^^osc>1ku6tm0mCkY$b4f^C}eeSY@3>!H|B^q@&I*I0o$i)$-+tt_DDvBo+oL(ATo0`cY~k8I&Ga_ z&h9-A|IPV@8W}(3k zha{O%w7_upM`$+jF??InjexDSfb9u|w3XCPr6W$O^<1pY zt|%(yB&QiUNp=WJ;_w=tw>ltuKY#nY;k+L1;juqt;TT>v{?BAu&_{uPp-(CT=O%zn z?lDJ2uRl^9yT%E^Ny=awWhF-&d-dJ+y$GY=zctXv%0+o<;=ecBye^H|u_2hi1mBvmaVrICE`KAf#o%>RWMxYQaW2L`5Q>tw&GK?b` zpT^9c!uco2chA>&(f<)9KsF=>;A?sr+OzCbG*tc2H(-VD*rj7L|2=hm$knMU$?2<< zZc+hXCVI_3%PB!tqCW?yJJr9Lgf}2nzo%$Yo<8>Z5A*nH(N9#&s!s+z=^*ROGxUqW z>F6{2p78bu7v1KMRKrD+=0I)HjnVd~sU+(2lbocb(p3+hG&KFLRzxW*>MOZ;0=~$( z@W3bPpQ7}RYY`LYm}!0JnX+$U^i%TtN41fj@!>|9z#^9&ySXM_Vs27JIkzJ!@fCiZ z5=bTC1zI$?@!0N6S)8(hslJp8e`R|u8oK!D)MWOlp8R9*kh4xmZ|vLPwOr0^v!ANw z&nlNva4I-Q&wn<~tV=fGtmU~%@?Dp^hS!h6l#9Z9Z{98*75_l=b$A2LB8@|_@9i4M z9k4pzw|0^~6x7jzM!HC=ggT&!TK2Tww9PT*!+*wtq~6z!e^49v`u@}S>US4sKeq1G z8R}+<&7;~t$rGAU*U9j%UUvDj(lR0G<_Uty0|8{SnLPN~P@vPO{w0uhu0%o=jl*nK zf%@T5GE)|Hm##qPwg}j_qA%4P&@zf;p`CS=B8CFe*Yf|vzOOE*gK8GxL^^fv#(2k7 z&b5GoTZ+o|BElM$8KcgypTGBmp@n1?3hwD%d)PGQhC^JgNTDw1w)Vw*s@MM0OrN_ zjc;u`IN#&D-Gu^Vc%oNImRN)XkRJHnni>9#q=ZZ4MIqI0#Gm15+))F$FlKT`n>lW_ zND_19yW%IlxCKH2i@6lvBRnH%YCHptZW%Jj-l{RVFq?Y>h3LRKAFa@yg$YZzoPMke zj(hlVjY=rbdxz}aUUX{;zbTkBiYg-@+ePIse7+~< zpHm^hOgE)ol~nwUOk6v8NdSV7R@P_7YTzk9@yrcal>VHU;wWWm{l`Er3k5%M_>3cM zkw=5S?e{QT;r#s*?2m+5a7Fm2(~{RwM3w2c_^*TX)1veM2*nOG$bCmxht2ix6Y3us zYxIfJY^9wVgdhJE`Wqj&FP1VC=T_ph-a4$u9Bh?qc(j@n1vua)Zi*O+jl@3JaEr+} z)pS)cv41G6GMVyGb68y9QweVnuGe5}0$$4+iVIct!=X2v?utrWGSA{B+Bvnej0BVX zvnfafpXH?RYl2yqF`HYLZ|`zO6jFaCJ&<|guMhk^y;{SEQfT~ytf4$cG}-!6p_wUP z_pe}wr0ctyuz6=S=Ug1E`R;eNUbhJzVHM$NY#DgqrX-GH(1fVG1+J(lT2x?->ShXc1)X& z2EAT_&9;9)Z|Dybk-l$!|3P4o$b^v1Zl7`aVeusCQ2Y98X@kMBPoF}+oN@6G7a0rc z{)_>kAN(|&Q1bkwKfWRZcht$3FCjdR!io6z=|+yjvbF9Jx!DzS}xqgCR&wErdIm++ZRDP{H)|8B3xd_;XB3@I({QU=o+*bXV5N9i%4 zUOvL!V~_cJ>=BxqAwoQ*)-N!gh)fOG3Oi_G&3qtuAz@_!I4F?61iFY9rd( z#3X`1V~TY7c-*k++Ogt?xOcgoNNR9||7R{=({V-X05;KDhuJIv72iDTl|>tOj!-7` zIyaMCX)X&3A;Uy3{zj|j?P7shJ{$Wl`6)lC4g?)4ltOaB={@1S`IjD({DkDoC4^l* zJ&w>~GPwqoN#Dw;Ac7Tne{$`RQWhpyoFsX^x9S1&^T;p7t{)ypmQEMTK*NcI7(8kj zg3#s^se|-_%WI4TgL>*fKPIb>1^BRXAog74>@e$SR%5d9deHj3_0 z!WuH(IBFX>rFrYL)iRbx@N+Y*Job*s?)^}a7>$lpc&8;Q=l=5cYkt^qtIHMad+p1d z4x^Lfn~taoA!)x(+?=?PXSpK`ODkK@C6U^m827<#gS?~r5VY3`L8=WqT=qTJj$BVG zK7a8S6oPiX(l-5aiZW?)vu*Y)CVV$E1E2M^AI>Isb_-PJ6BQ=?I|_fJPkL-XtKza4 zhQojNC8(aVCQRFv=HxdpZa(r?3w;_@W5tkkF244C%-)tbo6;7+w_uAK#0^tS_t*>0Zc*`ev6BB5=T zzC&)lBue9gj^z20%_5fe8RQTnZy~+0!T@3xrI9(H#0Ke3yDd*C&h{lC?2 zb)|H)Xb7#@inDds_w@SHuQWbBh0tNYT_0%0Z$*CC!oalWUxN25I|IfA!l`R|#ElTQ zRl_6TfMpJuXmvHIe9y*t!>#tqH|om;fMR?g;_u zDiwn_6Jb+WNxlonmpu4JY9ZMN>Q$s%ze%fy0!t0|V7S!3W^&NR6}?Bl#3cZY;_U!;;6v zq};hy$~$QzRLR7={8P1j5xT2@-9`P#=r0A5TJmA6bT}bY#JcDVTU&>BP+J=4N z)^g%;x%+c7S*3Lqw8V`_ofR_!aK)=%wr{R80s{$&)I>{d3WPzkvrz`}{R=6KgPThJ zegf%*Pv$18KKp1A5}~dsMrN}eE@HVzMReK>q5|cs2U<+QGh|6_jNR0%ZNc>3SKnTY zZ5^1`(U>2YMErf(?p5rxtrX3+SEB^w`od^_r zGo9$jIp+FH=P*=d^nY;?y;`F+RX)G7yF>pVo5Pyi=vvn|cfTxjaGeXKLmA$RDguLP zR#Fh@o?|W2^$+eD2V7nz)VD*YZq`z*TK-8Zc(w>xF{!KLeOhkZ zr-h$=_74`y!QH>oxq#O`|34w5r5^vW4!p{7L8j+e=AdUf)a<-W*3 zZTw1j(#mI=3=>Oj#bLnaU&b#DMcijQv#mrh9x6rwx{`g1j(+yYRWpA@$)-YK&U*=; z{AXzj;G`6YX#(-kZyN~mWfShPx^=;X!d364*a??73)GeR_LD*@Q*kxcP4W5zhCdQ^ZohD zV6N2LHylrG-VcUnbkzn^YZ>y)kh>~rm@+LZVKeu)O_D=0>fZVj$yn~3wpziWm-a(` z{9V4DqWrh;!)kc+@E1=!qJpy`9>MB^h~v-9jdTpp`g$c-_vO>Vob;P@NR8HXlg1QJ zrk6ff0YRmOodeV?g3*_m$SI$;_ZN~A13&4;_B;3-dWr7}h_)RY0|Kh7Zl8q}4F9V$ z&r@p_ID$fP8Rf}Lh7J{N?J0K=EZQwyIa^vn5#Ij^XckKle3At&KQz6l7x7LuX$iUL z- z!kUYbOo^T~4u^itT)NDWcOQtx+5eqUZf$JdTkP$MFdpXbf<_&H`0wD(k0!U(y>YD4(aZc?(XjHmM)QQIBTQ7 z`@Wy&Ip>V?{9m4T4#hDxd#}CLHRtu2^IKy2?K}w&80{{wNvz+4@Fj)q>QOkItV<<+ zS#bqR0G9d?arR>bk}j)>f|e$HVvRdZrG?qY2^g>f0`Rmk^{(%IcWGJOc=c|wBszVK zId$8|Zd+qDz?^N+nQQKXaU~!0;zwvKYX-PorLlizp#e9Ob8tuApv}1vb_y$awsYbt z=M{VG51E-9Z}sD8_&BxB=>_jX9kc=657xpI7O6|Ru$YIs#x$P8rd^?m-#m-_HVNkx z#hAt#Ck6#9w<2IZl;o`TVX1CtD9e4v4dJAr{&YN^BRuVE-aB;yf^>A~D${k^hN|s^Tbr@Ih@F%?5Vp(gj@nkXO^?CU z%)p&DQI(qW!zQkMs{%MD<@pV1xon|fS(17JE5g~G(hMex57tMrR1?-m`iLA0j^FxD z)SgzcT-DrOV{qF-JZm2f6%iF)Ttvg_xE(#^Nyd1eOU;Lkl*13CmeNe4uX2JUm4n47M(O7sby z=nNQr4t1D9>AP|D`0q>LnxUrgsE>L;9>8LiS?xG9Zj2Rop0m@G-&K(V-fBHN1oGAk zj*Y&4882Ujn@Oaq!_kIVMw6D$Y4K}@h5_-y;ZB0WB|Nd#?zEMy6PHetZjdS7KWEF0qiYrLmH z=qa3yTftK*xoSb$vd#G5IuwV=k!mw%LZmSdMSriExkmZ8;E}a9ix0w%gaDJT$W(9h*cE7%ZGh zHSl~YS=@|)S_jrlzqlPNmyPl8J^Uy#;`BZ~?U87FOO@vn?Bl=ZZ@Z>FEp{>$1V~p% zIYX7C`b(f_7$=7ET;;MdM?m`?!d7%zxCoI1dAz;jbSHx|RmtmO-Kuo$<|alwr`^Bm zPRm_Pn5Dy3(Rga52>4AKxcs^XV3W8<(R&+?JYlBjlvYC%Ss=GH1R+)_zS$!}fs2|@ z-qmp5!Vd!szt)}5N92}M8|qPtKM3hF_p5jfO@S~rPN!)RPuZp#p&vH zYZ5X(`3z;_>qd{Oqi%ELhBsZ8A8y4lwvNv+&4d)4FuN%#lyJqQ*)ni2tW&l#fI)0h zMpO&OqKV#wAh^Tyw^8qXoc4Q+pZRO&qBxrvnbqph$;V>)%X#>Pt{GU)sR}8K8nqJv zNk8!w@Q%m3QyG_EiZwntKv!!rXok4WXFZ6uO8R4OPn{K&VD^T2*Mw}UVo!EH$k*L;kJDGxpxVwJb#*hh>!n|x_}%e z;JkmuFvdxqyAx1O-zpAh7z#WRb*(3rnhk%d=E2KBJf04*>U$&)t+2V9 zzS8P`8NZ0&E15i}E&AIr)b;CI@9-Uff86<+zk^lE4iKHyLu5JVp|J(gD~pF^TDsXV z8k>j2J(v6>gON4q*+noyk)4T}e(zpAJG5hPa|S>2QURDaeVUn@^_74vS3>k(#)7K) zF?S6kuL0No>WJslV*?=;({WiV2{mZ0m(JoF_h&aRj>hE99~F3J{;ANfh*?vKQIK;a z7}IWp<;1NJXVv1b?&Ndhwvu)@GKFr)FRb-2r!9ItRpJ>o9G#PT#b*L31n$(R&+x*I z?{UB4T345T%apig!Z6gHNNn59=+h}HnGj5zg$dyW(a_(EhjiP44tm8|%JvRm@qTXe z0am@C5duzF-^cn6Lj2IW(N`l;0)qz?aJ=6d)k{t7q}`&~j?*I_Ty{+IWqiM#*_7?U zFxArrDF>gDC$jRYo&_?uqcDy*)T3aWa5bl343~5UBVTZEmk-e7sSnnaAS~3XiW}Xu z28Xz)fB(9|&Sau6tUs{B;Ghp0+`el{=dw^>yHI??n`lmAfngHLxdu8&%r>pS8U8&i zSNBPbH}`o=vXEJPIALgsTItcTMFB!cuAss}S!Q|?lKClfoL0~n^v2=KIR_Cblo3Ux zFjXTYQfG4xBQIq_j_u9XQ+*Erf8Bx|-v)hNpp4`Bed$21KWgWg$w=p1rv?i|6Ec!9 zc-y zn6{_*?C0RXkaMylzDp*5K~rPV}XPCB#-dKE#8&53Sy z1KyR{2iF)zS8Q1>H%X<9trN84lrsUy1t8&~Pw73d6BT2Z7>~7qX714bs)ia$)syX* z)@;X{RCnf7lHfDE;gC7J!+i(BE7?Jr6SPIqx7#^J`p8GvG23>f$dp9>3X9mvEk?Y+r>hH?>TQBaff7HNC zEY+1LQMzX)Zu;*Y#w5(diuw(ozS5s7!eo%5_0m#KUHpREyn>v3E1KPOs6(U)&O&l1 zy$fPNZ%+ASQbZej!P!}U@}La9+FsbtmaWKxes8bR(Kpa!uV5@?_Ly7}E!K6*u?eOUK;wTti_&Tn}&S@V17C68svCJ9(+(%jcm+ zkFx^RY#IU%lJQgJTw5d8GV)7}{0Vu=8bk1IiJI)b#l0PB1E{X?@JIUzMibZ71<5cTiusTfkK*Bt^vQ3VXd|(T z=RK@qGif6d_XBU18K_fhrhi3C1|HhC$k0ZA`b#&;G ze=%cQk8-AUMZ}X2g4f2T*pe6&;%`22()xqN%@#KdJs)DC(n4o__XJ7z9hUY7Xt}9) zlgf38SB0u48|qx3DeJgmR9@=0oqiSxljZ_TrCXL`D~;naa1Md6+?TY443|wdf@M&% zaU;DjMl~#)2bY!TpsMTL{?2DEyh9)}O%_tDI;Y1su}>IcN~S)$j=ahS z^@ofFVpOfc9yz8it~vycHSsjXPtLip7xs7u+l#+1^0mn!grSZB_J`o?L}_}NEWg8* zzla|zp9*xRHF&pQdfD;=p?RSYJsWDtQV&t@X6{|yF=kk4qqp~?BgYQt!eI3QKn1>e7s zH{4t0=#q@%CbLp7{dQN!bV3;HFu$tzXlI{Ia*vxd;Hp9+95nM9qg!gU^%lmR#y4^^ z9gF*xN}#9_;g;{f>aCsk)w4#G|6(&^yW@V(aB=j#@x4;~eSRLO2>nu33RETBcmO?k z5>7snd&25)O5sT2=be%D#WnV&WASUT$HabVwKpIzo)r+z$(~Xh&@Uo`Xhu8pmzT=`kT8}Jd5gM-8k}H+_@Mb+S#30uivtiWJx1--X!tO80ZxZEob!=5W7OVVdxe|!bd^@h*~pY@ z!y#542>$94t~?=F{I*WH&a)iZ=UHcxS{*|DoKVH|su${X7x*D6p#pUmxhSPGCYVB?knMzhPqS=+q^fw%^V zt%+Oqq41svX!7nfb0nxZv&d*Oh~WnUv`YitbJiwjUSolEsLgY&?A?mhc@6k|(#?0j z8uWzJm$vCoKB|Q*Opdy2ec`!-W9;*v^s@6{{(Z+WkFRq@MJ{33ItMLb>F;t&t0aBz zdE#oe_nn8ig=$)Re*uy8tpbs%I}W|u{Y4nZi zkq09SaAbphMK=8WI1+mf-+w**oB!S2IAXCe+g(?|Bc7CoJ?~mpj8( zXR;dS;SW2ex%E2lcSdFXkQOPCvwP&B>rhI&_C&0ec1P=&Q=G0UFehUc<@R{ws2k|* zfr}@eAijrkU45Muxd2$W#K1Y&^{Gr~5S6JHQtH+0crkiieer@7#z-f1mIFRtBnbiR zW0(Bm$unkS#JCj{sLYX9rylqH`}CB052J7sgo)!ZO7XS?W*;X>OMWie+V#*AZzwQ& zKNoN#N`GBMawmlfugmy-zK_SGCt$ZAD@DVz{1uD*@V21*sq`ERp*~f}$#1=RL8&$) zMc@8%AaC2pO6Y|7u@{<~B!nD%Vb9E6#MP&GEd1IQ=@>`xn}z9;<1EWiq+j13k_i>N z#NJW3$jizRL@2RC(T{*$3~^B5h1?3X4&^S1atNZms-(_zN|J!Ox+hR7lGuhtAcH(#T-P(ZolULqqxyq`7Ywc&5q1~>xIs0_6KL$qANz2=BTBIJc;U^j zyF2%!;Y3xPo=~+Q=Lh*w8jYaAYSBS7JDobOm6`ra+yYQ{LC=%rtnl6MfXsBw$)(N( zX)P_3ss;vQX2bceGL9u3jjShNK~3+t->c|*)ARLEOsZ3*8*Vg0d4rFu+dok4BFAG+ zX0QE|@iJ+_7~44o+`g}fYeN4gNktrBb&$`uJt%Q+QS-(-Jh2nPOcBU(18u9;a#pIN z8-oIvFkZkvJQc87!{9emq-*NJvJ8ito^wEq+F&3wI*@fMlNB4=#bu8&taiQ;~r{kkj)=@bD zi0Zn;Qoa@S*rt}1kSfz|%YTLR!Kuq6NO8QmI8WmJs*`yv;PK`XqalxfjzS2<+p_B) z>AzMlVcB4lhgVE_MPqKxcDKJtzesGL1~(i7>xg+MsJif8FMDZcJ;NQRXC1QtKxFvt zPYBPV(KXTo4*S~KovHY^ZX?(|;{i2<7yA$+BcEuz zDp6B*0)z;4yV?WE0k}ZgW|4x9Y~QB+r2WTgV+`$8 zC-vFv7&7)RrnYI4<*+t@1RH?sce_UFD)~_X<3ta@Xw0r!-QCx79=12b@iHVx$H#{= ztuC~shONc{>~ysnEGMk2>HFnzR(0?assblC5mRnl_bTTa#>Ub) zc95Q+vaK;bmcMbCusWn;iaf9yv^5Y!ya_-Wp{ucfVEtt*LZOysQujd-M z`bQlTES3<_sJx!yUgL7pC$ZS(&_CQc&}Ayf3j)mFFFc_1@$ssT>f$##mPI`)V^H@pI#bte=SHC7+`d}4cd#MxG6&5 zW|`zUqMLa!zr719kB>;f+O468GRCye*AWxGW?h0YMU{xGG+-z=1Yg}+&28vuu6=zv zy=t8yxnbWkE?2gaUf6Mxktzo@F#NV!Qm>>GD~6`3)aQK?!3;*mfljK|P0iE}_pn#5 zYSKmbLe9Bbsrl{ckl*!r&|I0)@QPGpPdxI;2i1Dp-7)=vn9p}d!W?x4@_^!c9~aqR z@VP4oiF8EKrJtlqAy({`fDoKAxj^%wgbr)qQZzY7*0^D+OM^kPE^B(FIO>Aq3Ziz5 zfyeWI33p|z-_p1j$;}-p67F`W2W5)}m21ZSlW8hv|9n&U<%g5gf5@%k2o>ISv}D

zhAmE+L-l2M7N~$5{?Y*lSkp==v$_ zRj4;xsQbn2UZIaP7KkX7e~k_7;@W*XaH<>a!RX&trby|!n?-I>J%qN99eukoS?&@m zTA?&A6?pa}Fjy$E$&Z|4^T_HN1G5iV^flA%aFttLX&ciecb!;ZZH^JBxYOo?&6{jx z-HQCDeRZbv_3Ak_sbUQdUoWYQ_Nk}evQtdm#;)(4wuH*%raLOrZrPEJ>_iV(AB%~y z+uWa8l0dEw1MuUqPJaJ*k=aB?j`HoJwBERWvEl_mZjmoM#Q@tO9Xef$Q$t+o44ejB z|850R_Ti(w2*<5^)Fql}>e6`nw!rDg`1#^@w_WbKAcBR53?3P}=#mw}LcExmL$bSt z`?AW~8-F$_$5~s;R6|W=3UltQ5;Rh7k2~+`+?P$&haQJEv^LSQvB$&w>6=s^tI1kA zPcTcGj?GJ}uPxTD5sM0BD@5KA>-Dacts)aLHpA8Ne&FLyYEk%;Bf9UfSHTX3U_Ak} zN)6@o?k2b!-y0!$Lk;+K5qs{1e}qr>mAtNXsg4ON2Xt)v*x)CA4d3#WH4|)18owLp zh8^XTwX#NhqptAuzgkONF6T>Tuu}Pkz0#rCE|Wr1GCNVXjLsS}i$w?uZVa`4(_3s| z%8JPw0rWNWfmdlwa?s8VpbML?j<>T9K{e@?Z#6}ri)KQnS>Td1hz7)Hf6U-0_tk1U zD|&5?V1Iuf8~xf?E;dcX97YOu`_}kt8iEe)zsRSG!-6Aq4|90NPVqMm06G1(X+Cw( zcPk>O3H4Uv?cQhFlDDhqBYhttF413;1v$+;sl4Xx&dwiVL>#;d=|(d2OE_Q{vt25z z63Gtzj5n!M(P{*gM%#saU;RLcV2!t5C#{Hq{Lz|tQMp7FV)t%p2f^GeRz*=a1K+Kf z|A6oOAtIMjQqUR7UC74;GOC)-EJ~6=eRAfOO9Rs=k#6ykU)Zi+xo>v#$x(9E*)k75 zD6{wC=W^8=Mm%B<2(-0RTct<~vM%|LEcILOp4O!6)dLlIPP1+t$O#_IRMl^gM@r!v)@$t`60^yVIh{EP|#1gptcx;Tf!2M7zU?% z4wVOADppmHm+|=D@R8Yv^E1Z6saq}gY*9jMctlqHw9-aoTt%imF|4XYGgLLVOFqPEAcW-PncERxDu?L-N)hv}| zEipsNI)t~597lS&MbbQ1SJ&aLju?Qb(%`v*h^nY;)Svr4hFL0g8DY$goEv*eYfo98 z*>STKLz0qE<&2Gqv)@)q?)1zOcO@@i%OWfL*vZutK@)A5Q6c7@bB`N2D)8%zsD&%w zXEe^%YcAPehHLtFKh~6cuL7(PZJq^MWPs=dv7rpk4+kRQ0S=l371&=jvLVVud6BIv zqJn29fy{N}wZo_cwql+O@ zDyM3T)9zuGyfCO{bngYytZKNCf0Hiqe8Sb*DQg=u{G@s$j(yYPMJ+{5Z*7+`&{nMQ z{?t-DXb=U_R%6dQGMoGEgbtU;tYLqxZ_eA5<}D2*+66sO8x}@bid@u$hQo`}lndO} z78Uz>p+(6pj_2`+Hw;+yp0}ghe>n=66Ks?8)GRG9J9f-HiXeExbND#UbQ8-Sxob{{ zN4I@kr69Q?lL(Z2xP7grcRYl%T%QsJ5gIzni9+D!7+nuYRhO#`;TkXZlwOW${>{rA z-qI?|C|C%ascd1wN0ul)u>#%m>pk0@pg9lgf%%<*16`qN|EMi- zoQ5J2JeH$H$1ZH)ys_MP*Ok+3a&!KZU^!dYyj^om-08wGxS0yFZag6DLsTNSz4>1fz_Z=_euokYS9gf-F zm}cv;eP&rQT%;HK{nrLG3GV}zQ^>y_SdFy=qoD#fAbb5oy63!-mF(oBEa^Xd9Ow{p zTz_cC?RnTaP&Mn>;cyojkB01ad(?ax4T6tBXnI{8D=BaeU34O>rs24hKbjQ)KbS(R z6k5}7qgRukeoftN^n#TEewP4lbVJ0C|HYBj2>+_2u+03Q#Y*>QELEV4~eI^#OcrmI?Sh!Np!@XeAa+j`dQU$R?x}C z6UYg&~Rq`rXJFC}L{mj^Wd5qzkH@9(OtG>{+EU8r{;#x2DsEo+|F z$wDFB@9wBJPT_K4w1@<)&qTNO`bo%hpcnNa3?}{!%y22YGzn1{Ex$=YP?HNQo*cVd z$9iIcD0er(WuX-OP%VCzI#8nHMWXd*f0I7_uF8JvKU?fxE6k_@Wz-HOIpmDebBa(xZ z#ez!*I?m^MOUyj!)RF8`Yqh_vM6v@(o(t#xS*ajnG!854B#$Q4pUGacvK5cU_|#NU z_dw={e@pO0MFx~q<9KNP%uMxWyz0hMd>|?een=OEYSvar2n& z+otC6oA>wSEH!Dr1<*>fUUD2S7x}K_DptygzbO2ce>gqB^oSdrt`K`ypll`=7|tZx z$nP6U2>;_UU*<&IvxaN&StvKgPEy_OJmL|WK*aJ>WClz~Sr;R`nKtjvE7PlKchTwq zwR9kGGy(sxR1ZZ@9B|xzoC7oO=l7NVWetqPFtwfxy~xWt#0VzH5|0w{rmw*ZVAA;pIQT5n`jDQiR=G+;I8jjVj~_fO%^{{Kbc{5>xv3-oe{<~=~; zyn_6?;qO?0R+;U#*OP7sXmb3au>)EhT`M&mSu=t}*NTRUm3) z<$_ejZ?!nJdJcqO{VnMcS?fpiwC3e$|#wJ>$Ye)iu z;go7kqKm+9A7N~8N6+Os=3)vtard{!$jtuLt|PoFe#J4kV3pM5un@n7Sx$j4P>$j0 z_o3=&?w033yNUA$*X@j~!+)VSN3lY$9Os%56@ya7>y*wt*)N=2C6zRkeD2Ih)iG`f zMC)}9NKBltds18`tGnn7A-@NL6=V-xZ#Ksm@VKNzm|x}8aJKgNJxjy6N&Z~8#xa$X zVyeVEMootjRnu*AzDd?JR&4KHw2}Q;Y@NKy-GV~jRDrldi#lnMOTJuu?JV_FqTuCL zM8eE;h5X0nUq>6o%CcpUu?2INTyT#bf4hPYqnjAQbmjQ?4{jP0XKE)0f(b416 zx`mBb-Xe-k>M2w1M91TTkn1H8Hw?yaRFKpJ+nt5*ljc`}b$$%%ZKJN+c1{dyDV5)Z z?OI3CCe|p`MNH2xn(|a4QdhaX*QLV^KPK#WPV&h67b2=&m7ZhNJt+;EONIKUnO&^& z%<=<5P~bN?bu$6bkP_Vi*(^`;KVZJ_2h{?|jz)++cw<3JY!A|Suxi@%fs!C0lx3Eq ze)Zxet=R#lnx+?-sUv%2#W!NtSQLp`YNiR+mgDx)(ONNy5b~icqWezEb!S?K^Nu4Z z+C$y=Fx>wlE)*UnX9&ZKM|>KPT%W;hIKLbiwNx1l+DmL@?G#gfeLLc(!C+T(MN8E& z;K66VXC2o_2zF%oua5i{QT5?zhP_oU$4$Oc^8)k;A4$c3pJ)|w+PoiAA_NS-lf9r~3FwK!b_#x8B5&bo9R)70W2n>JN_BY(bj7;Pyl0jV zQ4Z_S_MpC1-ECo&1szVRMg0e)&DGb@qkZ4zZ#OThC6c8yZriIY@4L6;H&!kgloMm* zngZ~XGaZo?uWnU3FC0pcPcv#0&LCWbUN4?C)LL4|DBt-n#JKsS-{%3BPD@w>d$jI|!m98@L~Fi9Jr|pjKJO)pUgU7Am*QJQZ}+~vKw;&EfEsnXOI%Re0Pi%n z0#s#D@ARuw;IH!(e>&#cyss<3TT7^e@Ex^G|K~j~Cl_#%+fjz{!Y5d;vU6y&LVs#2 z0lXjq(mw3NMls=>)O7p#mGw2D0MD6fIXTRb&hp}uw_5bV;xEq z6BKO=yh1L)cW?BFC{K=1Ke6eH?p%7JeTYLlU3Jf#@$l^e9X!5XfeWFRTK0{p#zdDu zwOD@pGiLomaYbSx;EcYXt}m%btQss~<_qmDx!e&pMW8p>g} zY#wYEv~_L~a>&Xl~{Vl7m^RgRn2W&;*JCN-6EJ>-t#JqIXTs2b%Xuua)dE%WOaG z%jovBubH3=;ls-;U@s^bB@&6umJRt_$aiy&bpA;F)|P6BN~;TetCwSF)tzkT%Joqn z8>}Qn&rK6aW8+?z&WL3jY%UGEW{)r1hux{%^pm*O>SkBYq>J1CqKfHX*@_H)^ELMU5jh?ndU(lavvt8xaW( zFsFZ8uuy!99v6u(+<5`uA+)rql$+pR7|PM5DDA#|{#Bgd16x zYfQoX48{A|=IUs%==YDZX36F{D~1bB{otkoM=Vjk1zA1TYURe@jr0xO$$IH@}N%mr#38_2uY z2ZH$z?h2xxHo(VwsU?ck=2iY+LY8~mD8_X=HFesu|8OYbgu));XHY7OhR)*$!QzkQ3?d z0ToqtQIDD^&Tn0^clX6LRN4JA9He23TU_JHF%OhM5zU>U1^)HNr7m||N~~lKLQxJz zDq!%{g4(+tjr_Ch$hYcd6=c<7RQH>>-!*v%dZl`;unLl4!_0}_5tNNES?!UvYYj66 zq%;7xKBu2$s}d7}dH>=3P12)V2(EqP>47peq*WLh6X+uQ@`s#2nNP$N!+ectz)w|@ z=liuDCAER1WOQmYSYPp&%*>Ll={IQRt%Dd&up*a0wvToMC}u+EL>^D=5_eW(v9zh6 z?Zhxx1aH_dFPz2hRp6w;EZl@Ky{KRCFWPLlpPry!M8(3Asp@q#W01(Ve9;v4$E$eOH1qgCn%&hYxr4Suk^2sZt0Fc=_46#UBjhpK_(lNl3`j!~NC_ z=xae54(fkL54sj2KJ=ba%#F0*J-&nSjxIo{m{_zto*_`M-72fc&6M{y5l2)IKNQaG z^*ze1^8iDiLM+(t8gE=5Sfa$GEF`C!;^lqN8%XFQ30+qe^O98YO;B_+x-6UGjrT9f# zH}t3tVwU8EdtTt0F&H~4Ru#ph3t!UaQ)L7;wE~=O6*5Cgn$QgZW1tmVFjqSG&v=>~ zI2twkYmEOu$+zq~h0Sv>5`Hxe+wKPqX-W!1x`qq!HOO+rF+jdPg*UgL);DoyxP&qQ zV>8EvLF%FQiMm^GV;i&}zMnZA;sTL@-LW7lIgh^4AS7wCCiyU%;8}=39i)$B1U3MeNXdWOu7X5DpLjzJ!xzB26VP(|Y5Q0wW0|XX%Wskl^*?BNBg0j3SH|X*4(50MBGRBpW#`d!CVUS}7H-l(UNs1{oq> zd0+tcBt-&NAFivt{y2KXd zE$n`rMqhsX`KP_dKi*ZN*2yV7qNJWl>uwrqvS}OfY#uCCp;fQ~S~)o5eEu7uzNgCr z=$UE5>vh#|RHqOLAk2Ly0_u=QDr`_a&VC@&uYGAu&$ZefdZ!*LnQHQGb``&WxFynK=m(x=7> zDe9k8$9@drN-tV(z^tFcofLj$Oe~F_F(pdWyS2gx*op~cN8ANEiBy=G3l$|giYoFM zcK`(Gk2geQ&e*N&<$}+_vv*35-6qv|ECa7SK#&b=mlb>bvXm&z9#-iGNDhy5I*9oX zEOIxnV4}P5CFdtR)u~8fQ@vdJXF00F!d&>6^@jw>4sJ*%67PNCi{vBu%&jo4PJ)kh zzSAIgHp3P=X^RAAxWOa;@VRa8>sGhT4-SdA2_uajhkhr7CBk#shL$VEBV+qvYPLpWE@KWA;E#$viet8;k@0LuS>y^T)Nz&O1E#npye> zti*7pz>zN%9un$urra^?FF4Qb@8ot}l9oC_GWe0MtG`)KA~%N_c`+(SKOepJ z;{XQ3=p@81#WkFC91f=jpA0vRw>dCd2K-z;f2N82rS#P96=bj+q9 z<2lpgEU7GBXJV6NcBPb}%0%x0yJ9F4_a*8daF}K5g4DS~8Xi`kf}**~bbFZA`1N6X zXyu<+`hRH;g}Iu=82;SOTpe6QJ z!|S+;B<=78Ie(LCy!sh*lDJ_ys>^@Q^FcI)!Jkk774Pu|JIyFSQXFENZKx@GT@BfBjvV#+^xZ*S-~o+576Br<$gc47nVIn|g4N`;xrZQi>^ zwQ&$F7y7?BY&X^$DxDz54nQnYo33kB`?WYo-}`#2W~ybD&741yhLlX{bK)wTX!hrS zBQ5-fMs+$`ZQXGi@y2;6`u;U=f^RsLAh-97Hy|`-PN0J=!XEV$&ssd5^B;|c%sMis zYqo(b4=4nnA5k9pYn45NsQMPF>_6ylni+pH9OJ31G8c$>oOuZsE`o>sded50ZS&X( zfcZyLv-crTKIz^OE)ZcH@_gP-OaWq7omYC6b{8L5QMx;{YCEP^ri8E;zhCCIwd!WZ z{s+V8q}ehUNgMzL6LHqwh-ZNo`|Td8aN~W>@ybzemy88o;$--W^F5XAcDX2tht*M; zYS{(iBu#kYf_uf|4SBZFn4-=pw}Oa%0StfBr$Mcm{R?i4-fdpo)SA~hNIGgQm$PHi zDRhro_lwc*XYckk89Qu8VNVTsq;@4N35(i(v^L|e)ed4+c68=YE$0iNEC?$HF@D?T zGz|T2@X*kRIlHHz{WXf{oG6%%&Ns_7nU{NzLar3+*cvOL8fZzO*p!|a_tDygsZ$ki zt2~?4a#EQ43h9S&!B@5yMKbuKt`1?H#JG0MwDqHUmIpLK6V}tC9iz|N9C^R;;jC1< zL5hETz_~54ezC@m?ovi!YHuvIab;ZDm?aux3t=&vk)A4OHqm1)q{~j)CbWcR)D-qCwO+Al zyL!|pW^*$WntOk)JQW!WLn$jkoG!lW`Do;y#GcsbF_$j?-&yO3kbR^6a%%xkgOiiu&Gpf4`NAeV-W{7eov+Qw5rwO zKiye_bHDshY0=!f6OTXOUZIXIv4~LvD02B0`-VxUGQ||Kx~DQ((d4`HE)v;IIwY9s z(uL)GPfIP`LJa0`S?clr6& z`Cdt1E5#N<$Ey}>phgm^Hpw^cXEC-_Pd8Slm@8ieZ*CjM80eG99tp*4|rAXs>Nj2hY`&$j>j(UYtjN zy0PpLH7*g1M*wK|AXmq%>20Ov`^};LU!y897vy~)DXf8+Ff6ezm@-?0EJOXPf@Sfu z=O~ntuD$vybgnxkrf>!O$`6c&x34FC-ZGoFHI_BX2Oaj!K0JifV8v|jSsS+q>d8tH ziTNq<+NxSDU=++0XDykXf8oRaiG{+D%@o)X6tWeO)V`-7k0buRdTeQ=jjz zCJy8*1pTi5=}yq3c6D2krK5nkB>HoX*)&rpxuV2-YmT()!M;r11T%zFK1sJzpEj0I z9dB7<;C{hhwPtO@Hi0LKOOs*(DBCq7(n+YXi1~HM67$SWa;~#qI4k?|jyV6UP9Zy( zBy7g9$9@C+P4_6;x2i_#HYialCFjK{&`h97MhK4>LMTv$q8|;@i=8k2<8PcVl})#6 zF8)*-pVV!KX2_q^G#bKj;{x$dZbtqogFs=7R+`3 zey1^{-`1XAJf*xnpS2XfK%08W+cV&8B(>`wDWf;~?3#XSJwNPl3+CSs7{MSxtMnZ4UfA?Q1r(urf85Z!U8ShkZ|HzRo_SE*XaUhfDMPr%bUH z{9#o}2#U-R*_wE!V<6Qnc~MqbP@#vVr>)<9Zd7!@Quns>L%zu~J)<~`54FVfdV_xE zl*=egkgViy&fPEbwwoJEklbU`<=v#iG19cg>d=SJFws{r*Yv%UTto}PSpI0 z_K7hU^mX*?9F5{y?1-fiD2khqin7h`4)xm#GV-cY{FQj(myu@Rk>rRY%H4!udd@Se znIADh$b*ip-tBo#K3O}ers(Tqvz1}iLb6A1!%NLg@sl+7^J(tR-hRiW4}3le>b-QR z1gCmfG4F}=09@j^*kGfS_e+<&d0xH22OtC0f2$4K>!*Omz3LxS#HkSU8?*RHAu*L+ zs7$u33-^nhe|h1xXk_#;4RyHdIf=w%^^|6#uD3bK-o;$m%6j9^`g3&!UpMG;20M0f z4J5J8u&)Q;)2%*aHro=Hp8|9phXNI0XBl_e)%B!{>2Dgqscc3RBze;~>iuJV*U{uw z`*`&uJPY@`t+F@kg^O}1?b~wCqt=%0ovmn3*{P|O8=;Q^{OIr8IZrD48Qshlgcerd zRdb5W;jDZ7uZdmE?eZ>%yK|aoblA}p{K5AhKgT|mnKayd+Y-%^80KHAH;F^6PBSw@ z8-(Uj{^4tHLMzI0(CQK6+S@eiKkZM6odjI2Q|7pi8}%W?-Q0%v!OVP@sVr=?soo&i zMH0n;f1~O5)n>yHSV*y2Jg`3Ir!IO@6=>?)U$ooo#OpE0DGZ_RehbG*lA>25MZnHi zE_-`U@Dq>_Pm)v}@3SCI#hrU_z~2=y`)jDs1lx#}{<{#x-&#Sq9A>T<4|8OU`NEqk?M!j3 zX0tj7o&Lpuf$}1K>-8CR&J&W3H(UI-WhC#m-Wjg36e!o5@X>lu<8G366^suaev(gq z>@I)g6Y4-t>CMT&=s!@^%07tUC@h&ujm;_R3}E1}xh9{4pauEPbagrI=gd(20@H#;6L0 zXun-(VGKgsJt8Jx)+11vtk_b5f7x_6rJ*FBG+G{qeV>)b@ny5alX?ozUd9r1><@Pn zt#T}V`@(77cIm9N&_W|L>2P(-v17ZE$O8bf+VIVhB^$97aZ2#Hc#eMc4^Er85!ckncPb;{M~Qm zs&xAg)m^7-mvs5DR~6yAN4wp`Xq9<77eZW9+Usc^w@=FM#z@>Snd&&SgBsEv?^hgm z=KLDRZ88AQ>h*^QGNMnoGUZ=g-VB|x%+b54zX~4njR0n6wVES(xhL@?N$P$B#wZ2g z1pY3uEtQ&VdS)^tMVa zFsX3R+#@q$490`~Dt}IkAN%X6sJ?GsUM5*>3Z3h9!fAZN{siV=D36g4u{F>VU7 zJ#KH{gJHXwGJkDrwFfko{nNc7>i}7!oZN`5fAjCJiasL}-@NUDPwZ-#=SYR63#D=MLJ=BzXg-i^A+cpUf^|F`y zvpu^>;Qz5sKVjpHNYTgwzVCrGgAo_$O{uG79A4pQjvPIkqdNVJ@+!LS&~i1j5g+W; zX-ggAtDli_Z*7&Xm>2rC>I99 z=4>=bo6|Y@RSXLn5G$6M{s{{?R0C7HIE5y%=Jki38|Hhopf4HsPBo3 zUIo}jF+pn|<)}xiq`+qrTEE5VUajy)1^-u>%h5lBxf@^{g`>Bp5VO#EXL7f zhJSW$;U*o!y!1i+#;*OrrG=uglx841WG^vs7DDv6D9o(c>C$sXI{@JVP;@bR*xL zMp*x*KILs)S~g4UC~xl}%Lmgkd1IN2ea2X^u!nNxJp~rB_SU!*KzRn+_Sj=B_# zZzsOU?i;#f$o&A$k5Na5Zx=mQ^n098gtClI*h*Qy*qm zC_NlzaMcU$kC1*SDT8g;Hga~M0_5VBkn7j8rA8K#y zt_rt%T!K7~+znPE+ox^E>-25vj%O7W{Aq3ue;1iwZdB*ejW^7(}$wM=azwi zIk{AiVjrW8UN$~^8Y|;vu9D=l|Cn2QE;J!Mh1UYOMospW)$FThH%c17U>EvG-;7u7 z@ImPz7{%CHIsdQQ|69+pIdu!R|4q)ntZvqY=9^`V+3GifgDGg3BAfR4oDBsG#J(?8Fl2x7mlI!lTd)}N)H>S6?9!b~K8S8R=W4|(GzhHQ|C ziEhr3a4K33tb^jl;De_GI`slkRI*O_3X$kzPl*-u3BWt{K>Se9R{${vP$EdrSz z0OvKu#Kk^Oa(LS~F#<;vCZdEa2b9HuuUCPxxKNoGQU4OGl98$@(brAX`X#;W_rIoF@pXjFelYAq z+o4{m8vAQ#e1DGxGozg0_4TtQo={{LQ3A5 zePD5`Pz|nV=d3!NWZghng1pr@w<|VOZ3?Ma7u?ZCmZkZhg|q(j)o&O=`s%~s;4|jJ zy1mTec`hPoEEX!Dm!0jXy?6c?!1Jvau7IUoaEmWYpz8EA@WEXUIkWvo+71mLICj?M z>+Hf+2eyV%O;q}>0DV(&ibkFz-Q0f#ORt_W{J(Ie^8MkE#uB=qs&(Dllmot|mPTw1 z*6L9>(_%c#=@46=^NR}$!LmJBpFyG+yc{sY0vnF=kq4sZZbCK;YxXIKZ2Wl8c*P+^+@!e)SMGoqfl2= zy7h=SN$FdJZ9xRBD+}f?+jK387Qb*Lrr0zyPf+yfb>rVow*JPC`MGqTw54b;PjbI1 z<0wAW#I-rU)Lpx!#V76$DpEAr6ZrAi+_x_Zg5V%0opL+kk1Xg7wBUNNNeN&aBHjLy zX6%iK)Ng|g#NeW1{x9m@GAORLYZrWiCJ-P5cMI-Lqrro_ySuw5XmEE64#C~s-K}wV z8rJ~RkmvorIcKU)oijhCYJSm0``&%udtI`w^^Okr)oh{B&+u$GSfSUAzm7uRc9;EU zmp1YRq7N6HBSfsdWR#YoWde$Z(pS30rFi4^0PO@7gqO zsSaX&>f`6{DfK&y0hbk6m`Lw!G7A0{83RuI@ARPXt|?ZS3h7?YPmk>10e&&t|0g8s z|0_4hTi+e{Q;J1E2!Q{9jfDSSyi5z(F~jU)?-8|0Ikn$?OcEh@cYS}K@-h1R1j1z# zagy|Xk7HEpSWj0e;cKQx#;9nWHj>vWl_IFVn|fEc-?!GuxGv~@fh2{+!R_h%n1>?? z8-93<-_NUFO**m!5CZc3DR|h|5E(~r?udo3lMurZcMBZ+Ek}>y`+5X~c;TesVO}nR zVr_;T>ZmVud|t@d;J%IbZFTB~qQ$VU$6?u4urNqAJQfy5x1EM8r_qpqC!w05WGN^C zEaIK4`D-eX3^{X6@Wr(N3QfxTACk=((KND$p^fyb@=;6#;rcA^y4!(R1>gu%eF!E$x-bf#A9Q*;mGcr`&u!in?%2+e24De z4`~FXB$P$5xa$3vB?go0VF=BkaX@>lVwLX(2)6Yc>{(O#dE3#_>?}?k^)?X=pP#6W zv+8Ym^77#fXVwSaJ`5@j{K-NpfWY(ObU$|qF4^+Rei{cjWQ4+_&g;5Iv&8ki?Y-ZfoZ5g`Wp=Hohmr<_5 z-npRUC&GAj;M({xJOyoGzRTmwyuf20*k~iW8vq!9D0-aR{71)n9Io(Yw&ODvCMgAN zGqzQ;3ZaDqJP-)R7a-UZ(Qa@~du`&l58AK-j&8(zD!?@qUegnD7U+wmr3BU%Q?&aK z-^eLA-wVbuXZu4R?9!6-etu!rluS2!WF1V<)^vW>I?gr4l7Vm{mnt9d05EU{>%P~5 znVud|x3!U#v1N*JHQgUdX$(%$;md?SWB?nM1p)(!Ze@en9x20Yiu`DS6X&Y5ESo#E zx&I4CL{&ws+`klI9M{GLTojkwRZS7IY=vw%n@8AcPca+VIG`rWc{DSzV)3?Wh(gnnc8O9J00PL+voW?VQ(xJcV23+pookHpW){Sv1fDp2=~T)_b7h!jyn>E@D>8kJ%s3t|4YI>ZDS2n|JyvbMFU%~wC1T#NfTTat4= zBFsNbn9UO!q$M4b2SAgwa@W9bH6J6FqyA5+ys$p((5hgL>O5;ZsvXY_63O}E#KQVA zh+ikfr~PmLRxo5;#z(`4*jPrayQFxdAf!OCXZG|8Tuc8VS969tEm_D*YH^#)^6rlo zJr&&TK^0$ZL{)lJb_%bMh_5*atercijK3xp+Hg{CaJHN3opdX@ZujCpj^`SFoG%IPil$+j) zd|*C&r&ZIdPVVuC=|4KT<-7+FMzqEQ66sZW-k-kWoM>2s&5=H^n9c?0-oV!Ll8?aN zP?v9s>HSDy=QeHq{QCpvdbr4)cOm@MAATpKkEQ|&<%mD_#OM}u(<#2$7PMagVmDO50F`EuYE{ z0DHpj8wIesEd2r=?fE)^WwJQFQkN$`eK929&%wsAry`z-g%D0zy2W>(W)^o*OCi2$ z4X#LK8k+}nA0BN&edd!PE%$4qdJpsDKTqdMG_A-o6iCYIWFlOx6M?*9BjVH!`f!{I z)189)V|FtTTGB6BD`9~9yFab5XW>I#z)d?2C6$tGIX>_ZR69LmHzN3#hw0tz_2I1| zS6PBa;XpnkO|t_ZiGSWUv7i6XoU-aFa8b+E{;_XjAqc#Pi2QqM-lp=}B5e@fx^z_A|t^~DBsS2+KU2YKH}{6CBj0pA((zqnF2A3zF#T~U$+ z5AAK&3wr7%BKzfBNKUV5ojHH(M@qN*cA{&8nu;F~aKP{TMHd{ld72R7{Ugml2HAY6 z%O`o^+`jf==FxQGL?lum};NZKpf5YC=aAmUmUXYkjGS%?w zEmlF&CT>L|)A@aoG~*~>(n%!L@%gcvq`Jae5Pe@%Yyj;M8z|dHi~>jg)}dhd9tT^n z51i|8S^*m*NblIiocD?O@tB4M^(_Z%Ac`D@U31r(v2$6^&um9?|Jr>-YyN$|5o7Fm zI&)4RIG_d0O}LKdf+b#e)RUTD`VsgK6pz*O$*ST4X-ruq!rPl~(xo2VQa&#@>Px-p z(>~}m$BUa_@#0ys;`-+;r2fg0-`J~%(*R^dFFJZYK?wYt^hy25?DeS?G`uwVGd$GK ztv1lQ%Pd$P5w1M?Pyb$nqt6H7+Ki16{vI$+C&Z&CHRC0?tE zV%vESbaJG30$SWZP!wA8UVeBOikD4)#bA)?|3klbj=1?XMR~K%5*4dX+?Vmn(kbnr z3vEzv>C_xb>yW>r{O>g@p&3C^H&vS38zTH(T;EK%}C!;^f$Kxzx_g6H>&Em!)}z~^D=Vq-k3v)IMwR)E6F2u=6On$ zP`4Ck1vO1TK8~??)K3L@xtSmRfgZ)EIcV&s{d0)|XH6T>7E#3MpEu5b)tc9T6!) zN!#%=3})N7Z1l;-b{T<0XmRAX75fVwOU@*p#71qJqiFR^1GlS0+Tu(>3%?T{#f6)# z1ZasDngo}ig`Yw3z2nq*B>HNvi$;zc9T!YM=E^|e?%h$dACEcii@Y;WN6g@(22YC2 z{3lS1>i&8?FZ);Fr>5wkIAuPBHu#kbi?{iX${k5$G6Me&G??{vYNh?Z4P%f&5Iko? zl{RGNEZHB?2v1`a*ZxlrHj10Nq47I76LX^SqJP!73^v%3^hQ-4(UJdXh&>{+B5_LS zgo1H?f5%Y{klOWyR5S0ryeu^n+!g+2_W485w!p6z)H_CNw2IFaU-VeB;@4H)ml{ur%cf)<_7Ha`%zw=D z4=`3Yt*T0)X(`Z&(<=hwhkkc53|2d$agrE3K=& z%qoacbWjec2dW%J`<&0kgGwIOvpewvnVa0n&L(D(O(KZ2yWsnwlbeRR(Nb^x+J?9}? z)bbHUVWvj4LUVY}-JsTju741H!``IvkQn=PpQ4)}S82#lKpX&(PGMx2mh^koXRlp0 zz<1(gwHdXx?et7<%lp(}gj*ye#w1Lt*B`$1wwXSE>ADK{JI;2yVU4N9g;ilqV{x?2 zp^$>phg}{P6w+qy$)Gus9@@}}6S%-ReNlfcSP31U)4pqxalxNy!a7d z15gc=vH^osd4(KQa19?U%u;$-$ z@z_!MHu=&9P0Y=qUjReWDuJAkBwanw8g>*kqimL=;S`n3;e{MYA>?((vBN9|D}MGF z1M^}wLXp{P<$aI=UI0S#N|b>#Z7f{3QV|!m@h2`>W?Ce9;JhyW4rTo(r68$+ia~8@L2%2B zG*Vpm2rDhaq*o$p&2mC`eB|{n>)@NX^A;;@Lk#0k+o-_4&?vsD4@T8ODGA9GWmf$Z zPrNN_yK23$OqN~exMx)+7(i#GamJ#m14Jms{(1?si@l-D2_Q(!S)$tP693L zYy7T?t49j>k(w@3`ss|Y3?0Z346`B5+vB7aQ2UPoOamG>gdB2-Km?o{))d!+yu=wK z@UHN>NK{)~&zJmu2SqW0k7$^d7w#vf*Bhh&#P@HHh{1*R2R{(gKr8}M;Psx$Fa7U` z-kuw?_5JNxG=p|u!EX`9n6S-3X!!bI3H$&5yiAu+T6In?kd%C(qd~TLD1r{ax^Iie z4E_@OeqRW~kZEvBg~c4!i2V2BA!Z0-Q4+gL^i*f7|1K@^wGgnQOLOh__vg zEbk0yM`h@XS#X*ZOk!(CzmMJNaiT}K=ep?i!D{A=Il9vitK4RBgy@1lFkc#GI?G8EI z@qg;n)Qh8Thd9wNS8hUkTs=fidkYTx0*#)6I++}l=DX}P=`F<9-j$F01$JCm=<%V9 zm~2{2qIuKc-gbJ(9_O?X%u1Kyedyeh*xgeknH#E7eP|hoWRrMb^jNY;;n9nhAhPA* zY}IPoYKzfo8hsy3leupM!niN@Wo|gi#ufJP5ev&jXW+9{}c_I9XXW#~~O zk=hC0s%lz0hb2AFg*g2nV5lzvF)xU$x%Vmt`@Cp(yEs7n0LNZireNw})A#Fx-WVHY z@6vTGR7wV2-CoQR$W&fDbG6uU;xSS^9*2@30S)8}vv7jk- z@eZ#`D2p;kewiseW18r)$0t=8=9WKwm2NM6wOEnmcjJWXVNaXFgLR&@SkaWD`fxx> zM#h@#dEL?$x>J>~84vYsZne_lg<<+PHI8N-N*`w%-PThSkNBy*8|uRZm%s$OpsAWB z)kTjKGJzuw>E{`HDx?tw>0|i>8*3%bom};{A5Fm&IJI;_8MWaQJSM^!iHa1Z=Df9a za$9jb1eDr%{lk6~mz#-^6f2qLu`3{Jxdf1+1f}HhO>unhDvJH8sI?L3u|3)3E+OJ8ZisWAUG9NmV(_j%APa7dkTb?;pNm)-&aO{z$ z5l1_w>$^gWQ|q)D*FC6`NR6{LSx<7q{N=?Q=RMCUN(uP#f^$d#E`)jfcgK_0IgkC1 z6frz8!Yk0NMV`L^nB%P2G8)!CLbkFS|FpTWcAd3()UIqph~)dShb7;W?e?wUHh91b z;$9&P=uwTpEZ;Oix$0b7+i`UQp(<)o9hTsUx=sPC9c0MqnyS$My%vh-Zc~$$%Z+5l zgwVYZ<#zsaXDR{!$-`3Sr>#m${A7UG;E*YFqv|9h?7pb#Yy-O-U2o90bov2SdX*2h zZmEVPO250Sz>49%m$w-gGd2}Pb)16{mfEBs#08&Y8Q{t!+h=S@UzOF3Qn|re`*Kxsc^TG`>=we;w%z+W6{RFP*o9k?WvfwB5ANa99eCEgF0DpOT|#%`H= zIRS3El8qy>HfMM*=={`ZfxG_pyZ6Wq^)RtX_VVdI%QnqWV6#-agd+?bTu)ulvG#~s{-1TT|VSxvfAF6$7ptTND<(eZ8YCfx@~5J@{j^!Ga-tx z)1|uivbMVPoR{_QJH=0CxjS@%bxV{Wm+LK@#zp(qZzr}jCi-Kol84Lt-l}GM^bj}l*!u9h7P}8QXTn&# zVU1eReNfe5C13miO3e&9xR948Koia_Avh@a@9e`fStA{F4)dFZGM@Oq3GNkwPExLk z1l8;QQL>k&m%A{+WPg_*nWHGeQhCUsjV}`Vt7!q^Wn-mKl^0$u0J-~Ae&hxxFO886 z4TWyshz*W}*spMdWE)59IHSpt54c$vjcm3qa&Hr*vt3%x$T0YoI&GkngSqRrCv3*f zo7TooSj{C@Rpp84bHOo!GPkZ{huHaZvT_5NiTUkZA9WD?NyU+1ABzqD$XuN53RB<0 zwQdxYKS>Y!O@=lGBDg}mmyTs3#knu0l4~x&BHhB)1I@=qk|im)+ib-k5Jb*;04L_q zAEO*pV4AY4BOz4)qSrsRBdKP7B8X}3N&rm)ZtmdA~ z6=h{=bSj}n_S)b>x5MZ4Ii3BwfXV%c8Ho}L+yd)6g!eenG8ERgyFdP2)T(`Q=w6ab z`UY;{vn>F)`91hgM#ORBb@o%ukKHRGU{iK!FxjliV!?`mZ}liFNDpc1H0FfH7;v|- zEa!K6?6j~*!?%W9&1&}f@0I~0_B9E{A`nl>gZpf7_2UZw_IJHov>V1X&fWTgtgpz^ z?tLg&W&r&lDiJN9H#W=#$X@5Gz?;)KcUCsPvv$(?6x_PeD$j##VL%1& zggsq1H=xj>xdQ)royXroefk$H!Q5;&EtvvS2}dJW!}v2ZZJ}qDu;#?HZnF{lZ=_(y zTD3Dg7ntFO+=NYg@vWIG!GdohMA<6glQ!F5r~#K-T$gpo*PVdeI?f??ezDG_ z8XV(QB{k`|mpM624G}W!Ev%LcTKw6!We*znm}ppuaFlonBBPhkz-5rTJqTa;*3WQF zBcFa+?(t&bbR1s#;Mc2CW(@Cp5E@yQMl=B=Qmi+DdIPfVAI{5-#_}Km|n2ZlD zA2sKu_rQ}&9WgK$-ENs&8H|sdy)an>`^+4uqxUR2LI5mru>o#4X-Q`zoQ7*64&fDO zyWRCx8Em+K;*SxP!ULhsNP4)t$-+r?*$~x2*lG^!*#`E`S03-fu__SO{6+P7!m-V3 z5QHy9f0>1QR$U)p*F<9HXxV;d1rW%5Ebrt7)-MYv;;|8R-vp9gxrj(yBUp^e&5QL%S9zxbwX+QlExU;OAvB?YAE8 z8|rtLt@jT+7t#vu;Av*iz`Kx0ZN^rRvFzc#M>Xl?uY_$4#6%g%jJjAysc5Xx+b?M} zGQE=dE(}$nE@LCKD63VH@@e3*#3>u0l**U(@RP~r=1wPv73IaairJ$@a&VQPF{&2{ z6I94N_?%e&g%nS3kOJ%Jd7(`+qR#b0jX?t6iZQP1vj^pX?u;7)6TT3;0 z*or*WtqUFj2wpRMjNRtEz7fH5x{twXh?E1GrhXEwT14>G!#EoZ<`QTI1U#8(c zTZZ4E!w_71Mfzt z=Cbgyou!XD#@6P1G7)cBsxF&8lAt)JP7<8|PY& zzl7jPZX`xhtFCm`X}u_b!dZ{_+FqbmuAosGs|hXk z^_AGSTaM6~gd!cdwm!pO$w<=Gd=T0QPvv z5DV6^vu()N*NT?H<6E5QH*bOA#`f#q7j6@>+JsB%aF(2ylKka<9WA?2Ec&w=*-L0v zOV=TP#6+Tn+Y3egHn1i!HLm!6H{b|65HV(%~%GbD8VU8~e7x+_a)(>9hWsCDBh5WeSYRXLC?5Wtf&H`LmXu%uT`84F=wDHZSK zlrv$v&zOFzx`jW)uDjb^N|@0U$*m?{Ht~RA!9i8H(0fZ2&Z?*3A>W}h_t_)5O7|0G zb!v%tE&N4mch0!{*=)%ft1F6ApuSP4V7Z@k-gWPedTNM*&3PS8iqGwsK;Bh2BCVBE zbMMgVGz-y?Ra2ow!3awH_@P#aMna^z%!3`Vf1(&fKY!^64w!8q3fJ!>Vl?FLw2T4i z7LzN9b)@`gxUWCHmk88Wf8=Mx*M0*&n8HeDK zDw^7-I6^nzuXa+(LcC>7y{vsmgnZCy)io8Mj2RRU(f7*ltifz}A*xT_RCv1(nAqeo9b)DNF>LH*O{Q>!7% zWVn~)ftcyA%%^bOhh53-HIVORgDd?ojYkRFI0GBb@af6-WL40U0EhnIJ> z2qKlrV)jRr{;_hMIQZSMK&3He=^_4A%M-6Prf7EhTmSlrsA}(+AvCcf-LuSQI-2gn&w|FkV--%8A^)-p|-o z>3cf1=@Tz-(uwMM$BDER$7LEynVoX3nS(E#{68tg`JD3SEM`~e@zPo!%w4!_aA^+~Ov zI`|N;aCrKA7GN>TA2S;XSuT|!{usXMeXE^&vE3u65bpEt8JyU%@?G((vJoF+7w)gq zZ@U~NK%bcOvUa%DEk)a;8nn%RDiJWDmzPYT0|Zg0z~!yENiJ3qDpX%leNP?rly|ix z-W&o=Vl%p^SF8HZp<6vK-GQXQzTwpvX5wo%T!uc(=8TUa-x&<_QXeYbx5DKQAUV}1 zGj1etZF|!l_~MjJnth80hfX&4L(S_Efajz~-^^-n5Hh4eO#Cb>Wj^5mFmYps^u6-D zgHP{U5;STvQj`AJo)+@E+;ARSCLA5BlI;mklRTSz;@9&xy}x-BK75kmv*RINXOgA&N*{U(aQPuGf-AYpZ_wBgl!82l0Q4paR?H$OoV?*bkwO& zJL{ZheJ$3Uv-s8`68uUJal>&Cn*AP6y^8T^vEuhragXh)(grFB+u zfdu!YNQiwic_q<1ry?FYAZgpySYf6kzqY2Xe7M#nQ!n3?=<6p-3R90TOMZ$VIrR8h zPMcsSwcu1JfpAsmsrg!PlH}{pfv+A?ekTblg>%#Bp8e@(LLP?I{-cgO95oNHQ-w8 ziXnx4!M@#jYyacMaVLh;N#j2H7F$FwP1*lXv=l|EaJL<^oaR!5+(WBg7MYqJph18} zP^KfF&>;(f&S`-4zpE>P6r9N88bnLQTON1|Ei8%$k+{$*B1mYNg6>A^1Z5Emb=_=_ zlaF3`nGNTQ2m1@$GSrFo7)%L!gB^--i@%FZ=l9;~QAY6!>%K5hnCLJQE&XMk6Nwfm zDRSk5<=QA znw|9(u_0puV74IEc5AOdl;d(;*Pe**Josc%z7p1!;KK3hc)-^v3o_yUPcg=Pfuf=G zkZ2#w$-v0*OZ9M?%uQf_q$U{XQ7*)2b zzA!!*Onp2Q7Y$$g;m=S%tIJpfaQFESF_T0R#ZpYx1WXyKwMN$`01`vr0}Bg87ajTG zFg!!7w>`B5n{tb;s0WB(s8tAmhJHPB4Lj99MLcEzV7CZC0rp`JB=1uU!vkJM8APsi<}moj-}! zOZ$=~+TA|Kk0D$3_?o=XB4J?{8arW(Y)}Z#-z$nCPsJ4Un2*S{A;V(rD~|bpUHaY( zlACB1(x;a`zkeA4S9=5F!M7b`B}uV)&-T5?Q&Da#%hj(WuzF-0UZ~CBtb3GLHz8sT zNF#JnZHBrLH5Ji)7C==FnVMkH9>{N2*OJh2X4Atd+STOdY1=AKR?muKRNi1nAN4+v z-5;ur7nUlKoj3}m9VJ!U*4MjeR7nfIzeuDc{;}!GP1LrLp|Xxt^D28M0Ik!lncEzQ z&Bxum?cPW|b8#m$_5p`2+umy$(&QShRFwU-Ocs;}tbS)=jr_OH)W4Fv<2Ie}2-vxC zJincV4_bg#r?-&C!uFn~H7S$N?%0zi8Q z4?=>Ogzf~S12adZP?jRveJ9P6kGREg1%ZN{281mwTeC#9S?G&w;zG&SvoH_8hbJ!o zF|hoWqeG?E;#w(C-dQ9HsPbx*|6^ZlUsWyYjXq;Wubj`PR)Bl|E%nqjN0HL{c!KO- zX83BhDV>u3yn_->EI*B+(CP>elPvj%2eG_VGTsyAOXPpm+D58i68V)FRu_4AbY0@SKeRX6;z|edhPr`*awngp z11Oqr<41$*0Gm#LZ*0JXi2lI-E`ewf_y3J z6Zy+w6?OMGV_vkCIBuxiQu6vL<>6+<2;Pptk^LfFA+2c5OO#__dg9?@`L53o`Lfe? zXG!{B4=R^yIlE?^!$3RScEnl-jd13gn(W#im`+%?+~6JOH<&$#5bT37q*(@M3HG=w zrK3?KO+VsyrQP2TV3xMB`#8Z!?s3)z2fHn_EC-@4HkzWYeJ>?%`GL>=(hV5E$1xb_ zXTy5uX;H%Wj{?T+4G??#xN8(`o{VO3Rb_a zWuy9N6gjZvn5u`=jq7KyMg2!&Z z2KEk7`FlqCh{|Kqc>vosFdVT7efyn=H@XnPg&{T=riRw~fZ@&AD)QpvG<3fQnJRg3 z{TsZ&DtW0Sj=Xj5pe-gZ!NZnfc+@nty6iZ*3^j6)ZKS%mO@htnZ*zRp8{YW*8D$K* z7A!=b4a}NrIA14=LD6>+z7Ji?R$rgf_N~aBQ;)i$1r4HNQ}av7#R*wmNN1>gO_5^L zO##xNfE=rR)T4kzxt5w>g0+Fs1S<;d=Rt~XH2*XM8iftM*l%X$Bbm&dP75FAi|Fp? zLOfBeMR;pNc}Mv(+Fs&b?j`g*Wk8cM82u66*32D$CYZK=ZiPl{@gAL(;clxEOHfl- zYu#+U{$|u+gum|p&L=`XK|3;2-EzgTPbe2TDO&;Z`1f|gezjJl$BdbLAL}VW?&YK^ zg{Pc#1usjwcQt5v==4J_9RnjSZXNwI@!WYhE)hBL^*>u9QG3>duOxR0WVd1Q``n(4 zhsgE)Z`4zL>WsTCj&l0~j{%OkuuK3LGB=1w+UGh`)8R6tN;x;$2?+_yQ7$-=?cQpz z`y?kZ64Zi`uc5E=vuom0F|Q$wwpt#oJxp+IdHViE_}7vcLxFak`3KZYyCjQk|)2;5LhAE-Q1(oe`)nxZK&fz+iV(@^OVMC3QrsW8EBbf`TInp^#Y2y4g3qLZHnl*+aJmJX9Q za!>(Vwf-a1Oq2Iz&M>yE^B**7{vnqJw)Twhjk1v1TJ{>JDf`Bw%Xyb87w%;*)H_^t zGC<^vJ#J$}e49DlvBNDuMEP7#hr$Q0Isy~;kv|#JSPJ6sFc51)D{KeFyd>d8DA)_g zdnC}%)!f>xWwe)Kv$kQWs?5_&m-jLOY&zse7_?+%nuBrDAq+;GgzoG)7c7d-!8AHQ zyk!{i{*hzO3Q1Xq0!0f0i5J*a3m%K1z(nlZ+}5*{Krn{Kx<95Jls;O;>%5<5gxb6k zo&g1J;L2U*#b3erEY=g+x2NHY^f_4N@<9?nkB=fB#sd_tr0?buaK|N0X7^*SVjqo+ zCFj&@5tHtVc8>fo#zxQ*#Ph9rxWtR1tpe2&@MY=O^P6?&SUb#k!5Cfdx^BhPn-q3c zMT6wfa@{S=rKBi3F7)L}%yCtS=h|)#j~BoA7HZF;oLCisk|8p3)_JqG(e#mGZo+SW zQdxDJYol>rKx6!=;wA)kt6)W9zGEbt zIZsBbY;)pX7>83*RFaXM+Zbe6WvAH$5*CgmOpRr6z%&jw{C&hy?*~@MqWVE3{j+pX zSK%ZX<^eZF+jSj*scf{J@S0a|n zTR%YANvQW%$H>F^8apWClmnPcKUUfqI*94J%+Mjz=m~yoz%#9XQ*E)jUq8ZzRtRZX zPqPsn=5MX}`0XW|pthW&_PVF)!~?rRB7Ghywv^$x5Q$rM<57zQ+t~uteLrzs`GKr< zv#0ks4gL{NbmsOvSQjuG@teJs828g(6@#c2B`QX*cW@>AY@p?Cyyn+M06Ma3UdKbN zwB;+rO!-Nm74%!~sG5jGnO>#vzZrh_4)(=BIgj5P9OOrtL#o@6^3lBYv1XZUew$)qQjkXc!j%LM3tN}F4LdHE`g!nzv+I47tJy6&{mX|e zYi6Aq*d$L|OA7uR@8i%W5I4O*Jx-07LI~_l{WkA?YXu0A=eINTtxn0FrxCTMi^)JV zd_9M=?={G)8ZLbEF55xm<%ppQp@%AT*b!eh3_;xQ1;1;Fsj|wJJ3<*_Xrnt>_1v8N zl>X#VrYD#C4$88O>3VSNUrygBm($k$!^nV}J^6&!)IR4)lI@EtKxQOeBbZwU4-WtH!sLL{) zh2of()2P+`*V|QEPNbl2^ZcHrG+L&PSIh54&R|A&L{|-S)mc8DSX7JBJ5^#(J-Gj= zs?>N?FrFPpqT1JM6J}TuJb%Dl1!P-MUgHrQ`{Mu8V9MS$K$ZQ1(tyM=;Qq{N`f!!$ zuVNJ2Uk0$i>LZ|-0rqYw?YEI!6%KXI#F3TKF7qH=tA=DhUSms(-fSaUdZyigkV`l$ zdk@tv(k4NiVN)>ral*@&Usd3T?m*E@Y=-}{6-({K)mt<8238`n0CyKzZ0k1r*gq0Gjxf@RS_+FWo7DUhIrcEN!;1cqf~Ffi^sxgx!opXlkwP7vIL zDUHBnjYOdnc1q1iSAIrI#{@N);+5zY?E9` zyFWC?sTJ~&@r7g+H$v5l?m&SNKHImC58o%63qC3l3}x26-?(SgQuXi|qbs0WP!WD+ zJNo+S`J$5#wXIQpJW-t80nW{?|21`mzqGFA-Z2ksELK(n#dTeORJ!4S4sRX#WTG1}v*K(|0TD`&3WX4r=ma5&w2SvsJrvNF+uNoSuH|`F2Oz;+d(>PVp#2C#f z2`5oY1@mRB6&Zy+`F@_rcu8kfxDpkJ|MP{f^?9M#ic-V%|V>4>|V3JJ&}5fM5T5?&rA0nq%jA7u-mVY7|%#Lg@E8 z!(aACyK>M-NLH0Q`Irlz35&1;eOP54Ily0jbARq0cM=OC1>;Xwd`0gX_Td`gjQLae zo5BesAQxCkI!U9cT%n=yTK%CL!t3fRyrz=olGUd82;dp=0eiUM?OW_+_wXFDodHj2 zY0%6n!RwC@qjp)!mhH!-RH#44uQRw`D7n~*hK=7WYsJIn2P|zS^A}zKj~+!LCz_$= z8y>GUA^c*E^^$ydYPs9!q4K;^Wm*l|aUvtB>pNebs6Er8trm*Vg6u>CS&ZXZb(^$e zbs80*?{DZhZy_Y|>s=njzyvF3{R~N+yKT#gGT)ujV5PR;5M`uQNc|Thj}$HJHlDRg zm^kR&Sy=BJE^VCfVKB9x4S^X^TO9jT9QkgS+&Deh=|(A=@UTullYQp06QBI}(CQy# zC->j);*t9s_Tr^XoG3u0+L@`c`;M@q#G#Vqt_wk1&kU60vSa77?5< zPdpaQS-H|di~k`V2NS1jzu@T2kY;gq2$o>#DL^lg*;ftbaUtxS+j}9Kt7?UDZQk-U zM4rIX1|51OIG3hbxAYrqFuZhl^pEoAROi7H-9S0HGnu0>Tp4MvjI<#*+$Kjkl%!Tu4VrnMqjxcYVi zNe_>#s@}z%F|JoL*PY*!co65}e`#KSgd63bn=~|JMz|<%YO@36U8tQT#QpAH21<{$ zv?wld+HpBg@`gyLkK#xOiekn4aq`5AS`|`dIpg!=S@r32!DhG+Qr4kI`$cP$aQVB< zM>r!Tsl7$AWYD*4TxGW48RDN!G!eC>pUm+>IaOknhnwGwYeG%a!uK40q$LSDr8FA+ zv_}{k@zPFBq6dHRc)Zi+#!Sqi{z~(9eGXygF;1+A7Q<;KR4q4Gwp&Ed? z^F!>$j@B}ThGg1BIF-1^sS*;oGvzRQ=u_gfdCCK)$u0HH_^HzJ!;LhUF);E7pBE7n zUb)SLCub6l`MK@~XX7Ay$+28ZapKiTX;pw9mjI^^m z4&DOGY%*LV!&S532t>Br!fBY? z<>pV&BCwhdiH{Z6;SAxIo!4cieA)zmx<>~VWg#?Np|z{h?umAuX0aI3j}+JX>DPpW zXS#d1Ew$-xbfj@;(N$$!XSL)0gYj)_z!?8G!tZi6?4Wk|h?CkgnS94@o7apLe|n0l zm*n+0=V10KogMH0Fs!zIOx>IMuD`a2@RtE8Wehdo9);Y)0=Uh*D;M4_&ODj<13=LX zO1Div^aW)vWOj`XWPUL@eBdrKNT(LSvfW$h_~ByY$46dAIc8%PjEnOj*oF6t+@TNQ z36+q5k*H<$dCyZ7Zx~jE>hhU9{W=x~FrNp%P^a69ZN!a6%Zp%X7QoKnOp6fmJt}bJ zDs{?$_lKJXn{i}a4l_0VHhwg|ZW>^-jgncu|K3cxJq#Nl&^En7@q%!n6K7+JDM~<` zwc+@S{$pI7gd+}y zufh++Iw&K^E|KA{O&`H=zLdK+?v4Agf;kUI+B7SZ;isksyouopL14R_UP_1at6h%1 z68luk_+OlD&m;g`o*iI!$}Vbxy2MWjj?u~}6jV>#6uI7{!0HVUX=`QLp$^)g(GwQ0 z20QBz=^}pIJKt+FNzeG^-dV9|s%Gq==JYE1XPZry$Jreje7_G?Gs&@{psefgab>4U z7#C39Amf;k>z3fMNh(mAc$VPod$Q%^?R>hGMSPr>4xbjUnA!|FnuV*i&PKewantPQ zRqKqTWM*8RL@u@&l%=_vVb6I84=E;xGGtUY2sEoXTW~#5%r+kbzq^+=IpDnq^*;8UB~vr8{zQGJ<%E_m;smce2; zv|7U(8SX!ZN1%ZFuVXTZnz=}L98OG zpp!rYQEJ}GWvm^cM2I6)B=ak7__)AJMV&QP1@LMlpZugcj`SCC+KHrC3O;0jGrj2g z-N*hK;zH+87|!e&emZe&#ImOp{Cqk;HuV>6pIlpmR?#3v!ubg=4>l9VxinV1Lfoa6 zftKR(J$+%kwJMz1peBf=b36=;t&VP(^Pg4E>g09ScE2RF79-09XA7?vqmXNlI{Y4X zls3C4$)9-dI7M}ZK;n|iQX)yb7o@TxiSlig_d=f-%xgZFJD@qOdH+Hy1=1i>tAQJM zY(4(<1wx5KTIF~J>kvQ=%N%oF;*|&u^4e6aI6hWK63xAIiH1(gr`AR*3Ax18B!;URJ z1JQ2l6R(E0v7oHaHY$HRm2hsm+B_0(cqgku`_q2Z2xhlkvozr@T>sYaAecr2FA}za zkLTZI)Ao0jiN{rfe1m8zxD~&1>flq&KKcuBIJn?na27K}29g=acQQ`-F05&x#0UNn zhM5^DAc{<;Sl;!eG@#;*Cg;$vkc3|Sr^H2=PXn_6jmXCkWysjf40XQFV8+TTQ%$}~ zdDE_adA`K0@WR-@3>{jLSO(`}-J|iV2ZXg4o@b;o%xN)}D*!W^arC>?1Bx$r!${$S zkh0i6;D$H;i$t@bT3gMoXTGC3m}XWp*!ZEKWT*ulFKt8EO~{VUpx`jd@FJ_*Mfqw1 zs*x1uUWMc`YnQ)Bxl!s^jm2!h!yE$F{-p}PTwpIpg8qb;$NMA6lsDeAcT&HQUnEr1Qh8_Is{1QC@m;b zq=hP=ROuxkAVrWKL3;0<1OlY+1z+Et-I<-8o$ue~@5z~*bDo_0d9M2^&wV+P9pbiv z80RAt_XB8MnLkC@xBB1;6&s>a?R@W`vZ3!f5{NuVw>TrdC^AEnIk0i}g(Xp-ad5xT zVo}q^FK&zfRrY;AsX8|FI%S}W^%l75593SR-F^ zVbi!PE8iR|I8xcX+8J{Pw<#I3A1`$-M>M&z$ZvHHQKdTng4ZAagFouJoAR3Kwut}t z-H&8mcgBwJy;qD4h#ETwSkK#w4P5ruq|b)H#og84zD7WFV@b_lXG7%hA!?>`1^uNB z9^S~|vUQ?Bg)^RL;%p#F5ADlZyH|-SjjJizfZSIf--aBVe&fAQ9N9n?*1B`aVqK5; zNKF|q1FYYY1;Sz64|42n?;cp#tC{(II`wb}o&F^Pb@fneSlv}Oo0NamDVhS>{!Ao~ zs~5hETP~>Sc;yy4&p+om2mOir@%z^qgSFhQHt?8-j!md%4c5@7@k+OXK-7A>vyQgL zL}`PM`3^v4Qn)RdGUXd|vNpM2+_J51GtaWWY7VJR(4g-$Ibgqhk^k_DiGFzrsWn;+ zb$a&zC5}vXfNCodDb@`D+8+MpobrtrKhRq01-ZS|%R#}vS|!M*TetpPCJGC$3O+%} zFS(nh04IV3E3VMYL(3>jr|t-q=Us+;>c1z>0vIyFF{sD^q`uklT4CmeF}_Ds(J1OqNJq<*=a3C zTq^N-_F?;zO`vH7ej|SJ;zHH+ha+yE868*c01GW>zsOMq<|uH~Ez&~pmq(A)re&wy z^`|e{-Z`Y(TxM3I*XF*Z_Ai_2EuoU5Imd{=9!?ktL!NE8A{og->D+?q83vF7ab1X>O?#26%-ru>Ajt{ zIP1>e)mkk^Uw!vqhJ(h2-)(n195I#4{#P40J+bu1^`n~KpZl0-of63}gBv01;ohP< zV>*d#8AKI)${(y}_F+`l`Rl6Fm#@~^;lrnlkBNfCXR^W*x19YS7x*KEz3N@VS8g`w zVl}OI?d2FlQ^)SgGv8Uxq|3MWa+KbcrS4g=0%u!-m@LHVixn>Piz<3boy!HKBfcb)aZyKzzL!>(_R z#0Qvq-U53L`br)aq#I$LS`fA%`R2%4Gl|GQzOYl5MFRt+=IcEgJhntBsZw)(Z6c(m z-Qt7jG8hqkT{p__>Q|bVHTfbqk>!peC0~u- zmm;64*CQH-xFR_I0=gmKi@!AP)s3nDK%ASI6@Ne-f=1TQB4J*ZljWStG`52Wz3z-X zy7jKpqy!F2N<*16oGLXNut}*l$39#+9g=W`=dDBQHohKq_tU8(pJX`(DGq$u8w*J8L+_6Rz zfpfapsl!p{A7yKY)Z*bu#}Sc-?)mhI7sa-Ps*9eDW3w`zoui?UxO9&33Sj3LPjMmE zH&$9}3hAo3aSzZ(>GPLCeKB=8MJbLgOHM-9MsQKnTgo%{X1M{!rOHAzAbL1+<*M{7 zGJ^eZK&;-;4LK%SKv^$^Ub1IyO$msORx%Q!7>`)~2Qd0B>a$L~7>kUt&mev3uF32+ ztViT4PCn_%JWaNoQIj^vp;yGT-tP-JvKa1Yr7u`ynu5Z-=l3iIBN#}Cn=_vaSq(;H zbVXYqM)Q^$iz4TX3)>NHlULYD5}NERINpJ#yxEs~+i4GGF92`?E%R$L8iwVi?dF|F z)@;EBs~Y+eGe-qKz70zs4%}mF!Q03uts~})NF_FYv;W>HD z>}0s~N_A0FJWh|<%9k67OK=#x0V=3=omT-> z!DZ3S$~3z{nBoA56Fs>4>SM;h3%Q1tT;JaIq`VBueN3-%TR%(5WH@5w@Z*8>yIf+_ z!2?KXOU(*s#cVVZ*0rHHKWXPE8gl35-B5Lgnq0E!@y+r|Q}Gt7kLj!$Q^eP1HZ3&g zj#)*c-%a;^*{nP7=spiUdxaaK90(VD-4>-_4WhL&wzBx`{oT=T3zYU>0~fGTa%!x# zVnAKIT{8Yii!E9{ja4UApx^L~6us3hJcn*WL-NaJH3n^mGV4#&CMqrBR< zpcCCoX|hAJGjj5b3n5l%K`(GJ`PpDGE!fMe&t0LKf=k2Th?KR8Z{vNB@kTx0@Y{*< zg<7@Hv1q#O_6sfGSu0-WA9fMWHjK|)Cu3KP-RA$}T30KG)0e+~OA}T9w^e~@Vx9tjnUfnv@A7*?XUNvsK2g+y zl!*8vCK8Y;u5Obr%VeF2{Ca=fxAI#=j=dsISW=<*z3 zB&3IRZP%EtIAFSLhbpO;y|O3kSc0M8A~puQ=kHv3%R^QkzUPY1~b*gSXu>ORj(XP}`K%{}`g`0Jof7 zt{((=31!{n4aMIVvah;whsOB7&mhX!K_g*1yV} z#fGSIro_OKQGDXBD-C$ZQF*ro{hv|}LydEnHJq$gYDF!1E0`;n2bpM#bQLQYQA>t_ zSIISmuGR834=@s2a_7b~)l78k+ar&Es7%{`0OSJtUf#J9xb(>h($Y^5SH!|5(W8{y z?cDVUK6sZc6*{pU|@s2mDpZ_eF@Dgd@-Arg4+heA?|!C!IgMar{Ys;JW&ZKtrVD zjMQ)VqIeTA|7NUH68!bm@A4NGxo?v}U~PQ7l=c|dTCXGTTE%o!o?}h&RmHHFiVJ7{w08GI zB4==*-<#Z$1@5n3k3Z^ZS!w0_VZsJE%RE}h%8>H*dad03o6ok+U!;>O9xCapOFBhg z+AN?pLj;e+Z5sYkol-o1^~LvJoePeolD7UA>f-+abN*k>NL~c-F;zHP?k{w>((eTAMGi!bwe(PSmKx4pb;dOgbW;-X`PN2GQ2bw3 z`Tf-3IzNv5od&pw`NUNq;s+F4*{^>)9v@xPe+beI{fXUulib7|;(wle0#s9T>`swL z-2%5crx7hOe64?Re$3ex&7Tgy*Z+t=0kUEk2&cJc9Sv5{Rtdtfuu`)S;MEpKo&CMN z!xqFv>jXfk#PpR_?C8_F$UO<@2{i$?3dk)eAdhOJr>^K#GnbHj`JG}a0M&Q=(^g3k zfRL?=t!%ZF2 z$Bi3YG+$eTCito~a%%e=gChpn8iz}*6&7Xt$#S!KWg3+b6WB)iF|5n`ZTGd5p} zSRn1#*9mt@?Be_AO2zxgq3dHZZ2g0?ACB4y+b0O?S0xTChV>foye0+Qsy8r|!9Hb?kv$VSUYNO9?lHnHk+r{jS zN+ECuUDpcq!wCP584TYt6_Hc1fUi8<3-~>OJ?!lh(q|=5J#(3*T_rz2(W|E_WVQ)b ziE}d0v6OM20{fjOYl>JS_0f^WX=!Qut>nCgBc4Ou{rx+K4)*p2tJnagCfq@93IM9) z?%`no0@2k@26mhUZCX+aNnGc}1CvA72)PFnRrZEu-ns(42@nz2-5G>Q-Rta$q_Hv! z*erg>$KA!r^A>L9Zp%AE6?8o$orVmK-?H5&4*tSQnrjmLvc^t3vdB=%|Hw%$vF`ks zovUlIVXDR1>DciPJ;)Vr-DCLK3Gd<5!G0HSuio#(PU4Nz|28_6PR9;gnLuH<2ZyG` z=111(DyObb(i1nQ`&r)|brj8)m6a-&q!^cPv;V|SE*1uUR>tFk+gOM02gr{?&lHZ+ zYvireG#@^s+G#@RR#jL3(34s1&uHg8JJx#i@wP_Br`et2($a`&7kYnd8=KDvYR?=! z9UV1i=Mr!*{;0O41Y{1uPR@Sbl=d#cuj@hECS6;&F*;P$%g{3nwCnNR>9EhAMXJT1 zn`X05{C~vp0>qPF2A(vwU>nPW;(Ks_GZf$#k2cjMFVD;SkQsiAX*1lx z>%A zBGwO{hzi1HPQ~C`KA@EBS_yry@XOV3jnU)$REIKFd6QwEYo(Yo&zAT4TfkwEesCKy>3PN6DCM zN^-A(_m9v^Z3#E^J|n`H&L5H6+93aJ|6-Hd6B@U&f-MqK)|Kovrao zHUOmQG&7EJBQ`&BIf^)c2KEyi@ms^j9P)(R&YC-RLJ>2aMPkW(sbgpRNn+H@D?6d zGzS)gOkoB#E8ZVK2Bb|;CRB8;=E3C1%zpSB#s@+@Yf)UHGieUCWd(^AWDZ_3sw2?( zgm0k&XU$+CW&q`rMKGkxs%Y!_w&Qdbt?vUTZ1oXfae6n})BSGufJTrnj1h2VOia!D z2*lAueTu?M41ujZdZFQrFs1O_-pm=(w6n4aM>EFmXraae_{BaRyYcG@?*z5l9IDB5 zzo8#z4%Xt-2k!=)R$vSZbV+sR!gL8RrPIRnsJKxN`HbU!pXSg{pKjcF6L9#;7)xWP z8OxCmx__=uIWq6hP%!SN)Uu!To^-aev-`%{yhqBV6hKUq&e$M@)p&Gq`^=Tp&OAXq zR(sBT8Hq#|q6)hb#ARh*N3kId=L7mwqS(~b)ORuVE;2K=`9MX{@B{4E7vnG3TUZ17EP(c5vZ)xT#g80y3CqG*#eG zb|7&=YAGYjEzpBMOO7>m3F{%o*zFjiG^|wxSw3Wfl;{CO^51VYLpdn*=RH0el*mKQ zrn*hDFRy1uqF=*yZG}pbEAtFSckK|`!Nz+$wG$cx3)Pv}azZuBz1gV<-jVowTso zyNb~Vnj4h)UK)e9B9X$|A&mjY>m!_d6H-z|E5?^6Y)X=42n}!Lj-qW1hOV_}6ABqd z%ib#M9rf@y2CzgTG^fDikKf{(D#-yB7Eu}eettg3wo-xLtdj$7=w%jXHm$$H6R^9B zGyc1@jY7lW5xS#jXEW$Jg2FQ=^NwQ@hyIg6HAQ#k%h#{O>%$fR#L~9lN?374&aiEC zWK&-*fy(JK_>KX6itG(RMXU?KEf=V-p{89K*crbV7Sq}+fcx1$i zOfqnriQ0#_LMpuYxH#LYAz5q!;U%aUqra)Yo5)ev`>V4Q?Zy<{XJa@i=$z0jnc8O) zfF9CvcVS3m4GbPasjVgwx4!+X!hZAlv5)(&>2oM?w8`k#bV$eJe{N;_~vS_FGpgBJ2Wc?K+WVGu~QMuozJa{Z> zbSpJB#v{==6H3mClA#|!bfLO@efjqbFsK7xi~ zGihn{eH;}(v{!rOa>C4p;@TO3a5Ja})!jHk(<$U&;m{Ygouu3#TGGliFBHlNxgk?(EHx@?F_`=y~V;)IxIPxl%sjHL*#A`Nw%janw_WzJXzb7@ajGzE;C<9@Ma-@y=BZe3mz_4 z&f)D32pWSN-)fd~yul-ay-=&vKq0zZe9>=JF9wyqf zo8>N0F$kHX%~Xs9-e1anxducH4HAp%HSr#klYcQ%yG6-#qg(G3raR2e&c2~fdw6?T zf7Mb|F#>zEO-=1&&z1z17V33$vhFxd5G{|>!YFk3Hmn$D8N3j)d^$@p0nw%N zcgJm-v{)%7(ba94K=oaj4_}4lxUM5ble!rmuMKh|3T8;CYzi^K%|j!7YYpA&gshoI zJ$h9joeA;7pF=IkXcd+1_t$lhoFl>ya`{xDYoA(#_HudnICgS4*X$pph!alLE@TU_ zWp5j}mhfDTNYcym#eRR8o4p76W$6NtohV06gd}50=#l25-apI8l5n^ zfDyLn9PX&fq`1&h&$C%TN|LxvJ!XZi(Kvv7*G|DK%`T3j&aPj|qAy?Pmbd{$k>d8! zES~XBq@<&mK@e+2qV7ft5e4UA55^Lf{e`{`d*665+T$#g5&XjEEUbBPb zw7>emGp$qQvD*LbXiZfe8jw6QLVJMRR6fbGpAZL{3$NvXD8UWy!FSM2gO4{(Nrgd5 zYd?N9G&Cp=FyZBRCrWwD!%B7h@@{8)p|@pZWg}?WwZwT{;U-Qpo{y}xoprSom(W4| z#Pu|FwM!9AQGd}+%RypUp&G1W;`T2R#kH(6{Nv{<2+WM(?mp80L_9w>yaBm*gN3RS zEhBMtn~9cwBw>X4^2VC(>ml^?-lbxljxVRzqt7Is^$!hGVgzIn;voswt7cr9 znqFUltPbxcU`cx5f$S~p7(HK8ghUyO=m_HX3EoykaH8c5BK(juu+DbeW{Y*w=xnKH z9{!@B#{E%Qi7$s4RDlOov;Azu9I$B9gbICPqz|*ilR4q58BwG5sL`ph(7@gFwh|k; zNBCUZio*vFf8*M3h9DR|9DO=_&?@M-M7VW)AA*U77pk~fY-;8J<~=q6Z#`#bxbEm< z@zQ2{6DJZ@eK3u2oTqeB9G1Rmg*o^^1B+8r&4Sz(Zt+@i@yH4*FygD#{Msev6Mag= z+NB3KJyNx9LQXvxPw@cH;5-nUs@}~fSy$eQgV3??2(9%@;RcGqauML8wO4v~qenVo zR@Uu@<>kT3L4*EBH3PsW{Qo4M%~(edDR1xjGD{;ABhPp*+* z>h()ym>NwN(NU#!#oM>!sUUW1CS~NRhuWBZ0~oX7lJc@Pv_EWTC{I@TWwkX`ClwVH z@&pMfp>$KHUwv?R_2C>)q*`POGU_52QPpy-hgOI*Z0WSL*Cy zT_%sAy@XUa)?ImgJS-;F>;ar79`Cgvgp`(w3q=V4UThuYWHOxg%?t#rNFOBpY1lcH z9{y?A&EEb*Ee`axjjNwer5lpcETny67Q;ycBK1_wVH%0OJJYS^xb-#XNPT`%icJe~5aX8uaTq=2d4on^)QnOnI?!)?Nrr z5}BF(L}gkqVC{tm-t?T0_tMVU#5iWWS=OHNI&Hk)SUdyOYML3HK^dNVflB#_`EhHP z!(xDCf4bn_wEOUk7k@4xG@$z!QCrPwHCEZ!F|Q?YepMW7xWC=mt*C{6by-mFh(BXC z{(r6$Zf>HyhB{eNAl@2LfrS{cYAFxeI~=oE&H5*An^OO$Rs}D6&3R^7dju^uRlei= zo1O2Ya)ew_e~0S7$?N8 dict: - data = {} - input_dim = self._event.args.get("Input Dims") - if input_dim: - data["Input Dims"] = input_dim - input_type = self._event.args.get("Input type") - if input_type: - data["Input type"] = input_type - return data - - @property - def data(self): - return {"Input Data": self.input_data, - "Host Self Duration(us)": round(self.host_self_dur, 2), - "Host Total Duration(us)": round(self.host_total_dur, 2), - "Device Self Duration(us)": round(self.device_self_dur, 2), - "Device Total Duration(us)": round(self.device_total_dur, 2)} - - @property - def info(self): - return {"id": self.node_id, - "node_type": self.MODULE_TYPE, - "data": self.data, - "upnode": self.parent_node.node_id if self.parent_node else "None", - "subnodes": [node.node_id for node in iter(self.child_nodes)]} - - @property - def is_root_node(self): - return self.node_id == Constant.NPU_ROOT_ID - - def update_child_nodes(self, node): - self._child_nodes.append(node) - - def update_kernel_total_list(self, kernel_list: list): - self._kernel_total_list.extend(kernel_list) diff --git a/profiler/module_visualization/graph_build/__init__.py b/profiler/module_visualization/graph_build/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/profiler/module_visualization/graph_build/fwd_module_node.py b/profiler/module_visualization/graph_build/fwd_module_node.py deleted file mode 100644 index 34d7ab829..000000000 --- a/profiler/module_visualization/graph_build/fwd_module_node.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from profiler.prof_common.base_node import BaseNode -from profiler.prof_common.trace_event_bean import TraceEventBean - - -class FwdModuleNode(BaseNode): - def __init__(self, event: TraceEventBean, parent_node=None): - super().__init__(event, parent_node) - self._bwd_op_list = [] - - @property - def bwd_op_list(self): - return self._bwd_op_list - - def update_bwd_op(self, bwd_op_list: list): - self._bwd_op_list.extend(bwd_op_list) diff --git a/profiler/module_visualization/graph_build/prof_graph_builder.py b/profiler/module_visualization/graph_build/prof_graph_builder.py deleted file mode 100644 index 83331b625..000000000 --- a/profiler/module_visualization/graph_build/prof_graph_builder.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from profiler.module_visualization.graph.prof_node import ProfNode -from profiler.module_visualization.graph_build.fwd_module_node import FwdModuleNode -from profiler.prof_common.tree_builder import TreeBuilder -from profiler.prof_common.trace_event_bean import TraceEventBean -from profiler.prof_common.constant import Constant -from profiler.module_visualization.prof_parse.prof_data_pre_process import ProfDataPreProcess - - -class ProfGraphBuilder: - def __init__(self, prof_data_path: str): - self._prof_data_path = prof_data_path - self._prof_data = {} - - @classmethod - def _create_event_bean_from_ops(cls, op_list: list, name: str) -> TraceEventBean: - min_start = min((op.start_time for op in iter(op_list))) - max_end = max((op.end_time for op in iter(op_list))) - # 以反向算子的区间作为反向module的区间范围，为了module包含算子，做了+1 +2处理 - return TraceEventBean({"ts": min_start - 1, "dur": float(max_end - min_start) + 2, "name": name}) - - @classmethod - def _trans_flow_to_dict(cls, flow_events: dict, end_events: list) -> dict: - end_event_dict = {} - for event in end_events: - end_event_dict[event.start_time] = event - result_data = {} - for flow in flow_events.values(): - start_point = flow.get("start") - end_point = flow.get("end") - if not start_point or not end_point: - continue - end_event = end_event_dict.get(end_point.start_time) - if end_event: - result_data.setdefault(start_point.start_time, []).append(end_event) - return result_data - - def build_graph(self): - self._prof_data = ProfDataPreProcess(self._prof_data_path).run() - all_data = [*self._prof_data.get(Constant.MODULE_EVENT, []), - *self.find_bwd_module(), - *self._prof_data.get(Constant.CPU_OP_EVENT, [])] - all_data.sort(key=lambda x: x.start_time) - name_dict = {} - for event in all_data: - order_id = name_dict.get(event.name, 0) - event.set_id(f"{event.name}_{order_id}") - name_dict[event.name] = order_id + 1 - root_node = TreeBuilder.build_tree(all_data, ProfNode, TraceEventBean({}, Constant.NPU_ROOT_ID)) - kernel_flow_dict = self._trans_flow_to_dict(self._prof_data.get(Constant.TORCH_TO_NPU_FLOW, {}), - self._prof_data.get(Constant.KERNEL_EVENT, [])) - for start_time, kernels in kernel_flow_dict.items(): - matched_node = root_node.binary_search(start_time) - while matched_node != Constant.INVALID_RETURN: - matched_node.update_kernel_total_list(kernels) - matched_node = matched_node.binary_search(start_time) - all_data = root_node.find_all_child_nodes() - all_data.append(root_node) - return all_data - - def find_bwd_module(self) -> list: - bwd_module_list = [] - fwdbwd_flow = self._prof_data.get(Constant.FWD_BWD_FLOW, {}) - module_list = self._prof_data.get(Constant.MODULE_EVENT, []) - cpu_op_list = self._prof_data.get(Constant.CPU_OP_EVENT, []) - if not fwdbwd_flow or not module_list or not cpu_op_list: - return bwd_module_list - fwd_tid = module_list[0].tid - bwd_tid = fwd_tid - for end_point in (flow.get("end") for flow in fwdbwd_flow.values()): - if end_point: - bwd_tid = end_point.tid - break - if fwd_tid == bwd_tid: - return bwd_module_list - # 将每一个反向包成一个module，名字叫“nn.Module: BACKWARD_0” - cpu_op_list.sort(key=lambda x: x.start_time) - pre_status = Constant.FWD_OR_OPT - bwd_op_list = [] - for op in cpu_op_list: - if op.tid == bwd_tid: - bwd_op_list.append(op) - pre_status = Constant.BACKWARD - elif pre_status == Constant.BACKWARD: - bwd_module_list.append(self._create_event_bean_from_ops(bwd_op_list, "nn.Module: BACKWARD")) - bwd_op_list.clear() - pre_status = Constant.FWD_OR_OPT - - # 通过连线匹配正向module，构建出反向的整体module关系 - root_node = TreeBuilder.build_tree(module_list, FwdModuleNode, TraceEventBean({})) - fwdbwd_flow_dict = self._trans_flow_to_dict(fwdbwd_flow, cpu_op_list) - for start_time, end_events in fwdbwd_flow_dict.items(): - matched_node = root_node.binary_search(start_time) - while matched_node != Constant.INVALID_RETURN: - matched_node.update_bwd_op(end_events) - matched_node = matched_node.binary_search(start_time) - all_nodes = root_node.find_all_child_nodes() - for module_node in all_nodes: - if module_node.bwd_op_list: - bwd_module_list.append( - self._create_event_bean_from_ops(module_node.bwd_op_list, f"{module_node.name} [BACKWARD]")) - return bwd_module_list diff --git a/profiler/module_visualization/prof_graph_export.py b/profiler/module_visualization/prof_graph_export.py deleted file mode 100644 index d336e97f7..000000000 --- a/profiler/module_visualization/prof_graph_export.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from datetime import datetime - -from profiler.prof_common.constant import Constant -from profiler.prof_common.file_reader import FileReader -from profiler.prof_common.path_manager import PathManager -from profiler.module_visualization.graph_build.prof_graph_builder import ProfGraphBuilder - - -class ProfGraphExport: - @staticmethod - def export_to_json(prof_data_path: str, output_path: str): - logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s") - try: - PathManager.input_path_common_check(prof_data_path) - PathManager.check_input_directory_path(output_path) - PathManager.make_dir_safety(output_path) - all_nodes = ProfGraphBuilder(prof_data_path).build_graph() - result_data = {"root": Constant.NPU_ROOT_ID, "node": {}} - for node in all_nodes: - result_data["node"][node.node_id] = node.info - file_name = "prof_graph_json_{}.vis".format(datetime.utcnow().strftime("%Y%m%d%H%M%S%f")[:-3]) - FileReader.write_json_file(output_path, result_data, file_name) - except RuntimeError as err: - logging.error(err) diff --git a/profiler/module_visualization/prof_parse/__init__.py b/profiler/module_visualization/prof_parse/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/profiler/module_visualization/prof_parse/prof_data_pre_process.py b/profiler/module_visualization/prof_parse/prof_data_pre_process.py deleted file mode 100644 index 9dc820e4c..000000000 --- a/profiler/module_visualization/prof_parse/prof_data_pre_process.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import os - -from profiler.prof_common.file_reader import FileReader -from profiler.prof_common.constant import Constant -from profiler.prof_common.trace_event_bean import TraceEventBean - - -class ProfDataPreProcess: - def __init__(self, prof_data_path: str): - self._prof_data_path = prof_data_path - self._trace_path = "" - self._kernel_pid = None - self._result_data = {Constant.CPU_OP_EVENT: [], Constant.MODULE_EVENT: [], Constant.KERNEL_EVENT: [], - Constant.TORCH_TO_NPU_FLOW: {}, Constant.FWD_BWD_FLOW: {}} - - def run(self) -> dict: - self._check_trace_path() - self._parse_trace_events() - self._check_result_data() - return self._result_data - - def _check_trace_path(self): - if os.path.isfile(self._prof_data_path): - (split_file_path, split_file_name) = os.path.split(self._prof_data_path) - (shot_name, extension) = os.path.splitext(split_file_name) - if extension != ".json": - msg = f"Invalid profiling path suffix: {self._prof_data_path}. " \ - f"You should input in a json file path, such as trace_view.json." - raise RuntimeError(msg) - self._trace_path = self._prof_data_path - return - ascend_output = os.path.join(self._prof_data_path, "ASCEND_PROFILER_OUTPUT") - profiler_output = ascend_output if os.path.isdir(ascend_output) else self._prof_data_path - json_path = os.path.join(profiler_output, "trace_view.json") - if not os.path.isfile(json_path): - msg = f"Invalid profiling path: {self._prof_data_path}. The data path should be the " \ - f"folder that ends with the ascend_pt collected by the Ascend PyTorch Profiler." - raise RuntimeError(msg) - self._trace_path = json_path - - def _parse_trace_events(self): - trace_data = FileReader.read_json_file(self._trace_path) - self._check_trace_data(trace_data) - iter_trace_data = iter(trace_data) - for event in iter_trace_data: - bean = TraceEventBean(event) - if bean.is_optimizer(): - self._result_data[Constant.MODULE_EVENT].append(bean) - elif bean.is_cpu_op(): - if not bean.is_step(): - self._result_data[Constant.CPU_OP_EVENT].append(bean) - elif bean.is_nn_module(): - self._result_data[Constant.MODULE_EVENT].append(bean) - elif bean.is_torch_to_npu(): - if bean.is_flow_start(): - self._result_data[Constant.TORCH_TO_NPU_FLOW].setdefault(bean.id, {})["start"] = bean - else: - self._result_data[Constant.TORCH_TO_NPU_FLOW].setdefault(bean.id, {})["end"] = bean - elif bean.is_fwd_bwd_flow(): - if bean.is_flow_start(): - self._result_data[Constant.FWD_BWD_FLOW].setdefault(bean.id, {})["start"] = bean - else: - self._result_data[Constant.FWD_BWD_FLOW].setdefault(bean.id, {})["end"] = bean - elif bean.is_kernel_event(self._kernel_pid): - self._result_data[Constant.KERNEL_EVENT].append(bean) - - def _check_trace_data(self, trace_data): - if not isinstance(trace_data, list): - msg = f"Invalid profiling data path, this feature only supports performance data " \ - f"collected by Ascend PyTorch Profiler." - raise RuntimeError(msg) - iter_trace_data = iter(trace_data) - for event in iter_trace_data: - bean = TraceEventBean(event) - if bean.is_npu_process(): - self._kernel_pid = bean.pid - break - if self._kernel_pid is None: - msg = f"There is no operator on the NPU side for this data, please check whether the NPU switch is enabled." - raise RuntimeError(msg) - - def _check_result_data(self): - if not self._result_data.get(Constant.CPU_OP_EVENT): - msg = f"This data does not have any aten operator, please make sure to enable the CPU switch." - raise RuntimeError(msg) - if not self._result_data.get(Constant.MODULE_EVENT): - msg = f"This data does not collect any modules, please make sure to turn on the with_stack switch." - raise RuntimeError(msg) -- Gitee From 5cc5e1603366d2d920bc714d46eafaa6261ccac0 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Tue, 30 Jul 2024 15:55:10 +0800 Subject: [PATCH 066/791] mindspore overflow check --- .../msprobe/core/common_config.py | 37 +++++----- .../msprobe/core/data_dump/data_collector.py | 29 +++++--- .../core/data_dump/data_processor/base.py | 45 ++++++------ .../core/data_dump/data_processor/factory.py | 12 ++-- .../data_processor/mindspore_processor.py | 69 +++++++++++++++++-- .../msprobe/mindspore/common/utils.py | 13 ++++ .../mindspore/debugger/debugger_config.py | 5 +- .../mindspore/debugger/precision_debugger.py | 2 +- .../msprobe/mindspore/ms_config.py | 30 ++++---- .../msprobe/mindspore/service.py | 19 ++--- .../test/mindspore_ut/test_ms_config.py | 4 +- 11 files changed, 181 insertions(+), 84 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index b4bf5cf28..d6c15e101 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -18,24 +18,27 @@ class CommonConfig: def _check_config(self): if self.task and self.task not in Const.TASK_LIST: - logger.error_log_with_exp( - "task is invalid, it should be one of {}".format(Const.TASK_LIST), MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("task is invalid, it should be one of {}".format(Const.TASK_LIST), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.rank is not None and not isinstance(self.rank, list): - logger.error_log_with_exp("rank is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("rank is invalid, it should be a list", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.step is not None and not isinstance(self.step, list): - logger.error_log_with_exp("step is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("step is invalid, it should be a list", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.level and self.level not in Const.LEVEL_LIST: - logger.error_log_with_exp( - "level is invalid, it should be one of {}".format(Const.LEVEL_LIST), MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("level is invalid, it should be one of {}".format(Const.LEVEL_LIST), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.seed is not None and not isinstance(self.seed, int): - logger.error_log_with_exp("seed is invalid, it should be an integer", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("seed is invalid, it should be an integer", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if not isinstance(self.is_deterministic, bool): - logger.error_log_with_exp( - "is_deterministic is invalid, it should be a boolean", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("is_deterministic is invalid, it should be a boolean", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if not isinstance(self.enable_dataloader, bool): - logger.error_log_with_exp( - "enable_dataloader is invalid, it should be a boolean", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) - + logger.error_log_with_exp("enable_dataloader is invalid, it should be a boolean", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + class BaseConfig: def __init__(self, json_config): @@ -50,9 +53,11 @@ class BaseConfig: def check_config(self): if self.scope is not None and not isinstance(self.scope, list): - logger.error_log_with_exp("scope is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("scope is invalid, it should be a list", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.list is not None and not isinstance(self.list, list): - logger.error_log_with_exp("list is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + logger.error_log_with_exp("list is invalid, it should be a list", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.data_mode is not None and not isinstance(self.data_mode, list): - logger.error_log_with_exp("data_mode is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) - + logger.error_log_with_exp("data_mode is invalid, it should be a list", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 800a2b81c..db437539a 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -1,7 +1,6 @@ - import os -from msprobe.core.data_dump.scope import build_scope, ListScope +from msprobe.core.data_dump.scope import build_scope, ListScope from msprobe.core.data_dump.json_writer import DataWriter from msprobe.core.common.log import logger from msprobe.core.common.const import Const @@ -21,7 +20,8 @@ class DataCollector: self.config = config self.data_writer = DataWriter() self.data_processor = DataProcessorFactory.create_processor(self.config, self.data_writer) - self.module_processor = DataProcessorFactory.get_module_processor(self.config.framework) if self.config.framework == Const.PT_FRAMEWORK else None + self.module_processor = DataProcessorFactory.get_module_processor(self.config.framework) \ + if self.config.framework == Const.PT_FRAMEWORK else None self.module_count = {} if self.config.task == Const.FREE_BENCHMARK: self.scope = build_scope(ListScope, self.config.scope, self.config.list) @@ -35,7 +35,7 @@ class DataCollector: @property def dump_file_path(self): return self.data_writer.dump_file_path - + @staticmethod def check_scope_and_pid(scope, name, pid): return (not scope or scope.check(name)) and pid == os.getpid() @@ -43,10 +43,10 @@ class DataCollector: @staticmethod def is_inplace(module): return getattr(module, "op_is_inplace", False) - + def if_return_forward_new_output(self): return self.data_processor.if_return_forward_new_output() - + def get_forward_new_output(self): return self.data_processor.get_forward_new_output() @@ -88,8 +88,11 @@ class DataCollector: else: data_info = self.data_processor.analyze_forward_inplace(name, module_input_output) if self.config.level == "L2": - return + return self.data_writer.update_stack(self.data_processor.analyze_api_call_stack(name)) + if self.data_processor.stop_run(): + self.handle_data(name, data_info, use_buffer=False) + raise Exception("[msprobe] exit") self.handle_data(name, data_info) def backward_data_collect(self, name, module, pid, module_input_output): @@ -98,6 +101,9 @@ class DataCollector: return data_info = self.data_processor.analyze_backward(name, module, module_input_output) + if self.data_processor.stop_run(): + self.handle_data(name, data_info, use_buffer=False) + raise Exception("[msprobe] exit") self.handle_data(name, data_info) def update_construct(self, name): @@ -105,12 +111,15 @@ class DataCollector: self.data_writer.update_construct({name: self.module_processor.api_parent_node}) self.data_writer.update_construct(self.module_processor.module_node) - def handle_data(self, name, data_info): + def handle_data(self, name, data_info, use_buffer=True): msg = f"msProbe is collecting data on {name}. " if data_info: msg = self.update_data(data_info, msg) logger.info(msg) - self.data_writer.flush_data_when_buffer_is_full() + if use_buffer: + self.data_writer.flush_data_when_buffer_is_full() + else: + self.write_json() def module_count_func(self, name, name_template): module_name = name.split(Const.SEP)[-3] @@ -135,6 +144,6 @@ class DataCollector: def update_dump_paths(self, *args): self.data_writer.update_dump_paths(*args) self.data_writer.initialize_json_file(task=self.config.task, level=self.config.level) - + def update_iter(self, current_iter): self.data_processor.update_iter(current_iter) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index a6858e8cb..cb7d31c60 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -35,11 +35,11 @@ class ModuleBackwardInputsOutputs: @property def grad_input_tuple(self): return convert_tuple(self.grad_input) - + @property def grad_output_tuple(self): - return convert_tuple(self.grad_output) - + return convert_tuple(self.grad_output) + class TensorStatInfo: def __init__(self, max_val=None, min_val=None, mean_val=None, norm_val=None): @@ -53,7 +53,7 @@ class BaseDataProcessor: _recursive_key_stack = [] special_type = (np.integer, np.floating, np.bool_, np.complexfloating, np.str_, np.byte, np.unicode_, bool, int, float, str, slice) - + def __init__(self, config, data_writer): self.data_writer = data_writer self.config = config @@ -65,11 +65,11 @@ class BaseDataProcessor: self.current_iter = 0 self._return_forward_new_output = False self._forward_new_output = None - + @property def data_path(self): return self.data_writer.dump_tensor_data_dir - + @staticmethod def analyze_api_call_stack(name): stack_str = [] @@ -87,7 +87,7 @@ class BaseDataProcessor: stack_str.append(stack_line) stack_info_struct = {name: stack_str} return stack_info_struct - + @staticmethod def _convert_numpy_to_builtin(arg): type_mapping = { @@ -103,15 +103,15 @@ class BaseDataProcessor: if isinstance(arg, numpy_type): return builtin_type(arg), type(arg).__name__ return arg, '' - + @staticmethod def _analyze_numpy(value, numpy_type): return {"type": numpy_type, "value": value} - + @classmethod def get_special_types(cls): return cls.special_type - + @classmethod def recursive_apply_transform(cls, args, transform): if isinstance(args, cls.get_special_types()): @@ -166,13 +166,14 @@ class BaseDataProcessor: return (Const.ALL in self.config.data_mode or forward_backward in self.config.data_mode or input_output in self.config.data_mode) - - def analyze_pre_forward(self, name, module,module_input_output: ModuleForwardInputsOutputs): + + def analyze_pre_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): pass - + def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): api_info_struct = {} - if self.is_dump_for_data_mode(Const.FORWARD, Const.INPUT): # check whether data_mode contains forward or input + # check whether data_mode contains forward or input + if self.is_dump_for_data_mode(Const.FORWARD, Const.INPUT): api_info_struct[name] = {} self.api_data_category = Const.INPUT args_info_list = self.analyze_element(module_input_output.args_tuple) @@ -181,13 +182,14 @@ class BaseDataProcessor: kwargs_info_list = self.analyze_element(module_input_output.kwargs) api_info_struct[name][Const.INPUT_KWARGS] = kwargs_info_list - if self.is_dump_for_data_mode(Const.FORWARD, Const.OUTPUT): # check whether data_mode contains forward or output + # check whether data_mode contains forward or output + if self.is_dump_for_data_mode(Const.FORWARD, Const.OUTPUT): api_info_struct[name] = api_info_struct.get(name, {}) self.api_data_category = Const.OUTPUT output_info_list = self.analyze_element(module_input_output.output_tuple) api_info_struct[name][Const.OUTPUT] = output_info_list return api_info_struct - + def analyze_pre_forward_inplace(self, name, module_input_output: ModuleForwardInputsOutputs): api_info_struct = {} if self.is_dump_for_data_mode(Const.FORWARD, Const.INPUT): @@ -199,7 +201,7 @@ class BaseDataProcessor: kwargs_info_list = self.analyze_element(module_input_output.kwargs) api_info_struct[name][Const.INPUT_KWARGS] = kwargs_info_list return api_info_struct - + def analyze_forward_inplace(self, name, module_input_output: ModuleForwardInputsOutputs): concat_args = module_input_output.concat_args_and_kwargs() api_info_struct = {} @@ -209,7 +211,7 @@ class BaseDataProcessor: output_info_list = self.analyze_element(concat_args) api_info_struct[name][Const.OUTPUT] = output_info_list return api_info_struct - + def analyze_backward(self, name, module, module_input_output: ModuleBackwardInputsOutputs): api_info_struct = {} if self.is_dump_for_data_mode(Const.BACKWARD, Const.OUTPUT): @@ -227,8 +229,11 @@ class BaseDataProcessor: return api_info_struct def get_save_file_path(self, suffix): - file_format = Const.PT_SUFFIX if self.config.framework == Const.PT_FRAMEWORK else Const.NUMPY_SUFFIX + file_format = Const.PT_SUFFIX if self.config.framework == Const.PT_FRAMEWORK else Const.NUMPY_SUFFIX dump_data_name = (self.current_api_or_module_name + Const.SEP + self.api_data_category + Const.SEP + suffix + file_format) file_path = os.path.join(self.data_writer.dump_tensor_data_dir, dump_data_name) - return dump_data_name, file_path \ No newline at end of file + return dump_data_name, file_path + + def stop_run(self): + return False diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/factory.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/factory.py index 86ef2115f..ad74acdee 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/factory.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/factory.py @@ -4,7 +4,7 @@ from msprobe.core.common.const import Const class DataProcessorFactory: _data_processor = {} _module_processor = {} - + @classmethod def register_processor(cls, framework, task, processor_class): key = (framework, task) @@ -13,7 +13,7 @@ class DataProcessorFactory: @classmethod def register_module_processor(cls, framework, processor_class): cls._module_processor[framework] = processor_class - + @classmethod def get_module_processor(cls, framework): processor_class = cls._module_processor.get(framework) @@ -39,7 +39,7 @@ class DataProcessorFactory: TensorDataProcessor as PytorchTensorDataProcessor, OverflowCheckDataProcessor as PytorchOverflowCheckDataProcessor, FreeBenchmarkDataProcessor as PytorchFreeBenchmarkDataProcessor, - KernelDumpDataProcessor as PytorchKernelDumpDataProcessor + KernelDumpDataProcessor as PytorchKernelDumpDataProcessor ) from ....pytorch.module_processer import ModuleProcesser cls.register_processor(Const.PT_FRAMEWORK, Const.STATISTICS, PytorchStatisticsDataProcessor) @@ -47,11 +47,13 @@ class DataProcessorFactory: cls.register_processor(Const.PT_FRAMEWORK, Const.OVERFLOW_CHECK, PytorchOverflowCheckDataProcessor) cls.register_processor(Const.PT_FRAMEWORK, Const.FREE_BENCHMARK, PytorchFreeBenchmarkDataProcessor) cls.register_processor(Const.PT_FRAMEWORK, Const.KERNEL_DUMP, PytorchKernelDumpDataProcessor) - cls.register_module_processor(Const.PT_FRAMEWORK, ModuleProcesser) + cls.register_module_processor(Const.PT_FRAMEWORK, ModuleProcesser) elif framework == Const.MS_FRAMEWORK: from .mindspore_processor import ( StatisticsDataProcessor as MindsporeStatisticsDataProcessor, - TensorDataProcessor as MindsporeTensorDataProcessor + TensorDataProcessor as MindsporeTensorDataProcessor, + OverflowCheckDataProcessor as MindsporeOverflowCheckDataProcessor ) cls.register_processor(Const.MS_FRAMEWORK, Const.STATISTICS, MindsporeStatisticsDataProcessor) cls.register_processor(Const.MS_FRAMEWORK, Const.TENSOR, MindsporeTensorDataProcessor) + cls.register_processor(Const.MS_FRAMEWORK, Const.OVERFLOW_CHECK, MindsporeOverflowCheckDataProcessor) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index a66cb9459..c208df7d9 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -19,7 +19,8 @@ from mindspore import ops import numpy as np from msprobe.core.common.const import Const -from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, TensorStatInfo +from msprobe.core.data_dump.data_processor.base import (BaseDataProcessor, TensorStatInfo, + ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs) from msprobe.core.common.file_check import path_len_exceeds_limit, change_mode, FileCheckConst from msprobe.mindspore.dump.hook_cell.wrap_functional import load_ops_functions from msprobe.mindspore.common.utils import convert_bf16_to_fp32 @@ -30,7 +31,7 @@ from msprobe.mindspore.dump.hook_cell.api_registry import api_register class MindsporeDataProcessor(BaseDataProcessor): mindspore_special_type = tuple([ms.Tensor]) ops_func, mint_ops_func, _ = load_ops_functions() - + def __init__(self, config, data_writer): super().__init__(config, data_writer) self.mindspore_object_key = { @@ -47,7 +48,7 @@ class MindsporeDataProcessor(BaseDataProcessor): @staticmethod def analyze_dtype_in_kwargs(element): return {"type": "mindspore.dtype", "value": str(element)} - + @staticmethod def _analyze_builtin(arg): single_arg = {} @@ -63,11 +64,11 @@ class MindsporeDataProcessor(BaseDataProcessor): single_arg.update({"type": type(arg).__name__}) single_arg.update({"value": arg}) return single_arg - + @classmethod def get_special_types(cls): return super().get_special_types() + cls.mindspore_special_type - + def get_stat_info(self, data): tensor_stat = TensorStatInfo() if data.numel() == 0: @@ -145,3 +146,61 @@ class TensorDataProcessor(MindsporeDataProcessor): else: logger.warning(f'The file path {file_path} length exceeds limit.') return single_arg + + +class OverflowCheckDataProcessor(MindsporeDataProcessor): + __slots__ = ["cached_tensors_and_file_paths"] + + def __init__(self, config, data_writer): + super().__init__(config, data_writer) + self.cached_tensors_and_file_paths = {} + self.real_overflow_dump_times = 0 + self.overflow_nums = config.overflow_nums + + def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): + self.has_overflow = False + api_info_struct = super().analyze_forward(name, module, module_input_output) + self.maybe_save_overflow_data() + return api_info_struct if self.has_overflow else None + + def analyze_backward(self, name, module, module_input_output: ModuleBackwardInputsOutputs): + self.has_overflow = False + api_info_struct = super().analyze_backward(name, module, module_input_output) + self.maybe_save_overflow_data() + return api_info_struct if self.has_overflow else None + + def maybe_save_overflow_data(self): + if self.has_overflow: + for file_path, tensor in self.cached_tensors_and_file_paths.items(): + tensor = convert_bf16_to_fp32(tensor) + np.save(file_path, tensor.asnumpy()) + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + self.real_overflow_dump_times += 1 + self.cached_tensors_and_file_paths = {} + + def stop_run(self): + if self.overflow_nums == -1: + return False + if self.real_overflow_dump_times >= self.overflow_nums: + logger.warning(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_times}") + return True + return False + + def _analyze_maybe_overflow_tensor(self, tensor_json): + if tensor_json['Max'] is None: + return + if np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']): + self.has_overflow = True + if np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']): + self.has_overflow = True + + def _analyze_tensor(self, tensor, suffix): + dump_data_name, file_path = self.get_save_file_path(suffix) + if not path_len_exceeds_limit(file_path): + self.cached_tensors_and_file_paths.update({file_path: tensor}) + else: + logger.warning(f'The file path {file_path} length exceeds limit.') + single_arg = super()._analyze_tensor(tensor, suffix) + self._analyze_maybe_overflow_tensor(single_arg) + single_arg.update({"data_name": dump_data_name}) + return single_arg diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index d02f38195..6abf0a1ee 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -29,3 +29,16 @@ def convert_bf16_to_fp32(tensor): tensor = tensor.to(ms.float32) return tensor + +class MsprobeStep(ms.train.Callback): + + def __init__(self, debugger): + super(MsprobeStep, self).__init__() + self.debugger = debugger + + def on_train_step_begin(self, run_context): + self.debugger.start() + + def on_train_step_end(self, run_context): + self.debugger.stop() + self.debugger.step() diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 04d66d6a2..0f0cdd905 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -19,9 +19,10 @@ class DebuggerConfig: self.level = DebuggerConfig.convert_map[common_config.level] self.level_ori = common_config.level self.list = [] if not task_config.list else task_config.list - self.scope =[] if not task_config.scope else task_config.scope - self.data_mode = [] if not task_config.data_mode else task_config.data_mode + self.scope = [] if not task_config.scope else task_config.scope + self.data_mode = [] if not task_config.data_mode else task_config.data_mode self.file_format = task_config.file_format + self.overflow_nums = 1 if not task_config.overflow_nums else task_config.overflow_nums self.check_mode = task_config.check_mode self.framework = Const.MS_FRAMEWORK self.summary_mode = task_config.summary_mode diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 30f7162ff..7082fc13e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -48,4 +48,4 @@ class PrecisionDebugger: def step(cls): if not cls._instance: raise Exception("PrecisionDebugger instance is not created.") - cls._instance.service.step() \ No newline at end of file + cls._instance.service.step() diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index 49ce4cf2c..c0ef6bb6c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -36,37 +36,39 @@ class StatisticsConfig(BaseConfig): raise Exception("summary_mode is invalid") -class OverflowCheck(BaseConfig): +class OverflowCheckConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) - self.file_format = None - self.check_mode = json_config.get("check_mode") + self.data_mode = ["all"] self._check_config() def _check_config(self): - if self.data_mode is not None and len(self.data_mode) > 0: - if len(self.data_mode) > 1 or self.data_mode[0] not in ["all", "input", "output"]: - raise Exception("data_mode must be all, input or output") + if self.overflow_nums is not None and not isinstance(self.overflow_nums, int): + raise Exception("overflow_nums is invalid, it should be an integer") + if self.overflow_nums is not None and self.overflow_nums != -1 and self.overflow_nums <= 0: + raise Exception("overflow_nums should be -1 or positive integer") if self.check_mode and self.check_mode not in ["all", "aicore", "atomic"]: raise Exception("check_mode is invalid") +TaskDict = { + Const.TENSOR: TensorConfig, + Const.STATISTICS: StatisticsConfig, + Const.OVERFLOW_CHECK: OverflowCheckConfig, +} + + def parse_common_config(json_config): return CommonConfig(json_config) def parse_task_config(task, json_config): - task_map = json_config[task] + task_map = json_config.get(task) if not task_map: task_map = dict() - if task == Const.TENSOR: - return TensorConfig(task_map) - elif task == Const.STATISTICS: - return StatisticsConfig(task_map) - elif task == Const.OVERFLOW_CHECK: - return OverflowCheck(task_map) - else: + if task not in TaskDict: raise Exception("task is invalid.") + return TaskDict.get(task)(task_map) def parse_json_config(json_file_path): diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index cb7a6f2a7..6d637ef02 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -69,10 +69,10 @@ class Service: def wrap_forward_hook(*args, **kwargs): return forward_hook(*args, **kwargs) - + def wrap_backward_hook(*args, **kwargs): return backward_hook(*args, **kwargs) - + return wrap_forward_hook, wrap_backward_hook def step(self): @@ -82,7 +82,7 @@ class Service: def start(self, model=None): self.model = model self.start_call = True - logger.info_on_rank_0("msprobe: debugger.start() is set successfully") + logger.info("msprobe: debugger.start() is set successfully") if self.config.step and self.current_iter > max(self.config.step): self.stop() raise Exception("msprobe: exit after iteration {}".format(max(self.config.step))) @@ -99,13 +99,15 @@ class Service: self.register_hook_new() self.first_start = False self.switch = True - logger.info_on_rank_0(f"Dump switch is turned on at step {self.current_iter}. ") + logger.info(f"Dump switch is turned on at step {self.current_iter}. ") self.create_dirs() - logger.info_on_rank_0(f"Dump data will be saved in {self.dump_iter_dir}.") + logger.info(f"Dump data will be saved in {self.dump_iter_dir}.") def stop(self): + logger.info("msprobe: debugger.stop() is set successfully. " + "Please set debugger.start() to turn on the dump switch again. ") if not self.start_call: - logger.error_on_rank_0("msprobe: debugger.start() is not set in the current scope.") + logger.error("msprobe: debugger.start() is not set in the current scope.") raise Exception("debugger.start() is not set in the current scope.") if self.config.step and self.current_iter not in self.config.step: return @@ -113,7 +115,6 @@ class Service: return self.switch = False self.start_call = False - logger.info_on_rank_0(f"msprobe: debugger.stop() is set successfully. Please set debugger.start() to turn on the dump switch again. ") self.data_collector.write_json() def create_dirs(self): @@ -138,9 +139,9 @@ class Service: construct_file_path = os.path.join(dump_dir, "construct.json") self.data_collector.update_dump_paths( dump_file_path, stack_file_path, construct_file_path, dump_data_dir, None) - + def register_hook_new(self): - logger.info_on_rank_0("The {} hook function is successfully mounted to the model.".format(self.config.task)) + logger.info("The {} hook function is successfully mounted to the model.".format(self.config.task)) if self.config.level == "L1": api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) api_register.api_set_hook_func() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py index 673386afb..30212d95e 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py @@ -19,7 +19,7 @@ from unittest.mock import patch, mock_open from msprobe.core.common.const import Const from msprobe.mindspore.ms_config import (parse_json_config, parse_task_config, - TensorConfig, StatisticsConfig, OverflowCheck) + TensorConfig, StatisticsConfig, OverflowCheckConfig) class TestMsConfig(TestCase): @@ -62,7 +62,7 @@ class TestMsConfig(TestCase): self.assertTrue(isinstance(task_config, StatisticsConfig)) task_config = parse_task_config("overflow_check", mock_json_config) - self.assertTrue(isinstance(task_config, OverflowCheck)) + self.assertTrue(isinstance(task_config, OverflowCheckConfig)) with self.assertRaises(Exception) as context: parse_task_config("free_benchmark", mock_json_config) -- Gitee From e594820221e683b2fbeadd57389099e6ee5616da Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 1 Aug 2024 14:09:48 +0800 Subject: [PATCH 067/791] list range dump bugfix when distribute api --- .../src/python/ptdbg_ascend/dump/utils.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py index 4ed099da2..7a5afd90b 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py @@ -21,6 +21,11 @@ range_begin_flag, range_end_flag = False, False def check_list_or_acl_mode(name_prefix): global dump_count for item in DumpUtil.dump_switch_scope: + if "pre_forward" in name_prefix: + parts = item.split(".") + rename = ".".join(parts[:-1]) + if name_prefix.startswith(rename): + return True if name_prefix.startswith(item): dump_count = dump_count + 1 return True @@ -30,7 +35,13 @@ def check_list_or_acl_mode(name_prefix): def check_range_mode(name_prefix): global range_begin_flag global range_end_flag - if name_prefix.startswith(DumpUtil.dump_switch_scope[0]): + if "Distributed" in DumpUtil.dump_switch_scope[0]: + parts = DumpUtil.dump_switch_scope[0].split(".") + rename = ".".join(parts[:-1]) + if name_prefix.startswith(rename): + range_begin_flag = True + return True + elif name_prefix.startswith(DumpUtil.dump_switch_scope[0]): range_begin_flag = True return True if name_prefix.startswith(DumpUtil.dump_switch_scope[1]): -- Gitee From e1117d57a53899f7b51f90a3d155caf97568c455 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 1 Aug 2024 14:21:51 +0800 Subject: [PATCH 068/791] review fix --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 0bdf8a4c7..f9773e337 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -647,9 +647,10 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): end_color=CompareConst.YELLOW, fill_type="solid") try: wb.save(file_path) - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) except Exception: logger.error('Save result file failed') + raise CompareException(CompareException.WRITE_FILE_ERROR) + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, -- Gitee From 2c5c55b2acb3fbdee0d4e4fd4fe3d89966fbc8d9 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 1 Aug 2024 14:29:25 +0800 Subject: [PATCH 069/791] review fix --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index f9773e337..a916ce7b4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -647,9 +647,9 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): end_color=CompareConst.YELLOW, fill_type="solid") try: wb.save(file_path) - except Exception: + except Exception as e: logger.error('Save result file failed') - raise CompareException(CompareException.WRITE_FILE_ERROR) + raise CompareException(CompareException.WRITE_FILE_ERROR) from e change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) -- Gitee From 8766fc7696889b81855046200c0824ad7af08b85 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 1 Aug 2024 16:09:26 +0800 Subject: [PATCH 070/791] bugfix import path fail --- .../msprobe/pytorch/bench_functions/npu_fusion_attention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index f4b639e2f..63f1fa2a3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -2,7 +2,7 @@ import torch import numpy as np from einops import rearrange -from api_accuracy_checker.common.utils import logger +from msprobe.pytorch.common.utils import logger gtype = torch.float64 # arm host必须选择float64，x86环境选择float32即可，64也行。arm计算很慢，s=8k的场景建议使用x86 softmax_build_mode = "QKV" # "MAX_SUM" -- Gitee From b12f6fbf67ef12129cce634bbff7f6e0354a64eb Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Thu, 1 Aug 2024 16:07:18 +0800 Subject: [PATCH 071/791] move specific cluster analysis to poc --- profiler/cli/cluster_cli.py | 4 +- profiler/cluster_analyse/README.md | 39 +-- .../analysis/analysis_facade.py | 12 - .../cluster_analyse/analysis/base_analysis.py | 153 ----------- .../analysis/cann_api_sum/__init__.py | 14 - .../analysis/cann_api_sum/cann_api_sum.py | 108 -------- .../analysis/cann_api_sum/stats.ipynb | 86 ------- .../analysis/cluster_display.py | 239 ------------------ .../analysis/compute_op_sum/__init__.py | 14 - .../analysis/compute_op_sum/compute_op_sum.py | 103 -------- .../analysis/compute_op_sum/stats.ipynb | 164 ------------ .../analysis/hccl_sum/__init__.py | 14 - .../analysis/hccl_sum/hccl_sum.py | 133 ---------- .../analysis/hccl_sum/stats.ipynb | 162 ------------ .../analysis/mstx_sum/__init__.py | 14 - .../analysis/mstx_sum/mstx_sum.py | 204 --------------- .../analysis/mstx_sum/stats.ipynb | 180 ------------- profiler/cluster_analyse/cluster_analysis.py | 75 +----- .../cluster_statistics_export/__init__.py | 14 - .../cann_api_sum_export.py | 65 ----- .../compute_op_sum_export.py | 49 ---- .../hccl_sum_export.py | 39 --- .../mstx_mark_export.py | 57 ----- .../mstx_step_export.py | 35 --- .../cluster_statistics_export/stats_export.py | 40 --- .../common_func/analysis_loader.py | 38 --- .../cluster_analyse/common_func/constant.py | 10 - .../cluster_analyse/common_func/context.py | 85 ------- .../cluster_analyse/common_func/db_manager.py | 7 - .../common_func/sql_extention_func.py | 73 ------ profiler/cluster_analyse/common_func/utils.py | 73 ------ 31 files changed, 17 insertions(+), 2286 deletions(-) delete mode 100644 profiler/cluster_analyse/analysis/cann_api_sum/__init__.py delete mode 100644 profiler/cluster_analyse/analysis/cann_api_sum/cann_api_sum.py delete mode 100644 profiler/cluster_analyse/analysis/cann_api_sum/stats.ipynb delete mode 100644 profiler/cluster_analyse/analysis/cluster_display.py delete mode 100644 profiler/cluster_analyse/analysis/compute_op_sum/__init__.py delete mode 100644 profiler/cluster_analyse/analysis/compute_op_sum/compute_op_sum.py delete mode 100644 profiler/cluster_analyse/analysis/compute_op_sum/stats.ipynb delete mode 100644 profiler/cluster_analyse/analysis/hccl_sum/__init__.py delete mode 100644 profiler/cluster_analyse/analysis/hccl_sum/hccl_sum.py delete mode 100644 profiler/cluster_analyse/analysis/hccl_sum/stats.ipynb delete mode 100644 profiler/cluster_analyse/analysis/mstx_sum/__init__.py delete mode 100644 profiler/cluster_analyse/analysis/mstx_sum/mstx_sum.py delete mode 100644 profiler/cluster_analyse/analysis/mstx_sum/stats.ipynb delete mode 100644 profiler/cluster_analyse/cluster_statistics_export/__init__.py delete mode 100644 profiler/cluster_analyse/cluster_statistics_export/cann_api_sum_export.py delete mode 100644 profiler/cluster_analyse/cluster_statistics_export/compute_op_sum_export.py delete mode 100644 profiler/cluster_analyse/cluster_statistics_export/hccl_sum_export.py delete mode 100644 profiler/cluster_analyse/cluster_statistics_export/mstx_mark_export.py delete mode 100644 profiler/cluster_analyse/cluster_statistics_export/mstx_step_export.py delete mode 100644 profiler/cluster_analyse/cluster_statistics_export/stats_export.py delete mode 100644 profiler/cluster_analyse/common_func/analysis_loader.py delete mode 100644 profiler/cluster_analyse/common_func/context.py delete mode 100644 profiler/cluster_analyse/common_func/sql_extention_func.py delete mode 100644 profiler/cluster_analyse/common_func/utils.py diff --git a/profiler/cli/cluster_cli.py b/profiler/cli/cluster_cli.py index 93a4a638f..c1563898d 100644 --- a/profiler/cli/cluster_cli.py +++ b/profiler/cli/cluster_cli.py @@ -21,7 +21,7 @@ sys.path.append(os.path.dirname(os.path.dirname(__file__))) from profiler.advisor.utils.tools import CONTEXT_SETTINGS, ClickAliasedGroup from profiler.advisor.utils.utils import debug_option from profiler.prof_common.constant import Constant -from profiler.cluster_analyse.cluster_analysis import ALL_FEATURE_LIST +from profiler.cluster_analyse.cluster_analysis import COMM_FEATURE_LIST from profiler.cluster_analyse.cluster_analysis import cluster_analysis_main @@ -33,7 +33,7 @@ context_settings['ignore_unknown_options'] = True short_help='Analyze cluster data to locate slow nodes and slow links.') @click.option('--profiling_path', '-d', type=click.Path(), required=True, help='path of the profiling data') -@click.option('--mode', '-m', type=click.Choice(ALL_FEATURE_LIST), default='all') +@click.option('--mode', '-m', type=click.Choice(COMM_FEATURE_LIST), default='all') @click.argument('args', nargs=-1) def cluster_cli(profiling_path, mode, args) -> None: required_args = ('-d', profiling_path, '-m', mode) diff --git a/profiler/cluster_analyse/README.md b/profiler/cluster_analyse/README.md index fdd43ca96..4a394e09a 100644 --- a/profiler/cluster_analyse/README.md +++ b/profiler/cluster_analyse/README.md @@ -54,10 +54,7 @@ experimental_config = torch_npu.profiler._ExperimentalConfig( | --------------------- | ------------------------------------------------------------ | -------- | | --collection_path或-d | 性能数据汇集目录，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 是 | | --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | - | --parallel_mode | 设置收集多卡、多节点db数据时的并发方式。取值为concurrent（使用concurrent.feature进程池实现并发）。
**只有-m配置cann_api_sum、compute_op_sum、hccl_sum、mstx_sum时可配置此参数。** | 否 | - | --export_type | 设置导出的数据形式。取值为db（.db格式文件）和notebook（Jupyter Notebook文件），默认值为db。
**只有-m配置cann_api_sum、compute_op_sum、hccl_sum、mstx_sum时可配置此参数。** | 否 | - | --rank_list | 对特定Rank上的数据进行统计，默认值为all（表示对所有Rank进行统计），须根据实际卡的Rank ID配置。应配置为大于等于0的整数，若所配置的值大于实际训练所运行的卡的Rank ID，则仅解析合法的RankID的数据，比如当前环境Rank ID为0到7，实际训练运行0到3卡，此时若配置Rank ID为0, 3, 4或不存在的10等其他值，则仅解析0和3。配置示例：--rank_list 0, 1, 2。
**只有-m配置cann_api_sum、compute_op_sum、hccl_sum、mstx_sum时可配置此参数。** | 否 | - | --top_num | 设置TopN耗时的通信算子的数量，默认值为15，配置示例：--top_num 20。
**只有-m配置hccl_sum时可配置此参数。** | 否 | + --mode参数说明： @@ -66,22 +63,7 @@ experimental_config = torch_npu.profiler._ExperimentalConfig( | communication_matrix | 解析通信矩阵数据。 | 否 | | communication_time | 解析通信耗时数据。 | 否 | | all | 同时解析通信矩阵communication_matrix和通信耗时数据communication_time，--mode参数默认值为all。 | 否 | - | cann_api_sum | 集群API性能数据汇总分析，输入性能数据需要基于ascend_pytorch_profiler_{rank_id}.db文件。--export_type为db时，输出交付件cluster_analysis.db；--export_type为notebook时，在cluster_analysis_output/CannApiSum目录下输出交付件stats.ipynb。 | 否 | - | compute_op_sum | 集群场景性能数据的device运行算子信息汇总分析，输入性能数据需要基于ascend_pytorch_profiler_{rank_id}.db文件。--export_type为db时，输出交付件cluster_analysis.db；--export_type为notebook时，在cluster_analysis_output/ComputeOpSum目录下输出交付件stats.ipynb。 | 否 | - | hccl_sum | 集合通信算子耗时分析，输入性能数据需要基于ascend_pytorch_profiler_{rank_id}.db文件。--export_type为db时，输出交付件cluster_analysis.db；--export_type为notebook时，在cluster_analysis_output/HcclSum目录下输出交付件stats.ipynb。 | 否 | - | mstx_sum | 集群场景mstx打点信息汇总分析，输入性能数据需要基于ascend_pytorch_profiler_{rank_id}.db文件。--export_type为db时，输出交付件cluster_analysis.db；--export_type为notebook时，在cluster_analysis_output/MstxSum目录下输出交付件stats.ipynb。 | 否 | - - --parallel_mode参数示例如下： - - ```bash - msprof-analyze cluster -d {cluster profiling data path} -m cann_api_sum --parallel_mode concurrent - ``` - - 或 - - ```bash - python3 cluster_analysis.py -d {cluster profiling data path} -m cann_api_sum --parallel_mode concurrent - ``` + ### 交付件 @@ -158,23 +140,6 @@ L列：Preparing，指迭代开始到首个计算或通信算子运行的时间 解析analysis.db或ascend_pytorch_profiler_{rank_id}.db生成的交付件，根据数据解析模式不同而解析不同的数据，可以使用MindStudio Insight工具展示。 -#### stats.ipynb - -- 数据解析模式为cann_api_sum时生成，保存在cluster_analysis_output/CannApiSum目录下。 - - 可使用jupyter notebook工具或MindStudio Insight工具打开，主要展示集群API耗时信息。 - -- 数据解析模式为compute_op_sum时生成，保存在cluster_analysis_output/ComputeOpSum目录下。 - - 可使用jupyter notebook工具或MindStudio Insight工具打开，主要展示集群计算算子耗时分析（将集群所有计算算子进行汇总并以图表展示），集群Rank计算算子耗时分析（将每个Rank的计算算子进行各自汇总）。 - -- 数据解析模式为hccl_sum时生成，保存在cluster_analysis_output/HcclSum目录下。 - - 可使用jupyter notebook工具或MindStudio Insight工具打开，主要展示集群通信算子耗时分析（将集群所有通信算子进行汇总并以图表展示），集群Rank通信算子耗时分析（将每个Rank的通信算子进行各自汇总）、Top通信算子信息展示。 - -- 数据解析模式为mstx_sum时生成，保存在cluster_analysis_output/MstxSum目录下。 - - 可使用jupyter notebook工具或MindStudio Insight工具打开，主要展示集群场景mstx打点信息，分为框架侧、CANN侧和Device侧三部分的打点信息。 diff --git a/profiler/cluster_analyse/analysis/analysis_facade.py b/profiler/cluster_analyse/analysis/analysis_facade.py index 435d77b21..c438fbcf3 100644 --- a/profiler/cluster_analyse/analysis/analysis_facade.py +++ b/profiler/cluster_analyse/analysis/analysis_facade.py @@ -19,8 +19,6 @@ from analysis.communication_analysis import CommunicationAnalysis from analysis.comm_matrix_analysis import CommMatrixAnalysis from analysis.step_trace_time_analysis import StepTraceTimeAnalysis from analysis.host_info_analysis import HostInfoAnalysis -from common_func.context import Context -from common_func.constant import Constant class AnalysisFacade: default_module = {CommunicationAnalysis, StepTraceTimeAnalysis, CommMatrixAnalysis, HostInfoAnalysis} @@ -38,13 +36,3 @@ class AnalysisFacade: for process in process_list: process.join() - - def recipe_analyze(self): - HostInfoAnalysis(self.params).run() - print("[INFO] Recipe analysis launched.") - try: - with Context.create_context(self.params.get(Constant.PARALLEL_MODE)) as context: - with self.params.get(Constant.RECIPE_CLASS)(self.params) as recipe: - recipe.run(context) - except Exception as e: - print("[ERROR] Recipe analysis launched failed, %s." % str(e)) diff --git a/profiler/cluster_analyse/analysis/base_analysis.py b/profiler/cluster_analyse/analysis/base_analysis.py index 7209e9b56..d7be4fc9c 100644 --- a/profiler/cluster_analyse/analysis/base_analysis.py +++ b/profiler/cluster_analyse/analysis/base_analysis.py @@ -22,8 +22,6 @@ from abc import abstractmethod from common_func.constant import Constant from common_func.file_manager import FileManager -from common_func.db_manager import DBManager -from common_func.utils import convert_unit from cluster_utils.data_transfer_adapter import DataTransferAdapter @@ -102,154 +100,3 @@ class BaseAnalysis: for rank_tup, group_dict in self.comm_ops_struct.items(): for step_id, communication_ops in group_dict.items(): self.compute_total_info(communication_ops) - - -class BaseRecipeAnalysis: - - UNIT = "Us" - DB_UNIT = "Ns" - - RANK_LIST = "rank_list" - - def __init__(self, params): - self._params = params - self._collection_dir = params.get(Constant.COLLECTION_PATH, "") - self._data_map = params.get(Constant.DATA_MAP, {}) - self._recipe_name = params.get(Constant.RECIPE_NAME, "") - self._mode = params.get(Constant.PARALLEL_MODE, "") - self._export_type = params.get(Constant.EXPORT_TYPE, "") - self._output_dir = None - self._rank_list = params.get(self.RANK_LIST, 'all') - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if self._params is not None and exc_type is not None: - print(f"[ERROR] Failed to exit analysis: {exc_val}") - traceback.print_exc(file=sys.stdout) - - def run(self, context): - pass - - @property - def base_dir(self): - return os.path.basename(os.path.dirname(__file__)) - - def _get_rank_db(self): - invalid_rank_id = [] - if self._rank_list == 'all': - rank_ids = list(self._data_map.keys()) - else: - rank_ids = [] - for rank_id in self._rank_list: - if rank_id in self._data_map.keys(): - rank_ids.append(rank_id) - else: - invalid_rank_id.append(str(rank_id)) - db_paths = [] - for rank_id in rank_ids: - rank_path = self._data_map[rank_id] - db_path = os.path.join(rank_path, Constant.SINGLE_OUTPUT, f"ascend_pytorch_profiler_{rank_id}.db") - if os.path.exists(db_path): - db_paths.append((rank_id, db_path)) - else: - print(f"[WARNING] DB file not found, rank id: {rank_id}, db path: {db_path}.") - if invalid_rank_id: - print(f"[WARNING] Invalid Rank id : [{','.join(invalid_rank_id)}].") - return db_paths - - def get_mode(self): - return self._mode - - def get_recipe_name(self): - return self._recipe_name - - def dump_data(self, data, file_name, table_name=None, index=True): - output_path = os.path.join(self._collection_dir, Constant.CLUSTER_ANALYSIS_OUTPUT) - if table_name: - result_db = os.path.join(output_path, file_name) - conn, cursor = DBManager.create_connect_db(result_db) - if isinstance(data, pd.DataFrame): - data.to_sql(table_name, conn, if_exists='replace', index=True) - else: - print(f"[ERROR] Unknown dump data type: {type(data)}") - DBManager.destroy_db_connect(conn, cursor) - else: - result_csv = os.path.join(output_path, file_name) - if isinstance(data, pd.DataFrame): - data = convert_unit(data, self.DB_UNIT, self.UNIT) - data.to_csv(result_csv, index=index) - else: - print(f"[ERROR] Unknown dump data type: {type(data)}") - - def _create_output_dir_name(self, name): - i = 1 - while os.path.exists(f"{name}-{i}"): - i += 1 - return f"{name}-{i}" - - def _create_unique_output_dir(self): - output_dir = os.path.join(self._collection_dir, Constant.CLUSTER_ANALYSIS_OUTPUT, self._recipe_name) - - if os.path.exists(output_dir): - return self._create_output_dir_name(output_dir) - return output_dir - - def _get_output_dir(self): - if self._output_dir is None: - self._output_dir = self._create_unique_output_dir() - os.makedirs(self._output_dir) - return self._output_dir - - def create_notebook(self, filename, notebook_template_dir=None, replace_dict=None): - if notebook_template_dir is None: - template_path = os.path.dirname(__file__) - else: - template_path = notebook_template_dir - output_path = os.path.join(self._get_output_dir(), filename) - template_file = os.path.join(template_path, self.base_dir, filename) - if replace_dict is None: - shutil.copy(template_file, output_path) - else: - with open(template_file, 'r') as f: - template_content = f.read() - for key, value in replace_dict.items(): - template_content = template_content.replace(str(key), str(value)) - with open(output_path, 'w') as f: - f.write(template_content) - print(f"[INFO] Notebook export path is: {self._get_output_dir()}") - - def add_helper_file(self, helper_file): - helper_output_path = os.path.join(self._get_output_dir(), helper_file) - helper_file_path = os.path.join(os.path.dirname(__file__), helper_file) - - if helper_file_path is not None: - shutil.copy(helper_file_path, helper_output_path) - - @staticmethod - def _filter_data(mapper_data): - return [(rank, data) for rank, data in mapper_data if data is not None and len(data) != 0] - - @classmethod - def add_parser_argument(cls, parser): - parser.add_argument("--rank_list", type=str, help="Rank id list", default='all') - - @classmethod - def parse_argument(cls, args_parsed) -> dict: - if args_parsed.rank_list == 'all': - return { - cls.RANK_LIST: 'all' - } - else: - rank_str_list = args_parsed.rank_list.split(",") - rank_list = [int(rank) for rank in rank_str_list if rank.isdigit()] - return { - cls.RANK_LIST: rank_list - } - - @classmethod - def get_extra_argument(cls, params) -> dict: - return { - cls.RANK_LIST: params.get(cls.RANK_LIST, "all") - } diff --git a/profiler/cluster_analyse/analysis/cann_api_sum/__init__.py b/profiler/cluster_analyse/analysis/cann_api_sum/__init__.py deleted file mode 100644 index 7101187a2..000000000 --- a/profiler/cluster_analyse/analysis/cann_api_sum/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/profiler/cluster_analyse/analysis/cann_api_sum/cann_api_sum.py b/profiler/cluster_analyse/analysis/cann_api_sum/cann_api_sum.py deleted file mode 100644 index db37b004b..000000000 --- a/profiler/cluster_analyse/analysis/cann_api_sum/cann_api_sum.py +++ /dev/null @@ -1,108 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pandas as pd - -from analysis.base_analysis import BaseRecipeAnalysis -from common_func.constant import Constant -from common_func.utils import stdev -from cluster_statistics_export.cann_api_sum_export import CannApiSumExport - - -class CannApiSum(BaseRecipeAnalysis): - - def __init__(self, params): - super().__init__(params) - print("[INFO] CannApiSum init.") - - @property - def base_dir(self): - return os.path.basename(os.path.dirname(__file__)) - - @staticmethod - def _mapper_func(data_map, analysis_class): - df = CannApiSumExport(data_map[1], analysis_class).read_export_db() - - if df is None or df.empty: - print(f"[WARNING] There is no stats data in {data_map[1]}.") - return None - return data_map[0], df - - def mapper_func(self, context): - return context.wait( - context.map( - self._mapper_func, - self._get_rank_db(), - analysis_class=self._recipe_name - ) - ) - - def reducer_func(self, mapper_res): - stats_rank_data = self._filter_data(mapper_res) - if not stats_rank_data: - print("[ERROR] Mapper data is None.") - return - stats_rank_data = [df.assign(rank=rank) for rank, df in stats_rank_data] - stats_rank_data = pd.concat(stats_rank_data) - stats_data = self._aggregate_stats(stats_rank_data) - if self._export_type == "db": - self.dump_data(stats_rank_data, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, "CannApiSumRank") - self.dump_data(stats_data, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, "CannApiSum") - elif self._export_type == "notebook": - self.dump_data(stats_rank_data, os.path.join(self._get_output_dir(), "rank_stats.csv"), index=False) - self.dump_data(stats_data, os.path.join(self._get_output_dir(), "all_stats.csv")) - self.save_notebook() - else: - print("[ERROR] Unknown export type.") - - def run(self, context): - mapper_res = self.mapper_func(context) - self.reducer_func(mapper_res) - - @staticmethod - def _aggregate_stats(stats_res): - grouped = stats_res.groupby("name") - res = {} - total_time = grouped["totalTimeNs"].sum() - res["timeRatio"] = total_time / total_time.sum() * 100.0 - res["totalTimeNs"] = total_time - res["totalCount"] = grouped["totalCount"].sum() - res["averageNs"] = res["totalTimeNs"] / res["totalCount"] - res["Q1Ns"] = grouped["Q1Ns"].min() - res["medNs"] = grouped["medNs"].median() - res["Q3Ns"] = grouped["Q3Ns"].max() - res["minNs"] = grouped["minNs"].min() - res["maxNs"] = grouped["maxNs"].max() - res["stdev"] = grouped.apply(lambda x: stdev(x, res)) - min_value = grouped["minNs"].min() - res["minRank"] = grouped.apply( - lambda x: ", ".join( - x.loc[x["minNs"] == min_value.loc[x.name], "rank"].astype(str) - ) - ) - max_value = grouped["maxNs"].max() - res["maxRank"] = grouped.apply( - lambda x: ", ".join( - x.loc[x["maxNs"] == max_value.loc[x.name], "rank"].astype(str) - ) - ) - res = pd.concat(res.values(), axis=1, keys=res.keys()).round(1) - res.sort_values(by="totalTimeNs", ascending=False, inplace=True) - return res - - def save_notebook(self): - self.create_notebook("stats.ipynb") - self.add_helper_file("cluster_display.py") diff --git a/profiler/cluster_analyse/analysis/cann_api_sum/stats.ipynb b/profiler/cluster_analyse/analysis/cann_api_sum/stats.ipynb deleted file mode 100644 index c97f039c5..000000000 --- a/profiler/cluster_analyse/analysis/cann_api_sum/stats.ipynb +++ /dev/null @@ -1,86 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# CANN_API_SUM" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, - "outputs": [], - "source": [ - "import pandas as pd\n", - "import plotly.offline as pyo\n", - "\n", - "from IPython.display import display, HTML\n", - "\n", - "import cluster_display\n", - "\n", - "display(HTML(\"\"))\n", - "pd.set_option('display.max_columns', None)\n", - "pd.set_option('display.max_rows', None)\n", - "pyo.init_notebook_mode()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "\n", - "\n", - "## 集群场景CANN层API统计分析\n", - "该分析脚本展示了集群场景的统计数据分析结果。需要注意以下几点：\n", - "1. 所有的时间信息单位是微秒(us);\n", - "2. Q1表示单个API耗时的25%分位数，最终结果取自所有卡的Q1值中最小值；\n", - "3. Q3表示单个API耗时的75%分位数，最终结果取自所有卡的Q3值中最大值；\n", - "4. 'minRank'展示了API最小耗时所在卡；\n", - "5. 'maxRank'展示了API最大耗时所在卡。" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "df = pd.read_csv(\"all_stats.csv\")\n", - "display(df)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cluster_display.display_box(df, xaxis_title=\"name\", yaxis_title=\"duration (ns)\")\n", - "cluster_display.display_stats_scatter(df, xaxis_title=\"name\", yaxis_title=\"duration (ns)\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "per_rank_df = pd.read_csv(\"rank_stats.csv\")\n", - "cluster_display.display_stats_per_operation(per_rank_df, xaxis_title='rank', yaxis_title='duration (ns)')" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/profiler/cluster_analyse/analysis/cluster_display.py b/profiler/cluster_analyse/analysis/cluster_display.py deleted file mode 100644 index 8fc6040cc..000000000 --- a/profiler/cluster_analyse/analysis/cluster_display.py +++ /dev/null @@ -1,239 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np -import pandas as pd -import plotly.graph_objects as go -from IPython.display import display, HTML -from ipywidgets import Dropdown, fixed, interact - - -def get_stats_cols(df): - cols = df.columns.tolist() - q1 = "Q1(Us)" if "Q1(Us)" in cols else "Q1~" - q3 = "Q3(Us)" if "Q3(Us)" in cols else "Q3~" - med = "med(Us)" if "med(Us)" in cols else "med~" - std = "stdev" if "stdev" in cols else "stdev~" - return q1, q3, med, std - - -def display_box(df, x=None, **layout_args): - if x is None: - x = df.columns[0] - q1, q3, med, std = get_stats_cols(df) - fig = go.Figure() - fig.add_trace( - go.Box( - x=df[x], - q1=df[q1], - median=df[med], - q3=df[q3], - sd=df[std], - lowerfence=df["minRank"], - upperfence=df["maxRank"] - ) - ) - fig.update_layout(**layout_args) - fig.show() - - -def display_stats_scatter(df, x=None, **layout_args): - if x is None: - x = df.columns[0] - q1, q3, med, _ = get_stats_cols(df) - fig = go.Figure() - col_names = [q1, med, q3, "minRank", "maxRank"] - for name in col_names: - fig.add_trace( - go.Scatter( - x=df[x], - y=df[name], - name=name - ) - ) - fig.update_layout(**layout_args) - fig.show() - - -def display_table_per_rank(df): - if df.empty: - display(df) - return - - rank_groups = df.groupby("rank") - def display_table(name): - rank_df = rank_groups.get_group(name) - rank_df = rank_df.drop(columns=["rank"]) - display(rank_df) - - dropdown = Dropdown( - options=rank_groups.groups.keys(), - description="rank:", - disabled=False, - ) - interact( - display_table, - name=dropdown - ) - - -def display_stats_per_operation(df, x=None, box=True, scatter=True, table=True, **layout_args): - if df.empty: - display(df) - return - - if x is None: - x = df.columns[0] - - op_groups = df.groupby(x) - - def display_graphs(name): - op_df = op_groups.get_group(name) - if table: - display(op_df.reset_index(drop=True).set_index("rank")) - if box: - display_box(op_df, x=op_df["rank"], **layout_args) - if scatter: - display_stats_scatter(op_df, x=op_df["rank"], **layout_args) - - operations = list(op_groups.groups.keys()) - - if len(operations) > 1: - dropdown = Dropdown( - options=operations, - description="Operation:", - disabled=False, - value=operations[1] - ) - interact( - display_graphs, - name=dropdown - ) - dropdown.value = operations[0] - else: - display_graphs(operations[0]) - - -def display_duration_boxplots(figs, stats_df: pd.DataFrame, orientation="v", title=None, - x_title="Names", y_title="Time", legend_title="Legend"): - mean_ds = stats_df.get("Mean(Us)", None) - min_ds = stats_df.get("Min(Us)", None) - max_ds = stats_df.get("Max(Us)", None) - q1_ds = stats_df.get("Q1(Us)", None) - median_ds = stats_df.get('Median(Us)', None) - q3_ds = stats_df.get('Q3(Us)', None) - return display_boxplot(figs, stats_df.index, min_ds, q1_ds, median_ds, q3_ds, max_ds, mean_ds, - orientation=orientation, title=title, x_title=x_title, y_title=y_title, - legend_title=legend_title) - - -def display_boxplot(figs, x_axis, min_ds, q1_ds, median_ds, q3_ds, max_ds, mean_ds, orientation="v", - title=None, x_title=None, y_title="Time", legend_title="Legend"): - fig = go.Figure() - fig.add_trace( - go.Box( - x=x_axis, - lowerfence=min_ds, - q1=q1_ds, - median=median_ds, - q3=q3_ds, - upperfence=max_ds, - mean=mean_ds - ) - ) - fig.update_traces(orientation=orientation) - fig.update_layout( - xaxis_title=x_title, yaxis_title=y_title, legend_title=legend_title, - title=title, height=1024 - ) - fig.show() - if isinstance(figs, list): - figs.append(fig) - return fig - - -def display_graph(figs, x_axis, y_axes, title=None, - x_title=None, y_title=None, legend_title="Legend"): - data = None - if isinstance(y_axes, pd.DataFrame): - data = y_axes.set_index(x_axis) - elif isinstance(y_axes, dict): - data = pd.DataFrame(y_axes, index=x_axis) - elif isinstance(y_axes, pd.Series): - data = pd.DataFrame({"": y_axes}, index=x_axis) - elif isinstance(y_axes, np.ndarray): - data = pd.DataFrame({"": pd.Series(y_axes)}, index=x_axis) - else: - return - - fig = data.plot.line() - fig.update_layout( - title=title, xaxis_title=x_title, yaxis_title=y_title, legend_title=legend_title - ) - fig.show() - if isinstance(figs, list): - figs.append(fig) - return fig - - -def display_stats_per_rank_groups_combobox(rank_stats_gdf): - names = list(rank_stats_gdf.groups.keys()) - if len(names) > 1: - dropdown = Dropdown( - options=names, layout={"width": "max-content"}, value=names[1] - ) - interact( - __display_stats_per_rank_group, - selected=dropdown, - rank_stats_gdf=fixed(rank_stats_gdf) - ) - dropdown.value = names[0] - elif len(names) == 1: - __display_stats_per_rank_group(names[0], rank_stats_gdf) - else: - print("cluster_display func:input rank_stats_gdf groups is null so no need to display") - - -def __display_stats_per_rank_group(selected, rank_stats_gdf): - df = rank_stats_gdf.get_group(selected) - df = df.reset_index(drop=True) - df = df.set_index(df["Rank"]) - display(df) - - figs = [] - display_duration_boxplots(figs, df, x_title="Ranks") - display_graph( - figs, - df.index, - df[["Q1(Us)", "Median(Us)", "Q3(Us)"]], - title="50% of Distribution", - x_title="Ranks" - ) - - -def display_stats_optional_combobox(options, display_func, args, description="Option:"): - if len(options) > 1: - dropdown = Dropdown( - options=options, layout={"width": "max-content"}, value=options[1], - description=description - ) - interact( - display_func, - selected=dropdown, - args=fixed(args) - ) - dropdown.value = options[0] - elif len(options) == 1: - display_func(options[0], args) diff --git a/profiler/cluster_analyse/analysis/compute_op_sum/__init__.py b/profiler/cluster_analyse/analysis/compute_op_sum/__init__.py deleted file mode 100644 index 7101187a2..000000000 --- a/profiler/cluster_analyse/analysis/compute_op_sum/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/profiler/cluster_analyse/analysis/compute_op_sum/compute_op_sum.py b/profiler/cluster_analyse/analysis/compute_op_sum/compute_op_sum.py deleted file mode 100644 index e71cf868a..000000000 --- a/profiler/cluster_analyse/analysis/compute_op_sum/compute_op_sum.py +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pandas as pd -from analysis.base_analysis import BaseRecipeAnalysis -from common_func.constant import Constant -from common_func.utils import describe_duration -from cluster_statistics_export.compute_op_sum_export import ComputeOpSumExport - - -class ComputeOpSum(BaseRecipeAnalysis): - - TABLE_ALL_RANK_STATS = "ComputeOpAllRankStats" - TABLE_PER_RANK_STATS_BY_OPTYPE = "ComputeOpPerRankStatsByOpType" - TABLE_PER_RANK_STATS_BY_OPNAME = "ComputeOpPerRankStatsByOpName" - - def __init__(self, params): - super().__init__(params) - print("[INFO] ComputeOpSum init.") - self.all_rank_stats = None - self.per_rank_stats_by_optype = None - self.per_rank_stats_by_opname = None - - @property - def base_dir(self): - return os.path.basename(os.path.dirname(__file__)) - - @staticmethod - def _mapper_func(data_map, analysis_class): - df = ComputeOpSumExport(data_map[1], analysis_class).read_export_db() - - if df is None or df.empty: - print(f"[WARNING] There is no stats data in {data_map[1]}.") - return None - - df["Rank"] = data_map[0] - return df - - def mapper_func(self, context): - return context.wait( - context.map( - self._mapper_func, - self._get_rank_db(), - analysis_class=self._recipe_name - ) - ) - - def reducer_func(self, mapper_res): - mapper_res = list(filter(lambda df: df is not None, mapper_res)) - if not mapper_res: - print("[ERROR] Mapper data is None.") - return - # get per rank stats by optype - self.per_rank_stats_by_optype = pd.concat( - describe_duration(df.groupby(["OpType", "TaskType"])["Duration"]).assign(Rank=df["Rank"][0]) for df in mapper_res) - self.per_rank_stats_by_optype.sort_values(by=["SumNs"], inplace=True, ascending=False) - - # get all rank stats by optype - all_op_data = pd.concat(mapper_res) - self.all_rank_stats = describe_duration(all_op_data.groupby(["OpType", "TaskType"])["Duration"]) - self.all_rank_stats.sort_values(by=["SumNs"], inplace=True, ascending=False) - - # get per rank stats by opname - self.per_rank_stats_by_opname = pd.concat( - describe_duration(df.groupby(["OpName", "OpType", "TaskType", "InputShapes"])["Duration"]).assign(Rank=df["Rank"][0]) for df in mapper_res) - self.per_rank_stats_by_opname.sort_values(by=["SumNs"], inplace=True, ascending=False) - - def run(self, context): - super().run(context) - mapper_res = self.mapper_func(context) - self.reducer_func(mapper_res) - - if self._export_type == "db": - self.save_db() - elif self._export_type == "notebook": - self.save_notebook() - else: - print("[ERROR] Unknown export type.") - - def save_notebook(self): - self.dump_data(self.all_rank_stats, os.path.join(self._get_output_dir(), "all_stats.csv")) - self.dump_data(self.per_rank_stats_by_optype, os.path.join(self._get_output_dir(), "rank_stats_by_optype.csv")) - self.dump_data(self.per_rank_stats_by_opname, os.path.join(self._get_output_dir(), "rank_stats_by_opname.csv")) - self.create_notebook("stats.ipynb") - self.add_helper_file("cluster_display.py") - - def save_db(self): - self.dump_data(self.all_rank_stats, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_ALL_RANK_STATS) - self.dump_data(self.per_rank_stats_by_optype, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_PER_RANK_STATS_BY_OPTYPE) - self.dump_data(self.per_rank_stats_by_opname, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_PER_RANK_STATS_BY_OPNAME) diff --git a/profiler/cluster_analyse/analysis/compute_op_sum/stats.ipynb b/profiler/cluster_analyse/analysis/compute_op_sum/stats.ipynb deleted file mode 100644 index c88d2684c..000000000 --- a/profiler/cluster_analyse/analysis/compute_op_sum/stats.ipynb +++ /dev/null @@ -1,164 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Compute Op Summary\n", - "\n", - "集群场景计算类算子数据分析\n", - "\n", - "主要包含以下3个统计内容：\n", - "1. 按算子类型和任务类型分组的，整个集群通信算子耗时的统计情况\n", - "2. 按算子类型和任务类型分组的，每个Rank上计算类算子的耗时情况\n", - "3. 按算子名称、任务类型、输入shape分组的，每个Rank上的计算类算子的耗时情况" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 数据准备" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import display, HTML\n", - "display(HTML(\"\"))\n", - "\n", - "import plotly.offline as pyo\n", - "\n", - "def is_lab_notebook():\n", - " import re\n", - " import psutil\n", - " return any(re.search('jupyter--lab-script', x) for x in psutil.Process().parent().cmdline())\n", - "\n", - "if is_lab_notebook():\n", - " pyo.init_notebook_mode()\n", - "\n", - "import pandas as pd\n", - "pd.options.plotting.backend = \"plotly\"\n", - "pd.set_option(\"display.max_rows\", 100)\n", - "pd.set_option(\"display.width\", 1000)\n", - "\n", - "import cluster_display\n", - "\n", - "all_stats_df = pd.read_csv(\"all_stats.csv\", index_col=\"OpType\")\n", - "rank_stats_by_optype_df = pd.read_csv(\"rank_stats_by_optype.csv\", index_col=\"OpType\")\n", - "rank_stats_by_opname_df = pd.read_csv(\"rank_stats_by_opname.csv\", index_col=\"OpName\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 计算类算子耗时分析\n", - "\n", - "将整个集群所有Rank的计算类算子进行汇总，按算子类型和任务类型分类，统计分析耗时情况，时间单位为微秒(us)\n", - "\n", - "包含以下统计项：\n", - "- Count：算子数量\n", - "- Mean：平均耗时\n", - "- Std：标准差\n", - "- Min：最小值\n", - "- Q1：四分之一分位数\n", - "- Median：中位数\n", - "- Q3：四分之三分位数\n", - "- Max：最大值\n", - "- Sum：总耗时" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "display(all_stats_df)\n", - "fig_all_rank = cluster_display.display_duration_boxplots(None, all_stats_df, x_title=\"OpType\")\n", - "fig_per_rank = cluster_display.display_graph(None, all_stats_df.index, all_stats_df[[\"Q1(Us)\", \"Median(Us)\", \"Q3(Us)\"]], title=\"50% of Distribution\", x_title=\"OpType\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 单个Rank的计算类算子基于算子类型的耗时分析\n", - "将集群内每个Rank的计算类算子进行汇总，按算子类型和任务类型分类，统计分析耗时情况，时间单位为微秒(us)\n", - "\n", - "包含以下统计项：\n", - "- Count：算子数量\n", - "- Mean：平均耗时\n", - "- Std：标准差\n", - "- Min：最小值\n", - "- Q1：四分之一分位数\n", - "- Median：中位数\n", - "- Q3：四分之三分位数\n", - "- Max：最大值\n", - "- Sum：总耗时" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "rank_stats_gdf = rank_stats_by_optype_df.groupby(rank_stats_by_optype_df.index)\n", - "cluster_display.display_stats_per_rank_groups_combobox(rank_stats_gdf)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 单个Rank的计算类算子基于算子名的耗时分析\n", - "\n", - "将集群内每个Rank的计算类算子进行汇总，按算子名称、任务类型、输入shape分类，统计分析耗时情况，时间单位为微秒(us)\n", - "\n", - "包含以下统计项：\n", - "- Count：算子数量\n", - "- Mean：平均耗时\n", - "- Std：标准差\n", - "- Min：最小值\n", - "- Q1：四分之一分位数\n", - "- Median：中位数\n", - "- Q3：四分之三分位数\n", - "- Max：最大值\n", - "- Sum：总耗时" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "rank_stats_gdf = rank_stats_by_opname_df.groupby(rank_stats_by_opname_df.index)\n", - "cluster_display.display_stats_per_rank_groups_combobox(rank_stats_gdf)" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/profiler/cluster_analyse/analysis/hccl_sum/__init__.py b/profiler/cluster_analyse/analysis/hccl_sum/__init__.py deleted file mode 100644 index 7101187a2..000000000 --- a/profiler/cluster_analyse/analysis/hccl_sum/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/profiler/cluster_analyse/analysis/hccl_sum/hccl_sum.py b/profiler/cluster_analyse/analysis/hccl_sum/hccl_sum.py deleted file mode 100644 index da0c575e4..000000000 --- a/profiler/cluster_analyse/analysis/hccl_sum/hccl_sum.py +++ /dev/null @@ -1,133 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pandas as pd -from analysis.base_analysis import BaseRecipeAnalysis -from common_func.constant import Constant -from common_func.utils import describe_duration -from cluster_statistics_export.hccl_sum_export import HcclSumExport - - -class HcclSum(BaseRecipeAnalysis): - - TABLE_ALL_RANK_STATS = "HcclAllRankStats" - TABLE_PER_RANK_STATS = "HcclPerRankStats" - TABLE_TOP_OP_STATS = "HcclTopOpStats" - - TOP_NUM = "top_num" - DEFAULT_TOP_NUM = 15 - - def __init__(self, params): - super().__init__(params) - print("[INFO] HcclSum init.") - self.per_rank_stats = None - self.all_rank_stats = None - self.top_op_stats = None - self.top_num = params.get(self.TOP_NUM, self.DEFAULT_TOP_NUM) - - @property - def base_dir(self): - return os.path.basename(os.path.dirname(__file__)) - - @staticmethod - def _mapper_func(data_map, analysis_class): - df = HcclSumExport(data_map[1], analysis_class).read_export_db() - - if df is None or df.empty: - print(f"[WARNING] There is no stats data in {data_map[1]}.") - return None - - df["Rank"] = data_map[0] - return df - - def mapper_func(self, context): - return context.wait( - context.map( - self._mapper_func, - self._get_rank_db(), - analysis_class=self._recipe_name - ) - ) - - def reducer_func(self, mapper_res): - mapper_res = list(filter(lambda df: df is not None, mapper_res)) - if not mapper_res: - print("[ERROR] Mapper data is None.") - return - self.per_rank_stats = pd.concat( - describe_duration(df.groupby("OpType")["Duration"]).assign(Rank=df["Rank"][0]) for df in mapper_res) - self.per_rank_stats.sort_values(by=["Rank"], inplace=True) - all_op_data = pd.concat(mapper_res) - self.all_rank_stats = describe_duration(all_op_data.groupby("OpType")["Duration"]) - grouped_op_stats = all_op_data.groupby("OpName") - self.top_op_stats = describe_duration(grouped_op_stats["Duration"]).nlargest(self.top_num, "MeanNs") - min_rank = [] - max_rank = [] - for op_name in self.top_op_stats.index: - df = grouped_op_stats.get_group(op_name) - min_rank.append(df[df["Duration"] == df["Duration"].min()]["Rank"].values[0]) - max_rank.append(df[df["Duration"] == df["Duration"].max()]["Rank"].values[0]) - self.top_op_stats["MinRank"] = min_rank - self.top_op_stats["MaxRank"] = max_rank - - def run(self, context): - super().run(context) - if self.top_num <= 0: - print(f"[WARNING] HcclSum: top_num is set to a invalid value, " - f"it will be reset to default value({self.DEFAULT_TOP_NUM}).") - self.top_num = self.DEFAULT_TOP_NUM - mapper_res = self.mapper_func(context) - self.reducer_func(mapper_res) - - if self._export_type == "db": - self.save_db() - elif self._export_type == "notebook": - self.save_notebook() - else: - print("[ERROR] Unknown export type.") - - def save_notebook(self): - self.dump_data(self.all_rank_stats, os.path.join(self._get_output_dir(), "all_stats.csv")) - self.dump_data(self.per_rank_stats, os.path.join(self._get_output_dir(), "rank_stats.csv")) - self.dump_data(self.top_op_stats, os.path.join(self._get_output_dir(), "top_op_stats.csv")) - self.create_notebook("stats.ipynb") - self.add_helper_file("cluster_display.py") - - def save_db(self): - self.dump_data(self.all_rank_stats, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_ALL_RANK_STATS) - self.dump_data(self.per_rank_stats, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_PER_RANK_STATS) - self.dump_data(self.top_op_stats, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_TOP_OP_STATS) - - @classmethod - def add_parser_argument(cls, parser): - BaseRecipeAnalysis.add_parser_argument(parser) - parser.add_argument("--top_num", type=int, help="Duration cost top count", default=cls.DEFAULT_TOP_NUM) - - @classmethod - def parse_argument(cls, args_parsed) -> dict: - argument_dict = BaseRecipeAnalysis.parse_argument(args_parsed) - argument_dict.update({ - cls.TOP_NUM: args_parsed.top_num - }) - return argument_dict - - @classmethod - def get_extra_argument(cls, params) -> dict: - argument_dict = BaseRecipeAnalysis.get_extra_argument(params) - argument_dict.update({ - cls.TOP_NUM: params.get(cls.TOP_NUM, cls.DEFAULT_TOP_NUM) - }) - return argument_dict diff --git a/profiler/cluster_analyse/analysis/hccl_sum/stats.ipynb b/profiler/cluster_analyse/analysis/hccl_sum/stats.ipynb deleted file mode 100644 index 87f8c6d73..000000000 --- a/profiler/cluster_analyse/analysis/hccl_sum/stats.ipynb +++ /dev/null @@ -1,162 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# HCCL Summary\n", - "\n", - "集群场景Hccl算子数据分析\n", - "\n", - "主要包含以下3个统计内容：\n", - "1. 按算子类型分组的，整个集群通信算子耗时的统计情况\n", - "2. 按算子类型分组的，每个Rank上通信算子的耗时情况\n", - "3. 整个集群平均耗时最久的TOP通信算子" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 数据准备" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import display, HTML\n", - "display(HTML(\"\"))\n", - "\n", - "import plotly.offline as pyo\n", - "\n", - "def is_lab_notebook():\n", - " import re\n", - " import psutil\n", - " return any(re.search('jupyter--lab-script', x) for x in psutil.Process().parent().cmdline())\n", - "\n", - "if is_lab_notebook():\n", - " pyo.init_notebook_mode()\n", - "\n", - "import pandas as pd\n", - "pd.options.plotting.backend = \"plotly\"\n", - "pd.set_option(\"display.max_rows\", 100)\n", - "pd.set_option(\"display.width\", 1000)\n", - "\n", - "import cluster_display\n", - "\n", - "all_stats_df = pd.read_csv(\"all_stats.csv\", index_col=\"OpType\")\n", - "rank_stats_df = pd.read_csv(\"rank_stats.csv\", index_col=\"OpType\")\n", - "top_op_stats_df = pd.read_csv(\"top_op_stats.csv\", index_col=\"OpName\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 集群通信算子耗时分析\n", - "\n", - "将整个集群所有Rank的通信算子进行汇总，按算子类型分类，统计分析耗时情况，时间单位为微秒(us)\n", - "\n", - "包含以下统计项：\n", - "- Count：算子数量\n", - "- Mean：平均耗时\n", - "- Std：标准差\n", - "- Min：最小值\n", - "- Q1：四分之一分位数\n", - "- Median：中位数\n", - "- Q3：四分之三分位数\n", - "- Max：最大值\n", - "- Sum：总耗时" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "display(all_stats_df)\n", - "fig_all_rank = cluster_display.display_duration_boxplots(None, all_stats_df, x_title=\"Hccl OpType\")\n", - "fig_per_rank = cluster_display.display_graph(None, all_stats_df.index, all_stats_df[[\"Q1(Us)\", \"Median(Us)\", \"Q3(Us)\"]], title=\"50% of Distribution\", x_title=\"Hccl OpType\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 集群Rank通信算子耗时分析\n", - "\n", - "将集群内每个Rank的通信算子进行汇总，按算子类型分类，统计分析耗时情况，时间单位为微秒(us)\n", - "\n", - "包含以下统计项：\n", - "- Count：算子数量\n", - "- Mean：平均耗时\n", - "- Std：标准差\n", - "- Min：最小值\n", - "- Q1：四分之一分位数\n", - "- Median：中位数\n", - "- Q3：四分之三分位数\n", - "- Max：最大值\n", - "- Sum：总耗时" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "rank_stats_gdf = rank_stats_df.groupby(rank_stats_df.index)\n", - "cluster_display.display_stats_per_rank_groups_combobox(rank_stats_gdf)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 集群TOP-N通信算子耗时分析\n", - "\n", - "统计集群内耗时最多的TOP-N通信算子，时间单位为微秒(us)\n", - "\n", - "包含以下统计项：\n", - "- Count：算子数量\n", - "- Mean：平均耗时\n", - "- Std：标准差\n", - "- Min：最小值\n", - "- Q1：四分之一分位数\n", - "- Median：中位数\n", - "- Q3：四分之三分位数\n", - "- Max：最大值\n", - "- Sum：总耗时\n", - "- MinRank：耗时最少算子所在的Rank\n", - "- MaxRank：耗时最长算子所在的Rank" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "display(top_op_stats_df)\n", - "fig_top_op = cluster_display.display_duration_boxplots(None, top_op_stats_df, x_title=\"Hccl OpName\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/profiler/cluster_analyse/analysis/mstx_sum/__init__.py b/profiler/cluster_analyse/analysis/mstx_sum/__init__.py deleted file mode 100644 index 7101187a2..000000000 --- a/profiler/cluster_analyse/analysis/mstx_sum/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/profiler/cluster_analyse/analysis/mstx_sum/mstx_sum.py b/profiler/cluster_analyse/analysis/mstx_sum/mstx_sum.py deleted file mode 100644 index 46a0e18ab..000000000 --- a/profiler/cluster_analyse/analysis/mstx_sum/mstx_sum.py +++ /dev/null @@ -1,204 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -import pandas as pd -from collections import namedtuple -from analysis.base_analysis import BaseRecipeAnalysis -from common_func.constant import Constant -from common_func.utils import describe_duration -from cluster_statistics_export.mstx_mark_export import MstxMarkExport -from cluster_statistics_export.mstx_step_export import MstxStepExport - - -MarkInfo = namedtuple("MarkInfo", ["name", "framework_duration", "cann_duration", "device_duration", - "tid", "start_ns"]) - - -def format_mark_info(df: pd.DataFrame, start_idx, stop_idx, name) -> MarkInfo: - start_series = df.iloc[start_idx] - stop_series = df.iloc[stop_idx] - return MarkInfo( - name=name, - framework_duration=float(stop_series["framework_ts"]-start_series["framework_ts"]), - cann_duration=float(stop_series["cann_ts"]-start_series["cann_ts"]), - device_duration=float(stop_series["device_ts"]-start_series["device_ts"]), - tid=start_series["tid"], - start_ns=start_series["cann_ts"] - ) - - -def rename_mark_msg_name(mark_stats_df: pd.DataFrame): - msg_idx_counter = {} - for idx, mark_info in enumerate(mark_stats_df.itertuples(index=False)): - msg_idx_counter.setdefault(mark_info.step_id, {}).setdefault(mark_info.name, []).append(idx) - for msg_dict in msg_idx_counter.values(): - for msg, idx_list in msg_dict.items(): - if len(idx_list) <= 1: - continue - for i, idx in enumerate(idx_list): - mark_stats_df.loc[idx, 'name'] = f"{msg}_{i}" - - -def compute_step_id(mark_stat, step_stats_df: pd.DataFrame): - for step_info in step_stats_df.itertuples(index=False): - if step_info.start_ns <= mark_stat.start_ns <= step_info.end_ns: - return step_info.step_id - print(f"[WARNING] {mark_stat.name} is not in any step.") - return 0 - - -def format_columns(df: pd.DataFrame): - formatted_df = df.rename( - { - "framework_duration": "FrameworkDurationNs", - "cann_duration": "CannDurationNs", - "device_duration": "DeviceDurationNs", - "duration": "DurationNs", - "step_id": "StepId", - "tid": "Tid", - "name": "Name" - }, - axis="columns" - ) - cols = [col for col in formatted_df.columns if not col.endswith("_ns") and col not in {"Tid"}] - return formatted_df[cols] - - -class MstxSum(BaseRecipeAnalysis): - - TABLE_FRAMEWORK_STATS = "MSTXAllFrameworkStats" - TABLE_CANN_STATS = "MSTXAllCannStats" - TABLE_DEVICE_STATS = "MSTXAllDeviceStats" - TABLE_MARK_STATS = "MSTXMarkStats" - - START_SUFFIX = "_start" - STOP_SUFFIX = "_stop" - - def __init__(self, params): - super().__init__(params) - print("[INFO] MstxSum init.") - self.mark_stats = None - self.all_fwk_stats = None - self.all_cann_stats = None - self.all_device_stats = None - - @property - def base_dir(self): - return os.path.basename(os.path.dirname(__file__)) - - @staticmethod - def _mapper_func(data_map, analysis_class): - step_df = MstxStepExport(data_map[1], analysis_class).read_export_db() - if step_df is None or step_df.empty: - step_df = pd.DataFrame({"start_ns": [0], "end_ns": [float("inf")], "step_id": [0]}) - mark_df = MstxMarkExport(data_map[1], analysis_class).read_export_db() - if mark_df is None or mark_df.empty: - print(f"[WARNING] There is no mark data in {data_map[1]}.") - return None - mark_df["framework_ts"] = mark_df["framework_ts"].astype("int64") - - mark_info = {} - mark_res = [] - mismatch_msg = [] - for idx, row in enumerate(mark_df.itertuples(index=False)): - if row.msg.endswith(MstxSum.START_SUFFIX): - msg = row.msg[:-len(MstxSum.START_SUFFIX)] - mark_info.setdefault(row.tid, {}).setdefault(msg, []).append(idx) - elif row.msg.endswith(MstxSum.STOP_SUFFIX): - msg = row.msg[:-len(MstxSum.STOP_SUFFIX)] - idx_list = mark_info.get(row.tid, {}).get(msg, []) - if not idx_list: - mismatch_msg.append((row.msg, idx)) - continue - start_idx = idx_list.pop() - mark_res.append(format_mark_info(mark_df, start_idx, idx, msg)) - - # 统计未匹配上的mark信息 - for msg_info in mark_info.values(): - for msg, idx_list in msg_info.items(): - if not idx_list: - continue - mismatch_msg.extend((msg + MstxSum.START_SUFFIX, idx) for idx in idx_list) - if mismatch_msg: - mismatch_msg.sort(key=lambda msg: msg[1]) - print(f"[WARNING] The following mark messages do not match anyone in " - f"rank {data_map[0]}: {','.join(msg[0] for msg in mismatch_msg)}.") - - mark_stats_df = pd.DataFrame(mark_res).assign(Rank=data_map[0]) - mark_stats_df["step_id"] = mark_stats_df.apply(compute_step_id, axis=1, step_stats_df=step_df) - rename_mark_msg_name(mark_stats_df) - mark_stats_df = format_columns(mark_stats_df).set_index("Name", drop=True) - return mark_stats_df - - def mapper_func(self, context): - return context.wait( - context.map( - self._mapper_func, - self._get_rank_db(), - analysis_class=self._recipe_name - ) - ) - - def reducer_func(self, mapper_res): - mapper_res = list(filter(lambda df: df is not None, mapper_res)) - if not mapper_res: - print("[ERROR] Mapper data is None.") - return - self.mark_stats = pd.concat(mapper_res) - all_fwk_stats = [] - all_cann_stats = [] - all_device_stats = [] - mark_step_df = self.mark_stats.groupby("StepId") - for step_id, df in mark_step_df: - name_gdf = df.groupby("Name") - fwk_stats = describe_duration(name_gdf["FrameworkDurationNs"]).assign(StepId=step_id) - fwk_stats.sort_values(by=["SumNs"], inplace=True, ascending=False) - all_fwk_stats.append(fwk_stats) - cann_stats = describe_duration(name_gdf["CannDurationNs"]).assign(StepId=step_id) - cann_stats.sort_values(by=["SumNs"], inplace=True, ascending=False) - all_cann_stats.append(cann_stats) - device_stats = describe_duration(name_gdf["DeviceDurationNs"]).assign(StepId=step_id) - device_stats.sort_values(by=["SumNs"], inplace=True, ascending=False) - all_device_stats.append(device_stats) - self.all_fwk_stats = pd.concat(all_fwk_stats) - self.all_cann_stats = pd.concat(all_cann_stats) - self.all_device_stats = pd.concat(all_device_stats) - - def run(self, context): - super().run(context) - mapper_res = self.mapper_func(context) - self.reducer_func(mapper_res) - - if self._export_type == "db": - self.save_db() - elif self._export_type == "notebook": - self.save_notebook() - else: - print("[ERROR] Unknown export type.") - - def save_notebook(self): - self.dump_data(self.mark_stats, os.path.join(self._get_output_dir(), "mark_stats.csv")) - self.dump_data(self.all_fwk_stats, os.path.join(self._get_output_dir(), "all_fwk_stats.csv")) - self.dump_data(self.all_cann_stats, os.path.join(self._get_output_dir(), "all_cann_stats.csv")) - self.dump_data(self.all_device_stats, os.path.join(self._get_output_dir(), "all_device_stats.csv")) - self.create_notebook("stats.ipynb") - self.add_helper_file("cluster_display.py") - - def save_db(self): - self.dump_data(self.mark_stats, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_MARK_STATS) - self.dump_data(self.all_fwk_stats, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_FRAMEWORK_STATS) - self.dump_data(self.all_cann_stats, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_CANN_STATS) - self.dump_data(self.all_device_stats, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, self.TABLE_DEVICE_STATS) diff --git a/profiler/cluster_analyse/analysis/mstx_sum/stats.ipynb b/profiler/cluster_analyse/analysis/mstx_sum/stats.ipynb deleted file mode 100644 index 84672bc72..000000000 --- a/profiler/cluster_analyse/analysis/mstx_sum/stats.ipynb +++ /dev/null @@ -1,180 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# MSTX Summary\n", - "\n", - "集群场景MSTX打点数据分析\n", - "\n", - "主要包含以下2个统计内容：\n", - "1. 按Step分组的，整个集群MSTX打点数据的统计情况\n", - "2. 按Name分组的，每个Rank上MSTX打点数据的统计情况" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 数据准备" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from IPython.display import display, HTML\n", - "display(HTML(\"\"))\n", - "\n", - "import plotly.offline as pyo\n", - "\n", - "def is_lab_notebook():\n", - " import re\n", - " import psutil\n", - " return any(re.search('jupyter--lab-script', x) for x in psutil.Process().parent().cmdline())\n", - "\n", - "if is_lab_notebook():\n", - " pyo.init_notebook_mode()\n", - "\n", - "import pandas as pd\n", - "pd.options.plotting.backend = \"plotly\"\n", - "pd.set_option(\"display.max_rows\", 100)\n", - "pd.set_option(\"display.width\", 1000)\n", - "\n", - "import cluster_display\n", - "\n", - "all_fwk_stats_gdf = pd.read_csv(\"all_fwk_stats.csv\", index_col=\"Name\").groupby(\"StepId\")\n", - "all_cann_stats_gdf = pd.read_csv(\"all_cann_stats.csv\", index_col=\"Name\").groupby(\"StepId\")\n", - "all_device_stats_gdf = pd.read_csv(\"all_device_stats.csv\", index_col=\"Name\").groupby(\"StepId\")\n", - "mark_stats_df = pd.read_csv(\"mark_stats.csv\", index_col=\"Name\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 集群MSTX数据分析\n", - "\n", - "将整个集群所有Rank的MSTX数据进行汇总，按Step划分，统计分析耗时情况，时间单位为微秒(us)\n", - "打点数据分为三种：\n", - "1. 框架侧耗时：Framework Time\n", - "2. Cann侧耗时：Cann Time\n", - "3. Device侧耗时：Devcie Time\n", - "\n", - "3种数据都包含以下统计项：\n", - "- Count：数量\n", - "- Mean：平均耗时\n", - "- Std：标准差\n", - "- Min：最小值\n", - "- Q1：四分之一分位数\n", - "- Median：中位数\n", - "- Q3：四分之三分位数\n", - "- Max：最大值\n", - "- Sum：总耗时" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def display_stats_mstx_step_combobox(selected, args):\n", - " step = selected\n", - " fwk_stats_gdf, cann_stats_gdf, device_stats_gdf = args\n", - " fwk_df = fwk_stats_gdf.get_group(step)\n", - " cann_df = cann_stats_gdf.get_group(step)\n", - " device_df = device_stats_gdf.get_group(step)\n", - " figs = []\n", - " display(HTML(\"

Framework Time Stats

\"))\n", - " display(fwk_df)\n", - " cluster_display.display_duration_boxplots(figs, fwk_df, title=\"Framework Time\", x_title=\"Name\", y_title=\"Time\")\n", - " display(HTML(\"

Cann Time Stats

\"))\n", - " display(cann_df)\n", - " cluster_display.display_duration_boxplots(figs, cann_df, title=\"Cann Time\", x_title=\"Name\", y_title=\"Time\")\n", - " display(HTML(\"

Device Time Stats

\"))\n", - " display(device_df)\n", - " cluster_display.display_duration_boxplots(figs, device_df, title=\"Device Time\", x_title=\"Name\", y_title=\"Time\")\n", - "\n", - "steps = list(all_fwk_stats_gdf.groups.keys())\n", - "if steps:\n", - " cluster_display.display_stats_optional_combobox(steps, display_stats_mstx_step_combobox, \n", - " [all_fwk_stats_gdf, all_cann_stats_gdf, all_device_stats_gdf], \"Step:\")\n", - "else:\n", - " print(\"There is no step in stats, so no need to display\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "### 集群Rank MSTX数据分析\n", - "\n", - "将集群内每个Rank的MSTX数据进行汇总，按打点Name分类，统计分析耗时情况，时间单位为微秒(us)\n", - "\n", - "包含以下统计项：\n", - "- Name：打点名称\n", - "- FrameworkDuration(Us)：框架侧耗时\n", - "- CannDuration(Us)：Cann侧耗时\n", - "- DeviceDuration(Us)：Device侧耗时\n", - "- Rank：Rank序号\n", - "- StepId：Step序号" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def display_mstx_duration_by_rank(selected, args):\n", - " mark_stats_gdf = args\n", - " df = mark_stats_gdf.get_group(selected).sort_values(\"Rank\")\n", - " display(df)\n", - " fwk_duration = []\n", - " cann_duration = []\n", - " device_duration = []\n", - " step_ids = []\n", - " for step_id, step_df in df.groupby(\"StepId\"):\n", - " fwk_duration.append((step_id, step_df[\"FrameworkDuration(Us)\"].values))\n", - " cann_duration.append((step_id, step_df[\"CannDuration(Us)\"].values))\n", - " device_duration.append((step_id, step_df[\"DeviceDuration(Us)\"].values))\n", - " step_ids.append(step_id)\n", - " fwk_df = pd.concat([pd.Series(dur, name=step_id) for step_id, dur in fwk_duration], axis=1)\n", - " cann_df = pd.concat([pd.Series(dur, name=step_id) for step_id, dur in cann_duration], axis=1)\n", - " device_df = pd.concat([pd.Series(dur, name=step_id) for step_id, dur in device_duration], axis=1)\n", - " figs = []\n", - " ranks = df[\"Rank\"].drop_duplicates()\n", - " cluster_display.display_graph(figs, ranks, fwk_df[step_ids],\n", - " title=\"Framework Time\", x_title=\"Rank\", y_title=\"Time\", legend_title=\"Step\")\n", - " cluster_display.display_graph(figs, ranks, cann_df[step_ids],\n", - " title=\"Cann Time\", x_title=\"Rank\", y_title=\"Time\", legend_title=\"Step\")\n", - " cluster_display.display_graph(figs, ranks, device_df[step_ids],\n", - " title=\"Device Time\", x_title=\"Rank\", y_title=\"Time\", legend_title=\"Step\")\n", - "\n", - "mark_stats_gdf = mark_stats_df.groupby(mark_stats_df.index)\n", - "names = list(mark_stats_gdf.groups.keys())\n", - "if steps:\n", - " cluster_display.display_stats_optional_combobox(names, display_mstx_duration_by_rank, mark_stats_gdf, \"Name:\")\n", - "else:\n", - " print(\"There is no mark name in stats, so no need to display\")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.12.1" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index a8d01dcfe..171417c88 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -22,36 +22,9 @@ from communication_group.communication_group_generator import CommunicationGroup from common_func.constant import Constant from common_func.file_manager import FileManager from common_func.path_manager import PathManager -from common_func import analysis_loader from analysis.analysis_facade import AnalysisFacade COMM_FEATURE_LIST = ['all', 'communication_time', 'communication_matrix'] -ALL_FEATURE_LIST = ['all', 'communication_time', 'communication_matrix', 'cann_api_sum', 'hccl_sum', 'compute_op_sum', - 'mstx_sum'] - - -def get_analysis_args(analysis_class, analysis_args): - parser = argparse.ArgumentParser(description="custom analysis args") - parser.add_argument("--parallel_mode", type=str, help="context mode", default="concurrent") - parser.add_argument("--export_type", type=str, help="export type", default="db") - analysis_class[1].add_parser_argument(parser) - return parser.parse_args(analysis_args) - -def parse_specific_params(analysis_name, analysis_args): - analysis_class = analysis_loader.get_class_from_name(analysis_name) - if not analysis_class: - print("[ERROR] undefined analysis.") - return None - - args_parsed = get_analysis_args(analysis_class, analysis_args) - specific_params = { - Constant.RECIPE_NAME: analysis_class[0], - Constant.RECIPE_CLASS: analysis_class[1], - Constant.PARALLEL_MODE: args_parsed.parallel_mode, - Constant.EXPORT_TYPE: args_parsed.export_type - } - specific_params.update(analysis_class[1].parse_argument(args_parsed)) - return specific_params class Interface: ASCEND_PT = "ascend_pt" @@ -96,51 +69,29 @@ class Interface: if data_type == Constant.INVALID: print("[ERROR] The current folder contains both DB and other files. Please check.") return - if self.analysis_mode not in COMM_FEATURE_LIST: - if data_type != Constant.DB: - print("[ERROR] The current analysis node only supports DB as input data. Please check.") - return - FileManager.create_output_dir(self.collection_path, is_overwrite=True) - params = { - Constant.COLLECTION_PATH: self.collection_path, - Constant.DATA_MAP: data_map, - Constant.DATA_TYPE: data_type, - Constant.RECIPE_NAME: self.origin_params.get(Constant.RECIPE_NAME, ""), - Constant.RECIPE_CLASS: self.origin_params.get(Constant.RECIPE_CLASS), - Constant.PARALLEL_MODE: self.origin_params.get(Constant.PARALLEL_MODE, ""), - Constant.EXPORT_TYPE: self.origin_params.get(Constant.EXPORT_TYPE, "") - } - params.update(params[Constant.RECIPE_CLASS].get_extra_argument(self.origin_params)) - AnalysisFacade(params).recipe_analyze() - else: - FileManager.create_output_dir(self.collection_path) - params = { - Constant.COLLECTION_PATH: self.collection_path, - Constant.DATA_MAP: data_map, - Constant.ANALYSIS_MODE: self.analysis_mode, - Constant.DATA_TYPE: data_type - } - comm_data_dict = CommunicationGroupGenerator(params).generate() - params[Constant.COMM_DATA_DICT] = comm_data_dict - AnalysisFacade(params).cluster_analyze() + FileManager.create_output_dir(self.collection_path) + params = { + Constant.COLLECTION_PATH: self.collection_path, + Constant.DATA_MAP: data_map, + Constant.ANALYSIS_MODE: self.analysis_mode, + Constant.DATA_TYPE: data_type + } + comm_data_dict = CommunicationGroupGenerator(params).generate() + params[Constant.COMM_DATA_DICT] = comm_data_dict + AnalysisFacade(params).cluster_analyze() def cluster_analysis_main(args=None): parser = argparse.ArgumentParser(description="cluster analysis module") parser.add_argument('-d', '--collection_path', type=str, required=True, help="profiling data path") - parser.add_argument('-m', '--mode', choices=ALL_FEATURE_LIST, + parser.add_argument('-m', '--mode', choices=COMM_FEATURE_LIST, default='all', help="different analysis mode") - args_parsed, args_remained = parser.parse_known_args(args=args) + args_parsed, _ = parser.parse_known_args(args=args) parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, Constant.ANALYSIS_MODE: args_parsed.mode } - if args_parsed.mode in COMM_FEATURE_LIST: - if args_remained: - print(f"[ERROR] The specific argument {args_remained} is not supported for communication analysis.") - return - else: - parameter.update(parse_specific_params(args_parsed.mode, args_remained)) + Interface(parameter).run() diff --git a/profiler/cluster_analyse/cluster_statistics_export/__init__.py b/profiler/cluster_analyse/cluster_statistics_export/__init__.py deleted file mode 100644 index 7101187a2..000000000 --- a/profiler/cluster_analyse/cluster_statistics_export/__init__.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. diff --git a/profiler/cluster_analyse/cluster_statistics_export/cann_api_sum_export.py b/profiler/cluster_analyse/cluster_statistics_export/cann_api_sum_export.py deleted file mode 100644 index 578ee937b..000000000 --- a/profiler/cluster_analyse/cluster_statistics_export/cann_api_sum_export.py +++ /dev/null @@ -1,65 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cluster_statistics_export.stats_export import StatsExport - -QUERY = """ -WITH - summary as ( - SELECT - name, - sum(endNs - startNs) AS duration, - count (*) AS num, - avg(endNs - startNs) AS avg_duration, - min(endNs - startNs) AS min_duration, - median(endNs - startNs) AS med_duration, - max(endNs - startNs) AS max_duration, - stdev(endNs - startNs) AS stdev_duration, - lower_quartile(endNs - startNs) AS lower_quartile_duration, - upper_quartile(endNs - startNs) AS upper_quartile_duration - FROM - CANN_API - GROUP BY name - ), - totals AS ( - SELECT sum(duration) AS total - FROM summary - ) -SELECT - ids.value AS "name", - round(summary.duration * 100.0 / (SELECT total FROM totals), 2) AS "durationRatio", - summary.duration AS "totalTimeNs", - summary.num AS "totalCount", - round(summary.avg_duration, 1) AS "averageNs", - round(summary.min_duration, 1) AS "minNs", - round(summary.lower_quartile_duration, 1) AS "Q1Ns", - round(summary.med_duration, 1) AS "medNs", - round(summary.upper_quartile_duration, 1) AS "Q3Ns", - round(summary.max_duration, 1) AS "maxNs", - round(summary.stdev_duration, 1) AS "stdev" -FROM - summary -LEFT JOIN - STRING_IDS AS ids - ON ids.id == summary.name -ORDER BY 2 DESC; - """ - - -class CannApiSumExport(StatsExport): - - def __init__(self, db_path, recipe_name): - super().__init__(db_path, recipe_name) - self._query = QUERY diff --git a/profiler/cluster_analyse/cluster_statistics_export/compute_op_sum_export.py b/profiler/cluster_analyse/cluster_statistics_export/compute_op_sum_export.py deleted file mode 100644 index d70c69610..000000000 --- a/profiler/cluster_analyse/cluster_statistics_export/compute_op_sum_export.py +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cluster_statistics_export.stats_export import StatsExport - - -QUERY = """ -SELECT - NAME_IDS.value AS "OpName", - OPTYPE_IDS.value AS "OpType", - TASKTYPE_IDS.value AS "TaskType", - INPUTSHAPES_IDS.value AS "InputShapes", - round(TASK.endNs - TASK.startNs) AS "Duration" -FROM - COMPUTE_TASK_INFO -LEFT JOIN TASK - ON TASK.globalTaskId == COMPUTE_TASK_INFO.globalTaskId -LEFT JOIN - STRING_IDS AS NAME_IDS - ON NAME_IDS.id == COMPUTE_TASK_INFO.name -LEFT JOIN - STRING_IDS AS OPTYPE_IDS - ON OPTYPE_IDS.id == COMPUTE_TASK_INFO.opType -LEFT JOIN - STRING_IDS AS TASKTYPE_IDS - ON TASKTYPE_IDS.id == COMPUTE_TASK_INFO.taskType -LEFT JOIN - STRING_IDS AS INPUTSHAPES_IDS - ON INPUTSHAPES_IDS.id == COMPUTE_TASK_INFO.inputShapes - """ - - -class ComputeOpSumExport(StatsExport): - - def __init__(self, db_path, recipe_name): - super().__init__(db_path, recipe_name) - self._query = QUERY diff --git a/profiler/cluster_analyse/cluster_statistics_export/hccl_sum_export.py b/profiler/cluster_analyse/cluster_statistics_export/hccl_sum_export.py deleted file mode 100644 index f695949de..000000000 --- a/profiler/cluster_analyse/cluster_statistics_export/hccl_sum_export.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cluster_statistics_export.stats_export import StatsExport - - -QUERY = """ -SELECT - NAME_IDS.value AS "OpName", - TYPE_IDS.value AS "OpType", - round(endNs - startNs) AS "Duration" -FROM - COMMUNICATION_OP -LEFT JOIN - STRING_IDS AS TYPE_IDS - ON TYPE_IDS.id == COMMUNICATION_OP.opType -LEFT JOIN - STRING_IDS AS NAME_IDS - ON NAME_IDS.id == COMMUNICATION_OP.opName - """ - - -class HcclSumExport(StatsExport): - - def __init__(self, db_path, recipe_name): - super().__init__(db_path, recipe_name) - self._query = QUERY diff --git a/profiler/cluster_analyse/cluster_statistics_export/mstx_mark_export.py b/profiler/cluster_analyse/cluster_statistics_export/mstx_mark_export.py deleted file mode 100644 index ac5355c02..000000000 --- a/profiler/cluster_analyse/cluster_statistics_export/mstx_mark_export.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cluster_statistics_export.stats_export import StatsExport - - -QUERY = """ -WITH - FRAMEWORK_API AS ( - SELECT - PYTORCH_API.startNs, - CONNECTION_IDS.connectionId - FROM - PYTORCH_API - LEFT JOIN - CONNECTION_IDS - ON PYTORCH_API.connectionId == CONNECTION_IDS.id - ) -SELECT - MSG_IDS.value AS "msg", - MSTX_EVENTS.startNs AS "cann_ts", - TASK.startNs AS "device_ts", - FRAMEWORK_API.startNs AS "framework_ts", - MSTX_EVENTS.globalTid AS "tid" -FROM - MSTX_EVENTS -LEFT JOIN - TASK - ON MSTX_EVENTS.connectionId == TASK.connectionId -LEFT JOIN - FRAMEWORK_API - ON MSTX_EVENTS.connectionId == FRAMEWORK_API.connectionId -LEFT JOIN - STRING_IDS AS MSG_IDS - ON MSTX_EVENTS.message == MSG_IDS.id -ORDER BY - MSTX_EVENTS.startNs - """ - - -class MstxMarkExport(StatsExport): - - def __init__(self, db_path, recipe_name): - super().__init__(db_path, recipe_name) - self._query = QUERY diff --git a/profiler/cluster_analyse/cluster_statistics_export/mstx_step_export.py b/profiler/cluster_analyse/cluster_statistics_export/mstx_step_export.py deleted file mode 100644 index c257ce675..000000000 --- a/profiler/cluster_analyse/cluster_statistics_export/mstx_step_export.py +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from cluster_statistics_export.stats_export import StatsExport - - -QUERY = """ -SELECT - id AS "step_id", - startNs AS "start_ns", - endNs AS "end_ns" -FROM - STEP_TIME -ORDER BY - startNs - """ - - -class MstxStepExport(StatsExport): - - def __init__(self, db_path, recipe_name): - super().__init__(db_path, recipe_name) - self._query = QUERY diff --git a/profiler/cluster_analyse/cluster_statistics_export/stats_export.py b/profiler/cluster_analyse/cluster_statistics_export/stats_export.py deleted file mode 100644 index e6d98f48e..000000000 --- a/profiler/cluster_analyse/cluster_statistics_export/stats_export.py +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import pandas as pd - -from common_func.db_manager import DBManager -from common_func.constant import Constant - - -class StatsExport: - - def __init__(self, db_path, analysis_class): - self._db_path = db_path - self._analysis_class = analysis_class - self._query = None - - def get_query(self): - return self._query - - def read_export_db(self): - query = self.get_query() - if query is None: - print(f"[ERROR] query is None.") - return - conn, cursor = DBManager.create_connect_db(self._db_path, Constant.ANALYSIS) - data = pd.read_sql(query, conn) - DBManager.destroy_db_connect(conn, cursor) - return data diff --git a/profiler/cluster_analyse/common_func/analysis_loader.py b/profiler/cluster_analyse/common_func/analysis_loader.py deleted file mode 100644 index 55e7dbc6e..000000000 --- a/profiler/cluster_analyse/common_func/analysis_loader.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import importlib -import inspect -import sys - -from common_func.constant import Constant -from analysis.base_analysis import BaseRecipeAnalysis - -def is_analysis_class(obj): - return inspect.isclass(obj) and issubclass(obj, BaseRecipeAnalysis) and obj != BaseRecipeAnalysis - -def get_class_from_name(analysis_name : str): - sys.path.append(Constant.ANALYSIS_PATH) - analysis_path = f"analysis.{analysis_name}.{analysis_name}" - module = None - try: - module = importlib.import_module(analysis_path) - except Exception as e: - print(f"[ERROR] {analysis_path} not find:{e}") - - specific_analysis = inspect.getmembers(module, is_analysis_class) - if not specific_analysis: - print(f"[ERROR] {analysis_name} not found.") - return specific_analysis[0] diff --git a/profiler/cluster_analyse/common_func/constant.py b/profiler/cluster_analyse/common_func/constant.py index 80f0374c1..2922d6a90 100644 --- a/profiler/cluster_analyse/common_func/constant.py +++ b/profiler/cluster_analyse/common_func/constant.py @@ -106,13 +106,3 @@ class Constant(object): CONFIG = "config" EXPER_CONFIG = "experimental_config" EXPORT_TYPE = "_export_type" - - # recipe config - ANALYSIS = "analysis" - RECIPE_NAME = "recipe_name" - RECIPE_CLASS = "recipe_class" - PARALLEL_MODE = "parallel_mode" - CLUSTER_CUSTOM_ANALYSE_PATH = os.path.abspath(os.path.dirname(__file__)) - ANALYSIS_PATH = os.path.join(CLUSTER_CUSTOM_ANALYSE_PATH, 'analysis') - - CONCURRENT_MODE = "concurrent" \ No newline at end of file diff --git a/profiler/cluster_analyse/common_func/context.py b/profiler/cluster_analyse/common_func/context.py deleted file mode 100644 index 4e3d544d3..000000000 --- a/profiler/cluster_analyse/common_func/context.py +++ /dev/null @@ -1,85 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import os -from functools import partial -from concurrent import futures -from common_func.constant import Constant - - -class Context(object): - """abstract base class""" - - ctx_map = None - - @classmethod - def create_context(cls, mode=Constant.CONCURRENT_MODE): - if cls.ctx_map is None: - keys = [Constant.CONCURRENT_MODE] - values = [ConcurrentContext] - cls.ctx_map = dict(zip(keys, values)) - - if mode not in cls.ctx_map: - raise NotImplementedError("mode must be in {}".format(keys)) - - return cls.ctx_map[mode]() - - def __init__(self): - print("[INFO] context {} initialized.".format(self._mode)) - - def __enter__(self): - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.close() - if exc_type is not None: - print(f"[ERROR] Failed to exit context: {exc_val}") - - def launch(self, func, *args, **kwargs): - raise NotImplementedError - - def map(self, func, *iterables, **kwargs): - raise NotImplementedError - - def wait(self, waitable): - raise NotImplementedError - -class ConcurrentContext(Context): - - def __init__(self, executor=None): - self._mode = Constant.CONCURRENT_MODE - super().__init__() - self._custom = executor is None - self._executor = executor or futures.ProcessPoolExecutor(max_workers=os.cpu_count()) - - def __enter__(self): - if self._executor is None: - raise RuntimeError("executor is None") - return self - - def close(self): - if self._custom: - self._executor.shutdown(wait=True) - self._executor = None - - def launch(self, func, *args, **kwargs): - return self._executor.submit(func, *args, **kwargs).result() - - def map(self, func, *iterables, **kwargs): - partial_func = partial(func, **kwargs) - return list(self._executor.map(partial_func, *iterables)) - - def wait(self, waitable): - return waitable \ No newline at end of file diff --git a/profiler/cluster_analyse/common_func/db_manager.py b/profiler/cluster_analyse/common_func/db_manager.py index c0d6ad89b..1aa7ed874 100644 --- a/profiler/cluster_analyse/common_func/db_manager.py +++ b/profiler/cluster_analyse/common_func/db_manager.py @@ -20,7 +20,6 @@ from common_func.constant import Constant from common_func.empty_class import EmptyClass from common_func.file_manager import check_db_path_valid from common_func.tables_config import TablesConfig -from common_func.sql_extention_func import SqlExtentionAggregateFunc class DBManager: """ @@ -42,12 +41,6 @@ class DBManager: print(f"[ERROR] {err}") return EmptyClass("empty conn"), EmptyClass("empty curs") try: - if mode == Constant.ANALYSIS: - try: - for func_name, params_count, class_name in SqlExtentionAggregateFunc: - conn.create_aggregate(func_name, params_count, class_name) - except sqlite3.Error as err: - print(f"[ERROR] {err}") if isinstance(conn, sqlite3.Connection): curs = conn.cursor() os.chmod(db_path, Constant.FILE_AUTHORITY) diff --git a/profiler/cluster_analyse/common_func/sql_extention_func.py b/profiler/cluster_analyse/common_func/sql_extention_func.py deleted file mode 100644 index 987a0d436..000000000 --- a/profiler/cluster_analyse/common_func/sql_extention_func.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np - - -class Median: - - def __init__(self) -> None: - self.data = [] - - def step(self, value) -> None: - self.data.append(value) - - def finalize(self): - return np.median(self.data) - - -class LowerQuartile: - - def __init__(self) -> None: - self.data = [] - - def step(self, value) -> None: - self.data.append(value) - - def finalize(self): - return np.quantile(self.data, 0.25) - - -class UpperQuartile: - - def __init__(self) -> None: - self.data = [] - - def step(self, value) -> None: - self.data.append(value) - - def finalize(self): - return np.quantile(self.data, 0.75) - - -class StandardDeviation: - - def __init__(self) -> None: - self.data = [] - - def step(self, value) -> None: - self.data.append(value) - - def finalize(self): - return np.std(self.data) - - -# func_name, params_count, class -SqlExtentionAggregateFunc = [ - ('median', 1, Median), - ('lower_quartile', 1, LowerQuartile), - ('upper_quartile', 1, UpperQuartile), - ('stdev', 1, StandardDeviation) -] diff --git a/profiler/cluster_analyse/common_func/utils.py b/profiler/cluster_analyse/common_func/utils.py deleted file mode 100644 index 0a20a5c23..000000000 --- a/profiler/cluster_analyse/common_func/utils.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import numpy as np -import pandas as pd - - -def format_columns(df: pd.DataFrame): - formatted_df = df.rename( - { - "25%": "Q1Ns", - "50%": "MedianNs", - "75%": "Q3Ns", - 0.25: "Q1Ns", - 0.5: "MedianNs", - 0.75: "Q3Ns", - "Q1": "Q1Ns", - "Q3": "Q3Ns", - "min": "MinNs", - "max": "MaxNs", - "median": "MedianNs", - "sum": "SumNs", - "std": "StdNs", - "mean": "MeanNs", - "count": "Count" - }, - axis="columns" - ) - - stats_cols = ["Count", "MeanNs", "StdNs", "MinNs", "Q1Ns", "MedianNs", "Q3Ns", "MaxNs", "SumNs"] - other_cols = [col for col in formatted_df.columns if col not in stats_cols] - return formatted_df[stats_cols + other_cols] - - -def describe_duration(series_groupby): - agg_df = series_groupby.agg(["min", "max", "count", "std", "mean", "sum"]) - quantile_df = series_groupby.quantile([0.25, 0.5, 0.75]) - - quantile_df = quantile_df.unstack() - quantile_df.columns = ["25%", "50%", "75%"] - - stats_df = pd.merge(agg_df, quantile_df, left_index=True, right_index=True) - formated_df = format_columns(stats_df) - formated_df.index.name = stats_df.index.name - return formated_df - - -def stdev(df, aggregated): - if len(df) <= 1: - return df["stdevNs"].iloc[0] - instance = aggregated["totalCount"].loc[df.name] - var_sum = np.dot(df["totalCount"] - 1, df["stdev"] ** 2) - deviation = df["averageNs"] - aggregated["averageNs"].loc[df.name] - dev_sum = np.dot(df["totalCount"], deviation ** 2) - return np.sqrt((var_sum + dev_sum) / (instance - 1)) - - -def convert_unit(df: pd.DataFrame, src_unit, dst_unit): - df.loc[:, df.columns.str.endswith(src_unit)] = df.loc[:, df.columns.str.endswith(src_unit)].apply(lambda x: x / 1000.0) - df = df.rename(columns=lambda x: x.replace(src_unit, "".join(["(", dst_unit, ")"]))) - return df -- Gitee From 1e172e4d6226f9738f2e4599cf327aa0fd7aed99 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 1 Aug 2024 17:08:31 +0800 Subject: [PATCH 072/791] compare command --- debug/accuracy_tools/msprobe/core/common/const.py | 1 + debug/accuracy_tools/msprobe/core/common/utils.py | 4 ++-- debug/accuracy_tools/msprobe/msprobe.py | 2 +- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 3 ++- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index b59536aa5..e3d3c4e01 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -20,6 +20,7 @@ class Const: DEFAULT_PATH = './' WHITE_LIST = 'white_list' BLACK_LIST = 'black_list' + DUMP_TENSOR_DATA = '/dump_tensor_data' # dump mode ALL = "all" diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 32aba8d8a..5662fed6b 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -496,8 +496,8 @@ def task_dumppath_get(input_param): else: logger.error(f"Compare is not required for overflow_check or free_benchmark.") raise CompareException(CompareException.INVALID_TASK_ERROR) - input_param['npu_dump_data_dir'] = npu_json_data['dump_data_dir'] - input_param['bench_dump_data_dir'] = bench_json_data['dump_data_dir'] + input_param['npu_dump_data_dir'] = os.path.dirname(npu_json_path) + Const.DUMP_TENSOR_DATA + input_param['bench_dump_data_dir'] = os.path.dirname(bench_json_path) + Const.DUMP_TENSOR_DATA return summary_compare, md5_compare diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 12b04920a..a815e7c53 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -37,7 +37,7 @@ def main(): help='Deep learning framework.') subparsers = parser.add_subparsers() subparsers.add_parser('parse') - compare_cmd_parser = subparsers.add_parser('run_ut') + compare_cmd_parser = subparsers.add_parser('compare') run_ut_cmd_parser = subparsers.add_parser('run_ut') multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut') api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare') diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 88c839511..7b9d4eca4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -15,6 +15,7 @@ # limitations under the License. """ +import argparse import json import multiprocessing import os.path @@ -1045,7 +1046,7 @@ def _compare_parser(parser): help=" The compare task result out path.", required=True) parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", help=" Whether to save stack info.", required=False) - parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_true", + parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", help=" Whether to give advisor.", required=False) parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", help=" Whether to perform a fuzzy match on the api name.", required=False) -- Gitee From bd91f277fca6dde8b447095ad988bf4776ec6fc8 Mon Sep 17 00:00:00 2001 From: hanqing Date: Thu, 1 Aug 2024 17:25:22 +0800 Subject: [PATCH 073/791] bugfix --- .../core/data_dump/data_processor/pytorch_processor.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 2712bac61..007fec809 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -147,10 +147,12 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_json.update({"Norm": tensor_stat.norm}) tensor_json.update({"requires_grad": tensor.requires_grad}) - if np.isinf(tensor_stat.max) or np.isnan(tensor_stat.max): - tensor_json['Max_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(tensor, "max") - if np.isinf(tensor_stat.min) or np.isnan(tensor_stat.min): - tensor_json['Min_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(tensor, "min") + if tensor_stat.max is not None: + if np.isinf(tensor_stat.max) or np.isnan(tensor_stat.max): + tensor_json['Max_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(tensor, "max") + if tensor_stat.min is not None: + if np.isinf(tensor_stat.min) or np.isnan(tensor_stat.min): + tensor_json['Min_except_inf_nan'] = self.handle_tensor_extremum_nan_inf(tensor, "min") if self.config.summary_mode == Const.MD5: tensor_md5 = self.get_md5_for_tensor(tensor) -- Gitee From 57708049aaeda067d92d79a757d9330c10bd7ab0 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 1 Aug 2024 17:29:29 +0800 Subject: [PATCH 074/791] compare command --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 7b9d4eca4..120ee8464 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -650,10 +650,7 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) -# def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, -# fuzzy_match=False): def compare(args): - #### 需要增加文件路径路径校验 with FileOpen(args.input_path, "r") as file: input_param = json.load(file) try: -- Gitee From 0eed10dd6692432b0cd9e175dad019514889580a Mon Sep 17 00:00:00 2001 From: l30044004 Date: Thu, 1 Aug 2024 19:48:58 +0800 Subject: [PATCH 075/791] =?UTF-8?q?=E5=A4=84=E7=90=86module=E7=BA=A7?= =?UTF-8?q?=E5=88=ABdump=EF=BC=8Cmodule=E8=BE=93=E5=87=BA=E8=A2=AB?= =?UTF-8?q?=E5=B0=81=E8=A3=85=E8=BF=87=E4=B8=8D=E6=98=AFtensor=E6=88=96?= =?UTF-8?q?=E5=8C=85=E5=90=ABtensor=E7=9A=84=E5=85=83=E7=BB=84=EF=BC=8Cdum?= =?UTF-8?q?p=E4=B8=8D=E5=88=B0=E6=95=B0=E6=8D=AE=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/module_processer.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/module_processer.py index f9368a087..9cee721db 100644 --- a/debug/accuracy_tools/msprobe/pytorch/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/module_processer.py @@ -26,9 +26,15 @@ class ModuleProcesser: def filter_tensor_and_tuple(func): @wraps(func) def wrap_by_filter_tensor_and_tuple(*args, **kwargs): - # setup_output_hook传入非tensor数据，工具后续dump会报错，处理方式是非tensor数据不传入 + # setup_output_hook传入非tensor数据，工具后续dump会报错，处理方式是解析非tensor数据的属性，对tensor属性挂hook # setup_output_hook定义为setup_output_hook(self, args)，因此处理第二个位置参数，即*args[1] if not isinstance(args[1], (torch.Tensor, tuple)): + for item_str in dir(args[1]): + item = getattr(args[1], item_str) + if isinstance(item, (torch.Tensor, tuple)): + args_new = (args[0], item) + result = func(*args_new, **kwargs) + setattr(args[1], item_str, result) return args[1] return func(*args, **kwargs) -- Gitee From 9ab00ace6c3bbdb588a7db4ff8a4cddc57d1d5f6 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 1 Aug 2024 20:05:41 +0800 Subject: [PATCH 076/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3importerror=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/mindspore/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/__init__.py b/debug/accuracy_tools/msprobe/mindspore/__init__.py index 077144429..d131591a3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/__init__.py +++ b/debug/accuracy_tools/msprobe/mindspore/__init__.py @@ -1,4 +1,3 @@ from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger -from .common.utils import seed_all from .compare.acc_compare import compare from .compare.distributed_compare import compare_distributed -- Gitee From 8dd843192c39f25e044be34d1f45ae96d7d12915 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 1 Aug 2024 20:07:41 +0800 Subject: [PATCH 077/791] bug fix --- .../pytorch/api_accuracy_checker/run_ut/data_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index b103643c0..b2eec691a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -94,7 +94,7 @@ def gen_real_tensor(data_path, convert_type): error_info = f"The file: {data_path} is not a pt or numpy file." raise CompareException(CompareException.INVALID_FILE_ERROR, error_info) if data_path.endswith('.pt'): - data = torch.load(data_path).cpu() + data = torch.load(data_path, map_location=torch.device('cpu')) else: data_np = numpy.load(data_path) data = torch.from_numpy(data_np) -- Gitee From f36064634f8ef3f237c7b1728e0b40e0d79f01e7 Mon Sep 17 00:00:00 2001 From: l30044004 Date: Thu, 1 Aug 2024 20:31:40 +0800 Subject: [PATCH 078/791] =?UTF-8?q?=E5=A4=84=E7=90=86module=E7=BA=A7?= =?UTF-8?q?=E5=88=ABdump=EF=BC=8Cmodule=E8=BE=93=E5=87=BA=E8=A2=AB?= =?UTF-8?q?=E5=B0=81=E8=A3=85=E8=BF=87=E4=B8=8D=E6=98=AFtensor=E6=88=96?= =?UTF-8?q?=E5=8C=85=E5=90=ABtensor=E7=9A=84=E5=85=83=E7=BB=84=EF=BC=8Cdum?= =?UTF-8?q?p=E4=B8=8D=E5=88=B0=E6=95=B0=E6=8D=AE=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/module_processer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/module_processer.py index 9cee721db..3e9969d32 100644 --- a/debug/accuracy_tools/msprobe/pytorch/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/module_processer.py @@ -31,7 +31,9 @@ class ModuleProcesser: if not isinstance(args[1], (torch.Tensor, tuple)): for item_str in dir(args[1]): item = getattr(args[1], item_str) - if isinstance(item, (torch.Tensor, tuple)): + # 处理tensor或者只包含tensor的元组 + if isinstance(item, torch.Tensor) or \ + (isinstance(item, tuple) and all(isinstance(x, torch.Tensor) for x in item)): args_new = (args[0], item) result = func(*args_new, **kwargs) setattr(args[1], item_str, result) -- Gitee From 71486de5222ae82330b7f22bd1b317b496bb6ea3 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 2 Aug 2024 08:56:15 +0800 Subject: [PATCH 079/791] compare command --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 120ee8464..eb2e957f6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -698,9 +698,9 @@ def compare_core(input_parma, output_path, **kwargs): check_file_not_exists(file_path) highlight_dict = {'red_rows': [], 'yellow_rows': []} - with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_json_path"), "r") as stack_json: + with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_path"), "r") as stack_json: result_df = compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, summary_compare, md5_compare) -- Gitee From 06d975f348c40ed57272233d27ed9980c78a131b Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 2 Aug 2024 09:03:28 +0800 Subject: [PATCH 080/791] review fix --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 2 +- .../msprobe/pytorch/compare/distributed_compare.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 0070b58e5..73bdadb7d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -656,7 +656,7 @@ def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) check_compare_param(input_parma, output_path, summary_compare, md5_compare) - except CompareException as error: + except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) compare_core(input_parma, output_path, stack_mode=stack_mode, diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 47a2864e5..caac13958 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -21,6 +21,7 @@ from msprobe.core.common.utils import CompareException, check_compare_param, \ check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid from msprobe.pytorch.compare.acc_compare import compare_core from msprobe.core.common.file_check import create_directory +from msprobe.core.common.exceptions import FileCheckException from msprobe.pytorch.common.log import logger @@ -103,7 +104,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) - except CompareException as error: + except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, -- Gitee From a304a065fcb35267ef9d205a8b4ecfedf1a02864 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 2 Aug 2024 09:24:09 +0800 Subject: [PATCH 081/791] compare command --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index eb2e957f6..9c36fb7a6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -671,8 +671,8 @@ def compare_core(input_parma, output_path, **kwargs): Compares data from multiple JSON files and generates a comparison report. Args: - input_parma (dict): A dictionary containing paths to JSON files ("npu_json_path", "bench_json_path", - "stack_json_path"). + input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", + "stack_path"). output_path (str): The path where the output Excel report will be saved. **kwargs: Additional keyword arguments including: - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. -- Gitee From ae0c13cdb738cf7b509173023376d242727394d0 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 2 Aug 2024 09:38:28 +0800 Subject: [PATCH 082/791] compare command --- .../msprobe/core/common/utils.py | 42 +++++++++---------- .../msprobe/pytorch/compare/acc_compare.py | 2 +- .../pytorch/compare/distributed_compare.py | 14 +++---- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 5662fed6b..37a7733e1 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -148,21 +148,21 @@ def check_summary_only_valid(summary_only): return summary_only -def check_compare_param(input_parma, output_path, stack_mode=False, summary_compare=False, md5_compare=False): - if not (isinstance(input_parma, dict) and isinstance(output_path, str)): +def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False): + if not (isinstance(input_param, dict) and isinstance(output_path, str)): logger.error("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) - check_file_or_directory_path(input_parma.get("npu_json_path"), False) - check_file_or_directory_path(input_parma.get("bench_json_path"), False) - check_file_or_directory_path(input_parma.get("stack_json_path"), False) + check_file_or_directory_path(input_param.get("npu_path"), False) + check_file_or_directory_path(input_param.get("bench_path"), False) + check_file_or_directory_path(input_param.get("stack_path"), False) if not summary_compare and not md5_compare: - check_file_or_directory_path(input_parma.get("npu_dump_data_dir"), True) - check_file_or_directory_path(input_parma.get("bench_dump_data_dir"), True) + check_file_or_directory_path(input_param.get("npu_dump_data_dir"), True) + check_file_or_directory_path(input_param.get("bench_dump_data_dir"), True) check_file_or_directory_path(output_path, True) - with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_json_path"), "r") as stack_json: - check_json_file(input_parma, npu_json, bench_json, stack_json) + with FileOpen(input_param.get("npu_path"), "r") as npu_json, \ + FileOpen(input_param.get("bench_path"), "r") as bench_json, \ + FileOpen(input_param.get("stack_path"), "r") as stack_json: + check_json_file(input_param, npu_json, bench_json, stack_json) def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False): @@ -201,9 +201,9 @@ def _check_json(json_file_handle, file_name): def check_json_file(input_param, npu_json, bench_json, stack_json): - _check_json(npu_json, input_param.get("npu_json_path")) - _check_json(bench_json, input_param.get("bench_json_path")) - _check_json(stack_json, input_param.get("stack_json_path")) + _check_json(npu_json, input_param.get("npu_path")) + _check_json(bench_json, input_param.get("bench_path")) + _check_json(stack_json, input_param.get("stack_path")) def check_file_size(input_file, max_size): @@ -472,14 +472,14 @@ def md5_find(data): def task_dumppath_get(input_param): - npu_json_path = input_param.get("npu_json_path", None) - bench_json_path = input_param.get("bench_json_path", None) - if not npu_json_path or not bench_json_path: + npu_path = input_param.get("npu_path", None) + bench_path = input_param.get("bench_path", None) + if not npu_path or not bench_path: logger.error(f"Please check the json path is valid.") raise CompareException(CompareException.INVALID_PATH_ERROR) - with FileOpen(npu_json_path, 'r') as npu_f: + with FileOpen(npu_path, 'r') as npu_f: npu_json_data = json.load(npu_f) - with FileOpen(bench_json_path, 'r') as bench_f: + with FileOpen(bench_path, 'r') as bench_f: bench_json_data = json.load(bench_f) if npu_json_data['task'] != bench_json_data['task']: logger.error(f"Please check the dump task is consistent.") @@ -496,8 +496,8 @@ def task_dumppath_get(input_param): else: logger.error(f"Compare is not required for overflow_check or free_benchmark.") raise CompareException(CompareException.INVALID_TASK_ERROR) - input_param['npu_dump_data_dir'] = os.path.dirname(npu_json_path) + Const.DUMP_TENSOR_DATA - input_param['bench_dump_data_dir'] = os.path.dirname(bench_json_path) + Const.DUMP_TENSOR_DATA + input_param['npu_dump_data_dir'] = os.path.dirname(npu_path) + Const.DUMP_TENSOR_DATA + input_param['bench_dump_data_dir'] = os.path.dirname(bench_path) + Const.DUMP_TENSOR_DATA return summary_compare, md5_compare diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 9c36fb7a6..f37282ff5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -657,7 +657,7 @@ def compare(args): summary_compare, md5_compare = task_dumppath_get(input_param) check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) create_directory(args.output_path) - check_compare_param(input_param, args.output_path, args.stack_mode, summary_compare, md5_compare) + check_compare_param(input_param, args.output_path, summary_compare, md5_compare) except CompareException as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 0298eca9e..fe8dcbfef 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -89,21 +89,21 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): n_dir = os.path.join(npu_dump_dir, nr) b_dir = os.path.join(bench_dump_dir, br) s_dir = b_dir - npu_json_path = extract_json(n_dir, stack_json=False) - bench_json_path = extract_json(b_dir, stack_json=False) - stack_json_path = extract_json(s_dir, stack_json=True) + npu_path = extract_json(n_dir, stack_json=False) + bench_path = extract_json(b_dir, stack_json=False) + stack_path = extract_json(s_dir, stack_json=True) dump_result_param = { - 'npu_json_path': npu_json_path, - 'bench_json_path': bench_json_path, - 'stack_json_path': stack_json_path, + 'npu_path': npu_path, + 'bench_path': bench_path, + 'stack_path': stack_path, 'is_print_compare_log': True } try: summary_compare, md5_compare = task_dumppath_get(dump_result_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(dump_result_param, output_path, stack_mode=stack_mode, summary_compare=summary_compare) + check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) except CompareException as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) -- Gitee From 499c8a1d91a82b929a111c8ffa686af064089f2a Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 2 Aug 2024 11:10:27 +0800 Subject: [PATCH 083/791] review fix --- .../ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py index 7a5afd90b..7cfac0d40 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py @@ -21,9 +21,9 @@ range_begin_flag, range_end_flag = False, False def check_list_or_acl_mode(name_prefix): global dump_count for item in DumpUtil.dump_switch_scope: - if "pre_forward" in name_prefix: - parts = item.split(".") - rename = ".".join(parts[:-1]) + if PRE_FORWARD in name_prefix: + parts = item.split(Const.DOT) + rename = Const.DOT.join(parts[:-1]) if name_prefix.startswith(rename): return True if name_prefix.startswith(item): @@ -36,8 +36,8 @@ def check_range_mode(name_prefix): global range_begin_flag global range_end_flag if "Distributed" in DumpUtil.dump_switch_scope[0]: - parts = DumpUtil.dump_switch_scope[0].split(".") - rename = ".".join(parts[:-1]) + parts = DumpUtil.dump_switch_scope[0].split(Const.DOT) + rename = Const.DOT.join(parts[:-1]) if name_prefix.startswith(rename): range_begin_flag = True return True -- Gitee From 71de52523bf5e046a4f5e24458afc525865d9ba1 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 2 Aug 2024 12:40:23 +0800 Subject: [PATCH 084/791] conflict fix --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 2 +- .../msprobe/pytorch/compare/distributed_compare.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index e8a3c8c05..c74f42daf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -662,7 +662,7 @@ def compare(args): check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) create_directory(args.output_path) check_compare_param(input_param, args.output_path, summary_compare, md5_compare) - except CompareException as error: + except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) compare_core(input_param, args.output_path, stack_mode=args.stack_mode, diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index f819ff794..f5d28de40 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -89,9 +89,9 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): for nr, br in zip(npu_ranks, bench_ranks): npu_data_dir = os.path.join(npu_dump_dir, nr) bench_data_dir = os.path.join(bench_dump_dir, br) - npu_json_path = extract_json(npu_data_dir, stack_json=False) - bench_json_path = extract_json(bench_data_dir, stack_json=False) - stack_json_path = extract_json(npu_data_dir, stack_json=True) + npu_path = extract_json(npu_data_dir, stack_json=False) + bench_path = extract_json(bench_data_dir, stack_json=False) + stack_path = extract_json(npu_data_dir, stack_json=True) dump_result_param = { 'npu_path': npu_path, -- Gitee From cb47e086b78847a8cf15374a425561d8f548283e Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 2 Aug 2024 14:03:43 +0800 Subject: [PATCH 085/791] ut fix --- .../msprobe/test/core_ut/common/test_utils.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index edd3eb53d..a1cd516c4 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -189,28 +189,28 @@ class TestUtils(TestCase): @patch.object(logger, "error") def test_check_compare_param(self, mock_error): params = { - "npu_json_path": "npu_json_path", - "bench_json_path": "bench_json_path", - "stack_json_path": "stack_json_path", + "npu_path": "npu_path", + "bench_path": "bench_path", + "stack_path": "stack_path", "npu_dump_data_dir": "npu_dump_data_dir", "bench_dump_data_dir": "bench_dump_data_dir" } call_args = [ - ("npu_json_path", False), - ("bench_json_path", False), - ("stack_json_path", False), + ("npu_path", False), + ("bench_path", False), + ("stack_path", False), ("npu_dump_data_dir", True), ("bench_dump_data_dir", True), ("output_path", True), - ("npu_json_path", False), - ("bench_json_path", False), - ("stack_json_path", False), + ("npu_path", False), + ("bench_path", False), + ("stack_path", False), ("output_path", True) ] with self.assertRaises(CompareException) as context: - check_compare_param("npu_json_path", "output_path") + check_compare_param("npu_path", "output_path") self.assertEqual(context.exception.code, CompareException.INVALID_PARAM_ERROR) mock_error.assert_called_with("Invalid input parameters") @@ -264,14 +264,14 @@ class TestUtils(TestCase): @patch("msprobe.core.common.utils._check_json") def test_check_json_file(self, _mock_check_json): input_param = { - "npu_json_path": "npu_json_path", - "bench_json_path": "bench_json_path", - "stack_json_path": "stack_json_path" + "npu_path": "npu_path", + "bench_path": "bench_path", + "stack_path": "stack_path" } check_json_file(input_param, "npu_json", "bench_json", "stack_json") - self.assertEqual(_mock_check_json.call_args_list[0][0], ("npu_json", "npu_json_path")) - self.assertEqual(_mock_check_json.call_args_list[1][0], ("bench_json", "bench_json_path")) - self.assertEqual(_mock_check_json.call_args_list[2][0], ("stack_json", "stack_json_path")) + self.assertEqual(_mock_check_json.call_args_list[0][0], ("npu_json", "npu_path")) + self.assertEqual(_mock_check_json.call_args_list[1][0], ("bench_json", "bench_path")) + self.assertEqual(_mock_check_json.call_args_list[2][0], ("stack_json", "stack_path")) @patch.object(logger, "error") def test_check_file_size(self, mock_error): @@ -307,8 +307,8 @@ class TestUtils(TestCase): @patch.object(logger, "error") def test_task_dumppath_get(self, mock_error): input_param = { - "npu_json_path": None, - "bench_json_path": "bench_json_path" + "npu_path": None, + "bench_path": "bench_path" } npu_json = { "task": Const.TENSOR, @@ -321,7 +321,7 @@ class TestUtils(TestCase): self.assertEqual(context.exception.code, CompareException.INVALID_PATH_ERROR) mock_error.assert_called_with("Please check the json path is valid.") - input_param["npu_json_path"] = "npu_json_path" + input_param["npu_path"] = "npu_path" with patch("msprobe.core.common.utils.FileOpen", mock_open(read_data="")), \ patch("msprobe.core.common.utils.json.load", return_value=npu_json): summary_compare, md5_compare = task_dumppath_get(input_param) -- Gitee From 7fd45281bf9abd3b08962d85cf50bda6234d303f Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Fri, 2 Aug 2024 14:46:23 +0800 Subject: [PATCH 086/791] bufgix: invalid url --- profiler/advisor/common/constant.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index 87245a43e..06186080d 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -48,8 +48,8 @@ NO_STACK_REASON_MAP = { TIMELINE_BACKWARD_NO_STACK_CODE: "Backward broadcast, without call stacks in profiling.", TIMELINE_ACL_TO_NPU_NO_STACK_CODE: "Incoming flow is 'acl_to_npu', without call stacks in profiling." } -TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc \ - /Samples%20of%20Fused%20Operator%20API%20Replacement.md" +TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/"\ + "Samples%20of%20Fused%20Operator%20API%20Replacement.md" AFFINITY_TRAINING_API = "Affinity training api" TIMELINE_WITH_STACK_DOC_URL = "https://www.hiascend.com/document/detail/zh/canncommercial/" \ "70RC1/modeldevpt/ptmigr/AImpug_0067.html" -- Gitee From a2e49b4bdbd44a7bf59c95e5530fc41b4cc05251 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 2 Aug 2024 15:40:59 +0800 Subject: [PATCH 087/791] bugfix: add file check when load or save --- .../grad_tool/grad_ms/grad_comparator.py | 13 +++++++++++-- debug/accuracy_tools/grad_tool/grad_ms/utils.py | 9 +++++++-- .../grad_tool/grad_pt/grad_comparator.py | 15 +++++++++++++-- .../grad_tool/grad_pt/grad_monitor.py | 7 +++++-- 4 files changed, 36 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py index 2bfeda438..72577bb7f 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py @@ -9,10 +9,19 @@ class MsGradComparator(BaseComparator): @classmethod def _load_grad_files(cls, grad_file1: str, grad_file2: str): + if not os.path.exists(grad_file1): + raise ValueError(f"file {grad_file1} not exists, please check the file path.") + if not os.path.exists(grad_file2): + raise ValueError(f"file {grad_file2} not exists, please check the file path.") + grad1_suffix = grad_file1.split(".")[-1] grad2_suffix = grad_file2.split(".")[-1] - grad1 = torch.load(grad_file1).numpy() if grad1_suffix == "pt" else np.load(grad_file1) - grad2 = torch.load(grad_file2).numpy() if grad2_suffix == "pt" else np.load(grad_file2) + + try: + grad1 = torch.load(grad_file1).numpy() if grad1_suffix == "pt" else np.load(grad_file1) + grad2 = torch.load(grad_file2).numpy() if grad2_suffix == "pt" else np.load(grad_file2) + except Exception as e: + raise RuntimeError("An unexpected error occurred: %s when loading grad_file." % str(e)) if grad1.shape != grad2.shape: raise RuntimeError(f"numpy shape is not equal: {grad_file1}, {grad_file2}") diff --git a/debug/accuracy_tools/grad_tool/grad_ms/utils.py b/debug/accuracy_tools/grad_tool/grad_ms/utils.py index c0efbdc7b..26f500cda 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/utils.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/utils.py @@ -21,6 +21,7 @@ level_adp = { }, } + def save_grad_direction(param_name, grad, save_path): if not os.path.exists(save_path): create_directory(save_path) @@ -34,9 +35,13 @@ def save_grad_direction(param_name, grad, save_path): grad_direction_tensor = grad > 0 grad_direction_ndarray = grad_direction_tensor.numpy() - np.save(save_filepath, grad_direction_ndarray) + try: + np.save(save_filepath, grad_direction_ndarray) + except Exception as e: + raise RuntimeError("An unexpected error occurred: %s when saving numpy to %s" % (str(e), save_filepath)) change_mode(save_filepath, 0o640) + def get_adapted_level(level: str): level_adapted = level_adp.get(level) - return level_adapted \ No newline at end of file + return level_adapted diff --git a/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py b/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py index d1229b93d..8540cf9ab 100644 --- a/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py +++ b/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py @@ -1,3 +1,5 @@ +import os + import torch from grad_tool.common.base_comparator import BaseComparator @@ -7,8 +9,17 @@ class PtGradComparator(BaseComparator): @classmethod def _load_grad_files(cls, grad_file1: str, grad_file2: str): - tensor1 = torch.load(grad_file1, map_location=torch.device("cpu")) - tensor2 = torch.load(grad_file2, map_location=torch.device("cpu")) + if not os.path.exists(grad_file1): + raise ValueError(f"file {grad_file1} not exists, please check the file path.") + if not os.path.exists(grad_file2): + raise ValueError(f"file {grad_file2} not exists, please check the file path.") + + try: + tensor1 = torch.load(grad_file1, map_location=torch.device("cpu")) + tensor2 = torch.load(grad_file2, map_location=torch.device("cpu")) + except Exception as e: + raise RuntimeError("An unexpected error occurred: %s when loading tensor." % str(e)) + if tensor1.shape != tensor2.shape: raise RuntimeError(f"tensor shape is not equal: {grad_file1}, {grad_file2}") if tensor1.dtype != torch.bool: diff --git a/debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py b/debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py index f3079e622..dd9ffb721 100644 --- a/debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py +++ b/debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py @@ -61,7 +61,10 @@ class PtGradientMonitor(BaseMonitor): param_grad = grad.clone().detach() is_positive = param_grad > 0 save_filepath = os.path.join(save_path, f"{param_name}.pt") - torch.save(is_positive, save_filepath) + try: + torch.save(is_positive, save_filepath) + except Exception as e: + raise RuntimeError("An unexpected error occurred: %s when saving tensor to %s" % (str(e), save_filepath)) change_mode(save_filepath, 0o640) def monitor(self, model): @@ -96,7 +99,7 @@ class PtGradientMonitor(BaseMonitor): output_lines.append(grad_info) if self._level_adp["have_grad_direction"]: PtGradientMonitor.save_grad_direction(param_name, grad, - f'{self._output_path}/rank{self._rank}/step{self._step}') + f'{self._output_path}/rank{self._rank}/step{self._step}') output_path = os.path.join(self._output_path, f"rank{getattr(self, '_rank')}", f"grad_summary_{self._step}.csv") write_csv(output_path, output_lines, -- Gitee From b49e2627a54ef953e4ce0233d4755ce9204c4343 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 2 Aug 2024 16:14:18 +0800 Subject: [PATCH 088/791] fix codecheck --- debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py | 2 +- debug/accuracy_tools/grad_tool/grad_ms/utils.py | 2 +- debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py | 2 +- debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py index 72577bb7f..3b930d4e2 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py @@ -21,7 +21,7 @@ class MsGradComparator(BaseComparator): grad1 = torch.load(grad_file1).numpy() if grad1_suffix == "pt" else np.load(grad_file1) grad2 = torch.load(grad_file2).numpy() if grad2_suffix == "pt" else np.load(grad_file2) except Exception as e: - raise RuntimeError("An unexpected error occurred: %s when loading grad_file." % str(e)) + raise RuntimeError(f"An unexpected error occurred: {e} when loading grad_file.") from e if grad1.shape != grad2.shape: raise RuntimeError(f"numpy shape is not equal: {grad_file1}, {grad_file2}") diff --git a/debug/accuracy_tools/grad_tool/grad_ms/utils.py b/debug/accuracy_tools/grad_tool/grad_ms/utils.py index 26f500cda..c8ee1fd1d 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/utils.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/utils.py @@ -38,7 +38,7 @@ def save_grad_direction(param_name, grad, save_path): try: np.save(save_filepath, grad_direction_ndarray) except Exception as e: - raise RuntimeError("An unexpected error occurred: %s when saving numpy to %s" % (str(e), save_filepath)) + raise RuntimeError(f"An unexpected error occurred: {e} when saving numpy to {save_filepath}") from e change_mode(save_filepath, 0o640) diff --git a/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py b/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py index 8540cf9ab..38f0e3215 100644 --- a/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py +++ b/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py @@ -18,7 +18,7 @@ class PtGradComparator(BaseComparator): tensor1 = torch.load(grad_file1, map_location=torch.device("cpu")) tensor2 = torch.load(grad_file2, map_location=torch.device("cpu")) except Exception as e: - raise RuntimeError("An unexpected error occurred: %s when loading tensor." % str(e)) + raise RuntimeError(f"An unexpected error occurred: {e} when loading tensor.") from e if tensor1.shape != tensor2.shape: raise RuntimeError(f"tensor shape is not equal: {grad_file1}, {grad_file2}") diff --git a/debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py b/debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py index dd9ffb721..2e1abde0d 100644 --- a/debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py +++ b/debug/accuracy_tools/grad_tool/grad_pt/grad_monitor.py @@ -64,7 +64,7 @@ class PtGradientMonitor(BaseMonitor): try: torch.save(is_positive, save_filepath) except Exception as e: - raise RuntimeError("An unexpected error occurred: %s when saving tensor to %s" % (str(e), save_filepath)) + raise RuntimeError(f"An unexpected error occurred: {e} when saving tensor to {save_filepath}") from e change_mode(save_filepath, 0o640) def monitor(self, model): -- Gitee From 43af20b2b5da77688b7be6f59dd304b40bf26067 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 2 Aug 2024 16:30:03 +0800 Subject: [PATCH 089/791] bug fix --- .../msprobe/core/data_dump/data_processor/pytorch_processor.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 007fec809..dbe6f0210 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -168,7 +168,8 @@ class TensorDataProcessor(PytorchDataProcessor): def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) if not path_len_exceeds_limit(file_path): - torch.save(tensor, file_path) + saved_tensor = tensor.contiguous().detach().cpu() + torch.save(saved_tensor, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) else: logger.warning(f'The file path {file_path} length exceeds limit.') -- Gitee From e35cdd627a584a4e6bde7e5f2f31fcca9a98894b Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Fri, 2 Aug 2024 17:15:24 +0800 Subject: [PATCH 090/791] Fix bug of inconsistent output for fix handler --- .../msprobe/pytorch/free_benchmark/main.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/main.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/main.py index 971776d13..69ece0a0c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/main.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/main.py @@ -10,7 +10,10 @@ from msprobe.pytorch.free_benchmark.common.enums import ( HandlerType, PerturbationMode, ) -from msprobe.pytorch.free_benchmark.common.params import data_pre_deal, make_handler_params +from msprobe.pytorch.free_benchmark.common.params import ( + data_pre_deal, + make_handler_params, +) from msprobe.pytorch.free_benchmark.compare.grad_saver import GradSaver from msprobe.pytorch.free_benchmark.perturbed_layers.layer_factory import LayerFactory from msprobe.pytorch.free_benchmark.result_handlers.handler_factory import ( @@ -70,9 +73,9 @@ class FreeBenchmarkCheck(ABC): layer.handle(data_params) handler_params = make_handler_params(name, self.config, self.current_iter) handler = FuzzHandlerFactory.create(handler_params) - handler.handle(data_params) - return data_params.perturbed_result, handler.get_unequal_rows() - + perturbed_output = handler.handle(data_params) + return perturbed_output, handler.get_unequal_rows() + def backward(self, name, module, grad_output): if not self.config.fuzz_stage == Const.BACKWARD: -- Gitee From 0119a2911802d3324fc914400e61108f5d3278a8 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Fri, 2 Aug 2024 17:33:37 +0800 Subject: [PATCH 091/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E6=96=87?= =?UTF-8?q?=E4=BB=B6=E8=AF=BB=E5=8F=96=E5=AD=98=E5=9C=A8=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E9=9A=90=E6=82=A3=E9=97=AE=E9=A2=98=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/utils.py | 9 -------- .../msprobe/core/data_dump/json_writer.py | 22 +++++++++---------- .../msprobe/pytorch/common/parse_json.py | 4 +++- 3 files changed, 14 insertions(+), 21 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 32aba8d8a..684b3d7ef 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -318,15 +318,6 @@ def execute_command(cmd): raise CompareException(CompareException.INVALID_DATA_ERROR) -def save_numpy_data(file_path, data): - """ - save_numpy_data - """ - if not os.path.exists(os.path.dirname(file_path)): - os.makedirs(os.path.dirname(file_path)) - np.save(file_path, data) - - def parse_value_by_comma(value): """ parse value by comma, like '1,2,4,8' diff --git a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py index c4b7fc11e..112e45171 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py @@ -4,7 +4,7 @@ import fcntl import json from pathlib import Path -from msprobe.core.common.file_check import change_mode +from msprobe.core.common.file_check import change_mode, FileOpen from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst @@ -30,20 +30,20 @@ class DataWriter: return is_exists = os.path.exists(file_path) append = "a+" if is_exists else "w+" - with os.fdopen( - os.open(file_path, Const.WRITE_FLAGS, FileCheckConst.DATA_FILE_AUTHORITY), append, newline="" - ) as csv_file: + with FileOpen(file_path, append) as csv_file: spawn_writer = csv.writer(csv_file) if not is_exists: spawn_writer.writerow(result_header) spawn_writer.writerows([result,]) + is_new_file = not is_exists + if is_new_file: + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) def initialize_json_file(self, **kwargs): kwargs.update({"dump_data_dir": self.dump_tensor_data_dir, Const.DATA: {}}) - with os.fdopen( - os.open(self.dump_file_path, Const.OVERWRITE_FLAGS, FileCheckConst.DATA_FILE_AUTHORITY), 'w' - ) as f: + with FileOpen(self.dump_file_path, 'w') as f: json.dump(kwargs, f) + change_mode(self.dump_file_path, FileCheckConst.DATA_FILE_AUTHORITY) if os.path.exists(self.stack_file_path): os.remove(self.stack_file_path) @@ -83,7 +83,7 @@ class DataWriter: def write_data_json(self, file_path): logger.info(f"dump.json is at {os.path.dirname(os.path.dirname(file_path))}. ") if Path(file_path).exists() and os.path.getsize(file_path) > 0: - with open(file_path, "r+") as f: + with FileOpen(file_path, "r+") as f: fcntl.flock(f, fcntl.LOCK_EX) data_to_write = json.load(f) fcntl.flock(f, fcntl.LOCK_UN) @@ -91,7 +91,7 @@ class DataWriter: self.init_json['data_path'] = self.dump_tensor_data_dir data_to_write = self.init_json data_to_write[Const.DATA].update(self.cache_data[Const.DATA]) - with open(file_path, 'w+') as f: + with FileOpen(file_path, 'w+') as f: fcntl.flock(f, fcntl.LOCK_EX) json.dump(data_to_write, f, indent=1) fcntl.flock(f, fcntl.LOCK_UN) @@ -99,13 +99,13 @@ class DataWriter: self.cache_data[Const.DATA].clear() def write_stack_info_json(self, file_path): - with open(file_path, 'w+') as f: + with FileOpen(file_path, 'w+') as f: fcntl.flock(f, fcntl.LOCK_EX) json.dump(self.cache_stack, f, indent=1) fcntl.flock(f, fcntl.LOCK_UN) def write_construct_info_json(self, file_path): - with open(file_path, 'w+') as f: + with FileOpen(file_path, 'w+') as f: fcntl.flock(f, fcntl.LOCK_EX) json.dump(self.cache_construct, f, indent=1) fcntl.flock(f, fcntl.LOCK_UN) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py b/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py index 22f798798..ccad90372 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py @@ -1,5 +1,7 @@ import json + from msprobe.core.common.exceptions import ParseJsonException +from msprobe.core.common.file_check import FileOpen def parse_json_info_forward_backward(json_path): @@ -11,7 +13,7 @@ def parse_json_info_forward_backward(json_path): api_name = '.'.join(name_struct[:-1]) return api_name - with open(json_path, 'r') as f: + with FileOpen(json_path, 'r') as f: dump_json = json.load(f) real_data_path = dump_json.get("dump_data_dir") -- Gitee From 5037a5463beed09495df83e8e70144f3d8c991a1 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Thu, 1 Aug 2024 19:21:58 +0800 Subject: [PATCH 092/791] fix backward input output --- .../msprobe/core/data_dump/data_processor/base.py | 12 ++++++------ .../pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 +- .../msprobe/pytorch/compare/acc_compare.py | 8 ++++---- debug/accuracy_tools/msprobe/pytorch/service.py | 3 ++- 4 files changed, 13 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index a6858e8cb..8dc9fd85e 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -212,17 +212,17 @@ class BaseDataProcessor: def analyze_backward(self, name, module, module_input_output: ModuleBackwardInputsOutputs): api_info_struct = {} - if self.is_dump_for_data_mode(Const.BACKWARD, Const.OUTPUT): + if self.is_dump_for_data_mode(Const.BACKWARD, Const.INPUT): api_info_struct[name] = {} - self.api_data_category = Const.OUTPUT + self.api_data_category = Const.INPUT input_info_list = self.analyze_element(module_input_output.grad_input_tuple) - api_info_struct[name][Const.GRAD_INPUT] = input_info_list + api_info_struct[name][Const.INPUT] = input_info_list - if self.is_dump_for_data_mode(Const.BACKWARD, Const.INPUT): + if self.is_dump_for_data_mode(Const.BACKWARD, Const.OUTPUT): api_info_struct[name] = api_info_struct.get(name, {}) - self.api_data_category = Const.INPUT + self.api_data_category = Const.OUTPUT output_info_list = self.analyze_element(module_input_output.grad_output_tuple) - api_info_struct[name][Const.GRAD_OUTPUT] = output_info_list + api_info_struct[name][Const.OUTPUT] = output_info_list return api_info_struct diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index a8ff9b599..440d2564c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -274,7 +274,7 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict if need_backward: if need_to_backward(grad_index, out): - backward_args = backward_content[api_full_name].get("grad_output") + backward_args = backward_content[api_full_name].get("input") grad = gen_args(backward_args, api_name, real_data_path=real_data_path)[0] bench_grad, _ = generate_cpu_params(grad, {}, False, api_name) bench_grad_out = run_backward(cpu_args, bench_grad, grad_index, out) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index ea9323ae0..e06522838 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -874,13 +874,13 @@ def read_op(op_data, op_name): op_parsed_list += output_parsed_list output_parsed_list.clear() if 'backward' in op_name: - if 'grad_input' in op_data: - input_item = op_data['grad_input'] + if 'input' in op_data: + input_item = op_data['input'] input_parsed_list = op_item_parse(input_item, op_name + '_input', None) op_parsed_list = input_parsed_list.copy() input_parsed_list.clear() - if 'grad_output' in op_data: - output_item = op_data['grad_output'] + if 'output' in op_data: + output_item = op_data['output'] output_parsed_list = op_item_parse(output_item, op_name + '_output', None) op_parsed_list += output_parsed_list output_parsed_list.clear() diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index daeda8898..9ec7ed451 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -67,7 +67,8 @@ class Service: if not self.switch: return if self.data_collector: - module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_input, grad_output=grad_output) + # 此处获取到的grad_input实际为反向过程的输出数据，grad_output为反向过程的输入数据，因此传入时调换顺序 + module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input) self.data_collector.backward_data_collect(api_or_module_name, module, pid, module_input_output) pid = os.getpid() -- Gitee From a800c3cfdd4797964c1971dbc42b7dc51bdf568a Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 3 Aug 2024 10:53:00 +0800 Subject: [PATCH 093/791] review fix --- .../msprobe/core/common/file_check.py | 20 +++++++++- .../msprobe/pytorch/compare/acc_compare.py | 39 +++++++------------ 2 files changed, 34 insertions(+), 25 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_check.py b/debug/accuracy_tools/msprobe/core/common/file_check.py index 36896cfbc..c567f9454 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_check.py +++ b/debug/accuracy_tools/msprobe/core/common/file_check.py @@ -262,4 +262,22 @@ def change_mode(path, mode): def path_len_exceeds_limit(file_path): return len(os.path.realpath(file_path)) > FileCheckConst.DIRECTORY_LENGTH or \ - len(os.path.basename(file_path)) > FileCheckConst.FILE_NAME_LENGTH \ No newline at end of file + len(os.path.basename(file_path)) > FileCheckConst.FILE_NAME_LENGTH + + +def check_file_type(path): + """ + Function Description: + determine if it is a file or a directory + Parameter: + path: path + Exception Description: + when neither a file nor a directory throw exception + """ + if os.path.isdir(path): + return FileCheckConst.DIR + elif os.path.isfile(path): + return FileCheckConst.FILE + else: + logger.error('Neither a file nor a directory.') + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index c74f42daf..0072d9432 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -36,7 +36,7 @@ from msprobe.pytorch.advisor.advisor import Advisor from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ format_value, check_file_not_exists, check_configuration_param, task_dumppath_get -from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, create_directory +from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, create_directory, check_file_type from msprobe.core.common.const import Const, CompareConst, FileCheckConst from msprobe.core.common.exceptions import FileCheckException @@ -657,17 +657,20 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): def compare(args): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) - try: - summary_compare, md5_compare = task_dumppath_get(input_param) - check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) - create_directory(args.output_path) - check_compare_param(input_param, args.output_path, summary_compare, md5_compare) - except (CompareException, FileCheckException) as error: - logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) - compare_core(input_param, args.output_path, stack_mode=args.stack_mode, - auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) + npu_path = input_param.get("npu_path", None) + bench_path = input_param.get("bench_path", None) + if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: + try: + summary_compare, md5_compare = task_dumppath_get(input_param) + check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) + create_directory(args.output_path) + check_compare_param(input_param, args.output_path, summary_compare, md5_compare) + except (CompareException, FileCheckException) as error: + logger.error('Compare failed. Please check the arguments and do it again!') + sys.exit(error.code) + compare_core(input_param, args.output_path, stack_mode=args.stack_mode, + auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) def compare_core(input_parma, output_path, **kwargs): @@ -1051,15 +1054,3 @@ def _compare_parser(parser): help=" Whether to give advisor.", required=False) parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", help=" Whether to perform a fuzzy match on the api name.", required=False) - - -def _compare(parser=None): - if not parser: - parser = argparse.ArgumentParser() - _compare_parser(parser) - args = parser.parse_args(sys.argv[1:]) - compare(args) - - -if __name__ == '__main__': - _compare() -- Gitee From dfff2102ba4ccf558334dd7903bda4a63fc959ef Mon Sep 17 00:00:00 2001 From: CSNIU Date: Sat, 3 Aug 2024 11:35:00 +0800 Subject: [PATCH 094/791] =?UTF-8?q?=E9=87=8D=E6=9E=84msprobe=E7=9A=84compa?= =?UTF-8?q?re=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/advisor/advisor.py | 124 ++++++ .../msprobe/core/advisor/advisor_const.py | 59 +++ .../msprobe/core/advisor/advisor_result.py | 58 +++ .../msprobe/core/common/utils.py | 2 +- .../msprobe/core/compare/acc_compare.py | 60 +++ .../msprobe/core/compare/check.py | 106 +++++ .../{pytorch => core}/compare/highlight.py | 0 .../{pytorch => core}/compare/mapping.yaml | 0 .../{pytorch => core}/compare/match.py | 0 .../{pytorch => core}/compare/npy_compare.py | 2 +- .../msprobe/core/compare/utils.py | 402 ++++++++++++++++++ .../msprobe/mindspore/compare/ms_compare.py | 201 +++++++++ 12 files changed, 1012 insertions(+), 2 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/core/advisor/advisor.py create mode 100644 debug/accuracy_tools/msprobe/core/advisor/advisor_const.py create mode 100644 debug/accuracy_tools/msprobe/core/advisor/advisor_result.py create mode 100644 debug/accuracy_tools/msprobe/core/compare/acc_compare.py create mode 100644 debug/accuracy_tools/msprobe/core/compare/check.py rename debug/accuracy_tools/msprobe/{pytorch => core}/compare/highlight.py (100%) rename debug/accuracy_tools/msprobe/{pytorch => core}/compare/mapping.yaml (100%) rename debug/accuracy_tools/msprobe/{pytorch => core}/compare/match.py (100%) rename debug/accuracy_tools/msprobe/{pytorch => core}/compare/npy_compare.py (99%) create mode 100644 debug/accuracy_tools/msprobe/core/compare/utils.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor.py b/debug/accuracy_tools/msprobe/core/advisor/advisor.py new file mode 100644 index 000000000..ec2773e6d --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +from msprobe.mindspore.advisor.advisor_result import AdvisorResult +from msprobe.mindspore.advisor.advisor_const import AdvisorConst +from msprobe.mindspore.common.log import logger +from msprobe.core.common.utils import CompareException +from msprobe.core.common.file_check import FileChecker +from msprobe.core.common.const import Const, CompareConst, FileCheckConst + +class Advisor: + """ + Class for generate advisor + """ + + def __init__(self, input_data, out_path=""): + self.input_data = input_data + self.out_path = os.path.realpath(out_path) + self.file_type = None + + @staticmethod + def deterministic_advisor(message, node_name): + for api_name in AdvisorConst.NEED_DETERMINISTIC_API: + if api_name in node_name: + return AdvisorConst.DETERMINISTIC_SUGGEST + return message + + @staticmethod + def batch_norm_advisor(message, node_name): + if AdvisorConst.FUNC_BATCH_NORM in node_name and AdvisorConst.FORWARD_INPUT_1 in node_name: + message = AdvisorConst.BATCH_NORM_SUGGEST + return message + + def analyze_unmatched(self, analyze_data): + if self.file_type == Const.ALL: + accuracy_unmatched = analyze_data[ + analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] + else: + accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) | + (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)] + num_unmatch = len(accuracy_unmatched) + if num_unmatch != 0: + for i in range(len(accuracy_unmatched)): + item = accuracy_unmatched.iloc[i] + logger.warning("The tensor name matches but the shape or dtype does not match: {}" + .format(item[CompareConst.NPU_NAME])) + + def gen_advisor_result(self, pd_data): + first_failing_data = pd_data.iloc[0] + node_name = first_failing_data[CompareConst.NPU_NAME] + index = first_failing_data['index'] + message = self.gen_advisor_message(node_name) + logger.warning("Find %s accuracy not reached, the line is %s" % (node_name, index)) + result = AdvisorResult(node_name, index, message) + return result + + def gen_advisor_message(self, node_name): + if AdvisorConst.FORWARD in node_name: + if AdvisorConst.INPUT in node_name: + message = AdvisorConst.FORWARD_INPUT_SUGGEST + else: + message = AdvisorConst.FORWARD_OUTPUT_SUGGEST + message = self.deterministic_advisor(message, node_name) + else: + if AdvisorConst.INPUT in node_name: + message = AdvisorConst.BACKWARD_INPUT_SUGGEST + else: + message = AdvisorConst.BACKWARD_OUTPUT_SUGGEST + message = self.deterministic_advisor(message, node_name) + message = self.batch_norm_advisor(message, node_name) + return message + + def analysis(self): + self._check_path_vaild() + analyze_data = self._parse_input_data() + logger.info("Start analyzing the comparison result: %s" % self.file_type) + self.analyze_unmatched(analyze_data) + if self.file_type == Const.ALL: + failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] + elif self.file_type == Const.MD5: + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF] + elif self.file_type == Const.SUMMARY: + failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING] + if failing_data.empty: + logger.info("All data from api input/output accuracy reached") + result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) + else: + result = self.gen_advisor_result(failing_data) + message_list = result.print_advisor_log() + result.gen_summary_file(self.out_path, message_list) + + def _parse_input_data(self): + data_columns = self.input_data.columns.values + if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): + self.file_type = Const.ALL + elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): + self.file_type = Const.MD5 + elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): + self.file_type = Const.SUMMARY + else: + logger.error('Compare result does not meet the required conditions.') + raise CompareException(CompareException.INVALID_DATA_ERROR) + df = self.input_data.reset_index() + return df + + def _check_path_vaild(self): + out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) + out_path_checker.common_check() diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor_const.py b/debug/accuracy_tools/msprobe/core/advisor/advisor_const.py new file mode 100644 index 000000000..737c67591 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor_const.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + + +class AdvisorConst: + """ + Class for advisor const + """ + + # text symbol + NEW_LINE = "\n" + COLON = ": " + + # advisor summary key + SUSPECT_NODES = "Suspect Nodes" + LINE = "Line" + ADVISOR_SUGGEST = "Expert Advice" + + NO_ERROR_API = "NA" + + # advisor message + NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements." + FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \ + "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \ + "3. The fault may be caused by memory corruption and further analysis is required." + FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation." + BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected." + BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation." + BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \ + "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \ + "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \ + "3. Use seed_all(mode=True) to enable deterministic computing." + DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \ + "can seed_all(mode=True) to enable deterministic computing." + + FUNC_BATCH_NORM = "Functional_batch_norm" + FORWARD_INPUT_1 = "forward_input.1" + NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"] + BATCH_NORM = "batch_norm" + + # name keyword + INPUT = "input" + OUTPUT = "output" + FORWARD = "forward" + BACKWARD = "backward" diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py new file mode 100644 index 000000000..5d59068fc --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import os +import time + +from msprobe.mindspore.advisor.advisor_const import AdvisorConst +from msprobe.mindspore.common.log import logger +from msprobe.core.common.const import Const, FileCheckConst +from msprobe.core.common.file_check import change_mode + + +class AdvisorResult: + """ + Class for generate advisor result + """ + + def __init__(self, node, line, message): + self.suspect_node = node + self.line = line + self.advisor_message = message + + @staticmethod + def gen_summary_file(out_path, message_list): + file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) + result_file = os.path.join(out_path, file_name) + try: + with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file: + output_file.truncate(0) + message_list = [message + AdvisorConst.NEW_LINE for message in message_list] + output_file.writelines(message_list) + change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY) + except IOError as io_error: + logger.error("Failed to save %s, the reason is %s." % (result_file, io_error)) + else: + logger.info("The advisor summary is saved in: %s" % result_file) + + def print_advisor_log(self): + logger.info("The summary of the expert advice is as follows: ") + message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line), + AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node, + AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message] + for message in message_list: + logger.info(message) + return message_list diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 37a7733e1..85003afad 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -513,4 +513,4 @@ def get_header_index(header_name, summary_compare=False): def convert_tuple(data): - return data if isinstance(data, tuple) else (data, ) + return data if isinstance(data, tuple) else (data, ) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py new file mode 100644 index 000000000..88a919555 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -0,0 +1,60 @@ +from msprobe.core.compare.check import check_op +from msprobe.mindspore.compare.ms_compare import MSComparator +from msprobe.core.common.const import Const, CompareConst +from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ + get_error_message +from msprobe.core.common.exceptions import FileCheckException + + +class Comparator: + def __init__(self): + pass + + def match_op(self,npu_queue, bench_queue, fuzzy_match): + for b_index, b_op in enumerate(bench_queue[0: -1]): + if check_op(npu_queue[-1], b_op, fuzzy_match): + return len(npu_queue) - 1, b_index + if check_op(npu_queue[-1], bench_queue[-1], fuzzy_match): + return len(npu_queue) - 1, len(bench_queue) - 1 + for n_index, n_op in enumerate(npu_queue[0: -1]): + if check_op(n_op, bench_queue[-1], fuzzy_match): + return n_index, len(bench_queue) - 1 + return -1, -1 + + def compare_by_op(op_name, op_name_mapping_dict, input_parma): + npu_bench_name_list = op_name_mapping_dict[op_name] + data_name = npu_bench_name_list[1] + error_file, relative_err, error_flag = None, None, False + if data_name == '-1' or data_name == -1: # 没有真实数据路径 + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + else: + try: + msComparator= MSComparator() + n_value = msComparator.read_npy_data(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) + b_value = msComparator.read_npy_data(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) + except IOError as error: + error_file = error.filename + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + except FileCheckException: + error_file = data_name + n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE + error_flag = True + + n_value, b_value, error_flag = get_error_type(n_value, b_value, error_flag) + if not error_flag: + relative_err = get_relative_err(n_value, b_value) + n_value, b_value = reshape_value(n_value, b_value) + + err_msg = get_error_message(n_value, b_value, op_name, error_flag, error_file=error_file) + result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=relative_err) + + if npu_bench_name_list[0] != npu_bench_name_list[1]: + err_msg += " Fuzzy matching data, the comparison accuracy may be affected." + result_list.append(err_msg) + return result_list + + +testComparator= Comparator() + diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py new file mode 100644 index 000000000..aab8cb50e --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -0,0 +1,106 @@ +from msprobe.core.compare.match import graph_mapping +from msprobe.core.common.log import logger +from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.core.compare.utils import rename_api + + + +def check_struct_match(npu_dict, bench_dict): + npu_struct_in = npu_dict.get("input_struct") + bench_struct_in = bench_dict.get("input_struct") + npu_struct_out = npu_dict.get("output_struct") + bench_struct_out = bench_dict.get("output_struct") + is_match = npu_struct_in == bench_struct_in and npu_struct_out == bench_struct_out + if not is_match: + if len(npu_struct_in) == 0 or len(bench_struct_in) == 0 or len(npu_struct_in) != len(bench_struct_in): + return False + struct_in_is_match = check_type_shape_match(npu_struct_in, bench_struct_in) + struct_out_is_match = check_type_shape_match(npu_struct_out, bench_struct_out) + is_match = struct_in_is_match and struct_out_is_match + return is_match + +def check_type_shape_match(npu_struct, bench_struct): + shape_type_match = False + for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct): + npu_type = npu_type_shape[0] + npu_shape = npu_type_shape[1] + bench_type = bench_type_shape[0] + bench_shape = bench_type_shape[1] + shape_match = npu_shape == bench_shape + type_match = npu_type == bench_type + if not type_match: + if ([npu_type, bench_type] in [["Float16", "Float32"], ["Float32", "Float16"]] )or ([npu_type, bench_type] in [["torch.float16", "torch.float32"], ["torch.float32", "torch.float16"], + ["torch.float16", "torch.bfloat16"], ["torch.bfloat16", "torch.float16"]]): + type_match = True + else: + type_match = False + shape_type_match = shape_match and type_match + if not shape_type_match: + return False + return shape_type_match + +def check_graph_mode(a_op_name, b_op_name): + if "Aten" in a_op_name and "Aten" not in b_op_name: + return True + if "Aten" not in a_op_name and "Aten" in b_op_name: + return True + return False + + +def check_op(npu_dict, bench_dict, fuzzy_match): + a_op_name = npu_dict["op_name"] + b_op_name = bench_dict["op_name"] + graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) + if graph_mode: + return graph_mapping.match(a_op_name[0], b_op_name[0]) + struct_match = check_struct_match(npu_dict, bench_dict) + if not fuzzy_match: + return a_op_name == b_op_name and struct_match + is_match = True + try: + is_match = fuzzy_check_op(a_op_name, b_op_name) + except Exception as err: + logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) + is_match = False + return is_match and struct_match + + + +def fuzzy_check_op(npu_name_list, bench_name_list): + if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list): + return False + is_match = True + for npu_name, bench_name in zip(npu_name_list, bench_name_list): + is_match = fuzzy_check_name(npu_name, bench_name) + if not is_match: + break + return is_match + +def fuzzy_check_name(npu_name, bench_name): + if "forward" in npu_name and "forward" in bench_name: + is_match = rename_api(npu_name, "forward") == rename_api(bench_name, "forward") + elif "backward" in npu_name and "backward" in bench_name: + is_match = rename_api(npu_name, "backward") == rename_api(bench_name, "backward") + else: + is_match = npu_name == bench_name + return is_match + + + +def check_accuracy(cos, max_abs_err): + if cos == CompareConst.SHAPE_UNMATCH: + return CompareConst.ACCURACY_CHECK_UNMATCH + if cos == CompareConst.NONE or max_abs_err == CompareConst.NONE: + return CompareConst.NONE + if cos == "N/A" or max_abs_err == "N/A": + return CompareConst.ACCURACY_CHECK_NO + try: + cos, max_abs_err = float(cos), float(max_abs_err) + except ValueError: + logger.warning("Cosine or MaxAbsErr can not get float value.") + return CompareConst.NONE + if cos < CompareConst.COS_THRESHOLD and max_abs_err > CompareConst.MAX_ABS_ERR_THRESHOLD: + return CompareConst.ACCURACY_CHECK_NO + if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: + return CompareConst.ACCURACY_CHECK_NO + return CompareConst.ACCURACY_CHECK_YES \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py similarity index 100% rename from debug/accuracy_tools/msprobe/pytorch/compare/highlight.py rename to debug/accuracy_tools/msprobe/core/compare/highlight.py diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml b/debug/accuracy_tools/msprobe/core/compare/mapping.yaml similarity index 100% rename from debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml rename to debug/accuracy_tools/msprobe/core/compare/mapping.yaml diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/match.py b/debug/accuracy_tools/msprobe/core/compare/match.py similarity index 100% rename from debug/accuracy_tools/msprobe/pytorch/compare/match.py rename to debug/accuracy_tools/msprobe/core/compare/match.py diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/npy_compare.py b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py similarity index 99% rename from debug/accuracy_tools/msprobe/pytorch/compare/npy_compare.py rename to debug/accuracy_tools/msprobe/core/compare/npy_compare.py index 5a0feb4cd..0c75076c5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/npy_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py @@ -2,7 +2,7 @@ import abc import numpy as np from msprobe.core.common.utils import format_value from msprobe.core.common.const import Const, CompareConst -from msprobe.pytorch.common.log import logger +from msprobe.core.common.log import logger def handle_inf_nan(n_value, b_value): diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py new file mode 100644 index 000000000..0ed0b4ebd --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -0,0 +1,402 @@ + +import numpy as np +from msprobe.core.common.log import logger +from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ + format_value, check_file_not_exists, check_configuration_param, task_dumppath_get +from msprobe.core.compare.check import check_accuracy + + +def rename_api(npu_name, process): + npu_split = npu_name.split(process) + torch_func_index, in_out = npu_split[0], npu_split[1] + torch_func_split = torch_func_index.rsplit(Const.SEP, 2) + torch_func = str(torch_func_split[0]) + str(in_out) + return torch_func + +def read_op(op_data, op_name): + op_parsed_list = [] + if 'forward' in op_name: + if 'input_args' in op_data: + input_item = op_data['input_args'] + input_parsed_list = op_item_parse(input_item, op_name + '_input', None) + op_parsed_list = input_parsed_list.copy() + input_parsed_list.clear() + if 'input_kwargs' in op_data: + kwargs_item = op_data['input_kwargs'] + if isinstance(kwargs_item, dict) and "type" in kwargs_item or isinstance(kwargs_item, list): + kwarg_parsed_list = op_item_parse(kwargs_item, op_name + '_input', None) + op_parsed_list += kwarg_parsed_list + kwarg_parsed_list.clear() + elif kwargs_item: + for kwarg in kwargs_item: + kwarg_parsed_list = op_item_parse(kwargs_item[kwarg], op_name + '_input.' + kwarg, None) + op_parsed_list += kwarg_parsed_list + kwarg_parsed_list.clear() + if 'output' in op_data: + output_item = op_data['output'] + output_parsed_list = op_item_parse(output_item, op_name + '_output', None) + op_parsed_list += output_parsed_list + output_parsed_list.clear() + if 'backward' in op_name: + if 'grad_input' in op_data: + input_item = op_data['grad_input'] + input_parsed_list = op_item_parse(input_item, op_name + '_input', None) + op_parsed_list = input_parsed_list.copy() + input_parsed_list.clear() + if 'grad_output' in op_data: + output_item = op_data['grad_output'] + output_parsed_list = op_item_parse(output_item, op_name + '_output', None) + op_parsed_list += output_parsed_list + output_parsed_list.clear() + return op_parsed_list + +def op_item_parse(item, op_name, index, item_list=None, top_bool=True): + if item_list is None: + item_list = [] + if item is None or (isinstance(item, dict) and not item): + if not top_bool: + tmp = {'full_op_name': op_name + '.' + str(index), 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, + 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'} + else: + tmp = {'full_op_name': op_name + '.0', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, + 'shape': None, 'md5': None, 'data_name': '-1'} + item_list.append(tmp) + return item_list + if index is None: + if isinstance(item, dict): + full_op_name = op_name + '.0' + else: + full_op_name = op_name + else: + full_op_name = op_name + Const.SEP + str(index) + if isinstance(item, dict): + if 'type' not in item: + for kwarg in item: + kwarg_parsed_list = op_item_parse(item[kwarg], op_name + Const.SEP + kwarg, None) + item_list += kwarg_parsed_list + kwarg_parsed_list.clear() + elif 'dtype' in item: + parsed_item = item + parsed_item['full_op_name'] = full_op_name + item_list.append(parsed_item) + elif 'type' in item: + parsed_item = {} + if item['type'] == 'torch.Size': + parsed_item['full_op_name'] = full_op_name + parsed_item['dtype'] = 'torch.Size' + parsed_item['shape'] = str(item['value']) + parsed_item['md5'] = None + parsed_item['Max'] = None + parsed_item['Min'] = None + parsed_item['Mean'] = None + parsed_item['Norm'] = None + parsed_item['data_name'] = '-1' + item_list.append(parsed_item) + elif item['type'] == 'slice': + parsed_item['full_op_name'] = full_op_name + parsed_item['dtype'] = 'slice' + parsed_item['shape'] = str(np.shape(np.array(item['value']))) + parsed_item['md5'] = None + parsed_item['Max'] = None + parsed_item['Min'] = None + parsed_item['Mean'] = None + parsed_item['Norm'] = None + parsed_item['data_name'] = '-1' + item_list.append(parsed_item) + else: + parsed_item['full_op_name'] = full_op_name + parsed_item['dtype'] = str(type(item['value'])) + parsed_item['shape'] = '[]' + parsed_item['md5'] = None + parsed_item['Max'] = item['value'] + parsed_item['Min'] = item['value'] + parsed_item['Mean'] = item['value'] + parsed_item['Norm'] = item['value'] + parsed_item['data_name'] = '-1' + item_list.append(parsed_item) + else: + resolve_api_special_parameters(item, full_op_name, item_list) + else: + for j, item_spec in enumerate(item): + op_item_parse(item_spec, full_op_name, j, item_list=item_list, top_bool=False) + return item_list + +def resolve_api_special_parameters(data_dict, full_op_name, item_list): + """ + Function Description: + 解析下面格式的数据, 是api参数的一种特殊格式 + { + "last_hidden_state": { + "type": "torch.Tensor", + "dtype": "torch.bfloat16", + ... + }, + "loss": { + "type": "torch.Tensor", + "dtype": "torch.float32", + ... + } + } + Parameter: + data_dict: 字典格式的数据 + full_op_name: 参数的全名字符串 + item_list: 参数信息集合 + """ + for key, value in data_dict.items(): + if isinstance(value, dict): + parsed_item = value + parts = full_op_name.split(".") + parts.insert(-1, key) + full_op_name_new = ".".join(parts) + parsed_item['full_op_name'] = full_op_name_new + item_list.append(parsed_item) + +@dataclass +class ComparisonResult: + cos_result: list + max_err_result: list + max_relative_err_result: list + err_msgs: list + one_thousand_err_ratio_result: list + five_thousand_err_ratio_result: list + + +def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=False): + def get_accuracy_core(n_start, n_len, b_start, b_len, key): + min_len = min(n_len, b_len) + npu_stack_info = n_dict.get("stack_info", None) + bench_stack_info = b_dict.get("stack_info", None) + has_stack = npu_stack_info and bench_stack_info + + all_mode_bool = not (summary_compare or md5_compare) + if all_mode_bool: + npu_data_name = n_dict.get("data_name", None) + bench_data_name = b_dict.get("data_name", None) + + for index in range(min_len): + + n_name = n_dict['op_name'][n_start + index] + b_name = b_dict['op_name'][b_start + index] + n_struct = n_dict[key][index] + b_struct = b_dict[key][index] + err_msg = "" + if md5_compare: + result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], + n_struct[2], b_struct[2], + CompareConst.PASS if n_struct[2] == b_struct[2] else CompareConst.DIFF] + if has_stack and index == 0 and key == "input_struct": + result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + result.append(result_item) + continue + + if summary_compare: + result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], + " ", " ", " ", " ", " ", " ", " ", " "] + else: + result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], + " ", " ", " ", " ", " "] + + npu_summary_data = n_dict.get("summary")[n_start + index] + result_item.extend(npu_summary_data) + bench_summary_data = b_dict.get("summary")[b_start + index] + result_item.extend(bench_summary_data) + + if summary_compare: + start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) + warning_flag = False + for i, (npu_val, bench_val) in enumerate(zip(npu_summary_data, bench_summary_data)): + if isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)): + diff = npu_val - bench_val + if bench_val != 0: + relative = str(abs((diff / bench_val) * 100)) + '%' + else: + relative = "N/A" + result_item[start_idx + i] = diff + result_item[start_idx + i + 4] = relative + magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) + if magnitude_diff > 0.5: + warning_flag = True + else: + result_item[start_idx + i] = CompareConst.NONE + accuracy_check = CompareConst.WARNING if warning_flag else "" + err_msg += "Need double check api accuracy." if warning_flag else "" + for i in range(start_idx, len(result_item)): + if str(result_item[i]) in ('inf', '-inf', 'nan'): + result_item[i] = f'{result_item[i]}\t' + + result_item.append(accuracy_check if summary_compare else CompareConst.ACCURACY_CHECK_YES) + result_item.append(err_msg) + if has_stack and index == 0 and key == "input_struct": + result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + if all_mode_bool: + result_item.append(npu_data_name[n_start + index]) + + result.append(result_item) + + if n_len > b_len: + for index in range(b_len, n_len): + n_name = n_dict['op_name'][n_start + index] + n_struct = n_dict[key][index] + if md5_compare: + result_item = [n_name, CompareConst.NAN, n_struct[0], CompareConst.NAN, + n_struct[1], CompareConst.NAN, n_struct[2], CompareConst.NAN, CompareConst.NAN] + result.append(result_item) + continue + result_item = [n_name, CompareConst.NAN, n_struct[0], CompareConst.NAN, + n_struct[1], CompareConst.NAN, " ", " ", " ", " ", " "] + summary_data = n_dict.get("summary")[n_start + index] + result_item.extend(summary_data) + summary_data = [CompareConst.NAN for _ in range(len(n_dict.get("summary")[0]))] + result_item.extend(summary_data) + + err_msg = "" + result_item.append(CompareConst.ACCURACY_CHECK_YES) + result_item.append(err_msg) + + if has_stack and index == 0 and key == "input_struct": + result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) + if all_mode_bool: + result_item.append(npu_data_name[n_start + index]) + + result.append(result_item) + + n_num = len(n_dict['op_name']) + b_num = len(b_dict['op_name']) + n_num_input = len([name for name in n_dict['op_name'] if 'input' in name]) + b_num_input = len([name for name in b_dict['op_name'] if 'input' in name]) + n_num_kwarg = len([name for name in n_dict['op_name'] if 'kwarg' in name]) + b_num_kwarg = len([name for name in b_dict['op_name'] if 'kwarg' in name]) + n_num_output = n_num - n_num_input - n_num_kwarg + b_num_output = b_num - b_num_input - b_num_kwarg + get_accuracy_core(0, n_num_input, 0, b_num_input, 'input_struct') + get_accuracy_core(n_num_input, n_num_kwarg, b_num_input, b_num_kwarg, "kwargs_struct") + get_accuracy_core(n_num_input + n_num_kwarg, n_num_output, b_num_input + b_num_kwarg, b_num_output, 'output_struct') + +def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): + index_out = 0 + npu_stack_info = n_dict.get("stack_info", None) + bench_name, bench_type, bench_shape = CompareConst.NAN, CompareConst.NAN, CompareConst.NAN + err_msg = CompareConst.NO_BENCH + accuracy_check_res = CompareConst.NAN + for index, n_name in enumerate(n_dict["op_name"]): + if n_name.find("input") != -1: + n_struct = n_dict["input_struct"][index] + else: + n_struct = n_dict["output_struct"][index_out] + index_out += 1 + + result_item = [n_name, bench_name, n_struct[0], bench_type, n_struct[1], bench_shape] + if md5_compare: + result_item.extend([CompareConst.NAN] * 3) + if npu_stack_info and index == 0: + result_item.extend(npu_stack_info) + result.append(result_item) + continue + if summary_compare: + result_item.extend([CompareConst.NAN] * 8) + else: + result_item.extend([CompareConst.NAN] * 5) + summary_data = n_dict.get("summary")[index] + result_item.extend(summary_data) + summary_data = [CompareConst.NAN] * 4 + result_item.extend(summary_data) + result_item.append(accuracy_check_res) + result_item.append(err_msg) + if npu_stack_info and index == 0: + result_item.extend(npu_stack_info) + if not md5_compare and not summary_compare and result_item[1] == CompareConst.NAN: + if index == 0: + result_item.extend(["-1"]) + else: + result_item.extend([CompareConst.NONE, "-1"]) + result.append(result_item) + + +def merge_tensor(tensor_list, summary_compare, md5_compare): + op_dict = {} + op_dict["op_name"] = [] + op_dict["input_struct"] = [] + op_dict["kwargs_struct"] = [] + op_dict["output_struct"] = [] + op_dict["summary"] = [] + op_dict["stack_info"] = [] + + all_mode_bool = not (summary_compare or md5_compare) + if all_mode_bool: + op_dict["data_name"] = [] + + for tensor in tensor_list: + if len(tensor) == 2: + op_dict['stack_info'].append(tensor['full_info']) + break + op_dict["op_name"].append(tensor['full_op_name']) + if not md5_compare: + if tensor['full_op_name'].find("input") != -1: + op_dict["input_struct"].append((tensor['dtype'], tensor['shape'])) + elif tensor['full_op_name'].find("kwarg") != -1: + op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'])) + elif tensor['full_op_name'].find("output") != -1: + op_dict["output_struct"].append((tensor['dtype'], tensor['shape'])) + else: + if tensor['full_op_name'].find("input") != -1: + op_dict["input_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) + elif tensor['full_op_name'].find("kwarg") != -1: + op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) + elif tensor['full_op_name'].find("output") != -1: + op_dict["output_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) + + op_dict["summary"].append([tensor['Max'], tensor['Min'], tensor['Mean'], tensor['Norm']]) + + if all_mode_bool: + op_dict["data_name"].append(tensor['data_name']) + + if not op_dict["kwargs_struct"]: + del op_dict["kwargs_struct"] + return op_dict if op_dict["op_name"] else {} + + + +def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): + """ + Save comparison results into the result DataFrame with thread safety. + Args: + offset: offset for index + result: data struct of ComparisonResult + result_df: result of DataFrame + lock: thread lock + + Returns: + comparison results in DataFrame + """ + + lock.acquire() + try: + for i, _ in enumerate(result.cos_result): + process_index = i + offset + result_df.loc[process_index, CompareConst.COSINE] = result.cos_result[i] + result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] + result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] + result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] + result_df.loc[process_index, CompareConst.ACCURACY] = check_accuracy(result.cos_result[i], result.max_err_result[i]) + result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = result.one_thousand_err_ratio_result[i] + result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = result.five_thousand_err_ratio_result[i] + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + finally: + lock.release() + + + + diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py new file mode 100644 index 000000000..0908c44c0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -0,0 +1,201 @@ +from msprobe.core.compare.acc_compare import Comparator +from msprobe.core.common.log import logger + + + + + + +import json +import multiprocessing +import os.path +import sys + +import numpy as np +import pandas as pd +import openpyxl +from openpyxl.styles import PatternFill +from collections import namedtuple +from dataclasses import dataclass + +from msprobe.mindspore.compare.match import graph_mapping +from msprobe.mindspore.compare.highlight import HighlightRules, get_header_index + +from msprobe.mindspore.advisor.advisor import Advisor +from msprobe.mindspore.common.log import logger +from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ + format_value, check_file_not_exists, check_configuration_param, task_dumppath_get +from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, create_directory +from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.core.common.exceptions import FileCheckException +from msprobe.core.compare.utils import ComparisonResult,_save_cmp_result,merge_tensor, get_un_match_accuracy,get_accuracy,read_op + + +class MSComparator (Comparator): + def __init__(self): + super().__init__() + + + def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): + cos_result = [] + max_err_result = [] + max_relative_err_result = [] + err_mess = [] + one_thousand_err_ratio_result = [] + five_thousand_err_ratio_result = [] + is_print_compare_log = input_parma.get("is_print_compare_log") + for i in range(len(result_df)): + op_name = result_df.iloc[i, 0] + if is_print_compare_log: + logger.info("start compare: {}".format(op_name)) + cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = self.compare_by_op( + op_name, dump_path_dict, input_parma) + if is_print_compare_log: + logger.info( + "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " + "five_thousand_err_ratio {}".format(op_name, cos_sim, max_abs_err, max_relative_err, err_msg, + one_thousand_err_ratio, five_thousand_err_ratio)) + cos_result.append(cos_sim) + max_err_result.append(max_abs_err) + max_relative_err_result.append(max_relative_err) + err_mess.append(err_msg) + one_thousand_err_ratio_result.append(one_thousand_err_ratio) + five_thousand_err_ratio_result.append(five_thousand_err_ratio) + + cr = ComparisonResult( + cos_result=cos_result, + max_err_result=max_err_result, + max_relative_err_result=max_relative_err_result, + err_msgs=err_mess, + one_thousand_err_ratio_result=one_thousand_err_ratio_result, + five_thousand_err_ratio_result=five_thousand_err_ratio_result + ) + + return _save_cmp_result(idx, cr, result_df, lock) + + def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): + npu_json_handle, bench_json_handle, stack_json_handle = file_handles + npu_json_data = json.load(npu_json_handle) + bench_json_data = json.load(bench_json_handle) + stack_json_data = json.load(stack_json_handle) + + if fuzzy_match: + logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") + + npu_ops_queue = [] + bench_ops_queue = [] + result = [] + + ops_npu_iter = iter(npu_json_data['data']) + ops_bench_iter = iter(bench_json_data['data']) + read_err_npu = True + read_err_bench = True + last_npu_ops_len = 0 + last_bench_ops_len = 0 + + while True: + if not read_err_npu and not read_err_bench: + break + try: + last_npu_ops_len = len(npu_ops_queue) + op_name_npu = next(ops_npu_iter) + read_err_npu = True + + npu_op_data = npu_json_data['data'][op_name_npu] + npu_op_parsed_list = read_op(npu_op_data, op_name_npu) + if op_name_npu in stack_json_data: + npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': stack_json_data[op_name_npu]}) + else: + npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': None}) + + npu_merge_list = merge_tensor(npu_op_parsed_list, summary_compare, md5_compare) + if npu_merge_list: + npu_ops_queue.append(npu_merge_list) + except StopIteration: + read_err_npu = False + try: + last_bench_ops_len = len(bench_ops_queue) + op_name_bench = next(ops_bench_iter) + + bench_op_data = bench_json_data['data'][op_name_bench] + bench_op_parsed_list = read_op(bench_op_data, op_name_bench) + if op_name_bench in stack_json_data: + bench_op_parsed_list.append( + {'full_op_name': op_name_bench, 'full_info': stack_json_data[op_name_bench]}) + else: + bench_op_parsed_list.append({'full_op_name': op_name_bench, 'full_info': None}) + + bench_merge_list = merge_tensor(bench_op_parsed_list, summary_compare, md5_compare) + if bench_merge_list: + bench_ops_queue.append(bench_merge_list) + except StopIteration: + read_err_bench = False + + # merge all boolean expressions + both_empty = not npu_ops_queue and not bench_ops_queue + no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) + if both_empty or no_change: + continue + + n_match_point, b_match_point = super().match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) + if n_match_point == -1 and b_match_point == -1: + continue + n_match_data = npu_ops_queue[n_match_point] + b_match_data = bench_ops_queue[b_match_point] + un_match_data = npu_ops_queue[0: n_match_point] + for npu_data in un_match_data: + get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) + get_accuracy(result, n_match_data, b_match_data, summary_compare, md5_compare) + del npu_ops_queue[0: n_match_point + 1] + del bench_ops_queue[0: b_match_point + 1] + if npu_ops_queue: + for npu_data in npu_ops_queue: + get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) + + header = [] + if md5_compare: + header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] + elif summary_compare: + header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] + else: + header = CompareConst.COMPARE_RESULT_HEADER[:] + + all_mode_bool = not (summary_compare or md5_compare) + if stack_mode: + if all_mode_bool: + header.append(CompareConst.STACK) + header.append(CompareConst.DATA_NAME) + else: + header.append(CompareConst.STACK) + else: + if all_mode_bool: + for row in result: + del row[-2] + header.append(CompareConst.DATA_NAME) + else: + for row in result: + del row[-1] + + result_df = pd.DataFrame(result, columns=header) + return result_df + + + def read_npy_data(self,dir_path, file_name): + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.NUMPY_SUFFIX, False) + data_path = path_checker.common_check() + data_value = np.load(data_path) # detach for less memory + if data_value.dtype == np.float16: + data_value=data_value.astype(np.float32) + + return data_value + + + + + + + + + \ No newline at end of file -- Gitee From 9953ba871d58076a0cb10afdb4d4b463004eb0b1 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Sat, 3 Aug 2024 15:49:49 +0800 Subject: [PATCH 095/791] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=AF=B9=E5=A4=96?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/compare/Multiprocessing_compute.py | 55 ++++++++ .../msprobe/core/compare/acc_compare.py | 2 +- .../msprobe/core/compare/highlight.py | 129 ++++++++++++++++++ .../msprobe/mindspore/compare/ms_compare.py | 91 +++++++++++- 4 files changed, 273 insertions(+), 4 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py diff --git a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py new file mode 100644 index 000000000..e0c52aa6f --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py @@ -0,0 +1,55 @@ + +import multiprocessing +import pandas as pd +from msprobe.core.common.log import logger +from msprobe.core.common.utils import CompareException + + + +def _handle_multi_process(func, input_parma, result_df, lock): + process_num = int((multiprocessing.cpu_count() + 1) / 2) + op_name_mapping_dict = read_dump_data(result_df) + + df_chunk_size = len(result_df) // process_num + if df_chunk_size > 0: + df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] + else: + df_chunks = [result_df] + + results = [] + pool = multiprocessing.Pool(process_num) + + def err_call(args): + logger.error('multiprocess compare failed! Reason: {}'.format(args)) + try: + pool.terminate() + except OSError as e: + logger.error("pool terminate failed") + + for process_idx, df_chunk in enumerate(df_chunks): + idx = df_chunk_size * process_idx + result = pool.apply_async(func, + args=(idx, op_name_mapping_dict, df_chunk, lock, input_parma), + error_callback=err_call) + results.append(result) + final_results = [r.get() for r in results] + pool.close() + pool.join() + return pd.concat(final_results, ignore_index=True) + +def read_dump_data(result_df): + try: + npu_dump_name_list = result_df.iloc[0:, 0].tolist() + npu_dump_tensor_list = result_df.iloc[0:, -1].tolist() + op_name_mapping_dict = {} + for index, _ in enumerate(npu_dump_name_list): + npu_dump_name = npu_dump_name_list[index] + npu_dump_tensor = npu_dump_tensor_list[index] + op_name_mapping_dict[npu_dump_name] = [npu_dump_tensor, npu_dump_tensor] + return op_name_mapping_dict + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 88a919555..dc581e70e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -21,7 +21,7 @@ class Comparator: return n_index, len(bench_queue) - 1 return -1, -1 - def compare_by_op(op_name, op_name_mapping_dict, input_parma): + def compare_by_op(self,op_name, op_name_mapping_dict, input_parma): npu_bench_name_list = op_name_mapping_dict[op_name] data_name = npu_bench_name_list[1] error_file, relative_err, error_flag = None, None, False diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 82f0022f8..17dee2f50 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -1,8 +1,16 @@ import math import abc import numpy as np +from collections import namedtuple +import openpyxl +from openpyxl.styles import PatternFill +from collections import namedtuple from msprobe.core.common.utils import get_header_index from msprobe.core.common.const import CompareConst +from msprobe.core.common.log import logger +from msprobe.core.common.utils import CompareException +from msprobe.core.common.file_check import change_mode +from msprobe.core.common.const import CompareConst, FileCheckConst class HighlightCheck(abc.ABC): @@ -98,3 +106,124 @@ class HighlightRules: "check_order_magnitude": CheckOrderMagnitude(), "check_max_relative_diff": CheckMaxRelativeDiff(), } + + +def find_error_rows(result, last_len, n_num_input, highlight_dict, summary_compare=False, md5_compare=False): + """找到单个API中需要高亮的行""" + if md5_compare: + return + npu_max_index = get_header_index('NPU max', summary_compare) + bench_max_index = get_header_index('Bench max', summary_compare) + max_diff_index = get_header_index('Max diff' if summary_compare else 'MaxAbsErr', summary_compare) + + red_lines, yellow_lines = [], [] + LineInfo = namedtuple('LineInfo', ['line_data', 'num_pointer']) + ApiInfo = namedtuple('ApiInfo', ['api_input', 'api_output', 'num_pointer']) + ColorColumns = namedtuple('ColorColumns', ['red', 'yellow']) + color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) + + # 对单行API的输入或输出进行误差判断 + for i, line in enumerate(result): + num = last_len + i + line_info = LineInfo(line_data=line, num_pointer=num) + for rule in HighlightRules.basic_rules.values(): + rule.apply(line_info, color_columns, summary_compare) + + # 对API的输出与输入比较，进行误差判断 + for n, api_out in enumerate(result[n_num_input:len(result)]): + num = last_len + n_num_input + n + if num in red_lines: + continue + if not isinstance(api_out[npu_max_index], (float, int)) \ + or not isinstance(api_out[bench_max_index], (float, int)) \ + or not isinstance(api_out[max_diff_index], (float, int)): + continue + for _, api_in in enumerate(result[0:n_num_input]): + if not isinstance(api_in[npu_max_index], (float, int)) \ + or not isinstance(api_in[bench_max_index], (float, int)) \ + or not isinstance(api_in[max_diff_index], (float, int)): + continue + + api_info = ApiInfo(api_input=api_in, api_output=api_out, num_pointer=num) + if summary_compare: + for rule in HighlightRules.summary_compare_rules.values(): + rule.apply(api_info, color_columns, summary_compare) + else: + for rule in HighlightRules.compare_rules.values(): + rule.apply(api_info, color_columns, summary_compare) + + highlight_dict.get('red_rows', []).extend(list(set(red_lines))) + highlight_dict.get('yellow_rows', []).extend(list(set(yellow_lines) - set(red_lines))) + + +def get_name_and_state(name): + """Get api/module name and state""" + if "input" in name: + api_name = name.split("input")[0] + state = "input" + else: + api_name = name.split("output")[0] + state = "output" + return api_name, state + +def find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare): + """将dataframe根据API分组，并找到有误差的算子用于高亮""" + result = result_df.values + start, input_num, output_num, end = 0, 0, 0, len(result_df) + last_api_name, last_state = None, None + num, last_len = 0, 0 + for res_i in result: + api_name, state = get_name_and_state(res_i[0]) + if last_api_name: + if api_name == last_api_name: + if state == last_state: + num += 1 + else: + input_num = num + num, last_state = 1, state + else: + output_num = num + find_error_rows(result[start:start + input_num + output_num], start, input_num, highlight_dict, + summary_compare, md5_compare) + num, last_api_name, last_state = 1, api_name, state + start += input_num + output_num + input_num, output_num = 1, 0 + else: + num, last_api_name, last_state = 1, api_name, state + if state: + if state == "input": + input_num = num + else: + output_num = num + find_error_rows(result[start:start + input_num + output_num], start, input_num, highlight_dict, summary_compare, md5_compare) + + +def highlight_rows_xlsx(result_df, highlight_dict, file_path): + """Write and highlight results in Excel""" + logger.info('Compare result is %s' % file_path) + + wb = openpyxl.Workbook() + ws = wb.active + + # write header + for j, col_name in enumerate(result_df.columns, start=1): + ws.cell(row=1, column=j, value=col_name) + + for i, row in enumerate(result_df.iterrows(), start=2): + for j, value in enumerate(row[1], start=1): + if not isinstance(value, (float, int)): + value = f'{str(value)}\t' if str(value) in ('inf', '-inf', 'nan') else str(value) + ws.cell(row=i, column=j, value=f'{str(value)}\t' if str(value) in ('inf', '-inf', 'nan') else value) + + if (i - 2) in highlight_dict['red_rows']: + ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.RED, + end_color=CompareConst.RED, fill_type="solid") + elif (i - 2) in highlight_dict['yellow_rows']: + ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.YELLOW, + end_color=CompareConst.YELLOW, fill_type="solid") + try: + wb.save(file_path) + except Exception as e: + logger.error('Save result file failed') + raise CompareException(CompareException.WRITE_FILE_ERROR) from e + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 0908c44c0..b7a839c22 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -29,6 +29,8 @@ from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, c from msprobe.core.common.const import Const, CompareConst, FileCheckConst from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.utils import ComparisonResult,_save_cmp_result,merge_tensor, get_un_match_accuracy,get_accuracy,read_op +from msprobe.core.compare.highlight import find_compare_result_error_rows,highlight_rows_xlsx +from msprobe.core.compare.Multiprocessing_compute import _handle_multi_process class MSComparator (Comparator): @@ -190,12 +192,95 @@ class MSComparator (Comparator): data_value=data_value.astype(np.float32) return data_value - - - + + def _do_multi_process(self,input_parma, result_df): + try: + result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + +def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, + fuzzy_match=False): + try: + summary_compare, md5_compare = task_dumppath_get(input_parma) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + create_directory(output_path) + check_compare_param(input_parma, output_path, stack_mode, summary_compare, md5_compare) + except CompareException as error: + logger.error('Compare failed. Please check the arguments and do it again!') + sys.exit(error.code) + compare_core(input_parma, output_path, stack_mode=stack_mode, + auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) + +# def compare(args): +# with FileOpen(args.input_path, "r") as file: +# input_param = json.load(file) +# try: +# summary_compare, md5_compare = task_dumppath_get(input_param) +# check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) +# create_directory(args.output_path) +# check_compare_param(input_param, args.output_path, summary_compare, md5_compare) +# except (CompareException, FileCheckException) as error: +# logger.error('Compare failed. Please check the arguments and do it again!') +# sys.exit(error.code) +# msComparator.compare_core(input_param, args.output_path, stack_mode=args.stack_mode, +# auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, +# md5_compare=md5_compare) + +def compare_core(input_parma, output_path, **kwargs): + """ + Compares data from multiple JSON files and generates a comparison report. + + Args: + input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", + "stack_path"). + output_path (str): The path where the output Excel report will be saved. + **kwargs: Additional keyword arguments including: + - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. + - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. + - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. + - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. + - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. + - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. + + Returns: + """ + # get kwargs or set default value + stack_mode = kwargs.get('stack_mode', False) + auto_analyze = kwargs.get('auto_analyze', True) + suffix = kwargs.get('suffix', '') + fuzzy_match = kwargs.get('fuzzy_match', False) + summary_compare = kwargs.get('summary_compare', False) + md5_compare = kwargs.get('md5_compare', False) + + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") + file_name = add_time_with_xlsx("compare_result" + suffix) + file_path = os.path.join(os.path.realpath(output_path), file_name) + check_file_not_exists(file_path) + highlight_dict = {'red_rows': [], 'yellow_rows': []} + msComparator= MSComparator() + with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_path"), "r") as stack_json: + result_df = msComparator.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, + summary_compare, md5_compare) + + if not md5_compare and not summary_compare: + result_df = msComparator._do_multi_process(input_parma, result_df) + find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) + highlight_rows_xlsx(result_df, highlight_dict, file_path) + if auto_analyze: + advisor = Advisor(result_df, output_path) + advisor.analysis() + + \ No newline at end of file -- Gitee From 47d1c0e4a01df1b8ff1840774b572483037c1815 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 3 Aug 2024 16:21:23 +0800 Subject: [PATCH 096/791] add mindspore compare command --- debug/accuracy_tools/msprobe/msprobe.py | 32 ++++++++++++++----------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index a815e7c53..e340c67eb 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -33,7 +33,7 @@ def main(): f"For any issue, refer README.md first", ) parser.set_defaults(print_help=parser.print_help) - parser.add_argument('-f', '--framework', required=True, choices=['pytorch'], + parser.add_argument('-f', '--framework', required=True, choices=['pytorch', 'mindspore'], help='Deep learning framework.') subparsers = parser.add_subparsers() subparsers.add_parser('parse') @@ -53,19 +53,23 @@ def main(): parser.print_help() sys.exit(0) args = parser.parse_args(sys.argv[1:]) - if sys.argv[3] == "run_ut": - run_ut_command(args) - elif sys.argv[3] == "parse": - cli_parse() - elif sys.argv[3] == "multi_run_ut": - config = prepare_config(args) - run_parallel_ut(config) - elif sys.argv[3] == "api_precision_compare": - _api_precision_compare_command(args) - elif sys.argv[3] == "run_overflow_check": - _run_overflow_check_command(args) - elif sys.argv[3] == "compare": - compare(args) + if sys.argv[2] == "pytorch": + if sys.argv[3] == "run_ut": + run_ut_command(args) + elif sys.argv[3] == "parse": + cli_parse() + elif sys.argv[3] == "multi_run_ut": + config = prepare_config(args) + run_parallel_ut(config) + elif sys.argv[3] == "api_precision_compare": + _api_precision_compare_command(args) + elif sys.argv[3] == "run_overflow_check": + _run_overflow_check_command(args) + elif sys.argv[3] == "compare": + compare(args) + else: + if sys.argv[3] == "compare": + pass if __name__ == "__main__": -- Gitee From 988ec62f3aa6869c3b08c0c1af097866d21a31af Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Fri, 2 Aug 2024 14:54:50 +0800 Subject: [PATCH 097/791] enable MS L2 task in graph mode --- .../msprobe/core/common/const.py | 18 ++++++++- .../mindspore/debugger/debugger_config.py | 10 ++--- .../mindspore/debugger/precision_debugger.py | 38 +++++++++++++++---- .../msprobe/mindspore/service.py | 5 ++- 4 files changed, 53 insertions(+), 18 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index b59536aa5..c1a453a21 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -81,12 +81,12 @@ class Const: INT_TYPE = [np.int32, np.int64] NPU = 'NPU' DISTRIBUTED = 'Distributed' - + INPLACE_LIST = [ "broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter", "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single", "all_to_all" ] - + CONVERT = { "int32_to_int64": ["torch.int32", "torch.int64"], } @@ -253,3 +253,17 @@ class OverflowConst: OVERFLOW_DEBUG_MODE_ENABLE = "OVERFLOW_DEBUG_MODE_ENABLE" OVERFLOW_ORIGINAL_MODE = 0 OVERFLOW_DEBUG_MODE = 1 + + +class MsConst: + CELL = "cell" + API = "api" + KERNEL = "kernel" + TOOL_LEVEL_DICT = { + "L0": CELL, + "L1": API, + "L2": KERNEL + } + PYNATIVE_MODE = "pynative" + GRAPH_GE_MODE = "graph_ge" + GRAPH_KBYK_MODE = "graph_kbyk" diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 0f0cdd905..23cb7294b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -1,14 +1,10 @@ import os + from msprobe.core.common.utils import Const +from msprobe.core.common.const import MsConst class DebuggerConfig: - convert_map = { - "L0": "cell", - "L1": "api", - "L2": 'kernel' - } - def __init__(self, common_config, task_config): self.dump_path = common_config.dump_path self.task = common_config.task @@ -16,7 +12,7 @@ class DebuggerConfig: self.step = [] if not common_config.step else common_config.step if not common_config.level: common_config.level = "L1" - self.level = DebuggerConfig.convert_map[common_config.level] + self.level = MsConst.TOOL_LEVEL_DICT.get(common_config.level, MsConst.API) self.level_ori = common_config.level self.list = [] if not task_config.list else task_config.list self.scope = [] if not task_config.scope else task_config.scope diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 7082fc13e..5475dc358 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -1,9 +1,12 @@ import os + import mindspore as ms + from msprobe.mindspore.service import Service from msprobe.mindspore.ms_config import parse_json_config from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.task_handler_factory import TaskHandlerFactory +from msprobe.core.common.const import MsConst class PrecisionDebugger: @@ -14,6 +17,8 @@ class PrecisionDebugger: cls._instance = super().__new__(cls) cls._instance.initialized = False cls._instance.config = None + cls.service = None + cls.first_start = False return cls._instance def __init__(self, config_path=None): @@ -24,28 +29,47 @@ class PrecisionDebugger: common_config, task_config = parse_json_config(config_path) self.config = DebuggerConfig(common_config, task_config) self.initialized = True - self.service = Service(self.config) + + @staticmethod + def _get_execution_mode(): + if ms.get_context("mode") == ms.GRAPH_MODE: + if ms.context.get_jit_config().get("jit_level") == "O2" or ms.get_context("jit_level") == "O2": + return MsConst.GRAPH_GE_MODE + else: + return MsConst.GRAPH_KBYK_MODE + else: + return MsConst.PYNATIVE_MODE @classmethod def start(cls): instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") - if ms.get_context("mode") == ms.PYNATIVE_MODE and instance.config.level_ori == "L1": + + instance.config.execution_mode = instance._get_execution_mode() + if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.level == MsConst.API: + if not instance.service: + instance.service = Service(instance.config) instance.service.start() else: - handler = TaskHandlerFactory.create(instance.config) - handler.handle() + if not instance.first_start: + handler = TaskHandlerFactory.create(instance.config) + handler.handle() + + instance.first_start = True @classmethod def stop(cls): instance = cls._instance if not instance: raise Exception("PrecisionDebugger instance is not created.") - instance.service.stop() + if instance.service: + instance.service.stop() @classmethod def step(cls): - if not cls._instance: + instance = cls._instance + if not instance: raise Exception("PrecisionDebugger instance is not created.") - cls._instance.service.step() + if instance.service: + instance.service.step() diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 87287aabd..50776aaf1 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -14,6 +14,7 @@ # ============================================================================ import os +import copy from pathlib import Path import functools from collections import defaultdict @@ -33,9 +34,9 @@ from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell class Service: def __init__(self, config): self.model = None - self.config = config + self.config = copy.deepcopy(config) self.config.level = self.config.level_ori - self.data_collector = build_data_collector(config) + self.data_collector = build_data_collector(self.config) self.switch = False self.current_iter = 0 self.first_start = True -- Gitee From e6b261c9d575872e4d167b110b0e762e5a942445 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Aug 2024 09:39:50 +0800 Subject: [PATCH 098/791] bugfix --- .../msprobe/core/data_dump/data_processor/pytorch_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index dbe6f0210..5672c3f9a 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -168,7 +168,7 @@ class TensorDataProcessor(PytorchDataProcessor): def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) if not path_len_exceeds_limit(file_path): - saved_tensor = tensor.contiguous().detach().cpu() + saved_tensor = tensor.contiguous().detach() torch.save(saved_tensor, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) else: -- Gitee From b3b74b656017069cc8d9cfaee192c7e21387c055 Mon Sep 17 00:00:00 2001 From: lijiaojiao Date: Fri, 2 Aug 2024 11:22:43 +0800 Subject: [PATCH 099/791] =?UTF-8?q?=E5=AE=89=E5=85=A8=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E5=8D=95=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grad_tool/common/base_comparator.py | 20 ++++++++++++++++--- .../accuracy_tools/grad_tool/common/utils.py | 3 ++- .../grad_tool/grad_ms/grad_analyzer.py | 5 ++++- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/common/base_comparator.py b/debug/accuracy_tools/grad_tool/common/base_comparator.py index d3254ae71..03f74a21e 100644 --- a/debug/accuracy_tools/grad_tool/common/base_comparator.py +++ b/debug/accuracy_tools/grad_tool/common/base_comparator.py @@ -7,7 +7,10 @@ import pandas as pd import matplotlib.pyplot as plt from grad_tool.common.constant import GradConst -from grad_tool.common.utils import write_csv, check_file_or_directory_path, print_info_log, create_directory +from grad_tool.common.utils import write_csv, check_file_or_directory_path, print_info_log, create_directory, print_error_log + +from ptdbg_ascend.src.python.ptdbg_ascend.common import file_check_util +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, check_path_pattern_valid, check_path_length class BaseComparator(ABC): @@ -85,8 +88,19 @@ class BaseComparator(ABC): picture_dir = os.path.join(output_dir, "similarities_picture") if not os.path.isdir(picture_dir): create_directory(picture_dir) - plt.savefig(os.path.join(picture_dir, f"{key}_similarities.png")) - plt.close() + file_path= os.path.join(picture_dir, f"{key}_similarities.png") + if os.path.exists(file_path): + raise ValueError(f"File {file_path} already exists") + check_path_length(file_path) + check_path_pattern_valid(file_path) + try: + plt.savefig(file_path) + plt.close() + except Exception as e: + error_message = "An unexpected error occurred: %s when savfig to %s" % (str(e), file_path) + print_error_log(error_message) + full_path = os.path.realpath(file_path) + file_check_util.change_mode(full_path, FileCheckConst.DATA_FILE_AUTHORITY) head_tuple = tuple(['step'] + [str(step) for step in steps]) write_csv(os.path.join(output_dir, "similarities.csv"), [[key] + value], head_tuple) diff --git a/debug/accuracy_tools/grad_tool/common/utils.py b/debug/accuracy_tools/grad_tool/common/utils.py index f40f8688c..fceda8ce0 100644 --- a/debug/accuracy_tools/grad_tool/common/utils.py +++ b/debug/accuracy_tools/grad_tool/common/utils.py @@ -7,6 +7,7 @@ import yaml import pandas as pd from grad_tool.common.constant import GradConst +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen def _print_log(level, msg, end='\n'): @@ -114,7 +115,7 @@ class ListCache(list): def get_config(filepath): - with open(filepath, 'r') as file: + with FileOpen(filepath, 'r') as file: config = yaml.safe_load(file) return config diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py index 895b8f2ae..c843df388 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py @@ -16,6 +16,7 @@ from grad_tool.common.utils import ListCache, print_warn_log from grad_tool.common.utils import create_directory, check_file_or_directory_path, write_csv from grad_tool.grad_ms.global_context import grad_context from grad_tool.grad_ms.global_context import GlobalContext +from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker def get_rank_id(): @@ -169,6 +170,8 @@ class CSVGenerator(Process): stat_data = None max_try = 10 while max_try: + file_path_checker = FileChecker(file_path, FileCheckConst.DIR,FileCheckConst.READ_ABLE) + file_path = file_path_checker.common_check() try: stat_data = np.load(file_path) return stat_data @@ -177,7 +180,7 @@ class CSVGenerator(Process): max_try -= 1 time.sleep(0.1) return stat_data - + def gen_csv_line(self, file_path: str, stat_data) -> None: shape_dim = int(stat_data[GradConst.SHAPE_DIM_IDX]) file_name = os.path.basename(file_path) -- Gitee From 4323775e44f5a3c6d6f118f670056166748f3777 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 5 Aug 2024 10:02:05 +0800 Subject: [PATCH 100/791] review fix --- .../accuracy_tools/msprobe/core/common/const.py | 2 +- .../accuracy_tools/msprobe/core/common/utils.py | 4 ++-- debug/accuracy_tools/msprobe/msprobe.py | 2 +- .../msprobe/pytorch/compare/acc_compare.py | 16 ++++++++++------ 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index e3d3c4e01..f563690ee 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -20,7 +20,7 @@ class Const: DEFAULT_PATH = './' WHITE_LIST = 'white_list' BLACK_LIST = 'black_list' - DUMP_TENSOR_DATA = '/dump_tensor_data' + DUMP_TENSOR_DATA = 'dump_tensor_data' # dump mode ALL = "all" diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 37a7733e1..1058f04b7 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -496,8 +496,8 @@ def task_dumppath_get(input_param): else: logger.error(f"Compare is not required for overflow_check or free_benchmark.") raise CompareException(CompareException.INVALID_TASK_ERROR) - input_param['npu_dump_data_dir'] = os.path.dirname(npu_path) + Const.DUMP_TENSOR_DATA - input_param['bench_dump_data_dir'] = os.path.dirname(bench_path) + Const.DUMP_TENSOR_DATA + input_param['npu_dump_data_dir'] = os.path.join(os.path.dirname(npu_path) + Const.DUMP_TENSOR_DATA) + input_param['bench_dump_data_dir'] = os.path.join(os.path.dirname(bench_path) + Const.DUMP_TENSOR_DATA) return summary_compare, md5_compare diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index a815e7c53..92d6bbfb6 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -65,7 +65,7 @@ def main(): elif sys.argv[3] == "run_overflow_check": _run_overflow_check_command(args) elif sys.argv[3] == "compare": - compare(args) + compare_cli(args) if __name__ == "__main__": diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 0072d9432..424aa3f7b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -654,22 +654,26 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) -def compare(args): +def compare_cli(args): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) + compare(input_param, output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) + + +def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: try: summary_compare, md5_compare = task_dumppath_get(input_param) - check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) - create_directory(args.output_path) - check_compare_param(input_param, args.output_path, summary_compare, md5_compare) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + create_directory(output_path) + check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) - compare_core(input_param, args.output_path, stack_mode=args.stack_mode, - auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, + compare_core(input_param, output_path, stack_mode=stack_mode, + auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) -- Gitee From b002457f8390947bc9cf7a9fc049c91b10d523ac Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 10:08:24 +0800 Subject: [PATCH 101/791] 82 --- .../overall_performance_comparator.py | 8 ++ .../compare_bean/profiling_info.py | 107 ++++++++---------- .../profiling_parser/gpu_profiling_parser.py | 21 +--- .../profiling_parser/npu_profiling_parser.py | 58 +++++----- .../compare_backend/utils/constant.py | 1 + .../compare_backend/utils/file_reader.py | 23 +++- 6 files changed, 109 insertions(+), 109 deletions(-) diff --git a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 7283c17b4..1c5cee43e 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -12,6 +12,14 @@ class OverallPerformanceComparator(BaseComparator): self._headers = [''] base_col = [f'{base_profiling_info.profiling_type}'] comp_col = [f'{comp_profiling_info.profiling_type}'] + if base_profiling_info.RDMA_bandwidth or comp_profiling_info.RDMA_bandwidth: + self._headers.extend(['RDMA Bandwidth(GB/s)']) + base_col.append(f'{base_profiling_info.RDMA_bandwidth:.3f}GB/s') + comp_col.append(f'{comp_profiling_info.RDMA_bandwidth:.3f}GB/s') + if base_profiling_info.SDMA_bandwidth or comp_profiling_info.SDMA_bandwidth: + self._headers.extend(['SDMA Bandwidth(GB/s)']) + base_col.append(f'{base_profiling_info.SDMA_bandwidth:.3f}GB/s') + comp_col.append(f'{comp_profiling_info.SDMA_bandwidth:.3f}GB/s') if not base_profiling_info.hide_op_details and not comp_profiling_info.hide_op_details: self._headers.extend(['Cube Time(Num)', 'Vector Time(Num)']) base_col.extend([f'{base_profiling_info.cube_time:.3f}s({base_profiling_info.cube_num})', diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index e0a80a4d3..2b966a449 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -8,8 +8,20 @@ class ProfilingInfo: def __init__(self, profiling_type: str): self.profiling_type = profiling_type - self.cube_time = 0.0 self.other_time = 0.0 + self.lccl_num = 0 + self.compute_time = 0.0 + self.communication_not_overlapped = 0.0 + self.wait_time = 0.0 + self.memory_used = 0.0 + self.e2e_time = 0.0 + self.scheduling_time = 0.0 + self.lccl_time = 0.0 + self.minimal_profiling = False + self.hide_op_details = False + self.is_level0 = False + + self.cube_time = 0.0 self.vec_time = 0.0 self.cube_num = 0 self.vec_num = 0 @@ -17,26 +29,14 @@ class ProfilingInfo: self.fa_num_fwd = 0 self.fa_num_bwd = 0 self.pa_num = 0 - self.lccl_num = 0 self.conv_time_fwd = 0.0 self.conv_time_bwd = 0.0 self.conv_num_fwd = 0 self.conv_num_bwd = 0 - self.compute_time = 0.0 - self.communication_not_overlapped = 0.0 - self.wait_time = 0.0 - self.memory_used = 0.0 - self.e2e_time = 0.0 self.sdma_time = 0.0 - self.scheduling_time = 0.0 self.fa_time_bwd = 0.0 self.pa_time = 0.0 - self.lccl_time = 0.0 self.fa_time_fwd = 0.0 - self.minimal_profiling = False - self.hide_op_details = False - self.is_level0 = False - # 性能拆解新指标 self.fa_time_fwd_cube = 0.0 self.fa_num_fwd_cube = 0 @@ -76,7 +76,8 @@ class ProfilingInfo: self.other_cube_time = 0.0 self.other_cube_num = 0 - + self.RDMA_bandwidth = 0.0 + self.SDMA_bandwidth = 0.0 @property def e2e_time_ms(self): return self.e2e_time * 10 ** 3 @@ -137,22 +138,6 @@ class ProfilingInfo: return sum((self.vector_num_trans, self.vector_num_notrans)) def trans_time_to_s(self): - self.cube_time = self.cube_time / 10 ** 6 - self.other_time = self.other_time / 10 ** 6 - self.vec_time = self.vec_time / 10 ** 6 - self.compute_time = self.compute_time / 10 ** 6 - self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 - self.wait_time = self.wait_time / 10 ** 6 - self.e2e_time = self.e2e_time / 10 ** 6 - self.sdma_time = self.sdma_time / 10 ** 6 - self.scheduling_time = self.scheduling_time / 10 ** 6 - self.fa_time_bwd = self.fa_time_bwd / 10 ** 6 - self.fa_time_fwd = self.fa_time_fwd / 10 ** 6 - self.pa_time = self.pa_time / 10 ** 6 - self.lccl_time = self.lccl_time / 10 ** 6 - self.conv_time_fwd = self.conv_time_fwd / 10 ** 6 - self.conv_time_bwd = self.conv_time_bwd / 10 ** 6 - # 新指标单位为ms self.fa_time_fwd_cube /= 10 ** 3 self.fa_time_bwd_cube /= 10 ** 3 @@ -171,6 +156,30 @@ class ProfilingInfo: self.page_attention_time /= 10 ** 3 self.other_cube_time /= 10 ** 3 + self.cube_time = (self.matmul_time_cube + self.matmul_time_vector + self.other_cube_time) / 1000 + self.vec_time = (self.vector_time_trans + self.vector_time_notrans) / 1000 + self.cube_num = (self.matmul_num_cube + self.matmul_num_vector + self.other_cube_num) + self.vec_num = (self.vector_num_trans + self.vector_num_notrans) + self.sdma_num = (self.sdma_num_tensor_move + self.sdma_num_stream) + self.fa_num_fwd = (self.fa_num_fwd_cube + self.fa_num_fwd_vector) + self.fa_num_bwd = (self.fa_num_bwd_cube + self.fa_num_bwd_vector) + self.pa_num = self.page_attention_num + self.conv_time_fwd = (self.conv_time_fwd_cube + self.conv_time_fwd_vector) / 1000 + self.conv_time_bwd = (self.conv_time_bwd_cube + self.conv_time_bwd_vector) / 1000 + self.conv_num_fwd = (self.conv_num_fwd_cube + self.conv_num_fwd_vector) + self.conv_num_bwd = (self.conv_num_bwd_cube + self.conv_num_bwd_vector) + self.sdma_time = (self.sdma_time_tensor_move + self.sdma_time_stream) / 1000 + self.fa_time_bwd = (self.fa_time_bwd_cube + self.fa_time_bwd_vector) / 1000 + self.pa_time = self.page_attention_time / 1000 + self.fa_time_fwd = (self.fa_time_fwd_cube + self.fa_time_fwd_vector) / 1000 + + self.other_time = self.other_time / 10 ** 6 + self.compute_time = self.compute_time / 10 ** 6 + self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 + self.wait_time = self.wait_time / 10 ** 6 + self.e2e_time = self.e2e_time / 10 ** 6 + self.scheduling_time = self.scheduling_time / 10 ** 6 + self.lccl_time = self.lccl_time / 10 ** 6 def calculate_other_time(self): self.other_time = max( [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - @@ -183,14 +192,6 @@ class ProfilingInfo: def calculate_schedule_time(self): self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time - self.communication_not_overlapped) - def update_fa_fwd_info(self, time: float): - self.fa_time_fwd += time - self.fa_num_fwd += 1 - - def update_fa_bwd_info(self, time: float): - self.fa_time_bwd += time - self.fa_num_bwd += 1 - def update_fa_fwd_cube_info(self, time: float): self.fa_time_fwd_cube += time self.fa_num_fwd_cube += 1 @@ -215,22 +216,10 @@ class ProfilingInfo: self.sdma_time_stream += time self.sdma_num_stream += num - def update_pa_info(self, time: float): - self.pa_time += time - self.pa_num += 1 - def update_lccl_info(self, time: float): self.lccl_time += time self.lccl_num += 1 - def update_conv_fwd_info(self, time: float): - self.conv_time_fwd += time - self.conv_num_fwd += 1 - - def update_conv_bwd_info(self, time: float): - self.conv_time_bwd += time - self.conv_num_bwd += 1 - def update_conv_bwd_cube_info(self, time: float): self.conv_time_bwd_cube += time self.conv_num_bwd_cube += 1 @@ -267,18 +256,6 @@ class ProfilingInfo: self.vector_time_notrans += time self.vector_num_notrans += 1 - def update_sdma_info(self, time: float, num: int = 1): - self.sdma_time += time - self.sdma_num += num - - def update_cube_info(self, time: float): - self.cube_time += time - self.cube_num += 1 - - def update_vec_info(self, time: float): - self.vec_time += time - self.vec_num += 1 - def update_other_cube_info(self, time: float): self.other_cube_time += time self.other_cube_num += 1 @@ -306,3 +283,9 @@ class ProfilingInfo: def is_not_minimal_profiling(self) -> bool: return self.profiling_type == Constant.NPU and not self.minimal_profiling + + def set_RDMA_bandwidth(self, bandwidth: float): + self.RDMA_bandwidth = bandwidth + + def set_SDMA_bandwidth(self, bandwidth: float): + self.SDMA_bandwidth = bandwidth \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 0aeeba83e..175b77603 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -61,9 +61,9 @@ class GPUProfilingParser(BaseProfilingParser): def _update_overall_metrics(self): self._calculate_performance_time() self.__parse_memory_reserved() + self._result_data.overall_metrics.trans_time_to_s() self._result_data.overall_metrics.calculate_vec_time() self._result_data.overall_metrics.calculate_schedule_time() - self._result_data.overall_metrics.trans_time_to_s() def _calculate_performance_time(self): min_ts = sys.float_info.max @@ -76,7 +76,6 @@ class GPUProfilingParser(BaseProfilingParser): min_ts = min(event.start_time, min_ts) max_ts = max(event.end_time, max_ts) if event.stream == self._compute_stream_id and self.__is_sdma_time(event.name): - self._result_data.overall_metrics.update_sdma_info(event.dur) self._result_data.overall_metrics.update_sdma_stream_info(event.dur) continue if not event.is_kernel_cat(): @@ -84,7 +83,6 @@ class GPUProfilingParser(BaseProfilingParser): self.__add_marks(event) if event.is_nccl_name(): continue - self.__add_compute_time(event, aten_events, flow_dict_new) self.categorize_computing_performance_data(event, flow_dict_new) self._aten_events = None self._result_data.overall_metrics.set_e2e_time(float(max_ts - min_ts)) @@ -104,23 +102,6 @@ class GPUProfilingParser(BaseProfilingParser): for timestep in range(int(event.start_time + 1), int(event.end_time + 1)): self._marks[str(timestep)] += -100 # mark this timestep in compute stream - def __add_compute_time(self, event: TraceEventBean, aten_events: list, flow_dict_new: dict): - if self.__is_flash_attention(event.name): - if event.is_backward(): - self._result_data.overall_metrics.update_fa_bwd_info(event.dur) - else: - self._result_data.overall_metrics.update_fa_fwd_info(event.dur) - elif any(cube_mark in event.lower_name for cube_mark in self.CUBE_MARK): - is_conv = self.__check_is_conv(event, aten_events, flow_dict_new) - if is_conv == "conv_fwd": - self._result_data.overall_metrics.update_conv_fwd_info(event.dur) - elif is_conv == "conv_bwd": - self._result_data.overall_metrics.update_conv_bwd_info(event.dur) - else: - self._result_data.overall_metrics.update_cube_info(event.dur) - else: - self._result_data.overall_metrics.update_vec_info(event.dur) - def __check_is_conv(self, event: TraceEventBean, aten_events: list, flow_dict_new: dict) -> str: flow_start_time = flow_dict_new.get(event.start_time) if not flow_start_time: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index cb25c252c..3c3f05427 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -22,6 +22,7 @@ class NPUProfilingParser(BaseProfilingParser): self._operator_memory_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "operator_memory.csv") self._memory_record_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "memory_record.csv") self._kernel_detail_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "kernel_details.csv") + self._communication_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "communication.json") self._info_json_path = path_dict.get(Constant.INFO_JSON_PATH, "") self._trace_events = [TraceEventBean(event) for event in self._trace_events] self._hccl_pid = None @@ -78,7 +79,6 @@ class NPUProfilingParser(BaseProfilingParser): print("[ERROR] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") return self._result_data.update_kernel_details(kernels_dict) - def _update_memory_list(self): try: memory_data = FileReader.read_csv_file(self._operator_memory_path, OperatorMemoryBean) @@ -121,6 +121,35 @@ class NPUProfilingParser(BaseProfilingParser): return self._dequeue_data[left].corr_id if self._dequeue_data[left].start_time <= ts_time <= \ self._dequeue_data[left].end_time else Constant.INVALID_VALUE + def _update_bandwidth(self): + try: + communication_json = FileReader.read_json_file(self._communication_path) + except FileNotFoundError: + print("[WARNING] The file communication.json does not exist.") + except Exception: + print("[ERROR] Failed to read communication.json.") + return + if not communication_json: + print("[WARNING] The JSON file is empty.") + return + for _, group_dict in communication_json.items(): + step_dict = group_dict.get("collective") + total_op_info = step_dict.get("Total Op Info", {}) + rdma_size_mb = rdma_time_ms = sdma_size_mb = sdma_time_ms = 0 + if "Communication Bandwidth Info" in total_op_info: + bandwidth_info = total_op_info["Communication Bandwidth Info"] + if "RDMA" in bandwidth_info: + rdma_info = bandwidth_info["RDMA"] + rdma_size_mb += rdma_info.get("Transit Size(MB)", 0) # 单位为 MB + rdma_time_ms += rdma_info.get("Transit Time(ms)", 0) # 单位为 MS + if "SDMA" in bandwidth_info: + sdma_info = bandwidth_info["SDMA"] + sdma_size_mb += sdma_info.get("Transit Size(MB)", 0) # 单位为 MB + sdma_time_ms += sdma_info.get("Transit Time(ms)", 0) # 单位为 MS + rdma_bandwidth = (rdma_size_mb / 1024) / (rdma_time_ms / 1000) if rdma_time_ms > 0 else 0 + sdma_bandwidth = (sdma_size_mb / 1024) / (sdma_time_ms / 1000) if sdma_time_ms > 0 else 0 + self._result_data.overall_metrics.set_RDMA_bandwidth(rdma_bandwidth) + self._result_data.overall_metrics.set_SDMA_bandwidth(sdma_bandwidth) def _update_overall_metrics(self): self.__parse_info_json() self.__parse_mem_csv() @@ -130,10 +159,11 @@ class NPUProfilingParser(BaseProfilingParser): self.__add_overlap_analysis_time() self._picking_notify_wait_event_and_not_overlap_event() self.__add_overlap_wait_time() + self._result_data.overall_metrics.trans_time_to_s() self._result_data.overall_metrics.calculate_other_time() self._result_data.overall_metrics.calculate_schedule_time() - self._result_data.overall_metrics.trans_time_to_s() + self._update_bandwidth() def _picking_notify_wait_event_and_not_overlap_event(self): self.notify_event_cache = [] self._not_overlaped_commu_event = [] @@ -271,28 +301,6 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.update_lccl_info(event.dur) def __parse_kernel_csv(self): - def __screen_data(kernel: KernelDetailsBean): - if kernel.is_flash_attention(): - if kernel.is_fa_bwd(): - self._result_data.overall_metrics.update_fa_bwd_info(kernel.duration) - else: - self._result_data.overall_metrics.update_fa_fwd_info(kernel.duration) - elif kernel.is_conv(): - if kernel.is_conv_bwd(): - self._result_data.overall_metrics.update_conv_bwd_info(kernel.duration) - else: - self._result_data.overall_metrics.update_conv_fwd_info(kernel.duration) - elif kernel.is_matmul(): - self._result_data.overall_metrics.update_cube_info(kernel.duration) - elif kernel.is_sdma(): - self._result_data.overall_metrics.update_sdma_info(kernel.duration) - elif kernel.is_page_attention(): - self._result_data.overall_metrics.update_pa_info(kernel.duration) - elif kernel.is_vector(): - self._result_data.overall_metrics.update_vec_info(kernel.duration) - else: - self._result_data.overall_metrics.update_cube_info(kernel.duration) - try: kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) except Exception: @@ -306,7 +314,6 @@ class NPUProfilingParser(BaseProfilingParser): for kernel in kernel_details: if kernel.is_invalid(): continue - __screen_data(kernel) self.categorize_computing_performance_data(kernel, flow_dict_new) def __parse_mem_csv(self): @@ -353,5 +360,4 @@ class NPUProfilingParser(BaseProfilingParser): compute_stream = event_wait_stream & ai_core_stream if event_wait_stream else ai_core_stream for stream in compute_stream: dur_list = sdma_dict.get(stream, []) - self._result_data.overall_metrics.update_sdma_info(sum(dur_list), len(dur_list)) self._result_data.overall_metrics.update_sdma_stream_info(sum(dur_list), len(dur_list)) diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index 252aa536e..80d7d5ee4 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -6,6 +6,7 @@ class Constant(object): MAX_PATH_LENGTH = 4096 MAX_FLOW_CAT_LEN = 20 MAX_FILE_SIZE = 1024 * 1024 * 1024 * 5 + MAX_JSON_SIZE = 1024 * 1024 * 1024 * 10 BYTE_TO_KB = 1024 YELLOW_COLOR = "FFFF00" GREEN_COLOR = "00FF00" diff --git a/profiler/compare_tools/compare_backend/utils/file_reader.py b/profiler/compare_tools/compare_backend/utils/file_reader.py index b4ae78638..99358368c 100644 --- a/profiler/compare_tools/compare_backend/utils/file_reader.py +++ b/profiler/compare_tools/compare_backend/utils/file_reader.py @@ -7,7 +7,28 @@ from compare_backend.utils.constant import Constant class FileReader: - + @classmethod + def read_json_file(cls, file_path: str, bean_class: any = None) -> any: + PathManager.check_path_readable(file_path) + if not os.path.isfile(file_path): + raise FileNotFoundError("File not exists.") + file_size = os.path.getsize(file_path) + if file_size <= 0: + return [] + if file_size > Constant.MAX_JSON_SIZE: + check_msg = input( + f"The file({file_path}) size exceeds the preset max value. Continue reading the file? [y/n]") + if check_msg.lower() != "y": + print(f"[WARNING] The user choose not to read the file: {file_path}") + return [] + result_data = [] + try: + with open(file_path, "r") as json_file: + result_data = json.loads(json_file.read()) + except Exception as e: + msg = f"Failed to read the file: {file_path}" + raise RuntimeError(msg) from e + return result_data @classmethod def read_trace_file(cls, file_path: str) -> any: PathManager.check_path_readable(file_path) -- Gitee From 01edd78abd452451530dfc1fe03aeed66b70f48f Mon Sep 17 00:00:00 2001 From: CSNIU Date: Mon, 5 Aug 2024 10:30:23 +0800 Subject: [PATCH 102/791] =?UTF-8?q?compare=E6=A8=A1=E5=9D=97=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E9=87=8D=E6=9E=84=E5=88=9D=E7=89=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/compare/Multiprocessing_compute.py | 67 ++++- .../msprobe/core/compare/acc_compare.py | 9 +- .../msprobe/core/compare/check.py | 18 -- .../msprobe/core/compare/utils.py | 49 +--- .../msprobe/mindspore/__init__.py | 4 +- .../msprobe/mindspore/compare/ms_compare.py | 48 +-- .../msprobe/pytorch/__init__.py | 4 +- .../msprobe/pytorch/compare/pt_comparator.py | 273 ++++++++++++++++++ 8 files changed, 365 insertions(+), 107 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py diff --git a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py index e0c52aa6f..9d8e9744e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py @@ -1,9 +1,10 @@ import multiprocessing import pandas as pd +from dataclasses import dataclass from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException - +from msprobe.core.common.const import CompareConst def _handle_multi_process(func, input_parma, result_df, lock): @@ -52,4 +53,66 @@ def read_dump_data(result_df): raise CompareException(CompareException.INVALID_DATA_ERROR) from e except IndexError as e: logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e \ No newline at end of file + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + + +@dataclass +class ComparisonResult: + cos_result: list + max_err_result: list + max_relative_err_result: list + err_msgs: list + one_thousand_err_ratio_result: list + five_thousand_err_ratio_result: list + +def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): + """ + Save comparison results into the result DataFrame with thread safety. + Args: + offset: offset for index + result: data struct of ComparisonResult + result_df: result of DataFrame + lock: thread lock + + Returns: + comparison results in DataFrame + """ + + lock.acquire() + try: + for i, _ in enumerate(result.cos_result): + process_index = i + offset + result_df.loc[process_index, CompareConst.COSINE] = result.cos_result[i] + result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] + result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] + result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] + result_df.loc[process_index, CompareConst.ACCURACY] = check_accuracy(result.cos_result[i], result.max_err_result[i]) + result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = result.one_thousand_err_ratio_result[i] + result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = result.five_thousand_err_ratio_result[i] + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + except IndexError as e: + logger.error('result dataframe elements can not be access.') + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e + finally: + lock.release() + +def check_accuracy(cos, max_abs_err): + if cos == CompareConst.SHAPE_UNMATCH: + return CompareConst.ACCURACY_CHECK_UNMATCH + if cos == CompareConst.NONE or max_abs_err == CompareConst.NONE: + return CompareConst.NONE + if cos == "N/A" or max_abs_err == "N/A": + return CompareConst.ACCURACY_CHECK_NO + try: + cos, max_abs_err = float(cos), float(max_abs_err) + except ValueError: + logger.warning("Cosine or MaxAbsErr can not get float value.") + return CompareConst.NONE + if cos < CompareConst.COS_THRESHOLD and max_abs_err > CompareConst.MAX_ABS_ERR_THRESHOLD: + return CompareConst.ACCURACY_CHECK_NO + if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: + return CompareConst.ACCURACY_CHECK_NO + return CompareConst.ACCURACY_CHECK_YES \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index dc581e70e..7d2be9c4c 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -1,6 +1,5 @@ from msprobe.core.compare.check import check_op -from msprobe.mindspore.compare.ms_compare import MSComparator -from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.const import CompareConst from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ get_error_message from msprobe.core.common.exceptions import FileCheckException @@ -30,9 +29,9 @@ class Comparator: error_flag = True else: try: - msComparator= MSComparator() - n_value = msComparator.read_npy_data(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) - b_value = msComparator.read_npy_data(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) + read_npy_data=getattr(self,"read_npy_data") + n_value = read_npy_data(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) + b_value = read_npy_data(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) except IOError as error: error_file = error.filename n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index aab8cb50e..a8ee3638a 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -1,6 +1,5 @@ from msprobe.core.compare.match import graph_mapping from msprobe.core.common.log import logger -from msprobe.core.common.const import Const, CompareConst, FileCheckConst from msprobe.core.compare.utils import rename_api @@ -87,20 +86,3 @@ def fuzzy_check_name(npu_name, bench_name): -def check_accuracy(cos, max_abs_err): - if cos == CompareConst.SHAPE_UNMATCH: - return CompareConst.ACCURACY_CHECK_UNMATCH - if cos == CompareConst.NONE or max_abs_err == CompareConst.NONE: - return CompareConst.NONE - if cos == "N/A" or max_abs_err == "N/A": - return CompareConst.ACCURACY_CHECK_NO - try: - cos, max_abs_err = float(cos), float(max_abs_err) - except ValueError: - logger.warning("Cosine or MaxAbsErr can not get float value.") - return CompareConst.NONE - if cos < CompareConst.COS_THRESHOLD and max_abs_err > CompareConst.MAX_ABS_ERR_THRESHOLD: - return CompareConst.ACCURACY_CHECK_NO - if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: - return CompareConst.ACCURACY_CHECK_NO - return CompareConst.ACCURACY_CHECK_YES \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 0ed0b4ebd..d213e0b46 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -1,10 +1,9 @@ +import os import numpy as np -from msprobe.core.common.log import logger from msprobe.core.common.const import Const, CompareConst -from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ - format_value, check_file_not_exists, check_configuration_param, task_dumppath_get -from msprobe.core.compare.check import check_accuracy + + def rename_api(npu_name, process): @@ -152,15 +151,6 @@ def resolve_api_special_parameters(data_dict, full_op_name, item_list): parsed_item['full_op_name'] = full_op_name_new item_list.append(parsed_item) -@dataclass -class ComparisonResult: - cos_result: list - max_err_result: list - max_relative_err_result: list - err_msgs: list - one_thousand_err_ratio_result: list - five_thousand_err_ratio_result: list - def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=False): def get_accuracy_core(n_start, n_len, b_start, b_len, key): @@ -363,39 +353,6 @@ def merge_tensor(tensor_list, summary_compare, md5_compare): -def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): - """ - Save comparison results into the result DataFrame with thread safety. - Args: - offset: offset for index - result: data struct of ComparisonResult - result_df: result of DataFrame - lock: thread lock - - Returns: - comparison results in DataFrame - """ - - lock.acquire() - try: - for i, _ in enumerate(result.cos_result): - process_index = i + offset - result_df.loc[process_index, CompareConst.COSINE] = result.cos_result[i] - result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] - result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] - result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] - result_df.loc[process_index, CompareConst.ACCURACY] = check_accuracy(result.cos_result[i], result.max_err_result[i]) - result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = result.one_thousand_err_ratio_result[i] - result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = result.five_thousand_err_ratio_result[i] - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - finally: - lock.release() diff --git a/debug/accuracy_tools/msprobe/mindspore/__init__.py b/debug/accuracy_tools/msprobe/mindspore/__init__.py index d131591a3..60bebb2ba 100644 --- a/debug/accuracy_tools/msprobe/mindspore/__init__.py +++ b/debug/accuracy_tools/msprobe/mindspore/__init__.py @@ -1,3 +1,3 @@ from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger -from .compare.acc_compare import compare -from .compare.distributed_compare import compare_distributed +# from .compare.acc_compare import compare +# from .compare.distributed_compare import compare_distributed diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index b7a839c22..9d1e1976b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,11 +1,3 @@ -from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.common.log import logger - - - - - - import json import multiprocessing import os.path @@ -13,24 +5,18 @@ import sys import numpy as np import pandas as pd -import openpyxl -from openpyxl.styles import PatternFill -from collections import namedtuple -from dataclasses import dataclass -from msprobe.mindspore.compare.match import graph_mapping -from msprobe.mindspore.compare.highlight import HighlightRules, get_header_index - -from msprobe.mindspore.advisor.advisor import Advisor -from msprobe.mindspore.common.log import logger +from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ - format_value, check_file_not_exists, check_configuration_param, task_dumppath_get -from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, create_directory + check_file_not_exists, check_configuration_param, task_dumppath_get +from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory from msprobe.core.common.const import Const, CompareConst, FileCheckConst -from msprobe.core.common.exceptions import FileCheckException + from msprobe.core.compare.utils import ComparisonResult,_save_cmp_result,merge_tensor, get_un_match_accuracy,get_accuracy,read_op from msprobe.core.compare.highlight import find_compare_result_error_rows,highlight_rows_xlsx from msprobe.core.compare.Multiprocessing_compute import _handle_multi_process +from msprobe.core.compare.acc_compare import Comparator +from msprobe.core.common.log import logger class MSComparator (Comparator): @@ -180,7 +166,15 @@ class MSComparator (Comparator): result_df = pd.DataFrame(result, columns=header) return result_df - + + + def _do_multi_process(self,input_parma, result_df): + try: + result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) @@ -194,23 +188,13 @@ class MSComparator (Comparator): return data_value - def _do_multi_process(self,input_parma, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - - - def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_parma) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(input_parma, output_path, stack_mode, summary_compare, md5_compare) + check_compare_param(input_parma, output_path, summary_compare, md5_compare) except CompareException as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) diff --git a/debug/accuracy_tools/msprobe/pytorch/__init__.py b/debug/accuracy_tools/msprobe/pytorch/__init__.py index 482e850f7..11193b39f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/__init__.py +++ b/debug/accuracy_tools/msprobe/pytorch/__init__.py @@ -1,4 +1,4 @@ from .debugger.precision_debugger import PrecisionDebugger from .common.utils import seed_all -from .compare.acc_compare import compare -from .compare.distributed_compare import compare_distributed +# from .compare.acc_compare import compare +# from .compare.distributed_compare import compare_distributed diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py new file mode 100644 index 000000000..5cf83762a --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py @@ -0,0 +1,273 @@ +import argparse +import json +import multiprocessing +import os.path +import sys +import torch +import numpy as np +import pandas as pd + +from msprobe.core.advisor.advisor import Advisor +from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ + check_file_not_exists, check_configuration_param, task_dumppath_get +from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory +from msprobe.core.common.const import Const, CompareConst, FileCheckConst + +from msprobe.core.compare.utils import ComparisonResult,_save_cmp_result,merge_tensor, get_un_match_accuracy,get_accuracy,read_op +from msprobe.core.compare.highlight import find_compare_result_error_rows,highlight_rows_xlsx +from msprobe.core.compare.Multiprocessing_compute import _handle_multi_process +from msprobe.core.compare.acc_compare import Comparator +from msprobe.core.common.log import logger + + +class PTComparator (Comparator): + def __init__(self): + super().__init__() + + + def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): + cos_result = [] + max_err_result = [] + max_relative_err_result = [] + err_mess = [] + one_thousand_err_ratio_result = [] + five_thousand_err_ratio_result = [] + is_print_compare_log = input_parma.get("is_print_compare_log") + for i in range(len(result_df)): + op_name = result_df.iloc[i, 0] + if is_print_compare_log: + logger.info("start compare: {}".format(op_name)) + cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = self.compare_by_op( + op_name, dump_path_dict, input_parma) + if is_print_compare_log: + logger.info( + "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " + "five_thousand_err_ratio {}".format(op_name, cos_sim, max_abs_err, max_relative_err, err_msg, + one_thousand_err_ratio, five_thousand_err_ratio)) + cos_result.append(cos_sim) + max_err_result.append(max_abs_err) + max_relative_err_result.append(max_relative_err) + err_mess.append(err_msg) + one_thousand_err_ratio_result.append(one_thousand_err_ratio) + five_thousand_err_ratio_result.append(five_thousand_err_ratio) + + cr = ComparisonResult( + cos_result=cos_result, + max_err_result=max_err_result, + max_relative_err_result=max_relative_err_result, + err_msgs=err_mess, + one_thousand_err_ratio_result=one_thousand_err_ratio_result, + five_thousand_err_ratio_result=five_thousand_err_ratio_result + ) + + return _save_cmp_result(idx, cr, result_df, lock) + + def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): + npu_json_handle, bench_json_handle, stack_json_handle = file_handles + npu_json_data = json.load(npu_json_handle) + bench_json_data = json.load(bench_json_handle) + stack_json_data = json.load(stack_json_handle) + + if fuzzy_match: + logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") + + npu_ops_queue = [] + bench_ops_queue = [] + result = [] + + ops_npu_iter = iter(npu_json_data['data']) + ops_bench_iter = iter(bench_json_data['data']) + read_err_npu = True + read_err_bench = True + last_npu_ops_len = 0 + last_bench_ops_len = 0 + + while True: + if not read_err_npu and not read_err_bench: + break + try: + last_npu_ops_len = len(npu_ops_queue) + op_name_npu = next(ops_npu_iter) + read_err_npu = True + + npu_op_data = npu_json_data['data'][op_name_npu] + npu_op_parsed_list = read_op(npu_op_data, op_name_npu) + if op_name_npu in stack_json_data: + npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': stack_json_data[op_name_npu]}) + else: + npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': None}) + + npu_merge_list = merge_tensor(npu_op_parsed_list, summary_compare, md5_compare) + if npu_merge_list: + npu_ops_queue.append(npu_merge_list) + except StopIteration: + read_err_npu = False + try: + last_bench_ops_len = len(bench_ops_queue) + op_name_bench = next(ops_bench_iter) + + bench_op_data = bench_json_data['data'][op_name_bench] + bench_op_parsed_list = read_op(bench_op_data, op_name_bench) + if op_name_bench in stack_json_data: + bench_op_parsed_list.append( + {'full_op_name': op_name_bench, 'full_info': stack_json_data[op_name_bench]}) + else: + bench_op_parsed_list.append({'full_op_name': op_name_bench, 'full_info': None}) + + bench_merge_list = merge_tensor(bench_op_parsed_list, summary_compare, md5_compare) + if bench_merge_list: + bench_ops_queue.append(bench_merge_list) + except StopIteration: + read_err_bench = False + + # merge all boolean expressions + both_empty = not npu_ops_queue and not bench_ops_queue + no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) + if both_empty or no_change: + continue + + n_match_point, b_match_point = super().match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) + if n_match_point == -1 and b_match_point == -1: + continue + n_match_data = npu_ops_queue[n_match_point] + b_match_data = bench_ops_queue[b_match_point] + un_match_data = npu_ops_queue[0: n_match_point] + for npu_data in un_match_data: + get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) + get_accuracy(result, n_match_data, b_match_data, summary_compare, md5_compare) + del npu_ops_queue[0: n_match_point + 1] + del bench_ops_queue[0: b_match_point + 1] + if npu_ops_queue: + for npu_data in npu_ops_queue: + get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) + + header = [] + if md5_compare: + header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] + elif summary_compare: + header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] + else: + header = CompareConst.COMPARE_RESULT_HEADER[:] + + all_mode_bool = not (summary_compare or md5_compare) + if stack_mode: + if all_mode_bool: + header.append(CompareConst.STACK) + header.append(CompareConst.DATA_NAME) + else: + header.append(CompareConst.STACK) + else: + if all_mode_bool: + for row in result: + del row[-2] + header.append(CompareConst.DATA_NAME) + else: + for row in result: + del row[-1] + + result_df = pd.DataFrame(result, columns=header) + return result_df + + + def read_npy_data(self,dir_path, file_name): + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.PT_SUFFIX, False) + data_path = path_checker.common_check() + data_value = torch.load(data_path, map_location=torch.device('cpu')).detach() # detach for less memory + if data_value.dtype == torch.bfloat16: + data_value = data_value.to(torch.float32) + data_value = data_value.numpy() + return data_value + + + def _do_multi_process(self,input_parma, result_df): + try: + result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + + +def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, + fuzzy_match=False): + try: + summary_compare, md5_compare = task_dumppath_get(input_parma) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + create_directory(output_path) + check_compare_param(input_parma, output_path, summary_compare, md5_compare) + except CompareException as error: + logger.error('Compare failed. Please check the arguments and do it again!') + sys.exit(error.code) + compare_core(input_parma, output_path, stack_mode=stack_mode, + auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) + +# def compare(args): +# with FileOpen(args.input_path, "r") as file: +# input_param = json.load(file) +# try: +# summary_compare, md5_compare = task_dumppath_get(input_param) +# check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) +# create_directory(args.output_path) +# check_compare_param(input_param, args.output_path, summary_compare, md5_compare) +# except (CompareException, FileCheckException) as error: +# logger.error('Compare failed. Please check the arguments and do it again!') +# sys.exit(error.code) +# msComparator.compare_core(input_param, args.output_path, stack_mode=args.stack_mode, +# auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, +# md5_compare=md5_compare) + +def compare_core(input_parma, output_path, **kwargs): + """ + Compares data from multiple JSON files and generates a comparison report. + + Args: + input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", + "stack_path"). + output_path (str): The path where the output Excel report will be saved. + **kwargs: Additional keyword arguments including: + - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. + - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. + - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. + - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. + - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. + - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. + + Returns: + """ + # get kwargs or set default value + stack_mode = kwargs.get('stack_mode', False) + auto_analyze = kwargs.get('auto_analyze', True) + suffix = kwargs.get('suffix', '') + fuzzy_match = kwargs.get('fuzzy_match', False) + summary_compare = kwargs.get('summary_compare', False) + md5_compare = kwargs.get('md5_compare', False) + + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") + file_name = add_time_with_xlsx("compare_result" + suffix) + file_path = os.path.join(os.path.realpath(output_path), file_name) + check_file_not_exists(file_path) + highlight_dict = {'red_rows': [], 'yellow_rows': []} + ptComparator= PTComparator() + with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_path"), "r") as stack_json: + result_df = ptComparator.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, + summary_compare, md5_compare) + + if not md5_compare and not summary_compare: + result_df = ptComparator._do_multi_process(input_parma, result_df) + find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) + highlight_rows_xlsx(result_df, highlight_dict, file_path) + if auto_analyze: + advisor = Advisor(result_df, output_path) + advisor.analysis() + + + + + + \ No newline at end of file -- Gitee From 59cbefe4a7e097400c270695920b6e7a92c6fb13 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Mon, 5 Aug 2024 10:33:31 +0800 Subject: [PATCH 103/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=A4=96=E9=83=A8?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/msprobe.py | 4 ++-- debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index e340c67eb..a27d3b55a 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -23,7 +23,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _ from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command from msprobe.pytorch.compare.acc_compare import _compare_parser, compare - +from msprobe.pytorch.compare.pt_comparator import pt_compare def main(): parser = argparse.ArgumentParser( @@ -66,7 +66,7 @@ def main(): elif sys.argv[3] == "run_overflow_check": _run_overflow_check_command(args) elif sys.argv[3] == "compare": - compare(args) + pt_compare(args) else: if sys.argv[3] == "compare": pass diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py index 5cf83762a..22dd2be4d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py @@ -191,7 +191,7 @@ class PTComparator (Comparator): -def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, +def pt_compare(input_parma, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_parma) -- Gitee From 5e93574699b3123e31fbb7dc32f2e2ee79725b19 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 5 Aug 2024 10:38:22 +0800 Subject: [PATCH 104/791] review fix --- debug/accuracy_tools/msprobe/msprobe.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 92d6bbfb6..4a6250039 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -22,7 +22,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _ _api_precision_compare_command from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command -from msprobe.pytorch.compare.acc_compare import _compare_parser, compare +from msprobe.pytorch.compare.acc_compare import _compare_parser, compare_cli def main(): -- Gitee From 36813c1635ea55b59f87a1a4b1aa35a7321170ae Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 5 Aug 2024 10:55:50 +0800 Subject: [PATCH 105/791] add complex method --- .../core/data_dump/data_processor/pytorch_processor.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 007fec809..4e39d862b 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -69,6 +69,12 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_stat.min = False not in data_clone elif not data_clone.shape: tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data_clone.item() + elif torch.is_complex(data_clone): + data_np = data_clone.cpu().numpy() + data_abs = np.abs(data_np) + tensor_stat.max = np.max(data_abs).item() + tensor_stat.min = np.min(data_abs).item() + tensor_stat.mean = np.mean(data_abs).item() else: if not data_clone.is_floating_point() or data_clone.dtype == torch.float64: data_clone = data_clone.float() -- Gitee From eb56c1da8cead969c2c2c499573ee24d8d44fa18 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 5 Aug 2024 10:57:04 +0800 Subject: [PATCH 106/791] review fix --- debug/accuracy_tools/msprobe/core/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 1058f04b7..f2b58dfad 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -496,8 +496,8 @@ def task_dumppath_get(input_param): else: logger.error(f"Compare is not required for overflow_check or free_benchmark.") raise CompareException(CompareException.INVALID_TASK_ERROR) - input_param['npu_dump_data_dir'] = os.path.join(os.path.dirname(npu_path) + Const.DUMP_TENSOR_DATA) - input_param['bench_dump_data_dir'] = os.path.join(os.path.dirname(bench_path) + Const.DUMP_TENSOR_DATA) + input_param['npu_dump_data_dir'] = os.path.join(os.path.dirname(npu_path), Const.DUMP_TENSOR_DATA) + input_param['bench_dump_data_dir'] = os.path.join(os.path.dirname(bench_path), Const.DUMP_TENSOR_DATA) return summary_compare, md5_compare -- Gitee From 58834d42f170f448168e9664ec873a4555562540 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 5 Aug 2024 11:07:25 +0800 Subject: [PATCH 107/791] review fix --- debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index 424aa3f7b..f0e56a609 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -657,7 +657,7 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): def compare_cli(args): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) - compare(input_param, output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) + compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): -- Gitee From 4023906e2328679c5a2343f6db5e9cf15362b9c2 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Mon, 5 Aug 2024 11:20:43 +0800 Subject: [PATCH 108/791] =?UTF-8?q?1.ai=20core=E9=99=8D=E9=A2=91=E5=88=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common_config.py | 4 ++-- .../core/data_dump/data_processor/pytorch_processor.py | 2 +- .../msprobe/pytorch/debugger/debugger_config.py | 2 +- debug/accuracy_tools/msprobe/pytorch/pt_config.py | 4 ++-- .../accuracy_tools/msprobe/test/core_ut/test_common_config.py | 2 +- .../accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py | 2 +- .../analyzer/computation/ai_core_freq/ai_core_freq_checker.py | 2 +- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index ed38eba00..b4bf5cf28 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -44,8 +44,8 @@ class BaseConfig: self.data_mode = json_config.get('data_mode') self.backward_input = json_config.get("backward_input") self.file_format = json_config.get("file_format") - self.summary_mode = json_config.get("summary_mode") - self.overflow_num = json_config.get("overflow_num") + self.summary_mode = json_config.get("summary_mode") + self.overflow_nums = json_config.get("overflow_nums") self.check_mode = json_config.get("check_mode") def check_config(self): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 8dac54fd2..2712bac61 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -182,7 +182,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): super().__init__(config, data_writer) self.cached_tensors_and_file_paths = {} self.real_overflow_dump_times = 0 - self.overflow_nums = config.overflow_num + self.overflow_nums = config.overflow_nums self.bits_for_overflow = 8 @staticmethod diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index cfc588e1e..f1289e9b0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -21,7 +21,7 @@ class DebuggerConfig: self.acl_config = common_config.acl_config if common_config.acl_config else "" self.is_forward_acl_dump = True self.summary_mode = task_config.summary_mode if task_config.summary_mode else Const.STATISTICS - self.overflow_num = task_config.overflow_num if task_config.overflow_num else 1 + self.overflow_nums = task_config.overflow_nums if task_config.overflow_nums else 1 self.framework = Const.PT_FRAMEWORK if self.task == Const.FREE_BENCHMARK: diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index a3d765f3a..ceec92a63 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -32,12 +32,12 @@ class StatisticsConfig(BaseConfig): class OverflowCheckConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) - self.overflow_num = json_config.get("overflow_nums") + self.overflow_nums = json_config.get("overflow_nums") self.check_mode = json_config.get("check_mode") self.check_overflow_config() def check_overflow_config(self): - if self.overflow_num is not None and not isinstance(self.overflow_num, int): + if self.overflow_nums is not None and not isinstance(self.overflow_nums, int): raise Exception("overflow_num is invalid") if self.check_mode is not None and self.check_mode not in ["all", "aicore", "atomic"]: raise Exception("check_mode is invalid") diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index 06c7378ed..8b2138a48 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -121,7 +121,7 @@ class TestCommonConfig(TestCase): self.assertIsNone(base_config.backward_input) self.assertIsNone(base_config.file_format) self.assertIsNone(base_config.summary_mode) - self.assertIsNone(base_config.overflow_num) + self.assertIsNone(base_config.overflow_nums) self.assertIsNone(base_config.check_mode) json_config.update({"scope": "Tensor_Add"}) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py index c344f0b66..470390d77 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_pt_config.py @@ -45,7 +45,7 @@ class TestPtConfig(TestCase): } } result = parse_task_config(Const.OVERFLOW_CHECK, overflow_check_config) - self.assertEqual(result.overflow_num, 1) + self.assertEqual(result.overflow_nums, 1) self.assertEqual(result.check_mode, "all") free_benchmark_config = { diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index 5ea4dbd75..7afa09cca 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -49,7 +49,7 @@ class AICoreFreqChecker: max_freq = max(self.DEFAULT_FREQ, convert_to_float(Config().get_config("aic_frequency"))) decrease_freq_ratio = sum(max_freq - freq for freq in freq_list) / (max_freq * len(freq_list)) - if decrease_freq_ratio >= self.DECREASE_FREQ_RATIO: + if decrease_freq_ratio >= Config().get_config("frequency_threshold"): self.ai_core_freq_issues = True self.decrease_freq_ops.append([op_name, op_count, op_total_duration, f"{round(decrease_freq_ratio, 4):.2%}", -- Gitee From 827f79700d59b4ce9da4fe2e0135de6aa5bac303 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 5 Aug 2024 14:19:30 +0800 Subject: [PATCH 109/791] distributed api input lack bugfix --- .../ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py index 7cfac0d40..80798ff41 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py @@ -22,8 +22,7 @@ def check_list_or_acl_mode(name_prefix): global dump_count for item in DumpUtil.dump_switch_scope: if PRE_FORWARD in name_prefix: - parts = item.split(Const.DOT) - rename = Const.DOT.join(parts[:-1]) + rename = item.rsplit(Const.DOT, 1)[0] if name_prefix.startswith(rename): return True if name_prefix.startswith(item): @@ -36,8 +35,7 @@ def check_range_mode(name_prefix): global range_begin_flag global range_end_flag if "Distributed" in DumpUtil.dump_switch_scope[0]: - parts = DumpUtil.dump_switch_scope[0].split(Const.DOT) - rename = Const.DOT.join(parts[:-1]) + rename = DumpUtil.dump_switch_scope[0].rsplit(Const.DOT, 1)[0] if name_prefix.startswith(rename): range_begin_flag = True return True -- Gitee From 3acd418630f918e27799a010c59f8f3cf72affa7 Mon Sep 17 00:00:00 2001 From: hanqing Date: Fri, 2 Aug 2024 18:54:43 +0800 Subject: [PATCH 110/791] safe --- .../accuracy_tools/grad_tool/common/utils.py | 3 ++- .../grad_tool/grad_ms/grad_analyzer.py | 1 + .../grad_tool/grad_ms/grad_comparator.py | 3 +++ .../grad_tool/grad_pt/grad_comparator.py | 3 +++ .../kj600/distributed/wrap_distributed.py | 5 ++-- .../kj600/unittest/test_basic_functions.py | 3 ++- .../kj600/kj600/unittest/test_cc_log_only.py | 3 ++- .../msprobe/pytorch/parse_tool/lib/compare.py | 27 ++++++++++--------- .../msprobe/pytorch/parse_tool/lib/config.py | 2 +- .../msprobe/pytorch/parse_tool/lib/utils.py | 5 +++- 10 files changed, 36 insertions(+), 19 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/common/utils.py b/debug/accuracy_tools/grad_tool/common/utils.py index f40f8688c..43b63676e 100644 --- a/debug/accuracy_tools/grad_tool/common/utils.py +++ b/debug/accuracy_tools/grad_tool/common/utils.py @@ -7,6 +7,7 @@ import yaml import pandas as pd from grad_tool.common.constant import GradConst +from msprobe.core.common.file_check import FileOpen def _print_log(level, msg, end='\n'): @@ -114,7 +115,7 @@ class ListCache(list): def get_config(filepath): - with open(filepath, 'r') as file: + with FileOpen(filepath, 'r') as file: config = yaml.safe_load(file) return config diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py index 895b8f2ae..a498b1a13 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py @@ -169,6 +169,7 @@ class CSVGenerator(Process): stat_data = None max_try = 10 while max_try: + check_file_or_directory_path(file_path) try: stat_data = np.load(file_path) return stat_data diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py index 2bfeda438..aa63614db 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_comparator.py @@ -3,6 +3,7 @@ import torch import numpy as np from grad_tool.common.base_comparator import BaseComparator +from grad_tool.common.utils import check_file_or_directory_path class MsGradComparator(BaseComparator): @@ -11,7 +12,9 @@ class MsGradComparator(BaseComparator): def _load_grad_files(cls, grad_file1: str, grad_file2: str): grad1_suffix = grad_file1.split(".")[-1] grad2_suffix = grad_file2.split(".")[-1] + check_file_or_directory_path(grad_file1) grad1 = torch.load(grad_file1).numpy() if grad1_suffix == "pt" else np.load(grad_file1) + check_file_or_directory_path(grad_file2) grad2 = torch.load(grad_file2).numpy() if grad2_suffix == "pt" else np.load(grad_file2) if grad1.shape != grad2.shape: diff --git a/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py b/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py index d1229b93d..6981bfcab 100644 --- a/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py +++ b/debug/accuracy_tools/grad_tool/grad_pt/grad_comparator.py @@ -1,13 +1,16 @@ import torch from grad_tool.common.base_comparator import BaseComparator +from grad_tool.common.utils import check_file_or_directory_path class PtGradComparator(BaseComparator): @classmethod def _load_grad_files(cls, grad_file1: str, grad_file2: str): + check_file_or_directory_path(grad_file1) tensor1 = torch.load(grad_file1, map_location=torch.device("cpu")) + check_file_or_directory_path(grad_file2) tensor2 = torch.load(grad_file2, map_location=torch.device("cpu")) if tensor1.shape != tensor2.shape: raise RuntimeError(f"tensor shape is not equal: {grad_file1}, {grad_file2}") diff --git a/debug/accuracy_tools/kj600/kj600/distributed/wrap_distributed.py b/debug/accuracy_tools/kj600/kj600/distributed/wrap_distributed.py index 4e2d5e175..ecbdd7578 100644 --- a/debug/accuracy_tools/kj600/kj600/distributed/wrap_distributed.py +++ b/debug/accuracy_tools/kj600/kj600/distributed/wrap_distributed.py @@ -5,6 +5,7 @@ import inspect import torch import torch.nn as nn import torch.distributed as dist +from msprobe.core.common.file_check import FileOpen from ..module_metric import get_metrics @@ -16,11 +17,11 @@ except ImportError: PREFIX_POST = "post" OpsPath = os.path.join(os.path.dirname(__file__), "distributed_ops.yaml") -with open(OpsPath) as f: +with FileOpen(OpsPath, 'r') as f: WrapDistributedOps = yaml.safe_load(f).get('distributed') StackBlackListPath = os.path.join(os.path.dirname(__file__), "stack_blacklist.yaml") -with open(StackBlackListPath) as f: +with FileOpen(StackBlackListPath, 'r') as f: StackBlackList = yaml.safe_load(f).get('stack') distributed_func = {} diff --git a/debug/accuracy_tools/kj600/kj600/unittest/test_basic_functions.py b/debug/accuracy_tools/kj600/kj600/unittest/test_basic_functions.py index b7cdd3385..1ff62c316 100644 --- a/debug/accuracy_tools/kj600/kj600/unittest/test_basic_functions.py +++ b/debug/accuracy_tools/kj600/kj600/unittest/test_basic_functions.py @@ -11,6 +11,7 @@ except ModuleNotFoundError: from kj600.module_hook import TrainerMon from tensorboard.backend.event_processing.event_accumulator import EventAccumulator +from msprobe.core.common.file_check import FileOpen class Model(torch.nn.Module): def __init__(self): @@ -37,7 +38,7 @@ def get_file_path(): def get_config(): os.environ["KJ600_OUTPUT_DIR"] = "./test_kj600_output" - with open("config_basic_functions.json", 'r') as file: + with FileOpen("config_basic_functions.json", 'r') as file: config_test = json.load(file) return config_test def get_tensorbaord(event_file_path): diff --git a/debug/accuracy_tools/kj600/kj600/unittest/test_cc_log_only.py b/debug/accuracy_tools/kj600/kj600/unittest/test_cc_log_only.py index d7508d4af..61224f2d4 100644 --- a/debug/accuracy_tools/kj600/kj600/unittest/test_cc_log_only.py +++ b/debug/accuracy_tools/kj600/kj600/unittest/test_cc_log_only.py @@ -7,9 +7,10 @@ from torch import distributed as dist import torch.multiprocessing as mp from kj600.module_hook import TrainerMon from kj600.unittest.cc_utils import * +from msprobe.core.common.file_check import FileOpen -with open(os.path.join(os.path.dirname(__file__), 'expected_cc_log.json')) as f: +with FileOpen(os.path.join(os.path.dirname(__file__), 'expected_cc_log.json'), 'r') as f: EXPECTED = json.load(f) def test_all_gather(context, rank, world_size, async_op): diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py index 85c4cde4d..2b091c59e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py @@ -83,16 +83,17 @@ class Compare: (left, right, save_txt, rl, al, diff_count) = args if left is None or right is None: raise ParseException("invalid input or output") - try: - left_data = np.load(left) - right_data = np.load(right) - except UnicodeError as e: - self.log.error("%s %s" % ("UnicodeError", str(e))) - self.log.warning("Please check the npy file") - raise ParseException(ParseException.PARSE_UNICODE_ERROR) from e - except IOError: - self.log.error("Failed to load npy %s or %s." % (left, right)) - raise ParseException(ParseException.PARSE_LOAD_NPY_ERROR) from e + if self.util.check_path_valid(left) and self.util.check_path_valid(right): + try: + left_data = np.load(left) + right_data = np.load(right) + except UnicodeError as e: + self.log.error("%s %s" % ("UnicodeError", str(e))) + self.log.warning("Please check the npy file") + raise ParseException(ParseException.PARSE_UNICODE_ERROR) from e + except IOError: + self.log.error("Failed to load npy %s or %s." % (left, right)) + raise ParseException(ParseException.PARSE_LOAD_NPY_ERROR) from e # save to txt if save_txt: @@ -157,8 +158,10 @@ class Compare: return res def compare_npy(self, file, bench_file, output_path): - data = np.load(file) - bench_data = np.load(bench_file) + if self.util.check_path_valid(file): + data = np.load(file) + if self.util.check_path_valid(bench_file): + bench_data = np.load(bench_file) shape, dtype = data.shape, data.dtype bench_shape, bench_dtype = bench_data.shape, bench_data.dtype filename = os.path.basename(file) diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/config.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/config.py index a745ff46f..a9a8b2b00 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/config.py @@ -33,7 +33,7 @@ class Const: OFFLINE_DUMP_CONVERT_PATTERN = \ r"^([A-Za-z0-9_-]+)\.([A-Za-z0-9_-]+)\.([0-9]+)(\.[0-9]+)?\.([0-9]{1,255})" \ r"\.([a-z]+)\.([0-9]{1,255})(\.[x0-9]+)?\.npy$" - NUMPY_PATTERN = r".*\.npy$" + NUMPY_PATTERN = r"^[\w\-_-]\.npy$" NPY_SUFFIX = ".npy" PKL_SUFFIX = ".pkl" DIRECTORY_LENGTH = 4096 diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py index 266e93fb3..3ed5a8a97 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py @@ -245,7 +245,10 @@ class Util: elif data.size % align != 0: pad_array = np.zeros((align - data.size % align,)) data = np.append(data, pad_array) - np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g') + try: + np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g') + except Exception as e: + self.log.error("An unexpected error occurred: %s when savetxt to %s" % (str(e)), dst_file) change_mode(dst_file, FileCheckConst.DATA_FILE_AUTHORITY) def list_convert_files(self, path, external_pattern=""): -- Gitee From bfed17ecf88c96d7895e4e66604f6e6841718013 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Mon, 5 Aug 2024 16:27:52 +0800 Subject: [PATCH 111/791] =?UTF-8?q?=20=20=E5=BE=AE=E9=87=8D=E6=9E=84=20?= =?UTF-8?q?=EF=BC=8C=E5=8F=A6=E5=A4=96=E5=8F=98=E6=9B=B4=E4=BA=86=E5=AF=B9?= =?UTF-8?q?=E5=A4=96=E7=9A=84=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/compare/ms_compare.py | 146 +++++++++--------- debug/accuracy_tools/msprobe/msprobe.py | 2 +- .../{pt_comparator.py => pt_compare.py} | 137 ++++++++-------- 3 files changed, 145 insertions(+), 140 deletions(-) rename debug/accuracy_tools/msprobe/pytorch/compare/{pt_comparator.py => pt_compare.py} (69%) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 9d1e1976b..c631655d9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -17,7 +17,7 @@ from msprobe.core.compare.highlight import find_compare_result_error_rows,highli from msprobe.core.compare.Multiprocessing_compute import _handle_multi_process from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger - +from msprobe.core.common.exceptions import FileCheckException class MSComparator (Comparator): def __init__(self): @@ -186,82 +186,84 @@ class MSComparator (Comparator): data_value=data_value.astype(np.float32) return data_value + + def compare_core(self,input_parma, output_path, **kwargs): + """ + Compares data from multiple JSON files and generates a comparison report. + + Args: + input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", + "stack_path"). + output_path (str): The path where the output Excel report will be saved. + **kwargs: Additional keyword arguments including: + - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. + - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. + - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. + - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. + - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. + - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. + + Returns: + """ + # get kwargs or set default value + stack_mode = kwargs.get('stack_mode', False) + auto_analyze = kwargs.get('auto_analyze', True) + suffix = kwargs.get('suffix', '') + fuzzy_match = kwargs.get('fuzzy_match', False) + summary_compare = kwargs.get('summary_compare', False) + md5_compare = kwargs.get('md5_compare', False) + + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") + file_name = add_time_with_xlsx("compare_result" + suffix) + file_path = os.path.join(os.path.realpath(output_path), file_name) + check_file_not_exists(file_path) + highlight_dict = {'red_rows': [], 'yellow_rows': []} + with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_path"), "r") as stack_json: + result_df = self.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, + summary_compare, md5_compare) + + if not md5_compare and not summary_compare: + result_df = self._do_multi_process(input_parma, result_df) + find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) + highlight_rows_xlsx(result_df, highlight_dict, file_path) + if auto_analyze: + advisor = Advisor(result_df, output_path) + advisor.analysis() + + +# def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, +# fuzzy_match=False): +# try: +# summary_compare, md5_compare = task_dumppath_get(input_parma) +# check_configuration_param(stack_mode, auto_analyze, fuzzy_match) +# create_directory(output_path) +# check_compare_param(input_parma, output_path, summary_compare, md5_compare) +# except CompareException as error: +# logger.error('Compare failed. Please check the arguments and do it again!') +# sys.exit(error.code) +# compare_core(input_parma, output_path, stack_mode=stack_mode, +# auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, +# md5_compare=md5_compare) - -def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, - fuzzy_match=False): +def ms_compare(args): + with FileOpen(args.input_path, "r") as file: + input_param = json.load(file) try: - summary_compare, md5_compare = task_dumppath_get(input_parma) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) - create_directory(output_path) - check_compare_param(input_parma, output_path, summary_compare, md5_compare) - except CompareException as error: + summary_compare, md5_compare = task_dumppath_get(input_param) + check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) + create_directory(args.output_path) + check_compare_param(input_param, args.output_path, summary_compare, md5_compare) + except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) - compare_core(input_parma, output_path, stack_mode=stack_mode, - auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) - -# def compare(args): -# with FileOpen(args.input_path, "r") as file: -# input_param = json.load(file) -# try: -# summary_compare, md5_compare = task_dumppath_get(input_param) -# check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) -# create_directory(args.output_path) -# check_compare_param(input_param, args.output_path, summary_compare, md5_compare) -# except (CompareException, FileCheckException) as error: -# logger.error('Compare failed. Please check the arguments and do it again!') -# sys.exit(error.code) -# msComparator.compare_core(input_param, args.output_path, stack_mode=args.stack_mode, -# auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, -# md5_compare=md5_compare) - -def compare_core(input_parma, output_path, **kwargs): - """ - Compares data from multiple JSON files and generates a comparison report. - - Args: - input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", - "stack_path"). - output_path (str): The path where the output Excel report will be saved. - **kwargs: Additional keyword arguments including: - - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. - - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. - - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. - - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. - - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. - - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. - - Returns: - """ - # get kwargs or set default value - stack_mode = kwargs.get('stack_mode', False) - auto_analyze = kwargs.get('auto_analyze', True) - suffix = kwargs.get('suffix', '') - fuzzy_match = kwargs.get('fuzzy_match', False) - summary_compare = kwargs.get('summary_compare', False) - md5_compare = kwargs.get('md5_compare', False) - - logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - file_name = add_time_with_xlsx("compare_result" + suffix) - file_path = os.path.join(os.path.realpath(output_path), file_name) - check_file_not_exists(file_path) - highlight_dict = {'red_rows': [], 'yellow_rows': []} msComparator= MSComparator() - with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_path"), "r") as stack_json: - result_df = msComparator.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, - summary_compare, md5_compare) - - if not md5_compare and not summary_compare: - result_df = msComparator._do_multi_process(input_parma, result_df) - find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) - highlight_rows_xlsx(result_df, highlight_dict, file_path) - if auto_analyze: - advisor = Advisor(result_df, output_path) - advisor.analysis() + msComparator.compare_core(input_param, args.output_path, stack_mode=args.stack_mode, + auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) + + diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index a27d3b55a..11c3899bd 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -23,7 +23,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _ from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command from msprobe.pytorch.compare.acc_compare import _compare_parser, compare -from msprobe.pytorch.compare.pt_comparator import pt_compare +from debug.accuracy_tools.msprobe.pytorch.compare.pt_compare import pt_compare def main(): parser = argparse.ArgumentParser( diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py similarity index 69% rename from debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py rename to debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 22dd2be4d..49fc5ed65 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_comparator.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -18,7 +18,7 @@ from msprobe.core.compare.highlight import find_compare_result_error_rows,highli from msprobe.core.compare.Multiprocessing_compute import _handle_multi_process from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger - +from msprobe.core.common.exceptions import FileCheckException class PTComparator (Comparator): def __init__(self): @@ -189,82 +189,85 @@ class PTComparator (Comparator): logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e + def compare_core(self,input_parma, output_path, **kwargs): + """ + Compares data from multiple JSON files and generates a comparison report. + Args: + input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", + "stack_path"). + output_path (str): The path where the output Excel report will be saved. + **kwargs: Additional keyword arguments including: + - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. + - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. + - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. + - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. + - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. + - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. -def pt_compare(input_parma, output_path, stack_mode=False, auto_analyze=True, - fuzzy_match=False): - try: - summary_compare, md5_compare = task_dumppath_get(input_parma) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) - create_directory(output_path) - check_compare_param(input_parma, output_path, summary_compare, md5_compare) - except CompareException as error: - logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) - compare_core(input_parma, output_path, stack_mode=stack_mode, - auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) + Returns: + """ + # get kwargs or set default value + stack_mode = kwargs.get('stack_mode', False) + auto_analyze = kwargs.get('auto_analyze', True) + suffix = kwargs.get('suffix', '') + fuzzy_match = kwargs.get('fuzzy_match', False) + summary_compare = kwargs.get('summary_compare', False) + md5_compare = kwargs.get('md5_compare', False) + + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") + file_name = add_time_with_xlsx("compare_result" + suffix) + file_path = os.path.join(os.path.realpath(output_path), file_name) + check_file_not_exists(file_path) + highlight_dict = {'red_rows': [], 'yellow_rows': []} + + with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_path"), "r") as stack_json: + result_df = self.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, + summary_compare, md5_compare) + + if not md5_compare and not summary_compare: + result_df = self._do_multi_process(input_parma, result_df) + find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) + highlight_rows_xlsx(result_df, highlight_dict, file_path) + if auto_analyze: + advisor = Advisor(result_df, output_path) + advisor.analysis() -# def compare(args): -# with FileOpen(args.input_path, "r") as file: -# input_param = json.load(file) + +# def pt_compare(input_parma, output_path, stack_mode=False, auto_analyze=True, +# fuzzy_match=False): # try: -# summary_compare, md5_compare = task_dumppath_get(input_param) -# check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) -# create_directory(args.output_path) -# check_compare_param(input_param, args.output_path, summary_compare, md5_compare) -# except (CompareException, FileCheckException) as error: +# summary_compare, md5_compare = task_dumppath_get(input_parma) +# check_configuration_param(stack_mode, auto_analyze, fuzzy_match) +# create_directory(output_path) +# check_compare_param(input_parma, output_path, summary_compare, md5_compare) +# except CompareException as error: # logger.error('Compare failed. Please check the arguments and do it again!') # sys.exit(error.code) -# msComparator.compare_core(input_param, args.output_path, stack_mode=args.stack_mode, -# auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, -# md5_compare=md5_compare) - -def compare_core(input_parma, output_path, **kwargs): - """ - Compares data from multiple JSON files and generates a comparison report. - - Args: - input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", - "stack_path"). - output_path (str): The path where the output Excel report will be saved. - **kwargs: Additional keyword arguments including: - - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. - - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. - - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. - - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. - - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. - - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. +# ptComparator= PTComparator() +# ptComparator.compare_core(input_parma, output_path, stack_mode=stack_mode, +# auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, +# md5_compare=md5_compare) - Returns: - """ - # get kwargs or set default value - stack_mode = kwargs.get('stack_mode', False) - auto_analyze = kwargs.get('auto_analyze', True) - suffix = kwargs.get('suffix', '') - fuzzy_match = kwargs.get('fuzzy_match', False) - summary_compare = kwargs.get('summary_compare', False) - md5_compare = kwargs.get('md5_compare', False) - logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - file_name = add_time_with_xlsx("compare_result" + suffix) - file_path = os.path.join(os.path.realpath(output_path), file_name) - check_file_not_exists(file_path) - highlight_dict = {'red_rows': [], 'yellow_rows': []} +def pt_compare(args): + with FileOpen(args.input_path, "r") as file: + input_param = json.load(file) + try: + summary_compare, md5_compare = task_dumppath_get(input_param) + check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) + create_directory(args.output_path) + check_compare_param(input_param, args.output_path, summary_compare, md5_compare) + except (CompareException, FileCheckException) as error: + logger.error('Compare failed. Please check the arguments and do it again!') + sys.exit(error.code) ptComparator= PTComparator() - with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_path"), "r") as stack_json: - result_df = ptComparator.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, - summary_compare, md5_compare) + ptComparator.compare_core(input_param, args.output_path, stack_mode=args.stack_mode, + auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) - if not md5_compare and not summary_compare: - result_df = ptComparator._do_multi_process(input_parma, result_df) - find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) - highlight_rows_xlsx(result_df, highlight_dict, file_path) - if auto_analyze: - advisor = Advisor(result_df, output_path) - advisor.analysis() -- Gitee From 92e35d3a4dfb0c0c024d629abc2996e1bd59b8f6 Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 16:45:56 +0800 Subject: [PATCH 112/791] 82 --- .../compare_bean/profiling_info.py | 77 ++++++++++++++----- .../profiling_parser/gpu_profiling_parser.py | 18 +++++ .../profiling_parser/npu_profiling_parser.py | 20 ++++- .../compare_bean/test_profiling_info.py | 54 +++++++++---- .../test_gpu_profiling_parser.py | 2 +- 5 files changed, 132 insertions(+), 39 deletions(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index 2b966a449..16bef2f0f 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -136,7 +136,15 @@ class ProfilingInfo: @property def vector_total_num(self): return sum((self.vector_num_trans, self.vector_num_notrans)) - + def trans_to_s(self): + self.cube_time /= 10 ** 3 + self.vec_time /= 10 ** 3 + self.conv_time_fwd /= 10 ** 3 + self.conv_time_bwd /= 10 ** 3 + self.sdma_time /= 10 ** 3 + self.fa_time_bwd /= 10 ** 3 + self.pa_time /= 10 ** 3 + self.fa_time_fwd /= 10 ** 3 def trans_time_to_s(self): # 新指标单位为ms self.fa_time_fwd_cube /= 10 ** 3 @@ -155,24 +163,6 @@ class ProfilingInfo: self.sdma_time_stream /= 10 ** 3 self.page_attention_time /= 10 ** 3 self.other_cube_time /= 10 ** 3 - - self.cube_time = (self.matmul_time_cube + self.matmul_time_vector + self.other_cube_time) / 1000 - self.vec_time = (self.vector_time_trans + self.vector_time_notrans) / 1000 - self.cube_num = (self.matmul_num_cube + self.matmul_num_vector + self.other_cube_num) - self.vec_num = (self.vector_num_trans + self.vector_num_notrans) - self.sdma_num = (self.sdma_num_tensor_move + self.sdma_num_stream) - self.fa_num_fwd = (self.fa_num_fwd_cube + self.fa_num_fwd_vector) - self.fa_num_bwd = (self.fa_num_bwd_cube + self.fa_num_bwd_vector) - self.pa_num = self.page_attention_num - self.conv_time_fwd = (self.conv_time_fwd_cube + self.conv_time_fwd_vector) / 1000 - self.conv_time_bwd = (self.conv_time_bwd_cube + self.conv_time_bwd_vector) / 1000 - self.conv_num_fwd = (self.conv_num_fwd_cube + self.conv_num_fwd_vector) - self.conv_num_bwd = (self.conv_num_bwd_cube + self.conv_num_bwd_vector) - self.sdma_time = (self.sdma_time_tensor_move + self.sdma_time_stream) / 1000 - self.fa_time_bwd = (self.fa_time_bwd_cube + self.fa_time_bwd_vector) / 1000 - self.pa_time = self.page_attention_time / 1000 - self.fa_time_fwd = (self.fa_time_fwd_cube + self.fa_time_fwd_vector) / 1000 - self.other_time = self.other_time / 10 ** 6 self.compute_time = self.compute_time / 10 ** 6 self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 @@ -180,6 +170,55 @@ class ProfilingInfo: self.e2e_time = self.e2e_time / 10 ** 6 self.scheduling_time = self.scheduling_time / 10 ** 6 self.lccl_time = self.lccl_time / 10 ** 6 + + def calculate_cube_time(self): + self.cube_time = self.matmul_time_cube + self.matmul_time_vector + self.other_cube_time + + def calculate_vec_time(self): + self.vec_time = self.vector_time_trans + self.vector_time_notrans + + def calculate_cube_num(self): + self.cube_num = self.matmul_num_cube + self.matmul_num_vector + self.other_cube_num + + def calculate_vec_num(self): + self.vec_num = self.vector_num_trans + self.vector_num_notrans + + def calculate_sdma_num(self): + self.sdma_num = self.sdma_num_tensor_move + self.sdma_num_stream + + def calculate_fa_num_fwd(self): + self.fa_num_fwd = self.fa_num_fwd_cube + self.fa_num_fwd_vector + + def calculate_fa_num_bwd(self): + self.fa_num_bwd = self.fa_num_bwd_cube + self.fa_num_bwd_vector + + def calculate_pa_num(self): + self.pa_num = self.page_attention_num + + def calculate_pa_time(self): + self.pa_num = self.page_attention_time + + def calculate_conv_time_fwd(self): + self.conv_time_fwd = self.conv_time_fwd_cube + self.conv_time_fwd_vector + + def calculate_conv_time_bwd(self): + self.conv_time_bwd = self.conv_time_bwd_cube + self.conv_time_bwd_vector + + def calculate_conv_num_fwd(self): + self.conv_num_fwd = self.conv_num_fwd_cube + self.conv_num_fwd_vector + + def calculate_conv_num_bwd(self): + self.conv_num_bwd = self.conv_num_bwd_cube + self.conv_num_bwd_vector + + def calculate_sdma_time(self): + self.sdma_time = self.sdma_time_tensor_move + self.sdma_time_stream + + def calculate_fa_time_fwd(self): + self.fa_time_fwd = self.fa_time_fwd_cube + self.fa_time_fwd_vector + + def calculate_fa_time_bwd(self): + self.fa_time_bwd = self.fa_time_bwd_cube + self.fa_time_bwd_vector + def calculate_other_time(self): self.other_time = max( [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 175b77603..4d4734a4c 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -58,12 +58,30 @@ class GPUProfilingParser(BaseProfilingParser): for record in addr_dict.values(): self._result_data.update_memory_list(record) + gpu + def _update_overall_metrics(self): self._calculate_performance_time() self.__parse_memory_reserved() self._result_data.overall_metrics.trans_time_to_s() + self._result_data.overall_metrics.calculate_cube_time() self._result_data.overall_metrics.calculate_vec_time() + self._result_data.overall_metrics.calculate_cube_num() + self._result_data.overall_metrics.calculate_vec_num() + self._result_data.overall_metrics.calculate_sdma_num() + self._result_data.overall_metrics.calculate_fa_num_fwd() + self._result_data.overall_metrics.calculate_fa_num_bwd() + self._result_data.overall_metrics.calculate_pa_num() + self._result_data.overall_metrics.calculate_pa_time() + self._result_data.overall_metrics.calculate_conv_time_fwd() + self._result_data.overall_metrics.calculate_conv_time_bwd() + self._result_data.overall_metrics.calculate_conv_num_fwd() + self._result_data.overall_metrics.calculate_conv_num_bwd() + self._result_data.overall_metrics.calculate_sdma_time() + self._result_data.overall_metrics.calculate_fa_time_fwd() + self._result_data.overall_metrics.calculate_fa_time_bwd() self._result_data.overall_metrics.calculate_schedule_time() + self._result_data.overall_metrics.trans_to_s() def _calculate_performance_time(self): min_ts = sys.float_info.max diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 3c3f05427..1d00332b4 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -159,10 +159,26 @@ class NPUProfilingParser(BaseProfilingParser): self.__add_overlap_analysis_time() self._picking_notify_wait_event_and_not_overlap_event() self.__add_overlap_wait_time() + self._result_data.overall_metrics.calculate_schedule_time() self._result_data.overall_metrics.trans_time_to_s() + self._result_data.overall_metrics.calculate_cube_time() + self._result_data.overall_metrics.calculate_vec_time() + self._result_data.overall_metrics.calculate_cube_num() + self._result_data.overall_metrics.calculate_vec_num() + self._result_data.overall_metrics.calculate_sdma_num() + self._result_data.overall_metrics.calculate_fa_num_fwd() + self._result_data.overall_metrics.calculate_fa_num_bwd() + self._result_data.overall_metrics.calculate_pa_num() + self._result_data.overall_metrics.calculate_pa_time() + self._result_data.overall_metrics.calculate_conv_time_fwd() + self._result_data.overall_metrics.calculate_conv_time_bwd() + self._result_data.overall_metrics.calculate_conv_num_fwd() + self._result_data.overall_metrics.calculate_conv_num_bwd() + self._result_data.overall_metrics.calculate_sdma_time() + self._result_data.overall_metrics.calculate_fa_time_fwd() + self._result_data.overall_metrics.calculate_fa_time_bwd() + self._result_data.overall_metrics.trans_to_s() self._result_data.overall_metrics.calculate_other_time() - self._result_data.overall_metrics.calculate_schedule_time() - self._update_bandwidth() def _picking_notify_wait_event_and_not_overlap_event(self): self.notify_event_cache = [] diff --git a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py index dc85b0af0..e6d543a77 100644 --- a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py +++ b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py @@ -36,40 +36,60 @@ class TestProfilingInfo(unittest.TestCase): def test_update_fa_fwd_info(self): info = ProfilingInfo("NPU") - info.update_fa_fwd_info(5) - info.update_fa_fwd_info(5) + info.fa_time_fwd_cube = 5 + info.fa_time_fwd_vector = 5 + info.fa_num_fwd_cube = 1 + info.fa_num_fwd_vector = 1 + info.calculate_fa_time_fwd() + info.calculate_fa_num_fwd() self.assertEqual(info.fa_time_fwd, 10) self.assertEqual(info.fa_num_fwd, 2) def test_update_fa_bwd_info(self): info = ProfilingInfo("NPU") - info.update_fa_bwd_info(5) - info.update_fa_bwd_info(5) + info.fa_time_bwd_cube = 5 + info.fa_time_bwd_vector = 5 + info.fa_num_bwd_cube = 1 + info.fa_num_bwd_vector = 1 + info.calculate_fa_time_bwd() + info.calculate_fa_num_bwd() self.assertEqual(info.fa_time_bwd, 10) self.assertEqual(info.fa_num_bwd, 2) def test_update_sdma_info(self): info = ProfilingInfo("NPU") - info.update_sdma_info(5) - self.assertEqual(info.sdma_time, 5) - self.assertEqual(info.sdma_num, 1) - info.update_sdma_info(5, 5) + info.sdma_time_tensor_move = 5 + info.sdma_time_stream = 5 + info.sdma_num_tensor_move = 5 + info.sdma_num_stream = 5 + info.calculate_sdma_time() + info.calculate_sdma_num() self.assertEqual(info.sdma_time, 10) - self.assertEqual(info.sdma_num, 6) + self.assertEqual(info.sdma_num, 10) def test_update_cube_info(self): info = ProfilingInfo("NPU") - info.update_cube_info(5) - info.update_cube_info(5) - self.assertEqual(info.cube_time, 10) - self.assertEqual(info.cube_num, 2) + info.matmul_time_cube = 1 + info.matmul_time_vector = 1 + info.other_cube_time = 1 + info.matmul_num_cube = 5 + info.matmul_num_vector = 5 + info.other_cube_num = 5 + info.calculate_cube_time() + info.calculate_cube_num() + self.assertEqual(info.cube_time, 3) + self.assertEqual(info.cube_num, 15) def test_update_vec_info(self): info = ProfilingInfo("NPU") - info.update_vec_info(5) - info.update_vec_info(5) - self.assertEqual(info.vec_time, 10) - self.assertEqual(info.vec_num, 2) + info.vector_time_trans = 1 + info.vector_time_notrans = 1 + info.vector_num_trans = 2 + info.vector_num_notrans = 2 + info.calculate_vec_time() + info.calculate_vec_num() + self.assertEqual(info.vec_time, 2) + self.assertEqual(info.vec_num, 4) def test_set_compute_time(self): info = ProfilingInfo("NPU") diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index d7cb3d058..50d60f39f 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -77,7 +77,7 @@ class TestGpuProfilingParser(unittest.TestCase): res._calculate_performance_time() self.assertEqual(res._result_data.overall_metrics.e2e_time, 98) self.assertEqual(res._result_data.overall_metrics.sdma_time, 4) - self.assertEqual(res._result_data.overall_metrics.sdma_num, 4) + self.assertEqual(res._result_data.overall_metrics.sdma_num, 0) self.assertEqual(res._result_data.overall_metrics.cube_time, 1) self.assertEqual(res._result_data.overall_metrics.cube_num, 1) self.assertEqual(res._result_data.overall_metrics.fa_time_fwd, 2) -- Gitee From 582a0992cf2436ec6d669921f66036b6d97fe43d Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 17:11:46 +0800 Subject: [PATCH 113/791] 82 --- .../compare_backend/compare_bean/profiling_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index 16bef2f0f..71fb3c4a8 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -196,7 +196,7 @@ class ProfilingInfo: self.pa_num = self.page_attention_num def calculate_pa_time(self): - self.pa_num = self.page_attention_time + self.pa_num = self.page_attention_num def calculate_conv_time_fwd(self): self.conv_time_fwd = self.conv_time_fwd_cube + self.conv_time_fwd_vector -- Gitee From f7370809ee7825b97fd799b73ef4f25b945070b2 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Thu, 25 Jul 2024 16:43:12 +0800 Subject: [PATCH 114/791] =?UTF-8?q?[msprobe]dump=E6=94=AF=E6=8C=81MindSpor?= =?UTF-8?q?e=E5=8A=A8=E6=80=81=E5=9B=BE=E5=9C=BA=E6=99=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/config/README.md | 108 +++++++++++---- .../msprobe/mindspore/doc/dump.md | 131 +++++++++++++++++- .../msprobe/pytorch/doc/dump.md | 6 +- 3 files changed, 210 insertions(+), 35 deletions(-) diff --git a/debug/accuracy_tools/msprobe/config/README.md b/debug/accuracy_tools/msprobe/config/README.md index 7b91bd26f..da8b67d53 100644 --- a/debug/accuracy_tools/msprobe/config/README.md +++ b/debug/accuracy_tools/msprobe/config/README.md @@ -2,17 +2,19 @@ 当前配置文件主要为PrecisionDebugger接口执行dump或无标杆比对操作时调用的配置，当PrecisionDebugger接口未指定该配置文件时，使用该文件的默认配置。配置文件详见[config.json](./config.json)。 +本文针对msprobe工具支持的PyTorch、MindSpore静态图和MindSpore静态图场景进行描述。 + ## 参数说明 ### **通用配置参数** | 参数名 | 说明 | 是否必选 | | ----------------- | ------------------------------------------------------------ | -------- | -| task | dump的任务类型，str类型。可取值"free_benchmark"（无标杆比对，仅PyTorch场景支持）、"statistics"（仅dump API统计信息，默认值）、"tensor"（dump API统计信息和完全复刻整网的API运行情况的真实数据）、"overflow_check"（溢出检测）。配置示例："task": "tensor"。根据task参数取值的不同，可以配置不同场景参数，详见：“**task配置为free_benchmark**”，“**task配置为statistics**”，“**task配置为tensor**”，“**task配置为overflow_check**”。 | 否 | -| dump_path | 设置dump数据目录路径，str类型。配置示例："dump_path": "./dump_path"。MindSpore场景仅支持绝对路径。 | 是 | +| task | dump的任务类型，str类型。可取值：
"free_benchmark"（无标杆比对）。
"statistics"（仅dump张量的统计信息，默认值）。
"tensor"（dump张量的统计信息和完整张量数据，MindSpore静态图场景仅dump完整张量数据）。
"overflow_check"（溢出检测）。
"run_ut"（精度预检配置）。
配置示例："task": "tensor"。
根据task参数取值的不同，可以配置不同场景参数，详见：“**task配置为free_benchmark**”，“**task配置为statistics**”，“**task配置为tensor**”，“**task配置为overflow_check**”，“**task配置为run_ut**”。 | 否 | +| dump_path | 设置dump数据目录路径，str类型。配置示例："dump_path": "./dump_path"。MindSpore静态图场景仅支持绝对路径。 | 是 | | rank | 指定对某张卡上的数据进行dump，list[int]类型，默认未配置（表示dump所有卡的数据），应配置为大于等于0的整数，且须配置实际可用的Rank ID。配置示例："rank": [1]。
对于PyTorch场景，Rank ID从0开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的Rank ID，则dump数据为空，比如当前环境Rank ID为0到7，实际训练运行0到3卡，此时若配置Rank ID为4或不存在的10等其他值，此时dump数据为空。
对于MindSpore场景，所有节点的Rank ID均从0开始计数，最大取值为每个节点可用卡总数-1，config.json配置一次rank参数对所有节点同时生效。 | 否 | | step | 指定dump某个step的数据，list[int]类型。默认未配置，表示dump所有step数据。dump特定step时，须指定为训练脚本中存在的step。step为list格式，可配置逐个step，例如："step": [0,1,2]。 | 否 | -| level | dump级别，str类型，根据不同级别dump不同数据。可取值"L0"（dump module模块级精度数据，仅PyTorch场景支持，使用背景详见“**模块级精度数据dump说明**”）、"L1"（dump API级精度数据，默认值）、"L2"（dump kernel级精度数据，须配置acl_config参数）、"mix"（dump module模块级和API级精度数据，即"L0"+"L1"，仅PyTorch场景支持）。配置示例："level": "L1"。 | 否 | +| level | dump级别，str类型，根据不同级别dump不同数据。可取值：
"L0"（dump module模块级精度数据，仅PyTorch场景支持，使用背景详见“**模块级精度数据dump说明**”）。
"L1"（dump API级精度数据，默认值，仅MindSpore动态图和PyTorch场景支持）。
"L2"（dump kernel级精度数据，仅MindSpore静态图和PyTorch场景支持，且PyTorch场景须配置acl_config参数）。
"mix"（dump module模块级和API级精度数据，即"L0"+"L1"，仅PyTorch场景支持）。
配置示例："level": "L1"。MindSpore动态图场景仅支持"L1"。 | 否 | | acl_config | kernel dump的配置文件，str类型。level取"L2"时，该参数必选；level为其他值时，该参数不选。参数示例：acl_config='./acl_config.json'。acl_config.json配置文件详细介绍请参见“**acl_config.json配置文件说明**”。 | 否 | | seed | 随机种子数，int类型，默认值为：1234，仅PyTorch场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见“**固定随机数范围**”。配置示例："seed": 1234。 | 否 | | is_deterministic | 确定性计算模式，bool类型，仅PyTorch场景支持。可取值true（开启）或false（关闭），默认关闭。配置示例："is_deterministic": true。
即使在相同的硬件和输入下，API多次执行的结果也可能不同，开启确定性计算是为了保证在相同的硬件和输入下，API多次执行的结果相同。
确定性计算会导致API执行性能降低，建议在发现模型多次执行结果不同的情况下开启。
rnn类算子、ReduceSum、ReduceMean等算子可能与确定性计算存在冲突，若开启确定性计算后多次执行的结果不相同，则考虑存在这些算子。 | 否 | @@ -63,27 +65,31 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 | 参数名 | 说明 | 是否必选 | | ------------ | ------------------------------------------------------------ | -------- | -| scope | PyTorch场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。 | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
MindSpore场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | -| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | -| summary_mode | 控制dump文件输出的模式，str类型，仅PyTorch场景支持，可取值md5（dump输出包含md5值以及API统计信息的dump.json文件，用于验证数据的完整性）、statistics（dump仅输出包含API统计信息的dump.json文件，默认值）。配置示例："summary_mode": "md5"。 | 否 | +| scope | PyTorch和MindSpore动态图场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。MindSpore动态图场景当前仅支持配置为API名。 | 否 | +| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch和MindSpore动态图场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch和MindSpore动态图场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
MindSpore静态图场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | +| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore静态图场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | +| summary_mode | 控制dump文件输出的模式，str类型，仅PyTorch和MindSpore动态图场景支持，可取值md5（dump输出包含md5值以及API统计信息的dump.json文件，用于验证数据的完整性）、statistics（dump仅输出包含API统计信息的dump.json文件，默认值）。配置示例："summary_mode": "md5"。 | 否 | ### task配置为tensor +MindSpore静态图场景仅dump完整张量数据。 + | 参数名 | 说明 | 是否必选 | | -------------- | ------------------------------------------------------------ | -------- | -| scope | PyTorch场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。 | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
PyTorch场景配置kernel_api，dump前向和反向API的kernel_api级别数据，其中dump反向API时需要配置**backward_input**参数。前向API配置示例："list": ["Tensor.permute.1.forward"]；反API配置示例："list": ["Tensor.permute.1.forward"], "backward.input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
MindSpore场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | +| scope | PyTorch和MindSpore动态图场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。MindSpore动态图场景当前仅支持配置为API名。 | 否 | +| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch和MindSpore动态图场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch和MindSpore动态图场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
PyTorch和MindSpore动态图场景配置kernel_api，dump前向和反向API的kernel_api级别数据，其中dump反向API时需要配置**backward_input**参数。前向API配置示例："list": ["Tensor.permute.1.forward"]；反API配置示例："list": ["Tensor.permute.1.forward"], "backward.input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
MindSpore静态图场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | | backward_input | 该输入文件为首次运行训练dump得到反向API输入的dump文件，str类型，仅PyTorch场景支持，默认未配置。例如若需要dump Functional.conv2d.1 API的反向过程的输入输出，则需要在dump目录下查找命名包含Functional.conv2d.1、backward和input字段的dump文件。配置示例："backward_input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"] | 否 | -| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | -| file_format | MindSpore场景真实tensor数据的保存格式，str类型，可取值"bin"（dump的tensor文件为二进制格式，"level": "L1"时不支持）、"npy"（dump的tensor文件后缀为.npy，默认值）。 | 否 | +| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore静态图场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | +| file_format | MindSpore静态图场景真实tensor数据的保存格式，str类型，可取值"bin"（dump的tensor文件为二进制格式，"level": "L1"时不支持）、"npy"（dump的tensor文件后缀为.npy，默认值）。 | 否 | ### task配置为overflow_check +MindSpore静态图场景的jit_level为O0/O1时，不支持该功能，须配置jit_level为O2。请参见[mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig)配置jit_config。 + | 参数名 | 说明 | 是否必选 | | ------------- | ------------------------------------------------------------ | -------- | -| overflow_nums | 控制溢出次数，int类型，仅PyTorch场景支持，表示第N次溢出时，停止训练，过程中检测到溢出API对应kernel数据均dump。配置示例："overflow_nums": 3。默认为1，即检测到1次溢出，训练停止，配置为-1时，表示持续检测溢出直到训练结束。 | 否 | -| check_mode | MindSpore场景kernel级别的溢出检测，str类型，可取值"aicore"（开启AI Core的溢出检测）、"atomic"（开启Atomic的溢出检测）、"all"（开启AI Core和Atomic的溢出检测，默认值）。配置示例"check_mode": "aicore"。 | 否 | +| overflow_nums | 控制溢出次数，int类型，仅MindSpore动态图和PyTorch场景支持，表示第N次溢出时，停止训练，过程中检测到溢出API对应kernel数据均dump。配置示例："overflow_nums": 3。默认为1，即检测到1次溢出，训练停止，配置为-1时，表示持续检测溢出直到训练结束。 | 否 | +| check_mode | MindSpore静态图场景kernel级别的溢出检测，str类型，可取值"aicore"（开启AI Core的溢出检测）、"atomic"（开启Atomic的溢出检测）、"all"（开启AI Core和Atomic的溢出检测，默认值）。配置示例"check_mode": "aicore"。 | 否 | ## 配置示例 @@ -180,7 +186,59 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 } ``` -### MindSpore场景task配置为statistics +### MindSpore静态图场景task配置为statistics + +```json +{ + "task": "statistics", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L2", + + "statistics": { + "list": [], + "data_mode": ["all"], + "summary_mode": "statistics" + } +} +``` + +### MindSpore静态图场景task配置为tensor + +```json +{ + "task": "tensor", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L2", + + "tensor": { + "list":[], + "data_mode": ["all"], + "backward_input": "" + } +} +``` + +### MindSpore静态图场景task配置为overflow_check + +```json +{ + "task": "overflow_check", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L2", + + "overflow_check": { + "check_mode": "all" + } +} +``` + +### MindSpore动态图场景task配置为statistics ```json { @@ -189,10 +247,9 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 "rank": [], "step": [], "level": "L1", - "seed": 1234, - "is_deterministic": false, "statistics": { + "scope": [], "list": [], "data_mode": ["all"], "summary_mode": "statistics" @@ -200,7 +257,7 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 } ``` -### MindSpore场景task配置为tensor +### MindSpore动态图场景task配置为tensor ```json { @@ -209,18 +266,16 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 "rank": [], "step": [], "level": "L1", - "seed": 1234, - "is_deterministic": false, "tensor": { + "scope": [], "list":[], "data_mode": ["all"], - "backward_input": "" } } ``` -### MindSpore场景task配置为overflow_check +### MindSpore动态图场景task配置为overflow_check ```json { @@ -229,12 +284,9 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 "rank": [], "step": [], "level": "L1", - "seed": 1234, - "is_deterministic": false, "overflow_check": { - "overflow_nums": 1, - "check_mode": "all" + "overflow_nums": 1 } } ``` @@ -255,7 +307,7 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 #### [config.json](./config.json)配置示例 -当level取"L2"时，须配置acl_config参数，并指定acl_config.json文件（用于指定L2 kernel级dump的配置），此时config.json文件配置示例如下： +当PyTorch场景level取"L2"时，须配置acl_config参数，并指定acl_config.json文件（用于指定L2 kernel级dump的配置），此时config.json文件配置示例如下： - 前向kernel dump配置示例： @@ -326,13 +378,13 @@ acl_config.json文件须自行创建，配置示例如下： | 字段名 | 说明 | | -------------- | ------------------------------------------------------------ | | dump_list | 待dump数据的API模型。为空，无需配置。 | -| dump_path | dump数据文件存储到运行环境的目录，主要用于指定kernel dump数据路径。支持配置绝对路径或相对路径。dump_path须为已存在目录。 | +| dump_path | dump数据文件存储到运行环境的目录，主要配置的是kernel级数据的存放路径。支持配置绝对路径或相对路径。dump_path须为已存在目录。 | | dump_mode | dump数据模式，配置如下： output：dump API的输出数据。默认值。 input：dump API的输入数据。 all：dump API的输入、输出数据。 | | dump_op_switch | 单API模型dump数据开关，配置如下：
off：关闭单API模型dump，默认值。
on：开启单API模型dump。 | **dump目录说明** -配置register_hook的dump_config后，采集的dump数据会在{dump_path}/{time}/{deviceid}/{model_id}目录下生成，例如“/home/HwHiAiUser/output/20200808163566/0/0” +配置acl_config.json后，采集的kernel级数据会在{dump_path}/{time}/{deviceid}/{model_id}目录下生成，例如“/home/HwHiAiUser/output/20200808163566/0/0” ``` ├── 20230131172437 diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md index 425d0683a..d346ff49c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md @@ -12,7 +12,7 @@ msprobe工具主要通过在训练脚本内添加dump接口并启动训练的方 通过加载dump配置文件的方式来确定dump操作的详细配置。 -可以在from msprobe.mindspore import PrecisionDebugger和模型初始化之间的任意位置添加该接口。 +PrecisionDebugger可以在from msprobe.mindspore import PrecisionDebugger之后的位置添加。详细使用可参考“**示例代码**”。 **原型** @@ -24,7 +24,7 @@ PrecisionDebugger(config_path=None) | 参数名 | 说明 | 是否必选 | | ----------- | ------------------------------------------------------------ | -------- | -| config_path | 指定dump配置文件路径，String类型。参数示例："./config.json"。未配置该路径时，默认使用[config.json](../../config)文件的默认配置。config.json文件可以配置更多参数，若需要进行更多场景的精度数据dump，建议配置[config.json](../../config/config.json)文件。 | 否 | +| config_path | 指定dump配置文件路径，String类型。参数示例："./config.json"。未配置该路径时，默认使用[config.json](../../config)文件的默认配置。config.json文件可以配置更多参数，若需要进行更多场景的精度数据dump，建议配置[config.json](../../config/config.json)文件。config.json文件的配置可参考《[配置文件说明](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/config/README.md)》。 | 否 | ### start函数 @@ -32,6 +32,8 @@ PrecisionDebugger(config_path=None) 启动函数。 +在模型初始化之后的位置添加。需要与stop函数一起添加在for循环内。 + **原型** ```Python @@ -40,8 +42,46 @@ debugger.start() 该函数为类函数，可以使用debugger.start()也可以使用PrecisionDebugger.start()。 +### stop函数 + +**功能说明** + +dump停止函数。 + +在**start**函数之后的任意位置添加。需要与start函数一起添加在for循环内。若需要dump反向数据，则需要添加在反向计算代码之后。 + +仅MindSpore动态图场景支持。 + +**原型** + +```Python +debugger.stop() +``` + +该函数为类函数，可以使用debugger.stop()也可以使用PrecisionDebugger.stop()。 + +### step函数 + +**功能说明** + +结束标识。 + +在最后一个**stop**函数后或一个step结束的位置添加。 + +仅MindSpore动态图场景支持。 + +**原型** + +```Python +debugger.step() +``` + +该函数为类函数，可以使用debugger.step()也可以使用PrecisionDebugger.step()。 + ## 示例代码 +### MindSpore静态图场景 + ```Python from msprobe.mindspore import PrecisionDebugger debugger = PrecisionDebugger(config_path="./config.json") @@ -51,15 +91,98 @@ debugger.start() ... ``` +### MindSpore动态图场景 + +```Python +import mindspore as ms +from msprobe.mindspore import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json") + +# 模型、损失函数的定义以及初始化等操作 +# ... + +# 数据集迭代的地方往往是模型开始训练的地方 +for data, label in data_loader: + debugger.start() # 开启数据dump + net = Model() + # 如下是模型每个step执行的逻辑 + grad_net = ms.grad(net)(data) + # ... + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` + ## dump结果文件介绍 +### MindSpore静态图场景 + 训练结束后，工具将dump的数据保存在dump_path参数指定的目录下。 -- level为L1时 +- jit_level为O0/O1时 dump结果目录请参见MindSpore官网中的《[同步Dump数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%90%8C%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)》。 -- level为L2时 +- jit_level为O2时 dump结果目录请参见MindSpore官网中的《[异步Dump数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%BC%82%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)》。 +jit_level请参见[mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig)配置jit_config。 + +### MindSpore动态图场景 + +训练结束后，工具将dump的数据保存在dump_path参数指定的目录下。 + +dump结果目录结构示例如下： + +```bash +├── dump_path +│ ├── step0 +│ | ├── rank0 +│ | │ ├── dump_tensor_data +| | | | ├── MintFunctional.relu.0.backward.input.0.npy +| | | | ├── Mint.abs.0.forward.input.0.npy +| | | | ├── Functional.split.0.forward.input.0.npy +| | | | ... +| | | | └── Tensor.__add__.0.forward.output.0.npy +│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） +│ | | ├── stack.json # 算子调用栈信息 +│ | | └── construct.json # 分层分级结构，level为L1时，construct.json内容为空 +│ | ├── rank1 +| | | ├── dump_tensor_data +| | | | └── ... +│ | | ├── dump.json +│ | | ├── stack.json +| | | └── construct.json +│ | ├── ... +│ | | +| | └── rank7 +│ ├── step1 +│ | ├── ... +│ ├── step2 +``` + +dump过程中，npy文件在对应算子或者模块被执行后就会落盘，而json文件则需要在正常执行PrecisionDebugger.stop()后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关npy文件，可能会导致json文件中数据丢失。 + +其中rank为设备上各卡的ID，每张卡上dump的数据会生成对应dump目录。非分布式场景下没有rank ID，目录名称为rank。 + +npy文件保存的前缀和MindSpore对应关系如下： + +| 前缀 | MindSpore模块 | +| -------------- | ---------------------------- | +| Tensor | mindspore.Tensor | +| Functional | mindspore.ops | +| Mint | mindspore.mint | +| MintFunctional | mindspore.mint.nn.functional | + +## 工具支持的API列表 + +msprobe工具维护固定的API支持列表，若需要删除或增加dump的API，可以在msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml文件内手动修改，如下示例： + +```bash +ops: # ops为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d +``` diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md index 7d0763b68..4bed0d03b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md @@ -158,7 +158,7 @@ dump结果目录结构示例如下： | | | | ├── MyModule.0.forward.input.pt # 开启模块级精度数据dump时存在模块级的dump数据文件 | | | | ... | | | | └── Fcuntion.linear.5.backward.output.pt -│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示2范数（平方根） +│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） │ | | ├── stack.json # 算子调用栈信息 │ | | └── construct.json # 分层分级结构 │ | ├── rank1 @@ -175,9 +175,9 @@ dump结果目录结构示例如下： │ ├── step2 ``` -dump过程中，pt文件在对应算子或者模块被执行后就会落盘，而json文件则需要在正常执行PrecisionDebugger.stop()后才会被落盘保存，异常的程序终止会保存终止前被执行算子的相关pt文件，但是不会生成json文件。 +dump过程中，pt文件在对应算子或者模块被执行后就会落盘，而json文件则需要在正常执行PrecisionDebugger.stop()后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关npy文件，可能会导致json文件中数据丢失。 -其中rank为设备上各卡的ID，每张卡上dump的数据会生成对应dump目录。 +其中rank为设备上各卡的ID，每张卡上dump的数据会生成对应dump目录。非分布式场景下没有rank ID，目录名称为rank。 pt文件保存的前缀和PyTorch对应关系如下： -- Gitee From b969d25d4c5f12f254472cfeca4e932b4efa44e8 Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 17:33:51 +0800 Subject: [PATCH 115/791] 82 --- .../compare_bean/test_profiling_info.py | 112 +++++++++--------- 1 file changed, 56 insertions(+), 56 deletions(-) diff --git a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py index e6d543a77..01cd0104d 100644 --- a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py +++ b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py @@ -34,62 +34,62 @@ class TestProfilingInfo(unittest.TestCase): info.calculate_schedule_time() self.assertEqual(info.scheduling_time, 2) - def test_update_fa_fwd_info(self): - info = ProfilingInfo("NPU") - info.fa_time_fwd_cube = 5 - info.fa_time_fwd_vector = 5 - info.fa_num_fwd_cube = 1 - info.fa_num_fwd_vector = 1 - info.calculate_fa_time_fwd() - info.calculate_fa_num_fwd() - self.assertEqual(info.fa_time_fwd, 10) - self.assertEqual(info.fa_num_fwd, 2) - - def test_update_fa_bwd_info(self): - info = ProfilingInfo("NPU") - info.fa_time_bwd_cube = 5 - info.fa_time_bwd_vector = 5 - info.fa_num_bwd_cube = 1 - info.fa_num_bwd_vector = 1 - info.calculate_fa_time_bwd() - info.calculate_fa_num_bwd() - self.assertEqual(info.fa_time_bwd, 10) - self.assertEqual(info.fa_num_bwd, 2) - - def test_update_sdma_info(self): - info = ProfilingInfo("NPU") - info.sdma_time_tensor_move = 5 - info.sdma_time_stream = 5 - info.sdma_num_tensor_move = 5 - info.sdma_num_stream = 5 - info.calculate_sdma_time() - info.calculate_sdma_num() - self.assertEqual(info.sdma_time, 10) - self.assertEqual(info.sdma_num, 10) - - def test_update_cube_info(self): - info = ProfilingInfo("NPU") - info.matmul_time_cube = 1 - info.matmul_time_vector = 1 - info.other_cube_time = 1 - info.matmul_num_cube = 5 - info.matmul_num_vector = 5 - info.other_cube_num = 5 - info.calculate_cube_time() - info.calculate_cube_num() - self.assertEqual(info.cube_time, 3) - self.assertEqual(info.cube_num, 15) - - def test_update_vec_info(self): - info = ProfilingInfo("NPU") - info.vector_time_trans = 1 - info.vector_time_notrans = 1 - info.vector_num_trans = 2 - info.vector_num_notrans = 2 - info.calculate_vec_time() - info.calculate_vec_num() - self.assertEqual(info.vec_time, 2) - self.assertEqual(info.vec_num, 4) + # def test_update_fa_fwd_info(self): + # info = ProfilingInfo("NPU") + # info.fa_time_fwd_cube = 5 + # info.fa_time_fwd_vector = 5 + # info.fa_num_fwd_cube = 1 + # info.fa_num_fwd_vector = 1 + # info.calculate_fa_time_fwd() + # info.calculate_fa_num_fwd() + # self.assertEqual(info.fa_time_fwd, 10) + # self.assertEqual(info.fa_num_fwd, 2) + # + # def test_update_fa_bwd_info(self): + # info = ProfilingInfo("NPU") + # info.fa_time_bwd_cube = 5 + # info.fa_time_bwd_vector = 5 + # info.fa_num_bwd_cube = 1 + # info.fa_num_bwd_vector = 1 + # info.calculate_fa_time_bwd() + # info.calculate_fa_num_bwd() + # self.assertEqual(info.fa_time_bwd, 10) + # self.assertEqual(info.fa_num_bwd, 2) + # + # def test_update_sdma_info(self): + # info = ProfilingInfo("NPU") + # info.sdma_time_tensor_move = 5 + # info.sdma_time_stream = 5 + # info.sdma_num_tensor_move = 5 + # info.sdma_num_stream = 5 + # info.calculate_sdma_time() + # info.calculate_sdma_num() + # self.assertEqual(info.sdma_time, 10) + # self.assertEqual(info.sdma_num, 10) + # + # def test_update_cube_info(self): + # info = ProfilingInfo("NPU") + # info.matmul_time_cube = 1 + # info.matmul_time_vector = 1 + # info.other_cube_time = 1 + # info.matmul_num_cube = 5 + # info.matmul_num_vector = 5 + # info.other_cube_num = 5 + # info.calculate_cube_time() + # info.calculate_cube_num() + # self.assertEqual(info.cube_time, 3) + # self.assertEqual(info.cube_num, 15) + # + # def test_update_vec_info(self): + # info = ProfilingInfo("NPU") + # info.vector_time_trans = 1 + # info.vector_time_notrans = 1 + # info.vector_num_trans = 2 + # info.vector_num_notrans = 2 + # info.calculate_vec_time() + # info.calculate_vec_num() + # self.assertEqual(info.vec_time, 2) + # self.assertEqual(info.vec_num, 4) def test_set_compute_time(self): info = ProfilingInfo("NPU") -- Gitee From 15ad414a912ba929f150c6fa92565f1df5d1215c Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 17:51:56 +0800 Subject: [PATCH 116/791] 82 --- .../profiling_parser/gpu_profiling_parser.py | 2 - .../compare_bean/test_profiling_info.py | 112 +++++++++--------- 2 files changed, 56 insertions(+), 58 deletions(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 4d4734a4c..bf5d39846 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -58,8 +58,6 @@ class GPUProfilingParser(BaseProfilingParser): for record in addr_dict.values(): self._result_data.update_memory_list(record) - gpu - def _update_overall_metrics(self): self._calculate_performance_time() self.__parse_memory_reserved() diff --git a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py index 01cd0104d..e6d543a77 100644 --- a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py +++ b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py @@ -34,62 +34,62 @@ class TestProfilingInfo(unittest.TestCase): info.calculate_schedule_time() self.assertEqual(info.scheduling_time, 2) - # def test_update_fa_fwd_info(self): - # info = ProfilingInfo("NPU") - # info.fa_time_fwd_cube = 5 - # info.fa_time_fwd_vector = 5 - # info.fa_num_fwd_cube = 1 - # info.fa_num_fwd_vector = 1 - # info.calculate_fa_time_fwd() - # info.calculate_fa_num_fwd() - # self.assertEqual(info.fa_time_fwd, 10) - # self.assertEqual(info.fa_num_fwd, 2) - # - # def test_update_fa_bwd_info(self): - # info = ProfilingInfo("NPU") - # info.fa_time_bwd_cube = 5 - # info.fa_time_bwd_vector = 5 - # info.fa_num_bwd_cube = 1 - # info.fa_num_bwd_vector = 1 - # info.calculate_fa_time_bwd() - # info.calculate_fa_num_bwd() - # self.assertEqual(info.fa_time_bwd, 10) - # self.assertEqual(info.fa_num_bwd, 2) - # - # def test_update_sdma_info(self): - # info = ProfilingInfo("NPU") - # info.sdma_time_tensor_move = 5 - # info.sdma_time_stream = 5 - # info.sdma_num_tensor_move = 5 - # info.sdma_num_stream = 5 - # info.calculate_sdma_time() - # info.calculate_sdma_num() - # self.assertEqual(info.sdma_time, 10) - # self.assertEqual(info.sdma_num, 10) - # - # def test_update_cube_info(self): - # info = ProfilingInfo("NPU") - # info.matmul_time_cube = 1 - # info.matmul_time_vector = 1 - # info.other_cube_time = 1 - # info.matmul_num_cube = 5 - # info.matmul_num_vector = 5 - # info.other_cube_num = 5 - # info.calculate_cube_time() - # info.calculate_cube_num() - # self.assertEqual(info.cube_time, 3) - # self.assertEqual(info.cube_num, 15) - # - # def test_update_vec_info(self): - # info = ProfilingInfo("NPU") - # info.vector_time_trans = 1 - # info.vector_time_notrans = 1 - # info.vector_num_trans = 2 - # info.vector_num_notrans = 2 - # info.calculate_vec_time() - # info.calculate_vec_num() - # self.assertEqual(info.vec_time, 2) - # self.assertEqual(info.vec_num, 4) + def test_update_fa_fwd_info(self): + info = ProfilingInfo("NPU") + info.fa_time_fwd_cube = 5 + info.fa_time_fwd_vector = 5 + info.fa_num_fwd_cube = 1 + info.fa_num_fwd_vector = 1 + info.calculate_fa_time_fwd() + info.calculate_fa_num_fwd() + self.assertEqual(info.fa_time_fwd, 10) + self.assertEqual(info.fa_num_fwd, 2) + + def test_update_fa_bwd_info(self): + info = ProfilingInfo("NPU") + info.fa_time_bwd_cube = 5 + info.fa_time_bwd_vector = 5 + info.fa_num_bwd_cube = 1 + info.fa_num_bwd_vector = 1 + info.calculate_fa_time_bwd() + info.calculate_fa_num_bwd() + self.assertEqual(info.fa_time_bwd, 10) + self.assertEqual(info.fa_num_bwd, 2) + + def test_update_sdma_info(self): + info = ProfilingInfo("NPU") + info.sdma_time_tensor_move = 5 + info.sdma_time_stream = 5 + info.sdma_num_tensor_move = 5 + info.sdma_num_stream = 5 + info.calculate_sdma_time() + info.calculate_sdma_num() + self.assertEqual(info.sdma_time, 10) + self.assertEqual(info.sdma_num, 10) + + def test_update_cube_info(self): + info = ProfilingInfo("NPU") + info.matmul_time_cube = 1 + info.matmul_time_vector = 1 + info.other_cube_time = 1 + info.matmul_num_cube = 5 + info.matmul_num_vector = 5 + info.other_cube_num = 5 + info.calculate_cube_time() + info.calculate_cube_num() + self.assertEqual(info.cube_time, 3) + self.assertEqual(info.cube_num, 15) + + def test_update_vec_info(self): + info = ProfilingInfo("NPU") + info.vector_time_trans = 1 + info.vector_time_notrans = 1 + info.vector_num_trans = 2 + info.vector_num_notrans = 2 + info.calculate_vec_time() + info.calculate_vec_num() + self.assertEqual(info.vec_time, 2) + self.assertEqual(info.vec_num, 4) def test_set_compute_time(self): info = ProfilingInfo("NPU") -- Gitee From 4c01fc9b4c1f82ef7a423bc49866cbdad44c0367 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 1 Aug 2024 20:40:46 +0800 Subject: [PATCH 117/791] add tensor_transport_layer for online api_accuracy_checker --- .../msprobe/core/common/const.py | 3 + .../api_accuracy_checker/common/config.py | 10 +- .../pytorch/api_accuracy_checker/config.yaml | 7 +- .../tensor_transport_layer/__init__.py | 0 .../tensor_transport_layer/attl.py | 187 +++++++++++ .../tensor_transport_layer/client.py | 310 ++++++++++++++++++ .../tensor_transport_layer/device_dispatch.py | 113 +++++++ .../tensor_transport_layer/server.py | 204 ++++++++++++ .../msprobe/pytorch/pt_config.py | 15 +- 9 files changed, 846 insertions(+), 3 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index df82455a6..119ad7d62 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -20,6 +20,9 @@ class Const: DEFAULT_PATH = './' WHITE_LIST = 'white_list' BLACK_LIST = 'black_list' + IS_ONLINE = False + NFS_PATH = "" + IS_BENCHMARK_DEVICE = True # dump mode ALL = "all" diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index 760e7c862..3c61624b6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -24,7 +24,13 @@ class Config: 'white_list': list, 'black_list': list, 'error_data_path': str, - 'precision': int + 'precision': int, + 'is_online': bool, + 'nfs_path': str, + 'is_benchmark_device': bool, + 'host': str, + 'port': int, + 'rank_list': list } if key not in validators: raise ValueError(f"{key} must be one of {validators.keys()}") @@ -38,6 +44,8 @@ class Config: RunUTConfig.check_filter_list_config(key, value) if key == 'error_data_path': RunUTConfig.check_error_data_path_config(value) + if key == 'nfs_path': + RunUTConfig.check_nfs_path_config(value) return value diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml index 2dac535dc..c2bb847b7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml @@ -2,4 +2,9 @@ white_list: [] black_list: [] error_data_path: './' precision: 14 - \ No newline at end of file +is_online: False +nfs_path: "" +is_benchmark_device: True +host: "" +port: -1 +rank_list: [0] diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py new file mode 100644 index 000000000..c4d5b76c5 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -0,0 +1,187 @@ +import io +import os.path +import time +import re +from pathlib import Path +from multiprocessing import Queue +from typing import Optional, Union, Dict, Any +from collections import namedtuple +from dataclasses import dataclass + +import torch + +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.client import TCPClient +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server import TCPServer +from msprobe.pytorch.common.utils import logger +from msprobe.core.common.utils import remove_path + + +ApiData = namedtuple('ApiData', ['name', 'args', 'kwargs', 'result', 'step', 'rank'], + defaults=['unknown', None, None, None, 0, 0]) +BufferType = Union[ApiData, Dict[str, Any], str] # Union[Tensor, Tuple[Optional[Tensor]]] + + +@dataclass +class ATTLConfig: + is_benchmark_device: bool + connect_ip: str + connect_port: int + # storage_config + nfs_path: str = None + check_sum: bool = True + queue_size: int = 50 + + +class ATTL: + def __init__(self, session_id: str, session_config: ATTLConfig, need_dump=True) -> None: + self.session_id = session_id + self.session_config = session_config + self.logger = logger + self.socket_manager = None + self.data_queue = Queue(maxsize=50) + self.dequeue_list = [] + self.message_end = False + self.kill_progress = False + self.check_attl_config() + if self.session_config.nfs_path: + self.nfs_path = Path(self.session_config.nfs_path) + elif self.session_config.is_benchmark_device: + + self.socket_manager = TCPServer(self.session_config.connect_port, + self.data_queue, + self.session_config.check_sum) + self.socket_manager.start() + elif need_dump: + self.socket_manager = TCPClient(self.session_config.connect_ip, + self.session_config.connect_port, + self.session_config.check_sum) + self.socket_manager.start() + + def check_attl_config(self): + if self.session_config.nfs_path: + if os.path.exists(self.session_config.nfs_path): + return + else: + raise Exception(f"nfs path {self.session_config.nfs_path} doesn't exists.") + ipv4_pattern = "([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])(\.([1-9]?\d|1\d{2}|2[0-4]\d|25[0-5])){3}$" + if not re.match(ipv4_pattern, self.session_config.connect_ip): + raise Exception(f"host {self.session_config.connect_ip} is invalid.") + if not (0 < self.session_config.connect_port <= 65535): + raise Exception(f"port {self.session_config.connect_port} is invalid.") + + def stop_serve(self): + if isinstance(self.socket_manager, TCPServer): + self.socket_manager.stop() + + def send(self, buffer: BufferType) -> None: + """ + npu major in 'send' (client) + """ + # know receiver receive and go next + if isinstance(buffer, ApiData): + buffer = move2target_device(buffer, torch.device('cpu')) + + if 'device' in buffer.kwargs: + buffer.kwargs.pop('device') + rank = buffer.rank if hasattr(buffer, "rank") else 0 + step = buffer.step if hasattr(buffer, "step") else 0 + io_buff = io.BytesIO() + torch.save(buffer, io_buff) + data = io_buff.getvalue() + self.socket_manager.add_to_sending_queue(data, rank=rank, step=step) + + def recv(self, timeout_ms=0) -> Optional[BufferType]: + buffer = None + while buffer is None: + if timeout_ms > 0: + time.sleep(timeout_ms / 1000.0) + if buffer is None and not self.data_queue.empty(): + buffer = self.data_queue.get() + break + if buffer is None and timeout_ms > 0: # timeout is the only case we give up and return None + break + if self.message_end and self.data_queue.empty(): + buffer = b"KILL_CONFIRM" + self.kill_progress = True + break + time.sleep(0.1) # waiting outside the lock before next attempt + if buffer is None: + # this is a result of a timeout + self.logger.info(f"RECEIVE API DATA TIMED OUT") + else: + if buffer == b"STOP_": + return "STOP_" + if buffer == b"KILL_": + self.message_end = True + return "STOP_" + if buffer == b"KILL_CONFIRM": + self.kill_progress = True + return "KILL_" + buffer = io.BytesIO(buffer) + try: + buffer = torch.load(buffer, map_location="cpu") + except Exception as e: + self.logger.error("there is something error. please check it. %s", e) + if isinstance(buffer, bytes): + return None + if isinstance(buffer, str): + return buffer + + return buffer + + def upload(self, buffer: BufferType): + if isinstance(buffer, ApiData): + buffer = move2target_device(buffer, torch.device('cpu')) + file_path = os.path.join(self.session_config.nfs_path, buffer.name + ".pt") + else: + file_path = os.path.join(self.session_config.nfs_path, buffer + f"_{int(time.time())}") + + torch.save(buffer, file_path) + + def download(self): + for file_type in ("start*", "*.pt", "end*"): + cur_file = next(self.nfs_path.glob(file_type), None) + if cur_file is not None: + break + + if cur_file is None: + return None + else: + buffer = torch.load(cur_file) + remove_path(cur_file) + return buffer + + +def move2device_exec(obj, device): + if isinstance(obj, (tuple, list)): + data_list = [move2device_exec(val, device) for val in obj] + return data_list if isinstance(obj, list) else tuple(data_list) + if isinstance(obj, dict): + return {key: move2device_exec(val, device) for key, val in obj.items()} + elif isinstance(obj, torch.Tensor): + obj = obj.detach() + if obj.device.type != device: + obj = obj.to(device) + return obj + elif "return_types" in str(type(obj)): + return move2device_exec(tuple(obj), device) + elif isinstance(obj, torch._C.device): + return torch.device(device) + else: + return obj + + +def move2target_device(buffer: ApiData, target_device): + # handle args + new_args = move2device_exec(buffer.args, target_device) + + # handle kwargs + new_kwargs = move2device_exec(buffer.kwargs, target_device) + + # handle result + new_results = move2device_exec(buffer.result, target_device) + + if target_device == torch.device('cpu') or target_device == "cpu": + return ApiData(buffer.name, tuple(new_args), new_kwargs, new_results, buffer.step, buffer.rank) + else: + return ApiData(buffer.name, tuple(new_args), new_kwargs, buffer.result, buffer.step, buffer.rank) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py new file mode 100644 index 000000000..5a436915c --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py @@ -0,0 +1,310 @@ +import hashlib +import io +import struct +import time +import os +import signal +import sys +from queue import Queue +from threading import Thread +from typing import Union + +from twisted.internet import reactor, protocol, endpoints +from twisted.protocols.basic import FileSender + +from msprobe.pytorch.common.utils import logger + + +class TCPDataItem: + def __init__(self, data, + sequence_number: int, + rank: int = 0, + step: int = 0): + self.raw_data = data + self.sequence_number = sequence_number + self.rank = rank + self.step = step + self.retry_times = 0 + self.pending_time = 0 + self.busy_time = 0 + + +class TCPClient: + MAX_SENDING_QUEUE_SIZE = 20 + ACK_SUCCESS = b"OK___" + ACK_ERROR = b"ERROR" + ACK_BUSY = b"BUSY_" + ACK_STOP = b"STOP_" + ACK_STOP_CONFIRM = b"OVER_" + ACK_KILL_PROCESS = b"KILL_" + + QUEUE_PENDING_TIME = 600 # 队列10分钟都处于阻塞状态，则终止sending进程 + RESEND_RETRY_TIMES = 2 # 最大重传数 + RESEND_TIMER_TIME = 5 # 接收ACK超时定时器 + RESEND_PENDING_TIME = 60 # 连续pending时间超过1分钟则放弃该数据 + + def __init__(self, host="localhost", port=8000, check_sum=False): + self.send_queue = Queue(self.MAX_SENDING_QUEUE_SIZE) + self.resend_dict = dict() + self.host = host + self.port = port + self.factory = None + self.sequence_number = 0 + self.signal_exit = False + self.tcp_manager = ClientProtocol(ack_queue_size=100, + chunk_size=655360, + check_sum=check_sum) + self.send_thread = Thread(target=self._sending_queue_data) + self.send_thread.setDaemon(True) + self.send_thread.start() + self.destroy_thread = Thread(target=self._destroy_queue_data) + self.destroy_thread.setDaemon(True) + self.destroy_thread.start() + + @staticmethod + def run_reactor(): + reactor.run(installSignalHandlers=False) + + def start(self): + def conn_callback(cur_protocol): + if cur_protocol.transport and cur_protocol.transport.getPeer().host == self.host: + logger.debug(f"Process: {os.getpid()} connects to server successfully.") + else: + logger.warning(f"Process: {os.getpid()} fails to connect to server. ") + raise ConnectionError(f"Failed to connect to {self.host}.") + + def conn_err_callback(failure): + self.signal_exit = True + time.sleep(1) + reactor.stop() + logger.error(f"Failed to connected {self.host} {self.port}. Reason is {failure.getErrorMessage()}") + os.kill(os.getpid(), signal.SIGKILL) + os.kill(os.getppid(), signal.SIGKILL) + + def cur_protocol(): + return self.tcp_manager + + self.factory = MessageClientFactory() + self.factory.protocol = cur_protocol + + endpoint = endpoints.TCP4ClientEndpoint(reactor, self.host, self.port) + d = endpoint.connect(self.factory) + d.addCallback(conn_callback) + d.addErrback(conn_err_callback) + + reactor_thread = Thread(target=self.run_reactor, daemon=True) + reactor_thread.start() + + def send_after_queue_empty(self, data): + while not self._ready_to_exit(): + self.add_to_sending_queue(data) + time.sleep(2) + + def check_client_alive(self): + return self.factory.num_connections > 0 + + def stop(self): + self.tcp_manager.connection_timeout() + + def send_stop_signal(self): + self.send_after_queue_empty(self.ACK_STOP) + while not self._ready_to_exit(): + if not self.check_client_alive(): + break + time.sleep(1) + while not self.tcp_manager.kill_process: + time.sleep(1) + + def add_to_sending_queue(self, data: Union[bytes, TCPDataItem], rank: int = 0, step: int = 0): + if self._ready_to_exit(): + return + + send_data = data + if not isinstance(data, TCPDataItem): + send_data = TCPDataItem(data=data, + sequence_number=self.sequence_number, + rank=rank, + step=step) + self.sequence_number += 1 + + self.send_queue.put(send_data, block=True, timeout=self.QUEUE_PENDING_TIME) + + def _send_data(self, data: TCPDataItem): + self.tcp_manager.send_wrapped_data(data.raw_data, + sequence_number=data.sequence_number, + rank=data.rank, + step=data.step + ) + + def _sending_queue_data(self): + while True: + if not self.tcp_manager.is_connected: + continue + + while self.send_queue.qsize() > 0: + if self._ready_to_exit(): + break + if len(self.resend_dict) < self.MAX_SENDING_QUEUE_SIZE: + data_obj = self.send_queue.get() + self._send_data(data_obj) + resend_key = str(data_obj.sequence_number) + "_" + str(data_obj.rank) + "_" + str(data_obj.step) + if resend_key not in self.resend_dict.keys(): + # Send data for the first time + self.resend_dict[resend_key] = data_obj + else: + time.sleep(0.1) + + if self._ready_to_exit(): + logger.debug("Successfully close sending process.") + break + time.sleep(0.1) + + def _destroy_queue_data(self): + while True: + if self._ready_to_exit(): + break + + while len(self.resend_dict) > 0 and self.tcp_manager.ack_queue.qsize() > 0: + ack_info, seq_number, rank, step = self.tcp_manager.ack_queue.get() + obj_key = str(seq_number) + "_" + str(rank) + "_" + str(step) + current_item = self.resend_dict.get(obj_key) + + if current_item is None: + continue + + if ack_info == self.ACK_SUCCESS: + self.resend_dict.pop(obj_key) + elif ack_info == self.ACK_BUSY: + logger.debug("RECV BUSY ACK") + if current_item.busy_time > 5: + self._resend_data(current_item) + else: + current_item.busy_time += 1 + elif ack_info == self.ACK_ERROR: + logger.debug("RECV ERROR ACK") + self._resend_data(current_item) + elif ack_info == self.ACK_STOP_CONFIRM: + logger.debug("RECV STOP ACK") + self.factory.num_connections -= 1 + + break + + time.sleep(0.1) + + def _resend_data(self, data: TCPDataItem): + if data.retry_times < self.RESEND_RETRY_TIMES: + data.retry_times += 1 + logger.debug(f"Resend data seq number: {data.sequence_number}") + self.add_to_sending_queue(data) + else: + self.resend_dict.pop(data.sequence_number) + logger.debug(f"SKIP send sequence number {data.sequence_number} after retry {data.retry_times} times!") + + def _pending_data(self, data: TCPDataItem): + if data.pending_time >= self.RESEND_PENDING_TIME: + self.resend_dict.pop(data.sequence_number) + logger.debug(f"SKIP send sequence number {data.sequence_number} after pending {data.pending_time} times!") + return + + # wait time is 100MB per second + pending_time = max(1, len(data.raw_data) // (2 ** 20 * 50)) + data.pending_time += pending_time + time.sleep(pending_time) + + def _ready_to_exit(self): + return self.signal_exit or self.tcp_manager.signal_exit + + +class ClientProtocol(protocol.Protocol): + TIMEOUT = 60 * 10 + + def __init__(self, ack_queue_size=100, chunk_size=65536, check_sum=False): + self.buffer = io.BytesIO() + self.is_connected = False + self.check_sum = check_sum + self.tell = 0 + self.ack_queue = Queue(maxsize=ack_queue_size) + self.file_sender = FileSender() + self.file_sender.CHUNK_SIZE = chunk_size + self.signal_exit = False + self.defer = None + self.kill_process = False + + def dataReceived(self, data): + if self.timeout_call.active(): + self.timeout_call.reset(self.TIMEOUT) + + self.buffer.seek(0, 2) + self.buffer.write(data) + self.buffer.seek(self.tell) + while True: + if len(self.buffer.getvalue()) >= 29: # 5 + 8 * 3 + ack = self.buffer.read(5) + seq_number = struct.unpack('!Q', self.buffer.read(8))[0] + rank = struct.unpack('!Q', self.buffer.read(8))[0] + step = struct.unpack('!Q', self.buffer.read(8))[0] + if ack == b"KILL_": + self.kill_process = True + logger.debug(f"接收到KILL信号, PID {os.getpid()}") + if ack == b"OVER_": + self.factory.num_connections -= 1 + self.tell += 29 + if not self.ack_queue.full(): + self.ack_queue.put((ack, seq_number, rank, step)) + self.buffer = io.BytesIO(self.buffer.getvalue()[self.tell:]) + self.tell = 0 + else: + time.sleep(0.1) + else: + break + + def send_wrapped_data(self, data, sequence_number: int = 0, rank: int = 0, step: int = 0): + length = len(data) + md5_hash = hashlib.md5(data).hexdigest() if self.check_sum else "" + while True: + if self.defer is None or self.defer.called: + self.defer = self.send_large_data( + length.to_bytes(8, byteorder='big') + + sequence_number.to_bytes(8, byteorder='big') + + rank.to_bytes(8, byteorder='big') + + step.to_bytes(8, byteorder='big') + + md5_hash.encode() + + data) + break + time.sleep(0.01) + + def send_large_data(self, data): + d = self.file_sender.beginFileTransfer(io.BytesIO(data), self.transport) + return d + + def connection_timeout(self): + if self.factory.num_connections <= 0: + return + + self.factory.num_connections -= 1 + logger.debug(f"超时退出{self.transport.addr}, PID {os.getpid()}") + self.transport.loseConnection() + + def connectionMade(self): + self.timeout_call = reactor.callLater(self.TIMEOUT, self.connection_timeout) + self.is_connected = True + self.factory.num_connections += 1 + logger.info("successfully connect server") + + def connectionLost(self, reason): + self.signal_exit = True + self.factory.num_connections -= 1 + logger.info("Lost connection with server") + + +class MessageClientFactory(protocol.ClientFactory): + def __init__(self): + self.num_connections = 0 + + def clientConnectionFailed(self, connector, reason): + logger.info(f"Fail to connection with server: {reason.getErrorMessage()}") + reactor.stop() + + def clientConnectionLost(self, connector, reason): + logger.info(f"Client lost connection with server: {reason.getErrorMessage()}") + reactor.stop() diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py new file mode 100644 index 000000000..cbc1b76fd --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -0,0 +1,113 @@ +import time + +import torch +import torch.multiprocessing as mp + +from msprobe.core.common.const import Const +from msprobe.pytorch.common.utils import logger +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import move2target_device + + +def run_ut_process(xpu_id, compare, consumer_queue, func, config): + """ When consumer_queue(shared with ConsumerDispatcher) is not empty, consume api data from consumer_queue. + :param xpu_id: int + :param compare: instance of Comparator + :param consumer_queue: shared queues of ConsumerDispatcher + :param func: run_touch_api_online + :param config: run_ut_config + :return: + """ + device = torch.device(f'cuda:{xpu_id}') + + while True: + if consumer_queue.empty(): + time.sleep(0.1) + continue + + api_data = consumer_queue.get() + if api_data == "KILL_": + # current consumer finish + return + + api_full_name = api_data.name + api_data = move2target_device(api_data, device) + try: + data_info = func(api_full_name, api_data, config.backward_content) + logger.debug(f"success exec in device {api_full_name}") + is_fwd_success, is_bwd_success = compare.compare_output(api_full_name, data_info) + logger.info(f"running api_full_name {api_full_name} ut, " + f"is_fwd_success: {is_fwd_success}, " + f"is_bwd_success: {is_bwd_success}") + except Exception as err: + [_, api_name, _] = api_full_name.split(Const.SEP) + if "expected scalar type Long" in str(err): + logger.warning(f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API " + f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.") + else: + logger.error(f"Run {api_full_name} UT Error: {str(err)}") + + compare.write_summary_csv((api_full_name, "SKIP", "SKIP", str(err), api_data.rank)) + + finally: + torch.cuda.empty_cache() + + +class ConsumerDispatcher: + def __init__(self, compare, capacity=10, num_workers=8, device: str = "gpu") -> None: + self.num_workers = num_workers + self.capacity = capacity + self.compare = compare + self.queues = [] + self.processes = [] + self.reverse_sort = False + self.pool = None + self.device = device + self.data_id = 0 + self.lock = mp.Lock() + self.result_queue = mp.Queue() + mp.set_start_method("spawn", force=True) + + def start(self, handle_func, config): + self.queues = [mp.Queue(maxsize=self.capacity) for _ in range(self.num_workers)] + for xpu_id, q in enumerate(self.queues): + p = mp.Process(name="run_ut_process", target=run_ut_process, + args=(xpu_id, self.compare, q, handle_func, config)) + + p.start() + self.processes.append(p) + logger.info("Successfully start unittest process.") + + def stop(self): + for q in self.queues: + while q.full(): + time.sleep(0.1) + q.put("KILL_") + + for p in self.processes: + p.join() + logger.info("Successfully stop unittest process.") + + def update_consume_queue(self, api_data): + while True: + index = self._choose_max_empty_site_strategy() + if index != -1: + q = self.queues[index] + q.put(api_data) + logger.debug(f"将{api_data.name}调度给第{index}个GPU") + break + logger.debug("所有的UT队列都已满, 阻塞中") + time.sleep(0.1) + + def _choose_max_empty_site_strategy(self): + maximum = 0 + index = -1 + # 充分利用多卡资源，防止任务过多分配给前面的卡 + _reverse = 1 if not self.reverse_sort else -1 + for i, q in enumerate(self.queues[::_reverse]): + empty_site = self.capacity - q.qsize() + if empty_site > maximum: + maximum = empty_site + index = i + index = len(self.queues) - index - 1 if index != -1 and self.reverse_sort else index + self.reverse_sort = not self.reverse_sort + return index diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py new file mode 100644 index 000000000..6dba19056 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py @@ -0,0 +1,204 @@ +import struct +import hashlib +import time +import io + +from threading import Thread +from twisted.internet import reactor, protocol, endpoints + +from msprobe.pytorch.common.utils import logger + + +class TCPServer: + def __init__(self, port, shared_queue, check_sum=False) -> None: + self.port = port + self.shared_queue = shared_queue + self.check_sum = check_sum + self.factory = MessageServerFactory() + self.reactor_thread = None + + @staticmethod + def run_reactor(): + reactor.run(installSignalHandlers=False) + + def start(self): + self.factory.protocol = self.build_protocol + endpoint = endpoints.TCP4ServerEndpoint(reactor, self.port) + endpoint.listen(self.factory) + self.reactor_thread = Thread(target=self.run_reactor, daemon=True) + self.reactor_thread.start() + + def is_running(self): + return not self.factory.is_all_connection_closed() + + def stop(self): + self.factory.doStop() + reactor.callFromThread(reactor.sigInt, 2) + self.reactor_thread.join() + + def build_protocol(self): + return ServerProtocol(self.shared_queue, self.check_sum) + + +class ServerProtocol(protocol.Protocol): + ACK_SUCCESS = b"OK___" + ACK_ERROR = b"ERROR" + ACK_BUSY = b"BUSY_" + ACK_STOP = b"STOP_" + ACK_STOP_CONFIRM = b"OVER_" + ACK_KILL_PROCESS = b"KILL_" + + def __init__(self, shared_queue, check_sum=False): + self.start_time = None + self.buffer = io.BytesIO() + self.consumer_queue = shared_queue + self.check_sum = check_sum + self.length_width = 8 + self.md5_width = 32 + self.obj_length = None + self.tell = 0 + self.obj_md5 = None + self.obj_body = None + self.sequence_number = -1 + self.rank = -1 + self.step = -1 + self.sequence_number_dict = dict() + + def connectionMade(self): + self.buffer = io.BytesIO() + self.obj_length = None + self.tell = 0 + self.obj_md5 = None + self.obj_body = None + self.factory.transport_dict[self.transport] = 1 + self.factory.transport_list.append(self.transport) + logger.info(f"Connected to {self.transport.getPeer()} successfully.") + + def connectionLost(self, reason): + self.factory.transport_dict.pop(self.transport, None) + if len(self.factory.transport_dict) == 0: + self.consumer_queue.put(self.ACK_KILL_PROCESS) + + logger.info(f"Lost connection with {self.transport.getPeer()}. Reason is: {reason} 与客户端 断开连接, " + f"current connection number is: {len(self.factory.transport_dict)}") + + def send_ack(self, ack_info): + ack_message = b"".join([ + ack_info, + self.sequence_number.to_bytes(8, byteorder='big'), + self.rank.to_bytes(8, byteorder='big'), + self.step.to_bytes(8, byteorder='big') + ]) + self.transport.write(ack_message) + + def post_process(self): + send_busy_ack = False + while self.consumer_queue.full(): + if not send_busy_ack: + self.send_ack(self.ACK_BUSY) + logger.debug("sending BUSY ACK") + send_busy_ack = True + time.sleep(0.1) + + obj_key = str(self.sequence_number) + "_" + str(self.rank) + "_" + str(self.step) + + recv_md5 = hashlib.md5(self.obj_body).hexdigest() + if self.check_sum and recv_md5 != self.obj_md5: + # when needs check md5 and check no pass, indicates received data error, send b"ERROR" to client. + logger.debug(f"Error:接收数据有问题，流水号{self.sequence_number}, expected {self.obj_md5}, but get {recv_md5}") + self.send_ack(self.ACK_ERROR) + else: + if self.obj_body == self.ACK_STOP: + self.handle_with_stop() + else: + self.send_ack(self.ACK_SUCCESS) + if obj_key in self.sequence_number_dict: + logger.debug(f"这是一次异常的重传，可以忽略。 {obj_key}, {self.sequence_number_dict}") + else: + self.sequence_number_dict[obj_key] = self.obj_md5 + self.consumer_queue.put(self.obj_body, block=True) + + self.reset_env() + finish_time = time.time() + logger.debug(f"finish_time: {finish_time - self.start_time}") + + def handle_with_stop(self): + logger.debug(f"接收到停止传输信号 TCP{self.transport.getPeer()}") + self.send_ack(self.ACK_STOP_CONFIRM) + if len(self.factory.transport_dict) == 0: + _rank, _step, _sequence_number = 0, 0, 100000000 + ack_kill = self.ACK_KILL_PROCESS + \ + _sequence_number.to_bytes(8, byteorder='big') + \ + _rank.to_bytes(8, byteorder='big') + \ + _step.to_bytes(8, byteorder='big') + for trans in self.factory.transport_list: + trans.write(ack_kill) + logger.debug(f"发送KILL信息给{self.transport.getPeer()}") + self.consumer_queue.put(self.ACK_KILL_PROCESS) + time.sleep(2) + + def reset_env(self): + self.obj_length = None + self.sequence_number = -1 + self.rank = -1 + self.step = -1 + self.obj_md5 = None + self.obj_body = None + + def dataReceived(self, data): + self.buffer.seek(0, 2) + self.buffer.write(data) + self.buffer.seek(self.tell) + + # The first data packet is packet header, it contains obj_length, sequence_number, rank, step + if self.obj_length is None and len(self.buffer.getvalue()) >= self.length_width * 4: + self.start_time = time.time() + self.obj_length = struct.unpack('!Q', self.buffer.read(self.length_width))[0] + self.sequence_number = struct.unpack('!Q', self.buffer.read(self.length_width))[0] + self.rank = struct.unpack('!Q', self.buffer.read(self.length_width))[0] + self.step = struct.unpack('!Q', self.buffer.read(self.length_width))[0] + self.tell += self.length_width * 4 + logger.debug( + f"流水号: {self.sequence_number}; RANK: {self.rank}; STEP: {self.step}; Length: {self.obj_length}") + + # If needs check md5 but not parse md5 yet, read 32b md5 values + check_sum_and_md5 = (self.check_sum + and self.obj_length is not None + and self.obj_md5 is None + and len(self.buffer.getvalue()) - self.tell >= self.md5_width) + if check_sum_and_md5: + self.obj_md5 = self.buffer.read(self.md5_width).decode() + self.tell += self.md5_width + logger.debug(f"MD5: {self.obj_md5}") + + current_length = len(self.buffer.getvalue()) - self.tell + if self.obj_length is not None and 0 < self.obj_length <= current_length: + # Current api data receive finished + self.obj_body = self.buffer.read(self.obj_length) + + self.tell += self.obj_length + self.buffer = io.BytesIO(self.buffer.getvalue()[self.tell:]) + self.buffer.seek(0) + self.tell = 0 + recv_data_time = time.time() + logger.debug(f"self.sequence_number {self.sequence_number} " + f"recv_data_time {recv_data_time - self.start_time}") + + if self.obj_body == self.ACK_STOP: + # Indicates the current TCP link receives a STOP signal and remove from the transport_dict + _transport = self.factory.transport_dict.pop(self.transport, None) + logger.debug(f"接收到b'STOP_' self.sequence_number {self.sequence_number} ") + self.post_process() + + +class MessageServerFactory(protocol.ServerFactory): + def __init__(self) -> None: + """ + transport_dict: links that have not completed data transmission. + transport_list: Records all TCP links. Appends TCP link to the transport list when a new TCP link is established. + """ + self.transport_dict = {} + self.transport_list = [] + + def is_all_connection_closed(self): + return len(self.transport_dict) == 0 diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index a3d765f3a..8fbe5dea0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -65,11 +65,18 @@ class FreeBenchmarkCheckConfig(BaseConfig): class RunUTConfig(BaseConfig): WrapApi = set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) + def __init__(self, json_config): super().__init__(json_config) self.white_list = json_config.get("white_list", Const.DEFAULT_LIST) self.black_list = json_config.get("black_list", Const.DEFAULT_LIST) self.error_data_path = json_config.get("error_data_path", Const.DEFAULT_PATH) + self.is_online = json_config.get("is_online", Const.IS_ONLINE) + self.nfs_path = json_config.get("nfs_path", Const.NFS_PATH) + self.is_benchmark_device = json_config.get("is_benchmark_device", Const.IS_BENCHMARK_DEVICE) + self.host = json_config.get("host", "") + self.port = json_config.get("port", -1) + self.rank_list = json_config.get("rank_list", Const.DEFAULT_LIST) self.check_run_ut_config() @classmethod @@ -86,11 +93,17 @@ class RunUTConfig(BaseConfig): def check_error_data_path_config(cls, error_data_path): if not os.path.exists(error_data_path): raise Exception("error_data_path: %s does not exist" % error_data_path) - + + @classmethod + def check_nfs_path_config(cls, nfs_path): + if nfs_path and not os.path.exists(nfs_path): + raise Exception("nfs_path: %s does not exist" % nfs_path) + def check_run_ut_config(self): RunUTConfig.check_filter_list_config(Const.WHITE_LIST, self.white_list) RunUTConfig.check_filter_list_config(Const.BLACK_LIST, self.black_list) RunUTConfig.check_error_data_path_config(self.error_data_path) + RunUTConfig.check_nfs_path_config(self.nfs_path) def parse_task_config(task, json_config): -- Gitee From 52a8d85a799ccf3c4b10984b2dcbb0b8f1b5cd0c Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 19:24:31 +0800 Subject: [PATCH 118/791] 82 --- .../compare_backend/compare_bean/profiling_info.py | 6 +++--- .../profiling_parser/test_gpu_profiling_parser.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index 71fb3c4a8..3722a44c3 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -224,9 +224,9 @@ class ProfilingInfo: [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - self.pa_time - self.vec_time - self.conv_time_fwd - self.conv_time_bwd]) - def calculate_vec_time(self): - self.vec_time = self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd \ - - self.conv_time_fwd - self.conv_time_bwd + # def calculate_vec_time(self): + # self.vec_time = self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd \ + # - self.conv_time_fwd - self.conv_time_bwd def calculate_schedule_time(self): self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time - self.communication_not_overlapped) diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index 50d60f39f..414a80ea9 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -76,7 +76,7 @@ class TestGpuProfilingParser(unittest.TestCase): res._marks = defaultdict(int) res._calculate_performance_time() self.assertEqual(res._result_data.overall_metrics.e2e_time, 98) - self.assertEqual(res._result_data.overall_metrics.sdma_time, 4) + self.assertEqual(res._result_data.overall_metrics.sdma_time, 0) self.assertEqual(res._result_data.overall_metrics.sdma_num, 0) self.assertEqual(res._result_data.overall_metrics.cube_time, 1) self.assertEqual(res._result_data.overall_metrics.cube_num, 1) -- Gitee From 5acf662b2d32e0c6296e9b1243a5a5b176d4525f Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 19:33:17 +0800 Subject: [PATCH 119/791] 82 --- .../profiling_parser/test_gpu_profiling_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index 414a80ea9..cef0fd9d3 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -76,8 +76,8 @@ class TestGpuProfilingParser(unittest.TestCase): res._marks = defaultdict(int) res._calculate_performance_time() self.assertEqual(res._result_data.overall_metrics.e2e_time, 98) - self.assertEqual(res._result_data.overall_metrics.sdma_time, 0) - self.assertEqual(res._result_data.overall_metrics.sdma_num, 0) + # self.assertEqual(res._result_data.overall_metrics.sdma_time, 0) + # self.assertEqual(res._result_data.overall_metrics.sdma_num, 0) self.assertEqual(res._result_data.overall_metrics.cube_time, 1) self.assertEqual(res._result_data.overall_metrics.cube_num, 1) self.assertEqual(res._result_data.overall_metrics.fa_time_fwd, 2) -- Gitee From 80fcbf44c97bbf80e927c0f41652449456ced21d Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 19:51:05 +0800 Subject: [PATCH 120/791] 82 --- .../compare_bean/profiling_info.py | 4 ---- .../test_gpu_profiling_parser.py | 20 +++++++++---------- 2 files changed, 10 insertions(+), 14 deletions(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index 3722a44c3..10ac47d6e 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -224,10 +224,6 @@ class ProfilingInfo: [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - self.pa_time - self.vec_time - self.conv_time_fwd - self.conv_time_bwd]) - # def calculate_vec_time(self): - # self.vec_time = self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd \ - # - self.conv_time_fwd - self.conv_time_bwd - def calculate_schedule_time(self): self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time - self.communication_not_overlapped) diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index cef0fd9d3..93c6e3855 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -76,16 +76,16 @@ class TestGpuProfilingParser(unittest.TestCase): res._marks = defaultdict(int) res._calculate_performance_time() self.assertEqual(res._result_data.overall_metrics.e2e_time, 98) - # self.assertEqual(res._result_data.overall_metrics.sdma_time, 0) - # self.assertEqual(res._result_data.overall_metrics.sdma_num, 0) - self.assertEqual(res._result_data.overall_metrics.cube_time, 1) - self.assertEqual(res._result_data.overall_metrics.cube_num, 1) - self.assertEqual(res._result_data.overall_metrics.fa_time_fwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_num_fwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_time_bwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_num_bwd, 2) - self.assertEqual(res._result_data.overall_metrics.vec_time, 2) - self.assertEqual(res._result_data.overall_metrics.vec_num, 2) # cun yi + self.assertEqual(res._result_data.overall_metrics.sdma_time, 0) + self.assertEqual(res._result_data.overall_metrics.sdma_num, 0) + self.assertEqual(res._result_data.overall_metrics.cube_time, 0) + self.assertEqual(res._result_data.overall_metrics.cube_num, 0) + self.assertEqual(res._result_data.overall_metrics.fa_time_fwd, 0) + self.assertEqual(res._result_data.overall_metrics.fa_num_fwd, 0) + self.assertEqual(res._result_data.overall_metrics.fa_time_bwd, 0) + self.assertEqual(res._result_data.overall_metrics.fa_num_bwd, 0) + self.assertEqual(res._result_data.overall_metrics.vec_time, 0) + self.assertEqual(res._result_data.overall_metrics.vec_num, 0) # cun yi self.assertEqual(res._result_data.overall_metrics.communication_not_overlapped, 2) self.assertEqual(res._result_data.overall_metrics.compute_time, 7) -- Gitee From 018f1bbbb8c468919a1f21bb4b7850be0d9531b2 Mon Sep 17 00:00:00 2001 From: lijiaojiao Date: Mon, 5 Aug 2024 19:57:14 +0800 Subject: [PATCH 121/791] =?UTF-8?q?=E3=80=90=E5=BC=80=E5=8F=91=E8=87=AA?= =?UTF-8?q?=E6=8F=90=E3=80=91=E3=80=90kj600=E3=80=91=E3=80=90=E5=AE=89?= =?UTF-8?q?=E5=85=A8=E3=80=91=E3=80=90=E4=B8=80=E8=88=AC=E3=80=91=E3=80=90?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E6=9C=AA=E6=A0=A1=E9=AA=8C=E3=80=91kj600?= =?UTF-8?q?=E4=B8=AD=E8=BE=93=E5=87=BA=E4=BB=B6=E6=96=87=E4=BB=B6=E6=9D=83?= =?UTF-8?q?=E9=99=90=E6=9C=AA=E6=8C=89=E8=A7=84=E5=AE=9A=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E6=88=90=E6=8C=87=E5=AE=9A=E6=9D=83=E9=99=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/kj600/kj600/module_hook.py | 18 ++++- debug/accuracy_tools/kj600/kj600/utils.py | 67 ++++++++++++++++++- 2 files changed, 80 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/kj600/kj600/module_hook.py b/debug/accuracy_tools/kj600/kj600/module_hook.py index 3b600b2b7..74ef684a6 100644 --- a/debug/accuracy_tools/kj600/kj600/module_hook.py +++ b/debug/accuracy_tools/kj600/kj600/module_hook.py @@ -14,7 +14,9 @@ from kj600.anomaly_detect import AnomalyScanner, SummaryWriterWithAD from kj600.anomaly_inform import AnomalyInformFactory from kj600.module_metric import get_metrics, write_metrics_tensorboard, get_summary_writer_tag_name, TensorMetrics from kj600.distributed.wrap_distributed import api_register, create_hooks, op_aggregate -from kj600.utils import print_warn_log, print_info_log, get_param_struct +from kj600.utils import print_warn_log, print_info_log, get_param_struct, check_path_length, check_path_pattern_valid, change_mode, FileCheckConst + + class ModuleHookContext: @@ -131,10 +133,20 @@ class TrainerMon: unique_id = str(uuid.uuid4())[:8] if dist.is_initialized(): if (dist.get_rank() in self.module_rank_list) or len(self.module_rank_list) == 0: + cur_path = os.path.join(output_base_dir, f"{cur_time}-rank{dist.get_rank()}-{unique_id}") + check_path_length(cur_path) + check_path_pattern_valid(cur_path) self.summary_writer = SummaryWriterWithAD( - os.path.join(output_base_dir, f"{cur_time}-rank{dist.get_rank()}-{unique_id}"), self.alert_rules, unique_id, anomaly_inform) + cur_path, self.alert_rules, unique_id, anomaly_inform) else: - self.summary_writer = SummaryWriterWithAD(os.path.join(output_base_dir, f"{cur_time}-{unique_id}"), self.alert_rules, unique_id, anomaly_inform) + cur_path = os.path.join(output_base_dir, f"{cur_time}-{unique_id}") + check_path_length(cur_path) + check_path_pattern_valid(cur_path) + self.summary_writer = SummaryWriterWithAD(cur_path, self.alert_rules, unique_id, anomaly_inform) + + full_path = os.path.realpath(cur_path) + change_mode(full_path,FileCheckConst.DATA_FILE_AUTHORITY) + # A HeatmapVisualizer instance is associated with an image self.update_heatmap_visualizer = defaultdict(HeatmapVisualizer) self.ratio_heatmap_visualizer = defaultdict(HeatmapVisualizer) diff --git a/debug/accuracy_tools/kj600/kj600/utils.py b/debug/accuracy_tools/kj600/kj600/utils.py index 53d47d998..0d300addf 100644 --- a/debug/accuracy_tools/kj600/kj600/utils.py +++ b/debug/accuracy_tools/kj600/kj600/utils.py @@ -8,6 +8,59 @@ FILE_NAME_MAX_LENGTH = 255 DIRECTORY_MAX_LENGTH = 4096 FILE_NAME_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" +class FileCheckConst: + """ + Class for file check const + """ + READ_ABLE = "read" + WRITE_ABLE = "write" + READ_WRITE_ABLE = "read and write" + DIRECTORY_LENGTH = 4096 + FILE_NAME_LENGTH = 255 + FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" + PKL_SUFFIX = ".pkl" + NUMPY_SUFFIX = ".npy" + JSON_SUFFIX = ".json" + PT_SUFFIX = ".pt" + CSV_SUFFIX = ".csv" + YAML_SUFFIX = ".yaml" + MAX_PKL_SIZE = 1 * 1024 * 1024 * 1024 + MAX_NUMPY_SIZE = 10 * 1024 * 1024 * 1024 + MAX_JSON_SIZE = 1 * 1024 * 1024 * 1024 + MAX_PT_SIZE = 10 * 1024 * 1024 * 1024 + MAX_CSV_SIZE = 1 * 1024 * 1024 * 1024 + MAX_YAML_SIZE = 10 * 1024 * 1024 + DIR = "dir" + FILE = "file" + DATA_DIR_AUTHORITY = 0o750 + DATA_FILE_AUTHORITY = 0o640 + FILE_SIZE_DICT = { + PKL_SUFFIX: MAX_PKL_SIZE, + NUMPY_SUFFIX: MAX_NUMPY_SIZE, + JSON_SUFFIX: MAX_JSON_SIZE, + PT_SUFFIX: MAX_PT_SIZE, + CSV_SUFFIX: MAX_CSV_SIZE, + YAML_SUFFIX: MAX_YAML_SIZE + } + +class FileCheckException(Exception): + """ + Class for File Check Exception + """ + NONE_ERROR = 0 + INVALID_PATH_ERROR = 1 + INVALID_FILE_TYPE_ERROR = 2 + INVALID_PARAM_ERROR = 3 + INVALID_PERMISSION_ERROR = 3 + + def __init__(self, code, error_info: str = ""): + super(FileCheckException, self).__init__() + self.code = code + self.error_info = error_info + + def __str__(self): + return self.error_info + def _print_log(level, msg, end='\n'): current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) pid = os.getgid() @@ -64,7 +117,7 @@ def check_path_length(path, name_length_limit=None): raise RuntimeError("The file path length exceeds limit.") -def check_path_pattern_vaild(path): +def check_path_pattern_valid(path): if not re.match(FILE_NAME_VALID_PATTERN, path): raise RuntimeError("The file path contains special characters.") @@ -95,7 +148,7 @@ def check_file_valid(path): check_link(path) real_path = os.path.realpath(path) check_path_length(real_path) - check_path_pattern_vaild(real_path) + check_path_pattern_valid(real_path) check_file_size(real_path) @@ -107,4 +160,14 @@ def check_file_valid_readable(path): def check_file_valid_writable(path): check_file_valid(path) check_path_writability(path) + + +def change_mode(path, mode): + if not os.path.exists(path) or os.path.islink(path): + return + try: + os.chmod(path, mode) + except PermissionError as ex: + print_error_log('Failed to change {} authority. {}'.format(path, str(ex))) + raise FileCheckException(FileCheckException.INVALID_PERMISSION_ERROR) from ex \ No newline at end of file -- Gitee From 95c35b0402ebfedd65e6647423fdc2a01371f92a Mon Sep 17 00:00:00 2001 From: CSNIU Date: Mon, 5 Aug 2024 20:39:45 +0800 Subject: [PATCH 122/791] importError bugfix --- .../msprobe/core/advisor/advisor.py | 6 +++--- .../msprobe/core/advisor/advisor_result.py | 4 ++-- .../msprobe/mindspore/__init__.py | 3 +-- .../mindspore/compare/distributed_compare.py | 7 ++++--- .../msprobe/mindspore/compare/ms_compare.py | 15 +++++++++++++- debug/accuracy_tools/msprobe/msprobe.py | 3 +-- .../msprobe/pytorch/__init__.py | 3 +-- .../pytorch/compare/distributed_compare.py | 8 ++++---- .../msprobe/pytorch/compare/pt_compare.py | 20 ++++++++++++++++--- 9 files changed, 47 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor.py b/debug/accuracy_tools/msprobe/core/advisor/advisor.py index ec2773e6d..9824ac22a 100644 --- a/debug/accuracy_tools/msprobe/core/advisor/advisor.py +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor.py @@ -17,9 +17,9 @@ import os -from msprobe.mindspore.advisor.advisor_result import AdvisorResult -from msprobe.mindspore.advisor.advisor_const import AdvisorConst -from msprobe.mindspore.common.log import logger +from msprobe.core.advisor.advisor_result import AdvisorResult +from msprobe.core.advisor.advisor_const import AdvisorConst +from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException from msprobe.core.common.file_check import FileChecker from msprobe.core.common.const import Const, CompareConst, FileCheckConst diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py index 5d59068fc..2bfea2eb9 100644 --- a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py @@ -17,8 +17,8 @@ import os import time -from msprobe.mindspore.advisor.advisor_const import AdvisorConst -from msprobe.mindspore.common.log import logger +from msprobe.core.advisor.advisor_const import AdvisorConst +from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.file_check import change_mode diff --git a/debug/accuracy_tools/msprobe/mindspore/__init__.py b/debug/accuracy_tools/msprobe/mindspore/__init__.py index 60bebb2ba..70be41497 100644 --- a/debug/accuracy_tools/msprobe/mindspore/__init__.py +++ b/debug/accuracy_tools/msprobe/mindspore/__init__.py @@ -1,3 +1,2 @@ from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger -# from .compare.acc_compare import compare -# from .compare.distributed_compare import compare_distributed +from .compare.distributed_compare import compare_distributed diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 4246bdd2b..303692dec 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -19,9 +19,9 @@ import sys import re from msprobe.core.common.utils import CompareException, check_compare_param, \ check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid -from msprobe.mindspore.compare.acc_compare import compare_core from msprobe.core.common.file_check import create_directory -from msprobe.mindspore.common.log import logger +from msprobe.core.common.log import logger +from msprobe.mindspore.compare.ms_compare import MSComparator def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): @@ -108,5 +108,6 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): except CompareException as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) - compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, + msComparator=MSComparator() + msComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index c631655d9..e0f3e481e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -267,6 +267,19 @@ def ms_compare(args): - + +def _compare_parser(parser): + parser.add_argument("-i", "--input_path", dest="input_path", type=str, + help=" The compare input path, a dict json.", required=True) + parser.add_argument("-o", "--output_path", dest="output_path", type=str, + help=" The compare task result out path.", required=True) + parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", + help=" Whether to save stack info.", required=False) + parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", + help=" Whether to give advisor.", required=False) + parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", + help=" Whether to perform a fuzzy match on the api name.", required=False) + + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 11c3899bd..19ebea2d6 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -22,8 +22,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _ _api_precision_compare_command from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command -from msprobe.pytorch.compare.acc_compare import _compare_parser, compare -from debug.accuracy_tools.msprobe.pytorch.compare.pt_compare import pt_compare +from msprobe.pytorch.compare.pt_compare import _compare_parser, pt_compare def main(): parser = argparse.ArgumentParser( diff --git a/debug/accuracy_tools/msprobe/pytorch/__init__.py b/debug/accuracy_tools/msprobe/pytorch/__init__.py index 11193b39f..c14d9701a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/__init__.py +++ b/debug/accuracy_tools/msprobe/pytorch/__init__.py @@ -1,4 +1,3 @@ from .debugger.precision_debugger import PrecisionDebugger from .common.utils import seed_all -# from .compare.acc_compare import compare -# from .compare.distributed_compare import compare_distributed +from .compare.distributed_compare import compare_distributed diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index f5d28de40..11e5193ec 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -19,11 +19,10 @@ import sys import re from msprobe.core.common.utils import CompareException, check_compare_param, \ check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid -from msprobe.pytorch.compare.acc_compare import compare_core from msprobe.core.common.file_check import create_directory from msprobe.core.common.exceptions import FileCheckException -from msprobe.pytorch.common.log import logger - +from msprobe.core.common.log import logger +from msprobe.pytorch.compare.pt_compare import PTComparator def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): def check_and_return_dir_contents(dump_dir, prefix): @@ -107,5 +106,6 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) - compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, + ptComparator=PTComparator() + ptComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 49fc5ed65..76d37ca93 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -13,9 +13,9 @@ from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, C from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory from msprobe.core.common.const import Const, CompareConst, FileCheckConst -from msprobe.core.compare.utils import ComparisonResult,_save_cmp_result,merge_tensor, get_un_match_accuracy,get_accuracy,read_op -from msprobe.core.compare.highlight import find_compare_result_error_rows,highlight_rows_xlsx -from msprobe.core.compare.Multiprocessing_compute import _handle_multi_process +from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op +from msprobe.core.compare.Multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process +from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException @@ -268,6 +268,20 @@ def pt_compare(args): auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) + +def _compare_parser(parser): + parser.add_argument("-i", "--input_path", dest="input_path", type=str, + help=" The compare input path, a dict json.", required=True) + parser.add_argument("-o", "--output_path", dest="output_path", type=str, + help=" The compare task result out path.", required=True) + parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", + help=" Whether to save stack info.", required=False) + parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", + help=" Whether to give advisor.", required=False) + parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", + help=" Whether to perform a fuzzy match on the api name.", required=False) + + -- Gitee From 16065e860f92179c99769a3897a2a244ef3fc015 Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Sat, 3 Aug 2024 16:54:16 +0800 Subject: [PATCH 123/791] grad tool --- debug/accuracy_tools/msprobe/__init__.py | 1 + .../accuracy_tools/msprobe/config/config.json | 8 + debug/accuracy_tools/msprobe/core/__init__.py | 0 .../msprobe/core/common/const.py | 3 +- .../msprobe/core/common/utils.py | 17 ++ .../msprobe/core/grad_probe/__init__.py | 0 .../msprobe/core/grad_probe/constant.py | 56 ++++++ .../msprobe/core/grad_probe/grad_compare.py | 180 ++++++++++++++++++ .../msprobe/core/grad_probe/utils.py | 9 + .../msprobe/pytorch/common/utils.py | 40 ++++ .../pytorch/debugger/precision_debugger.py | 13 ++ .../msprobe/pytorch/grad_probe/__init__.py | 0 .../pytorch/grad_probe/grad_monitor.py | 106 +++++++++++ .../pytorch/grad_probe/grad_stat_csv.py | 127 ++++++++++++ .../msprobe/pytorch/pt_config.py | 9 + .../pytorch_ut/grad_probe/test_grad_csv.py | 37 ++++ .../grad_probe/test_grad_monitor.py | 79 ++++++++ 17 files changed, 684 insertions(+), 1 deletion(-) create mode 100644 debug/accuracy_tools/msprobe/core/__init__.py create mode 100644 debug/accuracy_tools/msprobe/core/grad_probe/__init__.py create mode 100644 debug/accuracy_tools/msprobe/core/grad_probe/constant.py create mode 100644 debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py create mode 100644 debug/accuracy_tools/msprobe/core/grad_probe/utils.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/grad_probe/__init__.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py diff --git a/debug/accuracy_tools/msprobe/__init__.py b/debug/accuracy_tools/msprobe/__init__.py index e69de29bb..ade5d3d36 100644 --- a/debug/accuracy_tools/msprobe/__init__.py +++ b/debug/accuracy_tools/msprobe/__init__.py @@ -0,0 +1 @@ +from msprobe.core.grad_probe.grad_compare import GradComparator \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/config/config.json b/debug/accuracy_tools/msprobe/config/config.json index c6077b75a..ef0283ca2 100644 --- a/debug/accuracy_tools/msprobe/config/config.json +++ b/debug/accuracy_tools/msprobe/config/config.json @@ -29,5 +29,13 @@ "white_list": [], "black_list": [], "error_data_path": "./" + }, + "grad_probe": { + "level": "L1", + "param_list": [], + "rank": [], + "step": [], + "bounds": [-1, 0, 1], + "output_path": "./grad_output" } } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/__init__.py b/debug/accuracy_tools/msprobe/core/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index b59536aa5..b7d40ec63 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -61,13 +61,14 @@ class Const: ENV_ENABLE = "1" ENV_DISABLE = "0" MAX_SEED_VALUE = 4294967295 # 2**32 - 1 - TASK_LIST = ["tensor", "statistics", "overflow_check", "free_benchmark", "run_ut"] + TASK_LIST = ["tensor", "statistics", "overflow_check", "free_benchmark", "run_ut", "grad_probe"] LEVEL_LIST = ["L0", "L1", "L2", "mix"] STATISTICS = "statistics" TENSOR = "tensor" OVERFLOW_CHECK = "overflow_check" FREE_BENCHMARK = "free_benchmark" RUN_UT = "run_ut" + GRAD_PROBE = "grad_probe" ATTR_NAME_PREFIX = "wrap_" ATTR_NAME_PREFIX_LEN = len(ATTR_NAME_PREFIX) KERNEL_DUMP = "kernel_dump" diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 56f4d1e61..7818aaaf3 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -22,6 +22,7 @@ import stat import subprocess import time import json +import csv from datetime import datetime, timezone from pathlib import Path import numpy as np @@ -514,3 +515,19 @@ def get_header_index(header_name, summary_compare=False): def convert_tuple(data): return data if isinstance(data, tuple) else (data, ) + + +def write_csv(data, filepath): + with FileOpen(filepath, 'a+', encoding='utf-8-sig') as f: + writer = csv.writer(f) + writer.writerows(data) + + +def load_npy(filepath): + filepath = os.path.realpath(filepath) + check_file_or_directory_path(filepath) + try: + npy = np.load(filepath) + except Exception as e: + raise RuntimeError(f"load npy file {filepath} failed") from e + return npy diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/__init__.py b/debug/accuracy_tools/msprobe/core/grad_probe/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/constant.py b/debug/accuracy_tools/msprobe/core/grad_probe/constant.py new file mode 100644 index 000000000..38d33e988 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/grad_probe/constant.py @@ -0,0 +1,56 @@ + +class GradConst: + + FRAMEWORKS = {"PyTorch", "MindSpore"} + PYTORCH = "PyTorch" + MindSpore = "MindSpore" + + GRAD_FILE_SUFFIX = {"npy", "pt"} + NPY_SUFFIX = "npy" + PT_SUFFIX = "pt" + + # for callback + CURRENT_STEP = "current_step" + + PARAM_LIST = "param_list" + RANK = "rank" + STEP = "step" + BOUNDS = "bounds" + OUTPUT_PATH = "output_path" + + # level const + LEVEL = "level" + LEVEL0 = "L0" + LEVEL1 = "L1" + LEVEL2 = "L2" + SUPPORTED_LEVEL = {"L0", "L1", "L2"} + + # numpy coding + STEP_IDX = 0 + SHAPE_DIM_IDX = 4 + MAX_SIZE = 10 * 1024 * 1024 * 1024 + + # direction suffix + DIR_SUFFIX = "dir.npy" + + # file safty + DATA_DIR_AUTHORITY = 0o750 + DATA_FILE_AUTHORITY = 0o640 + DIRECTORY_LENGTH = 4096 + FILE_NAME_LENGTH = 255 + FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" + PARAM_VALID_PATTERN = r"^[a-zA-Z0-9.]+$" + DIR = "dir" + FILE = "file" + + STEP_FINISH = "step_finish" + + SUMMARY = "summary" + + # csv header entry + MD5 = "MD5" + DISTRIBUTION = "distribution" + SHAPE = "shape" + MAX = "max" + MIN = "min" + NORM = "norm" \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py new file mode 100644 index 000000000..26cba34f0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -0,0 +1,180 @@ +import os +from typing import List + +from tqdm import tqdm +import pandas as pd +import matplotlib.pyplot as plt + +from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.log import logger +from msprobe.core.common.utils import remove_path, write_csv, load_npy +from msprobe.core.grad_probe.constant import GradConst +from msprobe.pytorch.common.utils import load_pt + + +class GradComparator: + + @staticmethod + def _get_grad_weight_order(path1, path2): + for summary_file in os.listdir(path1): + if not summary_file.endswith(".csv"): + continue + if not os.path.exists(os.path.join(path2, summary_file)): + continue + summary_csv = pd.read_csv(os.path.join(path1, summary_file)) + return summary_csv["param_name"] + raise RuntimeError("no matched grad_summary.csv for comparison, please dump data in same configuration") + + @staticmethod + def _get_name_matched_grad_file(param_name, grad_files): + for grad_file in grad_files: + if param_name == grad_file[:grad_file.rfind('.')]: + return grad_file + raise RuntimeError("no matched grad_file for comparison, please dump data in same configuration") + + @classmethod + def compare_distributed(cls, path1: str, path2: str, output_dir: str): + ranks = cls._get_matched_dirs(path1, path2, "rank") + logger.info(f"the following ranks will be compared: {ranks}") + if not ranks: + raise RuntimeError("no matched ranks for comparison, please dump data in same configuration") + if not os.path.isdir(output_dir): + create_directory(output_dir) + for rank in tqdm(ranks, desc="rank"): + logger.info(f"now comparing rank {rank}:") + cls.compare(os.path.join(path1, f"rank{rank}"), + os.path.join(path2, f"rank{rank}"), + os.path.join(output_dir, f"rank{rank}")) + + @classmethod + def compare(cls, path1: str, path2: str, output_dir: str): + steps = cls._get_matched_dirs(path1, path2, "step") + if not steps: + raise RuntimeError("no matched steps for comparison, please dump data in same configuration") + similarities = cls._calculate_separated_similarities(path1, path2, steps) + if not os.path.isdir(output_dir): + create_directory(output_dir) + cls._save_similarities(similarities, steps, output_dir) + + @classmethod + def _get_matched_dirs(cls, path1: str, path2: str, dir_prefix): + check_file_or_directory_path(path1, isdir=True) + check_file_or_directory_path(path2, isdir=True) + dirs = [] + for dir_name in os.listdir(path1): + index = dir_name.replace(dir_prefix, "", 1) + if not dir_name.startswith(dir_prefix) or not index.isdigit(): + continue + + folder2 = os.path.join(path2, dir_name) + if not os.path.isdir(folder2): + continue + dirs.append(int(index)) + dirs = sorted(dirs) + return dirs + + @classmethod + def _save_similarities(cls, similarities: List[float], steps: List[int], output_dir: str): + if not similarities: + raise ValueError(f"length of similarities is 0") + result = [['step'] + [str(step) for step in steps]] + for key, value in tqdm(similarities.items(), desc="save similarities (by param)"): + if len(value) != len(steps): + raise RuntimeError(f"similarities length of {key}:{len(value)} not equal steps:{len(steps)}") + plt.plot(steps, value) + plt.xlabel('steps') + plt.ylabel('similarities') + plt.title(f'{key}_similarities') + picture_dir = os.path.join(output_dir, "similarities_picture") + if not os.path.isdir(picture_dir): + create_directory(picture_dir) + fig_save_path = os.path.join(picture_dir, f"{key}_similarities.png") + + check_path_before_create(fig_save_path) + try: + plt.savefig(fig_save_path) + except Exception as e: + raise RuntimeError(f"save plt figure {fig_save_path} failed") from e + plt.close() + + result.append([key] + value) + result_csv_path = os.path.join(output_dir, "similarities.csv") + if os.path.exists(result_csv_path): + logger.warning(f"{result_csv_path} will be recoverd") + remove_path(result_csv_path) + write_csv(result, result_csv_path) + + @classmethod + def _calculate_separated_similarities(cls, path1, path2, steps): + similarities = {} + logger.info(f"{len(steps)} steps will be compared") + grad_weight_order = cls._get_grad_weight_order(path1, path2) + for step in tqdm(steps, desc="culculate similarities (by step)"): + grad_files = cls._get_matched_grad_files(path1, path2, step) + same_count_summary = 0 + total_count_summary = 0 + for grad_name in grad_weight_order: + grad_file = cls._get_name_matched_grad_file(grad_name, grad_files) + grad1 = os.path.join(path1, f"step{step}", grad_file) + grad2 = os.path.join(path2, f"step{step}", grad_file) + same_count, total_count = cls._calculate_similarity(grad1, grad2) + same_count_summary += same_count + total_count_summary += total_count + idx = grad_file.rfind(".") + param_name = grad_file[:idx] + if param_name not in similarities: + similarities[param_name] = [] + if total_count == 0: + similarities[param_name].append(0) + else: + similarities[param_name].append(same_count / total_count) + if GradConst.SUMMARY not in similarities: + similarities[GradConst.SUMMARY] = [] + if total_count_summary == 0: + similarities[GradConst.SUMMARY].append(0) + else: + similarities[GradConst.SUMMARY].append(same_count_summary / total_count_summary) + return similarities + + @classmethod + def _get_matched_grad_files(cls, path1: str, path2: str, step: int): + path1 = os.path.join(path1, f"step{step}") + path2 = os.path.join(path2, f"step{step}") + check_file_or_directory_path(path1, isdir=True) + check_file_or_directory_path(path2, isdir=True) + grad_files = [] + for grad_file in os.listdir(path1): + splits = grad_file.split('.') + if len(splits) < 1 or splits[-1] not in GradConst.GRAD_FILE_SUFFIX: + continue + folder2 = os.path.join(path2, grad_file) + if not os.path.exists(folder2): + continue + grad_files.append(grad_file) + return sorted(grad_files) + + @classmethod + def _calculate_similarity(cls, grad_file1: str, grad_file2: str): + npy1, npy2 = cls._load_grad_files(grad_file1, grad_file2) + same_count = (npy1 == npy2).sum() + total_count = npy1.size + return same_count, total_count + + @classmethod + def _load_grad_files(cls, grad_file1: str, grad_file2: str): + if grad_file1.endswith('pt'): + grad1 = load_pt(grad_file1).numpy() + grad2 = load_pt(grad_file2).numpy() + else: + grad1 = load_npy(grad_file1) + grad2 = load_npy(grad_file2) + if grad1.shape != grad2.shape: + raise RuntimeError(f"tensor shape is not equal: {grad_file1}, {grad_file2}") + if grad1.dtype != bool: + raise TypeError(f"tensor type is not bool: {grad_file1}") + if grad2.dtype != bool: + raise TypeError(f"tensor type is not bool: {grad_file2}") + return grad1, grad2 + + diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py new file mode 100644 index 000000000..05dd9a568 --- /dev/null +++ b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py @@ -0,0 +1,9 @@ +def data_in_list_target(data, lst): + return not lst or len(lst) == 0 or data in lst + + +def check_numeral_list_ascend(lst): + if any(not isinstance(item, (int, float)) for item in lst): + raise Exception("The input list should only contain numbers") + if lst != sorted(lst): + raise Exception("The input list should be ascending") diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 181491488..ae8823de6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -23,6 +23,9 @@ import torch.distributed as dist import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError +from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create +from msprobe.core.common.file_check import FileCheckConst, change_mode + try: import torch_npu @@ -245,6 +248,43 @@ def get_tensor_rank(in_feat, out_feat): return tensor_rank +def get_rank_id(): + if torch.distributed.is_initialized(): + return torch.distributed.get_rank() + return 0 + + +def print_rank_0(message): + if dist.is_initialized(): + if dist.get_rank() == 0: + logger.info(message) + else: + logger.info(message) + + +def load_pt(pt_path, to_cpu=False): + pt_path = os.path.realpath(pt_path) + check_file_or_directory_path(pt_path) + try: + if to_cpu: + pt = torch.load(pt_path, map_location=torch.device("cpu")) + else: + pt = torch.load(pt_path) + except Exception as e: + raise RuntimeError(f"load pt file {pt_path} failed") from e + return pt + + +def save_pt(tensor, filepath): + filepath = os.path.realpath(filepath) + check_path_before_create(filepath) + try: + torch.save(tensor, filepath) + except Exception as e: + raise RuntimeError(f"save pt file {filepath} failed") from e + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + def _create_logger(level=logging.INFO): logger_ = logging.getLogger() logger_.setLevel(level) diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py index 6119bbd1d..012d42faf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py @@ -5,10 +5,13 @@ from msprobe.pytorch.service import Service from msprobe.pytorch.common.log import logger from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common.const import Const +from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor class PrecisionDebugger: _instance = None + tasks_not_need_debugger = [Const.GRAD_PROBE] def __new__(cls, *args, **kwargs): if cls._instance is None: @@ -31,6 +34,10 @@ class PrecisionDebugger: self.initialized = True self.model = self.check_model_valid(model) common_config, task_config = parse_json_config(config_path, task) + self.task = common_config.task + if self.task == Const.GRAD_PROBE: + GradientMonitor(task_config, model) + return if step: common_config.step = step self.config = DebuggerConfig( @@ -58,6 +65,8 @@ class PrecisionDebugger: @classmethod def start(cls): instance = cls._instance + if instance.task in PrecisionDebugger.tasks_not_need_debugger: + return if not instance: raise Exception("No instance of PrecisionDebugger found.") if instance.enable_dataloader: @@ -76,6 +85,8 @@ class PrecisionDebugger: @classmethod def stop(cls): instance = cls._instance + if instance.task in PrecisionDebugger.tasks_not_need_debugger: + return if not instance: raise Exception("PrecisionDebugger instance is not created.") if instance.enable_dataloader: @@ -85,6 +96,8 @@ class PrecisionDebugger: @classmethod def step(cls): + if cls._instance.task in PrecisionDebugger.tasks_not_need_debugger: + return if not cls._instance: raise Exception("PrecisionDebugger instance is not created.") cls._instance.service.step() diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/__init__.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py new file mode 100644 index 000000000..edd28635d --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -0,0 +1,106 @@ +import os +from collections import defaultdict + +import torch +from torch.optim.optimizer import register_optimizer_step_pre_hook +from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv +from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.log import logger +from msprobe.core.common.utils import remove_path, write_csv +from msprobe.pytorch.common.utils import get_rank_id, print_rank_0, save_pt + + +class GradientMonitor: + level_adp = { + "L0": { + "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": False + }, + "L1": { + "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": True + }, + "L2": { + "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": True + }, + } + + def __init__(self, config, model): + self._config = config._config + self._model = model + level = self._config.get("level") + if level not in GradientMonitor.level_adp: + raise Exception(f"level is valid, not in {GradientMonitor.level_adp.keys()}") + self._level_adp = GradientMonitor.level_adp[level] + self._param_list = self._config.get('param_list') + self._target_ranks = self._config.get("rank") + logger.info(f"target rank {self._target_ranks}") + self._target_step = self._config.get("step") + logger.info(f"target step {self._target_step}") + self._bounds = self._config.get("bounds") + check_numeral_list_ascend(self._bounds) + self._output_path = self._config.get("output_path") + if not os.path.exists(self._output_path): + create_directory(self._output_path) + else: + logger.warning(f"the file in {self._output_path} will be recoverd") + self._step = -1 + self._param2name = defaultdict(str) + self._monitor() + + @property + def output_path(self): + return self._output_path + + @staticmethod + def save_grad_direction(param_name, grad, save_path): + if not os.path.exists(save_path): + create_directory(save_path) + param_grad = grad.clone().detach() + is_positive = param_grad > 0 + save_filepath = os.path.join(save_path, f"{param_name}.pt") + save_pt(is_positive, save_filepath) + + def _monitor(self): + print_rank_0("> parameter names:") + for name, param in self._model.named_parameters(): + self._param2name[param] = name + print_rank_0(f"\t{name}") + setattr(self, "_rank", get_rank_id()) + if torch.distributed.is_initialized() and not data_in_list_target(getattr(self, "_rank"), self._target_ranks): + return + self._hook_optimizer() + + def _hook_optimizer(self): + def optimizer_pre_step_hook(optimizer, args, kargs): + self._step += 1 + if not data_in_list_target(self._step, self._target_step): + return + output_lines = [] + for param, param_name in self._param2name.items(): + if not data_in_list_target(param_name, self._param_list): + continue + grad = param.main_grad if hasattr(param, "main_grad") else param.grad + if grad is None: + logger.info(f"grad is None: {param_name}") + continue + grad_info = GradStatCsv.generate_csv_line(param_name, self._level_adp, grad, self._bounds) + output_lines.append(grad_info) + if self._level_adp["have_grad_direction"]: + GradientMonitor.save_grad_direction(param_name, grad, + f'{self._output_path}/rank{self._rank}/step{self._step}') + output_dirpath = os.path.join(self._output_path, f"rank{getattr(self, '_rank')}") + if not os.path.isdir(output_dirpath): + create_directory(output_dirpath) + output_path = os.path.join(output_dirpath, f"grad_summary_{self._step}.csv") + if os.path.exists(output_path): + logger.warning(f"{output_path} will be recoverd") + remove_path(output_path) + header_result = GradStatCsv.generate_csv_header(self._level_adp, self._bounds) + output_lines.insert(0, header_result) + write_csv(output_lines, output_path) + + register_optimizer_step_pre_hook(optimizer_pre_step_hook) diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py new file mode 100644 index 000000000..ae01b75ee --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py @@ -0,0 +1,127 @@ +from abc import ABC, abstractmethod +from collections import namedtuple +import hashlib +import torch +from msprobe.core.grad_probe.constant import GradConst + +CSV_header_input = namedtuple("CSV_header_input", ["bounds"]) +CSV_content_input = namedtuple("CSV_content_input", ["grad", "bounds"]) + + +class GradStatCsv: + csv = {} + + @staticmethod + def generate_csv_header(level, bounds): + header = ["param_name"] + for key in level["header"]: + csv_header_input = CSV_header_input(bounds=bounds) + header.extend(GradStatCsv.csv[key].generate_csv_header(csv_header_input)) + return header + + @staticmethod + def generate_csv_line(param_name, level, grad, bounds): + line = [param_name] + for key in level["header"]: + csv_content_input = CSV_content_input(grad=grad, bounds=bounds) + line.extend(GradStatCsv.csv[key].generate_csv_content(csv_content_input)) + return line + + +def register_csv_item(key, cls=None): + if cls is None: + # 无参数时，返回装饰器函数 + return lambda cls: register_csv_item(key, cls) + GradStatCsv.csv[key] = cls + return cls + + +class CsvItem(ABC): + @abstractmethod + def generate_csv_header(csv_header_input): + pass + + @abstractmethod + def generate_csv_content(csv_content_input): + pass + + +@register_csv_item(GradConst.MD5) +class CSV_md5(CsvItem): + def generate_csv_header(csv_header_input): + return ["MD5"] + + def generate_csv_content(csv_content_input): + grad = csv_content_input.grad + tensor_bytes = grad.cpu().detach().float().numpy().tobytes() + md5_hash = hashlib.md5(tensor_bytes) + return [md5_hash.hexdigest()] + + +@register_csv_item(GradConst.DISTRIBUTION) +class CSV_distribution(CsvItem): + def generate_csv_header(csv_header_input): + bounds = csv_header_input.bounds + intervals = [] + for i, _ in enumerate(bounds): + if i == 0: + intervals.append(f"(-inf, {bounds[i]}]") + else: + intervals.append(f"({bounds[i-1]}, {bounds[i]}]") + intervals.extend([f"({bounds[-1]}, inf)", "=0"]) + return intervals + + def generate_csv_content(csv_content_input): + grad = csv_content_input.grad + bounds = csv_content_input.bounds + grad = grad.cpu().detach() + if grad.dtype == torch.bfloat16: + grad = grad.to(torch.float32) + element_num = grad.numel() + grad_equal_0_num = (grad == 0).sum().item() + bound = torch.Tensor(bounds) + bucketsize_result = torch.bucketize(grad, bound) + interval_nums = [(bucketsize_result == i).sum().item() for i in range(len(bound) + 1)] + interval_nums.append(grad_equal_0_num) + return_list = [x / element_num if element_num != 0 else 0 for x in interval_nums] + return return_list + + +@register_csv_item(GradConst.MAX) +class CSV_max(CsvItem): + def generate_csv_header(csv_header_input): + return ["max"] + + def generate_csv_content(csv_content_input): + grad = csv_content_input.grad + return [torch.max(grad).cpu().detach().float().numpy().tolist()] + + +@register_csv_item(GradConst.MIN) +class CSV_max(CsvItem): + def generate_csv_header(csv_header_input): + return ["min"] + + def generate_csv_content(csv_content_input): + grad = csv_content_input.grad + return [torch.min(grad).cpu().detach().float().numpy().tolist()] + + +@register_csv_item(GradConst.NORM) +class CSV_max(CsvItem): + def generate_csv_header(csv_header_input): + return ["norm"] + + def generate_csv_content(csv_content_input): + grad = csv_content_input.grad + return [torch.norm(grad).cpu().detach().float().numpy().tolist()] + + +@register_csv_item(GradConst.SHAPE) +class CSV_shape(CsvItem): + def generate_csv_header(csv_header_input): + return ["shape"] + + def generate_csv_content(csv_content_input): + grad = csv_content_input.grad + return [list(grad.shape)] \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index ceec92a63..daba5476c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -93,6 +93,12 @@ class RunUTConfig(BaseConfig): RunUTConfig.check_error_data_path_config(self.error_data_path) +class GradToolConfig(BaseConfig): + def __init__(self, json_config): + super().__init__(json_config) + self._config = json_config + + def parse_task_config(task, json_config): default_dic = {} if task == Const.TENSOR: @@ -110,6 +116,9 @@ def parse_task_config(task, json_config): elif task == Const.RUN_UT: config_dic = json_config.get(Const.RUN_UT, default_dic) return RunUTConfig(config_dic) + elif task == Const.GRAD_PROBE: + config_dic = json_config.get(Const.GRAD_PROBE, default_dic) + return GradToolConfig(config_dic) else: return StatisticsConfig(default_dic) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py new file mode 100644 index 000000000..bd569f5a2 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py @@ -0,0 +1,37 @@ +# coding=utf-8 +import unittest +import os +import torch +from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv +from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor + +grad_tensor = torch.tensor([[-2, 2], [0.2, 0.3]]) + + +class TestGradCSV(unittest.TestCase): + def test_level_L0_header(self): + self.assertEqual(['param_name', 'MD5', 'max', 'min', 'norm', 'shape'], + GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L0"], [-1, 0, 1])) + + def test_level_L1_header(self): + self.assertEqual(['param_name', 'max', 'min', 'norm', 'shape'], + GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L1"], [-1, 0, 1])) + + def test_level_L2_header(self): + self.assertEqual(['param_name', '(-inf, -1]', '(-1, 0]', '(0, 1]', '(1, inf)', '=0', 'max', 'min', 'norm', 'shape'], + GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L2"], [-1, 0, 1])) + + def test_level_L0_content(self): + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L0"], grad_tensor, [-1, 0, 1]) + self.assertEqual(['model.conv2d', '678a6c7d9d9716682b56fda097d0936c', 2.0, -2.0, 2.851315498352051, [2, 2]], + generated_csv_line) + + def test_level_L1_content(self): + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L1"], grad_tensor, [-1, 0, 1]) + self.assertEqual(['model.conv2d', 2.0, -2.0, 2.851315498352051, [2, 2]], + generated_csv_line) + + def test_level_L2_content(self): + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L2"], grad_tensor, [-1, 0, 1]) + self.assertEqual(['model.conv2d', 0.25, 0.0, 0.5, 0.25, 0.0, 2.0, -2.0, 2.851315498352051, [2, 2]], + generated_csv_line) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py new file mode 100644 index 000000000..d79cca502 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py @@ -0,0 +1,79 @@ +import os +import shutil +import random +import unittest +import hashlib +import torch +import numpy as np +import torch.nn as nn +from msprobe.core.grad_probe.grad_compare import GradComparator +from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor +from msprobe.pytorch.pt_config import GradToolConfig + + +config_dict = { + "level": "L1", + "param_list": "", + "rank": [], + "step": [], + "bounds": [-1,0,1], + "output_path": "./grad_output" +} + +def seed_all(seed=1234, mode=False): + random.seed(seed) + os.environ['PYTHONHASHSEED'] = str(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.use_deterministic_algorithms(mode) + +seed_all() + + +inputs = [torch.rand(10, 10) for _ in range(10)] +labels = [torch.randint(0, 5, (10,)) for _ in range(10)] + + +class MockModule(nn.Module): + def __init__(self): + super().__init__() + self.linear = nn.Linear(10, 5) + self.relu = nn.ReLU() + + def forward(self, x): + x1 = self.linear(x) + x2 = self.relu(x1) + return x2 + + +def get_grad_monitor(): + loss_fun = nn.CrossEntropyLoss() + test_module = MockModule() + nn.init.constant_(test_module.linear.weight, 1.0) + nn.init.constant_(test_module.linear.bias, 1.0) + optimizer = torch.optim.SGD(test_module.parameters(), lr=1e-2) + + gm = GradientMonitor(GradToolConfig(config_dict), test_module) + + for input_data, label in zip(inputs, labels): + output = test_module(input_data) + loss = loss_fun(output, label) + optimizer.zero_grad() + loss.backward() + optimizer.step() + return gm + + +class TestGradMonitor(unittest.TestCase): + def test_compare(self): + gm = get_grad_monitor() + compare_output_path = os.path.join(gm.output_path, "grad_compare") + GradComparator.compare_distributed(gm.output_path, gm.output_path, + compare_output_path) + items = os.listdir(compare_output_path) + self.assertEqual(len(items), 1) + with open(os.path.join(compare_output_path, items[0], "similarities.csv"), 'r') as f: + data = f.read() + self.assertEqual(hashlib.md5(data.encode("utf-8")).hexdigest(), "138910fa9a4607d0adf6ff05e3753ed2") + shutil.rmtree(gm.output_path) + -- Gitee From ef21ed656d67624b18315ac28b5e29182ce67430 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Mon, 5 Aug 2024 21:59:10 +0800 Subject: [PATCH 124/791] =?UTF-8?q?buffix!=20importError=20fixed=EF=BC=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 7 +++---- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 2 -- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index e0f3e481e..21b1b9c24 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -2,7 +2,6 @@ import json import multiprocessing import os.path import sys - import numpy as np import pandas as pd @@ -12,9 +11,9 @@ from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, C from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory from msprobe.core.common.const import Const, CompareConst, FileCheckConst -from msprobe.core.compare.utils import ComparisonResult,_save_cmp_result,merge_tensor, get_un_match_accuracy,get_accuracy,read_op -from msprobe.core.compare.highlight import find_compare_result_error_rows,highlight_rows_xlsx -from msprobe.core.compare.Multiprocessing_compute import _handle_multi_process +from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op +from msprobe.core.compare.Multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process +from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 76d37ca93..081f5631d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,10 +1,8 @@ -import argparse import json import multiprocessing import os.path import sys import torch -import numpy as np import pandas as pd from msprobe.core.advisor.advisor import Advisor -- Gitee From 7a94897c5069d5752ba7a5a89607a9f19b3c4453 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 6 Aug 2024 09:38:12 +0800 Subject: [PATCH 125/791] compare_distributed cli add --- debug/accuracy_tools/msprobe/msprobe.py | 35 +++++++++++-------- .../msprobe/pytorch/compare/acc_compare.py | 31 ++++++---------- .../msprobe/pytorch/compare/compare_cli.py | 21 +++++++++++ 3 files changed, 52 insertions(+), 35 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 4a6250039..2963a52a0 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -22,7 +22,8 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _ _api_precision_compare_command from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command -from msprobe.pytorch.compare.acc_compare import _compare_parser, compare_cli +from msprobe.pytorch.compare.acc_compare import _compare_parser +from msprobe.pytorch.compare.compare_cli import compare_cli def main(): @@ -33,7 +34,7 @@ def main(): f"For any issue, refer README.md first", ) parser.set_defaults(print_help=parser.print_help) - parser.add_argument('-f', '--framework', required=True, choices=['pytorch'], + parser.add_argument('-f', '--framework', required=True, choices=['pytorch', 'mindspore'], help='Deep learning framework.') subparsers = parser.add_subparsers() subparsers.add_parser('parse') @@ -53,19 +54,23 @@ def main(): parser.print_help() sys.exit(0) args = parser.parse_args(sys.argv[1:]) - if sys.argv[3] == "run_ut": - run_ut_command(args) - elif sys.argv[3] == "parse": - cli_parse() - elif sys.argv[3] == "multi_run_ut": - config = prepare_config(args) - run_parallel_ut(config) - elif sys.argv[3] == "api_precision_compare": - _api_precision_compare_command(args) - elif sys.argv[3] == "run_overflow_check": - _run_overflow_check_command(args) - elif sys.argv[3] == "compare": - compare_cli(args) + if sys.argv[2] == "pytorch": + if sys.argv[3] == "run_ut": + run_ut_command(args) + elif sys.argv[3] == "parse": + cli_parse() + elif sys.argv[3] == "multi_run_ut": + config = prepare_config(args) + run_parallel_ut(config) + elif sys.argv[3] == "api_precision_compare": + _api_precision_compare_command(args) + elif sys.argv[3] == "run_overflow_check": + _run_overflow_check_command(args) + elif sys.argv[3] == "compare": + compare_cli(args) + else: + if sys.argv[3] == "compare": + pass if __name__ == "__main__": diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py index f0e56a609..fa5f8fbaf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py @@ -654,27 +654,18 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) -def compare_cli(args): - with FileOpen(args.input_path, "r") as file: - input_param = json.load(file) - compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) - - def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): - npu_path = input_param.get("npu_path", None) - bench_path = input_param.get("bench_path", None) - if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: - try: - summary_compare, md5_compare = task_dumppath_get(input_param) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) - create_directory(output_path) - check_compare_param(input_param, output_path, summary_compare, md5_compare) - except (CompareException, FileCheckException) as error: - logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) - compare_core(input_param, output_path, stack_mode=stack_mode, - auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) + try: + summary_compare, md5_compare = task_dumppath_get(input_param) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + create_directory(output_path) + check_compare_param(input_param, output_path, summary_compare, md5_compare) + except (CompareException, FileCheckException) as error: + logger.error('Compare failed. Please check the arguments and do it again!') + sys.exit(error.code) + compare_core(input_param, output_path, stack_mode=stack_mode, + auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) def compare_core(input_parma, output_path, **kwargs): diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py new file mode 100644 index 000000000..07e3b7710 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py @@ -0,0 +1,21 @@ +import json +from msprobe.core.common.file_check import FileOpen, check_file_type +from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.utils import CompareException +from msprobe.pytorch.compare.acc_compare import compare +from msprobe.pytorch.compare.distributed_compare import compare_distributed + + +def compare_cli(args): + with FileOpen(args.input_path, "r") as file: + input_param = json.load(file) + npu_path = input_param.get("npu_path", None) + bench_path = input_param.get("bench_path", None) + if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: + compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, + fuzzy_match=args.fuzzy_match) + elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: + compare_distributed(npu_path, bench_path, args.output_path) + else: + logger.error("The npu_path and bench_path need to be of the same type.") + raise CompareException(CompareException.INVALID_COMPARE_MODE) -- Gitee From 1943abaf50dfaf2ea8c9a569aa65101c0fea254c Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 6 Aug 2024 09:47:46 +0800 Subject: [PATCH 126/791] =?UTF-8?q?=E8=B6=85=E8=BF=87=E6=BA=A2=E5=87=BA?= =?UTF-8?q?=E6=AC=A1=E6=95=B0=E5=85=88=E8=90=BD=E7=9B=98=E5=86=8D=E9=80=80?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/data_dump/data_collector.py | 6 +++--- .../core/data_dump/data_processor/base.py | 3 --- .../data_processor/mindspore_processor.py | 17 +++++++++-------- .../data_processor/pytorch_processor.py | 18 ++++++++++-------- 4 files changed, 22 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index db437539a..aa93a1299 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -90,7 +90,7 @@ class DataCollector: if self.config.level == "L2": return self.data_writer.update_stack(self.data_processor.analyze_api_call_stack(name)) - if self.data_processor.stop_run(): + if self.data_processor.is_terminated: self.handle_data(name, data_info, use_buffer=False) raise Exception("[msprobe] exit") self.handle_data(name, data_info) @@ -101,7 +101,7 @@ class DataCollector: return data_info = self.data_processor.analyze_backward(name, module, module_input_output) - if self.data_processor.stop_run(): + if self.data_processor.is_terminated: self.handle_data(name, data_info, use_buffer=False) raise Exception("[msprobe] exit") self.handle_data(name, data_info) @@ -112,7 +112,7 @@ class DataCollector: self.data_writer.update_construct(self.module_processor.module_node) def handle_data(self, name, data_info, use_buffer=True): - msg = f"msProbe is collecting data on {name}. " + msg = f"msprobe is collecting data on {name}. " if data_info: msg = self.update_data(data_info, msg) logger.info(msg) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 2fbc86b56..765503061 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -234,6 +234,3 @@ class BaseDataProcessor: suffix + file_format) file_path = os.path.join(self.data_writer.dump_tensor_data_dir, dump_data_name) return dump_data_name, file_path - - def stop_run(self): - return False diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index c208df7d9..d8f7093fe 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -156,6 +156,15 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): self.cached_tensors_and_file_paths = {} self.real_overflow_dump_times = 0 self.overflow_nums = config.overflow_nums + + @property + def is_terminated(self): + if self.overflow_nums == -1: + return False + if self.real_overflow_dump_times >= self.overflow_nums: + logger.warning(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_times}") + return True + return False def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False @@ -178,14 +187,6 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): self.real_overflow_dump_times += 1 self.cached_tensors_and_file_paths = {} - def stop_run(self): - if self.overflow_nums == -1: - return False - if self.real_overflow_dump_times >= self.overflow_nums: - logger.warning(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_times}") - return True - return False - def _analyze_maybe_overflow_tensor(self, tensor_json): if tensor_json['Max'] is None: return diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 007fec809..f8bf38119 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -187,6 +187,15 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): self.overflow_nums = config.overflow_nums self.bits_for_overflow = 8 + @property + def is_terminated(self): + if self.overflow_nums == -1: + return False + if self.real_overflow_dump_times >= self.overflow_nums: + logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_times}") + return True + return False + @staticmethod def overflow_debug_mode_enable(): overflow_mode = os.getenv(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) @@ -209,16 +218,9 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): for file_path, tensor in self.cached_tensors_and_file_paths.items(): torch.save(tensor, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - self.inc_and_check_overflow_times() + self.real_overflow_dump_times += 1 self.cached_tensors_and_file_paths = {} - def inc_and_check_overflow_times(self): - self.real_overflow_dump_times += 1 - if self.overflow_nums == -1: - return - if self.real_overflow_dump_times >= self.overflow_nums: - raise MsprobeException(MsprobeException.OVERFLOW_NUMS_ERROR, str(self.real_overflow_dump_times)) - def check_overflow_npu(self): if self.overflow_debug_mode_enalbe(): float_status = torch.zeros(self.bits_for_overflow).npu() -- Gitee From 42c8654889a29c1b19ed99cd7b8fb7a088761391 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 6 Aug 2024 10:25:01 +0800 Subject: [PATCH 127/791] renew --- .../msprobe/core/data_dump/data_processor/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 765503061..e15000008 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -69,6 +69,10 @@ class BaseDataProcessor: @property def data_path(self): return self.data_writer.dump_tensor_data_dir + + @property + def is_terminated(self): + return False @staticmethod def analyze_api_call_stack(name): -- Gitee From 695e3b86ba4da1218abd98c7c9dbf34e16761755 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Mon, 5 Aug 2024 11:26:04 +0800 Subject: [PATCH 128/791] mindspore free benchmark --- .../msprobe/core/common/const.py | 74 +++++++++++ .../msprobe/core/common_config.py | 8 ++ .../mindspore/debugger/debugger_config.py | 26 +++- .../mindspore/debugger/precision_debugger.py | 11 +- .../mindspore/free_benchmark/__init__.py | 0 .../free_benchmark/api_pynative_self_check.py | 116 ++++++++++++++++++ .../free_benchmark/common/__init__.py | 0 .../mindspore/free_benchmark/common/config.py | 12 ++ .../free_benchmark/common/handler_params.py | 17 +++ .../mindspore/free_benchmark/common/utils.py | 71 +++++++++++ .../free_benchmark/data/support_wrap_ops.yaml | 0 .../free_benchmark/decorator/__init__.py | 0 .../free_benchmark/decorator/dec_forward.py | 42 +++++++ .../decorator/decorator_factory.py | 108 ++++++++++++++++ .../free_benchmark/handler/__init__.py | 0 .../free_benchmark/handler/base_handler.py | 91 ++++++++++++++ .../free_benchmark/handler/check_handler.py | 41 +++++++ .../free_benchmark/handler/fix_handler.py | 36 ++++++ .../free_benchmark/handler/handler_factory.py | 21 ++++ .../free_benchmark/perturbation/add_noise.py | 67 ++++++++++ .../perturbation/base_perturbation.py | 21 ++++ .../free_benchmark/perturbation/bit_noise.py | 63 ++++++++++ .../perturbation/improve_precision.py | 34 +++++ .../free_benchmark/perturbation/no_change.py | 12 ++ .../perturbation/perturbation_factory.py | 27 ++++ .../free_benchmark/self_check_tool_factory.py | 33 +++++ .../msprobe/mindspore/ms_config.py | 26 +++- .../msprobe/mindspore/runtime.py | 4 + .../msprobe/mindspore/task_handler_factory.py | 11 +- .../test/mindspore_ut/test_ms_config.py | 7 +- .../mindspore_ut/test_task_handler_factory.py | 4 +- 31 files changed, 972 insertions(+), 11 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/handler_params.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/check_handler.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/fix_handler.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/no_change.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/runtime.py diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index c1a453a21..6a262d4be 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -1,6 +1,11 @@ import os import stat + import numpy as np +try: + import mindspore as ms +except ImportError: + pass class Const: @@ -267,3 +272,72 @@ class MsConst: PYNATIVE_MODE = "pynative" GRAPH_GE_MODE = "graph_ge" GRAPH_KBYK_MODE = "graph_kbyk" + + +class MsFreeBenchmarkConst: + DEFAULT_DEVICE = "npu" + DEFAULT_STAGE = "forward" + DEFAULT_DUMP_LEVEL = "L1" + DEFAULT_PERT_TYPE = "improve_precision" + DEFAULT_HANDLER_TYPE = "check" + FIX_HANDLER_MODE = "fix" + ADD_NOISE = "add_noise" + BIT_NOISE = "bit_noise" + NO_CHANGE = "no_change", + IMPROVE_PRECISION = "improve_precision" + CHECK = "check" + FIX = "fix" + DEVICE_LIST = ["npu"] + STAGE_LIST = ["forward"] + DUMP_LEVEL_LIST = ["L1"] + PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE] + HANDLER_TYPE_LIST = [CHECK, FIX] + COMMUNICATION_API_LIST = [ + "mindspore.communication.comm_func.all_gather_into_tensor", + "mindspore.communication.comm_func.gather_into_tensor", + "mindspore.communication.comm_func.all_reduce", + "mindspore.communication.comm_func.reduce", + "mindspore.communication.comm_func.reduce_scatter_tensor" + ] + NO_CHANGE_ERROR_THRESHOLD = 1.0 + SYMBOL_FLIPPING_RATIO = 8.0 + OPS_PREFIX = "mindspore.ops." + Tensor_PREFIX = "mindspore.Tensor." + MINT_PREFIX = "mindspore.mint." + MINT_NN_FUNC_PREFIX = "mindspore.mint.nn.functional." + COMM_PREFIX = "mindspore.communication.comm_func." + + API_PREFIX_DICT = { + "ops": OPS_PREFIX, + "Tensor": Tensor_PREFIX, + "mint": MINT_PREFIX, + "mint.nn.functional": MINT_NN_FUNC_PREFIX, + "communication": COMM_PREFIX + } + + PERT_VALUE_DICT = { + ms.bfloat16: 1e-4, + ms.float16: 1e-6, + ms.float32: 1e-8, + ms.float64: 1e-16 + } + + ERROR_THRESHOLD = { + ms.float16: 1.002, + ms.float32: 1.0002 + } + + PERT_BIT_DICT = { + ms.float16: np.int16, + ms.float32: np.int32, + ms.float64: np.int64 + } + + MS_NUMPY_DTYPE_DICT = { + ms.int16: np.int16, + ms.int32: np.int32, + ms.int64: np.int64, + ms.float16: np.float16, + ms.float32: np.float32, + ms.float64: np.float64 + } diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index d6c15e101..688734be8 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -50,6 +50,14 @@ class BaseConfig: self.summary_mode = json_config.get("summary_mode") self.overflow_nums = json_config.get("overflow_nums") self.check_mode = json_config.get("check_mode") + self.fuzz_device = json_config.get("fuzz_device") + self.pert_mode = json_config.get("pert_mode") + self.handler_type = json_config.get("handler_type") + self.fuzz_level = json_config.get("fuzz_level") + self.fuzz_stage = json_config.get("fuzz_stage") + self.if_preheat = json_config.get("if_preheat") + self.preheat_step = json_config.get("preheat_step") + self.max_sample = json_config.get("max_sample") def check_config(self): if self.scope is not None and not isinstance(self.scope, list): diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 23cb7294b..b5c23ddf0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -1,11 +1,13 @@ import os +from pathlib import Path -from msprobe.core.common.utils import Const -from msprobe.core.common.const import MsConst +from msprobe.core.common.const import Const, MsConst, MsFreeBenchmarkConst +from msprobe.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create class DebuggerConfig: def __init__(self, common_config, task_config): + self.execution_mode = None self.dump_path = common_config.dump_path self.task = common_config.task self.rank = [] if not common_config.rank else common_config.rank @@ -23,6 +25,19 @@ class DebuggerConfig: self.framework = Const.MS_FRAMEWORK self.summary_mode = task_config.summary_mode self.check() + self._make_dump_path_if_not_exists() + + if self.task == Const.FREE_BENCHMARK: + self.pert_type = (MsFreeBenchmarkConst.DEFAULT_PERT_TYPE + if not task_config.pert_mode else task_config.pert_mode) + self.handler_type = (MsFreeBenchmarkConst.DEFAULT_HANDLER_TYPE + if not task_config.handler_type else task_config.handler_type) + if self.handler_type == MsFreeBenchmarkConst.FIX_HANDLER_MODE and \ + self.pert_type != MsFreeBenchmarkConst.DEFAULT_PERT_TYPE: + raise ValueError("pert_mode must be improve_precision or empty when handler_type is fix, " + f"but got {self.pert_type}.") + self.dump_level = MsFreeBenchmarkConst.DEFAULT_DUMP_LEVEL + self.stage = MsFreeBenchmarkConst.DEFAULT_STAGE def check(self): if not self.dump_path: @@ -50,3 +65,10 @@ class DebuggerConfig: for s in self.step: if not isinstance(s, int): raise ValueError(f"step element {s} should be int") + + def _make_dump_path_if_not_exists(self): + check_path_before_create(self.dump_path) + if not os.path.exists(self.dump_path): + Path(self.dump_path).mkdir(mode=0o750, exist_ok=True) + file_check = FileChecker(self.dump_path, FileCheckConst.DIR) + file_check.common_check() diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 5475dc358..fb2b906ce 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -6,7 +6,8 @@ from msprobe.mindspore.service import Service from msprobe.mindspore.ms_config import parse_json_config from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.task_handler_factory import TaskHandlerFactory -from msprobe.core.common.const import MsConst +from msprobe.core.common.const import Const, MsConst +from msprobe.mindspore.runtime import Runtime class PrecisionDebugger: @@ -29,6 +30,8 @@ class PrecisionDebugger: common_config, task_config = parse_json_config(config_path) self.config = DebuggerConfig(common_config, task_config) self.initialized = True + Runtime.step_count = 0 + Runtime.is_running = False @staticmethod def _get_execution_mode(): @@ -47,7 +50,8 @@ class PrecisionDebugger: raise Exception("No instance of PrecisionDebugger found.") instance.config.execution_mode = instance._get_execution_mode() - if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.level == MsConst.API: + if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.level == MsConst.API and \ + instance.config.task != Const.FREE_BENCHMARK: if not instance.service: instance.service = Service(instance.config) instance.service.start() @@ -57,6 +61,7 @@ class PrecisionDebugger: handler.handle() instance.first_start = True + Runtime.is_running = True @classmethod def stop(cls): @@ -65,6 +70,7 @@ class PrecisionDebugger: raise Exception("PrecisionDebugger instance is not created.") if instance.service: instance.service.stop() + Runtime.is_running = False @classmethod def step(cls): @@ -73,3 +79,4 @@ class PrecisionDebugger: raise Exception("PrecisionDebugger instance is not created.") if instance.service: instance.service.step() + Runtime.step_count += 1 diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/__init__.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py new file mode 100644 index 000000000..7f59556dc --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -0,0 +1,116 @@ +import os +import inspect +import importlib + +import yaml +import mindspore as ms +from mindspore.communication import comm_func + +from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.core.common.file_check import check_path_length, FileOpen +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.mindspore.free_benchmark.decorator.decorator_factory import decorate_forward_function + + +class ApiPyNativeSelFCheck: + def __init__(self, config: DebuggerConfig): + Config.is_enable = True + Config.handler_type = config.handler_type + Config.pert_type = config.pert_type + Config.stage = config.stage + Config.dump_level = config.dump_level + Config.steps = config.step + Config.ranks = config.rank + Config.dump_path = os.path.join(config.dump_path, "free_benchmark.csv") + check_path_length(Config.dump_path) + + self.api_list = config.list + all_api = get_supported_ops() + if not self.api_list: + self.api_list = all_api + else: + self.api_list = set(self.api_list) & all_api + + def handle(self): + for api_name in self.api_list: + hijack(api_name) + + +def get_supported_ops(): + supported_ops = [] + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(cur_path, "./data/support_wrap_ops.yaml") + + for k, v in MsFreeBenchmarkConst.API_PREFIX_DICT.items(): + with FileOpen(yaml_path, 'r') as f: + ops = yaml.safe_load(f).get(k) + if ops: + ops = [v + i for i in ops] + supported_ops += ops + + _all_functional_ops = [] + ms_ops = dir(ms.ops) + ms_ops = [MsFreeBenchmarkConst.OPS_PREFIX + i for i in ms_ops] + _all_functional_ops += ms_ops + + _all_functional_ops = [] + ms_tensor = dir(ms.Tensor) + ms_tensor = [MsFreeBenchmarkConst.Tensor_PREFIX + i for i in ms_tensor] + _all_functional_ops += ms_tensor + + ms_mint = dir(ms.mint) + ms_mint = [MsFreeBenchmarkConst.MINT_PREFIX + i for i in ms_mint] + _all_functional_ops += ms_mint + + ms_mint_nn_func = dir(ms.mint.nn.functional) + ms_mint_nn_func = [MsFreeBenchmarkConst.MINT_NN_FUNC_PREFIX + i for i in ms_mint_nn_func] + _all_functional_ops += ms_mint_nn_func + + ms_communication = dir(comm_func) + ms_communication = [MsFreeBenchmarkConst.COMM_PREFIX + i for i in ms_communication] + _all_functional_ops += ms_communication + + return set(supported_ops) & set(_all_functional_ops) + + +def get_decorate_func(): + return decorate_forward_function + + +def is_func_support_decorate(orig_func): + return not inspect.isclass(orig_func) and callable(orig_func) + + +def get_wrapper_obj(orig_func, api_name): + if is_func_support_decorate(orig_func): + wrapped_obj = get_decorate_func()(orig_func, api_name) + else: + wrapped_obj = orig_func + return wrapped_obj + + +def get_module(api_name): + func_name_list = api_name.split('.') + func_name = func_name_list[-1] + module_obj = importlib.import_module(func_name_list[0]) + for i, module_name in enumerate(func_name_list[1:-1]): + if not hasattr(module_obj, module_name): + importlib.import_module(f"{'.'.join(func_name_list[:i+2])}") + module_obj = getattr(module_obj, module_name) + orig_func = getattr(module_obj, func_name) + + return module_obj, orig_func + + +def hijack(api_name): + if len(api_name.strip()) == 0: + return + try: + func_name = api_name.split('.')[-1] + module_obj, origin_func = get_module(api_name) + wrapped_obj = get_wrapper_obj(origin_func, api_name) + setattr(module_obj, func_name, wrapped_obj) + except Exception as e: + logger.error(f"Failed decorator {api_name}: {e}") diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/__init__.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py new file mode 100644 index 000000000..4a22e203d --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py @@ -0,0 +1,12 @@ +from msprobe.core.common.const import MsFreeBenchmarkConst + + +class Config: + is_enable: bool = False + handler_type = MsFreeBenchmarkConst.DEFAULT_HANDLER_TYPE + pert_type = MsFreeBenchmarkConst.DEFAULT_PERT_TYPE + stage = MsFreeBenchmarkConst.DEFAULT_STAGE + dump_level = MsFreeBenchmarkConst.DEFAULT_DUMP_LEVEL + steps: list = [] + ranks: list = [] + dump_path: str = "" diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/handler_params.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/handler_params.py new file mode 100644 index 000000000..ae1733b98 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/handler_params.py @@ -0,0 +1,17 @@ +from typing import Optional, Any, Tuple, Dict, Callable + + +class HandlerParams: + """ + 参数结合体 + + """ + args: Optional[Tuple] = None + kwargs: Optional[Dict] = None + index: Optional[int] = None + original_result: Optional[Any] = None + fuzzed_result: Optional[Any] = None + is_consistent: Optional[bool] = True + save_flag: Optional[bool] = True + fuzzed_value: Optional[Any] = None + original_func: Optional[Callable] = None diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py new file mode 100644 index 000000000..3cc0f0789 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py @@ -0,0 +1,71 @@ +from typing import Any +from typing import Optional +from dataclasses import dataclass + +import mindspore as ms +from mindspore import Tensor + +from msprobe.mindspore.runtime import Runtime +from msprobe.core.common.const import MsFreeBenchmarkConst +from .config import Config +from .handler_params import HandlerParams + + +class Tools: + + @staticmethod + def get_first_tensor_dtype(tensor_seq: Any): + if isinstance(tensor_seq, Tensor): + return tensor_seq.dtype + if isinstance(tensor_seq, (list, tuple)): + for i in tensor_seq: + if isinstance(i, Tensor): + return i.dtype + raise Exception("The sequence does not contain tensors.") + + @staticmethod + def get_default_error_threshold(dtype): + if Config.pert_type == MsFreeBenchmarkConst.NO_CHANGE: + return MsFreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + return MsFreeBenchmarkConst.ERROR_THRESHOLD.get(dtype, MsFreeBenchmarkConst.ERROR_THRESHOLD.get(ms.float32)) + + +@dataclass +class UnequalRow: + rank: Optional[int] = None + pert_type: Optional[str] = None + stage: Optional[str] = None + step: Optional[int] = None + api_name: Optional[str] = None + max_rel: Optional[float] = None + dtype: Optional[str] = None + shape: Optional[str] = None + output_index: Optional[int] = None + + +def make_unequal_row( + api_name: str, + params: HandlerParams, + ratio: float = None, + index: int = None, +): + row = UnequalRow( + api_name=api_name, + pert_type=Config.pert_type, + output_index=index, + stage=Config.stage, + step=Runtime.step_count + ) + if isinstance(ratio, float): + row.max_rel = ratio - 1 + original_tensor = params.original_result + fuzzed_tensor = params.fuzzed_result + if index: + original_tensor = original_tensor[index] + fuzzed_tensor = fuzzed_tensor[index] + row.output_index = index + if isinstance(original_tensor, Tensor): + row.dtype = original_tensor.dtype + row.shape = original_tensor.shape + row.rank = Runtime.rank_id if Runtime.rank_id != -1 else None + return row diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/__init__.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py new file mode 100644 index 000000000..f745f711c --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py @@ -0,0 +1,42 @@ +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.mindspore.free_benchmark.handler.handler_factory import HandlerFactory +from msprobe.mindspore.free_benchmark.perturbation.perturbation_factory import PerturbationFactory + + +class ForwardSelfChecker: + + def __init__(self, api_name: str): + self.api_name = api_name + + def handle(self, params: HandlerParams): + """ + 装饰器实际执行逻辑 + + """ + perturbation = PerturbationFactory.create(self.api_name) + params.fuzzed_result = perturbation.handle(params) + params.original_result = params.original_func(*params.args, **params.kwargs) + if params.fuzzed_result is not False: + return self.deal_fuzzed_and_original_result(params) + return params.original_result + + def get_compare_data(self, params: HandlerParams): + if self.api_name not in MsFreeBenchmarkConst.COMMUNICATION_API_LIST: + return + # 以下为通讯类api处理逻辑 + params.fuzzed_result = params.fuzzed_value + if Config.pert_type == MsFreeBenchmarkConst.IMPROVE_PRECISION: + params.original_result = params.args + else: + params.original_result = params.args[params.index] + + def deal_fuzzed_and_original_result(self, params: HandlerParams): + original_result = params.original_result + self.get_compare_data(params) + handler = HandlerFactory.create(self.api_name) + result = handler.handle(params) + if self.api_name in MsFreeBenchmarkConst.COMMUNICATION_API_LIST: + result = original_result + return result diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py new file mode 100644 index 000000000..d5aeac5d4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py @@ -0,0 +1,108 @@ +import os +import sys +import traceback +from functools import wraps +from typing import Tuple, Dict, List + +from mindspore import ops + +from msprobe.mindspore.runtime import Runtime +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from .dec_forward import ForwardSelfChecker + + +def decorate(original_func, decorate_func, api_name=None): + """ + 总装饰器 + """ + @wraps(original_func) + def fuzz_wrapper(*args, **kwargs): + + def __exec_decorate_func(): + params = data_pre_deal(api_name, original_func, *args, **kwargs) + result = decorate_func(params) + return result + + try: + if Runtime.rank_id == -1: + Runtime.rank_id = os.environ.get("RANK_ID", -1) + if need_wrapper_func(api_name): + logger.info(f"[{api_name}] is checking.") + return __exec_decorate_func() + except Exception as e: + logger.error(f"[{api_name}] Error: {str(e)}") + logger.error(f"[{api_name}] Error detail: {traceback.format_exc()}") + + return original_func(*args, **kwargs) + + return fuzz_wrapper + + +def decorate_forward_function(func, api_name=None): + """ + 前向装饰器 + """ + + if not api_name: + api_name = func.__name__ + + def forward_func(params: HandlerParams): + forward = ForwardSelfChecker(api_name) + result = forward.handle(params) + return result + + return decorate(func, forward_func, api_name) + + +def stack_depth_check(api_name) -> bool: + nested_depth = 1 + frame = sys._getframe(1) + while frame: + if frame.f_code.co_name == "fuzz_wrapper": + nested_depth -= 1 + if nested_depth < 0: + logger.warning(f"[{api_name}] Stack full. Exit staking.") + return False + frame = frame.f_back + return True + + +def get_target_arg_index(args: Tuple) -> int: + """ + 类型校验 + + """ + for i, arg in enumerate(args): + if ops.is_tensor(arg): + if not ops.is_floating_point(arg): + continue + return i + if isinstance(arg, (List, Tuple, Dict)): + return i + return -1 + + +def data_pre_deal(api_name, func, *args, **kwargs): + params = HandlerParams() + params.args = args + params.kwargs = kwargs + params.original_func = func + index = get_target_arg_index(args) + if index == -1: + raise Exception(f"{api_name} has no supported input type") + params.index = index + return params + + +def need_wrapper_func(api_name): + if not (Runtime.is_running and Config.is_enable): + return False + if not stack_depth_check(api_name): + return False + if Config.steps and Runtime.step_count not in Config.steps: + return False + if Config.ranks and Runtime.rank_id != -1 and Runtime.rank_id not in Config.ranks: + return False + return True diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/__init__.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py new file mode 100644 index 000000000..85189a206 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py @@ -0,0 +1,91 @@ +import math +from abc import ABC, abstractmethod +from typing import Any, Tuple, Optional + +import mindspore as ms +from mindspore import Tensor, ops + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.utils import Tools +from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class BaseHandler(ABC): + + def __init__(self, api_name: str): + self.api_name = api_name + + @staticmethod + def pre_calculate(original_output, fuzzed_output): + abs_tol = MsFreeBenchmarkConst.PERT_VALUE_DICT.get(fuzzed_output.dtype, + MsFreeBenchmarkConst.PERT_VALUE_DICT.get(ms.float32)) + + return original_output.to(fuzzed_output.dtype), fuzzed_output, abs_tol + + @staticmethod + def get_threshold(dtype): + err = Tools.get_default_error_threshold(dtype) + return err + + @staticmethod + def convert_overflow_ratio_to_consistent(ratio): + if math.isnan(ratio) or math.isinf(ratio): + return MsFreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + return ratio + + @staticmethod + def get_endless_norm(first_tensor, second_tensor, abs_tol): + try: + ratio_tensor1 = ops.where(ops.abs(second_tensor) > abs_tol, ops.div(first_tensor, second_tensor), 1) + ratio_tensor2 = ops.where(ops.abs(first_tensor) > abs_tol, ops.div(second_tensor, first_tensor), 1) + except Exception as e: + logger.error(str(e)) + ratio_tensor1 = ops.where(ops.abs(second_tensor).to(ms.float32) > abs_tol, + ops.div(first_tensor.to(ms.float32), second_tensor.to(ms.float32)), 1) + ratio_tensor2 = ops.where(ops.abs(first_tensor).to(ms.float32) > abs_tol, + ops.div(second_tensor.to(ms.float32), first_tensor.to(ms.float32)), 1) + norm1 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor1)[0].to(ms.float32).item()) + norm2 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor2)[0].to(ms.float32).item()) + norm3 = BaseHandler.convert_overflow_ratio_to_consistent(ops.min(ratio_tensor1)[0].to(ms.float32).item()) + ratio = MsFreeBenchmarkConst.SYMBOL_FLIPPING_RATIO if norm3 < 0 else max(norm1, norm2) + + return ratio + + @staticmethod + def ratio_calculate(original_output, fuzzed_output) -> float: + try: + original_output, fuzzed_output, abs_tol = BaseHandler.pre_calculate(original_output, fuzzed_output) + except Exception as e: + logger.error(f"When computing ratio, y1 or y2 dtype is not supported {str(e)}") + return MsFreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + + abs_tol = abs_tol ** 0.5 + + return BaseHandler.get_endless_norm(original_output, fuzzed_output, abs_tol) + + @staticmethod + def npu_compare(original_output, fuzzed_output) -> Tuple[bool, Optional[float]]: + if not isinstance(fuzzed_output, Tensor): + logger.error(f"The compare for output type `{type(fuzzed_output)}` is not supported") + return True, 1.0 + + # 范数计算等 + err_thd = BaseHandler.get_threshold(original_output.dtype) + ratio = BaseHandler.ratio_calculate(original_output, fuzzed_output) + is_consistent = err_thd >= ratio >= 1.0 / err_thd + return is_consistent, ratio + + @staticmethod + def is_float_tensor(output) -> bool: + if isinstance(output, Tensor) and ops.is_floating_point(output): + return True + if isinstance(output, (list, tuple)): + for i in output: + if isinstance(i, Tensor) and ops.is_floating_point(i): + return True + return False + + @abstractmethod + def handle(self, params: HandlerParams) -> Any: + pass diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/check_handler.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/check_handler.py new file mode 100644 index 000000000..df80e76c0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/check_handler.py @@ -0,0 +1,41 @@ +from typing import Any +from dataclasses import asdict + +from mindspore import Tensor, ops + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.mindspore.free_benchmark.handler.base_handler import BaseHandler +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.mindspore.free_benchmark.common.utils import make_unequal_row +from msprobe.core.data_dump.json_writer import DataWriter + + +class CheckHandler(BaseHandler): + + def npu_compare_and_save(self, original_output, fuzzed_output, params: HandlerParams, output_index=None): + is_consistent, ratio = self.npu_compare(original_output, fuzzed_output) + params.is_consistent = params.is_consistent and is_consistent + if not is_consistent: + row = make_unequal_row(self.api_name, params, ratio, output_index) + data_dict = asdict(row) + DataWriter.write_data_to_csv( + data_dict.values(), + data_dict.keys(), + Config.dump_path + ) + logger.error(f"{self.api_name} is not consistent") + + def handle(self, params: HandlerParams) -> Any: + try: + if not self.is_float_tensor(params.fuzzed_result): + return params.original_result + if isinstance(params.fuzzed_result, Tensor): + self.npu_compare_and_save(params.original_result, params.fuzzed_result, params) + elif isinstance(params.fuzzed_result, (list, tuple)): + for i, item in enumerate(params.original_result): + if ops.is_tensor(item) and ops.is_floating_point(item): + self.npu_compare_and_save(item, params.fuzzed_result[i], params, output_index=i) + except Exception as e: + logger.error(str(e)) + return params.original_result diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/fix_handler.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/fix_handler.py new file mode 100644 index 000000000..2c377ba89 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/fix_handler.py @@ -0,0 +1,36 @@ +from typing import Any + +from mindspore import Tensor + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class FixHandler: + + def __init__(self, api_name: str): + self.api_name = api_name + + @staticmethod + def use_fuzzed_result(original_result, fuzzed_result): + if isinstance(original_result, Tensor): + return fuzzed_result.to(original_result.dtype) + if isinstance(original_result, dict): + dict_fixed_result = dict() + for k, v in original_result.items(): + dict_fixed_result[k] = FixHandler.use_fuzzed_result(v, fuzzed_result[k]) + return dict_fixed_result + if isinstance(original_result, (tuple, list)): + list_fixed_result = list() + for i, v in enumerate(original_result): + list_fixed_result.append(FixHandler.use_fuzzed_result(v, fuzzed_result[i])) + return type(original_result)(list_fixed_result) + return original_result + + def handle(self, params: HandlerParams) -> Any: + try: + return FixHandler.use_fuzzed_result(params.original_result, params.fuzzed_result) + except Exception as e: + logger.error(f"{self.api_name} failed to fix.") + logger.error(str(e)) + return params.original_result diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py new file mode 100644 index 000000000..8d709cb0d --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py @@ -0,0 +1,21 @@ +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.core.common.const import MsFreeBenchmarkConst +from .check_handler import CheckHandler +from .fix_handler import FixHandler + + +class HandlerFactory: + result_handlers = { + MsFreeBenchmarkConst.CHECK: CheckHandler, + MsFreeBenchmarkConst.FIX: FixHandler, + } + + @staticmethod + def create(api_name: str): + handler = HandlerFactory.result_handlers.get(Config.handler_type) + if handler: + return handler(api_name) + else: + logger.error(f"{Config.handler_type} is not supported.") + raise Exception diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py new file mode 100644 index 000000000..28969e453 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py @@ -0,0 +1,67 @@ +from typing import Any + +from mindspore import Tensor, ops + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.core.common.const import MsFreeBenchmarkConst + + +class AddNoisePerturbation(BasePerturbation): + + def handle(self, params: HandlerParams) -> Any: + """ + 返回增加扰动后的api输出 + + """ + params.fuzzed_value = self.add_noise(params.args[params.index]) + if not self.is_fuzzed: + logger.warning(f"{self.api_name} can not add noise.") + return False + return self.get_fuzzed_result(params) + + def add_noise(self, inputs) -> Any: + """ + 返回增加扰动后的api输入 + + """ + if isinstance(inputs, Tensor): + noise = self._get_noise(inputs) + if noise: + result = ops.where(ops.abs(inputs) > self.perturbation_value ** 0.5, + ops.add(noise, inputs), inputs) + result = result.type(dtype=inputs.dtype) + self.is_fuzzed = True + return result + + if isinstance(inputs, dict): + return {k: self.add_noise(v) for k, v in inputs.items()} + + if isinstance(inputs, (list, tuple)): + return [self.add_noise(v) for v in inputs] + + return inputs + + def _get_noise(self, input): + """ + 得到要添加的噪声值 + + """ + if self.is_fuzzed: + return False + if not ops.is_floating_point(input) or ops.numel(input) == 0: + return False + + pert_value = MsFreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype) + if not pert_value: + return False + else: + self.perturbation_value = pert_value + + max_val = ops.max(ops.abs(input))[0].item() + if max_val < pert_value: + return False + + noise = ops.full(input.shape, self.perturbation_value, dtype=input.dtype) + return noise diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py new file mode 100644 index 000000000..becfe2964 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py @@ -0,0 +1,21 @@ +from typing import Any + +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class BasePerturbation: + + def __init__(self, api_name: str): + self.api_name = api_name + self.is_fuzzed = False + self.perturbation_value = None + + @staticmethod + def get_fuzzed_result(params: HandlerParams): + args_front = params.args[:params.index] + args_rear = params.args[params.index + 1:] + fuzzed_result = params.original_func(*args_front, params.fuzzed_value, *args_rear, **params.kwargs) + return fuzzed_result + + def handler(self, params: HandlerParams) -> Any: + pass diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py new file mode 100644 index 000000000..13efb1f37 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py @@ -0,0 +1,63 @@ +from typing import Any + +import numpy as np +from mindspore import Tensor, ops + +from msprobe.mindspore.common.log import logger +from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation + + +class BitNoisePerturbation(BasePerturbation): + + def add_bit_noise(self, inputs) -> Any: + if isinstance(inputs, Tensor): + bit_len_type = self._get_bit_len_type(inputs) + if bit_len_type: + sub_normal_np = np.finfo(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)).smallest_normal + sub_normal = Tensor(sub_normal_np) + noise_type = list(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.keys())[ + list(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.values()).index(bit_len_type)] + noise = ops.full(inputs.shape, 1, dtype=noise_type) + input_np = inputs.asnumpy() + input_np_int = input_np.view(bit_len_type) + result = Tensor(input_np_int) + result = ops.where(ops.abs(inputs) > sub_normal, + ops.bitwise_xor(result, noise), result) + result_np = result.asnumpy() + result_np_float = result_np.view(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)) + self.is_fuzzed = True + return Tensor(result_np_float) + + if isinstance(inputs, dict): + return {k: self.add_bit_noise(v) for k, v in inputs.items()} + if isinstance(inputs, (tuple, list)): + return type(inputs)([self.add_bit_noise(v) for v in inputs]) + return inputs + + def handle(self, params: HandlerParams) -> any: + args = params.args + params.fuzzed_value = self.add_bit_noise(params.args[params.index]) + if not self.is_fuzzed: + logger.warning(f"{self.api_name} can not add bit noise.") + return False + params.args = args + return self.get_fuzzed_result(params) + + def _get_bit_len_type(self, input): + if self.is_fuzzed: + return False + if not isinstance(input, Tensor) or not ops.is_floating_point(input) or \ + input.numel() == 0: + return False + bit_len_type = MsFreeBenchmarkConst.PERT_BIT_DICT.get(input.dtype) + if not bit_len_type: + return False + pert_value = MsFreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype) + if not pert_value: + return False + max_val = ops.max(ops.abs(input))[0].item() + if max_val < pert_value: + return False + return bit_len_type diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py new file mode 100644 index 000000000..c32536187 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py @@ -0,0 +1,34 @@ +from typing import Any + +import mindspore as ms +from mindspore import Tensor, ops + +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.log import logger + + +class ImprovePrecisionPerturbation(BasePerturbation): + + def improve_tensor_precision(self, target_tensor): + if isinstance(target_tensor, Tensor) and ops.is_floating_point(target_tensor) and \ + target_tensor.dtype not in [ms.float64, ms.float32]: + self.is_fuzzed = True + return target_tensor.to(ms.float32) + if isinstance(target_tensor, dict): + return {k: self.improve_tensor_precision(v) for k, v in target_tensor.items()} + if isinstance(target_tensor, (tuple, list)): + return type(target_tensor)([self.improve_tensor_precision(v) for v in target_tensor]) + return target_tensor + + def handle(self, params: HandlerParams) -> Any: + args = self.improve_tensor_precision(params.args) + kwargs = self.improve_tensor_precision(params.kwargs) + fuzzed_value = args + if self.api_name in MsFreeBenchmarkConst.COMMUNICATION_API_LIST: + params.fuzzed_value = fuzzed_value + if not self.is_fuzzed: + logger.warning(f"{self.api_name} can not improve precision.") + return False + return params.original_func(*args, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/no_change.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/no_change.py new file mode 100644 index 000000000..fc844bfd6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/no_change.py @@ -0,0 +1,12 @@ +from typing import Any + +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class NoChangePerturbation(BasePerturbation): + + def handle(self, params: HandlerParams) -> Any: + params.fuzzed_value = params.args[params.index] + self.is_fuzzed = True + return self.get_fuzzed_result(params) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py new file mode 100644 index 000000000..01d1fa6e7 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py @@ -0,0 +1,27 @@ +from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.config import Config +from .add_noise import AddNoisePerturbation +from .bit_noise import BitNoisePerturbation +from .no_change import NoChangePerturbation +from .improve_precision import ImprovePrecisionPerturbation + + +class PerturbationFactory: + """ + 扰动工厂类 + + """ + perturbations = { + MsFreeBenchmarkConst.IMPROVE_PRECISION: ImprovePrecisionPerturbation, + MsFreeBenchmarkConst.ADD_NOISE: AddNoisePerturbation, + MsFreeBenchmarkConst.BIT_NOISE: BitNoisePerturbation, + MsFreeBenchmarkConst.NO_CHANGE: NoChangePerturbation, + } + + @staticmethod + def create(api_name: str): + perturbation = PerturbationFactory.perturbations.get(Config.pert_type) + if perturbation: + return perturbation(api_name) + else: + raise Exception(f'{Config.pert_type} is a invalid perturbation type') diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py new file mode 100644 index 000000000..c9a0d8a65 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py @@ -0,0 +1,33 @@ +from msprobe.core.common.const import MsConst +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelFCheck + + +class SelfCheckToolFactory: + tools = { + MsConst.CELL: { + MsConst.GRAPH_KBYK_MODE: None, + MsConst.GRAPH_GE_MODE: None, + MsConst.PYNATIVE_MODE: None + }, + MsConst.API: { + MsConst.GRAPH_KBYK_MODE: None, + MsConst.GRAPH_GE_MODE: None, + MsConst.PYNATIVE_MODE: ApiPyNativeSelFCheck + }, + MsConst.KERNEL: { + MsConst.GRAPH_KBYK_MODE: None, + MsConst.GRAPH_GE_MODE: None, + MsConst.PYNATIVE_MODE: None + } + } + + @staticmethod + def create(config: DebuggerConfig): + tool = SelfCheckToolFactory.tools.get(config.level) + if not tool: + raise Exception(f"{config.level} is not supported.") + tool = tool.get(config.execution_mode) + if not tool: + raise Exception(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.") + return tool(config) diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index c0ef6bb6c..56da70e16 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -1,7 +1,9 @@ import json + from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, MsFreeBenchmarkConst +from msprobe.mindspore.common.log import logger class TensorConfig(BaseConfig): @@ -51,10 +53,32 @@ class OverflowCheckConfig(BaseConfig): raise Exception("check_mode is invalid") +class FreeBenchmarkConfig(BaseConfig): + def __init__(self, task_config): + super().__init__(task_config) + self._check_config() + + def _check_config(self): + if self.fuzz_device and self.fuzz_device not in MsFreeBenchmarkConst.DEVICE_LIST: + raise Exception("fuzz_device must be npu or empty") + if self.pert_mode and self.pert_mode not in MsFreeBenchmarkConst.PERT_TYPE_LIST: + raise Exception("pert_mode must be improve_precision, add_noise, bit_noise , no_change or empty") + if self.handler_type and self.handler_type not in MsFreeBenchmarkConst.HANDLER_TYPE_LIST: + raise Exception("handler_type must be check or empty") + if self.fuzz_level and self.fuzz_level not in MsFreeBenchmarkConst.DUMP_LEVEL_LIST: + raise Exception("fuzz_level must be L1 or empty") + if self.fuzz_stage and self.fuzz_stage not in MsFreeBenchmarkConst.STAGE_LIST: + raise Exception("fuzz_stage must be forward or empty") + if self.if_preheat or self.preheat_step or self.max_sample: + logger.warning("'if_preheat', 'preheat_step' and 'max_sample' settings " + "are not supported for mindspore free benchmark task.") + + TaskDict = { Const.TENSOR: TensorConfig, Const.STATISTICS: StatisticsConfig, Const.OVERFLOW_CHECK: OverflowCheckConfig, + Const.FREE_BENCHMARK: FreeBenchmarkConfig } diff --git a/debug/accuracy_tools/msprobe/mindspore/runtime.py b/debug/accuracy_tools/msprobe/mindspore/runtime.py new file mode 100644 index 000000000..380b30d97 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/runtime.py @@ -0,0 +1,4 @@ +class Runtime: + step_count: int = 0 + rank_id: int = -1 + is_running: bool = False diff --git a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py index 7b7e6fd88..45fff4cd4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py @@ -1,17 +1,22 @@ +from msprobe.core.common.const import Const, MsConst from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.dump_tool_factory import DumpToolFactory from msprobe.mindspore.overflow_check.overflow_check_tool_factory import OverflowCheckToolFactory +from msprobe.mindspore.free_benchmark.self_check_tool_factory import SelfCheckToolFactory class TaskHandlerFactory: tasks = { - "tensor": DumpToolFactory, - "statistics": DumpToolFactory, - "overflow_check": OverflowCheckToolFactory + Const.TENSOR: DumpToolFactory, + Const.STATISTICS: DumpToolFactory, + Const.OVERFLOW_CHECK: OverflowCheckToolFactory, + Const.FREE_BENCHMARK: SelfCheckToolFactory } @staticmethod def create(config: DebuggerConfig): + if config.execution_mode == MsConst.PYNATIVE_MODE and config.task != Const.FREE_BENCHMARK: + raise Exception("Current Task can't run in pynative mode.") task = TaskHandlerFactory.tasks.get(config.task) if not task: raise Exception("valid task is needed.") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py index 30212d95e..fb408e83b 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py @@ -19,7 +19,7 @@ from unittest.mock import patch, mock_open from msprobe.core.common.const import Const from msprobe.mindspore.ms_config import (parse_json_config, parse_task_config, - TensorConfig, StatisticsConfig, OverflowCheckConfig) + TensorConfig, StatisticsConfig, OverflowCheckConfig, FreeBenchmarkConfig) class TestMsConfig(TestCase): @@ -64,6 +64,9 @@ class TestMsConfig(TestCase): task_config = parse_task_config("overflow_check", mock_json_config) self.assertTrue(isinstance(task_config, OverflowCheckConfig)) + task_config = parse_task_config("free_benchmark", mock_json_config) + self.assertTrue(isinstance(task_config, FreeBenchmarkConfig)) + with self.assertRaises(Exception) as context: - parse_task_config("free_benchmark", mock_json_config) + parse_task_config("unsupported_task", mock_json_config) self.assertEqual(str(context.exception), "task is invalid.") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py index 41be7b1db..699df3bae 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py @@ -21,6 +21,7 @@ from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.kernel_graph_dump import KernelGraphDump from msprobe.mindspore.task_handler_factory import TaskHandlerFactory +from msprobe.core.common.const import MsConst class TestTaskHandlerFactory(TestCase): @@ -43,6 +44,7 @@ class TestTaskHandlerFactory(TestCase): common_config = CommonConfig(json_config) task_config = BaseConfig(json_config) config = DebuggerConfig(common_config, task_config) + config.execution_mode = MsConst.GRAPH_GE_MODE handler = TaskHandlerFactory.create(config) self.assertTrue(isinstance(handler, KernelGraphDump)) @@ -52,7 +54,7 @@ class TestTaskHandlerFactory(TestCase): TaskHandlerFactory.create(config) self.assertEqual(str(context.exception), "Can not find task handler") - config.task = "free_benchmark" + config.task = "Free_benchmark" with self.assertRaises(Exception) as context: TaskHandlerFactory.create(config) self.assertEqual(str(context.exception), "valid task is needed.") -- Gitee From 8781b80be214a46828af172bbe09180cee3d54ec Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 6 Aug 2024 10:30:08 +0800 Subject: [PATCH 129/791] compare_distributed cli add --- debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py index 07e3b7710..9443e5ef0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py @@ -2,6 +2,7 @@ import json from msprobe.core.common.file_check import FileOpen, check_file_type from msprobe.core.common.const import FileCheckConst from msprobe.core.common.utils import CompareException +from msprobe.pytorch.common.log import logger from msprobe.pytorch.compare.acc_compare import compare from msprobe.pytorch.compare.distributed_compare import compare_distributed @@ -15,7 +16,8 @@ def compare_cli(args): compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: - compare_distributed(npu_path, bench_path, args.output_path) + kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} + compare_distributed(npu_path, bench_path, args.output_path, **kwargs) else: logger.error("The npu_path and bench_path need to be of the same type.") raise CompareException(CompareException.INVALID_COMPARE_MODE) -- Gitee From cc348a490bef55299f87957ced81328f438cb7f9 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Tue, 6 Aug 2024 10:45:29 +0800 Subject: [PATCH 130/791] mindspore free benchmark V1.1 --- .../free_benchmark/api_pynative_self_check.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index 7f59556dc..2c4c0c856 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -6,7 +6,7 @@ import yaml import mindspore as ms from mindspore.communication import comm_func -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.core.common.const import Const, MsFreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.config import Config from msprobe.core.common.file_check import check_path_length, FileOpen from msprobe.mindspore.common.log import logger @@ -41,7 +41,7 @@ class ApiPyNativeSelFCheck: def get_supported_ops(): supported_ops = [] cur_path = os.path.dirname(os.path.realpath(__file__)) - yaml_path = os.path.join(cur_path, "./data/support_wrap_ops.yaml") + yaml_path = os.path.join(cur_path, "data", "support_wrap_ops.yaml") for k, v in MsFreeBenchmarkConst.API_PREFIX_DICT.items(): with FileOpen(yaml_path, 'r') as f: @@ -55,7 +55,6 @@ def get_supported_ops(): ms_ops = [MsFreeBenchmarkConst.OPS_PREFIX + i for i in ms_ops] _all_functional_ops += ms_ops - _all_functional_ops = [] ms_tensor = dir(ms.Tensor) ms_tensor = [MsFreeBenchmarkConst.Tensor_PREFIX + i for i in ms_tensor] _all_functional_ops += ms_tensor @@ -92,12 +91,12 @@ def get_wrapper_obj(orig_func, api_name): def get_module(api_name): - func_name_list = api_name.split('.') + func_name_list = api_name.split(Const.SEP) func_name = func_name_list[-1] module_obj = importlib.import_module(func_name_list[0]) for i, module_name in enumerate(func_name_list[1:-1]): if not hasattr(module_obj, module_name): - importlib.import_module(f"{'.'.join(func_name_list[:i+2])}") + importlib.import_module(f"{Const.SEP.join(func_name_list[:i+2])}") module_obj = getattr(module_obj, module_name) orig_func = getattr(module_obj, func_name) @@ -105,10 +104,10 @@ def get_module(api_name): def hijack(api_name): - if len(api_name.strip()) == 0: + if not api_name.strip() == 0: return try: - func_name = api_name.split('.')[-1] + func_name = api_name.split(Const.SEP)[-1] module_obj, origin_func = get_module(api_name) wrapped_obj = get_wrapper_obj(origin_func, api_name) setattr(module_obj, func_name, wrapped_obj) -- Gitee From 895122ccd67c6d3f68dbf3fed5dcab1b90a5a637 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 6 Aug 2024 10:54:23 +0800 Subject: [PATCH 131/791] compare_distributed cli add --- debug/accuracy_tools/msprobe/msprobe.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 2963a52a0..d829d7b9e 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -68,9 +68,6 @@ def main(): _run_overflow_check_command(args) elif sys.argv[3] == "compare": compare_cli(args) - else: - if sys.argv[3] == "compare": - pass if __name__ == "__main__": -- Gitee From b5631743714b77de60c0811618f733f6dadba04c Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 6 Aug 2024 10:54:54 +0800 Subject: [PATCH 132/791] =?UTF-8?q?=E8=A1=A5=E5=85=85=E6=97=A0=E6=A0=87?= =?UTF-8?q?=E6=9D=86=E6=98=BE=E5=AD=98=E4=BC=98=E5=8C=96ut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../result_handlers/test_result_handler.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py index 399efeb42..a0beebec5 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py @@ -15,6 +15,7 @@ from msprobe.pytorch.free_benchmark.common.params import DataParams, make_handle from msprobe.pytorch.free_benchmark.result_handlers.handler_factory import ( FuzzHandlerFactory, ) +from msprobe.pytorch.free_benchmark.result_handlers.base_handler import FuzzHandler class Config(ABC): @@ -119,3 +120,18 @@ class TestFuzzHandler(TestCase): api_threshld, ThresholdConfig.DTYPE_PER_THD[torch.float16] ) + + def test_tensor_split_for_error_calculate(self): + tensor_size = 256 * 1024 * 1024 + origin_output = torch.randn(tensor_size, dtype=torch.float32) + perturbed_output = torch.randn(tensor_size, dtype=torch.float32) + + origin_output_chunks, perturbed_output_chunks = FuzzHandler.tensor_split_for_error_calculate( + origin_output, perturbed_output) + + self.assertEqual(len(origin_output_chunks), 64) + self.assertEqual(len(perturbed_output_chunks), 64) + for chunk in origin_output_chunks: + self.assertEqual(chunk.shape, (4 * 1024 * 1024,)) + for chunk in perturbed_output_chunks: + self.assertEqual(chunk.shape, (4 * 1024 * 1024,)) -- Gitee From f583b1adc4dfc1eb74bc763cf70968e7a2feb68f Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Tue, 6 Aug 2024 09:46:48 +0800 Subject: [PATCH 133/791] =?UTF-8?q?[mstt]=E6=B7=BB=E5=8A=A0=E5=85=AC?= =?UTF-8?q?=E7=BD=91URL?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 2 +- ...\205\254\347\275\221URL\350\257\264\346\230\216.md" | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) create mode 100644 "\345\205\254\347\275\221URL\350\257\264\346\230\216.md" diff --git a/profiler/README.md b/profiler/README.md index 549ffefc1..956764913 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -26,7 +26,7 @@ with torch_npu.profiler.profile( profile_memory=True, with_stack=True, experimental_config=experimental_config, - schedule=torch.profiler.schedule(wait=10, warmup=0, active=1, repeat=1), + schedule=torch_npu.profiler.schedule(wait=10, warmup=0, active=1, repeat=1), on_trace_ready=torch_npu.profiler.tensorboard_trace_handler("./profiling_data") ) as prof: # 模型训练代码 diff --git "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" new file mode 100644 index 000000000..abf8e1055 --- /dev/null +++ "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" @@ -0,0 +1,10 @@ +# 公网URL说明 + +| 软件类型 | 软件名 | 路径 | 类型 | 内容 | 用途说明 | +| -------- | -------------------------------------------------- | ------------------------------------ | -------- | ------------------------------------------------------------ | ----------------------- | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md"] | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html"] | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html"] | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning"] | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html"] | Advisor优化手段参考示例 | + -- Gitee From 5ad83e9faf0573ec74483574fa13716280e7624b Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Tue, 6 Aug 2024 11:32:20 +0800 Subject: [PATCH 134/791] mindspore free benchmark V1.2 --- .../mindspore/free_benchmark/decorator/decorator_factory.py | 1 - debug/accuracy_tools/msprobe/mindspore/ms_config.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py index d5aeac5d4..326758fde 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py @@ -63,7 +63,6 @@ def stack_depth_check(api_name) -> bool: if frame.f_code.co_name == "fuzz_wrapper": nested_depth -= 1 if nested_depth < 0: - logger.warning(f"[{api_name}] Stack full. Exit staking.") return False frame = frame.f_back return True diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index 56da70e16..ad5de0bf3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -64,7 +64,7 @@ class FreeBenchmarkConfig(BaseConfig): if self.pert_mode and self.pert_mode not in MsFreeBenchmarkConst.PERT_TYPE_LIST: raise Exception("pert_mode must be improve_precision, add_noise, bit_noise , no_change or empty") if self.handler_type and self.handler_type not in MsFreeBenchmarkConst.HANDLER_TYPE_LIST: - raise Exception("handler_type must be check or empty") + raise Exception("handler_type must be check, fix or empty") if self.fuzz_level and self.fuzz_level not in MsFreeBenchmarkConst.DUMP_LEVEL_LIST: raise Exception("fuzz_level must be L1 or empty") if self.fuzz_stage and self.fuzz_stage not in MsFreeBenchmarkConst.STAGE_LIST: -- Gitee From 9a89bf7e06cf4fc2be1bcda8b3c49625524cc436 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Tue, 6 Aug 2024 12:36:06 +0800 Subject: [PATCH 135/791] mindspore free benchmark V1.3 --- .../mindspore/free_benchmark/api_pynative_self_check.py | 2 +- .../free_benchmark/decorator/decorator_factory.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index 2c4c0c856..541ba14f4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -104,7 +104,7 @@ def get_module(api_name): def hijack(api_name): - if not api_name.strip() == 0: + if not api_name.strip(): return try: func_name = api_name.split(Const.SEP)[-1] diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py index 326758fde..c1cf50e9c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py @@ -28,7 +28,7 @@ def decorate(original_func, decorate_func, api_name=None): try: if Runtime.rank_id == -1: Runtime.rank_id = os.environ.get("RANK_ID", -1) - if need_wrapper_func(api_name): + if need_wrapper_func(): logger.info(f"[{api_name}] is checking.") return __exec_decorate_func() except Exception as e: @@ -56,7 +56,7 @@ def decorate_forward_function(func, api_name=None): return decorate(func, forward_func, api_name) -def stack_depth_check(api_name) -> bool: +def stack_depth_check() -> bool: nested_depth = 1 frame = sys._getframe(1) while frame: @@ -95,10 +95,10 @@ def data_pre_deal(api_name, func, *args, **kwargs): return params -def need_wrapper_func(api_name): +def need_wrapper_func(): if not (Runtime.is_running and Config.is_enable): return False - if not stack_depth_check(api_name): + if not stack_depth_check(): return False if Config.steps and Runtime.step_count not in Config.steps: return False -- Gitee From 90cf4a300e97169d20e9a47af9aea197f4e726c1 Mon Sep 17 00:00:00 2001 From: zhouyiyan Date: Mon, 5 Aug 2024 20:24:43 +0800 Subject: [PATCH 136/791] 82 --- .../overall_performance_comparator.py | 4 +- .../compare_bean/profiling_info.py | 144 ++++++++---------- .../profiling_parser/gpu_profiling_parser.py | 37 ++--- .../profiling_parser/npu_profiling_parser.py | 44 +++--- .../compare_bean/test_profiling_info.py | 54 +++---- .../test_gpu_profiling_parser.py | 20 +-- 6 files changed, 142 insertions(+), 161 deletions(-) diff --git a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 1c5cee43e..3a94527bb 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -13,11 +13,11 @@ class OverallPerformanceComparator(BaseComparator): base_col = [f'{base_profiling_info.profiling_type}'] comp_col = [f'{comp_profiling_info.profiling_type}'] if base_profiling_info.RDMA_bandwidth or comp_profiling_info.RDMA_bandwidth: - self._headers.extend(['RDMA Bandwidth(GB/s)']) + self._headers.extend(['RDMA Bandwidth']) base_col.append(f'{base_profiling_info.RDMA_bandwidth:.3f}GB/s') comp_col.append(f'{comp_profiling_info.RDMA_bandwidth:.3f}GB/s') if base_profiling_info.SDMA_bandwidth or comp_profiling_info.SDMA_bandwidth: - self._headers.extend(['SDMA Bandwidth(GB/s)']) + self._headers.extend(['SDMA Bandwidth']) base_col.append(f'{base_profiling_info.SDMA_bandwidth:.3f}GB/s') comp_col.append(f'{comp_profiling_info.SDMA_bandwidth:.3f}GB/s') if not base_profiling_info.hide_op_details and not comp_profiling_info.hide_op_details: diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index 10ac47d6e..fe5781426 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -8,20 +8,8 @@ class ProfilingInfo: def __init__(self, profiling_type: str): self.profiling_type = profiling_type - self.other_time = 0.0 - self.lccl_num = 0 - self.compute_time = 0.0 - self.communication_not_overlapped = 0.0 - self.wait_time = 0.0 - self.memory_used = 0.0 - self.e2e_time = 0.0 - self.scheduling_time = 0.0 - self.lccl_time = 0.0 - self.minimal_profiling = False - self.hide_op_details = False - self.is_level0 = False - self.cube_time = 0.0 + self.other_time = 0.0 self.vec_time = 0.0 self.cube_num = 0 self.vec_num = 0 @@ -29,14 +17,26 @@ class ProfilingInfo: self.fa_num_fwd = 0 self.fa_num_bwd = 0 self.pa_num = 0 + self.lccl_num = 0 self.conv_time_fwd = 0.0 self.conv_time_bwd = 0.0 self.conv_num_fwd = 0 self.conv_num_bwd = 0 + self.compute_time = 0.0 + self.communication_not_overlapped = 0.0 + self.wait_time = 0.0 + self.memory_used = 0.0 + self.e2e_time = 0.0 self.sdma_time = 0.0 + self.scheduling_time = 0.0 self.fa_time_bwd = 0.0 self.pa_time = 0.0 + self.lccl_time = 0.0 self.fa_time_fwd = 0.0 + self.minimal_profiling = False + self.hide_op_details = False + self.is_level0 = False + # 性能拆解新指标 self.fa_time_fwd_cube = 0.0 self.fa_num_fwd_cube = 0 @@ -78,6 +78,7 @@ class ProfilingInfo: self.other_cube_num = 0 self.RDMA_bandwidth = 0.0 self.SDMA_bandwidth = 0.0 + @property def e2e_time_ms(self): return self.e2e_time * 10 ** 3 @@ -136,16 +137,24 @@ class ProfilingInfo: @property def vector_total_num(self): return sum((self.vector_num_trans, self.vector_num_notrans)) - def trans_to_s(self): - self.cube_time /= 10 ** 3 - self.vec_time /= 10 ** 3 - self.conv_time_fwd /= 10 ** 3 - self.conv_time_bwd /= 10 ** 3 - self.sdma_time /= 10 ** 3 - self.fa_time_bwd /= 10 ** 3 - self.pa_time /= 10 ** 3 - self.fa_time_fwd /= 10 ** 3 + def trans_time_to_s(self): + self.cube_time = self.cube_time / 10 ** 6 + self.other_time = self.other_time / 10 ** 6 + self.vec_time = self.vec_time / 10 ** 6 + self.compute_time = self.compute_time / 10 ** 6 + self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 + self.wait_time = self.wait_time / 10 ** 6 + self.e2e_time = self.e2e_time / 10 ** 6 + self.sdma_time = self.sdma_time / 10 ** 6 + self.scheduling_time = self.scheduling_time / 10 ** 6 + self.fa_time_bwd = self.fa_time_bwd / 10 ** 6 + self.fa_time_fwd = self.fa_time_fwd / 10 ** 6 + self.pa_time = self.pa_time / 10 ** 6 + self.lccl_time = self.lccl_time / 10 ** 6 + self.conv_time_fwd = self.conv_time_fwd / 10 ** 6 + self.conv_time_bwd = self.conv_time_bwd / 10 ** 6 + # 新指标单位为ms self.fa_time_fwd_cube /= 10 ** 3 self.fa_time_bwd_cube /= 10 ** 3 @@ -163,70 +172,27 @@ class ProfilingInfo: self.sdma_time_stream /= 10 ** 3 self.page_attention_time /= 10 ** 3 self.other_cube_time /= 10 ** 3 - self.other_time = self.other_time / 10 ** 6 - self.compute_time = self.compute_time / 10 ** 6 - self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 - self.wait_time = self.wait_time / 10 ** 6 - self.e2e_time = self.e2e_time / 10 ** 6 - self.scheduling_time = self.scheduling_time / 10 ** 6 - self.lccl_time = self.lccl_time / 10 ** 6 - - def calculate_cube_time(self): - self.cube_time = self.matmul_time_cube + self.matmul_time_vector + self.other_cube_time - - def calculate_vec_time(self): - self.vec_time = self.vector_time_trans + self.vector_time_notrans - - def calculate_cube_num(self): - self.cube_num = self.matmul_num_cube + self.matmul_num_vector + self.other_cube_num - - def calculate_vec_num(self): - self.vec_num = self.vector_num_trans + self.vector_num_notrans - - def calculate_sdma_num(self): - self.sdma_num = self.sdma_num_tensor_move + self.sdma_num_stream - - def calculate_fa_num_fwd(self): - self.fa_num_fwd = self.fa_num_fwd_cube + self.fa_num_fwd_vector - - def calculate_fa_num_bwd(self): - self.fa_num_bwd = self.fa_num_bwd_cube + self.fa_num_bwd_vector - - def calculate_pa_num(self): - self.pa_num = self.page_attention_num - - def calculate_pa_time(self): - self.pa_num = self.page_attention_num - - def calculate_conv_time_fwd(self): - self.conv_time_fwd = self.conv_time_fwd_cube + self.conv_time_fwd_vector - - def calculate_conv_time_bwd(self): - self.conv_time_bwd = self.conv_time_bwd_cube + self.conv_time_bwd_vector - - def calculate_conv_num_fwd(self): - self.conv_num_fwd = self.conv_num_fwd_cube + self.conv_num_fwd_vector - - def calculate_conv_num_bwd(self): - self.conv_num_bwd = self.conv_num_bwd_cube + self.conv_num_bwd_vector - - def calculate_sdma_time(self): - self.sdma_time = self.sdma_time_tensor_move + self.sdma_time_stream - - def calculate_fa_time_fwd(self): - self.fa_time_fwd = self.fa_time_fwd_cube + self.fa_time_fwd_vector - - def calculate_fa_time_bwd(self): - self.fa_time_bwd = self.fa_time_bwd_cube + self.fa_time_bwd_vector def calculate_other_time(self): self.other_time = max( [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - self.pa_time - self.vec_time - self.conv_time_fwd - self.conv_time_bwd]) + def calculate_vec_time(self): + self.vec_time = self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd \ + - self.conv_time_fwd - self.conv_time_bwd + def calculate_schedule_time(self): self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time - self.communication_not_overlapped) + def update_fa_fwd_info(self, time: float): + self.fa_time_fwd += time + self.fa_num_fwd += 1 + + def update_fa_bwd_info(self, time: float): + self.fa_time_bwd += time + self.fa_num_bwd += 1 + def update_fa_fwd_cube_info(self, time: float): self.fa_time_fwd_cube += time self.fa_num_fwd_cube += 1 @@ -251,10 +217,22 @@ class ProfilingInfo: self.sdma_time_stream += time self.sdma_num_stream += num + def update_pa_info(self, time: float): + self.pa_time += time + self.pa_num += 1 + def update_lccl_info(self, time: float): self.lccl_time += time self.lccl_num += 1 + def update_conv_fwd_info(self, time: float): + self.conv_time_fwd += time + self.conv_num_fwd += 1 + + def update_conv_bwd_info(self, time: float): + self.conv_time_bwd += time + self.conv_num_bwd += 1 + def update_conv_bwd_cube_info(self, time: float): self.conv_time_bwd_cube += time self.conv_num_bwd_cube += 1 @@ -291,6 +269,18 @@ class ProfilingInfo: self.vector_time_notrans += time self.vector_num_notrans += 1 + def update_sdma_info(self, time: float, num: int = 1): + self.sdma_time += time + self.sdma_num += num + + def update_cube_info(self, time: float): + self.cube_time += time + self.cube_num += 1 + + def update_vec_info(self, time: float): + self.vec_time += time + self.vec_num += 1 + def update_other_cube_info(self, time: float): self.other_cube_time += time self.other_cube_num += 1 diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index bf5d39846..0aeeba83e 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -61,25 +61,9 @@ class GPUProfilingParser(BaseProfilingParser): def _update_overall_metrics(self): self._calculate_performance_time() self.__parse_memory_reserved() - self._result_data.overall_metrics.trans_time_to_s() - self._result_data.overall_metrics.calculate_cube_time() self._result_data.overall_metrics.calculate_vec_time() - self._result_data.overall_metrics.calculate_cube_num() - self._result_data.overall_metrics.calculate_vec_num() - self._result_data.overall_metrics.calculate_sdma_num() - self._result_data.overall_metrics.calculate_fa_num_fwd() - self._result_data.overall_metrics.calculate_fa_num_bwd() - self._result_data.overall_metrics.calculate_pa_num() - self._result_data.overall_metrics.calculate_pa_time() - self._result_data.overall_metrics.calculate_conv_time_fwd() - self._result_data.overall_metrics.calculate_conv_time_bwd() - self._result_data.overall_metrics.calculate_conv_num_fwd() - self._result_data.overall_metrics.calculate_conv_num_bwd() - self._result_data.overall_metrics.calculate_sdma_time() - self._result_data.overall_metrics.calculate_fa_time_fwd() - self._result_data.overall_metrics.calculate_fa_time_bwd() self._result_data.overall_metrics.calculate_schedule_time() - self._result_data.overall_metrics.trans_to_s() + self._result_data.overall_metrics.trans_time_to_s() def _calculate_performance_time(self): min_ts = sys.float_info.max @@ -92,6 +76,7 @@ class GPUProfilingParser(BaseProfilingParser): min_ts = min(event.start_time, min_ts) max_ts = max(event.end_time, max_ts) if event.stream == self._compute_stream_id and self.__is_sdma_time(event.name): + self._result_data.overall_metrics.update_sdma_info(event.dur) self._result_data.overall_metrics.update_sdma_stream_info(event.dur) continue if not event.is_kernel_cat(): @@ -99,6 +84,7 @@ class GPUProfilingParser(BaseProfilingParser): self.__add_marks(event) if event.is_nccl_name(): continue + self.__add_compute_time(event, aten_events, flow_dict_new) self.categorize_computing_performance_data(event, flow_dict_new) self._aten_events = None self._result_data.overall_metrics.set_e2e_time(float(max_ts - min_ts)) @@ -118,6 +104,23 @@ class GPUProfilingParser(BaseProfilingParser): for timestep in range(int(event.start_time + 1), int(event.end_time + 1)): self._marks[str(timestep)] += -100 # mark this timestep in compute stream + def __add_compute_time(self, event: TraceEventBean, aten_events: list, flow_dict_new: dict): + if self.__is_flash_attention(event.name): + if event.is_backward(): + self._result_data.overall_metrics.update_fa_bwd_info(event.dur) + else: + self._result_data.overall_metrics.update_fa_fwd_info(event.dur) + elif any(cube_mark in event.lower_name for cube_mark in self.CUBE_MARK): + is_conv = self.__check_is_conv(event, aten_events, flow_dict_new) + if is_conv == "conv_fwd": + self._result_data.overall_metrics.update_conv_fwd_info(event.dur) + elif is_conv == "conv_bwd": + self._result_data.overall_metrics.update_conv_bwd_info(event.dur) + else: + self._result_data.overall_metrics.update_cube_info(event.dur) + else: + self._result_data.overall_metrics.update_vec_info(event.dur) + def __check_is_conv(self, event: TraceEventBean, aten_events: list, flow_dict_new: dict) -> str: flow_start_time = flow_dict_new.get(event.start_time) if not flow_start_time: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 1d00332b4..5a556b8a6 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -79,6 +79,7 @@ class NPUProfilingParser(BaseProfilingParser): print("[ERROR] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") return self._result_data.update_kernel_details(kernels_dict) + def _update_memory_list(self): try: memory_data = FileReader.read_csv_file(self._operator_memory_path, OperatorMemoryBean) @@ -159,26 +160,9 @@ class NPUProfilingParser(BaseProfilingParser): self.__add_overlap_analysis_time() self._picking_notify_wait_event_and_not_overlap_event() self.__add_overlap_wait_time() + self._result_data.overall_metrics.calculate_other_time() self._result_data.overall_metrics.calculate_schedule_time() self._result_data.overall_metrics.trans_time_to_s() - self._result_data.overall_metrics.calculate_cube_time() - self._result_data.overall_metrics.calculate_vec_time() - self._result_data.overall_metrics.calculate_cube_num() - self._result_data.overall_metrics.calculate_vec_num() - self._result_data.overall_metrics.calculate_sdma_num() - self._result_data.overall_metrics.calculate_fa_num_fwd() - self._result_data.overall_metrics.calculate_fa_num_bwd() - self._result_data.overall_metrics.calculate_pa_num() - self._result_data.overall_metrics.calculate_pa_time() - self._result_data.overall_metrics.calculate_conv_time_fwd() - self._result_data.overall_metrics.calculate_conv_time_bwd() - self._result_data.overall_metrics.calculate_conv_num_fwd() - self._result_data.overall_metrics.calculate_conv_num_bwd() - self._result_data.overall_metrics.calculate_sdma_time() - self._result_data.overall_metrics.calculate_fa_time_fwd() - self._result_data.overall_metrics.calculate_fa_time_bwd() - self._result_data.overall_metrics.trans_to_s() - self._result_data.overall_metrics.calculate_other_time() self._update_bandwidth() def _picking_notify_wait_event_and_not_overlap_event(self): self.notify_event_cache = [] @@ -317,6 +301,28 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.update_lccl_info(event.dur) def __parse_kernel_csv(self): + def __screen_data(kernel: KernelDetailsBean): + if kernel.is_flash_attention(): + if kernel.is_fa_bwd(): + self._result_data.overall_metrics.update_fa_bwd_info(kernel.duration) + else: + self._result_data.overall_metrics.update_fa_fwd_info(kernel.duration) + elif kernel.is_conv(): + if kernel.is_conv_bwd(): + self._result_data.overall_metrics.update_conv_bwd_info(kernel.duration) + else: + self._result_data.overall_metrics.update_conv_fwd_info(kernel.duration) + elif kernel.is_matmul(): + self._result_data.overall_metrics.update_cube_info(kernel.duration) + elif kernel.is_sdma(): + self._result_data.overall_metrics.update_sdma_info(kernel.duration) + elif kernel.is_page_attention(): + self._result_data.overall_metrics.update_pa_info(kernel.duration) + elif kernel.is_vector(): + self._result_data.overall_metrics.update_vec_info(kernel.duration) + else: + self._result_data.overall_metrics.update_cube_info(kernel.duration) + try: kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) except Exception: @@ -330,6 +336,7 @@ class NPUProfilingParser(BaseProfilingParser): for kernel in kernel_details: if kernel.is_invalid(): continue + __screen_data(kernel) self.categorize_computing_performance_data(kernel, flow_dict_new) def __parse_mem_csv(self): @@ -376,4 +383,5 @@ class NPUProfilingParser(BaseProfilingParser): compute_stream = event_wait_stream & ai_core_stream if event_wait_stream else ai_core_stream for stream in compute_stream: dur_list = sdma_dict.get(stream, []) + self._result_data.overall_metrics.update_sdma_info(sum(dur_list), len(dur_list)) self._result_data.overall_metrics.update_sdma_stream_info(sum(dur_list), len(dur_list)) diff --git a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py index e6d543a77..dc85b0af0 100644 --- a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py +++ b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py @@ -36,60 +36,40 @@ class TestProfilingInfo(unittest.TestCase): def test_update_fa_fwd_info(self): info = ProfilingInfo("NPU") - info.fa_time_fwd_cube = 5 - info.fa_time_fwd_vector = 5 - info.fa_num_fwd_cube = 1 - info.fa_num_fwd_vector = 1 - info.calculate_fa_time_fwd() - info.calculate_fa_num_fwd() + info.update_fa_fwd_info(5) + info.update_fa_fwd_info(5) self.assertEqual(info.fa_time_fwd, 10) self.assertEqual(info.fa_num_fwd, 2) def test_update_fa_bwd_info(self): info = ProfilingInfo("NPU") - info.fa_time_bwd_cube = 5 - info.fa_time_bwd_vector = 5 - info.fa_num_bwd_cube = 1 - info.fa_num_bwd_vector = 1 - info.calculate_fa_time_bwd() - info.calculate_fa_num_bwd() + info.update_fa_bwd_info(5) + info.update_fa_bwd_info(5) self.assertEqual(info.fa_time_bwd, 10) self.assertEqual(info.fa_num_bwd, 2) def test_update_sdma_info(self): info = ProfilingInfo("NPU") - info.sdma_time_tensor_move = 5 - info.sdma_time_stream = 5 - info.sdma_num_tensor_move = 5 - info.sdma_num_stream = 5 - info.calculate_sdma_time() - info.calculate_sdma_num() + info.update_sdma_info(5) + self.assertEqual(info.sdma_time, 5) + self.assertEqual(info.sdma_num, 1) + info.update_sdma_info(5, 5) self.assertEqual(info.sdma_time, 10) - self.assertEqual(info.sdma_num, 10) + self.assertEqual(info.sdma_num, 6) def test_update_cube_info(self): info = ProfilingInfo("NPU") - info.matmul_time_cube = 1 - info.matmul_time_vector = 1 - info.other_cube_time = 1 - info.matmul_num_cube = 5 - info.matmul_num_vector = 5 - info.other_cube_num = 5 - info.calculate_cube_time() - info.calculate_cube_num() - self.assertEqual(info.cube_time, 3) - self.assertEqual(info.cube_num, 15) + info.update_cube_info(5) + info.update_cube_info(5) + self.assertEqual(info.cube_time, 10) + self.assertEqual(info.cube_num, 2) def test_update_vec_info(self): info = ProfilingInfo("NPU") - info.vector_time_trans = 1 - info.vector_time_notrans = 1 - info.vector_num_trans = 2 - info.vector_num_notrans = 2 - info.calculate_vec_time() - info.calculate_vec_num() - self.assertEqual(info.vec_time, 2) - self.assertEqual(info.vec_num, 4) + info.update_vec_info(5) + info.update_vec_info(5) + self.assertEqual(info.vec_time, 10) + self.assertEqual(info.vec_num, 2) def test_set_compute_time(self): info = ProfilingInfo("NPU") diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index 93c6e3855..d7cb3d058 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -76,16 +76,16 @@ class TestGpuProfilingParser(unittest.TestCase): res._marks = defaultdict(int) res._calculate_performance_time() self.assertEqual(res._result_data.overall_metrics.e2e_time, 98) - self.assertEqual(res._result_data.overall_metrics.sdma_time, 0) - self.assertEqual(res._result_data.overall_metrics.sdma_num, 0) - self.assertEqual(res._result_data.overall_metrics.cube_time, 0) - self.assertEqual(res._result_data.overall_metrics.cube_num, 0) - self.assertEqual(res._result_data.overall_metrics.fa_time_fwd, 0) - self.assertEqual(res._result_data.overall_metrics.fa_num_fwd, 0) - self.assertEqual(res._result_data.overall_metrics.fa_time_bwd, 0) - self.assertEqual(res._result_data.overall_metrics.fa_num_bwd, 0) - self.assertEqual(res._result_data.overall_metrics.vec_time, 0) - self.assertEqual(res._result_data.overall_metrics.vec_num, 0) # cun yi + self.assertEqual(res._result_data.overall_metrics.sdma_time, 4) + self.assertEqual(res._result_data.overall_metrics.sdma_num, 4) + self.assertEqual(res._result_data.overall_metrics.cube_time, 1) + self.assertEqual(res._result_data.overall_metrics.cube_num, 1) + self.assertEqual(res._result_data.overall_metrics.fa_time_fwd, 2) + self.assertEqual(res._result_data.overall_metrics.fa_num_fwd, 2) + self.assertEqual(res._result_data.overall_metrics.fa_time_bwd, 2) + self.assertEqual(res._result_data.overall_metrics.fa_num_bwd, 2) + self.assertEqual(res._result_data.overall_metrics.vec_time, 2) + self.assertEqual(res._result_data.overall_metrics.vec_num, 2) # cun yi self.assertEqual(res._result_data.overall_metrics.communication_not_overlapped, 2) self.assertEqual(res._result_data.overall_metrics.compute_time, 7) -- Gitee From 61853217cbf9078205a549e4ae9b05b29314ab9b Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 6 Aug 2024 14:16:26 +0800 Subject: [PATCH 137/791] =?UTF-8?q?=E8=A1=A5=E5=85=85=E6=97=A0=E6=A0=87?= =?UTF-8?q?=E6=9D=86=E6=98=BE=E5=AD=98=E4=BC=98=E5=8C=96ut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../free_benchmark/result_handlers/test_result_handler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py index a0beebec5..8be3be413 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py @@ -122,13 +122,16 @@ class TestFuzzHandler(TestCase): ) def test_tensor_split_for_error_calculate(self): + # 设置模拟的张量的大小 tensor_size = 256 * 1024 * 1024 origin_output = torch.randn(tensor_size, dtype=torch.float32) perturbed_output = torch.randn(tensor_size, dtype=torch.float32) + # 调用tensor_split_for_error_calculate方法 origin_output_chunks, perturbed_output_chunks = FuzzHandler.tensor_split_for_error_calculate( origin_output, perturbed_output) + # 验证返回的chunks数量和形状是否正确 self.assertEqual(len(origin_output_chunks), 64) self.assertEqual(len(perturbed_output_chunks), 64) for chunk in origin_output_chunks: -- Gitee From 3c67b1e347989843a3f36dd99cdfe9e1c44c7208 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 6 Aug 2024 14:35:23 +0800 Subject: [PATCH 138/791] bug fix no module error & parameters name contain special character --- debug/accuracy_tools/grad_tool/common/constant.py | 2 +- debug/accuracy_tools/grad_tool/common/utils.py | 3 +-- debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py | 3 --- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/common/constant.py b/debug/accuracy_tools/grad_tool/common/constant.py index 38d33e988..7904c1d42 100644 --- a/debug/accuracy_tools/grad_tool/common/constant.py +++ b/debug/accuracy_tools/grad_tool/common/constant.py @@ -39,7 +39,7 @@ class GradConst: DIRECTORY_LENGTH = 4096 FILE_NAME_LENGTH = 255 FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" - PARAM_VALID_PATTERN = r"^[a-zA-Z0-9.]+$" + PARAM_VALID_PATTERN = r"^[a-zA-Z0-9_.:-]+$" DIR = "dir" FILE = "file" diff --git a/debug/accuracy_tools/grad_tool/common/utils.py b/debug/accuracy_tools/grad_tool/common/utils.py index fceda8ce0..f40f8688c 100644 --- a/debug/accuracy_tools/grad_tool/common/utils.py +++ b/debug/accuracy_tools/grad_tool/common/utils.py @@ -7,7 +7,6 @@ import yaml import pandas as pd from grad_tool.common.constant import GradConst -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen def _print_log(level, msg, end='\n'): @@ -115,7 +114,7 @@ class ListCache(list): def get_config(filepath): - with FileOpen(filepath, 'r') as file: + with open(filepath, 'r') as file: config = yaml.safe_load(file) return config diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py index c843df388..fa794a681 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py @@ -16,7 +16,6 @@ from grad_tool.common.utils import ListCache, print_warn_log from grad_tool.common.utils import create_directory, check_file_or_directory_path, write_csv from grad_tool.grad_ms.global_context import grad_context from grad_tool.grad_ms.global_context import GlobalContext -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker def get_rank_id(): @@ -170,8 +169,6 @@ class CSVGenerator(Process): stat_data = None max_try = 10 while max_try: - file_path_checker = FileChecker(file_path, FileCheckConst.DIR,FileCheckConst.READ_ABLE) - file_path = file_path_checker.common_check() try: stat_data = np.load(file_path) return stat_data -- Gitee From 919a3767f6ce5052c19485f277f74f580b6f3c8f Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 6 Aug 2024 15:10:17 +0800 Subject: [PATCH 139/791] fix kwargs bug --- .../api_accuracy_checker/run_ut/data_generate.py | 14 ++++++++------ .../run_ut/test_data_generate.py | 2 +- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index b2eec691a..6f2b4801f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -257,12 +257,13 @@ def gen_args(args_info, api_name, need_grad=True, convert_type=None, real_data_p return args_result -def gen_kwargs(api_info, convert_type=None, real_data_path=None): +def gen_kwargs(api_info, api_name, convert_type=None, real_data_path=None): """ Function Description: Based on API basic information, generate input parameters: kwargs, for API forward running Parameter: api_info: API basic information. Dict + api_name: API name convert_type: convert ori_type to dist_type flag. real_data_path: the root directory for storing real data. """ @@ -270,11 +271,11 @@ def gen_kwargs(api_info, convert_type=None, real_data_path=None): kwargs_params = api_info.get("input_kwargs") for key, value in kwargs_params.items(): if isinstance(value, (list, tuple)): - kwargs_params[key] = gen_list_kwargs(value, convert_type, real_data_path) + kwargs_params[key] = gen_list_kwargs(value, api_name, convert_type, real_data_path) elif value is None: kwargs_params[key] = None elif value.get('type') in TENSOR_DATA_LIST or value.get('type').startswith("numpy"): - kwargs_params[key] = gen_data(value, True, convert_type, real_data_path) + kwargs_params[key] = gen_data(value, api_name, True, convert_type, real_data_path) elif value.get('type') in TORCH_TYPE: gen_torch_kwargs(kwargs_params, key, value) else: @@ -287,18 +288,19 @@ def gen_torch_kwargs(kwargs_params, key, value): kwargs_params[key] = eval(value.get('value')) -def gen_list_kwargs(kwargs_item_value, convert_type, real_data_path=None): +def gen_list_kwargs(kwargs_item_value, api_name, convert_type, real_data_path=None): """ Function Description: When kwargs value is list, generate the list of kwargs result Parameter: kwargs_item_value: kwargs value before to generate. List + api_name: API name convert_type: convert ori_type to dist_type flag. """ kwargs_item_result = [] for item in kwargs_item_value: if item.get('type') in TENSOR_DATA_LIST: - item_value = gen_data(item, False, convert_type, real_data_path) + item_value = gen_data(item, api_name, False, convert_type, real_data_path) elif item.get('type') == "torch.Size": item_value = torch.Size(item.get('value')) else: @@ -321,7 +323,7 @@ def gen_api_params(api_info, api_name, need_grad=True, convert_type=None, real_d if convert_type and convert_type not in Const.CONVERT: error_info = f"convert_type params not support {convert_type}." raise CompareException(CompareException.INVALID_PARAM_ERROR, error_info) - kwargs_params = gen_kwargs(api_info, convert_type, real_data_path) + kwargs_params = gen_kwargs(api_info, api_name, convert_type, real_data_path) if api_info.get("input_args"): args_params = gen_args(api_info.get("input_args"), api_name, need_grad, convert_type, real_data_path) else: diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py index f664dad19..d3a62e5e0 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py @@ -20,7 +20,7 @@ min_value = -1.444359375 class TestDataGenerateMethods(unittest.TestCase): def test_gen_api_params(self): api_info = copy.deepcopy(api_info_dict) - args_params, kwargs_params = gen_api_params(api_info, True, None, None) + args_params, kwargs_params = gen_api_params(api_info, "conv2d", True, None, None) max_diff = abs(args_params[0].max() - max_value) min_diff = abs(args_params[0].min() - min_value) self.assertEqual(len(args_params), 2) -- Gitee From 727e790b149ace81576ac9cf71f05501febf689c Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 6 Aug 2024 15:11:44 +0800 Subject: [PATCH 140/791] 82 --- .../profiling_parser/npu_profiling_parser.py | 2 +- .../compare_backend/utils/file_reader.py | 24 +------------------ 2 files changed, 2 insertions(+), 24 deletions(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 5a556b8a6..e4b6f1b1f 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -134,7 +134,7 @@ class NPUProfilingParser(BaseProfilingParser): print("[WARNING] The JSON file is empty.") return for _, group_dict in communication_json.items(): - step_dict = group_dict.get("collective") + step_dict = group_dict.get("collective", {}) total_op_info = step_dict.get("Total Op Info", {}) rdma_size_mb = rdma_time_ms = sdma_size_mb = sdma_time_ms = 0 if "Communication Bandwidth Info" in total_op_info: diff --git a/profiler/compare_tools/compare_backend/utils/file_reader.py b/profiler/compare_tools/compare_backend/utils/file_reader.py index 99358368c..e49445540 100644 --- a/profiler/compare_tools/compare_backend/utils/file_reader.py +++ b/profiler/compare_tools/compare_backend/utils/file_reader.py @@ -8,29 +8,7 @@ from compare_backend.utils.constant import Constant class FileReader: @classmethod - def read_json_file(cls, file_path: str, bean_class: any = None) -> any: - PathManager.check_path_readable(file_path) - if not os.path.isfile(file_path): - raise FileNotFoundError("File not exists.") - file_size = os.path.getsize(file_path) - if file_size <= 0: - return [] - if file_size > Constant.MAX_JSON_SIZE: - check_msg = input( - f"The file({file_path}) size exceeds the preset max value. Continue reading the file? [y/n]") - if check_msg.lower() != "y": - print(f"[WARNING] The user choose not to read the file: {file_path}") - return [] - result_data = [] - try: - with open(file_path, "r") as json_file: - result_data = json.loads(json_file.read()) - except Exception as e: - msg = f"Failed to read the file: {file_path}" - raise RuntimeError(msg) from e - return result_data - @classmethod - def read_trace_file(cls, file_path: str) -> any: + def read_json_file(cls, file_path: str) -> any: PathManager.check_path_readable(file_path) if not os.path.isfile(file_path): raise FileNotFoundError("File not exists.") -- Gitee From 226289ce12ed6742a175f243d001a82f7dcfc176 Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 6 Aug 2024 15:33:02 +0800 Subject: [PATCH 141/791] 82 --- .../compare_backend/profiling_parser/npu_profiling_parser.py | 2 +- profiler/compare_tools/compare_backend/utils/file_reader.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index e4b6f1b1f..02b6abdb0 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -124,7 +124,7 @@ class NPUProfilingParser(BaseProfilingParser): def _update_bandwidth(self): try: - communication_json = FileReader.read_json_file(self._communication_path) + communication_json = FileReader.read_trace_file(self._communication_path) except FileNotFoundError: print("[WARNING] The file communication.json does not exist.") except Exception: diff --git a/profiler/compare_tools/compare_backend/utils/file_reader.py b/profiler/compare_tools/compare_backend/utils/file_reader.py index e49445540..263888a3e 100644 --- a/profiler/compare_tools/compare_backend/utils/file_reader.py +++ b/profiler/compare_tools/compare_backend/utils/file_reader.py @@ -8,7 +8,7 @@ from compare_backend.utils.constant import Constant class FileReader: @classmethod - def read_json_file(cls, file_path: str) -> any: + def read_trace_file(cls, file_path: str) -> any: PathManager.check_path_readable(file_path) if not os.path.isfile(file_path): raise FileNotFoundError("File not exists.") -- Gitee From 923b572ae74e892a577a6e40bca7d92b22916d02 Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 6 Aug 2024 15:53:10 +0800 Subject: [PATCH 142/791] 82 --- .../comparator/overall_performance_comparator.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 3a94527bb..09d8688cf 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -12,14 +12,6 @@ class OverallPerformanceComparator(BaseComparator): self._headers = [''] base_col = [f'{base_profiling_info.profiling_type}'] comp_col = [f'{comp_profiling_info.profiling_type}'] - if base_profiling_info.RDMA_bandwidth or comp_profiling_info.RDMA_bandwidth: - self._headers.extend(['RDMA Bandwidth']) - base_col.append(f'{base_profiling_info.RDMA_bandwidth:.3f}GB/s') - comp_col.append(f'{comp_profiling_info.RDMA_bandwidth:.3f}GB/s') - if base_profiling_info.SDMA_bandwidth or comp_profiling_info.SDMA_bandwidth: - self._headers.extend(['SDMA Bandwidth']) - base_col.append(f'{base_profiling_info.SDMA_bandwidth:.3f}GB/s') - comp_col.append(f'{comp_profiling_info.SDMA_bandwidth:.3f}GB/s') if not base_profiling_info.hide_op_details and not comp_profiling_info.hide_op_details: self._headers.extend(['Cube Time(Num)', 'Vector Time(Num)']) base_col.extend([f'{base_profiling_info.cube_time:.3f}s({base_profiling_info.cube_num})', @@ -72,6 +64,14 @@ class OverallPerformanceComparator(BaseComparator): else: comp_col.extend( [f'{comp_profiling_info.communication_not_overlapped: .3f}s({comp_profiling_info.wait_time:.3f}s)']) + if base_profiling_info.RDMA_bandwidth or comp_profiling_info.RDMA_bandwidth: + self._headers.extend(['RDMA Bandwidth']) + base_col.append(f'{base_profiling_info.RDMA_bandwidth:.3f}GB/s') + comp_col.append(f'{comp_profiling_info.RDMA_bandwidth:.3f}GB/s') + if base_profiling_info.SDMA_bandwidth or comp_profiling_info.SDMA_bandwidth: + self._headers.extend(['SDMA Bandwidth']) + base_col.append(f'{base_profiling_info.SDMA_bandwidth:.3f}GB/s') + comp_col.append(f'{comp_profiling_info.SDMA_bandwidth:.3f}GB/s') if base_profiling_info.sdma_time or comp_profiling_info.sdma_time: self._headers.append('SDMA Time(Num)') base_col.append(f'{base_profiling_info.sdma_time:.3f}s({base_profiling_info.sdma_num})') -- Gitee From 90462ff04d9e958ac5e8777971c213ee1848fb84 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Tue, 6 Aug 2024 15:54:01 +0800 Subject: [PATCH 143/791] =?UTF-8?q?compare=5Fprocess=E5=87=BD=E6=95=B0clea?= =?UTF-8?q?ncode=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/compare/ms_compare.py | 63 ++++++++++++------- .../msprobe/pytorch/compare/pt_compare.py | 58 ++++++----------- 2 files changed, 62 insertions(+), 59 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 21b1b9c24..9e93a5159 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -60,6 +60,18 @@ class MSComparator (Comparator): return _save_cmp_result(idx, cr, result_df, lock) + + def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): + op_data = json_data['data'][op_name] + op_parsed_list = read_op(op_data, op_name) + if op_name in stack_json_data: + op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) + else: + op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) + + merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) + return merge_list + def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles npu_json_data = json.load(npu_json_handle) @@ -87,15 +99,7 @@ class MSComparator (Comparator): last_npu_ops_len = len(npu_ops_queue) op_name_npu = next(ops_npu_iter) read_err_npu = True - - npu_op_data = npu_json_data['data'][op_name_npu] - npu_op_parsed_list = read_op(npu_op_data, op_name_npu) - if op_name_npu in stack_json_data: - npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': stack_json_data[op_name_npu]}) - else: - npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': None}) - - npu_merge_list = merge_tensor(npu_op_parsed_list, summary_compare, md5_compare) + npu_merge_list = self.gen_merge_list(npu_json_data,op_name_npu,stack_json_data,summary_compare, md5_compare) if npu_merge_list: npu_ops_queue.append(npu_merge_list) except StopIteration: @@ -103,16 +107,7 @@ class MSComparator (Comparator): try: last_bench_ops_len = len(bench_ops_queue) op_name_bench = next(ops_bench_iter) - - bench_op_data = bench_json_data['data'][op_name_bench] - bench_op_parsed_list = read_op(bench_op_data, op_name_bench) - if op_name_bench in stack_json_data: - bench_op_parsed_list.append( - {'full_op_name': op_name_bench, 'full_info': stack_json_data[op_name_bench]}) - else: - bench_op_parsed_list.append({'full_op_name': op_name_bench, 'full_info': None}) - - bench_merge_list = merge_tensor(bench_op_parsed_list, summary_compare, md5_compare) + bench_merge_list = self.gen_merge_list(bench_json_data,op_name_bench,stack_json_data,summary_compare, md5_compare) if bench_merge_list: bench_ops_queue.append(bench_merge_list) except StopIteration: @@ -163,10 +158,36 @@ class MSComparator (Comparator): for row in result: del row[-1] - result_df = pd.DataFrame(result, columns=header) + result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df + + def make_result_table(self,result,md5_compare,summary_compare,stack_mode): + header = [] + if md5_compare: + header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] + elif summary_compare: + header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] + else: + header = CompareConst.COMPARE_RESULT_HEADER[:] - + all_mode_bool = not (summary_compare or md5_compare) + if stack_mode: + if all_mode_bool: + header.append(CompareConst.STACK) + header.append(CompareConst.DATA_NAME) + else: + header.append(CompareConst.STACK) + else: + if all_mode_bool: + for row in result: + del row[-2] + header.append(CompareConst.DATA_NAME) + else: + for row in result: + del row[-1] + result_df = pd.DataFrame(result, columns=header) + return result_df + def _do_multi_process(self,input_parma, result_df): try: result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 081f5631d..8207c7d64 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -59,6 +59,19 @@ class PTComparator (Comparator): ) return _save_cmp_result(idx, cr, result_df, lock) + + + def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): + op_data = json_data['data'][op_name] + op_parsed_list = read_op(op_data, op_name) + if op_name in stack_json_data: + op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) + else: + op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) + + merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) + return merge_list + def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles @@ -87,15 +100,7 @@ class PTComparator (Comparator): last_npu_ops_len = len(npu_ops_queue) op_name_npu = next(ops_npu_iter) read_err_npu = True - - npu_op_data = npu_json_data['data'][op_name_npu] - npu_op_parsed_list = read_op(npu_op_data, op_name_npu) - if op_name_npu in stack_json_data: - npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': stack_json_data[op_name_npu]}) - else: - npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': None}) - - npu_merge_list = merge_tensor(npu_op_parsed_list, summary_compare, md5_compare) + npu_merge_list = self.gen_merge_list(npu_json_data,op_name_npu,stack_json_data,summary_compare,md5_compare) if npu_merge_list: npu_ops_queue.append(npu_merge_list) except StopIteration: @@ -103,16 +108,7 @@ class PTComparator (Comparator): try: last_bench_ops_len = len(bench_ops_queue) op_name_bench = next(ops_bench_iter) - - bench_op_data = bench_json_data['data'][op_name_bench] - bench_op_parsed_list = read_op(bench_op_data, op_name_bench) - if op_name_bench in stack_json_data: - bench_op_parsed_list.append( - {'full_op_name': op_name_bench, 'full_info': stack_json_data[op_name_bench]}) - else: - bench_op_parsed_list.append({'full_op_name': op_name_bench, 'full_info': None}) - - bench_merge_list = merge_tensor(bench_op_parsed_list, summary_compare, md5_compare) + bench_merge_list =self.gen_merge_list(bench_json_data,op_name_bench,stack_json_data,summary_compare,md5_compare) if bench_merge_list: bench_ops_queue.append(bench_merge_list) except StopIteration: @@ -138,7 +134,11 @@ class PTComparator (Comparator): if npu_ops_queue: for npu_data in npu_ops_queue: get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - + + result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) + return result_df + + def make_result_table(self,result,md5_compare,summary_compare,stack_mode): header = [] if md5_compare: header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] @@ -162,11 +162,9 @@ class PTComparator (Comparator): else: for row in result: del row[-1] - result_df = pd.DataFrame(result, columns=header) return result_df - def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -234,22 +232,6 @@ class PTComparator (Comparator): advisor.analysis() -# def pt_compare(input_parma, output_path, stack_mode=False, auto_analyze=True, -# fuzzy_match=False): -# try: -# summary_compare, md5_compare = task_dumppath_get(input_parma) -# check_configuration_param(stack_mode, auto_analyze, fuzzy_match) -# create_directory(output_path) -# check_compare_param(input_parma, output_path, summary_compare, md5_compare) -# except CompareException as error: -# logger.error('Compare failed. Please check the arguments and do it again!') -# sys.exit(error.code) -# ptComparator= PTComparator() -# ptComparator.compare_core(input_parma, output_path, stack_mode=stack_mode, -# auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, -# md5_compare=md5_compare) - - def pt_compare(args): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) -- Gitee From b37b130e70dcbd9a220785ab8393e4136eaefe07 Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 6 Aug 2024 16:50:03 +0800 Subject: [PATCH 144/791] 82 --- .../compare_backend/profiling_parser/npu_profiling_parser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 02b6abdb0..1ae5b1fe6 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -147,8 +147,8 @@ class NPUProfilingParser(BaseProfilingParser): sdma_info = bandwidth_info["SDMA"] sdma_size_mb += sdma_info.get("Transit Size(MB)", 0) # 单位为 MB sdma_time_ms += sdma_info.get("Transit Time(ms)", 0) # 单位为 MS - rdma_bandwidth = (rdma_size_mb / 1024) / (rdma_time_ms / 1000) if rdma_time_ms > 0 else 0 - sdma_bandwidth = (sdma_size_mb / 1024) / (sdma_time_ms / 1000) if sdma_time_ms > 0 else 0 + rdma_bandwidth = rdma_size_mb / rdma_time_ms if rdma_time_ms > 0 else 0 + sdma_bandwidth = sdma_size_mb / sdma_time_ms if sdma_time_ms > 0 else 0 self._result_data.overall_metrics.set_RDMA_bandwidth(rdma_bandwidth) self._result_data.overall_metrics.set_SDMA_bandwidth(sdma_bandwidth) def _update_overall_metrics(self): -- Gitee From 67ff995a4569966d431b8b8e54955eb36a61182e Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 6 Aug 2024 17:00:15 +0800 Subject: [PATCH 145/791] backward fix --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 559dfdc0f..6418e8922 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -282,7 +282,7 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict if need_backward: if need_to_backward(grad_index, out): - backward_args = backward_content[api_full_name].get("input") + backward_args = backward_content[api_full_name].get("grad_input") grad = gen_args(backward_args, api_name, real_data_path=real_data_path)[0] bench_grad, _ = generate_cpu_params(grad, {}, False, api_name) bench_grad_out = run_backward(cpu_args, bench_grad, grad_index, out) -- Gitee From 286777d04ca0a315c06fa33a8d332cd526ef1861 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Tue, 6 Aug 2024 17:20:57 +0800 Subject: [PATCH 146/791] =?UTF-8?q?msprobe=E6=94=AF=E6=8C=81torch1.11?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../perturbed_layers/npu/add_noise.py | 2 +- .../perturbed_layers/npu/bit_noise.py | 2 +- .../perturbed_layers/npu/change_value.py | 2 +- .../perturbed_layers/npu/improve_precision.py | 2 +- .../perturbed_layers/npu/no_change.py | 2 +- .../perturbed_layers/run_cpu.py | 2 +- .../msprobe/pytorch/functional/dump_module.py | 2 +- .../pytorch/hook_module/hook_module.py | 11 +++++-- .../msprobe/pytorch/module_processer.py | 16 ++++++++- .../accuracy_tools/msprobe/pytorch/service.py | 33 +++++++++++++++---- 10 files changed, 56 insertions(+), 18 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py index a18ef1c51..2ccc2bfcf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/add_noise.py @@ -32,7 +32,7 @@ class AddNoiseLayer(NpuBaseLayer): return type(tensor_obj)([self.add_noise(value) for value in tensor_obj]) return tensor_obj - def handle(self, params: DataParams) -> torch.Any: + def handle(self, params: DataParams): """ 对输入添加扰动并返回 """ diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py index 45dea7b93..a0ac21691 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/bit_noise.py @@ -48,7 +48,7 @@ class BitNoiseLayer(NpuBaseLayer): return type(tensor_obj)([self.add_bit_noise(value) for value in tensor_obj]) return tensor_obj - def handle(self, params: DataParams) -> torch.Any: + def handle(self, params: DataParams): """ 对输入添加扰动并返回 """ diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py index 91085d57a..ae5bf9f03 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/change_value.py @@ -39,7 +39,7 @@ class ChangeValueLayer(NpuBaseLayer): return type(tensor_obj)([self.change_value(value) for value in tensor_obj]) return tensor_obj - def handle(self, params: DataParams) -> torch.Any: + def handle(self, params: DataParams): """ 对输入添加扰动并返回 """ diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py index ad6d8b898..53aa0d0d1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py @@ -32,7 +32,7 @@ class ImprovePrecisionLayer(NpuBaseLayer): ) return tensor_obj - def handle(self, params: DataParams) -> torch.Any: + def handle(self, params: DataParams): logger.info_on_rank_0( f"[msprobe] Free benchmark: Perturbation is " f"{PerturbationMode.IMPROVE_PRECISION} of {self.api_name}." diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py index a69c56002..fa775e00e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/no_change.py @@ -16,7 +16,7 @@ class NoChangeLayer(NpuBaseLayer): self.is_added = True return tensor_obj - def handle(self, params: DataParams) -> torch.Any: + def handle(self, params: DataParams): """ 对输入添加扰动并返回 """ diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py index d34ac9765..376f4ee3e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/run_cpu.py @@ -8,7 +8,7 @@ from msprobe.pytorch.free_benchmark.perturbed_layers.base_layer import BaseLayer class CpuLayer(BaseLayer): - def handle(self, params: DataParams) -> torch.Any: + def handle(self, params: DataParams): logger.info_on_rank_0( f"[msprobe] Free benchmark: Perturbation is to_cpu of {self.api_name}." diff --git a/debug/accuracy_tools/msprobe/pytorch/functional/dump_module.py b/debug/accuracy_tools/msprobe/pytorch/functional/dump_module.py index efb95c336..5d2e8d985 100644 --- a/debug/accuracy_tools/msprobe/pytorch/functional/dump_module.py +++ b/debug/accuracy_tools/msprobe/pytorch/functional/dump_module.py @@ -24,7 +24,7 @@ def module_dump(module, dump_name): dump_name = dump_name + Const.SEP + str(module_count.get(dump_name)) + Const.SEP pdg = PrecisionDebugger() - _, forward_hook, backward_hook = pdg.service.build_hook(BaseScope.Module_Type_Module, dump_name) + _, forward_hook, backward_hook, _ = pdg.service.build_hook(BaseScope.Module_Type_Module, dump_name) module.register_forward_hook(forward_hook, with_kwargs=True) module.register_full_backward_hook(backward_hook) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py index ff6427e51..4d8f48a4e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py @@ -23,6 +23,7 @@ import torch.nn as nn import torch.utils.hooks as full_hooks from msprobe.core.common.const import Const +torch_vsrsion_above_2 = torch.__version__.split('+')[0] > '2.0' class HOOKModule(nn.Module): @@ -48,9 +49,13 @@ class HOOKModule(nn.Module): else: HOOKModule.module_count[self.prefix] += 1 self.prefix = self.prefix + str(HOOKModule.module_count[self.prefix] - 1) + Const.SEP - forward_pre_hook, forward_hook, backward_hook = build_hook(self.prefix) - self.register_forward_pre_hook(forward_pre_hook, with_kwargs=True) - self.register_forward_hook(forward_hook, with_kwargs=True) + forward_pre_hook, forward_hook, backward_hook, _ = build_hook(self.prefix) + if torch_vsrsion_above_2: + self.register_forward_pre_hook(forward_pre_hook, with_kwargs=True) + self.register_forward_hook(forward_hook, with_kwargs=True) + else: + self.register_forward_pre_hook(forward_pre_hook) + self.register_forward_hook(forward_hook) self.register_backward_hook(backward_hook) def __call__(self, *input, **kwargs): diff --git a/debug/accuracy_tools/msprobe/pytorch/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/module_processer.py index 3e9969d32..c18288ef2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/module_processer.py @@ -109,7 +109,21 @@ class ModuleProcesser: if self.scope: self.scope.end_module(module.mindstudio_reserved_name) - if Const.START in start_or_stop: + def backward_hook(module, input, output=None): + try: + index = ModuleProcesser.module_count_func(name_prefix) + except IndexError as e: + index = None + pass + module.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index) + ModuleProcesser.module_node[full_name] = None + ModuleProcesser.api_parent_node = None + if self.scope: + self.scope.begin_module(full_name) + + if 'forward' in name_prefix and Const.START in start_or_stop: return pre_hook + elif 'backward' in name_prefix: + return backward_hook else: return end_hook diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 6b8d67abc..46d465714 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -2,6 +2,7 @@ import functools import os from pathlib import Path +import service from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException from msprobe.core.common.file_check import FileChecker, check_path_before_create @@ -14,6 +15,7 @@ from msprobe.pytorch.hook_module import remove_dropout from msprobe.pytorch.hook_module.api_registry import api_register from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.module_processer import ModuleProcesser +torch_vsrsion_above_2 = torch.__version__.split('+')[0] > '2.0' class Service: @@ -60,6 +62,20 @@ class Service: return self.data_collector.get_forward_new_output() return output + def forward_hook_torch_version_below_2(api_or_module_name, module, args, output): + if module_type == BaseScope.Module_Type_Module: + api_or_module_name = module.mindstudio_reserved_name + self.data_collector.visit_and_clear_overflow_status(api_or_module_name) + + if not self.switch: + return None + if self.data_collector: + module_input_output = ModuleForwardInputsOutputs(args=args, kwargs={}, output=output) + self.data_collector.forward_data_collect(api_or_module_name, module, pid, module_input_output) + if self.data_collector.if_return_forward_new_output(): + return self.data_collector.get_forward_new_output() + return output + def backward_hook(api_or_module_name, module, grad_input, grad_output): if module_type == BaseScope.Module_Type_Module: api_or_module_name = module.mindstudio_reserved_name @@ -78,7 +94,8 @@ class Service: pre_forward_hook = functools.partial(pre_hook, forward_name_template) forward_hook = functools.partial(forward_hook, forward_name_template) backward_hook = functools.partial(backward_hook, backward_name_template) - return pre_forward_hook, forward_hook, backward_hook + forward_hook_torch_version_below_2 = functools.partial(forward_hook_torch_version_below_2, forward_name_template) + return pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 def step(self): self.current_iter += 1 @@ -158,19 +175,21 @@ class Service: prefix = BaseScope.Module_Type_Module + Const.SEP + name + Const.SEP + \ module.__class__.__name__ + Const.SEP - pre_forward_hook, forward_hook, backward_hook = self.build_hook(BaseScope.Module_Type_Module, prefix) - module.register_forward_hook(forward_hook, with_kwargs=True) - module.register_full_backward_hook(backward_hook) - module.register_forward_pre_hook( self.module_processor.node_hook(prefix + Const.FORWARD, Const.START)) module.register_forward_hook( self.module_processor.node_hook(prefix + Const.FORWARD, Const.STOP)) - module.register_full_backward_pre_hook( - self.module_processor.node_hook(prefix + Const.BACKWARD, Const.START)) module.register_full_backward_hook( self.module_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) + pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 \ + = self.build_hook(BaseScope.Module_Type_Module, prefix) + if torch_vsrsion_above_2: + module.register_forward_hook(forward_hook, with_kwargs=True) + else: + module.register_forward_hook(forward_hook_torch_version_below_2) + module.register_full_backward_hook(backward_hook) + if self.config.level in ["mix", "L1", "L2"]: api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) api_register.api_modularity() -- Gitee From 9b4d9f63bef05cfedd73b4d52b6a9059978e1a9e Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Tue, 6 Aug 2024 17:29:08 +0800 Subject: [PATCH 147/791] mindspore free benchmark V1.4 --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- .../msprobe/mindspore/free_benchmark/perturbation/add_noise.py | 2 +- .../msprobe/mindspore/free_benchmark/perturbation/bit_noise.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 6a262d4be..b4baf4733 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -283,7 +283,7 @@ class MsFreeBenchmarkConst: FIX_HANDLER_MODE = "fix" ADD_NOISE = "add_noise" BIT_NOISE = "bit_noise" - NO_CHANGE = "no_change", + NO_CHANGE = "no_change" IMPROVE_PRECISION = "improve_precision" CHECK = "check" FIX = "fix" diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py index 28969e453..3d645a6f1 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py @@ -28,7 +28,7 @@ class AddNoisePerturbation(BasePerturbation): """ if isinstance(inputs, Tensor): noise = self._get_noise(inputs) - if noise: + if noise is not False: result = ops.where(ops.abs(inputs) > self.perturbation_value ** 0.5, ops.add(noise, inputs), inputs) result = result.type(dtype=inputs.dtype) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py index 13efb1f37..b682edf09 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py @@ -14,7 +14,7 @@ class BitNoisePerturbation(BasePerturbation): def add_bit_noise(self, inputs) -> Any: if isinstance(inputs, Tensor): bit_len_type = self._get_bit_len_type(inputs) - if bit_len_type: + if bit_len_type is not False: sub_normal_np = np.finfo(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)).smallest_normal sub_normal = Tensor(sub_normal_np) noise_type = list(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.keys())[ -- Gitee From 3069dc0d31de39ed8713643ab467a3f993881fdf Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Tue, 6 Aug 2024 18:18:31 +0800 Subject: [PATCH 148/791] mindspore free benchmark supported list --- .../accuracy_tools/msprobe/config/config.json | 12 + .../free_benchmark/data/support_wrap_ops.yaml | 842 ++++++++++++++++++ 2 files changed, 854 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml diff --git a/debug/accuracy_tools/msprobe/config/config.json b/debug/accuracy_tools/msprobe/config/config.json index ef0283ca2..8603771f8 100644 --- a/debug/accuracy_tools/msprobe/config/config.json +++ b/debug/accuracy_tools/msprobe/config/config.json @@ -37,5 +37,17 @@ "step": [], "bounds": [-1, 0, 1], "output_path": "./grad_output" + }, + "free_benchmark": { + "scope": [], + "list": [], + "fuzz_device": "npu", + "pert_mode": "improve_precision", + "handler_type": "check", + "fuzz_level": "L1", + "fuzz_stage": "forward", + "if_preheat": false, + "preheat_step": 15, + "max_sample": 20 } } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml new file mode 100644 index 000000000..cc802d381 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml @@ -0,0 +1,842 @@ +# List of apis that support self check + +communication: + - all_gather_into_tensor + - gather_into_tensor + - all_reduce + - reduce + - reduce_scatter_tensor + +ops: + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d + - adaptive_max_pool1d + - adaptive_max_pool2d + - avg_pool1d + - avg_pool2d + - avg_pool3d + - batch_norm + - bias_add + - ctc_greedy_decoder + - conv1d + - conv2d + - conv3d + - deformable_conv2d + - dense + - dropout + - dropout1d + - dropout2d + - dropout3d + - flatten + - fold + - fractional_max_pool3d + - lp_pool1d + - lp_pool2d + - lrn + - max_pool2d + - max_pool3d + - max_unpool1d + - max_unpool2d + - max_unpool3d + - unfold + - binary_cross_entropy + - binary_cross_entropy_with_logits + - cosine_embedding_loss + - cross_entropy + - ctc_loss + - gaussian_nll_loss + - hinge_embedding_loss + - huber_loss + - kl_div + - l1_loss + - margin_ranking_loss + - mse_loss + - multi_margin_loss + - multilabel_margin_loss + - multilabel_soft_margin_loss + - nll_loss + - smooth_l1_loss + - triplet_margin_loss + - elu + - fast_gelu + - gelu + - glu + - gumbel_softmax + - hardshrink + - hardsigmoid + - hardswish + - hardtanh + - leaky_relu + - log_softmax + - logsigmoid + - mish + - prelu + - relu + - relu6 + - rrelu + - selu + - sigmoid + - silu + - softmax + - softmin + - softshrink + - softsign + - tanh + - threshold + - cdist + - dist + - pdist + - choice_with_mask + - random_categorical + - log_uniform_candidate_sampler + - uniform_candidate_sampler + - affine_grid + - bounding_box_decode + - bounding_box_encode + - col2im + - check_valid + - crop_and_resize + - grid_sample + - interpolate + - iou + - pad + - padding + - pixel_shuffle + - pixel_unshuffle + - upsample + - abs + - absolute + - accumulate_n + - acos + - arccos + - acosh + - add + - addcdiv + - addcmul + - addmv + - addn + - angle + - arccosh + - arcsin + - arcsinh + - arctan + - arctanh + - arctan2 + - asin + - asinh + - atan + - atan2 + - atanh + - atleast_1d + - atleast_2d + - atleast_3d + - bessel_i0 + - bessel_i0e + - bessel_i1 + - bessel_i1e + - bessel_j0 + - bessel_j1 + - bessel_k0 + - bessel_k0e + - bessel_k1 + - bessel_k1e + - bessel_y0 + - bessel_y1 + - bitwise_and + - bitwise_left_shift + - bitwise_or + - bitwise_right_shift + - bitwise_xor + - ceil + - clamp + - clip + - combinations + - copysign + - cos + - cosh + - cosine_similarity + - cov + - diag_embed + - diff + - deg2rad + - digamma + - div + - divide + - erf + - erfc + - erfinv + - exp + - exp2 + - expm1 + - floor + - floor_div + - floor_mod + - float_power + - fmod + - frac + - gcd + - hypot + - igamma + - igammac + - imag + - i0 + - inv + - invert + - lcm + - ldexp + - lerp + - log + - log2 + - log10 + - log1p + - logaddexp + - logaddexp2 + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - mul + - multiply + - mvlgamma + - neg + - negative + - nextafter + - polar + - polygamma + - positive + - pow + - rad2deg + - ravel + - real + - reciprocal + - remainder + - rot90 + - round + - rsqrt + - sgn + - sign + - signbit + - sin + - sinc + - sinh + - sqrt + - square + - sub + - subtract + - t + - tan + - tanhshrink + - trapz + - tril_indices + - triu_indices + - true_divide + - trunc + - truncate_div + - truncate_mod + - xdivy + - xlogy + - zeta + - all + - amax + - amin + - aminmax + - any + - argmax + - argmin + - cummax + - cummin + - cumprod + - cumsum + - fmax + - histc + - logsumexp + - max + - mean + - median + - min + - norm + - prod + - std + - std_mean + - var + - var_mean + - argsort + - approximate_equal + - equal + - ge + - greater + - greater_equal + - gt + - intopk + - isclose + - isfinite + - isinf + - isnan + - isneginf + - isposinf + - isreal + - le + - less + - less_equal + - lt + - maximum + - minimum + - msort + - ne + - not_equal + - searchsorted + - topk + - bmm + - addbmm + - addmm + - baddbmm + - addr + - adjoint + - cholesky + - cholesky_solve + - batch_dot + - dot + - eig + - inner + - inverse + - geqrf + - ger + - kron + - lu_solve + - lu_unpack + - matmul + - matrix_solve + - matrix_band_part + - matrix_diag + - matrix_diag_part + - matrix_set_diag + - mm + - mv + - outer + - orgqr + - ormqr + - pinv + - svd + - tensor_dot + - logdet + - slogdet + - qr + - trace + - bartlett_window + - blackman_window + - hamming_window + - hann_window + - kaiser_window + - eye + - fill + - full + - full_like + - linspace + - logspace + - one_hot + - arange + - range + - heaviside + - bernoulli + - gamma + - laplace + - multinomial + - multinomial_with_replacement + - rand + - rand_like + - randint + - randint_like + - randn + - randn_like + - random_gamma + - random_poisson + - randperm + - standard_laplace + - standard_normal + - uniform + - argwhere + - batch_to_space_nd + - bincount + - block_diag + - broadcast_to + - cat + - channel_shuffle + - chunk + - column_stack + - concat + - conj + - count_nonzero + - deepcopy + - diag + - diagflat + - diagonal + - dyn_shape + - dsplit + - dstack + - einsum + - expand + - expand_dims + - flip + - fliplr + - flipud + - gather_d + - gather_elements + - gather_nd + - hsplit + - hstack + - masked_fill + - masked_select + - meshgrid + - moveaxis + - movedim + - narrow + - nan_to_num + - nansum + - normal + - nonzero + - population_count + - rank + - repeat_elements + - repeat_interleave + - reshape + - reverse + - reverse_sequence + - roll + - select + - sequence_mask + - shuffle + - size + - slice + - sort + - space_to_batch_nd + - sparse_segment_mean + - split + - squeeze + - stack + - strided_slice + - sum + - swapaxes + - swapdims + - tensor_split + - tile + - tril + - triu + - transpose + - unbind + - unique + - unique_consecutive + - unique_with_pad + - unsorted_segment_max + - unsorted_segment_min + - unsorted_segment_prod + - unsorted_segment_sum + - unsqueeze + - unstack + - view_as_real + - vsplit + - vstack + - where + - cross + - renorm + - tuple_to_array + - clip_by_global_norm + - clip_by_value + - derivative + - jet + +Tensor: + - __abs__ + - __add__ + - __and__ + - __iadd__ + - __ifloordiv__ + - __imatmul__ + - __imod__ + - __imul__ + - __isub__ + - __matmul__ + - __mod__ + - __mul__ + - __neg__ + - __or__ + - __pow__ + - __radd__ + - __rmatmul__ + - __rmod__ + - __rmul__ + - __rpow__ + - __rsub__ + - __sub__ + - __truediv__ + - __xor__ + - abs + - absolute + - acos + - acosh + - add + - addbmm + - addcdiv + - addcmul + - addmm + - addmv + - addr + - all + - amax + - amin + - any + - arccos + - arccosh + - argmax + - angle + - arcsin + - arcsinh + - arctan + - arctanh + - argmin + - argsort + - asin + - asinh + - atan + - atan2 + - atanh + - baddbmm + - bernoulli + - bincount + - bitwise_and + - bitwise_or + - bitwise_xor + - bmm + - broadcast_to + - ceil + - cholesky_solve + - cholesky + - clamp + - clip + - conj + - copysign + - cos + - cosh + - cross + - cummax + - cummin + - cumprod + - cumsum + - deg2rad + - diag + - diagflat + - diff + - digamma + - div + - divide + - equal + - erf + - erfc + - erfinv + - exp + - expand_as + - expm1 + - flip + - fliplr + - flipud + - float_power + - floor + - fmod + - frac + - gather_elements + - geqrf + - ger + - greater + - greater_equal + - half + - hardshrink + - heaviside + - histc + - hypot + - i0 + - igamma + - igammac + - imag + - index_add + - index_fill + - index_put + - index_select + - inner + - int + - inverse + - item + - lcm + - ldexp + - lerp + - log + - log10 + - log1p + - log2 + - logaddexp + - logaddexp2 + - logdet + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - logsumexp + - long + - masked_fill + - masked_scatter + - masked_select + - matmul + - max + - maximum + - mean + - median + - min + - minimum + - moveaxis + - movedim + - msort + - multinomial + - multiply + - mvlgamma + - nan_to_num + - nansum + - narrow + - neg + - negative + - nelement + - new_ones + - new_zeros + - nextafter + - norm + - nonzero + - not_equal + - ormqr + - permute + - pow + - prod + - qr + - ravel + - real + - reciprocal + - remainder + - renorm + - rad2deg + - tile + - repeat_interleave + - reshape + - reshape + - round + - rot90 + - rsqrt + - sum_to_size + - scatter + - sgn + - short + - sigmoid + - sign + - signbit + - sin + - sinc + - sinh + - slogdet + - sort + - split + - sqrt + - square + - squeeze + - std + - subtract + - subtract + - svd + - swapaxes + - swapdims + - t + - take + - tan + - tanh + - trace + - swapaxes + - tile + - topk + - tril + - tensor_split + - transpose + - true_divide + - trunc + - unbind + - unique_consecutive + - unsqueeze + - var + - view + - where + - xlogy + - from_numpy + - std + - take + - var + - all + - any + - copy + - diagonal + - flatten + - resize + - sum + +mint: + - abs + - absolute_import + - add + - add_ex + - all + - any + - any_ex + - arange + - argmax + - avg_pool2d + - baddbmm + - baddbmm_ex + - batch_norm + - binary_cross_entropy_with_logits + - bitwise_and + - bitwise_or + - bitwise_xor + - bmm + - broadcast_to + - cat + - cat_ex + - ceil + - chunk + - clamp + - conv2d + - conv_transpose2d + - cos + - cross + - cummax + - cummin + - cumsum + - div + - divide + - dropout + - embedding + - eq + - erf + - erfinv + - exp + - flatten + - flip + - flip_ex + - fold + - full + - gather + - gelu + - greater + - grid_sample + - group_norm + - gt + - index_select + - interpolate + - isclose + - isfinite + - layer_norm + - le + - leaky_relu + - less + - less_equal + - linear + - linspace + - log + - logical_and + - logical_not + - logical_or + - lt + - masked_select + - matmul + - max + - max_pool2d + - maximum + - mean + - mean_ex + - min + - minimum + - mul + - ne + - neg + - negative + - nonzero + - normal + - one_hot + - ones + - ones_ex + - ones_like + - pad + - permute + - permute_ex + - pow + - prod + - reciprocal + - relu + - remainder + - repeat_interleave + - rsqrt + - searchsorted + - sigmoid + - silu + - sin + - softmax + - softplus + - sort + - split + - sqrt + - sqrt_ex + - square + - stack + - sub + - sub_ex + - sum + - tanh + - tile + - topk + - tril + - triu + - unfold + - unique + - where + - xlogy + - zeros + - zeros_ex + - zeros_like + +mint.nn.functional: + - absolute_import + - avg_pool2d + - batch_norm + - batch_norm_ex + - bce_with_logits + - binary_cross_entropy_with_logits + - conv_transpose2d + - dense + - dropout + - embedding + - fold + - gelu + - grid_sample + - group_norm + - interpolate + - layer_norm + - leaky_relu + - linear + - max_pool2d + - max_pool2d_ex + - normal + - one_hot + - one_hot_ext + - pad + - relu + - sigmoid + - silu + - softmax + - softmax_ex + - softplus + - tanh + - unfold -- Gitee From 419d45c9906e26a24dbae3843b770524e6f53bca Mon Sep 17 00:00:00 2001 From: CSNIU Date: Tue, 6 Aug 2024 19:33:41 +0800 Subject: [PATCH 149/791] BugFix --- .../msprobe/mindspore/compare/acc_compare.py | 1033 ---------------- .../msprobe/mindspore/compare/compare_cli.py | 24 + .../mindspore/compare/distributed_compare.py | 2 +- .../msprobe/mindspore/compare/highlight.py | 100 -- .../msprobe/mindspore/compare/mapping.yaml | 607 ---------- .../msprobe/mindspore/compare/match.py | 36 - .../msprobe/mindspore/compare/ms_compare.py | 72 +- .../msprobe/mindspore/compare/npy_compare.py | 244 ---- debug/accuracy_tools/msprobe/msprobe.py | 7 +- .../msprobe/pytorch/compare/acc_compare.py | 1051 ----------------- .../msprobe/pytorch/compare/compare_cli.py | 7 +- .../pytorch/compare/distributed_compare.py | 1 + .../msprobe/pytorch/compare/pt_compare.py | 20 +- 13 files changed, 52 insertions(+), 3152 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/acc_compare.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/highlight.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/mapping.yaml delete mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/match.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/npy_compare.py delete mode 100644 debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/acc_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/acc_compare.py deleted file mode 100644 index 0464995d5..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/compare/acc_compare.py +++ /dev/null @@ -1,1033 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import json -import multiprocessing -import os.path -import sys - -import numpy as np -import pandas as pd -import openpyxl -from openpyxl.styles import PatternFill -from collections import namedtuple -from dataclasses import dataclass - -from msprobe.mindspore.compare.match import graph_mapping -from msprobe.mindspore.compare.highlight import HighlightRules, get_header_index -from msprobe.mindspore.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ - get_error_message -from msprobe.mindspore.advisor.advisor import Advisor -from msprobe.mindspore.common.log import logger -from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ - format_value, check_file_not_exists, check_configuration_param, task_dumppath_get -from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, create_directory -from msprobe.core.common.const import Const, CompareConst, FileCheckConst -from msprobe.core.common.exceptions import FileCheckException - - -def check_graph_mode(a_op_name, b_op_name): - if "Aten" in a_op_name and "Aten" not in b_op_name: - return True - if "Aten" not in a_op_name and "Aten" in b_op_name: - return True - return False - - -def check_op(npu_dict, bench_dict, fuzzy_match): - a_op_name = npu_dict["op_name"] - b_op_name = bench_dict["op_name"] - graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) - if graph_mode: - return graph_mapping.match(a_op_name[0], b_op_name[0]) - struct_match = check_struct_match(npu_dict, bench_dict) - if not fuzzy_match: - return a_op_name == b_op_name and struct_match - is_match = True - try: - is_match = fuzzy_check_op(a_op_name, b_op_name) - except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) - is_match = False - return is_match and struct_match - - -def check_struct_match(npu_dict, bench_dict): - npu_struct_in = npu_dict.get("input_struct") - bench_struct_in = bench_dict.get("input_struct") - npu_struct_out = npu_dict.get("output_struct") - bench_struct_out = bench_dict.get("output_struct") - is_match = npu_struct_in == bench_struct_in and npu_struct_out == bench_struct_out - if not is_match: - if len(npu_struct_in) == 0 or len(bench_struct_in) == 0 or len(npu_struct_in) != len(bench_struct_in): - return False - struct_in_is_match = check_type_shape_match(npu_struct_in, bench_struct_in) - struct_out_is_match = check_type_shape_match(npu_struct_out, bench_struct_out) - is_match = struct_in_is_match and struct_out_is_match - return is_match - - -def check_type_shape_match(npu_struct, bench_struct): - shape_type_match = False - for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct): - npu_type = npu_type_shape[0] - npu_shape = npu_type_shape[1] - bench_type = bench_type_shape[0] - bench_shape = bench_type_shape[1] - shape_match = npu_shape == bench_shape - type_match = npu_type == bench_type - if not type_match: - if [npu_type, bench_type] in [["Float16", "Float32"], ["Float32", "Float16"]]: - type_match = True - else: - type_match = False - shape_type_match = shape_match and type_match - if not shape_type_match: - return False - return shape_type_match - - -def fuzzy_check_op(npu_name_list, bench_name_list): - if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list): - return False - is_match = True - for npu_name, bench_name in zip(npu_name_list, bench_name_list): - is_match = fuzzy_check_name(npu_name, bench_name) - if not is_match: - break - return is_match - - -def fuzzy_check_name(npu_name, bench_name): - if "forward" in npu_name and "forward" in bench_name: - is_match = rename_api(npu_name, "forward") == rename_api(bench_name, "forward") - elif "backward" in npu_name and "backward" in bench_name: - is_match = rename_api(npu_name, "backward") == rename_api(bench_name, "backward") - else: - is_match = npu_name == bench_name - return is_match - - -def rename_api(npu_name, process): - npu_split = npu_name.split(process) - torch_func_index, in_out = npu_split[0], npu_split[1] - torch_func_split = torch_func_index.rsplit(Const.SEP, 2) - torch_func = str(torch_func_split[0]) + str(in_out) - return torch_func - - -def merge_tensor(tensor_list, summary_compare, md5_compare): - op_dict = {} - op_dict["op_name"] = [] - op_dict["input_struct"] = [] - op_dict["kwargs_struct"] = [] - op_dict["output_struct"] = [] - op_dict["summary"] = [] - op_dict["stack_info"] = [] - - all_mode_bool = not (summary_compare or md5_compare) - if all_mode_bool: - op_dict["data_name"] = [] - - for tensor in tensor_list: - if len(tensor) == 2: - op_dict['stack_info'].append(tensor['full_info']) - break - op_dict["op_name"].append(tensor['full_op_name']) - if not md5_compare: - if tensor['full_op_name'].find("input") != -1: - op_dict["input_struct"].append((tensor['dtype'], tensor['shape'])) - elif tensor['full_op_name'].find("kwarg") != -1: - op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'])) - elif tensor['full_op_name'].find("output") != -1: - op_dict["output_struct"].append((tensor['dtype'], tensor['shape'])) - else: - if tensor['full_op_name'].find("input") != -1: - op_dict["input_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - elif tensor['full_op_name'].find("kwarg") != -1: - op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - elif tensor['full_op_name'].find("output") != -1: - op_dict["output_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - - op_dict["summary"].append([tensor['Max'], tensor['Min'], tensor['Mean'], tensor['Norm']]) - - if all_mode_bool: - op_dict["data_name"].append(tensor['data_name']) - - if not op_dict["kwargs_struct"]: - del op_dict["kwargs_struct"] - return op_dict if op_dict["op_name"] else {} - - -def match_op(npu_queue, bench_queue, fuzzy_match): - for b_index, b_op in enumerate(bench_queue[0: -1]): - if check_op(npu_queue[-1], b_op, fuzzy_match): - return len(npu_queue) - 1, b_index - if check_op(npu_queue[-1], bench_queue[-1], fuzzy_match): - return len(npu_queue) - 1, len(bench_queue) - 1 - for n_index, n_op in enumerate(npu_queue[0: -1]): - if check_op(n_op, bench_queue[-1], fuzzy_match): - return n_index, len(bench_queue) - 1 - return -1, -1 - - -def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=False): - def get_accuracy_core(n_start, n_len, b_start, b_len, key): - min_len = min(n_len, b_len) - npu_stack_info = n_dict.get("stack_info", None) - bench_stack_info = b_dict.get("stack_info", None) - has_stack = npu_stack_info and bench_stack_info - - all_mode_bool = not (summary_compare or md5_compare) - if all_mode_bool: - npu_data_name = n_dict.get("data_name", None) - bench_data_name = b_dict.get("data_name", None) - - for index in range(min_len): - - n_name = n_dict['op_name'][n_start + index] - b_name = b_dict['op_name'][b_start + index] - n_struct = n_dict[key][index] - b_struct = b_dict[key][index] - err_msg = "" - if md5_compare: - result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], - n_struct[2], b_struct[2], - CompareConst.PASS if n_struct[2] == b_struct[2] else CompareConst.DIFF] - if has_stack and index == 0 and key == "input_struct": - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - result.append(result_item) - continue - - if summary_compare: - result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], - " ", " ", " ", " ", " ", " ", " ", " "] - else: - result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], - " ", " ", " ", " ", " "] - - npu_summary_data = n_dict.get("summary")[n_start + index] - result_item.extend(npu_summary_data) - bench_summary_data = b_dict.get("summary")[b_start + index] - result_item.extend(bench_summary_data) - - if summary_compare: - start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) - warning_flag = False - for i, (npu_val, bench_val) in enumerate(zip(npu_summary_data, bench_summary_data)): - if isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)): - diff = npu_val - bench_val - if bench_val != 0: - relative = str(abs((diff / bench_val) * 100)) + '%' - else: - relative = "N/A" - result_item[start_idx + i] = diff - result_item[start_idx + i + 4] = relative - magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) - if magnitude_diff > 0.5: - warning_flag = True - else: - result_item[start_idx + i] = CompareConst.NONE - accuracy_check = CompareConst.WARNING if warning_flag else "" - err_msg += "Need double check api accuracy." if warning_flag else "" - for i in range(start_idx, len(result_item)): - if str(result_item[i]) in ('inf', '-inf', 'nan'): - result_item[i] = f'{result_item[i]}\t' - - result_item.append(accuracy_check if summary_compare else CompareConst.ACCURACY_CHECK_YES) - result_item.append(err_msg) - if has_stack and index == 0 and key == "input_struct": - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - if all_mode_bool: - result_item.append(npu_data_name[n_start + index]) - - result.append(result_item) - - if n_len > b_len: - for index in range(b_len, n_len): - n_name = n_dict['op_name'][n_start + index] - n_struct = n_dict[key][index] - if md5_compare: - result_item = [n_name, CompareConst.NAN, n_struct[0], CompareConst.NAN, - n_struct[1], CompareConst.NAN, n_struct[2], CompareConst.NAN, CompareConst.NAN] - result.append(result_item) - continue - result_item = [n_name, CompareConst.NAN, n_struct[0], CompareConst.NAN, - n_struct[1], CompareConst.NAN, " ", " ", " ", " ", " "] - summary_data = n_dict.get("summary")[n_start + index] - result_item.extend(summary_data) - summary_data = [CompareConst.NAN for _ in range(len(n_dict.get("summary")[0]))] - result_item.extend(summary_data) - - err_msg = "" - result_item.append(CompareConst.ACCURACY_CHECK_YES) - result_item.append(err_msg) - - if has_stack and index == 0 and key == "input_struct": - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - if all_mode_bool: - result_item.append(npu_data_name[n_start + index]) - - result.append(result_item) - - n_num = len(n_dict['op_name']) - b_num = len(b_dict['op_name']) - n_num_input = len([name for name in n_dict['op_name'] if 'input' in name]) - b_num_input = len([name for name in b_dict['op_name'] if 'input' in name]) - n_num_kwarg = len([name for name in n_dict['op_name'] if 'kwarg' in name]) - b_num_kwarg = len([name for name in b_dict['op_name'] if 'kwarg' in name]) - n_num_output = n_num - n_num_input - n_num_kwarg - b_num_output = b_num - b_num_input - b_num_kwarg - get_accuracy_core(0, n_num_input, 0, b_num_input, 'input_struct') - get_accuracy_core(n_num_input, n_num_kwarg, b_num_input, b_num_kwarg, "kwargs_struct") - get_accuracy_core(n_num_input + n_num_kwarg, n_num_output, b_num_input + b_num_kwarg, b_num_output, 'output_struct') - - -def _do_multi_process(input_parma, result_df): - try: - result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - - -def read_dump_data(result_df): - try: - npu_dump_name_list = result_df.iloc[0:, 0].tolist() - npu_dump_tensor_list = result_df.iloc[0:, -1].tolist() - op_name_mapping_dict = {} - for index, _ in enumerate(npu_dump_name_list): - npu_dump_name = npu_dump_name_list[index] - npu_dump_tensor = npu_dump_tensor_list[index] - op_name_mapping_dict[npu_dump_name] = [npu_dump_tensor, npu_dump_tensor] - return op_name_mapping_dict - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - - -def _handle_multi_process(func, input_parma, result_df, lock): - process_num = int((multiprocessing.cpu_count() + 1) / 2) - op_name_mapping_dict = read_dump_data(result_df) - - df_chunk_size = len(result_df) // process_num - if df_chunk_size > 0: - df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] - else: - df_chunks = [result_df] - - results = [] - pool = multiprocessing.Pool(process_num) - - def err_call(args): - logger.error('multiprocess compare failed! Reason: {}'.format(args)) - try: - pool.terminate() - except OSError as e: - logger.error("pool terminate failed") - - for process_idx, df_chunk in enumerate(df_chunks): - idx = df_chunk_size * process_idx - result = pool.apply_async(func, - args=(idx, op_name_mapping_dict, df_chunk, lock, input_parma), - error_callback=err_call) - results.append(result) - final_results = [r.get() for r in results] - pool.close() - pool.join() - return pd.concat(final_results, ignore_index=True) - - -def compare_ops(idx, dump_path_dict, result_df, lock, input_parma): - cos_result = [] - max_err_result = [] - max_relative_err_result = [] - err_mess = [] - one_thousand_err_ratio_result = [] - five_thousand_err_ratio_result = [] - is_print_compare_log = input_parma.get("is_print_compare_log") - for i in range(len(result_df)): - op_name = result_df.iloc[i, 0] - if is_print_compare_log: - logger.info("start compare: {}".format(op_name)) - cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = compare_by_op( - op_name, dump_path_dict, input_parma) - if is_print_compare_log: - logger.info( - "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " - "five_thousand_err_ratio {}".format(op_name, cos_sim, max_abs_err, max_relative_err, err_msg, - one_thousand_err_ratio, five_thousand_err_ratio)) - cos_result.append(cos_sim) - max_err_result.append(max_abs_err) - max_relative_err_result.append(max_relative_err) - err_mess.append(err_msg) - one_thousand_err_ratio_result.append(one_thousand_err_ratio) - five_thousand_err_ratio_result.append(five_thousand_err_ratio) - - cr = ComparisonResult( - cos_result=cos_result, - max_err_result=max_err_result, - max_relative_err_result=max_relative_err_result, - err_msgs=err_mess, - one_thousand_err_ratio_result=one_thousand_err_ratio_result, - five_thousand_err_ratio_result=five_thousand_err_ratio_result - ) - - return _save_cmp_result(idx, cr, result_df, lock) - - -@dataclass -class ComparisonResult: - cos_result: list - max_err_result: list - max_relative_err_result: list - err_msgs: list - one_thousand_err_ratio_result: list - five_thousand_err_ratio_result: list - - -def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): - """ - Save comparison results into the result DataFrame with thread safety. - Args: - offset: offset for index - result: data struct of ComparisonResult - result_df: result of DataFrame - lock: thread lock - - Returns: - comparison results in DataFrame - """ - - lock.acquire() - try: - for i, _ in enumerate(result.cos_result): - process_index = i + offset - result_df.loc[process_index, CompareConst.COSINE] = result.cos_result[i] - result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] - result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] - result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] - result_df.loc[process_index, CompareConst.ACCURACY] = check_accuracy(result.cos_result[i], result.max_err_result[i]) - result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = result.one_thousand_err_ratio_result[i] - result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = result.five_thousand_err_ratio_result[i] - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - finally: - lock.release() - - -def check_accuracy(cos, max_abs_err): - if cos == CompareConst.SHAPE_UNMATCH: - return CompareConst.ACCURACY_CHECK_UNMATCH - if cos == CompareConst.NONE or max_abs_err == CompareConst.NONE: - return CompareConst.NONE - if cos == "N/A" or max_abs_err == "N/A": - return CompareConst.ACCURACY_CHECK_NO - try: - cos, max_abs_err = float(cos), float(max_abs_err) - except ValueError: - logger.warning("Cosine or MaxAbsErr can not get float value.") - return CompareConst.NONE - if cos < CompareConst.COS_THRESHOLD and max_abs_err > CompareConst.MAX_ABS_ERR_THRESHOLD: - return CompareConst.ACCURACY_CHECK_NO - if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: - return CompareConst.ACCURACY_CHECK_NO - return CompareConst.ACCURACY_CHECK_YES - - -def read_npy_data(dir_path, file_name): - data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.NUMPY_SUFFIX, False) - data_path = path_checker.common_check() - data_value = np.load(data_path) # detach for less memory - if data_value.dtype == np.float16: - data_value=data_value.astype(np.float32) - - return data_value - - -def compare_by_op(op_name, op_name_mapping_dict, input_parma): - npu_bench_name_list = op_name_mapping_dict[op_name] - data_name = npu_bench_name_list[1] - error_file, relative_err, error_flag = None, None, False - if data_name == '-1' or data_name == -1: # 没有真实数据路径 - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - else: - try: - n_value = read_npy_data(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) - b_value = read_npy_data(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) - except IOError as error: - error_file = error.filename - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - except FileCheckException: - error_file = data_name - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - - n_value, b_value, error_flag = get_error_type(n_value, b_value, error_flag) - if not error_flag: - relative_err = get_relative_err(n_value, b_value) - n_value, b_value = reshape_value(n_value, b_value) - - err_msg = get_error_message(n_value, b_value, op_name, error_flag, error_file=error_file) - result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=relative_err) - - if npu_bench_name_list[0] != npu_bench_name_list[1]: - err_msg += " Fuzzy matching data, the comparison accuracy may be affected." - result_list.append(err_msg) - return result_list - - -def handle_inf_nan(n_value, b_value): - n_inf = np.isinf(n_value) - b_inf = np.isinf(b_value) - n_nan = np.isnan(n_value) - b_nan = np.isnan(b_value) - - # merge boolean expressions - any_inf = np.any(n_inf) or np.any(b_inf) - any_nan = np.any(n_nan) or np.any(b_nan) - if any_inf or any_nan: - if np.array_equal(n_inf, b_inf) and np.array_equal(n_nan, b_nan): - n_value[n_inf] = 0 - b_value[b_inf] = 0 - n_value[n_nan] = 0 - b_value[b_nan] = 0 - else: - return CompareConst.NAN, CompareConst.NAN - return n_value, b_value - - -def find_error_rows(result, last_len, n_num_input, highlight_dict, summary_compare=False, md5_compare=False): - """找到单个API中需要高亮的行""" - if md5_compare: - return - npu_max_index = get_header_index('NPU max', summary_compare) - bench_max_index = get_header_index('Bench max', summary_compare) - max_diff_index = get_header_index('Max diff' if summary_compare else 'MaxAbsErr', summary_compare) - - red_lines, yellow_lines = [], [] - LineInfo = namedtuple('LineInfo', ['line_data', 'num_pointer']) - ApiInfo = namedtuple('ApiInfo', ['api_input', 'api_output', 'num_pointer']) - ColorColumns = namedtuple('ColorColumns', ['red', 'yellow']) - color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) - - # 对单行API的输入或输出进行误差判断 - for i, line in enumerate(result): - num = last_len + i - line_info = LineInfo(line_data=line, num_pointer=num) - for rule in HighlightRules.basic_rules.values(): - rule.apply(line_info, color_columns, summary_compare) - - # 对API的输出与输入比较，进行误差判断 - for n, api_out in enumerate(result[n_num_input:len(result)]): - num = last_len + n_num_input + n - if num in red_lines: - continue - if not isinstance(api_out[npu_max_index], (float, int)) \ - or not isinstance(api_out[bench_max_index], (float, int)) \ - or not isinstance(api_out[max_diff_index], (float, int)): - continue - for _, api_in in enumerate(result[0:n_num_input]): - if not isinstance(api_in[npu_max_index], (float, int)) \ - or not isinstance(api_in[bench_max_index], (float, int)) \ - or not isinstance(api_in[max_diff_index], (float, int)): - continue - - api_info = ApiInfo(api_input=api_in, api_output=api_out, num_pointer=num) - if summary_compare: - for rule in HighlightRules.summary_compare_rules.values(): - rule.apply(api_info, color_columns, summary_compare) - else: - for rule in HighlightRules.compare_rules.values(): - rule.apply(api_info, color_columns, summary_compare) - - highlight_dict.get('red_rows', []).extend(list(set(red_lines))) - highlight_dict.get('yellow_rows', []).extend(list(set(yellow_lines) - set(red_lines))) - - -def get_name_and_state(name): - """Get api/module name and state""" - if "input" in name: - api_name = name.split("input")[0] - state = "input" - else: - api_name = name.split("output")[0] - state = "output" - return api_name, state - - -def find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare): - """将dataframe根据API分组，并找到有误差的算子用于高亮""" - result = result_df.values - start, input_num, output_num, end = 0, 0, 0, len(result_df) - last_api_name, last_state = None, None - num, last_len = 0, 0 - for res_i in result: - api_name, state = get_name_and_state(res_i[0]) - if last_api_name: - if api_name == last_api_name: - if state == last_state: - num += 1 - else: - input_num = num - num, last_state = 1, state - else: - output_num = num - find_error_rows(result[start:start + input_num + output_num], start, input_num, highlight_dict, - summary_compare, md5_compare) - num, last_api_name, last_state = 1, api_name, state - start += input_num + output_num - input_num, output_num = 1, 0 - else: - num, last_api_name, last_state = 1, api_name, state - if state: - if state == "input": - input_num = num - else: - output_num = num - find_error_rows(result[start:start + input_num + output_num], start, input_num, highlight_dict, summary_compare, md5_compare) - - -def highlight_rows_xlsx(result_df, highlight_dict, file_path): - """Write and highlight results in Excel""" - logger.info('Compare result is %s' % file_path) - - wb = openpyxl.Workbook() - ws = wb.active - - # write header - for j, col_name in enumerate(result_df.columns, start=1): - ws.cell(row=1, column=j, value=col_name) - - for i, row in enumerate(result_df.iterrows(), start=2): - for j, value in enumerate(row[1], start=1): - if not isinstance(value, (float, int)): - value = f'{str(value)}\t' if str(value) in ('inf', '-inf', 'nan') else str(value) - ws.cell(row=i, column=j, value=f'{str(value)}\t' if str(value) in ('inf', '-inf', 'nan') else value) - - if (i - 2) in highlight_dict['red_rows']: - ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.RED, - end_color=CompareConst.RED, fill_type="solid") - elif (i - 2) in highlight_dict['yellow_rows']: - ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.YELLOW, - end_color=CompareConst.YELLOW, fill_type="solid") - wb.save(file_path) - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - - -def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, - fuzzy_match=False): - try: - summary_compare, md5_compare = task_dumppath_get(input_parma) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) - create_directory(output_path) - check_compare_param(input_parma, output_path, stack_mode, summary_compare, md5_compare) - except CompareException as error: - logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) - compare_core(input_parma, output_path, stack_mode=stack_mode, - auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) - - -def compare_core(input_parma, output_path, **kwargs): - """ - Compares data from multiple JSON files and generates a comparison report. - - Args: - input_parma (dict): A dictionary containing paths to JSON files ("npu_json_path", "bench_json_path", - "stack_json_path"). - output_path (str): The path where the output Excel report will be saved. - **kwargs: Additional keyword arguments including: - - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. - - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. - - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. - - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. - - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. - - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. - - Returns: - """ - # get kwargs or set default value - stack_mode = kwargs.get('stack_mode', False) - auto_analyze = kwargs.get('auto_analyze', True) - suffix = kwargs.get('suffix', '') - fuzzy_match = kwargs.get('fuzzy_match', False) - summary_compare = kwargs.get('summary_compare', False) - md5_compare = kwargs.get('md5_compare', False) - - logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - file_name = add_time_with_xlsx("compare_result" + suffix) - file_path = os.path.join(os.path.realpath(output_path), file_name) - check_file_not_exists(file_path) - highlight_dict = {'red_rows': [], 'yellow_rows': []} - - with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_json_path"), "r") as stack_json: - result_df = compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, - summary_compare, md5_compare) - - if not md5_compare and not summary_compare: - result_df = _do_multi_process(input_parma, result_df) - find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) - highlight_rows_xlsx(result_df, highlight_dict, file_path) - if auto_analyze: - advisor = Advisor(result_df, output_path) - advisor.analysis() - - -def parse(pkl_file, module_name_prefix): - if not isinstance(module_name_prefix, str): - logger.error("The parameter:module_name_prefix is not a string.") - raise CompareException(CompareException.INVALID_PARAM_ERROR) - with FileOpen(pkl_file, "r") as f: - done = False - title_printed = False - while not done: - pkl_line = f.readline() - if pkl_line == '\n': - continue - if len(pkl_line) == 0: - done = True - break - - msg = json.loads(pkl_line) - info_prefix = msg[0] - if not info_prefix.startswith(module_name_prefix): - continue - - if info_prefix.find("stack_info") != -1: - logger.info("\nTrace back({}):".format(msg[0])) - for item in reversed(msg[1]): - logger.info(" File \"{}\", line {}, in {}".format(item[0], item[1], item[2])) - logger.info(" {}".format(item[3])) - continue - if len(msg) > 5: - summary_info = " [{}][dtype: {}][shape: {}][max: {}][min: {}][mean: {}]" \ - .format(msg[0], msg[3], msg[4], msg[5][0], msg[5][1], msg[5][2]) - if not title_printed: - logger.info("\nStatistic Info:") - title_printed = True - logger.info(summary_info) - - -def op_item_parse(item, op_name, index, item_list=None, top_bool=True): - if item_list is None: - item_list = [] - if item is None or (isinstance(item, dict) and not item): - if not top_bool: - tmp = {'full_op_name': op_name + '.' + str(index), 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, - 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'} - else: - tmp = {'full_op_name': op_name + '.0', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, - 'shape': None, 'md5': None, 'data_name': '-1'} - item_list.append(tmp) - return item_list - if index is None: - if isinstance(item, dict): - full_op_name = op_name + '.0' - else: - full_op_name = op_name - else: - full_op_name = op_name + Const.SEP + str(index) - if isinstance(item, dict): - if 'type' not in item: - for kwarg in item: - kwarg_parsed_list = op_item_parse(item[kwarg], op_name + Const.SEP + kwarg, None) - item_list += kwarg_parsed_list - kwarg_parsed_list.clear() - elif 'dtype' in item: - parsed_item = item - parsed_item['full_op_name'] = full_op_name - item_list.append(parsed_item) - elif 'type' in item: - parsed_item = {} - if item['type'] == 'torch.Size': - parsed_item['full_op_name'] = full_op_name - parsed_item['dtype'] = 'torch.Size' - parsed_item['shape'] = str(item['value']) - parsed_item['md5'] = None - parsed_item['Max'] = None - parsed_item['Min'] = None - parsed_item['Mean'] = None - parsed_item['Norm'] = None - parsed_item['data_name'] = '-1' - item_list.append(parsed_item) - elif item['type'] == 'slice': - parsed_item['full_op_name'] = full_op_name - parsed_item['dtype'] = 'slice' - parsed_item['shape'] = str(np.shape(np.array(item['value']))) - parsed_item['md5'] = None - parsed_item['Max'] = None - parsed_item['Min'] = None - parsed_item['Mean'] = None - parsed_item['Norm'] = None - parsed_item['data_name'] = '-1' - item_list.append(parsed_item) - else: - parsed_item['full_op_name'] = full_op_name - parsed_item['dtype'] = str(type(item['value'])) - parsed_item['shape'] = '[]' - parsed_item['md5'] = None - parsed_item['Max'] = item['value'] - parsed_item['Min'] = item['value'] - parsed_item['Mean'] = item['value'] - parsed_item['Norm'] = item['value'] - parsed_item['data_name'] = '-1' - item_list.append(parsed_item) - else: - resolve_api_special_parameters(item, full_op_name, item_list) - else: - for j, item_spec in enumerate(item): - op_item_parse(item_spec, full_op_name, j, item_list=item_list, top_bool=False) - return item_list - - -def resolve_api_special_parameters(data_dict, full_op_name, item_list): - """ - Function Description: - 解析下面格式的数据, 是api参数的一种特殊格式 - { - "last_hidden_state": { - "type": "torch.Tensor", - "dtype": "torch.bfloat16", - ... - }, - "loss": { - "type": "torch.Tensor", - "dtype": "torch.float32", - ... - } - } - Parameter: - data_dict: 字典格式的数据 - full_op_name: 参数的全名字符串 - item_list: 参数信息集合 - """ - for key, value in data_dict.items(): - if isinstance(value, dict): - parsed_item = value - parts = full_op_name.split(".") - parts.insert(-1, key) - full_op_name_new = ".".join(parts) - parsed_item['full_op_name'] = full_op_name_new - item_list.append(parsed_item) - - -def read_op(op_data, op_name): - op_parsed_list = [] - if 'forward' in op_name: - if 'input_args' in op_data: - input_item = op_data['input_args'] - input_parsed_list = op_item_parse(input_item, op_name + '_input', None) - op_parsed_list = input_parsed_list.copy() - input_parsed_list.clear() - if 'input_kwargs' in op_data: - kwargs_item = op_data['input_kwargs'] - if isinstance(kwargs_item, dict) and "type" in kwargs_item or isinstance(kwargs_item, list): - kwarg_parsed_list = op_item_parse(kwargs_item, op_name + '_input', None) - op_parsed_list += kwarg_parsed_list - kwarg_parsed_list.clear() - elif kwargs_item: - for kwarg in kwargs_item: - kwarg_parsed_list = op_item_parse(kwargs_item[kwarg], op_name + '_input.' + kwarg, None) - op_parsed_list += kwarg_parsed_list - kwarg_parsed_list.clear() - if 'output' in op_data: - output_item = op_data['output'] - output_parsed_list = op_item_parse(output_item, op_name + '_output', None) - op_parsed_list += output_parsed_list - output_parsed_list.clear() - if 'backward' in op_name: - if 'grad_input' in op_data: - input_item = op_data['grad_input'] - input_parsed_list = op_item_parse(input_item, op_name + '_input', None) - op_parsed_list = input_parsed_list.copy() - input_parsed_list.clear() - if 'grad_output' in op_data: - output_item = op_data['grad_output'] - output_parsed_list = op_item_parse(output_item, op_name + '_output', None) - op_parsed_list += output_parsed_list - output_parsed_list.clear() - return op_parsed_list - - -def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): - npu_json_handle, bench_json_handle, stack_json_handle = file_handles - npu_json_data = json.load(npu_json_handle) - bench_json_data = json.load(bench_json_handle) - stack_json_data = json.load(stack_json_handle) - - if fuzzy_match: - logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") - - npu_ops_queue = [] - bench_ops_queue = [] - result = [] - - ops_npu_iter = iter(npu_json_data['data']) - ops_bench_iter = iter(bench_json_data['data']) - read_err_npu = True - read_err_bench = True - last_npu_ops_len = 0 - last_bench_ops_len = 0 - - while True: - if not read_err_npu and not read_err_bench: - break - try: - last_npu_ops_len = len(npu_ops_queue) - op_name_npu = next(ops_npu_iter) - read_err_npu = True - - npu_op_data = npu_json_data['data'][op_name_npu] - npu_op_parsed_list = read_op(npu_op_data, op_name_npu) - if op_name_npu in stack_json_data: - npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': stack_json_data[op_name_npu]}) - else: - npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': None}) - - npu_merge_list = merge_tensor(npu_op_parsed_list, summary_compare, md5_compare) - if npu_merge_list: - npu_ops_queue.append(npu_merge_list) - except StopIteration: - read_err_npu = False - try: - last_bench_ops_len = len(bench_ops_queue) - op_name_bench = next(ops_bench_iter) - - bench_op_data = bench_json_data['data'][op_name_bench] - bench_op_parsed_list = read_op(bench_op_data, op_name_bench) - if op_name_bench in stack_json_data: - bench_op_parsed_list.append( - {'full_op_name': op_name_bench, 'full_info': stack_json_data[op_name_bench]}) - else: - bench_op_parsed_list.append({'full_op_name': op_name_bench, 'full_info': None}) - - bench_merge_list = merge_tensor(bench_op_parsed_list, summary_compare, md5_compare) - if bench_merge_list: - bench_ops_queue.append(bench_merge_list) - except StopIteration: - read_err_bench = False - - # merge all boolean expressions - both_empty = not npu_ops_queue and not bench_ops_queue - no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) - if both_empty or no_change: - continue - - n_match_point, b_match_point = match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) - if n_match_point == -1 and b_match_point == -1: - continue - n_match_data = npu_ops_queue[n_match_point] - b_match_data = bench_ops_queue[b_match_point] - un_match_data = npu_ops_queue[0: n_match_point] - for npu_data in un_match_data: - get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - get_accuracy(result, n_match_data, b_match_data, summary_compare, md5_compare) - del npu_ops_queue[0: n_match_point + 1] - del bench_ops_queue[0: b_match_point + 1] - if npu_ops_queue: - for npu_data in npu_ops_queue: - get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - - header = [] - if md5_compare: - header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] - elif summary_compare: - header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] - else: - header = CompareConst.COMPARE_RESULT_HEADER[:] - - all_mode_bool = not (summary_compare or md5_compare) - if stack_mode: - if all_mode_bool: - header.append(CompareConst.STACK) - header.append(CompareConst.DATA_NAME) - else: - header.append(CompareConst.STACK) - else: - if all_mode_bool: - for row in result: - del row[-2] - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] - - result_df = pd.DataFrame(result, columns=header) - return result_df - - -def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): - index_out = 0 - npu_stack_info = n_dict.get("stack_info", None) - bench_name, bench_type, bench_shape = CompareConst.NAN, CompareConst.NAN, CompareConst.NAN - err_msg = CompareConst.NO_BENCH - accuracy_check_res = CompareConst.NAN - for index, n_name in enumerate(n_dict["op_name"]): - if n_name.find("input") != -1: - n_struct = n_dict["input_struct"][index] - else: - n_struct = n_dict["output_struct"][index_out] - index_out += 1 - - result_item = [n_name, bench_name, n_struct[0], bench_type, n_struct[1], bench_shape] - if md5_compare: - result_item.extend([CompareConst.NAN] * 3) - if npu_stack_info and index == 0: - result_item.extend(npu_stack_info) - result.append(result_item) - continue - if summary_compare: - result_item.extend([CompareConst.NAN] * 8) - else: - result_item.extend([CompareConst.NAN] * 5) - summary_data = n_dict.get("summary")[index] - result_item.extend(summary_data) - summary_data = [CompareConst.NAN] * 4 - result_item.extend(summary_data) - result_item.append(accuracy_check_res) - result_item.append(err_msg) - if npu_stack_info and index == 0: - result_item.extend(npu_stack_info) - if not md5_compare and not summary_compare and result_item[1] == CompareConst.NAN: - if index == 0: - result_item.extend(["-1"]) - else: - result_item.extend([CompareConst.NONE, "-1"]) - result.append(result_item) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py new file mode 100644 index 000000000..361e957f2 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py @@ -0,0 +1,24 @@ +import json +from msprobe.core.common.file_check import FileOpen, check_file_type +from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.utils import CompareException +from msprobe.core.common.log import logger +from msprobe.mindspore.compare.ms_compare import ms_compare +from msprobe.mindspore.compare.distributed_compare import compare_distributed + + +def compare_cli_ms(args): + with FileOpen(args.input_path, "r") as file: + input_param = json.load(file) + npu_path = input_param.get("npu_path", None) + bench_path = input_param.get("bench_path", None) + + if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: + ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, + fuzzy_match=args.fuzzy_match) + elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: + kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} + compare_distributed(npu_path, bench_path, args.output_path, **kwargs) + else: + logger.error("The npu_path and bench_path need to be of the same type.") + raise CompareException(CompareException.INVALID_COMPARE_MODE) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 303692dec..94d03f4f2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -57,8 +57,8 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): def extract_json(dirname, stack_json=False): json_path = '' for fname in os.listdir(dirname): - full_path = os.path.join(dirname, fname) if fname=="construct.json": continue + full_path = os.path.join(dirname, fname) if full_path.endswith('.json'): json_path = full_path if not stack_json and 'stack' not in json_path: diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/highlight.py b/debug/accuracy_tools/msprobe/mindspore/compare/highlight.py deleted file mode 100644 index 82f0022f8..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/compare/highlight.py +++ /dev/null @@ -1,100 +0,0 @@ -import math -import abc -import numpy as np -from msprobe.core.common.utils import get_header_index -from msprobe.core.common.const import CompareConst - - -class HighlightCheck(abc.ABC): - @abc.abstractmethod - def apply(self, info, color_columns, summary_compare): - raise NotImplementedError - - -class CheckOrderMagnitude(HighlightCheck): - """检查Max diff的数量级差异""" - def apply(self, info, color_columns, summary_compare=True): - api_in, api_out, num = info - max_diff_index = get_header_index('Max diff' if summary_compare else 'MaxAbsErr', summary_compare) - if abs(api_in[max_diff_index]) > abs(api_out[max_diff_index]): - return - in_order = 0 if abs(api_in[max_diff_index]) < 1 else math.log10(abs(api_in[max_diff_index])) - out_order = 0 if abs(api_out[max_diff_index]) < 1 else math.log10(abs(api_out[max_diff_index])) - if out_order - in_order >= CompareConst.ORDER_MAGNITUDE_DIFF_YELLOW: - color_columns.yellow.append(num) - - -class CheckOneThousandErrorRatio(HighlightCheck): - """检查千分误差比率""" - def apply(self, info, color_columns, summary_compare=True): - api_in, api_out, num = info - one_thousand_index = get_header_index('One Thousandth Err Ratio', summary_compare) - if not isinstance(api_in[one_thousand_index], (float, int)) or not isinstance(api_out[one_thousand_index], (float, int)): - return - if api_in[one_thousand_index] > CompareConst.ONE_THOUSAND_ERROR_IN_RED and api_out[one_thousand_index] < CompareConst.ONE_THOUSAND_ERROR_OUT_RED: - color_columns.red.append(num) - elif api_in[one_thousand_index] - api_out[one_thousand_index] > CompareConst.ONE_THOUSAND_ERROR_DIFF_YELLOW: - color_columns.yellow.append(num) - - -class CheckCosineSimilarity(HighlightCheck): - """检查余弦相似度""" - def apply(self, info, color_columns, summary_compare=True): - api_in, api_out, num = info - cosine_index = get_header_index('Cosine', summary_compare) - if not isinstance(api_in[cosine_index], (float, int)) or not isinstance(api_out[cosine_index], (float, int)): - return - if api_in[cosine_index] - api_out[cosine_index] > CompareConst.COSINE_DIFF_YELLOW: - color_columns.yellow.append(num) - - -class CheckMaxRelativeDiff(HighlightCheck): - """检查最大相对差异""" - def apply(self, info, color_columns, summary_compare=True): - api_in, api_out, num = info - max_diff_index = get_header_index('Max diff', summary_compare) - bench_max_index = get_header_index('Bench max', summary_compare) - input_max_relative_diff = np.abs(np.divide(api_in[max_diff_index], max(0.01, api_in[bench_max_index]))) - output_max_relative_diff = np.abs(np.divide(api_out[max_diff_index], max(0.01, api_out[bench_max_index]))) - if not isinstance(input_max_relative_diff, (float, int)) or not isinstance(output_max_relative_diff, - (float, int)): - return - if output_max_relative_diff > CompareConst.MAX_RELATIVE_OUT_RED: - color_columns.red.append(num) - elif output_max_relative_diff > CompareConst.MAX_RELATIVE_OUT_YELLOW and input_max_relative_diff < CompareConst.MAX_RELATIVE_IN_YELLOW: - color_columns.yellow.append(num) - - -class CheckOverflow(HighlightCheck): - """检查是否存在溢出""" - def apply(self, info, color_columns, summary_compare=True): - line, num = info - npu_max_index = get_header_index('NPU max', summary_compare) - npu_min_index = get_header_index('NPU min', summary_compare) - max_diff_index = get_header_index('Max diff' if summary_compare else 'MaxAbsErr', summary_compare) - if str(line[npu_max_index]) in CompareConst.OVERFLOW_LIST or str( - line[npu_min_index]) in CompareConst.OVERFLOW_LIST: - color_columns.red.append(num) - return - # check if Max_Diff > 1e+10 - if isinstance(line[max_diff_index], (float, int)) and line[max_diff_index] > CompareConst.MAX_DIFF_RED: - color_columns.red.append(num) - - -class HighlightRules: - """高亮规则集合，用于检查API的误差""" - # 适用于每行的规则 - basic_rules = { - "check_overflow": CheckOverflow() - } - - # 用于比较输入和输出的规则 - compare_rules = { - "check_order_magnitude": CheckOrderMagnitude(), - "check_one_thousand_error": CheckOneThousandErrorRatio(), - "check_cosine_similarity": CheckCosineSimilarity() - } - summary_compare_rules = { - "check_order_magnitude": CheckOrderMagnitude(), - "check_max_relative_diff": CheckMaxRelativeDiff(), - } diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/mapping.yaml b/debug/accuracy_tools/msprobe/mindspore/compare/mapping.yaml deleted file mode 100644 index eaffbe7a1..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/compare/mapping.yaml +++ /dev/null @@ -1,607 +0,0 @@ -__and__: __and__ -__iand__: __iand__ -__ilshift__: __ilshift__ -__ior__: __ior__ -__irshift__: __irshift__ -__ixor__: __ixor__ -__lshift__: __lshift__ -__or__: __or__ -__rshift__: __rshift__ -__xor__: __xor__ -_adaptive_avg_pool2d: adaptive_avg_pool2d -_adaptive_avg_pool3d: adaptive_avg_pool3d -_cdist_forward: cdist -_cudnn_rnn: rnn -_embedding_bag: embedding_bag -_fft_c2c: fft -_fft_c2r: rfft -_foreach_add_: _foreach_add_ -_foreach_addcdiv: _foreach_addcdiv -_foreach_copy_: _foreach_copy_ -_foreach_lerp_: _foreach_lerp_ -_foreach_maximum: _foreach_maximum -_foreach_mul: _foreach_mul -_foreach_neg_: _foreach_neg_ -_foreach_pow: _foreach_pow -_foreach_reciprocal_: _foreach_reciprocal_ -_foreach_sign: _foreach_sign -_foreach_sqrt: _foreach_sqrt -_foreach_sqrt_: _foreach_sqrt_ -_foreach_sub: _foreach_sub -_fused_adam: FusedAdam -_linalg_det: det -_linalg_eigh: eigh -_linalg_slogdet: slogdet -_linalg_svd: svd -_list_to_tensor: as_tensor -_log_softmax: log_softmax -_native_batch_norm_legit: batch_norm -_nested_tensor_from_tensor_list: _nested_tensor_from_tensor_list -_pdist_forward: pdist -_pin_memory: pin_memory -_reshape_alias: reshape -_resize_output_: resize_ -_softmax: softmax -_to_copy: to -abs: abs -abs_: abs_ -absolute: abs -absolute_: abs_ -acos: acos -acos_: acos_ -acosh: acosh -acosh_: acosh_ -adaptive_max_pool2d: adaptive_max_pool2d -adaptive_max_pool3d: adaptive_max_pool3d -add: add -add_: add_ -addbmm: addbmm -addbmm_: addbmm_ -addcdiv: addcdiv -addcdiv_: addcdiv_ -addcmul: addcmul -addcmul_: addcmul_ -addmm: addmm -addmm_: addmm_ -addmv: addmv -addmv_: addmv_ -addr: addr -affine_grid_generator: affine_grid -alias: alias -all: all -alpha_dropout: AlphaDropout -amax: amax -amin: amin -aminmax: aminmax -angle: angle -any: any -arange: arange -arccos: acos -arccos_: arccos_ -arccosh: arccosh -arccosh_: arccosh_ -arcsin: asin -arcsin_: arcsin_ -arcsinh: asinh -arcsinh_: arcsinh_ -arctan: atan -arctan2: atan2 -arctan2_: arctan2_ -arctan_: arctan_ -arctanh: arctanh -arctanh_: arctanh_ -argmax: argmax -argmin: argmin -argsort: argsort -as_strided: as_strided -asin: asin -asin_: asin_ -asinh: asinh -asinh_: asinh_ -atan: atan -atan2: atan2 -atan2_: atan2_ -atan_: atan_ -atanh: atanh -atanh_: atanh_ -avg_pool2d: avg_pool2d -avg_pool3d: avg_pool3d -baddbmm: baddbmm -baddbmm_: baddbmm_ -bernoulli: bernoulli -bernoulli_: bernoulli_ -binary_cross_entropy: BCELoss -binary_cross_entropy_with_logits: binary_cross_entropy_with_logits -bitwise_and: bitwise_and -bitwise_and_: bitwise_and_ -bitwise_left_shift: __lshift__ -bitwise_left_shift_: bitwise_left_shift_ -bitwise_not: bitwise_not -bitwise_not_: bitwise_not_ -bitwise_or: bitwise_or -bitwise_or_: bitwise_or_ -bitwise_right_shift: __rshift__ -bitwise_right_shift_: bitwise_right_shift_ -bitwise_xor: bitwise_xor -bitwise_xor_: bitwise_xor_ -bmm: bmm -broadcast_tensors: broadcast_tensors -bucketize: bucketize -cat: cat -cauchy: Cauchy -cauchy_: cauchy_ -ceil: ceil -ceil_: ceil_ -celu: celu -celu_: celu_ -cholesky: cholesky -cholesky_inverse: cholesky_inverse -cholesky_solve: cholesky_solve -clamp: clamp -clamp_: clamp_ -clamp_max: clamp_max -clamp_max_: clamp_max_ -clamp_min: clamp_min -clamp_min_: clamp_min_ -clip: clip -clip_: clip_ -clone: clone -col2im: col2im -complex: complex -conj_physical: conj -conj_physical_: conj_ -constant_pad_nd: pad -convolution: Conv2d -copy: copy_ -copy_: copy_ -copysign: copysign -copysign_: copysign_ -cos: cos -cos_: cos_ -cosh: cosh -cosh_: cosh_ -count_nonzero: count_nonzero -cudnn_batch_norm: BatchNorm2d -cummax: cummax -cummin: cummin -cumprod: cumprod -cumprod_: cumprod_ -cumsum: cumsum -cumsum_: cumsum_ -deg2rad: deg2rad -deg2rad_: deg2rad_ -detach: detach -diag: diag -diag_embed: diag_embed -diagonal: diagonal -diagonal_copy: diagonal -diagonal_scatter: diagonal -digamma: digamma -digamma_: digamma_ -dist: dist -div: div -div_: div_ -divide: div -divide_: divide_ -dot: dot -dropout: dropout -elu: ELU -elu_: elu_ -embedding: embedding -empty_like: empty_like -empty_strided: empty_strided -eq: eq -eq_: eq_ -erf: erf -erf_: erf_ -erfc: erfc -erfc_: erfc_ -erfinv: erfinv -erfinv_: erfinv_ -exp: exp -exp2: exp2 -exp2_: exp2_ -exp_: exp_ -expand: expand -expm1: expm1 -expm1_: expm1_ -exponential: Exponential -exponential_: exponential_ -eye: eye -fft_fft: fft -fft_fft2: fft2 -fft_fftn: fftn -fft_fftshift: fftshift -fft_hfft: hfft -fft_hfft2: hfft2 -fft_hfftn: hfftn -fft_ifft: ifft -fft_ifft2: ifft2 -fft_ifftn: ifftn -fft_ifftshift: ifftshift -fft_ihfft: ihfft -fft_ihfft2: ihfft2 -fft_ihfftn: ifftn -fft_irfft: irfft -fft_irfft2: irfft2 -fft_irfftn: irfftn -fft_rfft: rfft -fft_rfft2: rfft2 -fft_rfftn: rfftn -fill: fill_ -fill_: fill_ -fix: fix -fix_: fix_ -flip: flip -float_power_: float_power_ -floor: floor -floor_: floor_ -floor_divide: floor_divide -floor_divide_: floor_divide_ -fmax: fmax -fmin: fmin -fmod: fmod -fmod_: fmod_ -frac: frac -frac_: frac_ -full: full -full_like: full_like -gather: gather -gcd: gcd -gcd_: gcd_ -ge: ge -ge_: ge_ -gelu: GELU -gelu_: gelu_ -geometric: Geometric -geometric_: geometric_ -glu: glu -greater: gt -greater_: ge_ -greater_equal: ge -greater_equal_: ge_ -grid_sampler_2d: grid_sample -grid_sampler_3d: grid_sample -gru: GRU -gt: gt -gt_: gt_ -hardshrink: Hardshrink -hardsigmoid: hardsigmoid -hardsigmoid_: hardsigmoid_ -hardswish: hardswish -hardswish_: hardswish_ -hardtanh: hardtanh -hardtanh_: hardtanh_ -heaviside: heaviside -heaviside_: heaviside_ -hinge_embedding_loss: HingeEmbeddingLoss -huber_loss: huber_loss -hypot: hypot -hypot_: hypot_ -i0: i0 -i0_: i0_ -igamma: igamma -igamma_: igamma_ -igammac: igammac -igammac_: igammac_ -index: __getitem__ -index_add: index_add -index_add_: index_add_ -index_copy: index_copy_ -index_copy_: index_copy_ -index_fill: index_fill_ -index_fill_: index_fill_ -index_put: index_put_ -index_put_: index_put_ -index_reduce: index_select -index_select: index_select -is_pinned: is_pinned -is_same_size: is_same_size -isinf: isinf -isnan: isnan -isneginf: isneginf -isposinf: isposinf -istft: istft -item: item -lcm: lcm -lcm_: lcm_ -le: le -le_: le_ -leaky_relu: LeakyReLU -leaky_relu_: leaky_relu_ -lerp: lerp -lerp_: lerp_ -less: less -less_: less_ -less_equal: le -less_equal_: less_equal_ -lgamma: lgamma -lgamma_: lgamma_ -linalg_cholesky_ex: cholesky -linalg_cross: cross -linalg_householder_product: householder_product -linalg_inv_ex: inv -linalg_ldl_factor_ex: ldl -linalg_ldl_solve: ldl_solve -linalg_lu: lu -linalg_lu_factor_ex: lu_factor -linalg_lu_solve: lu_solve -linalg_matrix_exp: matrix_exp -linalg_qr: qr -linalg_solve_triangular: solve -linalg_vector_norm: norm -linspace: linspace -log: log -log10: log10 -log10_: log10_ -log1p: log1p -log1p_: log1p_ -log2: log2 -log2_: log2_ -log_: log_ -log_normal: LogNormal -log_sigmoid_forward: log_sigmoid -logaddexp: logaddexp -logaddexp2: logaddexp2 -_native_batch_norm_legit_functional: batch_norm -logcumsumexp: logcumsumexp -logical_and: logical_and -logical_and_: logical_and_ -logical_not: logical_not -logical_not_: logical_not_ -logical_or: logical_or -logical_or_: logical_or_ -logical_xor: logical_xor -logical_xor_: logical_xor_ -logit: logit -logit_: logit_ -logspace: logspace -logsumexp: logsumexp -lstm: LSTM -lt: lt -lt_: lt_ -lu_unpack: lu_unpack -margin_ranking_loss: margin_ranking_loss -masked_fill: masked_fill -masked_fill_: masked_fill_ -matmul: matmul -max: max -max_pool2d_with_indices: MaxPool2d -max_pool3d_with_indices: MaxPool3d -max_unpool2d: MaxUnpool2d -max_unpool3d: max_unpool3d -maximum: maximum -mean: mean -median: median -meshgrid: meshgrid -min: min -minimum: minimum -mish: Mish -mish_: mish_ -mm: mm -mode: mode -mse_loss: mse_loss -mul: mul -mul_: mul_ -multi_margin_loss: MultiMarginLoss -multilabel_margin_loss_forward: multilabel_margin_loss -multinomial: multinomial -multiply: multiply -multiply_: mul_ -mv: mv -mvlgamma: mvlgamma -mvlgamma_: mvlgamma_ -name: name -nan_to_num: nan_to_num -nan_to_num_: nan_to_num_ -nanmedian: nanmedian -nansum: nansum -narrow_copy: narrow -native_batch_norm: BatchNorm2d -native_dropout: dropout -native_group_norm: group_norm -native_layer_norm: LayerNorm -ne: ne -ne_: ne_ -neg: neg -neg_: neg_ -negative: neg -negative_: neg_ -new_empty: new_empty -new_empty_strided: new_empty_strided -new_full: new_full -new_ones: new_ones -new_zeros: new_zeros -nextafter: nextafter -nextafter_: nextafter_ -nll_loss: nll_loss -nll_loss2d_forward: NLLLoss2d -nll_loss_forward: NLLLoss -nonzero_static: nonzero -norm: norm -normal: normal -normal_: normal_ -not_equal: ne -not_equal_: ne_ -ones: ones -ones_like: ones_like -ormqr: ormqr -pairwise_distance: pairwise_distance -pdist: pdist -permute: permute -pin_memory: pin_memory -pixel_shuffle: PixelShuffle -polar: polar -polygamma: polygamma -positive: positive -pow: pow -pow_: pow_ -prelu: prelu -prod: prod -quantized_gru: GRU -quantized_lstm: LSTM -rad2deg: rad2deg -rad2deg_: rad2deg_ -rand: rand -rand_like: rand_like -randint: randint -randint_like: randint_like -randn: randn -randn_like: randn_like -randperm: randperm -reciprocal: reciprocal -reciprocal_: reciprocal_ -reflection_pad1d: reflection_pad1d -reflection_pad2d: reflection_pad2d -reflection_pad3d: ReflectionPad3d -relu: relu -relu6: relu6 -relu_: relu_ -remainder: remainder -remainder_: remainder_ -renorm: renorm -renorm_: renorm_ -repeat: repeat -repeat_interleave: repeat_interleave -replication_pad1d: ReplicationPad1d -replication_pad2d: replication_pad2d -replication_pad3d: replication_pad3d -resize_as_: resize_as_ -rnn_relu: RNN -rnn_tanh: RNN -roll: roll -rot90: rot90 -round: round -round_: round_ -rrelu_with_noise: RReLU -rrelu_with_noise_: rrelu_with_noise -rsqrt: rsqrt -rsqrt_: rsqrt_ -rsub: rsub -scalar_tensor: scalar_tensor -scatter: scatter_ -scatter_: scatter_ -scatter_add: scatter_add -scatter_add_: scatter_add_ -searchsorted: searchsorted -select: select -selu: selu -selu_: selu_ -sgn: sgn -sgn_: sgn_ -sigmoid: sigmoid -sigmoid_: sigmoid_ -sign: sign -sign_: sign_ -signbit: signbit -silu: silu -silu_: silu_ -sin: sin -sin_: sin_ -sinc: sinc -sinc_: sinc_ -sinh: sinh -sinh_: sinh_ -slice: slice -smooth_l1_loss: smooth_l1_loss -soft_margin_loss: soft_margin_loss -softplus: softplus -softshrink: softshrink -sort: sort -special_airy_ai: airy_ai -special_bessel_j0: j0 -special_bessel_j1: j1 -special_bessel_y0: y0 -special_bessel_y1: y1 -special_chebyshev_polynomial_t: chebyshev_t -special_chebyshev_polynomial_u: chebyshev_u -special_entr: entr -special_erfcx: erfcx -special_hermite_polynomial_h: hermite -special_hermite_polynomial_he: he -special_i0: i0 -special_i0e: i0e -special_i1: i1 -special_i1e: i1e -special_laguerre_polynomial_l: laguerre_l -special_log_ndtr: log_ndtr -special_modified_bessel_i0: i0 -special_modified_bessel_i1: i1 -special_modified_bessel_k0: k0 -special_modified_bessel_k1: i1 -special_ndtr: ndtr -special_ndtri: ndtri -special_scaled_modified_bessel_k0: i0e -special_scaled_modified_bessel_k1: scaled_modified_bessel_k1 -special_spherical_bessel_j0: spherical_jn -special_xlog1py: xlog1py -special_zeta: zeta -split: split -split_with_sizes: split -sqrt: sqrt -sqrt_: sqrt_ -square: square -square_: square_ -squeeze: squeeze -stack: stack -std: std -std_mean: std_mean -stft: stft -sub: sub -sub_: sub_ -subtract: sub -subtract_: subtract_ -sum: sum -t: t -t_: t_ -take: take -tan: tan -tan_: tan_ -tanh: tanh -tanh_: tanh_ -threshold: threshold -threshold_: threshold_ -to: to -topk: topk -trace: trace -transpose: transpose -transpose_: transpose_ -triangular_solve: triangular_solve -tril: tril -tril_: tril_ -tril_indices: tril_indices -triu: triu -triu_: triu_ -triu_indices: triu_indices -true_divide: true_divide -true_divide_: true_divide_ -trunc: trunc -trunc_: trunc_ -unbind: unbind -unfold: unfold -uniform: Uniform -uniform_: uniform_ -unsafe_chunk: unsafe_chunk -unsafe_split: split -unsafe_split_with_sizes: split_with_sizes -unsqueeze: unsqueeze -unsqueeze_: unsqueeze_ -upsample_bicubic2d: interpolate -upsample_bilinear2d: upsample_bilinear -upsample_nearest1d: interpolate -upsample_nearest2d: interpolate -upsample_nearest3d: interpolate -var: var -var_mean: var_mean -vdot: vdot -view: view -where: where -xlogy: xlogy -xlogy_: xlogy_ -zero: zeros -zero_: zero_ -zeros: zeros -zeros_like: zeros_like - - - diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/match.py b/debug/accuracy_tools/msprobe/mindspore/compare/match.py deleted file mode 100644 index 6347d8887..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/compare/match.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import yaml -from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import CompareException - - -class AtenIrMapping(): - def __init__(self): - cur_path = os.path.dirname(os.path.realpath(__file__)) - yaml_path = os.path.join(cur_path, "mapping.yaml") - with FileOpen(yaml_path, 'r') as f: - self.aten_mapping = yaml.safe_load(f) - - def match(self, op1, op2): - if "Aten" in op1 and "Aten" not in op2: - return self.match_op(op1, op2) - else: - return self.match_op(op2, op1) - - def match_op(self, aten_op, torch_op): - try: - aten_op_raw_name_overload = '_'.join(aten_op.split("_")[1:-3]) - aten_op_raw_name = aten_op_raw_name_overload.split('.')[0] - torch_op_raw_name = '_'.join(torch_op.split("_")[1:-3]).lower() - except IndexError as e: - err_msg = f"Dump op name format error: {aten_op}, {torch_op}. Your dump data may be corrupted." - raise CompareException.INVALID_DATA_ERROR(err_msg) from e - matching_op = self.aten_mapping.get(aten_op_raw_name) - if matching_op is None: - return False - if matching_op.lower() == torch_op_raw_name: - return True - return False - - -graph_mapping = AtenIrMapping() diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 9e93a5159..34d37b4fe 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -9,7 +9,7 @@ from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory -from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.core.common.const import CompareConst, FileCheckConst from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op from msprobe.core.compare.Multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process @@ -133,31 +133,6 @@ class MSComparator (Comparator): if npu_ops_queue: for npu_data in npu_ops_queue: get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - - header = [] - if md5_compare: - header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] - elif summary_compare: - header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] - else: - header = CompareConst.COMPARE_RESULT_HEADER[:] - - all_mode_bool = not (summary_compare or md5_compare) - if stack_mode: - if all_mode_bool: - header.append(CompareConst.STACK) - header.append(CompareConst.DATA_NAME) - else: - header.append(CompareConst.STACK) - else: - if all_mode_bool: - for row in result: - del row[-2] - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] - result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df @@ -253,53 +228,24 @@ class MSComparator (Comparator): advisor.analysis() -# def compare(input_parma, output_path, stack_mode=False, auto_analyze=True, -# fuzzy_match=False): -# try: -# summary_compare, md5_compare = task_dumppath_get(input_parma) -# check_configuration_param(stack_mode, auto_analyze, fuzzy_match) -# create_directory(output_path) -# check_compare_param(input_parma, output_path, summary_compare, md5_compare) -# except CompareException as error: -# logger.error('Compare failed. Please check the arguments and do it again!') -# sys.exit(error.code) -# compare_core(input_parma, output_path, stack_mode=stack_mode, -# auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, -# md5_compare=md5_compare) - -def ms_compare(args): - with FileOpen(args.input_path, "r") as file: - input_param = json.load(file) +def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) - check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) - create_directory(args.output_path) - check_compare_param(input_param, args.output_path, summary_compare, md5_compare) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + create_directory(output_path) + check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) - msComparator= MSComparator() - msComparator.compare_core(input_param, args.output_path, stack_mode=args.stack_mode, - auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) + msComparator=MSComparator() + msComparator.compare_core(input_param, output_path, stack_mode=stack_mode, + auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) -def _compare_parser(parser): - parser.add_argument("-i", "--input_path", dest="input_path", type=str, - help=" The compare input path, a dict json.", required=True) - parser.add_argument("-o", "--output_path", dest="output_path", type=str, - help=" The compare task result out path.", required=True) - parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", - help=" Whether to save stack info.", required=False) - parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", - help=" Whether to give advisor.", required=False) - parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", - help=" Whether to perform a fuzzy match on the api name.", required=False) - - \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/npy_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/npy_compare.py deleted file mode 100644 index 4ebe6296b..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/compare/npy_compare.py +++ /dev/null @@ -1,244 +0,0 @@ -import abc -import numpy as np -from msprobe.core.common.utils import format_value -from msprobe.core.common.const import Const, CompareConst -from msprobe.mindspore.common.log import logger - - -def handle_inf_nan(n_value, b_value): - """处理inf和nan的数据""" - n_inf = np.isinf(n_value) - b_inf = np.isinf(b_value) - n_nan = np.isnan(n_value) - b_nan = np.isnan(b_value) - n_invalid = np.any(n_inf) or np.any(n_nan) - b_invalid = np.any(b_inf) or np.any(b_nan) - if n_invalid or b_invalid: - if np.array_equal(n_inf, b_inf) and np.array_equal(n_nan, b_nan): - n_value[n_inf] = 0 - b_value[b_inf] = 0 - n_value[n_nan] = 0 - b_value[b_nan] = 0 - else: - return CompareConst.NAN, CompareConst.NAN - return n_value, b_value - - -def get_error_type(n_value, b_value, error_flag): - """判断数据是否有异常并返回异常的n_value, b_value，同时返回error_flag""" - if error_flag: - return CompareConst.READ_NONE, CompareConst.READ_NONE, True - if n_value.size == 0: # 判断读取到的数据是否为空 - return CompareConst.NONE, CompareConst.NONE, True - if n_value.shape != b_value.shape: # 判断NPU和bench的数据结构是否一致 - return CompareConst.SHAPE_UNMATCH, CompareConst.SHAPE_UNMATCH, True - if not n_value.shape: # 判断数据是否为标量 - return n_value, b_value, False - - n_value, b_value = handle_inf_nan(n_value, b_value) # 判断是否有nan/inf数据 - if n_value is CompareConst.NAN or b_value is CompareConst.NAN: - return CompareConst.NAN, CompareConst.NAN, True - return n_value, b_value, False - - -def reshape_value(n_value, b_value): - """返回reshape后的数据""" - if not n_value.shape: # 判断数据是否为标量 - if n_value.dtype == bool: - n_value = n_value.astype(float) - b_value = b_value.astype(float) - return n_value, b_value - - n_value = n_value.reshape(-1).astype(float) - b_value = b_value.reshape(-1).astype(float) - return n_value, b_value - - -def get_error_message(n_value, b_value, op_name, error_flag, error_file=None): - """获取异常情况的错误信息""" - if error_flag: - if n_value == CompareConst.READ_NONE: - if error_file: - return "Dump file: {} not found.".format(error_file) - return CompareConst.NO_BENCH - if n_value == CompareConst.NONE: - return "This is empty data, can not compare." - if n_value == CompareConst.SHAPE_UNMATCH: - return "Shape of NPU and bench Tensor do not match. Skipped." - if n_value == CompareConst.NAN: - return "The position of inf or nan in NPU and bench Tensor do not match." - else: - if not n_value.shape: - return "This is type of scalar data, can not compare." - if n_value.dtype != b_value.dtype: - logger.warning("Dtype of NPU and bench Tensor do not match: {}".format(op_name)) - return "Dtype of NPU and bench Tensor do not match." - return "" - - -class TensorComparisonBasic(abc.ABC): - """NPU和bench中npy数据的比较模板""" - @abc.abstractmethod - def apply(self, n_value, b_value, error_flag, relative_err=None): - raise NotImplementedError - - -class GetCosineSimilarity(TensorComparisonBasic): - """计算cosine相似度""" - @staticmethod - def correct_data(result): - if result == CompareConst.NAN: - return result - if float(result) > CompareConst.COSINE_THRESHOLD: - return 1.0 - return result - - def apply(self, n_value, b_value, error_flag, relative_err=None): - if error_flag: - if n_value == CompareConst.READ_NONE: - return CompareConst.NONE, '' - if n_value == CompareConst.NONE: - return CompareConst.UNSUPPORTED, '' - if n_value == CompareConst.SHAPE_UNMATCH: - return CompareConst.SHAPE_UNMATCH, '' - if n_value == CompareConst.NAN: - return "N/A", '' - - if not n_value.shape: - return CompareConst.UNSUPPORTED, '' - - with np.errstate(divide='ignore', invalid='ignore'): - if len(n_value) == 1: - return CompareConst.UNSUPPORTED, "This tensor is scalar." - num = n_value.dot(b_value) - a_norm = np.linalg.norm(n_value) - b_norm = np.linalg.norm(b_value) - - if a_norm <= Const.FLOAT_EPSILON and b_norm <= Const.FLOAT_EPSILON: - return 1.0, '' - if a_norm <= Const.FLOAT_EPSILON: - return CompareConst.NAN, 'Cannot compare by Cosine Similarity, All the data is Zero in npu dump data.' - if b_norm <= Const.FLOAT_EPSILON: - return CompareConst.NAN, 'Cannot compare by Cosine Similarity, All the data is Zero in Bench dump data.' - - cos = num / (a_norm * b_norm) - if np.isnan(cos): - return CompareConst.NAN, 'Cannot compare by Cosine Similarity, the dump data has NaN.' - result = format_value(cos) - result = self.correct_data(result) - return 1.0 if float(result) > 0.99999 else result, '' - - -class GetMaxAbsErr(TensorComparisonBasic): - """计算最大绝对误差""" - def apply(self, n_value, b_value, error_flag, relative_err=None): - if error_flag: - if n_value == CompareConst.READ_NONE: - return CompareConst.NONE, "" - if n_value == CompareConst.NONE: - return 0, "" - if n_value == CompareConst.SHAPE_UNMATCH: - return CompareConst.SHAPE_UNMATCH, "" - if n_value == CompareConst.NAN: - return "N/A", "" - - temp_res = n_value - b_value - max_value = np.max(np.abs(temp_res)) - return format_value(max_value), "" - - -def get_relative_err(n_value, b_value): - """计算相对误差""" - with np.errstate(divide='ignore', invalid='ignore'): - if b_value.dtype not in CompareConst.FLOAT_TYPE: - n_value, b_value = n_value.astype(float), b_value.astype(float) - zero_mask = (b_value == 0) - b_value[zero_mask] += np.finfo(b_value.dtype).eps - n_value[zero_mask] += np.finfo(b_value.dtype).eps - relative_err = np.divide((n_value - b_value), b_value) - return np.abs(relative_err) - - -class GetMaxRelativeErr(TensorComparisonBasic): - """计算最大相对误差""" - def apply(self, n_value, b_value, error_flag, relative_err=None): - if error_flag: - if n_value == CompareConst.READ_NONE: - return CompareConst.NONE, '' - if n_value == CompareConst.NONE: - return 0, '' - if n_value == CompareConst.SHAPE_UNMATCH: - return CompareConst.SHAPE_UNMATCH, '' - if n_value == CompareConst.NAN: - return "N/A", '' - - if relative_err is None: - relative_err = get_relative_err(n_value, b_value) - max_relative_err = np.max(np.abs(relative_err)) - if np.isnan(max_relative_err): - message = 'Cannot compare by MaxRelativeError, the data contains nan in dump data.' - return CompareConst.NAN, message - return format_value(max_relative_err), '' - - -class GetThousandErrRatio(TensorComparisonBasic): - """计算相对误差小于千分之一的比例""" - def apply(self, n_value, b_value, error_flag, relative_err=None): - if error_flag: - if n_value == CompareConst.READ_NONE: - return CompareConst.NONE, "" - if n_value == CompareConst.NONE: - return 0, "" - if n_value == CompareConst.SHAPE_UNMATCH: - return CompareConst.SHAPE_UNMATCH, "" - if n_value == CompareConst.NAN: - return "N/A", "" - - if not n_value.shape: - return CompareConst.NAN, "" - if relative_err is None: - relative_err = get_relative_err(n_value, b_value) - if not np.size(relative_err): - return CompareConst.NAN, "" - return format_value(np.sum(relative_err < CompareConst.THOUSAND_RATIO_THRESHOLD) / np.size(relative_err)), "" - - -class GetFiveThousandErrRatio(TensorComparisonBasic): - """计算相对误差小于千分之五的比例""" - def apply(self, n_value, b_value, error_flag, relative_err=None): - if error_flag: - if n_value == CompareConst.READ_NONE: - return CompareConst.NONE, "" - if n_value == CompareConst.NONE: - return 0, "" - if n_value == CompareConst.SHAPE_UNMATCH: - return CompareConst.SHAPE_UNMATCH, "" - if n_value == CompareConst.NAN: - return "N/A", "" - - if not n_value.shape: - return CompareConst.NAN, "" - if relative_err is None: - relative_err = get_relative_err(n_value, b_value) - if not np.size(relative_err): - return CompareConst.NAN, "" - return format_value(np.sum(relative_err < CompareConst.FIVE_THOUSAND_RATIO_THRESHOLD) / np.size(relative_err)), "" - - -class CompareOps: - compare_ops = { - "cosine_similarity": GetCosineSimilarity(), - "max_abs_error": GetMaxAbsErr(), - "max_relative_error": GetMaxRelativeErr(), - "one_thousand_err_ratio": GetThousandErrRatio(), - "five_thousand_err_ratio": GetFiveThousandErrRatio() - } - - -def compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=None): - result_list = [] - for op in CompareOps.compare_ops.values(): - result, msg = op.apply(n_value, b_value, error_flag, relative_err=relative_err) - err_msg += msg - result_list.append(result) - return result_list, err_msg diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index d829d7b9e..5146ee1ac 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -22,9 +22,9 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _ _api_precision_compare_command from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command -from msprobe.pytorch.compare.acc_compare import _compare_parser +from msprobe.pytorch.compare.pt_compare import _compare_parser from msprobe.pytorch.compare.compare_cli import compare_cli - +from msprobe.mindspore.compare.compare_cli import compare_cli_ms def main(): parser = argparse.ArgumentParser( @@ -68,7 +68,8 @@ def main(): _run_overflow_check_command(args) elif sys.argv[3] == "compare": compare_cli(args) - + else: + compare_cli_ms(args) if __name__ == "__main__": main() diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py deleted file mode 100644 index ee107fab4..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/compare/acc_compare.py +++ /dev/null @@ -1,1051 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2019-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import argparse -import json -import multiprocessing -import os.path -import sys -import torch -import numpy as np -import pandas as pd -import openpyxl -from openpyxl.styles import PatternFill -from collections import namedtuple -from dataclasses import dataclass - -from msprobe.pytorch.compare.match import graph_mapping -from msprobe.pytorch.compare.highlight import HighlightRules, get_header_index -from msprobe.pytorch.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ - get_error_message -from msprobe.pytorch.advisor.advisor import Advisor -from msprobe.pytorch.common.log import logger -from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ - format_value, check_file_not_exists, check_configuration_param, task_dumppath_get -from msprobe.core.common.file_check import FileChecker, change_mode, FileOpen, create_directory, check_file_type -from msprobe.core.common.const import Const, CompareConst, FileCheckConst -from msprobe.core.common.exceptions import FileCheckException - - -def check_graph_mode(a_op_name, b_op_name): - if "Aten" in a_op_name and "Aten" not in b_op_name: - return True - if "Aten" not in a_op_name and "Aten" in b_op_name: - return True - return False - - -def check_op(npu_dict, bench_dict, fuzzy_match): - a_op_name = npu_dict["op_name"] - b_op_name = bench_dict["op_name"] - graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) - if graph_mode: - return graph_mapping.match(a_op_name[0], b_op_name[0]) - struct_match = check_struct_match(npu_dict, bench_dict) - if not fuzzy_match: - return a_op_name == b_op_name and struct_match - is_match = True - try: - is_match = fuzzy_check_op(a_op_name, b_op_name) - except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) - is_match = False - return is_match and struct_match - - -def check_struct_match(npu_dict, bench_dict): - npu_struct_in = npu_dict.get("input_struct") - bench_struct_in = bench_dict.get("input_struct") - npu_struct_out = npu_dict.get("output_struct") - bench_struct_out = bench_dict.get("output_struct") - is_match = npu_struct_in == bench_struct_in and npu_struct_out == bench_struct_out - if not is_match: - if len(npu_struct_in) == 0 or len(bench_struct_in) == 0 or len(npu_struct_in) != len(bench_struct_in): - return False - struct_in_is_match = check_type_shape_match(npu_struct_in, bench_struct_in) - struct_out_is_match = check_type_shape_match(npu_struct_out, bench_struct_out) - is_match = struct_in_is_match and struct_out_is_match - return is_match - - -def check_type_shape_match(npu_struct, bench_struct): - shape_type_match = False - for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct): - npu_type = npu_type_shape[0] - npu_shape = npu_type_shape[1] - bench_type = bench_type_shape[0] - bench_shape = bench_type_shape[1] - shape_match = npu_shape == bench_shape - type_match = npu_type == bench_type - if not type_match: - if [npu_type, bench_type] in [["torch.float16", "torch.float32"], ["torch.float32", "torch.float16"], - ["torch.float16", "torch.bfloat16"], ["torch.bfloat16", "torch.float16"]]: - type_match = True - else: - type_match = False - shape_type_match = shape_match and type_match - if not shape_type_match: - return False - return shape_type_match - - -def fuzzy_check_op(npu_name_list, bench_name_list): - if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list): - return False - is_match = True - for npu_name, bench_name in zip(npu_name_list, bench_name_list): - is_match = fuzzy_check_name(npu_name, bench_name) - if not is_match: - break - return is_match - - -def fuzzy_check_name(npu_name, bench_name): - if "forward" in npu_name and "forward" in bench_name: - is_match = rename_api(npu_name, "forward") == rename_api(bench_name, "forward") - elif "backward" in npu_name and "backward" in bench_name: - is_match = rename_api(npu_name, "backward") == rename_api(bench_name, "backward") - else: - is_match = npu_name == bench_name - return is_match - - -def rename_api(npu_name, process): - npu_split = npu_name.split(process) - torch_func_index, in_out = npu_split[0], npu_split[1] - torch_func_split = torch_func_index.rsplit(Const.SEP, 2) - torch_func = str(torch_func_split[0]) + str(in_out) - return torch_func - - -def merge_tensor(tensor_list, summary_compare, md5_compare): - op_dict = {} - op_dict["op_name"] = [] - op_dict["input_struct"] = [] - op_dict["kwargs_struct"] = [] - op_dict["output_struct"] = [] - op_dict["summary"] = [] - op_dict["stack_info"] = [] - - all_mode_bool = not (summary_compare or md5_compare) - if all_mode_bool: - op_dict["data_name"] = [] - - for tensor in tensor_list: - if len(tensor) == 2: - op_dict['stack_info'].append(tensor['full_info']) - break - op_dict["op_name"].append(tensor['full_op_name']) - if not md5_compare: - if tensor['full_op_name'].find("input") != -1: - op_dict["input_struct"].append((tensor['dtype'], tensor['shape'])) - elif tensor['full_op_name'].find("kwarg") != -1: - op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'])) - elif tensor['full_op_name'].find("output") != -1: - op_dict["output_struct"].append((tensor['dtype'], tensor['shape'])) - else: - if tensor['full_op_name'].find("input") != -1: - op_dict["input_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - elif tensor['full_op_name'].find("kwarg") != -1: - op_dict["kwargs_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - elif tensor['full_op_name'].find("output") != -1: - op_dict["output_struct"].append((tensor['dtype'], tensor['shape'], tensor['md5'])) - - op_dict["summary"].append([tensor['Max'], tensor['Min'], tensor['Mean'], tensor['Norm']]) - - if all_mode_bool: - op_dict["data_name"].append(tensor['data_name']) - - if not op_dict["kwargs_struct"]: - del op_dict["kwargs_struct"] - return op_dict if op_dict["op_name"] else {} - - -def match_op(npu_queue, bench_queue, fuzzy_match): - for b_index, b_op in enumerate(bench_queue[0: -1]): - if check_op(npu_queue[-1], b_op, fuzzy_match): - return len(npu_queue) - 1, b_index - if check_op(npu_queue[-1], bench_queue[-1], fuzzy_match): - return len(npu_queue) - 1, len(bench_queue) - 1 - for n_index, n_op in enumerate(npu_queue[0: -1]): - if check_op(n_op, bench_queue[-1], fuzzy_match): - return n_index, len(bench_queue) - 1 - return -1, -1 - - -def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=False): - def get_accuracy_core(n_start, n_len, b_start, b_len, key): - min_len = min(n_len, b_len) - npu_stack_info = n_dict.get("stack_info", None) - bench_stack_info = b_dict.get("stack_info", None) - has_stack = npu_stack_info and bench_stack_info - - all_mode_bool = not (summary_compare or md5_compare) - if all_mode_bool: - npu_data_name = n_dict.get("data_name", None) - bench_data_name = b_dict.get("data_name", None) - - for index in range(min_len): - - n_name = n_dict['op_name'][n_start + index] - b_name = b_dict['op_name'][b_start + index] - n_struct = n_dict[key][index] - b_struct = b_dict[key][index] - err_msg = "" - if md5_compare: - result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], - n_struct[2], b_struct[2], - CompareConst.PASS if n_struct[2] == b_struct[2] else CompareConst.DIFF] - if has_stack and index == 0 and key == "input_struct": - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - result.append(result_item) - continue - - if summary_compare: - result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], - " ", " ", " ", " ", " ", " ", " ", " "] - else: - result_item = [n_name, b_name, n_struct[0], b_struct[0], n_struct[1], b_struct[1], - " ", " ", " ", " ", " "] - - npu_summary_data = n_dict.get("summary")[n_start + index] - result_item.extend(npu_summary_data) - bench_summary_data = b_dict.get("summary")[b_start + index] - result_item.extend(bench_summary_data) - - if summary_compare: - start_idx = CompareConst.SUMMARY_COMPARE_RESULT_HEADER.index(CompareConst.MAX_DIFF) - warning_flag = False - for i, (npu_val, bench_val) in enumerate(zip(npu_summary_data, bench_summary_data)): - if isinstance(npu_val, (float, int)) and isinstance(bench_val, (float, int)): - diff = npu_val - bench_val - if bench_val != 0: - relative = str(abs((diff / bench_val) * 100)) + '%' - else: - relative = "N/A" - result_item[start_idx + i] = diff - result_item[start_idx + i + 4] = relative - magnitude_diff = abs(diff) / (max(abs(npu_val), abs(bench_val)) + 1e-10) - if magnitude_diff > 0.5: - warning_flag = True - else: - result_item[start_idx + i] = CompareConst.NONE - accuracy_check = CompareConst.WARNING if warning_flag else "" - err_msg += "Need double check api accuracy." if warning_flag else "" - for i in range(start_idx, len(result_item)): - if str(result_item[i]) in ('inf', '-inf', 'nan'): - result_item[i] = f'{result_item[i]}\t' - - result_item.append(accuracy_check if summary_compare else CompareConst.ACCURACY_CHECK_YES) - result_item.append(err_msg) - if has_stack and index == 0 and key == "input_struct": - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - if all_mode_bool: - result_item.append(npu_data_name[n_start + index]) - - result.append(result_item) - - if n_len > b_len: - for index in range(b_len, n_len): - n_name = n_dict['op_name'][n_start + index] - n_struct = n_dict[key][index] - if md5_compare: - result_item = [n_name, CompareConst.NAN, n_struct[0], CompareConst.NAN, - n_struct[1], CompareConst.NAN, n_struct[2], CompareConst.NAN, CompareConst.NAN] - result.append(result_item) - continue - result_item = [n_name, CompareConst.NAN, n_struct[0], CompareConst.NAN, - n_struct[1], CompareConst.NAN, " ", " ", " ", " ", " "] - summary_data = n_dict.get("summary")[n_start + index] - result_item.extend(summary_data) - summary_data = [CompareConst.NAN for _ in range(len(n_dict.get("summary")[0]))] - result_item.extend(summary_data) - - err_msg = "" - result_item.append(CompareConst.ACCURACY_CHECK_YES) - result_item.append(err_msg) - - if has_stack and index == 0 and key == "input_struct": - result_item.extend(npu_stack_info) - else: - result_item.append(CompareConst.NONE) - if all_mode_bool: - result_item.append(npu_data_name[n_start + index]) - - result.append(result_item) - - n_num = len(n_dict['op_name']) - b_num = len(b_dict['op_name']) - n_num_input = len([name for name in n_dict['op_name'] if 'input' in name]) - b_num_input = len([name for name in b_dict['op_name'] if 'input' in name]) - n_num_kwarg = len([name for name in n_dict['op_name'] if 'kwarg' in name]) - b_num_kwarg = len([name for name in b_dict['op_name'] if 'kwarg' in name]) - n_num_output = n_num - n_num_input - n_num_kwarg - b_num_output = b_num - b_num_input - b_num_kwarg - get_accuracy_core(0, n_num_input, 0, b_num_input, 'input_struct') - get_accuracy_core(n_num_input, n_num_kwarg, b_num_input, b_num_kwarg, "kwargs_struct") - get_accuracy_core(n_num_input + n_num_kwarg, n_num_output, b_num_input + b_num_kwarg, b_num_output, 'output_struct') - - -def _do_multi_process(input_parma, result_df): - try: - result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - - -def read_dump_data(result_df): - try: - npu_dump_name_list = result_df.iloc[0:, 0].tolist() - npu_dump_tensor_list = result_df.iloc[0:, -1].tolist() - op_name_mapping_dict = {} - for index, _ in enumerate(npu_dump_name_list): - npu_dump_name = npu_dump_name_list[index] - npu_dump_tensor = npu_dump_tensor_list[index] - op_name_mapping_dict[npu_dump_name] = [npu_dump_tensor, npu_dump_tensor] - return op_name_mapping_dict - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - - -def _handle_multi_process(func, input_parma, result_df, lock): - process_num = int((multiprocessing.cpu_count() + 1) / 2) - op_name_mapping_dict = read_dump_data(result_df) - - df_chunk_size = len(result_df) // process_num - if df_chunk_size > 0: - df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] - else: - df_chunks = [result_df] - - results = [] - pool = multiprocessing.Pool(process_num) - - def err_call(args): - logger.error('multiprocess compare failed! Reason: {}'.format(args)) - try: - pool.terminate() - except OSError as e: - logger.error("pool terminate failed") - - for process_idx, df_chunk in enumerate(df_chunks): - idx = df_chunk_size * process_idx - result = pool.apply_async(func, - args=(idx, op_name_mapping_dict, df_chunk, lock, input_parma), - error_callback=err_call) - results.append(result) - final_results = [r.get() for r in results] - pool.close() - pool.join() - return pd.concat(final_results, ignore_index=True) - - -def compare_ops(idx, dump_path_dict, result_df, lock, input_parma): - cos_result = [] - max_err_result = [] - max_relative_err_result = [] - err_mess = [] - one_thousand_err_ratio_result = [] - five_thousand_err_ratio_result = [] - is_print_compare_log = input_parma.get("is_print_compare_log") - for i in range(len(result_df)): - op_name = result_df.iloc[i, 0] - if is_print_compare_log: - logger.info("start compare: {}".format(op_name)) - cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = compare_by_op( - op_name, dump_path_dict, input_parma) - if is_print_compare_log: - logger.info( - "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " - "five_thousand_err_ratio {}".format(op_name, cos_sim, max_abs_err, max_relative_err, err_msg, - one_thousand_err_ratio, five_thousand_err_ratio)) - cos_result.append(cos_sim) - max_err_result.append(max_abs_err) - max_relative_err_result.append(max_relative_err) - err_mess.append(err_msg) - one_thousand_err_ratio_result.append(one_thousand_err_ratio) - five_thousand_err_ratio_result.append(five_thousand_err_ratio) - - cr = ComparisonResult( - cos_result=cos_result, - max_err_result=max_err_result, - max_relative_err_result=max_relative_err_result, - err_msgs=err_mess, - one_thousand_err_ratio_result=one_thousand_err_ratio_result, - five_thousand_err_ratio_result=five_thousand_err_ratio_result - ) - - return _save_cmp_result(idx, cr, result_df, lock) - - -@dataclass -class ComparisonResult: - cos_result: list - max_err_result: list - max_relative_err_result: list - err_msgs: list - one_thousand_err_ratio_result: list - five_thousand_err_ratio_result: list - - -def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): - """ - Save comparison results into the result DataFrame with thread safety. - Args: - offset: offset for index - result: data struct of ComparisonResult - result_df: result of DataFrame - lock: thread lock - - Returns: - comparison results in DataFrame - """ - - lock.acquire() - try: - for i, _ in enumerate(result.cos_result): - process_index = i + offset - result_df.loc[process_index, CompareConst.COSINE] = result.cos_result[i] - result_df.loc[process_index, CompareConst.MAX_ABS_ERR] = result.max_err_result[i] - result_df.loc[process_index, CompareConst.MAX_RELATIVE_ERR] = result.max_relative_err_result[i] - result_df.loc[process_index, CompareConst.ERROR_MESSAGE] = result.err_msgs[i] - result_df.loc[process_index, CompareConst.ACCURACY] = check_accuracy(result.cos_result[i], result.max_err_result[i]) - result_df.loc[process_index, CompareConst.ONE_THOUSANDTH_ERR_RATIO] = result.one_thousand_err_ratio_result[i] - result_df.loc[process_index, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = result.five_thousand_err_ratio_result[i] - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - except IndexError as e: - logger.error('result dataframe elements can not be access.') - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - finally: - lock.release() - - -def check_accuracy(cos, max_abs_err): - if cos == CompareConst.SHAPE_UNMATCH: - return CompareConst.ACCURACY_CHECK_UNMATCH - if cos == CompareConst.NONE or max_abs_err == CompareConst.NONE: - return CompareConst.NONE - if cos == "N/A" or max_abs_err == "N/A": - return CompareConst.ACCURACY_CHECK_NO - try: - cos, max_abs_err = float(cos), float(max_abs_err) - except ValueError: - logger.warning("Cosine or MaxAbsErr can not get float value.") - return CompareConst.NONE - if cos < CompareConst.COS_THRESHOLD and max_abs_err > CompareConst.MAX_ABS_ERR_THRESHOLD: - return CompareConst.ACCURACY_CHECK_NO - if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: - return CompareConst.ACCURACY_CHECK_NO - return CompareConst.ACCURACY_CHECK_YES - - -def read_npy_data(dir_path, file_name): - data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.PT_SUFFIX, False) - data_path = path_checker.common_check() - data_value = torch.load(data_path, map_location=torch.device('cpu')).detach() # detach for less memory - if data_value.dtype == torch.bfloat16: - data_value = data_value.to(torch.float32) - data_value = data_value.numpy() - return data_value - - -def compare_by_op(op_name, op_name_mapping_dict, input_parma): - npu_bench_name_list = op_name_mapping_dict[op_name] - data_name = npu_bench_name_list[1] - error_file, relative_err, error_flag = None, None, False - if data_name == '-1' or data_name == -1: # 没有真实数据路径 - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - else: - try: - n_value = read_npy_data(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) - b_value = read_npy_data(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) - except IOError as error: - error_file = error.filename - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - except FileCheckException: - error_file = data_name - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - - n_value, b_value, error_flag = get_error_type(n_value, b_value, error_flag) - if not error_flag: - relative_err = get_relative_err(n_value, b_value) - n_value, b_value = reshape_value(n_value, b_value) - - err_msg = get_error_message(n_value, b_value, op_name, error_flag, error_file=error_file) - result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=relative_err) - - if npu_bench_name_list[0] != npu_bench_name_list[1]: - err_msg += " Fuzzy matching data, the comparison accuracy may be affected." - result_list.append(err_msg) - return result_list - - -def handle_inf_nan(n_value, b_value): - n_inf = np.isinf(n_value) - b_inf = np.isinf(b_value) - n_nan = np.isnan(n_value) - b_nan = np.isnan(b_value) - - # merge boolean expressions - any_inf = np.any(n_inf) or np.any(b_inf) - any_nan = np.any(n_nan) or np.any(b_nan) - if any_inf or any_nan: - if np.array_equal(n_inf, b_inf) and np.array_equal(n_nan, b_nan): - n_value[n_inf] = 0 - b_value[b_inf] = 0 - n_value[n_nan] = 0 - b_value[b_nan] = 0 - else: - return CompareConst.NAN, CompareConst.NAN - return n_value, b_value - - -def find_error_rows(result, last_len, n_num_input, highlight_dict, summary_compare=False, md5_compare=False): - """找到单个API中需要高亮的行""" - if md5_compare: - return - npu_max_index = get_header_index('NPU max', summary_compare) - bench_max_index = get_header_index('Bench max', summary_compare) - max_diff_index = get_header_index('Max diff' if summary_compare else 'MaxAbsErr', summary_compare) - - red_lines, yellow_lines = [], [] - LineInfo = namedtuple('LineInfo', ['line_data', 'num_pointer']) - ApiInfo = namedtuple('ApiInfo', ['api_input', 'api_output', 'num_pointer']) - ColorColumns = namedtuple('ColorColumns', ['red', 'yellow']) - color_columns = ColorColumns(red=red_lines, yellow=yellow_lines) - - # 对单行API的输入或输出进行误差判断 - for i, line in enumerate(result): - num = last_len + i - line_info = LineInfo(line_data=line, num_pointer=num) - for rule in HighlightRules.basic_rules.values(): - rule.apply(line_info, color_columns, summary_compare) - - # 对API的输出与输入比较，进行误差判断 - for n, api_out in enumerate(result[n_num_input:len(result)]): - num = last_len + n_num_input + n - if num in red_lines: - continue - if not isinstance(api_out[npu_max_index], (float, int)) \ - or not isinstance(api_out[bench_max_index], (float, int)) \ - or not isinstance(api_out[max_diff_index], (float, int)): - continue - for _, api_in in enumerate(result[0:n_num_input]): - if not isinstance(api_in[npu_max_index], (float, int)) \ - or not isinstance(api_in[bench_max_index], (float, int)) \ - or not isinstance(api_in[max_diff_index], (float, int)): - continue - - api_info = ApiInfo(api_input=api_in, api_output=api_out, num_pointer=num) - if summary_compare: - for rule in HighlightRules.summary_compare_rules.values(): - rule.apply(api_info, color_columns, summary_compare) - else: - for rule in HighlightRules.compare_rules.values(): - rule.apply(api_info, color_columns, summary_compare) - - highlight_dict.get('red_rows', []).extend(list(set(red_lines))) - highlight_dict.get('yellow_rows', []).extend(list(set(yellow_lines) - set(red_lines))) - - -def get_name_and_state(name): - """Get api/module name and state""" - if "input" in name: - api_name = name.split("input")[0] - state = "input" - else: - api_name = name.split("output")[0] - state = "output" - return api_name, state - - -def find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare): - """将dataframe根据API分组，并找到有误差的算子用于高亮""" - result = result_df.values - start, input_num, output_num, end = 0, 0, 0, len(result_df) - last_api_name, last_state = None, None - num, last_len = 0, 0 - for res_i in result: - api_name, state = get_name_and_state(res_i[0]) - if last_api_name: - if api_name == last_api_name: - if state == last_state: - num += 1 - else: - input_num = num - num, last_state = 1, state - else: - output_num = num - find_error_rows(result[start:start + input_num + output_num], start, input_num, highlight_dict, - summary_compare, md5_compare) - num, last_api_name, last_state = 1, api_name, state - start += input_num + output_num - input_num, output_num = 1, 0 - else: - num, last_api_name, last_state = 1, api_name, state - if state: - if state == "input": - input_num = num - else: - output_num = num - find_error_rows(result[start:start + input_num + output_num], start, input_num, highlight_dict, summary_compare, md5_compare) - - -def highlight_rows_xlsx(result_df, highlight_dict, file_path): - """Write and highlight results in Excel""" - logger.info('Compare result is %s' % file_path) - - wb = openpyxl.Workbook() - ws = wb.active - - # write header - for j, col_name in enumerate(result_df.columns, start=1): - ws.cell(row=1, column=j, value=col_name) - - for i, row in enumerate(result_df.iterrows(), start=2): - for j, value in enumerate(row[1], start=1): - if not isinstance(value, (float, int)): - value = f'{str(value)}\t' if str(value) in ('inf', '-inf', 'nan') else str(value) - ws.cell(row=i, column=j, value=f'{str(value)}\t' if str(value) in ('inf', '-inf', 'nan') else value) - - if (i - 2) in highlight_dict['red_rows']: - ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.RED, - end_color=CompareConst.RED, fill_type="solid") - elif (i - 2) in highlight_dict['yellow_rows']: - ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.YELLOW, - end_color=CompareConst.YELLOW, fill_type="solid") - try: - wb.save(file_path) - except Exception as e: - logger.error('Save result file failed') - raise CompareException(CompareException.WRITE_FILE_ERROR) from e - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - - -def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): - try: - summary_compare, md5_compare = task_dumppath_get(input_param) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) - create_directory(output_path) - check_compare_param(input_param, output_path, summary_compare, md5_compare) - except (CompareException, FileCheckException) as error: - logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) - compare_core(input_param, output_path, stack_mode=stack_mode, - auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) - - -def compare_core(input_parma, output_path, **kwargs): - """ - Compares data from multiple JSON files and generates a comparison report. - - Args: - input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", - "stack_path"). - output_path (str): The path where the output Excel report will be saved. - **kwargs: Additional keyword arguments including: - - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. - - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. - - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. - - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. - - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. - - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. - - Returns: - """ - # get kwargs or set default value - stack_mode = kwargs.get('stack_mode', False) - auto_analyze = kwargs.get('auto_analyze', True) - suffix = kwargs.get('suffix', '') - fuzzy_match = kwargs.get('fuzzy_match', False) - summary_compare = kwargs.get('summary_compare', False) - md5_compare = kwargs.get('md5_compare', False) - - logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - file_name = add_time_with_xlsx("compare_result" + suffix) - file_path = os.path.join(os.path.realpath(output_path), file_name) - check_file_not_exists(file_path) - highlight_dict = {'red_rows': [], 'yellow_rows': []} - - with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_path"), "r") as stack_json: - result_df = compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, - summary_compare, md5_compare) - - if not md5_compare and not summary_compare: - result_df = _do_multi_process(input_parma, result_df) - find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) - highlight_rows_xlsx(result_df, highlight_dict, file_path) - if auto_analyze: - advisor = Advisor(result_df, output_path) - advisor.analysis() - - -def parse(pkl_file, module_name_prefix): - if not isinstance(module_name_prefix, str): - logger.error("The parameter:module_name_prefix is not a string.") - raise CompareException(CompareException.INVALID_PARAM_ERROR) - with FileOpen(pkl_file, "r") as f: - done = False - title_printed = False - while not done: - pkl_line = f.readline() - if pkl_line == '\n': - continue - if len(pkl_line) == 0: - done = True - break - - msg = json.loads(pkl_line) - info_prefix = msg[0] - if not info_prefix.startswith(module_name_prefix): - continue - - if info_prefix.find("stack_info") != -1: - logger.info("\nTrace back({}):".format(msg[0])) - for item in reversed(msg[1]): - logger.info(" File \"{}\", line {}, in {}".format(item[0], item[1], item[2])) - logger.info(" {}".format(item[3])) - continue - if len(msg) > 5: - summary_info = " [{}][dtype: {}][shape: {}][max: {}][min: {}][mean: {}]" \ - .format(msg[0], msg[3], msg[4], msg[5][0], msg[5][1], msg[5][2]) - if not title_printed: - logger.info("\nStatistic Info:") - title_printed = True - logger.info(summary_info) - - -def op_item_parse(item, op_name, index, item_list=None, top_bool=True): - if item_list is None: - item_list = [] - if item is None or (isinstance(item, dict) and not item): - if not top_bool: - tmp = {'full_op_name': op_name + '.' + str(index), 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, - 'dtype': None, 'shape': None, 'md5': None, 'data_name': '-1'} - else: - tmp = {'full_op_name': op_name + '.0', 'Max': None, 'Min': None, 'Mean': None, 'Norm': None, 'dtype': None, - 'shape': None, 'md5': None, 'data_name': '-1'} - item_list.append(tmp) - return item_list - if index is None: - if isinstance(item, dict): - full_op_name = op_name + '.0' - else: - full_op_name = op_name - else: - full_op_name = op_name + Const.SEP + str(index) - if isinstance(item, dict): - if 'type' not in item: - for kwarg in item: - kwarg_parsed_list = op_item_parse(item[kwarg], op_name + Const.SEP + kwarg, None) - item_list += kwarg_parsed_list - kwarg_parsed_list.clear() - elif 'dtype' in item: - parsed_item = item - parsed_item['full_op_name'] = full_op_name - item_list.append(parsed_item) - elif 'type' in item: - parsed_item = {} - if item['type'] == 'torch.Size': - parsed_item['full_op_name'] = full_op_name - parsed_item['dtype'] = 'torch.Size' - parsed_item['shape'] = str(item['value']) - parsed_item['md5'] = None - parsed_item['Max'] = None - parsed_item['Min'] = None - parsed_item['Mean'] = None - parsed_item['Norm'] = None - parsed_item['data_name'] = '-1' - item_list.append(parsed_item) - elif item['type'] == 'slice': - parsed_item['full_op_name'] = full_op_name - parsed_item['dtype'] = 'slice' - parsed_item['shape'] = str(np.shape(np.array(item['value']))) - parsed_item['md5'] = None - parsed_item['Max'] = None - parsed_item['Min'] = None - parsed_item['Mean'] = None - parsed_item['Norm'] = None - parsed_item['data_name'] = '-1' - item_list.append(parsed_item) - else: - parsed_item['full_op_name'] = full_op_name - parsed_item['dtype'] = str(type(item['value'])) - parsed_item['shape'] = '[]' - parsed_item['md5'] = None - parsed_item['Max'] = item['value'] - parsed_item['Min'] = item['value'] - parsed_item['Mean'] = item['value'] - parsed_item['Norm'] = item['value'] - parsed_item['data_name'] = '-1' - item_list.append(parsed_item) - else: - resolve_api_special_parameters(item, full_op_name, item_list) - else: - for j, item_spec in enumerate(item): - op_item_parse(item_spec, full_op_name, j, item_list=item_list, top_bool=False) - return item_list - - -def resolve_api_special_parameters(data_dict, full_op_name, item_list): - """ - Function Description: - 解析下面格式的数据, 是api参数的一种特殊格式 - { - "last_hidden_state": { - "type": "torch.Tensor", - "dtype": "torch.bfloat16", - ... - }, - "loss": { - "type": "torch.Tensor", - "dtype": "torch.float32", - ... - } - } - Parameter: - data_dict: 字典格式的数据 - full_op_name: 参数的全名字符串 - item_list: 参数信息集合 - """ - for key, value in data_dict.items(): - if isinstance(value, dict): - parsed_item = value - parts = full_op_name.split(".") - parts.insert(-1, key) - full_op_name_new = ".".join(parts) - parsed_item['full_op_name'] = full_op_name_new - item_list.append(parsed_item) - - -def read_op(op_data, op_name): - op_parsed_list = [] - if 'forward' in op_name: - if 'input_args' in op_data: - input_item = op_data['input_args'] - input_parsed_list = op_item_parse(input_item, op_name + '_input', None) - op_parsed_list = input_parsed_list.copy() - input_parsed_list.clear() - if 'input_kwargs' in op_data: - kwargs_item = op_data['input_kwargs'] - if isinstance(kwargs_item, dict) and "type" in kwargs_item or isinstance(kwargs_item, list): - kwarg_parsed_list = op_item_parse(kwargs_item, op_name + '_input', None) - op_parsed_list += kwarg_parsed_list - kwarg_parsed_list.clear() - elif kwargs_item: - for kwarg in kwargs_item: - kwarg_parsed_list = op_item_parse(kwargs_item[kwarg], op_name + '_input.' + kwarg, None) - op_parsed_list += kwarg_parsed_list - kwarg_parsed_list.clear() - if 'output' in op_data: - output_item = op_data['output'] - output_parsed_list = op_item_parse(output_item, op_name + '_output', None) - op_parsed_list += output_parsed_list - output_parsed_list.clear() - if 'backward' in op_name: - if 'input' in op_data: - input_item = op_data['input'] - input_parsed_list = op_item_parse(input_item, op_name + '_input', None) - op_parsed_list = input_parsed_list.copy() - input_parsed_list.clear() - if 'output' in op_data: - output_item = op_data['output'] - output_parsed_list = op_item_parse(output_item, op_name + '_output', None) - op_parsed_list += output_parsed_list - output_parsed_list.clear() - return op_parsed_list - - -def compare_process(file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): - npu_json_handle, bench_json_handle, stack_json_handle = file_handles - npu_json_data = json.load(npu_json_handle) - bench_json_data = json.load(bench_json_handle) - stack_json_data = json.load(stack_json_handle) - - if fuzzy_match: - logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") - - npu_ops_queue = [] - bench_ops_queue = [] - result = [] - - ops_npu_iter = iter(npu_json_data['data']) - ops_bench_iter = iter(bench_json_data['data']) - read_err_npu = True - read_err_bench = True - last_npu_ops_len = 0 - last_bench_ops_len = 0 - - while True: - if not read_err_npu and not read_err_bench: - break - try: - last_npu_ops_len = len(npu_ops_queue) - op_name_npu = next(ops_npu_iter) - read_err_npu = True - - npu_op_data = npu_json_data['data'][op_name_npu] - npu_op_parsed_list = read_op(npu_op_data, op_name_npu) - if op_name_npu in stack_json_data: - npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': stack_json_data[op_name_npu]}) - else: - npu_op_parsed_list.append({'full_op_name': op_name_npu, 'full_info': None}) - - npu_merge_list = merge_tensor(npu_op_parsed_list, summary_compare, md5_compare) - if npu_merge_list: - npu_ops_queue.append(npu_merge_list) - except StopIteration: - read_err_npu = False - try: - last_bench_ops_len = len(bench_ops_queue) - op_name_bench = next(ops_bench_iter) - - bench_op_data = bench_json_data['data'][op_name_bench] - bench_op_parsed_list = read_op(bench_op_data, op_name_bench) - if op_name_bench in stack_json_data: - bench_op_parsed_list.append( - {'full_op_name': op_name_bench, 'full_info': stack_json_data[op_name_bench]}) - else: - bench_op_parsed_list.append({'full_op_name': op_name_bench, 'full_info': None}) - - bench_merge_list = merge_tensor(bench_op_parsed_list, summary_compare, md5_compare) - if bench_merge_list: - bench_ops_queue.append(bench_merge_list) - except StopIteration: - read_err_bench = False - - # merge all boolean expressions - both_empty = not npu_ops_queue and not bench_ops_queue - no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) - if both_empty or no_change: - continue - - n_match_point, b_match_point = match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) - if n_match_point == -1 and b_match_point == -1: - continue - n_match_data = npu_ops_queue[n_match_point] - b_match_data = bench_ops_queue[b_match_point] - un_match_data = npu_ops_queue[0: n_match_point] - for npu_data in un_match_data: - get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - get_accuracy(result, n_match_data, b_match_data, summary_compare, md5_compare) - del npu_ops_queue[0: n_match_point + 1] - del bench_ops_queue[0: b_match_point + 1] - if npu_ops_queue: - for npu_data in npu_ops_queue: - get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - - header = [] - if md5_compare: - header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] - elif summary_compare: - header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] - else: - header = CompareConst.COMPARE_RESULT_HEADER[:] - - all_mode_bool = not (summary_compare or md5_compare) - if stack_mode: - if all_mode_bool: - header.append(CompareConst.STACK) - header.append(CompareConst.DATA_NAME) - else: - header.append(CompareConst.STACK) - else: - if all_mode_bool: - for row in result: - del row[-2] - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] - - result_df = pd.DataFrame(result, columns=header) - return result_df - - -def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): - index_out = 0 - npu_stack_info = n_dict.get("stack_info", None) - bench_name, bench_type, bench_shape = CompareConst.NAN, CompareConst.NAN, CompareConst.NAN - err_msg = CompareConst.NO_BENCH - accuracy_check_res = CompareConst.NAN - for index, n_name in enumerate(n_dict["op_name"]): - if n_name.find("input") != -1: - n_struct = n_dict["input_struct"][index] - else: - n_struct = n_dict["output_struct"][index_out] - index_out += 1 - - result_item = [n_name, bench_name, n_struct[0], bench_type, n_struct[1], bench_shape] - if md5_compare: - result_item.extend([CompareConst.NAN] * 3) - if npu_stack_info and index == 0: - result_item.extend(npu_stack_info) - result.append(result_item) - continue - if summary_compare: - result_item.extend([CompareConst.NAN] * 8) - else: - result_item.extend([CompareConst.NAN] * 5) - summary_data = n_dict.get("summary")[index] - result_item.extend(summary_data) - summary_data = [CompareConst.NAN] * 4 - result_item.extend(summary_data) - result_item.append(accuracy_check_res) - result_item.append(err_msg) - if npu_stack_info and index == 0: - result_item.extend(npu_stack_info) - if not md5_compare and not summary_compare and result_item[1] == CompareConst.NAN: - if index == 0: - result_item.extend(["-1"]) - else: - result_item.extend([CompareConst.NONE, "-1"]) - result.append(result_item) - - -def _compare_parser(parser): - parser.add_argument("-i", "--input_path", dest="input_path", type=str, - help=" The compare input path, a dict json.", required=True) - parser.add_argument("-o", "--output_path", dest="output_path", type=str, - help=" The compare task result out path.", required=True) - parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", - help=" Whether to save stack info.", required=False) - parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", - help=" Whether to give advisor.", required=False) - parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", - help=" Whether to perform a fuzzy match on the api name.", required=False) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py index 9443e5ef0..155609f58 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py @@ -2,8 +2,8 @@ import json from msprobe.core.common.file_check import FileOpen, check_file_type from msprobe.core.common.const import FileCheckConst from msprobe.core.common.utils import CompareException -from msprobe.pytorch.common.log import logger -from msprobe.pytorch.compare.acc_compare import compare +from msprobe.core.common.log import logger +from msprobe.pytorch.compare.pt_compare import pt_compare from msprobe.pytorch.compare.distributed_compare import compare_distributed @@ -12,8 +12,9 @@ def compare_cli(args): input_param = json.load(file) npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) + if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: - compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, + pt_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 11e5193ec..aeea94945 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -57,6 +57,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): def extract_json(dirname, stack_json=False): json_path = '' for fname in os.listdir(dirname): + if fname=="construct.json": continue full_path = os.path.join(dirname, fname) if full_path.endswith('.json'): json_path = full_path diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 8207c7d64..b32e6df60 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -9,7 +9,7 @@ from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory -from msprobe.core.common.const import Const, CompareConst, FileCheckConst +from msprobe.core.common.const import CompareConst, FileCheckConst from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op from msprobe.core.compare.Multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process @@ -232,21 +232,19 @@ class PTComparator (Comparator): advisor.analysis() -def pt_compare(args): - with FileOpen(args.input_path, "r") as file: - input_param = json.load(file) +def pt_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) - check_configuration_param(args.stack_mode, args.auto_analyze, args.fuzzy_match) - create_directory(args.output_path) - check_compare_param(input_param, args.output_path, summary_compare, md5_compare) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + create_directory(output_path) + check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) - ptComparator= PTComparator() - ptComparator.compare_core(input_param, args.output_path, stack_mode=args.stack_mode, - auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) + ptComparator=PTComparator() + ptComparator.compare_core(input_param, output_path, stack_mode=stack_mode, + auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, + md5_compare=md5_compare) def _compare_parser(parser): -- Gitee From fdaacdbc710ba55867b426eec7512c2055016040 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 6 Aug 2024 19:50:20 +0800 Subject: [PATCH 150/791] add distributed async --- .../msprobe/pytorch/hook_module/wrap_distributed.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py index 6cf425441..3ca1db0f5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py @@ -57,7 +57,12 @@ class DistributedOPTemplate(HOOKModule): @torch_device_guard def forward(self, *args, **kwargs): - return distributed_func.get(self.op_name_)(*args, **kwargs) + if kwargs.get("async_op") or self.op_name_ in ["isend", "irecv"]: + handle = distributed_func.get(self.op_name_)(*args, **kwargs) + handle.wait() + return handle + else: + return distributed_func.get(self.op_name_)(*args, **kwargs) def wrap_distributed_op(op_name, hook): -- Gitee From e3e0bba6556479ccc51eb7dac83c72b7ecb9a81a Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 6 Aug 2024 19:50:32 +0800 Subject: [PATCH 151/791] add distributed --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 6418e8922..559dfdc0f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -282,7 +282,7 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict if need_backward: if need_to_backward(grad_index, out): - backward_args = backward_content[api_full_name].get("grad_input") + backward_args = backward_content[api_full_name].get("input") grad = gen_args(backward_args, api_name, real_data_path=real_data_path)[0] bench_grad, _ = generate_cpu_params(grad, {}, False, api_name) bench_grad_out = run_backward(cpu_args, bench_grad, grad_index, out) -- Gitee From 4c23304345096a98c4cb3fc52c310a53a25b4e1f Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Tue, 6 Aug 2024 19:57:33 +0800 Subject: [PATCH 152/791] mindspore free benchmark V1.5 --- .../msprobe/mindspore/free_benchmark/handler/base_handler.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py index 85189a206..375ed057a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py @@ -36,11 +36,10 @@ class BaseHandler(ABC): @staticmethod def get_endless_norm(first_tensor, second_tensor, abs_tol): - try: + if first_tensor.dtype != ms.bfloat16 and second_tensor.dtype != ms.bfloat16: ratio_tensor1 = ops.where(ops.abs(second_tensor) > abs_tol, ops.div(first_tensor, second_tensor), 1) ratio_tensor2 = ops.where(ops.abs(first_tensor) > abs_tol, ops.div(second_tensor, first_tensor), 1) - except Exception as e: - logger.error(str(e)) + else: ratio_tensor1 = ops.where(ops.abs(second_tensor).to(ms.float32) > abs_tol, ops.div(first_tensor.to(ms.float32), second_tensor.to(ms.float32)), 1) ratio_tensor2 = ops.where(ops.abs(first_tensor).to(ms.float32) > abs_tol, -- Gitee From 812979fde516f779c3b68107ecc0296622abdbf9 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Tue, 6 Aug 2024 20:05:01 +0800 Subject: [PATCH 153/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 46d465714..b417fa88d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -2,7 +2,7 @@ import functools import os from pathlib import Path -import service +import torch from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException from msprobe.core.common.file_check import FileChecker, check_path_before_create -- Gitee From 833896e5392e4ab6484d43c744d2521e42daf86a Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Tue, 6 Aug 2024 20:47:25 +0800 Subject: [PATCH 154/791] =?UTF-8?q?grad=20probe=E5=8F=AA=E6=94=AF=E6=8C=81?= =?UTF-8?q?torch2.0=E4=B9=8B=E5=90=8E=E7=9A=84=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/pytorch/grad_probe/grad_monitor.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index edd28635d..5f9ba8a46 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -2,7 +2,8 @@ import os from collections import defaultdict import torch -from torch.optim.optimizer import register_optimizer_step_pre_hook +if int(torch.__version__.split('.')[0]) >= 2: + from torch.optim.optimizer import register_optimizer_step_pre_hook from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target from msprobe.core.grad_probe.constant import GradConst @@ -102,5 +103,5 @@ class GradientMonitor: header_result = GradStatCsv.generate_csv_header(self._level_adp, self._bounds) output_lines.insert(0, header_result) write_csv(output_lines, output_path) - - register_optimizer_step_pre_hook(optimizer_pre_step_hook) + if int(torch.__version__.split('.')[0]) >= 2: + register_optimizer_step_pre_hook(optimizer_pre_step_hook) -- Gitee From 90d5caeb1b6ef202c9e251071b4836475b2ccb3c Mon Sep 17 00:00:00 2001 From: wuyulong11 <2284273586@qq.com> Date: Tue, 6 Aug 2024 21:56:49 +0800 Subject: [PATCH 155/791] =?UTF-8?q?=E3=80=90=E4=BF=AE=E6=94=B9=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E3=80=91=E3=80=90tbplugin=E3=80=91=E8=A1=A5=E5=85=85?= =?UTF-8?q?=E9=81=97=E6=BC=8F=E5=85=AC=E7=BD=91URL=E5=9C=B0=E5=9D=80=20?= =?UTF-8?q?=E3=80=90=E4=BF=AE=E6=94=B9=E4=BA=BA=E3=80=91=20wuyulong?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...7\275\221URL\350\257\264\346\230\216.xlsx" | Bin 16997 -> 17397 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git "a/plugins/tensorboard-plugins/tb_plugin/docs/\345\205\254\347\275\221URL\350\257\264\346\230\216.xlsx" "b/plugins/tensorboard-plugins/tb_plugin/docs/\345\205\254\347\275\221URL\350\257\264\346\230\216.xlsx" index b7a8bf1fd0e7eec640e46af76e16c6a228f335ba..fbe5a354ffba8619d9e93012d6fa3715e1f50e19 100644 GIT binary patch delta 10316 zcmaKSWl)|?vnCKExI>WO?!nyy!7aGEyX%bvcOE3TySqbhcXxMphs}H5ulDSz-P){Kft$fsYHNjH)AW*9M;5{725( zd%ODCvFO#eG{;WQbBX%Z6ZR*t17%Z-cvDDnJyFVprTN2R^?dmF$3Ky~)}~ovFqdXS z42(4T#fSL3P2>LXa9(H<{Yv_lb?{w^5$%h9`~2ZZ=zTH=tjJeK4T}y}{f)lx!ux6> zn?M{)RP>)&Sy5J!{NFM6YxSQ5;SlJ}n7Vd&N3I$ z21h0WHAVP*=b@h1JJYTeE+hJ$9Otf~#UbD>6ds3mt+55NOfyB7k6!tp)X6tO&6d%b zUW$m@j&OshB7T@aHqDfm`UHsq1_t&SELe71P%!)~OZ*cU*cb@>6F#7Ad(DpDcR_i^ zlO>&U|9NWB9xV>*nW|FI-JA%Lii3&_i7av_2AfFwl^9$hh0+iJT%&iKSN>O=hc z$V&j!WZ$b!Fr7E@hvyHg-L`^&w2O9saw>hO`EFq9qm4BBsxeI(%LXCvUTkR0+NX}X z-KIbq*EVt*PSS3=vkZtuI^*(+v3vXa^;*w%iP3Lv_zl5LOM%+uon6P>5M@sCq^ z4mDhNe;C;AB=J6OZyv$Ep(#J(cXM1`20>wCM$Z>bVxyNcLJ1(k>niIqg@qg+Wpc$68v&R~rw|i%s(<0xIsymnI z0)B^E7U0MS>cmM$F~dFHI1(Bz^JF)IDoL@6J0i6lcZST>3=CdJBOT=VnhcY@Y)3E0 zQ+6|?YuAATJb*h>Q}=s51c5MKEIDH+MgBf(`>lzTY<`e24Sbb#c%eOmHSGwFSQNT@ z17Uj_#bg?;|0Etl@XMXgE5@Cp4c|G?SJ5_;vxKM2iD>m6-!*w&^?^{{XsHXn_!JNQ>(|En7^S4B z3ib!x>JlM+u)!@duH-eTRcuI*FfG;nA!iX_61ivp;P6}amlY;mvlh}Ci&Kylu zb!b&o>2db<-lnFfQ58b~Gk`ftlRzdcqQ7y$8aUh+Iz6wr&7zXvMT3G}GwOGic)g6? zt%wi}-O-lbUKCY1JwElcPyEwY=ZUU)9bE3$Al2jmP9`zQmOKL`vV}Mo6B|TSqUjeL z7V2B$wKVTx4kCfp2+S=05KfjZ|zh>Bxys5vQMV5)yd1VoaM3_&ZaVn>pN;Oq4M#S$t!o;GfJ=oX8|$aQIghIi+lIk zJmgH}>>4rpVvYXXg4MSog4AZ3u+%GUUV>~og5hggvW=NE$Hf`6syM(z<~?tD>E?)* zrhKDb--{wu$Tl9}=e|@)njWyI`x$-IVYY94b>X#CX4{`v)We?;>`J3sj(5$FF`C6n z3qLLR5yj3W>BYO8w{bfmLdn+B_19q9IoaaIIx--=H*7?Hn&#{_k_$NLrDe4 z@s_(%tGK(dS*>TGQhAG*O#5bQDzEp96U${t zT;BT@;rn`;PR@cv*?eWrSOmLzRh`Vm9^8=n4XV@iv^%YOa~hBkFliEP!)LH>Xpr(6 zb%9-$PGaz^O_}tU>O3AmND4t!#3h$q;M84V<2Pv)Gu*{ZkRU9dED40dMh<3XN^@Mm zQMKgh8?Y+O8gm$CU|=|hT?}NA|8@VeD(y7uohv$eDr?o98rt+ zmNhtLTq7z3{JzB)VcEw#P-XHINuTO7pFVu?K?e#4!VBMSKeq;))$}Nor=6>5wBuEd<~{vy0?75UN`XC7KClMGFlS zW2uhSwhw}~ot-TFT46uk&Bjlq9UwMRpS|At%K=emVY2*d#T630)=t1Qn^~O}`iP(9 zWlK=OKRgy&xP%cw&IP2*$k|g)^bLu_5RT7rf3@p>wGWnmweb}~xqB&U*;)Y$Y;|_Q zmOz^|{K8|F6Cy7CLYZ>g=Vs_i%h?RwW7{@Qcq|VMr}hdepxhw^w8*@t_T_G}=WXOD<$h zmDLxo9gtLXo#5UbB_)2c-RbUgA>51<;$9x#I&raZ<>Re<2t0GfNN8Cn4t8wi}Z{$ujpOLMvMpEGHWDbZ!y1f?8 zPbZ6`wBX=5iXz@jEy;@&IJ4>nJFh$D-!)j?aIwUC8#)c9>+^}(Cd`8)tJgML60=;h z@LU81B~xBpO~$ydgLmq+pEX1Cl(J(51ue;AQaofI;Lb;8&^9$%&sFiyDtmfGGzTqZ zb=C3LDfe(+q_LFd?Axz!xdYnT2+nir2e=GDB~0bH6I|EhnyGc`?BYGo@~Y|16#{8@Bepo9C z+XS0ixze@Pyt{0^^Jn$>V758?gTHpv?hGelvgLEE(ZojTg!4a5fwf7dGThw#!gC#% zrVDoL24{LB4Y^ZHQ?nnuv^K}*&c3=;fW%2IM} zy1W^2r)gt*z||voIEP#uKB_XnVzo=K|01y=Ai*9}qFpBGK-}C#lUg$5mlH zccSZhF$hk;dI)JMCtGXP6^}uGJ6Covw~xkj)j@l^pCe_yWtJ6y_45=BBG>;f%CEf` z;OD6J*wSbo@DyF145m75P@qFsr3xZXc#5fuFnp3V9x_#$B~9Xhj{ddVyH?C3%!?A~ zOa7kkx;p>?ZN}-&TL2|v8F7l%>(T=96$8WjY3+dXOqjsSv;C5BU*_B38UtEnD`|g* z3F1HtjEd_05@`{&f5v9@_W2Mj!HXA3#;NOsqwE0hQuN!BX|ys0HgJD6xkDor>3Qfm z&wp=B=smGKU~-CnWz^}3YxjM$ww>brj7>k-fmGiB*LCLMbTG7^?=Q;^?@4wvX(25m z?t|KulaDu(b6D?IgagF{%M_+H)4&I5K{_PIop}kLK#QQq5rp&pClD=cF2vCXB8wJtwEtGb@CEBjg_sOT5Cao{`h1UHw=@ICGuE+UA#)D_sQ~U@1Sa&; zmxx9X1(PTQ4n>s947EKAeM=(ew^8CbFB16=(f_+5PEtsE&YS0~^_v*o3j6+?K!f!B zhJN~&9Rku5+`k>HOUGAlLMWBby0dV})w6Es@03xn3R{3^5gz6Lgk=PCjK}5|MZ#> zfc3KL8Rh(&EXo;;?Ee2*q0MRwL6bwPGksrRKmaBV&~UnP!hRMQ-BOk^A+ua`Pcf;D z-`SE=!Jaw=qV3<`b4lKbn`2sEU*sCz3X|a=r(Z7RM$|6jUcV%UsNF5e7g8b=ID==i z$Z&Pam-f!})Mv9Ho?^?ejXo>So~jrg2|MSV0x>uHS{l8Ow7;D&PCDK+Zaf&@L^ql% zlrDS+G_035Dg$;{Tl{nA(;7q-aOl4COFbwz161)F2y0pPF@^9dPO1Azb%RHgSEb_W z%^TYzSj&x)7oZ3}6*0%E+n6S3rJePJ(V6^R@dm;bIlH}e_t2k`r z`W>ZlZdyB~JfQPc~s18l%ZtM$Fo<9ZKs&Hh3d9SSCRQDm|sHCImeLi*11ZXKOuZXV;etx zJe{)Lf-qeGn<7cw5}?LW0_W1_lVE$2m7SF|Cz7l(QcK|ZSpR0$M;QA|NbLd%SGnXV z>%@ziMpIAQGeqI7sF_anMh$n(T!efCSbWbr>@U$Mw!Xw5ByQm<&aN#R3~+Oh-LPp1 z8eh&HvNhk^djg-pl^6OIJ zigiy+K77q82yTRBTqef5caWw`dx+`sYTVZ4mt1^e8y0lWH{M>4Zt|Yyn=rNk&0iVB z9VWcx_U|{rFLDogR684OgAzxmk&(^t$ zuOSAo&72)@v z;xMq(X~D2mP?%D~S|liTBBtpBIoyQt%_*7}Oo3@B4%+9X(i}WV58@6672z2 z;9;8GnX0AM)xkeRbt{E1xj)3TJfkO^)ZidrCgc~IdeQaDzC(ksSP`OP}Pg_l(z;h2bKg@#)ucgK3)mjD+ z5}exGxq+q69jM9sTK*Q8gWR5}T)PqJLQS7P@kHqKGiPy`y8yIlA!mlC=ltm7fqKLShQCK1x3~et^di z)b``dLBmyk?Z-*`3{?a4ub_>Ikc zFxITj{=~!k!u)X$dqAM77?5B`j?Mu`uZ$=og2_vzel%7Hu0aK7?`EI<-1>tLRR&hq zsu9Tr>7tDA+&iB_6=kRCZE7@X_?rkUmeSU9XDW>DcZ>X%%z`OlBx>esW@DC{$`Z4p za%(==i;DJ=ekGeRH)TGNvr`#pYjgek$n&!#y!w|19-Q~u!eudb4S>8ot7}L|(p756r^!U&LL_a=R^;i-U%k*jk zVh=V*W0icOO^#ghZ9Nkw9V@b!^M$zX6+zu{73W=2Ur(rWq-sIr{r=O@W_U+Jok?a7 z?q&z}6cBzQ1Xny#Cr}Wfux({w|4CAIsw=1_7lLgEA=rOc4-qbQA3BN%chN$BoQ>M;N!bi*kQ#3-Qb}h}kP^%<@+Es7{_5Q!anxLm@^?#ZT)oeJg=UUsIx5}QoPL2-;Hp#V*Guv5L!obwbLAi-c z_J((Lfk#va4r=l!p_jCT&?3n1>GGeR+~%@6h!~n7=;$1>@gZ^NaN54uSxw05dN5Kq zys*mj~@{)*I1dEB`LIkL-=#{b($%9pt<$c%e6EMVl zth}sP(>aKA@s9u$iU1?7&&(-qqGrl}59lNQJiaTM->6yHSo20^&CtBEhIv|k$we@Q zL5rrvJo%vW$Xp8WN`x2G-Y_DF#q7->(XV!0dl_W5d{4GJE0LGu;qyHMTZDvlNJz(w z=M6qhn83M>SRGrj+2;V~tmzbKB)hop?P$~|Y`feNq18ucAPT=TGyR3q0!6E)IR^a* z=0EL%U<(hFveh0PF6WIiVnaeBO{UK%sljIUCPLFFxe5b(Nf+dp~B6odDU=c(E7~}9QmIwo8oIU zz|Lg-8To*(X@IauidCM`3elQ&w4YR#aSf$JRk;d#fc=6&YL7K zaT)IK4(eO6k>QZ;L4+7d_A-nwe4<4=NgXH9=K70;1?MDxJo`pxHN}Ej)8WNA+p3aebHtvmE4z>dvcURtdl@p0Lk2tqK2_jX&$b;{yF-7e3 ztP%H7qVDDS-L-eJ6M89btel!$h_^X>et9urY=OvAG}(luru(v*A-GkC>QY=-x`bt{ zRtX`Hue@2SRzMwmLz!9NVKS?PUCO8C?DnAJc;eIKjcmqz`yptN0<4sGHcdU-mp%cN zU#Z_r7cG9u5#tYb2U2N1$1A!!7zYTZr@*@}^)qn#pPA%txGaP4kSp2F`K(6O+utY?}cV)&c~H1`E2KO|zwnWy-T_~oL-ip zX!VuJ!{HY)+6^3ARK&@BZ(Y+C{E z=yNjj$02X{^)3X;&X0T`%YKQU$x-GR=_ zL}q}>Yj#!ELAtQXSbgrvuOnv=SDmFTO&fMPR7Da`etGf(vll@r5?{5Zshr5H~njyLg-z_ zpu4OpaRoItrL5pk8cAqV=h6`+X?UF)p(=h{<-eoXxfkM^6`BNmbt|P?KhnL4O9{4D zp0HLU3h9C2EFNPpLa)>)J$$EeQ4-BkmFBA~%(Z0iQS^pqmA)3sahw8F)F|YOi*26X zlb23UtRcCR+!YSDgrOe0Lba&Z$5t;)#wVolZtC5ezC_$B){GfYWR7oIv8S_)@%Fp1 zYKt}-Ajx5{<&MbqCeDutN9+{1>ayRcrJxWjysM~!z+>HxHVMxeX{_VhvQyAL*!eAfqph3WF~P2w$O3B%sqk#0 zBa{s7!E+FsJb)8i?n4O+>ezZW<6S-Ee9PWaw;pL|GaQyueoil6K!%R8th1KC8!U%P z+p-IC-Z*X=A3SnT{X=tb>!C7`HpQen-n?%e?)LW}u|n>DQ`x|y{%Xp5d0axEqqvy9 z&}#Rsl4kUniUE@qn(id3x zEyZ;HrD4vcT$D!l58lumW-50UX(mgl(qF(Kn>Wh>?3-e!2swbEaR$@@FMKGPKhgVg=X@e;8pT9Nmxx1Dk>QpJ4ABK__k-{sfy~h^v%KurC zcwO(GA;$ytc&%z=BB5>jJNC8y7tWBb)NgZa`nP#49moHc&{8k(0s`hg=kZO*0D66~ z>jgBE<#xGFwjP>k4NI1|&O|2CCrj9yuokeY&4&5^ny|zHLrgQ^^a$B2Q7eIS>V3NQ zWZ^NV)8h@+_q<4`Bkpd!S$Q^rhlw9YpX;u5;# zCXa*=DxssYqo+swybY^RzE#W~^~#uS`-7zZ`AJc58igdxh*%U7RwRTfQJ*xkU)F*& z=Z6U?$qzHqoTj2kZBqn1Y+>GU>G_c|tsg9w2f}Ef`S5^w+;U5n#PHsvvM~1$a`CD1 za(-R@=QHo>vF@Fl*~=MvvWqD56%L7%Hyy^&O>T)TalX3hPYtUv-+>y}U-;dlg}~r6 z8xMFVtSI+dNOg$hw;++1Ca6_vTNh}ENA-YtY#*PV1|(>2bo90(?u@0YLug{e_~7rJ z_n((IcGZEZ5M5OsLH!zQ1o8Hw5+}KV1bNKkhi!SIo?@q@sRX+cSq=tmSY-vJUj&}g zN=8>7<4qgvYglV*{i(?6)0?K)Hd<3EQMbgC@r6Mccg=2w;<&j+&_iLAEP@O&qTdKG z;*{ZzKAZm*r|BkEMU~wzc55cm9WJga-YeN6l3NBo=KO6keyS!9>q#2uY0i~LGY60b4o=q zB4o{ln?}|dG`*O@pHJ}fHH3+!vGoI+x?zOI!@`Ot{dU~U4k$rl&mv4{IzWiJU_PQ& zk^(uPU>|TFr$DZ7M#MK1VG%kEb;PR!PM?iq{8I|wUSAqij1^Un;qf_n8f{0hXgqmN zk!s=ZSe&;t0md?UZ92d{d2KnMjx4?L^gwDFXob))X}sSIF>Sopqp#KUl&A22+ffBi z+FXceAi%#mohK+WcCke6|EHO*`MtIb1d!;A+MIR$YgXBQzk6B3t!X5Fa#FR7q7FcZ zK!XT(<=IV<7NR|_16@QTO|o*b;fj`iVrbe04;=Mkq>|^ZVE-cb>cv@v@}N&$?Z zs5Hy8?uA>0NQu`Sv1+le!}@KJ7!MsGt;Nql@P|-*0Z(m*Bgm}aLc>CMUcG00`u>W8 z;UH({>)WHHoksU4g=NE$pvW5i{$%1N;GwVf1*H3S{o1|(Wt10W&;0D-5r29r5KSou z5TL}|kHa2H$t(CL0UiL7Z~7Z(t@0JH0Um!HtM;@B0GvQk6H)g283I1|Ol)F>7-56!BlE|$kRX2y!~<8KtL-DuUuFm|U5RbKA?(cbw;KjMuj}2d zr{Igsw`X)DO>QG`b_qOtp_i1J>``M<1E?T?8IA9@&r0<5kD5q#LEG=hkm4K+B{auw z7j+ar<#-6%d0JvT?wp-*W5Tj91D<)>G?_dt3481l%yH-_`5>C%Eza-+rYm1ONN7c` zeviOf)Nh*c2JC-vk-w+H6nFni6k944OWP&+TRoh_WDDesHu^6R9*>V4urg@xUJ4lu zEFVK-uhxNG^2M_&ayEB~cY`wMK1O_UAcRO|=(8$Q<-|j+ZNaFm*4eTW-97%782m#o zH*YY(4T@;r6G}}VTJb-`nsNWGAY?TTqxoNpL@I};UI?eNR&lQh{$xepn?OP`*p4Su zpiS;CaE#^m5O$Q|Ku3XFY8j8U*Lb}nPcR}_`1j~$FyVf+q+E$=VUG?dz z@yJqfQtYj8z0QV*z*yn01aFs(_0UIXCQ#@VkJQZ)!6%{Ga&9j3oQh!$9LPN;%9RZ9}oS&@4IetO+%rISpUe%BgBM>L6t%I#*3zSh| zTUPlh<5OOdgI>Tt#Sddq`%#SAtnipUlm7APZz_ion<+Tdpw*JYDQJ?N5|FXzVL}Of z2wWE$?MO^dRFvGck|bW4=ktx}PYk7p9lIT9th6JxC(6UKg|vlke;E6*vNpjP!YDlp zQ<+60iE=1Y)qa8D#aihqM-)=u=<%5VQ~F0$sjY2;EvMOd(%@aJsLNH}pKk-E)I*&3 zY?#D36Y1=E3XX}zAcfR`b4z)f;cJD0e3)6&3WfB2{T3%DWzu?kY)|L?1WG8h=*zxw|GT4+Hof?N;?xS(o5HV776(4n9JL?<9^9Q^!6CR@+}+(}c;EZypP8DO zuCCsF*4}%y)T!!Ieb!Al^iBwL^>=8PQbLgia4Iw-s32KEbm`l077>CbQAg-D5Si0P zI&bB*T?aB7xoO6|7jU->cBz3lTq2H@jj2ai!cZBD(96AndN zpUms%pbxdkb3NbYMP{Zwm!bSL9#S#E3^VfO))YSNiIG05qspuj)1PzMrOMOl^OuUWcMH5%x^y~OHkrpRg95l{V_WpKr zIcH3oyVbIx`8)qPJh7P6?9H3Q3vX@?jotZVD0R{&n-=sk?H$Q)zxihLZ7pP<^=J$6 z^H?_m;S*%^`xPh0hV7i+CKy#R!M~-47)SR|g+_G@zlg5N`gz1&-N4Lj-M&R~(dE(= zqDD4<`h11F`AHeQ1r*@*0;??d0R|fi3JL)#TyazQsN;Z6>H`#16&U&h5oif`D2X3> z#&AQNz@V`r+Ur^!n;ByNfDj^v(oeZT{q>f6{=zE;J+S2Q)IBXJO1cfNqWO~zd%k1E zLkj!R?bA2v3gV&Yh(ZB0L+DBg)ofUm$HlGX(AmTmO%)%Tbt*3ourtGu?(jNX_a(;9^h;+RC6hNyVDFQL z%GbpqLFZ-oGas#2E%UBtPtQIV6|2~lwx!MucZLEB?Ca(=R2421j0j?LdLMW|u{(CT z8=o*7l#O%~_Uv=8oyA!S21Hh1k^co2nzmzNlZudl#emWjn(NrW89N_GJf zK;9ZHzsGmcr%mONT)6Y#Az^xUSd_LC1Bm&;)sVD}(@+tu%2^ZtK{RHa;n}VX0c&!6M_oX@>ZukR&pSZ8)T(E+q&RVR~qQSj!M5{zMm0_!T8# zu^8K^W=l9@C|sybuX!CXwEWJyF2UE7nYz%?-DPa~4ZJJR3yHk*DT1fc!L6nozNcUV z+OCIplJT-({;CAk7au80xF3qG(|9AL)U8U`)6wXO`OA6l0tRcX_mRGetKrCxs#nL6 z17H22OL5^2k;U;?VbTFF_%nmv*1ELC&#Gm4l-fzpf5`w8hr({N>Zds0$pAvNJ)NEl zQq!w1hzSIBesH+3NLX`~W~Zo7P&R_#3}jM}-kR%DGe*Ff{2P4i;(;1dT3!Hm<%uFA zRlD1%3s>aXqnDUkbuTj2q4aRnS=Ctplmv5iU~&gLdd)NR)7Fv!qwsL^!yd$e)k{+& zGo@|Gguj&tZHAda>jCJ`_x!N&(l_g*%yj@v#@$g|K{HYpc-~vdesgnWEMvKmEU^R$ z6L{R5I8C!Z9nqH7#V!`z6fc(%$MPr7aysloY>8)|WD6OX>Y4H`c6JySiyna1?D|@3 zdHH@m!)y(Vbv>FV1o;K35{n6MH`Hqo8V4h-wE{TepCc<)3mmbB9$^4V*jxPh(x<|?U$ z%9|iq+AOJ39iqdcW#(yW;?9Zdh>9GN>vmEtEcLX~UnN5KS2`-r@l^kJNgK3dI&9RR&Z?4brrH zH2qq*_-LTlZ}pYHUGrfN41ea%^rDWcc}ORKJ(|b(3a&~UJvkpe5AmmuUn}uzw;60V zFG#ku5Fyxq>*kb-MTcLyE_?O1FO<^KLVV&PGUwH=a3PLU^jnUa9@l)f{o0y_m8=z# zq!|&A6;dgAaqpfh!yqoCmkS!2wGQaVWYQXqd-pIq_y|gU#`;X`k3F=apg{s(A4sds z`+aamu(~AoRbK%-Gb7(1W3~t&5*6zh*i(|1}tx5s8X^#%|JiodPlY}t&UoI z2lP{NMm-Kn*~$=^TCUT9e_bN+$qTnyARJ>5_t^NB%1QhdNsB271`k`4p}wsS6vn-0 z_x{;d#S2ylowaL*OLTNJJU~)m6f7pnS6Zp$_xA>IktqAog;QpG6FU2O=rj917OAU$ z5a$3*_*7Cfyx$)g5zsAyXt3Zr1C6z-pGy&Ai&q!TBEBJ=2LbA1{{T@BGJ&hVtM3QA z_z$DxMLj=gOap^~I>D7jy17{FO=8+eo5J&^hb>jTs68g=59{+b0OSbX8p6Bwu9?n4 zL45veD;*%Kcjv9P_(Bc+><5+v~LL}LAs2d|D za;%5x0I*MoAV@_l`?k4X1{rdn}>w%T;W|iLi>padwe)>OWGalf@@Eu7UA(zm5Kj>ZF*;G}HxE1TyhQ zpb~V^$fUm&JjG${rthkG@pYFVs8Yo@prZ5Qlu3U(IC7c{mo+vin%_M#-z1X93X8AE zU>ygKb@{*vWmzxyk3`J7FEn!6>Bo;SW$c0dov+=2J(U+C2Wbn$jF4qPs}l6q-8JI= zsuQ$1Xs7Mk4O{{M5X zy6MgSz`yUy&KG=HbXj-MZ??XLUd1oSw?s2)!fHC&@NQ=~&dJ|DrViVBRe1*pO$szu&y<2h@G-IQrd89KMT2E z0c}#@F0-ytf9?^VGt4QBH_Y=i)YP-fKb2Ou+zCKxfj9cyEA<*9K1N!*_o+^((}uwg z?T%$X+*f&iDM~P5PV;z!3`QSYZ7W<+>pJM|kM>=&eqYT|SETP+>zo~xHjHtE_$UVL z?$nGuCD#LaN#dX7)rlb=>rcE3RH$A0%~fRZmf0i<57w+!mCUt*3UiG(G)I-5+tHjL z4wjYfOy+voZ46^mU(m~&BRe6`0e!8kLdr ziQUK9%zD%Fz=*YBqN;IaNkN9!+RW+YoCqNn*Ow#)TXTfQo@m!xWmztlT8 z2TjC!y~m7(7@+RITgKggFL=Ab{=xA3Pso<59R%k*pd!*Cfdxm>*>;L=$4B8Voo0_1 zb5>Lt{`{Pe`*!@sR0;n4yRgGa9O4Yg1nAm`=AHgwmy(PLFSp$T448ufPxYHPh~^+7 zdpy_M3FAq}Y-B&pd6_j_#47)cJpYoJW}(4HuVcE@+xcVn_bkc`e-EFAmJo>G*bhVI zjfQy83xomF@JWX3i2z;vAKHnXXl-NP%5zt&gzn%t)mEa_c|QjQq?nUYG5%Gy}(MZxX!BM>f4$BdZ*ud zr~mq$;5j_0G$`1t#hjFKi!OP``?lBs0V(@o3+K534(yDB1HvPlcb%|MVmRi1YFt=6 zhDFE&z^Fh66+?-r2a?eXf5ah+aG4Ol8wUOt%|DY|+Vs9XS+&{wZ}7jl{}Y=P%}r*u=pzFV;VC zDkE8F0(%QrV3mRY?@(w@4^2czUu2HGFez{UpB49=7(B7CJPiwj4o1hu0sXTF1h>1I z!4GYZt>tOs3X8O^TbqJ1tNL3*_OeQq_iPK7(TIjWx)jcUp=7S`*W-8n6Y(xbTDkNvoZg;S1@Ha*L0rMy5qv+U0 zZVKdf-JTj6nz+`$iurC}Fy>l|%2Qjmro0ix!Wd%UX@xzlr$!>^tmAyVh}%gbw{=Xb zspqgKD(+|5?^QfHi`w?6P(6_Y!BI|GqfYw$&6!F~_v@GNzP9-6HG>%Er82sayrH`{ z@JIi(>6Y16(Oo{R{l8W%s?!bCFLG*{3zB@b^mV*>au(lhhyy2!7H~l&B~i%kB3h%& z#W3m5rbfx1s5Sv0)>hhKY|p7`T>a^5)ruws8fFthXKiph_4u_!g*{kPfmo!lRI+nc zu+S@h86$68^nlm#`_$o7+GcCiR8eG_EM4o723Kj+ZJ$Zz*RPRgv=PtCGuG~ef0 zzw0z=v`iab354n7#gq%{biuOBxL>dAahG39;}3ik8V>-VY~3=0>x>MtR_UVL=<>mQ zb2k|y(Z?|6zZ3gS^)9j>WS)gS-R`7%%rH~CHGX8xb+bo{=Ov|HSE{e~KF%M=^W)zM z+X*oZCG1W(rq6v$&`&JhXkGaBu_#+@B68WlN1h z`hR_7dEyHL-E*G*?(d5yB^C53~a~JQ-5Z z0ABUrMh~0)&D>xU8)@L$I+BP>D%PZH=o=)YF1<1Y{j7p_M^hUt@toVq9d=W?^afGT z>^^#(ANDupQ;02zkqi#m^ycFl*<8o?jifRoSdBXFwpOoFV>hlwCyPFnUQ2Eu<8<8b zEpt7#ub`wzw8+U z%+4p41d=*>f>}63g^lW{2P7ycAMhjt6@X{}e9s^Z8gkz}=#BjKnInMjY+PWkV_^vr z?-S>;bQ#vTR8{hzprBD6uyiv25}fw9^0F%_p%K=xaF!ak)MR^D!=YYv=6A!eB=$C6 z10>P?Q3VzKT3tl)_-f3MLfD92k;Sv@2Cqqa>Sj=7_2_g+ydCyo{41ZB=!~mNYf-X` z!1QSuD5giVDlrNqk`|aEjyg!6Z53Sf|TSIKjiMyxO zs!70e-{VEWIN7z$pv_^_I+bOI|EN*nx0;4$<5Rw2-5z>!Bj4c@$7}>M-SOBFadWB` zj-^}ANgzSfcSl-ChxbXEhn#mT(VOvyZ2y-EP~uT%V|Tzl)>>;}af>qcZkf`Rh4|ak zB?e!YsU^^)gpF$fyuXsdR6feg4ig6t96$X{iv{P1Df7IfvRX1jeQvTzY` z^K5JbX2a`L34b3-&OF&r`o2{3VU$vdV#b3QqUI;eC;rp$AYC9@fK~8iG};P%UoE@{ z#1$g>2LTzbMKWFb+5Bl=ZOr4*RAPtk2j`|w*?;(JzVTH>SX>(B{q+Oh8PNf7% z1Dfr5Dn7M@u#8XU*Hm)p!Na0;6O1Vlu6T{DT8M;SnYs`$w58E$>Hz6n!)QK?B$IGW`x(q`{N&dMjN-pgQ@l(YZ>^KzBpYmRjq|A$vXW8@dp~Zk@lyC zc*ZF+`#pBC#b5DO$POWdRhoH5;KHLW<|0jP04>IgxgwS|oqA3^%ReK-xl%0}jgyCh z>G3{C7Zmmnn{yCoEQ0jE8Teqmf;6{!2qi9?2y|kn4C`v%ZD7phq4~*dna$RhLAiR= zLWFMVn9U_~UCAXzt+H9!4)qfCQc3bop9hHqBPR7yD&m=msj$QH7ub(R%(x7Fei~et zHopReU!(pqmL8$9C3XJz^7+9Ik?ka}a4k*F4(-1YSDu|TJP>8-#N)VHYp$Z-p?(!VLe;y<+Sw30v=7f^Fjok27^OnId#F=9x z87wY%lvkpgX^snWQ+9Unk$u*N+?nvWTD6&_!vojc*1zDXhIEEtYPy?1Hx)gGS3?mp zTf%w6!uYt5}BMr8eder28Hl}-9wcSL*gqT zhKML3MxTVkar5W-=8qfBmbHeKJECbie0WLHj0Vym@XQ>nf+}-(CFYb&fV4%`6muby zg!Ngp11?GSFUbTTxpx!OK#R=O@m;G59G5hb&|zLIarI6Z<0H8J^Y%we!CPS2Nc#7z3bRMc8wp_cbs1l#`#a2>7=eJjNeo=Dsds$Q9N>;|(Ej;Ydr;^-OYqMEo-cFFLf37SBf6jVu8vX)LM1CUN#v8%v2N|&~ z`Z)F0>G@XwSf~)dPkC>oM?~LKK~Y}hZ4K~t|4Hk(T?F@x9NH%mS5)=Pyv#hZ3A1NB zlZDsvxHbpOB(v0iwq^goL0p=UrXW!R>*Ng@7L}{v zzFZ(@U3TSu=&EpCh_?w_cf5dWwIgx2B5dQdnI$RoC0-gyP?@NSc_K(!h0_+9Atko9 zZ`gvRo{$}LQ^1PVm%QrW-XIJDdmZ#DH;=aS+789CZ}E3e!l)o;2xSUyk&zOJ6P1)H zYWSzTWb2pGI7#Q|Rq_&5i^WEx0@E&+g5NQW$_#^6*k65z>gWKJ&?2OC+@7MF|9bd<6g=z39b5nwU ziCI*@Qf{EGX~RVt{G@Jrt27{id800mjP|CThBWC41FH zqUa)dRN7`FjByIOqHbh+!{X3dTo(qaifV|DaK7PT?2P85;I(eUcxfVAeMqL={p?;MOVfm;wok<;ZPOmfo$nb*r}tz_L)ZCTW-2@eh_Rc0vz)GJ-5tER z%TMP>CIzK;xWGEFdmSn~H6E$qz9XF?hBm>ccF9CNy>zFr=uJx;g#w6wNC zX1M5e+iOj*%Qqz(wAWT>31S~lfpw(^x@uciP~4X)Qx*BYFKkI)XG&Y>pSbDXuUcX& zazA9l-vG=$>^NT9ao}Wrxrw~xQBHqRiXBAFk1slXCjCGE^hrAsG}^;MLH)u5Ba#z> zwDnxEq;Lb)N?*{KFH#R}MmV_(VpQYI=+&W}iJo+3(`;f1#^uL7?zC~0!uk?mtQg7> zC4yRJPjQc_I7=vD{K!F)pNEtlkbPUF%%T&`w}rh1qxnw{Tn}6y#vp7LuSsX9OseFc z&=q$s=bV5MaGajn>siC2~4Bok^a1j#LT$e`Ehkngq#<{!N2wN&W+ z(9LYLij@lb#0a|5zv2n1oL4#3pukeal`EI}p~s`3XLkvjhyVw7RdTvc@lzuKwDRKNoim za#Nd-cw=`Hi3V)bg>`AA%SK+~Zae3}Yihp8J1Azet~R&FL*PZ^9qfEHs|SfB8PlKU zMKC2X%4+ zz+PQA+lEq1h|vL}B7sCS;>saNyRRw&fJusiKP;NuMhW3QgL7>X9nqd0aayml z&$qHuEuE+3b4?_Db}83odrE{yZzX51fQt0lq}?0?D`vIiVsq9w8R?WS=r~sk5@aSa z2saef#V>u+i3MW|L$0LevmTn$lnAq)z=5)+O-GWu${emBR5y%G9TW}$O#d&UP3&&f z?Oorre+U?I@IV7WGzv=V;DLeoc+~9P3Fa&`A=+u4S^)s+D4<9g2k@QU2f<)G2nh#8 zx>zA`tX#ZnGr<@L>ygBqO7NVzGZr=m$2Iyhj1Mm^KQ$8yMp=xirk^OHE3^{VQQ)EF zV#U^PPvU4}>pmL3VrAtqx~(D`*r#WQXf;Yeuxx~E%{~RngeI@`R<^9v_}r1Ak?OO` z%ob}4A4D8Kcnz5>@TZ2sgYroPWRZ681zyWL9D-%^ZwCPFD<;TZCp3}`zOcL9L3M8pTy4?xOF0L4y`MDm z)|zhTUT&ax$3QHB0d$+;xFG4~Z@@9dv_Lr$mhM>;W4+(?XdKv>vxZZu5+!+V`$QPxZM>Ob)_w90}Ux<#Q7ngu0usTlkP3aJIQk5aLIhMb zfB*!g2*scoG*w@Jw{UKP!gFNg|3o?ppK`clms$Q2sEl?RFg{0x_f7urmahPm92igi zg_T&;pG4cTX%x1hv`fw#$BS`RfIj!}+Hv(cF4ahm$$evHb;Wn|=INqeAZNho1pe#v z^Pc8Q-_0D$(dZk{_QdhEV2CW#{9`&USQB)>D(1d$=nH*kD|l^A*p&0S=ow@9%f=DN z|9i3duHK(+sHnPXb?BibUNGzBI@#x_?WkfWdG)%q>arw$e0((P!8ic;5xIhtlv9f< z4rp%a0(^X4hNAIm#0A}+i7BnS09#1ncI^oQzN4CS=1p#dtE`LQa0nVjclu!(8Gs_W zvaI@ye#se@W=cqwsbh~M?%nhrV?P{6tqVr`AuGaa4WiX6tl=A8C?cU+THj9?<-zsV zF}4C>BxBGHkis-!HbGS#jDyv%mJ6XNv$Oh6kpJua}CvhS~p$9Z=-$LR)@XHy@YyCtu)_fzB zKIy*UE#trGkJqmHm=Ue*`V{p__PfF3-k(%ZeCbg1QfF~H%~li{PgJ1OC;m-`Dam@J z?LooLweCh*DF}w+iAR@NN3gG-K-M=ux`bd9(X`OFHZyWV%y;AM-&47+kM03F5eB#J zhSLffcIP|~{{(EpO#>RY?fHVze-_*xJn8Q|70W7_Y1p3wsqlp|V*pv3!prHUH)Rjz z>g2ek=%ZwZbe_?SEUmPh&7f1;ntf3==b=Sme&n2i>p(k0XUU0hIB$%GMvalKW;i=* zB%Rm!AvaV1fzBDxJjaB@uz?z4>_`{=0q#JZ_2!2}kuLfiv4efA4|s2!_vI$gWuxSF zRe14Ux1$kYme zf+Bdg{}gB_viG)~6Wk_zLiE4Bc%Y#EQ9S)q{tc*M0QZVe6aDvvp9U1vyF}wZ<==o< zGBBJd51{xHSVWWqF#ZXwM1TX16{QC#v4LAeJpr3s;4U6qu(TLG0Gs>0O9lQZ_VIt$ Pden*W!(H?L!}vb{h$=q7 -- Gitee From 8393d9ba7d39c0af725ed05cd2d7873e7d40aa8e Mon Sep 17 00:00:00 2001 From: l30036321 Date: Tue, 6 Aug 2024 22:03:30 +0800 Subject: [PATCH 156/791] L0 dump --- .../mindspore/debugger/precision_debugger.py | 6 +-- .../dump/hook_cell/cell_processor.py | 37 +++++++++++++++++++ .../msprobe/mindspore/service.py | 21 +++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/cell_processor.py diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 5475dc358..86c1c50da 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -41,16 +41,16 @@ class PrecisionDebugger: return MsConst.PYNATIVE_MODE @classmethod - def start(cls): + def start(cls, model): instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") instance.config.execution_mode = instance._get_execution_mode() - if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.level == MsConst.API: + if instance.config.execution_mode == MsConst.PYNATIVE_MODE: if not instance.service: instance.service = Service(instance.config) - instance.service.start() + instance.service.start(model) else: if not instance.first_start: handler = TaskHandlerFactory.create(instance.config) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/cell_processor.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/cell_processor.py new file mode 100644 index 000000000..9c7eaafd3 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/cell_processor.py @@ -0,0 +1,37 @@ + +from msprobe.core.data_dump.scope import ModuleRangeScope + + +class CellProcessor: + module_count = {} + + def __init__(self, scope): + if isinstance(scope, ModuleRangeScope): + self.scope = scope + else: + self.scope = None + + @staticmethod + def module_count_func(module_name): + if module_name not in ModuleProcessor.module_count: + CellProcessor.module_count[module_name] = 0 + else: + CellProcessor.module_count[module_name] += 1 + return CellProcessor.module_count[module_name] + + def node_hook(self, name_prefix, start_or_stop, **kwargs): + def pre_hook(module, input, output=None): + try: + index = self.module_count_func(name_prefix) + except IndexError as e: + index = None + pass + module.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index) + if self.scope: + self.scope.begin_module(full_name) + + def end_hook(module, input, output=None): + if self.scope: + self.scope.end_module(module.mindstudio_reserved_name) + + return pre_hook if Const.START in start_or_stop else end_hook diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 50776aaf1..b7699b336 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -29,6 +29,7 @@ from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.mindspore.dump.hook_cell.api_registry import api_register from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell +from msprobe.mindspore.dump.hook_cell.cell_processor import CellProcessor class Service: @@ -37,6 +38,7 @@ class Service: self.config = copy.deepcopy(config) self.config.level = self.config.level_ori self.data_collector = build_data_collector(self.config) + self.cell_processor = CellProcessor(self.data_collector.scope) self.switch = False self.current_iter = 0 self.first_start = True @@ -150,3 +152,22 @@ class Service: if self.config.level == "L1": api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) api_register.api_set_hook_func() + + if self.config.level == "L2": + for name, cell in self.model.cells_and_names(): + if cell == self.model: + continue + prefix = BaseScope.Module_Type_Module + Const.SEP + name + Const.SEP + \ + module.__class__.__name__ + Const.SEP + forward_hook, backward_hook = self.build_hook(BaseScope.Module_Type_Module, prefix) + cell.register_forward_hook(forward_hook) + cell.register_full_backward_hook(backward_hook) + + cell.register_forward_pre_hook( + self.module_processor.node_hook(prefix + Const.FORWARD, Const.START)) + cell.register_forward_hook( + self.module_processor.node_hook(prefix + Const.FORWARD, Const.STOP)) + cell.register_full_backward_pre_hook( + self.module_processor.node_hook(prefix + Const.BACKWARD, Const.START)) + cell.register_full_backward_hook( + self.module_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) -- Gitee From 66feb734d34737b9a3445214e62c007078d94ee8 Mon Sep 17 00:00:00 2001 From: zyy Date: Wed, 7 Aug 2024 09:59:19 +0800 Subject: [PATCH 157/791] 87 --- .../compare_backend/profiling_parser/npu_profiling_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 1ae5b1fe6..58f44f7f8 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -131,7 +131,7 @@ class NPUProfilingParser(BaseProfilingParser): print("[ERROR] Failed to read communication.json.") return if not communication_json: - print("[WARNING] The JSON file is empty.") + print("[WARNING] The communication.json file is empty.") return for _, group_dict in communication_json.items(): step_dict = group_dict.get("collective", {}) -- Gitee From a9df1860bc1552aea00940c53e836501665e927e Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Wed, 7 Aug 2024 10:24:39 +0800 Subject: [PATCH 158/791] mindspore free benchmark V1.6 --- .../msprobe/core/common/const.py | 87 ------------------- .../msprobe/mindspore/common/const.py | 85 ++++++++++++++++++ .../mindspore/debugger/debugger_config.py | 16 ++-- .../mindspore/debugger/precision_debugger.py | 3 +- .../free_benchmark/api_pynative_self_check.py | 15 ++-- .../mindspore/free_benchmark/common/config.py | 10 +-- .../mindspore/free_benchmark/common/utils.py | 8 +- .../free_benchmark/decorator/dec_forward.py | 8 +- .../free_benchmark/handler/base_handler.py | 12 +-- .../free_benchmark/handler/handler_factory.py | 6 +- .../free_benchmark/perturbation/add_noise.py | 4 +- .../free_benchmark/perturbation/bit_noise.py | 14 +-- .../perturbation/improve_precision.py | 4 +- .../perturbation/perturbation_factory.py | 10 +-- .../free_benchmark/self_check_tool_factory.py | 26 +++--- .../msprobe/mindspore/ms_config.py | 13 +-- .../msprobe/mindspore/task_handler_factory.py | 3 +- .../mindspore_ut/test_task_handler_factory.py | 4 +- 18 files changed, 166 insertions(+), 162 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/common/const.py diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index b4baf4733..3fbd0ab37 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -2,10 +2,6 @@ import os import stat import numpy as np -try: - import mindspore as ms -except ImportError: - pass class Const: @@ -258,86 +254,3 @@ class OverflowConst: OVERFLOW_DEBUG_MODE_ENABLE = "OVERFLOW_DEBUG_MODE_ENABLE" OVERFLOW_ORIGINAL_MODE = 0 OVERFLOW_DEBUG_MODE = 1 - - -class MsConst: - CELL = "cell" - API = "api" - KERNEL = "kernel" - TOOL_LEVEL_DICT = { - "L0": CELL, - "L1": API, - "L2": KERNEL - } - PYNATIVE_MODE = "pynative" - GRAPH_GE_MODE = "graph_ge" - GRAPH_KBYK_MODE = "graph_kbyk" - - -class MsFreeBenchmarkConst: - DEFAULT_DEVICE = "npu" - DEFAULT_STAGE = "forward" - DEFAULT_DUMP_LEVEL = "L1" - DEFAULT_PERT_TYPE = "improve_precision" - DEFAULT_HANDLER_TYPE = "check" - FIX_HANDLER_MODE = "fix" - ADD_NOISE = "add_noise" - BIT_NOISE = "bit_noise" - NO_CHANGE = "no_change" - IMPROVE_PRECISION = "improve_precision" - CHECK = "check" - FIX = "fix" - DEVICE_LIST = ["npu"] - STAGE_LIST = ["forward"] - DUMP_LEVEL_LIST = ["L1"] - PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE] - HANDLER_TYPE_LIST = [CHECK, FIX] - COMMUNICATION_API_LIST = [ - "mindspore.communication.comm_func.all_gather_into_tensor", - "mindspore.communication.comm_func.gather_into_tensor", - "mindspore.communication.comm_func.all_reduce", - "mindspore.communication.comm_func.reduce", - "mindspore.communication.comm_func.reduce_scatter_tensor" - ] - NO_CHANGE_ERROR_THRESHOLD = 1.0 - SYMBOL_FLIPPING_RATIO = 8.0 - OPS_PREFIX = "mindspore.ops." - Tensor_PREFIX = "mindspore.Tensor." - MINT_PREFIX = "mindspore.mint." - MINT_NN_FUNC_PREFIX = "mindspore.mint.nn.functional." - COMM_PREFIX = "mindspore.communication.comm_func." - - API_PREFIX_DICT = { - "ops": OPS_PREFIX, - "Tensor": Tensor_PREFIX, - "mint": MINT_PREFIX, - "mint.nn.functional": MINT_NN_FUNC_PREFIX, - "communication": COMM_PREFIX - } - - PERT_VALUE_DICT = { - ms.bfloat16: 1e-4, - ms.float16: 1e-6, - ms.float32: 1e-8, - ms.float64: 1e-16 - } - - ERROR_THRESHOLD = { - ms.float16: 1.002, - ms.float32: 1.0002 - } - - PERT_BIT_DICT = { - ms.float16: np.int16, - ms.float32: np.int32, - ms.float64: np.int64 - } - - MS_NUMPY_DTYPE_DICT = { - ms.int16: np.int16, - ms.int32: np.int32, - ms.int64: np.int64, - ms.float16: np.float16, - ms.float32: np.float32, - ms.float64: np.float64 - } diff --git a/debug/accuracy_tools/msprobe/mindspore/common/const.py b/debug/accuracy_tools/msprobe/mindspore/common/const.py new file mode 100644 index 000000000..08bb97649 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/common/const.py @@ -0,0 +1,85 @@ +import numpy as np +import mindspore as ms + + +class Const: + CELL = "cell" + API = "api" + KERNEL = "kernel" + TOOL_LEVEL_DICT = { + "L0": CELL, + "L1": API, + "L2": KERNEL + } + PYNATIVE_MODE = "pynative" + GRAPH_GE_MODE = "graph_ge" + GRAPH_KBYK_MODE = "graph_kbyk" + + +class FreeBenchmarkConst: + DEFAULT_DEVICE = "npu" + DEFAULT_STAGE = "forward" + DEFAULT_DUMP_LEVEL = "L1" + DEFAULT_PERT_TYPE = "improve_precision" + DEFAULT_HANDLER_TYPE = "check" + FIX_HANDLER_MODE = "fix" + ADD_NOISE = "add_noise" + BIT_NOISE = "bit_noise" + NO_CHANGE = "no_change" + IMPROVE_PRECISION = "improve_precision" + CHECK = "check" + FIX = "fix" + DEVICE_LIST = ["npu"] + STAGE_LIST = ["forward"] + DUMP_LEVEL_LIST = ["L1"] + PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE] + HANDLER_TYPE_LIST = [CHECK, FIX] + COMMUNICATION_API_LIST = [ + "mindspore.communication.comm_func.all_gather_into_tensor", + "mindspore.communication.comm_func.gather_into_tensor", + "mindspore.communication.comm_func.all_reduce", + "mindspore.communication.comm_func.reduce", + "mindspore.communication.comm_func.reduce_scatter_tensor" + ] + NO_CHANGE_ERROR_THRESHOLD = 1.0 + SYMBOL_FLIPPING_RATIO = 8.0 + OPS_PREFIX = "mindspore.ops." + Tensor_PREFIX = "mindspore.Tensor." + MINT_PREFIX = "mindspore.mint." + MINT_NN_FUNC_PREFIX = "mindspore.mint.nn.functional." + COMM_PREFIX = "mindspore.communication.comm_func." + + API_PREFIX_DICT = { + "ops": OPS_PREFIX, + "Tensor": Tensor_PREFIX, + "mint": MINT_PREFIX, + "mint.nn.functional": MINT_NN_FUNC_PREFIX, + "communication": COMM_PREFIX + } + + PERT_VALUE_DICT = { + ms.bfloat16: 1e-4, + ms.float16: 1e-6, + ms.float32: 1e-8, + ms.float64: 1e-16 + } + + ERROR_THRESHOLD = { + ms.float16: 1.002, + ms.float32: 1.0002 + } + + PERT_BIT_DICT = { + ms.float16: np.int16, + ms.float32: np.int32, + ms.float64: np.int64 + } + + MS_NUMPY_DTYPE_DICT = { + ms.int16: np.int16, + ms.int32: np.int32, + ms.int64: np.int64, + ms.float16: np.float16, + ms.float32: np.float32, + ms.float64: np.float64 + } diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index b5c23ddf0..54f640703 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -1,7 +1,9 @@ import os from pathlib import Path -from msprobe.core.common.const import Const, MsConst, MsFreeBenchmarkConst +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import Const as MsConst +from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create @@ -28,16 +30,16 @@ class DebuggerConfig: self._make_dump_path_if_not_exists() if self.task == Const.FREE_BENCHMARK: - self.pert_type = (MsFreeBenchmarkConst.DEFAULT_PERT_TYPE + self.pert_type = (FreeBenchmarkConst.DEFAULT_PERT_TYPE if not task_config.pert_mode else task_config.pert_mode) - self.handler_type = (MsFreeBenchmarkConst.DEFAULT_HANDLER_TYPE + self.handler_type = (FreeBenchmarkConst.DEFAULT_HANDLER_TYPE if not task_config.handler_type else task_config.handler_type) - if self.handler_type == MsFreeBenchmarkConst.FIX_HANDLER_MODE and \ - self.pert_type != MsFreeBenchmarkConst.DEFAULT_PERT_TYPE: + if self.handler_type == FreeBenchmarkConst.FIX_HANDLER_MODE and \ + self.pert_type != FreeBenchmarkConst.DEFAULT_PERT_TYPE: raise ValueError("pert_mode must be improve_precision or empty when handler_type is fix, " f"but got {self.pert_type}.") - self.dump_level = MsFreeBenchmarkConst.DEFAULT_DUMP_LEVEL - self.stage = MsFreeBenchmarkConst.DEFAULT_STAGE + self.dump_level = FreeBenchmarkConst.DEFAULT_DUMP_LEVEL + self.stage = FreeBenchmarkConst.DEFAULT_STAGE def check(self): if not self.dump_path: diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index fb2b906ce..6ef1966bc 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -6,7 +6,8 @@ from msprobe.mindspore.service import Service from msprobe.mindspore.ms_config import parse_json_config from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.task_handler_factory import TaskHandlerFactory -from msprobe.core.common.const import Const, MsConst +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.runtime import Runtime diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index 541ba14f4..bcfa31520 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -6,7 +6,8 @@ import yaml import mindspore as ms from mindspore.communication import comm_func -from msprobe.core.common.const import Const, MsFreeBenchmarkConst +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.config import Config from msprobe.core.common.file_check import check_path_length, FileOpen from msprobe.mindspore.common.log import logger @@ -43,7 +44,7 @@ def get_supported_ops(): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "data", "support_wrap_ops.yaml") - for k, v in MsFreeBenchmarkConst.API_PREFIX_DICT.items(): + for k, v in FreeBenchmarkConst.API_PREFIX_DICT.items(): with FileOpen(yaml_path, 'r') as f: ops = yaml.safe_load(f).get(k) if ops: @@ -52,23 +53,23 @@ def get_supported_ops(): _all_functional_ops = [] ms_ops = dir(ms.ops) - ms_ops = [MsFreeBenchmarkConst.OPS_PREFIX + i for i in ms_ops] + ms_ops = [FreeBenchmarkConst.OPS_PREFIX + i for i in ms_ops] _all_functional_ops += ms_ops ms_tensor = dir(ms.Tensor) - ms_tensor = [MsFreeBenchmarkConst.Tensor_PREFIX + i for i in ms_tensor] + ms_tensor = [FreeBenchmarkConst.Tensor_PREFIX + i for i in ms_tensor] _all_functional_ops += ms_tensor ms_mint = dir(ms.mint) - ms_mint = [MsFreeBenchmarkConst.MINT_PREFIX + i for i in ms_mint] + ms_mint = [FreeBenchmarkConst.MINT_PREFIX + i for i in ms_mint] _all_functional_ops += ms_mint ms_mint_nn_func = dir(ms.mint.nn.functional) - ms_mint_nn_func = [MsFreeBenchmarkConst.MINT_NN_FUNC_PREFIX + i for i in ms_mint_nn_func] + ms_mint_nn_func = [FreeBenchmarkConst.MINT_NN_FUNC_PREFIX + i for i in ms_mint_nn_func] _all_functional_ops += ms_mint_nn_func ms_communication = dir(comm_func) - ms_communication = [MsFreeBenchmarkConst.COMM_PREFIX + i for i in ms_communication] + ms_communication = [FreeBenchmarkConst.COMM_PREFIX + i for i in ms_communication] _all_functional_ops += ms_communication return set(supported_ops) & set(_all_functional_ops) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py index 4a22e203d..85f684d81 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py @@ -1,12 +1,12 @@ -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst class Config: is_enable: bool = False - handler_type = MsFreeBenchmarkConst.DEFAULT_HANDLER_TYPE - pert_type = MsFreeBenchmarkConst.DEFAULT_PERT_TYPE - stage = MsFreeBenchmarkConst.DEFAULT_STAGE - dump_level = MsFreeBenchmarkConst.DEFAULT_DUMP_LEVEL + handler_type = FreeBenchmarkConst.DEFAULT_HANDLER_TYPE + pert_type = FreeBenchmarkConst.DEFAULT_PERT_TYPE + stage = FreeBenchmarkConst.DEFAULT_STAGE + dump_level = FreeBenchmarkConst.DEFAULT_DUMP_LEVEL steps: list = [] ranks: list = [] dump_path: str = "" diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py index 3cc0f0789..3bb062800 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py @@ -6,7 +6,7 @@ import mindspore as ms from mindspore import Tensor from msprobe.mindspore.runtime import Runtime -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst from .config import Config from .handler_params import HandlerParams @@ -25,9 +25,9 @@ class Tools: @staticmethod def get_default_error_threshold(dtype): - if Config.pert_type == MsFreeBenchmarkConst.NO_CHANGE: - return MsFreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD - return MsFreeBenchmarkConst.ERROR_THRESHOLD.get(dtype, MsFreeBenchmarkConst.ERROR_THRESHOLD.get(ms.float32)) + if Config.pert_type == FreeBenchmarkConst.NO_CHANGE: + return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + return FreeBenchmarkConst.ERROR_THRESHOLD.get(dtype, FreeBenchmarkConst.ERROR_THRESHOLD.get(ms.float32)) @dataclass diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py index f745f711c..78661d7fc 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py @@ -1,5 +1,5 @@ from msprobe.mindspore.free_benchmark.common.config import Config -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams from msprobe.mindspore.free_benchmark.handler.handler_factory import HandlerFactory from msprobe.mindspore.free_benchmark.perturbation.perturbation_factory import PerturbationFactory @@ -23,11 +23,11 @@ class ForwardSelfChecker: return params.original_result def get_compare_data(self, params: HandlerParams): - if self.api_name not in MsFreeBenchmarkConst.COMMUNICATION_API_LIST: + if self.api_name not in FreeBenchmarkConst.COMMUNICATION_API_LIST: return # 以下为通讯类api处理逻辑 params.fuzzed_result = params.fuzzed_value - if Config.pert_type == MsFreeBenchmarkConst.IMPROVE_PRECISION: + if Config.pert_type == FreeBenchmarkConst.IMPROVE_PRECISION: params.original_result = params.args else: params.original_result = params.args[params.index] @@ -37,6 +37,6 @@ class ForwardSelfChecker: self.get_compare_data(params) handler = HandlerFactory.create(self.api_name) result = handler.handle(params) - if self.api_name in MsFreeBenchmarkConst.COMMUNICATION_API_LIST: + if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST: result = original_result return result diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py index 375ed057a..f35d23498 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py @@ -7,7 +7,7 @@ from mindspore import Tensor, ops from msprobe.mindspore.common.log import logger from msprobe.mindspore.free_benchmark.common.utils import Tools -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams @@ -18,8 +18,8 @@ class BaseHandler(ABC): @staticmethod def pre_calculate(original_output, fuzzed_output): - abs_tol = MsFreeBenchmarkConst.PERT_VALUE_DICT.get(fuzzed_output.dtype, - MsFreeBenchmarkConst.PERT_VALUE_DICT.get(ms.float32)) + abs_tol = FreeBenchmarkConst.PERT_VALUE_DICT.get(fuzzed_output.dtype, + FreeBenchmarkConst.PERT_VALUE_DICT.get(ms.float32)) return original_output.to(fuzzed_output.dtype), fuzzed_output, abs_tol @@ -31,7 +31,7 @@ class BaseHandler(ABC): @staticmethod def convert_overflow_ratio_to_consistent(ratio): if math.isnan(ratio) or math.isinf(ratio): - return MsFreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD return ratio @staticmethod @@ -47,7 +47,7 @@ class BaseHandler(ABC): norm1 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor1)[0].to(ms.float32).item()) norm2 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor2)[0].to(ms.float32).item()) norm3 = BaseHandler.convert_overflow_ratio_to_consistent(ops.min(ratio_tensor1)[0].to(ms.float32).item()) - ratio = MsFreeBenchmarkConst.SYMBOL_FLIPPING_RATIO if norm3 < 0 else max(norm1, norm2) + ratio = FreeBenchmarkConst.SYMBOL_FLIPPING_RATIO if norm3 < 0 else max(norm1, norm2) return ratio @@ -57,7 +57,7 @@ class BaseHandler(ABC): original_output, fuzzed_output, abs_tol = BaseHandler.pre_calculate(original_output, fuzzed_output) except Exception as e: logger.error(f"When computing ratio, y1 or y2 dtype is not supported {str(e)}") - return MsFreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD abs_tol = abs_tol ** 0.5 diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py index 8d709cb0d..bf8c681e5 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py @@ -1,14 +1,14 @@ from msprobe.mindspore.common.log import logger from msprobe.mindspore.free_benchmark.common.config import Config -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst from .check_handler import CheckHandler from .fix_handler import FixHandler class HandlerFactory: result_handlers = { - MsFreeBenchmarkConst.CHECK: CheckHandler, - MsFreeBenchmarkConst.FIX: FixHandler, + FreeBenchmarkConst.CHECK: CheckHandler, + FreeBenchmarkConst.FIX: FixHandler, } @staticmethod diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py index 3d645a6f1..2764d3d49 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py @@ -5,7 +5,7 @@ from mindspore import Tensor, ops from msprobe.mindspore.common.log import logger from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst class AddNoisePerturbation(BasePerturbation): @@ -53,7 +53,7 @@ class AddNoisePerturbation(BasePerturbation): if not ops.is_floating_point(input) or ops.numel(input) == 0: return False - pert_value = MsFreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype) + pert_value = FreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype) if not pert_value: return False else: diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py index b682edf09..65202e0f6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py @@ -4,7 +4,7 @@ import numpy as np from mindspore import Tensor, ops from msprobe.mindspore.common.log import logger -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation @@ -15,10 +15,10 @@ class BitNoisePerturbation(BasePerturbation): if isinstance(inputs, Tensor): bit_len_type = self._get_bit_len_type(inputs) if bit_len_type is not False: - sub_normal_np = np.finfo(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)).smallest_normal + sub_normal_np = np.finfo(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)).smallest_normal sub_normal = Tensor(sub_normal_np) - noise_type = list(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.keys())[ - list(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.values()).index(bit_len_type)] + noise_type = list(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.keys())[ + list(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.values()).index(bit_len_type)] noise = ops.full(inputs.shape, 1, dtype=noise_type) input_np = inputs.asnumpy() input_np_int = input_np.view(bit_len_type) @@ -26,7 +26,7 @@ class BitNoisePerturbation(BasePerturbation): result = ops.where(ops.abs(inputs) > sub_normal, ops.bitwise_xor(result, noise), result) result_np = result.asnumpy() - result_np_float = result_np.view(MsFreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)) + result_np_float = result_np.view(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)) self.is_fuzzed = True return Tensor(result_np_float) @@ -51,10 +51,10 @@ class BitNoisePerturbation(BasePerturbation): if not isinstance(input, Tensor) or not ops.is_floating_point(input) or \ input.numel() == 0: return False - bit_len_type = MsFreeBenchmarkConst.PERT_BIT_DICT.get(input.dtype) + bit_len_type = FreeBenchmarkConst.PERT_BIT_DICT.get(input.dtype) if not bit_len_type: return False - pert_value = MsFreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype) + pert_value = FreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype) if not pert_value: return False max_val = ops.max(ops.abs(input))[0].item() diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py index c32536187..f55a96aca 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py @@ -5,7 +5,7 @@ from mindspore import Tensor, ops from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.common.log import logger @@ -26,7 +26,7 @@ class ImprovePrecisionPerturbation(BasePerturbation): args = self.improve_tensor_precision(params.args) kwargs = self.improve_tensor_precision(params.kwargs) fuzzed_value = args - if self.api_name in MsFreeBenchmarkConst.COMMUNICATION_API_LIST: + if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST: params.fuzzed_value = fuzzed_value if not self.is_fuzzed: logger.warning(f"{self.api_name} can not improve precision.") diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py index 01d1fa6e7..6c8328dc2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py @@ -1,4 +1,4 @@ -from msprobe.core.common.const import MsFreeBenchmarkConst +from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.config import Config from .add_noise import AddNoisePerturbation from .bit_noise import BitNoisePerturbation @@ -12,10 +12,10 @@ class PerturbationFactory: """ perturbations = { - MsFreeBenchmarkConst.IMPROVE_PRECISION: ImprovePrecisionPerturbation, - MsFreeBenchmarkConst.ADD_NOISE: AddNoisePerturbation, - MsFreeBenchmarkConst.BIT_NOISE: BitNoisePerturbation, - MsFreeBenchmarkConst.NO_CHANGE: NoChangePerturbation, + FreeBenchmarkConst.IMPROVE_PRECISION: ImprovePrecisionPerturbation, + FreeBenchmarkConst.ADD_NOISE: AddNoisePerturbation, + FreeBenchmarkConst.BIT_NOISE: BitNoisePerturbation, + FreeBenchmarkConst.NO_CHANGE: NoChangePerturbation, } @staticmethod diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py index c9a0d8a65..e485887ce 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py @@ -1,24 +1,24 @@ -from msprobe.core.common.const import MsConst +from msprobe.mindspore.common.const import Const from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelFCheck class SelfCheckToolFactory: tools = { - MsConst.CELL: { - MsConst.GRAPH_KBYK_MODE: None, - MsConst.GRAPH_GE_MODE: None, - MsConst.PYNATIVE_MODE: None + Const.CELL: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: None }, - MsConst.API: { - MsConst.GRAPH_KBYK_MODE: None, - MsConst.GRAPH_GE_MODE: None, - MsConst.PYNATIVE_MODE: ApiPyNativeSelFCheck + Const.API: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: ApiPyNativeSelFCheck }, - MsConst.KERNEL: { - MsConst.GRAPH_KBYK_MODE: None, - MsConst.GRAPH_GE_MODE: None, - MsConst.PYNATIVE_MODE: None + Const.KERNEL: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: None } } diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index ad5de0bf3..05beeea32 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -2,7 +2,8 @@ import json from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.const import Const, MsFreeBenchmarkConst +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.common.log import logger @@ -59,15 +60,15 @@ class FreeBenchmarkConfig(BaseConfig): self._check_config() def _check_config(self): - if self.fuzz_device and self.fuzz_device not in MsFreeBenchmarkConst.DEVICE_LIST: + if self.fuzz_device and self.fuzz_device not in FreeBenchmarkConst.DEVICE_LIST: raise Exception("fuzz_device must be npu or empty") - if self.pert_mode and self.pert_mode not in MsFreeBenchmarkConst.PERT_TYPE_LIST: + if self.pert_mode and self.pert_mode not in FreeBenchmarkConst.PERT_TYPE_LIST: raise Exception("pert_mode must be improve_precision, add_noise, bit_noise , no_change or empty") - if self.handler_type and self.handler_type not in MsFreeBenchmarkConst.HANDLER_TYPE_LIST: + if self.handler_type and self.handler_type not in FreeBenchmarkConst.HANDLER_TYPE_LIST: raise Exception("handler_type must be check, fix or empty") - if self.fuzz_level and self.fuzz_level not in MsFreeBenchmarkConst.DUMP_LEVEL_LIST: + if self.fuzz_level and self.fuzz_level not in FreeBenchmarkConst.DUMP_LEVEL_LIST: raise Exception("fuzz_level must be L1 or empty") - if self.fuzz_stage and self.fuzz_stage not in MsFreeBenchmarkConst.STAGE_LIST: + if self.fuzz_stage and self.fuzz_stage not in FreeBenchmarkConst.STAGE_LIST: raise Exception("fuzz_stage must be forward or empty") if self.if_preheat or self.preheat_step or self.max_sample: logger.warning("'if_preheat', 'preheat_step' and 'max_sample' settings " diff --git a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py index 45fff4cd4..dfe2fbe2c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py @@ -1,4 +1,5 @@ -from msprobe.core.common.const import Const, MsConst +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.dump_tool_factory import DumpToolFactory from msprobe.mindspore.overflow_check.overflow_check_tool_factory import OverflowCheckToolFactory diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py index 699df3bae..cdc88a3be 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py @@ -21,7 +21,7 @@ from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.kernel_graph_dump import KernelGraphDump from msprobe.mindspore.task_handler_factory import TaskHandlerFactory -from msprobe.core.common.const import MsConst +from msprobe.mindspore.common.const import Const class TestTaskHandlerFactory(TestCase): @@ -44,7 +44,7 @@ class TestTaskHandlerFactory(TestCase): common_config = CommonConfig(json_config) task_config = BaseConfig(json_config) config = DebuggerConfig(common_config, task_config) - config.execution_mode = MsConst.GRAPH_GE_MODE + config.execution_mode = Const.GRAPH_GE_MODE handler = TaskHandlerFactory.create(config) self.assertTrue(isinstance(handler, KernelGraphDump)) -- Gitee From 5b64ac0fca7f1683de94cde1954c791639c58835 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 7 Aug 2024 10:27:58 +0800 Subject: [PATCH 159/791] =?UTF-8?q?=E5=B0=86is=5Fterminated=E6=8F=90?= =?UTF-8?q?=E5=8F=96=E5=88=B0BaseDataProcessor=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/data_dump/data_processor/base.py | 7 +++++++ .../data_dump/data_processor/mindspore_processor.py | 11 ----------- .../data_dump/data_processor/pytorch_processor.py | 11 ----------- 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index e15000008..80db0104b 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -65,6 +65,8 @@ class BaseDataProcessor: self.current_iter = 0 self._return_forward_new_output = False self._forward_new_output = None + self.real_overflow_dump_times = 0 + self.overflow_nums = config.overflow_nums @property def data_path(self): @@ -72,6 +74,11 @@ class BaseDataProcessor: @property def is_terminated(self): + if self.overflow_nums == -1: + return False + if self.real_overflow_dump_times >= self.overflow_nums: + logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_times}") + return True return False @staticmethod diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index d8f7093fe..877fc3a01 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -154,18 +154,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): def __init__(self, config, data_writer): super().__init__(config, data_writer) self.cached_tensors_and_file_paths = {} - self.real_overflow_dump_times = 0 - self.overflow_nums = config.overflow_nums - @property - def is_terminated(self): - if self.overflow_nums == -1: - return False - if self.real_overflow_dump_times >= self.overflow_nums: - logger.warning(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_times}") - return True - return False - def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False api_info_struct = super().analyze_forward(name, module, module_input_output) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index f8bf38119..191a33f9f 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -183,19 +183,8 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): def __init__(self, config, data_writer): super().__init__(config, data_writer) self.cached_tensors_and_file_paths = {} - self.real_overflow_dump_times = 0 - self.overflow_nums = config.overflow_nums self.bits_for_overflow = 8 - @property - def is_terminated(self): - if self.overflow_nums == -1: - return False - if self.real_overflow_dump_times >= self.overflow_nums: - logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_times}") - return True - return False - @staticmethod def overflow_debug_mode_enable(): overflow_mode = os.getenv(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) -- Gitee From 0fbac448466a36a3d81e03261dc9a4cd62d596e6 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Wed, 7 Aug 2024 10:42:47 +0800 Subject: [PATCH 160/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dhook=E6=B3=A8?= =?UTF-8?q?=E5=86=8C=E9=A1=BA=E5=BA=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/service.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index b417fa88d..79abfdc9e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -175,10 +175,6 @@ class Service: prefix = BaseScope.Module_Type_Module + Const.SEP + name + Const.SEP + \ module.__class__.__name__ + Const.SEP - module.register_forward_pre_hook( - self.module_processor.node_hook(prefix + Const.FORWARD, Const.START)) - module.register_forward_hook( - self.module_processor.node_hook(prefix + Const.FORWARD, Const.STOP)) module.register_full_backward_hook( self.module_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) @@ -190,6 +186,11 @@ class Service: module.register_forward_hook(forward_hook_torch_version_below_2) module.register_full_backward_hook(backward_hook) + module.register_forward_pre_hook( + self.module_processor.node_hook(prefix + Const.FORWARD, Const.START)) + module.register_forward_hook( + self.module_processor.node_hook(prefix + Const.FORWARD, Const.STOP)) + if self.config.level in ["mix", "L1", "L2"]: api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) api_register.api_modularity() -- Gitee From 5438d75958a2b0c5483fd864be60461b3e8d303f Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 7 Aug 2024 10:46:59 +0800 Subject: [PATCH 161/791] =?UTF-8?q?=E3=80=90BugFix=E3=80=91=E5=90=88?= =?UTF-8?q?=E5=B9=B6=E5=88=86=E6=94=AF=E5=AF=BC=E8=87=B4=E9=83=A8=E5=88=86?= =?UTF-8?q?=E6=94=B9=E5=8A=A8=E6=9C=AA=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/utils.py | 20 ++++++++++++---- .../mindspore/compare/distributed_compare.py | 23 +++++++++---------- debug/accuracy_tools/msprobe/msprobe.py | 2 +- .../msprobe/pytorch/compare/pt_compare.py | 11 --------- 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index d213e0b46..de4047fd9 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -38,13 +38,13 @@ def read_op(op_data, op_name): op_parsed_list += output_parsed_list output_parsed_list.clear() if 'backward' in op_name: - if 'grad_input' in op_data: - input_item = op_data['grad_input'] + if 'input' in op_data: + input_item = op_data['input'] input_parsed_list = op_item_parse(input_item, op_name + '_input', None) op_parsed_list = input_parsed_list.copy() input_parsed_list.clear() - if 'grad_output' in op_data: - output_item = op_data['grad_output'] + if 'output' in op_data: + output_item = op_data['output'] output_parsed_list = op_item_parse(output_item, op_name + '_output', None) op_parsed_list += output_parsed_list output_parsed_list.clear() @@ -352,7 +352,17 @@ def merge_tensor(tensor_list, summary_compare, md5_compare): return op_dict if op_dict["op_name"] else {} - +def _compare_parser(parser): + parser.add_argument("-i", "--input_path", dest="input_path", type=str, + help=" The compare input path, a dict json.", required=True) + parser.add_argument("-o", "--output_path", dest="output_path", type=str, + help=" The compare task result out path.", required=True) + parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", + help=" Whether to save stack info.", required=False) + parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", + help=" Whether to give advisor.", required=False) + parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", + help=" Whether to perform a fuzzy match on the api name.", required=False) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 94d03f4f2..cab07daec 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -20,10 +20,10 @@ import re from msprobe.core.common.utils import CompareException, check_compare_param, \ check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid from msprobe.core.common.file_check import create_directory +from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import MSComparator - def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): def check_and_return_dir_contents(dump_dir, prefix): """ @@ -87,25 +87,24 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): 'or use compare() api and manually match the ranks.') raise CompareException(CompareException.INVALID_PATH_ERROR) for nr, br in zip(npu_ranks, bench_ranks): - n_dir = os.path.join(npu_dump_dir, nr) - b_dir = os.path.join(bench_dump_dir, br) - s_dir = b_dir - npu_json_path = extract_json(n_dir, stack_json=False) - bench_json_path = extract_json(b_dir, stack_json=False) - stack_json_path = extract_json(s_dir, stack_json=True) + npu_data_dir = os.path.join(npu_dump_dir, nr) + bench_data_dir = os.path.join(bench_dump_dir, br) + npu_path = extract_json(npu_data_dir, stack_json=False) + bench_path = extract_json(bench_data_dir, stack_json=False) + stack_path = extract_json(npu_data_dir, stack_json=True) dump_result_param = { - 'npu_json_path': npu_json_path, - 'bench_json_path': bench_json_path, - 'stack_json_path': stack_json_path, + 'npu_path': npu_path, + 'bench_path': bench_path, + 'stack_path': stack_path, 'is_print_compare_log': True } try: summary_compare, md5_compare = task_dumppath_get(dump_result_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(dump_result_param, output_path, stack_mode=stack_mode, summary_compare=summary_compare) - except CompareException as error: + check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) + except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') sys.exit(error.code) msComparator=MSComparator() diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 5146ee1ac..4bc841654 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -22,7 +22,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _ _api_precision_compare_command from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command -from msprobe.pytorch.compare.pt_compare import _compare_parser +from msprobe.core.compare.utils import _compare_parser from msprobe.pytorch.compare.compare_cli import compare_cli from msprobe.mindspore.compare.compare_cli import compare_cli_ms diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index b32e6df60..75bc9d4f3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -247,17 +247,6 @@ def pt_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu md5_compare=md5_compare) -def _compare_parser(parser): - parser.add_argument("-i", "--input_path", dest="input_path", type=str, - help=" The compare input path, a dict json.", required=True) - parser.add_argument("-o", "--output_path", dest="output_path", type=str, - help=" The compare task result out path.", required=True) - parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", - help=" Whether to save stack info.", required=False) - parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", - help=" Whether to give advisor.", required=False) - parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", - help=" Whether to perform a fuzzy match on the api name.", required=False) -- Gitee From 848bec725e419784d40c6e9a2032d7ca7cdb3165 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Wed, 7 Aug 2024 10:52:37 +0800 Subject: [PATCH 162/791] =?UTF-8?q?=E4=BD=BF=E7=94=A8const=E5=B8=B8?= =?UTF-8?q?=E9=87=8F=E6=9B=BF=E6=8D=A2=E5=AD=97=E7=AC=A6=E4=B8=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/module_processer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/module_processer.py index c18288ef2..688c0b8c5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/module_processer.py @@ -121,9 +121,9 @@ class ModuleProcesser: if self.scope: self.scope.begin_module(full_name) - if 'forward' in name_prefix and Const.START in start_or_stop: + if Const.FORWARD in name_prefix and Const.START in start_or_stop: return pre_hook - elif 'backward' in name_prefix: + elif Const.BACKWARD in name_prefix: return backward_hook else: return end_hook -- Gitee From d3e579bd4f3521c9bc1d32f6fa284e0d1ad6192b Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Tue, 6 Aug 2024 20:06:44 +0800 Subject: [PATCH 163/791] bugfix: handle raise error --- profiler/cli/compare_cli.py | 5 +-- .../compare_backend/comparison_generator.py | 35 ++++++++++++++----- .../disaggregate/overall_perf_interface.py | 15 ++++++-- .../compare_backend/utils/args_manager.py | 11 +++--- .../compare_interface/comparison_interface.py | 1 - profiler/compare_tools/performance_compare.py | 1 - 6 files changed, 45 insertions(+), 23 deletions(-) diff --git a/profiler/cli/compare_cli.py b/profiler/cli/compare_cli.py index f9add948e..3a36d2cd9 100644 --- a/profiler/cli/compare_cli.py +++ b/profiler/cli/compare_cli.py @@ -44,7 +44,4 @@ from profiler.compare_tools.compare_backend.comparison_generator import Comparis @click.option('--gpu_flow_cat', type=str, default='', help="Identifier of the GPU connection.") def compare_cli(**kwargs) -> None: args = AnalyzeDict(kwargs) - try: - ComparisonGenerator(args).run() - except RuntimeError as e: - print(f"[ERROR] {e}") + ComparisonGenerator(args).run() diff --git a/profiler/compare_tools/compare_backend/comparison_generator.py b/profiler/compare_tools/compare_backend/comparison_generator.py index b07170b64..b4d17f88e 100644 --- a/profiler/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/compare_tools/compare_backend/comparison_generator.py @@ -12,13 +12,22 @@ class ComparisonGenerator: INTERFACE_DICT = {Constant.OVERALL_COMPARE: OverallInterface} def __init__(self, args): - self._args_manager = ArgsManager() - self._args_manager.init(args) + self._args_manager = ArgsManager(args) self._data_dict = {} def run(self): - self.load_data() - self.generate_compare_result() + try: + self._args_manager.init() + self.load_data() + self.generate_compare_result() + except NotImplementedError as e: + print(f"[ERROR] {e}") + except RuntimeError as e: + print(f"[ERROR] {e}") + except FileNotFoundError as e: + print(f"[ERROR] {e}") + except Exception as e: + print(f"[ERROR] {e}") def load_data(self): self._data_dict[Constant.BASE_DATA] = self.PARSER_DICT.get(self._args_manager.base_profiling_type)( @@ -37,8 +46,18 @@ class ComparisonGenerator: generator.join() def run_interface(self, compare_type: str) -> dict: - self.load_data() - interface = self.INTERFACE_DICT.get(compare_type) - if interface: - return interface(self._data_dict).run() + try: + self._args_manager.init() + self.load_data() + interface = self.INTERFACE_DICT.get(compare_type) + if interface: + return interface(self._data_dict).run() + except NotImplementedError as e: + print(f"[ERROR] {e}") + except RuntimeError as e: + print(f"[ERROR] {e}") + except FileNotFoundError as e: + print(f"[ERROR] {e}") + except Exception as e: + print(f"[ERROR] {e}") return {} diff --git a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index 7bac2b033..65524664e 100644 --- a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -15,9 +15,18 @@ class OverallPerfInterface: self._result_data = {} def run(self): - self._check_path() - self._load_data() - self._generate_result() + try: + self._check_path() + self._load_data() + self._generate_result() + except NotImplementedError as e: + print(f"[ERROR] {e}") + except RuntimeError as e: + print(f"[ERROR] {e}") + except FileNotFoundError as e: + print(f"[ERROR] {e}") + except Exception as e: + print(f"[ERROR] {e}") return self._result_data def _check_path(self): diff --git a/profiler/compare_tools/compare_backend/utils/args_manager.py b/profiler/compare_tools/compare_backend/utils/args_manager.py index ab9fb43a9..579bf9b99 100644 --- a/profiler/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/compare_tools/compare_backend/utils/args_manager.py @@ -11,17 +11,17 @@ class Singleton(object): self._cls = cls self._instance = {} - def __call__(self): + def __call__(self, args): if self._cls not in self._instance: - self._instance[self._cls] = self._cls() + self._instance[self._cls] = self._cls(args) return self._instance[self._cls] @Singleton class ArgsManager: - def __init__(self): - self._args = None + def __init__(self, args: any): + self._args = args self._base_path_dict = {} self._comparison_path_dict = {} @@ -114,8 +114,7 @@ class ArgsManager: path_dict.update({Constant.INFO_JSON_PATH: os.path.join(file_path, dir_name)}) return path_dict - def init(self, args: any): - self._args = args + def init(self): if self._args.max_kernel_num is not None and self._args.max_kernel_num <= Constant.LIMIT_KERNEL: msg = f"Invalid param, --max_kernel_num has to be greater than {Constant.LIMIT_KERNEL}" raise RuntimeError(msg) diff --git a/profiler/compare_tools/compare_interface/comparison_interface.py b/profiler/compare_tools/compare_interface/comparison_interface.py index 919095b31..b747aae47 100644 --- a/profiler/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/compare_tools/compare_interface/comparison_interface.py @@ -21,7 +21,6 @@ class ComparisonInterface: def compare(self, compare_type: str) -> dict: if compare_type == Constant.OVERALL_COMPARE: self._args.enable_profiling_compare = True - return ComparisonGenerator(self._args).run_interface(compare_type) def disaggregate_perf(self, compare_type: str) -> dict: diff --git a/profiler/compare_tools/performance_compare.py b/profiler/compare_tools/performance_compare.py index 7c9d60aac..7c3fcdb6e 100644 --- a/profiler/compare_tools/performance_compare.py +++ b/profiler/compare_tools/performance_compare.py @@ -31,7 +31,6 @@ def main(): ComparisonGenerator(args).run() - if __name__ == "__main__": start_time = datetime.datetime.now() main() -- Gitee From 739fe519dfc685eaa3c2b50479e834845069b527 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Wed, 7 Aug 2024 11:43:14 +0800 Subject: [PATCH 164/791] =?UTF-8?q?[compare=5Ftools]=E8=B5=84=E6=96=99?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E6=84=8F=E8=A7=81=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/compare_tools/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index b40f19e92..99559728d 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -196,7 +196,7 @@ MindSpore场景仅支持**总体性能**和**通信性能**的对比。 | Lccl Time(Num) | Lccl算子耗时，Num表示计算的次数。 | | Computing Time | 计算流耗时，计算流所有event耗时总和。如果有多条并发计算，计算流耗时对重叠部分只会计算一次。 | | Mem Usage | 内存使用。GPU上的内存使用可以使用nvidia-smi查看，NPU上的内存使用可以使用npu-smi查看，Profiling信息采集时打开profile_memory=True开关，mem usage显示的是memory_record里面的最大resevered值，一般来说是进程级内存。 | -| Uncovered Communication Time(Wait Time) | 通信未掩盖耗时，包含Wait Time（只有采集性能数据的Level等级为L1以上并且采集NPU数据时才会存在）为同步时间。 | +| Uncovered Communication Time(Wait Time) | 通信未掩盖耗时。Wait Time为卡间等待时间（Wait Time仅NPU场景才会存在）。 | | SDMA Time(Num) | 拷贝类任务耗时，Num表示计算的次数。 | | Free Time | 调度耗时 = E2E耗时 - 算子耗时 - 通信不可掩盖耗时。Free的定义为Device侧既不在通信又不在计算的时间，因此包含拷贝时间（SDMA Time）。 | | E2E Time(Not minimal profiling) | E2E总耗时，计算流端到端耗时。当存在Not minimal profiling时，表示该时间存在性能膨胀，会影响通信和调度耗时。 | @@ -221,7 +221,7 @@ Index列字段说明： | 字段 | | | 说明 | | ---------------------------- | ------------------ | ----------------------------------- | ------------------------------------------------------------ | -| Computing Time | | | 计算流耗时，计算流所有event耗时总和。如果有多条并发计算，计算流耗时对重叠部分只会计算一次。 | +| Computing Time | | | 计算流耗时，计算流所有event耗时总和。如果有多条并发计算，计算流耗时对重叠部分只会计算一次。
NPU场景下，仅当采集性能数据的Level等级为L1及以上且aic_metrics取值为PipeUtilization时才可拆分出Computing Time的二级字段Flash Attention、Conv等。 | | | Flash Attention | | Flash Attention算子。 | | | | Flash Attention (Forward) (Cube) | Flash Attention前向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | | | | Flash Attention (Forward) (Vector) | Flash Attention前向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | -- Gitee From 0de005cf96e5be201ce780ffbaf9d2632ba43753 Mon Sep 17 00:00:00 2001 From: zyy Date: Wed, 7 Aug 2024 11:49:11 +0800 Subject: [PATCH 165/791] profiling_compare_print --- .../compare_bean/profiling_info.py | 175 +++++++++--------- .../profiling_parser/gpu_profiling_parser.py | 19 -- .../profiling_parser/npu_profiling_parser.py | 24 --- .../compare_bean/test_profiling_info.py | 73 +++----- .../test_gpu_profiling_parser.py | 12 +- 5 files changed, 123 insertions(+), 180 deletions(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index fe5781426..9454064c0 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -8,31 +8,15 @@ class ProfilingInfo: def __init__(self, profiling_type: str): self.profiling_type = profiling_type - self.cube_time = 0.0 self.other_time = 0.0 - self.vec_time = 0.0 - self.cube_num = 0 - self.vec_num = 0 - self.sdma_num = 0 - self.fa_num_fwd = 0 - self.fa_num_bwd = 0 - self.pa_num = 0 self.lccl_num = 0 - self.conv_time_fwd = 0.0 - self.conv_time_bwd = 0.0 - self.conv_num_fwd = 0 - self.conv_num_bwd = 0 self.compute_time = 0.0 self.communication_not_overlapped = 0.0 self.wait_time = 0.0 self.memory_used = 0.0 self.e2e_time = 0.0 - self.sdma_time = 0.0 self.scheduling_time = 0.0 - self.fa_time_bwd = 0.0 - self.pa_time = 0.0 self.lccl_time = 0.0 - self.fa_time_fwd = 0.0 self.minimal_profiling = False self.hide_op_details = False self.is_level0 = False @@ -138,61 +122,78 @@ class ProfilingInfo: def vector_total_num(self): return sum((self.vector_num_trans, self.vector_num_notrans)) - def trans_time_to_s(self): - self.cube_time = self.cube_time / 10 ** 6 - self.other_time = self.other_time / 10 ** 6 - self.vec_time = self.vec_time / 10 ** 6 - self.compute_time = self.compute_time / 10 ** 6 - self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 - self.wait_time = self.wait_time / 10 ** 6 - self.e2e_time = self.e2e_time / 10 ** 6 - self.sdma_time = self.sdma_time / 10 ** 6 - self.scheduling_time = self.scheduling_time / 10 ** 6 - self.fa_time_bwd = self.fa_time_bwd / 10 ** 6 - self.fa_time_fwd = self.fa_time_fwd / 10 ** 6 - self.pa_time = self.pa_time / 10 ** 6 - self.lccl_time = self.lccl_time / 10 ** 6 - self.conv_time_fwd = self.conv_time_fwd / 10 ** 6 - self.conv_time_bwd = self.conv_time_bwd / 10 ** 6 + @property + def cube_time(self): + return (self.matmul_time_cube + self.matmul_time_vector + self.other_cube_time) / 1000 - # 新指标单位为ms - self.fa_time_fwd_cube /= 10 ** 3 - self.fa_time_bwd_cube /= 10 ** 3 - self.fa_time_fwd_vector /= 10 ** 3 - self.fa_time_bwd_vector /= 10 ** 3 - self.conv_time_fwd_cube /= 10 ** 3 - self.conv_time_bwd_cube /= 10 ** 3 - self.conv_time_fwd_vector /= 10 ** 3 - self.conv_time_bwd_vector /= 10 ** 3 - self.matmul_time_cube /= 10 ** 3 - self.matmul_time_vector /= 10 ** 3 - self.vector_time_trans /= 10 ** 3 - self.vector_time_notrans /= 10 ** 3 - self.sdma_time_tensor_move /= 10 ** 3 - self.sdma_time_stream /= 10 ** 3 - self.page_attention_time /= 10 ** 3 - self.other_cube_time /= 10 ** 3 + @property + def vec_time(self): + return (self.vector_time_trans + self.vector_time_notrans) / 1000 + + @property + def cube_num(self): + return self.matmul_num_cube + self.matmul_num_vector + self.other_cube_num + + @property + def vec_num(self): + return self.vector_num_trans + self.vector_num_notrans + + @property + def sdma_num(self): + return self.sdma_num_tensor_move + self.sdma_num_stream + + @property + def fa_num_fwd(self): + return self.fa_num_fwd_cube + self.fa_num_fwd_vector + + @property + def fa_num_bwd(self): + return self.fa_num_bwd_cube + self.fa_num_bwd_vector + + @property + def pa_num(self): + return self.page_attention_num + + @property + def pa_time(self): + return self.page_attention_time / 1000 + + @property + def conv_time_fwd(self): + return (self.conv_time_fwd_cube + self.conv_time_fwd_vector) / 1000 + + @property + def conv_time_bwd(self): + return (self.conv_time_bwd_cube + self.conv_time_bwd_vector) / 1000 + + @property + def conv_num_fwd(self): + return self.conv_num_fwd_cube + self.conv_num_fwd_vector + + @property + def conv_num_bwd(self): + return self.conv_num_bwd_cube + self.conv_num_bwd_vector + + @property + def sdma_time(self): + return (self.sdma_time_tensor_move + self.sdma_time_stream) / 1000 + + @property + def fa_time_fwd(self): + return (self.fa_time_fwd_cube + self.fa_time_fwd_vector) / 1000 + + @property + def fa_time_bwd(self): + return (self.fa_time_bwd_cube + self.fa_time_bwd_vector) / 1000 def calculate_other_time(self): self.other_time = max( [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - self.pa_time - self.vec_time - self.conv_time_fwd - self.conv_time_bwd]) - def calculate_vec_time(self): - self.vec_time = self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd \ - - self.conv_time_fwd - self.conv_time_bwd - def calculate_schedule_time(self): self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time - self.communication_not_overlapped) - def update_fa_fwd_info(self, time: float): - self.fa_time_fwd += time - self.fa_num_fwd += 1 - - def update_fa_bwd_info(self, time: float): - self.fa_time_bwd += time - self.fa_num_bwd += 1 - def update_fa_fwd_cube_info(self, time: float): self.fa_time_fwd_cube += time self.fa_num_fwd_cube += 1 @@ -217,22 +218,10 @@ class ProfilingInfo: self.sdma_time_stream += time self.sdma_num_stream += num - def update_pa_info(self, time: float): - self.pa_time += time - self.pa_num += 1 - def update_lccl_info(self, time: float): self.lccl_time += time self.lccl_num += 1 - def update_conv_fwd_info(self, time: float): - self.conv_time_fwd += time - self.conv_num_fwd += 1 - - def update_conv_bwd_info(self, time: float): - self.conv_time_bwd += time - self.conv_num_bwd += 1 - def update_conv_bwd_cube_info(self, time: float): self.conv_time_bwd_cube += time self.conv_num_bwd_cube += 1 @@ -269,18 +258,6 @@ class ProfilingInfo: self.vector_time_notrans += time self.vector_num_notrans += 1 - def update_sdma_info(self, time: float, num: int = 1): - self.sdma_time += time - self.sdma_num += num - - def update_cube_info(self, time: float): - self.cube_time += time - self.cube_num += 1 - - def update_vec_info(self, time: float): - self.vec_time += time - self.vec_num += 1 - def update_other_cube_info(self, time: float): self.other_cube_time += time self.other_cube_num += 1 @@ -313,4 +290,30 @@ class ProfilingInfo: self.RDMA_bandwidth = bandwidth def set_SDMA_bandwidth(self, bandwidth: float): - self.SDMA_bandwidth = bandwidth \ No newline at end of file + self.SDMA_bandwidth = bandwidth + + def trans_time_to_s(self): + # 新指标单位为ms + self.fa_time_fwd_cube /= 10 ** 3 + self.fa_time_bwd_cube /= 10 ** 3 + self.fa_time_fwd_vector /= 10 ** 3 + self.fa_time_bwd_vector /= 10 ** 3 + self.conv_time_fwd_cube /= 10 ** 3 + self.conv_time_bwd_cube /= 10 ** 3 + self.conv_time_fwd_vector /= 10 ** 3 + self.conv_time_bwd_vector /= 10 ** 3 + self.matmul_time_cube /= 10 ** 3 + self.matmul_time_vector /= 10 ** 3 + self.vector_time_trans /= 10 ** 3 + self.vector_time_notrans /= 10 ** 3 + self.sdma_time_tensor_move /= 10 ** 3 + self.sdma_time_stream /= 10 ** 3 + self.page_attention_time /= 10 ** 3 + self.other_cube_time /= 10 ** 3 + self.other_time = self.other_time / 10 ** 6 + self.compute_time = self.compute_time / 10 ** 6 + self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 + self.wait_time = self.wait_time / 10 ** 6 + self.e2e_time = self.e2e_time / 10 ** 6 + self.scheduling_time = self.scheduling_time / 10 ** 6 + self.lccl_time = self.lccl_time / 10 ** 6 diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 0aeeba83e..07943ba73 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -76,7 +76,6 @@ class GPUProfilingParser(BaseProfilingParser): min_ts = min(event.start_time, min_ts) max_ts = max(event.end_time, max_ts) if event.stream == self._compute_stream_id and self.__is_sdma_time(event.name): - self._result_data.overall_metrics.update_sdma_info(event.dur) self._result_data.overall_metrics.update_sdma_stream_info(event.dur) continue if not event.is_kernel_cat(): @@ -84,7 +83,6 @@ class GPUProfilingParser(BaseProfilingParser): self.__add_marks(event) if event.is_nccl_name(): continue - self.__add_compute_time(event, aten_events, flow_dict_new) self.categorize_computing_performance_data(event, flow_dict_new) self._aten_events = None self._result_data.overall_metrics.set_e2e_time(float(max_ts - min_ts)) @@ -104,23 +102,6 @@ class GPUProfilingParser(BaseProfilingParser): for timestep in range(int(event.start_time + 1), int(event.end_time + 1)): self._marks[str(timestep)] += -100 # mark this timestep in compute stream - def __add_compute_time(self, event: TraceEventBean, aten_events: list, flow_dict_new: dict): - if self.__is_flash_attention(event.name): - if event.is_backward(): - self._result_data.overall_metrics.update_fa_bwd_info(event.dur) - else: - self._result_data.overall_metrics.update_fa_fwd_info(event.dur) - elif any(cube_mark in event.lower_name for cube_mark in self.CUBE_MARK): - is_conv = self.__check_is_conv(event, aten_events, flow_dict_new) - if is_conv == "conv_fwd": - self._result_data.overall_metrics.update_conv_fwd_info(event.dur) - elif is_conv == "conv_bwd": - self._result_data.overall_metrics.update_conv_bwd_info(event.dur) - else: - self._result_data.overall_metrics.update_cube_info(event.dur) - else: - self._result_data.overall_metrics.update_vec_info(event.dur) - def __check_is_conv(self, event: TraceEventBean, aten_events: list, flow_dict_new: dict) -> str: flow_start_time = flow_dict_new.get(event.start_time) if not flow_start_time: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 58f44f7f8..29e9fea8d 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -301,28 +301,6 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.update_lccl_info(event.dur) def __parse_kernel_csv(self): - def __screen_data(kernel: KernelDetailsBean): - if kernel.is_flash_attention(): - if kernel.is_fa_bwd(): - self._result_data.overall_metrics.update_fa_bwd_info(kernel.duration) - else: - self._result_data.overall_metrics.update_fa_fwd_info(kernel.duration) - elif kernel.is_conv(): - if kernel.is_conv_bwd(): - self._result_data.overall_metrics.update_conv_bwd_info(kernel.duration) - else: - self._result_data.overall_metrics.update_conv_fwd_info(kernel.duration) - elif kernel.is_matmul(): - self._result_data.overall_metrics.update_cube_info(kernel.duration) - elif kernel.is_sdma(): - self._result_data.overall_metrics.update_sdma_info(kernel.duration) - elif kernel.is_page_attention(): - self._result_data.overall_metrics.update_pa_info(kernel.duration) - elif kernel.is_vector(): - self._result_data.overall_metrics.update_vec_info(kernel.duration) - else: - self._result_data.overall_metrics.update_cube_info(kernel.duration) - try: kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) except Exception: @@ -336,7 +314,6 @@ class NPUProfilingParser(BaseProfilingParser): for kernel in kernel_details: if kernel.is_invalid(): continue - __screen_data(kernel) self.categorize_computing_performance_data(kernel, flow_dict_new) def __parse_mem_csv(self): @@ -383,5 +360,4 @@ class NPUProfilingParser(BaseProfilingParser): compute_stream = event_wait_stream & ai_core_stream if event_wait_stream else ai_core_stream for stream in compute_stream: dur_list = sdma_dict.get(stream, []) - self._result_data.overall_metrics.update_sdma_info(sum(dur_list), len(dur_list)) self._result_data.overall_metrics.update_sdma_stream_info(sum(dur_list), len(dur_list)) diff --git a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py index dc85b0af0..59525f18f 100644 --- a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py +++ b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py @@ -4,28 +4,6 @@ from compare_backend.compare_bean.profiling_info import ProfilingInfo class TestProfilingInfo(unittest.TestCase): - def test_calculate_other_time(self): - info = ProfilingInfo("NPU") - info.compute_time = 10 - info.cube_time = 1 - info.fa_time_fwd = 2 - info.fa_time_bwd = 2 - info.vec_time = 3 - info.calculate_other_time() - self.assertEqual(info.other_time, 2) - info.vec_time = 7 - info.calculate_other_time() - self.assertEqual(info.other_time, 0) - - def test_calculate_vec_time(self): - info = ProfilingInfo("NPU") - info.compute_time = 10 - info.cube_time = 1 - info.fa_time_fwd = 2 - info.fa_time_bwd = 2 - info.calculate_vec_time() - self.assertEqual(info.vec_time, 5) - def test_calculate_schedule_time(self): info = ProfilingInfo("NPU") info.e2e_time = 10 @@ -36,41 +14,50 @@ class TestProfilingInfo(unittest.TestCase): def test_update_fa_fwd_info(self): info = ProfilingInfo("NPU") - info.update_fa_fwd_info(5) - info.update_fa_fwd_info(5) - self.assertEqual(info.fa_time_fwd, 10) + info.fa_time_fwd_cube = 5 + info.fa_time_fwd_vector = 5 + info.fa_num_fwd_cube = 1 + info.fa_num_fwd_vector = 1 + self.assertEqual(info.fa_time_fwd, 0.01) self.assertEqual(info.fa_num_fwd, 2) def test_update_fa_bwd_info(self): info = ProfilingInfo("NPU") - info.update_fa_bwd_info(5) - info.update_fa_bwd_info(5) - self.assertEqual(info.fa_time_bwd, 10) + info.fa_time_bwd_cube = 5 + info.fa_time_bwd_vector = 5 + info.fa_num_bwd_cube = 1 + info.fa_num_bwd_vector = 1 + self.assertEqual(info.fa_time_bwd, 0.01) self.assertEqual(info.fa_num_bwd, 2) def test_update_sdma_info(self): info = ProfilingInfo("NPU") - info.update_sdma_info(5) - self.assertEqual(info.sdma_time, 5) - self.assertEqual(info.sdma_num, 1) - info.update_sdma_info(5, 5) - self.assertEqual(info.sdma_time, 10) - self.assertEqual(info.sdma_num, 6) + info.sdma_time_tensor_move = 5 + info.sdma_time_stream = 5 + info.sdma_num_tensor_move = 5 + info.sdma_num_stream = 5 + self.assertEqual(info.sdma_time, 0.01) + self.assertEqual(info.sdma_num, 10) def test_update_cube_info(self): info = ProfilingInfo("NPU") - info.update_cube_info(5) - info.update_cube_info(5) - self.assertEqual(info.cube_time, 10) - self.assertEqual(info.cube_num, 2) + info.matmul_time_cube = 1 + info.matmul_time_vector = 1 + info.other_cube_time = 1 + info.matmul_num_cube = 5 + info.matmul_num_vector = 5 + info.other_cube_num = 5 + self.assertEqual(info.cube_time, 0.003) + self.assertEqual(info.cube_num, 15) def test_update_vec_info(self): info = ProfilingInfo("NPU") - info.update_vec_info(5) - info.update_vec_info(5) - self.assertEqual(info.vec_time, 10) - self.assertEqual(info.vec_num, 2) - + info.vector_time_trans = 1 + info.vector_time_notrans = 1 + info.vector_num_trans = 2 + info.vector_num_notrans = 2 + self.assertEqual(info.vec_time, 0.002) + self.assertEqual(info.vec_num, 4) def test_set_compute_time(self): info = ProfilingInfo("NPU") info.update_compute_time(1) diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index d7cb3d058..25293d64a 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -76,16 +76,12 @@ class TestGpuProfilingParser(unittest.TestCase): res._marks = defaultdict(int) res._calculate_performance_time() self.assertEqual(res._result_data.overall_metrics.e2e_time, 98) - self.assertEqual(res._result_data.overall_metrics.sdma_time, 4) + self.assertEqual(res._result_data.overall_metrics.sdma_time, 0.004) self.assertEqual(res._result_data.overall_metrics.sdma_num, 4) - self.assertEqual(res._result_data.overall_metrics.cube_time, 1) + self.assertEqual(res._result_data.overall_metrics.cube_time, 0.001) self.assertEqual(res._result_data.overall_metrics.cube_num, 1) - self.assertEqual(res._result_data.overall_metrics.fa_time_fwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_num_fwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_time_bwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_num_bwd, 2) - self.assertEqual(res._result_data.overall_metrics.vec_time, 2) - self.assertEqual(res._result_data.overall_metrics.vec_num, 2) # cun yi + self.assertEqual(res._result_data.overall_metrics.vec_time, 0.006) + self.assertEqual(res._result_data.overall_metrics.vec_num, 6) # cun yi self.assertEqual(res._result_data.overall_metrics.communication_not_overlapped, 2) self.assertEqual(res._result_data.overall_metrics.compute_time, 7) -- Gitee From 2ea4b74ebf1b1a4c80054a7dd8533bd92ebd7de0 Mon Sep 17 00:00:00 2001 From: zyy Date: Wed, 7 Aug 2024 15:24:51 +0800 Subject: [PATCH 166/791] 86 --- .../compare_bean/profiling_info.py | 64 +++++++++---------- .../compare_backend/utils/constant.py | 2 + 2 files changed, 34 insertions(+), 32 deletions(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index 9454064c0..c639aba5c 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -124,11 +124,12 @@ class ProfilingInfo: @property def cube_time(self): - return (self.matmul_time_cube + self.matmul_time_vector + self.other_cube_time) / 1000 + return ( + self.matmul_time_cube + self.matmul_time_vector + self.other_cube_time) / Constant.MILLISECONDS_TO_SECONDS @property def vec_time(self): - return (self.vector_time_trans + self.vector_time_notrans) / 1000 + return (self.vector_time_trans + self.vector_time_notrans) / Constant.MILLISECONDS_TO_SECONDS @property def cube_num(self): @@ -156,15 +157,15 @@ class ProfilingInfo: @property def pa_time(self): - return self.page_attention_time / 1000 + return self.page_attention_time / Constant.MILLISECONDS_TO_SECONDS @property def conv_time_fwd(self): - return (self.conv_time_fwd_cube + self.conv_time_fwd_vector) / 1000 + return (self.conv_time_fwd_cube + self.conv_time_fwd_vector) / Constant.MILLISECONDS_TO_SECONDS @property def conv_time_bwd(self): - return (self.conv_time_bwd_cube + self.conv_time_bwd_vector) / 1000 + return (self.conv_time_bwd_cube + self.conv_time_bwd_vector) / Constant.MILLISECONDS_TO_SECONDS @property def conv_num_fwd(self): @@ -176,16 +177,15 @@ class ProfilingInfo: @property def sdma_time(self): - return (self.sdma_time_tensor_move + self.sdma_time_stream) / 1000 + return (self.sdma_time_tensor_move + self.sdma_time_stream) / Constant.MILLISECONDS_TO_SECONDS @property def fa_time_fwd(self): - return (self.fa_time_fwd_cube + self.fa_time_fwd_vector) / 1000 + return (self.fa_time_fwd_cube + self.fa_time_fwd_vector) / Constant.MILLISECONDS_TO_SECONDS @property def fa_time_bwd(self): - return (self.fa_time_bwd_cube + self.fa_time_bwd_vector) / 1000 - + return (self.fa_time_bwd_cube + self.fa_time_bwd_vector) / Constant.MILLISECONDS_TO_SECONDS def calculate_other_time(self): self.other_time = max( [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - @@ -294,26 +294,26 @@ class ProfilingInfo: def trans_time_to_s(self): # 新指标单位为ms - self.fa_time_fwd_cube /= 10 ** 3 - self.fa_time_bwd_cube /= 10 ** 3 - self.fa_time_fwd_vector /= 10 ** 3 - self.fa_time_bwd_vector /= 10 ** 3 - self.conv_time_fwd_cube /= 10 ** 3 - self.conv_time_bwd_cube /= 10 ** 3 - self.conv_time_fwd_vector /= 10 ** 3 - self.conv_time_bwd_vector /= 10 ** 3 - self.matmul_time_cube /= 10 ** 3 - self.matmul_time_vector /= 10 ** 3 - self.vector_time_trans /= 10 ** 3 - self.vector_time_notrans /= 10 ** 3 - self.sdma_time_tensor_move /= 10 ** 3 - self.sdma_time_stream /= 10 ** 3 - self.page_attention_time /= 10 ** 3 - self.other_cube_time /= 10 ** 3 - self.other_time = self.other_time / 10 ** 6 - self.compute_time = self.compute_time / 10 ** 6 - self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 - self.wait_time = self.wait_time / 10 ** 6 - self.e2e_time = self.e2e_time / 10 ** 6 - self.scheduling_time = self.scheduling_time / 10 ** 6 - self.lccl_time = self.lccl_time / 10 ** 6 + self.fa_time_fwd_cube /= Constant.MILLISECONDS_TO_SECONDS + self.fa_time_bwd_cube /= Constant.MILLISECONDS_TO_SECONDS + self.fa_time_fwd_vector /= Constant.MILLISECONDS_TO_SECONDS + self.fa_time_bwd_vector /= Constant.MILLISECONDS_TO_SECONDS + self.conv_time_fwd_cube /= Constant.MILLISECONDS_TO_SECONDS + self.conv_time_bwd_cube /= Constant.MILLISECONDS_TO_SECONDS + self.conv_time_fwd_vector /= Constant.MILLISECONDS_TO_SECONDS + self.conv_time_bwd_vector /= Constant.MILLISECONDS_TO_SECONDS + self.matmul_time_cube /= Constant.MILLISECONDS_TO_SECONDS + self.matmul_time_vector /= Constant.MILLISECONDS_TO_SECONDS + self.vector_time_trans /= Constant.MILLISECONDS_TO_SECONDS + self.vector_time_notrans /= Constant.MILLISECONDS_TO_SECONDS + self.sdma_time_tensor_move /= Constant.MILLISECONDS_TO_SECONDS + self.sdma_time_stream /= Constant.MILLISECONDS_TO_SECONDS + self.page_attention_time /= Constant.MILLISECONDS_TO_SECONDS + self.other_cube_time /= Constant.MILLISECONDS_TO_SECONDS + self.other_time /= Constant.MICROSECONDS_TO_SECONDS + self.compute_time /= Constant.MICROSECONDS_TO_SECONDS + self.communication_not_overlapped /= Constant.MICROSECONDS_TO_SECONDS + self.wait_time /= Constant.MICROSECONDS_TO_SECONDS + self.e2e_time /= Constant.MICROSECONDS_TO_SECONDS + self.scheduling_time /= Constant.MICROSECONDS_TO_SECONDS + self.lccl_time /= Constant.MICROSECONDS_TO_SECONDS diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index 80d7d5ee4..724719920 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -16,6 +16,8 @@ class Constant(object): US_TO_MS = 1000 KB_TO_MB = 1024 INVALID_VALUE = -1 + MILLISECONDS_TO_SECONDS = 10 ** 3 + MICROSECONDS_TO_SECONDS = 10 ** 6 # epsilon EPS = 1e-15 -- Gitee From 75409358d6420e9ec63183fda5678b505f936198 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Mon, 5 Aug 2024 11:45:34 +0800 Subject: [PATCH 167/791] =?UTF-8?q?1.=E5=B0=8F=E5=8C=85=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/README.md | 12 +- profiler/advisor/analyzer/base_analyzer.py | 6 +- .../analyzer/communication/packet_analyzer.py | 46 +++++ .../analyzer/communication/packet_checker.py | 148 +++++++++++++++ profiler/advisor/common/analyzer_scopes.py | 15 ++ profiler/advisor/common/constant.py | 15 ++ .../dataset/cluster/hccl_collection.py | 78 ++++++++ .../advisor/dataset/communication/__init__.py | 0 .../communication/communication_dataset.py | 109 +++++++++++ .../html/templates/packet_analysis.html | 23 +++ profiler/advisor/img/cluster_2.png | Bin 0 -> 66543 bytes profiler/advisor/img/communication.png | Bin 0 -> 58862 bytes profiler/advisor/interface/interface.py | 19 +- profiler/advisor/rules/packet.yaml | 14 ++ .../test_packet_advice.py | 175 ++++++++++++++++++ 15 files changed, 656 insertions(+), 4 deletions(-) create mode 100644 profiler/advisor/analyzer/communication/packet_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/packet_checker.py create mode 100644 profiler/advisor/dataset/cluster/hccl_collection.py create mode 100644 profiler/advisor/dataset/communication/__init__.py create mode 100644 profiler/advisor/dataset/communication/communication_dataset.py create mode 100644 profiler/advisor/display/html/templates/packet_analysis.html create mode 100644 profiler/advisor/img/cluster_2.png create mode 100644 profiler/advisor/img/communication.png create mode 100644 profiler/advisor/rules/packet.yaml create mode 100644 profiler/test/ut/advisor/communication_advice/test_packet_advice.py diff --git a/profiler/advisor/README.md b/profiler/advisor/README.md index 770271105..3d0bd2b0c 100644 --- a/profiler/advisor/README.md +++ b/profiler/advisor/README.md @@ -67,12 +67,14 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 | overall | overall_summary | 计算、通信、空闲等维度对性能数据进行拆解 | | cluster | slow_rank | 慢卡识别 | | | slow_link | 慢链路识别 | +| | rdma_retransmission_analysis|RDMA通信重传检测 | | computing | aicpu | AI CPU调优 | | | dynamic_shape_analysis | 识别动态Shape算子 | | | block_dim_analysis | block dim算子调优 | | | operator_no_bound_analysis | operator no bound | | | graph | 融合算子图调优 | | | freq_analysis | AI Core算子降频分析 | +|communication|packet_analysis |通信小包检测 | | scheduling | timeline_fusion_ops | 亲和API替换调优 | | | timeline_op_dispatch | 识别算子下发问题(路径3/路径5) | @@ -126,11 +128,14 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 ![输入图片说明](./img/cluster.png) -cluster模块的分析包含快慢卡和快慢链路分析，仅识别问题，不提供调优建议。 +cluster模块的分析 +1. 包含快慢卡和快慢链路分析，仅识别问题，不提供调优建议。 +2. RDMA重传检测分析，识别发生重传的通信域并提供调优建议。 如下图示例，识别到当前训练任务的通信和下发（free较多说明存在任务下发存在问题）存在问题。 ![cluster_1](./img/cluster_1.png) - +如下图所示，识别到当前训练任务存在RDMA重传问题，并提供调优建议 +![cluster_2](./img/cluster_2.png) overall模块的分析包含当前训练任务慢卡的性能拆解，按照计算、通信和下发三个维度进行耗时的统计，可以基于该分析识别到训练性能瓶颈是计算、通信还是下发问题，同样不提供调优建议。 ![输入图片说明](./img/overall_0.png) @@ -159,6 +164,9 @@ computation模块从device计算性能维度进行分析，能够识别AI CPU、 ![computation_1](./img/computation_1.png) +communication模块从通信维度进行分析，目前支持通信小算子检测。 +![communication](./img/communication.png) + ## 工具使用（Jupyter Notebook方式） Jupyter Notebook使用方式如下： diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index ada1b0bf4..80368e1d6 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -81,7 +81,11 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): for dataset_cls in dataset_cls_list: if dataset_cls and callable(dataset_cls): - dataset = dataset_cls(collection_path=self.collection_path, data=self.dataset_list, **self.kwargs) + try: + dataset = dataset_cls(collection_path=self.collection_path, data=self.dataset_list, **self.kwargs) + except Exception as e: + logger.error(e) + continue key = dataset_cls.get_key() if key not in self.dataset_list: self.dataset_list[key] = [] diff --git a/profiler/advisor/analyzer/communication/packet_analyzer.py b/profiler/advisor/analyzer/communication/packet_analyzer.py new file mode 100644 index 000000000..73e5bc2bc --- /dev/null +++ b/profiler/advisor/analyzer/communication/packet_analyzer.py @@ -0,0 +1,46 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.communication.packet_checker import PacketChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset + +logger = logging.getLogger() + + +class PacketAnalyzer(BaseAnalyzer): + dataset_cls_list = [CommunicationDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = CommunicationDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseAnalyzer.check_data((CommunicationDataset.get_key(),)) + def optimize(self, **kwargs): + add_render_list = kwargs.get("add_render_list", True) + packet_checker = PacketChecker(**kwargs) + packet_checker.check_packet(self.dataset) + if not packet_checker.packet_issues: + return self.result + packet_checker.make_record(self.result) + self.html = packet_checker.make_render(self.html_render, add_render_list) + return self.result diff --git a/profiler/advisor/analyzer/communication/packet_checker.py b/profiler/advisor/analyzer/communication/packet_checker.py new file mode 100644 index 000000000..3d9ac81ff --- /dev/null +++ b/profiler/advisor/analyzer/communication/packet_checker.py @@ -0,0 +1,148 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.utils.utils import convert_to_float + +logger = logging.getLogger() + + +class Statistic: + def __init__(self, min_ratio, min_size, desc, type_): + self.issue = False + self.count = 0 + self.abnormal_count = 0 + self.abnormal_duration = 0 + self.abnormal_ratio = 0 + self.min_ratio = min_ratio + self.min_size = min_size + self.desc = desc + self.type = type_ + + def check_threshold(self): + if self.count and self.abnormal_count: + self.abnormal_ratio = self.abnormal_count / self.count + if self.abnormal_ratio > self.min_ratio: + self.issue = True + return self.issue + + def process(self, hccl_info): + info = dict() + if self.type == "SDMA": + info = hccl_info.sdma_info + elif self.type == "RDMA": + info = hccl_info.rdma_info + if info.get('Transit Size(MB)', 0): + packet_size = info.get('Transit Size(MB)', 0) + if packet_size < self.min_size: + self.abnormal_count += 1 + self.abnormal_duration += info.get('Transit Time(ms)', 0) + self.count += 1 + + def adapt(self, dst_headers: list, src_headers, datas: list): + if not self.issue: + return False + dst_headers.extend(src_headers) + datas.extend([self.count, self.abnormal_count, self.abnormal_ratio, self.abnormal_duration]) + self.desc = self.desc.format( + abnormal_sdma_ratio=f"{round(self.abnormal_ratio, 4):.2%}", + min_sdma_size=self.min_size, + abnormal_sdma_time=round(self.abnormal_duration, 4)) + return True + + +class PacketChecker: + def __init__(self, **kwargs): + self.packet_issues = False + self.desc = "" + self.sdma_desc = "" + self.rdma_desc = "" + self.suggestions = [] + self.min_sdma_size = 0 + self.min_rdma_size = 0 + self.min_sdma_ratio = 0 + self.min_rdma_ratio = 0 + self.step_id = kwargs.get("step") + self.stage = None + self.packet_issues = False + self._init_rule() + self.sdma_statistic = Statistic(self.min_sdma_ratio, self.min_sdma_size, self.sdma_desc, "SDMA") + self.rdma_statistic = Statistic(self.min_rdma_ratio, self.min_rdma_size, self.rdma_desc, "RDMA") + self.small_packet_detail = [] + self.headers = [] + self.sdma_headers = ["SDMA total count", "Small SDMA count", "Small SDMA ratio", "Small SDMA duration(ms)"] + self.rdma_headers = ["RDMA total count", "Small RDMA count", "Small RDMA ratio", "Small RDMA duration(ms)"] + + def check_packet(self, hccl_dataset: CommunicationDataset): + for step_id, hccl_list in hccl_dataset.hccl_dict.items(): + if self.step_id and step_id != self.step_id: + continue + for hccl_info in hccl_list: + self.sdma_statistic.process(hccl_info) + self.rdma_statistic.process(hccl_info) + self.sdma_statistic.check_threshold() + self.rdma_statistic.check_threshold() + if self.sdma_statistic.adapt(self.headers, self.sdma_headers, self.small_packet_detail): + self.packet_issues = True + self.desc += self.sdma_statistic.desc + if self.rdma_statistic.adapt(self.headers, self.rdma_headers, self.small_packet_detail): + self.packet_issues = True + self.desc += self.rdma_statistic.desc + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("Packet analysis", self.desc, self.suggestions) + result.add(OptimizeRecord(optimization_item)) + + sub_table_name = "Packet Analysis" if not self.stage else f"Stage-{self.stage}: Packet Analysis" + result.add_detail(sub_table_name, headers=self.headers) + result.add_detail(sub_table_name, detail=self.small_packet_detail) + + def make_render(self, html_render, add_render_list=True): + return html_render.render_template(key="communication", + template_dir="templates", + template_name="packet_analysis.html", + desc=self.desc, + solutions=self.solutions, + headers=self.headers, + data=self.small_packet_detail + ) + + def _init_rule(self): + syncbn_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "packet.yaml" + ) + + syncbn_rule = FileManager.read_yaml_file(syncbn_rule_path) + self.desc = syncbn_rule.get("problem") + self.sdma_desc = syncbn_rule.get("sdma_problem") + self.rdma_desc = syncbn_rule.get("rdma_problem") + self.min_sdma_size = convert_to_float(syncbn_rule.get("min_sdma_size")) + self.min_rdma_size = convert_to_float(syncbn_rule.get("min_rdma_size")) + self.min_sdma_ratio = convert_to_float(syncbn_rule.get("min_sdma_ratio")) + self.min_rdma_ratio = convert_to_float(syncbn_rule.get("min_rdma_ratio")) + + self.solutions = syncbn_rule.get("solutions") + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 52e3e0755..3876c0bac 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. class SupportedScopes: # used for specify fourth-level commands and define the key of the result dict @@ -6,6 +20,7 @@ class SupportedScopes: GRAPH = "graph" SLOW_RANK = "slow_rank" SLOW_LINK = "slow_link" + PACKET = "packet_analysis" OVER_ALL = "over_all" DYNAMIC_SHAPE_ANALYSIS = "dynamic_shape_analysis" AICPU_ANALYSIS = "aicpu_analysis" diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index 87245a43e..1399ca32c 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -123,6 +123,20 @@ MAX_RETRIES = 3 TIMEOUT = 3 ADVISOR_RULE_PATH = "ADVISOR_RULE_PATH" +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. CLOUD_RULE_PATH = "rules/cloud/" DEFAULT_RULE_PATH = "./rules/" @@ -137,6 +151,7 @@ CLUSTER_ANALYSIS_OUTPUT = "cluster_analysis_output" KERNEL_DETAILS_CSV = "kernel_details.csv" CLUSTER_STEP_TIME_CSV = "cluster_step_trace_time.csv" CLUSTER_COMM_JSON = "cluster_communication.json" +COMMUNICATION_JSON = "communication.json" BOTTLENECK = "bottleneck" DATA = "data" diff --git a/profiler/advisor/dataset/cluster/hccl_collection.py b/profiler/advisor/dataset/cluster/hccl_collection.py new file mode 100644 index 000000000..a9fa536ef --- /dev/null +++ b/profiler/advisor/dataset/cluster/hccl_collection.py @@ -0,0 +1,78 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +hccl info +""" +import logging + +logger = logging.getLogger() + + +class HcclInfo(): + def __init__(self, group: str, step: str, rank: str, op: str, rank_dict: dict) -> None: + self._group = group + self._step = step + self._rank = rank + self._name = op.split("@")[0] + self._elapse_time = self.get_elapse_time(rank_dict, "Elapse Time(ms)") + self._sdma_info = self.get_communication_info(rank_dict, "SDMA") + self._rdma_info = self.get_communication_info(rank_dict, "RDMA") + + @property + def group(self): + return self._group + + @property + def step(self): + return self._step + + @property + def rank(self): + return self._rank + + @property + def name(self): + return self._name + + @property + def rdma_info(self): + return self._rdma_info + + @property + def sdma_info(self): + return self._sdma_info + + @property + def elapse_time(self): + return self._elapse_time + + @staticmethod + def get_communication_info(rank_dict: dict, name: str): + communication_bandwidth_info = rank_dict.get('Communication Bandwidth Info', dict()) + return communication_bandwidth_info.get(name, dict()) + + @staticmethod + def get_elapse_time(rank_dict: dict, name: str): + communication_time_info = rank_dict.get('Communication Time Info', dict()) + return communication_time_info.get(name, "") + + def get_rdma_transmit_time(self): + return self.rdma_info.get('Transit Time(ms)', 0) + + def get_rdma_transit_size(self): + return self.rdma_info.get('Transit Size(MB)', 0) + + def get_rdma_bandwidth(self): + return self.rdma_info.get('Bandwidth(GB/s)', 0) diff --git a/profiler/advisor/dataset/communication/__init__.py b/profiler/advisor/dataset/communication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/dataset/communication/communication_dataset.py b/profiler/advisor/dataset/communication/communication_dataset.py new file mode 100644 index 000000000..6cfc87083 --- /dev/null +++ b/profiler/advisor/dataset/communication/communication_dataset.py @@ -0,0 +1,109 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from collections import defaultdict +from profiler.advisor.utils.utils import singleton +from profiler.advisor.common import constant as const +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo +from profiler.advisor.utils.utils import CheckPathAccess + +logger = logging.getLogger() + + +@singleton +class CommunicationDataset: + RANK = "rank" + + def __init__(self, collection_path, data: dict, **kwargs) -> None: + self.timeline_dir = collection_path + self.timeline_data_list = self.get_file_path_from_directory(self.timeline_dir, + lambda file: file.endswith(const.COMMUNICATION_JSON)) + self.hccl_dict = defaultdict(list) + self.step = kwargs.get("step") + if self.parse(): + key = self.get_key() + if key not in data: + data[key] = [] + data[key].append(self) + + @staticmethod + def load_json_data(json_path): + if not os.path.exists(json_path): + msg = "[ERROR] cluster_communication.json doesn't exist, terminate analysis." + raise RuntimeError(msg) + data = FileManager.read_json_file(json_path) + return data + + @staticmethod + @CheckPathAccess + def get_file_path_from_directory(path, check_func): + """ + get file from directory + """ + file_list = [] + + if not path: + return file_list + + if not os.path.isdir(path): + logger.warning("Expected existed directory, but got %s", path) + + for root, _, files in os.walk(path): + if root.endswith("cluster_analysis_output"): + continue + for filename in files: + filepath = os.path.join(root, filename) + if check_func(filename): + file_list.append(filepath) + return file_list + + @classmethod + def get_key(cls): + """ + get key of dataset + :return: key + """ + return cls.__module__.rsplit('.', maxsplit=1)[-1] + + def parse(self): + if len(self.timeline_data_list) == 0: + logger.warning("Please ensure communication.json in %s, skip timeline analysis.", self.timeline_dir) + return False + + if len(self.timeline_data_list) > 1: + logger.warning("Found multiple communication.json in %s, load the file of device 0 for analysis .", + self.timeline_dir) + + json_data = self.load_json_data(sorted(self.timeline_data_list)[0]) + self.process(json_data) + return True + + def process(self, communication_json: dict): + for step, step_dict in communication_json.items(): + for group, group_dict in step_dict.items(): + for op, op_dict in group_dict.items(): + self.process_hccl_info(group, step, op, op_dict) + + def process_hccl_info(self, group, step, op, op_dict): + try: + hccl_info = HcclInfo(group, step, "None", op, op_dict) + if self.hccl_dict.get(step) is None: + self.hccl_dict.setdefault(step, list()) + self.hccl_dict[step].append(hccl_info) + except ValueError as e: + msg = "[ERROR] Cluster_communication.json has invalid structure." + raise ValueError(msg) from e diff --git a/profiler/advisor/display/html/templates/packet_analysis.html b/profiler/advisor/display/html/templates/packet_analysis.html new file mode 100644 index 000000000..07189a926 --- /dev/null +++ b/profiler/advisor/display/html/templates/packet_analysis.html @@ -0,0 +1,23 @@ +
+

Packet Analysis

+
+ {{ desc }} + + + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+ +
+
diff --git a/profiler/advisor/img/cluster_2.png b/profiler/advisor/img/cluster_2.png new file mode 100644 index 0000000000000000000000000000000000000000..275b8709ef268d2348a6eb0da88ee262fd69372a GIT binary patch literal 66543 zcmdSBcUY6z6EKQDSJ_p{?z)N*0a1~qD^+S#7Ll&BNR3GE(n|=Su3)7@3=kqpks3OL z08vp|fY3t+g%BWw0D(XfLT+$%cYpV}f86gr-}ijqJg=3aRpyXwLH`RU#bn;;$@fsgy21L-mX!aO`@ zcntoyY8gJhG6l(ox!zaX6O0G&d#vZihbD)@ZF#58ScGdOow}22XPot+;E%_df_b@s z6Hh%)m7g$tB5D1>d_C6VetewVr(Z1}C~GCK%nJ(RRI}NijwSw<)cfP^-pj$TJC~*6 z71pA$jx^g<>UIl?#dO>bnE_9bY0c%K!fJk#C%LHYx7f3Nj>rD$^6-4RmUQ4-|Myc5 z_`Y?o^#APg{fVb>XAXYr-&EGv+Y`hR_Nm`*S8jQ16GF(UUi-xFw^t``+y3XTF3O7fH&7VXoMIs zfKeA+AL2U>6>gi=3+P$0LsZ~E8ZTVk-^t5b(!jG(WI*N|HX3q$Ga$sXth`P-k=;Jo z4K_y*9LqX0&l|wfL_$5f64@?s93wqu=uA(#tp%bh2Ao;-Oh3=xj*2Sn{?m!z7BIK) z*wc)-+q%6HbobrIcqLbl`Y)JwGO&emF_ht_2;Bi|igiy`QVhH)uSi0PT@>J3&v5h@ zHcDr#pG$OK4^k+MbaiNb=72Nb>yIwl*(_3Lj8+9N%>r)vK&vjV`g28M>ML_c(IaJs zTO}pt?B#)OPU2I0C1MKS%B>%^Y{3XjTl6KR4560WFe_{XxUp%*^-Tq;wChU@CsH2r zN+Tqz8GS(oC2!JGVx+|BEt2ANe~e;z>kc@B1exXF)McazuHL`cc&qUjcyuLhGhukx zZK>2dXiO01)^iD#cNZI}LvZPxrzvhdQOUyAH(nSv8*y1q4N|HF{i2Q8GT*eNF_>d%5Rp_zwOz!;Qg@=u!E}H?jEO9drkR<)VHkT`(V^?LW zl{jj8H^;U@4q?lw=~Y$o{G8wJzPhqi`pho=WFs9v+8IVlMhel_*NZOa<&`a2Syn|x1Nk5h-$^ayb7o>G8}fEKHNz;9{&XgImsS-;%BpwW+fb3B zOsgH#2d5dCd7io`n@#*OYMiYp@z7eRZeJ}NlPl-i^S&g>%lXnD+9`(h7Y~IjXKZj( zv`8@_!OXB3roQdg%F1NajI#~XP}fe<0Om+sikTz@jR5Qz#$_X}{xA7ot*Ft0NFrhR z8=aY@2D({^TIAj5m%W6s=Nz#YRE(sRf_j@|qw6ksr{sCtbO9rM;*fg`%hN3{<_MPV zds9kE-g{pxQKd%vipGK zALsS!STUPI^vC)OmZKNE=h%{xc4=FBukbp%Wi!4X+zR zq)B2e22T!Q_e3wQe>Mi}6k7Y*jyL=31oi3xXn5k%KyFcw`}{(DhVo{6*+>{o!@nfK zZqS--xAjoZ$fCI^0RE@xl=*aFjOpI)38s@Bi9KA)fE_FMrP*sLA8o^gJFX0|yi4j$ zt%@`2r33e1@p+~94AM-s6=!iEmlY%T`3#V-qwOw>kgB?H*kekSpWXNx68cumZz*SW z!26DLuZEI4zr5KL-WF(t)Im{CAZ3Qly1NnEs&z4hD!$Viji1qULeE5)5n&wx6;lz# zL>Wj^a}fGS2>SDwWuGK(6q5!}Z&;6x6oLlEkJsK{&KJX^mB@Q&n~l36Ue;}XqAs`%9I~js z7ec0$+O#OssP>r*!O8lm5F7^%$Mrg-d?IJ+fUUd+C4BJ**9&FYpCi!Gj!s!auERz_ zN{khfxe{fBk26?f-0)ga#l#_3CVR2JlS^X>XZAes$3(?K18YwI7%I*Yr56sMDE*0)d)f+cxrQ_7fs9&0z zYQ39fhOON`*br1_EXtT4s%aMNm0in(CY5?5K@H>Fl8#_Dg{y57>T+<1X}by)H4A^y zBq^|ts`e4#B3+y7lXT1LD|O!LK-dd(vDHMGP0PhfVTzO~YGhHOCh1M@<;&oAHQa8;?4TRFUcVugrR*?-{oMU4y$cDGJa)+>E9P zez(Z1x96cQsyFSYgM)8$xpvsLvgQXKo{GU?If!c(w$5>D(KU0mBy2n>U#Zy^M8vA(HBltT1*3i`R0J+eX?0P5!32(Pvjayz>A$`(p z!Pb>+RV2qS-Z>C zex7cQGm5Y5B6nUIqkK-;W6vIM&I~Q>s^NvXLE>z?u1_Z+<-(=8J1$KtaQ&S!C#eOZ zSD>|JDGCqwc|YCm_GPl|YPxmxr6Jd)=G3DPcwLKLhC_7GN27D}{e))}*N~arEqfpO<$kv0mtKjTet4*a*8_X{BMM6aCoYpLwiWN&3M~w zhpB2M8bYrJqy@z{iUXMux>>uWR+tgjJ$7P#-By1Y%BZjfph+p@Cs0XYq^?}o9*A{M z_?n2oW=qbDHn@V3?5N(*S~rnfde3$n1S<^H>~&C$>N){reIdL1nR8myW2*Nw&kiEJ zF1HM)@KbLt1N$_WWSkUUbgVBUiO)wTG$> z#dB&jC02S{%r*d4iS*gs6=}@lNEH^w&=QQyEhgv@g^9A^l$S;uH=IT4F6(hJ#KJKP z5$gi0D9S*{jowpsbz`*L<$h22#pP=C^=gvqK&gb>AcuO_`$SI<;?Vops$9@~&I0R8 zisfdpQP0M>?{8HXgI0^A7L|^pH*(s}TqRQAmuqB0aNbqJ=F`OO{$e4adfBl{;WrMe zngsOc8g<2kGoKxXxz0+>1rcj7L4!*~mI>0HPDc0ms@z#Kkn(3vll$Zj$~T;U1KH|H zs+R9en^^747o&c`BDqr5^EGmyj(+Jx_N+^0_ib!HF(U>EO${##P3I#|!<@4Cu8(MQ zTo#n6N*7_#kYJ{&KwujERLV2?(<2))BkE>;?e8`#o@=^G8h2vd0;EYb*`-tvNA6g> zYPt7xuJV+CaE5n_(%BxGlq?e7MXJ*d)ZTq?X=x@zlYBA!Zu#ly;9k+9%f7cu>LJ-d zdTnBabNM1E!4%|Ei=&6zMry5K@_=wx+@1BEH@0}idsjX`c!;&lA=JNln%|1ee3m($ zUn6^*!|KjZHEdG^{;?(f6m(Z z0CtOTan5ROD=AUrjQj+!T7R;O`-rIuhV^>fg29kwJ`oo-Po~O9@)_9Bptq8rz!E#_ zPV~Qp5Yd*ZGxgev^bI4$Jdm@v1J2^o=cmu7u8V7Ku@;}8i=t}ph2HloYFJ*ir`0+E z0>f<@Da*B{PixHN_d20Awl6o9`_!DNXcbV7Me>1om1rc_0rB}7{*JmD3W}vt%}zFq zy&5COgri;ODx&4ELp~a-Nak_l`;Kr}y-EedI6`0waPSXYu_89&O^mdTbsfWfSxxy| zChXW-oSCC--HU69CXDzxY#=*vd7`mdqb@pT_H$WGKTD0b#XlbCHnlu%sZiYgyabO^ z6CGc7TjrRF`Ka8cDmRuyR~hiWUR z_@@{RLA?6O%!Z6rkm=EohZ%SV4 zfwCE3u36Vn@&c+X#!K*qogGTG!;^#Z^J*K zMpmaKG@rgT6WtX&RNc{}s40{~f3iz^i&NUlWfYAcGlFyXcnBh4Kja#P8)9IEAR8M~ zQ5}ur?HRS|MjtYPC!YV%tfBJ8^krlEuVnM>5mnPh1U6?T!>U&0kR4IVJLt@of8dq( zS3py$?Tjw-5h(9CihTexAM=Upnw(D$!*`b!7FT&qVY~|+wL@!a33+vQ`l?DN1mYB| z2?hD>mhQ`Mk5vteReeyOsw;%s;pr}_A?g%_NN*_oPveoUT=*;Iu@ zr!~aB{a`4CL9ilb_)leIXrh!4UT}ob1ZTQtX-umk<`@)BZRwJ=Z*fc1CWrRWz91&S z)GOO9MVG0|mf5f&cCLmI*gmO!?~US7u16GFjWS2JvuOi1K$@IB!ts2e{Y806^N3((G)w`5X^2 zMzHl_1Sm&(RI3z3OZmkoeyg^qY57C9{K-XG zZNDgaCaKK$Q$@M7Qk%m|O!S~K({?{DHU3l>zmOMWpX2FVusUr6pdZK=`yC%EnCAYj z?~;Lo?+Fg;6Z~Rf!MbB6T+&elv7s~WiJQ@%Qk>6>x1d)~Y&6@E3cRX`t|NKetqZ-@ z4+n%ZDCmWGQIM!}u`YbeGv9utPR-sLm&YPHyg58kyK#3Gp_^&S)Q6 zHdmSZVl2ON91{7W!vtiR*KQrn2D3(MGJe)AbXHoAw=iG7-=+o{sT;2+%!JXJJp5KA z;LImK@niqQTVM0Y-wzw-6r)TBTSAKVT8s0{$WSBA<}fmeXVSt1rWu{ER5bs?e_cZ@lt0Gq9N7!B7>a@{-jth#3Sxwv=L8gtA>BRq-; zTa|=X>DF%oy$wax-^kYvAEn_Uj#{8S0*XN=Cb!AOK?E#lL1{2PUw_}buOV=~`**7& zCEKwXFGv|R>qkel=c$nVd^aNt1}?KFv;-?w6--p=jc9q+hzFK+t zbkn!Ub-^qrt!rFvrKw!H-X3aaJ1vj1(PjG4PKPAQ8xLO)L#=-L#4xaBT1Jf3@y^bS z;|MSmBFv^mv1E2{IoRidDFD`js0$q`mW4JJyH?-*!|3s6M6ucQxLA%*ezQfiF>c`t zYt?8tBjcsbtyI_%BwbMYS8LNZj4-T9kvF-8p9aIZy|n83;xc4?1#X$AOgqCeRt0hq*t-uboS9F>5bTS-oM%#QN=+d=T^?=!wGd|36LayxRhxj|! zP!0&qg*`27_zwL!u>4HVx}ophJn#iqolU;_NEnBuWHJ2O!U5bPMz5M5ZeYCaeL>^6 zY|H1S=Z!vIZg{)QOsfzK>h2V8{+Cp_hcZ@bQA(*KEB7Aec`~_bc^Hje85om$NNDag z*26Y?YODZBWgix}F$L*}yUT3{7?7GNld~r#ma&jg+$@*s=a`aV)NbEa{^-tK|{U01XNDea@+fE6>L%W zmRhebV3QKyc37;A;pDLA{@EU2WWmP7H4Y6^pxZH1MuUn@8z~R&uU0*O8Q7D6uet@A z@v4aGd1hoQCv%HKs1q*y4BCB>1ChXuDAy60P)0<*+`~8lPc$zK-wV1hnepk-Hjddi z)S|2jbGTUMneg!Z!JtK8-7Wh$StXh>x$ad6{>T}kLJ{Gw~LF2G>KXFPOb z%fUPg00L(*aKcAz91!XthwQOh%nr-L!M9ipnPpe5qd=N=Xg1K?>WDp-=({y=K!l#C zV^d#z!HCq;JA~`)m~Xd>#~WhO-Pe9%9~tkg`jx9X3$k)tH)ka}pB=P3+5!*KZM_4U zG72Mp7+f89DJ-7cn4g>+@HSq3sjfl9xSqH#54ao}b$EAUT-FhGUZp5|_micmpZ84F z^OoR`e#HdLT5by<#U(nW;6Qt#gGI6^*;|prEKz)OFNr!lUOHw4+kLAPv=opIA8JUu z(v$rYV=}vLn{sKRJA0^kxPK{TxnUEbq+(@IRW2Ro5NxYTzEm&Yoo$Cxq$gmMm`P!l zxJ_+xqj9O|aMk?##^Onp_2)gk!^>}5+vg`pn|Qjwr?QBNQ7cPf>D2QF zZkrhBP0AkZ0vR zZ_Vs6&S7@gr`f~aes0Q#A9yV|q~D^IkAJ47JiX-#S9(*aFNf;RyOj>FnP^EIarvBf z3%#;l5Zh`6XXcU>S4TyqC`m3MA<|+#OR#)oX-BMT*x*V*M{Q!ZVHlJ%pk$p#8*LIB6tEmgQv5u`96siDj1Y&F>UEa7V5_j@c$YSzS z@8~pP{1Q@rViK)PM551bPb+H>5p}4Yy}}m;pH8?1DhNk7myZp+a%IHwc{{f~XbkbE zfbcIiK7(BTKxOP{us)a(caE8M0248CR^-M!qxVqE<~FK+>KKb(_GL8ol^l6`ydEhe z5}EhqshHdDto%e}kY^V+_#V{H@c&HN1FWYz*PonTQVb8izByqH?NW;DI&SBI8AwBh z81w?&G?^be$brNYIu=XfZ&L;}^X=YQ zT&*{S_EQ_BcJ=2i+N0{w4yIa^5;)t?0*pRuQ8$F=T3dGJ{y|!W zd+=h$0dTWsqA}+DIUUCwzn+8d9n3>V-#vdB?kx(BFgQ#fWn0OxXDjS!YprmdfZ({( zRwG(#L;G2Gb$`QEco3wGPOc&-ha~1&Y|N3{U1r6nPVtIj@>br8tH*3iMa+M;Q!0;; zdMHt3Z;2sG4-{#=@sGT`Z7fPCb{O1G#foLD6M#wj<)<^M&V`#jd+t+7NULdt6j(Upe*EzvmHeg98(ehADaJ9gh3L62 z;!BkWuVxJhTDg3AjtL@m+?OJc-=6-g8@RV6NXnrx{UWU62e>JVaee@6-Gn(e!!cQ= z7m}_Qa<4Hd@Nh#;8{NndqvpUtE+qR4cu&`^<}z%?t>=PK9qD2ADWBujrUbc@cDzf& za$=#Ji2fqkRut1y6dr1D9XSakHXk7mOSr}#-=qXF3ytD^wLpHSVdC-rd-ja=^i~V& zDs90+akLwqa%fw3AfNE=dGnq=dM-P$_S7b|-}NY=P&bhey}`ZI^$TEhGoq2}%Bri) zw*yadRastkzFGZ_dyH6!G=4EFI#W!g*7JR}6rwSI_pLYpN~A>0^EwZN8Qm|0R+`aI zzI>t-lL@(yh+j*Dsexxk@VoJ}>XOhE0QcN<1vt(QnQDBI$+gNnp^v^EC-&ej@V(sO zrpqV=Hg~eP9*`&4$8m1~F|9cqhRmqWI^tPt$@RW@K6O9;dO*8kT_V~n?G9E|2@*va z_H>yP^uFb0s@0VO-Dy8u%9m2`J{MgP7<~Z&jVRDj23p}j`F_y%kiYh7cz_vBRwlwa~c5WVESLK}fbZ*%-ru$^tFJqxn`GB?B7~Ess z8cptAH2vWl{%-k8kZG=%HgxUoB)3=Z3=_8IZEf3F4D=Q_9j z&pGYZSbdV(;%HYw+!rfuyQ2_xFHuypxjQt*q2BLJ!J)q{^E|zuFY&GWfBunsJ9Mel zXR@LH_&4lheLV=}4*es(C_cp|`nWfGu2++`QjK3ItDhcd{YKUuO_6)=4yZ4`eG|B_ z*ss1)(V7WX$hfEb_2?L6;=DHQjUwXa+J3tDZNNJW_j%s=g)G0W+<4KU+O&Z}9~_Unj5Y)~=RoN;&0n>N z3p(@xn8PF#p-}h6D2=2TUkL_G8kDtC7(+3V1X@TrjO0zmf^= zt#SO2CLM@LQtru-&)@FHLVLJ4AnIqLHHFoDJVdi*Qs-z8y`;N$ ze57N7{YBaM+6hoEV5WD)EM3KKC@*kcEf6oz6w}w~@$6RdZRh+PuGHYYC^LB2x`acVRR=Z`hoE5Qy*6D-?qXws6rJS4&5qMeTl> zj^ca<4J;3e!~{g|5YG*uixNOBNz+1zlxYXy1jrmVr-K5UhPzKHe4}9HE{ph_55Wsn z8T3B|OBIy+@zQw%15z+B_-5~3TUb#4t6of#Xfu8%)K4eIBVqT-)yLunX!cy;f3uZX(hxi;et z5ruNU$-?ul$Wy7+zzTMLsHn0$SfkS2u&&OGA%&jZJy*BsC$MZ=f#!}o!M^dm-Y0hI zF7~M6Y4zQmG?l~+g8Qcj0LJ*sWI(`P2$}-~)vt`A%@V}wj;8XpizN%H9tU%-Tg!ss zYAG(1r&^JeheckFtXy{{VnEU$5TAce>F#{f&~Lw4&<2wMmjYeps(#fDveg^>^%nzc zYzhmdt#IyoLbo~?nJnpu^4BS6>Ble3%USz{6t7}0ke;80|MI47ewmlhXz^_UM&&eC z!D97h37ng_;DcMkw7^f#57}g$_Xc-7v_N(wtMPSmb~wef$=8P>RyrLn$6_OWMV0*UUMQz01p7+M=vln(Kg^q2MtPyp> z)DVAuY4`r7`h|#1P?nGdnEgkA&aUA{q3^g%(W}q2{<1XI>`G3=A_$Qc(UgPeyWIyd zh0+QNT~+Dr`H-tnjW7jV^24sc`B#G*`TQl-Ul@f-+X;|rkKxdahZCs+EX!90zB@avqv?a&y0kY1gU2h(QknR=GM8 z+&*KrF88kPP`PxR6xJ2rP>@{d`oy>}mz8NuCZ^WxR zfJZrufw02)#S|_-18(K)j6KkzF%!B?*MJ*)&gK>Nx+Ke}MfSvHH(zVn1~r0wgPH+p zQ)~0>Y3;R!>~J4sFt8#3=0`O{x=dV+%T%k9H)!4EpBiMnobJToHUq!SPCvyO8~7DJ z{{#e5^G{C`?6Gp@&37lAM?l=xMo|kp4^w-maDnrZ*rcN^$}SUndj#P_S09(FZ=_WR zy3`kJw4^`XsB?E&gzUY5YkOO_&iKNJ^XEIqOT&Gyie?9x$cMi!a3wJteWXnL8QRUM z!6swzjTd5Wr@6}5O_QU)?AS*y|Dyi8L8C~rg#xygn5~3OAJoKrH)6lbv!=M{;UhFu z*l`7==M4-8g8D|I{QE{#>GlW`30On<57Hqy-8u1w#f4XS1OxPZMrvKe?O|Pf-oo3q z$l%pf3k9GAhe~HeENs=PFX}!FWa>jysdix6HPMM{>QlT>nQC4M2JR@DVMHL(58(ng zNs)!(I{hk8U|2vwj2~^zT8DLW6-@rSjIW_)hN`nJw%UW8S~s2^t*AvTSAP)-n`fE% z>qmHtb-ghH`J?lQDmG}5t;>t8g~Zp?>5Q>zH&^Dcv(nCN8~n62HfaI{d3)!2+@T1{ zR+%N~y%d@~Hf&cG$o3}@7wjDSSl>ACy9Q;uU9dUyJ-_<4%;3hNw@h_i%j^p#vuQnE z$UFiE+Nc%NN~L5~-0{uXnf^GryHiyvd->I#f=uDyDVYDs}sXe-+Iz)P56=MvwfEMJyg8U<7an|L&V}%u_ zu$e4Zrp`Bs(R(`uCX`LwUP+(idI_q2lP|ZpJN4QV$wselPgJDtuJHmpAk(y*;MGZW zUeWM7hDJt%xw`Q+-UNBdcs_LE(;5cz)W``nwQ?!wd}#H~l)&8L9yNz|C-R{iv}DC*5T-nKokQ?>f~O5<%-x z+wT)8Tjc71);O+bdMl9|(C@duJk#-7e`JD)#(eBSfsTEcrU(rpw|~RHz~C?EZM*xd zpvZRdbswhpjE;zAsBwFONOd%udA>7_PRVlj`wJKk&-jQkeY(W~vrxI6Afj2|z@nG6 zEo`SHKWqy9X zl$4Y~yMPFNRF?0N#;Py>cng%Z&}Mw<0Y9055_0)AX{_(4az`I`RNT+H?FmbLCtXHU zt1q30aBZEz+;hG|;&F5l4VSsEY7zJzJ%d#PsTxIcN^H{@RZ-n=zp_bn!B z#>WFRR;famr7xK;zJt1J4FbUuM1m}< z5X=6t_}4nHuk)-NUtV55l>xhr)e+{#Y3@K$w3tgp>32f|dAJ`4`D59ks(zz=S|iO0 z31L*i)wqJA(%-|FE0uPC2Sd~s5q5vqwA26dsQTgx4mkR^+VK3VJbvX6eG7gv^M7H< zdhpN>kb$a6r473xpUL8eK094;{9LA7%t2HyV^frV)R9%`%tM3AYYsgjgJSQEgMGHv7)G` ztI=D5Cez_cQcKnuy1*!VQ@|xve_=I%2?Jsl`Ir!)6u0D6>@cv$9v0c_ZLE`HjTS zGh!lHOUDwrgpEYICXreQ5)5U^zVBc`PfiVDlHwOSurg!4v|;y8Z!DAo7LM{0&T3l( z5bKqe`%=Op5Z+ab$?+ShdiX*&E8WfrfBYtU@fJWU*du{Fj2>R~mu-+I^@is<%_+@v ztqgKht!S&Ax<|n`hRNQ-3EFNqOogM(Bg3HEB=71jY?C-?a-(|$Me!HwIs{XrCmxJn zC>DznZ_Zlqv8k0c(_)H61TV=HML>OZ1-(|{b>k5=3>B%S96)A?8&tgc=*&$2#$POt zWyhUq?X3#DRgR6a&0*j!pg692UO5iQR3Phi4uCafujs-1`hfq){CjUyHybrvABm2( zGAYg}Hy9oiBrYmM%LqkPxu1%ILJFuqxpw+{r<}*`;Ae;{iFJHpsPHX#+t0cwLnOvb zdB+k_I}6O1iqYxcrB)?wjdhC0iLAjufgJ2qvg8rqc23pumSy&*TmzRzgMKZ^B{k)x?%y8ou1;->w$C>%jcB=94 zpu+qEs$=$CBjqDaWjSi-Rd%-fhDVAeM-k87Hc)3!ZK)QtFUNY=Z&>aL-U63~u=Fb} z5ft5`u{;4e_PzH4hyDl+jrfZ{?^^PkFN^|~Ew;pzPU-SZSp;;Jp;+@wkzZ1^5IunT&Lt+BL+tCin@Q8B$>2hki=~Sy&A&FJ=lA z)2`#J9c>1J%d7bIZs4^E*;t1mjgNg@PqlUyX*QTLnT_#gxYiq~k2h1!wL50t5?kGs zm#I_5PsZpwzkLNU6^-$B!)K~17}?5UWcbT<_-^f1Eexx zb(f73bYl(_C>jm0hH!zj?znp&r_>xXEV5$C9FZjdmEj)t25r+{iES_dYqlw^wf~cRv|y?f%JHz=1V> zkg-rpJGCukKpm7MzP_sI6#c{szwiXCsyeD~)>EXfQhxB(fYL)?i1}4ad52iNxp8N1 z;rxcM@ZmM;-fF{X(9PO()JV;rzlIJlGMRn3?8Ajsd$Y|xa{G~~4&y#>? z@i6;fH_;W~1wQAq>aFMAET21uhR^O9QzUK<~(hSeFf<(>ZEP6bs>{ILjPy*sf(H&rd8 zxlaqCyM0wZ&nL0YeLH)PSP9t8ZR^>ln|`WD4dm1&`m*{6BqGxrur8ElaI&wkX zLy}Mtz4df=Rc+XYjzrBr652Pp6Y7vF+f|!U{$!Yj*uB1TO>i#VRXyR%Q9LEpxGxj`U#-f5V zamXLK_&7WK&Kx#|fwZy4Dq6Nw8(_-+8nq~rKZRx%{I(WCv^AzrZ>-Q1^P&=wPx zC-!|$wVblT{!)Luq)&XzK{i7Y%ZpoR98MSU&WcU>#VAs+#>b3Iqd)T#~*ute;PYbgy_Hahn?M(E2WPO`9h^VblKNxXYZMuGWl>%~NZ-pH}#Y>*)`%qj3iGF2=;zx>? z56XE(?FICJ*GdI0cb~pmT>s57aqZJ9lz5W3tCXk3m^N@oMi}J1ztJYK7;DP&z+Kglv>O+8qvU~qr zcUobOV6ls2h_DTetE$N(ekX$4kq?_GKME;Fdd z)*|Lg!8KX)X6?VM?EW|Oh>=m=xYZ%)=Fg_-)AtI4KbLCg_*v0}6+9<+bHr`*vrG}m zD+gcuek^#9n-UhxP@^LI$_VK{^wRHr{?nEZXO;$i^KQ-P*;9Sfl<6_K&in!<0eWya z|ItI&WW7gv#|Jcd&x_pDZ8n*{`Q(YjqbmEZCI2k$%VJq&10~}j3sWb2S(XEZDDMfZ zm8Yj`w0V}FW`&jSw0}9q!aKt_rAK&k{c~~}(lCdM8epi7Z7>xR58>IV zi-|Iy0B#Bd+F4%mzuv)9Q>rH|CcLR+2an_5jP?a3%PnqSNTirM0d%bDUu=4c`uVpZg}B-G1w0HmG><+n;~5;($;4t3W^^A`{0dK?}_;2>qTYKnCvOU zz?%gHRz4T?J9)CM#G^)!nIALZN)YNCFWmT4TxAnv8MziAl0H;P384TukBbkVuTx=t z94lrHSC0po4isXn>OJZ?un21#T6UmgQl2WiU)=LfpgyvNZDXa_c+E>=meZdkB`y5x z!kdh1J+_TEMPx>b7}-Yp^;*RYyAM%5^Ub!QsoL~%i~rI*0VTyeL&sbX6;?d1-bGX zz1<+PibR}*Tway!Un{e%HkiSTWeHKde7wx|Mh6>!%$GGQO@rQV7j+VFj=>x_VfF$*nEZ8B!j2_HjHMSOdNwY1tWKcg@u)1Fm?z8g1K9b z|DC@f%HE@Yft52*el*4MtvWE$J{5bt>K`)%?~d4ODsS2_v?;SD5{X4%Y;;#==PhgN zjH;?CW_QF!uhKpA;>IpDaB;D&Ea%K&1KsxJ$Oqq#AGv^B9^0qzX4hU2TDkp#$>&d4?ycGMj`acgdjAhP|vD9do$JhQjRZ)2^P>#W{s?q3n?r= zQ)uafArQ-}%V#V8HBq0;S2Pf1lPf;N`T5^9HSNa6#tyvagUY=XhZ(E=QR?C@qoxKz zqtV>g&(h9^xVm#iJL}H(d>$8gl`% z`}DfJT;9Ok)HFduQxkfX?}z=@zQ4D~2LW}DxQkC*SSaQSe${Q+uC4CwKF}{6vzM}-d<~Wcgg$T6O7!!v7$n!JCagU=LGI{^lZsp>KAf%PWsVh&DE#Q-R37M zVN6|}ORPBvBy`~4qZ~g&F9C7YU9sDT*go8Yb|}A+;_u2Y-)m~>hK1Er#lEA({nhRf zE}iA~Slc6)T;==qKHmYjLMQBV+m(OO&J+3UlE?d#qxhL#dO9G%Ky3*sS6gg{ZTT@=N|>*;em4%kWXiKP1iSiNkT+D@DFg*`k|k`r~A00 z7ga#^0gtY~IF+yIhjk#>2fd(@pwNuCQfxL5-}qA?@1@E5d*c5^@AJS<$3Y%FW--Ow zY#vv;aN8)M0ab{o2 z^b<~ti~oxPz|stdJwpscZxUf(I^#~zu&IQaoOg+ln7a8taa~tyL-K2m11qJthtkH*0&xEFMM#1~| zmEdzou3{a1FarO-#CcN$9L>mY+1)S-2@7GgHL|{hZ@Y?g%=X;;&(|8KZ}`K@2bGTL z!o-uVE<>EBzt+`$r|g}?yURy)!O>30|61pyOFw>3ukSxO`)eP;{qEY?@8{ZIN4xO- z*Cbck&it>u-68Rlxoe&EuvxP9NV~nfIzD*uqko;(T$beg^Dtotd2}!PmYC>RQO^Xu zxD~)mwgZinMBeNZ1WuI1p8%o~lFZD8f&TEKYeutX2r6oD3V-@X$d4lc?&e!_4Z!&r zvJo;xXN#;$PBiFmR7BK+O1FXjw+{HOk+jxk!GQoLgukTQ$BSowfBcC*rLlohSN6f{jb3 z!p(L5?Pa*T7@;)x0%z4*S7mAWA`n}7f7s2K3$NG?Ps}vSqpef6>)b_KFtP=KtvBnWS8`EirXF z!Re%rhf5!2*6sXa1wX}>?W?#bR(jHptC4&l{LQ6T7lHM^75i9U70k=jqYIO~dWSpx zi|+PvU6(&|oi|G#*ho0JJ5=(CI>43`V8;c+)(_C)G2f5k1GW?hIF~ zTjYm@z8sGWxs&^}ykux|DZL8PK0`AKCIgKJa?&x*q89IGphhVHV7A(RuBW|2P0^Z)6d2 zUcK7NJXoAXU_HBhI^1nlqVDvK`xp4Pd3PIh{0L&aW7;9bS3ad4$rIB2-BHJ`wlq|x z^Q*A$f8B$NC*?{a{QBvQ6zR!ulp!jR(=mXLI&+^ksHze(Ua%0udb7ts!=*Qk^Y1jf zmAcu4d1ooDfIHW1fMYFg$wabV@rZy*WK4)}{t@3*1rdRGA62{O=YGE#Qt9+_U9*k1 zPTeB>eh!jfc6iepJdi19p5@bPw(Uhb7JA%2bN=Y-ov!0-T^&Nq3c^;aiqa_aBcn*Nnafjaz&HV zZr{TI=Sz2NyDUGd_5wCJDCl7Ae8oxXcC>;7i}jg#+!uuD8=M3w25C-HAlI%VUSS(3 z@m>x6&1wkm{!AjTWjM$9g|Jtuik&&kLPS5zV@^qPc)_6Ewce_8Rs95EER?;FuHmZ_ zK2p(tL~g2~50vniuAaPhq*O0c3ldkXFOWfz=46YNA9deqKL)AsZ6$jNNTC?T4X!x_ z9Q;}6mh>}CL5ZaY772-oZ!7mAC>nqrq2ph*vurOYYuob19bFE&V7VM%CCzL!v3~aX zasp*wA*sngC+2ly^z71s`;5Js7nb^Ge`mLE+1ln%J2#JwYi%1)j!fg9x7RvlhB4rw zD!p8t*Jy6PXi(q#@^#f!RU6uF`>e*ce(fq7J>usrpmEsQ@z0-DpF$vi%Ur{6+=wW# zn#qeiMx4wgBwpT~u{&JiVLqQ2Gw?a0RTE$SI$#eelqS?A(l|RLiOf8?iU}=#id|s0 zQd$dhT9( z*BTy^WS_FJ;fiXqn1S4_dww+Zw`sa=nT@huK8VDd@y6#_Y27uesjZjRQzwYVat}i$ z29K&lyp()=HKnRd`lPzuGqGaJ_2=zd-LDN=_9vO2P>JZ5mJ@Zadkpdlm~zgvusQkj z$1?^Q+^GF|cJU`OUWF&-Z|#sfMXT7NKMpbCxP!W?yHrcfgRFON)(gBqm0QU%pNz;N zimJr-&K#ZzNW%|3RVm@VhpOT=-JS}p{LM^`Yd<FT7B$6gRr> zmBTL=mJx-)@J*WkoFAjagLXt0DUiW=n&-eR%j`z)v9NF1> z&&*!4X4cF)&$YJT5!KsrbW&kgzoe=(#@y|n9aPwlhc{K2R+NghhfR$u$3>R>7V5vI zsk;D|McQ3o44SPwCMq2pL^^K`R#a+5%PTchtYqiiGBJxp$IFqz%P*zq3Nl`QojNyJ z&b&@WuGz|~B^zbAQMZ(pBkgqj$HX)(4$8y~MRopBU-r`im+f0)g+{)VBGqs~!IgZj zX5X&CT=d;?pMHr{u#LWdh@Udx+GvH-#$}-5LO%SH*aySmPBm;}-?hLGqhi!HLia1r+V(z-?ljx8&l=S@s{jOy`=n=BK{PSXfxI+WnHYEx~2E@x#tL z=@?9$(8~(0GgATw)VR3;B$4h-!D`>?xuXXd#Em7tCU-9CE{$HjU~FVr*~S&Bp&nA^ zWVYWfr$fhhWpXED<2oS5;(m%*L4{hv@>r!&qEc;&%aCM*;Uu&7x5sO5nIX4v9wt>4b{5@oZt)nSz?MVlW(f0*4j=8@D zSp&^-#j`2|q|OfxCLpDDxe zDD0AgOeBALggPgWUHOMH@f68-UZM%0{d_Eu4}L3$&;k2$#HCLKRb6k7XY7__*r zED_4p4E#`q3IoD}Q!wxSuQh%uCt}pD#rQ1vLsz3(V6DU`)F`-LzY}4~8dP9uII0i% z@cElkQ)QnxrYqAy*tIVx!nnn$sx%N$)7-z&tAd(fgh4hq?fgSqEq#$9f`K9*N1p%s z#NC(af~OO0z9CE4Et@=QTH*r+;ii>9G1ewcI+}_Qdp7g(FDJ>sJhzEyK3VAMhg+=Q zWuK?CJiFN;(+9fazuWxSsbd(`l#wS_7_rE>6w3=laULv2)~rB3i~6Grr;w{} z1FFVAOsHp|A~}rDvo^eR>h%Rn!`UM8B)L#FC&|ZD-PWoM$NK6{M4U9UNb<`1e;4C7 zljo97!{;7F9#laLp`l5?aB|k7&iY%J%Ep0(4}1JE&jpOXjT)}P)DS1tox9HYe!X54 zFL72t1Cy}Sk$ql4Tql5*5jT;T=OY*L2uhwa=hzAbk^j{akwcA7Ny;xYamTq0OO%4B?14K-S6C|#r7n+&PkHzZoq@$H67P^M#@7+@N8t{$l-S8ewZ)fU71dFX^wQs$HV9E6J7h@~Vi2 z?$fJcv?O|4RWLc_T2hN$bR*4xNWabXvp1@QNfngl=O)YWeYa-mA=91x(MKBRPK!OD z{49DuSm*F8vw|n(!v~KKHJpbg6E$M_a?Uq+PYo;tQ_7RGcY$1sW3DtQR6>uJ9YIqPRCU1cB*HYZiAk|w0=x* zuU31#oQru>y=-kz({w^;_51ZKSV ztg%gu{KH>o3gPL6wh?+9t6P~CwSF<72YWQF`=y$xi~?^3s7sq1XT6N$=P_edQ-zJ* ziEiGA^N)20NlMnxZ>eK%U5^);W|Rh+`<;ny@PMC}YURy=Zz@`6MS%hKjpyM?!I0ds z7og;NfiGP1EFbnt-=IGpoHWV2XOZ?oB{r?ML@AP}|H+HLrS54`U;aJ!5606b5wq;F z;x6Y>rJf9i@sDc)7x*YVVZMd{R5-I z9PjEvJ}7L`k^2tHq*gyUU(!@-G-9gF_Be0huyy!E-9pYs{(B|q7b?awDA5njq1%lGtoZBa^n*_DxbxbpX0GGCaE zofj+D+PqEBfD&xE0)S%BBm~xkZYUJS*H6=B3#srdkFk+ojUMcF`QqE}q7Qqo!Q^hY zt0@&|*KhDe1)kn0;cu6#TY`QGS>lf&93hSj@0$?}pYPlY%GsZ)88Sb^H_^1;Nw@c_ zPi-aNS+JBkpEMgmIki8gxGQ~Nf}1~w0Y(^JQfD5EP)^zUNN?zvAVOt#wb5;WjzgAN z3NBQK)2}plx-2mI4+28Z6-c&%HNB z(LhJ=0`~I28uPUYP_TZCp3u*PgCND>(8r5+2ev1SpW56D`8g&&LC8JdVYx?L;vIg> z3+Q4G6t(=;R08PI^8a<5e>WV zM2dB^P)kITskXeYH-F^*<_q4YtIK1r8(S^a6|<(zbvr9Qzh|k)2bo7Pk`5@U?x_A7 znDV$Tzt&x^Cz+k82}8|2=O^Ierccj57fx-!O(qtD|IzZL5bFG&l59x@Y9rh_Vjm6s zDpxz8NSX~}!B!X@;pseK;#apR@GkdN_R?~HP718?+hl~15C@|PQ{9>4hFQFv!|{}H zD%)70f9_=A5e{N)=kMR)K$x&rLlnF!in=>QC!}#V+k4Xp{q&ewhy$=&@^B*d4gVCf zU*QR*`y@<;-*NXCEw%ly`fxMOq7AO*OiT5cgzQQH0uYP8FbyO`S4Uauw<2Vn=2&nW zAs=WKDc+CvF0>ZaKfzPxL`?$p z*2B|%@I6yW^(sK*vOWgn_r0agW;DFL%xUIFW7E>e$fE(zrzTdrDnfmjsyM8dsB?TN z6P}Lc0cEzJu}natG;T-)_u}k*7!avP+1uS(6bU@yp@<1J+Us2BgCX+g$lk}ct>m9G zO!n`Lw@1HfE!Rx(O01*B_S2tp*vwe+P#tw|Z7AFi7&uV^h(@SN8>e|uU;YzkMK{w~ z^**=ba1*NUoH{MF}Ay8-_5(e$4I$5Oz)81s%7-Z zHKpJre#3X}>L^5w)2woF^&|hOSq>?MLKb{ucYehYf8O?6sS#AA-TEEsE>t7_j@+qN ztX9^x1Z20sP<8goetj9_1FrTX-i75x`r~s0{SF0Gi;2b z&keuv)1uf_ptRyeL69&h(%22_OAcv9TQ;Q=dhaptd68;j@$lDeL&UIWWW8-4hC z5N5>AN}X&KX%Pa_%^#$|FCKTjT*YdwyM^3A_xk|`AgrN~QGJuCchNp_1zbSj7+Z(l zi&?6JQXh}%U$5MsNnC^wFL=eEhr?P5q zOf{^@2?rDljh*hTZc|e4f>ZAHizW)y+8^)nNRFTY9nAO&w*ua>{>G(mWqx}@vGhYP zxhPNLuwfipFDo{V3gyC{4Z8{D8O)jUd@VU>Lb=8{H@=KVlQ#$Pqjg#INAHWv7E=WMu-C;60UTmk%S}opi3v$lg zu&p%#e#QhZngb_Cc_PKPb|GZS58WnQkz|5mr8SrAoW$DjvWwYnBd;%B_*h)+X+->x zAYiBa*CerOfzlznPAE`uesxOkcX=3MQeMD=#~xZ}*z)LA{Bjhq&yA^vsl`fmS0m;+ ziX5H>xBqbo!r2%Kmf8ZfT2Yt94bHvE;H6Bs=V_M*Fz0op6>=+T^f*yP*X)@EOd4Lp zy8$uIhNL6D*69jw3)SD!SVzWOedeW1#L(w|yjEdvkwmvmmK7pKwYeV-kMMP=_n{BI zS^?#j1xl_^b}coRN|4n^v@kJb^XIgv{JWctrYNl`tUquXTN>cA}n zICyBT|KCJ{=dr)BFI-4VUN9BDqk^WR6BFeXQOtY{CoU!w;@$d~_~#i(o)!DwTEH_L zN&g9+K5^4J^x(^d)}0;!N0>!wdU_c(c7uY~I$`qf&@Y`)J z4Sj?-(#lzvee^HT%5Q6s- z(?G8N>;)li#{UfO6NXf{@5G4(>-KwQ>Cd~{&IZ1^5JD2|Z=u~vP+rHLHYEkoM zc-QyDI~xC?^i!`;etcaqxcPse;}bC0kDh%yzW z$_)rKApxMvw#v}jr-B4XP0Qoo1xN;bzqnuj)Mq}`Y(>EKD0*jS=XveZ zl;q^$MJKoBDjGKK@;{;7)KUmDF~UCN<3Q_XkB3xb_6=ff&Dh^8ovL)@{{W*h@dDH= z?0?=7K*gxjQ&UT-sL=Yt+)}hajXo8`N9?3RfB$~>&ECMzud%M4IA%}Z@*JE-vGr^8 zq5|mI*_HkMx*CRNevP1oe>MCBIND!OPEMd#IxUxaP1V|3SnURaC%wwQ_dxZdkGUW< zRw9X;3gUmLuODXlsx0zfJ*2Yscmrsz{1$0)@_Q-@OAT7nqe`$vMkd7l8tok_LxGK& za%sIcxSM1A=fs4@zv&hQU1#XT=H}*_!&$w&Q)kan(fxYSC#S^@#iTIxRj=`@YdJTl zU@KKjF(>}jT4m@KA|rp^-0>6fRcNDKU=$*PjqS?yAHQ;Q3)|W-_qc>@zlq14^q|fzwIQ+X6C*a>$8=ld>SgJhx{A^F~EaminLP&qBh1g3`k3wO( z-_6SC2|jiGU#bExgwhGUR=G)Y=3n9ni%rzI{I83`wErJxnEcb${{!R~o(AgM20UlG z0Nwq=A>jO_l6FrCx9-F3>(ZgQ78*7^Z;#+E>jCH}PVeB{&N%9&Ow#<4+mxRHGJ_k1wr$0Q(-)Z|5Qld$T$?S#bC9FA$k^9u)z}uMnLxm1ngK!e zL|DM3@9*^0g|=*;bai=nUnU42cZlQS97eLQQ^H{%q%*OfuNg6|J}Zi;gzs-a8P{bp zBG`lU5VSt%Vfesa*#ufq?q)KfU1sVyk5MD(dP6#6$bhC+P+&O<#|_dZE}eM1`jQsU zqO-aDjK3#pwg3z8f=qZ6NTYiYle*p^+H5NMQXEW#S43nj@rx62Eu@~3O`tw582)7* zC3Rqk$sPm1DHA7XPED@jkJ@5mTWvSqJlwQFg5HKGg)BF!l#L#4^;wL5l^lMdM3k#! z9NxG?7R%~hXFYLqmila`B2A}msMGCyG}TE^_{n@w6nd;6nX!EThRfM;Tu`4+Z<0|} z4>e)1Lq(^Xa*v$1>mn1CNr0z8CbrBywtiXqhg z)$+DVXuSDB>A@zZjC3Jd{4gwNH*}_fabtV0zBhfPI)vB&!9BWa>y@d?J}KZRVT2Jq z@#*fZbndriaZ-n}ZV@rwq#!qI{To9sE6tnqV%nuG;vWlTZUo+(G%nk$YEs|0899*` z^aiJf;DNh5Hg&5jJ=`weC5l6&dP9i>R*r~UI! z%8YGS&#BJY_dr=@l^|Ph@Fz?|}}nl$5lQKTL~% zqj>4WFE45~oNnKcV1t|m5Mr#dlNi$ceI0#$U^Bez$1Y>@gyd4T!NGJktnjhF;J#{g zmn4JV&z-AwrI#6VX&(gWG!(!bX-eJkp;8`7Cy84jSOp=EPvmxSTh&ZP#bZKz1nJ0W zK*T%8a6}UGMexo{zM-&$5Ka8@Ve2Pp*A*7lgG)HY@ZEbJQ!Ss$nH=C#tXoafV@326 z$Xle@2fVn@_C*$VjkGXtCe7rqfym59vNx4zc008Rsy29GL+oiCQMMdvh?3BUAJn+p zZ_kRf)#URECLAB7PoJyaon8%keSFS{)}bs)qq{NvQt#7dvCAOEH;swgRo3iDk87P6 z&fg2ihOqZKxEvkKzkhF_)yeMU;8HQ{Pv6C+pybey?AhdOuws{lN_ztM_Ec_~j_rDr%^&g0EL+`+bozOEM&Ems$_iX9jr(~Z-=g(m_@1Yidd25&IawlzE!dw$a zy(t*T;9C{VHFdDgWpt%#1TSR$ZOk-~V;7h$r*vE|8sMb7TdEU#tPRZSG}ugDyOX`V zs|qR;D@3-a#>E#%Txe$ex}?NoeC_?Z@F1Wv(@&<;XYSlc`C*Uyd34>Ab(e<5yVF6D zPv$XJZ5vPr(MGQI!^<)_7t$X0Y8)l=GkD$BELoqK9b&7PsQ4tMt{)x>ZG>;9F9r~` zw-;S+qo$jOZ#jlsqp&YKRHUsk4(xt z^Mo^(S=3tbZ!I1NZ5^q%ZWx_B0gq6Qc+QG5uFQSrC0lJjdY^rLohQ;92YMy^;Px-4 z?{dFP1D=X{nwbHtyOzOt)=?Falkegc?TH)oR=wF+NQYFE{I zb>QtX+VfFbVsrG&IyCfUntVIAOLioz;(oW}LqT4RX&rp1%V-}DO85mTt+gP8!l-WN^Xd68K$@g>@i1Ov3wPvsMOh?Vpo>~ zuGn4VyZWg2*($-M>*8YgArFqfBd8yl@ckl%-9l@}?r$`e1BUFMKBRd|lC zttQjwSke|vC=1d-0An)p`E-UKtb`Ec3sA&IrLS0SVSeDh-;S z|8c`selh#i>eD^2QI*;`y`Vf7TSctmli1(7yh+vNFtfs?iJLI=)%?dkP_^ z)-@jNUwBk=lHr0%__nrfKE5jU{km?Ijuji%j4*0l3s8Vjn>_PR`yRv^a>M;1TT5bQQU>$gH(IAiUfW+q6nG1N!S= zJT-ip`OG2Q)@14XK}h(FC3}BJ z4tY|tbXXBS&+u$}VfF2slgqrIEWvy6aJ^0EI1I4Dh_wBa?nDJxoSWeeT=b$S2`^+^ zTNoQKjLWx)gss?G_T?K5+cE68cTioZExU^yg$i^kAaGXkIne1`k<~vO^*q zZN8KwOjnnx=k{or*|xJ* zCYuF<7LI8$5wN@X^-^x}_UmqB7;?J|I9|v0UV;8neQx}FAx1a#PQjpIGdlaU?9-UeIW$Pc zw(F^SFa%d1B&Ww~%X?W9?{efC@PNe%8|?VNDvy0N2ZFn_JG6CTmAbOhn%9B__k7xN zlq_$*&>@%H8_!40&>_wN5@ln_us_}u;xKU``Ek@(d2^3DKf11&z^8$gJKRByn{AD;fPy$? zIi=h%fSJRjwe(Cx{9QxP4Cj^zE?uvf$-c458;Wbg6QV|N%>?ap3*|?Rg5=he9>o}8 zQV*|7l}?26>w>+r0n2)8sA)5WmLz5|L4kIP4d)L!VJ;i949|diacM zl8BFa%srm>`f5!M0kaqY9Uv1=`^#PxJ}W4$k*_koXvIf04NIZ7x2dL~HWF^rHNmCT zD7;b$iI5;d6Bua5u~KslXb5zCU_{4S+`@!wf$&Rjk#zuIILevn?qE0xit*jNv}A%RBs_&*R9D%6S-I*-I=I?RDSI ztl9w0s@-jC;t^c0-!) zQBM(w*y#m`7Y2TzzAcLBbu_ur#C~6>-0KERn=QdP}J?&ynf}8}-oiM@KFnfh_`Ukma<^XH4 z!xk#~Wyp;#OZOj*aT<-dPDFfdPD1lH%17=WHz9--RaU#*VSRsfLHgjc4);2^DAPVE zvd(@o5}Pzt1!SE$beb|YQzLReg3hMg74v1$nJ^^b1lOxexUsqaea*MNiJ1KjXMNpb zR+C*eR9}>i!h+4 zGue^VMu`g!?%wOQZ+xsVk{$33l`cI+r%~}+w|9Xc`+eyeMWfq+*PSiX0<1L~2kg%> z?)9zGE0lfXbZT6$f!#Xvz;D+B=~SpX_5F#JsqD`GubSUJOmo@+QVtHgVPa zQ+=vLj>>oM8i@nqWkmiH5?tGU-HgA{`2&PP%@E+rREPRdIyN2=T;VsM&3}N@*~z+8 z2$bN=?&giufT=HFG9(T#mjkYi)ek5Q`A}7N=_*xqw?5g0i-JX8F4O2y3a3!647ezM z#-}!4P!FSI&!iR7Ty!5hJIv8F&e0}{I~yz~JVwv#6eyP$6}5TGZ)Myu!ROAEcBiL3 zrH$vYi^>zdL5Gt``#M4283Mm&xoBvcn6}?M6ngcbOoU#bhatDxqBk6Lz+mLgm{B*$ z)#BbRDl+I-T#2}AQTi&OOB-9d(HsrC7ohN~R5pZngS~7H)4P{a`(RjB=*QG#S(T24 zqa53N&Qdb^@_Tt6Z~4#rOCY+EUwwMXuseB;&4|zVj~$cYW6IqPaosT7=lS5UHQZX9 zY~tHru!>o01UDEne)y5dP@Uyl`Cf5JClDKmTiPDuOV)Z;TpFikmsMKA5sg)k4^_}y zXw7^{>U$d!8xSuUN5ZHEaGvKiEt}zMD+l$0%kqbUfWA(-pz8y7tV7-fyu!Ai^nx$| zgnZYqv!;aJ@I;e_UH&(b?~4~0B^YJ+Rt^LX4o*2s9MGfFnGCBJ7b6dDjJSewqs2Nj^!DWpArt_@ zHoJn5lICp3w-&TFDNcJrS+D2rxy#Qh$95byTgtH2wgs@O3#C_>aw7P>cq^KV9#??g ztu(nfwm56{Z9cX%{qm9VDfj+nx$i^I*rxXi(kYEkbKM{;9b6fnGG0vaS0dSSESKU> zV}E5k$(yYDoY@>^x1Kb5447cA)OlQ&?G0YMjFT8cu<0gQ`1ip>*NO@xlo}QbboI)0 zwJPW{>MV$5r=e<71@JDGih$QaJ*#dow`xzIEVFN&^Gf!S*wtY9TcNm`VSZCI50Jf9 zpNJRQOJ~z-+qc}ot>p~#tx;v5+_08WlNt2>ZFb2IBc(@yFcuIWa)9( z^Lx0kYVhew<}$Otr4_=s7hzszvY)qa)%ys_Fxaaz19b3uMU*8kKMsQFW-O_Ihs&L) zsUQQXj+wHhmy~T(Fvj)i1I6dLD3wb0v1Su&NEzSeW5JOKG;Lpkaj7A;u|!daxdwkO z2D`|15tiHYiR|=f-THFZ^#Ssb2T-S7LECylr$Ok#TA3<9JsIvVvd8=gU5(eqb@2Fs zhO<5$IW;yDcN9kGrfrbU7@OUB8Oe}Rn^jElN91z0u7V~-Y%fpZo>Ipc?*KbNC(_qt z;2~@U{%Ky|vMlr*5BhM}EmnV1i8Rnru0cp?IK6AHZeU zhE8A_>e@LyPN|+$0ui3lWn+AwM4iEABlER&f>P>MjSUCV?e{VElR0QU30Qn22N*w`H6CZ{81j6AGRMT6J}7mzDhVYSU z&%Nq*?v=>Vv({W~Uj=P-$TQ!O{pWogde0Pgpm*G=cn8xnG(Y69p_f(BGqnh5u~-ys z{X7r0QPcz_T0ZUA;rl}*jEi{EtbH+UTv91mX^86YcsV^Dr2e{n7hZmyqJ6rvoIK^& z6O38Nhkc*Jw7F^z=ebAPTPN7VjNmAoM2!HaZBY7nt&|G_`Xgh)R<@>^y1`CKpHjut zJt3ut5OsI3pPT1U#|1v_)|;-D_SlZcu0c8+wl@~D2WM7$%bJ+JPaK z)}&E7VE=T(ZX)|7KB04eQUTx6Nq(!ObO&Du{HREW?RNssg5Yu2r$K7x zLQ10cuiWPH$Su>#mDl3##n&Qf`?xlhRz!QT#xVI|^-*E{aA))flammTpV0iCxBn8O zmR*9l+KbWF_l>%dEAyE}P5tkhMk@|D!fy%nssmyf>3sPWq!V-^)<--a3wtR0BA$HlPf8*`c>3+U~dd}u4(fPdmyEnIlo+yD7G>@K-Uc!Cy^lzoX zy^>fkpPRP_7bCtIltv@lj2%{{7FmDRq*8ZPM)EJMD;X;vlAHYCt{?3*Uwd7G^wT~z zW*>5RG;yMPa4FmMmX|~TKe~6^Q7|)-XS50BVq&E72$&(t^{7S(whFtKn6=qU{5i{3 zzt;B?yavR}gZ8`JD;;3T<>ir@N3CybdmTn*&t*e;VJ{Mh3=JNn$WQyoWC7=C^m5Z> zw-|WLptv0;@ME0b+wL~=~{TMq}zs$K#U>$eDNIp657O4)< zZkc=t?cQl`;rdQ7E#9uKej2z40f@fLJR6HDEtMVRf{xv;2WD?p3vMpTgRblwXYuH7 zCc8%!*(BIE7)^s!;XiToYtjL$LqfHfm#PVpKb)zqBM#KqGtZ_-zI$y^Fvo4`+KD<( zPJ-Udna4>9iug0J2NIp)g?3{Bjggif^U&Yv=aPP$ifu+i!*>JJ^$A7FHRqYk-5OcZdK&11>X);cCK3MCpRgloHjJg zgD#KP1Ysz0@{Yo}sKp;Z=H&MLCo48Q-$=D7F9t;3YYf|INamGXi&aWDY;1U%vZ|s< z2y%jYmSQqS@ECig3o*uzgG%%EkN+~T)I0#s_Es!Ev?cVOV>xZc+cvrSrQG~MwGn*>qPF&)IDa`re+(! zcNCoIY-%Fq$`Nrr;*wta*M)K~+Q}^>06(RS+8rz-%(ZdGO|^I^TJF@2Ql9sYuT3w_ zYWE6eoloH)%a;iDJHNG40tVR5>EbgSL7m_0^U6V`y(!fel^q-jAR#&b2Ja^len&SM z_EKY{Rd!@v1V#%x!#-zoM(nHIp}kW40j>>TW3LtAwssrK%-z5Bs3i&}ZxEZkKWc~3 zr>GQv+#mlMm<8k=HeoUv(C1{9<-|+8Eeu@GQ{YYwkEE*EYmB{v&6u#qz+3ye$*(x6 z(t^-8%d4@!J={t_F=fHe-db}6RzoBWI)dsb==9RjcM@611;Np$>9%N`6)cM$M|L;% znO4tx!hMHnT+y@Z`cR>M7@xELTveqQEk2sOJr<}1os zraC;RJ=}hpPDKTGlZ)!!4qvgPI<>EQmhyCDcN>18aX*P-5C^tX<%c^9=gESE%RHbM z2J0EMDK^xc*()Ck4C$Yg5^W(y&22b>D@f7KL1Y5UQ~%@oYxhdR$G=^8)vM-(DhvYy z2-jKE*^?teEvtfSIp7*RQPS^{;KfefPXR-D1R>2VNvU9{hpmV88%lOVbh{v zjMS{g&B3fr$ukI+^z$86FKV5i>As&`m2kHUk0MJBq@OsEe$9y*RCzP`hV5eL`>6t7 zQXfL{p;4QZ%pbVR-qjk(?Xu8sTD>nMl#ZC6JC$oX+p#+0cAQ8x@YlfUa#;nn*qgL@ z!Y5A}Szd2G zX8NlQvcKW=@7xf%Cl1_ts>o@ll=7xOBzEw(PoUT|p9fo!^IH33!!JM6EK4dMfHd9C z6Fi$8ZwDYwpID-)BYTm9rT>YEqt+|-P)g$;z>a!+MoTMWtJjvCr24PFi>Gv?r+g;L zPQIk)VPKbv{U7K0BDq4J+MN3zZp;ZzZ$IipTeJiP+B(A()6*`h?3&q zmHbN_%~pR?6_k18$ggSy_|6@n@U#aB03(g4njox)eLmNq9MTWcav+lUC$`w?{@=m4 zH&r--5UQvt#DUm@;aRk?SKFihJ<#Lj2v7^sN&4LH(~QKwR3J=yFEA3jAAE$j)UV1B_oK^z@Cf@3}d{ua9d=Uij9G~Z>$K2x^Olr9^@gFY0 zOcUjbooy05#)A^{1WAxz3jQLPSi9VeUyPVw&1;+Tm?S$qJwC`U%296)A!{Hl?4N{( zIna9t-&(xbs`pp3ev1F+u{+>x#}V(sUaQ`qK0X9_Y~7(ZwsO$jyYEiTJqxjn2~PtD z_#ZK}l3GOA6{s@3Fwxm&p*V3-(Bt^cmHUa)jQVHgnKl*goaV}zEc$k`>#!zGFZ5wo z>>~Spj=!o>hW_1SVcHV*6{exRTX6r^fa8T;6PuYZd zr4zlo{HjNAk-o$}x@~>v@o%%cC`umgLJt#%L6+{9bQ3kLilBwqukpkI&e@ye>p=+|r!|00h z&t(i$>^-&$g zR1{+hPYnmsqVlX4Eh$j=_gDnYnA7PVnrj5rN#RFGGoVAuf829a2|E(JenE3-G=VeS0sdN@ ziz%bv;#wk0Yi_0O_neGvOQ$~x89uUR{2>h~V-*Om$g9J8xt1c$W^!-W(#Y36C0|$i zBccPeG1smfD3OFSk{Q~0pIeb;c{SxzKa)sEa%Ibm>g9wb7E;Y#YH?GI35e>R|NRl` zQvE{=*>O2oehiWR~C@tXC3- z;M{YYk`~$FkkaEg{OUw$ag@7ub2He7dHyfPNtoK+#lPnQT{ui#4>3>DBl>4oa>yer zhD=hM080mQro)uc!M^Yn#N}YCh4GgGx3g_^S>?I{fQ6QW?cs zuTy&gq<9FE@(rq5+#=N1Cj8;0xphVVnc(IUVd$5b&|LDK5lKC8^kpf;zxw7!+p;N3 z?k8f6dWYfh?u{guT?1=hAYNZ_!MQ9~eH?-Y+�L$U?4PsZT0K;a6A^2oN}_OlcGX z-+H@G(L?y57m45yCMdwOL@Q2;gA}e9R2}8@*Xt8L-9E%BA&49K+$taO*dRS6@^*s8 zpEE<^#Kycy@p}$u-caZ>g16$Z!iu1(r7eOh?a^QT`~8zGd4~mlI1|GZEc@~HADRu* zYtUZkHZa7AGKk_@G7I|A$0nr&rP2slop57N#ly~ROg?EW=j%;ZiY(6`Do@y&yy_oP z&Q<~INM0GOLMTE*oe5r`{z9n*O={0uJ2+fgEi=L({;1Wb3(SA$Z~-^}7W!(J4MtU- z&@83Vx^ieYKF0i)`kh#Q_Mb+*<+UXm=fkf}hbJHkD1^N;m;cCHFq)$BKlniH|GDK) z3Evrb#QyyH>pN`nfAw_0Tkf|D!jPj2QU9t_xijPYMT1o&S>WIQe*bZpPgcGCw)#J> zg86-ia=gf&DdF$i=V<>H)SqKz_y4C)+i5hH^@6S#2h7%O2xWSYPRrwLO!W#2@&xcZ ziE+|;75ev<=5#doii*wihA4`H7n-nqAN22i&|eyy2W`%R-V|Vso$^{B?+xC z21`P^Dqw-DfTd+yxRAeZ)dJLN}!i%FKGM_vF3&9?I%{w)Nrjx8cTe7K9lGkTE_#MTD z#nlpKD{cA8_s;ii6*aw#@x!XsoyHumw9ciM#iXF{<`{Sb@$ZLwO3kW&>%#9&$}rX{ zFBYdm@hHX1L_MS%_%N6Eack?^zDU%`6El{(6$IO)8ol7lecWa>*nbUCv59SC8>VdG8Q2jtZ$VPkg8|uPT zww<$60AJ`nIfyHH!u-V)Bv$|nqVqZK5jWWssBuMOd=L1p|(D+T~G> zhX>3$*mr2`+F(3`0y<=p_#kRWy}!;|zl+0BMsv--M&E#MmBh|tA|E>CsYnU9bTJb3 z73yrWsEe#>v42BiO`Hj6kP>l|IiR!$=H>}RHKT{a#-FxyUeiG%n-=lWrc4UZqt%ZM zFWj3a`-rr}@(iIjc?=S&c?zi340Q`6#8lsLGEzJcC9qaZs%uC+|ZiAbTtGDx4 zyZiy)vghZc2X$5^>iq{=UG_SC)80#m+voQjgm-5 zhR0WNuG=@X1wqnYo01cP3vk*|yZ<%R!ryk}Kp(}%K#Zx6`9pZq^UHmdxq&O#%% zd*Mpn6We8GOGE?6{Iv7PPJ_f&N!LiEq@2&M}jybTA6-+26qQu1vI9z7dAgdYMdxmL|EM z?83X`hp)Jm0z+LaSxTy5c=jI+wG5%@p+>r0MPTwb=SLdKdG|Hhw5&GU2^{2*EsNQC z9CI2U(RmC!4^)o*d%8-p2Sl}t;iYW019GMlrS>tL%}^{g!-TJOGDHgXra92?TRoLe z+W0c}x1?M#UJGN+IZP^Mpabv{m^vlhTM7QeO{+2VULXf$kXO9DXVEpZ(d5*xB@Anh zPyXV6e%BDW{&LlA{Xw&7t7y62k0{34{^#x0*N^GK;!2at$krB}p+Ft1v5xQ8 z${Y>qIQew4dNdFoG;h-?DqPH2YQ6zST4p#})M^eLb0nnt#nvrjD`v2%{=Vdr+>_xc z^XC9^vF)O(K`BDQ_KO2rqW?JowlYjdDE_72;;%_K61#mE+oXq`SQmXdU3e@)(3({caY%Zc?YCiN zC$w%8ssdg|1}p7vUg-?!cT9kK97A%vjjbW4Ho66H6c%E~?oNSeOrEFw12V#NeE0#` zoP!BnIW^S%s^x`;Noy+v)qJh@y+DKM&`}334c*fm97a9@U>_p~nv89bh0e%vI!6a^ zd_KCbQ8~Jkq%Qk9WmMupZ@M|{l;h{)JlN0H$0*o})EAl;RAbi=UX=K`6Yu)yQv@_3yZv1g2mtx(i= zItMilTpg!?O|jL0p)i26Jfx({%AM4@5p3&3UX%PDPFciu4bmGxCc5dX+{koQs@4fGfhegVEA{fSW5P7y1B! z0Wn}~`};4`Ue_ZwFXWBX^(m}S>I!Dey)DlZ5#t&)cU*q%J#J`6k{v@dpL~2h?>~B; z0?F2cR;krQM)ii=h4y07a$dx@?ZH|&-hi@LX*9lNT7rH1+?E$)99&7bL#{#Alq6>? zy;XaOkevWiaiUU$F>xJ5AnK0#=9oU{8mZ%@NzJUka=bEHY}~&g1jeKwIvqFk(|ods%RJ z{0PI~eY#_`+-pK0}+-ZJ_e-hmL1EwEoXF)t-GFZejrV1(N?yx}neul}*(IN!{&*!PEK zB6vIW1_6UWSJv(*&uE16Tl5voRkuHSpr(0+695h}_uFRHiBH z90O@HXBevt9_;ZK_sw19U_qI=WUI#cW%+!oM293)hX$>1$J6_ve+5N2bwD?T$iem| z&s7TJI({o3wjXSZs2iVoa2LfhqgFyD32+G~x9!aoW!Om_lX{Omd$FpU)| zA6jKGcLVKfjwrfJt>sBYK1sxL%o@%ZRNpf zBDcJB4;0FSA+=ZE!Boe4F^xSN8LXWu8=}XjVny}y&&L&D}D>L5Bl*Vjq%5aXmncB_Vfo3{-wM%YjjkDfW3mm%_8xZ|D zrJ>=?)WLL0{YrqVf*Nl7$Ybvm~f$xv1c4}p15 zgk6+W$s1aTraRf;RNI+Kl5*3kpr{oXV3f3Voh%(uYem4Gz_D72zaF>T|s{s(F*5B@n7L~;bDIrKW{hW zTi#lHaV<<1hsM>*C%-N55V~lYVxOC3i~07VBc^gD10}4lPgwwx z%`OtvT$=O4>(bpG&h?HnH7hQIEBiW%r*SDV)aT(v?X~C2aJe%9b70YVSboNwr9Rk5_3#eGUA! z{D@Y~4hOZV}Ys_tMkm&XxHRypc->&jIE02!_|lGrmytZi`F6!(nPKk{+pw(#IfIYBC}wXKNi$_4H|_pFk(_ z>IS^k?1twaynuO8_g*tl!(EuZwD|GR@oB!9N?6FH-dypiu%`?HF%}X(?HW|8{doOE zBf&L4pfT0SlJ@eNTFQolm2?SL5I?Ke*8tK#{c_4A?e%Dz3I;_RM>A(_PVK0mm zE%a&4^_swDK3E?1qNyw;Y$`f5t-o*es48+ne3V`wXJ9s+GlLoh819rb_SQIMjv3!I zG1CoEaI}v4Fldg{xwl}Qxo^Ndl-+!K*dh1OMUm_p{P~%%>{`%mZROIB{n|Mku=Dzj ze{@3W#!X*K8O$IA0~Oil%jW<1wbeaK$}l%oK>Vo;&OR(%B$-$(v6g$Y*_o$8GkHyZ zuleJ(q&E~rCECwPiq4mhMZcFc^x$_85=hL4CfjM{mjBas&V=^ALn_OT_sj2`Iq~zz zVa>4oo0_iDqF1x=(+$t*?#KVYUoS*rj+YEV>Fl<=y8+KU&i`}z_uB_fU3voYzmEFv z{~*+YGm~uueqEO8zT)U2crK&kRP;z3=(gQD0R@g!FYY`$F_3?& zF1=~|KgWpr9kqQ{F>X^bOKXOl8PC(K(Pti!GGWk1HwaAgIMJfITm$2N%>du&oyK4t zAs(S38kCJIC9iR-0fKT_#d~>Zk7&OSd1UVf)v&^k8{Nz8y|~OPKJ5aU{`8~L_qv~5 zEf(fsvmZEAa`03<)o7(?Z&2$$o8P}ZiqMtvBw^kV1Y@zPWANbt2|c=ks8E`H*{8+0 z-0&%?48C-Men(XetN*rZHieb0Y%A-JWy~BFaU6jDx0AOzS+Cjkb~V-(t=lBd1ZI76 zn(EJ=k*#4hZ4f4Jkrqck<`v;cIE)iF-!ow*s>`3LytO1-o?o=xf@o0IfT7liYr!5l zToT^7XB$JL&0yvcHld3_x8|jp2QoStyCUD}w1>FOW=16Y?!(U`fPuaBDt2i8Ql1$XJKAxq)oe^*kh-gFuCr}fR+_zZXHoB^#Y~Zf@l2f z{51lHk_{4&eH%q!Gx^QCopI!6c!eK^&{vxxp@c$$4(JA&{yUdlm&_aslin(BC7QXJj*YWP6~t@U^lnC`FO4Dv zBcvs8Zd}jnOlhI<#oL}J#D%tLpOT%}nw7SY0uFw_i`;5c4mwSP03Bi|TA2{4`{|9B7>g7J zxHclWjdp{$?UqI?$BO+8N86{Zo&`PORL~sv${&BF-ETy`VXs9LzAKJb^|wxjarEYw zA6rIKj`P*0#)WD0b)vd8+a|~5W*3YFdfBiLs|Y-<+1w*YoSPe_LO|gZ^2S|njWS`sRFLL5K+Mkzd_WH5j)y}@WquD3oC9XO%8610>X<)J zb-p7lSn5NcWolc6KnIO`DZpeVgU!3*C|uoTFXM1iPEIs8KUoS3s!5ETCwp<)UC*NP zd@Ola9hZ)48-r)8*nRqo#=aNVNYwTjKCI8=9*GzL4O)!s+B8Z@mhUwh<g}ds(d+g&aW=n0ihl=L@ ztp(Vv+Y#2-3-VYVYuoKjo(SnfBpXF+sA}6rUoW|8>&KXWsj3Oe&l~a zlJ08nkY~r8U30P0#l!InHK^_s!D;}QR>0h?#93;Hw-RW zQQqVwNUW4?zV!fP(3yfM9OIfqSxd6MyQg!ujEXf6qhc|ReIYo7!$i*S(Ta4jFrf*!>%UW3-q0Bds`Pkz&AFcwfG8byp1D?mL~oj5`xbs7Daa(THS|L&D<24EG19HB7E3`Kq zA2mq9j2g6j^*41vp(cLGxe;YE%kFV@=SIs1ojn5!1~*xOo+*6ZrFdnR=%#qgja%d0b1QUaDz&uhNn@q*ny+4*8Puc-J zW$;3&eX>2hcwe<7UVZa(@mv@urfblRWJ2drlBGcG`2E|-W#v-j&sPtZ5(Vi^<!rhBk&GvXlDb0{_RIP!rU-{}!zS#@BYR5c9A{tFW1%XzS z*Q*|mAj=T-M*SH3p<=q`%v=wVE=bOr5^B@A;r}oI)Dbn`Gid1F__El0)?f6JH7#&9 z+Kj}xG!$`#ouwhp6nEggH>pq*tn1Gm8@h^3`x-MZ!=<+}E`GWC2-5DZkG4hl-QGy{Q;0i{g$otrj} z+H0_|mDD?j0jUAY1F!O?GTd>6Yj>Og0@2^ir&5@ZA(cLYnSKG`=mpP#1h<2DBg_%2 zxEX%~?yJ6DOt4jwQ)qCFI}Vz~A@De*_TeO})ORso#eW^>hx-#&w@>@Ga(86L5e}tG z&7R_n8x8oXmu$=l{=dNQ&Bawo_Lf-rv)j)jLcL{>?#?p2o0yL zhuhc5d@Xu672D=4R^SC}{iS^o3|Y59671Y_X zLm8>RQMWKiwqA3${IAl-wek=tqas226P*23CCg-qFx60Whd@XND(|;NlBg!;Ab?Rg40S00WE2(Amg|&ivnE;6xhtA`!KvDNnZ9-`3jWJb~ z(x83YLT^|yJ@fUN900o+M9|5bBtfZOWJF=~YrTXA7mxzJxmhMZQ` zuLVy?1ui>a6k>yr#Ub6wAx1_UPqiu|u4mernfp^dQKu_N3K{gjtEI4(CfzchB!atX zHnH3{lvM58gNMW}_#1UTiG0AFn5Q-pkPyaD)$>}tKqU*Q_vd{x_qxMhX%78wh)mW=>BV~{9pmZ}DHGF$XbV51mz}|uC=-AkxrK15$jyt+Bc2Tm zj&;jSuSKK8y@xyIRVV{B9%P9rSt#7rm@k(ezmOW}i@}TfWEzQ7q$(OTc3iGIgaN`1jfie@{$OMLDset3g>Qbl1yv9n(xAO>hdzTXV7mfP13u1*O-i@KQ znD9yuoOU?DzttT-eC4_6IhVZ$Ets=`-GNLfr5(NA)sf=o9w-4bMz`0=Vnni1j!%9n zcKKN=8ft4|IJJslF>>ZqlyR>>n9_vqIRb^BmkV;RC$Cc43iS7qkC&CAIXh=RVxA-7 z156y+;m>VX6Oj~wUP81Gwx`B7R&b0%MgF!drQTL>XBv$}o2>|rJN2gd1cEhT*SFBY zwnI_2QwZ|bi*3=)U0p)>b_1)RS$hjz@f-%!nQ|%|ZWbHTIwSMZOxBH8f*WrDoFZ{J zxA>rY4*#Wpvct%{cnH-X@zw;G++1+OJ}lwQiDhT{icgKx9*Kf~obDR=D`YS+(EryS zp6Hg40saO*_5f>&ybWdlK92O7Y5vG4#uKcj6rNRoj_KioqIo`Sd*b|mKJe}I!mY#; z-yGam>ptzNmtpS%XurgeAOCax1dj+m|Nq;+*ZKcdx%^*zy|48B2Gl%C-lwvrY|B?Z0r}IzHjfgS8YZ{6x%`+%s?q1H zqgVnTDCS)%E5rH}<^>pm>H0|H=+Xer;Jjh|+NIV?OzcNj=^s#HpF(X_ZVp3h6S&|9RaCY`xw>%6u7K?^A!Lxz4a zQkhLnSgu0IQPq^ZZKox*VSk*se1nDkYT#H_c(H;CGBase6~{bS%GvoXGo$A<;#+3l z<};nV$iR0)?cdf$86F^l5~$os;7HKE4H64ocl0D!hvWMH(C>4R7r8G4jEclJC2~f z&E1(ybv~9oJo@n(YxnDWbZqoB@B`qA@ZdSN%RlYHyYWtQ%TUmjbImsnTW%dd_m5)L zFA8yE>3|Zpukdgc*gV<>X%o){#R2Flhwdz0pu-TIix2fnf;D#^#=GRID1IHjJM%VP zlY4=)QKPn0CU$>GjNrWzj{XucrWnl4vzI}XUm1Q*dS_yGk2xa3Mb^FfPLZuD@Jq)y zqQ?2EIC)_|bL?>U#9e2&3dZXoACzmAQsDEAD2ko%JU|@QYs_xHzT&&q7yzS@Rs4T` zI?_-i801)RFq^MT+J{`85>^VS^t}S`763v(r%sdPsQb)uHZ$T==LmPS_yCjTRuzTR zODCNpYd&wO#6LnF*x&=zSO|oSMc2NKv*PoIZLb7WrD&7C(akU$7DBdr6In8wt?90l zRJkwX2_O64<*y~Sf~9d8WxhP@qc)lVQwBK1)rQt9YemOG*Y5NFF}UqEwJvnbRMoi= zbFwzj3a?aF#D2`A4su@FpAXE)NSAR=^Aa7BFUilG`W+$UhwF!2T&on#QfxZEg%~Y=VFnNqe1d)^-2CU){j7SgKh4}n8 zC8;Na$WilHYjkA*c&~b3OK#vf1ZTlI;oGv=bK_+1cQBdkst7}bYJ6Pzc#7(|HKCDL zm1h@dr2uye>e>_Ne-cUJ+89_G;9F?%AIN8J#d!rBz%h0oO&4nO!(+&f*#!rOO-M)I zuwe5HITdo{n;JgVY!C}S|u z*g~6&*JNijNP6cUo7&>kR}$0KU$D%^?n1^EOI@&*?lL^ryMRpXT;2o{2{VcB%aG%2 z&igg}LD}E;*slr#I;&rVS^%bIYt21eyF?_+E@7S43HnHORno;dJqLBo! z8TR`NzoeT-V?6XNcP`Z6(tQ0pTEX~SGBW$bsGlVw>lxAxM#GCB&W(mM&c+n`qj6Q2Fd$H{9ABYne6Z|I~L$?c!E_<`2B zZNv}6E|#&P5GUEU7V|eWUpwf`f*DYo+@f&vWs9zj*KqM}>ZRoZjD@$*AR`^19m~1f z)F+i0cQ1oy=LxY~;(4P)3BO>tPS)8|J+u>_?VqICYAf2+SK@CAmI^*gEh0jURu zIBLnQ+PA18Umi0ks>G-MI($?GW}i|H&)cc#0Kfh~0O4U%E#{`WJMV62!7wIth_)G$ zMZVq~n?!|_lO|du_*eIgK0*`l&6>~H%LrTXrN0Tlq9dXh5f!K+qz9mlSi^xR@vyDW zT&+amH9Y*@5S$B@ej+_`p&VmGaq*-Uk6ey2hVNHWb%KNC!o2S8m^Nq27DJ%mSxCqj zVr;r_hg+z4QDM-8Y&gi7QK^F2(@-!`G6~@Jw`zt+uXV41vcGrQ)R_cKZ=2wCL3u=7 z7;?NKJ{vDz5L!!9dXtU0hw&pDH&5}c*~^t1L*V}XOrWfAt7b)qwa15h_qYQ^4sCeu z`?4>suX#eD#Z?VaftQe((Oo)Rp#=sTAHZ(QhNO#qwI7@)q>{=<;eBaF5FN9)lHQMa zN|(#!dzZ$Ye7xiqIUH2F%B1ZCY&9N;z}8Zf9Oic3;itMHiwENZ9zhpX&Z%P-l4tEN^Y|FG$wwRa=UXQ&Dkt9aX_k>KEu)0~ ztk=56=@Em6Lb4N>@LGnG3OAwts&|wqU%mHQ5B5U0(- zUu0a_>!n@Umkaxnz(n>lBQ&?tabnFq9y-`;j#6C`I%^0QAE4@e=DrA*jm5%3Cb#k+ zdv%ZxAFAk_jVN-DyQWKK0{##0o``3$KTA)~H$EO3l%RtGu@5~;UjU;1;@DBLC~NT_(NUXoSuRQ?8^ zE-g34)c2sQW$g+|6bpjnCiQBSI}s1y)rpNe0d%}=^Gu3gi%g)xhyhV*$(2#n zWv_aNFtF|Bh}=K<0vhyL=+YsMZC&qpLth%(^F;nocuBepcbpS_lswGjhYx{C4uhG#Q!?UqO8{_*o4bZHZn6hBgiA$c@`JIuX}ylt zy#@^}fMayTi_Ao;W$RadTt;74nOC^>j;lBOZJch7Vy*qcY;H7~p)Y%v^wMcm-jKMW zcAQG@*>&6lY!LLE3_`7OQ_J4#mr&`<@VXkiL=FANA$KLq?j?uU+P@_tn`)C2{2=Me zht2aTH^#ing^B*W{^AVefx$m~CH0jq7DX=AirykhY*IqzV1!^Hw-rkn_1WjiLvq}F ztc|l>`|yiq+<%D5o(BMc8&haZCFaTsRWopTR}+{)R;>ic0euQqi4Fz3zITh0jfiIwd4hi)|}<;|CzK(!`Cwzo)XjtNPP{uCyLt zjE3FsGMvgiLvo&>We#jvY+y?I{iwVwwjMFRl{}C#q0rL23d%DaQMg{R`t?e$1I5_A zDY#-2gEPO2N%d=Ds*k)B(|iS9-T}3o3lh<4j#B=4rCA)7$9xq*Cp|*dQTMf$y-QV$ ze<;QB^v+`PrD!5%`_kSfy?&Ddx9dh)H%5{)GmSy}=9a|qpkrgVss61yi_wplqwN)l zS8jL`Wftq5BCDD0jt&p?&J&^p%l)jP27RX%dLXY7^kcaPB#pO%*TV*q9Z=%=^iA1R zNu&qWvmCI~=}wBaq70@uhc^N_zln8zINKi%Uw2tfz{beOhSV@82QY!`lFXFea{-Lj zYTwR)+dOGiAx8=O5TO${Z=`iS6{h91xK++DLZm#x?_%LFna3-1iY?6gYKN=rOLUBP z5v1$Lr`qss*`+v!&sYRz=jjCU#~V!R2@j-)(}y96l4+UKGtuMSHR2DZ!yYyEkzMrm z3y1gNsbyp?Vp?*I9X&&YJg~QW`Z)~9JrKE&+~mXb0B8nnMK|NBcl)aw|97J z`;N?OrcT53#hcaud$t6cU{qAkeU*avDO!y}0gvxo|`3yopzvA%J@#Hvde+db= zK9*MJypMV*Io_y@_8Thd=(TVX@oL)WrV-D2ERDSIHtlAheflKL39;G=zldttQQnd1 z4^k#Nog|%(t(>(Cgmy$H4HGPVlyyt-7ruB! znn7O>m+`w8*2u#ACf$5NS+4@AAMk1kuFLzkn=Z%?;sG#*n>Emr$Z zpN9)0LRrk4TEga3p6EGcgAeyRagG#OGuSScR#i$IzIyLvuqrFhu{#ydjxV;4uz2gSs9yF8OWilPCmt3Xt(|Nm& z?F1|jv^X0|h`7(OIBUn6K+5Bc)?YzyYUi8UzE9u*Q zof1ue+M~#U+N~dEY43`sH*^JX>Z{wOBSI+7ThyHAMWR5W$^yV@+-8AnRw8Id?|NZQ ziZW|8a*E|%tNp`4?D^uopAlXlwVAa{$W&tHCF@4v2N;saSc|Dq>>T|JqwvgKo9 zTMKk+ z$aAo0WL}*=yU?+|3BjK<(p4zsr>&XU&o3R$UQ_l4$a`Ov50J6q$_P@#Hhs3?K^MUT zB!pNV@I31H`6L(f)22XzW%muDFT{!#P(NsyV{0S6G8PC|oz>lOWfi+#=2TjCUEduW z+&`fjgc9zj!~A1E(0>XPjZ`4)S*&NkhVjd}llsy!>^gQgqzcq%xPU9mj*w;RZAoHF zj?nM)X>5<@F!HZVk6BL)du<0?^bHD>>fXdC1Q0C>8y~7+nhnX6&u37zy)j}NC;nQF z-GceXXQgwUXCIeq#_>y5vnaC4@)}}TAj{kmRDTGv0r-cQY&YGyO+&JK{BonM1aEKO z>RlRBF1D-5`s&u~u9%QPE7G_Btpx}IE{ehwbDQYJ08zHN*ah>i zM)AO<0=L;vn1v6l;MRS8Vg)MwE^geX%y%GFZ7M^F=c{>8Qir6AD*xR$bSU}Ril9k* zS!EZsqI(TEe>Dsr7Cn<%Zp{0J{;YTWmWgbY(JvIB(WZa-jLE2d2ZiiB1L>SIADcy{7WlbXMp3C+iqOY$Ir2Z1 z^FjTC@G)RI%WC*d!oq`EX%NuPSX*X8W&hAoVdE|$sQBxG&hta$2XOr3a+hdI%*hiU z`04d7#GtO0Qd5^VvSsv&tQzSrraDvD$({X-jPt0djj7W;&GDa)b0#?*ekjN@_X|m_hHu(;olXx~+zu*Iao~Wlz3W^D@L0Fp=4t~)V#tT{= ziyQB(EcuA^$^;kXouzHQdjL^z&x~sJDRS&c(ylKQX-rpwaUyrXtN8j)Bew|sX`d#4 zN_Y0^_NJ8`O>9Afx@2z7hqpQtUkk(Xlf)#uHFP@4I zd&ea>3XinhX4pSXO^OXt9tbs=)Eu)jcvYLGZ8G5LtDq&~Rf+4WsJ&#^t2aH7W~L|* zn}b9EkS7n;8Ekw})>Sh2MqW#(`d~*8DY{jVTq%c|+yWJ7Ec5$6swb^eki9*vY%%hqmQ6^M0$9$t(0$Ar*Jpf){V z!r2QE{S#M1nK>_m`E2cD$eA7+`&8kM-2FFlD^6>+Jf)kvJZMW(EHu!PW#tu!^n!eFzq$uH?mOI#^q3Y6;&<)38 zDm7WDNC<3+90bKhN#hL00>q7ByVqAj(C-CCITV$@7Hw(^pa&PEcRp|UJsY|pm@wJX zq%1RVDY(ruvWZ|gCNm)0v0Xp)g>5kPHg)R9n(~*F;nEe$>>TDB4_egMSohS*gDra93l=}d=GS6K z@+h;R(A`y`E*lm1JoVi9D#PYDe0)Qy-%bqEOwAl9`<|uU5Y85o!I4n^$Qvf=eNo5) z^Gv6AK9F8gV_q=Q1j?%9cKKcV#IvPT0J;9*(wcsO2WnKLr_ofy20xirnfX&uUSVZ` z0QZ&&PkFH=+V$Q~SsPO6j+ga`(XZVSeFDze^S)u6y3d-cD}A*24js=%Ph(*+;umMl z{2k26H5#TMkJ_wrg2?@k(Yx|1S%yHJ_}D?2C&4;s?Tbg%-$&o2{&Ff!i4=INOj^!n z*Sprby7zSaXq5wfbVnNBLt9?4#0P$E;7KLq?3sm!lrlvpEsd$?w}?B0fEb&21`k%u zFp(Xcu6BnE3rBcF@`G|Yh=idClJb@g;Q&LD=@pH9$(eU5B@oa|b1rw4FKd5z^FJ^WPef*Py#O%Z>LMC;Etgl^*e*1T))5? zu}PNWt=p45fG;Z&DQcwy9stS*DF)`|*??IK&%xdVCcxv5?}!3WPuT%3eG7(%nwczlwM`(KKCM}J&y3yDTj7cP zfu)O$DfMVxA@Gv?ieuJ})DglACLZQSWW=J=0)>*QGNn@57YoKh$wL#r4h6Cyvs$~# zrN!?M|Fn|5;Hc&<+t_yMZ*F{9*oVU}*J& zOhBB*qWxCb`4c|j&maH(s$)=BNbZcYS4bm5(6s@a^mi5N&Aw@$VT9j8*157WV%3!X zfEztuqn=THyy&IXKCl_2$vJM6)AntP|HQYe>%S>e9=3##ZiwAWYVQ6!NqX6tj?Y;U zDRo~Vg*j8*vF`ROg`I<#qLp|4G~VBoTx-F<%iC`nKr4(^`296dBU@Ad!EmtXX{pdi`?DwK~<%6D0v{Ve*h5qlcs8?h>VWM_4iz3=6eIb(yR z?gAXz-#gt z8qHywwbk;=Q*ZFH`K;a!+0}mbsnOd%AaA1>nMJIBRvdSK$_+$GKg8f9kAf)iFvkz9 z&ZigtKwH9C9%8wF^uX$v2`6txt7MuVf8p`nU^)EtegpH}&Fj%in%dsyj~E!9SjoZS z;se9v_MFLF9JHVAq{1_ikqc^rex+KE&ac__;SXo7mWJXF_Lv*oAFGX74o|k}Z`pPnoQIFz<-#xC-DYJTh8MGrRr(BEY1uuXc2L zD{kw7O3I&2me@)jO6Q{ph7ZH{Aa*n6PomxRLU#fQ-kH#&7#^+W)>_Tify#m}#uV_! zXrM|=Rb%m5I6y<`MNHV?OXzj1%hrnh*!SGQ#l8LxduqAIZPW|e^v@iw9k+K6EP5!5 z+O;x|L#F!u>HTRDPk<=tLT6m#!LPw9<-i@Fa=nXB_xvcID#$J~rt6DK<_Oss#H*@D z56-^$meCBti;vgkA{TzX2Krfu(7B4bx$cB+A`Ale1GT(Kzf|g=36H_Ex{pFV40fMY zgFnrPm}8DYq<<*CFX3tq6rJtiqu*$D1j=(b*KpWj|C_dlSMRF9*8ZxT(tKV@L>t?_ zj+R5lV4IqYY8MKZ%&>MyAiT~)hg5F^l$Yb_vXC)ZizsTGzBf}o3WF98ma2vi4f7^E zGH7wm>!gzrbtS3@zIZoN&I~P6CNhy$3bG|e!z!Q%+8+Pj*Q+nuDU)wE&lKw^?5l!S ztChdlZ*yX{d1J&+QB(Uh0}))tM+=4WKT|rd&(fz#h|;xpKwKWyL!;i{bV3-faxl-! zx{c0%x>%}|hOYq`>#ywI25p}`#1z1ms}1PfOSs6+GEsum#225$y(=_Dts za@NdQII(}ShpfkX)WbFt9}eWdTCJv(I)yi-mr zfLI$~UB)sVoR9TyQ!D@@zxaz=xL9dsPFiHQj4v%WX{==Jti<}1C2aMksG>3h#l_os zk_x@2mF#m45POC27W7<9`e6YAzx)*3R*D}m69uq(hcEn8AR$ahWanh7n6lzVgUUAE zr=6XT_g9Wnj$5qaRhNfdF0R_|A_fOqZ2032tjOV{MP<)Hk_>Mi%VqOLM6M1p~T(=G`-&9_vv$U;Ae5nJWcEC(+AMr^1 zH5)IRQrSL4(FiYO`z#4Eb2qaf1fXD?Jg$6u9^>&miC4z$cm>CjSsEp#YQ=4_)>|Uu zxeuzfWV-EnJZI-NV3Y9G#)a$f{K`>#qZnqI_k1XnonPau=Om#i?nB7(r1<--)jGRD zaFc_68R6tDiH~gR;rIM>lXg&ENaM9A7!f)YZzg=V*s?^Xsm1(waL^-dmwod+ds^4}%}P<_IyyDkrS$7wXG(jP zFI90c;+@8hsUM+4i_pX3hGP=%gc;e`H1`qvd=URA~eSOMU0~S(){5B8D z`N=@Rhpl_SoQS=bUUXL3)AdME#CH{xza0sRTTsJ`X8Yv_E@pDuhyP4y88pYKP&ZKp z+s6$(j}JUzy?h3Z;Vb6{#cxK})rZ$&V)g|PmfXwM`z4Ap&4>c`^WQ7j+Y(l`F}x<@ z?cWH3?<+N|pI&FdVOt%~eS$lP33{L;O5I_xN0yXIcrguc13^B5q+1@Y6)h81myV97 z8Zr37H87q4^CI%zy!GXh?<*4^3E(Ji>RTt5Gvf71IPq+eR({=+bC2&_X-i26Vi#M` z$`V9;&r1}w(FBUTNhst4u4}PzT<Fq7#djtem!SyhdWrNDIGMNs&wvZAMY`mlR zl-;PjtQiArP|xR6k0vPnIyUF%{e$Nd&x5E4<08{T^qZ>nKZ@=cP5qMy7HpVc;ze&)(@1DEL&2UEk zjv+g3j?o$rz;m`>iphPdNx}v7qs(p`S&ug05e@)Hg%MCru6$<~NE(4*-gl8@mMeFD z#BvbE=VdsKFfB6egR9z_czby8RS<7!LUS&7?;Jv1=c$|vbSV@jaY^Ivl z#Dwa)66&pqJc;)aKI2xTVW1Bv!$h)(UPV>+JUMRY{5(SD{EvC3;k)fmA!3I z<%R9!zQkRq$U9+0)#BlB-k1aE;rR;yYsM`J0Mh+#w}sJnfXtesS4DPmb5>ZHopf!G z^guZ>xVT6LgzwEx>y|!QBjbF#gZp@L2WH>_Y{^;NaLK-&xbujB9AM5Rm!Z?;tTw<= zsX5_Chf$?Swd3FvJisc`qut-eAxW`54RE#hfCtn$zR&Vb-h7~Z^0R#=%8&}?+3QcM z{C?P+3Rj-hL%*d5LLERtU|RmVrbaV9EK!2!xKYz3SM!9R3FWX z73(@%fMIeU?ZDnSys#k0ry{`F7`r6v#C|Ml$=U5nXpj5#$e|fzx8{xw;gRK;&gSkv zR5UoqIuB^RBCFvqdif039m$(wZ@^AJrkRqbx_21H0Iy1iCn!3D>?LVi$xNY;+|h>= z#fg_yrgnso=Y8(i=2!YcWtueBWCD3)Uae(thr-dov~!Brpi8y0r+j|O#~QtKKIahd zv_g+IeeU5GNj;5oxE0y~hMY|9xxJo!cLyQh0)HjT>%T9yS7&jm>JWLf;PonIQwg;s zt+>7NFcR%u7e4?seZ}TsbFoi9GSy90`!vhlTCDjguxq0}S+Al!+R(5=^tn4zga+a1 zy29F;j@5*>vNh*Osom*Ml2anD6JBoq*q7_3-44dn@+BmU7mqgD*hR|l!#}1@eKw82 z%^Bpp1OciBln0J4r8=w9p0qb$&0i&5)ew2NgYOTD#K%Vw^p>4V`s$tlv?fDHrFo{M z0(c2CU_-Z!7R~Mm4H)bIOd~==AwgiGy(O;5n=r`=QB;!8r%=7gg<{ zx(l^&t4|y)5-$94;kZR+;sr%}MC9ogttKw=O|L}wYH>BZi;4Vld(0qOq$G(dzvXn3 zbyBY|RAle_FY+TxdygH{^@sUw{WhkY7xsbIMH*nwzKoPy=LDQ=vuq}m?dnn!%^w$u zk)&lkz$YI*razr^!rOlmwR-7#dPz#X~bFP zKF*}TdGC;0(U+>Ih+F?S@e}hwRRh$CON0Q6$UW8 zZ22E+i?-t53i$JTshtxn-tyACvLDcIn!C1e`5w_eXXD=P0v>oQh|d|JB(KHIwV0jj zEfCZVkhq#pDAxGsObpY7<-H*#j4Q^`gx}7D)KM8OPAmAk>9r^iZSVxE+%)`KU-OK< zih@BE?*>zZP?GNs532z&D9E3iZG#5{RK6!4KX2>#j!N%7y-;pf62@OgL*thv{*!bf-gFSw{X9gst6L?lU6L(iO>u=PGLqFI%_crEb&)F?1ixflpIeYQZs6RZP|2n3+i{h_5t5l*`)vel% zoUDF!4}FjyuQN7(5piqA=86VM@|~H&?*a63{3_vAa`ha2xakLMj`JABZkrU+T+T;F(8i=hFi)?j6gf!<+e;LhZ! z9G1JM=~nP+b!mT>FEHBiy}@$u67zPfWsvh+U4-D_&DFK`x}W0@_^XgTkZdTr9ONsk z)!K8{n`Kds9z!18cY}Axa8SwZP7Ivsc_4-GmkUxmM%qEY^m{^-=ZgorgD+T(+je<~ z9&BAu`5JB)EgboWh@98Cb5p2c-VyL28Pxab>9ga>H>Z=k+wNRQrzIx~?6VWcb|QL2 z^_PQD`N8KuVr2K#6}p#8L`WCCAO9KfbBj*98wIlQx1VyD3hS`LJ9Ade(XHCObnEWA zjoTpJf;9cC$_wHn9XAJ_?H$DV*%4KDot}YTLh5&-WZ+A==XWXwkqluzux+KJ#~RWD zkERqlL>2Dut6p4AUJVgeSh8U4?({qbJM_>JFgw&@?AF&{*1sx10`E!OCoeeYn#2p( zxv^76Vh1HEuA)gvMq1TjF~B5^xE1sBX7|H)tO3%#16oSC?wwCwCHKxFuR8y4)tz-z zRNLSG2M`e@E+Hu((g@;!bP9+dFd&Vjq(cfr52X?!9nvY?CEd~uLpPGrT?51X4%d6X zzwiD2{XEZlo>{X7);hEHIeUNh`NZq&dGGd+As^k{9i~3ZbGchB&h) zBh}@N?gRD(7bOgE_T-lNIK``=v<5X<1s-DYmomJ0?tqL(CGoHcEmsTk+X&UC^dir$ zb(Z)Ln-ycbD<2AUnBkS9O6vE?mRiT@$XyK$YD&nLDx2>hz#oVYfy{P+nTj*uxPjDE=DRFP~64NMgGddhThA%}buh^!ClJSOHiGzq{m z4NT}b8x9H>{*V(Zkl>KYP1FS2`s#4C;ja?VyIVM4I!OKWxEQ--3)|bCyJ+y|R{li5 z;CSO50(YBxA{`FV183#ngB^41ii%@DBr1M=LoEbGrKR=#<8gWAFs?v z%uCcfUus`sfAD;ta9g~eztKo{kk=jEcaD=}SOYg2E- z$h3w7no;-T)Q(pa?6nkzF`t1~D*6kjBV7u|ANtL2qgR~=tr3kebrOtTri!q>m$;g5 zI_kw>kMYWhKl6j7;x_ELi(@|c602AVFHrN(%;ye^+@I{*(`}R{M+-kVqkcx-wmPr> zLuYZNqs?JApiiahW#h$S%z;NfrYD`R0o~nzZ0xqQ7eGX!h20Hf1mcl~861iM=2U*p zx~C!N`RMuVt;QJA@ak8xv$J1#u5~u~A}+)l>^8S`)T{T}A#aXIK9$Yse_7{vHjZ@> z)>U#C-|U22*mADx)Ffp^N6x8gV}7iuAb2pXDLGYcO~oOVb*)<5p2Wk_Ol=xoe{buj zdASJWaH7(!Q>QdVb!IUexm__Gk<_#g`;JehpDcD@$+>i7L5(-xRCv^iXz)r6tENn# zx9My%FqccC-CIks}dpOl|F{(=?iEtG#C5Cx)WZWqvBCtT|e{$*`d5!BxN*yblR$} z(}iwLMI`^^!}mmLXp{4ja+{HqE;j0yHO8Fj$(FqumDSbOQXl+PbH)ys9FyWMb4KF$ ze0}8eflB*1xk@7N(({4AVXc}(Qx!p-it%@eBKlf_C)4&-FUgD(OG{j8Qw5GwjRz!s zerJR~!Tl6c-Kq^e3+ybtw;JQl<4gii_s?w~zOERx&baZ;$1`)ltgsqO3pyhf;F76` zy_#VnKTX8$bbA=OcXc#li*AQ@?I1Gck)h~wLdY$R&$h?@-y>h2As0W%QSF~Lm*Y6= z>vbGudR@*eeC7f~;Q5Z9qx=9!9HDdU5opl0LzEu5oV zYGE*Jp3+!mpG@A|^I5Qs0wi%K0I&H~CD0aXgFt9S0@~E{n__xIPZ}>0K z{_ubI;piY&J=fNVlaBlS#oo@s9vu;;`0u70ex7=B^dg#fUbqvT73Z{;je@xcC;rmR z1%V91McUX{Q!c^@Dvs51h06`bTHp?=!DRi7QN!0K?IMTYZx?*`oAa`HH&Me+UD4yR zJ?}WzQ?J|a4>F{@*Dk4~sxZ6iHj3a$%ZmKD6$0z8UcIh^?3fFWp3C-qnxzeMY!jII zV5wBL&ed)10GAlAQx8a?&O|lz(+XvW>?FE^?tNx9=V-VTD7_^S;4_mmHC_I5S>L4Z zmc%umJW}%dz;EAf*t2T;1@WaDZ+N-5o&@?qGa{Om*@r2BXt)$1)L{qWuw&$oIxrd? zVW>#ypRcxd%RLWFBuG;2r~xUal#Uwp3*m35Tez3&lxw`9!TSSK3m5ZKm{Yo#b%2Rq z^U1qMFLAo)KF85*DEBrJu#FOx73(=#h$pbQ|FL>~|Kw&^Mt*wGj+k>~7Eb-RZQ2K! z_j3}Pc!8$vw}>tW8DjKhRMtu*5`r?H7JgtT|G<|wmt67862d4Xhc~~ngnGO=*%^!L zoybYvKfr2>h@8_TILQrl$m!YMlIgJD3FwICQ4cQ-(y!Rk?MUuR6f+5r$ljU7QhQ7gZafrUE4qW!&*G!6rY3&onQL3V+VsJAg$}`!_a4W4A1tB45}mtBU`Uz{R{Lr zl$q0a$uATz456h}9h<}SrrdKDOFkghIE$UZTQfOMa1E+P!JW=jXa7v{I#iqAODe}- z$~4BzAhNaV`wejVMy0vSYlj>qn~>G%<;vRqqM}*n%Q^Bfcs^uj=laX`FBdNF)rYSY z?#NRxDo86PU__eYs&KVR6>x) zkIv~ExZ`ZCm5|Ez>A2@tIn7=(UctTyMv-k5P7`M)>_R@lV`@hW4(D^7v#)CdASfZgEg|8%eGde= zF}yBbj{mHy_3E7&pM#w}@dM1NnQr`R_FJIdjzay4*#b@G$k!d(X=1}8Bz@Bjq7Qqg z2Iobeltah6PU*rbiZ#_)c&=~ejXUF$G(#<@Ij=i2Y(W2~LtBJt#k*6cb=$$p{9Q>Q zSMDD>80Mx~=vfj`&dj(jw3`M=MBX@353_03c)tNTXwG7KE2MkWa(1@3XsD>0Qa)OB zbKE24#2FPZb5#uenUwy4p*2tVyKp1kDnqc_#tZ2A=adq#I3j&sN71c2!OC7pOYpYs zmnADzLr)o({iKZ#9N=`MX;Zvf63pvmszRD7@o+HFrn)?%+*>WErS1tOgW2q+dN1WP z8wivdb)7b)BXfDOrLC9)QMZ)zFU(a*`qn)NGVF**eCaMoZ=Iu3U3E3rxy9ITaq1b; zxW}NL7g5K`IrD>;g>7lP;JO6|lY=n8NOJOrk#waNHb@>j0B`l(rl%kg>@AxwV9i{) z+Od3EQWJA}7Q>lLUzQU!Yj| zkQ4Dfo=3|HK&J4-=0&X}vc;)A#(n$isB5TOeF!0j7EwgIkEC%M;0_RW_a6ce?Oi{s zDkDRVwu96rl(QaGom_quWYd26#^gkxs$sHlmz*UkIRGjkOzEnd<88{A_+kt1M}IBa zi8-ZpRaJDr$8;7omJpwD@3iNGg)L_JEA=&#ta7(p_Wd&m*}ekUwo4yZj6*JdtUm8v zpf)+8>`65`r~L6liehKvT1iyl>~avw9sOCf!>btE0C#twLtG!D7)oz9Q&=#BGIP8> zx;Cofjw4vFdu!`E?7&@pAF&^v-0YeJ_*?7#Je1YTw>l7k{4YZ0K9IW^Om;$cWXR?87?7CcFBzJFzffwLqWI$9`bwu2>}3^Y@c3;UZ=IEn zM{;K`D*j8zdKJznBE3EU^N3;3LO^X!YU^xkHch+2hq-)yZu(*M-&!a}^GvQL&0Hz#eK6yNxdnPXiOK z@Il>bmgp&`{@tyyI)*wJcbGKNgd4s`KOfkc*UgfHT_Y`iNI$2fKt}%a ztW1PvN%xtHtG=q;n&YaTu;4f6l>2gcaRNrrNy^2-o(QYAF)a@H_5w`K<=b;H$2_n| zieD1y0{pKL8^@=+<4;xDqH2p4U392nZej~Jyl4+k1k)FIQ~YECmwdj{e0X4+_)(>ti{rHC zSNmr5Ouj`cQ{VdD0Adnb}lyX&ZN4p+hUvsW(UVt-f&mbMi~nzCAHvShtjt;{Pt@S#+$>3@(r4F zcBnIcGSKKt`Z6kpGI}YRqNBGhG?VLvcD6v$W{>+P6E3AjBYOBVq36z?C^ z-FIqcGQb{m*-3Q#A()^_rBN93kj_N282saY7&WZkvuU9?g>K{Z9w+SH%j`jid`(gP zlx|w7r&aHRf{bP3#UjT`F5E%UA56e+Ju{@D@AKHiJ(ih8OD|WEz11ZLkx}Z%-wbK1 z+t|sCiw^ESHFKCt8M6xM6P80T1Wy+dXh#dzXk?$3$HlIf@*KuP5S%y~CJ(0P)B2_{ zv|k1_Y#D58tgY^&qQfP^IFqB^RGg+*0Z#9G!M#&XMV7%_JGy8hS zEZmP0k7WW8(P*1btNZyC;v<~eP;@N*j*xSup6I0;!+PbVygYPomN$213h`(N?8=j# z#}kd{AR|;{;hEl>OWEW7I>ivY*`)?;J{0$#oG9^Z;x^A8Egg(lov%#OTj$1h2(mfJ zYyWcTz~-n49^cbMQOb`KnWBR!nsjUv!frxFu3nRJKms}7jo0unhDnPNwzQ_eQonW$H zeo`m9*qdsg0*QF0VoTU1^oaSh_g((em?lyR1CSxKJ!S*wHo5E@jL%#kfCv08D7`Zd zOxFjJ9$$-;R7h(Vs2IcROty!U+r%3aS%(|Vk7^HhpRUIL+iEj;0w9ezluaj~Q87Uu z=>l19RBC;w9{xAY?KEKR-rp}RH`*oAsKLA)njR~IV{`T81WI0eB!1Amg zeC(0`rRHd9u_j5W^R+Ts-fIjU*kF6mdp0w3yKN!n9l9ZL@9WOr5>_CsfSc+)N~_}3 zoO^mOfOY=HWE{a!-#24;dg|-B=O5D!IRoDF&sWJ($v*^Kawr)8u^ZNx=YQ?5_517p z&4(8*)~gn-14qJUGUa0g{z54?XZ9|ese!G35yv4s;9cy+Tx_Xv*dRW@2K|1Q`cD1@ z06_Zg|Do)1SMqu}%`CbP0cySJLl06!>IT8hLHA$gTVM|KZZ6mki zN_Sw-JYE%S^!#lj;4^TA9`gL6PY+_LIU&>_I-VE5DBAPD^EnT^b5uj8k3ng}VKdcl z*2_jH?@USCZPI1CXzy%mx`j#%s2;Rf$=u?Nm!3?d>*?{2&XPCo3QPng>^AXl0mYbh zXopA_aPl|7IN{-KyCk*A|5o^~Y^k=lfBvPlLH2K@{}dBV zU2KEhOnntl`iC@RTMl5q%M;Jk%N2%5A>V3eBY7i*a*j!KGe8BP56@-mUVA-cVwT06 zNT(JQlDL-Wn_d0xv;&b(3kmbsnJzFU@4QPVT@&-#{Lj(Z@t{=&E=snal#kWe>5z|R z6{dH*HypJ4o(x3VPN^_6Qf!&M{kY|(kxT^M0G5`EJ2VLwkX}2rdd0IQ{VlR>QSr%q zFKa^r(8HQO=o)NQ(XH&VCr!-J^37HsAH;-X!iIWh(}0|^_ZQ-IgSeTJ8DL5WL+6xn za3ux@7f^gWV+MeSD(V!I)Oo(kvxt0j9~(Cj6#b+!npM*o*`)Lfne5TgJR1ecRzV)| z7}q)-5KDzeDu%wC*=HdkOQUX1Hl5;*rtQXj{C4JNl>K+evi zmQJ@9UgUM~)k}3pgO}$92msSjHnSd$D5cR6{?H>_$k61(HMnU|II@G*g;pd7e z`i4}5CaA8x59u5vAAXWeQt!Vs7nc<1$~CoMc^umqE7J&rlqdFf8T*N4pC2V$@^T)- z?HAq4=f!F(&I!g}N~t^q+a8EsxmOjHer+>y$Tl|qvuS_{Z0T`HdlmoGvKL6N`44;2NNmfKI|WWpr2&=WLQIwntFuS+!uLLG+;Qs3*6O- zsO`DsB2gY=lgqQ*w`aJbQ~YZS3ljxt1p0;v>H&u2laoB#;ZteV)t)5UY{<# zwifhy3P1vr6d^~E{n?2+eb#>I>xI2vwHYk=S5H0|?1InrQS0TA+?bs~)?+FCsLi`O z@yFSti&3?X3+tig|H1?#g;LY@+vEqPtaKIMZ9|`pZn9YQ>Fugb^qcJ*l~pAXXoR#R zd=r=|QN+f~}C zEbDlZB!Q{_-I_jt4AQrbGiF~vr^t!~<%qt=c&{=5=t@T`sEaHgWSILfJnezv9G6WS zKx2Ns)rKczCa{b`709MZl!nvk;2+f?&4;8MgStmY>1hRea7-OI^3!UY1?OL=K*2gc zl9cF4sYUb?*islN`{0}6LSGY%iAq2H87@gC!@&DT^URkH57l$PtYO+y7t%X92NaAG@S77hAB- z#+@3;gzQZPyqA-%DzsN$K9jUt`F1epHmc&>hzI~hW7j2dXJDBvMZjooc_Zwno2Ls3 zsgHuX1W0a!!UA=|i^4oh^bVNl;7pG5>B*CQ{-A<80H#U<8|~V^!C7LL0DfX{_(yi? zG0l41U?!(Hn`(%Tb$-!_9DX5JP8zV7N~vjn9wvn&7WyCPV9pM`qafzro9&i}+gp8U znGB5{n0bt2QIFf__{JP!CX^qOjCa)#iGAhzn;kI9#SbT1+zr%TC(LU}z?-ttlz6JY zHwxUUmG+@2GlOU9+u&Gs9L`99e4ii!l&_< zA9d*QE1Yf*Yyr*!M}V;wcinUvBq`*H(~Blh{s{$15g^ zGsHxcptw5<7izlR{a(d*UGMX{|-prSI=|%ZL$02_};3JP*#23 zy37;{+4__0iu^6VqhhlL(bME{8qeo1`pvgDCMrJh&NoX=1O#m=X{WgeWMJi{#ftOw z6n`!0i`T2Pw)`6^M0!k*WdcwE7BgtU3P6R5LO1kekdrTSdDR19+5(P)qx5O}Yo8)d z5theq&AYZFa^pUvygCN@)+lW8&ao*bfrKSbcaJ;bv0^z)#rk@$Gs&kDLu(0a)^}Hr z9Ku+wfhj@n5xBBL&VNA-ipYIE)1F>ie(5uyfBj-W?m7Ie)ck8K_|}&_77CpqA&7X% zHd7#OfRRtt=dO9$D8}0$d71#c-d~av^)XO-P?0-OknIkBT~Wq7g%6J5*K52AHk(CwJ>JcouUM= zOPb#;F@s&Y`{`ZTC58zw!nAVet^og#c{GJY*}M>1w=H9|wRK$ruNDq;0d@3YAcLT> zq!tGFhrE#?5s~-m@`3<%rkR1(_SfOSF}o_wF+cg(!Ty1v8`3acXc7JFcv!9FOv&RJ zqxEzi(mvV+l{&-?A|}+fdAo#ovn-9%&`)T`$figJCOU%DE%<}K zkgip?KXrEI#)wFoc~xx80>?2zcA=P{8RJdXldVPVrc*cC(&~Vi><=a_M2Z z@DaP2o9_bdIHyqJFA7>QVwCM<2FuR-Jg;t$6uu>~d(NFLtHe4=SV@lIbBTgcp-1qc zgBs?T!@H^m=nY>^PqCZe0}qp8 z-mfTN-Xh`M*?Y-2KHJOcAwXh*L)p|lxku%_YWXU=-?G0U2vH~%ZXF0pRkq=l+~tER zEObKGJA>0f-qyNFDAktJJ{tf*RR4`2>Sl9`64uBJC~ zbfaN^BM875CGz)jcfU@tzhf0Esb9>sfZtt&dg*8KV}X z<2R|oCkp}F1;p%Ga>#~nm&ToY@GFw(u_nj_!FZR@NwUK^HjV7X9-kILlZ>@7N@X+E zt1m0v4ZZZ*aaW8;u*%je9DxBah%IY#`9O)zmC@L7AX9FhTuQ64Rj{5%UFfUEiCW~s zt&$+wSA+DYyrY$y)9ox@=aW=E!DoP$yb8QShJpO%s}K3D0>rs*eEQGsuZE)Gw{84n(V7{KQtJcU%gJRG)Inx37s1jIfJ5W(j?$-J~RgF z1#JIZuRCc(Oq(8ie3k+op2kS+#QY-TvqK?c#cUK(*rv%3I~nU6VcM1HxAvQvzTkpA z@_fKXtWU?cjPY(atDGcxdHOc}0yeF|dM+pZT}k-UOq8$nThc7=XP0RqnRMtW3W7#_?yG z|0j*mG?=}i5jR=*>!N>qzFo2s)}{O;^o@|GLO5y=mMbWr@$pd1iTCr9Qh%y{)r}y- zjT_1@Mko##d5{ZP-Xp}z95_O__#qClG1@YTn(S+^2yX=@X`Rw9x9ucq_4Lqk#%@l} z3s3lr_xUC9PE*WLLmAD#D9B?X;1_k?#QCphCS#`sh}z+!Drb`jS-E^cfh=O2t5QV9zbc*OHg_XNbe{} zhtPY82qX|72mwL}N$$p@=k*=;|L=dlJHCJ1G48l)494EsYwfkxo^!4>pJ%SgU-$Gh z&Yt2p#lXODR!j4a0RzMDW(*8RPyT)acrt_v3;`}DJvGg|7#Ns7AO0PQ6Jp|IV7SDf zbw}06Z)$B8o@hL3D@NOB_|vwC9IcH|AC_2$jaa-Ae+<9ubnZfhh+XEy^?AYftOcre z6{_MKCrvot-j#~}vvgBQQ|gO$JHO_?)URf+O3NScI1Je@pVjlInLl~{B>GnGMx?*O z(W6`!&$Qr(ip1w|v~plc5d!ZwzuJtc`5*$!?t5`F%@_Q6KOuhk^pE?HD~Dh6 zce$wA`P+}%jNfmw{(P){uI%US^`}wCf848_e=hvQ2NLL**Ys=ak6%=_uYzh6(bA6&W;sB-KeE}xaw?)NVs9=(OLGU??X8ftsY9ZMP6vlQI1K8=x|REsSZFmenvjHYl7?cu|unt)d~lH-yMbFkJlXQLAY!jYA+v(9^;)((2c z@k6DL+_09Yq`42+oT5ZuHPk*f%^?uB7R06RmY^#Q+;8Zemo#&8{+8|UiB3HD)PXmdOTQpilWR8+WYa$9tsyF3 zAn5F&CnzK>vTmMY;5%xT^r515Ins!!S<2T^^4cie*ZRXQ){JG2m^}5)&`;L>^A!qjG@}3ow5DRB3+XVp1-WKnvW` z@b}#Pd`5`N4J-!=M^rIhtg5WFF?p(*k3zZ7b3=|F=fGOcN!mR6IG~gK2+BjfZ_C>( zRmtc}O2KBAODv@b4HfNA#8S65XD}?Kc{+XUB<+Ip9JPD(wB(+JW15MZ`AKT3X(y;&K|crs@{I%pnr$=ON)<<7p(^Cf%vC z?zPrZNb(s@>1#+bW$FxjY4L?(?O8VEIQIT3=<76U<_In5E{aBKbGEcjqJ>6w$|dOa ztsFl(ze@96KwD|A){W4=&}Qgq9p!6~)uGfZmw|yZoO1FDXs)>A0(V43N72y*kDf?+ zfN|4S_H?=5&PyYvKVr@=i1ozf`Y<`NqM07M<%p&xT^M5<({kA71*d2w=;^F~GMi z6H$A)XUbK$`Od}G<#Lo?UWX}rw5YQOjxYryX6r7$?pb|IdfHL8Ms9c0 zjQPF-?7a4qkXq~W3VR1HUkW#o2)z~!!e?DmVw`6V?Dax~IR04W)wn}sjX>#i9) zq`{PBtr33yfruFo|7ua${h1BLlioj=TCvAo+s@ycHFQSQp79O*R*zVO0K8gm}{ z)!@Q3(zO{=7hFC1W@C+#w@_Y~kV}{&n{BrUt@r0VRhCDG+jf2L*f!z@#YU zTEbFYFJ-qynC`5`G49*orvh_Per)feuOyJm>wsng%((yR;g;I~5u}%0Q;X$3^Ij%* zeolKU(rY!amtmlx>asR=HtH&~Snq@@U7C%%eBnsqC=u1|ubtq_l`FRj_V{z(`Ph1`jOva9c8%8-jpep5wfLeY8Eu~6AO+;c=3fn-8!yN+Ga@}Tx2nNa{6aZ% z2}zmNkD^|mt%YG(mmbnDcYIJR7m8TZKMfetoMd023E`V5 zewEd#ax}z*nUhn3e%jRG6P_Chog8ZURNTD{a&Xh54JZg7OtsbGpB6LbR*6nut=(C= z!urb3$3ODZm+m8#tj z)nst%f%$}}#R=i6gSDfETWz777~9qoZR*z3)eVJfnKT^J%VlPopPoWdV7c>(&?Ifr z)SlL`G`&WAO%(0l=54C(k(4?p^+lklhY?~Yf^c93yf3Re$Ch@)$&Zg6q3)+;e{ZdCZs|v76w!kdpxVd*K z8KPz#XQf4J6o)r3n`|nr4Lyg(Rt!$OF%DNy2oP*2+{m;R?cGL)w0#t>r9I(=?Q|)C z5N@Uex#Xp3Nw{jO@~}W)&<2;JobMP+w02w5F&*q#qU__2Rj&MmxCLk_kpB=|*^pf4 zqH=K1ejKV;$nTMja?mnsbG$n^x*m$>aYHEvJ;aW{m=3ICr&QAv#vr?DLI%Az@9os2 z5T6Mk(USV~>ycpPD^?fIoyqI#%|)U!4I@P+C=)tX6XIWn?O4OHMSEirPw(AbZ>v7Im+0A8>$u$FY#K5?@kS-O`KWDOYK&zWVf*RMy9*z zJW@M2zO$7X7#RP^&dOw4;?2at!7RCZ2*uhTby61hiMMjRi#cG#)r%ED*_7Hj@b33n zQQ(htL^L;2gT0|SuTg}l@?e76hUdBcorAA!Py3jeyPr5HphEF2`TN#ZX*pNWR`C5^ zcQBon1ZMU9TQxYvJCg z#!Xt3RH6mHm3z@asDbKDJSN8h?(-myo6e&s7ppw^i0WX>9)M3`pQ>Egl9Jg8o|9WR z1y;a@DNfyg=`LDdy3RO9@CP?MqZD3ejOc!@HEN;6i)ZUYzb#jey8wdO=JZ?4HDl+8)Q4zoR_5nOTS(+r=zHO|Iv3wcd;;v;VD4C$o6p-n!HeSm$LQaf5ubsREV zStH)T8*w}RrQs!=d#Y*@E-frc{uFra)`BsOauR^Hd?ydLhx(Z^c4YUX4qglZ@0z=E z#p-iA#1@1=a&mT6e_VGD-pm(Xo`T##9VBeB4~a7v$~hT{*4yilN)T*{#-B6Di8{&% zHsSJ8TQfYXH`1*WN0&3y7_Z>1E|Q=!bH=VVOUPIe?-eyEz{J-4jHuh;0*=c^)Z=Xz zM|2b#iK#~o1KSgMpv;Q&WJ~!V8^N9!uA5Soxh!X<+}DbQjoHPE120jAG6T@)QtSDz zG5sj?s`|wbTigA(GIT(Gk={Beb=XS4nXtsR*=M$CL<+Gup^vU0hWdyY-h)jf>NI&e zxhN}NOVF$5W1f?`ri9&h3cjq*QqlA3hNL9z8#+r9cGqf`oH{4X*U_T5^HqV8kMNq0 zV4^q5w|!;{7_kQTLw%%Fl=g-!eK$WSp{V<=^%P1y#~p;m-NkO94)3!eOb62+Rt6VN z==D~hx8jW*mNDt^$?H`+3d7qI#V#~V~m}Vdx&We$_3K!gK zPaYL;oO*jdpHL^}BDsVAYjUvVdY_QBrWLRa-Hb&wUCBHF<5~XCj*CCyWf(w=*v)f|e& z$~)K@e`>ZIHCCFvX6a+V@Tz~UYvwhLF}J}?LFT?_-qpDS4{CwZZh9U&%)X%dk9$Q2 z%cZG9v+~YPLCgtfR^E#I3rtHlpZDp=+?B}|C`PUA6ReZ){sdcgzD%sLmJzi>CJZv306*O*A7 zQbH=smf#QbEoK}hCf)PjidLeSb@;aCH7VTsbKX}Ab6;+&RK(|cVOO+(DJvqv_ZxhK z-d|8ZUFiJCQRg5f^%k~8hwf+<-#E2BC)!%ZxYKo%-e9wv7Dx2zVn=pbBVjN{rG{HO zh*wL_z%rhD^4J)-G_VOWj@aLW!Ss~jK`xQS1eniXXPZU;$PR#+pyh+deBMl?Vyk^T zhps@lC!%4xua8?u=M!L2q{wpXPnRr-OdXdB4V7#J@Fm{Ovs-k6Oc6M}E^q_gkh@y$ zu~1)_t9sP8s!BF1-$tdfj;?n?ZA>=1zC+4>%&Y@aujII3vC$-X0I?f$=wzadT0q%6 zxD%NPHtjL=FUs~S*A*7vQtkD&Wg~`}S*JvUJj@p>EJpZaW0A7_ zt$(q5{^#vq`3cUJ-&Po#h>3D72DtS8NrqE?&xls@+4wJjvYbYnkldQ7U-nK z8U)i~<=4W{*nRNoN#478BM(H50o$qCkL?tozxLSWSt8I#UNy`6N!EU8_;^a_ zS-km)=T|CZvI$e#63sT}svhrU*RGXROTK7p-|tNe(jt(}aGjn2dsMzsiBOOippr(D zlpc)*Dx|c5Gq%*E*>~Ro99K!)*`vD*QLkvoD&;ey?Th=%v?JHCG+=6%be|qkM~gP1 zf7uP%sd|x*ap}r9n{O?pkRV*Az=vw$8%HX7@~%8klUN~WNNmrQ*zZ$&s+macmI)3- zR9{)>M*m)tm&A%Ev;HA%;X9-)YvQ|w9En-^@P$sHtuG>#lNVm5?K2nYk;KwDhbG>h znJlnr?y95X=QtizV$!W9`-CLuIfM{(0K*)@aTNPqJo5SkDN(hN_OFB4!tRf)G2RD{ z;cX@0em(U|`vfF`M^eg=x2Coxo8JeuW9Vh1TUJ3dU5EJxt$aSiS(}D@&|D{samUXU zDQ-n^>%GIQR?0#(dz48Ro%EAuGDoW=U2hLBl-D0+g-7;{K+!tu zq4IN<(<62mRm2#E_`a|%0Wv0#Q_+7ZYW>)vAfw|1L;$yaJdFY{vV_%`PfDcXwR-w) zKWR0m$Zx#=`8aW3NWrh{KudQ>uD?%6cV*{#bAT-g!5e3s?YNztL>!MTCnS_C3V?!y z!;uf-S(F^jC!?;)_VtHFCpbDSDGv$s>_e8HC;4x$VWp&nyYzW1V(PMcfN<59Cd@=s zAgx$5x0gNk2_Wc+^0x&*2+l@dd^yT?mQzf4V=18-aS0)FefFwFbRAO4qf(oAI+E>7 zktxHUGHYAw3C@H*S>M} zg~w+FhxfSBR?328XW<1o0Uh%p)#eyXe@IRTsvC!8nXx{Ua#E~h*2b=KrKnJP{>EGp z_cV8@U#@vf_3jIt`qJfmxtu_+Nh&Ef*rMk|v2$W>!%$F{Poh?L#sL zSd&;EbWN(clt0|Jv|{iH@4n|JrP^C!4RuTKx-0skF2Ng3k==sBh=GBTbcC))?FPl) zaqwu8e*jF?AFMyT4=>e^H+KADoB4PjWR;-6M!OCQUVLLI-*Cyftn5bfCh{&2*5c7T z44F>Y83k`I$Thm&1W3wO+jRN+e7j3UwKsY^nr=d@q=ksQ7SZjnz74BSCsL!P^nI8Q zMlc82y?#ZPIn84$WFWJJr@cnwi1mzPjA^k`d>DPhRJtgX#RTH22ELx!^$_kATpehx zs0LELW+gr99hifF``4ue`8&?~>2p|vQv)4!x@wWNf-u+9pkRCD-~eAGu_U>xUP$Pl zUK;gMB584_(RK3W*inZzJZ!zB`2uf2pZbM@`f6Fx{)m%oj)b9hZ&>RXR!Rv3GS#gs z?T}zQ+P@dn&SUvmE*XnEJuaf7)}gZc5;?k*W5MdftjVk>z{fnjZ)Z#|_Iwk~;=HFG zXqKFUL68u{o^nZt_N}L?d-#Yne=53PZ)j%7?h>L^KY{c>Fl&j_XsSn9W8?-uNHXS> z$CtsN!Qk!wlbISY%(Frq&dboFzpRClT`8w4*57Z;)=x;~ywdfGYabIRXW6kPvCbhu zsj8Eb@tun-C$>%FeQ8NIy&h4-W85an2x0r8&fPPs9-UuDxf#tcQOC!@>15q{GR4Wx zkR!O}W;NK?RM7#FK(FGfhil2d%M36s}pvMtAU< z-P4ah1uUXno$nNRmn%-qYbxuqnWq;;#gf=VT8xn-Og_xrpV=R2nUQyTNN5NQa+!;r zWGj$(IF!x48_QYdW)~xs;Dg$9glp3|`xxI^*Wv z`!xmCH`8*Y(=?^b)>oQ*9|w3I7!dANytDB126_AG9EtlDAvIMhqa#;xy8-}(~ z6vlmR=G*6@RLbWfa6f46U#oIn5A^24|T)764N0)Mu{{#B+?{Fw1h5^nyoSSiDTFOTOe*a5oVC6WCkHf^*nPC%RLhaq&X)D{r-ne8sH9k}P-ru{;4U~h33p~vzVxP( z4sFT0!=9p)BVRc_x5sppmAm^y&h9>=R}Wduu(CBx$k3}>nmjh%o|^-G6v`XW{&~d1 z&WDD7RK2+Gel~CHOiv=%F~DhVZ%lb-CQpu<9Kk&@YmKwsl9eOuEdWji3dGBRp!`g= z5N&m5F7RtK->G<`>v=uc>Rkd(ee&Z7nAm42Usmu67)o*;+skfR{<=EUaSiuo*0D6t zWpOV;>9NLAe%D^oxLYPc5NH!->8*1f{&41XQq^nM0!6=|9B|N@$ANM|_p2K(`YVBR zih#R%it~G1z6YCfsNII%JzGOlvnCmbJ0<;bfG*-k?GJhZ1_R>+JnCnC*l5pRkRWBE zsn2eltAv*Y^$c*B#H{mu!!3UcP}hD;iU!0w`U7u?79FLEFEHYjxnh?)7q79=bcOFk z6G^@BARLIaL~BSu@Ou&R1$D0Xihvk=M4hK2!NODH8n@X_Q$GGlCIqV>tK0kP8O8TZ zm`V12-u+sq_D28h=xK>sSO2i3e2b|H%Bi91YYWZ0nuHYbT8&0*2!13eNJq4B<%y!a zyG}v%`3K90-t1cFynB+o#~qgw-pouUXA8PsJuK}!Yn1GfS+V_ z%YM4t)`+P~y&Q_pii8gG($^w~8^n}k_km|#gw7GeP@fTN1G$WHYBN%!St`eM92?38 zQHRD!W&lxxhAX+ebN-;iLHP`b0+e-Xe|XcV9Cc~~HwXmE4!YC~CG$!xjq{a@GN4G1 zl(BVxw>YTC)_TbN|4m?%B(?!Mqdts5+BZoC#z#i z5@=oucl>^uX%kuMF<2=_Dljwq*19i+ntj+EjfFWsxGC!k(=9SgO2&iVpLb=0?AMw? z##+M-n-3#>r{`SV(RHSP!-UV46OCw*E=LZ}QNv77ho{2~6BmqjFrY%Dd7_!V$jm1C z8M>>pHNc-5LL)DXC)j^71^$?r4bo2Xz0C7Fe9!T4M*$qn?{ONs=<#;r4Zn`24(!ls zM@~*r^|Ld5i%E9n3zt2#tvw&m0jM&4Nb1MGqkEq%j8m)BZ+WD*J-dhZf?D7*mbe1dn1u?jaPXqD?+JW zruBqz{$pSgK7V#?$a=ULo;{q=VVQ;;o|k)`IspXy?_2FwLMdC%7;3U9ND^!1TvH-Zy%Q)Et>{9VoPQSbVV8(K(Z zxPh^8TsL)hbyJ=jwh62@Qkc|jZ7;82>Z}xFb98omY-Loqot)a&a1UD z_xN%%Wp?A(&+LFNRxYltJ)F>n=&`Xex48y6cX#*p1%fqE=Co=|u!4Jvtncf-jnxI>#%h#0WZ^q|KkNhOU`qU18}@0GdKolw&_Wa- z_utFJpYv-h*Fa7J|D#k08zINiiRAxM?!^CEwf~Wp(NYQ|O-Qdn?^V{(N0DMu$#~H{G3snUX;RtZdbUt{Kn4nT83o zc7>Oa(KXJkb(YoPGP%A>f||udLGfeqbuQb_GA_Pk5a!)=b{-7H&j1 z#E)^Og_lr*UNTQKwmJE)KM7iS89H3!96mlSokL4%9{~~VU2>_~X!(kVxo;@b$hkN~ z)BI-9jiAlf()kEST|ClXI2@gtE(R}p+A1;tJ`WcQhCt>f$~2 zOz;*y*iwBgCvfByV4`9I4>ZRrQm`^}hz z@`6GA^8Rkf-BxY0N?O*LTQ;k$XKpQ>`R=z5rM7E!n^h}Dn%s7J)}n*Ue3rOAgin>B z_H5|h-#?2M=Pm3ULVCZvGS^!G;Xwm`dND@Zpeq7&YJNU-?=o5CLwMtXkNYQy%R*IBgb2=b8W43z+={J=i@EQ_3pZQqh z+qzn7^l`Si)NEJG!uz-tfx~$LWv#x~e(>FU{g^iy>ODE}{4DjP=f%_hCHgRzk_IJ# z600~!x@D?Bykdg!s(Ao`*zQH%Z^)MW8Z$pxdtC7)u54NWx10vvS$M@Ir2_il;A}7s z*8ri@Q<%>Z<^tw%u2p``SHANpKb<)WXV4;d@Mo5wp?_`mOmaLK@??_L7Vq9s6?k5+EOb z7HRf_4`^TWvgLR)vu&d6w5)7%QoFh=b`9+X-ty#-T|NJEa#w*Bx$HZF!B!ka_NbKz zbR`S#6}<7^bPvR%#(;$)`EBZi{G^=5RRsMoL{>2*ho*o(>YL%qbW{- zl{*HZ_(7gGPqWO~9c)Q60FH{&{O+96HmF(bx?rs&pwHi>am@kEBCxn@i0)Jc`Dehf z;ksf==t2Q3x)=?Po?p0|ZN7b&M0i-L9^Ps+Y`ZK+4C|5$ps%lc-_vVahU1%2v)(e4DUh~nR*6e*fwYnMQ`QW#s5VV?K&Gi#rAvtAM!U0_47sdzF6CTdllxM z02#-nJpG$!pE@3IwA3k*ruHkcteN(|oo6=-G;skPARc3Ud8dQr zf)DQT&&za65_{>@g0EN2V|X0oRd|0)G{pTnncDu^ePg_(y0S96xLBmP zxR?xT-j5sm;dVo;7W-av5;YLp8Y|-^*8AHlvEAL>Z{y;!v$FWFUcE}OPkZo>i~jFP zYWy!c=AWD0e+f1I3g?#J0r2hW_QR5sW=19^F1kNtGjZJ9!om=MUokQNio0(E)NZc~ zYqV0?(5b8|0UySn3wH|%4&~}20qmDp^%Jr97+6?Xm?x(Y+c}~3UKSSP zI;@Tr%O1(NWKD0&-z7K-dMjfKJgg6H;b#(xvrSjhp64G{cbv6BXd*CNft1`Kzt!`#! zc8CPc%#y;w!ruD!I(-8FtLzZHI(HadU0PaCFfun!9vd_HCn0s>iHI7jgM-8Rk?^f8 zl!Mytj+t;wqkU~umff}Rry=UjW@b||P&BqL<%Q6XS!Vp67R&Y4yXe}W+@z}Wr zvPQ>i;aLh!gK~xHfc(aP+G;60O*IT%s*Xd3sMl{b=D&;emiDmyC!zMbo^8hao8F(m z7t5C2W?JYhr9b76X8K3Lxzc5mCNZgs(x29{mZ;LU8+RDH|6W44rwbj+oSZ$-8^NS{ zx2cM*wZm-6!%}(y(8Hl_jf{=6A2(A28uT=*4uTij8?U?`7@A47rtgop{A~P96tIBB zXS@eCf`UST@eJ*BfGaFJfld|v@rZs2Q&Ur0fK6moR!XPHLiH>xQa1OPvIDh`LZ>RU z{;@F8Z(|lvK;>7V&D5$4^(R{-%5iU*toDCaW^Qe*9H52|Pq((V-o{t;FO-UjaT_$r zT0Rf?=QGWrjXNeWS$`YPjT<*6bC0rify@5kTBWYiva;1;>_bz@eJQeqQESI9pVofy z_m=}$oU}U)#jRIk^ro?WR(3?n{@wva-hOb>@1N}DQrAkKUYifKm+Yh0rtP-BM{>Wv zUS)7A@I@E%)IZ4n*EKg0#x7iS?SF*i|9hzLJ=7`9n1g9;aJ}wMzWQeunyD*RWE>|?e{u^I^B{H~GL}V4>L^?8T_F$U7AT-A zv!(r=b}Jfc-e<3Ri2;d_SG%y4{tw64wqCP{SOj7Fiu@g%c>)}q(=Dh=y?ecvr*Fyp zJx_+xt6uEhX2|UFYNiL<+m7k=3*ogZNjp|0+X^m5dW7K*o5KRcwUU6~hkZi&IUg1< zEPQXe-=*chto#Auow^F3`aP)-%uG8)48|xNV^3Fmj6M0xFnX~M$J@!xy>WJL5hsaj z@=rowX32OE*C0#51L7>;ALWaGGPyR1SoP3&Q=aZr)WB_ZCb{)3%OL!hGkKckW2oL1 zM7=n3<5)}4#ezl%uaq-(=ZVCzD@HV(#oF$Y4uDs+}dcLDRt6~i5I5M4Miwnv(79R4*)=^ zcK8}@k*6oDsNYLpbo~L(;;RU`o#mj63TiTwysft$jg(-J!0&&yn0=tlmQn~=P?0@tv~eI$H0b}190uj!5x=KyyX>@+3>e7pg1 zJ7fVzCAPx+lR!`~w!dDHj{^12{-Mi{3buzxSgZ`6L@2%x%0w_X$<#)ElKWa?Kgc{S zi=>oDcZr8qK#h2#FEA=bK}&RpzS)EQs9C8&HbZ;E0(HN!jUob#3KaN6nyoBKbvnC) z7GfTe+huxo4T3id?e~sYG<)^F8=Y=0ir+CK60Pk=3*z!i5I4jc%dSCP+ZJ?OXKcP+}d9(`)NSSGc~>5Ghy}A z1FmQYO_@E!8CVD_NilU7Hb$Md4QL7*B5SA{ZW5m!^} zv}|wpd-PTH{nWS78oSf@KQy=$8Wu2xTVKP_uiCzGk)lO*i}Sj^M8Z5O><2~4%a(D_ zDyT=Wq_jA1dN$EFUE!~#i z<_s`LbJj^=9t;ADt&pcqnGG~DozB8ei$tQMz&X#CeHTy4)%Fh&Zr+Uvi`DPcs5U>9 zW&K8ZDS~S|mT^jrIR)IEW~$$t)|loxlV`M{b1_H1*}FJsi`eEL)Fv+HyL_aJ6Nt!+ zZ}f&AuitGF{;A*rPDwNG(a#|fsftPvp1Q^X)n+cG6>G0eacTgf{-Ni~NWp_3bv+=p zVb1Q%Grhi%ex9`8X+=}@0{wbHHm+eP4rMoxmkPg)gN)e^nu@t0&oA&cr$U?Kx2K!y zrAnyy@hR>f<{8Fse?LzjrbiiS#;uy_b#`eOX_+9o%chusKv4zTk3P(g zI>mFdT_ca=IxK#150yE=MIr|g#|j^hJ$hz1+w9eRxc9eujm25_MCkKwF=3OI6Ef-Z z)?Pc1m$)dj-LBlFP%d8=QCfHNQi}DOsPzIV9aEdF`C}e7ye8jc{&OWSpsudkhsf2! zf`|y-5&Cxku``8J78^-hjvOknZU)e>KwMnhA$SA=lK@8A?UEOd{Ruq&AEWXAS2@Ih z8i2+ATy6tjD9bA<8cF^ko6^%2RaHQ+^N{ceW_|=}mxpsB|Di~}`2cZh##CsjeCMNMf6d4urBIBnf-HJWq!|2FHN+hbFiAbe)YcXX&YHv%*iiX~j9&+8j zon>G1^!GELMq7*?3;Aj2W||ZkYd*;uM4Hg@^OX?MPe-1s zG+sJkda{rs_oYuVl)VO}_ChUYgBQ2f2!bHt=t-Q$?SG|5R$9i`BjLNWwSLG(KW&W^ zj5u}rw6T#?=vz~o;)asdH;S{^!RT_U_L(o`Uh14(o^9e zTncY#X>kfszaOpS-dUU8cAMkU@C(fkI;?j?)H6!&&01OE@%Sq>KbgRx$a#zS8IOOM zyo2s-uh zK+C-mjPFPJDEkk;G`R%QLn;>!=js>cUFR^l2dGx%QgX+*s#Y%gFeW%2XP4qSO25nZsebOU*!M+-VY7<%U*PRu^BLP>DKDZ2F~#CKmmXAplMFoG#r*#F zfBswZu=3lJOFx5Cp)h7|i?Zv%?P1r4oF_@^Pl0t_Ya^8KYxn3^$(N?Hy=<_8X0=tFFZQ zocbBh0P4}UhxNS;>(`8}?#LZ}@Q?C)nTqvJOKZV8(Gc2@r4AE&RP?h7I8WwMje23x zb!=9geodDPTyXE^ZoTFMZr*jn;X*Ozo>1FJ<`+gL@JvD;dv1@Ipvld1lnDs;ehryL zY3D$c+ZXK1Ud}%!-2X)9ayS8ykGM7UcTQBgGQ+ zPF2h5JU4VYhb`Sa*Qz5TZf-Y`quB&F{)RAa_o~$!9nPYm$mm^27OHfL1RY8ikS#PG z8gO-V9&|S}0&~p{cK#*PNNtfQhLi5hRO!{w<@$IpJRVb^JX+t{Az4&Z*w($bxHv{b z&Sv%@9wiX=rUm2LnhcT1d(-K!189@QF2*tN`zgtIp0RL(0k61PIYifR^k#1F4*6sy z?^OQySkpyo8G|CO4deA1ZQ}sd0rCz`U?e!ptfW}HTarMr6HP+3zg%hGO{54^QVaCL zXP#VsC!t&GAMb)H@Z7BxdjAcnJcd3C(!0L;eCxoz+$Ev9ZM+Nhss-URI(AmywYBM{ z`!AbNz)4lsm-lv*0=A~@q@}*>tKG~R>M4YEv>J1eiHzGLKEzKxFFa_2zx6c;zb$S0 zay+|WuiLViw(8Qj7ppPq{h1QTNF`b9^d>$$z95sMV~nXWJq@yWS9OE*xde1>WMJM< znm+i>fI9r>2s1jtx?no}kK%V~5t!q*1v$OK3H6{N7W)0-K!{1_4BqBZZBCy>f!Hzp z&ImdvfpEzxXkpmg5-*s0xlOPnJ$nQ3!6=-yIm-v>T3E9#4yU^=Z91@8SWhOdB!eDj ziCS%yf)&4V%omK_Kh2v$_Qx`w@!YguS&vg&e^akK0|6>noBHQAs zmzx3w_^rP@9ol0E2g{lKd%ZD;9W(oEMBVsHu$tFVW2A{{Hj6M#bHOR0&hl~?%0IvB z@_y$M?WeYCbRV* zpYP}jS_6E)2hwf|HhOnb$)Ni@tbr6352UHO4xg8FxRc*~mtv-&DT}E5Q*9lopSLvH zAuV8noqogN;5%rXEGNF7lw>EgmJv~vrti8CYO5tsK_?eKqJRTz-T1(M4=Q0Wm;KLs zczv0|DwN*!a$X<=|937xC68f3O&r|yWqY#mLT2&cf`xAa0pU|P6=B62FnVV8(W9`q z$sN4tY;$ml{FTQ4P`<}8GtHR0E?+%yfB7wTI#OH9r$>3>z1KoyXnrZ?t+ucrZxsu$ zpLu@+EBrQix7zq|%Qg9m_07;qpTPQ-m==v{*Y&SUQ@e~fi|!^xwK%Xfue0$mbNXmi zrtOU&O;=K|+p#;E9Vz3>(c{cFib~DF7}Qw*!J6TnVqMTk^7{H5$8q5ZcVrwnIe}Ej z_xScY@7mByB&a}I;p&YdHr|5prf6(t3-At*86FjJGGMZR#f`2EJrj>6n?{D{pcy{+`nT#puC0Pk04#>-=5MEx(6f zUaaWI1fCy5z@uE!8j)u&2YtOq5IR{Wp}!}R5|f|YxO`W)_SOi60`D$&ydET|WhDU* zHb7)){Ts5WAU}Fv=DGc>CrE@%+5MQ{8K$mPLx*Q-N1GZ>>^SD>EmnxfbzeFz_Ko=N zR3BJTNcrOxKy9JSPSb@cqg!T8vJ%4OE`H;=5-(10f_#Pu|6br3C48N?W4!Mm6z315 zwo9&ESC)FOQ;u{R5DVT)WDRz$YiA0l?Pzv9f<%YkEx<~U0)&htTRI;O4x)}y8}mf} zD9~OA7B@FWjD|nq@8f>_r&3f*(-ohgDER-<*wx_-QH|{WF|AlAf@XU_)(K6Cdd4Bji zbAjxM-Gl;{u`|XrzgT6_^n%ILxdk(Oof$;S`3qPn9ya4uUe1E6FDmH}U*}2mqQzuc zaOSSHe|14ofp}1XBS_r1ys@CYUd(wtw?g&bX0*K8X>Uv3zDgaQYLuUgb9wh_3Ii^b z-fi6}tx_V}xhwuluaRKKSh1@iRN34d--p(n+SS_9l7532LV1>W^bZH%O;>Wrx8B7g zz`1Ooxb08!jqr)iVDYclM1oPO&3+H#+v87MT}Wiq?ly8Pa+uaIJJJ5U+I7+}Wh6Pr z=1p_89>)5U(J#Us;5}z=`#Th2?nMjEuHquxn+ST!``7YbF5^hW*9{oGpH%nTDf7yC zM$$43QeCjLd{RAvaX0$Hx{JSSHnum=(&EhyF2;E@JW>&Lwr^dY=PyiiBPY)&HANu+ z5e_&EFCKsOK+tP3FF21$$NaiErSoXOqP2N3U*T&|fRR{$8=}WTNE$LJ3Ykg>FYqS* z0ZH1$A&GmIp|n3XKSXPQ2SGFCDz0<#S-Th78SAdd2X9AJ73Gbm3RL57=`#OYw)-!h zf{==nv(~ar_4O8qbZKpEn}mz8w7zYRRB{?*FURXA&=k15E#s)Gyn0=QEp|nwS0lxT(r&*B&wd zAA)2AlBDjbsy;b&>eTf!tsy7NYO+yp+BH=z`>?1N{4f|SAK=IU`_0u$8|sP`*dgPPt0Vb_^23w)1e7Su z*LRrX-=@N#nF&NOjz0+rJ9S#x@0-n8c8NPUB%CHS8N>t0h*7lveg2Fn(>EA?tc~{f^xc@2RV+ADZIRPZ_|R`d8Lia^63A zTzd3L2s|U(X+RwU5%>>7%$jE^yDlsoapJ*6mi(Kn_K#4I58oNOfpF{#ATn}Y*vQv3 zp{}m(_g`ZV9~p(ZR8`$%P6|z`eoT~Dw3M-tFw@k_co8@HzP^5%_RDHw$Z|Yx@Zb~c z7~-9_E#mo>={vi;k|U#yNnB>>>Ozmz>7wD8N%5a$2C`cNGIC$Ru-au)~< z-S{^sU{EO$x_&+59QwDX!a=u!_DaW47P)ZB?h3FpF8<8^1-klD#IER9imJm4&eHmz z_%I<!UoF(!dMfWa{ZUts35n0 zjb*)OyLkWDTv3n{eTVh!8hc1}}D9fyT85P?~d_} zd&an9+;g6PVB}fPs&mb`)^GjhT+@FcS+_^p1OE7r>3U#oi@Wv*um9cu`1QY86FFI> z%Mpw6|3AvQw!*E0FLNxqE7JqIf;68Vx$nooPeG{qHLU!#>_UyqH>VzHvYdB_OLZ6g zZE-7bSQp~yL7K726YCPv>bGLDFY#C_s3(}g)QVIk-f~|6RiB^Z^-C?=$an3Yq95m?e838+28XGmwka#gQnpV!(lg`=8M8W$v+ZcK&Gcr*m_YlGtq-o9Zdh*{ zCNaOjH}CIOrIwdn&yzg^ywrkFaNG-{ZK$6vBZLr+^S`70y|Z@!d&p`6JAq$21Lo4A zs5jPFiRvPP5zGnsre3EPP?I|*Wt(l~^>557e#%Pa*xAamT>t!N>I3NK)Qun!oqHBH zJsX!WUnBvu#YG?O8dk%cs?kB3n7$9vQ;7?NzkBePS-0nX@b~k?J&~2Z>mSR0l5(Fn z2^^KK1rg!<*`A9>WLX!iMB&X(L6~;#t+mmt~GOO z{PVH-`Mahp)3USY@7|9h;@o9KKxI6tQ8~e9sFZtD$58-bBgR!nF-y&P4MtmwFU!}T z1I&~S@oT=3{}Nd5iW6c~**HB>S#G3|=V^2QxD&9UtnTfy!*OF5J+E*aEA+p40!IxOzZG+T97<=BU1$_seRV@J& z;Fef4^yFSvt|?T9#S~DHbR%Yy>}We=+^Fv|{7oBHPrngt2x-s??Hb-I%h-fZ3Bk&~ z!~jF7I>2m%k?Q#p?ygM2?((ufY>JFTQA{~NX{~Sb84hh3^I&w7jx7G-mXU{^((%>TvQHS z!l?ViI=J_fWoJZku~QnycA%LLxtgi?NIwYxS$a1&^-PhVve;2CcF)p;-shEQrf?Fb zL)@#R%ecz9+Yo%|@&{Qs1AlhgUHk1$DlT8eXNfm_+>JlD(v@FB1<1};vGx@`drBfn zx*UzpWXB5^QghFifmuTmT%BOugQAm!rFBRL?%6>$O{bY_vd?dYHaJOy;p#0wtNtJ< zMu>m#3(o>MumO1FKg5HLH}8tSZi^!WgIcxyuEUUeL5XIdPX@I86taQWlF zUa}f{jNqWb49}gy`#8)&Q=id%ywC21nSPJ+Q2;EI*~t1T zR=UmY>g!Y0L9RHRepGJQBhF}%k+dP|0sX!CN&8*7b*m^QE*FR421@y|()LMF{V;6T zeR2arIn7KNh;BxSuFdY5Sx0O=o6*~_@E%mXY$LTPq+Q=oCMbd@ltX|4vh)hp`Kv=C zxb})6ycQd4Yw=XfzrsrQV+B#qU5j>{w3P1H(a}@@A(Ly!{89CwajYbP;P;lE~jDne)zS-pc-|Xo_?-%I`GR< zOjl52DfiA(KO;uSn`E(qxzCo51;W^k0&`Uh57{Ehu4h_Oh3I+t1T&yD8^2qb8{Oxg zW{Vi5)HhF(&<`slB5>tIxRGV3s^~s^Y@C83KKU>kZgu`{(n`1j#|@Q+TRl8b@9?5X zy3)ZxL4#Svo@=%VG;1B(jxu6mVwGyn1F^`tmEvok>K)M^y^q=X+e~3@8l_w8Ojk-h z5A8MR0VcYx?WU4rRm9Uy6(@^Do7a3Q0tnQ<9X!aB>*zMoNF49oajbrq}^ zuho58^SaDdu=S;`0M!xjRqh^oIBY*Y;@A7JGNN~gH?bXCk)%I6l&|l4V&*@DpY+w( zUk!~;DZ46}W3EDi5Kg^b?{4>`dE>1X4#aJ)t=i;eUVszQsiHbK8J2Hm3ct=2cv3{J zG?+!onU03Jp-0QTf_{b}!0#wjQ`W-7O5FWGpSc7c+HltMvH;$#!Jo2e#BH!=MtW(*||j zP6(M62u>YPlj&@FUExZ?*}EL~zpI~%MPvWofqa!yj8z1Rok4Nu@7`*FVw*`zNAfkH zJ2dsXpbpt7LNU$w?o2QNN>}vhzC-X{LFd-oAoh3jGsoj0$o2_dyuk$+VAUA$BDAs6 zSI-G>QbyXd6EA)6h}$SP?I}(DUcZq=SmTN$H}~E5x#4cK;}kL}8_UZZ1k|fk$fJTm zE^^dY_~(`682uqSuCEXFHw^fX$0QrNuep9FPSlxh7lX(+j9%i6gaTpxMQQ!}PGzxg z%5g-Ve&qLUIBtIhj;k%uPUYz}FXUb+vR_Ut#61y|7i8e>x6@YWoc_0K&6)ANaP!$+ zu<@H%;#*V6s;SD*49hU>md-ct@O?Pa@hfJiqqFD0C(95=h-VY&a8eEKt;bW|sfgF$ zNqDbwlv2vWs$cbMTXs?juL=Me4t9Pi(D2{ezDgj~3txH|(mP);o4h@q#vTt?D4dFR zOzgFyRwbi!5vTP^ykab)d+}pwN13y&ehPdu?Biuoh@)3#Uqb7$pMmdlgoF=@9w?;4 z6Vz2+KD92$cpORtKYrbjGj1yWz^azCmM|ldVJ?e(S*bzp@`Y? zeu(Er&~=-ZMemOB`7$RvgGY}_oqyURrA)+})aGm1MNhd96CaH3e~>jQmP&R`{J!l7 zZZoe0fpmOTf*l1u(H?DhN@e_qIDEaEbA|AQ=2d2W&?9|V1$`MM87j`ue@li>L;lTXOE}Hnr=n zHDnsc%$cqR>eldp0=3H-877;TpK*PCuG80f$QJrynjRTSR@Nww8{`O&F%F~~{f$=j zMHa8rVSBTEju64w+HE)?SLm7Qm*@K8AKazs`n>i%kj3Ldbx%ALUR+Cbfcvmc)T>1* z29tX;STyt$D-UK;kvMc**cfJTdr`gZP&X*$xvI{e_0+&7*M2q`7|>L$aDO6=eguGLw}#1ySukX;n=gw0hN}5+-nV$+#97$}RL?`F2E5TGOgY+h z8Z)x9Uvr=0;H3A0_uZ&pHWQ*Cs{_PR)FAexlcp4k%pdZ%RSf+4b&(pNujF^DSFb#; z`$%ZMRjstK9o%)kfnGK_=twC)uy*O%BW{+`^n0?!y~rd~v|HgdhQ8d{0=pFrBtc2QQY2ohgrziBsij`5_$t*^kj@}i??_jfXIJ7=mZ&J% zN!gBe!)>l*s*iBlnKC;P>E*U?Etq`DI7VL?uo6AhoRfe{8gkUq;i}koed_<9svSoA zvYEV|osum(&Ys^9348dTCZ+t>2$z<(F6AB>B!H3k;AA{YqXlPj?V^mpK?8cDxG5@9 zqiej1pITrPuEqVAZw*;bY!H``veSOZ%wp|F1^4AY+K=WuqydV<9-JGFPNkM9PFvbRC{4mo9OFx$pDCh$ zKXx>!d~fT1ubf`vBJHn>bsRdTeE(Z5zXgP;-~DQQ`>B^sE$7vZYvn6H1Uh!RB^z{} z{FMQD7tTedmt;H)Fbp(o;C=KDUAHcn@?UNly#=VC-7}*I!_) z|I2ULT3K5~bjxH^w44l;XJ%%GD?D&?aCq4@D2O8B5r!PMbKm}>XgiI^bd#rKc9Xf! zf>*2O;hP)RE}uc@8z-2N>Dc@7WUN--$mqq#2o||m#oJ^y5Sibl!*TnJz~s6ATLnxy zLP~mVejX+RENy2*raS-OzvSS_6sQumo7yj5<(|Q>Wa(Nz%l6l|U|Pz{v{x?Px$?M0 zv6J@YMQ)w~ERZcHFYf^@Ep2ICoo@Uj?UfmN!GBbAoqwusOYYe_B41lc$%{+xu}L3PS*Te4jFsmPzebMA)gG?U*Nm)m*dV^4hNTceb1LW&O+7{X#@0EE^dW$D+UXYaonr_BqSY3`ot=}diCnb*SwM!AvJ8*YmE+=$U~s_ zQ_i{UCJOQH`*Sa!Gk7}U*I#s3a(Q2kUp>6>C~Rohl8jB!>3(E8gROMAA99U;A8;I7 zSa3Q0hwRFM>i*!au)L0v{e$leLjv#=akL5lC+jVP-&QV!-t%3Sd~C=_xWj+Kv2sAm zGdOk(bEWo2uGMYlzbDl>3C1fW5>i)cbH@K6QRi6y_Yqkp%U_@V;sX3*O#UAz<^NUS zEfceb$zz9jPEC{2B#+n0F)kK4pWKcE)itQq!^!lK>t1=#$^2m6{JyPsoXm`)A#-`O z33*tLZ)?_*4IZEL`?&bEb(MnveQRH!#HviGZLBj;s}?}zYzM%FLoWI6@YR(y9beQ0 zTe!vUqpQI`DmK++w?jv#C^Su*1}t+H(AdFFEdb#+aXI6A2foDE&$m1d#L+b<9A@sWW zWrzsWcpvsy7_cAI6TY$${4w0HeoZ=)x?}i>Uz@F+mg5owOWhWI}w>qz%t3qQ9<5`B>_-*J64*E0(9JUWnPYoQD&Xp+fxU{7&DM2juU3oF!;UfKM;%HDeTt{XQn~wE zO|3&Hnx-1R)XuEk$6N&MALZ1k8qhj7*OOrWpuq~}soCx^5Ddb%I!iRgFv7ifMGHi1;UTu(NIopehz31C6GAQ>^ps1`kUBfoX zg^V-sBWmQOe17Ubq_{&YWr<2-bn(%e{O$MamR$BTDGKRS>a^bzEdt%p+#Rmx<6td_W>4d$j!h~tE_g~@~~@_ zEFj4>b$!ht{_IuN@Dgkl+i~-8e_~C}ooPH-gHk-ZD>I}X<=n1sQ*>Pf!JwWzF1Uog zI##`N0N(K`IBavWP`6|3cwhr|Zp*%7e|2LZ(yO(R%au6Qs#}yb%op-TY`lnBvmxjz z%Bb}2^j=L;)F}%_%FLsq51g&fQf1AO+&E&qNL0#dQztqXPrK=s5z-4ZIEswDq3@cN z)j+#tC`1729o&SfXRrt?Fq-O2W^fkSaihD{or zxSOf2v;44&sKzW^wpSA?l>1aSsZ?1iN*1TG#wOaxyJJ-Bg8zhE`ne{iF;*lZf#+bk zX9qMf&eQHxw{^=`oMjF-}gQ;R_U%u4Nc{(|@_{zL0F)>Rd3tQ8Bt^u$9WF6S4 z&(`_VxqH3B7B-)Y#P|HO4*X*eT302L`p6C}999m_7fncPfx+c{^P zMsgbfJl}uXGLZ>A!5WJ&)n)?>MSD%dG{r=#&+n8stT(Lg4Qf@Z%mP%41Y?3qT-raC zu4!#KWgu2{rqC@>W9RqDQ8-3SEo|s{J&s#%k#B*J4n1zg*-cRI1w=Za>b+uIw7j*F zim0<2HH$kaUU;C9#)7&C>jE_97>fSk!gT+*aTAQrtnN&WfoQk74;9RV=kEnfgIQOo zRM0Plx9obceFi<)+1q&Js!>qALAkkLhn^<>NZZe>(y>pmntso?FM4CaC4j>`vvmY% zWk>acb5ziUap)j$1v9_=k}DxfvCsqxJx1$U%aA&yuwFt$$rH_}B$9=Au#+lFZ>&43 z?vDb{QlUZlcFyG(bZxOt6>Pf(V%2{zx6Kyz+n5;n>p&3^*07=qn?28fo&nPX2kGhU z9ijA6)Ssu;aYpAp(;r$Gqpy9S7oDAjA{viU;suXUFTMN7%m2C!UNDO9*0|i?pNyoiTsJcsh?YDRfXU!Rr^j0=)%0EUvS> zDaJZYGF5{QxBU(MlyqD9->_mn5}1(r*u~*Vl)5fdDy_a-WXPS0dt+YQ&rLjFG1F_r zf6eD>elFlXt;;-))hv}`+oFuol2ty|TSlCP#kc5>gY4xILp=z~oFlxJ9(#kIA=&qW z{h}URJc7OnX6__F=?;z$E}uTt2cxi#{Z6#47sbBA%Cuy##!pSGvl$t9jg34N zPHPOUn)%0xrq-sXaU7J>RGKk3n(bS$P@X1~iD~HR>QA9S4^`wVV14gXor@QJ>NjXm z2p|vsR!QOl_EEAB+1qRQ%#L!xKVie0`BF)-GH(_R2RR#N95OST5`U%(i-;yCj*q0} zPit=56ya;fBU8spZT$s#Z;DMu%p;f4p;l2DZrkL57v(wM6&p?Yd>%`*4TQ1?rEt$LUPZi|-wma#y$M)T*?l z)6T_A>&l*UNaBR9Y4$fOx2(2uM?`KPv|cH~0`xnB~LcfUNvgkL6if8@#GnV(yR_Qipc4 zF}HIbDQR&#woe9w3!P#|XDomd(ARSY6NynratqDppDadJu_W48|2$Mk-Po9CG^L zrJhjqY^dE#ClgP^zKm?$FsYJW__|59>0JzZ!}Wp_jMFVT*{dM4;!KpWb1YwInn?kO^cI5&TnfJRvi{q>k$g|$85u2{RX@(3kISDkQoybO@1r>EIt)vv7m8c#FNDbGS${h}*^V zhd@;G(bQbV#~BEt%LwGxWVLqXkWS{U9NSN97^A32$fGWEG88Zz zLR*7dTJ`A*M{$NbH3##&PGD>{(foMgKwh}Bb9Bl9Rh4F048U(nwG{4j6<1HBA}&aP zRvC|%Aft&eBe^UBe`nvVT9ymN{uN# zw6PO+D)EsS{Yk!~Yi6dq!rEYtNp>~u&a;nGN52R&4NnPQyiW3_>$b}6MwKQqK*CZ@ z-S{|Sl%gN2VSvfw<95p_+Hx2FD6PU%*Pv-d*12qVzTslZYVH;bCyZ!O?A+nBoZV$f z^Gd}a!&&QT_UQBHa%9uc+<_XvIID z@ym&GQJz>tx?RV{x^=l0GJCu3tVrc)I!aWh+p`m9)cb9r4_&0~aaQg0mPw%^%}$@q zH8b>lNK>g*kiN@VE3TaxO&{Wwe2T58JzFk+OR!aV!^*qD{}B4KX*Oyyz+1KBrAI$CKWe*6I8c=S2BA1*Bv*Ymk58 zUX@fQckz5BH!#(VbY%I;neg^2l|w5U6|^ zrA+S%wCOm%Vggv{R0o+Mg`t`pm(jmR#jL17G9ybzfu>VC>D`<83tZ|8RPBpd`;x`Q z9a()HtH9DUBTk9pm{+!mDp?_%q}EsD_zyS`u4~_`a?ogvcw2Ux4@z9&8v$my+}9%8 zyH9&L-%~aFd=Ha3=*0AaQ-QjkWKl`Ds8C=$oJKKE}#6)t$7v$vR-KXMSB>C;EC@pKpQXM)uv>_Z>;e?ldkUORn{c zjlLA&rGjPgxsE!G)P^ROO}ot$IK(N#&;opDNltQI=1ui2}@VspRhmS5#R*yIn zJk%g))`~qoWV@xt^wP!L*Wx>gt6pO$z=LdSx7p8Tz0+{%Xzt$jB&wO}55Y{?DdPl#J$-zf^zcQwAdl|zL$3zdlz6Lx8>cTeo|QplWq)6y@b%`x~a zA%lV|ehS^)atD*ZGc1F)BZzxpprGL>%M{`&+_9<=(qaL=Tn95xjY>=2 zKqc&F3J3#u<>0kKm*w(;#9s>29(Y|x`@jrYaHd4RW|~N( zcm8Z)!|md3_{<(cOsD8lv?Omlx0(NZA7|+{fU}iO5vl8>2MY&Lr;U%SY`fZNmd(%q z7W=xzHPf6tXRL*d85uv~O{AEOo#TE>d=net`+nJ3dthVP z_&Kkd;@)Nd=UQe?0Zu37r(xMUJaF+$@iMN?{#F(@%5&$4rneMF)gtRj2BEo2mKx8M zANo%uSji1%B}mU|Al#vqP3DO#0s;Ej}V9X2t1ykKy^G$QXw7Ql1 zS%o@I=FzUrhuuJz6m|+X)mu^+_PTy`Y=B?$-S~9$G-$5Kx=h`zJ1?_W^;l^#CI5)& zw>dPE95!6ge<8)-KC8G-P5KudWLrj)l>b3vH&#ZIx!T-eA*UdvP0XbuUav_z0@9|g zmR{J~!mFPdtBQ6c@2)}hErTU3@D}T;XcsZ0Mr3D?j))8*W|Ww$vHHQJG*`$!IwdFa zvVVMHX-AB$zuWOiT$CQNp?_E|?B#U8$7~NRPgjIrmB-4C3wm?rfUnr%O=8{AWw-B^ z+GWDKQoh2DwT`e7B~i!YZ~Z#s+|oKtq1MJ=EpL!}p)_?1s+4(VM%SG$nUZk650YHr z$~jx5xN6c`FN8B7h`M<_GzS(#2Ukxl6QlfCj(_{~6})tq^4Ybqb1eK2s$hX!!kP#R zB<)M+=cLD-=jR3z0eGx5n6)d5b2;O4K_>R~Y4h|5&;q#$vL9;5}qgnK}B#|(|Eo$rYPv=E!A5$el$s0AH0v0A~$vJB+z>fSNl%sSO z>HGXLHt>_xtgp54bCFi%$5lhA&W0rP1ToXpKMM+ZF0kTEs*aF(dUqO+PWSgd;WZBM zth%6b0b=znqRkxq0NHGEK9H0Yad}}&rI~ZqTGaL}@mr!MeRI(-)GQzn6#eXvbFskJ zwd=IKOm$x6TakC8JSu-c*!k_{!Df5;-16Tp(o^jt9=&V zq>obACiC)7Kb*Q5M&xveU`X2OUr8<~N3aBI^qy?HJp?KLF8*XGK0e+vG_r-cuVWzI zxYi!CgWj;8;JO+tB-r-;%J_7-?`oNG^QdMMpwH1s371+pxQuu!-RM_%>#A$JO5$(+ z;rNZg6(nryYUe;N*lezzlpFCM zQBE*$9+rWs8z8gD;FLJ|pkZNNt=Fu!^zu!sRcELudNvwketDPxII?kO9coGhH-$QP z_SwCS2>{Xz%kN<)`z0->gxCZ0##gLZLZ3L)VhjXG$7Jg`aWWjuDe7lqLWg>NPOXHu zlpu^t|Im>+`yhB00y8~LE@1ot z)6zx!j_|;rus$Qu=k&4U{+vo&!Fm^@r_fbwWXP{H!~vdqN^&Cr-cNgpZ6EO>@P<3; z)N4ngVsBhDHLQzO#H+NtWgjffzZ#a$+EGG*G*ryB5kXnYvNoyYA<@JkbDh8(p&GC# z*6X^}8Mt|T7mXg?YF+Q3I<AZiz1-o+Y_KcRZb=@OTo zV{WfF--N5E<8WC&+!$Co0Ih@E)3CFzc9?(OAtWOfgdCS~^gNZp$zZH9Inee;Geg>~ z&|;=vu6ov(taUr{3r3xdLnzxP0?%P5KYwTBw`RUDUulkPlVtdmogAHAH`11DQT<6R zv(`+;yykN%S^;03YeBOh&h1t1|7gJ^N&MN+{^D@pf>}WYVpvSV#;>8*bZ%AX#aaay zVQom%BAAEQ2`14blw|8_e3D(qGC91h<8(e@>jv#HCpnyx!R*snrNG9$ z&&c4D&+o9d#GN&ns%P&HB@fkTL>`2YVWI;; zn7@3BzBfRoF2v^un9$Q8jA@K(U+~N1>ZmfT!C1j5x|J4Qw#tdeJOfmd3iIA&VU+ZX zjyeywS)l7)FpfzYAE`Ezh`Tt_xmTN=rPJPPsI;`!OxWQLD;Jubqw$n-7zmi1F7`Cu zkGDt|@*B%USp~|nb;{anTv*MJ`bj~(2yV9z*lQ!B)?t@yQoKZP=K?mQ{G0NXLRS#>@(^(0 zZWHP?>$_v(?|L%$V|m#%X62sDYl^jpxMxv>v^S6{*d}41+dtF&=}I8FDqwA<rVR&tRadB(&!ewAK8SR>3`+LtmUaywsvkZ`ee%*^6|VrMj`O??xA8eQnWnGOFp$7l1xWWUm`?Nv>@ps7e_UR^DG`RJ+oe%C4*qo;lceVi~0GA@EZol z@D`KG#kc^5*#eh-!`W`!G~_}zT25klzVe5y*RqBPj z(kPluhpp|?!}DXm`+A&MHq}uoy{qxhWOHgEp?y9UmHW^G4~YFF6W$cXf;2sj%4G=% z0O+R@1W$iY5B!|_SU2(s>smbG+1Kh$7Fj) zQ?2Y(zDx+dilnMmH?=qsxHnNd;Q!@7+iZahaGRB*@~E$IYL*$UoSiqUib^3?vq%gH zPO=;eWUUXWYQWyXzo{aYnWr9hK#q6&H|tMFj+?pm);)fYvyQBG=8MD21Uq|E_a~@T z#XtVGi|h~jQ{Sg%GtO^6M7fEr0C=0!E7pA9(`KZ!vZgvQ4&mnG%OHTyD-y;w9X{TP zdl8*+t(6?a9m=YREYuH6zF|B7_l9#ai$QliZm;?_OghdtaQ>TO7QUJV3lm{mM7n2z zU-^tGWT(0hdr|ItvwS}tSzI8!@LIh3on*5bbneTldPWvolv2J@m{BgX^Ygws)M$CP zP5TY@Z$^&n5ve~emd6RZQu+g$iA%RqaM22bA08##{r-Z5kAH_Nm&qx8Z+PaWa|UPa ztvuSw2TS#tH8%1=9)PtY?cy@6j%@!8@n(Vk_|hs#HlCuCSHV+)<7hsze2)% zu_TApM3CQn_rCwv=k$NaLHY5wOO@Q^zlkuyf4euv{vBrJE#=>5_OJh@$*}#yH}bz_ zeQ+{S_}{ny|8p^xAx94&qV7R~7}D_yuxj8eE}7-tpY?s4xV|~El1{!K4~y8o=lMpI zv`S8sF01x%s}9i(%@jOA528Q)$;h}H^Ji!nqgQ~FS>oOh7qIh!j{jMDu6&+CJm4l` z&g2kb$1HS1HA9dWH_0M-djtV(S` z$@llDX{n(_G}q~Iu5bB%RCWiwY}HKXN~Js^f-6X8CC4y`1yLc{YwETRE&hYPiqsx4 zmvx7HXg2Xu+Ai~J*l7R{#LPPi)RTKk!W=%%Tr@c%mvwHw=Fh?Zs#?25zPhrqH0Yk8 zZ;6@WgYtoGWxydD;N(}Z@2WBkMBKgC^EOy{dG}PH9o=;bmS8FMN5aTRkMDA-A)rKJ zja=@%kgu`S2(>@>8d;*704YHTS=X?z8>cr}A-k&5G_=!6i!2XUBgCouaxpL6hJ$!l zVWy!Z=+UoFd6-)Wweh{FB`Y3_XnaC*q9EiLq~f3b$xvq+KZ1ao%8-5D-b#J`*YLmG ziDr!~d`}%d%o?uj=;AZh9mw{L-R*Gi=-fvTz=B@dO>|N4lJ!wNmjX$K-^A@MxD zAwsaydv8AhGw1x-p#2b<{h8uJhU}e~$+;?aQ=VnwX+ni~4fYMC>x)&Gwovg}Fqf*% zM)R>LqWl)$?ZOd&u?OnRDe0Fc`@hB}|WwS#8m0g7_O=fq`yI%6iil}h|qb4quJkP{3A>t|5; z%NA7yTai7LUVV6HqcZ#A7LH3x#xHO;Hr;oaq zXJ7omG8ftqBy%Rd^SAyzobFg;s)0^(HP=;BiBr%X+2nr~Zy9IE z)5NTjUH5AD$HLzBsew8=H-*n;r8pPtLYtg;;&lV+5!U$*lCZFcEKmDmI z9?SX(cOE6cgZynD89!X^TnzK&4L5zmROVB0c>l!Dd@cMl3soPC-3P^gtJgO)EASz949Y2_3d!1$rNzrxB4dFOZ+AB z-wKXG{RccUG}VE-&rYn}8b01Q)vnq9F0!Y(+Ia8Q%bD9Nh~?d>Ni|dIK5|e>-#al! zOT4SfzH5mlyxI!$MB=G2b}=mMaBS!BTL(4)|C5)sTQ=x)XB-MW>N4CcXflv=G*?sX z^jweYI=#zEdK2t_=A-CSQhl;9D0s@p2=~~TZ#hhd9x>9BmH*dmSPK@CgKM=S?J2p z+Y01V=UiF;v^l@1@I$PSo2;bAU5w~Ig&dLmiv0R3GYa}&Y|OW7|GU=btOWA&KUM-5A65*PSfCsMvnw$rBO8D)qAz3X!Xo9$GeDddBhEBbVoAO^Pzs|d>&y2 za#d+52qZ^*Lal-a*N<9kbf9;0sRSI;42@Wgjb-5jBQ}y#^$CrNOo-%qfWUP77#>ly z+Mx~ZR&5>W!0hycP;Rj{FqQ*6xutT&7mfK6({+X5Yr&ZDy?<)DD+@H%iZ8yj}`{}{x6H? zr)MrO6?c64-}i{o(6q#*M23ZZo0+ZbTFi)=4WpaqXnr5k$aC4^N1t31b^7FzYKPV; z`|0Fz(HrjJ+0K@fj+(ExWXHDamOCf*zkAtnNQ_XUJmkxyzqJ;m z{*L9XTQ!a-65S%5kl(ETnJPK=W!$LWN~m&<<|uF}sIA3v<7dBFWCs7q>A^cQb)|00 z1z^uZXoi%SIl%`(nhm%?KGZ-cVU;4;Jz+`GK;-$t(D70~3WF{ea+%P-DHHssPlbs#dU8!CVaP5<jO7+d1j=(1DLGCU^cY zkf6H38X@smXG~CU3buJt3A?9I!}2*(yiz*q#)RQZO$_>J=2BBS%ueea zIAwwC8r|y3P9!@!IkEY*H;OPIq%&n=L$1=%VhMDPDS3Q4Hn;kx$&Lm>7e=jj(xF1E zvVl#F{Q!}NpqL*?KB`Q2VJZa%F*ZQ?9Gn)F(igJzPGme7OZ6eJY!^>(R4+cw3%YDk zXF&Wht_))h;w5AaoWw|C>)q#h+lD!A(9fw4;xk=hys((YSDDT!Qx)5tDlLs;MJTX1 zeMFq(c4IG5D+y!JUJN?j*i59*UGb1yh{8OShb|mgJdn?-pOFpK9o{Wq=$RO~3clxay>K=Ipq0CCD~o4;4`4q@+&yN`2I>=hcD`z> z&l~Qw@U93w0WcL~j=p#0p=jcy9^cLfRtJuX(zBQ1X<0n=nFK|jeinq_*7PEGSiu&L{4LQo&3|e0bl2Z?jGIq zSF6@AkD)V~c4U}SSGxyD*Z3&xA{u1H$r%s4!xbZA{XqP-JXgSjQ$3I|b97MxS7~U? z(SHAI?z}UsYupn;TV3b7SaHp7l*amK?Dvd^kQap0$_cdM)Glm<7ir+axZk36@g>Y? zA29I?IGFut(wVIy&z=*UBz1AbMvx1DSm?xCQBud{Rml%a9QxlyGBQ%tUhWY+nDiE$2+1YwM)i>`UCEGm$cIj3H}fA)T@W{>DtRZlgU0 zVO*l|oFCC!_eA|j?3xI&*vi%1>#YWW)>8N&kPrUBV8X3ngU+a!Xrp2}AqKD5){99E zap6&77oJQgB5Ea|oFqEA5)#210mUd?A+I_gQmZ3u&$`cQ(-d3XQ7F{r$eUFFBI^!X z%a`;eDG@bEI}gji40VPN2#tXa$JXXa?Uf>&Pt=?y#4^*Es{MAn=RBjH&<2M&O)m@_ zXH#0+ba-TJFH6jPLAZJfzO>_WN-By(ykfuuwscr6{ul240nT6pL)_lYqkA*CW4ya&JOb5#A)IWHt(IVyCYKfpe+*2H8otFX#m93;@Y$?4^d7Gn~I_l zau>7OwckWzH*Nj%ic>)RpIwXz5z~9md%U)pwsJZrOHNbA#c8Tp)2m(p70|lYDw!6M zt&F#(fiZi$w9^b7HAs7`eg$08#Je(NMXRq6A!eFhQYVd`3zGC&)EC`!&YpU zE1kll0JVt(vymZ0cOmaBF!0nd6HuTO@M8F7YSp4f@GT+wFL(qO898ITLhzJOOgaAl0{RQIY?np43WI$gFpShpP7D~Gv*rd(|Ww3HOI4hjgR zig;h9B~fb*{6k%xDJ7CDKzm2; z!iCCVl8b~Cb8mevWvkk)49v8#=-uQ^InY)=k{zKlI%1|SBYQhx>|Q#wp={9Fe5$>& zBEfh(?G;};gqxE+57}vgbZycY>6vs&xJ8SH_wXCmE}TAL-2S zxt+o^7t{wD+?~xZa?WJy3EmUWtlpYQMEj{>iD!Emqm7Hk^JBJF50d#j{H+^Lo*B+| zZM@gVV->oce8t%nKk2vOUa>H7los&H0WtmW?>;F(7s3|wJ}Y$L3R1HfHAzO79gsdAwA1PO%|VJ z`MDTbmssE}AkiNfaa62xF_KaFs(c~SdpAIQa@e&zCV&a4U^DPT;NNWaLRY) zod720Bb5_uzLldKif)(gX-n!7;#E?cQLY_MQDFfAd9O6iR(1V4>xo`f zepwAkNSKY$3&Re~6bOhtp4KG%Fxd1kck<8UdQcR@TrOJ*gt5eX=ruOj+DzxLmO2By zP|63rYs^D%r-12c9bySnc|zb;`}UpcI+AF;{*EYIBUdXcGA==6wr`qqddelLFRsGc zQjw)8v5~>0{?%Nm;WQ)u{x)8g|KPs3ops|H-KYaI2>er^RP?2VT}2sKYo0*F{~9we z%<^Dk3W&`OD5-*cG1gRF4gCt2kBP#)(mT z)}jYQXusW{Rni`4zQHb4HFE0~dCpJ1`MJXqv$xPf7gGX9b)wQ5$0)HYs#6F2uEa;` zm%Z$MI!HrA1@X4qw&WEE-Lb?ycDsbiYR4@nS@W#`UjB`lv`F^2ZI)bDV^I}Tj96Ok zwwX#}Wog{UwH2Ec5*K`19*`cju^usm_l8C{iP;vzm)?Enxjg4|AeGRcM4|t5X4yy@ zm<86KyW!FK=leUw+rR;5{|aVSkKVcV==fN4O>x z+2Ys5rBF}<5*}Vigq-c}DxBZ*?Qryeum{d&)q4odjX#5@-*I`OQfEnxYmHxYb}*`a zBEXKF$TkU=5Oov7t59$`9ps3hx*gOEdcIyiq6=I8{r$ynS4#0ksrHR-NWzFE| z;wXZEfPzRTN|oLTEkuU_lqw*-2?$6F(n3v8>0NpWBvJx|05J&=l8|t27@hB&bDit@ z-uL~^Ip2NV`~f$2*?X_O_WG^$TYGf}j?*7$kc>D*2fOXBDO#lo0yN;dxAsr$*~Zfv z;-@@R`!I69UaTKIoX?3FO`yGwTqH zt&l_Y{y4GR>L(K_FBrPLRcX*DSf2x_bf;g64F^plWhCaJRN zOIb;%xM;8~(VH}$Sc=lsRr^p}y@y=rlN}jK@7cajFrflVXSEZbOGtc|y}K)%vVU8* zzs=4rXgftgKo81C#XtmYDKC1)P?~W<4cp&|M0% zNg*C@nfs~5^Hv^;74QUY=K9xzUE$?{6|7_10AyU|196SFqzbEGIUATvT;D8yrpUs# zrxsX<{mEippWO(Q zMIs#wC)r-cyi$J)j$48&ncCkjkoJ`7FrWRToVJRT($AVZJZqw0`*?C~b{&0fa#guy zSKTY~yV)V|2iL&hJax1e>nl_9uXektkaZPqlaXnw4P+Q}2SBNjAlR1sF1KZc-IKK^ zyXcwX=pmH-CjpCwH-H55=n*bL{lrRO(muRa{joS6!F(}wtz1=T zF-5J5_NZjLV=Jc4W5FUdZKs+F2dlqzlqxYaOjQqK?kwY?+Mh_sQ;f`Gdir~dvBo1q?j#oxkw7z$Tfi6XwO|d%ruqGaI1)K+Bcyw;;q}q%AP<#gD1oN> zPt!Z0j0#rcZ3HT+NLs{xh586(7+zqL3xC;nGSx-AQlqAB*;JurQUu?m!QMRoteiPf z<%os2`m(Y0RG-1-6)z0tNF0VeJ>MuAGWD)Iy-rBRQU;lzp~;zR5IWMSCB@yGkT~vs z#+}rx?J_W?on4=MThQ#`w7pbG-iWbu){dqH{dLxIsFyniW0uRrn`FW&Lhbhs7#+;4 zt!D3Bti9?FZ?HUv=apNZ(IUKaaC4Y2Vq^YitN$Rkq0SoyfTf^Bwe8c=85SGa|MZAs{M`x7`G z0m2i_t-v|$aEUAF<@tQk%R6O@N^sQxPQcJRWa`U2n$tl_M;XO;ecdLp(6H2O_n$_e z=A9E*xLDWfmsDYLSldt?xhd58Ab}N~o2W8!^gZr)dVVG3-WCyZ1iUed z5A&G(_DN7zV)rKX`8bgj8gs3EcwC0}PH|27QeM%%J|%Z(`wx);dJ}s6kd#I z^T?W#uwQIZuc10f`Lb-xpKF{g(Vq@fE1=H2bX;9=2jq*-!Xmj(sWC+Mg^d zB>>mj!2|2LF(q@mBfvL2mEU=?c4xAHO)}}&aZ)q*b2YF6oZ#w>1fyT|2!!6(mN*G# z{pm#0Ru}-cXm;er`>LOBXVD!L?0TGj;?lPhk;;uXmeT zM3aw_;hM*BH1XpSy+0p$KpLqnP+uF)jb#$T&9sj`5I8qDXL%05-aiMFbvf>yM|o}0}<^yUyX_ztCKz^v7*M;qPLPzwv~n~yoDW7J@0 z!{#jq;qVM>{h$8H6k4z9`}~SgPp@hEuaIA=8eX^&eTf?6KuMu?j(Gbv)Dzyz+(z2d!MK%<*$(%*%;JV>W~NS$*s*VtO&i=F4<+RU=tcL&oqc`4edb4_L&PEd}T3SCQ9 z%46p|C3ekt1aYT=?JUv-N6U5-r7cf@;3zN}XnE-Jpa=u&5iW`gJsw~ndThtO5VGuj ziCV$oLo8mhr+tnXlw8R8N^5)1 z@Mg>&%{Fd!H}370^rYQT>CP?$Ygn@3K-u=%n&^4%0AuO~QR!0j(o*DPG1P085)`N9 zmU3${7*skWVcJ|%d$6WM^wbKoziMkzrGUO2;<`8MVa4B^E~Pajuy)i-8?b%cmEX)e z*QiQi4juc}82Z=%%6~Pp!)OdWw1iAF{Mt zMhD%Lx{0%{$p_C8wXYlo=eil?)~P|t)QjBcc65Qq%%@rAFKrqMi^?aUUss`LC> z3odw^{v^G$XmiFau_uk`u;o}N|APzAygMM>4dWhADevAF2ZIw0`o+L_+T(i<6 zdu*=z){^D*AzU~jSY5pG(H9#09F}q;lDru&7r!l7L256|Np7(WAG$aPULbi;_N#VM zIjQLFjGP(>n9daO17*EyAEo>iLXq-FRJ9;PWbJZn?bJ!4V-wgNG=_M(xtB+YMnNdEvz7eQYFajY`abJ|RB6Jw;`8 za=FG4lR0CmVpNOOmB&ak7K9Re6ca=T*g)z6R(l8@6b&S?X_S{kKAHg`bfS4Sjnl=Y z`P-7)(a1s=?5Qq@13Pzm1xFtjM_E z_cE?sg`HbEj#K}J0Q!Q@3P!rCO2>plA|DjfD6O-pf_`H0?eQ=lAOg&n`L<2}8qvBS zu;K}of#;fA^-|dAz_f0c-gFb3h3C68s`Wqe#1VsQ!a1Q;p@4Ek@-GeMwOXN7mZA`S8o1#)O5NMfp@g4zp3YAgqcaO7p@#fmCSRQJTw z|B@r_h?4^ZsB%Oi9atm1b{s$f-@M=LKkK5jqQnbl$D@K_B_a`YX0K(ohhiZS!adf zQui7=KE?N%d*%h==w{VRw!T>=xhM*9B-SSyxy3CciYt0pLmf1U-jyIO?wu#Iwu=>l z3;eBDmIkY1+ZW1r2gY`Zou62QY$2`G>YlND{WN;D+8T|p8S_$(C}_4ue!nqI?sHzN zuDFJX|HIdY9XsIX6{K`0>xx|mav#URP7t6@$M~CirhZUSIGkpwfQMZ-Y&q~)V5zFg zqkK}@4-q@J-Pbikv|+E#>;IIC5Dc;nR`+?YDITDP7DZqpp64`51d`Hx|&vj)gnNnlPhI7o@l;wsErs6L_`B~>@u&F#Kas#)*{t+FGvwz|T#ao5P_-%M&4EY@QW54$LOR0M^GE z8^a5VZNPy@A{-5^NRx0xMd*~a)*`oUyxLAWNFORT#Wa1S`Gy(m^@i~ob_4fFxh5%W zkD}P0E-V$QWYq>^Szh$2_Ea`t?|StakcAwf!l_?wEj}gQVL3!HBc>jYSh+Q%tP~L9 z58bQlQP&f@16rm|QdD-4@pzaG_Z7Vq0fUV@^K}Zy(Z}dV)6Hqs#V<6Oi-@QcxyZhh z9S88@VmPB1=XS53IEYk88}q!eIBS{z;9<0t`$;^-_GK)0@*$EOxQyN4wl)@&YUhcz zYKtO)K68tvi4C-WF+$LzHJ#oI?Iy6ZQ`{4xO?q(JDA0O%TkRI-hB3K5tzCLq89((Y z#>eh5y==$gko4)L5{J?Bd`U$e5zM~rY6*)D@cmHlYPdVaJ3FWE=26wR3js6>Q@D3? z*V5|hP0LUJKBZQ9dU&(V#^3LZBXg|d0>}w*uI&E!n&MK?3}Z&z|CxTG%hjA}Mxt4w&?i^<{44VgiXl_|v z*5q+JEJS0_1FvD9?|dtbwTcQ0zm7&8JW)HC-}qI-E14Bn9FwlMG2MdYAePYt*w67F z+_hF?R`*4GQ<(VO_j+?qmBs~uxxg+X zD~kkLo^>4m;WhZCIJ}rePD&IiPO6^B{&v6N3!MI!0_tNFgg6kPXopII#=9j-4aIQb zDa4XR_@_gudX}!z_|_@I(gfX$04D3d)LNo>ALx7;flgb!K$z= z8$nR$;Z_jk)q)KCvYpvx{9#jz^wimz`&G=T@(TW(@=>i0 zyZhIK*Y@ojtBV&Mw5T17{eY(~__R{8;->f}h8g^{eOCxYU+yNdPU_E;>ci^|?iTG$ z>*jw6=eu|KCzO%AQw-?BvB=)LKC*#c?o>!N#I&RDoz=ySdnxVCXZ6f4GlHhxCf#2P ztET^SntzAwc*caXw)dk0SGs1V4j+~uTK_~jn>*1VY)&YMXn&T|ZTNGRtSw09;}Y=e zHIz#EL5Or0DqSd$zuL5C!~M>ep}?An<;B|;>nBzc==};kD^4%YFG!7a{a%mTXd0i( zm_yJ9+$onU+8N$z7reM1!}mKvYw)V}?CiuwBdwe)1Sh+oV;kj2BYh2g+WPvg{NXL{ z$luJ}gPK;6GpsHQu;l`%Iahvi=1@&`v&Kfw3&sgQO|8!6)3o-awoMLv!h{@dsf2b) z*S54z89a7*pK9AklyZm#V8&$psEzL?Eui-78dYAN@%uG8HHX? zo$BFW5sU*Ts@aA^jx16RR`NZyWB9aVducYjZL@&hz_HL@bUSNl5@G5R($Xh@x=CKe z7y+^X?%LKd15?`wT}j+za;Pt0@}d&j%pT@%7~PU`K)?>&7I@Dsrr=?Zyl%g3kkC+%k8_G*r)z1W3pL_m_~ZMvQjF+retXF(t{|ol_R5k5#>zR(T^08_)uR8hUBPy47R#@ru#AIr|Ig z5nhbx?nsg*hmmHs1{(NB)$`Pmy2+$545(5`Th@iqY0P35P$LLB1dnvxNu{PvWTuT| z)>n{dR+aRN6Q{@4-+DC#79Pl9Uua!KSt33r1&*8ORB5!PgqWM(ETv{-7bE1i+_J4f zPkkFKT3x)2(kFVHD53lfv0azWaX%ErqQ-(yE}+?s3~RA!lD~DRP;-J*y7Q1sQB##w zI#SdhJns^4e<&O}iwGrto_g+VtIR1_{HOF|D19+#orEIp5lmiO&0n*B*A87o@*NH5 z2ZzllfHC4FUk`wqtdJFb3M`Vji-1th_^H9)Yy})XxfI5@tuLNX$=vH8eo5kHrOSgEZLLeCM&TDw zG`fEEgjnlts8J`(jKW27{nTi2-SC7~XGJ9KQD9e(bf;XRKup9TGw9MRT2SZ8>1qEk zGxv%axzK=Fx>y=is)Ci^EdR_~LIGeT6H!_3PzZOi5w>$+M0Qb+-O3OABEdAx&nMvA7#Xc-x28@Y1 zxSc#wb{#}tC{%6XZSDc_9O157O)J*p(VUwca0$#V#9)w9h#rlNyTrjr4t7Q;u=7Z) zBUUpX=@j6=B#?W|2Ynpb>BLZ9nZ1zC-aYQMaa!jqV8YtzTeK%{@aSu@RSKj-I%tU0-L^JN-^7%wt;t^75N&o25}3Vv<0dt< zq*D4N=^*qvcZFO-`YOa@9N&6iM-3GlSlK{DR$-7f7KOTBpBB*(#^C{DQe0kY*4DXi zyxf*ON+FJJ_P*t9F_b%9wYINGIU_w|LN`oMP-@cs7UBy zwK|#i>RvikJ$rf4K)O3=pw*6|k6b(EDeL|Z-q@v@9)T403t>laNV;3Gh3CZihx6Gj zwqJi@eHb-r_iePW32?q`wvKl7m7w|>f|{XTTInc(TMbr8&^@Bkc~-&RURkI6$3*^9 z8QHcP?%Y4y_~IFj={;iy^J##EdB2mR)nIjkM2+9S7TASx4gGk#l~$bzSNN0xyMdzwdFU$?{2T}=_EYM&=u zfL~YQh=f#lnBFqGib`+ei$tIpc3dao`i}nR;~4E9FYO-U82JF0z919$N@naoxgXCA z|0xXezuiasRm$Q^rr+9dAHbqNKAKJfH8ma_g4@M^@M6txaHE5DL?k$l+}VF0|A%u( zzQ5%^*4X*7AGB6j@e06XTlGyEz+M6Smh6xQ`&Js}pWR=OV1{0CvQ!Ie&9|~$wcDUt zujJ*&G)rCD+UreG@FgBRGcIj0C#29+L&Bv#n)_wjuJMl(-N^TF!=2(=6owiSAlV;#Ed*16hW*R+@Y<2-E| znx=t>3J{3qQa-tAx7RHznVnrH!^a+x9Wr2h6m5JN-$A@3)>KF-3tP!|Wn7rdx9X+> z*2=8=eH|^M+1Lu8ykgiHf82$wRji%@rO#dKO!um%(Iwc$m!k)_z8E4_&$_-CXvH`# z(t1&r;GWg4wsT*mj~gtWbVPP-QOAC|fQR>d>wbw#)J#=8z67ZCL5n`Su1tVb8f~W* zwY==9sy=kZ;Rb^li?8D5=`5hkA11@#qmFwa<87vDD6=64c2YuOTlN@F-gy zd^skqdR4hKC5McRzD(L7id|h3KO&6=b$0j5Pbhhg`%${Sm93(J26F}D3+%^>p_F2h z%c=wH4H|c{86XLNI&@ZYI>Zn&U_OZayuq&C*@=c!63Kt3Ev*F}vorKg(@a`3aJc)y zP}u!`S(~VfV-~5=Wxi;^o_@3{beN=;C$ZW$8n$U+W^<^AQ<>zb-jZ6i=%QCoypKo? zHa3M{cg%M)X3@jSaQQqD4%kZFy|FPp{g@;_DbfSCj!Ci~zqt|U7@CuiH1B{j2dM*m z!k)r%sgWGuQ{s1O#K!V8jLO{y{%lU39x#58w*4qmWVbr?0B4J+muYc6AWHQIs?1{L{NXq-kdF+Df@Cl!4p28OEUu$7MI8$+X(gs%MMg*XPPLQz@L; z%f_@2*Sw+#`YUb5t#M>YYTL#z;#fC+vH4RX?aMsXUMAe}Mz`&{&GVuW&G#eRf^)X{ zF=g2Zwv*>e64~yKj4pAVjkJ$b!sNwghUmm z^WNMWRtJa6M+GKo%V(NbG#CtYLL#Hb{${+|ofp4yMe~7uvT7l^MW_J=U3W0rdi%14 zD#H;Wb1@~|*>0p|Vhsk0>06Io^k6H=(&Jz)Dn3@IUEJ^ z&cf~8o|NI)PvSSG8Fm*FVEXN86~Fh2Ss#9xs~J7{cTS=Ax+6T}?v$N~ z4s(?aS*JeJ(4BYU8<4jJ<-eSNG2!58tawDBb)abNJo-hu;mlp-kG;na*K; zcAq`tK`r8ZkKKVDwb{&L;bMdge|k9TNmk3Sf+Qd=Em_zhm}WM5#(LQ=vTt;Vcq`&o za#9_8gw6oo1mJVZ@ElVt%Y6;kbO|P<_CfaX6h|XSxivs-8ve9Ft6zOF(cL^n*US`DUPuu^5vP(l& zPbzR_CYqsFSym&$Emc6G_A@`nLHVb1sVyNYHVU}#GB=8L!`$6+Mm|Kc#4|9YeL_39 zgr0CjDe(WWX{pkj|7?y$`jKsepY&0N#|eqwfmO8fXtg5uL9?aq0vRczWWNQ?QH1&^ zH+u?;h-FsOwQS+ImjIjd9rNZ9E7!)aw*3IIb)EJ1S1+r>5z-ya?Y!!ILM56^`DKe^ z5-1zQUZEwE9ADiYsZ#5qMC%vnI-OD!p~_d;sbhe*$#s+3k>;GXvRG4b!Q1p#3_6-L=5cNKZq8V9 z)sIy|bComo(+;F5K1|v;a4?XElP-zl}fo zt$ZHE0g>Ua%fG1{#OEPF56Z1C?pM>LEjigkbUbAAG1IPmZmX;AgHy`3p=)@DGx)vz z>G&41%b;x^*P}@y;vpiCoxK9t(CmbIMh!0;Csn$Z<}Mh?6C$`(s%tR=$BQ^?N8n|B zmm^v>R3Rz>As}bf+E&<=^@2xTg|GQ!N*{97V6WWkTJXBn!RB1v-GeUc#TNly#Cfx1 z>=(B-FR2hWTQh3ja)eBpCcpZ~xvqUqni=d%qM?fFAF^Dz^4Nnvz9!KRpjXc(Tpkekw$M8c1Gqb-QTBmA{FX4>7344* zjgSu69-4vS5%^`1GVA9dikT7(Ak)tVu*Nz#TGzJ0M1lXB=g9~6aid{5Oxm+E{Q8rW zBc?Bpw8uqRzvh3{PwXj;e^`Jt4A^&A52Aa$+Z5=3!s$di52d5DmYe9kKI_hwsgb$* zc*zo}hOWqsM7UWr*M+6oC#LM zZe1Kz9V+FN4z9Z9e$IQ-cu;MUFDbyce)O-vDN`=4keg8w$&%N~)7L!f&c55xNCh&g zy84QVJpDu6=0%dBElH&d^#GfbeDdM^A*6QIcE=@NB#+m?yx}c=7dG_#Ljo!!I(X7| zwqCq}?Ut*Z$976TUYEt~?$et&Mf`t}Zn}g0SmhMtRc%=ye~5C)KX)n%D3X+V#OHX;W`j<5&~VSB;0=fRVt?*>2soYlh4FMm**>_`~j zkWkJ&aIGl;W`CAo?5BGIn{Y{Guxwe@;~9k~m#@BHF`%@ovU39#i)QO7 zKDdxt)83Nq#9`I}$I?cg>#6sxRjd&50lU7m22A5Gl2=M>v*X^+SGT38JP?OA0u;n& z%U3t7hQLsKAe+s;{DehqE1^o%C4eU%+^o_F@ZA+=VlKDBrdxkd4pHl+ zM>BM&;*8^XlCS}1g<2Sm=+D?ou8gW{uKM^~Vha{oQ!u>8iJi{RA@>*W4-H8ru;l9f zZ00(dvV9L8TV~SO>hcF_$!e zoStXcUl-yo-JWJhdtC|n_Ib@TE%f5i+pgF zn(?aXem)w=rw||EKYH%7`;(zf*1J!_0bZWqt3JnvqP#0T!0Om6WVhEaZRfk}?e4Yr z1+uVroSyc!ErAohz!R67`V*_Oj`40$hA1aNU*wJ|>!3?bYK|t?GQeR9YT6r{_LzKO zZ2M--VEjbsV{$!5-(Rcy`(Z7ogOfZvOk{jCY`7v{X)BLNZ1CHYlr7vZJwPsho0z}R%h<5>E zu-mQLAT3^wb(G>Hz!&r4G)~2284~Gd*t5UwzW`*lk>&XJe$9=$bIj`OoK4~}NT&dn zhh%HQUe8~jJH1*2hC4f7{K`|qRVyi_2bG7m)z&s?q`o=4vIEV9uSigQ`6r5Bmdv}f($0ZFoy-_Non_lSJqP6deyrCZVAKd`&6SiN+~1V4_rX|NFI%wToJ80tI(gC zJ&oKoGGpgoP!j_jUy=W2;Q51H1@QnGPLnrj;0F5yeZu@q9{(!w1-Z%V&>TQUvDPl z+XQm~NhSa_t<+vVYHT{kMVAuifdERJU#p(GmJQqG(`DwM1!epzRg4q}83{>s5E-?$ z@;cgV<$|LC-w|cpUM9PujX4{Gm#+MM3?NsQdvXrXGK0QE`aE066E^4~D$el?6h6)7 zKO)W-hbZyb>tenbu%fuQOEt4JZObJBE{a2Gqyf!=sJ$FjuoSJG zZgY|S;eG)A#jF$oBW2W@SNa{+{%m+6BBU{F@EG}R|N7Gj3AzTM^5e19<7osqQA7no zH)Sr?oSBjO3uZ15nuCc7Zm}wunzuao4c};CCXDxUbn@_I72p#WFa5T^lPG0^|=J`4A@5dXo z9Q}FX);}8py#4L=5$W~#xcRStO7XmZ@&7nT+5Y;+Z|eH*|J0+4O$t&{^}6mK&sV@V z>vQ4suMys7OvD@>3h?v}NVn%Onc<&)6!Hn;JRNzPmyeH}*4TEH%`0U5DMn6~tNN1C zjjLn7V_&>z+h2|VhG9{TGGrFuHH;Z(<;ezZ6!UMy8_ zkATVl^y$+XHnw8pvLHdtLM2m?ruKG>`*(Tyc*^_R*H3Q-u$NT)?|muM!^NXtjJ}f3 z{Qes6r88&FynDo(!1+e6_(E()z$HU$@f|&r0*;@5bep#wzgo}@U1JF@6X3jd!4UWS zuI$~ke>}T}l`merSiE}YHx@a0d52rFzux9bJN@15`<#RDrQnXT|M~;5J9q2}t46yN zve)|SuJx5uKL+dLtD8#3pX}a?IV9!G5!NNSe+c@e;+BxeC6*|x`)h2`KhO9- zG2lSYYz+*vQA(GKIf93g{$4gI1|yeWy5|pG z8Lg~Fhsb*WIXp`_n3|_H8?Bbyw5kL5@_ZMPZ`r1@zqC0?dkp15cb^32eMt#)>~NlP zh#fuVT>>%UfZM%)Dwg=cdQakEH-6L%&ykNbJA3JU+e0s;aoIcLG6!$(8|@*BksN+w z1(4ZUqynEHCNDRXmMm-_=j5~(8SB^A>RejvvgCwC+fa!@iH-v!jm7k0Um&Jf*W@_$P68x#0>RAchAl4e7rNeQwQ7Jsvg<#<8q?*zHR|Fx4gZDMvKc&MeZED1_lZ9i+sSWz$-^SyE_ZO>h^W*9A zhWaSg^p)BHZs)0sI`dzSx-T82qrkN$j)&bt`A`T0X;vCZs!$nX#$Yq8)uQ>yJHj0Z zj*075JLu(@-8(?OiPlk&TpJ#qXXSS86Kp6c3)-2vgoRX)KC;XuK-=x7-A9A~FY}ga z$XNP27i~AEyhf$;13J}3-Bqfh$14y<-HXp`Z5QGb@EqY#ZHZpPRw`TUh?+S7TVr4z)nXc|H z(Fr-*S_%{09vRZa_tvI@hDm;g#`mARSF}P>*P6*V(nHEjU6Bx9p#M8E)s#wOnKQKZ zE@+ekqe+UhwQIf2n}%z7s$*31&@}6ISu>>loUrIjcJ85v>GqtZdNpn5ZXO&Ej}3WtAsB8H`*D#+J^woIG<9Es6~G|G4x!#0Z^|y}jOx z*4+bZ+VFpPKh~}eM%Ri}h$=&=IF>prwGY;^`|Wwx)3vSPD66SUt<|CrFec|#VM)*L zlJDmCQ>I`8C*24`9)kJmT*m9c4)%FLIIUsq=ufdiJx|6Z6`dJ!DMQv^6}X{BwK|`` z09-?p@9)ohZt_ZxONf?7d#VtJc;nSqXoSfSUz0D@fZ)P66T%kJTOHE#`OQG=yjjUC zFXdgEIN$c1#@-bqGpE>s+UJ-#X`TNjw9aptzuB?lTP%$XWlwsyR+e(CD%em z_TxcAKO0aW43wI!bMky>$k@1&su=_~C!tbt9Wv6mh60|BN)|@oLOemfQx20n3F|$F zZJHI+?gmCcx{(8rljyj8cc}X5M2X{Ke}KjQ*wj2Je=2T%{F^KvYfqZcZ2E8&zmUDW zYPv+OYHg=3#F9&5xKPkt&?3)1G$bS?C#fFt#71EM*6tX8U$S%+p8%P6M+3S2q1ifU zx1pJo7@aPJ3k`Ie3U?m5ccMs?Z_2Z;wLQ!yOY~xU4IHOGjbQ{J_a996A`|=>9SxbG z37GFH?HL-k?EI~uO4Pi zO*e8RKl?SNJJzqGe<124m##4;$09vkO3_-LxBQwjc^M`9lo+u(;Jd)$T-qGIA5u4* z51j!DAqfheH5b+$x8QczR&~XjqM|hef;73TbMaFAnts#cts+%0pQSS_G6Lp}Qs*iw z&Vu3+`d>|@w7%O0@?E>^Yj zu?Obg1=i3)%tBZRtw8*}|Eb%Fc=mUWn^(j^;Dv4b+t1s)&mqdjGDG z?R{B>1DjtkC?pReWn$kS&ju}niKrVG7x80$Em5rIu?NR&J7vmP^D``nn zMA1{v=-$R;)hm)*C?K3 zaPaRKU7R79eeeH|6(;h(=#o@}@q@&wd`*;JC({?cKz|iRR6XZ3kdU{@-)A6y=sEdW z(8zscxX5-ry^f44zj~MX;ijZ$l@V60rjGvJs}&4R@XDFa!sLDMFbbjY7cEo}p(?e3 z5>I4Q%@)GFkd$IKKkQxdxPm?H6)3z2*GQhKG1X`puPItvx8K{;>zPTuGUP(%9zXaI zrB;%<94p*cKggW+>w$*|wg`bW7qUqYk_&u8@{yWuiUj>>4nDs%5aTZLTgNsau9aZN z#dY5cjKtMFD}ljbbkZ-{&5EjmG*{2V(Pvj5ubCt!_-XPR`7mwtB42KJ#-xWq{*gPjYL?7H4Y6|0$Y>r6szOPh?S5- z`LHzKLt=va(0punk%SF}Ike9sd^D}M+ZcKN@dDm^OV-|LpznQrqQ|-5FLnb$h*#Rs z`e2m0?pVrDqm3p}mMgx_kJ|3S?;@|59`-CS&%90@D~w&>^CR!_iK;;v)alh)KfIA| zbx5t_kaL~mkQSp!X+Y|k07k=4+mcW2aeUEQQuV5R;zUqp%SL#B^H#igt%5GPQS4j} z=1t9A(>#boqK3M(S+1hwEGCt$f-np(Pt%baH%9z5kcQI8k<=w6RFDy=tlb@#Ii3ZN zKnIh1%pAw_mgD}gtUV~=E}dOI_je<1u|2BgF~y`|#jA(r#}fk{=yTaX4Oa7ImDfqVEfsFGLgEznrQWQl4F`|;)Al*R?ceN7J1`@Itk9tc zdiGW@YigN9j!&h%W!)yZqV_)L59`|XUGm+ARciBMYh7&}#6v*?sjf=dxvHDxRPVGJ z3|JzCzq+U{_iFFDKn~021DLRb{iL2+>>6r}VXW+ulQ!kI=D2uwpmu%Ur9El8-a072 zGl(!)32Il!&PS*FF4xy1{B)&S!+e}_W&WUW&u2_J!`G0}ip72uh4s?s!~gCL`3~dg*mPO=f%;lQOmXX($)Rdoddh zmKrSVij~t$#I}s6WuS!@G{eLe2rjyAK0(&bl82Tsx1ek1yteL-FyArbrUqTAYPPk= z^VWWNO5J+tR`3e3v=*XcM3eOek-1iYB4hbeB8h6>Wv@ZJ&G+jqY=#8MxLrJgx^cGXO}?TNy7L;UM6I92a|m*xgq zW@TlK&CFy(MMX_aP8I`&h#y?bNwzOM-kk5~eYgJWpzr@Xp8R?P0FD2?|D^vf37|Iy zMZR}H&IzD1uU|L~oWB+Izd2)pJ#wo#hxGB*Spl_-I}s6OXV^~J ze1G^YHUOEP?y}SCvO6+)kCEOvR?D3J7qMe+GQN%))npi#y#Y=DX$to}sL9YPzVht* zdr$P-plu_}uf-ukrBn5;KZ=wv+>PNXE*nd-1Maf1iQ2#exbTfIP}6qc>({U5A8&v8 zf$$O@+Y}eVU{b}#*V)+E%Bri4T;3J_Y9j3L5HPwgT{krrV*{6&OE`IZKllgFCoK2A zzH4K%;Qg`i5wDfP%A;z#r29$O+sR3KT&6$8+S%9uNb@2u?|Inwwms*K<5IpBS1BT6 zx%t+-!J)Uzz5h||z3vFEB2!7R)c%qmMJbU^>dnjqVh_M&v{n66r%h|@M z68$Gn22-)=;}}f!-&?x@Z*2p5{&IYLeDHu3uzo8da32U4FUA5;FXOz+U%+VDh7s>B z{^LP78mD13^5BG`JRs3=kc`^{k!=U0fq%gIH(pb*)J;sgCLl^cFX$B;6E(uMe9G^8 zL!s@n-`B$D9AANo$=A~d0AT+5?cLNdKE%l)4woO!w6C2Pd3f%X$zKw`1`H@yG+rsE zUqAXy$!*bJe)PuIPL-fEfg7Zgbh@uPbMEkCPDR>kW)r}s?-yp1EzFdwS?c@02o_WsU=Kb!^t_P;QSCktt zqgMayrKqGAuzR}eT-5&(`76bo=F<7+87KbtxOo1%*?#TfpDA91DJycR8~mJC+ww-3 z7xvHlhX&~X8Pwv5ImZ%p;>0(;cmJ;{CnEXs`?2X14^4E&UjF9b_8O%)N4T`{-R?Oj z=h5V=7D0RUy4l+cqj_`%kD)z&bsV3IBA%bMHVF6UVCJa*yguRRngIIE@B-af197!( zRdP6WfcsrQVaXl(f4_9O=nXX2$x%LcS&($9T(X2!#wlSs3TkaKR(Tt`psX+8l)I$_$9=`R<2)KFxhO_9245 znL!4>^E+h65D2;y6ZK|tna+6du41k4ALK=PbjWxOtvXg!~18;BIv?VqJ~+!XpLc2&ZHx-j1uXmbQh4zY5vAEeUm*TY`$Qmnm6v+)wU0#9A3I#?Yohg6@)mc+CMko>M* zCxqVs?f>C?wzX(YaTV_-(yUDzqps|wsLr*A-2mlRj_4-mxl^aAX1}%l*jn`Y^tmH* z)o&IVypsym&w@HzZs=&2;N}N>Cktsw5S!T@2TD*KBX_p`soF22wd>uLNU(5@4_Qan zxiU-3Y)S)t6E}$;vB(;M;?Zg!JOK!~7jGeN9|ocfZ*THbTiYts!3;ZxBA-B53zgpC zF@??sY9|K?pGkfLC3UFDaQ$}Od2NdAV(UORT!KE>7c}o_bYvzMH?J(zjgQHr%hwJA6TI5UoAm%;J*7hzp~xB8Y`5G zPFCiI2rGsy(}Tal2&-^un{4dbU-lZ_StV8t+l;&`ku!8NbX#862CIb6hE7~wP*rtRF*70{g?KemL_xdz z3W?ETA&t%yfy7JS`^@XiZ7yBXLAQ0I-CZeA&6RZMPTuAvgG+#4azL3}x0xkgo;LnW zZ#+BdFpHT$%~YiY5F#ufn7y5ZdP$1s$Q)f@2))2R`(d?$y2FhcXJmQSYIw-4C3>Vn zcZTMX&8iTgPYFn8!68XOeq2WkzHV`LR0ygmCw-~fX|!QG+k)=z)qh7fiZ+G_FnFvv zq1ty7DlX|Ih?#@?m;`PM5- z?&X($=T<+TJ_RVLJ_}YqYA|0d4d**{_-xx&QB_egKXN0}^igVKPrzXKyr__{L+kw4V4o%_{w z4U`m%Q3&6%ARxs26sr5+!hm3upkN~CKvlUHrDdGQM+|xt4XSsJu4Z6h z_3?CZ47r(gcGJ}BY42W6mJh1Gc1ii2q37qOnlHatGiSc}cW=7|xBAWNWfwzdUbZ-S zPO^J*%E8s|-7fE0{kb)D?aKLoI?nuk(i>Se4XLD^s^ez5E%j&k{>hd0xv;Thw`r>~ zyiVUvOFQTK{_G_CywhKllQoY%JG1}a@0mLn+kVfzWp(e}muXx69ekrx^?s6$P5Q0Q z-RF@yjQ>*7%k1~-KZOp+dw)K7)VP1&ah0>cKJE4Vxii1==bo9i95`nE%f~)FPi$X$ zrk>?~{nzp{mpHk8Zd&ult=P==Lx9=eHQLp``S)MzetkQ8O;_MryWHt@y7y<_p2*9; zJ^l9oSAOUJsq|jHRP~Ov^1iCK@qZmtSR<{XRug+)4c4cyaQww5b0Ty3+-b{}EdzEo zmMvVUSXx>N>4GeOG9v=s?xN`|8d4LBx)XF3K1+iPIl%p;wl=oS(8S1~e&E4_1W<>k zwdvprqf{p`VDmc-wsj$HvDeZ`;lF%cJ10F(tQ+&4mXJ{FM6=ZL{+WgOR&ze5Ur&lTB*Rv9Z#bWon=8QIbb94 z2Ml}mRqKm!CBMJty)=j`RIxjF?;_XCRVAQCcu4s4g};6QCjnJGfgLy_3kwUe4FaHh z0YIQ(acKC;6{}Z!uMF`5O=`S;9W1pn Date: Wed, 7 Aug 2024 15:35:53 +0800 Subject: [PATCH 168/791] =?UTF-8?q?=E4=BD=BF=E5=BD=93=E5=89=8D=E6=9A=82?= =?UTF-8?q?=E7=89=88=E6=9C=ACmindspore=E6=9A=82=E4=B8=8D=E6=94=AF=E6=8C=81?= =?UTF-8?q?dump=E6=95=B0=E6=8D=AE=E7=9A=84=E5=A4=9A=E5=8D=A1=E6=AF=94?= =?UTF-8?q?=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/exceptions.py | 2 +- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 2 +- debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py | 4 ++-- debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 2 +- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/exceptions.py b/debug/accuracy_tools/msprobe/core/common/exceptions.py index ea61f8cd5..eb314c7c6 100644 --- a/debug/accuracy_tools/msprobe/core/common/exceptions.py +++ b/debug/accuracy_tools/msprobe/core/common/exceptions.py @@ -85,4 +85,4 @@ class DistributedNotInitializedError(Exception): self.msg = msg def __str__(self): - return self.msg + return self.msg \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 7d2be9c4c..240c18f1f 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -1,5 +1,5 @@ from msprobe.core.compare.check import check_op -from msprobe.core.common.const import CompareConst +from msprobe.core.common.const import CompareConst from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ get_error_message from msprobe.core.common.exceptions import FileCheckException diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py index 361e957f2..175f52ade 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py @@ -17,8 +17,8 @@ def compare_cli_ms(args): ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: - kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} - compare_distributed(npu_path, bench_path, args.output_path, **kwargs) + logger.error('This function is not supported at this time.') + raise Exception("Mindspore Unsupport function compare_distributed.") else: logger.error("The npu_path and bench_path need to be of the same type.") raise CompareException(CompareException.INVALID_COMPARE_MODE) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 34d37b4fe..267aabff6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -12,7 +12,7 @@ from msprobe.core.common.file_check import FileChecker, FileOpen, create_directo from msprobe.core.common.const import CompareConst, FileCheckConst from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op -from msprobe.core.compare.Multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process +from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 75bc9d4f3..a43b2c2a1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -12,7 +12,7 @@ from msprobe.core.common.file_check import FileChecker, FileOpen, create_directo from msprobe.core.common.const import CompareConst, FileCheckConst from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op -from msprobe.core.compare.Multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process +from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger -- Gitee From 5cb77eae5b483c807b2abeac11f1fab77f509de0 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 7 Aug 2024 15:53:36 +0800 Subject: [PATCH 169/791] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E9=97=B4=E7=A9=BA=E8=A1=8C=EF=BC=8C=E7=A9=BA=E6=A0=BC=E6=95=B0?= =?UTF-8?q?=E9=87=8F=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/Multiprocessing_compute.py | 8 +++++--- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 2 ++ debug/accuracy_tools/msprobe/core/compare/check.py | 4 +++- debug/accuracy_tools/msprobe/core/compare/highlight.py | 5 +++-- debug/accuracy_tools/msprobe/core/compare/match.py | 2 ++ debug/accuracy_tools/msprobe/core/compare/utils.py | 6 ++++-- .../msprobe/mindspore/compare/compare_cli.py | 1 - .../msprobe/mindspore/compare/distributed_compare.py | 1 + .../msprobe/mindspore/compare/ms_compare.py | 5 +++++ .../msprobe/pytorch/compare/distributed_compare.py | 2 ++ .../accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 4 ++++ 11 files changed, 31 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py index 9d8e9744e..20e3c1d0c 100644 --- a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py @@ -3,8 +3,8 @@ import multiprocessing import pandas as pd from dataclasses import dataclass from msprobe.core.common.log import logger -from msprobe.core.common.utils import CompareException -from msprobe.core.common.const import CompareConst +from msprobe.core.common.utils import CompareException +from msprobe.core.common.const import CompareConst def _handle_multi_process(func, input_parma, result_df, lock): @@ -38,6 +38,7 @@ def _handle_multi_process(func, input_parma, result_df, lock): pool.join() return pd.concat(final_results, ignore_index=True) + def read_dump_data(result_df): try: npu_dump_name_list = result_df.iloc[0:, 0].tolist() @@ -55,7 +56,6 @@ def read_dump_data(result_df): logger.error('result dataframe elements can not be access.') raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - @dataclass class ComparisonResult: cos_result: list @@ -65,6 +65,7 @@ class ComparisonResult: one_thousand_err_ratio_result: list five_thousand_err_ratio_result: list + def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): """ Save comparison results into the result DataFrame with thread safety. @@ -99,6 +100,7 @@ def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): finally: lock.release() + def check_accuracy(cos, max_abs_err): if cos == CompareConst.SHAPE_UNMATCH: return CompareConst.ACCURACY_CHECK_UNMATCH diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 240c18f1f..1d11f120b 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -9,6 +9,7 @@ class Comparator: def __init__(self): pass + def match_op(self,npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): if check_op(npu_queue[-1], b_op, fuzzy_match): @@ -20,6 +21,7 @@ class Comparator: return n_index, len(bench_queue) - 1 return -1, -1 + def compare_by_op(self,op_name, op_name_mapping_dict, input_parma): npu_bench_name_list = op_name_mapping_dict[op_name] data_name = npu_bench_name_list[1] diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index a8ee3638a..97ddc26cd 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -3,7 +3,6 @@ from msprobe.core.common.log import logger from msprobe.core.compare.utils import rename_api - def check_struct_match(npu_dict, bench_dict): npu_struct_in = npu_dict.get("input_struct") bench_struct_in = bench_dict.get("input_struct") @@ -18,6 +17,7 @@ def check_struct_match(npu_dict, bench_dict): is_match = struct_in_is_match and struct_out_is_match return is_match + def check_type_shape_match(npu_struct, bench_struct): shape_type_match = False for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct): @@ -38,6 +38,7 @@ def check_type_shape_match(npu_struct, bench_struct): return False return shape_type_match + def check_graph_mode(a_op_name, b_op_name): if "Aten" in a_op_name and "Aten" not in b_op_name: return True @@ -75,6 +76,7 @@ def fuzzy_check_op(npu_name_list, bench_name_list): break return is_match + def fuzzy_check_name(npu_name, bench_name): if "forward" in npu_name and "forward" in bench_name: is_match = rename_api(npu_name, "forward") == rename_api(bench_name, "forward") diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 17dee2f50..21cab0839 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -9,8 +9,8 @@ from msprobe.core.common.utils import get_header_index from msprobe.core.common.const import CompareConst from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException -from msprobe.core.common.file_check import change_mode -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.file_check import change_mode +from msprobe.core.common.const import CompareConst, FileCheckConst class HighlightCheck(abc.ABC): @@ -166,6 +166,7 @@ def get_name_and_state(name): state = "output" return api_name, state + def find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare): """将dataframe根据API分组，并找到有误差的算子用于高亮""" result = result_df.values diff --git a/debug/accuracy_tools/msprobe/core/compare/match.py b/debug/accuracy_tools/msprobe/core/compare/match.py index 6347d8887..acab42585 100644 --- a/debug/accuracy_tools/msprobe/core/compare/match.py +++ b/debug/accuracy_tools/msprobe/core/compare/match.py @@ -11,12 +11,14 @@ class AtenIrMapping(): with FileOpen(yaml_path, 'r') as f: self.aten_mapping = yaml.safe_load(f) + def match(self, op1, op2): if "Aten" in op1 and "Aten" not in op2: return self.match_op(op1, op2) else: return self.match_op(op2, op1) + def match_op(self, aten_op, torch_op): try: aten_op_raw_name_overload = '_'.join(aten_op.split("_")[1:-3]) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index de4047fd9..55c1abd41 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -4,8 +4,6 @@ import numpy as np from msprobe.core.common.const import Const, CompareConst - - def rename_api(npu_name, process): npu_split = npu_name.split(process) torch_func_index, in_out = npu_split[0], npu_split[1] @@ -13,6 +11,7 @@ def rename_api(npu_name, process): torch_func = str(torch_func_split[0]) + str(in_out) return torch_func + def read_op(op_data, op_name): op_parsed_list = [] if 'forward' in op_name: @@ -50,6 +49,7 @@ def read_op(op_data, op_name): output_parsed_list.clear() return op_parsed_list + def op_item_parse(item, op_name, index, item_list=None, top_bool=True): if item_list is None: item_list = [] @@ -121,6 +121,7 @@ def op_item_parse(item, op_name, index, item_list=None, top_bool=True): op_item_parse(item_spec, full_op_name, j, item_list=item_list, top_bool=False) return item_list + def resolve_api_special_parameters(data_dict, full_op_name, item_list): """ Function Description: @@ -269,6 +270,7 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals get_accuracy_core(n_num_input, n_num_kwarg, b_num_input, b_num_kwarg, "kwargs_struct") get_accuracy_core(n_num_input + n_num_kwarg, n_num_output, b_num_input + b_num_kwarg, b_num_output, 'output_struct') + def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): index_out = 0 npu_stack_info = n_dict.get("stack_info", None) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py index 175f52ade..368a95310 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py @@ -4,7 +4,6 @@ from msprobe.core.common.const import FileCheckConst from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import ms_compare -from msprobe.mindspore.compare.distributed_compare import compare_distributed def compare_cli_ms(args): diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index cab07daec..0973e7ffe 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -24,6 +24,7 @@ from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import MSComparator + def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): def check_and_return_dir_contents(dump_dir, prefix): """ diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 267aabff6..b8f29745a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -72,6 +72,7 @@ class MSComparator (Comparator): merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) return merge_list + def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles npu_json_data = json.load(npu_json_handle) @@ -136,6 +137,7 @@ class MSComparator (Comparator): result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df + def make_result_table(self,result,md5_compare,summary_compare,stack_mode): header = [] if md5_compare: @@ -163,6 +165,7 @@ class MSComparator (Comparator): result_df = pd.DataFrame(result, columns=header) return result_df + def _do_multi_process(self,input_parma, result_df): try: result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) @@ -171,6 +174,7 @@ class MSComparator (Comparator): logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e + def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -182,6 +186,7 @@ class MSComparator (Comparator): return data_value + def compare_core(self,input_parma, output_path, **kwargs): """ Compares data from multiple JSON files and generates a comparison report. diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index aeea94945..f4596ba49 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -24,6 +24,7 @@ from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.pytorch.compare.pt_compare import PTComparator + def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): def check_and_return_dir_contents(dump_dir, prefix): """ @@ -54,6 +55,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): raise CompareException(CompareException.INVALID_PATH_ERROR) return contents + def extract_json(dirname, stack_json=False): json_path = '' for fname in os.listdir(dirname): diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index a43b2c2a1..fe2d4fd76 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -18,6 +18,7 @@ from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException + class PTComparator (Comparator): def __init__(self): super().__init__() @@ -138,6 +139,7 @@ class PTComparator (Comparator): result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df + def make_result_table(self,result,md5_compare,summary_compare,stack_mode): header = [] if md5_compare: @@ -165,6 +167,7 @@ class PTComparator (Comparator): result_df = pd.DataFrame(result, columns=header) return result_df + def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -185,6 +188,7 @@ class PTComparator (Comparator): logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e + def compare_core(self,input_parma, output_path, **kwargs): """ Compares data from multiple JSON files and generates a comparison report. -- Gitee From 0236e1f6af51262858c4cf1c1a457592014da5a6 Mon Sep 17 00:00:00 2001 From: yangxinxian <947098055@qq.com> Date: Wed, 7 Aug 2024 16:06:52 +0800 Subject: [PATCH 170/791] primitive op dump --- .../msprobe/core/common/const.py | 1 + .../msprobe/core/data_dump/data_collector.py | 16 ++ .../core/data_dump/data_processor/base.py | 40 +++++ .../data_processor/mindspore_processor.py | 5 +- .../mindspore/debugger/precision_debugger.py | 4 +- .../msprobe/mindspore/doc/dump.md | 12 +- .../msprobe/mindspore/service.py | 152 +++++++++++++++++- .../test/mindspore_ut/test_primitive_dump.py | 82 ++++++++++ 8 files changed, 304 insertions(+), 8 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 35946ca7c..929686920 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -16,6 +16,7 @@ class Const: OFF = 'OFF' BACKWARD = 'backward' FORWARD = 'forward' + PRIMITIVE_PREFIX = 'Primitive' DEFAULT_LIST = [] DEFAULT_PATH = './' WHITE_LIST = 'white_list' diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index db437539a..7acc607f1 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -106,6 +106,22 @@ class DataCollector: raise Exception("[msprobe] exit") self.handle_data(name, data_info) + def backward_input_data_collect(self, name, module, pid, module_input_output): + self.update_construct(name) + if not self.check_scope_and_pid(self.scope, name, pid): + return + + data_info = self.data_processor.analyze_backward_input(name, module, module_input_output) + self.handle_data(name, data_info) + + def backward_output_data_collect(self, name, module, pid, module_input_output): + self.update_construct(name) + if not self.check_scope_and_pid(self.scope, name, pid): + return + + data_info = self.data_processor.analyze_backward_output(name, module, module_input_output) + self.handle_data(name, data_info) + def update_construct(self, name): if self.config.level not in DataCollector.level_without_construct: self.data_writer.update_construct({name: self.module_processor.api_parent_node}) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 2fbc86b56..fcb522d11 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -41,6 +41,24 @@ class ModuleBackwardInputsOutputs: return convert_tuple(self.grad_output) +@dataclass +class ModuleBackwardInputs: + grad_input: Optional[Tuple] + + @property + def grad_input_tuple(self): + return convert_tuple(self.grad_input) + + +@dataclass +class ModuleBackwardOutputs: + grad_output: Optional[Tuple] + + @property + def grad_output_tuple(self): + return convert_tuple(self.grad_output) + + class TensorStatInfo: def __init__(self, max_val=None, min_val=None, mean_val=None, norm_val=None): self.max = max_val @@ -228,6 +246,28 @@ class BaseDataProcessor: return api_info_struct + def analyze_backward_input(self, name, module, + module_input_output: ModuleBackwardInputs): + api_info_struct = {} + if self.is_dump_for_data_mode(Const.BACKWARD, Const.INPUT): + api_info_struct[name] = {} + self.api_data_category = Const.INPUT + + input_info_list = self.analyze_element(module_input_output.grad_input_tuple) + api_info_struct[name][Const.INPUT] = input_info_list + return api_info_struct + + def analyze_backward_output(self, name, module, + module_input_output: ModuleBackwardOutputs): + api_info_struct = {} + if self.is_dump_for_data_mode(Const.BACKWARD, Const.OUTPUT): + api_info_struct[name] = {} + self.api_data_category = Const.OUTPUT + + output_info_list = self.analyze_element(module_input_output.grad_output_tuple) + api_info_struct[name][Const.OUTPUT] = output_info_list + return api_info_struct + def get_save_file_path(self, suffix): file_format = Const.PT_SUFFIX if self.config.framework == Const.PT_FRAMEWORK else Const.NUMPY_SUFFIX dump_data_name = (self.current_api_or_module_name + Const.SEP + self.api_data_category + Const.SEP + diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index c208df7d9..b28817e4a 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -74,8 +74,9 @@ class MindsporeDataProcessor(BaseDataProcessor): if data.numel() == 0: return tensor_stat elif data.dtype == ms.bool_: - tensor_stat.max = self.mint_ops_func["max"](data).item() - tensor_stat.min = self.mint_ops_func["min"](data).item() + data_np = data.asnumpy() + tensor_stat.max = np.max(data_np) + tensor_stat.min = np.min(data_np) elif not data.shape: tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data.item() elif data.dtype == ms.complex64 or data.dtype == ms.complex128: diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 5475dc358..40b44c57e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -41,7 +41,7 @@ class PrecisionDebugger: return MsConst.PYNATIVE_MODE @classmethod - def start(cls): + def start(cls, target=None): instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") @@ -50,7 +50,7 @@ class PrecisionDebugger: if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.level == MsConst.API: if not instance.service: instance.service = Service(instance.config) - instance.service.start() + instance.service.start(target) else: if not instance.first_start: handler = TaskHandlerFactory.create(instance.config) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md index 425d0683a..ef2431b9c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md @@ -35,10 +35,18 @@ PrecisionDebugger(config_path=None) **原型** ```Python -debugger.start() +debugger.start(model = None) ``` -该函数为类函数，可以使用debugger.start()也可以使用PrecisionDebugger.start()。 +该函数为类函数，可以使用debugger.start(model = None)也可以使用PrecisionDebugger.start(model = None) + + +**参数说明** + +| 参数名 | 说明 | 是否必选 | +| ----------- |---------------------------------------------------------------------------------------| -------- | +| model | 指具体的mindspore.nn.Cell，默认未配置，L1级别下传入model可以使能对primitive op的dump，否则无法dump primitive op。 | 否 | + ## 示例代码 diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 50776aaf1..4c2a4ef69 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -19,6 +19,9 @@ from pathlib import Path import functools from collections import defaultdict +from mindspore.common.tensor import Tensor +from mindspore import ops +from mindspore import nn from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope from msprobe.mindspore.common.utils import get_rank_if_initialized @@ -27,7 +30,9 @@ from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import Const from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.mindspore.dump.hook_cell.api_registry import api_register -from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs +from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs,\ + ModuleBackwardInputs, ModuleBackwardOutputs +from msprobe.core.common.exceptions import MsprobeException from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell @@ -41,9 +46,18 @@ class Service: self.current_iter = 0 self.first_start = True self.current_rank = None + self.primitive_counters = {} self.dump_iter_dir = None self.start_call = False + @staticmethod + def check_model_valid(model): + if not model or isinstance(model, nn.Cell): + return model + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, "model 参数必须是 mindspore.nn.Cell 类型。" + ) + def build_hook(self, module_type, name): def forward_hook(api_or_module_name, module, input, output): self.data_collector.visit_and_clear_overflow_status(api_or_module_name) @@ -79,13 +93,145 @@ class Service: return wrap_forward_hook, wrap_backward_hook + + def wrap_primitive(self, origin_func, primitive_name): + service_instance = self + + def create_backward_hook(captured_grads, num_tensors, updated_primitive_name, hook_type): + def backward_hook(grad): + captured_grads.append(grad) + backward_primitive_name = f"{updated_primitive_name}.{Const.BACKWARD}" + try: + if len(captured_grads) == num_tensors and hook_type == Const.INPUT: + service_instance.data_collector.visit_and_clear_overflow_status(backward_primitive_name) + new_module_input_output = ModuleBackwardOutputs(grad_output=tuple(captured_grads)) + service_instance.data_collector.backward_output_data_collect( + backward_primitive_name, service_instance, os.getpid(), new_module_input_output + ) + captured_grads.clear() + elif len(captured_grads) == num_tensors and hook_type == Const.OUTPUT: + service_instance.data_collector.visit_and_clear_overflow_status(backward_primitive_name) + new_module_input_output = ModuleBackwardInputs(grad_input=tuple(captured_grads)) + service_instance.data_collector.backward_input_data_collect( + backward_primitive_name, service_instance, os.getpid(), new_module_input_output + ) + captured_grads.clear() + + except Exception as exception: + raise Exception( + "This is a primitive op {hook_type}_backward dump error: {exception}," + " updated_primitive_name: {updated_primitive_name}".format( + hook_type=hook_type, exception=exception, backward_primitive_name=backward_primitive_name + ) + ) from exception + + return backward_hook + + def hook_primitive_inputs(args, captured_grads_input, updated_primitive_name): + hooked_inputs = [] + num_tensors = sum(isinstance(arg, Tensor) for arg in args) + input_backward_hook = create_backward_hook(captured_grads_input, num_tensors, updated_primitive_name, + Const.INPUT) + for _, arg in enumerate(args): + if isinstance(arg, Tensor): + arg_hooked = ops.HookBackward(input_backward_hook)(arg) + hooked_inputs.append(arg_hooked) + else: + hooked_inputs.append(arg) + return hooked_inputs + + def hook_primitive_outputs(out, captured_grads_output, updated_primitive_name): + if isinstance(out, tuple): + num_output_tensors = sum(isinstance(tensor, Tensor) for tensor in out) + else: + num_output_tensors = 1 + output_backward_hook = create_backward_hook(captured_grads_output, num_output_tensors, + updated_primitive_name, Const.OUTPUT) + + if isinstance(out, Tensor): + return ops.HookBackward(output_backward_hook)(out) + elif isinstance(out, tuple): + hooked_outputs = [] + for tensor in out: + if isinstance(tensor, Tensor): + hooked_outputs.append(ops.HookBackward(output_backward_hook)(tensor)) + else: + hooked_outputs.append(tensor) + return tuple(hooked_outputs) + return out + + def wrapped_primitive_call(instance_self, *args, **kwargs): + service_instance.update_primitive_counters(primitive_name) + current_count = service_instance.primitive_counters.get(primitive_name, 0) + updated_primitive_name = f"{Const.PRIMITIVE_PREFIX}.{primitive_name}.{current_count}" + + if not service_instance.switch: + return origin_func(*args, **kwargs) + + captured_grads_input, captured_grads_output = [], [] + + try: + hooked_inputs = hook_primitive_inputs(args, captured_grads_input, updated_primitive_name) + except Exception as exception: + raise Exception("This is a primitive op dump error during input hooking: {}," + " primitive_name: {}".format(exception, primitive_name)) from exception + + try: + out = origin_func(*hooked_inputs, **kwargs) + except Exception as exception: + raise Exception("This is a primitive op dump error during function call: {}," + " primitive_name: {}".format(exception, primitive_name)) from exception + + forward_primitive_name = f"{updated_primitive_name}.{Const.FORWARD}" + service_instance.data_collector.visit_and_clear_overflow_status(forward_primitive_name) + if service_instance.data_collector: + module_input_output = ModuleForwardInputsOutputs(args=hooked_inputs, kwargs=kwargs, output=out) + try: + service_instance.data_collector.forward_data_collect(forward_primitive_name, instance_self, + os.getpid(), module_input_output) + except Exception as exception: + raise Exception("This is a primitive op dump error during forward data collection: {}," + " primitive_name: {}".format(exception, primitive_name)) from exception + + if service_instance.data_collector.if_return_forward_new_output(): + out = service_instance.data_collector.get_forward_new_output() + + try: + out = hook_primitive_outputs(out, captured_grads_output, updated_primitive_name) + except Exception as exception: + raise Exception("This is a primitive op dump error during output hooking: {}," + " primitive_name: {}".format(exception, primitive_name)) from exception + + return out + + return wrapped_primitive_call + + def update_primitive_counters(self, primitive_name): + if primitive_name not in self.primitive_counters: + self.primitive_counters[primitive_name] = 0 + else: + self.primitive_counters[primitive_name] += 1 + + def register_hooks(self): + primitive_set = set() + for _, cell in self.model.cells_and_names(): + for pname, primitive in cell._primitives.items(): + primitive_set.add((pname, primitive)) + + for pname, primitive in primitive_set: + NewPrimitive = type('NewPrimitive', (primitive.__class__,), + {'__call__': self.wrap_primitive(primitive.__call__, pname)}) + primitive.__class__ = NewPrimitive + + def step(self): self.current_iter += 1 self.data_collector.update_iter(self.current_iter) HOOKCell.cell_count = defaultdict(int) + self.primitive_counters.clear() def start(self, model=None): - self.model = model + self.model = Service.check_model_valid(model) self.start_call = True logger.info("msprobe: debugger.start() is set successfully") if self.config.step and self.current_iter > max(self.config.step): @@ -150,3 +296,5 @@ class Service: if self.config.level == "L1": api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) api_register.api_set_hook_func() + if self.model: + self.register_hooks() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py new file mode 100644 index 000000000..25189a9b6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import os + +import unittest +from unittest.mock import Mock, patch +import copy +from msprobe.core.common.utils import Const +from msprobe.mindspore.service import Service +import mindspore +from mindspore.common.tensor import Tensor +from mindspore import ops +from mindspore import nn +from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common_config import CommonConfig, BaseConfig +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from unittest.mock import MagicMock +import numpy as np + + +class DummyModel(nn.Cell): + def __init__(self): + super(DummyModel, self).__init__() + self.dense = nn.Dense(2, 2) + + def construct(self, x): + return self.dense(x) +class TestService(unittest.TestCase): + def setUp(self): + json_config = { + "task": "statistics", + "dump_path": "/absolute_path", + "rank": [], + "step": [0, 2], + "level": "L1" + } + + common_config = CommonConfig(json_config) + task_config = BaseConfig(json_config) + config = DebuggerConfig(common_config, task_config) + self.service = Service(config) + self.service.model = Mock() + self.service.data_collector = Mock() + self.service.switch = True # Make sure the switch is on for testing + + def test_check_model_valid_none(self): + model = None + self.assertIsNone(self.service.check_model_valid(model)) + + def test_check_model_valid_valid_model(self): + model = DummyModel() + self.assertEqual(self.service.check_model_valid(model), model) + + def test_check_model_valid_invalid_model(self): + model = "invalid_model" + with self.assertRaises(MsprobeException) as context: + self.service.check_model_valid(model) + + # For the purpose of the test, let's also verify the expected exception message + expected_message = "[msprobe] 无效参数： model 参数必须是 mindspore.nn.Cell 类型。" + self.assertEqual(str(context.exception), expected_message) + + def test_update_primitive_counters(self): + primitive_name = "test_primitive" + self.service.update_primitive_counters(primitive_name) + self.assertEqual(self.service.primitive_counters[primitive_name], 0) + self.service.update_primitive_counters(primitive_name) + self.assertEqual(self.service.primitive_counters[primitive_name], 1) -- Gitee From 26c95c1f488917e7148d8a0f13a756863fe2a654 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Wed, 7 Aug 2024 16:11:39 +0800 Subject: [PATCH 171/791] =?UTF-8?q?=E4=BD=BF=E7=94=A8construct.json?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E5=8F=8D=E5=90=91=E6=98=A0=E5=B0=84=E5=85=B3?= =?UTF-8?q?=E7=B3=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/module_processer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/module_processer.py index 688c0b8c5..cd91eedc0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/module_processer.py @@ -116,7 +116,9 @@ class ModuleProcesser: index = None pass module.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index) - ModuleProcesser.module_node[full_name] = None + forward_full_name = full_name.replace(Const.BACKWARD, Const.FORWARD) + ModuleProcesser.module_node[full_name] = ModuleProcesser.module_node[forward_full_name].replace( + Const.FORWARD, Const.BACKWARD) if ModuleProcesser.module_node[forward_full_name] else None ModuleProcesser.api_parent_node = None if self.scope: self.scope.begin_module(full_name) -- Gitee From 588c50d542733d935db96713f24ae6ac65f4112f Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Wed, 7 Aug 2024 11:06:00 +0800 Subject: [PATCH 172/791] =?UTF-8?q?mindspore=E4=BE=A7=E6=A2=AF=E5=BA=A6?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E6=90=AC=E8=BF=81=E8=87=B3msprobe?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/config/config.json | 7 +- .../msprobe/core/common/utils.py | 24 +- .../msprobe/core/grad_probe/constant.py | 19 +- .../msprobe/core/grad_probe/grad_compare.py | 9 +- .../msprobe/core/grad_probe/utils.py | 43 ++++ .../mindspore/debugger/precision_debugger.py | 29 ++- .../msprobe/mindspore/grad_probe/__init__.py | 0 .../mindspore/grad_probe/global_context.py | 91 +++++++ .../mindspore/grad_probe/grad_analyzer.py | 231 ++++++++++++++++++ .../mindspore/grad_probe/grad_monitor.py | 27 ++ .../mindspore/grad_probe/grad_stat_csv.py | 132 ++++++++++ .../msprobe/mindspore/grad_probe/hook.py | 92 +++++++ .../msprobe/mindspore/grad_probe/utils.py | 29 +++ .../msprobe/mindspore/ms_config.py | 9 +- .../pytorch/debugger/precision_debugger.py | 10 +- .../pytorch/grad_probe/grad_monitor.py | 49 ++-- .../pytorch/grad_probe/grad_stat_csv.py | 14 +- .../msprobe/pytorch/pt_config.py | 4 +- .../pytorch_ut/grad_probe/test_grad_csv.py | 13 +- .../grad_probe/test_grad_monitor.py | 22 +- 20 files changed, 782 insertions(+), 72 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_monitor.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_stat_csv.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py diff --git a/debug/accuracy_tools/msprobe/config/config.json b/debug/accuracy_tools/msprobe/config/config.json index 8603771f8..bc9789a38 100644 --- a/debug/accuracy_tools/msprobe/config/config.json +++ b/debug/accuracy_tools/msprobe/config/config.json @@ -31,12 +31,9 @@ "error_data_path": "./" }, "grad_probe": { - "level": "L1", + "grad_level": "L1", "param_list": [], - "rank": [], - "step": [], - "bounds": [-1, 0, 1], - "output_path": "./grad_output" + "bounds": [-1, 0, 1] }, "free_benchmark": { "scope": [], diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index cde65dd0e..7a34a2411 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -27,7 +27,7 @@ from datetime import datetime, timezone from pathlib import Path import numpy as np -from msprobe.core.common.file_check import FileOpen, FileChecker +from msprobe.core.common.file_check import FileOpen, FileChecker, change_mode from msprobe.core.common.const import Const, FileCheckConst, CompareConst, OverflowConst from msprobe.core.common.log import logger @@ -258,6 +258,17 @@ def remove_path(path): raise CompareException(CompareException.INVALID_PATH_ERROR) from err +def move_file(src_path, dst_path): + check_file_or_directory_path(src_path) + check_path_before_create(dst_path) + try: + shutil.move(src_path, dst_path) + except Exception as e: + logger.error(f"move file {src_path} to {dst_path} failed") + raise RuntimeError(f"move file {src_path} to {dst_path} failed") from e + change_mode(dst_path, FileCheckConst.DATA_FILE_AUTHORITY) + + def get_dump_data_path(dump_dir): """ Function Description: @@ -515,10 +526,19 @@ def write_csv(data, filepath): def load_npy(filepath): - filepath = os.path.realpath(filepath) check_file_or_directory_path(filepath) try: npy = np.load(filepath) except Exception as e: raise RuntimeError(f"load npy file {filepath} failed") from e return npy + + +def save_npy(data, filepath): + filepath = os.path.realpath(filepath) + check_path_before_create(filepath) + try: + npy = np.save(filepath, data) + except Exception as e: + raise RuntimeError(f"save npy file {filepath} failed") from e + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/constant.py b/debug/accuracy_tools/msprobe/core/grad_probe/constant.py index 38d33e988..189ec2d11 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/constant.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/constant.py @@ -39,7 +39,7 @@ class GradConst: DIRECTORY_LENGTH = 4096 FILE_NAME_LENGTH = 255 FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" - PARAM_VALID_PATTERN = r"^[a-zA-Z0-9.]+$" + PARAM_VALID_PATTERN = r"^[a-zA-Z0-9_.]+$" DIR = "dir" FILE = "file" @@ -53,4 +53,19 @@ class GradConst: SHAPE = "shape" MAX = "max" MIN = "min" - NORM = "norm" \ No newline at end of file + NORM = "norm" + +level_adp = { + "L0": { + "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": False + }, + "L1": { + "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": True + }, + "L2": { + "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": True + }, + } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 26cba34f0..22acdf2fb 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -10,7 +10,6 @@ from msprobe.core.common.file_check import create_directory from msprobe.core.common.log import logger from msprobe.core.common.utils import remove_path, write_csv, load_npy from msprobe.core.grad_probe.constant import GradConst -from msprobe.pytorch.common.utils import load_pt class GradComparator: @@ -163,12 +162,8 @@ class GradComparator: @classmethod def _load_grad_files(cls, grad_file1: str, grad_file2: str): - if grad_file1.endswith('pt'): - grad1 = load_pt(grad_file1).numpy() - grad2 = load_pt(grad_file2).numpy() - else: - grad1 = load_npy(grad_file1) - grad2 = load_npy(grad_file2) + grad1 = load_npy(grad_file1) + grad2 = load_npy(grad_file2) if grad1.shape != grad2.shape: raise RuntimeError(f"tensor shape is not equal: {grad_file1}, {grad_file2}") if grad1.dtype != bool: diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py index 05dd9a568..f5db74baa 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py @@ -1,3 +1,8 @@ +import re +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.log import logger +from msprobe.core.common.utils import write_csv + def data_in_list_target(data, lst): return not lst or len(lst) == 0 or data in lst @@ -7,3 +12,41 @@ def check_numeral_list_ascend(lst): raise Exception("The input list should only contain numbers") if lst != sorted(lst): raise Exception("The input list should be ascending") + + +def check_param(param_name): + if not re.match(GradConst.PARAM_VALID_PATTERN, param_name): + raise RuntimeError("The parameter name contains special characters.") + + +def check_str(string, variable_name): + if not isinstance(string, str): + raise ValueError(f'The variable: "{variable_name}" is not a string.') + + +class ListCache(list): + threshold = 1000 + + def __init__(self, *args): + super().__init__(*args) + self._output_file = None + + def __del__(self): + self.flush() + + def flush(self): + if len(self) == 0: + return + if not self._output_file: + logger.warning("dumpfile path is not setted") + write_csv(self, self._output_file) + logger.info(f"write {len(self)} items to {self._output_file}.") + self.clear() + + def append(self, data): + list.append(self, data) + if len(self) >= ListCache.threshold: + self.flush() + + def set_output_file(self, output_file): + self._output_file = output_file diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 6ef1966bc..957af5643 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -10,11 +10,14 @@ from msprobe.core.common.const import Const from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.runtime import Runtime +from msprobe.mindspore.grad_probe.grad_monitor import GradientMonitor + class PrecisionDebugger: _instance = None + task_not_need_service = [Const.GRAD_PROBE] - def __new__(cls, config_path=None): + def __new__(cls, config_path=None, opt=None): if not cls._instance: cls._instance = super().__new__(cls) cls._instance.initialized = False @@ -26,11 +29,16 @@ class PrecisionDebugger: def __init__(self, config_path=None): if self.initialized: return + self.initialized = True if not config_path: config_path = os.path.join(os.path.dirname(__file__), "../../config/config.json") common_config, task_config = parse_json_config(config_path) + self.task = common_config.task + if self.task == Const.GRAD_PROBE: + self.gm = GradientMonitor(common_config, task_config) + return self.config = DebuggerConfig(common_config, task_config) - self.initialized = True + Runtime.step_count = 0 Runtime.is_running = False @@ -49,6 +57,8 @@ class PrecisionDebugger: instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") + if instance.task in PrecisionDebugger.task_not_need_service: + return instance.config.execution_mode = instance._get_execution_mode() if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.level == MsConst.API and \ @@ -69,6 +79,10 @@ class PrecisionDebugger: instance = cls._instance if not instance: raise Exception("PrecisionDebugger instance is not created.") + if instance.task == Const.GRAD_PROBE: + instance.gm.stop() + if instance.task in PrecisionDebugger.task_not_need_service: + return if instance.service: instance.service.stop() Runtime.is_running = False @@ -78,6 +92,17 @@ class PrecisionDebugger: instance = cls._instance if not instance: raise Exception("PrecisionDebugger instance is not created.") + if instance.task in PrecisionDebugger.task_not_need_service: + return if instance.service: instance.service.step() Runtime.step_count += 1 + + @classmethod + def monitor(cls, opt): + instance = cls._instance + if not instance: + raise Exception("PrecisionDebugger instance is not created.") + if instance.task != Const.GRAD_PROBE: + return + instance.gm.monitor(opt) diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/__init__.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py new file mode 100644 index 000000000..16d0bd0b8 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -0,0 +1,91 @@ +import os +import threading +from typing import Dict, Union + +from msprobe.core.grad_probe.utils import check_str +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.log import logger +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.utils import check_path_before_create + + +class GlobalContext: + + _instance = None + _instance_lock = threading.Lock() + _setting = { + GradConst.LEVEL: None, + GradConst.PARAM_LIST: None, + GradConst.STEP: None, + GradConst.RANK: None, + GradConst.CURRENT_STEP: 0, + GradConst.BOUNDS: [-10, -1, -0.1, -0.01, -0.001, 0, 0.001, 0.01, 0.1, 1, 10], + GradConst.OUTPUT_PATH: None + } + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance_lock.acquire() + cls._instance = object.__new__(cls) + cls._instance_lock.release() + return cls._instance + + def init_context(self, config_dict: Dict): + level = config_dict.get(GradConst.LEVEL) + check_str(level, variable_name = "level in yaml") + if level in GradConst.SUPPORTED_LEVEL: + self._setting[GradConst.LEVEL] = config_dict.get(GradConst.LEVEL) + else: + raise ValueError("Invalid level set in config yaml file, level option: L0, L1, L2") + + self._set_input_list(config_dict, GradConst.PARAM_LIST, str) + self._set_input_list(config_dict, GradConst.BOUNDS, float) + self._set_input_list(config_dict, GradConst.STEP, int) + self._set_input_list(config_dict, GradConst.RANK, int) + + output_path = config_dict.get(GradConst.OUTPUT_PATH) + check_str(output_path, variable_name = "output_path in yaml") + try: + check_path_before_create(output_path) + except RuntimeError as err: + raise ValueError(f"Invalid output_path: {output_path}. The error message is {err}.") from err + self._setting[GradConst.OUTPUT_PATH] = output_path + if not os.path.isdir(self._setting.get(GradConst.OUTPUT_PATH)): + create_directory(self._setting.get(GradConst.OUTPUT_PATH)) + else: + logger.warning("The output_path exists, the data will be covered.") + + def get_context(self, key: str): + if key not in self._setting: + logger.warning(f"Unrecognized {key}.") + return self._setting.get(key) + + def update_step(self): + self._setting[GradConst.CURRENT_STEP] += 1 + + def step_need_dump(self, step): + dump_step_list = self.get_context(GradConst.STEP) + return (not dump_step_list) or (step in dump_step_list) + + def rank_need_dump(self, rank): + dump_rank_list = self.get_context(GradConst.RANK) + return (not dump_rank_list) or (rank in dump_rank_list) + + def _set_input_list(self, config_dict: Dict, name: str, dtype: Union[int, str, float]): + value = config_dict.get(name) + if dtype == int: + type_str = "integer" + elif dtype == float: + type_str = "float" + else: + type_str = "string" + if value and isinstance(value, list): + for val in value: + if not isinstance(val, dtype): + logger.warning(f"Invalid {name} which must be None or list of {type_str}") + return + self._setting[name] = value + else: + logger.warning(f"{name} is None or not a list with valid items, use default value.") + +grad_context = GlobalContext() diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py new file mode 100644 index 000000000..2bdc11114 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -0,0 +1,231 @@ +import os +import time +from typing import List, Tuple +import multiprocessing +from multiprocessing import Process + +import numpy as np +import mindspore as ms +from mindspore.communication import get_rank +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter + +from msprobe.core.grad_probe.utils import ListCache +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.log import logger +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file +from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext + + +def get_rank_id(): + try: + rank_id = get_rank() + except Exception as err: + rank_id = 0 + return rank_id + + +@ms.jit +def grad_dump(dump_dir: str, g_name: str, dump_step: Parameter, grad: ms.Tensor, level: str, bounds: List): + ''' + Dump gradient statistic data. + level0: [step, max, min, norm, shape_dim, shape] + level1: [step, max, min, norm, shape_dim, shape] + grad_bool_data + level2: [step, max, min, norm, shape_dim, shape, dist_dim, dist] + grad_bool_data + ''' + dump_path = os.path.join(dump_dir, g_name) + dump_dir_path = dump_path + "_dir" + save_op = ms.ops.TensorDump() + + grad_flat = grad.reshape(-1) + max_val = grad_flat.max(axis=0).float() + min_val = grad_flat.min(axis=0).float() + norm_val = grad_flat.norm(ord=2).float() + shape = grad.shape + extrem_list = [dump_step[0].float(), max_val, min_val, norm_val] + extrem_stat = ms.ops.stack(extrem_list) + shape_list = [len(shape)] + list(shape) + shape_stat = ms.Tensor(shape_list).float() + level0_stat = ms.ops.concat((extrem_stat, shape_stat), axis=0) + level_stat = level0_stat + + if level == GradConst.LEVEL2: + zero_grad = (grad == 0).sum() + dist_dim = ms.Tensor([len(bounds) + 2]).float() + bucket_result = ms.ops.bucketize(grad.float(), bounds) + bucket_result = bucket_result.astype(ms.int8) + dist_stat = [(bucket_result == i).sum() for i in range(len(bounds) + 1)] + dist_stat.append(zero_grad) + dist_stat.append(ms.Tensor(1, dtype=ms.int64)) # make sure dist_stat is not empty + dist_stat = ms.ops.stack(dist_stat, axis=0).float() + level2_stat = ms.ops.concat((level0_stat, dist_dim, dist_stat), axis=0) + level_stat = level2_stat + + save_op(dump_path, level_stat) + if level == GradConst.LEVEL1 or level == GradConst.LEVEL2: + grad_direction = grad > 0 + save_op(dump_dir_path, grad_direction) + + +class CSVGenerator(Process): + + def __init__(self) -> None: + super().__init__() + self.dump_dir = None + self.save_dir = None + self.level = GradConst.LEVEL0 + self.cache_list = ListCache() + self.current_step = None + self.stop_event = None + self.last_finish = False + self.bounds = [-0.1, 0.0, 0.1], + + def init(self, context: GlobalContext): + rank_id = get_rank_id() + output_path = context.get_context(GradConst.OUTPUT_PATH) + self.level = context.get_context(GradConst.LEVEL) + self.bounds = context.get_context(GradConst.BOUNDS) + self.dump_dir = f"{output_path}/rank{rank_id}/Dump/" + self.save_dir = f"{output_path}/rank{rank_id}/" + self.current_step = None + self.stop_event = multiprocessing.Event() + self.last_finish = False + + def run(self): + while True: + if not os.path.exists(self.dump_dir): + time.sleep(0.1) + if self.stop_event.is_set(): + break + continue + npy_files = os.listdir(self.dump_dir) + npy_files.sort(key=lambda x: int(x.split("_")[0])) + self.traverse_files(npy_files) + empty = len(os.listdir(self.dump_dir)) == 0 + if self.stop_event.is_set() and empty and self.last_finish: + break + if os.path.exists(self.dump_dir): + remove_path(self.dump_dir) + + def stop(self): + self.stop_event.set() + + def traverse_files(self, npy_files: List): + for npy_file in npy_files: + file_path = os.path.join(self.dump_dir, npy_file) + while not os.path.exists(file_path): + time.sleep(0.01) + check_file_or_directory_path(file_path) + if GradConst.STEP_FINISH in npy_file: + self.cache_list.flush() + remove_path(file_path) + self.last_finish = True + elif file_path.split("_")[-1] == GradConst.DIR_SUFFIX: + prefix_idx = len(npy_file.split("_")[0]) + new_name = npy_file[prefix_idx + 1:].replace("_" + GradConst.DIR_SUFFIX, "." + GradConst.NPY_SUFFIX) + if not new_name: + raise RuntimeError("Invalid dump data name.") + if self.current_step is None: + raise RuntimeError("Current record step is None.") + step_dir = os.path.join(self.save_dir, f"step{self.current_step}") + if not os.path.exists(step_dir): + create_directory(step_dir) + dst_file = os.path.join(step_dir, new_name) + move_file(file_path, dst_file) + self.last_finish = False + elif file_path.split(".")[-1] == GradConst.NPY_SUFFIX: + stat_data = self.load_npy_data(file_path) + if stat_data is None: + continue + if not self.check_valid(stat_data): + os.remove(file_path) + continue + step = int(stat_data[GradConst.STEP_IDX]) + update_step = self.current_step is None or step != self.current_step + self.current_step = step + if update_step: + self.create_csv_file() + self.gen_csv_line(file_path, stat_data) + os.remove(file_path) + self.last_finish = False + + def check_valid(self, stat_data): + level = grad_context.get_context(GradConst.LEVEL) + try: + shape_dim = int(stat_data[GradConst.SHAPE_DIM_IDX]) + if level == GradConst.LEVEL2: + dist_dim = int(stat_data[shape_dim + GradConst.SHAPE_DIM_IDX + 1]) + length = shape_dim + dist_dim + 7 + else: + length = shape_dim + 5 + except IndexError as err: + return False + if length != len(stat_data): + return False + return True + + def load_npy_data(self, file_path: str): + stat_data = None + max_try = 10 + while max_try: + try: + stat_data = np.load(file_path) + return stat_data + except Exception as err: + logger.warning(f"load numpy file failed, retry...") + max_try -= 1 + time.sleep(0.1) + return stat_data + + def gen_csv_line(self, file_path: str, stat_data) -> None: + shape_dim = int(stat_data[GradConst.SHAPE_DIM_IDX]) + file_name = os.path.basename(file_path) + prefix_idx = len(file_name.split("_")[0]) + param_name = file_name[(prefix_idx + 1) : -(len(GradConst.NPY_SUFFIX) + 1)] + if not param_name: + raise RuntimeError("Invalid gradient statistic file name.") + csv_line = [param_name] + if self.level == GradConst.LEVEL2: + csv_line.extend(self.get_dist_data(shape_dim, stat_data)) + csv_line.extend(self.get_extrem_data(shape_dim, stat_data)) + self.cache_list.append(csv_line) + + def get_dist_data(self, shape_dim: int, stat_data: np.ndarray): + dist_data = stat_data[(shape_dim + GradConst.SHAPE_DIM_IDX + 2):-1] + element_num = dist_data.sum() - dist_data[-1] + if element_num != 0: + dist_data = dist_data / element_num + return list(dist_data) + + def get_extrem_data(self, shape_dim: int, stat_data: np.ndarray): + extrem_data = list(stat_data[(GradConst.STEP_IDX + 1):(GradConst.STEP_IDX + 4)]) + shape_data = stat_data[(GradConst.SHAPE_DIM_IDX + 1):(GradConst.SHAPE_DIM_IDX + shape_dim + 1)] + shape_data = list(shape_data.astype(int)) + extrem_data.append(shape_data) + return extrem_data + + def create_csv_file(self): + headers = ["Param_name"] + if self.level == GradConst.LEVEL2: + headers.extend(self.get_dist_header()) + headers.extend(self.get_extrem_headers()) + output_path = f"{self.save_dir}/grad_summary_{self.current_step}.csv" + write_csv([headers], output_path) + self.cache_list.set_output_file(output_path) + self.cache_list.clear() + + def get_extrem_headers(self) -> List[str]: + return ["Max", "Min", "Norm", "Shape"] + + def get_dist_header(self) -> List[str]: + intervals = [] + for i, _ in enumerate(self.bounds): + if i == 0: + intervals.append(f"(-inf, {self.bounds[i]}]") + else: + intervals.append(f"({self.bounds[i-1]}, {self.bounds[i]}]") + intervals.extend([f"({self.bounds[-1]}, inf)", "=0"]) + return intervals + +csv_generator = CSVGenerator() diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_monitor.py new file mode 100644 index 000000000..f1e082688 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_monitor.py @@ -0,0 +1,27 @@ +from msprobe.mindspore.grad_probe.global_context import grad_context +from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator +from msprobe.mindspore.grad_probe.hook import hook_optimizer +from msprobe.core.grad_probe.constant import GradConst + + +class GradientMonitor: + + def __init__(self, common_dict, task_config): + config = {} + config[GradConst.OUTPUT_PATH] = common_dict.dump_path + config[GradConst.STEP] = common_dict.step + config[GradConst.RANK] = common_dict.rank + config[GradConst.PARAM_LIST] = task_config.param_list + config[GradConst.LEVEL] = task_config.grad_level + config[GradConst.BOUNDS] = task_config.bounds + self.config = config + grad_context.init_context(self.config) + + @staticmethod + def monitor(opt): + csv_generator.init(grad_context) + hook_optimizer(opt) + + @staticmethod + def stop(): + csv_generator.stop() diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_stat_csv.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_stat_csv.py new file mode 100644 index 000000000..1c2b0ee3b --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_stat_csv.py @@ -0,0 +1,132 @@ +from abc import ABC, abstractmethod +import hashlib + +import mindspore +from mindspore import ops, Tensor +from msprobe.core.grad_probe.constant import GradConst + + +class CsvInput: + def __init__(self, param_name, grad, bounds): + self.param_name = param_name + self.grad = grad + self.bounds = bounds + +class GradStatCsv: + csv = {} + + @staticmethod + def get_csv_header(level, csv_input): + header = ["param_name"] + for key in level["header"]: + header.extend(GradStatCsv.csv[key].generate_csv_header(csv_input)) + return header + + @staticmethod + def get_csv_line(level, csv_input): + line = [csv_input.param_name] + for key in level["header"]: + line.extend(GradStatCsv.csv[key].generate_csv_content(csv_input)) + return line + + +def register_csv_item(key, cls=None): + if cls is None: + # 无参数时，返回装饰器函数 + return lambda cls: register_csv_item(key, cls) + GradStatCsv.csv[key] = cls + return cls + + +class CsvItem(ABC): + @staticmethod + @abstractmethod + def generate_csv_header(csv_input): + pass + + @staticmethod + @abstractmethod + def generate_csv_content(csv_input): + pass + + +@register_csv_item(GradConst.MD5) +class CsvMd5(CsvItem): + def generate_csv_header(csv_input): + return ["MD5"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + tensor_bytes = grad.float().numpy().tobytes() + md5_hash = hashlib.md5(tensor_bytes) + return [md5_hash.hexdigest()] + + +@register_csv_item(GradConst.DISTRIBUTION) +class CsvDistribution(CsvItem): + def generate_csv_header(csv_input): + bounds = csv_input.bounds + intervals = [] + if bounds: + intervals.append(f"(-inf, {bounds[0]}]") + for i in range(1, len(bounds)): + intervals.append(f"({bounds[i-1]}, {bounds[i]}]") + if intervals: + intervals.append(f"({bounds[-1]}, inf)") + intervals.append("=0") + + return intervals + + def generate_csv_content(csv_input): + grad = csv_input.grad + bounds = csv_input.bounds + if grad.dtype == mindspore.bfloat16: + grad = grad.to(mindspore.float32) + element_num = grad.numel() + grad_equal_0_num = (grad == 0).sum().item() + bucketsize_result = ops.bucketize(grad.float(), bounds) + bucketsize_result = bucketsize_result.astype(mindspore.int8) + interval_nums = [(bucketsize_result == i).sum().item() for i in range(len(bounds) + 1)] + interval_nums.append(grad_equal_0_num) + return_list = [x / element_num if element_num != 0 else 0 for x in interval_nums] + return return_list + + +@register_csv_item(GradConst.MAX) +class CsvMax(CsvItem): + def generate_csv_header(csv_input): + return ["max"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.amax(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.MIN) +class CsvMin(CsvItem): + def generate_csv_header(csv_input): + return ["min"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.amin(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.NORM) +class CsvNorm(CsvItem): + def generate_csv_header(csv_input): + return ["norm"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.norm(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.SHAPE) +class CsvShape(CsvItem): + def generate_csv_header(csv_input): + return ["shape"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [list(grad.shape)] \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py new file mode 100644 index 000000000..243fb33de --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py @@ -0,0 +1,92 @@ + +import os + +import mindspore +import mindspore as ms +from mindspore.common.api import jit +from mindspore.nn.optim.optimizer import Optimizer +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer + +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.log import logger + +from msprobe.core.common.utils import write_csv, remove_path +from msprobe.mindspore.grad_probe.global_context import grad_context +from msprobe.mindspore.grad_probe.grad_analyzer import grad_dump, get_rank_id +from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator +from msprobe.mindspore.grad_probe.grad_stat_csv import GradStatCsv, CsvInput +from msprobe.mindspore.grad_probe.utils import save_grad_direction, get_adapted_level + +class HookInput: + + ''' + HookInput is a class wrapping all the variables used for hooking optimizer + ''' + + def __init__(self, opt) -> None: + self.func = opt.construct + self.g_names = [param.name for param in opt._parameters] + self.param_list = grad_context.get_context(GradConst.PARAM_LIST) + self.rank_id = get_rank_id() + output_path = grad_context.get_context(GradConst.OUTPUT_PATH) + self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", "Dump") + self.save_dir = os.path.join(output_path, f"rank{self.rank_id}") + self.step_finish_flag = os.path.join(self.dump_dir, GradConst.STEP_FINISH) + if os.path.exists(self.save_dir): + logger.warning(f"Delete existing path {self.save_dir}.") + remove_path(self.save_dir) + self.level = grad_context.get_context(GradConst.LEVEL) + self.bounds = grad_context.get_context(GradConst.BOUNDS) + self.mode = mindspore.get_context("mode") + +def hook_graph_mode_optimizer(opt, hook_input): + @jit + def new_construct(self, gradients): + for index, grad_value in enumerate(gradients): + if hook_input.param_list and hook_input.g_names[index] not in hook_input.param_list: + continue + grad_dump(hook_input.dump_dir, hook_input.g_names[index], self.dump_step, + grad_value, hook_input.level, hook_input.bounds) + ms.ops.TensorDump()(hook_input.step_finish_flag, self.dump_step) + self.assignadd(self.dump_step, self.global_step_increase_tensor) + out = hook_input.func(gradients) + return out + + opt.dump_step = Parameter(initializer(0, [1], ms.int32), name="dump_step") + opt.construct = new_construct.__get__(opt, type(opt)) + csv_generator.start() + +def hook_pynative_optimizer(opt, hook_input): + level_adapted = get_adapted_level(hook_input.level) + + def hook_fn(cell, input): + gradients, = input + cur_step = grad_context.get_context(GradConst.CURRENT_STEP) + if grad_context.step_need_dump(cur_step) and grad_context.rank_need_dump(hook_input.rank_id): + output_lines = [] + for index, grad_value in enumerate(gradients): + param_name = hook_input.g_names[index] + if hook_input.param_list and param_name not in hook_input.param_list: + continue + csv_input = CsvInput(param_name, grad_value, hook_input.bounds) + grad_info = GradStatCsv.get_csv_line(level_adapted, csv_input) + output_lines.append(grad_info) + if level_adapted["have_grad_direction"]: + save_grad_direction(param_name, grad_value, os.path.join(hook_input.save_dir, f'step{cur_step}')) + output_csv_path = os.path.join(hook_input.save_dir, f"grad_summary_{cur_step}.csv") + dummy_csv_input = CsvInput(None, None, hook_input.bounds) + output_lines.insert(0, GradStatCsv.get_csv_header(level_adapted, dummy_csv_input)) + write_csv(output_lines, output_csv_path) + grad_context.update_step() + + opt.register_forward_pre_hook(hook_fn) + + +def hook_optimizer(opt: Optimizer): + hook_input = HookInput(opt) + + if hook_input.mode == mindspore.GRAPH_MODE: + hook_graph_mode_optimizer(opt, hook_input) + else: + hook_pynative_optimizer(opt, hook_input) diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py new file mode 100644 index 000000000..db0a36a02 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py @@ -0,0 +1,29 @@ +import os + +import numpy as np +import mindspore +from msprobe.core.grad_probe.constant import GradConst, level_adp +from msprobe.core.grad_probe.utils import check_param +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.utils import check_path_before_create, change_mode, check_file_or_directory_path, save_npy + + +def save_grad_direction(param_name, grad, save_path): + if not os.path.exists(save_path): + create_directory(save_path) + check_file_or_directory_path(save_path, isdir=True) + check_param(param_name) + save_filepath = os.path.join(save_path, f"{param_name}.npy") + check_path_before_create(save_filepath) + + if grad.dtype == mindspore.bfloat16: + grad = grad.to(mindspore.float32) + grad_direction_tensor = grad > 0 + grad_direction_ndarray = grad_direction_tensor.numpy() + + save_npy(grad_direction_ndarray, save_filepath) + + +def get_adapted_level(level: str): + level_adapted = level_adp.get(level) + return level_adapted \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index 05beeea32..0e7ce1529 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -73,13 +73,20 @@ class FreeBenchmarkConfig(BaseConfig): if self.if_preheat or self.preheat_step or self.max_sample: logger.warning("'if_preheat', 'preheat_step' and 'max_sample' settings " "are not supported for mindspore free benchmark task.") +class GradProbeConfig(BaseConfig): + def __init__(self, json_config): + super().__init__(json_config) + self.grad_level = json_config.get("grad_level") + self.param_list = json_config.get("param_list") + self.bounds = json_config.get("bounds") TaskDict = { Const.TENSOR: TensorConfig, Const.STATISTICS: StatisticsConfig, Const.OVERFLOW_CHECK: OverflowCheckConfig, - Const.FREE_BENCHMARK: FreeBenchmarkConfig + Const.FREE_BENCHMARK: FreeBenchmarkConfig, + Const.GRAD_PROBE: GradProbeConfig, } diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py index 012d42faf..8433f0af6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py @@ -36,7 +36,7 @@ class PrecisionDebugger: common_config, task_config = parse_json_config(config_path, task) self.task = common_config.task if self.task == Const.GRAD_PROBE: - GradientMonitor(task_config, model) + self.gm = GradientMonitor(common_config, task_config) return if step: common_config.step = step @@ -102,6 +102,14 @@ class PrecisionDebugger: raise Exception("PrecisionDebugger instance is not created.") cls._instance.service.step() + @classmethod + def monitor(cls, model): + if not cls._instance: + raise Exception("PrecisionDebugger instance is not created.") + if cls._instance.task != Const.GRAD_PROBE: + return + cls._instance.gm.monitor(model) + def iter_tracer(func): def func_wrapper(*args, **kwargs): diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index edd28635d..4bed1cc04 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -5,51 +5,34 @@ import torch from torch.optim.optimizer import register_optimizer_step_pre_hook from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target -from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.grad_probe.constant import GradConst, level_adp from msprobe.core.common.file_check import create_directory from msprobe.core.common.log import logger -from msprobe.core.common.utils import remove_path, write_csv +from msprobe.core.common.utils import remove_path, write_csv, save_npy from msprobe.pytorch.common.utils import get_rank_id, print_rank_0, save_pt class GradientMonitor: - level_adp = { - "L0": { - "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], - "have_grad_direction": False - }, - "L1": { - "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], - "have_grad_direction": True - }, - "L2": { - "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], - "have_grad_direction": True - }, - } - def __init__(self, config, model): - self._config = config._config - self._model = model - level = self._config.get("level") - if level not in GradientMonitor.level_adp: - raise Exception(f"level is valid, not in {GradientMonitor.level_adp.keys()}") - self._level_adp = GradientMonitor.level_adp[level] - self._param_list = self._config.get('param_list') - self._target_ranks = self._config.get("rank") + def __init__(self, common_config, task_config): + level = task_config.grad_level + if level not in level_adp: + raise Exception(f"level is valid, not in {level_adp.keys()}") + self._level_adp = level_adp[level] + self._param_list = task_config.param_list + self._target_ranks = common_config.rank logger.info(f"target rank {self._target_ranks}") - self._target_step = self._config.get("step") + self._target_step = common_config.step logger.info(f"target step {self._target_step}") - self._bounds = self._config.get("bounds") + self._bounds = task_config.bounds check_numeral_list_ascend(self._bounds) - self._output_path = self._config.get("output_path") + self._output_path = common_config.dump_path if not os.path.exists(self._output_path): create_directory(self._output_path) else: logger.warning(f"the file in {self._output_path} will be recoverd") self._step = -1 self._param2name = defaultdict(str) - self._monitor() @property def output_path(self): @@ -61,12 +44,12 @@ class GradientMonitor: create_directory(save_path) param_grad = grad.clone().detach() is_positive = param_grad > 0 - save_filepath = os.path.join(save_path, f"{param_name}.pt") - save_pt(is_positive, save_filepath) + save_filepath = os.path.join(save_path, f"{param_name}.npy") + save_npy(is_positive.numpy(), save_filepath) - def _monitor(self): + def monitor(self, model): print_rank_0("> parameter names:") - for name, param in self._model.named_parameters(): + for name, param in model.named_parameters(): self._param2name[param] = name print_rank_0(f"\t{name}") setattr(self, "_rank", get_rank_id()) diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py index ae01b75ee..757a1aebf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py @@ -63,13 +63,15 @@ class CSV_distribution(CsvItem): def generate_csv_header(csv_header_input): bounds = csv_header_input.bounds intervals = [] - for i, _ in enumerate(bounds): - if i == 0: - intervals.append(f"(-inf, {bounds[i]}]") - else: + if bounds: + intervals.append(f"(-inf, {bounds[0]}]") + for i in range(1, len(bounds)): intervals.append(f"({bounds[i-1]}, {bounds[i]}]") - intervals.extend([f"({bounds[-1]}, inf)", "=0"]) - return intervals + if intervals: + intervals.append(f"({bounds[-1]}, inf)") + intervals.append("=0") + + return intervals def generate_csv_content(csv_content_input): grad = csv_content_input.grad diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index daba5476c..2db6980bb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -96,7 +96,9 @@ class RunUTConfig(BaseConfig): class GradToolConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) - self._config = json_config + self.grad_level = json_config.get("grad_level") + self.param_list = json_config.get("param_list") + self.bounds = json_config.get("bounds") def parse_task_config(task, json_config): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py index bd569f5a2..f39d3f091 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py @@ -4,6 +4,7 @@ import os import torch from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor +from msprobe.core.grad_probe.constant import level_adp grad_tensor = torch.tensor([[-2, 2], [0.2, 0.3]]) @@ -11,27 +12,27 @@ grad_tensor = torch.tensor([[-2, 2], [0.2, 0.3]]) class TestGradCSV(unittest.TestCase): def test_level_L0_header(self): self.assertEqual(['param_name', 'MD5', 'max', 'min', 'norm', 'shape'], - GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L0"], [-1, 0, 1])) + GradStatCsv.generate_csv_header(level_adp["L0"], [-1, 0, 1])) def test_level_L1_header(self): self.assertEqual(['param_name', 'max', 'min', 'norm', 'shape'], - GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L1"], [-1, 0, 1])) + GradStatCsv.generate_csv_header(level_adp["L1"], [-1, 0, 1])) def test_level_L2_header(self): self.assertEqual(['param_name', '(-inf, -1]', '(-1, 0]', '(0, 1]', '(1, inf)', '=0', 'max', 'min', 'norm', 'shape'], - GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L2"], [-1, 0, 1])) + GradStatCsv.generate_csv_header(level_adp["L2"], [-1, 0, 1])) def test_level_L0_content(self): - generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L0"], grad_tensor, [-1, 0, 1]) + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", level_adp["L0"], grad_tensor, [-1, 0, 1]) self.assertEqual(['model.conv2d', '678a6c7d9d9716682b56fda097d0936c', 2.0, -2.0, 2.851315498352051, [2, 2]], generated_csv_line) def test_level_L1_content(self): - generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L1"], grad_tensor, [-1, 0, 1]) + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", level_adp["L1"], grad_tensor, [-1, 0, 1]) self.assertEqual(['model.conv2d', 2.0, -2.0, 2.851315498352051, [2, 2]], generated_csv_line) def test_level_L2_content(self): - generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L2"], grad_tensor, [-1, 0, 1]) + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", level_adp["L2"], grad_tensor, [-1, 0, 1]) self.assertEqual(['model.conv2d', 0.25, 0.0, 0.5, 0.25, 0.0, 2.0, -2.0, 2.851315498352051, [2, 2]], generated_csv_line) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py index d79cca502..607addd69 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py @@ -10,15 +10,24 @@ from msprobe.core.grad_probe.grad_compare import GradComparator from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor from msprobe.pytorch.pt_config import GradToolConfig +class config: + def __init__(self, config_dict): + for key, value in config_dict.items(): + setattr(self, key, value) -config_dict = { - "level": "L1", - "param_list": "", +common_config_dict = { "rank": [], "step": [], - "bounds": [-1,0,1], - "output_path": "./grad_output" + "dump_path": "./grad_output" +} +common_config = config(common_config_dict) + +task_config_dict = { + "grad_level": "L1", + "param_list": "", + "bounds": [-1,0,1] } +task_config = config(task_config_dict) def seed_all(seed=1234, mode=False): random.seed(seed) @@ -53,7 +62,8 @@ def get_grad_monitor(): nn.init.constant_(test_module.linear.bias, 1.0) optimizer = torch.optim.SGD(test_module.parameters(), lr=1e-2) - gm = GradientMonitor(GradToolConfig(config_dict), test_module) + gm = GradientMonitor(common_config, task_config) + gm.monitor(test_module) for input_data, label in zip(inputs, labels): output = test_module(input_data) -- Gitee From 7f7b1f6ee65a1d9ef7e029af39198bd3041d728d Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 7 Aug 2024 16:32:49 +0800 Subject: [PATCH 173/791] =?UTF-8?q?=E9=97=A8=E7=A6=81=E8=A6=81=E6=B1=82?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/compare/Multiprocessing_compute.py | 2 +- .../msprobe/core/compare/acc_compare.py | 6 ++-- .../msprobe/core/compare/highlight.py | 3 +- .../msprobe/core/compare/match.py | 2 -- .../msprobe/mindspore/compare/ms_compare.py | 34 ++++++++----------- .../msprobe/pytorch/__init__.py | 1 + .../msprobe/pytorch/compare/compare_cli.py | 4 +-- .../msprobe/pytorch/compare/pt_compare.py | 31 +++++++---------- 8 files changed, 33 insertions(+), 50 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py index 20e3c1d0c..da63005e5 100644 --- a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py @@ -1,7 +1,7 @@ import multiprocessing -import pandas as pd from dataclasses import dataclass +import pandas as pd from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException from msprobe.core.common.const import CompareConst diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 1d11f120b..084f8c9e9 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -6,10 +6,10 @@ from msprobe.core.common.exceptions import FileCheckException class Comparator: + def __init__(self): pass - def match_op(self,npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): if check_op(npu_queue[-1], b_op, fuzzy_match): @@ -21,7 +21,6 @@ class Comparator: return n_index, len(bench_queue) - 1 return -1, -1 - def compare_by_op(self,op_name, op_name_mapping_dict, input_parma): npu_bench_name_list = op_name_mapping_dict[op_name] data_name = npu_bench_name_list[1] @@ -55,7 +54,6 @@ class Comparator: err_msg += " Fuzzy matching data, the comparison accuracy may be affected." result_list.append(err_msg) return result_list - - + testComparator= Comparator() diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 21cab0839..802376347 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -1,10 +1,9 @@ import math import abc -import numpy as np from collections import namedtuple +import numpy as np import openpyxl from openpyxl.styles import PatternFill -from collections import namedtuple from msprobe.core.common.utils import get_header_index from msprobe.core.common.const import CompareConst from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/core/compare/match.py b/debug/accuracy_tools/msprobe/core/compare/match.py index acab42585..2a46105bd 100644 --- a/debug/accuracy_tools/msprobe/core/compare/match.py +++ b/debug/accuracy_tools/msprobe/core/compare/match.py @@ -10,7 +10,6 @@ class AtenIrMapping(): yaml_path = os.path.join(cur_path, "mapping.yaml") with FileOpen(yaml_path, 'r') as f: self.aten_mapping = yaml.safe_load(f) - def match(self, op1, op2): if "Aten" in op1 and "Aten" not in op2: @@ -18,7 +17,6 @@ class AtenIrMapping(): else: return self.match_op(op2, op1) - def match_op(self, aten_op, torch_op): try: aten_op_raw_name_overload = '_'.join(aten_op.split("_")[1:-3]) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index b8f29745a..23764a49d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -19,10 +19,10 @@ from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException class MSComparator (Comparator): + def __init__(self): super().__init__() - def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): cos_result = [] max_err_result = [] @@ -60,7 +60,6 @@ class MSComparator (Comparator): return _save_cmp_result(idx, cr, result_df, lock) - def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): op_data = json_data['data'][op_name] op_parsed_list = read_op(op_data, op_name) @@ -71,8 +70,7 @@ class MSComparator (Comparator): merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) return merge_list - - + def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles npu_json_data = json.load(npu_json_handle) @@ -135,8 +133,7 @@ class MSComparator (Comparator): for npu_data in npu_ops_queue: get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) - return result_df - + return result_df def make_result_table(self,result,md5_compare,summary_compare,stack_mode): header = [] @@ -163,17 +160,7 @@ class MSComparator (Comparator): for row in result: del row[-1] result_df = pd.DataFrame(result, columns=header) - return result_df - - - def _do_multi_process(self,input_parma, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - + return result_df def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) @@ -184,8 +171,7 @@ class MSComparator (Comparator): if data_value.dtype == np.float16: data_value=data_value.astype(np.float32) - return data_value - + return data_value def compare_core(self,input_parma, output_path, **kwargs): """ @@ -232,7 +218,15 @@ class MSComparator (Comparator): advisor = Advisor(result_df, output_path) advisor.analysis() - + def _do_multi_process(self,input_parma, result_df): + try: + result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) diff --git a/debug/accuracy_tools/msprobe/pytorch/__init__.py b/debug/accuracy_tools/msprobe/pytorch/__init__.py index c14d9701a..c4e426772 100644 --- a/debug/accuracy_tools/msprobe/pytorch/__init__.py +++ b/debug/accuracy_tools/msprobe/pytorch/__init__.py @@ -1,3 +1,4 @@ from .debugger.precision_debugger import PrecisionDebugger from .common.utils import seed_all from .compare.distributed_compare import compare_distributed +from .compare.pt_compare import compare \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py index 155609f58..b344d4efb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py @@ -3,7 +3,7 @@ from msprobe.core.common.file_check import FileOpen, check_file_type from msprobe.core.common.const import FileCheckConst from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger -from msprobe.pytorch.compare.pt_compare import pt_compare +from msprobe.pytorch.compare.pt_compare import compare from msprobe.pytorch.compare.distributed_compare import compare_distributed @@ -14,7 +14,7 @@ def compare_cli(args): bench_path = input_param.get("bench_path", None) if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: - pt_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, + compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index fe2d4fd76..43f628dd0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -23,7 +23,6 @@ class PTComparator (Comparator): def __init__(self): super().__init__() - def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): cos_result = [] max_err_result = [] @@ -61,7 +60,6 @@ class PTComparator (Comparator): return _save_cmp_result(idx, cr, result_df, lock) - def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): op_data = json_data['data'][op_name] op_parsed_list = read_op(op_data, op_name) @@ -71,8 +69,7 @@ class PTComparator (Comparator): op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) - return merge_list - + return merge_list def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles @@ -139,7 +136,6 @@ class PTComparator (Comparator): result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df - def make_result_table(self,result,md5_compare,summary_compare,stack_mode): header = [] if md5_compare: @@ -167,7 +163,6 @@ class PTComparator (Comparator): result_df = pd.DataFrame(result, columns=header) return result_df - def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -177,17 +172,7 @@ class PTComparator (Comparator): if data_value.dtype == torch.bfloat16: data_value = data_value.to(torch.float32) data_value = data_value.numpy() - return data_value - - - def _do_multi_process(self,input_parma, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - + return data_value def compare_core(self,input_parma, output_path, **kwargs): """ @@ -235,8 +220,16 @@ class PTComparator (Comparator): advisor = Advisor(result_df, output_path) advisor.analysis() - -def pt_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): + def _do_multi_process(self,input_parma, result_df): + try: + result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + +def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) -- Gitee From f523d2ac2450bb0fd4095075c6da450fd3f7ecee Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 7 Aug 2024 16:47:43 +0800 Subject: [PATCH 174/791] msprobe add online run_ut --- .../api_accuracy_checker/compare/compare.py | 39 ++++-- .../api_accuracy_checker/run_ut/run_ut.py | 132 ++++++++++++++++-- .../pytorch/debugger/debugger_config.py | 8 +- .../msprobe/pytorch/pt_config.py | 4 + .../accuracy_tools/msprobe/pytorch/service.py | 54 ++++++- 5 files changed, 204 insertions(+), 33 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index ee4958828..20f04b0cd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -33,16 +33,30 @@ class Comparator: COLUMN_BACKWARD_SUCCESS = "Backward Test Success" COLUMN_STACK_INFO = "Traceback callstack info" - def __init__(self, result_csv_path, details_csv_path, is_continue_run_ut, stack_info_json_path=None): - self.save_path = result_csv_path - self.detail_save_path = details_csv_path - if not is_continue_run_ut and not os.path.exists(self.save_path) and not os.path.exists(self.detail_save_path): + def __init__(self, result_csv_path, details_csv_path, is_continue_run_ut, stack_info_json_path=None, config=None): + self.save_path_str = result_csv_path + self.detail_save_path_str = details_csv_path + self.save_path_list = [result_csv_path] + self.detail_save_path_list = [details_csv_path] + + if config and config.online_config.is_online: + self.save_path_str = result_csv_path.replace(".csv", "_rank{}.csv") + self.detail_save_path_str = details_csv_path.replace(".csv", "_rank{}.csv") + self.save_path_list = [self.save_path_str.format(rank) for rank in config.online_config.rank_list] + self.detail_save_path_list = \ + [self.detail_save_path_str.format(rank) for rank in config.online_config.rank_list] + + if not is_continue_run_ut: self.write_csv_title() if stack_info_json_path: self.stack_info = get_json_contents(stack_info_json_path) else: self.stack_info = None + @staticmethod + def get_path_from_rank(rank, path_list, path_pattern): + return path_list[-1] if len(path_list) == 1 else path_pattern.format(rank) + @staticmethod def print_pretest_result(): logger.info("Successfully completed run_ut/multi_run_ut.") @@ -86,10 +100,11 @@ class Comparator: def write_csv_title(self): summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, self.COLUMN_BACKWARD_SUCCESS, "Message"]] - if not os.path.exists(self.save_path): - write_csv(summary_test_rows, self.save_path) - if not os.path.exists(self.detail_save_path): - write_csv(DETAIL_TEST_ROWS, self.detail_save_path) + for save_path, detail_save_path in zip(self.save_path_list, self.detail_save_path_list): + if not os.path.exists(save_path): + write_csv(summary_test_rows, save_path) + if not os.path.exists(detail_save_path): + write_csv(DETAIL_TEST_ROWS, detail_save_path) def write_summary_csv(self, test_result): test_rows = [] @@ -104,7 +119,8 @@ class Comparator: stack_info = "\n".join(self.stack_info[name]) df_row.append(stack_info) test_rows.append(df_row) - write_csv(test_rows, self.save_path) + save_path = self.get_path_from_rank(test_result[-1], self.save_path_list, self.save_path_str) + write_csv(test_rows, save_path) def write_detail_csv(self, test_result): test_rows = [] @@ -125,7 +141,10 @@ class Comparator: if isinstance(item, float) else item for item in test_subject] test_rows.append([subject] + list(test_subject)) - write_csv(test_rows, self.detail_save_path) + detail_save_path = self.get_path_from_rank(test_result[-1], + self.detail_save_path_list, + self.detail_save_path_str) + write_csv(test_rows, detail_save_path) def record_results(self, args): self.write_summary_csv(args) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index bca971116..04ad039b2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -36,14 +36,20 @@ from msprobe.core.common.file_check import FileOpen, FileChecker, \ from msprobe.pytorch.common.log import logger from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.const import Const, FileCheckConst, CompareConst +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTL, ATTLConfig, ApiData, move2device_exec +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.device_dispatch import ConsumerDispatcher + current_time = time.strftime("%Y%m%d%H%M%S") UT_ERROR_DATA_DIR = 'ut_error_data' + current_time RESULT_FILE_NAME = "accuracy_checking_result_" + current_time + ".csv" DETAILS_FILE_NAME = "accuracy_checking_details_" + current_time + ".csv" RunUTConfig = namedtuple('RunUTConfig', ['forward_content', 'backward_content', 'result_csv_path', 'details_csv_path', - 'save_error_data', 'is_continue_run_ut', 'real_data_path', 'white_list', - 'black_list', 'error_data_path']) + 'save_error_data', 'is_continue_run_ut', 'real_data_path', 'white_list', + 'black_list', 'error_data_path', 'online_config']) + +OnlineConfig = namedtuple('OnlineConfig', ['is_online', 'nfs_path', 'host', 'port', 'rank_list']) + not_backward_list = ['repeat_interleave'] not_detach_set = {'resize_', 'resize_as_', 'set_', 'transpose_', 't_', 'squeeze_', 'unsqueeze_'} not_raise_dtype_set = {'type_as'} @@ -140,7 +146,7 @@ def generate_cpu_params(input_args, input_kwargs, need_backward, api_name): elif isinstance(arg_in, torch.Tensor): if need_backward and arg_in.requires_grad: arg_in = deal_detach(raise_bench_data_dtype( - api_name, arg_in.clone(), raise_dtype=raise_dtype), to_detach).requires_grad_() + api_name, arg_in.clone(), raise_dtype=raise_dtype), to_detach).requires_grad_() temp_arg_in = arg_in * 1 arg_in = temp_arg_in.type_as(arg_in) arg_in.retain_grad() @@ -187,11 +193,25 @@ def run_ut(config): logger.info(f"UT task details will be saved in {config.details_csv_path}") if config.save_error_data: logger.info(f"UT task error_datas will be saved in {config.error_data_path}") - compare = Comparator(config.result_csv_path, config.details_csv_path, config.is_continue_run_ut) - with FileOpen(config.result_csv_path, 'r') as file: - csv_reader = csv.reader(file) - next(csv_reader) - api_name_set = {row[0] for row in csv_reader} + compare = Comparator(config.result_csv_path, config.details_csv_path, config.is_continue_run_ut, config=config) + + if config.online_config.is_online: + run_api_online(config, compare) + else: + with FileOpen(config.result_csv_path, 'r') as file: + csv_reader = csv.reader(file) + next(csv_reader) + api_name_set = {row[0] for row in csv_reader} + run_api_offline(config, compare, api_name_set) + for result_csv_path, details_csv_path in zip(compare.save_path_list, compare.detail_save_path_list): + change_mode(result_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) + change_mode(details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) + logger.info()(f"UT task result csv is saved in {result_csv_path}") + logger.info()(f"UT task details csv is saved in {details_csv_path}") + compare.print_pretest_result() + + +def run_api_offline(config, compare, api_name_set): for _, (api_full_name, api_info_dict) in enumerate(tqdm(config.forward_content.items(), **tqdm_params)): if api_full_name in api_name_set: continue @@ -223,9 +243,55 @@ def run_ut(config): else: torch.npu.empty_cache() gc.collect() - change_mode(compare.save_path, FileCheckConst.DATA_FILE_AUTHORITY) - change_mode(compare.detail_save_path, FileCheckConst.DATA_FILE_AUTHORITY) - compare.print_pretest_result() + + +def run_api_online(config, compare): + attl = init_attl(config.online_config) + dispatcher = ConsumerDispatcher(compare=compare) + dispatcher.start(handle_func=run_torch_api_online, config=config) + + def tcp_communication_flow(): + while True: + api_data = attl.recv() + if api_data == 'STOP_': + continue + if api_data == 'KILL_': + time.sleep(1) + logger.info("==========接收到STOP信号==========") + dispatcher.stop() + attl.stop_serve() + time.sleep(1) + break + if not isinstance(api_data, ApiData): + continue + api_full_name = api_data.name + + if config.white_list: + [_, api_name, _] = api_full_name.split(Const.SEP) + if api_name not in set(config.white_list): + continue + dispatcher.update_consume_queue(api_data) + + def shared_storage_communication_flow(): + flag_num = -1 + while True: + api_data = attl.download() + if api_data == "start": + if flag_num == -1: + flag_num += 1 + flag_num += 1 + if api_data == "end": + flag_num -= 1 + if flag_num == 0: + dispatcher.stop() + break + if isinstance(api_data, ApiData): + dispatcher.update_consume_queue(api_data) + + if config.nfs_path: + shared_storage_communication_flow() + else: + tcp_communication_flow() def is_unsupported_api(api_name): @@ -294,6 +360,20 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict return UtDataInfo(bench_grad_out, device_grad_out, device_out, out, bench_grad, in_fwd_data_list, backward_message) +def run_torch_api_online(api_full_name, api_data, backward_content): + in_fwd_data_list = [] + [api_type, api_name, _] = api_full_name.split(Const.SEP) + args, kwargs, out = api_data.args, api_data.kwargs, api_data.result + in_fwd_data_list.append(args) + in_fwd_data_list.append(kwargs) + if kwargs.get("device"): + del kwargs["device"] + + device_out = exec_api(api_type, api_name, args, kwargs) + device_out = move2device_exec(device_out, "cpu") + return UtDataInfo(None, None, out, device_out, None, in_fwd_data_list, None, rank=api_data.rank) + + def get_api_info(api_info_dict, api_name, real_data_path): convert_type, api_info_dict = api_info_preprocess(api_name, api_info_dict) need_grad = True @@ -357,11 +437,20 @@ def get_validated_details_csv_path(validated_result_csv_path): return validated_details_csv_path +def init_attl(config): + """config: OnlineConfig""" + attl = ATTL('gpu', ATTLConfig(is_benchmark_device=True, + connect_ip=config.host, + connect_port=config.port, + nfs_path=config.nfs_path)) + return attl + + def _run_ut_parser(parser): parser.add_argument("-api_info", "--api_info_file", dest="api_info_file", default="", type=str, - help=" The api param tool result file: generate from api param tool, " + help=" The api param tool result file: generate from api param tool, " "a json file.", - required=True) + required=False) parser.add_argument("-o", "--out_path", dest="out_path", default="", type=str, help=" The ut task result out path.", required=False) @@ -478,24 +567,37 @@ def run_ut_command(args): white_list = msCheckerConfig.white_list black_list = msCheckerConfig.black_list error_data_path = msCheckerConfig.error_data_path + is_online = msCheckerConfig.is_online + nfs_path = msCheckerConfig.nfs_path + host = msCheckerConfig.host + port = msCheckerConfig.port + rank_list = msCheckerConfig.rank_list if args.config_path: _, task_config = parse_json_config(args.config_path, Const.RUN_UT) white_list = task_config.white_list black_list = task_config.black_list error_data_path = task_config.error_data_path + is_online = task_config.is_online + nfs_path = task_config.nfs_path + host = task_config.host + port = task_config.port + rank_list = task_config.rank_list + if save_error_data: if args.result_csv_path: time_info = result_csv_path.split('.')[0].split('_')[-1] global UT_ERROR_DATA_DIR UT_ERROR_DATA_DIR = 'ut_error_data' + time_info error_data_path = initialize_save_error_data(error_data_path) + online_config = OnlineConfig(is_online, nfs_path, host, port, rank_list) run_ut_config = RunUTConfig(forward_content, backward_content, result_csv_path, details_csv_path, save_error_data, - args.result_csv_path, real_data_path, set(white_list), set(black_list), error_data_path) + args.result_csv_path, real_data_path, set(white_list), set(black_list), error_data_path, + online_config) run_ut(run_ut_config) class UtDataInfo: - def __init__(self, bench_grad, device_grad, device_output, bench_output, grad_in, in_fwd_data_list, + def __init__(self, bench_grad, device_grad, device_output, bench_output, grad_in, in_fwd_data_list, backward_message, rank=0): self.bench_grad = bench_grad self.device_grad = device_grad diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index cfc588e1e..04303c4f2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -35,7 +35,13 @@ class DebuggerConfig: "preheat_step": task_config.preheat_step if task_config.preheat_step else 15, "max_sample": task_config.max_sample if task_config.max_sample else 20, } - + + # dump api tensor and collaborate with online run_ut + self.online_run_ut = task_config.online_run_ut if task_config.online_run_ut else False + self.nfs_path = task_config.nfs_path if task_config.nfs_path else "" + self.host = task_config.host if task_config.host else "" + self.port = task_config.port if task_config.port else -1 + self.check() if self.step: self.step.sort() diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 8fbe5dea0..6bcd1a05e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -10,6 +10,10 @@ from msprobe.pytorch.hook_module.utils import WrapFunctionalOps, WrapTensorOps, class TensorConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) + self.online_run_ut = json_config.get("online_run_ut", False) + self.nfs_path = json_config.get("nfs_path", "") + self.host = json_config.get("host", "") + self.port = json_config.get("port", -1) self.check_config() self._check_file_format() diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index daeda8898..d74a9dc25 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -1,5 +1,6 @@ import functools import os +import time from pathlib import Path from msprobe.pytorch.common.log import logger @@ -13,6 +14,7 @@ from msprobe.pytorch.common.utils import get_rank_if_initialized from msprobe.pytorch.module_processer import ModuleProcesser from msprobe.pytorch.hook_module import remove_dropout from msprobe.pytorch.hook_module.api_registry import api_register +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTLConfig, ATTL, ApiData class Service: @@ -24,8 +26,20 @@ class Service: self.switch = False self.current_iter = 0 self.first_start = True - self.current_rank = None + try: + self.current_rank = get_rank_if_initialized() + except DistributedNotInitializedError: + self.current_rank = None self.dump_iter_dir = None + if self.config.online_run_ut: + attl_config = ATTLConfig(is_benchmark_device=False, + connect_ip=self.config.host, + connect_port=self.config.port, + nfs_path=self.config.nfs_path) + need_dump = self.current_rank in self.config.rank + self.attl = ATTL('npu', attl_config, need_dump=need_dump) + if self.config.nfs_path: + self.attl.upload("start") @staticmethod def forward_backward_dump_end(): @@ -52,6 +66,12 @@ class Service: if not self.switch: return None + + if self.config.online_run_ut: + api_data = ApiData(api_or_module_name, args, kwargs, output, self.current_iter, self.current_rank) + self.attl_send(api_data) + return None + if self.data_collector: module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=output) self.data_collector.forward_data_collect(api_or_module_name, module, pid, module_input_output) @@ -66,6 +86,13 @@ class Service: if not self.switch: return + + if self.config.online_run_ut: + api_data = ApiData(api_or_module_name, grad_input, None, grad_output, self.current_iter, + self.current_rank) + self.attl_send(api_data) + return None + if self.data_collector: module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_input, grad_output=grad_output) self.data_collector.backward_data_collect(api_or_module_name, module, pid, module_input_output) @@ -85,16 +112,22 @@ class Service: def start(self, model, api_origin=False): self.model = model if self.config.step and self.current_iter > max(self.config.step): + # send end or step signal + if self.config.online_run_ut: + if self.config.nfs_path: + self.attl.upload("end") + elif self.attl.socket_manager is not None: + logger.debug(f"进程{os.getpid()} 已完成,准备发送STOP信号") + self.attl.socket_manager.send_stop_signal() + else: + # current rank not need dump, wait + while True: + time.sleep(2) self.stop() raise Exception("msprobe: exit after iteration {}".format(max(self.config.step))) if self.config.step and self.current_iter not in self.config.step: return if self.first_start: - try: - self.current_rank = get_rank_if_initialized() - except DistributedNotInitializedError: - self.current_rank = None - if self.config.rank and self.current_rank not in self.config.rank: return self.register_hook_new() @@ -171,4 +204,11 @@ class Service: api_register.api_modularity() if Const.STATISTICS == self.config.task or Const.TENSOR == self.config.task: - remove_dropout() \ No newline at end of file + remove_dropout() + + def attl_send(self, api_data): + logger.info(f"tools is dumping api: {api_data.name}, rank: {self.current_rank}") + if self.config.nfs_path: + self.attl.upload(api_data) + else: + self.attl.send(api_data) -- Gitee From 2706526e790dc56ff76f07f856d5614c6cc8f633 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Wed, 7 Aug 2024 17:12:18 +0800 Subject: [PATCH 175/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dtorch2.x=E4=B8=8B?= =?UTF-8?q?=E7=9A=84construct.json=E4=B8=AD=E7=9A=84=E5=8F=8D=E5=90=91?= =?UTF-8?q?=E6=98=A0=E5=B0=84=E5=85=B3=E7=B3=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/pytorch/module_processer.py | 17 ++++++++++++----- debug/accuracy_tools/msprobe/pytorch/service.py | 11 +++++++++-- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/module_processer.py index cd91eedc0..8303ea814 100644 --- a/debug/accuracy_tools/msprobe/pytorch/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/module_processer.py @@ -5,6 +5,7 @@ from torch.utils.hooks import BackwardHook from msprobe.core.common.const import Const from msprobe.core.data_dump.scope import ModuleRangeScope +torch_version_above_2 = torch.__version__.split('+')[0] > '2.0' class ModuleProcesser: @@ -123,9 +124,15 @@ class ModuleProcesser: if self.scope: self.scope.begin_module(full_name) - if Const.FORWARD in name_prefix and Const.START in start_or_stop: - return pre_hook - elif Const.BACKWARD in name_prefix: - return backward_hook + if torch_version_above_2: + if Const.START in start_or_stop: + return pre_hook + else: + return end_hook else: - return end_hook + if Const.FORWARD in name_prefix and Const.START in start_or_stop: + return pre_hook + elif Const.BACKWARD in name_prefix: + return backward_hook + else: + return end_hook diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 79abfdc9e..840f97b9a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -15,7 +15,7 @@ from msprobe.pytorch.hook_module import remove_dropout from msprobe.pytorch.hook_module.api_registry import api_register from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.module_processer import ModuleProcesser -torch_vsrsion_above_2 = torch.__version__.split('+')[0] > '2.0' +torch_version_above_2 = torch.__version__.split('+')[0] > '2.0' class Service: @@ -180,9 +180,11 @@ class Service: pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 \ = self.build_hook(BaseScope.Module_Type_Module, prefix) - if torch_vsrsion_above_2: + if torch_version_above_2: module.register_forward_hook(forward_hook, with_kwargs=True) else: + module.register_full_backward_hook( + self.module_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) module.register_forward_hook(forward_hook_torch_version_below_2) module.register_full_backward_hook(backward_hook) @@ -190,6 +192,11 @@ class Service: self.module_processor.node_hook(prefix + Const.FORWARD, Const.START)) module.register_forward_hook( self.module_processor.node_hook(prefix + Const.FORWARD, Const.STOP)) + if torch_version_above_2: + module.register_full_backward_pre_hook( + self.module_processor.node_hook(prefix + Const.BACKWARD, Const.START)) + module.register_full_backward_hook( + self.module_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) if self.config.level in ["mix", "L1", "L2"]: api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) -- Gitee From c7cc3ac5e8588bd574e221e0f2cad7ddc45beef1 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Mon, 5 Aug 2024 15:10:24 +0800 Subject: [PATCH 176/791] =?UTF-8?q?rdma=E9=80=9A=E4=BF=A1=E9=87=8D?= =?UTF-8?q?=E4=BC=A0=E6=A3=80=E6=B5=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Communication_retransmission_analyzer.py | 46 +++++ .../Communication_retransmission_checker.py | 128 +++++++++++++ profiler/advisor/common/analyzer_scopes.py | 15 ++ .../dataset/cluster/cluster_dataset.py | 33 ++++ .../dataset/cluster/hccl_collection.py | 64 +++++++ ...communication_retransmission_analysis.html | 40 +++++ profiler/advisor/interface/interface.py | 16 ++ profiler/advisor/rules/rdma_analysis.yaml | 9 + .../test_rdma_retransmission_advice.py | 170 ++++++++++++++++++ 9 files changed, 521 insertions(+) create mode 100644 profiler/advisor/analyzer/cluster/Communication_retransmission_analyzer.py create mode 100644 profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py create mode 100644 profiler/advisor/dataset/cluster/hccl_collection.py create mode 100644 profiler/advisor/display/html/templates/communication_retransmission_analysis.html create mode 100644 profiler/advisor/rules/rdma_analysis.yaml create mode 100644 profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py diff --git a/profiler/advisor/analyzer/cluster/Communication_retransmission_analyzer.py b/profiler/advisor/analyzer/cluster/Communication_retransmission_analyzer.py new file mode 100644 index 000000000..3683ef1b4 --- /dev/null +++ b/profiler/advisor/analyzer/cluster/Communication_retransmission_analyzer.py @@ -0,0 +1,46 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.cluster.Communication_retransmission_checker import CommunicationRetransmissionChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset + +logger = logging.getLogger() + + +class RDMARetransmissionAnalyzer(BaseAnalyzer): + dataset_cls_list = [ClusterCommunicationDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = ClusterCommunicationDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseAnalyzer.check_data((ClusterCommunicationDataset.get_key(),)) + def optimize(self, **kwargs): + add_render_list = kwargs.get("add_render_list", True) + rdma_checker = CommunicationRetransmissionChecker(**kwargs) + rdma_checker.check_retransmission(self.dataset) + if not rdma_checker.rdma_issues: + return self.result + rdma_checker.make_record(self.result) + self.html = rdma_checker.make_render(self.html_render, add_render_list) + return self.result diff --git a/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py b/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py new file mode 100644 index 000000000..cc0f688e8 --- /dev/null +++ b/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py @@ -0,0 +1,128 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from typing import Dict, List +from collections import defaultdict +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo + +logger = logging.getLogger() + + +class GroupStatistic: + def __init__(self, min_transmission_time): + self.retransmission_issue = False + self.abnormal_op_dict: Dict[str, List] = dict() + + def add_op(self, op_name: str, hccl_info: HcclInfo): + if self.abnormal_op_dict.get(op_name) is None: + self.abnormal_op_dict.setdefault(op_name, []) + self.abnormal_op_dict.get(op_name).append([hccl_info.group, op_name, hccl_info.step, hccl_info.rank, + hccl_info.get_rdma_transit_size(), + hccl_info.get_rdma_transmit_time(), hccl_info.get_rdma_bandwidth()]) + + +class CommunicationRetransmissionChecker: + def __init__(self, **kwargs): + self.rdma_issues = False + self.desc = "" + self.sdma_desc = "" + self.rdma_desc = "" + self.suggestions = [] + self.abnormal_group_count = 0 + self.abnormal_rdma_list = [] + self.step_id = kwargs.get("step") + self.stage = None + self.group_statistics = defaultdict(GroupStatistic) + self.headers = ["Communication group", "Op name", "Step id", "Rank id", "RDMA transmit size(MB)", + "RDMA transmit time(ms)", "RDMA bandwidth"] + self._init_rule() + + def check_possible_retransmission_occurrence(self, hccl_list: List[HcclInfo]): + min_elapse_time = min(hccl.elapse_time for hccl in hccl_list) + max_transit_time = max(hccl.rdma_info.get('Transit Time(ms)', 0) for hccl in hccl_list) + if min_elapse_time < self.min_retransmission_time: # 检测是否是卡间不同步问题，而不是重传 + return False + return max_transit_time > self.min_retransmission_time + + def check_retransmission(self, hccl_dataset: ClusterCommunicationDataset): + """ + :Param event_dataset: dataset of timeline event + """ + for group_name, hccl_group_dict in hccl_dataset.hccl_dict.items(): + for op_name, hccl_op_dict in hccl_group_dict.items(): + for step_id, hccl_list in hccl_op_dict.items(): + if self.step_id and step_id != self.step_id: # 传输指定step（self.step_id）情况下，非目标step跳过 + continue + if not self.check_possible_retransmission_occurrence(hccl_list): + continue + self.rdma_issues = True + if self.group_statistics.get(group_name) is None: + self.group_statistics.setdefault(group_name, GroupStatistic(self.min_retransmission_time)) + self.abnormal_group_count += 1 + for hccl_info in hccl_list: + if hccl_info.rdma_info.get('Transit Size(MB)', 0): + transit_time = hccl_info.rdma_info.get('Transit Time(ms)', 0) + if transit_time > self.min_retransmission_time: + self.group_statistics.get(group_name).add_op(op_name, hccl_info) + if self.rdma_issues: + self.desc = self.desc.format(group_count=self.abnormal_group_count) + for _, group_statistic in self.group_statistics.items(): + for _, op_list in group_statistic.abnormal_op_dict.items(): + for op in op_list: + self.abnormal_rdma_list.append(op) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("Communication retransmission analysis", self.desc, self.suggestions) + result.add(OptimizeRecord(optimization_item)) + + sub_table_name = "Comm Retransmission Analysis" if not self.stage else f"Stage-{self.stage}: Comm Retransmission Analysis" + result.add_detail(sub_table_name, headers=self.headers) + + for row in self.abnormal_rdma_list: + result.add_detail(sub_table_name, detail=row) + + def make_render(self, html_render, add_render_list=True): + return html_render.render_template(key="cluster", + template_dir="templates", + template_name="communication_retransmission_analysis.html", + desc=self.desc, + solutions=self.solutions, + headers=self.headers, + data=self.abnormal_rdma_list + ) + + def _init_rule(self): + syncbn_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "rdma_analysis.yaml" + ) + + syncbn_rule = FileManager.read_yaml_file(syncbn_rule_path) + self.desc = syncbn_rule.get("problem") + self.min_retransmission_time = syncbn_rule.get("min_retransmission_time") + + self.solutions = syncbn_rule.get("solutions") + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 52e3e0755..db76aaa95 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. class SupportedScopes: # used for specify fourth-level commands and define the key of the result dict @@ -6,6 +20,7 @@ class SupportedScopes: GRAPH = "graph" SLOW_RANK = "slow_rank" SLOW_LINK = "slow_link" + COMMUNICATION_RETRANSMISSION_DETECTION = "communication_retransmission_analysis" OVER_ALL = "over_all" DYNAMIC_SHAPE_ANALYSIS = "dynamic_shape_analysis" AICPU_ANALYSIS = "aicpu_analysis" diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index e1163f1cd..b4956139c 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import logging import os @@ -10,6 +24,7 @@ from profiler.cluster_analyse.common_func.constant import Constant from collections import defaultdict from profiler.cluster_analyse.cluster_analysis import Interface from profiler.advisor.dataset.cluster.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo logger = logging.getLogger() @@ -114,6 +129,7 @@ class ClusterCommunicationDataset(ClusterDataset): self.SDMA_TIME_MS: 0, self.SDMA_SIZE_MB: 0, }) + self.hccl_dict = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) super().__init__(collection_path, data) @staticmethod @@ -136,9 +152,26 @@ class ClusterCommunicationDataset(ClusterDataset): def process(self, communication_json: dict): for comm_group, group_dict in communication_json.items(): + if self.hccl_dict.get(comm_group) is None: + self.hccl_dict.setdefault(comm_group, defaultdict(lambda: defaultdict(list))) for step, step_dict in group_dict.items(): for op, op_dict in step_dict.items(): self.compute_bandwidth(op_dict) + self.process_hccl_info(comm_group, step, op, op_dict) + + def process_hccl_info(self, group, step, op, op_dict): + op_name = op.split("@")[0] + for rank_id, rank_dict in op_dict.items(): + try: + hccl_info = HcclInfo(group, step, rank_id, op, rank_dict) + if self.hccl_dict[group].get(op_name) is None: + self.hccl_dict[group].setdefault(op_name, defaultdict(list)) + if self.hccl_dict[group][op_name].get(step) is None: + self.hccl_dict[group][op_name].setdefault(step, list()) + self.hccl_dict[group][op_name][step].append(hccl_info) + except ValueError as e: + msg = "[ERROR] Cluster_communication.json has invalid structure." + raise ValueError(msg) from e def compute_bandwidth(self, op_dict: dict): for rank_id, rank_dict in op_dict.items(): diff --git a/profiler/advisor/dataset/cluster/hccl_collection.py b/profiler/advisor/dataset/cluster/hccl_collection.py new file mode 100644 index 000000000..bd6de81f1 --- /dev/null +++ b/profiler/advisor/dataset/cluster/hccl_collection.py @@ -0,0 +1,64 @@ +""" +hccl info +""" +import logging + +logger = logging.getLogger() + + +class HcclInfo(): + def __init__(self, group: str, step: str, rank: str, op: str, rank_dict: dict) -> None: + self._group = group + self._step = step + self._rank = rank + self._name = op.split("@")[0] + self._elapse_time = self.get_elapse_time(rank_dict, "Elapse Time(ms)") + self._sdma_info = self.get_communication_info(rank_dict, "SDMA") + self._rdma_info = self.get_communication_info(rank_dict, "RDMA") + + @property + def group(self): + return self._group + + @property + def step(self): + return self._step + + @property + def rank(self): + return self._rank + + @property + def name(self): + return self._name + + @property + def rdma_info(self): + return self._rdma_info + + @property + def sdma_info(self): + return self._sdma_info + + @property + def elapse_time(self): + return self._elapse_time + + @staticmethod + def get_communication_info(rank_dict: dict, name: str): + communication_bandwidth_info = rank_dict.get('Communication Bandwidth Info', dict()) + return communication_bandwidth_info.get(name, dict()) + + @staticmethod + def get_elapse_time(rank_dict: dict, name: str): + communication_time_info = rank_dict.get('Communication Time Info', dict()) + return communication_time_info.get(name, "") + + def get_rdma_transmit_time(self): + return self.rdma_info.get('Transit Time(ms)', 0) + + def get_rdma_transit_size(self): + return self.rdma_info.get('Transit Size(MB)', 0) + + def get_rdma_bandwidth(self): + return self.rdma_info.get('Bandwidth(GB/s)', 0) diff --git a/profiler/advisor/display/html/templates/communication_retransmission_analysis.html b/profiler/advisor/display/html/templates/communication_retransmission_analysis.html new file mode 100644 index 000000000..75754fde7 --- /dev/null +++ b/profiler/advisor/display/html/templates/communication_retransmission_analysis.html @@ -0,0 +1,40 @@ + diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index 1d3872a17..61b572950 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import os from collections import OrderedDict import sys @@ -11,6 +25,7 @@ from profiler.advisor.analyzer.graph_fusion.graph_fusion_analyzer import FusionO from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.analyzer.cluster.slow_rank_analyser import SlowRankAnalyzer from profiler.advisor.analyzer.cluster.slow_link_analyser import SlowLinkAnalyzer +from profiler.advisor.analyzer.cluster.Communication_retransmission_analyzer import RDMARetransmissionAnalyzer from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer from profiler.advisor.analyzer.schedule.syncbn.syncbn_analyzer import SyncBNAnalyzer @@ -39,6 +54,7 @@ class Interface: "overall": OrderedDict({SupportedScopes.OVER_ALL: OverallSummaryAnalyzer}), "dataloader": OrderedDict({SupportedScopes.DATALOADER: DataloaderAnalyzer}), "cluster": OrderedDict({ + SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION: RDMARetransmissionAnalyzer, SupportedScopes.SLOW_RANK: SlowRankAnalyzer, SupportedScopes.SLOW_LINK: SlowLinkAnalyzer }) diff --git a/profiler/advisor/rules/rdma_analysis.yaml b/profiler/advisor/rules/rdma_analysis.yaml new file mode 100644 index 000000000..6c6062775 --- /dev/null +++ b/profiler/advisor/rules/rdma_analysis.yaml @@ -0,0 +1,9 @@ +problem: "RDMA communication retransmission occurs. A single retransmission takes more than 4s. Retransmission problems +are detected in {group_count} communication domains. \n +Advised to perform the following suggestions" +min_retransmission_time: 4000 #ms +solutions: + - check RDMA transmission time: + desc: "Check whether the transmission time of the RDMA operator that is suspected to be retransmitted is correct." + - Check the network configuration.: + desc: "Check the network configuration of the switch and compute node server." \ No newline at end of file diff --git a/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py b/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py new file mode 100644 index 000000000..eb383a659 --- /dev/null +++ b/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py @@ -0,0 +1,170 @@ +import os +import shutil +import stat +import json + +import unittest +from profiler.advisor.interface.interface import Interface +from profiler.advisor.common.analyzer_scopes import SupportedScopes + + +class TestRdmaAdvice(unittest.TestCase): + TMP_DIR = "./tmp/" + OUTPUT_DIR = "./tmp/cluster_analysis_output" + interface = None + err_interface = None + + def tearDown(self): + if os.path.exists(TestRdmaAdvice.TMP_DIR): + shutil.rmtree(TestRdmaAdvice.TMP_DIR) + if os.path.exists(TestRdmaAdvice.OUTPUT_DIR): + shutil.rmtree(TestRdmaAdvice.OUTPUT_DIR) + self.clear_htmls() + + def setUp(self): + if os.path.exists(TestRdmaAdvice.TMP_DIR): + shutil.rmtree(TestRdmaAdvice.TMP_DIR) + if not os.path.exists(TestRdmaAdvice.TMP_DIR): + os.makedirs(TestRdmaAdvice.TMP_DIR) + if not os.path.exists(TestRdmaAdvice.OUTPUT_DIR): + os.makedirs((TestRdmaAdvice.OUTPUT_DIR)) + self.clear_htmls() + + @classmethod + def clear_htmls(cls): + current_path = os.path.dirname(os.path.abspath(__file__)) + for filename in os.listdir(current_path): + # 检查文件是否以“mstt”开头 + if filename.startswith("mstt"): + # 构建文件的完整路径 + file_path = os.path.join(current_path, filename) + # 删除文件 + os.remove(file_path) + + @classmethod + def get_cluster_communication_view(cls): + data = {"p2p":{"step1" : { + "hcom_broadcast__844_0_1@13681369207305868844": { + "0": { + "Communication Time Info": { + "Start Timestamp(us)": 1713174287354248.0, + "Elapse Time(ms)": 4688, + "Transit Time(ms)": 0, + "Wait Time(ms)": 0.01162, + "Synchronization Time(ms)": 0.01162, + "Idle Time(ms)": 39.0606, + "Wait Time Ratio": 1.0, + "Synchronization Time Ratio": 1.0 + }, + "Communication Bandwidth Info": { + "RDMA": { + "Transit Size(MB)": 80, + "Transit Time(ms)": 4600, + "Bandwidth(GB/s)": 0.003, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "HCCS": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "PCIE": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SDMA": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SIO": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + } + } + }, + "16": { + "Communication Time Info": { + "Start Timestamp(us)": 1713174287186619.8, + "Elapse Time(ms)": 4788, + "Transit Time(ms)": 0.0013, + "Wait Time(ms)": 39.037240000000004, + "Synchronization Time(ms)": 39.03034, + "Idle Time(ms)": 167.66008000000002, + "Wait Time Ratio": 1.0, + "Synchronization Time Ratio": 1.0 + }, + "Communication Bandwidth Info": { + "RDMA": { + "Transit Size(MB)": 80, + "Transit Time(ms)": 4700, + "Bandwidth(GB/s)": 0.0033, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "HCCS": { + "Transit Size(MB)": 4e-05, + "Transit Time(ms)": 0.0013, + "Bandwidth(GB/s)": 0.0308, + "Large Packet Ratio": 0.0, + "Size Distribution": { + "4e-05": [ + 1, + 0.0013 + ] + } + }, + "PCIE": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SDMA": { + "Transit Size(MB)": 4e-05, + "Transit Time(ms)": 0.0013, + "Bandwidth(GB/s)": 0.0308, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SIO": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + } + } + }, + } + }}} + return data + + @classmethod + def create_communicaton_json(cls): + raw_data = cls.get_cluster_communication_view() + with os.fdopen(os.open(f"{TestRdmaAdvice.OUTPUT_DIR}/cluster_communication.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + def test_run_should_run_success_when_contain_cluster_communication_json(self): + self.create_communicaton_json() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "cluster" + scope = SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []))) + self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []).get('data'))) + result.clear() -- Gitee From 5a3c1f2bdb2850967a8e4c04ccb93a026bde5ea2 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 7 Aug 2024 17:19:04 +0800 Subject: [PATCH 177/791] rename multiprocess_compute --- .../{Multiprocessing_compute.py => multiprocessing_compute.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename debug/accuracy_tools/msprobe/core/compare/{Multiprocessing_compute.py => multiprocessing_compute.py} (100%) diff --git a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py similarity index 100% rename from debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py rename to debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py -- Gitee From 94f3f887b8b8fec384899810f3af928cd76413c2 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 7 Aug 2024 17:30:46 +0800 Subject: [PATCH 178/791] add online run_ut --- .../api_accuracy_checker/run_ut/run_ut.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 04ad039b2..3e5fae413 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -544,20 +544,24 @@ def run_ut_command(args): except Exception as error: logger.error(f"Set device id failed. device id is: {args.device_id}") raise NotImplementedError from error - check_link(args.api_info_file) - api_info = os.path.realpath(args.api_info_file) - check_file_suffix(api_info, FileCheckConst.JSON_SUFFIX) + + forward_content, backward_content, real_data_path = None, None, None + if args.api_info_file: + check_link(args.api_info_file) + api_info = os.path.realpath(args.api_info_file) + check_file_suffix(api_info, FileCheckConst.JSON_SUFFIX) + forward_content, backward_content, real_data_path = parse_json_info_forward_backward(api_info) + if args.filter_api: + logger.info("Start filtering the api in the forward_input_file.") + forward_content = preprocess_forward_content(forward_content) + logger.info("Finish filtering the api in the forward_input_file.") + out_path = os.path.realpath(args.out_path) if args.out_path else "./" check_path_before_create(out_path) create_directory(out_path) out_path_checker = FileChecker(out_path, FileCheckConst.DIR, ability=FileCheckConst.WRITE_ABLE) out_path = out_path_checker.common_check() save_error_data = args.save_error_data - forward_content, backward_content, real_data_path = parse_json_info_forward_backward(api_info) - if args.filter_api: - logger.info("Start filtering the api in the forward_input_file.") - forward_content = preprocess_forward_content(forward_content) - logger.info("Finish filtering the api in the forward_input_file.") result_csv_path = os.path.join(out_path, RESULT_FILE_NAME) details_csv_path = os.path.join(out_path, DETAILS_FILE_NAME) -- Gitee From e2120d0bce572fed561462caebca1b5bcba7a8a3 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Wed, 7 Aug 2024 17:42:22 +0800 Subject: [PATCH 179/791] =?UTF-8?q?=E5=88=A0=E9=99=A4=E9=87=8D=E5=A4=8D?= =?UTF-8?q?=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/service.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 840f97b9a..a7c8ea72c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -175,9 +175,6 @@ class Service: prefix = BaseScope.Module_Type_Module + Const.SEP + name + Const.SEP + \ module.__class__.__name__ + Const.SEP - module.register_full_backward_hook( - self.module_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) - pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 \ = self.build_hook(BaseScope.Module_Type_Module, prefix) if torch_version_above_2: -- Gitee From 72874dc0f3d3c85ca1ad3f6f80aa9b18f3a832a2 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 7 Aug 2024 18:20:56 +0800 Subject: [PATCH 180/791] add online run_ut --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 3e5fae413..a3cd6c81b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -288,7 +288,7 @@ def run_api_online(config, compare): if isinstance(api_data, ApiData): dispatcher.update_consume_queue(api_data) - if config.nfs_path: + if config.online_config.nfs_path: shared_storage_communication_flow() else: tcp_communication_flow() -- Gitee From 1b938e42518b8296068ae01e3d294597cdedf25c Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 7 Aug 2024 18:27:34 +0800 Subject: [PATCH 181/791] ptdbg dump bugfix when distributed api --- .../ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py index 80798ff41..583829074 100644 --- a/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py +++ b/debug/accuracy_tools/ptdbg_ascend/src/python/ptdbg_ascend/dump/utils.py @@ -21,7 +21,7 @@ range_begin_flag, range_end_flag = False, False def check_list_or_acl_mode(name_prefix): global dump_count for item in DumpUtil.dump_switch_scope: - if PRE_FORWARD in name_prefix: + if Const.PRE_FORWARD in name_prefix: rename = item.rsplit(Const.DOT, 1)[0] if name_prefix.startswith(rename): return True -- Gitee From 5f8c61f98986b9f2451fa87cf44a83769b8de87b Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 7 Aug 2024 18:36:50 +0800 Subject: [PATCH 182/791] =?UTF-8?q?clean=20code=E4=BB=A3=E7=A0=81=E4=BC=98?= =?UTF-8?q?=E5=8C=96=EF=BC=8Cut=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/acc_compare.py | 1 + .../msprobe/mindspore/__init__.py | 1 + .../msprobe/mindspore/compare/compare_cli.py | 4 +-- .../msprobe/mindspore/compare/ms_compare.py | 2 +- .../msprobe/pytorch/compare/pt_compare.py | 2 +- .../test/pytorch_ut/advisor/test_advisor.py | 4 +-- .../pytorch_ut/compare/test_acc_compare.py | 33 ++++++++++--------- .../test/pytorch_ut/compare/test_match.py | 2 +- 8 files changed, 27 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 084f8c9e9..015e33228 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -10,6 +10,7 @@ class Comparator: def __init__(self): pass + @classmethod def match_op(self,npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): if check_op(npu_queue[-1], b_op, fuzzy_match): diff --git a/debug/accuracy_tools/msprobe/mindspore/__init__.py b/debug/accuracy_tools/msprobe/mindspore/__init__.py index 70be41497..dfe872c52 100644 --- a/debug/accuracy_tools/msprobe/mindspore/__init__.py +++ b/debug/accuracy_tools/msprobe/mindspore/__init__.py @@ -1,2 +1,3 @@ from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger from .compare.distributed_compare import compare_distributed +from .compare.ms_compare import ms_compare \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py index 368a95310..23582592f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py @@ -16,8 +16,8 @@ def compare_cli_ms(args): ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: - logger.error('This function is not supported at this time.') - raise Exception("Mindspore Unsupport function compare_distributed.") + logger.error('Mindspore Unsupport function compare_distributed.') + raise Exception() else: logger.error("The npu_path and bench_path need to be of the same type.") raise CompareException(CompareException.INVALID_COMPARE_MODE) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 23764a49d..ddf51e314 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -235,7 +235,7 @@ def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) + raise Exception(error.code) msComparator=MSComparator() msComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 43f628dd0..88821f6fd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -237,7 +237,7 @@ def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) + raise Exception(error.code) ptComparator=PTComparator() ptComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/advisor/test_advisor.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/advisor/test_advisor.py index 176b80068..e140f8263 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/advisor/test_advisor.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/advisor/test_advisor.py @@ -7,8 +7,8 @@ from unittest.mock import patch import pandas -from msprobe.pytorch.advisor.advisor import Advisor -from msprobe.pytorch.advisor.advisor_const import AdvisorConst +from msprobe.core.advisor.advisor import Advisor +from msprobe.core.advisor.advisor_const import AdvisorConst class TestAdvisor(unittest.TestCase): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py index 288e259c0..b97dcc5d9 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py @@ -1,7 +1,10 @@ # coding=utf-8 import unittest import pandas as pd -from msprobe.pytorch.compare import acc_compare as compare +from msprobe.core.compare.check import check_graph_mode, check_op +from msprobe.core.compare.utils import merge_tensor, read_op, get_accuracy, rename_api +from msprobe.core.compare.acc_compare import Comparator +from msprobe.core.compare.highlight import find_error_rows,find_compare_result_error_rows npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], @@ -208,60 +211,60 @@ class TestUtilsMethods(unittest.TestCase): def test_check_graph_mode(self): op1 = "Aten" op2 = "torch" - self.assertTrue(compare.check_graph_mode(op1, op2)) - self.assertTrue(compare.check_graph_mode(op2, op1)) - self.assertFalse(compare.check_graph_mode(op1, op1)) - self.assertFalse(compare.check_graph_mode(op2, op2)) + self.assertTrue(check_graph_mode(op1, op2)) + self.assertTrue(check_graph_mode(op2, op1)) + self.assertFalse(check_graph_mode(op1, op1)) + self.assertFalse(check_graph_mode(op2, op2)) def test_check_op(self): fuzzy_match = False - result = compare.check_op(npu_dict, bench_dict, fuzzy_match) + result = check_op(npu_dict, bench_dict, fuzzy_match) self.assertEqual(result, True) def test_merge_tensor(self): - op_dict = compare.merge_tensor(tensor_list, True, False) + op_dict = merge_tensor(tensor_list, True, False) self.assertEqual(op_dict, result_op_dict) def test_read_op(self): - result = compare.read_op(op_data, op_name) + result = read_op(op_data, op_name) self.assertEqual(result, op_result) def test_match_op(self): fuzzy_match = False - a, b = compare.match_op([npu_dict], [bench_dict], fuzzy_match) + a, b = Comparator.match_op([npu_dict], [bench_dict], fuzzy_match) self.assertEqual(a, 0) self.assertEqual(b, 0) def test_get_accuracy(self): result = [] - compare.get_accuracy(result, npu_dict, bench_dict, highlight_dict) + get_accuracy(result, npu_dict, bench_dict, highlight_dict) self.assertEqual(result, o_result) def test_get_accuracy_graph_mode(self): result = [] - compare.get_accuracy(result, npu_dict_aten, bench_dict_functional, highlight_dict) + get_accuracy(result, npu_dict_aten, bench_dict_functional, highlight_dict) self.assertEqual(result, aten_result) def test_find_error_rows(self): summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] highlight_dict = {'red_rows': [], 'yellow_rows': []} - compare.find_error_rows(summary_result, 0, 1, highlight_dict, summary_compare=True) + find_error_rows(summary_result, 0, 1, highlight_dict, summary_compare=True) self.assertEqual(highlight_dict, {'red_rows': [], 'yellow_rows': []}) def test_find_compare_result_error_rows(self): result = [line_input, line_1, line_2, line_3] result_df = pd.DataFrame(result) highlight_dict = {'red_rows': [], 'yellow_rows': []} - compare.find_compare_result_error_rows(result_df, highlight_dict, False, False) + find_compare_result_error_rows(result_df, highlight_dict, False, False) self.assertEqual(highlight_dict, {'red_rows': [num_1, num_3], 'yellow_rows': [num_2]}) def test_rename_api(self): test_name_1 = "Distributed.broadcast.0.forward.input.0" expect_name_1 = "Distributed.broadcast.input.0" - actual_name_1 = compare.rename_api(test_name_1, "forward") + actual_name_1 = rename_api(test_name_1, "forward") self.assertEqual(actual_name_1, expect_name_1) test_name_2 = "Torch.sum.0.backward.output.0" expect_name_2 = "Torch.sum.output.0" - actual_name_2 = compare.rename_api(test_name_2, "backward") + actual_name_2 = rename_api(test_name_2, "backward") self.assertEqual(actual_name_2, expect_name_2) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py index ac28e994e..5dbe4453a 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py @@ -1,6 +1,6 @@ # coding=utf-8 import unittest -from msprobe.pytorch.compare import match +from msprobe.core.compare import match class TestMatch(unittest.TestCase): -- Gitee From 339f87beeb1c36913f58c1b613717e78dc238a78 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 7 Aug 2024 18:48:29 +0800 Subject: [PATCH 183/791] replace self.real_overflow_dump_times with self.real_overflow_dump_nums --- .../msprobe/core/data_dump/data_processor/base.py | 6 +++--- .../core/data_dump/data_processor/mindspore_processor.py | 2 +- .../core/data_dump/data_processor/pytorch_processor.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 80db0104b..c6dfcda2c 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -65,7 +65,7 @@ class BaseDataProcessor: self.current_iter = 0 self._return_forward_new_output = False self._forward_new_output = None - self.real_overflow_dump_times = 0 + self.real_overflow_dump_nums = 0 self.overflow_nums = config.overflow_nums @property @@ -76,8 +76,8 @@ class BaseDataProcessor: def is_terminated(self): if self.overflow_nums == -1: return False - if self.real_overflow_dump_times >= self.overflow_nums: - logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_times}") + if self.real_overflow_dump_nums >= self.overflow_nums: + logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_nums}") return True return False diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 877fc3a01..dd385209e 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -173,7 +173,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): tensor = convert_bf16_to_fp32(tensor) np.save(file_path, tensor.asnumpy()) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - self.real_overflow_dump_times += 1 + self.real_overflow_dump_nums += 1 self.cached_tensors_and_file_paths = {} def _analyze_maybe_overflow_tensor(self, tensor_json): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 191a33f9f..3c0305348 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -207,7 +207,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): for file_path, tensor in self.cached_tensors_and_file_paths.items(): torch.save(tensor, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - self.real_overflow_dump_times += 1 + self.real_overflow_dump_nums += 1 self.cached_tensors_and_file_paths = {} def check_overflow_npu(self): -- Gitee From aa41dbf108fa0fb3f96209e83095f4e0b1f926ab Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 7 Aug 2024 18:53:23 +0800 Subject: [PATCH 184/791] replace self.real_overflow_dump_nums with self.real_overflow_nums --- .../msprobe/core/data_dump/data_processor/base.py | 6 +++--- .../core/data_dump/data_processor/mindspore_processor.py | 2 +- .../core/data_dump/data_processor/pytorch_processor.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index c6dfcda2c..9acac5e8e 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -65,7 +65,7 @@ class BaseDataProcessor: self.current_iter = 0 self._return_forward_new_output = False self._forward_new_output = None - self.real_overflow_dump_nums = 0 + self.real_overflow_nums = 0 self.overflow_nums = config.overflow_nums @property @@ -76,8 +76,8 @@ class BaseDataProcessor: def is_terminated(self): if self.overflow_nums == -1: return False - if self.real_overflow_dump_nums >= self.overflow_nums: - logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_dump_nums}") + if self.real_overflow_nums >= self.overflow_nums: + logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") return True return False diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index dd385209e..1a31f935e 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -173,7 +173,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): tensor = convert_bf16_to_fp32(tensor) np.save(file_path, tensor.asnumpy()) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - self.real_overflow_dump_nums += 1 + self.real_overflow_nums += 1 self.cached_tensors_and_file_paths = {} def _analyze_maybe_overflow_tensor(self, tensor_json): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 3c0305348..f54d97155 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -207,7 +207,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): for file_path, tensor in self.cached_tensors_and_file_paths.items(): torch.save(tensor, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - self.real_overflow_dump_nums += 1 + self.real_overflow_nums += 1 self.cached_tensors_and_file_paths = {} def check_overflow_npu(self): -- Gitee From 780ae4373a80d835ef8f1903bf5187ed5a368a22 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 7 Aug 2024 19:07:10 +0800 Subject: [PATCH 185/791] =?UTF-8?q?compare=E5=87=BD=E6=95=B0=E4=B8=AD?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=BC=82=E5=B8=B8=E6=8D=95=E8=8E=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/compare/distributed_compare.py | 2 +- debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 2 +- .../msprobe/pytorch/compare/distributed_compare.py | 2 +- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 0973e7ffe..1e9586fba 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -107,7 +107,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) + raise CompareException(error.code) from error msComparator=MSComparator() msComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index ddf51e314..b42881ed4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -235,7 +235,7 @@ def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - raise Exception(error.code) + raise CompareException(error.code) from error msComparator=MSComparator() msComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index f4596ba49..05c274b15 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -108,7 +108,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) + raise CompareException(error.code) from error ptComparator=PTComparator() ptComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 88821f6fd..dd7f8fc17 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -237,7 +237,7 @@ def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - raise Exception(error.code) + raise CompareException(error.code) from error ptComparator=PTComparator() ptComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, -- Gitee From 68ae7ed988ef6703a640c1144d658af3e0de4022 Mon Sep 17 00:00:00 2001 From: wangqihui01 Date: Tue, 6 Aug 2024 10:21:15 +0800 Subject: [PATCH 186/791] =?UTF-8?q?=E6=94=AF=E6=8C=81=E7=8E=AF=E5=A2=83?= =?UTF-8?q?=E5=8F=98=E9=87=8F=E5=88=86=E6=9E=90=EF=BC=8C=E7=BB=99=E5=87=BA?= =?UTF-8?q?=E7=9B=B8=E5=85=B3=E5=BB=BA=E8=AE=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/README.md | 4 + .../overall/environment_variable_analyzer.py | 47 ++++++++ .../overall/environment_variable_checker.py | 102 ++++++++++++++++++ profiler/advisor/common/analyzer_scopes.py | 1 + profiler/advisor/common/constant.py | 1 + .../dataset/environment_variable_dataset.py | 47 ++++++++ .../html/templates/environment_variable.html | 21 ++++ profiler/advisor/img/env_var.png | Bin 0 -> 61753 bytes profiler/advisor/interface/interface.py | 6 +- .../rules/environment_variable_info.yaml | 42 ++++++++ ...347\275\221URL\350\257\264\346\230\216.md" | 1 + 11 files changed, 271 insertions(+), 1 deletion(-) create mode 100644 profiler/advisor/analyzer/overall/environment_variable_analyzer.py create mode 100644 profiler/advisor/analyzer/overall/environment_variable_checker.py create mode 100644 profiler/advisor/dataset/environment_variable_dataset.py create mode 100644 profiler/advisor/display/html/templates/environment_variable.html create mode 100644 profiler/advisor/img/env_var.png create mode 100644 profiler/advisor/rules/environment_variable_info.yaml diff --git a/profiler/advisor/README.md b/profiler/advisor/README.md index 04dd0b843..0f6a03807 100644 --- a/profiler/advisor/README.md +++ b/profiler/advisor/README.md @@ -65,6 +65,7 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 | dimension | mode | 参数释义 | | ---------- |---------------------------------------| ------------------------------------ | | overall | overall_summary | 计算、通信、空闲等维度对性能数据进行拆解 | +| | environment_variable_analysis | 环境变量设置推荐 | | cluster | slow_rank | 慢卡识别 | | | slow_link | 慢链路识别 | | | communication_retransmission_analysis |通信重传检测 | @@ -142,6 +143,9 @@ overall模块的分析包含当前训练任务慢卡的性能拆解，按照计 ![输入图片说明](./img/overall.png) +overall模块的environment_variable_analysis是对环境变量的设置做出推荐 +![env_var.png](img%2Fenv_var.png) + schedule模块包含亲和API、aclOpCompile、syncBatchNorm、SynchronizeStream等多项检测。 如下图示例，Operator Dispatch Issues提示需要在运行脚本的最开头添加如下代码用于消除aclOpCompile： diff --git a/profiler/advisor/analyzer/overall/environment_variable_analyzer.py b/profiler/advisor/analyzer/overall/environment_variable_analyzer.py new file mode 100644 index 000000000..3daaa3460 --- /dev/null +++ b/profiler/advisor/analyzer/overall/environment_variable_analyzer.py @@ -0,0 +1,47 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.prof_common.path_manager import PathManager +from profiler.advisor.dataset.environment_variable_dataset import EnvironmentVariableDataset +from profiler.advisor.analyzer.overall.environment_variable_checker import EnvironmentVariabelChecker + + +class EnvironmentVariabelAnalyzer(BaseAnalyzer): + dataset_cls_list = [EnvironmentVariableDataset] + + def __init__(self, collection_path: str, n_processes: int = 1, **kwargs): + super().__init__(collection_path, n_processes, **kwargs) + self.dataset = self.get_first_data_by_key(self.dataset_list, EnvironmentVariableDataset.get_key()) + + def optimize(self, **kwargs): + try: + PathManager.check_input_directory_path(self.collection_path) + except RuntimeError as e: + logging.error("Invalid path: %s", str(e)) + return self.result + self.collection_path = PathManager.get_realpath(self.collection_path) + checker = EnvironmentVariabelChecker() + checker.format_env_suggest(self.dataset) + checker.make_record(self.result) + checker.make_render(self.html_render) + return self.result + + def make_record(self): + pass + + def make_render(self): + pass diff --git a/profiler/advisor/analyzer/overall/environment_variable_checker.py b/profiler/advisor/analyzer/overall/environment_variable_checker.py new file mode 100644 index 000000000..ca316530d --- /dev/null +++ b/profiler/advisor/analyzer/overall/environment_variable_checker.py @@ -0,0 +1,102 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem +from profiler.advisor.result.item import OptimizeRecord +from profiler.advisor.common.analyzer_scopes import SupportedScopes +from profiler.advisor.display.html.render import HTMLRender + + +class EnvironmentVariabelChecker: + ENV_SUGGEST_CONDITION = { + "ASCEND_GLOBAL_LOG_LEVEL": lambda x: x != "" and x != 3, + "HCCL_RDAM_TC": lambda x: x != "", + "HCCL_RDMA_SL": lambda x: x != "", + "ACLNN_CACHE_LIMIT": lambda x: x == "" or (isinstance(x, int) and x < 10000), + "HOST_CACHE_CAPACITY": lambda x: x == "" or x == 0, + "ASCEND_ENHANCE_ENABLE": lambda x: x == 0, + "PYTORCH_NPU_ALLOC_CONF": lambda x: "expandable_segments:True" not in x, + "ASCEND_LAUNCH_BLOCKING": lambda x: x != 1, + } + + HEADERS = ["Environment", "Value", "Description", "Suggestion"] + + def __init__(self): + self.environment_info = self.read_environment_info() + self.env_suggest_csv = [] + self.env_suggest_html = [] + + @staticmethod + def read_environment_info(): + environment_variable_info_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "environment_variable_info.yaml" + ) + return FileManager.read_yaml_file(environment_variable_info_path) + + def format_env_suggest(self, data): + data = data.env_data.get('ENV_VARIABLES', {}) + for env, value in data.items(): + if not self.ENV_SUGGEST_CONDITION.get(env, lambda x: False)(value): + continue + desc = self.environment_info.get(env, {}).get("desc", "") + suggest = self.environment_info.get(env, {}).get("suggest", "") + self.env_suggest_csv += [ + [ + env, + value, + desc, + suggest, + ] + ] + self.env_suggest_html += [ + [ + env, + value, + desc.replace('\n', '
'), + self.environment_info.get(env, {}).get("suggest_html", suggest), + ] + ] + + def make_record(self, result: OptimizeResult): + if not self.env_suggest_csv: + return + desc = f"Describe and suggest the optimal environment variable settings" + suggestion = "Please set the optimal environment variable" + + optimization_item = OptimizeItem( + SupportedScopes.ENVIRONMENT_VARIABLE_ANALYSIS, + desc, + [suggestion] + ) + result.add(OptimizeRecord(optimization_item)) + result.add_detail(SupportedScopes.ENVIRONMENT_VARIABLE_ANALYSIS, headers=self.HEADERS) + for env_suggest in self.env_suggest_csv: + result.add_detail(SupportedScopes.ENVIRONMENT_VARIABLE_ANALYSIS, detail=env_suggest) + + def make_render(self, html_render: HTMLRender): + if not self.env_suggest_html: + return + html_render.render_template(key="overall", + template_dir="templates", + template_name="environment_variable.html", + result={ + "headers": self.HEADERS, + "data": self.env_suggest_html, + }) diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 3d20374d4..b947798c9 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -23,6 +23,7 @@ class SupportedScopes: COMMUNICATION_RETRANSMISSION_DETECTION = "communication_retransmission_analysis" PACKET = "packet_analysis" OVER_ALL = "over_all" + ENVIRONMENT_VARIABLE_ANALYSIS = "environment_variable_analysis" DYNAMIC_SHAPE_ANALYSIS = "dynamic_shape_analysis" AICPU_ANALYSIS = "aicpu_analysis" BLOCK_DIM_ANALYSIS = "block_dim_analysis" diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index cdc0dd4e5..c97cfbfd1 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -75,6 +75,7 @@ CANN_VERSION_C17 = '8.0.RC1' SUPPORTED_CANN_VERSION = [CANN_VERSION_C30, CANN_VERSION_C13, CANN_VERSION_C15, CANN_VERSION_C17] DEFAULT_CANN_VERSION = CANN_VERSION_C17 ASCEND_PYTORCH_PROFILER = "ascend_pytorch_profiler" +PROFILER_METADATA = "profiler_metadata.json" MSLITE = "mslite" MSPROF = "msprof" SUPPORTED_PROFILING_TYPE = [ASCEND_PYTORCH_PROFILER, MSLITE, MSPROF] diff --git a/profiler/advisor/dataset/environment_variable_dataset.py b/profiler/advisor/dataset/environment_variable_dataset.py new file mode 100644 index 000000000..577273ffe --- /dev/null +++ b/profiler/advisor/dataset/environment_variable_dataset.py @@ -0,0 +1,47 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import logging + +from profiler.advisor.common import constant +from profiler.cluster_analyse.common_func.file_manager import FileManager + + +class EnvironmentVariableDataset: + def __init__(self, collection_path, data: dict, **kwargs): + self.collection_path = collection_path + self.env_data = {} + self.read_data() + + @staticmethod + def get_env_data_file(collection_path: str) -> str: + for root, _, files in os.walk(collection_path): + for file_name in files: + if file_name == constant.PROFILER_METADATA: + return os.path.join(root, file_name) + return "" + + @classmethod + def get_key(cls): + return cls.__module__.rsplit('.', maxsplit=1)[-1] + + def read_data(self): + data_path = self.get_env_data_file(self.collection_path) + if not data_path: + return + try: + self.env_data = FileManager.read_json_file(data_path) + except RuntimeError as e: + logging.error("Read json failed. %s", str(e)) diff --git a/profiler/advisor/display/html/templates/environment_variable.html b/profiler/advisor/display/html/templates/environment_variable.html new file mode 100644 index 000000000..ab9509639 --- /dev/null +++ b/profiler/advisor/display/html/templates/environment_variable.html @@ -0,0 +1,21 @@ +
+

Environment Variable Issues

+
+ + + {% for header in result.get("headers") %} + + {% endfor %} + + + {% for row in result.get("data") %} + + {% for value in row %} + + {% endfor %} + + {% endfor %} + +
{{ header }}
{{ value|safe }}
+
+
\ No newline at end of file diff --git a/profiler/advisor/img/env_var.png b/profiler/advisor/img/env_var.png new file mode 100644 index 0000000000000000000000000000000000000000..a2c9b6f20e67600f09cff6f5269a464dd0010115 GIT binary patch literal 61753 zcmdqIXIN8d*EWo0#0KbCP^pdu6$O#r5|mL?#OOGJ(jp=tHT0SyI>0zeQ3Rw$2LTl! z(tDz0Bp`$UfrO5c5+Q^@LK4!y4LJ8R_xs-OkMDbq=Xj6f<5$@Gs_R;7UFSO2+7GW> zwo%x;Z?l}7oWh0kKmQ^p_rrvo+&Z0&8-OQ~+3W7f$=#E?@bl?kBVA`lSjnn=xw=v@ z1o`z~dDYXnXNY^dPB{MX)2Gj_c-NJGmUFB`qz#HVJ7hsB56`7}Z2^mJaS8Tf56LEqAY#Q13YrS$E!>qT$D zeUj2FGFl)5NeqCM0Dn#l{pl?$zdlUuXkEAb>%&^->zcA@l+1YL`RrZqa96T+3NDg% zW@*Y8w->~(BgMHigs)k7U3@!0O0-UB_QfAynnZ6%y^-KMniDooD?#mvBMMF}QS z=f_Y2TfJw?@3qFs@hg4rRm0%pDztnh@K!Wj9HjS*wRK{=BQHsQx%a&RkZ6DuA0ZRd z=V|g9JFNvini^hz=ocxhT^Za%Cg?P$kx;^GtNm^md%Lf^uo{4&IN4W^XrP}ZE0^7HRfI`|)cm4}F%+wb z>*o75R(wailnfRg@!h)8=tzu&OT}ly8n=%|iJ0@F4XmVKUYiMfZPd!^{(~7hE}LP_ zSk+N%u!liXqm}x~^Xd-;kCbg<#k*oHR{ra&?+jb_jv|0?QKQ;MjmzW(<{{bDpx-yViU1-b;!Jq)~ysKe8G z>0sr{@%U8ln7Pebt|uyU9it6~2k&C`wtBg`W=F7|wAkHufW#kw z4(#~4gHxl9!0!5jo4*1_^i=;XzWz}w=_92CePW=fnM3=QE3&cfnr&Jr4P30_rE-X0 zB=h7%Z(U1O9;PNnQyl4C-yt91;A3FAv9EXxU~`Vx+`XDfRqXq;$Ea zVs9%h+G5zHzcb8uc;_659k{eWCMD?8*?N;dtm?BlnYV#We9^G|UQOtFa>@R?=OCmS zYay?novqvoA(X>vX@MrZioM{=F*pA{Fn?c7)I{j`CUzY& z>BO{>&brH&hxK(}gHpDZ>^Onm;+1Ero}?Dlm14MRW-v|Em$KqL*2p`MTHN*-G}st% z1~Iv)bPlX&o*RK`D9O;+RI{`C%^goo8F#9d+fM7~UY8hF16%*y8p8|y6(%AORf3;I zZ?}*V@+0On;s#=1joPY+Qp@$1Vm4-#9U#T0d8!;}r!78#BbX|38F&GDSUI3+|K%a& z0L$R=U2oA+X&!gda|3(KOkJl`w_#_J9rBjZAX#zbpQ35t{nmi?5pVR(08X!DfZYw4 z;F(jbxKvu8(b!u{ejK?yuKHSAJupsNV%(lZjs6|5W2%Y}ohnp+*rXQUAbr>O^V>P3 z+G?|GLZl_VUWcRve7@E|yP{>EtMn^5p(5%qZ&dKJ3GSC0gue>tcDhDeV89uls&g{dZmF|8W?HU9G6O5On*#^(%Tb7|t3+gvE;0B8HzBF5l~Na+Q)~ zoT%ZdrmSVMo%T6Rn z$6|EFbaU2kHAo3#UZcI8Rt1%w%8`B2ZLUMhEX3k8hn1;s**uDh>L9u!)k7Aa7ZIE- zrxfJtjM^87o zE~@e3WJ6!LfTG4kwGH(!wW%Y%73SmND#{t|C2no-id$0X>M@JQ*ZoWq}9 zIH%6}XHVgs+N{i2_|a#{50UT=kkKCbSIM{^V-5-N8mk-istfjbmP%-}n3~c9C!0ne zeGH(CqL*-ZBPc0qhy0((I7JneW#A=uFZcM=q@HoIAgZI=)(wua!tVV{C=2+rzn`Fi zSe^gfA9aF**1B;EP3r^~j)1qTtSE?FVU}rhRvBqmYleZ!>UEoYgxKj@dTjMFl@o6N z8$11vDdhi@d3?}ZCxFOAxK#{tt>EV3pMcMmbth=VYonH|m$)jh-;8yks{%Z+J>;7< zPKH>Tf1p%-;?-ES6)1k;e&otQ>^?fxqtPfsF12f+f0Ieup}Y5ehB;|!pdUlbS20`l zAPXfL>b4#S;NHR+Yk~i!iSf=TvlSD#=f41G`v_PTWdJAsWG#r3X9pF1?etJv8Cz}( z8_jCmAr%*aPZ&HKOSb|g5mV<@k<{_e{|5Mm9ls-^{{`?-Q4`$yEfeGJvV{d%D1CLi zSEcSzrj1+%G&%1nO4h`uB$QmxXA&#BtdOUWVn8dNG5?-~fx&@)K15b_{NUV`El9U=pz=3q3VN8^Ag>63DGEtgkCWXb9GfL!s-WTb%vl$ zkv`W2&cs^NJ|xDpaAr4&9H?OvWpwngrWKy)C)149OIeCQ6CRQ8Q;QkFls5uNn_tk)HaH2=n=yi z>x$b!PB(ZnRAVz;`#dE+%B_BuR#Zg)t2b_b9HZ54SVPOSX>O_+tHct0hXOTKqrXI& z#2oeat~$(`5WMp{a?HsHMlkC|G)EjUYGT z5IsC(VB&^fi?HRRFjHgHz-7u@%SmFvo|&m@7@cF%2pFSkTC3fs5U-KMVpi^i#93*= zn6jd}LpGY=B91Khy}6uQ>b)o5IB%n}3eylj zSv}%522P?&rj-iaral63k}@T|X=%|iF(oA8M<41sCQ)G`8>KfN7sy%wq8`;})it^b zN#@nCmc?lFp@3niLhN|;sEphqbVT%cR1uT1b*7=0oZ;dwq3IHJ40)T`HkURTIAuc& z;oLHzY-BsV47kn8uuqm<6k!YkOtXe(?^^E7o28%g`zH34pBQ=SKl~fF1UxNe<7w$_ zr_f;4%O`61%&+b>keYnKsGm;6`yg_F;1fk5-CzQaI!!T8?xlGe0{H#-hht<$W1P7r@vFPDwec z8byE*p^A1jdN;%MDaN^!um>fk&UP`5oIPT01f)7)mrp#SK`~BCg;2P=6;`mxgAk;5p&!eepReiJ*ItkatavQPQs!LbRENo+hC9Ryc?&U+QKF}tMLb0}g}}oxGd1oa z&r2}w%~R{UvVlX1+$^;+sqXcwP#>%o9YkELYjTdp7wm4XKK?z#vJ8Z^7HpST zLHlOxxkxOXv%*Hhvrq~r_huCW3dQ?XxWWIMm@!w;Ih~EgPF=Z{$`9a3oTb{orP0;5 z%A29ynMjW;=9g$>FwK;Wdj92g5|roAAWI9`4t7SW-Z!O$9T#It>|Mv4n-?5w6y7!oS`1toOpI|HYA7{q zyhPxgRdatA!5c_fD@apT8PBk3z+-haHuKdH206&-he&J6t2HRw5cp!1n{O6`2$7bL z&-g7aFJHJ=audqgFm$&+Yw8*YB?>f2W%6&pFUo*h92k|9-1_otM$jj{1I zh4{XM?tjAi<4L$uZ#8m_4}z%TSBwamwIS1dqLV9d<*GVZF&q#`Y2Y>eOR$^=9MuNU zzf^)#JO6WS!}egNl#QVE_AEW5EjznbV&Z!PwfmmO1+iU*(2abV0Te2-Q2|^CY6)Iz zg4`$m8lsfoM6D^Xv^WRsrfTgJp7yi;9hWGh;h}y406eo1ziy6o8X8m0MjTc3wO}?f zS^zy>h@@-V67Vz?74lOiYqnzl4*)YlE_&}YRJUM-wFC+0xFKdZ@5<9EAX&ZWof)E? zLdAZpj%mv6F%c2>2Yil2a8&)0lA)k*02lo2ugL@M@?J?JYD^sqHC&k$wBtaE|dw^zqTd7t&9!2{`}GCrB*Cwo3)2ZooG&E8)tXs0(P!?L9XAEhi+B`+;h z;(NGG<2-&^ua)0qX>cLK_f!gOq-#J!5p3u>*9UN5&;V9UNMxo@%vq*}rt3BJ9SCb9 zxHOD^GfbFX!V2a&C?7}*;yGL%Q|3XY1mceXEd^Y6@CW{rlT!-$k`k5BD)*Xh9 zjCzAN8n~sSV?3C+&i*uTA}Y-&e0kZ_W%XVZhn`25l?L%h(srgk@R&qjz&`zQeEt~LayS%kA}0}J1If$;e0P!UOP4=Q51tQ-eDv4fD0FPrNof~N_koU0*4{CCmX_d%(ku7%l z3InYI>WJ3}x#l*)H_LraIpM~|rl-A0*Owuw>JsGv?JrwFGN#iaJmg^8MAf zLuA;1V+WdC+msnGx8R?A0_T}rG``&G`2}FSfx1|x;mqO#HyJcB84qPDK^yqpfm6S@ zVasx3;G{S$oS`6g5`bG7N^uFu;1bpjpr_oUfAG}*N7CiL8w)K@B==s<{8)+%2wU%b zj~$(ui2$|Y^#dCxV!|peuQa+>QeEC?v3X+bhD4_AQ0gp;6{93$vh1+cNY$?ZAZ~nd zVvpL;>7FAgs2HvFY*v$K?i*xHPv`0Y#N7{;WJ_R~mY6LT%+I|2cgCiV>Y}OfNliF0 zo{SfB=K_{Sb>{U9+@9lFiLDg)kTQ4I?5Zdv^ z-Bm4w@f=!|$@6rK)YM?qkpg$wHRZ!gM{<=W$I3MmAIJ8S+ym(enqro)(hW&l7=kPDF9odYmt&{zAHk9 zxzb(k#xoh5p1VSpy?iOIUe6Bl%lj8`ffQwrJw78?dIeX-9-gB7O()#RKyi!bMPm{j zF;zph2NWrjF!X^emxMr&k#kp1`K$oY!FdV>Wf^@@D&dbhX;dF3RvMGftP`{?EdZjt zTK!#SM=&^PbR;LyxY-KkwUo~Y>$(s8NZIa&)hZA46dzN;GbcWx`z zR#`V`tw8(lRhj9fB!g$8CzVj!zVl@E|6Rfi-Tr_U*PemT=0tKMO`cU%9APpMw3ne~ zP64FYz=<~iNQiB&?9H1H11TKU^K@_FS0L7Pm!^^VlN`VFggjUdm6VXsGaX{K*c+#S zvnll)5U2u4pIjd6Q<37XsbvkhCj~I+_Es`AFYpUGDWM``YFcDTdx zDeL~leRMK#ra@qleNJEso3ZG9p z{Nr~q*7^X~tl#lF9$JH6>dNWzw8h{9cTs{T@4>lR7vqcTmKJI7yfqD3S0p~^xydVZ z`90#H6mJ}z zg-pZ|IO9*^W?}-AY|uBeN{s_pA<4htp`ZDdPDUf?DvBjleYIs*hmrzDV#T8vRYk(r z=qyoV{fIW*b*t7Kt|<_7R@D4bJ}J5LC1s&5bbk)Q(_Lil8@j zsiA(+`V)oRqul%kmO-Iz2eM>3IS>kJ>f4>y&$987BF2y@03`xp`wkkd8@9Ej&Biqc z1G%I)`7C_-!TsN~>6_c%Il3|vFvGi$jlEkjMOl&>Psq|>va8>s``p)QjmrDnM0HH@ zmI?Csh{f(5ZGnb~+j!}HNY)_1SQouTYfg2n$o3y4~7uO)Gg->xtgwyy|qF<8`G8{;1%u#SQkK1VwWf z&j_;d)}%BNHEf~ray7;m<#k!sV@dn91ZKl<+6=zjnJ_@_nu+1n3a6B!)fJ2O#fI6DcH0Ot2ju-!5DjJHVO6nkBR`_8QVmS!AiggL1#cwA z%=sRV*q^fx%*C<3;AwH5F~vh&TmLYx;ZFj2wIFPgm6Sp#q>p``5$6|`@@JJ2l8s5Z zyldiII`o)n&D%Ahje1@5;hXzd<2Dgih+FBjI6vPm!M7>nsxQ7;*5@@DGZDjyfzHgQ~P zS6J4adcaMbf><@|zH4aR3KvWU^4^G=OY}Z8(r!f_iTbf=?+XoNyfTng`Q9XWf#lU}sV+#h(|}$#4>yUXORqj};I}k1chd*`D+6}EiWReeF$tRZ za%6ZYV*X=bJ@+rT{ukmp0SBD9c=bo;q(LAntfGJ~*Hsl}6i`x|$P$00Y5hU!gVd8i zBsN=c?(RU4wR_%pnvVGxF*~$BD}1SLIY;eHZ%(Z#X&M!t?mrlc6Gfh=1O`pZP;W~( z^#;i9j;H0YVg~6Dx`1j@!&7V2&ABU?#0Q%oUX}Z~Lwk^>hM=+A&*A*CDX7iC(^1z? zq`;IsLT84EZuflfHdQq-lU_o^S9;=ZoV9<^4l@sC;%P|ca)p}f26?<_ZmR zPLyEOf7eMMk3pK3T&7JXZD}`*dI?14%WGMX`kJ-Fjgs)8s(TTA+K|dTPf%CnK66-2 zIj-bZB=`An=B$mc&_@H=G(74|D6T$XS%9bfRklbiJY=lwh;q47QG~EA(ocbqUY7Z2 zj?v<=v2eHCHs~g23#3ET02eW^BT4$sLYBo_5Kwg0l*lLs#-TTa;DZ2}VZCx6rhKYg6-3d5-V zj@^?~INpPa{#Mt5z2lF0vw=|}S0nig25M4oHAbK8%&@I#D8~_tkTK7B&q-IgwMJ z0(pJ#*Ux4I4$fZYRxm|ws!l_-t^jHuI-?(Y#V}=TM^dHq{{HO`cuVU_rlC2*=?Nnz z76`XAT2Tk<8gUgxMJPd+H=^!A39h#7T@H}nk3K2evI3c-0UCVL$C`4pJpuw#Xw!>6 zRj<}l2)8yFza)3lmL5>@ObCApRe_gL{;U5RE+USEbbU@L9OAD0pG$!sg6 zj)O{hrz5gukwWgarS;cjL553zs74GG$3@zeSS`KHOMyJ#luoPdze`_Ut5YnT<}G=9 zF{Y_7!j3V0-pr3kWYT<|YRDQ3!j9XQ98cirTAJ?*n0Se`ru5#Qam2CeiVRDaTH^`= zr>1T2)?jiy#&2l145 z;#mO&StwLFanf>cH73AyS?y9@ z-4go%Ej?6Y%4ppA(yVC6G-;%Wz#%;&yW^z6Wv+^gOFr85U~V10>>avzUxbxMZ^WV>2FR+ZJm zO!g#+U+F2L7!Ri$V-STkMY)!{VQXi%7)$*1-{+Np?4yLr+uL@0Eh(Gihbr2JT#tow zxQiF~cV54k5h$O~joUHqUe0g9)7lEAkc1bw6(6c$XDLesE{goJPJWzIa*m&PDM;x4 z!7wePjH`YrL1FN^;L&5AY2?B*en990sbopqLZ1KGCow~s%q9~+mJ#_=Wp+NFKHDb$ z)?foxS3x6D2Xnkf=l6i0jLqP;XyIvhRGbJg+@|1zxr%^Pth8dq&V&1X!9=Y)4Qq!% zp!qdkS>Z|NhOja z)7S?}@#MCs!Nv!xZn^|QOJTN^FiGM)IOjqaRpF-*7SrMRpi#>5Xoi{&0+K1SUy2C9}O2Bw9 zrYZZjSj_A>Hjy5K20{_TDJV#}(?Swx5^62lB%Df)cXiRk`H+}EnIIGJU$u+yNH$ur z6857({5qdpR54>%lqPzLtq1PD9W~}->6!F7g#=1<(cYhRsYAutK{|7$+mbA#a1)V_ zXz87^Ba>64{;pIz@4k5$%Trfm45e1encn9pb)Sc z5U(2mO-DDyiep$<6<28uYKx4+lDVE-VPrm}|8ZJ8hTs;{ppcx6m^`qCD@wwhp8`ri zfVzj6XAaPaSoqF}Z>h;(MHY_lx zoy+w|^yk(D3$v*nJ#><3SVu8t2q^OKBPM9Tw=LI|B!-)fK8$3P)SPqY{S~{I-+y$a zYoOub%#aSWyP_KB+jCn2+-#~hmF830UJur+hhM@~ zv$g&(mo3<6tZ3+m^xsVv52m`_*`one=V*M&&Hdecl|*xTlWAL&$+aKkSz%ud>|I&F z`@KEKqq{fk{Z6)U&wZ~y`Dd~4|B%Z5Z>7ukPH$jqC26cuPsL9?NkjhcW5qx*^qM`} zs`lQx(d%)~?P|uFcpJ^J)JJ%>+ zq_En~ZoL)R>$y7Xa%1&aI7H{R96sM@b=Kv^^BX5(g14{RqyB$siI4pkmegqaXu+kW z4u!ETGV|OGuZhm`>PiuRnc;lK_(=Qbiqzd8J@4;H%aghGQ}@0wF5Lx1HJ-FQ zltLH&sdKIOb#9#-vUB7yi6#@ph{`98^Ie`N)OTD;Rf&7Zm@|(!YJ1le9gPT2Ansol z=VRu^3Cq)|ci~rR=Rm6O_xtESGDiGScq5x$dTnh~sqW0Fv$?Tw8@rLq4gyp7iO0#J z^z|%<0i6z8m%{eN!)~QxUIV^AbM@=BCI-}3PJCO|Rx9pLpDz~}Rj9q)-~MSTBg?#b zJkBGe^n^3C*W-_Uv2bqNU*F84&B_>#YaYGQxz@)GU7CqtF=otbj>pEXsK_}?*PW(T z3}QZD#NQS7p$}bp30w=2S#Ap7*Irs(18NFd$kEY%g4%SqX6UZ|NDW9t7ILcLE+zh> z-^_F6T0`CAA(76Yf$C|lrfha-nAbS+3;6M_wdI~pRU~59 z^1@%E`yP+q&Zks@9#9gz{ZLYulCVBstA@Ex{zov3uh(6a%6+YTo!F$WlaeDP`hwZ9 z!In_hyE~Sx;GDxtSL#wqBWC`zg_2X&*?R5PqloSO8N2&Q zqV*SB!3l-?xFx4Zp=xQQKjW;BLet9KIT0vIUTKXHv}8nx?4>iL!&~iUPF?qAz8UV< zPMDaq!t$CBx0^r?U@c`zS^&KsNqsPA)VEIHsK$DZ$t1nAg*7%mTd~`>8sD;isEgWu z19!wQJ=^)$ix7w1*w+&p{X@#}Q_5Lil`|<~@}tusqdme$5FEtSRwpMlEDia>M^ZHv zhiD=$@Sahw&Q#nP?a43=SFg%s{!(9-)i--X*9q!(QDcciEe}6;F;*Np;AwZ9;(B8n zaAwW);q+O3SDesZ@!@7p5fcs+pm2=Xj?N-gSLv{0{pDvvCv?Y#tLt#SRuAegxg3hT z>~o=xvU{!Im%=V*ss4!Iw6DMv%u25*+uzVRA$o$^jZP~ufNCSO&QpK5wxY$)byHxd zc^wzk9x7XjUO>GVdx^>*`p;|@rW$RIg)f}>RoruOfFF&jnrcsXO65TiG5u}5p4E1{ zb$0pOJuRGkDYCUHI0u=zt^fS!1&sWAIlz)`5-+_%FGN#n#BT^}sOEJ>aQ$dYW#ew; znZU47)3t&bqn#9E65VspwTx+d(Wq-=SHL+?%MB z8B-@_ukoVywbF-D5bC8J*+hATnNv;A;!~#KLJOt73Zea7bK>S6VNQwfwNiOzc(goQ zzas5(;{ZHmRo7i#lc$yG;uN5mDT$CR1H0(pUR>XdJo>t=+FP_%;Df44XLNfFnDtKs zGQ3uij)bvg3`1W>?QFx7I6)TmbEcu0=>F3LllLxw3o*k#JokHfw=jLfh5T^#`y=GC zJV1yb?PNQ2M`M;1h?y-7!!6bAZRz!xN~KN&C${o2_7cqK-w+~^oJ&JVR;YsDcpUMV8$+UxEfID04xAt78%ogR?qIevr@oi>pkLAM=q zBNeV^6Lqx&KM0a$FNY67H#;_4jl#%=-f^t{ts3+Eu!jsJl@u7XBvB2%1g# zB;G>4GbH8n=f3d^elW_czjXCP&iVVP)0K&RHH)0_#frNzO@ck_sIF9$YXGt|pac4j zNvs}XbPkow`nn6m!mjZU{l{kE`aDMja2F6NRgr#<1@Gx^rRoco;2#pWx*53Wa-rYrBy>8?$y zegr)zlxI(8yemvZl(i+Rx)*n5$3U37n~ z(*>8pyY?=kOVZ3z2*P*6oSi;n`c;xWD507PT~c*@2ENzh${$>89GME=xf2ZI3x>D) z_IkXn_aS*5Q;L7~Ycv1psDGy*Dq@QJBJ0pj&?8cR@@=D^QoJs&Sl^Ld^KTS@FJp}8 z;-7%{v8|qBXw7$QNUOHpFS3XgpMNuU5o#69yBSOij~yBSRK}&^Z~^!2`~*GPpRsv@ z9B#uc`z59#ip+B>1N5alu->aF>*^Sr)KoxnZ}z{F-wpE@sDOsxm+CU+rpi`?MV8%*iv3%me)IcBg1D+7_9+OmFM5WeeoVU zBw-o|J>{eJBX^iCWaQFU*4r& z>i+*P7*^DO^*glhnT={tFiL?y*WHSF^H1Dq{f-FaE~EbqPR8HJ20n^e{PUkEach|o zkPH2G_MiB4?`{9xt?RlT{S$WT|223#MgE2IKhdPu?^VE8PFA}A#IdApz7jj47Fw_$L@ zpEA0#U@=N8M%ysO9MTqYb-zHSHHQZ$aN@kJtLK@cE$)?dM#xd!Mw0y9ZB?fG|0$Qu z1|4Id9M-%UC&ivbX{;3};x@_}_nsF2Wfb2;^=Yl2XU>H^rgT2CLy#ViC5$j~2`wa1 z9A6+ubjel*zKQ%Vah{5BDUG9#Zwop=bE>j40v}}V;sqr78s2eP3g?@ib6Uuou5jJ$ zIXBq2{FUXujyk=L5$v_i1o>0=Xi=T{&@c`7OfrBrG(z9w|0$-B6n*n~oiArC@Ri4Z z9r{#hd&T_^7hl)x_PJbDXgmm~Pn*wh`r=`4;wY&_Ji=V0P+5}Db!01%9Dp)YB?#J# z9EOMqb2pTtQn>~qLuN&&O8Imj*SI4F{+7@X$T>94S~THqs`izXS6hyU*$*c7aJ=eA z1bTOMuW1CoX&h;*ewGeo8k*}X!oOOix%ZCdRr411;vjFy*QT-Wytz@678rg+%s^D9 zhyqWgQbe!1pwrS<5^Qwd9pYG&!Y64BB# z3fCL!aP%VbcvKniK?wvn8szdKYPG$PK^6%WLRU)N^NBJNr2MRQ4I#h zknY>4Y_@K=M@n@?R}PdV9~jVkY}U!yD{+}A5-+8bMDIvc7|%dhTL~QyRYs9t(Cer< zz;2q69t2!vQ%F{WkuLC&?0=b%$6UShX0hYbd-_|ZH*M@-a@&iZ zO0;2wcjFb+c?R$f;qgtUd>Nl9*`2A})9$0U!9S0O9J(m7M3mPJo^#K{zJwL9sDr;p z=YXrEexH*rOjg}-^veFk4fpf1eS_n8JR;O|MGU|{`P&8rht8rXuBXHD?8hcWm4^os zKJz-=|C1Hje5)#yy!vMF`|(WU^VwP{5cI7?A2&zquz|M9n;)GofMBNLc6vMSgo4!b z1p3$pw&8`1BhfvL-8SP!zMgS1yH8Q&pt~K}@r)9G-9;N(;l=63c1kMmtQ*+oFk>Gq zq(W&BVjrPO0J#NBTV9(kni0SG8odQ!V$^^mqlyqWp*BX)prNNAX20xR)BPEX5-cN1 ztUY9&{GTnjspndE+(`^9n>k6}G-2y@LtX6gw*7J;H?r9@mE@!-V++a=+Qd{HG!Js^ zXo_z2b;khzn2Ms$4FR_ep7HAl7B@8vrtH?v^Gtu8DQ;27Uu-#ZRxgD$GyXMN8$fZa zhR%-2|VQo`-)Opp^dF^6UZ$X;AxhCwZm;1k~%y{4bLA z7_E1f1Xre+VY~dR{0GLe65*M{E!d&xs9Pt|J!lTu;|eQ*(cc(WCqlSx>=6A{HEzU7 z(N@c7_%R-UHr0u#+)%9i7STzHB8L8Elb~^uV~)n`e3edkuS3edAbLw$*n5+afszUa zjjE|n)Qig5ApY~Ba%jo;lR0YZ(v8D}yXC`z<$cxLP2ZTW1He7ya&X1luFgTmH1^;h z^hla`Z*~(AZ7+P^;~6ws38+rHXxJV5RoKHnQ#V#b9Sd#ppI-3B0EuOpcUTNvFT&q- z`Q!GL+wBz0r4yTS;>?2W+-<)|l-7Ro&%#ep2aMPr!78Ev*D?%!sMF%)H|x#L4(h!qMlcrvD%~5k#|R`#b>Ay8ROlL#IJpJ{mqO^-qq9Eh z*%rMymByl`=a_Nw8M_=hBA4U@!f^zj-9q=l?bA=ZVcJj^yc??-8gn;B@>XG>C zj)~d?g4tL)9QCEwWN{hqr+7|6e@7 zR7v}Qa{1XK$o(#l6_~K6)N`_^*#;<-l_OL&4TX&2PR&%_=%4(+mt_igVLzHgFN3CK}o z6jfMGfTw|ua=oL~TRh5nn%yi&1(YjZ%15~Uh6K7$&&;OjSd*9pS zWYMbDXZtg^$OfzZgRJsyE(-r--k@I&lyDi1VSw@~N{_{!GLPfjDmsd^KStE2d%Z6) z_y+_7BwM9x!z@x~K+6X?Bip51y5-t&$JaTa6PeP?a5t6mpmXFR%2SUXiHj)0;l1l7 zd0)|$Gz+!8HN`)+79pU>&s^r$9X{mnV~_=hy!)3#NXgN`Gid&`>6*)eAXl(!TabTs zHV;2LEf8S!Ecs=TrO%|QU?qbde+V@V>|a|ALkLT%#1mEAkn%vRQM&IORcKL6cchHQ zFV0fa7pKPuQ=gHwbZl6$gN*UX9{%kzjp*k!I#9BYVSKJfsCuZ2Qyq4(cqlt0vESk! zXb|ubIVjW5*4=D$UWRLSykjo(^TjnDyD!(Qo9sbf@`8NvU`~mi&apbIu={kbuVwvt zW@&JvK1c5T&XlQkLy_)S@@L*IUXj7itZjN+DZ8l71oO_31`ufWTpSy%q5BviP15Jyd zN#&ZpKPfb9q21ze4fZhoihoqE$wQLTGqKgx3I%fh<(6d01%5=K&$mHv;A^h;?v&l4 zN;r8x6jpCJ2_o`~PK{lc_no}g=!D$bPK)x+-onu4dHD;X>;gbs^Nb!&K6@lJRz$r^6(1IZ6`NF5W4zE4?IjnembL;(T8>hhY9gZ7y@1GiT}h%7*_dEZ zN295|FKk`8%Jp1!NFi5v2dXW)j%=cMz8uK|j6%+@=^w!c;Er`Rus^+jqxQ@7b;P9i zrDi0PoyD2Id6ZlUY_w$fze{-Kkt&!{+)9>5xW89sl6>#di9$u!=YtCgbr zL%Yk%I~q||qwuCG7-Q%;rUKr`7F=W$=jHc>du6NVHBTVQNDowqsK7&(xbBR{^|$o0j2!DLZ8Bu$vezdUjkR*~Vf+ zYFKg-!N3m+R{S)c11wdeeR-+Dpl1L>2)@JgzapA)$|#+0d|3{iq;im-3yF@<+oSbc zKTuTu5g&3WlLl7VhT-gkwQq^WQBSLOQVRU-=xzAktP8JA9UPMPm+i|&!kofydjv9UUQ7Z5ZEF^vr;n%h>DNDo8s zXjpE;c=jcTmhtmH7%F3TG6%p_z>3`Ig zerjXb-g_m+|L!$!YC3F`Ts-Zn2haYkwxYT7%tnh%Y^T!>TibiOV5*K|-SDhh!xJ|n zINlQw&a&^^AIb2G=(MWY8IU~W$KuqYQv>w^-1oiU-_&7|+w^?-F5&LCnwqAJNOhzT zlEx(cW^|(uCynYbm+_9MZhU5H)9=OS2s+>h_n` zbu&GQHEmJ7ez5Qx*n0E@+!UR8yei9(vB8GG=rbZymmLk{lBiN7e%0QRJXHNAFGJ;h zroY~$?5M|E(4^`yN>hTdV|LBczoQ1Ay;tn;`2*tY^A8rDcz(@h@h`5E zS3S`^@kIG9<-}L6rM}LVqu?jyQ9Wx3NieVL8beb>DqjPL6-{YT2yn5 zC3H4iWM(bP4Om~iBcE1hVd&_qF;mPywBC>QhQ#pP!2yk)ukk^r)N0(YOLp-N)+Mc)uyVQaES`UgaA;%`J&hg1s_=8UBsC%9&wxKQ0BA z>_x3nmrp^QVfc~?ti@Z|R|q zXdnQ+lyLNIsqsKUDE2Gz6@JYjYa2#^@FPl$|KF9?akxxL_-j zLhrsZf#}_`OOC-RoJbAZHS{^z1IS2A(z)LCN+WOS?v>pjA3KPsFtF2wScfufPp<|y zZF2Sn50TC|^>Geq_?*biBk3bpo|25Lu*kf9od%s^2Y-1hj#nI=}WmYf|0=n^o zeTIKY&9=4a6esbM(S>p*?{cVkBg-T(_iDyP#xehEqsfyJyPBXuhk}z965DL(LGTUd z!GvZasU-hM;4i7snZKR6s;}Q&G@S0Y@p)qJ|=b5Ft`R3#fo2 zB{+_tQX-;)QX(zXBnq0Kl*j-H5J*5ufDi%+NeCgyd*Sok_c+h(_v86~e8=&AKhf*B z_P+LBd#!bz>)b1iI$`1`$6;4v(=hR#Q|%7N=#OCR+DY4^6R5LfCY@D~K$e`o)?L4f?gN zRNn?gdCpAWTOYvt+I6`@_SPaw0U5~Ow2tX-In~GTs+Veu&$rYDUvX0$BNixdC;p=+IQ~x!gM* zeQ(NDv-z8L)Ar1J(h#TJd|^jG>K!kLG>?lzvaZPXq`1Mk_a(|!%a%0KHS{Fw_a2(F zx%)D%KVVFu%}=8m!5zC0e9#J`J}7nz!f@B!%qb&3E45bYG`-i;bH$8>DG68n3m(+k zp2_@WmT{V8b014s;Q7b!iv8H;m&mtk7zaFAOR|BcTusM6LU3a7XVdWIijl5^W;gWe zo`tWuX^7u5-{Xn8PeWhHWjurI$*6EW`Z;RLt3A7<&`8^kOpA|qcWjrvQyxCv zNXxN8Jy5o}>~Gag#ZXbA`X~cm&N(RF4DdR~HrA@ig%=vgA2SG?soLDp@$_{wC=TIS)>f=g|mHVYm{a_72qL`Q1$=B zdw5`XAoM`qP5e7I>dIwjE(9U(hp41*jY?7B9Up8>Bq!8gOjZo0tN0FQ#~m+k79AB& zX|*qz>-%2)V+L8Jq9%NAZa9SSScX?qq^h}LPG|Z)yjA7=T0ojy_|SC26QIH!NdfZqhFXD&;YiFu+77!g<=4M9>e>n7sTzE?Rh-RBrA%mfP>h3Q@iwwui z3oh@T8#_g5S%Ix>A;Md}xZREkE`D2Q+HV!%s?I;Wpi1+I9~`4mp3|K;r6}5oye&!d z?sd4$nyK*>6P3upNsQ}KNX;Idi>$-H=ZW3tAd~Jrc$^?(OxqgdNm}YBE``5pV|t3Z z(8yP`hak)MzJ#imHQK_i53KxGokGFnTG0{fhkBxxnEQvsuh`|c-q_&n0`gCuP;;~ACym}Wg}*k9V(G|@lwFKale=LCSgl%{^Cw|<7xS9XxKM3n z{p}AwE_n@pMkEho=v~3i0!(P)gE6>fp~z1i`35O!HcAN^?$zux$-kRRD?ZIZ0IBnIaOS&lyqG-yT82y{u{7rA9>eynJWcnjGJ;1V7$OGJs&`9 z`M<-D+rqFMKeOr>^W|%Ew&mnm8yj!2Euv9epxtX5c%^Y%$*oqg%h&!42qK_bK>4P4rC`5`his=5ZP-Pb{N0u*Cr4E;Zx`En6>H1C^+~7rH#bkin-ODIIMt?!?t6j3 z?US07N^o)|D;ax@y+hW07*zK=aB3m@_@w$D+VSX3;~VB;be@o(@9J_8?;?ToQQms7 z3EkiU^+3?UtUB?7ncBPT9erq;w9vDpHL6I9H)I*6JvMCNo~u$94~XRU6-6KroGS$3 z8aZ!Rjltdx{DCKA{kOS)RGJdNYCrmTs$VK>MvL_+z^tCCZF&P8VZf z8S7PUaY|e(k1$@Bt$Xsv47DZS{>1*U}5U;N~KoD_oWgoC2y}NJid8U~5$=9%M2w~~`1W>Ea)-b(z-I$^q5!h~D))&zy)@Wa2_#Dz*NvVs1dDUF>{OpZ`6L#enjnCZGX1aTZDk)zsKR1w>tj$k zex5yfomI9RfToH6#8N}@U^$e-N@)}OmF$THKguE z1uCV@^eyg&)JG36T}e{h)9dXc5nCLTEv<(i9)GEJJ#5DgNW72}E1M$|EaCU#XUR==TSHdrrpL*cdtnYOU0{s`IL6#2x$qt++1%4-fmm5%*_4p2=_#{>x=~jvXwd+cuZHOFtr1( z3rgoCAF_NHxjj}%)4188H$bi}88|u(iqr|4I|o|yc?8bYbyXh%ebt^)7JZ8O$lOjf zM^;!-=Ux|mTUaV7dhtc;#4}v521L;jop1vqyeLu4Jp9)qMnYNMNW6BPck3V852Dh? zkz?hGH~XxIOv*`IrjBKtJ}FS*+dKo={V4TR`53n!Gz^}a1?OR01y!(85~2^^6PANa zt7J1#o*I>OuDSbI7HhbWaz4|C5YQKJMdmUhb)K*hyCj>&SxDVHQql%PJ`H9Z! z#=~*?2Rt;dLlVwq7GCMJ*V@~S4K{w`D==Np6kXTV$#?9ua7bdhA}2+S!%!9N^uU?Ft{cA9HeS(UGsw{V{=SPBO%qQ`9VRc#tP{|U4i(Ty zkW9ujNTG*1d+-p%I0$*6Xb{%XM<=|4oUIrTjx8lTUOvn`K=uC(g)_kmAe8GD^X;h` z6`kbOKE&HjKot_b0SRta`wgGSmWpL1Z@rx%vq%qNe(Ukp-Ej!^Ba|6&uY7I55@n9H z(M11C`@I97d!uAC5*1p#=hZL7VTaBO=|KfKl?;REbhq|~Lh8{fU)RIIMkwJGUCc6; zS!ulX%RDcf-ML`JELH5o47le7Uew?k#Nd2SiNEl#bpmtU6Fp}*i`;bB`0F$yLS4Z* zq_Xap3DV~8ZoA&>5qZSH=ivf7^Q33vwP0JiDry#^hOZR|uaAo%{5+dWEgvf0ek1$G ztHn(j-HvKk$YK+!U37YY*9}Xfxhzguj#X*~qIrY7=(F`^9Q--I**g}#(pB$Qk`MzD z^KYEjg{@BOc*Xfv!O!H5ywW}=oN4)jD>DF`8Eg}Jz24ML{@*_3@y!uwEzuxS-mnhsE_uv@ zcdcpgEfzi`$P==dX|=vFp^;vlR{cI*{V%wWydZCEU;4Iky(@v_5dBjyWnF`I9T3;P zG0$|$#EeqOUk`QShOSKRE>jNW+CL=$_q?m@rtDJ1@zDytMcs|F$sVsOD){F%yg93t zBc@UgV`B21FvGs81JOg!i`J&UrMiM3u@2h~k*e4Ap$nFe%skz--)ZasQSDWsScOb9 zdNXMTl}+n~k?`$az@NXxJ5nIDm-qmgh(y736B7z9k2W<6HcVa}Sl-_{Rt#QrkE&?r z%?$kksSz3u((9mQoVR%HiUl1eqFrLEuxi;}T@X8AuT+cP$d-JRdXWg$DfBQ3_!18b zRO!VUYX2~7AeKl*MN3ba@#Zs^Z)Kexxp9mCu5;z{3VxU+AA(*vtlSJMiWn#j48YM( zh&p%vU@3-7IOn(BU^g)`g-an@M&3XA1T9maGZOz5{|H>U)`um^D7k!Jy%Dx|Fq|2X zTzEbMT%#rK^6Bs9TdwtmXc%P{+xF#8y~c1Gh}NFtB7Mw3F2ZDv!-k(O&@fXR@5>XK zXef{CaTV?bofAmcWWYgaI|jc2i%EM_WNqu3o55~0DV%bgi%tHA>H7njmU8MtzWg7{ zH#7j}&UHe%5T0o5ljHPV_rX8etZ-j}qyT%Tfgtl|Yc2jh+Cai&oF$t}Zsdkqjb+*} zo%SPg{zPmjXl9*DFmhSX4L{`8)L6c3a423SchLEdsGP@9`?I6Y8$hOPLNo0%s_T%@ zrTzw;Hy>cQIjBxY(}qpsNdv%$6d%I253ZBo9Suae#|<16-yCK|=XM}%O57G~EH{3{ zGbc8w8K3K%TJEOdrh63Xo)Fz5&fhdnB|Xzo>cd-xND0W;MQZEpSyZfYStt6~XA~<; z3w3I+cIp^DYIe~&;ESdU`!)ZwM2-WDr<5Cf=R%-;?KZNrk8_gyuRcOhaZ}$U5bovW zP5`Q^toG|Zl)`%5mK(9LTESjP&HIG#wWF~L+-^yvKy?B(ZBSV2t;Y$odz;OIa$pbM zHH1Je=*TyarTwO+aNIa&dYQL|4p_YPNhw5&uTDxkD4p*E90?F(faYfRKt&*t6Kna7U`-~stg<0$yYbdva zBOP(j`TkcTebRF(GQJsyPZS1y1xJQp=)P~>I<$4X^yy`(wBBXi8#i6TorVeQk;EJ% zW=^e5MQorBDa7`jcvlPG6>BrFo>4_+wCzzXvCShxan^@l`hJEP z$s-!DDWW_N)bJ3ojS%+;==Jg9yDv7*GCL6}@|xCG8+sxMSyR8l4pUQa5A#@q9U_9+ zUiS$QLw*4$`dx#hUgt_XA1kc;2=(`>KxDiGn{>KrR)1FcK^m7Ry#Bof2~n6}pwRoe z*9c=&Vs~Mk)`SeFD2_*L!(%V5=hC7VVUMEZ5iFy$p*a|6#^O`c!q%AdvsN=s;d@sE z+6=tQROiQG#?!hCttNdJ*K3lrR8#g7?JJ+T~x86rLcnGv6lFXR7pHCnuI zdEtQt-Q$UPivTMDn+;6rQKJiyD7>~_lC=u}j%xc^jufNI50sYqUa*6ZI}4VE!4JzW zxatsds;bUo6f#`k?$AN1dRl1vtK0OrC-IA!tC=%rOXL|J!ijiwX0nxXZv6Q_$QqNC zYRo?`NzPh;n-3zib4kKO2;UlFno5)p6u;V$_8TIN+B+dt$%t&TVi#i>(xToYaplrv zXWg9!_KwCkj<#GhTp1$zVS7e91KF}ivbTA?=CgbgW_9i8;tt#FE~$;=Si+W9{`*5^ z?ZutD|132k^J;~a?*T9@qOZ(Ic(2#T>opMg2yMtb`hk5PO~>+U3Q5a0Nz zmPeg^6z>rm8Y51hUO$Ip;!HqowGUP~hGC*URk66&y) zQ>hEzbirTdP*2-p*9ubI$*v=M0MNcFC0GM)OKdhv0uN{eqSV2^RN=#O3k_|SW0GPy zqRE`iGik5hdOjh|V^_c#icZu?tAC$+m6;=Qg(-{`)<&wjL>OYR|Yb zs6m*bBJqchqR+%SNqyMs2Z~K}c5P0xjUiPTcNMtl>TF42fw$50Dpcx)1?Z2f^g%f~ zEvWoOtY+xR0^`%n5GY>npS#DLsZuJ5n5y58%pFox(Kud-Re1dC_CG)A!ndA5;mPZ` z@=-YD&cMGm2Np^0ja7CzRy?pFOsxWU%&sPeAhRA z%M$E~T=KvY2ZYuDtI=2#l}G{%0|B#&r6<)MMUCoPFy;R;cg^Lb|Jw_F%`yQN@o#z{ z_K>-ahF|t-T;ThRSIjvlwOOf%j zcgxc`F=Wo`1f1cewUBA8R<9#n8^*(4nld#0X%nN3LmL+w_e`Ht_>y!ZT&14^pw*|B z1dRUTL8{V&TDs~4kIV5q*cMs0Fu7Mp$1)pWB1(+1e;jQXa**ot3sk;Zg?**d+_D1Y zH|>M1^Ujq0@}Q%m1OJ7w{^d2k!o?Z`)>G`ZjobCO6`TMMuLgrQ?-PB_InMR~c6Khb z(HSUM@v>ZKHq+Vu8%pbJ5qb^V#0dY>v@OiWExPiU)PqWBWNCPut(8WNBwS5Ybb#>S zs}0w_#+6RhaTBR#_XBaJogEj|1?ZIy>pk6)4O2vW&zC*oy*b9T*rl!&}PY% zosP8)8Kdv>E-eiqn&)EKfsDvmj}+p-XQgpHi1@%;fwm#KhaDP)D&>9b-ZB(RcK^id z@Wy;K{pKK>nc6I7Yt-1#`?Kn$o#fm9+(Q>1rJ;yJx7{Vx>fa#)JdQvALGg->Q&WIT zXcqiU0taYl<3#B5RMJ@`pO>xsdm$lEDugjuj3`hmtIaQGp5cfZas2jlWje6Re_5r3 z=EjFB1^a9ozo_=QlYHkNgKq)IR_;fS+|}E@e?!4ltc6am{X62V+S>oqqojUc0@}*AKSW z)p>tPQI7jHVy@c$Dy1OW*0v#RyL@8Y|G!_`+}OCnBIlrVY`tyIY2$sQOUq*`G9A1+ z4ZjcOudP>!Z|)65^1Ca&6`?c$o_q^@?TY=!|E8SUAmlYn2TD2T=R`T#2&m3Ykd zZknpxxcw>!uXxq&m{s2xyI;WOzFEIO17i-i6;RJt^yM=b#7q>waa*gs8{Xt!7<*pk!K+Ui(7B2^W+vT1N7!fDCl*`n;0vitGN}_kk$`Co`o@^YFR|r zYmV1h%{J1BD`ROT@xz#n6Lp3c-WwI@Fm$SyyWS&gwl=w1)rGxYW&Q7c`(Y_8?%9;w zb==~|NlQczs=wH`pVjeUginau^O?o`)ezAYnWsFi(>b<#w=E+N-*Xpz8(J64G)-Y0 zksF!YIDWhS?wVt^tF#XQf=FxM1C6UVrIzas-8#Hbtv3-l!wkbR^RHCWrDGF3cePKJ z$BPs#golMRX^*!5n}ejENf$2IvACli?eW@Ny_02cJVT3ahko4h-t35K4l;MrNDR$x zmgc@AXgtU83^RRr<|UE0NC|sUxkKbauB2h0~(4Ai%aL7d_KaBm>#@w?6K zedIrP9)G;qP1~AU<#u;Uu%m(3 zuOwYu+I{3o-|;WcWv70}fsS{HEAcGv^1$mA8%kyRi&SVkv`jMY!}z&Ud&YaSc5Gl% zsBE&~YF(6VmYH^%;2=IOh_*KLm66PG>}=xsIvB!NjaP$3Q$JVyOtE<0wVf9cWJ5X9 zh=Y{?3tO=MTj!+M=fBItw0?9%Ftj@hu_s%xLz^u^D@0iT(4_&-G*dp`gAI@dK z%1pq#H>M}UUM{Skvm1g1t^_c`%Pog0+e|8U@`i|mLkZ!{`)qSTYfplV1(CFz-{wU7 z(nC3FuYs~i;PG$WrG3QZ^|@QiKVvpqvUNKY*OuSt+C1*+MZ&ihmJMR20=Fpd+Ix4% zf)~-kusqF}lZ$LK9gb8iA2a#o)|?=$Izk-v`^Zgg$6=?4Lz4@4!KIm3=le0F_j^&U zCfb2MelEj<0$oy*6q-zO@^U3~H3U3}C$%Bw-dC0oN*-fXn68}WotRA`%ZKEH;ur+8 zx}VUbR79ht0LkshTZU}+STWB*%2g8v%wazid_b@VgP^CEoY4!IqBPjlfnu&)1@sP z_tjF0IsT}UpmEBAVD?XAwJx>>grTbCEEd+T0u^s#hvAs==H~e;7f8XdSPdi&eHn(AHW!?& zN(h0b69N`3ZJ<0Com|0!`Y=7}<<~Opw@ckco%nHS$m3m)dW)9d{)6%+-3e;zQr3#D zF~Yqa_FE0yZXH17fc$j268ien@R#a?4iM`)X?dwCe(kk6WQbSEtE0ClG0=UMz5Kdz zk{=oTHd0${tek1e-_xbVREQ>)Z(U+a6y$5*zuT*prK65X^{+WMzn^XJO7^grBz?i(Zd-;R}& zQ{K)nP`*DenA?M~$)xUVlSxJ3%u2e%WB68lbJ8JSEUU0s_$wXJNF;Yrhk3b?tGh8RzO*o=S!|S@op6R6cD!Gp8VG30jN^_ zdP4uoJa0nyMP#G7oAz+cRMq*;t*6uEB1A#iV00>m7D@22l|3aLxhxPxi&jcQn@p5o z+B(|!SCK%51s=|ATssy2HAlu~NJ2e8+EujJ0x`i|Qz zSuV1TgY1!bEzm*TDB|`4lD0GyRles&DH-a0&B~&9tn$lTR3ExB!-NyB#g8l>L!`T+ zj%T`-PhfrE`k$UQPk}WsMyWF%muZMG-k{tMY85~vXxjU&DONUyB-f%KONe!7w~~;Q z-tH^Co4o^nyonw3(y5-a%%Qb~y`S78^IeqfEp=$KxeqpKi8A3h9?YVmI>F1MV4OA_ zi8C}=3HxYLiiM+c12B9FSx^s`+Nh5Sh-UZUYyMCx1x95a`*t&cyIWh1oSM?VJ(8?+ ze^~IG7zqb)=i(5KP1{Et29-No6+7xk8Hty%MMUhn6`IGa(G=ndO%3#C+{#Khes2Oj zfJWB1xZn+=i#??wQtM2TCbGLg2sfXM>|xNC?m;^v6W4twnz;DT=b|HP?EUu6W+)V8 zurGm#T78s+`vO;*u@OpAd?6%2H>9?@lv58Pj#v-_P_>G}Vi}?Atmx+t%?#)yo7)iKBOzqoF+ z;izm(&KKD8rs$hEMig+sI&QE=r#57Jy)%4h@G;X0+LS(OUO<0zkJ%E{W*GIwkxJA+ zMs!(_&U_sU(@zQ6aQe!Bj+WjxLDA~nVC8MnLce~vF|=4*j-o@Cu+vM3EVL+^Z?!VH=Sgu>b znLMD`ERW@ylQi0{l>z6l03gr)#$-Sp*JP@!rGq=iax~P$wp<)WA2<2!iq-5!9J=~u3gTHj*?iMV;4zDE0v{AZHcZFpzm(WXuf$)i6o70()BCnG z`}gCttoodxmmy848SiTC2l2ZB(OT!Yx(>7kr8aNR7q{tSeFFq)<||_$D3$0`S^y;3Db#3`L%CuL2I4Cw!S0Knj1T?2CN)p-DpnK&JAkW|sjM`S2eHu+BZF!AfcZ5PV87jDh@HmgBbv1@_7yS9aB< zdGk8%+fjWW9gaTu|JH5)FQUQE{ytIwQ0fH6F9ll*XX2Wy&&di0*mkDkRnX}ylZ`PM z>CpI5+0=9=)hQihBRl6*L2C-E8PMCv9{#6 z>1=N|U&|8g9$M^2tJu23uG$#RSZpBEysQwJo(`*EP5SKu0pN#WniX|x_5F0sY~2eR zg1zd!QB2yz@l3DFa|Su={Ow8>u*S%D3mV6tHxOCpPQ4^an*)8|@Gd7G#HsI2m-V!EBY|=wq(@QV1GZZK7|kAA%Z}IBJO04#=N%hI z7d3+;p4LlKmapi-B!2Ch`(oc!c`J9LwZiZ5f5|)!XP26?YvCkV4Y>28*VvSETQ4|v zkUnJ(bMNpfQ1P-7<3GU0N}0_9^d!$UbIe?Z9H5kWI7Y>NP%Xy3X1`}$;m|BVQF}2O zh&c~-yGf0&*R@=*K^55vDfWz`)jc3J+DJcC;S)JX<%_A=P%lnx@{8v;ct0*oc#AYU} zpNYI4M_D)^@3(8V9kH>SnqdSo!3I_I#Z1A&S%gpkHo+!)%d`4RD%ED-e<37ltnD*o?=(5M&!XxlTY;^Y7)u<@aK$2% zzm~h~zD>&1%G)iydAAx8-ZJMP3jRNCTWm4h^%4@JbgPFLCd3#8Jg8I%AL{bka9q~D9iVJt$o3_}<4Gn@LtvX88lCX+wN<++SV*U04OGwo9+0x8ZqlC1) z9`Kb6tx(nA#jrc?X|~IG)%H{_$hr)AFCqOp&_8Nj>fMz5#@OWS+aft%?bIFhsKS83os9B+4H*=>r)l0RHv zVvmj1?Jr`w7|UPys0QxbSmS@4x=Cgh&=!sSxW*d_&S>=_w=G@rI>o5MwTVInzo3$>iMwN+C@km9_-h5Fozg+J$zwQ{WJqOr zGdIG@=wo_->i}eS+rQx|S4+poA%|9~EW}UtPQ>^tdhPoZ}rk0=(LheOu4d)0e& zMffQ9WmjqE2IrTJT~B9x={ke)LMz=AHe|<_*w#saB~(w1>F}-_G4F`jlIaYsu8-2? z=*Ykt2t1Toq9kA~z%Yf&wcQ-DXBzJ~9)9l9dC$5X@#o-KkbX|}!qO%{Tq}12`7I&< zD*h4mDq9KGr(7Cpw=3;NL8F2-2e%t+p7TgEimdAs^Vf=gSATFFr10F|tIxJ;Fvv@F zS{RhcxYqH}+U@E!Trqe?Pzg2T`{%q`Sbc@vqifd+k_x}&|XblRGw2|F*XV%vIH)b<4AAJH)>x*Yj9>4St z{H7$*kZ_jtF3mTu#{d5>nDn7?1El1>Y-0dW6vJ!zX$!w~ z;LP+{>tpl<)u}LI{6u3!Nb&esa_AXBWV0M?{L$fFj@IUniThWAnlR+Ih>iHI2n|J>lvE>(@y<{?|!V!&}llQ zYe=lm#1!Q8kQ4R@4gZqB9#hl)Qr6*nXRsIhuRVdO7m?Pzt=1^-?1A83Z}-aoQmZIu zX5^XE`t0gyrp?k>_052qIvsRP=vVE&*u z1NB0pQ0s^aD>Q7*PiU_7&_NqaN!QOs_gD)dg394{+koFbdqX{%a89l{eRMEno~0~z zrT0~1GhS~<+Bd#PKZ|1w-PRQu8<0%fZGn_-lwL3e*_3@UcQTeR1_FP=o}C~_hZv6S zqE2;cJ|b44-CPmy=hutwq$Wq%6?;~ z(cjx(IhbN?H*QOBFsw5k9JI4n8#_2aYomIGIejvMjol|S2%HVxHTDp*J}gdP zGDs?A2J;O`#mldVQ$3>y;0I4-d;TEnoPjz`&kBB5m_CFIKONxUzUnB!yCF@tlw zk)>2tl|?owLXwG?;t`|Im5YuRw>6lCYi3Y0E@^5YR>WW|@n#L8>_I`AEl1;0>_MhL(;5w$5%1yEEbvv1ElZd@bxnFz==3||Ax$w>-i z*K_lv6s7l>rYzx`@N-;_y#%-i*rinpMuHkQRBo}*`>=7p%8_1UVsLrxuC-g?%WkYI z$b!T6Z{8;S%(o@1qgi14ra&SBXyb} zCaEJLc#M~0!@ibs&}|A>58xmin?T@DZgVBoy=&Mde$JHg@)}{XkL7TLh+G`jsSahR z6}N5c;L*Y5O~#BOHZf19e7>2YG>(=^L9&aIn;D_i(#IMcJB2Mz9e|~Ikmhk5_wn}> zf<0`Tw68m21c_sjj??G+Mnu-_nOLy=Tac>tB4)6 z3Lm*dBm0Pkg@nuNw+39lcIPLj+}F5kUIPJq{O2I@AhN0A5p9lK*ZQC;e`a|+`OFKV z2TR!Kkmbj2;;X4vLnY~lgZMUKnJ}4T>38y;i80zzdaWBc>=Jvnt9+n4dnK=gbTMB{ zStuP0K^4=977h-lgD~Ydf`w9GIft*lNU36*qngAf#F!bAdf{-)*2S%T6Cb66n2(5$ z-~+M>>_z6sXC0a+NZnQ`IM1sUu(*zHNJhNVjNrUmEa(0h!0 zS&l478zN`2R`j%5#mM!kd!#OB=U`GKL}FH=AD&M8@EfX6)r$Rn^oh66a3dxek>g0FcV{MSn z=8k?e8~X^vmyA8XkT!NC$2y-S?6-lu^V+#~%MkR$4kwM;(5#P=%2kX|o0(z5vB)FG< ze)Wo%fD23im5&%t@7``x@njffS-h8tZxE!lEp@!<_qy1C_SB2@oixuIgJY;q>$0%q!YOe2++Kbq}`PDs`VPNI@;+F}yMY zmDa8;{>vA6h~EIK^T%f^hAVN0`2Fa@*@&F@jslU%RQZb^Vr*3Br0VI&%fG3Ywxixv zs**0@Gk1dNC{m6NJ?s^#$h%z^<}_?%*xRtvdUtS!XNOS?1}d_woStvqtu&qnm7SNQ z`tohtu`31EpHKYP_Ak%B@!X6JGkRjM^VD{_f#++nB%Ed`3jQ6ZFO2kJ0v6lD~NX>Z7HZ&=laV;Sh=}sMnEtM&}}= zqCJup17H*@Ea?d!HTKGLwCk1KWRY{2Q#8}R#MUDnK2m7}5IDLRL%yv)s_AfNM_Rk*o3oTt!y_4`%Uta*$HO6O)i?FB6^JK_8NwC zVb^ef#M}YA3iIft+;VEz=sDnb2YXXyxHB8~sPu?W?z(2q+0!X3J@vM_FS3vRle)rg z+NZE=@hhmGhW{~r(lzszOT~6UvXzDMDbh^)W_cYAmg(V?XBqyn134>A`^d>sF*}L8 zM+1>(TSJ#uWv5lFT4VvrH1#JdFNPNCK9b*y5@WJvJMrXDaQ)NZj-_&rMo&U_{7>B|w)@Z?!5`3NLL8 zQ3!u~@W_Vfyw=krZmZDL>RV_Up4%8F1=*~UbNmh}&(e*C|@^fy}-H#0a(&h~1V_Y-CvqR-;o43L$ zOy%Dm1nQ`z!jP4Y4RS*rk5@~>hTk$r-!JF=FL>eqRh96pk91dprCOB8MMwSsK@|yRE&6%AOx?l1~_0&pli3tgAf2Zdff9 zhyJ5LP*K_rY4epF>RfnW;(E7YF4Is)csl|R#Bw?#R&7Z6TN?uC4Hlq9p9{%Z)4US< zGt$b}<+$Y2mT}QXbvYPt1p~Ms`VWR0h+rTjK5k`1=IAX>T8U1alRVE5n#(mu+5_*N zKlBf#yu={a&Z=4(a^5ftR9D)%B3JD!YU0byM*^Z2xwqB-P~lWI9+I_|(U5OVerVIN zi@KQ1x;R2tr0|hHxiPd12|MKysgUoLiToG5=9NMIa)z>sKEiPwTsv-~G>}^y4BV+} z)@oh--l%WC_x~;h;2*^X0JvXiO>IwIwYG|ars8ML7X z8K>J+1}0zVlMWvt!g5lJ$Cej(Fe03Jhy}jBzQzBB^$sRznuU7Cj-dfNibTkTN7H%@(4$Phl-{sRfk0oq9j@hyirW5?(y*T4e z+J^{4AadzLIC}mFR0HWpkfqQYk5ltYQ+1AFqwPB*O5U-Ah~*0ST2b$J-%d@M z3=4!W6(4=?d{2|MJ1M`PsCcK3{HQGk80YVSue6R%tA@ionu$B{hh?3qD*(^ zz^%=O=f|CeJH=k4NTf|0{A4=S%g&-MIDgzDJ(5+a#yXf;xiY=S0y*GxbEvvY{W`Z9 zNB_@byjp1LzenE#TUShoqn{}I@wD6M)u(p+>b*``YMTJvmcQKfJITp=0bM3-Ek69v zfzu|$Le;2ryINj@kqRsNm?v37N9b9&hY%=BxVprC5*5^>hfYbW>N6tM!rm@GDI*g`9Fw!F})2&M_ z&z@191!(6+Xnl$wBo$K8#sWu-%(D(wt1ywjL`iVa9lnwgCa!uBFLfF#L4jh$jwUL* zq$%diu)XB zir8C8E4$-9y}2=(?D=ClgzvgO(Cv-z`nvniK*o1`+p@@CLN5wf{6gYB=xEI@r5F}f z`JfNQyqxqNE?tt3w;E+8%$<8}F(ax4;NVRpwP#P2VmUaESMO5G!r6{Drl4GLwv#MS z=2eF9oGxtE3N>sMc+W6IU6M*<=hKF3?~(O`zgs?23qkC8I=WHhmpB1TE$h#B*Iuco z?(lnkqf=?Tyy3rQmJ-jBQQys4NGmrUdww_^;Zty?=7wrJjlEcb!Of9Qcc&4>OwT%htQW>Ue}`Iw93X~J`&8#wvJPw zIq;Cjgq0r)ZDluURkF|ZeNRqKOEKRq)Bzjlxd(A}mv&5Hw&G6{j8w~=H(Pv6+y*dI z>F&hXtKmip4Ynw57c4NObtW<fReI{MDucB8 zR<<~k?JhwMwYqDaGvJk?auAB=W7vW8{t2+cu0Ojd;HSi+d_x`hpof^sbheQ#4Z)l@ za48Fa{(^2fikf$#<2)_bhG~Vy0;@XWHWPS!@zFlVEWDuj*6AijxGgR3W?M0v zc6Z_N@!GuGPn!0TUeF!_KWlMz`?bi)iBR+L6Wy&*laOaj~PI4`#kYm}(nT*pJ8{=-iId}IJe zlDN=0;~@o8>*85MVj{IA^}FmruUF&xc^H;+qE-LcjD$};vf&y4b5JYAM}oCqcK~gZ znW;f00=s?z+3TA4AU+7JPD+SwR0{hqs$&wUllBA!P?y1HP(TE%*cX@2o;b`Wz93Z2Px601x8C7l%8Rdj9k1;+@A+P+z6mS0a zJsoDs7^{v=GUtcOXQVf;#*c60$MVY72v)7{47!-ei)zC6Gh0R*iRMKBKt|*9Uj(Cy zMy?ndIZr294O*l=c9=Qyh*gFfwqwVclJh9hJGS{}`Nulq)L&J$8y!HOV-a*g@Qslc ze(aS&H?9enGX(jS>o~zy&$LkPYY%t|FdI#<%r2*5FsBX;xYOx>2`VNV_9UHiTNb`| zYjWWJU(&;Et&-MVH1YVREv*<)eekr5?>tPu-mbssv4nSZ6!Ze@xN|M1?Yy5#_~Ae^ zMF3ew%9CtAtn9MvM$gFsS-F`Ojsy8hf zH!lu+*7teoX9MJFrtnblx%@laA*qR-G_qs58l|rYMu2mA>!Lvfq~nHtx{IR*dN1HN z8u@x%PG`>$(3>Bi!NDgpiOiG%-PCFznvB!_i-URwb3OKSqtAu&(ZR^whGt>0zhcgH$L*OjG(pgCG3wr6js3fL3)b%vnQ~mk1BXKb|pHCEmj80cKf2MAWF2XiI!p{F5 zT>J%k4kSS z-x@YY&(s|!ME!6nJ$NfK%pj{#>aP@EA{bwf0Z}}vNBQS81uu+(5v+2W=xHxzYtv-N zZ1m^CU}}79<~v8*f5e7IYfb%lK6c!Ibwn$X$L-8!UfGh(yr2kWvspXVI_N_?g$9cH zX|K?!ov;^8YOLr3d0p@Htn7<*xv1Zz>nYl}XBM<33K%CCpvADYBPU$rzMuqG)4z*F zFP2yS1JoOW$@)~KcPi)Fflsej$}353Xy}bkL$2%r&(seNelfYiy_v;3R=lX~f|f1G zI-qQvVfTAEC~aRbenuT#8&r5?*<{r2exW8(x!rYq&RF$MpLY&&s^WR^n~2{89;Te= zzayK0aP?JHl-0LhV;`zE^lDqUT{gc`sX5~C3=CI?eripgn@n_kfCVfMB6C?E0#VR- zP7QCeUZwSB1IpNY3!Kh{b-+gF7_r&15wz8-i1(8jQ&PU|umbp9jjIWNi7_Ak-r|({ zeQ)~BLY^vU?Tq9|XNrWysczo#>bY|L9qSfG+jjIZebU_1}1r{g7>lzN! z#3p&bK3DLn;eaTtB~TSCGt}wMWQI5sGZz50pGaoDceFe+-w|+aA;1s7InC9lKh)@4 zvnRUI;lti*2x>#qYK2oEni@C0wtr==pxXVwBlgJh>pLyKnmu!*^PD7@b&Oe~VHMAg zM=0N0VLn6YxdcgR?1CmkP0JkL@anp&J7@WwKhPdl+dM%!Wxx&i#Jn_zox{0HqKqvR zbY(?5lx4)RbIfM)_a&2T{*Kl_l$!~vM8}7SM2Od?*x}Ard&4OL&|5)Tv+w~=Jsqo& zi-F+|i89%S*q^w|;$z>HB%AMl#6APqAKSld^BMJJ?gwd96IK63ocRBw#M1w7 zKl}9N{2bi*e?IA3CM1P~Xd>mCG(H6a!+lKujjI1YpuCF^kA2X zdcTQ!pPb5(BzemDJRHawuK>UfpzbKWfeVRfg(Z%MA?wkdYt=K{u1LRBN>pm}qKQvE zfMtxkzEIfqXED^O1nHM4CVCUgSqmT>QShkPYcqJI!%T!FF2oG|9xi&f zDHU$7H75S7EjZ;(cRYkUDSvYCt%2`k9*mAEghCs!2oaK6jQ)~Nw=<)<#7|hAb!%-E zGYaQ>sT$ueX#E8crmPEWjyZ(%U}Agj=Gmi%Kg6le%c3tWtBhTzTK979(FmK;F}vl` zsy2azx%dO*BK#FmX71vmoGw!yI&E8L-n@U4SE_Titlt{ElP#~tZ)e>%#$+{l{*jK+ zgIWFAP7c^OYrn9i*Ywu5fmWGcZYpK!wxMn5N6fQj1dfRU3fNDwh-x99#+b*eJK|mF zuaZiPLhKpEMbfCu`bmdsnvTHY5_CTI2QZiYn~)4Z=(~T~wPDB)8I0A{V6a|X{{f$| z931FNB_ypCXq8Yr(4$Fd(l>S8g&nQEjRMd1qpBCTWN%*$C{!hbjOZms?Qrp5pmq}7V<+ARQ@yB6Kxb~7ft z$}<(1G*X??c(^xYDLO_&zmCh<(_k0EP)12J|jVUa# z0ncZ%bE<6sR^NXy)ARuoUw*s2x-b25_(`qzmQ}3ZUKOtVX8VSx##!%Ffu?rEy%e+i zQIm@&+H|#9T;;xVH$QWDhNOsJC%_M>59p#*EesEiK1#ea#wnKIY2b!V%4tcy-(eWoePbA#m z<(gbNSd-?k)_<`h{Ug}+&D~e$Cv(ThE6-rBemqZmVl@7~QQ;h#XaIBHEp`vI)^^Oc zRoq407voJZ$uLP=Pwdt>HYE^> z9&_is?T6Hh9U=GB+HyE6kUi!ep4Lg|VNA=!OM#)p`C-8UnI$G|<;)Z(cb3g*h!|+k zjmBg1WIw3VmgS==(j@ycANEsvml5)Tu=V9}fMD_8Vz}F|NAsn1-0985^@S!s<-qGV z7j1}ppS03jy=7ssR^`R(?g)|G>?|r-=lf4Kd>E*5F>}y{My9AX zT=TK*JYo@U4bF-N*9lA&7=ht-mbceXBM|p{!NdhcL`9^$W~q3BAe7HgEUG#PG|*o4 zsi&xaj7)k4^{nu8IIhmCqNW+Nnvhqmf|zjEBI(r zYfn`CfYe3c^Nb>3Dr*qXcEo@e5TH4wE+kxN6-;e>w&9{iy;fp??}i)hgcF_G5}UwL zPdN1Ed&}(oimv1b!K;F^lDMxb2RMG_A;n#hsq$)^VXdt;e{##Hr=V`~wYPY3s`pkk z93>e6aL6B$!<@2lEMJ;Vuzz^;Ktas&>I!-pApvE_pPqV~iYj37L(42C?ip34Dc{cc zD7?f>#TZu&{v&ex%Q<5j!cvId6^j7Ez{Cg!W;HHjy{YDC<1t_xe%lw&Iba&QdF{z_ zvv;k_L?obDlZ52kT+MErzc^>cNG_B+K+-;PDSUX@yClV-mE zxsVXm>Qt#IOc=H}!M}tCJ0dI6CsV<-V|F&Pe#m4};$V!X99hs%m^)M7lm&&<2L8;R z8Lq}>cKz~?8$D-A>T|2uV#iusWjS*y7M}2G=9RXTp(GHBXK@GpqGh=8BdR6~AoR{z zpX+&BBZ^bGdMHi)0}CTSEV+kPooqjX#g}e^)f)@yDj}<2piV@!dfk=V$ag2Y!*DyU zbLz>U%%rD$sHWuGQ8e5OV6K_?p3qf zT)|q8)*IdaDaMsJ*f-F;6|8Dn7sWX6@H{sEow49?QK@sK$|o`Q^`%mPah=f|nK3LL zn<4xH<*Y2_$-k%UQk%)5DPiSISv#o!br9!m?uEgv@90N3s;SW@WAku1Z~!s(${rvh zjK-6fI~a?w6acEA?cfi*#n`(G?e&+2ft9a1G)~}$ z#2lvwDahD5{Uks>@UFV(_x{m^I)it`KYS^_<1%7R)B6uyA69O{W>*a~Ypb6MFJ5LS z5};aB#`c`+UCnx6swqG|?D4!YEfC`Vpu!$n_!~ts?ej}agOVYZO*{aPYMjNB;ky~9 zafM#7!OC`k6>iu$!y=8@)?L)Rwb9wU0*Gd^^_^k(*4a zAIWH53_NL|8vhsd*I;Il{MBsT1YRogdHoT|sW03zYIXgzUZzArdTk=|_E!z>bPx?YM z5Y(5_C)2OASsZ7z|Lo>4HL(WXK)$aRN{zmQ#wgw+-}4h7XS^6H(!R*zmOlK1;*r# z)(f-TTqbD8iZrkPqnL<`OV8I4;hr}OPB&Wac~P;MUs`+-1UQNZ>ntx@9Xx8C0|g2A zm!G?oqVg9#?BuXZSBtM`;1qs*FDEG zKTzhxc}`HDBKj_^3U#8)vbSY}tb@$w3XOL+cZYURH?s;4L2AdS*mwIuNMl)l^?m4F zg88v-rv&aV#VoenG?yLUMrr*=VV-*k-E25v#F}`$nXxf%?8DClIS*lJdKYb;`7+8f zqpNupi9O*4x*9W^wruL|_T>Xk9o|^irDRTPNORK|1ax8xg|v-U<-I zvtnA$^&K2I{ZsJTx=BTN8{Ef>(y8$FpNC3b??-_NrQ^B1}gKuzGQ z%RJL04InuGA0VFkFWKyK(Q-O@zE<%SU>lqqz|K)j0AkhLS@~u8l&x~k*_kq@aUOs? z0SKeQg*G+(EC%~BK54v^LIr|w9R&*$yz;X;Nj}w0F9oT*ox?jsIXO}TL0~)C{wdlC z?1F`WN!^%D8npV~=R3GBG?TJ&s+u$>DLwqs=~BHTdqpuKWK9iZPqyru7~r#j9V1vd zo0({4)0OkAAr%XkG2w|>F??i?5Z#cN$7mnhxv9-JYf+p(@UItX1TPdS0J4z1j9v2! z@hfrVH5q^9>3xVB@6&j}0WNbq(Q~r0{wF;<0Y5M|=|&%g5WiZDYf7Np*+!)~HS2_L zm0I*yJh-{m($+OSZ`Z>Tn-2k7v`oFcH=TFA`P1fmi|#zz@?_buZP@_+O{;A7Mw7@n z3&zg_eg5GMWagoxQ~Lz9J@Y<#i06&|$)A9yc_eK}w+^a5{wZtSb5vhtbpq?hcGP)9?f62v4)1T_CLPJO2HN;$CqM`uyeAZ)dK%HJL>@@)ChY@QX%^hMB&AY?+t^ zChQo(KWB55x?#P0IqvNJgTf}9_sL|^`htAarV9jWh$kJfj~=mNxDeG7w;I@Sj)hh| zK&JTBVR}ALlq@`@kG`0|<>TauB~zAfjIRd`77YXZp>0OXYyBfR_lr2L?Gf?@!_=@S zoO0~0s?~7VWhz+1{PaI(F%X%id~aPQ%D=;P=~RZahv0lbVxLT!n?`Fj zDn+a~_80W%S95^#voGTTz{xApP*#-=v; z(ADS5=8QB*7wGe)j>?7qv!Ze{0EFi9)nB0TwcptjxJWY#MTE?0XfNe1dMVihILaS> zTW}?)#|4p-DAAB8O6#ia$W!1yuLb5p{F$WqHx1l>Qj&k9R~Pz6{~e!Fqkx9rv9M*I&M-*7ZowS40@D;dPLwnh=-0CG^TJEx!yCUV?$QUb(D?y@kUH4 zek`&-55U@~=8~S4zBHVhDX$)kT)*dx z&ML9n^IB2SRLbd(QQsSwV-`Q85cWI@B1 zp6@hV$y=k50nQ%rbVwIr>i&MX-Lzlw++5P*30d@#bX>6UJ|PJcbDuwFUnFp zj7EZw$l2gZH(-rxY|FS9=ZWj;89~bzw5kPEsA`!R`~>U;;XzijzCctkq>$r*R-|4i zPVEAz#J2dR!4VQ2qV^AKd~uebbH39*AMpvUHFeBxkK6Q+WD3vQ@tuiw3X{n+1Fnw+ z4QL$aP1my9B*)`PNOFRDIsv+7IELXA%varu4~3*(*c5A_8)c;N_=9hLs-V7N!?88? zr$8DSJ_vu~J$Nr?+kAQT|zJy$~XODFv>TqH4Bm-U%d`5on zWx{$uFb&8ygZ>>a|bGC-TzOrhBbZIb#`Z@cL zIOAfmPekT3q(1i~16qssy;)>AkRy@}(-{u$WY=5k!L_suct<(6eB{z}8R7mo4ucA7 z$Dd&GrFqsHoz+eFE3&n2zN@tGh4%M{ z^@NJR0jGo=>63eKW$>j&MND8EnGA2$&=F9KR_uHoXZozshV!y~yS6Yl zyE>C4;XkjL-TX0W@5ijwP4A1#ZTAF%q7ouGHT0=v7ZJn~CwE%2v|!Q25B~X7TfgK~ zFAdk?$_7JJ}Mc8L0RdlKZy7C8uP1ox>!ScrH3~fg-vXYDykJ; zKDcCe(99!66${`pnQD~7%t*n$o&4E!xU-{U)GFNBcLn9^x7%LcU=r z$2aZH>PDr_*~STdI~-Mv8(=u3T;D!do>|+~oSUYt3S~QG85sl{j}+Ll5_Y2e!-Z5? zT%N6N^ms?LIgTppmc)QfQfVvfD}OB@TWmZA+uM<(J^Gw_eQIuR+hiO59>|kbhA;f> zzMDkF8AS|Z%bj$$1s^Ygo?|}gDss{x{n2#BSz;VilZaY z?V+qTpZ5e>wzL;?m zoVFbwo(FV1TZR!A%O&Nokq7=jPiJl7qoNbWy*>X@Vq?i2T<1MrkKp~&o0<&XyMtP< zr^U~F&wjI-40G`_5bo{Hn9{IE$gQqAHNa|z);pt}hwlS5;M=p3-0_R$O1?uQ zwoiY*HryP3Tv;)c_l>EXUU>+T!5;_@BDcn8&K|sg>El1phPeqHDag-Drf3%B3Nhm) za?mNb;;T2!xA{n9Kas({Pnk{VIegXrutioVYjz+9W2@Jh1S@iHUtdlUVMnuz_`M2HpscH|+K zCDLjN6d%$=s4{Vw&Or1}D6@26`$X{a@RTQo(K_GQ+9Vih%xk1IHS-roP)SXo;~Ant zzO&;@HvN%98@?Cbk)!i#qLIq;#z~NBzjN1N&3h8rmbv}K<&O82??;|)QAsln zLN4t>P$g?N9@#!@am6cYH5L;aHY}LpXT)cPW^8ZB>4+jjV*RzHxUu00AC`bT_^eqZh%`NpBOiwX5bn=G8?Way$T+4+GqUDJ6*_1ESwN5wmZC8j#EI=>XV#eUI@I>ix z_Wn!{wxLADcF>4RLgD3Q=c1bzX=qFXH)y@6o{=z^AboLNN7P4KF1#%rCyd!}$O(|F z3zxWGtmQ-N^xBnD=wRi__z___(vv=uKyP{LQey;l-Vu46FMOlFTJVt+Y(^LN03}dQ z)UtkC3WOL!>$udl`eQespYoFn?UsdBhzqdR_k@iwh)m9+f1WV_21@3uT*0`Z$TjaQ z%vhPsl)TbF@6rG>YQm2ZiCdXi7wzk9@kmN*++d>%4>l5ign6oHJ^xE3m65nQ((DfY zoCwRzs}2))w`NHqvilB<`ehH;3$u`d(*x^CQ4cqjgG)9l>KeuOaC&BXZ8)+4UQ>=m z97MkCntyZ0wWfH@)QqyE?A;r5;?u@9zn)RH8+g7mDM|csdKQ>Q)0RJWG4u%g%yYP= ztucCF85GOw57Rk%YPxmhC!b74P$g8OE_0%RR8zFmz`9|1L{6}+eww6VczVw!~$Rb#8U`49YlD+h~{6QCH{cZc_s>MQEj)!;h z+(m~NY;%+KWpLJ#DGtxxI?SBe9m4TTaL$Ss#xiJ+Xf2P7d=1 z3wG10IgQ~w8DeG?U7RQogOQ$$QT*-Fxw7At{Nw71B zvukXMC046M=rNDCpc1tNq$28X{mNJv%OCVl;w~TN*ti%KuLI)0KUmZe2a?Qvw|_w| zW*tT0{BC)`HE^%X%R4l_zCMZJ*e#Y5%i<94s_Cgf*ehR{-Tkva$YWt1<4<7l|5Beu zJ~KlrzD%noCaw_UWL;l8>epy|%66NtC7oRu?p|T??9zhE3SDyD{ymRpx!W6GkYabIosk1HPLXzRL*~J>py)k zHxYMWsYK&yxc~f8*8gQGJ%KJObTmAfXMqfW?VSaY`Z0)q5!p>efn;zNX_!5nhvkNE zURc7MXmC<#R(=+QqJI{3Y3k7Kuwl^LN{!`z-wp4Cf03K~?1p9_74Xttd%oZ7 z>|Za}jR;Qv|4x4KnJIfz%Jt zsp7(@j)H1YF^Fs;b8aAu>T%TuGW3vu`~Cb`YpxoG=2cmW1hKviAMY@4&K_4gqD`~tKtz7xCE3u#)=F6`Zu;);FpSVH-~OBs_C2`*=zkFBYmZ zw)O;JD;jL~>awI~w@1<(Ue#2CSF1{9J3dCfm-Q{>T$bl;k)YT!(oABfs4Q&g zfV1Kq?KciifEBa)7a<7Ll6^%7ZU)|kwfZE64BMNV2G3aol^ zaF6+Z*3SXtUyq^q%0s!8i9EHfr(L3UN<> z{Fjl(F>#{Zx;BGhWT^7hmqzW|Jh~)?o$DyG&dLA_diANwN6g#Rg-)qXjF=9nL$3g$ z8W+Ys6mWk|olmTWF3cL*`(b<+-Pc=w`vi4Wm5(sg98j{kl0Ph%1*gp#;@s9zUen(2K5z1sj!jmhulLlhj{kUdfZ^)?M`!>|zF=#FoeI z!3fj2HNe3Zu8Q=RGVLOT-K^Qz2d$y_#Jij(V*^C8?^#A*(>t3WFr-NM{n{EBq0hct<@#<;4?mSF~%4 zdd_XFs32r5odQ22c`R%B)deEm@M`)a!HI&E=$&E7#b<@zO`>?Qm^0H zu~@#{wAN+PU|&k6hawZ61J6a#FZgdQuyXr(L(RJn7C*^spE03?8?5lEx@Vd{)?y`+ zRqY3`H^05aTs|y}(CUxOrBr`3Xk!V;6)=dY=!QZdm?Gf`E5;30-#cq+(hyT%eNJr4 zI1cX`@#l`%udfRa;JU<3l*Q1Z{!u#Pu1T!QmDMRRftP~ryHRAKiy+$0!15C(Q={Ye zu#$K)XVx~3v*uc^b#h#9#)*5elT=iX69y&OS!8ahWzQ%TmMeYvZhfgMf7_aj*uU)_ zEZ`2nlVFlU5=()(x5n$a1E(oJtQ0jCG|rL_?_zl-lYBse{=V5WuuDxrh7^X6Ckf_< zjsDuOB13VI33QkIlg=P6_R>2@J5%Ky)_CewEoJ$GLK~Cj_rP^`^1fF0ZlAm3@=D@v zH`lDe;&7OtR&P({aLij++69xHy8u%APTJJFad=^d@~0qE(7smeEL;61R$Sos&NXSw zvBtyN1(|6eu=vVG;@c2QLz4ktj(~O15HF)zZ(PjDAAzLsw4ij`>)z;{1cZS4$z^ z$d#YHo1ZUp{T*-p|FR(P-%&gG=T8Snf4XuqE^am!7(G3IxPPgUg!^LDcBrJ-*oLjZ z_#}X<)KbHA%fc6&&*0&7N*fzxr82i&06T!j(|n-t<@)17PgWUSKl7X7)9*D3n;gKb zd$2%>g|eN%FP@%nPxA9!_%$lJZ+RCP@H6JlhLEvN_jcsNa5K7C!^ldNpTa-tlZ$=ow@(dAzPE%gdafA!kp| zX;v*mKs4gYGv~#K|BGD(`@2kq$hImpUa}&<^IQH48rVNt&hjWt9a}GCczzu3OrD!M zIxA&zfR8`?-Dd^rkr1~o-Bul-w*CMpp#nN};q1fz3*rBNmcYU>PTKdKK}2AT6PLSH z!}OcK4Q#Ug$AL}9|8Zc``hObOH1Lmep#G2hS>D@j9vSU4I3FZ{l$B^qhq;M+tOr#2 z6~J}ZHP2m^y$CmPhop0OQ;IKKWtW!-RFvdNrt)|dUzQ#UJ#;ZY7+^@74hRp>_?j|5 zBIoy0jI1!DCl3#foT#!+4X~_JDZ84qzk#@3C2YwC4fAeLmQ0oX`L|cbcK|FlJN`Vu z2Eml!#X;rQC-F+Ej;u5bZA(BzTodS;Y;h1DWKr#bI(*ro*U~l{db+F2GsXNLJN)^_ zKp|TGMN(4mY4&aEFEr8zX=J8ja$@M6D(9LGD}aA@tmzg|P0s57kJ;4hhJp28j{`KF zSAEdDjv0m83O(CwDSNW`#g=aiH7^f(90-e2e>rA%M#3N3rztlt)nSmjU05-ycxNF@ z8bR|qWYW_Rw`h7l!?N3tp2Z_Se%88q_JsQ0TN2K1cMq*3$L9NXRaKts1HM5y|rYjQM{jKT?-L7&2GbC=J{rufOqs#@@MB#4-2k99ZcQ;9_4DJenVi zI`2R5F8iCgT=-sdw|Rj+#MUJc7z`7I)@#*Eo_M1j6lkJSt>G1DZYEAdol5fman}iA z>n?&VGYmP{&j|@Wsny7~yO=$e4h7rxr5fhfV96ClrOnU`$OzaL_Zzu98EgQ4`>MS1JT?jOis`GHjRbK(~8vRk_mY9DJ%gs*75 zY^gO)2;0AmYdCB(#lN#W%>Z?S>1V09SG-!v_$jp&3*D*69;&Jbkh0x$r;XN>zK=y# z=EwB5WU0}9qL6T|3aaMWcZ0Kdac?t^R_r}3IZ_=`F|~eBl4nEP+4@Vz!FFf=YK^KZ`IVt{IlOeY|F6e)^j_9q2CR z6(#qmwM-g6?XOi$*q;E>`z1e_G*y zGafQn#709?(x^#?*AiFWh(C6TsCzjNR^4nS)2XN*p5Dz5PocK<=W&HAhb4x?%m;17 z#{oPawR9*|JveKkKBBqN|D-7IRoriR}-{J$m{ngM7kW%p%y!%F^$>+u^; zd8JuL+;;_~oDXGrT_R4yvgrNf9{vE~5~OdHWQ22MvAbADi=pbY#uOwqS(Hm~;I2hq z+g6whUD-u2RAxKnp+ihd+;m)EaZ$(~S*fC?BVwQhR#P}yuWP@fld!*@!P;31%*u64 zr80|JzlQL2yds@>^qx)n2VR778lFCa1+)j&gDdR!Q!Gx4r)cAFg z7*^`{{lV7G55#&@SO*JQ==JJ3aU*8UFsNPqV9Z8qLtf@>(^w;q$%J6AnbizYQ_>uT z5amw4*nZ18uK%2|tvK$P#RgdEAJ;?tvlKB&D^a;N$H6Vexg8IO;ZPbtRAPADLK{D# z1O%Q5I0zcdL-2p42B|D8HLC6DHN0-XYHil5omVkw0PCTU4R{gee_AJYn1755UU{m` z1EU$|5Bvyw9JpZX{(bHDb_Xh*#~8AMJFVx2T+^fE@3SzolqDL9`EOqX--7l3{bPsy zGZVG01blYvkw4e}Bmt~jyaDj~rY%s9zm@M&61S9D!e;k$TA`ok4KfP(NZDuSoZ zjye&AE>Qm#J$kb()XCT7t~5#(IPfS76ER?wOKW0*@7f&!KtkzEzfV4O9eCtY@9Uy7 z+&oyoHAQ;_Wvpi^tOjjJsAx^X5z#m>$0MGZAK`A;xAUiFareE3< z-U1fZ%#!v4I&G_M^;_xTlITZfJD)jyY7wwtjXMwNZMm<0rkxMyi83`FYH@=%8jJB#4yaJ3YB2GhrD$^7v=Nrs#V9AA6e8b51+M zjbEBP_b}d?EEBVLo8{>Ezmw>JA&^?mk(~GE`tYVD$30W;Ab5ob$-3U{^>`hc-+I)E<^0|IY^Kx0jONVoTBvB4VbJFO8ToDj+6LtS6VDhs2LNOg;r5k zFNv4#fpnpl0n*(0ee+6Z;}J=pK3jZEk6tqwObdpxQ;UuW?oXcz@1&&JP3SpMt#3t5 zGk8U?Asgn%2E!G55S#nzTDh7B9rCR(c;y`Fzj&sChn#-bL5D~AwpGqI3gn< z`gwy{h9Kr!)V@Vmb)ON<5OjjL_SXvPm>xC3?4G2tH(Y*&aS55T$EEMdSW>lUK;1x= zJoO3RN+B?mSi02tfpNYe=L)V2&nj0RTOx64vQ>4GcC^|EnmhF>t+5(a6w;|5TJ(qfyELU}CM9ZKe0DiE+qXO36B zlk|wiW5w`G9z7Pw!t&^BB||1lE4C6uy~ad$5EO@xcf74KvXc8%=Vo=#`j4j@{&+&B zQ;&_PH0~>X>*(QY*5E$ojSKRJu#k_5S-mXInD`^0-x|ULZjPBc?)0X=BPX0XgYrUz zOAB<(A`{@{%CYskI+cw@V&NCu9Z<7D|uBO^}@MEhG7C@W!$?=#z-=< za1*otadS!s%H&XnRP{@pueru29)N;v2=8CSk><(#zUQpnc%c=@QZ&nni?g3#12{Fs zC0t)kGj5S-nUugB;Q!cnIPm^4dHc+Ce`&z+=lJPjZywBq-&`8}F`gE*tAx=vO7-II zt*a?cprXp7Pb-1hM5?I>7qi*kL;Q_{v;gNW58ORIHIRN3MmmIv}Xr~>hq9IfY8#bWE90s{;_1iP{0Mj)pTisef zjdFG}hi`JTH*wy9EH<3=9fM>ysCW@Q;=9%ppZPxv;+{`VtYuNFF|23JKewXXcPaLX z3|g%tE}9fafXC~n9fXEEQD%;)m_LO*iwMv}1vDL=^j7!O;wd4b zleoKRa(|z>>3+&JhoG(XPp9}r&2!w&9UM?E^aM*+Ng3J5agT6YUyma%duaVF|FJVT z&9CJMH_xzbGJH5;VLO>=gOba?1>xSiQm=l7!xP5+!<$)%}8b5=Abmg zQouZNtm0VMYvSAA$PNX=xdq~bZ1H=%<-}QCP;ky3$}> z0es4>iniMknwy_Bl5aF~zgUWcqE*n4NS*)P8C1S-YX`i!S;;Pa*oVMaG@1V7d^&~TN4&5mr#NFVGY=}W&;UPk!y|+K zEyf2G{?9Y_Uc9tK0|yu+2A%Nv{!wQrlK!NI;J~_%_jU<(P`p?6^af3V*P)=}_+b2Z z=$Ja$Zi;kStoPRX$~@)I-!r;>0eC|L>P0)h^TDK(e-~wN!n4SHQn)RF{5e+OHQjse zj`g=u2uY&_qv}%mSc_w`KtzDi1PS)C?%oIjAWkRF5>Xt3?^oXBg*~z8{H4ajWSXj( z7oRQ5y48$1esN6>Xg95{drsCa0D;M$llpryJD)Zj&ggGQm$mGJoYXW=qQ!Ch6@`{~ zUj(c>LTvvUI_!GQN5?*hTVPj{MYT(UdwT7(PkPnR+^v75|37p9a4lJGH@};Di2#Jz zzP^6X;=|}8(vV)q>L;*@(+R5&pWM%U&@!P)DwY_O^LlS@*-1o&(IaGsbx7-ugoNto-;jWBHO)dDHHmHH za9Ky$;v3<$os8tDkH(1eYqX~P{3xF)uy*REW~RNFl1THt0-*If#K$jZ1xFXX2)z?s zIOgzR@t2VpNkV8XBv`$p1l+F-AA^xBif&q&;ZQwH%jRm2t=FH0Ml?Ea_k!%8n(Ujd zkC=>VeSS13zs4xR*WMg|j%HD#M{%uUxRFaM6pl4n<-r-Zg%bV3YIjFy6%7qr6F)9Q zNRT0bS3PK3-KzP7bbs1#bP(RO9@(@hJzsDK<)iHv#~|=4PAfiJ-hM<}cCt~SKs|r$ z5N4&S+(af~ttgBZ=dXEJm?PIvg>9~sB!5QfAxT5TU?zzPj1mQ&H(NY7TToI|P2!*O zCKn3ZKb>fvjFsL~91$R=`@A~Ns>9)Ks<6=Zl1ar_?u*v_wZ%uV*~cpzKa5f*pa=+T zHP*UIa6y)k==hF}n;C3ClwdXakNXf8*E9)zV7oJ$`iT4B+;-hTAf1C8Fq)XGpuIhp z@JQvr|21A39!@MkWF!?Tp*dCF&9)z`%@U2w3nGw0qRh(jm$!+D(=KaX&06NRIzlLq z)PP|+cYGZ@kGy_dmZB+7?DX;sMEfQ<-&9q5~2X z(Ji~m?(R)`G*R^Jb79Ao*Mr3qytgqIi=WOcMPZ{uA%2G)pX1!;Wq0mSn( z2$tElp?JN@ap(Lz+3GYaU=$hqf|G&IoW!VCR>qT~Z9hCGU|`#2cM;utrE!F>$D}9s zG2`r0$1Fyrx{>Xj^>2x%+aq*_`2r&p7J=4Qa^m#klb& zDk^n>xexc>j$Oxnd~WhK(`{w@XO77QDW_>OTh(_;ngm?av7dlK#MezCA` zeYts$#r}~fY*cH(5kc%dsZX1@%_#pf^Ds`>V<}ZUG>5HGNJA_$?+SnV-C2ye2x*c^ ziGMi3&H26HhaYbu_vVZ}wGC@5TmtyOf*-)M*-FE{V2t{cQ^8Ev&7`K^<#{p;d*IFd z)?*Q4RlbIC`zGThpM*OUh|+l1SW|+yMjEXY)EKy+0nMZ;eZeu~F4Q!CPCc ztGi9P)=2kf3AgW=R0!b;U;5;|(ioRb;yWVUig+j(G&NfUi;M&R+TgKA`7LwoH0x~R zkmAQz4urF)JCrjfbl@I`PHb-ke>YN_JlAxVnl34yfGL#3O1w}NEEy!yjm0XvU0EVL zqYSxPC??~VeUqT@MS=B`6HhRYx7)4QF8Y+COMlD`O87BnWc_znws4}q-4|SJ3L^;` z{2~hbbulNu#TeipS~t%CCs3n|phaRc;J#6NOwRK}SDBB?ZB}&%9><+9>6KK+l{nPk zj(BVFtaUVSM-l$gBFPTl!h6wmmDm=QrJ)7;g-*NeT`|G(m&AVAv7W5cEpMG$Cc65V z0ZyLJQ5nH*#U#i*vH8J#V7ziJ<%i6&+!E!_p*S*Wr_bh_13|eZgQ=#O*A@8usW?4K zQ*3yr&}pnGqn{6m-5J~jxwuB~-)5iJQTMd+%B$UleY($EEW_UbTX>TfnVonXAJPAO zVfu6H&1^qPS6bxH*(J3q-g$n~|3bNE5h=HOExYR4?%#fI{=`b;@!OfxAKVo`U3oG} z`8Vgp>pNGqp8IwF@s1ysnVPCN4`llZAqO+t6^~9%b++iB{M*eKmp4z0I zkLz9+9WA|O*>>%x{`pkfd)r`rqZcFexAYuwJ2#n-Jko_enK zMSA(0qI0LCzI-+RcTZBgVR}F2_&;d!AmRS~`__%cd*|{@e(ew^!JPZN+-^{25B&Jo&VPB^d4UuCf6p{-6_1;}{Q6$ODFQOxyY7 z2Wt*LHq|j}dS%kM$8ekH->s&=7EzMiy&bCy;_FZCmAb#DZ{fcq?-}-h2IKF7hm7Ha z@t0z!uU4q$TK_trx+0gaulK@kNuBD0Wf$|m_Ai^(BT${#cIVg)wbFvTch(xmU-j+& zBeGBa#Ps=H{9o1U*&kd5Ucim%CY`37ud&aZ3-d1cf4*_ospkY zZ85>VS+?c;eBdDprwx1TPj85=yS4XU$X}`eG z?$(o5{4FJMdc(~d*Z7vE8}aNHu%1vKbNKz+;7`|Az{3zE>Mh83iI5zAAJ>cBN1b%q>2F-7PZd#u$!SKWxHm(U;g767+ zsXB0;?I(O%VHg>^AaNn&R6Pa;i}o|~=11Rhh}*};P!Wd`TJw=t55SKA1g=H;Z|^NH UEGBB}bOvOKr>mdKI;Vst0KZl<-2eap literal 0 HcmV?d00001 diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index 0b0bc2bb2..4908c275d 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -27,6 +27,7 @@ from profiler.advisor.analyzer.cluster.slow_rank_analyser import SlowRankAnalyze from profiler.advisor.analyzer.cluster.slow_link_analyser import SlowLinkAnalyzer from profiler.advisor.analyzer.cluster.Communication_retransmission_analyzer import RDMARetransmissionAnalyzer from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer +from profiler.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariabelAnalyzer from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer from profiler.advisor.analyzer.schedule.syncbn.syncbn_analyzer import SyncBNAnalyzer from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_analyzer import SynchronizeStreamAnalyzer @@ -54,7 +55,10 @@ class Interface: "communication": OrderedDict({ SupportedScopes.PACKET: PacketAnalyzer }), - "overall": OrderedDict({SupportedScopes.OVER_ALL: OverallSummaryAnalyzer}), + "overall": OrderedDict({ + SupportedScopes.ENVIRONMENT_VARIABLE_ANALYSIS: EnvironmentVariabelAnalyzer, + SupportedScopes.OVER_ALL: OverallSummaryAnalyzer, + }), "dataloader": OrderedDict({SupportedScopes.DATALOADER: DataloaderAnalyzer}), "cluster": OrderedDict({ SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION: RDMARetransmissionAnalyzer, diff --git a/profiler/advisor/rules/environment_variable_info.yaml b/profiler/advisor/rules/environment_variable_info.yaml new file mode 100644 index 000000000..b91f827ef --- /dev/null +++ b/profiler/advisor/rules/environment_variable_info.yaml @@ -0,0 +1,42 @@ +ASCEND_GLOBAL_LOG_LEVEL: + desc: "log level: 0-debug, 1-info, 2-warning, 3-error.\nDefault is error level." + suggest: "Debug or info level may lead to training performance degradation,\n + recommended setting error level by execute command 'export ASCEND_GLOBAL_LOG_LEVEL=3." +HCCL_RDAM_TC: + desc: "Configure the DSCP value of RoCE packets sent by the network port.\n + In the DS field of IP datagram header, the rightmost 6 bits are DSCP, and leftmost 2 bits are 0.\n + It should be set to DSCP * 4. Default value is 132, that is, DSCP is 33 (132=33*4)." + suggest: "Please refer to https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424" + suggest_html: "Please refer to LINK" +HCCL_RDMA_SL: + desc: "Specify the priority of the RDMA NIC.\n + The value must be the same as the PFC priority for the NIC.\n + Otherwise, the performance may deteriorate.\n + The value range is [0, 7], and default value is 4." + suggest: "Please refer to https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424" + suggest_html: "Please refer to LINK" +ACLNN_CACHE_LIMIT: + desc: "Number of cached aclnn operators." + suggest: "Setting a large number when alcnn and host bound, such as 'export ACLNN_CACHE_LIMIT=100000'" +HOST_CACHE_CAPACITY: + desc: "Enable dynamic shape cache.\n + The default value is 0, indicating that the data cache is disabled.\n + If it is set to a non-zero positive integer, for example, 10, the system caches the execution data of 10 inputs shapes that frequently occur recently.\n + When the cached shapes appear again, the host execution performance will be improved, but the host memory usage increase.\n + The specific increase is proportional to the value of the HOST_CACHE_CAPACITY and size of the model." + suggest: "Setting a non-zero number, such as 'export HOST_CACHE_CAPACITY=20'" +ASCEND_ENHANCE_ENABLE: + desc: "Enable hccl ffts+ mode. 0-disable, 1-enable" + suggest: "Recommend enable hccl ffts+ mode by execute command 'export ASCEND_ENHANCE_ENABLE=1'" +PYTORCH_NPU_ALLOC_CONF: + desc: "Controlling cache allocator behavior.\n + The optional parameter is max_split_size_mb, garbage_collection_threshold and expandable_segments.\n + 1. max_split_size_mb:v--the memory block that is greater than v will be not split.\n + 2. garbage_collection_threshold:t--after the threshold is set, if the NPU memory usage exceed threshold, the cached allocator starts to reclaim memory block. The range of t is (0.0, 1.0).\n + 3. expandable_segments:True/False--The default value is False. If True, this setting instructs cache allocator to create specific memory blocks that can be expanded later to better handle frequent changed in memory usage." + suggest: "export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True" +ASCEND_LAUNCH_BLOCKING: + desc: "Whether to enable the synchronization mode during operation execution.\n + When set to 1, force the operator to run in synchronous mode, making it easier to debug and track down problems in the code.\n + If the set to 0, the task is executed in asynchronous mode." + suggest: "export ASCEND_LAUNCH_BLOCKING=1" \ No newline at end of file diff --git "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" index abf8e1055..2db9b8786 100644 --- "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" +++ "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" @@ -7,4 +7,5 @@ | 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html"] | Advisor优化手段参考示例 | | 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning"] | Advisor优化手段参考示例 | | 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html"] | Advisor优化手段参考示例 | +| 文档 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424"] | Advisor优化手段参考示例 | -- Gitee From c0c04a0dd2204a42ca3e2e006950115ba6c08082 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Wed, 7 Aug 2024 19:42:42 +0800 Subject: [PATCH 187/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dut=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pytorch_ut/hook_module/test_hook_module.py | 14 ++++++++++++-- .../test/pytorch_ut/hook_module/test_wrap_aten.py | 5 +++++ .../hook_module/test_wrap_distributed.py | 6 ++++++ .../pytorch_ut/hook_module/test_wrap_tensor.py | 6 ++++++ .../test/pytorch_ut/hook_module/test_wrap_torch.py | 6 ++++++ 5 files changed, 35 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_hook_module.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_hook_module.py index 50783e5d7..96f4b4df2 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_hook_module.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_hook_module.py @@ -7,13 +7,18 @@ class TestHookModule(unittest.TestCase): def test_call_1(self): def forward_pre_hook(): return "result_input", "result_kwargs" + def forward_hook(): return 2 + def backward_hook(): pass + def forward_hook_torch_version_below_2(): + pass + def hook(prefix): - return forward_pre_hook, forward_hook, backward_hook + return forward_pre_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 HOOKModule.prefix_op_name_ = "123" test = HOOKModule(hook) test._call_func = Mock(return_value=1) @@ -23,13 +28,18 @@ class TestHookModule(unittest.TestCase): def test_call_2(self): def forward_pre_hook(nope, input, kwargs): return input, kwargs + def forward_hook(nope, input, kwargs, result): return input + def backward_hook(): pass + def forward_hook_torch_version_below_2(): + pass + def hook(prefix): - return forward_pre_hook, forward_hook, backward_hook + return forward_pre_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 HOOKModule.prefix_op_name_ = "123" input = 2 test = HOOKModule(hook) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py index 4940b07cb..aa559dbde 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py @@ -6,10 +6,15 @@ from msprobe.pytorch.hook_module.wrap_aten import AtenOPTemplate, AtenOPPacketTe def hook(name): def forward_pre_hook(nope, input, kwargs): return input, kwargs + def forward_hook(nope, input, kwargs, result): return 2 + def backward_hook(): pass + + def forward_hook_torch_version_below_2(): + pass return forward_pre_hook, forward_hook, backward_hook diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py index 9a375e45b..84b1ff993 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py @@ -6,10 +6,16 @@ class TestWrapDistributed(unittest.TestCase): def hook(name, prefix): def forward_pre_hook(nope, input, kwargs): return input, kwargs + def forward_hook(nope, input, kwargs, result): return 2 + def backward_hook(): pass + + def forward_hook_torch_version_below_2(): + pass + return forward_pre_hook, forward_hook, backward_hook def test_get_distributed_ops(self): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py index 61f76b0ca..ba9656f11 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py @@ -8,10 +8,16 @@ class TestWrapTensor(unittest.TestCase): def hook(name, prefix): def forward_pre_hook(nope, input, kwargs): return input, kwargs + def forward_hook(nope, input, kwargs, result): return 2 + def backward_hook(): pass + + def forward_hook_torch_version_below_2(): + pass + return forward_pre_hook, forward_hook, backward_hook def test_get_tensor_ops(self): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py index e1a3e7798..9bbd1e072 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py @@ -8,10 +8,16 @@ class TestWrapTorch(unittest.TestCase): def hook(name, prefix): def forward_pre_hook(nope, input, kwargs): return input, kwargs + def forward_hook(nope, input, kwargs, result): return 2 + def backward_hook(): pass + + def forward_hook_torch_version_below_2(): + pass + return forward_pre_hook, forward_hook, backward_hook def setUp(self): -- Gitee From becae25a860d128e1d79d8b7626f021699599d71 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Wed, 7 Aug 2024 19:53:41 +0800 Subject: [PATCH 188/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9torch=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/pytorch/hook_module/hook_module.py | 4 ++-- debug/accuracy_tools/msprobe/pytorch/module_processer.py | 4 ++-- debug/accuracy_tools/msprobe/pytorch/service.py | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py index 4d8f48a4e..aa724b50f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/hook_module.py @@ -23,7 +23,7 @@ import torch.nn as nn import torch.utils.hooks as full_hooks from msprobe.core.common.const import Const -torch_vsrsion_above_2 = torch.__version__.split('+')[0] > '2.0' +torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' class HOOKModule(nn.Module): @@ -50,7 +50,7 @@ class HOOKModule(nn.Module): HOOKModule.module_count[self.prefix] += 1 self.prefix = self.prefix + str(HOOKModule.module_count[self.prefix] - 1) + Const.SEP forward_pre_hook, forward_hook, backward_hook, _ = build_hook(self.prefix) - if torch_vsrsion_above_2: + if torch_version_above_or_equal_2: self.register_forward_pre_hook(forward_pre_hook, with_kwargs=True) self.register_forward_hook(forward_hook, with_kwargs=True) else: diff --git a/debug/accuracy_tools/msprobe/pytorch/module_processer.py b/debug/accuracy_tools/msprobe/pytorch/module_processer.py index 8303ea814..e6d2125e4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/module_processer.py +++ b/debug/accuracy_tools/msprobe/pytorch/module_processer.py @@ -5,7 +5,7 @@ from torch.utils.hooks import BackwardHook from msprobe.core.common.const import Const from msprobe.core.data_dump.scope import ModuleRangeScope -torch_version_above_2 = torch.__version__.split('+')[0] > '2.0' +torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' class ModuleProcesser: @@ -124,7 +124,7 @@ class ModuleProcesser: if self.scope: self.scope.begin_module(full_name) - if torch_version_above_2: + if torch_version_above_or_equal_2: if Const.START in start_or_stop: return pre_hook else: diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index a7c8ea72c..bbf432a72 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -15,7 +15,7 @@ from msprobe.pytorch.hook_module import remove_dropout from msprobe.pytorch.hook_module.api_registry import api_register from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.module_processer import ModuleProcesser -torch_version_above_2 = torch.__version__.split('+')[0] > '2.0' +torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' class Service: @@ -177,7 +177,7 @@ class Service: pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 \ = self.build_hook(BaseScope.Module_Type_Module, prefix) - if torch_version_above_2: + if torch_version_above_or_equal_2: module.register_forward_hook(forward_hook, with_kwargs=True) else: module.register_full_backward_hook( @@ -189,7 +189,7 @@ class Service: self.module_processor.node_hook(prefix + Const.FORWARD, Const.START)) module.register_forward_hook( self.module_processor.node_hook(prefix + Const.FORWARD, Const.STOP)) - if torch_version_above_2: + if torch_version_above_or_equal_2: module.register_full_backward_pre_hook( self.module_processor.node_hook(prefix + Const.BACKWARD, Const.START)) module.register_full_backward_hook( -- Gitee From 61a9e32e86ab0efc7a6de19d75ce7509cc5cb026 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Wed, 7 Aug 2024 19:56:49 +0800 Subject: [PATCH 189/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=86=92=E7=83=9F?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py | 2 +- .../test/pytorch_ut/hook_module/test_wrap_distributed.py | 2 +- .../msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py | 2 +- .../msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py index aa559dbde..f219e22e8 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_aten.py @@ -16,7 +16,7 @@ def hook(name): def forward_hook_torch_version_below_2(): pass - return forward_pre_hook, forward_hook, backward_hook + return forward_pre_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py index 84b1ff993..246feb56b 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_distributed.py @@ -16,7 +16,7 @@ class TestWrapDistributed(unittest.TestCase): def forward_hook_torch_version_below_2(): pass - return forward_pre_hook, forward_hook, backward_hook + return forward_pre_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 def test_get_distributed_ops(self): ops = get_distributed_ops() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py index ba9656f11..2aadc358a 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py @@ -18,7 +18,7 @@ class TestWrapTensor(unittest.TestCase): def forward_hook_torch_version_below_2(): pass - return forward_pre_hook, forward_hook, backward_hook + return forward_pre_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 def test_get_tensor_ops(self): result = get_tensor_ops() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py index 9bbd1e072..14b156e3b 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py @@ -18,7 +18,7 @@ class TestWrapTorch(unittest.TestCase): def forward_hook_torch_version_below_2(): pass - return forward_pre_hook, forward_hook, backward_hook + return forward_pre_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 def setUp(self): -- Gitee From 07749dae2c6bf4ba4111a0ee6b16e7e79e8292c4 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 00:45:53 +0800 Subject: [PATCH 190/791] add online run_ut --- .../tensor_transport_layer/attl.py | 6 +++++- debug/accuracy_tools/msprobe/pytorch/service.py | 12 ++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index c4d5b76c5..796d27728 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -86,7 +86,11 @@ class ATTL: rank = buffer.rank if hasattr(buffer, "rank") else 0 step = buffer.step if hasattr(buffer, "step") else 0 io_buff = io.BytesIO() - torch.save(buffer, io_buff) + try: + torch.save(buffer, io_buff) + except Exception as e: + logger.warning(f"buffer save failed: {e}") + return data = io_buff.getvalue() self.socket_manager.add_to_sending_queue(data, rank=rank, step=step) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index d74a9dc25..9b4afb756 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -36,7 +36,7 @@ class Service: connect_ip=self.config.host, connect_port=self.config.port, nfs_path=self.config.nfs_path) - need_dump = self.current_rank in self.config.rank + need_dump = len(self.config.rank) == 0 or self.current_rank in self.config.rank self.attl = ATTL('npu', attl_config, need_dump=need_dump) if self.config.nfs_path: self.attl.upload("start") @@ -68,7 +68,9 @@ class Service: return None if self.config.online_run_ut: - api_data = ApiData(api_or_module_name, args, kwargs, output, self.current_iter, self.current_rank) + if not self.data_collector.scope or self.data_collector.scope.check(api_or_module_name): + return None + api_data = ApiData(name[:-1], args, kwargs, output, self.current_iter, self.current_rank) self.attl_send(api_data) return None @@ -88,8 +90,9 @@ class Service: return if self.config.online_run_ut: - api_data = ApiData(api_or_module_name, grad_input, None, grad_output, self.current_iter, - self.current_rank) + if not self.data_collector.scope or self.data_collector.scope.check(api_or_module_name): + return None + api_data = ApiData(name[:-1], grad_input, None, grad_output, self.current_iter, self.current_rank) self.attl_send(api_data) return None @@ -208,6 +211,7 @@ class Service: def attl_send(self, api_data): logger.info(f"tools is dumping api: {api_data.name}, rank: {self.current_rank}") + api_data.rank = api_data.rank if api_data.rank else 0 if self.config.nfs_path: self.attl.upload(api_data) else: -- Gitee From a6fd8721767ea0b6bdb0a5f6d16af29cf89be2ad Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 09:16:19 +0800 Subject: [PATCH 191/791] add online run_ut --- .../pytorch/api_accuracy_checker/tensor_transport_layer/attl.py | 2 +- debug/accuracy_tools/msprobe/pytorch/service.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index 796d27728..e83fd2ecb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -83,7 +83,7 @@ class ATTL: if 'device' in buffer.kwargs: buffer.kwargs.pop('device') - rank = buffer.rank if hasattr(buffer, "rank") else 0 + rank = buffer.rank if hasattr(buffer, "rank") and buffer.rank is not None else 0 step = buffer.step if hasattr(buffer, "step") else 0 io_buff = io.BytesIO() try: diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 9b4afb756..681a444c0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -211,7 +211,6 @@ class Service: def attl_send(self, api_data): logger.info(f"tools is dumping api: {api_data.name}, rank: {self.current_rank}") - api_data.rank = api_data.rank if api_data.rank else 0 if self.config.nfs_path: self.attl.upload(api_data) else: -- Gitee From 052de6de0de2a2de81f3ec84b0ff72dd273eec80 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 09:19:03 +0800 Subject: [PATCH 192/791] add online run_ut --- debug/accuracy_tools/msprobe/pytorch/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 681a444c0..e18a0295f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -92,7 +92,7 @@ class Service: if self.config.online_run_ut: if not self.data_collector.scope or self.data_collector.scope.check(api_or_module_name): return None - api_data = ApiData(name[:-1], grad_input, None, grad_output, self.current_iter, self.current_rank) + api_data = ApiData(name[:-1], grad_input, {}, grad_output, self.current_iter, self.current_rank) self.attl_send(api_data) return None -- Gitee From 40eed90a6edce10fb77bce07a89e8f6268459128 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Thu, 8 Aug 2024 09:35:22 +0800 Subject: [PATCH 193/791] =?UTF-8?q?=E4=BC=98=E5=8C=96build=5Fhook=E5=87=BD?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/service.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index bbf432a72..bc41dad15 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -63,18 +63,7 @@ class Service: return output def forward_hook_torch_version_below_2(api_or_module_name, module, args, output): - if module_type == BaseScope.Module_Type_Module: - api_or_module_name = module.mindstudio_reserved_name - self.data_collector.visit_and_clear_overflow_status(api_or_module_name) - - if not self.switch: - return None - if self.data_collector: - module_input_output = ModuleForwardInputsOutputs(args=args, kwargs={}, output=output) - self.data_collector.forward_data_collect(api_or_module_name, module, pid, module_input_output) - if self.data_collector.if_return_forward_new_output(): - return self.data_collector.get_forward_new_output() - return output + return forward_hook(api_or_module_name, module, args, {}, output) def backward_hook(api_or_module_name, module, grad_input, grad_output): if module_type == BaseScope.Module_Type_Module: -- Gitee From 2564181cbed75c5fec875fedbf233114f0e73f62 Mon Sep 17 00:00:00 2001 From: zyy Date: Thu, 8 Aug 2024 09:35:32 +0800 Subject: [PATCH 194/791] 86 --- .../compare_backend/profiling_parser/gpu_profiling_parser.py | 1 - 1 file changed, 1 deletion(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 07943ba73..91b4094c2 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -61,7 +61,6 @@ class GPUProfilingParser(BaseProfilingParser): def _update_overall_metrics(self): self._calculate_performance_time() self.__parse_memory_reserved() - self._result_data.overall_metrics.calculate_vec_time() self._result_data.overall_metrics.calculate_schedule_time() self._result_data.overall_metrics.trans_time_to_s() -- Gitee From e4891794ba650e15280a476cd55d94a843e683ff Mon Sep 17 00:00:00 2001 From: h00613304 Date: Thu, 8 Aug 2024 09:52:59 +0800 Subject: [PATCH 195/791] =?UTF-8?q?=E4=BC=98=E5=8C=96build=5Fhook=E5=87=BD?= =?UTF-8?q?=E6=95=B0=E8=BF=94=E5=9B=9E=E4=B8=AA=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/service.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index bc41dad15..d264ab70d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -3,6 +3,7 @@ import os from pathlib import Path import torch +from collections import namedtuple from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException from msprobe.core.common.file_check import FileChecker, check_path_before_create @@ -17,6 +18,8 @@ from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.module_processer import ModuleProcesser torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' +HookFn = namedtuple('hookFn', ['pre_hook', 'forward_hook', 'backward_hook', 'forward_hook_torch_version_below_2']) + class Service: def __init__(self, config): @@ -84,7 +87,7 @@ class Service: forward_hook = functools.partial(forward_hook, forward_name_template) backward_hook = functools.partial(backward_hook, backward_name_template) forward_hook_torch_version_below_2 = functools.partial(forward_hook_torch_version_below_2, forward_name_template) - return pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2 + return HookFn(pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2) def step(self): self.current_iter += 1 -- Gitee From b602d05ae57fbacec9380eebcdba3c3c4e809906 Mon Sep 17 00:00:00 2001 From: h00613304 Date: Thu, 8 Aug 2024 10:19:45 +0800 Subject: [PATCH 196/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dbuild=5Fhook=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/service.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index d264ab70d..eb5fb861b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -83,11 +83,11 @@ class Service: pid = os.getpid() forward_name_template = name + Const.FORWARD backward_name_template = name + Const.BACKWARD - pre_forward_hook = functools.partial(pre_hook, forward_name_template) - forward_hook = functools.partial(forward_hook, forward_name_template) - backward_hook = functools.partial(backward_hook, backward_name_template) - forward_hook_torch_version_below_2 = functools.partial(forward_hook_torch_version_below_2, forward_name_template) - return HookFn(pre_forward_hook, forward_hook, backward_hook, forward_hook_torch_version_below_2) + pre_forward_hook_fn = functools.partial(pre_hook, forward_name_template) + forward_hook_fn = functools.partial(forward_hook, forward_name_template) + backward_hook_fn = functools.partial(backward_hook, backward_name_template) + forward_hook_torch_version_below_2_fn = functools.partial(forward_hook_torch_version_below_2, forward_name_template) + return HookFn(pre_forward_hook_fn, forward_hook_fn, backward_hook_fn, forward_hook_torch_version_below_2_fn) def step(self): self.current_iter += 1 -- Gitee From 62892977d6bbb69003e94bae75d037c99218d537 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 10:29:47 +0800 Subject: [PATCH 197/791] fix reviews --- debug/accuracy_tools/msprobe/core/common/const.py | 3 --- .../api_accuracy_checker/tensor_transport_layer/attl.py | 8 ++++++-- debug/accuracy_tools/msprobe/pytorch/pt_config.py | 6 +++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 34512ce04..2fe424a43 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -20,9 +20,6 @@ class Const: DEFAULT_PATH = './' WHITE_LIST = 'white_list' BLACK_LIST = 'black_list' - IS_ONLINE = False - NFS_PATH = "" - IS_BENCHMARK_DEVICE = True DUMP_TENSOR_DATA = 'dump_tensor_data' # dump mode diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index c4d5b76c5..21c5dafba 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -121,7 +121,7 @@ class ATTL: try: buffer = torch.load(buffer, map_location="cpu") except Exception as e: - self.logger.error("there is something error. please check it. %s", e) + self.logger.warning("there is something error. please check it. %s", e) if isinstance(buffer, bytes): return None if isinstance(buffer, str): @@ -147,7 +147,11 @@ class ATTL: if cur_file is None: return None else: - buffer = torch.load(cur_file) + buffer = None + try: + buffer = torch.load(cur_file) + except Exception as e: + self.logger.warning("there is something error. please check it. %s", e) remove_path(cur_file) return buffer diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 9dcd12c6e..5612a8734 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -71,9 +71,9 @@ class RunUTConfig(BaseConfig): self.white_list = json_config.get("white_list", Const.DEFAULT_LIST) self.black_list = json_config.get("black_list", Const.DEFAULT_LIST) self.error_data_path = json_config.get("error_data_path", Const.DEFAULT_PATH) - self.is_online = json_config.get("is_online", Const.IS_ONLINE) - self.nfs_path = json_config.get("nfs_path", Const.NFS_PATH) - self.is_benchmark_device = json_config.get("is_benchmark_device", Const.IS_BENCHMARK_DEVICE) + self.is_online = json_config.get("is_online", False) + self.nfs_path = json_config.get("nfs_path", "") + self.is_benchmark_device = json_config.get("is_benchmark_device", True) self.host = json_config.get("host", "") self.port = json_config.get("port", -1) self.rank_list = json_config.get("rank_list", Const.DEFAULT_LIST) -- Gitee From 9b08f2da4e4aa98fde5440b1e45e2b8d5ee9620a Mon Sep 17 00:00:00 2001 From: h00613304 Date: Thu, 8 Aug 2024 10:49:25 +0800 Subject: [PATCH 198/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9service=20import?= =?UTF-8?q?=E6=A8=A1=E5=9D=97=E5=AF=BC=E5=85=A5=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index eb5fb861b..bc363926f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -2,8 +2,8 @@ import functools import os from pathlib import Path -import torch from collections import namedtuple +import torch from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException from msprobe.core.common.file_check import FileChecker, check_path_before_create -- Gitee From e4e661b16491c78a82b26f87b31496fffcdff8a5 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 10:29:47 +0800 Subject: [PATCH 199/791] fix reviews --- debug/accuracy_tools/msprobe/core/common/const.py | 4 +--- .../api_accuracy_checker/tensor_transport_layer/attl.py | 8 ++++++-- debug/accuracy_tools/msprobe/pytorch/pt_config.py | 6 +++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 119ad7d62..573838d1e 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -20,9 +20,7 @@ class Const: DEFAULT_PATH = './' WHITE_LIST = 'white_list' BLACK_LIST = 'black_list' - IS_ONLINE = False - NFS_PATH = "" - IS_BENCHMARK_DEVICE = True + DUMP_TENSOR_DATA = 'dump_tensor_data' # dump mode ALL = "all" diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index e83fd2ecb..18099a3d3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -125,7 +125,7 @@ class ATTL: try: buffer = torch.load(buffer, map_location="cpu") except Exception as e: - self.logger.error("there is something error. please check it. %s", e) + self.logger.warning("there is something error. please check it. %s", e) if isinstance(buffer, bytes): return None if isinstance(buffer, str): @@ -151,7 +151,11 @@ class ATTL: if cur_file is None: return None else: - buffer = torch.load(cur_file) + buffer = None + try: + buffer = torch.load(cur_file) + except Exception as e: + self.logger.warning("there is something error. please check it. %s", e) remove_path(cur_file) return buffer diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 6bcd1a05e..ea84d5ca3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -75,9 +75,9 @@ class RunUTConfig(BaseConfig): self.white_list = json_config.get("white_list", Const.DEFAULT_LIST) self.black_list = json_config.get("black_list", Const.DEFAULT_LIST) self.error_data_path = json_config.get("error_data_path", Const.DEFAULT_PATH) - self.is_online = json_config.get("is_online", Const.IS_ONLINE) - self.nfs_path = json_config.get("nfs_path", Const.NFS_PATH) - self.is_benchmark_device = json_config.get("is_benchmark_device", Const.IS_BENCHMARK_DEVICE) + self.is_online = json_config.get("is_online", False) + self.nfs_path = json_config.get("nfs_path", "") + self.is_benchmark_device = json_config.get("is_benchmark_device", True) self.host = json_config.get("host", "") self.port = json_config.get("port", -1) self.rank_list = json_config.get("rank_list", Const.DEFAULT_LIST) -- Gitee From a16fb75dd2c131659ad7c5f33dffd0383cc15bf3 Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Tue, 6 Aug 2024 10:41:16 +0800 Subject: [PATCH 200/791] optimize_longest_common_subsequence --- .../data_prepare/operator_data_prepare.py | 27 ++- .../data_prepare/sequence_pre_matching.py | 162 ++++++++++++++++++ .../generator/detail_performance_generator.py | 158 ++++++----------- .../profiling_parser/base_profiling_parser.py | 7 +- .../compare_backend/utils/common_func.py | 17 +- .../compare_backend/utils/constant.py | 3 + .../compare_backend/utils/torch_op_node.py | 6 +- .../compare_backend/utils/tree_builder.py | 6 +- .../test_base_profiling_parser.py | 1 + .../compare_tools/utils/test_tree_builder.py | 8 +- 10 files changed, 258 insertions(+), 137 deletions(-) create mode 100644 profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py diff --git a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py index 3106527c4..59913528a 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py +++ b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py @@ -5,11 +5,11 @@ from compare_backend.utils.tree_builder import TreeBuilder class OperatorDataPrepare: def __init__(self, profiling_data: ProfilingResult): self.profiling_data = profiling_data + self._all_nodes = self._build_tree() + self._root_node = self._all_nodes[0] def get_top_layer_ops(self) -> any: - root_node = TreeBuilder.build_tree(self.profiling_data.torch_op_data, self.profiling_data.kernel_dict, - self.profiling_data.memory_list) - level1_child_nodes = root_node.child_nodes + level1_child_nodes = self._root_node.child_nodes result_data = [] for level1_node in level1_child_nodes: if level1_node.is_step_profiler(): @@ -19,18 +19,11 @@ class OperatorDataPrepare: return result_data def get_all_layer_ops(self) -> any: - root_node = TreeBuilder.build_tree(self.profiling_data.torch_op_data, [], []) - level1_child_nodes = root_node.child_nodes - node_queue = [] result_data = [] - for level1_node in level1_child_nodes: - if level1_node.is_step_profiler(): - node_queue.extend(level1_node.child_nodes) - else: - node_queue.append(level1_node) - while len(node_queue) > 0: - node = node_queue.pop(0) - result_data.append(node) - if node.child_nodes: - node_queue.extend(node.child_nodes) - return result_data \ No newline at end of file + if len(self._all_nodes) < 1: + return result_data + return list(filter(lambda x: not x.is_step_profiler(), self._all_nodes[1:])) + + def _build_tree(self): + return TreeBuilder.build_tree(self.profiling_data.torch_op_data, self.profiling_data.kernel_dict, + self.profiling_data.memory_list) diff --git a/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py b/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py new file mode 100644 index 000000000..c04d4c2b6 --- /dev/null +++ b/profiler/compare_tools/compare_backend/data_prepare/sequence_pre_matching.py @@ -0,0 +1,162 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from collections import deque + +from compare_backend.utils.name_function import NameFunction +from compare_backend.utils.common_func import longest_common_subsequence_matching +from compare_backend.utils.torch_op_node import TorchOpNode +from compare_backend.utils.module_node import ModuleNode + +from compare_backend.utils.constant import Constant + + +class SequencePreMatching: + OP_TYPE = 1 + MODULE_TYPE = 2 + + def __init__(self, args, base_bwd_tid=None, comparison_bwd_tid=None): + self._args = args + self._base_bwd_tid = base_bwd_tid + self._comparison_bwd_tid = comparison_bwd_tid + + @staticmethod + def _match_none_subsequence(base_ops: list, comparison_ops: list) -> list: + op_compare_result = [[op, None] for op in iter(base_ops)] + op_compare_result.extend([[None, op] for op in iter(comparison_ops)]) + return op_compare_result + + @staticmethod + def _split_operator_data(data_list, bwd_tid): + split_result = [] + if not data_list: + return split_result + data_list.sort(key=lambda x: x.start_time) + pre_tid = data_list[0].tid + part_data_dict = {Constant.IS_BWD: pre_tid == bwd_tid, Constant.OPS: []} + for op in data_list: + if op.tid == pre_tid or (pre_tid != bwd_tid and op.tid != bwd_tid): + part_data_dict[Constant.OPS].append(op) + else: + split_result.append(part_data_dict) + part_data_dict = {Constant.IS_BWD: op.tid == bwd_tid, Constant.OPS: [op]} + pre_tid = op.tid + split_result.append(part_data_dict) + return split_result + + def run(self, matching_type, base_data, comparison_data): + if matching_type == self.MODULE_TYPE: + return self._match_nn_module(base_data, comparison_data) + + if self._base_bwd_tid is None or self._comparison_bwd_tid is None: + return self._match_torch_op(base_data, comparison_data) + + base_data = self._split_operator_data(base_data, self._base_bwd_tid) + comparison_data = self._split_operator_data(comparison_data, self._comparison_bwd_tid) + if not base_data: + comparison_data_list = [] + for data in comparison_data: + comparison_data_list.extend(data.get(Constant.OPS, [])) + return self._match_torch_op([], comparison_data_list) + if not comparison_data: + base_data_list = [] + for data in base_data: + base_data_list.extend(data.get(Constant.OPS, [])) + return self._match_torch_op(base_data_list, []) + + result_data = [] + base_data_len, comparison_data_len = len(base_data), len(comparison_data) + if base_data[0].get(Constant.IS_BWD) == comparison_data[0].get(Constant.IS_BWD): + base_index, comparison_index = 0, 0 + elif base_data_len > comparison_data_len: + result_data.extend(self._match_torch_op(base_data[0].get(Constant.OPS, []), [])) + base_index, comparison_index = 1, 0 + else: + result_data.extend(self._match_torch_op([], comparison_data[0].get(Constant.OPS, []))) + base_index, comparison_index = 0, 1 + while base_index < base_data_len: + comparison_ops = [] if comparison_index >= comparison_data_len else comparison_data[ + comparison_index].get(Constant.OPS, []) + result_data.extend(self._match_torch_op(base_data[base_index].get(Constant.OPS, []), comparison_ops)) + base_index += 1 + comparison_index += 1 + while comparison_index < comparison_data_len: + result_data.extend(self._match_torch_op([], comparison_data[0].get(Constant.OPS, []))) + comparison_index += 1 + return result_data + + def _match_torch_op(self, base_ops, comparison_ops) -> list: + if not base_ops and not comparison_ops: + return [] + name_func = NameFunction(self._args).get_name_func() + op_compare_result = longest_common_subsequence_matching(base_ops, comparison_ops, name_func) \ + if not self._args.disable_details else self._match_none_subsequence(base_ops, comparison_ops) + if self._args.max_kernel_num is not None: + op_compare_result = self._drill_down(op_compare_result, name_func) + return op_compare_result + + def _drill_down(self, compare_result_data: list, name_func: any) -> list: + drill_down_result = [] + compare_result_data.reverse() + op_deque = deque(compare_result_data) + while op_deque: + match_data = op_deque.pop() + base_op = match_data[0] if match_data[0] else TorchOpNode() + comparison_op = match_data[1] if match_data[1] else TorchOpNode() + if not base_op.child_nodes or not comparison_op.child_nodes: + drill_down_result.append(match_data) + continue + if max(base_op.kernel_num, comparison_op.kernel_num) <= self._args.max_kernel_num: + drill_down_result.append(match_data) + continue + match_list = longest_common_subsequence_matching(base_op.child_nodes, + comparison_op.child_nodes, + name_func) \ + if not self._args.disable_details else self._match_none_subsequence(base_op.child_nodes, + comparison_op.child_nodes) + match_list.reverse() + op_deque.extend(match_list) + + return drill_down_result + + def _match_nn_module(self, base_root_node, comparison_root_node) -> list: + module_compare_result = [] + for index, base_node in enumerate(base_root_node): + comparison_node = comparison_root_node[index] if index < len(comparison_root_node) else None + if not base_node or not comparison_node: + continue + module_compare_result.extend(self._matching_all_modules(base_node, comparison_node)) + return module_compare_result + + def _matching_all_modules(self, base_node: ModuleNode, comparison_node: ModuleNode): + all_matched_modules = [] + matched_queue = deque() + matched_queue.append([base_node, comparison_node]) + while matched_queue: + matched_base_node, matched_comparison_node = matched_queue.popleft() + matched_node_list = self._matching_common_subsequence(matched_base_node, matched_comparison_node) + all_matched_modules.extend(matched_node_list) + for matched_node in matched_node_list: + matched_queue.append(matched_node) + return all_matched_modules + + def _matching_common_subsequence(self, base_node: ModuleNode, comparison_node: ModuleNode): + base_modules = base_node.child_nodes if base_node else [] + comparison_modules = comparison_node.child_nodes if comparison_node else [] + if not base_modules and not comparison_modules: + return [] + name_func = NameFunction(self._args).get_module_name + result = longest_common_subsequence_matching(base_modules, comparison_modules, name_func) \ + if not self._args.disable_details else self._match_none_subsequence(base_modules, comparison_modules) + return result diff --git a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py index 6fe693fb0..c0da4b65b 100644 --- a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -1,7 +1,5 @@ import os -from collections import deque from datetime import datetime -from queue import Queue from compare_backend.comparator.communication_comparator import CommunicationComparator from compare_backend.comparator.module_comparetor import ModuleComparator @@ -24,39 +22,25 @@ from compare_backend.compare_bean.overall_metrics_bean import OverallMetricsBean from compare_backend.data_prepare.module_data_prepare import ModuleDataPrepare from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare from compare_backend.generator.base_generator import BaseGenerator -from compare_backend.utils.common_func import longest_common_subsequence_matching from compare_backend.utils.constant import Constant -from compare_backend.utils.module_node import ModuleNode -from compare_backend.utils.name_function import NameFunction -from compare_backend.utils.torch_op_node import TorchOpNode from compare_backend.view.excel_view import ExcelView +from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching + class DetailPerformanceGenerator(BaseGenerator): def __init__(self, profiling_data_dict: dict, args: any): super().__init__(profiling_data_dict, args) - @classmethod - def _match_none_subsequence(cls, base_ops: list, comparison_ops: list) -> list: - op_compare_result = [[op, None] for op in iter(base_ops)] - op_compare_result.extend([[None, op] for op in iter(comparison_ops)]) - return op_compare_result - def compare(self): enable_compare = [self._args.enable_operator_compare, self._args.enable_memory_compare, self._args.enable_communication_compare, self._args.enable_api_compare, - self._args.enable_kernel_compare] + self._args.enable_kernel_compare, self._args.enable_profiling_compare] if any(enable_compare): print("[INFO] Start to compare performance detail data, please wait.") comparator_list = self._create_comparator() else: comparator_list = [] - if self._args.enable_profiling_compare: - overall_data = {Constant.BASE_DATA: self._profiling_data_dict.get(Constant.BASE_DATA).overall_metrics, - Constant.COMPARISON_DATA: self._profiling_data_dict.get( - Constant.COMPARISON_DATA).overall_metrics} - # overall 数据在最前面 - comparator_list.insert(0, OverallMetricsComparator(overall_data, OverallMetricsBean)) for comparator in comparator_list: self._result_data.update(comparator.generate_data()) @@ -71,45 +55,60 @@ class DetailPerformanceGenerator(BaseGenerator): def _create_comparator(self): comparator_list = [] - - op_compare_result = [] - - if self._args.enable_operator_compare: - module_compare_result = self.match_nn_module() if self._profiling_data_dict.get( - Constant.BASE_DATA).python_function_data and self._profiling_data_dict.get( - Constant.COMPARISON_DATA).python_function_data else [] - if not module_compare_result: - op_compare_result = self.match_torch_op() - - if self._args.enable_memory_compare and not op_compare_result: - op_compare_result = self.match_torch_op() - + # 总体性能拆解 + if self._args.enable_profiling_compare: + overall_data = { + Constant.BASE_DATA: self._profiling_data_dict.get(Constant.BASE_DATA).overall_metrics, + Constant.COMPARISON_DATA: self._profiling_data_dict.get(Constant.COMPARISON_DATA).overall_metrics + } + comparator_list.append(OverallMetricsComparator(overall_data, OverallMetricsBean)) + # 通信性能比对 if self._args.enable_communication_compare: communication_data = { Constant.BASE_DATA: self._profiling_data_dict.get(Constant.BASE_DATA).communication_dict, Constant.COMPARISON_DATA: self._profiling_data_dict.get(Constant.COMPARISON_DATA).communication_dict} comparator_list.append(CommunicationComparator(communication_data, CommunicationBean)) + # 算子性能比对-module级 + enable_operator_compare = False if self._args.enable_operator_compare: + module_compare_result = self._module_match() if module_compare_result: comparator_list.append(ModuleStatisticComparator(module_compare_result, ModuleStatisticBean)) if not self._args.disable_details: comparator_list.append(ModuleComparator(module_compare_result, ModuleCompareBean)) else: - comparator_list.append(OperatorStatisticComparator(op_compare_result, OperatorStatisticBean)) - if not self._args.disable_details: - comparator_list.append(OperatorComparator(op_compare_result, OperatorCompareBean)) + enable_operator_compare = True + + # build tree for operator_compare memory_compare and api_compare + base_op_prepare, comparison_op_prepare = None, None + if self._args.enable_memory_compare or self.enable_api_compare or enable_operator_compare: + base_op_prepare = OperatorDataPrepare(self._profiling_data_dict.get(Constant.BASE_DATA)) + comparison_op_prepare = OperatorDataPrepare(self._profiling_data_dict.get(Constant.COMPARISON_DATA)) + + # 算子性能比对-operator级 + op_compare_result = [] + if enable_operator_compare: + op_compare_result = self._operator_match(base_op_prepare.get_top_layer_ops(), + comparison_op_prepare.get_top_layer_ops()) + comparator_list.append(OperatorStatisticComparator(op_compare_result, OperatorStatisticBean)) + if not self._args.disable_details: + comparator_list.append(OperatorComparator(op_compare_result, OperatorCompareBean)) + # 算子内存比对 if self._args.enable_memory_compare: + if not op_compare_result: + op_compare_result = self._operator_match(base_op_prepare.get_top_layer_ops(), + comparison_op_prepare.get_top_layer_ops()) comparator_list.append(OperatorStatisticComparator(op_compare_result, MemoryStatisticBean)) if not self._args.disable_details: comparator_list.append(OperatorComparator(op_compare_result, MemoryCompareBean)) + # host api比对 if self._args.enable_api_compare: api_compare_result = { - Constant.BASE_DATA: OperatorDataPrepare( - self._profiling_data_dict.get(Constant.BASE_DATA)).get_all_layer_ops(), - Constant.COMPARISON_DATA: OperatorDataPrepare( - self._profiling_data_dict.get(Constant.COMPARISON_DATA)).get_all_layer_ops()} + Constant.BASE_DATA: base_op_prepare.get_all_layer_ops(), + Constant.COMPARISON_DATA: comparison_op_prepare.get_all_layer_ops()} comparator_list.append(ApiCompareComparator(api_compare_result, ApiCompareBean)) + # kernel比对 if self._args.enable_kernel_compare: kernel_compare_result = { Constant.BASE_DATA: self._profiling_data_dict.get(Constant.BASE_DATA).kernel_details, @@ -117,74 +116,19 @@ class DetailPerformanceGenerator(BaseGenerator): comparator_list.append(KernelCompareComparator(kernel_compare_result, KernelCompareBean)) return comparator_list - def match_torch_op(self) -> list: - base_ops = OperatorDataPrepare(self._profiling_data_dict.get(Constant.BASE_DATA)).get_top_layer_ops() - comparison_ops = OperatorDataPrepare( - self._profiling_data_dict.get(Constant.COMPARISON_DATA)).get_top_layer_ops() - if not base_ops and not comparison_ops: + def _module_match(self): + if not self._profiling_data_dict.get(Constant.BASE_DATA).python_function_data or not \ + self._profiling_data_dict.get(Constant.COMPARISON_DATA).python_function_data: return [] - name_func = NameFunction(self._args).get_name_func() - op_compare_result = longest_common_subsequence_matching(base_ops, comparison_ops, name_func) \ - if not self._args.disable_details else self._match_none_subsequence(base_ops, comparison_ops) - if self._args.max_kernel_num is not None: - op_compare_result = self._drill_down(op_compare_result, name_func) - return op_compare_result - - def _drill_down(self, compare_result_data: list, name_func: any) -> list: - drill_down_result = [] - compare_result_data.reverse() - op_deque = deque(compare_result_data) - while op_deque: - match_data = op_deque.pop() - base_op = match_data[0] if match_data[0] else TorchOpNode() - comparison_op = match_data[1] if match_data[1] else TorchOpNode() - if not base_op.child_nodes or not comparison_op.child_nodes: - drill_down_result.append(match_data) - continue - if max(base_op.kernel_num, comparison_op.kernel_num) <= self._args.max_kernel_num: - drill_down_result.append(match_data) - continue - match_list = longest_common_subsequence_matching(base_op.child_nodes, - comparison_op.child_nodes, - name_func) \ - if not self._args.disable_details else self._match_none_subsequence(base_op.child_nodes, - comparison_op.child_nodes) - match_list.reverse() - for data in match_list: - op_deque.append(data) - - return drill_down_result - - def match_nn_module(self) -> list: - module_compare_result = [] - base_root_node = ModuleDataPrepare(self._profiling_data_dict.get(Constant.BASE_DATA)).build_module_tree() + base_root_node = ModuleDataPrepare( + self._profiling_data_dict.get(Constant.BASE_DATA)).build_module_tree() comparison_root_node = ModuleDataPrepare( self._profiling_data_dict.get(Constant.COMPARISON_DATA)).build_module_tree() - for index, base_node in enumerate(base_root_node): - comparison_node = comparison_root_node[index] if index < len(comparison_root_node) else None - if not base_node or not comparison_node: - continue - module_compare_result.extend(self._matching_all_modules(base_node, comparison_node)) - return module_compare_result - - def _matching_all_modules(self, base_node: ModuleNode, comparison_node: ModuleNode): - all_matched_modules = [] - matched_queue = Queue() - matched_queue.put([base_node, comparison_node]) - while not matched_queue.empty(): - matched_base_node, matched_comparison_node = matched_queue.get() - matched_node_list = self._matching_common_subsequence(matched_base_node, matched_comparison_node) - all_matched_modules.extend(matched_node_list) - for matched_node in matched_node_list: - matched_queue.put(matched_node) - return all_matched_modules - - def _matching_common_subsequence(self, base_node: ModuleNode, comparison_node: ModuleNode): - base_modules = base_node.child_nodes if base_node else [] - comparison_modules = comparison_node.child_nodes if comparison_node else [] - if not base_modules and not comparison_modules: - return [] - name_func = NameFunction(self._args).get_module_name - result = longest_common_subsequence_matching(base_modules, comparison_modules, name_func) \ - if not self._args.disable_details else self._match_none_subsequence(base_modules, comparison_modules) - return result + return SequencePreMatching(self._args).run(SequencePreMatching.MODULE_TYPE, base_root_node, + comparison_root_node) + + def _operator_match(self, base_ops, comparison_ops): + base_bwd_tid = self._profiling_data_dict.get(Constant.BASE_DATA).bwd_tid + comparison_bwd_tid = self._profiling_data_dict.get(Constant.COMPARISON_DATA).bwd_tid + return SequencePreMatching(self._args, base_bwd_tid, comparison_bwd_tid).run(SequencePreMatching.OP_TYPE, + base_ops, comparison_ops) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 9daaa55ef..a2591dd0f 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -21,6 +21,7 @@ class ProfilingResult: self.python_function_data = [] self.fwdbwd_dict = {} self.kernel_details = {} + self.bwd_tid = None def update_torch_op_data(self, event: TraceEventBean): event.is_torch_op = True @@ -44,10 +45,13 @@ class ProfilingResult: def update_comm_task_data(self, comm_name: str, task_event: TraceEventBean): self.communication_dict.setdefault(comm_name, {}).setdefault("comm_task", {}).setdefault( task_event.name, []).append(task_event.dur) - + def update_kernel_details(self, kernels: dict): self.kernel_details = kernels + def update_bwd_tid(self, bwd_tid): + self.bwd_tid = bwd_tid + class BaseProfilingParser(ABC): @@ -115,6 +119,7 @@ class BaseProfilingParser(ABC): raise NotImplementedError("Function _get_dispatch_func need to be implemented.") def load_data(self) -> ProfilingResult: + self._result_data.update_bwd_tid(self._bwd_tid) self._dispatch_events() self._update_kernel_dict() self._update_communication_dict() diff --git a/profiler/compare_tools/compare_backend/utils/common_func.py b/profiler/compare_tools/compare_backend/utils/common_func.py index 68a1ab584..1ced3c0f8 100644 --- a/profiler/compare_tools/compare_backend/utils/common_func.py +++ b/profiler/compare_tools/compare_backend/utils/common_func.py @@ -41,6 +41,11 @@ def longest_common_subsequence_matching(base_ops: list, comparison_ops: list, na for index, value in enumerate(base_ops): result_data[index] = [value, None] return result_data + if not base_ops: + result_data = [None] * len(comparison_ops) + for index, value in enumerate(comparison_ops): + result_data[index] = [None, value] + return result_data comparison_len, base_len = len(comparison_ops), len(base_ops) if comparison_len * base_len > 50 * 10 ** 8: @@ -51,12 +56,12 @@ def longest_common_subsequence_matching(base_ops: list, comparison_ops: list, na cur_list = [0] * (base_len + 1) comparison_index = 1 - iter_comparison_data = iter(comparison_ops) - for comparison_data in iter_comparison_data: + all_base_data = [hash(name_func(op)) for op in base_ops] + all_comparison_data = [hash(name_func(op)) for op in comparison_ops] + for comparison_data in iter(all_comparison_data): base_index = 1 - iter_base_data = iter(base_ops) - for base_data in iter_base_data: - if name_func(comparison_data) == name_func(base_data): + for base_data in all_base_data: + if comparison_data == base_data: cur_list[base_index] = pre_list[base_index - 1] + 1 else: only_base = cur_list[base_index - 1] @@ -75,7 +80,7 @@ def longest_common_subsequence_matching(base_ops: list, comparison_ops: list, na while comparison_index > 0 and base_index > 0: base_data = base_ops[base_index - 1] comparison_data = comparison_ops[comparison_index - 1] - if name_func(base_data) == name_func(comparison_data): + if all_base_data[base_index - 1] == all_comparison_data[comparison_index - 1]: matched_op.append([base_data, comparison_data]) comparison_index -= 1 base_index -= 1 diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index 252aa536e..256dec117 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -91,3 +91,6 @@ class Constant(object): CPU_OP_MATMUL_MASK = ("aten::addmm", "aten::bmm", "aten::mm", "aten::matmul") KERNEL_CUBE_MASK = ("gemm", "conv", "cutlass", "wgrad") KERNEL_TRANS_MASK = ("cast", "transdata", "transpose") + + IS_BWD = "is_bwd" + OPS = "ops" diff --git a/profiler/compare_tools/compare_backend/utils/torch_op_node.py b/profiler/compare_tools/compare_backend/utils/torch_op_node.py index 69ee92d12..bb116a60c 100644 --- a/profiler/compare_tools/compare_backend/utils/torch_op_node.py +++ b/profiler/compare_tools/compare_backend/utils/torch_op_node.py @@ -24,6 +24,10 @@ class TorchOpNode: def name(self): return self._event.name + @property + def tid(self): + return self._event.tid + @property def input_shape(self): return str(self._event.args.get("Input Dims", Constant.NA)) @@ -67,7 +71,7 @@ class TorchOpNode: @property def api_dur(self): return self._event.dur - + @property def api_self_time(self): return self.api_dur - sum(child.api_dur for child in self._child_nodes) diff --git a/profiler/compare_tools/compare_backend/utils/tree_builder.py b/profiler/compare_tools/compare_backend/utils/tree_builder.py index d5aa787ac..b77011579 100644 --- a/profiler/compare_tools/compare_backend/utils/tree_builder.py +++ b/profiler/compare_tools/compare_backend/utils/tree_builder.py @@ -9,11 +9,13 @@ class TreeBuilder: @classmethod def build_tree(cls, event_list: list, kernel_dict: dict, memory_list: list) -> TorchOpNode: root_node = TorchOpNode() + all_nodes = [root_node] + ([None] * len(event_list)) all_event_list = [] all_event_list.extend(event_list) all_event_list.extend(memory_list) all_event_list.sort(key=lambda x: x.start_time) last_node = root_node + index = 1 for event in all_event_list: while last_node: if last_node != root_node and event.start_time > last_node.end_time: @@ -21,6 +23,8 @@ class TreeBuilder: continue if event.is_torch_op: tree_node = TorchOpNode(event, last_node) + all_nodes[index] = tree_node + index += 1 last_node.add_child_node(tree_node) last_node = tree_node if kernel_dict: @@ -29,7 +33,7 @@ class TreeBuilder: event.set_name(last_node.name) last_node.set_memory_allocated(event) break - return root_node + return all_nodes[:index] @classmethod def get_total_kernels(cls, root_node: TorchOpNode) -> list: diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py index 807346359..e84cfe048 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py @@ -26,6 +26,7 @@ class ProfilingParser(BaseProfilingParser): self._enable_communication_compare = True self._enable_kernel_compare = True self._enable_api_compare = True + self._bwd_tid = 1 def _update_kernel_details(self): pass diff --git a/profiler/test/ut/compare_tools/utils/test_tree_builder.py b/profiler/test/ut/compare_tools/utils/test_tree_builder.py index b9565b45e..326a424d3 100644 --- a/profiler/test/ut/compare_tools/utils/test_tree_builder.py +++ b/profiler/test/ut/compare_tools/utils/test_tree_builder.py @@ -18,11 +18,11 @@ class TestUtils(unittest.TestCase): for event in event_list: event.is_torch_op = True tree = TreeBuilder.build_tree(event_list, flow_kernel_dict, memory_allocated_list) - child_nodes = tree.child_nodes - self.assertEqual(len(tree._child_nodes), 2) + child_nodes = tree[0].child_nodes + self.assertEqual(len(tree[0].child_nodes), 2) self.assertEqual(child_nodes[0].start_time, 0) self.assertEqual(child_nodes[0].end_time, 1) self.assertEqual(child_nodes[0].kernel_num, 2) self.assertEqual(child_nodes[1].kernel_num, 0) - self.assertEqual(len(TreeBuilder.get_total_kernels(tree)), 2) - self.assertEqual(TreeBuilder.get_total_memory(tree)[0].size, 1) + self.assertEqual(len(TreeBuilder.get_total_kernels(tree[0])), 2) + self.assertEqual(TreeBuilder.get_total_memory(tree[0])[0].size, 1) -- Gitee From f9d3e0b8f55aa5a4a9a3d9a85a02c91cc5aca9f2 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 11:29:50 +0800 Subject: [PATCH 201/791] commit conflict --- debug/accuracy_tools/msprobe/pytorch/service.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 8d3e8fba2..c2749ecdf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -13,6 +13,9 @@ from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.utils import get_rank_if_initialized from msprobe.pytorch.hook_module import remove_dropout from msprobe.pytorch.hook_module.api_registry import api_register +from msprobe.pytorch.hook_module.hook_module import HOOKModule +from msprobe.pytorch.module_processer import ModuleProcesser +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTLConfig, ATTL, ApiData class Service: -- Gitee From 4a5629b81e91ae156fb2d896d6fe7e5de2d214f2 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 8 Aug 2024 11:39:04 +0800 Subject: [PATCH 202/791] change save tensor --- .../data_dump/data_processor/pytorch_processor.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 5672c3f9a..1c8ce67ce 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -12,6 +12,8 @@ from msprobe.core.common.const import Const, OverflowConst, FileCheckConst from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, ModuleBackwardInputsOutputs, \ ModuleForwardInputsOutputs, TensorStatInfo from msprobe.pytorch.free_benchmark import FreeBenchmarkCheck, UnequalRow +from msprobe.pytorch.common.utils import save_pt + try: import torch_npu @@ -167,12 +169,9 @@ class StatisticsDataProcessor(PytorchDataProcessor): class TensorDataProcessor(PytorchDataProcessor): def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) - if not path_len_exceeds_limit(file_path): - saved_tensor = tensor.contiguous().detach() - torch.save(saved_tensor, file_path) - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - else: - logger.warning(f'The file path {file_path} length exceeds limit.') + saved_tensor = tensor.contiguous().detach() + torch.save(saved_tensor, file_path) + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) single_arg = super()._analyze_tensor(tensor, suffix) single_arg.update({"data_name": dump_data_name}) return single_arg -- Gitee From 9bb7e3e82725cf14b56cfa13de58173b0dce47f3 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 8 Aug 2024 11:41:10 +0800 Subject: [PATCH 203/791] fix bug --- .../msprobe/core/data_dump/data_processor/pytorch_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 1c8ce67ce..b828d28de 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -170,7 +170,7 @@ class TensorDataProcessor(PytorchDataProcessor): def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) saved_tensor = tensor.contiguous().detach() - torch.save(saved_tensor, file_path) + save_pt(saved_tensor, file_path) change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) single_arg = super()._analyze_tensor(tensor, suffix) single_arg.update({"data_name": dump_data_name}) -- Gitee From 52c6fc63efc2734c1afaf5dc3351c3d32aec8237 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 13:00:57 +0800 Subject: [PATCH 204/791] fix online run_ut --- debug/accuracy_tools/msprobe/pytorch/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 54428806f..a9e446527 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -74,7 +74,7 @@ class Service: return None if self.config.online_run_ut: - if not self.data_collector.scope or self.data_collector.scope.check(api_or_module_name): + if self.data_collector.scope and not self.data_collector.scope.check(api_or_module_name): return None api_data = ApiData(name[:-1], args, kwargs, output, self.current_iter, self.current_rank) self.attl_send(api_data) @@ -99,7 +99,7 @@ class Service: return if self.config.online_run_ut: - if not self.data_collector.scope or self.data_collector.scope.check(api_or_module_name): + if self.data_collector.scope and not self.data_collector.scope.check(api_or_module_name): return None api_data = ApiData(name[:-1], grad_input, {}, grad_output, self.current_iter, self.current_rank) self.attl_send(api_data) -- Gitee From e20cfc1183fd98219706211abb1770b3b6dd0a6a Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 13:53:41 +0800 Subject: [PATCH 205/791] fix online run_ut --- debug/accuracy_tools/msprobe/pytorch/service.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index a9e446527..acea97815 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -60,6 +60,8 @@ class Service: if not self.switch: return args, kwargs + if self.config.online_run_ut: + return None, None if self.data_collector: module_input_output = ModuleForwardInputsOutputs(args=args, kwargs=kwargs, output=None) self.data_collector.pre_forward_data_collect(api_or_module_name, module, pid, module_input_output) @@ -100,10 +102,10 @@ class Service: if self.config.online_run_ut: if self.data_collector.scope and not self.data_collector.scope.check(api_or_module_name): - return None + return api_data = ApiData(name[:-1], grad_input, {}, grad_output, self.current_iter, self.current_rank) self.attl_send(api_data) - return None + return if self.data_collector: # 此处获取到的grad_input实际为反向过程的输出数据，grad_output为反向过程的输入数据，因此传入时调换顺序 @@ -134,7 +136,7 @@ class Service: if self.config.nfs_path: self.attl.upload("end") elif self.attl.socket_manager is not None: - logger.debug(f"进程{os.getpid()} 已完成,准备发送STOP信号") + logger.info(f"进程{os.getpid()} 已完成,准备发送STOP信号") self.attl.socket_manager.send_stop_signal() else: # current rank not need dump, wait @@ -153,7 +155,7 @@ class Service: api_register.api_modularity() self.switch = True logger.info_on_rank_0(f"Dump switch is turned on at step {self.current_iter}. ") - if self.config.level != "L2": + if self.config.level != "L2" and not self.config.online_run_ut: self.create_dirs() logger.info_on_rank_0(f"Dump data will be saved in {self.dump_iter_dir}.") @@ -165,6 +167,8 @@ class Service: if self.config.rank and self.current_rank not in self.config.rank: return self.switch = False + if self.config.online_run_ut: + return self.data_collector.write_json() def create_dirs(self): -- Gitee From 5f41cd17b76eea2f6d9ac2a1e68de5448c85f5ce Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 15:20:20 +0800 Subject: [PATCH 206/791] add tls online run_ut --- .../api_accuracy_checker/common/config.py | 6 ++-- .../pytorch/api_accuracy_checker/config.yaml | 2 +- .../api_accuracy_checker/run_ut/run_ut.py | 9 ++++-- .../tensor_transport_layer/attl.py | 13 +++++++-- .../tensor_transport_layer/client.py | 19 ++++++++++--- .../tensor_transport_layer/server.py | 20 +++++++++++-- .../tensor_transport_layer/ssl_config.py | 28 +++++++++++++++++++ .../pytorch/debugger/debugger_config.py | 1 + .../msprobe/pytorch/pt_config.py | 25 +++++++++++++++-- .../accuracy_tools/msprobe/pytorch/service.py | 3 +- 10 files changed, 107 insertions(+), 19 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index 3c61624b6..cf8af8d2c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -27,10 +27,10 @@ class Config: 'precision': int, 'is_online': bool, 'nfs_path': str, - 'is_benchmark_device': bool, 'host': str, 'port': int, - 'rank_list': list + 'rank_list': list, + 'tls_path': str } if key not in validators: raise ValueError(f"{key} must be one of {validators.keys()}") @@ -46,6 +46,8 @@ class Config: RunUTConfig.check_error_data_path_config(value) if key == 'nfs_path': RunUTConfig.check_nfs_path_config(value) + if key == 'tls_path': + RunUTConfig.check_tls_path_config(value) return value diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml index c2bb847b7..49f8a726d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/config.yaml @@ -4,7 +4,7 @@ error_data_path: './' precision: 14 is_online: False nfs_path: "" -is_benchmark_device: True host: "" port: -1 rank_list: [0] +tls_path: "" diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index aed8724a3..88e327e3c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -48,7 +48,7 @@ RunUTConfig = namedtuple('RunUTConfig', ['forward_content', 'backward_content', 'save_error_data', 'is_continue_run_ut', 'real_data_path', 'white_list', 'black_list', 'error_data_path', 'online_config']) -OnlineConfig = namedtuple('OnlineConfig', ['is_online', 'nfs_path', 'host', 'port', 'rank_list']) +OnlineConfig = namedtuple('OnlineConfig', ['is_online', 'nfs_path', 'host', 'port', 'rank_list', 'tls_path']) not_backward_list = ['repeat_interleave'] not_detach_set = {'resize_', 'resize_as_', 'set_', 'transpose_', 't_', 'squeeze_', 'unsqueeze_'} @@ -442,7 +442,8 @@ def init_attl(config): attl = ATTL('gpu', ATTLConfig(is_benchmark_device=True, connect_ip=config.host, connect_port=config.port, - nfs_path=config.nfs_path)) + nfs_path=config.nfs_path, + tls_path=config.tls_path)) return attl @@ -572,6 +573,7 @@ def run_ut_command(args): host = msCheckerConfig.host port = msCheckerConfig.port rank_list = msCheckerConfig.rank_list + tls_path = msCheckerConfig.tls_path if args.config_path: _, task_config = parse_json_config(args.config_path, Const.RUN_UT) white_list = task_config.white_list @@ -582,6 +584,7 @@ def run_ut_command(args): host = task_config.host port = task_config.port rank_list = task_config.rank_list + tls_path = task_config.tls_path if save_error_data: if args.result_csv_path: @@ -589,7 +592,7 @@ def run_ut_command(args): global UT_ERROR_DATA_DIR UT_ERROR_DATA_DIR = 'ut_error_data' + time_info error_data_path = initialize_save_error_data(error_data_path) - online_config = OnlineConfig(is_online, nfs_path, host, port, rank_list) + online_config = OnlineConfig(is_online, nfs_path, host, port, rank_list, tls_path) run_ut_config = RunUTConfig(forward_content, backward_content, result_csv_path, details_csv_path, save_error_data, args.result_csv_path, real_data_path, set(white_list), set(black_list), error_data_path, online_config) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index 18099a3d3..d3f506630 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -13,6 +13,7 @@ import torch from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.client import TCPClient from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server import TCPServer from msprobe.pytorch.common.utils import logger +from msprobe.pytorch.common.utils import save_pt from msprobe.core.common.utils import remove_path @@ -28,6 +29,7 @@ class ATTLConfig: connect_port: int # storage_config nfs_path: str = None + tls_path: str = None check_sum: bool = True queue_size: int = 50 @@ -49,12 +51,14 @@ class ATTL: self.socket_manager = TCPServer(self.session_config.connect_port, self.data_queue, - self.session_config.check_sum) + self.session_config.check_sum, + self.session_config.tls_path) self.socket_manager.start() elif need_dump: self.socket_manager = TCPClient(self.session_config.connect_ip, self.session_config.connect_port, - self.session_config.check_sum) + self.session_config.check_sum, + self.session_config.tls_path) self.socket_manager.start() def check_attl_config(self): @@ -140,7 +144,10 @@ class ATTL: else: file_path = os.path.join(self.session_config.nfs_path, buffer + f"_{int(time.time())}") - torch.save(buffer, file_path) + try: + save_pt(buffer, file_path) + except Exception as e: + self.logger.warning("there is something error in save_pt. please check it. %s", e) def download(self): for file_type in ("start*", "*.pt", "end*"): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py index 5a436915c..d9b0bd7d4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py @@ -9,10 +9,12 @@ from queue import Queue from threading import Thread from typing import Union -from twisted.internet import reactor, protocol, endpoints +from OpenSSL import SSL +from twisted.internet import ssl, reactor, protocol, endpoints from twisted.protocols.basic import FileSender from msprobe.pytorch.common.utils import logger +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.ssl_config import cipher_list class TCPDataItem: @@ -43,11 +45,12 @@ class TCPClient: RESEND_TIMER_TIME = 5 # 接收ACK超时定时器 RESEND_PENDING_TIME = 60 # 连续pending时间超过1分钟则放弃该数据 - def __init__(self, host="localhost", port=8000, check_sum=False): + def __init__(self, host="localhost", port=8000, check_sum=False, tls_path=None): self.send_queue = Queue(self.MAX_SENDING_QUEUE_SIZE) self.resend_dict = dict() self.host = host self.port = port + self.tls_path = tls_path self.factory = None self.sequence_number = 0 self.signal_exit = False @@ -86,8 +89,16 @@ class TCPClient: self.factory = MessageClientFactory() self.factory.protocol = cur_protocol - - endpoint = endpoints.TCP4ClientEndpoint(reactor, self.host, self.port) + if self.tls_path: + client_key = os.path.join(self.tls_path, "client.key") + client_crt = os.path.join(self.tls_path, "client.crt") + client_context_factory = ssl.DefaultOpenSSLContextFactory(client_key, client_crt, SSL.TLSv1_2_METHOD) + client_context_ = client_context_factory.getContext() + client_context_.set_cipher_list(cipher_list) + client_context_.set_options(SSL.OP_NO_RENEGOTIATION) + endpoint = endpoints.SSL4ClientEndpoint(reactor, self.host, self.port) + else: + endpoint = endpoints.TCP4ClientEndpoint(reactor, self.host, self.port) d = endpoint.connect(self.factory) d.addCallback(conn_callback) d.addErrback(conn_err_callback) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py index 6dba19056..f7883bc62 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py @@ -1,19 +1,23 @@ +import os.path import struct import hashlib import time import io +from OpenSSL import SSL from threading import Thread -from twisted.internet import reactor, protocol, endpoints +from twisted.internet import ssl, reactor, protocol, endpoints from msprobe.pytorch.common.utils import logger +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.ssl_config import cipher_list class TCPServer: - def __init__(self, port, shared_queue, check_sum=False) -> None: + def __init__(self, port, shared_queue, check_sum=False, tls_path=None) -> None: self.port = port self.shared_queue = shared_queue self.check_sum = check_sum + self.tls_path = tls_path self.factory = MessageServerFactory() self.reactor_thread = None @@ -23,7 +27,17 @@ class TCPServer: def start(self): self.factory.protocol = self.build_protocol - endpoint = endpoints.TCP4ServerEndpoint(reactor, self.port) + + if self.tls_path: + server_key = os.path.join(self.tls_path, "server.key") + server_crt = os.path.join(self.tls_path, "server.crt") + server_context_factory = ssl.DefaultOpenSSLContextFactory(server_key, server_crt, SSL.TLSv1_2_METHOD) + server_context_ = server_context_factory.getContext() + server_context_.set_cipher_list(cipher_list) + server_context_.set_options(SSL.OP_NO_RENEGOTIATION) + endpoint = endpoints.SSL4ServerEndpoint(reactor, self.port) + else: + endpoint = endpoints.TCP4ServerEndpoint(reactor, self.port) endpoint.listen(self.factory) self.reactor_thread = Thread(target=self.run_reactor, daemon=True) self.reactor_thread.start() diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py new file mode 100644 index 000000000..2bc200bac --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py @@ -0,0 +1,28 @@ +cipher_list = ":".join([ + 'TLS_DHE_RSA_WITH_AES_128_GCM_SHA256', + 'TLS_DHE_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_DHE_DSS_WITH_AES_128_GCM_SHA256', + 'TLS_DHE_DSS_WITH_AES_256_GCM_SHA384', + 'TLS_PSK_WITH_AES_256_GCM_SHA384', + 'TLS_DHE_PSK_WITH_AES_128_GCM_SHA256', + 'TLS_DHE_PSK_WITH_AES_256_GCM_SHA384', + 'TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256', + 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', + 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', + 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', + 'TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256', + 'TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256', + 'TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256', + 'TLS_DHE_RSA_WITH_AES_128_CCM', + 'TLS_DHE_RSA_WITH_AES_256_CCM' + 'TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256', + 'TLS_PSK_WITH_AES_256_CCM', + 'TLS_DHE_PSK_WITH_AES_128_CCM', + 'TLS_DHE_PSK_WITH_AES_256_CCM', + 'TLS_ECDHE_ECDSA_WITH_AES_128_CCM', + 'TLS_ECDHE_ECDSA_WITH_AES_256_CCM', + 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' +]).encode() diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index 2303c76fc..9bed41dba 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -39,6 +39,7 @@ class DebuggerConfig: # dump api tensor and collaborate with online run_ut self.online_run_ut = task_config.online_run_ut if task_config.online_run_ut else False self.nfs_path = task_config.nfs_path if task_config.nfs_path else "" + self.tls_path = task_config.tls_path if task_config.tls_path else "" self.host = task_config.host if task_config.host else "" self.port = task_config.port if task_config.port else -1 diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index f771473c1..115cb0001 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -14,13 +14,24 @@ class TensorConfig(BaseConfig): self.nfs_path = json_config.get("nfs_path", "") self.host = json_config.get("host", "") self.port = json_config.get("port", -1) + self.tls_path = json_config.get("tls_path", "") self.check_config() self._check_file_format() + self._check_tls_path_config() def _check_file_format(self): if self.file_format is not None and self.file_format not in ["npy", "bin"]: raise Exception("file_format is invalid") + def _check_tls_path_config(self): + if self.tls_path: + if not os.path.exists(self.tls_path): + raise Exception("tls_path: %s does not exist" % self.tls_path) + if not os.path.exists(os.path.join(self.tls_path, "client.key")): + raise Exception("tls_path does not contain client.key") + if not os.path.exists(os.path.join(self.tls_path, "client.crt")): + raise Exception("tls_path does not contain client.crt") + class StatisticsConfig(BaseConfig): def __init__(self, json_config): @@ -77,10 +88,9 @@ class RunUTConfig(BaseConfig): self.error_data_path = json_config.get("error_data_path", Const.DEFAULT_PATH) self.is_online = json_config.get("is_online", False) self.nfs_path = json_config.get("nfs_path", "") - self.is_benchmark_device = json_config.get("is_benchmark_device", True) self.host = json_config.get("host", "") self.port = json_config.get("port", -1) - self.rank_list = json_config.get("rank_list", Const.DEFAULT_LIST) + self.tls_path = json_config.get("tls_path", "") self.check_run_ut_config() @classmethod @@ -103,11 +113,22 @@ class RunUTConfig(BaseConfig): if nfs_path and not os.path.exists(nfs_path): raise Exception("nfs_path: %s does not exist" % nfs_path) + @classmethod + def check_tls_path_config(cls, tls_path): + if tls_path: + if not os.path.exists(tls_path): + raise Exception("tls_path: %s does not exist" % tls_path) + if not os.path.exists(os.path.join(tls_path, "server.key")): + raise Exception("tls_path does not contain server.key") + if not os.path.exists(os.path.join(tls_path, "server.crt")): + raise Exception("tls_path does not contain server.crt") + def check_run_ut_config(self): RunUTConfig.check_filter_list_config(Const.WHITE_LIST, self.white_list) RunUTConfig.check_filter_list_config(Const.BLACK_LIST, self.black_list) RunUTConfig.check_error_data_path_config(self.error_data_path) RunUTConfig.check_nfs_path_config(self.nfs_path) + RunUTConfig.check_tls_path_config(self.tls_path) class GradToolConfig(BaseConfig): diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index acea97815..afcac50db 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -41,7 +41,8 @@ class Service: attl_config = ATTLConfig(is_benchmark_device=False, connect_ip=self.config.host, connect_port=self.config.port, - nfs_path=self.config.nfs_path) + nfs_path=self.config.nfs_path, + tls_path=self.config.tls_path) need_dump = len(self.config.rank) == 0 or self.current_rank in self.config.rank self.attl = ATTL('npu', attl_config, need_dump=need_dump) if self.config.nfs_path: -- Gitee From b3844f0ee25bbffcd1d16035f134bfc8553f5ac7 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 15:22:42 +0800 Subject: [PATCH 207/791] =?UTF-8?q?=E8=A7=A3=E8=80=A6mindspore=E5=92=8Cpyt?= =?UTF-8?q?orch=E5=9C=A8=E8=A2=ABmsprobe=E8=B0=83=E7=94=A8=E6=97=B6?= =?UTF-8?q?=E4=BC=9A=E4=BA=92=E7=9B=B8=E4=BE=9D=E8=B5=96=E7=9A=84=E6=83=85?= =?UTF-8?q?=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/msprobe.py | 50 +++++++++++++++++-------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 4bc841654..802913814 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -15,16 +15,15 @@ import argparse import sys -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command -from msprobe.pytorch.parse_tool.cli import parse as cli_parse -from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut -from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, \ - _api_precision_compare_command -from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ - _run_overflow_check_command +import importlib.util from msprobe.core.compare.utils import _compare_parser -from msprobe.pytorch.compare.compare_cli import compare_cli -from msprobe.mindspore.compare.compare_cli import compare_cli_ms +from msprobe.core.common.log import logger + + +def is_module_available(module_name): + spec =importlib.util.find_spec(module_name) + return spec is not None + def main(): parser = argparse.ArgumentParser( @@ -33,6 +32,7 @@ def main(): "Providing one-site accuracy difference debugging toolkit for training on Ascend Devices.\n" f"For any issue, refer README.md first", ) + parser.set_defaults(print_help=parser.print_help) parser.add_argument('-f', '--framework', required=True, choices=['pytorch', 'mindspore'], help='Deep learning framework.') @@ -43,18 +43,32 @@ def main(): multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut') api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare') run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check') - _compare_parser(compare_cmd_parser) - _run_ut_parser(run_ut_cmd_parser) - _run_ut_parser(multi_run_ut_cmd_parser) multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, help='Number of splits for parallel processing. Range: 1-64') - _api_precision_compare_parser(api_precision_compare_cmd_parser) - _run_overflow_check_parser(run_overflow_check_cmd_parser) + + _compare_parser(compare_cmd_parser) + if len(sys.argv) == 1: parser.print_help() sys.exit(0) args = parser.parse_args(sys.argv[1:]) if sys.argv[2] == "pytorch": + if is_module_available("torch"): + from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command + from msprobe.pytorch.parse_tool.cli import parse as cli_parse + from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut + from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, \ + _api_precision_compare_command + from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ + _run_overflow_check_command + from msprobe.pytorch.compare.compare_cli import compare_cli + _run_ut_parser(run_ut_cmd_parser) + _run_ut_parser(multi_run_ut_cmd_parser) + _api_precision_compare_parser(api_precision_compare_cmd_parser) + _run_overflow_check_parser(run_overflow_check_cmd_parser) + else: + logger.error("Pytorch does not exit, please install pytorch library") + raise Exception() if sys.argv[3] == "run_ut": run_ut_command(args) elif sys.argv[3] == "parse": @@ -69,7 +83,13 @@ def main(): elif sys.argv[3] == "compare": compare_cli(args) else: - compare_cli_ms(args) + if is_module_available("mindspore"): + from msprobe.mindspore.compare.compare_cli import compare_cli_ms + else: + logger.error("Mindspore does not exit, please install mindspore library") + raise Exception() + if sys.argv[3] == "compare": + compare_cli_ms(args) if __name__ == "__main__": main() -- Gitee From 857954958b84b5d048545a759e7d86f68ab9c38d Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 8 Aug 2024 15:49:41 +0800 Subject: [PATCH 208/791] fix bug --- .../msprobe/core/data_dump/data_processor/pytorch_processor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index b828d28de..ba71fe1dc 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -171,7 +171,6 @@ class TensorDataProcessor(PytorchDataProcessor): dump_data_name, file_path = self.get_save_file_path(suffix) saved_tensor = tensor.contiguous().detach() save_pt(saved_tensor, file_path) - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) single_arg = super()._analyze_tensor(tensor, suffix) single_arg.update({"data_name": dump_data_name}) return single_arg -- Gitee From bf856dcb1f4d2d7a4ad3427bf32b0acabd34bd17 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Wed, 7 Aug 2024 16:22:18 +0800 Subject: [PATCH 209/791] Align input of backward --- .../data_processor/pytorch_processor.py | 2 +- .../free_benchmark/compare/grad_saver.py | 44 +++++++++---------- .../perturbed_layers/npu/improve_precision.py | 4 +- 3 files changed, 23 insertions(+), 27 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 007fec809..e7c8056b2 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -303,7 +303,7 @@ class FreeBenchmarkDataProcessor(PytorchDataProcessor): self._forward_new_output = new_output def analyze_backward(self, name, module, module_input_output: ModuleBackwardInputsOutputs): - self.checker.backward(name, module, module_input_output.grad_output) + self.checker.backward(name, module, module_input_output.grad_input) class KernelDumpDataProcessor(PytorchDataProcessor): diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py index 6781a1c2f..21f2b3b46 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py @@ -16,7 +16,6 @@ class GradSaver: self.handler_params = handler_params self.api_name = handler_params.api_name self.origin_func = origin_func - self.data_params = DataParams() self.is_compare = True self.kwargs = dict() self.perturbed_grad_input = tuple() @@ -62,27 +61,25 @@ class GradSaver: def compare_grad_results(self, handler, origin_grad, perturbed_grad, index): # TODO get dtype? - self.data_params.original_result = origin_grad - self.data_params.perturbed_result = perturbed_grad - self.data_params.grad_unequal_flag = False - self.data_params.valid_input_index = index + data_params = DataParams() + data_params.original_result = origin_grad + data_params.perturbed_result = perturbed_grad + data_params.grad_unequal_flag = False + data_params.valid_input_index = index try: - handler.handle(self.data_params) - if not self.data_params.is_consistent: + handler.handle(data_params) + if not data_params.is_consistent: self.is_compare = False - self.data_params.grad_unequal_flag = True - self.data_params.is_consistent = True - self.data_params.perturbed_result = self.perturbed_grad_input - self.data_params.original_result = self.origin_grad_input - handler.handle(self.data_params) + data_params.grad_unequal_flag = True + data_params.is_consistent = True + data_params.perturbed_result = self.perturbed_grad_input + data_params.original_result = self.origin_grad_input + handler.handle(data_params) except Exception as e: logger.warning_on_rank_0( f"[msprobe] Free benchmark: compare two vjp failed: api:{self.handler_params.api_name}." f"{e}" ) - # 在扰动前后输出对比后释放输出的引用 - self.data_params.perturbed_result = None - self.data_params.original_result = None def check_grad_input(self, origin_grad, new_grad_index): if self.perturbed_grad_input is None: @@ -164,20 +161,19 @@ class GradSaver: return grad_input def calculate_perturbed_grad_input(self, grad_output, need_grad_tensors, inner_args): - self.data_params.args = [need_grad_tensors, grad_output, inner_args] - self.data_params.kwargs = {} - self.data_params.valid_input_index = 0 - self.data_params.origin_func = self.get_grad_input_from_vjp + data_params = DataParams() + data_params.args = [need_grad_tensors, grad_output, inner_args] + data_params.kwargs = {} + data_params.valid_input_index = 0 + data_params.origin_func = self.get_grad_input_from_vjp layer = LayerFactory.create( self.handler_params.api_name, self.handler_params.fuzz_device, self.handler_params.pert_mode, ) - layer.handle(self.data_params) - # 在计算扰动输出之后，释放输入的引用 - self.data_params.args = None + layer.handle(data_params) # 确定扰动成功后，才会暂存 - if self.data_params.perturbed_result: + if data_params.perturbed_result: self.perturbed_grad_input = tuple( - [x.cpu() for x in self.data_params.perturbed_result] + [x.cpu() for x in data_params.perturbed_result] ) diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py index ad6d8b898..b455c202e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/perturbed_layers/npu/improve_precision.py @@ -17,7 +17,7 @@ class ImprovePrecisionLayer(NpuBaseLayer): and torch.is_floating_point(tensor_obj) and tensor_obj.dtype not in [torch.float32, torch.float64] ): - self._set_improve_valus(tensor_obj) + self._set_improve_values(tensor_obj) tensor_obj = self._change_dtype(tensor_obj) self.is_added = True return tensor_obj @@ -50,7 +50,7 @@ class ImprovePrecisionLayer(NpuBaseLayer): params.perturbed_result = params.origin_func(*new_args, **new_kwargs) return params.perturbed_result - def _set_improve_valus(self, inputs): + def _set_improve_values(self, inputs): if inputs.dtype in [torch.float16, torch.bfloat16]: self.perturbed_value = torch.float32 -- Gitee From 2c50fda1104810925fb566b150aca9467144f3a0 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 16:06:48 +0800 Subject: [PATCH 210/791] =?UTF-8?q?=E5=88=A0=E9=99=A4=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E5=AF=B9=E8=B1=A1=EF=BC=8C=E8=B0=83=E6=95=B4=E7=B1=BB=E6=96=B9?= =?UTF-8?q?=E6=B3=95=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 015e33228..df5ff18b2 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -11,7 +11,7 @@ class Comparator: pass @classmethod - def match_op(self,npu_queue, bench_queue, fuzzy_match): + def match_op(cls,npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): if check_op(npu_queue[-1], b_op, fuzzy_match): return len(npu_queue) - 1, b_index @@ -55,6 +55,4 @@ class Comparator: err_msg += " Fuzzy matching data, the comparison accuracy may be affected." result_list.append(err_msg) return result_list - -testComparator= Comparator() -- Gitee From b1b8c0390c6b1265d369a8268a341739e4dcd4ac Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 16:51:53 +0800 Subject: [PATCH 211/791] add tls online run_ut --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 4 ++-- .../api_accuracy_checker/tensor_transport_layer/client.py | 2 +- .../api_accuracy_checker/tensor_transport_layer/server.py | 2 +- debug/accuracy_tools/msprobe/pytorch/pt_config.py | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 88e327e3c..a1e2e64a9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -206,8 +206,8 @@ def run_ut(config): for result_csv_path, details_csv_path in zip(compare.save_path_list, compare.detail_save_path_list): change_mode(result_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) change_mode(details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) - logger.info()(f"UT task result csv is saved in {result_csv_path}") - logger.info()(f"UT task details csv is saved in {details_csv_path}") + logger.info(f"UT task result csv is saved in {result_csv_path}") + logger.info(f"UT task details csv is saved in {details_csv_path}") compare.print_pretest_result() diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py index d9b0bd7d4..df7abc188 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py @@ -96,7 +96,7 @@ class TCPClient: client_context_ = client_context_factory.getContext() client_context_.set_cipher_list(cipher_list) client_context_.set_options(SSL.OP_NO_RENEGOTIATION) - endpoint = endpoints.SSL4ClientEndpoint(reactor, self.host, self.port) + endpoint = endpoints.SSL4ClientEndpoint(reactor, self.host, self.port, client_context_factory) else: endpoint = endpoints.TCP4ClientEndpoint(reactor, self.host, self.port) d = endpoint.connect(self.factory) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py index f7883bc62..690ffea3e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py @@ -35,7 +35,7 @@ class TCPServer: server_context_ = server_context_factory.getContext() server_context_.set_cipher_list(cipher_list) server_context_.set_options(SSL.OP_NO_RENEGOTIATION) - endpoint = endpoints.SSL4ServerEndpoint(reactor, self.port) + endpoint = endpoints.SSL4ServerEndpoint(reactor, self.port, server_context_factory) else: endpoint = endpoints.TCP4ServerEndpoint(reactor, self.port) endpoint.listen(self.factory) diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 115cb0001..bb82f13b8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -90,6 +90,7 @@ class RunUTConfig(BaseConfig): self.nfs_path = json_config.get("nfs_path", "") self.host = json_config.get("host", "") self.port = json_config.get("port", -1) + self.rank_list = json_config.get("rank_list", Const.DEFAULT_LIST) self.tls_path = json_config.get("tls_path", "") self.check_run_ut_config() -- Gitee From abc25da9aa7f99d9445f9b493e9c47de3dcae00e Mon Sep 17 00:00:00 2001 From: Mrtutu Date: Tue, 6 Aug 2024 22:07:04 +0800 Subject: [PATCH 212/791] add parallel --- profiler/cluster_analyse/README.md | 6 + .../analysis/step_trace_time_analysis.py | 39 +++++- .../cluster_utils/parallel_algorithm.py | 120 ++++++++++++++++++ .../parallel_strategy_calculator.py | 119 +++++++++++++++++ .../cluster_analyse/common_func/constant.py | 3 + .../common_func/tables_config.py | 5 +- .../test_parallel_strategy_calculator.py | 46 +++++++ 7 files changed, 336 insertions(+), 2 deletions(-) create mode 100644 profiler/cluster_analyse/cluster_utils/parallel_algorithm.py create mode 100644 profiler/cluster_analyse/cluster_utils/parallel_strategy_calculator.py create mode 100644 profiler/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py diff --git a/profiler/cluster_analyse/README.md b/profiler/cluster_analyse/README.md index 4a394e09a..785056252 100644 --- a/profiler/cluster_analyse/README.md +++ b/profiler/cluster_analyse/README.md @@ -98,6 +98,12 @@ K列：Communication（Not Overlapped and Exclude Receive）指剔除recieve算 L列：Preparing，指迭代开始到首个计算或通信算子运行的时间。 +M列：DP Index，指集群数据按照并行策略切分后所属DP组的索引， 如果没有采集则不显示。 + +N列：PP Index，指集群数据按照并行策略切分后所属PP组的索引，如果没有采集则不显示。 + +O列：TP Index，指集群数据按照并行策略切分后所属TP组的索引，如果没有采集则不显示。 + **Tips**：先筛选B列type为stage， 看stage间是否有问题，再筛选B列type为rank，看rank是否有问题，根据以下几点排查。 * 根据Computing的时间差异判断是否有慢卡，或者有负载不均衡的现象。 diff --git a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py index 6a886fffa..617c0aafc 100644 --- a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py @@ -19,11 +19,14 @@ from common_func.db_manager import DBManager from common_func.constant import Constant from common_func.file_manager import FileManager from prof_bean.step_trace_time_bean import StepTraceTimeBean +from cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator class StepTraceTimeAnalysis: CLUSTER_TRACE_TIME_CSV = "cluster_step_trace_time.csv" CLUSTER_TRACE_TIME_TABLE = "ClusterStepTraceTime" + PROFILER_METADATA_JSON = "profiler_metadata.json" + PARALLEL_HEADERS = ["DP Index", "PP Index", "TP Index"] def __init__(self, param: dict): self.collection_path = param.get(Constant.COLLECTION_PATH) @@ -32,6 +35,7 @@ class StepTraceTimeAnalysis: self.step_time_dict = {} self.step_data_list = [] self.data_type = param.get(Constant.DATA_TYPE) + self.distributed_args = None @staticmethod def get_max_data_row(data_group_list: list): @@ -48,8 +52,35 @@ class StepTraceTimeAnalysis: def run(self): self.load_step_trace_time_data() self.analyze_step_time() + self.partition_ranks_data() self.dump_data() + def partition_ranks_data(self): + if not self.distributed_args: + return + + calculator = ParallelStrategyCalculator(**self.distributed_args) + parallelism_map = calculator.run() + + if len(parallelism_map) > len(self.step_time_dict): + missing_rank_ids = [rank_id for rank_id in range(len(parallelism_map)) + if rank_id not in self.step_time_dict] + print(f"[WARNING] Step trace data length should equal to real rank numbers, " + f"but get step data length = {len(self.step_time_dict)}, real rank numbers = {len(parallelism_map)}, " + f"maybe lost some rank ids = {missing_rank_ids}, please check your profiling data.") + + if len(parallelism_map) < len(self.step_time_dict): + print(f"[ERROR] Step trace data length should equal to real rank numbers, " + f"but get step data length = {len(self.step_time_dict)}, real rank numbers = {len(parallelism_map)}, " + f"maybe parallel params in profiler_metadata.json is error, please check your metadata data.") + self.distributed_args = None + return + + for step_data in self.step_data_list: + rank_id = step_data[2] + step_data.extend(list(parallelism_map[rank_id]) + if parallelism_map[rank_id] else ['NA'] * len(self.PARALLEL_HEADERS)) + def dump_data(self): if not self.step_data_list: print("[WARNING] Can't get step time info!") @@ -74,6 +105,10 @@ class StepTraceTimeAnalysis: def load_step_trace_time_data(self): for rank_id, profiling_dir_path in self.data_map.items(): + metadata_path = os.path.join(profiling_dir_path, self.PROFILER_METADATA_JSON) + if not self.distributed_args and os.path.exists(metadata_path): + metadata = FileManager.read_json_file(metadata_path) + self.distributed_args = metadata.get(Constant.DISTRIBUTED_ARGS, None) if metadata else None if self.data_type == Constant.TEXT: step_time_file = os.path.join(profiling_dir_path, Constant.SINGLE_OUTPUT, Constant.STEP_TIME_CSV) if os.path.exists(step_time_file): @@ -121,6 +156,8 @@ class StepTraceTimeAnalysis: def get_headers(self): if self.step_time_dict: for rank in self.step_time_dict: - if self.step_time_dict.get(rank): + if self.step_time_dict.get(rank) and self.distributed_args: + return self.step_time_dict[rank][0].all_headers + self.PARALLEL_HEADERS + elif self.step_time_dict.get(rank): return self.step_time_dict[rank][0].all_headers return [] diff --git a/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py b/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py new file mode 100644 index 000000000..9da829bbd --- /dev/null +++ b/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py @@ -0,0 +1,120 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from abc import ABC, abstractmethod + + +class ParallelAlgorithm(ABC): + @abstractmethod + def partition(self): + pass + + +class MegatronAlgorithm(ParallelAlgorithm): + def __init__(self, + world_size: int = 1, + tensor_model_parallel_size: int = 1, + pipeline_model_parallel_size: int = 1, + data_parallel_size: int = 1, + context_parallel_size: int = 1, + expert_model_parallel_size: int = 1, + **kwargs): + + if data_parallel_size % expert_model_parallel_size != 0: + raise RuntimeError( + f"data_parallel_size is not divisible by " + f"expert_model_parallel_size, get data_parallel_size = {data_parallel_size}, " + f"expert_model_parallel_size = {expert_model_parallel_size}" + ) + + if data_parallel_size * context_parallel_size % expert_model_parallel_size != 0: + raise RuntimeError( + f"data_parallel_size * context_parallel_size {data_parallel_size * context_parallel_size} " + f"is not divisible by expert_model_parallel_size " + ) + + if world_size != tensor_model_parallel_size * pipeline_model_parallel_size * data_parallel_size: + raise RuntimeError( + f"world_size must be equal to tensor_model_parallel_size * " + f"pipeline_model_parallel_size * data_parallel_size, but get world_size = {world_size}, " + f"tensor_model_parallel_size = {tensor_model_parallel_size}, " + f"pipeline_model_parallel_size = {pipeline_model_parallel_size}, " + f"data_parallel_size = {data_parallel_size}" + ) + + self.world_size = world_size + self.tensor_model_parallel_size = tensor_model_parallel_size + self.pipeline_model_parallel_size = pipeline_model_parallel_size + self.data_parallel_size = data_parallel_size + self.context_parallel_size = context_parallel_size + self.expert_model_parallel_size = expert_model_parallel_size + + self.num_tensor_model_parallel_groups = self.world_size // tensor_model_parallel_size + self.num_pipeline_model_parallel_groups = self.world_size // pipeline_model_parallel_size + self.num_data_parallel_groups = self.world_size // data_parallel_size + + self.all_data_parallel_group_ranks = [] + self.all_data_parallel_group_ranks_with_cp = [] + self.all_model_parallel_group_ranks = [] + self.all_tensor_model_parallel_ranks = [] + self.all_expert_parallel_ranks = [] + self.all_pipeline_model_parallel_ranks = [] + + def partition(self): + self._build_dp_group() + self._build_tp_group() + self._build_pp_group() + self._build_ep_group() + + def _build_dp_group(self): + # Build the data-parallel groups + for i in range(self.pipeline_model_parallel_size): + begin_rank = self.num_pipeline_model_parallel_groups * i + end_rank = self.num_pipeline_model_parallel_groups * (i + 1) + for k in range(self.tensor_model_parallel_size * self.context_parallel_size): + ranks = range(begin_rank + k, + end_rank, self.tensor_model_parallel_size * self.context_parallel_size) + self.all_data_parallel_group_ranks.append(list(ranks)) + + for k in range(self.tensor_model_parallel_size): + ranks_with_cp = range(begin_rank + k, + end_rank, self.tensor_model_parallel_size) + self.all_data_parallel_group_ranks_with_cp.append(list(ranks_with_cp)) + + # Build the model-parallel groups + for i in range(self.data_parallel_size): + ranks = [data_parallel_group_ranks[i] + for data_parallel_group_ranks in self.all_data_parallel_group_ranks] + self.all_model_parallel_group_ranks.append(list(ranks)) + + def _build_tp_group(self): + # Build the tensor model-parallel groups. + for i in range(self.num_tensor_model_parallel_groups): + ranks = range(i * self.tensor_model_parallel_size, + (i + 1) * self.tensor_model_parallel_size) + self.all_tensor_model_parallel_ranks.append(list(ranks)) + + def _build_pp_group(self): + # Build the pipeline model-parallel groups. + for p in range(self.num_pipeline_model_parallel_groups): + ranks = range(p, self.world_size, + self.num_pipeline_model_parallel_groups) + self.all_pipeline_model_parallel_ranks.append(list(ranks)) + + def _build_ep_group(self): + # Build the expert model-parallel groups. + for dp_cp_ranks in self.all_data_parallel_group_ranks_with_cp: + for i in range(0, len(dp_cp_ranks), self.expert_model_parallel_size): + ranks = dp_cp_ranks[i:i + self.expert_model_parallel_size] + self.all_expert_parallel_ranks.append(list(ranks)) diff --git a/profiler/cluster_analyse/cluster_utils/parallel_strategy_calculator.py b/profiler/cluster_analyse/cluster_utils/parallel_strategy_calculator.py new file mode 100644 index 000000000..0f0a1809d --- /dev/null +++ b/profiler/cluster_analyse/cluster_utils/parallel_strategy_calculator.py @@ -0,0 +1,119 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +from enum import Enum +from dataclasses import dataclass + +from .parallel_algorithm import MegatronAlgorithm + + +class ParallelAlgorithmType(Enum): + Megatron = 0 + + +@dataclass +class RankMetrics: + computing: float = 0.0 + communication: float = 0.0 + free: float = 0.0 + + +class RankNode: + def __init__(self, + index: int, + rank_ids: list, + category: str, + metrics: RankMetrics): + self.index = index + self.rank_ids = rank_ids + self.category = category + self.metrics = metrics + self.children = [] + + def add_child(self, child_node): + if isinstance(child_node, RankNode): + self.children.append(child_node) + else: + raise TypeError("Child must be an instance of TreeNode") + + +class ParallelStrategyCalculator: + ROOT_LABEL = "ROOT" + TP_LABEL = "TP" + PP_LABEL = "PP" + DP_LABEL = "DP" + + parallel_algorithms = { + ParallelAlgorithmType.Megatron: MegatronAlgorithm + } + + def __init__(self, + algorithm_type: ParallelAlgorithmType = ParallelAlgorithmType.Megatron, + **kwargs): + + self.algorithm = self.parallel_algorithms.get(algorithm_type, MegatronAlgorithm)(**kwargs) + + # result of partition rank id to DP Index, PP Index, TP Index + self.ranks_ptd_map = [None] * self.algorithm.world_size + self.root_node = None + + def run(self): + self.algorithm.partition() + self._build_tree() + self._dfs(self.root_node) + return self.ranks_ptd_map + + def _build_tree(self): + if not self.algorithm.all_model_parallel_group_ranks: + return + + self.root_node = RankNode(-1, self.algorithm.all_model_parallel_group_ranks, + ParallelStrategyCalculator.ROOT_LABEL, RankMetrics()) + + # DP Level + for i, dp_group in enumerate(self.algorithm.all_model_parallel_group_ranks): + dp_node = RankNode(i, dp_group, ParallelStrategyCalculator.DP_LABEL, RankMetrics()) + + # PP Level + for pp_idx, j in enumerate(range(0, len(dp_group), self.algorithm.tensor_model_parallel_size)): + pp_group = dp_group[j:j + self.algorithm.tensor_model_parallel_size] + pp_node = RankNode(pp_idx, pp_group, ParallelStrategyCalculator.PP_LABEL, RankMetrics()) + + # TP Level + for k, tp_rank in enumerate(pp_group): + tp_node = RankNode(k, [tp_rank], + ParallelStrategyCalculator.TP_LABEL, RankMetrics()) + pp_node.add_child(tp_node) + + dp_node.add_child(pp_node) + self.root_node.add_child(dp_node) + + def _dfs(self, + rank_node: RankNode, + parent_node: RankNode = None, + grandparent_node: RankNode = None): + + if rank_node is None: + return + + if not rank_node.children: + if rank_node.rank_ids: + self.ranks_ptd_map[rank_node.rank_ids[0]] = ( + grandparent_node.index, # DP Index + parent_node.index, # PP Index + rank_node.index # TP Index + ) + + for child in rank_node.children: + self._dfs(child, rank_node, parent_node) diff --git a/profiler/cluster_analyse/common_func/constant.py b/profiler/cluster_analyse/common_func/constant.py index 2922d6a90..a5b93b0ca 100644 --- a/profiler/cluster_analyse/common_func/constant.py +++ b/profiler/cluster_analyse/common_func/constant.py @@ -106,3 +106,6 @@ class Constant(object): CONFIG = "config" EXPER_CONFIG = "experimental_config" EXPORT_TYPE = "_export_type" + + # metadata key + DISTRIBUTED_ARGS = "distributed_args" diff --git a/profiler/cluster_analyse/common_func/tables_config.py b/profiler/cluster_analyse/common_func/tables_config.py index f01001451..7122d6461 100644 --- a/profiler/cluster_analyse/common_func/tables_config.py +++ b/profiler/cluster_analyse/common_func/tables_config.py @@ -59,7 +59,10 @@ class TablesConfig: ("stage", "NUMERIC, null"), ("bubble", "NUMERIC, null"), ("communication_not_overlapped_and_exclude_receive", "NUMERIC, null"), - ("preparing", "NUMERIC, null") + ("preparing", "NUMERIC, null"), + ("dp_index", "INTEGER, null"), + ("pp_index", "INTEGER, null"), + ("tp_index", "INTEGER, null") ], "HostInfoMap": [ ("hostUid", "INTEGER, null"), diff --git a/profiler/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py b/profiler/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py new file mode 100644 index 000000000..2eb8b300a --- /dev/null +++ b/profiler/test/ut/cluster_analyse/cluster_utils/test_parallel_strategy_calculator.py @@ -0,0 +1,46 @@ +import unittest + +from cluster_utils.parallel_strategy_calculator import ParallelStrategyCalculator + + +class TestParallelStrategyCalculator(unittest.TestCase): + def test_parallel_strategy_calculator_should_raise_runtime_error_when_dp4_ep3(self): + with self.assertRaises(RuntimeError): + calculator = ParallelStrategyCalculator( + world_size=16, + tensor_model_parallel_size=1, + pipeline_model_parallel_size=4, + data_parallel_size=4, + context_parallel_size=1, + expert_model_parallel_size=3) + + calculator.run() + + def test_parallel_strategy_calculator_should_raise_runtime_error_when_dp1_pp4_tp2_world_size16(self): + with self.assertRaises(RuntimeError): + calculator = ParallelStrategyCalculator( + world_size=16, + tensor_model_parallel_size=2, + pipeline_model_parallel_size=4, + data_parallel_size=1, + context_parallel_size=1, + expert_model_parallel_size=1) + + calculator.run() + + def test_parallel_strategy_calculator_dp2_pp4_tp2(self): + calculator = ParallelStrategyCalculator( + world_size=16, + tensor_model_parallel_size=2, + pipeline_model_parallel_size=4, + data_parallel_size=2, + context_parallel_size=1, + expert_model_parallel_size=1) + + # dp index, pp index, tp index + expected_res = [ + (0, 0, 0), (0, 0, 1), (1, 0, 0), (1, 0, 1), (0, 1, 0), (0, 1, 1), (1, 1, 0), (1, 1, 1), + (0, 2, 0), (0, 2, 1), (1, 2, 0), (1, 2, 1), (0, 3, 0), (0, 3, 1), (1, 3, 0), (1, 3, 1) + ] + res = calculator.run() + self.assertEqual(res, expected_res) -- Gitee From 3d9ae18ba5645028e1fef0846f4a3c6d2acda2c0 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 18:49:52 +0800 Subject: [PATCH 213/791] clean code --- .../api_accuracy_checker/tensor_transport_layer/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py index 690ffea3e..521f8d37f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py @@ -3,9 +3,9 @@ import struct import hashlib import time import io +from threading import Thread from OpenSSL import SSL -from threading import Thread from twisted.internet import ssl, reactor, protocol, endpoints from msprobe.pytorch.common.utils import logger -- Gitee From 579215f211a61dd5174e5113e0fa323ec30e6a88 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Thu, 8 Aug 2024 18:53:45 +0800 Subject: [PATCH 214/791] Adjust the ut of forward. --- .../msprobe/test/pytorch_ut/free_benchmark/test_main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/test_main.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/test_main.py index 4498a2af7..3fe3da9a0 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/test_main.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/test_main.py @@ -61,6 +61,7 @@ class TestInterface(TestCase): def testForwardFix(self): # 对于前向接口，在forward钩子中开启FIX，返回结果给hook的输出 + # 为了与下一层的输入对齐、应该转换为扰动前输出的dtype，否则可能报错 config = Config(Const.FORWARD, HandlerType.FIX) checker = FreeBenchmarkCheck(config) # 执行算子前向 @@ -76,7 +77,7 @@ class TestInterface(TestCase): kwargs={}, output=out, ) - self.assertEqual(result.dtype, torch.float32) + self.assertEqual(result.dtype, torch.float16) def testBackwardCheck(self): # 对于反向接口，在pre forward时暂存input, 然后在backwrad后进行对比 -- Gitee From ae3f2b1008df0d9b01e719e410f0077ed86fbd37 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 19:11:24 +0800 Subject: [PATCH 215/791] clean code --- .../tensor_transport_layer/ssl_config.py | 39 +++++++------------ 1 file changed, 13 insertions(+), 26 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py index 2bc200bac..8fafecac1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py @@ -1,28 +1,15 @@ cipher_list = ":".join([ - 'TLS_DHE_RSA_WITH_AES_128_GCM_SHA256', - 'TLS_DHE_RSA_WITH_AES_256_GCM_SHA384', - 'TLS_DHE_DSS_WITH_AES_128_GCM_SHA256', - 'TLS_DHE_DSS_WITH_AES_256_GCM_SHA384', - 'TLS_PSK_WITH_AES_256_GCM_SHA384', - 'TLS_DHE_PSK_WITH_AES_128_GCM_SHA256', - 'TLS_DHE_PSK_WITH_AES_256_GCM_SHA384', - 'TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256', - 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', - 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', - 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', - 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', - 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', - 'TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256', - 'TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256', - 'TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384', - 'TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256', - 'TLS_DHE_RSA_WITH_AES_128_CCM', - 'TLS_DHE_RSA_WITH_AES_256_CCM' - 'TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256', - 'TLS_PSK_WITH_AES_256_CCM', - 'TLS_DHE_PSK_WITH_AES_128_CCM', - 'TLS_DHE_PSK_WITH_AES_256_CCM', - 'TLS_ECDHE_ECDSA_WITH_AES_128_CCM', - 'TLS_ECDHE_ECDSA_WITH_AES_256_CCM', - 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' + 'TLS_DHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_DHE_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_DHE_DSS_WITH_AES_128_GCM_SHA256', 'TLS_DHE_DSS_WITH_AES_256_GCM_SHA384', + 'TLS_PSK_WITH_AES_256_GCM_SHA384', 'TLS_DHE_PSK_WITH_AES_128_GCM_SHA256', + 'TLS_DHE_PSK_WITH_AES_256_GCM_SHA384', 'TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256', + 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256', + 'TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384', + 'TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256', 'TLS_DHE_RSA_WITH_AES_128_CCM', + 'TLS_DHE_RSA_WITH_AES_256_CCM', 'TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256', + 'TLS_PSK_WITH_AES_256_CCM', 'TLS_DHE_PSK_WITH_AES_128_CCM', + 'TLS_DHE_PSK_WITH_AES_256_CCM', 'TLS_ECDHE_ECDSA_WITH_AES_128_CCM', + 'TLS_ECDHE_ECDSA_WITH_AES_256_CCM', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' ]).encode() -- Gitee From 2273affd8cff4048602cf817ab2a99922a324587 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Thu, 8 Aug 2024 19:18:56 +0800 Subject: [PATCH 216/791] --amend --- .../pytorch/free_benchmark/compare/grad_saver.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py index 21f2b3b46..1cf75524d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py @@ -2,7 +2,7 @@ import torch from msprobe.core.common.exceptions import FreeBenchmarkException from msprobe.pytorch.free_benchmark import logger from msprobe.pytorch.free_benchmark.common.constant import CommonField -from msprobe.pytorch.free_benchmark.common.params import DataParams, HandlerParams +from msprobe.pytorch.free_benchmark.common.params import DataParams, HandlerParams, data_pre_deal from msprobe.pytorch.free_benchmark.perturbed_layers.layer_factory import LayerFactory from msprobe.pytorch.free_benchmark.result_handlers.handler_factory import ( FuzzHandlerFactory, @@ -161,11 +161,12 @@ class GradSaver: return grad_input def calculate_perturbed_grad_input(self, grad_output, need_grad_tensors, inner_args): - data_params = DataParams() - data_params.args = [need_grad_tensors, grad_output, inner_args] - data_params.kwargs = {} - data_params.valid_input_index = 0 - data_params.origin_func = self.get_grad_input_from_vjp + data_params = data_pre_deal( + self.handler_params.api_name, + self.get_grad_input_from_vjp, + [need_grad_tensors, grad_output, inner_args], + {} + ) layer = LayerFactory.create( self.handler_params.api_name, self.handler_params.fuzz_device, -- Gitee From 1889c3cf532148e7b0344406ae1111804fc57891 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 19:46:59 +0800 Subject: [PATCH 217/791] clean code --- .../tensor_transport_layer/ssl_config.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py index 8fafecac1..8980723a3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py @@ -1,15 +1,13 @@ cipher_list = ":".join([ - 'TLS_DHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_DHE_RSA_WITH_AES_256_GCM_SHA384', - 'TLS_DHE_DSS_WITH_AES_128_GCM_SHA256', 'TLS_DHE_DSS_WITH_AES_256_GCM_SHA384', - 'TLS_PSK_WITH_AES_256_GCM_SHA384', 'TLS_DHE_PSK_WITH_AES_128_GCM_SHA256', + 'TLS_DHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_DHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_DHE_DSS_WITH_AES_128_GCM_SHA256', + 'TLS_DHE_DSS_WITH_AES_256_GCM_SHA384', 'TLS_PSK_WITH_AES_256_GCM_SHA384', 'TLS_DHE_PSK_WITH_AES_128_GCM_SHA256', 'TLS_DHE_PSK_WITH_AES_256_GCM_SHA384', 'TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384', - 'TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256', 'TLS_DHE_RSA_WITH_AES_128_CCM', - 'TLS_DHE_RSA_WITH_AES_256_CCM', 'TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256', - 'TLS_PSK_WITH_AES_256_CCM', 'TLS_DHE_PSK_WITH_AES_128_CCM', - 'TLS_DHE_PSK_WITH_AES_256_CCM', 'TLS_ECDHE_ECDSA_WITH_AES_128_CCM', - 'TLS_ECDHE_ECDSA_WITH_AES_256_CCM', 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' + 'TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256', 'TLS_DHE_RSA_WITH_AES_128_CCM', 'TLS_DHE_RSA_WITH_AES_256_CCM', + 'TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_PSK_WITH_AES_256_CCM', 'TLS_DHE_PSK_WITH_AES_128_CCM', + 'TLS_DHE_PSK_WITH_AES_256_CCM', 'TLS_ECDHE_ECDSA_WITH_AES_128_CCM', 'TLS_ECDHE_ECDSA_WITH_AES_256_CCM', + 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' ]).encode() -- Gitee From 534d47468f6bc2860bbb6ac2f060cb066e2733c2 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 19:57:34 +0800 Subject: [PATCH 218/791] =?UTF-8?q?=E5=8E=8B=E7=BC=A9=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/acc_compare.py | 52 ++ .../msprobe/core/compare/check.py | 8 +- .../msprobe/core/compare/highlight.py | 4 +- .../msprobe/core/compare/utils.py | 51 ++ .../msprobe/mindspore/__init__.py | 2 - .../msprobe/mindspore/advisor/advisor.py | 124 ---- .../mindspore/advisor/advisor_const.py | 59 -- .../mindspore/advisor/advisor_result.py | 58 -- .../msprobe/mindspore/compare/compare_cli.py | 6 +- .../mindspore/compare/distributed_compare.py | 52 +- .../msprobe/mindspore/compare/ms_compare.py | 56 +- .../msprobe/pytorch/advisor/advisor.py | 124 ---- .../msprobe/pytorch/advisor/advisor_const.py | 59 -- .../msprobe/pytorch/advisor/advisor_result.py | 58 -- .../pytorch/compare/distributed_compare.py | 53 +- .../msprobe/pytorch/compare/mapping.yaml | 607 ++++++++++++++++++ .../msprobe/pytorch/compare/match.py | 36 ++ .../msprobe/pytorch/compare/pt_compare.py | 56 +- .../test/pytorch_ut/compare/test_match.py | 2 +- 19 files changed, 768 insertions(+), 699 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py delete mode 100644 debug/accuracy_tools/msprobe/pytorch/advisor/advisor.py delete mode 100644 debug/accuracy_tools/msprobe/pytorch/advisor/advisor_const.py delete mode 100644 debug/accuracy_tools/msprobe/pytorch/advisor/advisor_result.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml create mode 100644 debug/accuracy_tools/msprobe/pytorch/compare/match.py diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index df5ff18b2..e46b81d41 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -1,8 +1,13 @@ +import multiprocessing +import pandas as pd from msprobe.core.compare.check import check_op from msprobe.core.common.const import CompareConst from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ get_error_message from msprobe.core.common.exceptions import FileCheckException +from msprobe.core.compare.utils import read_op, merge_tensor,CompareException +from msprobe.core.compare.multiprocessing_compute import _handle_multi_process +from msprobe.core.common.log import logger class Comparator: @@ -10,6 +15,16 @@ class Comparator: def __init__(self): pass + def _do_multi_process(self,input_parma, result_df): + try: + compare_ops=getattr(self,"compare_ops") + result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + @classmethod def match_op(cls,npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): @@ -56,3 +71,40 @@ class Comparator: result_list.append(err_msg) return result_list + def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): + op_data = json_data['data'][op_name] + op_parsed_list = read_op(op_data, op_name) + if op_name in stack_json_data: + op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) + else: + op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) + + merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) + return merge_list + + def make_result_table(self,result,md5_compare,summary_compare,stack_mode): + header = [] + if md5_compare: + header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] + elif summary_compare: + header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] + else: + header = CompareConst.COMPARE_RESULT_HEADER[:] + + all_mode_bool = not (summary_compare or md5_compare) + if stack_mode: + if all_mode_bool: + header.append(CompareConst.STACK) + header.append(CompareConst.DATA_NAME) + else: + header.append(CompareConst.STACK) + else: + if all_mode_bool: + for row in result: + del row[-2] + header.append(CompareConst.DATA_NAME) + else: + for row in result: + del row[-1] + result_df = pd.DataFrame(result, columns=header) + return result_df \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index 97ddc26cd..66a96d302 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -1,4 +1,4 @@ -from msprobe.core.compare.match import graph_mapping +from debug.accuracy_tools.msprobe.pytorch.compare.match import graph_mapping from msprobe.core.common.log import logger from msprobe.core.compare.utils import rename_api @@ -28,8 +28,10 @@ def check_type_shape_match(npu_struct, bench_struct): shape_match = npu_shape == bench_shape type_match = npu_type == bench_type if not type_match: - if ([npu_type, bench_type] in [["Float16", "Float32"], ["Float32", "Float16"]] )or ([npu_type, bench_type] in [["torch.float16", "torch.float32"], ["torch.float32", "torch.float16"], - ["torch.float16", "torch.bfloat16"], ["torch.bfloat16", "torch.float16"]]): + ms_type=[["Float16", "Float32"], ["Float32", "Float16"],["Float16", "BFloat16"],["BFloat16", "Float16"]] + torch_type=[["torch.float16", "torch.float32"], ["torch.float32", "torch.float16"], + ["torch.float16", "torch.bfloat16"], ["torch.bfloat16", "torch.float16"]] + if ([npu_type, bench_type] in ms_type)or ([npu_type, bench_type] in torch_type): type_match = True else: type_match = False diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 802376347..ef35fd061 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -4,10 +4,8 @@ from collections import namedtuple import numpy as np import openpyxl from openpyxl.styles import PatternFill -from msprobe.core.common.utils import get_header_index -from msprobe.core.common.const import CompareConst +from msprobe.core.common.utils import get_header_index, CompareException from msprobe.core.common.log import logger -from msprobe.core.common.utils import CompareException from msprobe.core.common.file_check import change_mode from msprobe.core.common.const import CompareConst, FileCheckConst diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 55c1abd41..909ab1e95 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -1,7 +1,58 @@ import os +import re import numpy as np from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.utils import CompareException, check_file_or_directory_path, check_regex_prefix_format_valid, logger + + +def extract_json(dirname, stack_json=False): + json_path = '' + for fname in os.listdir(dirname): + if fname=="construct.json": continue + full_path = os.path.join(dirname, fname) + if full_path.endswith('.json'): + json_path = full_path + if not stack_json and 'stack' not in json_path: + break + if stack_json and 'stack' in json_path: + break + + # Provide robustness on invalid directory inputs + if not json_path: + logger.error(f'No file is found in dump dir {dirname}. ') + raise CompareException(CompareException.NO_DUMP_FILE_ERROR) + return json_path + + +def check_and_return_dir_contents(dump_dir, prefix): + """ + check the given dump dir and validate files in dump dir by using the given prefix patterns to build a + pattern: ^{prefix}(?:0|[0-9][1-9]*)?$ + + Args: + dump_dir (str): dump dir + prefix (str): prefix for the patterns, prefix should be less than 20 characters and alphanumeric/-/_ only + + Returns: + content [list]: dir contents + Raises: + CompareException: invalid path + ValueError: prefix not match the patterns + + """ + check_regex_prefix_format_valid(prefix) + check_file_or_directory_path(dump_dir, True) + contents = os.listdir(dump_dir) + pattern = re.compile(rf'^{prefix}(?:0|[0-9][1-9]*)?$') + for name in contents: + if not pattern.match(name): + logger.error( + f"dump_dir contains '{name}'. Expected '{prefix}'. This name is not in the format of dump " + f"output. Please check and delete irrelevant files in {dump_dir} and try again." + ) + raise CompareException(CompareException.INVALID_PATH_ERROR) + return contents def rename_api(npu_name, process): diff --git a/debug/accuracy_tools/msprobe/mindspore/__init__.py b/debug/accuracy_tools/msprobe/mindspore/__init__.py index dfe872c52..3bf42d1e3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/__init__.py +++ b/debug/accuracy_tools/msprobe/mindspore/__init__.py @@ -1,3 +1 @@ from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger -from .compare.distributed_compare import compare_distributed -from .compare.ms_compare import ms_compare \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py deleted file mode 100644 index ec2773e6d..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os - -from msprobe.mindspore.advisor.advisor_result import AdvisorResult -from msprobe.mindspore.advisor.advisor_const import AdvisorConst -from msprobe.mindspore.common.log import logger -from msprobe.core.common.utils import CompareException -from msprobe.core.common.file_check import FileChecker -from msprobe.core.common.const import Const, CompareConst, FileCheckConst - -class Advisor: - """ - Class for generate advisor - """ - - def __init__(self, input_data, out_path=""): - self.input_data = input_data - self.out_path = os.path.realpath(out_path) - self.file_type = None - - @staticmethod - def deterministic_advisor(message, node_name): - for api_name in AdvisorConst.NEED_DETERMINISTIC_API: - if api_name in node_name: - return AdvisorConst.DETERMINISTIC_SUGGEST - return message - - @staticmethod - def batch_norm_advisor(message, node_name): - if AdvisorConst.FUNC_BATCH_NORM in node_name and AdvisorConst.FORWARD_INPUT_1 in node_name: - message = AdvisorConst.BATCH_NORM_SUGGEST - return message - - def analyze_unmatched(self, analyze_data): - if self.file_type == Const.ALL: - accuracy_unmatched = analyze_data[ - analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] - else: - accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) | - (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)] - num_unmatch = len(accuracy_unmatched) - if num_unmatch != 0: - for i in range(len(accuracy_unmatched)): - item = accuracy_unmatched.iloc[i] - logger.warning("The tensor name matches but the shape or dtype does not match: {}" - .format(item[CompareConst.NPU_NAME])) - - def gen_advisor_result(self, pd_data): - first_failing_data = pd_data.iloc[0] - node_name = first_failing_data[CompareConst.NPU_NAME] - index = first_failing_data['index'] - message = self.gen_advisor_message(node_name) - logger.warning("Find %s accuracy not reached, the line is %s" % (node_name, index)) - result = AdvisorResult(node_name, index, message) - return result - - def gen_advisor_message(self, node_name): - if AdvisorConst.FORWARD in node_name: - if AdvisorConst.INPUT in node_name: - message = AdvisorConst.FORWARD_INPUT_SUGGEST - else: - message = AdvisorConst.FORWARD_OUTPUT_SUGGEST - message = self.deterministic_advisor(message, node_name) - else: - if AdvisorConst.INPUT in node_name: - message = AdvisorConst.BACKWARD_INPUT_SUGGEST - else: - message = AdvisorConst.BACKWARD_OUTPUT_SUGGEST - message = self.deterministic_advisor(message, node_name) - message = self.batch_norm_advisor(message, node_name) - return message - - def analysis(self): - self._check_path_vaild() - analyze_data = self._parse_input_data() - logger.info("Start analyzing the comparison result: %s" % self.file_type) - self.analyze_unmatched(analyze_data) - if self.file_type == Const.ALL: - failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] - elif self.file_type == Const.MD5: - failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF] - elif self.file_type == Const.SUMMARY: - failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING] - if failing_data.empty: - logger.info("All data from api input/output accuracy reached") - result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) - else: - result = self.gen_advisor_result(failing_data) - message_list = result.print_advisor_log() - result.gen_summary_file(self.out_path, message_list) - - def _parse_input_data(self): - data_columns = self.input_data.columns.values - if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): - self.file_type = Const.ALL - elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): - self.file_type = Const.MD5 - elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): - self.file_type = Const.SUMMARY - else: - logger.error('Compare result does not meet the required conditions.') - raise CompareException(CompareException.INVALID_DATA_ERROR) - df = self.input_data.reset_index() - return df - - def _check_path_vaild(self): - out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) - out_path_checker.common_check() diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py deleted file mode 100644 index 737c67591..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - - -class AdvisorConst: - """ - Class for advisor const - """ - - # text symbol - NEW_LINE = "\n" - COLON = ": " - - # advisor summary key - SUSPECT_NODES = "Suspect Nodes" - LINE = "Line" - ADVISOR_SUGGEST = "Expert Advice" - - NO_ERROR_API = "NA" - - # advisor message - NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements." - FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \ - "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \ - "3. The fault may be caused by memory corruption and further analysis is required." - FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation." - BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected." - BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation." - BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \ - "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \ - "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \ - "3. Use seed_all(mode=True) to enable deterministic computing." - DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \ - "can seed_all(mode=True) to enable deterministic computing." - - FUNC_BATCH_NORM = "Functional_batch_norm" - FORWARD_INPUT_1 = "forward_input.1" - NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"] - BATCH_NORM = "batch_norm" - - # name keyword - INPUT = "input" - OUTPUT = "output" - FORWARD = "forward" - BACKWARD = "backward" diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py deleted file mode 100644 index 5d59068fc..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -import os -import time - -from msprobe.mindspore.advisor.advisor_const import AdvisorConst -from msprobe.mindspore.common.log import logger -from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.file_check import change_mode - - -class AdvisorResult: - """ - Class for generate advisor result - """ - - def __init__(self, node, line, message): - self.suspect_node = node - self.line = line - self.advisor_message = message - - @staticmethod - def gen_summary_file(out_path, message_list): - file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) - result_file = os.path.join(out_path, file_name) - try: - with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file: - output_file.truncate(0) - message_list = [message + AdvisorConst.NEW_LINE for message in message_list] - output_file.writelines(message_list) - change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY) - except IOError as io_error: - logger.error("Failed to save %s, the reason is %s." % (result_file, io_error)) - else: - logger.info("The advisor summary is saved in: %s" % result_file) - - def print_advisor_log(self): - logger.info("The summary of the expert advice is as follows: ") - message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line), - AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node, - AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message] - for message in message_list: - logger.info(message) - return message_list diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py index 23582592f..4a8149657 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py @@ -4,7 +4,7 @@ from msprobe.core.common.const import FileCheckConst from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import ms_compare - +from msprobe.mindspore.compare.distributed_compare import compare_distributed def compare_cli_ms(args): with FileOpen(args.input_path, "r") as file: @@ -16,8 +16,8 @@ def compare_cli_ms(args): ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: - logger.error('Mindspore Unsupport function compare_distributed.') - raise Exception() + kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} + compare_distributed(npu_path, bench_path, args.output_path, **kwargs) else: logger.error("The npu_path and bench_path need to be of the same type.") raise CompareException(CompareException.INVALID_COMPARE_MODE) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 1e9586fba..6f84a69e9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -15,64 +15,16 @@ # limitations under the License. """ import os -import sys -import re from msprobe.core.common.utils import CompareException, check_compare_param, \ - check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid + check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import create_directory from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import MSComparator +from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): - def check_and_return_dir_contents(dump_dir, prefix): - """ - check the given dump dir and validate files in dump dir by using the given prefix patterns to build a - pattern: ^{prefix}(?:0|[0-9][1-9]*)?$ - - Args: - dump_dir (str): dump dir - prefix (str): prefix for the patterns, prefix should be less than 20 characters and alphanumeric/-/_ only - - Returns: - content [list]: dir contents - Raises: - CompareException: invalid path - ValueError: prefix not match the patterns - - """ - check_regex_prefix_format_valid(prefix) - check_file_or_directory_path(dump_dir, True) - contents = os.listdir(dump_dir) - pattern = re.compile(rf'^{prefix}(?:0|[0-9][1-9]*)?$') - for name in contents: - if not pattern.match(name): - logger.error( - f"dump_dir contains '{name}'. Expected '{prefix}'. This name is not in the format of dump " - f"output. Please check and delete irrelevant files in {dump_dir} and try again." - ) - raise CompareException(CompareException.INVALID_PATH_ERROR) - return contents - - def extract_json(dirname, stack_json=False): - json_path = '' - for fname in os.listdir(dirname): - if fname=="construct.json": continue - full_path = os.path.join(dirname, fname) - if full_path.endswith('.json'): - json_path = full_path - if not stack_json and 'stack' not in json_path: - break - if stack_json and 'stack' in json_path: - break - - # Provide robustness on invalid directory inputs - if not json_path: - logger.error(f'No file is found in dump dir {dirname}. ') - raise CompareException(CompareException.NO_DUMP_FILE_ERROR) - return json_path - if kwargs.get('suffix'): logger.error("Argument 'suffix' is not supported for compare_distributed.") raise CompareException(CompareException.INVALID_PARAM_ERROR) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index b42881ed4..cba440fcc 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,18 +1,15 @@ import json -import multiprocessing import os.path -import sys import numpy as np -import pandas as pd from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.const import FileCheckConst -from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op -from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process +from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy +from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger @@ -59,17 +56,6 @@ class MSComparator (Comparator): ) return _save_cmp_result(idx, cr, result_df, lock) - - def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): - op_data = json_data['data'][op_name] - op_parsed_list = read_op(op_data, op_name) - if op_name in stack_json_data: - op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) - else: - op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) - - merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) - return merge_list def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles @@ -135,32 +121,7 @@ class MSComparator (Comparator): result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df - def make_result_table(self,result,md5_compare,summary_compare,stack_mode): - header = [] - if md5_compare: - header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] - elif summary_compare: - header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] - else: - header = CompareConst.COMPARE_RESULT_HEADER[:] - - all_mode_bool = not (summary_compare or md5_compare) - if stack_mode: - if all_mode_bool: - header.append(CompareConst.STACK) - header.append(CompareConst.DATA_NAME) - else: - header.append(CompareConst.STACK) - else: - if all_mode_bool: - for row in result: - del row[-2] - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] - result_df = pd.DataFrame(result, columns=header) - return result_df + def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) @@ -217,15 +178,6 @@ class MSComparator (Comparator): if auto_analyze: advisor = Advisor(result_df, output_path) advisor.analysis() - - def _do_multi_process(self,input_parma, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e - def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: diff --git a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor.py b/debug/accuracy_tools/msprobe/pytorch/advisor/advisor.py deleted file mode 100644 index b178664d9..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os - -from msprobe.pytorch.advisor.advisor_result import AdvisorResult -from msprobe.pytorch.advisor.advisor_const import AdvisorConst -from msprobe.pytorch.common.log import logger -from msprobe.core.common.utils import CompareException -from msprobe.core.common.file_check import FileChecker -from msprobe.core.common.const import Const, CompareConst, FileCheckConst - -class Advisor: - """ - Class for generate advisor - """ - - def __init__(self, input_data, out_path=""): - self.input_data = input_data - self.out_path = os.path.realpath(out_path) - self.file_type = None - - @staticmethod - def deterministic_advisor(message, node_name): - for api_name in AdvisorConst.NEED_DETERMINISTIC_API: - if api_name in node_name: - return AdvisorConst.DETERMINISTIC_SUGGEST - return message - - @staticmethod - def batch_norm_advisor(message, node_name): - if AdvisorConst.FUNC_BATCH_NORM in node_name and AdvisorConst.FORWARD_INPUT_1 in node_name: - message = AdvisorConst.BATCH_NORM_SUGGEST - return message - - def analyze_unmatched(self, analyze_data): - if self.file_type == Const.ALL: - accuracy_unmatched = analyze_data[ - analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] - else: - accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) | - (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)] - num_unmatch = len(accuracy_unmatched) - if num_unmatch != 0: - for i in range(len(accuracy_unmatched)): - item = accuracy_unmatched.iloc[i] - logger.warning("The tensor name matches but the shape or dtype does not match: {}" - .format(item[CompareConst.NPU_NAME])) - - def gen_advisor_result(self, pd_data): - first_failing_data = pd_data.iloc[0] - node_name = first_failing_data[CompareConst.NPU_NAME] - index = first_failing_data['index'] - message = self.gen_advisor_message(node_name) - logger.warning("Find %s accuracy not reached, the line is %s" % (node_name, index)) - result = AdvisorResult(node_name, index, message) - return result - - def gen_advisor_message(self, node_name): - if AdvisorConst.FORWARD in node_name: - if AdvisorConst.INPUT in node_name: - message = AdvisorConst.FORWARD_INPUT_SUGGEST - else: - message = AdvisorConst.FORWARD_OUTPUT_SUGGEST - message = self.deterministic_advisor(message, node_name) - else: - if AdvisorConst.INPUT in node_name: - message = AdvisorConst.BACKWARD_INPUT_SUGGEST - else: - message = AdvisorConst.BACKWARD_OUTPUT_SUGGEST - message = self.deterministic_advisor(message, node_name) - message = self.batch_norm_advisor(message, node_name) - return message - - def analysis(self): - self._check_path_vaild() - analyze_data = self._parse_input_data() - logger.info("Start analyzing the comparison result: %s" % self.file_type) - self.analyze_unmatched(analyze_data) - if self.file_type == Const.ALL: - failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] - elif self.file_type == Const.MD5: - failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF] - elif self.file_type == Const.SUMMARY: - failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING] - if failing_data.empty: - logger.info("All data from api input/output accuracy reached") - result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) - else: - result = self.gen_advisor_result(failing_data) - message_list = result.print_advisor_log() - result.gen_summary_file(self.out_path, message_list) - - def _parse_input_data(self): - data_columns = self.input_data.columns.values - if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): - self.file_type = Const.ALL - elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): - self.file_type = Const.MD5 - elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): - self.file_type = Const.SUMMARY - else: - logger.error('Compare result does not meet the required conditions.') - raise CompareException(CompareException.INVALID_DATA_ERROR) - df = self.input_data.reset_index() - return df - - def _check_path_vaild(self): - out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) - out_path_checker.common_check() diff --git a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_const.py b/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_const.py deleted file mode 100644 index 737c67591..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_const.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - - -class AdvisorConst: - """ - Class for advisor const - """ - - # text symbol - NEW_LINE = "\n" - COLON = ": " - - # advisor summary key - SUSPECT_NODES = "Suspect Nodes" - LINE = "Line" - ADVISOR_SUGGEST = "Expert Advice" - - NO_ERROR_API = "NA" - - # advisor message - NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements." - FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \ - "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \ - "3. The fault may be caused by memory corruption and further analysis is required." - FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation." - BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected." - BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation." - BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \ - "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \ - "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \ - "3. Use seed_all(mode=True) to enable deterministic computing." - DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \ - "can seed_all(mode=True) to enable deterministic computing." - - FUNC_BATCH_NORM = "Functional_batch_norm" - FORWARD_INPUT_1 = "forward_input.1" - NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"] - BATCH_NORM = "batch_norm" - - # name keyword - INPUT = "input" - OUTPUT = "output" - FORWARD = "forward" - BACKWARD = "backward" diff --git a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_result.py deleted file mode 100644 index 456f542e1..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_result.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -import os -import time - -from msprobe.pytorch.advisor.advisor_const import AdvisorConst -from msprobe.pytorch.common.log import logger -from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.file_check import change_mode - - -class AdvisorResult: - """ - Class for generate advisor result - """ - - def __init__(self, node, line, message): - self.suspect_node = node - self.line = line - self.advisor_message = message - - @staticmethod - def gen_summary_file(out_path, message_list): - file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) - result_file = os.path.join(out_path, file_name) - try: - with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file: - output_file.truncate(0) - message_list = [message + AdvisorConst.NEW_LINE for message in message_list] - output_file.writelines(message_list) - change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY) - except IOError as io_error: - logger.error("Failed to save %s, the reason is %s." % (result_file, io_error)) - else: - logger.info("The advisor summary is saved in: %s" % result_file) - - def print_advisor_log(self): - logger.info("The summary of the expert advice is as follows: ") - message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line), - AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node, - AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message] - for message in message_list: - logger.info(message) - return message_list diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 05c274b15..923c0044d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -15,65 +15,16 @@ # limitations under the License. """ import os -import sys -import re from msprobe.core.common.utils import CompareException, check_compare_param, \ - check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid + check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import create_directory from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.pytorch.compare.pt_compare import PTComparator +from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): - def check_and_return_dir_contents(dump_dir, prefix): - """ - check the given dump dir and validate files in dump dir by using the given prefix patterns to build a - pattern: ^{prefix}(?:0|[0-9][1-9]*)?$ - - Args: - dump_dir (str): dump dir - prefix (str): prefix for the patterns, prefix should be less than 20 characters and alphanumeric/-/_ only - - Returns: - content [list]: dir contents - Raises: - CompareException: invalid path - ValueError: prefix not match the patterns - - """ - check_regex_prefix_format_valid(prefix) - check_file_or_directory_path(dump_dir, True) - contents = os.listdir(dump_dir) - pattern = re.compile(rf'^{prefix}(?:0|[0-9][1-9]*)?$') - for name in contents: - if not pattern.match(name): - logger.error( - f"dump_dir contains '{name}'. Expected '{prefix}'. This name is not in the format of dump " - f"output. Please check and delete irrelevant files in {dump_dir} and try again." - ) - raise CompareException(CompareException.INVALID_PATH_ERROR) - return contents - - - def extract_json(dirname, stack_json=False): - json_path = '' - for fname in os.listdir(dirname): - if fname=="construct.json": continue - full_path = os.path.join(dirname, fname) - if full_path.endswith('.json'): - json_path = full_path - if not stack_json and 'stack' not in json_path: - break - if stack_json and 'stack' in json_path: - break - - # Provide robustness on invalid directory inputs - if not json_path: - logger.error(f'No file is found in dump dir {dirname}. ') - raise CompareException(CompareException.NO_DUMP_FILE_ERROR) - return json_path - if kwargs.get('suffix'): logger.error("Argument 'suffix' is not supported for compare_distributed.") raise CompareException(CompareException.INVALID_PARAM_ERROR) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml b/debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml new file mode 100644 index 000000000..eaffbe7a1 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml @@ -0,0 +1,607 @@ +__and__: __and__ +__iand__: __iand__ +__ilshift__: __ilshift__ +__ior__: __ior__ +__irshift__: __irshift__ +__ixor__: __ixor__ +__lshift__: __lshift__ +__or__: __or__ +__rshift__: __rshift__ +__xor__: __xor__ +_adaptive_avg_pool2d: adaptive_avg_pool2d +_adaptive_avg_pool3d: adaptive_avg_pool3d +_cdist_forward: cdist +_cudnn_rnn: rnn +_embedding_bag: embedding_bag +_fft_c2c: fft +_fft_c2r: rfft +_foreach_add_: _foreach_add_ +_foreach_addcdiv: _foreach_addcdiv +_foreach_copy_: _foreach_copy_ +_foreach_lerp_: _foreach_lerp_ +_foreach_maximum: _foreach_maximum +_foreach_mul: _foreach_mul +_foreach_neg_: _foreach_neg_ +_foreach_pow: _foreach_pow +_foreach_reciprocal_: _foreach_reciprocal_ +_foreach_sign: _foreach_sign +_foreach_sqrt: _foreach_sqrt +_foreach_sqrt_: _foreach_sqrt_ +_foreach_sub: _foreach_sub +_fused_adam: FusedAdam +_linalg_det: det +_linalg_eigh: eigh +_linalg_slogdet: slogdet +_linalg_svd: svd +_list_to_tensor: as_tensor +_log_softmax: log_softmax +_native_batch_norm_legit: batch_norm +_nested_tensor_from_tensor_list: _nested_tensor_from_tensor_list +_pdist_forward: pdist +_pin_memory: pin_memory +_reshape_alias: reshape +_resize_output_: resize_ +_softmax: softmax +_to_copy: to +abs: abs +abs_: abs_ +absolute: abs +absolute_: abs_ +acos: acos +acos_: acos_ +acosh: acosh +acosh_: acosh_ +adaptive_max_pool2d: adaptive_max_pool2d +adaptive_max_pool3d: adaptive_max_pool3d +add: add +add_: add_ +addbmm: addbmm +addbmm_: addbmm_ +addcdiv: addcdiv +addcdiv_: addcdiv_ +addcmul: addcmul +addcmul_: addcmul_ +addmm: addmm +addmm_: addmm_ +addmv: addmv +addmv_: addmv_ +addr: addr +affine_grid_generator: affine_grid +alias: alias +all: all +alpha_dropout: AlphaDropout +amax: amax +amin: amin +aminmax: aminmax +angle: angle +any: any +arange: arange +arccos: acos +arccos_: arccos_ +arccosh: arccosh +arccosh_: arccosh_ +arcsin: asin +arcsin_: arcsin_ +arcsinh: asinh +arcsinh_: arcsinh_ +arctan: atan +arctan2: atan2 +arctan2_: arctan2_ +arctan_: arctan_ +arctanh: arctanh +arctanh_: arctanh_ +argmax: argmax +argmin: argmin +argsort: argsort +as_strided: as_strided +asin: asin +asin_: asin_ +asinh: asinh +asinh_: asinh_ +atan: atan +atan2: atan2 +atan2_: atan2_ +atan_: atan_ +atanh: atanh +atanh_: atanh_ +avg_pool2d: avg_pool2d +avg_pool3d: avg_pool3d +baddbmm: baddbmm +baddbmm_: baddbmm_ +bernoulli: bernoulli +bernoulli_: bernoulli_ +binary_cross_entropy: BCELoss +binary_cross_entropy_with_logits: binary_cross_entropy_with_logits +bitwise_and: bitwise_and +bitwise_and_: bitwise_and_ +bitwise_left_shift: __lshift__ +bitwise_left_shift_: bitwise_left_shift_ +bitwise_not: bitwise_not +bitwise_not_: bitwise_not_ +bitwise_or: bitwise_or +bitwise_or_: bitwise_or_ +bitwise_right_shift: __rshift__ +bitwise_right_shift_: bitwise_right_shift_ +bitwise_xor: bitwise_xor +bitwise_xor_: bitwise_xor_ +bmm: bmm +broadcast_tensors: broadcast_tensors +bucketize: bucketize +cat: cat +cauchy: Cauchy +cauchy_: cauchy_ +ceil: ceil +ceil_: ceil_ +celu: celu +celu_: celu_ +cholesky: cholesky +cholesky_inverse: cholesky_inverse +cholesky_solve: cholesky_solve +clamp: clamp +clamp_: clamp_ +clamp_max: clamp_max +clamp_max_: clamp_max_ +clamp_min: clamp_min +clamp_min_: clamp_min_ +clip: clip +clip_: clip_ +clone: clone +col2im: col2im +complex: complex +conj_physical: conj +conj_physical_: conj_ +constant_pad_nd: pad +convolution: Conv2d +copy: copy_ +copy_: copy_ +copysign: copysign +copysign_: copysign_ +cos: cos +cos_: cos_ +cosh: cosh +cosh_: cosh_ +count_nonzero: count_nonzero +cudnn_batch_norm: BatchNorm2d +cummax: cummax +cummin: cummin +cumprod: cumprod +cumprod_: cumprod_ +cumsum: cumsum +cumsum_: cumsum_ +deg2rad: deg2rad +deg2rad_: deg2rad_ +detach: detach +diag: diag +diag_embed: diag_embed +diagonal: diagonal +diagonal_copy: diagonal +diagonal_scatter: diagonal +digamma: digamma +digamma_: digamma_ +dist: dist +div: div +div_: div_ +divide: div +divide_: divide_ +dot: dot +dropout: dropout +elu: ELU +elu_: elu_ +embedding: embedding +empty_like: empty_like +empty_strided: empty_strided +eq: eq +eq_: eq_ +erf: erf +erf_: erf_ +erfc: erfc +erfc_: erfc_ +erfinv: erfinv +erfinv_: erfinv_ +exp: exp +exp2: exp2 +exp2_: exp2_ +exp_: exp_ +expand: expand +expm1: expm1 +expm1_: expm1_ +exponential: Exponential +exponential_: exponential_ +eye: eye +fft_fft: fft +fft_fft2: fft2 +fft_fftn: fftn +fft_fftshift: fftshift +fft_hfft: hfft +fft_hfft2: hfft2 +fft_hfftn: hfftn +fft_ifft: ifft +fft_ifft2: ifft2 +fft_ifftn: ifftn +fft_ifftshift: ifftshift +fft_ihfft: ihfft +fft_ihfft2: ihfft2 +fft_ihfftn: ifftn +fft_irfft: irfft +fft_irfft2: irfft2 +fft_irfftn: irfftn +fft_rfft: rfft +fft_rfft2: rfft2 +fft_rfftn: rfftn +fill: fill_ +fill_: fill_ +fix: fix +fix_: fix_ +flip: flip +float_power_: float_power_ +floor: floor +floor_: floor_ +floor_divide: floor_divide +floor_divide_: floor_divide_ +fmax: fmax +fmin: fmin +fmod: fmod +fmod_: fmod_ +frac: frac +frac_: frac_ +full: full +full_like: full_like +gather: gather +gcd: gcd +gcd_: gcd_ +ge: ge +ge_: ge_ +gelu: GELU +gelu_: gelu_ +geometric: Geometric +geometric_: geometric_ +glu: glu +greater: gt +greater_: ge_ +greater_equal: ge +greater_equal_: ge_ +grid_sampler_2d: grid_sample +grid_sampler_3d: grid_sample +gru: GRU +gt: gt +gt_: gt_ +hardshrink: Hardshrink +hardsigmoid: hardsigmoid +hardsigmoid_: hardsigmoid_ +hardswish: hardswish +hardswish_: hardswish_ +hardtanh: hardtanh +hardtanh_: hardtanh_ +heaviside: heaviside +heaviside_: heaviside_ +hinge_embedding_loss: HingeEmbeddingLoss +huber_loss: huber_loss +hypot: hypot +hypot_: hypot_ +i0: i0 +i0_: i0_ +igamma: igamma +igamma_: igamma_ +igammac: igammac +igammac_: igammac_ +index: __getitem__ +index_add: index_add +index_add_: index_add_ +index_copy: index_copy_ +index_copy_: index_copy_ +index_fill: index_fill_ +index_fill_: index_fill_ +index_put: index_put_ +index_put_: index_put_ +index_reduce: index_select +index_select: index_select +is_pinned: is_pinned +is_same_size: is_same_size +isinf: isinf +isnan: isnan +isneginf: isneginf +isposinf: isposinf +istft: istft +item: item +lcm: lcm +lcm_: lcm_ +le: le +le_: le_ +leaky_relu: LeakyReLU +leaky_relu_: leaky_relu_ +lerp: lerp +lerp_: lerp_ +less: less +less_: less_ +less_equal: le +less_equal_: less_equal_ +lgamma: lgamma +lgamma_: lgamma_ +linalg_cholesky_ex: cholesky +linalg_cross: cross +linalg_householder_product: householder_product +linalg_inv_ex: inv +linalg_ldl_factor_ex: ldl +linalg_ldl_solve: ldl_solve +linalg_lu: lu +linalg_lu_factor_ex: lu_factor +linalg_lu_solve: lu_solve +linalg_matrix_exp: matrix_exp +linalg_qr: qr +linalg_solve_triangular: solve +linalg_vector_norm: norm +linspace: linspace +log: log +log10: log10 +log10_: log10_ +log1p: log1p +log1p_: log1p_ +log2: log2 +log2_: log2_ +log_: log_ +log_normal: LogNormal +log_sigmoid_forward: log_sigmoid +logaddexp: logaddexp +logaddexp2: logaddexp2 +_native_batch_norm_legit_functional: batch_norm +logcumsumexp: logcumsumexp +logical_and: logical_and +logical_and_: logical_and_ +logical_not: logical_not +logical_not_: logical_not_ +logical_or: logical_or +logical_or_: logical_or_ +logical_xor: logical_xor +logical_xor_: logical_xor_ +logit: logit +logit_: logit_ +logspace: logspace +logsumexp: logsumexp +lstm: LSTM +lt: lt +lt_: lt_ +lu_unpack: lu_unpack +margin_ranking_loss: margin_ranking_loss +masked_fill: masked_fill +masked_fill_: masked_fill_ +matmul: matmul +max: max +max_pool2d_with_indices: MaxPool2d +max_pool3d_with_indices: MaxPool3d +max_unpool2d: MaxUnpool2d +max_unpool3d: max_unpool3d +maximum: maximum +mean: mean +median: median +meshgrid: meshgrid +min: min +minimum: minimum +mish: Mish +mish_: mish_ +mm: mm +mode: mode +mse_loss: mse_loss +mul: mul +mul_: mul_ +multi_margin_loss: MultiMarginLoss +multilabel_margin_loss_forward: multilabel_margin_loss +multinomial: multinomial +multiply: multiply +multiply_: mul_ +mv: mv +mvlgamma: mvlgamma +mvlgamma_: mvlgamma_ +name: name +nan_to_num: nan_to_num +nan_to_num_: nan_to_num_ +nanmedian: nanmedian +nansum: nansum +narrow_copy: narrow +native_batch_norm: BatchNorm2d +native_dropout: dropout +native_group_norm: group_norm +native_layer_norm: LayerNorm +ne: ne +ne_: ne_ +neg: neg +neg_: neg_ +negative: neg +negative_: neg_ +new_empty: new_empty +new_empty_strided: new_empty_strided +new_full: new_full +new_ones: new_ones +new_zeros: new_zeros +nextafter: nextafter +nextafter_: nextafter_ +nll_loss: nll_loss +nll_loss2d_forward: NLLLoss2d +nll_loss_forward: NLLLoss +nonzero_static: nonzero +norm: norm +normal: normal +normal_: normal_ +not_equal: ne +not_equal_: ne_ +ones: ones +ones_like: ones_like +ormqr: ormqr +pairwise_distance: pairwise_distance +pdist: pdist +permute: permute +pin_memory: pin_memory +pixel_shuffle: PixelShuffle +polar: polar +polygamma: polygamma +positive: positive +pow: pow +pow_: pow_ +prelu: prelu +prod: prod +quantized_gru: GRU +quantized_lstm: LSTM +rad2deg: rad2deg +rad2deg_: rad2deg_ +rand: rand +rand_like: rand_like +randint: randint +randint_like: randint_like +randn: randn +randn_like: randn_like +randperm: randperm +reciprocal: reciprocal +reciprocal_: reciprocal_ +reflection_pad1d: reflection_pad1d +reflection_pad2d: reflection_pad2d +reflection_pad3d: ReflectionPad3d +relu: relu +relu6: relu6 +relu_: relu_ +remainder: remainder +remainder_: remainder_ +renorm: renorm +renorm_: renorm_ +repeat: repeat +repeat_interleave: repeat_interleave +replication_pad1d: ReplicationPad1d +replication_pad2d: replication_pad2d +replication_pad3d: replication_pad3d +resize_as_: resize_as_ +rnn_relu: RNN +rnn_tanh: RNN +roll: roll +rot90: rot90 +round: round +round_: round_ +rrelu_with_noise: RReLU +rrelu_with_noise_: rrelu_with_noise +rsqrt: rsqrt +rsqrt_: rsqrt_ +rsub: rsub +scalar_tensor: scalar_tensor +scatter: scatter_ +scatter_: scatter_ +scatter_add: scatter_add +scatter_add_: scatter_add_ +searchsorted: searchsorted +select: select +selu: selu +selu_: selu_ +sgn: sgn +sgn_: sgn_ +sigmoid: sigmoid +sigmoid_: sigmoid_ +sign: sign +sign_: sign_ +signbit: signbit +silu: silu +silu_: silu_ +sin: sin +sin_: sin_ +sinc: sinc +sinc_: sinc_ +sinh: sinh +sinh_: sinh_ +slice: slice +smooth_l1_loss: smooth_l1_loss +soft_margin_loss: soft_margin_loss +softplus: softplus +softshrink: softshrink +sort: sort +special_airy_ai: airy_ai +special_bessel_j0: j0 +special_bessel_j1: j1 +special_bessel_y0: y0 +special_bessel_y1: y1 +special_chebyshev_polynomial_t: chebyshev_t +special_chebyshev_polynomial_u: chebyshev_u +special_entr: entr +special_erfcx: erfcx +special_hermite_polynomial_h: hermite +special_hermite_polynomial_he: he +special_i0: i0 +special_i0e: i0e +special_i1: i1 +special_i1e: i1e +special_laguerre_polynomial_l: laguerre_l +special_log_ndtr: log_ndtr +special_modified_bessel_i0: i0 +special_modified_bessel_i1: i1 +special_modified_bessel_k0: k0 +special_modified_bessel_k1: i1 +special_ndtr: ndtr +special_ndtri: ndtri +special_scaled_modified_bessel_k0: i0e +special_scaled_modified_bessel_k1: scaled_modified_bessel_k1 +special_spherical_bessel_j0: spherical_jn +special_xlog1py: xlog1py +special_zeta: zeta +split: split +split_with_sizes: split +sqrt: sqrt +sqrt_: sqrt_ +square: square +square_: square_ +squeeze: squeeze +stack: stack +std: std +std_mean: std_mean +stft: stft +sub: sub +sub_: sub_ +subtract: sub +subtract_: subtract_ +sum: sum +t: t +t_: t_ +take: take +tan: tan +tan_: tan_ +tanh: tanh +tanh_: tanh_ +threshold: threshold +threshold_: threshold_ +to: to +topk: topk +trace: trace +transpose: transpose +transpose_: transpose_ +triangular_solve: triangular_solve +tril: tril +tril_: tril_ +tril_indices: tril_indices +triu: triu +triu_: triu_ +triu_indices: triu_indices +true_divide: true_divide +true_divide_: true_divide_ +trunc: trunc +trunc_: trunc_ +unbind: unbind +unfold: unfold +uniform: Uniform +uniform_: uniform_ +unsafe_chunk: unsafe_chunk +unsafe_split: split +unsafe_split_with_sizes: split_with_sizes +unsqueeze: unsqueeze +unsqueeze_: unsqueeze_ +upsample_bicubic2d: interpolate +upsample_bilinear2d: upsample_bilinear +upsample_nearest1d: interpolate +upsample_nearest2d: interpolate +upsample_nearest3d: interpolate +var: var +var_mean: var_mean +vdot: vdot +view: view +where: where +xlogy: xlogy +xlogy_: xlogy_ +zero: zeros +zero_: zero_ +zeros: zeros +zeros_like: zeros_like + + + diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/match.py b/debug/accuracy_tools/msprobe/pytorch/compare/match.py new file mode 100644 index 000000000..2a46105bd --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/match.py @@ -0,0 +1,36 @@ +import os +import yaml +from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import CompareException + + +class AtenIrMapping(): + def __init__(self): + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(cur_path, "mapping.yaml") + with FileOpen(yaml_path, 'r') as f: + self.aten_mapping = yaml.safe_load(f) + + def match(self, op1, op2): + if "Aten" in op1 and "Aten" not in op2: + return self.match_op(op1, op2) + else: + return self.match_op(op2, op1) + + def match_op(self, aten_op, torch_op): + try: + aten_op_raw_name_overload = '_'.join(aten_op.split("_")[1:-3]) + aten_op_raw_name = aten_op_raw_name_overload.split('.')[0] + torch_op_raw_name = '_'.join(torch_op.split("_")[1:-3]).lower() + except IndexError as e: + err_msg = f"Dump op name format error: {aten_op}, {torch_op}. Your dump data may be corrupted." + raise CompareException.INVALID_DATA_ERROR(err_msg) from e + matching_op = self.aten_mapping.get(aten_op_raw_name) + if matching_op is None: + return False + if matching_op.lower() == torch_op_raw_name: + return True + return False + + +graph_mapping = AtenIrMapping() diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index dd7f8fc17..35b59b69d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,18 +1,15 @@ import json -import multiprocessing import os.path -import sys import torch -import pandas as pd from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.const import FileCheckConst -from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op -from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process +from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy +from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger @@ -58,18 +55,7 @@ class PTComparator (Comparator): five_thousand_err_ratio_result=five_thousand_err_ratio_result ) - return _save_cmp_result(idx, cr, result_df, lock) - - def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): - op_data = json_data['data'][op_name] - op_parsed_list = read_op(op_data, op_name) - if op_name in stack_json_data: - op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) - else: - op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) - - merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) - return merge_list + return _save_cmp_result(idx, cr, result_df, lock) def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles @@ -136,33 +122,6 @@ class PTComparator (Comparator): result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df - def make_result_table(self,result,md5_compare,summary_compare,stack_mode): - header = [] - if md5_compare: - header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] - elif summary_compare: - header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] - else: - header = CompareConst.COMPARE_RESULT_HEADER[:] - - all_mode_bool = not (summary_compare or md5_compare) - if stack_mode: - if all_mode_bool: - header.append(CompareConst.STACK) - header.append(CompareConst.DATA_NAME) - else: - header.append(CompareConst.STACK) - else: - if all_mode_bool: - for row in result: - del row[-2] - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] - result_df = pd.DataFrame(result, columns=header) - return result_df - def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -220,13 +179,6 @@ class PTComparator (Comparator): advisor = Advisor(result_df, output_path) advisor.analysis() - def _do_multi_process(self,input_parma, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py index 5dbe4453a..6865845b3 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py @@ -1,6 +1,6 @@ # coding=utf-8 import unittest -from msprobe.core.compare import match +from debug.accuracy_tools.msprobe.pytorch.compare import match class TestMatch(unittest.TestCase): -- Gitee From 078feb7b85fed098a0be375a7df705876224d555 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 20:08:56 +0800 Subject: [PATCH 219/791] =?UTF-8?q?clean=20code=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/compare/utils.py | 3 ++- .../msprobe/mindspore/compare/ms_compare.py | 10 +--------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 909ab1e95..63b745432 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -9,7 +9,8 @@ from msprobe.core.common.utils import CompareException, check_file_or_directory_ def extract_json(dirname, stack_json=False): json_path = '' for fname in os.listdir(dirname): - if fname=="construct.json": continue + if fname == "construct.json": + continue full_path = os.path.join(dirname, fname) if full_path.endswith('.json'): json_path = full_path diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index cba440fcc..580cbc700 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -191,12 +191,4 @@ def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu msComparator=MSComparator() msComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) - - - - - - - - \ No newline at end of file + md5_compare=md5_compare) \ No newline at end of file -- Gitee From 140741c94356fd0be612824793d1b61631d419d1 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 20:09:49 +0800 Subject: [PATCH 220/791] codeclean --- .../msprobe/pytorch/debugger/debugger_config.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index 9bed41dba..7c32be7cc 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -36,12 +36,14 @@ class DebuggerConfig: "max_sample": task_config.max_sample if task_config.max_sample else 20, } - # dump api tensor and collaborate with online run_ut - self.online_run_ut = task_config.online_run_ut if task_config.online_run_ut else False - self.nfs_path = task_config.nfs_path if task_config.nfs_path else "" - self.tls_path = task_config.tls_path if task_config.tls_path else "" - self.host = task_config.host if task_config.host else "" - self.port = task_config.port if task_config.port else -1 + self.online_run_ut = False + if self.task == Const.TENSOR: + # dump api tensor and collaborate with online run_ut + self.online_run_ut = task_config.online_run_ut if task_config.online_run_ut else False + self.nfs_path = task_config.nfs_path if task_config.nfs_path else "" + self.tls_path = task_config.tls_path if task_config.tls_path else "" + self.host = task_config.host if task_config.host else "" + self.port = task_config.port if task_config.port else -1 self.check() if self.step: -- Gitee From 610fc0dc037ca5128887f985a3d86c67852d8f0c Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 8 Aug 2024 20:14:20 +0800 Subject: [PATCH 221/791] change number --- debug/accuracy_tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index afbf8feb3..70a69e9de 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -14,7 +14,7 @@ import setuptools -__version__ = '1.0.1' +__version__ = '1.0.2' INSTALL_REQUIRED = [ "wheel", -- Gitee From d2c6e6330df57eb572d04ae8e385bb240cfd6eb4 Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Thu, 8 Aug 2024 17:16:18 +0800 Subject: [PATCH 222/791] adapt optimizing --- profiler/cli/compare_cli.py | 3 ++ .../origin_data_bean/kernel_details_bean.py | 6 +++ .../compare_backend/comparison_generator.py | 8 +++- .../data_prepare/operator_data_prepare.py | 41 ++++++++++++++----- .../generator/detail_performance_generator.py | 8 +++- .../profiling_parser/base_profiling_parser.py | 6 ++- .../profiling_parser/gpu_profiling_parser.py | 4 +- .../profiling_parser/npu_profiling_parser.py | 14 +++++-- .../compare_backend/utils/args_manager.py | 25 ++++++++++- .../compare_backend/utils/compare_args.py | 6 ++- .../compare_backend/utils/constant.py | 2 + .../compare_backend/utils/torch_op_node.py | 5 +++ .../compare_interface/comparison_interface.py | 7 +++- profiler/compare_tools/performance_compare.py | 2 + .../test_base_profiling_parser.py | 1 + 15 files changed, 112 insertions(+), 26 deletions(-) diff --git a/profiler/cli/compare_cli.py b/profiler/cli/compare_cli.py index 3a36d2cd9..b18099897 100644 --- a/profiler/cli/compare_cli.py +++ b/profiler/cli/compare_cli.py @@ -42,6 +42,9 @@ from profiler.compare_tools.compare_backend.comparison_generator import Comparis required=False) @click.option('--use_input_shape', is_flag=True) @click.option('--gpu_flow_cat', type=str, default='', help="Identifier of the GPU connection.") +@click.option('--base_step', type=str, default='', help="基准性能数据指定比对step") +@click.option('--comparison_step', type=str, default='', help="比较性能数据指定比对step") + def compare_cli(**kwargs) -> None: args = AnalyzeDict(kwargs) ComparisonGenerator(args).run() diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py index c15396e9c..f29839724 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/kernel_details_bean.py @@ -18,6 +18,7 @@ class KernelDetailsBean: self._mac_time = 0.0 self._duration = 0.0 self._start_time = Decimal("0") + self._step_id = "" self.init() @property @@ -65,6 +66,10 @@ class KernelDetailsBean: @property def end_time(self) -> Decimal: return self.start_time + convert_to_decimal(self._duration) + + @property + def step_id(self) -> int: + return int(self._step_id) if self._step_id else Constant.VOID_STEP def is_hide_op_pmu(self): if "mac_time(us)" in self._data.keys() or "aiv_vec_time(us)" in self._data.keys(): @@ -119,4 +124,5 @@ class KernelDetailsBean: self._aicore_time = self._data.get("aicore_time(us)", "") self._mac_time = self._data.get('mac_time(us)', "") self._duration = self._data.get('Duration(us)', 0) + self._step_id = self._data.get('Step Id', "") self._start_time = Decimal(self._data.get("Start Time(us)", "0")) diff --git a/profiler/compare_tools/compare_backend/comparison_generator.py b/profiler/compare_tools/compare_backend/comparison_generator.py index b4d17f88e..bfbc1bb7b 100644 --- a/profiler/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/compare_tools/compare_backend/comparison_generator.py @@ -31,9 +31,13 @@ class ComparisonGenerator: def load_data(self): self._data_dict[Constant.BASE_DATA] = self.PARSER_DICT.get(self._args_manager.base_profiling_type)( - self._args_manager.args, self._args_manager.base_path_dict).load_data() + self._args_manager.args, + self._args_manager.base_path_dict, + self._args_manager.base_step).load_data() self._data_dict[Constant.COMPARISON_DATA] = self.PARSER_DICT.get(self._args_manager.comparison_profiling_type)( - self._args_manager.args, self._args_manager.comparison_path_dict).load_data() + self._args_manager.args, + self._args_manager.comparison_path_dict, + self._args_manager.comparison_step).load_data() def generate_compare_result(self): overall_data = {Constant.BASE_DATA: self._data_dict.get(Constant.BASE_DATA).overall_metrics, diff --git a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py index 59913528a..2df9ae43e 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py +++ b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py @@ -1,29 +1,48 @@ from compare_backend.profiling_parser.base_profiling_parser import ProfilingResult from compare_backend.utils.tree_builder import TreeBuilder - +from compare_backend.utils.constant import Constant class OperatorDataPrepare: - def __init__(self, profiling_data: ProfilingResult): + def __init__(self, profiling_data: ProfilingResult, specified_step_id: int = Constant.VOID_STEP): self.profiling_data = profiling_data self._all_nodes = self._build_tree() self._root_node = self._all_nodes[0] + self._specified_step_id = specified_step_id def get_top_layer_ops(self) -> any: - level1_child_nodes = self._root_node.child_nodes - result_data = [] - for level1_node in level1_child_nodes: - if level1_node.is_step_profiler(): - result_data.extend(level1_node.child_nodes) - else: - result_data.append(level1_node) - return result_data + if len(self._all_nodes) < 1: + return [] + return self._get_top_layers_ops_from_root_node(self._root_node.child_nodes) def get_all_layer_ops(self) -> any: result_data = [] if len(self._all_nodes) < 1: return result_data - return list(filter(lambda x: not x.is_step_profiler(), self._all_nodes[1:])) + if self._specified_step_id == Constant.VOID_STEP: + return list(filter(lambda x: not x.is_step_profiler(), self._all_nodes[1:])) + node_queue = self._get_top_layers_ops_from_root_node(self._root_node.child_nodes) + while len(node_queue) > 0: + node = node_queue.pop(0) + result_data.append(node) + if node.child_nodes: + node_queue.extend(node.child_nodes) + return result_data def _build_tree(self): return TreeBuilder.build_tree(self.profiling_data.torch_op_data, self.profiling_data.kernel_dict, self.profiling_data.memory_list) + + def _get_top_layers_ops_from_root_node(self, top_layers_nodes: list) -> list: + result_data = [] + for level1_node in top_layers_nodes: + if self._specified_step_id == Constant.VOID_STEP: + if level1_node.is_step_profiler(): + result_data.extend(level1_node.child_nodes) + else: + result_data.append(level1_node) + elif level1_node.is_step_profiler() and level1_node.get_step_id() == self._specified_step_id: + result_data.extend(level1_node.child_nodes) + if not result_data and self._specified_step_id != Constant.VOID_STEP: + print(f"[WARNING] There is no operator infomation for step {self._specified_step_id}, " \ + "please check whether the data contains this step.") + return result_data \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py index c0da4b65b..916c426c6 100644 --- a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -31,6 +31,8 @@ from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatchi class DetailPerformanceGenerator(BaseGenerator): def __init__(self, profiling_data_dict: dict, args: any): super().__init__(profiling_data_dict, args) + self._base_step_id = int(args.base_step) if args.base_step else Constant.VOID_STEP + self._comparison_step_id = int(args.comparison_step) if args.comparison_step else Constant.VOID_STEP def compare(self): enable_compare = [self._args.enable_operator_compare, self._args.enable_memory_compare, @@ -83,8 +85,10 @@ class DetailPerformanceGenerator(BaseGenerator): # build tree for operator_compare memory_compare and api_compare base_op_prepare, comparison_op_prepare = None, None if self._args.enable_memory_compare or self.enable_api_compare or enable_operator_compare: - base_op_prepare = OperatorDataPrepare(self._profiling_data_dict.get(Constant.BASE_DATA)) - comparison_op_prepare = OperatorDataPrepare(self._profiling_data_dict.get(Constant.COMPARISON_DATA)) + base_op_prepare = OperatorDataPrepare(self._profiling_data_dict.get(Constant.BASE_DATA), + self._base_step_id) + comparison_op_prepare = OperatorDataPrepare(self._profiling_data_dict.get(Constant.COMPARISON_DATA), + self._comparison_step_id) # 算子性能比对-operator级 op_compare_result = [] diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index a2591dd0f..6afc52ff9 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -55,7 +55,7 @@ class ProfilingResult: class BaseProfilingParser(ABC): - def __init__(self, args: any, path_dict: dict): + def __init__(self, args: any, path_dict: dict, step_id: int = Constant.VOID_STEP): self._args = args self._profiling_type = path_dict.get(Constant.PROFILING_TYPE) self._profiling_path = path_dict.get(Constant.PROFILING_PATH) @@ -80,6 +80,7 @@ class BaseProfilingParser(ABC): self._categorize_performance_index = 0 self._cpu_cube_op = None self._bwd_tid = None + self._step_id = step_id @property def cpu_cube_op(self): @@ -120,6 +121,9 @@ class BaseProfilingParser(ABC): def load_data(self) -> ProfilingResult: self._result_data.update_bwd_tid(self._bwd_tid) + if self._step_id != Constant.VOID_STEP and self._profiling_type == Constant.GPU: + msg = "[WARNING] step id is invalid in GPU data, please use this when comparing between NPU datas." + raise RuntimeError(msg) self._dispatch_events() self._update_kernel_dict() self._update_communication_dict() diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 91b4094c2..65fcc092f 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -13,8 +13,8 @@ class GPUProfilingParser(BaseProfilingParser): FLOW_CAT = ("async_gpu", "async_cpu_to_gpu", "ac2g", "async") TORCH_OP_CAT = ("cpu_op", "user_annotation", "cuda_runtime", "operator", "runtime") - def __init__(self, args: any, path_dict: dict): - super().__init__(args, path_dict) + def __init__(self, args: any, path_dict: dict, step_id: int = Constant.VOID_STEP): + super().__init__(args, path_dict, step_id) self._trace_events = [TraceEventBean(event) for event in self._trace_events.get("traceEvents", [])] self._flow_cat = (args.gpu_flow_cat,) if args.gpu_flow_cat else self.FLOW_CAT self._compute_stream_id = self._infer_compute_stream_id() diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 29e9fea8d..b763d8c9b 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -17,8 +17,8 @@ class NPUProfilingParser(BaseProfilingParser): ACTIVE_CPU = "ProfilerActivity.CPU" LEVEL_0 = "Level0" - def __init__(self, args: any, path_dict: dict): - super().__init__(args, path_dict) + def __init__(self, args: any, path_dict: dict, step_id: int = Constant.VOID_STEP): + super().__init__(args, path_dict, step_id) self._operator_memory_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "operator_memory.csv") self._memory_record_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "memory_record.csv") self._kernel_detail_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "kernel_details.csv") @@ -72,11 +72,17 @@ class NPUProfilingParser(BaseProfilingParser): for kernel in kernel_details: if kernel.is_invalid(): continue + if self._step_id != Constant.VOID_STEP and kernel.step_id != self._step_id: + continue input_shapes = kernel.input_shapes if kernel.input_shapes else 'N/A' kernels_dict.setdefault(kernel.op_type, {}).setdefault(input_shapes, []).append( [kernel.name, kernel.duration]) - if len(kernels_dict) == 1: - print("[ERROR] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") + if not kernels_dict: + if self._step_id != Constant.VOID_STEP: + print(f"[ERROR] There is no kernel details infomation for step {self._step_id}," \ + " please check whether the data contains this step.") + else: + print("[ERROR] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") return self._result_data.update_kernel_details(kernels_dict) diff --git a/profiler/compare_tools/compare_backend/utils/args_manager.py b/profiler/compare_tools/compare_backend/utils/args_manager.py index 579bf9b99..69136c4d7 100644 --- a/profiler/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/compare_tools/compare_backend/utils/args_manager.py @@ -24,6 +24,8 @@ class ArgsManager: self._args = args self._base_path_dict = {} self._comparison_path_dict = {} + self._base_step = Constant.VOID_STEP + self._comparison_step = Constant.VOID_STEP @property def args(self): @@ -53,6 +55,14 @@ class ArgsManager: def comparison_path_dict(self): return self._comparison_path_dict + @property + def base_step(self): + return self._base_step + + @property + def comparison_step(self): + return self._comparison_step + @property def enable_profiling_compare(self): return self._args.enable_profiling_compare @@ -88,6 +98,18 @@ class ArgsManager: PathManager.make_dir_safety(output_path) PathManager.check_path_writeable(output_path) + def get_step_args_with_validating(self): + if self._args.base_step and self._args.comparison_step: + if all([self._args.base_step.isdigit(), self._args.comparison_step.isdigit()]): + self._base_step = int(self._args.base_step) + self._comparison_step = int(self._args.comparison_step) + else: + msg = "Invalid param, base_step and comparison_step must be a number." + raise RuntimeError(msg) + elif any([self._args.base_step, self._args.comparison_step]): + msg = "Invalid param, base_step and comparison_step must be set at the same time." + raise RuntimeError(msg) + def parse_profiling_path(self, file_path: str): self.check_profiling_path(file_path) if os.path.isfile(file_path): @@ -134,7 +156,8 @@ class ArgsManager: self._args.enable_communication_compare = True self._args.enable_api_compare = True self._args.enable_kernel_compare = True - + + self.get_step_args_with_validating() base_profiling_path = PathManager.get_realpath(self._args.base_profiling_path) self.check_profiling_path(base_profiling_path) self._base_path_dict = self.parse_profiling_path(base_profiling_path) diff --git a/profiler/compare_tools/compare_backend/utils/compare_args.py b/profiler/compare_tools/compare_backend/utils/compare_args.py index 9e6291e89..36199b5b0 100644 --- a/profiler/compare_tools/compare_backend/utils/compare_args.py +++ b/profiler/compare_tools/compare_backend/utils/compare_args.py @@ -12,7 +12,9 @@ class Args: max_kernel_num: int = None, op_name_map: dict = {}, use_input_shape: bool = False, - gpu_flow_cat: str = ""): + gpu_flow_cat: str = "", + base_step: str = "", + comparison_step: str = ""): self.base_profiling_path = base_profiling_path self.comparison_profiling_path = comparison_profiling_path self.enable_profiling_compare = enable_profiling_compare @@ -26,3 +28,5 @@ class Args: self.op_name_map = op_name_map self.use_input_shape = use_input_shape self.gpu_flow_cat = gpu_flow_cat + self.base_step = base_step + self.comparison_step = comparison_step \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index dbac7ed32..08eb1792a 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -97,3 +97,5 @@ class Constant(object): IS_BWD = "is_bwd" OPS = "ops" + + VOID_STEP = -1 \ No newline at end of file diff --git a/profiler/compare_tools/compare_backend/utils/torch_op_node.py b/profiler/compare_tools/compare_backend/utils/torch_op_node.py index bb116a60c..06479462c 100644 --- a/profiler/compare_tools/compare_backend/utils/torch_op_node.py +++ b/profiler/compare_tools/compare_backend/utils/torch_op_node.py @@ -100,5 +100,10 @@ class TorchOpNode: def is_step_profiler(self) -> bool: return self._event.is_step_profiler() + def get_step_id(self) -> int: + if self.is_step_profiler(): + return int(self._event.name.split("#")[1]) + return Constant.VOID_STEP + def get_op_info(self) -> list: return [self.name, self.input_shape, self.input_type, self.call_stack] diff --git a/profiler/compare_tools/compare_interface/comparison_interface.py b/profiler/compare_tools/compare_interface/comparison_interface.py index b747aae47..68bbcc026 100644 --- a/profiler/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/compare_tools/compare_interface/comparison_interface.py @@ -12,11 +12,14 @@ from compare_backend.utils.constant import Constant class ComparisonInterface: - def __init__(self, base_profiling_path: str, comparison_profiling_path: str = ""): + def __init__(self, base_profiling_path: str, comparison_profiling_path: str = "", + base_step: str = "", comparison_step: str = ""): self.base_profiling_path = base_profiling_path if comparison_profiling_path: self._args = Args(base_profiling_path=base_profiling_path, - comparison_profiling_path=comparison_profiling_path) + comparison_profiling_path=comparison_profiling_path, + base_step=base_step, + comparison_step=comparison_step) def compare(self, compare_type: str) -> dict: if compare_type == Constant.OVERALL_COMPARE: diff --git a/profiler/compare_tools/performance_compare.py b/profiler/compare_tools/performance_compare.py index 7c3fcdb6e..dff87db2f 100644 --- a/profiler/compare_tools/performance_compare.py +++ b/profiler/compare_tools/performance_compare.py @@ -27,6 +27,8 @@ def main(): help="配置GPU与NPU等价的算子名称映射关系，以字典的形式传入") parser.add_argument("--use_input_shape", default=False, action='store_true', help="开启算子的精准匹配") parser.add_argument("--gpu_flow_cat", type=str, default='', help="gpu flow event的分类标识") + parser.add_argument("--base_step", type=str, default='', help="基准性能数据指定比对step") + parser.add_argument("--comparison_step", type=str, default='', help="比较性能数据指定比对step") args = parser.parse_args() ComparisonGenerator(args).run() diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py index e84cfe048..b78c59f1f 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_base_profiling_parser.py @@ -27,6 +27,7 @@ class ProfilingParser(BaseProfilingParser): self._enable_kernel_compare = True self._enable_api_compare = True self._bwd_tid = 1 + self._step_id = -1 def _update_kernel_details(self): pass -- Gitee From 6b67e0c79bcc452fcffdb41f48d6cfec7f8f8458 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 20:17:25 +0800 Subject: [PATCH 223/791] importerror fix --- debug/accuracy_tools/msprobe/core/compare/check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index 66a96d302..c9335ef98 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -1,4 +1,4 @@ -from debug.accuracy_tools.msprobe.pytorch.compare.match import graph_mapping +from msprobe.pytorch.compare.match import graph_mapping from msprobe.core.common.log import logger from msprobe.core.compare.utils import rename_api -- Gitee From 48fb639e1af9ed617ff7849d0eaae315d4ac658d Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 8 Aug 2024 21:02:03 +0800 Subject: [PATCH 224/791] add tls online run_ut --- .../tensor_transport_layer/ssl_config.py | 21 +++++------- .../accuracy_tools/msprobe/pytorch/service.py | 34 +++++++++++-------- 2 files changed, 29 insertions(+), 26 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py index 8980723a3..8e29cafd2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py @@ -1,13 +1,10 @@ cipher_list = ":".join([ - 'TLS_DHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_DHE_RSA_WITH_AES_256_GCM_SHA384', 'TLS_DHE_DSS_WITH_AES_128_GCM_SHA256', - 'TLS_DHE_DSS_WITH_AES_256_GCM_SHA384', 'TLS_PSK_WITH_AES_256_GCM_SHA384', 'TLS_DHE_PSK_WITH_AES_128_GCM_SHA256', - 'TLS_DHE_PSK_WITH_AES_256_GCM_SHA384', 'TLS_DHE_PSK_WITH_CHACHA20_POLY1305_SHA256', - 'TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384', - 'TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384', - 'TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_ECDHE_PSK_WITH_CHACHA20_POLY1305_SHA256', - 'TLS_ECDHE_PSK_WITH_AES_128_GCM_SHA256', 'TLS_ECDHE_PSK_WITH_AES_256_GCM_SHA384', - 'TLS_ECDHE_PSK_WITH_AES_128_CCM_SHA256', 'TLS_DHE_RSA_WITH_AES_128_CCM', 'TLS_DHE_RSA_WITH_AES_256_CCM', - 'TLS_DHE_RSA_WITH_CHACHA20_POLY1305_SHA256', 'TLS_PSK_WITH_AES_256_CCM', 'TLS_DHE_PSK_WITH_AES_128_CCM', - 'TLS_DHE_PSK_WITH_AES_256_CCM', 'TLS_ECDHE_ECDSA_WITH_AES_128_CCM', 'TLS_ECDHE_ECDSA_WITH_AES_256_CCM', - 'TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256' -]).encode() + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'ECDHE-ECDSA-CHACHA20-POLY1305', + 'ECDHE-RSA-CHACHA20-POLY1305', + 'DHE-RSA-AES128-GCM-SHA256', + 'DHE-RSA-AES256-GCM-SHA384' +]) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index afcac50db..187058bd7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -32,21 +32,9 @@ class Service: self.switch = False self.current_iter = 0 self.first_start = True - try: - self.current_rank = get_rank_if_initialized() - except DistributedNotInitializedError: - self.current_rank = None + self.current_rank = None self.dump_iter_dir = None - if self.config.online_run_ut: - attl_config = ATTLConfig(is_benchmark_device=False, - connect_ip=self.config.host, - connect_port=self.config.port, - nfs_path=self.config.nfs_path, - tls_path=self.config.tls_path) - need_dump = len(self.config.rank) == 0 or self.current_rank in self.config.rank - self.attl = ATTL('npu', attl_config, need_dump=need_dump) - if self.config.nfs_path: - self.attl.upload("start") + self.attl = None @staticmethod def forward_backward_dump_end(): @@ -148,6 +136,12 @@ class Service: if self.config.step and self.current_iter not in self.config.step: return if self.first_start: + try: + self.current_rank = get_rank_if_initialized() + except DistributedNotInitializedError: + self.current_rank = None + self.attl_init() + if self.config.rank and self.current_rank not in self.config.rank: return self.register_hook_new() @@ -235,6 +229,18 @@ class Service: if Const.STATISTICS == self.config.task or Const.TENSOR == self.config.task: remove_dropout() + def attl_init(self): + if self.config.online_run_ut: + attl_config = ATTLConfig(is_benchmark_device=False, + connect_ip=self.config.host, + connect_port=self.config.port, + nfs_path=self.config.nfs_path, + tls_path=self.config.tls_path) + need_dump = len(self.config.rank) == 0 or self.current_rank in self.config.rank + self.attl = ATTL('npu', attl_config, need_dump=need_dump) + if self.config.nfs_path: + self.attl.upload("start") + def attl_send(self, api_data): logger.info(f"tools is dumping api: {api_data.name}, rank: {self.current_rank}") if self.config.nfs_path: -- Gitee From 50f96e3634509d9c36be56d014094d0bed60b2dd Mon Sep 17 00:00:00 2001 From: makai Date: Thu, 8 Aug 2024 21:28:15 +0800 Subject: [PATCH 225/791] =?UTF-8?q?=E6=8A=8Abase=E7=B1=BB=E4=B8=AD?= =?UTF-8?q?=E7=9A=84is=5Fterminated=E5=B1=9E=E6=80=A7=E6=94=BE=E5=88=B0ms?= =?UTF-8?q?=E5=92=8CPt=E7=B1=BB=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/data_dump/data_processor/base.py | 7 ------- .../data_dump/data_processor/mindspore_processor.py | 11 +++++++++++ .../data_dump/data_processor/pytorch_processor.py | 13 ++++++++++++- 3 files changed, 23 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 9acac5e8e..e15000008 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -65,8 +65,6 @@ class BaseDataProcessor: self.current_iter = 0 self._return_forward_new_output = False self._forward_new_output = None - self.real_overflow_nums = 0 - self.overflow_nums = config.overflow_nums @property def data_path(self): @@ -74,11 +72,6 @@ class BaseDataProcessor: @property def is_terminated(self): - if self.overflow_nums == -1: - return False - if self.real_overflow_nums >= self.overflow_nums: - logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") - return True return False @staticmethod diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 1a31f935e..12875030c 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -154,6 +154,17 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): def __init__(self, config, data_writer): super().__init__(config, data_writer) self.cached_tensors_and_file_paths = {} + self.real_overflow_nums = 0 + self.overflow_nums = config.overflow_nums + + @property + def is_terminated(self): + if self.overflow_nums == -1: + return False + if self.real_overflow_nums >= self.overflow_nums: + logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") + return True + return False def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index f54d97155..8afe36bfd 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -184,7 +184,18 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): super().__init__(config, data_writer) self.cached_tensors_and_file_paths = {} self.bits_for_overflow = 8 - + self.real_overflow_nums = 0 + self.overflow_nums = config.overflow_nums + + @property + def is_terminated(self): + if self.overflow_nums == -1: + return False + if self.real_overflow_nums >= self.overflow_nums: + logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") + return True + return False + @staticmethod def overflow_debug_mode_enable(): overflow_mode = os.getenv(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) -- Gitee From 342e05c13b17b432ec607175f0f1c2074a9df0be Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 22:56:48 +0800 Subject: [PATCH 226/791] =?UTF-8?q?=E6=8A=8Amindspore=E5=92=8Cpytorch?= =?UTF-8?q?=E8=A7=A3=E8=80=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/acc_compare.py | 25 ++++++++++++++++--- .../msprobe/core/compare/check.py | 20 --------------- .../msprobe/mindspore/compare/ms_compare.py | 2 +- .../msprobe/pytorch/compare/pt_compare.py | 2 +- 4 files changed, 24 insertions(+), 25 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index e46b81d41..b999eab30 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -1,6 +1,5 @@ import multiprocessing import pandas as pd -from msprobe.core.compare.check import check_op from msprobe.core.common.const import CompareConst from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ get_error_message @@ -8,7 +7,7 @@ from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.utils import read_op, merge_tensor,CompareException from msprobe.core.compare.multiprocessing_compute import _handle_multi_process from msprobe.core.common.log import logger - +from msprobe.core.compare.check import check_graph_mode, check_struct_match, fuzzy_check_op class Comparator: @@ -24,8 +23,28 @@ class Comparator: logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e + def check_op(npu_dict, bench_dict, fuzzy_match): + a_op_name = npu_dict["op_name"] + b_op_name = bench_dict["op_name"] + graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) + + frame_name=getattr(self,"frame_name") + if frame_name == "PTComparator": + from msprobe.pytorch.compare.match import graph_mapping + if graph_mode: + return graph_mapping.match(a_op_name[0], b_op_name[0]) + struct_match = check_struct_match(npu_dict, bench_dict) + if not fuzzy_match: + return a_op_name == b_op_name and struct_match + is_match = True + try: + is_match = fuzzy_check_op(a_op_name, b_op_name) + except Exception as err: + logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) + is_match = False + return is_match and struct_match + - @classmethod def match_op(cls,npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): if check_op(npu_queue[-1], b_op, fuzzy_match): diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index c9335ef98..c243c0910 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -1,4 +1,3 @@ -from msprobe.pytorch.compare.match import graph_mapping from msprobe.core.common.log import logger from msprobe.core.compare.utils import rename_api @@ -49,25 +48,6 @@ def check_graph_mode(a_op_name, b_op_name): return False -def check_op(npu_dict, bench_dict, fuzzy_match): - a_op_name = npu_dict["op_name"] - b_op_name = bench_dict["op_name"] - graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) - if graph_mode: - return graph_mapping.match(a_op_name[0], b_op_name[0]) - struct_match = check_struct_match(npu_dict, bench_dict) - if not fuzzy_match: - return a_op_name == b_op_name and struct_match - is_match = True - try: - is_match = fuzzy_check_op(a_op_name, b_op_name) - except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) - is_match = False - return is_match and struct_match - - - def fuzzy_check_op(npu_name_list, bench_name_list): if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list): return False diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 580cbc700..be7439cb0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -18,7 +18,7 @@ from msprobe.core.common.exceptions import FileCheckException class MSComparator (Comparator): def __init__(self): - super().__init__() + self.frame_name=MSComparator.__name__ def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): cos_result = [] diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 35b59b69d..a947a12f6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -18,7 +18,7 @@ from msprobe.core.common.exceptions import FileCheckException class PTComparator (Comparator): def __init__(self): - super().__init__() + self.frame_name=PTComparator.__name__ def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): cos_result = [] -- Gitee From 5e5051d00c8a8f4103e220ac3ba8a183fe3fd14b Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 23:12:32 +0800 Subject: [PATCH 227/791] check_op bugfix --- .../accuracy_tools/msprobe/core/compare/acc_compare.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index b999eab30..960c42f54 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -23,7 +23,7 @@ class Comparator: logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e - def check_op(npu_dict, bench_dict, fuzzy_match): + def check_op(self, npu_dict, bench_dict, fuzzy_match): a_op_name = npu_dict["op_name"] b_op_name = bench_dict["op_name"] graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) @@ -45,14 +45,14 @@ class Comparator: return is_match and struct_match - def match_op(cls,npu_queue, bench_queue, fuzzy_match): + def match_op(self, npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): - if check_op(npu_queue[-1], b_op, fuzzy_match): + if self.check_op(npu_queue[-1], b_op, fuzzy_match): return len(npu_queue) - 1, b_index - if check_op(npu_queue[-1], bench_queue[-1], fuzzy_match): + if self.check_op(npu_queue[-1], bench_queue[-1], fuzzy_match): return len(npu_queue) - 1, len(bench_queue) - 1 for n_index, n_op in enumerate(npu_queue[0: -1]): - if check_op(n_op, bench_queue[-1], fuzzy_match): + if self.check_op(n_op, bench_queue[-1], fuzzy_match): return n_index, len(bench_queue) - 1 return -1, -1 -- Gitee From 806e4cf0dca51076a4f6e6478a2cad4ed38f644e Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 23:38:56 +0800 Subject: [PATCH 228/791] =?UTF-8?q?=E9=97=A8=E7=A6=81=E8=A7=A3=E5=86=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/acc_compare.py | 93 ++++++++++--------- .../pytorch_ut/compare/test_acc_compare.py | 6 +- .../test/pytorch_ut/compare/test_match.py | 2 +- 3 files changed, 53 insertions(+), 48 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 960c42f54..7705a748d 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -14,14 +14,45 @@ class Comparator: def __init__(self): pass - def _do_multi_process(self,input_parma, result_df): - try: - compare_ops=getattr(self,"compare_ops") - result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e + @classmethod + def make_result_table(cls,result,md5_compare,summary_compare,stack_mode): + header = [] + if md5_compare: + header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] + elif summary_compare: + header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] + else: + header = CompareConst.COMPARE_RESULT_HEADER[:] + + all_mode_bool = not (summary_compare or md5_compare) + if stack_mode: + if all_mode_bool: + header.append(CompareConst.STACK) + header.append(CompareConst.DATA_NAME) + else: + header.append(CompareConst.STACK) + else: + if all_mode_bool: + for row in result: + del row[-2] + header.append(CompareConst.DATA_NAME) + else: + for row in result: + del row[-1] + result_df = pd.DataFrame(result, columns=header) + return result_df + + @classmethod + def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): + op_data = json_data['data'][op_name] + op_parsed_list = read_op(op_data, op_name) + if op_name in stack_json_data: + op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) + else: + op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) + + merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) + return merge_list def check_op(self, npu_dict, bench_dict, fuzzy_match): a_op_name = npu_dict["op_name"] @@ -90,40 +121,12 @@ class Comparator: result_list.append(err_msg) return result_list - def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): - op_data = json_data['data'][op_name] - op_parsed_list = read_op(op_data, op_name) - if op_name in stack_json_data: - op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) - else: - op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) - - merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) - return merge_list - - def make_result_table(self,result,md5_compare,summary_compare,stack_mode): - header = [] - if md5_compare: - header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] - elif summary_compare: - header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] - else: - header = CompareConst.COMPARE_RESULT_HEADER[:] - - all_mode_bool = not (summary_compare or md5_compare) - if stack_mode: - if all_mode_bool: - header.append(CompareConst.STACK) - header.append(CompareConst.DATA_NAME) - else: - header.append(CompareConst.STACK) - else: - if all_mode_bool: - for row in result: - del row[-2] - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] - result_df = pd.DataFrame(result, columns=header) - return result_df \ No newline at end of file + def _do_multi_process(self,input_parma, result_df): + try: + compare_ops=getattr(self,"compare_ops") + result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py index b97dcc5d9..608f8ca9c 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py @@ -1,10 +1,11 @@ # coding=utf-8 import unittest import pandas as pd -from msprobe.core.compare.check import check_graph_mode, check_op +from msprobe.core.compare.check import check_graph_mode from msprobe.core.compare.utils import merge_tensor, read_op, get_accuracy, rename_api from msprobe.core.compare.acc_compare import Comparator from msprobe.core.compare.highlight import find_error_rows,find_compare_result_error_rows +from msprobe.core.compare.acc_compare import Comparator npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], @@ -218,7 +219,8 @@ class TestUtilsMethods(unittest.TestCase): def test_check_op(self): fuzzy_match = False - result = check_op(npu_dict, bench_dict, fuzzy_match) + Comparator=Comparator() + result = Comparator.check_op(npu_dict, bench_dict, fuzzy_match) self.assertEqual(result, True) def test_merge_tensor(self): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py index 6865845b3..aaa428726 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py @@ -1,6 +1,6 @@ # coding=utf-8 import unittest -from debug.accuracy_tools.msprobe.pytorch.compare import match +from accuracy_tools.msprobe.pytorch.compare import match class TestMatch(unittest.TestCase): -- Gitee From 7e4be399a8688858756a51fe9e8958f036a5f336 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 8 Aug 2024 23:49:50 +0800 Subject: [PATCH 229/791] =?UTF-8?q?=E9=97=A8=E7=A6=81DT=20fix?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/test/pytorch_ut/compare/test_match.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py index aaa428726..ac28e994e 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_match.py @@ -1,6 +1,6 @@ # coding=utf-8 import unittest -from accuracy_tools.msprobe.pytorch.compare import match +from msprobe.pytorch.compare import match class TestMatch(unittest.TestCase): -- Gitee From 2f5e7c1ad8acd8f897464db200ff2eb08e76a8f3 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Fri, 9 Aug 2024 00:08:07 +0800 Subject: [PATCH 230/791] =?UTF-8?q?DT=E9=87=8C=E9=9D=A2=E5=AF=B9=E8=B1=A1?= =?UTF-8?q?=E5=86=99=E9=94=99=E4=BA=86=EF=BC=8C=E5=B7=B2=E7=BB=8F=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/test/pytorch_ut/compare/test_acc_compare.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py index 608f8ca9c..f1ffefbd1 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py @@ -219,8 +219,8 @@ class TestUtilsMethods(unittest.TestCase): def test_check_op(self): fuzzy_match = False - Comparator=Comparator() - result = Comparator.check_op(npu_dict, bench_dict, fuzzy_match) + comparator=Comparator() + result = comparator.check_op(npu_dict, bench_dict, fuzzy_match) self.assertEqual(result, True) def test_merge_tensor(self): @@ -233,7 +233,8 @@ class TestUtilsMethods(unittest.TestCase): def test_match_op(self): fuzzy_match = False - a, b = Comparator.match_op([npu_dict], [bench_dict], fuzzy_match) + comparator=Comparator() + a, b = comparator.match_op([npu_dict], [bench_dict], fuzzy_match) self.assertEqual(a, 0) self.assertEqual(b, 0) -- Gitee From 8564d57eab6d09490bc89b39a101cb1e47068a2d Mon Sep 17 00:00:00 2001 From: CSNIU Date: Fri, 9 Aug 2024 01:04:45 +0800 Subject: [PATCH 231/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3check=5Fop=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/pytorch_ut/compare/test_acc_compare.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py index f1ffefbd1..b08b09c85 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py @@ -3,9 +3,8 @@ import unittest import pandas as pd from msprobe.core.compare.check import check_graph_mode from msprobe.core.compare.utils import merge_tensor, read_op, get_accuracy, rename_api -from msprobe.core.compare.acc_compare import Comparator from msprobe.core.compare.highlight import find_error_rows,find_compare_result_error_rows -from msprobe.core.compare.acc_compare import Comparator +from msprobe.pytorch.compare.pt_compare import PTComparator npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], @@ -219,8 +218,8 @@ class TestUtilsMethods(unittest.TestCase): def test_check_op(self): fuzzy_match = False - comparator=Comparator() - result = comparator.check_op(npu_dict, bench_dict, fuzzy_match) + ptComparator=PTComparator() + result = ptComparator.check_op(npu_dict, bench_dict, fuzzy_match) self.assertEqual(result, True) def test_merge_tensor(self): @@ -233,8 +232,8 @@ class TestUtilsMethods(unittest.TestCase): def test_match_op(self): fuzzy_match = False - comparator=Comparator() - a, b = comparator.match_op([npu_dict], [bench_dict], fuzzy_match) + ptComparator=PTComparator() + a, b = ptComparator.match_op([npu_dict], [bench_dict], fuzzy_match) self.assertEqual(a, 0) self.assertEqual(b, 0) -- Gitee From acd8afba63b360e3e1a7ee0241f8ef792f9cc559 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Fri, 9 Aug 2024 01:36:05 +0800 Subject: [PATCH 232/791] =?UTF-8?q?pytorch=E5=92=8Cms=E5=8A=9F=E8=83=BD?= =?UTF-8?q?=E8=A7=A3=E8=80=A6=EF=BC=8C=E8=A7=A3=E5=86=B3importError?= =?UTF-8?q?=EF=BC=8Cut=E6=8A=A5=E9=94=99=E7=AD=89=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grad_tool/common/constant.py | 2 +- .../accuracy_tools/grad_tool/common/utils.py | 3 +- .../grad_tool/grad_ms/grad_analyzer.py | 3 - .../accuracy_tools/msprobe/config/config.json | 19 +- .../msprobe/core/common/const.py | 16 +- .../msprobe/core/common/exceptions.py | 2 +- .../msprobe/core/common/utils.py | 24 +- .../msprobe/core/common_config.py | 8 + .../msprobe/core/compare/acc_compare.py | 93 +- .../msprobe/core/compare/check.py | 30 +- .../msprobe/core/compare/highlight.py | 12 +- .../msprobe/core/compare/match.py | 2 +- ..._compute.py => multiprocessing_compute.py} | 10 +- .../msprobe/core/compare/utils.py | 74 +- .../msprobe/core/data_dump/data_collector.py | 16 + .../core/data_dump/data_processor/base.py | 40 + .../data_processor/mindspore_processor.py | 5 +- .../msprobe/core/grad_probe/constant.py | 19 +- .../msprobe/core/grad_probe/grad_compare.py | 9 +- .../msprobe/core/grad_probe/utils.py | 43 + .../msprobe/mindspore/__init__.py | 1 - .../msprobe/mindspore/advisor/advisor.py | 124 --- .../mindspore/advisor/advisor_const.py | 59 -- .../mindspore/advisor/advisor_result.py | 58 -- .../msprobe/mindspore/common/const.py | 85 ++ .../msprobe/mindspore/compare/compare_cli.py | 3 +- .../mindspore/compare/distributed_compare.py | 76 +- .../msprobe/mindspore/compare/ms_compare.py | 81 +- .../mindspore/debugger/debugger_config.py | 28 +- .../mindspore/debugger/precision_debugger.py | 45 +- .../msprobe/mindspore/doc/dump.md | 12 +- .../mindspore/free_benchmark/__init__.py | 0 .../free_benchmark/api_pynative_self_check.py | 116 +++ .../free_benchmark/common/__init__.py | 0 .../mindspore/free_benchmark/common/config.py | 12 + .../free_benchmark/common/handler_params.py | 17 + .../mindspore/free_benchmark/common/utils.py | 71 ++ .../free_benchmark/data/support_wrap_ops.yaml | 842 ++++++++++++++++++ .../free_benchmark/decorator/__init__.py | 0 .../free_benchmark/decorator/dec_forward.py | 42 + .../decorator/decorator_factory.py | 107 +++ .../free_benchmark/handler/__init__.py | 0 .../free_benchmark/handler/base_handler.py | 90 ++ .../free_benchmark/handler/check_handler.py | 41 + .../free_benchmark/handler/fix_handler.py | 36 + .../free_benchmark/handler/handler_factory.py | 21 + .../free_benchmark/perturbation/add_noise.py | 67 ++ .../perturbation/base_perturbation.py | 21 + .../free_benchmark/perturbation/bit_noise.py | 63 ++ .../perturbation/improve_precision.py | 34 + .../free_benchmark/perturbation/no_change.py | 12 + .../perturbation/perturbation_factory.py | 27 + .../free_benchmark/self_check_tool_factory.py | 33 + .../msprobe/mindspore/grad_probe/__init__.py | 0 .../mindspore/grad_probe/global_context.py | 91 ++ .../mindspore/grad_probe/grad_analyzer.py | 231 +++++ .../mindspore/grad_probe/grad_monitor.py | 27 + .../mindspore/grad_probe/grad_stat_csv.py | 132 +++ .../msprobe/mindspore/grad_probe/hook.py | 92 ++ .../msprobe/mindspore/grad_probe/utils.py | 29 + .../msprobe/mindspore/ms_config.py | 32 + .../msprobe/mindspore/runtime.py | 4 + .../msprobe/mindspore/service.py | 152 +++- .../msprobe/mindspore/task_handler_factory.py | 12 +- debug/accuracy_tools/msprobe/msprobe.py | 52 +- .../msprobe/pytorch/__init__.py | 1 + .../msprobe/pytorch/advisor/advisor.py | 124 --- .../msprobe/pytorch/advisor/advisor_const.py | 59 -- .../msprobe/pytorch/advisor/advisor_result.py | 58 -- .../msprobe/pytorch/compare/compare_cli.py | 4 +- .../pytorch/compare/distributed_compare.py | 55 +- .../msprobe/pytorch/compare/mapping.yaml | 607 +++++++++++++ .../msprobe/pytorch/compare/match.py | 36 + .../msprobe/pytorch/compare/pt_compare.py | 84 +- .../pytorch/debugger/precision_debugger.py | 10 +- .../pytorch/grad_probe/grad_monitor.py | 56 +- .../pytorch/grad_probe/grad_stat_csv.py | 14 +- .../msprobe/pytorch/pt_config.py | 4 +- .../test/mindspore_ut/test_ms_config.py | 7 +- .../test/mindspore_ut/test_primitive_dump.py | 82 ++ .../mindspore_ut/test_task_handler_factory.py | 4 +- .../test/pytorch_ut/advisor/test_advisor.py | 4 +- .../pytorch_ut/compare/test_acc_compare.py | 35 +- .../result_handlers/test_result_handler.py | 19 + .../pytorch_ut/grad_probe/test_grad_csv.py | 13 +- .../grad_probe/test_grad_monitor.py | 22 +- ...7\275\221URL\350\257\264\346\230\216.xlsx" | Bin 16997 -> 17397 bytes profiler/advisor/README.md | 38 +- profiler/advisor/analyzer/base_analyzer.py | 6 +- .../Communication_retransmission_analyzer.py | 46 + .../Communication_retransmission_checker.py | 128 +++ .../analyzer/communication/packet_analyzer.py | 46 + .../analyzer/communication/packet_checker.py | 148 +++ .../ai_core_freq/ai_core_freq_checker.py | 2 +- profiler/advisor/common/analyzer_scopes.py | 16 + profiler/advisor/common/constant.py | 15 + .../dataset/cluster/cluster_dataset.py | 33 + .../dataset/cluster/hccl_collection.py | 78 ++ .../advisor/dataset/communication/__init__.py | 0 .../communication/communication_dataset.py | 109 +++ ...communication_retransmission_analysis.html | 40 + .../html/templates/packet_analysis.html | 23 + profiler/advisor/img/cluster_2.png | Bin 0 -> 66908 bytes profiler/advisor/img/communication.png | Bin 0 -> 58862 bytes profiler/advisor/interface/interface.py | 21 +- profiler/advisor/rules/packet.yaml | 14 + profiler/advisor/rules/rdma_analysis.yaml | 9 + profiler/cli/compare_cli.py | 5 +- .../overall_performance_comparator.py | 8 + .../compare_bean/profiling_info.py | 181 ++-- .../compare_backend/comparison_generator.py | 35 +- .../disaggregate/overall_perf_interface.py | 15 +- .../profiling_parser/gpu_profiling_parser.py | 20 - .../profiling_parser/npu_profiling_parser.py | 56 +- .../compare_backend/utils/args_manager.py | 11 +- .../compare_backend/utils/constant.py | 3 + .../compare_backend/utils/file_reader.py | 1 - .../compare_interface/comparison_interface.py | 1 - profiler/compare_tools/performance_compare.py | 1 - .../test_rdma_retransmission_advice.py | 170 ++++ .../test_packet_advice.py | 175 ++++ .../compare_bean/test_profiling_info.py | 73 +- .../test_gpu_profiling_parser.py | 12 +- 123 files changed, 5184 insertions(+), 1161 deletions(-) rename debug/accuracy_tools/msprobe/core/compare/{Multiprocessing_compute.py => multiprocessing_compute.py} (97%) delete mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/common/const.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/handler_params.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/check_handler.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/fix_handler.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/no_change.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_monitor.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_stat_csv.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/runtime.py delete mode 100644 debug/accuracy_tools/msprobe/pytorch/advisor/advisor.py delete mode 100644 debug/accuracy_tools/msprobe/pytorch/advisor/advisor_const.py delete mode 100644 debug/accuracy_tools/msprobe/pytorch/advisor/advisor_result.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml create mode 100644 debug/accuracy_tools/msprobe/pytorch/compare/match.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py create mode 100644 profiler/advisor/analyzer/cluster/Communication_retransmission_analyzer.py create mode 100644 profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py create mode 100644 profiler/advisor/analyzer/communication/packet_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/packet_checker.py create mode 100644 profiler/advisor/dataset/cluster/hccl_collection.py create mode 100644 profiler/advisor/dataset/communication/__init__.py create mode 100644 profiler/advisor/dataset/communication/communication_dataset.py create mode 100644 profiler/advisor/display/html/templates/communication_retransmission_analysis.html create mode 100644 profiler/advisor/display/html/templates/packet_analysis.html create mode 100644 profiler/advisor/img/cluster_2.png create mode 100644 profiler/advisor/img/communication.png create mode 100644 profiler/advisor/rules/packet.yaml create mode 100644 profiler/advisor/rules/rdma_analysis.yaml create mode 100644 profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py create mode 100644 profiler/test/ut/advisor/communication_advice/test_packet_advice.py diff --git a/debug/accuracy_tools/grad_tool/common/constant.py b/debug/accuracy_tools/grad_tool/common/constant.py index 38d33e988..7904c1d42 100644 --- a/debug/accuracy_tools/grad_tool/common/constant.py +++ b/debug/accuracy_tools/grad_tool/common/constant.py @@ -39,7 +39,7 @@ class GradConst: DIRECTORY_LENGTH = 4096 FILE_NAME_LENGTH = 255 FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" - PARAM_VALID_PATTERN = r"^[a-zA-Z0-9.]+$" + PARAM_VALID_PATTERN = r"^[a-zA-Z0-9_.:-]+$" DIR = "dir" FILE = "file" diff --git a/debug/accuracy_tools/grad_tool/common/utils.py b/debug/accuracy_tools/grad_tool/common/utils.py index fceda8ce0..f40f8688c 100644 --- a/debug/accuracy_tools/grad_tool/common/utils.py +++ b/debug/accuracy_tools/grad_tool/common/utils.py @@ -7,7 +7,6 @@ import yaml import pandas as pd from grad_tool.common.constant import GradConst -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileOpen def _print_log(level, msg, end='\n'): @@ -115,7 +114,7 @@ class ListCache(list): def get_config(filepath): - with FileOpen(filepath, 'r') as file: + with open(filepath, 'r') as file: config = yaml.safe_load(file) return config diff --git a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py index c843df388..fa794a681 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/grad_analyzer.py @@ -16,7 +16,6 @@ from grad_tool.common.utils import ListCache, print_warn_log from grad_tool.common.utils import create_directory, check_file_or_directory_path, write_csv from grad_tool.grad_ms.global_context import grad_context from grad_tool.grad_ms.global_context import GlobalContext -from ptdbg_ascend.src.python.ptdbg_ascend.common.file_check_util import FileCheckConst, FileChecker def get_rank_id(): @@ -170,8 +169,6 @@ class CSVGenerator(Process): stat_data = None max_try = 10 while max_try: - file_path_checker = FileChecker(file_path, FileCheckConst.DIR,FileCheckConst.READ_ABLE) - file_path = file_path_checker.common_check() try: stat_data = np.load(file_path) return stat_data diff --git a/debug/accuracy_tools/msprobe/config/config.json b/debug/accuracy_tools/msprobe/config/config.json index ef0283ca2..bc9789a38 100644 --- a/debug/accuracy_tools/msprobe/config/config.json +++ b/debug/accuracy_tools/msprobe/config/config.json @@ -31,11 +31,20 @@ "error_data_path": "./" }, "grad_probe": { - "level": "L1", + "grad_level": "L1", "param_list": [], - "rank": [], - "step": [], - "bounds": [-1, 0, 1], - "output_path": "./grad_output" + "bounds": [-1, 0, 1] + }, + "free_benchmark": { + "scope": [], + "list": [], + "fuzz_device": "npu", + "pert_mode": "improve_precision", + "handler_type": "check", + "fuzz_level": "L1", + "fuzz_stage": "forward", + "if_preheat": false, + "preheat_step": 15, + "max_sample": 20 } } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 2fe424a43..333757082 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -1,5 +1,6 @@ import os import stat + import numpy as np @@ -16,6 +17,7 @@ class Const: OFF = 'OFF' BACKWARD = 'backward' FORWARD = 'forward' + PRIMITIVE_PREFIX = 'Primitive' DEFAULT_LIST = [] DEFAULT_PATH = './' WHITE_LIST = 'white_list' @@ -255,17 +257,3 @@ class OverflowConst: OVERFLOW_DEBUG_MODE_ENABLE = "OVERFLOW_DEBUG_MODE_ENABLE" OVERFLOW_ORIGINAL_MODE = 0 OVERFLOW_DEBUG_MODE = 1 - - -class MsConst: - CELL = "cell" - API = "api" - KERNEL = "kernel" - TOOL_LEVEL_DICT = { - "L0": CELL, - "L1": API, - "L2": KERNEL - } - PYNATIVE_MODE = "pynative" - GRAPH_GE_MODE = "graph_ge" - GRAPH_KBYK_MODE = "graph_kbyk" diff --git a/debug/accuracy_tools/msprobe/core/common/exceptions.py b/debug/accuracy_tools/msprobe/core/common/exceptions.py index ea61f8cd5..eb314c7c6 100644 --- a/debug/accuracy_tools/msprobe/core/common/exceptions.py +++ b/debug/accuracy_tools/msprobe/core/common/exceptions.py @@ -85,4 +85,4 @@ class DistributedNotInitializedError(Exception): self.msg = msg def __str__(self): - return self.msg + return self.msg \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index cde65dd0e..7a34a2411 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -27,7 +27,7 @@ from datetime import datetime, timezone from pathlib import Path import numpy as np -from msprobe.core.common.file_check import FileOpen, FileChecker +from msprobe.core.common.file_check import FileOpen, FileChecker, change_mode from msprobe.core.common.const import Const, FileCheckConst, CompareConst, OverflowConst from msprobe.core.common.log import logger @@ -258,6 +258,17 @@ def remove_path(path): raise CompareException(CompareException.INVALID_PATH_ERROR) from err +def move_file(src_path, dst_path): + check_file_or_directory_path(src_path) + check_path_before_create(dst_path) + try: + shutil.move(src_path, dst_path) + except Exception as e: + logger.error(f"move file {src_path} to {dst_path} failed") + raise RuntimeError(f"move file {src_path} to {dst_path} failed") from e + change_mode(dst_path, FileCheckConst.DATA_FILE_AUTHORITY) + + def get_dump_data_path(dump_dir): """ Function Description: @@ -515,10 +526,19 @@ def write_csv(data, filepath): def load_npy(filepath): - filepath = os.path.realpath(filepath) check_file_or_directory_path(filepath) try: npy = np.load(filepath) except Exception as e: raise RuntimeError(f"load npy file {filepath} failed") from e return npy + + +def save_npy(data, filepath): + filepath = os.path.realpath(filepath) + check_path_before_create(filepath) + try: + npy = np.save(filepath, data) + except Exception as e: + raise RuntimeError(f"save npy file {filepath} failed") from e + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index d6c15e101..688734be8 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -50,6 +50,14 @@ class BaseConfig: self.summary_mode = json_config.get("summary_mode") self.overflow_nums = json_config.get("overflow_nums") self.check_mode = json_config.get("check_mode") + self.fuzz_device = json_config.get("fuzz_device") + self.pert_mode = json_config.get("pert_mode") + self.handler_type = json_config.get("handler_type") + self.fuzz_level = json_config.get("fuzz_level") + self.fuzz_stage = json_config.get("fuzz_stage") + self.if_preheat = json_config.get("if_preheat") + self.preheat_step = json_config.get("preheat_step") + self.max_sample = json_config.get("max_sample") def check_config(self): if self.scope is not None and not isinstance(self.scope, list): diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 7d2be9c4c..7705a748d 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -1,22 +1,89 @@ -from msprobe.core.compare.check import check_op -from msprobe.core.common.const import CompareConst +import multiprocessing +import pandas as pd +from msprobe.core.common.const import CompareConst from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ get_error_message from msprobe.core.common.exceptions import FileCheckException - +from msprobe.core.compare.utils import read_op, merge_tensor,CompareException +from msprobe.core.compare.multiprocessing_compute import _handle_multi_process +from msprobe.core.common.log import logger +from msprobe.core.compare.check import check_graph_mode, check_struct_match, fuzzy_check_op class Comparator: + def __init__(self): pass - def match_op(self,npu_queue, bench_queue, fuzzy_match): + @classmethod + def make_result_table(cls,result,md5_compare,summary_compare,stack_mode): + header = [] + if md5_compare: + header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] + elif summary_compare: + header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] + else: + header = CompareConst.COMPARE_RESULT_HEADER[:] + + all_mode_bool = not (summary_compare or md5_compare) + if stack_mode: + if all_mode_bool: + header.append(CompareConst.STACK) + header.append(CompareConst.DATA_NAME) + else: + header.append(CompareConst.STACK) + else: + if all_mode_bool: + for row in result: + del row[-2] + header.append(CompareConst.DATA_NAME) + else: + for row in result: + del row[-1] + result_df = pd.DataFrame(result, columns=header) + return result_df + + @classmethod + def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): + op_data = json_data['data'][op_name] + op_parsed_list = read_op(op_data, op_name) + if op_name in stack_json_data: + op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) + else: + op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) + + merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) + return merge_list + + def check_op(self, npu_dict, bench_dict, fuzzy_match): + a_op_name = npu_dict["op_name"] + b_op_name = bench_dict["op_name"] + graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) + + frame_name=getattr(self,"frame_name") + if frame_name == "PTComparator": + from msprobe.pytorch.compare.match import graph_mapping + if graph_mode: + return graph_mapping.match(a_op_name[0], b_op_name[0]) + struct_match = check_struct_match(npu_dict, bench_dict) + if not fuzzy_match: + return a_op_name == b_op_name and struct_match + is_match = True + try: + is_match = fuzzy_check_op(a_op_name, b_op_name) + except Exception as err: + logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) + is_match = False + return is_match and struct_match + + + def match_op(self, npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): - if check_op(npu_queue[-1], b_op, fuzzy_match): + if self.check_op(npu_queue[-1], b_op, fuzzy_match): return len(npu_queue) - 1, b_index - if check_op(npu_queue[-1], bench_queue[-1], fuzzy_match): + if self.check_op(npu_queue[-1], bench_queue[-1], fuzzy_match): return len(npu_queue) - 1, len(bench_queue) - 1 for n_index, n_op in enumerate(npu_queue[0: -1]): - if check_op(n_op, bench_queue[-1], fuzzy_match): + if self.check_op(n_op, bench_queue[-1], fuzzy_match): return n_index, len(bench_queue) - 1 return -1, -1 @@ -54,6 +121,12 @@ class Comparator: result_list.append(err_msg) return result_list - -testComparator= Comparator() - + def _do_multi_process(self,input_parma, result_df): + try: + compare_ops=getattr(self,"compare_ops") + result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index a8ee3638a..c243c0910 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -1,9 +1,7 @@ -from msprobe.core.compare.match import graph_mapping from msprobe.core.common.log import logger from msprobe.core.compare.utils import rename_api - def check_struct_match(npu_dict, bench_dict): npu_struct_in = npu_dict.get("input_struct") bench_struct_in = bench_dict.get("input_struct") @@ -18,6 +16,7 @@ def check_struct_match(npu_dict, bench_dict): is_match = struct_in_is_match and struct_out_is_match return is_match + def check_type_shape_match(npu_struct, bench_struct): shape_type_match = False for npu_type_shape, bench_type_shape in zip(npu_struct, bench_struct): @@ -28,8 +27,10 @@ def check_type_shape_match(npu_struct, bench_struct): shape_match = npu_shape == bench_shape type_match = npu_type == bench_type if not type_match: - if ([npu_type, bench_type] in [["Float16", "Float32"], ["Float32", "Float16"]] )or ([npu_type, bench_type] in [["torch.float16", "torch.float32"], ["torch.float32", "torch.float16"], - ["torch.float16", "torch.bfloat16"], ["torch.bfloat16", "torch.float16"]]): + ms_type=[["Float16", "Float32"], ["Float32", "Float16"],["Float16", "BFloat16"],["BFloat16", "Float16"]] + torch_type=[["torch.float16", "torch.float32"], ["torch.float32", "torch.float16"], + ["torch.float16", "torch.bfloat16"], ["torch.bfloat16", "torch.float16"]] + if ([npu_type, bench_type] in ms_type)or ([npu_type, bench_type] in torch_type): type_match = True else: type_match = False @@ -38,6 +39,7 @@ def check_type_shape_match(npu_struct, bench_struct): return False return shape_type_match + def check_graph_mode(a_op_name, b_op_name): if "Aten" in a_op_name and "Aten" not in b_op_name: return True @@ -46,25 +48,6 @@ def check_graph_mode(a_op_name, b_op_name): return False -def check_op(npu_dict, bench_dict, fuzzy_match): - a_op_name = npu_dict["op_name"] - b_op_name = bench_dict["op_name"] - graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) - if graph_mode: - return graph_mapping.match(a_op_name[0], b_op_name[0]) - struct_match = check_struct_match(npu_dict, bench_dict) - if not fuzzy_match: - return a_op_name == b_op_name and struct_match - is_match = True - try: - is_match = fuzzy_check_op(a_op_name, b_op_name) - except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) - is_match = False - return is_match and struct_match - - - def fuzzy_check_op(npu_name_list, bench_name_list): if len(npu_name_list) == 0 or len(bench_name_list) == 0 or len(npu_name_list) != len(bench_name_list): return False @@ -75,6 +58,7 @@ def fuzzy_check_op(npu_name_list, bench_name_list): break return is_match + def fuzzy_check_name(npu_name, bench_name): if "forward" in npu_name and "forward" in bench_name: is_match = rename_api(npu_name, "forward") == rename_api(bench_name, "forward") diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 17dee2f50..ef35fd061 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -1,16 +1,13 @@ import math import abc -import numpy as np from collections import namedtuple +import numpy as np import openpyxl from openpyxl.styles import PatternFill -from collections import namedtuple -from msprobe.core.common.utils import get_header_index -from msprobe.core.common.const import CompareConst +from msprobe.core.common.utils import get_header_index, CompareException from msprobe.core.common.log import logger -from msprobe.core.common.utils import CompareException -from msprobe.core.common.file_check import change_mode -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.file_check import change_mode +from msprobe.core.common.const import CompareConst, FileCheckConst class HighlightCheck(abc.ABC): @@ -166,6 +163,7 @@ def get_name_and_state(name): state = "output" return api_name, state + def find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare): """将dataframe根据API分组，并找到有误差的算子用于高亮""" result = result_df.values diff --git a/debug/accuracy_tools/msprobe/core/compare/match.py b/debug/accuracy_tools/msprobe/core/compare/match.py index 6347d8887..2a46105bd 100644 --- a/debug/accuracy_tools/msprobe/core/compare/match.py +++ b/debug/accuracy_tools/msprobe/core/compare/match.py @@ -10,7 +10,7 @@ class AtenIrMapping(): yaml_path = os.path.join(cur_path, "mapping.yaml") with FileOpen(yaml_path, 'r') as f: self.aten_mapping = yaml.safe_load(f) - + def match(self, op1, op2): if "Aten" in op1 and "Aten" not in op2: return self.match_op(op1, op2) diff --git a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py similarity index 97% rename from debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py rename to debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py index 9d8e9744e..da63005e5 100644 --- a/debug/accuracy_tools/msprobe/core/compare/Multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py @@ -1,10 +1,10 @@ import multiprocessing -import pandas as pd from dataclasses import dataclass +import pandas as pd from msprobe.core.common.log import logger -from msprobe.core.common.utils import CompareException -from msprobe.core.common.const import CompareConst +from msprobe.core.common.utils import CompareException +from msprobe.core.common.const import CompareConst def _handle_multi_process(func, input_parma, result_df, lock): @@ -38,6 +38,7 @@ def _handle_multi_process(func, input_parma, result_df, lock): pool.join() return pd.concat(final_results, ignore_index=True) + def read_dump_data(result_df): try: npu_dump_name_list = result_df.iloc[0:, 0].tolist() @@ -55,7 +56,6 @@ def read_dump_data(result_df): logger.error('result dataframe elements can not be access.') raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR) from e - @dataclass class ComparisonResult: cos_result: list @@ -65,6 +65,7 @@ class ComparisonResult: one_thousand_err_ratio_result: list five_thousand_err_ratio_result: list + def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): """ Save comparison results into the result DataFrame with thread safety. @@ -99,6 +100,7 @@ def _save_cmp_result(offset, result: ComparisonResult, result_df, lock): finally: lock.release() + def check_accuracy(cos, max_abs_err): if cos == CompareConst.SHAPE_UNMATCH: return CompareConst.ACCURACY_CHECK_UNMATCH diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index d213e0b46..63b745432 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -1,9 +1,59 @@ import os +import re import numpy as np from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.utils import CompareException, check_file_or_directory_path, check_regex_prefix_format_valid, logger +def extract_json(dirname, stack_json=False): + json_path = '' + for fname in os.listdir(dirname): + if fname == "construct.json": + continue + full_path = os.path.join(dirname, fname) + if full_path.endswith('.json'): + json_path = full_path + if not stack_json and 'stack' not in json_path: + break + if stack_json and 'stack' in json_path: + break + + # Provide robustness on invalid directory inputs + if not json_path: + logger.error(f'No file is found in dump dir {dirname}. ') + raise CompareException(CompareException.NO_DUMP_FILE_ERROR) + return json_path + + +def check_and_return_dir_contents(dump_dir, prefix): + """ + check the given dump dir and validate files in dump dir by using the given prefix patterns to build a + pattern: ^{prefix}(?:0|[0-9][1-9]*)?$ + + Args: + dump_dir (str): dump dir + prefix (str): prefix for the patterns, prefix should be less than 20 characters and alphanumeric/-/_ only + + Returns: + content [list]: dir contents + Raises: + CompareException: invalid path + ValueError: prefix not match the patterns + + """ + check_regex_prefix_format_valid(prefix) + check_file_or_directory_path(dump_dir, True) + contents = os.listdir(dump_dir) + pattern = re.compile(rf'^{prefix}(?:0|[0-9][1-9]*)?$') + for name in contents: + if not pattern.match(name): + logger.error( + f"dump_dir contains '{name}'. Expected '{prefix}'. This name is not in the format of dump " + f"output. Please check and delete irrelevant files in {dump_dir} and try again." + ) + raise CompareException(CompareException.INVALID_PATH_ERROR) + return contents def rename_api(npu_name, process): @@ -13,6 +63,7 @@ def rename_api(npu_name, process): torch_func = str(torch_func_split[0]) + str(in_out) return torch_func + def read_op(op_data, op_name): op_parsed_list = [] if 'forward' in op_name: @@ -38,18 +89,19 @@ def read_op(op_data, op_name): op_parsed_list += output_parsed_list output_parsed_list.clear() if 'backward' in op_name: - if 'grad_input' in op_data: - input_item = op_data['grad_input'] + if 'input' in op_data: + input_item = op_data['input'] input_parsed_list = op_item_parse(input_item, op_name + '_input', None) op_parsed_list = input_parsed_list.copy() input_parsed_list.clear() - if 'grad_output' in op_data: - output_item = op_data['grad_output'] + if 'output' in op_data: + output_item = op_data['output'] output_parsed_list = op_item_parse(output_item, op_name + '_output', None) op_parsed_list += output_parsed_list output_parsed_list.clear() return op_parsed_list + def op_item_parse(item, op_name, index, item_list=None, top_bool=True): if item_list is None: item_list = [] @@ -121,6 +173,7 @@ def op_item_parse(item, op_name, index, item_list=None, top_bool=True): op_item_parse(item_spec, full_op_name, j, item_list=item_list, top_bool=False) return item_list + def resolve_api_special_parameters(data_dict, full_op_name, item_list): """ Function Description: @@ -269,6 +322,7 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals get_accuracy_core(n_num_input, n_num_kwarg, b_num_input, b_num_kwarg, "kwargs_struct") get_accuracy_core(n_num_input + n_num_kwarg, n_num_output, b_num_input + b_num_kwarg, b_num_output, 'output_struct') + def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): index_out = 0 npu_stack_info = n_dict.get("stack_info", None) @@ -352,7 +406,17 @@ def merge_tensor(tensor_list, summary_compare, md5_compare): return op_dict if op_dict["op_name"] else {} - +def _compare_parser(parser): + parser.add_argument("-i", "--input_path", dest="input_path", type=str, + help=" The compare input path, a dict json.", required=True) + parser.add_argument("-o", "--output_path", dest="output_path", type=str, + help=" The compare task result out path.", required=True) + parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", + help=" Whether to save stack info.", required=False) + parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", + help=" Whether to give advisor.", required=False) + parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", + help=" Whether to perform a fuzzy match on the api name.", required=False) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index db437539a..7acc607f1 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -106,6 +106,22 @@ class DataCollector: raise Exception("[msprobe] exit") self.handle_data(name, data_info) + def backward_input_data_collect(self, name, module, pid, module_input_output): + self.update_construct(name) + if not self.check_scope_and_pid(self.scope, name, pid): + return + + data_info = self.data_processor.analyze_backward_input(name, module, module_input_output) + self.handle_data(name, data_info) + + def backward_output_data_collect(self, name, module, pid, module_input_output): + self.update_construct(name) + if not self.check_scope_and_pid(self.scope, name, pid): + return + + data_info = self.data_processor.analyze_backward_output(name, module, module_input_output) + self.handle_data(name, data_info) + def update_construct(self, name): if self.config.level not in DataCollector.level_without_construct: self.data_writer.update_construct({name: self.module_processor.api_parent_node}) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 2fbc86b56..fcb522d11 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -41,6 +41,24 @@ class ModuleBackwardInputsOutputs: return convert_tuple(self.grad_output) +@dataclass +class ModuleBackwardInputs: + grad_input: Optional[Tuple] + + @property + def grad_input_tuple(self): + return convert_tuple(self.grad_input) + + +@dataclass +class ModuleBackwardOutputs: + grad_output: Optional[Tuple] + + @property + def grad_output_tuple(self): + return convert_tuple(self.grad_output) + + class TensorStatInfo: def __init__(self, max_val=None, min_val=None, mean_val=None, norm_val=None): self.max = max_val @@ -228,6 +246,28 @@ class BaseDataProcessor: return api_info_struct + def analyze_backward_input(self, name, module, + module_input_output: ModuleBackwardInputs): + api_info_struct = {} + if self.is_dump_for_data_mode(Const.BACKWARD, Const.INPUT): + api_info_struct[name] = {} + self.api_data_category = Const.INPUT + + input_info_list = self.analyze_element(module_input_output.grad_input_tuple) + api_info_struct[name][Const.INPUT] = input_info_list + return api_info_struct + + def analyze_backward_output(self, name, module, + module_input_output: ModuleBackwardOutputs): + api_info_struct = {} + if self.is_dump_for_data_mode(Const.BACKWARD, Const.OUTPUT): + api_info_struct[name] = {} + self.api_data_category = Const.OUTPUT + + output_info_list = self.analyze_element(module_input_output.grad_output_tuple) + api_info_struct[name][Const.OUTPUT] = output_info_list + return api_info_struct + def get_save_file_path(self, suffix): file_format = Const.PT_SUFFIX if self.config.framework == Const.PT_FRAMEWORK else Const.NUMPY_SUFFIX dump_data_name = (self.current_api_or_module_name + Const.SEP + self.api_data_category + Const.SEP + diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index c208df7d9..b28817e4a 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -74,8 +74,9 @@ class MindsporeDataProcessor(BaseDataProcessor): if data.numel() == 0: return tensor_stat elif data.dtype == ms.bool_: - tensor_stat.max = self.mint_ops_func["max"](data).item() - tensor_stat.min = self.mint_ops_func["min"](data).item() + data_np = data.asnumpy() + tensor_stat.max = np.max(data_np) + tensor_stat.min = np.min(data_np) elif not data.shape: tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data.item() elif data.dtype == ms.complex64 or data.dtype == ms.complex128: diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/constant.py b/debug/accuracy_tools/msprobe/core/grad_probe/constant.py index 38d33e988..189ec2d11 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/constant.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/constant.py @@ -39,7 +39,7 @@ class GradConst: DIRECTORY_LENGTH = 4096 FILE_NAME_LENGTH = 255 FILE_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" - PARAM_VALID_PATTERN = r"^[a-zA-Z0-9.]+$" + PARAM_VALID_PATTERN = r"^[a-zA-Z0-9_.]+$" DIR = "dir" FILE = "file" @@ -53,4 +53,19 @@ class GradConst: SHAPE = "shape" MAX = "max" MIN = "min" - NORM = "norm" \ No newline at end of file + NORM = "norm" + +level_adp = { + "L0": { + "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": False + }, + "L1": { + "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": True + }, + "L2": { + "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], + "have_grad_direction": True + }, + } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 26cba34f0..22acdf2fb 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -10,7 +10,6 @@ from msprobe.core.common.file_check import create_directory from msprobe.core.common.log import logger from msprobe.core.common.utils import remove_path, write_csv, load_npy from msprobe.core.grad_probe.constant import GradConst -from msprobe.pytorch.common.utils import load_pt class GradComparator: @@ -163,12 +162,8 @@ class GradComparator: @classmethod def _load_grad_files(cls, grad_file1: str, grad_file2: str): - if grad_file1.endswith('pt'): - grad1 = load_pt(grad_file1).numpy() - grad2 = load_pt(grad_file2).numpy() - else: - grad1 = load_npy(grad_file1) - grad2 = load_npy(grad_file2) + grad1 = load_npy(grad_file1) + grad2 = load_npy(grad_file2) if grad1.shape != grad2.shape: raise RuntimeError(f"tensor shape is not equal: {grad_file1}, {grad_file2}") if grad1.dtype != bool: diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py index 05dd9a568..f5db74baa 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py @@ -1,3 +1,8 @@ +import re +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.log import logger +from msprobe.core.common.utils import write_csv + def data_in_list_target(data, lst): return not lst or len(lst) == 0 or data in lst @@ -7,3 +12,41 @@ def check_numeral_list_ascend(lst): raise Exception("The input list should only contain numbers") if lst != sorted(lst): raise Exception("The input list should be ascending") + + +def check_param(param_name): + if not re.match(GradConst.PARAM_VALID_PATTERN, param_name): + raise RuntimeError("The parameter name contains special characters.") + + +def check_str(string, variable_name): + if not isinstance(string, str): + raise ValueError(f'The variable: "{variable_name}" is not a string.') + + +class ListCache(list): + threshold = 1000 + + def __init__(self, *args): + super().__init__(*args) + self._output_file = None + + def __del__(self): + self.flush() + + def flush(self): + if len(self) == 0: + return + if not self._output_file: + logger.warning("dumpfile path is not setted") + write_csv(self, self._output_file) + logger.info(f"write {len(self)} items to {self._output_file}.") + self.clear() + + def append(self, data): + list.append(self, data) + if len(self) >= ListCache.threshold: + self.flush() + + def set_output_file(self, output_file): + self._output_file = output_file diff --git a/debug/accuracy_tools/msprobe/mindspore/__init__.py b/debug/accuracy_tools/msprobe/mindspore/__init__.py index 70be41497..3bf42d1e3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/__init__.py +++ b/debug/accuracy_tools/msprobe/mindspore/__init__.py @@ -1,2 +1 @@ from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger -from .compare.distributed_compare import compare_distributed diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py deleted file mode 100644 index ec2773e6d..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os - -from msprobe.mindspore.advisor.advisor_result import AdvisorResult -from msprobe.mindspore.advisor.advisor_const import AdvisorConst -from msprobe.mindspore.common.log import logger -from msprobe.core.common.utils import CompareException -from msprobe.core.common.file_check import FileChecker -from msprobe.core.common.const import Const, CompareConst, FileCheckConst - -class Advisor: - """ - Class for generate advisor - """ - - def __init__(self, input_data, out_path=""): - self.input_data = input_data - self.out_path = os.path.realpath(out_path) - self.file_type = None - - @staticmethod - def deterministic_advisor(message, node_name): - for api_name in AdvisorConst.NEED_DETERMINISTIC_API: - if api_name in node_name: - return AdvisorConst.DETERMINISTIC_SUGGEST - return message - - @staticmethod - def batch_norm_advisor(message, node_name): - if AdvisorConst.FUNC_BATCH_NORM in node_name and AdvisorConst.FORWARD_INPUT_1 in node_name: - message = AdvisorConst.BATCH_NORM_SUGGEST - return message - - def analyze_unmatched(self, analyze_data): - if self.file_type == Const.ALL: - accuracy_unmatched = analyze_data[ - analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] - else: - accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) | - (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)] - num_unmatch = len(accuracy_unmatched) - if num_unmatch != 0: - for i in range(len(accuracy_unmatched)): - item = accuracy_unmatched.iloc[i] - logger.warning("The tensor name matches but the shape or dtype does not match: {}" - .format(item[CompareConst.NPU_NAME])) - - def gen_advisor_result(self, pd_data): - first_failing_data = pd_data.iloc[0] - node_name = first_failing_data[CompareConst.NPU_NAME] - index = first_failing_data['index'] - message = self.gen_advisor_message(node_name) - logger.warning("Find %s accuracy not reached, the line is %s" % (node_name, index)) - result = AdvisorResult(node_name, index, message) - return result - - def gen_advisor_message(self, node_name): - if AdvisorConst.FORWARD in node_name: - if AdvisorConst.INPUT in node_name: - message = AdvisorConst.FORWARD_INPUT_SUGGEST - else: - message = AdvisorConst.FORWARD_OUTPUT_SUGGEST - message = self.deterministic_advisor(message, node_name) - else: - if AdvisorConst.INPUT in node_name: - message = AdvisorConst.BACKWARD_INPUT_SUGGEST - else: - message = AdvisorConst.BACKWARD_OUTPUT_SUGGEST - message = self.deterministic_advisor(message, node_name) - message = self.batch_norm_advisor(message, node_name) - return message - - def analysis(self): - self._check_path_vaild() - analyze_data = self._parse_input_data() - logger.info("Start analyzing the comparison result: %s" % self.file_type) - self.analyze_unmatched(analyze_data) - if self.file_type == Const.ALL: - failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] - elif self.file_type == Const.MD5: - failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF] - elif self.file_type == Const.SUMMARY: - failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING] - if failing_data.empty: - logger.info("All data from api input/output accuracy reached") - result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) - else: - result = self.gen_advisor_result(failing_data) - message_list = result.print_advisor_log() - result.gen_summary_file(self.out_path, message_list) - - def _parse_input_data(self): - data_columns = self.input_data.columns.values - if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): - self.file_type = Const.ALL - elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): - self.file_type = Const.MD5 - elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): - self.file_type = Const.SUMMARY - else: - logger.error('Compare result does not meet the required conditions.') - raise CompareException(CompareException.INVALID_DATA_ERROR) - df = self.input_data.reset_index() - return df - - def _check_path_vaild(self): - out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) - out_path_checker.common_check() diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py deleted file mode 100644 index 737c67591..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_const.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - - -class AdvisorConst: - """ - Class for advisor const - """ - - # text symbol - NEW_LINE = "\n" - COLON = ": " - - # advisor summary key - SUSPECT_NODES = "Suspect Nodes" - LINE = "Line" - ADVISOR_SUGGEST = "Expert Advice" - - NO_ERROR_API = "NA" - - # advisor message - NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements." - FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \ - "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \ - "3. The fault may be caused by memory corruption and further analysis is required." - FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation." - BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected." - BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation." - BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \ - "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \ - "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \ - "3. Use seed_all(mode=True) to enable deterministic computing." - DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \ - "can seed_all(mode=True) to enable deterministic computing." - - FUNC_BATCH_NORM = "Functional_batch_norm" - FORWARD_INPUT_1 = "forward_input.1" - NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"] - BATCH_NORM = "batch_norm" - - # name keyword - INPUT = "input" - OUTPUT = "output" - FORWARD = "forward" - BACKWARD = "backward" diff --git a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py deleted file mode 100644 index 5d59068fc..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/advisor/advisor_result.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -import os -import time - -from msprobe.mindspore.advisor.advisor_const import AdvisorConst -from msprobe.mindspore.common.log import logger -from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.file_check import change_mode - - -class AdvisorResult: - """ - Class for generate advisor result - """ - - def __init__(self, node, line, message): - self.suspect_node = node - self.line = line - self.advisor_message = message - - @staticmethod - def gen_summary_file(out_path, message_list): - file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) - result_file = os.path.join(out_path, file_name) - try: - with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file: - output_file.truncate(0) - message_list = [message + AdvisorConst.NEW_LINE for message in message_list] - output_file.writelines(message_list) - change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY) - except IOError as io_error: - logger.error("Failed to save %s, the reason is %s." % (result_file, io_error)) - else: - logger.info("The advisor summary is saved in: %s" % result_file) - - def print_advisor_log(self): - logger.info("The summary of the expert advice is as follows: ") - message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line), - AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node, - AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message] - for message in message_list: - logger.info(message) - return message_list diff --git a/debug/accuracy_tools/msprobe/mindspore/common/const.py b/debug/accuracy_tools/msprobe/mindspore/common/const.py new file mode 100644 index 000000000..08bb97649 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/common/const.py @@ -0,0 +1,85 @@ +import numpy as np +import mindspore as ms + + +class Const: + CELL = "cell" + API = "api" + KERNEL = "kernel" + TOOL_LEVEL_DICT = { + "L0": CELL, + "L1": API, + "L2": KERNEL + } + PYNATIVE_MODE = "pynative" + GRAPH_GE_MODE = "graph_ge" + GRAPH_KBYK_MODE = "graph_kbyk" + + +class FreeBenchmarkConst: + DEFAULT_DEVICE = "npu" + DEFAULT_STAGE = "forward" + DEFAULT_DUMP_LEVEL = "L1" + DEFAULT_PERT_TYPE = "improve_precision" + DEFAULT_HANDLER_TYPE = "check" + FIX_HANDLER_MODE = "fix" + ADD_NOISE = "add_noise" + BIT_NOISE = "bit_noise" + NO_CHANGE = "no_change" + IMPROVE_PRECISION = "improve_precision" + CHECK = "check" + FIX = "fix" + DEVICE_LIST = ["npu"] + STAGE_LIST = ["forward"] + DUMP_LEVEL_LIST = ["L1"] + PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE] + HANDLER_TYPE_LIST = [CHECK, FIX] + COMMUNICATION_API_LIST = [ + "mindspore.communication.comm_func.all_gather_into_tensor", + "mindspore.communication.comm_func.gather_into_tensor", + "mindspore.communication.comm_func.all_reduce", + "mindspore.communication.comm_func.reduce", + "mindspore.communication.comm_func.reduce_scatter_tensor" + ] + NO_CHANGE_ERROR_THRESHOLD = 1.0 + SYMBOL_FLIPPING_RATIO = 8.0 + OPS_PREFIX = "mindspore.ops." + Tensor_PREFIX = "mindspore.Tensor." + MINT_PREFIX = "mindspore.mint." + MINT_NN_FUNC_PREFIX = "mindspore.mint.nn.functional." + COMM_PREFIX = "mindspore.communication.comm_func." + + API_PREFIX_DICT = { + "ops": OPS_PREFIX, + "Tensor": Tensor_PREFIX, + "mint": MINT_PREFIX, + "mint.nn.functional": MINT_NN_FUNC_PREFIX, + "communication": COMM_PREFIX + } + + PERT_VALUE_DICT = { + ms.bfloat16: 1e-4, + ms.float16: 1e-6, + ms.float32: 1e-8, + ms.float64: 1e-16 + } + + ERROR_THRESHOLD = { + ms.float16: 1.002, + ms.float32: 1.0002 + } + + PERT_BIT_DICT = { + ms.float16: np.int16, + ms.float32: np.int32, + ms.float64: np.int64 + } + + MS_NUMPY_DTYPE_DICT = { + ms.int16: np.int16, + ms.int32: np.int32, + ms.int64: np.int64, + ms.float16: np.float16, + ms.float32: np.float32, + ms.float64: np.float64 + } diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py index 361e957f2..4a8149657 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py @@ -4,8 +4,7 @@ from msprobe.core.common.const import FileCheckConst from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import ms_compare -from msprobe.mindspore.compare.distributed_compare import compare_distributed - +from msprobe.mindspore.compare.distributed_compare import compare_distributed def compare_cli_ms(args): with FileOpen(args.input_path, "r") as file: diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 94d03f4f2..6f84a69e9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -15,63 +15,16 @@ # limitations under the License. """ import os -import sys -import re from msprobe.core.common.utils import CompareException, check_compare_param, \ - check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid + check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import create_directory +from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import MSComparator +from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): - def check_and_return_dir_contents(dump_dir, prefix): - """ - check the given dump dir and validate files in dump dir by using the given prefix patterns to build a - pattern: ^{prefix}(?:0|[0-9][1-9]*)?$ - - Args: - dump_dir (str): dump dir - prefix (str): prefix for the patterns, prefix should be less than 20 characters and alphanumeric/-/_ only - - Returns: - content [list]: dir contents - Raises: - CompareException: invalid path - ValueError: prefix not match the patterns - - """ - check_regex_prefix_format_valid(prefix) - check_file_or_directory_path(dump_dir, True) - contents = os.listdir(dump_dir) - pattern = re.compile(rf'^{prefix}(?:0|[0-9][1-9]*)?$') - for name in contents: - if not pattern.match(name): - logger.error( - f"dump_dir contains '{name}'. Expected '{prefix}'. This name is not in the format of dump " - f"output. Please check and delete irrelevant files in {dump_dir} and try again." - ) - raise CompareException(CompareException.INVALID_PATH_ERROR) - return contents - - def extract_json(dirname, stack_json=False): - json_path = '' - for fname in os.listdir(dirname): - if fname=="construct.json": continue - full_path = os.path.join(dirname, fname) - if full_path.endswith('.json'): - json_path = full_path - if not stack_json and 'stack' not in json_path: - break - if stack_json and 'stack' in json_path: - break - - # Provide robustness on invalid directory inputs - if not json_path: - logger.error(f'No file is found in dump dir {dirname}. ') - raise CompareException(CompareException.NO_DUMP_FILE_ERROR) - return json_path - if kwargs.get('suffix'): logger.error("Argument 'suffix' is not supported for compare_distributed.") raise CompareException(CompareException.INVALID_PARAM_ERROR) @@ -87,27 +40,26 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): 'or use compare() api and manually match the ranks.') raise CompareException(CompareException.INVALID_PATH_ERROR) for nr, br in zip(npu_ranks, bench_ranks): - n_dir = os.path.join(npu_dump_dir, nr) - b_dir = os.path.join(bench_dump_dir, br) - s_dir = b_dir - npu_json_path = extract_json(n_dir, stack_json=False) - bench_json_path = extract_json(b_dir, stack_json=False) - stack_json_path = extract_json(s_dir, stack_json=True) + npu_data_dir = os.path.join(npu_dump_dir, nr) + bench_data_dir = os.path.join(bench_dump_dir, br) + npu_path = extract_json(npu_data_dir, stack_json=False) + bench_path = extract_json(bench_data_dir, stack_json=False) + stack_path = extract_json(npu_data_dir, stack_json=True) dump_result_param = { - 'npu_json_path': npu_json_path, - 'bench_json_path': bench_json_path, - 'stack_json_path': stack_json_path, + 'npu_path': npu_path, + 'bench_path': bench_path, + 'stack_path': stack_path, 'is_print_compare_log': True } try: summary_compare, md5_compare = task_dumppath_get(dump_result_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(dump_result_param, output_path, stack_mode=stack_mode, summary_compare=summary_compare) - except CompareException as error: + check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) + except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) + raise CompareException(error.code) from error msComparator=MSComparator() msComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 34d37b4fe..be7439cb0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,27 +1,24 @@ import json -import multiprocessing import os.path -import sys import numpy as np -import pandas as pd from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.const import FileCheckConst -from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op -from msprobe.core.compare.Multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process +from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy +from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException class MSComparator (Comparator): - def __init__(self): - super().__init__() + def __init__(self): + self.frame_name=MSComparator.__name__ def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): cos_result = [] @@ -59,19 +56,7 @@ class MSComparator (Comparator): ) return _save_cmp_result(idx, cr, result_df, lock) - - - def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): - op_data = json_data['data'][op_name] - op_parsed_list = read_op(op_data, op_name) - if op_name in stack_json_data: - op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) - else: - op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) - - merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) - return merge_list - + def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles npu_json_data = json.load(npu_json_handle) @@ -134,42 +119,9 @@ class MSComparator (Comparator): for npu_data in npu_ops_queue: get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) - return result_df - - def make_result_table(self,result,md5_compare,summary_compare,stack_mode): - header = [] - if md5_compare: - header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] - elif summary_compare: - header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] - else: - header = CompareConst.COMPARE_RESULT_HEADER[:] - - all_mode_bool = not (summary_compare or md5_compare) - if stack_mode: - if all_mode_bool: - header.append(CompareConst.STACK) - header.append(CompareConst.DATA_NAME) - else: - header.append(CompareConst.STACK) - else: - if all_mode_bool: - for row in result: - del row[-2] - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] - result_df = pd.DataFrame(result, columns=header) - return result_df + return result_df - def _do_multi_process(self,input_parma, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e + def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) @@ -180,7 +132,7 @@ class MSComparator (Comparator): if data_value.dtype == np.float16: data_value=data_value.astype(np.float32) - return data_value + return data_value def compare_core(self,input_parma, output_path, **kwargs): """ @@ -226,8 +178,7 @@ class MSComparator (Comparator): if auto_analyze: advisor = Advisor(result_df, output_path) advisor.analysis() - - + def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) @@ -236,16 +187,8 @@ def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) + raise CompareException(error.code) from error msComparator=MSComparator() msComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) - - - - - - - - \ No newline at end of file + md5_compare=md5_compare) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 23cb7294b..54f640703 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -1,11 +1,15 @@ import os +from pathlib import Path -from msprobe.core.common.utils import Const -from msprobe.core.common.const import MsConst +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import Const as MsConst +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create class DebuggerConfig: def __init__(self, common_config, task_config): + self.execution_mode = None self.dump_path = common_config.dump_path self.task = common_config.task self.rank = [] if not common_config.rank else common_config.rank @@ -23,6 +27,19 @@ class DebuggerConfig: self.framework = Const.MS_FRAMEWORK self.summary_mode = task_config.summary_mode self.check() + self._make_dump_path_if_not_exists() + + if self.task == Const.FREE_BENCHMARK: + self.pert_type = (FreeBenchmarkConst.DEFAULT_PERT_TYPE + if not task_config.pert_mode else task_config.pert_mode) + self.handler_type = (FreeBenchmarkConst.DEFAULT_HANDLER_TYPE + if not task_config.handler_type else task_config.handler_type) + if self.handler_type == FreeBenchmarkConst.FIX_HANDLER_MODE and \ + self.pert_type != FreeBenchmarkConst.DEFAULT_PERT_TYPE: + raise ValueError("pert_mode must be improve_precision or empty when handler_type is fix, " + f"but got {self.pert_type}.") + self.dump_level = FreeBenchmarkConst.DEFAULT_DUMP_LEVEL + self.stage = FreeBenchmarkConst.DEFAULT_STAGE def check(self): if not self.dump_path: @@ -50,3 +67,10 @@ class DebuggerConfig: for s in self.step: if not isinstance(s, int): raise ValueError(f"step element {s} should be int") + + def _make_dump_path_if_not_exists(self): + check_path_before_create(self.dump_path) + if not os.path.exists(self.dump_path): + Path(self.dump_path).mkdir(mode=0o750, exist_ok=True) + file_check = FileChecker(self.dump_path, FileCheckConst.DIR) + file_check.common_check() diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 5475dc358..0b51efec8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -6,13 +6,18 @@ from msprobe.mindspore.service import Service from msprobe.mindspore.ms_config import parse_json_config from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.task_handler_factory import TaskHandlerFactory -from msprobe.core.common.const import MsConst +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import Const as MsConst +from msprobe.mindspore.runtime import Runtime + +from msprobe.mindspore.grad_probe.grad_monitor import GradientMonitor class PrecisionDebugger: _instance = None + task_not_need_service = [Const.GRAD_PROBE] - def __new__(cls, config_path=None): + def __new__(cls, config_path=None, opt=None): if not cls._instance: cls._instance = super().__new__(cls) cls._instance.initialized = False @@ -24,11 +29,18 @@ class PrecisionDebugger: def __init__(self, config_path=None): if self.initialized: return + self.initialized = True if not config_path: config_path = os.path.join(os.path.dirname(__file__), "../../config/config.json") common_config, task_config = parse_json_config(config_path) + self.task = common_config.task + if self.task == Const.GRAD_PROBE: + self.gm = GradientMonitor(common_config, task_config) + return self.config = DebuggerConfig(common_config, task_config) - self.initialized = True + + Runtime.step_count = 0 + Runtime.is_running = False @staticmethod def _get_execution_mode(): @@ -41,35 +53,56 @@ class PrecisionDebugger: return MsConst.PYNATIVE_MODE @classmethod - def start(cls): + def start(cls, target=None): instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") + if instance.task in PrecisionDebugger.task_not_need_service: + return instance.config.execution_mode = instance._get_execution_mode() - if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.level == MsConst.API: + if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.level == MsConst.API and \ + instance.config.task != Const.FREE_BENCHMARK: if not instance.service: instance.service = Service(instance.config) - instance.service.start() + instance.service.start(target) else: if not instance.first_start: handler = TaskHandlerFactory.create(instance.config) handler.handle() instance.first_start = True + Runtime.is_running = True @classmethod def stop(cls): instance = cls._instance if not instance: raise Exception("PrecisionDebugger instance is not created.") + if instance.task == Const.GRAD_PROBE: + instance.gm.stop() + if instance.task in PrecisionDebugger.task_not_need_service: + return if instance.service: instance.service.stop() + Runtime.is_running = False @classmethod def step(cls): instance = cls._instance if not instance: raise Exception("PrecisionDebugger instance is not created.") + if instance.task in PrecisionDebugger.task_not_need_service: + return if instance.service: instance.service.step() + Runtime.step_count += 1 + + @classmethod + def monitor(cls, opt): + instance = cls._instance + if not instance: + raise Exception("PrecisionDebugger instance is not created.") + if instance.task != Const.GRAD_PROBE: + return + instance.gm.monitor(opt) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md index 425d0683a..ef2431b9c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md @@ -35,10 +35,18 @@ PrecisionDebugger(config_path=None) **原型** ```Python -debugger.start() +debugger.start(model = None) ``` -该函数为类函数，可以使用debugger.start()也可以使用PrecisionDebugger.start()。 +该函数为类函数，可以使用debugger.start(model = None)也可以使用PrecisionDebugger.start(model = None) + + +**参数说明** + +| 参数名 | 说明 | 是否必选 | +| ----------- |---------------------------------------------------------------------------------------| -------- | +| model | 指具体的mindspore.nn.Cell，默认未配置，L1级别下传入model可以使能对primitive op的dump，否则无法dump primitive op。 | 否 | + ## 示例代码 diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/__init__.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py new file mode 100644 index 000000000..bcfa31520 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -0,0 +1,116 @@ +import os +import inspect +import importlib + +import yaml +import mindspore as ms +from mindspore.communication import comm_func + +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.core.common.file_check import check_path_length, FileOpen +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.mindspore.free_benchmark.decorator.decorator_factory import decorate_forward_function + + +class ApiPyNativeSelFCheck: + def __init__(self, config: DebuggerConfig): + Config.is_enable = True + Config.handler_type = config.handler_type + Config.pert_type = config.pert_type + Config.stage = config.stage + Config.dump_level = config.dump_level + Config.steps = config.step + Config.ranks = config.rank + Config.dump_path = os.path.join(config.dump_path, "free_benchmark.csv") + check_path_length(Config.dump_path) + + self.api_list = config.list + all_api = get_supported_ops() + if not self.api_list: + self.api_list = all_api + else: + self.api_list = set(self.api_list) & all_api + + def handle(self): + for api_name in self.api_list: + hijack(api_name) + + +def get_supported_ops(): + supported_ops = [] + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(cur_path, "data", "support_wrap_ops.yaml") + + for k, v in FreeBenchmarkConst.API_PREFIX_DICT.items(): + with FileOpen(yaml_path, 'r') as f: + ops = yaml.safe_load(f).get(k) + if ops: + ops = [v + i for i in ops] + supported_ops += ops + + _all_functional_ops = [] + ms_ops = dir(ms.ops) + ms_ops = [FreeBenchmarkConst.OPS_PREFIX + i for i in ms_ops] + _all_functional_ops += ms_ops + + ms_tensor = dir(ms.Tensor) + ms_tensor = [FreeBenchmarkConst.Tensor_PREFIX + i for i in ms_tensor] + _all_functional_ops += ms_tensor + + ms_mint = dir(ms.mint) + ms_mint = [FreeBenchmarkConst.MINT_PREFIX + i for i in ms_mint] + _all_functional_ops += ms_mint + + ms_mint_nn_func = dir(ms.mint.nn.functional) + ms_mint_nn_func = [FreeBenchmarkConst.MINT_NN_FUNC_PREFIX + i for i in ms_mint_nn_func] + _all_functional_ops += ms_mint_nn_func + + ms_communication = dir(comm_func) + ms_communication = [FreeBenchmarkConst.COMM_PREFIX + i for i in ms_communication] + _all_functional_ops += ms_communication + + return set(supported_ops) & set(_all_functional_ops) + + +def get_decorate_func(): + return decorate_forward_function + + +def is_func_support_decorate(orig_func): + return not inspect.isclass(orig_func) and callable(orig_func) + + +def get_wrapper_obj(orig_func, api_name): + if is_func_support_decorate(orig_func): + wrapped_obj = get_decorate_func()(orig_func, api_name) + else: + wrapped_obj = orig_func + return wrapped_obj + + +def get_module(api_name): + func_name_list = api_name.split(Const.SEP) + func_name = func_name_list[-1] + module_obj = importlib.import_module(func_name_list[0]) + for i, module_name in enumerate(func_name_list[1:-1]): + if not hasattr(module_obj, module_name): + importlib.import_module(f"{Const.SEP.join(func_name_list[:i+2])}") + module_obj = getattr(module_obj, module_name) + orig_func = getattr(module_obj, func_name) + + return module_obj, orig_func + + +def hijack(api_name): + if not api_name.strip(): + return + try: + func_name = api_name.split(Const.SEP)[-1] + module_obj, origin_func = get_module(api_name) + wrapped_obj = get_wrapper_obj(origin_func, api_name) + setattr(module_obj, func_name, wrapped_obj) + except Exception as e: + logger.error(f"Failed decorator {api_name}: {e}") diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/__init__.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py new file mode 100644 index 000000000..85f684d81 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/config.py @@ -0,0 +1,12 @@ +from msprobe.mindspore.common.const import FreeBenchmarkConst + + +class Config: + is_enable: bool = False + handler_type = FreeBenchmarkConst.DEFAULT_HANDLER_TYPE + pert_type = FreeBenchmarkConst.DEFAULT_PERT_TYPE + stage = FreeBenchmarkConst.DEFAULT_STAGE + dump_level = FreeBenchmarkConst.DEFAULT_DUMP_LEVEL + steps: list = [] + ranks: list = [] + dump_path: str = "" diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/handler_params.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/handler_params.py new file mode 100644 index 000000000..ae1733b98 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/handler_params.py @@ -0,0 +1,17 @@ +from typing import Optional, Any, Tuple, Dict, Callable + + +class HandlerParams: + """ + 参数结合体 + + """ + args: Optional[Tuple] = None + kwargs: Optional[Dict] = None + index: Optional[int] = None + original_result: Optional[Any] = None + fuzzed_result: Optional[Any] = None + is_consistent: Optional[bool] = True + save_flag: Optional[bool] = True + fuzzed_value: Optional[Any] = None + original_func: Optional[Callable] = None diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py new file mode 100644 index 000000000..3bb062800 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py @@ -0,0 +1,71 @@ +from typing import Any +from typing import Optional +from dataclasses import dataclass + +import mindspore as ms +from mindspore import Tensor + +from msprobe.mindspore.runtime import Runtime +from msprobe.mindspore.common.const import FreeBenchmarkConst +from .config import Config +from .handler_params import HandlerParams + + +class Tools: + + @staticmethod + def get_first_tensor_dtype(tensor_seq: Any): + if isinstance(tensor_seq, Tensor): + return tensor_seq.dtype + if isinstance(tensor_seq, (list, tuple)): + for i in tensor_seq: + if isinstance(i, Tensor): + return i.dtype + raise Exception("The sequence does not contain tensors.") + + @staticmethod + def get_default_error_threshold(dtype): + if Config.pert_type == FreeBenchmarkConst.NO_CHANGE: + return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + return FreeBenchmarkConst.ERROR_THRESHOLD.get(dtype, FreeBenchmarkConst.ERROR_THRESHOLD.get(ms.float32)) + + +@dataclass +class UnequalRow: + rank: Optional[int] = None + pert_type: Optional[str] = None + stage: Optional[str] = None + step: Optional[int] = None + api_name: Optional[str] = None + max_rel: Optional[float] = None + dtype: Optional[str] = None + shape: Optional[str] = None + output_index: Optional[int] = None + + +def make_unequal_row( + api_name: str, + params: HandlerParams, + ratio: float = None, + index: int = None, +): + row = UnequalRow( + api_name=api_name, + pert_type=Config.pert_type, + output_index=index, + stage=Config.stage, + step=Runtime.step_count + ) + if isinstance(ratio, float): + row.max_rel = ratio - 1 + original_tensor = params.original_result + fuzzed_tensor = params.fuzzed_result + if index: + original_tensor = original_tensor[index] + fuzzed_tensor = fuzzed_tensor[index] + row.output_index = index + if isinstance(original_tensor, Tensor): + row.dtype = original_tensor.dtype + row.shape = original_tensor.shape + row.rank = Runtime.rank_id if Runtime.rank_id != -1 else None + return row diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml new file mode 100644 index 000000000..cc802d381 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/data/support_wrap_ops.yaml @@ -0,0 +1,842 @@ +# List of apis that support self check + +communication: + - all_gather_into_tensor + - gather_into_tensor + - all_reduce + - reduce + - reduce_scatter_tensor + +ops: + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d + - adaptive_max_pool1d + - adaptive_max_pool2d + - avg_pool1d + - avg_pool2d + - avg_pool3d + - batch_norm + - bias_add + - ctc_greedy_decoder + - conv1d + - conv2d + - conv3d + - deformable_conv2d + - dense + - dropout + - dropout1d + - dropout2d + - dropout3d + - flatten + - fold + - fractional_max_pool3d + - lp_pool1d + - lp_pool2d + - lrn + - max_pool2d + - max_pool3d + - max_unpool1d + - max_unpool2d + - max_unpool3d + - unfold + - binary_cross_entropy + - binary_cross_entropy_with_logits + - cosine_embedding_loss + - cross_entropy + - ctc_loss + - gaussian_nll_loss + - hinge_embedding_loss + - huber_loss + - kl_div + - l1_loss + - margin_ranking_loss + - mse_loss + - multi_margin_loss + - multilabel_margin_loss + - multilabel_soft_margin_loss + - nll_loss + - smooth_l1_loss + - triplet_margin_loss + - elu + - fast_gelu + - gelu + - glu + - gumbel_softmax + - hardshrink + - hardsigmoid + - hardswish + - hardtanh + - leaky_relu + - log_softmax + - logsigmoid + - mish + - prelu + - relu + - relu6 + - rrelu + - selu + - sigmoid + - silu + - softmax + - softmin + - softshrink + - softsign + - tanh + - threshold + - cdist + - dist + - pdist + - choice_with_mask + - random_categorical + - log_uniform_candidate_sampler + - uniform_candidate_sampler + - affine_grid + - bounding_box_decode + - bounding_box_encode + - col2im + - check_valid + - crop_and_resize + - grid_sample + - interpolate + - iou + - pad + - padding + - pixel_shuffle + - pixel_unshuffle + - upsample + - abs + - absolute + - accumulate_n + - acos + - arccos + - acosh + - add + - addcdiv + - addcmul + - addmv + - addn + - angle + - arccosh + - arcsin + - arcsinh + - arctan + - arctanh + - arctan2 + - asin + - asinh + - atan + - atan2 + - atanh + - atleast_1d + - atleast_2d + - atleast_3d + - bessel_i0 + - bessel_i0e + - bessel_i1 + - bessel_i1e + - bessel_j0 + - bessel_j1 + - bessel_k0 + - bessel_k0e + - bessel_k1 + - bessel_k1e + - bessel_y0 + - bessel_y1 + - bitwise_and + - bitwise_left_shift + - bitwise_or + - bitwise_right_shift + - bitwise_xor + - ceil + - clamp + - clip + - combinations + - copysign + - cos + - cosh + - cosine_similarity + - cov + - diag_embed + - diff + - deg2rad + - digamma + - div + - divide + - erf + - erfc + - erfinv + - exp + - exp2 + - expm1 + - floor + - floor_div + - floor_mod + - float_power + - fmod + - frac + - gcd + - hypot + - igamma + - igammac + - imag + - i0 + - inv + - invert + - lcm + - ldexp + - lerp + - log + - log2 + - log10 + - log1p + - logaddexp + - logaddexp2 + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - mul + - multiply + - mvlgamma + - neg + - negative + - nextafter + - polar + - polygamma + - positive + - pow + - rad2deg + - ravel + - real + - reciprocal + - remainder + - rot90 + - round + - rsqrt + - sgn + - sign + - signbit + - sin + - sinc + - sinh + - sqrt + - square + - sub + - subtract + - t + - tan + - tanhshrink + - trapz + - tril_indices + - triu_indices + - true_divide + - trunc + - truncate_div + - truncate_mod + - xdivy + - xlogy + - zeta + - all + - amax + - amin + - aminmax + - any + - argmax + - argmin + - cummax + - cummin + - cumprod + - cumsum + - fmax + - histc + - logsumexp + - max + - mean + - median + - min + - norm + - prod + - std + - std_mean + - var + - var_mean + - argsort + - approximate_equal + - equal + - ge + - greater + - greater_equal + - gt + - intopk + - isclose + - isfinite + - isinf + - isnan + - isneginf + - isposinf + - isreal + - le + - less + - less_equal + - lt + - maximum + - minimum + - msort + - ne + - not_equal + - searchsorted + - topk + - bmm + - addbmm + - addmm + - baddbmm + - addr + - adjoint + - cholesky + - cholesky_solve + - batch_dot + - dot + - eig + - inner + - inverse + - geqrf + - ger + - kron + - lu_solve + - lu_unpack + - matmul + - matrix_solve + - matrix_band_part + - matrix_diag + - matrix_diag_part + - matrix_set_diag + - mm + - mv + - outer + - orgqr + - ormqr + - pinv + - svd + - tensor_dot + - logdet + - slogdet + - qr + - trace + - bartlett_window + - blackman_window + - hamming_window + - hann_window + - kaiser_window + - eye + - fill + - full + - full_like + - linspace + - logspace + - one_hot + - arange + - range + - heaviside + - bernoulli + - gamma + - laplace + - multinomial + - multinomial_with_replacement + - rand + - rand_like + - randint + - randint_like + - randn + - randn_like + - random_gamma + - random_poisson + - randperm + - standard_laplace + - standard_normal + - uniform + - argwhere + - batch_to_space_nd + - bincount + - block_diag + - broadcast_to + - cat + - channel_shuffle + - chunk + - column_stack + - concat + - conj + - count_nonzero + - deepcopy + - diag + - diagflat + - diagonal + - dyn_shape + - dsplit + - dstack + - einsum + - expand + - expand_dims + - flip + - fliplr + - flipud + - gather_d + - gather_elements + - gather_nd + - hsplit + - hstack + - masked_fill + - masked_select + - meshgrid + - moveaxis + - movedim + - narrow + - nan_to_num + - nansum + - normal + - nonzero + - population_count + - rank + - repeat_elements + - repeat_interleave + - reshape + - reverse + - reverse_sequence + - roll + - select + - sequence_mask + - shuffle + - size + - slice + - sort + - space_to_batch_nd + - sparse_segment_mean + - split + - squeeze + - stack + - strided_slice + - sum + - swapaxes + - swapdims + - tensor_split + - tile + - tril + - triu + - transpose + - unbind + - unique + - unique_consecutive + - unique_with_pad + - unsorted_segment_max + - unsorted_segment_min + - unsorted_segment_prod + - unsorted_segment_sum + - unsqueeze + - unstack + - view_as_real + - vsplit + - vstack + - where + - cross + - renorm + - tuple_to_array + - clip_by_global_norm + - clip_by_value + - derivative + - jet + +Tensor: + - __abs__ + - __add__ + - __and__ + - __iadd__ + - __ifloordiv__ + - __imatmul__ + - __imod__ + - __imul__ + - __isub__ + - __matmul__ + - __mod__ + - __mul__ + - __neg__ + - __or__ + - __pow__ + - __radd__ + - __rmatmul__ + - __rmod__ + - __rmul__ + - __rpow__ + - __rsub__ + - __sub__ + - __truediv__ + - __xor__ + - abs + - absolute + - acos + - acosh + - add + - addbmm + - addcdiv + - addcmul + - addmm + - addmv + - addr + - all + - amax + - amin + - any + - arccos + - arccosh + - argmax + - angle + - arcsin + - arcsinh + - arctan + - arctanh + - argmin + - argsort + - asin + - asinh + - atan + - atan2 + - atanh + - baddbmm + - bernoulli + - bincount + - bitwise_and + - bitwise_or + - bitwise_xor + - bmm + - broadcast_to + - ceil + - cholesky_solve + - cholesky + - clamp + - clip + - conj + - copysign + - cos + - cosh + - cross + - cummax + - cummin + - cumprod + - cumsum + - deg2rad + - diag + - diagflat + - diff + - digamma + - div + - divide + - equal + - erf + - erfc + - erfinv + - exp + - expand_as + - expm1 + - flip + - fliplr + - flipud + - float_power + - floor + - fmod + - frac + - gather_elements + - geqrf + - ger + - greater + - greater_equal + - half + - hardshrink + - heaviside + - histc + - hypot + - i0 + - igamma + - igammac + - imag + - index_add + - index_fill + - index_put + - index_select + - inner + - int + - inverse + - item + - lcm + - ldexp + - lerp + - log + - log10 + - log1p + - log2 + - logaddexp + - logaddexp2 + - logdet + - logical_and + - logical_not + - logical_or + - logical_xor + - logit + - logsumexp + - long + - masked_fill + - masked_scatter + - masked_select + - matmul + - max + - maximum + - mean + - median + - min + - minimum + - moveaxis + - movedim + - msort + - multinomial + - multiply + - mvlgamma + - nan_to_num + - nansum + - narrow + - neg + - negative + - nelement + - new_ones + - new_zeros + - nextafter + - norm + - nonzero + - not_equal + - ormqr + - permute + - pow + - prod + - qr + - ravel + - real + - reciprocal + - remainder + - renorm + - rad2deg + - tile + - repeat_interleave + - reshape + - reshape + - round + - rot90 + - rsqrt + - sum_to_size + - scatter + - sgn + - short + - sigmoid + - sign + - signbit + - sin + - sinc + - sinh + - slogdet + - sort + - split + - sqrt + - square + - squeeze + - std + - subtract + - subtract + - svd + - swapaxes + - swapdims + - t + - take + - tan + - tanh + - trace + - swapaxes + - tile + - topk + - tril + - tensor_split + - transpose + - true_divide + - trunc + - unbind + - unique_consecutive + - unsqueeze + - var + - view + - where + - xlogy + - from_numpy + - std + - take + - var + - all + - any + - copy + - diagonal + - flatten + - resize + - sum + +mint: + - abs + - absolute_import + - add + - add_ex + - all + - any + - any_ex + - arange + - argmax + - avg_pool2d + - baddbmm + - baddbmm_ex + - batch_norm + - binary_cross_entropy_with_logits + - bitwise_and + - bitwise_or + - bitwise_xor + - bmm + - broadcast_to + - cat + - cat_ex + - ceil + - chunk + - clamp + - conv2d + - conv_transpose2d + - cos + - cross + - cummax + - cummin + - cumsum + - div + - divide + - dropout + - embedding + - eq + - erf + - erfinv + - exp + - flatten + - flip + - flip_ex + - fold + - full + - gather + - gelu + - greater + - grid_sample + - group_norm + - gt + - index_select + - interpolate + - isclose + - isfinite + - layer_norm + - le + - leaky_relu + - less + - less_equal + - linear + - linspace + - log + - logical_and + - logical_not + - logical_or + - lt + - masked_select + - matmul + - max + - max_pool2d + - maximum + - mean + - mean_ex + - min + - minimum + - mul + - ne + - neg + - negative + - nonzero + - normal + - one_hot + - ones + - ones_ex + - ones_like + - pad + - permute + - permute_ex + - pow + - prod + - reciprocal + - relu + - remainder + - repeat_interleave + - rsqrt + - searchsorted + - sigmoid + - silu + - sin + - softmax + - softplus + - sort + - split + - sqrt + - sqrt_ex + - square + - stack + - sub + - sub_ex + - sum + - tanh + - tile + - topk + - tril + - triu + - unfold + - unique + - where + - xlogy + - zeros + - zeros_ex + - zeros_like + +mint.nn.functional: + - absolute_import + - avg_pool2d + - batch_norm + - batch_norm_ex + - bce_with_logits + - binary_cross_entropy_with_logits + - conv_transpose2d + - dense + - dropout + - embedding + - fold + - gelu + - grid_sample + - group_norm + - interpolate + - layer_norm + - leaky_relu + - linear + - max_pool2d + - max_pool2d_ex + - normal + - one_hot + - one_hot_ext + - pad + - relu + - sigmoid + - silu + - softmax + - softmax_ex + - softplus + - tanh + - unfold diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/__init__.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py new file mode 100644 index 000000000..78661d7fc --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py @@ -0,0 +1,42 @@ +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.mindspore.free_benchmark.handler.handler_factory import HandlerFactory +from msprobe.mindspore.free_benchmark.perturbation.perturbation_factory import PerturbationFactory + + +class ForwardSelfChecker: + + def __init__(self, api_name: str): + self.api_name = api_name + + def handle(self, params: HandlerParams): + """ + 装饰器实际执行逻辑 + + """ + perturbation = PerturbationFactory.create(self.api_name) + params.fuzzed_result = perturbation.handle(params) + params.original_result = params.original_func(*params.args, **params.kwargs) + if params.fuzzed_result is not False: + return self.deal_fuzzed_and_original_result(params) + return params.original_result + + def get_compare_data(self, params: HandlerParams): + if self.api_name not in FreeBenchmarkConst.COMMUNICATION_API_LIST: + return + # 以下为通讯类api处理逻辑 + params.fuzzed_result = params.fuzzed_value + if Config.pert_type == FreeBenchmarkConst.IMPROVE_PRECISION: + params.original_result = params.args + else: + params.original_result = params.args[params.index] + + def deal_fuzzed_and_original_result(self, params: HandlerParams): + original_result = params.original_result + self.get_compare_data(params) + handler = HandlerFactory.create(self.api_name) + result = handler.handle(params) + if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST: + result = original_result + return result diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py new file mode 100644 index 000000000..c1cf50e9c --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/decorator_factory.py @@ -0,0 +1,107 @@ +import os +import sys +import traceback +from functools import wraps +from typing import Tuple, Dict, List + +from mindspore import ops + +from msprobe.mindspore.runtime import Runtime +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from .dec_forward import ForwardSelfChecker + + +def decorate(original_func, decorate_func, api_name=None): + """ + 总装饰器 + """ + @wraps(original_func) + def fuzz_wrapper(*args, **kwargs): + + def __exec_decorate_func(): + params = data_pre_deal(api_name, original_func, *args, **kwargs) + result = decorate_func(params) + return result + + try: + if Runtime.rank_id == -1: + Runtime.rank_id = os.environ.get("RANK_ID", -1) + if need_wrapper_func(): + logger.info(f"[{api_name}] is checking.") + return __exec_decorate_func() + except Exception as e: + logger.error(f"[{api_name}] Error: {str(e)}") + logger.error(f"[{api_name}] Error detail: {traceback.format_exc()}") + + return original_func(*args, **kwargs) + + return fuzz_wrapper + + +def decorate_forward_function(func, api_name=None): + """ + 前向装饰器 + """ + + if not api_name: + api_name = func.__name__ + + def forward_func(params: HandlerParams): + forward = ForwardSelfChecker(api_name) + result = forward.handle(params) + return result + + return decorate(func, forward_func, api_name) + + +def stack_depth_check() -> bool: + nested_depth = 1 + frame = sys._getframe(1) + while frame: + if frame.f_code.co_name == "fuzz_wrapper": + nested_depth -= 1 + if nested_depth < 0: + return False + frame = frame.f_back + return True + + +def get_target_arg_index(args: Tuple) -> int: + """ + 类型校验 + + """ + for i, arg in enumerate(args): + if ops.is_tensor(arg): + if not ops.is_floating_point(arg): + continue + return i + if isinstance(arg, (List, Tuple, Dict)): + return i + return -1 + + +def data_pre_deal(api_name, func, *args, **kwargs): + params = HandlerParams() + params.args = args + params.kwargs = kwargs + params.original_func = func + index = get_target_arg_index(args) + if index == -1: + raise Exception(f"{api_name} has no supported input type") + params.index = index + return params + + +def need_wrapper_func(): + if not (Runtime.is_running and Config.is_enable): + return False + if not stack_depth_check(): + return False + if Config.steps and Runtime.step_count not in Config.steps: + return False + if Config.ranks and Runtime.rank_id != -1 and Runtime.rank_id not in Config.ranks: + return False + return True diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/__init__.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py new file mode 100644 index 000000000..f35d23498 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/base_handler.py @@ -0,0 +1,90 @@ +import math +from abc import ABC, abstractmethod +from typing import Any, Tuple, Optional + +import mindspore as ms +from mindspore import Tensor, ops + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.utils import Tools +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class BaseHandler(ABC): + + def __init__(self, api_name: str): + self.api_name = api_name + + @staticmethod + def pre_calculate(original_output, fuzzed_output): + abs_tol = FreeBenchmarkConst.PERT_VALUE_DICT.get(fuzzed_output.dtype, + FreeBenchmarkConst.PERT_VALUE_DICT.get(ms.float32)) + + return original_output.to(fuzzed_output.dtype), fuzzed_output, abs_tol + + @staticmethod + def get_threshold(dtype): + err = Tools.get_default_error_threshold(dtype) + return err + + @staticmethod + def convert_overflow_ratio_to_consistent(ratio): + if math.isnan(ratio) or math.isinf(ratio): + return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + return ratio + + @staticmethod + def get_endless_norm(first_tensor, second_tensor, abs_tol): + if first_tensor.dtype != ms.bfloat16 and second_tensor.dtype != ms.bfloat16: + ratio_tensor1 = ops.where(ops.abs(second_tensor) > abs_tol, ops.div(first_tensor, second_tensor), 1) + ratio_tensor2 = ops.where(ops.abs(first_tensor) > abs_tol, ops.div(second_tensor, first_tensor), 1) + else: + ratio_tensor1 = ops.where(ops.abs(second_tensor).to(ms.float32) > abs_tol, + ops.div(first_tensor.to(ms.float32), second_tensor.to(ms.float32)), 1) + ratio_tensor2 = ops.where(ops.abs(first_tensor).to(ms.float32) > abs_tol, + ops.div(second_tensor.to(ms.float32), first_tensor.to(ms.float32)), 1) + norm1 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor1)[0].to(ms.float32).item()) + norm2 = BaseHandler.convert_overflow_ratio_to_consistent(ops.max(ratio_tensor2)[0].to(ms.float32).item()) + norm3 = BaseHandler.convert_overflow_ratio_to_consistent(ops.min(ratio_tensor1)[0].to(ms.float32).item()) + ratio = FreeBenchmarkConst.SYMBOL_FLIPPING_RATIO if norm3 < 0 else max(norm1, norm2) + + return ratio + + @staticmethod + def ratio_calculate(original_output, fuzzed_output) -> float: + try: + original_output, fuzzed_output, abs_tol = BaseHandler.pre_calculate(original_output, fuzzed_output) + except Exception as e: + logger.error(f"When computing ratio, y1 or y2 dtype is not supported {str(e)}") + return FreeBenchmarkConst.NO_CHANGE_ERROR_THRESHOLD + + abs_tol = abs_tol ** 0.5 + + return BaseHandler.get_endless_norm(original_output, fuzzed_output, abs_tol) + + @staticmethod + def npu_compare(original_output, fuzzed_output) -> Tuple[bool, Optional[float]]: + if not isinstance(fuzzed_output, Tensor): + logger.error(f"The compare for output type `{type(fuzzed_output)}` is not supported") + return True, 1.0 + + # 范数计算等 + err_thd = BaseHandler.get_threshold(original_output.dtype) + ratio = BaseHandler.ratio_calculate(original_output, fuzzed_output) + is_consistent = err_thd >= ratio >= 1.0 / err_thd + return is_consistent, ratio + + @staticmethod + def is_float_tensor(output) -> bool: + if isinstance(output, Tensor) and ops.is_floating_point(output): + return True + if isinstance(output, (list, tuple)): + for i in output: + if isinstance(i, Tensor) and ops.is_floating_point(i): + return True + return False + + @abstractmethod + def handle(self, params: HandlerParams) -> Any: + pass diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/check_handler.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/check_handler.py new file mode 100644 index 000000000..df80e76c0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/check_handler.py @@ -0,0 +1,41 @@ +from typing import Any +from dataclasses import asdict + +from mindspore import Tensor, ops + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.mindspore.free_benchmark.handler.base_handler import BaseHandler +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.mindspore.free_benchmark.common.utils import make_unequal_row +from msprobe.core.data_dump.json_writer import DataWriter + + +class CheckHandler(BaseHandler): + + def npu_compare_and_save(self, original_output, fuzzed_output, params: HandlerParams, output_index=None): + is_consistent, ratio = self.npu_compare(original_output, fuzzed_output) + params.is_consistent = params.is_consistent and is_consistent + if not is_consistent: + row = make_unequal_row(self.api_name, params, ratio, output_index) + data_dict = asdict(row) + DataWriter.write_data_to_csv( + data_dict.values(), + data_dict.keys(), + Config.dump_path + ) + logger.error(f"{self.api_name} is not consistent") + + def handle(self, params: HandlerParams) -> Any: + try: + if not self.is_float_tensor(params.fuzzed_result): + return params.original_result + if isinstance(params.fuzzed_result, Tensor): + self.npu_compare_and_save(params.original_result, params.fuzzed_result, params) + elif isinstance(params.fuzzed_result, (list, tuple)): + for i, item in enumerate(params.original_result): + if ops.is_tensor(item) and ops.is_floating_point(item): + self.npu_compare_and_save(item, params.fuzzed_result[i], params, output_index=i) + except Exception as e: + logger.error(str(e)) + return params.original_result diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/fix_handler.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/fix_handler.py new file mode 100644 index 000000000..2c377ba89 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/fix_handler.py @@ -0,0 +1,36 @@ +from typing import Any + +from mindspore import Tensor + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class FixHandler: + + def __init__(self, api_name: str): + self.api_name = api_name + + @staticmethod + def use_fuzzed_result(original_result, fuzzed_result): + if isinstance(original_result, Tensor): + return fuzzed_result.to(original_result.dtype) + if isinstance(original_result, dict): + dict_fixed_result = dict() + for k, v in original_result.items(): + dict_fixed_result[k] = FixHandler.use_fuzzed_result(v, fuzzed_result[k]) + return dict_fixed_result + if isinstance(original_result, (tuple, list)): + list_fixed_result = list() + for i, v in enumerate(original_result): + list_fixed_result.append(FixHandler.use_fuzzed_result(v, fuzzed_result[i])) + return type(original_result)(list_fixed_result) + return original_result + + def handle(self, params: HandlerParams) -> Any: + try: + return FixHandler.use_fuzzed_result(params.original_result, params.fuzzed_result) + except Exception as e: + logger.error(f"{self.api_name} failed to fix.") + logger.error(str(e)) + return params.original_result diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py new file mode 100644 index 000000000..bf8c681e5 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/handler/handler_factory.py @@ -0,0 +1,21 @@ +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.mindspore.common.const import FreeBenchmarkConst +from .check_handler import CheckHandler +from .fix_handler import FixHandler + + +class HandlerFactory: + result_handlers = { + FreeBenchmarkConst.CHECK: CheckHandler, + FreeBenchmarkConst.FIX: FixHandler, + } + + @staticmethod + def create(api_name: str): + handler = HandlerFactory.result_handlers.get(Config.handler_type) + if handler: + return handler(api_name) + else: + logger.error(f"{Config.handler_type} is not supported.") + raise Exception diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py new file mode 100644 index 000000000..2764d3d49 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/add_noise.py @@ -0,0 +1,67 @@ +from typing import Any + +from mindspore import Tensor, ops + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.mindspore.common.const import FreeBenchmarkConst + + +class AddNoisePerturbation(BasePerturbation): + + def handle(self, params: HandlerParams) -> Any: + """ + 返回增加扰动后的api输出 + + """ + params.fuzzed_value = self.add_noise(params.args[params.index]) + if not self.is_fuzzed: + logger.warning(f"{self.api_name} can not add noise.") + return False + return self.get_fuzzed_result(params) + + def add_noise(self, inputs) -> Any: + """ + 返回增加扰动后的api输入 + + """ + if isinstance(inputs, Tensor): + noise = self._get_noise(inputs) + if noise is not False: + result = ops.where(ops.abs(inputs) > self.perturbation_value ** 0.5, + ops.add(noise, inputs), inputs) + result = result.type(dtype=inputs.dtype) + self.is_fuzzed = True + return result + + if isinstance(inputs, dict): + return {k: self.add_noise(v) for k, v in inputs.items()} + + if isinstance(inputs, (list, tuple)): + return [self.add_noise(v) for v in inputs] + + return inputs + + def _get_noise(self, input): + """ + 得到要添加的噪声值 + + """ + if self.is_fuzzed: + return False + if not ops.is_floating_point(input) or ops.numel(input) == 0: + return False + + pert_value = FreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype) + if not pert_value: + return False + else: + self.perturbation_value = pert_value + + max_val = ops.max(ops.abs(input))[0].item() + if max_val < pert_value: + return False + + noise = ops.full(input.shape, self.perturbation_value, dtype=input.dtype) + return noise diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py new file mode 100644 index 000000000..becfe2964 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/base_perturbation.py @@ -0,0 +1,21 @@ +from typing import Any + +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class BasePerturbation: + + def __init__(self, api_name: str): + self.api_name = api_name + self.is_fuzzed = False + self.perturbation_value = None + + @staticmethod + def get_fuzzed_result(params: HandlerParams): + args_front = params.args[:params.index] + args_rear = params.args[params.index + 1:] + fuzzed_result = params.original_func(*args_front, params.fuzzed_value, *args_rear, **params.kwargs) + return fuzzed_result + + def handler(self, params: HandlerParams) -> Any: + pass diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py new file mode 100644 index 000000000..65202e0f6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/bit_noise.py @@ -0,0 +1,63 @@ +from typing import Any + +import numpy as np +from mindspore import Tensor, ops + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation + + +class BitNoisePerturbation(BasePerturbation): + + def add_bit_noise(self, inputs) -> Any: + if isinstance(inputs, Tensor): + bit_len_type = self._get_bit_len_type(inputs) + if bit_len_type is not False: + sub_normal_np = np.finfo(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)).smallest_normal + sub_normal = Tensor(sub_normal_np) + noise_type = list(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.keys())[ + list(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.values()).index(bit_len_type)] + noise = ops.full(inputs.shape, 1, dtype=noise_type) + input_np = inputs.asnumpy() + input_np_int = input_np.view(bit_len_type) + result = Tensor(input_np_int) + result = ops.where(ops.abs(inputs) > sub_normal, + ops.bitwise_xor(result, noise), result) + result_np = result.asnumpy() + result_np_float = result_np.view(FreeBenchmarkConst.MS_NUMPY_DTYPE_DICT.get(inputs.dtype)) + self.is_fuzzed = True + return Tensor(result_np_float) + + if isinstance(inputs, dict): + return {k: self.add_bit_noise(v) for k, v in inputs.items()} + if isinstance(inputs, (tuple, list)): + return type(inputs)([self.add_bit_noise(v) for v in inputs]) + return inputs + + def handle(self, params: HandlerParams) -> any: + args = params.args + params.fuzzed_value = self.add_bit_noise(params.args[params.index]) + if not self.is_fuzzed: + logger.warning(f"{self.api_name} can not add bit noise.") + return False + params.args = args + return self.get_fuzzed_result(params) + + def _get_bit_len_type(self, input): + if self.is_fuzzed: + return False + if not isinstance(input, Tensor) or not ops.is_floating_point(input) or \ + input.numel() == 0: + return False + bit_len_type = FreeBenchmarkConst.PERT_BIT_DICT.get(input.dtype) + if not bit_len_type: + return False + pert_value = FreeBenchmarkConst.PERT_VALUE_DICT.get(input.dtype) + if not pert_value: + return False + max_val = ops.max(ops.abs(input))[0].item() + if max_val < pert_value: + return False + return bit_len_type diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py new file mode 100644 index 000000000..f55a96aca --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py @@ -0,0 +1,34 @@ +from typing import Any + +import mindspore as ms +from mindspore import Tensor, ops + +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.mindspore.common.log import logger + + +class ImprovePrecisionPerturbation(BasePerturbation): + + def improve_tensor_precision(self, target_tensor): + if isinstance(target_tensor, Tensor) and ops.is_floating_point(target_tensor) and \ + target_tensor.dtype not in [ms.float64, ms.float32]: + self.is_fuzzed = True + return target_tensor.to(ms.float32) + if isinstance(target_tensor, dict): + return {k: self.improve_tensor_precision(v) for k, v in target_tensor.items()} + if isinstance(target_tensor, (tuple, list)): + return type(target_tensor)([self.improve_tensor_precision(v) for v in target_tensor]) + return target_tensor + + def handle(self, params: HandlerParams) -> Any: + args = self.improve_tensor_precision(params.args) + kwargs = self.improve_tensor_precision(params.kwargs) + fuzzed_value = args + if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST: + params.fuzzed_value = fuzzed_value + if not self.is_fuzzed: + logger.warning(f"{self.api_name} can not improve precision.") + return False + return params.original_func(*args, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/no_change.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/no_change.py new file mode 100644 index 000000000..fc844bfd6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/no_change.py @@ -0,0 +1,12 @@ +from typing import Any + +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class NoChangePerturbation(BasePerturbation): + + def handle(self, params: HandlerParams) -> Any: + params.fuzzed_value = params.args[params.index] + self.is_fuzzed = True + return self.get_fuzzed_result(params) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py new file mode 100644 index 000000000..6c8328dc2 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py @@ -0,0 +1,27 @@ +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.mindspore.free_benchmark.common.config import Config +from .add_noise import AddNoisePerturbation +from .bit_noise import BitNoisePerturbation +from .no_change import NoChangePerturbation +from .improve_precision import ImprovePrecisionPerturbation + + +class PerturbationFactory: + """ + 扰动工厂类 + + """ + perturbations = { + FreeBenchmarkConst.IMPROVE_PRECISION: ImprovePrecisionPerturbation, + FreeBenchmarkConst.ADD_NOISE: AddNoisePerturbation, + FreeBenchmarkConst.BIT_NOISE: BitNoisePerturbation, + FreeBenchmarkConst.NO_CHANGE: NoChangePerturbation, + } + + @staticmethod + def create(api_name: str): + perturbation = PerturbationFactory.perturbations.get(Config.pert_type) + if perturbation: + return perturbation(api_name) + else: + raise Exception(f'{Config.pert_type} is a invalid perturbation type') diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py new file mode 100644 index 000000000..e485887ce --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/self_check_tool_factory.py @@ -0,0 +1,33 @@ +from msprobe.mindspore.common.const import Const +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.mindspore.free_benchmark.api_pynative_self_check import ApiPyNativeSelFCheck + + +class SelfCheckToolFactory: + tools = { + Const.CELL: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: None + }, + Const.API: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: ApiPyNativeSelFCheck + }, + Const.KERNEL: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: None + } + } + + @staticmethod + def create(config: DebuggerConfig): + tool = SelfCheckToolFactory.tools.get(config.level) + if not tool: + raise Exception(f"{config.level} is not supported.") + tool = tool.get(config.execution_mode) + if not tool: + raise Exception(f"Task free_benchmark is not supported in this mode: {config.execution_mode}.") + return tool(config) diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/__init__.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py new file mode 100644 index 000000000..16d0bd0b8 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -0,0 +1,91 @@ +import os +import threading +from typing import Dict, Union + +from msprobe.core.grad_probe.utils import check_str +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.log import logger +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.utils import check_path_before_create + + +class GlobalContext: + + _instance = None + _instance_lock = threading.Lock() + _setting = { + GradConst.LEVEL: None, + GradConst.PARAM_LIST: None, + GradConst.STEP: None, + GradConst.RANK: None, + GradConst.CURRENT_STEP: 0, + GradConst.BOUNDS: [-10, -1, -0.1, -0.01, -0.001, 0, 0.001, 0.01, 0.1, 1, 10], + GradConst.OUTPUT_PATH: None + } + + def __new__(cls, *args, **kwargs): + if cls._instance is None: + cls._instance_lock.acquire() + cls._instance = object.__new__(cls) + cls._instance_lock.release() + return cls._instance + + def init_context(self, config_dict: Dict): + level = config_dict.get(GradConst.LEVEL) + check_str(level, variable_name = "level in yaml") + if level in GradConst.SUPPORTED_LEVEL: + self._setting[GradConst.LEVEL] = config_dict.get(GradConst.LEVEL) + else: + raise ValueError("Invalid level set in config yaml file, level option: L0, L1, L2") + + self._set_input_list(config_dict, GradConst.PARAM_LIST, str) + self._set_input_list(config_dict, GradConst.BOUNDS, float) + self._set_input_list(config_dict, GradConst.STEP, int) + self._set_input_list(config_dict, GradConst.RANK, int) + + output_path = config_dict.get(GradConst.OUTPUT_PATH) + check_str(output_path, variable_name = "output_path in yaml") + try: + check_path_before_create(output_path) + except RuntimeError as err: + raise ValueError(f"Invalid output_path: {output_path}. The error message is {err}.") from err + self._setting[GradConst.OUTPUT_PATH] = output_path + if not os.path.isdir(self._setting.get(GradConst.OUTPUT_PATH)): + create_directory(self._setting.get(GradConst.OUTPUT_PATH)) + else: + logger.warning("The output_path exists, the data will be covered.") + + def get_context(self, key: str): + if key not in self._setting: + logger.warning(f"Unrecognized {key}.") + return self._setting.get(key) + + def update_step(self): + self._setting[GradConst.CURRENT_STEP] += 1 + + def step_need_dump(self, step): + dump_step_list = self.get_context(GradConst.STEP) + return (not dump_step_list) or (step in dump_step_list) + + def rank_need_dump(self, rank): + dump_rank_list = self.get_context(GradConst.RANK) + return (not dump_rank_list) or (rank in dump_rank_list) + + def _set_input_list(self, config_dict: Dict, name: str, dtype: Union[int, str, float]): + value = config_dict.get(name) + if dtype == int: + type_str = "integer" + elif dtype == float: + type_str = "float" + else: + type_str = "string" + if value and isinstance(value, list): + for val in value: + if not isinstance(val, dtype): + logger.warning(f"Invalid {name} which must be None or list of {type_str}") + return + self._setting[name] = value + else: + logger.warning(f"{name} is None or not a list with valid items, use default value.") + +grad_context = GlobalContext() diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py new file mode 100644 index 000000000..2bdc11114 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -0,0 +1,231 @@ +import os +import time +from typing import List, Tuple +import multiprocessing +from multiprocessing import Process + +import numpy as np +import mindspore as ms +from mindspore.communication import get_rank +from mindspore.ops import operations as P +from mindspore.common.parameter import Parameter + +from msprobe.core.grad_probe.utils import ListCache +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.log import logger +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file +from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext + + +def get_rank_id(): + try: + rank_id = get_rank() + except Exception as err: + rank_id = 0 + return rank_id + + +@ms.jit +def grad_dump(dump_dir: str, g_name: str, dump_step: Parameter, grad: ms.Tensor, level: str, bounds: List): + ''' + Dump gradient statistic data. + level0: [step, max, min, norm, shape_dim, shape] + level1: [step, max, min, norm, shape_dim, shape] + grad_bool_data + level2: [step, max, min, norm, shape_dim, shape, dist_dim, dist] + grad_bool_data + ''' + dump_path = os.path.join(dump_dir, g_name) + dump_dir_path = dump_path + "_dir" + save_op = ms.ops.TensorDump() + + grad_flat = grad.reshape(-1) + max_val = grad_flat.max(axis=0).float() + min_val = grad_flat.min(axis=0).float() + norm_val = grad_flat.norm(ord=2).float() + shape = grad.shape + extrem_list = [dump_step[0].float(), max_val, min_val, norm_val] + extrem_stat = ms.ops.stack(extrem_list) + shape_list = [len(shape)] + list(shape) + shape_stat = ms.Tensor(shape_list).float() + level0_stat = ms.ops.concat((extrem_stat, shape_stat), axis=0) + level_stat = level0_stat + + if level == GradConst.LEVEL2: + zero_grad = (grad == 0).sum() + dist_dim = ms.Tensor([len(bounds) + 2]).float() + bucket_result = ms.ops.bucketize(grad.float(), bounds) + bucket_result = bucket_result.astype(ms.int8) + dist_stat = [(bucket_result == i).sum() for i in range(len(bounds) + 1)] + dist_stat.append(zero_grad) + dist_stat.append(ms.Tensor(1, dtype=ms.int64)) # make sure dist_stat is not empty + dist_stat = ms.ops.stack(dist_stat, axis=0).float() + level2_stat = ms.ops.concat((level0_stat, dist_dim, dist_stat), axis=0) + level_stat = level2_stat + + save_op(dump_path, level_stat) + if level == GradConst.LEVEL1 or level == GradConst.LEVEL2: + grad_direction = grad > 0 + save_op(dump_dir_path, grad_direction) + + +class CSVGenerator(Process): + + def __init__(self) -> None: + super().__init__() + self.dump_dir = None + self.save_dir = None + self.level = GradConst.LEVEL0 + self.cache_list = ListCache() + self.current_step = None + self.stop_event = None + self.last_finish = False + self.bounds = [-0.1, 0.0, 0.1], + + def init(self, context: GlobalContext): + rank_id = get_rank_id() + output_path = context.get_context(GradConst.OUTPUT_PATH) + self.level = context.get_context(GradConst.LEVEL) + self.bounds = context.get_context(GradConst.BOUNDS) + self.dump_dir = f"{output_path}/rank{rank_id}/Dump/" + self.save_dir = f"{output_path}/rank{rank_id}/" + self.current_step = None + self.stop_event = multiprocessing.Event() + self.last_finish = False + + def run(self): + while True: + if not os.path.exists(self.dump_dir): + time.sleep(0.1) + if self.stop_event.is_set(): + break + continue + npy_files = os.listdir(self.dump_dir) + npy_files.sort(key=lambda x: int(x.split("_")[0])) + self.traverse_files(npy_files) + empty = len(os.listdir(self.dump_dir)) == 0 + if self.stop_event.is_set() and empty and self.last_finish: + break + if os.path.exists(self.dump_dir): + remove_path(self.dump_dir) + + def stop(self): + self.stop_event.set() + + def traverse_files(self, npy_files: List): + for npy_file in npy_files: + file_path = os.path.join(self.dump_dir, npy_file) + while not os.path.exists(file_path): + time.sleep(0.01) + check_file_or_directory_path(file_path) + if GradConst.STEP_FINISH in npy_file: + self.cache_list.flush() + remove_path(file_path) + self.last_finish = True + elif file_path.split("_")[-1] == GradConst.DIR_SUFFIX: + prefix_idx = len(npy_file.split("_")[0]) + new_name = npy_file[prefix_idx + 1:].replace("_" + GradConst.DIR_SUFFIX, "." + GradConst.NPY_SUFFIX) + if not new_name: + raise RuntimeError("Invalid dump data name.") + if self.current_step is None: + raise RuntimeError("Current record step is None.") + step_dir = os.path.join(self.save_dir, f"step{self.current_step}") + if not os.path.exists(step_dir): + create_directory(step_dir) + dst_file = os.path.join(step_dir, new_name) + move_file(file_path, dst_file) + self.last_finish = False + elif file_path.split(".")[-1] == GradConst.NPY_SUFFIX: + stat_data = self.load_npy_data(file_path) + if stat_data is None: + continue + if not self.check_valid(stat_data): + os.remove(file_path) + continue + step = int(stat_data[GradConst.STEP_IDX]) + update_step = self.current_step is None or step != self.current_step + self.current_step = step + if update_step: + self.create_csv_file() + self.gen_csv_line(file_path, stat_data) + os.remove(file_path) + self.last_finish = False + + def check_valid(self, stat_data): + level = grad_context.get_context(GradConst.LEVEL) + try: + shape_dim = int(stat_data[GradConst.SHAPE_DIM_IDX]) + if level == GradConst.LEVEL2: + dist_dim = int(stat_data[shape_dim + GradConst.SHAPE_DIM_IDX + 1]) + length = shape_dim + dist_dim + 7 + else: + length = shape_dim + 5 + except IndexError as err: + return False + if length != len(stat_data): + return False + return True + + def load_npy_data(self, file_path: str): + stat_data = None + max_try = 10 + while max_try: + try: + stat_data = np.load(file_path) + return stat_data + except Exception as err: + logger.warning(f"load numpy file failed, retry...") + max_try -= 1 + time.sleep(0.1) + return stat_data + + def gen_csv_line(self, file_path: str, stat_data) -> None: + shape_dim = int(stat_data[GradConst.SHAPE_DIM_IDX]) + file_name = os.path.basename(file_path) + prefix_idx = len(file_name.split("_")[0]) + param_name = file_name[(prefix_idx + 1) : -(len(GradConst.NPY_SUFFIX) + 1)] + if not param_name: + raise RuntimeError("Invalid gradient statistic file name.") + csv_line = [param_name] + if self.level == GradConst.LEVEL2: + csv_line.extend(self.get_dist_data(shape_dim, stat_data)) + csv_line.extend(self.get_extrem_data(shape_dim, stat_data)) + self.cache_list.append(csv_line) + + def get_dist_data(self, shape_dim: int, stat_data: np.ndarray): + dist_data = stat_data[(shape_dim + GradConst.SHAPE_DIM_IDX + 2):-1] + element_num = dist_data.sum() - dist_data[-1] + if element_num != 0: + dist_data = dist_data / element_num + return list(dist_data) + + def get_extrem_data(self, shape_dim: int, stat_data: np.ndarray): + extrem_data = list(stat_data[(GradConst.STEP_IDX + 1):(GradConst.STEP_IDX + 4)]) + shape_data = stat_data[(GradConst.SHAPE_DIM_IDX + 1):(GradConst.SHAPE_DIM_IDX + shape_dim + 1)] + shape_data = list(shape_data.astype(int)) + extrem_data.append(shape_data) + return extrem_data + + def create_csv_file(self): + headers = ["Param_name"] + if self.level == GradConst.LEVEL2: + headers.extend(self.get_dist_header()) + headers.extend(self.get_extrem_headers()) + output_path = f"{self.save_dir}/grad_summary_{self.current_step}.csv" + write_csv([headers], output_path) + self.cache_list.set_output_file(output_path) + self.cache_list.clear() + + def get_extrem_headers(self) -> List[str]: + return ["Max", "Min", "Norm", "Shape"] + + def get_dist_header(self) -> List[str]: + intervals = [] + for i, _ in enumerate(self.bounds): + if i == 0: + intervals.append(f"(-inf, {self.bounds[i]}]") + else: + intervals.append(f"({self.bounds[i-1]}, {self.bounds[i]}]") + intervals.extend([f"({self.bounds[-1]}, inf)", "=0"]) + return intervals + +csv_generator = CSVGenerator() diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_monitor.py new file mode 100644 index 000000000..f1e082688 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_monitor.py @@ -0,0 +1,27 @@ +from msprobe.mindspore.grad_probe.global_context import grad_context +from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator +from msprobe.mindspore.grad_probe.hook import hook_optimizer +from msprobe.core.grad_probe.constant import GradConst + + +class GradientMonitor: + + def __init__(self, common_dict, task_config): + config = {} + config[GradConst.OUTPUT_PATH] = common_dict.dump_path + config[GradConst.STEP] = common_dict.step + config[GradConst.RANK] = common_dict.rank + config[GradConst.PARAM_LIST] = task_config.param_list + config[GradConst.LEVEL] = task_config.grad_level + config[GradConst.BOUNDS] = task_config.bounds + self.config = config + grad_context.init_context(self.config) + + @staticmethod + def monitor(opt): + csv_generator.init(grad_context) + hook_optimizer(opt) + + @staticmethod + def stop(): + csv_generator.stop() diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_stat_csv.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_stat_csv.py new file mode 100644 index 000000000..1c2b0ee3b --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_stat_csv.py @@ -0,0 +1,132 @@ +from abc import ABC, abstractmethod +import hashlib + +import mindspore +from mindspore import ops, Tensor +from msprobe.core.grad_probe.constant import GradConst + + +class CsvInput: + def __init__(self, param_name, grad, bounds): + self.param_name = param_name + self.grad = grad + self.bounds = bounds + +class GradStatCsv: + csv = {} + + @staticmethod + def get_csv_header(level, csv_input): + header = ["param_name"] + for key in level["header"]: + header.extend(GradStatCsv.csv[key].generate_csv_header(csv_input)) + return header + + @staticmethod + def get_csv_line(level, csv_input): + line = [csv_input.param_name] + for key in level["header"]: + line.extend(GradStatCsv.csv[key].generate_csv_content(csv_input)) + return line + + +def register_csv_item(key, cls=None): + if cls is None: + # 无参数时，返回装饰器函数 + return lambda cls: register_csv_item(key, cls) + GradStatCsv.csv[key] = cls + return cls + + +class CsvItem(ABC): + @staticmethod + @abstractmethod + def generate_csv_header(csv_input): + pass + + @staticmethod + @abstractmethod + def generate_csv_content(csv_input): + pass + + +@register_csv_item(GradConst.MD5) +class CsvMd5(CsvItem): + def generate_csv_header(csv_input): + return ["MD5"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + tensor_bytes = grad.float().numpy().tobytes() + md5_hash = hashlib.md5(tensor_bytes) + return [md5_hash.hexdigest()] + + +@register_csv_item(GradConst.DISTRIBUTION) +class CsvDistribution(CsvItem): + def generate_csv_header(csv_input): + bounds = csv_input.bounds + intervals = [] + if bounds: + intervals.append(f"(-inf, {bounds[0]}]") + for i in range(1, len(bounds)): + intervals.append(f"({bounds[i-1]}, {bounds[i]}]") + if intervals: + intervals.append(f"({bounds[-1]}, inf)") + intervals.append("=0") + + return intervals + + def generate_csv_content(csv_input): + grad = csv_input.grad + bounds = csv_input.bounds + if grad.dtype == mindspore.bfloat16: + grad = grad.to(mindspore.float32) + element_num = grad.numel() + grad_equal_0_num = (grad == 0).sum().item() + bucketsize_result = ops.bucketize(grad.float(), bounds) + bucketsize_result = bucketsize_result.astype(mindspore.int8) + interval_nums = [(bucketsize_result == i).sum().item() for i in range(len(bounds) + 1)] + interval_nums.append(grad_equal_0_num) + return_list = [x / element_num if element_num != 0 else 0 for x in interval_nums] + return return_list + + +@register_csv_item(GradConst.MAX) +class CsvMax(CsvItem): + def generate_csv_header(csv_input): + return ["max"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.amax(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.MIN) +class CsvMin(CsvItem): + def generate_csv_header(csv_input): + return ["min"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.amin(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.NORM) +class CsvNorm(CsvItem): + def generate_csv_header(csv_input): + return ["norm"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [ops.norm(grad).float().numpy().tolist()] + + +@register_csv_item(GradConst.SHAPE) +class CsvShape(CsvItem): + def generate_csv_header(csv_input): + return ["shape"] + + def generate_csv_content(csv_input): + grad = csv_input.grad + return [list(grad.shape)] \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py new file mode 100644 index 000000000..243fb33de --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py @@ -0,0 +1,92 @@ + +import os + +import mindspore +import mindspore as ms +from mindspore.common.api import jit +from mindspore.nn.optim.optimizer import Optimizer +from mindspore.common.parameter import Parameter +from mindspore.common.initializer import initializer + +from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.common.log import logger + +from msprobe.core.common.utils import write_csv, remove_path +from msprobe.mindspore.grad_probe.global_context import grad_context +from msprobe.mindspore.grad_probe.grad_analyzer import grad_dump, get_rank_id +from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator +from msprobe.mindspore.grad_probe.grad_stat_csv import GradStatCsv, CsvInput +from msprobe.mindspore.grad_probe.utils import save_grad_direction, get_adapted_level + +class HookInput: + + ''' + HookInput is a class wrapping all the variables used for hooking optimizer + ''' + + def __init__(self, opt) -> None: + self.func = opt.construct + self.g_names = [param.name for param in opt._parameters] + self.param_list = grad_context.get_context(GradConst.PARAM_LIST) + self.rank_id = get_rank_id() + output_path = grad_context.get_context(GradConst.OUTPUT_PATH) + self.dump_dir = os.path.join(output_path, f"rank{self.rank_id}", "Dump") + self.save_dir = os.path.join(output_path, f"rank{self.rank_id}") + self.step_finish_flag = os.path.join(self.dump_dir, GradConst.STEP_FINISH) + if os.path.exists(self.save_dir): + logger.warning(f"Delete existing path {self.save_dir}.") + remove_path(self.save_dir) + self.level = grad_context.get_context(GradConst.LEVEL) + self.bounds = grad_context.get_context(GradConst.BOUNDS) + self.mode = mindspore.get_context("mode") + +def hook_graph_mode_optimizer(opt, hook_input): + @jit + def new_construct(self, gradients): + for index, grad_value in enumerate(gradients): + if hook_input.param_list and hook_input.g_names[index] not in hook_input.param_list: + continue + grad_dump(hook_input.dump_dir, hook_input.g_names[index], self.dump_step, + grad_value, hook_input.level, hook_input.bounds) + ms.ops.TensorDump()(hook_input.step_finish_flag, self.dump_step) + self.assignadd(self.dump_step, self.global_step_increase_tensor) + out = hook_input.func(gradients) + return out + + opt.dump_step = Parameter(initializer(0, [1], ms.int32), name="dump_step") + opt.construct = new_construct.__get__(opt, type(opt)) + csv_generator.start() + +def hook_pynative_optimizer(opt, hook_input): + level_adapted = get_adapted_level(hook_input.level) + + def hook_fn(cell, input): + gradients, = input + cur_step = grad_context.get_context(GradConst.CURRENT_STEP) + if grad_context.step_need_dump(cur_step) and grad_context.rank_need_dump(hook_input.rank_id): + output_lines = [] + for index, grad_value in enumerate(gradients): + param_name = hook_input.g_names[index] + if hook_input.param_list and param_name not in hook_input.param_list: + continue + csv_input = CsvInput(param_name, grad_value, hook_input.bounds) + grad_info = GradStatCsv.get_csv_line(level_adapted, csv_input) + output_lines.append(grad_info) + if level_adapted["have_grad_direction"]: + save_grad_direction(param_name, grad_value, os.path.join(hook_input.save_dir, f'step{cur_step}')) + output_csv_path = os.path.join(hook_input.save_dir, f"grad_summary_{cur_step}.csv") + dummy_csv_input = CsvInput(None, None, hook_input.bounds) + output_lines.insert(0, GradStatCsv.get_csv_header(level_adapted, dummy_csv_input)) + write_csv(output_lines, output_csv_path) + grad_context.update_step() + + opt.register_forward_pre_hook(hook_fn) + + +def hook_optimizer(opt: Optimizer): + hook_input = HookInput(opt) + + if hook_input.mode == mindspore.GRAPH_MODE: + hook_graph_mode_optimizer(opt, hook_input) + else: + hook_pynative_optimizer(opt, hook_input) diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py new file mode 100644 index 000000000..db0a36a02 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py @@ -0,0 +1,29 @@ +import os + +import numpy as np +import mindspore +from msprobe.core.grad_probe.constant import GradConst, level_adp +from msprobe.core.grad_probe.utils import check_param +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.utils import check_path_before_create, change_mode, check_file_or_directory_path, save_npy + + +def save_grad_direction(param_name, grad, save_path): + if not os.path.exists(save_path): + create_directory(save_path) + check_file_or_directory_path(save_path, isdir=True) + check_param(param_name) + save_filepath = os.path.join(save_path, f"{param_name}.npy") + check_path_before_create(save_filepath) + + if grad.dtype == mindspore.bfloat16: + grad = grad.to(mindspore.float32) + grad_direction_tensor = grad > 0 + grad_direction_ndarray = grad_direction_tensor.numpy() + + save_npy(grad_direction_ndarray, save_filepath) + + +def get_adapted_level(level: str): + level_adapted = level_adp.get(level) + return level_adapted \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index c0ef6bb6c..0e7ce1529 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -1,7 +1,10 @@ import json + from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.file_check import FileOpen from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.mindspore.common.log import logger class TensorConfig(BaseConfig): @@ -51,10 +54,39 @@ class OverflowCheckConfig(BaseConfig): raise Exception("check_mode is invalid") +class FreeBenchmarkConfig(BaseConfig): + def __init__(self, task_config): + super().__init__(task_config) + self._check_config() + + def _check_config(self): + if self.fuzz_device and self.fuzz_device not in FreeBenchmarkConst.DEVICE_LIST: + raise Exception("fuzz_device must be npu or empty") + if self.pert_mode and self.pert_mode not in FreeBenchmarkConst.PERT_TYPE_LIST: + raise Exception("pert_mode must be improve_precision, add_noise, bit_noise , no_change or empty") + if self.handler_type and self.handler_type not in FreeBenchmarkConst.HANDLER_TYPE_LIST: + raise Exception("handler_type must be check, fix or empty") + if self.fuzz_level and self.fuzz_level not in FreeBenchmarkConst.DUMP_LEVEL_LIST: + raise Exception("fuzz_level must be L1 or empty") + if self.fuzz_stage and self.fuzz_stage not in FreeBenchmarkConst.STAGE_LIST: + raise Exception("fuzz_stage must be forward or empty") + if self.if_preheat or self.preheat_step or self.max_sample: + logger.warning("'if_preheat', 'preheat_step' and 'max_sample' settings " + "are not supported for mindspore free benchmark task.") +class GradProbeConfig(BaseConfig): + def __init__(self, json_config): + super().__init__(json_config) + self.grad_level = json_config.get("grad_level") + self.param_list = json_config.get("param_list") + self.bounds = json_config.get("bounds") + + TaskDict = { Const.TENSOR: TensorConfig, Const.STATISTICS: StatisticsConfig, Const.OVERFLOW_CHECK: OverflowCheckConfig, + Const.FREE_BENCHMARK: FreeBenchmarkConfig, + Const.GRAD_PROBE: GradProbeConfig, } diff --git a/debug/accuracy_tools/msprobe/mindspore/runtime.py b/debug/accuracy_tools/msprobe/mindspore/runtime.py new file mode 100644 index 000000000..380b30d97 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/runtime.py @@ -0,0 +1,4 @@ +class Runtime: + step_count: int = 0 + rank_id: int = -1 + is_running: bool = False diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 50776aaf1..4c2a4ef69 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -19,6 +19,9 @@ from pathlib import Path import functools from collections import defaultdict +from mindspore.common.tensor import Tensor +from mindspore import ops +from mindspore import nn from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope from msprobe.mindspore.common.utils import get_rank_if_initialized @@ -27,7 +30,9 @@ from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import Const from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.mindspore.dump.hook_cell.api_registry import api_register -from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs +from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs,\ + ModuleBackwardInputs, ModuleBackwardOutputs +from msprobe.core.common.exceptions import MsprobeException from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell @@ -41,9 +46,18 @@ class Service: self.current_iter = 0 self.first_start = True self.current_rank = None + self.primitive_counters = {} self.dump_iter_dir = None self.start_call = False + @staticmethod + def check_model_valid(model): + if not model or isinstance(model, nn.Cell): + return model + raise MsprobeException( + MsprobeException.INVALID_PARAM_ERROR, "model 参数必须是 mindspore.nn.Cell 类型。" + ) + def build_hook(self, module_type, name): def forward_hook(api_or_module_name, module, input, output): self.data_collector.visit_and_clear_overflow_status(api_or_module_name) @@ -79,13 +93,145 @@ class Service: return wrap_forward_hook, wrap_backward_hook + + def wrap_primitive(self, origin_func, primitive_name): + service_instance = self + + def create_backward_hook(captured_grads, num_tensors, updated_primitive_name, hook_type): + def backward_hook(grad): + captured_grads.append(grad) + backward_primitive_name = f"{updated_primitive_name}.{Const.BACKWARD}" + try: + if len(captured_grads) == num_tensors and hook_type == Const.INPUT: + service_instance.data_collector.visit_and_clear_overflow_status(backward_primitive_name) + new_module_input_output = ModuleBackwardOutputs(grad_output=tuple(captured_grads)) + service_instance.data_collector.backward_output_data_collect( + backward_primitive_name, service_instance, os.getpid(), new_module_input_output + ) + captured_grads.clear() + elif len(captured_grads) == num_tensors and hook_type == Const.OUTPUT: + service_instance.data_collector.visit_and_clear_overflow_status(backward_primitive_name) + new_module_input_output = ModuleBackwardInputs(grad_input=tuple(captured_grads)) + service_instance.data_collector.backward_input_data_collect( + backward_primitive_name, service_instance, os.getpid(), new_module_input_output + ) + captured_grads.clear() + + except Exception as exception: + raise Exception( + "This is a primitive op {hook_type}_backward dump error: {exception}," + " updated_primitive_name: {updated_primitive_name}".format( + hook_type=hook_type, exception=exception, backward_primitive_name=backward_primitive_name + ) + ) from exception + + return backward_hook + + def hook_primitive_inputs(args, captured_grads_input, updated_primitive_name): + hooked_inputs = [] + num_tensors = sum(isinstance(arg, Tensor) for arg in args) + input_backward_hook = create_backward_hook(captured_grads_input, num_tensors, updated_primitive_name, + Const.INPUT) + for _, arg in enumerate(args): + if isinstance(arg, Tensor): + arg_hooked = ops.HookBackward(input_backward_hook)(arg) + hooked_inputs.append(arg_hooked) + else: + hooked_inputs.append(arg) + return hooked_inputs + + def hook_primitive_outputs(out, captured_grads_output, updated_primitive_name): + if isinstance(out, tuple): + num_output_tensors = sum(isinstance(tensor, Tensor) for tensor in out) + else: + num_output_tensors = 1 + output_backward_hook = create_backward_hook(captured_grads_output, num_output_tensors, + updated_primitive_name, Const.OUTPUT) + + if isinstance(out, Tensor): + return ops.HookBackward(output_backward_hook)(out) + elif isinstance(out, tuple): + hooked_outputs = [] + for tensor in out: + if isinstance(tensor, Tensor): + hooked_outputs.append(ops.HookBackward(output_backward_hook)(tensor)) + else: + hooked_outputs.append(tensor) + return tuple(hooked_outputs) + return out + + def wrapped_primitive_call(instance_self, *args, **kwargs): + service_instance.update_primitive_counters(primitive_name) + current_count = service_instance.primitive_counters.get(primitive_name, 0) + updated_primitive_name = f"{Const.PRIMITIVE_PREFIX}.{primitive_name}.{current_count}" + + if not service_instance.switch: + return origin_func(*args, **kwargs) + + captured_grads_input, captured_grads_output = [], [] + + try: + hooked_inputs = hook_primitive_inputs(args, captured_grads_input, updated_primitive_name) + except Exception as exception: + raise Exception("This is a primitive op dump error during input hooking: {}," + " primitive_name: {}".format(exception, primitive_name)) from exception + + try: + out = origin_func(*hooked_inputs, **kwargs) + except Exception as exception: + raise Exception("This is a primitive op dump error during function call: {}," + " primitive_name: {}".format(exception, primitive_name)) from exception + + forward_primitive_name = f"{updated_primitive_name}.{Const.FORWARD}" + service_instance.data_collector.visit_and_clear_overflow_status(forward_primitive_name) + if service_instance.data_collector: + module_input_output = ModuleForwardInputsOutputs(args=hooked_inputs, kwargs=kwargs, output=out) + try: + service_instance.data_collector.forward_data_collect(forward_primitive_name, instance_self, + os.getpid(), module_input_output) + except Exception as exception: + raise Exception("This is a primitive op dump error during forward data collection: {}," + " primitive_name: {}".format(exception, primitive_name)) from exception + + if service_instance.data_collector.if_return_forward_new_output(): + out = service_instance.data_collector.get_forward_new_output() + + try: + out = hook_primitive_outputs(out, captured_grads_output, updated_primitive_name) + except Exception as exception: + raise Exception("This is a primitive op dump error during output hooking: {}," + " primitive_name: {}".format(exception, primitive_name)) from exception + + return out + + return wrapped_primitive_call + + def update_primitive_counters(self, primitive_name): + if primitive_name not in self.primitive_counters: + self.primitive_counters[primitive_name] = 0 + else: + self.primitive_counters[primitive_name] += 1 + + def register_hooks(self): + primitive_set = set() + for _, cell in self.model.cells_and_names(): + for pname, primitive in cell._primitives.items(): + primitive_set.add((pname, primitive)) + + for pname, primitive in primitive_set: + NewPrimitive = type('NewPrimitive', (primitive.__class__,), + {'__call__': self.wrap_primitive(primitive.__call__, pname)}) + primitive.__class__ = NewPrimitive + + def step(self): self.current_iter += 1 self.data_collector.update_iter(self.current_iter) HOOKCell.cell_count = defaultdict(int) + self.primitive_counters.clear() def start(self, model=None): - self.model = model + self.model = Service.check_model_valid(model) self.start_call = True logger.info("msprobe: debugger.start() is set successfully") if self.config.step and self.current_iter > max(self.config.step): @@ -150,3 +296,5 @@ class Service: if self.config.level == "L1": api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) api_register.api_set_hook_func() + if self.model: + self.register_hooks() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py index 7b7e6fd88..dfe2fbe2c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py @@ -1,17 +1,23 @@ +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.dump_tool_factory import DumpToolFactory from msprobe.mindspore.overflow_check.overflow_check_tool_factory import OverflowCheckToolFactory +from msprobe.mindspore.free_benchmark.self_check_tool_factory import SelfCheckToolFactory class TaskHandlerFactory: tasks = { - "tensor": DumpToolFactory, - "statistics": DumpToolFactory, - "overflow_check": OverflowCheckToolFactory + Const.TENSOR: DumpToolFactory, + Const.STATISTICS: DumpToolFactory, + Const.OVERFLOW_CHECK: OverflowCheckToolFactory, + Const.FREE_BENCHMARK: SelfCheckToolFactory } @staticmethod def create(config: DebuggerConfig): + if config.execution_mode == MsConst.PYNATIVE_MODE and config.task != Const.FREE_BENCHMARK: + raise Exception("Current Task can't run in pynative mode.") task = TaskHandlerFactory.tasks.get(config.task) if not task: raise Exception("valid task is needed.") diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 5146ee1ac..802913814 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -15,16 +15,15 @@ import argparse import sys -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command -from msprobe.pytorch.parse_tool.cli import parse as cli_parse -from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut -from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, \ - _api_precision_compare_command -from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ - _run_overflow_check_command -from msprobe.pytorch.compare.pt_compare import _compare_parser -from msprobe.pytorch.compare.compare_cli import compare_cli -from msprobe.mindspore.compare.compare_cli import compare_cli_ms +import importlib.util +from msprobe.core.compare.utils import _compare_parser +from msprobe.core.common.log import logger + + +def is_module_available(module_name): + spec =importlib.util.find_spec(module_name) + return spec is not None + def main(): parser = argparse.ArgumentParser( @@ -33,6 +32,7 @@ def main(): "Providing one-site accuracy difference debugging toolkit for training on Ascend Devices.\n" f"For any issue, refer README.md first", ) + parser.set_defaults(print_help=parser.print_help) parser.add_argument('-f', '--framework', required=True, choices=['pytorch', 'mindspore'], help='Deep learning framework.') @@ -43,18 +43,32 @@ def main(): multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut') api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare') run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check') - _compare_parser(compare_cmd_parser) - _run_ut_parser(run_ut_cmd_parser) - _run_ut_parser(multi_run_ut_cmd_parser) multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, help='Number of splits for parallel processing. Range: 1-64') - _api_precision_compare_parser(api_precision_compare_cmd_parser) - _run_overflow_check_parser(run_overflow_check_cmd_parser) + + _compare_parser(compare_cmd_parser) + if len(sys.argv) == 1: parser.print_help() sys.exit(0) args = parser.parse_args(sys.argv[1:]) if sys.argv[2] == "pytorch": + if is_module_available("torch"): + from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command + from msprobe.pytorch.parse_tool.cli import parse as cli_parse + from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut + from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, \ + _api_precision_compare_command + from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ + _run_overflow_check_command + from msprobe.pytorch.compare.compare_cli import compare_cli + _run_ut_parser(run_ut_cmd_parser) + _run_ut_parser(multi_run_ut_cmd_parser) + _api_precision_compare_parser(api_precision_compare_cmd_parser) + _run_overflow_check_parser(run_overflow_check_cmd_parser) + else: + logger.error("Pytorch does not exit, please install pytorch library") + raise Exception() if sys.argv[3] == "run_ut": run_ut_command(args) elif sys.argv[3] == "parse": @@ -69,7 +83,13 @@ def main(): elif sys.argv[3] == "compare": compare_cli(args) else: - compare_cli_ms(args) + if is_module_available("mindspore"): + from msprobe.mindspore.compare.compare_cli import compare_cli_ms + else: + logger.error("Mindspore does not exit, please install mindspore library") + raise Exception() + if sys.argv[3] == "compare": + compare_cli_ms(args) if __name__ == "__main__": main() diff --git a/debug/accuracy_tools/msprobe/pytorch/__init__.py b/debug/accuracy_tools/msprobe/pytorch/__init__.py index c14d9701a..c4e426772 100644 --- a/debug/accuracy_tools/msprobe/pytorch/__init__.py +++ b/debug/accuracy_tools/msprobe/pytorch/__init__.py @@ -1,3 +1,4 @@ from .debugger.precision_debugger import PrecisionDebugger from .common.utils import seed_all from .compare.distributed_compare import compare_distributed +from .compare.pt_compare import compare \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor.py b/debug/accuracy_tools/msprobe/pytorch/advisor/advisor.py deleted file mode 100644 index b178664d9..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor.py +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - -import os - -from msprobe.pytorch.advisor.advisor_result import AdvisorResult -from msprobe.pytorch.advisor.advisor_const import AdvisorConst -from msprobe.pytorch.common.log import logger -from msprobe.core.common.utils import CompareException -from msprobe.core.common.file_check import FileChecker -from msprobe.core.common.const import Const, CompareConst, FileCheckConst - -class Advisor: - """ - Class for generate advisor - """ - - def __init__(self, input_data, out_path=""): - self.input_data = input_data - self.out_path = os.path.realpath(out_path) - self.file_type = None - - @staticmethod - def deterministic_advisor(message, node_name): - for api_name in AdvisorConst.NEED_DETERMINISTIC_API: - if api_name in node_name: - return AdvisorConst.DETERMINISTIC_SUGGEST - return message - - @staticmethod - def batch_norm_advisor(message, node_name): - if AdvisorConst.FUNC_BATCH_NORM in node_name and AdvisorConst.FORWARD_INPUT_1 in node_name: - message = AdvisorConst.BATCH_NORM_SUGGEST - return message - - def analyze_unmatched(self, analyze_data): - if self.file_type == Const.ALL: - accuracy_unmatched = analyze_data[ - analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_UNMATCH] - else: - accuracy_unmatched = analyze_data[(analyze_data[CompareConst.NPU_SHAPE] == CompareConst.NAN) | - (analyze_data[CompareConst.BENCH_SHAPE] == CompareConst.NAN)] - num_unmatch = len(accuracy_unmatched) - if num_unmatch != 0: - for i in range(len(accuracy_unmatched)): - item = accuracy_unmatched.iloc[i] - logger.warning("The tensor name matches but the shape or dtype does not match: {}" - .format(item[CompareConst.NPU_NAME])) - - def gen_advisor_result(self, pd_data): - first_failing_data = pd_data.iloc[0] - node_name = first_failing_data[CompareConst.NPU_NAME] - index = first_failing_data['index'] - message = self.gen_advisor_message(node_name) - logger.warning("Find %s accuracy not reached, the line is %s" % (node_name, index)) - result = AdvisorResult(node_name, index, message) - return result - - def gen_advisor_message(self, node_name): - if AdvisorConst.FORWARD in node_name: - if AdvisorConst.INPUT in node_name: - message = AdvisorConst.FORWARD_INPUT_SUGGEST - else: - message = AdvisorConst.FORWARD_OUTPUT_SUGGEST - message = self.deterministic_advisor(message, node_name) - else: - if AdvisorConst.INPUT in node_name: - message = AdvisorConst.BACKWARD_INPUT_SUGGEST - else: - message = AdvisorConst.BACKWARD_OUTPUT_SUGGEST - message = self.deterministic_advisor(message, node_name) - message = self.batch_norm_advisor(message, node_name) - return message - - def analysis(self): - self._check_path_vaild() - analyze_data = self._parse_input_data() - logger.info("Start analyzing the comparison result: %s" % self.file_type) - self.analyze_unmatched(analyze_data) - if self.file_type == Const.ALL: - failing_data = analyze_data[analyze_data[CompareConst.ACCURACY] == CompareConst.ACCURACY_CHECK_NO] - elif self.file_type == Const.MD5: - failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.DIFF] - elif self.file_type == Const.SUMMARY: - failing_data = analyze_data[analyze_data[CompareConst.RESULT] == CompareConst.WARNING] - if failing_data.empty: - logger.info("All data from api input/output accuracy reached") - result = AdvisorResult(AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERROR_API, AdvisorConst.NO_ERR_SUGGEST) - else: - result = self.gen_advisor_result(failing_data) - message_list = result.print_advisor_log() - result.gen_summary_file(self.out_path, message_list) - - def _parse_input_data(self): - data_columns = self.input_data.columns.values - if {CompareConst.ACCURACY, CompareConst.NPU_NAME}.issubset(data_columns): - self.file_type = Const.ALL - elif {CompareConst.RESULT, CompareConst.NPU_MD5}.issubset(data_columns): - self.file_type = Const.MD5 - elif {CompareConst.MAX_DIFF, CompareConst.RESULT}.issubset(data_columns): - self.file_type = Const.SUMMARY - else: - logger.error('Compare result does not meet the required conditions.') - raise CompareException(CompareException.INVALID_DATA_ERROR) - df = self.input_data.reset_index() - return df - - def _check_path_vaild(self): - out_path_checker = FileChecker(self.out_path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) - out_path_checker.common_check() diff --git a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_const.py b/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_const.py deleted file mode 100644 index 737c67591..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_const.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" - - -class AdvisorConst: - """ - Class for advisor const - """ - - # text symbol - NEW_LINE = "\n" - COLON = ": " - - # advisor summary key - SUSPECT_NODES = "Suspect Nodes" - LINE = "Line" - ADVISOR_SUGGEST = "Expert Advice" - - NO_ERROR_API = "NA" - - # advisor message - NO_ERR_SUGGEST = "All data in comparison result meets the accuracy requirements." - FORWARD_INPUT_SUGGEST = "1. Analyze the model to view the input source.\n" \ - "2. Check whether an inplace API causes the output result to overwrite the input result. That is, the fault is actually caused by a computation error.\n" \ - "3. The fault may be caused by memory corruption and further analysis is required." - FORWARD_OUTPUT_SUGGEST = "This is a forward API computation error. Check the computation implementation." - BACKWARD_INPUT_SUGGEST = "Check whether the forward computation result is affected." - BACKWARD_OUTPUT_SUGGEST = "This is a backward API computation error. Check the computation implementation." - BATCH_NORM_SUGGEST = "Torch API batch_norm input not fixed, the following suggestions may fix it:\n" \ - "1. If use torch.nn.functional.batch_norm, you can set parameter training=False.\n" \ - "2. If use torch.nn.BatchNormXXX, you can set parameter affine=False.\n" \ - "3. Use seed_all(mode=True) to enable deterministic computing." - DETERMINISTIC_SUGGEST = "This torch api may be uncertainty in the calculation, " \ - "can seed_all(mode=True) to enable deterministic computing." - - FUNC_BATCH_NORM = "Functional_batch_norm" - FORWARD_INPUT_1 = "forward_input.1" - NEED_DETERMINISTIC_API = ["conv2d", "conv3d", "matmul", "nll_loss", "layer_norm", "lstm"] - BATCH_NORM = "batch_norm" - - # name keyword - INPUT = "input" - OUTPUT = "output" - FORWARD = "forward" - BACKWARD = "backward" diff --git a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_result.py deleted file mode 100644 index 456f542e1..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/advisor/advisor_result.py +++ /dev/null @@ -1,58 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -""" -# Copyright (C) 2022-2024. Huawei Technologies Co., Ltd. All rights reserved. -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -""" -import os -import time - -from msprobe.pytorch.advisor.advisor_const import AdvisorConst -from msprobe.pytorch.common.log import logger -from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.file_check import change_mode - - -class AdvisorResult: - """ - Class for generate advisor result - """ - - def __init__(self, node, line, message): - self.suspect_node = node - self.line = line - self.advisor_message = message - - @staticmethod - def gen_summary_file(out_path, message_list): - file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) - result_file = os.path.join(out_path, file_name) - try: - with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file: - output_file.truncate(0) - message_list = [message + AdvisorConst.NEW_LINE for message in message_list] - output_file.writelines(message_list) - change_mode(result_file, FileCheckConst.DATA_FILE_AUTHORITY) - except IOError as io_error: - logger.error("Failed to save %s, the reason is %s." % (result_file, io_error)) - else: - logger.info("The advisor summary is saved in: %s" % result_file) - - def print_advisor_log(self): - logger.info("The summary of the expert advice is as follows: ") - message_list = [AdvisorConst.LINE + AdvisorConst.COLON + str(self.line), - AdvisorConst.SUSPECT_NODES + AdvisorConst.COLON + self.suspect_node, - AdvisorConst.ADVISOR_SUGGEST + AdvisorConst.COLON + self.advisor_message] - for message in message_list: - logger.info(message) - return message_list diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py index 155609f58..b344d4efb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py @@ -3,7 +3,7 @@ from msprobe.core.common.file_check import FileOpen, check_file_type from msprobe.core.common.const import FileCheckConst from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger -from msprobe.pytorch.compare.pt_compare import pt_compare +from msprobe.pytorch.compare.pt_compare import compare from msprobe.pytorch.compare.distributed_compare import compare_distributed @@ -14,7 +14,7 @@ def compare_cli(args): bench_path = input_param.get("bench_path", None) if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: - pt_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, + compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index aeea94945..923c0044d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -15,63 +15,16 @@ # limitations under the License. """ import os -import sys -import re from msprobe.core.common.utils import CompareException, check_compare_param, \ - check_configuration_param, task_dumppath_get, check_file_or_directory_path, check_regex_prefix_format_valid + check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import create_directory from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.pytorch.compare.pt_compare import PTComparator +from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json -def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): - def check_and_return_dir_contents(dump_dir, prefix): - """ - check the given dump dir and validate files in dump dir by using the given prefix patterns to build a - pattern: ^{prefix}(?:0|[0-9][1-9]*)?$ - - Args: - dump_dir (str): dump dir - prefix (str): prefix for the patterns, prefix should be less than 20 characters and alphanumeric/-/_ only - - Returns: - content [list]: dir contents - Raises: - CompareException: invalid path - ValueError: prefix not match the patterns - - """ - check_regex_prefix_format_valid(prefix) - check_file_or_directory_path(dump_dir, True) - contents = os.listdir(dump_dir) - pattern = re.compile(rf'^{prefix}(?:0|[0-9][1-9]*)?$') - for name in contents: - if not pattern.match(name): - logger.error( - f"dump_dir contains '{name}'. Expected '{prefix}'. This name is not in the format of dump " - f"output. Please check and delete irrelevant files in {dump_dir} and try again." - ) - raise CompareException(CompareException.INVALID_PATH_ERROR) - return contents - - def extract_json(dirname, stack_json=False): - json_path = '' - for fname in os.listdir(dirname): - if fname=="construct.json": continue - full_path = os.path.join(dirname, fname) - if full_path.endswith('.json'): - json_path = full_path - if not stack_json and 'stack' not in json_path: - break - if stack_json and 'stack' in json_path: - break - - # Provide robustness on invalid directory inputs - if not json_path: - logger.error(f'No file is found in dump dir {dirname}. ') - raise CompareException(CompareException.NO_DUMP_FILE_ERROR) - return json_path +def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): if kwargs.get('suffix'): logger.error("Argument 'suffix' is not supported for compare_distributed.") raise CompareException(CompareException.INVALID_PARAM_ERROR) @@ -106,7 +59,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) + raise CompareException(error.code) from error ptComparator=PTComparator() ptComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml b/debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml new file mode 100644 index 000000000..eaffbe7a1 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/mapping.yaml @@ -0,0 +1,607 @@ +__and__: __and__ +__iand__: __iand__ +__ilshift__: __ilshift__ +__ior__: __ior__ +__irshift__: __irshift__ +__ixor__: __ixor__ +__lshift__: __lshift__ +__or__: __or__ +__rshift__: __rshift__ +__xor__: __xor__ +_adaptive_avg_pool2d: adaptive_avg_pool2d +_adaptive_avg_pool3d: adaptive_avg_pool3d +_cdist_forward: cdist +_cudnn_rnn: rnn +_embedding_bag: embedding_bag +_fft_c2c: fft +_fft_c2r: rfft +_foreach_add_: _foreach_add_ +_foreach_addcdiv: _foreach_addcdiv +_foreach_copy_: _foreach_copy_ +_foreach_lerp_: _foreach_lerp_ +_foreach_maximum: _foreach_maximum +_foreach_mul: _foreach_mul +_foreach_neg_: _foreach_neg_ +_foreach_pow: _foreach_pow +_foreach_reciprocal_: _foreach_reciprocal_ +_foreach_sign: _foreach_sign +_foreach_sqrt: _foreach_sqrt +_foreach_sqrt_: _foreach_sqrt_ +_foreach_sub: _foreach_sub +_fused_adam: FusedAdam +_linalg_det: det +_linalg_eigh: eigh +_linalg_slogdet: slogdet +_linalg_svd: svd +_list_to_tensor: as_tensor +_log_softmax: log_softmax +_native_batch_norm_legit: batch_norm +_nested_tensor_from_tensor_list: _nested_tensor_from_tensor_list +_pdist_forward: pdist +_pin_memory: pin_memory +_reshape_alias: reshape +_resize_output_: resize_ +_softmax: softmax +_to_copy: to +abs: abs +abs_: abs_ +absolute: abs +absolute_: abs_ +acos: acos +acos_: acos_ +acosh: acosh +acosh_: acosh_ +adaptive_max_pool2d: adaptive_max_pool2d +adaptive_max_pool3d: adaptive_max_pool3d +add: add +add_: add_ +addbmm: addbmm +addbmm_: addbmm_ +addcdiv: addcdiv +addcdiv_: addcdiv_ +addcmul: addcmul +addcmul_: addcmul_ +addmm: addmm +addmm_: addmm_ +addmv: addmv +addmv_: addmv_ +addr: addr +affine_grid_generator: affine_grid +alias: alias +all: all +alpha_dropout: AlphaDropout +amax: amax +amin: amin +aminmax: aminmax +angle: angle +any: any +arange: arange +arccos: acos +arccos_: arccos_ +arccosh: arccosh +arccosh_: arccosh_ +arcsin: asin +arcsin_: arcsin_ +arcsinh: asinh +arcsinh_: arcsinh_ +arctan: atan +arctan2: atan2 +arctan2_: arctan2_ +arctan_: arctan_ +arctanh: arctanh +arctanh_: arctanh_ +argmax: argmax +argmin: argmin +argsort: argsort +as_strided: as_strided +asin: asin +asin_: asin_ +asinh: asinh +asinh_: asinh_ +atan: atan +atan2: atan2 +atan2_: atan2_ +atan_: atan_ +atanh: atanh +atanh_: atanh_ +avg_pool2d: avg_pool2d +avg_pool3d: avg_pool3d +baddbmm: baddbmm +baddbmm_: baddbmm_ +bernoulli: bernoulli +bernoulli_: bernoulli_ +binary_cross_entropy: BCELoss +binary_cross_entropy_with_logits: binary_cross_entropy_with_logits +bitwise_and: bitwise_and +bitwise_and_: bitwise_and_ +bitwise_left_shift: __lshift__ +bitwise_left_shift_: bitwise_left_shift_ +bitwise_not: bitwise_not +bitwise_not_: bitwise_not_ +bitwise_or: bitwise_or +bitwise_or_: bitwise_or_ +bitwise_right_shift: __rshift__ +bitwise_right_shift_: bitwise_right_shift_ +bitwise_xor: bitwise_xor +bitwise_xor_: bitwise_xor_ +bmm: bmm +broadcast_tensors: broadcast_tensors +bucketize: bucketize +cat: cat +cauchy: Cauchy +cauchy_: cauchy_ +ceil: ceil +ceil_: ceil_ +celu: celu +celu_: celu_ +cholesky: cholesky +cholesky_inverse: cholesky_inverse +cholesky_solve: cholesky_solve +clamp: clamp +clamp_: clamp_ +clamp_max: clamp_max +clamp_max_: clamp_max_ +clamp_min: clamp_min +clamp_min_: clamp_min_ +clip: clip +clip_: clip_ +clone: clone +col2im: col2im +complex: complex +conj_physical: conj +conj_physical_: conj_ +constant_pad_nd: pad +convolution: Conv2d +copy: copy_ +copy_: copy_ +copysign: copysign +copysign_: copysign_ +cos: cos +cos_: cos_ +cosh: cosh +cosh_: cosh_ +count_nonzero: count_nonzero +cudnn_batch_norm: BatchNorm2d +cummax: cummax +cummin: cummin +cumprod: cumprod +cumprod_: cumprod_ +cumsum: cumsum +cumsum_: cumsum_ +deg2rad: deg2rad +deg2rad_: deg2rad_ +detach: detach +diag: diag +diag_embed: diag_embed +diagonal: diagonal +diagonal_copy: diagonal +diagonal_scatter: diagonal +digamma: digamma +digamma_: digamma_ +dist: dist +div: div +div_: div_ +divide: div +divide_: divide_ +dot: dot +dropout: dropout +elu: ELU +elu_: elu_ +embedding: embedding +empty_like: empty_like +empty_strided: empty_strided +eq: eq +eq_: eq_ +erf: erf +erf_: erf_ +erfc: erfc +erfc_: erfc_ +erfinv: erfinv +erfinv_: erfinv_ +exp: exp +exp2: exp2 +exp2_: exp2_ +exp_: exp_ +expand: expand +expm1: expm1 +expm1_: expm1_ +exponential: Exponential +exponential_: exponential_ +eye: eye +fft_fft: fft +fft_fft2: fft2 +fft_fftn: fftn +fft_fftshift: fftshift +fft_hfft: hfft +fft_hfft2: hfft2 +fft_hfftn: hfftn +fft_ifft: ifft +fft_ifft2: ifft2 +fft_ifftn: ifftn +fft_ifftshift: ifftshift +fft_ihfft: ihfft +fft_ihfft2: ihfft2 +fft_ihfftn: ifftn +fft_irfft: irfft +fft_irfft2: irfft2 +fft_irfftn: irfftn +fft_rfft: rfft +fft_rfft2: rfft2 +fft_rfftn: rfftn +fill: fill_ +fill_: fill_ +fix: fix +fix_: fix_ +flip: flip +float_power_: float_power_ +floor: floor +floor_: floor_ +floor_divide: floor_divide +floor_divide_: floor_divide_ +fmax: fmax +fmin: fmin +fmod: fmod +fmod_: fmod_ +frac: frac +frac_: frac_ +full: full +full_like: full_like +gather: gather +gcd: gcd +gcd_: gcd_ +ge: ge +ge_: ge_ +gelu: GELU +gelu_: gelu_ +geometric: Geometric +geometric_: geometric_ +glu: glu +greater: gt +greater_: ge_ +greater_equal: ge +greater_equal_: ge_ +grid_sampler_2d: grid_sample +grid_sampler_3d: grid_sample +gru: GRU +gt: gt +gt_: gt_ +hardshrink: Hardshrink +hardsigmoid: hardsigmoid +hardsigmoid_: hardsigmoid_ +hardswish: hardswish +hardswish_: hardswish_ +hardtanh: hardtanh +hardtanh_: hardtanh_ +heaviside: heaviside +heaviside_: heaviside_ +hinge_embedding_loss: HingeEmbeddingLoss +huber_loss: huber_loss +hypot: hypot +hypot_: hypot_ +i0: i0 +i0_: i0_ +igamma: igamma +igamma_: igamma_ +igammac: igammac +igammac_: igammac_ +index: __getitem__ +index_add: index_add +index_add_: index_add_ +index_copy: index_copy_ +index_copy_: index_copy_ +index_fill: index_fill_ +index_fill_: index_fill_ +index_put: index_put_ +index_put_: index_put_ +index_reduce: index_select +index_select: index_select +is_pinned: is_pinned +is_same_size: is_same_size +isinf: isinf +isnan: isnan +isneginf: isneginf +isposinf: isposinf +istft: istft +item: item +lcm: lcm +lcm_: lcm_ +le: le +le_: le_ +leaky_relu: LeakyReLU +leaky_relu_: leaky_relu_ +lerp: lerp +lerp_: lerp_ +less: less +less_: less_ +less_equal: le +less_equal_: less_equal_ +lgamma: lgamma +lgamma_: lgamma_ +linalg_cholesky_ex: cholesky +linalg_cross: cross +linalg_householder_product: householder_product +linalg_inv_ex: inv +linalg_ldl_factor_ex: ldl +linalg_ldl_solve: ldl_solve +linalg_lu: lu +linalg_lu_factor_ex: lu_factor +linalg_lu_solve: lu_solve +linalg_matrix_exp: matrix_exp +linalg_qr: qr +linalg_solve_triangular: solve +linalg_vector_norm: norm +linspace: linspace +log: log +log10: log10 +log10_: log10_ +log1p: log1p +log1p_: log1p_ +log2: log2 +log2_: log2_ +log_: log_ +log_normal: LogNormal +log_sigmoid_forward: log_sigmoid +logaddexp: logaddexp +logaddexp2: logaddexp2 +_native_batch_norm_legit_functional: batch_norm +logcumsumexp: logcumsumexp +logical_and: logical_and +logical_and_: logical_and_ +logical_not: logical_not +logical_not_: logical_not_ +logical_or: logical_or +logical_or_: logical_or_ +logical_xor: logical_xor +logical_xor_: logical_xor_ +logit: logit +logit_: logit_ +logspace: logspace +logsumexp: logsumexp +lstm: LSTM +lt: lt +lt_: lt_ +lu_unpack: lu_unpack +margin_ranking_loss: margin_ranking_loss +masked_fill: masked_fill +masked_fill_: masked_fill_ +matmul: matmul +max: max +max_pool2d_with_indices: MaxPool2d +max_pool3d_with_indices: MaxPool3d +max_unpool2d: MaxUnpool2d +max_unpool3d: max_unpool3d +maximum: maximum +mean: mean +median: median +meshgrid: meshgrid +min: min +minimum: minimum +mish: Mish +mish_: mish_ +mm: mm +mode: mode +mse_loss: mse_loss +mul: mul +mul_: mul_ +multi_margin_loss: MultiMarginLoss +multilabel_margin_loss_forward: multilabel_margin_loss +multinomial: multinomial +multiply: multiply +multiply_: mul_ +mv: mv +mvlgamma: mvlgamma +mvlgamma_: mvlgamma_ +name: name +nan_to_num: nan_to_num +nan_to_num_: nan_to_num_ +nanmedian: nanmedian +nansum: nansum +narrow_copy: narrow +native_batch_norm: BatchNorm2d +native_dropout: dropout +native_group_norm: group_norm +native_layer_norm: LayerNorm +ne: ne +ne_: ne_ +neg: neg +neg_: neg_ +negative: neg +negative_: neg_ +new_empty: new_empty +new_empty_strided: new_empty_strided +new_full: new_full +new_ones: new_ones +new_zeros: new_zeros +nextafter: nextafter +nextafter_: nextafter_ +nll_loss: nll_loss +nll_loss2d_forward: NLLLoss2d +nll_loss_forward: NLLLoss +nonzero_static: nonzero +norm: norm +normal: normal +normal_: normal_ +not_equal: ne +not_equal_: ne_ +ones: ones +ones_like: ones_like +ormqr: ormqr +pairwise_distance: pairwise_distance +pdist: pdist +permute: permute +pin_memory: pin_memory +pixel_shuffle: PixelShuffle +polar: polar +polygamma: polygamma +positive: positive +pow: pow +pow_: pow_ +prelu: prelu +prod: prod +quantized_gru: GRU +quantized_lstm: LSTM +rad2deg: rad2deg +rad2deg_: rad2deg_ +rand: rand +rand_like: rand_like +randint: randint +randint_like: randint_like +randn: randn +randn_like: randn_like +randperm: randperm +reciprocal: reciprocal +reciprocal_: reciprocal_ +reflection_pad1d: reflection_pad1d +reflection_pad2d: reflection_pad2d +reflection_pad3d: ReflectionPad3d +relu: relu +relu6: relu6 +relu_: relu_ +remainder: remainder +remainder_: remainder_ +renorm: renorm +renorm_: renorm_ +repeat: repeat +repeat_interleave: repeat_interleave +replication_pad1d: ReplicationPad1d +replication_pad2d: replication_pad2d +replication_pad3d: replication_pad3d +resize_as_: resize_as_ +rnn_relu: RNN +rnn_tanh: RNN +roll: roll +rot90: rot90 +round: round +round_: round_ +rrelu_with_noise: RReLU +rrelu_with_noise_: rrelu_with_noise +rsqrt: rsqrt +rsqrt_: rsqrt_ +rsub: rsub +scalar_tensor: scalar_tensor +scatter: scatter_ +scatter_: scatter_ +scatter_add: scatter_add +scatter_add_: scatter_add_ +searchsorted: searchsorted +select: select +selu: selu +selu_: selu_ +sgn: sgn +sgn_: sgn_ +sigmoid: sigmoid +sigmoid_: sigmoid_ +sign: sign +sign_: sign_ +signbit: signbit +silu: silu +silu_: silu_ +sin: sin +sin_: sin_ +sinc: sinc +sinc_: sinc_ +sinh: sinh +sinh_: sinh_ +slice: slice +smooth_l1_loss: smooth_l1_loss +soft_margin_loss: soft_margin_loss +softplus: softplus +softshrink: softshrink +sort: sort +special_airy_ai: airy_ai +special_bessel_j0: j0 +special_bessel_j1: j1 +special_bessel_y0: y0 +special_bessel_y1: y1 +special_chebyshev_polynomial_t: chebyshev_t +special_chebyshev_polynomial_u: chebyshev_u +special_entr: entr +special_erfcx: erfcx +special_hermite_polynomial_h: hermite +special_hermite_polynomial_he: he +special_i0: i0 +special_i0e: i0e +special_i1: i1 +special_i1e: i1e +special_laguerre_polynomial_l: laguerre_l +special_log_ndtr: log_ndtr +special_modified_bessel_i0: i0 +special_modified_bessel_i1: i1 +special_modified_bessel_k0: k0 +special_modified_bessel_k1: i1 +special_ndtr: ndtr +special_ndtri: ndtri +special_scaled_modified_bessel_k0: i0e +special_scaled_modified_bessel_k1: scaled_modified_bessel_k1 +special_spherical_bessel_j0: spherical_jn +special_xlog1py: xlog1py +special_zeta: zeta +split: split +split_with_sizes: split +sqrt: sqrt +sqrt_: sqrt_ +square: square +square_: square_ +squeeze: squeeze +stack: stack +std: std +std_mean: std_mean +stft: stft +sub: sub +sub_: sub_ +subtract: sub +subtract_: subtract_ +sum: sum +t: t +t_: t_ +take: take +tan: tan +tan_: tan_ +tanh: tanh +tanh_: tanh_ +threshold: threshold +threshold_: threshold_ +to: to +topk: topk +trace: trace +transpose: transpose +transpose_: transpose_ +triangular_solve: triangular_solve +tril: tril +tril_: tril_ +tril_indices: tril_indices +triu: triu +triu_: triu_ +triu_indices: triu_indices +true_divide: true_divide +true_divide_: true_divide_ +trunc: trunc +trunc_: trunc_ +unbind: unbind +unfold: unfold +uniform: Uniform +uniform_: uniform_ +unsafe_chunk: unsafe_chunk +unsafe_split: split +unsafe_split_with_sizes: split_with_sizes +unsqueeze: unsqueeze +unsqueeze_: unsqueeze_ +upsample_bicubic2d: interpolate +upsample_bilinear2d: upsample_bilinear +upsample_nearest1d: interpolate +upsample_nearest2d: interpolate +upsample_nearest3d: interpolate +var: var +var_mean: var_mean +vdot: vdot +view: view +where: where +xlogy: xlogy +xlogy_: xlogy_ +zero: zeros +zero_: zero_ +zeros: zeros +zeros_like: zeros_like + + + diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/match.py b/debug/accuracy_tools/msprobe/pytorch/compare/match.py new file mode 100644 index 000000000..2a46105bd --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/compare/match.py @@ -0,0 +1,36 @@ +import os +import yaml +from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import CompareException + + +class AtenIrMapping(): + def __init__(self): + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(cur_path, "mapping.yaml") + with FileOpen(yaml_path, 'r') as f: + self.aten_mapping = yaml.safe_load(f) + + def match(self, op1, op2): + if "Aten" in op1 and "Aten" not in op2: + return self.match_op(op1, op2) + else: + return self.match_op(op2, op1) + + def match_op(self, aten_op, torch_op): + try: + aten_op_raw_name_overload = '_'.join(aten_op.split("_")[1:-3]) + aten_op_raw_name = aten_op_raw_name_overload.split('.')[0] + torch_op_raw_name = '_'.join(torch_op.split("_")[1:-3]).lower() + except IndexError as e: + err_msg = f"Dump op name format error: {aten_op}, {torch_op}. Your dump data may be corrupted." + raise CompareException.INVALID_DATA_ERROR(err_msg) from e + matching_op = self.aten_mapping.get(aten_op_raw_name) + if matching_op is None: + return False + if matching_op.lower() == torch_op_raw_name: + return True + return False + + +graph_mapping = AtenIrMapping() diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index b32e6df60..a947a12f6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,27 +1,24 @@ import json -import multiprocessing import os.path -import sys import torch -import pandas as pd from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.const import FileCheckConst -from msprobe.core.compare.utils import merge_tensor, get_un_match_accuracy, get_accuracy, read_op -from msprobe.core.compare.Multiprocessing_compute import ComparisonResult, _save_cmp_result, _handle_multi_process +from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy +from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException + class PTComparator (Comparator): def __init__(self): - super().__init__() - + self.frame_name=PTComparator.__name__ def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): cos_result = [] @@ -58,20 +55,7 @@ class PTComparator (Comparator): five_thousand_err_ratio_result=five_thousand_err_ratio_result ) - return _save_cmp_result(idx, cr, result_df, lock) - - - def gen_merge_list(self,json_data,op_name,stack_json_data,summary_compare,md5_compare): - op_data = json_data['data'][op_name] - op_parsed_list = read_op(op_data, op_name) - if op_name in stack_json_data: - op_parsed_list.append({'full_op_name': op_name, 'full_info': stack_json_data[op_name]}) - else: - op_parsed_list.append({'full_op_name': op_name, 'full_info': None}) - - merge_list = merge_tensor(op_parsed_list, summary_compare, md5_compare) - return merge_list - + return _save_cmp_result(idx, cr, result_df, lock) def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): npu_json_handle, bench_json_handle, stack_json_handle = file_handles @@ -138,33 +122,6 @@ class PTComparator (Comparator): result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df - def make_result_table(self,result,md5_compare,summary_compare,stack_mode): - header = [] - if md5_compare: - header = CompareConst.MD5_COMPARE_RESULT_HEADER[:] - elif summary_compare: - header = CompareConst.SUMMARY_COMPARE_RESULT_HEADER[:] - else: - header = CompareConst.COMPARE_RESULT_HEADER[:] - - all_mode_bool = not (summary_compare or md5_compare) - if stack_mode: - if all_mode_bool: - header.append(CompareConst.STACK) - header.append(CompareConst.DATA_NAME) - else: - header.append(CompareConst.STACK) - else: - if all_mode_bool: - for row in result: - del row[-2] - header.append(CompareConst.DATA_NAME) - else: - for row in result: - del row[-1] - result_df = pd.DataFrame(result, columns=header) - return result_df - def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -174,16 +131,7 @@ class PTComparator (Comparator): if data_value.dtype == torch.bfloat16: data_value = data_value.to(torch.float32) data_value = data_value.numpy() - return data_value - - - def _do_multi_process(self,input_parma, result_df): - try: - result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) - return result_df - except ValueError as e: - logger.error('result dataframe is not found.') - raise CompareException(CompareException.INVALID_DATA_ERROR) from e + return data_value def compare_core(self,input_parma, output_path, **kwargs): """ @@ -231,8 +179,9 @@ class PTComparator (Comparator): advisor = Advisor(result_df, output_path) advisor.analysis() - -def pt_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): + + +def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) @@ -240,24 +189,13 @@ def pt_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') - sys.exit(error.code) + raise CompareException(error.code) from error ptComparator=PTComparator() ptComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) -def _compare_parser(parser): - parser.add_argument("-i", "--input_path", dest="input_path", type=str, - help=" The compare input path, a dict json.", required=True) - parser.add_argument("-o", "--output_path", dest="output_path", type=str, - help=" The compare task result out path.", required=True) - parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", - help=" Whether to save stack info.", required=False) - parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", - help=" Whether to give advisor.", required=False) - parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", - help=" Whether to perform a fuzzy match on the api name.", required=False) diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py index 012d42faf..8433f0af6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/precision_debugger.py @@ -36,7 +36,7 @@ class PrecisionDebugger: common_config, task_config = parse_json_config(config_path, task) self.task = common_config.task if self.task == Const.GRAD_PROBE: - GradientMonitor(task_config, model) + self.gm = GradientMonitor(common_config, task_config) return if step: common_config.step = step @@ -102,6 +102,14 @@ class PrecisionDebugger: raise Exception("PrecisionDebugger instance is not created.") cls._instance.service.step() + @classmethod + def monitor(cls, model): + if not cls._instance: + raise Exception("PrecisionDebugger instance is not created.") + if cls._instance.task != Const.GRAD_PROBE: + return + cls._instance.gm.monitor(model) + def iter_tracer(func): def func_wrapper(*args, **kwargs): diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index edd28635d..36aec34e0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -2,54 +2,38 @@ import os from collections import defaultdict import torch -from torch.optim.optimizer import register_optimizer_step_pre_hook +if int(torch.__version__.split('.')[0]) >= 2: + from torch.optim.optimizer import register_optimizer_step_pre_hook from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target -from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.grad_probe.constant import GradConst, level_adp from msprobe.core.common.file_check import create_directory from msprobe.core.common.log import logger -from msprobe.core.common.utils import remove_path, write_csv +from msprobe.core.common.utils import remove_path, write_csv, save_npy from msprobe.pytorch.common.utils import get_rank_id, print_rank_0, save_pt class GradientMonitor: - level_adp = { - "L0": { - "header": [GradConst.MD5, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], - "have_grad_direction": False - }, - "L1": { - "header": [GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], - "have_grad_direction": True - }, - "L2": { - "header": [GradConst.DISTRIBUTION, GradConst.MAX, GradConst.MIN, GradConst.NORM, GradConst.SHAPE], - "have_grad_direction": True - }, - } - def __init__(self, config, model): - self._config = config._config - self._model = model - level = self._config.get("level") - if level not in GradientMonitor.level_adp: - raise Exception(f"level is valid, not in {GradientMonitor.level_adp.keys()}") - self._level_adp = GradientMonitor.level_adp[level] - self._param_list = self._config.get('param_list') - self._target_ranks = self._config.get("rank") + def __init__(self, common_config, task_config): + level = task_config.grad_level + if level not in level_adp: + raise Exception(f"level is valid, not in {level_adp.keys()}") + self._level_adp = level_adp[level] + self._param_list = task_config.param_list + self._target_ranks = common_config.rank logger.info(f"target rank {self._target_ranks}") - self._target_step = self._config.get("step") + self._target_step = common_config.step logger.info(f"target step {self._target_step}") - self._bounds = self._config.get("bounds") + self._bounds = task_config.bounds check_numeral_list_ascend(self._bounds) - self._output_path = self._config.get("output_path") + self._output_path = common_config.dump_path if not os.path.exists(self._output_path): create_directory(self._output_path) else: logger.warning(f"the file in {self._output_path} will be recoverd") self._step = -1 self._param2name = defaultdict(str) - self._monitor() @property def output_path(self): @@ -61,12 +45,12 @@ class GradientMonitor: create_directory(save_path) param_grad = grad.clone().detach() is_positive = param_grad > 0 - save_filepath = os.path.join(save_path, f"{param_name}.pt") - save_pt(is_positive, save_filepath) + save_filepath = os.path.join(save_path, f"{param_name}.npy") + save_npy(is_positive.numpy(), save_filepath) - def _monitor(self): + def monitor(self, model): print_rank_0("> parameter names:") - for name, param in self._model.named_parameters(): + for name, param in model.named_parameters(): self._param2name[param] = name print_rank_0(f"\t{name}") setattr(self, "_rank", get_rank_id()) @@ -102,5 +86,5 @@ class GradientMonitor: header_result = GradStatCsv.generate_csv_header(self._level_adp, self._bounds) output_lines.insert(0, header_result) write_csv(output_lines, output_path) - - register_optimizer_step_pre_hook(optimizer_pre_step_hook) + if int(torch.__version__.split('.')[0]) >= 2: + register_optimizer_step_pre_hook(optimizer_pre_step_hook) diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py index ae01b75ee..757a1aebf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_stat_csv.py @@ -63,13 +63,15 @@ class CSV_distribution(CsvItem): def generate_csv_header(csv_header_input): bounds = csv_header_input.bounds intervals = [] - for i, _ in enumerate(bounds): - if i == 0: - intervals.append(f"(-inf, {bounds[i]}]") - else: + if bounds: + intervals.append(f"(-inf, {bounds[0]}]") + for i in range(1, len(bounds)): intervals.append(f"({bounds[i-1]}, {bounds[i]}]") - intervals.extend([f"({bounds[-1]}, inf)", "=0"]) - return intervals + if intervals: + intervals.append(f"({bounds[-1]}, inf)") + intervals.append("=0") + + return intervals def generate_csv_content(csv_content_input): grad = csv_content_input.grad diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index daba5476c..2db6980bb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -96,7 +96,9 @@ class RunUTConfig(BaseConfig): class GradToolConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) - self._config = json_config + self.grad_level = json_config.get("grad_level") + self.param_list = json_config.get("param_list") + self.bounds = json_config.get("bounds") def parse_task_config(task, json_config): diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py index 30212d95e..fb408e83b 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py @@ -19,7 +19,7 @@ from unittest.mock import patch, mock_open from msprobe.core.common.const import Const from msprobe.mindspore.ms_config import (parse_json_config, parse_task_config, - TensorConfig, StatisticsConfig, OverflowCheckConfig) + TensorConfig, StatisticsConfig, OverflowCheckConfig, FreeBenchmarkConfig) class TestMsConfig(TestCase): @@ -64,6 +64,9 @@ class TestMsConfig(TestCase): task_config = parse_task_config("overflow_check", mock_json_config) self.assertTrue(isinstance(task_config, OverflowCheckConfig)) + task_config = parse_task_config("free_benchmark", mock_json_config) + self.assertTrue(isinstance(task_config, FreeBenchmarkConfig)) + with self.assertRaises(Exception) as context: - parse_task_config("free_benchmark", mock_json_config) + parse_task_config("unsupported_task", mock_json_config) self.assertEqual(str(context.exception), "task is invalid.") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py new file mode 100644 index 000000000..25189a9b6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import os + +import unittest +from unittest.mock import Mock, patch +import copy +from msprobe.core.common.utils import Const +from msprobe.mindspore.service import Service +import mindspore +from mindspore.common.tensor import Tensor +from mindspore import ops +from mindspore import nn +from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common_config import CommonConfig, BaseConfig +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from unittest.mock import MagicMock +import numpy as np + + +class DummyModel(nn.Cell): + def __init__(self): + super(DummyModel, self).__init__() + self.dense = nn.Dense(2, 2) + + def construct(self, x): + return self.dense(x) +class TestService(unittest.TestCase): + def setUp(self): + json_config = { + "task": "statistics", + "dump_path": "/absolute_path", + "rank": [], + "step": [0, 2], + "level": "L1" + } + + common_config = CommonConfig(json_config) + task_config = BaseConfig(json_config) + config = DebuggerConfig(common_config, task_config) + self.service = Service(config) + self.service.model = Mock() + self.service.data_collector = Mock() + self.service.switch = True # Make sure the switch is on for testing + + def test_check_model_valid_none(self): + model = None + self.assertIsNone(self.service.check_model_valid(model)) + + def test_check_model_valid_valid_model(self): + model = DummyModel() + self.assertEqual(self.service.check_model_valid(model), model) + + def test_check_model_valid_invalid_model(self): + model = "invalid_model" + with self.assertRaises(MsprobeException) as context: + self.service.check_model_valid(model) + + # For the purpose of the test, let's also verify the expected exception message + expected_message = "[msprobe] 无效参数： model 参数必须是 mindspore.nn.Cell 类型。" + self.assertEqual(str(context.exception), expected_message) + + def test_update_primitive_counters(self): + primitive_name = "test_primitive" + self.service.update_primitive_counters(primitive_name) + self.assertEqual(self.service.primitive_counters[primitive_name], 0) + self.service.update_primitive_counters(primitive_name) + self.assertEqual(self.service.primitive_counters[primitive_name], 1) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py index 41be7b1db..cdc88a3be 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py @@ -21,6 +21,7 @@ from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.kernel_graph_dump import KernelGraphDump from msprobe.mindspore.task_handler_factory import TaskHandlerFactory +from msprobe.mindspore.common.const import Const class TestTaskHandlerFactory(TestCase): @@ -43,6 +44,7 @@ class TestTaskHandlerFactory(TestCase): common_config = CommonConfig(json_config) task_config = BaseConfig(json_config) config = DebuggerConfig(common_config, task_config) + config.execution_mode = Const.GRAPH_GE_MODE handler = TaskHandlerFactory.create(config) self.assertTrue(isinstance(handler, KernelGraphDump)) @@ -52,7 +54,7 @@ class TestTaskHandlerFactory(TestCase): TaskHandlerFactory.create(config) self.assertEqual(str(context.exception), "Can not find task handler") - config.task = "free_benchmark" + config.task = "Free_benchmark" with self.assertRaises(Exception) as context: TaskHandlerFactory.create(config) self.assertEqual(str(context.exception), "valid task is needed.") diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/advisor/test_advisor.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/advisor/test_advisor.py index 176b80068..e140f8263 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/advisor/test_advisor.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/advisor/test_advisor.py @@ -7,8 +7,8 @@ from unittest.mock import patch import pandas -from msprobe.pytorch.advisor.advisor import Advisor -from msprobe.pytorch.advisor.advisor_const import AdvisorConst +from msprobe.core.advisor.advisor import Advisor +from msprobe.core.advisor.advisor_const import AdvisorConst class TestAdvisor(unittest.TestCase): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py index 288e259c0..b08b09c85 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py @@ -1,7 +1,10 @@ # coding=utf-8 import unittest import pandas as pd -from msprobe.pytorch.compare import acc_compare as compare +from msprobe.core.compare.check import check_graph_mode +from msprobe.core.compare.utils import merge_tensor, read_op, get_accuracy, rename_api +from msprobe.core.compare.highlight import find_error_rows,find_compare_result_error_rows +from msprobe.pytorch.compare.pt_compare import PTComparator npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], @@ -208,60 +211,62 @@ class TestUtilsMethods(unittest.TestCase): def test_check_graph_mode(self): op1 = "Aten" op2 = "torch" - self.assertTrue(compare.check_graph_mode(op1, op2)) - self.assertTrue(compare.check_graph_mode(op2, op1)) - self.assertFalse(compare.check_graph_mode(op1, op1)) - self.assertFalse(compare.check_graph_mode(op2, op2)) + self.assertTrue(check_graph_mode(op1, op2)) + self.assertTrue(check_graph_mode(op2, op1)) + self.assertFalse(check_graph_mode(op1, op1)) + self.assertFalse(check_graph_mode(op2, op2)) def test_check_op(self): fuzzy_match = False - result = compare.check_op(npu_dict, bench_dict, fuzzy_match) + ptComparator=PTComparator() + result = ptComparator.check_op(npu_dict, bench_dict, fuzzy_match) self.assertEqual(result, True) def test_merge_tensor(self): - op_dict = compare.merge_tensor(tensor_list, True, False) + op_dict = merge_tensor(tensor_list, True, False) self.assertEqual(op_dict, result_op_dict) def test_read_op(self): - result = compare.read_op(op_data, op_name) + result = read_op(op_data, op_name) self.assertEqual(result, op_result) def test_match_op(self): fuzzy_match = False - a, b = compare.match_op([npu_dict], [bench_dict], fuzzy_match) + ptComparator=PTComparator() + a, b = ptComparator.match_op([npu_dict], [bench_dict], fuzzy_match) self.assertEqual(a, 0) self.assertEqual(b, 0) def test_get_accuracy(self): result = [] - compare.get_accuracy(result, npu_dict, bench_dict, highlight_dict) + get_accuracy(result, npu_dict, bench_dict, highlight_dict) self.assertEqual(result, o_result) def test_get_accuracy_graph_mode(self): result = [] - compare.get_accuracy(result, npu_dict_aten, bench_dict_functional, highlight_dict) + get_accuracy(result, npu_dict_aten, bench_dict_functional, highlight_dict) self.assertEqual(result, aten_result) def test_find_error_rows(self): summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] highlight_dict = {'red_rows': [], 'yellow_rows': []} - compare.find_error_rows(summary_result, 0, 1, highlight_dict, summary_compare=True) + find_error_rows(summary_result, 0, 1, highlight_dict, summary_compare=True) self.assertEqual(highlight_dict, {'red_rows': [], 'yellow_rows': []}) def test_find_compare_result_error_rows(self): result = [line_input, line_1, line_2, line_3] result_df = pd.DataFrame(result) highlight_dict = {'red_rows': [], 'yellow_rows': []} - compare.find_compare_result_error_rows(result_df, highlight_dict, False, False) + find_compare_result_error_rows(result_df, highlight_dict, False, False) self.assertEqual(highlight_dict, {'red_rows': [num_1, num_3], 'yellow_rows': [num_2]}) def test_rename_api(self): test_name_1 = "Distributed.broadcast.0.forward.input.0" expect_name_1 = "Distributed.broadcast.input.0" - actual_name_1 = compare.rename_api(test_name_1, "forward") + actual_name_1 = rename_api(test_name_1, "forward") self.assertEqual(actual_name_1, expect_name_1) test_name_2 = "Torch.sum.0.backward.output.0" expect_name_2 = "Torch.sum.output.0" - actual_name_2 = compare.rename_api(test_name_2, "backward") + actual_name_2 = rename_api(test_name_2, "backward") self.assertEqual(actual_name_2, expect_name_2) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py index 399efeb42..8be3be413 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/free_benchmark/result_handlers/test_result_handler.py @@ -15,6 +15,7 @@ from msprobe.pytorch.free_benchmark.common.params import DataParams, make_handle from msprobe.pytorch.free_benchmark.result_handlers.handler_factory import ( FuzzHandlerFactory, ) +from msprobe.pytorch.free_benchmark.result_handlers.base_handler import FuzzHandler class Config(ABC): @@ -119,3 +120,21 @@ class TestFuzzHandler(TestCase): api_threshld, ThresholdConfig.DTYPE_PER_THD[torch.float16] ) + + def test_tensor_split_for_error_calculate(self): + # 设置模拟的张量的大小 + tensor_size = 256 * 1024 * 1024 + origin_output = torch.randn(tensor_size, dtype=torch.float32) + perturbed_output = torch.randn(tensor_size, dtype=torch.float32) + + # 调用tensor_split_for_error_calculate方法 + origin_output_chunks, perturbed_output_chunks = FuzzHandler.tensor_split_for_error_calculate( + origin_output, perturbed_output) + + # 验证返回的chunks数量和形状是否正确 + self.assertEqual(len(origin_output_chunks), 64) + self.assertEqual(len(perturbed_output_chunks), 64) + for chunk in origin_output_chunks: + self.assertEqual(chunk.shape, (4 * 1024 * 1024,)) + for chunk in perturbed_output_chunks: + self.assertEqual(chunk.shape, (4 * 1024 * 1024,)) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py index bd569f5a2..f39d3f091 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_csv.py @@ -4,6 +4,7 @@ import os import torch from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor +from msprobe.core.grad_probe.constant import level_adp grad_tensor = torch.tensor([[-2, 2], [0.2, 0.3]]) @@ -11,27 +12,27 @@ grad_tensor = torch.tensor([[-2, 2], [0.2, 0.3]]) class TestGradCSV(unittest.TestCase): def test_level_L0_header(self): self.assertEqual(['param_name', 'MD5', 'max', 'min', 'norm', 'shape'], - GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L0"], [-1, 0, 1])) + GradStatCsv.generate_csv_header(level_adp["L0"], [-1, 0, 1])) def test_level_L1_header(self): self.assertEqual(['param_name', 'max', 'min', 'norm', 'shape'], - GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L1"], [-1, 0, 1])) + GradStatCsv.generate_csv_header(level_adp["L1"], [-1, 0, 1])) def test_level_L2_header(self): self.assertEqual(['param_name', '(-inf, -1]', '(-1, 0]', '(0, 1]', '(1, inf)', '=0', 'max', 'min', 'norm', 'shape'], - GradStatCsv.generate_csv_header(GradientMonitor.level_adp["L2"], [-1, 0, 1])) + GradStatCsv.generate_csv_header(level_adp["L2"], [-1, 0, 1])) def test_level_L0_content(self): - generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L0"], grad_tensor, [-1, 0, 1]) + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", level_adp["L0"], grad_tensor, [-1, 0, 1]) self.assertEqual(['model.conv2d', '678a6c7d9d9716682b56fda097d0936c', 2.0, -2.0, 2.851315498352051, [2, 2]], generated_csv_line) def test_level_L1_content(self): - generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L1"], grad_tensor, [-1, 0, 1]) + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", level_adp["L1"], grad_tensor, [-1, 0, 1]) self.assertEqual(['model.conv2d', 2.0, -2.0, 2.851315498352051, [2, 2]], generated_csv_line) def test_level_L2_content(self): - generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", GradientMonitor.level_adp["L2"], grad_tensor, [-1, 0, 1]) + generated_csv_line = GradStatCsv.generate_csv_line("model.conv2d", level_adp["L2"], grad_tensor, [-1, 0, 1]) self.assertEqual(['model.conv2d', 0.25, 0.0, 0.5, 0.25, 0.0, 2.0, -2.0, 2.851315498352051, [2, 2]], generated_csv_line) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py index d79cca502..607addd69 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/grad_probe/test_grad_monitor.py @@ -10,15 +10,24 @@ from msprobe.core.grad_probe.grad_compare import GradComparator from msprobe.pytorch.grad_probe.grad_monitor import GradientMonitor from msprobe.pytorch.pt_config import GradToolConfig +class config: + def __init__(self, config_dict): + for key, value in config_dict.items(): + setattr(self, key, value) -config_dict = { - "level": "L1", - "param_list": "", +common_config_dict = { "rank": [], "step": [], - "bounds": [-1,0,1], - "output_path": "./grad_output" + "dump_path": "./grad_output" +} +common_config = config(common_config_dict) + +task_config_dict = { + "grad_level": "L1", + "param_list": "", + "bounds": [-1,0,1] } +task_config = config(task_config_dict) def seed_all(seed=1234, mode=False): random.seed(seed) @@ -53,7 +62,8 @@ def get_grad_monitor(): nn.init.constant_(test_module.linear.bias, 1.0) optimizer = torch.optim.SGD(test_module.parameters(), lr=1e-2) - gm = GradientMonitor(GradToolConfig(config_dict), test_module) + gm = GradientMonitor(common_config, task_config) + gm.monitor(test_module) for input_data, label in zip(inputs, labels): output = test_module(input_data) diff --git "a/plugins/tensorboard-plugins/tb_plugin/docs/\345\205\254\347\275\221URL\350\257\264\346\230\216.xlsx" "b/plugins/tensorboard-plugins/tb_plugin/docs/\345\205\254\347\275\221URL\350\257\264\346\230\216.xlsx" index b7a8bf1fd0e7eec640e46af76e16c6a228f335ba..fbe5a354ffba8619d9e93012d6fa3715e1f50e19 100644 GIT binary patch delta 10316 zcmaKSWl)|?vnCKExI>WO?!nyy!7aGEyX%bvcOE3TySqbhcXxMphs}H5ulDSz-P){Kft$fsYHNjH)AW*9M;5{725( zd%ODCvFO#eG{;WQbBX%Z6ZR*t17%Z-cvDDnJyFVprTN2R^?dmF$3Ky~)}~ovFqdXS z42(4T#fSL3P2>LXa9(H<{Yv_lb?{w^5$%h9`~2ZZ=zTH=tjJeK4T}y}{f)lx!ux6> zn?M{)RP>)&Sy5J!{NFM6YxSQ5;SlJ}n7Vd&N3I$ z21h0WHAVP*=b@h1JJYTeE+hJ$9Otf~#UbD>6ds3mt+55NOfyB7k6!tp)X6tO&6d%b zUW$m@j&OshB7T@aHqDfm`UHsq1_t&SELe71P%!)~OZ*cU*cb@>6F#7Ad(DpDcR_i^ zlO>&U|9NWB9xV>*nW|FI-JA%Lii3&_i7av_2AfFwl^9$hh0+iJT%&iKSN>O=hc z$V&j!WZ$b!Fr7E@hvyHg-L`^&w2O9saw>hO`EFq9qm4BBsxeI(%LXCvUTkR0+NX}X z-KIbq*EVt*PSS3=vkZtuI^*(+v3vXa^;*w%iP3Lv_zl5LOM%+uon6P>5M@sCq^ z4mDhNe;C;AB=J6OZyv$Ep(#J(cXM1`20>wCM$Z>bVxyNcLJ1(k>niIqg@qg+Wpc$68v&R~rw|i%s(<0xIsymnI z0)B^E7U0MS>cmM$F~dFHI1(Bz^JF)IDoL@6J0i6lcZST>3=CdJBOT=VnhcY@Y)3E0 zQ+6|?YuAATJb*h>Q}=s51c5MKEIDH+MgBf(`>lzTY<`e24Sbb#c%eOmHSGwFSQNT@ z17Uj_#bg?;|0Etl@XMXgE5@Cp4c|G?SJ5_;vxKM2iD>m6-!*w&^?^{{XsHXn_!JNQ>(|En7^S4B z3ib!x>JlM+u)!@duH-eTRcuI*FfG;nA!iX_61ivp;P6}amlY;mvlh}Ci&Kylu zb!b&o>2db<-lnFfQ58b~Gk`ftlRzdcqQ7y$8aUh+Iz6wr&7zXvMT3G}GwOGic)g6? zt%wi}-O-lbUKCY1JwElcPyEwY=ZUU)9bE3$Al2jmP9`zQmOKL`vV}Mo6B|TSqUjeL z7V2B$wKVTx4kCfp2+S=05KfjZ|zh>Bxys5vQMV5)yd1VoaM3_&ZaVn>pN;Oq4M#S$t!o;GfJ=oX8|$aQIghIi+lIk zJmgH}>>4rpVvYXXg4MSog4AZ3u+%GUUV>~og5hggvW=NE$Hf`6syM(z<~?tD>E?)* zrhKDb--{wu$Tl9}=e|@)njWyI`x$-IVYY94b>X#CX4{`v)We?;>`J3sj(5$FF`C6n z3qLLR5yj3W>BYO8w{bfmLdn+B_19q9IoaaIIx--=H*7?Hn&#{_k_$NLrDe4 z@s_(%tGK(dS*>TGQhAG*O#5bQDzEp96U${t zT;BT@;rn`;PR@cv*?eWrSOmLzRh`Vm9^8=n4XV@iv^%YOa~hBkFliEP!)LH>Xpr(6 zb%9-$PGaz^O_}tU>O3AmND4t!#3h$q;M84V<2Pv)Gu*{ZkRU9dED40dMh<3XN^@Mm zQMKgh8?Y+O8gm$CU|=|hT?}NA|8@VeD(y7uohv$eDr?o98rt+ zmNhtLTq7z3{JzB)VcEw#P-XHINuTO7pFVu?K?e#4!VBMSKeq;))$}Nor=6>5wBuEd<~{vy0?75UN`XC7KClMGFlS zW2uhSwhw}~ot-TFT46uk&Bjlq9UwMRpS|At%K=emVY2*d#T630)=t1Qn^~O}`iP(9 zWlK=OKRgy&xP%cw&IP2*$k|g)^bLu_5RT7rf3@p>wGWnmweb}~xqB&U*;)Y$Y;|_Q zmOz^|{K8|F6Cy7CLYZ>g=Vs_i%h?RwW7{@Qcq|VMr}hdepxhw^w8*@t_T_G}=WXOD<$h zmDLxo9gtLXo#5UbB_)2c-RbUgA>51<;$9x#I&raZ<>Re<2t0GfNN8Cn4t8wi}Z{$ujpOLMvMpEGHWDbZ!y1f?8 zPbZ6`wBX=5iXz@jEy;@&IJ4>nJFh$D-!)j?aIwUC8#)c9>+^}(Cd`8)tJgML60=;h z@LU81B~xBpO~$ydgLmq+pEX1Cl(J(51ue;AQaofI;Lb;8&^9$%&sFiyDtmfGGzTqZ zb=C3LDfe(+q_LFd?Axz!xdYnT2+nir2e=GDB~0bH6I|EhnyGc`?BYGo@~Y|16#{8@Bepo9C z+XS0ixze@Pyt{0^^Jn$>V758?gTHpv?hGelvgLEE(ZojTg!4a5fwf7dGThw#!gC#% zrVDoL24{LB4Y^ZHQ?nnuv^K}*&c3=;fW%2IM} zy1W^2r)gt*z||voIEP#uKB_XnVzo=K|01y=Ai*9}qFpBGK-}C#lUg$5mlH zccSZhF$hk;dI)JMCtGXP6^}uGJ6Covw~xkj)j@l^pCe_yWtJ6y_45=BBG>;f%CEf` z;OD6J*wSbo@DyF145m75P@qFsr3xZXc#5fuFnp3V9x_#$B~9Xhj{ddVyH?C3%!?A~ zOa7kkx;p>?ZN}-&TL2|v8F7l%>(T=96$8WjY3+dXOqjsSv;C5BU*_B38UtEnD`|g* z3F1HtjEd_05@`{&f5v9@_W2Mj!HXA3#;NOsqwE0hQuN!BX|ys0HgJD6xkDor>3Qfm z&wp=B=smGKU~-CnWz^}3YxjM$ww>brj7>k-fmGiB*LCLMbTG7^?=Q;^?@4wvX(25m z?t|KulaDu(b6D?IgagF{%M_+H)4&I5K{_PIop}kLK#QQq5rp&pClD=cF2vCXB8wJtwEtGb@CEBjg_sOT5Cao{`h1UHw=@ICGuE+UA#)D_sQ~U@1Sa&; zmxx9X1(PTQ4n>s947EKAeM=(ew^8CbFB16=(f_+5PEtsE&YS0~^_v*o3j6+?K!f!B zhJN~&9Rku5+`k>HOUGAlLMWBby0dV})w6Es@03xn3R{3^5gz6Lgk=PCjK}5|MZ#> zfc3KL8Rh(&EXo;;?Ee2*q0MRwL6bwPGksrRKmaBV&~UnP!hRMQ-BOk^A+ua`Pcf;D z-`SE=!Jaw=qV3<`b4lKbn`2sEU*sCz3X|a=r(Z7RM$|6jUcV%UsNF5e7g8b=ID==i z$Z&Pam-f!})Mv9Ho?^?ejXo>So~jrg2|MSV0x>uHS{l8Ow7;D&PCDK+Zaf&@L^ql% zlrDS+G_035Dg$;{Tl{nA(;7q-aOl4COFbwz161)F2y0pPF@^9dPO1Azb%RHgSEb_W z%^TYzSj&x)7oZ3}6*0%E+n6S3rJePJ(V6^R@dm;bIlH}e_t2k`r z`W>ZlZdyB~JfQPc~s18l%ZtM$Fo<9ZKs&Hh3d9SSCRQDm|sHCImeLi*11ZXKOuZXV;etx zJe{)Lf-qeGn<7cw5}?LW0_W1_lVE$2m7SF|Cz7l(QcK|ZSpR0$M;QA|NbLd%SGnXV z>%@ziMpIAQGeqI7sF_anMh$n(T!efCSbWbr>@U$Mw!Xw5ByQm<&aN#R3~+Oh-LPp1 z8eh&HvNhk^djg-pl^6OIJ zigiy+K77q82yTRBTqef5caWw`dx+`sYTVZ4mt1^e8y0lWH{M>4Zt|Yyn=rNk&0iVB z9VWcx_U|{rFLDogR684OgAzxmk&(^t$ zuOSAo&72)@v z;xMq(X~D2mP?%D~S|liTBBtpBIoyQt%_*7}Oo3@B4%+9X(i}WV58@6672z2 z;9;8GnX0AM)xkeRbt{E1xj)3TJfkO^)ZidrCgc~IdeQaDzC(ksSP`OP}Pg_l(z;h2bKg@#)ucgK3)mjD+ z5}exGxq+q69jM9sTK*Q8gWR5}T)PqJLQS7P@kHqKGiPy`y8yIlA!mlC=ltm7fqKLShQCK1x3~et^di z)b``dLBmyk?Z-*`3{?a4ub_>Ikc zFxITj{=~!k!u)X$dqAM77?5B`j?Mu`uZ$=og2_vzel%7Hu0aK7?`EI<-1>tLRR&hq zsu9Tr>7tDA+&iB_6=kRCZE7@X_?rkUmeSU9XDW>DcZ>X%%z`OlBx>esW@DC{$`Z4p za%(==i;DJ=ekGeRH)TGNvr`#pYjgek$n&!#y!w|19-Q~u!eudb4S>8ot7}L|(p756r^!U&LL_a=R^;i-U%k*jk zVh=V*W0icOO^#ghZ9Nkw9V@b!^M$zX6+zu{73W=2Ur(rWq-sIr{r=O@W_U+Jok?a7 z?q&z}6cBzQ1Xny#Cr}Wfux({w|4CAIsw=1_7lLgEA=rOc4-qbQA3BN%chN$BoQ>M;N!bi*kQ#3-Qb}h}kP^%<@+Es7{_5Q!anxLm@^?#ZT)oeJg=UUsIx5}QoPL2-;Hp#V*Guv5L!obwbLAi-c z_J((Lfk#va4r=l!p_jCT&?3n1>GGeR+~%@6h!~n7=;$1>@gZ^NaN54uSxw05dN5Kq zys*mj~@{)*I1dEB`LIkL-=#{b($%9pt<$c%e6EMVl zth}sP(>aKA@s9u$iU1?7&&(-qqGrl}59lNQJiaTM->6yHSo20^&CtBEhIv|k$we@Q zL5rrvJo%vW$Xp8WN`x2G-Y_DF#q7->(XV!0dl_W5d{4GJE0LGu;qyHMTZDvlNJz(w z=M6qhn83M>SRGrj+2;V~tmzbKB)hop?P$~|Y`feNq18ucAPT=TGyR3q0!6E)IR^a* z=0EL%U<(hFveh0PF6WIiVnaeBO{UK%sljIUCPLFFxe5b(Nf+dp~B6odDU=c(E7~}9QmIwo8oIU zz|Lg-8To*(X@IauidCM`3elQ&w4YR#aSf$JRk;d#fc=6&YL7K zaT)IK4(eO6k>QZ;L4+7d_A-nwe4<4=NgXH9=K70;1?MDxJo`pxHN}Ej)8WNA+p3aebHtvmE4z>dvcURtdl@p0Lk2tqK2_jX&$b;{yF-7e3 ztP%H7qVDDS-L-eJ6M89btel!$h_^X>et9urY=OvAG}(luru(v*A-GkC>QY=-x`bt{ zRtX`Hue@2SRzMwmLz!9NVKS?PUCO8C?DnAJc;eIKjcmqz`yptN0<4sGHcdU-mp%cN zU#Z_r7cG9u5#tYb2U2N1$1A!!7zYTZr@*@}^)qn#pPA%txGaP4kSp2F`K(6O+utY?}cV)&c~H1`E2KO|zwnWy-T_~oL-ip zX!VuJ!{HY)+6^3ARK&@BZ(Y+C{E z=yNjj$02X{^)3X;&X0T`%YKQU$x-GR=_ zL}q}>Yj#!ELAtQXSbgrvuOnv=SDmFTO&fMPR7Da`etGf(vll@r5?{5Zshr5H~njyLg-z_ zpu4OpaRoItrL5pk8cAqV=h6`+X?UF)p(=h{<-eoXxfkM^6`BNmbt|P?KhnL4O9{4D zp0HLU3h9C2EFNPpLa)>)J$$EeQ4-BkmFBA~%(Z0iQS^pqmA)3sahw8F)F|YOi*26X zlb23UtRcCR+!YSDgrOe0Lba&Z$5t;)#wVolZtC5ezC_$B){GfYWR7oIv8S_)@%Fp1 zYKt}-Ajx5{<&MbqCeDutN9+{1>ayRcrJxWjysM~!z+>HxHVMxeX{_VhvQyAL*!eAfqph3WF~P2w$O3B%sqk#0 zBa{s7!E+FsJb)8i?n4O+>ezZW<6S-Ee9PWaw;pL|GaQyueoil6K!%R8th1KC8!U%P z+p-IC-Z*X=A3SnT{X=tb>!C7`HpQen-n?%e?)LW}u|n>DQ`x|y{%Xp5d0axEqqvy9 z&}#Rsl4kUniUE@qn(id3x zEyZ;HrD4vcT$D!l58lumW-50UX(mgl(qF(Kn>Wh>?3-e!2swbEaR$@@FMKGPKhgVg=X@e;8pT9Nmxx1Dk>QpJ4ABK__k-{sfy~h^v%KurC zcwO(GA;$ytc&%z=BB5>jJNC8y7tWBb)NgZa`nP#49moHc&{8k(0s`hg=kZO*0D66~ z>jgBE<#xGFwjP>k4NI1|&O|2CCrj9yuokeY&4&5^ny|zHLrgQ^^a$B2Q7eIS>V3NQ zWZ^NV)8h@+_q<4`Bkpd!S$Q^rhlw9YpX;u5;# zCXa*=DxssYqo+swybY^RzE#W~^~#uS`-7zZ`AJc58igdxh*%U7RwRTfQJ*xkU)F*& z=Z6U?$qzHqoTj2kZBqn1Y+>GU>G_c|tsg9w2f}Ef`S5^w+;U5n#PHsvvM~1$a`CD1 za(-R@=QHo>vF@Fl*~=MvvWqD56%L7%Hyy^&O>T)TalX3hPYtUv-+>y}U-;dlg}~r6 z8xMFVtSI+dNOg$hw;++1Ca6_vTNh}ENA-YtY#*PV1|(>2bo90(?u@0YLug{e_~7rJ z_n((IcGZEZ5M5OsLH!zQ1o8Hw5+}KV1bNKkhi!SIo?@q@sRX+cSq=tmSY-vJUj&}g zN=8>7<4qgvYglV*{i(?6)0?K)Hd<3EQMbgC@r6Mccg=2w;<&j+&_iLAEP@O&qTdKG z;*{ZzKAZm*r|BkEMU~wzc55cm9WJga-YeN6l3NBo=KO6keyS!9>q#2uY0i~LGY60b4o=q zB4o{ln?}|dG`*O@pHJ}fHH3+!vGoI+x?zOI!@`Ot{dU~U4k$rl&mv4{IzWiJU_PQ& zk^(uPU>|TFr$DZ7M#MK1VG%kEb;PR!PM?iq{8I|wUSAqij1^Un;qf_n8f{0hXgqmN zk!s=ZSe&;t0md?UZ92d{d2KnMjx4?L^gwDFXob))X}sSIF>Sopqp#KUl&A22+ffBi z+FXceAi%#mohK+WcCke6|EHO*`MtIb1d!;A+MIR$YgXBQzk6B3t!X5Fa#FR7q7FcZ zK!XT(<=IV<7NR|_16@QTO|o*b;fj`iVrbe04;=Mkq>|^ZVE-cb>cv@v@}N&$?Z zs5Hy8?uA>0NQu`Sv1+le!}@KJ7!MsGt;Nql@P|-*0Z(m*Bgm}aLc>CMUcG00`u>W8 z;UH({>)WHHoksU4g=NE$pvW5i{$%1N;GwVf1*H3S{o1|(Wt10W&;0D-5r29r5KSou z5TL}|kHa2H$t(CL0UiL7Z~7Z(t@0JH0Um!HtM;@B0GvQk6H)g283I1|Ol)F>7-56!BlE|$kRX2y!~<8KtL-DuUuFm|U5RbKA?(cbw;KjMuj}2d zr{Igsw`X)DO>QG`b_qOtp_i1J>``M<1E?T?8IA9@&r0<5kD5q#LEG=hkm4K+B{auw z7j+ar<#-6%d0JvT?wp-*W5Tj91D<)>G?_dt3481l%yH-_`5>C%Eza-+rYm1ONN7c` zeviOf)Nh*c2JC-vk-w+H6nFni6k944OWP&+TRoh_WDDesHu^6R9*>V4urg@xUJ4lu zEFVK-uhxNG^2M_&ayEB~cY`wMK1O_UAcRO|=(8$Q<-|j+ZNaFm*4eTW-97%782m#o zH*YY(4T@;r6G}}VTJb-`nsNWGAY?TTqxoNpL@I};UI?eNR&lQh{$xepn?OP`*p4Su zpiS;CaE#^m5O$Q|Ku3XFY8j8U*Lb}nPcR}_`1j~$FyVf+q+E$=VUG?dz z@yJqfQtYj8z0QV*z*yn01aFs(_0UIXCQ#@VkJQZ)!6%{Ga&9j3oQh!$9LPN;%9RZ9}oS&@4IetO+%rISpUe%BgBM>L6t%I#*3zSh| zTUPlh<5OOdgI>Tt#Sddq`%#SAtnipUlm7APZz_ion<+Tdpw*JYDQJ?N5|FXzVL}Of z2wWE$?MO^dRFvGck|bW4=ktx}PYk7p9lIT9th6JxC(6UKg|vlke;E6*vNpjP!YDlp zQ<+60iE=1Y)qa8D#aihqM-)=u=<%5VQ~F0$sjY2;EvMOd(%@aJsLNH}pKk-E)I*&3 zY?#D36Y1=E3XX}zAcfR`b4z)f;cJD0e3)6&3WfB2{T3%DWzu?kY)|L?1WG8h=*zxw|GT4+Hof?N;?xS(o5HV776(4n9JL?<9^9Q^!6CR@+}+(}c;EZypP8DO zuCCsF*4}%y)T!!Ieb!Al^iBwL^>=8PQbLgia4Iw-s32KEbm`l077>CbQAg-D5Si0P zI&bB*T?aB7xoO6|7jU->cBz3lTq2H@jj2ai!cZBD(96AndN zpUms%pbxdkb3NbYMP{Zwm!bSL9#S#E3^VfO))YSNiIG05qspuj)1PzMrOMOl^OuUWcMH5%x^y~OHkrpRg95l{V_WpKr zIcH3oyVbIx`8)qPJh7P6?9H3Q3vX@?jotZVD0R{&n-=sk?H$Q)zxihLZ7pP<^=J$6 z^H?_m;S*%^`xPh0hV7i+CKy#R!M~-47)SR|g+_G@zlg5N`gz1&-N4Lj-M&R~(dE(= zqDD4<`h11F`AHeQ1r*@*0;??d0R|fi3JL)#TyazQsN;Z6>H`#16&U&h5oif`D2X3> z#&AQNz@V`r+Ur^!n;ByNfDj^v(oeZT{q>f6{=zE;J+S2Q)IBXJO1cfNqWO~zd%k1E zLkj!R?bA2v3gV&Yh(ZB0L+DBg)ofUm$HlGX(AmTmO%)%Tbt*3ourtGu?(jNX_a(;9^h;+RC6hNyVDFQL z%GbpqLFZ-oGas#2E%UBtPtQIV6|2~lwx!MucZLEB?Ca(=R2421j0j?LdLMW|u{(CT z8=o*7l#O%~_Uv=8oyA!S21Hh1k^co2nzmzNlZudl#emWjn(NrW89N_GJf zK;9ZHzsGmcr%mONT)6Y#Az^xUSd_LC1Bm&;)sVD}(@+tu%2^ZtK{RHa;n}VX0c&!6M_oX@>ZukR&pSZ8)T(E+q&RVR~qQSj!M5{zMm0_!T8# zu^8K^W=l9@C|sybuX!CXwEWJyF2UE7nYz%?-DPa~4ZJJR3yHk*DT1fc!L6nozNcUV z+OCIplJT-({;CAk7au80xF3qG(|9AL)U8U`)6wXO`OA6l0tRcX_mRGetKrCxs#nL6 z17H22OL5^2k;U;?VbTFF_%nmv*1ELC&#Gm4l-fzpf5`w8hr({N>Zds0$pAvNJ)NEl zQq!w1hzSIBesH+3NLX`~W~Zo7P&R_#3}jM}-kR%DGe*Ff{2P4i;(;1dT3!Hm<%uFA zRlD1%3s>aXqnDUkbuTj2q4aRnS=Ctplmv5iU~&gLdd)NR)7Fv!qwsL^!yd$e)k{+& zGo@|Gguj&tZHAda>jCJ`_x!N&(l_g*%yj@v#@$g|K{HYpc-~vdesgnWEMvKmEU^R$ z6L{R5I8C!Z9nqH7#V!`z6fc(%$MPr7aysloY>8)|WD6OX>Y4H`c6JySiyna1?D|@3 zdHH@m!)y(Vbv>FV1o;K35{n6MH`Hqo8V4h-wE{TepCc<)3mmbB9$^4V*jxPh(x<|?U$ z%9|iq+AOJ39iqdcW#(yW;?9Zdh>9GN>vmEtEcLX~UnN5KS2`-r@l^kJNgK3dI&9RR&Z?4brrH zH2qq*_-LTlZ}pYHUGrfN41ea%^rDWcc}ORKJ(|b(3a&~UJvkpe5AmmuUn}uzw;60V zFG#ku5Fyxq>*kb-MTcLyE_?O1FO<^KLVV&PGUwH=a3PLU^jnUa9@l)f{o0y_m8=z# zq!|&A6;dgAaqpfh!yqoCmkS!2wGQaVWYQXqd-pIq_y|gU#`;X`k3F=apg{s(A4sds z`+aamu(~AoRbK%-Gb7(1W3~t&5*6zh*i(|1}tx5s8X^#%|JiodPlY}t&UoI z2lP{NMm-Kn*~$=^TCUT9e_bN+$qTnyARJ>5_t^NB%1QhdNsB271`k`4p}wsS6vn-0 z_x{;d#S2ylowaL*OLTNJJU~)m6f7pnS6Zp$_xA>IktqAog;QpG6FU2O=rj917OAU$ z5a$3*_*7Cfyx$)g5zsAyXt3Zr1C6z-pGy&Ai&q!TBEBJ=2LbA1{{T@BGJ&hVtM3QA z_z$DxMLj=gOap^~I>D7jy17{FO=8+eo5J&^hb>jTs68g=59{+b0OSbX8p6Bwu9?n4 zL45veD;*%Kcjv9P_(Bc+><5+v~LL}LAs2d|D za;%5x0I*MoAV@_l`?k4X1{rdn}>w%T;W|iLi>padwe)>OWGalf@@Eu7UA(zm5Kj>ZF*;G}HxE1TyhQ zpb~V^$fUm&JjG${rthkG@pYFVs8Yo@prZ5Qlu3U(IC7c{mo+vin%_M#-z1X93X8AE zU>ygKb@{*vWmzxyk3`J7FEn!6>Bo;SW$c0dov+=2J(U+C2Wbn$jF4qPs}l6q-8JI= zsuQ$1Xs7Mk4O{{M5X zy6MgSz`yUy&KG=HbXj-MZ??XLUd1oSw?s2)!fHC&@NQ=~&dJ|DrViVBRe1*pO$szu&y<2h@G-IQrd89KMT2E z0c}#@F0-ytf9?^VGt4QBH_Y=i)YP-fKb2Ou+zCKxfj9cyEA<*9K1N!*_o+^((}uwg z?T%$X+*f&iDM~P5PV;z!3`QSYZ7W<+>pJM|kM>=&eqYT|SETP+>zo~xHjHtE_$UVL z?$nGuCD#LaN#dX7)rlb=>rcE3RH$A0%~fRZmf0i<57w+!mCUt*3UiG(G)I-5+tHjL z4wjYfOy+voZ46^mU(m~&BRe6`0e!8kLdr ziQUK9%zD%Fz=*YBqN;IaNkN9!+RW+YoCqNn*Ow#)TXTfQo@m!xWmztlT8 z2TjC!y~m7(7@+RITgKggFL=Ab{=xA3Pso<59R%k*pd!*Cfdxm>*>;L=$4B8Voo0_1 zb5>Lt{`{Pe`*!@sR0;n4yRgGa9O4Yg1nAm`=AHgwmy(PLFSp$T448ufPxYHPh~^+7 zdpy_M3FAq}Y-B&pd6_j_#47)cJpYoJW}(4HuVcE@+xcVn_bkc`e-EFAmJo>G*bhVI zjfQy83xomF@JWX3i2z;vAKHnXXl-NP%5zt&gzn%t)mEa_c|QjQq?nUYG5%Gy}(MZxX!BM>f4$BdZ*ud zr~mq$;5j_0G$`1t#hjFKi!OP``?lBs0V(@o3+K534(yDB1HvPlcb%|MVmRi1YFt=6 zhDFE&z^Fh66+?-r2a?eXf5ah+aG4Ol8wUOt%|DY|+Vs9XS+&{wZ}7jl{}Y=P%}r*u=pzFV;VC zDkE8F0(%QrV3mRY?@(w@4^2czUu2HGFez{UpB49=7(B7CJPiwj4o1hu0sXTF1h>1I z!4GYZt>tOs3X8O^TbqJ1tNL3*_OeQq_iPK7(TIjWx)jcUp=7S`*W-8n6Y(xbTDkNvoZg;S1@Ha*L0rMy5qv+U0 zZVKdf-JTj6nz+`$iurC}Fy>l|%2Qjmro0ix!Wd%UX@xzlr$!>^tmAyVh}%gbw{=Xb zspqgKD(+|5?^QfHi`w?6P(6_Y!BI|GqfYw$&6!F~_v@GNzP9-6HG>%Er82sayrH`{ z@JIi(>6Y16(Oo{R{l8W%s?!bCFLG*{3zB@b^mV*>au(lhhyy2!7H~l&B~i%kB3h%& z#W3m5rbfx1s5Sv0)>hhKY|p7`T>a^5)ruws8fFthXKiph_4u_!g*{kPfmo!lRI+nc zu+S@h86$68^nlm#`_$o7+GcCiR8eG_EM4o723Kj+ZJ$Zz*RPRgv=PtCGuG~ef0 zzw0z=v`iab354n7#gq%{biuOBxL>dAahG39;}3ik8V>-VY~3=0>x>MtR_UVL=<>mQ zb2k|y(Z?|6zZ3gS^)9j>WS)gS-R`7%%rH~CHGX8xb+bo{=Ov|HSE{e~KF%M=^W)zM z+X*oZCG1W(rq6v$&`&JhXkGaBu_#+@B68WlN1h z`hR_7dEyHL-E*G*?(d5yB^C53~a~JQ-5Z z0ABUrMh~0)&D>xU8)@L$I+BP>D%PZH=o=)YF1<1Y{j7p_M^hUt@toVq9d=W?^afGT z>^^#(ANDupQ;02zkqi#m^ycFl*<8o?jifRoSdBXFwpOoFV>hlwCyPFnUQ2Eu<8<8b zEpt7#ub`wzw8+U z%+4p41d=*>f>}63g^lW{2P7ycAMhjt6@X{}e9s^Z8gkz}=#BjKnInMjY+PWkV_^vr z?-S>;bQ#vTR8{hzprBD6uyiv25}fw9^0F%_p%K=xaF!ak)MR^D!=YYv=6A!eB=$C6 z10>P?Q3VzKT3tl)_-f3MLfD92k;Sv@2Cqqa>Sj=7_2_g+ydCyo{41ZB=!~mNYf-X` z!1QSuD5giVDlrNqk`|aEjyg!6Z53Sf|TSIKjiMyxO zs!70e-{VEWIN7z$pv_^_I+bOI|EN*nx0;4$<5Rw2-5z>!Bj4c@$7}>M-SOBFadWB` zj-^}ANgzSfcSl-ChxbXEhn#mT(VOvyZ2y-EP~uT%V|Tzl)>>;}af>qcZkf`Rh4|ak zB?e!YsU^^)gpF$fyuXsdR6feg4ig6t96$X{iv{P1Df7IfvRX1jeQvTzY` z^K5JbX2a`L34b3-&OF&r`o2{3VU$vdV#b3QqUI;eC;rp$AYC9@fK~8iG};P%UoE@{ z#1$g>2LTzbMKWFb+5Bl=ZOr4*RAPtk2j`|w*?;(JzVTH>SX>(B{q+Oh8PNf7% z1Dfr5Dn7M@u#8XU*Hm)p!Na0;6O1Vlu6T{DT8M;SnYs`$w58E$>Hz6n!)QK?B$IGW`x(q`{N&dMjN-pgQ@l(YZ>^KzBpYmRjq|A$vXW8@dp~Zk@lyC zc*ZF+`#pBC#b5DO$POWdRhoH5;KHLW<|0jP04>IgxgwS|oqA3^%ReK-xl%0}jgyCh z>G3{C7Zmmnn{yCoEQ0jE8Teqmf;6{!2qi9?2y|kn4C`v%ZD7phq4~*dna$RhLAiR= zLWFMVn9U_~UCAXzt+H9!4)qfCQc3bop9hHqBPR7yD&m=msj$QH7ub(R%(x7Fei~et zHopReU!(pqmL8$9C3XJz^7+9Ik?ka}a4k*F4(-1YSDu|TJP>8-#N)VHYp$Z-p?(!VLe;y<+Sw30v=7f^Fjok27^OnId#F=9x z87wY%lvkpgX^snWQ+9Unk$u*N+?nvWTD6&_!vojc*1zDXhIEEtYPy?1Hx)gGS3?mp zTf%w6!uYt5}BMr8eder28Hl}-9wcSL*gqT zhKML3MxTVkar5W-=8qfBmbHeKJECbie0WLHj0Vym@XQ>nf+}-(CFYb&fV4%`6muby zg!Ngp11?GSFUbTTxpx!OK#R=O@m;G59G5hb&|zLIarI6Z<0H8J^Y%we!CPS2Nc#7z3bRMc8wp_cbs1l#`#a2>7=eJjNeo=Dsds$Q9N>;|(Ej;Ydr;^-OYqMEo-cFFLf37SBf6jVu8vX)LM1CUN#v8%v2N|&~ z`Z)F0>G@XwSf~)dPkC>oM?~LKK~Y}hZ4K~t|4Hk(T?F@x9NH%mS5)=Pyv#hZ3A1NB zlZDsvxHbpOB(v0iwq^goL0p=UrXW!R>*Ng@7L}{v zzFZ(@U3TSu=&EpCh_?w_cf5dWwIgx2B5dQdnI$RoC0-gyP?@NSc_K(!h0_+9Atko9 zZ`gvRo{$}LQ^1PVm%QrW-XIJDdmZ#DH;=aS+789CZ}E3e!l)o;2xSUyk&zOJ6P1)H zYWSzTWb2pGI7#Q|Rq_&5i^WEx0@E&+g5NQW$_#^6*k65z>gWKJ&?2OC+@7MF|9bd<6g=z39b5nwU ziCI*@Qf{EGX~RVt{G@Jrt27{id800mjP|CThBWC41FH zqUa)dRN7`FjByIOqHbh+!{X3dTo(qaifV|DaK7PT?2P85;I(eUcxfVAeMqL={p?;MOVfm;wok<;ZPOmfo$nb*r}tz_L)ZCTW-2@eh_Rc0vz)GJ-5tER z%TMP>CIzK;xWGEFdmSn~H6E$qz9XF?hBm>ccF9CNy>zFr=uJx;g#w6wNC zX1M5e+iOj*%Qqz(wAWT>31S~lfpw(^x@uciP~4X)Qx*BYFKkI)XG&Y>pSbDXuUcX& zazA9l-vG=$>^NT9ao}Wrxrw~xQBHqRiXBAFk1slXCjCGE^hrAsG}^;MLH)u5Ba#z> zwDnxEq;Lb)N?*{KFH#R}MmV_(VpQYI=+&W}iJo+3(`;f1#^uL7?zC~0!uk?mtQg7> zC4yRJPjQc_I7=vD{K!F)pNEtlkbPUF%%T&`w}rh1qxnw{Tn}6y#vp7LuSsX9OseFc z&=q$s=bV5MaGajn>siC2~4Bok^a1j#LT$e`Ehkngq#<{!N2wN&W+ z(9LYLij@lb#0a|5zv2n1oL4#3pukeal`EI}p~s`3XLkvjhyVw7RdTvc@lzuKwDRKNoim za#Nd-cw=`Hi3V)bg>`AA%SK+~Zae3}Yihp8J1Azet~R&FL*PZ^9qfEHs|SfB8PlKU zMKC2X%4+ zz+PQA+lEq1h|vL}B7sCS;>saNyRRw&fJusiKP;NuMhW3QgL7>X9nqd0aayml z&$qHuEuE+3b4?_Db}83odrE{yZzX51fQt0lq}?0?D`vIiVsq9w8R?WS=r~sk5@aSa z2saef#V>u+i3MW|L$0LevmTn$lnAq)z=5)+O-GWu${emBR5y%G9TW}$O#d&UP3&&f z?Oorre+U?I@IV7WGzv=V;DLeoc+~9P3Fa&`A=+u4S^)s+D4<9g2k@QU2f<)G2nh#8 zx>zA`tX#ZnGr<@L>ygBqO7NVzGZr=m$2Iyhj1Mm^KQ$8yMp=xirk^OHE3^{VQQ)EF zV#U^PPvU4}>pmL3VrAtqx~(D`*r#WQXf;Yeuxx~E%{~RngeI@`R<^9v_}r1Ak?OO` z%ob}4A4D8Kcnz5>@TZ2sgYroPWRZ681zyWL9D-%^ZwCPFD<;TZCp3}`zOcL9L3M8pTy4?xOF0L4y`MDm z)|zhTUT&ax$3QHB0d$+;xFG4~Z@@9dv_Lr$mhM>;W4+(?XdKv>vxZZu5+!+V`$QPxZM>Ob)_w90}Ux<#Q7ngu0usTlkP3aJIQk5aLIhMb zfB*!g2*scoG*w@Jw{UKP!gFNg|3o?ppK`clms$Q2sEl?RFg{0x_f7urmahPm92igi zg_T&;pG4cTX%x1hv`fw#$BS`RfIj!}+Hv(cF4ahm$$evHb;Wn|=INqeAZNho1pe#v z^Pc8Q-_0D$(dZk{_QdhEV2CW#{9`&USQB)>D(1d$=nH*kD|l^A*p&0S=ow@9%f=DN z|9i3duHK(+sHnPXb?BibUNGzBI@#x_?WkfWdG)%q>arw$e0((P!8ic;5xIhtlv9f< z4rp%a0(^X4hNAIm#0A}+i7BnS09#1ncI^oQzN4CS=1p#dtE`LQa0nVjclu!(8Gs_W zvaI@ye#se@W=cqwsbh~M?%nhrV?P{6tqVr`AuGaa4WiX6tl=A8C?cU+THj9?<-zsV zF}4C>BxBGHkis-!HbGS#jDyv%mJ6XNv$Oh6kpJua}CvhS~p$9Z=-$LR)@XHy@YyCtu)_fzB zKIy*UE#trGkJqmHm=Ue*`V{p__PfF3-k(%ZeCbg1QfF~H%~li{PgJ1OC;m-`Dam@J z?LooLweCh*DF}w+iAR@NN3gG-K-M=ux`bd9(X`OFHZyWV%y;AM-&47+kM03F5eB#J zhSLffcIP|~{{(EpO#>RY?fHVze-_*xJn8Q|70W7_Y1p3wsqlp|V*pv3!prHUH)Rjz z>g2ek=%ZwZbe_?SEUmPh&7f1;ntf3==b=Sme&n2i>p(k0XUU0hIB$%GMvalKW;i=* zB%Rm!AvaV1fzBDxJjaB@uz?z4>_`{=0q#JZ_2!2}kuLfiv4efA4|s2!_vI$gWuxSF zRe14Ux1$kYme zf+Bdg{}gB_viG)~6Wk_zLiE4Bc%Y#EQ9S)q{tc*M0QZVe6aDvvp9U1vyF}wZ<==o< zGBBJd51{xHSVWWqF#ZXwM1TX16{QC#v4LAeJpr3s;4U6qu(TLG0Gs>0O9lQZ_VIt$ Pden*W!(H?L!}vb{h$=q7 diff --git a/profiler/advisor/README.md b/profiler/advisor/README.md index 770271105..04dd0b843 100644 --- a/profiler/advisor/README.md +++ b/profiler/advisor/README.md @@ -62,19 +62,21 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 #### 命令功能介绍 -| dimension | mode | 参数释义 | -| ---------- | -------------------------- | ---------------------------------------- | -| overall | overall_summary | 计算、通信、空闲等维度对性能数据进行拆解 | -| cluster | slow_rank | 慢卡识别 | -| | slow_link | 慢链路识别 | -| computing | aicpu | AI CPU调优 | -| | dynamic_shape_analysis | 识别动态Shape算子 | -| | block_dim_analysis | block dim算子调优 | -| | operator_no_bound_analysis | operator no bound | -| | graph | 融合算子图调优 | -| | freq_analysis | AI Core算子降频分析 | -| scheduling | timeline_fusion_ops | 亲和API替换调优 | -| | timeline_op_dispatch | 识别算子下发问题(路径3/路径5) | +| dimension | mode | 参数释义 | +| ---------- |---------------------------------------| ------------------------------------ | +| overall | overall_summary | 计算、通信、空闲等维度对性能数据进行拆解 | +| cluster | slow_rank | 慢卡识别 | +| | slow_link | 慢链路识别 | +| | communication_retransmission_analysis |通信重传检测 | +| computing | aicpu | AI CPU调优 | +| | dynamic_shape_analysis | 识别动态Shape算子 | +| | block_dim_analysis | block dim算子调优 | +| | operator_no_bound_analysis | operator no bound | +| | graph | 融合算子图调优 | +| | freq_analysis | AI Core算子降频分析 | +|communication| packet_analysis |通信小包检测 | +| scheduling | timeline_fusion_ops | 亲和API替换调优 | +| | timeline_op_dispatch | 识别算子下发问题(路径3/路径5) | - all @@ -126,11 +128,14 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 ![输入图片说明](./img/cluster.png) -cluster模块的分析包含快慢卡和快慢链路分析，仅识别问题，不提供调优建议。 +cluster模块的分析 +1. 包含快慢卡和快慢链路分析，仅识别问题，不提供调优建议。 +2. 通信重传检测分析，识别发生重传的通信域并提供调优建议。 如下图示例，识别到当前训练任务的通信和下发（free较多说明存在任务下发存在问题）存在问题。 ![cluster_1](./img/cluster_1.png) - +如下图所示，识别到当前训练任务存在通信重传问题，并提供调优建议 +![cluster_2](./img/cluster_2.png) overall模块的分析包含当前训练任务慢卡的性能拆解，按照计算、通信和下发三个维度进行耗时的统计，可以基于该分析识别到训练性能瓶颈是计算、通信还是下发问题，同样不提供调优建议。 ![输入图片说明](./img/overall_0.png) @@ -159,6 +164,9 @@ computation模块从device计算性能维度进行分析，能够识别AI CPU、 ![computation_1](./img/computation_1.png) +communication模块从通信维度进行分析，目前支持通信小算子检测。 +![communication](./img/communication.png) + ## 工具使用（Jupyter Notebook方式） Jupyter Notebook使用方式如下： diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index ada1b0bf4..80368e1d6 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -81,7 +81,11 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): for dataset_cls in dataset_cls_list: if dataset_cls and callable(dataset_cls): - dataset = dataset_cls(collection_path=self.collection_path, data=self.dataset_list, **self.kwargs) + try: + dataset = dataset_cls(collection_path=self.collection_path, data=self.dataset_list, **self.kwargs) + except Exception as e: + logger.error(e) + continue key = dataset_cls.get_key() if key not in self.dataset_list: self.dataset_list[key] = [] diff --git a/profiler/advisor/analyzer/cluster/Communication_retransmission_analyzer.py b/profiler/advisor/analyzer/cluster/Communication_retransmission_analyzer.py new file mode 100644 index 000000000..3683ef1b4 --- /dev/null +++ b/profiler/advisor/analyzer/cluster/Communication_retransmission_analyzer.py @@ -0,0 +1,46 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.cluster.Communication_retransmission_checker import CommunicationRetransmissionChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset + +logger = logging.getLogger() + + +class RDMARetransmissionAnalyzer(BaseAnalyzer): + dataset_cls_list = [ClusterCommunicationDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = ClusterCommunicationDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseAnalyzer.check_data((ClusterCommunicationDataset.get_key(),)) + def optimize(self, **kwargs): + add_render_list = kwargs.get("add_render_list", True) + rdma_checker = CommunicationRetransmissionChecker(**kwargs) + rdma_checker.check_retransmission(self.dataset) + if not rdma_checker.rdma_issues: + return self.result + rdma_checker.make_record(self.result) + self.html = rdma_checker.make_render(self.html_render, add_render_list) + return self.result diff --git a/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py b/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py new file mode 100644 index 000000000..cc0f688e8 --- /dev/null +++ b/profiler/advisor/analyzer/cluster/Communication_retransmission_checker.py @@ -0,0 +1,128 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from typing import Dict, List +from collections import defaultdict +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo + +logger = logging.getLogger() + + +class GroupStatistic: + def __init__(self, min_transmission_time): + self.retransmission_issue = False + self.abnormal_op_dict: Dict[str, List] = dict() + + def add_op(self, op_name: str, hccl_info: HcclInfo): + if self.abnormal_op_dict.get(op_name) is None: + self.abnormal_op_dict.setdefault(op_name, []) + self.abnormal_op_dict.get(op_name).append([hccl_info.group, op_name, hccl_info.step, hccl_info.rank, + hccl_info.get_rdma_transit_size(), + hccl_info.get_rdma_transmit_time(), hccl_info.get_rdma_bandwidth()]) + + +class CommunicationRetransmissionChecker: + def __init__(self, **kwargs): + self.rdma_issues = False + self.desc = "" + self.sdma_desc = "" + self.rdma_desc = "" + self.suggestions = [] + self.abnormal_group_count = 0 + self.abnormal_rdma_list = [] + self.step_id = kwargs.get("step") + self.stage = None + self.group_statistics = defaultdict(GroupStatistic) + self.headers = ["Communication group", "Op name", "Step id", "Rank id", "RDMA transmit size(MB)", + "RDMA transmit time(ms)", "RDMA bandwidth"] + self._init_rule() + + def check_possible_retransmission_occurrence(self, hccl_list: List[HcclInfo]): + min_elapse_time = min(hccl.elapse_time for hccl in hccl_list) + max_transit_time = max(hccl.rdma_info.get('Transit Time(ms)', 0) for hccl in hccl_list) + if min_elapse_time < self.min_retransmission_time: # 检测是否是卡间不同步问题，而不是重传 + return False + return max_transit_time > self.min_retransmission_time + + def check_retransmission(self, hccl_dataset: ClusterCommunicationDataset): + """ + :Param event_dataset: dataset of timeline event + """ + for group_name, hccl_group_dict in hccl_dataset.hccl_dict.items(): + for op_name, hccl_op_dict in hccl_group_dict.items(): + for step_id, hccl_list in hccl_op_dict.items(): + if self.step_id and step_id != self.step_id: # 传输指定step（self.step_id）情况下，非目标step跳过 + continue + if not self.check_possible_retransmission_occurrence(hccl_list): + continue + self.rdma_issues = True + if self.group_statistics.get(group_name) is None: + self.group_statistics.setdefault(group_name, GroupStatistic(self.min_retransmission_time)) + self.abnormal_group_count += 1 + for hccl_info in hccl_list: + if hccl_info.rdma_info.get('Transit Size(MB)', 0): + transit_time = hccl_info.rdma_info.get('Transit Time(ms)', 0) + if transit_time > self.min_retransmission_time: + self.group_statistics.get(group_name).add_op(op_name, hccl_info) + if self.rdma_issues: + self.desc = self.desc.format(group_count=self.abnormal_group_count) + for _, group_statistic in self.group_statistics.items(): + for _, op_list in group_statistic.abnormal_op_dict.items(): + for op in op_list: + self.abnormal_rdma_list.append(op) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("Communication retransmission analysis", self.desc, self.suggestions) + result.add(OptimizeRecord(optimization_item)) + + sub_table_name = "Comm Retransmission Analysis" if not self.stage else f"Stage-{self.stage}: Comm Retransmission Analysis" + result.add_detail(sub_table_name, headers=self.headers) + + for row in self.abnormal_rdma_list: + result.add_detail(sub_table_name, detail=row) + + def make_render(self, html_render, add_render_list=True): + return html_render.render_template(key="cluster", + template_dir="templates", + template_name="communication_retransmission_analysis.html", + desc=self.desc, + solutions=self.solutions, + headers=self.headers, + data=self.abnormal_rdma_list + ) + + def _init_rule(self): + syncbn_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "rdma_analysis.yaml" + ) + + syncbn_rule = FileManager.read_yaml_file(syncbn_rule_path) + self.desc = syncbn_rule.get("problem") + self.min_retransmission_time = syncbn_rule.get("min_retransmission_time") + + self.solutions = syncbn_rule.get("solutions") + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/analyzer/communication/packet_analyzer.py b/profiler/advisor/analyzer/communication/packet_analyzer.py new file mode 100644 index 000000000..73e5bc2bc --- /dev/null +++ b/profiler/advisor/analyzer/communication/packet_analyzer.py @@ -0,0 +1,46 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.communication.packet_checker import PacketChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset + +logger = logging.getLogger() + + +class PacketAnalyzer(BaseAnalyzer): + dataset_cls_list = [CommunicationDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = CommunicationDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseAnalyzer.check_data((CommunicationDataset.get_key(),)) + def optimize(self, **kwargs): + add_render_list = kwargs.get("add_render_list", True) + packet_checker = PacketChecker(**kwargs) + packet_checker.check_packet(self.dataset) + if not packet_checker.packet_issues: + return self.result + packet_checker.make_record(self.result) + self.html = packet_checker.make_render(self.html_render, add_render_list) + return self.result diff --git a/profiler/advisor/analyzer/communication/packet_checker.py b/profiler/advisor/analyzer/communication/packet_checker.py new file mode 100644 index 000000000..3d9ac81ff --- /dev/null +++ b/profiler/advisor/analyzer/communication/packet_checker.py @@ -0,0 +1,148 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.utils.utils import convert_to_float + +logger = logging.getLogger() + + +class Statistic: + def __init__(self, min_ratio, min_size, desc, type_): + self.issue = False + self.count = 0 + self.abnormal_count = 0 + self.abnormal_duration = 0 + self.abnormal_ratio = 0 + self.min_ratio = min_ratio + self.min_size = min_size + self.desc = desc + self.type = type_ + + def check_threshold(self): + if self.count and self.abnormal_count: + self.abnormal_ratio = self.abnormal_count / self.count + if self.abnormal_ratio > self.min_ratio: + self.issue = True + return self.issue + + def process(self, hccl_info): + info = dict() + if self.type == "SDMA": + info = hccl_info.sdma_info + elif self.type == "RDMA": + info = hccl_info.rdma_info + if info.get('Transit Size(MB)', 0): + packet_size = info.get('Transit Size(MB)', 0) + if packet_size < self.min_size: + self.abnormal_count += 1 + self.abnormal_duration += info.get('Transit Time(ms)', 0) + self.count += 1 + + def adapt(self, dst_headers: list, src_headers, datas: list): + if not self.issue: + return False + dst_headers.extend(src_headers) + datas.extend([self.count, self.abnormal_count, self.abnormal_ratio, self.abnormal_duration]) + self.desc = self.desc.format( + abnormal_sdma_ratio=f"{round(self.abnormal_ratio, 4):.2%}", + min_sdma_size=self.min_size, + abnormal_sdma_time=round(self.abnormal_duration, 4)) + return True + + +class PacketChecker: + def __init__(self, **kwargs): + self.packet_issues = False + self.desc = "" + self.sdma_desc = "" + self.rdma_desc = "" + self.suggestions = [] + self.min_sdma_size = 0 + self.min_rdma_size = 0 + self.min_sdma_ratio = 0 + self.min_rdma_ratio = 0 + self.step_id = kwargs.get("step") + self.stage = None + self.packet_issues = False + self._init_rule() + self.sdma_statistic = Statistic(self.min_sdma_ratio, self.min_sdma_size, self.sdma_desc, "SDMA") + self.rdma_statistic = Statistic(self.min_rdma_ratio, self.min_rdma_size, self.rdma_desc, "RDMA") + self.small_packet_detail = [] + self.headers = [] + self.sdma_headers = ["SDMA total count", "Small SDMA count", "Small SDMA ratio", "Small SDMA duration(ms)"] + self.rdma_headers = ["RDMA total count", "Small RDMA count", "Small RDMA ratio", "Small RDMA duration(ms)"] + + def check_packet(self, hccl_dataset: CommunicationDataset): + for step_id, hccl_list in hccl_dataset.hccl_dict.items(): + if self.step_id and step_id != self.step_id: + continue + for hccl_info in hccl_list: + self.sdma_statistic.process(hccl_info) + self.rdma_statistic.process(hccl_info) + self.sdma_statistic.check_threshold() + self.rdma_statistic.check_threshold() + if self.sdma_statistic.adapt(self.headers, self.sdma_headers, self.small_packet_detail): + self.packet_issues = True + self.desc += self.sdma_statistic.desc + if self.rdma_statistic.adapt(self.headers, self.rdma_headers, self.small_packet_detail): + self.packet_issues = True + self.desc += self.rdma_statistic.desc + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("Packet analysis", self.desc, self.suggestions) + result.add(OptimizeRecord(optimization_item)) + + sub_table_name = "Packet Analysis" if not self.stage else f"Stage-{self.stage}: Packet Analysis" + result.add_detail(sub_table_name, headers=self.headers) + result.add_detail(sub_table_name, detail=self.small_packet_detail) + + def make_render(self, html_render, add_render_list=True): + return html_render.render_template(key="communication", + template_dir="templates", + template_name="packet_analysis.html", + desc=self.desc, + solutions=self.solutions, + headers=self.headers, + data=self.small_packet_detail + ) + + def _init_rule(self): + syncbn_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "packet.yaml" + ) + + syncbn_rule = FileManager.read_yaml_file(syncbn_rule_path) + self.desc = syncbn_rule.get("problem") + self.sdma_desc = syncbn_rule.get("sdma_problem") + self.rdma_desc = syncbn_rule.get("rdma_problem") + self.min_sdma_size = convert_to_float(syncbn_rule.get("min_sdma_size")) + self.min_rdma_size = convert_to_float(syncbn_rule.get("min_rdma_size")) + self.min_sdma_ratio = convert_to_float(syncbn_rule.get("min_sdma_ratio")) + self.min_rdma_ratio = convert_to_float(syncbn_rule.get("min_rdma_ratio")) + + self.solutions = syncbn_rule.get("solutions") + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index 5ea4dbd75..7afa09cca 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -49,7 +49,7 @@ class AICoreFreqChecker: max_freq = max(self.DEFAULT_FREQ, convert_to_float(Config().get_config("aic_frequency"))) decrease_freq_ratio = sum(max_freq - freq for freq in freq_list) / (max_freq * len(freq_list)) - if decrease_freq_ratio >= self.DECREASE_FREQ_RATIO: + if decrease_freq_ratio >= Config().get_config("frequency_threshold"): self.ai_core_freq_issues = True self.decrease_freq_ops.append([op_name, op_count, op_total_duration, f"{round(decrease_freq_ratio, 4):.2%}", diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 52e3e0755..3d20374d4 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. class SupportedScopes: # used for specify fourth-level commands and define the key of the result dict @@ -6,6 +20,8 @@ class SupportedScopes: GRAPH = "graph" SLOW_RANK = "slow_rank" SLOW_LINK = "slow_link" + COMMUNICATION_RETRANSMISSION_DETECTION = "communication_retransmission_analysis" + PACKET = "packet_analysis" OVER_ALL = "over_all" DYNAMIC_SHAPE_ANALYSIS = "dynamic_shape_analysis" AICPU_ANALYSIS = "aicpu_analysis" diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index 06186080d..cdc0dd4e5 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -123,6 +123,20 @@ MAX_RETRIES = 3 TIMEOUT = 3 ADVISOR_RULE_PATH = "ADVISOR_RULE_PATH" +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. CLOUD_RULE_PATH = "rules/cloud/" DEFAULT_RULE_PATH = "./rules/" @@ -137,6 +151,7 @@ CLUSTER_ANALYSIS_OUTPUT = "cluster_analysis_output" KERNEL_DETAILS_CSV = "kernel_details.csv" CLUSTER_STEP_TIME_CSV = "cluster_step_trace_time.csv" CLUSTER_COMM_JSON = "cluster_communication.json" +COMMUNICATION_JSON = "communication.json" BOTTLENECK = "bottleneck" DATA = "data" diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index e1163f1cd..b4956139c 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -1,3 +1,17 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import logging import os @@ -10,6 +24,7 @@ from profiler.cluster_analyse.common_func.constant import Constant from collections import defaultdict from profiler.cluster_analyse.cluster_analysis import Interface from profiler.advisor.dataset.cluster.cluster_step_trace_time_bean import ClusterStepTraceTimeBean +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo logger = logging.getLogger() @@ -114,6 +129,7 @@ class ClusterCommunicationDataset(ClusterDataset): self.SDMA_TIME_MS: 0, self.SDMA_SIZE_MB: 0, }) + self.hccl_dict = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) super().__init__(collection_path, data) @staticmethod @@ -136,9 +152,26 @@ class ClusterCommunicationDataset(ClusterDataset): def process(self, communication_json: dict): for comm_group, group_dict in communication_json.items(): + if self.hccl_dict.get(comm_group) is None: + self.hccl_dict.setdefault(comm_group, defaultdict(lambda: defaultdict(list))) for step, step_dict in group_dict.items(): for op, op_dict in step_dict.items(): self.compute_bandwidth(op_dict) + self.process_hccl_info(comm_group, step, op, op_dict) + + def process_hccl_info(self, group, step, op, op_dict): + op_name = op.split("@")[0] + for rank_id, rank_dict in op_dict.items(): + try: + hccl_info = HcclInfo(group, step, rank_id, op, rank_dict) + if self.hccl_dict[group].get(op_name) is None: + self.hccl_dict[group].setdefault(op_name, defaultdict(list)) + if self.hccl_dict[group][op_name].get(step) is None: + self.hccl_dict[group][op_name].setdefault(step, list()) + self.hccl_dict[group][op_name][step].append(hccl_info) + except ValueError as e: + msg = "[ERROR] Cluster_communication.json has invalid structure." + raise ValueError(msg) from e def compute_bandwidth(self, op_dict: dict): for rank_id, rank_dict in op_dict.items(): diff --git a/profiler/advisor/dataset/cluster/hccl_collection.py b/profiler/advisor/dataset/cluster/hccl_collection.py new file mode 100644 index 000000000..a9fa536ef --- /dev/null +++ b/profiler/advisor/dataset/cluster/hccl_collection.py @@ -0,0 +1,78 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +hccl info +""" +import logging + +logger = logging.getLogger() + + +class HcclInfo(): + def __init__(self, group: str, step: str, rank: str, op: str, rank_dict: dict) -> None: + self._group = group + self._step = step + self._rank = rank + self._name = op.split("@")[0] + self._elapse_time = self.get_elapse_time(rank_dict, "Elapse Time(ms)") + self._sdma_info = self.get_communication_info(rank_dict, "SDMA") + self._rdma_info = self.get_communication_info(rank_dict, "RDMA") + + @property + def group(self): + return self._group + + @property + def step(self): + return self._step + + @property + def rank(self): + return self._rank + + @property + def name(self): + return self._name + + @property + def rdma_info(self): + return self._rdma_info + + @property + def sdma_info(self): + return self._sdma_info + + @property + def elapse_time(self): + return self._elapse_time + + @staticmethod + def get_communication_info(rank_dict: dict, name: str): + communication_bandwidth_info = rank_dict.get('Communication Bandwidth Info', dict()) + return communication_bandwidth_info.get(name, dict()) + + @staticmethod + def get_elapse_time(rank_dict: dict, name: str): + communication_time_info = rank_dict.get('Communication Time Info', dict()) + return communication_time_info.get(name, "") + + def get_rdma_transmit_time(self): + return self.rdma_info.get('Transit Time(ms)', 0) + + def get_rdma_transit_size(self): + return self.rdma_info.get('Transit Size(MB)', 0) + + def get_rdma_bandwidth(self): + return self.rdma_info.get('Bandwidth(GB/s)', 0) diff --git a/profiler/advisor/dataset/communication/__init__.py b/profiler/advisor/dataset/communication/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/dataset/communication/communication_dataset.py b/profiler/advisor/dataset/communication/communication_dataset.py new file mode 100644 index 000000000..6cfc87083 --- /dev/null +++ b/profiler/advisor/dataset/communication/communication_dataset.py @@ -0,0 +1,109 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from collections import defaultdict +from profiler.advisor.utils.utils import singleton +from profiler.advisor.common import constant as const +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo +from profiler.advisor.utils.utils import CheckPathAccess + +logger = logging.getLogger() + + +@singleton +class CommunicationDataset: + RANK = "rank" + + def __init__(self, collection_path, data: dict, **kwargs) -> None: + self.timeline_dir = collection_path + self.timeline_data_list = self.get_file_path_from_directory(self.timeline_dir, + lambda file: file.endswith(const.COMMUNICATION_JSON)) + self.hccl_dict = defaultdict(list) + self.step = kwargs.get("step") + if self.parse(): + key = self.get_key() + if key not in data: + data[key] = [] + data[key].append(self) + + @staticmethod + def load_json_data(json_path): + if not os.path.exists(json_path): + msg = "[ERROR] cluster_communication.json doesn't exist, terminate analysis." + raise RuntimeError(msg) + data = FileManager.read_json_file(json_path) + return data + + @staticmethod + @CheckPathAccess + def get_file_path_from_directory(path, check_func): + """ + get file from directory + """ + file_list = [] + + if not path: + return file_list + + if not os.path.isdir(path): + logger.warning("Expected existed directory, but got %s", path) + + for root, _, files in os.walk(path): + if root.endswith("cluster_analysis_output"): + continue + for filename in files: + filepath = os.path.join(root, filename) + if check_func(filename): + file_list.append(filepath) + return file_list + + @classmethod + def get_key(cls): + """ + get key of dataset + :return: key + """ + return cls.__module__.rsplit('.', maxsplit=1)[-1] + + def parse(self): + if len(self.timeline_data_list) == 0: + logger.warning("Please ensure communication.json in %s, skip timeline analysis.", self.timeline_dir) + return False + + if len(self.timeline_data_list) > 1: + logger.warning("Found multiple communication.json in %s, load the file of device 0 for analysis .", + self.timeline_dir) + + json_data = self.load_json_data(sorted(self.timeline_data_list)[0]) + self.process(json_data) + return True + + def process(self, communication_json: dict): + for step, step_dict in communication_json.items(): + for group, group_dict in step_dict.items(): + for op, op_dict in group_dict.items(): + self.process_hccl_info(group, step, op, op_dict) + + def process_hccl_info(self, group, step, op, op_dict): + try: + hccl_info = HcclInfo(group, step, "None", op, op_dict) + if self.hccl_dict.get(step) is None: + self.hccl_dict.setdefault(step, list()) + self.hccl_dict[step].append(hccl_info) + except ValueError as e: + msg = "[ERROR] Cluster_communication.json has invalid structure." + raise ValueError(msg) from e diff --git a/profiler/advisor/display/html/templates/communication_retransmission_analysis.html b/profiler/advisor/display/html/templates/communication_retransmission_analysis.html new file mode 100644 index 000000000..75754fde7 --- /dev/null +++ b/profiler/advisor/display/html/templates/communication_retransmission_analysis.html @@ -0,0 +1,40 @@ +
+

Communication Retransmission Analysis

+
+ {{ desc }} + + + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+

+ {{ desc }} + + + {% for header in headers %} + + {% endfor %} + + + {% for row in data %} + + {% for element in row %} + + {% endfor %} + + {% endfor %} +
{{ header }}
{{ element|safe }}
+ +
+
diff --git a/profiler/advisor/display/html/templates/packet_analysis.html b/profiler/advisor/display/html/templates/packet_analysis.html new file mode 100644 index 000000000..07189a926 --- /dev/null +++ b/profiler/advisor/display/html/templates/packet_analysis.html @@ -0,0 +1,23 @@ +
+

Packet Analysis

+
+ {{ desc }} + + + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+ +
+
diff --git a/profiler/advisor/img/cluster_2.png b/profiler/advisor/img/cluster_2.png new file mode 100644 index 0000000000000000000000000000000000000000..5cb7bd3ff9dbcc6ada325001f4fbe7cd79a6c51d GIT binary patch literal 66908 zcmdSA2UJs8+c1j5C@KOvDvAi`s32XXNsWq#bfrp1DWM~s5JFT$R1}mJsR8LVbfiW^ zYG~2|gb<_?NRT8Dk`VHT8J%~&yS{(jb^mq0yVl)HJSQj5+56c~+fU1@dxqLPhtC}5 z;NalVxqI7~gX5qy2gh&O2M@5H$n5~_*q1|(?^^nEaPWTJ``ec&$t%dgagIah_D$1} z$u%tGLz44TrCnKBxif>antX&m3D-8SFCR$$GRIxGkH4Y4THI6jc|m>V3m2=)d2XMq zE?;UXdyzZK{qev@9_!ba3VwIrRtdN~9N*@Bg%K?#EPgKi*fz7ZX^NBc$l<`jwg?fTK&A`czw%t-_QHYhqLzo zyv@A)oc|Z_$Ny8{kJ+N^If62ZrUH5=k}FDWRRZ;BfG~OI2&3+LF{)7=FZkXjden(n zEdrcOqZL}FU(8(zx`BAJGe!{j);W=z+b~(Awk1+^E%<(*9k@o+BkrQDanRsXaR8}G zV0Yl;3I?KN;cgHp!Gieq$eexOQre8HyfznQn;yxNdciwfV(F~S6yxiYLwrEepf`B> zkX`-tU}k`sM89y{n0uXj|Mm_!N9(P0vNik+Iy4o@RT^4<4FH+nkP6#PG)0tPD38#U z3D8P=2@U9@7(k_zt;#2^H8+Ugz^4{A(Sp(jf8*dN2bWqW;w>z4>7(t*43U%d?j$XN zaYw!aG#x*suvMJgQ1YX_p@fc&8d_*G6pVy8oAK{5>ioJnM+-5teF}x3Kwoz|+qC*% zLmk>F(RncLY@Bdo=#yY(iYaS$F>L)sk10a%y|rvW%5+7;q*O+DpbV{hwnTaMqiGX@ zIG~WMyt1YbdeoP+URtKJ8LyB`nlI5`so}ylZt8GwJfSuPL2c{2vt9Oh5u?~!U0$!T zwA5-HmAmX{ujh@b0Afe49K>F@P}Z!zbaw1gVX>aBUb>Z{rG09T-W}(mXAO$b97+M% zpW)nQJm!Wr^Y97BL+fzu-pZmt1E2Bhvmez+5w8Ql_F-g&WWYvr{rraF7Qrstf0f`k zgeKgF|IWb?0-N+;)5^g@;L7*!YRnOnJ}5annaf8weqUrnot}#maeeDaPZ>fo#J$C$ zr>DV_p8xr`{YClqK{et$X*!1ER-a zsf4_p9F*Tc8`84{3ZUN*g_~yNWi+pq>8Cd8HnN%Zqs&s6jlnoXB_Xe~HQRsmB(w(r z@i1>Y=j~+&@-p+gu{!X!jE0?>0(C+_Yf7V*H8SD^yNrFBFBVWNXm3i|&0=m$jy#sPtZSUynKS!Drq63@F7=v6Ytcv`K8eIqS6d2m44?<&wk)hR zr`}A3;2bTRoJ_36bYON1E6@>`|D-r(+faJ!K8}(AomgL7>{lh~F2pNX#v6EPkgEN! z&6X3?M4G}4-1*I1CVQ1Fw`~l@F6!*bKB?;nfQBl(9-<1jH%YG#HbPPAxlNiYWmZO{ z7g&r>wU8jNxw#8|Z!G|MJWPgzimqW>%nJ0jqC6b5kiiX zlj-^Zl`w+wzV*^jQ0ljBGY^zZ?@XRe{zO2!u+_!P2PDS=mZh^lpPQ!<(SiQiaQYF(6}|(9;N<|F&d55#S@S9M7r2T6lAI>alur zv_`!qp&^RWZ6<5sLT6T0F6ZJ$O1wiVJS=+v6S618(o()hC7ZA%cbad{BtOBHQ5+Xm z;GV#Hl`^BjaF|@ac}>WzW^d=fMI)pN8931MQZoFVf6Vyqla9(2*nBZJze0iu%uE(7 z%252B-%9A5ZGui)Om|Ms=+p8BG|+jr-sBoI5H(j~S$o}o-6pM`ZwkHGu)xwsT>?t? zdKZCQYlNdFc zP9lesbw#I?l~1Z$`K%UM_jg$id(?&i`zrCh_CZ>U1L2;6aG#A=YEbF~jXu)1dRZ&v+cce<9^H)npV_L5l&(cRx z0JuRVo9{BN;~3tAJiI|$T8Xx*#jY;`@%zc-@vc?B)$v_xUQgmJ7FDTkp+r!^y4nrf zQ`DPA8I{@{u=ZO$1nn`HlQmeV>|1TTg793mS=X&e6o5{S8lX+1TYgMUnF5rQ64|Ov z08;yiAGDqGr)8)6Hfx$0e))X-o_#$Df?Q4{Hm?%?kw9jjAMJl06C>F>MbSmydI z$Rnw5;LR&?RRO-i#%}t>wK22{@hie*_PB@DlnI;3f09p`F3|Nb#$AmJ^`dk+4g2J3 zSJZ_0Vfy@=sE&E~t8v<&;s>;&dN21Jgy_wz` ziL1r_=xxc`$sh$e+1{^=S{CO$b>^3(r;)c%F3vlNR3JDwGI$eAiLy*)FSLAB%0m4I zDo+E{sTAXRxPPQM%v!X+BpgTS=0~b(zd;ElzSUb(OY3gqE*n+0aKguX>_i>s;8nXl zM^u{Xe}b6QvrlGiZ&f7d9Y7)3x9^QBg!c-EM5Oa>q)2NOv$8E0HN! zcWHf6n!ZYK|0D|c-k_qUtR4il1h6|V!^9j&(JZSF2~A`JS-kw^I^WI^B|sDpP{?QmudPAhOJ~$#n zf6=Mz0ljX=6DbF-a-lNvt4)T~z zXB^T*8P}DFVx$&qcZEws-5!!F1tX%I9Ogz&3N_?}ZMWAuJhnWL@i<%tNh3}@_Mt9QU| zTJ1{M?n7xH-+2Dx`>ntvXDz{^2P!*)^_dm6x+vofM80C34&Dm%s8J>w?kW6P7w%iX z^Jdpzcqmxe^+88E7O(&pP9n=9)%omHrKA@>7JxO3_|srd=fr@N*0FORD|4&#KzqeBKBn_=DgdUZzIv;>R(IBvTlS#xn`w+6NYA zqoWjfE^lq`ysTdKRqvQ5b+gcT z*jhH}0gDPyk^^RnxZ|O3x)y)lI>aV+MM^jVu6VWwC}?+Ic&UUI-gj z8@<*O5hINg33-eX3uL+A_y|AHCDyj7`2C5=p$ikVtu-N^yj|AzH~xsWl}Tx5%+A-> zB_TDS;?Dc<@kduG*PgHXb($>!w}Gf(NAgqr|8dKSc>@UK0@8 zEZP%A(VKAL49Jb*?K+h*U|8f4H223bW{l^VOoO<9@%#lwCq)PJ5Tcs62qmtm>=4V| zz)e4S-m$!+-y(5ReR3!n?KE0h)SIq;36=&W2f=bD3WK_tv*|YWssR>;w60ZBv}K*s zm#(Mc9t!$qFf(p%6ezpD-xfhE+5}ilAJ>gtM_0lXb`+1SRHqvshU%&C=pZ8$F5Sq}~S8h<&r-X*`a@+!Fvu|&&lcLLEi$dL~p&tPe`u3gb{nQw~M zh=-mDbj$0_NG*CKrOoygPnc;Nur+zjFc*s&E-sPkk9t)0I1AVbHh+r!im*q(mJ&hb1K9tk62`xgc5{WD{l|VWk-oA zY5qSH52T+iN4KXnBS_oxic}Ht!}`$03F<%QQrqS?HnkTY6M?XoF949;&Y#9 zWxlN{50Hai2gt?enA9D){ypuDy7S$hQr5P{UJ ziM2}^Mx?>>zGKUr1+H&@J#t3nob6&p!Uo@(vweuAS5Q;u8k{p$w=(k~IT_z%4m&cw zs5v=mNiuC+yxE+R1j*zXL+6oRldokv+^gCP%v%9oj;|vF&5~7!O;ekV9q-460#rjB zBs2i>olJytJ*YPUtBUzzmr1#>An@Zo>Bwmn(Zb@sFHq!_IpCPQjovWnYjL*8w6>}j zlUiW?!e;YvPq?pjcnGS*)T`olZ;GjUsDD2=!0LcqD7Z|erIV54xY;-E#+T2+;s(Al z_`L*@tm`DW^X=b9R;{C!yX#!mTD>#a2a3{P2g=rUM!?GZoS@-5$Y|2c=E;fUFMJy| zFYA$y*N+w!8qmG(R{)$xp^846%asEe*zv$8oQMJdmm?YluiYhp%ttWo}TO(VG#ftro*BpK* z3hxUKd>C$yX|kL|P=oUIeKs4>r8-OoQRW`3sPL4FaqIBPio%~tHs9ivM>snuu8;Y#u>O>D53s{0^ou&?U|Ql9J7|21`jPyWF_uf` zD~{8Urb-hitzIJQu{D-EW$T9|D`yi4Q;Slw~5r=JFk~%^_%bx z=TQ;6ZiEOtzTQZxca~r7X-W5v2heF_hCGaY0ea-4EvFu_(ClbwRy;iByp-p%y<**7RZ zraE9?I9B6bcN^+3zUt4KVq6rYQqwK;to+3`B*>ZgVH353|BmtBy}!Tv?WG!8JWKvJ zjwj72e@kv?e69IfG^})zLRZ~*|2oaz%96a*pp_gjIg#9uDQbXy46${Gr@q(g z$iP0bj1U%sT)4pN+IH0)IB#vyuN&C*c_9U7wm7*8RN(uubzLEXZ<6F_y9@k&_H?uf zlb+WwGTKZnV>Q5h`nPqW4HO^V;tY3V{fpKGqxp-T8`FU%L{Gh7FkU_ud^uXcGhNR#O&ls zMT@9EqWMn~ZZ3*lXUwtu&mJHIf?3nfqz9aG@#yzbwZVW`Sj)EAmkTB5ARDahgy zeGD>iv)Pin>nYW>N4jwVw0kK@C+<4S&}^LJoIdBWbMG72tx1L!%qX^dzjmJw9uvrZ zYa^RDxd`m($VfSFYYmzcq0gJ756MUCVV@Pt;pHsW$4AEyf97wdBo^{~ho`VZ;*m7T zpzXSxN2Zo8hF;6}Q4(^#rp}_oT3j|e>kKlL0X1FieXNxiwwCfnwNhJG^`fP1dQVge z!+8ZLc`Y+hrWU=~y*Zk^v58Pj$LzQ~>H+xqw%nVR&J~Jlm~2npt|L@7mLPDH*-SNCHzK9mw*mvueISqwQL0Pk|U#uD}$GC z?TVw=*$MefySf>CnA_l0Wx8Cx(`*_$sY@`vGHw^2lGs@EC-&NWrM2AFj8~l{taS-U zDumP>;h>yJT4y>usKq+=82h#>tY0Y~%+!oEwb^ti17t%RJcE~p5}@BJr@(E?3Bm5x z8%(1!@vlpJJM0umbeo!*eLbXo0%JY;zXz^PhuUM6dZO%;jy0z+R^N~{VC*k^0e0Ls)CJ+(+CN-X!B&yevi+lbE+t9qreD%xn4h1w{9PHO(-W9mYKOkDZD{ zYTI6W9QGYjDk>+Imf*-G6u*aowndmiyfDJ62*oqURU2wQu}XJ}BE887b0#-?V?XK( zklXWul(%wxoI2tE7mi2+KIii{$SxaDJg%sd%hC+y|j5y%( z^|>{vt2+8xc_$HJwW1xhP!uK7UwZrwu|-=;pw?DzT-EnXYKiMMtd9PEOPJ9L37uL(?cux6*)1+J4mPN}RC^-oVZ=2QB1 z^KD`Frqk)&xU)%5xlWEK>=n0IN0bR4r|6#t{5vBxha9r=oAW^4P8(y5NsTsi76@-2Ab0hha;9uB zO{R;>?brpidpB_!Y%vQM?}seVeFgk}BvBLLb{cBVm%QqgdAiNyyk$QL$FVctE^9S> zi!pKut$~PDR?A*6p8OjK%CJM*fJm})88tC6u$D3w+ zQ`%AEQ>=AsZ5B^&$lWPFF94wtK=YlBL%|;3^O`G+woZRIG#s7*7EOnLhs*e=Af?x7 zd`qV+*2dLOu!~1N?thgs-S=vlmtXufSA#x_4MDqJrqKtq^GZ;Py2>Fo#AJmliLt#sQ(B`S z!f1+o;!v|1d^yoMd$%0-PTb=D>xP5l#{IoQ z_uoV7{GLhtYn+JObAQt{VmUQ<>7x;;KNGt}a9qZMCBO+GEyhikYKK@K} z0qu#--uQo|4$d&$$Cli`djb9jtL<})6_oeXr*8smx!D^}2(IWdthaK%oq$ycp$6^O znI61Yzm$Qp_G!*lVhi%#xFJZW8qiQ)D{u3|FMm^B&WuR0!=1eO;;))9A|-sL25pBz zcf37r{x>|wli-JN$X54TrKxwEd}>z+lQkZ6O!DNq*V`cW{D-A}f_-kJj9L3E^t%mL zhBN7&+>1;ey1(iHq&!plk=g43!*Q|=G|1?E7ELq{Mw0(K;Z17g4f5);k-%oOKc$$4*JFE)6 zw(q}fr6`>|DJLcW$NyOK#d5)+pce?IAxf6G$9NHq)}{ZcvC9lUB_@`kpa?NfgGsLY zzxcVyaq>awwLq2p^;4Y-y|Th_M24sWbmY&U->xM;=igEha~kQ6%4z)diUS|oj$M@a zd$c>`MV77vnHmD9IcJ#_#1)&NA6CrCk{zahVc3spH9DroOL8e;-MM3_=ZvDeu`&r$ z59FH0t((|)CySj;Z-HVIF1_6xzXz{@hN=(jdWMEmwukZ{+7M`{L}-m#*u+vEuwyAs zDmDJtDV+5^qFepUd^YJDU!b85qvIM3*o`3e#R}WS@#Bcg+~jzKP+b4pz^0vcO3Tc1 z3bA3u-)9GXyq-C$N-nYE{2S-byjO;ANviPC?|3&qw21+fpDTZN@lyIQ^Ycw*CI-;8 zKSE`l6iU%p0M~D%WhqlG?0*`0(ak=t9}5Qlu#oM}F$rB?u0wrisVFOD_}hdA`y7;# zPqa*GGitf0q^~A!En5SQ@jfCYALfmWbE^0LXW2ij#;3Gf;?~o`L$)yHsOt5UHYDk- z>DtiL7hmO-Vdseivj?d~jmY$>J2Aa4ug>{*>JEqZxGOdDDPI`fo{SFZQOH)FSEHlt@?u*isWKuQ?HN>c$8}0t|U9r%GB>L;!2U0%o{SQ z{)VBvD(AB`z#*ZqK%=ETM^QrK;t!}myylWE>{QTlmt@rpV`-toBB;8XpEx*($Q z=YBnz0K@i9ji&U0l0FiMx`+o+MOV@?xP+71xP%iOW7pz|48^q}LfcBbC!qQs`Oes- z33*vfDn;XCFsa8lU+C}EpL{G=yUjhY_{zdU$ty8|ukWNIS1xerVmjF6$GFe@GBds( zi}G6rBIEoA&Q>!2WUan^HmHd{8~Vjfx!T$hgfq|rt;$JL)R*YJA2cAtI4{`z_Qmc= zDX5p7a7w6IKjdQ`Xo=q7DXsF2#@QSnU@+0yAPRlJ=3FXnHGGX$Px-1@X&)D$dQr?S zRF8Z$AEG1{Ja*wHZN>xH#7;itqLH2wu4!T(ck$ASva4tKbyrVGa9KaMKhOvHc_|6F zFs?ateb;_y^^;A&urG%BU~~;=;{QNXU4`2H3YPJX70j4YYkcIj#Td^_+v(0QV2q)g zK6KU^8&YrL`rh2s*(E*lRwJiOY52JDvW(X~U+V@K;ACw-SR<Y< z512}YHqK2nRXns-9R<=5#ID#3Z%vQ~aOqonah=^zWlD*srur6RTc>`8tIoNDvQod< zQz6gj_sV>5duV58UGWyLCa+uvs0&RsD`PR!*a8?$|K+tcCJf$^9P4kC z0YaFIYV^sTl2X}_>=q55o$u|Y)0E9Ha_2yv*YRT|ea8W5?*v8cQxf-!6tGY|Qjnd+ zH1I}TY0|K1tVPn&TP5oj3l=QVQf0m0hF0o9x*h#n(o&2FHWkP!SJ`~q8qKo_avT*g z!99SB1D8rtg;7I^9;sCElHpvcOQHAAy?>S&`^yOnLqK1>6;yJW>v(I442K*L0hd(m zbzSZa)o!Ekow=|XPLaDy{pnTfEsVVCYIz}*fV+)BgrPFq#$dDBzny(H@RR@JuEqa3 z5AA<5^H#&)EjPEq|LpUoj!w6XG!(%tO5P=wAJCuomt78j&O1R#=`#LmYpo2%S(s%Ao^RGyvHruu_$PL{BQ3+ap z2icks+nZA*5TCGj&&Jyg*8-WWgHoea1Ez}4lx!d%jDWTAt3s+lz4Xo1_=|#*bv`NY zf4Nz%wsYfPMi3#TV_G$6?L!a&Q+GNQC!j*~dW&v3zfzHvb5T&y^KM+g^5FSZG{Wbw z)WKhN_H)viwu5Z9RWFKHXsE)mGa)G{kG*#x)qrvFD#;W0Pjw1*j4eXy8JUR_j~*by zoBs)QJUbV&B6)>WJP%6+tbO3+$%;%@K;5c@N%Osz@2$Rf^noibOHy;s^#z2a-ZUngll#@e z`JZ?@AnyG;1ft$fkL0gy8~PO?HJ#d99mF8iRr$8P$g3c#Lz=8z9}1o^%J@eczzv%` za#Eg`4@zGIZ+)oy=lRt!2+7FoV9c6KosP@U53yA%y8SAFk&=baXBIu$xE;YW>@9 z|ICQ_@6VGjq@!y$LxCfd6@lFITj5;cm*n!GTB?83_A?`G@C&gz2xPZCy4FhzG_$j= zRoT=bwPuK`F26f#fB${!f^Be9FVCiB{roFa4Gr0WCgXUeX?@*>#D&kXB8kWE95KLo zN2!8i5WY_KI5oYTdGT zs;91RdI)H-|Tox2-Zr-z%%Z1DS9>q`P|V@t6IVBZNF0_Nr}I&zXYzHek!Td zklA8uvh`}tp9G6q97IaG&28YTcD{)xUm>&`Ed8l&;!C4m!+(wo24#3u&b*(XO*}hg zT;*=kO>-EaFy|i%s0Fyj;79A*i9WrJ?3Qqd?{LSdmQ(>TmRHf@RQv?lUVrp;4ZB10 z>$^1holg`ZofwODPlQ5j`#-Hrc)>Baf&O}!A|)0QwfI^DdZ}4($=$L>-bjru0t(oO z8!36vSjeS7Y0GMh!I08UWDFeXJ2{Mun~q#xcLn{VIXI3@^Gd&)WzCdDH3qEDj)fwP zsMVs#{$3u7m6!W=xlKYg*W6FzkzcHvE`H?7FsXd{I7d=?5ZQX(_xGsK~Ri zrH^y$iNm2U>V$7%(EfC>)Tq}mB;8Zf;k*u>{AC@sG*8MfKs4=Q#8P%`w+b~>mCMUH zcPlSaSr{SE#~)y1y^6ws`nzJd^}FL@;j&^dALu>sLQ~&w+^}#emW$uKA2oW)ppVTDv&J5UkW9II?WLqS&k+X$cm`n@3U^DU+a-@K~DfMK6Yper$o9CZ0eWfkgY0J>#E zP-rFIgBP(rOdfr(obfu^yBkZ@A9%AeOpcF^$^*7($m%yM$L|BnICa^d6`WqMJfF7GEjwzkGzj__s0&$MCTu3q5KGCC^FO8 zAuDnLwEXQGHd?j0Bnr3V6$)1)K^Md0h^(=H(I{7Lj)I~ov zO^yMRA4T|WJ$n{}?bS=&&wsd?_Q*xetja$YU65nMtJ<<5r&pEf?u?=;jm$@a4Bkhc z#UM()<0Q$&D{uEjX!_9=h80J@XSmV6BZY^KCRt|m6t5eYL98bq17SZ8*tPEG{6W!Z z@@)3o3nl8qOrOTQoU8>^=xk1!-IWJ6G<}0q-4f_R)>Ol%irzSi#zx%KbuW{nv&2=| z?pFpzS?HxmjEKAN3?cAtjF-N2Nthx+f%hF6nUNcnC}I1rYvbmD_m!#b%?758MX`&p{W=)nQ`c*g*%l z+Ep>(vGOEMoq;2}Q@;BN4QkM-J6A7pWxUfv;N%}bseBy^1<#5~!)o%BpOnYPpvR+l z*OaassBaXW8v3*L9{ExD==Lqz;8|>O1+O0IQJ^DoIXQ!so%MX{(h6v8MPM|29~p5?iW-;J9j*sgx}Df=pYm-j1QoDm(yq^#tbhl#hp8iO$-elQ1Pgf4 zlCYkM)Ay>SB~^R;u>6e8P_vO?%koYMV^RS`>61V5BU~I3yPTFI1&NW;2{(u5h|0>? zrq5B=Q+C32!fV=nGrHn%@FYz>{pknvN5dsYuv|iMl^<(?)U=(N3W1%97kzwNhVA8Z zeyq`0FIKkX;{0SW3a)fXbp4K>`v2b4Y?nO^f%1pP?!#F z^;4{ZBaEFisSoB=(J|oiKsmkbkOmODfgDnMHUL*=#H*|VqHGwX4(u+G=3h?h#7#3} z^tuNYb2PfS%@EFn%(g1hC{Xa$QLI17f@{g54)89qa6k?*<}~sYTUCv3fALJu?`tRO z^;AalFxkht#kkT((2+|LR`((D;lk@g%Ty<1x=kB>6h)@57gbT}^hmRMvU%=Wg24UD zc1al`o;2en$QCBBQ*xw5DvB4M!4R=Y?zr@446BitD$U~?5(D_?fRE~j_bMhJj)i1;MH%Z2`1-38-JuQzP%3KeUG@`Ja z-T0@14O&3NI)9dj7x-4oZr>ubPQ!tnI}^D)<Xk(+Ij7gRT3HL+*MdjUjqh5%NcL z*TEt5vy6?i-+t`&Sa^``KmwC?8XmGBv8eA0?cRONO8cdB(-FPk`79Vd*s5>L9FQa^ ztD%sermg{FK6Yofdpr~3KDvg)@$SDoy}6;dp3gj>)c0|)`u6SbLr1mGCTUtdmrz;y z0ZspKAMVP0K7t6nnrMPdt?F-);Wn?$J@6kM8?kwev1$PGve4@bL=l62EqisDsC8UV z=31%(Xc@PZUzTbDJs>C*Niz;O&=aEbn^xuNx7(1N`dj{ufi5Fl%DV zZlf~!YxYp1wdDB}q*|(AzUAi~f2>$AhmGQ53^&a6Pk81Qy*wPCQF* zO=;Rdo$_bBz9=c&#WdpW*NSLpDgMZAELPGJRMG%hd7p*aG$6#uQ|DD%O3K*XFDNj7xjDSXe0guOl~{&?5!ij8vl!gUPoY3ucp3 zqz3MJ#Yzo~*K#}*-lozzM^KCq+9>or;L zud-g%{F`e2z_EywFPxjPM%_sDPxCC``t)3J(fSS!=|Axj%O`R#JU-@<;i2l=e76mH z&hO*Yd?c-JVAX1J>A*VwzE%M(@yzNPY5w2b)A!B0>_72nP~c|jh{($W;ZKg9j(XYJ zdDKv$|Fm}M?zQjRm=oio%Fs2;BpA_5ouvM5H_#cuRlin(qSm(HYBW|EW>MhA{+Q#F zc4TV;N$UL7$CW+ERaSTtK{eYHzuS=AcD(Xk{+DBAT-2>&*=9#64CA-EF{RL@_CYls zk&EJTMu}AO5ZJBjTeG}Es)9?kNK#+OdyDInr*51pTt!YEhsx+o+8cIL&DM4s0vV#s zi~YN4B1whk&M|}xJZu&k8m!Ea;{=}YiV6P&C6`JUtk;K)^POe;j%}qc6Mn_v930D- zgI0Ihwy5aLT=*n7s7`TIvAB)=WC~>Er5#()&MYM~RGjgOZPHC&*F945d($TKP6gEzqN)&8LMlPy!Ds8yrzB*J>kWTw{Vgqf|5LoA_}>2! zVX8v)h0sxAHFL-FCxc4|BO3KJ6svCm7&;YF?fab;3&M!9qh@4oFMt64Rkdw9m34#YSeB5RJ;{*4mp^X|8355XdQx)*(JJ8$QFcE6o=5&B|?KEVszYbyRXlP z?V2uVKEhZ{Z_1hOMgd2+b8`lQie09?ak5;K&LYe2=1MNs;EjahUUURgx-oFu$T{8I zyoeT&w*~h$4#vk$k`y4{ZE9pKvoF}I1bU(zQ<~f79UY12k0Og&(}5rBeq|Tf#Y`i| zyhu}l4~P{bjr;|2UJ(Uv)7A!U^wU03OxhKz_HIA2!1u-_6_m17wN&H2flaxlXcU2k zdI-kl&k7R9;zk7(kWymva1261UP|x$$=I$<2J^@Lg%|6G%>ojcUr?~LYX)$l^14fN zd6nf=qi2sTD_lxLZ0d}J!Y%SXcqdO%fxw>p zX9f0I0OThy2Bb||Z>v)Cg9kvMh=4mEuC3xa*6OLZp8 zHBg51oa-~Eh`pZh1%vWtc4siSOTk^T{9m7o5%YP%wBw4Qgi}PRk7q1vj&66ZrzCD!V%d z)%G3EIMP=;5iq@yK;8kRs|Fj|2L>0{xSIql&gS<$Nds~14dWFk4ar>Do* z#U<~WoE-DFQ*=KL&YRrk*T%`R>@O!Kdiwj#+46j`P)yE*jpy&ba+rIF8TXK-ROK$T zF{D&gRjmbHWJb1V%>BmRqO*&OgsP;Tx?*Z*Xy`>9HjCNPuysJ{=%;_}=kESFsl$8R z@9!UbvX9fLY9fJ<;hP0G=i2(n1=x^3)YTt!$&!s&Lu2Eie={e?#$oW5xp|VL!+BLq zqyBAsxbw|FU0eO`02^R0euSkbC+1FP2yap-M}OtW_Ex>iJ)pNOKpf>%lbr*eNqV`A zI%rk?p*v3YajV?wEgv5Q^;Z>S54v#ikp7R~WkYs1qn_-4l^I!Z%_>gG|LuQ~nVjgW zH(F1B9TYiQebo7mAWLkoHRsnDJasza1YPg*v)hBWZe2^6`Il(kP@&{=?3YV+`jraX zd*?2fi`C07-SJ{IZe$lB^-xoGf#JfP2vc7w-(i8M#>TehCuD}Bp5-lVyuvvuwIDigDD|$`q|C6a> z17;2=2NTm2eI`D#rIk&&g1>Z_>A3sbGk&RM0%7!9wzkf%jubZh4LAkCq9goN(&_kO z1l?@rmZIgU46zHz&i^KXMLGa#w=@8^2&$Dla*iz`$;oA)BXpASvEQX14BF@Q?cw-y z*fj0&g)KL7awdDmPsQQpqkno}_F!ZVvS$jGKK2b;nvsza?%Vsj^zA|WG1{*Q*jJmW z7CK%MvN>vjC#-B}tP)oT*phRwEgtk=OaZ;Yn|g#>{YB>fLLUHG^xtOT3HKAXT^#8>=%=|V8cX{yRf6k!%-`cV+(L+P6WzWRr&)eXd!dVSOdS zRO1)FWQ>28&YA2wES=_G{92}NKTBp<6q9LlGNJEYE6&htbyaecY+( zmbaj7=PrKz1Xg4)n<6Pel>)2UBBN8XfZ4$dMrYZz_mN5$MPd6zO-?1tOq zd+_d}wOv*oCfL0(;>54SG{==T($PYBzZXfe84rcak0!_kEf|3xaHH08Fj`KNLUmS> zBBwH29bJ~c3;t1_b*L&}od;Ao+;Bdz?30#On|`=A%3_0JGxU#FbMAK+tGmfI*zk3y z1s)3XD~)!DP=w2)2TzV%YV_+&DkS90!DCl3&O_BT(uo#(qk$Yy>RXqai4JCvnKIWZzGKv>6MR}b8`kvYF z+wSa^b4^hIa{pS0&s$+-d1G=uOajm;bnd$=xsZcalPB z`4~#Sp28%B=!SHZhB##lpLL{q1&te8qMoCfJ z_n!LRE8s8ew*jL$=rO^eVIP)vn}n%_^~3AJ+#cUB{$?6XZ?l*_YPx@{Neqiw!RyKL zx@X0eyClX_WHpfo7H%>-erLXm*bnVoDxGX*T6zIeJ}VPdQ){waqDW}mx-Di}@w1c5 z`}4WGopUm)1QA8|RkG^nta^^Iv<(2=C~agb9-3($$oHv-2{95mQot=(y=fEaEdpDW zsEXkgDz-05L92d5Awy8`i;RNq24>RXlYFT6*yB?pa|vV=pwM}}*s7WNk~JjYRtSwS z4aG=dcj~e?-=aKUgwEXS6w@}dZ@k%PxF7g^X4S=YY~+x{_Ug^biFpf|*qs*hRqF3K zFCys$=Oa^sI!hbEP0M7VX@-FiU;Rc!pOF|oPE-+;<_fN|La{>X7L8;_f3O|~r$-NG zZ9$=W7}m;3p;tu(OGTBRl~SKo5nh@@9t{s`VgZ6yIjM;05;KtxOs}woJ6qpUw&EKV z7A9?Y6c4l~kG((Ip_UUEd%~lfaoWo6Lt!rXH>!Ve_~w);`{$`EN+!{Eej%|Bd)`=^ zDt92ut`B64!42iz84aH)lmTudtbB$^0OlXpl__wOTijS##=fPqLP+Jum)e;qqqvJv zcv!zFq)U3igD!i@b+T(Fs<9U3mjG?EDP?#k*Y3{67Xh4GBKP6yZHqgOknF+OjN*?N z;ju5W7R9AI=UD{cqxuxY~&R2#T`x<7GfHfDz&ZQsVHMX0;(kJTts$IJkQ9c?=xD7$( z8W-s1>Zr%exrLa*!H8^akL}9N>dgdMxVQb*CyA(Jsm&{o-}xajO1R)VFAk_R+R&cx@NB#JzczvflBH1E;VR3njGZ7?9^aq#}uMIQ07BAofK|KHY0Ir z{q3bhEcGa6tsvH5OiGuVdpdwv*Clu+u*HA=2MS5=uMSAZw1VKT1 z4S@s_iqsH#h$Mu3@wE56_dj#*%y(z*{W8oj@a#O#Ui(>lt+m(s?ccVh+t>i#a&)tC zL(N`VDO)d!Iy}71bohNTaOIUTX%LQ8Vbfo^T>rF|cWpp$zMexvTqOo~Pj%isZE!!3 z4ZiCtQ~E-n(MUkp+OzQlq(%1NQ4-0z1kkM5Q2F?DF-Mqie4u!%EMVmXO)k-u{k|vr zD9f}}Ujueh9OBtWLS8zq)pkXY)Uas583woKWn)|{|W-=MuG6$PcQ6Yp9KA~!t_KGn+T1; zuF#tX7ceXgBVi!LIzDcJwE-kBTU4o_KnMk&07m;v_C!g02|6;X0-mOfgDhei^ zoKA6kZdDT(&ztNQ$u|%MIJ;}#WSt1AkE#k@0~Zg7<9X4fN}y3=ksqsHh`&6NqP4m> zX4sZ}(q!OfM5af*wA`9;+3|teIr?)4OPhBy$!_-RbS9}cPwt5+`3w6!uPx#))d|a+ z^8yBU<6|JI2f;U(YZm%I)%FLgWe=ydZW!E=an;>AYL?fa1@IH<(26va+gaV4TbJlhHRcO;WleZOCTTw_KQMC~k@+ z%825#>dVV@g+0SWAUmY{DV1Z zQWw2+ZLuznfyu4HN0*)&UzAo~X^5my_Owp&cAUL5qJiJcP_!*8Ycr~M^cf5!gkSY0 zqBWY5Q$L_6DEB`<>qJUc=mRrw_~*0Qas@@p*`eFMnAO@yeFqk2M59I6cKIKYj#b4` zSLe|3Ob_EDgU#1YoT{f~CA4*gz*AKkL3Gn^>idf(9cE8IZtyJ=)pdtJXxQkzWB3l&>EOlM(r$%;zPBY{zUd##E2(pbEi=5U9D?yzwl zL7NsT)z9BhN7G&u0{t%S+{TfkH(1X56q&Oc1HL=4y74DEZg$yuQ8To4sMuSmD*R+zSzI5;W50s?tx8D4{aTonlz0s-rOg`nq zZkyqK>NMog8RIIO%>;hMsl`vS)%(G(%iD~|A(&K|&l?5QR9L6Uu#i2-2aqJByJjj} zV7}bu(HH@O?c7h%F8RpBv61Uwqg%e&YNL?hy*}d!tBS%<+?3%RyF#U0etJX#N4Ukk|$0a)^8Ze-?o>7eYQblT@ zyG%03ay1p1$xvI6o5mjhikYPDMWce$E*f!TU2-(dc4?!A-DEO8HJTj^;PYh;6emA}2Ep6^F>+zfo0A4dCo8%}q1aGLrYlB*}V=8a+Fj#3g zlu2p`+l^03&eLgUHF_^=rc^vUfLwR($J5YZ=IrYWX0ui?fJZuC(16kdGAjSP8jXY0 zJtbOaQxj)aJ9D<*f_ozw2KBAO*q^N&K*|j3W+hpIh^S$sDVb;G!5iYf@bgP*^Pt7WQRreUK226Y17hPgD>w{vG>*rO;r@66T-X4^S?x|W5vVSQ_vP%Qbf{m6b)f8(h= zoHj|#n{|AXGs}A@hpIie-Pj6{z9@PS#c4H=Ec-Q$j!l%vy6)JgnR8;%8JM-5nWwF^j@P-7Di;u1^Ha~q=pr(jtP_C}t9SFwF8#Q}pA~YJ- z4GuI}W?G-=3Ox5^_#bxZ5eZzMc&WEJWnZets(e(VM8@vJ2R&HGOy2(BJb18QH}4j6 z!x${@X77Ut7F#c~ef7oDJ>u|_9IHMk(OD<99u+gN2Og%F3vN|h?9EfU=d#}4*Ic@!`aB#sy7{HO`-BuPm7>9;{YCe9fT&mHLB+Ye7?KeQaCvo1$MhCnEep#{ z9UbgA%P*dGM_KTSky~(H9QuGf9zy=6FL9%4`Q0YjHhdH9SRvYb2?hVkKrD zs1Eu#u&<*-Nl4?e^@z3S1Dq$DLc0=P=2?KTyt15eBpUXO(@58D8P2JL)vL)FmV zc}4h{M#%H@@m!`J5(TcsDuXYHHkNyQG>fp@g5sjG1qa_6#XlB?0cm=#blR=ij4Jk( zA{cXIwe+b;ksgnt3p*ur(EGJKif!j+OjMq4O;?2$vbPyAHVp3Sw`a*t58nsMxehRY zo-0>L3G^=I*@*%k!^rAUaq7ts zVgtB-mtjp+-|b%bxPC2jwgQj57b=AjUF@21l z>}QXiH7m z;ehYg9QtbC$hu26=doo_dIu%^sPekoC-Gw*D8{|gSVG}gFP8nwf5Lts8zt;_e_n}= zRD2v>HTCYvxa&7z=O!elPmk7hnbx!!fuB>%@dL+l*=sa(qnEzh*k{BA4I6nh4^%(? z5pmFf&=4u#gI#J2%2*Hg5)fFhfqrdx^W7y|uv1dnAC4P0t0ZqEQQWL=*HMDDFgZam zg141rN2auFb90PL?;4E_l|Xtw#vVWDuhlPKMHkKrBd31KwYRB)XAsH03XMg@wI7>1 z3GcSE8lPc8&=pOwzn#@m_$W+z5Yf4#0m%oCW|k_7JYAnrWLcG?%LvywS)T5Uzi~ZSMIk6y6^Xv)62hoY4&P31mN&+OUK8yP4x3`qE0C_W#tx%5+NJ+? zWMlAJsplW3*j!)@n%myf!mnP)g7QxAO&2wm1=GuqxFl+}J@1@hIxE zHjC-RD?GD|-m~#`WVeX6y9l9R;StBn9j{e@m;BaNo@hhPqZGUZP5V-|?VPZ`?7=149=w20C=Ekspmy z`75)SYIWM`#%talfL|il3D?PA?B(Q;(Xj2egH7M@DK}S&kP>jCWA%&+_Ca#<2h1Ln zG^&}>Ds4wih~HOEslLgDtd>)P3M%*Q6U9QF8sbFrN`zvg*Qh-nFQnt9=#(|@-JsTNhHcZERs00& z2|flm{!+Z*tnd0)13b*3~j9nz#-MZ{WroM+8^;8~oea)KkLV6zS^C z?=c0s+rbPDOnQ`}dr~>v3{^C*h+K&`ZM7QI@t@R5X=fYw5tC8TQSMPl2iAcoILz9b zXv-+B$dY14T#XC#kD62z2E8!ng*uRFpO5gqk$_A2aSo@RX+MCs(u=QLdrF9NVLZ^1}N6IMc#5ioKN9t8R9Y(F)D2Sq!;WG z#p)!YZhcf6!BvX=9tl64e05O&JQ|Gl0hc^4A&IH^@f{|~uEjA!)|J0?Z2(tIiN6e(pZ?}Lp z1+Xnye5`W%88?cZsvVKz$rRD3ce#tD<#I3c?mJI7F1K+#rHjX6eEMY!{KMO(jxr)v zw4JUD$d(sL=cRRQj8&P+n&zs3l}*`LG)anzI%Kd9X**2bH%6>LsCY8Lq5!ubD^sQA z>mw#JQ$b9{rZQt4fW@YjJ6p+ChLt=~t-(7|JIb|h{mCVqaR1ZwcOt_%hc^03B;XO3wE$CEd9z! zr97+~yxb$&lETD0)#>0KnmCLvJ|LF&&-L+pA-C^-I2mNos&wSrC+zUo=mLnI@cP%O z@hYEb8KW2Xr60_c-A5wBCl>ULU35Do?;`MgWA_CbRC1gp_KPrR`1Gi50wo9*hu?mp zcJp7Rf{n77W}v}=gt*k0gCl+Q@y{!g?a+djSL&K#UrLj9sa?C?Mr%9oIO##wxM!%c zaEQQ2D(xdFwN^9^4-MQ5R(o>l+$ZM@%!2@ml9%IHg6U2#0Ro#rD1-7gm@Mp2Gh7S! z6mdHJM4_we-tD=4?aFdtaaP-`OdrtY^lll&7rq+0VWxF-ITgpyj0&lqX(|(mnGoMz zYoF@nZoQRnoUl#c~-soPb}+fq#MU|tG2-vQdZ-BzF8tb1l7KO-%Bjv z`(qr1$uNaZ-;YhK*~vktd=Vb5IA0b~?x?HZu6GS`_FAGutXn{?VHb0IDENXN%cIT<(OZebz3T5WQQ$gx`9($S8B>zBwR7se!8(Wa&xj` zUN5oSr`a{Dj@$NAt;wx6TMIW9MGPAVR>e{Ho|Gz&uDg^Qj)IfNuQmSnMNAl*y9Z zU>|Oi8PyfVw(@}ViWX7#z_g95A8+xc{0CYQdK#J=p(D|#7M<#>tx_B@dL%YQAY3I8 ztgRNp6k;^UdXnld{McIS4YPtfs@?HoRdI);$=Qi!)}C)UDrm-xkpAwB(evsZW)RE26{M zOI_rMJ&7KclRo$)klQcPpjw|=rEBV3`78YB>|!YHZ!HpRHVf7gV8%(kX8ruunAPLr zO;{CdADqk$U*)#S(y8ysISY1~tfui7^P8&7Bu~AB!KNMNCy2T-QzXUWEL``;oas&6 zFUwODzIoZ0J;wpr*zL=F#a7??i(AhpW@iU~{<%({@;x*S6zcSO!1evXZ$4X73ACXT8Jb!iNft*f%FavZ~ud+AjrI>*TU+Ar;cP%$XAZaR_Z4`7}>rbiLFS@SEc#_ z#>=4KQn0J6#eRH=E;3{$%5WxXRU zf%USP+2;+bAh3OiML|E>dtF1F*BW!1bdq<`2)gzTp&>~c(1qJRXmG?a4+jS{6Q{2;HR_fKDX7t8^CMxwRpqfQ=%}!yH0_&hmX@kV#?j4Sx=gf-UPZjx9&pn>2YcF z5j&A3+eUrkKCl{v&LAV4jSd%*5U8o5SOJm{(*LLdfp#1#iy9sgPGNR9!7>k(IRd0*D>%X`cHBXz^_SO%es=Ve;du&E`hP!|8=CurKI!4{PE--wodPa7vyi_wo3Sh%w3_#|Grf4 zGDhp=%Sb^%K?2PEW`p_or%eYN^@nosz+Q`OGhLA+yQN>7(WHIGIFyv$b$}I8k zO*xc)#RvSMN<%t6dZ~|Ef83y;SzZH(2)Gvoo$%eJo|{s>&&UA!UXOID8JjZJ{4Yz**R|Y9ajkhbGM!W=Q7LO%_cvT?M(wiF_><0fD*m!zS+OE)C zIhCJT@90y$>*Hy^9+SigSa@qY9f3lb?a2O1Hm%{C=?hoPc26*LXRqCmyQXQ~z~Q>e z82-=PqO^T=$13#gllQ++FYE)KL{2sS3zZ@8_!9Ra1@A=xpuBtY^vgf9oC0U^TGu#x z)#kEM?!Vp*1@n}_@vrVuKqdc&BBuT~=GVUl_WuIM_yepJe}>=jfkXG#4@P5S{&G7v z?FX`A1NhU$G;eZMXt?AP3<9hTONClsMa|FEMo73^zNHf$7v(XH2vLeXhMze&NsD?sXwbS?hKNyC;&RpM}69&pxurFj=GIEN9Ms+Tk*qIa|2)BnfR0p#h zzS$8vwr?W51^!I@*|8y2rxC@W{T5Jw{})YrpGqz%exe}%5Z(k)5!Xi6%a;3e+OEW1COrNle2!Fc7F?ibbo`36qRDt zHXBjCuAy1vPdI^}nlHhO0>z7GeGz>$34N2KW|mDIQh}(w|4q1RIcbZHUsko9BBUL_ zrSHrc?iXY7SDB51wQuEo>oE%U{1GKlbZ2*&sAF5`=vFi08bXX6yBMtzHc(~?h!KFQ zE8IXOWLYIv_s`k#NECbWrRWZX55_MkF$yS^Y0U~xwIrube{M)4wVw8t$)dJC2dG2IeF#f-?gZ1e<`d5tBD>rsY&b^d^-_0BLgoNPFuy+q ztHr7fG@@=+ba`jjo_;IPh)JWNLvbiI5));ar91XSZQx2p$Fqelqmm8{4wI&}z@&7F(unXM8vAui`c z#h#~VVDAXC0reCiO;bg70M2W;UhVBBI0acnmvwS_4$gZo;|8$u!FqhH3>@lpFhY1O zsJ@&jCRnI!%-&Qjhg{Eov;0GK302U9vn5;94ZL@NuUpl5WKzbcg`l}ikb3*mXRzt z{;p%SA66?gPU;~n zCwZgL{)DPc9aqa;m z^DX_Z^#veT2k6jz*S0f;rR;CA%ihc?(wriS5x?sZq!y8-onvT|TtaN<(}Q-}=j*b) zI`lnQfoaF`6d&>K11PYDDxx}8lK_Ex7$XW34ZUr0LU8wbHD9%R5a{;fqxyXXyS~dkom7(0xSMF;*ifU zM&9tEd2Af{`w4DV7ApnD1q16{^=fBE*iB?PM}h%2`fk?7SW{aCtH3A4`Yv5>vXzk$ z&DMXN5U3D`Ef5#Lz{N7+0%?%6vqjI_pl=ERw;Z;vH+?GFJyophEj<(Z zj^aKuKDp-VYSf8w>V=ivi8=84pf=h;7-yrV`IC>abC4 z#ov7G_pb{UK4Zqd-kB^8H0{0CXD7eG?2ihrHQY0ojY<^|V(Hi_zcK^BGV}X+$#sd) zIr^(*JCi6kM;xM9eZU^Cj2aHi37NHR5KZYitrYan%G*t-?`tHYDF zUkbt|Rp9nMGeOahg*)9-Z)2Ko-PA!_^7ehP=9$`i`1O3j^UIII`5k*$6p}@qmz@Qy za;^$#$#_p?H6(}{O1bLGTmUxt#I7)H5gpDUtLQ613?`j){7~+JFtB0p%SrW@4x<$U znh7P2DnIub1{3e*Z6;vOI;mnUl0_{G9HJrE?}9u21e^4_*DjV0Ov_GC1)R6T+B@n{ zWwV^eP?<-U`J&Fkn->J{NQ|ltwnbPS{@D2W!q;A_yULNCC1f_Msgv7y*%c|jYAZka z^kV>)p)WR`t+JN%jHekUXKdRtzX(}-`?@;+PT4(SY`m@H?89Q zN9|V}22}~%WjR)6BTLMZULt&txzXO4(ob^HY?!U{da(kT2Wnm3d_fTIv5xfOM{$h; zY|d;P^1sv=)g+1_FnNQyBc)>M_@mn%an_A4nJSxSbKR!f<7czom)9CZbP>D^2G=ez z_X#I;2nc7O$dh9H1b4Ylcm|3xbAj#4#=cR->+C@tRt&`HjDmn-S2RIW~^w=k}`GpLG(=D4oV{%{%N7J-+~`H zYbnbShShXq(k`C0LB_70CZ+^y%BMvrb(7-W#A=^Cb46}UzPpu`K|S5>{da!D64Cq? z_XUhWM9b~#Pj@$TNYUH)u%1fdo6GrbK$rJbyliXR@g2+-UGRJuN?kE!8jIdB8gwQb z?F>7hyMnUmA*)1)Z#>S@P4V!xHYm`z+A(3RvbCs3LI*O9!=pr>53$~2oz8P@m&GIG z@r;oZc||>CM#~7Y#Hi{W&0((VL%!Mc&9j<2OrQMV9-b`40=_XD4FFfax4Kft0mCNH z!=wjltGAw`ir4CUu_CaNBfnj9z)Ce$p+$-4S@|OJX)Dj;b_R(}{dCzqQ>>JEXw-Gg^ft8P{0%GAmM9<0GF2wBIfYiZKgT!HgUO{* zxu@%Ux(L;rwrOB-JhfO~%58;i>Ss%(u1}JeFIx}zS&ayg2c^%8q7_tjjN*xSOw5a( zB|F#f!CYT)sva3jBgH3GM}nDx^uWMnV+B+^K9ax`?6xx!73yQ(!2xrrbvyr~hPMBp znO8T%>U*HV-My(ag+YLxih7|QBj%$mTH~Vn25UN)9?@@ysK1PRya!LcyfVJOm;M8c zM*p0cZg6^OM;HFwskp!{`aB|m5iW)#BJSKwdcn4x)j4(8!j3W1H7mK+-rbe@2#;wnqBL=V+$SH_vT9t zxQXkb!2pj92>0{`C9Ft?u{f(zs3tDMa!4EFYNWKx-sV({q*SlJ|gWCk?jgw@wqcMVsj(j2vvQ_ zdKeitidybv%GKp*?=@QMJ7{J9mQ+FWvm;?n3&yw{GjvW@k9x0tqja;KS~QeKNHLw} z1fp5dN%b~(d@1s~hY*Uk(7kLSF`~=|3>NLWufBX|epT9*7kOBS6XQ3s`%cbQ)(wbt zjwQ!<0L*^M2GtC>f+OvS?6-Y06xZ4ZyG=o{hUpur3ben)Xp$lYmoh|L-|ji*$Tk_q zfkb}f5&3kElgTt*t}+by1;CX7{Y$_N#%=ajCC#lSuCFenyx2L4S(#>ZMWlCHFJB9| zN~YI!!tqymFgK~*jkB#phabQFs7&)jHEAQ@yYSbVO6A*MO@tJnh8}juYMN>r*<(k7 zWrFn1fj58O@#KL_#zl(r0%RE2&IA+;(UoJNzq0{gIlYdVi+)nt#Jg-fFJYjIYqnn4 z&$uJLAZgW;MoumNPhL`T$7}GHw3;h9x^m%rB}fI8e5dF}jdGf^*_~}{&aJfwk>1&0 zCZa^INAR$7C)0f{y2B*>2de3-#d+IiLU3|qbRKocJ~+o>GaJS^EG1fgjv$)nXiV*I z%gHj$=;Fdzo0#n(T@Ze#-@#!by3!g!DL(Ekj+YR`r*+X$73O>H-csw z9o7O)j4w2c^Lg4oI1J{m7k?G^*ZJ%r>6yRALFcggXsKZ z?wSgg6vq)m@dWPo?bsLR5&}RaUm1$dY@LjXxKhD7dUO_=pTBDsy5Ymf+w%?OByxE$ z=j->JmOb=-2^av!+GRKHWxxZN_EH^8g3*ZK{E^p^|Y z$rT=2N_VL+S6@J$)l^ro;~84uFV>1aPjwV&FBGN2cl6>j6V+LIMC=<_v%#Rw!k$4t z@+*se*y_b9^siOS3+Y!Mp%7K|yU~J`N!^HNG8WFtza#tqHj7U{%U3ECDTKqJq7wYm zv37_hgv3q^jiab8>D#18FFcqk^?-eV=p(WDobMbwHP+`!;;yq5sh~=C6Wl`(#)fUnHFC@9xo3sa7P_|^cwVeY;GSQ1+Akp8(em)!Z zI-

&+fVx6DtB-QV(H|Pl>XC z^Bb3eAGS7nHndG48Wv8ql_Itu-%i$3YV_8^()07WYeXL}yQwHJOk=MW`hblnF4Ye+ zEA2P*VJhs%`$3Q#!=3x^@c47}RPLUp*E=yC+7S;NOe?#90r`>I))YVAG-3I!O*n_< zT8Fk~pzWg%jyX@^0o4cRCQe>2w{KRIs4ACHml~9wza8eN2)Ezaqrh@{g`K?%jT#Uw zso4<{8JPqdi0VaFjdmLQENLR%)1Dg&ku6Sru_&xlqAekIZ=EZbZcIHe(r-DrjFGpIEjl7 zza^>U?>kJXmsCw|y|(6AbT(d^e7YLIR=*1bv0ETZWoFQyN+Hq91_iT#8&#FH%~2nC z+iRPwPf5vcSi24@Dg{~gl6i!!KQ$9_rie500E^7u9xN`Q>o$;WbgstjXQ#_(BM2f_ zwp;^>pVLi3sacaQ{V1j!uL|`?dl^vDFS^l3#{zQKC;ipx#(I6rWS&FFTe7qdxHfJg zK<~H%`#WR+UCx$EeS#lcpnOiFBsXF_V#+5dT_!;?{WQyi)Uk0B68Kpy>Xval1h@Cb z5>fEZw0NlvdaD#oZ?Tnd6p&gKNPXr^dAdFOhm)eBOvgor zYygZ+G^A$BSaE2_4w%q=9VN4eT$3ewe0OE(p5X9-Qd>{#WOI6pR6I$(f?4|NFJ4?p@AU7Hu?AQGiIqXfunz>@wK9+v@XB z^Pryt5w4p`Nd)KhY^5s@_#*<7WXcKvZb)gbmwKE^_jaGtlPitK>(JHaHl&TCjn zUxBLXsICl+srOCL1J;0(-G0tBMlSizOgFt*k~Ynrn)FfAVeX@cJ|4|kwy$R4?@PO) z?lQzmOUwVr!P88w+cG>>hwt!cPa{RpPS5Gl18SC;MQtn}y`(=*sTWp5?ME5%{G~c?51QVyx$uxk;0w&ohbU3h!qA>wPsA2s4$wJKv6v z>?v*~2}xy90q$%PO(igCfJ>`!&PyBh45>~Ahi#Mtqgqbe0<#^r;(qQ6c|j3bWHn;L zH|hK4GsKANql3y!uFTN8o4{Xr93xe&9^_bOq0|jH=x3!Zo6b?9KX7}{*)n_iLrbq< zsXa6w0rD+M>_Om{WLWlxUN4dSvyOz@%q~!idVa8u6m)fBGM*m!urKij#cBxYjc~GZ zI9L5ba}H4VM|@Oc^fJkGRLkp^eG;bZTiRa;L1ps5hATbMZteUJm0??I!u<>{SObu- z=-$#1QmEib=-BSPNvA6x9JxxBtGKf;fvRg^owV20u`l$$1!Eefmcfq;$@y1 zq=va2$|1nO?4YUQM$*pOI{U)sxl-1+U*8O5J@WoKt&w;r! zZ)VhSMl9((*qM+l0_OcPdGd%u+ypd`cP*B|aYZ%9iRG>{T~#j{f}v0v1dj(BV(bHOyE z4DMjYCydU*f9VsV$oT~~Gr8T0o{pQ-Q*?N6Q=2Yj*PHr&T3=0{Rc1(ZW63zqCc}{Y zB?Pklb0EV|lJrv5xi(K=>W|dlWSIEH)B!a)Y6I)_+vX9WDNAL`gK=lX+q0KwV?G4X1lj!`c^Swx;)ZY(W*+1SS`JnDd41tJFy z#PM13rOJ~Go4`4K%^8g5JD-6H#pKA-q7dj8?KGF@IohuQ63~iSMFBt zbn1&qjSs$C<_xYER}538higS5X&^w}zB>SDg~ojA`%v^`+Rg=mI>vS6SE0GmAcDs+F}aYsL1pL2_V(5XKYjd~=yu9Wn}+-s-_8a%&41(LeOsURXhdWcbY*{@6njhX#VIui(w+(Y&Urp2c#kH&lvf$Z&^xC$2tG9XTY{gk^Na!BKxPhz%;L>jWtTw3aC0_0gI(%F~#VF`^PY!FXH8>;e89q_y5CfZKmQT)lr9%CJ zA-(#kvjR}`lPae67h+MJtDKb!lqc2eB~?{Q9e#ii=6&@$+pWyT$tTUWhGz zmCy|4^pGp8lRDKjRgS^HvNvX6qDvM7+VA(yTGcW{wO!L%dB~}EpD0t<4MJC&xw?7l z`yGiN`mX$d<$OD<#Iav%!Wy%e+S;Ru!M|)?%AdtlN|AXCe0SPMZ8H-=Ap=+_TLoU@ znP_XJl&Y63(m6l63lOgLI5{TR>Z#{1J3h5=$%!}>#;2V*v(fXbfPVO(58i^(5w$Z6 zO!B1}{aVHOXTdx4mC^L0%jpUAFUlc*{B0|UGn#8df>PxB zbt|B%2<3G2XJpH!&gRdbEB?6!t>f|VO>3X_A_p~|w0D$qJU@7I$+8u{xJF9bT|B*U zCRc!+vS&H;^hZ|DQ!@|6O{PmARWQvT^|5$*1*ow4gaw%`m2ZkkVKA#^{;ON>3AqU?0S8Xxg zOZH@uPN3v~-Hu(Zs$*aIzb-y~paJSw82Jy|FFJo z-L?K0!O+w9fALNb>X%0`i0lDW?tIB$h4*$#-u5aAk=7FpLgQ9C+~Ke9{qY{wqw19Y+`LT46F^O+ zuFU~tck@wD^zPW{&jW;?CPz15qc>%^En(u1_RpZ-e(5OXbEpVtz&PX6(8R2~Q)v6@ zgUpRs3%`)?!2b6aT{lNjbS-H{mrY#cCcx;RuQfwo4+6;1qQYpe{wqBqSFZ_^hh?K? z(l~lVOvtR`Y@L_I*pjRtiLl~N<{l4rd-^iQB7Lb3&?lbh2al0!U|iJ2$fvuKFO!z9 zNklCG|LAe(`t?6KI-bemkCvF;zNNF=JB^eFy6zz(!4=av+VmJ~7hUjP(JB=me3#O~ zYt563N&2*-iU!e(G?M!E8-v@)@r6g8i&tOn%F>O>C|qF7M&us{_x)_5*YG#nVH$<= z{mJ+JZ0;`xk6hU{o3d0OdKY4Q>1~%GZN*g+m;6@49Z^xtH>$U?gi{QN>L+@#&a`S2 z0(H#jymQ#Uj990AnU1&g>=HpwFTqX2=y%wBsQ7|HGEgo@K$lhF#H!;DDA!=kgIjJ+ zy(^EYPfu0_`GZx?66coH%A4=(9A3Qd*TFfzAQRV|=09TD7~8V^?DWJnaBfXfLx3@M zCxh+e_R1fX`cSdIM+ADP-{xMpXwlhc9F3E%U!nPZS2~>xN}d04?Q+XeQJ497~Z`mf1p#;s- zv)Q=~{qAxf%LcxC{G_Fdy8L zIyGvy`=Hd1OG9l73(V?g}y0*!v{p z6BKyV{$PMg>J@@eUp$n-z&cSNo9LxmNa_f9N!hgJ?lX#8|H)Qex>i>EEo+*J2xLLq5XYOA+g6S0@>)hmN5t@B-o1{I7dG z&k2@dyUj1%8S<)k=rZ~4(w_&+gxay*`;)pn z)Z!Hp%&`CSSdT?^Kby6V^uuVhKA@w|YEpmnbJK-C{j9+;!(VJg9MYc^LpV0dFnwk! z(o!cSP0DSnKP_hV|Kf*U`KO&y9CW`_CXll`l1{c59ZX=p@+X-O&r!&$f5v+krgJ94{J%(+zF+t+e9|+nzq|A2%>}>wvxds(VDG3|M|IXqKyhSDb2#L1?QvQ zYC`#`t_o`ZgkjgK`ZSQKDXP9fFJ7}`$>kr&R_L27_)_vO_~h=l+=vPw<(AwX=KS85 zGeaB%pIH>H^i23C1Yxjz`@5;?Z6}*-xj^(eY)xc>4+7}R?eu{Ovvxsi%h%9)A7|=>>$wGrf>NB)sD3^I44wU z_+fS0EF{pnx$(~+wq}ZQ4iCKxNb8YkVXC~GEnT%nC!1&2TUxO07iqD@FW60M3KZ{a z^R_ij$30W<3C9WV99469^Xsv$Yj=|ICl1Xo_ytVh3x&p=?k72(&JKZJcFy1r`+s;_ z?S_B8RABH$RdK;tf-EC>Q4;ex#-xwi`(}wQSH+^Hsbo(W?3IJCnYjVoDSJ~`2 z0OloyATbg90)eK)y}6XwIdTV~f=Y+KnxcB;rmZ>O#szeCy05%KkVH0JnfC z^1&mhocniNVC=pFjFh{-h1!0*-+B$SBGJLx%Ztx*nHxtJz6ysj`mjzA&WB6*<4WrS zNIITbUS7)0W)2UtJ}gdi-bWjCcHA31rtc&@*B1TU1P^Qs^sGM0IYepXpCr_8F;6Qg z8phrr<_Kvd4-mRmxmPsIe3?gO%`SXh^xnf{z-uQ39IFg0(2AeMSh1`Ojm$(>S5Wa9 zHG37&{+SbBj#YHg7G)=r>G)>ens^9Hs*mOo7O_ehwDt$;$nThz?USR{C?fp%+Kzs_ z3&^@}`IK2lzua>)&a?Ue3MVu&eFC;jeAlgF@$MJUU|G}a5lk&nyHUvxP-_3mB0vV! zs9s#d;uWm^;`8mwjj+a~7L6!j=wxpLMIxBO5zRhbnVCn^Nbwa7&{PUI*{nC!8#E_X zI5(cHGwp9tq2WqG!6;^Cq+DS&Rh@-l${Sib$tnD_S!r#1eWY0{yj9j}(`91>u%m<- zf&>E<;jkiU%tS|#F5)!nZBroR)#>!lQCt#HhaBKOV6kkywXkqo+8X2bKGCJObHBGU zcsWTCW2To^(|)*wE=a<3k0(iZwdHn)*GYi(U7P)UnCa7~TYVe#iL{5xGA#FK1a9lh z7;bj(?YwWzJ&}y^YI^-JXLMLwqO^zSmYyLQEZL?}%<2t&*O_}_zTq2_wsD00^c3Rx zv*PNvWmHE#FIq%w1QtBsE3-e4le8z_G3IFMd)gR)9+6vS-uan4Z@0Gia9*zyZ)8pU zH7uUK1Crt$1DoS!Frv5ei-*@-V(S8a%^+tMK^p=(K(hEGOuMWZ)>pCuFYU;_9nvJ= z+cyIbN5YI}T(&~k7Z|sYIR~+~7jch+Tk*E)ET?ZEooqOL)*xkm4drk^0JWzCq+XradB|;*cB!(Zd2^Az5}w-^d}B3^7pTE1;|Q}I-8ZCv5EJyh zALj0D_4>H+Tfkk;UpjXISFtst%k9|fq$GfxfPIv{vRQS1t?I}D-5-F&p(azxn?F0# zlsZ+<)Q-~+dQLpsYa1qxb~ae5%gKi&B=OaJ#PgC@}}r{iL#?9Ple5EIuK z|5q4qQ|%CuwZImCO+T?v`|V_Ib74wwenVOB#C3cla=+hcRN#6vg8 zfz-$If*_NfbP#EJ>*7WPJv>;erRcE$^^94Sh#h_rHxa`Pw4`--?@is;nP?dD(~TR; zHdiGr@Jd3oa4BKiRJ!yF6UKY}HMfSIGqONLk+Dyv%qpZ;xN~dk;jWr~wP$TwDYvOx z+KM-FP{Mu-ZaNnwMAqLZ9j@wdZxs%d(}HY523)=#)cCEme~wJMG{sfDaz=i@fpKeP zaEQ*AAWE4yZMmgcWhJqGDMI!sO>7dMac#N2RVWqF)rhH{r-KlDV}xNLp)rwu+ixh> zDp?KhaT8zFS#V8Y>&kjOAFVpl`A3W)x`5C4x@3qMgWJ?vWZKB)SkjS4kV=@|kolVv=`eiLki)K<(J-dmv;g^}}Wmy0L0TmHJ%A$7=3%yA11c=m7qyz|6MNyDqp?5)g2@n!m0F@FYK1KT26E3fFU(z}RfNHWQDC*Yj5ot~_Hg*<-Yhf)d#ktE{#X ziToQhQ*Q&}t_BosA{MAqa@JVuqE0Ev*01kU(Rb7oC+wEgA*p+K>mEE=sgiz z+n9Sm;L(T);`z2Gs&&B~_KlDG_dI6KV}&CrHTHHcPSAFECTwW2)Fwj_eouskmkzp7 zWYfEgSi>p{T@5xSyILYW6UM!mR}SCJDcf|$B;Obojq9x;SX*Kh}srHgc1D-}WJHb$}jt>VR>KAdK>PpYVSFgLQK zMqZ2~#hbAlF$xUEOg>6@PS-^=uwWm87B+=RvMD%sZV1OoLF=F_mVu8{KjJ8I#%Ge|D<>p z%qMQGhL0nXsk=9-PyaJlBqw?%o%L(V-fMJ&8J%{dR6r72HC$6D(SzwthFV@=iO4JA zA`Sbxny%mzg}1wVPt#30OL+Orw4MpWKXWbA9OEHO8b1_TB-%(e=$&|tD2?#UIh{|j z|E81zWIMEN#V&4R5#1 zN9ad{2fJu^6|!5kYL#$UmW- z4U-I7PDZ`yx%*z7QQzg`BeGd@HK4sMk@)!Bw(w-lf(;_96=vc7f7(I-z_6`va z@t5r3E0-9FFbA}y(Z1)^pi;km5BN;J@sZuD;2<%y$DK+dD@E~s-`7_`Sbu)Kx_Tp? zGgYx`dseef!U4_aA(7@Hc2S8X8Af~0?fxFM8w!AtqGBMfgd#4uZlW88E1huLw$J1h zI6T4{e%kJRWzIyAu>(QX0v>FO&w_iqtxN${ z*eJMbxhIWH!>_R_u0a$Y`!~){SsW?uCr<7_*8-WKjmo=;R@vyv5}l&0-iB@Wz37Ou z4YWl;TUy%biEDFEwY6PQrau8+P?Q^(M1+dOLnk4uV##*Ok}gBfpbU;dl=37tg9mc4%;3o^^JzQ7HHi6D66<>7q{7x5^a9VbO36@?CjnMYsCa0g%^ttA5<@On zsVJMmxCbqlFYfSIm(HJB{;s&T6z^H_%T(2^AE=|!tu#l^a?YpUhBVJT66?w5;gdK@ znf(wRLg2}78~$)a^Pfk)SNaQ|brGa$%_CN{w%i4Q@drCaP!lDyx}hKL?FA3xD(bRe zPyMCRA+i`{mNLg4XZ5FEbl{^nb~~SX?QLP0C3VTa71kx=8D0Pdj?_Yv(K(#P-|1mDxlv)& zd?VSdb^cw*#K-m?we?+}V3Q=4HB^^dVz6tHoP)kX`msvW7ivAESqw^HR12MwC@0fqMW>jQFg4T$$^s0hGeK=-tv^W8{NBC8&38s z|4hluo=A;qWRNpAnS4jAIz8d+TfRS3m{{hdpra8bZ=)7umM2avug?s?RDWCV^`T#J zGUPqIysI(K^gjMh2Hq6a;%u&<$=fgO!sX_{?S1P5&&PY`;|@D|v8S7DuD$n#5gNM< z3uCjd zpOW8PTDL~DDd|p$H+1le|RH@_F2stmDwJ-EH1{L91TajQW=lGny~)C?3lKTVBX zaWHcd_RY9mJ~=aX8gAEgTRY9_xBt;F5DXkLGq?K+VOYTKL(%L}5$x_B3T|&PvFQ!7d>Pq9axnG-G>7^TU>8#{mC} zJ%-=&@j!Me%{y2SeO52~M@hU=rT*mjBz(Di_NRmP@g` zKZ&dh+N?oq*wSL=v7UTS-jhv{A_g*sPI)6LdT7@BIz*w1_x zA1R_sJHEKl*2$6DFVa^x-nI;UFRv|WDlkzuUzGamgS9ivrf4Gi?0j<2qyHEPYyrK-%Rkc>XTc27v0Mm^7wzc_7B{*D$WN$Tj(pmR?S z-jVupI?T(?IQxz0opjIexQimrIl|j?Iaxbf^cT6B*@p}Td#dNt6vktAQao#q748_8FO-myy zLt|=J;|wMdT^}^$18gG0_LwyG&UPV|e7Y@Jn!y$m7FP@6cggprHo1q(aPc*3pQa0x z{JAku9`%;wq6$9I-U|>vlIV7ib~+iaPYW--e$=*NwHg9mrEdR#*X+;R1$QMChSQiy zi&GdxO7z#A7p6xG6;ul!?7tJ>W5-Hgrm7f)eI|Eb2; z%f~wk)OW^#E$!A%9Q3=Se}*_GG@Y6cTpL&ro1pJ>A#6EIWd{nqFwgG6-Y<2fo!-jv4qG$2`eita27@f z=E|V-en5WPb5cxsafgnxs*pnBE2NJvW@>W_>u~vvah*zRYH;wR*sRC$#~IY=ll&=ijGynr~j)qEIKdTjHlmkoHlm~sL7fxx8Ry!_Y71%&cHPUaS{yR_iz zipf;o4_i6sMOym)HH2`nt@pS>G-m`%H$y6aBZ0kwDkmp|WFaVYWZ0PmBBXZu(GV^d z0ZrmXczp6>)_8wcVf(KSq_~E|%4?4rxtWGU3G!}+ufu8OSD;;F?-XZ@G_>yY@@$Dd zg!hw_f>)SG%kD;fVTP>xr_L>hjq}4z#+*5m#UflW+#t-bED7ygqTc|<)M|kh$osxf z_q7#-ae3+ZV)VhbJjrze4rb(wH!wjgtLK;?ncG!zPs(8Vg{rAFE!sIpuEKV!u~#tYN&%@t zsZwnJ#%jvNOS9u?;y+E}R|2+*o8*HzbX!$2s)w^w%B*V(EaTDD;WV^JKxMC+ef=bY z=eF_uE^kBS<2ci^xIkcbC?Vm-Zt{X~dXyo6ea{BsT!$oi#9e?(D1p7RGQ*%*8VCP5 zlJ@zZK)4wN>_{zfFp4x_7aN9>UHfa*h4R1|QEdwvjIim`yzcgO3RvM7M zjmwizTp7;^$cgxVxHk_=gI!=P%47LS>>4apV&UDJ>3p?pWVbMI>0^${qOp0a%(_i1 z>H44=-0Akj{2}F~SD7u-4jkz$RP8LdH^a9v>(wr|Fa`Hf-sn!K<%}HY8o23TbFVX7 zm_%h6aW%FKr)Q;Lm>(E|{rI;<3jRH!S~$EewYt|KLG6Ax?E-PskiyZ496ddl7WrzN z#ua&Ec7lmG@^73C$P#&ONp_mxz(%j&zYgUKwG3RxTS8xS`d2}_Nt)^cGwkXZP1lrm zkJf$Xht9`~hZfIUndF0oXgy&(1b*Fox#ig=+8yvX8CWGrgpDMX>wpYH&$Aa~GZ@0+nyck*{zfH}WI|-=w$zUAxMv7@90%4KK-CLB ztiFB0U}=DbIzE;ntVirAG$!ube3nWuoZRL8T-gz8T8#??B70jMvH=k!HeT$7uVDcq zS?@?V&f+p>U{6R=l7JtvQ&=lr?cCY5$bZ8BE8swwJ25BNloYu0Bw8Pc280%kSoVbq zKfACF`?=rMAo^6*o8fQ6L=O-Yqj4?{&)6YUOm8mF;!*?Xzt}+>+|fPOwSmYzX@Hy> z<`gbC)iN#Z%3tiK>Uux2r;u{Vb1(N{7!M?!ewOrzc^^2cxa`AF?g@sM8iwv%TSBli zV@E#uCR&zbrnYR9@q&i}z7y^8JG8Cdy~w!?Ts>jLSEAGWh}fC33hFLCTt2QfAQ@6H z7|6yyIpR-c+rBNxg&1CPMT%hU-?6_7EH#c#b_uKO6))vH${#|ucDyrn4y15RHk)CuMNr);=!4f_z-{QjkF_oa_x?n}!vF%Wwi#9uUv z>K>{PdSQK2@XMic4fv}KH6$f!p)@sj;aGeP6lvPze^&PmuUDc9sP6znOc%Wn;{wH`D5UnlvCZ)=`u8Z(dJVV_Ns8im0X@aub@;^Dc{hs|p0M3kv2dlz<+nOf z?aunqULplE+UG;U)-%(m4rY~xroA9m6nztL%;A^$>QcAi+Q;NgQEEJN2~yzpZe04e zx4*KM_dB+}vDL9I2fRvp|M%&I?HSjSY^BY%xO9fS_UnYsTP%P$1ZGwo8%xS;*>}97 zCNGgJ_{@L|%0z*%g(@jxEVC+6v>L9?=auEeh`B~KX|n>KmNGxi)N6)KP$g=MxI(42 z(PytVHqUf!{E%&a3-qW)97(_Nj~S_5t9|09{${X{!p=IhjRrznPesMItV6x5#-V0I za8^YZC?GKLYND#qlXB7aLiXWFo2hTgs*Z{YzaI<7K^IJ`pmyD$9ThV$<}r?p-3{n_ z>~QhdHS)8ou&?23KJIxP?p>(v{uih1fv#O4t9Y2zO^ro=$^>?2x;#QOLsX-gQc1#-g zu;36I$HxC53ZEAJukFoXE+WV#2NjmC4j~%KeL^1I)?SFn~WQ zG24_4?Ox6z1u(n%Vt`7-maZce=a=OxAgZkqAU{$OpbxhFT~}0ayA6MvtXM&KU#Y{19E-pVKO0*kuiVg$cGcffTJVhehk& zD?@X8(p_sZo4JWml!(A&F%jVirf_U!FeFsm}KtWK?m9s9-O3E*ekgxUe&m#xJ zRc<<=s=@D~tgN9`i4R5w{@e-9>lgoNXAruQ2-*|FA?kgE^oW{7j%U#*WzxWRU6X-;k5mUN?F?UNbiUlNnqXyd(8$K64=U0m4SkWGC~5Ee=Qzg zwvOE}jXUZmS!G(V(nB*8w!1aUWCHi+P4gS?sDBx`NC@Mjd=%-7y$}{A$wMo6OH=!2 z30>ZuQf;AAa&A9#nhK-YDjywCZ=ZZ3`L7hd1tVnH#(iHJD1cXfx$fP+`p=u*PiU`Z zI0IT=*n|fC(51gwtbhM9SqfkWSqyaHb#w52lyE&+-a)%U{(mgw*s)|J6TpSiu>{te z$nO%{lE6Z%5&p*=9y?Zb86ZP_y=?xk%2V@q#>Aw^y zXg_gVP2u;b{U??B`L%x#rT=lszytTV`J4ao{C~yL`Y*lA|2=}(|Nqf|Sl53iBk%t^ z1AgNam9Jua0^mT*cd&-(K24Mb%C$NGL<7JzExRtwt$f$wBsoJ#AkqPU)T41uTL8&! zD(5&i-8bQ@{dNH6&2!W7Lt$}=P%S$pWoH2>ZwxV306H{3Wz~Z+2&Vl^de)5Hwliot z3=bIrZEgn^D2Z9)FUj5+l1Waaq|vbSv)0z3V<-%Srer9@yMfh&jdd-^zok`-B^!=E zI`%9A-1oqg)2Ub0DegKUQge@YjooSFq2sR;y371lzfL$+gZT&E$v<-KU1>8UdJppI zn$Fyu>NeLn(L9bR4^O{PG1{B5MRndF`snF=?A*E;M;kBIM%z%f>%PC~V;#2?Kk852 zNE!3dD=^fj=(~lE?2<+`EZe)1t`{|^7IuylR&?qp2hwHHBG-{UEfR#*p3X{X z9ZfY{typPp+46dxa->9p?-s@KZ=*BSm;x7Y7iMsSSlNprrO$wZz`4Oi-P2`lHRC3Q zFPy`+%ZK^c6Jutp;up&;w&I-cBv%OBQzb;$i5xDxMPlJM4RAa$E<*Q8xkOyBGkl{? zPu0`VcI86D#e~p`1#w8f9Qmz@nIKCPP`lM?eH^NENiX)17O60ePQf-X?jWy&cG|m# ziwJS8Qr?Exm8${ujgPI<<62RwlbV?=s<(P;bZC1=6j{Zy?%w5+jQi0^8@G_j?8Y$|nxz#N8NNj5_w)78<(wK8UEw0`YFH z{nZ^s&Xo1DE_+JV#U(7=0RdET+w|1FIRL;)u`Rq3DJht*BOQPIm^%evLVA*e2i+6L zyxQ}L>1(dSka8r8L2=pc6KGxTffsdt&7(CSzI@x8Ib;>LWJ4&{>yb>)-uAwJRU_8R zeX4kQi~~Hk6o36fwRU>#@8ZB>5A#SJhEOPa5q>ejzf@gV6Ck2CBhLM7&!mMS-E9N;rcekH z^CBt56R(ZzbPlxu)IpcpGpPfUA|9%^#~K(WmT!sii(<8F$$UkA+5pQ?$L z_m#`nr)KL=kY_OFnDY!$VG5ov(Bmx!3wCo{OkX3qBRlXS2AWKXJU)L%?Vv;{oM|Fw z5^tfErTLN4QqA8L_6@0*Si|FP_?i)}2g#R_HV>E5(;~l9$fvx5!m#X92tVZ5CN77r)`u6tD8^V>>qr0Aaw>{rKujxgE z7go!L!{Q0fF*!wAl(y=@n@Aa*WgV)XD!KnSH7s`;IAU}^(R{|6;o0(8KF6OojTYf` zrn&O2c*{ZduIZ44V}_GMIpRR!J5ftBjVn#8i780r;ib@sfI?dzYtmYi4kCuXDJnNW z`!%^mNX6iGNOH9@RiiP-Y}ez@8r8K$KLRwV<7qu$g=#&QjNiX%A3O#Zt1XiEsZ>O>-f!q+UQR41zkLO2T!Woe2!TSr2i*K>6Zvxvkse=WU zI^}*<%9@X|3;PtqHZ>*&brP2C?OR^!#(C#B@$cX3NL;b5@wKrlJm}R>;Z}FMD;7!R zQ(OF#THsl*A+cagcu_v;d0;A6BGA8!|moBa0RWf*#zyJc5Nl8O5l(;#1&^v2gD4-82<7r45O3!vL0RBw^kl_C3bIw~cGnik7-c zl#~EBBz+(#yZAdo_h8p_kFm0Xw-UFXsceeZPH*5;^nR<4tCXfe4JSL`3(hW5&4KN^VWjhy;PKx%^i-X_0vV|dQJ6uF-drc2D!(74I)6u*yffb)Z81yFPD%?z`Eu2o6_CIE$d>uEV1ggH z`mqvnebkVB{g{zfn9h^9hrPb5muqk?m?}rAZq%$OGdQxbF-c>j@o3uDsH}h zg&5Wjtelf@I9jmcrZ+ak`tU+{y)5+`Yl@84z|!&=vHQ+a6|-v>@79ec(@+vdpRvh+e$?rF>?S(3B;x-fuYPW?mZf;cRx@N{- zdS~nN)R^|T?Bc8Bj91mwHM^b=(kSWeHXW=G2CLpw_;7bJa&vI!Rl`HVZR#pESwdEy)}*bIFvD(#nn&P)Ed8Gh*iMtkExIZN?NPOb<%j0=1Yk&^)Q6{}N^K1+_ar$m8?P z%qw!rIzW!_t7Cf2UHh7W5-e%ZC^SAKLR4Ac+VarojIbmtC~`5R@+u)rP}$&iL~?aj z)8e6mb4d9Bs@U|%!)>RIfa+awWVRgozIZO8h-W!l9B!CEe<+ysa0>s0HQ^Q}HId-y zfa4Kg3mXnP`mq1o(L(`8FGJ@@%|7(Gd3`xhuJxWh7q_jDd4Yt5Ha=JX&sKOttk~_Y zt~_K!NJCtwHWv==p&gn1vNccJY+6VN^TMesT!k+_ey@{Tet4|#r<6j&iU+-`MBJxm z>!{{wqqljvV3u)QN_CgTL=f>4t;}y^EC0B;X%|xnQ?7JZsQUSc^Qx^ohdUwZsLUBOtcV&(=J$ zykmKw4URzV(_U|UarZB-D+_}wM}-!g@{3ADsur(j&s^amFS2d)I6U6C8ys8W*d+3; zscIa^M+N{h&2?n9(#(=Kg3Aesf4lk|z)8zbmj*_b^jAN6HGJ)CW@lFbf|h>amM@<; z`cmEUn_f8OyL0^8($OEfcc#vhw#tXCTHo-{I6)U^th5w|qo9D_u(-Wr;7 zc;2ox5Cu{MtKK%{5i2w9bS#9LJ_>nnAX4~+L*U`Ry`II-y?RnV%gC4Y$1d&Tmju`n z;w{Bw@|sPcrMG3w*jgv&(Li4mo2P$!#?D%Sa{2bR>gs3hyPKt9K{wV5yjEc52S!B} zMYjM6u9$v4U}NQtV_J{YO!Q-Q#P!N9zX9ZOztQrwDH%Fd&C7(&w|w!t$9_4CQ!njHD_YAtDJdHdMqY7K)FP`sQ zRK|@d-kR{6h?D|YB&f8VkzAASSAUzIYgziTQim;JPzIO8?-2=ZI9dI?S0-BIg3U}hNM>d@Othzy`MJ_m4*I{u>^$Uue4_ks| zczMjqPubf)4>rRC z)B!NW&5qd1fA_L2AhszUZg?su_z2)YEZ+_*-MF2*YD#6GUxFR?aT0DA_!+vSVqFn3 z_jQAGqln*Mhb&JILU&MwDt3LfA1~A-FqG2tm9)t!rv3DSssQUB2#tkp;j%#HW6z6I z-9E=$H#UZ<@4j5|QiBmLAUuMt6ba)tgNPXx;c$pd1D<;eL=1k1Lw1+hhnH-}E~dl# zPdf|^@Tm&ts7$w*frMb*8IBe3X_+3%UE)Fa!jn;3;x z2Ikd!PWL{=Ly~KQV54St@<5+Ld1C-P>eGYxie9&`F1B^#U$?U|g$8Ia#b9***;IH839)J#U4TL@!w^=D?XNHU& zTS-t}Ta5C|0x?)z%v9<(wbJoV@;=O?1lj5Fimlzv)_w1jdQtF7rIbtSKi>TCkst?K z#O-{&|7#!K-;1$K&?beCleE?)45Pc!MDzusJ48lO*mgo54*6b@l_`Yse(LNUrGQbr z`|Bx|K5&sS<>@AF%Y)6{&H&2T`cFmD`Jr7&=FuXOY?sN_s9Ad4-C6SnQ{6!;%;ze) zp+#m_WwRTINy!ea6Hp0Oa7{|Piz(}DmR8HsZeG0;=M+x9ngBN6U9|_HNV)g27B#v@ zFYhyR{TafS1&Srv4$LCw%LsrKsWy#fd|{XkI9s`UW@wKKqxiWR#T0UQ$s~9Nu3=|K zQ&9DgFl@%a#gl5~`RsquzxR=1E)&T(Z&YwZUG8B1rB6dU(*ACO30t$#U}btujd>59 zY7`&Ms$yWWC2X9r%L<7Y(dC!qRFwqkNoM{L9J>_b#l}thzHa^xV_j%vncf8db~FnN zlV41eo4z)<*23Nq)OxjHl5J=85;E4TDSLQ{`^2#uGR{`Tn{5&JXHA_7m~M72vE(?T z?R_J?nuJ@lY1$Mej#ECklG>Qoc z_?;{6zZjQhthy~9uDUXg8FrDHa=wkD>i%5GytqonZxwgD1<*ZV8OFT|Ox9MwIqr9Q z>N(#pM9!9bs)lGtjcvSL6elA>JOX6@vL$^o%ST%~wL^P4H78Eu=Uh`2X-y?#2bO>I z!nt%}w(oAAEedNb7v~%Wo|9nF8w%A`zSJTUVS>tfs9Vc zU|$&#A!YoszdzNua``b-Hi08+)o#x+`Jy@#kVWkE8@&p6xD>|sAeI(){_G}BoX^~S zqn!i)ZsX{=02@?+ z)6jcs`iZnbuBL&dQMR1JN2+y=+$VT2QasFMvbLVP8%Wx7z9XW|r{O1_0VAJY972qh zX*)B~e(eAJw!=Zf{51X7{sIGkENk}at&o)TB%WtXo}?(0)6oTH32 zMrRIZbZk{sFs&#C-6#mM9F`avvP>2ww;8(ltqaOJ=rF%lN@WS^>>D`GhYP+wqmbYE z(|x#YYpe^LM-?xW5XM2kX78mG&o$5b&suoku>BXog`WWV=5P<)&*89I= zZ7Y$CIBsE$1pwPNT~|g^>a_oe%(Hs|=@a_`0w9o5&$j)0kb%-&plmUR@ ze`0Bjjmf~H+kl#jMXv{>$qcIJ2{a|2nIl?Wp0k!^KXzt%m3~~SWtI(Cq?c~TiVpPzOw?jK0&P~k6AeWOM}D04n0CkHc3LZ|IIM`(^);N z$$Eafu`sMYS?7D)vOOskg9FZ4O!YgqNjeb#3wWZ0$PWckJ(4>{kzBg_dEW+=wjs2!+&q# z`PU22g@FST&;8Hc+$T_g?tT!{$a7lc%N$0PAs!7>!V!B|&|9%%|Na(fXFJx;{` zX+t2pk7$4{z-h|;Vo1`GIZlLEsD|og9)DjApB{vla)^~SM;y~>YW@^q;MO`bKPg@{ z2W$;IdSxiado($5*`4r8Rknc5+mRcwWm9*1;Cz}x-yh#2UmY5-a++ev&eRRA&gsg- zjZZ>fUc)5)PA<6fUb#OW*E*0>FE({Nm(x=QzVldQc2*$xsB{DWnS)2+#eDAWA6+Ee zT8g9eonHGieExua`qCO`f=;WU@LYND^ub=3t>mqDD%n&` z{5gUNgKl74!h8^3i7tl)O4SpRcGOraCFYG=fCH!BJRIIc92o5VO39U*OOvJ0I8?;j zY~#_S*0SpzLgyFOD*I@-oE~P5h=%y;^=(0z>2TiJVxBl!}{R?iJiY*}vG@QZXwyS0^dPNNvBg35`(M6cE;_h8*) zCDOGKQ{a}Mnx?|+9LWLh`$?gP+2O3?&Qu6!=xdw7@|b5Qs7XfV1L&)SY;D9@l-pZL zi=W#|?g=RM-IyM~ue}qP50pbQru4nlO-2r1Hetd#IP|D5rl;sl_+~kZ^TT7V^Z{>U z*qW*yAdmzj+BEfsmL9FkMY(81dfu=Q1|*4wSN{w?NHPF;Q(E4ui}LjWA0E?*%nHHa zh#*W&lkudmIq%Vi)hm~)=JLTr;#(0tW7^BwvUMmB&WVhm8|QOdlXEjk>CnGqL2 zO5N`4v(vsj7<&4w0zr+jT{)_?#>=}Vkkpxx#7i>dI4zR9`ppkwP*%_yB0hPx+q<2kJTZ7K^vIXIYeoV+$l+H~ zI2o6+ZjFRm_nl3)`|dr7f<}*Q#3~(RhSQge_E!f1g(d8>qiJ;D#1eDRc5H;l){?(q z?=eGRmT(kL#I#VEi=7CgcIf8Tu4q0QMB0lGjxap6vnAGXE=NscM{e$in44=v-+iWp zLslj}4?=0~WyhY|j-KW{9FLRJ3L|7U9e*_pF|j?}qsSR`Yf-JD#cAFlfJ$bQ#@}{qZiyI}FUnhCYID3_`Nd^|PzQU5nISQE zM_xVhBrd$3y$|=qK*>pWT)MBz<%^bWtX@Kd%bLruTB5g8BT2P*e?)7JvAZ~`oTBng zD(wCR(1m6Ak@Z0cpei<+-@`WAvfiYsNboI?+k_m>K+H2l{DoA_V>jFl(&&5lIL-;% zqQ;nCW?wKc)?62ykY#(JK1kvM8Dv(4m2YY69*C!AA zZm5&^LsKqZ7jHdCSl|8yD8YZly29fu+)>zXP4(GI#G<6lF2a0+16mfpwz;G{F;XCk(@&W- z-?EOx#=YB~uSX@1=L2$3C*Yh;R4LE{b5pmG__ur8+N1a1RICIKhy3|Tx_DmW!i)J9 zIj**mobA;48{aSMQ#j_j)0|_Vo&DKW*&sB@w}bp4mzHAaX!5m2v6p%#x+t5RC$|<4 z*>1tyTQC{t{qyp}Xz|*VTNtSlk||YXo@dU|gXyqXP0g?;J#J?B#D1%4nf-g0AN1*3 zf@W{vcb*dpUE{YX)T-@j3+U*!-4EzQA2_N#_-M;VS0PVjhWE|<=D1%b z=}BrMJ>LR_wvLo5K=OqKt$+_YvTF4;V4`-0uXT>`CNvddIR_XA@a=o}S_^z|yg6ID zQo@x3BL<*CMwXNg6(0@JorEGatnV$2QWY1F*)CyrE0Y6m=2maQt2eeoc$VoN%VoY| zM zc9yox6A^jWAQR`i9KUnXIqHuMnRE*}kXc?h(OmMw%v#`(mA!_tp5$T?$)agh!>W;oNd>P*ZfKW`XXrydxVkcgml##y{y6hvqp2%wMF zn}96ePslW`B!N#Qgq1q2a)@rrA74MJ3xu~CPSRR1s>)re_a8jsnmt)EK92d}TQ+<0 znNq>z?Mh9yrx_+O@18z+bXm=V!!2dTH@h8HCen^|N}44gy&<-ia#gTf80XZ}9e5}@JxybmU7oGT zi?A{nNnB6wZKzh*4fT>ZF|5dzYzZST^YS+tRZdpptZgpQ_R6O(r?k%rAeEY(!Wc|6 zR8$oxA!X=e(7U%Mi8s7Y)5T0Z6G9KwTvr>P#J-C*90Nto_ONnC_??Y46rBq)s7^H= z3BA8@O*{5;bD~tC8}?JRR<@U7F&O*l6rI(}v@~91J3P;##QM=yt|Qy$GDgp)C^7MC z7dlW-Qr4Hcd!0eZ2G%Imc(QrlPdrL->5XXJAp*(Y>j@ z?Jkv{Q)NO{=adVT28*}0c-sm>Dwkq1YgE+ta_X5U?&Tx(8y+3Oq=FxtPx!7*3BBr4 z7-LYc!4ZA0e`H=`PLkZr*!Wh{V|O~2;0lkh%n4h8|Jc#)ZnXj($jA9^O%daH@IRhz zJt&Hx|7GM}@&x7QG3G|}2FKYMR?xG8kS*PKt3QaD=49~~60-}ynOrC&3DP`mf7|Wm zP95|yW?EU0gMGy0t&eYv&min>Vm8X0PCc?v_ge~mVDuxlB+@JMr?t)wG>ECny z+#>jT5d;jbOE|{h)rB_d7(Glb2T$RMncb82KcAU5AM{xq-yI)&a<=bQSmwF?Ta1&B z-t;D0JV*nKcUw^?v;qWr+7aUapfBeUWz;BFordRKiWNU2R3_g#yO)CaFb!AbaPaVSP;u@KIB&|KBLwU2%&NbD zLTskIg5&Qk?38!z4NFeCml#aZd<7iN>Fty!r1|?`B2Y(!D^{R#f#3r-m-&3Q@v#hi$Ufl^}mk>Wkdr3$gykMg3?KeIH2|8j5&Pe2n_4u5`sd2bd^ z?t^zv7t@s!4xGD3e=(-Y@H`uL79LpdJR+ht6uNzP^3BoHp&!NPpKjE4%hhTUl^=uk z<*CQD9Pf%p?HCQD7Vf)#EW%#|bY8Z#_E~#<)I)V;0=NdfyCfObNSTlQo$t8YqT7S1 z=!xOl4J9w7dc}}}^);mpzdNzp>J#-wuHrY$ZhvHZa=LXWg#hza9Af-}>E4&-M1}9k zfuX+Do2qc-WhwQ;UjN*!MEswXv6fEXL$hGGD17c@C8bUMxozD;+(=VgpZ(%Dv1v2A zstZa*h7?@4aO}ln0j@bA)dnW+KYx^mc;+1KH($1aoou->-!d}SJxpEz>reCJ*s%9o zrbmt@^xg~F&B-=#)?@Tq#4X?P;wOjf5#9_}iv2ZqB`q-k6Y&u~k`Q_tcF%4=%*K`5 zg{teD;_3N?Q{&^))cX=20yD)g8YZ!B^m`hx6)O8`hrIKcS>}&QO90m@e~c9ro7#5y z+!=DYJ@ee+i5SciM`m)t?02nfJH;P#&#TMk?T1m?B{z&c4(eh&zl8dUGp^}NjWogA z7}ns7+%fp&!`@G-F3XG9gyn+~^Yg$=j&(3VBTQUnc9v=9_|vQItzVqIGskA*Ih8ar zqx>#>9U-AZyc*W)nD*K!;H*23R`|1B#p+zR^wCOyeMx)IgW+r1`<_X{jYpuo%@Q4% z&OhEF0x;Ga9IK}A03+*3h}YE4AJ6%}*>2BPqR%H?x2;#{WzE`0((k?&u#J|qxYU|| ziwp~VZ-FbvA@-H)gRAD$g20Zw2N%q1s%}Jpl+kOU`~-m^QWE0x*>C;Cal4spj45fc zobdVRihkn$_S7#%pHB

)khK+sO!FX`g#O=OyF>ZT)qk`-&iyr=hBEH02quIdWb` zFeY~*Hbw5(f|e<+GZ3e$lfmD=u<;2hRB}FonzduLv`R;LESre+t$&Qhy$Ol1nAu6e zLh?k0H+TY7ZWuE2|IW{O248ArhbtC8GZku`eRFt-0LQkTda!vLH%n|C_Tb~^Z9Cf; zOhXT?{9Mc;L>yMDPUwpfbzNf%%Hf13LWpg7iT-2noG5FiN&4haMa9^8V25Zqk{ zg1bv_2m}ch+%0%;*TI5ou;38f8FY|gnAhxm_P*zw_v+nxbzj}OU0pR()!oz6|N7Tj z-}?U5Ye`{;V9-kyjL6w=SyO zG~kTyzWoB_v%ZLSXL{23W9m+`ZXR-GBX1?dY zi4g_WTV5Y&9FItBA9il3)FQSJK4W(2kcqWABIb)0jCmnfctOm8Xz`pzxEvERX9ru+ zGD&Qy`yfK22x=m)dQ;GxCpjNE9t+t;E+WYob^QXB=aIJrUwfBtUK}^#S?9#rb&izu z9OpZ=G%E83=|oVrp4X}(yP#%wj(Hj_X7 zPC)LiTkav^vq9q?)ay%_6+unUPKeJFRBa2fgdD##E8DE4J4b`UEEW;%Qj&~Gk90bn zP~U&80lkzY&a)7Yvht)x^HOdb_Q|pv8fVMt z5b-KUOfD<-=p6IS(i)`llVYB^Vn>xy3icFAdY$JI% zDQnDy!@y1#aq%9N&BN7)>Lbv?-)gr3`WSWhXEe<9&DK(UXgV@ZuVB`AhgM)?!@k0* zWOJ*Y26PN@WWhIZe`NhP{BUflt0cMK)xKQ))Zc|aP|k*KbzpVmdb0{+5Fm8JrFIxY zH57Xw&TCE&OYB|XxvG7-ygk1pa(d4*vD9gOVWdX!_8nLPIV8R2pPuz^{(59e6Zvyp zXN8$+$m&vWQEvRDb_Gll0uH5q&-^uJ)xzDue5J=zI)rZnEPUGYhnZ3~Oo?9S1Hgu= z_5RcX2~|qN8qEZ?toZ|4-{2nK;stT-=GI@{>PAalWPlR!^CTNxrwL*j^?F>u7_X0z z@`MnKI9OU$fN3W|J!rA?WZ?b(1oU?`;wQhy_d}dR|FDUe~XcLHE`x3CM zsFe1Y@Xr2Vm2l-bBDpzjyzs5-<*fUJW<8I6FkQ>S`<0Mh;cRY}!MLUD4s1l9ZsDa- z3kyr>TWMt`9w&j>-$(D;8FZ}XAA55e89M)@ryZdsk7IHW8D<>3tj5b@NF32=Hp=AT z-PXtcG>r|y?r#tZE-wcM3+ltEc>5X!kBBDR`R5dRyXt*ei-ye(Ew;NOzT2dt8@x?A zaHo6x!u@%{0(a2tGE?b3mjlM&hFe^{{-bS7x9$Cc?jDVBtEMkdKB(HKFda6teJim2 zc~(8@sNM6MI?0%KHkWRVZrbyMhnazcubL52pLN?6JClY1P$O(OIMqvs5lCp2ly|=9 zlv)GbV8NgVvrUklDs?`KV=EtB80joIT9NBiWbmL&o*r!fcAvg39U74;I8n;@o}oB- zxzSLH`PPm1kjI?Ghq2{-V0}`5MCfd1Y{Hanl-Fqo>l$30=0-@mO(^b(f=tM}FwUje zH=A)UqyQsXG6GXZXJb|*F5GtI@fjCcFoq<8q8qLy78bq$DC2-T=1K_SuzNEv*8uI7 zS3HgNJ6QxrY%y{DGIF7`KpB&lDNojWzV7C8u=Xvl=Rg)gqw=KXc0*C!I(URd1mhMD z93fyP|C&dd)SYnbhr*V7+IqRoL1ew5a6x2s=5DpdUa{+K=jfP2-Px1VP<<=nOpV^z z--!8U;Sw{MJ978WT!1AwrgUg(W%9DC?2TG0b>Q@f_}Y(xDamW?VlzBUeTClH`WmSI zcM9WkY3u$fDjEwERT|jG0N)jTgNtDQ1M-tv))T$VwOC|bG2EKB>}iM{ItHcBvoDFj zjdau0WI+Q_y#D@s`n!00fci4n4eE{^XCheKwMvzQjO^d*6MJ|K_Zrsa$Z^v;p3SWh#=4^b`1Dj_%g4$)4UmI zH$)I0F5-nEVvUl}FzRDeF0{CW_^^YOuXb?Ck_mfxjIJrqX6D{aw2JNbIh<9%?NC<< z8Y+zqdQgMnTY$WO5PC6dm%*Y*f%PY~y<=bBkF}o-327-sooT{Di5=gCY0d#>9WW&)}wF#f66R{{mn8o?vVYLEVL)ki-S&EGf5@BXCFnY7MX_o@~2!GfdM2EgLE;X_b~IW;q=&u$SHu{%NT-0HvU<9|K?6c>Lc zf6vdv0GO?3v$!MwAmIHz|A74qP5unH_CGd1RluA5)294({GThdNCGk6KOKGXU%U8U zr0Dq1D+1WITO;X=yNEaK658a+&yh*7psd}V+Y`c7Y(Rc@yCHTZI_UV1z2&lMyh89l z-eUuDSpL8M&Hvs3sxTbdJK0MpnP#=}dDb$&Gm*e#B{^}3Qy*-sl@Mghx*FEttEHEa zS2pI}>hrfOOn}MqHREcGrS^QPkf&b1w!_o;>YRJHHEH5DV&lz8O;AaR_=2zP?qnlR zN^Q@_{aYN!07P{#7-WEs-7TshBfGHSG5L!pJ0W&uFVbnaVdJ*>ZMs?0*X0Fgo(kTRFn1a8c%#8OG#1KMTzzl)Es=Ox>A2?rlHCMf| z1~wSFCvwa!1M@mQ4mxre^QqrQe+<9oQmPQv6Gf7>pCr7Z4WtQuJXSf5(c=JOqNHZd z{kXET7t24oZr4}WqIruh7d%40rE8t9pkrBWnl64y!qqlbti@Q&f`Nuog5i%M(TtK= zE4C{7+k;ant9lKuX(-CI$VNZ$>F(LgG=Rr5`s0NR9FbMm*Ors1dnZCsz<5^uvbI2b zZ5nK=LYKMs%{R)FYfuZMh{EAty7FiX4Ja~YwcTf(6=7L##yp%78zkTPi6{CavZ@Kj zW{R3+F!;WjgCXU&@kRVQ@7h(K`sw_M)pRTc3OWOaq{=u( zr0oMSuV}@Llw*k4yyw8yt0Sbdqf@kdGBI2zOgPqf(dyGz2p_qYqi<})9Ho?l(oqfjh$=RrY@f`wD$MZx7{BkJwJ$< zVkIBgn1rKd(u0^-VK8-X{#@NM4d%2*AT^a@M56)elHEd!6y+=h2ZP;P0cbv+u~E`?umE(viDm=J;~(kPI*maY?cYapF z9AMO9OTDZFW8%$ET7pl8m(ru$S+AC@&XTf|Fd$Cyr)h(IMnOUam!~?HTx!>X{TX&j zc4kIb=|!n8m0w`l9hbW2%kb3&XoiT63d6Yjb$d`dy@F(^|zAh0l`E8vvOU2!~Uu{PWZRedSJ5I z3#Gm}{tf$Dy*loD7}{F5D?8^aSd1mbwTIYj)30K?1S7Vp@n56%u9I-t9CzPL% z2Npl4&fJEV*yqaiM=eEhzH1TkTkA!TJe70C0C6NdeX}}A9QGBLfr3r}&)=VUMp2pMi4yb3b#Y)auK3>56^-5Rsy9 z9c_DFJeovQLznzb!uMher_XmDhnJGW~~(4Z>%0uE7Hh35=VNMG9qZ)hf0VuX@%(;TpA1 z8X2WzOHLSrQ@@1P(t4@Oi>OYUZamaqu0Ta34`le#1vraKs!t#X+BqZ1I(#3~>CuZV zyVSE^t#HguKd+`jFKQAji?;8b4I@TxHAAcbqo!k`o4J739nI0>wg>3=Ge+P0AdsRw zsV0u)jZ*YuVlzh&=z#;{aQrx49b@BDqw=pT9+NpZUiaI-e`TVLBYC+g(Hs|1ybl7= zyZfnBT&Azj5*KA#_&ZK>A8H)t8k1YZPHio~M9V$&lN4QP^P~5T=Nyz!)xX-imT>&$ z-n*SoHpfjAIp>Xyb-0+ewT&K`XGFDx^be3sCE+KIe9uj3Uly2!aBQ?tdVanLp|(&z z{m#s6nm#(f553LlD6=+bS=s^guQeuu19plKgb^Vj30^b}0;I@~G%fkkt93eT*%uLW zt6PZA;0;H;5w{;pBi}ZR?FDk8UwZqW@_WxI|4LEYN-i6(i?CPUPza=}h!)yv7IHL} zjA#tZj8L&{Tv623`tf+!hnZfVtltv=U?L{oU+h<3gW^VJH+h~KRV7TO=CEQL5Z8pu z>kMRQI4hBYazk=I$D6PDY&gOrNH<-sT$mMIfGP1=Pt-a>1GUycFFVIbI`=xN=67t! zX^x)do)Zt)T*>Y*%@S;<;>(8zjF=`yYKgC2hPJQl7pRzAh6NRTHRjXd591pvUT?jBYwY+rlHpdfsIk&C6i0-!M)mTvg)c@bl+$v29Xss+m{IlSNL^ zVi?Ocs)zy3Zi@0gD~8b)mNqLo(KBc@mDCG-38o0%fJQF&Cye{b3AVL31=hA15Lk?l zsck1}dLGS)+!ts51_YdGIOw}=mxYwQEM+u=4AA_TI-kWrbk=CTTuBqUIZDtI?DK#& zxOsrHkbc{8Dd*OWT!z>|kVISYH#Nf9FbYj~hg<{M`;3(0AF;5Rh)^py5v=kZhSjW8 zurFKPySeklCO=I2wxA>973k_Y5fPTroQ_US>+k}*NXb-^^DfZ@6@-U6{sX*6{KVbQ zcy3dHy6OS6*VoQht==jJn*=+*8iX1%k)!*9zxPA0MaU3)4YZ#mdZ27uEp!^dajBOi zZRAKVDX>c8bk-)10l*%Ki--UW!> z7AmZRW0m2q#GIn&i!>3ywOc41k;C6z=Am{fbiA(FfcQe~hK$nQ{`{G<7n*<6nELBy z^v~ku2A%ZO+~}8j;FCaVxhPs+IqX~J*{LFoh*zE-{El)KyfyQU`6WuHVazGpRvh5O z2`uF>uPIL8H;C%p^5mH*-5wX*I2jA~+e`h;%e2wsF^YN)sK-3S}5Ty%E zl=_^zd((P8>{tk5V~?b?xOP^5xp|7Bi4xNt2taQw^`QonY$69Fe#EkNpo1Q$5H$ZT z*J0;MDTm~`=e!@WY3xbgbU12RCnBCsy_0Qvpq|Jym@a9hxRtCsbUKsl%1pQO39XnN zS8yV4;hIuQA{?@*zpZwdcZ5iYX8OvSp{i2~%d{Tri!LcSg)uYJjTWicL?1k4@VxP< zO)pbA%+uy-bCCCd$Y~Aw)_26fYUeWd;!8?=hw|mq1tyAGy%Et)^YFa961PL`$vtbC z$J>D|(JI3hKuwU;poCbygNs8}SPH?+yW!n|Df7M)prze(g+rU8nI+IfD`sN}=y<*5 zNaOqoaFoMe;Lh|K^~Tl<9Jn8gc^!!o)ED;A==~877yKIe|$iFc(0Ag=O!ZO<;Jq#a&AKNW3&AC2ukt1mrUE2#bC3+qGyv&m>|7V0B? zklzfd-{sv7l3E)aNf!Y6_85AGsQ!VljrWT(x|ZA03de;^07rdzpC%EoN`GxdR2G!X zcwBowDc{2wS#tr1`%L<5MvdaSJHza+bAX~2&VudqlA|eRO>v&4W0`8hcI%TvJD6Wn zEO4}DXAl!Z5EG?##NaLf*Ft9md~=bdZM(?srTR;n%Y$*`@deEsYx!?a6PdqnoFEVb zBn5_*sKcH34z|SNpnxOTNRYVJ0szF_TSU-j^xhJG-G7i|{@q{SfzSWe5dYs|XRod2 z%RX=g`FAEP;Fdn1B=6rFQ$xCdTUp`7Dd;)9{zH(=baxnG=>H+n@8~Yk?|%$~@o$g! z|ID4Ve_<H-s|y&Ztn1us=-o5jxCwMx%c z;X*6Sy~{_S5`%2bZHtJ|X5A>)^998{k;^8apMh-?a_O59p@)ee&_6I1&C-%*#P#=F z?EY2p39%e*;m-crpW*%@A1uQob8QIi)s^(qrX~AQ3`Lq1Zp&7f^<{%#crxhKI2y^& zcDq1#FKK7*d-!4%q1e7q_8vNXg63=#g~<+3;YuMj-SCjecv}-P%=Rau(GY%IIG=>K%y7u0iN;Z_MCJ{VYVAQ{FR@NiP(NDdef>Ose9X8^if){q`yvKz8f z+Y2`RBk0!~u$(?D`7`Ke=6C&2a)uJ?$>(xm{d3+3vb4pQL7f2t%j{T@j% zyp3yyBBGV%Uo9BW1k3lPNxB`OBh(ipo(*IZ7vt5R=P+hM&X*5i0kH&)hx6I~fjK|TdHf^}^GqQrE?Ge`Ru0)We$ zmD^z}Vt>m6ET{jq1k1p}z%!(#X$0vaI(@Ovfw7YB%n&=4nMpEHi!D)G8`Hb!rP><- z6xZ)~TWhIl)^f7pvP)z>b)K;dH<>~)Q2VI(YbJb#k^ zx0@}6*I`2>P_Cf(IqCcdSzKN$Y!i7VLF*zO&OSr^deKpp_JzEX6nV+s-kviUg^AP*v{iNZ0UiY^kk6{(~NlSS{Ua|G^ze(oPYq+0r6QZ5k;Nz%?v@0 zEJ$?|EN`|H3ROw9N!$K|25=jBgElj8A!dbpsW?&BOr&n>lJe79lAa`;h*Qol)1hQQY^~iM;J#rBf`pWQg`c% z;_rxn6f#8>+BHOC#6j0wzw(kpfJ(GxJQsA-*qPxeM?MCJ<-kL82Ims*g*3-t>^DA@ zA+GePzXjuq>$VmVAX@CpRg}Mt}d*bxQ{NNkz{)-D>_df5&u{d4xpeCA~ zx}0Dh-{Jt$tFhk*tx@j^J=4{@G#JJRWy1%(xAxwTzA++su0Q1h^g223;2sHr4ko=w z1DeW4*WK2z&152$*5)j!PZYd4nQWHwdqmpck8Ng{^pF$02Q&uVl6T1MeK@E>^lQ!6 zPmNLKyB_OQ%I~8)1nH6~?VHp4ewVX=DhwCiWa~Y*nW(K~({obK(1-5eny$&?7WU8j zE(spasc?uBApKHYGk>n38Qmh#N=a;^WGIM!p_s+PYb!7I<7EzrxlA&qhrnBQtYlNa z+;C|X-XxJJ)<xIOFmt>JsxCk*yCF&!z}@4DB%e2E6i;Q^n!uc0gMUothQT>>sTc4Rs7D+Ydpe zO!ObctPS(e`iyj_hs`&^{;kj-q-PC(a|Jbte(^7rWbYRdo9#n35n8%CPu`c}cSIiJT2YuaT=7uENvtQ(76YySB=XR~D);)#!v)0D=fazrA zitpmq&%Qhn%2}i{RL^~(7OTNUcF60=meYQ*FV;|!J+I#Q{n#^`*RkW$7BB+!npf{Y zjGq?eiGQJRNcA`3qKdfi*%;W2)zO88_wmM}ffPe@EXv}1D)jpEh&(?#kL6IzcLsw> z9sogN5u4$mvO4A}g8)cGn&S6UCLcQYO(%;z1;STurLB6?7`gedt*Obig*r1uJ7{Az zT4pb2OuDQ9i*);gCnJ$A7z>9DNLSjFA2f=BBkRTs2p5G~Y3K|i>TXN_0M49&ye@xn z399MS#K^hcA1?73skP6plLW^gNZZ5P> z;tgDBZ;z!v%Efot)Ejor+x>bb4E4ltu_>9pvYnaNs=>Wi&p~1f#Z8+o>Ila*Hg(KP=Pf=d5B>*huppF`8BTL!o#r^G6V)BhMfPiGnR+AarF#@w#p)aDqKi*)i#EFD zJkO`*d+WAE@Cqms69`|Z@NBCD+wfjUS6H6j^Up0EtvTncR@*| z84g*9;icd3h>Di9NJ)>~4f7Q~q<(|j&K|{Pq`uu`Pt{TT1SGwqLv7dA5V{2VDx1#F zLN=AYNawsc&iX(eG#5tnC6Z)jUcpPaW9AdL{k8uyRxMe{r!QO3-@ynnshL2U(vsSm z?+I&vL1a--?4QZJdFAYz9T>#Wkf`D|8lm{ZU7q8aaD{NlPwyf&fLs88jKmXSFBRo; z#)+q%EmY-2Z=43wYw;;PT+IM-VS1i}#1+GdXI-kKLPo1;iaV{U(H%A?!~b)Smg%RK z_Ub~eWI@1uwLng?;#350DOMysbyea^KehH!d5L6Y&@*v-o(~-|#<=v#v!Sl{{klXE zza=Q?!}j0fm9+3Gzore0fAq2?5S`#5ie}0!VSz#A zEa79%?o&gT@q?+V74~(fY7S0o|EWTsRBaj@Urx0XgvxUqnbi6s2g<>s+k?n5#y29{ z#!odCF6-|+sUCDk{V>B@Wl^Pce7KmTE%Bc?hRg3|m{H-&MkDSDWlaUZ&+U!I9*-uj zbH#h&R4tJKhW@87VWed-Kr&bv_3T!sisxaS&aL27%s?*yzju_Ste(#YPpHQYkFKb& z<{wM&=*5`&o-`_CubWe6!}7=q^#sv+VwZzzfPQCzY-wklC_oR`{&%)P<)7Gwv>v4M zySeJ&WTqPUTF%jxsY)lLq~0~t9&wD z^Lsfa^ZArI&R+X()%BFB^BLAOtAT(jSluq*`+D{q`QKVjwceAu?@BeKbnrpsqeUv& zeV>ao6h79*KH?)hT9diuD9Mt>L=~7l%OozlPi*Bfmqc0Q3J7$s?5-{Vh;cejK6yID zlh0&6_@xdWib&ByS3sxDcv5TB#7GpW1PW-rp$W6#(?Rkg%=oth&*n#fhtL;xE7B3< z8UBNa7{=d)UCY9VKFtz*2{)2VA$e2y;q(er0_r#HyaRRYY-cSO1XD??MeJC@GwX|I;^bH!w#6aSO9s<#UptPZ)_o@Lc8-WHX?tL5yeYOo)XRM#884dY!vk{ zewQ~~S@}a9&Sh>O^@bG@Pi_C!2)z&c0OmVyt(r4A^A33j`T|RQcaXUKe`6$MGZ{XF z6XAxRUlRqOR6PBc0JjhbX%mhJf!XN;*6`Jy`)mNX&F0&hQhuEAeqda~`d=7ycN2m% z!N(?+uXE6#U;mAbDCTOd5ENseh;unB{DX(k&|kRYApqM)p1;oickm2)p$06SZhVmG zjiRs8cc%m(_IJf^y=5mqU&7w}W8weqMb^|9RfQzXkFg|6~68 kf0c0fOLqS`4b+XqB%%0zog3Hs>h2<`_wr(8q6VM-7ZO_h?f?J) literal 0 HcmV?d00001 diff --git a/profiler/advisor/img/communication.png b/profiler/advisor/img/communication.png new file mode 100644 index 0000000000000000000000000000000000000000..ba7c753f6de93cdd483b04c16bcb41002a1432ef GIT binary patch literal 58862 zcmeFY2UJsA*EWjB!m)wA3LLr$ic+OZ4TuOFno^|&kP>=O2t5QV9zbc*OHg_XNbe{} zhtPY82qX|72mwL}N$$p@=k*=;|L=dlJHCJ1G48l)494EsYwfkxo^!4>pJ%SgU-$Gh z&Yt2p#lXODR!j4a0RzMDW(*8RPyT)acrt_v3;`}DJvGg|7#Ns7AO0PQ6Jp|IV7SDf zbw}06Z)$B8o@hL3D@NOB_|vwC9IcH|AC_2$jaa-Ae+<9ubnZfhh+XEy^?AYftOcre z6{_MKCrvot-j#~}vvgBQQ|gO$JHO_?)URf+O3NScI1Je@pVjlInLl~{B>GnGMx?*O z(W6`!&$Qr(ip1w|v~plc5d!ZwzuJtc`5*$!?t5`F%@_Q6KOuhk^pE?HD~Dh6 zce$wA`P+}%jNfmw{(P){uI%US^`}wCf848_e=hvQ2NLL**Ys=ak6%=_uYzh6(bA6&W;sB-KeE}xaw?)NVs9=(OLGU??X8ftsY9ZMP6vlQI1K8=x|REsSZFmenvjHYl7?cu|unt)d~lH-yMbFkJlXQLAY!jYA+v(9^;)((2c z@k6DL+_09Yq`42+oT5ZuHPk*f%^?uB7R06RmY^#Q+;8Zemo#&8{+8|UiB3HD)PXmdOTQpilWR8+WYa$9tsyF3 zAn5F&CnzK>vTmMY;5%xT^r515Ins!!S<2T^^4cie*ZRXQ){JG2m^}5)&`;L>^A!qjG@}3ow5DRB3+XVp1-WKnvW` z@b}#Pd`5`N4J-!=M^rIhtg5WFF?p(*k3zZ7b3=|F=fGOcN!mR6IG~gK2+BjfZ_C>( zRmtc}O2KBAODv@b4HfNA#8S65XD}?Kc{+XUB<+Ip9JPD(wB(+JW15MZ`AKT3X(y;&K|crs@{I%pnr$=ON)<<7p(^Cf%vC z?zPrZNb(s@>1#+bW$FxjY4L?(?O8VEIQIT3=<76U<_In5E{aBKbGEcjqJ>6w$|dOa ztsFl(ze@96KwD|A){W4=&}Qgq9p!6~)uGfZmw|yZoO1FDXs)>A0(V43N72y*kDf?+ zfN|4S_H?=5&PyYvKVr@=i1ozf`Y<`NqM07M<%p&xT^M5<({kA71*d2w=;^F~GMi z6H$A)XUbK$`Od}G<#Lo?UWX}rw5YQOjxYryX6r7$?pb|IdfHL8Ms9c0 zjQPF-?7a4qkXq~W3VR1HUkW#o2)z~!!e?DmVw`6V?Dax~IR04W)wn}sjX>#i9) zq`{PBtr33yfruFo|7ua${h1BLlioj=TCvAo+s@ycHFQSQp79O*R*zVO0K8gm}{ z)!@Q3(zO{=7hFC1W@C+#w@_Y~kV}{&n{BrUt@r0VRhCDG+jf2L*f!z@#YU zTEbFYFJ-qynC`5`G49*orvh_Per)feuOyJm>wsng%((yR;g;I~5u}%0Q;X$3^Ij%* zeolKU(rY!amtmlx>asR=HtH&~Snq@@U7C%%eBnsqC=u1|ubtq_l`FRj_V{z(`Ph1`jOva9c8%8-jpep5wfLeY8Eu~6AO+;c=3fn-8!yN+Ga@}Tx2nNa{6aZ% z2}zmNkD^|mt%YG(mmbnDcYIJR7m8TZKMfetoMd023E`V5 zewEd#ax}z*nUhn3e%jRG6P_Chog8ZURNTD{a&Xh54JZg7OtsbGpB6LbR*6nut=(C= z!urb3$3ODZm+m8#tj z)nst%f%$}}#R=i6gSDfETWz777~9qoZR*z3)eVJfnKT^J%VlPopPoWdV7c>(&?Ifr z)SlL`G`&WAO%(0l=54C(k(4?p^+lklhY?~Yf^c93yf3Re$Ch@)$&Zg6q3)+;e{ZdCZs|v76w!kdpxVd*K z8KPz#XQf4J6o)r3n`|nr4Lyg(Rt!$OF%DNy2oP*2+{m;R?cGL)w0#t>r9I(=?Q|)C z5N@Uex#Xp3Nw{jO@~}W)&<2;JobMP+w02w5F&*q#qU__2Rj&MmxCLk_kpB=|*^pf4 zqH=K1ejKV;$nTMja?mnsbG$n^x*m$>aYHEvJ;aW{m=3ICr&QAv#vr?DLI%Az@9os2 z5T6Mk(USV~>ycpPD^?fIoyqI#%|)U!4I@P+C=)tX6XIWn?O4OHMSEirPw(AbZ>v7Im+0A8>$u$FY#K5?@kS-O`KWDOYK&zWVf*RMy9*z zJW@M2zO$7X7#RP^&dOw4;?2at!7RCZ2*uhTby61hiMMjRi#cG#)r%ED*_7Hj@b33n zQQ(htL^L;2gT0|SuTg}l@?e76hUdBcorAA!Py3jeyPr5HphEF2`TN#ZX*pNWR`C5^ zcQBon1ZMU9TQxYvJCg z#!Xt3RH6mHm3z@asDbKDJSN8h?(-myo6e&s7ppw^i0WX>9)M3`pQ>Egl9Jg8o|9WR z1y;a@DNfyg=`LDdy3RO9@CP?MqZD3ejOc!@HEN;6i)ZUYzb#jey8wdO=JZ?4HDl+8)Q4zoR_5nOTS(+r=zHO|Iv3wcd;;v;VD4C$o6p-n!HeSm$LQaf5ubsREV zStH)T8*w}RrQs!=d#Y*@E-frc{uFra)`BsOauR^Hd?ydLhx(Z^c4YUX4qglZ@0z=E z#p-iA#1@1=a&mT6e_VGD-pm(Xo`T##9VBeB4~a7v$~hT{*4yilN)T*{#-B6Di8{&% zHsSJ8TQfYXH`1*WN0&3y7_Z>1E|Q=!bH=VVOUPIe?-eyEz{J-4jHuh;0*=c^)Z=Xz zM|2b#iK#~o1KSgMpv;Q&WJ~!V8^N9!uA5Soxh!X<+}DbQjoHPE120jAG6T@)QtSDz zG5sj?s`|wbTigA(GIT(Gk={Beb=XS4nXtsR*=M$CL<+Gup^vU0hWdyY-h)jf>NI&e zxhN}NOVF$5W1f?`ri9&h3cjq*QqlA3hNL9z8#+r9cGqf`oH{4X*U_T5^HqV8kMNq0 zV4^q5w|!;{7_kQTLw%%Fl=g-!eK$WSp{V<=^%P1y#~p;m-NkO94)3!eOb62+Rt6VN z==D~hx8jW*mNDt^$?H`+3d7qI#V#~V~m}Vdx&We$_3K!gK zPaYL;oO*jdpHL^}BDsVAYjUvVdY_QBrWLRa-Hb&wUCBHF<5~XCj*CCyWf(w=*v)f|e& z$~)K@e`>ZIHCCFvX6a+V@Tz~UYvwhLF}J}?LFT?_-qpDS4{CwZZh9U&%)X%dk9$Q2 z%cZG9v+~YPLCgtfR^E#I3rtHlpZDp=+?B}|C`PUA6ReZ){sdcgzD%sLmJzi>CJZv306*O*A7 zQbH=smf#QbEoK}hCf)PjidLeSb@;aCH7VTsbKX}Ab6;+&RK(|cVOO+(DJvqv_ZxhK z-d|8ZUFiJCQRg5f^%k~8hwf+<-#E2BC)!%ZxYKo%-e9wv7Dx2zVn=pbBVjN{rG{HO zh*wL_z%rhD^4J)-G_VOWj@aLW!Ss~jK`xQS1eniXXPZU;$PR#+pyh+deBMl?Vyk^T zhps@lC!%4xua8?u=M!L2q{wpXPnRr-OdXdB4V7#J@Fm{Ovs-k6Oc6M}E^q_gkh@y$ zu~1)_t9sP8s!BF1-$tdfj;?n?ZA>=1zC+4>%&Y@aujII3vC$-X0I?f$=wzadT0q%6 zxD%NPHtjL=FUs~S*A*7vQtkD&Wg~`}S*JvUJj@p>EJpZaW0A7_ zt$(q5{^#vq`3cUJ-&Po#h>3D72DtS8NrqE?&xls@+4wJjvYbYnkldQ7U-nK z8U)i~<=4W{*nRNoN#478BM(H50o$qCkL?tozxLSWSt8I#UNy`6N!EU8_;^a_ zS-km)=T|CZvI$e#63sT}svhrU*RGXROTK7p-|tNe(jt(}aGjn2dsMzsiBOOippr(D zlpc)*Dx|c5Gq%*E*>~Ro99K!)*`vD*QLkvoD&;ey?Th=%v?JHCG+=6%be|qkM~gP1 zf7uP%sd|x*ap}r9n{O?pkRV*Az=vw$8%HX7@~%8klUN~WNNmrQ*zZ$&s+macmI)3- zR9{)>M*m)tm&A%Ev;HA%;X9-)YvQ|w9En-^@P$sHtuG>#lNVm5?K2nYk;KwDhbG>h znJlnr?y95X=QtizV$!W9`-CLuIfM{(0K*)@aTNPqJo5SkDN(hN_OFB4!tRf)G2RD{ z;cX@0em(U|`vfF`M^eg=x2Coxo8JeuW9Vh1TUJ3dU5EJxt$aSiS(}D@&|D{samUXU zDQ-n^>%GIQR?0#(dz48Ro%EAuGDoW=U2hLBl-D0+g-7;{K+!tu zq4IN<(<62mRm2#E_`a|%0Wv0#Q_+7ZYW>)vAfw|1L;$yaJdFY{vV_%`PfDcXwR-w) zKWR0m$Zx#=`8aW3NWrh{KudQ>uD?%6cV*{#bAT-g!5e3s?YNztL>!MTCnS_C3V?!y z!;uf-S(F^jC!?;)_VtHFCpbDSDGv$s>_e8HC;4x$VWp&nyYzW1V(PMcfN<59Cd@=s zAgx$5x0gNk2_Wc+^0x&*2+l@dd^yT?mQzf4V=18-aS0)FefFwFbRAO4qf(oAI+E>7 zktxHUGHYAw3C@H*S>M} zg~w+FhxfSBR?328XW<1o0Uh%p)#eyXe@IRTsvC!8nXx{Ua#E~h*2b=KrKnJP{>EGp z_cV8@U#@vf_3jIt`qJfmxtu_+Nh&Ef*rMk|v2$W>!%$F{Poh?L#sL zSd&;EbWN(clt0|Jv|{iH@4n|JrP^C!4RuTKx-0skF2Ng3k==sBh=GBTbcC))?FPl) zaqwu8e*jF?AFMyT4=>e^H+KADoB4PjWR;-6M!OCQUVLLI-*Cyftn5bfCh{&2*5c7T z44F>Y83k`I$Thm&1W3wO+jRN+e7j3UwKsY^nr=d@q=ksQ7SZjnz74BSCsL!P^nI8Q zMlc82y?#ZPIn84$WFWJJr@cnwi1mzPjA^k`d>DPhRJtgX#RTH22ELx!^$_kATpehx zs0LELW+gr99hifF``4ue`8&?~>2p|vQv)4!x@wWNf-u+9pkRCD-~eAGu_U>xUP$Pl zUK;gMB584_(RK3W*inZzJZ!zB`2uf2pZbM@`f6Fx{)m%oj)b9hZ&>RXR!Rv3GS#gs z?T}zQ+P@dn&SUvmE*XnEJuaf7)}gZc5;?k*W5MdftjVk>z{fnjZ)Z#|_Iwk~;=HFG zXqKFUL68u{o^nZt_N}L?d-#Yne=53PZ)j%7?h>L^KY{c>Fl&j_XsSn9W8?-uNHXS> z$CtsN!Qk!wlbISY%(Frq&dboFzpRClT`8w4*57Z;)=x;~ywdfGYabIRXW6kPvCbhu zsj8Eb@tun-C$>%FeQ8NIy&h4-W85an2x0r8&fPPs9-UuDxf#tcQOC!@>15q{GR4Wx zkR!O}W;NK?RM7#FK(FGfhil2d%M36s}pvMtAU< z-P4ah1uUXno$nNRmn%-qYbxuqnWq;;#gf=VT8xn-Og_xrpV=R2nUQyTNN5NQa+!;r zWGj$(IF!x48_QYdW)~xs;Dg$9glp3|`xxI^*Wv z`!xmCH`8*Y(=?^b)>oQ*9|w3I7!dANytDB126_AG9EtlDAvIMhqa#;xy8-}(~ z6vlmR=G*6@RLbWfa6f46U#oIn5A^24|T)764N0)Mu{{#B+?{Fw1h5^nyoSSiDTFOTOe*a5oVC6WCkHf^*nPC%RLhaq&X)D{r-ne8sH9k}P-ru{;4U~h33p~vzVxP( z4sFT0!=9p)BVRc_x5sppmAm^y&h9>=R}Wduu(CBx$k3}>nmjh%o|^-G6v`XW{&~d1 z&WDD7RK2+Gel~CHOiv=%F~DhVZ%lb-CQpu<9Kk&@YmKwsl9eOuEdWji3dGBRp!`g= z5N&m5F7RtK->G<`>v=uc>Rkd(ee&Z7nAm42Usmu67)o*;+skfR{<=EUaSiuo*0D6t zWpOV;>9NLAe%D^oxLYPc5NH!->8*1f{&41XQq^nM0!6=|9B|N@$ANM|_p2K(`YVBR zih#R%it~G1z6YCfsNII%JzGOlvnCmbJ0<;bfG*-k?GJhZ1_R>+JnCnC*l5pRkRWBE zsn2eltAv*Y^$c*B#H{mu!!3UcP}hD;iU!0w`U7u?79FLEFEHYjxnh?)7q79=bcOFk z6G^@BARLIaL~BSu@Ou&R1$D0Xihvk=M4hK2!NODH8n@X_Q$GGlCIqV>tK0kP8O8TZ zm`V12-u+sq_D28h=xK>sSO2i3e2b|H%Bi91YYWZ0nuHYbT8&0*2!13eNJq4B<%y!a zyG}v%`3K90-t1cFynB+o#~qgw-pouUXA8PsJuK}!Yn1GfS+V_ z%YM4t)`+P~y&Q_pii8gG($^w~8^n}k_km|#gw7GeP@fTN1G$WHYBN%!St`eM92?38 zQHRD!W&lxxhAX+ebN-;iLHP`b0+e-Xe|XcV9Cc~~HwXmE4!YC~CG$!xjq{a@GN4G1 zl(BVxw>YTC)_TbN|4m?%B(?!Mqdts5+BZoC#z#i z5@=oucl>^uX%kuMF<2=_Dljwq*19i+ntj+EjfFWsxGC!k(=9SgO2&iVpLb=0?AMw? z##+M-n-3#>r{`SV(RHSP!-UV46OCw*E=LZ}QNv77ho{2~6BmqjFrY%Dd7_!V$jm1C z8M>>pHNc-5LL)DXC)j^71^$?r4bo2Xz0C7Fe9!T4M*$qn?{ONs=<#;r4Zn`24(!ls zM@~*r^|Ld5i%E9n3zt2#tvw&m0jM&4Nb1MGqkEq%j8m)BZ+WD*J-dhZf?D7*mbe1dn1u?jaPXqD?+JW zruBqz{$pSgK7V#?$a=ULo;{q=VVQ;;o|k)`IspXy?_2FwLMdC%7;3U9ND^!1TvH-Zy%Q)Et>{9VoPQSbVV8(K(Z zxPh^8TsL)hbyJ=jwh62@Qkc|jZ7;82>Z}xFb98omY-Loqot)a&a1UD z_xN%%Wp?A(&+LFNRxYltJ)F>n=&`Xex48y6cX#*p1%fqE=Co=|u!4Jvtncf-jnxI>#%h#0WZ^q|KkNhOU`qU18}@0GdKolw&_Wa- z_utFJpYv-h*Fa7J|D#k08zINiiRAxM?!^CEwf~Wp(NYQ|O-Qdn?^V{(N0DMu$#~H{G3snUX;RtZdbUt{Kn4nT83o zc7>Oa(KXJkb(YoPGP%A>f||udLGfeqbuQb_GA_Pk5a!)=b{-7H&j1 z#E)^Og_lr*UNTQKwmJE)KM7iS89H3!96mlSokL4%9{~~VU2>_~X!(kVxo;@b$hkN~ z)BI-9jiAlf()kEST|ClXI2@gtE(R}p+A1;tJ`WcQhCt>f$~2 zOz;*y*iwBgCvfByV4`9I4>ZRrQm`^}hz z@`6GA^8Rkf-BxY0N?O*LTQ;k$XKpQ>`R=z5rM7E!n^h}Dn%s7J)}n*Ue3rOAgin>B z_H5|h-#?2M=Pm3ULVCZvGS^!G;Xwm`dND@Zpeq7&YJNU-?=o5CLwMtXkNYQy%R*IBgb2=b8W43z+={J=i@EQ_3pZQqh z+qzn7^l`Si)NEJG!uz-tfx~$LWv#x~e(>FU{g^iy>ODE}{4DjP=f%_hCHgRzk_IJ# z600~!x@D?Bykdg!s(Ao`*zQH%Z^)MW8Z$pxdtC7)u54NWx10vvS$M@Ir2_il;A}7s z*8ri@Q<%>Z<^tw%u2p``SHANpKb<)WXV4;d@Mo5wp?_`mOmaLK@??_L7Vq9s6?k5+EOb z7HRf_4`^TWvgLR)vu&d6w5)7%QoFh=b`9+X-ty#-T|NJEa#w*Bx$HZF!B!ka_NbKz zbR`S#6}<7^bPvR%#(;$)`EBZi{G^=5RRsMoL{>2*ho*o(>YL%qbW{- zl{*HZ_(7gGPqWO~9c)Q60FH{&{O+96HmF(bx?rs&pwHi>am@kEBCxn@i0)Jc`Dehf z;ksf==t2Q3x)=?Po?p0|ZN7b&M0i-L9^Ps+Y`ZK+4C|5$ps%lc-_vVahU1%2v)(e4DUh~nR*6e*fwYnMQ`QW#s5VV?K&Gi#rAvtAM!U0_47sdzF6CTdllxM z02#-nJpG$!pE@3IwA3k*ruHkcteN(|oo6=-G;skPARc3Ud8dQr zf)DQT&&za65_{>@g0EN2V|X0oRd|0)G{pTnncDu^ePg_(y0S96xLBmP zxR?xT-j5sm;dVo;7W-av5;YLp8Y|-^*8AHlvEAL>Z{y;!v$FWFUcE}OPkZo>i~jFP zYWy!c=AWD0e+f1I3g?#J0r2hW_QR5sW=19^F1kNtGjZJ9!om=MUokQNio0(E)NZc~ zYqV0?(5b8|0UySn3wH|%4&~}20qmDp^%Jr97+6?Xm?x(Y+c}~3UKSSP zI;@Tr%O1(NWKD0&-z7K-dMjfKJgg6H;b#(xvrSjhp64G{cbv6BXd*CNft1`Kzt!`#! zc8CPc%#y;w!ruD!I(-8FtLzZHI(HadU0PaCFfun!9vd_HCn0s>iHI7jgM-8Rk?^f8 zl!Mytj+t;wqkU~umff}Rry=UjW@b||P&BqL<%Q6XS!Vp67R&Y4yXe}W+@z}Wr zvPQ>i;aLh!gK~xHfc(aP+G;60O*IT%s*Xd3sMl{b=D&;emiDmyC!zMbo^8hao8F(m z7t5C2W?JYhr9b76X8K3Lxzc5mCNZgs(x29{mZ;LU8+RDH|6W44rwbj+oSZ$-8^NS{ zx2cM*wZm-6!%}(y(8Hl_jf{=6A2(A28uT=*4uTij8?U?`7@A47rtgop{A~P96tIBB zXS@eCf`UST@eJ*BfGaFJfld|v@rZs2Q&Ur0fK6moR!XPHLiH>xQa1OPvIDh`LZ>RU z{;@F8Z(|lvK;>7V&D5$4^(R{-%5iU*toDCaW^Qe*9H52|Pq((V-o{t;FO-UjaT_$r zT0Rf?=QGWrjXNeWS$`YPjT<*6bC0rify@5kTBWYiva;1;>_bz@eJQeqQESI9pVofy z_m=}$oU}U)#jRIk^ro?WR(3?n{@wva-hOb>@1N}DQrAkKUYifKm+Yh0rtP-BM{>Wv zUS)7A@I@E%)IZ4n*EKg0#x7iS?SF*i|9hzLJ=7`9n1g9;aJ}wMzWQeunyD*RWE>|?e{u^I^B{H~GL}V4>L^?8T_F$U7AT-A zv!(r=b}Jfc-e<3Ri2;d_SG%y4{tw64wqCP{SOj7Fiu@g%c>)}q(=Dh=y?ecvr*Fyp zJx_+xt6uEhX2|UFYNiL<+m7k=3*ogZNjp|0+X^m5dW7K*o5KRcwUU6~hkZi&IUg1< zEPQXe-=*chto#Auow^F3`aP)-%uG8)48|xNV^3Fmj6M0xFnX~M$J@!xy>WJL5hsaj z@=rowX32OE*C0#51L7>;ALWaGGPyR1SoP3&Q=aZr)WB_ZCb{)3%OL!hGkKckW2oL1 zM7=n3<5)}4#ezl%uaq-(=ZVCzD@HV(#oF$Y4uDs+}dcLDRt6~i5I5M4Miwnv(79R4*)=^ zcK8}@k*6oDsNYLpbo~L(;;RU`o#mj63TiTwysft$jg(-J!0&&yn0=tlmQn~=P?0@tv~eI$H0b}190uj!5x=KyyX>@+3>e7pg1 zJ7fVzCAPx+lR!`~w!dDHj{^12{-Mi{3buzxSgZ`6L@2%x%0w_X$<#)ElKWa?Kgc{S zi=>oDcZr8qK#h2#FEA=bK}&RpzS)EQs9C8&HbZ;E0(HN!jUob#3KaN6nyoBKbvnC) z7GfTe+huxo4T3id?e~sYG<)^F8=Y=0ir+CK60Pk=3*z!i5I4jc%dSCP+ZJ?OXKcP+}d9(`)NSSGc~>5Ghy}A z1FmQYO_@E!8CVD_NilU7Hb$Md4QL7*B5SA{ZW5m!^} zv}|wpd-PTH{nWS78oSf@KQy=$8Wu2xTVKP_uiCzGk)lO*i}Sj^M8Z5O><2~4%a(D_ zDyT=Wq_jA1dN$EFUE!~#i z<_s`LbJj^=9t;ADt&pcqnGG~DozB8ei$tQMz&X#CeHTy4)%Fh&Zr+Uvi`DPcs5U>9 zW&K8ZDS~S|mT^jrIR)IEW~$$t)|loxlV`M{b1_H1*}FJsi`eEL)Fv+HyL_aJ6Nt!+ zZ}f&AuitGF{;A*rPDwNG(a#|fsftPvp1Q^X)n+cG6>G0eacTgf{-Ni~NWp_3bv+=p zVb1Q%Grhi%ex9`8X+=}@0{wbHHm+eP4rMoxmkPg)gN)e^nu@t0&oA&cr$U?Kx2K!y zrAnyy@hR>f<{8Fse?LzjrbiiS#;uy_b#`eOX_+9o%chusKv4zTk3P(g zI>mFdT_ca=IxK#150yE=MIr|g#|j^hJ$hz1+w9eRxc9eujm25_MCkKwF=3OI6Ef-Z z)?Pc1m$)dj-LBlFP%d8=QCfHNQi}DOsPzIV9aEdF`C}e7ye8jc{&OWSpsudkhsf2! zf`|y-5&Cxku``8J78^-hjvOknZU)e>KwMnhA$SA=lK@8A?UEOd{Ruq&AEWXAS2@Ih z8i2+ATy6tjD9bA<8cF^ko6^%2RaHQ+^N{ceW_|=}mxpsB|Di~}`2cZh##CsjeCMNMf6d4urBIBnf-HJWq!|2FHN+hbFiAbe)YcXX&YHv%*iiX~j9&+8j zon>G1^!GELMq7*?3;Aj2W||ZkYd*;uM4Hg@^OX?MPe-1s zG+sJkda{rs_oYuVl)VO}_ChUYgBQ2f2!bHt=t-Q$?SG|5R$9i`BjLNWwSLG(KW&W^ zj5u}rw6T#?=vz~o;)asdH;S{^!RT_U_L(o`Uh14(o^9e zTncY#X>kfszaOpS-dUU8cAMkU@C(fkI;?j?)H6!&&01OE@%Sq>KbgRx$a#zS8IOOM zyo2s-uh zK+C-mjPFPJDEkk;G`R%QLn;>!=js>cUFR^l2dGx%QgX+*s#Y%gFeW%2XP4qSO25nZsebOU*!M+-VY7<%U*PRu^BLP>DKDZ2F~#CKmmXAplMFoG#r*#F zfBswZu=3lJOFx5Cp)h7|i?Zv%?P1r4oF_@^Pl0t_Ya^8KYxn3^$(N?Hy=<_8X0=tFFZQ zocbBh0P4}UhxNS;>(`8}?#LZ}@Q?C)nTqvJOKZV8(Gc2@r4AE&RP?h7I8WwMje23x zb!=9geodDPTyXE^ZoTFMZr*jn;X*Ozo>1FJ<`+gL@JvD;dv1@Ipvld1lnDs;ehryL zY3D$c+ZXK1Ud}%!-2X)9ayS8ykGM7UcTQBgGQ+ zPF2h5JU4VYhb`Sa*Qz5TZf-Y`quB&F{)RAa_o~$!9nPYm$mm^27OHfL1RY8ikS#PG z8gO-V9&|S}0&~p{cK#*PNNtfQhLi5hRO!{w<@$IpJRVb^JX+t{Az4&Z*w($bxHv{b z&Sv%@9wiX=rUm2LnhcT1d(-K!189@QF2*tN`zgtIp0RL(0k61PIYifR^k#1F4*6sy z?^OQySkpyo8G|CO4deA1ZQ}sd0rCz`U?e!ptfW}HTarMr6HP+3zg%hGO{54^QVaCL zXP#VsC!t&GAMb)H@Z7BxdjAcnJcd3C(!0L;eCxoz+$Ev9ZM+Nhss-URI(AmywYBM{ z`!AbNz)4lsm-lv*0=A~@q@}*>tKG~R>M4YEv>J1eiHzGLKEzKxFFa_2zx6c;zb$S0 zay+|WuiLViw(8Qj7ppPq{h1QTNF`b9^d>$$z95sMV~nXWJq@yWS9OE*xde1>WMJM< znm+i>fI9r>2s1jtx?no}kK%V~5t!q*1v$OK3H6{N7W)0-K!{1_4BqBZZBCy>f!Hzp z&ImdvfpEzxXkpmg5-*s0xlOPnJ$nQ3!6=-yIm-v>T3E9#4yU^=Z91@8SWhOdB!eDj ziCS%yf)&4V%omK_Kh2v$_Qx`w@!YguS&vg&e^akK0|6>noBHQAs zmzx3w_^rP@9ol0E2g{lKd%ZD;9W(oEMBVsHu$tFVW2A{{Hj6M#bHOR0&hl~?%0IvB z@_y$M?WeYCbRV* zpYP}jS_6E)2hwf|HhOnb$)Ni@tbr6352UHO4xg8FxRc*~mtv-&DT}E5Q*9lopSLvH zAuV8noqogN;5%rXEGNF7lw>EgmJv~vrti8CYO5tsK_?eKqJRTz-T1(M4=Q0Wm;KLs zczv0|DwN*!a$X<=|937xC68f3O&r|yWqY#mLT2&cf`xAa0pU|P6=B62FnVV8(W9`q z$sN4tY;$ml{FTQ4P`<}8GtHR0E?+%yfB7wTI#OH9r$>3>z1KoyXnrZ?t+ucrZxsu$ zpLu@+EBrQix7zq|%Qg9m_07;qpTPQ-m==v{*Y&SUQ@e~fi|!^xwK%Xfue0$mbNXmi zrtOU&O;=K|+p#;E9Vz3>(c{cFib~DF7}Qw*!J6TnVqMTk^7{H5$8q5ZcVrwnIe}Ej z_xScY@7mByB&a}I;p&YdHr|5prf6(t3-At*86FjJGGMZR#f`2EJrj>6n?{D{pcy{+`nT#puC0Pk04#>-=5MEx(6f zUaaWI1fCy5z@uE!8j)u&2YtOq5IR{Wp}!}R5|f|YxO`W)_SOi60`D$&ydET|WhDU* zHb7)){Ts5WAU}Fv=DGc>CrE@%+5MQ{8K$mPLx*Q-N1GZ>>^SD>EmnxfbzeFz_Ko=N zR3BJTNcrOxKy9JSPSb@cqg!T8vJ%4OE`H;=5-(10f_#Pu|6br3C48N?W4!Mm6z315 zwo9&ESC)FOQ;u{R5DVT)WDRz$YiA0l?Pzv9f<%YkEx<~U0)&htTRI;O4x)}y8}mf} zD9~OA7B@FWjD|nq@8f>_r&3f*(-ohgDER-<*wx_-QH|{WF|AlAf@XU_)(K6Cdd4Bji zbAjxM-Gl;{u`|XrzgT6_^n%ILxdk(Oof$;S`3qPn9ya4uUe1E6FDmH}U*}2mqQzuc zaOSSHe|14ofp}1XBS_r1ys@CYUd(wtw?g&bX0*K8X>Uv3zDgaQYLuUgb9wh_3Ii^b z-fi6}tx_V}xhwuluaRKKSh1@iRN34d--p(n+SS_9l7532LV1>W^bZH%O;>Wrx8B7g zz`1Ooxb08!jqr)iVDYclM1oPO&3+H#+v87MT}Wiq?ly8Pa+uaIJJJ5U+I7+}Wh6Pr z=1p_89>)5U(J#Us;5}z=`#Th2?nMjEuHquxn+ST!``7YbF5^hW*9{oGpH%nTDf7yC zM$$43QeCjLd{RAvaX0$Hx{JSSHnum=(&EhyF2;E@JW>&Lwr^dY=PyiiBPY)&HANu+ z5e_&EFCKsOK+tP3FF21$$NaiErSoXOqP2N3U*T&|fRR{$8=}WTNE$LJ3Ykg>FYqS* z0ZH1$A&GmIp|n3XKSXPQ2SGFCDz0<#S-Th78SAdd2X9AJ73Gbm3RL57=`#OYw)-!h zf{==nv(~ar_4O8qbZKpEn}mz8w7zYRRB{?*FURXA&=k15E#s)Gyn0=QEp|nwS0lxT(r&*B&wd zAA)2AlBDjbsy;b&>eTf!tsy7NYO+yp+BH=z`>?1N{4f|SAK=IU`_0u$8|sP`*dgPPt0Vb_^23w)1e7Su z*LRrX-=@N#nF&NOjz0+rJ9S#x@0-n8c8NPUB%CHS8N>t0h*7lveg2Fn(>EA?tc~{f^xc@2RV+ADZIRPZ_|R`d8Lia^63A zTzd3L2s|U(X+RwU5%>>7%$jE^yDlsoapJ*6mi(Kn_K#4I58oNOfpF{#ATn}Y*vQv3 zp{}m(_g`ZV9~p(ZR8`$%P6|z`eoT~Dw3M-tFw@k_co8@HzP^5%_RDHw$Z|Yx@Zb~c z7~-9_E#mo>={vi;k|U#yNnB>>>Ozmz>7wD8N%5a$2C`cNGIC$Ru-au)~< z-S{^sU{EO$x_&+59QwDX!a=u!_DaW47P)ZB?h3FpF8<8^1-klD#IER9imJm4&eHmz z_%I<!UoF(!dMfWa{ZUts35n0 zjb*)OyLkWDTv3n{eTVh!8hc1}}D9fyT85P?~d_} zd&an9+;g6PVB}fPs&mb`)^GjhT+@FcS+_^p1OE7r>3U#oi@Wv*um9cu`1QY86FFI> z%Mpw6|3AvQw!*E0FLNxqE7JqIf;68Vx$nooPeG{qHLU!#>_UyqH>VzHvYdB_OLZ6g zZE-7bSQp~yL7K726YCPv>bGLDFY#C_s3(}g)QVIk-f~|6RiB^Z^-C?=$an3Yq95m?e838+28XGmwka#gQnpV!(lg`=8M8W$v+ZcK&Gcr*m_YlGtq-o9Zdh*{ zCNaOjH}CIOrIwdn&yzg^ywrkFaNG-{ZK$6vBZLr+^S`70y|Z@!d&p`6JAq$21Lo4A zs5jPFiRvPP5zGnsre3EPP?I|*Wt(l~^>557e#%Pa*xAamT>t!N>I3NK)Qun!oqHBH zJsX!WUnBvu#YG?O8dk%cs?kB3n7$9vQ;7?NzkBePS-0nX@b~k?J&~2Z>mSR0l5(Fn z2^^KK1rg!<*`A9>WLX!iMB&X(L6~;#t+mmt~GOO z{PVH-`Mahp)3USY@7|9h;@o9KKxI6tQ8~e9sFZtD$58-bBgR!nF-y&P4MtmwFU!}T z1I&~S@oT=3{}Nd5iW6c~**HB>S#G3|=V^2QxD&9UtnTfy!*OF5J+E*aEA+p40!IxOzZG+T97<=BU1$_seRV@J& z;Fef4^yFSvt|?T9#S~DHbR%Yy>}We=+^Fv|{7oBHPrngt2x-s??Hb-I%h-fZ3Bk&~ z!~jF7I>2m%k?Q#p?ygM2?((ufY>JFTQA{~NX{~Sb84hh3^I&w7jx7G-mXU{^((%>TvQHS z!l?ViI=J_fWoJZku~QnycA%LLxtgi?NIwYxS$a1&^-PhVve;2CcF)p;-shEQrf?Fb zL)@#R%ecz9+Yo%|@&{Qs1AlhgUHk1$DlT8eXNfm_+>JlD(v@FB1<1};vGx@`drBfn zx*UzpWXB5^QghFifmuTmT%BOugQAm!rFBRL?%6>$O{bY_vd?dYHaJOy;p#0wtNtJ< zMu>m#3(o>MumO1FKg5HLH}8tSZi^!WgIcxyuEUUeL5XIdPX@I86taQWlF zUa}f{jNqWb49}gy`#8)&Q=id%ywC21nSPJ+Q2;EI*~t1T zR=UmY>g!Y0L9RHRepGJQBhF}%k+dP|0sX!CN&8*7b*m^QE*FR421@y|()LMF{V;6T zeR2arIn7KNh;BxSuFdY5Sx0O=o6*~_@E%mXY$LTPq+Q=oCMbd@ltX|4vh)hp`Kv=C zxb})6ycQd4Yw=XfzrsrQV+B#qU5j>{w3P1H(a}@@A(Ly!{89CwajYbP;P;lE~jDne)zS-pc-|Xo_?-%I`GR< zOjl52DfiA(KO;uSn`E(qxzCo51;W^k0&`Uh57{Ehu4h_Oh3I+t1T&yD8^2qb8{Oxg zW{Vi5)HhF(&<`slB5>tIxRGV3s^~s^Y@C83KKU>kZgu`{(n`1j#|@Q+TRl8b@9?5X zy3)ZxL4#Svo@=%VG;1B(jxu6mVwGyn1F^`tmEvok>K)M^y^q=X+e~3@8l_w8Ojk-h z5A8MR0VcYx?WU4rRm9Uy6(@^Do7a3Q0tnQ<9X!aB>*zMoNF49oajbrq}^ zuho58^SaDdu=S;`0M!xjRqh^oIBY*Y;@A7JGNN~gH?bXCk)%I6l&|l4V&*@DpY+w( zUk!~;DZ46}W3EDi5Kg^b?{4>`dE>1X4#aJ)t=i;eUVszQsiHbK8J2Hm3ct=2cv3{J zG?+!onU03Jp-0QTf_{b}!0#wjQ`W-7O5FWGpSc7c+HltMvH;$#!Jo2e#BH!=MtW(*||j zP6(M62u>YPlj&@FUExZ?*}EL~zpI~%MPvWofqa!yj8z1Rok4Nu@7`*FVw*`zNAfkH zJ2dsXpbpt7LNU$w?o2QNN>}vhzC-X{LFd-oAoh3jGsoj0$o2_dyuk$+VAUA$BDAs6 zSI-G>QbyXd6EA)6h}$SP?I}(DUcZq=SmTN$H}~E5x#4cK;}kL}8_UZZ1k|fk$fJTm zE^^dY_~(`682uqSuCEXFHw^fX$0QrNuep9FPSlxh7lX(+j9%i6gaTpxMQQ!}PGzxg z%5g-Ve&qLUIBtIhj;k%uPUYz}FXUb+vR_Ut#61y|7i8e>x6@YWoc_0K&6)ANaP!$+ zu<@H%;#*V6s;SD*49hU>md-ct@O?Pa@hfJiqqFD0C(95=h-VY&a8eEKt;bW|sfgF$ zNqDbwlv2vWs$cbMTXs?juL=Me4t9Pi(D2{ezDgj~3txH|(mP);o4h@q#vTt?D4dFR zOzgFyRwbi!5vTP^ykab)d+}pwN13y&ehPdu?Biuoh@)3#Uqb7$pMmdlgoF=@9w?;4 z6Vz2+KD92$cpORtKYrbjGj1yWz^azCmM|ldVJ?e(S*bzp@`Y? zeu(Er&~=-ZMemOB`7$RvgGY}_oqyURrA)+})aGm1MNhd96CaH3e~>jQmP&R`{J!l7 zZZoe0fpmOTf*l1u(H?DhN@e_qIDEaEbA|AQ=2d2W&?9|V1$`MM87j`ue@li>L;lTXOE}Hnr=n zHDnsc%$cqR>eldp0=3H-877;TpK*PCuG80f$QJrynjRTSR@Nww8{`O&F%F~~{f$=j zMHa8rVSBTEju64w+HE)?SLm7Qm*@K8AKazs`n>i%kj3Ldbx%ALUR+Cbfcvmc)T>1* z29tX;STyt$D-UK;kvMc**cfJTdr`gZP&X*$xvI{e_0+&7*M2q`7|>L$aDO6=eguGLw}#1ySukX;n=gw0hN}5+-nV$+#97$}RL?`F2E5TGOgY+h z8Z)x9Uvr=0;H3A0_uZ&pHWQ*Cs{_PR)FAexlcp4k%pdZ%RSf+4b&(pNujF^DSFb#; z`$%ZMRjstK9o%)kfnGK_=twC)uy*O%BW{+`^n0?!y~rd~v|HgdhQ8d{0=pFrBtc2QQY2ohgrziBsij`5_$t*^kj@}i??_jfXIJ7=mZ&J% zN!gBe!)>l*s*iBlnKC;P>E*U?Etq`DI7VL?uo6AhoRfe{8gkUq;i}koed_<9svSoA zvYEV|osum(&Ys^9348dTCZ+t>2$z<(F6AB>B!H3k;AA{YqXlPj?V^mpK?8cDxG5@9 zqiej1pITrPuEqVAZw*;bY!H``veSOZ%wp|F1^4AY+K=WuqydV<9-JGFPNkM9PFvbRC{4mo9OFx$pDCh$ zKXx>!d~fT1ubf`vBJHn>bsRdTeE(Z5zXgP;-~DQQ`>B^sE$7vZYvn6H1Uh!RB^z{} z{FMQD7tTedmt;H)Fbp(o;C=KDUAHcn@?UNly#=VC-7}*I!_) z|I2ULT3K5~bjxH^w44l;XJ%%GD?D&?aCq4@D2O8B5r!PMbKm}>XgiI^bd#rKc9Xf! zf>*2O;hP)RE}uc@8z-2N>Dc@7WUN--$mqq#2o||m#oJ^y5Sibl!*TnJz~s6ATLnxy zLP~mVejX+RENy2*raS-OzvSS_6sQumo7yj5<(|Q>Wa(Nz%l6l|U|Pz{v{x?Px$?M0 zv6J@YMQ)w~ERZcHFYf^@Ep2ICoo@Uj?UfmN!GBbAoqwusOYYe_B41lc$%{+xu}L3PS*Te4jFsmPzebMA)gG?U*Nm)m*dV^4hNTceb1LW&O+7{X#@0EE^dW$D+UXYaonr_BqSY3`ot=}diCnb*SwM!AvJ8*YmE+=$U~s_ zQ_i{UCJOQH`*Sa!Gk7}U*I#s3a(Q2kUp>6>C~Rohl8jB!>3(E8gROMAA99U;A8;I7 zSa3Q0hwRFM>i*!au)L0v{e$leLjv#=akL5lC+jVP-&QV!-t%3Sd~C=_xWj+Kv2sAm zGdOk(bEWo2uGMYlzbDl>3C1fW5>i)cbH@K6QRi6y_Yqkp%U_@V;sX3*O#UAz<^NUS zEfceb$zz9jPEC{2B#+n0F)kK4pWKcE)itQq!^!lK>t1=#$^2m6{JyPsoXm`)A#-`O z33*tLZ)?_*4IZEL`?&bEb(MnveQRH!#HviGZLBj;s}?}zYzM%FLoWI6@YR(y9beQ0 zTe!vUqpQI`DmK++w?jv#C^Su*1}t+H(AdFFEdb#+aXI6A2foDE&$m1d#L+b<9A@sWW zWrzsWcpvsy7_cAI6TY$${4w0HeoZ=)x?}i>Uz@F+mg5owOWhWI}w>qz%t3qQ9<5`B>_-*J64*E0(9JUWnPYoQD&Xp+fxU{7&DM2juU3oF!;UfKM;%HDeTt{XQn~wE zO|3&Hnx-1R)XuEk$6N&MALZ1k8qhj7*OOrWpuq~}soCx^5Ddb%I!iRgFv7ifMGHi1;UTu(NIopehz31C6GAQ>^ps1`kUBfoX zg^V-sBWmQOe17Ubq_{&YWr<2-bn(%e{O$MamR$BTDGKRS>a^bzEdt%p+#Rmx<6td_W>4d$j!h~tE_g~@~~@_ zEFj4>b$!ht{_IuN@Dgkl+i~-8e_~C}ooPH-gHk-ZD>I}X<=n1sQ*>Pf!JwWzF1Uog zI##`N0N(K`IBavWP`6|3cwhr|Zp*%7e|2LZ(yO(R%au6Qs#}yb%op-TY`lnBvmxjz z%Bb}2^j=L;)F}%_%FLsq51g&fQf1AO+&E&qNL0#dQztqXPrK=s5z-4ZIEswDq3@cN z)j+#tC`1729o&SfXRrt?Fq-O2W^fkSaihD{or zxSOf2v;44&sKzW^wpSA?l>1aSsZ?1iN*1TG#wOaxyJJ-Bg8zhE`ne{iF;*lZf#+bk zX9qMf&eQHxw{^=`oMjF-}gQ;R_U%u4Nc{(|@_{zL0F)>Rd3tQ8Bt^u$9WF6S4 z&(`_VxqH3B7B-)Y#P|HO4*X*eT302L`p6C}999m_7fncPfx+c{^P zMsgbfJl}uXGLZ>A!5WJ&)n)?>MSD%dG{r=#&+n8stT(Lg4Qf@Z%mP%41Y?3qT-raC zu4!#KWgu2{rqC@>W9RqDQ8-3SEo|s{J&s#%k#B*J4n1zg*-cRI1w=Za>b+uIw7j*F zim0<2HH$kaUU;C9#)7&C>jE_97>fSk!gT+*aTAQrtnN&WfoQk74;9RV=kEnfgIQOo zRM0Plx9obceFi<)+1q&Js!>qALAkkLhn^<>NZZe>(y>pmntso?FM4CaC4j>`vvmY% zWk>acb5ziUap)j$1v9_=k}DxfvCsqxJx1$U%aA&yuwFt$$rH_}B$9=Au#+lFZ>&43 z?vDb{QlUZlcFyG(bZxOt6>Pf(V%2{zx6Kyz+n5;n>p&3^*07=qn?28fo&nPX2kGhU z9ijA6)Ssu;aYpAp(;r$Gqpy9S7oDAjA{viU;suXUFTMN7%m2C!UNDO9*0|i?pNyoiTsJcsh?YDRfXU!Rr^j0=)%0EUvS> zDaJZYGF5{QxBU(MlyqD9->_mn5}1(r*u~*Vl)5fdDy_a-WXPS0dt+YQ&rLjFG1F_r zf6eD>elFlXt;;-))hv}`+oFuol2ty|TSlCP#kc5>gY4xILp=z~oFlxJ9(#kIA=&qW z{h}URJc7OnX6__F=?;z$E}uTt2cxi#{Z6#47sbBA%Cuy##!pSGvl$t9jg34N zPHPOUn)%0xrq-sXaU7J>RGKk3n(bS$P@X1~iD~HR>QA9S4^`wVV14gXor@QJ>NjXm z2p|vsR!QOl_EEAB+1qRQ%#L!xKVie0`BF)-GH(_R2RR#N95OST5`U%(i-;yCj*q0} zPit=56ya;fBU8spZT$s#Z;DMu%p;f4p;l2DZrkL57v(wM6&p?Yd>%`*4TQ1?rEt$LUPZi|-wma#y$M)T*?l z)6T_A>&l*UNaBR9Y4$fOx2(2uM?`KPv|cH~0`xnB~LcfUNvgkL6if8@#GnV(yR_Qipc4 zF}HIbDQR&#woe9w3!P#|XDomd(ARSY6NynratqDppDadJu_W48|2$Mk-Po9CG^L zrJhjqY^dE#ClgP^zKm?$FsYJW__|59>0JzZ!}Wp_jMFVT*{dM4;!KpWb1YwInn?kO^cI5&TnfJRvi{q>k$g|$85u2{RX@(3kISDkQoybO@1r>EIt)vv7m8c#FNDbGS${h}*^V zhd@;G(bQbV#~BEt%LwGxWVLqXkWS{U9NSN97^A32$fGWEG88Zz zLR*7dTJ`A*M{$NbH3##&PGD>{(foMgKwh}Bb9Bl9Rh4F048U(nwG{4j6<1HBA}&aP zRvC|%Aft&eBe^UBe`nvVT9ymN{uN# zw6PO+D)EsS{Yk!~Yi6dq!rEYtNp>~u&a;nGN52R&4NnPQyiW3_>$b}6MwKQqK*CZ@ z-S{|Sl%gN2VSvfw<95p_+Hx2FD6PU%*Pv-d*12qVzTslZYVH;bCyZ!O?A+nBoZV$f z^Gd}a!&&QT_UQBHa%9uc+<_XvIID z@ym&GQJz>tx?RV{x^=l0GJCu3tVrc)I!aWh+p`m9)cb9r4_&0~aaQg0mPw%^%}$@q zH8b>lNK>g*kiN@VE3TaxO&{Wwe2T58JzFk+OR!aV!^*qD{}B4KX*Oyyz+1KBrAI$CKWe*6I8c=S2BA1*Bv*Ymk58 zUX@fQckz5BH!#(VbY%I;neg^2l|w5U6|^ zrA+S%wCOm%Vggv{R0o+Mg`t`pm(jmR#jL17G9ybzfu>VC>D`<83tZ|8RPBpd`;x`Q z9a()HtH9DUBTk9pm{+!mDp?_%q}EsD_zyS`u4~_`a?ogvcw2Ux4@z9&8v$my+}9%8 zyH9&L-%~aFd=Ha3=*0AaQ-QjkWKl`Ds8C=$oJKKE}#6)t$7v$vR-KXMSB>C;EC@pKpQXM)uv>_Z>;e?ldkUORn{c zjlLA&rGjPgxsE!G)P^ROO}ot$IK(N#&;opDNltQI=1ui2}@VspRhmS5#R*yIn zJk%g))`~qoWV@xt^wP!L*Wx>gt6pO$z=LdSx7p8Tz0+{%Xzt$jB&wO}55Y{?DdPl#J$-zf^zcQwAdl|zL$3zdlz6Lx8>cTeo|QplWq)6y@b%`x~a zA%lV|ehS^)atD*ZGc1F)BZzxpprGL>%M{`&+_9<=(qaL=Tn95xjY>=2 zKqc&F3J3#u<>0kKm*w(;#9s>29(Y|x`@jrYaHd4RW|~N( zcm8Z)!|md3_{<(cOsD8lv?Omlx0(NZA7|+{fU}iO5vl8>2MY&Lr;U%SY`fZNmd(%q z7W=xzHPf6tXRL*d85uv~O{AEOo#TE>d=net`+nJ3dthVP z_&Kkd;@)Nd=UQe?0Zu37r(xMUJaF+$@iMN?{#F(@%5&$4rneMF)gtRj2BEo2mKx8M zANo%uSji1%B}mU|Al#vqP3DO#0s;Ej}V9X2t1ykKy^G$QXw7Ql1 zS%o@I=FzUrhuuJz6m|+X)mu^+_PTy`Y=B?$-S~9$G-$5Kx=h`zJ1?_W^;l^#CI5)& zw>dPE95!6ge<8)-KC8G-P5KudWLrj)l>b3vH&#ZIx!T-eA*UdvP0XbuUav_z0@9|g zmR{J~!mFPdtBQ6c@2)}hErTU3@D}T;XcsZ0Mr3D?j))8*W|Ww$vHHQJG*`$!IwdFa zvVVMHX-AB$zuWOiT$CQNp?_E|?B#U8$7~NRPgjIrmB-4C3wm?rfUnr%O=8{AWw-B^ z+GWDKQoh2DwT`e7B~i!YZ~Z#s+|oKtq1MJ=EpL!}p)_?1s+4(VM%SG$nUZk650YHr z$~jx5xN6c`FN8B7h`M<_GzS(#2Ukxl6QlfCj(_{~6})tq^4Ybqb1eK2s$hX!!kP#R zB<)M+=cLD-=jR3z0eGx5n6)d5b2;O4K_>R~Y4h|5&;q#$vL9;5}qgnK}B#|(|Eo$rYPv=E!A5$el$s0AH0v0A~$vJB+z>fSNl%sSO z>HGXLHt>_xtgp54bCFi%$5lhA&W0rP1ToXpKMM+ZF0kTEs*aF(dUqO+PWSgd;WZBM zth%6b0b=znqRkxq0NHGEK9H0Yad}}&rI~ZqTGaL}@mr!MeRI(-)GQzn6#eXvbFskJ zwd=IKOm$x6TakC8JSu-c*!k_{!Df5;-16Tp(o^jt9=&V zq>obACiC)7Kb*Q5M&xveU`X2OUr8<~N3aBI^qy?HJp?KLF8*XGK0e+vG_r-cuVWzI zxYi!CgWj;8;JO+tB-r-;%J_7-?`oNG^QdMMpwH1s371+pxQuu!-RM_%>#A$JO5$(+ z;rNZg6(nryYUe;N*lezzlpFCM zQBE*$9+rWs8z8gD;FLJ|pkZNNt=Fu!^zu!sRcELudNvwketDPxII?kO9coGhH-$QP z_SwCS2>{Xz%kN<)`z0->gxCZ0##gLZLZ3L)VhjXG$7Jg`aWWjuDe7lqLWg>NPOXHu zlpu^t|Im>+`yhB00y8~LE@1ot z)6zx!j_|;rus$Qu=k&4U{+vo&!Fm^@r_fbwWXP{H!~vdqN^&Cr-cNgpZ6EO>@P<3; z)N4ngVsBhDHLQzO#H+NtWgjffzZ#a$+EGG*G*ryB5kXnYvNoyYA<@JkbDh8(p&GC# z*6X^}8Mt|T7mXg?YF+Q3I<AZiz1-o+Y_KcRZb=@OTo zV{WfF--N5E<8WC&+!$Co0Ih@E)3CFzc9?(OAtWOfgdCS~^gNZp$zZH9Inee;Geg>~ z&|;=vu6ov(taUr{3r3xdLnzxP0?%P5KYwTBw`RUDUulkPlVtdmogAHAH`11DQT<6R zv(`+;yykN%S^;03YeBOh&h1t1|7gJ^N&MN+{^D@pf>}WYVpvSV#;>8*bZ%AX#aaay zVQom%BAAEQ2`14blw|8_e3D(qGC91h<8(e@>jv#HCpnyx!R*snrNG9$ z&&c4D&+o9d#GN&ns%P&HB@fkTL>`2YVWI;; zn7@3BzBfRoF2v^un9$Q8jA@K(U+~N1>ZmfT!C1j5x|J4Qw#tdeJOfmd3iIA&VU+ZX zjyeywS)l7)FpfzYAE`Ezh`Tt_xmTN=rPJPPsI;`!OxWQLD;Jubqw$n-7zmi1F7`Cu zkGDt|@*B%USp~|nb;{anTv*MJ`bj~(2yV9z*lQ!B)?t@yQoKZP=K?mQ{G0NXLRS#>@(^(0 zZWHP?>$_v(?|L%$V|m#%X62sDYl^jpxMxv>v^S6{*d}41+dtF&=}I8FDqwA<rVR&tRadB(&!ewAK8SR>3`+LtmUaywsvkZ`ee%*^6|VrMj`O??xA8eQnWnGOFp$7l1xWWUm`?Nv>@ps7e_UR^DG`RJ+oe%C4*qo;lceVi~0GA@EZol z@D`KG#kc^5*#eh-!`W`!G~_}zT25klzVe5y*RqBPj z(kPluhpp|?!}DXm`+A&MHq}uoy{qxhWOHgEp?y9UmHW^G4~YFF6W$cXf;2sj%4G=% z0O+R@1W$iY5B!|_SU2(s>smbG+1Kh$7Fj) zQ?2Y(zDx+dilnMmH?=qsxHnNd;Q!@7+iZahaGRB*@~E$IYL*$UoSiqUib^3?vq%gH zPO=;eWUUXWYQWyXzo{aYnWr9hK#q6&H|tMFj+?pm);)fYvyQBG=8MD21Uq|E_a~@T z#XtVGi|h~jQ{Sg%GtO^6M7fEr0C=0!E7pA9(`KZ!vZgvQ4&mnG%OHTyD-y;w9X{TP zdl8*+t(6?a9m=YREYuH6zF|B7_l9#ai$QliZm;?_OghdtaQ>TO7QUJV3lm{mM7n2z zU-^tGWT(0hdr|ItvwS}tSzI8!@LIh3on*5bbneTldPWvolv2J@m{BgX^Ygws)M$CP zP5TY@Z$^&n5ve~emd6RZQu+g$iA%RqaM22bA08##{r-Z5kAH_Nm&qx8Z+PaWa|UPa ztvuSw2TS#tH8%1=9)PtY?cy@6j%@!8@n(Vk_|hs#HlCuCSHV+)<7hsze2)% zu_TApM3CQn_rCwv=k$NaLHY5wOO@Q^zlkuyf4euv{vBrJE#=>5_OJh@$*}#yH}bz_ zeQ+{S_}{ny|8p^xAx94&qV7R~7}D_yuxj8eE}7-tpY?s4xV|~El1{!K4~y8o=lMpI zv`S8sF01x%s}9i(%@jOA528Q)$;h}H^Ji!nqgQ~FS>oOh7qIh!j{jMDu6&+CJm4l` z&g2kb$1HS1HA9dWH_0M-djtV(S` z$@llDX{n(_G}q~Iu5bB%RCWiwY}HKXN~Js^f-6X8CC4y`1yLc{YwETRE&hYPiqsx4 zmvx7HXg2Xu+Ai~J*l7R{#LPPi)RTKk!W=%%Tr@c%mvwHw=Fh?Zs#?25zPhrqH0Yk8 zZ;6@WgYtoGWxydD;N(}Z@2WBkMBKgC^EOy{dG}PH9o=;bmS8FMN5aTRkMDA-A)rKJ zja=@%kgu`S2(>@>8d;*704YHTS=X?z8>cr}A-k&5G_=!6i!2XUBgCouaxpL6hJ$!l zVWy!Z=+UoFd6-)Wweh{FB`Y3_XnaC*q9EiLq~f3b$xvq+KZ1ao%8-5D-b#J`*YLmG ziDr!~d`}%d%o?uj=;AZh9mw{L-R*Gi=-fvTz=B@dO>|N4lJ!wNmjX$K-^A@MxD zAwsaydv8AhGw1x-p#2b<{h8uJhU}e~$+;?aQ=VnwX+ni~4fYMC>x)&Gwovg}Fqf*% zM)R>LqWl)$?ZOd&u?OnRDe0Fc`@hB}|WwS#8m0g7_O=fq`yI%6iil}h|qb4quJkP{3A>t|5; z%NA7yTai7LUVV6HqcZ#A7LH3x#xHO;Hr;oaq zXJ7omG8ftqBy%Rd^SAyzobFg;s)0^(HP=;BiBr%X+2nr~Zy9IE z)5NTjUH5AD$HLzBsew8=H-*n;r8pPtLYtg;;&lV+5!U$*lCZFcEKmDmI z9?SX(cOE6cgZynD89!X^TnzK&4L5zmROVB0c>l!Dd@cMl3soPC-3P^gtJgO)EASz949Y2_3d!1$rNzrxB4dFOZ+AB z-wKXG{RccUG}VE-&rYn}8b01Q)vnq9F0!Y(+Ia8Q%bD9Nh~?d>Ni|dIK5|e>-#al! zOT4SfzH5mlyxI!$MB=G2b}=mMaBS!BTL(4)|C5)sTQ=x)XB-MW>N4CcXflv=G*?sX z^jweYI=#zEdK2t_=A-CSQhl;9D0s@p2=~~TZ#hhd9x>9BmH*dmSPK@CgKM=S?J2p z+Y01V=UiF;v^l@1@I$PSo2;bAU5w~Ig&dLmiv0R3GYa}&Y|OW7|GU=btOWA&KUM-5A65*PSfCsMvnw$rBO8D)qAz3X!Xo9$GeDddBhEBbVoAO^Pzs|d>&y2 za#d+52qZ^*Lal-a*N<9kbf9;0sRSI;42@Wgjb-5jBQ}y#^$CrNOo-%qfWUP77#>ly z+Mx~ZR&5>W!0hycP;Rj{FqQ*6xutT&7mfK6({+X5Yr&ZDy?<)DD+@H%iZ8yj}`{}{x6H? zr)MrO6?c64-}i{o(6q#*M23ZZo0+ZbTFi)=4WpaqXnr5k$aC4^N1t31b^7FzYKPV; z`|0Fz(HrjJ+0K@fj+(ExWXHDamOCf*zkAtnNQ_XUJmkxyzqJ;m z{*L9XTQ!a-65S%5kl(ETnJPK=W!$LWN~m&<<|uF}sIA3v<7dBFWCs7q>A^cQb)|00 z1z^uZXoi%SIl%`(nhm%?KGZ-cVU;4;Jz+`GK;-$t(D70~3WF{ea+%P-DHHssPlbs#dU8!CVaP5<jO7+d1j=(1DLGCU^cY zkf6H38X@smXG~CU3buJt3A?9I!}2*(yiz*q#)RQZO$_>J=2BBS%ueea zIAwwC8r|y3P9!@!IkEY*H;OPIq%&n=L$1=%VhMDPDS3Q4Hn;kx$&Lm>7e=jj(xF1E zvVl#F{Q!}NpqL*?KB`Q2VJZa%F*ZQ?9Gn)F(igJzPGme7OZ6eJY!^>(R4+cw3%YDk zXF&Wht_))h;w5AaoWw|C>)q#h+lD!A(9fw4;xk=hys((YSDDT!Qx)5tDlLs;MJTX1 zeMFq(c4IG5D+y!JUJN?j*i59*UGb1yh{8OShb|mgJdn?-pOFpK9o{Wq=$RO~3clxay>K=Ipq0CCD~o4;4`4q@+&yN`2I>=hcD`z> z&l~Qw@U93w0WcL~j=p#0p=jcy9^cLfRtJuX(zBQ1X<0n=nFK|jeinq_*7PEGSiu&L{4LQo&3|e0bl2Z?jGIq zSF6@AkD)V~c4U}SSGxyD*Z3&xA{u1H$r%s4!xbZA{XqP-JXgSjQ$3I|b97MxS7~U? z(SHAI?z}UsYupn;TV3b7SaHp7l*amK?Dvd^kQap0$_cdM)Glm<7ir+axZk36@g>Y? zA29I?IGFut(wVIy&z=*UBz1AbMvx1DSm?xCQBud{Rml%a9QxlyGBQ%tUhWY+nDiE$2+1YwM)i>`UCEGm$cIj3H}fA)T@W{>DtRZlgU0 zVO*l|oFCC!_eA|j?3xI&*vi%1>#YWW)>8N&kPrUBV8X3ngU+a!Xrp2}AqKD5){99E zap6&77oJQgB5Ea|oFqEA5)#210mUd?A+I_gQmZ3u&$`cQ(-d3XQ7F{r$eUFFBI^!X z%a`;eDG@bEI}gji40VPN2#tXa$JXXa?Uf>&Pt=?y#4^*Es{MAn=RBjH&<2M&O)m@_ zXH#0+ba-TJFH6jPLAZJfzO>_WN-By(ykfuuwscr6{ul240nT6pL)_lYqkA*CW4ya&JOb5#A)IWHt(IVyCYKfpe+*2H8otFX#m93;@Y$?4^d7Gn~I_l zau>7OwckWzH*Nj%ic>)RpIwXz5z~9md%U)pwsJZrOHNbA#c8Tp)2m(p70|lYDw!6M zt&F#(fiZi$w9^b7HAs7`eg$08#Je(NMXRq6A!eFhQYVd`3zGC&)EC`!&YpU zE1kll0JVt(vymZ0cOmaBF!0nd6HuTO@M8F7YSp4f@GT+wFL(qO898ITLhzJOOgaAl0{RQIY?np43WI$gFpShpP7D~Gv*rd(|Ww3HOI4hjgR zig;h9B~fb*{6k%xDJ7CDKzm2; z!iCCVl8b~Cb8mevWvkk)49v8#=-uQ^InY)=k{zKlI%1|SBYQhx>|Q#wp={9Fe5$>& zBEfh(?G;};gqxE+57}vgbZycY>6vs&xJ8SH_wXCmE}TAL-2S zxt+o^7t{wD+?~xZa?WJy3EmUWtlpYQMEj{>iD!Emqm7Hk^JBJF50d#j{H+^Lo*B+| zZM@gVV->oce8t%nKk2vOUa>H7los&H0WtmW?>;F(7s3|wJ}Y$L3R1HfHAzO79gsdAwA1PO%|VJ z`MDTbmssE}AkiNfaa62xF_KaFs(c~SdpAIQa@e&zCV&a4U^DPT;NNWaLRY) zod720Bb5_uzLldKif)(gX-n!7;#E?cQLY_MQDFfAd9O6iR(1V4>xo`f zepwAkNSKY$3&Re~6bOhtp4KG%Fxd1kck<8UdQcR@TrOJ*gt5eX=ruOj+DzxLmO2By zP|63rYs^D%r-12c9bySnc|zb;`}UpcI+AF;{*EYIBUdXcGA==6wr`qqddelLFRsGc zQjw)8v5~>0{?%Nm;WQ)u{x)8g|KPs3ops|H-KYaI2>er^RP?2VT}2sKYo0*F{~9we z%<^Dk3W&`OD5-*cG1gRF4gCt2kBP#)(mT z)}jYQXusW{Rni`4zQHb4HFE0~dCpJ1`MJXqv$xPf7gGX9b)wQ5$0)HYs#6F2uEa;` zm%Z$MI!HrA1@X4qw&WEE-Lb?ycDsbiYR4@nS@W#`UjB`lv`F^2ZI)bDV^I}Tj96Ok zwwX#}Wog{UwH2Ec5*K`19*`cju^usm_l8C{iP;vzm)?Enxjg4|AeGRcM4|t5X4yy@ zm<86KyW!FK=leUw+rR;5{|aVSkKVcV==fN4O>x z+2Ys5rBF}<5*}Vigq-c}DxBZ*?Qryeum{d&)q4odjX#5@-*I`OQfEnxYmHxYb}*`a zBEXKF$TkU=5Oov7t59$`9ps3hx*gOEdcIyiq6=I8{r$ynS4#0ksrHR-NWzFE| z;wXZEfPzRTN|oLTEkuU_lqw*-2?$6F(n3v8>0NpWBvJx|05J&=l8|t27@hB&bDit@ z-uL~^Ip2NV`~f$2*?X_O_WG^$TYGf}j?*7$kc>D*2fOXBDO#lo0yN;dxAsr$*~Zfv z;-@@R`!I69UaTKIoX?3FO`yGwTqH zt&l_Y{y4GR>L(K_FBrPLRcX*DSf2x_bf;g64F^plWhCaJRN zOIb;%xM;8~(VH}$Sc=lsRr^p}y@y=rlN}jK@7cajFrflVXSEZbOGtc|y}K)%vVU8* zzs=4rXgftgKo81C#XtmYDKC1)P?~W<4cp&|M0% zNg*C@nfs~5^Hv^;74QUY=K9xzUE$?{6|7_10AyU|196SFqzbEGIUATvT;D8yrpUs# zrxsX<{mEippWO(Q zMIs#wC)r-cyi$J)j$48&ncCkjkoJ`7FrWRToVJRT($AVZJZqw0`*?C~b{&0fa#guy zSKTY~yV)V|2iL&hJax1e>nl_9uXektkaZPqlaXnw4P+Q}2SBNjAlR1sF1KZc-IKK^ zyXcwX=pmH-CjpCwH-H55=n*bL{lrRO(muRa{joS6!F(}wtz1=T zF-5J5_NZjLV=Jc4W5FUdZKs+F2dlqzlqxYaOjQqK?kwY?+Mh_sQ;f`Gdir~dvBo1q?j#oxkw7z$Tfi6XwO|d%ruqGaI1)K+Bcyw;;q}q%AP<#gD1oN> zPt!Z0j0#rcZ3HT+NLs{xh586(7+zqL3xC;nGSx-AQlqAB*;JurQUu?m!QMRoteiPf z<%os2`m(Y0RG-1-6)z0tNF0VeJ>MuAGWD)Iy-rBRQU;lzp~;zR5IWMSCB@yGkT~vs z#+}rx?J_W?on4=MThQ#`w7pbG-iWbu){dqH{dLxIsFyniW0uRrn`FW&Lhbhs7#+;4 zt!D3Bti9?FZ?HUv=apNZ(IUKaaC4Y2Vq^YitN$Rkq0SoyfTf^Bwe8c=85SGa|MZAs{M`x7`G z0m2i_t-v|$aEUAF<@tQk%R6O@N^sQxPQcJRWa`U2n$tl_M;XO;ecdLp(6H2O_n$_e z=A9E*xLDWfmsDYLSldt?xhd58Ab}N~o2W8!^gZr)dVVG3-WCyZ1iUed z5A&G(_DN7zV)rKX`8bgj8gs3EcwC0}PH|27QeM%%J|%Z(`wx);dJ}s6kd#I z^T?W#uwQIZuc10f`Lb-xpKF{g(Vq@fE1=H2bX;9=2jq*-!Xmj(sWC+Mg^d zB>>mj!2|2LF(q@mBfvL2mEU=?c4xAHO)}}&aZ)q*b2YF6oZ#w>1fyT|2!!6(mN*G# z{pm#0Ru}-cXm;er`>LOBXVD!L?0TGj;?lPhk;;uXmeT zM3aw_;hM*BH1XpSy+0p$KpLqnP+uF)jb#$T&9sj`5I8qDXL%05-aiMFbvf>yM|o}0}<^yUyX_ztCKz^v7*M;qPLPzwv~n~yoDW7J@0 z!{#jq;qVM>{h$8H6k4z9`}~SgPp@hEuaIA=8eX^&eTf?6KuMu?j(Gbv)Dzyz+(z2d!MK%<*$(%*%;JV>W~NS$*s*VtO&i=F4<+RU=tcL&oqc`4edb4_L&PEd}T3SCQ9 z%46p|C3ekt1aYT=?JUv-N6U5-r7cf@;3zN}XnE-Jpa=u&5iW`gJsw~ndThtO5VGuj ziCV$oLo8mhr+tnXlw8R8N^5)1 z@Mg>&%{Fd!H}370^rYQT>CP?$Ygn@3K-u=%n&^4%0AuO~QR!0j(o*DPG1P085)`N9 zmU3${7*skWVcJ|%d$6WM^wbKoziMkzrGUO2;<`8MVa4B^E~Pajuy)i-8?b%cmEX)e z*QiQi4juc}82Z=%%6~Pp!)OdWw1iAF{Mt zMhD%Lx{0%{$p_C8wXYlo=eil?)~P|t)QjBcc65Qq%%@rAFKrqMi^?aUUss`LC> z3odw^{v^G$XmiFau_uk`u;o}N|APzAygMM>4dWhADevAF2ZIw0`o+L_+T(i<6 zdu*=z){^D*AzU~jSY5pG(H9#09F}q;lDru&7r!l7L256|Np7(WAG$aPULbi;_N#VM zIjQLFjGP(>n9daO17*EyAEo>iLXq-FRJ9;PWbJZn?bJ!4V-wgNG=_M(xtB+YMnNdEvz7eQYFajY`abJ|RB6Jw;`8 za=FG4lR0CmVpNOOmB&ak7K9Re6ca=T*g)z6R(l8@6b&S?X_S{kKAHg`bfS4Sjnl=Y z`P-7)(a1s=?5Qq@13Pzm1xFtjM_E z_cE?sg`HbEj#K}J0Q!Q@3P!rCO2>plA|DjfD6O-pf_`H0?eQ=lAOg&n`L<2}8qvBS zu;K}of#;fA^-|dAz_f0c-gFb3h3C68s`Wqe#1VsQ!a1Q;p@4Ek@-GeMwOXN7mZA`S8o1#)O5NMfp@g4zp3YAgqcaO7p@#fmCSRQJTw z|B@r_h?4^ZsB%Oi9atm1b{s$f-@M=LKkK5jqQnbl$D@K_B_a`YX0K(ohhiZS!adf zQui7=KE?N%d*%h==w{VRw!T>=xhM*9B-SSyxy3CciYt0pLmf1U-jyIO?wu#Iwu=>l z3;eBDmIkY1+ZW1r2gY`Zou62QY$2`G>YlND{WN;D+8T|p8S_$(C}_4ue!nqI?sHzN zuDFJX|HIdY9XsIX6{K`0>xx|mav#URP7t6@$M~CirhZUSIGkpwfQMZ-Y&q~)V5zFg zqkK}@4-q@J-Pbikv|+E#>;IIC5Dc;nR`+?YDITDP7DZqpp64`51d`Hx|&vj)gnNnlPhI7o@l;wsErs6L_`B~>@u&F#Kas#)*{t+FGvwz|T#ao5P_-%M&4EY@QW54$LOR0M^GE z8^a5VZNPy@A{-5^NRx0xMd*~a)*`oUyxLAWNFORT#Wa1S`Gy(m^@i~ob_4fFxh5%W zkD}P0E-V$QWYq>^Szh$2_Ea`t?|StakcAwf!l_?wEj}gQVL3!HBc>jYSh+Q%tP~L9 z58bQlQP&f@16rm|QdD-4@pzaG_Z7Vq0fUV@^K}Zy(Z}dV)6Hqs#V<6Oi-@QcxyZhh z9S88@VmPB1=XS53IEYk88}q!eIBS{z;9<0t`$;^-_GK)0@*$EOxQyN4wl)@&YUhcz zYKtO)K68tvi4C-WF+$LzHJ#oI?Iy6ZQ`{4xO?q(JDA0O%TkRI-hB3K5tzCLq89((Y z#>eh5y==$gko4)L5{J?Bd`U$e5zM~rY6*)D@cmHlYPdVaJ3FWE=26wR3js6>Q@D3? z*V5|hP0LUJKBZQ9dU&(V#^3LZBXg|d0>}w*uI&E!n&MK?3}Z&z|CxTG%hjA}Mxt4w&?i^<{44VgiXl_|v z*5q+JEJS0_1FvD9?|dtbwTcQ0zm7&8JW)HC-}qI-E14Bn9FwlMG2MdYAePYt*w67F z+_hF?R`*4GQ<(VO_j+?qmBs~uxxg+X zD~kkLo^>4m;WhZCIJ}rePD&IiPO6^B{&v6N3!MI!0_tNFgg6kPXopII#=9j-4aIQb zDa4XR_@_gudX}!z_|_@I(gfX$04D3d)LNo>ALx7;flgb!K$z= z8$nR$;Z_jk)q)KCvYpvx{9#jz^wimz`&G=T@(TW(@=>i0 zyZhIK*Y@ojtBV&Mw5T17{eY(~__R{8;->f}h8g^{eOCxYU+yNdPU_E;>ci^|?iTG$ z>*jw6=eu|KCzO%AQw-?BvB=)LKC*#c?o>!N#I&RDoz=ySdnxVCXZ6f4GlHhxCf#2P ztET^SntzAwc*caXw)dk0SGs1V4j+~uTK_~jn>*1VY)&YMXn&T|ZTNGRtSw09;}Y=e zHIz#EL5Or0DqSd$zuL5C!~M>ep}?An<;B|;>nBzc==};kD^4%YFG!7a{a%mTXd0i( zm_yJ9+$onU+8N$z7reM1!}mKvYw)V}?CiuwBdwe)1Sh+oV;kj2BYh2g+WPvg{NXL{ z$luJ}gPK;6GpsHQu;l`%Iahvi=1@&`v&Kfw3&sgQO|8!6)3o-awoMLv!h{@dsf2b) z*S54z89a7*pK9AklyZm#V8&$psEzL?Eui-78dYAN@%uG8HHX? zo$BFW5sU*Ts@aA^jx16RR`NZyWB9aVducYjZL@&hz_HL@bUSNl5@G5R($Xh@x=CKe z7y+^X?%LKd15?`wT}j+za;Pt0@}d&j%pT@%7~PU`K)?>&7I@Dsrr=?Zyl%g3kkC+%k8_G*r)z1W3pL_m_~ZMvQjF+retXF(t{|ol_R5k5#>zR(T^08_)uR8hUBPy47R#@ru#AIr|Ig z5nhbx?nsg*hmmHs1{(NB)$`Pmy2+$545(5`Th@iqY0P35P$LLB1dnvxNu{PvWTuT| z)>n{dR+aRN6Q{@4-+DC#79Pl9Uua!KSt33r1&*8ORB5!PgqWM(ETv{-7bE1i+_J4f zPkkFKT3x)2(kFVHD53lfv0azWaX%ErqQ-(yE}+?s3~RA!lD~DRP;-J*y7Q1sQB##w zI#SdhJns^4e<&O}iwGrto_g+VtIR1_{HOF|D19+#orEIp5lmiO&0n*B*A87o@*NH5 z2ZzllfHC4FUk`wqtdJFb3M`Vji-1th_^H9)Yy})XxfI5@tuLNX$=vH8eo5kHrOSgEZLLeCM&TDw zG`fEEgjnlts8J`(jKW27{nTi2-SC7~XGJ9KQD9e(bf;XRKup9TGw9MRT2SZ8>1qEk zGxv%axzK=Fx>y=is)Ci^EdR_~LIGeT6H!_3PzZOi5w>$+M0Qb+-O3OABEdAx&nMvA7#Xc-x28@Y1 zxSc#wb{#}tC{%6XZSDc_9O157O)J*p(VUwca0$#V#9)w9h#rlNyTrjr4t7Q;u=7Z) zBUUpX=@j6=B#?W|2Ynpb>BLZ9nZ1zC-aYQMaa!jqV8YtzTeK%{@aSu@RSKj-I%tU0-L^JN-^7%wt;t^75N&o25}3Vv<0dt< zq*D4N=^*qvcZFO-`YOa@9N&6iM-3GlSlK{DR$-7f7KOTBpBB*(#^C{DQe0kY*4DXi zyxf*ON+FJJ_P*t9F_b%9wYINGIU_w|LN`oMP-@cs7UBy zwK|#i>RvikJ$rf4K)O3=pw*6|k6b(EDeL|Z-q@v@9)T403t>laNV;3Gh3CZihx6Gj zwqJi@eHb-r_iePW32?q`wvKl7m7w|>f|{XTTInc(TMbr8&^@Bkc~-&RURkI6$3*^9 z8QHcP?%Y4y_~IFj={;iy^J##EdB2mR)nIjkM2+9S7TASx4gGk#l~$bzSNN0xyMdzwdFU$?{2T}=_EYM&=u zfL~YQh=f#lnBFqGib`+ei$tIpc3dao`i}nR;~4E9FYO-U82JF0z919$N@naoxgXCA z|0xXezuiasRm$Q^rr+9dAHbqNKAKJfH8ma_g4@M^@M6txaHE5DL?k$l+}VF0|A%u( zzQ5%^*4X*7AGB6j@e06XTlGyEz+M6Smh6xQ`&Js}pWR=OV1{0CvQ!Ie&9|~$wcDUt zujJ*&G)rCD+UreG@FgBRGcIj0C#29+L&Bv#n)_wjuJMl(-N^TF!=2(=6owiSAlV;#Ed*16hW*R+@Y<2-E| znx=t>3J{3qQa-tAx7RHznVnrH!^a+x9Wr2h6m5JN-$A@3)>KF-3tP!|Wn7rdx9X+> z*2=8=eH|^M+1Lu8ykgiHf82$wRji%@rO#dKO!um%(Iwc$m!k)_z8E4_&$_-CXvH`# z(t1&r;GWg4wsT*mj~gtWbVPP-QOAC|fQR>d>wbw#)J#=8z67ZCL5n`Su1tVb8f~W* zwY==9sy=kZ;Rb^li?8D5=`5hkA11@#qmFwa<87vDD6=64c2YuOTlN@F-gy zd^skqdR4hKC5McRzD(L7id|h3KO&6=b$0j5Pbhhg`%${Sm93(J26F}D3+%^>p_F2h z%c=wH4H|c{86XLNI&@ZYI>Zn&U_OZayuq&C*@=c!63Kt3Ev*F}vorKg(@a`3aJc)y zP}u!`S(~VfV-~5=Wxi;^o_@3{beN=;C$ZW$8n$U+W^<^AQ<>zb-jZ6i=%QCoypKo? zHa3M{cg%M)X3@jSaQQqD4%kZFy|FPp{g@;_DbfSCj!Ci~zqt|U7@CuiH1B{j2dM*m z!k)r%sgWGuQ{s1O#K!V8jLO{y{%lU39x#58w*4qmWVbr?0B4J+muYc6AWHQIs?1{L{NXq-kdF+Df@Cl!4p28OEUu$7MI8$+X(gs%MMg*XPPLQz@L; z%f_@2*Sw+#`YUb5t#M>YYTL#z;#fC+vH4RX?aMsXUMAe}Mz`&{&GVuW&G#eRf^)X{ zF=g2Zwv*>e64~yKj4pAVjkJ$b!sNwghUmm z^WNMWRtJa6M+GKo%V(NbG#CtYLL#Hb{${+|ofp4yMe~7uvT7l^MW_J=U3W0rdi%14 zD#H;Wb1@~|*>0p|Vhsk0>06Io^k6H=(&Jz)Dn3@IUEJ^ z&cf~8o|NI)PvSSG8Fm*FVEXN86~Fh2Ss#9xs~J7{cTS=Ax+6T}?v$N~ z4s(?aS*JeJ(4BYU8<4jJ<-eSNG2!58tawDBb)abNJo-hu;mlp-kG;na*K; zcAq`tK`r8ZkKKVDwb{&L;bMdge|k9TNmk3Sf+Qd=Em_zhm}WM5#(LQ=vTt;Vcq`&o za#9_8gw6oo1mJVZ@ElVt%Y6;kbO|P<_CfaX6h|XSxivs-8ve9Ft6zOF(cL^n*US`DUPuu^5vP(l& zPbzR_CYqsFSym&$Emc6G_A@`nLHVb1sVyNYHVU}#GB=8L!`$6+Mm|Kc#4|9YeL_39 zgr0CjDe(WWX{pkj|7?y$`jKsepY&0N#|eqwfmO8fXtg5uL9?aq0vRczWWNQ?QH1&^ zH+u?;h-FsOwQS+ImjIjd9rNZ9E7!)aw*3IIb)EJ1S1+r>5z-ya?Y!!ILM56^`DKe^ z5-1zQUZEwE9ADiYsZ#5qMC%vnI-OD!p~_d;sbhe*$#s+3k>;GXvRG4b!Q1p#3_6-L=5cNKZq8V9 z)sIy|bComo(+;F5K1|v;a4?XElP-zl}fo zt$ZHE0g>Ua%fG1{#OEPF56Z1C?pM>LEjigkbUbAAG1IPmZmX;AgHy`3p=)@DGx)vz z>G&41%b;x^*P}@y;vpiCoxK9t(CmbIMh!0;Csn$Z<}Mh?6C$`(s%tR=$BQ^?N8n|B zmm^v>R3Rz>As}bf+E&<=^@2xTg|GQ!N*{97V6WWkTJXBn!RB1v-GeUc#TNly#Cfx1 z>=(B-FR2hWTQh3ja)eBpCcpZ~xvqUqni=d%qM?fFAF^Dz^4Nnvz9!KRpjXc(Tpkekw$M8c1Gqb-QTBmA{FX4>7344* zjgSu69-4vS5%^`1GVA9dikT7(Ak)tVu*Nz#TGzJ0M1lXB=g9~6aid{5Oxm+E{Q8rW zBc?Bpw8uqRzvh3{PwXj;e^`Jt4A^&A52Aa$+Z5=3!s$di52d5DmYe9kKI_hwsgb$* zc*zo}hOWqsM7UWr*M+6oC#LM zZe1Kz9V+FN4z9Z9e$IQ-cu;MUFDbyce)O-vDN`=4keg8w$&%N~)7L!f&c55xNCh&g zy84QVJpDu6=0%dBElH&d^#GfbeDdM^A*6QIcE=@NB#+m?yx}c=7dG_#Ljo!!I(X7| zwqCq}?Ut*Z$976TUYEt~?$et&Mf`t}Zn}g0SmhMtRc%=ye~5C)KX)n%D3X+V#OHX;W`j<5&~VSB;0=fRVt?*>2soYlh4FMm**>_`~j zkWkJ&aIGl;W`CAo?5BGIn{Y{Guxwe@;~9k~m#@BHF`%@ovU39#i)QO7 zKDdxt)83Nq#9`I}$I?cg>#6sxRjd&50lU7m22A5Gl2=M>v*X^+SGT38JP?OA0u;n& z%U3t7hQLsKAe+s;{DehqE1^o%C4eU%+^o_F@ZA+=VlKDBrdxkd4pHl+ zM>BM&;*8^XlCS}1g<2Sm=+D?ou8gW{uKM^~Vha{oQ!u>8iJi{RA@>*W4-H8ru;l9f zZ00(dvV9L8TV~SO>hcF_$!e zoStXcUl-yo-JWJhdtC|n_Ib@TE%f5i+pgF zn(?aXem)w=rw||EKYH%7`;(zf*1J!_0bZWqt3JnvqP#0T!0Om6WVhEaZRfk}?e4Yr z1+uVroSyc!ErAohz!R67`V*_Oj`40$hA1aNU*wJ|>!3?bYK|t?GQeR9YT6r{_LzKO zZ2M--VEjbsV{$!5-(Rcy`(Z7ogOfZvOk{jCY`7v{X)BLNZ1CHYlr7vZJwPsho0z}R%h<5>E zu-mQLAT3^wb(G>Hz!&r4G)~2284~Gd*t5UwzW`*lk>&XJe$9=$bIj`OoK4~}NT&dn zhh%HQUe8~jJH1*2hC4f7{K`|qRVyi_2bG7m)z&s?q`o=4vIEV9uSigQ`6r5Bmdv}f($0ZFoy-_Non_lSJqP6deyrCZVAKd`&6SiN+~1V4_rX|NFI%wToJ80tI(gC zJ&oKoGGpgoP!j_jUy=W2;Q51H1@QnGPLnrj;0F5yeZu@q9{(!w1-Z%V&>TQUvDPl z+XQm~NhSa_t<+vVYHT{kMVAuifdERJU#p(GmJQqG(`DwM1!epzRg4q}83{>s5E-?$ z@;cgV<$|LC-w|cpUM9PujX4{Gm#+MM3?NsQdvXrXGK0QE`aE066E^4~D$el?6h6)7 zKO)W-hbZyb>tenbu%fuQOEt4JZObJBE{a2Gqyf!=sJ$FjuoSJG zZgY|S;eG)A#jF$oBW2W@SNa{+{%m+6BBU{F@EG}R|N7Gj3AzTM^5e19<7osqQA7no zH)Sr?oSBjO3uZ15nuCc7Zm}wunzuao4c};CCXDxUbn@_I72p#WFa5T^lPG0^|=J`4A@5dXo z9Q}FX);}8py#4L=5$W~#xcRStO7XmZ@&7nT+5Y;+Z|eH*|J0+4O$t&{^}6mK&sV@V z>vQ4suMys7OvD@>3h?v}NVn%Onc<&)6!Hn;JRNzPmyeH}*4TEH%`0U5DMn6~tNN1C zjjLn7V_&>z+h2|VhG9{TGGrFuHH;Z(<;ezZ6!UMy8_ zkATVl^y$+XHnw8pvLHdtLM2m?ruKG>`*(Tyc*^_R*H3Q-u$NT)?|muM!^NXtjJ}f3 z{Qes6r88&FynDo(!1+e6_(E()z$HU$@f|&r0*;@5bep#wzgo}@U1JF@6X3jd!4UWS zuI$~ke>}T}l`merSiE}YHx@a0d52rFzux9bJN@15`<#RDrQnXT|M~;5J9q2}t46yN zve)|SuJx5uKL+dLtD8#3pX}a?IV9!G5!NNSe+c@e;+BxeC6*|x`)h2`KhO9- zG2lSYYz+*vQA(GKIf93g{$4gI1|yeWy5|pG z8Lg~Fhsb*WIXp`_n3|_H8?Bbyw5kL5@_ZMPZ`r1@zqC0?dkp15cb^32eMt#)>~NlP zh#fuVT>>%UfZM%)Dwg=cdQakEH-6L%&ykNbJA3JU+e0s;aoIcLG6!$(8|@*BksN+w z1(4ZUqynEHCNDRXmMm-_=j5~(8SB^A>RejvvgCwC+fa!@iH-v!jm7k0Um&Jf*W@_$P68x#0>RAchAl4e7rNeQwQ7Jsvg<#<8q?*zHR|Fx4gZDMvKc&MeZED1_lZ9i+sSWz$-^SyE_ZO>h^W*9A zhWaSg^p)BHZs)0sI`dzSx-T82qrkN$j)&bt`A`T0X;vCZs!$nX#$Yq8)uQ>yJHj0Z zj*075JLu(@-8(?OiPlk&TpJ#qXXSS86Kp6c3)-2vgoRX)KC;XuK-=x7-A9A~FY}ga z$XNP27i~AEyhf$;13J}3-Bqfh$14y<-HXp`Z5QGb@EqY#ZHZpPRw`TUh?+S7TVr4z)nXc|H z(Fr-*S_%{09vRZa_tvI@hDm;g#`mARSF}P>*P6*V(nHEjU6Bx9p#M8E)s#wOnKQKZ zE@+ekqe+UhwQIf2n}%z7s$*31&@}6ISu>>loUrIjcJ85v>GqtZdNpn5ZXO&Ej}3WtAsB8H`*D#+J^woIG<9Es6~G|G4x!#0Z^|y}jOx z*4+bZ+VFpPKh~}eM%Ri}h$=&=IF>prwGY;^`|Wwx)3vSPD66SUt<|CrFec|#VM)*L zlJDmCQ>I`8C*24`9)kJmT*m9c4)%FLIIUsq=ufdiJx|6Z6`dJ!DMQv^6}X{BwK|`` z09-?p@9)ohZt_ZxONf?7d#VtJc;nSqXoSfSUz0D@fZ)P66T%kJTOHE#`OQG=yjjUC zFXdgEIN$c1#@-bqGpE>s+UJ-#X`TNjw9aptzuB?lTP%$XWlwsyR+e(CD%em z_TxcAKO0aW43wI!bMky>$k@1&su=_~C!tbt9Wv6mh60|BN)|@oLOemfQx20n3F|$F zZJHI+?gmCcx{(8rljyj8cc}X5M2X{Ke}KjQ*wj2Je=2T%{F^KvYfqZcZ2E8&zmUDW zYPv+OYHg=3#F9&5xKPkt&?3)1G$bS?C#fFt#71EM*6tX8U$S%+p8%P6M+3S2q1ifU zx1pJo7@aPJ3k`Ie3U?m5ccMs?Z_2Z;wLQ!yOY~xU4IHOGjbQ{J_a996A`|=>9SxbG z37GFH?HL-k?EI~uO4Pi zO*e8RKl?SNJJzqGe<124m##4;$09vkO3_-LxBQwjc^M`9lo+u(;Jd)$T-qGIA5u4* z51j!DAqfheH5b+$x8QczR&~XjqM|hef;73TbMaFAnts#cts+%0pQSS_G6Lp}Qs*iw z&Vu3+`d>|@w7%O0@?E>^Yj zu?Obg1=i3)%tBZRtw8*}|Eb%Fc=mUWn^(j^;Dv4b+t1s)&mqdjGDG z?R{B>1DjtkC?pReWn$kS&ju}niKrVG7x80$Em5rIu?NR&J7vmP^D``nn zMA1{v=-$R;)hm)*C?K3 zaPaRKU7R79eeeH|6(;h(=#o@}@q@&wd`*;JC({?cKz|iRR6XZ3kdU{@-)A6y=sEdW z(8zscxX5-ry^f44zj~MX;ijZ$l@V60rjGvJs}&4R@XDFa!sLDMFbbjY7cEo}p(?e3 z5>I4Q%@)GFkd$IKKkQxdxPm?H6)3z2*GQhKG1X`puPItvx8K{;>zPTuGUP(%9zXaI zrB;%<94p*cKggW+>w$*|wg`bW7qUqYk_&u8@{yWuiUj>>4nDs%5aTZLTgNsau9aZN z#dY5cjKtMFD}ljbbkZ-{&5EjmG*{2V(Pvj5ubCt!_-XPR`7mwtB42KJ#-xWq{*gPjYL?7H4Y6|0$Y>r6szOPh?S5- z`LHzKLt=va(0punk%SF}Ike9sd^D}M+ZcKN@dDm^OV-|LpznQrqQ|-5FLnb$h*#Rs z`e2m0?pVrDqm3p}mMgx_kJ|3S?;@|59`-CS&%90@D~w&>^CR!_iK;;v)alh)KfIA| zbx5t_kaL~mkQSp!X+Y|k07k=4+mcW2aeUEQQuV5R;zUqp%SL#B^H#igt%5GPQS4j} z=1t9A(>#boqK3M(S+1hwEGCt$f-np(Pt%baH%9z5kcQI8k<=w6RFDy=tlb@#Ii3ZN zKnIh1%pAw_mgD}gtUV~=E}dOI_je<1u|2BgF~y`|#jA(r#}fk{=yTaX4Oa7ImDfqVEfsFGLgEznrQWQl4F`|;)Al*R?ceN7J1`@Itk9tc zdiGW@YigN9j!&h%W!)yZqV_)L59`|XUGm+ARciBMYh7&}#6v*?sjf=dxvHDxRPVGJ z3|JzCzq+U{_iFFDKn~021DLRb{iL2+>>6r}VXW+ulQ!kI=D2uwpmu%Ur9El8-a072 zGl(!)32Il!&PS*FF4xy1{B)&S!+e}_W&WUW&u2_J!`G0}ip72uh4s?s!~gCL`3~dg*mPO=f%;lQOmXX($)Rdoddh zmKrSVij~t$#I}s6WuS!@G{eLe2rjyAK0(&bl82Tsx1ek1yteL-FyArbrUqTAYPPk= z^VWWNO5J+tR`3e3v=*XcM3eOek-1iYB4hbeB8h6>Wv@ZJ&G+jqY=#8MxLrJgx^cGXO}?TNy7L;UM6I92a|m*xgq zW@TlK&CFy(MMX_aP8I`&h#y?bNwzOM-kk5~eYgJWpzr@Xp8R?P0FD2?|D^vf37|Iy zMZR}H&IzD1uU|L~oWB+Izd2)pJ#wo#hxGB*Spl_-I}s6OXV^~J ze1G^YHUOEP?y}SCvO6+)kCEOvR?D3J7qMe+GQN%))npi#y#Y=DX$to}sL9YPzVht* zdr$P-plu_}uf-ukrBn5;KZ=wv+>PNXE*nd-1Maf1iQ2#exbTfIP}6qc>({U5A8&v8 zf$$O@+Y}eVU{b}#*V)+E%Bri4T;3J_Y9j3L5HPwgT{krrV*{6&OE`IZKllgFCoK2A zzH4K%;Qg`i5wDfP%A;z#r29$O+sR3KT&6$8+S%9uNb@2u?|Inwwms*K<5IpBS1BT6 zx%t+-!J)Uzz5h||z3vFEB2!7R)c%qmMJbU^>dnjqVh_M&v{n66r%h|@M z68$Gn22-)=;}}f!-&?x@Z*2p5{&IYLeDHu3uzo8da32U4FUA5;FXOz+U%+VDh7s>B z{^LP78mD13^5BG`JRs3=kc`^{k!=U0fq%gIH(pb*)J;sgCLl^cFX$B;6E(uMe9G^8 zL!s@n-`B$D9AANo$=A~d0AT+5?cLNdKE%l)4woO!w6C2Pd3f%X$zKw`1`H@yG+rsE zUqAXy$!*bJe)PuIPL-fEfg7Zgbh@uPbMEkCPDR>kW)r}s?-yp1EzFdwS?c@02o_WsU=Kb!^t_P;QSCktt zqgMayrKqGAuzR}eT-5&(`76bo=F<7+87KbtxOo1%*?#TfpDA91DJycR8~mJC+ww-3 z7xvHlhX&~X8Pwv5ImZ%p;>0(;cmJ;{CnEXs`?2X14^4E&UjF9b_8O%)N4T`{-R?Oj z=h5V=7D0RUy4l+cqj_`%kD)z&bsV3IBA%bMHVF6UVCJa*yguRRngIIE@B-af197!( zRdP6WfcsrQVaXl(f4_9O=nXX2$x%LcS&($9T(X2!#wlSs3TkaKR(Tt`psX+8l)I$_$9=`R<2)KFxhO_9245 znL!4>^E+h65D2;y6ZK|tna+6du41k4ALK=PbjWxOtvXg!~18;BIv?VqJ~+!XpLc2&ZHx-j1uXmbQh4zY5vAEeUm*TY`$Qmnm6v+)wU0#9A3I#?Yohg6@)mc+CMko>M* zCxqVs?f>C?wzX(YaTV_-(yUDzqps|wsLr*A-2mlRj_4-mxl^aAX1}%l*jn`Y^tmH* z)o&IVypsym&w@HzZs=&2;N}N>Cktsw5S!T@2TD*KBX_p`soF22wd>uLNU(5@4_Qan zxiU-3Y)S)t6E}$;vB(;M;?Zg!JOK!~7jGeN9|ocfZ*THbTiYts!3;ZxBA-B53zgpC zF@??sY9|K?pGkfLC3UFDaQ$}Od2NdAV(UORT!KE>7c}o_bYvzMH?J(zjgQHr%hwJA6TI5UoAm%;J*7hzp~xB8Y`5G zPFCiI2rGsy(}Tal2&-^un{4dbU-lZ_StV8t+l;&`ku!8NbX#862CIb6hE7~wP*rtRF*70{g?KemL_xdz z3W?ETA&t%yfy7JS`^@XiZ7yBXLAQ0I-CZeA&6RZMPTuAvgG+#4azL3}x0xkgo;LnW zZ#+BdFpHT$%~YiY5F#ufn7y5ZdP$1s$Q)f@2))2R`(d?$y2FhcXJmQSYIw-4C3>Vn zcZTMX&8iTgPYFn8!68XOeq2WkzHV`LR0ygmCw-~fX|!QG+k)=z)qh7fiZ+G_FnFvv zq1ty7DlX|Ih?#@?m;`PM5- z?&X($=T<+TJ_RVLJ_}YqYA|0d4d**{_-xx&QB_egKXN0}^igVKPrzXKyr__{L+kw4V4o%_{w z4U`m%Q3&6%ARxs26sr5+!hm3upkN~CKvlUHrDdGQM+|xt4XSsJu4Z6h z_3?CZ47r(gcGJ}BY42W6mJh1Gc1ii2q37qOnlHatGiSc}cW=7|xBAWNWfwzdUbZ-S zPO^J*%E8s|-7fE0{kb)D?aKLoI?nuk(i>Se4XLD^s^ez5E%j&k{>hd0xv;Thw`r>~ zyiVUvOFQTK{_G_CywhKllQoY%JG1}a@0mLn+kVfzWp(e}muXx69ekrx^?s6$P5Q0Q z-RF@yjQ>*7%k1~-KZOp+dw)K7)VP1&ah0>cKJE4Vxii1==bo9i95`nE%f~)FPi$X$ zrk>?~{nzp{mpHk8Zd&ult=P==Lx9=eHQLp``S)MzetkQ8O;_MryWHt@y7y<_p2*9; zJ^l9oSAOUJsq|jHRP~Ov^1iCK@qZmtSR<{XRug+)4c4cyaQww5b0Ty3+-b{}EdzEo zmMvVUSXx>N>4GeOG9v=s?xN`|8d4LBx)XF3K1+iPIl%p;wl=oS(8S1~e&E4_1W<>k zwdvprqf{p`VDmc-wsj$HvDeZ`;lF%cJ10F(tQ+&4mXJ{FM6=ZL{+WgOR&ze5Ur&lTB*Rv9Z#bWon=8QIbb94 z2Ml}mRqKm!CBMJty)=j`RIxjF?;_XCRVAQCcu4s4g};6QCjnJGfgLy_3kwUe4FaHh z0YIQ(acKC;6{}Z!uMF`5O=`S;9W1pn None: args = AnalyzeDict(kwargs) - try: - ComparisonGenerator(args).run() - except RuntimeError as e: - print(f"[ERROR] {e}") + ComparisonGenerator(args).run() diff --git a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py index 7283c17b4..09d8688cf 100644 --- a/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py +++ b/profiler/compare_tools/compare_backend/comparator/overall_performance_comparator.py @@ -64,6 +64,14 @@ class OverallPerformanceComparator(BaseComparator): else: comp_col.extend( [f'{comp_profiling_info.communication_not_overlapped: .3f}s({comp_profiling_info.wait_time:.3f}s)']) + if base_profiling_info.RDMA_bandwidth or comp_profiling_info.RDMA_bandwidth: + self._headers.extend(['RDMA Bandwidth']) + base_col.append(f'{base_profiling_info.RDMA_bandwidth:.3f}GB/s') + comp_col.append(f'{comp_profiling_info.RDMA_bandwidth:.3f}GB/s') + if base_profiling_info.SDMA_bandwidth or comp_profiling_info.SDMA_bandwidth: + self._headers.extend(['SDMA Bandwidth']) + base_col.append(f'{base_profiling_info.SDMA_bandwidth:.3f}GB/s') + comp_col.append(f'{comp_profiling_info.SDMA_bandwidth:.3f}GB/s') if base_profiling_info.sdma_time or comp_profiling_info.sdma_time: self._headers.append('SDMA Time(Num)') base_col.append(f'{base_profiling_info.sdma_time:.3f}s({base_profiling_info.sdma_num})') diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index e0a80a4d3..c639aba5c 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -8,31 +8,15 @@ class ProfilingInfo: def __init__(self, profiling_type: str): self.profiling_type = profiling_type - self.cube_time = 0.0 self.other_time = 0.0 - self.vec_time = 0.0 - self.cube_num = 0 - self.vec_num = 0 - self.sdma_num = 0 - self.fa_num_fwd = 0 - self.fa_num_bwd = 0 - self.pa_num = 0 self.lccl_num = 0 - self.conv_time_fwd = 0.0 - self.conv_time_bwd = 0.0 - self.conv_num_fwd = 0 - self.conv_num_bwd = 0 self.compute_time = 0.0 self.communication_not_overlapped = 0.0 self.wait_time = 0.0 self.memory_used = 0.0 self.e2e_time = 0.0 - self.sdma_time = 0.0 self.scheduling_time = 0.0 - self.fa_time_bwd = 0.0 - self.pa_time = 0.0 self.lccl_time = 0.0 - self.fa_time_fwd = 0.0 self.minimal_profiling = False self.hide_op_details = False self.is_level0 = False @@ -76,6 +60,8 @@ class ProfilingInfo: self.other_cube_time = 0.0 self.other_cube_num = 0 + self.RDMA_bandwidth = 0.0 + self.SDMA_bandwidth = 0.0 @property def e2e_time_ms(self): @@ -136,61 +122,78 @@ class ProfilingInfo: def vector_total_num(self): return sum((self.vector_num_trans, self.vector_num_notrans)) - def trans_time_to_s(self): - self.cube_time = self.cube_time / 10 ** 6 - self.other_time = self.other_time / 10 ** 6 - self.vec_time = self.vec_time / 10 ** 6 - self.compute_time = self.compute_time / 10 ** 6 - self.communication_not_overlapped = self.communication_not_overlapped / 10 ** 6 - self.wait_time = self.wait_time / 10 ** 6 - self.e2e_time = self.e2e_time / 10 ** 6 - self.sdma_time = self.sdma_time / 10 ** 6 - self.scheduling_time = self.scheduling_time / 10 ** 6 - self.fa_time_bwd = self.fa_time_bwd / 10 ** 6 - self.fa_time_fwd = self.fa_time_fwd / 10 ** 6 - self.pa_time = self.pa_time / 10 ** 6 - self.lccl_time = self.lccl_time / 10 ** 6 - self.conv_time_fwd = self.conv_time_fwd / 10 ** 6 - self.conv_time_bwd = self.conv_time_bwd / 10 ** 6 + @property + def cube_time(self): + return ( + self.matmul_time_cube + self.matmul_time_vector + self.other_cube_time) / Constant.MILLISECONDS_TO_SECONDS - # 新指标单位为ms - self.fa_time_fwd_cube /= 10 ** 3 - self.fa_time_bwd_cube /= 10 ** 3 - self.fa_time_fwd_vector /= 10 ** 3 - self.fa_time_bwd_vector /= 10 ** 3 - self.conv_time_fwd_cube /= 10 ** 3 - self.conv_time_bwd_cube /= 10 ** 3 - self.conv_time_fwd_vector /= 10 ** 3 - self.conv_time_bwd_vector /= 10 ** 3 - self.matmul_time_cube /= 10 ** 3 - self.matmul_time_vector /= 10 ** 3 - self.vector_time_trans /= 10 ** 3 - self.vector_time_notrans /= 10 ** 3 - self.sdma_time_tensor_move /= 10 ** 3 - self.sdma_time_stream /= 10 ** 3 - self.page_attention_time /= 10 ** 3 - self.other_cube_time /= 10 ** 3 + @property + def vec_time(self): + return (self.vector_time_trans + self.vector_time_notrans) / Constant.MILLISECONDS_TO_SECONDS + + @property + def cube_num(self): + return self.matmul_num_cube + self.matmul_num_vector + self.other_cube_num + + @property + def vec_num(self): + return self.vector_num_trans + self.vector_num_notrans + + @property + def sdma_num(self): + return self.sdma_num_tensor_move + self.sdma_num_stream + + @property + def fa_num_fwd(self): + return self.fa_num_fwd_cube + self.fa_num_fwd_vector + @property + def fa_num_bwd(self): + return self.fa_num_bwd_cube + self.fa_num_bwd_vector + + @property + def pa_num(self): + return self.page_attention_num + + @property + def pa_time(self): + return self.page_attention_time / Constant.MILLISECONDS_TO_SECONDS + + @property + def conv_time_fwd(self): + return (self.conv_time_fwd_cube + self.conv_time_fwd_vector) / Constant.MILLISECONDS_TO_SECONDS + + @property + def conv_time_bwd(self): + return (self.conv_time_bwd_cube + self.conv_time_bwd_vector) / Constant.MILLISECONDS_TO_SECONDS + + @property + def conv_num_fwd(self): + return self.conv_num_fwd_cube + self.conv_num_fwd_vector + + @property + def conv_num_bwd(self): + return self.conv_num_bwd_cube + self.conv_num_bwd_vector + + @property + def sdma_time(self): + return (self.sdma_time_tensor_move + self.sdma_time_stream) / Constant.MILLISECONDS_TO_SECONDS + + @property + def fa_time_fwd(self): + return (self.fa_time_fwd_cube + self.fa_time_fwd_vector) / Constant.MILLISECONDS_TO_SECONDS + + @property + def fa_time_bwd(self): + return (self.fa_time_bwd_cube + self.fa_time_bwd_vector) / Constant.MILLISECONDS_TO_SECONDS def calculate_other_time(self): self.other_time = max( [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - self.pa_time - self.vec_time - self.conv_time_fwd - self.conv_time_bwd]) - def calculate_vec_time(self): - self.vec_time = self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd \ - - self.conv_time_fwd - self.conv_time_bwd - def calculate_schedule_time(self): self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time - self.communication_not_overlapped) - def update_fa_fwd_info(self, time: float): - self.fa_time_fwd += time - self.fa_num_fwd += 1 - - def update_fa_bwd_info(self, time: float): - self.fa_time_bwd += time - self.fa_num_bwd += 1 - def update_fa_fwd_cube_info(self, time: float): self.fa_time_fwd_cube += time self.fa_num_fwd_cube += 1 @@ -215,22 +218,10 @@ class ProfilingInfo: self.sdma_time_stream += time self.sdma_num_stream += num - def update_pa_info(self, time: float): - self.pa_time += time - self.pa_num += 1 - def update_lccl_info(self, time: float): self.lccl_time += time self.lccl_num += 1 - def update_conv_fwd_info(self, time: float): - self.conv_time_fwd += time - self.conv_num_fwd += 1 - - def update_conv_bwd_info(self, time: float): - self.conv_time_bwd += time - self.conv_num_bwd += 1 - def update_conv_bwd_cube_info(self, time: float): self.conv_time_bwd_cube += time self.conv_num_bwd_cube += 1 @@ -267,18 +258,6 @@ class ProfilingInfo: self.vector_time_notrans += time self.vector_num_notrans += 1 - def update_sdma_info(self, time: float, num: int = 1): - self.sdma_time += time - self.sdma_num += num - - def update_cube_info(self, time: float): - self.cube_time += time - self.cube_num += 1 - - def update_vec_info(self, time: float): - self.vec_time += time - self.vec_num += 1 - def update_other_cube_info(self, time: float): self.other_cube_time += time self.other_cube_num += 1 @@ -306,3 +285,35 @@ class ProfilingInfo: def is_not_minimal_profiling(self) -> bool: return self.profiling_type == Constant.NPU and not self.minimal_profiling + + def set_RDMA_bandwidth(self, bandwidth: float): + self.RDMA_bandwidth = bandwidth + + def set_SDMA_bandwidth(self, bandwidth: float): + self.SDMA_bandwidth = bandwidth + + def trans_time_to_s(self): + # 新指标单位为ms + self.fa_time_fwd_cube /= Constant.MILLISECONDS_TO_SECONDS + self.fa_time_bwd_cube /= Constant.MILLISECONDS_TO_SECONDS + self.fa_time_fwd_vector /= Constant.MILLISECONDS_TO_SECONDS + self.fa_time_bwd_vector /= Constant.MILLISECONDS_TO_SECONDS + self.conv_time_fwd_cube /= Constant.MILLISECONDS_TO_SECONDS + self.conv_time_bwd_cube /= Constant.MILLISECONDS_TO_SECONDS + self.conv_time_fwd_vector /= Constant.MILLISECONDS_TO_SECONDS + self.conv_time_bwd_vector /= Constant.MILLISECONDS_TO_SECONDS + self.matmul_time_cube /= Constant.MILLISECONDS_TO_SECONDS + self.matmul_time_vector /= Constant.MILLISECONDS_TO_SECONDS + self.vector_time_trans /= Constant.MILLISECONDS_TO_SECONDS + self.vector_time_notrans /= Constant.MILLISECONDS_TO_SECONDS + self.sdma_time_tensor_move /= Constant.MILLISECONDS_TO_SECONDS + self.sdma_time_stream /= Constant.MILLISECONDS_TO_SECONDS + self.page_attention_time /= Constant.MILLISECONDS_TO_SECONDS + self.other_cube_time /= Constant.MILLISECONDS_TO_SECONDS + self.other_time /= Constant.MICROSECONDS_TO_SECONDS + self.compute_time /= Constant.MICROSECONDS_TO_SECONDS + self.communication_not_overlapped /= Constant.MICROSECONDS_TO_SECONDS + self.wait_time /= Constant.MICROSECONDS_TO_SECONDS + self.e2e_time /= Constant.MICROSECONDS_TO_SECONDS + self.scheduling_time /= Constant.MICROSECONDS_TO_SECONDS + self.lccl_time /= Constant.MICROSECONDS_TO_SECONDS diff --git a/profiler/compare_tools/compare_backend/comparison_generator.py b/profiler/compare_tools/compare_backend/comparison_generator.py index b07170b64..b4d17f88e 100644 --- a/profiler/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/compare_tools/compare_backend/comparison_generator.py @@ -12,13 +12,22 @@ class ComparisonGenerator: INTERFACE_DICT = {Constant.OVERALL_COMPARE: OverallInterface} def __init__(self, args): - self._args_manager = ArgsManager() - self._args_manager.init(args) + self._args_manager = ArgsManager(args) self._data_dict = {} def run(self): - self.load_data() - self.generate_compare_result() + try: + self._args_manager.init() + self.load_data() + self.generate_compare_result() + except NotImplementedError as e: + print(f"[ERROR] {e}") + except RuntimeError as e: + print(f"[ERROR] {e}") + except FileNotFoundError as e: + print(f"[ERROR] {e}") + except Exception as e: + print(f"[ERROR] {e}") def load_data(self): self._data_dict[Constant.BASE_DATA] = self.PARSER_DICT.get(self._args_manager.base_profiling_type)( @@ -37,8 +46,18 @@ class ComparisonGenerator: generator.join() def run_interface(self, compare_type: str) -> dict: - self.load_data() - interface = self.INTERFACE_DICT.get(compare_type) - if interface: - return interface(self._data_dict).run() + try: + self._args_manager.init() + self.load_data() + interface = self.INTERFACE_DICT.get(compare_type) + if interface: + return interface(self._data_dict).run() + except NotImplementedError as e: + print(f"[ERROR] {e}") + except RuntimeError as e: + print(f"[ERROR] {e}") + except FileNotFoundError as e: + print(f"[ERROR] {e}") + except Exception as e: + print(f"[ERROR] {e}") return {} diff --git a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index 7bac2b033..65524664e 100644 --- a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -15,9 +15,18 @@ class OverallPerfInterface: self._result_data = {} def run(self): - self._check_path() - self._load_data() - self._generate_result() + try: + self._check_path() + self._load_data() + self._generate_result() + except NotImplementedError as e: + print(f"[ERROR] {e}") + except RuntimeError as e: + print(f"[ERROR] {e}") + except FileNotFoundError as e: + print(f"[ERROR] {e}") + except Exception as e: + print(f"[ERROR] {e}") return self._result_data def _check_path(self): diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 0aeeba83e..91b4094c2 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -61,7 +61,6 @@ class GPUProfilingParser(BaseProfilingParser): def _update_overall_metrics(self): self._calculate_performance_time() self.__parse_memory_reserved() - self._result_data.overall_metrics.calculate_vec_time() self._result_data.overall_metrics.calculate_schedule_time() self._result_data.overall_metrics.trans_time_to_s() @@ -76,7 +75,6 @@ class GPUProfilingParser(BaseProfilingParser): min_ts = min(event.start_time, min_ts) max_ts = max(event.end_time, max_ts) if event.stream == self._compute_stream_id and self.__is_sdma_time(event.name): - self._result_data.overall_metrics.update_sdma_info(event.dur) self._result_data.overall_metrics.update_sdma_stream_info(event.dur) continue if not event.is_kernel_cat(): @@ -84,7 +82,6 @@ class GPUProfilingParser(BaseProfilingParser): self.__add_marks(event) if event.is_nccl_name(): continue - self.__add_compute_time(event, aten_events, flow_dict_new) self.categorize_computing_performance_data(event, flow_dict_new) self._aten_events = None self._result_data.overall_metrics.set_e2e_time(float(max_ts - min_ts)) @@ -104,23 +101,6 @@ class GPUProfilingParser(BaseProfilingParser): for timestep in range(int(event.start_time + 1), int(event.end_time + 1)): self._marks[str(timestep)] += -100 # mark this timestep in compute stream - def __add_compute_time(self, event: TraceEventBean, aten_events: list, flow_dict_new: dict): - if self.__is_flash_attention(event.name): - if event.is_backward(): - self._result_data.overall_metrics.update_fa_bwd_info(event.dur) - else: - self._result_data.overall_metrics.update_fa_fwd_info(event.dur) - elif any(cube_mark in event.lower_name for cube_mark in self.CUBE_MARK): - is_conv = self.__check_is_conv(event, aten_events, flow_dict_new) - if is_conv == "conv_fwd": - self._result_data.overall_metrics.update_conv_fwd_info(event.dur) - elif is_conv == "conv_bwd": - self._result_data.overall_metrics.update_conv_bwd_info(event.dur) - else: - self._result_data.overall_metrics.update_cube_info(event.dur) - else: - self._result_data.overall_metrics.update_vec_info(event.dur) - def __check_is_conv(self, event: TraceEventBean, aten_events: list, flow_dict_new: dict) -> str: flow_start_time = flow_dict_new.get(event.start_time) if not flow_start_time: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index cb25c252c..29e9fea8d 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -22,6 +22,7 @@ class NPUProfilingParser(BaseProfilingParser): self._operator_memory_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "operator_memory.csv") self._memory_record_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "memory_record.csv") self._kernel_detail_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "kernel_details.csv") + self._communication_path = os.path.join(path_dict.get(Constant.ASCEND_OUTPUT_PATH, ""), "communication.json") self._info_json_path = path_dict.get(Constant.INFO_JSON_PATH, "") self._trace_events = [TraceEventBean(event) for event in self._trace_events] self._hccl_pid = None @@ -121,6 +122,35 @@ class NPUProfilingParser(BaseProfilingParser): return self._dequeue_data[left].corr_id if self._dequeue_data[left].start_time <= ts_time <= \ self._dequeue_data[left].end_time else Constant.INVALID_VALUE + def _update_bandwidth(self): + try: + communication_json = FileReader.read_trace_file(self._communication_path) + except FileNotFoundError: + print("[WARNING] The file communication.json does not exist.") + except Exception: + print("[ERROR] Failed to read communication.json.") + return + if not communication_json: + print("[WARNING] The communication.json file is empty.") + return + for _, group_dict in communication_json.items(): + step_dict = group_dict.get("collective", {}) + total_op_info = step_dict.get("Total Op Info", {}) + rdma_size_mb = rdma_time_ms = sdma_size_mb = sdma_time_ms = 0 + if "Communication Bandwidth Info" in total_op_info: + bandwidth_info = total_op_info["Communication Bandwidth Info"] + if "RDMA" in bandwidth_info: + rdma_info = bandwidth_info["RDMA"] + rdma_size_mb += rdma_info.get("Transit Size(MB)", 0) # 单位为 MB + rdma_time_ms += rdma_info.get("Transit Time(ms)", 0) # 单位为 MS + if "SDMA" in bandwidth_info: + sdma_info = bandwidth_info["SDMA"] + sdma_size_mb += sdma_info.get("Transit Size(MB)", 0) # 单位为 MB + sdma_time_ms += sdma_info.get("Transit Time(ms)", 0) # 单位为 MS + rdma_bandwidth = rdma_size_mb / rdma_time_ms if rdma_time_ms > 0 else 0 + sdma_bandwidth = sdma_size_mb / sdma_time_ms if sdma_time_ms > 0 else 0 + self._result_data.overall_metrics.set_RDMA_bandwidth(rdma_bandwidth) + self._result_data.overall_metrics.set_SDMA_bandwidth(sdma_bandwidth) def _update_overall_metrics(self): self.__parse_info_json() self.__parse_mem_csv() @@ -133,7 +163,7 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.calculate_other_time() self._result_data.overall_metrics.calculate_schedule_time() self._result_data.overall_metrics.trans_time_to_s() - + self._update_bandwidth() def _picking_notify_wait_event_and_not_overlap_event(self): self.notify_event_cache = [] self._not_overlaped_commu_event = [] @@ -271,28 +301,6 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.update_lccl_info(event.dur) def __parse_kernel_csv(self): - def __screen_data(kernel: KernelDetailsBean): - if kernel.is_flash_attention(): - if kernel.is_fa_bwd(): - self._result_data.overall_metrics.update_fa_bwd_info(kernel.duration) - else: - self._result_data.overall_metrics.update_fa_fwd_info(kernel.duration) - elif kernel.is_conv(): - if kernel.is_conv_bwd(): - self._result_data.overall_metrics.update_conv_bwd_info(kernel.duration) - else: - self._result_data.overall_metrics.update_conv_fwd_info(kernel.duration) - elif kernel.is_matmul(): - self._result_data.overall_metrics.update_cube_info(kernel.duration) - elif kernel.is_sdma(): - self._result_data.overall_metrics.update_sdma_info(kernel.duration) - elif kernel.is_page_attention(): - self._result_data.overall_metrics.update_pa_info(kernel.duration) - elif kernel.is_vector(): - self._result_data.overall_metrics.update_vec_info(kernel.duration) - else: - self._result_data.overall_metrics.update_cube_info(kernel.duration) - try: kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) except Exception: @@ -306,7 +314,6 @@ class NPUProfilingParser(BaseProfilingParser): for kernel in kernel_details: if kernel.is_invalid(): continue - __screen_data(kernel) self.categorize_computing_performance_data(kernel, flow_dict_new) def __parse_mem_csv(self): @@ -353,5 +360,4 @@ class NPUProfilingParser(BaseProfilingParser): compute_stream = event_wait_stream & ai_core_stream if event_wait_stream else ai_core_stream for stream in compute_stream: dur_list = sdma_dict.get(stream, []) - self._result_data.overall_metrics.update_sdma_info(sum(dur_list), len(dur_list)) self._result_data.overall_metrics.update_sdma_stream_info(sum(dur_list), len(dur_list)) diff --git a/profiler/compare_tools/compare_backend/utils/args_manager.py b/profiler/compare_tools/compare_backend/utils/args_manager.py index ab9fb43a9..579bf9b99 100644 --- a/profiler/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/compare_tools/compare_backend/utils/args_manager.py @@ -11,17 +11,17 @@ class Singleton(object): self._cls = cls self._instance = {} - def __call__(self): + def __call__(self, args): if self._cls not in self._instance: - self._instance[self._cls] = self._cls() + self._instance[self._cls] = self._cls(args) return self._instance[self._cls] @Singleton class ArgsManager: - def __init__(self): - self._args = None + def __init__(self, args: any): + self._args = args self._base_path_dict = {} self._comparison_path_dict = {} @@ -114,8 +114,7 @@ class ArgsManager: path_dict.update({Constant.INFO_JSON_PATH: os.path.join(file_path, dir_name)}) return path_dict - def init(self, args: any): - self._args = args + def init(self): if self._args.max_kernel_num is not None and self._args.max_kernel_num <= Constant.LIMIT_KERNEL: msg = f"Invalid param, --max_kernel_num has to be greater than {Constant.LIMIT_KERNEL}" raise RuntimeError(msg) diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index 252aa536e..724719920 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -6,6 +6,7 @@ class Constant(object): MAX_PATH_LENGTH = 4096 MAX_FLOW_CAT_LEN = 20 MAX_FILE_SIZE = 1024 * 1024 * 1024 * 5 + MAX_JSON_SIZE = 1024 * 1024 * 1024 * 10 BYTE_TO_KB = 1024 YELLOW_COLOR = "FFFF00" GREEN_COLOR = "00FF00" @@ -15,6 +16,8 @@ class Constant(object): US_TO_MS = 1000 KB_TO_MB = 1024 INVALID_VALUE = -1 + MILLISECONDS_TO_SECONDS = 10 ** 3 + MICROSECONDS_TO_SECONDS = 10 ** 6 # epsilon EPS = 1e-15 diff --git a/profiler/compare_tools/compare_backend/utils/file_reader.py b/profiler/compare_tools/compare_backend/utils/file_reader.py index b4ae78638..263888a3e 100644 --- a/profiler/compare_tools/compare_backend/utils/file_reader.py +++ b/profiler/compare_tools/compare_backend/utils/file_reader.py @@ -7,7 +7,6 @@ from compare_backend.utils.constant import Constant class FileReader: - @classmethod def read_trace_file(cls, file_path: str) -> any: PathManager.check_path_readable(file_path) diff --git a/profiler/compare_tools/compare_interface/comparison_interface.py b/profiler/compare_tools/compare_interface/comparison_interface.py index 919095b31..b747aae47 100644 --- a/profiler/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/compare_tools/compare_interface/comparison_interface.py @@ -21,7 +21,6 @@ class ComparisonInterface: def compare(self, compare_type: str) -> dict: if compare_type == Constant.OVERALL_COMPARE: self._args.enable_profiling_compare = True - return ComparisonGenerator(self._args).run_interface(compare_type) def disaggregate_perf(self, compare_type: str) -> dict: diff --git a/profiler/compare_tools/performance_compare.py b/profiler/compare_tools/performance_compare.py index 7c9d60aac..7c3fcdb6e 100644 --- a/profiler/compare_tools/performance_compare.py +++ b/profiler/compare_tools/performance_compare.py @@ -31,7 +31,6 @@ def main(): ComparisonGenerator(args).run() - if __name__ == "__main__": start_time = datetime.datetime.now() main() diff --git a/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py b/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py new file mode 100644 index 000000000..eb383a659 --- /dev/null +++ b/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py @@ -0,0 +1,170 @@ +import os +import shutil +import stat +import json + +import unittest +from profiler.advisor.interface.interface import Interface +from profiler.advisor.common.analyzer_scopes import SupportedScopes + + +class TestRdmaAdvice(unittest.TestCase): + TMP_DIR = "./tmp/" + OUTPUT_DIR = "./tmp/cluster_analysis_output" + interface = None + err_interface = None + + def tearDown(self): + if os.path.exists(TestRdmaAdvice.TMP_DIR): + shutil.rmtree(TestRdmaAdvice.TMP_DIR) + if os.path.exists(TestRdmaAdvice.OUTPUT_DIR): + shutil.rmtree(TestRdmaAdvice.OUTPUT_DIR) + self.clear_htmls() + + def setUp(self): + if os.path.exists(TestRdmaAdvice.TMP_DIR): + shutil.rmtree(TestRdmaAdvice.TMP_DIR) + if not os.path.exists(TestRdmaAdvice.TMP_DIR): + os.makedirs(TestRdmaAdvice.TMP_DIR) + if not os.path.exists(TestRdmaAdvice.OUTPUT_DIR): + os.makedirs((TestRdmaAdvice.OUTPUT_DIR)) + self.clear_htmls() + + @classmethod + def clear_htmls(cls): + current_path = os.path.dirname(os.path.abspath(__file__)) + for filename in os.listdir(current_path): + # 检查文件是否以“mstt”开头 + if filename.startswith("mstt"): + # 构建文件的完整路径 + file_path = os.path.join(current_path, filename) + # 删除文件 + os.remove(file_path) + + @classmethod + def get_cluster_communication_view(cls): + data = {"p2p":{"step1" : { + "hcom_broadcast__844_0_1@13681369207305868844": { + "0": { + "Communication Time Info": { + "Start Timestamp(us)": 1713174287354248.0, + "Elapse Time(ms)": 4688, + "Transit Time(ms)": 0, + "Wait Time(ms)": 0.01162, + "Synchronization Time(ms)": 0.01162, + "Idle Time(ms)": 39.0606, + "Wait Time Ratio": 1.0, + "Synchronization Time Ratio": 1.0 + }, + "Communication Bandwidth Info": { + "RDMA": { + "Transit Size(MB)": 80, + "Transit Time(ms)": 4600, + "Bandwidth(GB/s)": 0.003, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "HCCS": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "PCIE": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SDMA": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SIO": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + } + } + }, + "16": { + "Communication Time Info": { + "Start Timestamp(us)": 1713174287186619.8, + "Elapse Time(ms)": 4788, + "Transit Time(ms)": 0.0013, + "Wait Time(ms)": 39.037240000000004, + "Synchronization Time(ms)": 39.03034, + "Idle Time(ms)": 167.66008000000002, + "Wait Time Ratio": 1.0, + "Synchronization Time Ratio": 1.0 + }, + "Communication Bandwidth Info": { + "RDMA": { + "Transit Size(MB)": 80, + "Transit Time(ms)": 4700, + "Bandwidth(GB/s)": 0.0033, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "HCCS": { + "Transit Size(MB)": 4e-05, + "Transit Time(ms)": 0.0013, + "Bandwidth(GB/s)": 0.0308, + "Large Packet Ratio": 0.0, + "Size Distribution": { + "4e-05": [ + 1, + 0.0013 + ] + } + }, + "PCIE": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SDMA": { + "Transit Size(MB)": 4e-05, + "Transit Time(ms)": 0.0013, + "Bandwidth(GB/s)": 0.0308, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SIO": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + } + } + }, + } + }}} + return data + + @classmethod + def create_communicaton_json(cls): + raw_data = cls.get_cluster_communication_view() + with os.fdopen(os.open(f"{TestRdmaAdvice.OUTPUT_DIR}/cluster_communication.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + def test_run_should_run_success_when_contain_cluster_communication_json(self): + self.create_communicaton_json() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "cluster" + scope = SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []))) + self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []).get('data'))) + result.clear() diff --git a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py b/profiler/test/ut/advisor/communication_advice/test_packet_advice.py new file mode 100644 index 000000000..a8fd4549e --- /dev/null +++ b/profiler/test/ut/advisor/communication_advice/test_packet_advice.py @@ -0,0 +1,175 @@ +import os +import shutil +import stat +import json + +import unittest +from profiler.advisor.interface.interface import Interface +from profiler.advisor.common.analyzer_scopes import SupportedScopes + + +class TestPacketAdvice(unittest.TestCase): + TMP_DIR = "./ascend_pt" + OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" + interface = None + err_interface = None + + def tearDown(self): + if os.path.exists(TestPacketAdvice.TMP_DIR): + shutil.rmtree(TestPacketAdvice.TMP_DIR) + self.clear_htmls() + + def setUp(self): + if os.path.exists(TestPacketAdvice.TMP_DIR): + shutil.rmtree(TestPacketAdvice.TMP_DIR) + if not os.path.exists(TestPacketAdvice.TMP_DIR): + os.makedirs(TestPacketAdvice.TMP_DIR) + if not os.path.exists(TestPacketAdvice.OUTPUT_DIR): + os.makedirs(TestPacketAdvice.OUTPUT_DIR) + self.clear_htmls() + + @classmethod + def clear_htmls(cls): + current_path = os.path.dirname(os.path.abspath(__file__)) + for filename in os.listdir(current_path): + # 检查文件是否以“att”开头 + if filename.startswith("mstt"): + # 构建文件的完整路径 + file_path = os.path.join(current_path, filename) + # 删除文件 + os.remove(file_path) + + @classmethod + def get_communication_view(cls): + data = {"step1":{"collective" : { + "hcom_broadcast__844_1_1@13681369207305868844": { + "Communication Time Info": { + "Start Timestamp(us)": 1713174287407957.0, + "Elapse Time(ms)": 0.06086, + "Transit Time(ms)": 0.00126, + "Wait Time(ms)": 0.014939999999999998, + "Synchronization Time(ms)": 0.00714, + "Idle Time(ms)": 0.044660000000000005, + "Wait Time Ratio": 0.9222, + "Synchronization Time Ratio": 0.85 + }, + "Communication Bandwidth Info": { + "RDMA": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "HCCS": { + "Transit Size(MB)": 0.028575999999999997, + "Transit Time(ms)": 0.008620000000000001, + "Bandwidth(GB/s)": 3.3151, + "Large Packet Ratio": 0.0, + "Size Distribution": { + "0.004224": [ + 6, + 0.00736 + ], + "0.003232": [ + 1, + 0.00126 + ] + } + }, + "PCIE": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SDMA": { + "Transit Size(MB)": 0.028575999999999997, + "Transit Time(ms)": 0.008620000000000001, + "Bandwidth(GB/s)": 3.3151, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SIO": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + } + } + }, + "hcom_allReduce__844_2_1@13681369207305868844": { + "Communication Time Info": { + "Start Timestamp(us)": 1713174287432401.2, + "Elapse Time(ms)": 2.9042, + "Transit Time(ms)": 1.35236, + "Wait Time(ms)": 1.47632, + "Synchronization Time(ms)": 1.44524, + "Idle Time(ms)": 0.07551999999999981, + "Wait Time Ratio": 0.5219, + "Synchronization Time Ratio": 0.5166 + }, + "Communication Bandwidth Info": { + "RDMA": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "HCCS": { + "Transit Size(MB)": 176.16076799999996, + "Transit Time(ms)": 9.55658, + "Bandwidth(GB/s)": 18.4335, + "Large Packet Ratio": 0.0, + "Size Distribution": { + "12.582912": [ + 14, + 9.55658 + ] + } + }, + "PCIE": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SDMA": { + "Transit Size(MB)": 176.16076799999996, + "Transit Time(ms)": 9.55658, + "Bandwidth(GB/s)": 18.4335, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SIO": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + } + } + }, + }}} + return data + + @classmethod + def create_communicaton_json(cls): + raw_data = cls.get_communication_view() + with os.fdopen(os.open(f"{TestPacketAdvice.OUTPUT_DIR}/communication.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + def test_run_should_run_success_when_ascend_pt_contain_communication_json(self): + self.create_communicaton_json() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "communication" + scope = SupportedScopes.PACKET + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(2, len(result.data.get("Packet Analysis", []))) + self.assertEqual(1, len(result.data.get("Packet Analysis", []).get('data'))) + result.clear() diff --git a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py index dc85b0af0..59525f18f 100644 --- a/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py +++ b/profiler/test/ut/compare_tools/compare_bean/test_profiling_info.py @@ -4,28 +4,6 @@ from compare_backend.compare_bean.profiling_info import ProfilingInfo class TestProfilingInfo(unittest.TestCase): - def test_calculate_other_time(self): - info = ProfilingInfo("NPU") - info.compute_time = 10 - info.cube_time = 1 - info.fa_time_fwd = 2 - info.fa_time_bwd = 2 - info.vec_time = 3 - info.calculate_other_time() - self.assertEqual(info.other_time, 2) - info.vec_time = 7 - info.calculate_other_time() - self.assertEqual(info.other_time, 0) - - def test_calculate_vec_time(self): - info = ProfilingInfo("NPU") - info.compute_time = 10 - info.cube_time = 1 - info.fa_time_fwd = 2 - info.fa_time_bwd = 2 - info.calculate_vec_time() - self.assertEqual(info.vec_time, 5) - def test_calculate_schedule_time(self): info = ProfilingInfo("NPU") info.e2e_time = 10 @@ -36,41 +14,50 @@ class TestProfilingInfo(unittest.TestCase): def test_update_fa_fwd_info(self): info = ProfilingInfo("NPU") - info.update_fa_fwd_info(5) - info.update_fa_fwd_info(5) - self.assertEqual(info.fa_time_fwd, 10) + info.fa_time_fwd_cube = 5 + info.fa_time_fwd_vector = 5 + info.fa_num_fwd_cube = 1 + info.fa_num_fwd_vector = 1 + self.assertEqual(info.fa_time_fwd, 0.01) self.assertEqual(info.fa_num_fwd, 2) def test_update_fa_bwd_info(self): info = ProfilingInfo("NPU") - info.update_fa_bwd_info(5) - info.update_fa_bwd_info(5) - self.assertEqual(info.fa_time_bwd, 10) + info.fa_time_bwd_cube = 5 + info.fa_time_bwd_vector = 5 + info.fa_num_bwd_cube = 1 + info.fa_num_bwd_vector = 1 + self.assertEqual(info.fa_time_bwd, 0.01) self.assertEqual(info.fa_num_bwd, 2) def test_update_sdma_info(self): info = ProfilingInfo("NPU") - info.update_sdma_info(5) - self.assertEqual(info.sdma_time, 5) - self.assertEqual(info.sdma_num, 1) - info.update_sdma_info(5, 5) - self.assertEqual(info.sdma_time, 10) - self.assertEqual(info.sdma_num, 6) + info.sdma_time_tensor_move = 5 + info.sdma_time_stream = 5 + info.sdma_num_tensor_move = 5 + info.sdma_num_stream = 5 + self.assertEqual(info.sdma_time, 0.01) + self.assertEqual(info.sdma_num, 10) def test_update_cube_info(self): info = ProfilingInfo("NPU") - info.update_cube_info(5) - info.update_cube_info(5) - self.assertEqual(info.cube_time, 10) - self.assertEqual(info.cube_num, 2) + info.matmul_time_cube = 1 + info.matmul_time_vector = 1 + info.other_cube_time = 1 + info.matmul_num_cube = 5 + info.matmul_num_vector = 5 + info.other_cube_num = 5 + self.assertEqual(info.cube_time, 0.003) + self.assertEqual(info.cube_num, 15) def test_update_vec_info(self): info = ProfilingInfo("NPU") - info.update_vec_info(5) - info.update_vec_info(5) - self.assertEqual(info.vec_time, 10) - self.assertEqual(info.vec_num, 2) - + info.vector_time_trans = 1 + info.vector_time_notrans = 1 + info.vector_num_trans = 2 + info.vector_num_notrans = 2 + self.assertEqual(info.vec_time, 0.002) + self.assertEqual(info.vec_num, 4) def test_set_compute_time(self): info = ProfilingInfo("NPU") info.update_compute_time(1) diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py index d7cb3d058..25293d64a 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_gpu_profiling_parser.py @@ -76,16 +76,12 @@ class TestGpuProfilingParser(unittest.TestCase): res._marks = defaultdict(int) res._calculate_performance_time() self.assertEqual(res._result_data.overall_metrics.e2e_time, 98) - self.assertEqual(res._result_data.overall_metrics.sdma_time, 4) + self.assertEqual(res._result_data.overall_metrics.sdma_time, 0.004) self.assertEqual(res._result_data.overall_metrics.sdma_num, 4) - self.assertEqual(res._result_data.overall_metrics.cube_time, 1) + self.assertEqual(res._result_data.overall_metrics.cube_time, 0.001) self.assertEqual(res._result_data.overall_metrics.cube_num, 1) - self.assertEqual(res._result_data.overall_metrics.fa_time_fwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_num_fwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_time_bwd, 2) - self.assertEqual(res._result_data.overall_metrics.fa_num_bwd, 2) - self.assertEqual(res._result_data.overall_metrics.vec_time, 2) - self.assertEqual(res._result_data.overall_metrics.vec_num, 2) # cun yi + self.assertEqual(res._result_data.overall_metrics.vec_time, 0.006) + self.assertEqual(res._result_data.overall_metrics.vec_num, 6) # cun yi self.assertEqual(res._result_data.overall_metrics.communication_not_overlapped, 2) self.assertEqual(res._result_data.overall_metrics.compute_time, 7) -- Gitee From a7c8b2bc0dec5e875704201a85421bd82bf1dba8 Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Fri, 9 Aug 2024 16:21:18 +0800 Subject: [PATCH 233/791] grad_probe READMD --- debug/accuracy_tools/msprobe/README.md | 12 +- .../msprobe/doc/grad_probe/grad_probe.md | 207 ++++++++++++++++++ .../msprobe/doc/grad_probe/img/image-1.png | Bin 0 -> 42344 bytes .../msprobe/doc/grad_probe/img/image-2.png | Bin 0 -> 26563 bytes .../msprobe/doc/grad_probe/img/image-3.png | Bin 0 -> 22581 bytes .../msprobe/doc/grad_probe/img/image-4.png | Bin 0 -> 22779 bytes .../msprobe/doc/grad_probe/img/image.png | Bin 0 -> 11977 bytes 7 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md create mode 100644 debug/accuracy_tools/msprobe/doc/grad_probe/img/image-1.png create mode 100644 debug/accuracy_tools/msprobe/doc/grad_probe/img/image-2.png create mode 100644 debug/accuracy_tools/msprobe/doc/grad_probe/img/image-3.png create mode 100644 debug/accuracy_tools/msprobe/doc/grad_probe/img/image-4.png create mode 100644 debug/accuracy_tools/msprobe/doc/grad_probe/img/image.png diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 42743c507..a89592499 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -21,7 +21,9 @@ Successfully installed mindstudio_probe-{version} ``` ### 下载whl包安装 -1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、torch、tqdm依赖。 +1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、matplotlib依赖。 + + 根据自己的环境选择安装 torch、mindspore。 若环境中已安装部分依赖，不需要重复安装。 @@ -177,6 +179,14 @@ Required-by: MindSpore场景：暂不支持。 +6. 执行梯度采集和比对。 + + 用于采集梯度数据并进行梯度相似度比对。可以精准定位问题出现的step。 + + 详见[梯度状态监测工具](./doc/grad_probe/grad_probe.md)。 + + + 上述流程中的工具均为msprobe工具的子工具，使用相同的命令行，格式如下： 精度预检工具 diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md b/debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md new file mode 100644 index 000000000..fcbd2f123 --- /dev/null +++ b/debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md @@ -0,0 +1,207 @@ +# Ascend模型梯度状态监测工具 + +梯度状态监测工具提供了两种能力： + +- 将模型权重的梯度数据导出。这种功能可以将模型权重的梯度值以统计量的形式采集出来，用以分析问题。 +- 将两份梯度数据进行相似度对比。在有标杆问题中，可以确认训练过程中精度问题出现的step，以及抓取反向过程中的问题。 + +工具支持PyTorch版本：2.0/2.1/2.2；支持MindSpore版本：r2.3。 + +## 工具特性 + +- 使用便捷，无需在训练流程里插入代码 +- 可以精准定位问题出现的step + +## 使用方式 + +### 梯度数据导出 + +1. 创建配置文件config.json，样例如下： + + ```json + { + "task": "grad_probe", + "dump_path": "./dump_path", + "rank": [], + "step": [], + "grad_probe": { + "grad_level": "L1", + "param_list": [], + "bounds": [-1, 0, 1] + } + } + ``` + > step指的是优化器被调用的次数（并非模型跑的step，某些step，例如loss为nan时，不会调用优化器） + + **参数说明** + + | 参数 | 说明 | 输入类型 | 是否必选 | + |--------------------------------|-----------------------------------|-----------------|----------| + | task | 填为"grad_probe"。 | str | 是 | + | grad_level | 输出级别。决定导出数据的详细程度，级别越大导出数据越详细。可取值：L0, L1, L2|str | 是 | + | param_list | 权重名称列表，表示需要监控的权重。列表为空就表示监控所有权重。 | List[str] | 是 | + | rank | rank id列表，在多卡场景下，表示需要导出梯度数据的进程的rank id。列表为空就表示导出所有rank的数据。（MindSpore静态图模式下，当前暂不支持指定rank功能） | List[int] | 是 | + | step | step列表，表示需要导出数据的step列表。列表为空就表示导出所有step的数据。（MindSpore静态图模式下，当前暂不支持指定step功能） | List[int] | 是 | + | bounds | 区间列表，用来划分区间以统计数值的分布。需要保证由数据小到大排列。可以使用默认值[-1, 0, 1] | List[float] | 是 | + | dump_path | 输出目录。如果不存在就会创建一个新目录。 | str | 是 | + + **不同级别的level的导出数据** + + + | 级别 | 特征数据表头 | 是否有方向数据 | + | ---- | ------------------------------------------------------------ | -------------- | + | L0 | ("param_name", "MD5", "max", "min", "norm", "shape") | 否 | + | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | + | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | + + intervals就是根据值分布bounds划分出的区间。 + MindSpore静态图模式下，L0级别中暂不支持"MD5" + + **方向数据解释** + + 因为模型的参数往往非常大，所以存储真实数据是不可接受的，这里折衷一下，只存储梯度数据的正负号（一个布尔值），也就是方向。 + + **bounds和值分布解释** + + + 值分布：梯度数据落在各个区间的元素个数占总元素个数的比例。 + + bounds：一个列表，用来划分出区间以统计值分布。例如传入bounds = [-10, 0, 10]，此时有一个 grad_value: Tensor = [9.3 , 5.4, -1.0, -12.3]，依据 bounds 划分出 (-inf, -10]、(-10, 0]、(0, 10]、(10, inf) 四个区间，然后统计grad_value里的数据落在每个区间内的个数，得到 1、1、2、0。如下图所示： + ![Alt text](img/image-1.png) + +2. 插入代码。示例代码如下： + +- PyTorch框架：模型构造完成后，传入config.json的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将`模型`作为参数传入。 +```python +from msprobe.pytorch import PrecisionDebugger +debugger = PrecisionDebugger("config_json_path") +debugger.monitor(model) +``` +- MindSpore框架：优化器构造完成后，传入config.json的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将`优化器`作为参数传入。 +```python +from msprobe.mindspore import PrecisionDebugger +debugger = PrecisionDebugger("config_json_path") +debugger.monitor(optimizer) +``` + +3. 结束监控（MindSpore静态图模式下需要） + + 在训练结束之后，调用stop接口 + +```python +gm.stop() +``` + +### 输出结果 +**输出目录结构**（以level配置L2为例） + +```bash +{dump_path} + ├── rank{rank_id} + │ ├── grad_summary_{step}.csv + │ ├── step{step} + │ │ ├── {param_name}.npy +``` ++ {timestamp}：梯度工具导出数据的时候会在output_path下生成一个时间戳目录，然后在这个时间戳目录下输出结果。 ++ rank_{rank_id}：在分布式场景下，会记录卡的rank_id。非分布式场景下，如果是CPU则记录进程号，如果是CPU或GPU则记录卡号 ++ grad_summary_{step}.csv：会分step记录每一步的梯度数据统计值。 ++ step_{step}：这个目录下会存放该step的梯度的方向数据。 ++ {param_name}.pt(npy)：模型参数的梯度方向数据，PyTorch保存的是pt文件，MindSpore是npy文件。 + +**grad_summary_{step}.csv** + +样例如下： + +![Alt text](img/image.png) + +| 字段 | 含义 | +| --------------------- | ------------------------------------------------------------| +| Param_name | 模型参数名称。 | +| MD5 | 梯度数据的MD5值。 | +| (-inf, -0.01]...[0.01, inf) | 梯度值落在区间内的元素个数占总元素的比例。 | +| =0 | 梯度为0的元素个数占总元素的比例。 | +| Max | 最大值。 | +| Min | 最小值。 | +| Norm | L2norm值。 | +| Shape | 形状。 | + +### 梯度相似度比对 + +会根据所导出的权重，分step比对梯度相似度，输出每个权重的梯度相似度和总的梯度相似度。单个权重的梯度相似度为两份方向数据的重合度，总的梯度相似度为每个权重的梯度相似度按元素个数加权。 + +#### 前提条件 + +- 相同配置下，以Level为L1或L2分别采集npu和gpu环境下的梯度数据。 +- 将两份梯度数据传到同一环境下。 + +#### 使用方式 + + +新建如下Python脚本，传入npu和gpu的dump_path以及输出目录，比对结果输出目录不存在的话会新建： + +```python +from msprobe import * +GradComparator.compare_distributed("配置文件里写的dump_path", + "配置文件里写的dump_path", + "比对结果输出目录") +``` + + +### 比对结果 + +**输出目录结构** + +如下为多卡比对结果，单卡则没有rank_{rank_id}这一级目录。 + +```bash +比对结果输出目录 + ├── rank{rank_id} + │ ├── similarities.csv + │ └── similarities_picture + │ ├── {param_name}.png + │ └── summary_similarities.png +``` + +**问题界定** + +原则：对于任意权重，第0步的梯度相似度低于0.97，或者某一步的梯度相似度下降超过0.03，认为这一步存在精度问题。例子如下： + +- 第0步相似度低于0.97 + +![Alt text](img/image-3.png) + +- 第3步相似度下降超过0.03 + +![Alt text](img/image-4.png) + +- 正常情况 + +![Alt text](img/image-2.png) + +这个原则是一个经验性的指标，并不是严格的标注，还需要结合实际情况具体分析。 + +## 公开接口 + +**接口说明** + +```python +PrecisionDebugger.monitor(module) +``` + +| 参数 | 说明 | 是否必选 | +| ----- | -------------------- | -------- | +| module |Pytorch框架下传入模型，必须是torch.nn.Module；MindSpore框架下传入优化器。 | 是 | + + +**接口说明** + +```python +GradComparator.compare_distributed(dump_path1, dump_path2, output_path) +``` + +| 参数 | 说明 | 是否必选 | +| ----- | -------------------- | -------- | +| dump_path1 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path。 | 是 | +| dump_path2 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path，与dump_path1可以互换。 | 是 | +| output_path |输出结果目录，不存在会新建。 | 是 | + + +# FAQ diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-1.png b/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-1.png new file mode 100644 index 0000000000000000000000000000000000000000..bee75b8b42e4d63137c554cb703ddd7f70d8c1ce GIT binary patch literal 42344 zcmXtgbzIZ`8!aKCqhWOS=#-8DqmfSO?v(D7?h@&emQqqeknZjV>F&Ot@9*BrAHJr} z?Y;M&IOja)F;ZDk8Vy7Qf`fxYla+y~!ok6J0*?#dAppN7DeIWQ!BN1;!o<`);g9q^ zd^IGe+TXWDnRp?+ft9HiMMMM#YpXCvP!&*sqVlFA-n~O$9Emd$gF?Uk92W%1C9m4P zqrARZ9lBk8T;KBD%?ZWC8uGb7?l2z{sjs&`89h3xJYhWdD84DV4xB~&gkdF|V2SZA z(JM9XbpWTpZ`vDr2w#Q#lZnJfQTIm8*sjF4_mOoAb}%Fac*y}LzN2eziG0ihDx~BAm^wRwdS37!3X0}@ zgdn&D3-N-l``6yGq6DZ(@aaQga?k#v3l?3wm$G2FFWttGCb+}ul8K6ZBsY)GW}W-4 z!Id*M*PcQ$_3Su%@I7-*e0IM+tvK*r$POO65BYa%Wl#^ufiFV-P%I(X2_gM!U2(`0 z&&O0rq6`0uidiAUM}nT>+`95M1m2Lp8XUIudU);`QMYsqS--shZQ$r@oFUOmf{uIv zoI-_W=h~ZpcEjnbpG*@8wT8N;Cf*^f9&p^tdu@CbviNW$b(Q>i@^T8fQC~&e_&$jS z+9+xWK9-7J*B_&VXB_l!{$=bQNtikWOchm+AS`P!e*mQj99D+>?J?oWoM~vwwUa&RF1#jk$-cW>7E%1q02Kf5(iC zn?Uv*8=;dxwpnJ8b}J3-l6cD8A??S=Bx_FSf3Je*-bxyltw@h7-(~)-h-T1?lQQ=b zRG$*I#n7bp!!u>+*rHkvggwd%Tyh>8as+CV9_uLlLUhP;lifd~aFYK2Ls; zQ5Kc-SsQp&nx2-3i+f)Z&5|iu%^gNl`0BZ61neD`<^&*6%&T3NBfDs%3nNtZ@3g7y z5FQ7&&J*BS@uGXg-)bib6;67BcT?vMW2Fbe*!X`g*qysI>#;_StRY#^V80I&4}wx) zKh}S$k}*P#;&#uIk&r>$xqVEuu3y+x5y{2KNR)vUe0Ai^P^JKbkV3?Y?r}w}m^*z7 z&XR)>w%>G=_L~)+aYZOZx-{#B0x>9(%LWEvg~4dB6CtxqTw5x`kua*S+Ijz61rlqt znaSdjt??`n1kruXr6sAbj-a?jHHeZHd?cFVDMeI??+7VCKC(!91u|UNk-&ilI>|cI z-Wc+0VW7V*!a0xDf>w~g+Ti5jW*JKdYYYuoLUQP!p8rP1P#wY z0eWIY(!YSpfS=KC@8o?V{vKH6q`&u1rH2zjR~3dONu$vqJ)=*nVjb}O3=9UNE9(u|5mLJlDc;L7&d2@$8BQ6qr8Yid0`Z6 z_FcW#I(^zMPiqsd@AP)7Z`|e=RS=?o(YatCH@aD1mjgE#v|z_mAQMFk-?xe>Y6!98 z%t+{L*AOEY{m2Zpff+M}BWwdnuz&7(mic9iGwHujPqwFXatK5S1*$!emEekm3}%wt zPR(x+yH=pQ#)W|0u;NxVeUYL*QXsOXqv*4!a7ri90yUYMTu^zM_B9p!_0**~f{K9$ zB&ZCg%hHzY+HlH?8QS3_K)oCEDu$4}8a8oIl8Zq0Y~tL6-te2n*h$!`B?OPkP#sBh znL8%K6Ql$V4h}|z%Y&`u)ZN{AlwHs(3Y-q2yI=$SF`vclBnKK`&a4BHFmOX>6fRAA z2+SD^tGMM%EOyvQ|L)n}OfF@?PUEL(?rlAx@^$Wo9D+h#>E$TBMxntqn|EsaJ>^Ru8 z{FHvo7}Gnh0y+#RT<{Vx1px>_f!!Ms(gO-Q_;Q@|nD07QqkIdD5H-MG7?Iz~1ieSa z%Y>@dI58;Ks6aT)b=Mt&jsI6v&j^8l!(aY74*`E6#^HQVk2ieQ!HyjAeZmSR0+a&w zGL8-49)V&S*i9L!QB$K(BdVb?9Lj-JbT%}?jYGqvl9j>F&%Yg-DyQnQ*;D8g)!72P zUq1raNFN4N`KLvoCzB}V73r~&c<`TZs_;D->GGXNfZib3MT7Cd@92^vod)1^(OPNV zW<#QTg2^=XyGp@GMi`Q~XNqe5Ogtb1JqUw}6AV$VK~gf1G49|4FWoUXCk>5^N#2pz z!lvYGk>U6EF2l0-%6As34xI2|!9OfzAj~D@WGdVh6Fo8f-^d}YDv5Wt&rn|9bDea{ z9QVGgeCLQaH+u=8_7MX{{lmb|Am__J8yh|!*X{zzAzsH>!GiQbaOqd-mi;->B|J8R z8qWcj)~(<5T)QDp=9_kZ46uthIVbIT{DA6bV`Jm(?F~FoJennEzzo~Msvb#8O?A7m z6Ie&OMEp=fCYa_jOCDoDs{SkJfNRuVr^BWvDs~?ae`IgmON9+T@n=@~L(nHKMRI=s z!L9fkRIRDz>B0)FD(z;ModE^JS69I55wzu7Ji+H^MTP? zPyf#s`wAcODwx|VH>cdMb+2r1ZNKR#Q$WV+|o8=UIY zt4jJD{gWm`xZ1%Nn5{PK@V|U~JGx95>4~rCUoAQH<(Cb2g?{Tzh>~!l!x+bMYu>zD z*0M<6&{pGg*=L{Z#SgZ#UUjBxE7EnikDB~k9Einm!k?cyx7+OtQ(ta8*2c-Vogi=a zq(gjo9p;Tj{~=Pi)go|j-<~}3Hy^6qc6_jo>n3hXCZ&gVTVmMz{`(_HwEF2XP1k0o z<5$n7;aJ{}XxzN~yw|F0MgbD^G&2z{=Uoh>ca{hSrLaVJy}@BbNsxk3&HMn( zjuW5um-ouOr2`5D^O69Rv!?_>k$m<*%9ox(G(Dl+8wnw_Wj!U4!Q>E-lR;0q)^=%u zUahF4Y)`2X4K87Q&IcbyoWv60oH0> zCi~Y4C7ytXnMRD`me>zwEi}z?!i7@vbz0ORHa+%aJr!o-)AecbnAOaXTK<1zZ-+3U%Q^|5bJQ5 zgxA`33;^kBBTQtZn?Sug)qbZpv~nvL>S`MwaLIgR?ZRGOwD`Lz0~DpaZ5A_PNJ7R= zM@L8NybEr@Q)4+ZmQwf{7M}*2Kkw=V)vO(U*xfeE5=>7@O3KLCN6XTxt&K$YtlHll z{hq~d`g5`4(9Iv22pwtU;Yy)Rgn((NjYj>hWO2oCBehvYR5|y&z7|opjhlxSc}_|q zbB?wB*979F)+tDm3bvt#MEtY9??yqsrDLj-lHtaR8N;mngmwr0s=vK$Y-Qr%c;b?@ zHjvLNk0kA9KBD*4>B2^Dd1{Gc%t^mf7?JMulitqsE9Z5yf#2+pCu67hKA(W6>=hg| z_M-WqoYgDYWI7N3*Dd4mu?O={zZf+{KD9_YGh7BU`riGGDlznT+pm)>e01~oKkMUf zC?z5j>^=W_PvIw68cCuBwqrN*T;)0>*_RpdIq1aTea22R}_ zr+?bcnlA;}AgBX~3j%+j&GR}mXXtyLYEXSe#%*@eLhgNi!tdwE4I5&M#z#AzwSWG- zPn~RVJo_BRqVtk09q-TXIZy2O&^B8k-RQtJ^6BRv;;6wMfMv-F3hmVAuNWLX*qr7_ ztiC!qExFGyNojbdb9{biZTsTKK|O*uv&#TVMVNLy>Mkm2^<}3fwh5qdx!846;w=}k zAMJud6jH!>B^ak9ub-9-PGh*53Ry!iq4?lDTwuTH%$(4Y+E1lhErVGAz#I^?f43wB1GJ1!hk$L{j~ z;ihv)$yB7`=hl^*dGdn~845Cc*WOLRCVUC9DMPp)X1H(wepWSi$dXq2Z_}l@Kd{Gy z67d|eAPT^BR7?P3h~b#2q?-kz0W20T8c7&2o)*lLp#U+h)#m#P6RV=rX^G z^Fp%FW1U;vrzV)$Pv(k4K!JTGh;1Zd6j^OBbfcQb$7cP{ zR7NerC`$z%ceRGwHr z@DuV?o@GeD^OKL_%d9=R$M<>1J<|1&=DST6(r12I;pkQq+f=P1;qO zxV&_p1?LV7)n#4UaP|m_W-_~^xuEZuX(>)mM325AtBJe>s1S<7O|1+D71SBp?znU8 zir*+>3SWh^@hgnpa1lad^x-ejl!vb#pEa5H6S&hn}s@cIvCR9da` z#jjQK>Tq658GR%ScPO8;8~>T@-x2WoG=XNjc6Y4gqgNZ4?=F9n_k*-Tz0%12;aQ}b zgI-vei;3uE>NNZ`f9*8f%JX94k>(41F{!W z+p|RacFwT?J-|QXQMQDh%weTPb|lmGitK4)qkaN2js(F|ELg(AWS{DCeu8>S3XN32 z)vCkakF2dxXUl?R_0h$iJI{>S0wS}Z@bHICbfN0BlxqR+^^ccZf#7&s;Aiw9&k*$R z5T}98d*?9`zN?u&!%X&>?T9ah+do6c9K(GMe)&@{$ivF**2={WACZYnW2tn0;IFo7 z>38{WmDM8Vo(k=CqndJ)(alLtBP|(!6Zjr*xkQBiXqL7>hStE))3Yy3jC^yn{Cy;Q zrb5%0 zb@!<^Mtl@nD}R_?KzaJ}TGAz0)thxxIXrx29^G;HWYjIv-DkNFfRXGehATo%P2IBO zDiA9rp*UNHugh(-d~{O(eXXkWkI`9$u|ApS_JY1;okYIV_I&VHxV8ketNy2O(*O+N zt8oea@FRXQv|k>p%NZr0we!`tw<7HQ6jT}wt@Sr;O6VT{^1E^kMq0>DmS1=M7&^l1 zha|(sM4ET2Se52VH7%EquIYyEBZ4R29yl+=e#sghj*$p=M?08oIi6y35Y2zJRlgge zgL+Ac^|1=jY}gPDlRq)GQC*z&(x|vmOT)a2*!Fs1h(g(t7hA|g@tZp)?`%1x z$OQ;7QrrL{Icz4R9Ev1XNU)$*VC9pWuGbZfumJyGY$Orf zS2^_`=g?c5+6t5ZeB*)){UxgTp*jnXWx{^1*$x2vVz^2zBw*Ux)} z@fOVbpNgdl=lZJ9rKzgjPBm-A<(1fNP$kEPhjWRO!FlVV-uXPZ=1VABmHvuX`DJT@MycGxxc;>?i8|N2T8WiTN_wzbeXm!|QW%v=Gf> zMpz&og>aRSH2ZYIbF4P9yHzaJm5=;Q)9373HfR~is^7`=WV!kCzu^pkh%ATM zPI8VgB20U~U^0-PXD_tw-1nWfZEeIoP8Hz)Wx&K#cthHQyY1duKY7ytw4RmfG*46K zqtEq0gN5|&NI!B5=yA`X4An)#TIR@6d6rELdzbflvJ9NgUMQ`8rDu@9XcOhNGhD{s z1vKub%us+ESkS3E>R_pR;zdY{g-OHvr$Tr@QK)iqU{^x*!7hbPreFw79;j9oFRf@k z?d}aXf{Q$OSe2q8N_8aE9v_kv+(yN#S{GoZ8zBz!?%e4pN0k8+LX8C8aP4BstCm|w zasq8Oq|~G%eM~RFw(Z-PZ%YV2zjQ#;FNf1(5!Oei4E_2*-*P6O#Cx0~!q8(VTS{WOaV#HbN9XBq# z_kw%gd`N#T*g=VP16}>3W?Kn4pmqmi0?8u}J1n>C=?DsFtpj7|DPbAPD<%D;r6z0) zK}Nd&*sG$Fd}JMnX%UB?P~7vdlp2x0tt{Vdusot?9;ekaAorp9H-DNTE&`2_?oB-V z5U(w-uc)kjs=PU>x5W2ElS`)yD^)R#7W? zL3Z0-59{xHWXFF8)Rx>&C;Aj-L|>ashCA`ekCJ}U&va2j``(UNCgBf{u;&Kkz~S+t z>e(E;_&vsNe^po@Bz*LE0r;KNt()h-I;tnL&XT|0uUqYJuZ@i~PB^mVXO!vZeKXOw zn9lenadwela65PfppGyKt2Pu@&F_olQQG&a){ znqoNp@mm7E6XfxAz8b~Jm zNQ?GZCpXfInP;49z9=1%IbC+Gx10X>`=ysj7`feZZiQE)(&$!_M8{If!{J`xb%H*2 zqLstz-S|{y5^~4Q;Gy3G+TOFvQXe@XyUk>6%w8r=>vz<0y|OOej;v@H)sX+!m5nlb zL!aksc*M6tFWcSmqD%GzE%vMHUajyX<>L9{=i6*2zuQJ8m{CH%|Ho8VY}QT&v*lqeO12^>Jm1irQrz){3TrCT(D3wXd;9)y z-A-2-{@}^EXAaHb$3lTdrhCV;@I^oMssEGP_ZguApH^bqpM2(MIm;cF*FTin=G&xS z;SpP0_niV-`$WD0!!WNG{r+Wj&wD2oe|676rUkrI($Y<^0okW>ZRH##vFA5YB^E-S zsnWYoAU`PUF; z3?>bgI(iKiFX^K6zP%)MhxoyBcJ&XG3<>rEj8I$CMlLKRkmXb!9fp@zMD;@{$-5-x zSD!x-SIo@;a}BgWyZFjsIn!H}L0Ccbp;Y*(uk>|rKLnY#`crB~hMPangV3%jI$M^g z<-v5Byp}zmt!tG*tMTbX%%$2~R=at3xwTr!XpXSQ;R$tk!>L}gYuVDF#}bR+EQ{eU zugn2L;rpGxUvi~@X#>OWxg=+ct#b3J5~W-Lx07$V4Mi4GBk2AeqKA7qjwQaI`M4hV zOdm4TGSg_sqS7)1K2K~4ODip|Olhr^!xQ6hBqYg&OU`B&H|*?mT}Amb=zlU^{d*5_Ps|ehg!a` z&mLU6EO%bJVrs+GCVgMbNew;CyMN%QC>-s;V(`;L^txQH&s*)KZ;$7l*S~Vjg5vv9 zQ~h0522+R9*$iu`yz%{eqHu@OSeBY^^Jez%%)9GO@tL`re`}hKj()Ta`1-+iak`8P zk9GOCZS@}_zfV~1A(tg*9c@Vi3y>b9XD25#)Ji!94`;0X{FzPs7_5D)&G#AQSo=GV zo%iuFY$`D<)kJ+{H~Q=ph-x@tMrx4G@v8{O2!$_6qe?Juyf0LNbX+ogT;^bpE5QR3 zC!hOD4wWKAi4V;qmN4(He9DP5NFF@f$zanV)3vB-zIcIZ_LrOBpLQrJoK$VF0dA|! zZ=gkp$~)Ak@n;g|u1F0U%tkEkdnm_$BQnTCfg3QgbIiQ5$hJvU5(sVjSn|Et{1tD4nhsR>vQ$F~-!s2IQElLOb#31v zDsil{MKH)?XB5BBY>XCu)mlVlaMYZ921@T z_~PszP4I#rBQxd}gqdXo&5J4JAo@4Di_&ZG!Q<(M)62D8CuaXUK62>)_X1d3c95O< z7Jk>~3h?=oeJLl$KNE3P!&<;}S6PctT!%Pgm-merr!u34;Hk|%FELl-#c1Znw6W+W7Kju+@|t4*lIN{lb_$ptiqgMXtWEYzhu zuLK(^M+bQ(_(I4nkfL_?)t^)XA(c;SDq?eGA)==XKK=M8uC7=-A#lX zp8O^h-WK+;>&sa|_M6C5m=G9gHMLelaOMf$qFjeknS`4-rIacHB`4_r-Zn`JH2aIj5f#qT+&F z4mC#3Z$Si5o?JUUwaU{#4-8og=Dfxju)Fi)V?h)AHoAJjU2+_9!Gh!R_1Q-njg;je zmp&}?@Mc6wBd4>mwxa)@?Xg$K0vl+f@vun(bG_c7t0!$LQhI}J+4ESkwC)`FNM5Y% zO7KQ9dD_~}vssAOoOiRwu8D1G<|#=dHXE?MQ<~_I^4iwvlb}RWF+f%*iV3JKlG&oOx) zg5!#`WKzF?PAb4{+8g8dctfdHfSE{!dBaHV9qfRdvMldnz;Kgf1ZO-yUqKTcaRpi^ zeW6RDI^d+9?EW^OJraY0mi-PQm0+>$q#%;_?)@7C>>gi2-JGCb_qekP3BmA4*4m0# zf!|Qsg9gHrhC?aeX0V4*j2JDlhtDG>IM&JKI3T1CUerVEM~qpBO>Rhoy(c?}SFi3s>-CWlQzq^K$lfOY6r26%<5nP&DUO(JXc?8QCC0!J^ z7@lkA`@z3iy))F4h0w(b{X<^+Prt0E@^R?1IMnb9DtAc}4OG}<2I9;Ky!cvzI27&f zep0G&)D4^U$${x93;!0jZW6BdcfAGGdHYR^k}ySF(JsN98{QxUd8!9dPk!d?ba?l# zS!M^kIhp)i*m7nL^Je~BGSzIq)G$#fz2<#b6^4ROCgjO(fqs;<2hT#~#~l~)ua^dw z8<U64i$D4ezH?YNXx-SQr`&p99hE2YWWWLHCl7S^hw`NNSWtg zQ~i!WVbyq`19W=s>3*`zZ}Kw2>ib7Q|0h1pD^X~m`2Lp`Mn@{KslN?+2DF=1 zWcv$BeSVRfxCpC-Q2pWEus8cBN{V@!A@F|%B)>ttN@Q?*6fR-}Y0LuCEw@)N3NAk5 zM268aSkYdzgV-!O@v!2PcoVh1zP}>=#>R!>jw7qOE{x_?5^>8tkLeWKD{-wY=g87`OODY>Mh4O8+hlpY<51P(@N{DWwLzQ-IfnrI8 zLEA&QMg>)3KRGzhp!)W2B#zp5g)(5sXJP{UHIqiUddOQesW_619RHX1LUO_kpbzjw z*<@!J{U5Wd?Nd^oH=ZGSv5>51BRZNa9MTlRlE zKK*?;nn*2;P>U&N^hnMCH9j|)T5Ld7_?WUn7JJ^^xgygaafL0ZS`Lu{B=!}~ z1d;B3V$`=%0~W@!abyv@Fd$O?sk__xiYEb%9g6KbnkfYeD&eM7$mCCLW|xSCpz`AwMz z@!>CylAho7Om{ejsM`Vfa0gQ!%ysN8cqpf&gdBBBz1~sk0idDk-xq^bjjJt2e-H={ z3~_tj_r9^Q#9BzpI!ERv^NRrUdP#}7)4rZcFVNEa& z92bzAxmgwlNqC>;0NlY!i`yZtmR{o5uXlj9MTGj6QvwXwPd1%FmP9u6pkSZD6oxr{ z4`AqGyHK~YP;W^KqTMk`QIlcS2Bh_G-vr!`<^@lG*DGXj3S|FK&KJKvoXcf5?pkg0 znVOnfz>M5n+^tJ;J2z8c^{u1FNZiin?Gq=XszqLRk+K&ZmOrYff$T z*$9)L5Nm;Jex;4RQ6=0>Te7InFM_EM&Z&Dcu7Bba98SIqcun=sLl39s=Whlu5%8wK zTRXPj(`~{zi#&-Dm#&?5Jf5_=3FsHgrT!bql&(Wk+t6p7dwG7eoheUENI1)VeX?9? zu)l8`BqUZ`Tt*1$ERs$B#%-;dv!f);VURcL<)uN;QF9 zBF>QP7vg**!x~3ZlPcQfG!l2qunw0|1-;ta=K&h}R&VThgnSBw!}reVUL^RSe^U|b zleEg$OzLMK)i(lOfcSK}m6(=XfL?q1W%Ah9=w^xd2`IXN$B3BXIKU-mK~STK0!aX5 zb-(u!7w-bT84k04Ou%n>xNbGwAs1+pDR&+8kS-p!E0Wf#w{Iu80@EPWx(iAI<8nuC zYBe{Gjp%iDDahcaW`KTuyDt~?@^1_E&znxlv#J4c<{cn2swjLe$`hVY6jz-z5KB>D zNAn|mKv7Y(PNL1QKu9Az`<(F%E&I&jO!x#fALC;239)8HX51p?u`AWCH4zsV-=8dA z3wZUjDJ*CF%?+t7%IXHGx4Iu=V`DRF{rK&)A$|tsN=JvRJTs|F#MqY?YyZ6E`N885cbtI@zSmj)A1`xz2SBF50^=^5{Ts6<FMe1?d>mU#3tkdF55D&;_XwN zpl2v4x-=lr00(2#ti0Qb7X{uGux#iy*fsBe&vX^g-x*BBXZaKwXx5~j6Am}h-+y<~ zdYa?88Cmh81r!wke5mB4B>$&#X@jJ+QKfrS%Ug{4AgVf)?wVbCDq(-h+GnY6L9O{> z)Hp%fU%!|m3ovuO#zb5x^FniDgYg0tjNf_HVqY8CMJohA4hb#4vG?0rs*MA2Z#{cKF*1&0AX#`rCgf& zM~SdL`U0o3@=rM#M`(2{Dm1i(e@&x7pwJ->Rmq+0?fK&y0OQS8Xt7aJ=9>*9$;!&s zd<_D3PY<iEe&X1!ywZO%fejFXP6oLN3CaLb5({~2zubhyk<8cK9I@TCdsjfe zz*;bmi;XlGSVOa53iGZRK_Jm&s3u4>7-S-pA(tY!!jLViIE(t~@fzWo*YxX4|0fx) zgxna%{{9^N8008LWQGD(Mly&ggSa(SxD5N|2GUlL315VGgzQXdA}3j?jhRv;6(Xk^ zt{eYWP(j9Bx-;=*95N#%XSOcP`zTO3>0H1(Soo5@2XV($JVvp4t`+F(#E!qQb}tZ- z(amUdNEVI0K$PEk4kEZjywPjPAFD zNdF`{|hQDAD`7Ht%CnEXf#_8ALI$t+`9UDF^~zw zAZ)GO?{URr4Q-5O??}JJ75K`7Qj;ANqKa?X8iNd8NFhB!0THY`K*VyqT#%M7H@?UU zD!=f!6#gZ@M>&B~l0t)=n!P66^Xhn60Rg$Y)_JQ}E|u{Q8PIdI{~GmbmHw3Z%of=> z+@E6iS%e_AcQ-eHthMWx?OcHw8P+}kAA#&8;&rH;+8<13>u@{N95rr61dP&n(dIzm zAr*9IHu%lA{m*Sqe=c7l9QZwj9nXPktPC4@O&(8YSim3BQeTu1wvUuxUdElbgjoA* z?#NNfBS#uh-=pV)7_XjQb%-G>qzdD{)b+k2~l$8uXKW`HeRoEWnF z@obXb-i({Bj@(7QbP>9N0gKoc5=ri@6CHRfXG>QyRcAir$tr3wOt5(80t{C2Ep`9% zu3L;|>Ew$e0jTDz8>9PsRyZ0rJ^gO;Zca$Wzi@K@2>IAOym^gh{{VOQneyQEb(75nxcxtOW`HPP2 z3<1Rvs7)kZ5>fhgRw*(%B_GJ9tGY9{!~ip z?LRmT&-#VmuKSZ@!rtmX^aurfQW-T2{O@;Y<C_X5nU){6`%2u5F?R5#9 zMl4UlS1@~YXDGentP2I`jUju#Zx(HRuJ&rp2C(@u2FIq|X8@Q*S=F9z(T!U73dte`7C3D+kn1xFCF&tH9I@c9P zc2(c3@v}DD4IQ?RM#Ih;xdIKZaIo|k5?@$4uM%K5&kijvr{yx&pN=fOkeNn0?Yb0~ z5&C${D%8r2-K=ohRVc+z&d16Z0fk>R3@*mb5F$)RJ`vhm`f05OG-v?)3?$J4#F}Lh zC!%GYL5Og^)q^cA3DBBNxQTEDm0MR~0) z@Iv)Iw+?T;5?Rkt&k}PH>Jt1z^nlyXx_qR7*fkkIMOGZ~;4=<=?Jdn~w2|_#3aYL8%JF6R!Q?I_y{rv$j{uUY z%{lD?Pu55ys-pPN-=sxR;@VIoIRlnF{>bniK5`mUA=#9^Uof#pWlsjch73Sy4Hf09 zPDXB6(f*|oXU4WQT{|{XF-C?f_Sxfe1ag!wC!7X~-DO1Dds)!#T_6syhT=swQfLrq z0<=zPu!;-`XXny~3%M9rzLZxThN5z>L!+goWg+S=E@d+LeLTjzjac#CrHyoAFWh<7~WX|h)Mhk##&j6bp7#@DZ zpiKZJpRi${y?DXs(`VIYrD7F+QwGn#>RTz@5C-vbA8RohfeWSO*E$atTKGq_YnP%L zXZvxGiD3t`=}oM;zG8_QFz>w{emC#kMm7oSO-rx+M!2Hin`ZeAa9wGi%3~+yOoQ+e zb2i$|NqoK}rq}K);JUZ!7mTxT0tA%@nFe!cwfH0?-OxbTJ&(ewB^$5g-x|AmUiD@fq*A zL~i~~14M2_5DhJ2S49BHFf`$Tcxg;X!jW%{(46K{!H3h71b1m(>2%in4iY#SEH7c2FSR`@6ug9Wfsjl)kMn}W`GR15G z%2_?Y#|)S~;RUT1>dcoK9YrliNrYC!LNH=azk#Y3poJKq$piI32^#C`j^S__?iSgX z#4=K}%A6~zk0IGL0 zwRGHX05vp87ftQw)!;L#m%1I!qTn(2_xA_JB&aF_-z@?*-7)pH{22O+w6p>7 z`SA$KLF^~82Iy<+eu@IC(5zX@q(|6N zMOxQLrpH$JghpTw0@qGP;$rAVg{yBeL^i5D=m<*Zlpm{siG)K_Ze&d7#QFX1laZ)$ zl7^>Yusxu=C&4$M>%-ebMO;BboZO9c3+9tKlO4Xivkd3V()wC_4pnV}h$8o;+Bap+w#MIgcRQF4IwIzEAR!6Yzzui^0xIOs zpFaUb$^T-2zO#$A-FoUf4xK_^Rq|lF>*JJr-q8&JO#Wvp{Ra1+_yp*0tokiKRSKmN z@(ZP6#h)k-R#dCQjaGp2P-EPE0?0`PlF{PE;Q=o|W>E5Cp(XE1(iFcR4}PcoCM+b^ z>3=&G@6JQp1SZdF#fc15pp<_#DKMeztVj&NewqvgW&-42=7`1Y&gFdq-N!w?{uI!OW1M99e}aE(G~x@Dxn@cIH> zuSo)vNa8}$hmhVH{ecpZeV6HO{$P+-R(h7sLIx369mJeMMe(dXhdRqe{1f|ZOd&dx z7Yfj;ctS+g^FKdRiCPse&R*Jc06K}dj~BqXmo{I_spo&LYYO~BI1StDN%5D`3D+K(FE zjUJAu@DDH109U3CpE!{Wy6{7kfed0vN9{xP&O!*0M2EjR@WNB*LhnCLG0q{&$i#FA zK>8}f4iN`JKzB`JGbBTn?g17n{;zzhS+)4>gDtzw5ibBwPV`{Fq2PG|4u+kkCR5N9 zp{}BL2hb_T!$DxsIO^*zNIfaSr?MBT&nU9k{J|l(pF@ zt`b?-^*xVL@1pY`=v4VgEV!+Kg`nJOBX$`M$NGhuk^~=Fe$?*tancM}lLn7J(#Qtsm7VBbt_ERO$T*cGETOBb7@V&~uLNO_#lRLL;J%YzVFr0gBjst=+?bwL}tWBayBD8o3?#$4sGuzrha8 z2RZ-<2&*I^ua>t1Fr8mt7Ua$Qsc$EbNSBPxEhPmK7tIKIfdcv(Fc$}IrcJR#UO^!= zS{1kwK$qgP8qWi?W1qIwkxbrnCM{N=gB!GYGim+!Yqr$nYziP@01xMi_<7FgbJUS| zG)H_LkfhN{&_l=WAsm&YNoPlN$YYuhhyCG2IdZn<`WvKoXNJLqQe;+xfL_5Ux4M>2 z(uIYFn!t3e#s5EFClH~h)m;+ga;Z(9e2$=G~~lCy7cGyjpK*B68<0A1wz-7JJ) zh-m-N-yGbdge4U)bBqc`aI%QChkE?`scFTzv6 zB{P8ar`vq^C2k}8KObI_abH9dDm#vxKzZj!8uo-n5L>FItXx6qrn7;pxtbU}3|J1` zp)qqSqF9pDSNbAL1MD6~<4BB#dwz=w42KO6IAmcY!KeL2<8}Qma>MI&v5m*_NzZF1 zHQ_7ovQu}dO5t~f3Dq@ zRbyH)B3vb%LcQG7rAUnUFbGW+%{V7A(}}GM%=$O4@9)3!EVLT2V`+fV(fS#efH4XmO#GeMY%F1`^Z)k(NP^=+`bDuhzrM4(@3*uv zi`-So=nz=?4nE(=kXTG8KQ*l2z63y zxItq|#lbG%Efo?BNzuAn^Z1&>olv7WCot|Qyi@Nwm~0pmDFD$dsSO3B+@45b%I6%_ zsRwmnfU@ABAg6|9BorvS1gm*E$*6+!RLx40lIn9yJCp-WzC1a@GfGmX=N_S{yVfG4 z;84FJs7Hx2KsCZcsDb|#AVz8nXgQx6>>^DI05%Ksgb%=|2o%wOUjyXXa(cMNPgo>2 zFqkyfzt~;gIyW;j13Yq+zz+up2Q4iUQ}nPEqY(Z-NxdEQEN&H_Bp;tWg!#+Qx2zt& zmy_ke@^ZKaU;KCoQ1cjY2@6WG0ww;b;U@XGoXpckdB3?<1H!24qehq(- zfOI)Uc1?d&QZoE$rSf=lyn$_4s+;*IjqVss&s*o-y*P1~G>{-;)J z8KNJY51X-(>xVy=B$`rt1|X4=f8(QztRLCgq(x&KlwuY74F=i1&~0EO#KvBLl868C zs22EO?;7HO!2EZq{_tXFM4mb>y%z)gFw}~fNakR~Q6`Cz-)f->lSUz%@$IxEL9YkE ztUXWLIaye$trlxQJ(+kIv{jKR4Ff77Ohv{o4Vu{fR6jC}NHF?eVw{0ao=xVk5ocPq z>83)Y#wSkJ3RPdrs3Zg#nJor#JLH3+r26$fZERut$OjEJz^ zEP6gE`4rXyuv!3({$#E^ZDeQqvm1&f2s8o4r$xO#QN>j>o3*sv9BBEc7MPR+%{Y|C zmB(Ph@UdT+i0s#ujG6J2d@t!c+Q=n|$)5wPvKf;YxVi7E2Z>9Sa-hUa?=NQ*ZKzzv3ZFJ{j%a1oEJ}R zW+7EJ(!%bqY9lHYp)&EH3vZB%>#jbUlU* zsc4%NULVW?=@1gCGg z))dtxgzH&|r}16!#NOl8jN(Sz=Tfx~?YfF)qN#qJRJRCYA|+kF6R zgyAw}Mn^N7O_Dw-D5+UMU6Frw6BF^T=Eg(PELU4#IsX{q#EjPmLE*&!FSqtn%qC#aLx5NF3?%iW?4$weuDsz#%_k>?S6M-(*;!h#o+PoCY(5b~R&f zwON#WQ{cR$u?`ddn@(sVM^I0gIt~dAr1llqXkyJCdiyWoB@3zR_)W?YLJH|3=rv$M zdY*ILn(}=sLzdrKz-9gL3UnDzN5R?XqsPE7W+OOHxRg=mdg%nCgS4%ykx)v`8p(Xj zWS(g{WXhD#g9OpBy}~LXhXGI+LDe801~m02GexN5?oK?b*YkR7K*r*B+y&!_*<6g1 zv@t1Q3kgi`1@!Dcl>iZ&{xgtVCIlXl_@rQf$`l(HXWM?Q)>lo!@4?2WD1@*PXw9(< zO3vb$gTHg-t-xmk2cr3HqZtnMn_@T=(IH^==Qio0E#Qezd&B3w8utq02s>lb(lIvq z4}D}Ds$%%!dnv={Rl&X@En1+GO%~lC!qKv7lkmw+Jf|}OBBAzU$G>u*^RuL~f)@irV19VnuN=hUW z0U{KkVp&FV1(-lavE^nwrcemuPJ3)UO~_voF&#f460|D9(XxR6S8|q$@M>E8&$b7M zN(QfM;r{zig`>3-0<8`^zreIl@$xj>NSc)Uve4c{7V6AD?m#h}OBl;K+YMI$Bh;<^ z&-S)a=E&{wGOmm6!avarY&)R5ubepm=CIqvwzPZ@(-2+TKNICA{G=?S5a3<@xh!D& z`Ka^+`ipw+XOF(y9qvE<2$N-H@0%9Enlbk82I4aMqOb)`Tk)j|vC0I?WMhQ7pc+_! z(1qWq?GOt*P#iURDFh=Sc5?pY!Oru1G;^rm>)uhZ|MH&fq(2qnb3aXlOEHxirv6|d z=l{V+>)ZR5ucWK`I|V-iTBLK}<1VOZ8I*J1`j02P*SNS@w$%f)22jWu5=4a%M*C(Z zuz>h$%|VC~`EL`p_)U8zij^xwtCV~X7MPCRRLks*N(0A0a7FPEw<@s;?%R3iL_N3@U^M zVmSj<7|JlK(%d`j6P20vu_@S5NA<~`A)=gp!iKBF(~!2adqvzYxZxR@G1-^G>WbGzyq zAwhu$QjM~FUbwX-|J__=6n`v|CNPJ(oE*#@L0;A;>PZ@)jWHy0CuM?(*n+Xg1FoGn z-@uDpH;i@ur~9J%TZv>UnfJp5GgL=7LB2Kg?YECFYd$CP`AW2PU^8%C`_>5hdAsBH zS`9}OQGBeRJp`xl6GOfv%@32`t=ybkTBHO@5XC~I%+^b6A-mM_sYE{biREG*}pw_HG0iO_1xQKDsMuLmkOFgrnm^my7k^fiiF^Y3*kN%1Z$ zL59x4uwkaT>qdR+S-cpuT|R1XFR#lH9-Em`9JPEH79A9?Az%(b<(D2C3zcvd3pc6f zeO~~g1Hg46g{gFeleEPV>GWjXwFBX)l`lX zzy>Ch^CQpaAK8=pFMTPNhoY4e6+6>|3b9s^l5R0^1XS+~L#p1y32{LwCY!9KXT(u^ zU4CzXxly9k$S7tCT0+f68-7qZ#?dG`pqJ2ZIT%Q_rv(%UW$?MzfMf60Wq7eYtYo`r zmrF_a^DI3=Y2et!6Z8~bN3uzbtCf0!c~fBHaddF_XYAs}dB5D7(r@|1A}C@axA}uC^oS-+tQ7ql`u~&KBnIaHfJ6>V z4F_OYqq|nP94~<_6Aa%c)BL0*hNTc+GJ&ic&U@ zHizwyVPq+dzmmmb$rH^%k&wqQT#HinbEt+7^mT1%ih1ZF$MaBp+<1D6nTF&K>=Z8n zq1>;6%1;c+J`-kgHjRLF`}T6b7cBs;An=|_HS7N->3I*Ysk4P~?vo+Jq$m>7wuu$r z(a_K&(W}w}nG0M;;9`cR<0h(07x_c?w`aorNyp8dQ4p+u%s+4w0n}Clu!k=kK=N6% zf{_P^QbmuUE7N4oj%Y3>snYV2JWn$p|Zj3qh<lr{RMhJzFqEK|X-W>o|9{1y= zdix$-AP|9N31ABJV9%$@iIl=B{F3~YKe+?(F|aD z&4FZ`&fy?n*CL}*7XFch5%VjA#VH^n{|CpYU3>za-#@M!&|plRPZz-2K_L;~GoQ`} zHNC&qmu=>`rjfT6M6OSoq;chH;-DvW3DVvL@eTuR#-(6&RN_*UOuZx_N2EX$#YgB$ z&y7|Gg=D@FYz<=@UA7QFj^IW{zf6MR91zX7Rq(N|4(WJ&XoK|&8yRDgEOvj_Z`KF? z-u@A}VPCO-30D0QpbZ?zNVNK>SwCiCs85YIrX^AET{Dc!oOQjKlIl=IdY6<-u$fAR z$zY_GS=4JJUOxgARZME}jVZ%oT8s;=5DiCoiBP8FX;UQf&?1POxrtC%eM#PU z?fesFkxEg3Rc*y#YGRU$_diiq`qf`T47lvm#;9q?5J%qL0pIQ?Z9sRGnPGV^Mzoio?hUP@!dV*>8g52bea2 z3$XfTIB35e}WoHo!duTu+F~Qj%nXg2ka- z`RPOfh9wC2P3;9qC)59U7RNf}PaCtUZt}jMNdY7TK*O6f>n$euE|L1;)=mKc{X3b( z@T6%AC1e~Z%0TS|<>EQ;qPYI%?3_;{O#c4;nOKf$;03^zUh7?u>6DJPHJuF<|psV$oS%+DstVaE3r#efN+om4j3gjDFV7bk$0 zlF2n9Ky!lrDU|KKO##Ez;ZQ}XT__z!+m|RQu!oIjxZ#HIlXd~}Ldd=S&uXh{Z$uja zPJvNpQtQ6i7YfcglSU1ti~c{?2OJ^=P(_<;J0+wukHv-WL3}`}@wMj#MRZ6 zpdoCS9VKYh21cCD>(}fD-~GOg@l_;CWg=nsg(bUKR987SP4~&0l4=s0Vuq@O;X*wV zp?HSe*1^+_2gnW}_U#N+q__tXzhh!2`-PKD+a0T9vtbJ?ao60|r%AnzaK`j(ocTi| zt{cF2LzXwYoo@nXVt=jk6`uT+0=N31A^4DFvav zg#xOL^=b<{V-Ee#xdW$Ehy+0xdU`KND0&LFSfusQuEw(hAxL~@Y!rlI#ND&6%(RBx zzszcUdnx>!V+aD$dl#ah@<9llzbK#AbP_EL*}ynSzJusSDS0+4qO3{{V|MbVs*tPw zMmH6%IWYnJy8y@pVA!Eq=l)YZ0~cxUCI;dR zhV1UVa~O<-jQf|wAn#24AR|31&=080@nbTj7Tv-`7Ln+LLSx(CD~~4e_n4fO#`niU zA}D>Ko&4$o7#GZrQd)RIglQ}ucSNP807!DvA3ul~rK>+2A?#ww`TpXhq@)8~ukyNk zaP^*7EW9)sv1%0!Kc&n-;vQNYqzeVtwhvWUl%t()^%tAa@Nd(imGf(k6a42Fo1hfQ z;IJmZP5wz6f9ZD|+sT5_FD;cr4P8v6Q8Y7*nGg8?Wt-OHe}rhQ2{P>v_j_e95eW&C zRRodH8Gb=C_m%nS);=FBZSwRKfqIUX~jtPH!(?Fai;+;n?0yCX+rFhxBYlb%yo{_dzRVO(dnJ zwgAK$Y`9>3z4=*Mv8ka}Yci%(QfD&F2pHAM-*1auLasW&pD9TaG9=zx4OaUx_IbH1 zfp_G)4hWOL3#tc_D~Z=xcgd$Bv$R6mr%%wdVw7#45)J$tQ!IF3}gJBGCbQP1C^?@9*9)Z+F z7MjoHUjr{!nhO@ljrVjk!>EE!0A|CgjiE(B3yJpM4om^LoTJx6Z!KtKB6(Rsll%aR zo*ao>K4R1v2ZUj0*^2ERqtsfFZU|+$p&+JyXvApKviA)^&cNJL5o$oLnmx+a(m=SuFANr=e<>*AHqw1*PUC=7R< z-~UE+NaFfkac;E9n;o7htL(aX>@FCdv*U?nzNe*?drPkuh(rUMT+I7a2q0r)W`l^v z8?`Z1HYEKq;+zs)0mh-q>0Y&m6O14Vz85nJ$ULrTtRS~Vgp_~dd((5fF0J;AWN&7k z(IFpBJU{*MHdX256udv!YhRiA3itW-KkxNm|3MKW`%K>;&=h9o`%)B%s92mR0dAUa zEM)q*Drng4#9DhI7_fj4!bKS)APt#Y98Qa;q?G*BOC}uyf5eK8cF0vGX+G;Hp#_7F7$K}OK&(g zpr3yGX!^3me%N^HWq6d+TLed=#z7B^H!5a<2seeHJz2XdyJ}|luOb)Gw|Ne%GHvOo zl+eM+ZZKRQ@u_qnX`my+%fe}s_k{H3#N$UP5gJ6;;<}grIXXDe;UVBGQ@saStJSl> z_jU#l9VL}A982&Yif=W+2|?q5rQ{D`z4ZOiBqP3K@gD<0#<%^HZJF#debHu$l7J;r zBkP^S(`(3;Lg_3N)|;NaR}I6vR{!^UoI$drtkN@k;fxbXw5jC;;{+#e2w2)J(Zfp9 zT-~5iVEMu45e*%!lpRf%*CdN2-EA)>9M(PX@4)anfpldD&$(n{yhd&6JJkkp_WuUR zTcO5h|8)dKGx!KW65eOW{{4jCUb=2C)dZJ~lY2MFW!TRp$){lhx(#U*iTT_y;_=al z1xL#mJK$>!@moMlGlBy`{`RNFSn3%t&=xA`8iS?0;Brnyg++?+G9_k@7WxaoxCqST z{D08D7SzCU`FqU?=e_{=z)dK?htaD|e6+#t`0?4>juuw{%u-!uZdSk^AYNTB9quBBcrGmO+m;5SR!m;`i6DqMnn8w%Vv zB`~C;y1_Y0{HhYwIBu-v`01&Db$qN-Y2%xqoR`m&WR~?p%ksgsOeZ%M3UmN|iDxJA zZ745>AF3TCQ3HX391*w0erw)t<)T%MCuLZesoXiLmc#U}&+|R6=y$L4XV%~mnsP{l z&sNe+Hx+5OQ=_X~Rbeb>E}9cw9Gj~-6cVG_EgbA4+vjfY>9lC%lFz1a*Q~hFk>4or zB%2EObnT(K@##`}z4-q5TDN#K_4@FH6F$3l`|iPc@M0|Z14`7y-tGO9ZY65aD$4Q| z{#x1-Nc-LsE}W<}9Fwl?Pj$Ze&D83W!M-HCxOJ^j-f*dU_XNJIdE?8&NFs&lxT%L> z3hme03`uur`d^_Ff?ijgh&Z-^*N&rir=87 zLWYwZ6KsA83aI=ue^;1RisIv{#EkWUA}cs9PU#D!h@bE4P=l&r#}gf1n7#pRCGX+# z`PQ2EUWUfVoR+ro%rQt>!4vd;>JgVcQsDd`679P#5ebq7-tU^vbjmw>RFjEF6Tt@c z=J>8P{5`Y}{`g@(eSaHuVSYS~-D@a%D(jo~`&FEGB%OM>cKCwyW+L6a%W4sXH{)k4 zB(|$Qtau3ED%_drBmm!I)3}s~n2H`%aI&nuv?J>=juVS5pCpU_pjQ1j)#m?L0N3v* z&+CWWD2h%$NWX8Q#^lQW*?V@rOyw4|;#;$f37I)yGs(!jISB6kj+)i_U>ZC~f&iT~ z@dPH2?1wGA*}h}|hN=zGc69hlV%xr4BtP02EY!=@*0SJ>QN_%jsCu&8zziArN+67En)ZiF5o@oi=ks6!MLL${pCXdL!y*b zsuHQi&b0$Bhc6};-{FZ^v%n~G$ZZm{v0=Uy|7pqPfajXks^jE|!+C=!(#|hF=c8rC zHP5q(r3g)aYdDzdxCw#khuwdl2z_AGUq zio)CE1xy_qDtTYz1vq!#OKvR6(YhW~qFeTb9JO2Sb7PMVTpZYZS z-cf_c`TM2Uyj0Y+{kZHCP9=g)Eyoeb>%)Wt%t;5?AH#1)M~fq!PxhnJ?A0Slp2#Eu z=1=J#Y?#FRo13jO4v!kA1BSHgdCrnfy$L!=;pjEF=D+<3+s93~nLVi{`y&wtKf|#r z_&y-Vw)#ukn;R=+Ued&&?1XhD-7V#CUN!F4m2&iMai%sYiUgT7;v+@=ICHk?rnP+b z2^a61#oKZ|s@yYnVdC6W4Xanf%lJu5f!D%OO!GSAPMgu(J#6l? zc3e2G6Ro9*{IZcjKZkaSYz=HT^a3?m%div4hswu|G!l;fbZX3idpMrM`EkNdFHy5! zk;(Xm0|S{uQ}{@zm3m0XUrtGI=EoC+Fc&lK?p`n?VmBOQvTdz?YUf#&(k&8_uwJyj z{%}IUwPaILcK@@uA^QE!T6Phigv0u8&!@#8f_BK+0BpwMRo$34559Kw9L^NEvvFub zhNL5Y+sH?hAmHFS_t{>{x2G4)#z_vC1aqP*t_Y7<9ITFQD568v=+p)-SS^%HuBI=Vog@O9)z&R(X2Z}_8lp-hg_kkQj< z{Hp#2{9qjCGn=<^l#F=OPC+|e$M~le19_eo}(Cf)OYU@f^r*+PR_am`6W?4Ja2~IYgIxV)@#%AU>|UFQ-LQW z&|%lCAO>gbHVH6VE0ui4Kwq{zSqMDhaZAKy-3gA;3zg@ZTIuoF@zKF`39f87>E^gs&X7c{nC$Z8V(f@mT2gYX{rAM!^0{JH zowGe!FS*%gSmw1Antkj;Sq)8)O>L1xx_8g#ReY9%wNjM$HWg%`CV8T^Z?mP03HS|H zFGtxo2dY0ck{HkBm`zk+b%b0|Gf#{9{rRJd9@TFU!_u{3$8!%>(xS4mvX(-H_Y~qy z@8{ySgTU%m-;mSXT#l*4OrSrx+2w0Lj%6mT5c$V>KYlMMOHm_QhZ90j&#$m$%hgrSJMwf2r}_0Je|56x>ZSD8>-pYH*O+sNA6KMV zVaVfBIYtWQmeEDchn!h!qtxGm?Ofk}#i#8YBLqQZ$&$ykDP@yQn~p?&8{{0z z>=2+Yx7{rHoKnp|!?d!pC`;dxK)`Mk=olC~7`?oIVvB?1y>#U%_mR|30-WIk* z9Ub+Ep;oF|uBIpdWpfv^G)P`adXGuZC(fx8mR-0p@`wt*-lBZ^YrOUShG$V#+Ir97 zwpWz6g5Mtf23opST%&n^`k>;-ux_x#X6M(Z6QT=k+QF}EnF6kd3!zrUJuiB-8ner3 z%``*K&vU&cG%l2Vr+*UCzwz?g?2XSfI@Zf<2nyb(%0}b3%sBeo?!>^3=Mt6I@v>W7 z7IwX3Cqg!I2%nj^iL^Pw&?;A>D;gU#R`}J9vdeJUp^;_PJbhb5yWxyM9`l~^Vls4w zvu;VC2uTU&1Ji=5=tD-Zgk#nxiPp1nP8Uk53@|W-%cQ&Fc>X6d!xO}KaF#pDPK?ig zDdRR(2yS}1JN5CwCaULuR)B|l;>Xo3{Y^h+B0I#Xr?BjXqH|~biKeMgO*6Thw4$ZP zV7EBzK`lsR>UVNy_sl0s^}IKC7PtZo-m6W{u=uaj7F#`ssul%G2Gw<3@`T?SgRWro z19`qVbq@sMWcvTWiJ-KcC&vQ z1E8Rw9kk-V;p$GaT3(&CQO8IR^P_hPnB1~7N4jnI=)S~4e~hcmLS63}=`#pbE{f3I ztmD3sTQl*S{mXHjG)C7rhBT3qyD_kziA_3>Gu5zOsmGYUe3c82bSLwWUwo6srTf0n)&g)XhvPRQk{;TOID`no2 zPA*lje{0AXJDc+Bnn(rg@p~MEnJvvVTlU->TtD=b1{W&e#o4WksalJkTwhiwI5PcVoIXoiNjJ4wvM+4=S=h85 zIYD2vC>sJfCV0X*Wo~S*;jU_I?hNV&8Y6!UT~iMaiBGsJgpbbd*b|PBD%V-7m}vB<2$SLYrCKbkzgsEZW1(=h$hGP|yR!__f_F>Q4?#$26XY#dp{`%!> zYNcZ;+B>$u9RRC1|!GyB_2yrRjwe%f%Y4(fw4f=Bn2cO`j zaNu>sl_1dhHyl!E?t`)mhe` zfBm-8S13Ny{hEgDG{oipX2v&IF^wBztnEVa1xOk;;uZt9b5-y zQ8%4kdqhagEZ=x7^_eFTQj-2`rS|DpDSDbla6rqRr@-ezE4S6-Gjkl$4~d;(;HvGs zx9G3n`~B9R*(_d-Vd<*Z6;Y&y^u1`0xdxYQ_f)LS`V3Bk;PPhm{zJf^X1gxk*5f!~ z>!k?=vU(}Ic~NDj2eifSL+GQ-0obBBaUzQiDS4ckzz zr?{OmT%KqjH0B<39nY%HmM+`CDQZ<9c^Iie2RbhSi`BkqfWZC3pnhKR#C+D%<*{0J zRUso|iOxawu{)K!kXiBf)9K~_ElK7FGdUoziOa8YQ+>tDB zA&^)%>~GHwt{sX}wq!Itt~}b4C%o-$m%#Y>M^;vcESH;`8z`|#+b_({zaw~X%p5lD z!X!su7<2r8utcyi@msXJlFHHs^b8qRs+BT~d!W7;5lISP%^x`7x(t5fWXP7JoIPY$ zBr%4$yT1*=dRAt3GM9LmzKqdhjGX;@ZJ;J!_E^^UjJrL`h5>(Pxi@yP=qO)4yXi>g z&Kp=(f_+ikuRxfmSfa}}`a}O!p-mH^S9Opnh2tvtDi@cW8(%cyDshl!~&yQ)~yn*P09r4V-d1AZJ$*3ZvEpX@9y z)RI{cHIAZ715Ip>8cW5EE7@{NerKP+Lko3j4CT! z&Uz?wW~3!2-d}D#lS{Z5_m@B1xi0i`OR0p}ESq@WraB}}~#1u?wT3-mh zocnw?onByL`t-*G-tl>H@BNq?oy`n0@8 zO)gU|8#z5rx9vR9bseAk_!=jlEE@U^lxbTyt(UeB4i0v9plzqJ)w4%#aD3yIlr>wv zrXQaBGXiH_M(^@|Pl3L?p|MeM*H%5dW=j&q%`-+(Wt#ELrLT*6_6$vAWRqe#VGY#r zuUk7}b+wtRT?VBo{{ut%c7PQx#g-`mc;&(8*)JE58Y z>%Pdp?MXDy!NDIqyb--%=76Wd_VG$k@ND)Q!(^Q2ri2@%bbKnOj-P-4&P)GA#x`7Q zqjl80f8QHZb|ZaL>}v$I_vwVrH&YOdZQ~~hPB+C9xlFM+vJIqMiquaR&(9JEX_Z1$ zA3N>`Gw4m?4xk(P+M%%3j3Wp+$^u2bB;$rZM}ITf4f}BYu5C@nkH zlbk6{?*HQI%TeomIu4L0M#QyV&|Mz&uQq1GK!h=uOHM(JyAR`6;$>`8?!R?PJxRRq z#ZwAzzA{NI9f@6{W)*j)57Gp=#tP+Xf>M{RH)nA_onZommy|D#T}5r`qV3D)58~mc3+{wTR~R$zkD{d}e08@9Ni1{4}pqm+C&5Dvck~{E(PA zZWTY|p!)s$`hLmx*voCs`F{7Yz`FCW=d}WoozTgmK#ft|*C^yf2w7Obt?*gL&34aR z(5l(20qu>Z&tdjVe2H`U5hrK1=H4Ra=^S_-N#$10o;DA=eW;IdtFM^Z(A&rM-M528 zq}@Uuv+`iPyQV})*D;+S^uS+=e++} zE5P99ie5_nvNa87G$rxAEIiQXrz%eXyuZYThVOVqCt|1baNoV~;1xB7(rj!RkJnwk zOY`*@alv+i; zVDqj+eC2_uTi?DpCqB0|7b2s_JNX9uyc9VY4!A`1=2<3X6twQ z6%C#_nqZIi|8B7J@$nj792w4^sLszN@Q=Bsv6C6SI(w_h1LuTGle~J+`L}0(FmDHb8?& z+EnNK^5@dhQfgkPFazh?c{Z11IqDK$z$mVABtq2r8437^#m8ll|niDH@HIm(xALk@ntSRCIOD*%?RZ{p>d%^07P8Ej&|~fC3(bT{+m|LCSIrF#(tKbzXK{TqoangH&AFO= zJzvZyP5OX;xiy(#dV=`2=Ol7&;jV8>-L|@Q>E-^k(n7C?3cFE7;2kS&0Y6up`@`HP zKEJ?ixIlRO@UmENDyILq68l)zd-y z42#`ut;0H)c8c`uqEfW-m4gt`kyS^nrD*M!tyvj=#%6urYO7W*``>Mi1DY&O$BlwF z*@AuOf7$+8%ce`=&4injwqQ&#c0jBiDgZu$$7N%oIZq}6*r-D$n{t= zme|Z5NJpdj_8G7;Y`h`+WKzrRF&nsE_ZSUts)zd~DMA~I@mTZ#MbbS|XH!c>=hFil z=j1Z~y;Fm>p%TNLDQY3xb@VmSBy-x>?k@)3y$|OH$|)14h^|XSdDGF%b~cW)h1Z1B zr#P=I7l>LczdUW%jM(tlBq`>trGEJWpc{g=+S)4bn}~7R7n&bA0qqcYI8K4_b$L$Z zkAL&ye_PwL_;tJp=PZ+`H-;9lwkx>od~QBT7*%qUZf`lX-#mQ(P&Y0JFQD_M1%&wU*~mQm$3#c!H>OB2fwB0}WkmKq&B zrJly~_;Q|#YJt5$ZGq2y`%W^0 z=?xxyb+SyXGvA`;$iw6r>FMOc69G=nCqvGokTZqOb3CXuZ^93>iW-OL?oaT=GR2^) zGnAep?h*UD=g+)esiob1LKQ8RDbVNF!5P}J$hJwNu5gY3XbWSVaG^dK;Q<~)?9 z@$~3%?L!)PyJ!8>gpx8NGJq49BIm!Kn)Y{WvsV9hGujx_pV^AKst0ED{iutom*FfKc&+{r%^uQ8k ztb!p=rt&ZibMw9iGQp%|5cY_gR&>1T%Eb!g`5P~2=3oj+piDjZe$YD zThhBe=$hK_QK`dUQ4XN(qI{S!Vok^hf6#ZSl99X$YzN~A`ah{L8CdOP?Qu06Sd%P1 zi*u`cH9Q1=|1J?|BVd0WDU}2GsPnYMpnO4btRMPn?%tEsjpZ9T-rZ!Lil*nutNd}& zuVrTz***~JxfpD#Ye&`RW)JV!XOh2m=a73n25EjcOXj!`gG}LNB|d!We&;9rqCd59 z*e>dj)rcoYO;zgFU!&R|VU;>aEYj?(IaZQ9wEf9-CJ z?67h1s_Jb;7F^Vi!*%KV2zSe*N?xDmYOzJ!LYa@#hU+fZ_3s?j0ug47kbLRde7I~b z8`x4Bn6~RmZw@#F6+0)RvnSV^;wj|$q4I_)e~x5w8B3=n_l5O%mda6oa_kPON`8il zIQWxLSH(~BTQk>+snq!Ngx`JQFMw;Nn&clphG&y&C?5aq54TBlMd_1@ifOw+-cM^e z(@dyqFITNlYh<4T?t}e3M&2+KXAF{uyrit5xUsR0W2ua3;Hq4P#NyPFlCJVF=5W6M z39$gYxe5a%KM|vz5Jx#sS(&mJu;Sh*)haJ*YiX&ft}c#YBHZ&6bxfJ54L4n1tM2UV zWULaE|A8|i_o#2HE|C|DBpu#6)^;;zYRxMh?Ca7${8WiGKdUzPd-gVrO0zxl$xhWY zBCS(c^oHs|!0te*NSR|^RsFM?%VmK{xOlmmvl_O<0}K`iy#0=N^O=OMCvU4d{7?S| ziTbkK+`7Vs#Ce8Tse-dYp;pE=kW}R086H*&ej|p{lPQ1J&Pcnx5*EgPy1WSbfsa}E z!Qon=)5&`ADra0^Se;SZ#a`3?fJ_*BrSowjO0ZK|`+3`Zw)_VJqUmUAHD6nC6VKy7 z#!6d}tF00DSSGsn)ep>wWoroeOXMs+Z(r zU!s*Qr{EH34T9nfaMI+t&OJOmJ^L&c44KW?R(1GQ*cZcChPbS)tv)Qiv5pFdZ+Cx< z(LUE?Y|Cyh57)R3;Mk@w(w$xTtgfNly?c?JyVi83>29n)*Ax1&=5V-@F1xC7Ba+mf zGGR4b0;PG>Y7ZGm%NiNSGW?6KM|sVx2a9D5VS5u=%gfHxh_!HNWiBorf2dv}5GOt_ zEzc-x>8P%)R48F~rcDJpY{>r&lblZ+a~-c(??M(*nl-3iFQu*5Jia`9H05~=b2j3x zj165%{mWo!k;>at1^p-4ae%^(Q&ENyeS?~!=?o`ad~)pXF5Dx>NJ^R%Pkmc+bwf)_ zDigV*-+5sPGj+hvr*74WS<4D+c9FcrZwy7sW7irRvFCy5;l3Xkaj%@55bKygq#Pwi zbMz-YEghYCS~O{4zuYY?od)9IaUon?23!}XYrGF?DNT*d8OA0LDwVaJboc+q0?dMk z4mO9q{aekOdT+prA0hGFe$}2y)+$bqLG;~Z8`*}uy#zAImr_Xon@_*0f?bGR^5(|0 zWi3m|9Lnq6pz_R3=6L*aD|lF~Flpy0Jy(I_;dnZlPlC-t1Y3&LVOA=DVF}ZmdoyHrPo$xzd@RN z6al-zoXiCEfC4%Pb;9ODJws+%;~i{tR94oh<}EuRhv&yvgtK!^R+niuHzB)2@aS}7 zwRP37ReOvIhdEBCQP;(Xva%EGd6qGl*^RDQzt)iMdX{XZsnZuKn7Z1Bn=xMbcThpG zqsV3!L>x`m=5bvWIZ>I+22-x%b?Ci5x?6ts*%B_E1wztlNlMT?Q@)zzb2l|d?u>^m z$(Q;>WM&9C%@81nEff-){T;uve9pIWb&tL#Zq#=VGaj}--mt4B;9l$)&HwQxDps?c zrc-&=-iq9g?S3$5+Whj9_E+cQcJnH~l2WFPDi_=_ub6>Y%8!A&Ygr%Si|1W5@c(bB zQd^;1?&E&2oHu7Qn$pVe>%Y2pn!*NmN=sTr0$3-a1D_7rFh5Dv=B zZQS!?3zd^YdEQvF?^0;7k8wt>ON*rJ?)DT(&c14awN{pgr=dZzrRvg7iS)Ds%>bb!D! z{z2YWl+Lh6*9m%4i};_m=VZ0kN}CK65(G``my*x)e#sf37O~48ehA5RVN=uo$Wty? zn^`&df``d`2N@(U9L0cDpi;6g*~J!?n+B(R1t*;7hWAE^E9JRM4dJ;T zs~J7mHqiiIjRV0$Ta5b=ogIyWKO#2^I(6s->-)_hhZq*M8CEJ{OMnu;nJQLLEnaH^ zPEfI5S-Li(WW6$eg3L9|irB+rnNTPNM5oTP4f%CO&sOdMHiJ^MFGD1xdiLL>G2`8Kk^8{Xq;5t~( zHcbxDd01g8B||od=^Z(5<%Hnia!Dbd*rH)bV5ez+F(RlnJu9EFI+?j}?F63q#nF*-$awX`no{sEe>|D}0IMTY5n(!JQ1Y68QHYFlj7)^f$ zNZL^#{;nemF^Ld#21&_rVL@3!LsQ0fxbg2x5n{117){e6k>^qHR5O|(3hCQ1TtR|= z$$klfV}1Vve*aBKPQar$X*VOn$K%|tPeSENT35B@+HKd6vgLYy=xs8uVmDfEoW$Tw6uV+hb)=2+OU zG9EbW-E5I8O;K>rXhG;19Q@(5L?1xED~RBKFGNlBESyS=%AB|lvHfD7efDAXC)e3N ze0UsO5pgn2F_t8>BYyDNkg&iw;(~is2{7k{2D_i(e6z=y!*d|sO@(eD{9wzf35b8{xG8z0gQOB&Z>=+h5hNGQ3-rTtUI_RFG1#m7kwo7vy|d zZS4^zHljbgLRL6*w26G@nS@`Ntnx&7Wn4p497ygrQkJdLbm&Fagm?Y8HcwnEMd|xZ z!sGu1?i~^0Hxu_I>0KZ&UN)16M*S6gpi+TA`J%08#~&Neu?I?e1b$wiZ*Wf1rV!f; zeqs9eBF+;U6gJt=cKxq zS2^j3&cED%$_ewaj0CANA0Cef2`Te|v`i)wi^Y%@i^V__OJ@oMgj$%e5>qII$mMdo zC@z-^D49$ql}aIvD5+Em6@iAB&*5;Oq*N+JweedTh2M%qB9t$cO0jTnZ!d_FFds~# zCV-;bM=qD!MG1vM8x*t#WMb`dxty*GMe+H3k%&FR)8ZEQh#?&Uc%ryaVSFQQN=3da>~F0d&i5((WGbibnlC`u#}Ib3*$ z3s0ydtuH=*=M|{GvdD+ z_`4%KV-B2C=}27Qhp+;zx}Uko#jU8GMb6fg;^X1#eIobwstpz>P2RwBwJi7M+O}itK7JAI$mRi35E%F1*!hjUO@WzrcXPO`e|a8y}7Vh2S$1-q2m` zyLQR`vp2N{GM+yK19~$()pc`dJ4a03%4tyLo{9JHj%XKMB;A%W_}2Jpbi6yKhhTF13%cfbNa^v>G_gn1y@p5cm&k4#P7j?LQoaNTfBmio~C7n;3)`~gEVr0!`&>%L;`9c zv`1oU6otgp5Ksvy{1&3&ur8E_D0t_h4a%-4xYq?GX%rB`0~0m204e~Xe~5wvFw$eC zSeO?Kb=7RChhQGZR~0x0C80&scJrd||9w~t=0S19^8%n?OX zmm`W+fX;;Leef+RK+D9=qEVom&0iYTpIhFhF#XS%;X^-Pa7d{hh>@Ofn8U}e4w#7w zz)eAb;J}7EkuM~@d_w`k=_aC4*w+*zeIF&hjaFK%wzK`3(}G1+Y$2j(MtZw-;kc!H z7VDx%KFuwU4??4`0A>n0OSY_Xa`W0+Sk)?%ll#B$m^c~f=|E3d)Cv`VTiQ@BjtZC~ z_+0qZ2&Yj}sTAN^tu`ZV*D9~jR<@Antd-xtw0qUMfSN90e^J=2w7tS7h5N7A8mIvF zpp8NM@81I*D3$^jP;YSxPrpR$9fvXDFDkA7iw(@*kcsr)&4iVs%^m`M3y?lS{nM{< zzB3m-j&b2Nf2%Midf7<#Y@Z9y{iCPwJRV_+A+*^#=2c6~O)#2`3Ec=fY6Y)F!c|UH z=hM+czVzL8RI1TiEJPHb-gE^D4+vqopw;VN<)rJ9NF<0WHi+&>4pSUFf{GFrctM1@L!lK%r9F>fzk` z!0*DpGXv__1I0mtX7NvO;mtblg%doJu1XXNh&ETM)E|xu-}vN)v(w6)(t4FfqY!o{ zc(2~J|B^y)gccacfTBV8GYu$&`{T`lKX?<#?wz7PeiMnlB!Jf#=nDy`KYgc)Qz1? z8I4A~vkW!`ni9W-wu9R~^p#k=+zO`921Q@3rBQg17Pezvfx?Hv3%SsUs2E=Ug-st2 zg($nX+W(Zh@TdS@00#pK3eXpUAqV=%=G*sm7asZ%lzGKncxWiPi}6|PCX`AgNCpE6 zWpG^Cg+OM0>Y+X7iq#szNDn!9ks0>8UxB3?7E$l71 zEZ7V5Xu(XoD7*lU33z-&;dCHVPf&kwUvff1LR?(j4(7A>>`kdc4UoV3E<8RyJ@IiY z!Nl)nHJv`XH#i_5Vb`95ibk!`NZ*CWDTdujn+t3K*fYgerBTo<^bLCZci|nU@Qzb> zaJO-<3PZBcC8YNdF!KXBg$D=r?;ZzAZxeyb49|tmIPPZJ1V!MM&R=v2j~kd z=I!=Ce~(bvBoPXYFXEaVS2@X;!E3g#8i!#-?6tX8YTfc{S!Q1bo) z90u>lfWkfzvMwVdJ-9#M+tNhwNcBuJJR=1+jiSlG?V^xknkLhxOG01-Xo&g9NKe;= z-_oM+p^!`)yfZ9}Y}GV#IHG7aZ_K0_>1jG^k zC1&s6{xj!R4d`ovRwg4tjD_RrsdOooj%HX&{2qb7A zBDup)ykYVw|FS>FrL;ba_QBj6Yv(r{2^gObbT@GXtfzIdTCydWE5 zR^eC;3PW51h=O;iLrM4+PT?U0cod6@sZ=T~2{8~500EhB z3J)P8_$_&nJ%z%0=}deUL}3nuWgrS-ETGweTN7FX6o7L;Pyi)h0#Nv}ZxjWL^aG*b z6cyx~Os4)QV5Elv&}BBFFca8z;TdTZgciWJ*u^voYo{}5lt?6kGB~af(qA#i0Y-XA z0~;=kCG2Z>ivDnHPizYCEBlMWUV~T;D1dM{$$bTi&*#IpbQjazfujZdP$+EkUUHffy zv;n12^z6dA@EPd-uxmge!T=uLE(#wCjlNCmqt&C^aUc{_WMe?FFp7d$3Mh;Zg^4#Z zptOC#G%X4)fI?lU06jerMJF(A*MI^eJw7ZZ5QPaak)RI*EkR+NQEKLTC1lbllu0Mv z$biy7NZU1_v?=TlfgDr})uvIjEylivQDg(B4YXa9T?0y^uxBB*2IM1(UIf@h(FPPz zXh5N}KqmfFgB)xDG)DB)!%Q@V^kk%`1C7GA!l_52Xah>4a26w+MxnW&CqAl28&KMU zW4F?4J8S`Zl}4k`fYLvf_6?}Rg?G5{e`^ZY7|_3|0qv>G^_V&Cvq6Kt`10$71J?z;y>Orercf>B91dOc{ilNm4H`6b z>Y`FX->a2V_9T7r*{6e<&(t|{yEJ|8{CFn@)L58%ar&5F1`irE_?Ia;^$HsUT6A&8 z*zdj^G-%NG!zZ51eQq*axLmGI(6rZYJ}5kR@ZeFdd-ax=q^^Hr9B9e4lnFn6J800L ziL+N0R(6^#go>{GatxDQ9K?G9}qP`InYG?Me%q%A;UjUYXvOvP#7i936yUaC6!81 zCXJ%gAY8_9*Ml^vREp0c5D1_EFw)y-f>`>YWHOmRAb=`_LSau&4-~_3MWX;HY_IlT z>$)i{mCY5FWE>kYdfc(RN*Y!2yk@meQb|>ZjMKV()6_wOrt(D+nM@kHEj(yj^wsR_ z>({U6-p}t56Y_IV01{%c82bVh5Q#)|yJ4Dc7EIG993kD^tkQxTPV;9ZpU>qA*xNVG z{AK8Z7A~Rmhk7Lv$uc+Zth_oQmwo9#$d6x*DsN?XcXzX@3j$Wo_m4=B0j4Vkps*sV4r1Vr(x?|uLESG!K%>uzg`^>uUF6y3%qlllcp8oXh#|F+%N zuV2s3&dz&O*exKm_Vf^_+;WnX8JbSj2(QDEGPy^YhyGgS^RD^DJ(j4 zH}r71TtQQ*-+IqecPmUU__=M?t>gh2z9kFuP*$rI9e>zNgm1N4EtJ%1wOG>3e5_XM zxsYW)&kIvAI}7a((r1jAQqsz!R-oz-PnY!jmEf@I^)Fx5_r6x3W=)UB+;K7IpD+() zync4?icMi1JPACk;l-Qg9E8fGeH=HD=xGMi;D7uk(uMT%^MemGi;2at7`0E**7^GR z8yO2qn?jiC&u+wg`t2C0-efiL6QVs65|i@s9+#IlXiUU@Kzq_CbTv`l6!1X=FuCvG9de-{VfSNeoE9NHbKa-#TxU#BQqIh}#=Y1Ja zW%IpBqeka8ib3b1TNmc8@~ZFPzH0*t$4q#8UnY~maUOm5-e54m8Jy8*L}Csgt%1Tw z`aq))G7ySp2S82)yw6YblF$ku3QkKw0lO&7q=^`46moe$;0TDqrT|t~2pPdlqtQsC zunIa8o`q&ep@~u8!UoiZC^%1tOl%57*@%K-Kqi6&5GV@BVnJ;b1!tW$ne?+DEr?#P zN11j}mBj6fI@x!dyV~59{%+cu$Ms?4H6FdEU{rvqy7SD8wNlF!vBNOtm zLD^)&)fdR52`2157pkq%XmEsdH&^c8FgYUTh+1!iMq4p<^z!ITot}9Fh6@Aax73yS zy18tONRcSXyP7-dYNNdTd;@}gH*H+F$n98WvDRR~p2d+#H*9}05re^i3>pyV26f>q zbDc3H>{4l8rl{I&!Q|t2N?)Q>-FxFA{QZOd{kN`gc1=8ZhASn+BJ>c#;R=y9@GWgF z&>$cm5EvhP7Hlh|1)(RtR7z-40ixAvdrB^k8#bwkC54Jg?`BM!yXt9OyTNFrDM%Cy z9Ur2gFg68TCxc8HC6~*wE^J}@r|>{lhNM_XX9q%Ig%3mkkVYyth#P=Rxm*qj8igeR zj!aXSkE~V@PGKV-qAMU1A`BpnC}HfFvl{*-)OcZd_#SQLT|M!kNwHmE*^*OuB6}F^VxXaVDd#s_?4YGN<6gLoFP! z*=*+Vcsi|?$KiBzbhNiLr3bDUF@9FJSc7%p(1H*psEfzrL4qE`z?ls#4BtYxf~cj_ zevHT{hkT=?W{uP2^!rsW3!C)Z9yV#WuCCh`PE4A$D2yY2ps5G)?M`86NqiQZ z7ueJ^N+OZ;sb?g=Kt0bcjT<`gX}1hApXX*yU$nNoi3R^eQP{2Y#Mf#GFP{yW9?VV1 zhd?xFA5=!G)&BQ0(!=w^s@}-#C;=}w?egIM2+|J0uZ;}~`T#P~y>bvLvf%yV;fP_(}@f$m0OXOAX;s0tEzNfY3YW&jBZ3!Afp9?=} z{OFjhMtWqn6keP-amt}Hx1~xovz_~LH?uEBQdc`>$PW)Ygft4;zpSnNy-6H znznqG=_7J0x?Z~Q47okDo3N|dd(o0TCv(W|iPiePTzG9q!IUw-WL5NqUOdb`v(Ur0 zi6wZ~E(6Ee*5yYNh@=UTP+r=S}F*f{8Q+TbPG99 z((rg>Qq@%y&z`sRUS$V(MdYv9VlkVH2EB>dDwT-^(|#TsaXy zAKWQcnoLHew07*MQI9&LX0u7FQR@swlTkf?%&?TKQWZM>Oc;~NWY!xDTCL7#&`N60 z{XAquJx_t%fj{kk(12z}yMHm!Pps8j%%;*Ssne&gXyA~jAemUKA(=*nQpr?mX{#w) z;dj6(`lY@`kHJB|Xqw4&j*v%n3IAZ0yHlVOf#T75@!sFW2#)Zc< zIik?jg1stO2BNF}yDq$4#~)UOAc|heqYDo^gHQl9p+#XP?ZU$b7NF2q{aqKHZVL2S z?7Hx@(}XDWv)K8u?3c<%VoA$yPA zy>rFS+avBomO`V~Yc*#QSov)P<`dgqYQlh59`bM;c%+O=!W-zn4@NVCwpi?}sJ zyYRT10UO8Ak~9ikc-o-Zcj2J~Zj{o8NVvHR7plNE;jXaFo-2Hdd(v=l1ex?95{<&+ zAo`F5H9OGXqPB0(LJWw?}JUk*IA}T6s-;GLvK)`Bi*%KGx>*M7g5_05hrbwlu z&x>eHP=VJ!M50BN=Ve7lZVe0!Ja96-OCUBFjarTR?EW36FJ%ik%_sKl^!M}e2?|O) zaH6u2MNV^=4|X>_VQ~`C=3<~jBvcz`@j!=2P~*#!YON&W_}-8p|E=Ls8FwBEWaRZt zV9-z$_Qgw-(MTU6VFS`u4v(Jb14H|VNDdbsBJa?1#{~Qyf`5mOl4B3_199P@U#wPx zQmObYp|U}&=-WAkALbV&$-NqqHks5)rJM1AP(32mwiElR+XDDs?)LK&y?YKj^|U8!~E>Nk`0_$pmd>u~<|Zg@K9m z(y3G`rCg!a8qMYpAr2Ipf*5G6O2f1a&EbUS9F90p(ol{#(EfW~|Ko9>Fn{pxKzHHk zsg9Rnx@Oh(a=Lnt>dM*1>tFQOI#kqp&G30iv)9RN(a)>1k0k3PmBe zJUlF@ihpfd6;aqkG$TEt?258sq{kLOnM$P+JBt=&H;t&*VWjUb!7fV8eEJurbFe*8 zfj45Lw<`*#gI#WUo5n;1Fku%(k1JXNqG(P2I3qnuL;8A5&{LRxl>n^?j&Pi7cBe2r z2m%p>2^#e}jP$5BO2BTl;g*Mz^c1G+!XBhiYPH%C#^hk6f5}5Z$DqKrIxLPby36iL z4Kv^GDLkyj@ULS9>R5rkpB1QM57e;-3Z@X8SvsN%NDmPzfPyZA9MsA#p)$=aPowBG$b1E%G8F#M2$k(V z0=5SB1^AWljCQG!>F%IqVwy%_Un9w}4IeY2P!oDwA-6m|hT$;UW<1cDSQip1(>&QW zl-pP@k`vRD2s7af9hApON)I6#h4TeY!(red3a2ngLj|y{;AGqeMVkvOiR#gW%Cu>M zAOWZdQ8b}4s!F5a>>d=r0`^bgZ3=x*1{BAZT?0y+LTnnly+{u}8ihvZcMT}^@q00# z^pvADc{@gWS^`=PG`n#9xxd+^`{H*EC=~c_HK25N(9MGBKhc1qDWu0RZ7Z=i=}bhu zjsZor(HPN_2s8gW1BwdZEdFy1DE`u*e7gn|7B^7%4GgHmg?G5{4i_Ft@L(|f6>Lln z7am(xBog7~HQWe*f8x>k-@t|c=%bI|M9E=59R}24Kph4Y#sC;HM;MbrL0%SMV;glk z9oWo|KKkhYeZ+k5RNj$rw1A@p94+8zf!}Qbu$+Ac6xX4@3v(23w1A@p94+ua-U47f dKla<>2g literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-2.png b/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-2.png new file mode 100644 index 0000000000000000000000000000000000000000..587ffc560fadcfb6600fd0b528845753fca53c82 GIT binary patch literal 26563 zcmcG#1yGw^+wTj-El{9%kOHN+mO`Mxin|mkUc5L2ZE#7E;_kFaaV=h?6nA%NaQEV{ zL*MUt-sjzCpFQ81`Oe%k+zGQLS-I9)*SgpL`u#&xl%(;nDX@`{knm(>UaKJ?p+FF? zlQ7W{e~a&mmXMHWkz`+sX}BZr=`Tg7b4MOX47|zAo!WS{j^}wDBKgsbqyn?TQYVxk zQ#+9CXQ(7*g`4w`wdopz((cd8DWwle`BD^0DJ{P_BTSeCnTlF`?o-rnZ?DdFOl{Lc znQS*8$6vHrCs|Pbb;-vdf3w~`{(A8o3=7wa&-A|30X{mloVmYUx_<-P4+NZA-do;X zwW>miY2uOpb+MqlpBi5iJ5q&ix|8D--JJFf$pRbhFEj5|VM;*IU(FDAKMcBuws=Y% z(nsVja7*upFAyx0INune$t$OM!p6_=@}2#U_tFURrPtL;REKbCrho z)!K$704U&me`a{!K~=|T*j&B@gH$GzxK$#W=#}EmCk?zeD=l?F-f}V%?P~dbYh`N0 z>^d-DZl+mu;r6YbzZPd2%>WzhV{hMow$Abn;$p`C@?_|-d@ECY`owmxo~@mRclSm8 z_h*#>E4%OSxTV%Mo&$g&=K_iIB}62)haV%4V4d?3mBOLM0!udi@2j{|Cpzah-PO~J zc(QmLUXyEw>Z3?quA{r$meeDoK#a= z66=9z9AAebhYVx;01Z8zf$Z3{`tj@!=lkg(2#&_{_ZnHpqZ{>o zj|^4Qe0V@^!tve{SS%e#y^uSOXSH>YE#Q zrfSc)Rp5JIc%m=M@*)hp?eM3{?Fc1FnZi1cP5Yeu5>x92i7dD zmikp{d!mU2_Aq}daoIXbkzo-eA*{9hKhg}^v&o90~>zW@t_hsX;Sq+v%U&+nLl z>aV_Kx+*3Q=2l6Obyz9Z*? zBph;Jjeb#%w}|(H(WA&$SSM@`vm9`k*XuFbw-xzL>a(^{ZFWY!Cpmj>kAf5Uual4DTd?panX$|7!No*qz{^>B6!GlsHp@3iK!U zwOu*e+N z0iK@u`R_g5ll1}0ldCgp6vMY3m&dstNAMtzyPJBC-@C~kSEmKY2UkZcUF#!Xq}OTg zFJO1f=zrzl|Gb|wpfD|DqrjBznIDDvKFz{>{%|`aAh>9`zixrGc8<8KK(~ikz@f@O zKTno;8kfy&ZL7k+8a}jt`e;0hkXO$=2d++vgh;&w^eW1^$6o;YKvu;!sKm&UqK{Sn z++O=qXuJ!?iPV5@KgvRE0WPT91sdMt*+nL~1^@o;EX$Cpc9wWb32_feYRy!)d@kJ!(@yB2D)h@{)n|1?udi{0vGds@h zSE_NIU6%joa2h_MzaW~zgRoJds?^C};wNX-FWh7&5%U<*o#S|=!GG1ogQ?*-}O8H7{H6p!CNbt zN)_4wY?w>6c6Pp=y7?~HEP){>4g^_kTBrRx;%sP$pUobWH#S_sJNBLjy2KpWs0R(N zpR@h+t;=nDc#S8yFO=J3zN2?&VX~NLV-nMmIMWWsF@Nf9572Lz*w*wY#%OpO#C}sZ z-F{Lw8B6(GQ!b6Ealv+=I>Ib>?_TdKpVU9X?B|I3RqTbqRk%E6C!+%jzh3oRc)d2% za;i{5dgN>1M0Sy{Ny5&spD)J2n4tAn)Epp4ApiR!e6m^hQaeC>quwoIdDdp;ca9K1 z1x<$N$-gJW<7pJXFc)oXCG2T?%Rb~Z(lGq^+((+B_-H+)=$p|Cs4*8rkw%?lm?K9o zmH_&atW=Dg2Y$y|Lp87K^FRi_lq>E0)M=GK>BbdS~2ZL zN3Sxrx^dSz{0LcZx1i%&a-e;f3Uu9h6W2YJ6-AnfsqzHfD@_pMm+hYBv=(XhnMgWa zh=r-VipfZ{%2NBEpKtsS484mq9O5)$m(4&Wb|W-FPEGIU3TPHeN5-@-L_mCBBw&F* z5^bMV)fhTEI)0X5+8s8=J4Ea_pR{~=Tn-91bcZs708N5@yq9Cg(fGpl`hPsk`=wxT6r-# zphU{y(b0mS`*}n&rA4Hk-Iz5FT#H+_okHUu>0;Rk5os3(GYc)`+P-DABTH|PIR`1t zeLAKSUSgHuiWDj>q~d}=NM!o#ZiQyNrbQbnDzMx6rcL>{!5==hHGC#y`(h(P2$GTV z!82|#yD@+J8^yoFuYyX3$OG@NuG*hwU5ukWJYQ~Oj;j~ke3j7Zl2iY8d_^A*+7w2S$|(5=Rg=KXaC8@&_I>N zQ2$uI@<1$;a{o$K&tv(=zCim_6pw|wJxH1b?1HR<*vUYT&NNs=wudJwMm7E! z+t-M(V>GWjlw;|T|=f}Pp{v#X;=!pAWJMIqjj z3$*l&F&IYod9lKy1?w>phB0}I+L_!cTUS8qdzp;cIN@n3`JYxF^PDHq@HGoDO}^an z7Fx)t^P(1rO-$W>(-{CMYNM%-Ub~6_$3DB>}Py>Fj^jPj56WT-l{dVu*WF%4@ zKK8jqQl1sKADq!o!AWYUJaZ$+AtL^={42jA_jUI12n&raPh(FpVd|b8rP)h8j*RXZ z0dW3m%S6kQOMgMYn*XbUPZR4w@Ugw&1sl!v_s%Qnb-K>ZUS^tFCQACX5*WaiGIf4a zOakM^>SaPliqBT#GPFQkZ2nX-*{g~GSoe8_Rneq~xuuTTtGy;7Hw#z9uEW&ZFiXWi z?LQGE{DRq|j{HwhFgE=)%j6owL#-#AcOfCqnD?h4^YXVqPjj&*)qq0St@rm$lB&-{ z8{T8})-OxC^%AzHCHew5Yk7&-;I>Rvy^z|Sd@Hw&8T1wB_%&S)FxU6-!;`HZDk)oa z-R2m;38@Vi?d!>Ud7C*^=v$r|hRe`uklG4Qh*(z&6I$+_`Ofmm@@Lsp9qmN<(tF#ZkAf&`#Fxr7zB3QH_2Ir&UQzZ$$J^P<+ zzWK56o)461ab9cJt3OB47f=^BaO7|4J4UX84(YVnKgXY`|)_lgxO;Z|)#h$D;OA z+d+ZlnqhzX6EAhdwJTmobdM$j_s}2PLK|QKM5r{FRsxi<)(rj4! z$a?3Cq^#TK`a6O}OoV&4#{6xU=ZOhcPApb=zrzz(Y`9Se7G~4Vn|b}}BHdiq50*PU z^eeus;t}a2N`m*=gTM;hXHL6W zRVxEM5+NH=s{B&I_o~LxH{x7fp<@wA4WES-UHqnmihYm^NR-5-r=Q1tQuRPl)z0H^ zuz9Rg+Tv3PB7_nJmX=UJ3vdH{Yo z%iT#{(glWXeMB4BXotHio~)6nL4Uh#+VTI!e74|c9dtgMZ9){1%BRw$(?MO+XsmQ( zq1NLI>GiR#F?Ky7!q<#}2erh~18Lo@*XxEO96JX+;hhzfuQxmERtO*#d9a#h;qnz? zpuYB-FV(Xk3IV#go8%KTR`wz*>zZH1-Vyl&5)tHeJsC6|mSb@^9EPgDwrYG>$W)q- zCl=C_bCQ8f6-_H2PzOJz%4E_n)pW5sr7A^ebA0^zBI3CLkJqxEYI4;X-&I~j>^MMT zuz4u@n{20Uw>}q=Z|CGNj*F=8qBX1~tXiZMqDz-J zp*c0S1;|}@ZSk*mQtwpZyxUYDHT|@i+=S0CuS6t`S7;ulks00Js+>of9ByLX8pDCA zHh&u}uNQ(CEPkw^M+WakzmJyW10PbH-MG1Kn|dP7l{wctZK z#gH~}kpV_SQTk1ErWieZquJM_H%XuN1gL&&#NEtw+<4xDUIm@l@_=IjsA=y%n^o~% z_8!d)WjB83*9tfo@~f(?m0)s8#=$7ZKr**TQ*A>-TC1d+dGi`v`iQ81vKfU~#<*47 z(^~B}fGIg{H6(3pa!>-;*Nq(KTz!1SB^^rCd6NH=bR32FJ4=mL_AIY{WGMl~KpHYZ1=D7OAq zK-T5TbN5Rd??A76IM#3%ZdDQk%<>gqzf&KXlZR7;#QQYTNR#2KDsL_Q<>jrYmsI$~ z%8gDTPP1cf;?$E&eBT*Y!lQ05ilzpb3i^HZj_Ujz=}KqUP5DeB6q$->E{b}fI43zi z49UI~z*7y%p(gy4k3jbJ5nQFl6Zt*tjhnKhh;y96yD!*V1!#7O;R#GpPh2<$KR7%U zy|V0EmR7~y>plsNjTIm{|5X-D^NYscMuEXf4YUgLrVD$sm4@08yee&(Aem0w0=PjA z;QFQh$_Uxn^~4K~SV@gnwOY-lz}~AGS|1JXPGG~U^!o2WP}#VVnaNWju8y{NE+CK+ z9Z%=Q^3f3~W|4^8NAVD4p_&3?Y>yGqkQd819lx~cbb}!1=20$cY^9cvRpgW=9*e!! z3MvOvB4Kgf+>0pWM}&r3SqjAZxr@W3xx#GCBY+p-S$cRVow24lvd~9{T;1IjDw95H z$KqN=O{llV z9n17F9X(b~!D^5oYh=TzE^45iaui=u@3%_&C42go89&Uv+S!-mYow#BmW(6HivLl( zuiCZFge0M&+B{Oa<_4r!_)ue_CFfr5rs{$}J@MDDpJ*%9Ps6!#8s>_*R6u@R`)*4e zDb>pL^Mjz6tzR)&SHO=yzh324j55K$(N9TK(0mDwUlWH(r?d=;o{ zqY6kT7NN}!_5sxrzM@sdMP~3mCW&0o6*d} zR1LJBvFTDVZ$p`M279wjV#!RNUPn`+)W^68z5I5^8h<3EV)-Ik_hY_X{wRLML2gZs zfSB>Te!2_#$bxte8IFMmlrS)O)~0#;s7jT*(%jO?eBo7q^$UpZuehS>*J zhQnMi;pQ>b-gY?UUJgLNmc!T6+CRAh?!zu`=@&?G1?CsKA(H##Z*1e`4sUDSeI&1Bip>?o{Srcp+uWS z5jz5zGNxvZRl{4S#7#R>8TW6K#vl~W(s<_Q+}A;A<1TR87hh);GK<1TpGV~e4*G18@=Mm2%g7dWUK^+Gsl!XHictA}NWAD1sg z%sF3ll>W@N9$811yrRRnYQr6(Z)*e6hoaG9Qsz2Or$Kw?CXe_M_Z}g*ptfW_W zX%I2zaG{M)-rd-%C~KhJP(C|U8z^$rM0Vuo{?`^Q{To+lJBAUGAWh>VA}|NqxnOo_ zlycKgLH2`yY07*6;*mSwHV5c|_C29Dsfa0>H3{V?FlGR$ZZz_$jy}5ljq=6csrn-&8&f3%^!jUEo}d!T)KzG*&2elx>#FRm0r^%^i1qg;bEt zVM>$XyoS(+f%heMwmpemOv$|T?}5khO#w3-!)^O%?o!_UJ#$^z&arZ*rT60ydA6Rm$lfl`#F=17DmQhQ>%K}qCd|TP{VXzfg5xn=bH!{eMY; zYctL=HG*qUs!3Kglc;ob$qm$W)u?P~8Z(3-<^HC4m{dcn&dW9~SdAe91$(E}UW=Lz zgSHD+i^H_|T*WL&a!^K|#g_^0Cr6~qB@MnMCfqZJGFY+d`Zd{9iA1uILKtn4OpS#f zL>hcY5-Yhj0vj|)<~%i)z5l@hsITnYMtk-W*ucJn z!s){+EQL3yu~_8!IBkI|vi0#aQsGM6V6=epC>ut++ihec%Mm%ynx7+fhp(J9;S)Yh z*e+|_>&~YKSpiMAQ*!4>!xepdT8YL~G_Pw;SFa$Nm+u(zJP8QZ^?k!Y70Uu(rqv~- z$0JJpSG6y|?5L(wuPXOk1$E9avj3sliKcW}$u=!fFc(j=UzriuvZyB+N9GyF!z#K!^-bCwG8! zvxk0S0r-*^eE|ag8J-ya>M(;nAx)V{JzyL96%5Qp$hg)We`&P5krG<@2L|eqcBnv6 zTYTGB9spXgFqTK80j)7zY7g%@@_6G%sw9^On`;P^Ia?#7p(XH}>CfmBm#RL{&?lLo z&c6@$7H5bu+sXC0{mIT0=gG;2^2tgU{>kmN+sWSC%agMmt&_Dr(v!;*i<7M}rjw%; z>62gW7$lBzso;| z|53*HjnT-XACtzH8S#=^Kb9!Ow$L&!bF;sVa_k`$2d@?=_5=)6yiN0)5T-1L$RK?l+AL-;7-qBV~p-u?Py|z@-_T`0_>(6r|+bQH5R?sZ1DTy{g_Yp zh4o6;TK-se*Tib=Tpeu%2md=|olN&R$*iMDM1XOgOA{75ygTg?xls=3mE6y_5r zdAc-Fkh1#SR0vueil`8(H4vsgkuLEV zU*4Sxv*Tu69NF;hsxOKM{8rKD|79mX+@m4QJ6ELI{6+-HXYhjTg9UtYmrAsAA^T&E z70$h815w`^9OblZAh3Tf$nVLN;~Qu=vQAmS&%7IUJ?|8gdT#HxH?@cIU0RtEZXaw* zOJjh7%%v43FIR^ra?fs$00x0~nM-c~5x-n6G=!aaTK^rTZV8 z1m)gE!{6<_+kE&{vNpFoPZJ-MuMgevdox~rjumxt9svggI%{vJe`$T#9V{h{Y&~Nx zq~h&nOpb+PhmAq>hPilBeomOFzfbI%=nH6>PPjek=t^3sR&vFtQxf}&1z086yxyYu zYFlx^P-!Zt3@^MOYqfvlxLH>$XnIoD0XK1at4OFtL$i16azpiwI1>z0avG0V9Y#iR zFLwUs-xg>C)crjjyWDR-+v;!&nmPNt6g5l^SSg`+=vGymzP~sZr$;k`CaZtzuZZZ)(O&UeRbp#h<8RutikPqv^lrzbN zN=7?Y9Xfb2JB%Nosp_-t!a+i_x#TOkDp&2uvO=E_NU$N2%Ij`^9R{z;mw-ug-BTZQ zGqUg?J@s9GR+qF|<8y0Q$NRLz7zpHlsai6?-yi55^HUDh*G9wX2Gv)ew+ro=luegn z!cGWfSE)%f$Za)zzyEZ2G7-Zy?Q=V7(W5v`U1nULPOCBhv(_GKJ~V8`dxT$!CPqwTQN+S8@(Cbc;}xJVMp+Y41_ClzQq9wpmAKY8Fqv+~rz_@2vZB_?X-FhLR| z=8*)ahyhb#2WU5zS)O3E#0%p&CNB#0Xs)aFr=6BL5@`Lf;WHu*uwgxeXzfJCeiWrNPUcYBRIV$`%KIb}C?W~z2$ zJ#UuB_|cC(Cce%jmhN-EA|GH+%K>@4`aQCoY|eg&Kh4{3NzSaWlWm-!)fv?n%j@aU zBEjgSqxq_xfJ;Sx`S6brfvfUXsWbCDJqmV)rD)$Hz!;c78D;wx4Ucqf%;Fti?dsPK zjNJ5KEHP1KZ5ykT#2rbsMD#F3owhIuiwK+N>5k--g{no{BjRbpTn{?ME~zm=*v~QA zSml&q)#g^xai^LKBx<9Bn2u+qiTn9JB$4>=ayf8sz!5t%bSotM$*O(D1+l3CRbNmO zu3xwhry<_WbFc|OU%=DD+OD*$e}Y>il&~1wMNE~Q(hX49lF_1|L8uM-w>H%S*n6HU zRKDn*hgP<3?~}w0G%9m|_u{6);&NIF>3-zX3>~ekUFUHrrznb(JrUAE6|aOIn~K9J zSvrV7Mu{vN@?6xG`h5M{Pj3>A%P*va#tC3fFK`J#PVJI#A`p3a;VP?Qzi8;~YDyNL zMEXA2vsm438H|@J$wU;_2A`EfzwBK%Sl%ikqRy3Kly21 z%3aOCO}esZ%8)}KRKP!Tc}?W9T!$2lEI=c3)NW~2JC@5%lIhO#ZGHZ;&hug$F zRvr2qvT39>2yAR>esv508k=J1*8VOM@_xxcHboa(hcoGvB4b5TJ(oTiDY1daWD0%f zDl23BL!bOY{LS!p+59uIqyb^sbQx9mgi-mUY+6*rDZLORhK*rwWoX`k z`ABD{MJ*YXQUENwM)EN$pb>+!f(A5~e>TSD-PDJ+dlrjul(7TTrY>#_AbM=!x>-T!wwMx8VR#g@vwYa%r)TYTR- z8boQy6e0bv^byL>DRF+d8ztQ#BlGLdqWQ&(X!!>f!4?*5retYjSBGiKodLC)Y*vx_ zfldiP=)%;Dvvkx<+z%qc*RcPah@fcJ^eSN#z1w~QZKgNNUGJOxVpfYv&4(+cC?S3l zDul=RT|V)nxjLcG*(6#o8e4$pTye;^Lp39BPa^mvX#_;WT1l$%n{{TPzi0(UO z326n0R!A<)P@?l%v#gm$crO)P^ z&HUJpJAexCxIWVQ+u6HO3oZ;e7ULY0_0_1%nPs?QZ&#r3c+8&M)g^pogXY-p<*pEh zivLo^ZIvEWw5zq2><2k7Yols@^`Rq(&AXiQmdnWBki&)Fy#4|=+{_VLeTnhipmPXW z>SznHd6$A^!rehmc=KhI&Rre(Te zKI*{3N)>UWun9ZulVZL5#y=&6p}2{vyFo;y!4-Vi_n8?e?j@!Y(N_c5FdhHo$FjMtnY*m5tYfv;QmoF>xWoI289C*-4O_I0`Iv&*2xMGi5t@Q2Aer5#aH`jR; zpZkv}^&a+|63*HKD688)@A?X7-_&0vvWjRTP0x4OrgdIeMTlsLOzohGVBk|4B2p8q z9bHLxk5d3r9>ZO)+}zIc^mYY3ApgP__5waCKU&^QH0q}~Y7x($S;2%~;w&J{i{5_v z-VYYkinUe-N>Gx4jr;wlVz5!bn?cVZ>Rf~5_LRUEhMP~SOXln;;`D(@Z!~3u;1+zFh2Ma^AtVYHj zrcNEZTgObq1Hm?U$LqNZGx#jyfpq&9A5es#+maAuTPlKTOGZ!=DF~V^2|-Sz+AlRH zBj||~1Z10pAShCmGr`FSiXsJ{2AqT-DN@}I7LyS)MaoP2#Uun#k*ZZ@kc^-zQb;)s zk`QD?s>M)7GJ>v1VN%XWLJ$_I(vj2%aB#Tp{}p$G^|b~pWfxu7{0D{5>W%43S9srx z`5bI!Lo4|=hVVarkpCZt;)c8d3j>|Snv~9Cb#5HrV>#v#iVnUVRW(9ZKlEK=Dd(RRx!z13o_;9OZd(g~*A{MJ6PQFf$145Q=&sWc`Z<2_eNha+#6hy` zM{9W|%G<4s5RapJ-m|b|)-C;;H@#g-kZi3is+*n>uLg5=v|KhsnL zqpW)9A;k+q8Lc1bisxPduL_#UgjK;;An?3D$;swyK?9ZBeC;8|AM} zntR~`^nEhFt$vALOrRM{TG6e3dlOp`HG3zYK(o7Wk#qd(kO)@8Io#gThkaJFW-*PZ zYc}(wrczc3)<|ec(0xvX1>^cPU3C0p`$IRpgp2n4lywZI#$Yltyv*>MpGII+{$7uX zehx3#S?}t(VYa~HfH6$bL46EYtG1`p(EikiI1HlT<_Al27;BxPfN%W0-%UR-h-anw zv)HjaRGVbxb=X2^-1thO%aUdA{m=T}pMRS}Bp+&WSh=O?jMcAI_rZJV=I$C^BU~J{ zxLOzQ@MSxpiWQ3TM1dra4;y31!sL)YdKM*)d?u)i1RTn zH}<5-i&hv%V2)qB+P5C1&?{+M_)8FkGrU?6G+p(3m0n53swDJon_|VP5T|L;_!dXz zn*&^*)W>)!vX@dps&5=E;bmdh@<)+m%DNjteYy_hJl!G``3Bn0o8qYLIRkta7hdoF zlYCIq&)9gfVcW|)QsP8|nxjNb#v?F8MIwuq!PAa^Y#xo;mjN+;$zJ_JJZ$e>NJe*n zizu&WeVeRPU)$r9w>M#(+~KO#_m~NO6e-kC^+j!zyNR~D_=?rv+e<-Tdt-j0_o5p( zqF;N=l09TjY50hJ@>lQG6cOig^43VJ1+(ZOzRuN}MR#`Nj9&82T+8~wV<4yxPBX7;K?)J#1%l5Ck zl+cS0P}|J0{qyJU3XDWA`?TDg!RdaFV%gM^erNK+pN(#G*Rrcd>wX|f>844*QPf%a z2xlC_IM7u2-3>m3H#hDzM~4etwvJ) z}d+yWN%zSYmplbvVBuIvr#)O@8K=2|hN zrYeBD#WE?OX#44rsPDyM2`NnLewciP`4piuZ0Xo)s^7G2LxL5+(ay88r|Mli{Ag=b zB~9S|Xn5ce;-L)fEZhIMp4`ysgB>e*Uyp}?qIkwTA$DBCQXUggcx+-ti}rP?LKH?f z(w4ha^YhxfwMi9~|BKv^`-_5vFOxXW%qnRuo_qCp{NG|l-1Cky=X@nKgaQ;osIUcPgS3xpSodjTA@XP5QPoe{Q#jP30wY)T88RX zjIFrU_9E%8jA=OPOL{rPmEBS5;=dxyaH2-d-sAPCF1H3m}>Vk1L+ zpcorOkzwg0W^wT)5R(PWn;qC>35_k>jgJ1Ru&b&j?u>6cAS_B$Y=`aRn)S>bbzAO; z4h5B=IS?!P;f%~-p`yE>`KMGqyyx(o=@38Zwb_%9o=?aj%0@KkRt5JtJaOhSZe3i- z^N+|iwzx9rKXFik*8Wt`mZl_k?BNU(vQ{ypLHt&aySncVaMg-6NI$Z!8l#m>v}ll6 z(!|59GB65pwoRZvv*pixjP)SK1HNt|g$yvD`E9ixQV5ra>SFZBv7Z1cF zHw$V>lE2!-p1`S8O2VhfxmM7F zvtMBQ^I9l&a9{p8O|NfXO&$!U&t^*u8jofgS6BFn&20gp8~h4);y1{D+RqS1(v z6r3)8iYswDU_3@Y=xrq}^xt0ysCjn#pBltv2=&QmgC*J6E$^WkBED#AT5=XgvR{4- z&67R}?CPCY`;%RY(~F z7*5Vo>9D<@w^G1sBV||0(KKPuw+^t&YK^?OMy5I`$=){)*Jv)ChHYM>ti6niJA_6Z) zhOl@~mv_i&QJ2sF_W_u_cI{Gqa-sXaU;$76R04+^_UR$qcAL+62 zi>hDF#FxJHkszyhL1X;OEhmIP;TZIM0c&>NH(=^~$_qc;^5^9~p4IEa@(p%=YbQuM zIpC7xlU{-Y+8MVTWrwd5erAXDqH~#G7;+N%(Fon9B~e=zHPLStn(ox z%6!~&{)~D6lS{gQfT=^MlWOa0IEpEqvzm-l0>^~>iY3|3=BM_X!?@m`r!$Aqop}0G zwz(K2*f&9!Os1#U^8Eg(ZH13S^5>g?%&6_FoTlmEA`hB}xI4l2AP2)U@kixnYl9!Z5SfzaZK(e; zq1|13zcIWvwHb?G)lit z^9AZmUaS74U!NN>PUK<*fez9$E z%XBK(miMf7>%AuB19=i*^3Fr)fh5#mlHgF#GlQsKM9gGO_!ED0_5mjm^+k9A8cMuCmWxob`WEfvy}`Rqy4e-J?VSFmNaj zQTo1s+R#nnIo(u?o^+1M6Ib?MD{4n~+OqgZpd4>39(Q5C!i}qi0oxW4Np@J0;rm8KIkd7e=qQD+a`DHH=X^eNchNMG;T_w zJn&kQ3N(RMh*M0p@P)%@g@ba%wTNrviQ0U39nzMPi^K!Qp!71hz6^PVKmFb}KgB?{ z%hS`geQ=)p)x;0=ZIrwhN>?W1%L*0PVF?M_sH_gI0n|ZLia&DJ-EDH2Zhy@Av$(gc zZBbiupi&ubHp*Q!T^v?)mTe&~Qf!fY#`cl*;2(013l&N}fsaL%6+)}mFM^?uh1g~> zjCdVxc{srsy%;tm1T;ZC(oFZN;e-pzM9;w=KoBo!wz2f@4y%NkDS17my=}czwk_XsBeR&uM11TrUVSjn0DHYh_y^xni+I~m zzw$ISf&nO|L^?#x{!iDk1G)R55X5Ck^FJIY)A-0-ug?t0kvsCk@pAsQp!~N1g8y;d z;lFLh+j8BsD=UMeE#9fEPpo(5xD8Br!#-?opU(7g9I3r?J94fP?MHA6pGGL;2(%wg ztv!is&-6G>Et;a7wVN&0WEu`@C8JvEYMhR2z6Ya}Q^e$hp5TLA{#44#IHqgU81P`d z^%@rJc5%PGI_*ll6eSSmlOV@43(w^9NrtJQSh$1q((IhU=ouhBEXtMb1w)Fs?c#6Okkh5?8%b6gB zM=@H_*weo)1AMv^_-PCSV(0sXo2tg2ZhG2n&7A0dnN$7(`gU!`3`wvrb*GYz3DS$ON ze6pOg%eurHMVT6il~wLZ$s07hFsOVAJNzIsL+62QYG`2e_mEMMMt_KQHduS?{jF<% z*YgNdFY|LRcYu0~ATt2KVp~%6)Ac|*O(WuGMH4j^bA)eoswqYXd0vj4A^I_fPwcov z0HS_ASjH_3z}-LA7(4+f9M>VXa#O#Kwd%Z^DHp>cDC+{9>)-{|Gn$nPgqcWd{p$CH z5Co|eyY@gUi*1zMP;bV6aNUY^dVfikag(WHVNzj7`@j>1$@w(L~x(@MJn7{#KMAq6vT{#;t|ii7lBtw)_} zIPu(nDGoeg)6;nZueuS|3#DMUH~$-x;DGrqI>$m77A{<#ubyDiVI{4gZ}LyiMbi$P zbN3qIxoCtSSOiL@2Npq^Av8NUt?Tl~luEift@5IlV{zUU_Ymt~P{^W`Nm}HK@i0lH z7k;!{rVf>YY?_ZQEFzE*&I+rRF|J9phPkGnPU*66$pxkf$xHj(#;YOn%=T3isy{RTJGa)RjlOOF zz!Eox{pJIez`s>5%lfQjS4rA|SQH^8kpCM>fVaziRH!sUKnWo80V-0Kq`_$&<#_>D zIf?@3F1pP@1$s!u#12TZd3*VOwe^+;;M3 zrDf&Y{EGfuC<_8QW~^o|xaIop>#QS!nKCD@*E)|Q>xiutOujJR~^Y#b?28!R2q=s@9%s-B@NsLag=5>fY`>e94RBm4u1 z@sPOP2uG-@+@Z?(e+-xpo-z}=??e_NF-|T(@0AZu-!>KjKypzrU%~+D6=WpOPV(@& z8+ZM}q;ftF9%ERKgY{4YhT3#%Fr)yeMuV!8ZbxW0GMSeNN~n1Ce8V!{L(wsm(Mkp$ zru>}$<%+!BP|5D{Yle+qd{`c#d(~W7!r3vByy&A0V2|W9Wvyr30VU`p#0Wk?tZl|q zJEb+h@AWgg5HK@fTRhyBF{J;SL(KN+*|cvVCW#|6~*9)N6EHDpiRpxl(kU#^C{@O6}P%EKGws68L?% z42~~9j^~R#_WMGfHq(R}mS29N<=8ph~6R>^0FZ)CeFG5Ap)Pl|>!r!t&c8uInj3!8dKso-YET-5du6>$Bx(XJ! z#Aa-Qt($dx!;tKgw9Hnh6^-o(H%{P;XckR;q1I+KyO0XP48i_iW{42A+L`V_o!rG% z`9J;y383eLM4*k32*~^JNFOAEXZ2sh*=qVl<*w64aa|hzm?Chulm5~P9y}_x{|}u& zMrjkTDLhjbx>ku22VX*tFHYAA~=gGU+-Y>*NOmxu>kk%mI8464&s z@n%Ia{EFS{f1r!iiTBM|vSh`bIx+PRX zX%-P!Iu?lq36W-%kWMM-29d^~B&8%JcFFI6_kH_3?>yhk_xu;J*PmT zf|DDAswf|Vh7P|Khs@8=_6DwVje>Zagpp9cH%iQ!m(xIJ1#vEWQye;+!&%P*o52vA ziY~0&;`(t1%u|87G*}+Qmg_(wp>&`8M!Y33AJ>>E-W0{bB(6`H;XrzL@5p~>OB-1X z9=eA$(Sy~Ct?r3fkh42Za*uw` zXv;3C?9Lmqy{=D3W+Y#SJWD2JQil;;^_^&M&OW~Qk|K2vgp(P-Py3Yj%ybn;`cl1o zwAG*f1c7A6ZeOY?MVfifHlg~79gjToU4FL2+6@n}xI zOYSK!l!%E^P`syw=uKt37mcx3O#z1--6)l>FQ%M#9f|$rk`hvl+QRj#BBRUQ`#3k{ zLNQ@2iSaf%mZx2NlzWe`E=PzYS-na@a-RKS5bzKL1_9+fJ&sSzIW=pVn4Eo&3lHKT z_SR_260x3&PeabVknhj%O+DnA>=JJv%y$V+Lo237J%gso%T?F2=7xMiu|DxsX@gU9 z*639DP)0Ri?TPFII|ylf1IgQqTj@7F?90MiVq3ZT^ib#7)K+j*d`lOd&8s8Y(IV_? z9}#wSzC%fS9U#)~P9ku(k@3u#@dBH5R6`KM>lK-LH1{iWa_JrdUmZ-)^OuAsEr@nA z&5@3Z5CHfreIkNkDGHe-2d;^_8PPBZtc<=!$nXpLA~zEl*%jkaE=ECSw2=iE5O4dr z!QRqT(gZjvTS3EHQDQznB;l@Ynb(waZg{I-kwx6}?D)z?y83}YQMH_Ig~X-*Vg=w6 zQBBn?SXD7!ArzL3!6Wq46W~r^OpA3myN-^#m94lv6ie1%%Y^B%uV!+i0Cs@7roVOo z4pc(-b=zoq~!F_YU>YLT+9qn7XFV^R}} ztx|UN<2c6gdnNdT1`>`H<493k*8rXHAgw-N2{Mws!erpHMBZYpOS-as(ykLJ zyo`&FyBhOx-#Yo~y%$7Lv}v0Ro9LNjf0eLLDn_1v@e8T=kFbzSZ(q(K&6JPx^k-_~ zql_J-{N0k>Z`LXq+?0z${p#dsDD>$L6F^QsnzKy1<&%6`~a@` zuT5XD3oY0;I?Bnh0=o`h3tSDDdXN}2_0xU2EUvvoZ{;m1-kW>P0SaR~vM;1`C=9+< ztTV*c>~WFW&B{O`t3w1>1ZE8PG*rI+1{YL76L`ig>B9+RsHOn9IGUCHt*JT&ofm@! z=W37bB&h1B5-b=lXP4?N*MXzlFx|od1BfHjZkH?lBmN$;Pmw|_lC%a?ni7v1oNp7~ z`O4W1s`(H=IF65=HYC*b6GHJ#-dB(9RWyXyc72RDr)mb7%RVk+pY*5;-;EZEfy7+^ z3#qzSY7<~|KS^RsNhd-DL=Gg-ANQdg3JI1xcp~syWkX5 zyic;kX04*dY+Pgev{VW0@8T6!Va%kwln29B%Yvu3o(y}TF| zUynd=>7*47pM6sS?ND$1qPhF-XpLdm$0c)iqBgQPr;yWRHL==~1SQp-u{ZC~HtZx+ zIQ|IzzNz@&5VN8ZY9o)lT0P-x_S0c)X{$P1s*WJw5z(0!l>^h`@9Rz{x5h#vJK03t zh^}`xX?+_~9R7+=ceG zo446KZEDO@-^Z5rEwdPr^_nI|Aku3u`3zaZDsO`kb{(_AjE8v|K<=PBcj3X>){p|n zqT?9z))ybA@+T7zQsOQ)Gis*g5`%idQ%9@rfPq(0I(Z+z^rN2EfZh)vVgy}0IXWW; z1s#vDn~2Zt7BL;jit@bC?d8l#itSDb3@N|Y%n z`sJL(&&zwb$rhOoq;vu=%;3n4^>Y01>Cv~{Qm#Hz#N~C# zEhNm(X2rQoTtEl&ALp~ZbHr|w6qr1-#2U0uN_gyufrJ2afFN<>D@96B*N@QB7gO3C=rE$CA0hLL|94M!d@ZEt*Rc}dmLCDOSncjk4h-L>{yZO>}T?z>BU zz>n~2pOa_Fvsu*!pOevg$-31WS#q~n*Aa>{lDfq><*t^8NmR`+Av+M!z z3XN_|H=j>046+2fD{jfn^rtc#?Pp*8;1~ckDBfVrmr@PDU_IMV1*U4KX!Lbvdb6>ap?$5aj2q5;q*)xWwadvlZ6nzs15bWfNwk zb?Ng3q74Vz*RC3{2Cad`>LYQ3Y5-b6o8*8K4@|SrRj!)`WE&)maHQt2UlQvx(2-)VEjhDk` zu8}IY^zq|WkEl#k+=?mkk;|;>bew^%I=xy*A6jmGP0F-rU+N8d;UnLbMscWz+JSNn=t|3y5?)y>H`L*n^&X8IA6zV4L+$% znFkr<2{ZqCkuv&wci;kchoyZhp=3Bj0cJ;dup+#3Bo!N_8HQ_`~Hi=&EQr$@$yE9G$~L8Gg#{R{`PIBF6X5 z^IT(lqc-`7U)KcM6F#$r+5?e>-M$;{UhDKRMJ6Y$0t8WdhGKS*MT+jL*7liIX3+1% zUPhTid!KJuUIGhjNXoT`7VKlXi&8Q>vDZez{{a@DM=EXj0I%?M!keh;nRJbaz@#w7 zj-YOP0NlU6xkyRA?h+0^3e`;$y>die*6UF;r2?mkJkV7+W*o$9yWR;J4ajR6(WReq z(%4pj_gaf(3QL|9!(hy=DC*+N;#CHl$0mt>?v;1rjva~jwzx*CFlX#V- z013@hI5hM_IPEEL9gD+yh#ivJe`5`5^M|4%X%Ckigw2L=1gt3AZL@hjS1jX9erpvz zL!Yx{@~?4S8PxISlD-}y%O~lS$7?K?>oF`)xZa~o=b8|2Q<;f*WHDd)**`~!Sfq?w zWr_M7DkAG2ufiqlilVB=NR$plu*=SXF89P59!TvZYx$#3w?Lw^VaVW#q-eKEB1s3< z=U(z>=mn{sJuSvn;uqR#MV4;MgQXC_4i09GQ4vnNMiFzLRFvO3iXZqTZ%mg^s3JNkduYtJkVgcEuNUj?*AHgY&wDU`;il zrO^&)=A{DkKolrWEwkwSW>ZdW9O)S3`H!<_F-g)eV+_WRz^(Dhldn5t9RPSvQ`c((WgRKSBeW^ zL~T!zXIbwCt1s4D)%T%I@0rP&d1C1dwC~OY+d%W|CM7X8MMn&Vg`eG zG6GI#!-yK1v6!ajzI@)LY~$DQB>k1(JWj=^;iCujP;#op^j4$VwZrls{cs(rrR*vw z?O1Ev_Ll2b(N3xF6Hi|K3Q5rN1ee+R?G0eBxy{D+GIX;t6qd!4F=d({yc(lz2k19Y z$E3PI#K9)JxcHD#+KQ~>ntC_K(0rT@h!Sy}Fbi?Sw1%%h#d>JQsx~ z5RNAm<#sYxXqMl3dw=kgT}j(Y zanAdcwcb&oi%6@?H;BL54MQl$qeekZ*Kp?6pWix&#pj?eO7-%6+Br0cKno#XAT6Mfrpz*%8; zjIy>Sk~1UCGXN4X6Vi*(C*6rNE0Wn&Eq(4SswQHx!zgGjCl3HdYzZ>jk4pYTFP(AD zqUu>wWR~-IzSj1`0+zeir^|Z22C!J)9-PD*O7E16~pRKmt{vUW-QZ^+ABmkr;9En zBQ=P2w`PEI=qO!XMad*8RmqUQQTDF%2O36?=N7tCZptWvx-d)*I1mnz?NoC=x`k3m z6P?Hq#Wt4)qevviXj(%Aau~;Q(ab{rp4&F@=D>DixX&1a`OBWmh=MGLwpZY zykTt1SWhncZ|+#cCVD`GR1!yn@#%#oJv?B5_X-(sOvy4momXmpJH{I*!p78NxOCua zzJJoX2L5L!g8x5-VIbWt0C*XqTQiNqfSYmGd8R=S@H0e8A2bR9j>aAS2Mq#%ry)9+ zAtemB8h4=?l7fJ*AriwVRRQd;uBW;rdIbOJhb$?Ouvx+I(DClS27Cig$*~ml&J!u_ z!~4FZh3d+s9WI7SgLm-<%UuI(^+sv)e88{FuSQmu%Xi*ZzZGYT;9No?JD|`NY$! zLAyi$G@LK`nzm6b?^^An8!mQ)!OI=tuCN`YtBxXrwwQ!t{VcXz5 zfd8;;$9{*J8geoOX#Ewo_94=Tm^{ zP}EHH!3X~%pl{DLKx$RZ2%A9kOxuR*ryw#7x!V)F8%yrxyYW$>n$nh4W9Ti;?`<>_ z+324@28W{m)4fA&4cJ<(LXy+tg$OEIfhL7I>tOwt@4wBwEbiNX^piEwoOH+Z!&7gV zWu{PVJ!MS<<{9^AVjck=+(Hr4{9l$sH0*S=V|jmi%xUz19rk2|PaK=Q9a_Z@?mMQQ z6izWoNKf!AU3I$J;fS79&{+rLD-L7z#c!|e^yiVXYCXGM#Z*bDsl4XDg1(LpUI@3s z6%SgGPe=1UgHt^Ug}CaURqha!0Xyq~+seVWblg0oIE)#~YQI<9b6|@FmjFKidl{U+ zf2S^yrMbQ;r3DpXI(K09)Fu0XX8{rhHVpXHk_h?a)R!ij#dclJZ!iB(b=5WoxFcdL z0PlQj+`lRJx+>kl*fTYKM4RE@0w0 z8(Ibizj9)H>?x7Fum06;wZpl57YJe@m$=VoJnIH`-Xj!50i?KA!4QjWdZZ@!g@+_3 zyLZVXvf|yM%N~On0T1h-O}Z@oG;KfC&HQ`8f0Dy4T}0qdv)5;mgOz?zL13R?Anvcc zpR=f?3vldP+v9&Gj;Ix6w?`aIQ83isI?pdAYPf#_*+w}C9fK$#>xd!{1Jg^H80d4t z;bwPb5w}`i{=-4*?xdc|1;Fp6KGzCsS#6KG*Jjkgdz0I{aaVW~h_CjK?{93oesn+# zfWiO;Mb`+sA}%vi&=!|iR=vg*UkTo{(_(4!#HobkCDYH=uhlc zOa>2W>w-glmPppzj$P{8(ycLn%a|v9KG{EjeRBX){q@*);By3%exPS=?n$Gx>0ZVR z>M31s0uxlYVb|>@T>fc)21z-?>B28+bkLaltIPn!(B{}~0-fVV-e$l#ZV{eFEi|fHzX|8nG)qktpd>k!^^*R#kP&Pe%_`c;0S)=x^8fj71*4m4{+*u_?dWb`acf>$!| z7pUU~sF@Z^hiqT>XX2kQ2}xtghRPs8k8-Y38b7HphEpxDTKvP5#&{q zqY=hr4*51_6!GW8m*m9jBGEc$q?iyL*T}o}0p9`T5=YGJkP{q@J3vb(c$OGz*`7Te zJE17=C2%`uH=V$C;>{jCvZ%Ft89aW*(BzO;hB#N)QR}nqxV3709*YB_%oOn-DBWtuLMizPg=c}U+S%7)3spYHc1RwP^N?AGTN1eLAGO_SBB zULNiR&#MtUFr-~6T3nZu@aB)w%cF7mp?M_cd)f8BY%aG1%Uh1k;;E2OSGlT0`4h2} zKi-^87oSoJR+_hD7bi(wrIO%T56yN|&8xmK@h}jtEHGB&Xz_!PZDfr3SRUN!HJG+& z{=8+U{OXl(V&9Q7jxUkL24NPbOF#Dt;DCYoUdXp?N1x&2GpuA@`gJtBpA34_3mz>F z^I?3SzyKK>3ic6Gftua-c4^>N`1=!}Wrx1Fu;0;s5{u literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-3.png b/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-3.png new file mode 100644 index 0000000000000000000000000000000000000000..4c280ac00f8972b15e06b3528a17c6d7600c96f7 GIT binary patch literal 22581 zcmd43Wl$X5+Ng~~aDvMK2?-Khf`-8%I0S+_B)B^nWN;@0OK^gP-~`tpxVyW%3_AE3 zvY%(~XV-auoUiJfs;|0gW`6W^ufBWDy;fiMwI)nONd^a#0uuoN0Y~<&YCjtU3f~=&Nx(DKc{!$4cXUvgAS|L!rbW3=U^6Y{z0$-lA9HZPqI~<>x z$Mg1z)y?^gsqWq${&E1rx61sqi}AN1gX^Vtzt*N@KYny{ci)P>x@>v4X^}ZN z`r4>%*fsI8ZSI&C!0~7Ms*B!VTCkV7VhY>mmR9t=J~J=Jn{F7tJKsw}b(6u(bB~Ri zD8PHs)^*JJ*H;h#94&bz=)o~7zha7c?smp9veDfhPDLI}!D*;&dC33S@WdYvE|mVb ztFn>#^;o`$4Br0z4ZJmCWGCM7;Qo+!qe`^PsF&~XcqH>bv-uR+)u-Zr0ilr!$Bc*+7Y4@K_RM6_rxH~L83 zT4KtI?ewSNt#GarEORXE_^q!$CqBBd>lfu4cjxG`tbUt3V6{RLapqoC@5aaAFn_6j zT-PvDyEpOv_x5gYG4JURLB3zicv{D5uYNIao`~7)ydL2~tE1cN2ePRQxCvr&aN8fzHggGoA-rLW2qPLz7W##+f!9L?pr>&Zlw{Ovc0XVy6 zMwb~6R~fvmR1W^-2%EKNGsi&1M#nF|4dccvdwbSRoIEiE9bWk;+8xn4d|ToaK?NaD zPB699rF)z2svnOa>9xpL&GzrsWFP^J@|9kf#lhx388JLwgg5NiSOe7)BQ6wbH28T( zR1rOtd(HiDGc8_0CAO`Onp^}5Kw-4x0uSAHSZL8~$=onGR z`zJW9eaiq@i1kMtc%i}lh^0(Otj%N)m4fp|*j;~_Ir3ClZYw%}#Gbd+C9?!Me|5@X z>kU<|ntMU?en5XT-hOWyBzAn1KN*nGpkw5|R#I9rf3E0_9q-W)bEXDChw)E9lvbVX z!TL;(XL#(~x<>A1*(H@f+LmWfYM_1}71w*1+ z-UPi-=FMK>6aq3(PY*;7g`vB_Lp%*^v4~_>B=@aaE9|O{dz$g-$-b3X8utGE!P5Nz zQUf{8emU{r^%wxLZ9}xyg0L3Q#gk&qJLR(M_ZVuNrSb-6KFaBW1vYDpV1i+=V1-xahV)drmDu}ONIiUiE(O04vhT>YhK z#P{+~Fz)wS79)`JIn6Ec+*f&Kl5OS$Khw0e3oak0{{BV(Hs&rFn1cI;3cyd?vZ;>T zpa#k3M;Be@I=!4#LZvy-GA5{@W0QXD7p2O^v63l#FMvYmZfbw}cVEkG1UxBF;U_=U znr(ArR7jLebXc!A^8>LCb!N|)!J`Zoe*8JX+kBP%m7z?{l>!DS0k9m$Zs97g3&Fz- z{}I6e4TSu?@ILNW>iXDI%(ozRnjt5T9oiOY#uRCW*u78&38C#lEV}i&clFl(qJq=H zbl7Ftc=xg686ofVsgYUZJLw3*XT8vUK3b~5gJ zkc8$qx&GSiFPZ+2e%fVJ0}s}dlwO(TZg3< zU-zYEZ)d|6A6LU>kRv>Wxn%&oooGaST#ge?FOQalV2q1%HTDZBeD?EuQ>B2$ZyOtZ zNrxQ~q=#3hTZc2h%nx_Q@^QZ(bj8p#$iz_B`QEP4T$q!6#r>bQ`D|P&?=D;f7;y1#dP+@FZxbM*W2 z<)Dx7U0NxD!O8%zSIIaH*8Z-CcFDhv_pifBq2!ppXYt4)4_4MchUZi@nap(N^Dd+Y zL4!o=f1F*E0C7i)7|XySctwr05uVb=34UzJ-IYU1qLVf#{@qBwXDX7Sf*KwEq{+sx zaa8+%H_;7a-)W%AM_Rg0bN8``GKxc5VO4(cI@UDMp zopgZvI<_W@ljQkqI7$3zJz{ra+w+;r8fNqw1_sg^=BR*W$fm~Nw2w$E1Na;$*oYDk z_?r?cJUnTb`XTwId%+#cFra$6TOEwwy+L2-D9N6+tnZw2ig@d!GGZcQu1YtYc8baJ zX+{Zz%}3yV5lN^Hl~ThSBw4T3UVK*xYwKfAlpy0U&=ogpSS(p6Q*f;^MXYnu4*%zU zyTf?f^|K|2yEu#g8{`Shp~v3&&;*)gQU!_ru3A?ODM5-Bx`W^n2}@@`vyd>&JeY13 zINYG9xowl{e(K*q5QLAI=Y*K(zlET)Zp!2e7&tMzSHdt2^05! z_Qw5S_}z80-oUmd2JvI<6KRu_`F|lT6nUEW8~3!+L+2nCYj zMyunRG2oiPry#NAF8xlN)Kl$7(P*CPhDQ0yt^OqC)_=nOSM=*2d%HXGmQEIak(N0_?Mgeer8^k8(wqF^3ka4AQ^&o`S0_r~WID;qd0AubFc9uv0GS zv4LvjI~jK4cXLAK`#Ko#-aR*1Uv+|lrJj1U+03S&rWmZAeQ(bUf=CTJt-7q;Yv6^q6T)}#>E#Xio5ke+yqRQ3qva;w+??yc zeUdXN_v2tL(Dm{ElxHARL(h4DZSeWyF2=Yse13WSCxPExop#;b-_6}!pZDF}UX9(s zpl#j{_Y2-P7X#jR*Aw1XCmr7Rw=>??XFcAxuo3taskO=XVX^7cIF6?tVE$nsH-5L$R=()#6ew>i}ZyWe# z1dLu~su^R+I`y9(h7G=vLZEtgqbUtJW->(<_C zz_)BImV9?YZaZz!^4Gb)1%0290=*cU`+>Q$9SH(bsr44K^~S9tXP!W=d5ISD4^OuZ z>USjG`82wvc{f%M3{q>k?x>}FLDyX+a@_sClWN-Bq0DH$OSbj&VN_M+E zqQ*{`C3~rgDhJ1Q!@g+%m&VVa{u%dT@ik=N?NQl#E$b-}TaeCKjoCzRh`j=yT^J3M zA@0oWOiavkw8Zf|CSvN`;pWO6$HwUbhUkfF1c3m;?Si*Z3E1wU%Lg%8^m2&kcS9GznVXCK;2=(_T{qT&o|p6l7A_EZo!9L1l4g^A+CnrG zgii~aGXM?BwmdBsi%FC-N-B8&bG_6HS~mnmwSB*1-CdJw=>OiqV_x(uyTGtl_q`BO zLzFJ7hFzQ6c4egygZs|5x2BP%6n6++o{IoV`Q(Ov>T@N1o`b{wR-@dwxsf2e*4~n9 zZx<8G5JIXFExovyL=xt0(i@|RTxG9I(pax>e%F!800Tv$qL0j#m0<&bqOa0%iEbQ7 zhgB=pX#y`7uc?$3HLG|K)aCOn&nQquM=LqoN8f4}v0XPVM2m%e4AIs#MV6cGS*7#9 zp;A%UevKqKwq2Eo2c}!_ZsoeI!J{$YYq1`S->vNBJt5W>V#S$62l>U z8@rvD^@5dA%W;-qOeJLN8!OoXZX7%9#P2XUKZ9($vt7y`zb$8xh5%Kji%PlKcAz3O z-K<>1JZpPohRBFv%QX)`hj$~8E6vi5@}_jqSA~A6&`fi-S|_cBX{fz=mrI31=-n5W zqMr+HGg4ucmDqy8g0tNr&0muEzew#3TjwNK$5{HTCRdXc9J^P*;;C*z*;hmitiETckG=Fw=tP+VRLpi54Y z9l-pXBhqi_7j_=((O!N?Y}yC`y_m%QW51|sp&J^&v2#jCVVfK;){4eDba_%>Bzo1W`N2TVNB^2415QeA{< zETrm+D75R1L!b1?GE^j$ZSI*p(SZE`&2Qkrk}Etjt(1V;TPHvusDf5!^tFk(6S+fs zv?l|g&}wNsJ=Px*^N|!egTE$5hO@^MWdg)2UB(SbW@CI;w{c*~Dq!B>;znQ-xV~lv zg|e+l-p4AFk$w^D4=_x`1Fk|y9AV;CKc<854&)nvoOhU?Gn*ZS{5+BMVWvPgqcW_F zPn$IRI^UYBL4|8--9wi_hq>JmH3}^*AUC{yyOM?XTc@u0X9rZ%1;|XAPmKuu`Ugb` zCCD;|_oR=-CCs`Iu_tFm=!j&E5hQ@Zg#$Wre7}c-pHQb^vUFNnCuJehR}-Y>iZa{d z$-hnINPDmXiGAW%=^Bk>Z)z#42cMsDrPAjrlxL8j`4{$UkHOYmZvlHE<6+kbsI)Xg zpGmPJ?L#)0K*8cD=21XaoLTWmZ#r<^Hhz`E=q%M7R0Jp8Qnk~x7lE9+|6QVw)Lhn7 zQDyYAseBH4Cv=_CzT}~GyK35HZm}hYHyomt!T?`3pGA z1a}5aKsT`7h%A3~dCh{)GEJpgsEoioYmj=m9O!r9M6#tsnnB_d=xeAh67*EQoKwQ# zSqk2KNl>yrtbGuZ6jea1<+^e1i!ZCMzYijPxv3Sp-+lSL_o{R0E#EfsqS5^OozfMV z4?pHqbiTV_%Iyx>f#C^rVGz8)2-4ai>yB0kll_JZRvl;de=6Cf-6(SsFd8L;tttSW ziqRHS5k-u@xK;gP7JpaX#_u_@9O2qO_bqPM2j)G!&S^F$1!zhfS0`@bT*C8&R%cKWf_*KN+yWA z74J-GN$)EA8r8_C)O<1uq592<=+QPzrjkHLyuWCZ-zJCoxH)h&{oVJ0X<>u-6Zqt2=phmIv@^U~s#II|#b?$85COHGZRbCp1jBORwQ?? zlnH*{kN=hA`U}sCH>MKzCYXvUoi>PRIf#N%7wRNu$r9=5SB&s8MENiRiqf}v8iL%c zO>Yqp)kwYH9%H+BalcmGN_j(YmC##pR-M*k%WvX(Yq+EJp2-N5_vI&!f5psCp4PmC zjARB)agwSFS=I&IF(ePgZOS+$l8gcvu~{%xpC^5uT0{I(sj{)UY}r{=cC5<5?}plC z;x5WBfiL8A%*Qg2qQZb_8DB--w@B;iM($y9Nu9g^dV?3z#TM?t3*FK#;jSvJ%lUMG zXJ5PNjGNeyh<%$(e!B~P`GtE>TH`dw!}oF52z_?VYK~_XLS8(!v??`}#4nXT6ZAXj zW6{^uox5|I4)yDguo0i_*QTOPJ!wd}?44uN*n5Tr*~S4xl1;>5{c_yn!05`!q{O^s z2u$T97WhzDC5vTomCIY@!Wvaws;4$@=GH$pmms)EEnk7`WM3^RkdV0ylwHcC-mM-D zHYCk-(;E@uC9$MDOYT-~FsqD4)oMV;*biURr*lw92^uuYU)F|P+)n*Y-5j(2&E2LS zTLWROS;;A;GnDK4x!Ba%lBl6WMK^8Xm33G|7@moX<}7@kUHsD4(o*h$A!3N=eu4`n zzzP`qzUWK-`3fQBD?N+)oLqoGi&1kfL-)3;ir8^>oIQYSmXsz+lhqdK!N~dN*+ylB zn%z%33s=lDL|Q9Ky|Z_|@aHW`cW>8?68p*;QzhlO4RWDSpB;CtAI_V^A?fx(Nx_iV zHw164xHrxaJvVfjsa`cAw}#u>g9FelSsFD0QIWvrPaGO>myRc7b;+nsP?^s=IyMiRPuN;|$6I3{V+iJ4ib&#wF} zY|tUnzV|k%@AI5t34>FS|0xoCmRt;WCA9y;=N!jh|DO1$Fop^-IK_z-xgl!fnH7c3oiX#j_}u7U=^(BIEA7+-@>jh_?Y!}C_Qd?8 zVnG5!+45_wUMS66rc{0jko!b{tU4(qm$iVOX$|sMlQ9@e?@=z4xB*LnD@{6p~&JbG+8|LmIL*p+Z*)?0fOR z5(p7ltTu~T*c#fKaQ*aLv6f?#)E6Rvpg+At!ZBOH2`u>GM_mRVi)T5BD#=Cq3A=M7Y1y2whm?T7UR*-@>_;A`(r&`-ZpY)UG0{nAbh7 zOrQ3iqYOAw`)5~GIu-hS_B|{8j`udzc|;^-OPW@RG6T~`vWp)r`(dh=<|^Ox(&yOH za{9>*Dc$>7#P0ZN1Y574{Dt=lnWe7`Ie`_>ooBQS+hGS~x};rtCoix1VorkMcEEre zj;k|(Ait07gNYG3vrOBv=8jQr^nH00NAG&{6p-UpllWPvi^aO8B_*>0Q+AcGfbB6r~d?RgCMG4b|UZO2vlOF-f2+Hx5YCD0Z0pF^!4EwvmDWr!TYne^dQso~+)P{tvg zIUn9!9v#8~^&uQiAND1&AD(Uv9ggKIAFg%99JYnv9p2qs9>QxohX->thXW~mhv$1! zhZBXGhyMe`KJ1BOIE1cs9gbwn+zYvh2HugQ{ijIy&rg@x$fV_P{^9!H{9^$hA!PhM zfq(ZvK|-@O#V4B!%&=bu~^=KXPfk1Q&7Y9@YBA@9`fKZ z4rF6Hi8bwhCN$YjDJxflrnY!1gAaI&18Zf4N{sZiLhV^=j_o&&PEO*PxFlHTHGOr? z_h9oSs(d|LieIB_PPt0&D`ECyhBP$i*S5D#cRP=K{(K@q*Ve8RFM6r=v4UENbAwPv z(mzt!172`*h}d9LqMiO)rF*DD+V%V8ZTcezI2YuV{9*K?&XxX~Ozpf(ON^*NT?BAz zm@?OR&(1rb#tshOUvlqmq?-%^A6d^n^n~pX|8T29ZI~o{<<#KX=Cm`p*UIX3uFKd`lgzH!hyA9826UDji0rK%y zWX$Q3o?e6P*LLyGq<4KJ_rgyGOJ7-wM5Db=C_*}#3I zvOx_kAV|>%-AvY&!aqU(T-WwkE39=`IU?-!m4zkF>}Bgu`3wq|H|zV4_dO4A*4yVi z<)ue%*S0`o_qPjBlf70GQ*uI&=P*Ux@N2FE*}*%K{)Wm~ajTLiQ%|13a+Zw1(skj5 ztyHFm4xINy*)E6}f*|FF;edLK_(SV=w>z&j)JBy^Xo58A9N%@I^LS{fBMFsQBYuBl z%L~$3EB&@aK_gJ=)ISBq3Y7)HFHHyHmI=Wn(_XH=L_$lL|JlAx_I1@0Ynpy(6wu(v zHCM90L46Qdoh0#zk3&uC!xn8qt1O{L7l~!fh|FFX&EgT!EIgw*hDNX$T(GA4`>#LH z8JY=E#5`A7Obm#N;Ht590=1edZRCyhf5>MP=H}rN*@`vAuU)$il19JUPoOy#o)GDpNO*@1sWK2)9w+7Tu^G#VJV{^ z!J2MfZB!|w(Cnv<#lRNe5>jzA!KV^5R*4FFih6IgCh$zP(F(=QvYAZx#~_mmebRf1 zLff0J1EyZ(8qImOEPWOyD{>TgZ6>zp_RD}V6Xo&H)OAi|)|+noGsDc{qAwa8Fvp!&ZJC6=C|9?O+%c7< zoFLk&p0Mj3!tzTjqfkHnuRKY}?q34@V(T3u{SU_#vTSwW^vQsr-)kHi19lN7BVHu~JcoAzOrNT6frM}#=U&&w-B;P< z+8L_!IYr9^&VHSf!)+D!N!4ntmBq?r>M5*mWn|bENK`0C$%Yh|u=vz9tDfsS%T2Q~ z&bi9n{Fv+SlmKd*vCKGexf$CGdVLXV+s7V-7ZzXEWzxNDAT#rKzVRR;j8Q1N!vBO? zmyar;kl?JOkVKl9oeX9h1{7+Z43W|tL(yueyPZv3vWkLynSD`?SdO8UXb-dnGRQP7 z=?FvKUxI`;C1eg{NyG04f@3nM3+!1t4aK%5%Jmpi$Y@GY9+pe1j^kiw=x$~5=y7nn zqqqd>39mOSD9La}_#KDgAR?*kkUKiSPI}}@BWz)Yyo_R5Q}^UqKgJahK9`=i>GZpC z<_%A>Ng(%+(~M(P?l#J$zEkwdh2|UCG(Fd>$w4q81urDOt zSbEQ5suJKe(7Q|KrTKx$Sp%$He&7IXrm#l?a;Q7bTD z$-M%0fbxu`?o#k*GE2GcF$GJbH^jAr5mIQxmwZ5c{|Yqp?Pk=Ro?%^~MDv@MUcSZ- zkddF@{qF8x%>=ZFN|+8NPka$$IoLo@s>T2x83v456t83V$aT7~H7kqBMcA^LppKlT zNI*f=MBLt(!fB^Y5&8n2N5x2PVc}UMlEV2Spq5V%3E2g%`MlBvpbt~`a zm2^_#=~)eT)K1}1j!S8xdEL-wlg}b}W52_6 zd)afl!W~ps<;jF*Nt!;|0{F|us3=E77B_Jf!iJJ>GS7c5hjWR~O zBTp>Z)?&)YjZpffJ9{X3U%~qp2{85In}O0~Ha$(?#XIf9#ZC5_h#Gc=Jdp#A*``up>3qn^%7lJB=#{Qyau*|T9wXjb_mcm|!b1y^*d}SMNZaZ&f z*%00G1jZzkJciuxSR^8WD?sjtXwfFU%Czm-$=7U)Y(vvxE^;kn;%VHqdF)8X>^R0< z8{$S=L#bVSYLQ!+uh+WHpX996Onl2alX*r@wSk=Lw*`E}3(sXEf8^6prK&Tg#xhhY zvPlfC1k(jtiYm4;nJzG>yzG1$z8a3`hwEy_{E57qkyPAS*3GXe{@TTM zx5++{r%$5D`%?M^_>I+KPC=g{*^aS7gjrW~C!;$9SLyA`dL1@iSEU}?7d4jgZpHhv zEz?X*Ha`bwqNlhH60@HTZ+YZh<=c{qbsiVJ5hPe){OH({&LC&)he5i#F=kIAs0#!c*M?F`eoXGlLB;<=mnm6j#O z!wd#PVt$Kk3#`zbX9*-uz!fx_+hi`E`g<1?c+?L+u4!oQus|o|KvN|I~S7thM@v0L!|0KdvpN5Z*KgqMV9N*Um;-*85FcHydyGdxslS z*wRk(-rXW9$4De&-M%u!VptD^LwLjoCX2I;AZ~Qa$V*i3U-%c9!z{;7qWc*?wVWU; z5i~>9N3$wPIq^Su50lK|49!}CFsZq&EL|-`M6Jc_6T}R<9^(#YzWnAD1uNrLzY^GQ zw!AxN7vkvqz8i!Z2ERvDgMR)sj?QqxSMkj!(|}^jku99}%LRc)8gD+41Jex8bk8&AEE;Q@edT5LUlC#Q z4X@~i_dMjhOJ}8iUjDdIMZcAl-z4`i3;ZzY8rLnf!@A9N&BwZ6w}IY2NRpwtKJMfA z@GIyc5cHudx5)Bq+rh@l8TxkVKpi;!74VzB_yem^b_*rqIeVo@>Al0fakH6(LchKq zNfdO1$`{g7xrfrC3ityt10gv6HM54;`$iv3F-pTjraHz%4g(|OkDzPFbRdZ)o2^in z?vj;exj8LkIr8Fz9!qXhj&DlLFb)F#@AvP!MhSL?QM7Yyi#T#4Cok43zj1z5*eb_L zr(+X!u4eSKS!T-?+0oKRn;NU*0qts$|E+apT)}m%w9DjA4(1kkR{6ucx&72HwBAcB zo8a?rl=B2tGksu_{@TeXLEFe0dq_1ZL#CzLP-hMoIb4SZGQ6GP>Q~^uH3W0$<&oj# z))39*&87Y2-W2=g>6Y^4S{L5s-HrR@!5rV^`JU#Zh+l$Y1%)|a~z%$Fw{@|UX}SeLg~&X@Z$+?QuNYM1LhgqJX=>E-qa zoi7n#XqiYG#N~h5mVI27pS$hvmY4jOWL8FrwETItg$pTInVhnl6Rz+HBAx3Mh>%QA zKr8hAX_JIMdQw`);@sF7^~?VTB{8UOu8l1JWo0~vYbUQr8i%pl^4UDvx{eFzRE83+ zO~LvA_P*Eu5MVMx;Vx4+DmzcxaRJuiU-C)ry_S(?SDzEkzihBSS`y=P_&{dTiZ#Fi z+V$uAcM}`X-QXHAi-z48%r5OVPyaR=PJfP7o1xjIE6Ae*m<0$ojMj41%wWhP5Qw`y zE?qv#u^$UzZaDv>(d=h;4;|(eSnE;`8SQ$Z%eoJSTLo3y9+0d2P&HD;EgLNAG6She zuZ>?3X<=!AJ8?KPKw3x7JB`83=hdNM3_c=e(}zK7FTx@;3==?P0mrIm`}V2GI#+ar zr>kSXOTUPOwaDGQXm_Xr%$soX3lI)Q3^iXP7fo!!!hJl|b*>s3_3{=zK2_BBsj;Tg zHKWR8=N1$BdCqV`W103tLDq| zT)Rm>E1Elc8>)wjv6cFTdPJnw^JNkX>`!5f=!I?LTHPS&JtI zvrbz0}xrZvGygRsUQ((s`b_`8OAOz&l1uvDorKnDq(5-69^ zD>SuRn2-@cXyY}&*LS+r)>S9yb8sv8Af_B*`*XvCf zNpU)mlO*RzB>x`aE-33;VX*Sb_0ms~poifRm%ACJvWB{5r^12GVqC!VdDb`4Lnn-P zRS^w>e_FioEudAJp(VSI`jHacY`2f%wn>uQVI=43E!j;C4td@jHz+T#dA}p8jnDam z3Knd|rheve|s00nOW#xr<0GDGvbAzc*?Y*P(^9pq+XbW=Vcx)rfc;4}12Iq}f zvt;HPQ=^Ex%4qgw;K1vM1Gnvb2Aua~lcrBtaV{!Bewn85p_`(ZMKL=me`3qSfR`5T zO}I)<<2;9@;n0wQ+rBfGnO=_+iA4E|=E?ZN1x=%x!`X#-09zR{?bxJ<%~hZ6d%c~L zeJ=^ZK6dn)#Yb51s6;%%f{MPZ>Q$bfOSI9cOhV|i?aS+{k|?U@&kuO7CnDSNj@&VA z)q1!2+ofI%;dVL1t`K<)KFBn8yS*L{LVeXc$0cIm&_F%SiK`Xc?ph2dDD+Z0h0+Z6 zF{b!l8`nVrB!5u@@Y<`;sH5<)0Js;S>T9$-c|5w^R%tNfQ17=HNLbC}5E}mMgd46! z_>`MEB%1y~2IgaCRDpnL+*@Yv?SDfEhb{M zc=A}B0v`B0N+eJ^AxXR}djKkYPDh>dzzGL7AfrT3e@~pOVi~zfhybkWSG_^(P9KEhFig#&Nexw z8e$!tHWG}tVw!PHxIEoJj|B4V{p1JH;3|R8K6aYI&&ni{30Fkt80RF1dtV(<@I0qB zJl=hh4n^|eNqTV{w&waKJWPgunhes3&^JPEdp^(SsfwV`9&w(`-sS^*p6e&XnEI>DNj^!8)(1HO?~8P~N$AqEQ9eUA39 zbQ&JDNb|Rer^R+*InJt*Ew?^ah#1|YAgN4sQ`{T1*3jtsyf;<}?M&{V%yahx&-fz@ zpm?`I{j{zPb?n3U*d0K`@I4YKds|ryab##VmGb=wfDJ%2yq!ryI!cB<$H}Eni zTjYU7%B%-kh~D5{P7|F0^(Yh@D{==F_aq+aQm7MEJ(Umh;%V9GMRURs$M}cT@OXGG zM*o4VAO}rlNpg6cJ&2QHG#E5zl56petj@m9&2Wrh76 zQUBN&0OP|OlhEZ=`@B>YlCht7(5K9+R^i zcO`kWIad>^;Ez_*H-yiO>X@ISJOZ%-Ce$}HYFvU~~$1YOISF$w( z`wWn*^D4y(ccdC{zftZY8SknkX}7cltbO+n^9xBia>Tvh^<9xA@n^8z6Boi^4w}dx= zNG#{*us=xbNd)T%erFGEAh(iN`Wr+jm|?7^E+o z8IQ!7m=B~486?3T1g<693 z?JEd+?#s0W21d|G7&0@b!6T(GdG#+!K@s-qVN(e_O1frg6LDTTK2j#e^?flMQlf?K zAugi>e`-^6Faj}Nz0G*|!+35rQob0@E(ThMr?60;x=`dq*K~c^_nS{)Rq^-v52fG| zD%l25MpWW>DX!=1bLn&;73s5vj*Svs|6CIsV4%>}o@H?fcElQDsM=~F+2qMEAXFkh zgo{&@O0$9|AClGzcBK#r6%Tix(-@QOwi~HfC&4Epl+}sdYj39Q7isGsVrA;iX)Wob>P@>7=)6mM;zl!|`BOqZ&1~YTtdt z?=T-4c8As>YpwvD3oiQGEWV&;9dngPxpq5XoX+5o1V$Owp*` zyfu}%B{B|xEMGmHFKOcgxsH+qG8@7mJ4yV3Yf`e$J~aE*g+rLO0Pl#JdD@=G69RU- zKV5Kr#vdO|;T>b2jfPP^W2RUsDYnuy>j}5^&oqGQ}4@I&InL7xcsKz)R0ZWPO~<> zyBss-Jw*P;=%x9bi`A`kMGR4> zbDGEjIKM#QF51|_#kUB|JHvvkIV)+QK|2135bXxWcz=lyMo(?VL~zMKgWyIAL6slt zSUR5x;X`FEW0J`<`4QHAXJrqMF~@#@3Wj>?2!t=y_JOfSf%kQ)j&1dc<%0?@G=@1L0>*`-lV}ZS zFopXN#CSa^tYqbUy0k!v-{;X$4K|UcDpIyViId!Hq7Av!HCx%9+t{XGyQus!QXl;C zll51=-k?fciDP}t*YU98GSXQ4!8clNB<)4diUwtZO)uVEi%4tMT=nwDl*a*?uyGyfI9m0!s7`--&V; zNkJYljsMmmuJEx8xGo@{s{Oox;)&4{S1LYKayS`+ga086&5#nMt)@-tdGgtS*++@` z7y1*83H(|z6eUj<#VlL7ZxIA}oL1H{95NFLD0w_d^+U zHj}Or>OGzRT4LAm3<0;`j+xE~7GDmg`hSQP{;$)IG2-b3Jwt(eg4Q72B3>n*!V)){ z7E{gqF<*5;5j!d%?LyIK%rd;RJhO(U7=P9U?9f;f>7{SmUYD8I_zd;NUaTyWh^?VJ zG1HV4*OZ}a=wGYnuZnH`aR&~pu-uCCE!&#CN~*Nw_j0|H%60daPVnK?T4S#(O&=b; zt8K?3y(>sL`xty(2MU|ak|Ib0sgn$J_U2}dCNw(72E9gWoo4zXqZGmmuS73B2kL^P zK{W|@>PsaAy8d)2dpajG8yifQ1tP?EUm?E3QOO6^(52oC$_x^3SMB{$n&NwBP(5

;TM|v%{HI$8Z0SoaOGeU zr!QBx6urhMP9J^a&w>X4k8tJpKVN8=_1E8ff?A6wOscr}a1dB7RHNVMn-?=`vimK( z5CMU7{qcVR?3Y*ng)dzH3tuE86>o(|l|Pf#Spin%w<7cEo;LZj8umhPT|U;i%|Rlu zmKfL1hsbGSN?Ct&TT{RpQ}U<+=QbJ5^q+}#YBw80|ihY zyzsHgK?Y~vD*I(5`rQLth^YPQ^LNUR>rvd?dyUE_7;2EM5m1;Lb}zxQg?u8d0xxjp z?Yf@$()e5!%;hQv9?Lhp`GYtnNcMGNn7f;{1?*1{+z-ly8uZ^FOm$gQ3_-_Tl;n|n zt6Y+$OJmi(PI z(Cv`V#vAkc(6)n=OsF)S2YL20sC&pG8uh9px>Y-xI3K`?lLmeB3pIPe_yjdahu?7| zS18`Qv{^uN3cdmW&`ILJfCDRx+6t6}{><9?%VQ8^vn-NR&~cKw+PMgid0iPHrNkOQ zJG|jhA;9t)3ATmkAI}+&ewFnw3SX8{AzdX%2e9&-%`aj;A{05>ZCzgbw))20G?~i) z6Z`wE_{z+&>c+s8=|1Ca7ii+b($;nRXc}e%Q;0Wg7i~2cxwBXy^-6@fpt}|BB{cm0 zHLeL?Cb)-nRv&HhUCboT@N46{$=^qyc0WX-3854V*l&$p6U0_PA_aq}O`!bCg)md5 zKh)uef3U2xfCGa7ne`}pY8U-oa8ISuTqRc-asm>AmQ+^dHLrzH?s7a?zNUa-nNtnf zq#Yn&jY`=tm}&@P(CjmD^U&$y1{%KFe#wVbqG$!@uT znP=cVHYCXSCSmv0&=YrkXUyeyzxLwY za3s1JA|gv@MY!C=G=8O<4RZ(u@W5?oS+;EqBW1mSY01w|1`pO$6)L1O2Fy)SQc#-S zVugRjY{|5z!1+vFzb;7X-``r?!)*b|;;6h=Ifl+d$mnT3`?tTdoDz&oG4|ojrs8BM z-3`O=o?i5}fpis0cg@c_l!!9I&YqQ00B?3>utc`ic9muR{9^*ip;eV5pDD+dVV;vK z;0P*EKYKI?29UpfB57$Rj*B2RBpN^$r>x<%FkYR+_~g0KJH4A<$FW4bkY|Q_AypXJ zsg>mPYb3U|*Oq?Yt^>Tl8fltgYCmO5d73kYdCp9=JCha-wE$gSUgSz~@A9UVv5Rv$0=m7?5{@%{JzZHOS#m3ZY(||nqds}W1)~w!BP~# z2IVM|m0kK6COjmA-W+qUj%CwOJb-;3e8o!luE{w9m(i7X9PiR7Gc3Jtj+u-!n(RvlH9tamSVSde#~--hGTQ0~c5E|bm#_-W zDl%c|LYG&yyt7ZW2*@V4{kS__n!55`FV2Ujq&7~@r%5aXG-|$|_ZMW~IBX`{`N!gl zxpkG8V@0S^oQ?^LyAcl%?EQ8B%gD;f*seG6p-+6qniTHjbz(poI~9{mQ8YXE4 zO-nQb2^zw!_(Yc6JL_2(uWy~cWc@)+Fk}TOiT0O^6-r*tamAN1mYH7kpT@XrF##}A zoe@_>Lj~U~tx8sE1VIC78D8Dmp7VXNW$qqnq_^=8OeX?n%R;#iVb>$8hE?9h_qtaG zJGf-9>TCNA^y}LPR*=ca@W(tG9D9R?ELq0BWlPdfF=P$L z5Jp5}UuPs^KPn?MreVGx=W*|;^SF=4y}#VYHHC8jow*XZuY`IPI!ai8^_G9!l-tVN>C_R6>Ih0a_-rM&0KDZexP z+W;i$xLUb}ii^2?v5c_3CDQjYGqgbW>_4muxo9aELDO0y^7!9uL)#0gKUp1JkCzuby`wR8_E2kMP(wfYGXAymAXP;Az($_=|v}Pw408lLhm@6M$ zQ})w1_H-fSRiya_VqsKh>JCZi-m}m4y1|l4ly&@*`;kIx#TfTXO$YhhwD-mV6F0V; z{FiDu-F?%Y1rwbOIhRiP%##*%uN2~fC0%7(;rMXqCd|ilX)XJWIDbWrBE{wlj;Z); zM7t|>*8rDZvE~1{Loq_Vn9yj?JY*eC#7IHIU0$+Z{CKA3zj_r_=RrbKDd!Ajg8rHw z2rt<<1JeV`lZwso5$%fjma6*nX{6zoDf_2)hJHKNwULu!Ls!HMzAQ}>+^J0i^=0lP zj67>F{<@}h`rX;vUvp;-M{-=m{wiENCJx;!FW-ZueHRQ&PfgLxwnxk?EZ>o?|*t%G{INq9J*xuCyyOyFW7+wfWPVu=A^GL*_dMOK4jOoYpLM z;v}PKtM{3Amp5<3ZOLwL`2N*d1)At4a0N5leo7 z>g2^D76dVZ3NgpO?!U9)u$20y`)NuDtFl%6#}N*OH1a%Nt~GR-4j>Nkw&BO~eBti; zdtU|gv|IyUbw{7Ac)oS*rlm~hqq%;t$`SqkP#@qB=;u&PHUI->00&281F9VE$XVSN zn&U8d&$=nDa?xxQiTH&rk$amqz%v>vZ0NxEJ6_X9D^^SsFhkQjZWru`444?_1ZUIW*!ZwAITG%JTZ~ zK1+Q0U56HhqO$wIh)U#O(EAni`%Lk2ADzH<8aLyD#PRPn7l??1bl0@W4EU|PP!va0 zeq;hFmYMf-rd^C##duBi8J;9?p)ll~vT-~HZs!&f0tAEX?2w!*x5vLNh=f-5`w^!) zFX!1}XjooaOzfWW`>qGthjW6vaz+g*&iCgx0b_#At;w$P@wt10+AlY1;OgdO{32x* z_-FueXfqJ7L^>niLHW>aeFzIAnkjOQF4f=N5sKJsYK?UjdBh3rOqcZhVn@&y-%P<~ z2$cVkfL7$&#t;`{!PioVIgEOp`jW&sC8HWlF0j{)zC%3o`<}&JdogGW{RHWMhLjFc zTNhAO-ocPGUn=D#=5pt$no!~(LUxO4yuW19fGc+ zAY2kQio_$y)hnw9Jv$>NbwM61#c4A`;Jwy=>kizqKr*H;(6KZAVTaHTPA3Or60h)9 z>5fP2Y0vt`Q`o`e&YL=^E}Jq))<<=QeID!~lKVScXL8Ye0X{T2VLI+(86~0Z5GrpDLm|AAmw z$^t)a4LmuJJTXXv7ymiv-HrKsv1Z~>;i`y*CpIO4jbK`DdBj8({kUFiY9Zh;s5&%k zatT5-pg{+IlNAMv*Hp~mGqk}Mdhhq(eQu;W85!n8u#4Ud;{%hn&bJ{~(b^JRpq}>? zHaFhT`no`t)C0;o>g1mLb2_E)INKi>Y{+h0RqM{cT+65MwVZ)z3&oBm<=o-SEKmO~ z<5g2ADyujr{;~g5T=Av*wRia%a8=SdW=+mV{0dq^b`Fkt@$5|eSvus$bb`QnAVhrH zxcftNC?7LE6>fGQXJ$h-B6CH66Y0_-$ad;d;jDn)YiC`VOCRr*Dn_k1wm7ap9<5Nc z>Ns_ian&7ubuH_2rL9L{C7i+C?la{->OO^9Y!R5134Zy|0N^TZQXiNR+`4OdD(;T# zW#CdRbVHSY0rcg5W@AKHSjlK$E4avY>XUNl=1LMgCW_xIeYb{+Zf-vl#qG>{oArv_ zPUQC#Ho+91Gf6{dnz^ThWhMU#V`SCs=K+w{bP?E1q^0nV}EaB=dhIs?b94=9p@p4 zO?JIePq^8(b$-9GihDu7M*iWgIF}#!s*cz4z%5%$W_~1H)dW$E9KLt>a6FlZc(96Q zUJrN{LzLf7Uqmgp+ciXO2uJdvU=yz?2Gcj8uK%&-Ii?H9&$QSo$vEQFf_A|H%`myM zifgM-&R61w$<_IM%#sq5Vi|@+UF%WaE>FR&0iRD$s89dxqa~GxNJ&yeajmlOBWdyW zeh$LJVl&PO5f4I^SFVo8IJIS3VcQ^tSJXAjOSSJuQ06@dZJ<+3?U63?ID(BD!X3fJ zRj6_B&D?!s!Vx^*M5Fh=vyoFX3V(#R)^Rwwczt_CKza%+1L;@z%9JypXLB-2R}FfN zm;AfqS5-KE4K#@VzNHgcm!c-4eUffdOtse>PPLA}IepV>*J0NU6Zs0ngm!)n=?&f> zz-sLU{1TL{BRQT{`4O8zzx6pdUlqI2T(4?zo;##6##r67UVxCkp|KpuQM%Ce?nZ-h z{*9Zxkv}l0d4|x1LW(3KXL~D?${#Vz#kxT-hPlr-?W(~#Eq=vRqxN{arN?qJ(B!Ox zURhqtGmFKVP$9i#IALs!rWc4(Vp!E*I;#>WAnW;j^@}f^!mit5#a@_Z|D;2cTuW(7 zqI)>)P=9T_y2=-D5(<#3y8Du?bK=i8I|Lkl@nH8a?mV6}(mWtK#R~qI@~iqekC*~( z2?r`Z)Tl9w$UB?z(^q*KAjF=(>O>`6&)~B6(8)?^^|5bUvQs=bYP*Af3T>mN8s2~6 z2WNsYr;Z!X6$(R5oX`pb%Xl_}0n}R>ORDIBAREIb2&|cH@o0~bd*+=<6im%Ma`7Re zo57xf!*p2A7LAUqmq3TuThrGnB)T`{WYfg$Dzi|Al3SF5z-$&N+XqIRhPYFsLcHOf zRa#}q{L{Ff7O2bSg266_RQLqIq^(96kOApG-OfrxC}qHdW!}^d);mSFDs28)Focq` zE@^c9eS;_pA(kYvQVKulhjo`MSG?lpd@luAv|0k4|Cf@kt9|Z)5^XR(_58%GN5Td* zwWN8(xatg))u=~%(d@9~q#M%Yu;;zL8GHIbvUQyBG0Ex~W28eM8baUaJ*&|izKEkb zuYa28)V5wIcznI<3Cn=~Xvgg_G=0<2{qRA}&z60_O0Y(_u>))(FY{Ao3e9zxgL6#Y zTS*?mS4Or;p+vY}4exNSIb`Kg^=fyWQT=m4Q#cXww6u z&f(LomA5n*rqf&x;RX2lNy_h~bUtp%GbAg1k1*MM)D-Wrf*6xPFf*ch1zhVdv3=KJ z#!Dgq4V_%n`r0)%w?YM&w~T*725>WOxnYa=QgLLcj~50?uhfu5Y~B zWxeBTz``A<6P?UoHE1>LaMd*H!Jsh$Y9e13&S zRNeE=a#PRux&)qq*kt#G2iJ+tw9<;=I zOVuWKGtm@E1ah(HV8lgd%i&ee8&A4jkJ{?WOFAXqK9OC6{=#D#B)4lj`0{|2U>D-1 z?P*tS8^;wqQgJcsyWnQ}V<3#?h58W|WPv^YKs}MIHa`Lojnn@i8eiy6LXO?HdDz@U zB0sSCyLNX3tqB7vf8LjNhu;Du>easkgC#vcEAcS#N&-YwG7fu!p#-C_GPL9U{}JA+eD z&0CAQDDtYRs`Y4J$A!k{_eXU1-+V*;yZ!!qGLs@ZvAi2wf=iZp*I(amUFo&?%iHD-PbK^I`5OfYT@56!dEi- zp+zY_$_n|@|9g~V;pBpJgn{>a3wJ8ex=>KgMefb+PhG~Yfq7`1ag$G}^zC}z6R1=r zM9MW@S1073OGPRnt~AvqQg>icTwF{Fd-&T?D4@-X8rC4@u{e1Pm%JBvGJ04meh0smlU|)yT36W6Ru6lMBGV<4yF7aqy|!vKi0xYc69L0&8CP$we8#H z&B+#V84CWG!+8H`e9c?K7vbF8z1#17I^YzC3KmwS)x6~S9PxDgK1DnsyNe=qNxtC< z$7RABcr>F$Sbx{U9$F2f}6mEcmi24gksDC{U7 zIBj1P=bKjoLGX>fu9XJzd z>-Wpj(%qcGlAA78d?~>1n_zNHwvx7z0}Ln~TeJTDc>O&$?UzkAJx#s)v;Od>RO=Io zWe4gs*ua=rTaSEgowBn9zC}=PKQwRv3h>A?OBD0CKT*5S>b~c$nH3Fg-na=K>KpytoJlwr& z&&56l`_iHSJ9cLkE@qQKI)AP;map^PB=>9H!+c<&yl7hYy^RONgrKL<_Gy{XSxCd! zv9F^MYaHRFc!n+ogNT zxddYAH|#4#JQ2%?w{o_APIo*j3f+CxrK9g@Q=h@M_|8zTgy=APz;_&tsgXmy&Urpj zkI~8c#!oc)m%gBL{QE9npijy5CHJt>fmq@oa3oXf;~4n_~(vl%r06M z!pM2n-S&5bE!HLUb$9MBkMgK{P&+)=qmB`CY0$TWH-Sks=5VU3X2+Vwhuw+%~Jt-1hJ84(@&2!AMV>C(s~? z$p;_+8F;gZr`v(B3WXLffeuEBLbPH6SGwUjTLQjU*m?3@iw+Q!suUnPXIQP2KaV37z#}quxD9@r;W=ALNb5uKwkBXEZ7&0+BC$ zcA>suBsrJ9BkW5crwzRR&)b0>w8Uv=WoKpoiKETliO5^RJl09u@Nw}EC#)ibp>jU) ze#^0DGHWl6C_6S5`{#UbVfS;Juy_B&#dLlrV{<>M4660^&4ru31w@d%j7V8-U1O$^ z{1bl{ntY#J+|CZbB>ST1Bzw;f7W%3j7kVd(jr$ULjC(f*V0}4iu-=X+y1rn1y58&a z1KioqcV~MBDC8T6&@|m_<+p^bAx!04rNVJZl|SAFNO5DRtUMx}Di$aqJcoApxs9nyu7$wZjQ9RD*H>43zdGpuF{+u)Y zi=X*hQURFhr)A9p*EYGMJVtf3_*FS*otS0HkAK~#6d`g~HM=FS6}2sAek(L;SyL`O zjlFAttbeYe=93b=j*fx%w63R z5(RCBO)UM2(ZCbBc)HiDGOH1Kj(m;nkh&d?vf66i#KintfJbT<=R{s8=Is0b_5tFy z{ig^;tX6I;8wA5_R%mLUlq ze69KoYiV}0eG9B=bp!41sxg1Vc@kou2x`BB1lP?M^p%HOWHCOf@v{m4YbyLTBUUiw z+s8b)SafB01^uy0mcGwf_D(I!)@eZjp|6+=`VG;jK~()Tm>&-np(~#sOHbDXfaZFh zJKWcieq|Q%q{Z4G;qFVZH?3tvc~c_z`tPap&~01r zPfKVX#W+_Os<3GPFlo&Lv7DG8b98!D9>=z5O=1B*f|1BSra0R?TQXoG0>Z^S(CY_- z{8!?Mr@jW<^+W2K--^4`9Lfm~eancXji;PEB0fyl6D;`GtZU{)od5aTE;ce&5zg&d zET~@dW0$nT8}+2ekIf8at))xFYP0WJ?mzKMe&{E@4nrU&XiRTmDASWjtbqHt!i=_o7>d`{( z%34P-ZkEjU^s4Bed%AwH2#!$Ovf}ZYyfjGU}u8b=?Xf_*l_K!S1szxP7?Kp6Re zqnq5iCgdpHPaq7BoI&5k`nGGWX8)jtr3zqf%mSA5&S=TMM}6k0Q)PS3J(fy&&kcDW z@ys6N2#cSYYMc8MeAGn~??;XW;Q^MYKQc~>m~nMq;Sg>@STvDpHJ+{hwe+FS$KgK?7*Uyyx<{zq9wby|VDR zyRq@PxiD??y?1EzzP4=ixqaUVzI+Gty|V*)Uzr1aZmfae3zPXKU&r}|k5==I-nR4g zUS>v3KK4cpA1sU-!8S(qo~CJ%z7A;;A1%{>-tW`Iz24DE`qo&KqZ0k3q^mwC_j?k z$V!8YYfAfQfJbWpYI^EL|4h(DJOcHVdc5y{2kU;NhTmRUiWj*c4dY$2vlGg<;YqWL^P0Q6$JC6fvV!)#<%@~_X{vPl^b2MDSgt%O_tDBc)rMB&vz1d* z$oULj?=@WX}qL(vIl!DA)AuIAhKT{zS$NrR5FL&v6Oil5puBu|iFUh-^XMsjEv^r?a?I$8683(Cr0js5 zuJE{&xh%Q&Y3Y>1_-pspm+K)Ag?{Y3H<rY^}la5P?_USD|X_RkS@Wn{(FDWsyIC~+5m=kXd zkFN*bBie<&y_-Knou@s6_uK8pmCBS!JSs%xXVD&>R>Q?t;Lp?UrDN+iux~wPD|$1~ zIKrHgS?xXI%yR0nnasFsKPqxC5$0gZA?CUd?Ke!dkzW6))v>Qnds7>cY$EL_k^C~d z&QMY%^~u7H$rtXQ;EphSWuE6Z;1B%4uV+RRiZfep&{A+84Q|7sqc-`^PT8bmUoH#h%NK~TZ@Bjchfhr>>?*|L4F zNTT_hP?Qcuk)YqpnJ!F`Yo|27m*wXNeZ}`JS7?Y8@XKm5uMB60md!+pBlW4|S}_SC zGECWMAhNaC0bUm%96BQ*U)vFqZo%u~*E(fj>#k-`sF0WUVtu*tH@*#S$JQ^mvjU9} zka0%wd=pB!XKs4qq>-@~7VH}ou$@_R=mP<4?v_Z@`T2i!P+C?+O8+%NRoIrPfuGu|+|40ni7($bc;diG=B?jpb?IrK}LzTlc0)GH);steNe{`8IP+TYII7N{UCtMd}_x z&Qz$2Z5r!Uv0!tMy8L*F>|}uV+FV_s`~GC+@-u=_5ACjCF6V;~g8%_&dZHzbC6y$W zDKne)g!m!SfEIxOphKACu)3}`q4-qV5WJ&P2 zne!Kjdp5k$S+#c*I-5<*oPhn#1eZB9QJL=ME}|P!m@_2o2FY}%a5+1t(@V8h9xOmJ zL$bERmC(m+{qf1hAZ@U;C`x!-s4enNSG)q&D0vr-j&+ zoMOy^2KOYV?#An_EcxKGhHs>&6&Vcn0p!X4{S3(-uhdtuRN+IStb24hGX!oQPonx} zYG`FdTll@`oPOh73YkNqHeWtl=_E9v#8nMO>A$_xeMVb2Ss^ZjHSlOo5IdY@kTB0j zQXgEctCCKNIXjkX_8=zP>N=SMX zRNh6x&xY9W#!RTvOx?wZY21F62995zxMh3p=mvhKJXtwduRPqs5z9l)u~gOo^6<9d zM)!5ZP+c5H1fTe3EpAoieJ&fqq<(*qR@8T{#T{)Fp&4%;TNXTI%#KN6Y0>#_h^(mO% zTA-)?X5leg(7i5Fj>hOj+Vaeceiv zFM+;p&v2~44xF+ireJFc!Csdh3Ky^KCyoXB;I1n}H~)Za*?A{%`h+uup3@U(bnq#b zier}uBa^S`Y3(J`SQEXDxP!@Ck`rfDc}B|x*PmDZO)0%qylPi+F<2BcreAX-tmZLr z>(7$J%^~EjNd@FIc(2DHu}vmow9CtKY{Mfsv)q(iYEm47_8`9kKTu;d@PhwBI$j2F z{R<=+-Dq)Zv-cKU@|CyNQeGw=@#PLqCz{brV1 z70Q+5##y%ev?d~N!`m4SNVG$<@2$|88S|}RC`jGJEzg;0z zx9(AW9IZNyQy8}_i;jte9#Te6$x_ZuW$8xfqvWLv{()H!C-G$1QdeD+G>IPk`UIT_ zHb{$8t#PklfvLh=gQlPR=$$x7#%w{!UctHC+nBttDI`{lB}+k(d|KIy=}l*>U{^Fc zn>MpDTU0rii0&veS&KYVEH)#+5$dhRmt$JcE-7mqcUaxcTl-@ zbHa{TyM)T0Z>e42_L-%}8HIMZoart1s9hx`j=yY8+K2fkYS5DLuzu1I3sa*aH3GG! z|F$Z;SF|9q+YOXkCENV=xLW38b5FOydvW(lwVfPOWb&!x97@1U;M1VW5A8JeUrkAg zw8&9e3q-u8SWRe;6NZ(JzR{zKU8f-~P{2*`=uY@mmN=)6aV^qzVZ@Pyh)Q@dYTWV; zj_Ze-?Uefz@vnCrYZgcLdOdEQg!K(9IFuXV1M>P!M#?3<+QH1b?*O|@_=Cd+Y5jWT zwVaSD-|~x-W8=-T<_~<(lQ*3ZG4TLEY9=k8Zn_28Wj8e$WmH^LoOYL&LQmj%-KwNk zH_BIFGAtUe+%>XT{j>1e{9|!bZ7f)+Su#M(y$;}!7N2fzZX&%kV(-ZN)Fn!nJIo{~ zhg7w?gOO@i{hq4vs#UwgCFmJdl@x$&Le>DF7Dg^A(Z+Vz`{ z*pD;)7i(8gHKr{4bR~GOB9og<)%xE0XzQ0+uPkqFwvr2^BVtK*nk*oOQ^EP2X%*{7 zxnndq*DXCMZr!uy$jHiy4Egq)j~n~k9ps~LvUqahxju#dWQfl@iPFV5sH1__*@C>B zO}UGE=#-E*3Qw#09QT_uvd-((QTiU^OWM2;Pdy^s!y zt*dG73=pXdVt1u*eQejYQc2@;X&WshEHxY@H5g@&pb&Wbxv!}OB}5oHZrBc)anoKP z4dvq1U*nUT)5S6f&d@Dldt(qM1N?uMdhET(Nx}aT*Fl2PotTABnxsN-VRr^o|7@Ze zJL4++BO-A$fuw`ax!}mG2C>?TQM!1eH}ZTEsbg3iNE4W~nG53Oqko^24US(`a6`UN z3JQ#!C62KxUn}IAskw`>{&d#>G908{>95K7fL$V86Di{MZUtM{s6`ED_8|QNOk`X6 zEqT*bF5$y^Q$2po-I_~G7=HOt?=}6eUBBXtcG2Pa@h~hr;r*7)Tp47Y zD`|QUPJwC3x&#(_A_;16_|<9E_dU~7x(%S6E-m!OSO)F+vf8OgG1jKPk zJcb#emGTKIt5uvRe>1MHr@7*DfWY~5$HEpbK6|11JR_2Fvm18yHw&y zfxVo#U%HtKO6+$AL=@v){kWZ>f{{J?`6No&3>*wrxCp1|*_}+@nXd^T&9(WN?^g0o z?tbz!g@tQRiCd&1j9h}AhUoImG}u2)Tw`v|S-@;iCC9QJhDoh{_rq+<&sCmAFRPDy zvx{+Tjz(Ajea>~hBYY!V;xJA0YtBfQha&`rcOVhR4s?qnfe8U`JNS3p<~l}KT=wz4 z9M0o@r}5nSR>(4d$7KEb<)R&d$a?9htbtlu*_N`!i8*0)siggv94bZFpd=~!m`L$? z+<#Dl#s(Tx-1{!?E3Xt2d6VzGP?OIszohS-prp?YF9MDWFEn}!Ao7RLjT^yyFiBq_ z7|@#^k;N3G1A=)E7Mg?+IVgdL#HZtYJzueLlTfiy13!;(qacq_JuiZ)3lF482&ln; z&(+ez`4EI%D2i5sAD<2=h)*kCay~09xAi}9{om+bJr=p^>rnOVFIk)oKtjlV7VsY$ zB1I_j8nron_#b-m|0^n<{y^q*$kBN~^}2>BJHP4V!nLwhM%c2 zp2yKUg^S=0F&TbIKJ+Xv26Ubmh$|#wPhcwYX!h~27RI~V08E}Uso5*;+ee!Hw)v86 z@Aj*5jY;9~CorD-O?>w*k+96y<` zCCW#;tF{-`0~jO(8CtmR47TjA_Qq&DlTTdu>#_AutA?X!U}wxL&%qAktaYo~#Mz89 z-Vz5F?~fL;eG`u++SL+Z$QuYSV)Qm?Y^=hWJl;6!X`T9bZNWRA#n6+4nF%b@`j-bG}*wxBYXcSvA%2?%SEw?W*$#ND}+yX8|Hn)A~6Zmr|{Kk+u~QFhCkr+N2*g8hC23|+jMM2`dHQzd!JsH;(WV z$D6&U`Sslr^%WiayC0nK-_m{W=399iBg$4(qOQ_`hRc|t@YYZ$%iZg3gZM?H{gk93 z*yKv@8NdzmWbOoi)V=(I>8H7<+VYPJiVf;`6&h|^yeV-9A+WM)n5lE&$@qnJBF08> zI80W#C1;@~7E&o8>d}ltd;6Bt#3XAbt-2dE{HPqwF4=Lj9|{{3vK6>AOH)60X^AJt zh&=IqR^Eaq(;Gm{QV_$nT$FJ-%_vzvM?3xDxKkvsjjZP_HVYQA4o|_X6kP#JQ$u72 z5vyZQS7w-i@D2jiu?Dn3&Qg^H4gk`J$?p_KK(wOhk#gL|B2JYLRODRrFm!SEtAz1G%M`iqHN{+o#q+@6H|e1dH>5M&pL!EDb+i}Wv1`~I`)Rk>uJ|YwOZVjq?_v(F=m1v5S6xM9-5cXMMycMR3*66Yh!Jk51p4c_m-8i1ivh?OW=++XX$}`8SIIMLhB!2xCI+?O=(oVZ& zjWAU`a&?+vih=JYOkcIMW0|hSnB?e-pEs!5od4`|70-+fdSx0Z)3BljnPSu`OS6vV z1^9~~a*HvB(S)ygeqQ3)2Kt<$gE(lyn#%^c``)$EXT*GRh%e5BaRW_ORGdv`coy5Y zWcl|yt~ZztOIwz916XNs!Sx{Fp*9IiS*u~IHNp6%sl!rkw6R|gAjFWpvtA?t+e-j7 z<_H5Mg)NB%bKbyn-`REO4bn4QJ~+6dsV{`qv8teA18YoW#7zB%64i8F^a@&m$;+tU zjAj;a&@^;eq-sa;^7&@^Hop}@gT<0dC0IAC+vWV^?f}YuWlu>-hC00poOrkG#zL_Q%v`6>E3ix?7$uhg>X-l*c@Twaw9&SYo zQ{&^P>#MfI&i@N4m;)1@MBD45;gYY7SiKXhTFq_4$bx>xl9p1_^|XN}?!44UL=Qvc z7~k-Lq+gjgY+iA0bLxhFqPZNdkIp7$f4mn%o$TE*tiQ&w#S(@eu2bUC^&M+M$zw!P zE((9SL>{;qaKr@;-BNA|Jrzu^le~@~Zw~soruAc+aFTVVc-Chz&NDLAQ6oT{(*H11SPekpPu=Rvvlnhh~AJ`yA zCx4RvUC;ZjBkm}P%}~Lu-Le2>Cg?&Q0VLA9uxV{EB4r*)W6^$#*8q1SAveJXwW%sVo5TlHj&)y}ovu=CO~d`DAjw z!uX(c{tc?=YS&7Jzp{WcMrh7&;X3B`-eR-%e*K?+MXL33U~y95VP9`8rw7O4?S3mv zX2~OFdpw}=felC!C>2utVri(>I%%xJYD1OZA?EDpx$rwh@c31+*0W#e+CI6nj|g4U z&lj1`WUa2teW+gXPHO&$wTvFl^H)A5n|>UL*`<%GQ6STfIwE7R)$$6&8FU5r+y2zw znBy(oik10|N6|~Sp583Wl?*1B(lGo1aM`m>Px!qc#uacf_m(Jv&srx*GX+dAt>mQ# zPHlS2<1&4!@(Q66tU_wMKV4p(KHc)*>`eHH~je)c5Lqqkvcl%{h0xnG+eL zelBQ9-zY2+Q>1@*v0f1wW%5yY)lmrSS7dh{bZDGzu-e2A)BrF`8DeIhDqWlaJajmg zrUPUVWMKp;hEtbUx-EmX|J#=iDgv$=&g{=Do?EunIZO|5;R8xO-$@2%W@~B5=n(=h zV4$mpRl28<@wj{lC64KXJu-G}qYw}6+PqnlyeC0s9J!vg1HULjSm>;nmGW>60<>PYtYIe;b|MX3gT_!Qo*xvX~+w& z>d=@*^3pMkE~E&R)2~~_y~4Jr%Q--;F*Ey$02rC~zv=rp%3nVj(D0U~&3ZvM!6g?b zlm|>vZDrzV&FwQTB3Ce^;{3}W`L0ZQrS=20UB}WXCOqpT#_s2l;O4ZDjkW&y)I5e97B6_=PE z@17*h`x_&}d^Yyax(mp#T?Xa~Yd)JLZ=GT;O=j1p{y_^$f6xMM%dWT}SROTUgBefj zju7q689$KU_fd7%+*@}8?W;aADvn07O~H!>EQgt;31$qNGVxCgBpe81EaZ`bvr`Aw zO1k369)=DP#ATl0^JD&Xf$0!yo@!TmuN1nHq?;D|u*8jZMehh4dK31LpV6fmvbZSOa)0hZ z)oc}9Q;Km;&EZQ3b@h9lT}v<8J)cljX*ZCE&3YTNJ96^VJdosRJ_9%QN%!kW>}4!&M_`Fs?~Hsw4eBGJhT)2H%bFfLm#e9NvAF)&XLo>RC!CH{ z`H*q>MF4z>@etod)y^|6cwLvQLoU^KaOcP@gv|$!8S7!kK0a$?oILhY{Mx%zalgS< z#Qm_X7r_zg#2^qhIbW=B7v75&$G)HW1P~{cofCFc5+QaO4T|Ot{8$q)SI_0n5LE&k z)MG3s-CrgXf+s`h(3x!GW`P8wgumX3(Wsgln0^pQu6>BiJ8WT+{`UFtkp2o3gy zVI{z(rH!wS6%AX59=i?^odcjZXyUv`Ys(_|pw}1CIQFea)519_< zdUx(XuqWmAkj8YUmkwipWEIfN7nSv}jfbdMv1i~QpOXsLhROK$H~-x3p?NSvJZ66@ z=peM=I<^0LXL>hxw=*|ar#&|}mm@b9N3G!dyE*XnSs(cJY7BgN+y=h8nF3#(bb)U! zhQJqxzv_MO=j%V7_t$%0kJo#_+v^c_UHymC?s_m{_u+ZeBJO)PEB^7USKRw*RNU+M zx46&EBtiu26bD}nihCY5)ApdsoT%}sBlOMx{ip^|iiAPyY6tT3{+8<=j4HceB#Nx7 z;&8uZ&31H+2d;8|&~%eG)Fg!dB~#4fOW^;Y4iqn>BH&^s|4}9$A7?}w&Xl5rAHx1m zEfI3X-z^oPbszfS%6}{jaSQ1h?f*J<{s5^53**cGdkX^raNk>yA=2|y_1E09po!l= zqq`06WCyY@l#^|ReEPCjf4wztF*joQ%H7rmfCswC2m5#^>|iZ=?u zeAVd_wkDGU$c3&0J@pq_+(VToZ-(nG^5fL5T?CTb-tliKi|_#Jy777SQdc;R*NztA zxJAz|5Z1wCE?>tA1<=H0litdt?38g5+B84S@oO%}E{gPur;-<`FIMf8Huk#<5Vm+5 z5DGAPr>E)J`RNd~xr&QJ%fm6_k4W$e7kxN3wF`%bDta04U|m3;EM$F>&e*Xud**~b zp{eyHqwnC;ShRQqk$coJ-Ghnp;9JbG{@?l*Rp<-le8Ob;f7C%D+*hAr29qh{+$fA%CO?M~V%J>7y&Ipy-F3w3R zK53jegnQWkvF@-=X^O#%NKYiH^A8%@bCb>wU^ZlEtJt3MsC^MjQspSNpM6)ECi-l? z9YU_4oh33!KuGq(h8magkW=+XV*3!UHhr5Zm536T*>g1*E*;~M%!)OISKKl$*AmQh z4=(Rq$$?B>vp;qC)!M?Ds6o64*a5G{V?f9lUz}-$*Jsz(B3LnKR(h)2{0TL=Xy_K} z<9p6W9$ytblk`}e{Z%xVRns^J16k<+TV5kg-l==4Al`A+XeonBlFX-UmR3Fc@8udh zhfta+$bKNu!cFe4z7Ox@#7r!RCaT^7xKdDm6|<++10(%y<9o>Q#8*M(_kNDX=;mal zr7F%JqJ8Iq-T1vL&+D14cpb_eMGU1tzp-|doiIOr&9o8^4i_)U3P><(vyoFaG!s{G z^Pd0|_#)?#sn+i-KRLV2t*w*(2;)GqWwn!SO^4_wV1~sQGMh{af39 z@&bC!>$4AG8}b-gRYCkDi;OC6+#G()AtC_4b@=&9zltx8{Z3iZ3=HwzFotAggh zFp}(B42x$14=lnyBy^UFeSsb!ACU4S124GH<3$bBG90%>6!v&mcMl|g3YB$*CzMjT zTtgK@-=K|Disy0r8EySy+mg6X>3J1ZUtv)f{0unGCGGJY8)C#B93~|V$U!VghC1mcl*zfX=7~*NTa(BBv)aRc zu%A=~uKcm-idfIJBLCgTRV!hHqdfe!ps2U-_DlDj`+oGVMO0F;{PM9B)kh9b+t`o+ zvb5$&?_mWirh`R$Qy26e%9sq$v^~q`ZRGzkD@Mk%VF+Tuh9DMer>2o4Ea4k!Aw0SG z@$mzF7YJx^8??464td6)RthCU$I=P;1@v*#a=&?+~W|v#}}k4qLWqoKw^nuBh_3O(N9w^;#x-8n?_(tIQjNMAwo( zWA*S>1~|Omp`@6(oK@Hcm^eAM*Bm|p434>%PNxXs<$MV+e$nQYk)>2FxjaPPMvA?7jpH+&!9pX-2`4AZCU3!TG@(Xgluo$4Je+#%LqO%J_>-w^3gS}u)~6^sK0MBlPdUY(kyTE1?rh;b9>0I zcfB~#N$WPU8`iB>55TD)HbeDgC=pCv4~7uBe%XUF#;SZ#>R1%D};%>+PB zy@0{mWTU@h$eC)QzswO5<$3(7=FytXMRsj^*#Xpr)#$Yc4leg^SzR&pI7|^jIP>z4uQ56(aff?wu z1y*NCw|{i8RIa22`lb=gXVzv`tJ7nddtnQrWY3z$*~A`hH9lT?PpT6+O!Iw=ROF+# z@7(!p2s%`bHM1-H({NM=VAg>U19SUp&ogh62q%$Z6&lf8UtD8}cllA~QfAB=!N_}= z)D$9I)6-u1^_^R6s`5g8 zke{SSmUbZCDAGmul=)*&AY&j5ipZ0x94(eU`mCyL<;$#7w?H-ro3rR=Bh70HZqf<| zSt41yb?5z!2Qk8%JaKzzjl|QIf9M833q!)twwYc-3?oY$*2$X=w^}3;))i!LP=HKgWEwI0@&<0C zJ1X1?DTHQN$cwDO?3bk)D;+{B{9)1CSDE0Td&0{ppABx%Jl=UK;Lv4Df@gH9w08tv zaL7P-)@o-pxJIY7M0frL`t@@PH)>)pdum;gUdunsUtCRo#c8`jSBT{t4+bY&5ZEzt@I~k~n@8Sl+Ns1B1$j$o;_;k0R_8vsqCW zEl1=H!>f#Uho7i_Mu~q31Ai&CFZmY6l#(f81IG5INZF6S{NqyCXPS|^3oNy29r;WP zLA~fuzvI(V2=tpZRo4vMKT{r+#=_&pSngu(vEXQ!BU{$ZVx}CSY>s#c;x!p`PhC3r zR~~)jm;>prsT&(LHkZvw<&_)!oqJ=cxo7xm_wwcgS=RA1Xi-nXo?=RwKcc)?;IXw#q^o;Nx1~RMQB(AZvA)FZQg=7E(u`>;d5!l!y$dpXz zll0vev(nGeVKA`@!HkhdqVqiu975tP5;KF$D(u3~7Jrr5Qe9c2ELo2X_zK!Nf1#fm zL+1^59+Pju5?w;9r_#-+YaO1(hdg;TLVrkz|DsOQ?qf`fCl{%0ad_Vv{iI{aH&2^} zKmbo77%yMU4%W}R*zQ4hzwv4pImuFPHkonl&kweX+|0Q|@~H}LjyAN=bsD%Daf301 zO*fhDRWn*DygZx7t!`+twu$G4{KR95Law8`%XxC|V3Gy5cR4LB9$%|?>98{W@w}r~ZAS_=G6wloQ4Ej=2rls7Yo-7F(&MJjC&hu`$6;ce6{Zv8q!!@mFRPIg_zejk4fy^-Tnv_8lBlLGVZ;ML6G zMHErud%3{A29pA=bS@s-uR8LYLm6G;Mt)*sGTKwk|4+!F+UF@rD3X5jrehd+D_ZQQ zVk*yItlW<}fP*DY?N)shQoI;eNH?s=9Qoqr8kiks;7hF- z@VncO@clUe98p#XM^tIUuTE{@+hZK?<25DtavL`M|4>^9-yCB6ySfks;34(pjt%{x z2>O3Lo>MkM0p52u7q{O6(cXzHl=i}13gQj^DI2T~#`KfUy6?th1zOnAzeJSYFpBb1 zB?3PDKLU^+#{!X2Vjgb2f^Ys^ob&Y2dobCzQmp?ljsLG(4MZa}VlP6TU+vg*Iy%o9 zdWz0y_iOS;-t&zA#*S`nEA8V4qP)eS;I7^0a{YYyhY_q|)2L;ztgOLWk0m$)hS#p< z3*C{BD1JPAFF*^|ryJ&)cqNCO=evFgOfZ&LJ(2g1XrM4$k`GaG8()59vtYS9K0P7$ z!esVBnFr4}b@jbV!L$*$!HBzIkJ*s&bq&+h_#+QYvhG$2S%XN4OXgw!q&{=9u&&Gr z^qG9Fk{cKPEyFehF0`nh*Ace;o$z+=*Ltn?y&YUSUQFd%lm6{O{QOA5KP*v^p{;`YeMJ*Jd{&YsnTJ{z4<3&o~A7=9^;>mDFYFF>J z*51>Fz=&214BX$T5y=TkWkzb24D5G!c<#U6%cqH1*A(mh>5|GZw3!4Y)Ftj!@x0@G zV?g`Jz7P3$WVG`@{j{M^G-r z9xahaa|6%ZvPS610kjr?<8`Uo-m#o|LRz8nji;>$-aLNYVUdCls|1%{2;l1Ujo=*?UbTcCrG&+tOff?+pamnQx7zts?D!Wv@HjW_#Cg%a*o6JZ}0uL#~m zQo>?-5%~c>g?}naBc=C1(?GVvbANglSAi z-{&YMY)~YhYD9>2Tek^bzO3GKIsa7ilKNDC$1N2PPQzGHd7wDdwzEUAebXRGO6j;^nj)I9B*_)~!6BD$D&N zBIq!#_;J`o=98&H&M!WLzZRN`yGwf+JeGStP7F9k=7_!7R0w+}U6LQlQXFNrSiOv> z8ln78)d+I(OFKJF2<pT++p2p)^LKHg(zl74VX6Za-Tz z%R#<~w&rcp2I)r23{tl|`wx>BMQUuZA95KBCc6eb5e!lBWQuCZKX8>F?!}Pia z?-9-@iyO^ed>dV>X``#O(GBARTwnrSKS(H>5>>odNruy`ed>_#+TgqBaJSpD5OOz| z7WrCJ{Qfo!7Zw8wUI{Cq3xMRztp-f2O{ZVPA5psRYbSDm{39pYy5G)4ihh*s;D4Q0 ziZ!TW!e?XZA@yklspUOx|7LxSg00QCsrE@My*AOupx_ur_CW=u_225z(4{g+^|xIT zA&relqcMG?V8+!|!*}`{VtHt!Niq5H%r=4>choouX+%_@Y0S!bv7{ zbU~?z0r21FE%B_~gFQx{C+yBz`h%EfkV9jnQY94n< zWc%NRA-BH~g&`S$A++-Oc26=i#*oaPDNIun45E+TzW)Fn<~GILGXFNTYWC#%S+6<> z_Q#y4W>_hsXB;H|Lni$Fe6qqmX}!8BRX&*nbek0(qI&gMA08Dca)bJ}QhPry)?RD96aWS1pR46_!#+o!x3PINmP{?VY5J1C@8zXN$DO z2r?a^owjwgty?O}I1ChCn23tv=O}b)+T{cxuVImLrwB$rL+BK%;97oZTD#{Z)r%f{ z*xmi(t;49m|ErhtifU?I*DwST6zNEZ0MbReR86EwH7Fec0RbUOl|U5fO;IG$5fTuo zN;6WVDAKzDf+9^52t^R-CG<04uf24geJ;+`9=XX?=J@BBV}5_$|9Kxf(eQHZEH)ZH z!M3Ud?dd;C9|p~__UV1h`7ZFZ)5&s^0Jq}sg&HrtBcG|{P>tu%S5lj#Z>K8hDt+`4 zy#drdW?%@)5~Sg9e&FdNjza4}rE1=J3yEKfMzo-SrY6G-D5uO8SzsN0RolU<%~eUJ z@h01R2Wg1%?k*_V9${>(;9aLJiIVDExrK_3)|ZY~c;@PsD%F###&P>2n4GJW-TZyR zX6oZ0lwoIUN#8Hv3PgDSnm^i9OIiA+!IJ}vZMkmaNqlr#@Iy}2B5VcA2cg1YDzs=A z2t(ICm7vubnRv_7J8-f0_`gl)Bavtg4g~8gos~}`wKU6>@IJZkOyRHygC+PBy7aEb zjcmw))4;u+q|)4tes+waCb!)cE$W}i>K@&&&(9mS0t}XmxqwsuMy=rM9Q5V2v>yTb zzrLU0oBYGOpo7pT=ike^Q&=+TYMw}pX-UxHLnh$5M4p~33&PnI9jIZ$ zlFfmHg3&zkFrl~!Mf}y`sQCzQ!zsbzlL(3>>oDxe`>aNWfxsQ@y&6wnDE)%;*ol<@ zlsIm}nz2Ht8mRSE;P*SN+5-$`rT7(6Gb_Hs_6^K6-isH#r|7u;6^);a=)2L?Ju$jF z6%apA>Lm}-Y7_gJXl0LLxLcDL+-p&q7aaF1DOf?IDoZsm*TcqTdqKj8y&J}#rI~fj z46fpEVoGh>RVh%sUr=fd&UlcpaAz5S;}kzd8;b?G`<@|MF(A5iU3XZ3hxc2uwaZ7W z`e&RS$b?BqgLZGuY0NHc%VOjM5J8+9?&QW%J^W%}e-Mz!@5sa^i7*fl&ww+ z&(ZKSJ@ZCez2(de6B`rsRSuGto-4w@4}Z!epa_>_f8Aa;Z6f6sBV3-|Hq(qF=~NwC z0o$$3@57sRE64Yu-U29r=IBw8!)jW%4gX$~r*fJ1Y)rNQN=}l_I-UtjGw`OmVJ7HB?RD&ONl0{L3TG`$Ee@i2K3VT`4_Kb&sdLZ z3C;gNEBshNiT9z*9dz}Kf1oEBxF5_5?s;f3+*5fQI_ghOe%n%Wr+=EweVdsU7Vj;-!pOdH&A3Oo%a51HCJxC1y_Ya}A* z^dVu5RX!9pwZXnOIqXV3A=Z<(*z_>p-yyju6BCvLs?_~y*#mpAe5N>!OgN- z{GG#7VF9m-xPex&MwI+iv+M$%F}$VTeH~Tg3<+e}m)|nLQ}1gf74H1KQ|HP0HktD! znCs9RvavB@TPodTfNQ-y??=B}NXm;Jg7)72}RaR7F$&AyqN=Ko&c{__(Rd#^2@GN<2iQKJ-AlkuH&n z6r}O-QKSc=FOmW4r=5M}8TBs9ZG(qb?HdF->DIOh<<$f(y}%}6BglB>dX`yqm1(?< z2J_o0`J>ovq`qDCX9hQ{M9pTlBB+C>-Q(E@0d)Ybh+H~^D}Fs&uEz@ZIppv{$Bt9+ zn{f~~o%oElVxnr@wInn|S?y+dd1Q?JD|e+4l*x@L?q8>bx|F*v%;}9plPbMn5=0X` z{EgG4`s;2St$(2vBAMqig=B$nVTGSv)*~e-BC?C=-OF$(MH(|*LD2h&R-CiN2lQgQhf&WXo z&<-cTsnyQ!ZI-NHfm^y;Lo8VuGqCN#E-PE_1&1hy^!YBKVre{dzwSKrlK(xch&XF- z?prFn#&!Be0H?4t&b&IV6!u(hDBAg)FN*2!Fahw7{Uc1cfM_#qp?1Tui!#!W-LFty zc#o!$?#P>n(CaX6)or78It|v=3^pM5%#3v^E{OOq9`9!`+XNB>R~nXzLLqM@THaNw zxx==`Fd|>Ri^h0HmdO}78DK5X-KYQi{ISCua_HPTt|6dvOf944Z zKIT1v(I*}oe;D|;g(3m$w=kCm|Hrj;jtP@qr)`~LWy=aEI171f%oi5$S{OP9RAdLt zeg_^`k3BvjTvxe7#Ubl3%fD}MG^XxnQF6^vDpW?=VtOT@`@zCo#Xpy)ivlzyUt(Y~ z)-dV%TT$cnO?f`4ZiZiND)2n1;saGmiPlo0&1KXJ5{p|BwvoXjB3~ica%hT84u0sQyHgID*oM-je?U4tgZXX`65Mr``O!jv1P!U?zW%YdsAeF zO7PrRHh!?+_p^Xj139(Lv>C-vxx}*3#5Im^r~F8_M!Nxk!6U`E=sZMh`IRu{OniL3 z79~Hjd^uWXe*1JWd7$;->KswXhmZW-E3g4svI$j@p~kv(zf0{Mo^0Rt?{+STJC*J% z<-Fq9ZAo4fCp2-4BrS1yFO>G&#>!j9l&7e9l^_?!@)-~+fAh(( zVR^Y!w_{JLsIaK(v)FA;mQfm>ndfF!>CK_u2ddz7bv1~+>eLB?|ye%x33(BiWXgomj>tt8qrb z23Na5-`(lNEHr!jYkOOX4jl|M^l=hlfXwZ^kZY$VQ5t+Me12?8nz2bxoKcUiNJn!U zf>wV;N=K;NaGxm=X?-+NV*LB1t>SkqI@Y+;f|+!(@DT$?r{O@v6&-vhE<$~R3(EOW zx0wj{fOQ@hnu3>{C+(XpV|QaXWpP_{ty%exg2v_1ija<+4)A=Jxqtl|si2c&AcdT~ z)2Gt8mE%LduQU8|oz+1*(5^PJH`Et-$Ft00L)o9qe0nFN?n`nbDsSz=D_t77f zvf}FY+uu|^)a8xZ_y*P71zT84^#EH@kgxZCp|2Q5gi(@~PeR4j^J2(SC@EEzaXq6h zW9=-ICt%S?C`}vJx&wEs{qu>KJRCTU@Q7BAF^PyARO5y|a@mup>|b4XoUTo-fV%W~ z=0U#1t1u#fLXy8xx)Vm7vO#ac^wJOPh~aylUQ1T_PID?tSh`)7kV$2f`R<}B#D7_% z>EXg){Ml~yrADR-(uX1QW#9X^`0QnmJpw=Lx997=6!Ihl0x9|EOGCOdRP%EfQ#3V> z!z_+H*z|^FRWl(M@0mv*sSgkRb*nl+E`LF$7#RK`{~Z22u*&um4u0W#diSfj&mD;s z|7Ft?3k9Y*hGLlJ%r(HRSn{`9QR=^~5kt9v1h)$~57n>#`5D8U+6~#?a$^IzPE!I> PV+ws8n0BeAL-_vy`qB$W literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image.png b/debug/accuracy_tools/msprobe/doc/grad_probe/img/image.png new file mode 100644 index 0000000000000000000000000000000000000000..5a498f5d2a77d8808d36d5c12e52c87809137bd0 GIT binary patch literal 11977 zcmd^lWl&sQw$WgS!R~G&I&|@ZiC%f#B{0g1bXwA-GFJaCaEq z`%O*NH*?>6zdQHeOx3QvR-IaPWIbm;`<(TxFjZw)EDTZ%1Ox;udAau=5fD%#o_z^4 zq~|%L6$6NXK#d^(UP8kI@gPIE^pjvxzwmJ=V0G0p1K{)rN42rL>rMBIN2HhJnx=dc zK~h&=SXe=u*(@{M<59aks^=8eLURyoZSiVZSfqU5*_#SIR)8I? z`Q)ldNNExVENvj=Eg)vCBtM|`Z$dLBkH=k+bn%ffD_;Lm$AAwN2z8xgATi^ z8Bw5iLAMEkV5(?-mQ@3~Uxnn*J5ZTCBAhf^Y-@Ef)LaPzFoSl%@)9maZuhaHQO4Q| zDB>dA`S~*{=CXplu@7ju|Faz%=)dQeRf~zZM17Ce3Phn|nEg!&T>wy&qrha*>wmn2 z3nOU%M=t=0dYuVe1xvGeD<*yHuanZot;M$F#9XDl5MApt=9wQDf~ z%#d3v_zRd&Eg55V5ye~+?Z0$*Kn!*~uGyKn?DzCACjN8}4f<<-xLT=fXy9MXE)8hH zfN`y`Gqym~G-tFt9xrms+u4tdL>=v%PxQ1qZ`KL=*brbFBtgVmr8*KGkb-nA)5l$;(K3>IVX}dEI~z6;^6pgF;4C9PkQ+%^V`b**}*>7h}9K zlzWy9neA`f$-dhpY@|xU)r;Pg+W0xtN9B9H21I|JO`B^Na46vS87^4Ww2uA0?fUP( zmFFycWnCH~eKR~ank0LB{Rcmp=&Ai0Ml+loUQ!spy6D-A#hcmg3BT1#%Kt@i#2_4_ z46ow0L1U;UhTF58)2<5KY8Be1AP@m7;0}Tz3cB}ya!nB+&Tbz)fe(VkB=;RV?L_RZ8@$To; z5a#hYcSOUGu4l7C9@STM%c=adg1aA%?t0<4HS%eDG;9WR!d`6~TMDAP1CREJBuM^{ zjQzaWt^?MS>`#RM3w=1;+Q+pjQh1AefTu)r)uobI zbmK)Q>!}71KVEu1Z$gs(jBzCpH24@>yJn|>sY3! zaP9H_T&>auo2bp%gR)rq$wHoS!@XG3S6oi{mc&vXzfs44ZZ@1v`=(+;=4o}R8O8qH z`Cs=YBQI1J7(%#x%FBHQm_oma70;ifw_44uQt&5@HAEHOt3Zjd0xSe|6XV^o%q zh=)IA+zW*94YIHlnq-B-I6Ass)e!!v-e%+bDA2nlqIT2@qKmln{n{>e;9=Ftq&a z&_-<54Q9j5suASkavU?tsvi}2l@-&4)8fX1_Ic`Sa^FOLq5*d5TU}TfWgRR0XnLQ_ z4!L>${`co0tW`>2?WYQ{$F5}gS;VH1b5UpPRRP2+VVO^UXf>Lf^3Fz$$k%*zmD#MB zc7y)V-2ok)D)W|2j~PHDY-Ol%P)atG$FNHe`*Z@iHRzr%SVe5G)lXBS_&o36s<~hG z?jzv@wl-DP(M-9ZyC?5!N9P-*@$#pLvNPJi-;!P4ha)#Gr$@EtJSI)RlXRyA+}-6> z#@HlH)=+Fo!Nj`qJ6%Sa2+R@toGptL{8vJmEF*BmAu5`8_eq|YlvE#Twn z*gBzZLsG?l`VJ_2N&YZYlTMY}N4B@Qyhch#&$82p@k^+;M zP#BIeXKN9+X!EV{etewM9=2xhq)}ExJbU=*l1D_y=3N}#`}Ts6g$Q0!R85t_sZ_}) zpFnr@NUs#%@LnRl)e-3-TS=tKf)q?M=@fcDCkfKQeBjI41}TnL;knEjYy`1t5Nsvi zOl-i+h%nHQW^>T4EgswZr`i4aHCpzWP&)_e(D3TRaHzP= z7<@7Ci{Zvft4@Bh$(SYiOdfM5>rHn<`_sk>5&mOgWhl(mu&Oo3A+oKvD}gX(Lg%Ea1lx!=nj=upgpX^)s2!yX##%@Oso^|Gyl z65|xMAcD1L*MH(cbX!#B#FyZL@dt8x=@~yE{G8A_Yyn-KmKMewW0`6A)Ng|L2+X>R z8^(kYcRbNM;??|fY44yL{6OFKgra^L_eae$p>*?`5r0@r6oy3%!|)Y)#?XHrCmdNgOAy8Yt{t0 z3{jj$q^s(E7we?JaTUIqEG|FbRW*|wY|(;$6lG1XIex86S!dcYERU^@v&M83ehA_V zCW57lxV$}eugsi!{L!b(uml?tkHPIwFWHSUO&hu7Oaz;DuZu~aldJ~TLRJn*AyyHH4J5C`>oUS_vLsUk zSD?sig4GH`xQ^M$!9yy@{^Ogo4dD}Xq3-xETm<#+3PjoVQ|5r9i+wGc5p8n#^IH^r z+Ulh8l4puEEX7vj8;kK@pfyd~HGNJ>k4t`8tGD)J?mqXH~plD5l7Rr2(}TbGYbI z$77kj$!8$(Ga1u{`D!WfUL-JpZCP!X?9Gj*DAKGj>a7?2ULMTxWFw1C7~_oZxpf?$ z=HjbM5tUYmIvRznjB^oKBmQo;?>zS^Fu^11mVl9YrRyzq8*6~<^_2qbybC(`$TkEA z=k)hCl3o@CqlC^o5?p@Jz}p9UTVXVIDJ&~FW1G(-88~Rwkrl>hNO<4@)#ViakjxAM z4;y?#iHUY%kd5oUu2K?NM-uH72^qt0VRS~r+g__dMx_2(_c;)t( zfdTlIOEriC8DMsX>>|rI;lBjz8UgyReRt-2KS&5X7fWz! z4PNo@$?SFbV7AmvMtIXO#%wdnTtUk=#PD<}dzK`Vutd(L!`@ z=!Rx+*=W?LjuJPHLeidh;4#3_;11rf!5mK<134K{xQ_dvOy`ir+v4J)rp@3>tYTZ1 zEdj-*18IY{Q4A{@H7t>q-O7HWzu-1x;Ioe49MU}?3kyM4DRq7(0_FhfC+tyDwuuSG zs#N1w7I4v2>#6k0U58974852TmtFEQL1J4A*wrI(V2d&mV6?z@+vy6LqYylbOW7pi zc}ZRSbv62;$5ZxvMdfsf#a+|b?EZ7a7w=L~Pm3TxTFo%GqX|M;?>1Z<%n zRPAgvkQw}NFlK=qf#a*1SL=ry9k+|jznkUC6AJ=^H%vwy;rmp+>@l9Vn~n>D^`1Lj znbL2=fRG{PEOo}VvWjeZi`Efm-mM}=saCRnA@7A7xd-0oSjcReaphZ@a~kL z27-$wL&!Jkt}u>Cmmt?n=ax1oUob^UqOzC@%Pgp>qxVIG9BFnTagl-@4dz-w=2VJB zh*4aK?^#qhkmncorkBsXPnTv+F>xxEBPMo>=`Y|F{WsO2QU9@AUt;|Ix{t~!J)>Z? zWbRQZS?2xYNYNQrut|}rt&I|1+_tjqr5~QP>mo18FK#|{jl7j?%x<2v%|w@});;wK z2SK!Ssh=iuv2zg#4f(6*C!RHo*s;r2EhUzyn9>#HH^zq_c*LU^aDOGO$4RVdd>I6) z4-gZBKiq+1ZzpYZAcxY>Hz397%*rqC6&M{w_*_>~UAqR@4 z<&rc{jKI(ndqhqZ`Bt)s45oqPSU!~0rGzGBnAtC<_AQXdVpPJGYCMRDsgSSKUa!=G zaw%1^?PtnDS1P+HPv>HDf>zxO#44EU$Bzd;H~A8+6MfxoDk8X2QRURdU;9F0nuyDw zmNfGcrkw>R#QQDQvKuY$U|DJdC z_*W%?DXu0=Q5&zsb&xf+Ki_BSa}PyAvv@ZoyU*yjPx23X()%Yez&rlp4?ih!mPpw$ zsG>^5nilcttSZ*&L1zT2-6M)>uOU02Hr_+W-ez{JhI z1}j1J<)wD;K5T&OEXIWTK096plx%6AAzO$W+nvRDk+>h=R>@{H>i|Dt_& z8g##dE5(mb{{T`RwrqA#E$wt^b#-0;PJ}O}A_ONZVzXKsSwwvt1#9;&GLZf@YMhgM zjRCrNF$C}WiZ874jf>6-gSVN*H3K?bWFu~Dz};L_sIc<+Q{T|I8VjGARtc+B93vZ+ z6^4x(e_?9v%v4ig0<2D8v^9D?)wM(q1=FC5IDmsSD8kJ3jXEoorS80bObY&HLUcK6 z?l4BCeC=#(*&?EVqgPW+EEQWu7w8uXBb;#UEUi1EXi~1#uQ8e5*f(B&RY20VYe1{y zAeV`JlQN=EP#9fRqEET}v8y3vs$-5!$IWGukpbcZ9aof|*Nsh8{L@=bZJIUl2^6j$ zQ)`iCUF>IBi`@(?ACJV4trc=QeD}pffqir%cl(Z?MV^|!C|ctqkhF;R3E~9HgGA%7 zD7bF!6xH3vr&%=}OS^?lm);nw4WLLbse_9xQUVTEq?lAl{>a=DqLPPe?*JAc(yQkOJttF@c+>PyNxiI3BLGz(YQ7`V02nPFqu< z5bu{DC!y6Ao@UvZ>-0>)l}4`2XkiQYNx?y&>MVkR==ND7_j(TufAcEBfC+B#tFE7u z@i<;ktP!j~t?%+v?L|!HdRg4VbzGh+Y|OYdcgvF1OKJaSg4T=9MRUW~{F~;)cxCUe z?rgv}a6GwMvL{H7R6~(TB@%|`Iq$#_s%VDjALbz@b=dsS+%%2f_FTWnbV&kztx4L^ zO3e%hfckV>PG>RV`$WOR0aI?GI`@u@9rYijkW5K$XCMR2b`IXBd`zMEd1O-0S{Otr zB+g|@4uY$HI-~KcIXJgy1Hi?bm$8$wA zHSZr5VG+5K(B7%Z)&a7EP*arMcNJg#Mj z_j2I^S+WO11>u`q-#koP z7&0BW_s_;r3gdjnBJA0YvLmQus;KBKUWy##Wtsp}Yo@he^!;O)DdLC{_>S)S(nAqIAlXM=8c4Ij4Z)?Oi7)DHTPHtK6Tm`xuI1&I;kax{GmiCiq)7UxD z`0MruLxPK-oHNm{qp|5L*HuaxoQwlu^|y9*PJzx{gEQD;QvzmD*Z zjI=1On7+`_>J1c7>OU4ZWDgKb5xv$25|j?FWO)`a{m9oKonOX&(5{DDPLLoEgDcpp z3Df2E%JJ#0L-GS`WTn8wO!F4TDYazLfS2+re3902=x}d-o{ub@L~~ES3`;RggBxX% zv)q4|KYkD^U?$NnYv*wLm5Kj>jy(d)-cjDR;+W>arXXMqqT0w7o#S8@j5xGNZ{_G~5kLP$RJ46bt@s8?(K`pERhwcf2x-&T>6gIBv(mqDu3Ar=YF zUD5keuTalQ$a?W2u#b7h=BF@6jBz_i+nO$s%Cblm^!5b{mEYD|h)te|>=qF#U33G2D&YE6|5KDj<(ws&+}OPlGsDb5PC zoAUll*>oJcPzpixi!u=Kiz{vLtsJ%u9f^9;0imnPnC^ry{frv@htK3366yda&(;m= zc#p|he6l&*0f7+&@&U$ZxC@DA|6`jRcKHft3Chx1Ys9E`oDrMYjbb%uu7LrSjZJj! z)IEKZhy+)&$Tt8biOA%4KnQeMh@lKIYAhg1X|19EGgQLDam{N770ahWN9UM)`F~~! zWnW$?1e{S<{kx%Vb&S+)Ae8uowb9$w-sQBO5R zKIaYm^IwFWc=n9?SzhUs5*(=Gq$3jUB|rR^u@d>pC*pEEj>TI({Iyq^Csg=f)-G|@ zU;Y2_m?BG3SItyUCwuM74;4GacaO=&eh*veG2yN?`N1ItE5?4ezRe@{vI~4%pE`}( z43ip+v8yvHmpK_l+8*t{Ui(OL^+K}h8x}C2@DYlCDZhc9jT|^j#=f_SU1NR_F4y>_ z`cN2P%G=}WR$Xvy8FRa)vG8DO=BTLV;pVcmCAiLWM@LrimnTfBh;tHh{>@ZmBG9TDbA_Z}w?KD{MC}@8_~R`!grh7j!__ zs(mgxAyf_J<$w8#58L45;fDLT_}KF=4?x*oxF+x{J^kj>5u4wE(x(9Mki`e=%P2QJ zj42EMk3|~EqyZQ^f`R@@Bu!H{U1<>2hHqrrqhwMYR z6_aosEW#s0%Sx(G!XxOIZ|La+H?#Q;8~yGdaDMajZ|wj084$|p3LhOU-Y2m9%pNBA zXKKcAQ1j~8gq`2BW`rf~*H}uCO@6Y($5D@-0gCPoH=NNcvl5<0wZHW9ee_9cl$~0t=z+!SUc7l&~4y zKpopiy{v-2O8CtVOX}x4=ma9?^GDQfohW*NVDD^3*X_J2M~dkKs-l68jh(MmfSh*Dl< z3UksFc1?{mQ{ZHrxA~l0`90i@QMIcY3FtDN#buO;^XS%@m`A(b;a8U1ez^Z8ne7OsXoQcy6 zo6XW5w_j_a4X_I2E3D>-_Q}oAMrM1c!cPnOdZ4TwFSOV!`ax?Sk=tw%K`I^>5iV!P zz^MBPc*pcuxj4+zWwjkd=}Vblg;;C>PPGd-eUR%}vs2z|3bhFCRI(x#bvYE?XcJsn zwG4c40BoR`%R4)hBSvo`{+2^C!@<0!K2@U2GcZaPfEc`Ff~jk!hIK1&;N6 z+g_%U_jW!7@|HHuvDeeR{A*77gO=VG-wvr2Nwi5rWuQHOIGhSludey3t1U3ThI>)) zR%5VUig)TF+NdyY4qk(4JAktJEI80~ZiB|#Vv&c_^Q;lK-^H@Bb1j1 zs08gD<=_f+?o5@le+=`jVhdo9IF3aItXN4yKb z9t+fp*yJlw5KN5pmCEf8i4TZ^v%D{*Y%<=}>-Nd7tjywei`QnDq-{C%j2K;f0W$9?&Y&ReEH?B<0u90ew^Ng)$-0$87$p+w%q)%8b# zv6+<2S^TLsOT!=T0M^VBeJCyfs?h>ui5ARQrr$Ctynqjw>9gmjZNCp&CD*Oe*#qvR z{lb>;9!kt6oDhJmZ!>ZQNsOi{)Xe@Xk!suhWk;(?6W#0Zl$s696DLLB5VRf;gb<+2B?-%PSte?lb=$z zuZ_cbQEFTx>t*b(d)Q`}Y|ZvI1$Q*qw^C+qc80eV#S^H|QU|OmF`?3YcGJBxt-(<}`B41+)%KREihe`B7rS&b7RwYHiF}buBZ_-C4 z@O#WaJ)`6IJZ)E6>$Be!+{4~7g*PAh&@ses&%#q$LZd1D_J=XnZ5WU)c|b|t9YCH~ zY?5beLS1vSic^bcqPucL`G94D+qlP=R!j7e^T*@W;*G2Kszasu}?WAmOih==RaOFP^UI`n@Q32w;ah-?n=k#&vS1MY-?4|mS54N4eb+( zEZ?npbOuD_hhM=IQ3bW)jUE(Bd2T>=6AtW;`OcO%yC$9!!_K`oy`DUV1?B7WwI8@{ zbecz9InP|uze6?3#&`(hIw+#s?vUU`s}H>a!9m$fz_%zBec2R1mjF41cg_M^Hp=!W zUeWCJy&p?#loQZkvAy*=3jCvCc%A?BKC&l4Jx>mMY}M-A(7L2#^UlXULk;)N5Bu}F zGD#wpCR~!`z&Rav+t4_9@XjtHZ!USGVT!?t4l&u{LYt{uAnu5i{_|;RN?wy~eF0m1 zjDE(1oKa%Gqo%+DHh*pHwDLVHe$<|GIXhl8?$>Ykos?z{Ks&a_PS2ieCngbnDEVGo zJlD@)_mu$sZ>_(HNm12Cq^!i_$(FHxIq&G7&B_JVA;$)wFg1|02`IKyO*Erp;jvOS z?fb`;wNfkTMhp-~=xVxPPTEplZoT25Do>1ve-x75_1cN7BVzLOubn(?iyQRbW#J}t zrrAK@-|T=iJXoJP@PAzyl9_v6d-jq3XEG+jrGaP4AhPto{Kh}i;BSHo6*4X3z18A{ zii$8snDYC{pbw~4%I50icQV2is9N{CBNzpSw2Pw0#7j7N;v)nLs$1y&QsBg3 z_Dm$j810$O2(or+IG2+8{RC2&@SYpH%wb$lLiz(duWwkiH+YA=kMzB9qWydYJbNZ% zzT1wa#p2#O{o?lhVW2wb8%lBZYI}1>Xarp@F1QOTHJ=RC_)T9$fP_BN%uw7a>YK=g z_d#UQ^4|l41nW-5Gq~iH(^9C!8B1L)KWYnMD4Zrgdh<>7M1eI)mZ7c%xeTzTL(%+; zyk8=jO>LGaurxWao=4NeRcs}vGst1PgzcYu@%H8Rz~vs;e}+;YtHiCoTRaZP#MTmK zkLv&1|CN+NAiq}BoeZT3Cffu3dYgjynNf;w(wgD&Gq>{zhB%!N!K$wga4xbxPh_59 zN|fj3lw54kS)caMii)@#@)e#;m0fO>o-+~eW7@6xyxqXYq!ijyHGgbAeo3B_JNoOY z62>!=`IJz)7b}dYL~*&Roqi7cxs#0#6D!S}TEjW=eLTTDvKwwXz`ofHC$it?B2ViB zWh9q_5_~TNJQKXqc2tq>gIu>T%t==Vi6|-rcIV z^X}^eC65yipxAsomp7UUTEA3hVjKGExPG^Pp?~$mg(l!Ao6SLaqGI!962zB4)eFEf z4<-oIiS8&lDNO7Hewu_L!$ym+HDAD`uG7c@!F(kb;i$ArD%a|WV^`&Nq14L^#{Kqw zV+Pyojt=+hY2m%V6kZ!xXWK;NP#TA5WYtUjJCk0jnw^{Bstc#H9fGH!EEY5zUil|? z$1yembXZt1`LOjWxzOs^Bh8ewt*q%p<{YW1*=L9Agi=2~mfCYg{8~JlUY^(PxCa~A zbjhg?qz$X;xq`z`;*q3>Q>F6+-O%K-;v2Wk0lrjDn25_GA<#IH^=D-honV&TfJLAC zmPh;VV)%=Pb01J&*iml;mJNZvahm!?1kt|Ic)Nze4C$S`|E9D$T)x%_$Qn@?P_5J4 z-{oi+5yWXUR<pV5i$!z6jkw9T`KmD8`t(K1!|MTDf7o zQ&FjQD$I?KCS!+#QsM{!V3=O?*4FQ~^?e~+oQoP2b>mwS3&=!f6p~r$Y`s9T*G^9u zO#_R^j?hhZf}s80iMm(nG80h1P4!su)y8_J;uajx%bp~03^ ziBSB(uT>f(L5CqV%xKUEk0wGQV7F957WhQH!lSx^%0CC?C`-If<3)!qB1ysvxP(cf z4oK@ADnlm(SBHTJO88Ae9z^*J}Gy}H!29w zem()wFaL27>whb96%l*vp!!cgS5mD1D~-_qZb@m9<>3jLJ}h1RZq4YqMIAw2TKRp2 Iq)EX403JAP(EtDd literal 0 HcmV?d00001 -- Gitee From c4eb0fc6443e997c7695128eb626805433c09f63 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Fri, 9 Aug 2024 16:50:27 +0800 Subject: [PATCH 234/791] =?UTF-8?q?[profiler\compare=5Ftools]=E6=89=93?= =?UTF-8?q?=E5=B1=8F=E7=BB=93=E6=9E=9C=E6=96=B0=E5=A2=9ERDMA=20SDMA?= =?UTF-8?q?=E5=B8=A6=E5=AE=BD=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/compare_tools/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index 99559728d..97dcf5b19 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -197,6 +197,8 @@ MindSpore场景仅支持**总体性能**和**通信性能**的对比。 | Computing Time | 计算流耗时，计算流所有event耗时总和。如果有多条并发计算，计算流耗时对重叠部分只会计算一次。 | | Mem Usage | 内存使用。GPU上的内存使用可以使用nvidia-smi查看，NPU上的内存使用可以使用npu-smi查看，Profiling信息采集时打开profile_memory=True开关，mem usage显示的是memory_record里面的最大resevered值，一般来说是进程级内存。 | | Uncovered Communication Time(Wait Time) | 通信未掩盖耗时。Wait Time为卡间等待时间（Wait Time仅NPU场景才会存在）。 | +| RDMA Bandwidth(GB/s) | RDMA带宽，单位GB/s。 | +| SDMA Bandwidth(GB/s) | SDMA带宽，单位GB/s。 | | SDMA Time(Num) | 拷贝类任务耗时，Num表示计算的次数。 | | Free Time | 调度耗时 = E2E耗时 - 算子耗时 - 通信不可掩盖耗时。Free的定义为Device侧既不在通信又不在计算的时间，因此包含拷贝时间（SDMA Time）。 | | E2E Time(Not minimal profiling) | E2E总耗时，计算流端到端耗时。当存在Not minimal profiling时，表示该时间存在性能膨胀，会影响通信和调度耗时。 | -- Gitee From db172b35c5014f6469120cf7163026933282088d Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 9 Aug 2024 17:31:30 +0800 Subject: [PATCH 235/791] compare compatibility --- .../msprobe/core/common/utils.py | 22 +++++++++++-------- .../msprobe/pytorch/compare/compare_cli.py | 5 ++++- .../pytorch/compare/distributed_compare.py | 9 ++++---- .../msprobe/pytorch/compare/pt_compare.py | 19 ++++++++-------- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 7a34a2411..cfc4fbd74 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -153,16 +153,16 @@ def check_compare_param(input_param, output_path, summary_compare=False, md5_com if not (isinstance(input_param, dict) and isinstance(output_path, str)): logger.error("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) - check_file_or_directory_path(input_param.get("npu_path"), False) - check_file_or_directory_path(input_param.get("bench_path"), False) - check_file_or_directory_path(input_param.get("stack_path"), False) + check_file_or_directory_path(input_param.get("npu_json_path"), False) + check_file_or_directory_path(input_param.get("bench_json_path"), False) + check_file_or_directory_path(input_param.get("stack_json_path"), False) if not summary_compare and not md5_compare: check_file_or_directory_path(input_param.get("npu_dump_data_dir"), True) check_file_or_directory_path(input_param.get("bench_dump_data_dir"), True) check_file_or_directory_path(output_path, True) - with FileOpen(input_param.get("npu_path"), "r") as npu_json, \ - FileOpen(input_param.get("bench_path"), "r") as bench_json, \ - FileOpen(input_param.get("stack_path"), "r") as stack_json: + with FileOpen(input_param.get("npu_json_path"), "r") as npu_json, \ + FileOpen(input_param.get("bench_json_path"), "r") as bench_json, \ + FileOpen(input_param.get("stack_json_path"), "r") as stack_json: check_json_file(input_param, npu_json, bench_json, stack_json) @@ -474,9 +474,13 @@ def md5_find(data): return False -def task_dumppath_get(input_param): - npu_path = input_param.get("npu_path", None) - bench_path = input_param.get("bench_path", None) +def task_dumppath_get(input_param, framework="mindspore"): + if framework == "mindspore": + npu_path = input_param.get("npu_path", None) + bench_path = input_param.get("bench_path", None) + else: + npu_path = input_param.get("npu_json_path", None) + bench_path = input_param.get("bench_json_path", None) if not npu_path or not bench_path: logger.error(f"Please check the json path is valid.") raise CompareException(CompareException.INVALID_PATH_ERROR) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py index b344d4efb..8d5e048fb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py @@ -12,8 +12,11 @@ def compare_cli(args): input_param = json.load(file) npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) - + if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: + input_param["npu_json_path"] = input_param.pop("npu_path") + input_param["bench_json_path"] = input_param.pop("bench_path") + input_param["stack_json_path"] = input_param.pop("stack_path") compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 923c0044d..0b82c2d30 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -47,13 +47,14 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): stack_path = extract_json(npu_data_dir, stack_json=True) dump_result_param = { - 'npu_path': npu_path, - 'bench_path': bench_path, - 'stack_path': stack_path, + 'npu_json_path': npu_path, + 'bench_json_path': bench_path, + 'stack_json_path': stack_path, 'is_print_compare_log': True } try: - summary_compare, md5_compare = task_dumppath_get(dump_result_param) + framework = "pytorch" + summary_compare, md5_compare = task_dumppath_get(dump_result_param, framework) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index a947a12f6..41b8229fd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -165,9 +165,9 @@ class PTComparator (Comparator): check_file_not_exists(file_path) highlight_dict = {'red_rows': [], 'yellow_rows': []} - with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_path"), "r") as stack_json: + with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_json_path"), "r") as stack_json: result_df = self.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, summary_compare, md5_compare) @@ -179,11 +179,11 @@ class PTComparator (Comparator): advisor = Advisor(result_df, output_path) advisor.analysis() - - + def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: - summary_compare, md5_compare = task_dumppath_get(input_param) + framework = "pytorch" + summary_compare, md5_compare = task_dumppath_get(input_param, framework) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) check_compare_param(input_param, output_path, summary_compare, md5_compare) @@ -198,9 +198,8 @@ def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy - - - - \ No newline at end of file + + + -- Gitee From 399fdf2a05f2b62cb7ec6057a27b373c5d374fdb Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 9 Aug 2024 17:35:21 +0800 Subject: [PATCH 236/791] codeclean --- .../tensor_transport_layer/device_dispatch.py | 2 ++ .../accuracy_tools/msprobe/pytorch/service.py | 19 +++++++++---------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index cbc1b76fd..58b1353ff 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -43,6 +43,8 @@ def run_ut_process(xpu_id, compare, consumer_queue, func, config): if "expected scalar type Long" in str(err): logger.warning(f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API " f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.") + elif Const.DISTRIBUTED in str(err): + logger.info(f"{api_name} is not supported for run ut. SKIP.") else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 187058bd7..12844f600 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -120,17 +120,9 @@ class Service: def start(self, model, api_origin=False): self.model = model if self.config.step and self.current_iter > max(self.config.step): - # send end or step signal if self.config.online_run_ut: - if self.config.nfs_path: - self.attl.upload("end") - elif self.attl.socket_manager is not None: - logger.info(f"进程{os.getpid()} 已完成,准备发送STOP信号") - self.attl.socket_manager.send_stop_signal() - else: - # current rank not need dump, wait - while True: - time.sleep(2) + # send stop signal if online_run_ut + self.attl_stop() self.stop() raise Exception("msprobe: exit after iteration {}".format(max(self.config.step))) if self.config.step and self.current_iter not in self.config.step: @@ -247,3 +239,10 @@ class Service: self.attl.upload(api_data) else: self.attl.send(api_data) + + def attl_stop(self): + if self.config.nfs_path: + self.attl.upload("end") + elif self.attl.socket_manager is not None: + logger.info(f"pid: {os.getpid()} finished, start send STOP signal.") + self.attl.socket_manager.send_stop_signal() -- Gitee From 59a13a6beb19b4003b6257ecf3f7840152389c27 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 9 Aug 2024 17:46:21 +0800 Subject: [PATCH 237/791] cleancode --- .../pytorch/api_accuracy_checker/tensor_transport_layer/attl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index d3f506630..e699bc554 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -93,7 +93,7 @@ class ATTL: try: torch.save(buffer, io_buff) except Exception as e: - logger.warning(f"buffer save failed: {e}") + logger.info(f"{buffer.name} can not be saved, skip: {e}") return data = io_buff.getvalue() self.socket_manager.add_to_sending_queue(data, rank=rank, step=step) -- Gitee From ea6248599cef1127062fd6b7dbfc682758234bf9 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 9 Aug 2024 17:56:59 +0800 Subject: [PATCH 238/791] cleancode --- .../tensor_transport_layer/device_dispatch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 58b1353ff..42f34bfc6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -39,11 +39,11 @@ def run_ut_process(xpu_id, compare, consumer_queue, func, config): f"is_fwd_success: {is_fwd_success}, " f"is_bwd_success: {is_bwd_success}") except Exception as err: - [_, api_name, _] = api_full_name.split(Const.SEP) + [api_type, api_name, _] = api_full_name.split(Const.SEP) if "expected scalar type Long" in str(err): logger.warning(f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API " f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.") - elif Const.DISTRIBUTED in str(err): + elif api_type in [Const.DISTRIBUTED]: logger.info(f"{api_name} is not supported for run ut. SKIP.") else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") -- Gitee From 66c5b7e42591b80ab799fb8039fb0e10b9945be0 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 9 Aug 2024 18:32:52 +0800 Subject: [PATCH 239/791] compare compatibility --- .../msprobe/core/common/utils.py | 27 +++++++++++++------ .../pytorch/compare/distributed_compare.py | 2 +- .../msprobe/pytorch/compare/pt_compare.py | 2 +- 3 files changed, 21 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index cfc4fbd74..246af98a3 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -149,21 +149,32 @@ def check_summary_only_valid(summary_only): return summary_only -def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False): +def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False, framework="mindspore"): if not (isinstance(input_param, dict) and isinstance(output_path, str)): logger.error("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) - check_file_or_directory_path(input_param.get("npu_json_path"), False) - check_file_or_directory_path(input_param.get("bench_json_path"), False) - check_file_or_directory_path(input_param.get("stack_json_path"), False) + if framework == "mindspore": + check_file_or_directory_path(input_param.get("npu_path"), False) + check_file_or_directory_path(input_param.get("bench_path"), False) + check_file_or_directory_path(input_param.get("stack_path"), False) + else: + check_file_or_directory_path(input_param.get("npu_json_path"), False) + check_file_or_directory_path(input_param.get("bench_json_path"), False) + check_file_or_directory_path(input_param.get("stack_json_path"), False) if not summary_compare and not md5_compare: check_file_or_directory_path(input_param.get("npu_dump_data_dir"), True) check_file_or_directory_path(input_param.get("bench_dump_data_dir"), True) check_file_or_directory_path(output_path, True) - with FileOpen(input_param.get("npu_json_path"), "r") as npu_json, \ - FileOpen(input_param.get("bench_json_path"), "r") as bench_json, \ - FileOpen(input_param.get("stack_json_path"), "r") as stack_json: - check_json_file(input_param, npu_json, bench_json, stack_json) + if framework == "mindspore": + with FileOpen(input_param.get("npu_path"), "r") as npu_json, \ + FileOpen(input_param.get("bench_path"), "r") as bench_json, \ + FileOpen(input_param.get("stack_path"), "r") as stack_json: + check_json_file(input_param, npu_json, bench_json, stack_json) + else: + with FileOpen(input_param.get("npu_json_path"), "r") as npu_json, \ + FileOpen(input_param.get("bench_json_path"), "r") as bench_json, \ + FileOpen(input_param.get("stack_json_path"), "r") as stack_json: + check_json_file(input_param, npu_json, bench_json, stack_json) def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False): diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 0b82c2d30..7270b6830 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -57,7 +57,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): summary_compare, md5_compare = task_dumppath_get(dump_result_param, framework) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) + check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare, framework=framework) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 41b8229fd..13db2b4c5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -186,7 +186,7 @@ def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy summary_compare, md5_compare = task_dumppath_get(input_param, framework) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(input_param, output_path, summary_compare, md5_compare) + check_compare_param(input_param, output_path, summary_compare, md5_compare, framework=framework) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error -- Gitee From 1ff667eeca90e1956d9d717fa8c03466d6e41ec1 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Sat, 10 Aug 2024 09:39:03 +0800 Subject: [PATCH 240/791] ms overflow check ut --- .../data_processor/mindspore_processor.py | 4 +- .../mindspore/debugger/precision_debugger.py | 4 +- .../msprobe/mindspore/ms_config.py | 4 +- .../msprobe/mindspore/service.py | 15 +- .../msprobe/test/core_ut/common/test_utils.py | 38 ++--- .../data_dump/data_processor/test_factory.py} | 40 +++-- .../test_mindspore_processor.py | 145 ++++++++++++++++++ .../core_ut/data_dump/test_data_collector.py | 71 +++++++++ .../test/mindspore_ut/common/test_ms_utils.py | 55 +++++++ .../debugger/test_debugger_config.py | 80 ++++++++++ .../{ => debugger}/test_precision_debugger.py | 69 +++++++-- .../test/mindspore_ut/test_ms_config.py | 20 ++- 12 files changed, 478 insertions(+), 67 deletions(-) rename debug/accuracy_tools/msprobe/test/{mindspore_ut/test_debugger_config.py => core_ut/data_dump/data_processor/test_factory.py} (38%) create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/common/test_ms_utils.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_debugger_config.py rename debug/accuracy_tools/msprobe/test/mindspore_ut/{ => debugger}/test_precision_debugger.py (37%) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 6f32b1ec2..8d0966909 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -157,7 +157,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): self.cached_tensors_and_file_paths = {} self.real_overflow_nums = 0 self.overflow_nums = config.overflow_nums - + @property def is_terminated(self): if self.overflow_nums == -1: @@ -166,7 +166,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") return True return False - + def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False api_info_struct = super().analyze_forward(name, module, module_input_output) diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 0b51efec8..04cc3345c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -38,7 +38,7 @@ class PrecisionDebugger: self.gm = GradientMonitor(common_config, task_config) return self.config = DebuggerConfig(common_config, task_config) - + Runtime.step_count = 0 Runtime.is_running = False @@ -97,7 +97,7 @@ class PrecisionDebugger: if instance.service: instance.service.step() Runtime.step_count += 1 - + @classmethod def monitor(cls, opt): instance = cls._instance diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index 0e7ce1529..fb78a5f6c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -63,7 +63,7 @@ class FreeBenchmarkConfig(BaseConfig): if self.fuzz_device and self.fuzz_device not in FreeBenchmarkConst.DEVICE_LIST: raise Exception("fuzz_device must be npu or empty") if self.pert_mode and self.pert_mode not in FreeBenchmarkConst.PERT_TYPE_LIST: - raise Exception("pert_mode must be improve_precision, add_noise, bit_noise , no_change or empty") + raise Exception("pert_mode must be improve_precision, add_noise, bit_noise, no_change or empty") if self.handler_type and self.handler_type not in FreeBenchmarkConst.HANDLER_TYPE_LIST: raise Exception("handler_type must be check, fix or empty") if self.fuzz_level and self.fuzz_level not in FreeBenchmarkConst.DUMP_LEVEL_LIST: @@ -73,6 +73,8 @@ class FreeBenchmarkConfig(BaseConfig): if self.if_preheat or self.preheat_step or self.max_sample: logger.warning("'if_preheat', 'preheat_step' and 'max_sample' settings " "are not supported for mindspore free benchmark task.") + + class GradProbeConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 4c2a4ef69..29881e738 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -22,6 +22,7 @@ from collections import defaultdict from mindspore.common.tensor import Tensor from mindspore import ops from mindspore import nn + from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope from msprobe.mindspore.common.utils import get_rank_if_initialized @@ -30,7 +31,7 @@ from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import Const from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.mindspore.dump.hook_cell.api_registry import api_register -from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs,\ +from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs, \ ModuleBackwardInputs, ModuleBackwardOutputs from msprobe.core.common.exceptions import MsprobeException from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell @@ -93,7 +94,6 @@ class Service: return wrap_forward_hook, wrap_backward_hook - def wrap_primitive(self, origin_func, primitive_name): service_instance = self @@ -118,12 +118,8 @@ class Service: captured_grads.clear() except Exception as exception: - raise Exception( - "This is a primitive op {hook_type}_backward dump error: {exception}," - " updated_primitive_name: {updated_primitive_name}".format( - hook_type=hook_type, exception=exception, backward_primitive_name=backward_primitive_name - ) - ) from exception + raise Exception(f"This is a primitive op {hook_type}_backward dump error: {exception}," + f" updated_primitive_name: {updated_primitive_name}") from exception return backward_hook @@ -223,7 +219,6 @@ class Service: {'__call__': self.wrap_primitive(primitive.__call__, pname)}) primitive.__class__ = NewPrimitive - def step(self): self.current_iter += 1 self.data_collector.update_iter(self.current_iter) @@ -297,4 +292,4 @@ class Service: api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) api_register.api_set_hook_func() if self.model: - self.register_hooks() \ No newline at end of file + self.register_hooks() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index a1cd516c4..a02a402f6 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -# Copyright (C) 2022-2023. Huawei Technologies Co., Ltd. All rights reserved. +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -23,24 +23,24 @@ from unittest.mock import patch, MagicMock, mock_open from msprobe.core.common.log import logger from msprobe.core.common.const import Const from msprobe.core.common.utils import (CompareException, - check_seed_all, - check_inplace_op, - make_dump_path_if_not_exists, - check_mode_valid, - check_switch_valid, - check_dump_mode_valid, - check_summary_mode_valid, - check_summary_only_valid, - check_file_or_directory_path, - check_compare_param, - check_configuration_param, - is_starts_with, - _check_json, - check_json_file, - check_file_size, - check_regex_prefix_format_valid, - get_dump_data_path, - task_dumppath_get) + check_seed_all, + check_inplace_op, + make_dump_path_if_not_exists, + check_mode_valid, + check_switch_valid, + check_dump_mode_valid, + check_summary_mode_valid, + check_summary_only_valid, + check_file_or_directory_path, + check_compare_param, + check_configuration_param, + is_starts_with, + _check_json, + check_json_file, + check_file_size, + check_regex_prefix_format_valid, + get_dump_data_path, + task_dumppath_get) from msprobe.core.common.file_check import FileCheckConst diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_debugger_config.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_factory.py similarity index 38% rename from debug/accuracy_tools/msprobe/test/mindspore_ut/test_debugger_config.py rename to debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_factory.py index 5187d3951..2f4f25300 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_debugger_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_factory.py @@ -14,29 +14,25 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -from unittest import TestCase +import unittest +from unittest.mock import patch +from msprobe.core.data_dump.data_processor.factory import DataProcessorFactory from msprobe.core.common.const import Const -from msprobe.core.common_config import CommonConfig, BaseConfig -from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.core.data_dump.data_processor.mindspore_processor import ( + StatisticsDataProcessor as MindsporeStatisticsDataProcessor, + TensorDataProcessor as MindsporeTensorDataProcessor, + OverflowCheckDataProcessor as MindsporeOverflowCheckDataProcessor +) -class TestDebuggerConfig(TestCase): - def test_init(self): - json_config = { - "dump_path": "/absolute_path", - "rank": [], - "step": [], - "level": "L0" - } - common_config = CommonConfig(json_config) - task_config = BaseConfig(json_config) - debugger_config = DebuggerConfig(common_config, task_config) - self.assertEqual(debugger_config.task, Const.STATISTICS) - self.assertEqual(debugger_config.file_format, "npy") - self.assertEqual(debugger_config.check_mode, "all") - - common_config.dump_path = "./path" - with self.assertRaises(Exception) as context: - DebuggerConfig(common_config, task_config) - self.assertEqual(str(context.exception), "Dump path must be absolute path.") +class TestDataProcessorFactory(unittest.TestCase): + def test_register_processors(self): + with patch.object(DataProcessorFactory, "register_processor") as mock_register: + DataProcessorFactory.register_processors(Const.MS_FRAMEWORK) + self.assertEqual(mock_register.call_args_list[0][0], + (Const.MS_FRAMEWORK, Const.STATISTICS, MindsporeStatisticsDataProcessor)) + self.assertEqual(mock_register.call_args_list[1][0], + (Const.MS_FRAMEWORK, Const.TENSOR, MindsporeTensorDataProcessor)) + self.assertEqual(mock_register.call_args_list[2][0], + (Const.MS_FRAMEWORK, Const.OVERFLOW_CHECK, MindsporeOverflowCheckDataProcessor)) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py new file mode 100644 index 000000000..046388741 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import unittest +from unittest.mock import patch + +from mindspore import Tensor +import numpy as np + +from msprobe.core.data_dump.data_processor.base import BaseDataProcessor +from msprobe.core.data_dump.data_processor.mindspore_processor import MindsporeDataProcessor, OverflowCheckDataProcessor +from msprobe.core.common.const import FileCheckConst + + +class TestOverflowCheckDataProcessor(unittest.TestCase): + def setUp(self): + class Config: + def __init__(self): + self.overflow_nums = 1 + self.data_processor = OverflowCheckDataProcessor(Config(), None) + + def test___init__(self): + self.assertEqual(self.data_processor.cached_tensors_and_file_paths, {}) + self.assertEqual(self.data_processor.real_overflow_nums, 0) + self.assertEqual(self.data_processor.overflow_nums, 1) + + def test_analyze_forward(self): + def func(_): + self.data_processor.has_overflow = True + with patch.object(BaseDataProcessor, "analyze_forward", return_value={"min", 0}): + with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data"): + api_info = self.data_processor.analyze_forward("name", "module", "module_input_output") + self.assertFalse(self.data_processor.has_overflow) + self.assertIsNone(api_info) + with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data", new=func): + api_info = self.data_processor.analyze_forward("name", "module", "module_input_output") + self.assertTrue(self.data_processor.has_overflow) + self.assertEqual(api_info, {"min", 0}) + + def test_analyze_backward(self): + def func(_): + self.data_processor.has_overflow = True + with patch.object(BaseDataProcessor, "analyze_backward", return_value={"min", 0}): + with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data"): + api_info = self.data_processor.analyze_backward("name", "module", "module_input_output") + self.assertFalse(self.data_processor.has_overflow) + self.assertIsNone(api_info) + with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data", new=func): + api_info = self.data_processor.analyze_backward("name", "module", "module_input_output") + self.assertTrue(self.data_processor.has_overflow) + self.assertEqual(api_info, {"min", 0}) + + @patch("msprobe.core.data_dump.data_processor.mindspore_processor.np.save") + @patch("msprobe.core.data_dump.data_processor.mindspore_processor.change_mode") + def test_maybe_save_overflow_data(self, mock_change_mode, mock_save): + self.data_processor.has_overflow = True + tensor1 = Tensor(1) + tensor2 = Tensor(2) + self.data_processor.cached_tensors_and_file_paths = {"tensor1": tensor1, "tensor2": tensor2} + with patch("mindspore.Tensor.asnumpy", return_value="npy"): + self.data_processor.maybe_save_overflow_data() + self.assertEqual(mock_save.call_args_list[0][0], + ("tensor1", "npy")) + self.assertEqual(mock_save.call_args_list[1][0], + ("tensor2", "npy")) + self.assertEqual(mock_change_mode.call_args_list[0][0], + ("tensor1", FileCheckConst.DATA_FILE_AUTHORITY)) + self.assertEqual(mock_change_mode.call_args_list[1][0], + ("tensor2", FileCheckConst.DATA_FILE_AUTHORITY)) + + @patch("msprobe.core.data_dump.data_processor.mindspore_processor.logger.info") + def test_is_terminated(self, mock_info): + self.data_processor.overflow_nums = -1 + self.assertFalse(self.data_processor.is_terminated) + self.data_processor.real_overflow_nums = 2 + self.data_processor.overflow_nums = 2 + self.assertTrue(self.data_processor.is_terminated) + mock_info.assert_called_with("[msprobe] 超过预设溢出次数 当前溢出次数: 2") + self.data_processor.overflow_nums = 3 + self.assertFalse(self.data_processor.is_terminated) + + def test__analyze_maybe_overflow_tensor(self): + self.data_processor.has_overflow = False + tensor_json = {"Max": None, "Min": 0} + self.data_processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertFalse(self.data_processor.has_overflow) + tensor_json.update({"Max": -np.inf}) + self.data_processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.data_processor.has_overflow) + self.data_processor.has_overflow = False + tensor_json.update({"Max": np.inf}) + self.data_processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.data_processor.has_overflow) + self.data_processor.has_overflow = False + tensor_json.update({"Max": np.nan}) + self.data_processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.data_processor.has_overflow) + tensor_json.update({"Max": 0}) + self.data_processor.has_overflow = False + tensor_json.update({"Min": -np.inf}) + self.data_processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.data_processor.has_overflow) + self.data_processor.has_overflow = False + tensor_json.update({"Min": np.inf}) + self.data_processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.data_processor.has_overflow) + self.data_processor.has_overflow = False + tensor_json.update({"Min": np.nan}) + self.data_processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.data_processor.has_overflow) + + @patch("msprobe.core.data_dump.data_processor.mindspore_processor.logger.warning") + @patch.object(OverflowCheckDataProcessor, "get_save_file_path") + @patch.object(MindsporeDataProcessor, "_analyze_tensor") + def test__analyze_tensor(self, mock_super, mock_get_file_path, mock_warning): + mock_get_file_path.return_value = ("dump_data_name", "file_path") + single_arg = {"Max": None} + mock_super.return_value = single_arg + + with patch("msprobe.core.data_dump.data_processor.mindspore_processor.path_len_exceeds_limit", + return_value=False): + ret = self.data_processor._analyze_tensor("tensor", "suffix") + self.assertEqual(self.data_processor.cached_tensors_and_file_paths, {"file_path": "tensor"}) + mock_warning.assert_not_called() + mock_super.assert_called_with("tensor", "suffix") + self.assertEqual(ret.get("Max"), None) + self.assertEqual(ret.get("data_name"), "dump_data_name") + + with patch("msprobe.core.data_dump.data_processor.mindspore_processor.path_len_exceeds_limit", + return_value=True): + self.data_processor._analyze_tensor("tensor", "suffix") + mock_warning.assert_called_with("The file path file_path length exceeds limit.") diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index eedbe5be7..15a0883f5 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -1,3 +1,20 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + import unittest from unittest.mock import patch, mock_open, MagicMock @@ -5,6 +22,9 @@ from msprobe.core.common.utils import Const from msprobe.core.data_dump.data_collector import DataCollector from msprobe.pytorch.debugger.debugger_config import DebuggerConfig from msprobe.pytorch.pt_config import parse_json_config +from msprobe.core.data_dump.json_writer import DataWriter +from msprobe.core.data_dump.data_processor.base import BaseDataProcessor +from msprobe.core.data_dump.data_processor.pytorch_processor import StatisticsDataProcessor class TestDataCollector(unittest.TestCase): @@ -45,3 +65,54 @@ class TestDataCollector(unittest.TestCase): self.data_collector.pre_forward_data_collect(name, None, pid, None) self.data_collector.check_scope_and_pid.assert_called_once_with( self.data_collector.scope, "TestModule.backward", 123) + + def test_handle_data(self): + with patch.object(DataCollector, "update_data", return_value="msg") as mock_update_data, \ + patch.object(DataCollector, "write_json") as mock_write_json, \ + patch("msprobe.core.data_dump.data_collector.logger.info") as mock_info, \ + patch("msprobe.core.data_dump.json_writer.DataWriter.flush_data_when_buffer_is_full") as mock_flush: + self.data_collector.handle_data("Tensor.add", {"min": 0}) + msg = "msprobe is collecting data on Tensor.add. " + mock_update_data.assert_called_with({"min": 0}, msg) + mock_info.assert_called_with("msg") + mock_flush.assert_called() + mock_write_json.assert_not_called() + + mock_update_data.reset_mock() + mock_info.reset_mock() + mock_flush.reset_mock() + self.data_collector.handle_data("Tensor.add", {}, use_buffer=False) + mock_update_data.assert_not_called() + mock_info.assert_not_called() + mock_write_json.assert_called() + + @patch.object(DataCollector, "update_construct") + @patch.object(DataWriter, "update_stack") + @patch.object(BaseDataProcessor, "analyze_api_call_stack") + @patch.object(DataCollector, "handle_data") + def test_forward_data_collect(self, mock_handle_data, _, __, ___): + with patch.object(DataCollector, "check_scope_and_pid", return_value=True), \ + patch.object(DataCollector, "is_inplace", return_value=False), \ + patch.object(StatisticsDataProcessor, "analyze_forward", return_value={}): + with patch.object(StatisticsDataProcessor, "is_terminated", return_value=True), \ + self.assertRaises(Exception) as context: + self.data_collector.forward_data_collect("name", "module", "pid", "module_input_output") + mock_handle_data.assert_called_with("name", {}, use_buffer=False) + self.assertEqual(str(context.exception), "[msprobe] exit") + + self.data_collector.forward_data_collect("name", "module", "pid", "module_input_output") + mock_handle_data.assert_called_with("name", {}) + + @patch.object(DataCollector, "update_construct") + @patch.object(DataCollector, "handle_data") + def test_backward_data_collect(self, mock_handle_data, _): + with patch.object(DataCollector, "check_scope_and_pid", return_value=True), \ + patch.object(StatisticsDataProcessor, "analyze_backward", return_value={}): + with patch.object(StatisticsDataProcessor, "is_terminated", return_value=True), \ + self.assertRaises(Exception) as context: + self.data_collector.backward_data_collect("name", "module", "pid", "module_input_output") + mock_handle_data.assert_called_with("name", {}, use_buffer=False) + self.assertEqual(str(context.exception), "[msprobe] exit") + + self.data_collector.backward_data_collect("name", "module", "pid", "module_input_output") + mock_handle_data.assert_called_with("name", {}) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/common/test_ms_utils.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/common/test_ms_utils.py new file mode 100644 index 000000000..96f2daf20 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/common/test_ms_utils.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import unittest + +from msprobe.mindspore.common.utils import MsprobeStep + + +class TestMsprobeStep(unittest.TestCase): + def setUp(self): + class Debugger: + def __init__(self): + self.start_called = False + self.stop_called = False + self.step_called = False + self.stop_called_first = False + + def start(self): + self.start_called = True + + def stop(self): + self.stop_called = True + + def step(self): + if self.stop_called: + self.stop_called_first = True + self.step_called = True + debugger = Debugger() + self.msprobe_step = MsprobeStep(debugger) + + def test_on_train_step_begin(self): + self.msprobe_step.on_train_step_begin("run_context") + self.assertTrue(self.msprobe_step.debugger.start_called) + self.assertFalse(self.msprobe_step.debugger.stop_called) + self.assertFalse(self.msprobe_step.debugger.step_called) + + def test_on_train_step_end(self): + self.msprobe_step.on_train_step_end("run_context") + self.assertFalse(self.msprobe_step.debugger.start_called) + self.assertTrue(self.msprobe_step.debugger.stop_called) + self.assertTrue(self.msprobe_step.debugger.step_called) + self.assertTrue(self.msprobe_step.debugger.stop_called_first) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_debugger_config.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_debugger_config.py new file mode 100644 index 000000000..980663237 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_debugger_config.py @@ -0,0 +1,80 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import unittest +from unittest.mock import patch + +from msprobe.core.common.const import Const, FileCheckConst +from msprobe.mindspore.common.const import FreeBenchmarkConst +from msprobe.core.common_config import CommonConfig, BaseConfig +from msprobe.mindspore.debugger.debugger_config import DebuggerConfig + + +class TestDebuggerConfig(unittest.TestCase): + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def test_init(self, _): + json_config = { + "dump_path": "/absolute_path", + "rank": [], + "step": [], + "level": "L2" + } + common_config = CommonConfig(json_config) + task_config = BaseConfig(json_config) + debugger_config = DebuggerConfig(common_config, task_config) + self.assertEqual(debugger_config.task, Const.STATISTICS) + self.assertEqual(debugger_config.file_format, "npy") + self.assertEqual(debugger_config.check_mode, "all") + self.assertEqual(debugger_config.overflow_nums, 1) + + common_config.dump_path = "./path" + with self.assertRaises(Exception) as context: + DebuggerConfig(common_config, task_config) + self.assertEqual(str(context.exception), "Dump path must be absolute path.") + + common_config.dump_path = "./path" + with self.assertRaises(Exception) as context: + DebuggerConfig(common_config, task_config) + self.assertEqual(str(context.exception), "Dump path must be absolute path.") + + common_config.level = "L1" + common_config.task = Const.FREE_BENCHMARK + debugger_config = DebuggerConfig(common_config, task_config) + self.assertEqual(debugger_config.pert_type, FreeBenchmarkConst.DEFAULT_PERT_TYPE) + self.assertEqual(debugger_config.handler_type, FreeBenchmarkConst.DEFAULT_HANDLER_TYPE) + self.assertEqual(debugger_config.dump_level, FreeBenchmarkConst.DEFAULT_DUMP_LEVEL) + self.assertEqual(debugger_config.stage, FreeBenchmarkConst.DEFAULT_STAGE) + + task_config.handler_type = FreeBenchmarkConst.FIX + task_config.pert_mode = FreeBenchmarkConst.ADD_NOISE + with self.assertRaises(Exception) as context: + DebuggerConfig(common_config, task_config) + self.assertEqual(str(context.exception), + "pert_mode must be improve_precision or empty when handler_type is fix, " + f"but got {FreeBenchmarkConst.ADD_NOISE}.") + + @patch("msprobe.mindspore.debugger.debugger_config.os.path.exists", return_value=False) + def test__make_dump_path_if_not_exists(self, _): + json_config = {"dump_path": "/absolute_path"} + common_config = CommonConfig(json_config) + task_config = BaseConfig(json_config) + with patch("msprobe.mindspore.debugger.debugger_config.check_path_before_create") as mock_check_path, \ + patch("msprobe.mindspore.debugger.debugger_config.Path.mkdir") as mock_mkdir, \ + patch("msprobe.mindspore.debugger.debugger_config.FileChecker") as mock_checker: + DebuggerConfig(common_config, task_config) + mock_check_path.assert_called_with(json_config.get("dump_path")) + mock_mkdir.assert_called_with(mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) + mock_checker.assert_called_with(common_config.dump_path, FileCheckConst.DIR) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_precision_debugger.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_precision_debugger.py similarity index 37% rename from debug/accuracy_tools/msprobe/test/mindspore_ut/test_precision_debugger.py rename to debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_precision_debugger.py index 425ed3040..ee9970f51 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_precision_debugger.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_precision_debugger.py @@ -14,16 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -from unittest import TestCase -from unittest.mock import patch +import unittest +from unittest.mock import patch, MagicMock from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.debugger.precision_debugger import PrecisionDebugger +from msprobe.mindspore.runtime import Runtime +from msprobe.mindspore.common.const import Const as MsConst +from msprobe.core.common.const import Const -class TestPrecisionDebugger(TestCase): - def test_start(self): +class TestPrecisionDebugger(unittest.TestCase): + + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def test_start(self, _): class Handler: called = False @@ -35,22 +40,68 @@ class TestPrecisionDebugger(TestCase): "dump_path": "/absolute_path", "rank": [], "step": [], - "level": "L0" + "level": "L1" } common_config = CommonConfig(json_config) task_config = BaseConfig(json_config) handler = Handler() - with patch("msprobe.mindspore.debugger.precision_debugger.parse_json_config", - return_value=[common_config, task_config]), \ + mock_get_mode = MagicMock() + mock_parse_json_config = MagicMock() + with patch("msprobe.mindspore.debugger.precision_debugger.parse_json_config", new=mock_parse_json_config), \ + patch.object(PrecisionDebugger, "_get_execution_mode", new=mock_get_mode), \ patch("msprobe.mindspore.debugger.precision_debugger.TaskHandlerFactory.create", return_value=handler): + mock_get_mode.return_value = MsConst.GRAPH_GE_MODE + mock_parse_json_config.return_value = [common_config, task_config] debugger = PrecisionDebugger() + self.assertEqual(Runtime.step_count, 0) + self.assertFalse(Runtime.is_running) debugger.start() - self.assertTrue(isinstance(debugger.config, DebuggerConfig)) - self.assertTrue(Handler.called) + self.assertTrue(Runtime.is_running) + self.assertTrue(isinstance(debugger.config, DebuggerConfig)) + self.assertTrue(Handler.called) + + mock_get_mode.return_value = MsConst.PYNATIVE_MODE + with patch("msprobe.mindspore.debugger.precision_debugger.Service") as mock_Service: + debugger = PrecisionDebugger() + debugger.start() + service = mock_Service.return_value + mock_Service.assert_called_with(debugger.config) + service.start.assert_called_with(None) PrecisionDebugger._instance = None with self.assertRaises(Exception) as context: debugger.start() self.assertEqual(str(context.exception), "No instance of PrecisionDebugger found.") + + with patch("msprobe.mindspore.debugger.precision_debugger.parse_json_config", new=mock_parse_json_config), \ + patch.object(PrecisionDebugger, "_get_execution_mode", new=mock_get_mode), \ + patch("msprobe.mindspore.debugger.precision_debugger.TaskHandlerFactory.create", return_value=handler): + common_config.task = Const.FREE_BENCHMARK + mock_get_mode.return_value = MsConst.PYNATIVE_MODE + mock_parse_json_config.return_value = [common_config, task_config] + Handler.called = False + debugger = PrecisionDebugger() + debugger.start() + self.assertTrue(Handler.called) + + def test_stop_step(self): + class MockPrecisionDebugger: + def __init__(self): + self.task = Const.TENSOR + self.service = None + PrecisionDebugger._instance = None + with self.assertRaises(Exception) as context: + PrecisionDebugger.stop() + self.assertEqual(str(context.exception), "PrecisionDebugger instance is not created.") + with self.assertRaises(Exception) as context: + PrecisionDebugger.step() + self.assertEqual(str(context.exception), "PrecisionDebugger instance is not created.") + PrecisionDebugger._instance = MockPrecisionDebugger() + Runtime.is_running = True + PrecisionDebugger.stop() + self.assertFalse(Runtime.is_running) + Runtime.step_count = 0 + PrecisionDebugger.step() + self.assertEqual(Runtime.step_count, 1) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py index fb408e83b..4954acc11 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_ms_config.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -from unittest import TestCase +import unittest from unittest.mock import patch, mock_open from msprobe.core.common.const import Const @@ -22,7 +22,7 @@ from msprobe.mindspore.ms_config import (parse_json_config, parse_task_config, TensorConfig, StatisticsConfig, OverflowCheckConfig, FreeBenchmarkConfig) -class TestMsConfig(TestCase): +class TestMsConfig(unittest.TestCase): def test_parse_json_config(self): mock_json_data = { "dump_path": "./dump/", @@ -64,6 +64,22 @@ class TestMsConfig(TestCase): task_config = parse_task_config("overflow_check", mock_json_config) self.assertTrue(isinstance(task_config, OverflowCheckConfig)) + mock_json_config.update({"overflow_check": {"overflow_nums": "1"}}) + with self.assertRaises(Exception) as context: + task_config = parse_task_config("overflow_check", mock_json_config) + self.assertEqual(str(context.exception), "overflow_nums is invalid, it should be an integer") + + mock_json_config.update({"overflow_check": {"overflow_nums": 0}}) + with self.assertRaises(Exception) as context: + task_config = parse_task_config("overflow_check", mock_json_config) + self.assertEqual(str(context.exception), "overflow_nums should be -1 or positive integer") + + mock_json_config.update({"overflow_check": {"overflow_nums": 1}}) + mock_json_config.update({"overflow_check": {"check_mode": "core"}}) + with self.assertRaises(Exception) as context: + task_config = parse_task_config("overflow_check", mock_json_config) + self.assertEqual(str(context.exception), "check_mode is invalid") + task_config = parse_task_config("free_benchmark", mock_json_config) self.assertTrue(isinstance(task_config, FreeBenchmarkConfig)) -- Gitee From 0f9eeee01001fc7ce7825208b7bbdc73119864e2 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 10 Aug 2024 18:02:41 +0800 Subject: [PATCH 241/791] fuzzy match bugfix --- .../msprobe/core/compare/acc_compare.py | 16 +++++----- .../msprobe/core/compare/npy_compare.py | 6 ++-- .../msprobe/core/compare/utils.py | 12 ++++---- .../msprobe/mindspore/compare/ms_compare.py | 21 +++++++------ .../msprobe/pytorch/compare/pt_compare.py | 30 +++++++------------ 5 files changed, 39 insertions(+), 46 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 7705a748d..05eceb4f5 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -1,6 +1,6 @@ import multiprocessing import pandas as pd -from msprobe.core.common.const import CompareConst +from msprobe.core.common.const import CompareConst, Const from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ get_error_message from msprobe.core.common.exceptions import FileCheckException @@ -9,6 +9,7 @@ from msprobe.core.compare.multiprocessing_compute import _handle_multi_process from msprobe.core.common.log import logger from msprobe.core.compare.check import check_graph_mode, check_struct_match, fuzzy_check_op + class Comparator: def __init__(self): @@ -74,8 +75,7 @@ class Comparator: logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) is_match = False return is_match and struct_match - - + def match_op(self, npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): if self.check_op(npu_queue[-1], b_op, fuzzy_match): @@ -87,8 +87,8 @@ class Comparator: return n_index, len(bench_queue) - 1 return -1, -1 - def compare_by_op(self,op_name, op_name_mapping_dict, input_parma): - npu_bench_name_list = op_name_mapping_dict[op_name] + def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): + npu_bench_name_list = op_name_mapping_dict[npu_op_name] data_name = npu_bench_name_list[1] error_file, relative_err, error_flag = None, None, False if data_name == '-1' or data_name == -1: # 没有真实数据路径 @@ -97,8 +97,8 @@ class Comparator: else: try: read_npy_data=getattr(self,"read_npy_data") - n_value = read_npy_data(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) - b_value = read_npy_data(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) + n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.PT_SUFFIX) + b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.PT_SUFFIX) except IOError as error: error_file = error.filename n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE @@ -113,7 +113,7 @@ class Comparator: relative_err = get_relative_err(n_value, b_value) n_value, b_value = reshape_value(n_value, b_value) - err_msg = get_error_message(n_value, b_value, op_name, error_flag, error_file=error_file) + err_msg = get_error_message(n_value, b_value, npu_op_name, error_flag, error_file=error_file) result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=relative_err) if npu_bench_name_list[0] != npu_bench_name_list[1]: diff --git a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py index 0c75076c5..279f950f9 100644 --- a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py @@ -5,7 +5,7 @@ from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.log import logger -def handle_inf_nan(n_value, b_value): +def handle_inf_nan(n_value, b_value): """处理inf和nan的数据""" n_inf = np.isinf(n_value) b_inf = np.isinf(b_value) @@ -54,7 +54,7 @@ def reshape_value(n_value, b_value): return n_value, b_value -def get_error_message(n_value, b_value, op_name, error_flag, error_file=None): +def get_error_message(n_value, b_value, npu_op_name, error_flag, error_file=None): """获取异常情况的错误信息""" if error_flag: if n_value == CompareConst.READ_NONE: @@ -71,7 +71,7 @@ def get_error_message(n_value, b_value, op_name, error_flag, error_file=None): if not n_value.shape: return "This is type of scalar data, can not compare." if n_value.dtype != b_value.dtype: - logger.warning("Dtype of NPU and bench Tensor do not match: {}".format(op_name)) + logger.warning("Dtype of NPU and bench Tensor do not match: {}".format(npu_op_name)) return "Dtype of NPU and bench Tensor do not match." return "" diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 63b745432..142269724 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -69,34 +69,34 @@ def read_op(op_data, op_name): if 'forward' in op_name: if 'input_args' in op_data: input_item = op_data['input_args'] - input_parsed_list = op_item_parse(input_item, op_name + '_input', None) + input_parsed_list = op_item_parse(input_item, op_name + '.input', None) op_parsed_list = input_parsed_list.copy() input_parsed_list.clear() if 'input_kwargs' in op_data: kwargs_item = op_data['input_kwargs'] if isinstance(kwargs_item, dict) and "type" in kwargs_item or isinstance(kwargs_item, list): - kwarg_parsed_list = op_item_parse(kwargs_item, op_name + '_input', None) + kwarg_parsed_list = op_item_parse(kwargs_item, op_name + '.input', None) op_parsed_list += kwarg_parsed_list kwarg_parsed_list.clear() elif kwargs_item: for kwarg in kwargs_item: - kwarg_parsed_list = op_item_parse(kwargs_item[kwarg], op_name + '_input.' + kwarg, None) + kwarg_parsed_list = op_item_parse(kwargs_item[kwarg], op_name + '.input.' + kwarg, None) op_parsed_list += kwarg_parsed_list kwarg_parsed_list.clear() if 'output' in op_data: output_item = op_data['output'] - output_parsed_list = op_item_parse(output_item, op_name + '_output', None) + output_parsed_list = op_item_parse(output_item, op_name + '.output', None) op_parsed_list += output_parsed_list output_parsed_list.clear() if 'backward' in op_name: if 'input' in op_data: input_item = op_data['input'] - input_parsed_list = op_item_parse(input_item, op_name + '_input', None) + input_parsed_list = op_item_parse(input_item, op_name + '.input', None) op_parsed_list = input_parsed_list.copy() input_parsed_list.clear() if 'output' in op_data: output_item = op_data['output'] - output_parsed_list = op_item_parse(output_item, op_name + '_output', None) + output_parsed_list = op_item_parse(output_item, op_name + '.output', None) op_parsed_list += output_parsed_list output_parsed_list.clear() return op_parsed_list diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index be7439cb0..278e42a23 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -15,29 +15,31 @@ from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException + class MSComparator (Comparator): def __init__(self): self.frame_name=MSComparator.__name__ - def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): + def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): cos_result = [] max_err_result = [] max_relative_err_result = [] err_mess = [] one_thousand_err_ratio_result = [] five_thousand_err_ratio_result = [] - is_print_compare_log = input_parma.get("is_print_compare_log") + is_print_compare_log = input_param.get("is_print_compare_log") for i in range(len(result_df)): - op_name = result_df.iloc[i, 0] + npu_op_name = result_df.iloc[i, 0] + bench_op_name = result_df.iloc[i, 1] if is_print_compare_log: - logger.info("start compare: {}".format(op_name)) + logger.info("start compare: {}".format(npu_op_name)) cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = self.compare_by_op( - op_name, dump_path_dict, input_parma) + npu_op_name, bench_op_name, dump_path_dict, input_param) if is_print_compare_log: logger.info( "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " - "five_thousand_err_ratio {}".format(op_name, cos_sim, max_abs_err, max_relative_err, err_msg, + "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, err_msg, one_thousand_err_ratio, five_thousand_err_ratio)) cos_result.append(cos_sim) max_err_result.append(max_abs_err) @@ -178,7 +180,8 @@ class MSComparator (Comparator): if auto_analyze: advisor = Advisor(result_df, output_path) advisor.analysis() - + + def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) @@ -188,7 +191,7 @@ def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - msComparator=MSComparator() + msComparator = MSComparator() msComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, - md5_compare=md5_compare) \ No newline at end of file + md5_compare=md5_compare) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index a947a12f6..afef69ad9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -20,24 +20,25 @@ class PTComparator (Comparator): def __init__(self): self.frame_name=PTComparator.__name__ - def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): + def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): cos_result = [] max_err_result = [] max_relative_err_result = [] err_mess = [] one_thousand_err_ratio_result = [] five_thousand_err_ratio_result = [] - is_print_compare_log = input_parma.get("is_print_compare_log") + is_print_compare_log = input_param.get("is_print_compare_log") for i in range(len(result_df)): - op_name = result_df.iloc[i, 0] + npu_op_name = result_df.iloc[i, 0] + bench_op_name = result_df.iloc[i, 1] if is_print_compare_log: - logger.info("start compare: {}".format(op_name)) + logger.info("start compare: {}".format(npu_op_name)) cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = self.compare_by_op( - op_name, dump_path_dict, input_parma) + npu_op_name, bench_op_name, dump_path_dict, input_param) if is_print_compare_log: logger.info( "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " - "five_thousand_err_ratio {}".format(op_name, cos_sim, max_abs_err, max_relative_err, err_msg, + "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, err_msg, one_thousand_err_ratio, five_thousand_err_ratio)) cos_result.append(cos_sim) max_err_result.append(max_abs_err) @@ -104,7 +105,7 @@ class PTComparator (Comparator): if both_empty or no_change: continue - n_match_point, b_match_point = super().match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) + n_match_point, b_match_point = super().match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) if n_match_point == -1 and b_match_point == -1: continue n_match_data = npu_ops_queue[n_match_point] @@ -179,8 +180,7 @@ class PTComparator (Comparator): advisor = Advisor(result_df, output_path) advisor.analysis() - - + def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) @@ -190,17 +190,7 @@ def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - ptComparator=PTComparator() + ptComparator = PTComparator() ptComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) - - - - - - - - - - \ No newline at end of file -- Gitee From 73608d4911e5359aabad784e4df2ac043cded081 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 10 Aug 2024 18:32:29 +0800 Subject: [PATCH 242/791] fuzzy match bugfix ut bugfix --- .../msprobe/test/pytorch_ut/compare/test_acc_compare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py index b08b09c85..4480784e1 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py @@ -195,15 +195,15 @@ op_name = "Tensor.add_0.0.forward" op_result = [ {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward_input.0'}, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.input.0'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, - 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.forward_input.1'}, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.forward.input.1'}, {'full_op_name': 'Tensor.add_0.0.forward_input.alpha.0', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward_output.0'}] + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.output.0'}] class TestUtilsMethods(unittest.TestCase): -- Gitee From 7b61cedf331c66c6864d185cf8ba7600e7494145 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 10 Aug 2024 18:41:42 +0800 Subject: [PATCH 243/791] ut bugfix --- .../msprobe/test/pytorch_ut/compare/test_acc_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py index 4480784e1..ad8e9453e 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py @@ -199,7 +199,7 @@ op_result = [ {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.forward.input.1'}, - {'full_op_name': 'Tensor.add_0.0.forward_input.alpha.0', 'dtype': "", 'shape': '[]', 'md5': None, + {'full_op_name': 'Tensor.add_0.0.forward.input.alpha.0', 'dtype': "", 'shape': '[]', 'md5': None, 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, -- Gitee From d167e40e551908ec9df2296c77b5c05c5cbd4cd8 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Sun, 11 Aug 2024 21:41:39 +0800 Subject: [PATCH 244/791] data dump ut --- .../core/data_dump/data_processor/base.py | 3 + .../data_processor/mindspore_processor.py | 3 - .../data_processor/pytorch_processor.py | 5 +- .../test_data_processor/test_base.py | 210 ++++++++ .../test_pytorch_processor.py | 456 ++++++++++++++++++ 5 files changed, 670 insertions(+), 7 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_base.py create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_pytorch_processor.py diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index fcb522d11..54a9b927e 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -187,6 +187,9 @@ class BaseDataProcessor: def analyze_pre_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): pass + + def analyze_element(self, element): + return self.recursive_apply_transform(element, self.analyze_single_element) def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): api_info_struct = {} diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index b28817e4a..9eda4e2c6 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -110,9 +110,6 @@ class MindsporeDataProcessor(BaseDataProcessor): return self._analyze_builtin(element) return {} - def analyze_element(self, element): - return self.recursive_apply_transform(element, self.analyze_single_element) - def _analyze_tensor(self, tensor, suffix): tensor_stat = self.get_stat_info(tensor) tensor_json = { diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 1af076650..ae7d37f35 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -138,9 +138,6 @@ class PytorchDataProcessor(BaseDataProcessor): return self._analyze_builtin(element) return {} - def analyze_element(self, element): - return self.recursive_apply_transform(element, self.analyze_single_element) - def _analyze_tensor(self, tensor, suffix): tensor_stat = self.get_stat_info(tensor) tensor_json = {} @@ -226,7 +223,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): raise MsprobeException(MsprobeException.OVERFLOW_NUMS_ERROR, str(self.real_overflow_dump_times)) def check_overflow_npu(self): - if self.overflow_debug_mode_enalbe(): + if self.overflow_debug_mode_enable(): float_status = torch.zeros(self.bits_for_overflow).npu() result = torch_npu.npu_get_float_status(float_status, OverflowConst.OVERFLOW_DEBUG_MODE) if result.cpu()[0] != 0: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_base.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_base.py new file mode 100644 index 000000000..3e343528c --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_base.py @@ -0,0 +1,210 @@ +import unittest +from unittest.mock import patch, MagicMock +import os + +import numpy as np +from msprobe.core.common.log import logger +from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs, \ + TensorStatInfo, BaseDataProcessor + + +class TestModuleForwardInputsOutputs(unittest.TestCase): + + @patch('msprobe.core.common.utils.convert_tuple') + def test_args_tuple(self, mock_convert_tuple): + mock_convert_tuple.return_value = (1, 2, 3) + module = ModuleForwardInputsOutputs(args=(1, 2, 3), kwargs=None, output=None) + self.assertEqual(module.args_tuple, (1, 2, 3)) + + @patch('msprobe.core.common.utils.convert_tuple') + def test_output_tuple(self, mock_convert_tuple): + mock_convert_tuple.return_value = (4, 5, 6) + module = ModuleForwardInputsOutputs(args=None, kwargs=None, output=(4, 5, 6)) + self.assertEqual(module.output_tuple, (4, 5, 6)) + + def test_concat_args_and_kwargs(self): + module = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3, 'b': 4}, output=None) + self.assertEqual(module.concat_args_and_kwargs(), (1, 2, 3, 4)) + + +class TestModuleBackwardInputsOutputs(unittest.TestCase): + + @patch('msprobe.core.common.utils.convert_tuple') + def test_grad_input_tuple(self, mock_convert_tuple): + mock_convert_tuple.return_value = (1, 2, 3) + module = ModuleBackwardInputsOutputs(grad_output=None, grad_input=(1, 2, 3)) + self.assertEqual(module.grad_input_tuple, (1, 2, 3)) + + @patch('msprobe.core.common.utils.convert_tuple') + def test_grad_output_tuple(self, mock_convert_tuple): + mock_convert_tuple.return_value = (4, 5, 6) + module = ModuleBackwardInputsOutputs(grad_output=(4, 5, 6), grad_input=None) + self.assertEqual(module.grad_output_tuple, (4, 5, 6)) + + +class TestTensorStatInfo(unittest.TestCase): + + def test_tensor_stat_info(self): + tensor_info = TensorStatInfo(max_val=10, min_val=1, mean_val=5, norm_val=3) + self.assertEqual(tensor_info.max, 10) + self.assertEqual(tensor_info.min, 1) + self.assertEqual(tensor_info.mean, 5) + self.assertEqual(tensor_info.norm, 3) + + +class TestBaseDataProcessor(unittest.TestCase): + + def setUp(self): + self.config = MagicMock() + self.data_writer = MagicMock() + self.processor = BaseDataProcessor(self.config, self.data_writer) + + self.processor.current_api_or_module_name = "test_api" + self.processor.api_data_category = "input" + + @patch('inspect.stack') + def test_analyze_api_call_stack(self, mock_stack): + mock_stack.return_value = [ + (None, 'file0.py', 0, 'function0', ['code line 0'], None), + (None, 'file1.py', 10, 'function1', ['code line 1'], None), + (None, 'file2.py', 20, 'function2', ['code line 2'], None), + (None, 'file3.py', 30, 'function3', ['code line 3'], None), + (None, 'file4.py', 40, 'function4', ['code line 4'], None), + (None, 'file5.py', 50, 'function5', ['code line 5'], None), + (None, 'file6.py', 60, 'function6', ['code line 6'], None), + (None, 'file7.py', 70, 'function7', ['code line 7'], None), + ] + result = BaseDataProcessor.analyze_api_call_stack('test_stack') + expected_output = { + 'test_stack': [ + 'File file5.py, line 50, in function5, \n code line 5', + 'File file6.py, line 60, in function6, \n code line 6', + 'File file7.py, line 70, in function7, \n code line 7', + ] + } + self.assertEqual(result, expected_output) + + def test_convert_numpy_to_builtin(self): + self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.int32(5)), (5, 'int32')) + self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.float64(3.14)), (3.14, 'float64')) + self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.bool_(True)), (True, 'bool_')) + self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.str_('test')), ('test', 'str_')) + self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(5), (5, '')) + + def test_analyze_numpy(self): + result = BaseDataProcessor._analyze_numpy(5, 'int32') + self.assertEqual(result, {'type': 'int32', 'value': 5}) + + def test_get_special_types(self): + self.assertIn(int, BaseDataProcessor.get_special_types()) + + def test_recursive_apply_transform(self): + transform = lambda x, _: x * 2 + self.assertEqual(BaseDataProcessor.recursive_apply_transform(2, transform), 4) + self.assertEqual(BaseDataProcessor.recursive_apply_transform([1, 2], transform), [2, 4]) + self.assertEqual(BaseDataProcessor.recursive_apply_transform((1, 2), transform), (2, 4)) + self.assertEqual(BaseDataProcessor.recursive_apply_transform({'a': 1}, transform), {'a': 2}) + + @patch.object(logger, 'warning') + def test_recursive_apply_transform_with_warning(self, mock_logger): + transform = lambda x, _: x * 2 + BaseDataProcessor.recursive_apply_transform({1, 2, 3}, transform) + print(mock_logger.call_args_list) + mock_logger.assert_called_with(f"Data type {type({1, 2, 3})} is not supported.") + + def test_if_return_forward_new_output(self): + self.processor._return_forward_new_output = True + self.assertTrue(self.processor.if_return_forward_new_output()) + + def test_get_forward_new_output(self): + self.processor._return_forward_new_output = True + self.processor._forward_new_output = "new_output" + self.assertEqual(self.processor.get_forward_new_output(), "new_output") + self.assertFalse(self.processor._return_forward_new_output) + + def test_update_iter(self): + self.processor.update_iter(5) + self.assertEqual(self.processor.current_iter, 5) + + def test_visit_and_clear_overflow_status(self): + self.processor.has_overflow = True + self.processor.visit_and_clear_overflow_status("new_api") + self.assertFalse(self.processor.has_overflow) + self.assertEqual(self.processor.current_api_or_module_name, "new_api") + + def test_is_dump_for_data_mode(self): + self.config.data_mode = ["all"] + self.assertTrue(self.processor.is_dump_for_data_mode("forward", "input")) + self.config.data_mode = ["forward"] + self.assertTrue(self.processor.is_dump_for_data_mode("forward", "input")) + self.config.data_mode = ["input"] + self.assertTrue(self.processor.is_dump_for_data_mode("forward", "input")) + self.config.data_mode = ["backward"] + self.assertFalse(self.processor.is_dump_for_data_mode("forward", "input")) + + @patch.object(BaseDataProcessor, 'analyze_element') + def test_analyze_forward(self, mock_analyze_element): + mock_analyze_element.side_effect = lambda args: args + module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=(4, 5)) + self.config.data_mode = ["all"] + result = self.processor.analyze_forward("test_forward", None, module_io) + expected = { + "test_forward": { + "input_args": (1, 2), + "input_kwargs": {'a': 3}, + "output": (4, 5) + } + } + self.assertEqual(result, expected) + + @patch.object(BaseDataProcessor, 'analyze_element') + def test_analyze_pre_forward_inplace(self, mock_analyze_element): + mock_analyze_element.side_effect = lambda args: args + module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=None) + self.config.data_mode = ["all"] + result = self.processor.analyze_pre_forward_inplace("test_pre_forward", module_io) + expected = { + "test_pre_forward": { + "input_args": (1, 2), + "input_kwargs": {'a': 3} + } + } + self.assertEqual(result, expected) + + @patch.object(BaseDataProcessor, 'analyze_element') + def test_analyze_forward_inplace(self, mock_analyze_element): + mock_analyze_element.side_effect = lambda args: args + module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=None) + self.config.data_mode = ["all"] + result = self.processor.analyze_forward_inplace("test_forward_inplace", module_io) + expected = { + "test_forward_inplace": { + "output": (1, 2, 3) + } + } + self.assertEqual(result, expected) + + @patch.object(BaseDataProcessor, 'analyze_element') + def test_analyze_backward(self, mock_analyze_element): + mock_analyze_element.side_effect = lambda args: args + module_io = ModuleBackwardInputsOutputs(grad_input=(1, 2), grad_output=(3, 4)) + self.config.data_mode = ["all"] + result = self.processor.analyze_backward("test_backward", None, module_io) + expected = { + "test_backward": { + "input": (1, 2), + "output": (3, 4) + } + } + self.assertEqual(result, expected) + + def test_get_save_file_path(self): + self.config.framework = "pytorch" + result = self.processor.get_save_file_path("suffix") + expected_file_name = "test_api.input.suffix.pt" + expected_file_path = os.path.join(self.data_writer.dump_tensor_data_dir, expected_file_name) + self.assertEqual(result, (expected_file_name, expected_file_path)) + + +if __name__ == '__main__': + unittest.main() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_pytorch_processor.py new file mode 100644 index 000000000..94d126a4b --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_pytorch_processor.py @@ -0,0 +1,456 @@ +import sys +import unittest +from unittest.mock import patch, MagicMock, Mock +import zlib + +# mock_torch_npu = MagicMock() +# modules = { +# 'torch_npu': mock_torch_npu, +# 'torch_npu.npu': mock_torch_npu.npu, +# } +# patcher = patch.dict('sys.modules', modules) +# patcher.start() + +import torch +from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs, BaseDataProcessor +from msprobe.core.data_dump.data_processor.pytorch_processor import ( + PytorchDataProcessor, + FreeBenchmarkDataProcessor, + TensorDataProcessor, + KernelDumpDataProcessor, + OverflowCheckDataProcessor +) + +from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common.file_check import change_mode, path_len_exceeds_limit +from msprobe.core.common.const import Const, OverflowConst, FileCheckConst +sys.modules['torch_npu'] = Mock() +sys.modules['torch_npu.npu'] = Mock() +sys.modules['torch_npu._C'] = Mock() +torch_npu = sys.modules['torch_npu'] + + + +class TestPytorchDataProcessor(unittest.TestCase): + + def setUp(self): + self.config = MagicMock() + self.data_writer = MagicMock() + self.processor = PytorchDataProcessor(self.config, self.data_writer) + + def test_get_md5_for_tensor(self): + tensor = torch.tensor([1, 2, 3]) + expected_md5 = zlib.crc32(tensor.numpy().tobytes()) + self.assertEqual(self.processor.get_md5_for_tensor(tensor), f"{expected_md5:08x}") + + def test_analyze_device_in_kwargs(self): + device = torch.device('npu:0') + result = self.processor.analyze_device_in_kwargs(device) + expected = {'type': 'torch.device', 'value': 'cuda:0'} + self.assertEqual(result, expected) + + def test_analyze_dtype_in_kwargs(self): + dtype = torch.float32 + result = self.processor.analyze_dtype_in_kwargs(dtype) + expected = {'type': 'torch.dtype', 'value': 'torch.float32'} + self.assertEqual(result, expected) + + def test_get_stat_info_float(self): + tensor = torch.tensor([1.0, 2.0, 3.0]) + result = self.processor.get_stat_info(tensor) + self.assertEqual(result.max, 3.0) + self.assertEqual(result.min, 1.0) + self.assertEqual(result.mean, 2.0) + self.assertEqual(result.norm, torch.norm(tensor).item()) + + def test_get_stat_info_int(self): + tensor = torch.tensor([1, 2, 3], dtype=torch.int32) + result = self.processor.get_stat_info(tensor) + self.assertEqual(result.max, 3) + self.assertEqual(result.min, 1) + self.assertEqual(result.mean, 2) + self.assertEqual(result.norm, torch.norm(tensor.float()).item()) + + def test_get_stat_info_empty(self): + tensor = torch.tensor([]) + result = self.processor.get_stat_info(tensor) + self.assertIsNone(result.max) + self.assertIsNone(result.min) + self.assertIsNone(result.mean) + self.assertIsNone(result.norm) + + def test_get_stat_info_bool(self): + tensor = torch.tensor([True, False, True]) + result = self.processor.get_stat_info(tensor) + self.assertEqual(result.max, True) + self.assertEqual(result.min, False) + self.assertIsNone(result.mean) + self.assertIsNone(result.norm) + + def test_handle_tensor_extremum_nan_inf_all_nan(self): + tensor = torch.tensor([float('nan'), float('nan')]) + result = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') + self.assertTrue(torch.isnan(result)) + + def test_handle_tensor_extremum_nan_inf_all_inf(self): + tensor = torch.tensor([float('inf'), float('inf')]) + result = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') + self.assertTrue(torch.isinf(result)) + + def test_handle_tensor_extremum_nan_inf_all_negative_inf(self): + tensor = torch.tensor([float('-inf'), float('-inf')]) + result = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') + self.assertTrue(torch.isinf(result) and result < 0) + + def test_handle_tensor_extremum_nan_inf_mixed(self): + tensor = torch.tensor([1.0, float('nan'), 3.0, float('-inf'), 2.0]) + result_max = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') + result_min = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') + self.assertEqual(result_max, 3.0) + self.assertEqual(result_min, 1.0) + + def test_handle_tensor_extremum_nan_inf_mixed_with_inf(self): + tensor = torch.tensor([1.0, float('nan'), 3.0, float('inf'), 2.0]) + result_max = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') + result_min = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') + self.assertEqual(result_max, 3.0) + self.assertEqual(result_min, 1.0) + + def test_handle_tensor_extremum_nan_inf_no_inf_nan(self): + tensor = torch.tensor([1.0, 2.0, 3.0]) + result_max = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') + result_min = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') + self.assertEqual(result_max, 3.0) + self.assertEqual(result_min, 1.0) + + def test_handle_tensor_extremum_nan_inf_all_inf_nan(self): + tensor = torch.tensor([float('nan'), float('inf'), float('-inf')]) + result_max = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') + result_min = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') + self.assertTrue(torch.isnan(result_max)) + self.assertTrue(torch.isnan(result_min)) + + def test_analyze_builtin(self): + result = self.processor._analyze_builtin(slice(1, 10, 2)) + expected = {'type': 'slice', 'value': [1, 10, 2]} + self.assertEqual(result, expected) + + def test_analyze_torch_size(self): + size = torch.Size([3, 4, 5]) + result = self.processor._analyze_torch_size(size) + expected = {'type': 'torch.Size', 'value': [3, 4, 5]} + self.assertEqual(result, expected) + + def test_get_special_types(self): + special_types = self.processor.get_special_types() + self.assertIn(torch.Tensor, special_types) + + @patch.object(PytorchDataProcessor, 'get_md5_for_tensor') + def test_analyze_tensor(self, get_md5_for_tensor): + get_md5_for_tensor.return_value = 'mocked_md5' + tensor = torch.tensor([1.0, 2.0, 3.0]) + self.config.summary_mode = 'md5' + result = self.processor._analyze_tensor(tensor, 'suffix') + expected = { + 'type': 'torch.Tensor', + 'dtype': str(tensor.dtype), + 'shape': tensor.shape, + 'Max': 3.0, + 'Min': 1.0, + 'Mean': 2.0, + 'Norm': torch.norm(tensor).item(), + 'requires_grad': tensor.requires_grad, + 'md5': 'mocked_md5' + } + self.assertDictEqual(expected, result) + + def test_analyze_tensor_with_empty_tensor(self): + tensor = torch.tensor([]) + result = self.processor._analyze_tensor(tensor, 'suffix') + self.assertEqual(result['Max'], None) + self.assertEqual(result['Min'], None) + self.assertEqual(result['Mean'], None) + self.assertEqual(result['Norm'], None) + + def test_analyze_tensor_with_inf_and_nan(self): + tensor = torch.tensor([1.0, float('inf'), float('nan'), -float('inf')]) + result = self.processor._analyze_tensor(tensor, 'suffix') + self.assertEqual(result['Max_except_inf_nan'], 1.0) + self.assertEqual(result['Min_except_inf_nan'], 1.0) + + +class TestTensorDataProcessor(unittest.TestCase): + + def setUp(self): + self.config = MagicMock() + self.data_writer = MagicMock() + self.processor = TensorDataProcessor(self.config, self.data_writer) + + @patch('torch.save') + @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) + @patch('msprobe.core.common.file_check.change_mode') + def test_analyze_tensor(self, mock_change_mode, mock_save): + tensor = torch.tensor([1.0, 2.0, 3.0]) + suffix = 'suffix' + result = self.processor._analyze_tensor(tensor, suffix) + mock_save.assert_called_once() + mock_change_mode.assert_called_once() + self.assertIn('data_name', result) + + +class TestOverflowCheckDataProcessor(unittest.TestCase): + + def setUp(self): + self.config = MagicMock() + self.config.overflow_nums = 1 + self.data_writer = MagicMock() + self.processor = OverflowCheckDataProcessor(self.config, self.data_writer) + self.current_api_or_module_name = "test_api_name" + self.api_data_category = "input" + sys.modules['torch_npu'] = Mock() + sys.modules['torch_npu.npu'] = Mock() + sys.modules['torch_npu._C'] = Mock() + + @patch('torch.save') + @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) + @patch('msprobe.core.common.file_check.change_mode') + def test_maybe_save_overflow_data_and_check_overflow_times(self, mock_change_mode, mock_path_len_exceeds_limit, mock_save): + self.processor.has_overflow = True + self.processor.cached_tensors_and_file_paths = {'dummy_path': torch.tensor([1.0, 2.0, 3.0])} + + # First call should save the tensor and not raise an exception + self.processor.maybe_save_overflow_data_and_check_overflow_times() + mock_save.assert_called_once() + mock_path_len_exceeds_limit.assert_called_once() + mock_change_mode.assert_called_once() + + # Second call should raise an exception due to overflow nums limit + with self.assertRaises(MsprobeException) as context: + self.processor.maybe_save_overflow_data_and_check_overflow_times() + + self.assertEqual(str(context.exception), MsprobeException.OVERFLOW_NUMS_ERROR) + + def test_inc_and_check_overflow_times(self): + self.processor.real_overflow_dump_times = 0 + self.processor.overflow_nums = 1 + self.processor.has_overflow = True + + # First increment should not raise an exception + self.processor.inc_and_check_overflow_times() + self.assertEqual(self.processor.real_overflow_dump_times, 1) + + # Second increment should raise an exception + with self.assertRaises(MsprobeException) as context: + self.processor.inc_and_check_overflow_times() + + self.assertEqual(str(context.exception), MsprobeException.OVERFLOW_NUMS_ERROR) + + @patch('os.getenv', return_value=Const.ENV_ENABLE) + def test_overflow_debug_mode_enable(self, mock_getenv): + result = self.processor.overflow_debug_mode_enable() + self.assertTrue(result) + mock_getenv.assert_called_once_with(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) + + @patch('numpy.isinf', return_value=True) + @patch('numpy.isnan', return_value=False) + def test_analyze_maybe_overflow_tensor(self, mock_isnan, mock_isinf): + tensor_json = {'Max': float('inf'), 'Min': 1.0} + self.processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.processor.has_overflow) + + @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) + @patch.object(BaseDataProcessor, 'get_save_file_path', return_value=['test_api_name', 'test_api_name.0.forward.input.pt']) + def test_analyze_tensor(self, mock_path_len_exceeds_limit, mock_get_save_file_path): + tensor = torch.tensor([1.0, 2.0, 3.0]) + suffix = 'suffix' + with patch.object(PytorchDataProcessor, '_analyze_tensor', return_value={'Max': 3.0, 'Min': 1.0}) as mock_super_analyze_tensor: + result = self.processor._analyze_tensor(tensor, suffix) + mock_super_analyze_tensor.assert_called_once_with(tensor, suffix) + self.assertIn('data_name', result) + self.assertTrue(self.processor.has_overflow) + + @patch.object(PytorchDataProcessor, 'analyze_element', return_value=['mocked_result']) + def test_analyze_backward(self, mock_analyze_element): + module_io = ModuleBackwardInputsOutputs(grad_output=(1, 2), grad_input=(3, 4)) + self.config.data_mode = ["all"] + result = self.processor.analyze_backward("test_backward", None, module_io) + expected = { + "test_backward": { + "grad_input": ['mocked_result'], + "grad_output": ['mocked_result'] + } + } + self.assertEqual(result, expected) + + @patch.object(PytorchDataProcessor, 'analyze_element', return_value=['mocked_result']) + def test_analyze_forward(self, mock_analyze_element): + module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=(4, 5)) + self.config.data_mode = ["all"] + result = self.processor.analyze_forward("test_forward", None, module_io) + expected = { + "test_forward": { + "input_args": ['mocked_result'], + "input_kwargs": ['mocked_result'], + "output": ['mocked_result'] + } + } + self.assertEqual(result, expected) + +class TestFreeBenchmarkDataProcessor(unittest.TestCase): + + def setUp(self): + self.config = MagicMock() + self.data_writer = MagicMock() + self.processor = FreeBenchmarkDataProcessor(self.config, self.data_writer) + + @patch('msprobe.pytorch.free_benchmark.FreeBenchmarkCheck.pre_forward') + def test_analyze_pre_forward(self, mock_pre_forward): + module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=None) + self.processor.analyze_pre_forward('test_pre_forward', None, module_io) + mock_pre_forward.assert_called_once() + + @patch('msprobe.pytorch.free_benchmark.FreeBenchmarkCheck.forward', return_value=(None, [])) + def test_analyze_forward(self, mock_forward): + module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=(4, 5)) + self.processor.analyze_forward('test_forward', None, module_io) + mock_forward.assert_called_once() + + @patch('msprobe.pytorch.free_benchmark.FreeBenchmarkCheck.backward') + def test_analyze_backward(self, mock_backward): + module_io = ModuleBackwardInputsOutputs(grad_output=(torch.tensor([1.0, 2.0]),), grad_input=None) + self.processor.analyze_backward('test_backward', None, module_io) + mock_backward.assert_called_once() + + +# class TestKernelDumpDataProcessor(unittest.TestCase): + +# def setUp(self): +# self.config = MagicMock() +# self.config.is_forward_acl_dump = True +# self.config.acl_config = "dummy_acl_config" +# self.config.backward_input = {'test_module': 'dummy_path'} +# self.data_writer = MagicMock() +# self.processor = KernelDumpDataProcessor(self.config, self.data_writer) + +# @patch('torch_npu.npu.synchronize') +# @patch('torch_npu.npu.init_dump') +# @patch('torch_npu.npu.set_dump') +# @patch('torch_npu.npu.finalize_dump') +# def test_forward_acl_dump(self, mock_finalize_dump, mock_set_dump, mock_init_dump, mock_synchronize): +# module = MagicMock() +# module.forward = MagicMock(return_value=torch.tensor([1.0, 2.0, 3.0])) +# module_io = MagicMock() +# module_io.args = (1, 2) +# module_io.kwargs = {'a': 3} + +# KernelDumpDataProcessor.forward_init_status = False + +# self.processor.forward_acl_dump('test_module', module, module_io) + +# mock_synchronize.assert_called() +# mock_init_dump.assert_called_once_with() +# mock_set_dump.assert_called_once_with("dummy_acl_config") +# mock_finalize_dump.assert_called_once_with() +# module.forward.assert_called_with(1, 2, a=3) + +# @patch('torch_npu.npu.synchronize') +# @patch('torch_npu.npu.init_dump') +# @patch('torch_npu.npu.set_dump') +# @patch('torch_npu.npu.finalize_dump') +# @patch('torch.load', return_value=torch.tensor([1.0, 2.0, 3.0])) +# def test_dump_mode_backward_acl_dump(self, mock_load, mock_finalize_dump, mock_set_dump, mock_init_dump, mock_synchronize): +# module = MagicMock() +# module.forward = MagicMock(return_value=torch.tensor([1.0, 2.0, 3.0])) +# module_io = MagicMock() +# module_io.args = (1, 2) +# module_io.kwargs = {'a': 3} + +# KernelDumpDataProcessor.forward_init_status = False + +# self.processor.dump_mode_backward_acl_dump('test_module', module, module_io) + +# mock_synchronize.assert_called() +# mock_init_dump.assert_called_once_with() +# mock_set_dump.assert_called_once_with("dummy_acl_config") +# mock_finalize_dump.assert_called_once_with() +# mock_load.assert_called_once_with('dummy_path') +# module.forward.assert_called_with(1, 2, a=3) + +# def test_op_need_trigger(self): +# self.assertTrue(self.processor.op_need_trigger('Tensor.__getitem__.')) +# self.assertFalse(self.processor.op_need_trigger('SomeOtherOp')) + +# @patch.object(KernelDumpDataProcessor, 'forward_acl_dump') +# @patch.object(KernelDumpDataProcessor, 'dump_mode_backward_acl_dump') +# def test_analyze_forward(self, mock_dump_mode_backward_acl_dump, mock_forward_acl_dump): +# self.processor.analyze_forward('test_module', MagicMock(), MagicMock()) +# mock_forward_acl_dump.assert_called_once() +# mock_dump_mode_backward_acl_dump.assert_not_called() + +# self.config.is_forward_acl_dump = False +# self.processor.analyze_forward('test_module', MagicMock(), MagicMock()) +# mock_dump_mode_backward_acl_dump.assert_called_once() +# mock_forward_acl_dump.assert_called_once() # 因为已经被调用过一次 + +# @patch('torch.Tensor.backward') +# def test_acl_backward_dump_status(self, mock_backward): +# output = torch.tensor([1.0, 2.0, 3.0]) +# grad = torch.tensor([0.1, 0.1, 0.1]) +# self.assertTrue(self.processor.acl_backward_dump_status(output, grad, 'test_module')) +# mock_backward.assert_called_once_with(grad, retain_graph=True) + +# output = [torch.tensor([1.0, 2.0, 3.0])] +# self.assertTrue(self.processor.acl_backward_dump_status(output, grad, 'test_module')) +# mock_backward.assert_called_with(grad, retain_graph=True) + +# output = [torch.tensor([1.0, 2.0, 3.0])] +# self.assertFalse(self.processor.acl_backward_dump_status(output, grad, 'unknown_module')) + +# def tearDown(self): +# KernelDumpDataProcessor.forward_init_status = False + +# patcher.stop() +# class TestKernelDumpDataProcessor(unittest.TestCase): + +# def setUp(self): +# self.config = MagicMock() +# self.data_writer = MagicMock() +# self.processor = KernelDumpDataProcessor(self.config, self.data_writer) + +# @patch('torch_npu.npu.synchronize') +# @patch('torch_npu.npu.init_dump') +# @patch('torch_npu.npu.set_dump') +# @patch('torch_npu.npu.finalize_dump') +# def test_forward_acl_dump(self, mock_finalize_dump, mock_set_dump, mock_init_dump, mock_synchronize): +# module = MagicMock() +# module.forward = MagicMock(return_value=torch.tensor([1.0, 2.0, 3.0])) +# module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=None) +# self.processor.forward_acl_dump('test_module', module, module_io) +# mock_synchronize.assert_called() +# mock_init_dump.assert_called_once() +# mock_set_dump.assert_called_once() +# mock_finalize_dump.assert_called_once() + +# @patch('torch_npu.npu.synchronize') +# @patch('torch_npu.npu.init_dump') +# @patch('torch_npu.npu.set_dump') +# @patch('torch_npu.npu.finalize_dump') +# @patch('torch.load', return_value=torch.tensor([1.0, 2.0, 3.0])) +# def test_dump_mode_backward_acl_dump(self, mock_load, mock_finalize_dump, mock_set_dump, mock_init_dump, mock_synchronize): +# module = MagicMock() +# module.forward = MagicMock(return_value=torch.tensor([1.0, 2.0, 3.0])) +# module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=None) +# self.config.backward_input = {'test_module': 'dummy_path'} +# self.processor.dump_mode_backward_acl_dump('test_module', module, module_io) +# mock_synchronize.assert_called() +# mock_init_dump.assert_called_once() +# mock_set_dump.assert_called_once() +# mock_finalize_dump.assert_called_once() +# mock_load.assert_called_once() + +# def test_op_need_trigger(self): +# self.assertTrue(self.processor.op_need_trigger('Tensor.__getitem__.')) +# self.assertFalse(self.processor.op_need_trigger('SomeOtherOp')) + +if __name__ == '__main__': + unittest.main() -- Gitee From 23e2bd22ec90d3582c49b7a51557ce3fc1f5576f Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 12 Aug 2024 09:22:01 +0800 Subject: [PATCH 245/791] clean code --- .../tensor_transport_layer/device_dispatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 42f34bfc6..1a5462203 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -44,7 +44,7 @@ def run_ut_process(xpu_id, compare, consumer_queue, func, config): logger.warning(f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API " f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.") elif api_type in [Const.DISTRIBUTED]: - logger.info(f"{api_name} is not supported for run ut. SKIP.") + logger.info(f"{api_full_name} is not supported for run ut. SKIP.") else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") -- Gitee From dcad560ec2ed0de2fa9e98331bee4c51f0e1ed5d Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 12 Aug 2024 09:30:52 +0800 Subject: [PATCH 246/791] fuzzy match bugfix --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 05eceb4f5..25824721e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -116,7 +116,7 @@ class Comparator: err_msg = get_error_message(n_value, b_value, npu_op_name, error_flag, error_file=error_file) result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=relative_err) - if npu_bench_name_list[0] != npu_bench_name_list[1]: + if npu_op_name != bench_op_name: err_msg += " Fuzzy matching data, the comparison accuracy may be affected." result_list.append(err_msg) return result_list -- Gitee From 4d1ce5ce0870f16654ccab9e204c53f665eee90e Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Mon, 12 Aug 2024 10:08:41 +0800 Subject: [PATCH 247/791] =?UTF-8?q?[profiler\compare=5Ftools]=E6=AF=94?= =?UTF-8?q?=E5=AF=B9=E5=B7=A5=E5=85=B7=E8=B5=84=E6=96=99=E6=96=B0=E5=A2=9E?= =?UTF-8?q?base=5Fstep=E5=92=8Ccomparison=5Fstep=E5=8F=82=E6=95=B0?= =?UTF-8?q?=E6=8F=8F=E8=BF=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/compare_tools/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index 97dcf5b19..2de6d2300 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -148,6 +148,8 @@ python performance_compare.py [基准性能数据文件所在路径] [比对性 | --enable_kernel_compare | 开启kernel性能比对。仅针对NPU与NPU比对的场景。需要使用性能数据中的kernel_details.csv文件。 | 否 | | --enable_api_compare | 开启API性能比对。需要使用性能数据中的trace_view.csv文件。 | 否 | | --disable_details | 隐藏明细比对，只进行统计级比对。 | 否 | +| --base_step | 基准性能数据step ID，配置后使用基准性能数据对应step的数据进行比对。为整数，需配置实际数据存在的step ID，默认未配置，比对所有性能数据，需要与--comparsion_step同时配置。配置示例：--base_step=1。 | 否 | +| --comparsion_step | 比对性能数据step ID，配置后使用比对性能数据对应step的数据进行比对。为整数，需配置实际数据存在的step ID，默认未配置，比对所有性能数据，需要与--base_step同时配置。配置示例：--comparsion_step=1。 | 否 | 说明：以上开关均不设置的情况下，**工具默认开启所有的性能比对**，当用户设置了以上开关，则按照用户设置的开关进行性能比对，示例如下： -- Gitee From 31fc53261e5ab7c5b7f646bbabfdf1fd495589e7 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 12 Aug 2024 11:21:24 +0800 Subject: [PATCH 248/791] online_run_ut add black_list --- .../api_accuracy_checker/run_ut/run_ut.py | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index a1e2e64a9..90c6e9454 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -219,9 +219,7 @@ def run_api_offline(config, compare, api_name_set): continue [_, api_name, _] = api_full_name.split(Const.SEP) try: - if config.black_list and api_name in config.black_list: - continue - if config.white_list and api_name not in config.white_list: + if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue data_info = run_torch_api(api_full_name, config.real_data_path, config.backward_content, api_info_dict) is_fwd_success, is_bwd_success = compare.compare_output(api_full_name, data_info) @@ -265,11 +263,9 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - - if config.white_list: - [_, api_name, _] = api_full_name.split(Const.SEP) - if api_name not in set(config.white_list): - continue + [_, api_name, _] = api_full_name.split(Const.SEP) + if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): + continue dispatcher.update_consume_queue(api_data) def shared_storage_communication_flow(): @@ -285,8 +281,13 @@ def run_api_online(config, compare): if flag_num == 0: dispatcher.stop() break - if isinstance(api_data, ApiData): - dispatcher.update_consume_queue(api_data) + if not isinstance(api_data, ApiData): + continue + api_full_name = api_data.name + [_, api_name, _] = api_full_name.split(Const.SEP) + if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): + continue + dispatcher.update_consume_queue(api_data) if config.online_config.nfs_path: shared_storage_communication_flow() @@ -294,6 +295,19 @@ def run_api_online(config, compare): tcp_communication_flow() +def blacklist_and_whitelist_filter(api_name, black_list, white_list): + """ + run api(api_name) if api_name not in black_list and in white_list. + If api is both in black_list and black_list, black_list first. + return: False for exec api, True for not exec + """ + if black_list and api_name in black_list: + return True + if white_list and api_name not in white_list: + return True + return False + + def is_unsupported_api(api_name): split_name = api_name.split(Const.SEP)[0] flag = split_name in [Const.NPU, Const.DISTRIBUTED] -- Gitee From 3d8d53876a351b04fbb2d5dc15b81ef663da4a17 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 12 Aug 2024 11:42:32 +0800 Subject: [PATCH 249/791] add annotation --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 90c6e9454..c06d8707e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -556,6 +556,8 @@ def run_ut_command(args): logger.error(f"Set device id failed. device id is: {args.device_id}") raise NotImplementedError from error + # 在线预检场景下，不需要外出输出api信息，forward_content, backward_content, real_data_path设置为None + # 离线场景下，forward_content, backward_content, real_data_path从api_info_file中解析 forward_content, backward_content, real_data_path = None, None, None if args.api_info_file: check_link(args.api_info_file) -- Gitee From 67c6240737cae49fa834e418202ae16aeaa58b14 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 12 Aug 2024 14:28:00 +0800 Subject: [PATCH 250/791] skip Distributed api --- .../api_accuracy_checker/tensor_transport_layer/attl.py | 2 +- debug/accuracy_tools/msprobe/pytorch/service.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index e699bc554..9ff0ad703 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -93,7 +93,7 @@ class ATTL: try: torch.save(buffer, io_buff) except Exception as e: - logger.info(f"{buffer.name} can not be saved, skip: {e}") + self.logger.info(f"{buffer.name} can not be saved, skip: {e}") return data = io_buff.getvalue() self.socket_manager.add_to_sending_queue(data, rank=rank, step=step) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 12844f600..980c7d840 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -235,6 +235,10 @@ class Service: def attl_send(self, api_data): logger.info(f"tools is dumping api: {api_data.name}, rank: {self.current_rank}") + api_type, _, _ = api_data.name.split(Const.SEP) + if api_type in [Const.DISTRIBUTED]: + logger.info(f"api {api_data.name} is not supported, skip") + return if self.config.nfs_path: self.attl.upload(api_data) else: -- Gitee From 4e4f1b91f87ba4642b67adc67a24f55f817a2164 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 12 Aug 2024 14:28:42 +0800 Subject: [PATCH 251/791] compare compatibility bugfix --- debug/accuracy_tools/msprobe/core/common/utils.py | 10 +++++----- .../msprobe/pytorch/compare/distributed_compare.py | 3 ++- .../msprobe/pytorch/compare/pt_compare.py | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 246af98a3..e960dc98b 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -149,11 +149,11 @@ def check_summary_only_valid(summary_only): return summary_only -def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False, framework="mindspore"): +def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False, framework=Const.MS_FRAMEWORK): if not (isinstance(input_param, dict) and isinstance(output_path, str)): logger.error("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) - if framework == "mindspore": + if framework == Const.MS_FRAMEWORK: check_file_or_directory_path(input_param.get("npu_path"), False) check_file_or_directory_path(input_param.get("bench_path"), False) check_file_or_directory_path(input_param.get("stack_path"), False) @@ -165,7 +165,7 @@ def check_compare_param(input_param, output_path, summary_compare=False, md5_com check_file_or_directory_path(input_param.get("npu_dump_data_dir"), True) check_file_or_directory_path(input_param.get("bench_dump_data_dir"), True) check_file_or_directory_path(output_path, True) - if framework == "mindspore": + if framework == Const.MS_FRAMEWORK: with FileOpen(input_param.get("npu_path"), "r") as npu_json, \ FileOpen(input_param.get("bench_path"), "r") as bench_json, \ FileOpen(input_param.get("stack_path"), "r") as stack_json: @@ -485,8 +485,8 @@ def md5_find(data): return False -def task_dumppath_get(input_param, framework="mindspore"): - if framework == "mindspore": +def task_dumppath_get(input_param, framework=Const.MS_FRAMEWORK): + if framework == Const.MS_FRAMEWORK: npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) else: diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 7270b6830..ad3ef2e01 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -20,6 +20,7 @@ from msprobe.core.common.utils import CompareException, check_compare_param, \ from msprobe.core.common.file_check import create_directory from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger +from msprobe.core.common.const import Const from msprobe.pytorch.compare.pt_compare import PTComparator from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json @@ -53,7 +54,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): 'is_print_compare_log': True } try: - framework = "pytorch" + framework = Const.PT_FRAMEWORK summary_compare, md5_compare = task_dumppath_get(dump_result_param, framework) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 13db2b4c5..0e9a0e025 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -6,7 +6,7 @@ from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory -from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.const import FileCheckConst, Const from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result @@ -182,7 +182,7 @@ class PTComparator (Comparator): def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: - framework = "pytorch" + framework = Const.PT_FRAMEWORK summary_compare, md5_compare = task_dumppath_get(input_param, framework) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) -- Gitee From 3cc44bee53ad0efd888d55d897414a792b9657fb Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Mon, 12 Aug 2024 14:39:58 +0800 Subject: [PATCH 252/791] =?UTF-8?q?=E5=88=A0=E9=99=A4=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E4=B8=AD=E7=9A=84todo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/kj600/kj600/distributed/wrap_distributed.py | 2 +- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 +- .../msprobe/pytorch/free_benchmark/compare/grad_saver.py | 1 - .../pytorch/free_benchmark/result_handlers/handler_factory.py | 1 - 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/kj600/kj600/distributed/wrap_distributed.py b/debug/accuracy_tools/kj600/kj600/distributed/wrap_distributed.py index 80f978c94..77fd7924f 100644 --- a/debug/accuracy_tools/kj600/kj600/distributed/wrap_distributed.py +++ b/debug/accuracy_tools/kj600/kj600/distributed/wrap_distributed.py @@ -142,7 +142,7 @@ def op_aggregate(op, tensorlist): return max(tensorlist) if op == 'norm': return sum(tensorlist) - if op == 'zeros': # TODO wrong + if op == 'zeros': return sum(tensorlist) / len(tensorlist) if len(tensorlist) != 0 else 0 return torch.nan diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 559dfdc0f..96ace0f68 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -195,7 +195,7 @@ def run_ut(config): for _, (api_full_name, api_info_dict) in enumerate(tqdm(config.forward_content.items(), **tqdm_params)): if api_full_name in api_name_set: continue - if is_unsupported_api(api_full_name): # TODO run_ut does not support to the npu fusion api and distributed api + if is_unsupported_api(api_full_name): continue [_, api_name, _] = api_full_name.split(Const.SEP) try: diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py index 1cf75524d..e58223e59 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/compare/grad_saver.py @@ -60,7 +60,6 @@ class GradSaver: _index += 1 def compare_grad_results(self, handler, origin_grad, perturbed_grad, index): - # TODO get dtype? data_params = DataParams() data_params.original_result = origin_grad data_params.perturbed_result = perturbed_grad diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py index 5ee968c6a..46efd8283 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/result_handlers/handler_factory.py @@ -22,7 +22,6 @@ class FuzzHandlerFactory: handler = FuzzHandlerFactory.result_handlers.get(params.handler_type) else: handler = FuzzHandlerFactory.result_handlers.get(HandlerType.PREHEAT) - # TODO if not handler: raise FreeBenchmarkException( FreeBenchmarkException.UnsupportedType, -- Gitee From 20ffb0ecdecfbd4abf00e13c164963c255437204 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 12 Aug 2024 14:51:44 +0800 Subject: [PATCH 253/791] compare compatibility bugfix --- .../msprobe/pytorch/compare/distributed_compare.py | 5 ++--- .../msprobe/pytorch/compare/pt_compare.py | 14 ++------------ 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index ad3ef2e01..f07895810 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -54,11 +54,10 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): 'is_print_compare_log': True } try: - framework = Const.PT_FRAMEWORK - summary_compare, md5_compare = task_dumppath_get(dump_result_param, framework) + summary_compare, md5_compare = task_dumppath_get(dump_result_param, framework=Const.PT_FRAMEWORK) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare, framework=framework) + check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare, framework=Const.PT_FRAMEWORK) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 0e9a0e025..3e28db33f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -182,11 +182,10 @@ class PTComparator (Comparator): def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: - framework = Const.PT_FRAMEWORK - summary_compare, md5_compare = task_dumppath_get(input_param, framework) + summary_compare, md5_compare = task_dumppath_get(input_param, framework=Const.PT_FRAMEWORK) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(input_param, output_path, summary_compare, md5_compare, framework=framework) + check_compare_param(input_param, output_path, summary_compare, md5_compare, framework=Const.PT_FRAMEWORK) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error @@ -194,12 +193,3 @@ def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy ptComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) - - - - - - - - - -- Gitee From 6c68ed1a00ed413343dffb6543d14b85ac90f954 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 12 Aug 2024 15:04:20 +0800 Subject: [PATCH 254/791] compare compatibility bugfix --- .../msprobe/pytorch/compare/distributed_compare.py | 4 ++-- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index f07895810..5a6bfd15e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -61,6 +61,6 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - ptComparator=PTComparator() - ptComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, + pt_comparator = PTComparator() + pt_comparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 3e28db33f..c3d9e4c6d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -189,7 +189,7 @@ def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - ptComparator=PTComparator() - ptComparator.compare_core(input_param, output_path, stack_mode=stack_mode, + pt_comparator = PTComparator() + pt_comparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) -- Gitee From 3bd8119a1e6be1157b19e7989527702786af7d16 Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Mon, 12 Aug 2024 15:13:03 +0800 Subject: [PATCH 255/791] =?UTF-8?q?=E9=85=8D=E7=BD=AE=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E5=86=85=E5=AE=B9=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/mindspore/ms_config.py | 15 ++++++++++++--- debug/accuracy_tools/msprobe/pytorch/pt_config.py | 15 ++++++++++++--- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index 0e7ce1529..9bde39949 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -5,6 +5,8 @@ from msprobe.core.common.file_check import FileOpen from msprobe.core.common.const import Const from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.common.log import logger +from msprobe.core.grad_probe.constant import level_adp +from msprobe.core.grad_probe.utils import check_numeral_list_ascend class TensorConfig(BaseConfig): @@ -76,9 +78,16 @@ class FreeBenchmarkConfig(BaseConfig): class GradProbeConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) - self.grad_level = json_config.get("grad_level") - self.param_list = json_config.get("param_list") - self.bounds = json_config.get("bounds") + self.grad_level = json_config.get("grad_level", "L1") + self.param_list = json_config.get("param_list", []) + self.bounds = json_config.get("bounds", []) + + def _check_config(self): + if self.grad_level not in level_adp.keys(): + raise Exception(f"grad_level must be one of {level_adp.keys()}") + if not isinstance(self.param_list, list): + raise Exception(f"param_list must be a list") + check_numeral_list_ascend(self.bounds) TaskDict = { diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 2acd0ec53..49f5e56f5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -5,6 +5,8 @@ from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.file_check import FileOpen from msprobe.core.common.const import Const from msprobe.pytorch.hook_module.utils import WrapFunctionalOps, WrapTensorOps, WrapTorchOps +from msprobe.core.grad_probe.constant import level_adp +from msprobe.core.grad_probe.utils import check_numeral_list_ascend class TensorConfig(BaseConfig): @@ -109,9 +111,16 @@ class RunUTConfig(BaseConfig): class GradToolConfig(BaseConfig): def __init__(self, json_config): super().__init__(json_config) - self.grad_level = json_config.get("grad_level") - self.param_list = json_config.get("param_list") - self.bounds = json_config.get("bounds") + self.grad_level = json_config.get("grad_level", "L1") + self.param_list = json_config.get("param_list", []) + self.bounds = json_config.get("bounds", []) + + def _check_config(self): + if self.grad_level not in level_adp.keys(): + raise Exception(f"grad_level must be one of {level_adp.keys()}") + if not isinstance(self.param_list, list): + raise Exception(f"param_list must be a list") + check_numeral_list_ascend(self.bounds) def parse_task_config(task, json_config): -- Gitee From 24b190619d2ba895f0c4e8f224ab534317eefdf5 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Sat, 10 Aug 2024 00:17:07 +0800 Subject: [PATCH 256/791] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=96=99=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E3=80=91=E5=9C=A8msprobe=E6=A1=86=E6=9E=B6=E4=B8=8B?= =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E7=B2=BE=E5=BA=A6=E6=AF=94=E5=AF=B9=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E7=9A=84=E8=B5=84=E6=96=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/README.md | 2 +- .../msprobe/mindspore/doc/compare.md | 58 +++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/doc/compare.md diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 42743c507..d0f0212cd 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -159,7 +159,7 @@ Required-by: PyTorch场景：详见[PyTorch_精度比对工具](./pytorch/doc/ptdbg_ascend_overview.md)。 - MindSpore场景：暂不支持。 + MindSpore场景：详见[MindSpore_精度比对工具](./mindspore/doc/compare.md)。 5. 执行溢出解析。 diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md new file mode 100644 index 000000000..f252fffe1 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md @@ -0,0 +1,58 @@ +# 精度比对工具 +msprobe精度比对工具主要通过对同一个模型，在两个不同的MindSpore环境下，输入相同的训练数据，在分别得到dump数据后，对这两个dump数据进行全量自动对比，从而快速定位不同版本之间的精度问题。 + +执行精度比对操作需要安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 + +## 命令行方式比对 + +精度比对工具目前使用方式为命令行形式，对MindSpore的dump数据仅支持单卡。 + +请先参见《[精度数据采集](./dump.md)》完成不同环境下MindSpore精度数据的采集。 + +### 操作步骤 + +1. 使用MindSpore进行dump，得到不同框架版本的dump数据。 + +2. 创建比对文件，文件内容及示例请参见“**比对文件**”。 + +3. 执行如下示例命令进行比对： + + ```shell + msprobe -f mindspore compare -i ./compare.json -o ./output -s + ``` + + **完整参数说明** + + | 参数名 | 说明 | 是否必选 | + | ------------------ | ------------------------------------------------------------ | -------- | + | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | + | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -a或--auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + +4. 查看比对结果，请详见PyTorch目录下的《[精度比对工具](../../pytorch/doc/ptdbg_ascend_compare.md)》的“比对结果分析”章节。 + +### 比对文件 + +以在当前目录创建./compare.json为例，单卡场景示例如下： + + + ```json + { + "npu_path": "./npu_dump/dump.json", + "bench_path": "./bench_dump/dump.json", + "stack_path": "./npu_dump/stack.json", + "is_print_compare_log": True + } + ``` + + +**参数说明** + +| 参数名 | 说明 | 是否必选 | +| -------------------- | ------------------------------------------------------------ | ------------------ | +| npu_path | 配置NPU环境下的dump.json文件（单卡场景）。数据类型：str。 | 是 | +| bench_path | 配置CPU、GPU或NPU环境下的dump.json文件（单卡场景）。数据类型：str。 | 是 | +| stack_path | 配置NPU dump目录下的stack.json文件。数据类型：str。 | 是| +| is_print_compare_log | 配置是否开启日志打屏。可取值True或False，默认为True。数据类型：bool | 否 | \ No newline at end of file -- Gitee From 10a875727152d516f81d1be44302b0f69e4b8545 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Sat, 10 Aug 2024 16:51:35 +0800 Subject: [PATCH 257/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91mindpsore?= =?UTF-8?q?=E7=B2=BE=E5=BA=A6=E6=AF=94=E5=AF=B9=E6=A8=A1=E5=9D=97=E4=BB=A3?= =?UTF-8?q?=E7=A0=81=E9=87=8D=E6=9E=84=E6=A3=80=E8=A7=86=E6=84=8F=E8=A7=81?= =?UTF-8?q?=E6=95=B4=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/acc_compare.py | 9 +- .../msprobe/core/compare/mapping.yaml | 607 ------------------ .../msprobe/core/compare/match.py | 36 -- .../msprobe/core/compare/utils.py | 2 +- .../mindspore/compare/distributed_compare.py | 2 +- .../msprobe/mindspore/compare/ms_compare.py | 27 +- debug/accuracy_tools/msprobe/msprobe.py | 42 +- .../pytorch/compare/distributed_compare.py | 2 +- .../msprobe/pytorch/compare/pt_compare.py | 20 +- 9 files changed, 49 insertions(+), 698 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/core/compare/mapping.yaml delete mode 100644 debug/accuracy_tools/msprobe/core/compare/match.py diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 7705a748d..aa2016247 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -4,7 +4,7 @@ from msprobe.core.common.const import CompareConst from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ get_error_message from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.compare.utils import read_op, merge_tensor,CompareException +from msprobe.core.compare.utils import read_op, merge_tensor, CompareException from msprobe.core.compare.multiprocessing_compute import _handle_multi_process from msprobe.core.common.log import logger from msprobe.core.compare.check import check_graph_mode, check_struct_match, fuzzy_check_op @@ -59,7 +59,7 @@ class Comparator: b_op_name = bench_dict["op_name"] graph_mode = check_graph_mode(a_op_name[0], b_op_name[0]) - frame_name=getattr(self,"frame_name") + frame_name = getattr(self,"frame_name") if frame_name == "PTComparator": from msprobe.pytorch.compare.match import graph_mapping if graph_mode: @@ -75,7 +75,6 @@ class Comparator: is_match = False return is_match and struct_match - def match_op(self, npu_queue, bench_queue, fuzzy_match): for b_index, b_op in enumerate(bench_queue[0: -1]): if self.check_op(npu_queue[-1], b_op, fuzzy_match): @@ -96,7 +95,7 @@ class Comparator: error_flag = True else: try: - read_npy_data=getattr(self,"read_npy_data") + read_npy_data = getattr(self,"read_npy_data") n_value = read_npy_data(input_parma.get("npu_dump_data_dir"), npu_bench_name_list[0]) b_value = read_npy_data(input_parma.get("bench_dump_data_dir"), npu_bench_name_list[1]) except IOError as error: @@ -123,7 +122,7 @@ class Comparator: def _do_multi_process(self,input_parma, result_df): try: - compare_ops=getattr(self,"compare_ops") + compare_ops = getattr(self,"compare_ops") result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) return result_df except ValueError as e: diff --git a/debug/accuracy_tools/msprobe/core/compare/mapping.yaml b/debug/accuracy_tools/msprobe/core/compare/mapping.yaml deleted file mode 100644 index eaffbe7a1..000000000 --- a/debug/accuracy_tools/msprobe/core/compare/mapping.yaml +++ /dev/null @@ -1,607 +0,0 @@ -__and__: __and__ -__iand__: __iand__ -__ilshift__: __ilshift__ -__ior__: __ior__ -__irshift__: __irshift__ -__ixor__: __ixor__ -__lshift__: __lshift__ -__or__: __or__ -__rshift__: __rshift__ -__xor__: __xor__ -_adaptive_avg_pool2d: adaptive_avg_pool2d -_adaptive_avg_pool3d: adaptive_avg_pool3d -_cdist_forward: cdist -_cudnn_rnn: rnn -_embedding_bag: embedding_bag -_fft_c2c: fft -_fft_c2r: rfft -_foreach_add_: _foreach_add_ -_foreach_addcdiv: _foreach_addcdiv -_foreach_copy_: _foreach_copy_ -_foreach_lerp_: _foreach_lerp_ -_foreach_maximum: _foreach_maximum -_foreach_mul: _foreach_mul -_foreach_neg_: _foreach_neg_ -_foreach_pow: _foreach_pow -_foreach_reciprocal_: _foreach_reciprocal_ -_foreach_sign: _foreach_sign -_foreach_sqrt: _foreach_sqrt -_foreach_sqrt_: _foreach_sqrt_ -_foreach_sub: _foreach_sub -_fused_adam: FusedAdam -_linalg_det: det -_linalg_eigh: eigh -_linalg_slogdet: slogdet -_linalg_svd: svd -_list_to_tensor: as_tensor -_log_softmax: log_softmax -_native_batch_norm_legit: batch_norm -_nested_tensor_from_tensor_list: _nested_tensor_from_tensor_list -_pdist_forward: pdist -_pin_memory: pin_memory -_reshape_alias: reshape -_resize_output_: resize_ -_softmax: softmax -_to_copy: to -abs: abs -abs_: abs_ -absolute: abs -absolute_: abs_ -acos: acos -acos_: acos_ -acosh: acosh -acosh_: acosh_ -adaptive_max_pool2d: adaptive_max_pool2d -adaptive_max_pool3d: adaptive_max_pool3d -add: add -add_: add_ -addbmm: addbmm -addbmm_: addbmm_ -addcdiv: addcdiv -addcdiv_: addcdiv_ -addcmul: addcmul -addcmul_: addcmul_ -addmm: addmm -addmm_: addmm_ -addmv: addmv -addmv_: addmv_ -addr: addr -affine_grid_generator: affine_grid -alias: alias -all: all -alpha_dropout: AlphaDropout -amax: amax -amin: amin -aminmax: aminmax -angle: angle -any: any -arange: arange -arccos: acos -arccos_: arccos_ -arccosh: arccosh -arccosh_: arccosh_ -arcsin: asin -arcsin_: arcsin_ -arcsinh: asinh -arcsinh_: arcsinh_ -arctan: atan -arctan2: atan2 -arctan2_: arctan2_ -arctan_: arctan_ -arctanh: arctanh -arctanh_: arctanh_ -argmax: argmax -argmin: argmin -argsort: argsort -as_strided: as_strided -asin: asin -asin_: asin_ -asinh: asinh -asinh_: asinh_ -atan: atan -atan2: atan2 -atan2_: atan2_ -atan_: atan_ -atanh: atanh -atanh_: atanh_ -avg_pool2d: avg_pool2d -avg_pool3d: avg_pool3d -baddbmm: baddbmm -baddbmm_: baddbmm_ -bernoulli: bernoulli -bernoulli_: bernoulli_ -binary_cross_entropy: BCELoss -binary_cross_entropy_with_logits: binary_cross_entropy_with_logits -bitwise_and: bitwise_and -bitwise_and_: bitwise_and_ -bitwise_left_shift: __lshift__ -bitwise_left_shift_: bitwise_left_shift_ -bitwise_not: bitwise_not -bitwise_not_: bitwise_not_ -bitwise_or: bitwise_or -bitwise_or_: bitwise_or_ -bitwise_right_shift: __rshift__ -bitwise_right_shift_: bitwise_right_shift_ -bitwise_xor: bitwise_xor -bitwise_xor_: bitwise_xor_ -bmm: bmm -broadcast_tensors: broadcast_tensors -bucketize: bucketize -cat: cat -cauchy: Cauchy -cauchy_: cauchy_ -ceil: ceil -ceil_: ceil_ -celu: celu -celu_: celu_ -cholesky: cholesky -cholesky_inverse: cholesky_inverse -cholesky_solve: cholesky_solve -clamp: clamp -clamp_: clamp_ -clamp_max: clamp_max -clamp_max_: clamp_max_ -clamp_min: clamp_min -clamp_min_: clamp_min_ -clip: clip -clip_: clip_ -clone: clone -col2im: col2im -complex: complex -conj_physical: conj -conj_physical_: conj_ -constant_pad_nd: pad -convolution: Conv2d -copy: copy_ -copy_: copy_ -copysign: copysign -copysign_: copysign_ -cos: cos -cos_: cos_ -cosh: cosh -cosh_: cosh_ -count_nonzero: count_nonzero -cudnn_batch_norm: BatchNorm2d -cummax: cummax -cummin: cummin -cumprod: cumprod -cumprod_: cumprod_ -cumsum: cumsum -cumsum_: cumsum_ -deg2rad: deg2rad -deg2rad_: deg2rad_ -detach: detach -diag: diag -diag_embed: diag_embed -diagonal: diagonal -diagonal_copy: diagonal -diagonal_scatter: diagonal -digamma: digamma -digamma_: digamma_ -dist: dist -div: div -div_: div_ -divide: div -divide_: divide_ -dot: dot -dropout: dropout -elu: ELU -elu_: elu_ -embedding: embedding -empty_like: empty_like -empty_strided: empty_strided -eq: eq -eq_: eq_ -erf: erf -erf_: erf_ -erfc: erfc -erfc_: erfc_ -erfinv: erfinv -erfinv_: erfinv_ -exp: exp -exp2: exp2 -exp2_: exp2_ -exp_: exp_ -expand: expand -expm1: expm1 -expm1_: expm1_ -exponential: Exponential -exponential_: exponential_ -eye: eye -fft_fft: fft -fft_fft2: fft2 -fft_fftn: fftn -fft_fftshift: fftshift -fft_hfft: hfft -fft_hfft2: hfft2 -fft_hfftn: hfftn -fft_ifft: ifft -fft_ifft2: ifft2 -fft_ifftn: ifftn -fft_ifftshift: ifftshift -fft_ihfft: ihfft -fft_ihfft2: ihfft2 -fft_ihfftn: ifftn -fft_irfft: irfft -fft_irfft2: irfft2 -fft_irfftn: irfftn -fft_rfft: rfft -fft_rfft2: rfft2 -fft_rfftn: rfftn -fill: fill_ -fill_: fill_ -fix: fix -fix_: fix_ -flip: flip -float_power_: float_power_ -floor: floor -floor_: floor_ -floor_divide: floor_divide -floor_divide_: floor_divide_ -fmax: fmax -fmin: fmin -fmod: fmod -fmod_: fmod_ -frac: frac -frac_: frac_ -full: full -full_like: full_like -gather: gather -gcd: gcd -gcd_: gcd_ -ge: ge -ge_: ge_ -gelu: GELU -gelu_: gelu_ -geometric: Geometric -geometric_: geometric_ -glu: glu -greater: gt -greater_: ge_ -greater_equal: ge -greater_equal_: ge_ -grid_sampler_2d: grid_sample -grid_sampler_3d: grid_sample -gru: GRU -gt: gt -gt_: gt_ -hardshrink: Hardshrink -hardsigmoid: hardsigmoid -hardsigmoid_: hardsigmoid_ -hardswish: hardswish -hardswish_: hardswish_ -hardtanh: hardtanh -hardtanh_: hardtanh_ -heaviside: heaviside -heaviside_: heaviside_ -hinge_embedding_loss: HingeEmbeddingLoss -huber_loss: huber_loss -hypot: hypot -hypot_: hypot_ -i0: i0 -i0_: i0_ -igamma: igamma -igamma_: igamma_ -igammac: igammac -igammac_: igammac_ -index: __getitem__ -index_add: index_add -index_add_: index_add_ -index_copy: index_copy_ -index_copy_: index_copy_ -index_fill: index_fill_ -index_fill_: index_fill_ -index_put: index_put_ -index_put_: index_put_ -index_reduce: index_select -index_select: index_select -is_pinned: is_pinned -is_same_size: is_same_size -isinf: isinf -isnan: isnan -isneginf: isneginf -isposinf: isposinf -istft: istft -item: item -lcm: lcm -lcm_: lcm_ -le: le -le_: le_ -leaky_relu: LeakyReLU -leaky_relu_: leaky_relu_ -lerp: lerp -lerp_: lerp_ -less: less -less_: less_ -less_equal: le -less_equal_: less_equal_ -lgamma: lgamma -lgamma_: lgamma_ -linalg_cholesky_ex: cholesky -linalg_cross: cross -linalg_householder_product: householder_product -linalg_inv_ex: inv -linalg_ldl_factor_ex: ldl -linalg_ldl_solve: ldl_solve -linalg_lu: lu -linalg_lu_factor_ex: lu_factor -linalg_lu_solve: lu_solve -linalg_matrix_exp: matrix_exp -linalg_qr: qr -linalg_solve_triangular: solve -linalg_vector_norm: norm -linspace: linspace -log: log -log10: log10 -log10_: log10_ -log1p: log1p -log1p_: log1p_ -log2: log2 -log2_: log2_ -log_: log_ -log_normal: LogNormal -log_sigmoid_forward: log_sigmoid -logaddexp: logaddexp -logaddexp2: logaddexp2 -_native_batch_norm_legit_functional: batch_norm -logcumsumexp: logcumsumexp -logical_and: logical_and -logical_and_: logical_and_ -logical_not: logical_not -logical_not_: logical_not_ -logical_or: logical_or -logical_or_: logical_or_ -logical_xor: logical_xor -logical_xor_: logical_xor_ -logit: logit -logit_: logit_ -logspace: logspace -logsumexp: logsumexp -lstm: LSTM -lt: lt -lt_: lt_ -lu_unpack: lu_unpack -margin_ranking_loss: margin_ranking_loss -masked_fill: masked_fill -masked_fill_: masked_fill_ -matmul: matmul -max: max -max_pool2d_with_indices: MaxPool2d -max_pool3d_with_indices: MaxPool3d -max_unpool2d: MaxUnpool2d -max_unpool3d: max_unpool3d -maximum: maximum -mean: mean -median: median -meshgrid: meshgrid -min: min -minimum: minimum -mish: Mish -mish_: mish_ -mm: mm -mode: mode -mse_loss: mse_loss -mul: mul -mul_: mul_ -multi_margin_loss: MultiMarginLoss -multilabel_margin_loss_forward: multilabel_margin_loss -multinomial: multinomial -multiply: multiply -multiply_: mul_ -mv: mv -mvlgamma: mvlgamma -mvlgamma_: mvlgamma_ -name: name -nan_to_num: nan_to_num -nan_to_num_: nan_to_num_ -nanmedian: nanmedian -nansum: nansum -narrow_copy: narrow -native_batch_norm: BatchNorm2d -native_dropout: dropout -native_group_norm: group_norm -native_layer_norm: LayerNorm -ne: ne -ne_: ne_ -neg: neg -neg_: neg_ -negative: neg -negative_: neg_ -new_empty: new_empty -new_empty_strided: new_empty_strided -new_full: new_full -new_ones: new_ones -new_zeros: new_zeros -nextafter: nextafter -nextafter_: nextafter_ -nll_loss: nll_loss -nll_loss2d_forward: NLLLoss2d -nll_loss_forward: NLLLoss -nonzero_static: nonzero -norm: norm -normal: normal -normal_: normal_ -not_equal: ne -not_equal_: ne_ -ones: ones -ones_like: ones_like -ormqr: ormqr -pairwise_distance: pairwise_distance -pdist: pdist -permute: permute -pin_memory: pin_memory -pixel_shuffle: PixelShuffle -polar: polar -polygamma: polygamma -positive: positive -pow: pow -pow_: pow_ -prelu: prelu -prod: prod -quantized_gru: GRU -quantized_lstm: LSTM -rad2deg: rad2deg -rad2deg_: rad2deg_ -rand: rand -rand_like: rand_like -randint: randint -randint_like: randint_like -randn: randn -randn_like: randn_like -randperm: randperm -reciprocal: reciprocal -reciprocal_: reciprocal_ -reflection_pad1d: reflection_pad1d -reflection_pad2d: reflection_pad2d -reflection_pad3d: ReflectionPad3d -relu: relu -relu6: relu6 -relu_: relu_ -remainder: remainder -remainder_: remainder_ -renorm: renorm -renorm_: renorm_ -repeat: repeat -repeat_interleave: repeat_interleave -replication_pad1d: ReplicationPad1d -replication_pad2d: replication_pad2d -replication_pad3d: replication_pad3d -resize_as_: resize_as_ -rnn_relu: RNN -rnn_tanh: RNN -roll: roll -rot90: rot90 -round: round -round_: round_ -rrelu_with_noise: RReLU -rrelu_with_noise_: rrelu_with_noise -rsqrt: rsqrt -rsqrt_: rsqrt_ -rsub: rsub -scalar_tensor: scalar_tensor -scatter: scatter_ -scatter_: scatter_ -scatter_add: scatter_add -scatter_add_: scatter_add_ -searchsorted: searchsorted -select: select -selu: selu -selu_: selu_ -sgn: sgn -sgn_: sgn_ -sigmoid: sigmoid -sigmoid_: sigmoid_ -sign: sign -sign_: sign_ -signbit: signbit -silu: silu -silu_: silu_ -sin: sin -sin_: sin_ -sinc: sinc -sinc_: sinc_ -sinh: sinh -sinh_: sinh_ -slice: slice -smooth_l1_loss: smooth_l1_loss -soft_margin_loss: soft_margin_loss -softplus: softplus -softshrink: softshrink -sort: sort -special_airy_ai: airy_ai -special_bessel_j0: j0 -special_bessel_j1: j1 -special_bessel_y0: y0 -special_bessel_y1: y1 -special_chebyshev_polynomial_t: chebyshev_t -special_chebyshev_polynomial_u: chebyshev_u -special_entr: entr -special_erfcx: erfcx -special_hermite_polynomial_h: hermite -special_hermite_polynomial_he: he -special_i0: i0 -special_i0e: i0e -special_i1: i1 -special_i1e: i1e -special_laguerre_polynomial_l: laguerre_l -special_log_ndtr: log_ndtr -special_modified_bessel_i0: i0 -special_modified_bessel_i1: i1 -special_modified_bessel_k0: k0 -special_modified_bessel_k1: i1 -special_ndtr: ndtr -special_ndtri: ndtri -special_scaled_modified_bessel_k0: i0e -special_scaled_modified_bessel_k1: scaled_modified_bessel_k1 -special_spherical_bessel_j0: spherical_jn -special_xlog1py: xlog1py -special_zeta: zeta -split: split -split_with_sizes: split -sqrt: sqrt -sqrt_: sqrt_ -square: square -square_: square_ -squeeze: squeeze -stack: stack -std: std -std_mean: std_mean -stft: stft -sub: sub -sub_: sub_ -subtract: sub -subtract_: subtract_ -sum: sum -t: t -t_: t_ -take: take -tan: tan -tan_: tan_ -tanh: tanh -tanh_: tanh_ -threshold: threshold -threshold_: threshold_ -to: to -topk: topk -trace: trace -transpose: transpose -transpose_: transpose_ -triangular_solve: triangular_solve -tril: tril -tril_: tril_ -tril_indices: tril_indices -triu: triu -triu_: triu_ -triu_indices: triu_indices -true_divide: true_divide -true_divide_: true_divide_ -trunc: trunc -trunc_: trunc_ -unbind: unbind -unfold: unfold -uniform: Uniform -uniform_: uniform_ -unsafe_chunk: unsafe_chunk -unsafe_split: split -unsafe_split_with_sizes: split_with_sizes -unsqueeze: unsqueeze -unsqueeze_: unsqueeze_ -upsample_bicubic2d: interpolate -upsample_bilinear2d: upsample_bilinear -upsample_nearest1d: interpolate -upsample_nearest2d: interpolate -upsample_nearest3d: interpolate -var: var -var_mean: var_mean -vdot: vdot -view: view -where: where -xlogy: xlogy -xlogy_: xlogy_ -zero: zeros -zero_: zero_ -zeros: zeros -zeros_like: zeros_like - - - diff --git a/debug/accuracy_tools/msprobe/core/compare/match.py b/debug/accuracy_tools/msprobe/core/compare/match.py deleted file mode 100644 index 2a46105bd..000000000 --- a/debug/accuracy_tools/msprobe/core/compare/match.py +++ /dev/null @@ -1,36 +0,0 @@ -import os -import yaml -from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import CompareException - - -class AtenIrMapping(): - def __init__(self): - cur_path = os.path.dirname(os.path.realpath(__file__)) - yaml_path = os.path.join(cur_path, "mapping.yaml") - with FileOpen(yaml_path, 'r') as f: - self.aten_mapping = yaml.safe_load(f) - - def match(self, op1, op2): - if "Aten" in op1 and "Aten" not in op2: - return self.match_op(op1, op2) - else: - return self.match_op(op2, op1) - - def match_op(self, aten_op, torch_op): - try: - aten_op_raw_name_overload = '_'.join(aten_op.split("_")[1:-3]) - aten_op_raw_name = aten_op_raw_name_overload.split('.')[0] - torch_op_raw_name = '_'.join(torch_op.split("_")[1:-3]).lower() - except IndexError as e: - err_msg = f"Dump op name format error: {aten_op}, {torch_op}. Your dump data may be corrupted." - raise CompareException.INVALID_DATA_ERROR(err_msg) from e - matching_op = self.aten_mapping.get(aten_op_raw_name) - if matching_op is None: - return False - if matching_op.lower() == torch_op_raw_name: - return True - return False - - -graph_mapping = AtenIrMapping() diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 63b745432..510403bf3 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -65,7 +65,7 @@ def rename_api(npu_name, process): def read_op(op_data, op_name): - op_parsed_list = [] + op_parsed_list = Const.DEFAULT_LIST if 'forward' in op_name: if 'input_args' in op_data: input_item = op_data['input_args'] diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 6f84a69e9..08f0a03ec 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -60,6 +60,6 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - msComparator=MSComparator() + msComparator = MSComparator() msComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index be7439cb0..a4736a91b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,24 +1,23 @@ import json import os.path import numpy as np - from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory from msprobe.core.common.const import FileCheckConst - +from msprobe.core.common.log import logger +from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.common.log import logger -from msprobe.core.common.exceptions import FileCheckException + class MSComparator (Comparator): def __init__(self): - self.frame_name=MSComparator.__name__ + self.frame_name = MSComparator.__name__ def compare_ops(self,idx, dump_path_dict, result_df, lock, input_parma): cos_result = [] @@ -47,12 +46,12 @@ class MSComparator (Comparator): five_thousand_err_ratio_result.append(five_thousand_err_ratio) cr = ComparisonResult( - cos_result=cos_result, - max_err_result=max_err_result, - max_relative_err_result=max_relative_err_result, - err_msgs=err_mess, - one_thousand_err_ratio_result=one_thousand_err_ratio_result, - five_thousand_err_ratio_result=five_thousand_err_ratio_result + cos_result = cos_result, + max_err_result = max_err_result, + max_relative_err_result = max_relative_err_result, + err_msgs = err_mess, + one_thousand_err_ratio_result = one_thousand_err_ratio_result, + five_thousand_err_ratio_result = five_thousand_err_ratio_result ) return _save_cmp_result(idx, cr, result_df, lock) @@ -121,8 +120,6 @@ class MSComparator (Comparator): result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) return result_df - - def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -130,7 +127,7 @@ class MSComparator (Comparator): data_path = path_checker.common_check() data_value = np.load(data_path) # detach for less memory if data_value.dtype == np.float16: - data_value=data_value.astype(np.float32) + data_value = data_value.astype(np.float32) return data_value @@ -188,7 +185,7 @@ def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - msComparator=MSComparator() + msComparator = MSComparator() msComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 802913814..54b4a12d0 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -43,32 +43,32 @@ def main(): multi_run_ut_cmd_parser = subparsers.add_parser('multi_run_ut') api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare') run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check') - multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, - help='Number of splits for parallel processing. Range: 1-64') - _compare_parser(compare_cmd_parser) + is_torch_available=is_module_available("torch") + if is_torch_available: + from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command + from msprobe.pytorch.parse_tool.cli import parse as cli_parse + from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut + from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, \ + _api_precision_compare_command + from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ + _run_overflow_check_command + from msprobe.pytorch.compare.compare_cli import compare_cli + _run_ut_parser(run_ut_cmd_parser) + _run_ut_parser(multi_run_ut_cmd_parser) + multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, + help='Number of splits for parallel processing. Range: 1-64') + _api_precision_compare_parser(api_precision_compare_cmd_parser) + _run_overflow_check_parser(run_overflow_check_cmd_parser) if len(sys.argv) == 1: parser.print_help() sys.exit(0) args = parser.parse_args(sys.argv[1:]) if sys.argv[2] == "pytorch": - if is_module_available("torch"): - from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command - from msprobe.pytorch.parse_tool.cli import parse as cli_parse - from msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut import prepare_config, run_parallel_ut - from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import _api_precision_compare_parser, \ - _api_precision_compare_command - from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ - _run_overflow_check_command - from msprobe.pytorch.compare.compare_cli import compare_cli - _run_ut_parser(run_ut_cmd_parser) - _run_ut_parser(multi_run_ut_cmd_parser) - _api_precision_compare_parser(api_precision_compare_cmd_parser) - _run_overflow_check_parser(run_overflow_check_cmd_parser) - else: - logger.error("Pytorch does not exit, please install pytorch library") - raise Exception() + if not is_torch_available: + logger.error("PyTorch does not exit, please install PyTorch library") + raise Exception("PyTorch does not exit, please install PyTorch library") if sys.argv[3] == "run_ut": run_ut_command(args) elif sys.argv[3] == "parse": @@ -86,8 +86,8 @@ def main(): if is_module_available("mindspore"): from msprobe.mindspore.compare.compare_cli import compare_cli_ms else: - logger.error("Mindspore does not exit, please install mindspore library") - raise Exception() + logger.error("MindSpore does not exit, please install MindSpore library") + raise Exception("MindSpore does not exit, please install MindSpore library") if sys.argv[3] == "compare": compare_cli_ms(args) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 923c0044d..22d0598ed 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -60,6 +60,6 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - ptComparator=PTComparator() + ptComparator = PTComparator() ptComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index a947a12f6..1cc6301c5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,19 +1,17 @@ import json import os.path import torch - from msprobe.core.advisor.advisor import Advisor from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ check_file_not_exists, check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory from msprobe.core.common.const import FileCheckConst - +from msprobe.core.common.log import logger +from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.common.log import logger -from msprobe.core.common.exceptions import FileCheckException class PTComparator (Comparator): @@ -47,12 +45,12 @@ class PTComparator (Comparator): five_thousand_err_ratio_result.append(five_thousand_err_ratio) cr = ComparisonResult( - cos_result=cos_result, - max_err_result=max_err_result, + cos_result = cos_result, + max_err_result = max_err_result, max_relative_err_result=max_relative_err_result, - err_msgs=err_mess, - one_thousand_err_ratio_result=one_thousand_err_ratio_result, - five_thousand_err_ratio_result=five_thousand_err_ratio_result + err_msgs = err_mess, + one_thousand_err_ratio_result = one_thousand_err_ratio_result, + five_thousand_err_ratio_result = five_thousand_err_ratio_result ) return _save_cmp_result(idx, cr, result_df, lock) @@ -92,7 +90,7 @@ class PTComparator (Comparator): try: last_bench_ops_len = len(bench_ops_queue) op_name_bench = next(ops_bench_iter) - bench_merge_list =self.gen_merge_list(bench_json_data,op_name_bench,stack_json_data,summary_compare,md5_compare) + bench_merge_list = self.gen_merge_list(bench_json_data,op_name_bench,stack_json_data,summary_compare,md5_compare) if bench_merge_list: bench_ops_queue.append(bench_merge_list) except StopIteration: @@ -190,7 +188,7 @@ def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - ptComparator=PTComparator() + ptComparator = PTComparator() ptComparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) -- Gitee From e7ba347874eddb1390dbd15e85d64fd7b484dff0 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Wed, 7 Aug 2024 21:51:58 +0800 Subject: [PATCH 258/791] =?UTF-8?q?[msprobe]compare=E6=AF=94=E5=AF=B9?= =?UTF-8?q?=E8=83=BD=E5=8A=9B=E6=8F=90=E4=BE=9B=E5=91=BD=E4=BB=A4=E8=A1=8C?= =?UTF-8?q?=E5=B7=A5=E5=85=B7=E8=B5=84=E6=96=99=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/README.md | 13 ++- .../pytorch/doc/ptdbg_ascend_compare.md | 86 ++++++++++++++++--- .../pytorch/doc/ptdbg_ascend_quickstart.md | 6 +- 3 files changed, 90 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 42743c507..129bfbf13 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -21,7 +21,12 @@ Successfully installed mindstudio_probe-{version} ``` ### 下载whl包安装 -1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、torch、tqdm依赖。 +1. 使用pip命令安装依赖： + + 1. 根据实际环境安装torch或mindspore + + 2. 安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、einops、matplotlib + 若环境中已安装部分依赖，不需要重复安装。 @@ -193,6 +198,12 @@ msprobe -f multi_run_ut [-h] msprobe -f api_precision_compare [-h] ``` +精度比对工具 + +```bash +msprobe -f compare [-h] +``` + 溢出解析工具 ```bash diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md index 4bd05c73e..e265a6af4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md @@ -1,10 +1,10 @@ # **精度比对工具** -## CPU或GPU与NPU精度数据比对 +## 总体说明 -### 总体说明 +- 本节主要介绍通过命令行方式和比对函数方式进行CPU或GPU与NPU的精度数据比对，执行精度比对操作前需要先完成CPU或GPU与NPU的精度数据dump，参见《[精度数据采集](./dump.md)》。 -- 本节主要介绍CPU或GPU与NPU精度数据比对的函数以及示例，执行精度比对操作前需要先完成CPU或GPU与NPU的精度数据dump，详见《[精度数据采集](./dump.md)》。 +- 训练精度一体化工具msprobe，使用子命令compare进行比对，可支持单卡和多卡场景的精度数据比对。 - 比对函数均通过单独创建精度比对脚本执行，可支持单卡和多卡场景的精度数据比对。 @@ -12,12 +12,76 @@ 用户环境性能弱于标准约束或非独占使用的比对速度酌情向下浮动。比对速度的计算方式：两份比对文件大小/比对耗时。 -### 约束 +## 约束 - NPU自研API，在CPU或GPU若没有对应的API，该API的dump数据不比对。 - NPU与CPU或GPU的计算结果误差可能会随着模型的执行不断累积，最终会出现同一个API因为输入的数据差异较大而无法比对的情况。 - CPU或GPU与NPU中两个相同的API会因为调用次数不同导致无法比对或比对到错误的API，不影响整体运行，该API忽略。 +## 命令行方式比对 + +将CPU或GPU与NPU的dump文件进行比对，支持单卡和多卡，可同时比对多卡的dump数据。多机场景需要每个设备单独执行比对操作。 + +请先参见《[精度数据采集](./dump.md)》完成CPU或GPU与NPU的精度数据dump。 + +### 操作步骤 + +1. 创建比对文件，文件内容及示例请参见“**比对文件**”。 + +2. 执行如下示例命令进行比对： + + ```shell + msprobe -f pytorch compare -i ./compare.json -o ./output -s + ``` + + **完整参数说明** + + | 参数名 | 说明 | 是否必选 | + | ------------------ | ------------------------------------------------------------ | -------- | + | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | + | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -a或--auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + +3. 查看比对结果，请参见“**比对结果分析**”。 + +### 比对文件 + +以在当前目录创建./compare.json为例。 + +- 单卡场景示例 + + ```json + { + "npu_path": "./npu_dump/dump.json", + "bench_path": "./bench_dump/dump.json", + "stack_path": "./npu_dump/stack.json", + "is_print_compare_log": True + } + ``` + +- 多卡场景示例 + + ```json + { + "npu_path": "./npu_dump/step0", + "bench_path": "./bench_dump/step0", + "is_print_compare_log": True + } + ``` + +**参数说明** + +| 参数名 | 说明 | 是否必选 | +| -------------------- | ------------------------------------------------------------ | ------------------ | +| npu_path | 配置NPU环境下的dump.json文件（单卡场景）或真实数据目录（多卡场景）。数据类型：str。 | 是 | +| bench_path | 配置CPU、GPU或NPU环境下的dump.json文件（单卡场景）或真实数据目录（多卡场景）。数据类型：str。 | 是 | +| stack_path | 配置NPU dump目录下的stack.json文件。数据类型：str。 | 单卡必选，多卡不选 | +| is_print_compare_log | 配置是否开启日志打屏。可取值True或False。数据类型：bool | 否 | + +## 比对函数方式比对 + ### compare_distributed **功能说明** @@ -66,7 +130,7 @@ compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_mat | 参数名 | 说明 | 是否必选 | | ------------ | ------------------------------------------------------------ | -------- | -| input_param | 配置dump数据文件及目录。数据类型：dict。配置参数包括：
"npu_json_path"：指定NPU dump目录下的dump.json文件。参数示例："npu_json_path": "./npu_dump/dump.json"。必选。
"bench_json_path"：指定CPU、GPU或NPU dump目录下的dump.json文件。参数示例："bench_json_path": "./gpu_dump/dump.json"。必选。
"stack_json_path"：指定NPU dump目录下的stack.json文件。参数示例："stack_json_path": "./npu_dump/stack.json"。可选。
"is_print_compare_log"：配置是否开启日志打屏。可取值True或False。可选。 | 是 | +| input_param | 配置dump数据文件及目录。数据类型：dict。配置参数包括：
"npu_json_path"：指定NPU dump目录下的dump.json文件。参数示例："npu_json_path": "./npu_dump/dump.json"。必选。
"bench_json_path"：指定CPU、GPU或NPU dump目录下的dump.json文件。参数示例："bench_json_path": "./bench_dump/dump.json"。必选。
"stack_json_path"：指定NPU dump目录下的stack.json文件。参数示例："stack_json_path": "./npu_dump/stack.json"。可选。
"is_print_compare_log"：配置是否开启日志打屏。可取值True或False。可选。 | 是 | | output_path | 配置比对结果文件存盘目录。参数示例：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。数据类型：str。 | 是 | | stack_mode | 配置stack_mode的开关。仅当配置"stack_json_path"需要开启。可取值True或False，参数示例：stack_mode=True，默认为False。数据类型：bool。 | 否 | | auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。可取值True或False，参数示例：auto_analyze=False，默认为True。数据类型：bool。 | 否 | @@ -78,13 +142,13 @@ compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_mat ```Python from msprobe.pytorch import compare -dump_result_param={ +input_param={ "npu_json_path": "./npu_dump/dump.json", -"bench_json_path": "./gpu_dump/dump.json", +"bench_json_path": "./bench_dump/dump.json", "stack_json_path": "./npu_dump/stack.json", "is_print_compare_log": True } -compare(dump_result_param, output_path="./output", stack_mode=True) +compare(input_param, output_path="./output", stack_mode=True) ``` ### 统计量比对 @@ -97,13 +161,13 @@ compare(dump_result_param, output_path="./output", stack_mode=True) ```Python from msprobe.pytorch import compare -dump_result_param={ +input_param={ "npu_json_path": "./npu_dump/dump.json", -"bench_json_path": "./gpu_dump/dump.json", +"bench_json_path": "./bench_dump/dump.json", "stack_json_path": "./npu_dump/stack.json", "is_print_compare_log": True } -compare(dump_result_param, output_path="./output", stack_mode=True) +compare(input_param, output_path="./output", stack_mode=True) ``` **比对结果** diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_quickstart.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_quickstart.md index 4b6ac9de2..1a581226f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_quickstart.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_quickstart.md @@ -83,13 +83,13 @@ PyTorch训练场景的精度问题分析建议参考以下思路进行精度比 ```python from msprobe.pytorch import compare - dump_result_param={ + input_param={ "npu_json_path": "./npu_dump/dump.json", - "bench_json_path": "./gpu_dump/dump.json", + "bench_json_path": "./bench_dump/dump.json", "stack_json_path": "./npu_dump/stack.json", "is_print_compare_log": True } - compare(dump_result_param, output_path="./output", stack_mode=True) + compare(input_param, output_path="./output", stack_mode=True) ``` 执行比对： -- Gitee From 2015e4be559827c4944c1ea5dd140c72f31d50ff Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Tue, 13 Aug 2024 10:02:04 +0800 Subject: [PATCH 259/791] =?UTF-8?q?[msprobe]=E8=BF=AD=E4=BB=A3=E4=BA=8C?= =?UTF-8?q?=E5=87=BA=E5=8C=85=E6=9B=B4=E6=96=B0=E8=B5=84=E6=96=99=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/README.md | 27 ++++++++++++-------------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 42743c507..13baaebf6 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -29,13 +29,14 @@ Successfully installed mindstudio_probe-{version} 请通过下表链接下载工具whl包。 - | 版本 | 发布日期 | 支持PyTorch版本 | 下载链接 | 校验码 | - | ----- | ---------- | --------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | - | 1.0.1 | 2024-07-25 | 2.0/2.1/2.2 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | - | 1.0 | 2024-07-09 | 2.0/2.1/2.2 | [ascend_training_accuracy_tools-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/1.0/ascend_training_accuracy_tools-1.0-py3-none-any.whl) | 5016dfe886c5d340ec6f60a959673355855f313c91f100680da814efb49f8e81 | - | 0.0.3 | 2024-06-11 | 2.0/2.1/2.2 | [ascend_training_accuracy_tools-0.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.3-py3-none-any.whl) | f46d9714704859e2d67861a65bbb3c76b0a250cf6e238b978b5b959ab1fe125a | - | 0.0.2 | 2024-05-23 | 2.0/2.1/2.2 | [ascend_training_accuracy_tools-0.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.2-py3-none-any.whl) | 2e35809bde559e9c4d2f16a02ccde779ed9e436bb65fded0b7ebaf6ac2c88d93 | - | 0.0.1 | 2024-03-15 | 2.0/2.1 | [ascend_training_accuracy_tools-0.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.1-py3-none-any.whl) | 5801510d4e827e4859bc9a5aca021e4d30c2ea42d60a4c8ad0c2baab1b7782c9 | + | 版本 | 发布日期 | 支持PyTorch版本 | 支持MindSpore版本 | 下载链接 | 校验码 | + | ----- | ---------- | ---------------- | ----------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | + | 1.0.2 | 2024-08-09 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.2-py3-none-any.whl) | e4a980e5d98c426ce5ce9842520d9bc031d3b3de621c74b3d59414cc6e238e0e | + | 1.0.1 | 2024-07-25 | 2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | + | 1.0 | 2024-07-09 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/1.0/ascend_training_accuracy_tools-1.0-py3-none-any.whl) | 5016dfe886c5d340ec6f60a959673355855f313c91f100680da814efb49f8e81 | + | 0.0.3 | 2024-06-11 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-0.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.3-py3-none-any.whl) | f46d9714704859e2d67861a65bbb3c76b0a250cf6e238b978b5b959ab1fe125a | + | 0.0.2 | 2024-05-23 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-0.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.2-py3-none-any.whl) | 2e35809bde559e9c4d2f16a02ccde779ed9e436bb65fded0b7ebaf6ac2c88d93 | + | 0.0.1 | 2024-03-15 | 2.0/2.1 | - | [ascend_training_accuracy_tools-0.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.1-py3-none-any.whl) | 5801510d4e827e4859bc9a5aca021e4d30c2ea42d60a4c8ad0c2baab1b7782c9 | 3. whl包校验。 @@ -205,15 +206,11 @@ msprobe -f run_overflow_check [-h] msprobe -f parse [-h] ``` -| 参数 | 说明 | -| ---- | ------------------------------------------------------ | -| -f | 框架，请按所使用框架配置，当前支持pytorch或mindspore。 | -| -h | 帮助信息。 | +| 参数 | 说明 | +| ---- | ------------------------------------------------------------ | +| -f | 框架，请按所使用框架配置，当前支持配置为：pytorch、mindspore。 | +| -h | 帮助信息。 | ## 贡献 push代码前，请务必保证已经完成了基础功能测试和网络测试。 - -## Release Notes - -Release Notes请参见[RELEASE](RELEASE.md)。 \ No newline at end of file -- Gitee From 4eadb8d4c8aa2099cf8ac7180436b883b17a9e8d Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Tue, 13 Aug 2024 10:08:10 +0800 Subject: [PATCH 260/791] =?UTF-8?q?[msprobe]=E8=BF=AD=E4=BB=A3=E4=BA=8C?= =?UTF-8?q?=E5=87=BA=E5=8C=85=E6=9B=B4=E6=96=B0=E8=B5=84=E6=96=99=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md | 2 +- debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md index 41b97098a..bb0f09a03 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md @@ -8,7 +8,7 @@ **真实数据模式**：精度预检工具支持随机生成模式和真实数据模式，即在预检dump时可以选择由工具构造随机数进行输入获得dump数据或选择获取真实输入数据进行预检dump操作；随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 -**工具支持PyTorch版本**：2.0/2.1/2.2。 +**工具支持PyTorch版本**：1.11/2.0/2.1/2.2。 **工具特性** diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md b/debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md index b8c9c3b4c..f56165b25 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md @@ -2,7 +2,7 @@ 针对训练过程中的溢出检测场景（当《[精度数据采集](./dump.md)》开启溢出检测dump时），对于输入正常但输出存在溢出的API，会在训练执行目录下将溢出的API信息按照前向和反向分类，dump并保存为`dump.json`，前向过程溢出的API可通过该工具对`dump.json`进行解析，输出溢出API为正常溢出还是非正常溢出，从而帮助用户快速判断。 -工具支持PyTorch版本：1.11.0/2.0/2.1/2.2。 +工具支持PyTorch版本：1.11/2.0/2.1/2.2。 操作步骤如下： -- Gitee From 410d78552167c962c8b08aed4cfba39479bfd30f Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:13:14 +0000 Subject: [PATCH 261/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/common/utils.py | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index b6e893296..8c714b56b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -14,10 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -import json import os import re -import csv import torch @@ -38,12 +36,6 @@ class DumpException(CompareException): pass -def write_csv(data, filepath): - with FileOpen(filepath, 'a', encoding='utf-8-sig') as f: - writer = csv.writer(f) - writer.writerows(data) - - def check_object_type(check_object, allow_type): """ Function Description: @@ -93,24 +85,6 @@ def check_file_or_directory_path(path, isdir=False): raise CompareException(CompareException.INVALID_PATH_ERROR) -def get_json_contents(file_path): - ops = get_file_content_bytes(file_path) - try: - json_obj = json.loads(ops) - except ValueError as error: - logger.error('Failed to load "%s". %s' % (file_path, str(error))) - raise CompareException(CompareException.INVALID_FILE_ERROR) from error - if not isinstance(json_obj, dict): - logger.error('Json file %s, content is not a dictionary!' % file_path) - raise CompareException(CompareException.INVALID_FILE_ERROR) - return json_obj - - -def get_file_content_bytes(file): - with FileOpen(file, 'rb') as file_handle: - return file_handle.read() - - class SoftlinkCheckException(Exception): pass -- Gitee From 8b6d54ce3549f621d8826f0c54e35737c27048cd Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:13:35 +0000 Subject: [PATCH 262/791] update debug/accuracy_tools/msprobe/pytorch/common/utils.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/common/utils.py | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index ae8823de6..9cf326ea2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -18,13 +18,15 @@ import logging import os import random import stat +import csv +import json import torch import torch.distributed as dist import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create -from msprobe.core.common.file_check import FileCheckConst, change_mode +from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, CompareException +from msprobe.core.common.file_check import FileCheckConst, change_mode, FileOpen try: @@ -294,5 +296,23 @@ def _create_logger(level=logging.INFO): return logger_ +def get_json_contents(file_path): + ops = get_file_content_bytes(file_path) + try: + json_obj = json.loads(ops) + except ValueError as error: + logger.error('Failed to load "%s". %s' % (file_path, str(error))) + raise CompareException(CompareException.INVALID_FILE_ERROR) from error + if not isinstance(json_obj, dict): + logger.error('Json file %s, content is not a dictionary!' % file_path) + raise CompareException(CompareException.INVALID_FILE_ERROR) + return json_obj + + +def get_file_content_bytes(file): + with FileOpen(file, 'rb') as file_handle: + return file_handle.read() + + log_level = logging.DEBUG if os.environ.get("API_ACCURACY_CHECK_LOG_LEVEL") == "1" else logging.INFO logger = _create_logger(log_level) -- Gitee From 9fda1f498b70c85299695e377ec0bbc30a46eb5e Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:13:57 +0000 Subject: [PATCH 263/791] update debug/accuracy_tools/msprobe/core/common/utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/core/common/utils.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 7a34a2411..c957b3622 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -520,9 +520,14 @@ def convert_tuple(data): def write_csv(data, filepath): + is_first_create = False + if not os.path.exists(filepath): + is_first_create = True with FileOpen(filepath, 'a+', encoding='utf-8-sig') as f: writer = csv.writer(f) writer.writerows(data) + if is_first_create: + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) def load_npy(filepath): -- Gitee From b080634d4b0680d81fa29fff4055f6507293ee7c Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:14:32 +0000 Subject: [PATCH 264/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/api_precision_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 73bf7c2b8..3d751ab15 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -7,7 +7,7 @@ from collections import namedtuple import torch import pandas as pd -from msprobe.pytorch.api_accuracy_checker.common.utils import write_csv +from msprobe.core.common.utils import write_csv from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ -- Gitee From 45cc29e55d75024b9fe6a2c67e657c78036c5277 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:15:38 +0000 Subject: [PATCH 265/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/api_accuracy_checker/compare/compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 20f04b0cd..bdb3d8fee 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -4,7 +4,7 @@ from collections import namedtuple import torch import numpy as np from msprobe.pytorch.common.log import logger -from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents, write_csv +from msprobe.core.common.utils import write_csv from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \ DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, BinaryStandardApi, \ ULPStandardApi, ThousandthStandardApi, apis_threshold @@ -15,6 +15,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get check_small_value, check_norm_value, get_abs_bench_with_eps, get_ulp_err from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.core.common.const import Const, CompareConst +from msprobe.pytorch.common.utils import get_json_contents ResultInfo = namedtuple('ResultInfo', ['full_api_name', 'fwd_success_status', 'bwd_success_status', -- Gitee From 9766d845bf7eed4c6d36048575c8e3c3c5221b33 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:15:59 +0000 Subject: [PATCH 266/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py. Signed-off-by: jiangchangting1 --- .../pytorch/api_accuracy_checker/run_ut/run_overflow_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index 732745ee8..20e6112d6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -11,7 +11,7 @@ else: import torch from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import exec_api, generate_device_params, get_api_info -from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents +from msprobe.pytorch.common.utils import get_json_contents from msprobe.core.common.file_check import check_link from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward -- Gitee From f0d1d6e9e833402d696ca22a5677025a5fa0af6e Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:19:40 +0000 Subject: [PATCH 267/791] update ut Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/common/test_common_utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py index 56d100f0a..ef6eea318 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py @@ -1,8 +1,12 @@ + +import json +import csv import unittest from unittest.mock import patch from msprobe.pytorch.api_accuracy_checker.common.utils import * - +from msprobe.pytorch.common.utils import get_json_contents +from msprobe.core.common.utils import write_csv class TestUtils(unittest.TestCase): -- Gitee From 6142aa7b2d13520b4dcc6314d058348bc1a87d44 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:20:08 +0000 Subject: [PATCH 268/791] update ut Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/run_ut/test_data_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py index d3a62e5e0..37e871b6c 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py @@ -4,7 +4,7 @@ import unittest import copy from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import * -from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents +from msprobe.pytorch.common.utils import get_json_contents base_dir = os.path.dirname(os.path.realpath(__file__)) forward_file = os.path.join(base_dir, "forward.json") -- Gitee From 847cfa04564acafdb2b45605f347e58c21367faa Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 13 Aug 2024 03:20:31 +0000 Subject: [PATCH 269/791] update ut Signed-off-by: jiangchangting1 --- .../test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py index bc643794a..a413524de 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py @@ -5,7 +5,7 @@ import unittest import torch from unittest.mock import patch, DEFAULT from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import * -from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents +from msprobe.pytorch.common.utils import get_json_contents base_dir = os.path.dirname(os.path.realpath(__file__)) forward_file = os.path.join(base_dir, "forward.json") -- Gitee From 9e35e8181bb5155a449d8bbdc53e4811b6313f9b Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 12 Aug 2024 15:25:39 +0800 Subject: [PATCH 270/791] add online run_ut readme --- debug/accuracy_tools/msprobe/config/README.md | 42 ++-- .../doc/api_accuracy_checker_online.md | 187 ++++++++++++++++++ debug/accuracy_tools/setup.py | 4 +- 3 files changed, 218 insertions(+), 15 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md diff --git a/debug/accuracy_tools/msprobe/config/README.md b/debug/accuracy_tools/msprobe/config/README.md index 7d11a3652..27f48a888 100644 --- a/debug/accuracy_tools/msprobe/config/README.md +++ b/debug/accuracy_tools/msprobe/config/README.md @@ -95,13 +95,20 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 ### task配置为tensor -| 参数名 | 说明 | 是否必选 | -| -------------- | ------------------------------------------------------------ | -------- | -| scope | PyTorch场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。 | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
PyTorch场景配置kernel_api，dump前向和反向API的kernel_api级别数据，其中dump反向API时需要配置**backward_input**参数。前向API配置示例："list": ["Tensor.permute.1.forward"]；反API配置示例："list": ["Tensor.permute.1.forward"], "backward.input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
MindSpore场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | -| backward_input | 该输入文件为首次运行训练dump得到反向API输入的dump文件，str类型，仅PyTorch场景支持，默认未配置。例如若需要dump Functional.conv2d.1 API的反向过程的输入输出，则需要在dump目录下查找命名包含Functional.conv2d.1、backward和input字段的dump文件。配置示例："backward_input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"] | 否 | -| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | -| file_format | MindSpore场景真实tensor数据的保存格式，str类型，可取值"bin"（dump的tensor文件为二进制格式，"level": "L1"时不支持）、"npy"（dump的tensor文件后缀为.npy，默认值）。 | 否 | +| 参数名 | 说明 | 是否必选 | +|----------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------| +| scope | PyTorch场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。 | 否 | +| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
PyTorch场景配置kernel_api，dump前向和反向API的kernel_api级别数据，其中dump反向API时需要配置**backward_input**参数。前向API配置示例："list": ["Tensor.permute.1.forward"]；反API配置示例："list": ["Tensor.permute.1.forward"], "backward.input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
MindSpore场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | +| backward_input | 该输入文件为首次运行训练dump得到反向API输入的dump文件，str类型，仅PyTorch场景支持，默认未配置。例如若需要dump Functional.conv2d.1 API的反向过程的输入输出，则需要在dump目录下查找命名包含Functional.conv2d.1、backward和input字段的dump文件。配置示例："backward_input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"] | 否 | +| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | +| file_format | MindSpore场景真实tensor数据的保存格式，str类型，可取值"bin"（dump的tensor文件为二进制格式，"level": "L1"时不支持）、"npy"（dump的tensor文件后缀为.npy，默认值）。 | 否 | +| online_run_ut | 在线预检模式开关，bool类型，可取值true（开启）、false（关闭），默认未配置，表示关闭。配置为true表示开启在线预检。 | 否 | +| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。仅在online_run_ut字段配置为true时生效，未配置该参数后host和port不生效。 | 否 | +| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的局域网IP地址。仅在online_run_ut字段配置为true时生效，局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | +| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的端口号。仅在online_run_ut字段配置为true时生效，局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | + +说明： +online_run_ut、nfs_path、host、port等字段仅在线预检场景NPU机器生效，详细说明见[《在线精度预检》](../pytorch/doc/api_accuracy_checker_online.md) ### task配置为overflow_check @@ -114,13 +121,20 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 仅PyTorch场景支持。 -| 参数名称 | 说明 | 是否必选 | -| --------------- | ------------------------------------------------------------ | -------- | -| white_list | API dump白名单，仅对指定的API进行dump。配置示例："white_list": ["conv1d", "conv2d"]。默认未配置白名单，即dump全量API数据。 | 否 | -| black_list | API dump黑名单，被指定的API不进行dump。配置示例："black_list": ["conv1d", "conv2d"]。默认未配置黑名单，即dump全量API数据。 | 否 | -| error_data_path | 配置保存精度未达标的API输入输出数据路径，默认为当前路径。配置示例"error_data_path": "./"。 | 否 | - -说明：white_list和black_list同时配置时，二者配置的API名单若无交集，则白名单生效，若API名单存在交集，则白名单排除的部分以及交集的API不进行dump。 +| 参数名称 | 说明 | 是否必选 | +|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------| +| white_list | API dump白名单，仅对指定的API进行dump。配置示例："white_list": ["conv1d", "conv2d"]。默认未配置白名单，即dump全量API数据。 | 否 | +| black_list | API dump黑名单，被指定的API不进行dump。配置示例："black_list": ["conv1d", "conv2d"]。默认未配置黑名单，即dump全量API数据。 | 否 | +| error_data_path | 配置保存精度未达标的API输入输出数据路径，默认为当前路径。配置示例"error_data_path": "./"。 | 否 | +| is_online | 在线预检模式开关，bool类型，可取值true（开启）、false（关闭），默认关闭。 | 否 | +| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效，仅在is_online字段配置为true时生效。 | 否 | +| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机地址127.0.0.1或本机局域网IP。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。仅在is_online字段配置为true时生效。 | 否 | +| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机可用端口。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。仅在is_online字段配置为true时生效。 | 否 | +| rank_list | 指定在线预检的Rank ID，默认值为[0]，list[int]类型，应配置为大于等于0的整数，且须根据实际卡的Rank ID配置，若所配置的值大于实际训练所运行的卡的Rank ID，则在线预检输出数据为空。GPU和NPU须配置一致。仅在is_online字段配置为true时生效。 | 否 | + +说明： +
（1）white_list和black_list同时配置时，二者配置的API名单若无交集，则白名单生效，若API名单存在交集，则白名单排除的部分以及交集的API不进行dump。 +
（2）is_online、nfs_path、host、port、rank_list等字段仅在线预检场景GPU机器生效，详细说明见[《在线精度预检》](../pytorch/doc/api_accuracy_checker_online.md) ## 配置示例 diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md new file mode 100644 index 000000000..5050689a3 --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md @@ -0,0 +1,187 @@ +# 在线精度预检 + +为了应对大模型场景下，通过离线预检方式dump API输入输出数据导致的存储资源紧张问题，提供在线精度预检功能。本功能实现在执行NPU训练操作的过程中，通过TCP/IP协议在NPU +Host与GPU Host设备间建立连接，将NPU上对应API的输入数据在GPU设备上运行，将两份输出数据进行比对，得到预检比对结果，从而减少数据dump的步骤，降低存储资源的占用。 + +## 在线精度预检流程 + +在线精度预检当前支持**局域网场景**和**共享存储场景**，请根据不同的场景选择对应的配置。 + +在线精度预检操作流程如下： + +1. 准备GPU和NPU可正常运行的训练环境，PyTorch版本大于等于2.0，并保证两台Host在同一局域网内可正常通信或能通过共享存储进行通信。 +2. GPU和NPU Host设备上同时安装msprobe工具，详见《[MindStudio精度调试工具](./../../README.md) + 》，其中在线预检要多安装twisted、pyOpenSSL依赖，该依赖为Python模块。 +3. 分别配置GPU侧、NPU侧的config.json文件。 +4. 在GPU侧运行msprobe -f pytorch run_ut -config ./config.json。 +5. 在NPU侧配置训练脚本。 +6. 在NPU侧执行训练。 + +## 在线精度预检操作指导 + +### 配置config.json文件 + +安装完成预检工具后，需要分别在GPU和NPU环境下分别配置config.json。其中需要重点关注文件中的is_online、is_benchmark_device、host和port参数的配置，保障在线预检时GPU和NPU两台设备间的通信正常。 + +#### GPU侧在线预检配置说明 + +| 参数名称 | 说明 | 是否必选 | +|-----------------|-----------------------------------------------------------------------------------------------------------------------|------| +| task | 任务名称，str类型，配置为run_ut表示预检任务。通过其他字段is_online判断离线预检、在线预检任务。 | 是 | +| white_list | 预检的API白名单，list[str]类型。参数示例：white_list=["conv1d", "conv2d"]。默认未配置白名单，即预检全量API数据。 | 否 | +| black_list | 预检的API黑名单，list[str]类型。参数示例：white_list=["conv1d", "conv2d"]。默认未配置黑名单，即预检全量API数据。 | 否 | +| error_data_path | 配置保存精度未达标的API输入输出数据路径，str类型。在线预检模式下该参数不生效。 | 否 | +| is_online | 在线预检模式开关，bool类型，可取值True（开启）、False（关闭），默认关闭。 | 是 | +| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效。 | 否 | +| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机地址127.0.0.1或本机局域网IP。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | +| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机可用端口。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | +| rank_list | 指定在线预检的Rank ID，默认值为[0]，list[int]类型，应配置为大于等于0的整数，且须根据实际卡的Rank ID配置，若所配置的值大于实际训练所运行的卡的Rank ID，则在线预检输出数据为空。GPU和NPU须配置一致。 | 是 | + +#### NPU侧在线预检配置说明 + +| 参数名称 | 说明 | 是否必选 | +|------------------|--------------------------------------------------------------------------------------------------------|------| +| task | 任务名称，str类型，配置为tensor表示dump API统计信息和完全复刻整网的API运行情况的真实数据。通过字段onlin_run_ut判断是否使用在线预检功能。 | 是 | +| dump_path | dump路径，str类型，配置为合法路径即可，兼容tensor任务静态检查 | 是 | +| level | dump级别，str类型，在线预检时配置为L1，表示dump API级精度数据。在线预检可不配置，默认取值L1。 | 是 | +| rank | 指定对某张卡上的数据进行dump，list[int]类型，默认未配置(表示dump所有卡的数据)，需要与GPU侧配置项rank_list保持一致。 | 否 | +| step | 指定dump某个step的数据，list[int]类型，默认未配置，表示dump所有step的数据。dump特定step时，须指定为训练脚本中存在的step。 | 否 | +| seed | 随机种子数，int类型，默认值为1234。通过固定随机数保证模型的输入或输出一致。 | 否 | +| is_deterministic | 确定性计算模式，bool类型，可取值true（开启）或false（关闭），默认关闭。 | 否 | +| scope | dump范围，list[str]类型，默认未配置（list也未配置时师表dump所有api的额数据），配置方式参考[《config配置文件说明》](./../../config/README.md) | 否 | +| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有api的数据），配置方式参考[《config配置文件说明》](./../../config/README.md) | 否 | +| online_run_ut | 在线预检模式开关，bool类型，可取值True（开启）、False（关闭），默认关闭。 | 是 | +| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效。 | 否 | +| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的局域网IP地址。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | +| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的端口号。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | + +#### 局域网场景配置示例 + +若复制下列示例，请删除注释后使用。 + +GPU侧： + +```json +{ + "task": "run_ut", + "run_ut": { + "white_list": [], + "black_list": [], + "error_data_path": "./", + "is_online": true, + "nfs_path": "", + "host": "127.0.0.1", + "port": 59208, + "rank_list": [0] + } +} +``` + +NPU侧： + +```json +{ + "task": "tensor", + "dump_path": "./dump_path", + "rank": [0], + "step": [0], + "level": "L1", + "seed": 1234, + "is_deterministic": true, + "tensor": { + "scope": [], + "list": [], + "online_run_ut": true, + "nfs_path": "", + "host": "xx.xx.xx.x", + "port": 59208 + } +} +``` + +#### 共享存储场景配置示例 + +若复制下列示例，请删除注释后使用。 + +GPU侧： + +```json +{ + "task": "run_ut", + "run_ut": { + "white_list": [], + "black_list": [], + "error_data_path": "./", + "is_online": true, + "nfs_path": "/nfs/xxx/data", + "host": "", + "port": -1, + "rank_list": [0] + } +} +``` + +NPU侧： + +```json +{ + "task": "tensor", + "dump_path": "./dump_path", + "rank": [0], + "step": [0], + "level": "L1", + "seed": 1234, + "is_deterministic": true, + "tensor": { + "scope": [], + "list": [], + "online_run_ut": true, + "nfs_path": "/nfs/xxx/data", + "host": "", + "port": -1 + } +} +``` + +### 在GPU侧运行run_ut + +由于GPU侧为通信接收端，需先于NPU侧执行run_ut操作，命令如下： + +```bash +msprobe -f pytorch run_ut -config ./config.json +``` + +GPU侧配置好config.json文件后执行run_ut命令，此时GPU处于预检等待状态： + +- 局域网场景：当NPU侧启动训练后将预检的API输入和输出数据发送到GPU侧时，GPU启动预检操作。 +- 共享存储场景：当NPU侧启动训练后将预检的API输入和输出数据发送到共享存储时，GPU启动预检操作。 + +### 在NPU侧配置训练脚本 + +在NPU训练脚本中添加如下代码以获取run_ut操作的预检API输入和输出数据： + +```python +from msprobe.pytorch import PrecisionDebugger + +debugger = PrecisionDebugger("config.json") +... + +debugger.start() + +... + +debugger.stop() +debugger.step() +``` + +### 在NPU侧执行训练脚本 + +配置完NPU侧训练脚本后即可执行训练脚本，命令示例如下： + +```bash +bash train.sh +``` + +训练脚本执行完毕后，在GPU侧dump_path目录下生成比对结果文件，详细介绍请参见《[精度预检工具](./api_accuracy_checker.md)》中的” +**预检结果**“。 + diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index afbf8feb3..30e353b81 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -23,7 +23,9 @@ INSTALL_REQUIRED = [ "pyyaml", "rich", "tqdm", - "openpyxl" + "openpyxl", + "pyOpenSSL", + "matplotlib" ] EXCLUDE_PKGS = [ -- Gitee From 52e21e10a4a5f33dffcfcc8d9e5f0bd569571628 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 13 Aug 2024 11:35:27 +0800 Subject: [PATCH 271/791] update msprobe setup requirements --- debug/accuracy_tools/msprobe/README.md | 2 +- debug/accuracy_tools/setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index a89592499..8a889fb82 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -21,7 +21,7 @@ Successfully installed mindstudio_probe-{version} ``` ### 下载whl包安装 -1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、matplotlib依赖。 +1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、pyOpenSSL、twisted、matplotlib依赖。 根据自己的环境选择安装 torch、mindspore。 diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index 8ca7116de..b41f9da15 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -25,6 +25,7 @@ INSTALL_REQUIRED = [ "tqdm", "openpyxl", "pyOpenSSL", + "twisted", "matplotlib" ] -- Gitee From 68d57533b7e1b98ef27b1d902f7fcfa23d7cae9e Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 13 Aug 2024 14:28:08 +0800 Subject: [PATCH 272/791] add einops --- debug/accuracy_tools/msprobe/README.md | 2 +- debug/accuracy_tools/setup.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 8a889fb82..63a420c25 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -21,7 +21,7 @@ Successfully installed mindstudio_probe-{version} ``` ### 下载whl包安装 -1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、pyOpenSSL、twisted、matplotlib依赖。 +1. 使用pip命令安装einops、numpy、openpyxl、pandas、PyYAML、rich、tqdm、pyOpenSSL、twisted、matplotlib依赖。 根据自己的环境选择安装 torch、mindspore。 diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index b41f9da15..c394420e9 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -18,6 +18,7 @@ __version__ = '1.0.2' INSTALL_REQUIRED = [ "wheel", + "einops", "numpy", "pandas >= 1.3.5", "pyyaml", -- Gitee From fe6720e625ebbe1e9696dffbbef3b66784d10260 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 13 Aug 2024 16:39:13 +0800 Subject: [PATCH 273/791] fix conlict --- debug/accuracy_tools/msprobe/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 63a420c25..a89592499 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -21,7 +21,7 @@ Successfully installed mindstudio_probe-{version} ``` ### 下载whl包安装 -1. 使用pip命令安装einops、numpy、openpyxl、pandas、PyYAML、rich、tqdm、pyOpenSSL、twisted、matplotlib依赖。 +1. 使用pip命令安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、matplotlib依赖。 根据自己的环境选择安装 torch、mindspore。 -- Gitee From ea61e904bd66cb5b53c83a2141d12f119657a78b Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 13 Aug 2024 16:41:02 +0800 Subject: [PATCH 274/791] fix conlict --- debug/accuracy_tools/msprobe/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index bbc825325..f752ca8f5 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -25,7 +25,7 @@ Successfully installed mindstudio_probe-{version} 1. 根据实际环境安装torch或mindspore - 2. 安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、einops、matplotlib + 2. 安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、einops、matplotlib、pyOpenSSL、twisted 若环境中已安装部分依赖，不需要重复安装。 -- Gitee From 5527d194982bc6faf8f277a7479dc1580146018c Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 13 Aug 2024 17:33:09 +0800 Subject: [PATCH 275/791] fuzzy match msprobe npy read bugfix --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index e99e194e2..65b3096a8 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -97,8 +97,13 @@ class Comparator: else: try: read_npy_data = getattr(self, "read_npy_data") - n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.PT_SUFFIX) - b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.PT_SUFFIX) + frame_name = getattr(self, "frame_name") + if frame_name == "MSComparator": + n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.NUMPY_SUFFIX) + b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.NUMPY_SUFFIX) + else: + n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.PT_SUFFIX) + b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.PT_SUFFIX) except IOError as error: error_file = error.filename n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE -- Gitee From 99c65b18c5849f6145e79491a47e625d5d5b97d3 Mon Sep 17 00:00:00 2001 From: lijiaojiao Date: Tue, 13 Aug 2024 20:56:54 +0800 Subject: [PATCH 276/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E7=9B=AE?= =?UTF-8?q?=E5=BD=95=E6=9D=83=E9=99=90=E8=AE=BE=E7=BD=AE=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E6=BD=9C=E5=9C=A8=E7=9A=84cur=5Fdir=E6=9C=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/kj600/kj600/module_hook.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/kj600/kj600/module_hook.py b/debug/accuracy_tools/kj600/kj600/module_hook.py index 74ef684a6..c0741b360 100644 --- a/debug/accuracy_tools/kj600/kj600/module_hook.py +++ b/debug/accuracy_tools/kj600/kj600/module_hook.py @@ -132,8 +132,8 @@ class TrainerMon: cur_time = datetime.now().strftime('%b%d_%H-%M-%S') unique_id = str(uuid.uuid4())[:8] if dist.is_initialized(): + cur_path = os.path.join(output_base_dir, f"{cur_time}-rank{dist.get_rank()}-{unique_id}") if (dist.get_rank() in self.module_rank_list) or len(self.module_rank_list) == 0: - cur_path = os.path.join(output_base_dir, f"{cur_time}-rank{dist.get_rank()}-{unique_id}") check_path_length(cur_path) check_path_pattern_valid(cur_path) self.summary_writer = SummaryWriterWithAD( @@ -145,7 +145,7 @@ class TrainerMon: self.summary_writer = SummaryWriterWithAD(cur_path, self.alert_rules, unique_id, anomaly_inform) full_path = os.path.realpath(cur_path) - change_mode(full_path,FileCheckConst.DATA_FILE_AUTHORITY) + change_mode(full_path,FileCheckConst.DATA_DIR_AUTHORITY) # A HeatmapVisualizer instance is associated with an image self.update_heatmap_visualizer = defaultdict(HeatmapVisualizer) -- Gitee From 7b92bf34e80dc32e4ac161317c27d78a82bb2ec0 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Tue, 13 Aug 2024 21:09:22 +0800 Subject: [PATCH 277/791] =?UTF-8?q?=E6=A3=80=E8=A7=86=E6=84=8F=E8=A7=81?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/utils.py | 30 +++++++++---------- .../mindspore/compare/distributed_compare.py | 4 +-- .../msprobe/mindspore/compare/ms_compare.py | 4 +-- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 510403bf3..ef1604c0f 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -66,14 +66,14 @@ def rename_api(npu_name, process): def read_op(op_data, op_name): op_parsed_list = Const.DEFAULT_LIST - if 'forward' in op_name: - if 'input_args' in op_data: - input_item = op_data['input_args'] + if Const.FORWARD in op_name: + if Const.INPUT in op_data: + input_item = op_data[Const.INPUT] input_parsed_list = op_item_parse(input_item, op_name + '_input', None) op_parsed_list = input_parsed_list.copy() input_parsed_list.clear() - if 'input_kwargs' in op_data: - kwargs_item = op_data['input_kwargs'] + if Const.INPUT_KWARGS in op_data: + kwargs_item = op_data[Const.INPUT_KWARGS] if isinstance(kwargs_item, dict) and "type" in kwargs_item or isinstance(kwargs_item, list): kwarg_parsed_list = op_item_parse(kwargs_item, op_name + '_input', None) op_parsed_list += kwarg_parsed_list @@ -83,19 +83,19 @@ def read_op(op_data, op_name): kwarg_parsed_list = op_item_parse(kwargs_item[kwarg], op_name + '_input.' + kwarg, None) op_parsed_list += kwarg_parsed_list kwarg_parsed_list.clear() - if 'output' in op_data: - output_item = op_data['output'] + if Const.OUTPUT in op_data: + output_item = op_data[Const.OUTPUT] output_parsed_list = op_item_parse(output_item, op_name + '_output', None) op_parsed_list += output_parsed_list output_parsed_list.clear() - if 'backward' in op_name: - if 'input' in op_data: - input_item = op_data['input'] + if Const.BACKWARD in op_name: + if Const.INPUT in op_data: + input_item = op_data[Const.INPUT] input_parsed_list = op_item_parse(input_item, op_name + '_input', None) op_parsed_list = input_parsed_list.copy() input_parsed_list.clear() - if 'output' in op_data: - output_item = op_data['output'] + if Const.OUTPUT in op_data: + output_item = op_data[Const.OUTPUT] output_parsed_list = op_item_parse(output_item, op_name + '_output', None) op_parsed_list += output_parsed_list output_parsed_list.clear() @@ -198,7 +198,7 @@ def resolve_api_special_parameters(data_dict, full_op_name, item_list): for key, value in data_dict.items(): if isinstance(value, dict): parsed_item = value - parts = full_op_name.split(".") + parts = full_op_name.split(Const.SEP) parts.insert(-1, key) full_op_name_new = ".".join(parts) parsed_item['full_op_name'] = full_op_name_new @@ -312,8 +312,8 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals n_num = len(n_dict['op_name']) b_num = len(b_dict['op_name']) - n_num_input = len([name for name in n_dict['op_name'] if 'input' in name]) - b_num_input = len([name for name in b_dict['op_name'] if 'input' in name]) + n_num_input = len([name for name in n_dict['op_name'] if Const.INPUT in name]) + b_num_input = len([name for name in b_dict['op_name'] if Const.INPUT in name]) n_num_kwarg = len([name for name in n_dict['op_name'] if 'kwarg' in name]) b_num_kwarg = len([name for name in b_dict['op_name'] if 'kwarg' in name]) n_num_output = n_num - n_num_input - n_num_kwarg diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 08f0a03ec..c4054ed05 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -60,6 +60,6 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - msComparator = MSComparator() - msComparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, + ms_comparator = MSComparator() + ms_comparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index a4736a91b..a11be362f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -185,7 +185,7 @@ def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - msComparator = MSComparator() - msComparator.compare_core(input_param, output_path, stack_mode=stack_mode, + ms_comparator = MSComparator() + ms_comparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) \ No newline at end of file -- Gitee From dc892985a4d56b6a1c0e4e0c53460d77fe24b09a Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 14 Aug 2024 02:04:47 +0800 Subject: [PATCH 278/791] =?UTF-8?q?=E7=BB=93=E5=90=88=E7=82=9C=E5=93=A5?= =?UTF-8?q?=E4=BB=A3=E7=A0=81=E7=9A=84=E6=A3=80=E8=A7=86=E6=84=8F=E8=A7=81?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/utils.py | 36 ++-- .../msprobe/core/compare/acc_compare.py | 167 +++++++++++++++++- .../{pytorch => core}/compare/compare_cli.py | 19 +- .../msprobe/mindspore/compare/compare_cli.py | 23 --- .../mindspore/compare/distributed_compare.py | 2 +- .../msprobe/mindspore/compare/ms_compare.py | 162 +---------------- debug/accuracy_tools/msprobe/msprobe.py | 13 +- .../msprobe/pytorch/compare/pt_compare.py | 163 +---------------- 8 files changed, 199 insertions(+), 386 deletions(-) rename debug/accuracy_tools/msprobe/{pytorch => core}/compare/compare_cli.py (56%) delete mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index e960dc98b..26f0f6905 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -153,28 +153,20 @@ def check_compare_param(input_param, output_path, summary_compare=False, md5_com if not (isinstance(input_param, dict) and isinstance(output_path, str)): logger.error("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) - if framework == Const.MS_FRAMEWORK: - check_file_or_directory_path(input_param.get("npu_path"), False) - check_file_or_directory_path(input_param.get("bench_path"), False) - check_file_or_directory_path(input_param.get("stack_path"), False) - else: - check_file_or_directory_path(input_param.get("npu_json_path"), False) - check_file_or_directory_path(input_param.get("bench_json_path"), False) - check_file_or_directory_path(input_param.get("stack_json_path"), False) + + check_file_or_directory_path(input_param.get("npu_path"), False) + check_file_or_directory_path(input_param.get("bench_path"), False) + check_file_or_directory_path(input_param.get("stack_path"), False) if not summary_compare and not md5_compare: check_file_or_directory_path(input_param.get("npu_dump_data_dir"), True) check_file_or_directory_path(input_param.get("bench_dump_data_dir"), True) check_file_or_directory_path(output_path, True) - if framework == Const.MS_FRAMEWORK: - with FileOpen(input_param.get("npu_path"), "r") as npu_json, \ - FileOpen(input_param.get("bench_path"), "r") as bench_json, \ - FileOpen(input_param.get("stack_path"), "r") as stack_json: - check_json_file(input_param, npu_json, bench_json, stack_json) - else: - with FileOpen(input_param.get("npu_json_path"), "r") as npu_json, \ - FileOpen(input_param.get("bench_json_path"), "r") as bench_json, \ - FileOpen(input_param.get("stack_json_path"), "r") as stack_json: - check_json_file(input_param, npu_json, bench_json, stack_json) + + with FileOpen(input_param.get("npu_path"), "r") as npu_json, \ + FileOpen(input_param.get("bench_path"), "r") as bench_json, \ + FileOpen(input_param.get("stack_path"), "r") as stack_json: + check_json_file(input_param, npu_json, bench_json, stack_json) + def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False): @@ -486,12 +478,8 @@ def md5_find(data): def task_dumppath_get(input_param, framework=Const.MS_FRAMEWORK): - if framework == Const.MS_FRAMEWORK: - npu_path = input_param.get("npu_path", None) - bench_path = input_param.get("bench_path", None) - else: - npu_path = input_param.get("npu_json_path", None) - bench_path = input_param.get("bench_json_path", None) + npu_path = input_param.get("npu_path", None) + bench_path = input_param.get("bench_path", None) if not npu_path or not bench_path: logger.error(f"Please check the json path is valid.") raise CompareException(CompareException.INVALID_PATH_ERROR) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 65b3096a8..be749e5aa 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -1,14 +1,19 @@ import multiprocessing +import os +import json import pandas as pd +from msprobe.core.common.file_check import FileOpen from msprobe.core.common.const import CompareConst, Const -from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ - get_error_message from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.compare.utils import read_op, merge_tensor, CompareException -from msprobe.core.compare.multiprocessing_compute import _handle_multi_process from msprobe.core.common.log import logger +from msprobe.core.common.utils import add_time_with_xlsx, CompareException, check_file_not_exists from msprobe.core.compare.check import check_graph_mode, check_struct_match, fuzzy_check_op - +from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx +from msprobe.core.compare.utils import read_op, merge_tensor, CompareException, get_un_match_accuracy, get_accuracy +from msprobe.core.compare.multiprocessing_compute import _handle_multi_process, ComparisonResult, _save_cmp_result +from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ + get_error_message +from msprobe.core.advisor.advisor import Advisor class Comparator: @@ -87,6 +92,71 @@ class Comparator: return n_index, len(bench_queue) - 1 return -1, -1 + def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): + npu_json_handle, bench_json_handle, stack_json_handle = file_handles + npu_json_data = json.load(npu_json_handle) + bench_json_data = json.load(bench_json_handle) + stack_json_data = json.load(stack_json_handle) + + if fuzzy_match: + logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") + + npu_ops_queue = [] + bench_ops_queue = [] + result = [] + + ops_npu_iter = iter(npu_json_data['data']) + ops_bench_iter = iter(bench_json_data['data']) + read_err_npu = True + read_err_bench = True + last_npu_ops_len = 0 + last_bench_ops_len = 0 + + while True: + if not read_err_npu and not read_err_bench: + break + try: + last_npu_ops_len = len(npu_ops_queue) + op_name_npu = next(ops_npu_iter) + read_err_npu = True + npu_merge_list = self.gen_merge_list(npu_json_data,op_name_npu,stack_json_data,summary_compare,md5_compare) + if npu_merge_list: + npu_ops_queue.append(npu_merge_list) + except StopIteration: + read_err_npu = False + try: + last_bench_ops_len = len(bench_ops_queue) + op_name_bench = next(ops_bench_iter) + bench_merge_list = self.gen_merge_list(bench_json_data,op_name_bench,stack_json_data,summary_compare,md5_compare) + if bench_merge_list: + bench_ops_queue.append(bench_merge_list) + except StopIteration: + read_err_bench = False + + # merge all boolean expressions + both_empty = not npu_ops_queue and not bench_ops_queue + no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) + if both_empty or no_change: + continue + + n_match_point, b_match_point = self.match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) + if n_match_point == -1 and b_match_point == -1: + continue + n_match_data = npu_ops_queue[n_match_point] + b_match_data = bench_ops_queue[b_match_point] + un_match_data = npu_ops_queue[0: n_match_point] + for npu_data in un_match_data: + get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) + get_accuracy(result, n_match_data, b_match_data, summary_compare, md5_compare) + del npu_ops_queue[0: n_match_point + 1] + del bench_ops_queue[0: b_match_point + 1] + if npu_ops_queue: + for npu_data in npu_ops_queue: + get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) + + result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) + return result_df + def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): npu_bench_name_list = op_name_mapping_dict[npu_op_name] data_name = npu_bench_name_list[1] @@ -126,10 +196,93 @@ class Comparator: result_list.append(err_msg) return result_list + def compare_core(self,input_parma, output_path, **kwargs): + """ + Compares data from multiple JSON files and generates a comparison report. + + Args: + input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", + "stack_path"). + output_path (str): The path where the output Excel report will be saved. + **kwargs: Additional keyword arguments including: + - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. + - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. + - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. + - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. + - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. + - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. + + Returns: + """ + # get kwargs or set default value + stack_mode = kwargs.get('stack_mode', False) + auto_analyze = kwargs.get('auto_analyze', True) + suffix = kwargs.get('suffix', '') + fuzzy_match = kwargs.get('fuzzy_match', False) + summary_compare = kwargs.get('summary_compare', False) + md5_compare = kwargs.get('md5_compare', False) + + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") + file_name = add_time_with_xlsx("compare_result" + suffix) + file_path = os.path.join(os.path.realpath(output_path), file_name) + check_file_not_exists(file_path) + highlight_dict = {'red_rows': [], 'yellow_rows': []} + + with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ + FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ + FileOpen(input_parma.get("stack_json_path"), "r") as stack_json: + result_df = self.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, + summary_compare, md5_compare) + + if not md5_compare and not summary_compare: + result_df = self._do_multi_process(input_parma, result_df) + find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) + highlight_rows_xlsx(result_df, highlight_dict, file_path) + if auto_analyze: + advisor = Advisor(result_df, output_path) + advisor.analysis() + + def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): + cos_result = [] + max_err_result = [] + max_relative_err_result = [] + err_mess = [] + one_thousand_err_ratio_result = [] + five_thousand_err_ratio_result = [] + is_print_compare_log = input_param.get("is_print_compare_log") + for i in range(len(result_df)): + npu_op_name = result_df.iloc[i, 0] + bench_op_name = result_df.iloc[i, 1] + if is_print_compare_log: + logger.info("start compare: {}".format(npu_op_name)) + cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = self.compare_by_op( + npu_op_name, bench_op_name, dump_path_dict, input_param) + if is_print_compare_log: + logger.info( + "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " + "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, err_msg, + one_thousand_err_ratio, five_thousand_err_ratio)) + cos_result.append(cos_sim) + max_err_result.append(max_abs_err) + max_relative_err_result.append(max_relative_err) + err_mess.append(err_msg) + one_thousand_err_ratio_result.append(one_thousand_err_ratio) + five_thousand_err_ratio_result.append(five_thousand_err_ratio) + + cr = ComparisonResult( + cos_result = cos_result, + max_err_result = max_err_result, + max_relative_err_result=max_relative_err_result, + err_msgs = err_mess, + one_thousand_err_ratio_result = one_thousand_err_ratio_result, + five_thousand_err_ratio_result = five_thousand_err_ratio_result + ) + + return _save_cmp_result(idx, cr, result_df, lock) + def _do_multi_process(self,input_parma, result_df): try: - compare_ops = getattr(self,"compare_ops") - result_df = _handle_multi_process(compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) + result_df = _handle_multi_process(self.compare_ops, input_parma, result_df, multiprocessing.Manager().RLock()) return result_df except ValueError as e: logger.error('result dataframe is not found.') diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py similarity index 56% rename from debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py rename to debug/accuracy_tools/msprobe/core/compare/compare_cli.py index 8d5e048fb..e4e257bc6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -3,11 +3,9 @@ from msprobe.core.common.file_check import FileOpen, check_file_type from msprobe.core.common.const import FileCheckConst from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger -from msprobe.pytorch.compare.pt_compare import compare -from msprobe.pytorch.compare.distributed_compare import compare_distributed -def compare_cli(args): +def compare_cli(args,frame_name): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) npu_path = input_param.get("npu_path", None) @@ -17,11 +15,22 @@ def compare_cli(args): input_param["npu_json_path"] = input_param.pop("npu_path") input_param["bench_json_path"] = input_param.pop("bench_path") input_param["stack_json_path"] = input_param.pop("stack_path") - compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, + if frame_name == "pytorch": + from msprobe.pytorch.compare.pt_compare import compare + from msprobe.pytorch.compare.distributed_compare import compare_distributed + compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, + fuzzy_match=args.fuzzy_match) + else: + from msprobe.mindspore.compare.ms_compare import ms_compare + from msprobe.mindspore.compare.distributed_compare import ms_compare_distributed + ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} - compare_distributed(npu_path, bench_path, args.output_path, **kwargs) + if frame_name == "pytorch": + compare_distributed(npu_path, bench_path, args.output_path, **kwargs) + else: + ms_compare_distributed(npu_path, bench_path, args.output_path, **kwargs) else: logger.error("The npu_path and bench_path need to be of the same type.") raise CompareException(CompareException.INVALID_COMPARE_MODE) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py b/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py deleted file mode 100644 index 4a8149657..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/compare/compare_cli.py +++ /dev/null @@ -1,23 +0,0 @@ -import json -from msprobe.core.common.file_check import FileOpen, check_file_type -from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.utils import CompareException -from msprobe.core.common.log import logger -from msprobe.mindspore.compare.ms_compare import ms_compare -from msprobe.mindspore.compare.distributed_compare import compare_distributed - -def compare_cli_ms(args): - with FileOpen(args.input_path, "r") as file: - input_param = json.load(file) - npu_path = input_param.get("npu_path", None) - bench_path = input_param.get("bench_path", None) - - if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: - ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, - fuzzy_match=args.fuzzy_match) - elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: - kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} - compare_distributed(npu_path, bench_path, args.output_path, **kwargs) - else: - logger.error("The npu_path and bench_path need to be of the same type.") - raise CompareException(CompareException.INVALID_COMPARE_MODE) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index c4054ed05..b10b17128 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -24,7 +24,7 @@ from msprobe.mindspore.compare.ms_compare import MSComparator from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json -def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): +def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): if kwargs.get('suffix'): logger.error("Argument 'suffix' is not supported for compare_distributed.") raise CompareException(CompareException.INVALID_PARAM_ERROR) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 457c28c1c..3807acb99 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,127 +1,17 @@ -import json import os.path import numpy as np -from msprobe.core.advisor.advisor import Advisor -from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ - check_file_not_exists, check_configuration_param, task_dumppath_get -from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory +from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, task_dumppath_get +from msprobe.core.common.file_check import FileChecker, create_directory from msprobe.core.common.const import FileCheckConst from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy -from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result -from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator - class MSComparator (Comparator): - def __init__(self): self.frame_name = MSComparator.__name__ - def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): - cos_result = [] - max_err_result = [] - max_relative_err_result = [] - err_mess = [] - one_thousand_err_ratio_result = [] - five_thousand_err_ratio_result = [] - is_print_compare_log = input_param.get("is_print_compare_log") - for i in range(len(result_df)): - npu_op_name = result_df.iloc[i, 0] - bench_op_name = result_df.iloc[i, 1] - if is_print_compare_log: - logger.info("start compare: {}".format(npu_op_name)) - cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = self.compare_by_op( - npu_op_name, bench_op_name, dump_path_dict, input_param) - if is_print_compare_log: - logger.info( - "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " - "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, err_msg, - one_thousand_err_ratio, five_thousand_err_ratio)) - cos_result.append(cos_sim) - max_err_result.append(max_abs_err) - max_relative_err_result.append(max_relative_err) - err_mess.append(err_msg) - one_thousand_err_ratio_result.append(one_thousand_err_ratio) - five_thousand_err_ratio_result.append(five_thousand_err_ratio) - - cr = ComparisonResult( - cos_result = cos_result, - max_err_result = max_err_result, - max_relative_err_result = max_relative_err_result, - err_msgs = err_mess, - one_thousand_err_ratio_result = one_thousand_err_ratio_result, - five_thousand_err_ratio_result = five_thousand_err_ratio_result - ) - - return _save_cmp_result(idx, cr, result_df, lock) - - def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): - npu_json_handle, bench_json_handle, stack_json_handle = file_handles - npu_json_data = json.load(npu_json_handle) - bench_json_data = json.load(bench_json_handle) - stack_json_data = json.load(stack_json_handle) - - if fuzzy_match: - logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") - - npu_ops_queue = [] - bench_ops_queue = [] - result = [] - - ops_npu_iter = iter(npu_json_data['data']) - ops_bench_iter = iter(bench_json_data['data']) - read_err_npu = True - read_err_bench = True - last_npu_ops_len = 0 - last_bench_ops_len = 0 - - while True: - if not read_err_npu and not read_err_bench: - break - try: - last_npu_ops_len = len(npu_ops_queue) - op_name_npu = next(ops_npu_iter) - read_err_npu = True - npu_merge_list = self.gen_merge_list(npu_json_data,op_name_npu,stack_json_data,summary_compare, md5_compare) - if npu_merge_list: - npu_ops_queue.append(npu_merge_list) - except StopIteration: - read_err_npu = False - try: - last_bench_ops_len = len(bench_ops_queue) - op_name_bench = next(ops_bench_iter) - bench_merge_list = self.gen_merge_list(bench_json_data,op_name_bench,stack_json_data,summary_compare, md5_compare) - if bench_merge_list: - bench_ops_queue.append(bench_merge_list) - except StopIteration: - read_err_bench = False - - # merge all boolean expressions - both_empty = not npu_ops_queue and not bench_ops_queue - no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) - if both_empty or no_change: - continue - - n_match_point, b_match_point = super().match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) - if n_match_point == -1 and b_match_point == -1: - continue - n_match_data = npu_ops_queue[n_match_point] - b_match_data = bench_ops_queue[b_match_point] - un_match_data = npu_ops_queue[0: n_match_point] - for npu_data in un_match_data: - get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - get_accuracy(result, n_match_data, b_match_data, summary_compare, md5_compare) - del npu_ops_queue[0: n_match_point + 1] - del bench_ops_queue[0: b_match_point + 1] - if npu_ops_queue: - for npu_data in npu_ops_queue: - get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) - return result_df - def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -130,55 +20,9 @@ class MSComparator (Comparator): data_value = np.load(data_path) # detach for less memory if data_value.dtype == np.float16: data_value = data_value.astype(np.float32) - return data_value - def compare_core(self,input_parma, output_path, **kwargs): - """ - Compares data from multiple JSON files and generates a comparison report. - - Args: - input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", - "stack_path"). - output_path (str): The path where the output Excel report will be saved. - **kwargs: Additional keyword arguments including: - - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. - - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. - - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. - - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. - - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. - - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. - - Returns: - """ - # get kwargs or set default value - stack_mode = kwargs.get('stack_mode', False) - auto_analyze = kwargs.get('auto_analyze', True) - suffix = kwargs.get('suffix', '') - fuzzy_match = kwargs.get('fuzzy_match', False) - summary_compare = kwargs.get('summary_compare', False) - md5_compare = kwargs.get('md5_compare', False) - - logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - file_name = add_time_with_xlsx("compare_result" + suffix) - file_path = os.path.join(os.path.realpath(output_path), file_name) - check_file_not_exists(file_path) - highlight_dict = {'red_rows': [], 'yellow_rows': []} - with FileOpen(input_parma.get("npu_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_path"), "r") as stack_json: - result_df = self.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, - summary_compare, md5_compare) - - if not md5_compare and not summary_compare: - result_df = self._do_multi_process(input_parma, result_df) - find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) - highlight_rows_xlsx(result_df, highlight_dict, file_path) - if auto_analyze: - advisor = Advisor(result_df, output_path) - advisor.analysis() - - + def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 54b4a12d0..af8034077 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -18,13 +18,12 @@ import sys import importlib.util from msprobe.core.compare.utils import _compare_parser from msprobe.core.common.log import logger - +from msprobe.core.compare.compare_cli import compare_cli def is_module_available(module_name): spec =importlib.util.find_spec(module_name) return spec is not None - def main(): parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, @@ -53,7 +52,7 @@ def main(): _api_precision_compare_command from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command - from msprobe.pytorch.compare.compare_cli import compare_cli + _run_ut_parser(run_ut_cmd_parser) _run_ut_parser(multi_run_ut_cmd_parser) multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, @@ -81,15 +80,13 @@ def main(): elif sys.argv[3] == "run_overflow_check": _run_overflow_check_command(args) elif sys.argv[3] == "compare": - compare_cli(args) + compare_cli(args,"pytorch") else: - if is_module_available("mindspore"): - from msprobe.mindspore.compare.compare_cli import compare_cli_ms - else: + if not is_module_available("mindspore"): logger.error("MindSpore does not exit, please install MindSpore library") raise Exception("MindSpore does not exit, please install MindSpore library") if sys.argv[3] == "compare": - compare_cli_ms(args) + compare_cli(args,"mindspore") if __name__ == "__main__": main() diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 82c58029e..d95666b01 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,126 +1,17 @@ -import json + import os.path import torch -from msprobe.core.advisor.advisor import Advisor -from msprobe.core.common.utils import check_compare_param, add_time_with_xlsx, CompareException, \ - check_file_not_exists, check_configuration_param, task_dumppath_get -from msprobe.core.common.file_check import FileChecker, FileOpen, create_directory from msprobe.core.common.const import FileCheckConst, Const from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.compare.utils import get_un_match_accuracy, get_accuracy -from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _save_cmp_result -from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.acc_compare import Comparator - +from msprobe.core.common.utils import create_directory, check_configuration_param, task_dumppath_get, check_compare_param, FileChecker +from msprobe.core.common.utils import CompareException class PTComparator (Comparator): def __init__(self): self.frame_name=PTComparator.__name__ - def compare_ops(self, idx, dump_path_dict, result_df, lock, input_param): - cos_result = [] - max_err_result = [] - max_relative_err_result = [] - err_mess = [] - one_thousand_err_ratio_result = [] - five_thousand_err_ratio_result = [] - is_print_compare_log = input_param.get("is_print_compare_log") - for i in range(len(result_df)): - npu_op_name = result_df.iloc[i, 0] - bench_op_name = result_df.iloc[i, 1] - if is_print_compare_log: - logger.info("start compare: {}".format(npu_op_name)) - cos_sim, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg = self.compare_by_op( - npu_op_name, bench_op_name, dump_path_dict, input_param) - if is_print_compare_log: - logger.info( - "[{}] Compare result: cosine {}, max_abs_err {}, max_relative_err {}, {}, one_thousand_err_ratio {}, " - "five_thousand_err_ratio {}".format(npu_op_name, cos_sim, max_abs_err, max_relative_err, err_msg, - one_thousand_err_ratio, five_thousand_err_ratio)) - cos_result.append(cos_sim) - max_err_result.append(max_abs_err) - max_relative_err_result.append(max_relative_err) - err_mess.append(err_msg) - one_thousand_err_ratio_result.append(one_thousand_err_ratio) - five_thousand_err_ratio_result.append(five_thousand_err_ratio) - - cr = ComparisonResult( - cos_result = cos_result, - max_err_result = max_err_result, - max_relative_err_result=max_relative_err_result, - err_msgs = err_mess, - one_thousand_err_ratio_result = one_thousand_err_ratio_result, - five_thousand_err_ratio_result = five_thousand_err_ratio_result - ) - - return _save_cmp_result(idx, cr, result_df, lock) - - def compare_process(self,file_handles, stack_mode, fuzzy_match, summary_compare=False, md5_compare=False): - npu_json_handle, bench_json_handle, stack_json_handle = file_handles - npu_json_data = json.load(npu_json_handle) - bench_json_data = json.load(bench_json_handle) - stack_json_data = json.load(stack_json_handle) - - if fuzzy_match: - logger.warning("This task uses fuzzy matching, which may affect the accuracy of the comparison.") - - npu_ops_queue = [] - bench_ops_queue = [] - result = [] - - ops_npu_iter = iter(npu_json_data['data']) - ops_bench_iter = iter(bench_json_data['data']) - read_err_npu = True - read_err_bench = True - last_npu_ops_len = 0 - last_bench_ops_len = 0 - - while True: - if not read_err_npu and not read_err_bench: - break - try: - last_npu_ops_len = len(npu_ops_queue) - op_name_npu = next(ops_npu_iter) - read_err_npu = True - npu_merge_list = self.gen_merge_list(npu_json_data,op_name_npu,stack_json_data,summary_compare,md5_compare) - if npu_merge_list: - npu_ops_queue.append(npu_merge_list) - except StopIteration: - read_err_npu = False - try: - last_bench_ops_len = len(bench_ops_queue) - op_name_bench = next(ops_bench_iter) - bench_merge_list = self.gen_merge_list(bench_json_data,op_name_bench,stack_json_data,summary_compare,md5_compare) - if bench_merge_list: - bench_ops_queue.append(bench_merge_list) - except StopIteration: - read_err_bench = False - - # merge all boolean expressions - both_empty = not npu_ops_queue and not bench_ops_queue - no_change = (len(npu_ops_queue) == last_npu_ops_len) and (len(bench_ops_queue) == last_bench_ops_len) - if both_empty or no_change: - continue - - n_match_point, b_match_point = super().match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) - if n_match_point == -1 and b_match_point == -1: - continue - n_match_data = npu_ops_queue[n_match_point] - b_match_data = bench_ops_queue[b_match_point] - un_match_data = npu_ops_queue[0: n_match_point] - for npu_data in un_match_data: - get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - get_accuracy(result, n_match_data, b_match_data, summary_compare, md5_compare) - del npu_ops_queue[0: n_match_point + 1] - del bench_ops_queue[0: b_match_point + 1] - if npu_ops_queue: - for npu_data in npu_ops_queue: - get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - - result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) - return result_df - def read_npy_data(self,dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, @@ -132,53 +23,7 @@ class PTComparator (Comparator): data_value = data_value.numpy() return data_value - def compare_core(self,input_parma, output_path, **kwargs): - """ - Compares data from multiple JSON files and generates a comparison report. - - Args: - input_parma (dict): A dictionary containing paths to JSON files ("npu_path", "bench_path", - "stack_path"). - output_path (str): The path where the output Excel report will be saved. - **kwargs: Additional keyword arguments including: - - stack_mode (bool, optional): Enables stack mode comparison. Defaults to False. - - auto_analyze (bool, optional): If True, triggers automatic analysis after comparison. Defaults to True. - - suffix (str, optional): Suffix to append to the output file name. Defaults to ''. - - fuzzy_match (bool, optional): Enables fuzzy matching during comparison. Defaults to False. - - summary_compare (bool, optional): Enables summary comparison mode. Defaults to False. - - md5_compare (bool, optional): Enables MD5 comparison. Defaults to False. - - Returns: - """ - # get kwargs or set default value - stack_mode = kwargs.get('stack_mode', False) - auto_analyze = kwargs.get('auto_analyze', True) - suffix = kwargs.get('suffix', '') - fuzzy_match = kwargs.get('fuzzy_match', False) - summary_compare = kwargs.get('summary_compare', False) - md5_compare = kwargs.get('md5_compare', False) - - logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - file_name = add_time_with_xlsx("compare_result" + suffix) - file_path = os.path.join(os.path.realpath(output_path), file_name) - check_file_not_exists(file_path) - highlight_dict = {'red_rows': [], 'yellow_rows': []} - - with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ - FileOpen(input_parma.get("bench_json_path"), "r") as bench_json, \ - FileOpen(input_parma.get("stack_json_path"), "r") as stack_json: - result_df = self.compare_process([npu_json, bench_json, stack_json], stack_mode, fuzzy_match, - summary_compare, md5_compare) - - if not md5_compare and not summary_compare: - result_df = self._do_multi_process(input_parma, result_df) - find_compare_result_error_rows(result_df, highlight_dict, summary_compare, md5_compare) - highlight_rows_xlsx(result_df, highlight_dict, file_path) - if auto_analyze: - advisor = Advisor(result_df, output_path) - advisor.analysis() - - + def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param, framework=Const.PT_FRAMEWORK) -- Gitee From 7734593d5a2cc9f710f74911eb12124b12832d9d Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 14 Aug 2024 03:02:36 +0800 Subject: [PATCH 279/791] =?UTF-8?q?=E5=AD=97=E6=AE=B5bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/utils.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 26f0f6905..e7fc2500d 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -154,17 +154,17 @@ def check_compare_param(input_param, output_path, summary_compare=False, md5_com logger.error("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) - check_file_or_directory_path(input_param.get("npu_path"), False) - check_file_or_directory_path(input_param.get("bench_path"), False) - check_file_or_directory_path(input_param.get("stack_path"), False) + check_file_or_directory_path(input_param.get("npu_json_path"), False) + check_file_or_directory_path(input_param.get("bench_json_path"), False) + check_file_or_directory_path(input_param.get("stack_json_path"), False) if not summary_compare and not md5_compare: check_file_or_directory_path(input_param.get("npu_dump_data_dir"), True) check_file_or_directory_path(input_param.get("bench_dump_data_dir"), True) check_file_or_directory_path(output_path, True) - with FileOpen(input_param.get("npu_path"), "r") as npu_json, \ - FileOpen(input_param.get("bench_path"), "r") as bench_json, \ - FileOpen(input_param.get("stack_path"), "r") as stack_json: + with FileOpen(input_param.get("npu_json_path"), "r") as npu_json, \ + FileOpen(input_param.get("bench_json_path"), "r") as bench_json, \ + FileOpen(input_param.get("stack_json_path"), "r") as stack_json: check_json_file(input_param, npu_json, bench_json, stack_json) @@ -205,9 +205,9 @@ def _check_json(json_file_handle, file_name): def check_json_file(input_param, npu_json, bench_json, stack_json): - _check_json(npu_json, input_param.get("npu_path")) - _check_json(bench_json, input_param.get("bench_path")) - _check_json(stack_json, input_param.get("stack_path")) + _check_json(npu_json, input_param.get("npu_json_path")) + _check_json(bench_json, input_param.get("bench_json_path")) + _check_json(stack_json, input_param.get("stack_json_path")) def check_file_size(input_file, max_size): @@ -478,8 +478,8 @@ def md5_find(data): def task_dumppath_get(input_param, framework=Const.MS_FRAMEWORK): - npu_path = input_param.get("npu_path", None) - bench_path = input_param.get("bench_path", None) + npu_path = input_param.get("npu_json_path", None) + bench_path = input_param.get("bench_json_path", None) if not npu_path or not bench_path: logger.error(f"Please check the json path is valid.") raise CompareException(CompareException.INVALID_PATH_ERROR) -- Gitee From daca5b2feaf7c7f31f79af426c1a6ff79de8f5d4 Mon Sep 17 00:00:00 2001 From: yangxinxian <947098055@qq.com> Date: Wed, 14 Aug 2024 09:39:37 +0800 Subject: [PATCH 280/791] =?UTF-8?q?=E6=8F=90=E4=BA=A4PR=E6=A8=A1=E6=9D=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitee/PULL_REQUEST_TEMPLATE.zh-CN.md | 55 +++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 .gitee/PULL_REQUEST_TEMPLATE.zh-CN.md diff --git a/.gitee/PULL_REQUEST_TEMPLATE.zh-CN.md b/.gitee/PULL_REQUEST_TEMPLATE.zh-CN.md new file mode 100644 index 000000000..a4049752f --- /dev/null +++ b/.gitee/PULL_REQUEST_TEMPLATE.zh-CN.md @@ -0,0 +1,55 @@ +# PR 合入模板 + +## 1. 修改描述 +- **修改原因：** +- **修改内容：** + +--- + +## 2. 功能验证 +- [ ] **功能自验** +- [ ] **本地自验用例截图**（请确保不体现个人信息） +- [ ] **冒烟是否通过** + +--- + +## 3. 代码检视 +- **要求：** + - 合入代码大于 200 行，需三人以上会议检视。 + - 检视密度≥2个/100行。 + - 检视缺陷密度达不到要求的需给出说明。 + - 大于 1000 行代码原则上不允许合入，需进行备案。 +- [ ] **是否经过代码检视** +- [ ] **是否具备UT测试用例看护** + +--- + +## 4. 安全自检 +- **典型安全编码问题 ** +- [ ] **若涉及对外接口，是否已校验外部数据** +- [ ] **MR 标题和描述是否按格式填写** +- [ ] **是否进行空指针校验** +- [ ] **是否进行返回值校验** +- [ ] **是否正确考虑文件权限配置** +- [ ] **是否充分考虑接口的异常场景** +- [ ] **是否正确记录错误日志** +- [ ] **若涉及正则表达式，是否对正则表达式做 ReDos 校验** +- [ ] **若涉及运算，是否存在整数溢出、除零等风险** + +--- + +## 5. 变更知会 +- **资料修改：** +- **变更通知（消息知会 + 邮件知会）：** + +--- + +## 6. 冒烟修改 +- **PR 来源：** + - [ ] 问题单 + - [ ] 需求特性 + - [ ] 其他 +- [ ] **是否存在冒烟可以拦截却未拦截的情况** +- [ ] **是否需要添加冒烟：** + +--- \ No newline at end of file -- Gitee From c0816e54d0ff326b63ecf035c5df82df52f5ecd4 Mon Sep 17 00:00:00 2001 From: lijiaojiao Date: Tue, 13 Aug 2024 21:44:30 +0800 Subject: [PATCH 281/791] =?UTF-8?q?=E3=80=90UT=E3=80=91=E8=A1=A5=E5=85=85o?= =?UTF-8?q?nline=5Fdispatch=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pytorch/online_dispatch/dispatch.py | 24 +-- .../pytorch/online_dispatch/dump_compare.py | 26 +-- .../test_compare_online_dispatch.py | 101 ++++++++++ .../online_dispatch/test_dump_compare.py | 183 ++++++++++++++++++ .../online_dispatch/test_single_compare.py | 52 +++++ .../test_utils_online_dispatch.py | 99 ++++++++++ 6 files changed, 460 insertions(+), 25 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_dump_compare.py create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_single_compare.py create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index 898df30b9..bf5cf1194 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -49,7 +49,7 @@ class PtdbgDispatch(TorchDispatchMode): self.single_api_index_dict = {} self.device_dump_path_cpu = None self.device_dump_path_npu = None - self.all_summery = [] + self.all_summary = [] self.call_stack_list = [] self.process_num = process_num self.filter_dump_api() @@ -90,12 +90,12 @@ class PtdbgDispatch(TorchDispatchMode): if self.process_num > 0: self.pool.close() self.pool.join() - summery_path = os.path.join(self.root_cpu_path, f'summary.json') - if not os.path.exists(summery_path): + summary_path = os.path.join(self.root_cpu_path, f'summary.json') + if not os.path.exists(summary_path): logger_error("Please check train log, An exception may have occurred!") return - check_file_or_directory_path(summery_path, False) - fp_handle = open(summery_path, "r") + check_file_or_directory_path(summary_path, False) + fp_handle = open(summary_path, "r") while True: json_line_data = fp_handle.readline() if json_line_data == '\n': @@ -103,7 +103,7 @@ class PtdbgDispatch(TorchDispatchMode): if len(json_line_data) == 0: break msg = json.loads(json_line_data) - self.all_summery[msg[0]] = msg[1] + self.all_summary[msg[0]] = msg[1] fp_handle.close() if self.debug_flag: @@ -111,9 +111,9 @@ class PtdbgDispatch(TorchDispatchMode): output_num = 0 total_num = 0 - for list_data in self.all_summery: + for list_data in self.all_summary: for data in list_data: - logger_debug(f'summery: Device[{self.device_id}], Pid[{os.getpid()}], Data[{data}]') + logger_debug(f'summary: Device[{self.device_id}], Pid[{os.getpid()}], Data[{data}]') if "_input" in data[CompareConst.NPU_NAME]: input_num = input_num + 1 if "_output" in data[CompareConst.NPU_NAME]: @@ -175,16 +175,16 @@ class PtdbgDispatch(TorchDispatchMode): cpu_out = cpu_out.float() if self.process_num == 0: - self.all_summery.append([]) - data_info = DisPatchDataInfo(cpu_args, cpu_kwargs, self.all_summery, func, npu_out_cpu, cpu_out, self.lock) + self.all_summary.append([]) + data_info = DisPatchDataInfo(cpu_args, cpu_kwargs, self.all_summary, func, npu_out_cpu, cpu_out, self.lock) dispatch_workflow(run_param, data_info) else: self.lock.acquire() - self.all_summery.append([]) + self.all_summary.append([]) self.lock.release() run_param.process_flag = True if self.check_fun(func, run_param): - data_info = DisPatchDataInfo(cpu_args, cpu_kwargs, self.all_summery, None, npu_out_cpu, cpu_out, + data_info = DisPatchDataInfo(cpu_args, cpu_kwargs, self.all_summary, None, npu_out_cpu, cpu_out, self.lock) self.pool.apply_async(func=dispatch_multiprocess, args=(run_param, data_info), error_callback=error_call) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py index f83b6fc9f..4d0453308 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py @@ -32,10 +32,10 @@ class DispatchRunParam: class DisPatchDataInfo: - def __init__(self, cpu_args, cpu_kwargs, all_summery, func, npu_out_cpu, cpu_out, lock): + def __init__(self, cpu_args, cpu_kwargs, all_summary, func, npu_out_cpu, cpu_out, lock): self.cpu_args = cpu_args self.cpu_kwargs = cpu_kwargs - self.all_summery = all_summery + self.all_summary = all_summary self.func = func self.npu_out_cpu = npu_out_cpu self.cpu_out = cpu_out @@ -87,24 +87,24 @@ def dump_data(data, prefix, dump_path): elif support_basic_type(data): if isinstance(data, torch.Tensor) and data.is_meta: return - # dump data may greater than summery_list collect + # dump data may greater than summary_list collect np_save_data(data, prefix, dump_path) -def save_temp_summery(api_index, single_api_summery, path, lock): - summery_path = os.path.join(path, f'summery.json') +def save_temp_summary(api_index, single_api_summary, path, lock): + summary_path = os.path.join(path, f'summary.json') lock.acquire() - with FileOpen(summery_path, "a") as f: - json.dump([api_index, single_api_summery], f) + with FileOpen(summary_path, "a") as f: + json.dump([api_index, single_api_summary], f) f.write('\n') lock.release() def dispatch_workflow(run_param: DispatchRunParam, data_info: DisPatchDataInfo): cpu_args, cpu_kwargs = data_info.cpu_args, data_info.cpu_kwargs - all_summery, func = data_info.all_summery, data_info.func + all_summary, func = data_info.all_summary, data_info.func npu_out_cpu, cpu_out, lock = data_info.npu_out_cpu, data_info.cpu_out, data_info.lock - single_api_summery = [] + single_api_summary = [] prefix_input = f'{run_param.aten_api}_{run_param.single_api_index}_input' prefix_output = f'{run_param.aten_api}_{run_param.single_api_index}_output' @@ -127,9 +127,9 @@ def dispatch_workflow(run_param: DispatchRunParam, data_info: DisPatchDataInfo): dump_data(npu_out_cpu, prefix_output, run_param.root_npu_path) if run_param.process_num == 0: - all_summery[run_param.api_index - 1] = copy.deepcopy(single_api_summery) + all_summary[run_param.api_index - 1] = copy.deepcopy(single_api_summary) else: - save_temp_summery(run_param.api_index - 1, single_api_summery, run_param.root_cpu_path, lock) + save_temp_summary(run_param.api_index - 1, single_api_summary, run_param.root_cpu_path, lock) def get_torch_func(run_param): @@ -156,10 +156,10 @@ def error_call(err): logger.error(f'multiprocess {err}') -def save_csv(all_summery, call_stack_list, csv_path): +def save_csv(all_summary, call_stack_list, csv_path): df = pd.DataFrame(columns=CSV_COLUMN_NAME) - for index, list_data in enumerate(all_summery): + for index, list_data in enumerate(all_summary): for data in list_data: csv_row_data = {CompareConst.NPU_NAME: data[CompareConst.NPU_NAME], CompareConst.BENCH_NAME: data[CompareConst.BENCH_NAME], diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py new file mode 100644 index 000000000..4db0981b2 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py @@ -0,0 +1,101 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import torch +from msprobe.pytorch.online_dispatch.compare import get_json_contents, Saver + +import json +import csv +import os +import logging +import threading +from pathlib import Path +import pandas as pd +from unittest.mock import Mock, patch +from msprobe.core.common.utils import CompareException + +class TestCompare(unittest.TestCase): + def setUp(self): + self.dict_json_path = "./dict.json" + self.list_json_path = "./list.json" + Path(self.dict_json_path).touch() + Path(self.list_json_path).touch() + + def tearDown(self): + if os.path.exists(self.dict_json_path): + os.remove(self.dict_json_path) + if os.path.exists(self.list_json_path): + os.remove(self.list_json_path) + + def test_get_json_contents_when_get_json(self): + data = {"one":1} + with open(self.dict_json_path,'w') as f: + json.dump(data, f) + self.assertEqual(get_json_contents(self.dict_json_path),data) + + @patch('msprobe.core.common.log.BaseLogger.error') + def test_get_json_contents_when_get_list(self,mock_error): + data = [1,2] + with open(self.list_json_path,'w') as f: + json.dump(data, f) + with self.assertRaises(CompareException) as context: + get_json_contents(self.list_json_path) + self.assertEqual(context.exception.code, CompareException.INVALID_FILE_ERROR) + mock_error.assert_called_once_with('Json file %s, content is not a dictionary!' % self.list_json_path) + +class TestSaver(unittest.TestCase): + def setUp(self): + self.save_path = "./saver_save.csv" + self.detail_save_path = "./saver_detail.csv" + self.saver = Saver(self.save_path,self.detail_save_path,False) + Path(self.save_path).touch() + Path(self.detail_save_path).touch() + + def tearDown(self): + if os.path.exists(self.save_path): + os.remove(self.save_path) + if os.path.exists(self.detail_save_path): + os.remove(self.detail_save_path) + + def test_write_csv_title(self): + self.saver.write_csv_title() + mock_data_save = {self.saver.COLUMN_API_NAME:{}, + self.saver.COLUMN_FORWARD_SUCCESS:{}, + self.saver.COLUMN_BACKWARD_SUCCESS:{}, + "Message":{}} + mock_data_detail = {'Npu Name': {}, 'Bench Dtype': {}, 'NPU Dtype': {}, 'Shape': {}, 'error_balance': {}, 'max_abs_diff': {}, 'max_abs_idx': {}, 'max_rel_diff': {}, 'max_rel_idx': {}, 'eb_thd': {}, 'error_thd': {}, 'Status': {}, 'Message': {}} + self.assertEqual(pd.read_csv(self.save_path).to_dict(), mock_data_save) + self.assertEqual(pd.read_csv(self.detail_save_path).to_dict(), mock_data_detail) + + def test_write_summary_csv(self): + mock_test_result = Mock() + mock_test_result.api_name = "api_name" + mock_test_result.is_fwd_success = "SKIP" + mock_test_result.is_bwd_success = "NOSKIP" + mock_test_result.fwd_compare_alg_results = "result" + self.saver.write_summary_csv(mock_test_result) + mock_data_save = {"api_name":{}, "SKIP":{}, "NOSKIP":{}, "result":{}} + self.assertTrue(pd.read_csv(self.save_path).to_dict()==mock_data_save) + + def test_write_detail_csv(self): + mock_test_result = Mock() + mock_test_result.api_name = "api_name" + mock_test_result.fwd_compare_alg_results = ["f"] + mock_test_result.bwd_compare_alg_results = ["b"] + self.saver.write_detail_csv(mock_test_result) + mock_data_detail = {'api_name.forward.output.0': {0: 'api_name.backward.output.0'}, 'f': {0: 'b'}} + + self.assertTrue(pd.read_csv(self.detail_save_path).to_dict()==mock_data_detail) + diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_dump_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_dump_compare.py new file mode 100644 index 000000000..1393884ce --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_dump_compare.py @@ -0,0 +1,183 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import torch +import json +import os +import threading +from pathlib import Path +from unittest.mock import Mock, patch +import pandas as pd + +from msprobe.core.common.const import CompareConst +from msprobe.pytorch.online_dispatch.dump_compare import support_basic_type, dump_data, save_temp_summary, dispatch_workflow, get_torch_func, dispatch_multiprocess, error_call, save_csv + + + +class TestDumpCompare(unittest.TestCase): + def setUp(self): + self.summary_path = "summary.json" + Path(self.summary_path).touch() + self.csv_path = "test_save_csv.csv" + Path(self.csv_path).touch() + self.data = {CompareConst.NPU_NAME: 1, + CompareConst.BENCH_NAME: 1, + CompareConst.NPU_DTYPE: 1, + CompareConst.BENCH_DTYPE: 11, + CompareConst.NPU_SHAPE: 1, + CompareConst.BENCH_SHAPE: 1, + CompareConst.NPU_MAX: 1, + CompareConst.NPU_MIN: 1, + CompareConst.NPU_MEAN: 1, + CompareConst.BENCH_MAX: 1, + CompareConst.BENCH_MIN: 1, + CompareConst.BENCH_MEAN: 1, + CompareConst.COSINE: 1, + CompareConst.MAX_ABS_ERR: 1, + CompareConst.MAX_RELATIVE_ERR: 1, + CompareConst.ACCURACY: 1, + CompareConst.ERROR_MESSAGE: 1} + self.data_gt = {CompareConst.NPU_NAME: 1, + CompareConst.BENCH_NAME: 1, + CompareConst.NPU_DTYPE: 1, + CompareConst.BENCH_DTYPE: 11, + CompareConst.NPU_SHAPE: 1, + CompareConst.BENCH_SHAPE: 1, + CompareConst.NPU_MAX: 1, + CompareConst.NPU_MIN: 1, + CompareConst.NPU_MEAN: 1, + CompareConst.BENCH_MAX: 1, + CompareConst.BENCH_MIN: 1, + CompareConst.BENCH_MEAN: 1, + CompareConst.COSINE: 1, + CompareConst.MAX_ABS_ERR: 1, + CompareConst.MAX_RELATIVE_ERR: 1, + CompareConst.ACCURACY: 1, + CompareConst.STACK: 2, + CompareConst.ERROR_MESSAGE: 1} + + def tearDown(self): + if os.path.exists(self.summary_path): + os.remove(self.summary_path) + if os.path.exists(self.csv_path): + os.remove(self.csv_path) + + def test_support_basic_type_should_return_true_when_is_instance(self): + self.assertTrue(support_basic_type(2.3)) + + def test_support_basic_type_should_return_false_when_isnot_instance(self): + self.assertFalse(support_basic_type("abcde")) + + def test_save_temp_summary(self): + api_index='1' + single_api_summary="conv2d" + path = '' + data = [] + lock=threading.Lock() + + save_temp_summary(api_index=api_index,single_api_summary=single_api_summary,path=path,lock=lock) + + with open(self.summary_path, 'r') as f: + content = f.readlines() + for line in content: + data.append(json.loads(line)) + self.assertEqual([['1','conv2d']],data) + + @patch('msprobe.pytorch.online_dispatch.dump_compare.dump_data') + @patch('msprobe.pytorch.online_dispatch.dump_compare.save_temp_summary') + def test_dispatch_workflow_should_dump_when_flag_is_True(self,mock_save_temp_summary,mock_dump_data): + mock_run_param = Mock() + mock_run_param.aten_api="aten_api" + mock_run_param.single_api_index="single_api_index" + mock_run_param.root_npu_path="" + mock_data_info = Mock() + mock_data_info.cpu_args=None + mock_data_info.cpu_kwargs=[] + + mock_run_param.dump_flag=True + mock_run_param.process_num = 0 + mock_run_param.api_index = 1 + mock_data_info.all_summary=[1] + + dispatch_workflow(mock_run_param, mock_data_info) + mock_dump_data.assert_called() + mock_save_temp_summary.assert_not_called() + + @patch('msprobe.pytorch.online_dispatch.dump_compare.dump_data') + @patch('msprobe.pytorch.online_dispatch.dump_compare.save_temp_summary') + def test_dispatch_workflow_should_not_dump_when_flag_is_false(self,mock_save_temp_summary,mock_dump_data): + mock_run_param = Mock() + mock_run_param.aten_api="aten_api" + mock_run_param.single_api_index="single_api_index" + mock_run_param.root_npu_path="" + mock_data_info = Mock() + mock_data_info.cpu_args=None + mock_data_info.cpu_kwargs=[] + + mock_run_param.dump_flag=False + mock_run_param.auto_dump_flag=False + mock_run_param.process_num = 1 + mock_run_param.api_index = 1 + mock_data_info.all_summary=[1] + + dispatch_workflow(mock_run_param, mock_data_info) + mock_dump_data.assert_not_called() + mock_save_temp_summary.assert_called() + + def test_get_torch_func_should_return_None_when_outside_input(self): + mock_run_param = Mock() + mock_run_param.func_namespace="new_attr1" + mock_run_param.aten_api="new_attr2" + mock_run_param.aten_api_overload_name="new_attr3" + self.assertIsNone(get_torch_func(mock_run_param)) + + def test_get_torch_func_should_return_None_when_inside_input(self): + mock_run_param = Mock() + mock_run_param.func_namespace="aten" + mock_run_param.aten_api="add" + mock_run_param.aten_api_overload_name="Scalar" + self.assertEqual(get_torch_func(mock_run_param),torch.ops.aten.add.Scalar) + + @patch('msprobe.core.common.log.BaseLogger.error') + def test_dispatch_multiprocess_should_logger_error_when_wrong_api_input(self,mock_error): + mock_run_param = Mock() + mock_run_param.func_namespace="new_attr1" + mock_run_param.aten_api="new_attr2" + mock_run_param.aten_api_overload_name="new_attr3" + mock_dispatch_data_info=Mock() + dispatch_multiprocess(mock_run_param,mock_dispatch_data_info) + mock_error.assert_called_once_with(f'can not find suitable call api:{mock_run_param.aten_api}') + + @patch('msprobe.pytorch.online_dispatch.dump_compare.dispatch_workflow') + def test_dispatch_multiprocess_should_workflow_when_right_api_input(self,mock_workflow): + mock_run_param = Mock() + mock_run_param.func_namespace="aten" + mock_run_param.aten_api="add" + mock_run_param.aten_api_overload_name="Scalar" + mock_dispatch_data_info=Mock() + mock_workflow.return_value=1 + dispatch_multiprocess(mock_run_param,mock_dispatch_data_info) + mock_workflow.assert_called_once_with(mock_run_param,mock_dispatch_data_info) + + @patch('msprobe.core.common.log.BaseLogger.error') + def test_error_call(self,mock_error): + error_call("messages") + mock_error.assert_called_once_with("multiprocess messages") + + def test_save_csv(self): + save_csv([[self.data]],[2],self.csv_path) + df = pd.read_csv(self.csv_path) + df_gt = pd.DataFrame.from_dict(self.data_gt, orient='index').T + self.assertTrue((df.all()==df_gt.all()).all()) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_single_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_single_compare.py new file mode 100644 index 000000000..9aaa14af6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_single_compare.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import torch +import logging +from unittest.mock import Mock, patch +from msprobe.pytorch.online_dispatch.single_compare import SingleBenchmarkCompareStandard,SingleBenchmarkAccuracyResult, SingleBenchmarkAccuracyCompare + +class TestSingleBenchmarkCompareStandard(unittest.TestCase): + def setUp(self): + self.single_benchmark_compare_standard = SingleBenchmarkCompareStandard() + @patch('logging.warning') + def test_get_error_thd_when_input_f64(self,mock_warning): + self.single_benchmark_compare_standard.get_error_thd(torch.float64) + mock_warning.assert_called_once_with("the output data of fp64 uses the same standard as fp32.") + + @patch('logging.error') + def test_get_error_thd_when_input_bool(self,mock_error): + self.single_benchmark_compare_standard.get_error_thd(torch.bool) + mock_error.assert_called_once_with("Single benchmark compare only supports floating point " + "in fp16, bf16, fp32. ") + + def test_get_eb_thd_when_input_f16(self): + self.assertEqual(self.single_benchmark_compare_standard.get_eb_thd(torch.float16),2 ** -10) + + def test_get_eb_thd_when_input_bool(self): + self.assertIsNone(self.single_benchmark_compare_standard.get_eb_thd(torch.bool)) + + +class TestSingleBenchmarkAccuracyResult(unittest.TestCase): + def setUp(self): + self.single_benchmark_accuracy_result = SingleBenchmarkAccuracyResult(True,1,1,1,1,1) + + def test_get_result_result_false(self): + self.single_benchmark_accuracy_result.get_result(0.5,0.5) + self.assertEqual(self.single_benchmark_accuracy_result.result,False) + + def test_get_result_result_true(self): + self.single_benchmark_accuracy_result.get_result(2,2) + self.assertEqual(self.single_benchmark_accuracy_result.result,True) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py new file mode 100644 index 000000000..90a646956 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py @@ -0,0 +1,99 @@ +# Copyright (c) 2024-2024 Huawei Technologies Co., Ltd. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest +import inspect +import numpy as np +import os +import sys +import torch +import psutil +import logging +from pathlib import Path +from unittest.mock import patch, MagicMock + +from msprobe.pytorch.online_dispatch.utils import COLOR_RED, COLOR_CYAN, COLOR_YELLOW, COLOR_RESET, COMPARE_LOGO, get_callstack, np_save_data, data_to_cpu, logger_debug, logger_info, logger_warn, logger_error, logger_user, logger_logo, DispatchException + +cpu_device = torch._C.device("cpu") + +class FakeData: + def init(self): + self.numpy=np.random.rand(5,5) + +class FakeDataNoNumpy: + def init(self): + self.data=np.random.rand(5,5) + +class TestUtils(unittest.TestCase): + def setUp(self): + self.stack=inspect.stack() + self.data_path="" + self.file_name="data" + self.data=FakeData() + self.data_nonumpy=FakeDataNoNumpy() + self.dispatch_exception=DispatchException(err_code=1, err_msg="messages") + Path(os.path.join(self.data_path, f'{self.file_name}.npy')).touch() + def tearDown(self): + if os.path.exists(os.path.join(self.data_path, f'{self.file_name}.npy')): + os.remove(os.path.join(self.data_path, f'{self.file_name}.npy')) + + @patch('msprobe.core.common.file_check.change_mode') + def test_np_save_data_should_error_when_input_wrong(self,mock_change_mode): + np_save_data(self.data_nonumpy,self.file_name,self.data_path) + mock_change_mode.assert_not_called() + + def test_data_to_cpu_should_return_tensor_copy_when_input_tensor(self): + data = torch.tensor([1,2],device=cpu_device,dtype=torch.float16) + deep=1 + data_cpu=[] + self.assertEqual(data_to_cpu(data,deep,data_cpu).all(),data.clone().detach().float().all()) + + def test_data_to_cpu_should_return_list_when_input_list(self): + data=[1,2] + deep=0 + data_cpu=[] + self.assertEqual(data_to_cpu(data,deep,data_cpu), data) + + @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') + def test_logger_debug(self,mock_inf0): + logger_debug("messages") + mock_inf0.return_value.assert_called_once_with("DEBUG messages") + + @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') + def test_logger_info(self,mock_info): + logger_info("messages") + mock_info.return_value.assert_called_once_with("INFO messages") + + @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') + def test_logger_warn(self,mock_info): + logger_warn("messages") + mock_info.return_value.assert_called_once_with(f'{COLOR_YELLOW}WARNING messages {COLOR_RESET}') + + @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') + def test_logger_error(self,mock_info): + logger_error("messages") + mock_info.return_value.assert_called_once_with(f'{COLOR_RED}ERROR messages {COLOR_RESET}') + + @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') + def test_logger_user(self,mock_info): + logger_user("messages") + mock_info.return_value.assert_called_once_with("messages") + + @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') + def test_logger_logo(self,mock_info): + logger_logo() + mock_info.return_value.assert_called_once_with(f'{COLOR_CYAN}{COMPARE_LOGO} {COLOR_RESET}') + + def test_str(self): + self.assertEqual(self.dispatch_exception.__str__(),"messages") \ No newline at end of file -- Gitee From 7edeed617fe0d1a53570c8552fc5c7c5386c82b3 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Tue, 13 Aug 2024 11:07:12 +0800 Subject: [PATCH 282/791] o dump ut --- .../test_base.py | 7 +- .../test_mindspore_processor.py | 111 ++++++- .../test_pytorch_processor.py | 308 +++++------------- 3 files changed, 183 insertions(+), 243 deletions(-) rename debug/accuracy_tools/msprobe/test/core_ut/data_dump/{test_data_processor => data_processor}/test_base.py (99%) rename debug/accuracy_tools/msprobe/test/core_ut/data_dump/{test_data_processor => data_processor}/test_pytorch_processor.py (45%) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_base.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py similarity index 99% rename from debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_base.py rename to debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py index 3e343528c..886d4b124 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_base.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py @@ -58,7 +58,7 @@ class TestBaseDataProcessor(unittest.TestCase): self.config = MagicMock() self.data_writer = MagicMock() self.processor = BaseDataProcessor(self.config, self.data_writer) - + self.data_writer.dump_tensor_data_dir = "./dump_data" self.processor.current_api_or_module_name = "test_api" self.processor.api_data_category = "input" @@ -109,7 +109,6 @@ class TestBaseDataProcessor(unittest.TestCase): def test_recursive_apply_transform_with_warning(self, mock_logger): transform = lambda x, _: x * 2 BaseDataProcessor.recursive_apply_transform({1, 2, 3}, transform) - print(mock_logger.call_args_list) mock_logger.assert_called_with(f"Data type {type({1, 2, 3})} is not supported.") def test_if_return_forward_new_output(self): @@ -204,7 +203,3 @@ class TestBaseDataProcessor(unittest.TestCase): expected_file_name = "test_api.input.suffix.pt" expected_file_path = os.path.join(self.data_writer.dump_tensor_data_dir, expected_file_name) self.assertEqual(result, (expected_file_name, expected_file_path)) - - -if __name__ == '__main__': - unittest.main() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py index 046388741..1bc2d125c 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -15,16 +15,121 @@ # limitations under the License. """ import unittest -from unittest.mock import patch +from unittest.mock import patch, MagicMock +import zlib +import mindspore as ms from mindspore import Tensor import numpy as np -from msprobe.core.data_dump.data_processor.base import BaseDataProcessor -from msprobe.core.data_dump.data_processor.mindspore_processor import MindsporeDataProcessor, OverflowCheckDataProcessor +from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, TensorStatInfo +from msprobe.core.data_dump.data_processor.mindspore_processor import ( + MindsporeDataProcessor, + TensorDataProcessor, + OverflowCheckDataProcessor +) from msprobe.core.common.const import FileCheckConst +class TestMindsporeDataProcessor(unittest.TestCase): + def setUp(self): + self.config = MagicMock() + self.data_writer = MagicMock() + self.processor = MindsporeDataProcessor(self.config, self.data_writer) + self.tensor = ms.Tensor(np.array([1.0, 2.0, 3.0]).astype(np.float32)) + + def test_get_md5_for_tensor(self): + tensor = ms.Tensor([1.0, 2.0, 3.0], dtype=ms.bfloat16) + expected_crc32 = zlib.crc32(np.array([1.0, 2.0, 3.0], dtype=np.float32).tobytes()) + expected_md5 = f"{expected_crc32:08x}" + result = self.processor.get_md5_for_tensor(tensor) + self.assertEqual(result, expected_md5) + + def test_analyze_builtin(self): + test_slice = slice(1, 3, None) + expected_result = {"type": "slice", "value": [1, 3, None]} + result = self.processor._analyze_builtin(test_slice) + self.assertEqual(result, expected_result) + + test_int = 42 + expected_result = {"type": "int", "value": 42} + result = self.processor._analyze_builtin(test_int) + self.assertEqual(result, expected_result) + + def test_get_stat_info_float(self): + tensor = ms.Tensor([1.0, 2.0, 3.0]) + result = self.processor.get_stat_info(tensor) + self.assertEqual(result.max, 3.0) + self.assertEqual(result.min, 1.0) + self.assertEqual(result.mean, 2.0) + self.assertEqual(result.norm, ms.ops.norm(tensor).item()) + + def test_get_stat_info_int(self): + tensor = ms.Tensor([1, 2, 3], dtype=ms.int32) + result = self.processor.get_stat_info(tensor) + self.assertEqual(result.max, 3) + self.assertEqual(result.min, 1) + self.assertEqual(result.mean, 2) + self.assertEqual(result.norm, ms.ops.norm(tensor).item()) + + def test_get_stat_info_bool(self): + tensor = ms.Tensor([True, False, True]) + result = self.processor.get_stat_info(tensor) + self.assertEqual(result.max, True) + self.assertEqual(result.min, False) + self.assertIsNone(result.mean) + self.assertIsNone(result.norm) + + @patch.object(MindsporeDataProcessor, 'get_md5_for_tensor') + def test__analyze_tensor(self, get_md5_for_tensor): + get_md5_for_tensor.return_value = "test_md5" + tensor = ms.Tensor(np.array([1, 2, 3], dtype=np.int32)) + self.config.summary_mode = 'md5' + suffix = "test_tensor" + expected_result = { + 'type': 'mindspore.Tensor', + 'dtype': 'Int32', + 'shape': (3,), + 'Max': 3, + 'Min': 1, + 'Mean': 2, + 'Norm': ms.ops.norm(tensor).item(), + 'md5': 'test_md5', + } + result = self.processor._analyze_tensor(tensor, suffix) + self.assertEqual(result, expected_result) + + +class TestTensorDataProcessor(unittest.TestCase): + + def setUp(self): + self.config = MagicMock() + self.data_writer = MagicMock() + self.processor = TensorDataProcessor(self.config, self.data_writer) + self.data_writer.dump_tensor_data_dir = "./dump_data" + self.processor.current_api_or_module_name = "test_api" + self.processor.api_data_category = "input" + + @patch('numpy.save') + def test_analyze_tensor(self, mock_save): + self.config.framework = "mindspore" + tensor = ms.Tensor([1.0, 2.0, 3.0]) + suffix = 'suffix' + result = self.processor._analyze_tensor(tensor, suffix) + mock_save.assert_called_once() + expected = { + 'type': 'mindspore.Tensor', + 'dtype': str(tensor.dtype), + 'shape': tensor.shape, + 'Max': 3.0, + 'Min': 1.0, + 'Mean': 2.0, + 'Norm': ms.ops.norm(tensor).item(), + 'data_name': 'test_api.input.suffix.npy' + } + self.assertEqual(expected, result) + + class TestOverflowCheckDataProcessor(unittest.TestCase): def setUp(self): class Config: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py similarity index 45% rename from debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_pytorch_processor.py rename to debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index 94d126a4b..89c1d9e84 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -3,32 +3,18 @@ import unittest from unittest.mock import patch, MagicMock, Mock import zlib -# mock_torch_npu = MagicMock() -# modules = { -# 'torch_npu': mock_torch_npu, -# 'torch_npu.npu': mock_torch_npu.npu, -# } -# patcher = patch.dict('sys.modules', modules) -# patcher.start() - import torch +import numpy as np + from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutputs, ModuleForwardInputsOutputs, BaseDataProcessor from msprobe.core.data_dump.data_processor.pytorch_processor import ( PytorchDataProcessor, FreeBenchmarkDataProcessor, TensorDataProcessor, - KernelDumpDataProcessor, OverflowCheckDataProcessor ) -from msprobe.core.common.exceptions import MsprobeException -from msprobe.core.common.file_check import change_mode, path_len_exceeds_limit -from msprobe.core.common.const import Const, OverflowConst, FileCheckConst -sys.modules['torch_npu'] = Mock() -sys.modules['torch_npu.npu'] = Mock() -sys.modules['torch_npu._C'] = Mock() -torch_npu = sys.modules['torch_npu'] - +from msprobe.core.common.const import Const, OverflowConst class TestPytorchDataProcessor(unittest.TestCase): @@ -40,11 +26,11 @@ class TestPytorchDataProcessor(unittest.TestCase): def test_get_md5_for_tensor(self): tensor = torch.tensor([1, 2, 3]) - expected_md5 = zlib.crc32(tensor.numpy().tobytes()) - self.assertEqual(self.processor.get_md5_for_tensor(tensor), f"{expected_md5:08x}") + expected_hash = zlib.crc32(tensor.numpy().tobytes()) + self.assertEqual(self.processor.get_md5_for_tensor(tensor), f"{expected_hash:08x}") def test_analyze_device_in_kwargs(self): - device = torch.device('npu:0') + device = torch.device('cuda:0') result = self.processor.analyze_device_in_kwargs(device) expected = {'type': 'torch.device', 'value': 'cuda:0'} self.assertEqual(result, expected) @@ -89,46 +75,46 @@ class TestPytorchDataProcessor(unittest.TestCase): def test_handle_tensor_extremum_nan_inf_all_nan(self): tensor = torch.tensor([float('nan'), float('nan')]) - result = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') - self.assertTrue(torch.isnan(result)) + result = self.processor.handle_tensor_extremum_nan_inf(tensor, 'max') + self.assertTrue(np.isnan(result)) def test_handle_tensor_extremum_nan_inf_all_inf(self): tensor = torch.tensor([float('inf'), float('inf')]) - result = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') - self.assertTrue(torch.isinf(result)) + result = self.processor.handle_tensor_extremum_nan_inf(tensor, 'max') + self.assertTrue(np.isinf(result)) def test_handle_tensor_extremum_nan_inf_all_negative_inf(self): tensor = torch.tensor([float('-inf'), float('-inf')]) - result = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') - self.assertTrue(torch.isinf(result) and result < 0) + result = self.processor.handle_tensor_extremum_nan_inf(tensor, 'min') + self.assertTrue(np.isinf(result) and result < 0) def test_handle_tensor_extremum_nan_inf_mixed(self): tensor = torch.tensor([1.0, float('nan'), 3.0, float('-inf'), 2.0]) - result_max = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') - result_min = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') + result_max = self.processor.handle_tensor_extremum_nan_inf(tensor, 'max') + result_min = self.processor.handle_tensor_extremum_nan_inf(tensor, 'min') self.assertEqual(result_max, 3.0) self.assertEqual(result_min, 1.0) def test_handle_tensor_extremum_nan_inf_mixed_with_inf(self): tensor = torch.tensor([1.0, float('nan'), 3.0, float('inf'), 2.0]) - result_max = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') - result_min = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') + result_max = self.processor.handle_tensor_extremum_nan_inf(tensor, 'max') + result_min = self.processor.handle_tensor_extremum_nan_inf(tensor, 'min') self.assertEqual(result_max, 3.0) self.assertEqual(result_min, 1.0) def test_handle_tensor_extremum_nan_inf_no_inf_nan(self): tensor = torch.tensor([1.0, 2.0, 3.0]) - result_max = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') - result_min = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') + result_max = self.processor.handle_tensor_extremum_nan_inf(tensor, 'max') + result_min = self.processor.handle_tensor_extremum_nan_inf(tensor, 'min') self.assertEqual(result_max, 3.0) self.assertEqual(result_min, 1.0) def test_handle_tensor_extremum_nan_inf_all_inf_nan(self): tensor = torch.tensor([float('nan'), float('inf'), float('-inf')]) - result_max = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'max') - result_min = PytorchDataProcessor.handle_tensor_extremum_nan_inf(tensor, 'min') - self.assertTrue(torch.isnan(result_max)) - self.assertTrue(torch.isnan(result_min)) + result_max = self.processor.handle_tensor_extremum_nan_inf(tensor, 'max') + result_min = self.processor.handle_tensor_extremum_nan_inf(tensor, 'min') + self.assertTrue(np.isinf(result_max)) + self.assertTrue(np.isinf(result_min)) def test_analyze_builtin(self): result = self.processor._analyze_builtin(slice(1, 10, 2)) @@ -185,17 +171,29 @@ class TestTensorDataProcessor(unittest.TestCase): self.config = MagicMock() self.data_writer = MagicMock() self.processor = TensorDataProcessor(self.config, self.data_writer) + self.data_writer.dump_tensor_data_dir = "./dump_data" + self.processor.current_api_or_module_name = "test_api" + self.processor.api_data_category = "input" @patch('torch.save') - @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) - @patch('msprobe.core.common.file_check.change_mode') - def test_analyze_tensor(self, mock_change_mode, mock_save): + def test_analyze_tensor(self, mock_save): + self.config.framework = "pytorch" tensor = torch.tensor([1.0, 2.0, 3.0]) suffix = 'suffix' result = self.processor._analyze_tensor(tensor, suffix) mock_save.assert_called_once() - mock_change_mode.assert_called_once() - self.assertIn('data_name', result) + expected = { + 'type': 'torch.Tensor', + 'dtype': 'torch.float32', + 'shape': tensor.shape, + 'Max': 3.0, + 'Min': 1.0, + 'Mean': 2.0, + 'Norm': torch.norm(tensor).item(), + 'requires_grad': False, + 'data_name': 'test_api.input.suffix.pt' + } + self.assertEqual(expected, result) class TestOverflowCheckDataProcessor(unittest.TestCase): @@ -212,38 +210,14 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): sys.modules['torch_npu._C'] = Mock() @patch('torch.save') - @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) - @patch('msprobe.core.common.file_check.change_mode') - def test_maybe_save_overflow_data_and_check_overflow_times(self, mock_change_mode, mock_path_len_exceeds_limit, mock_save): + def test_maybe_save_overflow_data_and_check_overflow_times(self, mock_save): self.processor.has_overflow = True + self.processor.real_overflow_nums = 0 self.processor.cached_tensors_and_file_paths = {'dummy_path': torch.tensor([1.0, 2.0, 3.0])} - - # First call should save the tensor and not raise an exception self.processor.maybe_save_overflow_data_and_check_overflow_times() mock_save.assert_called_once() - mock_path_len_exceeds_limit.assert_called_once() - mock_change_mode.assert_called_once() - - # Second call should raise an exception due to overflow nums limit - with self.assertRaises(MsprobeException) as context: - self.processor.maybe_save_overflow_data_and_check_overflow_times() - - self.assertEqual(str(context.exception), MsprobeException.OVERFLOW_NUMS_ERROR) - - def test_inc_and_check_overflow_times(self): - self.processor.real_overflow_dump_times = 0 - self.processor.overflow_nums = 1 - self.processor.has_overflow = True - - # First increment should not raise an exception - self.processor.inc_and_check_overflow_times() - self.assertEqual(self.processor.real_overflow_dump_times, 1) - - # Second increment should raise an exception - with self.assertRaises(MsprobeException) as context: - self.processor.inc_and_check_overflow_times() - - self.assertEqual(str(context.exception), MsprobeException.OVERFLOW_NUMS_ERROR) + self.processor.maybe_save_overflow_data_and_check_overflow_times() + self.assertEqual(self.processor.real_overflow_nums, 2) @patch('os.getenv', return_value=Const.ENV_ENABLE) def test_overflow_debug_mode_enable(self, mock_getenv): @@ -251,50 +225,50 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.assertTrue(result) mock_getenv.assert_called_once_with(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) - @patch('numpy.isinf', return_value=True) - @patch('numpy.isnan', return_value=False) - def test_analyze_maybe_overflow_tensor(self, mock_isnan, mock_isinf): + @patch('msprobe.core.data_dump.data_processor.pytorch_processor.is_gpu', return_value=True) + def test_analyze_maybe_overflow_tensor(self, _): tensor_json = {'Max': float('inf'), 'Min': 1.0} self.processor._analyze_maybe_overflow_tensor(tensor_json) self.assertTrue(self.processor.has_overflow) @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) @patch.object(BaseDataProcessor, 'get_save_file_path', return_value=['test_api_name', 'test_api_name.0.forward.input.pt']) - def test_analyze_tensor(self, mock_path_len_exceeds_limit, mock_get_save_file_path): + def test_analyze_tensor(self, mock_path_len_exceeds_limit, _): tensor = torch.tensor([1.0, 2.0, 3.0]) suffix = 'suffix' + expected = {'Max': 3.0, 'Min': 1.0, 'data_name': 'test_api_name'} with patch.object(PytorchDataProcessor, '_analyze_tensor', return_value={'Max': 3.0, 'Min': 1.0}) as mock_super_analyze_tensor: result = self.processor._analyze_tensor(tensor, suffix) mock_super_analyze_tensor.assert_called_once_with(tensor, suffix) - self.assertIn('data_name', result) + mock_path_len_exceeds_limit.assert_called_once() + self.assertEqual(expected, result) + + def test_analyze_backward(self): + def func(_): + self.processor.has_overflow = True + with patch.object(BaseDataProcessor, "analyze_backward", return_value={"min", 0}): + with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times"): + api_info = self.processor.analyze_backward("name", "module", "module_input_output") + self.assertFalse(self.processor.has_overflow) + self.assertIsNone(api_info) + with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times", new=func): + api_info = self.processor.analyze_backward("name", "module", "module_input_output") self.assertTrue(self.processor.has_overflow) + self.assertEqual(api_info, {"min", 0}) + + def test_analyze_forward(self): + def func(_): + self.processor.has_overflow = True + with patch.object(BaseDataProcessor, "analyze_forward", return_value={"min", 0}): + with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times"): + api_info = self.processor.analyze_forward("name", "module", "module_input_output") + self.assertFalse(self.processor.has_overflow) + self.assertIsNone(api_info) + with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times", new=func): + api_info = self.processor.analyze_forward("name", "module", "module_input_output") + self.assertTrue(self.processor.has_overflow) + self.assertEqual(api_info, {"min", 0}) - @patch.object(PytorchDataProcessor, 'analyze_element', return_value=['mocked_result']) - def test_analyze_backward(self, mock_analyze_element): - module_io = ModuleBackwardInputsOutputs(grad_output=(1, 2), grad_input=(3, 4)) - self.config.data_mode = ["all"] - result = self.processor.analyze_backward("test_backward", None, module_io) - expected = { - "test_backward": { - "grad_input": ['mocked_result'], - "grad_output": ['mocked_result'] - } - } - self.assertEqual(result, expected) - - @patch.object(PytorchDataProcessor, 'analyze_element', return_value=['mocked_result']) - def test_analyze_forward(self, mock_analyze_element): - module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=(4, 5)) - self.config.data_mode = ["all"] - result = self.processor.analyze_forward("test_forward", None, module_io) - expected = { - "test_forward": { - "input_args": ['mocked_result'], - "input_kwargs": ['mocked_result'], - "output": ['mocked_result'] - } - } - self.assertEqual(result, expected) class TestFreeBenchmarkDataProcessor(unittest.TestCase): @@ -320,137 +294,3 @@ class TestFreeBenchmarkDataProcessor(unittest.TestCase): module_io = ModuleBackwardInputsOutputs(grad_output=(torch.tensor([1.0, 2.0]),), grad_input=None) self.processor.analyze_backward('test_backward', None, module_io) mock_backward.assert_called_once() - - -# class TestKernelDumpDataProcessor(unittest.TestCase): - -# def setUp(self): -# self.config = MagicMock() -# self.config.is_forward_acl_dump = True -# self.config.acl_config = "dummy_acl_config" -# self.config.backward_input = {'test_module': 'dummy_path'} -# self.data_writer = MagicMock() -# self.processor = KernelDumpDataProcessor(self.config, self.data_writer) - -# @patch('torch_npu.npu.synchronize') -# @patch('torch_npu.npu.init_dump') -# @patch('torch_npu.npu.set_dump') -# @patch('torch_npu.npu.finalize_dump') -# def test_forward_acl_dump(self, mock_finalize_dump, mock_set_dump, mock_init_dump, mock_synchronize): -# module = MagicMock() -# module.forward = MagicMock(return_value=torch.tensor([1.0, 2.0, 3.0])) -# module_io = MagicMock() -# module_io.args = (1, 2) -# module_io.kwargs = {'a': 3} - -# KernelDumpDataProcessor.forward_init_status = False - -# self.processor.forward_acl_dump('test_module', module, module_io) - -# mock_synchronize.assert_called() -# mock_init_dump.assert_called_once_with() -# mock_set_dump.assert_called_once_with("dummy_acl_config") -# mock_finalize_dump.assert_called_once_with() -# module.forward.assert_called_with(1, 2, a=3) - -# @patch('torch_npu.npu.synchronize') -# @patch('torch_npu.npu.init_dump') -# @patch('torch_npu.npu.set_dump') -# @patch('torch_npu.npu.finalize_dump') -# @patch('torch.load', return_value=torch.tensor([1.0, 2.0, 3.0])) -# def test_dump_mode_backward_acl_dump(self, mock_load, mock_finalize_dump, mock_set_dump, mock_init_dump, mock_synchronize): -# module = MagicMock() -# module.forward = MagicMock(return_value=torch.tensor([1.0, 2.0, 3.0])) -# module_io = MagicMock() -# module_io.args = (1, 2) -# module_io.kwargs = {'a': 3} - -# KernelDumpDataProcessor.forward_init_status = False - -# self.processor.dump_mode_backward_acl_dump('test_module', module, module_io) - -# mock_synchronize.assert_called() -# mock_init_dump.assert_called_once_with() -# mock_set_dump.assert_called_once_with("dummy_acl_config") -# mock_finalize_dump.assert_called_once_with() -# mock_load.assert_called_once_with('dummy_path') -# module.forward.assert_called_with(1, 2, a=3) - -# def test_op_need_trigger(self): -# self.assertTrue(self.processor.op_need_trigger('Tensor.__getitem__.')) -# self.assertFalse(self.processor.op_need_trigger('SomeOtherOp')) - -# @patch.object(KernelDumpDataProcessor, 'forward_acl_dump') -# @patch.object(KernelDumpDataProcessor, 'dump_mode_backward_acl_dump') -# def test_analyze_forward(self, mock_dump_mode_backward_acl_dump, mock_forward_acl_dump): -# self.processor.analyze_forward('test_module', MagicMock(), MagicMock()) -# mock_forward_acl_dump.assert_called_once() -# mock_dump_mode_backward_acl_dump.assert_not_called() - -# self.config.is_forward_acl_dump = False -# self.processor.analyze_forward('test_module', MagicMock(), MagicMock()) -# mock_dump_mode_backward_acl_dump.assert_called_once() -# mock_forward_acl_dump.assert_called_once() # 因为已经被调用过一次 - -# @patch('torch.Tensor.backward') -# def test_acl_backward_dump_status(self, mock_backward): -# output = torch.tensor([1.0, 2.0, 3.0]) -# grad = torch.tensor([0.1, 0.1, 0.1]) -# self.assertTrue(self.processor.acl_backward_dump_status(output, grad, 'test_module')) -# mock_backward.assert_called_once_with(grad, retain_graph=True) - -# output = [torch.tensor([1.0, 2.0, 3.0])] -# self.assertTrue(self.processor.acl_backward_dump_status(output, grad, 'test_module')) -# mock_backward.assert_called_with(grad, retain_graph=True) - -# output = [torch.tensor([1.0, 2.0, 3.0])] -# self.assertFalse(self.processor.acl_backward_dump_status(output, grad, 'unknown_module')) - -# def tearDown(self): -# KernelDumpDataProcessor.forward_init_status = False - -# patcher.stop() -# class TestKernelDumpDataProcessor(unittest.TestCase): - -# def setUp(self): -# self.config = MagicMock() -# self.data_writer = MagicMock() -# self.processor = KernelDumpDataProcessor(self.config, self.data_writer) - -# @patch('torch_npu.npu.synchronize') -# @patch('torch_npu.npu.init_dump') -# @patch('torch_npu.npu.set_dump') -# @patch('torch_npu.npu.finalize_dump') -# def test_forward_acl_dump(self, mock_finalize_dump, mock_set_dump, mock_init_dump, mock_synchronize): -# module = MagicMock() -# module.forward = MagicMock(return_value=torch.tensor([1.0, 2.0, 3.0])) -# module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=None) -# self.processor.forward_acl_dump('test_module', module, module_io) -# mock_synchronize.assert_called() -# mock_init_dump.assert_called_once() -# mock_set_dump.assert_called_once() -# mock_finalize_dump.assert_called_once() - -# @patch('torch_npu.npu.synchronize') -# @patch('torch_npu.npu.init_dump') -# @patch('torch_npu.npu.set_dump') -# @patch('torch_npu.npu.finalize_dump') -# @patch('torch.load', return_value=torch.tensor([1.0, 2.0, 3.0])) -# def test_dump_mode_backward_acl_dump(self, mock_load, mock_finalize_dump, mock_set_dump, mock_init_dump, mock_synchronize): -# module = MagicMock() -# module.forward = MagicMock(return_value=torch.tensor([1.0, 2.0, 3.0])) -# module_io = ModuleForwardInputsOutputs(args=(1, 2), kwargs={'a': 3}, output=None) -# self.config.backward_input = {'test_module': 'dummy_path'} -# self.processor.dump_mode_backward_acl_dump('test_module', module, module_io) -# mock_synchronize.assert_called() -# mock_init_dump.assert_called_once() -# mock_set_dump.assert_called_once() -# mock_finalize_dump.assert_called_once() -# mock_load.assert_called_once() - -# def test_op_need_trigger(self): -# self.assertTrue(self.processor.op_need_trigger('Tensor.__getitem__.')) -# self.assertFalse(self.processor.op_need_trigger('SomeOtherOp')) - -if __name__ == '__main__': - unittest.main() -- Gitee From 12c570a823ad8631178906d437782963eb93a2e2 Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Wed, 14 Aug 2024 14:35:24 +0800 Subject: [PATCH 283/791] bug_fix_for_fa_mm_conv_recoginze --- .../profiling_parser/base_profiling_parser.py | 5 ++--- .../profiling_parser/gpu_profiling_parser.py | 9 ++++++--- .../profiling_parser/npu_profiling_parser.py | 14 ++++++++------ 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 6afc52ff9..b56b30936 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -136,14 +136,13 @@ class BaseProfilingParser(ABC): self._check_result_data() return self._result_data - def categorize_computing_performance_data(self, tk: (TraceEventBean, KernelDetailsBean), flow_dict_new: dict): + def categorize_computing_performance_data(self, tk: (TraceEventBean, KernelDetailsBean), flow_start_time): if tk.is_page_attention(): self._result_data.overall_metrics.update_page_attention_info(tk.dur) return if tk.is_sdma(): self._result_data.overall_metrics.update_sdma_tensor_move_info(tk.dur) return - flow_start_time = flow_dict_new.get(tk.start_time) if flow_start_time: while self._categorize_performance_index < len(self.cpu_cube_op): cur_op = self.cpu_cube_op[self._categorize_performance_index] @@ -183,7 +182,7 @@ class BaseProfilingParser(ABC): 判断fa/conv/matmul/vector使用cpu_op """ if cpu_op.is_fa_for_cpu_op(): - if self._is_backward(cpu_op): + if cpu_op.is_bwd_for_cpu_op(): if tk.is_cube_kernel_cat(): self._result_data.overall_metrics.update_fa_bwd_cube_info(tk.dur) else: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 65fcc092f..a6b9f020c 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -68,8 +68,8 @@ class GPUProfilingParser(BaseProfilingParser): min_ts = sys.float_info.max max_ts = sys.float_info.min self._trace_events.sort(key=lambda x: x.start_time) - aten_events = [event for event in self._trace_events if event.name.startswith("aten::")] flow_dict_new = self._get_flow_time_dict() + computing_events = [] for event in self._trace_events: if event.stream: min_ts = min(event.start_time, min_ts) @@ -82,8 +82,11 @@ class GPUProfilingParser(BaseProfilingParser): self.__add_marks(event) if event.is_nccl_name(): continue - self.categorize_computing_performance_data(event, flow_dict_new) - self._aten_events = None + computing_events.append(event) + ordered_computing_events = sorted( + ((flow_dict_new.get(event.start_time, 0), event) for event in computing_events), key=lambda x: x[0]) + for flow_start_time, event in ordered_computing_events: + self.categorize_computing_performance_data(event, flow_start_time) self._result_data.overall_metrics.set_e2e_time(float(max_ts - min_ts)) self.__add_compute_and_overlap_time() diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index b763d8c9b..4139a4b6d 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -80,7 +80,7 @@ class NPUProfilingParser(BaseProfilingParser): if not kernels_dict: if self._step_id != Constant.VOID_STEP: print(f"[ERROR] There is no kernel details infomation for step {self._step_id}," \ - " please check whether the data contains this step.") + " please check whether the data contains this step.") else: print("[ERROR] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") return @@ -157,6 +157,7 @@ class NPUProfilingParser(BaseProfilingParser): sdma_bandwidth = sdma_size_mb / sdma_time_ms if sdma_time_ms > 0 else 0 self._result_data.overall_metrics.set_RDMA_bandwidth(rdma_bandwidth) self._result_data.overall_metrics.set_SDMA_bandwidth(sdma_bandwidth) + def _update_overall_metrics(self): self.__parse_info_json() self.__parse_mem_csv() @@ -170,6 +171,7 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.calculate_schedule_time() self._result_data.overall_metrics.trans_time_to_s() self._update_bandwidth() + def _picking_notify_wait_event_and_not_overlap_event(self): self.notify_event_cache = [] self._not_overlaped_commu_event = [] @@ -316,11 +318,11 @@ class NPUProfilingParser(BaseProfilingParser): self._result_data.overall_metrics.hide_op_details = True return flow_dict_new = self._get_flow_time_dict() - kernel_details.sort(key=lambda x: x.start_time) - for kernel in kernel_details: - if kernel.is_invalid(): - continue - self.categorize_computing_performance_data(kernel, flow_dict_new) + ordered_computing_events = sorted( + ((flow_dict_new.get(kernel.start_time, 0), kernel) for kernel in kernel_details if not kernel.is_invalid()), + key=lambda x: x[0]) + for flow_start_time, event in ordered_computing_events: + self.categorize_computing_performance_data(event, flow_start_time) def __parse_mem_csv(self): try: -- Gitee From 402107adde37f63da230f2024bedc45a04f10a3c Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 14 Aug 2024 14:59:23 +0800 Subject: [PATCH 284/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=A4=9A=E4=BD=99?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/utils.py | 4 ++-- .../msprobe/mindspore/compare/distributed_compare.py | 6 +++--- .../msprobe/pytorch/compare/distributed_compare.py | 4 ++-- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 4 ++-- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index e7fc2500d..7fa78c95d 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -149,7 +149,7 @@ def check_summary_only_valid(summary_only): return summary_only -def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False, framework=Const.MS_FRAMEWORK): +def check_compare_param(input_param, output_path, summary_compare=False, md5_compare=False): if not (isinstance(input_param, dict) and isinstance(output_path, str)): logger.error("Invalid input parameters") raise CompareException(CompareException.INVALID_PARAM_ERROR) @@ -477,7 +477,7 @@ def md5_find(data): return False -def task_dumppath_get(input_param, framework=Const.MS_FRAMEWORK): +def task_dumppath_get(input_param): npu_path = input_param.get("npu_json_path", None) bench_path = input_param.get("bench_json_path", None) if not npu_path or not bench_path: diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index b10b17128..9f5dcb9d4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -47,9 +47,9 @@ def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): stack_path = extract_json(npu_data_dir, stack_json=True) dump_result_param = { - 'npu_path': npu_path, - 'bench_path': bench_path, - 'stack_path': stack_path, + 'npu_json_path': npu_path, + 'bench_json_path': bench_path, + 'stack_json_path': stack_path, 'is_print_compare_log': True } try: diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index 5a6bfd15e..a90213da4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -54,10 +54,10 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): 'is_print_compare_log': True } try: - summary_compare, md5_compare = task_dumppath_get(dump_result_param, framework=Const.PT_FRAMEWORK) + summary_compare, md5_compare = task_dumppath_get(dump_result_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare, framework=Const.PT_FRAMEWORK) + check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index d95666b01..d5d0555ff 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -26,10 +26,10 @@ class PTComparator (Comparator): def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: - summary_compare, md5_compare = task_dumppath_get(input_param, framework=Const.PT_FRAMEWORK) + summary_compare, md5_compare = task_dumppath_get(input_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) - check_compare_param(input_param, output_path, summary_compare, md5_compare, framework=Const.PT_FRAMEWORK) + check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error -- Gitee From 69643cf6000c076b5595b1ef61a1688602829c8c Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 14 Aug 2024 15:21:59 +0800 Subject: [PATCH 285/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=A4=9A=E5=8D=A1?= =?UTF-8?q?=E5=8C=85=E5=BC=95=E7=94=A8bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/core/compare/compare_cli.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index e4e257bc6..be1c9bcba 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -11,18 +11,20 @@ def compare_cli(args,frame_name): npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) + if frame_name == "pytorch": + from msprobe.pytorch.compare.pt_compare import compare + from msprobe.pytorch.compare.distributed_compare import compare_distributed + else: + from msprobe.mindspore.compare.ms_compare import ms_compare + from msprobe.mindspore.compare.distributed_compare import ms_compare_distributed if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: input_param["npu_json_path"] = input_param.pop("npu_path") input_param["bench_json_path"] = input_param.pop("bench_path") input_param["stack_json_path"] = input_param.pop("stack_path") if frame_name == "pytorch": - from msprobe.pytorch.compare.pt_compare import compare - from msprobe.pytorch.compare.distributed_compare import compare_distributed compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) else: - from msprobe.mindspore.compare.ms_compare import ms_compare - from msprobe.mindspore.compare.distributed_compare import ms_compare_distributed ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: -- Gitee From d4f88cd879866e50f0f20e078096ad5361faa0e7 Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Wed, 14 Aug 2024 15:50:54 +0800 Subject: [PATCH 286/791] delete_master_branch_redundant_code --- profiler/prof_common/base_node.py | 78 ------------------------ profiler/prof_common/file_reader.py | 59 ------------------ profiler/prof_common/trace_event_bean.py | 69 --------------------- profiler/prof_common/tree_builder.py | 33 ---------- profiler/prof_common/utils.py | 25 -------- 5 files changed, 264 deletions(-) delete mode 100644 profiler/prof_common/base_node.py delete mode 100644 profiler/prof_common/file_reader.py delete mode 100644 profiler/prof_common/trace_event_bean.py delete mode 100644 profiler/prof_common/tree_builder.py delete mode 100644 profiler/prof_common/utils.py diff --git a/profiler/prof_common/base_node.py b/profiler/prof_common/base_node.py deleted file mode 100644 index b7cd67800..000000000 --- a/profiler/prof_common/base_node.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from math import ceil -from queue import Queue - -from decimal import Decimal - -from profiler.prof_common.constant import Constant -from profiler.prof_common.trace_event_bean import TraceEventBean - - -class BaseNode: - def __init__(self, event: TraceEventBean, parent_node=None): - self._event = event - self._parent_node = parent_node - self._child_nodes = [] - - @property - def parent_node(self): - return self._parent_node - - @property - def child_nodes(self): - return self._child_nodes - - @property - def name(self): - return self._event.name - - @property - def start_time(self) -> Decimal: - return self._event.start_time - - @property - def end_time(self) -> Decimal: - return self._event.end_time - - def update_child_nodes(self, node): - self._child_nodes.append(node) - - def binary_search(self, ts_time): - if not self.child_nodes: - return Constant.INVALID_RETURN - right = len(self.child_nodes) - 1 - left = 0 - while right > left: - mid = left + ceil((right - left) / 2) - if ts_time >= self.child_nodes[mid].start_time: - left = mid - else: - right = mid - 1 - if self.child_nodes[left].start_time < ts_time < self.child_nodes[left].end_time: - return self.child_nodes[left] - return Constant.INVALID_RETURN - - def find_all_child_nodes(self) -> list: - result_data = [] - node_queue = Queue() - for child_node in self.child_nodes: - node_queue.put(child_node) - while not node_queue.empty(): - tree_node = node_queue.get() - result_data.append(tree_node) - for child_node in tree_node.child_nodes: - node_queue.put(child_node) - return result_data diff --git a/profiler/prof_common/file_reader.py b/profiler/prof_common/file_reader.py deleted file mode 100644 index d8a9c8fb4..000000000 --- a/profiler/prof_common/file_reader.py +++ /dev/null @@ -1,59 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import json -import logging -import os - -from profiler.prof_common.path_manager import PathManager -from profiler.prof_common.constant import Constant - - -class FileReader: - DATA_FILE_AUTHORITY = 0o640 - DATA_DIR_AUTHORITY = 0o750 - - @classmethod - def read_json_file(cls, file_path: str) -> any: - PathManager.check_path_readable(file_path) - if not os.path.isfile(file_path): - raise FileNotFoundError("File not exists.") - file_size = os.path.getsize(file_path) - if file_size <= 0: - return [] - if file_size > Constant.MAX_FILE_SIZE_5_GB: - msg = f"The file({file_path}) size exceeds the preset max value, failed to read the file." - raise RuntimeError(msg) - try: - with open(file_path, "rt") as file: - json_data = json.loads(file.read()) - except Exception as e: - msg = f"Can't read file: {file_path}" - raise RuntimeError(msg) from e - return json_data - - @classmethod - def write_json_file(cls, output_path: str, data: dict, file_name: str, format_json: bool = False) -> None: - if not data: - return - output_file = os.path.join(output_path, file_name) - PathManager.check_path_writeable(output_path) - try: - with os.fdopen( - os.open(output_file, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY), 'w' - ) as file: - indent = 4 if format_json else None - file.write(json.dumps(data, indent=indent)) - except Exception as e: - raise RuntimeError(f"Can't create the file: {output_path}") from e diff --git a/profiler/prof_common/trace_event_bean.py b/profiler/prof_common/trace_event_bean.py deleted file mode 100644 index 2d4b96e4f..000000000 --- a/profiler/prof_common/trace_event_bean.py +++ /dev/null @@ -1,69 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from decimal import Decimal - -from profiler.prof_common.utils import convert_to_decimal -from profiler.prof_common.analyze_dict import AnalyzeDict - - -class TraceEventBean(AnalyzeDict): - def __init__(self, data: dict, unique_id: int = None): - super().__init__(data) - self._id = unique_id - - @property - def unique_id(self): - return self._id - - @property - def start_time(self) -> Decimal: - return convert_to_decimal(self.ts) - - @property - def end_time(self) -> Decimal: - return self.start_time + convert_to_decimal(self.dur) - - def set_id(self, name_id): - self._id = name_id - - def is_cpu_op(self): - return self.cat == "cpu_op" - - def is_optimizer(self): - return self.cat == "cpu_op" and self.name.lower().startswith("optimizer") - - def is_nn_module(self): - return self.cat == "python_function" and self.name.lower().startswith("nn.module") - - def is_step(self): - return self.name.lower().startswith("profilerstep#") - - def is_torch_to_npu(self): - return self.cat == "async_npu" - - def is_fwd_bwd_flow(self): - return self.cat == "fwdbwd" - - def is_flow_start(self): - return self.ph == "s" - - def is_flow_end(self): - return self.ph == "f" - - def is_kernel_event(self, kernel_pid): - return self.ph == "X" and self.pid == kernel_pid - - def is_npu_process(self): - return self.ph == "M" and self.name == "process_name" and self.args.get("name", "") == "Ascend Hardware" diff --git a/profiler/prof_common/tree_builder.py b/profiler/prof_common/tree_builder.py deleted file mode 100644 index b7d3e1baf..000000000 --- a/profiler/prof_common/tree_builder.py +++ /dev/null @@ -1,33 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from profiler.prof_common.trace_event_bean import TraceEventBean - - -class TreeBuilder: - @staticmethod - def build_tree(event_list: list, node_class: any, root_bean: any): - root_node = node_class(root_bean) - event_list.sort(key=lambda x: x.start_time) - last_node = root_node - for event in event_list: - while last_node: - if last_node != root_node and event.start_time > last_node.end_time: - last_node = last_node.parent_node - continue - tree_node = node_class(event, last_node) - last_node.update_child_nodes(tree_node) - last_node = tree_node - break - return root_node diff --git a/profiler/prof_common/utils.py b/profiler/prof_common/utils.py deleted file mode 100644 index a9db41ad0..000000000 --- a/profiler/prof_common/utils.py +++ /dev/null @@ -1,25 +0,0 @@ -# Copyright (c) 2024 Huawei Technologies Co., Ltd -# All rights reserved. -# -# Licensed under the BSD 3-Clause License (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://opensource.org/licenses/BSD-3-Clause -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from decimal import Decimal - - -def convert_to_decimal(data: any) -> Decimal: - try: - decimal_value = Decimal(data) - except Exception: - logging.error('Invalid profiling data which failed to convert data to decimal.') - return 0.0 - return decimal_value -- Gitee From 18c8a3fa5e6311aaade79078a01a0ab73fef1e46 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Wed, 14 Aug 2024 08:22:14 +0000 Subject: [PATCH 287/791] update debug/accuracy_tools/msprobe/pytorch/common/utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 9cf326ea2..e29c78bcf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -301,10 +301,10 @@ def get_json_contents(file_path): try: json_obj = json.loads(ops) except ValueError as error: - logger.error('Failed to load "%s". %s' % (file_path, str(error))) + logger.error('Failed to load "%s". %s', file_path, str(error)) raise CompareException(CompareException.INVALID_FILE_ERROR) from error if not isinstance(json_obj, dict): - logger.error('Json file %s, content is not a dictionary!' % file_path) + logger.error('Json file %s, content is not a dictionary!', file_path) raise CompareException(CompareException.INVALID_FILE_ERROR) return json_obj -- Gitee From 4a8d88403fbd7d101b0f63543904b03eb5ec57e2 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Mon, 12 Aug 2024 20:29:00 +0800 Subject: [PATCH 288/791] =?UTF-8?q?=E9=99=8D=E9=A2=91=E5=88=86=E6=9E=90?= =?UTF-8?q?=E9=A2=9D=E5=AE=9A=E9=A2=91=E7=8E=87bug=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../analyzer/computation/ai_core_freq/ai_core_freq_checker.py | 3 +-- profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py | 2 +- .../advisor/dataset/communication/communication_dataset.py | 2 +- profiler/advisor/dataset/timeline_event_dataset.py | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index 7afa09cca..5bfa5adc4 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -10,7 +10,6 @@ logger = logging.getLogger() class AICoreFreqChecker: - DEFAULT_FREQ = 1800 DECREASE_FREQ_RATIO = 0.05 SHOW_TOPK_OPS = 10 TOTAL_DURATION_INDEX = 2 @@ -46,7 +45,7 @@ class AICoreFreqChecker: op_count = op_info.get("count", 0) op_total_duration = round(op_info.get("dur", 0), 2) - max_freq = max(self.DEFAULT_FREQ, convert_to_float(Config().get_config("aic_frequency"))) + max_freq = convert_to_float(Config().get_config("aic_frequency")) decrease_freq_ratio = sum(max_freq - freq for freq in freq_list) / (max_freq * len(freq_list)) if decrease_freq_ratio >= Config().get_config("frequency_threshold"): diff --git a/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py b/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py index c99baea65..96cfe2248 100644 --- a/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py +++ b/profiler/advisor/dataset/ai_core_freq/ai_core_freq_dataset.py @@ -69,7 +69,7 @@ class AICoreFreqDataset: return False if len(self.timeline_data_list) > 1: - logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis .", + logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis.", self.timeline_dir) _ = parse_json_with_generator(sorted(self.timeline_data_list)[0], self._add_event) diff --git a/profiler/advisor/dataset/communication/communication_dataset.py b/profiler/advisor/dataset/communication/communication_dataset.py index 6cfc87083..0a8c9e4a1 100644 --- a/profiler/advisor/dataset/communication/communication_dataset.py +++ b/profiler/advisor/dataset/communication/communication_dataset.py @@ -85,7 +85,7 @@ class CommunicationDataset: return False if len(self.timeline_data_list) > 1: - logger.warning("Found multiple communication.json in %s, load the file of device 0 for analysis .", + logger.warning("Found multiple communication.json in %s, load the file of device 0 for analysis.", self.timeline_dir) json_data = self.load_json_data(sorted(self.timeline_data_list)[0]) diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/advisor/dataset/timeline_event_dataset.py index 1504e65f5..2956e2075 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/advisor/dataset/timeline_event_dataset.py @@ -170,7 +170,7 @@ class TimelineEventDataset: return False if len(self.timeline_data_list) > 1: - logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis .", + logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis.", self.timeline_dir) result = self.parse_data_with_generator(self._add_event) -- Gitee From 6c8fb0e410edcde3b11f07dd088a5a874a0e3eaf Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Mon, 12 Aug 2024 15:17:15 +0800 Subject: [PATCH 289/791] Supplement the important parameters of the setup.py required by the publishing community --- profiler/LICENSE | 201 +++++++++++++++++++++++++++++++++++++++++++ profiler/setup.py | 7 ++ profiler/version.txt | 2 +- 3 files changed, 209 insertions(+), 1 deletion(-) create mode 100644 profiler/LICENSE diff --git a/profiler/LICENSE b/profiler/LICENSE new file mode 100644 index 000000000..261eeb9e9 --- /dev/null +++ b/profiler/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/profiler/setup.py b/profiler/setup.py index 19c8729d9..4a625f6c9 100644 --- a/profiler/setup.py +++ b/profiler/setup.py @@ -28,6 +28,12 @@ setup( name="msprof-analyze", version=version, description="MindStudio Profiler Analyze Tools", + long_description="msprof-analyze provides statistics, analysis, and related tuning suggestions for the " + "performance data collected in training and large model scenarios. The main functional modules" + " include: performance comparison, performance analysis, and cluster analysis.", + url="https://gitee.com/ascend/mstt/tree/master/profiler", + author="MindStudio", + author_email="pmail_mindstudio@huawei.com", package_dir={"": root_path}, packages=find_packages(root_path), include_package_data=False, @@ -35,6 +41,7 @@ setup( install_requires=requires, package_data={'': ['*.json', '*.ini', '*.txt', '*.yaml', '*.html', '*.ipynb']}, tests_require=tests_requires, + license='Apache License 2.0', entry_points=""" [console_scripts] msprof-analyze=profiler.cli.entrance:msprof_analyze_cli diff --git a/profiler/version.txt b/profiler/version.txt index 8428158dc..cb174d58a 100644 --- a/profiler/version.txt +++ b/profiler/version.txt @@ -1 +1 @@ -1.1.2 \ No newline at end of file +1.2.1 \ No newline at end of file -- Gitee From c8bedf6f5d4a3097739d3b67569d3da6eef030bd Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Wed, 14 Aug 2024 09:37:34 +0000 Subject: [PATCH 290/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 7e5891b5a..4343e630b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -20,7 +20,7 @@ from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Message, hf_32_standard_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args -from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents, api_info_preprocess, \ +from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ initialize_save_path, UtDataProcessor from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn @@ -34,6 +34,7 @@ from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward from msprobe.core.common.file_check import FileOpen, FileChecker, \ change_mode, check_file_suffix, check_link, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger +from msprobe.pytorch.common.utils import get_json_contents from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTL, ATTLConfig, ApiData, move2device_exec -- Gitee From fd9850bed58dac63f1004dcaca8b106733f6fb1f Mon Sep 17 00:00:00 2001 From: Mrtutu Date: Tue, 13 Aug 2024 22:50:37 +0800 Subject: [PATCH 291/791] fix tuple --- .../cluster_analyse/analysis/step_trace_time_analysis.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py index 617c0aafc..f18c8aaca 100644 --- a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py @@ -78,8 +78,13 @@ class StepTraceTimeAnalysis: for step_data in self.step_data_list: rank_id = step_data[2] - step_data.extend(list(parallelism_map[rank_id]) - if parallelism_map[rank_id] else ['NA'] * len(self.PARALLEL_HEADERS)) + if isinstance(rank_id, int): + # type is rank, rank_id is int + step_data.extend(list(parallelism_map[rank_id]) + if parallelism_map[rank_id] else ['NA'] * len(self.PARALLEL_HEADERS)) + else: + # type is stage, rank_id is tuple + step_data.extend(['NA'] * len(self.PARALLEL_HEADERS)) def dump_data(self): if not self.step_data_list: -- Gitee From f779dd0ad6d0b3b593784bdef30c3fe8e46bdfc8 Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Wed, 14 Aug 2024 14:58:45 +0800 Subject: [PATCH 292/791] bugfix: adapt overall_perf_interface --- profiler/compare_tools/compare_backend/utils/args_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/compare_tools/compare_backend/utils/args_manager.py b/profiler/compare_tools/compare_backend/utils/args_manager.py index 69136c4d7..f8b8b5047 100644 --- a/profiler/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/compare_tools/compare_backend/utils/args_manager.py @@ -11,7 +11,7 @@ class Singleton(object): self._cls = cls self._instance = {} - def __call__(self, args): + def __call__(self, args = None): if self._cls not in self._instance: self._instance[self._cls] = self._cls(args) return self._instance[self._cls] -- Gitee From 32b89b716baaad3176d72f7aff12813bcf943c36 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 14 Aug 2024 19:52:11 +0800 Subject: [PATCH 293/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E9=97=A8=E7=A6=81bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/compare/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 8a4c5fb55..ff1974f51 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -67,8 +67,8 @@ def rename_api(npu_name, process): def read_op(op_data, op_name): op_parsed_list = Const.DEFAULT_LIST if Const.FORWARD in op_name: - if Const.INPUT in op_data: - input_item = op_data[Const.INPUT] + if Const.INPUT_ARGS in op_data: + input_item = op_data[Const.INPUT_ARGS] input_parsed_list = op_item_parse(input_item, op_name + '.input', None) op_parsed_list = input_parsed_list.copy() input_parsed_list.clear() -- Gitee From 0665165fbb60754e5aeef95add62f5cc75c4d842 Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Thu, 15 Aug 2024 10:43:07 +0800 Subject: [PATCH 294/791] bug_fix_for_compare_tools --- .../generator/detail_performance_generator.py | 2 +- .../profiling_parser/gpu_profiling_parser.py | 9 --------- profiler/compare_tools/compare_backend/utils/constant.py | 6 +++--- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py index 916c426c6..5bacc24ed 100644 --- a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -84,7 +84,7 @@ class DetailPerformanceGenerator(BaseGenerator): # build tree for operator_compare memory_compare and api_compare base_op_prepare, comparison_op_prepare = None, None - if self._args.enable_memory_compare or self.enable_api_compare or enable_operator_compare: + if self._args.enable_memory_compare or self._args.enable_api_compare or enable_operator_compare: base_op_prepare = OperatorDataPrepare(self._profiling_data_dict.get(Constant.BASE_DATA), self._base_step_id) comparison_op_prepare = OperatorDataPrepare(self._profiling_data_dict.get(Constant.COMPARISON_DATA), diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index a6b9f020c..8dfa7cd53 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -7,8 +7,6 @@ from compare_backend.utils.constant import Constant class GPUProfilingParser(BaseProfilingParser): - CUBE_MARK = ['gemm', 'conv', 'cutlass', 'wgrad'] - FA_MARK_LIST = [['fmha', 'kernel'], ['flash', 'kernel'], ['attention', 'kernel']] SDMA_MARK_LIST = ['htod', 'dtod', 'dtoh', 'memset (device)'] FLOW_CAT = ("async_gpu", "async_cpu_to_gpu", "ac2g", "async") TORCH_OP_CAT = ("cpu_op", "user_annotation", "cuda_runtime", "operator", "runtime") @@ -22,13 +20,6 @@ class GPUProfilingParser(BaseProfilingParser): self._aten_index = 0 self._find_bwd_tid() - @classmethod - def __is_flash_attention(cls, name: str): - for fa_mark in cls.FA_MARK_LIST: - if not [1 for mark in fa_mark if mark not in name.lower()]: - return True - return False - @classmethod def __is_sdma_time(cls, name: str): return any(mask in name.lower() for mask in cls.SDMA_MARK_LIST) diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index 08eb1792a..ffe6a906e 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -64,7 +64,7 @@ class Constant(object): MODULE_TOP_TABLE = "ModuleCompareStatistic" OVERALL_METRICS_TABLE = "OverallMetrics" API_TABLE = "ApiCompare" - KERNEL_TABLE = "KernelCompare" + KERNEL_TABLE = "KernelCompare" # memory SIZE = "Size(KB)" @@ -87,7 +87,7 @@ class Constant(object): # compare type OVERALL_COMPARE = "overall" - BWD_LIST = ["bwd", "backward", "back"] + BWD_LIST = ["bwd", "backward", "back", "grad"] CPU_OP_FA_MASK = ("flash_attention", "fusion_attention", "flashattn", "xformers_flash", "efficient_attention") CPU_OP_CONV = "aten::conv" @@ -98,4 +98,4 @@ class Constant(object): IS_BWD = "is_bwd" OPS = "ops" - VOID_STEP = -1 \ No newline at end of file + VOID_STEP = -1 -- Gitee From 5d3f3add4803d4bd40799d2e37e8f2915e547517 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Thu, 15 Aug 2024 11:43:24 +0800 Subject: [PATCH 295/791] =?UTF-8?q?[profiler]=E8=BF=AD=E4=BB=A3=E4=BA=8C?= =?UTF-8?q?=E5=87=BA=E5=8C=85=E8=B5=84=E6=96=99=E8=A1=A5=E5=85=85=E4=B8=8B?= =?UTF-8?q?=E8=BD=BD=E9=93=BE=E6=8E=A5=E4=BB=A5=E5=8F=8A=E6=96=B0=E5=A2=9E?= =?UTF-8?q?msprof-analyze=E7=9A=84pip=E5=91=BD=E4=BB=A4=E8=A1=8C=E5=AE=89?= =?UTF-8?q?=E8=A3=85=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/profiler/README.md b/profiler/README.md index 956764913..411fe2a43 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -81,7 +81,23 @@ ascend pytorch profiler数据目录结构如下： ## 工具安装 -性能工具的安装方式包括：**下载whl包安装**和**源代码编译安装**。 +性能工具的安装方式包括：**pip安装**、**下载whl包安装**和**源代码编译安装**。 + +### pip安装 + +```shell +pip install msprof-analyze +``` + +使用`pip install msprof-analyze==版本号`可安装指定版本的包，支持1.2.1及之后版本，版本号参见“**下载whl包安装**”。 + +pip命令会自动安装最新的包及其配套依赖。 + +提示如下信息则表示安装成功。 + +```bash +Successfully installed msprof-analyze-{version} +``` #### 下载whl包安装 @@ -91,6 +107,7 @@ ascend pytorch profiler数据目录结构如下： | profiler版本 | 发布日期 | 下载链接 | 校验码 | | ------------ | ---------- | ------------------------------------------------------------ | ------------------------------------------------------------ | + | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | | 1.1.2 | 2024-07-12 | [msprof_analyze-1.1.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.2/msprof_analyze-1.1.2-py3-none-any.whl) | af62125b1f9348bf491364e03af712fc6d0282ccee3fb07458bc9bbef82dacc6 | | 1.1.1 | 2024-06-20 | [msprof_analyze-1.1.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.1/msprof_analyze-1.1.1-py3-none-any.whl) | 76aad967a3823151421153d368d4d2f8e5cfbcb356033575e0b8ec5acea8e5e4 | -- Gitee From 1438c08492286d47123a949754767b1112f8ae6a Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Thu, 15 Aug 2024 11:46:38 +0800 Subject: [PATCH 296/791] =?UTF-8?q?[msit]=E5=88=A0=E9=99=A4=E7=BB=91?= =?UTF-8?q?=E6=A0=B8=E5=B7=A5=E5=85=B7=E9=93=BE=E6=8E=A5=E8=A1=A5=E5=85=85?= =?UTF-8?q?advisor=E5=B7=A5=E5=85=B7=E9=93=BE=E6=8E=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d88a843a9..601deec66 100644 --- a/README.md +++ b/README.md @@ -62,9 +62,10 @@ MindStudio Training Tools，MindStudio训练工具链。针对训练&大模型 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合MindStudio Insight的集群分析功能使用。 -3. [affinity_cpu_bind (亲和性cpu绑核工具) ](https://gitee.com/ascend/mstt/tree/master/profiler/affinity_cpu_bind) +3. [advisor](https://gitee.com/ascend/mstt/blob/master/profiler/advisor) + + 将Ascend PyTorch Profiler或者msprof采集的PyThon场景性能数据进行分析，并输出性能调优建议。 - 提供亲和性CPU绑核能力，改善host_bound调度问题。 ### [Tensorboard](https://gitee.com/ascend/mstt/tree/master/plugins/tensorboard-plugins/tb_plugin) @@ -78,10 +79,10 @@ MindStudio Training Tools工具版本分支的维护阶段如下： | **状态** | **时间** | **说明** | | ------------------- | -------- | ------------------------------------------------ | -| 计划 | 1—3 个月 | 计划特性 | +| 计划 | 1~3 个月 | 计划特性 | | 开发 | 3个月 | 开发特性 | -| 维护 | 6—12个月 | 合入所有已解决的问题并发布版本 | -| 无维护 | 0—3 个月 | 合入所有已解决的问题，无专职维护人员，无版本发布 | +| 维护 | 6~12个月 | 合入所有已解决的问题并发布版本 | +| 无维护 | 0~3 个月 | 合入所有已解决的问题，无专职维护人员，无版本发布 | | 生命周期终止（EOL） | N/A | 分支不再接受任何修改 | ## 现有分支的维护状态 -- Gitee From c912142e2765767bae80bff2c9eb11373fe1dd18 Mon Sep 17 00:00:00 2001 From: makai Date: Thu, 15 Aug 2024 15:13:41 +0800 Subject: [PATCH 297/791] =?UTF-8?q?=E6=B7=BB=E5=8A=A0load=5Fyaml=E5=87=BD?= =?UTF-8?q?=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/pytorch/common/utils.py | 14 +++++++++++++- .../msprobe/pytorch/hook_module/utils.py | 12 ++++++------ debug/accuracy_tools/msprobe/pytorch/pt_config.py | 4 ++-- 3 files changed, 21 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index ae8823de6..b304c8a12 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -18,13 +18,14 @@ import logging import os import random import stat +import yaml import torch import torch.distributed as dist import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create -from msprobe.core.common.file_check import FileCheckConst, change_mode +from msprobe.core.common.file_check import FileChecker, FileOpen, FileCheckConst, change_mode try: @@ -283,6 +284,17 @@ def save_pt(tensor, filepath): except Exception as e: raise RuntimeError(f"save pt file {filepath} failed") from e change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + +def load_yaml(yaml_path): + path_checker = FileChecker(yaml_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.YAML_SUFFIX) + checked_path = path_checker.common_check() + try: + with FileOpen(checked_path, "r") as f: + Ops = yaml.safe_load(f) + except Exception as e: + raise RuntimeError(f"load yaml file {yaml_path} failed") from e + return Ops def _create_logger(level=logging.INFO): diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py index c1e581675..bd5e6e669 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py @@ -16,14 +16,14 @@ """ import os -import yaml +from msprobe.pytorch.common.utils import load_yaml -from msprobe.core.common.file_check import FileOpen -cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - Ops = yaml.safe_load(f) +def get_Ops(): + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") + Ops = load_yaml(yaml_path) WrapFunctionalOps = Ops.get('functional') WrapTensorOps = Ops.get('tensor') WrapTorchOps = Ops.get('torch') + return set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 68225f8a8..adfe12f15 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -4,7 +4,7 @@ import os from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.file_check import FileOpen from msprobe.core.common.const import Const -from msprobe.pytorch.hook_module.utils import WrapFunctionalOps, WrapTensorOps, WrapTorchOps +from msprobe.pytorch.hook_module.utils import get_Ops from msprobe.core.grad_probe.constant import level_adp from msprobe.core.grad_probe.utils import check_numeral_list_ascend @@ -81,7 +81,7 @@ class FreeBenchmarkCheckConfig(BaseConfig): class RunUTConfig(BaseConfig): - WrapApi = set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) + WrapApi = get_Ops() def __init__(self, json_config): super().__init__(json_config) -- Gitee From e747a02a665c09e2f91b66c1f0d4304ad8f9851f Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 14 Aug 2024 20:55:05 +0800 Subject: [PATCH 298/791] =?UTF-8?q?ut=E9=80=82=E9=85=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/compare_cli.py | 13 +++++++------ debug/accuracy_tools/msprobe/msprobe.py | 19 ++++++++++--------- .../msprobe/pytorch/compare/pt_compare.py | 1 - .../msprobe/test/core_ut/common/test_utils.py | 18 +++++++++--------- 4 files changed, 26 insertions(+), 25 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index be1c9bcba..ac96a65de 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -1,17 +1,18 @@ import json from msprobe.core.common.file_check import FileOpen, check_file_type -from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.const import FileCheckConst, Const from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger -def compare_cli(args,frame_name): + +def compare_cli(args): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) - - if frame_name == "pytorch": + frame_name =args.framework + if frame_name ==Const.PT_FRAMEWORK: from msprobe.pytorch.compare.pt_compare import compare from msprobe.pytorch.compare.distributed_compare import compare_distributed else: @@ -21,7 +22,7 @@ def compare_cli(args,frame_name): input_param["npu_json_path"] = input_param.pop("npu_path") input_param["bench_json_path"] = input_param.pop("bench_path") input_param["stack_json_path"] = input_param.pop("stack_path") - if frame_name == "pytorch": + if frame_name == Const.PT_FRAMEWORK: compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) else: @@ -29,7 +30,7 @@ def compare_cli(args,frame_name): fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} - if frame_name == "pytorch": + if frame_name == Const.PT_FRAMEWORK: compare_distributed(npu_path, bench_path, args.output_path, **kwargs) else: ms_compare_distributed(npu_path, bench_path, args.output_path, **kwargs) diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index af8034077..97f90a64d 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -19,6 +19,7 @@ import importlib.util from msprobe.core.compare.utils import _compare_parser from msprobe.core.common.log import logger from msprobe.core.compare.compare_cli import compare_cli +from msprobe.core.common.const import Const def is_module_available(module_name): spec =importlib.util.find_spec(module_name) @@ -33,7 +34,7 @@ def main(): ) parser.set_defaults(print_help=parser.print_help) - parser.add_argument('-f', '--framework', required=True, choices=['pytorch', 'mindspore'], + parser.add_argument('-f', '--framework', required=True, choices=[Const.PT_FRAMEWORK, Const.MS_FRAMEWORK], help='Deep learning framework.') subparsers = parser.add_subparsers() subparsers.add_parser('parse') @@ -64,10 +65,10 @@ def main(): parser.print_help() sys.exit(0) args = parser.parse_args(sys.argv[1:]) - if sys.argv[2] == "pytorch": + if sys.argv[2] == Const.PT_FRAMEWORK: if not is_torch_available: - logger.error("PyTorch does not exit, please install PyTorch library") - raise Exception("PyTorch does not exit, please install PyTorch library") + logger.error("PyTorch does not exist, please install PyTorch library") + raise Exception("PyTorch does not exist, please install PyTorch library") if sys.argv[3] == "run_ut": run_ut_command(args) elif sys.argv[3] == "parse": @@ -80,13 +81,13 @@ def main(): elif sys.argv[3] == "run_overflow_check": _run_overflow_check_command(args) elif sys.argv[3] == "compare": - compare_cli(args,"pytorch") + compare_cli(args) else: - if not is_module_available("mindspore"): - logger.error("MindSpore does not exit, please install MindSpore library") - raise Exception("MindSpore does not exit, please install MindSpore library") + if not is_module_available(Const.MS_FRAMEWORK): + logger.error("MindSpore does not exist, please install MindSpore library") + raise Exception("MindSpore does not exist, please install MindSpore library") if sys.argv[3] == "compare": - compare_cli(args,"mindspore") + compare_cli(args) if __name__ == "__main__": main() diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index d5d0555ff..c0319c64d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,4 +1,3 @@ - import os.path import torch from msprobe.core.common.const import FileCheckConst, Const diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index a02a402f6..0007312a7 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -189,9 +189,9 @@ class TestUtils(TestCase): @patch.object(logger, "error") def test_check_compare_param(self, mock_error): params = { - "npu_path": "npu_path", - "bench_path": "bench_path", - "stack_path": "stack_path", + "npu_json_path": "npu_path", + "bench_json_path": "bench_path", + "stack_json_path": "stack_path", "npu_dump_data_dir": "npu_dump_data_dir", "bench_dump_data_dir": "bench_dump_data_dir" } @@ -264,9 +264,9 @@ class TestUtils(TestCase): @patch("msprobe.core.common.utils._check_json") def test_check_json_file(self, _mock_check_json): input_param = { - "npu_path": "npu_path", - "bench_path": "bench_path", - "stack_path": "stack_path" + "npu_json_path": "npu_path", + "bench_json_path": "bench_path", + "stack_json_path": "stack_path" } check_json_file(input_param, "npu_json", "bench_json", "stack_json") self.assertEqual(_mock_check_json.call_args_list[0][0], ("npu_json", "npu_path")) @@ -307,8 +307,8 @@ class TestUtils(TestCase): @patch.object(logger, "error") def test_task_dumppath_get(self, mock_error): input_param = { - "npu_path": None, - "bench_path": "bench_path" + "npu_json_path": None, + "bench_json_path": "bench_path" } npu_json = { "task": Const.TENSOR, @@ -321,7 +321,7 @@ class TestUtils(TestCase): self.assertEqual(context.exception.code, CompareException.INVALID_PATH_ERROR) mock_error.assert_called_with("Please check the json path is valid.") - input_param["npu_path"] = "npu_path" + input_param["npu_json_path"] = "npu_path" with patch("msprobe.core.common.utils.FileOpen", mock_open(read_data="")), \ patch("msprobe.core.common.utils.json.load", return_value=npu_json): summary_compare, md5_compare = task_dumppath_get(input_param) -- Gitee From 9e945209dc92b710bc2020a590577491d3ae870a Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 15 Aug 2024 19:39:29 +0800 Subject: [PATCH 299/791] run_overflow_check: fix output of exec_api is None error --- .../pytorch/api_accuracy_checker/run_ut/run_overflow_check.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index 732745ee8..f1b4fc3a0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -83,6 +83,10 @@ def run_torch_api(api_full_name, api_info_dict, real_data_path): del kwargs["device"] out = exec_api(api_type, api_name, args, kwargs) npu_out = exec_api(api_type, api_name, npu_args, npu_kwargs) + if not out and not npu_out: + logger.warning("The %s overflow is a normal overflow, out and npu_out is None." % api_full_name) + return + cpu_overflow = check_data_overflow(out) npu_overflow = torch_npu.npu.utils.npu_check_overflow(npu_out) if cpu_overflow == npu_overflow: -- Gitee From 1b0c6ec1712c96e8841bc8fe335520a06889f105 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 15 Aug 2024 20:22:36 +0800 Subject: [PATCH 300/791] fix reviews --- .../pytorch/api_accuracy_checker/run_ut/run_overflow_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index f1b4fc3a0..2e892c276 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -83,7 +83,7 @@ def run_torch_api(api_full_name, api_info_dict, real_data_path): del kwargs["device"] out = exec_api(api_type, api_name, args, kwargs) npu_out = exec_api(api_type, api_name, npu_args, npu_kwargs) - if not out and not npu_out: + if out is None and npu_out is None: logger.warning("The %s overflow is a normal overflow, out and npu_out is None." % api_full_name) return -- Gitee From 5425ed0ff292cd583042bb9a4e25bb8dea696ae1 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 15 Aug 2024 12:57:58 +0000 Subject: [PATCH 301/791] update UT Signed-off-by: jiangchangting1 --- .../common/test_common_utils.py | 25 ------------------- 1 file changed, 25 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py index ef6eea318..5802103c2 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py @@ -5,28 +5,10 @@ import unittest from unittest.mock import patch from msprobe.pytorch.api_accuracy_checker.common.utils import * -from msprobe.pytorch.common.utils import get_json_contents from msprobe.core.common.utils import write_csv class TestUtils(unittest.TestCase): - @patch('msprobe.pytorch.api_accuracy_checker.common.utils.get_file_content_bytes') - def test_get_json_contents_should_raise_exception(self, mock_get_file_content_bytes): - mock_get_file_content_bytes.return_value = 'not a dict' - with self.assertRaises(CompareException) as ce: - get_json_contents('') - self.assertEqual(ce.exception.code, CompareException.INVALID_FILE_ERROR) - - def test_get_json_contents_should_return_json_obj(self): - test_dict = {"key": "value"} - file_name = 'test.json' - - fd = os.open(file_name, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644) - with os.fdopen(fd, 'w') as f: - json.dump(test_dict, f) - self.assertEqual(get_json_contents(file_name), test_dict) - os.remove(file_name) - def test_write_csv(self): test_file_name = 'test.csv' test_data = [["name", "age"], ["Alice", "20"], ["Bob", "30"]] @@ -59,13 +41,6 @@ class TestUtils(unittest.TestCase): self.assertTrue(os.path.exists(test_dir_name)) os.rmdir(test_dir_name) - def test_get_file_content_bytes(self): - fd = os.open('test.txt', os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644) - with os.fdopen(fd, 'w') as f: - f.write("Hello, World!") - self.assertEqual(get_file_content_bytes('test.txt'), b"Hello, World!") - os.remove('test.txt') - @patch('os.path.exists') def test_check_file_or_dir_path_should_raise_exe_when_dir_path_not_existed(self, mock_path_exists): mock_path_exists.return_value = False -- Gitee From 6b27f6ce90c291f6090c7cb0005dacff662e3db5 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 15 Aug 2024 13:00:44 +0000 Subject: [PATCH 302/791] update debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py. Signed-off-by: jiangchangting1 --- .../msprobe/test/core_ut/common/test_utils.py | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index a02a402f6..c77cfa556 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -40,7 +40,9 @@ from msprobe.core.common.utils import (CompareException, check_file_size, check_regex_prefix_format_valid, get_dump_data_path, - task_dumppath_get) + task_dumppath_get, + get_json_contents, + get_file_content_bytes) from msprobe.core.common.file_check import FileCheckConst @@ -343,3 +345,26 @@ class TestUtils(TestCase): task_dumppath_get(input_param) self.assertEqual(context.exception.code, CompareException.INVALID_TASK_ERROR) mock_error.assert_called_with("Compare is not required for overflow_check or free_benchmark.") + + def test_get_json_contents_should_raise_exception(self, mock_get_file_content_bytes): + mock_get_file_content_bytes.return_value = 'not a dict' + with self.assertRaises(CompareException) as ce: + get_json_contents('') + self.assertEqual(ce.exception.code, CompareException.INVALID_FILE_ERROR) + + def test_get_json_contents_should_return_json_obj(self): + test_dict = {"key": "value"} + file_name = 'test.json' + + fd = os.open(file_name, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644) + with os.fdopen(fd, 'w') as f: + json.dump(test_dict, f) + self.assertEqual(get_json_contents(file_name), test_dict) + os.remove(file_name) + + def test_get_file_content_bytes(self): + fd = os.open('test.txt', os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o644) + with os.fdopen(fd, 'w') as f: + f.write("Hello, World!") + self.assertEqual(get_file_content_bytes('test.txt'), b"Hello, World!") + os.remove('test.txt') -- Gitee From 5bb9dda9790479a1cd5edc00e07e43ad249829a4 Mon Sep 17 00:00:00 2001 From: makai Date: Thu, 15 Aug 2024 22:31:59 +0800 Subject: [PATCH 303/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++-- .../msprobe/pytorch/hook_module/utils.py | 10 +++++----- debug/accuracy_tools/msprobe/pytorch/pt_config.py | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index b304c8a12..048f3e07b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -291,10 +291,10 @@ def load_yaml(yaml_path): checked_path = path_checker.common_check() try: with FileOpen(checked_path, "r") as f: - Ops = yaml.safe_load(f) + yaml_data = yaml.safe_load(f) except Exception as e: raise RuntimeError(f"load yaml file {yaml_path} failed") from e - return Ops + return yaml_data def _create_logger(level=logging.INFO): diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py index bd5e6e669..53814ce12 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py @@ -19,11 +19,11 @@ import os from msprobe.pytorch.common.utils import load_yaml -def get_Ops(): +def get_ops(): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") - Ops = load_yaml(yaml_path) - WrapFunctionalOps = Ops.get('functional') - WrapTensorOps = Ops.get('tensor') - WrapTorchOps = Ops.get('torch') + ops = load_yaml(yaml_path) + WrapFunctionalOps = ops.get('functional') + WrapTensorOps = ops.get('tensor') + WrapTorchOps = ops.get('torch') return set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index adfe12f15..f86ac1580 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -4,7 +4,7 @@ import os from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.core.common.file_check import FileOpen from msprobe.core.common.const import Const -from msprobe.pytorch.hook_module.utils import get_Ops +from msprobe.pytorch.hook_module.utils import get_ops from msprobe.core.grad_probe.constant import level_adp from msprobe.core.grad_probe.utils import check_numeral_list_ascend @@ -81,7 +81,7 @@ class FreeBenchmarkCheckConfig(BaseConfig): class RunUTConfig(BaseConfig): - WrapApi = get_Ops() + WrapApi = get_ops() def __init__(self, json_config): super().__init__(json_config) -- Gitee From 40ebc37c5edd37a7da24114eb372aad89a7e1237 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 16 Aug 2024 10:22:57 +0800 Subject: [PATCH 304/791] safety bugfix --- .../accuracy_tools/msprobe/core/common/utils.py | 16 ++++++++++++++++ .../msprobe/core/compare/acc_compare.py | 15 ++++++++------- .../msprobe/core/compare/compare_cli.py | 5 ++--- .../msprobe/core/compare/highlight.py | 13 ++++--------- .../msprobe/pytorch/compare/pt_compare.py | 8 +++++--- 5 files changed, 35 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 7fa78c95d..cdc0ebe5a 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -545,3 +545,19 @@ def save_npy(data, filepath): except Exception as e: raise RuntimeError(f"save npy file {filepath} failed") from e change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + +def save_workbook(workbook, file_path): + """ + 保存工作簿到指定的文件路径 + workbook: 要保存的工作簿对象 + file_path: 文件保存路径 + """ + file_path = os.path.realpath(file_path) + check_path_before_create(file_path) + try: + workbook.save(file_path) + except Exception as e: + logger.error(f'Save result file "{os.path.basename(file_path)}" failed') + raise CompareException(CompareException.WRITE_FILE_ERROR) from e + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index be749e5aa..c1991616b 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -15,6 +15,7 @@ from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, get_error_message from msprobe.core.advisor.advisor import Advisor + class Comparator: def __init__(self): @@ -154,7 +155,7 @@ class Comparator: for npu_data in npu_ops_queue: get_un_match_accuracy(result, npu_data, md5_compare, summary_compare) - result_df = self.make_result_table(result,md5_compare,summary_compare,stack_mode) + result_df = self.make_result_table(result, md5_compare, summary_compare, stack_mode) return result_df def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): @@ -196,7 +197,7 @@ class Comparator: result_list.append(err_msg) return result_list - def compare_core(self,input_parma, output_path, **kwargs): + def compare_core(self, input_parma, output_path, **kwargs): """ Compares data from multiple JSON files and generates a comparison report. @@ -270,12 +271,12 @@ class Comparator: five_thousand_err_ratio_result.append(five_thousand_err_ratio) cr = ComparisonResult( - cos_result = cos_result, - max_err_result = max_err_result, + cos_result=cos_result, + max_err_result=max_err_result, max_relative_err_result=max_relative_err_result, - err_msgs = err_mess, - one_thousand_err_ratio_result = one_thousand_err_ratio_result, - five_thousand_err_ratio_result = five_thousand_err_ratio_result + err_msgs=err_mess, + one_thousand_err_ratio_result=one_thousand_err_ratio_result, + five_thousand_err_ratio_result=five_thousand_err_ratio_result ) return _save_cmp_result(idx, cr, result_df, lock) diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index ac96a65de..2e7eb2ddf 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -5,14 +5,13 @@ from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger - def compare_cli(args): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) - frame_name =args.framework - if frame_name ==Const.PT_FRAMEWORK: + frame_name = args.framework + if frame_name == Const.PT_FRAMEWORK: from msprobe.pytorch.compare.pt_compare import compare from msprobe.pytorch.compare.distributed_compare import compare_distributed else: diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index ef35fd061..8f0a625be 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -4,10 +4,9 @@ from collections import namedtuple import numpy as np import openpyxl from openpyxl.styles import PatternFill -from msprobe.core.common.utils import get_header_index, CompareException +from msprobe.core.common.utils import get_header_index, save_workbook from msprobe.core.common.log import logger -from msprobe.core.common.file_check import change_mode -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.const import CompareConst class HighlightCheck(abc.ABC): @@ -219,9 +218,5 @@ def highlight_rows_xlsx(result_df, highlight_dict, file_path): elif (i - 2) in highlight_dict['yellow_rows']: ws.cell(row=i, column=j).fill = PatternFill(start_color=CompareConst.YELLOW, end_color=CompareConst.YELLOW, fill_type="solid") - try: - wb.save(file_path) - except Exception as e: - logger.error('Save result file failed') - raise CompareException(CompareException.WRITE_FILE_ERROR) from e - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + + save_workbook(wb, file_path) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index c0319c64d..788d8d6ff 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -4,14 +4,16 @@ from msprobe.core.common.const import FileCheckConst, Const from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.common.utils import create_directory, check_configuration_param, task_dumppath_get, check_compare_param, FileChecker +from msprobe.core.common.utils import create_directory, check_configuration_param, task_dumppath_get, \ + check_compare_param, FileChecker from msprobe.core.common.utils import CompareException + class PTComparator (Comparator): def __init__(self): - self.frame_name=PTComparator.__name__ + self.frame_name = PTComparator.__name__ - def read_npy_data(self,dir_path, file_name): + def read_npy_data(self, dir_path, file_name): data_path = os.path.join(dir_path, file_name) path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.PT_SUFFIX, False) -- Gitee From e9f9906482a6cfda9e97a5899f9fe4ab0411a084 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Tue, 13 Aug 2024 10:59:03 +0800 Subject: [PATCH 305/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E6=BA=A2=E5=87=BA=E6=A3=80=E6=9F=A5=E5=9C=A8=E9=80=9A?= =?UTF-8?q?=E4=BF=A1=E7=AE=97=E5=AD=90=E6=BA=A2=E5=87=BA=E5=90=8E=E6=97=A0?= =?UTF-8?q?=E6=B3=95dump=E4=B8=8Bpt=E6=96=87=E4=BB=B6=E7=9A=84=E9=94=99?= =?UTF-8?q?=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/data_dump/data_collector.py | 7 ++--- .../data_processor/pytorch_processor.py | 28 +++++++++++++------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 2ac077dca..1b2ae513d 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -71,12 +71,11 @@ class DataCollector: backward_name = name.replace(Const.FORWARD, Const.BACKWARD) if self.check_scope_and_pid(self.scope, backward_name, pid): self.data_processor.analyze_pre_forward(backward_name, module, module_input_output) - if not self.is_inplace(module): + if not self.is_inplace(module) or not self.check_scope_and_pid(self.scope, name, pid): return logger.info(f"API {name} is inplace.") - if self.check_scope_and_pid(self.scope, name, pid): - data_info = self.data_processor.analyze_pre_forward_inplace(name, module_input_output) - self.update_data(data_info) + data_info = self.data_processor.analyze_pre_forward_inplace(name, module_input_output) + self.handle_data(name, data_info) def forward_data_collect(self, name, module, pid, module_input_output): self.update_construct(name) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index a58997fa2..3419cbb85 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -1,3 +1,4 @@ +import copy import os import zlib from dataclasses import asdict @@ -5,7 +6,6 @@ from typing import List import numpy as np import torch -from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common.file_check import path_len_exceeds_limit, change_mode from msprobe.core.common.log import logger from msprobe.core.common.const import Const, OverflowConst, FileCheckConst @@ -85,7 +85,7 @@ class PytorchDataProcessor(BaseDataProcessor): tensor_stat.mean = torch._C._VariableFunctionsClass.mean(data_clone).item() tensor_stat.norm = torch._C._VariableFunctionsClass.norm(data_clone).item() return tensor_stat - + @staticmethod def handle_tensor_extremum_nan_inf(tensor, operator): data_clone = tensor.detach() @@ -101,7 +101,7 @@ class PytorchDataProcessor(BaseDataProcessor): data_no_nan = data_clone[~data_nan] return torch._C._VariableFunctionsClass.max(data_no_nan).item() if operator == 'max' else \ torch._C._VariableFunctionsClass.min(data_no_nan).item() - + @staticmethod def _analyze_builtin(arg): single_arg = {} @@ -117,7 +117,7 @@ class PytorchDataProcessor(BaseDataProcessor): single_arg.update({"type": type(arg).__name__}) single_arg.update({"value": arg}) return single_arg - + @staticmethod def _analyze_torch_size(arg): return {"type": "torch.Size", "value": list(arg)} @@ -188,7 +188,8 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): self.bits_for_overflow = 8 self.real_overflow_nums = 0 self.overflow_nums = config.overflow_nums - + self.forward_inplace_inputs = None + @property def is_terminated(self): if self.overflow_nums == -1: @@ -197,12 +198,24 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") return True return False - + @staticmethod def overflow_debug_mode_enable(): overflow_mode = os.getenv(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) return overflow_mode == Const.ENV_ENABLE + def analyze_pre_forward_inplace(self, name, module_input_output: ModuleForwardInputsOutputs): + self.forward_inplace_inputs = copy.deepcopy(module_input_output) + return None + + def analyze_forward_inplace(self, name, module_input_output: ModuleForwardInputsOutputs): + module_input_output.output = module_input_output.concat_args_and_kwargs() + module_input_output.args = self.forward_inplace_inputs.args + module_input_output.kwargs = self.forward_inplace_inputs.kwargs + # release memory used by forward inputs + self.forward_inplace_inputs = None + return self.analyze_forward(name, None, module_input_output) + def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False api_info_struct = super().analyze_forward(name, module, module_input_output) @@ -218,8 +231,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): def maybe_save_overflow_data_and_check_overflow_times(self): if self.has_overflow: for file_path, tensor in self.cached_tensors_and_file_paths.items(): - torch.save(tensor, file_path) - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + save_pt(tensor, file_path) self.real_overflow_nums += 1 self.cached_tensors_and_file_paths = {} -- Gitee From 251bf4ba6b57957df217ac6b83275b76cdee7fb3 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 16 Aug 2024 12:38:46 +0800 Subject: [PATCH 306/791] =?UTF-8?q?compare=E8=B5=84=E6=96=99=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/doc/compare.md | 16 ++++++++-------- .../pytorch/doc/ptdbg_ascend_compare.md | 18 +++++++++--------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md index f252fffe1..802f70267 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md @@ -23,13 +23,13 @@ msprobe精度比对工具主要通过对同一个模型，在两个不同的Mind **完整参数说明** - | 参数名 | 说明 | 是否必选 | - | ------------------ | ------------------------------------------------------------ | -------- | - | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | - | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -a或--auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | 参数名 | 说明 | 是否必选 | + | ------------------ |----------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | + | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | + | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -a或--auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，表示开启，通过配置该参数关闭。 | 否 | + | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | 4. 查看比对结果，请详见PyTorch目录下的《[精度比对工具](../../pytorch/doc/ptdbg_ascend_compare.md)》的“比对结果分析”章节。 @@ -43,7 +43,7 @@ msprobe精度比对工具主要通过对同一个模型，在两个不同的Mind "npu_path": "./npu_dump/dump.json", "bench_path": "./bench_dump/dump.json", "stack_path": "./npu_dump/stack.json", - "is_print_compare_log": True + "is_print_compare_log": true } ``` diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md index e265a6af4..80228bab0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md @@ -36,13 +36,13 @@ **完整参数说明** - | 参数名 | 说明 | 是否必选 | - | ------------------ | ------------------------------------------------------------ | -------- | - | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | - | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -a或--auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | 参数名 | 说明 | 是否必选 | + | ------------------ |----------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | + | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | + | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -a或--auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，表示开启，通过配置该参数关闭。 | 否 | + | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | 3. 查看比对结果，请参见“**比对结果分析**”。 @@ -57,7 +57,7 @@ "npu_path": "./npu_dump/dump.json", "bench_path": "./bench_dump/dump.json", "stack_path": "./npu_dump/stack.json", - "is_print_compare_log": True + "is_print_compare_log": true } ``` @@ -67,7 +67,7 @@ { "npu_path": "./npu_dump/step0", "bench_path": "./bench_dump/step0", - "is_print_compare_log": True + "is_print_compare_log": true } ``` -- Gitee From 1e98472f52bd18f5aa8ac9311bb2ce8f1cc8260b Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 12:54:41 +0800 Subject: [PATCH 307/791] =?UTF-8?q?=E6=90=AC=E8=BF=81=E5=87=BD=E6=95=B0loa?= =?UTF-8?q?d=5Fyaml=E5=88=B0msprobe/core/common/utils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/utils.py | 12 ++++++++++++ .../accuracy_tools/msprobe/pytorch/common/utils.py | 14 +------------- .../msprobe/pytorch/hook_module/utils.py | 2 +- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index e960dc98b..c0c229882 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -23,6 +23,7 @@ import subprocess import time import json import csv +import yaml from datetime import datetime, timezone from pathlib import Path import numpy as np @@ -557,3 +558,14 @@ def save_npy(data, filepath): except Exception as e: raise RuntimeError(f"save npy file {filepath} failed") from e change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + +def load_yaml(yaml_path): + path_checker = FileChecker(yaml_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.YAML_SUFFIX) + checked_path = path_checker.common_check() + try: + with FileOpen(checked_path, "r") as f: + yaml_data = yaml.safe_load(f) + except Exception as e: + raise RuntimeError(f"load yaml file {yaml_path} failed") from e + return yaml_data diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 048f3e07b..3c7465c14 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -18,14 +18,13 @@ import logging import os import random import stat -import yaml import torch import torch.distributed as dist import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create -from msprobe.core.common.file_check import FileChecker, FileOpen, FileCheckConst, change_mode +from msprobe.core.common.file_check import FileCheckConst, change_mode try: @@ -286,17 +285,6 @@ def save_pt(tensor, filepath): change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) -def load_yaml(yaml_path): - path_checker = FileChecker(yaml_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.YAML_SUFFIX) - checked_path = path_checker.common_check() - try: - with FileOpen(checked_path, "r") as f: - yaml_data = yaml.safe_load(f) - except Exception as e: - raise RuntimeError(f"load yaml file {yaml_path} failed") from e - return yaml_data - - def _create_logger(level=logging.INFO): logger_ = logging.getLogger() logger_.setLevel(level) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py index 53814ce12..dd5ae4c29 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py @@ -16,7 +16,7 @@ """ import os -from msprobe.pytorch.common.utils import load_yaml +from msprobe.core.common.utils import load_yaml def get_ops(): -- Gitee From 88128608532dcfe9d4670a78a0d0436a3d225df3 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Fri, 16 Aug 2024 05:03:15 +0000 Subject: [PATCH 308/791] =?UTF-8?q?Update=20debug/accuracy=5Ftools/msprobe?= =?UTF-8?q?/pytorch/hook=5Fmodule/utils.py=20=E5=91=BD=E5=90=8D=E8=A7=84?= =?UTF-8?q?=E8=8C=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py index dd5ae4c29..0c02067b5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py @@ -23,7 +23,7 @@ def get_ops(): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") ops = load_yaml(yaml_path) - WrapFunctionalOps = ops.get('functional') + wrap_functional_ops = ops.get('functional') WrapTensorOps = ops.get('tensor') WrapTorchOps = ops.get('torch') return set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) -- Gitee From eff728a2b0af0e5f6cc39d36cab6b971bf5a17a6 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 13:11:23 +0800 Subject: [PATCH 309/791] =?UTF-8?q?=E8=A7=84=E8=8C=83=E5=91=BD=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py index dd5ae4c29..dbcc9bfa3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py @@ -23,7 +23,7 @@ def get_ops(): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") ops = load_yaml(yaml_path) - WrapFunctionalOps = ops.get('functional') - WrapTensorOps = ops.get('tensor') - WrapTorchOps = ops.get('torch') - return set(WrapFunctionalOps) | set(WrapTensorOps) | set(WrapTorchOps) + wrap_functional = ops.get('functional') + wrap_tensor = ops.get('tensor') + wrap_torch = ops.get('torch') + return set(wrap_functional) | set(wrap_tensor) | set(wrap_torch) -- Gitee From d2ba7e143987607038a41c1921257dd816c9f6d3 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 13:23:14 +0800 Subject: [PATCH 310/791] =?UTF-8?q?=E8=B0=83=E6=95=B4=E5=AF=BC=E5=85=A5?= =?UTF-8?q?=E5=BA=93=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index c0c229882..48c0c3a38 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -23,9 +23,9 @@ import subprocess import time import json import csv -import yaml from datetime import datetime, timezone from pathlib import Path +import yaml import numpy as np from msprobe.core.common.file_check import FileOpen, FileChecker, change_mode -- Gitee From f1554cf53865aed978836e604b585e8468427d75 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Fri, 16 Aug 2024 14:18:25 +0800 Subject: [PATCH 311/791] =?UTF-8?q?=E6=97=A0=E7=94=A8=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cli/analyze_cli.py | 1 - 1 file changed, 1 deletion(-) diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index f400a265b..b90132fd6 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -73,7 +73,6 @@ def analyze_cli(**kwargs): type=click.Choice(constant.SUPPORTED_TORCH_VERSION, case_sensitive=False), default=constant.DEFAULT_TORCH_VERSION, help='The runtime torch version, which can be detected by exec command "pip show torch"') -# @click.option('--is_inference', is_flag=True, help="Enable performance analysis of inference task") @click.option("-pt", "--profiling_type", metavar="", -- Gitee From 79cad6aa04a90c2261eb6eeecbe31e1ee6c22a92 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 14:27:28 +0800 Subject: [PATCH 312/791] =?UTF-8?q?=E5=88=A0=E9=99=A4=E7=A9=BA=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 3c7465c14..ae8823de6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -283,8 +283,8 @@ def save_pt(tensor, filepath): except Exception as e: raise RuntimeError(f"save pt file {filepath} failed") from e change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) - - + + def _create_logger(level=logging.INFO): logger_ = logging.getLogger() logger_.setLevel(level) -- Gitee From e799da5b4a9a9222a92d68f02d49b493adec1974 Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Thu, 15 Aug 2024 16:02:31 +0800 Subject: [PATCH 313/791] update_overall_metrics_sheet_index --- profiler/compare_tools/README.md | 62 +++++------ .../compare_bean/overall_metrics_bean.py | 100 +++++++++++------- .../compare_bean/profiling_info.py | 36 +++++-- .../disaggregate/overall_perf_interface.py | 8 +- .../compare_backend/utils/excel_config.py | 12 ++- 5 files changed, 130 insertions(+), 88 deletions(-) diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index 2de6d2300..535bff4be 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -223,36 +223,38 @@ MindSpore场景仅支持**总体性能**和**通信性能**的对比。 Index列字段说明： -| 字段 | | | 说明 | -| ---------------------------- | ------------------ | ----------------------------------- | ------------------------------------------------------------ | -| Computing Time | | | 计算流耗时，计算流所有event耗时总和。如果有多条并发计算，计算流耗时对重叠部分只会计算一次。
NPU场景下，仅当采集性能数据的Level等级为L1及以上且aic_metrics取值为PipeUtilization时才可拆分出Computing Time的二级字段Flash Attention、Conv等。 | -| | Flash Attention | | Flash Attention算子。 | -| | | Flash Attention (Forward) (Cube) | Flash Attention前向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | -| | | Flash Attention (Forward) (Vector) | Flash Attention前向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | -| | | Flash Attention (Backward) (Cube) | Flash Attention反向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | -| | | Flash Attention (Backward) (Vector) | Flash Attention反向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | -| | Conv | | Conv算子。 | -| | | Conv (Forward) (Cube) | Conv前向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | -| | | Conv (Forward) (Vector) | Conv前向Vector算子。Conv前向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | -| | | Conv (Backward) (Cube) | Conv反向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | -| | | Conv (Backward) (Vector) | Conv反向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | -| | Matmul | | Matmul算子。 | -| | | Matmul (Cube) | Matmul算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | -| | | Matmul (Vector) | Matmul算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | -| | Paged Attention | | Paged Attention算子。 | -| | Vector | | Vector算子。 | -| | | Vector (Trans) | 转换类Vector算子，主要包含Cast、TransPose、TransData算子。（仅针对NPU数据） | -| | | Vector ( No Trans) | 非转换类Vector算子。 | -| | Cube | | 未识别出Flash Attention、Conv和Matmul的Cube算子。 | -| | SDMA (Tensor Move) | | 拷贝类任务。 | -| | Other | | AI CPU、DSA等其他算子。 | -| Uncovered Communication Time | | | 通信未掩盖耗时，包含卡间等待时间。 | -| | Wait | | 卡间同步等待耗时。（仅针对NPU数据） | -| | Transmit | | 通信传输耗时。 | -| Free Time | | | 调度耗时 = E2E耗时 - 算子耗时 - 通信不可掩盖耗时。Free的定义为Device侧既不在通信又不在计算的时间，因此包含拷贝时间（SDMA Time）。 | -| | SDMA | | NPU为除Tensor Move外的拷贝类任务，GPU为所有拷贝类任务。 | -| | Free | | 排除SDMA的空闲耗时。 | -| E2E Time | | | E2E总耗时，计算流端到端耗时。当存在Not minimal profiling时，表示该时间存在性能膨胀，会影响通信和调度耗时。 | +| 字段 | | | 说明 | +| ---------------------------- |:--------------------------| ----------------------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------| +| Computing Time | | | 计算流耗时，计算流所有event耗时总和。如果有多条并发计算，计算流耗时对重叠部分只会计算一次。
NPU场景下，仅当采集性能数据的Level等级为L1及以上且aic_metrics取值为PipeUtilization时才可拆分出Computing Time的二级字段Flash Attention、Conv等。 | +| | Flash Attention (Forward) | | Flash Attention前向算子。 | +| | | Flash Attention (Forward) (Cube) | Flash Attention前向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Flash Attention (Forward) (Vector) | Flash Attention前向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | Flash Attention (Backward)| | Flash Attention反向算子。 | +| | | Flash Attention (Backward) (Cube) | Flash Attention反向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Flash Attention (Backward) (Vector) | Flash Attention反向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | Conv (Forward) | | Conv前向算子。 | +| | | Conv (Forward) (Cube) | Conv前向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Conv (Forward) (Vector) | Conv前向Vector算子。Conv前向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | Conv (Backward) | | Conv反向算子。 | +| | | Conv (Backward) (Cube) | Conv反向算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Conv (Backward) (Vector) | Conv反向算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | Matmul | | Matmul算子。 | +| | | Matmul (Cube) | Matmul算子下发的所有Cube类Kernel的总耗时，一般为执行该算子核心计算的算子。 | +| | | Matmul (Vector) | Matmul算子下发的所有Vector类Kernel的总耗时，一般为插入的转换类算子，如TransData。 | +| | Paged Attention | | Paged Attention算子。 | +| | Vector | | Vector算子。 | +| | | Vector (Trans) | 转换类Vector算子，主要包含Cast、TransPose、TransData算子。（仅针对NPU数据） | +| | | Vector ( No Trans) | 非转换类Vector算子。 | +| | Cube | | 未识别出Flash Attention、Conv和Matmul的Cube算子。 | +| | SDMA (Tensor Move) | | 拷贝类任务。 | +| | Other | | AI CPU、DSA等其他算子。 | +| Uncovered Communication Time | | | 通信未掩盖耗时，包含卡间等待时间。 | +| | Wait | | 卡间同步等待耗时。（仅针对NPU数据） | +| | Transmit | | 通信传输耗时。 | +| Free Time | | | 调度耗时 = E2E耗时 - 算子耗时 - 通信不可掩盖耗时。Free的定义为Device侧既不在通信又不在计算的时间，因此包含拷贝时间（SDMA Time）。 | +| | SDMA | | NPU为除Tensor Move外的拷贝类任务，GPU为所有拷贝类任务。 | +| | Free | | 排除SDMA的空闲耗时。 | +| E2E Time | | | E2E总耗时，计算流端到端耗时。当存在Not minimal profiling时，表示该时间存在性能膨胀，会影响通信和调度耗时。 | 可以采取最简性能数据采集的方式来减少E2E耗时的性能膨胀，示例代码如下： diff --git a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py b/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py index 544f8f523..aec94a0e1 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py @@ -55,14 +55,16 @@ class OverallMetricsInfo: self._profiling_info = profiling_info self._overall_metrics_data_map = { ExcelConfig.COMPUTING: self.computing_data, - ExcelConfig.FA: self.fa_data, + ExcelConfig.FA_FWD: self.fa_fwd_data, ExcelConfig.FA_FWD_CUBE: self.fa_fwd_cube_data, ExcelConfig.FA_FWD_VECTOR: self.fa_fwd_vector_data, + ExcelConfig.FA_BWD: self.fa_bwd_data, ExcelConfig.FA_BWD_CUBE: self.fa_bwd_cube_data, ExcelConfig.FA_BWD_VECTOR: self.fa_bwd_vector_data, - ExcelConfig.CONV: self.conv_data, + ExcelConfig.CONV_FWD: self.conv_fwd_data, ExcelConfig.CONV_FWD_CUBE: self.conv_fwd_cube_data, ExcelConfig.CONV_FWD_VECTOR: self.conv_fwd_vector_data, + ExcelConfig.CONV_BWD: self.conv_bwd_data, ExcelConfig.CONV_BWD_CUBE: self.conv_bwd_cube_data, ExcelConfig.CONV_BWD_VECTOR: self.conv_bwd_vector_data, ExcelConfig.MM: self.mm_data, @@ -84,6 +86,12 @@ class OverallMetricsInfo: ExcelConfig.E2E_TIME: self.e2e_time_data } + @property + def e2e_time(self): + if isclose(self._profiling_info.e2e_time_ms, 0): + raise RuntimeError("Invalid E2E Time.") + return self._profiling_info.e2e_time_ms + @property def overall_metrics(self): return self._overall_metrics_data_map @@ -91,165 +99,179 @@ class OverallMetricsInfo: @property def computing_data(self): return [self._profiling_info.compute_time_ms, - self._profiling_info.compute_time_ms / self._profiling_info.e2e_time_ms, - sum((self._profiling_info.fa_total_num, self._profiling_info.conv_total_num, + self._profiling_info.compute_time_ms / self.e2e_time, + sum((self._profiling_info.fa_fwd_num, self._profiling_info.fa_bwd_num, + self._profiling_info.conv_fwd_num, self._profiling_info.conv_bwd_num, self._profiling_info.mm_total_num, self._profiling_info.vector_total_num, self._profiling_info.sdma_num_tensor_move, self._profiling_info.other_cube_num, self._profiling_info.page_attention_num))] @property - def fa_data(self): - return [self._profiling_info.fa_total_time, - self._profiling_info.fa_total_time / self._profiling_info.e2e_time_ms, - self._profiling_info.fa_total_num] + def fa_fwd_data(self): + return [self._profiling_info.fa_fwd_time, + self._profiling_info.fa_fwd_time / self.e2e_time, + self._profiling_info.fa_fwd_num] + + @property + def fa_bwd_data(self): + return [self._profiling_info.fa_bwd_time, + self._profiling_info.fa_bwd_time / self.e2e_time, + self._profiling_info.fa_bwd_num] @property def fa_fwd_cube_data(self): return [self._profiling_info.fa_time_fwd_cube, - self._profiling_info.fa_time_fwd_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_time_fwd_cube / self.e2e_time, self._profiling_info.fa_num_fwd_cube] @property def fa_fwd_vector_data(self): return [self._profiling_info.fa_time_fwd_vector, - self._profiling_info.fa_time_fwd_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_time_fwd_vector / self.e2e_time, self._profiling_info.fa_num_fwd_vector] @property def fa_bwd_cube_data(self): return [self._profiling_info.fa_time_bwd_cube, - self._profiling_info.fa_time_bwd_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_time_bwd_cube / self.e2e_time, self._profiling_info.fa_num_bwd_cube] @property def fa_bwd_vector_data(self): return [self._profiling_info.fa_time_bwd_vector, - self._profiling_info.fa_time_bwd_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.fa_time_bwd_vector / self.e2e_time, self._profiling_info.fa_num_bwd_vector] @property - def conv_data(self): - return [self._profiling_info.conv_total_time, - self._profiling_info.conv_total_time / self._profiling_info.e2e_time_ms, - self._profiling_info.conv_total_num] + def conv_fwd_data(self): + return [self._profiling_info.conv_fwd_time, + self._profiling_info.conv_fwd_time / self.e2e_time, + self._profiling_info.conv_fwd_num] + + @property + def conv_bwd_data(self): + return [self._profiling_info.conv_bwd_time, + self._profiling_info.conv_bwd_time / self.e2e_time, + self._profiling_info.conv_bwd_num] @property def conv_fwd_cube_data(self): return [self._profiling_info.conv_time_fwd_cube, - self._profiling_info.conv_time_fwd_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_time_fwd_cube / self.e2e_time, self._profiling_info.conv_num_fwd_cube] @property def conv_fwd_vector_data(self): return [self._profiling_info.conv_time_fwd_vector, - self._profiling_info.conv_time_fwd_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_time_fwd_vector / self.e2e_time, self._profiling_info.conv_num_fwd_vector] @property def conv_bwd_cube_data(self): return [self._profiling_info.conv_time_bwd_cube, - self._profiling_info.conv_time_bwd_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_time_bwd_cube / self.e2e_time, self._profiling_info.conv_num_bwd_cube] @property def conv_bwd_vector_data(self): return [self._profiling_info.conv_time_bwd_vector, - self._profiling_info.conv_time_bwd_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.conv_time_bwd_vector / self.e2e_time, self._profiling_info.conv_num_bwd_vector] @property def mm_data(self): return [self._profiling_info.mm_total_time, - self._profiling_info.mm_total_time / self._profiling_info.e2e_time_ms, + self._profiling_info.mm_total_time / self.e2e_time, self._profiling_info.mm_total_num] @property def mm_cube_data(self): return [self._profiling_info.matmul_time_cube, - self._profiling_info.matmul_time_cube / self._profiling_info.e2e_time_ms, + self._profiling_info.matmul_time_cube / self.e2e_time, self._profiling_info.matmul_num_cube] @property def mm_vector_data(self): return [self._profiling_info.matmul_time_vector, - self._profiling_info.matmul_time_vector / self._profiling_info.e2e_time_ms, + self._profiling_info.matmul_time_vector / self.e2e_time, self._profiling_info.matmul_num_vector] @property def pa_data(self): return [self._profiling_info.page_attention_time, - self._profiling_info.page_attention_time / self._profiling_info.e2e_time_ms, + self._profiling_info.page_attention_time / self.e2e_time, self._profiling_info.page_attention_num] @property def vector_data(self): return [self._profiling_info.vector_total_time, - self._profiling_info.vector_total_time / self._profiling_info.e2e_time_ms, + self._profiling_info.vector_total_time / self.e2e_time, self._profiling_info.vector_total_num] @property def vector_trans_data(self): return [self._profiling_info.vector_time_trans, - self._profiling_info.vector_time_trans / self._profiling_info.e2e_time_ms, + self._profiling_info.vector_time_trans / self.e2e_time, self._profiling_info.vector_num_trans] @property def vector_no_trans_data(self): return [self._profiling_info.vector_time_notrans, - self._profiling_info.vector_time_notrans / self._profiling_info.e2e_time_ms, + self._profiling_info.vector_time_notrans / self.e2e_time, self._profiling_info.vector_num_notrans] @property def cube_data(self): return [self._profiling_info.other_cube_time, - self._profiling_info.other_cube_time / self._profiling_info.e2e_time_ms, + self._profiling_info.other_cube_time / self.e2e_time, self._profiling_info.other_cube_num] @property def sdma_tm_data(self): return [self._profiling_info.sdma_time_tensor_move, - self._profiling_info.sdma_time_tensor_move / self._profiling_info.e2e_time_ms, + self._profiling_info.sdma_time_tensor_move / self.e2e_time, self._profiling_info.sdma_num_tensor_move] @property def other_data(self): other_time = max((0, - self._profiling_info.compute_time_ms - self._profiling_info.fa_total_time - - self._profiling_info.conv_total_time - self._profiling_info.mm_total_time - + self._profiling_info.compute_time_ms - self._profiling_info.fa_fwd_time - + self._profiling_info.fa_bwd_time - self._profiling_info.conv_fwd_time - + self._profiling_info.conv_bwd_time - self._profiling_info.mm_total_time - self._profiling_info.vector_total_time - self._profiling_info.sdma_time_tensor_move - self._profiling_info.other_cube_time - self._profiling_info.page_attention_time)) - return [other_time, other_time / self._profiling_info.e2e_time_ms, "/"] + return [other_time, other_time / self.e2e_time, "/"] @property def communication_data(self): return [self._profiling_info.communication_not_overlapped_ms, - self._profiling_info.communication_not_overlapped_ms / self._profiling_info.e2e_time_ms, "/"] + self._profiling_info.communication_not_overlapped_ms / self.e2e_time, "/"] @property def wait_data(self): return [self._profiling_info.wait_time_ms, - self._profiling_info.wait_time_ms / self._profiling_info.e2e_time_ms, "/"] + self._profiling_info.wait_time_ms / self.e2e_time, "/"] @property def transmit_data(self): return [self._profiling_info.transmit_time_ms, - self._profiling_info.transmit_time_ms / self._profiling_info.e2e_time_ms, "/"] + self._profiling_info.transmit_time_ms / self.e2e_time, "/"] @property def free_time_data(self): return [self._profiling_info.free_time_ms, - self._profiling_info.free_time_ms / self._profiling_info.e2e_time_ms, "/"] + self._profiling_info.free_time_ms / self.e2e_time, "/"] @property def sdma_data(self): return [self._profiling_info.sdma_time_stream, - self._profiling_info.sdma_time_stream / self._profiling_info.e2e_time_ms, "/"] + self._profiling_info.sdma_time_stream / self.e2e_time, "/"] @property def free_data(self): free = self._profiling_info.free_time_ms - self._profiling_info.sdma_time_stream - return [free, free / self._profiling_info.e2e_time_ms, "/"] + return [free, free / self.e2e_time, "/"] @property def e2e_time_data(self): - return [self._profiling_info.e2e_time_ms, 1, "/"] + return [self.e2e_time, 1, "/"] diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index c639aba5c..e2891ecc4 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -88,23 +88,36 @@ class ProfilingInfo: return (self.communication_not_overlapped - self.wait_time) * 10 ** 3 @property - def fa_total_time(self): - return sum((self.fa_time_fwd_cube, self.fa_time_fwd_vector, self.fa_time_bwd_cube, self.fa_time_bwd_vector)) + def fa_fwd_time(self): + return self.fa_time_fwd_cube + self.fa_time_fwd_vector @property - def fa_total_num(self): - return sum((self.fa_num_fwd_cube, self.fa_num_fwd_vector, self.fa_num_bwd_cube, self.fa_num_bwd_vector)) + def fa_bwd_time(self): + return self.fa_time_bwd_cube + self.fa_time_bwd_vector @property - def conv_total_time(self): - return sum( - (self.conv_time_fwd_cube, self.conv_time_fwd_vector, self.conv_time_bwd_cube, - self.conv_time_bwd_vector)) + def fa_fwd_num(self): + return self.fa_num_fwd_cube + self.fa_num_fwd_vector + + @property + def fa_bwd_num(self): + return self.fa_num_bwd_cube + self.fa_num_bwd_vector @property - def conv_total_num(self): - return sum((self.conv_num_fwd_cube, self.conv_num_fwd_vector, self.conv_num_bwd_cube, - self.conv_num_bwd_vector)) + def conv_fwd_time(self): + return self.conv_time_fwd_cube + self.conv_time_fwd_vector + + @property + def conv_bwd_time(self): + return self.conv_time_bwd_cube + self.conv_time_bwd_vector + + @property + def conv_fwd_num(self): + return self.conv_num_fwd_cube + self.conv_num_fwd_vector + + @property + def conv_bwd_num(self): + return self.conv_num_bwd_cube + self.conv_num_bwd_vector @property def mm_total_time(self): @@ -186,6 +199,7 @@ class ProfilingInfo: @property def fa_time_bwd(self): return (self.fa_time_bwd_cube + self.fa_time_bwd_vector) / Constant.MILLISECONDS_TO_SECONDS + def calculate_other_time(self): self.other_time = max( [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - diff --git a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py index 65524664e..ca7271f18 100644 --- a/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py +++ b/profiler/compare_tools/compare_backend/disaggregate/overall_perf_interface.py @@ -48,15 +48,15 @@ class OverallPerfInterface: "computing_time_ms": overall_data.compute_time_ms, "uncovered_communication_time_ms": overall_data.communication_not_overlapped_ms, "free_time_ms": overall_data.free_time_ms}, - "computing_time_disaggregate": {"fa_time_ms": overall_data.fa_total_time, - "conv_time_ms": overall_data.conv_total_time, + "computing_time_disaggregate": {"fa_time_ms": overall_data.fa_fwd_time + overall_data.fa_bwd_time, + "conv_time_ms": overall_data.conv_fwd_time + overall_data.conv_bwd_time, "matmul_time_ms": overall_data.mm_total_time, "page_attention_time_ms": overall_data.page_attention_time, "vector_time_ms": overall_data.vector_total_time, "tensor_move_time_ms": overall_data.sdma_time_tensor_move, "other_cube_time_ms": overall_data.other_cube_time}, - "computing_num_disaggregate": {"fa_num": overall_data.fa_total_num, - "conv_num": overall_data.conv_total_num, + "computing_num_disaggregate": {"fa_num": overall_data.fa_fwd_num + overall_data.fa_bwd_num, + "conv_num": overall_data.conv_fwd_num + overall_data.conv_bwd_num, "matmul_num": overall_data.mm_total_num, "page_attention_num": overall_data.page_attention_num, "vector_num": overall_data.vector_total_num, diff --git a/profiler/compare_tools/compare_backend/utils/excel_config.py b/profiler/compare_tools/compare_backend/utils/excel_config.py index b6be0ae2e..975b2a532 100644 --- a/profiler/compare_tools/compare_backend/utils/excel_config.py +++ b/profiler/compare_tools/compare_backend/utils/excel_config.py @@ -249,15 +249,17 @@ class ExcelConfig(object): # computing time COMPUTING = "Computing Time" - FA = "\tFlash Attention" + FA_FWD = "\tFlash Attention (Forward)" FA_FWD_CUBE = "\t\tFlash Attention (Forward) (Cube)" FA_FWD_VECTOR = "\t\tFlash Attention (Forward) (Vector)" + FA_BWD = "\tFlash Attention (Backward)" FA_BWD_CUBE = "\t\tFlash Attention (Backward) (Cube)" FA_BWD_VECTOR = "\t\tFlash Attention (Backward) (Vector)" - CONV = "\tConv" + CONV_FWD = "\tConv (Forward)" CONV_FWD_CUBE = "\t\tConv (Forward) (Cube)" CONV_FWD_VECTOR = "\t\tConv (Forward) (Vector)" + CONV_BWD = "\tConv (Backward)" CONV_BWD_CUBE = "\t\tConv (Backward) (Cube)" CONV_BWD_VECTOR = "\t\tConv (Backward) (Vector)" @@ -293,8 +295,10 @@ class ExcelConfig(object): COMMUNICATION_TIME: CellFormatType.BLUE_NORMAL, FREE_TIME: CellFormatType.BLUE_NORMAL, E2E_TIME: CellFormatType.BLUE_NORMAL, - FA: CellFormatType.LIGHT_BLUE_NORMAL, - CONV: CellFormatType.LIGHT_BLUE_NORMAL, + FA_FWD: CellFormatType.LIGHT_BLUE_NORMAL, + FA_BWD: CellFormatType.LIGHT_BLUE_NORMAL, + CONV_FWD: CellFormatType.LIGHT_BLUE_NORMAL, + CONV_BWD: CellFormatType.LIGHT_BLUE_NORMAL, MM: CellFormatType.LIGHT_BLUE_NORMAL, PA: CellFormatType.LIGHT_BLUE_NORMAL, VECTOR: CellFormatType.LIGHT_BLUE_NORMAL, -- Gitee From bcf11a4d4d2b16ae152d6172f211dc2fc260fbd2 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Fri, 16 Aug 2024 15:38:11 +0800 Subject: [PATCH 314/791] fix write json bug --- .../data_dump/data_processor/mindspore_processor.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index eb76e7bf9..2abb294f6 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -75,16 +75,16 @@ class MindsporeDataProcessor(BaseDataProcessor): return tensor_stat elif data.dtype == ms.bool_: data_np = data.asnumpy() - tensor_stat.max = np.max(data_np) - tensor_stat.min = np.min(data_np) + tensor_stat.max = np.max(data_np).item() + tensor_stat.min = np.min(data_np).item() elif not data.shape: tensor_stat.max = tensor_stat.min = tensor_stat.mean = tensor_stat.norm = data.item() elif data.dtype == ms.complex64 or data.dtype == ms.complex128: data_abs = np.abs(data.asnumpy()) - tensor_stat.max = np.max(data_abs) - tensor_stat.min = np.min(data_abs) - tensor_stat.mean = np.mean(data_abs) - tensor_stat.norm = np.linalg.norm(data_abs) + tensor_stat.max = np.max(data_abs).item() + tensor_stat.min = np.min(data_abs).item() + tensor_stat.mean = np.mean(data_abs).item() + tensor_stat.norm = np.linalg.norm(data_abs).item() else: if data.dtype == ms.bfloat16 or not ops.is_floating_point(data): data = data.to(ms.float32) -- Gitee From 76a3c15b76a6ec3a9f2a0a41abda1a3c4413ed96 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 16:53:35 +0800 Subject: [PATCH 315/791] =?UTF-8?q?=E5=B0=86msprobe=E4=B8=AD=E5=8A=A0?= =?UTF-8?q?=E8=BD=BDyaml=E7=9A=84=E7=A7=81=E6=9C=89=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E9=83=BD=E4=BD=BF=E7=94=A8load=5Fyaml=E6=9B=BF=E6=8D=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dump/hook_cell/wrap_functional.py | 19 ++++++++----------- .../mindspore/dump/hook_cell/wrap_tensor.py | 14 +++++--------- .../free_benchmark/api_pynative_self_check.py | 8 ++++---- .../api_accuracy_checker/common/config.py | 6 ++---- .../compare/compare_utils.py | 19 ++++++++----------- .../msprobe/pytorch/compare/match.py | 6 ++---- .../msprobe/pytorch/hook_module/wrap_aten.py | 13 +++++-------- .../pytorch/hook_module/wrap_distributed.py | 8 +++----- .../pytorch/hook_module/wrap_functional.py | 10 +++------- .../pytorch/hook_module/wrap_npu_custom.py | 10 ++++------ .../pytorch/hook_module/wrap_tensor.py | 10 ++++------ .../msprobe/pytorch/hook_module/wrap_torch.py | 10 ++++------ .../msprobe/pytorch/hook_module/wrap_vf.py | 13 +++++-------- .../pytorch/online_dispatch/dispatch.py | 15 ++++++--------- .../hook_module/test_wrap_tensor.py | 1 - .../pytorch_ut/hook_module/test_wrap_torch.py | 1 - 16 files changed, 63 insertions(+), 100 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py index be3d1bd25..489ca55bb 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py @@ -14,11 +14,9 @@ # ============================================================================ import os -import yaml import mindspore as ms from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell -from msprobe.core.common.utils import Const -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import Const, load_yaml cur_path = os.path.dirname(os.path.realpath(__file__)) @@ -34,15 +32,14 @@ def load_ops_functions(): def get_functional_ops(): ops_func, mint_ops_func, mint_func_ops_func = load_ops_functions() - with FileOpen(yaml_path, 'r') as f: - config = yaml.safe_load(f) - WrapFunctionalOps = config.get("ops") - WrapMintOps = config.get("mint.ops") - WrapMintFunctionalOps = config.get("mint.nn.functional") + config = load_yaml(yaml_path) + wrap_functional = config.get("ops") + wrap_mint = config.get("mint.ops") + wrap_mint_functional = config.get("mint.nn.functional") return ( - set(WrapFunctionalOps) & set(ops_func.keys()), - set(WrapMintOps) & set(mint_ops_func.keys()), - set(WrapMintFunctionalOps) & set(mint_func_ops_func.keys()) + set(wrap_functional) & set(ops_func.keys()), + set(wrap_mint) & set(mint_ops_func.keys()), + set(wrap_mint_functional) & set(mint_func_ops_func.keys()) ) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py index ae6a9a979..7002e8b7d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py @@ -14,17 +14,14 @@ # ============================================================================ import os -import yaml import mindspore as ms - from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell -from msprobe.core.common.utils import Const -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import Const, load_yaml + cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - WrapTensorOps = yaml.safe_load(f).get('tensor') + TensorFunc = {} for f in dir(ms.Tensor): @@ -32,9 +29,9 @@ for f in dir(ms.Tensor): def get_tensor_ops(): - global WrapTensorOps + wrap_tensor_ops = load_yaml(yaml_path) _tensor_ops = dir(ms.Tensor) - return set(WrapTensorOps) & set(_tensor_ops) + return set(wrap_tensor_ops) & set(_tensor_ops) class HOOKTensor(object): @@ -55,7 +52,6 @@ class TensorOPTemplate(HOOKCell): def wrap_tensor_op(op_name, hook): def tensor_op_template(*args, **kwargs): return TensorOPTemplate(op_name, hook)(*args, **kwargs) - return tensor_op_template diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index bcfa31520..510d5044f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -2,14 +2,14 @@ import os import inspect import importlib -import yaml import mindspore as ms from mindspore.communication import comm_func +from msprobe.core.common.utils import load_yaml from msprobe.core.common.const import Const from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.config import Config -from msprobe.core.common.file_check import check_path_length, FileOpen +from msprobe.core.common.file_check import check_path_length from msprobe.mindspore.common.log import logger from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.free_benchmark.decorator.decorator_factory import decorate_forward_function @@ -44,9 +44,9 @@ def get_supported_ops(): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "data", "support_wrap_ops.yaml") + yaml_data = load_yaml(yaml_path) for k, v in FreeBenchmarkConst.API_PREFIX_DICT.items(): - with FileOpen(yaml_path, 'r') as f: - ops = yaml.safe_load(f).get(k) + ops = yaml_data.get(k) if ops: ops = [v + i for i in ops] supported_ops += ops diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index cf8af8d2c..c7bf5947b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -1,15 +1,13 @@ import os -import yaml from msprobe.pytorch.api_accuracy_checker.common.utils import check_file_or_directory_path -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import load_yaml from msprobe.pytorch.pt_config import RunUTConfig class Config: def __init__(self, yaml_file): check_file_or_directory_path(yaml_file, False) - with FileOpen(yaml_file, 'r') as file: - config = yaml.safe_load(file) + config = load_yaml(yaml_file) self.config = {key: self.validate(key, value) for key, value in config.items()} def __getattr__(self, item): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py index 5c7e86ff3..9c986b8ca 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py @@ -4,11 +4,10 @@ import math import numpy as np import torch -import yaml -from msprobe.core.common.utils import CompareException + +from msprobe.core.common.utils import CompareException, load_yaml from msprobe.core.common.const import Const from msprobe.pytorch.common.log import logger -from msprobe.core.common.file_check import FileOpen current_time = time.strftime("%Y%m%d%H%M%S") @@ -22,17 +21,15 @@ BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_C cur_path = os.path.dirname(os.path.realpath(__file__)) standard_yaml_path = os.path.join(cur_path, "api_precision_standard.yaml") -with FileOpen(standard_yaml_path, 'r') as f: - Apis = yaml.safe_load(f) - AbsoluteStandardApi = Apis.get('AbsoluteThreshStandard') - BinaryStandardApi = Apis.get('BinaryCompareStandard') - ULPStandardApi = Apis.get('ULPStandard') - ThousandthStandardApi = Apis.get('ThousandthStandard') +apis = load_yaml(standard_yaml_path) +AbsoluteStandardApi = apis.get('AbsoluteThreshStandard') +BinaryStandardApi = apis.get('BinaryCompareStandard') +ULPStandardApi = apis.get('ULPStandard') +ThousandthStandardApi = apis.get('ThousandthStandard') threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml") -with FileOpen(threshold_yaml_path, 'r') as f: - apis_threshold = yaml.safe_load(f) +apis_threshold = load_yaml(threshold_yaml_path) DETAIL_TEST_ROWS = [[ diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/match.py b/debug/accuracy_tools/msprobe/pytorch/compare/match.py index 2a46105bd..5958697f2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/match.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/match.py @@ -1,15 +1,13 @@ import os -import yaml from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import CompareException +from msprobe.core.common.utils import CompareException, load_yaml class AtenIrMapping(): def __init__(self): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "mapping.yaml") - with FileOpen(yaml_path, 'r') as f: - self.aten_mapping = yaml.safe_load(f) + self.aten_mapping = load_yaml(yaml_path) def match(self, op1, op2): if "Aten" in op1 and "Aten" not in op2: diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py index a02abbe5f..16b5a4ef4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py @@ -18,20 +18,17 @@ import os import torch -import yaml - from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import load_yaml from msprobe.pytorch.function_factory import npu_custom_grad_functions cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - Ops = yaml.safe_load(f) - WrapAtenOps = Ops.get('aten') - WhiteAtenOps = Ops.get('white_aten_ops', []) +ops = load_yaml(yaml_path) +wrap_aten_ops = ops.get('aten') +white_aten_ops = ops.get('white_aten_ops', []) aten_func = {} @@ -69,7 +66,7 @@ class AtenOPTemplate(HOOKModule): if isinstance(self.op, str): if self.op in npu_custom_grad_functions: return npu_custom_grad_functions[self.op](*args, **kwargs) - if self.op in WhiteAtenOps: + if self.op in white_aten_ops: return eval(f"torch.ops.aten.{self.op}")(*args, **kwargs) if self.op not in aten_func: raise Exception(f"Skip op[{self.op}] accuracy check, because the op is not " diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py index 3ca1db0f5..512c1a1ae 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py @@ -18,18 +18,16 @@ import os from functools import wraps import torch.distributed as dist -import yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import load_yaml cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - WrapDistributedOps = yaml.safe_load(f).get('distributed') distributed_func = {} @@ -38,9 +36,9 @@ for f in dir(dist): def get_distributed_ops(): - global WrapDistributedOps _all_distributed_ops = dir(dist) - return set(WrapDistributedOps) & set(_all_distributed_ops) + wrap_distributed_ops = load_yaml(yaml_path) + return set(wrap_distributed_ops) & set(_all_distributed_ops) class HOOKDistributedOP(object): diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py index fd7610ca8..f92acac47 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py @@ -16,15 +16,13 @@ """ import os - import torch -import yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const from msprobe.pytorch.common.log import logger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import load_yaml def remove_dropout(): @@ -66,14 +64,12 @@ def remove_dropout(): cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - WrapFunctionalOps = yaml.safe_load(f).get('functional') def get_functional_ops(): - global WrapFunctionalOps + wrap_functional_ops = load_yaml(yaml_path).get('functional') _all_functional_ops = dir(torch.nn.functional) - return set(WrapFunctionalOps) & set(_all_functional_ops) + return set(wrap_functional_ops) & set(_all_functional_ops) TorchFunctions = {func: getattr(torch.nn.functional, func) for func in get_functional_ops()} diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index 8a67ed942..747b165a4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -17,18 +17,16 @@ import os import torch -import yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard, torch_without_guard_version from msprobe.core.common.const import Const -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import load_yaml from msprobe.pytorch.function_factory import npu_custom_functions cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - WrapNpuOps = yaml.safe_load(f).get('torch_npu') + try: import torch_npu @@ -39,12 +37,12 @@ else: def get_npu_ops(): - global WrapNpuOps if torch_without_guard_version: _npu_ops = dir(torch.ops.npu) else: _npu_ops = dir(torch_npu._C._VariableFunctionsClass) - return set(WrapNpuOps) & set(_npu_ops) + wrap_npu_ops = load_yaml(yaml_path).get('torch_npu') + return set(wrap_npu_ops) & set(_npu_ops) class HOOKNpuOP(object): diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py index 3e26ae3be..d3a5c399b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py @@ -18,23 +18,21 @@ import os import torch -import yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard, parameter_adapter from msprobe.core.common.const import Const -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import load_yaml + cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - WrapTensorOps = yaml.safe_load(f).get('tensor') def get_tensor_ops(): - global WrapTensorOps _tensor_ops = dir(torch.Tensor) - return set(WrapTensorOps) & set(_tensor_ops) + wrap_tensor_ops = load_yaml(yaml_path).get('tensor') + return set(wrap_tensor_ops) & set(_tensor_ops) TensorOps = {op: getattr(torch.Tensor, op) for op in get_tensor_ops()} diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py index 486ddda49..92f4bbca0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py @@ -16,25 +16,23 @@ """ import os - import torch -import yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import load_yaml + cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - WrapTorchOps = yaml.safe_load(f).get('torch') def get_torch_ops(): - global WrapTorchOps _torch_ops = [] - for operation in WrapTorchOps: + wrap_torch_ops = load_yaml(yaml_path).get('torch') + for operation in wrap_torch_ops: if '.' in operation: operation_sub_module_name, operation_sub_op = operation.rsplit('.', 1) operation_sub_module = getattr(torch, operation_sub_module_name) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py index d78beb2a6..21e0ebfdc 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py @@ -16,24 +16,21 @@ """ import os - import torch -import yaml +from msprobe.core.common.const import Const +from msprobe.core.common.utils import load_yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule -from msprobe.core.common.file_check import FileOpen from msprobe.pytorch.common.utils import torch_device_guard -from msprobe.core.common.const import Const + cur_path = os.path.dirname(os.path.realpath(__file__)) yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") -with FileOpen(yaml_path, 'r') as f: - WrapVfOps = yaml.safe_load(f).get('_VF') def get_vf_ops(): - global WrapVfOps - return WrapVfOps + wrap_vf_ops = load_yaml(yaml_path).get('_VF') + return wrap_vf_ops class HOOKVfOP(object): diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index bf5cf1194..b38f6613a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -4,7 +4,6 @@ import json from pathlib import Path from multiprocessing import Manager, Pool -import yaml import torch from torch.utils._python_dispatch import TorchDispatchMode @@ -21,8 +20,7 @@ from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, from .utils import get_callstack, data_to_cpu, logger_debug, logger_error, logger_warn, logger_logo, get_sys_info, \ DispatchException from .compare import Comparator -from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create +from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, load_yaml from msprobe.core.common.const import Const, CompareConst current_time = time.strftime("%Y%m%d%H%M%S") @@ -70,7 +68,7 @@ class PtdbgDispatch(TorchDispatchMode): self.aten_ops_blacklist = [] self.npu_adjust_autogard = [] yaml_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "torch_ops_config.yaml") - self.load_yaml_file(yaml_path) + self.get_ops(yaml_path) self.lock = None if process_num > 0: @@ -214,11 +212,10 @@ class PtdbgDispatch(TorchDispatchMode): dir_name = f'msprobe_{tag}_rank{self.device_id}_{time_now}' return dir_name - def load_yaml_file(self, file_path): - with FileOpen(file_path, 'r') as f: - yaml_file = yaml.safe_load(f) - self.aten_ops_blacklist = yaml_file.get('aten_ops_blacklist') - self.npu_adjust_autogard = yaml_file.get('npu_adjust_autogard') + def get_ops(self, file_path): + yaml_file = load_yaml(file_path) + self.aten_ops_blacklist = yaml_file.get('aten_ops_blacklist') + self.npu_adjust_autogard = yaml_file.get('npu_adjust_autogard') def filter_dump_api(self): if self.dump_mode != Const.LIST or not self.dump_api_list: diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py index 2aadc358a..6868c5bda 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_tensor.py @@ -1,6 +1,5 @@ import unittest import torch -import yaml from msprobe.pytorch.hook_module.wrap_tensor import get_tensor_ops, HOOKTensor, TensorOPTemplate, wrap_tensor_op, wrap_tensor_ops_and_bind class TestWrapTensor(unittest.TestCase): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py index 14b156e3b..e0e4d000c 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/hook_module/test_wrap_torch.py @@ -1,6 +1,5 @@ import unittest import torch -import yaml from msprobe.pytorch.hook_module.wrap_torch import * class TestWrapTorch(unittest.TestCase): -- Gitee From 177d8162b87968e4c50c75d63d7e4d375057aefc Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 17:25:09 +0800 Subject: [PATCH 316/791] =?UTF-8?q?=E6=A3=80=E6=9F=A5=E5=8F=98=E9=87=8F?= =?UTF-8?q?=E5=90=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py | 4 ++-- .../msprobe/pytorch/online_dispatch/dispatch.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py index 16b5a4ef4..b63bd9d02 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py @@ -37,9 +37,9 @@ for f in dir(torch.ops.aten): def get_aten_ops(): - global WrapAtenOps + global wrap_aten_ops _all_aten_ops = dir(torch.ops.aten) - return set(WrapAtenOps) & set(_all_aten_ops) + return set(wrap_aten_ops) & set(_all_aten_ops) class HOOKAtenOP(object): diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index b38f6613a..aaae8640c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -15,13 +15,14 @@ except ImportError: else: is_npu = True +from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, load_yaml +from msprobe.core.common.const import Const, CompareConst from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ DispatchRunParam, DisPatchDataInfo from .utils import get_callstack, data_to_cpu, logger_debug, logger_error, logger_warn, logger_logo, get_sys_info, \ DispatchException from .compare import Comparator -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, load_yaml -from msprobe.core.common.const import Const, CompareConst + current_time = time.strftime("%Y%m%d%H%M%S") RESULT_FILE_NAME = "accuracy_checking_result_" + current_time + ".csv" -- Gitee From bc7087e61b6a5b83a8d578ab3d65c149558f9bd2 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 16 Aug 2024 17:41:04 +0800 Subject: [PATCH 317/791] api unmatch nan changed to None --- .../msprobe/core/compare/utils.py | 18 +++++++++++------- debug/accuracy_tools/msprobe/msprobe.py | 2 +- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index ff1974f51..f905ea91f 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -326,9 +326,9 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): index_out = 0 npu_stack_info = n_dict.get("stack_info", None) - bench_name, bench_type, bench_shape = CompareConst.NAN, CompareConst.NAN, CompareConst.NAN + bench_name, bench_type, bench_shape = CompareConst.NONE, CompareConst.NONE, CompareConst.NONE err_msg = CompareConst.NO_BENCH - accuracy_check_res = CompareConst.NAN + accuracy_check_res = CompareConst.NONE for index, n_name in enumerate(n_dict["op_name"]): if n_name.find("input") != -1: n_struct = n_dict["input_struct"][index] @@ -338,24 +338,28 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): result_item = [n_name, bench_name, n_struct[0], bench_type, n_struct[1], bench_shape] if md5_compare: - result_item.extend([CompareConst.NAN] * 3) + result_item.extend([CompareConst.NONE] * 3) if npu_stack_info and index == 0: result_item.extend(npu_stack_info) + else: + result_item.append(CompareConst.NONE) result.append(result_item) continue if summary_compare: - result_item.extend([CompareConst.NAN] * 8) + result_item.extend([CompareConst.NONE] * 8) else: - result_item.extend([CompareConst.NAN] * 5) + result_item.extend([CompareConst.NONE] * 5) summary_data = n_dict.get("summary")[index] result_item.extend(summary_data) - summary_data = [CompareConst.NAN] * 4 + summary_data = [CompareConst.NONE] * 4 result_item.extend(summary_data) result_item.append(accuracy_check_res) result_item.append(err_msg) if npu_stack_info and index == 0: result_item.extend(npu_stack_info) - if not md5_compare and not summary_compare and result_item[1] == CompareConst.NAN: + else: + result_item.append(CompareConst.NONE) + if not md5_compare and not summary_compare and result_item[1] == CompareConst.NONE: if index == 0: result_item.extend(["-1"]) else: diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 97f90a64d..dd80d89c3 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -44,7 +44,7 @@ def main(): api_precision_compare_cmd_parser = subparsers.add_parser('api_precision_compare') run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check') _compare_parser(compare_cmd_parser) - is_torch_available=is_module_available("torch") + is_torch_available = is_module_available("torch") if is_torch_available: from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command from msprobe.pytorch.parse_tool.cli import parse as cli_parse -- Gitee From 6b5a7ac506378bf865a30a05ce0702590f1b02df Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 19:31:09 +0800 Subject: [PATCH 318/791] =?UTF-8?q?=E6=A3=80=E6=9F=A5=E5=8F=98=E9=87=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/dump/hook_cell/wrap_tensor.py | 2 +- .../compare/api_precision_compare.py | 10 +++++----- .../pytorch/api_accuracy_checker/compare/compare.py | 12 ++++++------ .../api_accuracy_checker/compare/compare_utils.py | 8 ++++---- .../msprobe/pytorch/hook_module/wrap_distributed.py | 3 +-- 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py index 7002e8b7d..a9facd218 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py @@ -29,7 +29,7 @@ for f in dir(ms.Tensor): def get_tensor_ops(): - wrap_tensor_ops = load_yaml(yaml_path) + wrap_tensor_ops = load_yaml(yaml_path).get('tensor') _tensor_ops = dir(ms.Tensor) return set(wrap_tensor_ops) & set(_tensor_ops) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 73bf7c2b8..312fe5935 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -11,7 +11,7 @@ from msprobe.pytorch.api_accuracy_checker.common.utils import write_csv from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ - ApiPrecisionCompareColumn, AbsoluteStandardApi, BinaryStandardApi, ULPStandardApi, ThousandthStandardApi, \ + ApiPrecisionCompareColumn, absolute_standard_api, binary_standard_api, ulp_standard_api, thousandth_standard_api, \ BINARY_COMPARE_UNSUPPORT_LIST, ULP_COMPARE_SUPPORT_LIST, convert_str_to_float, CompareMessage, is_inf_or_nan, \ check_inf_or_nan from msprobe.pytorch.api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn @@ -315,14 +315,14 @@ def analyse_csv(npu_data, gpu_data, config): write_detail_csv(compare_column.to_column_value(), config.details_csv_path) else: compare_column.api_name = full_api_name_with_direction_status - if api_name in ThousandthStandardApi: + if api_name in thousandth_standard_api: new_status = record_thousandth_threshold_result(compare_column, row_npu) elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or \ - api_name in BinaryStandardApi: + api_name in binary_standard_api: new_status = record_binary_consistency_result(api_name, compare_column, row_npu) - elif api_name in AbsoluteStandardApi: + elif api_name in absolute_standard_api: new_status = record_absolute_threshold_result(compare_column, row_npu) - elif api_name in ULPStandardApi and \ + elif api_name in ulp_standard_api and \ row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in ULP_COMPARE_SUPPORT_LIST: us = ULPStandard(full_api_name_with_direction_status, row_npu, row_gpu) new_status = record_ulp_compare_result(compare_column, us) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 20f04b0cd..155a02d59 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -6,8 +6,8 @@ import numpy as np from msprobe.pytorch.common.log import logger from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents, write_csv from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \ - DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, AbsoluteStandardApi, BinaryStandardApi, \ - ULPStandardApi, ThousandthStandardApi, apis_threshold + DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, absolute_standard_api, binary_standard_api, \ + ulp_standard_api, thousandth_standard_api, apis_threshold from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, \ get_mean_rel_err, get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ @@ -280,15 +280,15 @@ class Comparator: abs_bench, abs_bench_with_eps = get_abs_bench_with_eps(bench_output, dtype) abs_err = get_abs_err(bench_output, device_output) rel_err_orign = get_rel_err_origin(abs_err, abs_bench_with_eps) - if api_name in ThousandthStandardApi: + if api_name in thousandth_standard_api: thousand_res, thousand_status = get_rel_err_ratio(rel_err_orign, CompareConst.THOUSAND_RATIO_THRESHOLD) compare_column.rel_err_thousandth = thousand_res if str(dtype) in BENCHMARK_COMPARE_SUPPORT_LIST: both_finite_mask, inf_nan_mask = get_finite_and_infinite_mask(bench_output, device_output) - if api_name in BinaryStandardApi: + if api_name in binary_standard_api: err_rate, _, _ = self._compare_bool_tensor(bench_output, device_output) compare_column.error_rate = err_rate - elif api_name in AbsoluteStandardApi: + elif api_name in absolute_standard_api: small_value_threshold, small_value_atol, rtol = self._get_absolute_threshold_attribute( api_name, str(dtype)) rel_err = abs_err / abs_bench_with_eps @@ -298,7 +298,7 @@ class Comparator: dtype, rtol) compare_column.rel_err_ratio = check_norm_value(normal_value_mask, rel_err, rtol) compare_column.abs_err_ratio = check_small_value(abs_err, small_value_mask, small_value_atol) - elif api_name in ULPStandardApi: + elif api_name in ulp_standard_api: if bench_output.size == 0: compare_column.max_ulp_error = 0 compare_column.mean_ulp_error = 0 diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py index 9c986b8ca..00f002c17 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py @@ -22,10 +22,10 @@ BINARY_COMPARE_UNSUPPORT_LIST = BENCHMARK_COMPARE_SUPPORT_LIST + API_PRECISION_C cur_path = os.path.dirname(os.path.realpath(__file__)) standard_yaml_path = os.path.join(cur_path, "api_precision_standard.yaml") apis = load_yaml(standard_yaml_path) -AbsoluteStandardApi = apis.get('AbsoluteThreshStandard') -BinaryStandardApi = apis.get('BinaryCompareStandard') -ULPStandardApi = apis.get('ULPStandard') -ThousandthStandardApi = apis.get('ThousandthStandard') +absolute_standard_api = apis.get('AbsoluteThreshStandard') +binary_standard_api = apis.get('BinaryCompareStandard') +ulp_standard_api = apis.get('ULPStandard') +thousandth_standard_api = apis.get('ThousandthStandard') threshold_yaml_path = os.path.join(cur_path, "api_precision_threshold.yaml") diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py index 512c1a1ae..88a8526ad 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py @@ -22,7 +22,6 @@ import torch.distributed as dist from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const -from msprobe.core.common.file_check import FileOpen from msprobe.core.common.utils import load_yaml @@ -37,7 +36,7 @@ for f in dir(dist): def get_distributed_ops(): _all_distributed_ops = dir(dist) - wrap_distributed_ops = load_yaml(yaml_path) + wrap_distributed_ops = load_yaml(yaml_path).get('distributed') return set(wrap_distributed_ops) & set(_all_distributed_ops) -- Gitee From b45b8f1579b0c5fcbf1108c4e33a111b6ea7be50 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 19:42:28 +0800 Subject: [PATCH 319/791] del spaces --- .../pytorch/api_accuracy_checker/compare/compare_utils.py | 1 - debug/accuracy_tools/msprobe/pytorch/compare/match.py | 1 - debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py | 1 - 3 files changed, 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py index 00f002c17..4c2b921e1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py @@ -2,7 +2,6 @@ import time import os import math -import numpy as np import torch from msprobe.core.common.utils import CompareException, load_yaml diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/match.py b/debug/accuracy_tools/msprobe/pytorch/compare/match.py index 5958697f2..ac445ad8e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/match.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/match.py @@ -1,5 +1,4 @@ import os -from msprobe.core.common.file_check import FileOpen from msprobe.core.common.utils import CompareException, load_yaml diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py index 92f4bbca0..8aad3daff 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py @@ -21,7 +21,6 @@ import torch from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const -from msprobe.core.common.file_check import FileOpen from msprobe.core.common.utils import load_yaml -- Gitee From f21e534139372325c3031d4395f2c2d9d9d1e06d Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 16 Aug 2024 20:42:09 +0800 Subject: [PATCH 320/791] del spaces --- debug/accuracy_tools/msprobe/core/common/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 48c0c3a38..25c7ead8b 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -18,7 +18,6 @@ import collections import os import re import shutil -import stat import subprocess import time import json @@ -29,7 +28,7 @@ import yaml import numpy as np from msprobe.core.common.file_check import FileOpen, FileChecker, change_mode -from msprobe.core.common.const import Const, FileCheckConst, CompareConst, OverflowConst +from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.core.common.log import logger -- Gitee From 863bbbc4b78cd3355a187c070b94a14aeb266d39 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 17 Aug 2024 10:10:21 +0800 Subject: [PATCH 321/791] =?UTF-8?q?=E8=BF=87=E6=BB=A4=E6=B6=88=E6=81=AF?= =?UTF-8?q?=E4=B8=AD=E7=9A=84=E7=89=B9=E6=AE=8A=E5=AD=97=E7=AC=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/const.py | 7 +++++++ debug/accuracy_tools/msprobe/core/common/log.py | 15 ++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 333757082..90fcadc7d 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -257,3 +257,10 @@ class OverflowConst: OVERFLOW_DEBUG_MODE_ENABLE = "OVERFLOW_DEBUG_MODE_ENABLE" OVERFLOW_ORIGINAL_MODE = 0 OVERFLOW_DEBUG_MODE = 1 + + +class MsgConst: + """ + Class for log messages const + """ + SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index f31dad64d..637d00303 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -1,6 +1,8 @@ import os import time import sys +from msprobe.core.common.const import MsgConst + class BaseLogger: def __init__(self): @@ -20,12 +22,23 @@ class BaseLogger: def get_rank(self): return self.rank + def filter_special_chars(func): + def wrapper(self, msg): + if any(char in msg for char in MsgConst.SPECIAL_CHAR): + for char in MsgConst.SPECIAL_CHAR: + msg = msg.replace(char, '_') + return func(self, msg) + return wrapper + + @filter_special_chars def info(self, msg): self._print_log(self.info_level, msg) + @filter_special_chars def error(self, msg): self._print_log(self.error_level, msg) + @filter_special_chars def warning(self, msg): self._print_log(self.warning_level, msg) @@ -52,4 +65,4 @@ class BaseLogger: raise exception -logger = BaseLogger() \ No newline at end of file +logger = BaseLogger() -- Gitee From feede96289bba36bb318a4742f794b9f33993da6 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 17 Aug 2024 10:41:13 +0800 Subject: [PATCH 322/791] check --- debug/accuracy_tools/msprobe/core/data_dump/data_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 2ac077dca..aecf6095f 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -128,8 +128,8 @@ class DataCollector: self.data_writer.update_construct(self.module_processor.module_node) def handle_data(self, name, data_info, use_buffer=True): - msg = f"msprobe is collecting data on {name}. " if data_info: + msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) logger.info(msg) if use_buffer: -- Gitee From 40730854f044c0600a161b5710f31f3fdd4022af Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Fri, 16 Aug 2024 15:33:02 +0800 Subject: [PATCH 323/791] change kbyk dump level --- .../msprobe/core/common/const.py | 8 +- .../msprobe/mindspore/common/const.py | 12 +-- .../mindspore/debugger/debugger_config.py | 3 +- .../mindspore/dump/dump_tool_factory.py | 41 +++++----- .../{api_kbk_dump.py => kernel_kbyk_dump.py} | 74 +++++++++++-------- .../overflow_check_tool_factory.py | 32 ++++---- .../msprobe/mindspore/task_handler_factory.py | 5 +- .../mindspore_ut/test_dump_tool_factory.py | 21 ++++-- .../mindspore_ut/test_kernel_graph_dump.py | 4 +- .../test_kernel_graph_overflow_check.py | 4 +- ...i_kbk_dump.py => test_kernel_kbyk_dump.py} | 22 +++--- .../test_overflow_check_tool_factory.py | 12 ++- .../test/mindspore_ut/test_primitive_dump.py | 18 ++--- .../mindspore_ut/test_task_handler_factory.py | 6 +- 14 files changed, 141 insertions(+), 121 deletions(-) rename debug/accuracy_tools/msprobe/mindspore/dump/{api_kbk_dump.py => kernel_kbyk_dump.py} (31%) rename debug/accuracy_tools/msprobe/test/mindspore_ut/{test_api_kbk_dump.py => test_kernel_kbyk_dump.py} (67%) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 333757082..d169eeaa7 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -64,14 +64,18 @@ class Const: ENV_ENABLE = "1" ENV_DISABLE = "0" MAX_SEED_VALUE = 4294967295 # 2**32 - 1 - TASK_LIST = ["tensor", "statistics", "overflow_check", "free_benchmark", "run_ut", "grad_probe"] - LEVEL_LIST = ["L0", "L1", "L2", "mix"] STATISTICS = "statistics" TENSOR = "tensor" OVERFLOW_CHECK = "overflow_check" FREE_BENCHMARK = "free_benchmark" RUN_UT = "run_ut" GRAD_PROBE = "grad_probe" + TASK_LIST = [TENSOR, STATISTICS, OVERFLOW_CHECK, FREE_BENCHMARK, RUN_UT, GRAD_PROBE] + LEVEL_L0 = "L0" + LEVEL_L1 = "L1" + LEVEL_L2 = "L2" + LEVEL_MIX = "mix" + LEVEL_LIST = [LEVEL_L0, LEVEL_L1, LEVEL_L2, LEVEL_MIX] ATTR_NAME_PREFIX = "wrap_" ATTR_NAME_PREFIX_LEN = len(ATTR_NAME_PREFIX) KERNEL_DUMP = "kernel_dump" diff --git a/debug/accuracy_tools/msprobe/mindspore/common/const.py b/debug/accuracy_tools/msprobe/mindspore/common/const.py index 08bb97649..454b48372 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/const.py @@ -1,15 +1,17 @@ import numpy as np import mindspore as ms +from msprobe.core.common.const import Const as CoreCost + class Const: CELL = "cell" API = "api" KERNEL = "kernel" TOOL_LEVEL_DICT = { - "L0": CELL, - "L1": API, - "L2": KERNEL + CoreCost.LEVEL_L0: CELL, + CoreCost.LEVEL_L1: API, + CoreCost.LEVEL_L2: KERNEL } PYNATIVE_MODE = "pynative" GRAPH_GE_MODE = "graph_ge" @@ -19,7 +21,7 @@ class Const: class FreeBenchmarkConst: DEFAULT_DEVICE = "npu" DEFAULT_STAGE = "forward" - DEFAULT_DUMP_LEVEL = "L1" + DEFAULT_DUMP_LEVEL = CoreCost.LEVEL_L1 DEFAULT_PERT_TYPE = "improve_precision" DEFAULT_HANDLER_TYPE = "check" FIX_HANDLER_MODE = "fix" @@ -31,7 +33,7 @@ class FreeBenchmarkConst: FIX = "fix" DEVICE_LIST = ["npu"] STAGE_LIST = ["forward"] - DUMP_LEVEL_LIST = ["L1"] + DUMP_LEVEL_LIST = [CoreCost.LEVEL_L1] PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE] HANDLER_TYPE_LIST = [CHECK, FIX] COMMUNICATION_API_LIST = [ diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 54f640703..78dc253fa 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -14,8 +14,7 @@ class DebuggerConfig: self.task = common_config.task self.rank = [] if not common_config.rank else common_config.rank self.step = [] if not common_config.step else common_config.step - if not common_config.level: - common_config.level = "L1" + common_config.level = Const.LEVEL_L1 if not common_config.level else common_config.level self.level = MsConst.TOOL_LEVEL_DICT.get(common_config.level, MsConst.API) self.level_ori = common_config.level self.list = [] if not task_config.list else task_config.list diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py index 2c4579b0e..1e4b06a38 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py @@ -1,24 +1,25 @@ +from msprobe.mindspore.common.const import Const from msprobe.mindspore.debugger.debugger_config import DebuggerConfig -from msprobe.mindspore.dump.api_kbk_dump import ApiKbkDump +from msprobe.mindspore.dump.kernel_kbyk_dump import KernelKbykDump from msprobe.mindspore.dump.kernel_graph_dump import KernelGraphDump class DumpToolFactory: tools = { - "cell": { - "kbk": None, - "graph": None, - "pynative": None + Const.CELL: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: None }, - "api": { - "kbk": ApiKbkDump, - "graph": None, - "pynative": None + Const.API: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: None }, - "kernel": { - "kbk": None, - "graph": KernelGraphDump, - "pynative": None + Const.KERNEL: { + Const.GRAPH_KBYK_MODE: KernelKbykDump, + Const.GRAPH_GE_MODE: KernelGraphDump, + Const.PYNATIVE_MODE: None } } @@ -26,13 +27,9 @@ class DumpToolFactory: def create(config: DebuggerConfig): tool = DumpToolFactory.tools.get(config.level) if not tool: - raise Exception("valid level is needed.") - if config.level == "api": - tool = tool.get("kbk") - elif config.level == "kernel": - tool = tool.get("graph") - elif config.level == "cell": - raise Exception("Cell dump in not supported now.") + raise Exception("Valid level is needed.") + tool = tool.get(config.execution_mode) if not tool: - raise Exception("Data dump in not supported in this mode.") - return tool(config) \ No newline at end of file + raise Exception(f"Data dump is not supported in {config.execution_mode} mode " + f"when dump level is {config.level}.") + return tool(config) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/api_kbk_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py similarity index 31% rename from debug/accuracy_tools/msprobe/mindspore/dump/api_kbk_dump.py rename to debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py index 5c7af45d7..54e57c508 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/api_kbk_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py @@ -1,55 +1,65 @@ import os import json + from msprobe.core.common.utils import make_dump_path_if_not_exists from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.const import Const + +class KernelKbykDump: + COMMON_SETTINGS = "common_dump_settings" + E2E_SETTINGS = "e2e_dump_settings" -class ApiKbkDump: def __init__(self, config: DebuggerConfig): self.dump_json = dict() - self.dump_json["common_dump_settings"] = dict() - self.dump_json["common_dump_settings"]["dump_mode"] = 0 - self.dump_json["common_dump_settings"]["path"] = "" - self.dump_json["common_dump_settings"]["net_name"] = "Net" - self.dump_json["common_dump_settings"]["iteration"] = "all" - self.dump_json["common_dump_settings"]["saved_data"] = "statistic" - self.dump_json["common_dump_settings"]["input_output"] = 0 - self.dump_json["common_dump_settings"]["kernels"] = [] - self.dump_json["common_dump_settings"]["support_device"] = [0,1,2,3,4,5,6,7] - self.dump_json["e2e_dump_settings"] = dict() - self.dump_json["e2e_dump_settings"]["enable"] = True - self.dump_json["e2e_dump_settings"]["trans_flag"] = True - - - if len(config.list) > 0: - self.dump_json["common_dump_settings"]["dump_mode"] = 1 - self.dump_json["common_dump_settings"]["kernels"] = config.list - self.dump_json["common_dump_settings"]["path"] = config.dump_path - if len(config.step) > 0: + common_set = dict() + e2e_set = dict() + + common_set = dict() + common_set["dump_mode"] = 0 + common_set["path"] = "" + common_set["net_name"] = "Net" + common_set["iteration"] = "all" + common_set["saved_data"] = "statistic" + common_set["input_output"] = 0 + common_set["kernels"] = [] + common_set["support_device"] = [0, 1, 2, 3, 4, 5, 6, 7] + e2e_set = dict() + e2e_set["enable"] = True + e2e_set["trans_flag"] = True + + if config.list: + common_set["dump_mode"] = 1 + common_set["kernels"] = config.list + common_set["path"] = config.dump_path + if config.step: step_str = "" for s in config.step: step_str += (str(s) + '|') - self.dump_json["common_dump_settings"]["iteration"] = step_str[:-1] - if len(config.rank) > 0: - self.dump_json["common_dump_settings"]["support_device"] = config.rank - if config.task == "tensor": - self.dump_json["common_dump_settings"]["saved_data"] = "tensor" + common_set["iteration"] = step_str[:-1] + if config.rank: + common_set["support_device"] = config.rank + if config.task == Const.TENSOR: + common_set["saved_data"] = Const.TENSOR if len(config.data_mode) == 1: - if config.data_mode[0] == "input": - self.dump_json["common_dump_settings"]["input_output"] = 1 - if config.data_mode[0] == "output": - self.dump_json["common_dump_settings"]["input_output"] = 2 + if config.data_mode[0] == Const.INPUT: + common_set["input_output"] = 1 + if config.data_mode[0] == Const.OUTPUT: + common_set["input_output"] = 2 + + self.dump_json[KernelKbykDump.COMMON_SETTINGS] = common_set + self.dump_json[KernelKbykDump.E2E_SETTINGS] = e2e_set def handle(self): - json_path = self.dump_json["common_dump_settings"]["path"] + json_path = self.dump_json[KernelKbykDump.COMMON_SETTINGS]["path"] make_dump_path_if_not_exists(json_path) - json_path = os.path.join(json_path, "api_kbk_dump.json") + json_path = os.path.join(json_path, "kernel_kbyk_dump.json") with FileOpen(json_path, 'w') as f: json.dump(self.dump_json, f) logger.info(json_path + " has been created.") - os.environ["GRAPH_OP_RUN"] = "1" + os.environ["MINDSPORE_DUMP_CONFIG"] = json_path if "MS_ACL_DUMP_CFG_PATH" in os.environ: del os.environ["MS_ACL_DUMP_CFG_PATH"] diff --git a/debug/accuracy_tools/msprobe/mindspore/overflow_check/overflow_check_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/overflow_check/overflow_check_tool_factory.py index d809c7142..5d71caf30 100644 --- a/debug/accuracy_tools/msprobe/mindspore/overflow_check/overflow_check_tool_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/overflow_check/overflow_check_tool_factory.py @@ -1,23 +1,24 @@ +from msprobe.mindspore.common.const import Const from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.overflow_check.kernel_graph_overflow_check import KernelGraphOverflowCheck class OverflowCheckToolFactory: tools = { - "cell": { - "kbk": None, - "graph": None, - "pynative": None + Const.CELL: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: None }, - "api": { - "kbk": None, - "graph": None, - "pynative": None + Const.API: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: None, + Const.PYNATIVE_MODE: None }, - "kernel": { - "kbk": None, - "graph": KernelGraphOverflowCheck, - "pynative": None + Const.KERNEL: { + Const.GRAPH_KBYK_MODE: None, + Const.GRAPH_GE_MODE: KernelGraphOverflowCheck, + Const.PYNATIVE_MODE: None } } @@ -25,8 +26,9 @@ class OverflowCheckToolFactory: def create(config: DebuggerConfig): tool = OverflowCheckToolFactory.tools.get(config.level) if not tool: - raise Exception("valid level is needed.") - tool = tool.get("graph") + raise Exception("Valid level is needed.") + tool = tool.get(config.execution_mode) if not tool: - raise Exception("Overflow check in not supported in this mode.") + raise Exception(f"Overflow check is not supported in {config.execution_mode} mode " + f"when level is {config.level}.") return tool(config) diff --git a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py index dfe2fbe2c..45344fad1 100644 --- a/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/task_handler_factory.py @@ -1,5 +1,4 @@ from msprobe.core.common.const import Const -from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.dump_tool_factory import DumpToolFactory from msprobe.mindspore.overflow_check.overflow_check_tool_factory import OverflowCheckToolFactory @@ -16,11 +15,9 @@ class TaskHandlerFactory: @staticmethod def create(config: DebuggerConfig): - if config.execution_mode == MsConst.PYNATIVE_MODE and config.task != Const.FREE_BENCHMARK: - raise Exception("Current Task can't run in pynative mode.") task = TaskHandlerFactory.tasks.get(config.task) if not task: - raise Exception("valid task is needed.") + raise Exception("Valid task is needed.") handler = task.create(config) if not handler: raise Exception("Can not find task handler") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py index fb88d7bbb..3b5282f96 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py @@ -15,15 +15,17 @@ # limitations under the License. """ from unittest import TestCase +from unittest.mock import patch +from msprobe.mindspore.common.const import Const from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.dump.dump_tool_factory import DumpToolFactory class TestDumpToolFactory(TestCase): - - def test_create(self): + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def test_create(self, _): json_config = { "task": "statistics", "dump_path": "/absolute_path", @@ -39,13 +41,20 @@ class TestDumpToolFactory(TestCase): config.level = "module" with self.assertRaises(Exception) as context: DumpToolFactory.create(config) - self.assertEqual(str(context.exception), "valid level is needed.") + self.assertEqual(str(context.exception), "Valid level is needed.") + + config.level = Const.KERNEL + with self.assertRaises(Exception) as context: + DumpToolFactory.create(config) + self.assertEqual(str(context.exception), "Data dump is not supported in None mode when dump level is kernel.") - config.level = "cell" + config.execution_mode = Const.GRAPH_GE_MODE + config.level = Const.CELL with self.assertRaises(Exception) as context: DumpToolFactory.create(config) - self.assertEqual(str(context.exception), "Cell dump in not supported now.") + self.assertEqual(str(context.exception), "Data dump is not supported in graph_ge mode when dump level is cell.") - config.level = "kernel" + config.execution_mode = Const.GRAPH_KBYK_MODE + config.level = Const.KERNEL dumper = DumpToolFactory.create(config) self.assertEqual(dumper.dump_json["common_dump_settings"]["net_name"], "Net") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py index e691a2c7e..1d308c18d 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py @@ -25,8 +25,8 @@ from msprobe.mindspore.dump.kernel_graph_dump import KernelGraphDump class TestKernelGraphDump(TestCase): - - def test_handle(self): + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def test_handle(self, _): json_config = { "task": "tensor", "dump_path": "/absolute_path", diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py index a93fab021..71e78cc79 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py @@ -25,8 +25,8 @@ from msprobe.mindspore.overflow_check.kernel_graph_overflow_check import KernelG class TestKernelGraphOverflowCheck(TestCase): - - def test_handle(self): + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def test_handle(self, _): json_config = { "task": "overflow_check", "dump_path": "/absolute_path", diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_api_kbk_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py similarity index 67% rename from debug/accuracy_tools/msprobe/test/mindspore_ut/test_api_kbk_dump.py rename to debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py index 7411018ff..d680032b8 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_api_kbk_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py @@ -21,31 +21,31 @@ from unittest.mock import patch from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig -from msprobe.mindspore.dump.api_kbk_dump import ApiKbkDump +from msprobe.mindspore.dump.kernel_kbyk_dump import KernelKbykDump -class TestApiKbkDump(TestCase): - - def test_handle(self): +class TestKernelKbykDump(TestCase): + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def test_handle(self, _): json_config = { "task": "statistics", "dump_path": "/absolute_path", "rank": [], "step": [0, 2], - "level": "L1" + "level": "L2" } common_config = CommonConfig(json_config) task_config = BaseConfig(json_config) config = DebuggerConfig(common_config, task_config) - dumper = ApiKbkDump(config) + dumper = KernelKbykDump(config) self.assertEqual(dumper.dump_json["common_dump_settings"]["iteration"], "0|2") os.environ["MS_ACL_DUMP_CFG_PATH"] = "path" - with patch("msprobe.mindspore.dump.api_kbk_dump.make_dump_path_if_not_exists"), \ - patch("msprobe.mindspore.dump.api_kbk_dump.FileOpen"), \ - patch("msprobe.mindspore.dump.api_kbk_dump.json.dump"), \ - patch("msprobe.mindspore.dump.api_kbk_dump.logger.info"): + with patch("msprobe.mindspore.dump.kernel_kbyk_dump.make_dump_path_if_not_exists"), \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.FileOpen"), \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.json.dump"), \ + patch("msprobe.mindspore.dump.kernel_kbyk_dump.logger.info") as mock_info: dumper.handle() - self.assertEqual(os.environ.get("GRAPH_OP_RUN"), "1") + mock_info.assert_called_with("/absolute_path/kernel_kbyk_dump.json has been created.") self.assertEqual(os.environ.get("MS_ACL_DUMP_CFG_PATH"), None) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py index 47da051d4..90ffbcb97 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py @@ -15,15 +15,17 @@ # limitations under the License. """ from unittest import TestCase +from unittest.mock import patch +from msprobe.mindspore.common.const import Const from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.overflow_check.overflow_check_tool_factory import OverflowCheckToolFactory class TestOverflowCheckToolFactory(TestCase): - - def test_create(self): + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def test_create(self, _): json_config = { "task": "overflow_check", "dump_path": "/absolute_path", @@ -39,12 +41,14 @@ class TestOverflowCheckToolFactory(TestCase): config.level = "module" with self.assertRaises(Exception) as context: OverflowCheckToolFactory.create(config) - self.assertEqual(str(context.exception), "valid level is needed.") + self.assertEqual(str(context.exception), "Valid level is needed.") + config.execution_mode = Const.GRAPH_GE_MODE config.level = "cell" with self.assertRaises(Exception) as context: OverflowCheckToolFactory.create(config) - self.assertEqual(str(context.exception), "Overflow check in not supported in this mode.") + self.assertEqual(str(context.exception), + f"Overflow check is not supported in {config.execution_mode} mode when level is {config.level}.") config.level = "kernel" dumper = OverflowCheckToolFactory.create(config) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py index 25189a9b6..b85fbacd4 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py @@ -14,22 +14,15 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -import os - import unittest from unittest.mock import Mock, patch -import copy -from msprobe.core.common.utils import Const -from msprobe.mindspore.service import Service -import mindspore -from mindspore.common.tensor import Tensor -from mindspore import ops + from mindspore import nn + +from msprobe.mindspore.service import Service from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig -from unittest.mock import MagicMock -import numpy as np class DummyModel(nn.Cell): @@ -39,8 +32,11 @@ class DummyModel(nn.Cell): def construct(self, x): return self.dense(x) + + class TestService(unittest.TestCase): - def setUp(self): + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def setUp(self, _): json_config = { "task": "statistics", "dump_path": "/absolute_path", diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py index cdc88a3be..6dc6322cc 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py @@ -25,8 +25,8 @@ from msprobe.mindspore.common.const import Const class TestTaskHandlerFactory(TestCase): - - def test_create(self): + @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + def test_create(self, _): class HandlerFactory: def create(self): return None @@ -57,4 +57,4 @@ class TestTaskHandlerFactory(TestCase): config.task = "Free_benchmark" with self.assertRaises(Exception) as context: TaskHandlerFactory.create(config) - self.assertEqual(str(context.exception), "valid task is needed.") + self.assertEqual(str(context.exception), "Valid task is needed.") -- Gitee From 189c23e8e7c3cbac5e9838c31a804ead0eb1c9e4 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 17 Aug 2024 10:59:02 +0800 Subject: [PATCH 324/791] renew --- debug/accuracy_tools/msprobe/core/common/log.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 637d00303..0f950d9ad 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -1,6 +1,7 @@ import os import time import sys +from functools import wraps from msprobe.core.common.const import MsgConst @@ -23,6 +24,7 @@ class BaseLogger: return self.rank def filter_special_chars(func): + @wraps(func) def wrapper(self, msg): if any(char in msg for char in MsgConst.SPECIAL_CHAR): for char in MsgConst.SPECIAL_CHAR: -- Gitee From 0bdd78cf6a3aab20c584c5b83b99140e1eefff64 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 17 Aug 2024 03:39:47 +0000 Subject: [PATCH 325/791] update debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py. Signed-off-by: jiangchangting1 --- .../msprobe/test/core_ut/common/test_utils.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index c77cfa556..d01b22804 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -40,11 +40,10 @@ from msprobe.core.common.utils import (CompareException, check_file_size, check_regex_prefix_format_valid, get_dump_data_path, - task_dumppath_get, - get_json_contents, - get_file_content_bytes) -from msprobe.core.common.file_check import FileCheckConst + task_dumppath_get) +from msprobe.core.common.file_check import FileCheckConst +from msprobe.pytorch.common.utils import get_json_contents, get_file_content_bytes class TestUtils(TestCase): @patch.object(logger, "error") -- Gitee From c047ef177c6f0bd137ebd835f91caa525da809a5 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 17 Aug 2024 14:30:36 +0800 Subject: [PATCH 326/791] compute_element + type_mapping + utils + compute_element ut --- .../msprobe/core/common/exceptions.py | 10 +- .../api_accuracy_checker/__init__.py | 0 .../api_accuracy_checker/compute_element.py | 203 ++++++++++++++++++ .../api_accuracy_checker/type_mapping.py | 86 ++++++++ .../mindspore/api_accuracy_checker/utils.py | 56 +++++ .../api_accuracy_checker/files/input.npy | Bin 0 -> 152 bytes .../test_compute_element.py | 74 +++++++ 7 files changed, 428 insertions(+), 1 deletion(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/__init__.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/input.npy create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py diff --git a/debug/accuracy_tools/msprobe/core/common/exceptions.py b/debug/accuracy_tools/msprobe/core/common/exceptions.py index eb314c7c6..507d47151 100644 --- a/debug/accuracy_tools/msprobe/core/common/exceptions.py +++ b/debug/accuracy_tools/msprobe/core/common/exceptions.py @@ -85,4 +85,12 @@ class DistributedNotInitializedError(Exception): self.msg = msg def __str__(self): - return self.msg \ No newline at end of file + return self.msg + +class ApiAccuracyCheckerException(CodedException): + ParseJsonFailed = 0 + UnsupportType = 1 + err_strs = { + ParseJsonFailed: "[msprobe] Api Accuracy Checker parse json failed: ", + UnsupportType: "[msprobe] Api Accuracy Checker get unsupported type: ", + } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/__init__.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py new file mode 100644 index 000000000..07478be0e --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -0,0 +1,203 @@ +import os + +import mindspore +import torch +import numpy as np + +from msprobe.core.common.log import logger +from msprobe.core.common.exceptions import ApiAccuracyCheckerException +from msprobe.core.common.utils import load_npy +from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np_type, api_info_type_str_to_type, + ms_dtype_to_dtype_str, torch_dtype_to_dtype_str, + dtype_str_to_ms_dtype, dtype_str_to_np_dtype, + dtype_str_to_torch_dtype, DEFAULT_CONSTRUCT_NP_DTYPE) +from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context + + +class MstensorMetaData: + def __init__(self, dtype, npy_path, maximum, minimum, shape) -> None: + self.dtype = dtype + self.npy_path = npy_path + self.maximum = maximum + self.minimum = minimum + self.shape = shape + +class ComputeElement: + def __init__(self, compute_element_info=None, parameter=None): + if parameter is not None: + self._init_with_parameter(parameter) + elif isinstance(compute_element_info, (list, dict)): + self._init_from_compute_element_info(compute_element_info) + else: + logger.error_log_with_exp( + "ComputeElement.__init__ failed: not init with parameter or compute_element info is not (list, dict)", + ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + + def _init_from_compute_element_info(self, compute_element_info): + ''' + Args: + compute_element_info: Union[list, dict] + is_constructed: boolean + + Return: + void + + init member attributes: self.shape, self.dtype_str, self.parameter + ''' + if isinstance(compute_element_info, list): + self.shape = tuple() + self.dtype_str = "tuple" + self.parameter = tuple(ComputeElement(compute_element_info=sub_info).get_parameter() + for sub_info in compute_element_info) + else: + type_str = check_and_get_from_json_dict(compute_element_info, "type", "type field in api_info.json", + accepted_type=str, accepted_value=api_info_type_str_to_type.keys()) + + if type_str == "mindspore.Tensor": + self._init_from_mstensor_compute_element_info(compute_element_info) + else: # type_str in ("slice", "int", "float", "bool") + value = check_and_get_from_json_dict(compute_element_info, "value", "value field in api_info.json") + self.shape = tuple() + self.dtype_str = type_str + self.parameter = slice(*tuple(value)) if type_str == "slice" else value + + def _init_from_mstensor_compute_element_info(self, compute_element_info): + ''' + do not load real tensor, only record meta data + ''' + dtype_str = check_and_get_from_json_dict(compute_element_info, "dtype", "dtype field in api_info.json", + accepted_type=str, accepted_value=dtype_str_to_ms_dtype.keys()) + shape = check_and_get_from_json_dict(compute_element_info, "shape", "shape field in api_info.json", + accepted_type=(list,)) + if global_context.get_is_constructed(): + maximum = check_and_get_from_json_dict(compute_element_info, "Max", "Max field in api_info.json", + accepted_type=(int, float)) + minimum = check_and_get_from_json_dict(compute_element_info, "Min", "Min field in api_info.json", + accepted_type=(int, float)) + + npy_path = None + else: + maximum, minimum = None, None + data_name = check_and_get_from_json_dict(compute_element_info, "data_name", + "data_name field in api_info.json", accepted_type=(str,)) + npy_path = os.path.join(global_context.get_dump_data_dir(), data_name) + mstensor_meta_data = MstensorMetaData(dtype_str, npy_path, maximum, minimum, shape) + self.parameter = mstensor_meta_data + self.dtype_str = dtype_str + self.shape = tuple(shape) + + def _init_with_parameter(self, parameter): + self.parameter = parameter + if isinstance(parameter, mindspore.Tensor): + self.shape = tuple(parameter.shape) + self.dtype_str = ms_dtype_to_dtype_str.get(parameter.dtype) + elif isinstance(parameter, torch.Tensor): + self.shape = tuple(parameter.shape) + self.dtype_str = torch_dtype_to_dtype_str.get(parameter.dtype) + elif isinstance(parameter, (int, float, str, slice, tuple)): + self.shape = tuple() + self.dtype_str = "tuple" if isinstance(parameter, tuple) else api_info_type_str_to_type.get(type(parameter)) + else: + err_msg = "ComputeElement._init_with_parameter failed: " \ + "parameter type is not in (int, float, str, slice, torch.Tensor, mindspore.Tensor)" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + + def get_parameter(self, get_origin=True, get_mindspore_tensor=True): + ''' + Args: + get_origin: boolean + get_mindspore_tensor: boolean + + Return: + parameter: Union[int, float, str, slice,tuple, torch.Tensor, mindspore.Tensor] + ''' + if isinstance(self.parameter, (int, float, str, slice, torch.Tensor, tuple, mindspore.Tensor)): + parameter_tmp = self.parameter + elif isinstance(self.parameter, MstensorMetaData): + mstensor_meta_data = self.parameter + ms_dtype = dtype_str_to_ms_dtype.get(mstensor_meta_data.dtype_str) + if global_context.get_is_constructed(): + np_dtype = dtype_str_to_np_dtype.get(mstensor_meta_data.dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + ndarray = self._construct_ndarray(mstensor_meta_data.shape, mstensor_meta_data.maximum, + mstensor_meta_data.minimum, np_dtype) + else: + ndarray = load_npy(mstensor_meta_data.npy_path) + parameter_tmp = mindspore.Tensor(ndarray, dtype=ms_dtype) + else: + err_msg = "ComputeElement.get_parameter failed: self.parameter type is not in " \ + "(int, float, str, slice, torch.Tensor, mindspore.Tensor, MstensorMetaData)" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + + # if necessary, do transfer + if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and not get_mindspore_tensor: + parameter = self._transfer_to_torch_tensor(parameter_tmp) + elif not get_origin and isinstance(parameter, torch.Tensor) and get_mindspore_tensor: + parameter = self._transfer_to_mindspore_tensor(parameter_tmp) + else: + parameter = parameter_tmp + + return parameter + + def get_shape(self): + return self.shape + + def get_dtype(self): + return self.dtype_str + + + def _transfer_to_torch_tensor(self, ms_tensor): + ''' + Args: + ms_tensor: mindspore.Tensor + Return: + torch_tensor: torch.Tensor + ''' + ms_dtype = ms_tensor.dtype + dtype_str = ms_dtype_to_dtype_str.get(ms_dtype) + if dtype_str not in dtype_str_to_torch_dtype: + err_msg = f"ComputeElement._transfer_to_torch_tensor failed: no matching torch dtype for {dtype_str}" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + else: + torch_dtype = dtype_str_to_torch_dtype.get(dtype_str) + np_ndarray_float64 = ms_tensor.as_type(mindspore.float64).numpy() + torch_tensor = torch.from_numpy(np_ndarray_float64).to(torch_dtype) + return torch_tensor + + def _transfer_to_mindspore_tensor(self, torch_tensor): + ''' + Args: + torch_tensor: torch.Tensor + + Return: + ms_tensor: mindspore.Tensor + ''' + torch_dtype = torch_tensor.dtype + dtype_str = torch_dtype_to_dtype_str.get(torch_dtype) + if dtype_str not in dtype_str_to_ms_dtype: + err_msg = \ + f"ComputeElement._transfer_to_mindspore_tensor failed: no matching mindspore dtype for {dtype_str}" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + else: + ms_dtype = dtype_str_to_ms_dtype.get(dtype_str) + np_ndarray_float64 = torch_tensor.to(torch.float64, copy=True).numpy() + ms_tensor = mindspore.Tensor.from_numpy(np_ndarray_float64).astype(ms_dtype) + return ms_tensor + + def _convert_inf_to_real_num(self, value, dtype_str): + if value == float("inf"): + np_dtype = dtype_str_to_np_type.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + value = np.finfo(np_dtype).max + elif value == float("-inf"): + np_dtype = dtype_str_to_np_type.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + value = np.finfo(np_dtype).min + return value + + def _construct_ndarray(self, shape, maximum, minimum, np_dtype): + shape = tuple(shape) + if np_dtype == np.bool_: + ndarray = np.random.rand(*shape) > 0.5 + else: + maximum = self._convert_inf_to_real_num(maximum, np_dtype) + minimum = self._convert_inf_to_real_num(minimum, np_dtype) + ndarray = np.random.uniform(minimum, maximum, shape).astype(np_dtype) + return ndarray \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py new file mode 100644 index 000000000..c196d338c --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py @@ -0,0 +1,86 @@ +import mindspore.dtype as mstype +import numpy as np +import mindspore +import torch + +INT8 = "Int8" +UINT8 = "UInt8" +INT16 = "Int16" +UINT16 = "UInt16" +INT32 = "Int32" +UINT32 = "UInt32" +INT64 = "Int64" +UINT64 = "UInt64" +FLOAT16 = "Float16" +FLOAT32 = "Float32" +FLOAT64 = "Float64" +BOOL = "Bool" +BFLOAT16 = "BFloat16" +INT4 = "Int4" + + +dtype_str_to_ms_dtype = { + INT8: mstype.int8, + UINT8: mstype.uint8, + INT16: mstype.int16, + UINT16: mstype.uint16, + INT32: mstype.int32, + UINT32: mstype.uint32, + INT64: mstype.int64, + UINT64: mstype.uint64, + FLOAT16: mstype.float16, + FLOAT32: mstype.float32, + FLOAT64: mstype.float64, + BOOL: mstype.bool_, + BFLOAT16: mstype.bfloat16, + INT4: mstype.qint4x2 +} +ms_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_ms_dtype.items()} + + +dtype_str_to_np_dtype = { + INT8: np.int8, + UINT8: np.uint8, + INT16: np.int16, + UINT16: np.uint16, + INT32: np.int32, + UINT32: np.uint32, + INT64: np.int64, + UINT64: np.uint64, + FLOAT16: np.float16, + FLOAT32: np.float32, + FLOAT64: np.float64, + BOOL: np.bool_ +} +np_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_np_dtype.items()} + +dtype_str_to_torch_dtype = { + INT8: torch.int8, + UINT8: torch.uint8, + INT16: torch.int16, + INT32: torch.int32, + INT64: torch.int64, + FLOAT16: torch.float16, + FLOAT32: torch.float32, + FLOAT64: torch.float64, + BOOL: torch.bool, + BFLOAT16: torch.bfloat16, +} +torch_dtype_to_dtype_str = {value: key for key, value in dtype_str_to_torch_dtype.items()} + +MINDSPORE_TENSOR_TYPE_STR = "mindspore.Tensor" +BOOL_TYPE_STR = "bool" +INT_TYPE_STR = "int" +FLOAT_TYPE_STR = "float" +SLICE_TYPE_STR = "slice" + +api_info_type_str_to_type = { + MINDSPORE_TENSOR_TYPE_STR: mindspore.Tensor, + BOOL_TYPE_STR: bool, + INT_TYPE_STR: int, + FLOAT_TYPE_STR: float, + SLICE_TYPE_STR: slice, +} +type_to_api_info_type_str = {value: key for key, value in api_info_type_str_to_type.items()} + +DEFAULT_CONSTRUCT_NP_DTYPE = np.float64 \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py new file mode 100644 index 000000000..95f2ccdbb --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py @@ -0,0 +1,56 @@ +from msprobe.core.common.exceptions import ApiAccuracyCheckerException +from msprobe.core.common.log import logger + +def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_type=None, accepted_value=None): + ''' + Args: + dict_instance: dict, dict parsed from input json + key: str + key_description: str + accepted_type: tuple + accepted_value: Union[tuple, list] + + Return: + value, the corresponding value of "key" in "dict_instance" + + Exception: + raise ApiAccuracyCheckerException.ParseJsonFailed error when + 1. dict_instance is not a dict + 2. value is None + 3. value is not accepted type + 4. value is not accepted value + ''' + parse_failed_exception = ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed) + if not isinstance(dict_instance, dict): + logger.error_log_with_exp("check_and_get_from_json_dict failed: input is not a dict", parse_failed_exception) + value = dict_instance.get(key) + if value is None: + logger.error_log_with_exp(f"check_and_get_from_json_dict failed: {key_description} is missing", + parse_failed_exception) + elif accepted_type is not None and not isinstance(value, accepted_type): + logger.error_log_with_exp( + f"check_and_get_from_json_dict failed: {key_description} is not accepted type: {accepted_type}", + parse_failed_exception) + elif accepted_value is not None and not value in accepted_value: + logger.error_log_with_exp( + f"check_and_get_from_json_dict failed: {key_description} is not accepted value: {accepted_value}", + parse_failed_exception) + return value + +class GlobalContext: + def __init__(self): + self.is_constructed = True + self.dump_data_dir = "" + + def init(self, is_constructed, dump_data_dir): + self.is_constructed = is_constructed + self.dump_data_dir = dump_data_dir + + def get_dump_data_dir(self): + return self.dump_data_dir + + def get_is_constructed(self): + return self.dump_data_dir + + +global_context = GlobalContext() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/input.npy b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/input.npy new file mode 100644 index 0000000000000000000000000000000000000000..a01ef37867a9c44a7a57b5007b165d738d2cc38d GIT binary patch literal 152 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlV+l>qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= kXCxM+0{I$7Its>`ItsN4WCJb+h6Z~E1_lQp0%8Ok01Y4>Y5)KL literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py new file mode 100644 index 000000000..82dd33d52 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -0,0 +1,74 @@ +import sys +import logging +import os + +import pytest +import mindspore +import torch +import numpy as np + +from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement +from msprobe.mindspore.api_accuracy_checker.type_mapping import FLOAT32, FLOAT_TYPE_STR +from msprobe.mindspore.api_accuracy_checker.utils import global_context + +logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + + + +class TestClass: + @classmethod + def setup_class(cls): + """ + class level setup_class + """ + cls.init(TestClass) + + def init(self): + global_context.init(False, os.path.join(directory, "files")) + self.ndarray = np.array([[1, 2, 3], [1, 2, 3]], dtype=np.float32) + self.ms_tensor = mindspore.Tensor(self.ndarray) + self.torch_tensor = torch.Tensor(self.ndarray) + self.tensor_shape = (2, 3) + self.float_instance = 12.0 + + pass + + def test_init_with_parameter_mstensor(self): + # input_parameter, origin_parameter, mstensor_parameter, torchtensor_parameter, shape, dtype_str + parameter_results_mapping = [ + [self.ms_tensor, self.ms_tensor, self.ms_tensor, self.torch_tensor, self.tensor_shape, FLOAT32], + [self.torch_tensor, self.torch_tensor, self.ms_tensor, self.torch_tensor, self.tensor_shape, FLOAT32], + [self.float_instance, self.float_instance, self.float_instance, self.float_instance, tuple(), FLOAT_TYPE_STR], + + ] + for parameter_result in parameter_results_mapping: + input_parameter, origin_parameter, mstensor_parameter, torchtensor_parameter, shape, dtype_str = parameter_result + + compute_element = ComputeElement(parameter=input_parameter) + + assert compute_element.get_parameter(get_origin=True) == origin_parameter + assert compute_element.get_parameter(get_origin=False, get_mindspore_tensor=True) == mstensor_parameter + assert compute_element.get_parameter(get_origin=False, get_mindspore_tensor=False) == torchtensor_parameter + assert compute_element.get_shape() == shape + assert compute_element.get_dtype() == dtype_str + + def test_init_with_compute_element_info(self): + compute_element_info = { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape":[2, 3], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + } + compute_element = ComputeElement(compute_element_info=compute_element_info) + assert compute_element.get_parameter(get_origin=True) == self.ms_tensor + assert compute_element.get_shape() == self.tensor_shape + assert compute_element.get_dtype() == FLOAT32 + + + -- Gitee From 7808b5fa96da3041c5d735792e70c692caa8f92c Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 17 Aug 2024 14:52:07 +0800 Subject: [PATCH 327/791] =?UTF-8?q?=E5=88=86=E6=AD=A5load?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/utils.py | 9 ++++++--- .../msprobe/mindspore/dump/hook_cell/wrap_tensor.py | 3 ++- .../msprobe/pytorch/hook_module/wrap_distributed.py | 3 ++- .../msprobe/pytorch/hook_module/wrap_functional.py | 3 ++- .../msprobe/pytorch/hook_module/wrap_npu_custom.py | 3 ++- .../msprobe/pytorch/hook_module/wrap_tensor.py | 3 ++- .../msprobe/pytorch/hook_module/wrap_torch.py | 3 ++- .../msprobe/pytorch/hook_module/wrap_vf.py | 3 ++- 8 files changed, 20 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 25c7ead8b..fa35a1382 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -545,7 +545,8 @@ def load_npy(filepath): try: npy = np.load(filepath) except Exception as e: - raise RuntimeError(f"load npy file {filepath} failed") from e + logger.error(f"The numpy file failed to load. Please check the path: {filepath}.") + raise RuntimeError(f"Load numpy file {filepath} failed.") from e return npy @@ -555,7 +556,8 @@ def save_npy(data, filepath): try: npy = np.save(filepath, data) except Exception as e: - raise RuntimeError(f"save npy file {filepath} failed") from e + logger.error(f"The numpy file failed to save. Please check the path: {filepath}.") + raise RuntimeError(f"Save numpy file {filepath} failed.") from e change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) @@ -566,5 +568,6 @@ def load_yaml(yaml_path): with FileOpen(checked_path, "r") as f: yaml_data = yaml.safe_load(f) except Exception as e: - raise RuntimeError(f"load yaml file {yaml_path} failed") from e + logger.error(f"The yaml file failed to load. Please check the path: {checked_path}.") + raise RuntimeError(f"Load yaml file {checked_path} failed.") from e return yaml_data diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py index a9facd218..7c919a1b6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py @@ -29,7 +29,8 @@ for f in dir(ms.Tensor): def get_tensor_ops(): - wrap_tensor_ops = load_yaml(yaml_path).get('tensor') + yaml_data = load_yaml(yaml_path) + wrap_tensor_ops = yaml_data.get('tensor') _tensor_ops = dir(ms.Tensor) return set(wrap_tensor_ops) & set(_tensor_ops) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py index 88a8526ad..1f720a32b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py @@ -36,7 +36,8 @@ for f in dir(dist): def get_distributed_ops(): _all_distributed_ops = dir(dist) - wrap_distributed_ops = load_yaml(yaml_path).get('distributed') + yaml_data = load_yaml(yaml_path) + wrap_distributed_ops = yaml_data.get('distributed') return set(wrap_distributed_ops) & set(_all_distributed_ops) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py index f92acac47..95715ec1a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py @@ -67,7 +67,8 @@ yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") def get_functional_ops(): - wrap_functional_ops = load_yaml(yaml_path).get('functional') + yaml_data = load_yaml(yaml_path) + wrap_functional_ops = yaml_data.get('functional') _all_functional_ops = dir(torch.nn.functional) return set(wrap_functional_ops) & set(_all_functional_ops) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index 747b165a4..c2300470e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -41,7 +41,8 @@ def get_npu_ops(): _npu_ops = dir(torch.ops.npu) else: _npu_ops = dir(torch_npu._C._VariableFunctionsClass) - wrap_npu_ops = load_yaml(yaml_path).get('torch_npu') + yaml_data = load_yaml(yaml_path) + wrap_npu_ops = yaml_data.get('torch_npu') return set(wrap_npu_ops) & set(_npu_ops) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py index d3a5c399b..90bb3c61b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py @@ -31,7 +31,8 @@ yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") def get_tensor_ops(): _tensor_ops = dir(torch.Tensor) - wrap_tensor_ops = load_yaml(yaml_path).get('tensor') + yaml_data = load_yaml(yaml_path) + wrap_tensor_ops = yaml_data.get('tensor') return set(wrap_tensor_ops) & set(_tensor_ops) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py index 8aad3daff..32d086a6d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py @@ -30,7 +30,8 @@ yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") def get_torch_ops(): _torch_ops = [] - wrap_torch_ops = load_yaml(yaml_path).get('torch') + yaml_data = load_yaml(yaml_path) + wrap_torch_ops = yaml_data.get('torch') for operation in wrap_torch_ops: if '.' in operation: operation_sub_module_name, operation_sub_op = operation.rsplit('.', 1) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py index 21e0ebfdc..022535824 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py @@ -29,7 +29,8 @@ yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") def get_vf_ops(): - wrap_vf_ops = load_yaml(yaml_path).get('_VF') + yaml_data = load_yaml(yaml_path) + wrap_vf_ops = yaml_data.get('_VF') return wrap_vf_ops -- Gitee From 466991a204fc4fa2baf23ec46c85839a52c7dcfe Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 17 Aug 2024 15:36:34 +0800 Subject: [PATCH 328/791] bug fix --- .../api_accuracy_checker/compute_element.py | 23 ++--- .../api_accuracy_checker/type_mapping.py | 5 +- .../mindspore/api_accuracy_checker/utils.py | 2 +- .../test_compute_element.py | 87 +++++++++++++++++-- 4 files changed, 95 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 07478be0e..11cca2bc6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -7,16 +7,17 @@ import numpy as np from msprobe.core.common.log import logger from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.utils import load_npy -from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np_type, api_info_type_str_to_type, +from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np_dtype, api_info_type_str_to_type, ms_dtype_to_dtype_str, torch_dtype_to_dtype_str, dtype_str_to_ms_dtype, dtype_str_to_np_dtype, - dtype_str_to_torch_dtype, DEFAULT_CONSTRUCT_NP_DTYPE) + dtype_str_to_torch_dtype, DEFAULT_CONSTRUCT_NP_DTYPE, + TUPLE_TYPE_STR, MINDSPORE_TENSOR_TYPE_STR) from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context class MstensorMetaData: - def __init__(self, dtype, npy_path, maximum, minimum, shape) -> None: - self.dtype = dtype + def __init__(self, dtype_str, npy_path, maximum, minimum, shape) -> None: + self.dtype_str = dtype_str self.npy_path = npy_path self.maximum = maximum self.minimum = minimum @@ -46,14 +47,14 @@ class ComputeElement: ''' if isinstance(compute_element_info, list): self.shape = tuple() - self.dtype_str = "tuple" + self.dtype_str = TUPLE_TYPE_STR self.parameter = tuple(ComputeElement(compute_element_info=sub_info).get_parameter() for sub_info in compute_element_info) else: type_str = check_and_get_from_json_dict(compute_element_info, "type", "type field in api_info.json", accepted_type=str, accepted_value=api_info_type_str_to_type.keys()) - if type_str == "mindspore.Tensor": + if type_str == MINDSPORE_TENSOR_TYPE_STR: self._init_from_mstensor_compute_element_info(compute_element_info) else: # type_str in ("slice", "int", "float", "bool") value = check_and_get_from_json_dict(compute_element_info, "value", "value field in api_info.json") @@ -96,7 +97,7 @@ class ComputeElement: self.dtype_str = torch_dtype_to_dtype_str.get(parameter.dtype) elif isinstance(parameter, (int, float, str, slice, tuple)): self.shape = tuple() - self.dtype_str = "tuple" if isinstance(parameter, tuple) else api_info_type_str_to_type.get(type(parameter)) + self.dtype_str = TUPLE_TYPE_STR if isinstance(parameter, tuple) else api_info_type_str_to_type.get(type(parameter)) else: err_msg = "ComputeElement._init_with_parameter failed: " \ "parameter type is not in (int, float, str, slice, torch.Tensor, mindspore.Tensor)" @@ -131,7 +132,7 @@ class ComputeElement: # if necessary, do transfer if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and not get_mindspore_tensor: parameter = self._transfer_to_torch_tensor(parameter_tmp) - elif not get_origin and isinstance(parameter, torch.Tensor) and get_mindspore_tensor: + elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and get_mindspore_tensor: parameter = self._transfer_to_mindspore_tensor(parameter_tmp) else: parameter = parameter_tmp @@ -159,7 +160,7 @@ class ComputeElement: logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) else: torch_dtype = dtype_str_to_torch_dtype.get(dtype_str) - np_ndarray_float64 = ms_tensor.as_type(mindspore.float64).numpy() + np_ndarray_float64 = ms_tensor.astype(mindspore.float64).numpy() torch_tensor = torch.from_numpy(np_ndarray_float64).to(torch_dtype) return torch_tensor @@ -185,10 +186,10 @@ class ComputeElement: def _convert_inf_to_real_num(self, value, dtype_str): if value == float("inf"): - np_dtype = dtype_str_to_np_type.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) value = np.finfo(np_dtype).max elif value == float("-inf"): - np_dtype = dtype_str_to_np_type.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) value = np.finfo(np_dtype).min return value diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py index c196d338c..219d06afc 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py @@ -1,4 +1,4 @@ -import mindspore.dtype as mstype +from mindspore.common import dtype as mstype import numpy as np import mindspore import torch @@ -73,6 +73,8 @@ BOOL_TYPE_STR = "bool" INT_TYPE_STR = "int" FLOAT_TYPE_STR = "float" SLICE_TYPE_STR = "slice" +TUPLE_TYPE_STR = "tuple" +STR_TYPE_STR = "str" api_info_type_str_to_type = { MINDSPORE_TENSOR_TYPE_STR: mindspore.Tensor, @@ -80,6 +82,7 @@ api_info_type_str_to_type = { INT_TYPE_STR: int, FLOAT_TYPE_STR: float, SLICE_TYPE_STR: slice, + STR_TYPE_STR: str, } type_to_api_info_type_str = {value: key for key, value in api_info_type_str_to_type.items()} diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py index 95f2ccdbb..333e7cc60 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py @@ -50,7 +50,7 @@ class GlobalContext: return self.dump_data_dir def get_is_constructed(self): - return self.dump_data_dir + return self.is_constructed global_context = GlobalContext() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py index 82dd33d52..506893633 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -8,7 +8,8 @@ import torch import numpy as np from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement -from msprobe.mindspore.api_accuracy_checker.type_mapping import FLOAT32, FLOAT_TYPE_STR +from msprobe.mindspore.api_accuracy_checker.type_mapping import (FLOAT32, FLOAT_TYPE_STR, INT_TYPE_STR, + TUPLE_TYPE_STR, STR_TYPE_STR, SLICE_TYPE_STR) from msprobe.mindspore.api_accuracy_checker.utils import global_context logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') @@ -33,30 +34,45 @@ class TestClass: self.ms_tensor = mindspore.Tensor(self.ndarray) self.torch_tensor = torch.Tensor(self.ndarray) self.tensor_shape = (2, 3) - self.float_instance = 12.0 - pass - def test_init_with_parameter_mstensor(self): + def test_init_with_parameter_tensor(self): # input_parameter, origin_parameter, mstensor_parameter, torchtensor_parameter, shape, dtype_str parameter_results_mapping = [ [self.ms_tensor, self.ms_tensor, self.ms_tensor, self.torch_tensor, self.tensor_shape, FLOAT32], [self.torch_tensor, self.torch_tensor, self.ms_tensor, self.torch_tensor, self.tensor_shape, FLOAT32], - [self.float_instance, self.float_instance, self.float_instance, self.float_instance, tuple(), FLOAT_TYPE_STR], - ] for parameter_result in parameter_results_mapping: input_parameter, origin_parameter, mstensor_parameter, torchtensor_parameter, shape, dtype_str = parameter_result compute_element = ComputeElement(parameter=input_parameter) + assert (compute_element.get_parameter(get_origin=True) == origin_parameter).all() + assert (compute_element.get_parameter(get_origin=False, get_mindspore_tensor=True) == mstensor_parameter).all() + assert (compute_element.get_parameter(get_origin=False, get_mindspore_tensor=False) == torchtensor_parameter).all() + assert compute_element.get_shape() == shape + assert compute_element.get_dtype() == dtype_str + + def test_init_with_parameter_other_type(self): + # input_parameter, origin_parameter, shape, dtype_str + parameter_results_mapping = { + [1, 1, tuple(), INT_TYPE_STR], + [1.0, 1.0, tuple(), FLOAT_TYPE_STR], + ["string", "string", tuple(), STR_TYPE_STR], + [slice(1, 10, 2), slice(1, 10, 2), tuple(), SLICE_TYPE_STR], + [tuple([1, 2]), tuple([1, 2]), tuple(), TUPLE_TYPE_STR], + } + + for parameter_result in parameter_results_mapping: + input_parameter, origin_parameter, shape, dtype_str = parameter_result + + compute_element = ComputeElement(parameter=input_parameter) + assert compute_element.get_parameter(get_origin=True) == origin_parameter - assert compute_element.get_parameter(get_origin=False, get_mindspore_tensor=True) == mstensor_parameter - assert compute_element.get_parameter(get_origin=False, get_mindspore_tensor=False) == torchtensor_parameter assert compute_element.get_shape() == shape assert compute_element.get_dtype() == dtype_str - def test_init_with_compute_element_info(self): + def test_init_with_compute_element_info_mstensor(self): compute_element_info = { "type": "mindspore.Tensor", "dtype": "Float32", @@ -70,5 +86,58 @@ class TestClass: assert compute_element.get_shape() == self.tensor_shape assert compute_element.get_dtype() == FLOAT32 + def test_init_with_compute_element_info_mstensor_constructed(self): + global_context.is_constructed = True + compute_element_info = { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape":[2, 3], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + } + compute_element = ComputeElement(compute_element_info=compute_element_info) + parameter = compute_element.get_parameter(get_origin=True) + assert (parameter <= 3.0).all() + assert (parameter >= 1.0).all() + assert compute_element.get_shape() == self.tensor_shape + assert compute_element.get_dtype() == FLOAT32 + + def test_init_with_compute_element_info_tuple(self): + global_context.is_constructed = False + compute_element_info = [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape":[2, 3], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + }, + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape":[2, 3], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + }, + ] + compute_element = ComputeElement(compute_element_info=compute_element_info) + parameter = compute_element.get_parameter(get_origin=True) + assert (parameter[0] == self.ms_tensor).all() + assert (parameter[1] == self.ms_tensor).all() + assert compute_element.get_shape() == tuple() + assert compute_element.get_dtype() == TUPLE_TYPE_STR + def test_init_with_compute_element_info_int(self): + compute_element_info = { + "type": "int", + "value": -1, + } + compute_element = ComputeElement(compute_element_info=compute_element_info) + parameter = compute_element.get_parameter(get_origin=True) + assert parameter == -1 + assert compute_element.get_shape() == tuple() + assert compute_element.get_dtype() == INT_TYPE_STR -- Gitee From 434af14de573fe24871d5fdf0ceef4f3eb6472e9 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 17 Aug 2024 15:46:28 +0800 Subject: [PATCH 329/791] bug fix1 --- .../mindspore/api_accuracy_checker/compute_element.py | 7 ++++--- .../api_accuracy_checker/test_compute_element.py | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 11cca2bc6..ee2ed175c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -10,8 +10,9 @@ from msprobe.core.common.utils import load_npy from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np_dtype, api_info_type_str_to_type, ms_dtype_to_dtype_str, torch_dtype_to_dtype_str, dtype_str_to_ms_dtype, dtype_str_to_np_dtype, - dtype_str_to_torch_dtype, DEFAULT_CONSTRUCT_NP_DTYPE, - TUPLE_TYPE_STR, MINDSPORE_TENSOR_TYPE_STR) + dtype_str_to_torch_dtype, type_to_api_info_type_str, + DEFAULT_CONSTRUCT_NP_DTYPE, TUPLE_TYPE_STR, + MINDSPORE_TENSOR_TYPE_STR) from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context @@ -97,7 +98,7 @@ class ComputeElement: self.dtype_str = torch_dtype_to_dtype_str.get(parameter.dtype) elif isinstance(parameter, (int, float, str, slice, tuple)): self.shape = tuple() - self.dtype_str = TUPLE_TYPE_STR if isinstance(parameter, tuple) else api_info_type_str_to_type.get(type(parameter)) + self.dtype_str = TUPLE_TYPE_STR if isinstance(parameter, tuple) else type_to_api_info_type_str.get(type(parameter)) else: err_msg = "ComputeElement._init_with_parameter failed: " \ "parameter type is not in (int, float, str, slice, torch.Tensor, mindspore.Tensor)" diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py index 506893633..7253f802a 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -55,13 +55,13 @@ class TestClass: def test_init_with_parameter_other_type(self): # input_parameter, origin_parameter, shape, dtype_str - parameter_results_mapping = { + parameter_results_mapping = [ [1, 1, tuple(), INT_TYPE_STR], [1.0, 1.0, tuple(), FLOAT_TYPE_STR], ["string", "string", tuple(), STR_TYPE_STR], [slice(1, 10, 2), slice(1, 10, 2), tuple(), SLICE_TYPE_STR], [tuple([1, 2]), tuple([1, 2]), tuple(), TUPLE_TYPE_STR], - } + ] for parameter_result in parameter_results_mapping: input_parameter, origin_parameter, shape, dtype_str = parameter_result @@ -73,6 +73,7 @@ class TestClass: assert compute_element.get_dtype() == dtype_str def test_init_with_compute_element_info_mstensor(self): + global_context.is_constructed = False compute_element_info = { "type": "mindspore.Tensor", "dtype": "Float32", @@ -82,7 +83,7 @@ class TestClass: "data_name": "input.npy" } compute_element = ComputeElement(compute_element_info=compute_element_info) - assert compute_element.get_parameter(get_origin=True) == self.ms_tensor + assert (compute_element.get_parameter(get_origin=True) == self.ms_tensor).all() assert compute_element.get_shape() == self.tensor_shape assert compute_element.get_dtype() == FLOAT32 -- Gitee From d95db16d8d6253b4169ce87c8f52db023d282982 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 17 Aug 2024 16:17:57 +0800 Subject: [PATCH 330/791] compare algorithm half done --- .../base_compare_algorithm.py | 226 ++++++++++++++++++ .../mindspore/api_accuracy_checker/const.py | 9 + 2 files changed, 235 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py new file mode 100644 index 000000000..838e77b74 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -0,0 +1,226 @@ +from abc import ABC, abstractmethod + +import mindspore +import torch +import numpy as np + +from msprobe.core.common.log import logger +from msprobe.mindspore.api_accuracy_checker.const import (COSINE_SIMILARITY, MAX_ABSOLUTE_DIFF, MAX_RELATIVE_DIFF, + PASS, ERROR, SKIP) + + + +class CompareResult: + def __init__(self, compare_value, pass_status, err_msg): + self.compare_value = compare_value + self.pass_status = pass_status + self.err_msg = err_msg + +class BaseCompareAlgorithm(ABC): + def __call__(self, bench_compute_element, tested_compute_element): + ''' + Args: + bench_compute_element: ComputeElement + tested_compute_element: ComputeElement + + Return: + compare_result: CompareResult + ''' + if self.check_validity(bench_compute_element, tested_compute_element): + compare_value = self.run_compare(bench_compute_element, tested_compute_element) + pass_status = self.check_pass(compare_value) + else: + #todo warning + compare_value = None + pass_status = SKIP + + err_msg = self.generate_err_msg(pass_status) + + compare_result = CompareResult(compare_value, pass_status, err_msg) + return compare_result + + @abstractmethod + def check_validity(self, bench_compute_element, tested_compute_element): + ''' + Args: + bench_compute_element: ComputeElement + tested_compute_element: ComputeElement + + Return: + check_res: boolean + ''' + raise NotImplementedError + + @abstractmethod + def run_compare(self, bench_compute_element, tested_compute_element): + ''' + Args: + bench_compute_element: ComputeElement + tested_compute_element: ComputeElement + + Return: + compare_value: float/int + ''' + raise NotImplementedError + + @abstractmethod + def check_pass(self, compare_value): + ''' + Args: + compare_value: float/int + + Return: + pass_status: str + ''' + raise NotImplementedError + + @abstractmethod + def generate_err_msg(self, pass_status): + ''' + Args: + pass_status: str + + Return: + err_msg: str + ''' + raise NotImplementedError + + + def _convert_to_np_float64_ndarray(self, tensor): + if isinstance(tensor, mindspore.Tensor): + ndarray = tensor.as_type(mindspore.float64).numpy() + elif isinstance(tensor, torch.Tensor): + ndarray = tensor.to(torch.float64, copy=True).numpy() + else: + raise ValueError #todo not tensor + return ndarray + + def _check_two_tensor(self, bench_compute_element, tested_compute_element): + bench_parameter = bench_compute_element.get_parameter() + tested_parameter = tested_compute_element.get_parameter() + + bench_is_tensor = isinstance(bench_parameter, (mindspore.Tensor, torch.Tensor)) + tested_is_tensosr = isinstance(tested_parameter, (mindspore.Tensor, torch.Tensor)) + shape_same = bench_compute_element.get_shape() == tested_compute_element.get_shape() + + return bench_is_tensor and tested_is_tensosr and shape_same + + + +class CosineSimilarityCompareAlgorithm(BaseCompareAlgorithm): + def __init__(self) -> None: + super().__init__() + self.pass_threshold = 0.99 + + def check_validity(self, bench_compute_element, tested_compute_element): + return self._check_two_tensor(bench_compute_element, tested_compute_element) + + def run_compare(self, bench_compute_element, tested_compute_element): + bench_ndarray = self._convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self._convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + + bench_norm = np.linalg.norm(bench_ndarray) + tested_norm = np.linalg.norm(tested_ndarray) + dot_product = np.dot(bench_ndarray.flatten(), tested_ndarray.flatten()) + cosine_similarity = dot_product / (bench_norm * tested_norm) + + return cosine_similarity + + @abstractmethod + def check_pass(self, compare_value): + if compare_value > self.pass_threshold: + return PASS + else: + return ERROR + + @abstractmethod + def generate_err_msg(self, pass_status): + if pass_status == PASS: + err_msg = "" + elif pass_status == SKIP: + err_msg = "two inputs are not valid for computing cosine similarity, skip comparing" + elif pass_status == ERROR: + err_msg = f"cosine similarity is less than threshold: {self.pass_threshold}" + else: + raise ValueError #todo + return err_msg + +class MaxAbsoluteDiffCompareAlgorithm(BaseCompareAlgorithm): + def __init__(self) -> None: + super().__init__() + self.pass_threshold = 0.001 + + def check_validity(self, bench_compute_element, tested_compute_element): + return self._check_two_tensor(bench_compute_element, tested_compute_element) + + def run_compare(self, bench_compute_element, tested_compute_element): + bench_ndarray = self._convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self._convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + + max_absolute_diff = np.max(np.abs(bench_ndarray - tested_ndarray)) + return max_absolute_diff + + @abstractmethod + def check_pass(self, compare_value): + if compare_value < self.pass_threshold: + return PASS + else: + return ERROR + + @abstractmethod + def generate_err_msg(self, pass_status): + if pass_status == PASS: + err_msg = "" + elif pass_status == SKIP: + err_msg = "two inputs are not valid for computing max absolute difference, skip comparing" + elif pass_status == ERROR: + err_msg = f"max absolute difference is greater than threshold: {self.pass_threshold}" + else: + raise ValueError #todo + return err_msg + +class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): + def __init__(self) -> None: + super().__init__() + self.pass_threshold = 0.01 + self.epsilon = 1e-8 + + def check_validity(self, bench_compute_element, tested_compute_element): + return self._check_two_tensor(bench_compute_element, tested_compute_element) + + def run_compare(self, bench_compute_element, tested_compute_element): + bench_ndarray = self._convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self._convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + + abs_diff = np.abs(bench_ndarray - tested_ndarray) + bench_ndarray_nonzero = bench_ndarray + (bench_ndarray == 0) * self.epsilon # prevent division by 0 + + max_relative_diff = np.max(abs_diff / bench_ndarray_nonzero) + return max_relative_diff + + @abstractmethod + def check_pass(self, compare_value): + if compare_value < self.pass_threshold: + return PASS + else: + return ERROR + + @abstractmethod + def generate_err_msg(self, pass_status): + if pass_status == PASS: + err_msg = "" + elif pass_status == SKIP: + err_msg = "two inputs are not valid for computing max relative difference, skip comparing" + elif pass_status == ERROR: + err_msg = f"max relative difference is greater than threshold: {self.pass_threshold}" + else: + raise ValueError #todo + return err_msg + + + +compare_algorithms = { + COSINE_SIMILARITY: CosineSimilarityCompareAlgorithm(), + MAX_ABSOLUTE_DIFF: MaxAbsoluteDiffCompareAlgorithm(), + MAX_RELATIVE_DIFF: MaxRelativeDiffCompareAlgorithm(), +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py new file mode 100644 index 000000000..c7ca0550d --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -0,0 +1,9 @@ + +# base_compare_algorithm +COSINE_SIMILARITY = "cosine similarity" +MAX_ABSOLUTE_DIFF = "max absolute difference" +MAX_RELATIVE_DIFF = "max relative difference" + +PASS = "pass" +ERROR = "error" +SKIP = "skip" -- Gitee From cf3ec10624c1fa78c2d361ed242b70785ec8aca3 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 17 Aug 2024 16:27:44 +0800 Subject: [PATCH 331/791] cleancode --- .../api_accuracy_checker/compute_element.py | 207 +++++++++--------- .../mindspore/api_accuracy_checker/utils.py | 2 +- 2 files changed, 106 insertions(+), 103 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index ee2ed175c..d171925cc 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -35,6 +35,108 @@ class ComputeElement: "ComputeElement.__init__ failed: not init with parameter or compute_element info is not (list, dict)", ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + @staticmethod + def transfer_to_torch_tensor(ms_tensor): + ''' + Args: + ms_tensor: mindspore.Tensor + Return: + torch_tensor: torch.Tensor + ''' + ms_dtype = ms_tensor.dtype + dtype_str = ms_dtype_to_dtype_str.get(ms_dtype) + if dtype_str not in dtype_str_to_torch_dtype: + err_msg = f"ComputeElement.transfer_to_torch_tensor failed: no matching torch dtype for {dtype_str}" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + else: + torch_dtype = dtype_str_to_torch_dtype.get(dtype_str) + np_ndarray_float64 = ms_tensor.astype(mindspore.float64).numpy() + torch_tensor = torch.from_numpy(np_ndarray_float64).to(torch_dtype) + return torch_tensor + + @staticmethod + def transfer_to_mindspore_tensor(torch_tensor): + ''' + Args: + torch_tensor: torch.Tensor + + Return: + ms_tensor: mindspore.Tensor + ''' + torch_dtype = torch_tensor.dtype + dtype_str = torch_dtype_to_dtype_str.get(torch_dtype) + if dtype_str not in dtype_str_to_ms_dtype: + err_msg = \ + f"ComputeElement._transfer_to_mindspore_tensor failed: no matching mindspore dtype for {dtype_str}" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + else: + ms_dtype = dtype_str_to_ms_dtype.get(dtype_str) + np_ndarray_float64 = torch_tensor.to(torch.float64, copy=True).numpy() + ms_tensor = mindspore.Tensor.from_numpy(np_ndarray_float64).astype(ms_dtype) + return ms_tensor + + @staticmethod + def convert_inf_to_real_num(value, dtype_str): + if value == float("inf"): + np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + value = np.finfo(np_dtype).max + elif value == float("-inf"): + np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + value = np.finfo(np_dtype).min + return value + + def get_parameter(self, get_origin=True, get_mindspore_tensor=True): + ''' + Args: + get_origin: boolean + get_mindspore_tensor: boolean + + Return: + parameter: Union[int, float, str, slice,tuple, torch.Tensor, mindspore.Tensor] + ''' + if isinstance(self.parameter, (int, float, str, slice, torch.Tensor, tuple, mindspore.Tensor)): + parameter_tmp = self.parameter + elif isinstance(self.parameter, MstensorMetaData): + mstensor_meta_data = self.parameter + ms_dtype = dtype_str_to_ms_dtype.get(mstensor_meta_data.dtype_str) + if global_context.get_is_constructed(): + np_dtype = dtype_str_to_np_dtype.get(mstensor_meta_data.dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + ndarray = self._construct_ndarray(mstensor_meta_data.shape, mstensor_meta_data.maximum, + mstensor_meta_data.minimum, np_dtype) + else: + ndarray = load_npy(mstensor_meta_data.npy_path) + parameter_tmp = mindspore.Tensor(ndarray, dtype=ms_dtype) + else: + err_msg = "ComputeElement.get_parameter failed: self.parameter type is not in " \ + "(int, float, str, slice, torch.Tensor, mindspore.Tensor, MstensorMetaData)" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + + # if necessary, do transfer + if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and not get_mindspore_tensor: + parameter = self.transfer_to_torch_tensor(parameter_tmp) + elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and get_mindspore_tensor: + parameter = self.transfer_to_mindspore_tensor(parameter_tmp) + else: + parameter = parameter_tmp + + return parameter + + def get_shape(self): + return self.shape + + def get_dtype(self): + return self.dtype_str + + def _construct_ndarray(self, shape, maximum, minimum, np_dtype): + shape = tuple(shape) + if np_dtype == np.bool_: + ndarray = np.random.rand(*shape) > 0.5 + else: + maximum = self.convert_inf_to_real_num(maximum, np_dtype) + minimum = self.convert_inf_to_real_num(minimum, np_dtype) + ndarray = np.random.uniform(minimum, maximum, shape).astype(np_dtype) + return ndarray + def _init_from_compute_element_info(self, compute_element_info): ''' Args: @@ -98,108 +200,9 @@ class ComputeElement: self.dtype_str = torch_dtype_to_dtype_str.get(parameter.dtype) elif isinstance(parameter, (int, float, str, slice, tuple)): self.shape = tuple() - self.dtype_str = TUPLE_TYPE_STR if isinstance(parameter, tuple) else type_to_api_info_type_str.get(type(parameter)) + self.dtype_str =\ + TUPLE_TYPE_STR if isinstance(parameter, tuple) else type_to_api_info_type_str.get(type(parameter)) else: err_msg = "ComputeElement._init_with_parameter failed: " \ "parameter type is not in (int, float, str, slice, torch.Tensor, mindspore.Tensor)" - logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) - - def get_parameter(self, get_origin=True, get_mindspore_tensor=True): - ''' - Args: - get_origin: boolean - get_mindspore_tensor: boolean - - Return: - parameter: Union[int, float, str, slice,tuple, torch.Tensor, mindspore.Tensor] - ''' - if isinstance(self.parameter, (int, float, str, slice, torch.Tensor, tuple, mindspore.Tensor)): - parameter_tmp = self.parameter - elif isinstance(self.parameter, MstensorMetaData): - mstensor_meta_data = self.parameter - ms_dtype = dtype_str_to_ms_dtype.get(mstensor_meta_data.dtype_str) - if global_context.get_is_constructed(): - np_dtype = dtype_str_to_np_dtype.get(mstensor_meta_data.dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) - ndarray = self._construct_ndarray(mstensor_meta_data.shape, mstensor_meta_data.maximum, - mstensor_meta_data.minimum, np_dtype) - else: - ndarray = load_npy(mstensor_meta_data.npy_path) - parameter_tmp = mindspore.Tensor(ndarray, dtype=ms_dtype) - else: - err_msg = "ComputeElement.get_parameter failed: self.parameter type is not in " \ - "(int, float, str, slice, torch.Tensor, mindspore.Tensor, MstensorMetaData)" - logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) - - # if necessary, do transfer - if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and not get_mindspore_tensor: - parameter = self._transfer_to_torch_tensor(parameter_tmp) - elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and get_mindspore_tensor: - parameter = self._transfer_to_mindspore_tensor(parameter_tmp) - else: - parameter = parameter_tmp - - return parameter - - def get_shape(self): - return self.shape - - def get_dtype(self): - return self.dtype_str - - - def _transfer_to_torch_tensor(self, ms_tensor): - ''' - Args: - ms_tensor: mindspore.Tensor - Return: - torch_tensor: torch.Tensor - ''' - ms_dtype = ms_tensor.dtype - dtype_str = ms_dtype_to_dtype_str.get(ms_dtype) - if dtype_str not in dtype_str_to_torch_dtype: - err_msg = f"ComputeElement._transfer_to_torch_tensor failed: no matching torch dtype for {dtype_str}" - logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) - else: - torch_dtype = dtype_str_to_torch_dtype.get(dtype_str) - np_ndarray_float64 = ms_tensor.astype(mindspore.float64).numpy() - torch_tensor = torch.from_numpy(np_ndarray_float64).to(torch_dtype) - return torch_tensor - - def _transfer_to_mindspore_tensor(self, torch_tensor): - ''' - Args: - torch_tensor: torch.Tensor - - Return: - ms_tensor: mindspore.Tensor - ''' - torch_dtype = torch_tensor.dtype - dtype_str = torch_dtype_to_dtype_str.get(torch_dtype) - if dtype_str not in dtype_str_to_ms_dtype: - err_msg = \ - f"ComputeElement._transfer_to_mindspore_tensor failed: no matching mindspore dtype for {dtype_str}" - logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) - else: - ms_dtype = dtype_str_to_ms_dtype.get(dtype_str) - np_ndarray_float64 = torch_tensor.to(torch.float64, copy=True).numpy() - ms_tensor = mindspore.Tensor.from_numpy(np_ndarray_float64).astype(ms_dtype) - return ms_tensor - - def _convert_inf_to_real_num(self, value, dtype_str): - if value == float("inf"): - np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) - value = np.finfo(np_dtype).max - elif value == float("-inf"): - np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) - value = np.finfo(np_dtype).min - return value - - def _construct_ndarray(self, shape, maximum, minimum, np_dtype): - shape = tuple(shape) - if np_dtype == np.bool_: - ndarray = np.random.rand(*shape) > 0.5 - else: - maximum = self._convert_inf_to_real_num(maximum, np_dtype) - minimum = self._convert_inf_to_real_num(minimum, np_dtype) - ndarray = np.random.uniform(minimum, maximum, shape).astype(np_dtype) - return ndarray \ No newline at end of file + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py index 333e7cc60..8740afa17 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py @@ -31,7 +31,7 @@ def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_t logger.error_log_with_exp( f"check_and_get_from_json_dict failed: {key_description} is not accepted type: {accepted_type}", parse_failed_exception) - elif accepted_value is not None and not value in accepted_value: + elif accepted_value is not None and value not in accepted_value: logger.error_log_with_exp( f"check_and_get_from_json_dict failed: {key_description} is not accepted value: {accepted_value}", parse_failed_exception) -- Gitee From ed87d2d8c7dd2757c5e7bbcff85230d2e8b276a7 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 17 Aug 2024 16:28:37 +0800 Subject: [PATCH 332/791] =?UTF-8?q?=E6=92=A4=E9=94=80if=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/log.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 0f950d9ad..3fc3951c3 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -26,9 +26,8 @@ class BaseLogger: def filter_special_chars(func): @wraps(func) def wrapper(self, msg): - if any(char in msg for char in MsgConst.SPECIAL_CHAR): - for char in MsgConst.SPECIAL_CHAR: - msg = msg.replace(char, '_') + for char in MsgConst.SPECIAL_CHAR: + msg = msg.replace(char, '_') return func(self, msg) return wrapper -- Gitee From c167dccc796e06058ddb839db31e1e4ac1729d7b Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Sat, 17 Aug 2024 15:45:20 +0800 Subject: [PATCH 333/791] update_wait_time --- .../origin_data_bean/trace_event_bean.py | 7 +- .../compare_bean/overall_metrics_bean.py | 155 ++++++++++----- .../compare_bean/profiling_info.py | 19 +- .../overall_performance_generator.py | 5 + .../profiling_parser/base_profiling_parser.py | 5 +- .../profiling_parser/npu_profiling_parser.py | 176 ++++++++++-------- .../compare_backend/utils/constant.py | 2 + .../compare_backend/utils/excel_config.py | 4 +- .../test_npu_profiling_parser.py | 5 +- 9 files changed, 244 insertions(+), 134 deletions(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py index 245b51d10..26d5bc447 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/origin_data_bean/trace_event_bean.py @@ -148,10 +148,13 @@ class TraceEventBean: return self.lower_cat == "dequeue" def is_process_meta(self) -> bool: - return self.is_m_mode() and self._name == "process_name" + return self._name == "process_name" def is_thread_meta(self) -> bool: - return self.is_m_mode() and self._name == "thread_name" + return self._name == "thread_name" + + def is_thread_sort_meta(self) -> bool: + return self._name == "thread_sort_index" def is_communication_op_thread(self) -> bool: return self._args.get("name", "").find("Communication") != -1 diff --git a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py b/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py index aec94a0e1..0c96c5f58 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/overall_metrics_bean.py @@ -17,7 +17,7 @@ from math import isclose from compare_backend.compare_bean.profiling_info import ProfilingInfo from compare_backend.utils.common_func import calculate_diff_ratio from compare_backend.utils.constant import Constant -from compare_backend.utils.excel_config import ExcelConfig +from compare_backend.utils.excel_config import ExcelConfig, CellFormatType class OverallMetricsBean: @@ -28,19 +28,51 @@ class OverallMetricsBean: def __init__(self, base_info: ProfilingInfo, comparison_info: ProfilingInfo): self._base_data = OverallMetricsInfo(base_info).overall_metrics self._comparison_data = OverallMetricsInfo(comparison_info).overall_metrics + if not any((base_info.is_not_minimal_profiling(), comparison_info.is_not_minimal_profiling())): + self.TABLE_NAME += ' (Minimal Prof)' @property def rows(self): rows_data = [] - for index, base_data in self._base_data.items(): - comparison_data = self._comparison_data.get(index) - row = self.get_row_data(index, base_data, comparison_data) + rows_data.extend( + self._get_rows(self._base_data.get("before_group", {}), self._comparison_data.get("before_group", {}))) + base_group_data = self._base_data.get("group", {}) + comparison_group_data = self._comparison_data.get("group", {}) + default_value = [0, 0, "/"] + for group_name, base_data in base_group_data.items(): + comparison_data = comparison_group_data.pop(group_name, {}) + self._append_data(rows_data, self._get_row_data(group_name, base_data.get("group", default_value), + comparison_data.get("group", default_value))) + self._append_data(rows_data, + self._get_row_data(ExcelConfig.WAIT, base_data.get(ExcelConfig.WAIT, default_value), + comparison_data.get(ExcelConfig.WAIT, default_value))) + self._append_data(rows_data, + self._get_row_data(ExcelConfig.TRANSMIT, + base_data.get(ExcelConfig.TRANSMIT, default_value), + comparison_data.get(ExcelConfig.TRANSMIT, default_value))) + for group_name, comparison_data in comparison_group_data.items(): + self._append_data(rows_data, self._get_row_data(group_name, default_value, + comparison_data.get("group", default_value))) + self._append_data(rows_data, self._get_row_data(ExcelConfig.WAIT, default_value, + comparison_data.get(ExcelConfig.WAIT, default_value))) + self._append_data(rows_data, self._get_row_data(ExcelConfig.TRANSMIT, default_value, + comparison_data.get(ExcelConfig.TRANSMIT, default_value))) + rows_data.extend( + self._get_rows(self._base_data.get("after_group", {}), self._comparison_data.get("after_group", {}))) + return rows_data + + @classmethod + def _get_rows(cls, base_data_dict, comparison_data_dict): + rows_data = [] + for index, base_data in base_data_dict.items(): + comparison_data = comparison_data_dict.get(index) + row = cls._get_row_data(index, base_data, comparison_data) if row: rows_data.append(row) return rows_data - @staticmethod - def get_row_data(index, base_data, comparison_data): + @classmethod + def _get_row_data(cls, index, base_data, comparison_data): if isclose(base_data[0], 0) and isclose(comparison_data[0], 0): return [] row_data = [index] @@ -49,52 +81,28 @@ class OverallMetricsBean: row_data.extend(calculate_diff_ratio(base_data[0], comparison_data[0])) return row_data + @classmethod + def _append_data(cls, all_data, data): + if not data: + return + all_data.append(data) + class OverallMetricsInfo: def __init__(self, profiling_info: ProfilingInfo): self._profiling_info = profiling_info - self._overall_metrics_data_map = { - ExcelConfig.COMPUTING: self.computing_data, - ExcelConfig.FA_FWD: self.fa_fwd_data, - ExcelConfig.FA_FWD_CUBE: self.fa_fwd_cube_data, - ExcelConfig.FA_FWD_VECTOR: self.fa_fwd_vector_data, - ExcelConfig.FA_BWD: self.fa_bwd_data, - ExcelConfig.FA_BWD_CUBE: self.fa_bwd_cube_data, - ExcelConfig.FA_BWD_VECTOR: self.fa_bwd_vector_data, - ExcelConfig.CONV_FWD: self.conv_fwd_data, - ExcelConfig.CONV_FWD_CUBE: self.conv_fwd_cube_data, - ExcelConfig.CONV_FWD_VECTOR: self.conv_fwd_vector_data, - ExcelConfig.CONV_BWD: self.conv_bwd_data, - ExcelConfig.CONV_BWD_CUBE: self.conv_bwd_cube_data, - ExcelConfig.CONV_BWD_VECTOR: self.conv_bwd_vector_data, - ExcelConfig.MM: self.mm_data, - ExcelConfig.MM_CUBE: self.mm_cube_data, - ExcelConfig.MM_VECTOR: self.mm_vector_data, - ExcelConfig.PA: self.pa_data, - ExcelConfig.VECTOR: self.vector_data, - ExcelConfig.VECTOR_TRANS: self.vector_trans_data, - ExcelConfig.VECTOR_NO_TRANS: self.vector_no_trans_data, - ExcelConfig.CUBE: self.cube_data, - ExcelConfig.SDMA_TM: self.sdma_tm_data, - ExcelConfig.OTHER: self.other_data, - ExcelConfig.COMMUNICATION_TIME: self.communication_data, - ExcelConfig.WAIT: self.wait_data, - ExcelConfig.TRANSMIT: self.transmit_data, - ExcelConfig.FREE_TIME: self.free_time_data, - ExcelConfig.SDMA: self.sdma_data, - ExcelConfig.FREE: self.free_data, - ExcelConfig.E2E_TIME: self.e2e_time_data - } + self._comm_group_list = list(profiling_info.communication_group_time.keys()) + self._overall_metrics_data = self._init_overall_metrics_data() @property def e2e_time(self): if isclose(self._profiling_info.e2e_time_ms, 0): - raise RuntimeError("Invalid E2E Time.") + return float("inf") return self._profiling_info.e2e_time_ms @property def overall_metrics(self): - return self._overall_metrics_data_map + return self._overall_metrics_data @property def computing_data(self): @@ -247,16 +255,6 @@ class OverallMetricsInfo: return [self._profiling_info.communication_not_overlapped_ms, self._profiling_info.communication_not_overlapped_ms / self.e2e_time, "/"] - @property - def wait_data(self): - return [self._profiling_info.wait_time_ms, - self._profiling_info.wait_time_ms / self.e2e_time, "/"] - - @property - def transmit_data(self): - return [self._profiling_info.transmit_time_ms, - self._profiling_info.transmit_time_ms / self.e2e_time, "/"] - @property def free_time_data(self): return [self._profiling_info.free_time_ms, @@ -275,3 +273,60 @@ class OverallMetricsInfo: @property def e2e_time_data(self): return [self.e2e_time, 1, "/"] + + def communication_data_by_group(self, group_name: str): + return [self._profiling_info.get_communication_time_by_group(group_name), + self._profiling_info.get_communication_time_by_group(group_name) / self.e2e_time, + "/"] + + def wait_data_by_group(self, group_name: str): + return [self._profiling_info.get_wait_time_by_group(group_name), + self._profiling_info.get_wait_time_by_group(group_name) / self.e2e_time, "/"] + + def transmit_data_by_group(self, group_name: str): + return [self._profiling_info.get_transmit_time_by_group(group_name), + self._profiling_info.get_transmit_time_by_group(group_name) / self.e2e_time, "/"] + + def _init_overall_metrics_data(self): + overall_metrics_data = {"before_group": { + ExcelConfig.COMPUTING: self.computing_data, + ExcelConfig.FA_FWD: self.fa_fwd_data, + ExcelConfig.FA_FWD_CUBE: self.fa_fwd_cube_data, + ExcelConfig.FA_FWD_VECTOR: self.fa_fwd_vector_data, + ExcelConfig.FA_BWD: self.fa_bwd_data, + ExcelConfig.FA_BWD_CUBE: self.fa_bwd_cube_data, + ExcelConfig.FA_BWD_VECTOR: self.fa_bwd_vector_data, + ExcelConfig.CONV_FWD: self.conv_fwd_data, + ExcelConfig.CONV_FWD_CUBE: self.conv_fwd_cube_data, + ExcelConfig.CONV_FWD_VECTOR: self.conv_fwd_vector_data, + ExcelConfig.CONV_BWD: self.conv_bwd_data, + ExcelConfig.CONV_BWD_CUBE: self.conv_bwd_cube_data, + ExcelConfig.CONV_BWD_VECTOR: self.conv_bwd_vector_data, + ExcelConfig.MM: self.mm_data, + ExcelConfig.MM_CUBE: self.mm_cube_data, + ExcelConfig.MM_VECTOR: self.mm_vector_data, + ExcelConfig.PA: self.pa_data, + ExcelConfig.VECTOR: self.vector_data, + ExcelConfig.VECTOR_TRANS: self.vector_trans_data, + ExcelConfig.VECTOR_NO_TRANS: self.vector_no_trans_data, + ExcelConfig.CUBE: self.cube_data, + ExcelConfig.SDMA_TM: self.sdma_tm_data, + ExcelConfig.OTHER: self.other_data, + ExcelConfig.COMMUNICATION_TIME: self.communication_data} + } + if self._comm_group_list: + for group_name in self._comm_group_list: + group_name_index = f"\t{group_name}" + ExcelConfig.ROW_STYLE_MAP[group_name_index] = CellFormatType.LIGHT_BLUE_NORMAL + overall_metrics_data.setdefault("group", {})[group_name_index] = { + "group": self.communication_data_by_group(group_name), + ExcelConfig.WAIT: self.wait_data_by_group(group_name), + ExcelConfig.TRANSMIT: self.transmit_data_by_group(group_name) + } + overall_metrics_data["after_group"] = { + ExcelConfig.FREE_TIME: self.free_time_data, + ExcelConfig.SDMA: self.sdma_data, + ExcelConfig.FREE: self.free_data, + ExcelConfig.E2E_TIME: self.e2e_time_data + } + return overall_metrics_data diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index e2891ecc4..7985501db 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -63,6 +63,9 @@ class ProfilingInfo: self.RDMA_bandwidth = 0.0 self.SDMA_bandwidth = 0.0 + # 按group展示通信的卡间等待和传输耗时 + self.communication_group_time = {} + @property def e2e_time_ms(self): return self.e2e_time * 10 ** 3 @@ -291,8 +294,10 @@ class ProfilingInfo: def update_comm_not_overlap(self, time: float): self.communication_not_overlapped += time - def update_comm_not_overlap_wait_time(self, time: float): - self.wait_time = time + def update_communication_group_time(self, time_dict: dict): + self.communication_group_time = time_dict + for time in time_dict.values(): + self.wait_time += time.get(Constant.WAIT_TIME, 0) def set_memory_used(self, memory: float): self.memory_used = memory @@ -331,3 +336,13 @@ class ProfilingInfo: self.e2e_time /= Constant.MICROSECONDS_TO_SECONDS self.scheduling_time /= Constant.MICROSECONDS_TO_SECONDS self.lccl_time /= Constant.MICROSECONDS_TO_SECONDS + + def get_wait_time_by_group(self, group_name: str): + return self.communication_group_time.get(group_name, {}).get(Constant.WAIT_TIME, 0) / 10 ** 3 + + def get_transmit_time_by_group(self, group_name: str): + return self.communication_group_time.get(group_name, {}).get(Constant.TRANSMIT_TIME, 0) / 10 ** 3 + + def get_communication_time_by_group(self, group_name: str): + return (self.communication_group_time.get(group_name, {}).get(Constant.WAIT_TIME, 0) + + self.communication_group_time.get(group_name, {}).get(Constant.TRANSMIT_TIME, 0)) / 10 ** 3 diff --git a/profiler/compare_tools/compare_backend/generator/overall_performance_generator.py b/profiler/compare_tools/compare_backend/generator/overall_performance_generator.py index 9fe31d0ea..72cac6cf6 100644 --- a/profiler/compare_tools/compare_backend/generator/overall_performance_generator.py +++ b/profiler/compare_tools/compare_backend/generator/overall_performance_generator.py @@ -1,3 +1,5 @@ +import logging + from compare_backend.comparator.overall_performance_comparator import OverallPerformanceComparator from compare_backend.compare_bean.profiling_info import ProfilingInfo from compare_backend.generator.base_generator import BaseGenerator @@ -17,3 +19,6 @@ class OverallPerformanceGenerator(BaseGenerator): if not self._result_data: return ScreenView(self._result_data).generate_view() + logging.info("The OverallMetrics sheet page is more comprehensive for the disaggregate of performance data, " + "and it is recommended to view the overall performance comparison results from " + "the performance_comparison_result_*.xlsx.") diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index b56b30936..44f0e7105 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -322,9 +322,10 @@ class BaseProfilingParser(ABC): print(f"[WARNING] Can't find any communication op in the file: {self._profiling_path}") if self._enable_kernel_compare and not self._result_data.kernel_details: if self._profiling_type == Constant.GPU: - print(f"[WARNING] kernel compare between GPU data and NPU data is not supported.") + print(f"[WARNING] kernel compare only support between NPU data and NPU data.") else: - print(f"[WARNING] Can't find any kernel details in the file: {self._profiling_path}") + print(f"[WARNING] Can't find any valid kernels in the file: {self._profiling_path}. Please " + f"make sure that the profiling data is greater than level0 and aic_metrics=PipeUtilization.") def _read_trace_event(self): try: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 4139a4b6d..a5c9d4675 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -32,9 +32,32 @@ class NPUProfilingParser(BaseProfilingParser): self._enqueue_dict = {} self._dequeue_data = [] self._overlap_analysis = [] + self._group_comm_tid_dict = {} + self._hccl_tid_name_dict = {} self._dispatch_func = self._get_dispatch_func() self._filter_meta_id() + @staticmethod + def __calculate_overlap_time_with_uncovered_communication(uncovered_communication_events: list, events: list): + overlap_time = 0 + events.sort(key=lambda x: x.start_time) + index = 0 + for comm_event in uncovered_communication_events: + pre_overlap_ts = comm_event.start_time + while index < len(events): + event = events[index] + if event.end_time <= comm_event.start_time: + index += 1 + continue + if event.start_time >= comm_event.end_time: + break + if event.end_time >= comm_event.end_time: + overlap_time += comm_event.end_time - max(event.start_time, pre_overlap_ts) + break + overlap_time += event.end_time - max(event.start_time, pre_overlap_ts) + index += 1 + return float(overlap_time) + def _get_dispatch_func(self): func_list = set() if self._enable_memory_compare or self._enable_operator_compare or self._enable_profiling_compare: @@ -79,8 +102,8 @@ class NPUProfilingParser(BaseProfilingParser): [kernel.name, kernel.duration]) if not kernels_dict: if self._step_id != Constant.VOID_STEP: - print(f"[ERROR] There is no kernel details infomation for step {self._step_id}," \ - " please check whether the data contains this step.") + print(f"[ERROR] There is no kernel details information for step {self._step_id}, " + f"please check whether the data contains this step.") else: print("[ERROR] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") return @@ -165,73 +188,54 @@ class NPUProfilingParser(BaseProfilingParser): self.__add_lccl_time() self.__add_sdma_time() self.__add_overlap_analysis_time() - self._picking_notify_wait_event_and_not_overlap_event() - self.__add_overlap_wait_time() + self.__add_communication_wait_time() self._result_data.overall_metrics.calculate_other_time() self._result_data.overall_metrics.calculate_schedule_time() self._result_data.overall_metrics.trans_time_to_s() self._update_bandwidth() - def _picking_notify_wait_event_and_not_overlap_event(self): - self.notify_event_cache = [] - self._not_overlaped_commu_event = [] - for event in self._comm_task_list: - if event.name == 'Notify_Wait' and event.args.get('rdma_type', 0) != 'RDMA_PAYLOAD_CHECK' \ - and event.args.get('rdma_type', 0) != 'RDMA_PAYLOAD_ACK': - self.notify_event_cache.append(event) - for event in self._overlap_analysis: - if event.is_comm_not_overlap(): - self._not_overlaped_commu_event.append(event) - self._not_overlaped_commu_event.sort(key=lambda x: x.start_time) - - def __add_overlap_wait_time(self): - notify_wait_event_dict = dict() - for notify_event in self.notify_event_cache: - if notify_event.tid in notify_wait_event_dict: - notify_wait_event_dict[notify_event.tid].append(notify_event) + def __add_communication_wait_time(self): + """ + 按group统计uncovered communication time的卡间等待时间、传输时间。选择传输时间最长的plane作为该group的卡间等待时间、传输时间。 + 卡间等待时间用Notify_Wait任务（部分做rdma传输的Notify_Wait任务除外）计算，传输时间=通信时间-卡间等待时间。 + rdma传输有两种范式，一种是RDMASend、RDMASend、Notify_Wait、RDMASend、Notify_Wait，里面的notify wait都是传输时间； + 还有一种是RDMASend、RDMASend、Notify_Wait, 这个notify wait也是传输时间。 + 因此，满足前2个task为RDMASend、RDMASend的Notify_Wait不计入卡间等待时间， + 满足前4个task为RDMASend、RDMASend、Notify_Wait、RDMASend的Notify_Wait不计入卡间等待时间。 + """ + notify_wait_task_group_by_tid = {} + self._comm_task_list.sort(key=lambda x: x.start_time) + last_4_task_mode_dict = {} # 前4个task的类型，R代表RDMASend/N代表Notify_Wait/O代表Other + for task_event in self._comm_task_list: + last_4_task_mode = last_4_task_mode_dict.get(task_event.tid) + if task_event.name == 'RDMASend': + last_4_task_mode_dict[task_event.tid] = f"{last_4_task_mode[1:]}R" if last_4_task_mode else "OOOR" + elif task_event.name == 'Notify_Wait': + if not last_4_task_mode or last_4_task_mode != "RRNR" and last_4_task_mode[2:] != "RR": + notify_wait_task_group_by_tid.setdefault(task_event.tid, []).append(task_event) + last_4_task_mode_dict[task_event.tid] = f"{last_4_task_mode[1:]}N" if last_4_task_mode else "OOON" else: - notify_wait_event_dict[notify_event.tid] = [notify_event] - - if self._result_data.overall_metrics.is_level0: - return - - total_time = 0 - for commu_event in self._not_overlaped_commu_event: - wait_time_list = [0] - commu_event_start_time = float(commu_event.start_time) - commu_event_end_time = float(commu_event.start_time) + commu_event.dur - - for plane_id, events in notify_wait_event_dict.items(): - wait_time = 0 - idx = 0 - for notify_event in events: - notify_event_start_time = float(notify_event.start_time) - notify_event_end_time = float(notify_event.start_time) + notify_event.dur - if notify_event_start_time < commu_event_start_time and notify_event_end_time > \ - commu_event_end_time: - wait_time = commu_event_end_time - commu_event_start_time - break - elif notify_event_start_time < commu_event_start_time <= notify_event_end_time <= \ - commu_event_end_time: - wait_time += notify_event_end_time - commu_event_start_time - idx += 1 - elif commu_event_start_time <= notify_event_start_time <= commu_event_end_time < \ - notify_event_end_time: - wait_time += commu_event_end_time - notify_event_start_time - break - elif notify_event_start_time >= commu_event_start_time and notify_event_end_time <= \ - commu_event_end_time: - wait_time += notify_event_end_time - notify_event_start_time - idx += 1 - elif notify_event_end_time < commu_event_start_time: - idx += 1 - else: - break - - wait_time_list.append(wait_time) - notify_wait_event_dict[plane_id] = notify_wait_event_dict[plane_id][idx:] - total_time += max(wait_time_list) - self._result_data.overall_metrics.update_comm_not_overlap_wait_time(total_time) + last_4_task_mode_dict[task_event.tid] = f"{last_4_task_mode[1:]}O" if last_4_task_mode else "OOOO" + uncovered_communication_events = list(filter(lambda x: x.is_comm_not_overlap(), self._overlap_analysis)) + group_comm_time_dict = {} + for comm_tid, tid_list in self._group_comm_tid_dict.items(): + min_wait_time = float("inf") + min_wait_tid = None + for tid in tid_list: + notify_wait_time = sum((event.dur for event in notify_wait_task_group_by_tid.get(tid, []))) + if notify_wait_time < min_wait_time: + min_wait_time = notify_wait_time + min_wait_tid = tid + notify_wait_events = notify_wait_task_group_by_tid.get(min_wait_tid, []) + communication_op_events = list(filter(lambda x: x.tid == comm_tid, self._comm_list)) + wait_time = self.__calculate_overlap_time_with_uncovered_communication(uncovered_communication_events, + notify_wait_events) + uncovered_communication_time = self.__calculate_overlap_time_with_uncovered_communication( + uncovered_communication_events, communication_op_events) + group_comm_time_dict[self._hccl_tid_name_dict.get(comm_tid)] = { + Constant.WAIT_TIME: wait_time, + Constant.TRANSMIT_TIME: uncovered_communication_time - wait_time} + self._result_data.overall_metrics.update_communication_group_time(group_comm_time_dict) def _picking_hccl_event(self, event: TraceEventBean): if event.pid != self._hccl_pid or not event.is_x_mode(): @@ -267,24 +271,46 @@ class NPUProfilingParser(BaseProfilingParser): return event.lower_cat == self.TORCH_OP_CAT def _filter_meta_id(self): + thread_events, thread_sort_events = [], [] for event in self._trace_events: if event.is_fwdbwd() and event.is_flow_end(): self._bwd_tid = event.tid - if not event.is_process_meta(): + if not event.is_m_mode(): continue - if event.is_hccl_process_name(): - self._hccl_pid = event.pid - elif event.is_npu_process_name(): - self._kernel_pid = event.pid - elif event.is_overlap_process_name(): - self._overlap_pid = event.pid - if not self._enable_communication_compare: + if event.is_process_meta(): + if event.is_hccl_process_name(): + self._hccl_pid = event.pid + elif event.is_npu_process_name(): + self._kernel_pid = event.pid + elif event.is_overlap_process_name(): + self._overlap_pid = event.pid + if event.is_thread_meta(): + thread_events.append(event) + if event.is_thread_sort_meta(): + thread_sort_events.append(event) + + if not self._enable_communication_compare and not self._enable_profiling_compare: return - for event in self._trace_events: - if not event.is_thread_meta(): + # 获取hccl bar的所有thread信息 + tid_index_dict = {} + for event in thread_events: + if event.pid == self._hccl_pid: + self._hccl_tid_name_dict[event.tid] = event.args.get("name", "") + for event in thread_sort_events: + if event.pid == self._hccl_pid: + tid_index_dict[event.args.get("sort_index", 0)] = event.tid + ordered_index = sorted(tid_index_dict.keys()) + cur_tid = None + for index in ordered_index: + tid = tid_index_dict.get(index) + tid_name = self._hccl_tid_name_dict.get(tid, "") + if "Communication" in tid_name: + self._hccl_op_tid_list.append(tid) + self._group_comm_tid_dict.setdefault(tid, []) + cur_tid = tid continue - if event.pid == self._hccl_pid and event.is_communication_op_thread(): - self._hccl_op_tid_list.append(event.tid) + if tid_name: + self._group_comm_tid_dict.setdefault(cur_tid, []).append(tid) def __parse_info_json(self): try: diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index ffe6a906e..8dc503763 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -38,6 +38,8 @@ class Constant(object): # excel headers BASE_PROFILING = 'Base Profiling: ' COMPARISON_PROFILING = 'Comparison Profiling: ' + WAIT_TIME = "wait" + TRANSMIT_TIME = "transmit" # compare type OPERATOR_COMPARE = "OperatorCompare" diff --git a/profiler/compare_tools/compare_backend/utils/excel_config.py b/profiler/compare_tools/compare_backend/utils/excel_config.py index 975b2a532..403702b20 100644 --- a/profiler/compare_tools/compare_backend/utils/excel_config.py +++ b/profiler/compare_tools/compare_backend/utils/excel_config.py @@ -279,8 +279,8 @@ class ExcelConfig(object): # communication time COMMUNICATION_TIME = "Uncovered Communication Time" - WAIT = "\tWait" - TRANSMIT = "\tTransmit" + WAIT = "\t\tWait" + TRANSMIT = "\t\tTransmit" # free time FREE_TIME = "Free Time" diff --git a/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py b/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py index 0e3965247..3d9ff4512 100644 --- a/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py +++ b/profiler/test/ut/compare_tools/profiling_parser/test_npu_profiling_parser.py @@ -17,7 +17,8 @@ class TestNPUProfilingParser(unittest.TestCase): meta_events = [{"ph": "M", "name": "process_name", "pid": 7, "tid": 3, "args": {"name": "HCCL"}}, {"ph": "M", "name": "process_name", "pid": 9, "tid": 3, "args": {"name": "Overlap Analysis"}}, {"ph": "M", "name": "process_name", "pid": 5, "tid": 3, "args": {"name": "Ascend Hardware"}}, - {"ph": "M", "name": "thread_name", "pid": 7, "tid": 3, "args": {"name": "Communication"}}] + {"ph": "M", "name": "thread_name", "pid": 7, "tid": 3, "args": {"name": "Communication"}}, + {"ph": "M", "name": "thread_sort_index", "pid": 7, "tid": 3, "args": {"sort_index": 0}}] def test_update_memory_list_when_invalid_path(self): with patch("compare_backend.profiling_parser.base_profiling_parser.BaseProfilingParser.__init__"), \ @@ -125,6 +126,8 @@ class TestNPUProfilingParser(unittest.TestCase): res = NPUProfilingParser({}, {}) res._trace_events = [TraceEventBean(event) for event in self.meta_events] res._hccl_op_tid_list = [] + res._hccl_tid_name_dict = {} + res._group_comm_tid_dict = {} res._enable_communication_compare = True res._filter_meta_id() self.assertEqual(res._hccl_pid, 7) -- Gitee From 346c231d393cdf3897392b5bc4cc9219daf53503 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 17 Aug 2024 08:47:35 +0000 Subject: [PATCH 334/791] update debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index d01b22804..edd8f67af 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -16,6 +16,7 @@ """ import os import uuid +import json from unittest import TestCase from unittest.mock import patch, MagicMock, mock_open -- Gitee From 0f5452784908fb926071eaae7ebb7bfe73357e60 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Sat, 17 Aug 2024 08:59:56 +0000 Subject: [PATCH 335/791] update debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index edd8f67af..ec6f375bf 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -345,7 +345,8 @@ class TestUtils(TestCase): task_dumppath_get(input_param) self.assertEqual(context.exception.code, CompareException.INVALID_TASK_ERROR) mock_error.assert_called_with("Compare is not required for overflow_check or free_benchmark.") - + + @patch('msprobe.pytorch.common.utils.get_file_content_bytes') def test_get_json_contents_should_raise_exception(self, mock_get_file_content_bytes): mock_get_file_content_bytes.return_value = 'not a dict' with self.assertRaises(CompareException) as ce: -- Gitee From 90f7c41e11bc1b1babd5986e8519d00201079af9 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Sat, 17 Aug 2024 17:05:22 +0800 Subject: [PATCH 336/791] mindspore free benchmark introduction --- debug/accuracy_tools/msprobe/config/README.md | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/msprobe/config/README.md b/debug/accuracy_tools/msprobe/config/README.md index b8357855b..7720ddbdf 100644 --- a/debug/accuracy_tools/msprobe/config/README.md +++ b/debug/accuracy_tools/msprobe/config/README.md @@ -45,26 +45,26 @@ Location字段为msprobe工具的安装路径，那么config.json文件位置为 ### task配置为free_benchmark -仅PyTorch场景支持。 +仅PyTorch场景与MindSpore动态图场景支持，且"level"需为"L1"。 task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通过对当前模型API的输入添加扰动因子，二次执行，将得到的输出与未添加扰动因子前的输出进行比对，从而**得出该模型中可能因迁移等变化导致精度降低的API**。 -无标杆比对优势在于省去了从GPU环境获取dump数据并执行的步骤，也省去了在NPU环境执行dump的操作，降低了精度比对的操作难度。 +无标杆比对优势在于省去了从CPU/GPU环境获取标杆数据的步骤，也省去了在NPU环境执行dump的操作，降低了精度比对的操作难度。 建议配置白名单（配置scope或list）控制少量API进行无标杆比对，一次对过多API执行无标杆比对可能导致显存溢出或性能膨胀。 | 参数名 | 说明 | 是否必选 | | ------------ | ------------------------------------------------------------ | -------- | -| scope | PyTorch场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。与list参数不能同时配置。 | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
PyTorch场景配置kernel_api，dump前向和反向API的kernel_api级别数据，其中dump反向API时需要配置**backward_input**参数。前向API配置示例："list": ["Tensor.permute.1.forward"]；反向API配置示例："list": ["Tensor.permute.1.forward"], "backward.input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
与scope参数不能同时配置。 | 否 | -| fuzz_device | 标杆设备，str类型。可取值：
"npu"：无标杆，通过添加扰动因子进行比对，默认值。
"cpu"：以CPU为标杆，pert_mode须配置为"to_cpu"。
配置示例："fuzz_device": "cpu"。 | 否 | -| pert_mode | 无标杆扰动因子，str类型。可取值：
"improve_precision"：对输入做升精度，默认值。
"add_noise"：对输入增加噪声。
"no_change"：不加扰动直接二次执行。
"bit_noise"：输入的末位比特翻转。
"change_value"：输入的张量首尾值调换。
"to_cpu"：在CPU等价执行。
配置示例："pert_mode": "to_cpu"。 | 否 | -| handler_type | 处理类型，可取值："check"（进行无标杆比对检查，默认值）、"fix"（将扰动后的API输出结果覆盖原始API输出结果，尝试将Loss曲线恢复正常，该模式下不支持预热if_preheat）。配置示例："handler_type": "fix"。 | 否 | -| fuzz_level | 无标杆数据dump级别，即选择比对结果文件应输出的表头属性，当前仅支持取值为："L1"。输出结果详见“**无标杆比对数据存盘格式**”。 | 否 | -| fuzz_stage | 前反向，选择对API前向或反向进行无标杆比对，可取值："forward"（前向，默认值）、"backward"（反向）。配置示例："fuzz_stage": "backward"。 | 否 | -| if_preheat | 预热功能，开启功能后工具可以根据每次迭代的输出调整精度算法的阈值，从而更准确找出存在精度问题的API，bool类型。可取值true（开启）或false（关闭），默认关闭。配置示例："if_preheat": "true"。"handler_type": "fix"不支持预热。 | 否 | -| preheat_step | 开启预热的迭代数量，int类型，默认值为15。须配置"if_preheat": "true"。 | 否 | -| max_sample | 每个算子预热的采样次数的最大阈值，int类型，默认值为20。须配置"if_preheat": "true"。 | 否 | +| scope | 自定义检测API列表（仅PyTorch场景支持），list[str]类型，默认值为空列表，当list也为空列表时，表示检测所有API。需要在[]内配置具体API名（在工具dump结果中查看）。配置示例："scope": ["Torch.matmul.0", "Tensor.pow.4"]。与list参数不能同时配置。 | 否 | +| list | 自定义检测API类型或API名称，list[str]类型，默认值为空列表，表示检测所有API（PyTorch场景下还需scope也为空列表）。包含如下配置方法：
PyTorch场景下，指定某一类API，对某一类的API进行无标杆比对。配置示例："list": ["relu"]。
MindSpore场景下，指定API名称，对列表中的API进行检测。配置示例："list": ["mindspore.mint.div", "mindspore.ops.bmm", "mindspore.Tensor.\_\_add\_\_"]。
与scope参数不能同时配置。 | 否 | +| fuzz_device | 标杆设备，str类型。可取值：
"npu"：无标杆，通过添加扰动因子进行比对，默认值。
"cpu"：以CPU为标杆，pert_mode须配置为"to_cpu"（仅PyTorch场景支持）。
配置示例："fuzz_device": "npu"。 | 否 | +| pert_mode | 无标杆扰动因子，str类型。可取值：
"improve_precision"：对输入做升精度，默认值。
"add_noise"：对输入增加噪声。
"no_change"：不加扰动直接二次执行。
"bit_noise"：输入的末位比特翻转。
"change_value"：输入的张量首尾值调换（仅PyTorch场景支持）。
"to_cpu"：在CPU等价执行（仅PyTorch场景支持）。
配置示例："pert_mode": "improve_precision"。 | 否 | +| handler_type | 处理类型，可取值："check"（进行无标杆比对检查，默认值）、"fix"（将扰动后的API输出结果覆盖原始API输出结果，尝试将Loss曲线恢复正常，该模式下不支持预热if_preheat与反向过程）。配置示例："handler_type": "fix"。 | 否 | +| fuzz_level | 无标杆数据dump级别，即选择比对结果文件应输出的表头属性，当前仅支持取值为："L1"（默认值）。输出结果详见“**无标杆比对数据存盘格式**”。 | 否 | +| fuzz_stage | 前反向，选择对API前向或反向进行无标杆比对，可取值："forward"（前向，默认值）、"backward"（反向，仅PyTorch场景支持）。配置示例："fuzz_stage": "forward"。 | 否 | +| if_preheat | 预热功能（仅PyTorch场景支持），开启功能后工具可以根据每次迭代的输出调整精度算法的阈值，从而更准确找出存在精度问题的API，bool类型。可取值true（开启）或false（关闭），默认关闭。配置示例："if_preheat": "true"。"handler_type": "fix"不支持预热。 | 否 | +| preheat_step | 开启预热的迭代数量（仅PyTorch场景支持），int类型，默认值为15。须配置"if_preheat": "true"。 | 否 | +| max_sample | 每个算子预热的采样次数的最大阈值（仅PyTorch场景支持），int类型，默认值为20。须配置"if_preheat": "true"。 | 否 | #### 无标杆比对数据存盘格式 @@ -82,7 +82,7 @@ task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通 | max_rel | 输出对比最大相对误差，float类型。 | | dtype | 输入的dtype，string类型。 | | shape | 输入的shape，tuple类型。 | -| Output_index | 如果输出为列表或元组，其中一个元素检测不一致，则会有该元素的index，否则为空，int类型。 | +| output_index | 如果输出为列表或元组，其中一个元素检测不一致，则会有该元素的index，否则为空，int类型。 | ### task配置为statistics @@ -307,6 +307,27 @@ MindSpore静态图场景的jit_level为O0/O1时，不支持该功能，须配置 } ``` +### MindSpore动态图场景task配置为free_benchmark + +```json +{ + "task": "free_benchmark", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + + "free_benchmark": { + "list": ["mindspore.ops.add"], + "fuzz_device": "npu", + "pert_mode": "improve_precision", + "handler_type": "check", + "fuzz_level": "L1", + "fuzz_stage": "forward" + } +} +``` + ### MindSpore动态图场景task配置为statistics ```json -- Gitee From 57098f38f79bea83960a3475cda9c52f2ae0db81 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 17 Aug 2024 18:11:28 +0800 Subject: [PATCH 337/791] compare_algorithm ut + api_runner --- .../msprobe/core/common/exceptions.py | 4 + .../api_accuracy_checker/api_runner.py | 133 ++++++++++++++++++ .../base_compare_algorithm.py | 67 +++++---- .../mindspore/api_accuracy_checker/const.py | 8 ++ .../mindspore/api_accuracy_checker/utils.py | 7 + .../test_compare_algorithm.py | 51 +++++++ 6 files changed, 239 insertions(+), 31 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py diff --git a/debug/accuracy_tools/msprobe/core/common/exceptions.py b/debug/accuracy_tools/msprobe/core/common/exceptions.py index 507d47151..5f67686d0 100644 --- a/debug/accuracy_tools/msprobe/core/common/exceptions.py +++ b/debug/accuracy_tools/msprobe/core/common/exceptions.py @@ -90,7 +90,11 @@ class DistributedNotInitializedError(Exception): class ApiAccuracyCheckerException(CodedException): ParseJsonFailed = 0 UnsupportType = 1 + WrongValue = 2 + ApiWrong = 3 err_strs = { ParseJsonFailed: "[msprobe] Api Accuracy Checker parse json failed: ", UnsupportType: "[msprobe] Api Accuracy Checker get unsupported type: ", + WrongValue: "[msprobe] Api Accuracy Checker get wrong value: ", + ApiWrong: "[msprobe] Api Accuracy Checker something wrong with api: ", } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py new file mode 100644 index 000000000..d464928db --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -0,0 +1,133 @@ + + +import mindspore +import torch +from mindspore import ops + +from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement +from msprobe.mindspore.api_accuracy_checker.const import IS_MINDSPORE_API, IS_TORCH_API, MINT, MINT_FUNCTIONAL +from msprobe.core.common.exceptions import ApiAccuracyCheckerException +from msprobe.core.common.log import logger +from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple + + +class ApiRunner: + def __init__(self) -> None: + self.api_parent_module_mapping = { + (MINT, IS_MINDSPORE_API): mindspore.mint, + (MINT, IS_TORCH_API): torch, + (MINT_FUNCTIONAL, IS_MINDSPORE_API): mindspore.mint.nn.functional, + (MINT_FUNCTIONAL, IS_TORCH_API): torch.nn.functional + } + + def __call__(self, inputs, api_name_str, kwargs, gradient_inputs=None, + is_forward=True, is_mindspore_api=IS_MINDSPORE_API): + ''' + Args: + inputs: List[ComputeElement] + api_name_str: str + kwargs: dict + gradient_inputs: Union[List[ComputeElement], None] + is_forward: boolean + is_mindspore_api: boolean + + Return: + outputs: list[ComputeElement] + + Description: + run mindspore.mint/torch api + ''' + api_type_str, api_sub_name = self.get_info_from_name(api_name_str) + api_instance = self._get_api_instance(api_type_str, api_sub_name, is_mindspore_api) + + self._run_api(api_instance, inputs, kwargs, gradient_inputs, is_forward, is_mindspore_api) + + @classmethod + def get_info_from_name(cls, api_name_str): + ''' + Args: + api_name_str: str, the key of data dict in api_info.json. e.g. "MintFunctional.relu.0" + + Return: + api_type_str: str, Union["MintFunctional", "Mint"] + api_sub_name: str, e.g. "relu" + ''' + api_name_list = api_name_str.split('.') + if len(api_name_list) != 4: + err_msg = f"ApiRunner.get_info_from_name failed: api_name_str: {api_name_str} is not in defined format" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) + api_type_str, api_sub_name = api_name_list[0], api_name_list[1] + if api_type_str not in [MINT, MINT_FUNCTIONAL]: + err_msg = f"ApiRunner.get_info_from_name failed: not mint or mint.nn.functional api" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) + + return api_type_str, api_sub_name + + def _get_api_instance(self, api_type_str, api_sub_name, is_mindspore_api): + ''' + Args: + api_type_str: str, Union["MintFunctional", "Mint"] + api_sub_name: str, e.g. "relu" + is_mindspore_api: boolean + + Return: + api_instance: function object + + Description: + get mindspore.mint/torch api fucntion + mindspore.mint.{api_sub_name} <--> torch.{api_sub_name} + mindspore.mint.nn.functional.{api_sub_name} <--> torch.nn.functional.{api_sub_name} + ''' + + api_parent_module = self.api_parent_module_mapping.get((api_type_str, is_mindspore_api)) + module_str = "mindspore.mint." if is_mindspore_api else "torch." + submodule_str = "nn.functional." if api_type_str == MINT_FUNCTIONAL else "" + full_api_name = module_str + submodule_str + api_sub_name + if not hasattr(api_parent_module, api_sub_name): + err_msg = f"ApiRunner._get_api_instance failed: {full_api_name} is not found" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ApiWrong)) + + api_instance = getattr(api_parent_module, api_sub_name) + if not callable(api_instance): + err_msg = f"ApiRunner._get_api_instance failed: {full_api_name} is not callable" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ApiWrong)) + + return api_instance + + def _run_api(self, api_instance, inputs, kwargs, gradient_inputs, is_forward, is_mindspore_api): + inputs = tuple(compute_element.get_parameter(get_origin=False, get_mindspore_tensor=is_mindspore_api) + for compute_element in inputs) + + if is_forward: + forward_result = api_instance(*inputs, **kwargs) # can be single tensor or tuple + forward_result_tuple = convert_to_tuple(forward_result) + res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in forward_result_tuple] + else: + if gradient_inputs is None: + err_msg = f"ApiRunner._run_api failed: run backward api but gradient_inputs is missing" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) + gradient_inputs = \ + tuple(compute_element.get_parameter(get_origin=False, get_mindspore_tensor=is_mindspore_api) + for compute_element in gradient_inputs) + if is_mindspore_api: + grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_instance) + backward_result = grad_func(*inputs, **kwargs, gradient_inputs) # can be single tensor or tuple + backward_result_tuple = convert_to_tuple(backward_result) + res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_tuple] + else: + #set requires_grad + for tensor in inputs: + if hasattr(tensor, "requires_grad"): + setattr(tensor, "requires_grad", True) + forward_result = api_instance(*inputs, **kwargs) + forward_result.backward(gradient_inputs) + backward_result_list = [] + for tensor in inputs: + if hasattr(tensor, "grad"): + backward_result_list.append(getattr(tensor, "grad")) + res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_list] + + return res_compute_element_list + + +api_runner = ApiRunner() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index 838e77b74..c2cefa70b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -4,19 +4,24 @@ import mindspore import torch import numpy as np +from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.const import (COSINE_SIMILARITY, MAX_ABSOLUTE_DIFF, MAX_RELATIVE_DIFF, PASS, ERROR, SKIP) - class CompareResult: def __init__(self, compare_value, pass_status, err_msg): self.compare_value = compare_value self.pass_status = pass_status self.err_msg = err_msg + class BaseCompareAlgorithm(ABC): + def __init__(self) -> None: + super().__init__() + self.compare_algorithm_name = None + def __call__(self, bench_compute_element, tested_compute_element): ''' Args: @@ -30,7 +35,7 @@ class BaseCompareAlgorithm(ABC): compare_value = self.run_compare(bench_compute_element, tested_compute_element) pass_status = self.check_pass(compare_value) else: - #todo warning + logger.warning(f"not suitable for computing {self.compare_algorithm_name}, skip this.") compare_value = None pass_status = SKIP @@ -85,55 +90,54 @@ class BaseCompareAlgorithm(ABC): ''' raise NotImplementedError - - def _convert_to_np_float64_ndarray(self, tensor): + @classmethod + def convert_to_np_float64_ndarray(tensor): if isinstance(tensor, mindspore.Tensor): - ndarray = tensor.as_type(mindspore.float64).numpy() + ndarray = tensor.astype(mindspore.float64).numpy() elif isinstance(tensor, torch.Tensor): ndarray = tensor.to(torch.float64, copy=True).numpy() else: - raise ValueError #todo not tensor + err_msg = "BaseCompareAlgorithm.convert_to_np_float64_ndarray failed: " \ + "input is not mindspore.Tensor or torch.Tensor" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) return ndarray - def _check_two_tensor(self, bench_compute_element, tested_compute_element): + @classmethod + def check_two_tensor(bench_compute_element, tested_compute_element): bench_parameter = bench_compute_element.get_parameter() tested_parameter = tested_compute_element.get_parameter() bench_is_tensor = isinstance(bench_parameter, (mindspore.Tensor, torch.Tensor)) tested_is_tensosr = isinstance(tested_parameter, (mindspore.Tensor, torch.Tensor)) shape_same = bench_compute_element.get_shape() == tested_compute_element.get_shape() - return bench_is_tensor and tested_is_tensosr and shape_same - class CosineSimilarityCompareAlgorithm(BaseCompareAlgorithm): def __init__(self) -> None: super().__init__() - self.pass_threshold = 0.99 + self.compare_algorithm_name = COSINE_SIMILARITY + self.pass_threshold = 0.9999 def check_validity(self, bench_compute_element, tested_compute_element): - return self._check_two_tensor(bench_compute_element, tested_compute_element) + return self.check_two_tensor(bench_compute_element, tested_compute_element) def run_compare(self, bench_compute_element, tested_compute_element): - bench_ndarray = self._convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) - tested_ndarray = self._convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + bench_ndarray = self.convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) bench_norm = np.linalg.norm(bench_ndarray) tested_norm = np.linalg.norm(tested_ndarray) dot_product = np.dot(bench_ndarray.flatten(), tested_ndarray.flatten()) cosine_similarity = dot_product / (bench_norm * tested_norm) - return cosine_similarity - @abstractmethod def check_pass(self, compare_value): if compare_value > self.pass_threshold: return PASS else: return ERROR - @abstractmethod def generate_err_msg(self, pass_status): if pass_status == PASS: err_msg = "" @@ -142,32 +146,33 @@ class CosineSimilarityCompareAlgorithm(BaseCompareAlgorithm): elif pass_status == ERROR: err_msg = f"cosine similarity is less than threshold: {self.pass_threshold}" else: - raise ValueError #todo + logger.warning(f"unseen pass_status: {pass_status}") + err_msg = "" return err_msg + class MaxAbsoluteDiffCompareAlgorithm(BaseCompareAlgorithm): def __init__(self) -> None: super().__init__() + self.compare_algorithm_name = MAX_ABSOLUTE_DIFF self.pass_threshold = 0.001 def check_validity(self, bench_compute_element, tested_compute_element): - return self._check_two_tensor(bench_compute_element, tested_compute_element) + return self.check_two_tensor(bench_compute_element, tested_compute_element) def run_compare(self, bench_compute_element, tested_compute_element): - bench_ndarray = self._convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) - tested_ndarray = self._convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + bench_ndarray = self.convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) max_absolute_diff = np.max(np.abs(bench_ndarray - tested_ndarray)) return max_absolute_diff - @abstractmethod def check_pass(self, compare_value): if compare_value < self.pass_threshold: return PASS else: return ERROR - @abstractmethod def generate_err_msg(self, pass_status): if pass_status == PASS: err_msg = "" @@ -176,36 +181,36 @@ class MaxAbsoluteDiffCompareAlgorithm(BaseCompareAlgorithm): elif pass_status == ERROR: err_msg = f"max absolute difference is greater than threshold: {self.pass_threshold}" else: - raise ValueError #todo + logger.warning(f"unseen pass_status: {pass_status}") + err_msg = "" return err_msg + class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): def __init__(self) -> None: super().__init__() + self.compare_algorithm_name = MAX_RELATIVE_DIFF self.pass_threshold = 0.01 self.epsilon = 1e-8 def check_validity(self, bench_compute_element, tested_compute_element): - return self._check_two_tensor(bench_compute_element, tested_compute_element) + return self.check_two_tensor(bench_compute_element, tested_compute_element) def run_compare(self, bench_compute_element, tested_compute_element): - bench_ndarray = self._convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) - tested_ndarray = self._convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) + bench_ndarray = self.convert_to_np_float64_ndarray(bench_compute_element.get_parameter()) + tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) abs_diff = np.abs(bench_ndarray - tested_ndarray) bench_ndarray_nonzero = bench_ndarray + (bench_ndarray == 0) * self.epsilon # prevent division by 0 - max_relative_diff = np.max(abs_diff / bench_ndarray_nonzero) return max_relative_diff - @abstractmethod def check_pass(self, compare_value): if compare_value < self.pass_threshold: return PASS else: return ERROR - @abstractmethod def generate_err_msg(self, pass_status): if pass_status == PASS: err_msg = "" @@ -214,11 +219,11 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): elif pass_status == ERROR: err_msg = f"max relative difference is greater than threshold: {self.pass_threshold}" else: - raise ValueError #todo + logger.warning(f"unseen pass_status: {pass_status}") + err_msg = "" return err_msg - compare_algorithms = { COSINE_SIMILARITY: CosineSimilarityCompareAlgorithm(), MAX_ABSOLUTE_DIFF: MaxAbsoluteDiffCompareAlgorithm(), diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py index c7ca0550d..794416980 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -7,3 +7,11 @@ MAX_RELATIVE_DIFF = "max relative difference" PASS = "pass" ERROR = "error" SKIP = "skip" + + +#api_runner +IS_MINDSPORE_API = True +IS_TORCH_API = False + +MINT = "Mint" +MINT_FUNCTIONAL = "MintFunctional" \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py index 8740afa17..54abf59c2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py @@ -37,6 +37,13 @@ def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_t parse_failed_exception) return value +def convert_to_tuple(input): + if isinstance(input, (tuple, list)): + return tuple(input) + else: + input_list = [input] + return tuple(input_list) + class GlobalContext: def __init__(self): self.is_constructed = True diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py new file mode 100644 index 000000000..9aba764fc --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py @@ -0,0 +1,51 @@ +import sys +import logging +import os + +import pytest +import mindspore +import torch +from unittest.mock import MagicMock + +from msprobe.mindspore.api_accuracy_checker.base_compare_algorithm import compare_algorithms +from msprobe.mindspore.api_accuracy_checker.const import COSINE_SIMILARITY, MAX_ABSOLUTE_DIFF, MAX_RELATIVE_DIFF, ERROR + +logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + + +@pytest.fixture +def mock_mstensor_compute_element(): + mock = MagicMock() + mock.get_parameter.return_value = mindspore.Tensor([1., 1.9, 3.], dtype=mindspore.float32) + mock.get_shape.return_value = (3,) + return mock + +@pytest.fixture +def mock_torchtensor_compute_element(): + mock = MagicMock() + mock.get_parameter.return_value = torch.Tensor([1., 2., 3.], dtype=torch.float32) + mock.get_shape.return_value = (3,) + return mock + + +class TestClass: + + def test_cosine_similarity(self, mock_torchtensor_compute_element, mock_mstensor_compute_element): + compare_result = compare_algorithms[COSINE_SIMILARITY](mock_torchtensor_compute_element, mock_mstensor_compute_element) + assert abs(compare_result.compare_value - 0.9997375534689601) < 1e-5 + assert compare_result.pass_status == ERROR + + + def test_max_absolute_difference(self, mock_torchtensor_compute_element, mock_mstensor_compute_element): + compare_result = compare_algorithms[MAX_ABSOLUTE_DIFF](mock_torchtensor_compute_element, mock_mstensor_compute_element) + assert abs(compare_result.compare_value - 0.1) < 1e-5 + assert compare_result.pass_status == ERROR + + def test_max_relative_difference(self, mock_torchtensor_compute_element, mock_mstensor_compute_element): + compare_result = compare_algorithms[MAX_RELATIVE_DIFF](mock_torchtensor_compute_element, mock_mstensor_compute_element) + assert abs(compare_result.compare_value - 0.05) < 1e-5 + assert compare_result.pass_status == ERROR \ No newline at end of file -- Gitee From 7caf7eb778475724a8f9e2f7027308e18516fa27 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 19 Aug 2024 09:39:43 +0800 Subject: [PATCH 338/791] =?UTF-8?q?=E6=9B=B4=E6=96=B0UT?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/test/core_ut/test_log.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 1687c48d0..31d56acb0 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -33,18 +33,18 @@ class TestLog(TestCase): @patch.object(BaseLogger, "_print_log") def test_print_info_log(self, mock__print_log): - logger.info("info_msg") - mock__print_log.assert_called_with("INFO", "info_msg") + logger.info("\n\n\ninfo_msg") + mock__print_log.assert_called_with("INFO", "___info_msg") @patch.object(BaseLogger, "_print_log") def test_print_warn_log(self, mock__print_log): - logger.warning("warn_msg") - mock__print_log.assert_called_with("WARNING", "warn_msg") + logger.warning("\n\n\nwarn_msg") + mock__print_log.assert_called_with("WARNING", "___warn_msg") @patch.object(BaseLogger, "_print_log") def test_print_error_log(self, mock__print_log): - logger.error("error_msg") - mock__print_log.assert_called_with("ERROR", "error_msg") + logger.error("\n\n\nerror_msg") + mock__print_log.assert_called_with("ERROR", "___error_msg") @patch.object(BaseLogger, "error") def test_error_log_with_exp(self, mock_error): -- Gitee From fa3d47e7117de765a2f3ca340bf26340f8b646f5 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 19 Aug 2024 10:04:03 +0800 Subject: [PATCH 339/791] =?UTF-8?q?=E6=9B=B4=E6=96=B0var.=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 3fc3951c3..ed0776291 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -25,11 +25,11 @@ class BaseLogger: def filter_special_chars(func): @wraps(func) - def wrapper(self, msg): + def func_level(self, msg): for char in MsgConst.SPECIAL_CHAR: msg = msg.replace(char, '_') return func(self, msg) - return wrapper + return func_level @filter_special_chars def info(self, msg): -- Gitee From 0d1ae72750ecd7150864e3e2f2e627ac8544a259 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 19 Aug 2024 10:44:06 +0800 Subject: [PATCH 340/791] change cli parameter --- .../msprobe/core/compare/compare_cli.py | 16 ++++++++-------- .../accuracy_tools/msprobe/core/compare/utils.py | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index ac96a65de..f5222fcf3 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -5,14 +5,14 @@ from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger - def compare_cli(args): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) - frame_name =args.framework - if frame_name ==Const.PT_FRAMEWORK: + frame_name = args.framework + auto_analyze = not args.compare_only + if frame_name == Const.PT_FRAMEWORK: from msprobe.pytorch.compare.pt_compare import compare from msprobe.pytorch.compare.distributed_compare import compare_distributed else: @@ -23,13 +23,13 @@ def compare_cli(args): input_param["bench_json_path"] = input_param.pop("bench_path") input_param["stack_json_path"] = input_param.pop("stack_path") if frame_name == Const.PT_FRAMEWORK: - compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, - fuzzy_match=args.fuzzy_match) + compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=auto_analyze, + fuzzy_match=args.fuzzy_match) else: - ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, - fuzzy_match=args.fuzzy_match) + ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=auto_analyze, + fuzzy_match=args.fuzzy_match) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: - kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} + kwargs = {"stack_mode": args.stack_mode, "auto_analyze": auto_analyze, "fuzzy_match": args.fuzzy_match} if frame_name == Const.PT_FRAMEWORK: compare_distributed(npu_path, bench_path, args.output_path, **kwargs) else: diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index f905ea91f..d890973aa 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -417,7 +417,7 @@ def _compare_parser(parser): help=" The compare task result out path.", required=True) parser.add_argument("-s", "--stack_mode", dest="stack_mode", action="store_true", help=" Whether to save stack info.", required=False) - parser.add_argument("-a", "--auto_analyze", dest="auto_analyze", action="store_false", + parser.add_argument("-c", "--compare_only", dest="compare_only", action="store_true", help=" Whether to give advisor.", required=False) parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", help=" Whether to perform a fuzzy match on the api name.", required=False) -- Gitee From d5085bdf022f772919799850852dbaf99e43c011 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 19 Aug 2024 11:16:07 +0800 Subject: [PATCH 341/791] api unmatched none to n/a --- debug/accuracy_tools/msprobe/core/common/const.py | 1 + debug/accuracy_tools/msprobe/core/compare/utils.py | 14 +++++++------- .../msprobe/pytorch/doc/ptdbg_ascend_compare.md | 14 +++++++------- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 333757082..0a409650e 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -176,6 +176,7 @@ class CompareConst: WARNING = 'Warning' ERROR = 'error' SKIP = 'SKIP' + N_A = 'N/A' BFLOAT16_MIN = -3.3895313892515355e+38 BFLOAT16_MAX = 3.3895313892515355e+38 BFLOAT16_EPS = 3.90625e-3 # 2 ** -8 diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index d890973aa..bb0ab4014 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -326,9 +326,9 @@ def get_accuracy(result, n_dict, b_dict, summary_compare=False, md5_compare=Fals def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): index_out = 0 npu_stack_info = n_dict.get("stack_info", None) - bench_name, bench_type, bench_shape = CompareConst.NONE, CompareConst.NONE, CompareConst.NONE + bench_name, bench_type, bench_shape = CompareConst.N_A, CompareConst.N_A, CompareConst.N_A err_msg = CompareConst.NO_BENCH - accuracy_check_res = CompareConst.NONE + accuracy_check_res = CompareConst.N_A for index, n_name in enumerate(n_dict["op_name"]): if n_name.find("input") != -1: n_struct = n_dict["input_struct"][index] @@ -338,7 +338,7 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): result_item = [n_name, bench_name, n_struct[0], bench_type, n_struct[1], bench_shape] if md5_compare: - result_item.extend([CompareConst.NONE] * 3) + result_item.extend([CompareConst.N_A] * 3) if npu_stack_info and index == 0: result_item.extend(npu_stack_info) else: @@ -346,12 +346,12 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): result.append(result_item) continue if summary_compare: - result_item.extend([CompareConst.NONE] * 8) + result_item.extend([CompareConst.N_A] * 8) else: - result_item.extend([CompareConst.NONE] * 5) + result_item.extend([CompareConst.N_A] * 5) summary_data = n_dict.get("summary")[index] result_item.extend(summary_data) - summary_data = [CompareConst.NONE] * 4 + summary_data = [CompareConst.N_A] * 4 result_item.extend(summary_data) result_item.append(accuracy_check_res) result_item.append(err_msg) @@ -359,7 +359,7 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): result_item.extend(npu_stack_info) else: result_item.append(CompareConst.NONE) - if not md5_compare and not summary_compare and result_item[1] == CompareConst.NONE: + if not md5_compare and not summary_compare and result_item[1] == CompareConst.N_A: if index == 0: result_item.extend(["-1"]) else: diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md index 80228bab0..7c378a584 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md @@ -36,13 +36,13 @@ **完整参数说明** - | 参数名 | 说明 | 是否必选 | - | ------------------ |----------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | - | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -a或--auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，表示开启，通过配置该参数关闭。 | 否 | - | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | 参数名 | 说明 | 是否必选 | + |-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | + | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | + | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -c或--compare_only | 是否自动精度分析。未配置默认开启，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，表示开启，通过配置该参数关闭自动精度分析，仅输出比对结果表格。 | 否 | + | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | 3. 查看比对结果，请参见“**比对结果分析**”。 -- Gitee From 29394cfc47ea24162a02220a53ed7c9d829ff628 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 19 Aug 2024 11:21:40 +0800 Subject: [PATCH 342/791] compare readme update --- .../msprobe/mindspore/doc/compare.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md index 802f70267..b97a86fcd 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md @@ -23,13 +23,13 @@ msprobe精度比对工具主要通过对同一个模型，在两个不同的Mind **完整参数说明** - | 参数名 | 说明 | 是否必选 | - | ------------------ |----------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | - | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -a或--auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，表示开启，通过配置该参数关闭。 | 否 | - | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | 参数名 | 说明 | 是否必选 | + |-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | + | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | + | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -c或--compare_only | 是否自动精度分析，未配置默认开启，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，表示开启，通过配置该参数关闭自动精度分析，仅输出比对结果表格。 | 否 | + | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | 4. 查看比对结果，请详见PyTorch目录下的《[精度比对工具](../../pytorch/doc/ptdbg_ascend_compare.md)》的“比对结果分析”章节。 -- Gitee From 7904b7f703b09293061d8d60853081ae49ef1055 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 19 Aug 2024 11:33:09 +0800 Subject: [PATCH 343/791] bugfix1 --- .../api_accuracy_checker/api_runner.py | 47 ++++++++++--------- .../mindspore/api_accuracy_checker/const.py | 9 ++-- 2 files changed, 32 insertions(+), 24 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index d464928db..b9a6e7c9a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -5,7 +5,8 @@ import torch from mindspore import ops from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement -from msprobe.mindspore.api_accuracy_checker.const import IS_MINDSPORE_API, IS_TORCH_API, MINT, MINT_FUNCTIONAL +from msprobe.mindspore.api_accuracy_checker.const import (MINDSPORE_PLATFORM, TORCH_PLATFORM, MINT, MINT_FUNCTIONAL, + FORWARD_API) from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple @@ -14,19 +15,19 @@ from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple class ApiRunner: def __init__(self) -> None: self.api_parent_module_mapping = { - (MINT, IS_MINDSPORE_API): mindspore.mint, - (MINT, IS_TORCH_API): torch, - (MINT_FUNCTIONAL, IS_MINDSPORE_API): mindspore.mint.nn.functional, - (MINT_FUNCTIONAL, IS_TORCH_API): torch.nn.functional + (MINT, MINDSPORE_PLATFORM): mindspore.mint, + (MINT, TORCH_PLATFORM): torch, + (MINT_FUNCTIONAL, MINDSPORE_PLATFORM): mindspore.mint.nn.functional, + (MINT_FUNCTIONAL, TORCH_PLATFORM): torch.nn.functional } def __call__(self, inputs, api_name_str, kwargs, gradient_inputs=None, - is_forward=True, is_mindspore_api=IS_MINDSPORE_API): + forward_or_backward=FORWARD_API, api_platform=MINDSPORE_PLATFORM): ''' Args: inputs: List[ComputeElement] api_name_str: str - kwargs: dict + kwargs: dict{str: ComputeElement} gradient_inputs: Union[List[ComputeElement], None] is_forward: boolean is_mindspore_api: boolean @@ -38,15 +39,15 @@ class ApiRunner: run mindspore.mint/torch api ''' api_type_str, api_sub_name = self.get_info_from_name(api_name_str) - api_instance = self._get_api_instance(api_type_str, api_sub_name, is_mindspore_api) + api_instance = self.get_api_instance(api_type_str, api_sub_name, api_platform) - self._run_api(api_instance, inputs, kwargs, gradient_inputs, is_forward, is_mindspore_api) + self.run_api(api_instance, inputs, kwargs, gradient_inputs, forward_or_backward, api_platform) @classmethod def get_info_from_name(cls, api_name_str): ''' Args: - api_name_str: str, the key of data dict in api_info.json. e.g. "MintFunctional.relu.0" + api_name_str: str, the key of data dict in api_info.json. e.g. "MintFunctional.relu.0.backward" Return: api_type_str: str, Union["MintFunctional", "Mint"] @@ -63,7 +64,8 @@ class ApiRunner: return api_type_str, api_sub_name - def _get_api_instance(self, api_type_str, api_sub_name, is_mindspore_api): + @classmethod + def get_api_instance(cls, api_type_str, api_sub_name, api_platform): ''' Args: api_type_str: str, Union["MintFunctional", "Mint"] @@ -79,37 +81,40 @@ class ApiRunner: mindspore.mint.nn.functional.{api_sub_name} <--> torch.nn.functional.{api_sub_name} ''' - api_parent_module = self.api_parent_module_mapping.get((api_type_str, is_mindspore_api)) - module_str = "mindspore.mint." if is_mindspore_api else "torch." + api_parent_module = cls.api_parent_module_mapping.get((api_type_str, api_platform)) + module_str = "mindspore.mint." if api_platform == MINDSPORE_PLATFORM else "torch." submodule_str = "nn.functional." if api_type_str == MINT_FUNCTIONAL else "" full_api_name = module_str + submodule_str + api_sub_name if not hasattr(api_parent_module, api_sub_name): - err_msg = f"ApiRunner._get_api_instance failed: {full_api_name} is not found" + err_msg = f"ApiRunner.get_api_instance failed: {full_api_name} is not found" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ApiWrong)) api_instance = getattr(api_parent_module, api_sub_name) if not callable(api_instance): - err_msg = f"ApiRunner._get_api_instance failed: {full_api_name} is not callable" + err_msg = f"ApiRunner.get_api_instance failed: {full_api_name} is not callable" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ApiWrong)) return api_instance - def _run_api(self, api_instance, inputs, kwargs, gradient_inputs, is_forward, is_mindspore_api): - inputs = tuple(compute_element.get_parameter(get_origin=False, get_mindspore_tensor=is_mindspore_api) + @classmethod + def run_api(cls, api_instance, inputs, kwargs, gradient_inputs, forward_or_backward, api_platform): + inputs = tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) for compute_element in inputs) + kwargs = {key: value.get_parameter(get_origin=False, tensor_platform=api_platform) + for key, value in kwargs.items()} - if is_forward: + if forward_or_backward == FORWARD_API: forward_result = api_instance(*inputs, **kwargs) # can be single tensor or tuple forward_result_tuple = convert_to_tuple(forward_result) res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in forward_result_tuple] else: if gradient_inputs is None: - err_msg = f"ApiRunner._run_api failed: run backward api but gradient_inputs is missing" + err_msg = f"ApiRunner.run_api failed: run backward api but gradient_inputs is missing" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) gradient_inputs = \ - tuple(compute_element.get_parameter(get_origin=False, get_mindspore_tensor=is_mindspore_api) + tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) for compute_element in gradient_inputs) - if is_mindspore_api: + if api_platform == MINDSPORE_PLATFORM: grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_instance) backward_result = grad_func(*inputs, **kwargs, gradient_inputs) # can be single tensor or tuple backward_result_tuple = convert_to_tuple(backward_result) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py index 794416980..7f02da8cd 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -10,8 +10,11 @@ SKIP = "skip" #api_runner -IS_MINDSPORE_API = True -IS_TORCH_API = False +MINDSPORE_PLATFORM = "mindspore_platform" +TORCH_PLATFORM = "torch_platform" MINT = "Mint" -MINT_FUNCTIONAL = "MintFunctional" \ No newline at end of file +MINT_FUNCTIONAL = "MintFunctional" + +FORWARD_API = "forward_api" +BACKWARD_API = "backward_api" \ No newline at end of file -- Gitee From 54f0f962ef7e63b1366e453bdd6125fc574597e8 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 19 Aug 2024 11:44:49 +0800 Subject: [PATCH 344/791] api unmatch output result improve --- debug/accuracy_tools/msprobe/core/compare/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index bb0ab4014..493224685 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -349,10 +349,10 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): result_item.extend([CompareConst.N_A] * 8) else: result_item.extend([CompareConst.N_A] * 5) - summary_data = n_dict.get("summary")[index] - result_item.extend(summary_data) - summary_data = [CompareConst.N_A] * 4 - result_item.extend(summary_data) + npu_summary_data = n_dict.get("summary")[index] + result_item.extend(npu_summary_data) + bench_summary_data = [CompareConst.N_A] * 4 + result_item.extend(bench_summary_data) result_item.append(accuracy_check_res) result_item.append(err_msg) if npu_stack_info and index == 0: -- Gitee From 9084b9a5b0e2302c039120d1b0f3e071fc3dbb1b Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:08:29 +0000 Subject: [PATCH 345/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/common/utils.py | 34 ------------------- 1 file changed, 34 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 8c714b56b..fdd37337d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -51,40 +51,6 @@ def check_object_type(check_object, allow_type): raise CompareException(CompareException.INVALID_DATA_ERROR) -def check_file_or_directory_path(path, isdir=False): - """ - Function Description: - check whether the path is valid - Parameter: - path: the path to check - isdir: the path is dir or file - Exception Description: - when invalid data throw exception - """ - if isdir: - if not os.path.exists(path): - logger.error('The path {} is not exist.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not os.path.isdir(path): - logger.error('The path {} is not a directory.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not os.access(path, os.W_OK): - logger.error( - 'The path {} does not have permission to write. Please check the path permission'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - else: - if not os.path.isfile(path): - logger.error('{} is an invalid file or non-exist.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not os.access(path, os.R_OK): - logger.error( - 'The path {} does not have permission to read. Please check the path permission'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - class SoftlinkCheckException(Exception): pass -- Gitee From 17dfe0fac317dd1bd9bb6009b26b237f204008ba Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:09:39 +0000 Subject: [PATCH 346/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/api_accuracy_checker/common/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index cf8af8d2c..1ae9070f9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -1,6 +1,6 @@ import os import yaml -from msprobe.pytorch.api_accuracy_checker.common.utils import check_file_or_directory_path +from msprobe.pytorch.core.common.utils import check_file_or_directory_path from msprobe.core.common.file_check import FileOpen from msprobe.pytorch.pt_config import RunUTConfig -- Gitee From 804018fad22b8ecb7cd75708f7709abf685d700f Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:11:06 +0000 Subject: [PATCH 347/791] update debug/accuracy_tools/msprobe/pytorch/common/utils.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/common/utils.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index e29c78bcf..378ff1a3d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -295,24 +295,6 @@ def _create_logger(level=logging.INFO): logger_.addHandler(ch) return logger_ - -def get_json_contents(file_path): - ops = get_file_content_bytes(file_path) - try: - json_obj = json.loads(ops) - except ValueError as error: - logger.error('Failed to load "%s". %s', file_path, str(error)) - raise CompareException(CompareException.INVALID_FILE_ERROR) from error - if not isinstance(json_obj, dict): - logger.error('Json file %s, content is not a dictionary!', file_path) - raise CompareException(CompareException.INVALID_FILE_ERROR) - return json_obj - - -def get_file_content_bytes(file): - with FileOpen(file, 'rb') as file_handle: - return file_handle.read() - log_level = logging.DEBUG if os.environ.get("API_ACCURACY_CHECK_LOG_LEVEL") == "1" else logging.INFO logger = _create_logger(log_level) -- Gitee From 39616dec95b25cab3a345fd23ce848edc99f5688 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:11:56 +0000 Subject: [PATCH 348/791] update debug/accuracy_tools/msprobe/core/common/utils.py. Signed-off-by: jiangchangting1 --- .../msprobe/core/common/utils.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index c957b3622..990ede26b 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -547,3 +547,21 @@ def save_npy(data, filepath): except Exception as e: raise RuntimeError(f"save npy file {filepath} failed") from e change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + +def get_json_contents(file_path): + ops = get_file_content_bytes(file_path) + try: + json_obj = json.loads(ops) + except ValueError as error: + logger.error('Failed to load "%s". %s', file_path, str(error)) + raise CompareException(CompareException.INVALID_FILE_ERROR) from error + if not isinstance(json_obj, dict): + logger.error('Json file %s, content is not a dictionary!', file_path) + raise CompareException(CompareException.INVALID_FILE_ERROR) + return json_obj + + +def get_file_content_bytes(file): + with FileOpen(file, 'rb') as file_handle: + return file_handle.read() -- Gitee From 4a9d5e5688e609272b839e68010c8aa64441023a Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:13:12 +0000 Subject: [PATCH 349/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index bdb3d8fee..39755a6c5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -15,7 +15,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get check_small_value, check_norm_value, get_abs_bench_with_eps, get_ulp_err from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.core.common.const import Const, CompareConst -from msprobe.pytorch.common.utils import get_json_contents +from msprobe.core.common.utils import get_json_contents ResultInfo = namedtuple('ResultInfo', ['full_api_name', 'fwd_success_status', 'bwd_success_status', -- Gitee From 624bdf525a304759f18be7c9e574b2798b83dd0c Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:13:41 +0000 Subject: [PATCH 350/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py. Signed-off-by: jiangchangting1 --- .../pytorch/api_accuracy_checker/run_ut/run_overflow_check.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index 20e6112d6..4170f338f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -11,7 +11,7 @@ else: import torch from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import exec_api, generate_device_params, get_api_info -from msprobe.pytorch.common.utils import get_json_contents +from msprobe.core.common.utils import get_json_contents from msprobe.core.common.file_check import check_link from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward -- Gitee From 9d5912706c15758aed73b93c6d407ca3ab7a420d Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:14:15 +0000 Subject: [PATCH 351/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 4343e630b..38e878f7b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -34,7 +34,7 @@ from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward from msprobe.core.common.file_check import FileOpen, FileChecker, \ change_mode, check_file_suffix, check_link, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import get_json_contents +from msprobe.core.common.utils import get_json_contents from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTL, ATTLConfig, ApiData, move2device_exec -- Gitee From e9b1f77d889b0d7461841d33096bb0a1778a4220 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:16:52 +0000 Subject: [PATCH 352/791] update debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index ec6f375bf..9676bd312 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -44,7 +44,7 @@ from msprobe.core.common.utils import (CompareException, task_dumppath_get) from msprobe.core.common.file_check import FileCheckConst -from msprobe.pytorch.common.utils import get_json_contents, get_file_content_bytes +from msprobe.core.common.utils import get_json_contents, get_file_content_bytes class TestUtils(TestCase): @patch.object(logger, "error") -- Gitee From 6564081faec74a61066edc6c8ab8b62c93d1f456 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:18:09 +0000 Subject: [PATCH 353/791] update ut Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/run_ut/test_data_generate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py index 37e871b6c..8d8815d02 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py @@ -4,7 +4,7 @@ import unittest import copy from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import * -from msprobe.pytorch.common.utils import get_json_contents +from msprobe.core.common.utils import get_json_contents base_dir = os.path.dirname(os.path.realpath(__file__)) forward_file = os.path.join(base_dir, "forward.json") -- Gitee From aa58b82432441884293491e73d4bde4d2a02bb22 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:18:27 +0000 Subject: [PATCH 354/791] update debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py. Signed-off-by: jiangchangting1 --- .../test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py index a413524de..5be41d78f 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py @@ -5,7 +5,7 @@ import unittest import torch from unittest.mock import patch, DEFAULT from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import * -from msprobe.pytorch.common.utils import get_json_contents +from msprobe.core.common.utils import get_json_contents base_dir = os.path.dirname(os.path.realpath(__file__)) forward_file = os.path.join(base_dir, "forward.json") -- Gitee From 18526b60199133b54ed0f114425dd0b0710cc87b Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:31:11 +0000 Subject: [PATCH 355/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/api_accuracy_checker/common/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index 032770710..14478dfef 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -1,6 +1,6 @@ import os import yaml -from msprobe.pytorch.core.common.utils import check_file_or_directory_path +from msprobe.core.common.utils import check_file_or_directory_path from msprobe.core.common.utils import load_yaml from msprobe.pytorch.pt_config import RunUTConfig -- Gitee From 5b98d7181b17c89968c5bdcf0df60ddbdaad0dd3 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 06:53:16 +0000 Subject: [PATCH 356/791] update debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index b624345b8..5151d6f26 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -346,7 +346,7 @@ class TestUtils(TestCase): self.assertEqual(context.exception.code, CompareException.INVALID_TASK_ERROR) mock_error.assert_called_with("Compare is not required for overflow_check or free_benchmark.") - @patch('msprobe.pytorch.common.utils.get_file_content_bytes') + @patch('msprobe.core.common.utils.get_file_content_bytes') def test_get_json_contents_should_raise_exception(self, mock_get_file_content_bytes): mock_get_file_content_bytes.return_value = 'not a dict' with self.assertRaises(CompareException) as ce: -- Gitee From 5e279b935f8b4704e483e38a04e0941cf9504a0f Mon Sep 17 00:00:00 2001 From: qianggee Date: Mon, 19 Aug 2024 07:01:13 +0000 Subject: [PATCH 357/791] clear unused code --- debug/accuracy_tools/kj600/kj600/anomaly_inform.py | 1 - debug/accuracy_tools/kj600/kj600/optimizer_collect.py | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/debug/accuracy_tools/kj600/kj600/anomaly_inform.py b/debug/accuracy_tools/kj600/kj600/anomaly_inform.py index 301ac7692..485c06d4d 100644 --- a/debug/accuracy_tools/kj600/kj600/anomaly_inform.py +++ b/debug/accuracy_tools/kj600/kj600/anomaly_inform.py @@ -1,6 +1,5 @@ import smtplib from email.mime.text import MIMEText -import sqlite3 from datetime import datetime, timedelta from kj600.database import Database, ExceptionMessage diff --git a/debug/accuracy_tools/kj600/kj600/optimizer_collect.py b/debug/accuracy_tools/kj600/kj600/optimizer_collect.py index 285f17ca6..6a06c8d7c 100644 --- a/debug/accuracy_tools/kj600/kj600/optimizer_collect.py +++ b/debug/accuracy_tools/kj600/kj600/optimizer_collect.py @@ -1,10 +1,9 @@ from collections import defaultdict import torch import torch.distributed as dist -from kj600.visualizer import HeatmapVisualizer -def print_rank_0(message, debug=False, force=False): +def print_rank_0(message): if dist.is_initialized(): if dist.get_rank() == 0: print(message) -- Gitee From 94804d09ea46f1c221323d42010e97e132741046 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 07:03:02 +0000 Subject: [PATCH 358/791] update Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/common/test_common_utils.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py index 5802103c2..989a7ad0c 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py @@ -29,12 +29,6 @@ class TestUtils(unittest.TestCase): except Exception as e: self.fail(f"check_object_type raised exception {e}") - def test_check_file_or_directory_path(self): - try: - check_file_or_directory_path(__file__) - except Exception as e: - self.fail(f"check_file_or_directory_path raised exception {e}") - def test_create_directory(self): test_dir_name = 'test_dir' create_directory(test_dir_name) -- Gitee From 401242b4d0c7c8b02644c9ce5d29ab7c49eed26e Mon Sep 17 00:00:00 2001 From: qianggee Date: Mon, 19 Aug 2024 07:05:19 +0000 Subject: [PATCH 359/791] add file check --- .../accuracy_tools/kj600/kj600/file_check.py | 283 ++++++++++++++++++ .../accuracy_tools/kj600/kj600/module_hook.py | 9 +- .../kj600/kj600/module_spec_verifier.py | 7 - 3 files changed, 288 insertions(+), 11 deletions(-) create mode 100644 debug/accuracy_tools/kj600/kj600/file_check.py diff --git a/debug/accuracy_tools/kj600/kj600/file_check.py b/debug/accuracy_tools/kj600/kj600/file_check.py new file mode 100644 index 000000000..c567f9454 --- /dev/null +++ b/debug/accuracy_tools/kj600/kj600/file_check.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2022-2023. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" +import os +import re + +from msprobe.core.common.log import logger +from msprobe.core.common.exceptions import FileCheckException +from msprobe.core.common.const import FileCheckConst + + +class FileChecker: + """ + The class for check file. + + Attributes: + file_path: The file or dictionary path to be verified. + path_type: file or dictionary + ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability + file_type(str): The correct file type for file + """ + def __init__(self, file_path, path_type, ability=None, file_type=None, is_script=True): + self.file_path = file_path + self.path_type = self._check_path_type(path_type) + self.ability = ability + self.file_type = file_type + self.is_script = is_script + + @staticmethod + def _check_path_type(path_type): + if path_type not in [FileCheckConst.DIR, FileCheckConst.FILE]: + logger.error(f'The path_type must be {FileCheckConst.DIR} or {FileCheckConst.FILE}.') + raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR) + return path_type + + def common_check(self): + """ + 功能：用户校验基本文件权限：软连接、文件长度、是否存在、读写权限、文件属组、文件特殊字符 + 注意：文件后缀的合法性，非通用操作，可使用其他独立接口实现 + """ + check_path_exists(self.file_path) + check_link(self.file_path) + self.file_path = os.path.realpath(self.file_path) + check_path_length(self.file_path) + check_path_type(self.file_path, self.path_type) + self.check_path_ability() + if self.is_script: + check_path_owner_consistent(self.file_path) + check_path_pattern_vaild(self.file_path) + check_common_file_size(self.file_path) + check_file_suffix(self.file_path, self.file_type) + return self.file_path + + def check_path_ability(self): + if self.ability == FileCheckConst.WRITE_ABLE: + check_path_writability(self.file_path) + if self.ability == FileCheckConst.READ_ABLE: + check_path_readability(self.file_path) + if self.ability == FileCheckConst.READ_WRITE_ABLE: + check_path_readability(self.file_path) + check_path_writability(self.file_path) + + +class FileOpen: + """ + The class for open file by a safe way. + + Attributes: + file_path: The file or dictionary path to be opened. + mode(str): The file open mode + """ + SUPPORT_READ_MODE = ["r", "rb"] + SUPPORT_WRITE_MODE = ["w", "wb", "a", "ab"] + SUPPORT_READ_WRITE_MODE = ["r+", "rb+", "w+", "wb+", "a+", "ab+"] + + def __init__(self, file_path, mode, encoding='utf-8'): + self.file_path = file_path + self.mode = mode + self.encoding = encoding + self._handle = None + + def __enter__(self): + self.check_file_path() + binary_mode = "b" + if binary_mode not in self.mode: + self._handle = open(self.file_path, self.mode, encoding=self.encoding) + else: + self._handle = open(self.file_path, self.mode) + return self._handle + + def __exit__(self, exc_type, exc_val, exc_tb): + if self._handle: + self._handle.close() + + def check_file_path(self): + support_mode = self.SUPPORT_READ_MODE + self.SUPPORT_WRITE_MODE + self.SUPPORT_READ_WRITE_MODE + if self.mode not in support_mode: + logger.error("File open not support %s mode" % self.mode) + raise FileCheckException(FileCheckException.ILLEGAL_PARAM_ERROR) + check_link(self.file_path) + self.file_path = os.path.realpath(self.file_path) + check_path_length(self.file_path) + self.check_ability_and_owner() + check_path_pattern_vaild(self.file_path) + if os.path.exists(self.file_path): + check_common_file_size(self.file_path) + + def check_ability_and_owner(self): + if self.mode in self.SUPPORT_READ_MODE: + check_path_exists(self.file_path) + check_path_readability(self.file_path) + check_path_owner_consistent(self.file_path) + if self.mode in self.SUPPORT_WRITE_MODE and os.path.exists(self.file_path): + check_path_writability(self.file_path) + check_path_owner_consistent(self.file_path) + if self.mode in self.SUPPORT_READ_WRITE_MODE and os.path.exists(self.file_path): + check_path_readability(self.file_path) + check_path_writability(self.file_path) + check_path_owner_consistent(self.file_path) + + +def check_link(path): + abs_path = os.path.abspath(path) + if os.path.islink(abs_path): + logger.error('The file path {} is a soft link.'.format(path)) + raise FileCheckException(FileCheckException.SOFT_LINK_ERROR) + + +def check_path_length(path, name_length=None): + file_max_name_length = name_length if name_length else FileCheckConst.FILE_NAME_LENGTH + if len(path) > FileCheckConst.DIRECTORY_LENGTH or \ + len(os.path.basename(path)) > file_max_name_length: + logger.error('The file path length exceeds limit.') + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) + + +def check_path_exists(path): + if not os.path.exists(path): + logger.error('The file path %s does not exist.' % path) + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) + + +def check_path_readability(path): + if not os.access(path, os.R_OK): + logger.error('The file path %s is not readable.' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_path_writability(path): + if not os.access(path, os.W_OK): + logger.error('The file path %s is not writable.' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_path_executable(path): + if not os.access(path, os.X_OK): + logger.error('The file path %s is not executable.' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_other_user_writable(path): + st = os.stat(path) + if st.st_mode & 0o002: + logger.error('The file path %s may be insecure because other users have write permissions. ' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_path_owner_consistent(path): + file_owner = os.stat(path).st_uid + if file_owner != os.getuid(): + logger.error('The file path %s may be insecure because is does not belong to you.' % path) + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) + + +def check_path_pattern_vaild(path): + if not re.match(FileCheckConst.FILE_VALID_PATTERN, path): + logger.error('The file path %s contains special characters.' %(path)) + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) + + +def check_file_size(file_path, max_size): + file_size = os.path.getsize(file_path) + if file_size >= max_size: + logger.error(f'The size of file path {file_path} exceeds {max_size} bytes.') + raise FileCheckException(FileCheckException.FILE_TOO_LARGE_ERROR) + + +def check_common_file_size(file_path): + if os.path.isfile(file_path): + for suffix, max_size in FileCheckConst.FILE_SIZE_DICT.items(): + if file_path.endswith(suffix): + check_file_size(file_path, max_size) + break + + +def check_file_suffix(file_path, file_suffix): + if file_suffix: + if not file_path.endswith(file_suffix): + logger.error(f"The {file_path} should be a {file_suffix} file!") + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) + + +def check_path_type(file_path, file_type): + if file_type == FileCheckConst.FILE: + if not os.path.isfile(file_path): + logger.error(f"The {file_path} should be a file!") + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) + if file_type == FileCheckConst.DIR: + if not os.path.isdir(file_path): + logger.error(f"The {file_path} should be a dictionary!") + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) + + +def create_directory(dir_path): + """ + Function Description: + creating a directory with specified permissions + Parameter: + dir_path: directory path + Exception Description: + when invalid data throw exception + """ + dir_path = os.path.realpath(dir_path) + try: + os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) + except OSError as ex: + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, + 'Failed to create {}. Please check the path permission or disk space .{}'.format(dir_path, str(ex))) from ex + + +def check_path_before_create(path): + if path_len_exceeds_limit(path): + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, 'The file path length exceeds limit.') + + if not re.match(FileCheckConst.FILE_PATTERN, os.path.realpath(path)): + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, + 'The file path {} contains special characters.'.format(path)) + + +def change_mode(path, mode): + if not os.path.exists(path) or os.path.islink(path): + return + try: + os.chmod(path, mode) + except PermissionError as ex: + raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR, + 'Failed to change {} authority. {}'.format(path, str(ex))) from ex + + +def path_len_exceeds_limit(file_path): + return len(os.path.realpath(file_path)) > FileCheckConst.DIRECTORY_LENGTH or \ + len(os.path.basename(file_path)) > FileCheckConst.FILE_NAME_LENGTH + + +def check_file_type(path): + """ + Function Description: + determine if it is a file or a directory + Parameter: + path: path + Exception Description: + when neither a file nor a directory throw exception + """ + if os.path.isdir(path): + return FileCheckConst.DIR + elif os.path.isfile(path): + return FileCheckConst.FILE + else: + logger.error('Neither a file nor a directory.') + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) diff --git a/debug/accuracy_tools/kj600/kj600/module_hook.py b/debug/accuracy_tools/kj600/kj600/module_hook.py index c0741b360..930bc1d7a 100644 --- a/debug/accuracy_tools/kj600/kj600/module_hook.py +++ b/debug/accuracy_tools/kj600/kj600/module_hook.py @@ -6,8 +6,8 @@ from datetime import datetime import torch import torch.distributed as dist from torch.optim.optimizer import register_optimizer_step_pre_hook, register_optimizer_step_post_hook -from kj600.module_spec_verifier import get_config, validate_config_spec -from kj600.optimizer_collect import MixPrecsionOptimizerMon, print_rank_0, OptimizerMonFactory, MegatronDistributedOptimizerMon +from kj600.module_spec_verifier import validate_config_spec +from kj600.optimizer_collect import MixPrecsionOptimizerMon, print_rank_0, OptimizerMonFactory from kj600.features import eff_rank, get_sign_matches from kj600.visualizer import HeatmapVisualizer from kj600.anomaly_detect import AnomalyScanner, SummaryWriterWithAD @@ -15,7 +15,7 @@ from kj600.anomaly_inform import AnomalyInformFactory from kj600.module_metric import get_metrics, write_metrics_tensorboard, get_summary_writer_tag_name, TensorMetrics from kj600.distributed.wrap_distributed import api_register, create_hooks, op_aggregate from kj600.utils import print_warn_log, print_info_log, get_param_struct, check_path_length, check_path_pattern_valid, change_mode, FileCheckConst - +from kj600.file_check import FileOpen @@ -84,7 +84,8 @@ class TrainerMon: self.optimizer_context = defaultdict(OptimizerContext) self.cc_context = defaultdict(CommunicationContext) self.params_have_main_grad = params_have_main_grad - self.config = get_config(config_file_path) + with FileOpen(config_file_path, 'r') as f: + self.config = json.load(config_file_path) self.module_rank_list = self.config.get("module_ranks", []) self.eps = self.config.get('eps', 1e-8) self.ops = self.config.get('ops', []) diff --git a/debug/accuracy_tools/kj600/kj600/module_spec_verifier.py b/debug/accuracy_tools/kj600/kj600/module_spec_verifier.py index 395aa82f1..66ea28059 100644 --- a/debug/accuracy_tools/kj600/kj600/module_spec_verifier.py +++ b/debug/accuracy_tools/kj600/kj600/module_spec_verifier.py @@ -2,15 +2,8 @@ import json import re import abc import torch -from kj600.utils import check_file_valid_readable -def get_config(file_path='config.json'): - check_file_valid_readable(file_path) - with open(file_path, 'r') as file: - config = json.load(file) - return config - # 用于存储所有validator实现类的注册表 config_validator_registry = {} -- Gitee From f5c66f4218e1e4c4846e46160d439bc147849da7 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 19 Aug 2024 15:16:55 +0800 Subject: [PATCH 360/791] code review --- .../api_accuracy_checker/compute_element.py | 57 ++++++++++++------- .../mindspore/api_accuracy_checker/const.py | 3 + .../api_accuracy_checker/type_mapping.py | 27 ++++++++- 3 files changed, 65 insertions(+), 22 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index d171925cc..71515fc28 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -11,8 +11,10 @@ from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np ms_dtype_to_dtype_str, torch_dtype_to_dtype_str, dtype_str_to_ms_dtype, dtype_str_to_np_dtype, dtype_str_to_torch_dtype, type_to_api_info_type_str, - DEFAULT_CONSTRUCT_NP_DTYPE, TUPLE_TYPE_STR, - MINDSPORE_TENSOR_TYPE_STR) + DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE, TUPLE_TYPE_STR, + MINDSPORE_TENSOR_TYPE_STR, float_dtype_str_list, + int_dtype_str_list) +from msprobe.mindspore.api_accuracy_checker.const import MINDSPORE_PLATFORM, TORCH_PLATFORM from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context @@ -26,6 +28,7 @@ class MstensorMetaData: class ComputeElement: def __init__(self, compute_element_info=None, parameter=None): + self.supported_parameter_type = tuple(api_info_type_str_to_type.keys()) + tuple([torch.Tensor, tuple]) if parameter is not None: self._init_with_parameter(parameter) elif isinstance(compute_element_info, (list, dict)): @@ -50,8 +53,15 @@ class ComputeElement: logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) else: torch_dtype = dtype_str_to_torch_dtype.get(dtype_str) - np_ndarray_float64 = ms_tensor.astype(mindspore.float64).numpy() - torch_tensor = torch.from_numpy(np_ndarray_float64).to(torch_dtype) + + if dtype_str in float_dtype_str_list: + middle_dtype = mindspore.float64 + elif dtype_str in int_dtype_str_list: + middle_dtype = mindspore.int64 + else: + middle_dtype = mindspore.uint64 + np_ndarray = ms_tensor.astype(middle_dtype).numpy() + torch_tensor = torch.from_numpy(np_ndarray).to(torch_dtype) return torch_tensor @staticmethod @@ -71,21 +81,26 @@ class ComputeElement: logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) else: ms_dtype = dtype_str_to_ms_dtype.get(dtype_str) - np_ndarray_float64 = torch_tensor.to(torch.float64, copy=True).numpy() - ms_tensor = mindspore.Tensor.from_numpy(np_ndarray_float64).astype(ms_dtype) + + if dtype_str in float_dtype_str_list: + middle_dtype = torch.float64 + elif dtype_str in int_dtype_str_list: + middle_dtype = torch.int64 + np_ndarray = torch_tensor.to(middle_dtype, copy=True).numpy() + ms_tensor = mindspore.Tensor.from_numpy(np_ndarray).astype(ms_dtype) return ms_tensor @staticmethod def convert_inf_to_real_num(value, dtype_str): if value == float("inf"): - np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE) value = np.finfo(np_dtype).max elif value == float("-inf"): - np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + np_dtype = dtype_str_to_np_dtype.get(dtype_str, DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE) value = np.finfo(np_dtype).min return value - def get_parameter(self, get_origin=True, get_mindspore_tensor=True): + def get_parameter(self, get_origin=True, tensor_platform=MINDSPORE_PLATFORM): ''' Args: get_origin: boolean @@ -94,13 +109,13 @@ class ComputeElement: Return: parameter: Union[int, float, str, slice,tuple, torch.Tensor, mindspore.Tensor] ''' - if isinstance(self.parameter, (int, float, str, slice, torch.Tensor, tuple, mindspore.Tensor)): + if isinstance(self.parameter, self.supported_parameter_type): parameter_tmp = self.parameter elif isinstance(self.parameter, MstensorMetaData): mstensor_meta_data = self.parameter ms_dtype = dtype_str_to_ms_dtype.get(mstensor_meta_data.dtype_str) if global_context.get_is_constructed(): - np_dtype = dtype_str_to_np_dtype.get(mstensor_meta_data.dtype_str, DEFAULT_CONSTRUCT_NP_DTYPE) + np_dtype = dtype_str_to_np_dtype.get(mstensor_meta_data.dtype_str, DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE) ndarray = self._construct_ndarray(mstensor_meta_data.shape, mstensor_meta_data.maximum, mstensor_meta_data.minimum, np_dtype) else: @@ -108,13 +123,13 @@ class ComputeElement: parameter_tmp = mindspore.Tensor(ndarray, dtype=ms_dtype) else: err_msg = "ComputeElement.get_parameter failed: self.parameter type is not in " \ - "(int, float, str, slice, torch.Tensor, mindspore.Tensor, MstensorMetaData)" + "(int, float, str, slice, bool, torch.Tensor, mindspore.Tensor, MstensorMetaData)" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) # if necessary, do transfer - if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and not get_mindspore_tensor: + if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and tensor_platform == TORCH_PLATFORM: parameter = self.transfer_to_torch_tensor(parameter_tmp) - elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and get_mindspore_tensor: + elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and tensor_platform ==MINDSPORE_PLATFORM: parameter = self.transfer_to_mindspore_tensor(parameter_tmp) else: parameter = parameter_tmp @@ -192,17 +207,17 @@ class ComputeElement: def _init_with_parameter(self, parameter): self.parameter = parameter + if not isinstance(parameter, self.supported_parameter_type): + err_msg = "ComputeElement._init_with_parameter failed: " \ + "parameter type is not in (int, float, str, slice, bool, torch.Tensor, mindspore.Tensor)" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) if isinstance(parameter, mindspore.Tensor): self.shape = tuple(parameter.shape) self.dtype_str = ms_dtype_to_dtype_str.get(parameter.dtype) elif isinstance(parameter, torch.Tensor): self.shape = tuple(parameter.shape) self.dtype_str = torch_dtype_to_dtype_str.get(parameter.dtype) - elif isinstance(parameter, (int, float, str, slice, tuple)): - self.shape = tuple() - self.dtype_str =\ - TUPLE_TYPE_STR if isinstance(parameter, tuple) else type_to_api_info_type_str.get(type(parameter)) else: - err_msg = "ComputeElement._init_with_parameter failed: " \ - "parameter type is not in (int, float, str, slice, torch.Tensor, mindspore.Tensor)" - logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) \ No newline at end of file + self.shape = tuple() + self.dtype_str = \ + TUPLE_TYPE_STR if isinstance(parameter, tuple) else type_to_api_info_type_str.get(type(parameter)) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py new file mode 100644 index 000000000..c46372358 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -0,0 +1,3 @@ +# compute_element +MINDSPORE_PLATFORM = "mindspore_platform" +TORCH_PLATFORM = "torch_platform" \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py index 219d06afc..e622d06b4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py @@ -86,4 +86,29 @@ api_info_type_str_to_type = { } type_to_api_info_type_str = {value: key for key, value in api_info_type_str_to_type.items()} -DEFAULT_CONSTRUCT_NP_DTYPE = np.float64 \ No newline at end of file +DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE = np.float64 +DEFAULT_CONSTRUCT_NP_INT_DTYPE = np.float64 +DEFAULT_CONSTRUCT_NP_UINT_DTYPE = np.float64 + +float_dtype_str_list = [ + FLOAT16, + FLOAT32, + FLOAT64, + BFLOAT16, +] + +int_dtype_str_list = [ + INT8, + INT16, + INT32, + INT64, + BOOL, + INT4, +] + +uint_dtype_str_list = [ + UINT8, + UINT16, + UINT32, + UINT64, +] \ No newline at end of file -- Gitee From 029174678c9f95a351161bc690cd86b43020389e Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 19 Aug 2024 15:23:33 +0800 Subject: [PATCH 361/791] bug fic --- .../mindspore/api_accuracy_checker/compute_element.py | 2 +- .../api_accuracy_checker/test_compute_element.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 71515fc28..6d171ae43 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -28,7 +28,7 @@ class MstensorMetaData: class ComputeElement: def __init__(self, compute_element_info=None, parameter=None): - self.supported_parameter_type = tuple(api_info_type_str_to_type.keys()) + tuple([torch.Tensor, tuple]) + self.supported_parameter_type = tuple(type_to_api_info_type_str.keys()) + tuple([torch.Tensor, tuple]) if parameter is not None: self._init_with_parameter(parameter) elif isinstance(compute_element_info, (list, dict)): diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py index 7253f802a..9e88657e3 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -11,6 +11,7 @@ from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElemen from msprobe.mindspore.api_accuracy_checker.type_mapping import (FLOAT32, FLOAT_TYPE_STR, INT_TYPE_STR, TUPLE_TYPE_STR, STR_TYPE_STR, SLICE_TYPE_STR) from msprobe.mindspore.api_accuracy_checker.utils import global_context +from msprobe.mindspore.api_accuracy_checker.const import MINDSPORE_PLATFORM, TORCH_PLATFORM logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') logger = logging.getLogger(__name__) @@ -48,8 +49,8 @@ class TestClass: compute_element = ComputeElement(parameter=input_parameter) assert (compute_element.get_parameter(get_origin=True) == origin_parameter).all() - assert (compute_element.get_parameter(get_origin=False, get_mindspore_tensor=True) == mstensor_parameter).all() - assert (compute_element.get_parameter(get_origin=False, get_mindspore_tensor=False) == torchtensor_parameter).all() + assert (compute_element.get_parameter(get_origin=False, tensor_platform=MINDSPORE_PLATFORM) == mstensor_parameter).all() + assert (compute_element.get_parameter(get_origin=False, tensor_platform=TORCH_PLATFORM) == torchtensor_parameter).all() assert compute_element.get_shape() == shape assert compute_element.get_dtype() == dtype_str -- Gitee From 7b0602d192e11245ab882e7e339ac977b48e53a8 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 19 Aug 2024 15:33:00 +0800 Subject: [PATCH 362/791] api unmatch output result improve --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 2 +- debug/accuracy_tools/msprobe/core/compare/utils.py | 5 +---- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index be749e5aa..2882fe0b3 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -191,7 +191,7 @@ class Comparator: err_msg = get_error_message(n_value, b_value, npu_op_name, error_flag, error_file=error_file) result_list, err_msg = compare_ops_apply(n_value, b_value, error_flag, err_msg, relative_err=relative_err) - if npu_op_name != bench_op_name: + if npu_op_name != bench_op_name and bench_op_name != CompareConst.N_A: err_msg += " Fuzzy matching data, the comparison accuracy may be affected." result_list.append(err_msg) return result_list diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 493224685..c22dbedda 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -360,10 +360,7 @@ def get_un_match_accuracy(result, n_dict, md5_compare, summary_compare): else: result_item.append(CompareConst.NONE) if not md5_compare and not summary_compare and result_item[1] == CompareConst.N_A: - if index == 0: - result_item.extend(["-1"]) - else: - result_item.extend([CompareConst.NONE, "-1"]) + result_item.extend(["-1"]) result.append(result_item) -- Gitee From 2d0f7cada8ce9ccbab9062df8201962fec0fa286 Mon Sep 17 00:00:00 2001 From: qianggee Date: Mon, 19 Aug 2024 07:42:12 +0000 Subject: [PATCH 363/791] filter special char in log --- .../accuracy_tools/kj600/kj600/module_hook.py | 8 ++--- .../kj600/kj600/optimizer_collect.py | 8 ----- debug/accuracy_tools/kj600/kj600/utils.py | 36 +++++++++++++++++-- 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/kj600/kj600/module_hook.py b/debug/accuracy_tools/kj600/kj600/module_hook.py index 930bc1d7a..0e5a20e7c 100644 --- a/debug/accuracy_tools/kj600/kj600/module_hook.py +++ b/debug/accuracy_tools/kj600/kj600/module_hook.py @@ -7,14 +7,14 @@ import torch import torch.distributed as dist from torch.optim.optimizer import register_optimizer_step_pre_hook, register_optimizer_step_post_hook from kj600.module_spec_verifier import validate_config_spec -from kj600.optimizer_collect import MixPrecsionOptimizerMon, print_rank_0, OptimizerMonFactory +from kj600.optimizer_collect import MixPrecsionOptimizerMon, OptimizerMonFactory from kj600.features import eff_rank, get_sign_matches from kj600.visualizer import HeatmapVisualizer from kj600.anomaly_detect import AnomalyScanner, SummaryWriterWithAD from kj600.anomaly_inform import AnomalyInformFactory from kj600.module_metric import get_metrics, write_metrics_tensorboard, get_summary_writer_tag_name, TensorMetrics from kj600.distributed.wrap_distributed import api_register, create_hooks, op_aggregate -from kj600.utils import print_warn_log, print_info_log, get_param_struct, check_path_length, check_path_pattern_valid, change_mode, FileCheckConst +from kj600.utils import print_warn_log, print_info_log, print_rank_0, get_param_struct, check_path_length, check_path_pattern_valid, change_mode, FileCheckConst from kj600.file_check import FileOpen @@ -85,7 +85,7 @@ class TrainerMon: self.cc_context = defaultdict(CommunicationContext) self.params_have_main_grad = params_have_main_grad with FileOpen(config_file_path, 'r') as f: - self.config = json.load(config_file_path) + self.config = json.load(f) self.module_rank_list = self.config.get("module_ranks", []) self.eps = self.config.get('eps', 1e-8) self.ops = self.config.get('ops', []) @@ -265,7 +265,7 @@ class TrainerMon: context = self.optimizer_context[optimizer] if self.print_struct and not all(value == {} for value in self.module_struct.values()) and not self.struct_printed: self._smallest_rank_print("> module struct:") - self._smallest_rank_print(json.dumps(self.module_struct, indent=4)) + self._smallest_rank_print(json.dumps(self.module_struct)) self.struct_printed = True if not self.cc_log_only: raise Exception("exit after first step when print model struct") diff --git a/debug/accuracy_tools/kj600/kj600/optimizer_collect.py b/debug/accuracy_tools/kj600/kj600/optimizer_collect.py index 6a06c8d7c..b5337aa01 100644 --- a/debug/accuracy_tools/kj600/kj600/optimizer_collect.py +++ b/debug/accuracy_tools/kj600/kj600/optimizer_collect.py @@ -3,14 +3,6 @@ import torch import torch.distributed as dist -def print_rank_0(message): - if dist.is_initialized(): - if dist.get_rank() == 0: - print(message) - else: - print(message) - - class MixPrecsionOptimizerMon: wrapped_optimizer = None diff --git a/debug/accuracy_tools/kj600/kj600/utils.py b/debug/accuracy_tools/kj600/kj600/utils.py index 0d300addf..856687c3e 100644 --- a/debug/accuracy_tools/kj600/kj600/utils.py +++ b/debug/accuracy_tools/kj600/kj600/utils.py @@ -2,12 +2,32 @@ import os import time import sys import re +from functools import wraps +from torch import distributed as dist FILE_MAX_SIZE = 10 * 1024 * 1024 * 1024 FILE_NAME_MAX_LENGTH = 255 DIRECTORY_MAX_LENGTH = 4096 FILE_NAME_VALID_PATTERN = r"^[a-zA-Z0-9_.:/-]+$" + + +class MsgConst: + """ + Class for log messages const + """ + SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] + + +def filter_special_chars(func): + @wraps(func) + def func_level(msg): + for char in MsgConst.SPECIAL_CHAR: + msg = msg.replace(char, '_') + return func(msg) + return func_level + + class FileCheckConst: """ Class for file check const @@ -61,6 +81,15 @@ class FileCheckException(Exception): def __str__(self): return self.error_info + +def print_rank_0(message): + if dist.is_initialized(): + if dist.get_rank() == 0: + print(message) + else: + print(message) + + def _print_log(level, msg, end='\n'): current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(int(time.time()))) pid = os.getgid() @@ -68,16 +97,18 @@ def _print_log(level, msg, end='\n'): sys.stdout.flush() -def print_info_log(info_msg, end='\n'): +@filter_special_chars +def print_info_log(info_msg): """ Function Description: print info log. Parameter: info_msg: the info message. """ - _print_log("INFO", info_msg, end=end) + _print_log("INFO", info_msg) +@filter_special_chars def print_error_log(error_msg): """ Function Description: @@ -88,6 +119,7 @@ def print_error_log(error_msg): _print_log("ERROR", error_msg) +@filter_special_chars def print_warn_log(warn_msg): """ Function Description: -- Gitee From 1593a24503aec68c97648b6140730cf64dcb275c Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 07:47:29 +0000 Subject: [PATCH 364/791] update Signed-off-by: jiangchangting1 --- .../common/test_common_utils.py | 31 ------------------- 1 file changed, 31 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py index 989a7ad0c..00e3e9333 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py @@ -35,37 +35,6 @@ class TestUtils(unittest.TestCase): self.assertTrue(os.path.exists(test_dir_name)) os.rmdir(test_dir_name) - @patch('os.path.exists') - def test_check_file_or_dir_path_should_raise_exe_when_dir_path_not_existed(self, mock_path_exists): - mock_path_exists.return_value = False - with self.assertRaises(CompareException) as ce: - check_file_or_directory_path('', isdir=True) - self.assertEqual(ce.exception.code, CompareException.INVALID_PATH_ERROR) - - @patch('os.path.exists') - @patch('os.path.isdir') - @patch('os.access') - def test_check_file_or_dir_path_should_pass_when_path_is_dir(self, mock_os_access, mock_path_is_dir, - mock_path_exists): - mock_os_access.return_value = True - mock_path_is_dir.return_value = True - mock_path_exists.return_value = True - check_file_or_directory_path('', isdir=True) - - @patch('os.path.isfile') - @patch('os.access') - def test_check_file_or_dir_path_should_raise_exe_when_file_not_access(self, mock_os_access, mock_path_is_file): - mock_os_access.return_value = False - mock_path_is_file.return_value = True - with self.assertRaises(CompareException) as ce: - check_file_or_directory_path('', isdir=False) - self.assertEqual(ce.exception.code, CompareException.INVALID_PATH_ERROR) - - def test_check_file_or_dir_path_should_pass_when_path_is_file(self): - with unittest.mock.patch('os.path.isfile', return_value=True), \ - unittest.mock.patch('os.access', return_value=True): - check_file_or_directory_path('', isdir=False) - def test_api_info_preprocess_no_conversion_needed(self): api_name = 'linear' original_api_info = {'key': 'value'} -- Gitee From e27a08b38d18dffccb40e5120b43b94d6b34d2ff Mon Sep 17 00:00:00 2001 From: lijiaojiao Date: Thu, 15 Aug 2024 18:42:44 +0800 Subject: [PATCH 365/791] =?UTF-8?q?=E3=80=90=E9=97=AE=E9=A2=98=E5=8D=95?= =?UTF-8?q?=E3=80=91=E6=97=A0=E7=94=A8=E4=BB=A3=E7=A0=81=E4=BB=A5=E5=8F=8A?= =?UTF-8?q?=E8=BE=93=E5=87=BA=E4=BB=B6=E6=9D=83=E9=99=90=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pytorch/online_dispatch/compare.py | 6 +-- .../pytorch/online_dispatch/dispatch.py | 40 ++++++++--------- .../pytorch/online_dispatch/dump_compare.py | 42 +++-------------- .../pytorch/online_dispatch/single_compare.py | 10 ++--- .../msprobe/pytorch/online_dispatch/utils.py | 45 +------------------ .../online_dispatch/test_dump_compare.py | 12 +---- .../test_utils_online_dispatch.py | 32 +------------ 7 files changed, 38 insertions(+), 149 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py index 048ab3f90..985e0f743 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py @@ -6,10 +6,9 @@ import json from collections import namedtuple from rich.table import Table from rich.console import Console +from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.file_check import FileOpen, change_mode from .single_compare import single_benchmark_compare_wrap -from .utils import DispatchException -from msprobe.core.common.const import CompareConst -from msprobe.core.common.file_check import FileOpen from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException @@ -42,6 +41,7 @@ def write_csv(data, filepath): with FileOpen(filepath, 'a', encoding='utf-8-sig') as f: writer = csv.writer(f) writer.writerows(data) + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) class Saver: diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index aaae8640c..f63460950 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -17,10 +17,10 @@ else: from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, load_yaml from msprobe.core.common.const import Const, CompareConst +from msprobe.pytorch.common.log import logger from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ DispatchRunParam, DisPatchDataInfo -from .utils import get_callstack, data_to_cpu, logger_debug, logger_error, logger_warn, logger_logo, get_sys_info, \ - DispatchException +from .utils import get_callstack, data_to_cpu, get_sys_info, DispatchException, COMPARE_LOGO from .compare import Comparator @@ -32,12 +32,12 @@ DETAILS_FILE_NAME = "accuracy_checking_details_" + current_time + ".csv" class PtdbgDispatch(TorchDispatchMode): def __init__(self, dump_mode=Const.OFF, api_list=None, debug=False, dump_path=None, tag=None, process_num=0): super(PtdbgDispatch, self).__init__() - logger_logo() + logger.info(COMPARE_LOGO) if not is_npu: - logger_error("Please confirm you run environment installed torch_npu!") + logger.error("Please confirm you run environment installed torch_npu!") return if dump_path is None: - logger_error("Please set dump_path when dump_mode is config!") + logger.error("Please set dump_path when dump_mode is config!") check_file_or_directory_path(dump_path, True) self.device_id = torch_npu._C._npu_getDevice() @@ -75,7 +75,7 @@ class PtdbgDispatch(TorchDispatchMode): if process_num > 0: self.pool = Pool(process_num) if debug: - logger_debug(f'Main pid:{os.getpid()} device:{self.device_id} dump_list:{self.dump_api_list} ' + logger.info(f'Main pid:{os.getpid()} device:{self.device_id} dump_list:{self.dump_api_list} ' f'dump_mode:{self.dump_mode} cpu_path[{self.root_cpu_path}], npu_path[{self.root_npu_path}], ' f'process[{process_num}]') @@ -84,14 +84,14 @@ class PtdbgDispatch(TorchDispatchMode): if not is_npu: return - logger_debug(f'start write compare csv: Rank[{self.device_id}], Pid[{os.getpid()}') + logger.info(f'start write compare csv: Rank[{self.device_id}], Pid[{os.getpid()}') if self.process_num > 0: self.pool.close() self.pool.join() summary_path = os.path.join(self.root_cpu_path, f'summary.json') if not os.path.exists(summary_path): - logger_error("Please check train log, An exception may have occurred!") + logger.error("Please check train log, An exception may have occurred!") return check_file_or_directory_path(summary_path, False) fp_handle = open(summary_path, "r") @@ -112,18 +112,18 @@ class PtdbgDispatch(TorchDispatchMode): for list_data in self.all_summary: for data in list_data: - logger_debug(f'summary: Device[{self.device_id}], Pid[{os.getpid()}], Data[{data}]') + logger.info(f'summary: Device[{self.device_id}], Pid[{os.getpid()}], Data[{data}]') if "_input" in data[CompareConst.NPU_NAME]: input_num = input_num + 1 if "_output" in data[CompareConst.NPU_NAME]: output_num = output_num + 1 total_num = total_num + 1 - logger_debug(f'Dispatch exit: Device[{self.device_id}], Pid[{os.getpid()} Input[{input_num}] ' + logger.info(f'Dispatch exit: Device[{self.device_id}], Pid[{os.getpid()} Input[{input_num}] ' f'Output[{output_num}] Total[{total_num}] API_Total[{self.api_index}]]') def __torch_dispatch__(self, func, types, args=(), kwargs=None): if not is_npu: - logger_error("Please confirm you run environment installed torch_npu!") + logger.error("Please confirm you run environment installed torch_npu!") return func(*args, **kwargs) func_name_split_list = func.__name__.split(".") @@ -131,7 +131,7 @@ class PtdbgDispatch(TorchDispatchMode): try: aten_api_overload_name = func_name_split_list[1] except IndexError: - logger_error(f"Please check the func name {func.__name__}!") + logger.error(f"Please check the func name {func.__name__}!") return func(*args, **kwargs) self.enable_autogard(aten_api) @@ -150,7 +150,7 @@ class PtdbgDispatch(TorchDispatchMode): run_param = self.get_run_param(aten_api, func.__name__, aten_api_overload_name) if self.debug_flag: - logger_debug(f'Dispatch Info: Rank[{self.device_id}], Pid[{os.getpid()}], Func[{func.__name__}], ' + logger.info(f'Dispatch Info: Rank[{self.device_id}], Pid[{os.getpid()}], Func[{func.__name__}], ' f'Name[{run_param.aten_api}_{run_param.single_api_index}], ' f'Count[{self.api_index}], Sys[{get_sys_info()}]') @@ -188,7 +188,7 @@ class PtdbgDispatch(TorchDispatchMode): self.pool.apply_async(func=dispatch_multiprocess, args=(run_param, data_info), error_callback=error_call) else: - logger_error("can not get correct function please set process_num=0") + logger.error("can not get correct function please set process_num=0") return npu_out @staticmethod @@ -207,7 +207,7 @@ class PtdbgDispatch(TorchDispatchMode): time.sleep(1) time_now = time.strftime("%Y%m%d%H%M%S", time.localtime(time.time())) if tag is None or not isinstance(tag, str): - logger_warn('There is not tag or the type of tag is not string.') + logger.warning('There is not tag or the type of tag is not string.') dir_name = f'msprobe_rank{self.device_id}_{time_now}' else: dir_name = f'msprobe_{tag}_rank{self.device_id}_{time_now}' @@ -228,7 +228,7 @@ class PtdbgDispatch(TorchDispatchMode): if aten_api in aten_api_list: dump_api_list.append(aten_api) else: - logger_warn(f'{aten_api} is not aten api will not dump, please refer to torch.ops.aten') + logger.warning(f'{aten_api} is not aten api will not dump, please refer to torch.ops.aten') self.dump_api_list = dump_api_list def get_run_param(self, aten_api, func_name, aten_api_overload_name): @@ -255,16 +255,16 @@ class PtdbgDispatch(TorchDispatchMode): def check_param(self): if self.dump_mode not in Const.ONLINE_DUMP_MODE: - logger_error('The parameter "dump mode" can only be one of {}.'.format(Const.ONLINE_DUMP_MODE)) + logger.error('The parameter "dump mode" can only be one of {}.'.format(Const.ONLINE_DUMP_MODE)) raise DispatchException(DispatchException.INVALID_PARAMETER) if not isinstance(self.dump_api_list, list): - logger_error('The type of parameter "api_list" can only be list.') + logger.error('The type of parameter "api_list" can only be list.') raise DispatchException(DispatchException.INVALID_PARAMETER) if not isinstance(self.debug_flag, bool): - logger_error('The type of parameter "debug" can only be bool.') + logger.error('The type of parameter "debug" can only be bool.') raise DispatchException(DispatchException.INVALID_PARAMETER) if not isinstance(self.process_num, int) or self.process_num < 0: - logger_error('The type of parameter "process_num" can only be int and it should not be less than 0.') + logger.error('The type of parameter "process_num" can only be int and it should not be less than 0.') raise DispatchException(DispatchException.INVALID_PARAMETER) def enable_autogard(self, aten_api): diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py index 4d0453308..5b2e94e3c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py @@ -5,11 +5,10 @@ from datetime import datetime, timezone import pandas as pd import torch -from .utils import np_save_data, logger_debug, logger_error, logger_warn, logger_user, COLOR_RED, COLOR_GREEN, \ - COLOR_RESET, CSV_COLUMN_NAME -from msprobe.core.common.file_check import FileOpen, change_mode -from msprobe.core.common.const import CompareConst, FileCheckConst, Const from msprobe.pytorch.common.log import logger +from msprobe.core.common.file_check import FileOpen +from .utils import np_save_data + class DispatchRunParam: def __init__(self, debug_flag, device_id, root_npu_path, root_cpu_path, process_num, comparator): @@ -57,7 +56,7 @@ class TimeStatistics: def __enter__(self): if self.debug: self.time = datetime.now(tz=timezone.utc) - logger_debug(f'Time[{self.tag}]-ENTER: Dev[{self.device}], Pid[{os.getpid()}], Fun[{self.fun}], ' \ + logger.info(f'Time[{self.tag}]-ENTER: Dev[{self.device}], Pid[{os.getpid()}], Fun[{self.fun}], ' \ f'Id[{self.index}]') def __exit__(self, exc_type, exc_val, exc_tb): @@ -68,9 +67,9 @@ class TimeStatistics: hot_time_cost = "Hotspot " + time_cost if cost_time.total_seconds() > self.timeout: - logger_debug(hot_time_cost) + logger.info(hot_time_cost) else: - logger_debug(time_cost) + logger.info(time_cost) def support_basic_type(data): @@ -155,32 +154,3 @@ def dispatch_multiprocess(run_param, dispatch_data_info): def error_call(err): logger.error(f'multiprocess {err}') - -def save_csv(all_summary, call_stack_list, csv_path): - df = pd.DataFrame(columns=CSV_COLUMN_NAME) - - for index, list_data in enumerate(all_summary): - for data in list_data: - csv_row_data = {CompareConst.NPU_NAME: data[CompareConst.NPU_NAME], - CompareConst.BENCH_NAME: data[CompareConst.BENCH_NAME], - CompareConst.NPU_DTYPE: data[CompareConst.NPU_DTYPE], - CompareConst.BENCH_DTYPE: data[CompareConst.BENCH_DTYPE], - CompareConst.NPU_SHAPE: data[CompareConst.NPU_SHAPE], - CompareConst.BENCH_SHAPE: data[CompareConst.BENCH_SHAPE], - CompareConst.NPU_MAX: data[CompareConst.NPU_MAX], - CompareConst.NPU_MIN: data[CompareConst.NPU_MIN], - CompareConst.NPU_MEAN: data[CompareConst.NPU_MEAN], - CompareConst.BENCH_MAX: data[CompareConst.BENCH_MAX], - CompareConst.BENCH_MIN: data[CompareConst.BENCH_MIN], - CompareConst.BENCH_MEAN: data[CompareConst.BENCH_MEAN], - CompareConst.COSINE: data[CompareConst.COSINE], - CompareConst.MAX_ABS_ERR: data[CompareConst.MAX_ABS_ERR], - CompareConst.MAX_RELATIVE_ERR: data[CompareConst.MAX_RELATIVE_ERR], - CompareConst.ACCURACY: data[CompareConst.ACCURACY], - CompareConst.STACK: call_stack_list[index], - CompareConst.ERROR_MESSAGE: data[CompareConst.ERROR_MESSAGE]} - row_df = pd.DataFrame.from_dict(csv_row_data, orient='index').T - df = pd.concat([df, row_df]) - - df.to_csv(csv_path, index=False) - change_mode(csv_path, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py index aa0afa4e4..1408b4778 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py @@ -3,15 +3,15 @@ from functools import wraps import torch from prettytable import PrettyTable from collections import namedtuple -from .utils import logger_user, logger_debug +from msprobe.pytorch.common.log import logger def func_log_wrapper(): def _out_wrapper(func): @wraps(func) def _in_wrapper(*kargs, **kwargs): - logger_debug("start to run: {}".format(func.__name__)) + logger.info(f"start to run: {func.__name__}") x = func(*kargs, **kwargs) - logger_debug("end to run: {}".format(func.__name__)) + logger.info(f"end to run: {func.__name__}") return x return _in_wrapper @@ -165,7 +165,7 @@ class SingleBenchmarkAccuracyCompare: def compute_binary_diff(cls, npu_out, bench_out): result = torch.equal(npu_out, bench_out) if result: - logger_user("二进制精度比对通过, 无需单标杆比对法验证") + logger.info("二进制精度比对通过, 无需单标杆比对法验证") return SingleBenchmarkAccuracyResult(result=result, max_abs_diff=0, max_rel_diff=0, error_balance=0) @classmethod @@ -301,7 +301,7 @@ class SingleBenchSummary: table.add_row(["max_rel_diff", self.max_rel_diff, self.error_thd]) table.add_row(["max_rel_idx", self.max_rel_idx, "-"]) - logger_user(table) + logger.info(table) def to_column_value(self): return [self.bench_dtype, self.npu_dtype, self.shape, self.error_balance, diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index fec3e0b00..596ab090a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -1,6 +1,5 @@ import os import inspect -import logging import psutil import torch import numpy as np @@ -14,6 +13,7 @@ else: from msprobe.core.common.const import CompareConst, FileCheckConst from msprobe.core.common.file_check import change_mode +from msprobe.core.common.log import logger cpu_device = torch._C.device("cpu") COLOR_RED = '\033[31m' @@ -77,7 +77,7 @@ def np_save_data(data, file_name, data_path): np.save(dump_path, data) change_mode(dump_path, FileCheckConst.DATA_FILE_AUTHORITY) except Exception as e: - logger_error("save numpy failed, error: {}".format(e)) + logger.error("save numpy failed, error: {}".format(e)) finally: pass @@ -124,47 +124,6 @@ def data_to_cpu(data, deep, data_cpu): return data -def get_mp_logger(): - logger = logging.getLogger(__name__) - if not logger.handlers: - logger.setLevel(logging.INFO) - handler = logging.StreamHandler() - formatter = logging.Formatter('%(asctime)s %(message)s') - logger.propagate = True - handler.setFormatter(formatter) - logger.addHandler(handler) - return logger.info - - -def logger_debug(mesg): - logger = get_mp_logger() - logger(f'DEBUG ' + mesg) - - -def logger_info(mesg): - logger = get_mp_logger() - logger(f'INFO ' + mesg) - - -def logger_warn(mesg): - logger = get_mp_logger() - logger(f'{COLOR_YELLOW}WARNING {mesg} {COLOR_RESET}') - - -def logger_error(mesg): - logger = get_mp_logger() - logger(f'{COLOR_RED}ERROR {mesg} {COLOR_RESET}') - - -def logger_user(mesg): - logger = get_mp_logger() - logger(mesg) - - -def logger_logo(): - logger_user(f'{COLOR_CYAN}{COMPARE_LOGO} {COLOR_RESET}') - - def get_sys_info(): mem = psutil.virtual_memory() cpu_percent = psutil.cpu_percent(interval=1) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_dump_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_dump_compare.py index 1393884ce..4c1a0f7df 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_dump_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_dump_compare.py @@ -22,7 +22,7 @@ from unittest.mock import Mock, patch import pandas as pd from msprobe.core.common.const import CompareConst -from msprobe.pytorch.online_dispatch.dump_compare import support_basic_type, dump_data, save_temp_summary, dispatch_workflow, get_torch_func, dispatch_multiprocess, error_call, save_csv +from msprobe.pytorch.online_dispatch.dump_compare import support_basic_type, dump_data, save_temp_summary, dispatch_workflow, get_torch_func, dispatch_multiprocess, error_call @@ -30,8 +30,6 @@ class TestDumpCompare(unittest.TestCase): def setUp(self): self.summary_path = "summary.json" Path(self.summary_path).touch() - self.csv_path = "test_save_csv.csv" - Path(self.csv_path).touch() self.data = {CompareConst.NPU_NAME: 1, CompareConst.BENCH_NAME: 1, CompareConst.NPU_DTYPE: 1, @@ -71,8 +69,6 @@ class TestDumpCompare(unittest.TestCase): def tearDown(self): if os.path.exists(self.summary_path): os.remove(self.summary_path) - if os.path.exists(self.csv_path): - os.remove(self.csv_path) def test_support_basic_type_should_return_true_when_is_instance(self): self.assertTrue(support_basic_type(2.3)) @@ -175,9 +171,3 @@ class TestDumpCompare(unittest.TestCase): def test_error_call(self,mock_error): error_call("messages") mock_error.assert_called_once_with("multiprocess messages") - - def test_save_csv(self): - save_csv([[self.data]],[2],self.csv_path) - df = pd.read_csv(self.csv_path) - df_gt = pd.DataFrame.from_dict(self.data_gt, orient='index').T - self.assertTrue((df.all()==df_gt.all()).all()) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py index 90a646956..764d844ad 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py @@ -23,7 +23,7 @@ import logging from pathlib import Path from unittest.mock import patch, MagicMock -from msprobe.pytorch.online_dispatch.utils import COLOR_RED, COLOR_CYAN, COLOR_YELLOW, COLOR_RESET, COMPARE_LOGO, get_callstack, np_save_data, data_to_cpu, logger_debug, logger_info, logger_warn, logger_error, logger_user, logger_logo, DispatchException +from msprobe.pytorch.online_dispatch.utils import COLOR_RED, COLOR_CYAN, COLOR_YELLOW, COLOR_RESET, COMPARE_LOGO, np_save_data, data_to_cpu, DispatchException cpu_device = torch._C.device("cpu") @@ -65,35 +65,5 @@ class TestUtils(unittest.TestCase): data_cpu=[] self.assertEqual(data_to_cpu(data,deep,data_cpu), data) - @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') - def test_logger_debug(self,mock_inf0): - logger_debug("messages") - mock_inf0.return_value.assert_called_once_with("DEBUG messages") - - @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') - def test_logger_info(self,mock_info): - logger_info("messages") - mock_info.return_value.assert_called_once_with("INFO messages") - - @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') - def test_logger_warn(self,mock_info): - logger_warn("messages") - mock_info.return_value.assert_called_once_with(f'{COLOR_YELLOW}WARNING messages {COLOR_RESET}') - - @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') - def test_logger_error(self,mock_info): - logger_error("messages") - mock_info.return_value.assert_called_once_with(f'{COLOR_RED}ERROR messages {COLOR_RESET}') - - @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') - def test_logger_user(self,mock_info): - logger_user("messages") - mock_info.return_value.assert_called_once_with("messages") - - @patch('msprobe.pytorch.online_dispatch.utils.get_mp_logger') - def test_logger_logo(self,mock_info): - logger_logo() - mock_info.return_value.assert_called_once_with(f'{COLOR_CYAN}{COMPARE_LOGO} {COLOR_RESET}') - def test_str(self): self.assertEqual(self.dispatch_exception.__str__(),"messages") \ No newline at end of file -- Gitee From 7a45d30a7100b5a071026acb178f2866a465f3be Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 19 Aug 2024 17:09:10 +0800 Subject: [PATCH 366/791] update setup install --- debug/accuracy_tools/msprobe/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 211943e3c..fe447bbc6 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -97,11 +97,12 @@ Successfully installed mindstudio_probe-{version} 3. 安装msprobe ```shell - python setup.py install + python setup.py bdist_wheel + pip install dist/mindstudio_probe*.whl ``` 提示出现如下信息则表示源码安装成功。 ```shell - Finished processing dependencies for mindstudio-probe=={version} + Successfully installed ... mindstudio_probe-{version} ... ``` ### 查看msprobe工具信息 -- Gitee From 3f8a20bea8ba811486189175632b6de9add2e62e Mon Sep 17 00:00:00 2001 From: jijiarong Date: Tue, 13 Aug 2024 14:58:18 +0800 Subject: [PATCH 367/791] mindspore graph compare --- .../core/compare/multiprocessing_compute.py | 31 +- .../msprobe/core/compare/npy_compare.py | 53 +++ .../mindspore/compare/ms_graph_compare.py | 336 ++++++++++++++++++ 3 files changed, 419 insertions(+), 1 deletion(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py diff --git a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py index da63005e5..c39176fe2 100644 --- a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py @@ -1,6 +1,8 @@ import multiprocessing from dataclasses import dataclass +from functools import partial +import numpy as np import pandas as pd from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException @@ -39,6 +41,33 @@ def _handle_multi_process(func, input_parma, result_df, lock): return pd.concat(final_results, ignore_index=True) +def _ms_graph_handle_multi_process(func, result_df, mode): + process_num = int((multiprocessing.cpu_count() + 1) / 2) + df_chunk_size = len(result_df) // process_num + if df_chunk_size > 0: + df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] + else: + df_chunks = [result_df] + + results = [] + pool = multiprocessing.Pool(process_num) + + def err_call(args): + logger.error('multiprocess compare failed! Reason: {}'.format(args)) + try: + pool.terminate() + except OSError as e: + logger.error("pool terminate failed") + + for _, df_chunk in enumerate(df_chunks): + result = pool.apply_async(func, args=(df_chunk, mode), error_callback=err_call) + results.append(result) + final_results = [r.get() for r in results] + pool.close() + pool.join() + return pd.concat(final_results, ignore_index=True) + + def read_dump_data(result_df): try: npu_dump_name_list = result_df.iloc[0:, 0].tolist() @@ -117,4 +146,4 @@ def check_accuracy(cos, max_abs_err): return CompareConst.ACCURACY_CHECK_NO if cos < CompareConst.COS_MAX_THRESHOLD or max_abs_err > CompareConst.MAX_ABS_ERR_MAX_THRESHOLD: return CompareConst.ACCURACY_CHECK_NO - return CompareConst.ACCURACY_CHECK_YES \ No newline at end of file + return CompareConst.ACCURACY_CHECK_YES diff --git a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py index 279f950f9..39bb87826 100644 --- a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py @@ -76,6 +76,59 @@ def get_error_message(n_value, b_value, npu_op_name, error_flag, error_file=None return "" +def npy_data_check(n_value, b_value): + error_flag = False + error_message = "" + if n_value is None or b_value is None: + error_flag = True + error_message += "Dump file not found." + + if not error_flag: + if n_value.size == 0 or b_value.size == 0: + error_flag = True + error_message += "This is empty data, can not compare." + + if not error_flag: + if not n_value.shape or not b_value.shape: + error_flag = True + error_message += "This is type of scalar data, can not compare." + if n_value.shape != b_value.shape: + error_flag = True + error_message += "Shape of NPU and bench Tensor do not match." + if n_value.dtype != b_value.dtype: + error_flag = True + error_message += "Dtype of NPU and bench Tensor do not match. Skipped." + + if not error_flag: + n_value, b_value = handle_inf_nan(n_value, b_value) # 判断是否有nan/inf数据 + if n_value is CompareConst.NAN or b_value is CompareConst.NAN: + error_flag = True + error_message += "The position of inf or nan in NPU and bench Tensor do not match." + + return error_flag, error_message + + +def statistics_data_check(result_dict): + error_flag = False + error_message = "" + if result_dict['NPU Name'] is None or result_dict['NPU Name'] is None: + error_flag = True + error_message += "Dump file not found." + + if not error_flag: + if not result_dict['NPU Tensor Shape'] or not result_dict['Bench Tensor Shape']: + error_flag = True + error_message = "This is type of scalar data, can not compare." + if result_dict['NPU Tensor Shape'] != result_dict['Bench Tensor Shape']: + error_flag = True + error_message += "This is type of scalar data, can not compare." + if result_dict['Bench Dtype'] != result_dict['Bench Dtype']: + error_flag = True + error_message += "Dtype of NPU and bench Tensor do not match. Skipped." + + return error_flag, error_message + + class TensorComparisonBasic(abc.ABC): """NPU和bench中npy数据的比较模板""" @abc.abstractmethod diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py new file mode 100644 index 000000000..16d09403b --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -0,0 +1,336 @@ +import csv +import glob +import os +import sys + +import numpy as np +import pandas as pd +from msprobe.core.common.const import CompareConst +from msprobe.core.common.exceptions import FileCheckException +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.log import logger +from msprobe.core.common.utils import add_time_with_xlsx, CompareException +from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy +from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply + + +class row_data: + def __init__(self, mode): + self.basic_data = { + CompareConst.NPU_NAME: None, CompareConst.BENCH_NAME: None, CompareConst.NPU_DTYPE: None, CompareConst.BENCH_DTYPE: None, CompareConst.NPU_SHAPE: None, + CompareConst.BENCH_SHAPE: None, CompareConst.NPU_MAX: None, CompareConst.NPU_MIN: None, CompareConst.NPU_MEAN: None, CompareConst.NPU_NORM: None, + CompareConst.BENCH_MAX: None, CompareConst.BENCH_MIN: None, CompareConst.BENCH_MEAN: None, CompareConst.BENCH_NORM: None, + CompareConst.ACCURACY: '', CompareConst.ERROR_MESSAGE: '' + } + self.npy_data = { + CompareConst.COSINE: None, CompareConst.MAX_ABS_ERR: None, CompareConst.MAX_RELATIVE_ERR: None, CompareConst.ONE_THOUSANDTH_ERR_RATIO: None, + CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: None + } + self.statistic_data = { + CompareConst.MAX_DIFF: None, CompareConst.MIN_DIFF: None, CompareConst.MEAN_DIFF: None, CompareConst.NORM_DIFF: None, CompareConst.MAX_RELATIVE_ERR: None, + CompareConst.MIN_RELATIVE_ERR: None, CompareConst.MEAN_RELATIVE_ERR: None, CompareConst.NORM_RELATIVE_ERR: None + } + if mode == 'NPY_MODE': + self.data = {**self.basic_data, **self.npy_data} + else: + self.data = {**self.basic_data, **self.statistic_data} + + def __call__(self): + return self.data + + +def generate_step(npu_path, rank_id): + step_set = set() + rank_path = os.path.join(npu_path, f"rank_{rank_id}") + for path in os.listdir(rank_path): + if path not in ["execution_order", "graphs"]: + data_path = os.path.join(rank_path, path) + for graph_path in os.listdir(data_path): + step_set.update([int(i) for i in os.listdir(os.path.join(data_path, graph_path))]) + return sorted(list(step_set)) + + +def generate_path_by_rank_step(base_path, rank_id, step_id): + path_with_rank_id = os.path.join(base_path, f"rank_{rank_id}") + for path in os.listdir(path_with_rank_id): + if path not in ["execution_order", "graphs"]: + # TODO: graph id path need remove + return os.path.join(path_with_rank_id, path, "*", str(step_id)) + logger.error(f"Data_path {path_with_rank_id} is not exist.") + return '' + + +def generate_data_name(data_path): + data_list = [] + + mapping_path = os.path.join(data_path, "mapping.csv") + statistic_path = os.path.join(data_path, "statistic.csv") + npy_path = os.path.join(data_path, "*.npy") + + mapping_file_list = glob.glob(mapping_path) + statistic_file_list = glob.glob(statistic_path) + npy_file_list = glob.glob(npy_path) + + mapping_exist = True if mapping_file_list else False + statistic_exist = True if statistic_file_list else False + npy_exist = True if npy_file_list else False + + if npy_exist: + mapping_dict = [] + if mapping_exist: + for mapping_file in mapping_file_list: + with open(mapping_file, "r") as f: + csv_reader = csv.reader(f, delimiter=",") + header = next(csv_reader) + for row in csv_reader: + mapping_dict[row[0]] = row[1] + for data in npy_file_list: + if data in mapping_dict: + split_list = mapping_dict[data].split(".") + else: + split_list = data.split(".") + compare_key = f"{split_list[1]}.{split_list[2]}.{split_list[3]}.{split_list[5]}.{split_list[6]}" + timestamp = int(split_list[4]) + + data_list.append([os.path.join(data_path, data), compare_key, timestamp]) + elif statistic_exist: + statistic_data_list = [] + for statistic_file in statistic_file_list: + with open(statistic_file, "r") as f: + csv_reader = csv.reader(f, delimiter=",") + header = next(csv_reader) + header_index = {'Data Type': None, 'Shape': None, 'Max Value': None, 'Min Value': None, + 'Avg Value': None, 'L2Norm Value': None} + for key in header_index.keys(): + for index, value in enumerate(header): + if key == value: + header_index[key] = index + for key in header_index.keys(): + if header_index[key] is None: + logger.error(f"Data_path {data_path} has no key {key}") + raise FileCheckException(f"Data_path {data_path} has no key {key}") + statistic_data_list.extend([row for row in csv_reader]) + + for data in statistic_data_list: + compare_key = f"{data[1]}.{data[2]}.{data[3]}.{data[5]}" + timestamp = int(data[4]) + data_list.append( + [os.path.join(data_path, statistic_path), compare_key, timestamp, data[header_index['Data Type']], + data[header_index['Shape']], data[header_index['Max Value']], data[header_index['Min Value']], + data[header_index['Avg Value']], data[header_index['L2Norm Value']]]) + + if npy_exist: + mode = "NPY_MODE" + elif statistic_exist: + mode = "STATISTIC_MODE" + else: + mode = "ERROR_MODE" + logger.error(f"Error mode.") + return mode, data_list + + +def read_npy_data(data_path): + try: + data_value = np.load(data_path) + if data_value.dtype == np.float16: + data_value = data_value.astype(np.float32) + except FileNotFoundError as e: + data_value = None + return data_value + + +class GraphMSComparator: + def __init__(self, input_param, output_path): + self.output_path = output_path + self.base_npu_path = input_param.get('npu_path', None) + self.base_bench_path = input_param.get('bench_path', None) + self.rank_list = input_param.get('rank_list', []) + self.step_list = input_param.get('step_list', []) + + @staticmethod + def compare_ops(compare_result_db, mode): + + def npy_mode_compute(row): + result_dict = row_data('NPY_MODE')() + n_value = None + b_value = None + + if os.path.exists(row[CompareConst.NPU_NAME]): + n_value = read_npy_data(row[CompareConst.NPU_NAME]) + result_dict[CompareConst.NPU_NAME] = row[CompareConst.NPU_NAME] + result_dict[CompareConst.NPU_DTYPE] = n_value.dtype + result_dict[CompareConst.NPU_SHAPE] = n_value.shape + result_dict[CompareConst.NPU_MAX] = np.max(n_value) + result_dict[CompareConst.NPU_MIN] = np.min(n_value) + result_dict[CompareConst.NPU_MEAN] = np.mean(n_value) + result_dict[CompareConst.NPU_NORM] = np.linalg.norm(n_value) + + if os.path.exists(row[CompareConst.BENCH_NAME]): + b_value = read_npy_data(row[CompareConst.BENCH_NAME]) + result_dict[CompareConst.BENCH_NAME] = row[CompareConst.BENCH_NAME] + result_dict[CompareConst.BENCH_DTYPE] = b_value.dtype + result_dict[CompareConst.BENCH_SHAPE] = b_value.shape + result_dict[CompareConst.BENCH_MAX] = np.max(b_value) + result_dict[CompareConst.BENCH_MIN] = np.min(b_value) + result_dict[CompareConst.BENCH_MEAN] = np.mean(b_value) + result_dict[CompareConst.BENCH_NORM] = np.linalg.norm(b_value) + + error_flag, error_message = npy_data_check(n_value, b_value) + result_dict[CompareConst.ERROR_MESSAGE] = error_message + + if not error_flag: + n_value, b_value = reshape_value(n_value, b_value) + result_list, err_msg = compare_ops_apply(n_value, b_value, False, "") + result_dict[CompareConst.COSINE] = result_list[0] + result_dict[CompareConst.MAX_ABS_ERR] = result_list[1] + result_dict[CompareConst.MAX_RELATIVE_ERR] = result_list[2] + result_dict[CompareConst.ONE_THOUSANDTH_ERR_RATIO] = result_list[3] + result_dict[CompareConst.FIVE_THOUSANDTHS_ERR_RATIO] = result_list[4] + result_dict[CompareConst.ACCURACY] = check_accuracy(result_list[0], result_list[1]) + result_dict[CompareConst.ERROR_MESSAGE] = err_msg + + return pd.Series(result_dict) + + def statistic_mode_compute(row): + result_dict = row_data('STATISTIC')() + + result_dict[CompareConst.NPU_NAME] = row[CompareConst.NPU_NAME] + result_dict[CompareConst.NPU_DTYPE] = row[CompareConst.NPU_DTYPE] + result_dict[CompareConst.NPU_SHAPE] = row[CompareConst.NPU_SHAPE] + result_dict[CompareConst.NPU_MAX] = np.float32(row[CompareConst.NPU_MAX]) + result_dict[CompareConst.NPU_MIN] = np.float32(row[CompareConst.NPU_MIN]) + result_dict[CompareConst.NPU_MEAN] = np.float32(row[CompareConst.NPU_MEAN]) + result_dict[CompareConst.NPU_NORM] = np.float32(row[CompareConst.NPU_NORM]) + + result_dict[CompareConst.BENCH_NAME] = row[CompareConst.BENCH_NAME] + result_dict[CompareConst.BENCH_DTYPE] = row[CompareConst.BENCH_DTYPE] + result_dict[CompareConst.BENCH_SHAPE] = row[CompareConst.BENCH_SHAPE] + result_dict[CompareConst.BENCH_MAX] = np.float32(row[CompareConst.BENCH_MAX]) + result_dict[CompareConst.BENCH_MIN] = np.float32(row[CompareConst.BENCH_MIN]) + result_dict[CompareConst.BENCH_MEAN] = np.float32(row[CompareConst.BENCH_MEAN]) + result_dict[CompareConst.BENCH_NORM] = np.float32(row[CompareConst.BENCH_NORM]) + + error_flag, error_message = statistics_data_check(result_dict) + result_dict[CompareConst.ERROR_MESSAGE] += error_message + if not error_flag: + # TODO: check Algorithms + result_dict[CompareConst.MAX_DIFF] = np.abs( + result_dict[CompareConst.NPU_MAX] - result_dict[CompareConst.BENCH_MAX]) + result_dict[CompareConst.MIN_DIFF] = np.abs( + result_dict[CompareConst.NPU_MIN] - result_dict[CompareConst.BENCH_MIN]) + result_dict[CompareConst.MEAN_DIFF] = np.abs( + result_dict[CompareConst.NPU_MEAN] - result_dict[CompareConst.BENCH_MEAN]) + result_dict[CompareConst.NORM_DIFF] = np.abs( + result_dict[CompareConst.NPU_NORM] - result_dict[CompareConst.BENCH_NORM]) + result_dict[CompareConst.MAX_RELATIVE_ERR] = result_dict[CompareConst.MAX_DIFF] / result_dict[ + CompareConst.BENCH_MAX] if result_dict[CompareConst.BENCH_MAX] > 0 else 0 + result_dict[CompareConst.MAX_RELATIVE_ERR] = str(result_dict[CompareConst.MAX_RELATIVE_ERR] * 100) + "%" + result_dict[CompareConst.MIN_RELATIVE_ERR] = result_dict[CompareConst.MIN_DIFF] / result_dict[ + CompareConst.BENCH_MIN] if result_dict[CompareConst.BENCH_MIN] > 0 else 0 + result_dict[CompareConst.MIN_RELATIVE_ERR] = str(result_dict[CompareConst.MIN_RELATIVE_ERR] * 100) + "%" + result_dict[CompareConst.MEAN_RELATIVE_ERR] = result_dict[CompareConst.MEAN_DIFF] / result_dict[ + CompareConst.BENCH_MEAN] if result_dict[CompareConst.BENCH_MEAN] > 0 else 0 + result_dict[CompareConst.MEAN_RELATIVE_ERR] = str( + result_dict[CompareConst.MEAN_RELATIVE_ERR] * 100) + "%" + result_dict[CompareConst.NORM_RELATIVE_ERR] = result_dict[CompareConst.NORM_DIFF] / result_dict[ + CompareConst.BENCH_NORM] if result_dict[CompareConst.BENCH_NORM] > 0 else 0 + result_dict[CompareConst.NORM_RELATIVE_ERR] = str( + result_dict[CompareConst.NORM_RELATIVE_ERR] * 100) + "%" + magnitude_diff = result_dict[CompareConst.MAX_DIFF] / ( + max(result_dict[CompareConst.NPU_MAX], result_dict[CompareConst.BENCH_MAX]) + 1e-10) + if magnitude_diff > 0.5: + result_dict[CompareConst.ACCURACY] = 'No' + else: + result_dict[CompareConst.ACCURACY] = 'Yes' + + return pd.Series(result_dict) + + if mode == "NPY_MODE": + compare_result_db = compare_result_db.apply(npy_mode_compute, axis=1) + else: + compare_result_db = compare_result_db.apply(statistic_mode_compute, axis=1) + return compare_result_db + + def compare_core(self): + logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") + + # split by rank and step + if not self.rank_list: + self.rank_list = [int(i.split("_")[-1]) for i in os.listdir(self.base_npu_path)] + for rank_id in self.rank_list: + if not self.step_list: + self.step_list = generate_step(self.base_npu_path, rank_id) + for step_id in self.step_list: + compare_result_df, mode = self.compare_process(rank_id, step_id) + if isinstance(compare_result_df, list): + is_empty = not compare_result_df + elif isinstance(compare_result_df, pd.DataFrame): + is_empty = compare_result_df.empty + else: + is_empty = True + if is_empty or not mode: + continue + compare_result_df = self._do_multi_process(compare_result_df, mode) + compare_result_name = add_time_with_xlsx(f"compare_result_{str(rank_id)}_{str(rank_id)}") + compare_result_path = os.path.join(os.path.realpath(self.output_path), f"{compare_result_name}") + # TODO:重新排布 + compare_result_df.to_excel(compare_result_path, index=False) + logger.info(f"Compare rank: {rank_id} step: {step_id} finish. Compare result: {compare_result_path}.") + + def compare_process(self, rank_id, step_id): + # generate data_path + npu_data_path = generate_path_by_rank_step(self.base_npu_path, rank_id, step_id) + bench_data_path = generate_path_by_rank_step(self.base_bench_path, rank_id, step_id) + if not npu_data_path or not bench_data_path: + return [], '' + + # generate file name + npu_mode, npu_data_list = generate_data_name(npu_data_path) + match_mode, match_data_list = generate_data_name(bench_data_path) + + if npu_mode == "ERROR_MODE" or match_mode == "ERROR_MODE": + logger.error(f"Data_path {npu_data_path} or {bench_data_path} is not exist.") + return [], '' + if npu_mode != match_mode: + logger.error(f"NPU mode {npu_mode} not equal to MATCH mode {match_mode}.") + return [], '' + + if npu_mode == 'NPY_MODE': + npu_data_df = pd.DataFrame(npu_data_list, columns=[CompareConst.NPU_NAME, 'Compare Key', 'TimeStamp']) + bench_data_df = pd.DataFrame(match_data_list, columns=[CompareConst.BENCH_NAME, 'Compare Key', 'TimeStamp']) + else: + npu_data_df = pd.DataFrame(npu_data_list, + columns=[CompareConst.NPU_NAME, 'Compare Key', 'TimeStamp', CompareConst.NPU_DTYPE, CompareConst.NPU_SHAPE, + CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM]) + bench_data_df = pd.DataFrame(match_data_list, + columns=[CompareConst.BENCH_NAME, 'Compare Key', 'TimeStamp', CompareConst.BENCH_DTYPE, + CompareConst.BENCH_SHAPE, CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN, + CompareConst.BENCH_NORM]) + + npu_data_df['Local Index'] = npu_data_df.sort_values('TimeStamp').groupby('Compare Key').cumcount() + bench_data_df['Local Index'] = bench_data_df.sort_values('TimeStamp').groupby('Compare Key').cumcount() + + compare_result_df = pd.merge(npu_data_df, bench_data_df, on=['Compare Key', 'Local Index'], how='outer') + + compare_result_df[CompareConst.NPU_NAME] = compare_result_df[CompareConst.NPU_NAME].fillna('') + compare_result_df[CompareConst.BENCH_NAME] = compare_result_df[CompareConst.BENCH_NAME].fillna('') + + return compare_result_df, npu_mode + + def _do_multi_process(self, result_df, mode): + try: + result_df = _ms_graph_handle_multi_process(self.compare_ops, result_df, mode) + return result_df + except ValueError as e: + logger.error('result dataframe is not found.') + raise CompareException(CompareException.INVALID_DATA_ERROR) from e + + +def ms_graph_compare(inputs, outputs): + try: + create_directory(outputs) + except (CompareException, FileCheckException) as error: + logger.error('Compare failed. Please check the arguments and do it again!') + return + msComparator = GraphMSComparator(inputs, outputs) + msComparator.compare_core() -- Gitee From dc5ca41662ca4527655f9cf1f88fffec4a657929 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 10:07:01 +0000 Subject: [PATCH 368/791] update debug/accuracy_tools/msprobe/core/common/utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/core/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 6ffe2402c..423afeaab 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -559,10 +559,10 @@ def get_json_contents(file_path): try: json_obj = json.loads(ops) except ValueError as error: - logger.error('Failed to load "%s". %s', file_path, str(error)) + logger.error('Failed to load "%s". %s' % (file_path, str(error)) raise CompareException(CompareException.INVALID_FILE_ERROR) from error if not isinstance(json_obj, dict): - logger.error('Json file %s, content is not a dictionary!', file_path) + logger.error('Json file %s, content is not a dictionary!' % file_path) raise CompareException(CompareException.INVALID_FILE_ERROR) return json_obj -- Gitee From 788799000a446cde7272550503bbb75869e24c39 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 10:18:09 +0000 Subject: [PATCH 369/791] update debug/accuracy_tools/msprobe/core/common/utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/core/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 423afeaab..def30f316 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -559,7 +559,7 @@ def get_json_contents(file_path): try: json_obj = json.loads(ops) except ValueError as error: - logger.error('Failed to load "%s". %s' % (file_path, str(error)) + logger.error('Failed to load %s. %s' % (file_path, str(error)) raise CompareException(CompareException.INVALID_FILE_ERROR) from error if not isinstance(json_obj, dict): logger.error('Json file %s, content is not a dictionary!' % file_path) -- Gitee From cb9b008e83bb25524a373a17a3183af3a34ecfc1 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 10:25:54 +0000 Subject: [PATCH 370/791] update debug/accuracy_tools/msprobe/core/common/utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/core/common/utils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index def30f316..caa699a02 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -559,10 +559,8 @@ def get_json_contents(file_path): try: json_obj = json.loads(ops) except ValueError as error: - logger.error('Failed to load %s. %s' % (file_path, str(error)) raise CompareException(CompareException.INVALID_FILE_ERROR) from error if not isinstance(json_obj, dict): - logger.error('Json file %s, content is not a dictionary!' % file_path) raise CompareException(CompareException.INVALID_FILE_ERROR) return json_obj -- Gitee From f52c91eb6dc2106daf7114ff59687577a73a7996 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 19 Aug 2024 18:51:35 +0800 Subject: [PATCH 371/791] check path of config_json and api_info_file --- .../api_accuracy_checker/run_ut/multi_run_ut.py | 2 +- .../pytorch/api_accuracy_checker/run_ut/run_ut.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index 9acb5ee64..64669e56b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -117,7 +117,7 @@ def run_parallel_ut(config): for api_info in config.api_files: cmd = create_cmd(api_info, next(device_id_cycle)) - process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1) + process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, bufsize=1, shell=False) processes.append(process) threading.Thread(target=read_process_output, args=(process,), daemon=True).start() diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 7e5891b5a..e6786af49 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -560,10 +560,10 @@ def run_ut_command(args): # 离线场景下，forward_content, backward_content, real_data_path从api_info_file中解析 forward_content, backward_content, real_data_path = None, None, None if args.api_info_file: - check_link(args.api_info_file) - api_info = os.path.realpath(args.api_info_file) - check_file_suffix(api_info, FileCheckConst.JSON_SUFFIX) - forward_content, backward_content, real_data_path = parse_json_info_forward_backward(api_info) + api_info_file_checker = FileChecker(file_path = args.api_info_file, path_type = FileCheckConst.FILE, \ + ability = FileCheckConst.READ_ABLE, file_type = FileCheckConst.JSON_SUFFIX) + checked_api_info = api_info_file_checker.common_check() + forward_content, backward_content, real_data_path = parse_json_info_forward_backward(checked_api_info) if args.filter_api: logger.info("Start filtering the api in the forward_input_file.") forward_content = preprocess_forward_content(forward_content) @@ -591,7 +591,10 @@ def run_ut_command(args): rank_list = msCheckerConfig.rank_list tls_path = msCheckerConfig.tls_path if args.config_path: - _, task_config = parse_json_config(args.config_path, Const.RUN_UT) + config_path_checker = FileChecker(args.config_path, FileCheckConst.FILE, + FileCheckConst.READ_ABLE, FileCheckConst.JSON_SUFFIX) + checked_config_path = config_path_checker.common_check() + _, task_config = parse_json_config(checked_config_path, Const.RUN_UT) white_list = task_config.white_list black_list = task_config.black_list error_data_path = task_config.error_data_path -- Gitee From 16d0b5ce9fa7934230172352b4b5ea9344c4c3fc Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 19 Aug 2024 18:54:32 +0800 Subject: [PATCH 372/791] check path of config_json and api_info_file --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index e6786af49..d55e38ac9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -560,8 +560,8 @@ def run_ut_command(args): # 离线场景下，forward_content, backward_content, real_data_path从api_info_file中解析 forward_content, backward_content, real_data_path = None, None, None if args.api_info_file: - api_info_file_checker = FileChecker(file_path = args.api_info_file, path_type = FileCheckConst.FILE, \ - ability = FileCheckConst.READ_ABLE, file_type = FileCheckConst.JSON_SUFFIX) + api_info_file_checker = FileChecker(file_path = args.api_info_file, path_type = FileCheckConst.FILE, + ability = FileCheckConst.READ_ABLE, file_type = FileCheckConst.JSON_SUFFIX) checked_api_info = api_info_file_checker.common_check() forward_content, backward_content, real_data_path = parse_json_info_forward_backward(checked_api_info) if args.filter_api: -- Gitee From f5aad7487c2c604a3990aa0d3177b043ec921a50 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 19 Aug 2024 19:51:51 +0800 Subject: [PATCH 373/791] api_info + api_info_ut + api_runner_ut --- .../api_accuracy_checker/api_info.py | 73 ++++++++++++++++ .../mindspore/api_accuracy_checker/const.py | 6 +- .../api_accuracy_checker/test_api_info.py | 80 +++++++++++++++++ .../api_accuracy_checker/test_api_runner.py | 86 +++++++++++++++++++ 4 files changed, 244 insertions(+), 1 deletion(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py new file mode 100644 index 000000000..64609a1e3 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py @@ -0,0 +1,73 @@ +from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement +from msprobe.mindspore.api_accuracy_checker.const import FORWARD_API, BACKWARD_API, INPUT, OUTPUT +from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict +from msprobe.core.common.exceptions import ApiAccuracyCheckerException +from msprobe.core.common.log import logger + +class ApiInfo: + def __init__(self, api_name): + self.api_name = api_name + self.forward_info = None + self.backward_info = None + self.has_forward_info = False + self.has_backward_info = False + + def load_forward_info(self, forward_info_dict): + self.forward_info = forward_info_dict + self.has_forward_info = True + + def load_backward_info(self, backward_info_dict): + self.backward_info = backward_info_dict + self.has_backward_info = True + + def check_forward_info(self): + return self.has_forward_info + + def check_backward_info(self): + return self.has_backward_info + + def get_compute_element_list(self, forward_or_backward, input_or_output): + ''' + Args: + forward_or_backward: str, Union["forward_api", "backward_api"] + input_or_output: str, Union["input", "output"] + + Return: + compute_element_list: List[ComputeElement] + ''' + mapping = { + (FORWARD_API, INPUT): [self.forward_info, "input_args", + f"input_args field of {self.api_name} forward api in api_info.json"], + (FORWARD_API, OUTPUT): [self.forward_info, "output", + f"output field of {self.api_name} forward api in api_info.json"], + (BACKWARD_API, INPUT): [self.backward_info, "input", + f"input field of {self.api_name} backward api in api_info.json"], + (BACKWARD_API, OUTPUT): [self.backward_info, "output", + f"output field of {self.api_name} backward api in api_info.json"] + } + dict_instance, key, key_desc = mapping[(forward_or_backward, input_or_output)] + compute_element_info_list = check_and_get_from_json_dict(dict_instance, key, key_desc, accepted_type=list) + compute_element_list = [ComputeElement(compute_element_info=compute_element_info) + for compute_element_info in compute_element_info_list] + return compute_element_list + + def get_kwargs(self): + ''' + Return: + kwargs_compute_element_dict: dict{str: ComputeElement} + ''' + kwargs_dict = check_and_get_from_json_dict(self.forward_info, "input_kwargs", + "input_kwargs in api_info.json", accepted_type=dict) + for key_str, compute_element_info in kwargs_dict.items(): + if not isinstance(key_str, str): + err_msg = "ApiInfo.get_kwargs failed: compute_element_dict key is not a string" + logger.error_log_with_exp(err_msg, + ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed)) + if not isinstance(compute_element_info, (list, dict)): + err_msg = "ApiInfo.get_kwargs failed: compute_element_dict value is not a list or dict" + logger.error_log_with_exp(err_msg, + ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed)) + kwargs_compute_element_dict = {key_str: ComputeElement(compute_element_info=compute_element_info) + for key_str, compute_element_info in kwargs_dict} + return kwargs_compute_element_dict + diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py index 11a87dec6..96ebc073e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -18,4 +18,8 @@ MINT = "Mint" MINT_FUNCTIONAL = "MintFunctional" FORWARD_API = "forward_api" -BACKWARD_API = "backward_api" \ No newline at end of file +BACKWARD_API = "backward_api" + +#api_info +INPUT = "input" +OUTPUT = "output" \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py new file mode 100644 index 000000000..1346ceaf9 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py @@ -0,0 +1,80 @@ +import sys +import logging +import os + +import pytest +import mindspore + +from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo +from msprobe.mindspore.api_accuracy_checker.utils import global_context + +logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + + + +class TestClass: + @classmethod + def setup_class(cls): + """ + class level setup_class + """ + cls.init(TestClass) + + def init(self): + global_context.init(False, os.path.join(directory, "files")) + + def test_get_compute_element_list(self): + # first load forward backward api_info + forward_api_info_dict = { + "input_args": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy", + } + ], + "input_kwargs": { + "approximate": { + "type": "str", + "value": "tanh", + } + }, + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy", + } + ], + } + + api_info = ApiInfo("MintFuntional.gelu.0.forward") + api_info.load_forward_info(forward_api_info_dict) + + assert api_info.check_forward_info == True + assert api_info.check_backward_info == False + + input_compute_element_list = api_info.get_compute_element_list("forward_api", "input") + parameter_real = input_compute_element_list[0].get_parameter() + parameter_target = mindspore.Tensor([1., 2., 3.]) + assert parameter_real == parameter_target + + kwargs_compute_element_dict = api_info.get_kwargs() + assert kwargs_compute_element_dict.get("approximate").get_parameter() == "tanh" + diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py new file mode 100644 index 000000000..4b8ea36aa --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py @@ -0,0 +1,86 @@ +import sys +import logging +import os + +import pytest +import mindspore +import torch +from unittest.mock import MagicMock + +from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner +from msprobe.mindspore.api_accuracy_checker.const import MINDSPORE_PLATFORM, TORCH_PLATFORM, FORWARD_API, BACKWARD_API + +logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + + +# 创建一个包含if判断的mock实例的fixture +@pytest.fixture +def mock_compute_element_input_instance(): + mock = MagicMock() + def side_effect(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([1., 2., 3.]) + else: + return torch.Tensor([1., 2., 3.]) + mock.get_parameter.side_effect = side_effect + return mock + +@pytest.fixture +def mock_compute_element_kwargs_instance(): + mock = MagicMock() + mock.get_parameter.return_value = "tanh" + return mock + +@pytest.fixture +def mock_compute_element_result_instance(): + mock = MagicMock() + def side_effect(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([1., 2., 3.]) + else: + return torch.Tensor([1., 2., 3.]) + mock.get_parameter.side_effect = side_effect + return mock + +class TestClass: + + def test_run_api(self): + kwargs = {"approximate": mock_compute_element_kwargs_instance} + inputs = [mock_compute_element_input_instance] + gradient_inputs = [mock_compute_element_input_instance] + result = [mock_compute_element_result_instance] + + + # api_instance, inputs, kwargs, gradient_inputs, forward_or_backward, api_platform, result + test_cases = [ + [mindspore.mint.nn.functional.gelu, inputs, kwargs, gradient_inputs, FORWARD_API, MINDSPORE_PLATFORM, result], + [mindspore.mint.nn.functional.gelu, inputs, kwargs, gradient_inputs, BACKWARD_API, MINDSPORE_PLATFORM, result], + [torch.nn.functional.gelu, inputs, kwargs, gradient_inputs, FORWARD_API, TORCH_PLATFORM, result], + [torch.nn.functional.gelu, inputs, kwargs, gradient_inputs, BACKWARD_API, TORCH_PLATFORM, result], + ] + for test_case in test_cases: + api_instance, inputs_target, kwargs_target, gradient_inputs_target, forward_or_backward, api_platform, results_target = test_case + results_real = api_runner.run_api(api_instance, inputs_target, kwargs_target, gradient_inputs_target, forward_or_backward, api_platform) + for res_real, res_target in zip(results_real, results_target): + assert res_real.get_parameter() == res_target.get_parameter() + + + def test_get_api_instance(self): + #api_type_str, api_sub_name, api_platform, result_api + test_cases = [ + ["MintFunctional", "relu", MINDSPORE_PLATFORM, mindspore.mint.nn.functional.relu], + ["MintFunctional", "relu", TORCH_PLATFORM, torch.nn.functional.relu] + ] + for test_case in test_cases: + api_type_str, api_sub_name, api_platform, result_api = test_case + assert api_runner.get_api_instance(api_type_str, api_sub_name, api_platform) == result_api + + def test_get_info_from_name(self): + api_name = "MintFunctional.relu.0.backward" + api_type_str, api_sub_name = api_runner.get_info_from_name(api_name_str=api_name) + assert api_type_str == "MintFunctional" + assert api_sub_name == "relu" -- Gitee From 49e682b2f9fa47eb3fbfe5edb6109773face7229 Mon Sep 17 00:00:00 2001 From: zyy Date: Mon, 19 Aug 2024 19:54:17 +0800 Subject: [PATCH 374/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=87=AA=E5=AE=9A?= =?UTF-8?q?=E4=B9=89=E8=BE=93=E5=87=BA=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dataset/cluster/cluster_dataset.py | 12 +++++++----- profiler/cli/analyze_cli.py | 4 +++- profiler/cli/cluster_cli.py | 6 ++++-- .../cluster_analyse/analysis/base_analysis.py | 5 +++-- .../analysis/comm_matrix_analysis.py | 2 +- .../analysis/communication_analysis.py | 2 +- .../analysis/host_info_analysis.py | 2 +- .../analysis/step_trace_time_analysis.py | 5 +++-- profiler/cluster_analyse/cluster_analysis.py | 19 +++++++++++++------ .../cluster_analyse/common_func/constant.py | 1 + .../base_communication_group.py | 1 + .../communication_db_group.py | 2 +- .../communication_json_group.py | 2 +- 13 files changed, 40 insertions(+), 23 deletions(-) diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index b4956139c..445d4c87e 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -32,14 +32,15 @@ logger = logging.getLogger() class ClusterDataset(Dataset): def __init__(self, collection_path, data: dict, **kwargs) -> None: + self.cluster_analysis_output_path = kwargs.get(Constant.CLUSTER_ANALYSIS_OUTPUT_PATH, collection_path) super().__init__(collection_path, data) def is_cluster_analysis_output_exist(self): """ check whether input path is valid """ - for file in os.listdir(self.collection_path): - if file == 'cluster_analysis_output': + for filename in os.listdir(self.cluster_analysis_output_path): + if filename == 'cluster_analysis_output': logger.info("[INFO]Cluster has been analyzed " "because of the existence of cluster analysis output directory.") logger.info("[INFO]Skip Cluster analyze backend.") @@ -51,7 +52,8 @@ class ClusterDataset(Dataset): return parameter = { Constant.COLLECTION_PATH: self.collection_path, - Constant.ANALYSIS_MODE: "all" + Constant.ANALYSIS_MODE: "all", + Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: self.cluster_analysis_output_path } print("[INFO] cluster analysis is in the process, please wait...") try: @@ -82,7 +84,7 @@ class ClusterStepTraceTimeDataset(ClusterDataset): def __init__(self, collection_path: str, data: dict, **kwargs): self._step_dict = defaultdict() - super().__init__(collection_path, data) + super().__init__(collection_path, data, **kwargs) def _parse(self): self.cluster_analyze() @@ -130,7 +132,7 @@ class ClusterCommunicationDataset(ClusterDataset): self.SDMA_SIZE_MB: 0, }) self.hccl_dict = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - super().__init__(collection_path, data) + super().__init__(collection_path, data, **kwargs) @staticmethod def compute_ratio(dividend: float, divisor: float): diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index f400a265b..c57251e5d 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -64,6 +64,8 @@ def analyze_cli(**kwargs): help='Directory of profiling data') @click.option('--benchmark_profiling_path', '-bp', 'benchmark_profiling_path', type=click.Path(), help='Directory of benchmark profiling data, used for compare performance') +@click.option('--output_path', '-o', 'cluster_analysis_output_path', type=click.Path(), + help='Path of cluster analysis output') @click.option('--cann_version', '-cv', 'cann_version', type=click.Choice(constant.SUPPORTED_CANN_VERSION, case_sensitive=False), default=constant.DEFAULT_CANN_VERSION, @@ -131,4 +133,4 @@ def analyze_schedule(**kwargs) -> None: help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @debug_option def analyze_computation(**kwargs) -> None: - _analyze(["computation"], **kwargs) \ No newline at end of file + _analyze(["computation"], **kwargs) diff --git a/profiler/cli/cluster_cli.py b/profiler/cli/cluster_cli.py index c1563898d..22760b0bf 100644 --- a/profiler/cli/cluster_cli.py +++ b/profiler/cli/cluster_cli.py @@ -34,7 +34,9 @@ context_settings['ignore_unknown_options'] = True @click.option('--profiling_path', '-d', type=click.Path(), required=True, help='path of the profiling data') @click.option('--mode', '-m', type=click.Choice(COMM_FEATURE_LIST), default='all') +@click.option('--output_path', '-o', 'cluster_analysis_output_path', type=click.Path(), + help='Path of cluster analysis output') @click.argument('args', nargs=-1) -def cluster_cli(profiling_path, mode, args) -> None: - required_args = ('-d', profiling_path, '-m', mode) +def cluster_cli(profiling_path, mode, cluster_analysis_output_path, args) -> None: + required_args = ('-d', profiling_path, '-m', mode, '-cp', cluster_analysis_output_path) cluster_analysis_main(required_args + args) diff --git a/profiler/cluster_analyse/analysis/base_analysis.py b/profiler/cluster_analyse/analysis/base_analysis.py index d7be4fc9c..a0e3aa4fd 100644 --- a/profiler/cluster_analyse/analysis/base_analysis.py +++ b/profiler/cluster_analyse/analysis/base_analysis.py @@ -29,6 +29,7 @@ class BaseAnalysis: MAX_RANKS = 1000 def __init__(self, param: dict): self.collection_path = param.get(Constant.COLLECTION_PATH) + self.cluster_analysis_output_path = param.get(Constant.CLUSTER_ANALYSIS_OUTPUT_PATH) self.data_map = param.get(Constant.DATA_MAP) self.data_type = param.get(Constant.DATA_TYPE) self.communication_ops = [] @@ -68,7 +69,7 @@ class BaseAnalysis: self.dump_json() else: if len(self.data_map) >= self.MAX_RANKS: - print("[WARNING]The number of ranks is too large to dump to db, it will be dumped to json file.") + print("[WARNING]The number of ranks is too large to dump to db, it will be dumped to json file.") self.dump_json() else: self.dump_db() @@ -81,7 +82,7 @@ class BaseAnalysis: output_comm_data = {} for key in self.comm_ops_struct: output_comm_data[str(key)] = self.comm_ops_struct.get(key) - FileManager.create_json_file(self.collection_path, output_comm_data, self.SAVED_JSON) + FileManager.create_json_file(self.cluster_analysis_output_path, output_comm_data, self.SAVED_JSON) def split_op_by_group(self): for single_op in self.communication_ops: diff --git a/profiler/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/cluster_analyse/analysis/comm_matrix_analysis.py index 8dc04471f..5d674aa52 100644 --- a/profiler/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/cluster_analyse/analysis/comm_matrix_analysis.py @@ -30,7 +30,7 @@ class CommMatrixAnalysis(BaseAnalysis): def dump_db(self): res_comm_matrix = self.adapter.transfer_matrix_from_json_to_db(self.comm_ops_struct) - output_path = os.path.join(self.collection_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + output_path = os.path.join(self.cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) DBManager.create_tables(result_db, self.COMMUNICATION_MATRIX_TABLE) conn, cursor = DBManager.create_connect_db(result_db) diff --git a/profiler/cluster_analyse/analysis/communication_analysis.py b/profiler/cluster_analyse/analysis/communication_analysis.py index 3f0a9b417..40af81084 100644 --- a/profiler/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/cluster_analyse/analysis/communication_analysis.py @@ -30,7 +30,7 @@ class CommunicationAnalysis(BaseAnalysis): def dump_db(self): res_comm_time, res_comm_bandwidth = self.adapter.transfer_comm_from_json_to_db(self.comm_ops_struct) - output_path = os.path.join(self.collection_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + output_path = os.path.join(self.cluster_analysis_output, Constant.CLUSTER_ANALYSIS_OUTPUT) result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) DBManager.create_tables(result_db, self.COMMUNICATION_TIME_TABLE, self.COMMUNICATION_BANDWIDTH_TABLE) conn, cursor = DBManager.create_connect_db(result_db) diff --git a/profiler/cluster_analyse/analysis/host_info_analysis.py b/profiler/cluster_analyse/analysis/host_info_analysis.py index 563711080..6fcbb0122 100644 --- a/profiler/cluster_analyse/analysis/host_info_analysis.py +++ b/profiler/cluster_analyse/analysis/host_info_analysis.py @@ -37,7 +37,7 @@ class HostInfoAnalysis(BaseAnalysis): self.dump_db() def dump_db(self): - output_path = os.path.join(self.collection_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + output_path = os.path.join(self.cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) conn, curs = DBManager.create_connect_db(result_db) if not (conn and curs): diff --git a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py index 617c0aafc..1b2dc0da8 100644 --- a/profiler/cluster_analyse/analysis/step_trace_time_analysis.py +++ b/profiler/cluster_analyse/analysis/step_trace_time_analysis.py @@ -30,6 +30,7 @@ class StepTraceTimeAnalysis: def __init__(self, param: dict): self.collection_path = param.get(Constant.COLLECTION_PATH) + self.cluster_analysis_output_path = param.get(Constant.CLUSTER_ANALYSIS_OUTPUT_PATH) self.data_map = param.get(Constant.DATA_MAP) self.communication_group = param.get(Constant.COMM_DATA_DICT, {}).get(Constant.COMMUNICATION_GROUP) self.step_time_dict = {} @@ -87,9 +88,9 @@ class StepTraceTimeAnalysis: return if self.data_type == Constant.TEXT: headers = self.get_headers() - FileManager.create_csv_file(self.collection_path, self.step_data_list, self.CLUSTER_TRACE_TIME_CSV, headers) + FileManager.create_csv_file(self.cluster_analysis_output_path, self.step_data_list, self.CLUSTER_TRACE_TIME_CSV, headers) else: - output_path = os.path.join(self.collection_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + output_path = os.path.join(self.cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) DBManager.create_tables(result_db, self.CLUSTER_TRACE_TIME_TABLE) column_len = DBManager.get_table_column_count(result_db, self.CLUSTER_TRACE_TIME_TABLE) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 171417c88..11c52ae5b 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -40,7 +40,13 @@ class Interface: self.communication_ops = [] self.matrix_ops = [] self.origin_params = params + self.cluster_analysis_output_path = self.get_cluster_analysis_output_path(params) + def get_cluster_analysis_output_path(self, params): + cluster_analysis_output_path = params.get(Constant.CLUSTER_ANALYSIS_OUTPUT_PATH) + if cluster_analysis_output_path: + return PathManager.get_realpath(cluster_analysis_output_path) + return self.collection_path def allocate_prof_data(self): ascend_pt_dirs = [] ascend_ms_dirs = [] @@ -58,10 +64,10 @@ class Interface: print("[ERROR] Can not analyze pytorch and mindspore meantime.") return [] return (pt_data_map, data_type) if pt_data_map else (ms_data_map, Constant.TEXT) - def run(self): PathManager.check_input_directory_path(self.collection_path) PathManager.check_path_owner_consistent(self.collection_path) + PathManager.check_path_writeable(self.cluster_analysis_output_path) data_map, data_type = self.allocate_prof_data() if not data_map: print("[WARNING] Can not get rank info or profiling data.") @@ -69,31 +75,32 @@ class Interface: if data_type == Constant.INVALID: print("[ERROR] The current folder contains both DB and other files. Please check.") return - FileManager.create_output_dir(self.collection_path) + FileManager.create_output_dir(self.cluster_analysis_output_path) params = { Constant.COLLECTION_PATH: self.collection_path, Constant.DATA_MAP: data_map, Constant.ANALYSIS_MODE: self.analysis_mode, - Constant.DATA_TYPE: data_type + Constant.DATA_TYPE: data_type, + Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: self.cluster_analysis_output_path, } comm_data_dict = CommunicationGroupGenerator(params).generate() params[Constant.COMM_DATA_DICT] = comm_data_dict AnalysisFacade(params).cluster_analyze() - def cluster_analysis_main(args=None): parser = argparse.ArgumentParser(description="cluster analysis module") parser.add_argument('-d', '--collection_path', type=str, required=True, help="profiling data path") parser.add_argument('-m', '--mode', choices=COMM_FEATURE_LIST, default='all', help="different analysis mode") + parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') args_parsed, _ = parser.parse_known_args(args=args) parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, - Constant.ANALYSIS_MODE: args_parsed.mode + Constant.ANALYSIS_MODE: args_parsed.mode, + Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: args_parsed.output_path } Interface(parameter).run() - if __name__ == "__main__": cluster_analysis_main() diff --git a/profiler/cluster_analyse/common_func/constant.py b/profiler/cluster_analyse/common_func/constant.py index a5b93b0ca..54e419910 100644 --- a/profiler/cluster_analyse/common_func/constant.py +++ b/profiler/cluster_analyse/common_func/constant.py @@ -68,6 +68,7 @@ class Constant(object): COMMUNICATION_OPS = "communication_ops" MATRIX_OPS = "matrix_ops" COLLECTION_PATH = "collection_path" + CLUSTER_ANALYSIS_OUTPUT_PATH = "output_path" COMMUNICATION_GROUP = "communication_group" TRANSPORT_TYPE = "Transport Type" COMM_DATA_DICT = "comm_data_dict" diff --git a/profiler/cluster_analyse/communication_group/base_communication_group.py b/profiler/cluster_analyse/communication_group/base_communication_group.py index 55f6801c2..91c4af775 100644 --- a/profiler/cluster_analyse/communication_group/base_communication_group.py +++ b/profiler/cluster_analyse/communication_group/base_communication_group.py @@ -26,6 +26,7 @@ from cluster_utils.data_transfer_adapter import DataTransferAdapter class BaseCommunicationGroup: def __init__(self, params: dict): self.collection_path = params.get(Constant.COLLECTION_PATH) + self.cluster_analysis_output_path = params.get(Constant.CLUSTER_ANALYSIS_OUTPUT_PATH) self.data_map = params.get(Constant.DATA_MAP) self.data_type = params.get(Constant.DATA_TYPE) self.analysis_mode = params.get(Constant.ANALYSIS_MODE) diff --git a/profiler/cluster_analyse/communication_group/communication_db_group.py b/profiler/cluster_analyse/communication_group/communication_db_group.py index 510dcd971..f39bf65d9 100644 --- a/profiler/cluster_analyse/communication_group/communication_db_group.py +++ b/profiler/cluster_analyse/communication_group/communication_db_group.py @@ -38,7 +38,7 @@ class CommunicationDBGroup(BaseCommunicationGroup): return rank_id, comm_data, comm_matrix_data def dump_data(self): - output_path = os.path.join(self.collection_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + output_path = os.path.join(self.cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) res = [] for data_type, data_list in self.communication_group.items(): diff --git a/profiler/cluster_analyse/communication_group/communication_json_group.py b/profiler/cluster_analyse/communication_group/communication_json_group.py index f6e01e3ab..b609b5de0 100644 --- a/profiler/cluster_analyse/communication_group/communication_json_group.py +++ b/profiler/cluster_analyse/communication_group/communication_json_group.py @@ -27,7 +27,7 @@ class CommunicationJsonGroup(BaseCommunicationGroup): super().__init__(params) def dump_data(self): - FileManager.create_json_file(self.collection_path, self.communication_group, self.COMMUNICATION_GROUP_JSON) + FileManager.create_json_file(self.cluster_analysis_output_path, self.communication_group, self.COMMUNICATION_GROUP_JSON) def read_communication_func(self: any, params: tuple): if len(params) < 3: -- Gitee From 152f2ffe601d245b9a32f9ad7983f319d293c8ed Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Sat, 17 Aug 2024 16:00:47 +0800 Subject: [PATCH 375/791] =?UTF-8?q?=E3=80=90=E4=BC=98=E5=8C=96=E3=80=91?= =?UTF-8?q?=E5=86=97=E4=BD=99=E4=BB=A3=E7=A0=81=E5=88=A0=E9=99=A4=E4=BB=A5?= =?UTF-8?q?=E5=8F=8A=E6=95=B0=E5=80=BC=E5=AE=89=E5=85=A8=E9=9A=90=E6=82=A3?= =?UTF-8?q?=E9=97=AE=E9=A2=98=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/data_dump/data_collector.py | 20 ------ .../data_processor/pytorch_processor.py | 10 ++- .../api_accuracy_checker/compare/algorithm.py | 7 +-- .../api_accuracy_checker/compare/compare.py | 61 ++++++++++--------- 4 files changed, 42 insertions(+), 56 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 1b2ae513d..ce9fb0314 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -136,26 +136,6 @@ class DataCollector: else: self.write_json() - def module_count_func(self, name, name_template): - module_name = name.split(Const.SEP)[-3] - if "forward" in name_template: - if module_name not in self.module_count: - self.module_count[module_name] = [0, [0]] - else: - if self.module_count[module_name][-1] and \ - self.module_count[module_name][0] != self.module_count[module_name][-1][-1]: - self.module_count[module_name][-1].pop() - self.module_count[module_name][0] += 1 - self.module_count[module_name][-1].append(self.module_count[module_name][0]) - index = self.module_count[module_name][0] - else: - backward_stack = self.module_count[module_name][-1] if module_name in self.module_count else [] - if not backward_stack: - index = "abnormal" - else: - index = backward_stack.pop() - return index - def update_dump_paths(self, *args): self.data_writer.update_dump_paths(*args) self.data_writer.initialize_json_file(task=self.config.task, level=self.config.level) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 3419cbb85..1e529d2b4 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -262,9 +262,13 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): if np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']): self.has_overflow = True else: - self.has_overflow = self.check_overflow_npu() - if self.has_overflow: - self.clear_overflow_npu() + try: + self.has_overflow = self.check_overflow_npu() + if self.has_overflow: + self.clear_overflow_npu() + except Exception as e: + logger.error(f"Overflow check failed, the current environment may be abnormal.") + raise RuntimeError(f"overflow check failed") from e def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/algorithm.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/algorithm.py index 1bb19cc04..4f7fa14d3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/algorithm.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/algorithm.py @@ -6,9 +6,6 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import ULP_PARAM from msprobe.core.common.const import CompareConst -DEFAULT_THRESHOLD = 1 - - #cos def cosine_sim(bench_output, device_output): msg = "" @@ -197,8 +194,8 @@ def check_norm_value(normal_value_mask, rel_err, rtol): def get_ulp_err(bench_output, device_output, dtype): parameters = ULP_PARAMETERS.get(dtype) - min_eb = parameters.get('min_eb', DEFAULT_THRESHOLD)[0] - exponent_num = parameters.get('exponent_num', DEFAULT_THRESHOLD)[0] + min_eb = parameters.get('min_eb')[0] + exponent_num = parameters.get('exponent_num')[0] abs_bench = np.abs(bench_output) eb = np.where(abs_bench == 0, 0, np.floor(np.log2(abs_bench))) eb = np.maximum(eb, min_eb) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 155a02d59..78945b505 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -1,27 +1,28 @@ # 进行比对及结果展示 import os from collections import namedtuple -import torch + import numpy as np -from msprobe.pytorch.common.log import logger +import torch +from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.utils import CompareException +from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.api_accuracy_checker.common.utils import get_json_contents, write_csv -from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \ - DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, absolute_standard_api, binary_standard_api, \ - ulp_standard_api, thousandth_standard_api, apis_threshold -from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, \ get_mean_rel_err, get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask, check_inf_nan_value, \ check_small_value, check_norm_value, get_abs_bench_with_eps, get_ulp_err -from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig -from msprobe.core.common.const import Const, CompareConst - +from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn +from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \ + DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, absolute_standard_api, binary_standard_api, \ + ulp_standard_api, thousandth_standard_api, apis_threshold +from msprobe.pytorch.common.log import logger ResultInfo = namedtuple('ResultInfo', ['full_api_name', 'fwd_success_status', 'bwd_success_status', 'fwd_compare_alg_results', 'bwd_compare_alg_results', 'rank']) -INDEX_TEST_RESULT__GROUP = 3 +INDEX_TEST_RESULT_GROUP = 3 INDEX_FIRST_GROUP = 0 INDEX_MESSAGE = -1 @@ -60,7 +61,7 @@ class Comparator: @staticmethod def print_pretest_result(): logger.info("Successfully completed run_ut/multi_run_ut.") - + @staticmethod def _compare_dropout(bench_output, device_output): tensor_num = bench_output.numel() @@ -89,7 +90,7 @@ class Comparator: error_rate = float(error_nums / bench_output.size) result = CompareConst.PASS if error_rate == 0 else CompareConst.ERROR return error_rate, result, "" - + @staticmethod def _get_absolute_threshold_attribute(api_name, dtype): small_value_threshold = apis_threshold.get(api_name).get(dtype).get('small_value') @@ -108,26 +109,31 @@ class Comparator: def write_summary_csv(self, test_result): test_rows = [] - if self.stack_info: - test_rows[0].append(self.COLUMN_STACK_INFO) - - name = test_result[0] - df_row = list(test_result[:INDEX_TEST_RESULT__GROUP]) - if test_result[1] == "SKIP": - df_row.append(test_result[INDEX_TEST_RESULT__GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE]) - if self.stack_info: - stack_info = "\n".join(self.stack_info[name]) - df_row.append(stack_info) - test_rows.append(df_row) - save_path = self.get_path_from_rank(test_result[-1], self.save_path_list, self.save_path_str) + try: + name = test_result[0] + df_row = list(test_result[:INDEX_TEST_RESULT_GROUP]) + if test_result[1] == "SKIP": + df_row.append(test_result[INDEX_TEST_RESULT_GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE]) + if self.stack_info: + stack_info = "\n".join(self.stack_info[name]) + df_row.append(stack_info) + test_rows.append(df_row) + save_path = self.get_path_from_rank(test_result[-1], self.save_path_list, self.save_path_str) + except IndexError as e: + logger.error("List index out of bounds when writing summary CSV.") + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR, "list index out of bounds") from e write_csv(test_rows, save_path) def write_detail_csv(self, test_result): test_rows = [] + try: + subject_prefix = test_result[0] + fwd_result = test_result[3] + bwd_result = test_result[4] + except IndexError as e: + logger.error("List index out of bounds when writing detail CSV.") + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR, "list index out of bounds") from e - subject_prefix = test_result[0] - fwd_result = test_result[3] - bwd_result = test_result[4] if isinstance(fwd_result, list): for i, test_subject in enumerate(fwd_result): subject = subject_prefix + ".forward.output." + str(i) @@ -140,7 +146,6 @@ class Comparator: test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) if isinstance(item, float) else item for item in test_subject] test_rows.append([subject] + list(test_subject)) - detail_save_path = self.get_path_from_rank(test_result[-1], self.detail_save_path_list, self.detail_save_path_str) -- Gitee From 095c5a041b45fb6271b9437bc4a4b116d0accb89 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Mon, 19 Aug 2024 15:04:00 +0800 Subject: [PATCH 376/791] use common save_npy function --- .../accuracy_tools/grad_tool/grad_ms/utils.py | 5 ++-- .../msprobe/core/common/utils.py | 2 +- .../data_processor/mindspore_processor.py | 17 ++++------- .../msprobe/mindspore/common/utils.py | 13 +++++++++ .../mindspore/debugger/debugger_config.py | 2 +- .../test_mindspore_processor.py | 29 +++++++------------ 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/grad_ms/utils.py b/debug/accuracy_tools/grad_tool/grad_ms/utils.py index c8ee1fd1d..2f60ffd6f 100644 --- a/debug/accuracy_tools/grad_tool/grad_ms/utils.py +++ b/debug/accuracy_tools/grad_tool/grad_ms/utils.py @@ -3,8 +3,9 @@ import os import numpy as np import mindspore from grad_tool.common.constant import GradConst -from grad_tool.common.utils import (print_warn_log, create_directory, change_mode, check_file_or_directory_path, +from grad_tool.common.utils import (create_directory, change_mode, check_file_or_directory_path, path_valid_check, check_param) +from msprobe.core.common.const import FileCheckConst level_adp = { "L0": { @@ -39,7 +40,7 @@ def save_grad_direction(param_name, grad, save_path): np.save(save_filepath, grad_direction_ndarray) except Exception as e: raise RuntimeError(f"An unexpected error occurred: {e} when saving numpy to {save_filepath}") from e - change_mode(save_filepath, 0o640) + change_mode(save_filepath, FileCheckConst.DATA_FILE_AUTHORITY) def get_adapted_level(level: str): diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 6794e2227..ff2fa5949 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -542,7 +542,7 @@ def save_npy(data, filepath): filepath = os.path.realpath(filepath) check_path_before_create(filepath) try: - npy = np.save(filepath, data) + np.save(filepath, data) except Exception as e: logger.error(f"The numpy file failed to save. Please check the path: {filepath}.") raise RuntimeError(f"Save numpy file {filepath} failed.") from e diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 2abb294f6..b9747a88e 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -14,6 +14,7 @@ # ============================================================================ import zlib + import mindspore as ms from mindspore import ops import numpy as np @@ -21,9 +22,9 @@ import numpy as np from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import (BaseDataProcessor, TensorStatInfo, ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs) -from msprobe.core.common.file_check import path_len_exceeds_limit, change_mode, FileCheckConst +from msprobe.core.common.file_check import path_len_exceeds_limit from msprobe.mindspore.dump.hook_cell.wrap_functional import load_ops_functions -from msprobe.mindspore.common.utils import convert_bf16_to_fp32 +from msprobe.mindspore.common.utils import convert_bf16_to_fp32, save_tensor_as_npy from msprobe.mindspore.common.log import logger from msprobe.mindspore.dump.hook_cell.api_registry import api_register @@ -136,13 +137,7 @@ class TensorDataProcessor(MindsporeDataProcessor): dump_data_name, file_path = self.get_save_file_path(suffix) single_arg = super()._analyze_tensor(tensor, suffix) single_arg.update({"data_name": dump_data_name}) - if not path_len_exceeds_limit(file_path): - tensor = convert_bf16_to_fp32(tensor) - saved_tensor = tensor.asnumpy() - np.save(file_path, saved_tensor) - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - else: - logger.warning(f'The file path {file_path} length exceeds limit.') + save_tensor_as_npy(tensor, file_path) return single_arg @@ -179,9 +174,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): def maybe_save_overflow_data(self): if self.has_overflow: for file_path, tensor in self.cached_tensors_and_file_paths.items(): - tensor = convert_bf16_to_fp32(tensor) - np.save(file_path, tensor.asnumpy()) - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + save_tensor_as_npy(tensor, file_path) self.real_overflow_nums += 1 self.cached_tensors_and_file_paths = {} diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index 6abf0a1ee..dc4444e78 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -14,7 +14,11 @@ # ============================================================================ import mindspore as ms + from msprobe.core.common.exceptions import DistributedNotInitializedError +from msprobe.core.common.file_check import path_len_exceeds_limit +from msprobe.core.common.utils import save_npy +from msprobe.core.common.log import logger def get_rank_if_initialized(): @@ -30,6 +34,15 @@ def convert_bf16_to_fp32(tensor): return tensor +def save_tensor_as_npy(tensor, file_path): + if not path_len_exceeds_limit(file_path): + tensor = convert_bf16_to_fp32(tensor) + saved_tensor = tensor.asnumpy() + save_npy(saved_tensor, file_path) + else: + logger.warning(f'The file path {file_path} length exceeds limit.') + + class MsprobeStep(ms.train.Callback): def __init__(self, debugger): diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 78dc253fa..0e37086f7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -70,6 +70,6 @@ class DebuggerConfig: def _make_dump_path_if_not_exists(self): check_path_before_create(self.dump_path) if not os.path.exists(self.dump_path): - Path(self.dump_path).mkdir(mode=0o750, exist_ok=True) + Path(self.dump_path).mkdir(mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) file_check = FileChecker(self.dump_path, FileCheckConst.DIR) file_check.common_check() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py index 1bc2d125c..e097f9c61 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -22,13 +22,12 @@ import mindspore as ms from mindspore import Tensor import numpy as np -from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, TensorStatInfo +from msprobe.core.data_dump.data_processor.base import BaseDataProcessor from msprobe.core.data_dump.data_processor.mindspore_processor import ( MindsporeDataProcessor, TensorDataProcessor, OverflowCheckDataProcessor -) -from msprobe.core.common.const import FileCheckConst +) class TestMindsporeDataProcessor(unittest.TestCase): @@ -65,7 +64,7 @@ class TestMindsporeDataProcessor(unittest.TestCase): self.assertEqual(result.norm, ms.ops.norm(tensor).item()) def test_get_stat_info_int(self): - tensor = ms.Tensor([1, 2, 3], dtype=ms.int32) + tensor = ms.Tensor([1, 2, 3], dtype=ms.int32) result = self.processor.get_stat_info(tensor) self.assertEqual(result.max, 3) self.assertEqual(result.min, 1) @@ -79,7 +78,7 @@ class TestMindsporeDataProcessor(unittest.TestCase): self.assertEqual(result.min, False) self.assertIsNone(result.mean) self.assertIsNone(result.norm) - + @patch.object(MindsporeDataProcessor, 'get_md5_for_tensor') def test__analyze_tensor(self, get_md5_for_tensor): get_md5_for_tensor.return_value = "test_md5" @@ -110,7 +109,7 @@ class TestTensorDataProcessor(unittest.TestCase): self.processor.current_api_or_module_name = "test_api" self.processor.api_data_category = "input" - @patch('numpy.save') + @patch('msprobe.core.data_dump.data_processor.mindspore_processor.save_tensor_as_npy') def test_analyze_tensor(self, mock_save): self.config.framework = "mindspore" tensor = ms.Tensor([1.0, 2.0, 3.0]) @@ -128,7 +127,7 @@ class TestTensorDataProcessor(unittest.TestCase): 'data_name': 'test_api.input.suffix.npy' } self.assertEqual(expected, result) - + class TestOverflowCheckDataProcessor(unittest.TestCase): def setUp(self): @@ -168,23 +167,17 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.assertTrue(self.data_processor.has_overflow) self.assertEqual(api_info, {"min", 0}) - @patch("msprobe.core.data_dump.data_processor.mindspore_processor.np.save") - @patch("msprobe.core.data_dump.data_processor.mindspore_processor.change_mode") - def test_maybe_save_overflow_data(self, mock_change_mode, mock_save): + @patch("msprobe.core.data_dump.data_processor.mindspore_processor.save_tensor_as_npy") + def test_maybe_save_overflow_data(self, mock_save): self.data_processor.has_overflow = True tensor1 = Tensor(1) tensor2 = Tensor(2) self.data_processor.cached_tensors_and_file_paths = {"tensor1": tensor1, "tensor2": tensor2} - with patch("mindspore.Tensor.asnumpy", return_value="npy"): - self.data_processor.maybe_save_overflow_data() + self.data_processor.maybe_save_overflow_data() self.assertEqual(mock_save.call_args_list[0][0], - ("tensor1", "npy")) + (tensor1, "tensor1")) self.assertEqual(mock_save.call_args_list[1][0], - ("tensor2", "npy")) - self.assertEqual(mock_change_mode.call_args_list[0][0], - ("tensor1", FileCheckConst.DATA_FILE_AUTHORITY)) - self.assertEqual(mock_change_mode.call_args_list[1][0], - ("tensor2", FileCheckConst.DATA_FILE_AUTHORITY)) + (tensor2, "tensor2")) @patch("msprobe.core.data_dump.data_processor.mindspore_processor.logger.info") def test_is_terminated(self, mock_info): -- Gitee From e51f5c1e6791e0d017c2ee0c6bfc2822c69baf8a Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Mon, 19 Aug 2024 13:23:36 +0000 Subject: [PATCH 377/791] update debug/accuracy_tools/msprobe/core/common/utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/core/common/utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index caa699a02..ff73ea9f9 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -523,14 +523,12 @@ def convert_tuple(data): def write_csv(data, filepath): - is_first_create = False - if not os.path.exists(filepath): - is_first_create = True + is_first_create = os.path.exists(filepath) with FileOpen(filepath, 'a+', encoding='utf-8-sig') as f: writer = csv.writer(f) writer.writerows(data) - if is_first_create: - change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + if is_first_create: + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) def load_npy(filepath): @@ -559,8 +557,10 @@ def get_json_contents(file_path): try: json_obj = json.loads(ops) except ValueError as error: + logger.error('Failed to load json.') raise CompareException(CompareException.INVALID_FILE_ERROR) from error if not isinstance(json_obj, dict): + logger.error('Json file content is not a dictionary!') raise CompareException(CompareException.INVALID_FILE_ERROR) return json_obj -- Gitee From 06bc9c337d45a54d9e36fea3399dd5e8554d0463 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 19 Aug 2024 22:00:35 +0800 Subject: [PATCH 378/791] bug fix --- .../api_accuracy_checker/api_info.py | 2 +- .../api_accuracy_checker/api_runner.py | 20 +++++++---- .../base_compare_algorithm.py | 4 +-- .../api_accuracy_checker/test_api_info.py | 6 ++-- .../api_accuracy_checker/test_api_runner.py | 33 +++++++++++++------ 5 files changed, 42 insertions(+), 23 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py index 64609a1e3..ea5f9e1c7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py @@ -68,6 +68,6 @@ class ApiInfo: logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.ParseJsonFailed)) kwargs_compute_element_dict = {key_str: ComputeElement(compute_element_info=compute_element_info) - for key_str, compute_element_info in kwargs_dict} + for key_str, compute_element_info in kwargs_dict.items()} return kwargs_compute_element_dict diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index b9a6e7c9a..d3f0c4e56 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -64,8 +64,7 @@ class ApiRunner: return api_type_str, api_sub_name - @classmethod - def get_api_instance(cls, api_type_str, api_sub_name, api_platform): + def get_api_instance(self, api_type_str, api_sub_name, api_platform): ''' Args: api_type_str: str, Union["MintFunctional", "Mint"] @@ -81,7 +80,7 @@ class ApiRunner: mindspore.mint.nn.functional.{api_sub_name} <--> torch.nn.functional.{api_sub_name} ''' - api_parent_module = cls.api_parent_module_mapping.get((api_type_str, api_platform)) + api_parent_module = self.api_parent_module_mapping.get((api_type_str, api_platform)) module_str = "mindspore.mint." if api_platform == MINDSPORE_PLATFORM else "torch." submodule_str = "nn.functional." if api_type_str == MINT_FUNCTIONAL else "" full_api_name = module_str + submodule_str + api_sub_name @@ -111,12 +110,19 @@ class ApiRunner: if gradient_inputs is None: err_msg = f"ApiRunner.run_api failed: run backward api but gradient_inputs is missing" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) - gradient_inputs = \ - tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) - for compute_element in gradient_inputs) + if len(gradient_inputs) == 1: + gradient_inputs = gradient_inputs[0].get_parameter(get_origin=False, tensor_platform=api_platform) + else: + gradient_inputs = \ + tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) + for compute_element in gradient_inputs) if api_platform == MINDSPORE_PLATFORM: + if kwargs != {}: + err_msg = f"ApiRunner.run_api failed: backward api with kwargs is currently not supported." + logger.error_log_with_exp(err_msg, + ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_instance) - backward_result = grad_func(*inputs, **kwargs, gradient_inputs) # can be single tensor or tuple + backward_result = grad_func(*inputs, gradient_inputs, **kwargs) # can be single tensor or tuple backward_result_tuple = convert_to_tuple(backward_result) res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_tuple] else: diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index c2cefa70b..d4fc578cd 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -91,7 +91,7 @@ class BaseCompareAlgorithm(ABC): raise NotImplementedError @classmethod - def convert_to_np_float64_ndarray(tensor): + def convert_to_np_float64_ndarray(cls, tensor): if isinstance(tensor, mindspore.Tensor): ndarray = tensor.astype(mindspore.float64).numpy() elif isinstance(tensor, torch.Tensor): @@ -103,7 +103,7 @@ class BaseCompareAlgorithm(ABC): return ndarray @classmethod - def check_two_tensor(bench_compute_element, tested_compute_element): + def check_two_tensor(cls, bench_compute_element, tested_compute_element): bench_parameter = bench_compute_element.get_parameter() tested_parameter = tested_compute_element.get_parameter() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py index 1346ceaf9..9392780ae 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py @@ -67,13 +67,13 @@ class TestClass: api_info = ApiInfo("MintFuntional.gelu.0.forward") api_info.load_forward_info(forward_api_info_dict) - assert api_info.check_forward_info == True - assert api_info.check_backward_info == False + assert api_info.check_forward_info() == True + assert api_info.check_backward_info() == False input_compute_element_list = api_info.get_compute_element_list("forward_api", "input") parameter_real = input_compute_element_list[0].get_parameter() parameter_target = mindspore.Tensor([1., 2., 3.]) - assert parameter_real == parameter_target + assert (parameter_real == parameter_target).all() kwargs_compute_element_dict = api_info.get_kwargs() assert kwargs_compute_element_dict.get("approximate").get_parameter() == "tanh" diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py index 4b8ea36aa..bdbbf51c2 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py @@ -36,37 +36,50 @@ def mock_compute_element_kwargs_instance(): return mock @pytest.fixture -def mock_compute_element_result_instance(): +def mock_compute_element_forward_result_instance(): mock = MagicMock() def side_effect(**kwargs): if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: - return mindspore.Tensor([1., 2., 3.]) + return mindspore.Tensor([8.41192007e-01, 1.95459759e+00, 2.99636269e+00]) else: - return torch.Tensor([1., 2., 3.]) + return torch.Tensor([8.41192007e-01, 1.95459759e+00, 2.99636269e+00]) + mock.get_parameter.side_effect = side_effect + return mock + +@pytest.fixture +def mock_compute_element_backward_result_instance(): + mock = MagicMock() + def side_effect(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([1.0833155, 2.1704636, 3.0358372]) + else: + return torch.Tensor([1.0833155, 2.1704636, 3.0358372]) mock.get_parameter.side_effect = side_effect return mock class TestClass: - def test_run_api(self): + def test_run_api(selfmock_compute_element_kwargs_instance, mock_compute_element_input_instance, + mock_compute_element_forward_result_instance, mock_compute_element_backward_result_instance): kwargs = {"approximate": mock_compute_element_kwargs_instance} inputs = [mock_compute_element_input_instance] gradient_inputs = [mock_compute_element_input_instance] - result = [mock_compute_element_result_instance] + forward_result = [mock_compute_element_forward_result_instance] + backward_result = [mock_compute_element_backward_result_instance] # api_instance, inputs, kwargs, gradient_inputs, forward_or_backward, api_platform, result test_cases = [ - [mindspore.mint.nn.functional.gelu, inputs, kwargs, gradient_inputs, FORWARD_API, MINDSPORE_PLATFORM, result], - [mindspore.mint.nn.functional.gelu, inputs, kwargs, gradient_inputs, BACKWARD_API, MINDSPORE_PLATFORM, result], - [torch.nn.functional.gelu, inputs, kwargs, gradient_inputs, FORWARD_API, TORCH_PLATFORM, result], - [torch.nn.functional.gelu, inputs, kwargs, gradient_inputs, BACKWARD_API, TORCH_PLATFORM, result], + [mindspore.mint.nn.functional.gelu, inputs, kwargs, gradient_inputs, FORWARD_API, MINDSPORE_PLATFORM, forward_result], + [mindspore.mint.nn.functional.gelu, inputs, {}, gradient_inputs, BACKWARD_API, MINDSPORE_PLATFORM, backward_result], + [torch.nn.functional.gelu, inputs, kwargs, gradient_inputs, FORWARD_API, TORCH_PLATFORM, forward_result], + [torch.nn.functional.gelu, inputs, {}, gradient_inputs, BACKWARD_API, TORCH_PLATFORM, backward_result], ] for test_case in test_cases: api_instance, inputs_target, kwargs_target, gradient_inputs_target, forward_or_backward, api_platform, results_target = test_case results_real = api_runner.run_api(api_instance, inputs_target, kwargs_target, gradient_inputs_target, forward_or_backward, api_platform) for res_real, res_target in zip(results_real, results_target): - assert res_real.get_parameter() == res_target.get_parameter() + assert (abs(res_real.get_parameter() - res_target.get_parameter(tensor_platform=api_platform)) < 1e-5).all() def test_get_api_instance(self): -- Gitee From 503d6647581792662d5e5573cff956eb744dbe5d Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 19 Aug 2024 22:04:32 +0800 Subject: [PATCH 379/791] bug fix2 --- .../test/mindspore_ut/api_accuracy_checker/test_api_runner.py | 2 +- .../api_accuracy_checker/test_compare_algorithm.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py index bdbbf51c2..d035be420 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py @@ -59,7 +59,7 @@ def mock_compute_element_backward_result_instance(): class TestClass: - def test_run_api(selfmock_compute_element_kwargs_instance, mock_compute_element_input_instance, + def test_run_api(self, mock_compute_element_kwargs_instance, mock_compute_element_input_instance, mock_compute_element_forward_result_instance, mock_compute_element_backward_result_instance): kwargs = {"approximate": mock_compute_element_kwargs_instance} inputs = [mock_compute_element_input_instance] diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py index 9aba764fc..e7b1e7181 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py @@ -20,14 +20,14 @@ directory = os.path.dirname(file_path) @pytest.fixture def mock_mstensor_compute_element(): mock = MagicMock() - mock.get_parameter.return_value = mindspore.Tensor([1., 1.9, 3.], dtype=mindspore.float32) + mock.get_parameter.return_value = mindspore.Tensor([1., 1.9, 3.]) mock.get_shape.return_value = (3,) return mock @pytest.fixture def mock_torchtensor_compute_element(): mock = MagicMock() - mock.get_parameter.return_value = torch.Tensor([1., 2., 3.], dtype=torch.float32) + mock.get_parameter.return_value = torch.Tensor([1., 2., 3.]) mock.get_shape.return_value = (3,) return mock -- Gitee From d699ddcaa434b89620aa34f08d3fd17d7972ddb5 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Mon, 19 Aug 2024 22:06:26 +0800 Subject: [PATCH 380/791] cross frame compare --- .../msprobe/core/compare/acc_compare.py | 9 +- .../msprobe/core/compare/check.py | 23 ++++- .../msprobe/core/compare/compare_cli.py | 10 ++- .../msprobe/core/compare/utils.py | 3 + .../msprobe/mindspore/compare/ms_compare.py | 86 +++++++++++++++---- 5 files changed, 110 insertions(+), 21 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index be749e5aa..77ea7464c 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -18,7 +18,7 @@ from msprobe.core.advisor.advisor import Advisor class Comparator: def __init__(self): - pass + pass @classmethod def make_result_table(cls,result,md5_compare,summary_compare,stack_mode): @@ -169,8 +169,11 @@ class Comparator: read_npy_data = getattr(self, "read_npy_data") frame_name = getattr(self, "frame_name") if frame_name == "MSComparator": - n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.NUMPY_SUFFIX) - b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.NUMPY_SUFFIX) + n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.NUMPY_SUFFIX, load_pt=False) + if self.cross_frame: + b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.PT_SUFFIX, load_pt=True) + else: + b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.NUMPY_SUFFIX, load_pt=False) else: n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.PT_SUFFIX) b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.PT_SUFFIX) diff --git a/debug/accuracy_tools/msprobe/core/compare/check.py b/debug/accuracy_tools/msprobe/core/compare/check.py index c243c0910..39bafe3b3 100644 --- a/debug/accuracy_tools/msprobe/core/compare/check.py +++ b/debug/accuracy_tools/msprobe/core/compare/check.py @@ -2,11 +2,32 @@ from msprobe.core.common.log import logger from msprobe.core.compare.utils import rename_api -def check_struct_match(npu_dict, bench_dict): +dtype_mapping = { + "Int8": "torch.int8", + "UInt8": "torch.uint8", + "Int16": "torch.int16", + "UInt16": "torch.uint16", + "Int32": "torch.int32", + "UInt32": "torch.uint32", + "Int64": "torch.int64", + "UInt64": "torch.uint64", + "Float16": "torch.float16", + "Float32": "torch.float32", + "Float64": "torch.float64", + "Bool": "torch.bool", + "BFloat16": "torch.bfloat16" + } + + +def check_struct_match(npu_dict, bench_dict, cross_frame=False): npu_struct_in = npu_dict.get("input_struct") bench_struct_in = bench_dict.get("input_struct") npu_struct_out = npu_dict.get("output_struct") bench_struct_out = bench_dict.get("output_struct") + + if cross_frame: + npu_struct_in = [(dtype_mapping.get(item[0], item[0]), item[1]) for item in npu_struct_in] + npu_struct_out = [(dtype_mapping.get(item[0], item[0]), item[1]) for item in npu_struct_out] is_match = npu_struct_in == bench_struct_in and npu_struct_out == bench_struct_out if not is_match: if len(npu_struct_in) == 0 or len(bench_struct_in) == 0 or len(npu_struct_in) != len(bench_struct_in): diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index ac96a65de..80a98a13b 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -26,8 +26,14 @@ def compare_cli(args): compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, fuzzy_match=args.fuzzy_match) else: - ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, - fuzzy_match=args.fuzzy_match) + kwargs = { + "stack_mode": args.stack_mode, + "auto_analyze": args.auto_analyze, + "fuzzy_match": args.fuzzy_match, + "cell_mapping": args.cell_mapping, + } + + ms_compare(input_param, args.output_path, **kwargs) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} if frame_name == Const.PT_FRAMEWORK: diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index ff1974f51..b04a1aed7 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -417,6 +417,9 @@ def _compare_parser(parser): help=" Whether to give advisor.", required=False) parser.add_argument("-f", "--fuzzy_match", dest="fuzzy_match", action="store_true", help=" Whether to perform a fuzzy match on the api name.", required=False) + parser.add_argument("-cm", "--cell_mapping", dest="cell_mapping", type=str, nargs='?', const=True, + help=" The cell mapping file path.", required=False) + diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 3807acb99..bbd32a9d6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,30 +1,86 @@ import os.path import numpy as np -from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, task_dumppath_get +from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, \ + task_dumppath_get, load_yaml from msprobe.core.common.file_check import FileChecker, create_directory -from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.const import FileCheckConst, Const from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.compare.acc_compare import Comparator +from msprobe.core.compare.acc_compare import Comparator +from msprobe.core.common.utils import CompareException +from msprobe.core.compare.check import check_struct_match, fuzzy_check_op class MSComparator (Comparator): - def __init__(self): + def __init__(self, cell_mapping=None, api_mapping=None): self.frame_name = MSComparator.__name__ + self.cell_mapping = cell_mapping + self.api_mapping = api_mapping + self.cross_frame = cell_mapping is not None or api_mapping is not None + self.cell_mapping_dict = self.load_mapping_file(self.cell_mapping) + self.api_mapping_dict = self.load_mapping_file(self.api_mapping) + + def load_mapping_file(self, mapping_file): + if isinstance(self.cell_mapping, str): + mapping_dict = load_yaml(mapping_file) + else: + mapping_dict = {} + return mapping_dict + + def process_cell_mapping(self, a_op_name): + a_op_name = [op_name.replace("Cell", "Module", 1) for op_name in a_op_name] + if self.cell_mapping_dict: + for index, op_name in enumerate(a_op_name): + cell_name = op_name.split(Const.SEP, 1)[-1].rsplit(Const.SEP, 4)[0] + if cell_name in self.cell_mapping_dict: + a_op_name[index] = op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) + return a_op_name + + + def check_op(self, npu_dict, bench_dict, fuzzy_match): + a_op_name = npu_dict["op_name"] + b_op_name = bench_dict["op_name"] + + if self.api_mapping is not None: + pass + if self.cell_mapping is not None: + a_op_name = self.process_cell_mapping(a_op_name) + + struct_match = check_struct_match(npu_dict, bench_dict, cross_frame=self.cross_frame) + if not fuzzy_match: + return a_op_name == b_op_name and struct_match + is_match = True + try: + is_match = fuzzy_check_op(a_op_name, b_op_name) + except Exception as err: + logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) + is_match = False + return is_match and struct_match - def read_npy_data(self,dir_path, file_name): + def read_npy_data(self, dir_path, file_name, load_pt=False): data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.NUMPY_SUFFIX, False) - data_path = path_checker.common_check() - data_value = np.load(data_path) # detach for less memory - if data_value.dtype == np.float16: - data_value = data_value.astype(np.float32) + if load_pt: + import torch + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.PT_SUFFIX, False) + data_path = path_checker.common_check() + data_value = torch.load(data_path, map_location=torch.device('cpu')).detach() # detach for less memory + if data_value.dtype == torch.bfloat16: + data_value = data_value.to(torch.float32) + data_value = data_value.numpy() + else: + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.NUMPY_SUFFIX, False) + data_path = path_checker.common_check() + data_value = np.load(data_path) return data_value - - -def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): + +def ms_compare(input_param, output_path, **kwargs): try: + stack_mode = kwargs.get('stack_mode', False) + auto_analyze = kwargs.get('auto_analyze', True) + fuzzy_match = kwargs.get('fuzzy_match', False) + cell_mapping = kwargs.get('cell_mapping', None) summary_compare, md5_compare = task_dumppath_get(input_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) @@ -32,7 +88,7 @@ def ms_compare(input_param, output_path, stack_mode=False, auto_analyze=True, fu except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - ms_comparator = MSComparator() + ms_comparator = MSComparator(cell_mapping) ms_comparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) -- Gitee From 4722ec1371a8446dbc5f785a608d61a60227cceb Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 20 Aug 2024 09:16:53 +0800 Subject: [PATCH 381/791] api unmatch output result improve --- debug/accuracy_tools/msprobe/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 211943e3c..42a65dc58 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -163,7 +163,7 @@ Required-by: 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位训练场景下的精度问题。 - PyTorch场景：详见[PyTorch_精度比对工具](./pytorch/doc/ptdbg_ascend_overview.md)。 + PyTorch场景：详见[PyTorch_精度比对工具](./pytorch/doc/ptdbg_ascend_compare.md)。 MindSpore场景：详见[MindSpore_精度比对工具](./mindspore/doc/compare.md)。 -- Gitee From e6b04f913d13d69e5f46984b6995334dd11da25e Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 20 Aug 2024 10:06:21 +0800 Subject: [PATCH 382/791] fix online run_ut csv print --- .../pytorch/api_accuracy_checker/run_ut/run_ut.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 7e5891b5a..10012d49f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -189,8 +189,13 @@ def generate_cpu_params(input_args, input_kwargs, need_backward, api_name): def run_ut(config): logger.info("start UT test") - logger.info(f"UT task result will be saved in {config.result_csv_path}") - logger.info(f"UT task details will be saved in {config.details_csv_path}") + if config.online_config.is_online: + logger.info(f"UT task result will be saved in {config.result_csv_path}".replace(".csv", "_rank*.csv")) + logger.info(f"UT task details will be saved in {config.details_csv_path}".replace(".csv", "_rank*.csv")) + else: + logger.info(f"UT task result will be saved in {config.result_csv_path}") + logger.info(f"UT task details will be saved in {config.details_csv_path}") + if config.save_error_data: logger.info(f"UT task error_datas will be saved in {config.error_data_path}") compare = Comparator(config.result_csv_path, config.details_csv_path, config.is_continue_run_ut, config=config) -- Gitee From c9bd32b8bc66bfc0da15c7add2a53092519f4827 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 20 Aug 2024 10:58:24 +0800 Subject: [PATCH 383/791] fix run_ut result.csv message loss --- .../tensor_transport_layer/device_dispatch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 1a5462203..3d73d4adf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -48,7 +48,7 @@ def run_ut_process(xpu_id, compare, consumer_queue, func, config): else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") - compare.write_summary_csv((api_full_name, "SKIP", "SKIP", str(err), api_data.rank)) + compare.write_summary_csv((api_full_name, "SKIP", "SKIP", [[str(err)]], api_data.rank)) finally: torch.cuda.empty_cache() -- Gitee From fdd9b2164438860ed94546f7b49c9f0a50acff1d Mon Sep 17 00:00:00 2001 From: zhaolei Date: Thu, 15 Aug 2024 20:45:21 +0800 Subject: [PATCH 384/791] =?UTF-8?q?gc=E6=A3=80=E6=B5=8B=E5=88=86=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../advisor/analyzer/schedule/gc/__init__.py | 0 .../analyzer/schedule/gc/gc_analyzer.py | 42 +++++++ .../analyzer/schedule/gc/gc_checker.py | 113 +++++++++++++++++ profiler/advisor/common/analyzer_scopes.py | 1 + .../advisor/dataset/timeline_event_dataset.py | 13 +- .../advisor/display/html/templates/gc.html | 37 ++++++ profiler/advisor/interface/interface.py | 4 +- profiler/advisor/rules/gc.yaml | 11 ++ profiler/advisor/utils/utils.py | 18 +++ .../advisor/schedule_advice/test_gc_advice.py | 116 ++++++++++++++++++ 10 files changed, 352 insertions(+), 3 deletions(-) create mode 100644 profiler/advisor/analyzer/schedule/gc/__init__.py create mode 100644 profiler/advisor/analyzer/schedule/gc/gc_analyzer.py create mode 100644 profiler/advisor/analyzer/schedule/gc/gc_checker.py create mode 100644 profiler/advisor/display/html/templates/gc.html create mode 100644 profiler/advisor/rules/gc.yaml create mode 100644 profiler/test/ut/advisor/schedule_advice/test_gc_advice.py diff --git a/profiler/advisor/analyzer/schedule/gc/__init__.py b/profiler/advisor/analyzer/schedule/gc/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py b/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py new file mode 100644 index 000000000..432179302 --- /dev/null +++ b/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py @@ -0,0 +1,42 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.schedule.gc.gc_checker import GcChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset + +logger = logging.getLogger() + + +class GcAnalyzer(BaseAnalyzer): + dataset_cls_list = [TimelineEventDataset] + + def __init__(self, collection_path, **kwargs): + super().__init__(collection_path, **kwargs) + self.result = OptimizeResult() + self.html_render = HTMLRender() + key = TimelineEventDataset.get_key() + self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) + + @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + def optimize(self, **kwargs): + gc_checker = GcChecker() + gc_checker.check_gc(self.timeline_event_dataset, rank_id=kwargs.get("rank_id"), stage=kwargs.get("stage")) + gc_checker.make_record(self.result) + gc_checker.make_render(self.html_render) + return self.result diff --git a/profiler/advisor/analyzer/schedule/gc/gc_checker.py b/profiler/advisor/analyzer/schedule/gc/gc_checker.py new file mode 100644 index 000000000..05ef28760 --- /dev/null +++ b/profiler/advisor/analyzer/schedule/gc/gc_checker.py @@ -0,0 +1,113 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os + +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.utils.utils import convert_to_float, convert_to_int + +logger = logging.getLogger() + + +class GcChecker: + + def __init__(self): + self.stage = None + self.rank_id = None + self.optimization_item = [] + self.gc_issues = False + self.desc = "" + self.suggestions = [] + self.solutions = None + self.gc_threshold = 0 + self.gc_topk_num = 0 + self.abnormal_gc_count = 0 + self.abnormal_gc_duration = 0 + self.abnormal_gc_list = [] + self.headers = ["timestamp", "duration(us)"] + self._init_rule() + + def check_gc(self, event_dataset: TimelineEventDataset, rank_id=None, stage=None): + """ + :Param event_dataset: dataset of timeline event + """ + if not hasattr(event_dataset, "gc_events"): + logger.debug("Skip gc checker, because no gc event found") + return + self.rank_id = rank_id + self.stage = stage + for gc_event in event_dataset.gc_events: + if convert_to_float(gc_event.dur) >= self.gc_threshold: + self.gc_issues = True + self.abnormal_gc_count += 1 + self.abnormal_gc_duration += convert_to_float(gc_event.dur) + self.abnormal_gc_list.append([gc_event.ts, gc_event.dur]) + self.abnormal_gc_duration = round(self.abnormal_gc_duration / 1000, 4) + self.abnormal_gc_list.sort(key=lambda x: x[1], reverse=True) + self.desc = self.desc.format(gc_count=self.abnormal_gc_count, gc_total_time=self.abnormal_gc_duration) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + if not self.gc_issues: + return + + self.optimization_item.append(OptimizeItem("gc", self.desc, self.suggestions)) + for optimization in self.optimization_item: + result.add(OptimizeRecord(optimization)) + if self.rank_id is not None: + self.headers = ["Rank id"] + self.headers + sub_table_name = "GcAnalysis" if not self.stage else f"Stage-{self.stage}: GcAnalysis" + result.add_detail(sub_table_name, headers=self.headers) + + for row in self.abnormal_gc_list: + if self.rank_id is not None: + row = [self.rank_id] + row + result.add_detail(sub_table_name, detail=row) + + def make_render(self, html_render): + if not self.gc_issues: + return + show_num = min(self.gc_topk_num, self.abnormal_gc_count) + html_render.render_template(key="schedule", + template_dir="templates", + template_name="gc.html", + desc=self.desc, + solutions=self.solutions, + headers=self.headers, + datas=self.abnormal_gc_list[:show_num], + num=show_num) + + def _init_rule(self): + gc_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))), + "rules", + "gc.yaml" + ) + + gc_rule = FileManager.read_yaml_file(gc_rule_path) + + self.gc_threshold = convert_to_float(gc_rule.get("gc_threshold", 0)) + self.gc_topk_num = convert_to_int(gc_rule.get("top_num", 0)) + self.desc = gc_rule.get("problem", "") + + self.solutions = gc_rule.get("solutions", []) + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index b947798c9..652e10b08 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -33,3 +33,4 @@ class SupportedScopes: SYNCBN = "syncbn" SYNCHRONIZE_STREAM = "synchronize_stream" FREQ_ANALYSIS = "freq_analysis" + GC_ANALYSIS = "gc_analysis" diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/advisor/dataset/timeline_event_dataset.py index 2956e2075..c76314641 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/advisor/dataset/timeline_event_dataset.py @@ -1,4 +1,3 @@ -import json import logging import os from typing import List, Any @@ -6,7 +5,6 @@ import traceback import ijson from tqdm import tqdm -import yaml from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent @@ -89,6 +87,7 @@ class TimelineEventDataset: self._optimizer: List[Any] = [] self._dataloader: List[Any] = [] self._sync_batchnorm: List[Any] = [] + self._gc: List[Any] = [] self._synchronize_stream = SynchronizeStreamCollector() self.timeline_dir = collection_path self.timeline_data_list = get_file_path_from_directory(collection_path, @@ -151,6 +150,10 @@ class TimelineEventDataset: def sync_batchnorm(self): return self._sync_batchnorm + @property + def gc_events(self): + return self._gc + @property def synchronize_stream(self): return self._synchronize_stream @@ -226,6 +229,10 @@ class TimelineEventDataset: if event.name == const.OP_COMPILE_NAME or event.args.get("id") == const.OP_COMPILE_ID: self._ops_compile.update(event) + def _add_gc(self, event: TimelineEvent): + if event.get("cat") and event.get("cat").lower() == 'gc': + self._gc.append(event) + def _add_optimizer(self, event: TimelineEvent): self._optimizer.append(TimelineEvent({"name": event.name, "dataset_index": event.dataset_index})) @@ -260,6 +267,8 @@ class TimelineEventDataset: self._add_dataloader(event) # for analysis of syncBatchNorm operator, prompt users to replace source code of torch_npu's syncbn self._add_sync_batchnorm(event) + # for analysis of GcAnalyzer + self._add_gc(event) def _add_event(self, index, event): event["dataset_index"] = index diff --git a/profiler/advisor/display/html/templates/gc.html b/profiler/advisor/display/html/templates/gc.html new file mode 100644 index 000000000..e6357c922 --- /dev/null +++ b/profiler/advisor/display/html/templates/gc.html @@ -0,0 +1,37 @@ + +

+

GC Analysis

+
+ {{ desc }} + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+ The details of top {{ num }} garbage collection events are as follows: +

+ + + {% for header in headers %} + + {% endfor %} + + + {% for row in datas %} + + {% for element in row %} + + {% endfor %} + + {% endfor %} +
{{ header }}
{{ element|safe }}
+
+
diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index 4908c275d..3fa10db64 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -34,6 +34,7 @@ from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_an from profiler.advisor.analyzer.dataloader.dataloader_analyzer import DataloaderAnalyzer from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_analyzer import AICoreFreqAnalyzer from profiler.advisor.analyzer.communication.packet_analyzer import PacketAnalyzer +from profiler.advisor.analyzer.schedule.gc.gc_analyzer import GcAnalyzer class Interface: @@ -42,7 +43,8 @@ class Interface: SupportedScopes.SYNCBN: SyncBNAnalyzer, SupportedScopes.TIMELINE_OP_DISPATCH: OpDispatchAnalyzer, SupportedScopes.SYNCHRONIZE_STREAM: SynchronizeStreamAnalyzer, - SupportedScopes.TIMELINE_FUSION_OPS: TimelineFusionOpsAnalyzer + SupportedScopes.TIMELINE_FUSION_OPS: TimelineFusionOpsAnalyzer, + SupportedScopes.GC_ANALYSIS: GcAnalyzer }), "computation": OrderedDict({ SupportedScopes.DYNAMIC_SHAPE_ANALYSIS: DynamicShapeAnalyzer, diff --git a/profiler/advisor/rules/gc.yaml b/profiler/advisor/rules/gc.yaml new file mode 100644 index 000000000..fad9b3523 --- /dev/null +++ b/profiler/advisor/rules/gc.yaml @@ -0,0 +1,11 @@ +problem: "Abnormal garbage collection (GC) event is detected for {gc_count} times, and the total time is {gc_total_time} ms\n. +The GC operation is time-consuming and blocks the entire process. As a result, some steps in the model training process take a longer time than other steps." +gc_threshold: 1000 #us +top_num: 10 +solutions: + - memory management: + desc: "implement effective Python memory management; release memory promptly when not in use to avoid long-term retention; avoid circular references between objects." + - adjusting the GC threshold: + desc: "use gc.set_threshold() to adjust the garbage collection threshold can delay garbage collection, but this is a temporary solution." + - disable GC: + desc: "use gc.disable() to disable GC, note that this is a temporary solution." diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index 83f304c2d..d0a132167 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -92,6 +92,12 @@ def singleton(cls): _instance[cls][collection_path] = cls(*args, **kw) return _instance[cls].get(collection_path) + def reset_all_instances(): + """ + 用于ut使用，清空单例类，防止ut不同测试用例间相互干扰 + """ + _instance.clear() + # 保留原始类的属性和方法 _singleton.__name__ = cls.__name__ _singleton.__module__ = cls.__module__ @@ -110,6 +116,9 @@ def singleton(cls): continue setattr(_singleton, function_obj.__name__, function_obj) + _singleton.reset_all_instances = reset_all_instances + singleton.reset_all_instances = reset_all_instances + return _singleton @@ -608,3 +617,12 @@ def convert_to_float(num): logger.error(f"Can not convert %ss to float", num) pass return 0 + + +def convert_to_int(data: any) -> int: + try: + int_value = int(data) + except ValueError: + logger.error(f"Can not convert %ss to float.", data) + return 0 + return int_value diff --git a/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py b/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py new file mode 100644 index 000000000..f18311ab1 --- /dev/null +++ b/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py @@ -0,0 +1,116 @@ +import os +import shutil +import stat +import json + +import unittest +from profiler.advisor.interface.interface import Interface +from profiler.advisor.common.analyzer_scopes import SupportedScopes +from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset + + +class TestGcAdvice(unittest.TestCase): + TMP_DIR = "./ascend_pt" + OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" + interface = None + + def tearDown(self): + if os.path.exists(TestGcAdvice.TMP_DIR): + shutil.rmtree(TestGcAdvice.TMP_DIR) + self.clear_htmls() + TimelineEventDataset.reset_all_instances() + + def setUp(self): + if os.path.exists(TestGcAdvice.TMP_DIR): + shutil.rmtree(TestGcAdvice.TMP_DIR) + if not os.path.exists(TestGcAdvice.TMP_DIR): + os.makedirs(TestGcAdvice.TMP_DIR) + if not os.path.exists(TestGcAdvice.OUTPUT_DIR): + os.makedirs(TestGcAdvice.OUTPUT_DIR) + self.clear_htmls() + + @classmethod + def clear_htmls(cls): + current_path = os.path.dirname(os.path.abspath(__file__)) + for filename in os.listdir(current_path): + # 检查文件是否以“att”开头 + if filename.startswith("mstt"): + # 构建文件的完整路径 + file_path = os.path.join(current_path, filename) + # 删除文件 + os.remove(file_path) + + @classmethod + def create_trace_view_with_gc_events(cls): + # Python pid + py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} + # Python GC pid + py_gc_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 2, "args": {"name": "Python GC"}} + # ascend pid + ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} + # ascend pid + cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} + # ascend hardware ops + ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, + "args": {"Task Type": "AI_CORE"}} + ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, + "args": {"Task Type": "AI_CORE"}} + gc_event1 = {"ph": "X", "name": "GC", "ts": "1699529622103750", "dur": 1500, "tid": 3, "pid": 4, "cat": "GC", + "args": {}} + gc_event2 = {"ph": "X", "name": "GC", "ts": "1699529623104750", "dur": 50, "tid": 3, "pid": 4, "cat": "GC", + "args": {}} + gc_event3 = {"ph": "X", "name": "GC", "ts": "1699529623105750", "dur": 50000, "tid": 3, "pid": 4, "cat": "GC", + "args": {}} + # flow event + flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} + flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} + + raw_data = [ + py_pid_data, py_gc_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, gc_event1, gc_event2, + gc_event3, flow_event_s, flow_event_e + ] + with os.fdopen(os.open(f"{TestGcAdvice.OUTPUT_DIR}/trace_view.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + @classmethod + def create_trace_view_without_gc_events(cls): + # Python pid + py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} + # ascend pid + ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} + # ascend pid + cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} + # ascend hardware ops + ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, + "args": {"Task Type": "AI_CORE"}} + ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, + "args": {"Task Type": "AI_CORE"}} + # flow event + flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} + flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} + + raw_data = [ + py_pid_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, flow_event_s, flow_event_e + ] + with os.fdopen(os.open(f"{TestGcAdvice.OUTPUT_DIR}/trace_view.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + def test_run_should_run_success_when_trace_view_contain_gc_events(self): + self.create_trace_view_with_gc_events() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "schedule" + scope = SupportedScopes.GC_ANALYSIS + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(2, len(result.data.get("GcAnalysis", dict).get("data", []))) + result.clear() + + def test_run_should_run_success_when_trace_view_not_contain_gc_events(self): + self.create_trace_view_without_gc_events() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = "schedule" + scope = SupportedScopes.GC_ANALYSIS + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(0, len(result.data.get("GcAnalysis", []))) + result.clear() -- Gitee From 20273276dba4ce5a49f1f27e3e51671d3c0aa60c Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 20 Aug 2024 03:47:20 +0000 Subject: [PATCH 385/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py. Signed-off-by: jiangchangting1 --- .../msprobe/pytorch/api_accuracy_checker/compare/compare.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 4725d3e42..18387b88e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -3,10 +3,9 @@ import os from collections import namedtuple import numpy as np -from msprobe.core.common.utils import write_csv, get_json_contents +from msprobe.core.common.utils import write_csv, get_json_contents, CompareException import torch from msprobe.core.common.const import Const, CompareConst -from msprobe.core.common.utils import CompareException from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, \ get_mean_rel_err, get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask, check_inf_nan_value, \ -- Gitee From bc658569b07eb2dc4277e8a2e6c8bf637ea932c1 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 20 Aug 2024 14:37:54 +0800 Subject: [PATCH 386/791] api unmatch output result improve --- .../msprobe/mindspore/doc/compare.md | 14 +++++++------- .../msprobe/pytorch/doc/ptdbg_ascend_compare.md | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md index b97a86fcd..01b4da70b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md @@ -23,13 +23,13 @@ msprobe精度比对工具主要通过对同一个模型，在两个不同的Mind **完整参数说明** - | 参数名 | 说明 | 是否必选 | - |-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | - | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -c或--compare_only | 是否自动精度分析，未配置默认开启，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，表示开启，通过配置该参数关闭自动精度分析，仅输出比对结果表格。 | 否 | - | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | 参数名 | 说明 | 是否必选 | + |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | + | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | + | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -c或--compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | + | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | 4. 查看比对结果，请详见PyTorch目录下的《[精度比对工具](../../pytorch/doc/ptdbg_ascend_compare.md)》的“比对结果分析”章节。 diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md index 7c378a584..c3c154b56 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md @@ -36,13 +36,13 @@ **完整参数说明** - | 参数名 | 说明 | 是否必选 | - |-------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | - | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -c或--compare_only | 是否自动精度分析。未配置默认开启，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，表示开启，通过配置该参数关闭自动精度分析，仅输出比对结果表格。 | 否 | - | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | 参数名 | 说明 | 是否必选 | + |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | + | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | + | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -c或--compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | + | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | 3. 查看比对结果，请参见“**比对结果分析**”。 -- Gitee From b30781cab43b80a0b43fa383abf6ff0e23a7040a Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 20 Aug 2024 15:11:33 +0800 Subject: [PATCH 387/791] fix pip install msprobe method --- debug/accuracy_tools/msprobe/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index fe447bbc6..26171923e 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -98,7 +98,8 @@ Successfully installed mindstudio_probe-{version} ```shell python setup.py bdist_wheel - pip install dist/mindstudio_probe*.whl + cd dist + pip install mindstudio_probe*.whl ``` 提示出现如下信息则表示源码安装成功。 ```shell -- Gitee From 517fa7ae20417f52bc072336b5e5781057a3c6be Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 20 Aug 2024 15:17:00 +0800 Subject: [PATCH 388/791] pytest -> unittest + api_input_aggregation --- .../api_accuracy_checker/api_runner.py | 78 +++++++++++-------- .../api_accuracy_checker/test_api_info.py | 24 +++--- .../api_accuracy_checker/test_api_runner.py | 19 +++-- .../test_compare_algorithm.py | 52 ++++++------- .../test_compute_element.py | 58 +++++++------- 5 files changed, 117 insertions(+), 114 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index d3f0c4e56..140413eb2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -12,6 +12,19 @@ from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple +class ApiInputAggregation: + def __init__(self, inputs, kwargs, gradient_inputs) -> None: + ''' + Args: + inputs: List[ComputeElement] + kwargs: dict{str: ComputeElement} + gradient_inputs: Union[List[ComputeElement], None] + ''' + self.inputs = inputs + self.kwargs = kwargs + self.gradient_inputs = gradient_inputs + + class ApiRunner: def __init__(self) -> None: self.api_parent_module_mapping = { @@ -21,16 +34,14 @@ class ApiRunner: (MINT_FUNCTIONAL, TORCH_PLATFORM): torch.nn.functional } - def __call__(self, inputs, api_name_str, kwargs, gradient_inputs=None, - forward_or_backward=FORWARD_API, api_platform=MINDSPORE_PLATFORM): + def __call__(self, api_input_aggregation, api_name_str, forward_or_backward=FORWARD_API, + api_platform=MINDSPORE_PLATFORM): ''' Args: - inputs: List[ComputeElement] - api_name_str: str - kwargs: dict{str: ComputeElement} - gradient_inputs: Union[List[ComputeElement], None] - is_forward: boolean - is_mindspore_api: boolean + api_input_aggregation: ApiInputAggregation + api_name_str: str, e.g. "MintFunctional.relu.0" + forward_or_backward: str, Union["forward_api", "backward_api"] + api_platform: str, Union["mindspore_api", "torch_api"] Return: outputs: list[ComputeElement] @@ -39,22 +50,22 @@ class ApiRunner: run mindspore.mint/torch api ''' api_type_str, api_sub_name = self.get_info_from_name(api_name_str) - api_instance = self.get_api_instance(api_type_str, api_sub_name, api_platform) + api_instance = self._get_api_instance(api_type_str, api_sub_name, api_platform) - self.run_api(api_instance, inputs, kwargs, gradient_inputs, forward_or_backward, api_platform) + self.run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform) - @classmethod - def get_info_from_name(cls, api_name_str): + @staticmethod + def get_info_from_name(api_name_str): ''' Args: - api_name_str: str, the key of data dict in api_info.json. e.g. "MintFunctional.relu.0.backward" + api_name_str: str, the trimmed key of data dict in api_info.json. e.g. "MintFunctional.relu.0" Return: api_type_str: str, Union["MintFunctional", "Mint"] api_sub_name: str, e.g. "relu" ''' api_name_list = api_name_str.split('.') - if len(api_name_list) != 4: + if len(api_name_list) != 3: err_msg = f"ApiRunner.get_info_from_name failed: api_name_str: {api_name_str} is not in defined format" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) api_type_str, api_sub_name = api_name_list[0], api_name_list[1] @@ -64,12 +75,12 @@ class ApiRunner: return api_type_str, api_sub_name - def get_api_instance(self, api_type_str, api_sub_name, api_platform): + def _get_api_instance(self, api_type_str, api_sub_name, api_platform): ''' Args: api_type_str: str, Union["MintFunctional", "Mint"] api_sub_name: str, e.g. "relu" - is_mindspore_api: boolean + api_platform: str: Union["mindpore_api", "torch_api"] Return: api_instance: function object @@ -95,12 +106,13 @@ class ApiRunner: return api_instance - @classmethod - def run_api(cls, api_instance, inputs, kwargs, gradient_inputs, forward_or_backward, api_platform): + @staticmethod + def run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform): inputs = tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) - for compute_element in inputs) + for compute_element in api_input_aggregation.inputs) kwargs = {key: value.get_parameter(get_origin=False, tensor_platform=api_platform) - for key, value in kwargs.items()} + for key, value in api_input_aggregation.kwargs.items()} + gradient_inputs = api_input_aggregation.gradient_inputs if forward_or_backward == FORWARD_API: forward_result = api_instance(*inputs, **kwargs) # can be single tensor or tuple @@ -110,19 +122,15 @@ class ApiRunner: if gradient_inputs is None: err_msg = f"ApiRunner.run_api failed: run backward api but gradient_inputs is missing" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) - if len(gradient_inputs) == 1: - gradient_inputs = gradient_inputs[0].get_parameter(get_origin=False, tensor_platform=api_platform) - else: - gradient_inputs = \ - tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) - for compute_element in gradient_inputs) + gradient_inputs = tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) + for compute_element in gradient_inputs) if api_platform == MINDSPORE_PLATFORM: - if kwargs != {}: - err_msg = f"ApiRunner.run_api failed: backward api with kwargs is currently not supported." - logger.error_log_with_exp(err_msg, - ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) - grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_instance) - backward_result = grad_func(*inputs, gradient_inputs, **kwargs) # can be single tensor or tuple + if len(gradient_inputs) == 1: + gradient_inputs = gradient_inputs[0] + def api_with_kwargs(forward_inputs): + api_instance(forward_inputs, **kwargs) + grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_with_kwargs) + backward_result = grad_func(*inputs, gradient_inputs) # can be single tensor or tuple backward_result_tuple = convert_to_tuple(backward_result) res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_tuple] else: @@ -130,8 +138,10 @@ class ApiRunner: for tensor in inputs: if hasattr(tensor, "requires_grad"): setattr(tensor, "requires_grad", True) - forward_result = api_instance(*inputs, **kwargs) - forward_result.backward(gradient_inputs) + forward_results = api_instance(*inputs, **kwargs) + forward_results = convert_to_tuple(forward_results) + for forward_res, gradient_in in zip(forward_results, gradient_inputs): + forward_res.backward(gradient_in) backward_result_list = [] for tensor in inputs: if hasattr(tensor, "grad"): diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py index 9392780ae..311819796 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py @@ -1,8 +1,7 @@ +import unittest import sys import logging import os - -import pytest import mindspore from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo @@ -14,17 +13,12 @@ logger = logging.getLogger(__name__) file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) - - -class TestClass: +class TestApiInfo(unittest.TestCase): @classmethod - def setup_class(cls): + def setUpClass(cls): """ class level setup_class """ - cls.init(TestClass) - - def init(self): global_context.init(False, os.path.join(directory, "files")) def test_get_compute_element_list(self): @@ -64,17 +58,19 @@ class TestClass: ], } - api_info = ApiInfo("MintFuntional.gelu.0.forward") + api_info = ApiInfo("MintFuntional.gelu.0") api_info.load_forward_info(forward_api_info_dict) - assert api_info.check_forward_info() == True - assert api_info.check_backward_info() == False + self.assertTrue(api_info.check_forward_info()) + self.assertFalse(api_info.check_backward_info()) input_compute_element_list = api_info.get_compute_element_list("forward_api", "input") parameter_real = input_compute_element_list[0].get_parameter() parameter_target = mindspore.Tensor([1., 2., 3.]) - assert (parameter_real == parameter_target).all() + self.assertTrue((parameter_real == parameter_target).all()) kwargs_compute_element_dict = api_info.get_kwargs() - assert kwargs_compute_element_dict.get("approximate").get_parameter() == "tanh" + self.assertEqual(kwargs_compute_element_dict.get("approximate").get_parameter(), "tanh") +if __name__ == '__main__': + unittest.main() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py index d035be420..b326f1a25 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py @@ -7,7 +7,7 @@ import mindspore import torch from unittest.mock import MagicMock -from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner +from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation from msprobe.mindspore.api_accuracy_checker.const import MINDSPORE_PLATFORM, TORCH_PLATFORM, FORWARD_API, BACKWARD_API logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') @@ -67,17 +67,20 @@ class TestClass: forward_result = [mock_compute_element_forward_result_instance] backward_result = [mock_compute_element_backward_result_instance] + forward_api_input_aggregation = ApiInputAggregation(inputs, kwargs, None) + backward_api_input_aggregation = ApiInputAggregation(inputs, {}, gradient_inputs) - # api_instance, inputs, kwargs, gradient_inputs, forward_or_backward, api_platform, result + + # api_instance, api_input_aggregation, forward_or_backward, api_platform, result test_cases = [ - [mindspore.mint.nn.functional.gelu, inputs, kwargs, gradient_inputs, FORWARD_API, MINDSPORE_PLATFORM, forward_result], - [mindspore.mint.nn.functional.gelu, inputs, {}, gradient_inputs, BACKWARD_API, MINDSPORE_PLATFORM, backward_result], - [torch.nn.functional.gelu, inputs, kwargs, gradient_inputs, FORWARD_API, TORCH_PLATFORM, forward_result], - [torch.nn.functional.gelu, inputs, {}, gradient_inputs, BACKWARD_API, TORCH_PLATFORM, backward_result], + [mindspore.mint.nn.functional.gelu, forward_api_input_aggregation, FORWARD_API, MINDSPORE_PLATFORM, forward_result], + [mindspore.mint.nn.functional.gelu, backward_api_input_aggregation, BACKWARD_API, MINDSPORE_PLATFORM, backward_result], + [torch.nn.functional.gelu, forward_api_input_aggregation, FORWARD_API, TORCH_PLATFORM, forward_result], + [torch.nn.functional.gelu, backward_api_input_aggregation, BACKWARD_API, TORCH_PLATFORM, backward_result], ] for test_case in test_cases: - api_instance, inputs_target, kwargs_target, gradient_inputs_target, forward_or_backward, api_platform, results_target = test_case - results_real = api_runner.run_api(api_instance, inputs_target, kwargs_target, gradient_inputs_target, forward_or_backward, api_platform) + api_instance, api_input_aggregation, forward_or_backward, api_platform, results_target = test_case + results_real = api_runner.run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform) for res_real, res_target in zip(results_real, results_target): assert (abs(res_real.get_parameter() - res_target.get_parameter(tensor_platform=api_platform)) < 1e-5).all() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py index e7b1e7181..35b00bc40 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py @@ -1,8 +1,7 @@ +import unittest import sys import logging import os - -import pytest import mindspore import torch from unittest.mock import MagicMock @@ -17,35 +16,30 @@ file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) -@pytest.fixture -def mock_mstensor_compute_element(): - mock = MagicMock() - mock.get_parameter.return_value = mindspore.Tensor([1., 1.9, 3.]) - mock.get_shape.return_value = (3,) - return mock - -@pytest.fixture -def mock_torchtensor_compute_element(): - mock = MagicMock() - mock.get_parameter.return_value = torch.Tensor([1., 2., 3.]) - mock.get_shape.return_value = (3,) - return mock - +class TestCompareAlgorithms(unittest.TestCase): -class TestClass: + def setUp(self): + self.mock_torchtensor_compute_element = MagicMock() + self.mock_torchtensor_compute_element.get_parameter.return_value = torch.Tensor([1., 2., 3.]) + self.mock_torchtensor_compute_element.get_shape.return_value = (3,) + self.mock_mstensor_compute_element = MagicMock() + self.mock_mstensor_compute_element.get_parameter.return_value = mindspore.Tensor([1., 1.9, 3.]) + self.mock_mstensor_compute_element.get_shape.return_value = (3,) - def test_cosine_similarity(self, mock_torchtensor_compute_element, mock_mstensor_compute_element): - compare_result = compare_algorithms[COSINE_SIMILARITY](mock_torchtensor_compute_element, mock_mstensor_compute_element) - assert abs(compare_result.compare_value - 0.9997375534689601) < 1e-5 - assert compare_result.pass_status == ERROR + def test_cosine_similarity(self): + compare_result = compare_algorithms[COSINE_SIMILARITY](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) + self.assertAlmostEqual(compare_result.compare_value, 0.9997375534689601, places=5) + self.assertEqual(compare_result.pass_status, ERROR) + def test_max_absolute_difference(self): + compare_result = compare_algorithms[MAX_ABSOLUTE_DIFF](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) + self.assertAlmostEqual(compare_result.compare_value, 0.1, places=5) + self.assertEqual(compare_result.pass_status, ERROR) - def test_max_absolute_difference(self, mock_torchtensor_compute_element, mock_mstensor_compute_element): - compare_result = compare_algorithms[MAX_ABSOLUTE_DIFF](mock_torchtensor_compute_element, mock_mstensor_compute_element) - assert abs(compare_result.compare_value - 0.1) < 1e-5 - assert compare_result.pass_status == ERROR + def test_max_relative_difference(self): + compare_result = compare_algorithms[MAX_RELATIVE_DIFF](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) + self.assertAlmostEqual(compare_result.compare_value, 0.05, places=5) + self.assertEqual(compare_result.pass_status, ERROR) - def test_max_relative_difference(self, mock_torchtensor_compute_element, mock_mstensor_compute_element): - compare_result = compare_algorithms[MAX_RELATIVE_DIFF](mock_torchtensor_compute_element, mock_mstensor_compute_element) - assert abs(compare_result.compare_value - 0.05) < 1e-5 - assert compare_result.pass_status == ERROR \ No newline at end of file +if __name__ == '__main__': + unittest.main() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py index 9e88657e3..12714ac34 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -1,8 +1,7 @@ +import unittest import sys import logging import os - -import pytest import mindspore import torch import numpy as np @@ -20,14 +19,13 @@ file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) - -class TestClass: +class TestComputeElement(unittest.TestCase): @classmethod - def setup_class(cls): + def setUpClass(cls): """ class level setup_class """ - cls.init(TestClass) + cls.init(TestComputeElement) def init(self): global_context.init(False, os.path.join(directory, "files")) @@ -48,11 +46,11 @@ class TestClass: compute_element = ComputeElement(parameter=input_parameter) - assert (compute_element.get_parameter(get_origin=True) == origin_parameter).all() - assert (compute_element.get_parameter(get_origin=False, tensor_platform=MINDSPORE_PLATFORM) == mstensor_parameter).all() - assert (compute_element.get_parameter(get_origin=False, tensor_platform=TORCH_PLATFORM) == torchtensor_parameter).all() - assert compute_element.get_shape() == shape - assert compute_element.get_dtype() == dtype_str + self.assertTrue((compute_element.get_parameter(get_origin=True) == origin_parameter).all()) + self.assertTrue((compute_element.get_parameter(get_origin=False, tensor_platform=MINDSPORE_PLATFORM) == mstensor_parameter).all()) + self.assertTrue((compute_element.get_parameter(get_origin=False, tensor_platform=TORCH_PLATFORM) == torchtensor_parameter).all()) + self.assertEqual(compute_element.get_shape(), shape) + self.assertEqual(compute_element.get_dtype(), dtype_str) def test_init_with_parameter_other_type(self): # input_parameter, origin_parameter, shape, dtype_str @@ -69,9 +67,9 @@ class TestClass: compute_element = ComputeElement(parameter=input_parameter) - assert compute_element.get_parameter(get_origin=True) == origin_parameter - assert compute_element.get_shape() == shape - assert compute_element.get_dtype() == dtype_str + self.assertEqual(compute_element.get_parameter(get_origin=True), origin_parameter) + self.assertEqual(compute_element.get_shape(), shape) + self.assertEqual(compute_element.get_dtype(), dtype_str) def test_init_with_compute_element_info_mstensor(self): global_context.is_constructed = False @@ -84,9 +82,9 @@ class TestClass: "data_name": "input.npy" } compute_element = ComputeElement(compute_element_info=compute_element_info) - assert (compute_element.get_parameter(get_origin=True) == self.ms_tensor).all() - assert compute_element.get_shape() == self.tensor_shape - assert compute_element.get_dtype() == FLOAT32 + self.assertTrue((compute_element.get_parameter(get_origin=True) == self.ms_tensor).all()) + self.assertEqual(compute_element.get_shape(), self.tensor_shape) + self.assertEqual(compute_element.get_dtype(), FLOAT32) def test_init_with_compute_element_info_mstensor_constructed(self): global_context.is_constructed = True @@ -100,10 +98,10 @@ class TestClass: } compute_element = ComputeElement(compute_element_info=compute_element_info) parameter = compute_element.get_parameter(get_origin=True) - assert (parameter <= 3.0).all() - assert (parameter >= 1.0).all() - assert compute_element.get_shape() == self.tensor_shape - assert compute_element.get_dtype() == FLOAT32 + self.assertTrue((parameter <= 3.0).all()) + self.assertTrue((parameter >= 1.0).all()) + self.assertEqual(compute_element.get_shape(), self.tensor_shape) + self.assertEqual(compute_element.get_dtype(), FLOAT32) def test_init_with_compute_element_info_tuple(self): global_context.is_constructed = False @@ -127,11 +125,10 @@ class TestClass: ] compute_element = ComputeElement(compute_element_info=compute_element_info) parameter = compute_element.get_parameter(get_origin=True) - assert (parameter[0] == self.ms_tensor).all() - assert (parameter[1] == self.ms_tensor).all() - assert compute_element.get_shape() == tuple() - assert compute_element.get_dtype() == TUPLE_TYPE_STR - + self.assertTrue((parameter[0] == self.ms_tensor).all()) + self.assertTrue((parameter[1] == self.ms_tensor).all()) + self.assertEqual(compute_element.get_shape(), tuple()) + self.assertEqual(compute_element.get_dtype(), TUPLE_TYPE_STR) def test_init_with_compute_element_info_int(self): compute_element_info = { @@ -140,6 +137,9 @@ class TestClass: } compute_element = ComputeElement(compute_element_info=compute_element_info) parameter = compute_element.get_parameter(get_origin=True) - assert parameter == -1 - assert compute_element.get_shape() == tuple() - assert compute_element.get_dtype() == INT_TYPE_STR + self.assertEqual(parameter, -1) + self.assertEqual(compute_element.get_shape(), tuple()) + self.assertEqual(compute_element.get_dtype(), INT_TYPE_STR) + +if __name__ == '__main__': + unittest.main() -- Gitee From c44079ba245cb49aa533136dda3a21ef4ccef59d Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 20 Aug 2024 15:37:19 +0800 Subject: [PATCH 389/791] api_runner fix --- .../api_accuracy_checker/test_api_runner.py | 157 +++++++++++------- 1 file changed, 99 insertions(+), 58 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py index b326f1a25..dad6d6047 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py @@ -1,8 +1,7 @@ +import unittest import sys import logging import os - -import pytest import mindspore import torch from unittest.mock import MagicMock @@ -16,67 +15,109 @@ logger = logging.getLogger(__name__) file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) - -# 创建一个包含if判断的mock实例的fixture -@pytest.fixture -def mock_compute_element_input_instance(): - mock = MagicMock() - def side_effect(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: - return mindspore.Tensor([1., 2., 3.]) - else: - return torch.Tensor([1., 2., 3.]) - mock.get_parameter.side_effect = side_effect - return mock - -@pytest.fixture -def mock_compute_element_kwargs_instance(): - mock = MagicMock() - mock.get_parameter.return_value = "tanh" - return mock - -@pytest.fixture -def mock_compute_element_forward_result_instance(): - mock = MagicMock() - def side_effect(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: - return mindspore.Tensor([8.41192007e-01, 1.95459759e+00, 2.99636269e+00]) - else: - return torch.Tensor([8.41192007e-01, 1.95459759e+00, 2.99636269e+00]) - mock.get_parameter.side_effect = side_effect - return mock - -@pytest.fixture -def mock_compute_element_backward_result_instance(): - mock = MagicMock() - def side_effect(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: - return mindspore.Tensor([1.0833155, 2.1704636, 3.0358372]) - else: - return torch.Tensor([1.0833155, 2.1704636, 3.0358372]) - mock.get_parameter.side_effect = side_effect - return mock - -class TestClass: - - def test_run_api(self, mock_compute_element_kwargs_instance, mock_compute_element_input_instance, - mock_compute_element_forward_result_instance, mock_compute_element_backward_result_instance): - kwargs = {"approximate": mock_compute_element_kwargs_instance} - inputs = [mock_compute_element_input_instance] - gradient_inputs = [mock_compute_element_input_instance] - forward_result = [mock_compute_element_forward_result_instance] - backward_result = [mock_compute_element_backward_result_instance] +def func(x_1, x_2, opt="opt1"): + y_1 = x_1*2 + 1 + if opt == "opt1": + y_2 = x_1 + x_2 + else: + y_2 = x_1*2 + x_2 + return y_1, y_2 + +def side_effect_forward_input_1(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([1., 2., 3.]) + else: + return torch.Tensor([1., 2., 3.]) + +def side_effect_forward_input_2(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([1.1, 2., 3.]) + else: + return torch.Tensor([1.1, 2., 3.]) + +def side_effect_forward_output_1(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([3., 5., 7.]) + else: + return torch.Tensor([3., 5., 7.]) + +def side_effect_forward_output_2(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([2.1, 4., 6.]) + else: + return torch.Tensor([2.1, 4., 6.]) + +def side_effect_backward_input_1(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([1., 2., 3.]) + else: + return torch.Tensor([1., 2., 3.]) + +def side_effect_backward_input_2(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([1.11, 2., 3.]) + else: + return torch.Tensor([1.11, 2., 3.]) + +def side_effect_backward_output_1(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([3.11, 6., 9.]) + else: + return torch.Tensor([3.11, 6., 9.]) + +def side_effect_backward_output_2(**kwargs): + if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + return mindspore.Tensor([1.11, 2., 3.]) + else: + return torch.Tensor([1.11, 2., 3.]) + + +class TestApiRunner(unittest.TestCase): + + def setUp(self): + self.mock_compute_element_kwargs_instance = MagicMock() + self.mock_compute_element_kwargs_instance.get_parameter.return_value = "opt1" + + self.mock_compute_element_forward_input_1_instance = MagicMock() + self.mock_compute_element_forward_input_1_instance.get_parameter.side_effect = side_effect_forward_input_1 + + self.mock_compute_element_forward_input_2_instance = MagicMock() + self.mock_compute_element_forward_input_2_instance.get_parameter.side_effect = side_effect_forward_input_2 + + self.mock_compute_element_backward_input_1_instance = MagicMock() + self.mock_compute_element_backward_input_1_instance.get_parameter.side_effect = side_effect_backward_input_1 + + self.mock_compute_element_backward_input_2_instance = MagicMock() + self.mock_compute_element_backward_input_2_instance.get_parameter.side_effect = side_effect_backward_input_2 + + self.mock_compute_element_forward_output_1_instance = MagicMock() + self.mock_compute_element_forward_output_1_instance.get_parameter.side_effect = side_effect_forward_output_1 + + self.mock_compute_element_forward_output_2_instance = MagicMock() + self.mock_compute_element_forward_output_2_instance.get_parameter.side_effect = side_effect_forward_output_2 + + self.mock_compute_element_backward_output_1_instance = MagicMock() + self.mock_compute_element_backward_output_1_instance.get_parameter.side_effect = side_effect_backward_output_1 + + self.mock_compute_element_backward_output_2_instance = MagicMock() + self.mock_compute_element_backward_output_2_instance.get_parameter.side_effect = side_effect_backward_output_2 + + def test_run_api(self): + kwargs = {"opt": self.mock_compute_element_kwargs_instance} + inputs = [self.mock_compute_element_forward_input_1_instance, self.mock_compute_element_forward_input_2_instance] + gradient_inputs = [self.mock_compute_element_backward_input_1_instance, self.mock_compute_element_backward_input_2_instance] + forward_result = [self.mock_compute_element_forward_output_1_instance, self.mock_compute_element_forward_output_2_instance] + backward_result = [self.mock_compute_element_backward_output_1_instance, self.mock_compute_element_backward_output_2_instance] forward_api_input_aggregation = ApiInputAggregation(inputs, kwargs, None) backward_api_input_aggregation = ApiInputAggregation(inputs, {}, gradient_inputs) - # api_instance, api_input_aggregation, forward_or_backward, api_platform, result test_cases = [ - [mindspore.mint.nn.functional.gelu, forward_api_input_aggregation, FORWARD_API, MINDSPORE_PLATFORM, forward_result], - [mindspore.mint.nn.functional.gelu, backward_api_input_aggregation, BACKWARD_API, MINDSPORE_PLATFORM, backward_result], - [torch.nn.functional.gelu, forward_api_input_aggregation, FORWARD_API, TORCH_PLATFORM, forward_result], - [torch.nn.functional.gelu, backward_api_input_aggregation, BACKWARD_API, TORCH_PLATFORM, backward_result], + [func, forward_api_input_aggregation, FORWARD_API, MINDSPORE_PLATFORM, forward_result], + [func, backward_api_input_aggregation, BACKWARD_API, MINDSPORE_PLATFORM, backward_result], + [func, forward_api_input_aggregation, FORWARD_API, TORCH_PLATFORM, forward_result], + [func, backward_api_input_aggregation, BACKWARD_API, TORCH_PLATFORM, backward_result], ] for test_case in test_cases: api_instance, api_input_aggregation, forward_or_backward, api_platform, results_target = test_case @@ -96,7 +137,7 @@ class TestClass: assert api_runner.get_api_instance(api_type_str, api_sub_name, api_platform) == result_api def test_get_info_from_name(self): - api_name = "MintFunctional.relu.0.backward" + api_name = "MintFunctional.relu.0" api_type_str, api_sub_name = api_runner.get_info_from_name(api_name_str=api_name) assert api_type_str == "MintFunctional" assert api_sub_name == "relu" -- Gitee From 5c0fd7814d6b4053aab43ca4150ef8a9a0d713db Mon Sep 17 00:00:00 2001 From: yangxinxian <947098055@qq.com> Date: Tue, 20 Aug 2024 15:43:39 +0800 Subject: [PATCH 390/791] =?UTF-8?q?=E3=80=90msprobe=E3=80=91=E3=80=90featu?= =?UTF-8?q?re=E3=80=91=E6=8C=87=E5=AE=9Astep=E9=9C=80=E6=B1=82model.train?= =?UTF-8?q?=E6=96=B9=E5=BC=8F=E4=BD=BF=E7=94=A8=E6=96=87=E6=A1=A3=E4=BB=A5?= =?UTF-8?q?=E5=8F=8AUT=E6=B7=BB=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mindspore/debugger/debugger_config.py | 4 ++-- .../msprobe/mindspore/doc/dump.md | 17 ++++++++++++++ .../test/mindspore_ut/test_primitive_dump.py | 22 +++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index fa8cc85a9..24b61f3df 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -63,8 +63,8 @@ class DebuggerConfig: def _check_step(self): for s in self.step: - if not isinstance(s, int): - raise ValueError(f"step element {s} should be int") + if not isinstance(s, int) or s < 0: + raise ValueError(f"step element {s} must be a positive integer.") def _make_dump_path_if_not_exists(self): check_path_before_create(self.dump_path) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md index 0d45e1b14..c9b8c38f2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md @@ -101,6 +101,8 @@ debugger.start() ### MindSpore动态图场景 +当使用模型使用for循环时，在每个迭代的开始插入debugger.start()，在每个迭代的结束插入debugger.stop()与debugger.step()： + ```Python import mindspore as ms from msprobe.mindspore import PrecisionDebugger @@ -122,6 +124,21 @@ for data, label in data_loader: debugger.step() # 结束一个step的dump ``` +当使用模型的train方法而非for循环时，可以通过在callbacks参数中传入MsprobeStep(debugger)： + +```Python +from msprobe.mindspore.common.utils import MsprobeStep +from msprobe.mindspore import PrecisionDebugger + +# 初始化PrecisionDebugger +debugger = PrecisionDebugger(config_path="./config.json") + +# 自动在每个step开始时调用start()，在每个step结束时调用stop()和step()。 +# 这意味着您无需手动在循环内添加start、stop和step函数，框架会自动完成数据的dump操作。 +trainer.train(1, dataset_train, callbacks=[loss_monior, MsprobeStep(debugger)]) + +``` + ## dump结果文件介绍 ### MindSpore静态图场景 diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py index b85fbacd4..aa6372fa3 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py @@ -23,6 +23,8 @@ from msprobe.mindspore.service import Service from msprobe.core.common.exceptions import MsprobeException from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig +from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell +from collections import defaultdict class DummyModel(nn.Cell): @@ -76,3 +78,23 @@ class TestService(unittest.TestCase): self.assertEqual(self.service.primitive_counters[primitive_name], 0) self.service.update_primitive_counters(primitive_name) self.assertEqual(self.service.primitive_counters[primitive_name], 1) + + def test_step_updates_iteration(self): + initial_iter = self.service.current_iter + self.service.step() + self.assertEqual(self.service.current_iter, initial_iter + 1) + + @patch.object(HOOKCell, 'cell_count', new_callable=lambda: defaultdict(int)) + def test_step_resets_counters(self, _): + # 假设在 step 调用之前已经有一些 primitive_counters + self.service.primitive_counters["test_primitive"] = 5 + self.service.step() + self.assertEqual(self.service.primitive_counters, {}) + self.assertEqual(HOOKCell.cell_count, defaultdict(int)) + + def test_step_calls_update_iter(self): + # 检查是否在调用 step 时调用了 update_iter + with patch.object(self.service.data_collector, 'update_iter') as mock_update_iter: + initial_iter = self.service.current_iter + self.service.step() + mock_update_iter.assert_called_once_with(initial_iter + 1) \ No newline at end of file -- Gitee From bb82442df7bd2b914629a4ddcae8cee233bda67a Mon Sep 17 00:00:00 2001 From: yangxinxian <947098055@qq.com> Date: Tue, 20 Aug 2024 15:57:52 +0800 Subject: [PATCH 391/791] =?UTF-8?q?=E6=95=B0=E6=8D=AE=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E4=B8=BAfloat16=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/data_dump/data_processor/base.py | 10 ++++++++++ .../data_dump/data_processor/mindspore_processor.py | 8 ++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index b0ebe5da9..f595aedda 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -110,6 +110,16 @@ class BaseDataProcessor: stack_info_struct = {name: stack_str} return stack_info_struct + @staticmethod + def transfer_type(data): + dtype = str(type(data)) + if 'int' in dtype: + return int(data) + elif 'float' in dtype: + return float(data) + else: + return data + @staticmethod def _convert_numpy_to_builtin(arg): type_mapping = { diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index b9747a88e..c822322b7 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -117,10 +117,10 @@ class MindsporeDataProcessor(BaseDataProcessor): 'type': 'mindspore.Tensor', 'dtype': str(tensor.dtype), 'shape': tensor.shape, - 'Max': tensor_stat.max, - 'Min': tensor_stat.min, - 'Mean': tensor_stat.mean, - 'Norm': tensor_stat.norm + 'Max': self.transfer_type(tensor_stat.max), + 'Min': self.transfer_type(tensor_stat.min), + 'Mean': self.transfer_type(tensor_stat.mean), + 'Norm': self.transfer_type(tensor_stat.norm), } if self.config.summary_mode == Const.MD5: tensor_md5 = self.get_md5_for_tensor(tensor) -- Gitee From 16c81bc970f76069214add34b85a81b07fd8d955 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Tue, 20 Aug 2024 08:12:24 +0000 Subject: [PATCH 392/791] update debug/accuracy_tools/msprobe/core/common/utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/core/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 5a41f1337..c390703ad 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -527,7 +527,7 @@ def write_csv(data, filepath): with FileOpen(filepath, 'a+', encoding='utf-8-sig') as f: writer = csv.writer(f) writer.writerows(data) - if is_first_create: + if not is_first_create: change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) -- Gitee From cf04bb4017b569c4e9e5de268be351e58b4a9e21 Mon Sep 17 00:00:00 2001 From: hanqing Date: Mon, 19 Aug 2024 15:12:51 +0800 Subject: [PATCH 393/791] addressing security issues --- .../msprobe/core/common/utils.py | 22 ++++ .../msprobe/pytorch/parse_tool/lib/compare.py | 23 ++-- .../msprobe/pytorch/parse_tool/lib/config.py | 1 + .../pytorch/parse_tool/lib/parse_tool.py | 8 +- .../msprobe/pytorch/parse_tool/lib/utils.py | 109 ++++-------------- .../pytorch/parse_tool/lib/visualization.py | 7 +- 6 files changed, 72 insertions(+), 98 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 7fa78c95d..13bb7d4a8 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -304,6 +304,7 @@ def create_directory(dir_path): when invalid data throw exception """ if not os.path.exists(dir_path): + check_path_before_create(dir_path) try: os.makedirs(dir_path, mode=0o700) except OSError as ex: @@ -523,9 +524,12 @@ def convert_tuple(data): def write_csv(data, filepath): + exist = os.path.exists(filepath) with FileOpen(filepath, 'a+', encoding='utf-8-sig') as f: writer = csv.writer(f) writer.writerows(data) + if not exist: + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) def load_npy(filepath): @@ -545,3 +549,21 @@ def save_npy(data, filepath): except Exception as e: raise RuntimeError(f"save npy file {filepath} failed") from e change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + +def save_npy_to_txt(self, data, dst_file='', align=0): + if os.path.exists(dst_file): + self.log.info("Dst file %s exists, will not save new one.", dst_file) + return + shape = data.shape + data = data.flatten() + if align == 0: + align = 1 if len(shape) == 0 else shape[-1] + elif data.size % align != 0: + pad_array = np.zeros((align - data.size % align,)) + data = np.append(data, pad_array) + check_path_before_create(dst_file) + try: + np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g') + except Exception as e: + self.log.error("An unexpected error occurred: %s when savetxt to %s" % (str(e)), dst_file) + change_mode(dst_file, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py index 2b091c59e..b24065915 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py @@ -22,6 +22,8 @@ from collections import namedtuple from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException +from msprobe.core.common.utils import create_directory, write_csv, save_npy_to_txt +from msprobe.core.common.file_check import FileChecker class Compare: @@ -36,7 +38,7 @@ class Compare: self.log.info("Compare finished!!") def compare_vector(self, my_dump_path, golden_dump_path, result_dir, msaccucmp_path): - self.util.create_dir(result_dir) + create_directory(result_dir) self.util.check_path_valid(result_dir) call_msaccucmp = self.util.check_msaccucmp(msaccucmp_path) cmd = '%s %s compare -m %s -g %s -out %s' % ( @@ -65,7 +67,7 @@ class Compare: self.util.print_panel("\n".join(summary_txt)) def convert(self, dump_file, data_format, output, msaccucmp_path): - self.util.create_dir(output) + create_directory(output) self.util.check_path_valid(output) call_msaccucmp = self.util.check_msaccucmp(msaccucmp_path) if data_format: @@ -97,8 +99,8 @@ class Compare: # save to txt if save_txt: - self.util.save_npy_to_txt(left_data, left + ".txt") - self.util.save_npy_to_txt(right_data, right + ".txt") + save_npy_to_txt(left_data, left + ".txt") + save_npy_to_txt(right_data, right + ".txt") # compare data (total_cnt, all_close, cos_sim, err_percent) = self.do_compare_data(left_data, right_data, rl, al, diff_count) content = ['Left:', ' ├─ NpyFile: %s' % left] @@ -184,7 +186,7 @@ class Compare: rel_diff_max = np.max(rel_error) compare_result = [[filename, bench_filename, data_mean, bench_data_mean, md5_consistency, abs_diff_max, rel_diff_max]] - self.util.write_csv(compare_result, output_path) + write_csv(compare_result, output_path) def compare_all_file_in_directory(self, my_dump_dir, golden_dump_dir, output_path): if not (self.util.is_subdir_count_equal(my_dump_dir, golden_dump_dir) @@ -231,7 +233,7 @@ class Compare: "Max Abs Error", "Max Relative Error" ]] - self.util.write_csv(title_rows, output_path) + write_csv(title_rows, output_path) my_ordered_subdirs = self.util.get_sorted_subdirectories_names(my_dump_dir) golden_ordered_subdirs = self.util.get_sorted_subdirectories_names(golden_dump_dir) @@ -249,7 +251,9 @@ class Compare: def convert_api_dir_to_npy(self, dump_dir, param, output_dir, msaccucmp_path): dump_dir = self.util.path_strip(dump_dir) - for root, _, files in os.walk(dump_dir): + for root, _, files in os.walk(dump_dir, topdown=True): + path_checker = FileChecker(root) + path_checker.common_check() for file in files: file_path = os.path.join(root, file) file_name = os.path.basename(file_path) @@ -260,3 +264,8 @@ class Compare: timestamp = parts[-1] output_path = os.path.join(output_dir, op_name, timestamp) self.convert_dump_to_npy(file_path, param, output_path, msaccucmp_path) + path_depth = root.count(os.sep) + if path_depth <= Const.MAX_TRAVERSAL_DEPTH: + yield root, _, files + else: + _[:] = [] diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/config.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/config.py index a9a8b2b00..176295ad9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/config.py @@ -38,6 +38,7 @@ class Const: PKL_SUFFIX = ".pkl" DIRECTORY_LENGTH = 4096 FILE_NAME_LENGTH = 255 + MAX_TRAVERSAL_DEPTH = 5 FILE_PATTERN = r'^[a-zA-Z0-9_./-]+$' ONE_GB = 1 * 1024 * 1024 * 1024 TEN_GB = 10 * 1024 * 1024 * 1024 diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py index 9a47dc54c..67c981412 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py @@ -23,7 +23,7 @@ from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.compare import Compare from msprobe.pytorch.parse_tool.lib.visualization import Visualization from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception, ParseException - +from msprobe.core.common.utils import create_directory class ParseTool: def __init__(self): @@ -33,7 +33,7 @@ class ParseTool: @catch_exception def prepare(self): - self.util.create_dir(Const.DATA_ROOT_DIR) + create_directory(Const.DATA_ROOT_DIR) @catch_exception def do_vector_compare(self, args): @@ -112,8 +112,8 @@ class ParseTool: args = parser.parse_args(argv) self.util.check_path_valid(args.my_dump_path) self.util.check_path_valid(args.golden_dump_path) - self.util.check_path_format(args.my_dump_path, Const.NPY_SUFFIX) - self.util.check_path_format(args.golden_dump_path, Const.NPY_SUFFIX) + self.util.check_file_path_format(args.my_dump_path, Const.NPY_SUFFIX) + self.util.check_file_path_format(args.golden_dump_path, Const.NPY_SUFFIX) compare_data_args = namedtuple('compare_data_args', ['my_dump_path', 'golden_dump_path', 'save', 'rtol', 'atol', 'count']) compare_data_args.__new__.__defaults__ = (False, 0.001, 0.001, 20) res = compare_data_args(args.my_dump_path, args.golden_dump_path, args.save, args.rtol, args.atol, args.count) diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py index 17a01f20f..66d1dee95 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py @@ -31,8 +31,8 @@ from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException from msprobe.core.common.file_check import change_mode, check_other_user_writable,\ check_path_executable, check_path_owner_consistent from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create +from msprobe.core.common.file_check import FileOpen, FileChecker +from msprobe.core.common.utils import check_file_or_directory_path from msprobe.pytorch.common.log import logger @@ -57,12 +57,7 @@ except ImportError as err: class Util: def __init__(self): self.ms_accu_cmp = None - logging.basicConfig( - level=Const.LOG_LEVEL, - format="%(asctime)s (%(process)d) -[%(levelname)s]%(message)s", - datefmt="%Y-%m-%d %H:%M:%S" - ) - self.log = logging.getLogger() + self.log = logger self.python = sys.executable @staticmethod @@ -128,17 +123,6 @@ class Util: md5_hash = hashlib.md5(np_bytes) return md5_hash.hexdigest() - @staticmethod - def write_csv(self, data, filepath): - need_change_mode = False - if not os.path.exists(filepath): - need_change_mode = True - with FileOpen(filepath, 'a') as f: - writer = csv.writer(f) - writer.writerows(data) - if need_change_mode: - change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) - @staticmethod def deal_with_dir_or_file_inconsistency(self, output_path): if os.path.exists(output_path): @@ -160,10 +144,17 @@ class Util: @staticmethod def dir_contains_only(self, path, endfix): - for _, _, files in os.walk(path): + for root, _, files in os.walk(path, topdown=True): + path_checker = FileChecker(root) + path_checker.common_check() for file in files: if not file.endswith(endfix): return False + path_depth = root.count(os.sep) + if path_depth <= Const.MAX_TRAVERSAL_DEPTH: + yield root, _, files + else: + _[:] = [] return True @staticmethod @@ -188,7 +179,7 @@ class Util: if not cmd: self.log.error("Commond is None") return -1 - self.log.debug("[RUN CMD]: %s", cmd) + self.log.info("[RUN CMD]: %s", cmd) cmd = cmd.split(" ") complete_process = subprocess.run(cmd, shell=False) return complete_process.returncode @@ -208,7 +199,7 @@ class Util: "Check msaccucmp failed in dir %s. This is not a correct msaccucmp file" % target_file) raise ParseException(ParseException.PARSE_MSACCUCMP_ERROR) result = subprocess.run( - [self.python, target_file, "--help"], stdout=subprocess.PIPE) + [self.python, target_file, "--help"], stdout=subprocess.PIPE, shell=False) if result.returncode == 0: self.log.info("Check [%s] success.", target_file) else: @@ -217,41 +208,12 @@ class Util: raise ParseException(ParseException.PARSE_MSACCUCMP_ERROR) return target_file - def create_dir(self, path): - path = self.path_strip(path) - if os.path.exists(path): - return - self.check_path_name(path) - try: - os.makedirs(path, mode=FileCheckConst.DATA_DIR_AUTHORITY) - except OSError as e: - self.log.error("Failed to create %s.", path) - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) from e - def gen_npy_info_txt(self, source_data): (shape, dtype, max_data, min_data, mean) = \ self.npy_info(source_data) return \ '[Shape: %s] [Dtype: %s] [Max: %s] [Min: %s] [Mean: %s]' % (shape, dtype, max_data, min_data, mean) - def save_npy_to_txt(self, data, dst_file='', align=0): - if os.path.exists(dst_file): - self.log.info("Dst file %s exists, will not save new one.", dst_file) - return - shape = data.shape - data = data.flatten() - if align == 0: - align = 1 if len(shape) == 0 else shape[-1] - elif data.size % align != 0: - pad_array = np.zeros((align - data.size % align,)) - data = np.append(data, pad_array) - check_path_before_create(dst_file) - try: - np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g') - except Exception as e: - self.log.error("An unexpected error occurred: %s when savetxt to %s" % (str(e)), dst_file) - change_mode(dst_file, FileCheckConst.DATA_FILE_AUTHORITY) - def list_convert_files(self, path, external_pattern=""): return self.list_file_with_pattern( path, Const.OFFLINE_DUMP_CONVERT_PATTERN, external_pattern, self._gen_npu_dump_convert_file_info @@ -278,27 +240,8 @@ class Util: def check_path_valid(self, path): path = self.path_strip(path) - if not path or not os.path.exists(path): - self.log.error("The path %s does not exist." % path) - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) - if os.path.islink(path): - self.log.error('The file path {} is a soft link.'.format(path)) - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) - if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ - Const.FILE_NAME_LENGTH: - self.log.error('The file path length exceeds limit.') - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) - if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): - self.log.error('The file path {} contains special characters.'.format(path)) - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) - if os.path.isfile(path): - file_size = os.path.getsize(path) - if path.endswith(Const.PKL_SUFFIX) and file_size > Const.ONE_GB: - self.log.error('The file {} size is greater than 1GB.'.format(path)) - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) - if path.endswith(Const.NPY_SUFFIX) and file_size > Const.TEN_GB: - self.log.error('The file {} size is greater than 10GB.'.format(path)) - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) + path_checker = FileChecker(path) + path_checker.common_check() return True def check_files_in_path(self, path): @@ -326,17 +269,24 @@ class Util: self.check_path_valid(path) file_list = {} re_pattern = re.compile(pattern) - for dir_path, _, file_names in os.walk(path, followlinks=True): + for dir_path, _, file_names in os.walk(path, topdown=True): + path_checker = FileChecker(dir) + path_checker.common_check() for name in file_names: match = re_pattern.match(name) if not match: continue - if extern_pattern != '' and not re.match(extern_pattern, name): + if extern_pattern != '' and re_pattern.match(extern_pattern) and not re.match(extern_pattern, name): continue file_list[name] = gen_info_func(name, match, dir_path) + path_depth = dir_path.count(os.sep) + if path_depth <= Const.MAX_TRAVERSAL_DEPTH: + yield dir_path, _, file_names + else: + _[:] = [] return file_list - def check_path_format(self, path, suffix): + def check_file_path_format(self, path, suffix): if os.path.isfile(path): if not path.endswith(suffix): self.log.error("%s is not a %s file." % (path, suffix)) @@ -348,15 +298,6 @@ class Util: self.log.error("The file path %s is invalid" % path) raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) - def check_path_name(self, path): - if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ - Const.FILE_NAME_LENGTH: - self.log.error('The file path length exceeds limit.') - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) - if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): - self.log.error('The file path {} contains special characters.'.format(path)) - raise ParseException(ParseException.PARSE_INVALID_PATH_ERROR) - def check_str_param(self, param): if len(param) > Const.FILE_NAME_LENGTH: self.log.error('The parameter length exceeds limit') diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py index 5e37b58d0..2a274f3ca 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py @@ -21,6 +21,7 @@ from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.utils import save_npy_to_txt class Visualization: @@ -43,18 +44,18 @@ class Visualization: summary = ['[yellow]%s[/yellow]' % self.util.gen_npy_info_txt(np_data), 'Path: %s' % target_file, "TextFile: %s.txt" % target_file] self.util.print_panel(self.util.create_columns([table, "\n".join(summary)]), target_file) - self.util.save_npy_to_txt(np_data, target_file + ".txt") + save_npy_to_txt(np_data, target_file + ".txt") def print_npy_data(self, file_name): file_name = self.util.path_strip(file_name) self.util.check_path_valid(file_name) - self.util.check_path_format(file_name, Const.NPY_SUFFIX) + self.util.check_file_path_format(file_name, Const.NPY_SUFFIX) return self.print_npy_summary(file_name) def parse_pkl(self, path, api_name): path = self.util.path_strip(path) self.util.check_path_valid(path) - self.util.check_path_format(path, Const.PKL_SUFFIX) + self.util.check_file_path_format(path, Const.PKL_SUFFIX) self.util.check_str_param(api_name) with FileOpen(path, "r") as pkl_handle: title_printed = False -- Gitee From ac0de8ae07e026946cc434cc3078e8dfb72d16f1 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 20 Aug 2024 17:48:09 +0800 Subject: [PATCH 394/791] cleancode --- .../api_accuracy_checker/api_info.py | 25 +-- .../api_accuracy_checker/api_runner.py | 40 ++--- .../base_compare_algorithm.py | 152 +++++++----------- .../api_accuracy_checker/compute_element.py | 8 +- .../mindspore/api_accuracy_checker/const.py | 32 ++-- .../api_accuracy_checker/type_mapping.py | 12 +- .../api_accuracy_checker/test_api_info.py | 2 +- .../api_accuracy_checker/test_api_runner.py | 30 ++-- .../test_compare_algorithm.py | 14 +- .../test_compute_element.py | 6 +- 10 files changed, 145 insertions(+), 176 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py index ea5f9e1c7..901e5162a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py @@ -1,5 +1,6 @@ from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement -from msprobe.mindspore.api_accuracy_checker.const import FORWARD_API, BACKWARD_API, INPUT, OUTPUT +from msprobe.core.common.const import Const +from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger @@ -29,23 +30,23 @@ class ApiInfo: def get_compute_element_list(self, forward_or_backward, input_or_output): ''' Args: - forward_or_backward: str, Union["forward_api", "backward_api"] + forward_or_backward: str, Union["forward", "backward"] input_or_output: str, Union["input", "output"] Return: compute_element_list: List[ComputeElement] ''' mapping = { - (FORWARD_API, INPUT): [self.forward_info, "input_args", - f"input_args field of {self.api_name} forward api in api_info.json"], - (FORWARD_API, OUTPUT): [self.forward_info, "output", - f"output field of {self.api_name} forward api in api_info.json"], - (BACKWARD_API, INPUT): [self.backward_info, "input", - f"input field of {self.api_name} backward api in api_info.json"], - (BACKWARD_API, OUTPUT): [self.backward_info, "output", - f"output field of {self.api_name} backward api in api_info.json"] + (Const.FORWARD, Const.INPUT): [self.forward_info, MsApiAccuracyCheckerConst.API_INFO_FORWARD_INPUT, + f"input_args field of {self.api_name} forward api in api_info.json"], + (Const.FORWARD, Const.OUTPUT): [self.forward_info, MsApiAccuracyCheckerConst.API_INFO_FORWARD_OUTPUT, + f"output field of {self.api_name} forward api in api_info.json"], + (Const.BACKWARD, Const.INPUT): [self.backward_info, MsApiAccuracyCheckerConst.API_INFO_BACKWARD_INPUT, + f"input field of {self.api_name} backward api in api_info.json"], + (Const.BACKWARD, Const.OUTPUT): [self.backward_info, MsApiAccuracyCheckerConst.API_INFO_BACKWARD_OUTPUT, + f"output field of {self.api_name} backward api in api_info.json"] } - dict_instance, key, key_desc = mapping[(forward_or_backward, input_or_output)] + dict_instance, key, key_desc = mapping.get((forward_or_backward, input_or_output)) compute_element_info_list = check_and_get_from_json_dict(dict_instance, key, key_desc, accepted_type=list) compute_element_list = [ComputeElement(compute_element_info=compute_element_info) for compute_element_info in compute_element_info_list] @@ -56,7 +57,7 @@ class ApiInfo: Return: kwargs_compute_element_dict: dict{str: ComputeElement} ''' - kwargs_dict = check_and_get_from_json_dict(self.forward_info, "input_kwargs", + kwargs_dict = check_and_get_from_json_dict(self.forward_info, MsApiAccuracyCheckerConst.API_INFO_FORWARD_KWARGS, "input_kwargs in api_info.json", accepted_type=dict) for key_str, compute_element_info in kwargs_dict.items(): if not isinstance(key_str, str): diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index 140413eb2..8147440d7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -5,11 +5,12 @@ import torch from mindspore import ops from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement -from msprobe.mindspore.api_accuracy_checker.const import (MINDSPORE_PLATFORM, TORCH_PLATFORM, MINT, MINT_FUNCTIONAL, - FORWARD_API) +from msprobe.core.common.const import Const +from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple +from msprobe.mindspore.api_accuracy_checker.type_mapping import api_parent_module_mapping class ApiInputAggregation: @@ -28,20 +29,20 @@ class ApiInputAggregation: class ApiRunner: def __init__(self) -> None: self.api_parent_module_mapping = { - (MINT, MINDSPORE_PLATFORM): mindspore.mint, - (MINT, TORCH_PLATFORM): torch, - (MINT_FUNCTIONAL, MINDSPORE_PLATFORM): mindspore.mint.nn.functional, - (MINT_FUNCTIONAL, TORCH_PLATFORM): torch.nn.functional + (MsApiAccuracyCheckerConst.MINT, Const.MS_FRAMEWORK): mindspore.mint, + (MsApiAccuracyCheckerConst.MINT, Const.PT_FRAMEWORK): torch, + (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional, + (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional } - def __call__(self, api_input_aggregation, api_name_str, forward_or_backward=FORWARD_API, - api_platform=MINDSPORE_PLATFORM): + def __call__(self, api_input_aggregation, api_name_str, forward_or_backward=Const.FORWARD, + api_platform=Const.MS_FRAMEWORK): ''' Args: api_input_aggregation: ApiInputAggregation api_name_str: str, e.g. "MintFunctional.relu.0" - forward_or_backward: str, Union["forward_api", "backward_api"] - api_platform: str, Union["mindspore_api", "torch_api"] + forward_or_backward: str, Union["forward", "backward"] + api_platform: str, Union["mindspore", "torch"] Return: outputs: list[ComputeElement] @@ -50,7 +51,7 @@ class ApiRunner: run mindspore.mint/torch api ''' api_type_str, api_sub_name = self.get_info_from_name(api_name_str) - api_instance = self._get_api_instance(api_type_str, api_sub_name, api_platform) + api_instance = self.get_api_instance(api_type_str, api_sub_name, api_platform) self.run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform) @@ -69,18 +70,19 @@ class ApiRunner: err_msg = f"ApiRunner.get_info_from_name failed: api_name_str: {api_name_str} is not in defined format" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) api_type_str, api_sub_name = api_name_list[0], api_name_list[1] - if api_type_str not in [MINT, MINT_FUNCTIONAL]: + if api_type_str not in [MsApiAccuracyCheckerConst.MINT, MsApiAccuracyCheckerConst.MINT_FUNCTIONAL]: err_msg = f"ApiRunner.get_info_from_name failed: not mint or mint.nn.functional api" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) return api_type_str, api_sub_name - def _get_api_instance(self, api_type_str, api_sub_name, api_platform): + @staticmethod + def get_api_instance(api_type_str, api_sub_name, api_platform): ''' Args: api_type_str: str, Union["MintFunctional", "Mint"] api_sub_name: str, e.g. "relu" - api_platform: str: Union["mindpore_api", "torch_api"] + api_platform: str: Union["mindpore", "torch"] Return: api_instance: function object @@ -91,9 +93,9 @@ class ApiRunner: mindspore.mint.nn.functional.{api_sub_name} <--> torch.nn.functional.{api_sub_name} ''' - api_parent_module = self.api_parent_module_mapping.get((api_type_str, api_platform)) - module_str = "mindspore.mint." if api_platform == MINDSPORE_PLATFORM else "torch." - submodule_str = "nn.functional." if api_type_str == MINT_FUNCTIONAL else "" + api_parent_module = api_parent_module_mapping.get((api_type_str, api_platform)) + module_str = "mindspore.mint." if api_platform == Const.MS_FRAMEWORK else "torch." + submodule_str = "nn.functional." if api_type_str == MsApiAccuracyCheckerConst.MINT_FUNCTIONAL else "" full_api_name = module_str + submodule_str + api_sub_name if not hasattr(api_parent_module, api_sub_name): err_msg = f"ApiRunner.get_api_instance failed: {full_api_name} is not found" @@ -114,7 +116,7 @@ class ApiRunner: for key, value in api_input_aggregation.kwargs.items()} gradient_inputs = api_input_aggregation.gradient_inputs - if forward_or_backward == FORWARD_API: + if forward_or_backward == Const.FORWARD: forward_result = api_instance(*inputs, **kwargs) # can be single tensor or tuple forward_result_tuple = convert_to_tuple(forward_result) res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in forward_result_tuple] @@ -124,7 +126,7 @@ class ApiRunner: logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) gradient_inputs = tuple(compute_element.get_parameter(get_origin=False, tensor_platform=api_platform) for compute_element in gradient_inputs) - if api_platform == MINDSPORE_PLATFORM: + if api_platform == Const.MS_FRAMEWORK: if len(gradient_inputs) == 1: gradient_inputs = gradient_inputs[0] def api_with_kwargs(forward_inputs): diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index d4fc578cd..8f7f3d2f4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -6,9 +6,7 @@ import numpy as np from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger -from msprobe.mindspore.api_accuracy_checker.const import (COSINE_SIMILARITY, MAX_ABSOLUTE_DIFF, MAX_RELATIVE_DIFF, - PASS, ERROR, SKIP) - +from msprobe.core.common.const import CompareConst class CompareResult: def __init__(self, compare_value, pass_status, err_msg): @@ -21,6 +19,25 @@ class BaseCompareAlgorithm(ABC): def __init__(self) -> None: super().__init__() self.compare_algorithm_name = None + self.err_msg_mapping = { + CompareConst.COSINE: { + CompareConst.PASS: "", + CompareConst.ERROR: f"cosine similarity is less than threshold: {CompareConst.COSINE_THRESHOLD}", + CompareConst.SKIP: "two inputs are not valid for computing cosine similarity, skip comparing", + }, + CompareConst.MAX_ABS_ERR: { + CompareConst.PASS: "", + CompareConst.ERROR: "max absolute difference is greater than " \ + f"threshold: {CompareConst.THOUSAND_RATIO_THRESHOLD}", + CompareConst.SKIP: "two inputs are not valid for computing max absolute difference, skip comparing", + }, + CompareConst.MAX_RELATIVE_ERR: { + CompareConst.PASS: "", + CompareConst.ERROR: "max relative difference is greater than " \ + f"threshold: {CompareConst.THOUSAND_RATIO_THRESHOLD}", + CompareConst.SKIP: "two inputs are not valid for computing max relative difference, skip comparing", + }, + } def __call__(self, bench_compute_element, tested_compute_element): ''' @@ -37,13 +54,35 @@ class BaseCompareAlgorithm(ABC): else: logger.warning(f"not suitable for computing {self.compare_algorithm_name}, skip this.") compare_value = None - pass_status = SKIP + pass_status = CompareConst.SKIP - err_msg = self.generate_err_msg(pass_status) + err_msg = self.err_msg_mapping.get(self.compare_algorithm_name).get(pass_status) compare_result = CompareResult(compare_value, pass_status, err_msg) return compare_result + @staticmethod + def convert_to_np_float64_ndarray(tensor): + if isinstance(tensor, mindspore.Tensor): + ndarray = tensor.astype(mindspore.float64).numpy() + elif isinstance(tensor, torch.Tensor): + ndarray = tensor.to(torch.float64, copy=True).numpy() + else: + err_msg = "BaseCompareAlgorithm.convert_to_np_float64_ndarray failed: " \ + "input is not mindspore.Tensor or torch.Tensor" + logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) + return ndarray + + @staticmethod + def check_two_tensor(bench_compute_element, tested_compute_element): + bench_parameter = bench_compute_element.get_parameter() + tested_parameter = tested_compute_element.get_parameter() + + bench_is_tensor = isinstance(bench_parameter, (mindspore.Tensor, torch.Tensor)) + tested_is_tensosr = isinstance(tested_parameter, (mindspore.Tensor, torch.Tensor)) + shape_same = bench_compute_element.get_shape() == tested_compute_element.get_shape() + return bench_is_tensor and tested_is_tensosr and shape_same + @abstractmethod def check_validity(self, bench_compute_element, tested_compute_element): ''' @@ -79,45 +118,11 @@ class BaseCompareAlgorithm(ABC): ''' raise NotImplementedError - @abstractmethod - def generate_err_msg(self, pass_status): - ''' - Args: - pass_status: str - - Return: - err_msg: str - ''' - raise NotImplementedError - - @classmethod - def convert_to_np_float64_ndarray(cls, tensor): - if isinstance(tensor, mindspore.Tensor): - ndarray = tensor.astype(mindspore.float64).numpy() - elif isinstance(tensor, torch.Tensor): - ndarray = tensor.to(torch.float64, copy=True).numpy() - else: - err_msg = "BaseCompareAlgorithm.convert_to_np_float64_ndarray failed: " \ - "input is not mindspore.Tensor or torch.Tensor" - logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) - return ndarray - - @classmethod - def check_two_tensor(cls, bench_compute_element, tested_compute_element): - bench_parameter = bench_compute_element.get_parameter() - tested_parameter = tested_compute_element.get_parameter() - - bench_is_tensor = isinstance(bench_parameter, (mindspore.Tensor, torch.Tensor)) - tested_is_tensosr = isinstance(tested_parameter, (mindspore.Tensor, torch.Tensor)) - shape_same = bench_compute_element.get_shape() == tested_compute_element.get_shape() - return bench_is_tensor and tested_is_tensosr and shape_same - class CosineSimilarityCompareAlgorithm(BaseCompareAlgorithm): def __init__(self) -> None: super().__init__() - self.compare_algorithm_name = COSINE_SIMILARITY - self.pass_threshold = 0.9999 + self.compare_algorithm_name = CompareConst.COSINE def check_validity(self, bench_compute_element, tested_compute_element): return self.check_two_tensor(bench_compute_element, tested_compute_element) @@ -133,29 +138,16 @@ class CosineSimilarityCompareAlgorithm(BaseCompareAlgorithm): return cosine_similarity def check_pass(self, compare_value): - if compare_value > self.pass_threshold: - return PASS - else: - return ERROR - - def generate_err_msg(self, pass_status): - if pass_status == PASS: - err_msg = "" - elif pass_status == SKIP: - err_msg = "two inputs are not valid for computing cosine similarity, skip comparing" - elif pass_status == ERROR: - err_msg = f"cosine similarity is less than threshold: {self.pass_threshold}" + if compare_value > CompareConst.COSINE_THRESHOLD: + return CompareConst.PASS else: - logger.warning(f"unseen pass_status: {pass_status}") - err_msg = "" - return err_msg + return CompareConst.ERROR class MaxAbsoluteDiffCompareAlgorithm(BaseCompareAlgorithm): def __init__(self) -> None: super().__init__() - self.compare_algorithm_name = MAX_ABSOLUTE_DIFF - self.pass_threshold = 0.001 + self.compare_algorithm_name = CompareConst.MAX_ABS_ERR def check_validity(self, bench_compute_element, tested_compute_element): return self.check_two_tensor(bench_compute_element, tested_compute_element) @@ -168,29 +160,16 @@ class MaxAbsoluteDiffCompareAlgorithm(BaseCompareAlgorithm): return max_absolute_diff def check_pass(self, compare_value): - if compare_value < self.pass_threshold: - return PASS + if compare_value < CompareConst.THOUSAND_RATIO_THRESHOLD: + return CompareConst.PASS else: - return ERROR - - def generate_err_msg(self, pass_status): - if pass_status == PASS: - err_msg = "" - elif pass_status == SKIP: - err_msg = "two inputs are not valid for computing max absolute difference, skip comparing" - elif pass_status == ERROR: - err_msg = f"max absolute difference is greater than threshold: {self.pass_threshold}" - else: - logger.warning(f"unseen pass_status: {pass_status}") - err_msg = "" - return err_msg + return CompareConst.ERROR class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): def __init__(self) -> None: super().__init__() - self.compare_algorithm_name = MAX_RELATIVE_DIFF - self.pass_threshold = 0.01 + self.compare_algorithm_name = CompareConst.MAX_RELATIVE_ERR self.epsilon = 1e-8 def check_validity(self, bench_compute_element, tested_compute_element): @@ -206,26 +185,15 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): return max_relative_diff def check_pass(self, compare_value): - if compare_value < self.pass_threshold: - return PASS - else: - return ERROR - - def generate_err_msg(self, pass_status): - if pass_status == PASS: - err_msg = "" - elif pass_status == SKIP: - err_msg = "two inputs are not valid for computing max relative difference, skip comparing" - elif pass_status == ERROR: - err_msg = f"max relative difference is greater than threshold: {self.pass_threshold}" + if compare_value < CompareConst.THOUSAND_RATIO_THRESHOLD: + return CompareConst.PASS else: - logger.warning(f"unseen pass_status: {pass_status}") - err_msg = "" - return err_msg + return CompareConst.ERROR + compare_algorithms = { - COSINE_SIMILARITY: CosineSimilarityCompareAlgorithm(), - MAX_ABSOLUTE_DIFF: MaxAbsoluteDiffCompareAlgorithm(), - MAX_RELATIVE_DIFF: MaxRelativeDiffCompareAlgorithm(), + CompareConst.COSINE: CosineSimilarityCompareAlgorithm(), + CompareConst.MAX_ABS_ERR: MaxAbsoluteDiffCompareAlgorithm(), + CompareConst.MAX_RELATIVE_ERR: MaxRelativeDiffCompareAlgorithm(), } \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 6d171ae43..534b1ae51 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -14,7 +14,7 @@ from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np DEFAULT_CONSTRUCT_NP_FLOAT_DTYPE, TUPLE_TYPE_STR, MINDSPORE_TENSOR_TYPE_STR, float_dtype_str_list, int_dtype_str_list) -from msprobe.mindspore.api_accuracy_checker.const import MINDSPORE_PLATFORM, TORCH_PLATFORM +from msprobe.core.common.const import Const from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context @@ -100,7 +100,7 @@ class ComputeElement: value = np.finfo(np_dtype).min return value - def get_parameter(self, get_origin=True, tensor_platform=MINDSPORE_PLATFORM): + def get_parameter(self, get_origin=True, tensor_platform=Const.MS_FRAMEWORK): ''' Args: get_origin: boolean @@ -127,9 +127,9 @@ class ComputeElement: logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.UnsupportType)) # if necessary, do transfer - if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and tensor_platform == TORCH_PLATFORM: + if not get_origin and isinstance(parameter_tmp, mindspore.Tensor) and tensor_platform == Const.PT_FRAMEWORK: parameter = self.transfer_to_torch_tensor(parameter_tmp) - elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and tensor_platform ==MINDSPORE_PLATFORM: + elif not get_origin and isinstance(parameter_tmp, torch.Tensor) and tensor_platform ==Const.MS_FRAMEWORK: parameter = self.transfer_to_mindspore_tensor(parameter_tmp) else: parameter = parameter_tmp diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py index 96ebc073e..0d7e67fcf 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -1,25 +1,13 @@ -# base_compare_algorithm -COSINE_SIMILARITY = "cosine similarity" -MAX_ABSOLUTE_DIFF = "max absolute difference" -MAX_RELATIVE_DIFF = "max relative difference" +class MsApiAccuracyCheckerConst: + #api_info + API_INFO_FORWARD_INPUT = "input_args" + API_INFO_FORWARD_OUTPUT = "output" + API_INFO_FORWARD_KWARGS = "input_kwargs" + API_INFO_BACKWARD_INPUT = "input" + API_INFO_BACKWARD_OUTPUT = "output" -PASS = "pass" -ERROR = "error" -SKIP = "skip" + #api_runner + MINT = "Mint" + MINT_FUNCTIONAL = "MintFunctional" -# compute_element -MINDSPORE_PLATFORM = "mindspore_platform" -TORCH_PLATFORM = "torch_platform" - - -#api_runner -MINT = "Mint" -MINT_FUNCTIONAL = "MintFunctional" - -FORWARD_API = "forward_api" -BACKWARD_API = "backward_api" - -#api_info -INPUT = "input" -OUTPUT = "output" \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py index e622d06b4..44989f0b6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py @@ -3,6 +3,9 @@ import numpy as np import mindspore import torch +from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst +from msprobe.core.common.const import Const + INT8 = "Int8" UINT8 = "UInt8" INT16 = "Int16" @@ -111,4 +114,11 @@ uint_dtype_str_list = [ UINT16, UINT32, UINT64, -] \ No newline at end of file +] + +api_parent_module_mapping = { + (MsApiAccuracyCheckerConst.MINT, Const.MS_FRAMEWORK): mindspore.mint, + (MsApiAccuracyCheckerConst.MINT, Const.PT_FRAMEWORK): torch, + (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional, + (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py index 311819796..d2c62bc4f 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py @@ -64,7 +64,7 @@ class TestApiInfo(unittest.TestCase): self.assertTrue(api_info.check_forward_info()) self.assertFalse(api_info.check_backward_info()) - input_compute_element_list = api_info.get_compute_element_list("forward_api", "input") + input_compute_element_list = api_info.get_compute_element_list("forward", "input") parameter_real = input_compute_element_list[0].get_parameter() parameter_target = mindspore.Tensor([1., 2., 3.]) self.assertTrue((parameter_real == parameter_target).all()) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py index dad6d6047..dac2b9b36 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_runner.py @@ -7,7 +7,7 @@ import torch from unittest.mock import MagicMock from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation -from msprobe.mindspore.api_accuracy_checker.const import MINDSPORE_PLATFORM, TORCH_PLATFORM, FORWARD_API, BACKWARD_API +from msprobe.core.common.const import Const logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') logger = logging.getLogger(__name__) @@ -24,49 +24,49 @@ def func(x_1, x_2, opt="opt1"): return y_1, y_2 def side_effect_forward_input_1(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + if kwargs.get("tensor_platform") == Const.MS_FRAMEWORK: return mindspore.Tensor([1., 2., 3.]) else: return torch.Tensor([1., 2., 3.]) def side_effect_forward_input_2(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + if kwargs.get("tensor_platform") == Const.MS_FRAMEWORK: return mindspore.Tensor([1.1, 2., 3.]) else: return torch.Tensor([1.1, 2., 3.]) def side_effect_forward_output_1(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + if kwargs.get("tensor_platform") == Const.MS_FRAMEWORK: return mindspore.Tensor([3., 5., 7.]) else: return torch.Tensor([3., 5., 7.]) def side_effect_forward_output_2(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + if kwargs.get("tensor_platform") == Const.MS_FRAMEWORK: return mindspore.Tensor([2.1, 4., 6.]) else: return torch.Tensor([2.1, 4., 6.]) def side_effect_backward_input_1(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + if kwargs.get("tensor_platform") == Const.MS_FRAMEWORK: return mindspore.Tensor([1., 2., 3.]) else: return torch.Tensor([1., 2., 3.]) def side_effect_backward_input_2(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + if kwargs.get("tensor_platform") == Const.MS_FRAMEWORK: return mindspore.Tensor([1.11, 2., 3.]) else: return torch.Tensor([1.11, 2., 3.]) def side_effect_backward_output_1(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + if kwargs.get("tensor_platform") == Const.MS_FRAMEWORK: return mindspore.Tensor([3.11, 6., 9.]) else: return torch.Tensor([3.11, 6., 9.]) def side_effect_backward_output_2(**kwargs): - if kwargs.get("tensor_platform") == MINDSPORE_PLATFORM: + if kwargs.get("tensor_platform") == Const.MS_FRAMEWORK: return mindspore.Tensor([1.11, 2., 3.]) else: return torch.Tensor([1.11, 2., 3.]) @@ -114,10 +114,10 @@ class TestApiRunner(unittest.TestCase): # api_instance, api_input_aggregation, forward_or_backward, api_platform, result test_cases = [ - [func, forward_api_input_aggregation, FORWARD_API, MINDSPORE_PLATFORM, forward_result], - [func, backward_api_input_aggregation, BACKWARD_API, MINDSPORE_PLATFORM, backward_result], - [func, forward_api_input_aggregation, FORWARD_API, TORCH_PLATFORM, forward_result], - [func, backward_api_input_aggregation, BACKWARD_API, TORCH_PLATFORM, backward_result], + [func, forward_api_input_aggregation, Const.FORWARD, Const.MS_FRAMEWORK, forward_result], + [func, backward_api_input_aggregation, Const.BACKWARD, Const.MS_FRAMEWORK, backward_result], + [func, forward_api_input_aggregation, Const.FORWARD, Const.PT_FRAMEWORK, forward_result], + [func, backward_api_input_aggregation, Const.BACKWARD, Const.PT_FRAMEWORK, backward_result], ] for test_case in test_cases: api_instance, api_input_aggregation, forward_or_backward, api_platform, results_target = test_case @@ -129,8 +129,8 @@ class TestApiRunner(unittest.TestCase): def test_get_api_instance(self): #api_type_str, api_sub_name, api_platform, result_api test_cases = [ - ["MintFunctional", "relu", MINDSPORE_PLATFORM, mindspore.mint.nn.functional.relu], - ["MintFunctional", "relu", TORCH_PLATFORM, torch.nn.functional.relu] + ["MintFunctional", "relu", Const.MS_FRAMEWORK, mindspore.mint.nn.functional.relu], + ["MintFunctional", "relu", Const.PT_FRAMEWORK, torch.nn.functional.relu] ] for test_case in test_cases: api_type_str, api_sub_name, api_platform, result_api = test_case diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py index 35b00bc40..70f9401eb 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py @@ -7,7 +7,7 @@ import torch from unittest.mock import MagicMock from msprobe.mindspore.api_accuracy_checker.base_compare_algorithm import compare_algorithms -from msprobe.mindspore.api_accuracy_checker.const import COSINE_SIMILARITY, MAX_ABSOLUTE_DIFF, MAX_RELATIVE_DIFF, ERROR +from msprobe.core.common.const import CompareConst logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') logger = logging.getLogger(__name__) @@ -27,19 +27,19 @@ class TestCompareAlgorithms(unittest.TestCase): self.mock_mstensor_compute_element.get_shape.return_value = (3,) def test_cosine_similarity(self): - compare_result = compare_algorithms[COSINE_SIMILARITY](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) + compare_result = compare_algorithms[CompareConst.COSINE](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) self.assertAlmostEqual(compare_result.compare_value, 0.9997375534689601, places=5) - self.assertEqual(compare_result.pass_status, ERROR) + self.assertEqual(compare_result.pass_status, CompareConst.ERROR) def test_max_absolute_difference(self): - compare_result = compare_algorithms[MAX_ABSOLUTE_DIFF](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) + compare_result = compare_algorithms[CompareConst.MAX_ABS_ERR](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) self.assertAlmostEqual(compare_result.compare_value, 0.1, places=5) - self.assertEqual(compare_result.pass_status, ERROR) + self.assertEqual(compare_result.pass_status, CompareConst.ERROR) def test_max_relative_difference(self): - compare_result = compare_algorithms[MAX_RELATIVE_DIFF](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) + compare_result = compare_algorithms[CompareConst.MAX_RELATIVE_ERR](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) self.assertAlmostEqual(compare_result.compare_value, 0.05, places=5) - self.assertEqual(compare_result.pass_status, ERROR) + self.assertEqual(compare_result.pass_status, CompareConst.ERROR) if __name__ == '__main__': unittest.main() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py index 12714ac34..c64bf9952 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -10,7 +10,7 @@ from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElemen from msprobe.mindspore.api_accuracy_checker.type_mapping import (FLOAT32, FLOAT_TYPE_STR, INT_TYPE_STR, TUPLE_TYPE_STR, STR_TYPE_STR, SLICE_TYPE_STR) from msprobe.mindspore.api_accuracy_checker.utils import global_context -from msprobe.mindspore.api_accuracy_checker.const import MINDSPORE_PLATFORM, TORCH_PLATFORM +from msprobe.core.common.const import Const logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') logger = logging.getLogger(__name__) @@ -47,8 +47,8 @@ class TestComputeElement(unittest.TestCase): compute_element = ComputeElement(parameter=input_parameter) self.assertTrue((compute_element.get_parameter(get_origin=True) == origin_parameter).all()) - self.assertTrue((compute_element.get_parameter(get_origin=False, tensor_platform=MINDSPORE_PLATFORM) == mstensor_parameter).all()) - self.assertTrue((compute_element.get_parameter(get_origin=False, tensor_platform=TORCH_PLATFORM) == torchtensor_parameter).all()) + self.assertTrue((compute_element.get_parameter(get_origin=False, tensor_platform=Const.MS_FRAMEWORK) == mstensor_parameter).all()) + self.assertTrue((compute_element.get_parameter(get_origin=False, tensor_platform=Const.PT_FRAMEWORK) == torchtensor_parameter).all()) self.assertEqual(compute_element.get_shape(), shape) self.assertEqual(compute_element.get_dtype(), dtype_str) -- Gitee From bd5501d61a5acd92b4b92b96719abaabc8fc4058 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Tue, 20 Aug 2024 19:36:28 +0800 Subject: [PATCH 395/791] add explain --- debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index bbd32a9d6..6bdd4f672 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -31,6 +31,8 @@ class MSComparator (Comparator): a_op_name = [op_name.replace("Cell", "Module", 1) for op_name in a_op_name] if self.cell_mapping_dict: for index, op_name in enumerate(a_op_name): + # get cell name & class name from op_name + # Cell.fc1.Dense.forward.0.input.0 cell_name = op_name.split(Const.SEP, 1)[-1].rsplit(Const.SEP, 4)[0] if cell_name in self.cell_mapping_dict: a_op_name[index] = op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) -- Gitee From e194e02a2e7f1f53ab4bb1c2a0698a8255627983 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Tue, 20 Aug 2024 20:06:20 +0800 Subject: [PATCH 396/791] cosine near 1 processing improve --- debug/accuracy_tools/msprobe/core/compare/npy_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py index 279f950f9..1b0208346 100644 --- a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py @@ -90,7 +90,7 @@ class GetCosineSimilarity(TensorComparisonBasic): if result == CompareConst.NAN: return result if float(result) > CompareConst.COSINE_THRESHOLD: - return 1.0 + return round(float(result), 6) return result def apply(self, n_value, b_value, error_flag, relative_err=None): -- Gitee From 6fe2b1061a5172cf9a64e79a0a5305eb01bb0d97 Mon Sep 17 00:00:00 2001 From: wuyulong11 <2284273586@qq.com> Date: Tue, 20 Aug 2024 14:36:56 +0800 Subject: [PATCH 397/791] =?UTF-8?q?=E3=80=90=E4=BF=AE=E6=94=B9=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E3=80=91=E3=80=90tbplugin=E3=80=91=E5=B0=86=E5=85=AC?= =?UTF-8?q?=E7=BD=91URL=E5=86=99=E5=85=A5=E9=85=8D=E7=BD=AE=E6=96=87?= =?UTF-8?q?=E4=BB=B6=20=E3=80=90=E4=BF=AE=E6=94=B9=E4=BA=BA=E3=80=91=20wuy?= =?UTF-8?q?ulong?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tb_plugin/config/config.ini | 11 ++++++++++ ...7\275\221URL\350\257\264\346\230\216.xlsx" | Bin 17397 -> 17513 bytes .../tensorboard-plugins/tb_plugin/setup.py | 12 +++++++---- .../tb_plugin/torch_tb_profiler/io/file.py | 7 ------ .../torch_tb_profiler/profiler/data.py | 20 ++++++++++-------- .../torch_tb_profiler/profiler/tensor_core.py | 2 -- .../tb_plugin/torch_tb_profiler/utils.py | 1 - 7 files changed, 30 insertions(+), 23 deletions(-) create mode 100644 plugins/tensorboard-plugins/tb_plugin/config/config.ini diff --git a/plugins/tensorboard-plugins/tb_plugin/config/config.ini b/plugins/tensorboard-plugins/tb_plugin/config/config.ini new file mode 100644 index 000000000..500d472d2 --- /dev/null +++ b/plugins/tensorboard-plugins/tb_plugin/config/config.ini @@ -0,0 +1,11 @@ +[URL] +pytorch_data_loading_url = https://pytorch.org/docs/stable/data.html#single-and-multi-process-data-loading +pytorch_amp_url = https://pytorch.org/docs/stable/amp.html +pytorch_ckp_url = https://pytorch.org/docs/stable/checkpoint.html +cuda_nn_ddp_instead_url = https://pytorch.org/docs/stable/notes/cuda.html#cuda-nn-ddp-instead +compress_url = https://pytorch.org/docs/stable/ddp_comm_hooks.html +grad_acc_url = https://towardsdatascience.com/what-is-gradient-accumulation-in-deep-learning-ec034122cfa +lamb_url = https://nvidia.github.io/apex/optimizers.html#apex.optimizers.FusedLAMB +repository_url = https://gitee.com/ascend/att/tree/master/plugins/tensorboard-plugins/tb_plugin +[EMAIL] +author_email = pmail_mindstudio@huawei.com \ No newline at end of file diff --git "a/plugins/tensorboard-plugins/tb_plugin/docs/\345\205\254\347\275\221URL\350\257\264\346\230\216.xlsx" "b/plugins/tensorboard-plugins/tb_plugin/docs/\345\205\254\347\275\221URL\350\257\264\346\230\216.xlsx" index fbe5a354ffba8619d9e93012d6fa3715e1f50e19..de0bb25fe155aa188e5670a377311e96168586e8 100644 GIT binary patch delta 10411 zcmZvCbx@s6^Cj*Y{NnEJ&c)p&$i>|qf?XtN@JoWbCb+u=LU4E2;O@S8zxS)(?$++q zRCm?s?lV1qJXQ0|oZCp~!zk#Q6li$f=cIRTNIEnVIHO@sP;wOrdr0kna(0U&n?ir5KmiRGR*)9n(#R@wT91z^3d~6SsQ4 zsBc3VJG-J;`T>bZ+juA{O27w&0d6zq2J~Jo$&SU)KVoqhIKNc(c`z>z0p_i~`{;>T z(?nBP;FjGG2rSl0bd+onF3xmNi#L=!Fb^J`&NsQ3=%L=lbtPChTqr0gB&bN$Z3)5_QVTg)D5zryG%N{t#ra5zr zwO1Fp-RdHQCE?@r-X{7cfaX#ki?$jxo^8mQ?ywu43THg}VS0}#*<<4z{Px|#hJ{ui zJE=oMiRM%43a*Ub$aO2Yn{sx<05hy0B7~|@>$@%%jFO3X)2Vq)$@ol;s~UEd@j6Bc<5OJ6y0T3 z3fN$tBEuN_w_e`q`;jOI8nVjVw@@ft6Wh0BjM!AZ54FJ#r-5!@;?l^D!835*2G72R z3PC`tuvn`SDskjstZe+Kx%+g6lE6ss+(rIrpmZ(M$gxw->;Pg zt#{Sk5&`!+FlX`gl8iHyS`nG<%*RU_sg*pCyUVPDz>0aV5nN;pQi(rE<+{!+fiWH-CO*V(quFX`2 zRvbz<^c>n?(-(u-h4NP!DAN~1vLh!y$vU*~W{{Ard98nN(HgiTsv)& zY;So7ql0{IuMinfB-0NqQdsfv4=ril)#gf%4ZM?z1U78JEMFq3D5Q9FdS{j4j|sQi zxZ%nS2+P>9K);)zrQ zAA1ySgvO*C!98ye-UO(LTmi3Z3o4v`nmXC3E17Nrg5(Lae9Wnj-d@~qk8h_-d%oKI zhu-P4baG(>zwVVCV6tIP9%r3#mU>H)bEtxpfYy>c(9vtJOR6l*IW;NX3V zPA)D%F7u}R&F`RyJ{6qG`OHsdC1jNCP8x44a@2q5Kp!NSZA4d35WcM%)kPN0MsIdg zH`#zwP@O$_$6IbmkAaSCU9HBuSNA32vwqgP22i#>E6=6w9IzNxk78p4Qe|X zvzJ6=d;LMqZA2w2ZHtj4DrY$?LO2 z`9>JqTUuJ^O=rQgB!V;zrS{MLza4Y)#;)ub_W7cXR(Bx6pFaBqZd){|O zC8!B74w3=O7U;#SpW&Rosm-Ur6Wl*Y6ZfJ1QY|#zcu2GIRkdhx@pjqV&My`Gy!dX} z59Pnch2YfPAjxd&9m#?FIj1Ik*25%1X3R;koju_Vu?mZHu6N5o<~7CgnRMcz6d<}^vM_yduu4EG zjNkrX24g|MZVN!3QUxsBTUI{@Nwy+XI~>eoEciy^S6u=R0Xzm((R)3-z96ZYzdXc^ zu5p;MePE#(?DU-?$5wnpC94p@K!v~I^S|Nef5M6jc~h|lEV>K}n_lRwt6KnJ#z#`cD}#LPKIfX{9k_h}vDo4C%(TV&UQnuYf%pCl>k?f;Iu3I{ z7c*08Ha=rb8SU11VYOYVpC9T!`)h0u!{#mDV)+N293e&CJ2h*2h--IuTrliAfF?>=~i925K0QH!X+5QEH^M*f_z0 zh{pp*2I?6hKuw>}dF!;_O5z@;vE*e%w(RGx*!dkhcQUYmSVZv-BR!Cp`h7z|Mp* zsf2W5$AUY(*GL_i{8I#~y**<2V^Vbw**U(eE$*p5$-4dK#O}Qm%Tje6KFe746p&!Q z!96m(amOOZy_yKzPIJskc77Bn{=Sr*?Cf8nl;PF;z$6yL!iM zKS|3~5u^pzB4^3K@Q;H7+D~AY$P%5BMcRj_4q>J*-1xOTl@ipT;UgH@s&h)g~7$6~1~_p{Y!4Pk{T6mtaeB zcA&^7GxVw5?TM}+$maI;gqXjLsllaxxW)n@|5Db>r5@``MFWF@Fp` znHL8W@kpT-&xul1=A08B%&)50YIhdkyQTWp{s5o3B zyO}~cq!Sl_{urAVXxL5SixK%(XYE%DzlV2iJH2bsapPZ_F0f-wO z9{BHWJZh{q)*tO$vS+R40-UF18}F0Owjbv3a+GXtc4KNkwSxY%9P!3BFGR$S1ZMx0{^(t@bP#m$6R@a?~<8^friVSNBXkIJWTjvTG>6sw>FU zyD3s;+vHm5l6QH3LcLjiIlT{Ddf9O@0M8zICFy3~?H_Ln#GUopqL#91Xd@^kO-u*s zoROpK@k=fElI$L45P%LJX<6N$6km=sbkCX}YQoXdr9dn(9nJ%V4GhD5pd{yR)1xxu z)JUI|wXRQUW{P^abr=C!I#yP}r`c@Wiy-x%hQjt>pBa=xDY6+s=*h||-n}X&@YN~7 zTc9C{htNkoy~itSqfWzVPLK43cB*1!Zvs3w8@9FLC4<9}Xm}9w>iiFkpa6dG_11M} zJ(3S)W*tjLz`Y6b)V^q^VK%d6mrwY3Km;={C}UNW-|A~+BlQSZIc%G|Li#*(T1BSH zWopKc7Ihm>?CV&?nxP4AopcQ<@HkdseSxx@1&%{6UmPZ{--_q=}<)~{pKZCPAa37@U@ss~H(=p}s0r)#&d*>a0K@b9B!DXQ~b zS6}iDN@`jV-*~^6DV>1W{BhXPj8en)i?|tnBjyEGQU2(@SN~7 zC#Xr=*5vDYh*ADedZsXQf5CzRGIBa%Jlk8FkC_b(sQ2+dwW2V+;crU{y7443SGjaD z-GzZB`RdyClK1zid-*wzc*_bZ{seEBx9>@B8_f@w?Tnr8*e1ooPknH|z}2260SX|y z^wf8w^O-Pv3pH$vC(yTXQeh#qPBmsno5E1GzM1TMO$=iER72`tf3`{F2{>udM z%0o;lhNp|Y0LkOaS{6G8M=%BTuYcaCOXAj@|?X zjdWrD^Sj2hMD609r$BoVHN`Z!cb-!=Wf_LkRb`WmIKKT?>xngTVbdvzuCTWRk<~ZE zM6!A&z#IZfxB#TR1qljDnHYjh$wDbp&@Zj~V~~$ZLRVYnjG1$ll2%3!M4%rdbOsrK&0Ori*uB%Vs+zb59d$U7-f^I0y1dJQz?>d zk;n-rIohiK4*2ukr!?sCi^BLXq^#SAC2bj`cc;EhYxwr4Kom_Lx zEH6<8etN|^3T5*fNq5XfDZr{hZ3IyAe(>Ebgqo0M2ID-=&eG6pLCk+Vt{MBD4-`?B z(4n_}sN{17RiBHZcfV{upz`q!;cpgN_6#snGY>)^rQ-&0f0tT{f+E@i*oTAia-y0( zvo{$c3P<|PW8QaZW->>)tp=qJ#SQaHVdHCUJ#}Xwn#efPw&xB^Nn~-!Mf!_SJ*wBFg}rKJ%}C_&fb4sULyA! zD{@qGvcBBWq|yR0zV}r#n&Aq5$yg0e(Q>cJ-z}3E7wDs;6#N(_UDO&v9feSrk>;V| zJVg>?IiQ$qQpP$Y9I?_Y92{8mE3J9A)StPZN}m6_Lt04qY^1d!9iLPcY6Gt@1wLPt z4R}ij6spE-r?8mIU}1IqeXF_^#=B5@e{6}uddhUX`5O%8myq*Mt}@D4*^ z@W+hs+G;^UPPJwH%)?b%x=h{?21W;;nz`IS;xjO5)qCaRe!0fMn-93K>zQHoMZ3%I zt)ccd;i)$Pt6hPq7JWI$8bhcFL5NPBM|D#4e+#P$kY_2%7jo1N$!unJvVGb)HIl%L|rd2t=fkn9CM=Z=7sUFY_ zcVOVR>tzOETcUeM^6e{A)^TMdfcfHL9~!dbkW1S^Ef`1+f5BAe zG|KZyBq%}`5%#;55vADFeglR9nRGrsS|X>|H=>Dz{a9tcfDVofB{EV;%1%KN4jH;@ z9GLnM+5#9Lf~`-bX$px!C<1y-CaEDJ`&E)H`;e9lTWB&g%jv?f$K}gS>z9461)&}F z6)NR0IQDk}D`)uGrX>9tBXEt8rbq3Rj;c>IJ+|%~4xYgr+#Hk~ul63G`5W(n8n1jY z-ce&Of@`FGupN|spw>8+byAQ$*EX!Y1Od^5P&FDZUaBW%)la*gj`)uB1)%LYz z?qxr|=#ATmcevCP!*bgkmo-rH`;E~Og?i~9Nu#D(RHK?T$D4Lle~+E5#(~b#(=!^o z{t(YncGk}K-t9>CVngvo`^w7u>Ux9^xsRqnc5Zi_N=l_$lTY9`GY<=YUpmworTzU> zRktvk%&qhHplx(VMXwdkP+FYlb9xYJM*|-V*KK7i?3-f)a`$Id`xb(lX zY{EC0tbe(03Iv7-%Zk_kEK`Efi*x{TFNL>U66=6om}C*Z3K~Um=!T&*jQaV(dZ8uA z6GWJP#@{Sh7GVb8^Uaer2Kl)9zCLYbJKS~>k9GFvX)6CRA3Ca&g&izHDNpu`Y1UWw z5mtBMB+W3Dji${9Pg>jB?FcIzi0$#o>mohv-BIi!O`Bs2(Lhj?r@^F6qDKvoSqrz- z;G3D=U%@>0Rl{2sXMct&#nFmY+yns{%oY~>TgL%!tlFGc2L47-mi^$+cGNoUhf7F=f+~$rSkpFFh_Z^; z%>BgZp0Nw{B+(z4lE@I0Sw#AI#Y?9@U}1k}@8yqWtUab2ZAweSDRtqV`1|gM0d!hD zgb5`|3?{=n(V9&n(BhkxGuBN|fW&|~cyu$EZf*~7y5uxVtxCI4(w+Ko3&M4{W|KCV zS}LkFAOVkyj;zn(QG}_m7G@mxDm~hh4fdM0?6{`1#@`Q_%+loV0ZKE4RkFx15tKO+ zXO*1%aKW9T^;xwyRCR#rX5lN8j+@EH=s%Sj%LygTa{XuzypBiU|NEVwv3{KK2^v0# z;y&{RGR6$RIZjyli1!}f|8CkEiW!wI5PzKkd1B@Qr_*IW_ILNR-Y<9U3oS&KLLy74 zd4sa=&vQIX?(S|2a|&0RI`U8N9PER%wD8I|yoJ5r^Gbqzx?43gzUI5!3772LRc81( zqR$-*o4gL+O^SP0WJZ45W9}&sulOF>CB8Vk7jY_8ZiB(U7BUwp3|cr&7t8WPY~AXq zA01BwW8G$DRoN9AO*A|+?bOUUl$uR*cap~;5IZ&Va|E_q2{(b2KYdrM?!0{hZrqN3 zgZ)AFX)ckQM78#8$!UVL4FtydxVP%O;B2S`_@g8^(dLA6u05B1`c}4Z@+eh&8vhie z!dsw((thoS$o5AkoM_EeIB$S;k9fGz#kjo$yryYP{abvo$HoNgu=`Z)R73vM-|Wn2 z&xRZRc;wagnm)JQYdu>i=_EQGrNC#qhEC^3{0+n+SFJMb5gug1&QGxav)x$lqbkNf9z zLmg<(=>tZgZY!QA-)u@XA7R$ztE&b*-M0LByF%&ZJh1Yxt67bAn9v7UEKP776e4^P zD>IfIb$L|Ei?}1ic)L%d#%(Hju;v}Xp$*|E4qF^LC0TkO9a9`x&}s8~I@}>I#UIkA zQ$s)fvdW~FFRa3GKMBodl$RPu<9Z4$9+R-j*SGzq)Zgl{a$WuzJsp4ZGZG66CjAUd zCjW4|lB3Uu5S1&JB(`kq?m2y*{Bc>uictGJ87ZdEsQ z=XD4Arr?=lfTqLav-{Z$V$P?m1v?!@ISx5*kY1VmKuH*_5s;}oVw<^V-)JzD%Jfxnj$v1iy&LJOJY2D{C=9S%moMZ5 zhVD=O@=KA^WG?QdPD_FUEMtv7cN;-8WTKRo^wRh_-FxcWMuw6D26u8d_!|fHqXV+X zry*W;dB7fTg=KW^u5zc4j$> zVR|VFN%OB~v$yz7+B*=Al%NH<WMnk?j$&_l5E|*5XaR5)O^q&a5@QQ<+q&5Jt3ux4CmfnWvZn&O=1x@^fbMdQkX$))w%7bC|%%*Zg zCGXH5`B=6WX8F`VIuiG3@K-Ud&A8EOhd+sm<1V=}tyy`}X4y~1Dd|b-%;K7vew)PA zf9d8vYGkQa@M*sMBlV5vVJVzguj4CqAQI2sceCgxqmEkYI1R2a%fHVh6sdFtaF{4V z_Q;1wc4=}fgq(!`jyWW zB>PqR7y7c&U)LCRL&N=pmWiO~+84gzvvrCFt6P6Kr%H)|z`W)GrKO@$tWv&^jjmF@ zk&U7f$>oVIS%sCi66S!Fw*%&MvgdkTF1y0}DT;0S+Hx3m&R^3w>U;`JN4iTefj0^@ z;a>K0?XNfVulcrA2^J+xU8)q7ACdf98{3v3%xf}=*$aoZ-F z-mLwn4|!uJrObZ${F5wZ#*4}q784{VhH^DpA*VNK%V4?Sm++Fv%^&K7`ZV7dh#SMd zj>s;V@73)uqXUpHb86E>Q}slGxP5`S)Q-XmV(CEiz>uZc59PKnp($Z}4^b=r1#rcQ z1oRF^4~H}Lzg@0L3L9jbV2P~?W4UEq-_O3--2d)=+I#fxMjetz-?6+oy%AVi0+S`= zzKPIKoPx7)CR6ja{~q0MNIolz#eYuR9k)9m7In+uoIZVC#+0&~q`{^QRG?8Enw8># zf|Zuq5J)R{Jny_loS^Rx#2tpVqyXIg3U|%aIXF`j7ZiTIdf7T0xnCFiG6P{_YEBG1 zqmmVWyx*8mTStaZUZX}pEi?ex%X83muBg?|4Ktf2)DVFlKgfw6ugO?Xl(5>i*v6(qik`By9ZK$^q2LK=ViXR=NmGO$w zlGAV+l|XR1K$HsUYH7!VsCePjz#69bQ|W3~zFK$MU!Vd#0b&73q8}8IJ3P7t{jknh zgz}f^2YBPM{f@wxrGN6wWtruZGXH_3t0-lv;FvY6S)4pPP)7xhZi&`IWd|xuM~dcQ;Tc^UOg*v+;F12j#^&N zO-PQhsbk)YN>F7<0Zxh7lo+CY{rh+9RgW)ga#oPa#+-yP`r_aD}?cKlsEt94uu#+4i* z6l9+iFwvSNGtASB_BUoG6ebX@;k4#^IP`O~zab>q3#(mMPrjl#A23+znEJl&6%^SBq-6c@PqBZ|R`g6010WvxA2r zT0?Y=`2JZefzZeCI7;;F3WcdT1A$|`?R(wzoU3^{j%UHiXLBm7^TJO7Mdn+*T3~*$ zS)7=nFW^Cxxt?FgHg=J-dD5UAgSCKT)y%B({${SXO*^Br-Sp;wibpwjSY;l#4 zU&S>@EDaH=(zScf^e*pZm8jLHKR1|+WQ+j^>g0QFZQ z9ndiHqL3OfaTsoX$fXz;j2=IPRy_H?O1utnA>#iLl_5ew;rx}F{WGAUDE`_52)2X- zx+ww_l&|AoB^opgE))a{0Kobu0}ARN2Nw7z1_4SDKyoDj&?S&I2|4Hx2n(#!9SZ6nN!vf;-*wn2A^kwQ|NgF?1My&RSs)0Kg3yW(2}wQ} zHvmKpfCq__WP%}Lg0xBsL+e6r|Jq>)rxZQ$|9tnDP*BwWlKz+VW aP*8;bS57D@Bo7q;*^t6Qco6$1^1lEa^76+3 delta 10284 zcmZvCbx>bDw=PiJog&5Ei@O(!ySux)Zlt*L!`-b&afjmW?(XizZ{P1d=bpKeok{kx zXC;3mGb{5Phk@URg8xbchw8T*NoEJ7fztvR^{&o8I6fuubE8&JcLwj-A#hlJ&tHG< z&^$j8yZ)N#+~s{C*|2uX@dS3LYHpQi4oP7kMwPU@a8#yQgpl~>cl4gEd9FCjmBlb4 z6Rly{5dmNGL?8mZ54z-lvY~Z7Lbp;>$5Oz7KqN9ozw99!%Jp&MlGF7-lRtvUfrcos zl>ieH`+H7KjE%G?HSS@(;gb+NB7+5U_b%V)brYxm?E7SZFt@W67Ba^M+W^10^fiUt zMTa`5=JKV2DfIJKAVz=TF}Am?jAgJ2lGyF6WM81dI0+x>=HPep+=$AbCI!i_Vg)_m zD8!)VD8FyKG?V-1x^*HI#NW~qJhgO$I6Z|U6VPw8wn5gJ<`@dGs~=Q)MJ8x@vU)Si zQ3*Ry9uU7s9w$-Ea}=aML1Kb|fx&`>%IydVMV{wMd;$X-2Yo;g0J`=!90dKBROh_8 zGU*Sn(@T!%3E0on)k2<@#E{gS)a1zI(YtXt#4;}z(u)lkN;$eVI|B)(r|vNy5;w;_ zf}m!{K25@z!qMNnzu6pi6a{76bO+Kh7{V?0f;0Zu$zZIR(q^)55&`dJ#-?ojnrJ)i ziew4xqi2z%9p=kxz^d`if&0CqQVt{=nVq72-52Q~;=5mR9SK{72Hap`3v{=E32M*b z#v7iGgF9WMz9$_mqd2#;mFEI(?;+~Z{+M~2rhbc1vWrl}^<1J0TBt$|6l5*y5a?i; zBbLP#p8@9RyK|O7HuRJ^`Ar8fc>DzWnSf49XNVoeaRKZ{z|m_Y^6Z-PPPQQ@$)!6v z^maru{kveWI9DwN-fkq%cpX*ec+-mx-#kk?lzTEw*9v_g z;Aq6v2c)AH;OABgW=go z)R9R!m5CQPg^w8ea_{$wdGBn;e*yGYwa?}+rF7zWxf`ME3?Jjmm!^j}<&>Bz zjz|5PRVOD{I^xgsXC7%)gMp(MEFMLU%3C(f4{F+mAw%1e4^PW|PO`EDgbQ2=Fm+iUl>Gb!W14x(4QeMjX1YONphV-V!Cv1iLS0b zye6jnXYTI7uC}*H9a9i1h$TjwP&OiJplQ(-I64qMyQsR$rIzGFhk{!-8E}<+y^7td ziV_Ro)s@*<5>q=nIrDZ*{@q{ijiG%LS{cwN)$9aLE;+@XHVY-XjWiz@A3|KN9S|B3 z?q3_QCX3kJs<#QlUY~Osb0b}?Fv|RZ!_~$z1Rh{5d{M=?$Gddkc}2#1rajy7c(q%n z5B(Ji{_*E8Q6*$BFfCzFIs!Y8p&^y-a^aNL*CMbng-L%c63&xQyYq6|M)W-%f!?5q zHETnZ9{BzK#|;M|j_1#FsC|afU12#7QxoT|gOdPZnXMY*?fHq>>;C)s^1g=(=doKF z_P*MBRE{eD`}V^m&?=BPzvxw|!b*y?`fS#2m%grwI=8H4JL9EvF_#RILxjPzZh$}0 zEt)C9UcnT0@$*BonepMZW7kpLECAUf;(cQ1k_JT~GJ>~)+7RUd&$&XB9M!y9afULj zfr8?-w^G847TSo6Yh6CVJbJ>B8#?mM*-Yo9S@d5CX0q>vE6caXbhMS54Te6H8N&97 zfB?^xTFT6zRlQ2=ai_(B>Gh@0a)tdsVQH^GQm8wvekJ}5WA<1s8y&)o&_@ghx0Dax zO5x_+q$m}COM4Ws4D2cv7tGTvP9OCb$YuJu?WD{Oo3eN-hVA6rk_uFgkB&-y?x1Ho z|8Zna#nzEYPDMX$HJ^NRsCcwic<97tVO&@ci`UXoHPRclfC42Qn!s1-PNU}O!Cn+) z_Sr|aK&TYnZh$>czUgqW8>5Kv)NHMRm0ImBYAW-qy}1Hl@QfSJZA?Wg*nrC5m()sw|~&4BzN3t zgprZ)0&WQiW>xs{pkkZzC-oeoUQ+;Re~mIe(%Z+0DpsLd-e>FqZiPOFUJ8Nn;u;PH zYMBi7{<&|^CMS3NW;nfeYlkq$!q_8k$#U)#pQsLT(sJx!`~pdb?~W}rZbB<24E&+h z6miARGFWZu4B3zxmfsMe?64D+6Y+(AFDQN<@&FZRFv3CV$gKoKJ2*X#CTrJ3HWDjI zJ6WO0eqxP%m8!btz*?4`ont=YOxFk#ICqHVJQS{94=0`tWkU}S7H6%A*L4hmwx63S z`BLRL)59)6ts5jh+K{)=_QMHDZ*i*fOVu?pgU)Wy47)|W4#sGJ^;K&~@jp0L+jvA# zA%JT!84F6@vnt0~_`*&+i3$r1H8pmKp{V!+qlQ2<=;>A>Ln-19*)U zn!^;G;!Btt@g{nvwjzyw=rLj-E?oluWNBy%+D>0TM7-BqTx%1(o4GJR98g2aFrJG^ zUzb7ryN$ovK2ETBg}=MAc84LeQtQ?He9}>sm%@uavfAVjjUu&}J6+LGwth%j)qRS0 zf1Hy1$$qz|-;HQ1T9{{LV*Aw1%AKFD`Z4(26*H?N+K`QpcJrJ6ePjfn*E?sW>%2(t zvx}bDe2STw9hvxSdEzQSeP~YW#+kvylX@XL&rpN3*ww{S0@CBPWML*v0<{$v-&qXl zc6wPsyx5h^Ak=lkx#+&p`j(qD-q+Y=D9eyv+&*am97VIP#hQfmhL!g+BqWXM@_H)H za|67~pyRv+n)fFMc1Q@pvH}+6W8NX&LUcA=bCd1-FM&B#Z=a}^kmcNN5Gh&)#3cu) z3oQ!pb=ysK*)sVeN5!?}_GZG9sf*(U-+<`l8g_Z~N1f48#`h%AW%VL+X3ktrvgv$; zVjNGw2aBjHDSc2kHg7SkN0+`KCXWUp^K+|MNU(VX*kc%C3)?q;Z!y$DHw4aaty?Vw zq9!_|x_tzsb$OO}YHSxS^xZE;p-I?}Va=5Q`FfkaL>$K3g{qUKV=R`t9{StEJQ>R! zi<}@_fVWr(h2ei7zw}{(U!XbS$e?>6P zwMl|H2iEWI+ptowF3V-E1bV;e?-J5wpY6T{Q887Jr0cvcFQQyCGQOYH4Z6-o0D>>i zj?1R~Id4PjjOfvAWCPh|NQ12~YU&TmWTiBL*;_R`7sGIbFFvH%XYP~Ea)W%!v2V-f zv8t3fz{B;_F0FL5_mTI4z=J7~@8rs$*%`*QNtZXC!?&@z4$Aj)cEeC7G9x_q*~hb? z@B#k6AiI30d9h?AbWC`U8rLr3e!k4E5q;ZHPLz|Z(^$65gCAtYS&*FfmgW3{twLVM z5UvMfoQzkM2(KjwuPF$x(@^hw5XT=#tU4^Qf!k3dmuxRp;EIeR&7M^#!bJa@r z0RmDHJg^i@SjC^1RtOb~I1C-&Eq{tMweWhO7NG<()o zg8Oeg+@^D28|op8jUDJ07B15A?9+kE#S_VqhhGOOXay4w7MZ_f`4*%&33|oEWyF5j z^Nw*%C694MCx7_=c4&*b642D}+HC)47!VN_F3@qcBzTU%Zk6rs zQY7PB;BClmM>5TxZ6AAHnKxZEG8%EgHw|KG@wYa4A?wbCsJUbuq|^@`Q(ga*&}`Y< z8O3HXL3XX7zG=mcoyn(0asSr(7ExBM>Wop`boGUL@oJqed$2k|Qp9Wa9F zPEmtxGy6xWph=dB(GcHJBff)j@(!xUCQD8Bi%?NHW!F4okw@1WUD2e-F|B>#%*jmp zPAlR}F2+_XF0rEzh9F5X>ML_%DiZr+Gri2_n(G0i+-Z`XCfLmNcieyZ#fq} zv`pFty53<*UnT7<>Ngtr8HS#fME5{iw6AvNgrk&Cfzb`$RfEpK(Dkccu^SZAVK;ZRhc0bizn zYWCr8QAKzwBI`Ce(X)#@ZQe^lUs&t0p|I@c7vH$3f3f-YdVE{>w9t&XW2pTjo21i> zuhQ}THuOdQ^0ITSt1#JXH#gajyYgZ8^o5h?;w|RJHR_wKwNBE$i7MpFi_G~3ci9cZ zAWmWkG<{W(mlD)EL>1+nc{z(dX-!O^%cS!hN~h!Jo~;ufTwNT{`DZ@hoR+12`7k?^ zK!U*PT6g&OP(#$QxX;4t!ba@kOKeZJU!#hJe$qyOPF@cuZIF>ScYZ!05hXsB#l5*k zMtwcPV@!{77?T^+eR00_bNw)basy2h#vK;?w<1axXFpBClVP>Ithsvk?M;Ob{9_rw751{4A=BU%{m5|W?# zjZ$XQP7<1D^+j^mQ%Sy;b3Cc7=IbLgNYx!)RDVL042vty{rCw9+w=h*M_6ZpHxCV8 z{bc|*6BZ2Y0}muYLQj%_PC;1<(T^kMUr`z{r*72DOFp==zC1A&kskymWwk5`N+h8G z2;220CNiB5GPyUOCM<5&RTptJyQ45pvSib|c-RB^BUk$9a%348^yVm)$aIJ&#gXr? zmQ|2$=h~FJ-si~=jUX4iUTw-re|$6=I{XaO?Q=abM(IqXc8>SYsjzX#F*Lb_ALwz7 zlHti>)e&o-3;Xs--hV@fksaM*?-5)8z=@WLDlVqvPD{sB9>09(@#dsN`pRxO6mL=Q zc;^XVn;yc(Fs8 zsudA$aTZYQ7?`=}*^tLwEGG1<0z&$gYOZ@^{@zd*$TdPJ2LoqgEeOs;dQ&W3JS|Qf z=^%n+2<}AWuHq=g9UCjhPf~Kz-66FF5bV2%p@Dk_NbvCo&@se#OIC&x?A+g+sS~j1 zX#6@GsUJVvZC-#q#N!yRE9G2NqGk}U=T{j*aVVjTCrFJTwlEGYjDXkF0K$Cu)h8o` z=;EXXsdNR9jkJW|d$Y~43`&9O;1o$zvgyhZq>&R0Svhtap4e3%B~MSc2PKoeg7)=< zy0&~vmqZUsTf!3}Rn>^nu*bbrAcH3a?;Z-$<(GgPq{s)gXgeA8lOA{|wvS}o5?EXuHxi8vgK};%MW~vpg8#Z>5~jGHjgJjm z7ALVj!7+eJ8Dzo@%aZOPW}*7`+B_C0)FdJV1rz9XVPVm9TL8EpwpbP@9{{6*BqaNCSYEE7>&9Ga5#*{O%9} zTYRLFtMTe|yJ(sf9~K^MHh)IV2(@rD6P`gWh~giM``!3j1VmP)JYg>$% z7^+F2n=@3w(G)40h5!255?3ZEQTRigj_MXEHC}G*tClOHPO1QK9C)!}PN3BYH=Fx= z^aG)`5yC1Z9#Eat3DcQzcAQd`bq}XPQ@aj_ue1uZ@-M(EcZyye#k(uPOPqnfUorU9 zEXw|y&A+0P!aVHPkh9g-oloC|aB%6d1pN#*u+-6Dla$o719 z(%g=Zj)e3KA;L^?lx2G17c1RO={$wDG+Zhvz99YW-4Be-X^V%nWg&?3dPgPM@Ij^) zqQn2n{H)1eX+H4mJCiT}Y}%Kw66)Z*xBAYfnoP`f%(e4L2>BO`0{ET=bJT9{I!Qkj z+J0fceMc7uk&n{m>Y3T4M7tC0%ZnLPD@37^*%llP{pYo8q3wD!x3ZF-%h;wG)ewS3 zs#|p$#WX$&QE>~SC2L?urr(;%U&bOg`k0YZ%6%~ zcNhesP8GX1gP#qJ8#(4kn0^;QE&ai1kIC5b1$n^zTkpZZ!NaBg_QjjpfExe zwz#Y6Az#{MuRZq`)l)D^sL#)mu8c-Jz;caQ9Sr$bGC4477l~!`~z5stZGv{=ogincE4r;&*qwO5|I?sK25v zc?~r_t*YqsGn&Y*-mNo6%J?QDN?qdQSKzKe*M69LZg>jt#p5Ua#ZC28 zK^)c#!&NrUXo69#_4DYR)=gO~S6zm`x}?CGqgTlnflcN{Jl}a*O@mUQtjzA|J#G2y z)E1H_#Z&QUTLkJ)ceoDC#`xN$*~Fv_{%wP2^XI4srP^^L%AAQU8;&g2alQc$HeIn6 zBV>6@)*?98l*z(M8A3kd`*I|_9tdA$bF*@U`Ez2RX4xxW~ z_h-Ma`%DWD!NO%#1@Qn62F9og28Q(aUEjlo$=Suz#?+b7!`5bA$2N9V1M5Ae5BR0i z4bvx++2@%7QlV!7WIz)>J4_B29gUYQ6RtbQec>f!?0&qLll+IB%d3mA&y7y2E)vPC zr9Yo!WmU(72amOaf?A)~-xkDGRmHQv9^o~zGM}E*a4S_``rOm4l3gxw*b}A~mw-if zmsSyaZ8*@S;JYG^cYa-Q2{{0}xFcw?_i#8N+!GgFWNRe@MCO{Dp=9X}pF`Lca6>Eo zsNg`I+wT^9Ye!sfdE1({qmAvxBhsqRS(S?@&@tBawhH${l~9@64k50aC(RQ>$DSF# zX%Fwb)CM!Bne``H4s0Vm{@yhb8O{n{$VegOoWN!ZbV{1K1 z=61CR;fd7ZNmoU7R%kDOpaNd*Pj92=3SP|3PPiI(X$f<`sKu%cMuAiaBlH?omBdUl3k91_-_TK!a=XRAz?A@{X%T!rHc z)h=TKD&`?9rLh(UCMqWS%A0lcwKVXgL^?f_u%D#KS@M{mZlx?UsJ<6eVK|*7WYK&V zow#FtPAD1|Hmy;$Q61@PLC{|R`2#8KVD|?seQ_vAGoTVbX&rMlOUxkhRl8XFu+QFg zLASIn9+l1+F$mequf;HftYYEET>G3Fv${qLkl1PJ*PN{7U?N@whoCj)Sf+iZTfzPn)BZo3^!?hc*S~dW*{#o#R6XnEimtMf89`h zRHJ6uyw$uDP$ zV)_$!&ZWf11)hk`OMVB7fbpU#G!SQOL_>`8q>7v2Y+Gn1!Xt7kNE;0!QpP~zz{rYH zxr?Y$y;I5?^U0oT|Bb8(`=lf^gGw4TEVWl-!a%6u>RfK04rR+>~rKr9L_T0B-yl3}z z?rN5S{4&OJl~XePO^<18i$`)>g1^4zQ{!6PH=x%22SLwR2{1In&I{fJC&sfLRud-m zHAM8K8ETEj-VGY!k7m#Uj-Ov|BQmrv21ffaPxkWl5i~%8loDb5HYEn7%r% zkYTMYqC`h&xr_W@k^Pi`9QmD74@_c11Tee!YExORn{s}8C7Xj$V;>SNJm;T zu1~+m9JW+{gUM?-unhjBDL^~;`l;x5kq!-)W{y&62!d4QymD!rD0xfCmWgdPZ6B5h z>?uK!mI#Rqj$v?f4~+0cL`3OSz^;eIAr(mcS(F(-*8{>dg^G}LQWPl^9fKYc{!Zt_ z{KHXJ;UiGTe0tyvdAO!3(gco%GN3Z-mSm2HQ=9rXmQ-)d?!5couB+Vfx{emG1luQ z*i9_jEH^(7zI0gyQ`;eQ@VE~%gQ8#+=LdyPAMO&A7eg9r`ne{{VQ?p;1?$Ukgx(es zn{+A;2gd6Xd&tWH+L?+0FTWY^fJO}D9g{TEVv1vjW11E;{&HHT)~?Wb5NQ)8BiV4q zuEV*B7_diXI&y}zmAC*Q96|91J+&W=qOgGrj|k)Y^qun?1}Y6jf?Qp1?v9stn>=F_ zSBytPqH7HY(ny+t$Nst(kpA1vYsV&(Nnwa1%d?wT;@O#CER{S!hzi6#Ou!vUD<}r0 z0A9!<#YCYZ-<=NG?xM4)$03g&e9uDz*envl5}s+KhUY}tz&?qIZgQpOJzaL(!c3NQ z`C<-%n-JqY^amScsBT@z3v%=R&9r?y`nVwwIRj#(ZHn{NB@+>Rd)%Cq-++Tm*dV3y z57+y>6O}Cg&f)o(1soH}6<>ak<&wQ|ib;U9678P5ho3?LMCv1Q7Sj0EkF#ru+_k$2 zV@15ImHm1O8i7Mb5i3965R_#`_WP-c*Ws-YL6(~^xsr@5AwNP64vAu% zh|$fl<)8MjkU&hNLwCRHonz2nUY@v>%BhD$-1ayo-2<^x=S7Z|NwLd;2w35$&S8yHLSM5Y2j=z4< zsnZckd;d|z|CElW>y}E@j3hPN2Dt+0WB+<0;PX@178YNw|@2$%CV37<(oAWg~tHkgPUuJZ}4UH-VMJI@j+dYSGm z?t4saCnWH$TF3nwj>$YqwmDFEN&Z6=+k!*4&OFN2Ua(zL{pq^t*ji~y{H`dowi}UL(Q0b%g)0io1k!c??hQ;GS=20LXD-@|N5iaz`n zY0eGkRsxBc;*EJZ4|Vi&j4z2ctl3k8ZfDXR6XFe7zMhtz$T(lsl*-yWxC zBaDRU!KQLkBFlk0BE%b1Z+|4mdkU)U03QqZh{Dk{6`>PT&Vd{+qqE#b+-V!+Vus>I z_K4=Uz$A*=zL8*b#H`DTwwchXL^?_Lp|~iwCZwXDBu=9`oEP5wj)U{i#Evy-hX#>X1Qout+5sIeVyc{3PM2mN{`-M$Tl8irdDtb*0WluqS zCxUtI1YXMv-gVyN>dwZCkGm_qu)qS74(?4*k)Z#{zH(O)E68}pKnv=VEmAmo82i3I zeIc6Lc6dOk58)}DK-~kkRnB@5;CGi^c%YTV6~fF(@l_5s&uISe`0_f+;D!5T_}O7B z^;lDwEjN^vaK<9X;!F9i+W7O$2=@N@3GB>;s>4L9ijWJ;;>Cj9edhhXXKTD72521iU2<2;(ZzFaT=vQW&RA_^i!rkmJmO7CkxBiV?6Sq15dzdtuM z-wAJ^_+bSmQ|Y2Q*36;&lSWaYO8qeRT*;T&wC&d>4(73gO`P}~iKjh2p!*y7X|HpW zh(IUTBvZ*|)h&*0t{(erY1TDI^Vi}9<;5)A?$f_Ko##Cs?<<6VxLcri0lYdh{&sMP zIX)1%pdiF52gp>A1>%DPlqQ()UyL|rAs+nyP3?jK1H<@R82;11!HEBkC!k;ypl0|xf_|FHi+0A(ZKVJk>O{oN5581~;!8ww2UhrmD5 F{{rEo)Gq)4 diff --git a/plugins/tensorboard-plugins/tb_plugin/setup.py b/plugins/tensorboard-plugins/tb_plugin/setup.py index 3c0900612..61d29c402 100644 --- a/plugins/tensorboard-plugins/tb_plugin/setup.py +++ b/plugins/tensorboard-plugins/tb_plugin/setup.py @@ -21,9 +21,14 @@ import os import pathlib import subprocess +from configparser import ConfigParser + import setuptools +config = ConfigParser() +config.read('./config/config.ini') + def read(rel_path): here = os.path.abspath(os.path.dirname(__file__)) with open(os.path.join(here, rel_path)) as fp: @@ -83,11 +88,10 @@ setuptools.setup( name="torch-tb-profiler-ascend", version=get_version(os.path.join('torch_tb_profiler', '__init__.py')), description="PyTorch Ascend Profiler TensorBoard Plugin", - long_description="PyTorch Ascend Profiler TensorBoard Plugin : \ - https://gitee.com/ascend/att/tree/master/plugins/tensorboard-plugins/tb_plugin", - url="https://gitee.com/ascend/att/tree/master/plugins/tensorboard-plugins/tb_plugin", + long_description=f"PyTorch Ascend Profiler TensorBoard Plugin: {config.get('URL', 'repository_url')}", + url=config.get('URL', 'repository_url'), author="Ascend Team", - author_email="pmail_mindstudio@huawei.com", + author_email=config.get('EMAIL', 'author_email'), cmdclass={ "build_fe": build_fe }, diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/file.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/file.py index dc9abb056..a833902a2 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/file.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/file.py @@ -160,10 +160,6 @@ class LocalFileSystem(LocalPath, BaseFileSystem): return StatData(file_length) def walk(self, top, topdown=True, onerror=None): - # Note on followlinks=True: per the tensorboard documentation [1], users are encouraged to - # use symlink trees to have fine-grained control over the filesystem layout of runs. To - # support such trees, we must follow links. - # [1] https://github.com/tensorflow/tensorboard/blob/master/README.md#logdir--logdir_spec-legacy-mode yield from os.walk(top, topdown, onerror, followlinks=True) @@ -261,9 +257,6 @@ class S3FileSystem(RemotePath, BaseFileSystem): def download_file(self, file_to_download, file_to_save): logger.info("s3: starting downloading file %s as %s" % (file_to_download, file_to_save)) - # Use boto3.resource instead of boto3.client('s3') to support minio. - # https://docs.min.io/docs/how-to-use-aws-sdk-for-python-with-minio-server.html - # To support minio, the S3_ENDPOINT need to be set like: S3_ENDPOINT=http://localhost:9000 s3 = boto3.resource("s3", endpoint_url=self._s3_endpoint) bucket, path = self.bucket_and_path(file_to_download) s3.Bucket(bucket).download_file(path, file_to_save) diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py index d6f9bb245..ac06a2640 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py @@ -26,6 +26,7 @@ import re import tempfile from json.decoder import JSONDecodeError from typing import Dict, List, Optional +from configparser import ConfigParser from .op_tree import OpTreeBuilder from .. import io, utils @@ -44,6 +45,8 @@ from .tensor_cores_parser import TensorCoresParser from .trace import BaseEvent, EventTypes, MemoryEvent logger = utils.get_logger() +config = ConfigParser() +config.read('../../config/config.ini') class RunProfileData(object): @@ -363,7 +366,7 @@ class RunProfileData(object): dataloader_ratio = self.avg_costs.costs[ProfileRole.DataLoader] / self.avg_costs.costs[ProfileRole.Total] if dataloader_ratio > 0.05: percentage = dataloader_ratio * 100 - url = 'https://pytorch.org/docs/stable/data.html#single-and-multi-process-data-loading' + url = config.get('URL', 'pytorch_data_loading_url') self.recommendations.append( f'This run has high time cost on input data loading. {percentage:.1f}% of the step ' + "time is in DataLoader. You could try to set num_workers on DataLoader's construction " + @@ -375,12 +378,11 @@ class RunProfileData(object): if self.device_props: # Tensor Cores feature is available on GPU cards with compute capability >= 7.0 - # https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#features-and-technical-specifications major = self.device_props[0].get('computeMajor') # If it's a pure CPU run, then self.tc_used_ratio is None, this rule will not be triggered. if major is not None and major >= 7: if math.isclose(self.tc_used_ratio, 0.0) and self.tc_eligible_ops_kernel_ratio > 0.0: - url = 'https://pytorch.org/docs/stable/amp.html' + url = config.get('URL', 'pytorch_amp_url') self.recommendations.append( f'Kernels with {round(self.tc_eligible_ops_kernel_ratio * 100)}%' ' time are launched by Tensor Cores eligible operators. ' @@ -395,8 +397,8 @@ class RunProfileData(object): if total_mem is not None and peak_mem > total_mem * 0.9: percentage = peak_mem / total_mem * 100 if total_mem > 0 else 0 total_mem_gb = total_mem / 1024 / 1024 / 1024 - ckp_url = 'https://pytorch.org/docs/stable/checkpoint.html' - amp_url = 'https://pytorch.org/docs/stable/amp.html' + ckp_url = config.get('URL', 'pytorch_ckp_url') + amp_url = config.get('URL', 'pytorch_amp_url') self.recommendations.append( f'Device memory usage is at the limit of device memory capacity ' f'({percentage:.1f}% of {total_mem_gb:.1f}GB on GPU{dev_id}). ' @@ -406,7 +408,7 @@ class RunProfileData(object): def _analyze_distributed_metrics(self): if self.use_dp and len(self.used_devices) > 1: - url = 'https://pytorch.org/docs/stable/notes/cuda.html#cuda-nn-ddp-instead' + url = config.get('URL', 'cuda_nn_ddp_instead_url') self.recommendations.append( f"It is recommended to {href('use DistributedDataParallel instead of DataParallel', url)}" ' to do multi-GPU training.') @@ -428,9 +430,9 @@ class RunProfileData(object): communication_ratio = self.avg_costs.costs[ProfileRole.Communication] / self.avg_costs.costs[ProfileRole.Total] if communication_ratio > 0.1: percentage = communication_ratio * 100 - compress_url = 'https://pytorch.org/docs/stable/ddp_comm_hooks.html', - grad_acc_url = 'https://towardsdatascience.com/what-is-gradient-accumulation-in-deep-learning-ec034122cfa' - lamb_url = 'https://nvidia.github.io/apex/optimizers.html#apex.optimizers.FusedLAMB' + compress_url = config.get('URL', 'compress_url') + grad_acc_url = config.get('URL', 'grad_acc_url') + lamb_url = config.get('URL', 'lamb_url') self.recommendations.append( f'This run has high time cost on communication. {percentage:.1f}% of the step time is in ' f"communication. You could try {href('Gradient Compression', compress_url)} or " diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/tensor_core.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/tensor_core.py index 3a69cf70b..501e2076c 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/tensor_core.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/tensor_core.py @@ -8,7 +8,6 @@ class TC_Allowlist_Meta(type): class TC_Allowlist(metaclass=TC_Allowlist_Meta): - # Refer to https://github.com/NVIDIA/PyProf/blob/fd1b2902e3306119eee40ba6b6e8b2f816920c29/pyprof/prof/tc.py#L19 allowlist = ['h884', 's884', 'h1688', 's1688', 'hmma', 'i8816', '16816', 'dgrad_1x1_stride_2x2', 'first_layer_wgrad_kernel', 'conv1x1', 'conv2d_c1_k1', 'direct_group', 'xmma_implicit_gemm', @@ -25,7 +24,6 @@ class TC_Allowlist(metaclass=TC_Allowlist_Meta): class TC_OP_Allowlist(metaclass=TC_Allowlist_Meta): - # Refer to https://github.com/pytorch/pytorch/blob/69b2bf70f9c0e591ce5e566afa59e19618031ead/aten/src/ATen/autocast_mode.cpp#L290-L351 # noqa: E501 allowlist = ['aten::_convolution', 'aten::conv1d', 'aten::conv2d', 'aten::conv3d', 'aten::conv_tbc', 'aten::conv_transpose1d', 'aten::conv_transpose2d', 'aten::conv_transpose3d', 'aten::convolution', 'aten::cudnn_convolution', 'aten::cudnn_convolution_transpose', diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/utils.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/utils.py index 8f4189d76..21909d9c2 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/utils.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/utils.py @@ -76,7 +76,6 @@ class Canonicalizer: input_time_metric='us', input_memory_metric='B'): # raw timestamp is in microsecond - # https://github.com/pytorch/pytorch/blob/v1.9.0/torch/csrc/autograd/profiler_kineto.cpp#L33 time_metric_to_factor = { 'us': 1, 'ms': 1e3, -- Gitee From 9b6c9b92af4195e5dbc5ea70e49b13da2032dc5d Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 20 Aug 2024 20:31:01 +0800 Subject: [PATCH 398/791] bug fix --- .../msprobe/mindspore/api_accuracy_checker/api_runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index 8147440d7..9fc126759 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -129,8 +129,8 @@ class ApiRunner: if api_platform == Const.MS_FRAMEWORK: if len(gradient_inputs) == 1: gradient_inputs = gradient_inputs[0] - def api_with_kwargs(forward_inputs): - api_instance(forward_inputs, **kwargs) + def api_with_kwargs(*forward_inputs): + return api_instance(*forward_inputs, **kwargs) grad_func = ops.GradOperation(get_all=True, sens_param=True)(api_with_kwargs) backward_result = grad_func(*inputs, gradient_inputs) # can be single tensor or tuple backward_result_tuple = convert_to_tuple(backward_result) -- Gitee From 808c485d8d3b3e20fbb9dfa02793df88ba4c2f37 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 20 Aug 2024 20:34:12 +0800 Subject: [PATCH 399/791] =?UTF-8?q?=E6=97=A0=E6=A0=87=E6=9D=86=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E5=BA=94=E7=94=A8=E5=9C=BA=E6=99=AF=E5=92=8C=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E5=9F=BA=E7=BA=BF=E6=8A=A5=E5=91=8A=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...72\347\272\277\346\212\245\345\221\212.md" | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 "debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" new file mode 100644 index 000000000..65034d7fb --- /dev/null +++ "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" @@ -0,0 +1,61 @@ +# 无标杆工具应用场景和性能基线报告 + +## 环境信息 + +NPU：Atlas A2 训练系列产品 + +CPU： + +![输入图片说明](img/cpu_info.png) + +Torch：2.1.0 + +CANN：8.0.T5 + +除上述环境信息影响性能外，API的数量、种类以及Shape都会对性能产生影响，因此本次选取指定网络进行测试。 + +## 模型信息和性能基线 + +大模型在使用msprobe工具dump数据时，建议先简化模型层数，减少dump数据量。 + +以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 + + +### YOLOV5 + +主要数据类型：FLOAT32 + + + +### LLaMA2 + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16 + +性能基线报告 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.24 | 13.69 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | +| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | +| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | +| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | +| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | +| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1.66 | softmax本身为高精度，跳过 | +| check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | +| check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | +| check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | +| check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | +| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | +| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.375 | 1 | | +| check | 前 | ["matmul"] | change_value | | | | | | +| check | 反 | ["softmax"] | to_cpu | | | | | 不建议整网 | +| check | 反 | ["softmax"] | improve_precision | | | | | 不建议整网 | +| check | 反 | ["softmax"] | add_noise | | | | | 不建议整网 | +| check | 反 | ["softmax"] | bit_noise | | | | | 不建议整网 | +| check | 反 | ["softmax"] | change_value | | | | | 不建议整网 | +| fix | 前 | ["softmax"] | to_cpu | | | | | 不支持整网、不支持反向 | +| fix | 前 | ["softmax"] | improve_precision | | | | | 不支持整网、不支持反向 | +| 预热 | 前 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | +| 预热 | 反 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | + -- Gitee From 3db117695dd7ff0c33651452e933d5abd9692ef4 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 20 Aug 2024 20:43:43 +0800 Subject: [PATCH 400/791] cleancode --- .../mindspore/api_accuracy_checker/api_info.py | 18 +++++++----------- .../api_accuracy_checker/api_runner.py | 2 +- .../mindspore/api_accuracy_checker/const.py | 7 ------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py index 901e5162a..4308d411c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py @@ -10,22 +10,18 @@ class ApiInfo: self.api_name = api_name self.forward_info = None self.backward_info = None - self.has_forward_info = False - self.has_backward_info = False def load_forward_info(self, forward_info_dict): self.forward_info = forward_info_dict - self.has_forward_info = True def load_backward_info(self, backward_info_dict): self.backward_info = backward_info_dict - self.has_backward_info = True def check_forward_info(self): - return self.has_forward_info + return self.forward_info != None def check_backward_info(self): - return self.has_backward_info + return self.backward_info != None def get_compute_element_list(self, forward_or_backward, input_or_output): ''' @@ -37,13 +33,13 @@ class ApiInfo: compute_element_list: List[ComputeElement] ''' mapping = { - (Const.FORWARD, Const.INPUT): [self.forward_info, MsApiAccuracyCheckerConst.API_INFO_FORWARD_INPUT, + (Const.FORWARD, Const.INPUT): [self.forward_info, Const.INPUT_ARGS, f"input_args field of {self.api_name} forward api in api_info.json"], - (Const.FORWARD, Const.OUTPUT): [self.forward_info, MsApiAccuracyCheckerConst.API_INFO_FORWARD_OUTPUT, + (Const.FORWARD, Const.OUTPUT): [self.forward_info, Const.OUTPUT, f"output field of {self.api_name} forward api in api_info.json"], - (Const.BACKWARD, Const.INPUT): [self.backward_info, MsApiAccuracyCheckerConst.API_INFO_BACKWARD_INPUT, + (Const.BACKWARD, Const.INPUT): [self.backward_info, Const.INPUT, f"input field of {self.api_name} backward api in api_info.json"], - (Const.BACKWARD, Const.OUTPUT): [self.backward_info, MsApiAccuracyCheckerConst.API_INFO_BACKWARD_OUTPUT, + (Const.BACKWARD, Const.OUTPUT): [self.backward_info, Const.OUTPUT, f"output field of {self.api_name} backward api in api_info.json"] } dict_instance, key, key_desc = mapping.get((forward_or_backward, input_or_output)) @@ -57,7 +53,7 @@ class ApiInfo: Return: kwargs_compute_element_dict: dict{str: ComputeElement} ''' - kwargs_dict = check_and_get_from_json_dict(self.forward_info, MsApiAccuracyCheckerConst.API_INFO_FORWARD_KWARGS, + kwargs_dict = check_and_get_from_json_dict(self.forward_info, Const.INPUT_KWARGS, "input_kwargs in api_info.json", accepted_type=dict) for key_str, compute_element_info in kwargs_dict.items(): if not isinstance(key_str, str): diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index 9fc126759..62286dc0f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -65,7 +65,7 @@ class ApiRunner: api_type_str: str, Union["MintFunctional", "Mint"] api_sub_name: str, e.g. "relu" ''' - api_name_list = api_name_str.split('.') + api_name_list = api_name_str.split(Const.SEP) if len(api_name_list) != 3: err_msg = f"ApiRunner.get_info_from_name failed: api_name_str: {api_name_str} is not in defined format" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py index 0d7e67fcf..74079aafb 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -1,11 +1,4 @@ class MsApiAccuracyCheckerConst: - #api_info - API_INFO_FORWARD_INPUT = "input_args" - API_INFO_FORWARD_OUTPUT = "output" - API_INFO_FORWARD_KWARGS = "input_kwargs" - API_INFO_BACKWARD_INPUT = "input" - API_INFO_BACKWARD_OUTPUT = "output" - #api_runner MINT = "Mint" MINT_FUNCTIONAL = "MintFunctional" -- Gitee From abe8336c7e8e555659241780837adca8266609a3 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 20 Aug 2024 20:54:05 +0800 Subject: [PATCH 401/791] cleancode --- .../mindspore/api_accuracy_checker/api_runner.py | 14 ++++++-------- .../mindspore/api_accuracy_checker/type_mapping.py | 12 +----------- 2 files changed, 7 insertions(+), 19 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index 62286dc0f..5473a9ae5 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -25,16 +25,14 @@ class ApiInputAggregation: self.kwargs = kwargs self.gradient_inputs = gradient_inputs +api_parent_module_mapping = { + (MsApiAccuracyCheckerConst.MINT, Const.MS_FRAMEWORK): mindspore.mint, + (MsApiAccuracyCheckerConst.MINT, Const.PT_FRAMEWORK): torch, + (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional, + (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional +} class ApiRunner: - def __init__(self) -> None: - self.api_parent_module_mapping = { - (MsApiAccuracyCheckerConst.MINT, Const.MS_FRAMEWORK): mindspore.mint, - (MsApiAccuracyCheckerConst.MINT, Const.PT_FRAMEWORK): torch, - (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional, - (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional - } - def __call__(self, api_input_aggregation, api_name_str, forward_or_backward=Const.FORWARD, api_platform=Const.MS_FRAMEWORK): ''' diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py index 44989f0b6..e622d06b4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/type_mapping.py @@ -3,9 +3,6 @@ import numpy as np import mindspore import torch -from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst -from msprobe.core.common.const import Const - INT8 = "Int8" UINT8 = "UInt8" INT16 = "Int16" @@ -114,11 +111,4 @@ uint_dtype_str_list = [ UINT16, UINT32, UINT64, -] - -api_parent_module_mapping = { - (MsApiAccuracyCheckerConst.MINT, Const.MS_FRAMEWORK): mindspore.mint, - (MsApiAccuracyCheckerConst.MINT, Const.PT_FRAMEWORK): torch, - (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional, - (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional -} \ No newline at end of file +] \ No newline at end of file -- Gitee From 1201f94959517e44626937c9098f8a00e5219528 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 20 Aug 2024 20:55:34 +0800 Subject: [PATCH 402/791] cleancode2 --- .../msprobe/mindspore/api_accuracy_checker/api_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py index 4308d411c..657b5f9b7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py @@ -18,10 +18,10 @@ class ApiInfo: self.backward_info = backward_info_dict def check_forward_info(self): - return self.forward_info != None + return self.forward_info is not None def check_backward_info(self): - return self.backward_info != None + return self.backward_info is not None def get_compute_element_list(self, forward_or_backward, input_or_output): ''' -- Gitee From a2fee150e9dc53a34bb963405be6d92104a3fe51 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 20 Aug 2024 21:00:50 +0800 Subject: [PATCH 403/791] =?UTF-8?q?=E6=97=A0=E6=A0=87=E6=9D=86=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E5=BA=94=E7=94=A8=E5=9C=BA=E6=99=AF=E5=92=8C=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E5=9F=BA=E7=BA=BF=E6=8A=A5=E5=91=8A=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...5\345\237\272\347\272\277\346\212\245\345\221\212.md" | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" index 65034d7fb..7e14a37eb 100644 --- "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" +++ "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" @@ -32,6 +32,7 @@ CANN：8.0.T5 NUM_LAYER：1，1卡，主要数据类型：FLOAT16 性能基线报告 +其中耗时为训练10步，去除第一步耗时所得的平均每步耗时。 | 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | |--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| @@ -48,12 +49,12 @@ NUM_LAYER：1，1卡，主要数据类型：FLOAT16 | check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | | check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | | check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.375 | 1 | | -| check | 前 | ["matmul"] | change_value | | | | | | -| check | 反 | ["softmax"] | to_cpu | | | | | 不建议整网 | -| check | 反 | ["softmax"] | improve_precision | | | | | 不建议整网 | +| check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | +| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | +| check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | | check | 反 | ["softmax"] | add_noise | | | | | 不建议整网 | | check | 反 | ["softmax"] | bit_noise | | | | | 不建议整网 | -| check | 反 | ["softmax"] | change_value | | | | | 不建议整网 | +| check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | | fix | 前 | ["softmax"] | to_cpu | | | | | 不支持整网、不支持反向 | | fix | 前 | ["softmax"] | improve_precision | | | | | 不支持整网、不支持反向 | | 预热 | 前 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | -- Gitee From bdfd4f8c0be368a79b7c94b33e83ffdc64b9fc4e Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 20 Aug 2024 21:02:31 +0800 Subject: [PATCH 404/791] bug fix --- .../msprobe/mindspore/api_accuracy_checker/api_info.py | 1 - .../msprobe/mindspore/api_accuracy_checker/api_runner.py | 1 - 2 files changed, 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py index 657b5f9b7..47c33aeab 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py @@ -1,6 +1,5 @@ from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement from msprobe.core.common.const import Const -from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index 5473a9ae5..ed92be83c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -10,7 +10,6 @@ from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerCon from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple -from msprobe.mindspore.api_accuracy_checker.type_mapping import api_parent_module_mapping class ApiInputAggregation: -- Gitee From 2c1a3007872e3a08bd49f7a0612d1c752dc344dc Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 20 Aug 2024 21:12:58 +0800 Subject: [PATCH 405/791] add online precision compare --- .../compare/api_precision_compare.py | 20 +++ .../api_accuracy_checker/compare/compare.py | 56 +++++---- .../tensor_transport_layer/device_dispatch.py | 119 ++++++++++++++---- 3 files changed, 149 insertions(+), 46 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 3de8774d1..0e48142ee 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -289,6 +289,26 @@ def api_precision_compare(config): change_mode(config.details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) +def online_api_precision_compare(npu_data, gpu_data, rank): + result_csv_path = os.path.join("./", API_PRECISION_COMPARE_RESULT_FILE_NAME).replace(".csv", f"_rank{rank}.csv") + details_csv_path = os.path.join("./", API_PRECISION_COMPARE_DETAILS_FILE_NAME).replace(".csv", f"_rank{rank}.csv") + detail_csv_title = [ApiPrecisionCompareColumn.get_detail_csv_title()] + result_csv_title = [ApiPrecisionCompareColumn.get_result_csv_title()] + if not os.path.exists(result_csv_path): + write_csv(result_csv_title, result_csv_path) + if not os.path.exists(details_csv_path): + write_csv(detail_csv_title, details_csv_path) + config = CompareConfig("", "", result_csv_path, details_csv_path) + try: + check_csv_columns(npu_data.columns, "npu_csv") + check_csv_columns(gpu_data.columns, "gpu_csv") + analyse_csv(npu_data, gpu_data, config) + except Exception as err: + logger.error(f"Online api precision compare Error: %s" % str(err)) + change_mode(result_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) + change_mode(details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) + + def analyse_csv(npu_data, gpu_data, config): forward_status, backward_status = [], [] last_api_name, last_api_dtype = None, None diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 18387b88e..188dbed92 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -98,6 +98,32 @@ class Comparator: rtol = apis_threshold.get(api_name).get(dtype).get('rtol') return small_value_threshold, small_value_atol, rtol + @staticmethod + def get_run_ut_detail(test_result): + """get run_ut detail before write to csv, called by online run_ut""" + test_rows = [] + try: + subject_prefix = test_result[0] + fwd_result = test_result[3] + bwd_result = test_result[4] + except IndexError as e: + logger.error("List index out of bounds when writing detail CSV.") + raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR, "list index out of bounds") from e + + if isinstance(fwd_result, list): + for i, test_subject in enumerate(fwd_result): + subject = subject_prefix + ".forward.output." + str(i) + test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) + if isinstance(item, float) else item for item in test_subject] + test_rows.append([subject] + list(test_subject)) + if isinstance(bwd_result, list): + for i, test_subject in enumerate(bwd_result): + subject = subject_prefix + ".backward.output." + str(i) + test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) + if isinstance(item, float) else item for item in test_subject] + test_rows.append([subject] + list(test_subject)) + return test_rows + def write_csv_title(self): summary_test_rows = [[self.COLUMN_API_NAME, self.COLUMN_FORWARD_SUCCESS, self.COLUMN_BACKWARD_SUCCESS, "Message"]] @@ -125,27 +151,7 @@ class Comparator: write_csv(test_rows, save_path) def write_detail_csv(self, test_result): - test_rows = [] - try: - subject_prefix = test_result[0] - fwd_result = test_result[3] - bwd_result = test_result[4] - except IndexError as e: - logger.error("List index out of bounds when writing detail CSV.") - raise CompareException(CompareException.INDEX_OUT_OF_BOUNDS_ERROR, "list index out of bounds") from e - - if isinstance(fwd_result, list): - for i, test_subject in enumerate(fwd_result): - subject = subject_prefix + ".forward.output." + str(i) - test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) - if isinstance(item, float) else item for item in test_subject] - test_rows.append([subject] + list(test_subject)) - if isinstance(bwd_result, list): - for i, test_subject in enumerate(bwd_result): - subject = subject_prefix + ".backward.output." + str(i) - test_subject = ["{:.{}f}".format(item, msCheckerConfig.precision) - if isinstance(item, float) else item for item in test_subject] - test_rows.append([subject] + list(test_subject)) + test_rows = self.get_run_ut_detail(test_result) detail_save_path = self.get_path_from_rank(test_result[-1], self.detail_save_path_list, self.detail_save_path_str) @@ -155,7 +161,10 @@ class Comparator: self.write_summary_csv(args) self.write_detail_csv(args) - def compare_output(self, full_api_name, data_info): + def compare_output(self, full_api_name, data_info, is_online=False): + """Get compare result and write to result and detail csv. + is_online: bool, default False. True: called by online api precision compare, only compare without write to csv. + """ _, api_name, _ = full_api_name.split(Const.SEP) bench_output, device_output = data_info.bench_output, data_info.device_output bench_grad, device_grad = data_info.bench_grad, data_info.device_grad @@ -184,6 +193,9 @@ class Comparator: fwd_compare_alg_results, bwd_compare_alg_results, data_info.rank) + if is_online: + # get run_ut compare detail + return self.get_run_ut_detail(result_info) self.record_results(result_info) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS \ or bwd_success_status == CompareConst.SPACE diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 3d73d4adf..51076422c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -1,12 +1,20 @@ import time +import pandas as pd import torch import torch.multiprocessing as mp from msprobe.core.common.const import Const +from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import online_api_precision_compare +from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import DETAIL_TEST_ROWS, thousandth_standard_api, \ + binary_standard_api, absolute_standard_api +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import exec_api, UtDataInfo from msprobe.pytorch.common.utils import logger from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import move2target_device +# NPU vs GPU api list +CompareApi = set(absolute_standard_api) | set(binary_standard_api) | set(thousandth_standard_api) + def run_ut_process(xpu_id, compare, consumer_queue, func, config): """ When consumer_queue(shared with ConsumerDispatcher) is not empty, consume api data from consumer_queue. @@ -17,7 +25,7 @@ def run_ut_process(xpu_id, compare, consumer_queue, func, config): :param config: run_ut_config :return: """ - device = torch.device(f'cuda:{xpu_id}') + gpu_device = torch.device(f'cuda:{xpu_id}') while True: if consumer_queue.empty(): @@ -29,29 +37,92 @@ def run_ut_process(xpu_id, compare, consumer_queue, func, config): # current consumer finish return - api_full_name = api_data.name - api_data = move2target_device(api_data, device) - try: - data_info = func(api_full_name, api_data, config.backward_content) - logger.debug(f"success exec in device {api_full_name}") - is_fwd_success, is_bwd_success = compare.compare_output(api_full_name, data_info) - logger.info(f"running api_full_name {api_full_name} ut, " - f"is_fwd_success: {is_fwd_success}, " - f"is_bwd_success: {is_bwd_success}") - except Exception as err: - [api_type, api_name, _] = api_full_name.split(Const.SEP) - if "expected scalar type Long" in str(err): - logger.warning(f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API " - f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.") - elif api_type in [Const.DISTRIBUTED]: - logger.info(f"{api_full_name} is not supported for run ut. SKIP.") - else: - logger.error(f"Run {api_full_name} UT Error: {str(err)}") - - compare.write_summary_csv((api_full_name, "SKIP", "SKIP", [[str(err)]], api_data.rank)) - - finally: - torch.cuda.empty_cache() + _, api_name, _ = api_data.name.split(Const.SEP) + if api_name in CompareApi: + # NPU vs GPU + online_compare(api_data, gpu_device, compare, func, config) + else: + # NPUvsCPU vs GPUvsCPU + online_precision_compare(api_data, gpu_device, compare, func, config) + + +def online_precision_compare(api_data, device, compare, func, config): + """online run_ut for precision_compare: NPUvsCPU vs GPUvsCPU + 1. get NPUvsCPU compare result + 2. get GPUvsCPU compare result + 3. call online_api_precision_compare + """ + + api_full_name = api_data.name + [api_type, api_name, _] = api_full_name.split(Const.SEP) + npu_args, npu_kwargs, npu_out = api_data.args, api_data.kwargs, api_data.result + + if npu_kwargs.get("device"): + del npu_kwargs["device"] + + try: + # NPU vs CPU + cpu_out = exec_api(api_type, api_name, npu_args, npu_kwargs) + npu_data_info = UtDataInfo(None, None, npu_out, cpu_out, None, [], None, rank=api_data.rank) + logger.debug(f"success exec run_ut in cpu device {api_full_name}") + npu_detail = compare.compare_output(api_full_name, npu_data_info, True) + npu_data = pd.DataFrame(npu_detail, columns=DETAIL_TEST_ROWS[-1]) + + # GPU vs CPU + api_data_gpu = move2target_device(api_data, device) # args, kwargs -> gpu, result -> npu + data_info = func(api_full_name, api_data_gpu, config.backward_content) + gpu_out = data_info.bench_output + gpu_data_info = UtDataInfo(None, None, gpu_out, cpu_out, None, [], None, rank=api_data.rank) + logger.debug(f"success exec run_ut in gpu device {api_full_name}") + gpu_detail = compare.compare_output(api_full_name, gpu_data_info, True) + gpu_data = pd.DataFrame(gpu_detail, columns=DETAIL_TEST_ROWS[-1]) + + # NPUvsCPU vs GPUvsCPU + online_api_precision_compare(npu_data, gpu_data, api_data.rank) + + except Exception as err: + if "expected scalar type Long" in str(err): + logger.warning( + f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API " + f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.") + elif api_type in [Const.DISTRIBUTED]: + logger.info(f"{api_full_name} is not supported for run ut. SKIP.") + else: + logger.error(f"Run {api_full_name} UT Error: {str(err)}") + + compare.write_summary_csv((api_full_name, "SKIP", "SKIP", [[str(err)]], api_data.rank)) + + finally: + torch.cuda.empty_cache() + + +def online_compare(api_data, device, compare, func, config): + """online run_ut for compare：NPU vs GPU""" + + api_full_name = api_data.name + api_data = move2target_device(api_data, device) + try: + data_info = func(api_full_name, api_data, config.backward_content) + logger.debug(f"success exec run_ut in device {api_full_name}") + is_fwd_success, is_bwd_success = compare.compare_output(api_full_name, data_info) + logger.info(f"running api_full_name {api_full_name} ut, " + f"is_fwd_success: {is_fwd_success}, " + f"is_bwd_success: {is_bwd_success}") + except Exception as err: + [api_type, api_name, _] = api_full_name.split(Const.SEP) + if "expected scalar type Long" in str(err): + logger.warning( + f"API {api_name} not support int32 tensor in CPU, please add {api_name} to CONVERT_API " + f"'int32_to_int64' list in accuracy_tools/msprobe/core/common/const.py file.") + elif api_type in [Const.DISTRIBUTED]: + logger.info(f"{api_full_name} is not supported for run ut. SKIP.") + else: + logger.error(f"Run {api_full_name} UT Error: {str(err)}") + + compare.write_summary_csv((api_full_name, "SKIP", "SKIP", str(err), api_data.rank)) + + finally: + torch.cuda.empty_cache() class ConsumerDispatcher: -- Gitee From 7bcf738d685a66091bf486809bbc505574abecbc Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Mon, 19 Aug 2024 20:16:22 +0800 Subject: [PATCH 406/791] bugfix: raise error when lacking step data --- .../data_prepare/operator_data_prepare.py | 4 ++++ .../profiling_parser/gpu_profiling_parser.py | 2 +- .../profiling_parser/npu_profiling_parser.py | 17 +++++++++-------- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py index 2df9ae43e..cc0f923a3 100644 --- a/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py +++ b/profiler/compare_tools/compare_backend/data_prepare/operator_data_prepare.py @@ -26,6 +26,10 @@ class OperatorDataPrepare: result_data.append(node) if node.child_nodes: node_queue.extend(node.child_nodes) + if not result_data: + msg = f"There is no operator event data for step {self._specified_step_id}, " \ + "please check whether the data contains this step." + raise RuntimeError(msg) return result_data def _build_tree(self): diff --git a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py index 8dfa7cd53..c2e5a1480 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/gpu_profiling_parser.py @@ -160,7 +160,7 @@ class GPUProfilingParser(BaseProfilingParser): if event.is_kernel_except_nccl() and event.stream: kernel_stream_ids.append(event.stream) if not kernel_stream_ids: - raise RuntimeError('[ERROR] The profiling data does not contain kernel running data.') + raise RuntimeError('The profiling data does not contain kernel running data.') counter = Counter(kernel_stream_ids) return counter.most_common(1)[0][0] diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index a5c9d4675..8f2714d9c 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -102,10 +102,11 @@ class NPUProfilingParser(BaseProfilingParser): [kernel.name, kernel.duration]) if not kernels_dict: if self._step_id != Constant.VOID_STEP: - print(f"[ERROR] There is no kernel details information for step {self._step_id}, " - f"please check whether the data contains this step.") + msg = f"There is no kernel details information for step {self._step_id}," \ + " please check whether the data contains this step." + raise RuntimeError(msg) else: - print("[ERROR] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") + print("[WARNING] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") return self._result_data.update_kernel_details(kernels_dict) @@ -316,7 +317,7 @@ class NPUProfilingParser(BaseProfilingParser): try: json_data = FileReader.read_trace_file(self._info_json_path) except Exception: - print('[WARNING] Failed to read profiler_info.json.') + print('[ERROR] Failed to read profiler_info.json.') return if not isinstance(json_data, dict) or not json_data: print('[WARNING] Invalid profiler info.') @@ -338,7 +339,7 @@ class NPUProfilingParser(BaseProfilingParser): try: kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) except Exception: - print('[WARNING] Npu kernel details csv file is not available.') + print('[ERROR] Npu kernel details csv file is not available.') return if not kernel_details or kernel_details[0].is_hide_op_pmu(): self._result_data.overall_metrics.hide_op_details = True @@ -354,16 +355,16 @@ class NPUProfilingParser(BaseProfilingParser): try: memory_record = FileReader.read_csv_file(self._memory_record_path, MemoryRecordBean) except FileNotFoundError: - print('[INFO] Npu memory record csv file is not available.') + print('[WARNING] Npu memory record csv file is not available.') except Exception: - print('[WARNING] Load memory info failed.') + print('[ERROR] Load memory info failed.') else: memory_used = max([memory.total_reserved_mb for memory in memory_record]) / 1024 self._result_data.overall_metrics.set_memory_used(memory_used) def __add_overlap_analysis_time(self): if not self._overlap_analysis: - print('[ERROR] Failed to get overlap analysis data.') + print('[WARNING] Failed to get overlap analysis data.') return min_ts = sys.float_info.max max_ts = sys.float_info.min -- Gitee From 0d6a2911e0422dcdf244eeac4c5611edabc40131 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 20 Aug 2024 21:32:56 +0800 Subject: [PATCH 407/791] fix codecheck --- .../msprobe/pytorch/api_accuracy_checker/compare/compare.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 188dbed92..9c9723ed2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -99,7 +99,7 @@ class Comparator: return small_value_threshold, small_value_atol, rtol @staticmethod - def get_run_ut_detail(test_result): + def _get_run_ut_detail(test_result): """get run_ut detail before write to csv, called by online run_ut""" test_rows = [] try: @@ -151,7 +151,7 @@ class Comparator: write_csv(test_rows, save_path) def write_detail_csv(self, test_result): - test_rows = self.get_run_ut_detail(test_result) + test_rows = self._get_run_ut_detail(test_result) detail_save_path = self.get_path_from_rank(test_result[-1], self.detail_save_path_list, self.detail_save_path_str) @@ -195,7 +195,7 @@ class Comparator: data_info.rank) if is_online: # get run_ut compare detail - return self.get_run_ut_detail(result_info) + return self._get_run_ut_detail(result_info) self.record_results(result_info) return fwd_success_status == CompareConst.PASS, bwd_success_status == CompareConst.PASS \ or bwd_success_status == CompareConst.SPACE -- Gitee From 4c9cdbb2723514f9b23909614e4219911effb144 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 20 Aug 2024 22:27:11 +0800 Subject: [PATCH 408/791] alter tls --- .../api_accuracy_checker/tensor_transport_layer/client.py | 8 +++++--- .../tensor_transport_layer/ssl_config.py | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py index df7abc188..ced71179c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py @@ -137,8 +137,10 @@ class TCPClient: rank=rank, step=step) self.sequence_number += 1 - - self.send_queue.put(send_data, block=True, timeout=self.QUEUE_PENDING_TIME) + try: + self.send_queue.put(send_data, block=True, timeout=self.QUEUE_PENDING_TIME) + except Exception as e: + logger.debug(f"send_queue put send_data timeout, {str(e)}") def _send_data(self, data: TCPDataItem): self.tcp_manager.send_wrapped_data(data.raw_data, @@ -305,7 +307,7 @@ class ClientProtocol(protocol.Protocol): def connectionLost(self, reason): self.signal_exit = True self.factory.num_connections -= 1 - logger.info("Lost connection with server") + logger.info(f"Lost connection with server, reason is : {reason}") class MessageClientFactory(protocol.ClientFactory): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py index 8e29cafd2..b6e815e63 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/ssl_config.py @@ -7,4 +7,4 @@ cipher_list = ":".join([ 'ECDHE-RSA-CHACHA20-POLY1305', 'DHE-RSA-AES128-GCM-SHA256', 'DHE-RSA-AES256-GCM-SHA384' -]) +]).encode() -- Gitee From cb0ff1ec59306df4d19ded8462f47bfff647c014 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 20 Aug 2024 22:29:33 +0800 Subject: [PATCH 409/791] Solving circular dependencies --- .../compare/api_precision_compare.py | 2 +- .../run_ut/multi_run_ut.py | 5 +- .../run_ut/run_overflow_check.py | 3 +- .../api_accuracy_checker/run_ut/run_ut.py | 67 +----------------- .../run_ut/run_ut_utils.py | 69 ++++++++++++++++++- .../tensor_transport_layer/device_dispatch.py | 2 +- .../compare/test_compare.py | 2 +- .../run_ut/test_run_ut.py | 3 +- 8 files changed, 80 insertions(+), 73 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 0e48142ee..5dbbd122a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -15,7 +15,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECI BINARY_COMPARE_UNSUPPORT_LIST, ULP_COMPARE_SUPPORT_LIST, convert_str_to_float, CompareMessage, is_inf_or_nan, \ check_inf_or_nan from msprobe.pytorch.api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validated_result_csv_path from msprobe.core.common.file_check import FileChecker, change_mode, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index 64669e56b..c83fb5fb8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -9,8 +9,9 @@ import threading from collections import namedtuple from itertools import cycle from tqdm import tqdm -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, get_validated_result_csv_path, \ - get_validated_details_csv_path, preprocess_forward_content +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, preprocess_forward_content +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validated_result_csv_path, \ + get_validated_details_csv_path from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.common import parse_json_info_forward_backward from msprobe.core.common.file_check import FileChecker, check_file_suffix, check_link, FileOpen, \ diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index ecb441716..98e5eaab3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -10,7 +10,8 @@ else: is_gpu = False import torch from tqdm import tqdm -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import exec_api, generate_device_params, get_api_info +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import generate_device_params, get_api_info +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import exec_api from msprobe.core.common.utils import get_json_contents from msprobe.core.common.file_check import check_link from msprobe.pytorch.common.log import logger diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index a03b9a1f8..8930e853a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -1,7 +1,6 @@ import argparse import os import csv -import re import sys import time import gc @@ -18,21 +17,17 @@ else: import torch from tqdm import tqdm -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Message, hf_32_standard_api +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Message, hf_32_standard_api, UtDataInfo, \ + get_validated_result_csv_path, get_validated_details_csv_path, exec_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ initialize_save_path, UtDataProcessor from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn -from msprobe.pytorch.hook_module.wrap_tensor import TensorOPTemplate -from msprobe.pytorch.hook_module.wrap_functional import FunctionalOPTemplate -from msprobe.pytorch.hook_module.wrap_torch import TorchOPTemplate -from msprobe.pytorch.hook_module.wrap_npu_custom import NpuOPTemplate -from msprobe.pytorch.hook_module.wrap_aten import AtenOPTemplate from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward from msprobe.core.common.file_check import FileOpen, FileChecker, \ - change_mode, check_file_suffix, check_link, check_path_before_create, create_directory + change_mode, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import get_json_contents from msprobe.pytorch.pt_config import parse_json_config @@ -77,25 +72,6 @@ tqdm_params = { } -def exec_api(api_type, api_name, args, kwargs): - if api_type == "Functional": - functional_api = FunctionalOPTemplate(api_name, str, False) - out = functional_api.forward(*args, **kwargs) - if api_type == "Tensor": - tensor_api = TensorOPTemplate(api_name, str, False) - out = tensor_api.forward(*args, **kwargs) - if api_type == "Torch": - torch_api = TorchOPTemplate(api_name, str, False) - out = torch_api.forward(*args, **kwargs) - if api_type == "Aten": - torch_api = AtenOPTemplate(api_name, None, False) - out = torch_api.forward(*args, **kwargs) - if api_type == "NPU": - torch_api = NpuOPTemplate(api_name, None, False) - out = torch_api.forward(*args, **kwargs) - return out - - def deal_detach(arg, to_detach=True): return arg.detach() if to_detach else arg @@ -433,30 +409,6 @@ def initialize_save_error_data(error_data_path): return error_data_path -def get_validated_result_csv_path(result_csv_path, mode): - if mode not in ['result', 'detail']: - raise ValueError("The csv mode must be result or detail") - result_csv_path_checker = FileChecker(result_csv_path, FileCheckConst.FILE, ability=FileCheckConst.READ_WRITE_ABLE, - file_type=FileCheckConst.CSV_SUFFIX) - validated_result_csv_path = result_csv_path_checker.common_check() - if mode == 'result': - result_csv_name = os.path.basename(validated_result_csv_path) - pattern = r"^accuracy_checking_result_\d{14}\.csv$" - if not re.match(pattern, result_csv_name): - raise ValueError("When continue run ut, please do not modify the result csv name.") - return validated_result_csv_path - - -def get_validated_details_csv_path(validated_result_csv_path): - result_csv_name = os.path.basename(validated_result_csv_path) - details_csv_name = result_csv_name.replace('result', 'details') - details_csv_path = os.path.join(os.path.dirname(validated_result_csv_path), details_csv_name) - details_csv_path_checker = FileChecker(details_csv_path, FileCheckConst.FILE, - ability=FileCheckConst.READ_WRITE_ABLE, file_type=FileCheckConst.CSV_SUFFIX) - validated_details_csv_path = details_csv_path_checker.common_check() - return validated_details_csv_path - - def init_attl(config): """config: OnlineConfig""" attl = ATTL('gpu', ATTLConfig(is_benchmark_device=True, @@ -624,19 +576,6 @@ def run_ut_command(args): run_ut(run_ut_config) -class UtDataInfo: - def __init__(self, bench_grad, device_grad, device_output, bench_output, grad_in, in_fwd_data_list, - backward_message, rank=0): - self.bench_grad = bench_grad - self.device_grad = device_grad - self.device_output = device_output - self.bench_output = bench_output - self.grad_in = grad_in - self.in_fwd_data_list = in_fwd_data_list - self.backward_message = backward_message - self.rank = rank - - if __name__ == '__main__': _run_ut() logger.info("UT task completed.") diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index d78642f21..84db853b9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -1,7 +1,74 @@ +import os +import re + +from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.file_check import FileChecker +from msprobe.pytorch.hook_module.wrap_aten import AtenOPTemplate +from msprobe.pytorch.hook_module.wrap_functional import FunctionalOPTemplate +from msprobe.pytorch.hook_module.wrap_npu_custom import NpuOPTemplate +from msprobe.pytorch.hook_module.wrap_tensor import TensorOPTemplate +from msprobe.pytorch.hook_module.wrap_torch import TorchOPTemplate + hf_32_standard_api = ["conv1d", "conv2d"] class Backward_Message: MULTIPLE_BACKWARD_MESSAGE = "Multiple backward is not supported." UNSUPPORT_BACKWARD_MESSAGE = "function with out=... arguments don't support automatic differentiation, skip backward." - NO_BACKWARD_RESULT_MESSAGE = "function backward result is None, skip backward." \ No newline at end of file + NO_BACKWARD_RESULT_MESSAGE = "function backward result is None, skip backward." + + +class UtDataInfo: + def __init__(self, bench_grad, device_grad, device_output, bench_output, grad_in, in_fwd_data_list, + backward_message, rank=0): + self.bench_grad = bench_grad + self.device_grad = device_grad + self.device_output = device_output + self.bench_output = bench_output + self.grad_in = grad_in + self.in_fwd_data_list = in_fwd_data_list + self.backward_message = backward_message + self.rank = rank + + +def get_validated_result_csv_path(result_csv_path, mode): + if mode not in ['result', 'detail']: + raise ValueError("The csv mode must be result or detail") + result_csv_path_checker = FileChecker(result_csv_path, FileCheckConst.FILE, ability=FileCheckConst.READ_WRITE_ABLE, + file_type=FileCheckConst.CSV_SUFFIX) + validated_result_csv_path = result_csv_path_checker.common_check() + if mode == 'result': + result_csv_name = os.path.basename(validated_result_csv_path) + pattern = r"^accuracy_checking_result_\d{14}\.csv$" + if not re.match(pattern, result_csv_name): + raise ValueError("When continue run ut, please do not modify the result csv name.") + return validated_result_csv_path + + +def get_validated_details_csv_path(validated_result_csv_path): + result_csv_name = os.path.basename(validated_result_csv_path) + details_csv_name = result_csv_name.replace('result', 'details') + details_csv_path = os.path.join(os.path.dirname(validated_result_csv_path), details_csv_name) + details_csv_path_checker = FileChecker(details_csv_path, FileCheckConst.FILE, + ability=FileCheckConst.READ_WRITE_ABLE, file_type=FileCheckConst.CSV_SUFFIX) + validated_details_csv_path = details_csv_path_checker.common_check() + return validated_details_csv_path + + +def exec_api(api_type, api_name, args, kwargs): + if api_type == "Functional": + functional_api = FunctionalOPTemplate(api_name, str, False) + out = functional_api.forward(*args, **kwargs) + if api_type == "Tensor": + tensor_api = TensorOPTemplate(api_name, str, False) + out = tensor_api.forward(*args, **kwargs) + if api_type == "Torch": + torch_api = TorchOPTemplate(api_name, str, False) + out = torch_api.forward(*args, **kwargs) + if api_type == "Aten": + torch_api = AtenOPTemplate(api_name, None, False) + out = torch_api.forward(*args, **kwargs) + if api_type == "NPU": + torch_api = NpuOPTemplate(api_name, None, False) + out = torch_api.forward(*args, **kwargs) + return out diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 51076422c..ea8f83862 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -8,7 +8,7 @@ from msprobe.core.common.const import Const from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import online_api_precision_compare from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import DETAIL_TEST_ROWS, thousandth_standard_api, \ binary_standard_api, absolute_standard_api -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import exec_api, UtDataInfo +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import UtDataInfo, exec_api from msprobe.pytorch.common.utils import logger from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import move2target_device diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/compare/test_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/compare/test_compare.py index e1e6d51de..825806baf 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/compare/test_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/compare/test_compare.py @@ -9,7 +9,7 @@ import torch.nn.functional from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import UtDataInfo +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import UtDataInfo current_time = time.strftime("%Y%m%d%H%M%S") RESULT_FILE_NAME = "accuracy_checking_result_" + current_time + ".csv" diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py index 5be41d78f..3ade53454 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py @@ -1,11 +1,10 @@ # coding=utf-8 -import os import copy import unittest -import torch from unittest.mock import patch, DEFAULT from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import * from msprobe.core.common.utils import get_json_contents +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import UtDataInfo, exec_api base_dir = os.path.dirname(os.path.realpath(__file__)) forward_file = os.path.join(base_dir, "forward.json") -- Gitee From ff7d8bc25651df6854862c32ede40d8f44cd1496 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Wed, 21 Aug 2024 09:11:06 +0800 Subject: [PATCH 410/791] test --- ...47\350\203\275\345\237\272\347\272\277.md" | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 "\346\227\240\346\240\207\346\235\206\346\200\247\350\203\275\345\237\272\347\272\277.md" diff --git "a/\346\227\240\346\240\207\346\235\206\346\200\247\350\203\275\345\237\272\347\272\277.md" "b/\346\227\240\346\240\207\346\235\206\346\200\247\350\203\275\345\237\272\347\272\277.md" new file mode 100644 index 000000000..981d00b34 --- /dev/null +++ "b/\346\227\240\346\240\207\346\235\206\346\200\247\350\203\275\345\237\272\347\272\277.md" @@ -0,0 +1,145 @@ +# 无标杆工具应用场景和性能基线报告 + +## 环境信息 + +NPU：Atlas A2 训练系列产品 + +CPU： + +![输入图片说明](img/cpu_info.png) + +Torch：2.1.0 + +CANN：8.0.T5 + +除上述环境信息影响性能外，API的数量、种类以及Shape都会对性能产生影响，因此本次选取不同场景网络和不同算子进行测试。 + +## 模型信息和性能基线 + +大模型在使用msprobe工具dump数据时，建议先简化模型层数，减少dump数据量。 + +以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 + + +### LLaMA2-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink +其中，softmax算子为FLOAT32，输入输出均为2G大小，为模型最大显存开销的API。 +在该模型下、对无标杆工具处理模式、插装范围、扰动方式组合下性能和显存基线进行覆盖。 + +性能基线报告 +其中耗时为训练10步，去除第一步耗时所得的平均每步耗时。 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.24 | 13.69 | 1 | 1 | 混精模式基线 | +| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1.66 | softmax本身为高精度，跳过 | +| check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | +| check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | +| check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | +| check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | +| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | +| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.375 | 1 | | +| check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | +| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | +| check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | +| check | 反 | ["softmax"] | add_noise | | | | | 不建议整网 | +| check | 反 | ["softmax"] | bit_noise | | | | | 不建议整网 | +| check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | +| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | +| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | +| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | +| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | +| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | +| fix | 前 | ["softmax"] | to_cpu | | | | | 不支持整网、不支持反向 | +| fix | 前 | ["softmax"] | improve_precision | | | | | 不支持整网、不支持反向 | +| 预热 | 前 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | +| 预热 | 反 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | + +### Aquila2-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | | | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | | | | | | +| check | 反 | [] | add_noise | | | | | | +| fix | 前 | [] | improve_precision | | | | | | + +### Baichuan-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | | | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | | | | | | +| check | 反 | [] | add_noise | | | | | | +| fix | 前 | [] | improve_precision | | | | | | + +### Bloom-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | | | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | | | | | | +| check | 反 | [] | add_noise | | | | | | +| fix | 前 | [] | improve_precision | | | | | | + +### Interlm-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | | | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | | | | | | +| check | 反 | [] | add_noise | | | | | | +| fix | 前 | [] | improve_precision | | | | | | + +### Qwen-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | | | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | | | | | | +| check | 反 | [] | add_noise | | | | | | +| fix | 前 | [] | improve_precision | | | | | | + +### Gemma-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | | | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | | | | | | +| check | 反 | [] | add_noise | | | | | | +| fix | 前 | [] | improve_precision | | | | | | + +### ResNet50-Cifar +1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch。 +主要算子为conv2d，每个step有51个, 因此对conv2d进行检测。 +CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step信息和反向信息，且该框架下性能暂不保证）。 +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.09 | 7.63 | 1 | 1 | 基线 | +| check | 前 | ["conv2d"] | improve_precision | 0.889 | 7.94 | 9.81 | 1.04 | | +| fix | 前 | ["conv2d"] | improve_precision | 0.328 | 7.47 | 3.64 | 0.91 | | +| fix | 前 | ["conv2d"] | to_cpu | 12.23 | 7.47 | 135.88 | 0.91 | | + +### OpenSora1.0 + +1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | | | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | | | | | | +| check | 反 | [] | add_noise | | | | | | +| fix | 前 | [] | improve_precision | | | | | | \ No newline at end of file -- Gitee From 4270d9040bf83f9c069ed87e3b403e83800eec60 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Wed, 21 Aug 2024 10:13:15 +0800 Subject: [PATCH 411/791] api_accuracy_checker class half --- .../api_accuracy_checker.py | 66 +++++++++++++++++++ .../mindspore/api_accuracy_checker/const.py | 8 ++- 2 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py new file mode 100644 index 000000000..6dbb7145c --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -0,0 +1,66 @@ +from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.const import Const +from msprobe.core.common.log import logger +from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo +from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation +from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst +from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context +import json + + +class ApiAccuracyChecker: + def __init__(self): + self.api_infos = dict() + self.results = None + + def parse(self, api_info_path): + with FileOpen(api_info_path, "r") as f: + api_info_dict = json.load(f) + + # init global context + task = check_and_get_from_json_dict(api_info_dict, MsApiAccuracyCheckerConst.TASK_FIELD, + "task field in api_info.json",accepted_type=str, + accepted_value=(MsApiAccuracyCheckerConst.STATISTICS_TASK, + MsApiAccuracyCheckerConst.TENSOR_TASK)) + is_constructed = task == MsApiAccuracyCheckerConst.STATISTICS_TASK + if not is_constructed: + dump_data_dir = check_and_get_from_json_dict(api_info_dict, MsApiAccuracyCheckerConst.DUMP_DATA_DIR_FIELD, + "dump_data_dir field in api_info.json", accepted_type=str) + else: + dump_data_dir = "" + global_context.init(is_constructed, dump_data_dir) + + api_info_data = check_and_get_from_json_dict(api_info_dict, MsApiAccuracyCheckerConst.DATA_FIELD, + "data field in api_info.json", accepted_type=dict) + for api_name, api_info in api_info_data.items(): + forbackward_str = api_name.split(".")[-1] + if forbackward_str not in (Const.FORWARD, Const.BACKWARD): + logger.warning(f"api: {api_name} is not recognized as forward api or backward api, skip this.") + api_name = Const.SEP.join(api_name.split(".")[:-1]) # www.xxx.yyy.zzz --> www.xxx.yyy + if api_name not in self.api_infos: + self.api_infos[api_name] = ApiInfo(api_name) + + if forbackward_str == Const.FORWARD: + self.api_infos[api_name].load_forward_info(api_info) + else: + self.api_infos[api_name].load_backward_info(api_info) + + + def run_and_compare(self): + for api_name_str, api_info in self.api_infos: + if not api_info.check_forward_info(): + logger.warning(f"api: {api_name_str} is lack of forward infomation skip checking") + continue + # first run forapi_infoward and compare + inputs = api_info.get_compute_element_list(Const.FORWARD, Const.INPUT) + kwargs =api_info.get_kwargs() + + if global_context.get_is_constructed(): + tested + + def record(self, compare_result_collection): + pass + + + def to_csv(self): + pass \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py index 74079aafb..69b52ea61 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -1,6 +1,10 @@ class MsApiAccuracyCheckerConst: - #api_runner + # api_info field MINT = "Mint" MINT_FUNCTIONAL = "MintFunctional" - + TASK_FIELD = "task" + STATISTICS_TASK = "statistics" + TENSOR_TASK = "tensor" + DUMP_DATA_DIR_FIELD = "dump_data_dir" + DATA_FIELD = "data" \ No newline at end of file -- Gitee From 4e975d5c53ef7e00b9eebe44cf6d18b3b2353248 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Wed, 21 Aug 2024 10:14:10 +0800 Subject: [PATCH 412/791] part2 code view --- .../mindspore/api_accuracy_checker/base_compare_algorithm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index 8f7f3d2f4..ced2371f0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -79,9 +79,9 @@ class BaseCompareAlgorithm(ABC): tested_parameter = tested_compute_element.get_parameter() bench_is_tensor = isinstance(bench_parameter, (mindspore.Tensor, torch.Tensor)) - tested_is_tensosr = isinstance(tested_parameter, (mindspore.Tensor, torch.Tensor)) + tested_is_tensor = isinstance(tested_parameter, (mindspore.Tensor, torch.Tensor)) shape_same = bench_compute_element.get_shape() == tested_compute_element.get_shape() - return bench_is_tensor and tested_is_tensosr and shape_same + return bench_is_tensor and tested_is_tensor and shape_same @abstractmethod def check_validity(self, bench_compute_element, tested_compute_element): -- Gitee From c000ccf4e227d4e43fe14fab8f91cfd2378cb904 Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Tue, 20 Aug 2024 17:29:43 +0800 Subject: [PATCH 413/791] adapter advisor --- .../compare_backend/comparison_generator.py | 2 + .../interface/compare_interface.py | 47 +++++++++++++++++++ .../compare_backend/utils/compare_args.py | 2 + .../compare_backend/utils/constant.py | 1 + .../compare_interface/comparison_interface.py | 9 ++++ 5 files changed, 61 insertions(+) create mode 100644 profiler/compare_tools/compare_backend/interface/compare_interface.py diff --git a/profiler/compare_tools/compare_backend/comparison_generator.py b/profiler/compare_tools/compare_backend/comparison_generator.py index bfbc1bb7b..356917215 100644 --- a/profiler/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/compare_tools/compare_backend/comparison_generator.py @@ -1,6 +1,7 @@ from compare_backend.generator.detail_performance_generator import DetailPerformanceGenerator from compare_backend.generator.overall_performance_generator import OverallPerformanceGenerator from compare_backend.interface.overall_interface import OverallInterface +from compare_backend.interface.compare_interface import CompareInterface from compare_backend.profiling_parser.gpu_profiling_parser import GPUProfilingParser from compare_backend.profiling_parser.npu_profiling_parser import NPUProfilingParser from compare_backend.utils.constant import Constant @@ -56,6 +57,7 @@ class ComparisonGenerator: interface = self.INTERFACE_DICT.get(compare_type) if interface: return interface(self._data_dict).run() + return CompareInterface(self._data_dict, self._args_manager).run() except NotImplementedError as e: print(f"[ERROR] {e}") except RuntimeError as e: diff --git a/profiler/compare_tools/compare_backend/interface/compare_interface.py b/profiler/compare_tools/compare_backend/interface/compare_interface.py new file mode 100644 index 000000000..67c5db67f --- /dev/null +++ b/profiler/compare_tools/compare_backend/interface/compare_interface.py @@ -0,0 +1,47 @@ +from compare_backend.utils.constant import Constant + +from compare_backend.comparator.operator_comparator import OperatorComparator +from compare_backend.comparator.api_compare_comparator import ApiCompareComparator +from compare_backend.comparator.kernel_compare_comparator import KernelCompareComparator +from compare_backend.compare_bean.operator_compare_bean import OperatorCompareBean +from compare_backend.compare_bean.api_compare_bean import ApiCompareBean +from compare_backend.compare_bean.kernel_compare_bean import KernelCompareBean +from compare_backend.data_prepare.operator_data_prepare import OperatorDataPrepare +from compare_backend.utils.constant import Constant +from compare_backend.data_prepare.sequence_pre_matching import SequencePreMatching + +class CompareInterface: + def __init__(self, data_dict: dict, args_manager: any): + self._data_dict = data_dict + self._args_manager = args_manager + + def run(self): + if self._args_manager.enable_kernel_compare: + kernel_compare_result = { + Constant.BASE_DATA: self._data_dict.get(Constant.BASE_DATA).kernel_details, + Constant.COMPARISON_DATA: self._data_dict.get(Constant.COMPARISON_DATA).kernel_details} + return KernelCompareComparator(kernel_compare_result, KernelCompareBean).generate_data() + + base_op_prepare = OperatorDataPrepare(self._data_dict.get(Constant.BASE_DATA), + self._args_manager.base_step) + comparison_op_prepare = OperatorDataPrepare(self._data_dict.get(Constant.COMPARISON_DATA), + self._args_manager.comparison_step) + + if self._args_manager.enable_api_compare: + api_compare_result = { + Constant.BASE_DATA: base_op_prepare.get_all_layer_ops(), + Constant.COMPARISON_DATA: comparison_op_prepare.get_all_layer_ops()} + return ApiCompareComparator(api_compare_result, ApiCompareBean).generate_data() + + if self._args_manager.enable_operator_compare: + op_compare_result = self._operator_match(base_op_prepare.get_top_layer_ops(), + comparison_op_prepare.get_top_layer_ops()) + return OperatorComparator(op_compare_result, OperatorCompareBean).generate_data() + return {} + + def _operator_match(self, base_ops, comparison_ops): + base_bwd_tid = self._data_dict.get(Constant.BASE_DATA).bwd_tid + comparison_bwd_tid = self._data_dict.get(Constant.COMPARISON_DATA).bwd_tid + return SequencePreMatching(self._args_manager.args, base_bwd_tid, comparison_bwd_tid).run(SequencePreMatching.OP_TYPE, + base_ops, comparison_ops) + diff --git a/profiler/compare_tools/compare_backend/utils/compare_args.py b/profiler/compare_tools/compare_backend/utils/compare_args.py index 36199b5b0..b3171711e 100644 --- a/profiler/compare_tools/compare_backend/utils/compare_args.py +++ b/profiler/compare_tools/compare_backend/utils/compare_args.py @@ -8,6 +8,7 @@ class Args: enable_communication_compare: bool = False, enable_api_compare: bool = False, enable_kernel_compare: bool = False, + disable_details: bool = False, output_path: str = "", max_kernel_num: int = None, op_name_map: dict = {}, @@ -23,6 +24,7 @@ class Args: self.enable_communication_compare = enable_communication_compare self.enable_api_compare = enable_api_compare self.enable_kernel_compare = enable_kernel_compare + self.disable_details = disable_details self.output_path = output_path self.max_kernel_num = max_kernel_num self.op_name_map = op_name_map diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index 8dc503763..51c7ce737 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -46,6 +46,7 @@ class Constant(object): MEMORY_COMPARE = "MemoryCompare" API_COMPARE = "ApiCompare" KERNEL_COMPARE = "KernelCompare" + # sheet name OPERATOR_SHEET = "OperatorCompare" MEMORY_SHEET = "MemoryCompare" diff --git a/profiler/compare_tools/compare_interface/comparison_interface.py b/profiler/compare_tools/compare_interface/comparison_interface.py index 68bbcc026..9f593e19e 100644 --- a/profiler/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/compare_tools/compare_interface/comparison_interface.py @@ -24,6 +24,15 @@ class ComparisonInterface: def compare(self, compare_type: str) -> dict: if compare_type == Constant.OVERALL_COMPARE: self._args.enable_profiling_compare = True + elif compare_type == Constant.KERNEL_COMPARE: + self._args.enable_kernel_compare = True + elif compare_type == Constant.API_COMPARE: + self._args.enable_api_compare = True + elif compare_type == Constant.OPERATOR_COMPARE: + self._args.enable_operator_compare = True + else: + print('[ERROR] Invalid compare_type value: {compare_type} which not supported.') + return {} return ComparisonGenerator(self._args).run_interface(compare_type) def disaggregate_perf(self, compare_type: str) -> dict: -- Gitee From d93918c4ea7c0ac2c1a15d8ff2c7d43fef255d14 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Wed, 21 Aug 2024 05:58:28 +0000 Subject: [PATCH 414/791] update debug/accuracy_tools/msprobe/pytorch/common/utils.py. Signed-off-by: jiangchangting1 --- .../accuracy_tools/msprobe/pytorch/common/utils.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 378ff1a3d..05c075ebf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -36,13 +36,12 @@ except ImportError: else: is_gpu = False -torch_without_guard_version_list = ['2.1', '2.2'] -for version in torch_without_guard_version_list: - if torch.__version__.startswith(version): - torch_without_guard_version = True - break - else: - torch_without_guard_version = False + +if torch.__version__ >= '2.1': + torch_without_guard_version = True +else: + torch_without_guard_version = False + if not is_gpu and not torch_without_guard_version: from torch_npu.utils.device_guard import torch_device_guard as torch_npu_device_guard -- Gitee From 8f67ce8624063b3927882e864eac0bf7d1bc6cf4 Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Wed, 21 Aug 2024 07:00:43 +0000 Subject: [PATCH 415/791] update debug/accuracy_tools/msprobe/pytorch/common/utils.py. Signed-off-by: jiangchangting1 --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 05c075ebf..a8e05402f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -37,10 +37,7 @@ else: is_gpu = False -if torch.__version__ >= '2.1': - torch_without_guard_version = True -else: - torch_without_guard_version = False +torch_without_guard_version = torch.__version__ >= '2.1' if not is_gpu and not torch_without_guard_version: -- Gitee From af6c9c7d901774941c0aed613c9b9f68479ed0b4 Mon Sep 17 00:00:00 2001 From: zyy Date: Wed, 21 Aug 2024 15:22:46 +0800 Subject: [PATCH 416/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9cluster=5Fanalysis=5F?= =?UTF-8?q?output=E4=B8=BAcluster=5Fanalysis=5Foutput=5Fpath?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/analysis/communication_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/analysis/communication_analysis.py b/profiler/cluster_analyse/analysis/communication_analysis.py index 40af81084..5d642cddd 100644 --- a/profiler/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/cluster_analyse/analysis/communication_analysis.py @@ -30,7 +30,7 @@ class CommunicationAnalysis(BaseAnalysis): def dump_db(self): res_comm_time, res_comm_bandwidth = self.adapter.transfer_comm_from_json_to_db(self.comm_ops_struct) - output_path = os.path.join(self.cluster_analysis_output, Constant.CLUSTER_ANALYSIS_OUTPUT) + output_path = os.path.join(self.cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) DBManager.create_tables(result_db, self.COMMUNICATION_TIME_TABLE, self.COMMUNICATION_BANDWIDTH_TABLE) conn, cursor = DBManager.create_connect_db(result_db) -- Gitee From 77016f65615ac7b6b459ea8da23d020c52e064e6 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 21 Aug 2024 15:41:06 +0800 Subject: [PATCH 417/791] clean code --- .../compare/api_precision_compare.py | 28 +++++++++++++++---- .../tensor_transport_layer/device_dispatch.py | 24 ++++++++++------ 2 files changed, 38 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 5dbbd122a..0737c30b7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -19,7 +19,7 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validat from msprobe.core.common.file_check import FileChecker, change_mode, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.const import CompareConst, FileCheckConst, Const CompareConfig = namedtuple('CompareConfig', ['npu_csv_path', 'gpu_csv_path', 'result_csv_path', 'details_csv_path']) BenchmarkInf_Nan_Consistency = namedtuple('BenchmarkInf_Nan_Consistency', ['small_value_inf_nan_consistency', @@ -289,9 +289,10 @@ def api_precision_compare(config): change_mode(config.details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) -def online_api_precision_compare(npu_data, gpu_data, rank): - result_csv_path = os.path.join("./", API_PRECISION_COMPARE_RESULT_FILE_NAME).replace(".csv", f"_rank{rank}.csv") - details_csv_path = os.path.join("./", API_PRECISION_COMPARE_DETAILS_FILE_NAME).replace(".csv", f"_rank{rank}.csv") +def online_api_precision_compare(online_config): + rank = online_config.rank + result_csv_path = os.path.join("./", online_config.result_csv_path).replace(".csv", f"_rank{rank}.csv") + details_csv_path = os.path.join("./", online_config.details_csv_path).replace(".csv", f"_rank{rank}.csv") detail_csv_title = [ApiPrecisionCompareColumn.get_detail_csv_title()] result_csv_title = [ApiPrecisionCompareColumn.get_result_csv_title()] if not os.path.exists(result_csv_path): @@ -300,6 +301,7 @@ def online_api_precision_compare(npu_data, gpu_data, rank): write_csv(detail_csv_title, details_csv_path) config = CompareConfig("", "", result_csv_path, details_csv_path) try: + npu_data, gpu_data = online_config.npu_data, online_config.gpu_data check_csv_columns(npu_data.columns, "npu_csv") check_csv_columns(gpu_data.columns, "gpu_csv") analyse_csv(npu_data, gpu_data, config) @@ -311,13 +313,14 @@ def online_api_precision_compare(npu_data, gpu_data, rank): def analyse_csv(npu_data, gpu_data, config): forward_status, backward_status = [], [] - last_api_name, last_api_dtype = None, None + last_api_name, last_api_dtype, last_api_full_name = None, None, None for _, row_npu in npu_data.iterrows(): message = '' compare_column = ApiPrecisionOutputColumn() full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status] - _, api_name, _, direction_status, _, _ = full_api_name_with_direction_status.split(".") + api_type, api_name, api_nums, direction_status, _, _ = full_api_name_with_direction_status.split(Const.SEP) + api_full_name = Const.SEP.join([api_type, api_name, api_nums]) if row_gpu.empty: logger.warning(f'This API : {full_api_name_with_direction_status} does not exist in the GPU data.') continue @@ -355,6 +358,7 @@ def analyse_csv(npu_data, gpu_data, config): if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) + print_test_success(api_full_name, "skip", "skip") forward_status, backward_status = [], [] message = '' else: @@ -362,11 +366,13 @@ def analyse_csv(npu_data, gpu_data, config): backward_result = get_api_checker_result(backward_status) message += CompareMessage.get(last_api_name, "") if forward_result == CompareConst.ERROR else "" write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path) + print_test_success(api_full_name, forward_result, backward_result) forward_status, backward_status = [], [] message = '' is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in API_PRECISION_COMPARE_UNSUPPORT_LIST last_api_name = api_name + last_api_full_name = api_full_name last_api_dtype = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] if not is_supported: @@ -383,11 +389,21 @@ def analyse_csv(npu_data, gpu_data, config): if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) + print_test_success(last_api_full_name, "skip", "skip") else: forward_result = get_api_checker_result(forward_status) backward_result = get_api_checker_result(backward_status) message += CompareMessage.get(last_api_name, "") if forward_result == CompareConst.ERROR else "" write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path) + print_test_success(last_api_full_name, forward_result, backward_result) + + +def print_test_success(api_full_name, forward_result, backward_result): + is_fwd_success = (forward_result == CompareConst.PASS) + is_bwd_success = (backward_result == CompareConst.PASS or backward_result == CompareConst.SPACE) + logger.info(f"running api_full_name {api_full_name} compare, " + f"is_fwd_success: {is_fwd_success}, " + f"is_bwd_success: {is_bwd_success}") def check_error_rate(npu_error_rate): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index ea8f83862..7d03066f6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -1,4 +1,5 @@ import time +from collections import namedtuple import pandas as pd import torch @@ -9,12 +10,19 @@ from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import o from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import DETAIL_TEST_ROWS, thousandth_standard_api, \ binary_standard_api, absolute_standard_api from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import UtDataInfo, exec_api -from msprobe.pytorch.common.utils import logger +from msprobe.pytorch.common.log import logger from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import move2target_device # NPU vs GPU api list CompareApi = set(absolute_standard_api) | set(binary_standard_api) | set(thousandth_standard_api) +current_time = time.strftime("%Y%m%d%H%M%S") +ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + ".csv" +ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + ".csv" + +OnlineApiPrecisionCompareConfig = namedtuple('OnlineApiPrecisionCompareConfig', + ['npu_data', 'gpu_data', 'rank', 'result_csv_path', 'details_csv_path']) + def run_ut_process(xpu_id, compare, consumer_queue, func, config): """ When consumer_queue(shared with ConsumerDispatcher) is not empty, consume api data from consumer_queue. @@ -64,7 +72,6 @@ def online_precision_compare(api_data, device, compare, func, config): # NPU vs CPU cpu_out = exec_api(api_type, api_name, npu_args, npu_kwargs) npu_data_info = UtDataInfo(None, None, npu_out, cpu_out, None, [], None, rank=api_data.rank) - logger.debug(f"success exec run_ut in cpu device {api_full_name}") npu_detail = compare.compare_output(api_full_name, npu_data_info, True) npu_data = pd.DataFrame(npu_detail, columns=DETAIL_TEST_ROWS[-1]) @@ -73,12 +80,14 @@ def online_precision_compare(api_data, device, compare, func, config): data_info = func(api_full_name, api_data_gpu, config.backward_content) gpu_out = data_info.bench_output gpu_data_info = UtDataInfo(None, None, gpu_out, cpu_out, None, [], None, rank=api_data.rank) - logger.debug(f"success exec run_ut in gpu device {api_full_name}") gpu_detail = compare.compare_output(api_full_name, gpu_data_info, True) gpu_data = pd.DataFrame(gpu_detail, columns=DETAIL_TEST_ROWS[-1]) # NPUvsCPU vs GPUvsCPU - online_api_precision_compare(npu_data, gpu_data, api_data.rank) + precision_compare_config = OnlineApiPrecisionCompareConfig(npu_data, gpu_data, api_data.rank, + ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME, + ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME) + online_api_precision_compare(precision_compare_config) except Exception as err: if "expected scalar type Long" in str(err): @@ -103,7 +112,6 @@ def online_compare(api_data, device, compare, func, config): api_data = move2target_device(api_data, device) try: data_info = func(api_full_name, api_data, config.backward_content) - logger.debug(f"success exec run_ut in device {api_full_name}") is_fwd_success, is_bwd_success = compare.compare_output(api_full_name, data_info) logger.info(f"running api_full_name {api_full_name} ut, " f"is_fwd_success: {is_fwd_success}, " @@ -119,7 +127,7 @@ def online_compare(api_data, device, compare, func, config): else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") - compare.write_summary_csv((api_full_name, "SKIP", "SKIP", str(err), api_data.rank)) + compare.write_summary_csv((api_full_name, "SKIP", "SKIP", [[str(err)]], api_data.rank)) finally: torch.cuda.empty_cache() @@ -159,6 +167,8 @@ class ConsumerDispatcher: for p in self.processes: p.join() logger.info("Successfully stop unittest process.") + logger.info(f"Api_precision_compare task result will be saved in {ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME}".replace(".csv", "_rank*.csv")) + logger.info(f"Api_precision_compare task details will be saved in {ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME}".replace(".csv", "_rank*.csv")) def update_consume_queue(self, api_data): while True: @@ -166,9 +176,7 @@ class ConsumerDispatcher: if index != -1: q = self.queues[index] q.put(api_data) - logger.debug(f"将{api_data.name}调度给第{index}个GPU") break - logger.debug("所有的UT队列都已满, 阻塞中") time.sleep(0.1) def _choose_max_empty_site_strategy(self): -- Gitee From 76e355df6002207a236009ca1b0506f5b5d15dee Mon Sep 17 00:00:00 2001 From: CSNIU Date: Wed, 21 Aug 2024 15:52:56 +0800 Subject: [PATCH 418/791] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B7=A8=E6=A1=86?= =?UTF-8?q?=E6=9E=B6api=E7=B2=BE=E5=BA=A6=E6=AF=94=E5=AF=B9=E5=8A=9F?= =?UTF-8?q?=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/compare/compare_cli.py | 1 + .../msprobe/core/compare/utils.py | 2 ++ .../msprobe/mindspore/compare/ms_compare.py | 30 +++++++++++++++++-- 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index 37383e532..ebaee114d 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -31,6 +31,7 @@ def compare_cli(args): "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match, "cell_mapping": args.cell_mapping, + "api_mapping": args.api_mapping, } ms_compare(input_param, args.output_path, **kwargs) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index b4bdc0e8d..ad985865c 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -420,6 +420,8 @@ def _compare_parser(parser): help=" Whether to perform a fuzzy match on the api name.", required=False) parser.add_argument("-cm", "--cell_mapping", dest="cell_mapping", type=str, nargs='?', const=True, help=" The cell mapping file path.", required=False) + parser.add_argument("-am", "--api_mapping", dest="api_mapping", type=str, nargs='?', const=True, + help=" The api mapping file path.", required=False) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index bbd32a9d6..1d0e4e629 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -19,9 +19,16 @@ class MSComparator (Comparator): self.cross_frame = cell_mapping is not None or api_mapping is not None self.cell_mapping_dict = self.load_mapping_file(self.cell_mapping) self.api_mapping_dict = self.load_mapping_file(self.api_mapping) + if api_mapping is not None: + self.ms_to_pt_mapping = self.load_internal_api() + + def load_internal_api(self): + cur_path = os.path.dirname(os.path.realpath(__file__)) + yaml_path=os.path.join(cur_path,"ms_to_pt_api_dict.yaml") + return load_yaml(yaml_path) def load_mapping_file(self, mapping_file): - if isinstance(self.cell_mapping, str): + if isinstance(mapping_file, str): mapping_dict = load_yaml(mapping_file) else: mapping_dict = {} @@ -75,12 +82,31 @@ class MSComparator (Comparator): data_value = np.load(data_path) return data_value + def api_replace(self, a_op_name, target, para): + for i in range(len(a_op_name)): + a_op_name[i] = a_op_name[i].replace(target, para) + return a_op_name + + def process_api_mapping(self, a_op_name, b_op_name): + ms_api_para_list = a_op_name[0].split(Const.SEP) + ms_api_name= ms_api_para_list[0] + Const.SEP + ms_api_para_list[1] + pt_api_para_list = b_op_name[0].split(Const.SEP) + pt_api_name= pt_api_para_list[0] + Const.SEP + pt_api_para_list[1] + if ms_api_para_list[0] == "Mint": + return self.api_replace(a_op_name, "Mint", "Torch") + if ms_api_para_list[0] == "MintFunctional": + return self.api_replace(a_op_name, "MintFunctional", "Functional") + if self.ms_to_pt_mapping.get(ms_api_name) == pt_api_name: + return self.api_replace(a_op_name, ms_api_name, pt_api_name) + + def ms_compare(input_param, output_path, **kwargs): try: stack_mode = kwargs.get('stack_mode', False) auto_analyze = kwargs.get('auto_analyze', True) fuzzy_match = kwargs.get('fuzzy_match', False) cell_mapping = kwargs.get('cell_mapping', None) + api_mapping = kwargs.get('api_mapping', None) summary_compare, md5_compare = task_dumppath_get(input_param) check_configuration_param(stack_mode, auto_analyze, fuzzy_match) create_directory(output_path) @@ -88,7 +114,7 @@ def ms_compare(input_param, output_path, **kwargs): except (CompareException, FileCheckException) as error: logger.error('Compare failed. Please check the arguments and do it again!') raise CompareException(error.code) from error - ms_comparator = MSComparator(cell_mapping) + ms_comparator = MSComparator(cell_mapping, api_mapping) ms_comparator.compare_core(input_param, output_path, stack_mode=stack_mode, auto_analyze=auto_analyze, fuzzy_match=fuzzy_match, summary_compare=summary_compare, md5_compare=md5_compare) -- Gitee From 086ea7ea9771619cd61a1866dd3ede1710031d82 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 21 Aug 2024 15:58:08 +0800 Subject: [PATCH 419/791] =?UTF-8?q?=E6=97=A0=E6=A0=87=E6=9D=86=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E5=BA=94=E7=94=A8=E5=9C=BA=E6=99=AF=E5=92=8C=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E5=9F=BA=E7=BA=BF=E6=8A=A5=E5=91=8A=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...72\347\272\277\346\212\245\345\221\212.md" | 115 +++++++++++++++--- 1 file changed, 99 insertions(+), 16 deletions(-) diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" index 7e14a37eb..3e704d9e2 100644 --- "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" +++ "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" @@ -12,7 +12,7 @@ Torch：2.1.0 CANN：8.0.T5 -除上述环境信息影响性能外，API的数量、种类以及Shape都会对性能产生影响，因此本次选取指定网络进行测试。 +除上述环境信息影响性能外，API的数量、种类以及Shape都会对性能产生影响，因此本次选取不同场景网络和不同算子进行测试。 ## 模型信息和性能基线 @@ -21,15 +21,12 @@ CANN：8.0.T5 以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 -### YOLOV5 -主要数据类型：FLOAT32 +### LLaMA2-7B - - -### LLaMA2 - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16 +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink +其中，softmax算子为FLOAT32，输入输出均为2G大小，为模型最大显存开销的API。 +在该模型下、对无标杆工具处理模式、插装范围、扰动方式组合下性能和显存基线进行覆盖。 性能基线报告 其中耗时为训练10步，去除第一步耗时所得的平均每步耗时。 @@ -42,21 +39,107 @@ NUM_LAYER：1，1卡，主要数据类型：FLOAT16 | check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | | check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | | check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | -| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1.66 | softmax本身为高精度，跳过 | +| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1 | softmax本身为高精度，跳过 | | check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | | check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | | check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | | check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | | check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | -| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.375 | 1 | | +| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.38 | 1 | | | check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | | check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | | check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | -| check | 反 | ["softmax"] | add_noise | | | | | 不建议整网 | -| check | 反 | ["softmax"] | bit_noise | | | | | 不建议整网 | | check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | -| fix | 前 | ["softmax"] | to_cpu | | | | | 不支持整网、不支持反向 | -| fix | 前 | ["softmax"] | improve_precision | | | | | 不支持整网、不支持反向 | -| 预热 | 前 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | -| 预热 | 反 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | +| fix | 前 | ["softmax"] | to_cpu | 9.70 | 16.67 | 40.42 | 1.22 | 不支持整网、不支持反向 | +| fix | 前 | ["softmax"] | improve_precision | 0.26 | 14.67 | 1.08 | 1.07 | 不支持整网、不支持反向 | +| 预热 | 前 | [] | improve_precision | 155.07 | 24.79 | 646.13 | 1.81 | 低精度模型基线、只测预热的迭代 | +| 预热 | 反 | [] | improve_precision | 72.29 | 22.01 | 301.21 | 1.61 | 低精度模型基线、只测预热的迭代 | +### Aquila2-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.17 | 13.66 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.57 | 14.24 | 9.24 | 1.04 | | +| check | 反 | [] | add_noise | 21.05 | 14.19 | 123.82 | 1.04 | | +| fix | 前 | [] | improve_precision | 0.95 | 15.55 | 5.59 | 1.14 | | + +### Baichuan2-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.26 | 12.12 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.02 | 12.27 | 3.92 | 1.01 | | +| check | 反 | [] | add_noise | 11.15 | 12.67 | 42.88 | 1.05 | | +| fix | 前 | [] | improve_precision | 0.95 | 12.82 | 3.65 | 1.06 | | + +### Bloom-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.14 | 9.51 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.64 | 11.58 | 11.71 | 1.22 | | +| check | 反 | [] | add_noise | 17.15 | 9.51 | 122.5 | 1 | | +| fix | 前 | [] | improve_precision | 0.87 | 10.62 | 6.21 | 1.12 | | + +### Interlm-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.13 | 10.76 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.19 | 11.68 | 9.15 | 1.09 | | +| check | 反 | [] | add_noise | 11.69 | 10.89 | 89.92 | 1.01 | | +| fix | 前 | [] | improve_precision | 0.75 | 11.68 | 5.77 | 1.09 | | + +### Qwen-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.28 | 18.41 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 2.34 | 23.18 | 8.36 | 1.26 | | +| check | 反 | [] | add_noise | 22.07 | 19.47 | 78.82 | 1.06 | | +| fix | 前 | [] | improve_precision | 1.31 | 21.11 | 4.68 | 1.15 | | + +### Gemma-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.15 | 11.06 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.49 | 13.17 | 9.93 | 1.19 | | +| check | 反 | [] | add_noise | 16.69 | 11.06 | 111.27 | 1 | | +| fix | 前 | [] | improve_precision | 0.87 | 12.25 | 5.8 | 1.11 | | + +### ResNet50-Cifar +1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch。 +主要算子为conv2d，每个step有51个, 因此对conv2d进行检测。 +CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step信息和反向信息，且该框架下性能暂不保证）。 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.09 | 7.63 | 1 | 1 | 基线 | +| check | 前 | ["conv2d"] | improve_precision | 0.889 | 7.94 | 9.81 | 1.04 | | +| fix | 前 | ["conv2d"] | improve_precision | 0.328 | 7.47 | 3.64 | 0.91 | | +| fix | 前 | ["conv2d"] | to_cpu | 12.23 | 7.47 | 135.88 | 0.91 | | + +### OpenSora1.0 + +1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | | | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | | | | | | +| check | 反 | [] | add_noise | | | | | | +| fix | 前 | [] | improve_precision | | | | | | \ No newline at end of file -- Gitee From db071dca6142e2a961b41e053480771d8ac01a2a Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 21 Aug 2024 16:15:30 +0800 Subject: [PATCH 420/791] clean code --- .../compare/api_precision_compare.py | 4 ++-- .../tensor_transport_layer/device_dispatch.py | 24 +++++++++++-------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 0737c30b7..1528f02ab 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -291,8 +291,8 @@ def api_precision_compare(config): def online_api_precision_compare(online_config): rank = online_config.rank - result_csv_path = os.path.join("./", online_config.result_csv_path).replace(".csv", f"_rank{rank}.csv") - details_csv_path = os.path.join("./", online_config.details_csv_path).replace(".csv", f"_rank{rank}.csv") + result_csv_path = os.path.join("./", online_config.result_csv_path).replace("_rank*.csv", f"_rank{rank}.csv") + details_csv_path = os.path.join("./", online_config.details_csv_path).replace("_rank*.csv", f"_rank{rank}.csv") detail_csv_title = [ApiPrecisionCompareColumn.get_detail_csv_title()] result_csv_title = [ApiPrecisionCompareColumn.get_result_csv_title()] if not os.path.exists(result_csv_path): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 7d03066f6..9277c2a0a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -17,20 +17,21 @@ from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import mov CompareApi = set(absolute_standard_api) | set(binary_standard_api) | set(thousandth_standard_api) current_time = time.strftime("%Y%m%d%H%M%S") -ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + ".csv" -ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + ".csv" +ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME = "api_precision_compare_result_" + current_time + "_rank*.csv" +ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_" + current_time + "_rank*.csv" OnlineApiPrecisionCompareConfig = namedtuple('OnlineApiPrecisionCompareConfig', ['npu_data', 'gpu_data', 'rank', 'result_csv_path', 'details_csv_path']) -def run_ut_process(xpu_id, compare, consumer_queue, func, config): +def run_ut_process(xpu_id, compare, consumer_queue, func, config, api_precision_csv_file): """ When consumer_queue(shared with ConsumerDispatcher) is not empty, consume api data from consumer_queue. :param xpu_id: int :param compare: instance of Comparator :param consumer_queue: shared queues of ConsumerDispatcher :param func: run_touch_api_online :param config: run_ut_config + :param api_precision_csv_file: list, length is 2, result file name and details file name :return: """ gpu_device = torch.device(f'cuda:{xpu_id}') @@ -51,10 +52,10 @@ def run_ut_process(xpu_id, compare, consumer_queue, func, config): online_compare(api_data, gpu_device, compare, func, config) else: # NPUvsCPU vs GPUvsCPU - online_precision_compare(api_data, gpu_device, compare, func, config) + online_precision_compare(api_data, gpu_device, compare, func, config, api_precision_csv_file) -def online_precision_compare(api_data, device, compare, func, config): +def online_precision_compare(api_data, device, compare, func, config, api_precision_csv_file): """online run_ut for precision_compare: NPUvsCPU vs GPUvsCPU 1. get NPUvsCPU compare result 2. get GPUvsCPU compare result @@ -84,9 +85,9 @@ def online_precision_compare(api_data, device, compare, func, config): gpu_data = pd.DataFrame(gpu_detail, columns=DETAIL_TEST_ROWS[-1]) # NPUvsCPU vs GPUvsCPU + result_file_name, details_file_name = api_precision_csv_file precision_compare_config = OnlineApiPrecisionCompareConfig(npu_data, gpu_data, api_data.rank, - ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME, - ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME) + result_file_name, details_file_name) online_api_precision_compare(precision_compare_config) except Exception as err: @@ -150,12 +151,15 @@ class ConsumerDispatcher: def start(self, handle_func, config): self.queues = [mp.Queue(maxsize=self.capacity) for _ in range(self.num_workers)] + api_precision_csv_file = [ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME, ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME] for xpu_id, q in enumerate(self.queues): p = mp.Process(name="run_ut_process", target=run_ut_process, - args=(xpu_id, self.compare, q, handle_func, config)) + args=(xpu_id, self.compare, q, handle_func, config, api_precision_csv_file)) p.start() self.processes.append(p) + logger.info(f"Api_precision_compare task result will be saved in {ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME}") + logger.info(f"Api_precision_compare task details will be saved in {ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME}") logger.info("Successfully start unittest process.") def stop(self): @@ -167,8 +171,8 @@ class ConsumerDispatcher: for p in self.processes: p.join() logger.info("Successfully stop unittest process.") - logger.info(f"Api_precision_compare task result will be saved in {ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME}".replace(".csv", "_rank*.csv")) - logger.info(f"Api_precision_compare task details will be saved in {ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME}".replace(".csv", "_rank*.csv")) + logger.info(f"Api_precision_compare task result is saved in {ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME}") + logger.info(f"Api_precision_compare task details is saved in {ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME}") def update_consume_queue(self, api_data): while True: -- Gitee From 13f80098eb4d2649e51172583684055fcf5aab9f Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 21 Aug 2024 16:18:43 +0800 Subject: [PATCH 421/791] add statement that data in compare result is string, etc. --- .../accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md index c3c154b56..ffff1f44f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md @@ -234,6 +234,9 @@ PyTorch精度比对是以CPU或GPU的计算结果为标杆，通过计算精度 - 真实数据One Thousandth Err Ratio的input - output > 0.1 - 真实数据Cosine的input - output > 0.1 +### **Shape为[]时，统计量列说明** +当NPU Tensor Shape列为[]时，表示标量或0维张量，统计量列（NPU max、NPU min、NPU mean、NPU l2norm）展示相同的唯一元素。Bench同理。 + # FAQ [FAQ](./FAQ.md) -- Gitee From b6c22b45c865cf8a0f46791dd7ed4c7e0a88716b Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 21 Aug 2024 16:19:42 +0800 Subject: [PATCH 422/791] =?UTF-8?q?=E6=97=A0=E6=A0=87=E6=9D=86=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E5=BA=94=E7=94=A8=E5=9C=BA=E6=99=AF=E5=92=8C=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E5=9F=BA=E7=BA=BF=E6=8A=A5=E5=91=8A=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...72\347\272\277\346\212\245\345\221\212.md" | 34 +++++++++++-------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" index 3e704d9e2..bf78f78c2 100644 --- "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" +++ "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" @@ -26,6 +26,7 @@ CANN：8.0.T5 NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink 其中，softmax算子为FLOAT32，输入输出均为2G大小，为模型最大显存开销的API。 + 在该模型下、对无标杆工具处理模式、插装范围、扰动方式组合下性能和显存基线进行覆盖。 性能基线报告 @@ -34,26 +35,26 @@ NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/Model | 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | |--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| | / | / | / | / | 0.24 | 13.69 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | -| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | -| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | -| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | -| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | | check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1 | softmax本身为高精度，跳过 | | check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | | check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | | check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | | check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | -| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | +| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | | check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.38 | 1 | | | check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | -| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | +| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | +| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | +| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | +| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | +| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | | check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | | check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | +| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | | fix | 前 | ["softmax"] | to_cpu | 9.70 | 16.67 | 40.42 | 1.22 | 不支持整网、不支持反向 | | fix | 前 | ["softmax"] | improve_precision | 0.26 | 14.67 | 1.08 | 1.07 | 不支持整网、不支持反向 | | 预热 | 前 | [] | improve_precision | 155.07 | 24.79 | 646.13 | 1.81 | 低精度模型基线、只测预热的迭代 | -| 预热 | 反 | [] | improve_precision | 72.29 | 22.01 | 301.21 | 1.61 | 低精度模型基线、只测预热的迭代 | +| 预热 | 反 | [] | improve_precision | 72.29 | 22.01 | 301.21 | 1.61 | 低精度模型基线、只测预热的迭代，grad_output为高精度的算子跳过 | ### Aquila2-7B @@ -124,7 +125,7 @@ NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/Model ### ResNet50-Cifar 1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch。 主要算子为conv2d，每个step有51个, 因此对conv2d进行检测。 -CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step信息和反向信息，且该框架下性能暂不保证）。 +CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step信息和反向信息）。 | 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | |--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| @@ -135,11 +136,16 @@ CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step ### OpenSora1.0 -1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch +4卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch + +每张卡每个step中linear算子个数为257个，FA算子个数为83（FA算子反向无效）。 | 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | |--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | | | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | | | | | | -| check | 反 | [] | add_noise | | | | | | -| fix | 前 | [] | improve_precision | | | | | | \ No newline at end of file +| / | / | / | / | 0.99 | 17.61 | 1 | 1 | 混精模式基线 | +| check | 前 | ["linear","npu_fusion_attention"] | improve_precision | 3.88 | 17.61 | 3.92 | 1 | | +| check | 前 | ["linear","npu_fusion_attention"] | add_noise | 3.46 | 17.61 | 3.49 | 1 | | +| check | 反 | ["linear"] | improve_precision | 12.61 | 17.61 | 12.74 | 1 | | +| check | 反 | ["linear"] | add_noise | 9.8 | 17.61 | 9.90 | 1 | | +| fix | 前 | ["linear"] | to_cpu | 18.83 | 17.61 | 19.02 | 1 | | +| fix | 前 | ["linear"] | improve_precision | 2.83 | 17.61 | 2.86 | 1 | | -- Gitee From 4ee831c74690875b3bdf79c74f475d6fa74f88a8 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 21 Aug 2024 16:24:23 +0800 Subject: [PATCH 423/791] =?UTF-8?q?=E6=97=A0=E6=A0=87=E6=9D=86=E5=B7=A5?= =?UTF-8?q?=E5=85=B7=E5=BA=94=E7=94=A8=E5=9C=BA=E6=99=AF=E5=92=8C=E6=80=A7?= =?UTF-8?q?=E8=83=BD=E5=9F=BA=E7=BA=BF=E6=8A=A5=E5=91=8A=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename "debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" => "debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\234\272\346\231\257\351\252\214\350\257\201\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" (99%) diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\234\272\346\231\257\351\252\214\350\257\201\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" similarity index 99% rename from "debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" rename to "debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\234\272\346\231\257\351\252\214\350\257\201\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" index bf78f78c2..8862c5b76 100644 --- "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\272\224\347\224\250\345\234\272\346\231\257\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" +++ "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\234\272\346\231\257\351\252\214\350\257\201\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" @@ -1,4 +1,4 @@ -# 无标杆工具应用场景和性能基线报告 +# 无标杆工具场景验证和性能基线报告 ## 环境信息 -- Gitee From ad76f7c024eff3ee4af96d35ae611a66e78c3b2f Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Wed, 21 Aug 2024 16:47:55 +0800 Subject: [PATCH 424/791] api_checker code done --- .../msprobe/core/common/utils.py | 6 + .../api_accuracy_checker.py | 179 ++++++++++++++++-- .../mindspore/api_accuracy_checker/const.py | 17 +- .../mindspore/api_accuracy_checker/main.py | 16 ++ debug/accuracy_tools/msprobe/msprobe.py | 13 +- 5 files changed, 214 insertions(+), 17 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/main.py diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 7fa78c95d..72f9cf253 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -527,6 +527,12 @@ def write_csv(data, filepath): writer = csv.writer(f) writer.writerows(data) +def write_whole_csv(data_list, filepath): + with FileOpen(filepath, 'w', encoding='utf-8-sig') as f: + writer = csv.writer(f) + for data in data_list: + writer.writerows(data) + def load_npy(filepath): check_file_or_directory_path(filepath) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 6dbb7145c..20ee1535e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -1,17 +1,39 @@ +import json +import os + from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.const import Const +from msprobe.core.common.utils import write_whole_csv, add_time_as_suffix +from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation +from msprobe.mindspore.api_accuracy_checker.base_compare_algorithm import compare_algorithms from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context -import json + + +class BasicInfoAndStatus: + def __init__(self, api_name, bench_dtype, tested_dtype, shape, status, err_msg) -> None: + self.api_name = api_name + self.bench_dtype = bench_dtype + self.tested_dtype = tested_dtype + self.shape = shape + self.status = status + self.err_msg = err_msg + +class ResultCsvEntry: + def __init__(self) -> None: + self.forward_pass_status = None + self.backward_pass_status = None + self.forward_err_msg = "" + self.backward_err_msg = "" + self.overall_err_msg = None class ApiAccuracyChecker: def __init__(self): self.api_infos = dict() - self.results = None + self.results = dict() def parse(self, api_info_path): with FileOpen(api_info_path, "r") as f: @@ -45,22 +67,153 @@ class ApiAccuracyChecker: else: self.api_infos[api_name].load_backward_info(api_info) + def run_and_compare_helper(self, api_info, api_name_str, api_input_aggregation, forward_or_backward): + ''' + Args: + api_info: ApiInfo + api_name_str: str + api_input_aggregation: ApiInputAggregation + forward_or_backward: str: Union["forward", "backward"] + + Description: + get mindspore api output, run torch api and get output. + compare output. + record compare result. + ''' + # get output + if global_context.get_is_constructed(): + # constructed situation, need use constructed input to run mindspore api getting tested_output + tested_outputs = api_runner(api_input_aggregation, api_name_str, forward_or_backward, Const.MS_FRAMEWORK) + else: + tested_outputs = api_info.get_compute_element_list(forward_or_backward, Const.OUTPUT) + bench_outputs = api_runner(api_input_aggregation, api_name_str, Const.FORWARD, Const.PT_FRAMEWORK) + + # compare output + for i, (bench_out, tested_out) in enumerate(zip(bench_outputs, tested_outputs)): + api_name_with_slot = Const.SEP.join([api_name_str, forward_or_backward, Const.OUTPUT, str(i)]) + bench_dtype = bench_out.get_dtype() + tested_dtype = tested_out.get_dtype() + shape = bench_out.get_shape() + + compare_result_dict = dict() + for compare_algorithm_name, compare_algorithm in compare_algorithms.items(): + compare_result = compare_algorithm(bench_out, tested_out) + compare_result_dict[compare_algorithm_name] = compare_result + + if compare_result_dict.get(CompareConst.COSINE).pass_status == CompareConst.PASS and \ + compare_result_dict.get(CompareConst.MAX_ABS_ERR).pass_status == CompareConst.PASS: + status = CompareConst.PASS + err_msg = "" + else: + status = CompareConst.ERROR + err_msg = compare_result_dict.get(CompareConst.COSINE).err_msg + \ + compare_result_dict.get(CompareConst.MAX_ABS_ERR).err_msg + self.record(api_name_str, forward_or_backward, + BasicInfoAndStatus(api_name_with_slot, bench_dtype, tested_dtype, shape, status, err_msg), + compare_result_dict) def run_and_compare(self): for api_name_str, api_info in self.api_infos: if not api_info.check_forward_info(): - logger.warning(f"api: {api_name_str} is lack of forward infomation skip checking") + logger.warning(f"api: {api_name_str} is lack of forward infomation, skip forward and backward check") continue - # first run forapi_infoward and compare - inputs = api_info.get_compute_element_list(Const.FORWARD, Const.INPUT) - kwargs =api_info.get_kwargs() + forward_inputs = api_info.get_compute_element_list(Const.FORWARD, Const.INPUT) + kwargs = api_info.get_kwargs(api_info, api_name_str, ) + forward_inputs_aggregation = ApiInputAggregation(forward_inputs, kwargs, None) + self.run_and_compare_helper(api_info, api_name_str, forward_inputs_aggregation, Const.FORWARD) - if global_context.get_is_constructed(): - tested + if not api_info.check_backward_info(): + logger.warning(f"api: {api_name_str} is lack of backward infomation, skip backward check") + continue + gradient_inputs = api_info.get_compute_element_list(Const.BACKWARD, Const.INPUT) + backward_inputs_aggregation = ApiInputAggregation(forward_inputs, kwargs, gradient_inputs) + self.run_and_compare_helper(api_info, api_name_str, backward_inputs_aggregation, Const.BACKWARD) + + def record(self, api_real_name, forward_or_backward, basic_info, compare_result_dict): + key = tuple([api_real_name, forward_or_backward]) + if key not in self.results: + self.results[key] = [] + self.results[key].append(tuple([basic_info, compare_result_dict])) + + + def to_detail_csv(self, csv_dir): + # detail_csv + detail_csv = [] + detail_csv_header_basic_info = [ + MsApiAccuracyCheckerConst.DETAIL_CSV_API_NAME, + MsApiAccuracyCheckerConst.DETAIL_CSV_BENCH_DTYPE, + MsApiAccuracyCheckerConst.DETAIL_CSV_TESTED_DTYPE, + MsApiAccuracyCheckerConst.DETAIL_CSV_SHAPE, + ] + detail_csv_header_compare_result = list(compare_algorithms.keys()) + detail_csv_header_status = [ + MsApiAccuracyCheckerConst.DETAIL_CSV_PASS_STATUS, + MsApiAccuracyCheckerConst.DETAIL_CSV_MESSAGE, + ] + + detail_csv_header = detail_csv_header_basic_info + detail_csv_header_compare_result + detail_csv_header_status + detail_csv.append(detail_csv_header) + + for _, results in self.results.items(): + # detail csv + for res in results: + basic_info, compare_result_dict = res + csv_row_basic_info = \ + [basic_info.api_name, basic_info.bench_dtype, basic_info.tested_dtype, basic_info.shape] + csv_row_compare_result = list(compare_result_dict.get(algorithm_name).compare_value \ + for algorithm_name in detail_csv_header_compare_result) + csv_row_status = [basic_info.status, basic_info.err_msg] + csv_row = csv_row_basic_info + csv_row_compare_result + csv_row_status + detail_csv.append(csv_row) - def record(self, compare_result_collection): - pass + file_name = os.path.join(csv_dir, add_time_as_suffix(MsApiAccuracyCheckerConst.DETAIL_CSV_FILE_NAME)) + write_whole_csv(detail_csv, file_name) - def to_csv(self): - pass \ No newline at end of file + def to_result_csv(self, csv_dir): + result_csv_dict = dict() + for key, results in self.results.items(): + api_real_name, forward_or_backward = key + forward_or_backward_pass_status = CompareConst.PASS + forward_or_backward_overall_err_msg = "" + # detail csv + for res in results: + basic_info, _ = res + if basic_info.status != CompareConst.PASS: + forward_or_backward_pass_status = CompareConst.ERROR + forward_or_backward_overall_err_msg += basic_info.err_msg + forward_or_backward_overall_err_msg = \ + "" if forward_or_backward_pass_status == CompareConst.PASS else forward_or_backward_overall_err_msg + + #result_csv_dict + if api_real_name not in result_csv_dict: + result_csv_dict[api_real_name] = ResultCsvEntry() + if forward_or_backward == Const.FORWARD: + result_csv_dict[api_real_name].forward_pass_status = forward_or_backward_pass_status + result_csv_dict[api_real_name].forward_err_msg = forward_or_backward_overall_err_msg + else: + result_csv_dict[api_real_name].backward_pass_status = forward_or_backward_pass_status + result_csv_dict[api_real_name].backward_err_msg = forward_or_backward_overall_err_msg + + #result_csv + result_csv = [] + result_csv_header = [ + MsApiAccuracyCheckerConst.DETAIL_CSV_API_NAME, + MsApiAccuracyCheckerConst.RESULT_CSV_FORWARD_TEST_SUCCESS, + MsApiAccuracyCheckerConst.RESULT_CSV_BACKWARD_TEST_SUCCESS, + MsApiAccuracyCheckerConst.DETAIL_CSV_MESSAGE, + ] + result_csv.append(result_csv_header) + + for api_name, result_csv_entry in result_csv_dict: + if result_csv_entry.forward_pass_status == CompareConst.PASS and \ + result_csv_entry.backward_pass_status == CompareConst.PASS: + overall_err_msg = "" + else: + overall_err_msg = result_csv_entry.forward_err_msg + result_csv_entry.backward_err_msg + row = [api_name, result_csv_entry.forward_pass_status, + result_csv_entry.backward_pass_status, overall_err_msg] + result_csv.append(row) + + file_name = os.path.join(csv_dir, add_time_as_suffix(MsApiAccuracyCheckerConst.RESULT_CSV_FILE_NAME)) + write_whole_csv(result_csv, file_name) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py index 69b52ea61..9886576f5 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -7,4 +7,19 @@ class MsApiAccuracyCheckerConst: STATISTICS_TASK = "statistics" TENSOR_TASK = "tensor" DUMP_DATA_DIR_FIELD = "dump_data_dir" - DATA_FIELD = "data" \ No newline at end of file + DATA_FIELD = "data" + + #detail_csv + DETAIL_CSV_API_NAME = "API Name" + DETAIL_CSV_BENCH_DTYPE = "Bench Dtype" + DETAIL_CSV_TESTED_DTYPE = "Tested Dtype" + DETAIL_CSV_SHAPE = "Shape" + DETAIL_CSV_PASS_STATUS = "Status" + DETAIL_CSV_MESSAGE = "Message" + DETAIL_CSV_FILE_NAME = "accuracy_checking_details" + + #result_csv + RESULT_CSV_FORWARD_TEST_SUCCESS = "Forward Test Success" + RESULT_CSV_BACKWARD_TEST_SUCCESS = "Backward Test Success" + RESULT_CSV_FILE_NAME = "accuracy_checking_result" + diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/main.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/main.py new file mode 100644 index 000000000..db830e76b --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/main.py @@ -0,0 +1,16 @@ +from msprobe.mindspore.api_accuracy_checker.api_accuracy_checker import ApiAccuracyChecker + +def add_api_accuracy_checker_argument(parser): + parser.add_argument("-api_info", "--api_info_file", dest="api_info_file", type=str, required=True, + help=" The api param tool result file: generate from api param tool, " + "a json file.") + parser.add_argument("-o", "--out_path", dest="out_path", default="./", type=str, required=False, + help=" The ut task result out path.") + + +def api_checker_main(args): + api_accuracy_checker = ApiAccuracyChecker() + api_accuracy_checker.parse(args.api_info_file) + api_accuracy_checker.run_and_compare() + api_accuracy_checker.to_detail_csv(args.out_path) + api_accuracy_checker.to_result_csv(args.out_path) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 97f90a64d..e6488816f 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -32,7 +32,7 @@ def main(): "Providing one-site accuracy difference debugging toolkit for training on Ascend Devices.\n" f"For any issue, refer README.md first", ) - + parser.set_defaults(print_help=parser.print_help) parser.add_argument('-f', '--framework', required=True, choices=[Const.PT_FRAMEWORK, Const.MS_FRAMEWORK], help='Deep learning framework.') @@ -45,6 +45,7 @@ def main(): run_overflow_check_cmd_parser = subparsers.add_parser('run_overflow_check') _compare_parser(compare_cmd_parser) is_torch_available=is_module_available("torch") + is_mindspore_available = is_module_available("mindspore") if is_torch_available: from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import _run_ut_parser, run_ut_command from msprobe.pytorch.parse_tool.cli import parse as cli_parse @@ -53,14 +54,17 @@ def main(): _api_precision_compare_command from msprobe.pytorch.api_accuracy_checker.run_ut.run_overflow_check import _run_overflow_check_parser, \ _run_overflow_check_command - + _run_ut_parser(run_ut_cmd_parser) _run_ut_parser(multi_run_ut_cmd_parser) multi_run_ut_cmd_parser.add_argument('-n', '--num_splits', type=int, choices=range(1, 65), default=8, help='Number of splits for parallel processing. Range: 1-64') _api_precision_compare_parser(api_precision_compare_cmd_parser) _run_overflow_check_parser(run_overflow_check_cmd_parser) - + elif is_mindspore_available: + from msprobe.mindspore.api_accuracy_checker.main import add_api_accuracy_checker_argument + add_api_accuracy_checker_argument(run_ut_cmd_parser) + if len(sys.argv) == 1: parser.print_help() sys.exit(0) @@ -88,6 +92,9 @@ def main(): raise Exception("MindSpore does not exist, please install MindSpore library") if sys.argv[3] == "compare": compare_cli(args) + elif sys.argv[3] == "run_ut": + from msprobe.mindspore.api_accuracy_checker.main import api_checker_main + api_checker_main(args) if __name__ == "__main__": main() -- Gitee From a093f09b004d872adfedcbd22176ab6b45cff652 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Wed, 21 Aug 2024 16:17:17 +0800 Subject: [PATCH 425/791] fix save func --- .../msprobe/core/compare/acc_compare.py | 2 +- .../msprobe/mindspore/compare/ms_compare.py | 19 +++++++------------ 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 78f782dbe..f963272ae 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -172,7 +172,7 @@ class Comparator: if frame_name == "MSComparator": n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.NUMPY_SUFFIX) if self.cross_frame: - b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.PT_SUFFIX, load_pt=True) + b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.PT_SUFFIX, load_pt_file=True) else: b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_op_name + Const.NUMPY_SUFFIX) else: diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index a5d7466e0..be0947989 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,7 +1,7 @@ import os.path import numpy as np from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, \ - task_dumppath_get, load_yaml + task_dumppath_get, load_yaml, load_npy from msprobe.core.common.file_check import FileChecker, create_directory from msprobe.core.common.const import FileCheckConst, Const from msprobe.core.common.log import logger @@ -11,7 +11,7 @@ from msprobe.core.common.utils import CompareException from msprobe.core.compare.check import check_struct_match, fuzzy_check_op -class MSComparator (Comparator): +class MSComparator(Comparator): def __init__(self, cell_mapping=None, api_mapping=None): self.frame_name = MSComparator.__name__ self.cell_mapping = cell_mapping @@ -58,22 +58,17 @@ class MSComparator (Comparator): is_match = False return is_match and struct_match - def read_npy_data(self, dir_path, file_name, load_pt=False): + def read_npy_data(self, dir_path, file_name, load_pt_file=False): data_path = os.path.join(dir_path, file_name) - if load_pt: + if load_pt_file: import torch - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.PT_SUFFIX, False) - data_path = path_checker.common_check() - data_value = torch.load(data_path, map_location=torch.device('cpu')).detach() # detach for less memory + from msprobe.pytorch.common.utils import load_pt + data_value = load_pt(data_path).detach() if data_value.dtype == torch.bfloat16: data_value = data_value.to(torch.float32) data_value = data_value.numpy() else: - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.NUMPY_SUFFIX, False) - data_path = path_checker.common_check() - data_value = np.load(data_path) + data_value = load_npy(data_path) return data_value -- Gitee From 15baa6c872ba35042271c0c6da9aa9dfbacb2d3f Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 21 Aug 2024 17:45:27 +0800 Subject: [PATCH 426/791] clean code --- .../compare/api_precision_compare.py | 6 ++-- .../tensor_transport_layer/device_dispatch.py | 30 +++++++++++-------- 2 files changed, 21 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 1528f02ab..7cc1d6b5e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -291,8 +291,8 @@ def api_precision_compare(config): def online_api_precision_compare(online_config): rank = online_config.rank - result_csv_path = os.path.join("./", online_config.result_csv_path).replace("_rank*.csv", f"_rank{rank}.csv") - details_csv_path = os.path.join("./", online_config.details_csv_path).replace("_rank*.csv", f"_rank{rank}.csv") + result_csv_path = os.path.join(Const.DEFAULT_PATH, online_config.result_csv_path).replace("_rank*.csv", f"_rank{rank}.csv") + details_csv_path = os.path.join(Const.DEFAULT_PATH, online_config.details_csv_path).replace("_rank*.csv", f"_rank{rank}.csv") detail_csv_title = [ApiPrecisionCompareColumn.get_detail_csv_title()] result_csv_title = [ApiPrecisionCompareColumn.get_result_csv_title()] if not os.path.exists(result_csv_path): @@ -306,7 +306,7 @@ def online_api_precision_compare(online_config): check_csv_columns(gpu_data.columns, "gpu_csv") analyse_csv(npu_data, gpu_data, config) except Exception as err: - logger.error(f"Online api precision compare Error: %s" % str(err)) + logger.error(f"Online api precision compare Error: {str(err)}") change_mode(result_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) change_mode(details_csv_path, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 9277c2a0a..6d544dc60 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -22,15 +22,15 @@ ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME = "api_precision_compare_details_ OnlineApiPrecisionCompareConfig = namedtuple('OnlineApiPrecisionCompareConfig', ['npu_data', 'gpu_data', 'rank', 'result_csv_path', 'details_csv_path']) +# namedtuple of [instance of Comparator, func of run_touch_api_online, config of run_ut_config] +CommonCompareConfig = namedtuple('CommonCompareConfig', ['compare', 'handle_func', 'config']) -def run_ut_process(xpu_id, compare, consumer_queue, func, config, api_precision_csv_file): +def run_ut_process(xpu_id, consumer_queue, common_config, api_precision_csv_file): """ When consumer_queue(shared with ConsumerDispatcher) is not empty, consume api data from consumer_queue. :param xpu_id: int - :param compare: instance of Comparator :param consumer_queue: shared queues of ConsumerDispatcher - :param func: run_touch_api_online - :param config: run_ut_config + :param common_config: namedtuple of CommonCompareConfig :param api_precision_csv_file: list, length is 2, result file name and details file name :return: """ @@ -49,19 +49,23 @@ def run_ut_process(xpu_id, compare, consumer_queue, func, config, api_precision_ _, api_name, _ = api_data.name.split(Const.SEP) if api_name in CompareApi: # NPU vs GPU - online_compare(api_data, gpu_device, compare, func, config) + online_compare(api_data, gpu_device, common_config) else: # NPUvsCPU vs GPUvsCPU - online_precision_compare(api_data, gpu_device, compare, func, config, api_precision_csv_file) + online_precision_compare(api_data, gpu_device, common_config, api_precision_csv_file) -def online_precision_compare(api_data, device, compare, func, config, api_precision_csv_file): +def online_precision_compare(api_data, device, common_config, api_precision_csv_file): """online run_ut for precision_compare: NPUvsCPU vs GPUvsCPU 1. get NPUvsCPU compare result 2. get GPUvsCPU compare result 3. call online_api_precision_compare + :param api_data + :param device + :param common_config: namedtuple of CommonCompareConfig + :param api_precision_csv_file: [result_file_name, details_file_name] """ - + compare, func, config = common_config.compare, common_config.handle_func, common_config.config api_full_name = api_data.name [api_type, api_name, _] = api_full_name.split(Const.SEP) npu_args, npu_kwargs, npu_out = api_data.args, api_data.kwargs, api_data.result @@ -106,9 +110,10 @@ def online_precision_compare(api_data, device, compare, func, config, api_precis torch.cuda.empty_cache() -def online_compare(api_data, device, compare, func, config): - """online run_ut for compare：NPU vs GPU""" - +def online_compare(api_data, device, common_config): + """online run_ut for compare：NPU vs GPU + """ + compare, func, config = common_config.compare, common_config.handle_func, common_config.config api_full_name = api_data.name api_data = move2target_device(api_data, device) try: @@ -152,9 +157,10 @@ class ConsumerDispatcher: def start(self, handle_func, config): self.queues = [mp.Queue(maxsize=self.capacity) for _ in range(self.num_workers)] api_precision_csv_file = [ONLINE_API_PRECISION_COMPARE_RESULT_FILE_NAME, ONLINE_API_PRECISION_COMPARE_DETAILS_FILE_NAME] + common_config = CommonCompareConfig(self.compare, handle_func, config) for xpu_id, q in enumerate(self.queues): p = mp.Process(name="run_ut_process", target=run_ut_process, - args=(xpu_id, self.compare, q, handle_func, config, api_precision_csv_file)) + args=(xpu_id, q, common_config, api_precision_csv_file)) p.start() self.processes.append(p) -- Gitee From 018363070c671041253d3b1953f329c139ce753a Mon Sep 17 00:00:00 2001 From: wuyuhan Date: Wed, 21 Aug 2024 15:55:16 +0800 Subject: [PATCH 427/791] analyzer controller --- .../advisor/analyzer/analyzer_controller.py | 557 ++++++++++++++++++ profiler/advisor/analyzer/base_analyzer.py | 55 +- .../analyzer/cluster/slow_rank_analyzer.py | 216 +++++++ profiler/advisor/analyzer/memory/__init__.py | 0 .../analyzer/memory/memory_analyzer.py | 38 ++ .../dispatch/timeline_op_dispatch_analyzer.py | 60 +- .../fusion_ops/fusion_ops_analyzer.py | 22 +- .../fusion_ops/timeline_api_stack_checker.py | 22 +- .../schedule/syncbn/syncbn_analyzer.py | 16 +- .../schedule/syncbn/syncbn_checker.py | 11 +- .../synchronize_stream_analyzer.py | 15 +- .../synchronize_stream_checker.py | 11 +- .../schedule/timeline_base_checker.py | 14 +- profiler/advisor/common/analyzer_scopes.py | 2 + .../advisor/common/async_analysis_status.py | 7 + profiler/advisor/common/constant.py | 32 +- .../test_dataloader_checker.py | 65 -- .../timeline_advice/test_syncbn_checker.py | 62 -- .../test_synchronize_stream.py | 55 -- .../test_timeline_op_compile_checker.py | 46 -- .../test_rdma_retransmission_advice.py | 170 ------ .../test_packet_advice.py | 175 ------ .../compute_advice/test_frequency_advice.py | 145 ----- .../advisor/schedule_advice/test_gc_advice.py | 116 ---- 24 files changed, 974 insertions(+), 938 deletions(-) create mode 100644 profiler/advisor/analyzer/analyzer_controller.py create mode 100644 profiler/advisor/analyzer/cluster/slow_rank_analyzer.py create mode 100644 profiler/advisor/analyzer/memory/__init__.py create mode 100644 profiler/advisor/analyzer/memory/memory_analyzer.py create mode 100644 profiler/advisor/common/async_analysis_status.py delete mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py delete mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py delete mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py delete mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py delete mode 100644 profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py delete mode 100644 profiler/test/ut/advisor/communication_advice/test_packet_advice.py delete mode 100644 profiler/test/ut/advisor/compute_advice/test_frequency_advice.py delete mode 100644 profiler/test/ut/advisor/schedule_advice/test_gc_advice.py diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py new file mode 100644 index 000000000..e8704542e --- /dev/null +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -0,0 +1,557 @@ +import copy +import logging +import json +import sys +import os +import multiprocessing as mp +from pathlib import Path +from multiprocessing import Manager + +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "compare_tools")) +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "cluster_analyse")) + +from profiler.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer +from profiler.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer +from profiler.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer +from profiler.advisor.config.config import Config +from profiler.advisor.common.analyzer_scopes import SupportedScopes +from profiler.advisor.common.async_analysis_status import AsyncAnalysisStatus +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterDataset +from profiler.advisor.utils.utils import Timer, safe_index, safe_division +from profiler.advisor.interface.interface import Interface +from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from profiler.prof_common.path_manager import PathManager + +logger = logging.getLogger() + + +class AnalyzerController: + CLUSTER_RANK_THRESHOLD = 2 + + def __init__(self): + self.dimensions = Interface.all_dimension + self.kwargs = {} + self.slow_rank_analyzer = None + self.slow_link_analyzer = None + self.cluster_local_data_map = {} + self.default_rank_id = None + self.rank_id_map = {} + self._is_cluster = False + self.analysis_process_resp = Manager().dict() + + @staticmethod + def _check_profiling_path_valid(profiling_path): + PathManager.input_path_common_check(profiling_path) + + if not Path(profiling_path).exists(): + logger.error("Profiling path is not existed. Invalid profiling path: %s", profiling_path) + return False + return True + + @staticmethod + def _get_step_rank_for_cluster_statistic_diff(target_cluster_statistic_data, benchmark_cluster_statistic_data, + headers, dimension, get_max=False): + if dimension not in headers: + logger.error("Error dimension %s for cluster statistics data, optionals are %s.", dimension, headers) + return None, None, None + + dimension_index = safe_index(headers, dimension) + diff_record = [] + # 对比目标profiling和benchmark profiling 每张卡的计算和下发和带宽，取计算、下发、带宽差异最大的卡进行下一步分析 + for target_row_data, benchmark_row_data in zip(target_cluster_statistic_data, benchmark_cluster_statistic_data): + target_data = safe_index(target_row_data, dimension_index) + benchmark_data = safe_index(benchmark_row_data, dimension_index) + if not isinstance(target_data, (int, float)) or not isinstance(benchmark_data, (int, float)): + continue + diff_record.append(target_data - benchmark_data) + + if SlowRankAnalyzer.compute_max_gap_ratio(diff_record, safe_division(sum(diff_record), len( + diff_record))) < SlowRankAnalyzer.RATIO_THRESHOLD: + return None, None, None + + value = max(diff_record) if get_max else min(diff_record) + value_index = safe_index(diff_record, value) + + step_value_index = safe_index(headers, "step") + rank_id_value_index = safe_index(headers, "rank_id") + step = safe_index(safe_index(target_cluster_statistic_data, value_index, []), step_value_index) + benchmark_step = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), step_value_index) + target_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) + benchmark_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) + + if target_rank_id != benchmark_rank_id: + logger.error( + "Rank ids of target profiling must keep the same as benchmark profiling, skip cluster comparison") + return None, None, None + + return step, benchmark_step, target_rank_id + + def do_analysis(self, dimensions, **kwargs): + pid = os.getpid() + resp = {"id": pid} + try: + self._do_analysis(dimensions, pid=pid, resp=resp, **kwargs) + except Exception as e: + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED, error_msg=str(e)) + logger.error(e) + raise RuntimeError(e) + + def async_do_analysis(self, dimensions, **kwargs): + # 异步分析，用于部署服务，通过接口查询异步作业状态 + async_analysis_process = mp.Process(target=self.do_analysis, args=(dimensions,), kwargs=kwargs, + name="Async advisor performance analysis") + async_analysis_process.start() + return async_analysis_process + + def get_response_by_pid(self, pid): + return self.analysis_process_resp.get(pid) + + def single_rank_analysis(self, profiling_path, benchmark_profiling_path=None): + job_list = [] + + profiling_path = self._get_profiling_path_by_rank(profiling_path) + benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) + + # 单卡场景无集群分析 + for dim in [Interface.CLUSTER]: + if dim in self.dimensions: + self.dimensions.remove(dim) + + for dimension in self.dimensions: + dimension_analysis_func_name = f"{dimension}_analysis" + if not hasattr(self, dimension_analysis_func_name): + continue + logger.info("Start %s analysis", dimension) + job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + + if benchmark_profiling_path: + # kernel/api 比对 + job_list += self._single_profiling_comparison(profiling_path, benchmark_profiling_path) + else: + # 单卡性能拆解 + self.overall(profiling_path) + return job_list + + def cluster_analysis(self, profiling_path, benchmark_profiling_path=None): + job_list = [] + + # 单集群profiling分析：下发、通信、计算、显存/内存 + for dimension in self.dimensions: + dimension_analysis_func_name = f"cluster_{dimension}_analysis" + if not hasattr(self, dimension_analysis_func_name): + continue + logger.info("Start cluster %s analysis", dimension) + job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + + if benchmark_profiling_path: + # 两个集群profiling比对分析 + job_list += self._cluster_profiling_comparison(profiling_path, benchmark_profiling_path) + else: + self.overall(profiling_path) + return job_list + + def overall(self, profiling_path): + from profiler.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariabelAnalyzer + env_analyzer = EnvironmentVariabelAnalyzer(profiling_path) + env_analyzer.optimize() + + if self._is_cluster: + self.slow_rank_analyzer.optimize(template_key=Interface.OVERALL) + self.slow_link_analyzer.optimize(template_key=Interface.OVERALL) + else: + from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer + overall_analyzer = OverallSummaryAnalyzer(profiling_path) + overall_analyzer.optimize() + + def schedule_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None): + # 任意单卡的下发分析 + + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.SCHEDULE]: + for scope in Interface.get_scope(dimension): + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def computation_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None, stage=None): + # 任意单卡的计算分析 + + kwargs = copy.deepcopy(self.kwargs) + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + kwargs["stage"] = stage + job_list = [] + + for dimension in [Interface.COMPUTATION]: + for scope in Interface.get_scope(dimension): + if scope == SupportedScopes.STAGE_COMPUTE: + continue + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def memory_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None): + # 任意单卡的内存分析 + + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.MEMORY]: + for scope in Interface.get_scope(dimension): + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def communication_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None, bandwidth_type=None): + + job_list = [] + supported_trans_type = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + if bandwidth_type is not None and bandwidth_type not in supported_trans_type: + logger.error("Error transit type %s, optionals are %s", bandwidth_type, supported_trans_type) + return job_list + + bandwidth_type_list = [bandwidth_type] if bandwidth_type is not None else supported_trans_type + + for bandwidth_type in bandwidth_type_list: + job_list += getattr(self, f"_communication_{bandwidth_type.lower()}_analysis")(profiling_path, + benchmark_profiling_path, + step, benchmark_step) + + return job_list + + def cluster_schedule_analysis(self, profiling_path): + # 目标集群profiling数据下发分析，不包含两个集群profiling数据的比对分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) + + info_msg = f"Maximum free for rank {slow_rank_id}" + if slow_step: + info_msg += f" and step {slow_step}" + logger.info(info_msg) + + job_list += self.schedule_analysis(analysis_profiling_path, step=slow_step) + return job_list + + def cluster_communication_analysis(self, profiling_path): + job_list = [] + + for dimension in [Interface.COMMUNICATION]: + for scope in Interface.get_scope(dimension): + analyzer_class = Interface.get_analyzer(dimension, scope) + if hasattr(analyzer_class, "requires_cluster_dataset") and getattr(analyzer_class, + "requires_cluster_dataset"): + + # 如果不依赖数据集，或者依赖的是ClusterDataset，则不用根据带宽确定需要分析的特定rank + kwargs = copy.deepcopy(self.kwargs) + kwargs["profiling_path"] = profiling_path + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + else: + # 非ClusterDataset场景，需要根据带宽大小分析特定的rank + for bandwidth_type in [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA]: + global_step_rank = self.slow_link_analyzer.get_global_step_rank(bandwidth_type) + # 获取带宽最小的卡进行分析 + target_rank_id = global_step_rank.get("minimum", {}).get("rank_id") or self.default_rank_id + step = global_step_rank.get("minimum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, target_rank_id) + + info_msg = f"Minimum {bandwidth_type} bandwidth for rank {target_rank_id} " + if step: + info_msg += f"and step {step}" + logger.info(info_msg) + + job_list += self.communication_analysis(analysis_profiling_path, step=step, + bandwidth_type=bandwidth_type) + + return job_list + + def cluster_computation_analysis(self, profiling_path): + # 目标集群profiling数据计算分析，不包含两个集群profiling数据的比对分析；如果有pp stage，则对不同stage进行计算分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.COMPUTE) + stage_step_rank = self.slow_rank_analyzer.get_stage_step_rank(SlowRankAnalyzer.COMPUTE) + + if stage_step_rank: + # 对不同pp stage取min max进行分析 + logger.info("Analysis steps and ranks of different pipeline parallel stages are %s", + json.dumps(stage_step_rank)) + + stages_profiling_path = [] + for stage, step_rank_info in stage_step_rank.items(): + rank_id = step_rank_info.get("maximum", {}).get("rank_id") + step = step_rank_info.get("maximum", {}).get("step") + + info_msg = f"For {stage}, slow rank is {rank_id}" + if step: + info_msg += f", step is {step}" + logger.info(info_msg) + + stages_profiling_path.append( + dict( + stage=stage, + rank_id=rank_id, + step=step, + profiling_path=self._get_profiling_path_by_rank(profiling_path, rank_id) + ) + ) + Interface.add_analyzer(Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, PPStageComputationAnalyzer) + kwargs = {"stages_profiling_path": stages_profiling_path, "profiling_path": profiling_path} + + job_list.append((Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, Interface(**kwargs), kwargs)) + else: + # 不区分stage，对所有卡取Min max进行分析 + logger.info("Without pipeline parallel stage, Global analysis steps and ranks is %s", + json.dumps(global_step_rank)) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + # 如果没有标杆profiling数据的rank id，说明没有快慢卡问题，直接对默认rank id进行分析，因此这里取值为None + fast_rank_id = global_step_rank.get("minimum", {}).get("rank_id") + fast_step = global_step_rank.get("minimum", {}).get("step") + + info_msg = f"Maximum computation time for rank {slow_rank_id}" + if slow_step: + info_msg += f" and step {slow_step}, " + if fast_rank_id: + info_msg += f"minimum computation time for rank {fast_rank_id}" + if fast_step: + info_msg += f" and step {fast_step}" + logger.info(info_msg) + + job_list += self.computation_analysis( + self._get_profiling_path_by_rank(profiling_path, slow_rank_id), + self._get_profiling_path_by_rank(profiling_path, fast_rank_id), + slow_step, + fast_step + ) + + return job_list + + def cluster_memory_analysis(self, profiling_path): + # 目标集群profiling数据内存分析，当前memory识别的两个算子，导致的问题都是大的free，因此选择FREE最慢的卡进行分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) + + info_msg = f"Maximum free for rank {slow_rank_id} " + if slow_step: + info_msg += f"and step {slow_step}" + logger.info(info_msg) + + job_list += self.memory_analysis(analysis_profiling_path, step=slow_step) + return job_list + + def _do_analysis(self, dimensions, **kwargs): + self.dimensions = dimensions + self.kwargs = kwargs + result_list = [] + profiling_path = self.kwargs.get("profiling_path") + benchmark_profiling_path = self.kwargs.get("benchmark_profiling_path") + pid = self.kwargs.get("pid") + resp = self.kwargs.get("resp") + + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.ANALYZING) + + if not self._check_profiling_path_valid(profiling_path): + error_msg = f"Got invalid argument '-d/--profiling_path' {profiling_path}, skip analysis" + self._update_analysis_process_resp(pid, resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + if benchmark_profiling_path and not self._check_profiling_path_valid(benchmark_profiling_path): + error_msg = f"Got invalid argument '-bp/--benchmark_profiling_path' {benchmark_profiling_path}, skip analysis" + self._update_analysis_process_resp(pid, resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + + self._is_cluster = self._is_cluster_profiling(profiling_path) + if not self._is_cluster: + job_list = self.single_rank_analysis(profiling_path, benchmark_profiling_path) + else: + job_list = self.cluster_analysis(profiling_path, benchmark_profiling_path) + + for i, (dimension, scope, interface, kwargs) in enumerate(job_list[::-1]): + result_list.append( + interface.get_result(dimension, scope, render_html=i == len(job_list) - 1, output_dict=False, + **kwargs) + ) + + for result in result_list[::-1]: + if result and hasattr(result, "show"): + result.show() + break + self._get_analysis_success_resp(pid, resp) + + def _communication_rdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None): + # 小包分析 + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.COMMUNICATION]: + for scope in Interface.get_scope(dimension): + if scope != SupportedScopes.PACKET: + continue + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + + return job_list + + def _communication_sdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None): + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + return job_list + + def _single_profiling_comparison(self, profiling_path, benchmark_profiling_path, step=None, + benchmark_step=None): + # TODO 基于compare tools 对比计算下发 + kwargs = copy.deepcopy(self.kwargs) + return [] + + def _cluster_profiling_comparison(self, profiling_path, benchmark_profiling_path): + # 从计算、下发和通信三个维度对集群profiling数据进行对比 + + job_list = [] + benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) + benchmark_slow_rank_analyzer = SlowRankAnalyzer(benchmark_profiling_path) + benchmark_slow_link_analyzer = SlowLinkAnalyzer(benchmark_profiling_path) + + # 计算和下发分析 + job_list += self._cluster_data_comparison(profiling_path, + benchmark_profiling_path, + self.slow_rank_analyzer, + benchmark_slow_rank_analyzer, + get_max=True) + + # 通信分析 + job_list += self._cluster_data_comparison(profiling_path, + benchmark_profiling_path, + self.slow_link_analyzer, + benchmark_slow_link_analyzer, + get_max=False) + return job_list + + def _cluster_data_comparison(self, profiling_path, benchmark_profiling_path, target_cluster_analyzer, + benchmark_cluster_analyzer, get_max=False): + # #low rank/slow link结果逐行对比获取差值最大的rank和step进行单卡分析 + job_list = [] + + if isinstance(target_cluster_analyzer, SlowRankAnalyzer): + comparison_dims = [SlowRankAnalyzer.COMPUTE, SlowRankAnalyzer.FREE] + elif isinstance(target_cluster_analyzer, SlowLinkAnalyzer): + comparison_dims = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + else: + return job_list + + target_data = target_cluster_analyzer.format_datas.get("data", []) + benchmark_data = benchmark_cluster_analyzer.format_datas.get("data", []) + headers = benchmark_cluster_analyzer.format_datas.get("headers", []) + + if len(target_data) != len(benchmark_data): + logger.warning( + "The product of ranks and steps of Benchmark profiling is not equals to target profiling, skip cluster comparison.") + return job_list + + for dimension in comparison_dims: + step, benchmark_step, rank_id_for_comparison = AnalyzerController._get_step_rank_for_cluster_statistic_diff( + target_data, + benchmark_data, + headers, + dimension, + get_max=get_max + ) + rank_profiling_path = self._get_profiling_path_by_rank(profiling_path, rank_id_for_comparison) + rank_benchmark_profiling_path = self._get_profiling_path_by_rank( + benchmark_profiling_path, + rank_id_for_comparison + ) + + job_list += self._single_profiling_comparison( + rank_profiling_path, + rank_benchmark_profiling_path, + step, + benchmark_step + ) + return job_list + + def _is_cluster_profiling(self, profiling_path): + path_list = [os.path.join(profiling_path, dir_name) for dir_name in os.listdir(profiling_path)] + ascend_pt_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_pt")] + data_processor = PytorchDataPreprocessor(ascend_pt_dirs) + + self.cluster_local_data_map[profiling_path] = data_processor.get_data_map() + + if not self.cluster_local_data_map or not self.cluster_local_data_map.get(profiling_path): + return False + + self.default_rank_id = list(self.cluster_local_data_map[profiling_path].keys())[0] + + self.slow_rank_analyzer = SlowRankAnalyzer(profiling_path) + self.slow_link_analyzer = SlowLinkAnalyzer(profiling_path) + return len(self.cluster_local_data_map[profiling_path]) >= self.CLUSTER_RANK_THRESHOLD + + def _get_profiling_path_by_rank(self, profiling_path, rank_id=None): + + if not profiling_path: + return profiling_path + + return self._get_target_profiling_path_for_local(profiling_path, rank_id) + + def _get_target_profiling_path_for_local(self, profiling_path, rank_id): + rank_id_map = self.cluster_local_data_map.get(profiling_path, {}) + if rank_id is None or not rank_id_map: + return profiling_path + + if rank_id in rank_id_map: + return rank_id_map.get(rank_id) + + local_first_rank_id = sorted(list(map(int, rank_id_map.keys())))[0] + logger.warning("Target rank id %s does not exist in local profiling data %s, use rank %s for analysis", + rank_id, profiling_path, local_first_rank_id) + return rank_id_map.get(local_first_rank_id) + + def _update_analysis_process_resp(self, pid, resp, **kwargs): + if kwargs: + resp.update(kwargs) + self.analysis_process_resp[pid] = resp + + def _get_analysis_success_resp(self, pid, resp): + html_path = os.path.join(Config().work_path, f"mstt_advisor_{Timer().strftime}.html") + xlsx_path = os.path.join(Config().work_path, f"mstt_advisor_{Timer().strftime}.xlsx") + result_files = {"html": html_path, "xlsx": xlsx_path} + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.SUCCESS, result_files=result_files) \ No newline at end of file diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index 80368e1d6..6347839b1 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -22,12 +22,16 @@ from profiler.advisor.common.version_control import VersionControl from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from profiler.advisor.utils.utils import safe_division logger = logging.getLogger() class BaseAnalyzer(VersionControl, metaclass=ABCMeta): _SUPPORT_VERSIONS = constant.SUPPORTED_CANN_VERSION + ANALYZER_HIGH_PRIORITY_TIME_RATIO = 0.05 + ANALYZER_MEDIUM_PRIORITY_TIME_RATIO = 0.03 dataset_cls_list = [] @@ -43,6 +47,18 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): self.result = OptimizeResult() self.record_list: Dict[str, List] = {} + @staticmethod + def get_first_data_by_key(data, key) -> Union[Dataset, None]: + """ + get the first member from data with key + :param data: input data + :param key: data key + :return: the first dataset in dataset list + """ + if key in data and len(data[key]) > 0: + return data[key][0] + return None + @classmethod def check_data(cls, data_list: tuple): """ @@ -63,7 +79,7 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): return None logger.info("Enable analysis %s with %s", self.__class__.__name__, ",".join(data_list)) - return func(self) + return func(self, **kwargs) return wrapper @@ -73,6 +89,10 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): def optimize(self, **kwargs): pass + @abstractmethod + def get_priority(self): + pass + def init_dataset_list(self)->None: dataset_cls_list = self.dataset_cls_list if len(dataset_cls_list) == 0: @@ -91,14 +111,25 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): self.dataset_list[key] = [] self.dataset_list[key].append(dataset) - @staticmethod - def get_first_data_by_key(data, key) -> Union[Dataset, None]: - """ - get the first member from data with key - :param data: input data - :param key: data key - :return: the first dataset in dataset list - """ - if key in data and len(data[key]) > 0: - return data[key][0] - return None + def init_dataset_list(self) -> None: + dataset_cls_list = self.dataset_cls_list + if len(dataset_cls_list) == 0: + logger.warning(f"Analyzer: %s don't rely on any dataset!", self.__class__.__name__) + return + + for dataset_cls in dataset_cls_list: + if dataset_cls and callable(dataset_cls): + dataset = dataset_cls(collection_path=self.collection_path, data=self.dataset_list, **self.kwargs) + key = dataset_cls.get_key() + if key not in self.dataset_list: + self.dataset_list[key] = [] + self.dataset_list[key].append(dataset) + + def get_priority_by_time_ratio(self, dur, step_dur): + time_ratio = safe_division(dur, step_dur) + if time_ratio >= self.ANALYZER_HIGH_PRIORITY_TIME_RATIO: + return PriorityBackgroundColor.high + elif time_ratio >= self.ANALYZER_MEDIUM_PRIORITY_TIME_RATIO: + return PriorityBackgroundColor.medium + else: + return PriorityBackgroundColor.low diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py new file mode 100644 index 000000000..a1971baf9 --- /dev/null +++ b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py @@ -0,0 +1,216 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.common import constant +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset +from profiler.advisor.utils.utils import safe_index + +logger = logging.getLogger() + + +class SlowRankAnalyzer(BaseAnalyzer): + SLOW_RANK_ANALYSIS = "slow rank" + RANK = "rank" + RATIO_THRESHOLD = 0.05 + BOTTLENECK_LIST = ['Computing', 'Communication', "Free"] + dataset_cls_list = [ClusterStepTraceTimeDataset] + COMPUTE = "compute(us)" + FREE = "free(us)" + COMMUNICATION = "communication(us)" + + def __init__(self, collection_path, n_processes: int = 1, **kwargs): + super().__init__(collection_path, n_processes, **kwargs) + key = ClusterStepTraceTimeDataset.get_key() + self.step_trace_class = self.get_first_data_by_key(self.dataset_list, key) + self.step_trace_dict = self.step_trace_class.get_data() + self.stages = self.step_trace_class.get_stages() + self.result = OptimizeResult() + self.bottelneck = '' + self.suggestion = '' + self._steps = set() + if self.step_trace_dict is not None: + self.format_datas = self.format_details() + + @property + def steps(self): + return sorted(list(self._steps)) + + @staticmethod + def compute_max_gap_ratio(data: list, mean: float): + if mean == 0: + return 0 + else: + return (max(data) - min(data)) / mean + + def optimize(self, **kwargs): + if self.step_trace_dict is None: + logger.error("slow_rank 分析失败，原因是数据加载失败，请检查你的cluster_analysis_outpu文件夹 \ + 如不关心这类数据请忽略") + return self.result + self.process() + self.make_record() + self.make_render(kwargs.get("template_key")) + return self.result + + def process(self): + total_time_list = [sum(data_tuple) for rank_id, data_tuple in self.step_trace_dict.items()] + if total_time_list: + mean_total_time = sum(total_time_list) / len(total_time_list) + for i in range(len(self.BOTTLENECK_LIST)): + self.produce_bottleneck(self.step_trace_dict, i, mean_total_time) + + if not self.bottelneck: + self.bottelneck = "There is no slow rank issues" + + def produce_bottleneck(self, step_dict: dict, produce_type: int, mean_total_time: float): + data_list = [data_tuple[produce_type] for rank_id, data_tuple in step_dict.items()] + max_ratio = self.compute_max_gap_ratio(data_list, mean_total_time) + if max_ratio > self.RATIO_THRESHOLD: + self.bottelneck += f'{self.BOTTLENECK_LIST[produce_type]} \n' \ + f' has some issues in the cluster, \n' \ + f' because the max difference of {self.BOTTLENECK_LIST[produce_type]} time \n' \ + f' has reached {round(max_ratio * mean_total_time / 1000, 3)}ms. \n' + + def make_record(self): + """ + make record for what and how to optimize + """ + + optimization_item = OptimizeItem( + SlowRankAnalyzer.SLOW_RANK_ANALYSIS, + self.bottelneck, + self.suggestion + ) + self.result.add(OptimizeRecord(optimization_item)) + + data_list = self.format_datas.get("data", []) + headers = self.format_datas.get("headers", []) + for data in data_list: + self.result.add_detail(SlowRankAnalyzer.SLOW_RANK_ANALYSIS, headers, data) + + def format_details(self): + details_dict = {} + headers = ["step", "rank_id", "compute(us)", "communication(us)", "free(us)"] + data_list = [] + for key, value in self.step_trace_dict.items(): + step, rank_id = key.split(constant.STEP_RANK_SEP) + data_list.append([int(step), int(rank_id)] + value) + if step and step not in self._steps: + self._steps.add(step) + + details_dict["headers"] = headers + details_dict["data"] = sorted(data_list, key=lambda x: (x[0], x[1])) + return details_dict + + def make_render(self, template_key="cluster"): + result_for_html = { + "Description": self.bottelneck, + "suggestion": self.suggestion, + "details": [self.format_datas] + } + + self.html_render.render_template(key=template_key, + title=SlowRankAnalyzer.SLOW_RANK_ANALYSIS, + template_dir="templates", + template_name="cluster_analysis.html", + cann_version=self.cann_version, + torch_version=self.torch_version, + result=result_for_html) + + def get_global_step_rank(self, dimension): + global_step_rank = {} + + headers = self.format_datas.get("headers") + + dimension_index = safe_index(headers, dimension) + rank_id_index = safe_index(headers, "rank_id") + step_index = safe_index(headers, "step") + if dimension_index is None or rank_id_index is None: + return global_step_rank + + data_list = [tuple_list[dimension_index] for tuple_list in self.format_datas.get("data")] + max_time, min_time = max(data_list), min(data_list) + + if self.compute_max_gap_ratio(data_list, sum(data_list) / len( + data_list)) < self.RATIO_THRESHOLD: + return global_step_rank + max_time_index = data_list.index(max_time) + min_time_index = data_list.index(min_time) + + max_time_rank_id = self.format_datas.get("data")[max_time_index][rank_id_index] + min_time_rank_id = self.format_datas.get("data")[min_time_index][rank_id_index] + + if step_index is not None: + max_time_step = self.format_datas.get("data")[max_time_index][step_index] + min_time_step = self.format_datas.get("data")[min_time_index][step_index] + else: + max_time_step, min_time_step = constant.DEFAULT_STEP, constant.DEFAULT_STEP + + global_step_rank["maximum"] = {"rank_id": max_time_rank_id, "step": max_time_step} + global_step_rank["minimum"] = {"rank_id": min_time_rank_id, "step": min_time_step} + + return global_step_rank + + def get_stage_step_rank(self, dimension): + stage_step_rank = {} + + headers = self.format_datas.get("headers") + dimension_index = safe_index(headers, dimension) + rank_id_index = safe_index(headers, "rank_id") + step_index = safe_index(headers, "step") + if dimension_index is None or rank_id_index is None: + return stage_step_rank + + rank_list = [tuple_list[rank_id_index] for tuple_list in self.format_datas.get("data")] + cost_time_list = [tuple_list[dimension_index] for tuple_list in self.format_datas.get("data")] + + if step_index is not None: + step_list = [tuple_list[step_index] for tuple_list in self.format_datas.get("data")] + else: + step_list = [constant.DEFAULT_STEP] * len(rank_list) + + for index, stage in enumerate(self.stages): + tmp_step_list, tmp_rank_list, tmp_time_list = [], [], [] + for step, rank_id, time in zip(step_list, rank_list, cost_time_list): + if rank_id not in stage: + continue + + tmp_step_list.append(step) + tmp_rank_list.append(rank_id) + tmp_time_list.append(time) + + if self.compute_max_gap_ratio(tmp_time_list, sum(tmp_time_list) / len( + tmp_time_list)) < self.RATIO_THRESHOLD: + continue + + max_time, min_time = max(tmp_time_list), min(tmp_time_list) + max_time_index, min_time_index = tmp_time_list.index(max_time), tmp_time_list.index(min_time) + + stage_key = f"stage-{index}" + stage_step_rank[stage_key] = {} + stage_step_rank[stage_key]["maximum"] = {"rank_id": tmp_rank_list[max_time_index], + "step": tmp_step_list[max_time_index]} + stage_step_rank[stage_key]["minimum"] = {"rank_id": tmp_rank_list[min_time_index], + "step": tmp_step_list[min_time_index]} + + return stage_step_rank + + def get_priority(self): + pass \ No newline at end of file diff --git a/profiler/advisor/analyzer/memory/__init__.py b/profiler/advisor/analyzer/memory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/memory/memory_analyzer.py b/profiler/advisor/analyzer/memory/memory_analyzer.py new file mode 100644 index 000000000..cd7b0a242 --- /dev/null +++ b/profiler/advisor/analyzer/memory/memory_analyzer.py @@ -0,0 +1,38 @@ +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.memory.memory_checker import MemoryOpsChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor + +logger = logging.getLogger() + + +class MemoryAnalyzer(BaseAnalyzer): + dataset_cls_list = [ScheduleAnalysisDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = ScheduleAnalysisDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) + def optimize(self, **kwargs): + memory_checker = MemoryOpsChecker() + memory_checker.check_memory_ops(self.dataset) + memory_checker.make_record(self.result) + memory_checker.make_render(self.html_render, priority=self.get_priority(memory_checker.max_mem_op_dur)) + return self.result + + def get_priority(self, max_mem_op_dur): + step_duration = getattr(self.dataset, "step_duration", None) + ratio = self.get_priority_by_time_ratio(max_mem_op_dur, step_duration) + + if step_duration is None: + return PriorityBackgroundColor.low + + return ratio diff --git a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py index 0e62a3ff0..58b2c301b 100644 --- a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py +++ b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py @@ -16,26 +16,26 @@ # limitations under the License. import logging - from profiler.advisor.common import constant as const from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.result.result import OptimizeResult from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() class OpDispatchAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] """ operator dispatch optimizer """ def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.dataset = self.get_first_data_by_key(self.dataset_list, key) self.result = OptimizeResult() self.html_render = HTMLRender() @@ -54,21 +54,21 @@ class OpDispatchAnalyzer(BaseAnalyzer): self.make_render(self.html_render) return self.result - def get_op_compile_info(self, event_dataset: TimelineEventDataset): - """ - :Param event_dataset: dataset of timeline event - """ - if hasattr(event_dataset, "ops_compile"): - self._op_compile = getattr(event_dataset, "ops_compile") - if not self._op_compile or self._op_compile.total_count < const.MAX_OP_COMPILE_NUM: - return + def get_op_compile_info(self, event_dataset: ScheduleAnalysisDataset): + """ + :Param event_dataset: dataset of timeline event + """ + if hasattr(event_dataset, "ops_compile"): + self._op_compile = getattr(event_dataset, "ops_compile") + if not self._op_compile or self._op_compile.total_count < const.MAX_OP_COMPILE_NUM: + return - self._issues_record.append(['operator dispatch', - const.OP_COMPILE_ID, - self._op_compile.total_count, - self._op_compile.total_time]) - else: - logger.debug("Skip operator compile checker, because no op_compile attr find.") + self._issues_record.append(['operator dispatch', + const.OP_COMPILE_ID, + self._op_compile.total_count, + self._op_compile.total_time]) + else: + logger.debug("Skip operator compile checker, because no op_compile attr find.") def make_record(self, result: OptimizeResult): """ @@ -77,8 +77,9 @@ class OpDispatchAnalyzer(BaseAnalyzer): if not self._op_compile or len(self._issues_record) <= 0: return desc = f"Found {self._op_compile.total_count} operator compile issues." - suggestion = (f"Please use `torch_npu.npu.set_compile_mode(jit_compile=False)` to disable jit compile " - f"in dynamic shape usage.") + suggestion = ("Please place the following code at the entrance of the python script to disable jit compile. " \ + "Code: `torch_npu.npu.set_compile_mode(jit_compile=False); " + "torch_npu.npu.config.allow_internal_format = False`") self.optimization_item.append(OptimizeItem("Operator dispatch", desc, [suggestion])) for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) @@ -87,7 +88,7 @@ class OpDispatchAnalyzer(BaseAnalyzer): for op_info in self._issues_record: result.add_detail('operator dispatch', detail=op_info) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): issues = [] optimizations = [] for optimization in self.optimization_item: @@ -97,11 +98,20 @@ class OpDispatchAnalyzer(BaseAnalyzer): )) for record in self._issues_record: issues.append(dict(issue=record[0], - op_name=record[1], - counts=record[2], - total_time=record[3])) + op_name=record[1], + counts=record[2], + total_time=record[3])) html_render.render_template(key="schedule", template_dir="templates", template_name="operator_dispatch.html", issues=issues, - optimizers=optimizations) + optimizers=optimizations, + priority_background_color=self.get_priority()) + + def get_priority(self): + step_duration = getattr(self.dataset, "step_duration", None) + op_compile_total_dur = getattr(self._op_compile, "total_time", None) + if step_duration is None or op_compile_total_dur is None: + return PriorityBackgroundColor.low + + return self.get_priority_by_time_ratio(op_compile_total_dur, step_duration) diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index c1eb24b8e..305d23994 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -8,25 +8,29 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.common import constant as const from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import format_timeline_result from profiler.advisor.common.timeline.fusion_ops_db import init_timeline_ops_db +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() class TimelineFusionOpsAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs): super().__init__(collection_path, n_processes, **kwargs) self._matched_op_index = {} if self.n_processes <= 1 else multiprocessing.Manager().dict() self.matched_op_stacks = {} self.empty_stacks = True - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) + def get_priority(self): + return PriorityBackgroundColor.low + def optimize(self, **kwargs): for mode in [const.ATEN.lower(), const.OPTIMIZER.lower()]: @@ -154,8 +158,9 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): timeline_profiling_doc_url=const.TIMELINE_WITH_STACK_DOC_URL ) + sheet_name = "Affinity apis" optimization_item = OptimizeItem( - SupportedScopes.TIMELINE_FUSION_OPS, + sheet_name, desc, [suggestion] ) @@ -163,16 +168,16 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): self.result.add(OptimizeRecord(optimization_item)) record_title = ["Affinity API", "Code stacks", "Stack called counts"] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, headers=record_title) + self.result.add_detail(sheet_name, headers=record_title) for api_name, stacks_info in format_timeline_result(self.matched_op_stacks).items(): if not stacks_info: detail = [api_name, "null", "null"] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, detail=detail) + self.result.add_detail(sheet_name, detail=detail) else: for stack in stacks_info: detail = [api_name, *stack] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, detail=detail) + self.result.add_detail(sheet_name, detail=detail) def make_render(self): format_result_for_html = format_timeline_result(dict(self.matched_op_stacks), dump_html=True) @@ -185,7 +190,8 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): empty_stacks=self.empty_stacks, with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, api_doc_url=const.TIMELINE_API_DOC_URL, - result=format_result_for_html) + result=format_result_for_html, + priority_background_color=self.get_priority()) def query_stack(self, event_dataset): if all([len(matched_index) == 0 for matched_index in self._matched_op_index.values()]): diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py b/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py index f684a4892..92425910b 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py @@ -3,7 +3,7 @@ from typing import List from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import get_analyze_processes, ParallelJob @@ -21,7 +21,8 @@ class OpStackFinder: self.task_type = None self.matched_index = set() - def get_api_stack_by_op(self, event_dataset: TimelineEventDataset, op_name: List[str] = None, task_type: str = None, + def get_api_stack_by_op(self, event_dataset: ComputationAnalysisDataset, op_name: List[str] = None, + task_type: str = None, disable_multiprocess=False): """ :Param event_dataset: dataset of timeline event @@ -82,7 +83,13 @@ class OpStackFinder: for op_info in self._stack_record: result.add_detail('operator stacks', detail=op_info) - def _get_api_stack_by_op(self, event_dataset: TimelineEventDataset, op_name: str, task_type: str): + def query_stack(self, event_dataset: ComputationAnalysisDataset): + + if not event_dataset.dataset_len: + return + _ = event_dataset.parse_data_with_generator(self._query_stack_by_matched_index) + + def _get_api_stack_by_op(self, event_dataset: ComputationAnalysisDataset, op_name: str, task_type: str): for _, src_op_event in event_dataset.ops_with_task_type.items(): op_task_type = src_op_event.get(const.TASK_TYPE) @@ -110,6 +117,7 @@ class OpStackFinder: task_id = src_op_event.task_id if not task_id: continue + self.matched_index.add(dst_op_index) if dst_op_index not in self._task_id_record: self._task_id_record[dst_op_index] = [] @@ -122,7 +130,7 @@ class OpStackFinder: if not dst_op_event: return const.TIMELINE_BACKWARD_NO_STACK_CODE - return dst_op_event.get("dataset_index") + return int(dst_op_event.get("dataset_index")) def _query_index_by_acl_to_npu(self, acl_to_npu_event): if acl_to_npu_event: @@ -148,6 +156,7 @@ class OpStackFinder: return None event = TimelineEvent(event) stack = event.args.get(const.CALL_STACKS) + stack = stack if stack else const.NO_STACK_REASON_MAP.get(const.TIMELINE_BACKWARD_NO_STACK_CODE) for matched_op_info in self._task_id_record.get(index, []): self._stack_record.append([*matched_op_info, stack]) @@ -156,8 +165,3 @@ class OpStackFinder: self._stack_record.append([*matched_op_info, const.NO_STACK_REASON_MAP.get(const.TIMELINE_ACL_TO_NPU_NO_STACK_CODE)]) return None - - def query_stack(self, event_dataset: TimelineEventDataset): - if not event_dataset.dataset_len: - return - _ = event_dataset.parse_data_with_generator(self._query_stack_by_matched_index) diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py index 2786a7840..df8c22fa5 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py @@ -1,30 +1,32 @@ import logging -from typing import List, Dict, Any - from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() class SyncBNAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, **kwargs): super().__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): syncbn_checker = SyncBNChecker() syncbn_checker.check_syncbn(self.timeline_event_dataset) syncbn_checker.make_record(self.result) - syncbn_checker.make_render(self.html_render) + syncbn_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + def get_priority(self): + return PriorityBackgroundColor.high \ No newline at end of file diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py index c0e10448f..e83a15491 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py @@ -1,7 +1,7 @@ import logging import os -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.cluster_analyse.common_func.file_manager import FileManager @@ -20,7 +20,7 @@ class SyncBNChecker: self.max_syncbn_num = None self._init_rule() - def check_syncbn(self, event_dataset: TimelineEventDataset): + def check_syncbn(self, event_dataset: ScheduleAnalysisDataset): """ :Param event_dataset: dataset of timeline event """ @@ -43,14 +43,17 @@ class SyncBNChecker: for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.syncbn_issues: return + + priority = kwargs.get("priority") html_render.render_template(key="schedule", template_dir="templates", template_name="sync_batchnorm.html", desc=self.desc, - solutions=self.solutions) + solutions=self.solutions, + priority_background_color=priority) def _init_rule(self): syncbn_rule_path = os.path.join( diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py index d8906504c..61ec7d1fa 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py @@ -5,28 +5,33 @@ from typing import List, Dict, Any from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() class SynchronizeStreamAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, **kwargs): super().__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): synchronize_stream_checker = SynchronizeStreamChecker() synchronize_stream_checker.check_synchronize(self.timeline_event_dataset, kwargs.get("profiling_with_stack")) synchronize_stream_checker.make_record(self.result) - synchronize_stream_checker.make_render(self.html_render) + synchronize_stream_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + + def get_priority(self): + return PriorityBackgroundColor.low \ No newline at end of file diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py index 83ddd80a0..7af46f766 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py @@ -1,7 +1,7 @@ import logging from profiler.advisor.common import constant as const -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.analyzer.schedule.timeline_base_checker import TimelineBaseChecker @@ -21,7 +21,7 @@ class SynchronizeStreamChecker(TimelineBaseChecker): self.solutions = [] self.max_synchronize_num = None - def check_synchronize(self, event_dataset: TimelineEventDataset, profiling_with_stack=None): + def check_synchronize(self, event_dataset: ScheduleAnalysisDataset, profiling_with_stack=None): """ :Param event_dataset: dataset of timeline event """ @@ -73,10 +73,10 @@ class SynchronizeStreamChecker(TimelineBaseChecker): for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.synchronize_issues: return - + priority = kwargs.get("priority") format_result_for_html = format_timeline_result(dict(self.matched_op_stacks), dump_html=True) html_render.render_template(key="schedule", template_dir="templates", @@ -86,4 +86,5 @@ class SynchronizeStreamChecker(TimelineBaseChecker): result=format_result_for_html, with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, empty_stacks=self.empty_stacks, - framework_black_list=self.framework_black_list) + framework_black_list=self.framework_black_list, + priority_background_color=priority) diff --git a/profiler/advisor/analyzer/schedule/timeline_base_checker.py b/profiler/advisor/analyzer/schedule/timeline_base_checker.py index 8bc691502..f481733d4 100644 --- a/profiler/advisor/analyzer/schedule/timeline_base_checker.py +++ b/profiler/advisor/analyzer/schedule/timeline_base_checker.py @@ -4,7 +4,7 @@ import logging from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult logger = logging.getLogger() @@ -19,19 +19,11 @@ class TimelineBaseChecker(ABC): self.empty_stacks = True self.framework_black_list = False - @abstractmethod - def make_record(self, result: OptimizeResult): - pass - - @abstractmethod - def make_render(self, html_render): - pass - - def query_stack(self, event_dataset: TimelineEventDataset = None, profiling_with_stack: str = None): + def query_stack(self, event_dataset: ScheduleAnalysisDataset = None, profiling_with_stack: str = None): if all([len(matched_index) == 0 for matched_index in self._matched_op_index.values()]): return - event_dataset = event_dataset if not profiling_with_stack else TimelineEventDataset( + event_dataset = event_dataset if not profiling_with_stack else ScheduleAnalysisDataset( collection_path=profiling_with_stack, data={}, _datasets={}, analysis_mode="fusion_ops", build_dataset=False) diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 652e10b08..72b8dd3df 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -33,4 +33,6 @@ class SupportedScopes: SYNCBN = "syncbn" SYNCHRONIZE_STREAM = "synchronize_stream" FREQ_ANALYSIS = "freq_analysis" + MEMORY = "memory" + STAGE_COMPUTE = "stage_compute" GC_ANALYSIS = "gc_analysis" diff --git a/profiler/advisor/common/async_analysis_status.py b/profiler/advisor/common/async_analysis_status.py new file mode 100644 index 000000000..f67ca235a --- /dev/null +++ b/profiler/advisor/common/async_analysis_status.py @@ -0,0 +1,7 @@ +class AsyncAnalysisStatus: + FAILED = "failed" + SUCCESS = "success" + ANALYZING = "analyzing" + + FAILED_STATUS_CODE = 400 + NON_FAILED_STATUS_CODE = 200 diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index c97cfbfd1..298e94fc1 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -33,6 +33,7 @@ TASK_TYPE = "Task Type" CPU_OP = "cpu_op" AI_CORE = "AI_CORE" AI_CPU = "AI_CPU" +MIX_AIC = "MIX_AIC" CALL_STACKS = "Call stack" INPUT_DIMS = "Input Dims" OP_SEP = "-" @@ -48,8 +49,7 @@ NO_STACK_REASON_MAP = { TIMELINE_BACKWARD_NO_STACK_CODE: "Backward broadcast, without call stacks in profiling.", TIMELINE_ACL_TO_NPU_NO_STACK_CODE: "Incoming flow is 'acl_to_npu', without call stacks in profiling." } -TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/"\ - "Samples%20of%20Fused%20Operator%20API%20Replacement.md" +TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" AFFINITY_TRAINING_API = "Affinity training api" TIMELINE_WITH_STACK_DOC_URL = "https://www.hiascend.com/document/detail/zh/canncommercial/" \ "70RC1/modeldevpt/ptmigr/AImpug_0067.html" @@ -124,20 +124,6 @@ MAX_RETRIES = 3 TIMEOUT = 3 ADVISOR_RULE_PATH = "ADVISOR_RULE_PATH" -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. CLOUD_RULE_PATH = "rules/cloud/" DEFAULT_RULE_PATH = "./rules/" @@ -156,7 +142,17 @@ COMMUNICATION_JSON = "communication.json" BOTTLENECK = "bottleneck" DATA = "data" - +ADVISOR_ANALYSIS_OUTPUT_DIR = "advisor_analysis_result" +DEFAULT_PROCESSES = 8 +CLUSTER_ANALYSIS_FILE_PATTERN = [r'profiler_info_\d+\.json', "step_trace_time.csv", "communication.json", + "communication_matrix.json"] +ANALYSIS_OUTPUT_PATH = "ANALYSIS_OUTPUT_PATH" +DEFAULT_RANK_FOR_PROFILING_ANALYSIS = 0 +PROFILER_INFO_FILE_PATTERN = r"profiler_info_(\d+)\.json" +DISABLE_STREAMINIG_READER = "DISABLE_STREAMINIG_READER" FRAMEWORK_STACK_BLACK_LIST = ["torch", "torch_npu", "megatron", "deepspeed"] DISABLE_STREAMING_READER = "DISABLE_STREAMING_READER" -MAX_FILE_SIZE = 10**10 +MAX_FILE_SIZE = 10 ** 10 +MAX_NUM_PROCESSES = 4 +DEFAULT_STEP = "-1" +STEP_RANK_SEP = "_" diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py deleted file mode 100644 index 3d8e22b7c..000000000 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py +++ /dev/null @@ -1,65 +0,0 @@ -import unittest -import os -import sys -import yaml - -from profiler.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestDataloaderChecker(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - def setUp(self) -> None: - rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), - "advisor", "rules", "dataloader.yaml") - - with open(rule_path, "rb") as file: - self.rule = yaml.safe_load(file) - - def test_no_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) * 1000 - dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=True) - - checker = DataloaderChecker() - checker.check_slow_dataloader(dataset) - self.assertFalse(checker.dataloader_issues) - - def test_no_slow_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) * 1000 - dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=False) - checker = DataloaderChecker() - checker.check_slow_dataloader(dataset) - self.assertFalse(checker.dataloader_issues) - - def test_found_slow_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") + 1) * 1000 - dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=False) - checker = DataloaderChecker() - checker.check_slow_dataloader(dataset) - self.assertTrue(checker.dataloader_issues) - - desc = self.rule.get("problem").format(dataloader_duration=dataloader_duration / 1000, - dataloader_duration_threshold=self.rule.get( - "dataloader_duration_threshold")) - - self.assertEqual(desc, checker.desc) - - def _get_mock_dataset(self, dur, is_empty_dataset=False): - dataset = TimelineEvent() - if is_empty_dataset: - return dataset - - dataset["dataloader"] = [TimelineEvent({"dur": dur, "name": "dataloader"})] - return dataset - - -if __name__ == '__main__': - tester = TestDataloaderChecker() - tester.test_no_dataloader() - tester.test_no_slow_dataloader() - tester.test_found_slow_dataloader() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py deleted file mode 100644 index d1df810a0..000000000 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py +++ /dev/null @@ -1,62 +0,0 @@ -import unittest -import os -import sys -import yaml - -from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestSyncBNChecker(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - def setUp(self) -> None: - rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), - "advisor", "rules", "sync_batchnorm.yaml") - - with open(rule_path, "rb") as file: - self.rule = yaml.safe_load(file) - - def test_no_syncbn(self): - dataset = self._get_mock_dataset(1, is_empty_dataset=True) - - checker = SyncBNChecker() - checker.check_syncbn(dataset) - self.assertFalse(checker.syncbn_issues) - - def test_syncbn_not_reach_threshold(self): - dataset = self._get_mock_dataset(self.rule.get("max_syncbn_num") - 1, is_empty_dataset=False) - checker = SyncBNChecker() - checker.check_syncbn(dataset) - self.assertFalse(checker.syncbn_issues) - - def test_found_slow_dataloader(self): - dataset = self._get_mock_dataset(self.rule.get("max_syncbn_num") + 1, is_empty_dataset=False) - checker = SyncBNChecker() - checker.check_syncbn(dataset) - self.assertTrue(checker.syncbn_issues) - - desc = self.rule.get("problem").format(syncbn_num=self.rule.get("max_syncbn_num") + 1) - - self.assertEqual(desc, checker.desc) - - def _get_mock_dataset(self, syncbn_num, is_empty_dataset=False): - dataset = TimelineEvent() - if is_empty_dataset: - return dataset - - dataset["sync_batchnorm"] = [] - for _ in range(syncbn_num): - dataset["sync_batchnorm"].append(TimelineEvent({"name": "SyncBatchNorm"})) - return dataset - - -if __name__ == '__main__': - tester = TestSyncBNChecker() - tester.test_no_syncbn() - tester.test_syncbn_not_reach_threshold() - tester.test_found_slow_dataloader() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py deleted file mode 100644 index 360363ce3..000000000 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -import os -import sys -import yaml - -from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestSynchronizeChecker(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - def setUp(self) -> None: - rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), - "advisor", "rules", "synchronize.yaml") - - with open(rule_path, "rb") as file: - self.rule = yaml.safe_load(file) - - def test_no_synchronize_stream(self): - dataset = self._get_mock_dataset(1, [], is_empty_dataset=True) - - checker = SynchronizeStreamChecker() - checker.check_synchronize(dataset) - self.assertFalse(checker.synchronize_issues) - - def test_max_synchronize_stream(self): - dataset = self._get_mock_dataset(100, [], is_empty_dataset=False) - checker = SynchronizeStreamChecker() - checker.check_synchronize(dataset) - self.assertFalse(checker.synchronize_issues) - - def _get_mock_dataset(self, total_count, slow_synchronize_stream, is_empty_dataset=False): - dataset = TimelineEvent() - if is_empty_dataset: - return dataset - - dataset["synchronize_stream"] = TimelineEvent( - dict( - total_count=total_count, - slow_synchronize_stream=slow_synchronize_stream, - rule=dict(max_synchronize_num=10, problem="", solutions=[]), - ) - ) - return dataset - - -if __name__ == '__main__': - tester = TestSynchronizeChecker() - tester.test_no_synchronize_stream() - tester.test_max_synchronize_stream() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py deleted file mode 100644 index 9060bfb8d..000000000 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py +++ /dev/null @@ -1,46 +0,0 @@ -import unittest -import os -import sys - -work_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))))) -sys.path.insert(0, work_path) -from unittest.mock import patch -from profiler.advisor.analyzer.schedule import dispatch -from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset -from profiler.advisor.display.html.render import HTMLRender -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestOperatorDispatchAnalyzer(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - @patch("profiler.advisor.common.constant.MAX_OP_COMPILE_NUM", 5) - def test_ops_dispatch_analyzer(self): - kwargs = {"analysis_mode": "all"} - data_root_dir = os.path.dirname(os.path.realpath(__file__)) - op_dispatch_analyzer = OpDispatchAnalyzer(data_root_dir, **kwargs) - - results = op_dispatch_analyzer.optimize(**kwargs) - self.assertTrue(results.page_dict) - self.assertIsNotNone(results.sheet_recorder.sheet_data.get("operator dispatch")) - - @patch("profiler.advisor.common.constant.MAX_OP_COMPILE_NUM", 5) - def test_ops_dispatch_make_render(self): - kwargs = {"analysis_mode": "timeline"} - data_root_dir = os.path.dirname(os.path.realpath(__file__)) - op_dispatch = OpDispatchAnalyzer(data_root_dir, **kwargs) - event_dataset = op_dispatch.get_first_data_by_key(op_dispatch.dataset_list, TimelineEventDataset.get_key()) - - op_dispatch.get_op_compile_info(event_dataset) - html_render = HTMLRender() - op_dispatch.make_render(html_render) - self.assertTrue(len(html_render.render_list) >= 1) - - -if __name__ == '__main__': - tester = TestOperatorDispatchAnalyzer() - tester.test_ops_dispatch_analyzer() - tester.test_ops_dispatch_make_render() diff --git a/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py b/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py deleted file mode 100644 index eb383a659..000000000 --- a/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py +++ /dev/null @@ -1,170 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes - - -class TestRdmaAdvice(unittest.TestCase): - TMP_DIR = "./tmp/" - OUTPUT_DIR = "./tmp/cluster_analysis_output" - interface = None - err_interface = None - - def tearDown(self): - if os.path.exists(TestRdmaAdvice.TMP_DIR): - shutil.rmtree(TestRdmaAdvice.TMP_DIR) - if os.path.exists(TestRdmaAdvice.OUTPUT_DIR): - shutil.rmtree(TestRdmaAdvice.OUTPUT_DIR) - self.clear_htmls() - - def setUp(self): - if os.path.exists(TestRdmaAdvice.TMP_DIR): - shutil.rmtree(TestRdmaAdvice.TMP_DIR) - if not os.path.exists(TestRdmaAdvice.TMP_DIR): - os.makedirs(TestRdmaAdvice.TMP_DIR) - if not os.path.exists(TestRdmaAdvice.OUTPUT_DIR): - os.makedirs((TestRdmaAdvice.OUTPUT_DIR)) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“mstt”开头 - if filename.startswith("mstt"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def get_cluster_communication_view(cls): - data = {"p2p":{"step1" : { - "hcom_broadcast__844_0_1@13681369207305868844": { - "0": { - "Communication Time Info": { - "Start Timestamp(us)": 1713174287354248.0, - "Elapse Time(ms)": 4688, - "Transit Time(ms)": 0, - "Wait Time(ms)": 0.01162, - "Synchronization Time(ms)": 0.01162, - "Idle Time(ms)": 39.0606, - "Wait Time Ratio": 1.0, - "Synchronization Time Ratio": 1.0 - }, - "Communication Bandwidth Info": { - "RDMA": { - "Transit Size(MB)": 80, - "Transit Time(ms)": 4600, - "Bandwidth(GB/s)": 0.003, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "HCCS": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "PCIE": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SDMA": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SIO": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - } - } - }, - "16": { - "Communication Time Info": { - "Start Timestamp(us)": 1713174287186619.8, - "Elapse Time(ms)": 4788, - "Transit Time(ms)": 0.0013, - "Wait Time(ms)": 39.037240000000004, - "Synchronization Time(ms)": 39.03034, - "Idle Time(ms)": 167.66008000000002, - "Wait Time Ratio": 1.0, - "Synchronization Time Ratio": 1.0 - }, - "Communication Bandwidth Info": { - "RDMA": { - "Transit Size(MB)": 80, - "Transit Time(ms)": 4700, - "Bandwidth(GB/s)": 0.0033, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "HCCS": { - "Transit Size(MB)": 4e-05, - "Transit Time(ms)": 0.0013, - "Bandwidth(GB/s)": 0.0308, - "Large Packet Ratio": 0.0, - "Size Distribution": { - "4e-05": [ - 1, - 0.0013 - ] - } - }, - "PCIE": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SDMA": { - "Transit Size(MB)": 4e-05, - "Transit Time(ms)": 0.0013, - "Bandwidth(GB/s)": 0.0308, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SIO": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - } - } - }, - } - }}} - return data - - @classmethod - def create_communicaton_json(cls): - raw_data = cls.get_cluster_communication_view() - with os.fdopen(os.open(f"{TestRdmaAdvice.OUTPUT_DIR}/cluster_communication.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_contain_cluster_communication_json(self): - self.create_communicaton_json() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "cluster" - scope = SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []))) - self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []).get('data'))) - result.clear() diff --git a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py b/profiler/test/ut/advisor/communication_advice/test_packet_advice.py deleted file mode 100644 index a8fd4549e..000000000 --- a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py +++ /dev/null @@ -1,175 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes - - -class TestPacketAdvice(unittest.TestCase): - TMP_DIR = "./ascend_pt" - OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" - interface = None - err_interface = None - - def tearDown(self): - if os.path.exists(TestPacketAdvice.TMP_DIR): - shutil.rmtree(TestPacketAdvice.TMP_DIR) - self.clear_htmls() - - def setUp(self): - if os.path.exists(TestPacketAdvice.TMP_DIR): - shutil.rmtree(TestPacketAdvice.TMP_DIR) - if not os.path.exists(TestPacketAdvice.TMP_DIR): - os.makedirs(TestPacketAdvice.TMP_DIR) - if not os.path.exists(TestPacketAdvice.OUTPUT_DIR): - os.makedirs(TestPacketAdvice.OUTPUT_DIR) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“att”开头 - if filename.startswith("mstt"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def get_communication_view(cls): - data = {"step1":{"collective" : { - "hcom_broadcast__844_1_1@13681369207305868844": { - "Communication Time Info": { - "Start Timestamp(us)": 1713174287407957.0, - "Elapse Time(ms)": 0.06086, - "Transit Time(ms)": 0.00126, - "Wait Time(ms)": 0.014939999999999998, - "Synchronization Time(ms)": 0.00714, - "Idle Time(ms)": 0.044660000000000005, - "Wait Time Ratio": 0.9222, - "Synchronization Time Ratio": 0.85 - }, - "Communication Bandwidth Info": { - "RDMA": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "HCCS": { - "Transit Size(MB)": 0.028575999999999997, - "Transit Time(ms)": 0.008620000000000001, - "Bandwidth(GB/s)": 3.3151, - "Large Packet Ratio": 0.0, - "Size Distribution": { - "0.004224": [ - 6, - 0.00736 - ], - "0.003232": [ - 1, - 0.00126 - ] - } - }, - "PCIE": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SDMA": { - "Transit Size(MB)": 0.028575999999999997, - "Transit Time(ms)": 0.008620000000000001, - "Bandwidth(GB/s)": 3.3151, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SIO": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - } - } - }, - "hcom_allReduce__844_2_1@13681369207305868844": { - "Communication Time Info": { - "Start Timestamp(us)": 1713174287432401.2, - "Elapse Time(ms)": 2.9042, - "Transit Time(ms)": 1.35236, - "Wait Time(ms)": 1.47632, - "Synchronization Time(ms)": 1.44524, - "Idle Time(ms)": 0.07551999999999981, - "Wait Time Ratio": 0.5219, - "Synchronization Time Ratio": 0.5166 - }, - "Communication Bandwidth Info": { - "RDMA": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "HCCS": { - "Transit Size(MB)": 176.16076799999996, - "Transit Time(ms)": 9.55658, - "Bandwidth(GB/s)": 18.4335, - "Large Packet Ratio": 0.0, - "Size Distribution": { - "12.582912": [ - 14, - 9.55658 - ] - } - }, - "PCIE": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SDMA": { - "Transit Size(MB)": 176.16076799999996, - "Transit Time(ms)": 9.55658, - "Bandwidth(GB/s)": 18.4335, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SIO": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - } - } - }, - }}} - return data - - @classmethod - def create_communicaton_json(cls): - raw_data = cls.get_communication_view() - with os.fdopen(os.open(f"{TestPacketAdvice.OUTPUT_DIR}/communication.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_ascend_pt_contain_communication_json(self): - self.create_communicaton_json() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "communication" - scope = SupportedScopes.PACKET - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("Packet Analysis", []))) - self.assertEqual(1, len(result.data.get("Packet Analysis", []).get('data'))) - result.clear() diff --git a/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py b/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py deleted file mode 100644 index 51acf3b8e..000000000 --- a/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes - - -class TestFrequencyAdvice(unittest.TestCase): - TMP_DIR = "./ascend_pt" - OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" - DEVICE_DIR = "./ascend_pt/PROF_000001_20240415174447255_OAANHDOMMJMHGIFC/device_0" - interface = None - err_interface = None - - def tearDown(self): - if os.path.exists(TestFrequencyAdvice.TMP_DIR): - shutil.rmtree(TestFrequencyAdvice.TMP_DIR) - self.clear_htmls() - - def setUp(self): - if os.path.exists(TestFrequencyAdvice.TMP_DIR): - shutil.rmtree(TestFrequencyAdvice.TMP_DIR) - if not os.path.exists(TestFrequencyAdvice.TMP_DIR): - os.makedirs(TestFrequencyAdvice.TMP_DIR) - if not os.path.exists(TestFrequencyAdvice.OUTPUT_DIR): - os.makedirs(TestFrequencyAdvice.OUTPUT_DIR) - if not os.path.exists(TestFrequencyAdvice.DEVICE_DIR): - os.makedirs(TestFrequencyAdvice.DEVICE_DIR) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“att”开头 - if filename.startswith("att"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def get_basic_trace_view(cls): - # Python pid - py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} - # ascend pid - ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} - # ascend pid - cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} - # ascend hardware ops - ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - # flow event - flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} - flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} - return [py_pid_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, flow_event_s, flow_event_e] - - @classmethod - def create_info_json(cls): - info = { - "DeviceInfo": [ - { - "id": 7, - "env_type": 3, - "ctrl_cpu_id": "ARMv8_Cortex_A55", - "ctrl_cpu_core_num": 1, - "ctrl_cpu_endian_little": 1, - "ts_cpu_core_num": 0, - "ai_cpu_core_num": 6, - "ai_core_num": 25, - "ai_cpu_core_id": 2, - "ai_core_id": 0, - "aicpu_occupy_bitmap": 252, - "ctrl_cpu": "0", - "ai_cpu": "2,3,4,5,6", - "aiv_num": 50, - "hwts_frequency": "49.999001", - "aic_frequency": "1850", - "aiv_frequency": "1850" - } - ] - } - with os.fdopen(os.open(f"{TestFrequencyAdvice.DEVICE_DIR}/info.json.0", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(info)) - - @classmethod - def create_non_910B_trace_view(cls): - basic_info = cls.get_basic_trace_view() - - # python ops - py_event1 = {"ph": "X", "cat": "python_function", "name": "aten::slice", "ts": "200", "dur": 100, "tid": 2, - "pid": 1, - "args": {"Call stack": "/root/test/slice.py(116);\r\n/root/torch/module.py"}} - py_event2 = {"ph": "X", "cat": "python_function", "name": "slice", "ts": "199", "dur": 200, "tid": 2, "pid": 1, - "args": {"Call stack": "/root/test/slice.py(116);\r\n/root/torch/module.py"}} - raw_data = [ - *basic_info, py_event1, py_event2 - ] - with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/trace_view.json", - # with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/msprof_20240415174455.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - @classmethod - def create_910B_trace_view(cls): - basic_info = cls.get_basic_trace_view() - - # python ops - py_event1 = {"name": "AI Core Freq", "ts": "1699529623106000.061", "pid": 682820896, "tid": 0, - "args": {"MHz": 1850}, "ph": "C"} - py_event2 = {"name": "AI Core Freq", "ts": "1699529623106770.541", "pid": 682820896, "tid": 0, - "args": {"MHz": 800}, "ph": "C"} - raw_data = [ - *basic_info, py_event1, py_event2 - ] - - with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/trace_view.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_msprof_not_contain_frequency_data(self): - self.create_info_json() - self.create_non_910B_trace_view() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "computation" - scope = SupportedScopes.FREQ_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(0, len(result.data.get("AI Core Frequency", []))) - result.clear() - - def test_run_should_run_success_when_trace_view_contain_frequency_data(self): - self.create_info_json() - self.create_910B_trace_view() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "computation" - scope = SupportedScopes.FREQ_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("AI Core Frequency", dict).get("data", []))) - result.clear() diff --git a/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py b/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py deleted file mode 100644 index f18311ab1..000000000 --- a/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py +++ /dev/null @@ -1,116 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset - - -class TestGcAdvice(unittest.TestCase): - TMP_DIR = "./ascend_pt" - OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" - interface = None - - def tearDown(self): - if os.path.exists(TestGcAdvice.TMP_DIR): - shutil.rmtree(TestGcAdvice.TMP_DIR) - self.clear_htmls() - TimelineEventDataset.reset_all_instances() - - def setUp(self): - if os.path.exists(TestGcAdvice.TMP_DIR): - shutil.rmtree(TestGcAdvice.TMP_DIR) - if not os.path.exists(TestGcAdvice.TMP_DIR): - os.makedirs(TestGcAdvice.TMP_DIR) - if not os.path.exists(TestGcAdvice.OUTPUT_DIR): - os.makedirs(TestGcAdvice.OUTPUT_DIR) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“att”开头 - if filename.startswith("mstt"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def create_trace_view_with_gc_events(cls): - # Python pid - py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} - # Python GC pid - py_gc_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 2, "args": {"name": "Python GC"}} - # ascend pid - ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} - # ascend pid - cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} - # ascend hardware ops - ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - gc_event1 = {"ph": "X", "name": "GC", "ts": "1699529622103750", "dur": 1500, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - gc_event2 = {"ph": "X", "name": "GC", "ts": "1699529623104750", "dur": 50, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - gc_event3 = {"ph": "X", "name": "GC", "ts": "1699529623105750", "dur": 50000, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - # flow event - flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} - flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} - - raw_data = [ - py_pid_data, py_gc_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, gc_event1, gc_event2, - gc_event3, flow_event_s, flow_event_e - ] - with os.fdopen(os.open(f"{TestGcAdvice.OUTPUT_DIR}/trace_view.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - @classmethod - def create_trace_view_without_gc_events(cls): - # Python pid - py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} - # ascend pid - ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} - # ascend pid - cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} - # ascend hardware ops - ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - # flow event - flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} - flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} - - raw_data = [ - py_pid_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, flow_event_s, flow_event_e - ] - with os.fdopen(os.open(f"{TestGcAdvice.OUTPUT_DIR}/trace_view.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_trace_view_contain_gc_events(self): - self.create_trace_view_with_gc_events() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "schedule" - scope = SupportedScopes.GC_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("GcAnalysis", dict).get("data", []))) - result.clear() - - def test_run_should_run_success_when_trace_view_not_contain_gc_events(self): - self.create_trace_view_without_gc_events() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "schedule" - scope = SupportedScopes.GC_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(0, len(result.data.get("GcAnalysis", []))) - result.clear() -- Gitee From e7e9090b7729cef10b8266ea30bf6a4772522a33 Mon Sep 17 00:00:00 2001 From: wuyuhan Date: Wed, 21 Aug 2024 20:45:21 +0800 Subject: [PATCH 428/791] =?UTF-8?q?=E8=AE=A1=E7=AE=97,=E9=80=9A=E4=BF=A1,d?= =?UTF-8?q?ataloader=E7=AD=89analyzer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../advisor/analyzer/analyzer_controller.py | 557 ++++++++++++++++++ profiler/advisor/analyzer/base_analyzer.py | 55 +- .../analyzer/cluster/slow_rank_analyzer.py | 216 +++++++ .../base_communication_analyzer.py | 8 + .../analyzer/communication/packet/__init__.py | 0 .../{ => packet}/packet_analyzer.py | 18 +- .../{ => packet}/packet_checker.py | 9 +- .../communication/retransmission/__init__.py | 0 .../communication_retransmission_analyzer.py | 52 ++ .../communication_retransmission_checker.py | 129 ++++ .../ai_core_freq/ai_core_freq_analyzer.py | 22 +- .../ai_core_freq/ai_core_freq_checker.py | 34 +- .../computation/aicpu/aicpu_checker.py | 28 +- .../computation/bound/block_dim_checker.py | 14 +- .../bound/operator_bound_checker.py | 14 +- .../op_compile/dynamic_shape_checker.py | 17 +- .../analyzer/computation/operator_checker.py | 42 +- .../pp_stage_computation_analyzer.py | 106 ++++ .../computation/profiling_analyzer.py | 27 +- .../dataloader/dataloader_analyzer.py | 14 +- .../analyzer/dataloader/dataloader_checker.py | 12 +- .../graph_fusion/graph_fusion_analyzer.py | 21 +- profiler/advisor/analyzer/memory/__init__.py | 0 .../analyzer/memory/memory_analyzer.py | 38 ++ .../overall/environment_variable_analyzer.py | 4 + .../overall/overall_summary_analyzer.py | 5 +- .../dispatch/timeline_op_dispatch_analyzer.py | 60 +- .../fusion_ops/fusion_ops_analyzer.py | 22 +- .../fusion_ops/timeline_api_stack_checker.py | 22 +- .../schedule/syncbn/syncbn_analyzer.py | 16 +- .../schedule/syncbn/syncbn_checker.py | 11 +- .../synchronize_stream_analyzer.py | 15 +- .../synchronize_stream_checker.py | 11 +- .../schedule/timeline_base_checker.py | 14 +- profiler/advisor/common/analyzer_scopes.py | 2 + .../advisor/common/async_analysis_status.py | 7 + profiler/advisor/common/constant.py | 32 +- .../dataset/cluster/cluster_dataset.py | 50 +- .../cluster/cluster_step_trace_time_bean.py | 3 + .../advisor/dataset/timeline_event_dataset.py | 367 ++++-------- .../dataset/timeline_op_collector/__init__.py | 0 .../timeline_op_collector.py | 376 ++++++++++++ .../test_dataloader_checker.py | 65 -- .../timeline_advice/test_syncbn_checker.py | 62 -- .../test_synchronize_stream.py | 55 -- .../test_timeline_op_compile_checker.py | 46 -- .../test_rdma_retransmission_advice.py | 170 ------ .../test_packet_advice.py | 175 ------ .../compute_advice/test_frequency_advice.py | 145 ----- .../advisor/schedule_advice/test_gc_advice.py | 116 ---- 50 files changed, 1979 insertions(+), 1305 deletions(-) create mode 100644 profiler/advisor/analyzer/analyzer_controller.py create mode 100644 profiler/advisor/analyzer/cluster/slow_rank_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/base_communication_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/packet/__init__.py rename profiler/advisor/analyzer/communication/{ => packet}/packet_analyzer.py (74%) rename profiler/advisor/analyzer/communication/{ => packet}/packet_checker.py (96%) create mode 100644 profiler/advisor/analyzer/communication/retransmission/__init__.py create mode 100644 profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py create mode 100644 profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py create mode 100644 profiler/advisor/analyzer/memory/__init__.py create mode 100644 profiler/advisor/analyzer/memory/memory_analyzer.py create mode 100644 profiler/advisor/common/async_analysis_status.py create mode 100644 profiler/advisor/dataset/timeline_op_collector/__init__.py create mode 100644 profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py delete mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py delete mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py delete mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py delete mode 100644 profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py delete mode 100644 profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py delete mode 100644 profiler/test/ut/advisor/communication_advice/test_packet_advice.py delete mode 100644 profiler/test/ut/advisor/compute_advice/test_frequency_advice.py delete mode 100644 profiler/test/ut/advisor/schedule_advice/test_gc_advice.py diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py new file mode 100644 index 000000000..e8704542e --- /dev/null +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -0,0 +1,557 @@ +import copy +import logging +import json +import sys +import os +import multiprocessing as mp +from pathlib import Path +from multiprocessing import Manager + +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "compare_tools")) +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "cluster_analyse")) + +from profiler.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer +from profiler.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer +from profiler.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer +from profiler.advisor.config.config import Config +from profiler.advisor.common.analyzer_scopes import SupportedScopes +from profiler.advisor.common.async_analysis_status import AsyncAnalysisStatus +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterDataset +from profiler.advisor.utils.utils import Timer, safe_index, safe_division +from profiler.advisor.interface.interface import Interface +from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from profiler.prof_common.path_manager import PathManager + +logger = logging.getLogger() + + +class AnalyzerController: + CLUSTER_RANK_THRESHOLD = 2 + + def __init__(self): + self.dimensions = Interface.all_dimension + self.kwargs = {} + self.slow_rank_analyzer = None + self.slow_link_analyzer = None + self.cluster_local_data_map = {} + self.default_rank_id = None + self.rank_id_map = {} + self._is_cluster = False + self.analysis_process_resp = Manager().dict() + + @staticmethod + def _check_profiling_path_valid(profiling_path): + PathManager.input_path_common_check(profiling_path) + + if not Path(profiling_path).exists(): + logger.error("Profiling path is not existed. Invalid profiling path: %s", profiling_path) + return False + return True + + @staticmethod + def _get_step_rank_for_cluster_statistic_diff(target_cluster_statistic_data, benchmark_cluster_statistic_data, + headers, dimension, get_max=False): + if dimension not in headers: + logger.error("Error dimension %s for cluster statistics data, optionals are %s.", dimension, headers) + return None, None, None + + dimension_index = safe_index(headers, dimension) + diff_record = [] + # 对比目标profiling和benchmark profiling 每张卡的计算和下发和带宽，取计算、下发、带宽差异最大的卡进行下一步分析 + for target_row_data, benchmark_row_data in zip(target_cluster_statistic_data, benchmark_cluster_statistic_data): + target_data = safe_index(target_row_data, dimension_index) + benchmark_data = safe_index(benchmark_row_data, dimension_index) + if not isinstance(target_data, (int, float)) or not isinstance(benchmark_data, (int, float)): + continue + diff_record.append(target_data - benchmark_data) + + if SlowRankAnalyzer.compute_max_gap_ratio(diff_record, safe_division(sum(diff_record), len( + diff_record))) < SlowRankAnalyzer.RATIO_THRESHOLD: + return None, None, None + + value = max(diff_record) if get_max else min(diff_record) + value_index = safe_index(diff_record, value) + + step_value_index = safe_index(headers, "step") + rank_id_value_index = safe_index(headers, "rank_id") + step = safe_index(safe_index(target_cluster_statistic_data, value_index, []), step_value_index) + benchmark_step = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), step_value_index) + target_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) + benchmark_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) + + if target_rank_id != benchmark_rank_id: + logger.error( + "Rank ids of target profiling must keep the same as benchmark profiling, skip cluster comparison") + return None, None, None + + return step, benchmark_step, target_rank_id + + def do_analysis(self, dimensions, **kwargs): + pid = os.getpid() + resp = {"id": pid} + try: + self._do_analysis(dimensions, pid=pid, resp=resp, **kwargs) + except Exception as e: + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED, error_msg=str(e)) + logger.error(e) + raise RuntimeError(e) + + def async_do_analysis(self, dimensions, **kwargs): + # 异步分析，用于部署服务，通过接口查询异步作业状态 + async_analysis_process = mp.Process(target=self.do_analysis, args=(dimensions,), kwargs=kwargs, + name="Async advisor performance analysis") + async_analysis_process.start() + return async_analysis_process + + def get_response_by_pid(self, pid): + return self.analysis_process_resp.get(pid) + + def single_rank_analysis(self, profiling_path, benchmark_profiling_path=None): + job_list = [] + + profiling_path = self._get_profiling_path_by_rank(profiling_path) + benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) + + # 单卡场景无集群分析 + for dim in [Interface.CLUSTER]: + if dim in self.dimensions: + self.dimensions.remove(dim) + + for dimension in self.dimensions: + dimension_analysis_func_name = f"{dimension}_analysis" + if not hasattr(self, dimension_analysis_func_name): + continue + logger.info("Start %s analysis", dimension) + job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + + if benchmark_profiling_path: + # kernel/api 比对 + job_list += self._single_profiling_comparison(profiling_path, benchmark_profiling_path) + else: + # 单卡性能拆解 + self.overall(profiling_path) + return job_list + + def cluster_analysis(self, profiling_path, benchmark_profiling_path=None): + job_list = [] + + # 单集群profiling分析：下发、通信、计算、显存/内存 + for dimension in self.dimensions: + dimension_analysis_func_name = f"cluster_{dimension}_analysis" + if not hasattr(self, dimension_analysis_func_name): + continue + logger.info("Start cluster %s analysis", dimension) + job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + + if benchmark_profiling_path: + # 两个集群profiling比对分析 + job_list += self._cluster_profiling_comparison(profiling_path, benchmark_profiling_path) + else: + self.overall(profiling_path) + return job_list + + def overall(self, profiling_path): + from profiler.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariabelAnalyzer + env_analyzer = EnvironmentVariabelAnalyzer(profiling_path) + env_analyzer.optimize() + + if self._is_cluster: + self.slow_rank_analyzer.optimize(template_key=Interface.OVERALL) + self.slow_link_analyzer.optimize(template_key=Interface.OVERALL) + else: + from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer + overall_analyzer = OverallSummaryAnalyzer(profiling_path) + overall_analyzer.optimize() + + def schedule_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None): + # 任意单卡的下发分析 + + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.SCHEDULE]: + for scope in Interface.get_scope(dimension): + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def computation_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None, stage=None): + # 任意单卡的计算分析 + + kwargs = copy.deepcopy(self.kwargs) + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + kwargs["stage"] = stage + job_list = [] + + for dimension in [Interface.COMPUTATION]: + for scope in Interface.get_scope(dimension): + if scope == SupportedScopes.STAGE_COMPUTE: + continue + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def memory_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None): + # 任意单卡的内存分析 + + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.MEMORY]: + for scope in Interface.get_scope(dimension): + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def communication_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None, bandwidth_type=None): + + job_list = [] + supported_trans_type = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + if bandwidth_type is not None and bandwidth_type not in supported_trans_type: + logger.error("Error transit type %s, optionals are %s", bandwidth_type, supported_trans_type) + return job_list + + bandwidth_type_list = [bandwidth_type] if bandwidth_type is not None else supported_trans_type + + for bandwidth_type in bandwidth_type_list: + job_list += getattr(self, f"_communication_{bandwidth_type.lower()}_analysis")(profiling_path, + benchmark_profiling_path, + step, benchmark_step) + + return job_list + + def cluster_schedule_analysis(self, profiling_path): + # 目标集群profiling数据下发分析，不包含两个集群profiling数据的比对分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) + + info_msg = f"Maximum free for rank {slow_rank_id}" + if slow_step: + info_msg += f" and step {slow_step}" + logger.info(info_msg) + + job_list += self.schedule_analysis(analysis_profiling_path, step=slow_step) + return job_list + + def cluster_communication_analysis(self, profiling_path): + job_list = [] + + for dimension in [Interface.COMMUNICATION]: + for scope in Interface.get_scope(dimension): + analyzer_class = Interface.get_analyzer(dimension, scope) + if hasattr(analyzer_class, "requires_cluster_dataset") and getattr(analyzer_class, + "requires_cluster_dataset"): + + # 如果不依赖数据集，或者依赖的是ClusterDataset，则不用根据带宽确定需要分析的特定rank + kwargs = copy.deepcopy(self.kwargs) + kwargs["profiling_path"] = profiling_path + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + else: + # 非ClusterDataset场景，需要根据带宽大小分析特定的rank + for bandwidth_type in [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA]: + global_step_rank = self.slow_link_analyzer.get_global_step_rank(bandwidth_type) + # 获取带宽最小的卡进行分析 + target_rank_id = global_step_rank.get("minimum", {}).get("rank_id") or self.default_rank_id + step = global_step_rank.get("minimum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, target_rank_id) + + info_msg = f"Minimum {bandwidth_type} bandwidth for rank {target_rank_id} " + if step: + info_msg += f"and step {step}" + logger.info(info_msg) + + job_list += self.communication_analysis(analysis_profiling_path, step=step, + bandwidth_type=bandwidth_type) + + return job_list + + def cluster_computation_analysis(self, profiling_path): + # 目标集群profiling数据计算分析，不包含两个集群profiling数据的比对分析；如果有pp stage，则对不同stage进行计算分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.COMPUTE) + stage_step_rank = self.slow_rank_analyzer.get_stage_step_rank(SlowRankAnalyzer.COMPUTE) + + if stage_step_rank: + # 对不同pp stage取min max进行分析 + logger.info("Analysis steps and ranks of different pipeline parallel stages are %s", + json.dumps(stage_step_rank)) + + stages_profiling_path = [] + for stage, step_rank_info in stage_step_rank.items(): + rank_id = step_rank_info.get("maximum", {}).get("rank_id") + step = step_rank_info.get("maximum", {}).get("step") + + info_msg = f"For {stage}, slow rank is {rank_id}" + if step: + info_msg += f", step is {step}" + logger.info(info_msg) + + stages_profiling_path.append( + dict( + stage=stage, + rank_id=rank_id, + step=step, + profiling_path=self._get_profiling_path_by_rank(profiling_path, rank_id) + ) + ) + Interface.add_analyzer(Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, PPStageComputationAnalyzer) + kwargs = {"stages_profiling_path": stages_profiling_path, "profiling_path": profiling_path} + + job_list.append((Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, Interface(**kwargs), kwargs)) + else: + # 不区分stage，对所有卡取Min max进行分析 + logger.info("Without pipeline parallel stage, Global analysis steps and ranks is %s", + json.dumps(global_step_rank)) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + # 如果没有标杆profiling数据的rank id，说明没有快慢卡问题，直接对默认rank id进行分析，因此这里取值为None + fast_rank_id = global_step_rank.get("minimum", {}).get("rank_id") + fast_step = global_step_rank.get("minimum", {}).get("step") + + info_msg = f"Maximum computation time for rank {slow_rank_id}" + if slow_step: + info_msg += f" and step {slow_step}, " + if fast_rank_id: + info_msg += f"minimum computation time for rank {fast_rank_id}" + if fast_step: + info_msg += f" and step {fast_step}" + logger.info(info_msg) + + job_list += self.computation_analysis( + self._get_profiling_path_by_rank(profiling_path, slow_rank_id), + self._get_profiling_path_by_rank(profiling_path, fast_rank_id), + slow_step, + fast_step + ) + + return job_list + + def cluster_memory_analysis(self, profiling_path): + # 目标集群profiling数据内存分析，当前memory识别的两个算子，导致的问题都是大的free，因此选择FREE最慢的卡进行分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) + + info_msg = f"Maximum free for rank {slow_rank_id} " + if slow_step: + info_msg += f"and step {slow_step}" + logger.info(info_msg) + + job_list += self.memory_analysis(analysis_profiling_path, step=slow_step) + return job_list + + def _do_analysis(self, dimensions, **kwargs): + self.dimensions = dimensions + self.kwargs = kwargs + result_list = [] + profiling_path = self.kwargs.get("profiling_path") + benchmark_profiling_path = self.kwargs.get("benchmark_profiling_path") + pid = self.kwargs.get("pid") + resp = self.kwargs.get("resp") + + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.ANALYZING) + + if not self._check_profiling_path_valid(profiling_path): + error_msg = f"Got invalid argument '-d/--profiling_path' {profiling_path}, skip analysis" + self._update_analysis_process_resp(pid, resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + if benchmark_profiling_path and not self._check_profiling_path_valid(benchmark_profiling_path): + error_msg = f"Got invalid argument '-bp/--benchmark_profiling_path' {benchmark_profiling_path}, skip analysis" + self._update_analysis_process_resp(pid, resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + + self._is_cluster = self._is_cluster_profiling(profiling_path) + if not self._is_cluster: + job_list = self.single_rank_analysis(profiling_path, benchmark_profiling_path) + else: + job_list = self.cluster_analysis(profiling_path, benchmark_profiling_path) + + for i, (dimension, scope, interface, kwargs) in enumerate(job_list[::-1]): + result_list.append( + interface.get_result(dimension, scope, render_html=i == len(job_list) - 1, output_dict=False, + **kwargs) + ) + + for result in result_list[::-1]: + if result and hasattr(result, "show"): + result.show() + break + self._get_analysis_success_resp(pid, resp) + + def _communication_rdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None): + # 小包分析 + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.COMMUNICATION]: + for scope in Interface.get_scope(dimension): + if scope != SupportedScopes.PACKET: + continue + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + + return job_list + + def _communication_sdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None): + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + return job_list + + def _single_profiling_comparison(self, profiling_path, benchmark_profiling_path, step=None, + benchmark_step=None): + # TODO 基于compare tools 对比计算下发 + kwargs = copy.deepcopy(self.kwargs) + return [] + + def _cluster_profiling_comparison(self, profiling_path, benchmark_profiling_path): + # 从计算、下发和通信三个维度对集群profiling数据进行对比 + + job_list = [] + benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) + benchmark_slow_rank_analyzer = SlowRankAnalyzer(benchmark_profiling_path) + benchmark_slow_link_analyzer = SlowLinkAnalyzer(benchmark_profiling_path) + + # 计算和下发分析 + job_list += self._cluster_data_comparison(profiling_path, + benchmark_profiling_path, + self.slow_rank_analyzer, + benchmark_slow_rank_analyzer, + get_max=True) + + # 通信分析 + job_list += self._cluster_data_comparison(profiling_path, + benchmark_profiling_path, + self.slow_link_analyzer, + benchmark_slow_link_analyzer, + get_max=False) + return job_list + + def _cluster_data_comparison(self, profiling_path, benchmark_profiling_path, target_cluster_analyzer, + benchmark_cluster_analyzer, get_max=False): + # #low rank/slow link结果逐行对比获取差值最大的rank和step进行单卡分析 + job_list = [] + + if isinstance(target_cluster_analyzer, SlowRankAnalyzer): + comparison_dims = [SlowRankAnalyzer.COMPUTE, SlowRankAnalyzer.FREE] + elif isinstance(target_cluster_analyzer, SlowLinkAnalyzer): + comparison_dims = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + else: + return job_list + + target_data = target_cluster_analyzer.format_datas.get("data", []) + benchmark_data = benchmark_cluster_analyzer.format_datas.get("data", []) + headers = benchmark_cluster_analyzer.format_datas.get("headers", []) + + if len(target_data) != len(benchmark_data): + logger.warning( + "The product of ranks and steps of Benchmark profiling is not equals to target profiling, skip cluster comparison.") + return job_list + + for dimension in comparison_dims: + step, benchmark_step, rank_id_for_comparison = AnalyzerController._get_step_rank_for_cluster_statistic_diff( + target_data, + benchmark_data, + headers, + dimension, + get_max=get_max + ) + rank_profiling_path = self._get_profiling_path_by_rank(profiling_path, rank_id_for_comparison) + rank_benchmark_profiling_path = self._get_profiling_path_by_rank( + benchmark_profiling_path, + rank_id_for_comparison + ) + + job_list += self._single_profiling_comparison( + rank_profiling_path, + rank_benchmark_profiling_path, + step, + benchmark_step + ) + return job_list + + def _is_cluster_profiling(self, profiling_path): + path_list = [os.path.join(profiling_path, dir_name) for dir_name in os.listdir(profiling_path)] + ascend_pt_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_pt")] + data_processor = PytorchDataPreprocessor(ascend_pt_dirs) + + self.cluster_local_data_map[profiling_path] = data_processor.get_data_map() + + if not self.cluster_local_data_map or not self.cluster_local_data_map.get(profiling_path): + return False + + self.default_rank_id = list(self.cluster_local_data_map[profiling_path].keys())[0] + + self.slow_rank_analyzer = SlowRankAnalyzer(profiling_path) + self.slow_link_analyzer = SlowLinkAnalyzer(profiling_path) + return len(self.cluster_local_data_map[profiling_path]) >= self.CLUSTER_RANK_THRESHOLD + + def _get_profiling_path_by_rank(self, profiling_path, rank_id=None): + + if not profiling_path: + return profiling_path + + return self._get_target_profiling_path_for_local(profiling_path, rank_id) + + def _get_target_profiling_path_for_local(self, profiling_path, rank_id): + rank_id_map = self.cluster_local_data_map.get(profiling_path, {}) + if rank_id is None or not rank_id_map: + return profiling_path + + if rank_id in rank_id_map: + return rank_id_map.get(rank_id) + + local_first_rank_id = sorted(list(map(int, rank_id_map.keys())))[0] + logger.warning("Target rank id %s does not exist in local profiling data %s, use rank %s for analysis", + rank_id, profiling_path, local_first_rank_id) + return rank_id_map.get(local_first_rank_id) + + def _update_analysis_process_resp(self, pid, resp, **kwargs): + if kwargs: + resp.update(kwargs) + self.analysis_process_resp[pid] = resp + + def _get_analysis_success_resp(self, pid, resp): + html_path = os.path.join(Config().work_path, f"mstt_advisor_{Timer().strftime}.html") + xlsx_path = os.path.join(Config().work_path, f"mstt_advisor_{Timer().strftime}.xlsx") + result_files = {"html": html_path, "xlsx": xlsx_path} + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.SUCCESS, result_files=result_files) \ No newline at end of file diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index 80368e1d6..6347839b1 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -22,12 +22,16 @@ from profiler.advisor.common.version_control import VersionControl from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from profiler.advisor.utils.utils import safe_division logger = logging.getLogger() class BaseAnalyzer(VersionControl, metaclass=ABCMeta): _SUPPORT_VERSIONS = constant.SUPPORTED_CANN_VERSION + ANALYZER_HIGH_PRIORITY_TIME_RATIO = 0.05 + ANALYZER_MEDIUM_PRIORITY_TIME_RATIO = 0.03 dataset_cls_list = [] @@ -43,6 +47,18 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): self.result = OptimizeResult() self.record_list: Dict[str, List] = {} + @staticmethod + def get_first_data_by_key(data, key) -> Union[Dataset, None]: + """ + get the first member from data with key + :param data: input data + :param key: data key + :return: the first dataset in dataset list + """ + if key in data and len(data[key]) > 0: + return data[key][0] + return None + @classmethod def check_data(cls, data_list: tuple): """ @@ -63,7 +79,7 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): return None logger.info("Enable analysis %s with %s", self.__class__.__name__, ",".join(data_list)) - return func(self) + return func(self, **kwargs) return wrapper @@ -73,6 +89,10 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): def optimize(self, **kwargs): pass + @abstractmethod + def get_priority(self): + pass + def init_dataset_list(self)->None: dataset_cls_list = self.dataset_cls_list if len(dataset_cls_list) == 0: @@ -91,14 +111,25 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): self.dataset_list[key] = [] self.dataset_list[key].append(dataset) - @staticmethod - def get_first_data_by_key(data, key) -> Union[Dataset, None]: - """ - get the first member from data with key - :param data: input data - :param key: data key - :return: the first dataset in dataset list - """ - if key in data and len(data[key]) > 0: - return data[key][0] - return None + def init_dataset_list(self) -> None: + dataset_cls_list = self.dataset_cls_list + if len(dataset_cls_list) == 0: + logger.warning(f"Analyzer: %s don't rely on any dataset!", self.__class__.__name__) + return + + for dataset_cls in dataset_cls_list: + if dataset_cls and callable(dataset_cls): + dataset = dataset_cls(collection_path=self.collection_path, data=self.dataset_list, **self.kwargs) + key = dataset_cls.get_key() + if key not in self.dataset_list: + self.dataset_list[key] = [] + self.dataset_list[key].append(dataset) + + def get_priority_by_time_ratio(self, dur, step_dur): + time_ratio = safe_division(dur, step_dur) + if time_ratio >= self.ANALYZER_HIGH_PRIORITY_TIME_RATIO: + return PriorityBackgroundColor.high + elif time_ratio >= self.ANALYZER_MEDIUM_PRIORITY_TIME_RATIO: + return PriorityBackgroundColor.medium + else: + return PriorityBackgroundColor.low diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py new file mode 100644 index 000000000..a1971baf9 --- /dev/null +++ b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py @@ -0,0 +1,216 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.common import constant +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset +from profiler.advisor.utils.utils import safe_index + +logger = logging.getLogger() + + +class SlowRankAnalyzer(BaseAnalyzer): + SLOW_RANK_ANALYSIS = "slow rank" + RANK = "rank" + RATIO_THRESHOLD = 0.05 + BOTTLENECK_LIST = ['Computing', 'Communication', "Free"] + dataset_cls_list = [ClusterStepTraceTimeDataset] + COMPUTE = "compute(us)" + FREE = "free(us)" + COMMUNICATION = "communication(us)" + + def __init__(self, collection_path, n_processes: int = 1, **kwargs): + super().__init__(collection_path, n_processes, **kwargs) + key = ClusterStepTraceTimeDataset.get_key() + self.step_trace_class = self.get_first_data_by_key(self.dataset_list, key) + self.step_trace_dict = self.step_trace_class.get_data() + self.stages = self.step_trace_class.get_stages() + self.result = OptimizeResult() + self.bottelneck = '' + self.suggestion = '' + self._steps = set() + if self.step_trace_dict is not None: + self.format_datas = self.format_details() + + @property + def steps(self): + return sorted(list(self._steps)) + + @staticmethod + def compute_max_gap_ratio(data: list, mean: float): + if mean == 0: + return 0 + else: + return (max(data) - min(data)) / mean + + def optimize(self, **kwargs): + if self.step_trace_dict is None: + logger.error("slow_rank 分析失败，原因是数据加载失败，请检查你的cluster_analysis_outpu文件夹 \ + 如不关心这类数据请忽略") + return self.result + self.process() + self.make_record() + self.make_render(kwargs.get("template_key")) + return self.result + + def process(self): + total_time_list = [sum(data_tuple) for rank_id, data_tuple in self.step_trace_dict.items()] + if total_time_list: + mean_total_time = sum(total_time_list) / len(total_time_list) + for i in range(len(self.BOTTLENECK_LIST)): + self.produce_bottleneck(self.step_trace_dict, i, mean_total_time) + + if not self.bottelneck: + self.bottelneck = "There is no slow rank issues" + + def produce_bottleneck(self, step_dict: dict, produce_type: int, mean_total_time: float): + data_list = [data_tuple[produce_type] for rank_id, data_tuple in step_dict.items()] + max_ratio = self.compute_max_gap_ratio(data_list, mean_total_time) + if max_ratio > self.RATIO_THRESHOLD: + self.bottelneck += f'{self.BOTTLENECK_LIST[produce_type]} \n' \ + f' has some issues in the cluster, \n' \ + f' because the max difference of {self.BOTTLENECK_LIST[produce_type]} time \n' \ + f' has reached {round(max_ratio * mean_total_time / 1000, 3)}ms. \n' + + def make_record(self): + """ + make record for what and how to optimize + """ + + optimization_item = OptimizeItem( + SlowRankAnalyzer.SLOW_RANK_ANALYSIS, + self.bottelneck, + self.suggestion + ) + self.result.add(OptimizeRecord(optimization_item)) + + data_list = self.format_datas.get("data", []) + headers = self.format_datas.get("headers", []) + for data in data_list: + self.result.add_detail(SlowRankAnalyzer.SLOW_RANK_ANALYSIS, headers, data) + + def format_details(self): + details_dict = {} + headers = ["step", "rank_id", "compute(us)", "communication(us)", "free(us)"] + data_list = [] + for key, value in self.step_trace_dict.items(): + step, rank_id = key.split(constant.STEP_RANK_SEP) + data_list.append([int(step), int(rank_id)] + value) + if step and step not in self._steps: + self._steps.add(step) + + details_dict["headers"] = headers + details_dict["data"] = sorted(data_list, key=lambda x: (x[0], x[1])) + return details_dict + + def make_render(self, template_key="cluster"): + result_for_html = { + "Description": self.bottelneck, + "suggestion": self.suggestion, + "details": [self.format_datas] + } + + self.html_render.render_template(key=template_key, + title=SlowRankAnalyzer.SLOW_RANK_ANALYSIS, + template_dir="templates", + template_name="cluster_analysis.html", + cann_version=self.cann_version, + torch_version=self.torch_version, + result=result_for_html) + + def get_global_step_rank(self, dimension): + global_step_rank = {} + + headers = self.format_datas.get("headers") + + dimension_index = safe_index(headers, dimension) + rank_id_index = safe_index(headers, "rank_id") + step_index = safe_index(headers, "step") + if dimension_index is None or rank_id_index is None: + return global_step_rank + + data_list = [tuple_list[dimension_index] for tuple_list in self.format_datas.get("data")] + max_time, min_time = max(data_list), min(data_list) + + if self.compute_max_gap_ratio(data_list, sum(data_list) / len( + data_list)) < self.RATIO_THRESHOLD: + return global_step_rank + max_time_index = data_list.index(max_time) + min_time_index = data_list.index(min_time) + + max_time_rank_id = self.format_datas.get("data")[max_time_index][rank_id_index] + min_time_rank_id = self.format_datas.get("data")[min_time_index][rank_id_index] + + if step_index is not None: + max_time_step = self.format_datas.get("data")[max_time_index][step_index] + min_time_step = self.format_datas.get("data")[min_time_index][step_index] + else: + max_time_step, min_time_step = constant.DEFAULT_STEP, constant.DEFAULT_STEP + + global_step_rank["maximum"] = {"rank_id": max_time_rank_id, "step": max_time_step} + global_step_rank["minimum"] = {"rank_id": min_time_rank_id, "step": min_time_step} + + return global_step_rank + + def get_stage_step_rank(self, dimension): + stage_step_rank = {} + + headers = self.format_datas.get("headers") + dimension_index = safe_index(headers, dimension) + rank_id_index = safe_index(headers, "rank_id") + step_index = safe_index(headers, "step") + if dimension_index is None or rank_id_index is None: + return stage_step_rank + + rank_list = [tuple_list[rank_id_index] for tuple_list in self.format_datas.get("data")] + cost_time_list = [tuple_list[dimension_index] for tuple_list in self.format_datas.get("data")] + + if step_index is not None: + step_list = [tuple_list[step_index] for tuple_list in self.format_datas.get("data")] + else: + step_list = [constant.DEFAULT_STEP] * len(rank_list) + + for index, stage in enumerate(self.stages): + tmp_step_list, tmp_rank_list, tmp_time_list = [], [], [] + for step, rank_id, time in zip(step_list, rank_list, cost_time_list): + if rank_id not in stage: + continue + + tmp_step_list.append(step) + tmp_rank_list.append(rank_id) + tmp_time_list.append(time) + + if self.compute_max_gap_ratio(tmp_time_list, sum(tmp_time_list) / len( + tmp_time_list)) < self.RATIO_THRESHOLD: + continue + + max_time, min_time = max(tmp_time_list), min(tmp_time_list) + max_time_index, min_time_index = tmp_time_list.index(max_time), tmp_time_list.index(min_time) + + stage_key = f"stage-{index}" + stage_step_rank[stage_key] = {} + stage_step_rank[stage_key]["maximum"] = {"rank_id": tmp_rank_list[max_time_index], + "step": tmp_step_list[max_time_index]} + stage_step_rank[stage_key]["minimum"] = {"rank_id": tmp_rank_list[min_time_index], + "step": tmp_step_list[min_time_index]} + + return stage_step_rank + + def get_priority(self): + pass \ No newline at end of file diff --git a/profiler/advisor/analyzer/communication/base_communication_analyzer.py b/profiler/advisor/analyzer/communication/base_communication_analyzer.py new file mode 100644 index 000000000..95a830e47 --- /dev/null +++ b/profiler/advisor/analyzer/communication/base_communication_analyzer.py @@ -0,0 +1,8 @@ +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer + + +class BaseCommunicationAnalyzer(BaseAnalyzer): + requires_cluster_dataset = True + + def __init__(self, collection_path, n_processes: int = 1, **kwargs): + super().__init__(collection_path, n_processes, **kwargs) diff --git a/profiler/advisor/analyzer/communication/packet/__init__.py b/profiler/advisor/analyzer/communication/packet/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/communication/packet_analyzer.py b/profiler/advisor/analyzer/communication/packet/packet_analyzer.py similarity index 74% rename from profiler/advisor/analyzer/communication/packet_analyzer.py rename to profiler/advisor/analyzer/communication/packet/packet_analyzer.py index 73e5bc2bc..e77ea7780 100644 --- a/profiler/advisor/analyzer/communication/packet_analyzer.py +++ b/profiler/advisor/analyzer/communication/packet/packet_analyzer.py @@ -14,17 +14,19 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.communication.packet_checker import PacketChecker +from profiler.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer +from profiler.advisor.analyzer.communication.packet.packet_checker import PacketChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset +from profiler.advisor.result.result import OptimizeResult logger = logging.getLogger() -class PacketAnalyzer(BaseAnalyzer): +class PacketAnalyzer(BaseCommunicationAnalyzer): dataset_cls_list = [CommunicationDataset] + requires_cluster_dataset = False def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) @@ -34,7 +36,7 @@ class PacketAnalyzer(BaseAnalyzer): self.html_render = HTMLRender() self.html = None - @BaseAnalyzer.check_data((CommunicationDataset.get_key(),)) + @BaseCommunicationAnalyzer.check_data((CommunicationDataset.get_key(),)) def optimize(self, **kwargs): add_render_list = kwargs.get("add_render_list", True) packet_checker = PacketChecker(**kwargs) @@ -42,5 +44,9 @@ class PacketAnalyzer(BaseAnalyzer): if not packet_checker.packet_issues: return self.result packet_checker.make_record(self.result) - self.html = packet_checker.make_render(self.html_render, add_render_list) + self.html = packet_checker.make_render(self.html_render, add_render_list, priority=self.get_priority()) return self.result + + def get_priority(self): + # 提升1% ~ 3% + return PriorityBackgroundColor.low diff --git a/profiler/advisor/analyzer/communication/packet_checker.py b/profiler/advisor/analyzer/communication/packet/packet_checker.py similarity index 96% rename from profiler/advisor/analyzer/communication/packet_checker.py rename to profiler/advisor/analyzer/communication/packet/packet_checker.py index 3d9ac81ff..d270667cd 100644 --- a/profiler/advisor/analyzer/communication/packet_checker.py +++ b/profiler/advisor/analyzer/communication/packet/packet_checker.py @@ -116,19 +116,20 @@ class PacketChecker: result.add_detail(sub_table_name, headers=self.headers) result.add_detail(sub_table_name, detail=self.small_packet_detail) - def make_render(self, html_render, add_render_list=True): + def make_render(self, html_render, add_render_list=True, **kwargs): + priority = kwargs.get("priority") return html_render.render_template(key="communication", template_dir="templates", template_name="packet_analysis.html", desc=self.desc, solutions=self.solutions, headers=self.headers, - data=self.small_packet_detail - ) + data=self.small_packet_detail, + priority_background_color=priority) def _init_rule(self): syncbn_rule_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))), "rules", "packet.yaml" ) diff --git a/profiler/advisor/analyzer/communication/retransmission/__init__.py b/profiler/advisor/analyzer/communication/retransmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py new file mode 100644 index 000000000..78cade900 --- /dev/null +++ b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer +from profiler.advisor.analyzer.communication.retransmission.communication_retransmission_checker import \ + CommunicationRetransmissionChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from profiler.advisor.result.result import OptimizeResult + +logger = logging.getLogger() + + +class RDMARetransmissionAnalyzer(BaseCommunicationAnalyzer): + dataset_cls_list = [ClusterCommunicationDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = ClusterCommunicationDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseCommunicationAnalyzer.check_data((ClusterCommunicationDataset.get_key(),)) + def optimize(self, **kwargs): + add_render_list = kwargs.get("add_render_list", True) + rdma_checker = CommunicationRetransmissionChecker(**kwargs) + rdma_checker.check_retransmission(self.dataset) + if not rdma_checker.rdma_issues: + return self.result + rdma_checker.make_record(self.result) + self.html = rdma_checker.make_render(self.html_render, add_render_list, priority=self.get_priority()) + return self.result + + def get_priority(self): + # 单次重传最少4s，高优先级 + return PriorityBackgroundColor.high diff --git a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py new file mode 100644 index 000000000..4431ccce4 --- /dev/null +++ b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py @@ -0,0 +1,129 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from typing import Dict, List +from collections import defaultdict +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo + +logger = logging.getLogger() + + +class GroupStatistic: + def __init__(self, min_transmission_time): + self.retransmission_issue = False + self.abnormal_op_dict: Dict[str, List] = dict() + + def add_op(self, op_name: str, hccl_info: HcclInfo): + if self.abnormal_op_dict.get(op_name) is None: + self.abnormal_op_dict.setdefault(op_name, []) + self.abnormal_op_dict.get(op_name).append([hccl_info.group, op_name, hccl_info.step, hccl_info.rank, + hccl_info.get_rdma_transit_size(), + hccl_info.get_rdma_transmit_time(), hccl_info.get_rdma_bandwidth()]) + + +class CommunicationRetransmissionChecker: + def __init__(self, **kwargs): + self.rdma_issues = False + self.desc = "" + self.sdma_desc = "" + self.rdma_desc = "" + self.suggestions = [] + self.abnormal_group_count = 0 + self.abnormal_rdma_list = [] + self.step_id = kwargs.get("step") + self.stage = None + self.group_statistics = defaultdict(GroupStatistic) + self.headers = ["Communication group", "Op name", "Step id", "Rank id", "RDMA transmit size(MB)", + "RDMA transmit time(ms)", "RDMA bandwidth"] + self._init_rule() + + def check_possible_retransmission_occurrence(self, hccl_list: List[HcclInfo]): + min_elapse_time = min(hccl.elapse_time for hccl in hccl_list) + max_transit_time = max(hccl.rdma_info.get('Transit Time(ms)', 0) for hccl in hccl_list) + if min_elapse_time < self.min_retransmission_time: # 检测是否是卡间不同步问题，而不是重传 + return False + return max_transit_time > self.min_retransmission_time + + def check_retransmission(self, hccl_dataset: ClusterCommunicationDataset): + """ + :Param event_dataset: dataset of timeline event + """ + for group_name, hccl_group_dict in hccl_dataset.hccl_dict.items(): + for op_name, hccl_op_dict in hccl_group_dict.items(): + for step_id, hccl_list in hccl_op_dict.items(): + if self.step_id and step_id != self.step_id: # 传输指定step（self.step_id）情况下，非目标step跳过 + continue + if not self.check_possible_retransmission_occurrence(hccl_list): + continue + self.rdma_issues = True + if self.group_statistics.get(group_name) is None: + self.group_statistics.setdefault(group_name, GroupStatistic(self.min_retransmission_time)) + self.abnormal_group_count += 1 + for hccl_info in hccl_list: + if hccl_info.rdma_info.get('Transit Size(MB)', 0): + transit_time = hccl_info.rdma_info.get('Transit Time(ms)', 0) + if transit_time > self.min_retransmission_time: + self.group_statistics.get(group_name).add_op(op_name, hccl_info) + if self.rdma_issues: + self.desc = self.desc.format(group_count=self.abnormal_group_count) + for _, group_statistic in self.group_statistics.items(): + for _, op_list in group_statistic.abnormal_op_dict.items(): + for op in op_list: + self.abnormal_rdma_list.append(op) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("Communication retransmission analysis", self.desc, self.suggestions) + result.add(OptimizeRecord(optimization_item)) + + sub_table_name = "Comm Retransmission Analysis" if not self.stage else f"Stage-{self.stage}: Comm Retransmission Analysis" + result.add_detail(sub_table_name, headers=self.headers) + + for row in self.abnormal_rdma_list: + result.add_detail(sub_table_name, detail=row) + + def make_render(self, html_render, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="communication", + template_dir="templates", + template_name="communication_retransmission_analysis.html", + desc=self.desc, + solutions=self.solutions, + headers=self.headers, + data=self.abnormal_rdma_list, + priority_background_color=priority) + + def _init_rule(self): + syncbn_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))), + "rules", + "rdma_analysis.yaml" + ) + + syncbn_rule = FileManager.read_yaml_file(syncbn_rule_path) + self.desc = syncbn_rule.get("problem") + self.min_retransmission_time = syncbn_rule.get("min_retransmission_time") + + self.solutions = syncbn_rule.get("solutions") + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py index 4f25deff7..bc0841152 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py @@ -3,34 +3,40 @@ import logging from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_checker import AICoreFreqChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.ai_core_freq.ai_core_freq_dataset import AICoreFreqDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset +from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser from profiler.advisor.config.config import Config logger = logging.getLogger() class AICoreFreqAnalyzer(BaseAnalyzer): - dataset_cls_list = [AICoreFreqDataset] + dataset_cls_list = [ComputationAnalysisDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) - key = AICoreFreqDataset.get_key() + key = ComputationAnalysisDataset.get_key() self.dataset = self.get_first_data_by_key(self.dataset_list, key) self.result = OptimizeResult() self.html_render = HTMLRender() self.html = None + info = DeviceInfoParser(collection_path) + info.parse_data() - @BaseAnalyzer.check_data((AICoreFreqDataset.get_key(),)) + @BaseAnalyzer.check_data((ComputationAnalysisDataset.get_key(),)) def optimize(self, **kwargs): if not Config().get_config("aic_frequency"): logger.warning("Can not find ai core frequency in info.json*, please check data integrity.") return self.result + add_render_list = kwargs.get("add_render_list", True) ai_core_freq_checker = AICoreFreqChecker() - ai_core_freq_checker.check_ai_core_freq(self.dataset) - if not ai_core_freq_checker.ai_core_freq_issues: - return self.result + ai_core_freq_checker.check_ai_core_freq(self.dataset, rank_id=kwargs.get("rank_id"), stage=kwargs.get("stage")) ai_core_freq_checker.make_record(self.result) - self.html = ai_core_freq_checker.make_render(self.html_render, add_render_list) + self.html = ai_core_freq_checker.make_render(self.html_render, add_render_list, priority=self.get_priority()) return self.result + + def get_priority(self): + return PriorityBackgroundColor.high \ No newline at end of file diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index 5bfa5adc4..c8a94287d 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -1,6 +1,6 @@ import logging -from profiler.advisor.dataset.ai_core_freq.ai_core_freq_dataset import AICoreFreqDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.config.config import Config @@ -26,7 +26,7 @@ class AICoreFreqChecker: self.rank_id = None self.stage = None - def check_ai_core_freq(self, event_dataset: AICoreFreqDataset, rank_id=None, stage=None): + def check_ai_core_freq(self, event_dataset: ComputationAnalysisDataset, rank_id=None, stage=None): """ :Param event_dataset: dataset of timeline event """ @@ -60,6 +60,8 @@ class AICoreFreqChecker: self.decrease_freq_ops.sort(key= lambda x: (x[self.TOTAL_DURATION_INDEX], x[self.DECREASE_FREQ_RATIO_INDEX]), reverse=True) + if not self.ai_core_freq_issues: + return self.desc = (f"{len(self.decrease_freq_ops)} operators are found during frequency reduction, and the reduction " f"ratio is larger than {self.DECREASE_FREQ_RATIO}.") @@ -71,22 +73,29 @@ class AICoreFreqChecker: """ make record for what and how to optimize """ - optimization_item = OptimizeItem("AI Core Frequency", self.desc, [self.suggestions]) + if not self.ai_core_freq_issues: + return self.ai_core_freq_issues + + sheet_name = "AI Core Frequency" + if self.rank_id is not None: + sheet_name = f"rank {self.rank_id} AI Core Frequency".capitalize() + + optimization_item = OptimizeItem(sheet_name, self.desc, [self.suggestions]) result.add(OptimizeRecord(optimization_item)) self.headers = ["Operator name", "Count", "Total duration(us)", "AI CORE frequency decreased ratio", "Average frequency", "Max frequency", "Min frequency"] - if self.rank_id: - self.headers = ["Rank id"] + self.headers - sub_table_name = "AI Core Frequency" if not self.stage else f"Stage-{self.stage}: AI Core Frequency" - result.add_detail(sub_table_name, headers=self.headers) + result.add_detail(sheet_name, headers=self.headers) for row in self.decrease_freq_ops: - if self.rank_id: - row = [self.rank_id] + row - result.add_detail(sub_table_name, detail=row) + result.add_detail(sheet_name, detail=row) + return True + + def make_render(self, html_render, add_render_list=True, **kwargs): + if not self.ai_core_freq_issues: + return self.ai_core_freq_issues - def make_render(self, html_render, add_render_list=True): + priority = kwargs.get("priority") if self.SHOW_TOPK_OPS: self.desc += f" Only show {self.SHOW_TOPK_OPS} operators here, see latest mstt_advisor.xlsx for details." return html_render.render_template(key="computation", @@ -96,4 +105,5 @@ class AICoreFreqChecker: suggestion=self.suggestions, headers=self.headers, data=self.decrease_freq_ops[:self.SHOW_TOPK_OPS], - add_render_list=add_render_list) + add_render_list=add_render_list, + priority_background_color=priority) diff --git a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py b/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py index 0caede4b8..394ad74fd 100644 --- a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py +++ b/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py @@ -8,7 +8,7 @@ from profiler.advisor.analyzer.schedule.fusion_ops.timeline_api_stack_checker im from profiler.advisor.common import constant from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset from profiler.cluster_analyse.common_func.file_manager import FileManager @@ -30,6 +30,8 @@ class AicpuChecker(OperatorChecker): self.aicpu_rules: Dict = {} self.aicpu_checker: Dict = {} self.load_aicpu_rules() + self.total_task_duration = 0.0 + self.aicpu_task_duration = 0.0 def _check_data(self, profiling_data: ProfilingDataset) -> bool: if not self._check_summary(profiling_data): @@ -88,7 +90,7 @@ class AicpuChecker(OperatorChecker): def get_opeartor_stack_info(api_stack_finder: OpStackFinder, op_name_list: list) -> list: data: Dict[str, Dataset] = {} - event_dataset = TimelineEventDataset(collection_path=profiling_data.collection_path, data=data, task_type=constant.AI_CPU) + event_dataset = ComputationAnalysisDataset(collection_path=profiling_data.collection_path, data=data, task_type=constant.AI_CPU) # disable multiprocessing, avoid cost time of enable new process for light task api_stack_finder.get_api_stack_by_op(event_dataset, op_name_list, constant.AI_CPU, @@ -96,14 +98,16 @@ class AicpuChecker(OperatorChecker): return api_stack_finder._stack_record self._op_list = [] - total_task_duration = 0.0 + max_task_duration = 0.0 for op_info in op_summary.op_list: + task_duration = float(op_info.task_duration) + if self._check_operator(op_info): self._op_list.append(op_info) + self.aicpu_task_duration += task_duration - task_duration = float(op_info.task_duration) - total_task_duration += task_duration + self.total_task_duration += task_duration max_task_duration = max(max_task_duration, task_duration) if (not self._op_list) or (max_task_duration < self._MIN_TASK_DURATION): return False @@ -145,11 +149,15 @@ class AicpuChecker(OperatorChecker): ",".join(double_type_ai_cpu_operator))) return True - def make_render(self, html_render, record): - html_render.render_template(key="computation", - template_dir="templates", - template_name="operator_ai_cpu.html", - format_result=self.format_operator_result(record, constant.OPERATOR_LIST_UNLIMIT)) + def make_render(self, html_render, record, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="computation", + template_dir="templates", + template_name="operator_ai_cpu.html", + format_result=self.format_operator_result(record, + constant.OPERATOR_LIST_UNLIMIT), + add_render_list=add_render_list, + priority_background_color=priority) def format_operator_result(self, record, limit): """ diff --git a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py index 7a873c656..5b358ebaa 100644 --- a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py +++ b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py @@ -45,11 +45,15 @@ class BlockDimChecker(OperatorChecker): "task duration are as follows:\n" return True - def make_render(self, html_render, record): - html_render.render_template(key="computation", - template_dir="templates", - template_name="operator_block_dim.html", - format_result=self.format_operator_result(record, constant.OPERATOR_OUT_TOPK)) + def make_render(self, html_render, record, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="computation", + template_dir="templates", + template_name="operator_block_dim.html", + format_result=self.format_operator_result(record, + constant.OPERATOR_OUT_TOPK), + add_render_list=add_render_list, + priority_background_color=priority) def _check_operator(self, op_info) -> bool: if op_info.task_type not in ["AI_CORE", "AI_VECTOR_CORE", "MIX_AIC"]: diff --git a/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py b/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py index a22b380f9..2096e9ffa 100644 --- a/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py +++ b/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py @@ -46,8 +46,12 @@ class OperatorBoundChecker(OperatorChecker): return False return True - def make_render(self, html_render, record): - html_render.render_template(key="computation", - template_dir="templates", - template_name="operator_no_bound.html", - format_result=self.format_operator_result(record, constant.OPERATOR_OUT_TOPK)) + def make_render(self, html_render, record, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="computation", + template_dir="templates", + template_name="operator_no_bound.html", + format_result=self.format_operator_result(record, + constant.OPERATOR_OUT_TOPK), + add_render_list=add_render_list, + priority_background_color=priority) \ No newline at end of file diff --git a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py b/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py index 86d3bac4f..2521b6e7e 100644 --- a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py +++ b/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py @@ -27,11 +27,13 @@ class DynamicShapeChecker(OperatorChecker): def check(self, profiling_database) -> bool: return self.is_dynamic_shape(profiling_database) - def make_record(self, profiling_database) -> OptimizeRecord: + def make_record(self, profiling_database, rank_id=None) -> OptimizeRecord: """ make record for what and how to optimize """ + if rank_id is not None: + self._PROBLEM = f"rank {rank_id} ".capitalize() + self._PROBLEM.lower() optimization_item = OptimizeItem( self._PROBLEM, self._description, @@ -58,8 +60,11 @@ class DynamicShapeChecker(OperatorChecker): format_result = {"record": record.__dict__, "suggestion": '
'.join(release_suggestion_list)} return format_result - def make_render(self, html_render, record): - html_render.render_template(key="computation", - template_dir="templates", - template_name="operator_dynamic_shape.html", - format_result=self.format_operator_result(record)) + def make_render(self, html_render, record, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="computation", + template_dir="templates", + template_name="operator_dynamic_shape.html", + format_result=self.format_operator_result(record), + add_render_list=add_render_list, + priority_background_color=priority) diff --git a/profiler/advisor/analyzer/computation/operator_checker.py b/profiler/advisor/analyzer/computation/operator_checker.py index 64618b56a..e24eae1d0 100644 --- a/profiler/advisor/analyzer/computation/operator_checker.py +++ b/profiler/advisor/analyzer/computation/operator_checker.py @@ -40,6 +40,23 @@ class OperatorChecker(VersionControl): self.cann_version = cann_version self._op_list: List[OpInfo] = [] + @staticmethod + def get_ratio(op_info: OpInfo, attr: str) -> float: + if not op_info.has_attr(attr): + return 0 + value = op_info.get_attr(attr) + if not value or value == "N/A": + return 0 + return float(value) + + @classmethod + def get_name(cls): + """ + get name of checker + :return: checker name + """ + return cls._PROBLEM + def check(self, profiling_data: ProfilingDataset) -> bool: """ check if any operator need optimize @@ -77,12 +94,16 @@ class OperatorChecker(VersionControl): return True return False - def make_record(self, profiling_data: ProfilingDataset): + def make_record(self, profiling_data: ProfilingDataset, rank_id=None): """ Make record for what and how to optimize :param profiling_data: profiling data :return: optimize record """ + + if rank_id is not None: + self._PROBLEM = f"rank {rank_id} ".capitalize() + self._PROBLEM.lower() + task_duration_list = [float(op_info.get_attr("task_duration")) for op_info in self._op_list if hasattr(op_info, "get_attr")] total_cost_time = sum(task_duration_list) @@ -239,14 +260,6 @@ class OperatorChecker(VersionControl): """Get node views.""" return [] - @classmethod - def get_name(cls): - """ - get name of checker - :return: checker name - """ - return cls._PROBLEM - def get_incomes(self) -> float: """get incomes""" incomes = 0.0 @@ -269,16 +282,7 @@ class OperatorChecker(VersionControl): logger.warning(self.SKIP_CHECK_MSG, self._CHECKER, "op summary") return False return True - - @staticmethod - def get_ratio(op_info: OpInfo, attr: str) -> float: - if not op_info.has_attr(attr): - return 0 - value = op_info.get_attr(attr) - if not value or value == "N/A": - return 0 - return float(value) - + def get_details(self) -> list: """ get details of operator to be optimized diff --git a/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py b/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py new file mode 100644 index 000000000..bc02b4c3e --- /dev/null +++ b/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py @@ -0,0 +1,106 @@ +import logging +import os +from multiprocessing import Manager + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.common.analyzer_scopes import SupportedScopes +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from profiler.advisor.interface.interface import Interface +from profiler.advisor.utils.utils import ParallelJob +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.common import constant as const + +logger = logging.getLogger() + + +class PPStageComputationAnalyzer(BaseAnalyzer): + + def __init__(self, collection_path, **kwargs): + super().__init__(collection_path, **kwargs) + self.collection_path = collection_path + self._stages_rendered_html = Manager().list() + self._multiprocess_result = Manager().dict() + # html render不能序列化，无法用多进程，放到optimize里面初始化 + self.html_render = None + self.result = None + + @staticmethod + def _get_valid_sheet_name(sheet_name, prefix): + if not sheet_name.lower().startswith(prefix.lower()): + sheet_name = f"{prefix} {sheet_name}" + return sheet_name + + def optimize(self, stages_profiling_path, **kwargs): + pp_stage_processes = min(int(os.getenv("PP_STAGE_ANALYSIS_PROCESSES", 0)), len(stages_profiling_path), + const.MAX_NUM_PROCESSES) + if pp_stage_processes <= 1: + for stage_profiling_path in stages_profiling_path: + self._optimize(**stage_profiling_path) + else: + logger.info("Start to parallel analysis of pp stages, number of processes is %s", pp_stage_processes) + parallel_stage_analysis_job = ParallelJob(self._optimize, stages_profiling_path, + "Computation analysis of Pipeline parallel stages") + parallel_stage_analysis_job.start(pp_stage_processes) + self._merge_multiprocess_result() + + self.make_render() + self.html_render = HTMLRender() + return self.result + + def make_render(self): + HTMLRender().render_template(key="computation", + template_dir="templates", + template_name="pp_stage_computation_analysis.html", + stages_rendered_html=list(self._stages_rendered_html), + priority_background_color=PriorityBackgroundColor.high) + + def get_priority(self): + pass + + def _optimize(self, profiling_path, **kwargs): + stage_html_record = dict(stage=kwargs.get("stage"), rank_id=kwargs.get("rank_id"), step=kwargs.get("step")) + kwargs["add_render_list"] = False + + # stage 并行分析时，避免调用本身，即SupportedScopes.STAGE_COMPUTE + scopes = Interface.get_scope(Interface.COMPUTATION) + stage_analyzer_list = [Interface.get_analyzer(Interface.COMPUTATION, scope) for scope in scopes if + scope != SupportedScopes.STAGE_COMPUTE] + + for analyzer_cls in stage_analyzer_list: + analyzer = analyzer_cls(collection_path=profiling_path, **kwargs) + result = analyzer.optimize(**kwargs) + if hasattr(result, "data") and result.data: + self.result = result + if hasattr(analyzer, "html") and analyzer.html: + if "html_list" not in stage_html_record: + stage_html_record["html_list"] = [] + stage_html_record["html_list"].append(analyzer.html) + self._stages_rendered_html.append(stage_html_record) + self._multiprocess_result[f"rank {kwargs.get('rank_id')}".capitalize()] = result.data + + def _merge_multiprocess_result(self): + self.result = OptimizeResult() + for key, result_data in self._multiprocess_result.items(): + problem_data = result_data.get("problems", {}).get("data", []) + if not problem_data: + continue + + for row in problem_data: + if len(row) < 3: + continue + issue_name, desc, suggestion = row[:3] + sheet_name = PPStageComputationAnalyzer._get_valid_sheet_name(issue_name, key) + optimization_item = OptimizeItem(sheet_name, desc, [suggestion]) + self.result.add(OptimizeRecord(optimization_item)) + del result_data["problems"] + + for issue_name, issue_details in result_data.items(): + headers = issue_details.get("headers", []) + data = issue_details.get("data", []) + sheet_name = PPStageComputationAnalyzer._get_valid_sheet_name(issue_name, key) + self.result.add_detail(sheet_name, headers=headers) + + for row in data: + self.result.add_detail(sheet_name, detail=row) diff --git a/profiler/advisor/analyzer/computation/profiling_analyzer.py b/profiler/advisor/analyzer/computation/profiling_analyzer.py index 2021bcd57..b29373e87 100644 --- a/profiler/advisor/analyzer/computation/profiling_analyzer.py +++ b/profiler/advisor/analyzer/computation/profiling_analyzer.py @@ -8,6 +8,7 @@ from profiler.advisor.analyzer.computation.bound.block_dim_checker import BlockD from profiler.advisor.analyzer.computation.bound.operator_bound_checker import OperatorBoundChecker from profiler.advisor.analyzer.computation.op_compile.dynamic_shape_checker import DynamicShapeChecker from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset @@ -22,6 +23,7 @@ class ProfilingAnalyzer(BaseAnalyzer, ABC): self.checker = OperatorChecker(self.cann_version) self.html_render = HTMLRender() self.result = OptimizeResult() + self.html = None @BaseAnalyzer.check_data((ProfilingDataset.get_key(),)) def optimize(self, **kwargs) -> OptimizeResult: @@ -32,22 +34,29 @@ class ProfilingAnalyzer(BaseAnalyzer, ABC): """ profiling_data = self.get_first_data_by_key(self.dataset_list, ProfilingDataset.get_key()) checker = self.checker + rank_id = kwargs.get("rank_id") + + add_render_list = kwargs.get("add_render_list", True) + if not checker.pre_check(profiling_data): return self.result if checker.check(profiling_data): # add record - record = checker.make_record(profiling_data) - checker.make_render(self.html_render, record) + record = checker.make_record(profiling_data, rank_id) + self.html = checker.make_render(self.html_render, record, add_render_list, + priority=self.get_priority(checker)) self.result.add(record) # add details details = checker.get_details() if details: for i, detail in enumerate(details): + sheet_name = checker.get_name() if rank_id is None else \ + f"rank {rank_id} ".capitalize() + checker.get_name() if i == 0: # the first row is header - self.result.add_detail(checker.get_name(), headers=detail) + self.result.add_detail(sheet_name, headers=detail) else: - self.result.add_detail(checker.get_name(), detail=detail) + self.result.add_detail(sheet_name, detail=detail) # add tune op list tune_op_list = checker.get_tune_op_list() if tune_op_list: @@ -55,11 +64,13 @@ class ProfilingAnalyzer(BaseAnalyzer, ABC): return self.result - def make_record(self): - pass + def get_priority(self, checker): + if "aicpu" not in checker.__class__.__name__.lower(): + return PriorityBackgroundColor.low - def make_render(self): - pass + aicpu_duration = getattr(checker, "aicpu_task_duration", 0.0) + total_duration = getattr(checker, "total_task_duration", 0.0) + return self.get_priority_by_time_ratio(aicpu_duration, total_duration) class DynamicShapeAnalyzer(ProfilingAnalyzer): diff --git a/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py b/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py index 291c3a1f9..3d1a537c2 100644 --- a/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py +++ b/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py @@ -5,26 +5,30 @@ from typing import List, Dict, Any from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() class DataloaderAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.dataset = self.get_first_data_by_key(self.dataset_list, key) self.result = OptimizeResult() self.html_render = HTMLRender() - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): dataloader_checker = DataloaderChecker() dataloader_checker.check_slow_dataloader(self.dataset) dataloader_checker.make_record(self.result) - dataloader_checker.make_render(self.html_render) + dataloader_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + def get_priority(self): + return PriorityBackgroundColor.high diff --git a/profiler/advisor/analyzer/dataloader/dataloader_checker.py b/profiler/advisor/analyzer/dataloader/dataloader_checker.py index eb1886284..f392a0838 100644 --- a/profiler/advisor/analyzer/dataloader/dataloader_checker.py +++ b/profiler/advisor/analyzer/dataloader/dataloader_checker.py @@ -3,7 +3,7 @@ import re import logging import yaml -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.cluster_analyse.common_func.file_manager import FileManager @@ -22,7 +22,7 @@ class DataloaderChecker: self.dataloader_duration_threshold = None self._init_rule() - def check_slow_dataloader(self, event_dataset: TimelineEventDataset): + def check_slow_dataloader(self, event_dataset: ScheduleAnalysisDataset): """ :Param event_dataset: dataset of timeline event """ @@ -32,7 +32,7 @@ class DataloaderChecker: return for event in event_dataset.dataloader: - dataloader_duration = float(event.dur) / 1000 + dataloader_duration = float(event.dur) if dataloader_duration < self.dataloader_duration_threshold: continue self.desc = self.desc.format(dataloader_duration=dataloader_duration, @@ -53,14 +53,16 @@ class DataloaderChecker: for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.dataloader_issues: return + priority = kwargs.get("priority") html_render.render_template(key="dataloader", template_dir="templates", template_name="slow_dataloader.html", desc=self.desc, - suggestions=self.suggestions) + suggestions=self.suggestions, + priority_background_color=priority) def _init_rule(self): dataloader_rule_path = os.path.join( diff --git a/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py b/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py index 326be83b8..e9dcd263d 100644 --- a/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py +++ b/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py @@ -20,17 +20,22 @@ class FusionOPAnalyzer(BaseAnalyzer): super(FusionOPAnalyzer, self).__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - + self.html = None + @BaseAnalyzer.check_data((GraphDataset.get_key(),)) def optimize(self, **kwargs): """ :return: result """ - self._check(self.dataset_list.get("GraphDataset"), self.dataset_list.get("ProfilingDataset")) + self._check(self.dataset_list.get("GraphDataset"), self.dataset_list.get("ProfilingDataset"), + kwargs.get("add_render_list")) return self.result - def _check(self, graph_data: List[GraphDataset], - profiling_data: List[ProfilingDataset] = None) -> None: + def get_priority(self): + pass + + def _check(self, graph_data: List[GraphDataset], profiling_data: List[ProfilingDataset] = None, + add_render_list=True) -> None: if len(graph_data) == 0 or graph_data[0].is_empty(): return for _, rule in self.RULES.items(): @@ -40,10 +45,4 @@ class FusionOPAnalyzer(BaseAnalyzer): else: checker.find_fusion_matched_issues_with_times(graph_data, profiling_data) checker.make_record(self.result) - checker.make_render(self.html_render) - - def make_record(self): - pass - - def make_render(self): - pass + self.html = checker.make_render(self.html_render, add_render_list) \ No newline at end of file diff --git a/profiler/advisor/analyzer/memory/__init__.py b/profiler/advisor/analyzer/memory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/memory/memory_analyzer.py b/profiler/advisor/analyzer/memory/memory_analyzer.py new file mode 100644 index 000000000..cd7b0a242 --- /dev/null +++ b/profiler/advisor/analyzer/memory/memory_analyzer.py @@ -0,0 +1,38 @@ +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.memory.memory_checker import MemoryOpsChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor + +logger = logging.getLogger() + + +class MemoryAnalyzer(BaseAnalyzer): + dataset_cls_list = [ScheduleAnalysisDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = ScheduleAnalysisDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) + def optimize(self, **kwargs): + memory_checker = MemoryOpsChecker() + memory_checker.check_memory_ops(self.dataset) + memory_checker.make_record(self.result) + memory_checker.make_render(self.html_render, priority=self.get_priority(memory_checker.max_mem_op_dur)) + return self.result + + def get_priority(self, max_mem_op_dur): + step_duration = getattr(self.dataset, "step_duration", None) + ratio = self.get_priority_by_time_ratio(max_mem_op_dur, step_duration) + + if step_duration is None: + return PriorityBackgroundColor.low + + return ratio diff --git a/profiler/advisor/analyzer/overall/environment_variable_analyzer.py b/profiler/advisor/analyzer/overall/environment_variable_analyzer.py index 3daaa3460..c4468c36d 100644 --- a/profiler/advisor/analyzer/overall/environment_variable_analyzer.py +++ b/profiler/advisor/analyzer/overall/environment_variable_analyzer.py @@ -18,6 +18,7 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.prof_common.path_manager import PathManager from profiler.advisor.dataset.environment_variable_dataset import EnvironmentVariableDataset from profiler.advisor.analyzer.overall.environment_variable_checker import EnvironmentVariabelChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor class EnvironmentVariabelAnalyzer(BaseAnalyzer): @@ -40,6 +41,9 @@ class EnvironmentVariabelAnalyzer(BaseAnalyzer): checker.make_render(self.html_render) return self.result + def get_priority(self): + return PriorityBackgroundColor.high + def make_record(self): pass diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py index 8e93dbda7..fe43072a8 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py @@ -23,7 +23,7 @@ from profiler.compare_tools.compare_interface.comparison_interface import Compar class OverallSummaryAnalyzer(BaseAnalyzer): - OVERALL_SUMMARY_ANALYZER = "overall_summary_analysis" + OVERALL_SUMMARY_ANALYZER = "overall summary" advice_map = { "Computing Time": "if you want more detailed advice please go to mstt_advisor_*.html", "Uncovered Communication Time": "if you want more detailed advice please go to mstt_advisor_*.html", @@ -233,6 +233,9 @@ class OverallSummaryAnalyzer(BaseAnalyzer): torch_version=self.torch_version, result=result_for_html) + def get_priority(self): + pass + def get_profile_path(collection_path): for root, dirs, files in os.walk(collection_path): diff --git a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py index 0e62a3ff0..58b2c301b 100644 --- a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py +++ b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py @@ -16,26 +16,26 @@ # limitations under the License. import logging - from profiler.advisor.common import constant as const from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.result.result import OptimizeResult from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() class OpDispatchAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] """ operator dispatch optimizer """ def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.dataset = self.get_first_data_by_key(self.dataset_list, key) self.result = OptimizeResult() self.html_render = HTMLRender() @@ -54,21 +54,21 @@ class OpDispatchAnalyzer(BaseAnalyzer): self.make_render(self.html_render) return self.result - def get_op_compile_info(self, event_dataset: TimelineEventDataset): - """ - :Param event_dataset: dataset of timeline event - """ - if hasattr(event_dataset, "ops_compile"): - self._op_compile = getattr(event_dataset, "ops_compile") - if not self._op_compile or self._op_compile.total_count < const.MAX_OP_COMPILE_NUM: - return + def get_op_compile_info(self, event_dataset: ScheduleAnalysisDataset): + """ + :Param event_dataset: dataset of timeline event + """ + if hasattr(event_dataset, "ops_compile"): + self._op_compile = getattr(event_dataset, "ops_compile") + if not self._op_compile or self._op_compile.total_count < const.MAX_OP_COMPILE_NUM: + return - self._issues_record.append(['operator dispatch', - const.OP_COMPILE_ID, - self._op_compile.total_count, - self._op_compile.total_time]) - else: - logger.debug("Skip operator compile checker, because no op_compile attr find.") + self._issues_record.append(['operator dispatch', + const.OP_COMPILE_ID, + self._op_compile.total_count, + self._op_compile.total_time]) + else: + logger.debug("Skip operator compile checker, because no op_compile attr find.") def make_record(self, result: OptimizeResult): """ @@ -77,8 +77,9 @@ class OpDispatchAnalyzer(BaseAnalyzer): if not self._op_compile or len(self._issues_record) <= 0: return desc = f"Found {self._op_compile.total_count} operator compile issues." - suggestion = (f"Please use `torch_npu.npu.set_compile_mode(jit_compile=False)` to disable jit compile " - f"in dynamic shape usage.") + suggestion = ("Please place the following code at the entrance of the python script to disable jit compile. " \ + "Code: `torch_npu.npu.set_compile_mode(jit_compile=False); " + "torch_npu.npu.config.allow_internal_format = False`") self.optimization_item.append(OptimizeItem("Operator dispatch", desc, [suggestion])) for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) @@ -87,7 +88,7 @@ class OpDispatchAnalyzer(BaseAnalyzer): for op_info in self._issues_record: result.add_detail('operator dispatch', detail=op_info) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): issues = [] optimizations = [] for optimization in self.optimization_item: @@ -97,11 +98,20 @@ class OpDispatchAnalyzer(BaseAnalyzer): )) for record in self._issues_record: issues.append(dict(issue=record[0], - op_name=record[1], - counts=record[2], - total_time=record[3])) + op_name=record[1], + counts=record[2], + total_time=record[3])) html_render.render_template(key="schedule", template_dir="templates", template_name="operator_dispatch.html", issues=issues, - optimizers=optimizations) + optimizers=optimizations, + priority_background_color=self.get_priority()) + + def get_priority(self): + step_duration = getattr(self.dataset, "step_duration", None) + op_compile_total_dur = getattr(self._op_compile, "total_time", None) + if step_duration is None or op_compile_total_dur is None: + return PriorityBackgroundColor.low + + return self.get_priority_by_time_ratio(op_compile_total_dur, step_duration) diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index c1eb24b8e..305d23994 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -8,25 +8,29 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.common import constant as const from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import format_timeline_result from profiler.advisor.common.timeline.fusion_ops_db import init_timeline_ops_db +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() class TimelineFusionOpsAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs): super().__init__(collection_path, n_processes, **kwargs) self._matched_op_index = {} if self.n_processes <= 1 else multiprocessing.Manager().dict() self.matched_op_stacks = {} self.empty_stacks = True - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) + def get_priority(self): + return PriorityBackgroundColor.low + def optimize(self, **kwargs): for mode in [const.ATEN.lower(), const.OPTIMIZER.lower()]: @@ -154,8 +158,9 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): timeline_profiling_doc_url=const.TIMELINE_WITH_STACK_DOC_URL ) + sheet_name = "Affinity apis" optimization_item = OptimizeItem( - SupportedScopes.TIMELINE_FUSION_OPS, + sheet_name, desc, [suggestion] ) @@ -163,16 +168,16 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): self.result.add(OptimizeRecord(optimization_item)) record_title = ["Affinity API", "Code stacks", "Stack called counts"] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, headers=record_title) + self.result.add_detail(sheet_name, headers=record_title) for api_name, stacks_info in format_timeline_result(self.matched_op_stacks).items(): if not stacks_info: detail = [api_name, "null", "null"] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, detail=detail) + self.result.add_detail(sheet_name, detail=detail) else: for stack in stacks_info: detail = [api_name, *stack] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, detail=detail) + self.result.add_detail(sheet_name, detail=detail) def make_render(self): format_result_for_html = format_timeline_result(dict(self.matched_op_stacks), dump_html=True) @@ -185,7 +190,8 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): empty_stacks=self.empty_stacks, with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, api_doc_url=const.TIMELINE_API_DOC_URL, - result=format_result_for_html) + result=format_result_for_html, + priority_background_color=self.get_priority()) def query_stack(self, event_dataset): if all([len(matched_index) == 0 for matched_index in self._matched_op_index.values()]): diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py b/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py index f684a4892..92425910b 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py @@ -3,7 +3,7 @@ from typing import List from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import get_analyze_processes, ParallelJob @@ -21,7 +21,8 @@ class OpStackFinder: self.task_type = None self.matched_index = set() - def get_api_stack_by_op(self, event_dataset: TimelineEventDataset, op_name: List[str] = None, task_type: str = None, + def get_api_stack_by_op(self, event_dataset: ComputationAnalysisDataset, op_name: List[str] = None, + task_type: str = None, disable_multiprocess=False): """ :Param event_dataset: dataset of timeline event @@ -82,7 +83,13 @@ class OpStackFinder: for op_info in self._stack_record: result.add_detail('operator stacks', detail=op_info) - def _get_api_stack_by_op(self, event_dataset: TimelineEventDataset, op_name: str, task_type: str): + def query_stack(self, event_dataset: ComputationAnalysisDataset): + + if not event_dataset.dataset_len: + return + _ = event_dataset.parse_data_with_generator(self._query_stack_by_matched_index) + + def _get_api_stack_by_op(self, event_dataset: ComputationAnalysisDataset, op_name: str, task_type: str): for _, src_op_event in event_dataset.ops_with_task_type.items(): op_task_type = src_op_event.get(const.TASK_TYPE) @@ -110,6 +117,7 @@ class OpStackFinder: task_id = src_op_event.task_id if not task_id: continue + self.matched_index.add(dst_op_index) if dst_op_index not in self._task_id_record: self._task_id_record[dst_op_index] = [] @@ -122,7 +130,7 @@ class OpStackFinder: if not dst_op_event: return const.TIMELINE_BACKWARD_NO_STACK_CODE - return dst_op_event.get("dataset_index") + return int(dst_op_event.get("dataset_index")) def _query_index_by_acl_to_npu(self, acl_to_npu_event): if acl_to_npu_event: @@ -148,6 +156,7 @@ class OpStackFinder: return None event = TimelineEvent(event) stack = event.args.get(const.CALL_STACKS) + stack = stack if stack else const.NO_STACK_REASON_MAP.get(const.TIMELINE_BACKWARD_NO_STACK_CODE) for matched_op_info in self._task_id_record.get(index, []): self._stack_record.append([*matched_op_info, stack]) @@ -156,8 +165,3 @@ class OpStackFinder: self._stack_record.append([*matched_op_info, const.NO_STACK_REASON_MAP.get(const.TIMELINE_ACL_TO_NPU_NO_STACK_CODE)]) return None - - def query_stack(self, event_dataset: TimelineEventDataset): - if not event_dataset.dataset_len: - return - _ = event_dataset.parse_data_with_generator(self._query_stack_by_matched_index) diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py index 2786a7840..df8c22fa5 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py @@ -1,30 +1,32 @@ import logging -from typing import List, Dict, Any - from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() class SyncBNAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, **kwargs): super().__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): syncbn_checker = SyncBNChecker() syncbn_checker.check_syncbn(self.timeline_event_dataset) syncbn_checker.make_record(self.result) - syncbn_checker.make_render(self.html_render) + syncbn_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + def get_priority(self): + return PriorityBackgroundColor.high \ No newline at end of file diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py index c0e10448f..e83a15491 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py @@ -1,7 +1,7 @@ import logging import os -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.cluster_analyse.common_func.file_manager import FileManager @@ -20,7 +20,7 @@ class SyncBNChecker: self.max_syncbn_num = None self._init_rule() - def check_syncbn(self, event_dataset: TimelineEventDataset): + def check_syncbn(self, event_dataset: ScheduleAnalysisDataset): """ :Param event_dataset: dataset of timeline event """ @@ -43,14 +43,17 @@ class SyncBNChecker: for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.syncbn_issues: return + + priority = kwargs.get("priority") html_render.render_template(key="schedule", template_dir="templates", template_name="sync_batchnorm.html", desc=self.desc, - solutions=self.solutions) + solutions=self.solutions, + priority_background_color=priority) def _init_rule(self): syncbn_rule_path = os.path.join( diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py index d8906504c..61ec7d1fa 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py @@ -5,28 +5,33 @@ from typing import List, Dict, Any from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() class SynchronizeStreamAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, **kwargs): super().__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): synchronize_stream_checker = SynchronizeStreamChecker() synchronize_stream_checker.check_synchronize(self.timeline_event_dataset, kwargs.get("profiling_with_stack")) synchronize_stream_checker.make_record(self.result) - synchronize_stream_checker.make_render(self.html_render) + synchronize_stream_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + + def get_priority(self): + return PriorityBackgroundColor.low \ No newline at end of file diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py index 83ddd80a0..7af46f766 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py @@ -1,7 +1,7 @@ import logging from profiler.advisor.common import constant as const -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.analyzer.schedule.timeline_base_checker import TimelineBaseChecker @@ -21,7 +21,7 @@ class SynchronizeStreamChecker(TimelineBaseChecker): self.solutions = [] self.max_synchronize_num = None - def check_synchronize(self, event_dataset: TimelineEventDataset, profiling_with_stack=None): + def check_synchronize(self, event_dataset: ScheduleAnalysisDataset, profiling_with_stack=None): """ :Param event_dataset: dataset of timeline event """ @@ -73,10 +73,10 @@ class SynchronizeStreamChecker(TimelineBaseChecker): for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.synchronize_issues: return - + priority = kwargs.get("priority") format_result_for_html = format_timeline_result(dict(self.matched_op_stacks), dump_html=True) html_render.render_template(key="schedule", template_dir="templates", @@ -86,4 +86,5 @@ class SynchronizeStreamChecker(TimelineBaseChecker): result=format_result_for_html, with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, empty_stacks=self.empty_stacks, - framework_black_list=self.framework_black_list) + framework_black_list=self.framework_black_list, + priority_background_color=priority) diff --git a/profiler/advisor/analyzer/schedule/timeline_base_checker.py b/profiler/advisor/analyzer/schedule/timeline_base_checker.py index 8bc691502..f481733d4 100644 --- a/profiler/advisor/analyzer/schedule/timeline_base_checker.py +++ b/profiler/advisor/analyzer/schedule/timeline_base_checker.py @@ -4,7 +4,7 @@ import logging from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult logger = logging.getLogger() @@ -19,19 +19,11 @@ class TimelineBaseChecker(ABC): self.empty_stacks = True self.framework_black_list = False - @abstractmethod - def make_record(self, result: OptimizeResult): - pass - - @abstractmethod - def make_render(self, html_render): - pass - - def query_stack(self, event_dataset: TimelineEventDataset = None, profiling_with_stack: str = None): + def query_stack(self, event_dataset: ScheduleAnalysisDataset = None, profiling_with_stack: str = None): if all([len(matched_index) == 0 for matched_index in self._matched_op_index.values()]): return - event_dataset = event_dataset if not profiling_with_stack else TimelineEventDataset( + event_dataset = event_dataset if not profiling_with_stack else ScheduleAnalysisDataset( collection_path=profiling_with_stack, data={}, _datasets={}, analysis_mode="fusion_ops", build_dataset=False) diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 652e10b08..72b8dd3df 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -33,4 +33,6 @@ class SupportedScopes: SYNCBN = "syncbn" SYNCHRONIZE_STREAM = "synchronize_stream" FREQ_ANALYSIS = "freq_analysis" + MEMORY = "memory" + STAGE_COMPUTE = "stage_compute" GC_ANALYSIS = "gc_analysis" diff --git a/profiler/advisor/common/async_analysis_status.py b/profiler/advisor/common/async_analysis_status.py new file mode 100644 index 000000000..f67ca235a --- /dev/null +++ b/profiler/advisor/common/async_analysis_status.py @@ -0,0 +1,7 @@ +class AsyncAnalysisStatus: + FAILED = "failed" + SUCCESS = "success" + ANALYZING = "analyzing" + + FAILED_STATUS_CODE = 400 + NON_FAILED_STATUS_CODE = 200 diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index c97cfbfd1..298e94fc1 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -33,6 +33,7 @@ TASK_TYPE = "Task Type" CPU_OP = "cpu_op" AI_CORE = "AI_CORE" AI_CPU = "AI_CPU" +MIX_AIC = "MIX_AIC" CALL_STACKS = "Call stack" INPUT_DIMS = "Input Dims" OP_SEP = "-" @@ -48,8 +49,7 @@ NO_STACK_REASON_MAP = { TIMELINE_BACKWARD_NO_STACK_CODE: "Backward broadcast, without call stacks in profiling.", TIMELINE_ACL_TO_NPU_NO_STACK_CODE: "Incoming flow is 'acl_to_npu', without call stacks in profiling." } -TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/"\ - "Samples%20of%20Fused%20Operator%20API%20Replacement.md" +TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" AFFINITY_TRAINING_API = "Affinity training api" TIMELINE_WITH_STACK_DOC_URL = "https://www.hiascend.com/document/detail/zh/canncommercial/" \ "70RC1/modeldevpt/ptmigr/AImpug_0067.html" @@ -124,20 +124,6 @@ MAX_RETRIES = 3 TIMEOUT = 3 ADVISOR_RULE_PATH = "ADVISOR_RULE_PATH" -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. CLOUD_RULE_PATH = "rules/cloud/" DEFAULT_RULE_PATH = "./rules/" @@ -156,7 +142,17 @@ COMMUNICATION_JSON = "communication.json" BOTTLENECK = "bottleneck" DATA = "data" - +ADVISOR_ANALYSIS_OUTPUT_DIR = "advisor_analysis_result" +DEFAULT_PROCESSES = 8 +CLUSTER_ANALYSIS_FILE_PATTERN = [r'profiler_info_\d+\.json', "step_trace_time.csv", "communication.json", + "communication_matrix.json"] +ANALYSIS_OUTPUT_PATH = "ANALYSIS_OUTPUT_PATH" +DEFAULT_RANK_FOR_PROFILING_ANALYSIS = 0 +PROFILER_INFO_FILE_PATTERN = r"profiler_info_(\d+)\.json" +DISABLE_STREAMINIG_READER = "DISABLE_STREAMINIG_READER" FRAMEWORK_STACK_BLACK_LIST = ["torch", "torch_npu", "megatron", "deepspeed"] DISABLE_STREAMING_READER = "DISABLE_STREAMING_READER" -MAX_FILE_SIZE = 10**10 +MAX_FILE_SIZE = 10 ** 10 +MAX_NUM_PROCESSES = 4 +DEFAULT_STEP = "-1" +STEP_RANK_SEP = "_" diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index 445d4c87e..e268b4092 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -15,6 +15,7 @@ import logging import os +import re from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.utils.utils import singleton @@ -81,9 +82,11 @@ class ClusterDataset(Dataset): @singleton class ClusterStepTraceTimeDataset(ClusterDataset): RANK = "rank" + STAGE = "stage" def __init__(self, collection_path: str, data: dict, **kwargs): self._step_dict = defaultdict() + self._stages = [] super().__init__(collection_path, data, **kwargs) def _parse(self): @@ -101,14 +104,31 @@ class ClusterStepTraceTimeDataset(ClusterDataset): step_dict = defaultdict(lambda: [0, 0, 0]) for step_bean in step_data: if step_bean.type == self.RANK: - step_dict[step_bean.index][0] += step_bean.compute - step_dict[step_bean.index][1] += step_bean.communication - step_dict[step_bean.index][2] += step_bean.free + step_rank_record = [] + step = str(step_bean.step).replace(" ", "") or str(const.DEFAULT_STEP) + rank = str(step_bean.index).replace(" ", "") + if step: + step_rank_record.append(step) + if rank: + step_rank_record.append(rank) + + step_rank_index = const.STEP_RANK_SEP.join(step_rank_record) + step_dict[step_rank_index][0] += step_bean.compute + step_dict[step_rank_index][1] += step_bean.communication + step_dict[step_rank_index][2] += step_bean.free + if step_bean.type == self.STAGE: + stage = sorted(list(map(int, re.findall(r'\d+', step_bean.stage)))) + if stage in self._stages: + continue + self._stages.append(stage) return step_dict def get_data(self): return self._step_dict + def get_stages(self): + return sorted(self._stages) + @singleton class ClusterCommunicationDataset(ClusterDataset): @@ -158,7 +178,7 @@ class ClusterCommunicationDataset(ClusterDataset): self.hccl_dict.setdefault(comm_group, defaultdict(lambda: defaultdict(list))) for step, step_dict in group_dict.items(): for op, op_dict in step_dict.items(): - self.compute_bandwidth(op_dict) + self.compute_bandwidth(step.lower().lstrip("step") or str(const.DEFAULT_STEP), op_dict) self.process_hccl_info(comm_group, step, op, op_dict) def process_hccl_info(self, group, step, op, op_dict): @@ -175,7 +195,7 @@ class ClusterCommunicationDataset(ClusterDataset): msg = "[ERROR] Cluster_communication.json has invalid structure." raise ValueError(msg) from e - def compute_bandwidth(self, op_dict: dict): + def compute_bandwidth(self, step, op_dict: dict): for rank_id, rank_dict in op_dict.items(): try: rank = int(rank_id) @@ -184,17 +204,17 @@ class ClusterCommunicationDataset(ClusterDataset): raise ValueError(msg) from e for comm_type, bw_dict in rank_dict.get(self.COMMUNICATION_BANDWIDTH_INFO, {}).items(): if comm_type == self.SDMA: - self.rank_bw_dict[rank][self.SDMA_SIZE_MB] += bw_dict.get(self.TRANSIT_SIZE) - self.rank_bw_dict[rank][self.SDMA_TIME_MS] += bw_dict.get(self.TRANSIT_TIME) + self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.SDMA_SIZE_MB] += bw_dict.get(self.TRANSIT_SIZE) + self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.SDMA_TIME_MS] += bw_dict.get(self.TRANSIT_TIME) if comm_type == self.RDMA: - self.rank_bw_dict[rank][self.RDMA_SIZE_MB] += bw_dict.get(self.TRANSIT_SIZE) - self.rank_bw_dict[rank][self.RDMA_TIME_MS] += bw_dict.get(self.TRANSIT_TIME) - - for rank, rank_dict in self.rank_bw_dict.items(): - self.rank_bw_dict[rank][self.RDMA_BANDWIDTH] = self.compute_ratio( - self.rank_bw_dict[rank][self.RDMA_SIZE_MB], self.rank_bw_dict[rank][self.RDMA_TIME_MS]) - self.rank_bw_dict[rank][self.SDMA_BANDWIDTH] = self.compute_ratio( - self.rank_bw_dict[rank][self.SDMA_SIZE_MB], self.rank_bw_dict[rank][self.SDMA_TIME_MS]) + self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.RDMA_SIZE_MB] += bw_dict.get(self.TRANSIT_SIZE) + self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.RDMA_TIME_MS] += bw_dict.get(self.TRANSIT_TIME) + + for step_rank in self.rank_bw_dict.keys(): + self.rank_bw_dict[step_rank][self.RDMA_BANDWIDTH] = self.compute_ratio( + self.rank_bw_dict[step_rank][self.RDMA_SIZE_MB], self.rank_bw_dict[step_rank][self.RDMA_TIME_MS]) + self.rank_bw_dict[step_rank][self.SDMA_BANDWIDTH] = self.compute_ratio( + self.rank_bw_dict[step_rank][self.SDMA_SIZE_MB], self.rank_bw_dict[step_rank][self.SDMA_TIME_MS]) def get_data(self): return self.rank_bw_dict diff --git a/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py b/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py index b108fc77a..8ae0e55f2 100644 --- a/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py +++ b/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py @@ -65,3 +65,6 @@ class ClusterStepTraceTimeBean: msg = "[ERROR] Cluster step trace time.csv has invalid value in column 'Free'." raise ValueError(msg) from e + @property + def stage(self) -> int: + return self._data.get(self.INDEX) diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/advisor/dataset/timeline_event_dataset.py index c76314641..44bbc141d 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/advisor/dataset/timeline_event_dataset.py @@ -1,101 +1,47 @@ +import inspect import logging -import os -from typing import List, Any import traceback +from collections import OrderedDict import ijson from tqdm import tqdm from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.utils.utils import get_file_path_from_directory, check_path_valid, singleton -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.utils.utils import get_file_path_from_directory, check_path_valid, singleton, convert_to_float +from profiler.advisor.dataset.timeline_op_collector.timeline_op_collector import ( + OpCompileCollector, + SynchronizeStreamCollector, + MemCollector, + DataloaderCollector, + SyncBNCollector, + AtenCollector, + OptimizerCollector, + FrequencyCollector, + SpecificTaskTypeOpCollector, + TorchToNpuCollector, + AclToNpuCollector, + OpStackCollector, + StepCollector, + GcCollector +) logger = logging.getLogger() -class OpCompileCollector: - def __init__(self): - self._total_op_compile_counter = 0 - self._total_op_compile_time = 0.0 +class BaseTimelineEventDataset: + PROFILER_STEP_PREFIX = "ProfilerStep" - @property - def total_time(self): - return self._total_op_compile_time - - @property - def total_count(self): - return self._total_op_compile_counter - - def is_empty(self): - return self._total_op_compile_counter == 0 - - def update(self, event: TimelineEvent): - self._total_op_compile_time += float(event.dur) - self._total_op_compile_counter += 1 - - def unset(self): - self._total_op_compile_counter = 0 - self._total_op_compile_time = 0.0 - - -class SynchronizeStreamCollector: - - def __init__(self): - self._synchronize_stream_count = 0 - self._slow_synchronize_stream = [] - self.rule = SynchronizeStreamCollector._load_rule() - - @property - def total_count(self): - return self._synchronize_stream_count - - @property - def slow_synchronize_stream(self): - return self._slow_synchronize_stream - - @staticmethod - def _load_rule(): - sync_stream_rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "rules", - "synchronize.yaml") - - sync_stream_rule = FileManager.read_yaml_file(sync_stream_rule_path) - return sync_stream_rule - - def update_sync_stream_count(self): - self._synchronize_stream_count += 1 - - def append_slow_sync_stream(self, event): - if float(event.dur) / 1000 >= self.rule.get("slow_synchronize_threshold", 10): - self._slow_synchronize_stream.append(event) - - def unset(self): - self._synchronize_stream_count = 0 - self._slow_synchronize_stream = [] - - -@singleton -class TimelineEventDataset: + collector_map = {} def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: - self._ops_with_task_type = {} - self._ops_with_stack = {} - self._ops_compile = OpCompileCollector() - self._torch_to_npu = {} - self._acl_to_npu = set() - self._aten: List[Any] = [] - self._optimizer: List[Any] = [] - self._dataloader: List[Any] = [] - self._sync_batchnorm: List[Any] = [] - self._gc: List[Any] = [] - self._synchronize_stream = SynchronizeStreamCollector() self.timeline_dir = collection_path + self.profiler_step = [] self.timeline_data_list = get_file_path_from_directory(collection_path, lambda file: file.endswith("trace_view.json")) self.dataset_len = None - self.analysis_mode = kwargs.get("analysis_mode") - self.task_type = kwargs.get("task_type") - + self.step = kwargs.get("step") + self.step_duration = None if not build_dataset: return @@ -105,59 +51,6 @@ class TimelineEventDataset: data[key] = [] data[key].append(self) - if self.analysis_mode in ["op_stack", "all"]: - self._task_op_names = list(set([event_key.split("-")[0] for event_key in self._ops_with_task_type.keys()])) - - self._post_process() - - @property - def ops_with_stack(self): - return self._ops_with_stack - - @property - def ops_compile(self): - return self._ops_compile - - @property - def torch_to_npu(self): - return self._torch_to_npu - - @property - def acl_to_npu(self): - return self._acl_to_npu - - @property - def ops_with_task_type(self): - return self._ops_with_task_type - - @property - def task_op_names(self): - return self._task_op_names - - @property - def optimizer(self): - return self._optimizer - - @property - def aten(self): - return self._aten - - @property - def dataloader(self): - return self._dataloader - - @property - def sync_batchnorm(self): - return self._sync_batchnorm - - @property - def gc_events(self): - return self._gc - - @property - def synchronize_stream(self): - return self._synchronize_stream - @classmethod def get_key(cls): """ @@ -166,6 +59,23 @@ class TimelineEventDataset: """ return cls.__module__.rsplit('.', maxsplit=1)[-1] + def get_post_process_kwargs(self, func_name): + kwargs = {} + if func_name == FrequencyCollector.__name__: + ops_with_task_type = getattr(self, "ops_with_task_type", {}).values() + kwargs["ai_core_ops"] = [op for op in ops_with_task_type if + op.get(const.TASK_TYPE) in [const.AI_CORE, const.MIX_AIC]] + return kwargs + + def add_event(self, index, event): + event["dataset_index"] = index + if not isinstance(event, TimelineEvent): + event = TimelineEvent(event) + + for _, collector in self.collector_map.items(): + collector.add_op(event) + return True + def parse(self): if len(self.timeline_data_list) == 0: @@ -173,10 +83,10 @@ class TimelineEventDataset: return False if len(self.timeline_data_list) > 1: - logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis.", + logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis .", self.timeline_dir) - result = self.parse_data_with_generator(self._add_event) + result = self.parse_data_with_generator(self.add_event) if not self.dataset_len: self.dataset_len = len(result) @@ -202,137 +112,100 @@ class TimelineEventDataset: timeline_data_path) return result - def _add_ops_with_task_type(self, event): - key = f"{event.name}-{event.ts}" - self._ops_with_task_type[key] = TimelineEvent( - { - const.TASK_TYPE: event.args.get(const.TASK_TYPE), - "task_id": event.args.get("Task Id"), - "tid": event.tid, - "name": event.name, - "ts": str(event.ts) - } - ) - - def _add_ops_with_stack(self, event): - self._ops_with_stack[str(event.ts)] = TimelineEvent({"name": event.name, "dataset_index": event.dataset_index}) - - def _add_torch_to_npu(self, event): - key = f"{event.ph}-{event.id}" - self._torch_to_npu[key] = TimelineEvent({"tid": event.tid, "ts": str(event.ts)}) - - def _add_acl_to_npu(self, event): - # op with task type equals to ai_cpu which derived from acl_to_npu do not have stacks - self._acl_to_npu.add(str(event.ts)) - - def _add_op_compile(self, event: TimelineEvent): - if event.name == const.OP_COMPILE_NAME or event.args.get("id") == const.OP_COMPILE_ID: - self._ops_compile.update(event) - - def _add_gc(self, event: TimelineEvent): - if event.get("cat") and event.get("cat").lower() == 'gc': - self._gc.append(event) - - def _add_optimizer(self, event: TimelineEvent): - self._optimizer.append(TimelineEvent({"name": event.name, "dataset_index": event.dataset_index})) - - def _add_aten(self, event: TimelineEvent): - self._aten.append(TimelineEvent({ - "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur - })) - - def _add_dataloader(self, event: TimelineEvent): - if "dataloader" in event.name.lower(): - self._dataloader.append(TimelineEvent({ - "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur, - "stack": event.args.get("Call stack") - })) - - def _add_sync_batchnorm(self, event: TimelineEvent): - if event.name.lower() == "syncbatchnorm": - self._sync_batchnorm.append(TimelineEvent({ - "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur - })) - - def _add_synchronize(self, event: TimelineEvent): - if event.name.startswith(const.SYNC_STREAM): - self._synchronize.append(TimelineEvent({ - "name": event.name, "ts": event.ts, "dur": event.dur - })) - - def _add_specific_operator(self, event): - # for analysis of operator aclOpCompile, enable jit_compILE=False - self._add_op_compile(event) - # for analysis of slow dataloader.__next__ - self._add_dataloader(event) - # for analysis of syncBatchNorm operator, prompt users to replace source code of torch_npu's syncbn - self._add_sync_batchnorm(event) - # for analysis of GcAnalyzer - self._add_gc(event) - - def _add_event(self, index, event): - event["dataset_index"] = index - if not isinstance(event, TimelineEvent): - event = TimelineEvent(event) - - self._add_specific_operator(event) - - if self.analysis_mode == "fusion_ops": - self._add_event_for_fusion_ops(event) - elif self.analysis_mode == "op_stack": - self._add_event_for_op_stack(event) + def _get_target_ops_by_step(self, op_list): + target_ops = [] + if not self.profiler_step: + return op_list + if not self.step or f"ProfilerStep#{self.step}" not in [event.name for event in self.profiler_step]: + target_ops = op_list + if self.profiler_step: + self.step_duration = convert_to_float(self.profiler_step[-1].dur) else: - self._add_event_for_fusion_ops(event) - self._add_event_for_op_stack(event) - return True - - def _add_event_for_fusion_ops(self, event): - if event.name.lower().startswith(f"{const.ATEN}{const.ATEN_SEP}") or event.name.lower().startswith( - f"{const.NPU}{const.ATEN_SEP}"): - self._add_aten(event) - return - - # 检查cann层同步操作，根据时间窗口索引到host侧的aten算子并给出堆栈 - if event.name.startswith(const.SYNC_STREAM): - self._add_aten(event) + for step_event in self.profiler_step: + if step_event.name != f"ProfilerStep#{self.step}": + continue + self.step_duration = convert_to_float(step_event.dur) + for op_event in op_list: + if step_event.ts_include(op_event): + target_ops.append(op_event) + target_ops.sort(key=lambda x: convert_to_float(x.ts)) + return target_ops + + def _collector_post_process(self): + # 按step过滤collector中的算子，并将过滤后的算子设置为当前dataset的property，与原始TimelineEventDataset的property保持一致 + for collector_name, collector in self.collector_map.items(): + logger.debug("Start post process for operator collector: %s", collector_name) + if collector.require_filter_by_step: + logger.debug("Operator Collector %s requires filter ops by step %s", collector_name, self.step) + target_op_list = self._get_target_ops_by_step(collector.op_list) + else: + logger.debug("Operator Collector %s use operators of all step for analysis", collector_name) + target_op_list = collector.op_list - if event.name.startswith(f"{const.OPTIMIZER}.{const.OPTIMIZER_STEP}{const.OPTIMIZER_SEP}"): - self._add_optimizer(event) - return + logger.debug("Source number of ops is %s, number of ops after filtered by rank is %s", + len(collector.op_list), len(target_op_list)) - def _add_event_for_op_stack(self, event): - if event.name.lower() == const.TORCH_TO_NPU: - self._add_torch_to_npu(event) - return + collector_kwargs = self.get_post_process_kwargs(collector_name) + collector.post_process(target_op_list, **collector_kwargs) + for property_name, property_value in collector.attribute_to_dataset.items(): + setattr(self, property_name, property_value) - if event.args.get(const.CALL_STACKS): - self._add_ops_with_stack(event) - return - if event.args.get(const.TASK_TYPE) and event.args.get(const.TASK_TYPE) in [const.AI_CORE, const.AI_CPU]: - self._add_ops_with_task_type(event) - return +@singleton +class ScheduleAnalysisDataset(BaseTimelineEventDataset): + collector_map = OrderedDict( + StepCollector=StepCollector(), + MemCollector=MemCollector(), + OpCompileCollector=OpCompileCollector(), + SynchronizeStreamCollector=SynchronizeStreamCollector(), + DataloaderCollector=DataloaderCollector(), + SyncBNCollector=SyncBNCollector(), + AtenCollector=AtenCollector(), + OptimizerCollector=OptimizerCollector(), + GcCollector=GcCollector() + ) - if event.name and event.ts and event.name == const.ACL_TO_NPU: - self._add_acl_to_npu(event) - return + def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: + super().__init__(collection_path, data, build_dataset, **kwargs) + self.aten = None + self.synchronize_stream = None + self._collector_post_process() + self._post_process() def _post_process(self): # eliminate sub aten operator of the first level aten operator by 'ts' and 'dur', # keep the first level aten operator contiguous formated_atens = [] - for event in sorted(self._aten, key=lambda x: x.get("ts", -1)): + if not hasattr(self, "aten") or not hasattr(self, "synchronize_stream"): + return + + for event in sorted(self.aten, key=lambda x: x.get("ts", -1)): if event.name.startswith(const.ATEN): if not formated_atens or not formated_atens[-1].ts_include(event): formated_atens.append(event) elif event.name.startswith(const.SYNC_STREAM): - self._synchronize_stream.update_sync_stream_count() - if formated_atens[-1].ts_include(event): + self.synchronize_stream.update_sync_stream_count() + if formated_atens and formated_atens[-1].ts_include(event): # 使用aten算子的索引，用于查询堆栈 event["dataset_index"] = formated_atens[-1].get("dataset_index") - self._synchronize_stream.append_slow_sync_stream(event) + self.synchronize_stream.append_slow_sync_stream(event) else: continue - self._aten = formated_atens + self.aten = formated_atens + + +class ComputationAnalysisDataset(BaseTimelineEventDataset): + collector_map = OrderedDict( + StepCollector=StepCollector(), + SpecificTaskTypeOpCollector=SpecificTaskTypeOpCollector(), + TorchToNpuCollector=TorchToNpuCollector(), + AclToNpuCollector=AclToNpuCollector(), + OpStackCollector=OpStackCollector(), + FrequencyCollector=FrequencyCollector(), + ) + + def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: + super().__init__(collection_path, data, build_dataset, **kwargs) + self._collector_post_process() diff --git a/profiler/advisor/dataset/timeline_op_collector/__init__.py b/profiler/advisor/dataset/timeline_op_collector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py new file mode 100644 index 000000000..56e6165dd --- /dev/null +++ b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py @@ -0,0 +1,376 @@ +import logging +import math +import os +from abc import abstractmethod, ABCMeta + +from profiler.advisor.common import constant as const +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.advisor.utils.utils import convert_to_float +from profiler.cluster_analyse.common_func.file_manager import FileManager + +logger = logging.getLogger() + + +class BaseOpCollector(metaclass=ABCMeta): + + def __init__(self): + self.attribute_to_dataset = {} + self.op_list = [] + self.require_filter_by_step = True + + @abstractmethod + def add_op(self): + """ add timeline event into self.op_list, and then will filter event in self.op_list by specific step + """ + pass + + @abstractmethod + def post_process(self): + """ convert self.op_list to required format like dict, set and so on and then record the final object into + self.attribute_to_dataset which used to set property of timeline event dataset + """ + pass + + +class StepCollector(BaseOpCollector): + KEY_WORD = "ProfilerStep" + + def __init__(self): + super().__init__() + self.require_filter_by_step = False + + def add_op(self, event): + if event.name.startswith(self.KEY_WORD): + self.op_list.append(event) + + def post_process(self, *args, **kwargs): + self.attribute_to_dataset["profiler_step"] = self.op_list + + +class OpCompileCollector(BaseOpCollector): + def __init__(self): + super().__init__() + self._total_op_compile_counter = 0 + self._total_op_compile_time = 0.0 + + @property + def total_time(self): + return self._total_op_compile_time + + @property + def total_count(self): + return self._total_op_compile_counter + + def is_empty(self): + return self._total_op_compile_counter == 0 + + def update(self, event: TimelineEvent): + self._total_op_compile_time += float(event.dur) + self._total_op_compile_counter += 1 + + def unset(self): + self._total_op_compile_counter = 0 + self._total_op_compile_time = 0.0 + + def add_op(self, event): + if event.name == const.OP_COMPILE_NAME or event.args.get("id") == const.OP_COMPILE_ID: + self.op_list.append(event) + + def post_process(self, target_op_list, **kwargs): + for op in target_op_list: + self.update(op) + + self.attribute_to_dataset["ops_compile"] = self + + +class SynchronizeStreamCollector(BaseOpCollector): + + def __init__(self): + super().__init__() + self._synchronize_stream_count = 0 + self._slow_synchronize_stream = [] + self.rule = SynchronizeStreamCollector._load_rule() + + @property + def total_count(self): + return self._synchronize_stream_count + + @property + def slow_synchronize_stream(self): + return self._slow_synchronize_stream + + @staticmethod + def _load_rule(): + sync_stream_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "synchronize.yaml") + + sync_stream_rule = FileManager.read_yaml_file(sync_stream_rule_path) + return sync_stream_rule + + def update_sync_stream_count(self): + self._synchronize_stream_count += 1 + + def append_slow_sync_stream(self, event): + if float(event.dur) / 1000 >= self.rule.get("slow_synchronize_threshold", 10): + self._slow_synchronize_stream.append(event) + + def unset(self): + self._synchronize_stream_count = 0 + self._slow_synchronize_stream = [] + + def add_op(self, event): + return self.op_list + + def post_process(self, *args, **kwargs): + self.attribute_to_dataset["synchronize_stream"] = self + + +class MemCollector(BaseOpCollector): + MEMORY_OP_NAME = ["AscendCL@aclMallocMemInner", "AscendCL@aclrtFreePhysical"] + + def __init__(self): + super().__init__() + self.mem_op_info = {} + self.rule = self._load_rule() + + @staticmethod + def _load_rule(): + memory_rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "memory.yaml") + + memory_rule = FileManager.read_yaml_file(memory_rule_path) + return memory_rule + + def add_op(self, event): + if event.name not in self.MEMORY_OP_NAME: + return + self.op_list.append(event) + + def post_process(self, target_op_list, **kwargs): + for op in target_op_list: + if op.name not in self.mem_op_info: + self.mem_op_info[op.name] = dict(count=0, total_dur=0) + self.mem_op_info[op.name]["count"] += 1 + self.mem_op_info[op.name]["total_dur"] += float(op.dur) + + self.attribute_to_dataset["memory_ops"] = self + + +class DataloaderCollector(BaseOpCollector): + key_word = "dataloader" + + def __init__(self): + super().__init__() + + def add_op(self, event): + if self.key_word in event.name.lower(): + self.op_list.append(TimelineEvent({ + "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur, + "stack": event.args.get("Call stack") + })) + + def post_process(self, *args, **kwargs): + self.attribute_to_dataset["dataloader"] = self.op_list + + +class SyncBNCollector(BaseOpCollector): + key_word = "syncbatchnorm" + + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name.lower() == self.key_word: + self.op_list.append(TimelineEvent({ + "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur + })) + + def post_process(self, target_op_list, **kwargs): + self.attribute_to_dataset["sync_batchnorm"] = target_op_list + + +class AtenCollector(BaseOpCollector): + + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name.lower().startswith(f"{const.ATEN}{const.ATEN_SEP}") or event.name.lower().startswith( + f"{const.NPU}{const.ATEN_SEP}"): + self._add_aten(event) + return + + # 检查cann层同步操作，根据时间窗口索引到host侧的aten算子并给出堆栈 + if event.name.startswith(const.SYNC_STREAM): + self._add_aten(event) + + def post_process(self, target_op_list, **kwargs): + self.attribute_to_dataset["aten"] = target_op_list + + def _add_aten(self, event: TimelineEvent): + self.op_list.append(TimelineEvent({ + "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur + })) + + +class OptimizerCollector(BaseOpCollector): + + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name.startswith(f"{const.OPTIMIZER}.{const.OPTIMIZER_STEP}{const.OPTIMIZER_SEP}"): + self.op_list.append(TimelineEvent( + {"name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur})) + + def post_process(self, target_op_list, **kwargs): + self.attribute_to_dataset["optimizer"] = target_op_list + + +class FrequencyCollector(BaseOpCollector): + KEY_WORD = "AI Core Freq" + + def __init__(self): + super().__init__() + self._previous_freq_index = -1 + + @staticmethod + def get_op_frequency(ai_core_ops, ai_core_freq): + ai_core_freq.sort(key=lambda x: float(x.ts)) + op_freq_record = {} + + op_index, freq_index = 0, 0 + while op_index < len(ai_core_ops) and freq_index < len(ai_core_freq): + op_event = ai_core_ops[op_index] + op_end_time = convert_to_float(op_event.ts) + convert_to_float(op_event.dur) + op_freq_list = [] + while freq_index < len(ai_core_freq): + freq_event = ai_core_freq[freq_index] + if convert_to_float(freq_event.end) < op_end_time: + op_freq_list.append(convert_to_float(freq_event.args.MHz)) + freq_index += 1 + continue + elif convert_to_float(freq_event.ts) < op_end_time: + if op_event.name not in op_freq_record: + op_freq_record[op_event.name] = {"count": 0, "dur": 0, "freq_list": []} + op_freq_record[op_event.name]["count"] += 1 + op_freq_record[op_event.name]["dur"] += convert_to_float(op_event.dur) + op_freq_list.append(convert_to_float(freq_event.args.MHz)) + op_freq_record[op_event.name]["freq_list"].append(min(op_freq_list)) + break + else: + break + + op_index += 1 + return op_freq_record + + def add_op(self, event): + if event.name == self.KEY_WORD: + if self._previous_freq_index != -1: + self.op_list[self._previous_freq_index]["end"] = event.get("ts", float(math.inf)) + self._previous_freq_index += 1 + event.setdefault("end", float(math.inf)) + self.op_list.append(event) + + def post_process(self, target_op_list, **kwargs): + ai_core_ops = kwargs.get("ai_core_ops", []) + if not ai_core_ops: + return + ai_core_ops.sort(key=lambda x: float(x.ts)) + op_freq = FrequencyCollector.get_op_frequency(ai_core_ops, target_op_list) + self.attribute_to_dataset["op_freq"] = op_freq + + +class SpecificTaskTypeOpCollector(BaseOpCollector): + + def __init__(self, op_type_list=None): + super().__init__() + self.op_type_list = op_type_list if op_type_list else [const.AI_CPU, const.AI_CORE, const.MIX_AIC] + + def add_op(self, event): + if event.args.get(const.TASK_TYPE) and event.args.get(const.TASK_TYPE) in self.op_type_list: + self.op_list.append( + TimelineEvent( + { + const.TASK_TYPE: event.args.get(const.TASK_TYPE), + "task_id": event.args.get("Task Id"), + "tid": event.tid, + "name": event.name, + "ts": str(event.ts), + "dur": str(event.dur) + } + ) + ) + + def post_process(self, target_op_list, **kwargs): + op_map = dict() + for op in target_op_list: + key = f"{op.name}-{op.ts}" + op_map[key] = op + + self.attribute_to_dataset["ops_with_task_type"] = op_map + self.attribute_to_dataset["task_op_names"] = list( + set([event_key.split("-")[0] for event_key in op_map.keys()])) + + +class TorchToNpuCollector(BaseOpCollector): + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name.lower() == const.TORCH_TO_NPU: + self.op_list.append(TimelineEvent({"tid": event.tid, "ts": str(event.ts), "ph": event.ph, "id": event.id})) + + def post_process(self, target_op_list, **kwargs): + op_map = dict() + for op in target_op_list: + key = f"{op.ph}-{op.id}" + op_map[key] = op + + self.attribute_to_dataset["torch_to_npu"] = op_map + + +class AclToNpuCollector(BaseOpCollector): + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name and event.ts and event.name == const.ACL_TO_NPU: + self.op_list.append(TimelineEvent({"ts": event.ts})) + + def post_process(self, target_op_list, **kwargs): + op_record = set(str(op.ts) for op in target_op_list) + self.attribute_to_dataset["acl_to_npu"] = op_record + + +class OpStackCollector(BaseOpCollector): + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.args.get(const.CALL_STACKS): + self.op_list.append( + TimelineEvent({"name": event.name, "dataset_index": event.dataset_index, "ts": event.ts})) + + def post_process(self, target_op_list, **kwargs): + op_map = dict() + for op in target_op_list: + op_map[str(op.ts)] = op + + self.attribute_to_dataset["ops_with_stack"] = op_map + + +class GcCollector(BaseOpCollector): + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.cat and isinstance(event.cat, str) and event.cat.lower() == "gc": + self.op_list.append(TimelineEvent( + {"name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur})) + + def post_process(self, target_op_list, **kwargs): + self.attribute_to_dataset["gc_events"] = self.op_list diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py deleted file mode 100644 index 3d8e22b7c..000000000 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py +++ /dev/null @@ -1,65 +0,0 @@ -import unittest -import os -import sys -import yaml - -from profiler.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestDataloaderChecker(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - def setUp(self) -> None: - rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), - "advisor", "rules", "dataloader.yaml") - - with open(rule_path, "rb") as file: - self.rule = yaml.safe_load(file) - - def test_no_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) * 1000 - dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=True) - - checker = DataloaderChecker() - checker.check_slow_dataloader(dataset) - self.assertFalse(checker.dataloader_issues) - - def test_no_slow_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) * 1000 - dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=False) - checker = DataloaderChecker() - checker.check_slow_dataloader(dataset) - self.assertFalse(checker.dataloader_issues) - - def test_found_slow_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") + 1) * 1000 - dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=False) - checker = DataloaderChecker() - checker.check_slow_dataloader(dataset) - self.assertTrue(checker.dataloader_issues) - - desc = self.rule.get("problem").format(dataloader_duration=dataloader_duration / 1000, - dataloader_duration_threshold=self.rule.get( - "dataloader_duration_threshold")) - - self.assertEqual(desc, checker.desc) - - def _get_mock_dataset(self, dur, is_empty_dataset=False): - dataset = TimelineEvent() - if is_empty_dataset: - return dataset - - dataset["dataloader"] = [TimelineEvent({"dur": dur, "name": "dataloader"})] - return dataset - - -if __name__ == '__main__': - tester = TestDataloaderChecker() - tester.test_no_dataloader() - tester.test_no_slow_dataloader() - tester.test_found_slow_dataloader() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py deleted file mode 100644 index d1df810a0..000000000 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py +++ /dev/null @@ -1,62 +0,0 @@ -import unittest -import os -import sys -import yaml - -from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestSyncBNChecker(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - def setUp(self) -> None: - rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), - "advisor", "rules", "sync_batchnorm.yaml") - - with open(rule_path, "rb") as file: - self.rule = yaml.safe_load(file) - - def test_no_syncbn(self): - dataset = self._get_mock_dataset(1, is_empty_dataset=True) - - checker = SyncBNChecker() - checker.check_syncbn(dataset) - self.assertFalse(checker.syncbn_issues) - - def test_syncbn_not_reach_threshold(self): - dataset = self._get_mock_dataset(self.rule.get("max_syncbn_num") - 1, is_empty_dataset=False) - checker = SyncBNChecker() - checker.check_syncbn(dataset) - self.assertFalse(checker.syncbn_issues) - - def test_found_slow_dataloader(self): - dataset = self._get_mock_dataset(self.rule.get("max_syncbn_num") + 1, is_empty_dataset=False) - checker = SyncBNChecker() - checker.check_syncbn(dataset) - self.assertTrue(checker.syncbn_issues) - - desc = self.rule.get("problem").format(syncbn_num=self.rule.get("max_syncbn_num") + 1) - - self.assertEqual(desc, checker.desc) - - def _get_mock_dataset(self, syncbn_num, is_empty_dataset=False): - dataset = TimelineEvent() - if is_empty_dataset: - return dataset - - dataset["sync_batchnorm"] = [] - for _ in range(syncbn_num): - dataset["sync_batchnorm"].append(TimelineEvent({"name": "SyncBatchNorm"})) - return dataset - - -if __name__ == '__main__': - tester = TestSyncBNChecker() - tester.test_no_syncbn() - tester.test_syncbn_not_reach_threshold() - tester.test_found_slow_dataloader() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py deleted file mode 100644 index 360363ce3..000000000 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -import os -import sys -import yaml - -from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker -from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestSynchronizeChecker(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - def setUp(self) -> None: - rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), - "advisor", "rules", "synchronize.yaml") - - with open(rule_path, "rb") as file: - self.rule = yaml.safe_load(file) - - def test_no_synchronize_stream(self): - dataset = self._get_mock_dataset(1, [], is_empty_dataset=True) - - checker = SynchronizeStreamChecker() - checker.check_synchronize(dataset) - self.assertFalse(checker.synchronize_issues) - - def test_max_synchronize_stream(self): - dataset = self._get_mock_dataset(100, [], is_empty_dataset=False) - checker = SynchronizeStreamChecker() - checker.check_synchronize(dataset) - self.assertFalse(checker.synchronize_issues) - - def _get_mock_dataset(self, total_count, slow_synchronize_stream, is_empty_dataset=False): - dataset = TimelineEvent() - if is_empty_dataset: - return dataset - - dataset["synchronize_stream"] = TimelineEvent( - dict( - total_count=total_count, - slow_synchronize_stream=slow_synchronize_stream, - rule=dict(max_synchronize_num=10, problem="", solutions=[]), - ) - ) - return dataset - - -if __name__ == '__main__': - tester = TestSynchronizeChecker() - tester.test_no_synchronize_stream() - tester.test_max_synchronize_stream() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py b/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py deleted file mode 100644 index 9060bfb8d..000000000 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py +++ /dev/null @@ -1,46 +0,0 @@ -import unittest -import os -import sys - -work_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))))) -sys.path.insert(0, work_path) -from unittest.mock import patch -from profiler.advisor.analyzer.schedule import dispatch -from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset -from profiler.advisor.display.html.render import HTMLRender -from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env - - -class TestOperatorDispatchAnalyzer(unittest.TestCase): - @classmethod - def tearDownClass(cls) -> None: - recover_env() - - @patch("profiler.advisor.common.constant.MAX_OP_COMPILE_NUM", 5) - def test_ops_dispatch_analyzer(self): - kwargs = {"analysis_mode": "all"} - data_root_dir = os.path.dirname(os.path.realpath(__file__)) - op_dispatch_analyzer = OpDispatchAnalyzer(data_root_dir, **kwargs) - - results = op_dispatch_analyzer.optimize(**kwargs) - self.assertTrue(results.page_dict) - self.assertIsNotNone(results.sheet_recorder.sheet_data.get("operator dispatch")) - - @patch("profiler.advisor.common.constant.MAX_OP_COMPILE_NUM", 5) - def test_ops_dispatch_make_render(self): - kwargs = {"analysis_mode": "timeline"} - data_root_dir = os.path.dirname(os.path.realpath(__file__)) - op_dispatch = OpDispatchAnalyzer(data_root_dir, **kwargs) - event_dataset = op_dispatch.get_first_data_by_key(op_dispatch.dataset_list, TimelineEventDataset.get_key()) - - op_dispatch.get_op_compile_info(event_dataset) - html_render = HTMLRender() - op_dispatch.make_render(html_render) - self.assertTrue(len(html_render.render_list) >= 1) - - -if __name__ == '__main__': - tester = TestOperatorDispatchAnalyzer() - tester.test_ops_dispatch_analyzer() - tester.test_ops_dispatch_make_render() diff --git a/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py b/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py deleted file mode 100644 index eb383a659..000000000 --- a/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py +++ /dev/null @@ -1,170 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes - - -class TestRdmaAdvice(unittest.TestCase): - TMP_DIR = "./tmp/" - OUTPUT_DIR = "./tmp/cluster_analysis_output" - interface = None - err_interface = None - - def tearDown(self): - if os.path.exists(TestRdmaAdvice.TMP_DIR): - shutil.rmtree(TestRdmaAdvice.TMP_DIR) - if os.path.exists(TestRdmaAdvice.OUTPUT_DIR): - shutil.rmtree(TestRdmaAdvice.OUTPUT_DIR) - self.clear_htmls() - - def setUp(self): - if os.path.exists(TestRdmaAdvice.TMP_DIR): - shutil.rmtree(TestRdmaAdvice.TMP_DIR) - if not os.path.exists(TestRdmaAdvice.TMP_DIR): - os.makedirs(TestRdmaAdvice.TMP_DIR) - if not os.path.exists(TestRdmaAdvice.OUTPUT_DIR): - os.makedirs((TestRdmaAdvice.OUTPUT_DIR)) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“mstt”开头 - if filename.startswith("mstt"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def get_cluster_communication_view(cls): - data = {"p2p":{"step1" : { - "hcom_broadcast__844_0_1@13681369207305868844": { - "0": { - "Communication Time Info": { - "Start Timestamp(us)": 1713174287354248.0, - "Elapse Time(ms)": 4688, - "Transit Time(ms)": 0, - "Wait Time(ms)": 0.01162, - "Synchronization Time(ms)": 0.01162, - "Idle Time(ms)": 39.0606, - "Wait Time Ratio": 1.0, - "Synchronization Time Ratio": 1.0 - }, - "Communication Bandwidth Info": { - "RDMA": { - "Transit Size(MB)": 80, - "Transit Time(ms)": 4600, - "Bandwidth(GB/s)": 0.003, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "HCCS": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "PCIE": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SDMA": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SIO": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - } - } - }, - "16": { - "Communication Time Info": { - "Start Timestamp(us)": 1713174287186619.8, - "Elapse Time(ms)": 4788, - "Transit Time(ms)": 0.0013, - "Wait Time(ms)": 39.037240000000004, - "Synchronization Time(ms)": 39.03034, - "Idle Time(ms)": 167.66008000000002, - "Wait Time Ratio": 1.0, - "Synchronization Time Ratio": 1.0 - }, - "Communication Bandwidth Info": { - "RDMA": { - "Transit Size(MB)": 80, - "Transit Time(ms)": 4700, - "Bandwidth(GB/s)": 0.0033, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "HCCS": { - "Transit Size(MB)": 4e-05, - "Transit Time(ms)": 0.0013, - "Bandwidth(GB/s)": 0.0308, - "Large Packet Ratio": 0.0, - "Size Distribution": { - "4e-05": [ - 1, - 0.0013 - ] - } - }, - "PCIE": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SDMA": { - "Transit Size(MB)": 4e-05, - "Transit Time(ms)": 0.0013, - "Bandwidth(GB/s)": 0.0308, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SIO": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - } - } - }, - } - }}} - return data - - @classmethod - def create_communicaton_json(cls): - raw_data = cls.get_cluster_communication_view() - with os.fdopen(os.open(f"{TestRdmaAdvice.OUTPUT_DIR}/cluster_communication.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_contain_cluster_communication_json(self): - self.create_communicaton_json() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "cluster" - scope = SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []))) - self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []).get('data'))) - result.clear() diff --git a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py b/profiler/test/ut/advisor/communication_advice/test_packet_advice.py deleted file mode 100644 index a8fd4549e..000000000 --- a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py +++ /dev/null @@ -1,175 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes - - -class TestPacketAdvice(unittest.TestCase): - TMP_DIR = "./ascend_pt" - OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" - interface = None - err_interface = None - - def tearDown(self): - if os.path.exists(TestPacketAdvice.TMP_DIR): - shutil.rmtree(TestPacketAdvice.TMP_DIR) - self.clear_htmls() - - def setUp(self): - if os.path.exists(TestPacketAdvice.TMP_DIR): - shutil.rmtree(TestPacketAdvice.TMP_DIR) - if not os.path.exists(TestPacketAdvice.TMP_DIR): - os.makedirs(TestPacketAdvice.TMP_DIR) - if not os.path.exists(TestPacketAdvice.OUTPUT_DIR): - os.makedirs(TestPacketAdvice.OUTPUT_DIR) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“att”开头 - if filename.startswith("mstt"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def get_communication_view(cls): - data = {"step1":{"collective" : { - "hcom_broadcast__844_1_1@13681369207305868844": { - "Communication Time Info": { - "Start Timestamp(us)": 1713174287407957.0, - "Elapse Time(ms)": 0.06086, - "Transit Time(ms)": 0.00126, - "Wait Time(ms)": 0.014939999999999998, - "Synchronization Time(ms)": 0.00714, - "Idle Time(ms)": 0.044660000000000005, - "Wait Time Ratio": 0.9222, - "Synchronization Time Ratio": 0.85 - }, - "Communication Bandwidth Info": { - "RDMA": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "HCCS": { - "Transit Size(MB)": 0.028575999999999997, - "Transit Time(ms)": 0.008620000000000001, - "Bandwidth(GB/s)": 3.3151, - "Large Packet Ratio": 0.0, - "Size Distribution": { - "0.004224": [ - 6, - 0.00736 - ], - "0.003232": [ - 1, - 0.00126 - ] - } - }, - "PCIE": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SDMA": { - "Transit Size(MB)": 0.028575999999999997, - "Transit Time(ms)": 0.008620000000000001, - "Bandwidth(GB/s)": 3.3151, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SIO": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - } - } - }, - "hcom_allReduce__844_2_1@13681369207305868844": { - "Communication Time Info": { - "Start Timestamp(us)": 1713174287432401.2, - "Elapse Time(ms)": 2.9042, - "Transit Time(ms)": 1.35236, - "Wait Time(ms)": 1.47632, - "Synchronization Time(ms)": 1.44524, - "Idle Time(ms)": 0.07551999999999981, - "Wait Time Ratio": 0.5219, - "Synchronization Time Ratio": 0.5166 - }, - "Communication Bandwidth Info": { - "RDMA": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "HCCS": { - "Transit Size(MB)": 176.16076799999996, - "Transit Time(ms)": 9.55658, - "Bandwidth(GB/s)": 18.4335, - "Large Packet Ratio": 0.0, - "Size Distribution": { - "12.582912": [ - 14, - 9.55658 - ] - } - }, - "PCIE": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SDMA": { - "Transit Size(MB)": 176.16076799999996, - "Transit Time(ms)": 9.55658, - "Bandwidth(GB/s)": 18.4335, - "Large Packet Ratio": 0, - "Size Distribution": {} - }, - "SIO": { - "Transit Size(MB)": 0, - "Transit Time(ms)": 0, - "Bandwidth(GB/s)": 0, - "Large Packet Ratio": 0, - "Size Distribution": {} - } - } - }, - }}} - return data - - @classmethod - def create_communicaton_json(cls): - raw_data = cls.get_communication_view() - with os.fdopen(os.open(f"{TestPacketAdvice.OUTPUT_DIR}/communication.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_ascend_pt_contain_communication_json(self): - self.create_communicaton_json() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "communication" - scope = SupportedScopes.PACKET - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("Packet Analysis", []))) - self.assertEqual(1, len(result.data.get("Packet Analysis", []).get('data'))) - result.clear() diff --git a/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py b/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py deleted file mode 100644 index 51acf3b8e..000000000 --- a/profiler/test/ut/advisor/compute_advice/test_frequency_advice.py +++ /dev/null @@ -1,145 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes - - -class TestFrequencyAdvice(unittest.TestCase): - TMP_DIR = "./ascend_pt" - OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" - DEVICE_DIR = "./ascend_pt/PROF_000001_20240415174447255_OAANHDOMMJMHGIFC/device_0" - interface = None - err_interface = None - - def tearDown(self): - if os.path.exists(TestFrequencyAdvice.TMP_DIR): - shutil.rmtree(TestFrequencyAdvice.TMP_DIR) - self.clear_htmls() - - def setUp(self): - if os.path.exists(TestFrequencyAdvice.TMP_DIR): - shutil.rmtree(TestFrequencyAdvice.TMP_DIR) - if not os.path.exists(TestFrequencyAdvice.TMP_DIR): - os.makedirs(TestFrequencyAdvice.TMP_DIR) - if not os.path.exists(TestFrequencyAdvice.OUTPUT_DIR): - os.makedirs(TestFrequencyAdvice.OUTPUT_DIR) - if not os.path.exists(TestFrequencyAdvice.DEVICE_DIR): - os.makedirs(TestFrequencyAdvice.DEVICE_DIR) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“att”开头 - if filename.startswith("att"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def get_basic_trace_view(cls): - # Python pid - py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} - # ascend pid - ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} - # ascend pid - cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} - # ascend hardware ops - ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - # flow event - flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} - flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} - return [py_pid_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, flow_event_s, flow_event_e] - - @classmethod - def create_info_json(cls): - info = { - "DeviceInfo": [ - { - "id": 7, - "env_type": 3, - "ctrl_cpu_id": "ARMv8_Cortex_A55", - "ctrl_cpu_core_num": 1, - "ctrl_cpu_endian_little": 1, - "ts_cpu_core_num": 0, - "ai_cpu_core_num": 6, - "ai_core_num": 25, - "ai_cpu_core_id": 2, - "ai_core_id": 0, - "aicpu_occupy_bitmap": 252, - "ctrl_cpu": "0", - "ai_cpu": "2,3,4,5,6", - "aiv_num": 50, - "hwts_frequency": "49.999001", - "aic_frequency": "1850", - "aiv_frequency": "1850" - } - ] - } - with os.fdopen(os.open(f"{TestFrequencyAdvice.DEVICE_DIR}/info.json.0", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(info)) - - @classmethod - def create_non_910B_trace_view(cls): - basic_info = cls.get_basic_trace_view() - - # python ops - py_event1 = {"ph": "X", "cat": "python_function", "name": "aten::slice", "ts": "200", "dur": 100, "tid": 2, - "pid": 1, - "args": {"Call stack": "/root/test/slice.py(116);\r\n/root/torch/module.py"}} - py_event2 = {"ph": "X", "cat": "python_function", "name": "slice", "ts": "199", "dur": 200, "tid": 2, "pid": 1, - "args": {"Call stack": "/root/test/slice.py(116);\r\n/root/torch/module.py"}} - raw_data = [ - *basic_info, py_event1, py_event2 - ] - with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/trace_view.json", - # with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/msprof_20240415174455.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - @classmethod - def create_910B_trace_view(cls): - basic_info = cls.get_basic_trace_view() - - # python ops - py_event1 = {"name": "AI Core Freq", "ts": "1699529623106000.061", "pid": 682820896, "tid": 0, - "args": {"MHz": 1850}, "ph": "C"} - py_event2 = {"name": "AI Core Freq", "ts": "1699529623106770.541", "pid": 682820896, "tid": 0, - "args": {"MHz": 800}, "ph": "C"} - raw_data = [ - *basic_info, py_event1, py_event2 - ] - - with os.fdopen(os.open(f"{TestFrequencyAdvice.OUTPUT_DIR}/trace_view.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_msprof_not_contain_frequency_data(self): - self.create_info_json() - self.create_non_910B_trace_view() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "computation" - scope = SupportedScopes.FREQ_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(0, len(result.data.get("AI Core Frequency", []))) - result.clear() - - def test_run_should_run_success_when_trace_view_contain_frequency_data(self): - self.create_info_json() - self.create_910B_trace_view() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "computation" - scope = SupportedScopes.FREQ_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("AI Core Frequency", dict).get("data", []))) - result.clear() diff --git a/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py b/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py deleted file mode 100644 index f18311ab1..000000000 --- a/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py +++ /dev/null @@ -1,116 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset - - -class TestGcAdvice(unittest.TestCase): - TMP_DIR = "./ascend_pt" - OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" - interface = None - - def tearDown(self): - if os.path.exists(TestGcAdvice.TMP_DIR): - shutil.rmtree(TestGcAdvice.TMP_DIR) - self.clear_htmls() - TimelineEventDataset.reset_all_instances() - - def setUp(self): - if os.path.exists(TestGcAdvice.TMP_DIR): - shutil.rmtree(TestGcAdvice.TMP_DIR) - if not os.path.exists(TestGcAdvice.TMP_DIR): - os.makedirs(TestGcAdvice.TMP_DIR) - if not os.path.exists(TestGcAdvice.OUTPUT_DIR): - os.makedirs(TestGcAdvice.OUTPUT_DIR) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“att”开头 - if filename.startswith("mstt"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def create_trace_view_with_gc_events(cls): - # Python pid - py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} - # Python GC pid - py_gc_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 2, "args": {"name": "Python GC"}} - # ascend pid - ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} - # ascend pid - cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} - # ascend hardware ops - ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - gc_event1 = {"ph": "X", "name": "GC", "ts": "1699529622103750", "dur": 1500, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - gc_event2 = {"ph": "X", "name": "GC", "ts": "1699529623104750", "dur": 50, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - gc_event3 = {"ph": "X", "name": "GC", "ts": "1699529623105750", "dur": 50000, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - # flow event - flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} - flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} - - raw_data = [ - py_pid_data, py_gc_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, gc_event1, gc_event2, - gc_event3, flow_event_s, flow_event_e - ] - with os.fdopen(os.open(f"{TestGcAdvice.OUTPUT_DIR}/trace_view.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - @classmethod - def create_trace_view_without_gc_events(cls): - # Python pid - py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} - # ascend pid - ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} - # ascend pid - cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} - # ascend hardware ops - ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - # flow event - flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} - flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} - - raw_data = [ - py_pid_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, flow_event_s, flow_event_e - ] - with os.fdopen(os.open(f"{TestGcAdvice.OUTPUT_DIR}/trace_view.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_trace_view_contain_gc_events(self): - self.create_trace_view_with_gc_events() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "schedule" - scope = SupportedScopes.GC_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("GcAnalysis", dict).get("data", []))) - result.clear() - - def test_run_should_run_success_when_trace_view_not_contain_gc_events(self): - self.create_trace_view_without_gc_events() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "schedule" - scope = SupportedScopes.GC_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(0, len(result.data.get("GcAnalysis", []))) - result.clear() -- Gitee From 259ae094720b71809f69bd753e6803767dcf9c8f Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Wed, 21 Aug 2024 22:18:01 +0800 Subject: [PATCH 429/791] cleancode + bug fix --- .../msprobe/core/common/const.py | 27 ++++++++ .../msprobe/core/common/utils.py | 6 +- .../api_accuracy_checker.py | 61 ++++++++++--------- .../api_accuracy_checker/api_runner.py | 17 +++--- .../base_compare_algorithm.py | 7 +-- .../mindspore/api_accuracy_checker/const.py | 4 +- 6 files changed, 76 insertions(+), 46 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 333757082..7da6f1d06 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -257,3 +257,30 @@ class OverflowConst: OVERFLOW_DEBUG_MODE_ENABLE = "OVERFLOW_DEBUG_MODE_ENABLE" OVERFLOW_ORIGINAL_MODE = 0 OVERFLOW_DEBUG_MODE = 1 + +class MsCompareConst: + # api_info field + MINT = "Mint" + MINT_FUNCTIONAL = "MintFunctional" + + TASK_FIELD = "task" + STATISTICS_TASK = "statistics" + TENSOR_TASK = "tensor" + DUMP_DATA_DIR_FIELD = "dump_data_dir" + DATA_FIELD = "data" + + #detail_csv + DETAIL_CSV_API_NAME = "API Name" + DETAIL_CSV_BENCH_DTYPE = "Bench Dtype" + DETAIL_CSV_TESTED_DTYPE = "Tested Dtype" + DETAIL_CSV_SHAPE = "Shape" + DETAIL_CSV_PASS_STATUS = "Status" + DETAIL_CSV_MESSAGE = "Message" + DETAIL_CSV_FILE_NAME = "accuracy_checking_details" + + #result_csv + RESULT_CSV_FORWARD_TEST_SUCCESS = "Forward Test Success" + RESULT_CSV_BACKWARD_TEST_SUCCESS = "Backward Test Success" + RESULT_CSV_FILE_NAME = "accuracy_checking_result" + + EPSILON = 1e-8 \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 72f9cf253..83f74f1fb 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -527,11 +527,11 @@ def write_csv(data, filepath): writer = csv.writer(f) writer.writerows(data) -def write_whole_csv(data_list, filepath): +def write_new_csv(data, filepath): with FileOpen(filepath, 'w', encoding='utf-8-sig') as f: writer = csv.writer(f) - for data in data_list: - writer.writerows(data) + writer.writerows(data) + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) def load_npy(filepath): diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 20ee1535e..d2db4f10c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -2,13 +2,12 @@ import json import os from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import write_whole_csv, add_time_as_suffix -from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.utils import write_new_csv, add_time_as_suffix +from msprobe.core.common.const import Const, CompareConst, MsCompareConst from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation from msprobe.mindspore.api_accuracy_checker.base_compare_algorithm import compare_algorithms -from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context @@ -40,25 +39,29 @@ class ApiAccuracyChecker: api_info_dict = json.load(f) # init global context - task = check_and_get_from_json_dict(api_info_dict, MsApiAccuracyCheckerConst.TASK_FIELD, + task = check_and_get_from_json_dict(api_info_dict, MsCompareConst.TASK_FIELD, "task field in api_info.json",accepted_type=str, - accepted_value=(MsApiAccuracyCheckerConst.STATISTICS_TASK, - MsApiAccuracyCheckerConst.TENSOR_TASK)) - is_constructed = task == MsApiAccuracyCheckerConst.STATISTICS_TASK + accepted_value=(MsCompareConst.STATISTICS_TASK, + MsCompareConst.TENSOR_TASK)) + is_constructed = task == MsCompareConst.STATISTICS_TASK if not is_constructed: - dump_data_dir = check_and_get_from_json_dict(api_info_dict, MsApiAccuracyCheckerConst.DUMP_DATA_DIR_FIELD, + dump_data_dir = check_and_get_from_json_dict(api_info_dict, MsCompareConst.DUMP_DATA_DIR_FIELD, "dump_data_dir field in api_info.json", accepted_type=str) else: dump_data_dir = "" global_context.init(is_constructed, dump_data_dir) - api_info_data = check_and_get_from_json_dict(api_info_dict, MsApiAccuracyCheckerConst.DATA_FIELD, + api_info_data = check_and_get_from_json_dict(api_info_dict, MsCompareConst.DATA_FIELD, "data field in api_info.json", accepted_type=dict) for api_name, api_info in api_info_data.items(): - forbackward_str = api_name.split(".")[-1] + is_mint = api_name.split(Const.SEP)[0] in \ + (MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL) + if not is_mint: + continue + forbackward_str = api_name.split(Const.SEP)[-1] if forbackward_str not in (Const.FORWARD, Const.BACKWARD): logger.warning(f"api: {api_name} is not recognized as forward api or backward api, skip this.") - api_name = Const.SEP.join(api_name.split(".")[:-1]) # www.xxx.yyy.zzz --> www.xxx.yyy + api_name = Const.SEP.join(api_name.split(Const.SEP)[:-1]) # www.xxx.yyy.zzz --> www.xxx.yyy if api_name not in self.api_infos: self.api_infos[api_name] = ApiInfo(api_name) @@ -86,7 +89,7 @@ class ApiAccuracyChecker: tested_outputs = api_runner(api_input_aggregation, api_name_str, forward_or_backward, Const.MS_FRAMEWORK) else: tested_outputs = api_info.get_compute_element_list(forward_or_backward, Const.OUTPUT) - bench_outputs = api_runner(api_input_aggregation, api_name_str, Const.FORWARD, Const.PT_FRAMEWORK) + bench_outputs = api_runner(api_input_aggregation, api_name_str, forward_or_backward, Const.PT_FRAMEWORK) # compare output for i, (bench_out, tested_out) in enumerate(zip(bench_outputs, tested_outputs)): @@ -113,12 +116,12 @@ class ApiAccuracyChecker: compare_result_dict) def run_and_compare(self): - for api_name_str, api_info in self.api_infos: + for api_name_str, api_info in self.api_infos.items(): if not api_info.check_forward_info(): logger.warning(f"api: {api_name_str} is lack of forward infomation, skip forward and backward check") continue forward_inputs = api_info.get_compute_element_list(Const.FORWARD, Const.INPUT) - kwargs = api_info.get_kwargs(api_info, api_name_str, ) + kwargs = api_info.get_kwargs() forward_inputs_aggregation = ApiInputAggregation(forward_inputs, kwargs, None) self.run_and_compare_helper(api_info, api_name_str, forward_inputs_aggregation, Const.FORWARD) @@ -140,15 +143,15 @@ class ApiAccuracyChecker: # detail_csv detail_csv = [] detail_csv_header_basic_info = [ - MsApiAccuracyCheckerConst.DETAIL_CSV_API_NAME, - MsApiAccuracyCheckerConst.DETAIL_CSV_BENCH_DTYPE, - MsApiAccuracyCheckerConst.DETAIL_CSV_TESTED_DTYPE, - MsApiAccuracyCheckerConst.DETAIL_CSV_SHAPE, + MsCompareConst.DETAIL_CSV_API_NAME, + MsCompareConst.DETAIL_CSV_BENCH_DTYPE, + MsCompareConst.DETAIL_CSV_TESTED_DTYPE, + MsCompareConst.DETAIL_CSV_SHAPE, ] detail_csv_header_compare_result = list(compare_algorithms.keys()) detail_csv_header_status = [ - MsApiAccuracyCheckerConst.DETAIL_CSV_PASS_STATUS, - MsApiAccuracyCheckerConst.DETAIL_CSV_MESSAGE, + MsCompareConst.DETAIL_CSV_PASS_STATUS, + MsCompareConst.DETAIL_CSV_MESSAGE, ] detail_csv_header = detail_csv_header_basic_info + detail_csv_header_compare_result + detail_csv_header_status @@ -166,8 +169,8 @@ class ApiAccuracyChecker: csv_row = csv_row_basic_info + csv_row_compare_result + csv_row_status detail_csv.append(csv_row) - file_name = os.path.join(csv_dir, add_time_as_suffix(MsApiAccuracyCheckerConst.DETAIL_CSV_FILE_NAME)) - write_whole_csv(detail_csv, file_name) + file_name = os.path.join(csv_dir, add_time_as_suffix(MsCompareConst.DETAIL_CSV_FILE_NAME)) + write_new_csv(detail_csv, file_name) def to_result_csv(self, csv_dir): @@ -198,14 +201,14 @@ class ApiAccuracyChecker: #result_csv result_csv = [] result_csv_header = [ - MsApiAccuracyCheckerConst.DETAIL_CSV_API_NAME, - MsApiAccuracyCheckerConst.RESULT_CSV_FORWARD_TEST_SUCCESS, - MsApiAccuracyCheckerConst.RESULT_CSV_BACKWARD_TEST_SUCCESS, - MsApiAccuracyCheckerConst.DETAIL_CSV_MESSAGE, + MsCompareConst.DETAIL_CSV_API_NAME, + MsCompareConst.RESULT_CSV_FORWARD_TEST_SUCCESS, + MsCompareConst.RESULT_CSV_BACKWARD_TEST_SUCCESS, + MsCompareConst.DETAIL_CSV_MESSAGE, ] result_csv.append(result_csv_header) - for api_name, result_csv_entry in result_csv_dict: + for api_name, result_csv_entry in result_csv_dict.items(): if result_csv_entry.forward_pass_status == CompareConst.PASS and \ result_csv_entry.backward_pass_status == CompareConst.PASS: overall_err_msg = "" @@ -215,5 +218,5 @@ class ApiAccuracyChecker: result_csv_entry.backward_pass_status, overall_err_msg] result_csv.append(row) - file_name = os.path.join(csv_dir, add_time_as_suffix(MsApiAccuracyCheckerConst.RESULT_CSV_FILE_NAME)) - write_whole_csv(result_csv, file_name) \ No newline at end of file + file_name = os.path.join(csv_dir, add_time_as_suffix(MsCompareConst.RESULT_CSV_FILE_NAME)) + write_new_csv(result_csv, file_name) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index ed92be83c..ff754ca20 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -5,8 +5,7 @@ import torch from mindspore import ops from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement -from msprobe.core.common.const import Const -from msprobe.mindspore.api_accuracy_checker.const import MsApiAccuracyCheckerConst +from msprobe.core.common.const import Const, MsCompareConst from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple @@ -25,10 +24,10 @@ class ApiInputAggregation: self.gradient_inputs = gradient_inputs api_parent_module_mapping = { - (MsApiAccuracyCheckerConst.MINT, Const.MS_FRAMEWORK): mindspore.mint, - (MsApiAccuracyCheckerConst.MINT, Const.PT_FRAMEWORK): torch, - (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional, - (MsApiAccuracyCheckerConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional + (MsCompareConst.MINT, Const.MS_FRAMEWORK): mindspore.mint, + (MsCompareConst.MINT, Const.PT_FRAMEWORK): torch, + (MsCompareConst.MINT_FUNCTIONAL, Const.MS_FRAMEWORK): mindspore.mint.nn.functional, + (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional } class ApiRunner: @@ -50,7 +49,7 @@ class ApiRunner: api_type_str, api_sub_name = self.get_info_from_name(api_name_str) api_instance = self.get_api_instance(api_type_str, api_sub_name, api_platform) - self.run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform) + return self.run_api(api_instance, api_input_aggregation, forward_or_backward, api_platform) @staticmethod def get_info_from_name(api_name_str): @@ -67,7 +66,7 @@ class ApiRunner: err_msg = f"ApiRunner.get_info_from_name failed: api_name_str: {api_name_str} is not in defined format" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) api_type_str, api_sub_name = api_name_list[0], api_name_list[1] - if api_type_str not in [MsApiAccuracyCheckerConst.MINT, MsApiAccuracyCheckerConst.MINT_FUNCTIONAL]: + if api_type_str not in [MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL]: err_msg = f"ApiRunner.get_info_from_name failed: not mint or mint.nn.functional api" logger.error_log_with_exp(err_msg, ApiAccuracyCheckerException(ApiAccuracyCheckerException.WrongValue)) @@ -92,7 +91,7 @@ class ApiRunner: api_parent_module = api_parent_module_mapping.get((api_type_str, api_platform)) module_str = "mindspore.mint." if api_platform == Const.MS_FRAMEWORK else "torch." - submodule_str = "nn.functional." if api_type_str == MsApiAccuracyCheckerConst.MINT_FUNCTIONAL else "" + submodule_str = "nn.functional." if api_type_str == MsCompareConst.MINT_FUNCTIONAL else "" full_api_name = module_str + submodule_str + api_sub_name if not hasattr(api_parent_module, api_sub_name): err_msg = f"ApiRunner.get_api_instance failed: {full_api_name} is not found" diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index ced2371f0..0670d2789 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -6,7 +6,7 @@ import numpy as np from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger -from msprobe.core.common.const import CompareConst +from msprobe.core.common.const import CompareConst, MsCompareConst class CompareResult: def __init__(self, compare_value, pass_status, err_msg): @@ -134,7 +134,7 @@ class CosineSimilarityCompareAlgorithm(BaseCompareAlgorithm): bench_norm = np.linalg.norm(bench_ndarray) tested_norm = np.linalg.norm(tested_ndarray) dot_product = np.dot(bench_ndarray.flatten(), tested_ndarray.flatten()) - cosine_similarity = dot_product / (bench_norm * tested_norm) + cosine_similarity = (MsCompareConst.EPSILON + dot_product) / (MsCompareConst.EPSILON + bench_norm * tested_norm) return cosine_similarity def check_pass(self, compare_value): @@ -170,7 +170,6 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): def __init__(self) -> None: super().__init__() self.compare_algorithm_name = CompareConst.MAX_RELATIVE_ERR - self.epsilon = 1e-8 def check_validity(self, bench_compute_element, tested_compute_element): return self.check_two_tensor(bench_compute_element, tested_compute_element) @@ -180,7 +179,7 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) abs_diff = np.abs(bench_ndarray - tested_ndarray) - bench_ndarray_nonzero = bench_ndarray + (bench_ndarray == 0) * self.epsilon # prevent division by 0 + bench_ndarray_nonzero = bench_ndarray + (bench_ndarray == 0) * MsCompareConst.EPSILON # prevent division by 0 max_relative_diff = np.max(abs_diff / bench_ndarray_nonzero) return max_relative_diff diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py index 9886576f5..cc6943a05 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py @@ -1,4 +1,4 @@ -class MsApiAccuracyCheckerConst: +class MsCompareConst: # api_info field MINT = "Mint" MINT_FUNCTIONAL = "MintFunctional" @@ -23,3 +23,5 @@ class MsApiAccuracyCheckerConst: RESULT_CSV_BACKWARD_TEST_SUCCESS = "Backward Test Success" RESULT_CSV_FILE_NAME = "accuracy_checking_result" + EPSILON = 1e-8 + -- Gitee From 30f87f4caa74b006e0cf281c0a9cd41df394e038 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Wed, 21 Aug 2024 22:29:31 +0800 Subject: [PATCH 430/791] bug fix --- .../mindspore/api_accuracy_checker/base_compare_algorithm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index 0670d2789..6b48f60a4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -179,7 +179,7 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) abs_diff = np.abs(bench_ndarray - tested_ndarray) - bench_ndarray_nonzero = bench_ndarray + (bench_ndarray == 0) * MsCompareConst.EPSILON # prevent division by 0 + bench_ndarray_nonzero = np.abs(bench_ndarray) + (bench_ndarray == 0) * MsCompareConst.EPSILON # prevent division by 0 max_relative_diff = np.max(abs_diff / bench_ndarray_nonzero) return max_relative_diff -- Gitee From bf1c58fcadcd35554bbb386ec57106408992aa1c Mon Sep 17 00:00:00 2001 From: Henry Date: Wed, 7 Aug 2024 11:48:07 +0800 Subject: [PATCH 431/791] pynative + jit --- .../mindspore/debugger/precision_debugger.py | 2 +- .../msprobe/mindspore/doc/dump.md | 6 +- .../mindspore/dump/dump_tool_factory.py | 2 +- .../mindspore/dump/hook_cell/api_registry.py | 2 +- .../msprobe/mindspore/dump/jit_dump.py | 54 ++++++++ .../msprobe/mindspore/service.py | 21 ++- .../mindspore_ut/debugger/test_jit_dump.py | 129 ++++++++++++++++++ 7 files changed, 211 insertions(+), 5 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_jit_dump.py diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index d2a5e8d2d..0b3fec632 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -61,7 +61,7 @@ class PrecisionDebugger: return instance.config.execution_mode = instance._get_execution_mode() - if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.task != Const.FREE_BENCHMARK: + if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.task != Const.FREE_BENCHMARK and instance.config.level != "kernel": if not instance.service: instance.service = Service(instance.config) instance.service.start(target) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md index 0d45e1b14..908b40d5e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md @@ -152,8 +152,9 @@ dump结果目录结构示例如下： | | | | ├── MintFunctional.relu.0.backward.input.0.npy | | | | ├── Mint.abs.0.forward.input.0.npy | | | | ├── Functional.split.0.forward.input.0.npy +| | | | ├── Tensor.__add__.0.forward.output.0.npy | | | | ... -| | | | └── Tensor.__add__.0.forward.output.0.npy +| | | | └── Jit.AlexNet.0.forward.input.0.npy │ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） │ | | ├── stack.json # 算子调用栈信息 │ | | └── construct.json # 分层分级结构，level为L1时，construct.json内容为空 @@ -175,6 +176,8 @@ dump过程中，npy文件在对应算子或者模块被执行后就会落盘， 其中rank为设备上各卡的ID，每张卡上dump的数据会生成对应dump目录。非分布式场景下没有rank ID，目录名称为rank。 +动态图场景下使能PSJit或PIJit，装饰特定Cell或function，被装饰的部分会全部/部分使能静态图流程。PSJit场景下config.json文件配置level为L1时，被PSJit装饰的部分也作为API被dump到对应目录；若配置level为L2时，则只会dump用户网络中静态图流程下的相关kernel。PIJit场景开启dump工具后，会被还原为动态图，按API粒度进行dump。 + npy文件保存的前缀和MindSpore对应关系如下： | 前缀 | MindSpore模块 | @@ -183,6 +186,7 @@ npy文件保存的前缀和MindSpore对应关系如下： | Functional | mindspore.ops | | Mint | mindspore.mint | | MintFunctional | mindspore.mint.nn.functional | +| Jit | mindspore.jit | ## 工具支持的API列表 diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py b/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py index 1e4b06a38..138dcb60d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/dump_tool_factory.py @@ -19,7 +19,7 @@ class DumpToolFactory: Const.KERNEL: { Const.GRAPH_KBYK_MODE: KernelKbykDump, Const.GRAPH_GE_MODE: KernelGraphDump, - Const.PYNATIVE_MODE: None + Const.PYNATIVE_MODE: KernelKbykDump } } diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py index 5508416fd..584bf91c1 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py @@ -98,7 +98,7 @@ class ApiRegistry: self.mint_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKMintOP, attr_name) for attr_name in dir(HOOKMintNNFunctionalOP): if attr_name.startswith(Const.ATTR_NAME_PREFIX): - self.mint_func_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKMintNNFunctionalOP, attr_name) + self.mint_func_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKMintNNFunctionalOP, attr_name) api_register = ApiRegistry() diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py new file mode 100644 index 000000000..c2dc7f5a0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py @@ -0,0 +1,54 @@ +import os +from mindspore.common.api import _MindsporeFunctionExecutor +from mindspore._c_expression import PyNativeExecutor_ +from msprobe.mindspore.dump.hook_cell.api_registry import api_register +from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs + + +def dump_jit(name, in_feat, out_feat, is_forward): + pid = os.getpid() + ori_args = str(type(name)) + index = ori_args.find("__main__.") + if index!= -1: + result = ori_args[(index + len("__main__.")):-2] + if is_forward: + name_template = "Jit." + result + ".forward" + else: + name_template = "Jit." + result + ".backward" + JitDump.data_collector.visit_and_clear_overflow_status(name_template) + if JitDump.data_collector: + module_input_output = ModuleForwardInputsOutputs(args=in_feat, kwargs={}, output=out_feat) + JitDump.data_collector.forward_data_collect(name_template, {}, pid, module_input_output) + + +class JitDump(_MindsporeFunctionExecutor): + dump_config = None + jit_enable = False + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._executor = PyNativeExecutor_.get_instance() + + def __call__(self, *args, **kwargs): + api_register.api_set_ori_func() + out = super().__call__(*args, **kwargs) + dump_jit(args[0], args[1], out, True) + JitDump.jit_enable = True + api_register.api_set_hook_func() + return out + + @classmethod + def set_config(cls, value): + cls.dump_config = value + + @classmethod + def set_data_collector(cls, value): + cls.data_collector = value + + def grad(self, obj, grad, weights, grad_position, *args, **kwargs): + if JitDump.jit_enable: + api_register.api_set_ori_func() + output = self._executor.grad(grad, obj, weights, grad_position, *args, *(kwargs.values())) + if JitDump.jit_enable: + dump_jit(obj, args, output, False) + api_register.api_set_hook_func() + return output \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index bd87effd9..016c68782 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -19,9 +19,16 @@ from pathlib import Path import functools from collections import defaultdict +import mindspore as ms from mindspore.common.tensor import Tensor from mindspore import ops from mindspore import nn +try: + from mindspore.common._pijit_context import PIJitCaptureContext + pijit_label = True +except ImportError: + pijit_label = False + from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope @@ -36,6 +43,7 @@ from msprobe.core.data_dump.data_processor.base import ModuleBackwardInputsOutpu from msprobe.core.common.exceptions import MsprobeException from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell from msprobe.mindspore.cell_processor import CellProcessor +from msprobe.mindspore.dump.jit_dump import JitDump class Service: @@ -267,6 +275,14 @@ class Service: logger.info(f"Dump switch is turned on at step {self.current_iter}. ") self.create_dirs() logger.info(f"Dump data will be saved in {self.dump_iter_dir}.") + if self.config.level == "L1": + JitDump.set_config(self.config) + JitDump.set_data_collector(self.data_collector) + ms.common.api._MindsporeFunctionExecutor = JitDump + ms.common.api._PyNativeExecutor.grad = JitDump.grad + if pijit_label: + PIJitCaptureContext.__enter__ = self.empty + PIJitCaptureContext.__exit__ = self.empty def stop(self): logger.info("msprobe: debugger.stop() is set successfully. " @@ -304,6 +320,9 @@ class Service: construct_file_path = os.path.join(dump_dir, "construct.json") self.data_collector.update_dump_paths( dump_file_path, stack_file_path, construct_file_path, dump_data_dir, None) + + def empty(self, *args, **kwargs): + pass def register_hook_new(self): logger.info("The {} hook function is successfully mounted to the model.".format(self.config.task)) @@ -332,4 +351,4 @@ class Service: cell.register_backward_pre_hook( self.cell_processor.node_hook(prefix + Const.BACKWARD, Const.START)) cell.register_backward_hook( - self.cell_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) + self.cell_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_jit_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_jit_dump.py new file mode 100644 index 000000000..3666e05cd --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_jit_dump.py @@ -0,0 +1,129 @@ +import numpy as np +import os +from unittest.mock import patch, MagicMock +import mindspore as ms +import mindspore.common.dtype as mstype +import mindspore.nn as nn +import mindspore.ops as ops +from mindspore.common.tensor import Tensor +from mindspore import jit +from msprobe.mindspore import PrecisionDebugger +from msprobe.core.common_config import CommonConfig, BaseConfig + +def conv(in_channels, out_channels, kernel_size, stride=1, padding=0, pad_mode="valid", has_bias=True): + return nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size, stride=stride, padding=padding, +has_bias=has_bias, pad_mode=pad_mode) + +def fc_with_initialize(input_channels, out_channels, has_bias=True): + return nn.Dense(input_channels, out_channels, has_bias=has_bias) + +class DataNormTranspose(nn.Cell): + """Normalize an tensor image with mean and standard deviation. + + Given mean: (R, G, B) and std: (R, G, B), + will normalize each channel of the torch.*Tensor, i.e. + channel = (channel - mean) / std + + Args: + mean (sequence): Sequence of means for R, G, B channels respectively. + std (sequence): Sequence of standard deviations for R, G, B channels + respectively. + """ + + def __init__(self, dataset_name='imagenet'): + super(DataNormTranspose, self).__init__() + # Computed from random subset of ImageNet training images + if dataset_name == 'imagenet': + self.mean = Tensor(np.array([0.485 * 255, 0.456 * 255, 0.406 * 255]).reshape((1, 1, 1, 3)), mstype.float32) + self.std = Tensor(np.array([0.229 * 255, 0.224 * 255, 0.225 * 255]).reshape((1, 1, 1, 3)), mstype.float32) + else: + self.mean = Tensor(np.array([0.4914, 0.4822, 0.4465]).reshape((1, 1, 1, 3)), mstype.float32) + self.std = Tensor(np.array([0.2023, 0.1994, 0.2010]).reshape((1, 1, 1, 3)), mstype.float32) + + def construct(self, x): + x = (x - self.mean) / self.std + x = ops.transpose(x, (0, 3, 1, 2)) + return x + +class AlexNet(nn.Cell): + """ + Alexnet + """ + + def __init__(self, num_classes=10, channel=3, phase='train', include_top=True, dataset_name='imagenet'): + super(AlexNet, self).__init__() + self.data_trans = DataNormTranspose(dataset_name=dataset_name) + self.conv1 = conv(channel, 64, 11, stride=4, pad_mode="same", has_bias=True) + self.conv2 = conv(64, 128, 5, pad_mode="same", has_bias=True) + self.conv3 = conv(128, 192, 3, pad_mode="same", has_bias=True) + self.conv4 = conv(192, 256, 3, pad_mode="same", has_bias=True) + self.conv5 = conv(256, 256, 3, pad_mode="same", has_bias=True) + self.relu = nn.ReLU() + nn.BatchNorm2d + self.max_pool2d = nn.MaxPool2d(kernel_size=3, stride=2, pad_mode='valid') + self.include_top = include_top + if self.include_top: + dropout_ratio = 0.65 + if phase == 'test': + dropout_ratio = 1.0 + self.flatten = nn.Flatten() + self.fc1 = fc_with_initialize(6 * 6 * 256, 4096) + self.fc2 = fc_with_initialize(4096, 4096) + self.fc3 = fc_with_initialize(4096, num_classes) + self.dropout = nn.Dropout(p=1 - dropout_ratio) + + @jit + def construct(self, x): + """define network""" + x = self.data_trans(x) + x = self.conv1(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv2(x) + x = self.relu(x) + x = self.max_pool2d(x) + x = self.conv3(x) + x = self.relu(x) + x = self.conv4(x) + x = self.relu(x) + x = self.conv5(x) + x = self.relu(x) + x = self.max_pool2d(x) + if not self.include_top: + return x + x = self.flatten(x) + x = self.fc1(x) + x = self.relu(x) + x = self.dropout(x) + x = self.fc2(x) + x = self.relu(x) + x = self.dropout(x) + x = self.fc3(x) + x = ops.celu(x, 2.0) + return x + +if __name__ == "__main__": + json_config = { + "task": "statistics", + "dump_path": "/absolute_path", + "rank": [], + "step": [], + "level": "L1" + } + + common_config = CommonConfig(json_config) + task_config = BaseConfig(json_config) + mock_parse_json_config = MagicMock() + mock_parse_json_config.return_value = [common_config, task_config] + debugger = PrecisionDebugger() + ms.set_context(mode=ms.PYNATIVE_MODE) + net = AlexNet() + debugger.start() + ops.relu(ms.Tensor(np.random.random([1, 227, 227, 3]).astype(np.float32))) + grad_net = ms.grad(net, None, net.trainable_params()) + output = grad_net(ms.Tensor(np.random.random([1, 227, 227, 3]).astype(np.float32))) + debugger.stop() + expected_file_count = 5 + dir_path = "/absolute_path/step0/rank/dump_tensor_data/" + actual_file_count = len(os.listdir(dir_path)) + assert actual_file_count == expected_file_count -- Gitee From cee6d83212f5dd708762e0f4c273a5c777a3677f Mon Sep 17 00:00:00 2001 From: wuyuhan Date: Thu, 22 Aug 2024 09:26:48 +0800 Subject: [PATCH 432/791] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= =?UTF-8?q?=E5=92=8C=E6=A8=A1=E7=89=88=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../advisor/analyzer/analyzer_controller.py | 557 ++++++++++++++++++ profiler/advisor/analyzer/base_analyzer.py | 55 +- .../analyzer/cluster/slow_link_analyzer.py | 191 ++++++ .../analyzer/cluster/slow_rank_analyzer.py | 216 +++++++ .../base_communication_analyzer.py | 8 + .../analyzer/communication/packet/__init__.py | 0 .../{ => packet}/packet_analyzer.py | 18 +- .../{ => packet}/packet_checker.py | 9 +- .../communication/retransmission/__init__.py | 0 .../communication_retransmission_analyzer.py | 52 ++ .../communication_retransmission_checker.py | 129 ++++ .../ai_core_freq/ai_core_freq_analyzer.py | 22 +- .../ai_core_freq/ai_core_freq_checker.py | 34 +- .../computation/aicpu/aicpu_checker.py | 28 +- .../computation/bound/block_dim_checker.py | 14 +- .../bound/operator_bound_checker.py | 14 +- .../op_compile/dynamic_shape_checker.py | 17 +- .../analyzer/computation/operator_checker.py | 42 +- .../pp_stage_computation_analyzer.py | 106 ++++ .../computation/profiling_analyzer.py | 27 +- .../dataloader/dataloader_analyzer.py | 14 +- .../analyzer/dataloader/dataloader_checker.py | 12 +- .../graph_fusion/graph_fusion_analyzer.py | 21 +- profiler/advisor/analyzer/memory/__init__.py | 0 .../analyzer/memory/memory_analyzer.py | 38 ++ .../advisor/analyzer/memory/memory_checker.py | 76 +++ .../overall/environment_variable_analyzer.py | 4 + .../overall/overall_summary_analyzer.py | 5 +- .../dispatch/timeline_op_dispatch_analyzer.py | 60 +- .../fusion_ops/fusion_ops_analyzer.py | 22 +- .../fusion_ops/timeline_api_stack_checker.py | 22 +- .../analyzer/schedule/gc/gc_analyzer.py | 15 +- .../analyzer/schedule/gc/gc_checker.py | 10 +- .../schedule/syncbn/syncbn_analyzer.py | 16 +- .../schedule/syncbn/syncbn_checker.py | 11 +- .../synchronize_stream_analyzer.py | 15 +- .../synchronize_stream_checker.py | 11 +- .../schedule/timeline_base_checker.py | 14 +- profiler/advisor/common/analyzer_scopes.py | 2 + .../advisor/common/async_analysis_status.py | 7 + profiler/advisor/common/constant.py | 32 +- .../dataset/cluster/cluster_dataset.py | 50 +- .../cluster/cluster_step_trace_time_bean.py | 3 + .../advisor/dataset/timeline_event_dataset.py | 367 ++++-------- .../dataset/timeline_op_collector/__init__.py | 0 .../timeline_op_collector.py | 376 ++++++++++++ .../display/html/priority_background_color.py | 4 + profiler/advisor/display/html/render.py | 51 +- .../display/html/templates/affinity_api.html | 8 +- .../html/templates/ai_core_frequency.html | 2 +- .../advisor/display/html/templates/main.html | 34 +- .../display/html/templates/memory.html | 18 + .../html/templates/operator_ai_cpu.html | 2 +- .../html/templates/operator_block_dim.html | 2 +- .../html/templates/operator_dispatch.html | 2 +- .../templates/operator_dynamic_shape.html | 2 +- .../html/templates/operator_no_bound.html | 2 +- .../pp_stage_computation_analysis.html | 19 + profiler/advisor/interface/interface.py | 56 +- profiler/advisor/result/result.py | 6 + profiler/advisor/rules/dataloader.yaml | 3 +- profiler/advisor/rules/memory.yaml | 7 + .../advisor/rules/timeline_fusion_ops.yaml | 18 +- profiler/advisor/utils/utils.py | 33 +- profiler/cli/analyze_cli.py | 46 +- .../test_analyzer_controller.py | 189 ++++++ .../test_packet_advice.py | 2 +- .../test_rdma_retransmission_advice.py | 2 +- .../test_pp_stage_computation_analyzer.py | 55 ++ .../advisor/schedule_advice/test_gc_advice.py | 116 ---- .../test_dataloader_checker.py | 10 +- .../timeline_advice/test_memory_op_checker.py | 62 ++ .../timeline_advice/test_syncbn_checker.py | 2 +- .../test_synchronize_stream.py | 2 +- .../test_timeline_op_collector.py | 144 +++++ .../test_timeline_op_compile_checker.py | 9 +- .../advisor/timeline_advice/trace_view.json | 1 + 77 files changed, 2935 insertions(+), 716 deletions(-) create mode 100644 profiler/advisor/analyzer/analyzer_controller.py create mode 100644 profiler/advisor/analyzer/cluster/slow_link_analyzer.py create mode 100644 profiler/advisor/analyzer/cluster/slow_rank_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/base_communication_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/packet/__init__.py rename profiler/advisor/analyzer/communication/{ => packet}/packet_analyzer.py (74%) rename profiler/advisor/analyzer/communication/{ => packet}/packet_checker.py (96%) create mode 100644 profiler/advisor/analyzer/communication/retransmission/__init__.py create mode 100644 profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py create mode 100644 profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py create mode 100644 profiler/advisor/analyzer/memory/__init__.py create mode 100644 profiler/advisor/analyzer/memory/memory_analyzer.py create mode 100644 profiler/advisor/analyzer/memory/memory_checker.py create mode 100644 profiler/advisor/common/async_analysis_status.py create mode 100644 profiler/advisor/dataset/timeline_op_collector/__init__.py create mode 100644 profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py create mode 100644 profiler/advisor/display/html/priority_background_color.py create mode 100644 profiler/advisor/display/html/templates/memory.html create mode 100644 profiler/advisor/display/html/templates/pp_stage_computation_analysis.html create mode 100644 profiler/advisor/rules/memory.yaml create mode 100644 profiler/test/ut/advisor/analyzer_controller/test_analyzer_controller.py rename profiler/test/ut/advisor/{cluster_advice => communication_advice}/test_rdma_retransmission_advice.py (99%) create mode 100644 profiler/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py delete mode 100644 profiler/test/ut/advisor/schedule_advice/test_gc_advice.py rename profiler/test/ut/advisor/{advisor_backend => }/timeline_advice/test_dataloader_checker.py (92%) create mode 100644 profiler/test/ut/advisor/timeline_advice/test_memory_op_checker.py rename profiler/test/ut/advisor/{advisor_backend => }/timeline_advice/test_syncbn_checker.py (95%) rename profiler/test/ut/advisor/{advisor_backend => }/timeline_advice/test_synchronize_stream.py (95%) create mode 100644 profiler/test/ut/advisor/timeline_advice/test_timeline_op_collector.py rename profiler/test/ut/advisor/{advisor_backend => }/timeline_advice/test_timeline_op_compile_checker.py (85%) create mode 100644 profiler/test/ut/advisor/timeline_advice/trace_view.json diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py new file mode 100644 index 000000000..e8704542e --- /dev/null +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -0,0 +1,557 @@ +import copy +import logging +import json +import sys +import os +import multiprocessing as mp +from pathlib import Path +from multiprocessing import Manager + +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "compare_tools")) +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "cluster_analyse")) + +from profiler.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer +from profiler.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer +from profiler.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer +from profiler.advisor.config.config import Config +from profiler.advisor.common.analyzer_scopes import SupportedScopes +from profiler.advisor.common.async_analysis_status import AsyncAnalysisStatus +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterDataset +from profiler.advisor.utils.utils import Timer, safe_index, safe_division +from profiler.advisor.interface.interface import Interface +from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor +from profiler.prof_common.path_manager import PathManager + +logger = logging.getLogger() + + +class AnalyzerController: + CLUSTER_RANK_THRESHOLD = 2 + + def __init__(self): + self.dimensions = Interface.all_dimension + self.kwargs = {} + self.slow_rank_analyzer = None + self.slow_link_analyzer = None + self.cluster_local_data_map = {} + self.default_rank_id = None + self.rank_id_map = {} + self._is_cluster = False + self.analysis_process_resp = Manager().dict() + + @staticmethod + def _check_profiling_path_valid(profiling_path): + PathManager.input_path_common_check(profiling_path) + + if not Path(profiling_path).exists(): + logger.error("Profiling path is not existed. Invalid profiling path: %s", profiling_path) + return False + return True + + @staticmethod + def _get_step_rank_for_cluster_statistic_diff(target_cluster_statistic_data, benchmark_cluster_statistic_data, + headers, dimension, get_max=False): + if dimension not in headers: + logger.error("Error dimension %s for cluster statistics data, optionals are %s.", dimension, headers) + return None, None, None + + dimension_index = safe_index(headers, dimension) + diff_record = [] + # 对比目标profiling和benchmark profiling 每张卡的计算和下发和带宽，取计算、下发、带宽差异最大的卡进行下一步分析 + for target_row_data, benchmark_row_data in zip(target_cluster_statistic_data, benchmark_cluster_statistic_data): + target_data = safe_index(target_row_data, dimension_index) + benchmark_data = safe_index(benchmark_row_data, dimension_index) + if not isinstance(target_data, (int, float)) or not isinstance(benchmark_data, (int, float)): + continue + diff_record.append(target_data - benchmark_data) + + if SlowRankAnalyzer.compute_max_gap_ratio(diff_record, safe_division(sum(diff_record), len( + diff_record))) < SlowRankAnalyzer.RATIO_THRESHOLD: + return None, None, None + + value = max(diff_record) if get_max else min(diff_record) + value_index = safe_index(diff_record, value) + + step_value_index = safe_index(headers, "step") + rank_id_value_index = safe_index(headers, "rank_id") + step = safe_index(safe_index(target_cluster_statistic_data, value_index, []), step_value_index) + benchmark_step = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), step_value_index) + target_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) + benchmark_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) + + if target_rank_id != benchmark_rank_id: + logger.error( + "Rank ids of target profiling must keep the same as benchmark profiling, skip cluster comparison") + return None, None, None + + return step, benchmark_step, target_rank_id + + def do_analysis(self, dimensions, **kwargs): + pid = os.getpid() + resp = {"id": pid} + try: + self._do_analysis(dimensions, pid=pid, resp=resp, **kwargs) + except Exception as e: + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED, error_msg=str(e)) + logger.error(e) + raise RuntimeError(e) + + def async_do_analysis(self, dimensions, **kwargs): + # 异步分析，用于部署服务，通过接口查询异步作业状态 + async_analysis_process = mp.Process(target=self.do_analysis, args=(dimensions,), kwargs=kwargs, + name="Async advisor performance analysis") + async_analysis_process.start() + return async_analysis_process + + def get_response_by_pid(self, pid): + return self.analysis_process_resp.get(pid) + + def single_rank_analysis(self, profiling_path, benchmark_profiling_path=None): + job_list = [] + + profiling_path = self._get_profiling_path_by_rank(profiling_path) + benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) + + # 单卡场景无集群分析 + for dim in [Interface.CLUSTER]: + if dim in self.dimensions: + self.dimensions.remove(dim) + + for dimension in self.dimensions: + dimension_analysis_func_name = f"{dimension}_analysis" + if not hasattr(self, dimension_analysis_func_name): + continue + logger.info("Start %s analysis", dimension) + job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + + if benchmark_profiling_path: + # kernel/api 比对 + job_list += self._single_profiling_comparison(profiling_path, benchmark_profiling_path) + else: + # 单卡性能拆解 + self.overall(profiling_path) + return job_list + + def cluster_analysis(self, profiling_path, benchmark_profiling_path=None): + job_list = [] + + # 单集群profiling分析：下发、通信、计算、显存/内存 + for dimension in self.dimensions: + dimension_analysis_func_name = f"cluster_{dimension}_analysis" + if not hasattr(self, dimension_analysis_func_name): + continue + logger.info("Start cluster %s analysis", dimension) + job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + + if benchmark_profiling_path: + # 两个集群profiling比对分析 + job_list += self._cluster_profiling_comparison(profiling_path, benchmark_profiling_path) + else: + self.overall(profiling_path) + return job_list + + def overall(self, profiling_path): + from profiler.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariabelAnalyzer + env_analyzer = EnvironmentVariabelAnalyzer(profiling_path) + env_analyzer.optimize() + + if self._is_cluster: + self.slow_rank_analyzer.optimize(template_key=Interface.OVERALL) + self.slow_link_analyzer.optimize(template_key=Interface.OVERALL) + else: + from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer + overall_analyzer = OverallSummaryAnalyzer(profiling_path) + overall_analyzer.optimize() + + def schedule_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None): + # 任意单卡的下发分析 + + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.SCHEDULE]: + for scope in Interface.get_scope(dimension): + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def computation_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None, stage=None): + # 任意单卡的计算分析 + + kwargs = copy.deepcopy(self.kwargs) + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + kwargs["stage"] = stage + job_list = [] + + for dimension in [Interface.COMPUTATION]: + for scope in Interface.get_scope(dimension): + if scope == SupportedScopes.STAGE_COMPUTE: + continue + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def memory_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None): + # 任意单卡的内存分析 + + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.MEMORY]: + for scope in Interface.get_scope(dimension): + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + return job_list + + def communication_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None, bandwidth_type=None): + + job_list = [] + supported_trans_type = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + if bandwidth_type is not None and bandwidth_type not in supported_trans_type: + logger.error("Error transit type %s, optionals are %s", bandwidth_type, supported_trans_type) + return job_list + + bandwidth_type_list = [bandwidth_type] if bandwidth_type is not None else supported_trans_type + + for bandwidth_type in bandwidth_type_list: + job_list += getattr(self, f"_communication_{bandwidth_type.lower()}_analysis")(profiling_path, + benchmark_profiling_path, + step, benchmark_step) + + return job_list + + def cluster_schedule_analysis(self, profiling_path): + # 目标集群profiling数据下发分析，不包含两个集群profiling数据的比对分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) + + info_msg = f"Maximum free for rank {slow_rank_id}" + if slow_step: + info_msg += f" and step {slow_step}" + logger.info(info_msg) + + job_list += self.schedule_analysis(analysis_profiling_path, step=slow_step) + return job_list + + def cluster_communication_analysis(self, profiling_path): + job_list = [] + + for dimension in [Interface.COMMUNICATION]: + for scope in Interface.get_scope(dimension): + analyzer_class = Interface.get_analyzer(dimension, scope) + if hasattr(analyzer_class, "requires_cluster_dataset") and getattr(analyzer_class, + "requires_cluster_dataset"): + + # 如果不依赖数据集，或者依赖的是ClusterDataset，则不用根据带宽确定需要分析的特定rank + kwargs = copy.deepcopy(self.kwargs) + kwargs["profiling_path"] = profiling_path + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + else: + # 非ClusterDataset场景，需要根据带宽大小分析特定的rank + for bandwidth_type in [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA]: + global_step_rank = self.slow_link_analyzer.get_global_step_rank(bandwidth_type) + # 获取带宽最小的卡进行分析 + target_rank_id = global_step_rank.get("minimum", {}).get("rank_id") or self.default_rank_id + step = global_step_rank.get("minimum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, target_rank_id) + + info_msg = f"Minimum {bandwidth_type} bandwidth for rank {target_rank_id} " + if step: + info_msg += f"and step {step}" + logger.info(info_msg) + + job_list += self.communication_analysis(analysis_profiling_path, step=step, + bandwidth_type=bandwidth_type) + + return job_list + + def cluster_computation_analysis(self, profiling_path): + # 目标集群profiling数据计算分析，不包含两个集群profiling数据的比对分析；如果有pp stage，则对不同stage进行计算分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.COMPUTE) + stage_step_rank = self.slow_rank_analyzer.get_stage_step_rank(SlowRankAnalyzer.COMPUTE) + + if stage_step_rank: + # 对不同pp stage取min max进行分析 + logger.info("Analysis steps and ranks of different pipeline parallel stages are %s", + json.dumps(stage_step_rank)) + + stages_profiling_path = [] + for stage, step_rank_info in stage_step_rank.items(): + rank_id = step_rank_info.get("maximum", {}).get("rank_id") + step = step_rank_info.get("maximum", {}).get("step") + + info_msg = f"For {stage}, slow rank is {rank_id}" + if step: + info_msg += f", step is {step}" + logger.info(info_msg) + + stages_profiling_path.append( + dict( + stage=stage, + rank_id=rank_id, + step=step, + profiling_path=self._get_profiling_path_by_rank(profiling_path, rank_id) + ) + ) + Interface.add_analyzer(Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, PPStageComputationAnalyzer) + kwargs = {"stages_profiling_path": stages_profiling_path, "profiling_path": profiling_path} + + job_list.append((Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, Interface(**kwargs), kwargs)) + else: + # 不区分stage，对所有卡取Min max进行分析 + logger.info("Without pipeline parallel stage, Global analysis steps and ranks is %s", + json.dumps(global_step_rank)) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + # 如果没有标杆profiling数据的rank id，说明没有快慢卡问题，直接对默认rank id进行分析，因此这里取值为None + fast_rank_id = global_step_rank.get("minimum", {}).get("rank_id") + fast_step = global_step_rank.get("minimum", {}).get("step") + + info_msg = f"Maximum computation time for rank {slow_rank_id}" + if slow_step: + info_msg += f" and step {slow_step}, " + if fast_rank_id: + info_msg += f"minimum computation time for rank {fast_rank_id}" + if fast_step: + info_msg += f" and step {fast_step}" + logger.info(info_msg) + + job_list += self.computation_analysis( + self._get_profiling_path_by_rank(profiling_path, slow_rank_id), + self._get_profiling_path_by_rank(profiling_path, fast_rank_id), + slow_step, + fast_step + ) + + return job_list + + def cluster_memory_analysis(self, profiling_path): + # 目标集群profiling数据内存分析，当前memory识别的两个算子，导致的问题都是大的free，因此选择FREE最慢的卡进行分析 + + job_list = [] + global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) + slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + slow_step = global_step_rank.get("maximum", {}).get("step") + analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) + + info_msg = f"Maximum free for rank {slow_rank_id} " + if slow_step: + info_msg += f"and step {slow_step}" + logger.info(info_msg) + + job_list += self.memory_analysis(analysis_profiling_path, step=slow_step) + return job_list + + def _do_analysis(self, dimensions, **kwargs): + self.dimensions = dimensions + self.kwargs = kwargs + result_list = [] + profiling_path = self.kwargs.get("profiling_path") + benchmark_profiling_path = self.kwargs.get("benchmark_profiling_path") + pid = self.kwargs.get("pid") + resp = self.kwargs.get("resp") + + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.ANALYZING) + + if not self._check_profiling_path_valid(profiling_path): + error_msg = f"Got invalid argument '-d/--profiling_path' {profiling_path}, skip analysis" + self._update_analysis_process_resp(pid, resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + if benchmark_profiling_path and not self._check_profiling_path_valid(benchmark_profiling_path): + error_msg = f"Got invalid argument '-bp/--benchmark_profiling_path' {benchmark_profiling_path}, skip analysis" + self._update_analysis_process_resp(pid, resp, error_msg=error_msg, + status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.FAILED) + logger.error(error_msg) + return + + self._is_cluster = self._is_cluster_profiling(profiling_path) + if not self._is_cluster: + job_list = self.single_rank_analysis(profiling_path, benchmark_profiling_path) + else: + job_list = self.cluster_analysis(profiling_path, benchmark_profiling_path) + + for i, (dimension, scope, interface, kwargs) in enumerate(job_list[::-1]): + result_list.append( + interface.get_result(dimension, scope, render_html=i == len(job_list) - 1, output_dict=False, + **kwargs) + ) + + for result in result_list[::-1]: + if result and hasattr(result, "show"): + result.show() + break + self._get_analysis_success_resp(pid, resp) + + def _communication_rdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None): + # 小包分析 + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + for dimension in [Interface.COMMUNICATION]: + for scope in Interface.get_scope(dimension): + if scope != SupportedScopes.PACKET: + continue + interface = Interface(**kwargs) + job_list.append((dimension, scope, interface, kwargs)) + + return job_list + + def _communication_sdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, + benchmark_step=None): + kwargs = copy.deepcopy(self.kwargs) + job_list = [] + return job_list + + def _single_profiling_comparison(self, profiling_path, benchmark_profiling_path, step=None, + benchmark_step=None): + # TODO 基于compare tools 对比计算下发 + kwargs = copy.deepcopy(self.kwargs) + return [] + + def _cluster_profiling_comparison(self, profiling_path, benchmark_profiling_path): + # 从计算、下发和通信三个维度对集群profiling数据进行对比 + + job_list = [] + benchmark_profiling_path = self._get_profiling_path_by_rank(benchmark_profiling_path) + benchmark_slow_rank_analyzer = SlowRankAnalyzer(benchmark_profiling_path) + benchmark_slow_link_analyzer = SlowLinkAnalyzer(benchmark_profiling_path) + + # 计算和下发分析 + job_list += self._cluster_data_comparison(profiling_path, + benchmark_profiling_path, + self.slow_rank_analyzer, + benchmark_slow_rank_analyzer, + get_max=True) + + # 通信分析 + job_list += self._cluster_data_comparison(profiling_path, + benchmark_profiling_path, + self.slow_link_analyzer, + benchmark_slow_link_analyzer, + get_max=False) + return job_list + + def _cluster_data_comparison(self, profiling_path, benchmark_profiling_path, target_cluster_analyzer, + benchmark_cluster_analyzer, get_max=False): + # #low rank/slow link结果逐行对比获取差值最大的rank和step进行单卡分析 + job_list = [] + + if isinstance(target_cluster_analyzer, SlowRankAnalyzer): + comparison_dims = [SlowRankAnalyzer.COMPUTE, SlowRankAnalyzer.FREE] + elif isinstance(target_cluster_analyzer, SlowLinkAnalyzer): + comparison_dims = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + else: + return job_list + + target_data = target_cluster_analyzer.format_datas.get("data", []) + benchmark_data = benchmark_cluster_analyzer.format_datas.get("data", []) + headers = benchmark_cluster_analyzer.format_datas.get("headers", []) + + if len(target_data) != len(benchmark_data): + logger.warning( + "The product of ranks and steps of Benchmark profiling is not equals to target profiling, skip cluster comparison.") + return job_list + + for dimension in comparison_dims: + step, benchmark_step, rank_id_for_comparison = AnalyzerController._get_step_rank_for_cluster_statistic_diff( + target_data, + benchmark_data, + headers, + dimension, + get_max=get_max + ) + rank_profiling_path = self._get_profiling_path_by_rank(profiling_path, rank_id_for_comparison) + rank_benchmark_profiling_path = self._get_profiling_path_by_rank( + benchmark_profiling_path, + rank_id_for_comparison + ) + + job_list += self._single_profiling_comparison( + rank_profiling_path, + rank_benchmark_profiling_path, + step, + benchmark_step + ) + return job_list + + def _is_cluster_profiling(self, profiling_path): + path_list = [os.path.join(profiling_path, dir_name) for dir_name in os.listdir(profiling_path)] + ascend_pt_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_pt")] + data_processor = PytorchDataPreprocessor(ascend_pt_dirs) + + self.cluster_local_data_map[profiling_path] = data_processor.get_data_map() + + if not self.cluster_local_data_map or not self.cluster_local_data_map.get(profiling_path): + return False + + self.default_rank_id = list(self.cluster_local_data_map[profiling_path].keys())[0] + + self.slow_rank_analyzer = SlowRankAnalyzer(profiling_path) + self.slow_link_analyzer = SlowLinkAnalyzer(profiling_path) + return len(self.cluster_local_data_map[profiling_path]) >= self.CLUSTER_RANK_THRESHOLD + + def _get_profiling_path_by_rank(self, profiling_path, rank_id=None): + + if not profiling_path: + return profiling_path + + return self._get_target_profiling_path_for_local(profiling_path, rank_id) + + def _get_target_profiling_path_for_local(self, profiling_path, rank_id): + rank_id_map = self.cluster_local_data_map.get(profiling_path, {}) + if rank_id is None or not rank_id_map: + return profiling_path + + if rank_id in rank_id_map: + return rank_id_map.get(rank_id) + + local_first_rank_id = sorted(list(map(int, rank_id_map.keys())))[0] + logger.warning("Target rank id %s does not exist in local profiling data %s, use rank %s for analysis", + rank_id, profiling_path, local_first_rank_id) + return rank_id_map.get(local_first_rank_id) + + def _update_analysis_process_resp(self, pid, resp, **kwargs): + if kwargs: + resp.update(kwargs) + self.analysis_process_resp[pid] = resp + + def _get_analysis_success_resp(self, pid, resp): + html_path = os.path.join(Config().work_path, f"mstt_advisor_{Timer().strftime}.html") + xlsx_path = os.path.join(Config().work_path, f"mstt_advisor_{Timer().strftime}.xlsx") + result_files = {"html": html_path, "xlsx": xlsx_path} + self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.SUCCESS, result_files=result_files) \ No newline at end of file diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index 80368e1d6..6347839b1 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -22,12 +22,16 @@ from profiler.advisor.common.version_control import VersionControl from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from profiler.advisor.utils.utils import safe_division logger = logging.getLogger() class BaseAnalyzer(VersionControl, metaclass=ABCMeta): _SUPPORT_VERSIONS = constant.SUPPORTED_CANN_VERSION + ANALYZER_HIGH_PRIORITY_TIME_RATIO = 0.05 + ANALYZER_MEDIUM_PRIORITY_TIME_RATIO = 0.03 dataset_cls_list = [] @@ -43,6 +47,18 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): self.result = OptimizeResult() self.record_list: Dict[str, List] = {} + @staticmethod + def get_first_data_by_key(data, key) -> Union[Dataset, None]: + """ + get the first member from data with key + :param data: input data + :param key: data key + :return: the first dataset in dataset list + """ + if key in data and len(data[key]) > 0: + return data[key][0] + return None + @classmethod def check_data(cls, data_list: tuple): """ @@ -63,7 +79,7 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): return None logger.info("Enable analysis %s with %s", self.__class__.__name__, ",".join(data_list)) - return func(self) + return func(self, **kwargs) return wrapper @@ -73,6 +89,10 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): def optimize(self, **kwargs): pass + @abstractmethod + def get_priority(self): + pass + def init_dataset_list(self)->None: dataset_cls_list = self.dataset_cls_list if len(dataset_cls_list) == 0: @@ -91,14 +111,25 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): self.dataset_list[key] = [] self.dataset_list[key].append(dataset) - @staticmethod - def get_first_data_by_key(data, key) -> Union[Dataset, None]: - """ - get the first member from data with key - :param data: input data - :param key: data key - :return: the first dataset in dataset list - """ - if key in data and len(data[key]) > 0: - return data[key][0] - return None + def init_dataset_list(self) -> None: + dataset_cls_list = self.dataset_cls_list + if len(dataset_cls_list) == 0: + logger.warning(f"Analyzer: %s don't rely on any dataset!", self.__class__.__name__) + return + + for dataset_cls in dataset_cls_list: + if dataset_cls and callable(dataset_cls): + dataset = dataset_cls(collection_path=self.collection_path, data=self.dataset_list, **self.kwargs) + key = dataset_cls.get_key() + if key not in self.dataset_list: + self.dataset_list[key] = [] + self.dataset_list[key].append(dataset) + + def get_priority_by_time_ratio(self, dur, step_dur): + time_ratio = safe_division(dur, step_dur) + if time_ratio >= self.ANALYZER_HIGH_PRIORITY_TIME_RATIO: + return PriorityBackgroundColor.high + elif time_ratio >= self.ANALYZER_MEDIUM_PRIORITY_TIME_RATIO: + return PriorityBackgroundColor.medium + else: + return PriorityBackgroundColor.low diff --git a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py b/profiler/advisor/analyzer/cluster/slow_link_analyzer.py new file mode 100644 index 000000000..438d3f55e --- /dev/null +++ b/profiler/advisor/analyzer/cluster/slow_link_analyzer.py @@ -0,0 +1,191 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from collections import defaultdict +from typing import Dict, List +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.common import constant +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from profiler.advisor.utils.utils import safe_index + +logger = logging.getLogger() + + +class SlowLinkAnalyzer(BaseAnalyzer): + RDMA_TIME_MS = "RDMA time(ms)" + RDMA_SIZE_MB = "RDMA size(mb)" + SDMA_TIME_MS = "SDMA time(ms)" + SDMA_SIZE_MB = "SDMA size(mb)" + RDMA_BANDWIDTH = "RDMA bandwidth(GB/s)" + SDMA_BANDWIDTH = "SDMA bandwidth(GB/s)" + COMMUNICATION_BANDWIDTH_INFO = "Communication Bandwidth Info" + TRANSIT_TIME = "Transit Time(ms)" + TRANSIT_SIZE = "Transit Size(MB)" + SDMA = "SDMA" + RDMA = "RDMA" + SLOW_LINK_ANALYSIS = "slow link" + RATIO_THRESHOLD = 0.05 + dataset_cls_list = [ClusterCommunicationDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs): + super().__init__(collection_path, n_processes, **kwargs) + key = ClusterCommunicationDataset.get_key() + self.communication_data_class = self.get_first_data_by_key(self.dataset_list, key) + self.rank_bw_dict = self.communication_data_class.get_data() + self.result = OptimizeResult() + self.bottelneck = '' + self.suggestion = '' + if self.rank_bw_dict is not None: + self.format_datas = self.format_details() + + @staticmethod + def compute_max_gap_ratio(data: list, mean: float): + if mean == 0: + return 0 + else: + return (max(data) - min(data)) / mean + + def optimize(self, **kwargs): + if self.rank_bw_dict is None: + logger.error("Slow link analysis failed due to data loading failure. \ + Please check your cluster_analysis_output folder. \ + If you are not concerned about this type of data, please ignore this message.") + return self.result + self.process() + self.make_record() + self.make_render(kwargs.get("template_key")) + return self.result + + def process(self): + if self.rank_bw_dict: + self.produce_bottleneck(self.RDMA_BANDWIDTH) + self.produce_bottleneck(self.SDMA_BANDWIDTH) + + def produce_bottleneck(self, link_type: str): + data_list = [rank_dict.get(link_type, 0) for rank_id, rank_dict in self.rank_bw_dict.items()] + if len(data_list) > 0: + avg_bw = round(sum(data_list) / len(data_list), 3) + else: + logger.info("The slow link (identified bottleneck) cannot provide a bottleneck \ + because the analysis data is missing bandwidth information.") + return + self.bottelneck += f'{link_type}: \n' \ + f' The average is {avg_bw}, \n' \ + f' while the maximum is {round(max(data_list), 3)}GB/s \n' \ + f' and the minimum is {round(min(data_list), 3)}GB/s. \n' \ + f' the difference is {round(max(data_list) - min(data_list), 3)}GB/s. \n' + + def format_details(self): + if not self.rank_bw_dict: + return { + "headers": [], + "data": [] + } + + details_dict = {} + headers = list({k for rank_bw_value in self.rank_bw_dict.values() for k in rank_bw_value.keys()}) + headers.sort() + + data_list = [] + for step_rank, rank_bw in self.rank_bw_dict.items(): + step_rank_list = list(map(int, step_rank.split(constant.STEP_RANK_SEP))) + value_list = [rank_bw.get(i, 0) for i in headers] + data_list.append(step_rank_list + value_list) + data_list.sort(key=lambda x: (x[0], x[1])) # 按rank_id排序 + + details_dict["headers"] = ["step", "rank_id"] + headers + details_dict["data"] = data_list + + return details_dict + + def make_record(self): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem( + SlowLinkAnalyzer.SLOW_LINK_ANALYSIS, + self.bottelneck, + self.suggestion + ) + self.result.add(OptimizeRecord(optimization_item)) + + data_list = self.format_datas.get("data", []) + headers = self.format_datas.get("headers", []) + for data in data_list: + self.result.add_detail(SlowLinkAnalyzer.SLOW_LINK_ANALYSIS, headers, data) + + def make_render(self, template_key="cluster"): + result_for_html = { + "Description": self.bottelneck, + "suggestion": self.suggestion, + "details": [self.format_datas] + } + + self.html_render.render_template(key=template_key, + title=SlowLinkAnalyzer.SLOW_LINK_ANALYSIS, + template_dir="templates", + template_name="cluster_analysis.html", + cann_version=self.cann_version, + torch_version=self.torch_version, + result=result_for_html) + + def get_global_step_rank(self, bindwidth_type): + global_step_rank = {} + bindwidth_key_map = {self.RDMA: self.RDMA_BANDWIDTH, self.SDMA: self.SDMA_BANDWIDTH} + + if bindwidth_type not in bindwidth_key_map: + raise RuntimeError(f"Error bindwidth type {bindwidth_type}, optionals are {bindwidth_key_map.keys()}") + + headers = self.format_datas.get("headers") + + bindwidth_index = safe_index(headers, bindwidth_key_map.get(bindwidth_type)) + + if bindwidth_index is not None: + data_list = [tuple_list[bindwidth_index] for tuple_list in self.format_datas.get("data", [])] + max_bandwidth, min_bandwidth = max(data_list), min(data_list) + + if self.compute_max_gap_ratio(data_list, sum(data_list) / len( + data_list)) < self.RATIO_THRESHOLD: + return global_step_rank + + max_bandwidth_index = data_list.index(max_bandwidth) + min_bandwidth_index = data_list.index(min_bandwidth) + + rank_id_index = safe_index(headers, "rank_id") + step_index = safe_index(headers, "step") + + if rank_id_index is None: + return global_step_rank + + max_bandwidth_rank_id = self.format_datas.get("data")[max_bandwidth_index][rank_id_index] + min_bandwidth_rank_id = self.format_datas.get("data")[min_bandwidth_index][rank_id_index] + + if step_index is None: + max_bandwidth_step, min_bandwidth_step = constant.DEFAULT_STEP, constant.DEFAULT_STEP + else: + max_bandwidth_step = self.format_datas.get("data")[max_bandwidth_index][step_index] + min_bandwidth_step = self.format_datas.get("data")[min_bandwidth_index][step_index] + + global_step_rank["maximum"] = {"rank_id": max_bandwidth_rank_id, "step": max_bandwidth_step} + global_step_rank["minimum"] = {"rank_id": min_bandwidth_rank_id, "step": min_bandwidth_step} + + return global_step_rank + + def get_priority(self): + pass \ No newline at end of file diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py new file mode 100644 index 000000000..a1971baf9 --- /dev/null +++ b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py @@ -0,0 +1,216 @@ +# Copyright (c) 2023, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.common import constant +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset +from profiler.advisor.utils.utils import safe_index + +logger = logging.getLogger() + + +class SlowRankAnalyzer(BaseAnalyzer): + SLOW_RANK_ANALYSIS = "slow rank" + RANK = "rank" + RATIO_THRESHOLD = 0.05 + BOTTLENECK_LIST = ['Computing', 'Communication', "Free"] + dataset_cls_list = [ClusterStepTraceTimeDataset] + COMPUTE = "compute(us)" + FREE = "free(us)" + COMMUNICATION = "communication(us)" + + def __init__(self, collection_path, n_processes: int = 1, **kwargs): + super().__init__(collection_path, n_processes, **kwargs) + key = ClusterStepTraceTimeDataset.get_key() + self.step_trace_class = self.get_first_data_by_key(self.dataset_list, key) + self.step_trace_dict = self.step_trace_class.get_data() + self.stages = self.step_trace_class.get_stages() + self.result = OptimizeResult() + self.bottelneck = '' + self.suggestion = '' + self._steps = set() + if self.step_trace_dict is not None: + self.format_datas = self.format_details() + + @property + def steps(self): + return sorted(list(self._steps)) + + @staticmethod + def compute_max_gap_ratio(data: list, mean: float): + if mean == 0: + return 0 + else: + return (max(data) - min(data)) / mean + + def optimize(self, **kwargs): + if self.step_trace_dict is None: + logger.error("slow_rank 分析失败，原因是数据加载失败，请检查你的cluster_analysis_outpu文件夹 \ + 如不关心这类数据请忽略") + return self.result + self.process() + self.make_record() + self.make_render(kwargs.get("template_key")) + return self.result + + def process(self): + total_time_list = [sum(data_tuple) for rank_id, data_tuple in self.step_trace_dict.items()] + if total_time_list: + mean_total_time = sum(total_time_list) / len(total_time_list) + for i in range(len(self.BOTTLENECK_LIST)): + self.produce_bottleneck(self.step_trace_dict, i, mean_total_time) + + if not self.bottelneck: + self.bottelneck = "There is no slow rank issues" + + def produce_bottleneck(self, step_dict: dict, produce_type: int, mean_total_time: float): + data_list = [data_tuple[produce_type] for rank_id, data_tuple in step_dict.items()] + max_ratio = self.compute_max_gap_ratio(data_list, mean_total_time) + if max_ratio > self.RATIO_THRESHOLD: + self.bottelneck += f'{self.BOTTLENECK_LIST[produce_type]} \n' \ + f' has some issues in the cluster, \n' \ + f' because the max difference of {self.BOTTLENECK_LIST[produce_type]} time \n' \ + f' has reached {round(max_ratio * mean_total_time / 1000, 3)}ms. \n' + + def make_record(self): + """ + make record for what and how to optimize + """ + + optimization_item = OptimizeItem( + SlowRankAnalyzer.SLOW_RANK_ANALYSIS, + self.bottelneck, + self.suggestion + ) + self.result.add(OptimizeRecord(optimization_item)) + + data_list = self.format_datas.get("data", []) + headers = self.format_datas.get("headers", []) + for data in data_list: + self.result.add_detail(SlowRankAnalyzer.SLOW_RANK_ANALYSIS, headers, data) + + def format_details(self): + details_dict = {} + headers = ["step", "rank_id", "compute(us)", "communication(us)", "free(us)"] + data_list = [] + for key, value in self.step_trace_dict.items(): + step, rank_id = key.split(constant.STEP_RANK_SEP) + data_list.append([int(step), int(rank_id)] + value) + if step and step not in self._steps: + self._steps.add(step) + + details_dict["headers"] = headers + details_dict["data"] = sorted(data_list, key=lambda x: (x[0], x[1])) + return details_dict + + def make_render(self, template_key="cluster"): + result_for_html = { + "Description": self.bottelneck, + "suggestion": self.suggestion, + "details": [self.format_datas] + } + + self.html_render.render_template(key=template_key, + title=SlowRankAnalyzer.SLOW_RANK_ANALYSIS, + template_dir="templates", + template_name="cluster_analysis.html", + cann_version=self.cann_version, + torch_version=self.torch_version, + result=result_for_html) + + def get_global_step_rank(self, dimension): + global_step_rank = {} + + headers = self.format_datas.get("headers") + + dimension_index = safe_index(headers, dimension) + rank_id_index = safe_index(headers, "rank_id") + step_index = safe_index(headers, "step") + if dimension_index is None or rank_id_index is None: + return global_step_rank + + data_list = [tuple_list[dimension_index] for tuple_list in self.format_datas.get("data")] + max_time, min_time = max(data_list), min(data_list) + + if self.compute_max_gap_ratio(data_list, sum(data_list) / len( + data_list)) < self.RATIO_THRESHOLD: + return global_step_rank + max_time_index = data_list.index(max_time) + min_time_index = data_list.index(min_time) + + max_time_rank_id = self.format_datas.get("data")[max_time_index][rank_id_index] + min_time_rank_id = self.format_datas.get("data")[min_time_index][rank_id_index] + + if step_index is not None: + max_time_step = self.format_datas.get("data")[max_time_index][step_index] + min_time_step = self.format_datas.get("data")[min_time_index][step_index] + else: + max_time_step, min_time_step = constant.DEFAULT_STEP, constant.DEFAULT_STEP + + global_step_rank["maximum"] = {"rank_id": max_time_rank_id, "step": max_time_step} + global_step_rank["minimum"] = {"rank_id": min_time_rank_id, "step": min_time_step} + + return global_step_rank + + def get_stage_step_rank(self, dimension): + stage_step_rank = {} + + headers = self.format_datas.get("headers") + dimension_index = safe_index(headers, dimension) + rank_id_index = safe_index(headers, "rank_id") + step_index = safe_index(headers, "step") + if dimension_index is None or rank_id_index is None: + return stage_step_rank + + rank_list = [tuple_list[rank_id_index] for tuple_list in self.format_datas.get("data")] + cost_time_list = [tuple_list[dimension_index] for tuple_list in self.format_datas.get("data")] + + if step_index is not None: + step_list = [tuple_list[step_index] for tuple_list in self.format_datas.get("data")] + else: + step_list = [constant.DEFAULT_STEP] * len(rank_list) + + for index, stage in enumerate(self.stages): + tmp_step_list, tmp_rank_list, tmp_time_list = [], [], [] + for step, rank_id, time in zip(step_list, rank_list, cost_time_list): + if rank_id not in stage: + continue + + tmp_step_list.append(step) + tmp_rank_list.append(rank_id) + tmp_time_list.append(time) + + if self.compute_max_gap_ratio(tmp_time_list, sum(tmp_time_list) / len( + tmp_time_list)) < self.RATIO_THRESHOLD: + continue + + max_time, min_time = max(tmp_time_list), min(tmp_time_list) + max_time_index, min_time_index = tmp_time_list.index(max_time), tmp_time_list.index(min_time) + + stage_key = f"stage-{index}" + stage_step_rank[stage_key] = {} + stage_step_rank[stage_key]["maximum"] = {"rank_id": tmp_rank_list[max_time_index], + "step": tmp_step_list[max_time_index]} + stage_step_rank[stage_key]["minimum"] = {"rank_id": tmp_rank_list[min_time_index], + "step": tmp_step_list[min_time_index]} + + return stage_step_rank + + def get_priority(self): + pass \ No newline at end of file diff --git a/profiler/advisor/analyzer/communication/base_communication_analyzer.py b/profiler/advisor/analyzer/communication/base_communication_analyzer.py new file mode 100644 index 000000000..95a830e47 --- /dev/null +++ b/profiler/advisor/analyzer/communication/base_communication_analyzer.py @@ -0,0 +1,8 @@ +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer + + +class BaseCommunicationAnalyzer(BaseAnalyzer): + requires_cluster_dataset = True + + def __init__(self, collection_path, n_processes: int = 1, **kwargs): + super().__init__(collection_path, n_processes, **kwargs) diff --git a/profiler/advisor/analyzer/communication/packet/__init__.py b/profiler/advisor/analyzer/communication/packet/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/communication/packet_analyzer.py b/profiler/advisor/analyzer/communication/packet/packet_analyzer.py similarity index 74% rename from profiler/advisor/analyzer/communication/packet_analyzer.py rename to profiler/advisor/analyzer/communication/packet/packet_analyzer.py index 73e5bc2bc..e77ea7780 100644 --- a/profiler/advisor/analyzer/communication/packet_analyzer.py +++ b/profiler/advisor/analyzer/communication/packet/packet_analyzer.py @@ -14,17 +14,19 @@ # limitations under the License. import logging -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.analyzer.communication.packet_checker import PacketChecker +from profiler.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer +from profiler.advisor.analyzer.communication.packet.packet_checker import PacketChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset +from profiler.advisor.result.result import OptimizeResult logger = logging.getLogger() -class PacketAnalyzer(BaseAnalyzer): +class PacketAnalyzer(BaseCommunicationAnalyzer): dataset_cls_list = [CommunicationDataset] + requires_cluster_dataset = False def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) @@ -34,7 +36,7 @@ class PacketAnalyzer(BaseAnalyzer): self.html_render = HTMLRender() self.html = None - @BaseAnalyzer.check_data((CommunicationDataset.get_key(),)) + @BaseCommunicationAnalyzer.check_data((CommunicationDataset.get_key(),)) def optimize(self, **kwargs): add_render_list = kwargs.get("add_render_list", True) packet_checker = PacketChecker(**kwargs) @@ -42,5 +44,9 @@ class PacketAnalyzer(BaseAnalyzer): if not packet_checker.packet_issues: return self.result packet_checker.make_record(self.result) - self.html = packet_checker.make_render(self.html_render, add_render_list) + self.html = packet_checker.make_render(self.html_render, add_render_list, priority=self.get_priority()) return self.result + + def get_priority(self): + # 提升1% ~ 3% + return PriorityBackgroundColor.low diff --git a/profiler/advisor/analyzer/communication/packet_checker.py b/profiler/advisor/analyzer/communication/packet/packet_checker.py similarity index 96% rename from profiler/advisor/analyzer/communication/packet_checker.py rename to profiler/advisor/analyzer/communication/packet/packet_checker.py index 3d9ac81ff..d270667cd 100644 --- a/profiler/advisor/analyzer/communication/packet_checker.py +++ b/profiler/advisor/analyzer/communication/packet/packet_checker.py @@ -116,19 +116,20 @@ class PacketChecker: result.add_detail(sub_table_name, headers=self.headers) result.add_detail(sub_table_name, detail=self.small_packet_detail) - def make_render(self, html_render, add_render_list=True): + def make_render(self, html_render, add_render_list=True, **kwargs): + priority = kwargs.get("priority") return html_render.render_template(key="communication", template_dir="templates", template_name="packet_analysis.html", desc=self.desc, solutions=self.solutions, headers=self.headers, - data=self.small_packet_detail - ) + data=self.small_packet_detail, + priority_background_color=priority) def _init_rule(self): syncbn_rule_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))), "rules", "packet.yaml" ) diff --git a/profiler/advisor/analyzer/communication/retransmission/__init__.py b/profiler/advisor/analyzer/communication/retransmission/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py new file mode 100644 index 000000000..78cade900 --- /dev/null +++ b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_analyzer.py @@ -0,0 +1,52 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer +from profiler.advisor.analyzer.communication.retransmission.communication_retransmission_checker import \ + CommunicationRetransmissionChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from profiler.advisor.result.result import OptimizeResult + +logger = logging.getLogger() + + +class RDMARetransmissionAnalyzer(BaseCommunicationAnalyzer): + dataset_cls_list = [ClusterCommunicationDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = ClusterCommunicationDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseCommunicationAnalyzer.check_data((ClusterCommunicationDataset.get_key(),)) + def optimize(self, **kwargs): + add_render_list = kwargs.get("add_render_list", True) + rdma_checker = CommunicationRetransmissionChecker(**kwargs) + rdma_checker.check_retransmission(self.dataset) + if not rdma_checker.rdma_issues: + return self.result + rdma_checker.make_record(self.result) + self.html = rdma_checker.make_render(self.html_render, add_render_list, priority=self.get_priority()) + return self.result + + def get_priority(self): + # 单次重传最少4s，高优先级 + return PriorityBackgroundColor.high diff --git a/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py new file mode 100644 index 000000000..4431ccce4 --- /dev/null +++ b/profiler/advisor/analyzer/communication/retransmission/communication_retransmission_checker.py @@ -0,0 +1,129 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from typing import Dict, List +from collections import defaultdict +from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo + +logger = logging.getLogger() + + +class GroupStatistic: + def __init__(self, min_transmission_time): + self.retransmission_issue = False + self.abnormal_op_dict: Dict[str, List] = dict() + + def add_op(self, op_name: str, hccl_info: HcclInfo): + if self.abnormal_op_dict.get(op_name) is None: + self.abnormal_op_dict.setdefault(op_name, []) + self.abnormal_op_dict.get(op_name).append([hccl_info.group, op_name, hccl_info.step, hccl_info.rank, + hccl_info.get_rdma_transit_size(), + hccl_info.get_rdma_transmit_time(), hccl_info.get_rdma_bandwidth()]) + + +class CommunicationRetransmissionChecker: + def __init__(self, **kwargs): + self.rdma_issues = False + self.desc = "" + self.sdma_desc = "" + self.rdma_desc = "" + self.suggestions = [] + self.abnormal_group_count = 0 + self.abnormal_rdma_list = [] + self.step_id = kwargs.get("step") + self.stage = None + self.group_statistics = defaultdict(GroupStatistic) + self.headers = ["Communication group", "Op name", "Step id", "Rank id", "RDMA transmit size(MB)", + "RDMA transmit time(ms)", "RDMA bandwidth"] + self._init_rule() + + def check_possible_retransmission_occurrence(self, hccl_list: List[HcclInfo]): + min_elapse_time = min(hccl.elapse_time for hccl in hccl_list) + max_transit_time = max(hccl.rdma_info.get('Transit Time(ms)', 0) for hccl in hccl_list) + if min_elapse_time < self.min_retransmission_time: # 检测是否是卡间不同步问题，而不是重传 + return False + return max_transit_time > self.min_retransmission_time + + def check_retransmission(self, hccl_dataset: ClusterCommunicationDataset): + """ + :Param event_dataset: dataset of timeline event + """ + for group_name, hccl_group_dict in hccl_dataset.hccl_dict.items(): + for op_name, hccl_op_dict in hccl_group_dict.items(): + for step_id, hccl_list in hccl_op_dict.items(): + if self.step_id and step_id != self.step_id: # 传输指定step（self.step_id）情况下，非目标step跳过 + continue + if not self.check_possible_retransmission_occurrence(hccl_list): + continue + self.rdma_issues = True + if self.group_statistics.get(group_name) is None: + self.group_statistics.setdefault(group_name, GroupStatistic(self.min_retransmission_time)) + self.abnormal_group_count += 1 + for hccl_info in hccl_list: + if hccl_info.rdma_info.get('Transit Size(MB)', 0): + transit_time = hccl_info.rdma_info.get('Transit Time(ms)', 0) + if transit_time > self.min_retransmission_time: + self.group_statistics.get(group_name).add_op(op_name, hccl_info) + if self.rdma_issues: + self.desc = self.desc.format(group_count=self.abnormal_group_count) + for _, group_statistic in self.group_statistics.items(): + for _, op_list in group_statistic.abnormal_op_dict.items(): + for op in op_list: + self.abnormal_rdma_list.append(op) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("Communication retransmission analysis", self.desc, self.suggestions) + result.add(OptimizeRecord(optimization_item)) + + sub_table_name = "Comm Retransmission Analysis" if not self.stage else f"Stage-{self.stage}: Comm Retransmission Analysis" + result.add_detail(sub_table_name, headers=self.headers) + + for row in self.abnormal_rdma_list: + result.add_detail(sub_table_name, detail=row) + + def make_render(self, html_render, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="communication", + template_dir="templates", + template_name="communication_retransmission_analysis.html", + desc=self.desc, + solutions=self.solutions, + headers=self.headers, + data=self.abnormal_rdma_list, + priority_background_color=priority) + + def _init_rule(self): + syncbn_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))), + "rules", + "rdma_analysis.yaml" + ) + + syncbn_rule = FileManager.read_yaml_file(syncbn_rule_path) + self.desc = syncbn_rule.get("problem") + self.min_retransmission_time = syncbn_rule.get("min_retransmission_time") + + self.solutions = syncbn_rule.get("solutions") + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py index 4f25deff7..bc0841152 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py @@ -3,34 +3,40 @@ import logging from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_checker import AICoreFreqChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.ai_core_freq.ai_core_freq_dataset import AICoreFreqDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset +from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser from profiler.advisor.config.config import Config logger = logging.getLogger() class AICoreFreqAnalyzer(BaseAnalyzer): - dataset_cls_list = [AICoreFreqDataset] + dataset_cls_list = [ComputationAnalysisDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) - key = AICoreFreqDataset.get_key() + key = ComputationAnalysisDataset.get_key() self.dataset = self.get_first_data_by_key(self.dataset_list, key) self.result = OptimizeResult() self.html_render = HTMLRender() self.html = None + info = DeviceInfoParser(collection_path) + info.parse_data() - @BaseAnalyzer.check_data((AICoreFreqDataset.get_key(),)) + @BaseAnalyzer.check_data((ComputationAnalysisDataset.get_key(),)) def optimize(self, **kwargs): if not Config().get_config("aic_frequency"): logger.warning("Can not find ai core frequency in info.json*, please check data integrity.") return self.result + add_render_list = kwargs.get("add_render_list", True) ai_core_freq_checker = AICoreFreqChecker() - ai_core_freq_checker.check_ai_core_freq(self.dataset) - if not ai_core_freq_checker.ai_core_freq_issues: - return self.result + ai_core_freq_checker.check_ai_core_freq(self.dataset, rank_id=kwargs.get("rank_id"), stage=kwargs.get("stage")) ai_core_freq_checker.make_record(self.result) - self.html = ai_core_freq_checker.make_render(self.html_render, add_render_list) + self.html = ai_core_freq_checker.make_render(self.html_render, add_render_list, priority=self.get_priority()) return self.result + + def get_priority(self): + return PriorityBackgroundColor.high \ No newline at end of file diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index 5bfa5adc4..c8a94287d 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -1,6 +1,6 @@ import logging -from profiler.advisor.dataset.ai_core_freq.ai_core_freq_dataset import AICoreFreqDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.config.config import Config @@ -26,7 +26,7 @@ class AICoreFreqChecker: self.rank_id = None self.stage = None - def check_ai_core_freq(self, event_dataset: AICoreFreqDataset, rank_id=None, stage=None): + def check_ai_core_freq(self, event_dataset: ComputationAnalysisDataset, rank_id=None, stage=None): """ :Param event_dataset: dataset of timeline event """ @@ -60,6 +60,8 @@ class AICoreFreqChecker: self.decrease_freq_ops.sort(key= lambda x: (x[self.TOTAL_DURATION_INDEX], x[self.DECREASE_FREQ_RATIO_INDEX]), reverse=True) + if not self.ai_core_freq_issues: + return self.desc = (f"{len(self.decrease_freq_ops)} operators are found during frequency reduction, and the reduction " f"ratio is larger than {self.DECREASE_FREQ_RATIO}.") @@ -71,22 +73,29 @@ class AICoreFreqChecker: """ make record for what and how to optimize """ - optimization_item = OptimizeItem("AI Core Frequency", self.desc, [self.suggestions]) + if not self.ai_core_freq_issues: + return self.ai_core_freq_issues + + sheet_name = "AI Core Frequency" + if self.rank_id is not None: + sheet_name = f"rank {self.rank_id} AI Core Frequency".capitalize() + + optimization_item = OptimizeItem(sheet_name, self.desc, [self.suggestions]) result.add(OptimizeRecord(optimization_item)) self.headers = ["Operator name", "Count", "Total duration(us)", "AI CORE frequency decreased ratio", "Average frequency", "Max frequency", "Min frequency"] - if self.rank_id: - self.headers = ["Rank id"] + self.headers - sub_table_name = "AI Core Frequency" if not self.stage else f"Stage-{self.stage}: AI Core Frequency" - result.add_detail(sub_table_name, headers=self.headers) + result.add_detail(sheet_name, headers=self.headers) for row in self.decrease_freq_ops: - if self.rank_id: - row = [self.rank_id] + row - result.add_detail(sub_table_name, detail=row) + result.add_detail(sheet_name, detail=row) + return True + + def make_render(self, html_render, add_render_list=True, **kwargs): + if not self.ai_core_freq_issues: + return self.ai_core_freq_issues - def make_render(self, html_render, add_render_list=True): + priority = kwargs.get("priority") if self.SHOW_TOPK_OPS: self.desc += f" Only show {self.SHOW_TOPK_OPS} operators here, see latest mstt_advisor.xlsx for details." return html_render.render_template(key="computation", @@ -96,4 +105,5 @@ class AICoreFreqChecker: suggestion=self.suggestions, headers=self.headers, data=self.decrease_freq_ops[:self.SHOW_TOPK_OPS], - add_render_list=add_render_list) + add_render_list=add_render_list, + priority_background_color=priority) diff --git a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py b/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py index 0caede4b8..394ad74fd 100644 --- a/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py +++ b/profiler/advisor/analyzer/computation/aicpu/aicpu_checker.py @@ -8,7 +8,7 @@ from profiler.advisor.analyzer.schedule.fusion_ops.timeline_api_stack_checker im from profiler.advisor.common import constant from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset from profiler.cluster_analyse.common_func.file_manager import FileManager @@ -30,6 +30,8 @@ class AicpuChecker(OperatorChecker): self.aicpu_rules: Dict = {} self.aicpu_checker: Dict = {} self.load_aicpu_rules() + self.total_task_duration = 0.0 + self.aicpu_task_duration = 0.0 def _check_data(self, profiling_data: ProfilingDataset) -> bool: if not self._check_summary(profiling_data): @@ -88,7 +90,7 @@ class AicpuChecker(OperatorChecker): def get_opeartor_stack_info(api_stack_finder: OpStackFinder, op_name_list: list) -> list: data: Dict[str, Dataset] = {} - event_dataset = TimelineEventDataset(collection_path=profiling_data.collection_path, data=data, task_type=constant.AI_CPU) + event_dataset = ComputationAnalysisDataset(collection_path=profiling_data.collection_path, data=data, task_type=constant.AI_CPU) # disable multiprocessing, avoid cost time of enable new process for light task api_stack_finder.get_api_stack_by_op(event_dataset, op_name_list, constant.AI_CPU, @@ -96,14 +98,16 @@ class AicpuChecker(OperatorChecker): return api_stack_finder._stack_record self._op_list = [] - total_task_duration = 0.0 + max_task_duration = 0.0 for op_info in op_summary.op_list: + task_duration = float(op_info.task_duration) + if self._check_operator(op_info): self._op_list.append(op_info) + self.aicpu_task_duration += task_duration - task_duration = float(op_info.task_duration) - total_task_duration += task_duration + self.total_task_duration += task_duration max_task_duration = max(max_task_duration, task_duration) if (not self._op_list) or (max_task_duration < self._MIN_TASK_DURATION): return False @@ -145,11 +149,15 @@ class AicpuChecker(OperatorChecker): ",".join(double_type_ai_cpu_operator))) return True - def make_render(self, html_render, record): - html_render.render_template(key="computation", - template_dir="templates", - template_name="operator_ai_cpu.html", - format_result=self.format_operator_result(record, constant.OPERATOR_LIST_UNLIMIT)) + def make_render(self, html_render, record, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="computation", + template_dir="templates", + template_name="operator_ai_cpu.html", + format_result=self.format_operator_result(record, + constant.OPERATOR_LIST_UNLIMIT), + add_render_list=add_render_list, + priority_background_color=priority) def format_operator_result(self, record, limit): """ diff --git a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py index 7a873c656..5b358ebaa 100644 --- a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py +++ b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py @@ -45,11 +45,15 @@ class BlockDimChecker(OperatorChecker): "task duration are as follows:\n" return True - def make_render(self, html_render, record): - html_render.render_template(key="computation", - template_dir="templates", - template_name="operator_block_dim.html", - format_result=self.format_operator_result(record, constant.OPERATOR_OUT_TOPK)) + def make_render(self, html_render, record, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="computation", + template_dir="templates", + template_name="operator_block_dim.html", + format_result=self.format_operator_result(record, + constant.OPERATOR_OUT_TOPK), + add_render_list=add_render_list, + priority_background_color=priority) def _check_operator(self, op_info) -> bool: if op_info.task_type not in ["AI_CORE", "AI_VECTOR_CORE", "MIX_AIC"]: diff --git a/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py b/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py index a22b380f9..2096e9ffa 100644 --- a/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py +++ b/profiler/advisor/analyzer/computation/bound/operator_bound_checker.py @@ -46,8 +46,12 @@ class OperatorBoundChecker(OperatorChecker): return False return True - def make_render(self, html_render, record): - html_render.render_template(key="computation", - template_dir="templates", - template_name="operator_no_bound.html", - format_result=self.format_operator_result(record, constant.OPERATOR_OUT_TOPK)) + def make_render(self, html_render, record, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="computation", + template_dir="templates", + template_name="operator_no_bound.html", + format_result=self.format_operator_result(record, + constant.OPERATOR_OUT_TOPK), + add_render_list=add_render_list, + priority_background_color=priority) \ No newline at end of file diff --git a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py b/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py index 86d3bac4f..2521b6e7e 100644 --- a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py +++ b/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py @@ -27,11 +27,13 @@ class DynamicShapeChecker(OperatorChecker): def check(self, profiling_database) -> bool: return self.is_dynamic_shape(profiling_database) - def make_record(self, profiling_database) -> OptimizeRecord: + def make_record(self, profiling_database, rank_id=None) -> OptimizeRecord: """ make record for what and how to optimize """ + if rank_id is not None: + self._PROBLEM = f"rank {rank_id} ".capitalize() + self._PROBLEM.lower() optimization_item = OptimizeItem( self._PROBLEM, self._description, @@ -58,8 +60,11 @@ class DynamicShapeChecker(OperatorChecker): format_result = {"record": record.__dict__, "suggestion": '
'.join(release_suggestion_list)} return format_result - def make_render(self, html_render, record): - html_render.render_template(key="computation", - template_dir="templates", - template_name="operator_dynamic_shape.html", - format_result=self.format_operator_result(record)) + def make_render(self, html_render, record, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="computation", + template_dir="templates", + template_name="operator_dynamic_shape.html", + format_result=self.format_operator_result(record), + add_render_list=add_render_list, + priority_background_color=priority) diff --git a/profiler/advisor/analyzer/computation/operator_checker.py b/profiler/advisor/analyzer/computation/operator_checker.py index 64618b56a..e24eae1d0 100644 --- a/profiler/advisor/analyzer/computation/operator_checker.py +++ b/profiler/advisor/analyzer/computation/operator_checker.py @@ -40,6 +40,23 @@ class OperatorChecker(VersionControl): self.cann_version = cann_version self._op_list: List[OpInfo] = [] + @staticmethod + def get_ratio(op_info: OpInfo, attr: str) -> float: + if not op_info.has_attr(attr): + return 0 + value = op_info.get_attr(attr) + if not value or value == "N/A": + return 0 + return float(value) + + @classmethod + def get_name(cls): + """ + get name of checker + :return: checker name + """ + return cls._PROBLEM + def check(self, profiling_data: ProfilingDataset) -> bool: """ check if any operator need optimize @@ -77,12 +94,16 @@ class OperatorChecker(VersionControl): return True return False - def make_record(self, profiling_data: ProfilingDataset): + def make_record(self, profiling_data: ProfilingDataset, rank_id=None): """ Make record for what and how to optimize :param profiling_data: profiling data :return: optimize record """ + + if rank_id is not None: + self._PROBLEM = f"rank {rank_id} ".capitalize() + self._PROBLEM.lower() + task_duration_list = [float(op_info.get_attr("task_duration")) for op_info in self._op_list if hasattr(op_info, "get_attr")] total_cost_time = sum(task_duration_list) @@ -239,14 +260,6 @@ class OperatorChecker(VersionControl): """Get node views.""" return [] - @classmethod - def get_name(cls): - """ - get name of checker - :return: checker name - """ - return cls._PROBLEM - def get_incomes(self) -> float: """get incomes""" incomes = 0.0 @@ -269,16 +282,7 @@ class OperatorChecker(VersionControl): logger.warning(self.SKIP_CHECK_MSG, self._CHECKER, "op summary") return False return True - - @staticmethod - def get_ratio(op_info: OpInfo, attr: str) -> float: - if not op_info.has_attr(attr): - return 0 - value = op_info.get_attr(attr) - if not value or value == "N/A": - return 0 - return float(value) - + def get_details(self) -> list: """ get details of operator to be optimized diff --git a/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py b/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py new file mode 100644 index 000000000..bc02b4c3e --- /dev/null +++ b/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py @@ -0,0 +1,106 @@ +import logging +import os +from multiprocessing import Manager + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.common.analyzer_scopes import SupportedScopes +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from profiler.advisor.interface.interface import Interface +from profiler.advisor.utils.utils import ParallelJob +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.common import constant as const + +logger = logging.getLogger() + + +class PPStageComputationAnalyzer(BaseAnalyzer): + + def __init__(self, collection_path, **kwargs): + super().__init__(collection_path, **kwargs) + self.collection_path = collection_path + self._stages_rendered_html = Manager().list() + self._multiprocess_result = Manager().dict() + # html render不能序列化，无法用多进程，放到optimize里面初始化 + self.html_render = None + self.result = None + + @staticmethod + def _get_valid_sheet_name(sheet_name, prefix): + if not sheet_name.lower().startswith(prefix.lower()): + sheet_name = f"{prefix} {sheet_name}" + return sheet_name + + def optimize(self, stages_profiling_path, **kwargs): + pp_stage_processes = min(int(os.getenv("PP_STAGE_ANALYSIS_PROCESSES", 0)), len(stages_profiling_path), + const.MAX_NUM_PROCESSES) + if pp_stage_processes <= 1: + for stage_profiling_path in stages_profiling_path: + self._optimize(**stage_profiling_path) + else: + logger.info("Start to parallel analysis of pp stages, number of processes is %s", pp_stage_processes) + parallel_stage_analysis_job = ParallelJob(self._optimize, stages_profiling_path, + "Computation analysis of Pipeline parallel stages") + parallel_stage_analysis_job.start(pp_stage_processes) + self._merge_multiprocess_result() + + self.make_render() + self.html_render = HTMLRender() + return self.result + + def make_render(self): + HTMLRender().render_template(key="computation", + template_dir="templates", + template_name="pp_stage_computation_analysis.html", + stages_rendered_html=list(self._stages_rendered_html), + priority_background_color=PriorityBackgroundColor.high) + + def get_priority(self): + pass + + def _optimize(self, profiling_path, **kwargs): + stage_html_record = dict(stage=kwargs.get("stage"), rank_id=kwargs.get("rank_id"), step=kwargs.get("step")) + kwargs["add_render_list"] = False + + # stage 并行分析时，避免调用本身，即SupportedScopes.STAGE_COMPUTE + scopes = Interface.get_scope(Interface.COMPUTATION) + stage_analyzer_list = [Interface.get_analyzer(Interface.COMPUTATION, scope) for scope in scopes if + scope != SupportedScopes.STAGE_COMPUTE] + + for analyzer_cls in stage_analyzer_list: + analyzer = analyzer_cls(collection_path=profiling_path, **kwargs) + result = analyzer.optimize(**kwargs) + if hasattr(result, "data") and result.data: + self.result = result + if hasattr(analyzer, "html") and analyzer.html: + if "html_list" not in stage_html_record: + stage_html_record["html_list"] = [] + stage_html_record["html_list"].append(analyzer.html) + self._stages_rendered_html.append(stage_html_record) + self._multiprocess_result[f"rank {kwargs.get('rank_id')}".capitalize()] = result.data + + def _merge_multiprocess_result(self): + self.result = OptimizeResult() + for key, result_data in self._multiprocess_result.items(): + problem_data = result_data.get("problems", {}).get("data", []) + if not problem_data: + continue + + for row in problem_data: + if len(row) < 3: + continue + issue_name, desc, suggestion = row[:3] + sheet_name = PPStageComputationAnalyzer._get_valid_sheet_name(issue_name, key) + optimization_item = OptimizeItem(sheet_name, desc, [suggestion]) + self.result.add(OptimizeRecord(optimization_item)) + del result_data["problems"] + + for issue_name, issue_details in result_data.items(): + headers = issue_details.get("headers", []) + data = issue_details.get("data", []) + sheet_name = PPStageComputationAnalyzer._get_valid_sheet_name(issue_name, key) + self.result.add_detail(sheet_name, headers=headers) + + for row in data: + self.result.add_detail(sheet_name, detail=row) diff --git a/profiler/advisor/analyzer/computation/profiling_analyzer.py b/profiler/advisor/analyzer/computation/profiling_analyzer.py index 2021bcd57..b29373e87 100644 --- a/profiler/advisor/analyzer/computation/profiling_analyzer.py +++ b/profiler/advisor/analyzer/computation/profiling_analyzer.py @@ -8,6 +8,7 @@ from profiler.advisor.analyzer.computation.bound.block_dim_checker import BlockD from profiler.advisor.analyzer.computation.bound.operator_bound_checker import OperatorBoundChecker from profiler.advisor.analyzer.computation.op_compile.dynamic_shape_checker import DynamicShapeChecker from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset @@ -22,6 +23,7 @@ class ProfilingAnalyzer(BaseAnalyzer, ABC): self.checker = OperatorChecker(self.cann_version) self.html_render = HTMLRender() self.result = OptimizeResult() + self.html = None @BaseAnalyzer.check_data((ProfilingDataset.get_key(),)) def optimize(self, **kwargs) -> OptimizeResult: @@ -32,22 +34,29 @@ class ProfilingAnalyzer(BaseAnalyzer, ABC): """ profiling_data = self.get_first_data_by_key(self.dataset_list, ProfilingDataset.get_key()) checker = self.checker + rank_id = kwargs.get("rank_id") + + add_render_list = kwargs.get("add_render_list", True) + if not checker.pre_check(profiling_data): return self.result if checker.check(profiling_data): # add record - record = checker.make_record(profiling_data) - checker.make_render(self.html_render, record) + record = checker.make_record(profiling_data, rank_id) + self.html = checker.make_render(self.html_render, record, add_render_list, + priority=self.get_priority(checker)) self.result.add(record) # add details details = checker.get_details() if details: for i, detail in enumerate(details): + sheet_name = checker.get_name() if rank_id is None else \ + f"rank {rank_id} ".capitalize() + checker.get_name() if i == 0: # the first row is header - self.result.add_detail(checker.get_name(), headers=detail) + self.result.add_detail(sheet_name, headers=detail) else: - self.result.add_detail(checker.get_name(), detail=detail) + self.result.add_detail(sheet_name, detail=detail) # add tune op list tune_op_list = checker.get_tune_op_list() if tune_op_list: @@ -55,11 +64,13 @@ class ProfilingAnalyzer(BaseAnalyzer, ABC): return self.result - def make_record(self): - pass + def get_priority(self, checker): + if "aicpu" not in checker.__class__.__name__.lower(): + return PriorityBackgroundColor.low - def make_render(self): - pass + aicpu_duration = getattr(checker, "aicpu_task_duration", 0.0) + total_duration = getattr(checker, "total_task_duration", 0.0) + return self.get_priority_by_time_ratio(aicpu_duration, total_duration) class DynamicShapeAnalyzer(ProfilingAnalyzer): diff --git a/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py b/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py index 291c3a1f9..3d1a537c2 100644 --- a/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py +++ b/profiler/advisor/analyzer/dataloader/dataloader_analyzer.py @@ -5,26 +5,30 @@ from typing import List, Dict, Any from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.dataloader.dataloader_checker import DataloaderChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() class DataloaderAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.dataset = self.get_first_data_by_key(self.dataset_list, key) self.result = OptimizeResult() self.html_render = HTMLRender() - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): dataloader_checker = DataloaderChecker() dataloader_checker.check_slow_dataloader(self.dataset) dataloader_checker.make_record(self.result) - dataloader_checker.make_render(self.html_render) + dataloader_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + def get_priority(self): + return PriorityBackgroundColor.high diff --git a/profiler/advisor/analyzer/dataloader/dataloader_checker.py b/profiler/advisor/analyzer/dataloader/dataloader_checker.py index eb1886284..f392a0838 100644 --- a/profiler/advisor/analyzer/dataloader/dataloader_checker.py +++ b/profiler/advisor/analyzer/dataloader/dataloader_checker.py @@ -3,7 +3,7 @@ import re import logging import yaml -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.cluster_analyse.common_func.file_manager import FileManager @@ -22,7 +22,7 @@ class DataloaderChecker: self.dataloader_duration_threshold = None self._init_rule() - def check_slow_dataloader(self, event_dataset: TimelineEventDataset): + def check_slow_dataloader(self, event_dataset: ScheduleAnalysisDataset): """ :Param event_dataset: dataset of timeline event """ @@ -32,7 +32,7 @@ class DataloaderChecker: return for event in event_dataset.dataloader: - dataloader_duration = float(event.dur) / 1000 + dataloader_duration = float(event.dur) if dataloader_duration < self.dataloader_duration_threshold: continue self.desc = self.desc.format(dataloader_duration=dataloader_duration, @@ -53,14 +53,16 @@ class DataloaderChecker: for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.dataloader_issues: return + priority = kwargs.get("priority") html_render.render_template(key="dataloader", template_dir="templates", template_name="slow_dataloader.html", desc=self.desc, - suggestions=self.suggestions) + suggestions=self.suggestions, + priority_background_color=priority) def _init_rule(self): dataloader_rule_path = os.path.join( diff --git a/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py b/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py index 326be83b8..e9dcd263d 100644 --- a/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py +++ b/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py @@ -20,17 +20,22 @@ class FusionOPAnalyzer(BaseAnalyzer): super(FusionOPAnalyzer, self).__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - + self.html = None + @BaseAnalyzer.check_data((GraphDataset.get_key(),)) def optimize(self, **kwargs): """ :return: result """ - self._check(self.dataset_list.get("GraphDataset"), self.dataset_list.get("ProfilingDataset")) + self._check(self.dataset_list.get("GraphDataset"), self.dataset_list.get("ProfilingDataset"), + kwargs.get("add_render_list")) return self.result - def _check(self, graph_data: List[GraphDataset], - profiling_data: List[ProfilingDataset] = None) -> None: + def get_priority(self): + pass + + def _check(self, graph_data: List[GraphDataset], profiling_data: List[ProfilingDataset] = None, + add_render_list=True) -> None: if len(graph_data) == 0 or graph_data[0].is_empty(): return for _, rule in self.RULES.items(): @@ -40,10 +45,4 @@ class FusionOPAnalyzer(BaseAnalyzer): else: checker.find_fusion_matched_issues_with_times(graph_data, profiling_data) checker.make_record(self.result) - checker.make_render(self.html_render) - - def make_record(self): - pass - - def make_render(self): - pass + self.html = checker.make_render(self.html_render, add_render_list) \ No newline at end of file diff --git a/profiler/advisor/analyzer/memory/__init__.py b/profiler/advisor/analyzer/memory/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/memory/memory_analyzer.py b/profiler/advisor/analyzer/memory/memory_analyzer.py new file mode 100644 index 000000000..cd7b0a242 --- /dev/null +++ b/profiler/advisor/analyzer/memory/memory_analyzer.py @@ -0,0 +1,38 @@ +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.analyzer.memory.memory_checker import MemoryOpsChecker +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor + +logger = logging.getLogger() + + +class MemoryAnalyzer(BaseAnalyzer): + dataset_cls_list = [ScheduleAnalysisDataset] + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + key = ScheduleAnalysisDataset.get_key() + self.dataset = self.get_first_data_by_key(self.dataset_list, key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) + def optimize(self, **kwargs): + memory_checker = MemoryOpsChecker() + memory_checker.check_memory_ops(self.dataset) + memory_checker.make_record(self.result) + memory_checker.make_render(self.html_render, priority=self.get_priority(memory_checker.max_mem_op_dur)) + return self.result + + def get_priority(self, max_mem_op_dur): + step_duration = getattr(self.dataset, "step_duration", None) + ratio = self.get_priority_by_time_ratio(max_mem_op_dur, step_duration) + + if step_duration is None: + return PriorityBackgroundColor.low + + return ratio diff --git a/profiler/advisor/analyzer/memory/memory_checker.py b/profiler/advisor/analyzer/memory/memory_checker.py new file mode 100644 index 000000000..8dec295c9 --- /dev/null +++ b/profiler/advisor/analyzer/memory/memory_checker.py @@ -0,0 +1,76 @@ +import os +import re +import logging +import yaml + +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset, MemCollector +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager + +logger = logging.getLogger() + + +class MemoryOpsChecker: + + def __init__(self): + + self.memory_issues = False + self.optimization_item = [] + self.desc = "" + self.suggestions = [] + self.memory_ops_duration_threshold = None + self.max_mem_op_dur = 0 + + def check_memory_ops(self, event_dataset: ScheduleAnalysisDataset): + """ + :Param event_dataset: dataset of timeline event + """ + if not hasattr(event_dataset, "memory_ops") or not getattr(event_dataset, "memory_ops") or \ + not event_dataset.memory_ops.mem_op_info: + logger.debug("Skip slow memory ops checker, because no memory ops: %s", MemCollector.MEMORY_OP_NAME) + return + + rule = event_dataset.memory_ops.rule + max_dur_thres = rule.get("max_total_duration") + raw_problem = rule.get("problem") + + for memory_op_name, memory_op_info in event_dataset.memory_ops.mem_op_info.items(): + op_dur = memory_op_info.get("total_dur") + op_count = memory_op_info.get("count") + if op_dur < max_dur_thres: + continue + if op_dur > self.max_mem_op_dur: + self.max_mem_op_dur = op_dur + + self.memory_issues = True + self.desc += raw_problem.format(memory_op_num=op_count, memory_op_name=memory_op_name, + memory_op_dur=op_dur) + " " + for solution in rule.get("solutions", []): + if memory_op_name not in solution: + continue + suggestion = solution.get(memory_op_name, {}).get("desc") + + self.suggestions.append(f"{suggestion} for optimize memory operator {memory_op_name}") + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + if not self.memory_issues: + return + + self.optimization_item.append(OptimizeItem("Memory", self.desc, self.suggestions)) + for optimization in self.optimization_item: + result.add(OptimizeRecord(optimization)) + + def make_render(self, html_render, **kwargs): + if not self.memory_issues: + return + priority = kwargs.get("priority") + html_render.render_template(key="memory", + template_dir="templates", + template_name="memory.html", + desc=self.desc, + suggestions=self.suggestions, + priority_background_color=priority) diff --git a/profiler/advisor/analyzer/overall/environment_variable_analyzer.py b/profiler/advisor/analyzer/overall/environment_variable_analyzer.py index 3daaa3460..c4468c36d 100644 --- a/profiler/advisor/analyzer/overall/environment_variable_analyzer.py +++ b/profiler/advisor/analyzer/overall/environment_variable_analyzer.py @@ -18,6 +18,7 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.prof_common.path_manager import PathManager from profiler.advisor.dataset.environment_variable_dataset import EnvironmentVariableDataset from profiler.advisor.analyzer.overall.environment_variable_checker import EnvironmentVariabelChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor class EnvironmentVariabelAnalyzer(BaseAnalyzer): @@ -40,6 +41,9 @@ class EnvironmentVariabelAnalyzer(BaseAnalyzer): checker.make_render(self.html_render) return self.result + def get_priority(self): + return PriorityBackgroundColor.high + def make_record(self): pass diff --git a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py index 8e93dbda7..fe43072a8 100644 --- a/profiler/advisor/analyzer/overall/overall_summary_analyzer.py +++ b/profiler/advisor/analyzer/overall/overall_summary_analyzer.py @@ -23,7 +23,7 @@ from profiler.compare_tools.compare_interface.comparison_interface import Compar class OverallSummaryAnalyzer(BaseAnalyzer): - OVERALL_SUMMARY_ANALYZER = "overall_summary_analysis" + OVERALL_SUMMARY_ANALYZER = "overall summary" advice_map = { "Computing Time": "if you want more detailed advice please go to mstt_advisor_*.html", "Uncovered Communication Time": "if you want more detailed advice please go to mstt_advisor_*.html", @@ -233,6 +233,9 @@ class OverallSummaryAnalyzer(BaseAnalyzer): torch_version=self.torch_version, result=result_for_html) + def get_priority(self): + pass + def get_profile_path(collection_path): for root, dirs, files in os.walk(collection_path): diff --git a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py index 0e62a3ff0..58b2c301b 100644 --- a/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py +++ b/profiler/advisor/analyzer/schedule/dispatch/timeline_op_dispatch_analyzer.py @@ -16,26 +16,26 @@ # limitations under the License. import logging - from profiler.advisor.common import constant as const from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.result.result import OptimizeResult from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() class OpDispatchAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] """ operator dispatch optimizer """ def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: super().__init__(collection_path, n_processes, **kwargs) - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.dataset = self.get_first_data_by_key(self.dataset_list, key) self.result = OptimizeResult() self.html_render = HTMLRender() @@ -54,21 +54,21 @@ class OpDispatchAnalyzer(BaseAnalyzer): self.make_render(self.html_render) return self.result - def get_op_compile_info(self, event_dataset: TimelineEventDataset): - """ - :Param event_dataset: dataset of timeline event - """ - if hasattr(event_dataset, "ops_compile"): - self._op_compile = getattr(event_dataset, "ops_compile") - if not self._op_compile or self._op_compile.total_count < const.MAX_OP_COMPILE_NUM: - return + def get_op_compile_info(self, event_dataset: ScheduleAnalysisDataset): + """ + :Param event_dataset: dataset of timeline event + """ + if hasattr(event_dataset, "ops_compile"): + self._op_compile = getattr(event_dataset, "ops_compile") + if not self._op_compile or self._op_compile.total_count < const.MAX_OP_COMPILE_NUM: + return - self._issues_record.append(['operator dispatch', - const.OP_COMPILE_ID, - self._op_compile.total_count, - self._op_compile.total_time]) - else: - logger.debug("Skip operator compile checker, because no op_compile attr find.") + self._issues_record.append(['operator dispatch', + const.OP_COMPILE_ID, + self._op_compile.total_count, + self._op_compile.total_time]) + else: + logger.debug("Skip operator compile checker, because no op_compile attr find.") def make_record(self, result: OptimizeResult): """ @@ -77,8 +77,9 @@ class OpDispatchAnalyzer(BaseAnalyzer): if not self._op_compile or len(self._issues_record) <= 0: return desc = f"Found {self._op_compile.total_count} operator compile issues." - suggestion = (f"Please use `torch_npu.npu.set_compile_mode(jit_compile=False)` to disable jit compile " - f"in dynamic shape usage.") + suggestion = ("Please place the following code at the entrance of the python script to disable jit compile. " \ + "Code: `torch_npu.npu.set_compile_mode(jit_compile=False); " + "torch_npu.npu.config.allow_internal_format = False`") self.optimization_item.append(OptimizeItem("Operator dispatch", desc, [suggestion])) for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) @@ -87,7 +88,7 @@ class OpDispatchAnalyzer(BaseAnalyzer): for op_info in self._issues_record: result.add_detail('operator dispatch', detail=op_info) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): issues = [] optimizations = [] for optimization in self.optimization_item: @@ -97,11 +98,20 @@ class OpDispatchAnalyzer(BaseAnalyzer): )) for record in self._issues_record: issues.append(dict(issue=record[0], - op_name=record[1], - counts=record[2], - total_time=record[3])) + op_name=record[1], + counts=record[2], + total_time=record[3])) html_render.render_template(key="schedule", template_dir="templates", template_name="operator_dispatch.html", issues=issues, - optimizers=optimizations) + optimizers=optimizations, + priority_background_color=self.get_priority()) + + def get_priority(self): + step_duration = getattr(self.dataset, "step_duration", None) + op_compile_total_dur = getattr(self._op_compile, "total_time", None) + if step_duration is None or op_compile_total_dur is None: + return PriorityBackgroundColor.low + + return self.get_priority_by_time_ratio(op_compile_total_dur, step_duration) diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index c1eb24b8e..305d23994 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -8,25 +8,29 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.common import constant as const from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import format_timeline_result from profiler.advisor.common.timeline.fusion_ops_db import init_timeline_ops_db +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() class TimelineFusionOpsAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, n_processes: int = 1, **kwargs): super().__init__(collection_path, n_processes, **kwargs) self._matched_op_index = {} if self.n_processes <= 1 else multiprocessing.Manager().dict() self.matched_op_stacks = {} self.empty_stacks = True - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) + def get_priority(self): + return PriorityBackgroundColor.low + def optimize(self, **kwargs): for mode in [const.ATEN.lower(), const.OPTIMIZER.lower()]: @@ -154,8 +158,9 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): timeline_profiling_doc_url=const.TIMELINE_WITH_STACK_DOC_URL ) + sheet_name = "Affinity apis" optimization_item = OptimizeItem( - SupportedScopes.TIMELINE_FUSION_OPS, + sheet_name, desc, [suggestion] ) @@ -163,16 +168,16 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): self.result.add(OptimizeRecord(optimization_item)) record_title = ["Affinity API", "Code stacks", "Stack called counts"] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, headers=record_title) + self.result.add_detail(sheet_name, headers=record_title) for api_name, stacks_info in format_timeline_result(self.matched_op_stacks).items(): if not stacks_info: detail = [api_name, "null", "null"] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, detail=detail) + self.result.add_detail(sheet_name, detail=detail) else: for stack in stacks_info: detail = [api_name, *stack] - self.result.add_detail(SupportedScopes.TIMELINE_FUSION_OPS, detail=detail) + self.result.add_detail(sheet_name, detail=detail) def make_render(self): format_result_for_html = format_timeline_result(dict(self.matched_op_stacks), dump_html=True) @@ -185,7 +190,8 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): empty_stacks=self.empty_stacks, with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, api_doc_url=const.TIMELINE_API_DOC_URL, - result=format_result_for_html) + result=format_result_for_html, + priority_background_color=self.get_priority()) def query_stack(self, event_dataset): if all([len(matched_index) == 0 for matched_index in self._matched_op_index.values()]): diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py b/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py index f684a4892..92425910b 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/timeline_api_stack_checker.py @@ -3,7 +3,7 @@ from typing import List from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ComputationAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import get_analyze_processes, ParallelJob @@ -21,7 +21,8 @@ class OpStackFinder: self.task_type = None self.matched_index = set() - def get_api_stack_by_op(self, event_dataset: TimelineEventDataset, op_name: List[str] = None, task_type: str = None, + def get_api_stack_by_op(self, event_dataset: ComputationAnalysisDataset, op_name: List[str] = None, + task_type: str = None, disable_multiprocess=False): """ :Param event_dataset: dataset of timeline event @@ -82,7 +83,13 @@ class OpStackFinder: for op_info in self._stack_record: result.add_detail('operator stacks', detail=op_info) - def _get_api_stack_by_op(self, event_dataset: TimelineEventDataset, op_name: str, task_type: str): + def query_stack(self, event_dataset: ComputationAnalysisDataset): + + if not event_dataset.dataset_len: + return + _ = event_dataset.parse_data_with_generator(self._query_stack_by_matched_index) + + def _get_api_stack_by_op(self, event_dataset: ComputationAnalysisDataset, op_name: str, task_type: str): for _, src_op_event in event_dataset.ops_with_task_type.items(): op_task_type = src_op_event.get(const.TASK_TYPE) @@ -110,6 +117,7 @@ class OpStackFinder: task_id = src_op_event.task_id if not task_id: continue + self.matched_index.add(dst_op_index) if dst_op_index not in self._task_id_record: self._task_id_record[dst_op_index] = [] @@ -122,7 +130,7 @@ class OpStackFinder: if not dst_op_event: return const.TIMELINE_BACKWARD_NO_STACK_CODE - return dst_op_event.get("dataset_index") + return int(dst_op_event.get("dataset_index")) def _query_index_by_acl_to_npu(self, acl_to_npu_event): if acl_to_npu_event: @@ -148,6 +156,7 @@ class OpStackFinder: return None event = TimelineEvent(event) stack = event.args.get(const.CALL_STACKS) + stack = stack if stack else const.NO_STACK_REASON_MAP.get(const.TIMELINE_BACKWARD_NO_STACK_CODE) for matched_op_info in self._task_id_record.get(index, []): self._stack_record.append([*matched_op_info, stack]) @@ -156,8 +165,3 @@ class OpStackFinder: self._stack_record.append([*matched_op_info, const.NO_STACK_REASON_MAP.get(const.TIMELINE_ACL_TO_NPU_NO_STACK_CODE)]) return None - - def query_stack(self, event_dataset: TimelineEventDataset): - if not event_dataset.dataset_len: - return - _ = event_dataset.parse_data_with_generator(self._query_stack_by_matched_index) diff --git a/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py b/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py index 432179302..3d142819d 100644 --- a/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py +++ b/profiler/advisor/analyzer/schedule/gc/gc_analyzer.py @@ -1,4 +1,3 @@ -# Copyright (c) 2024, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -18,25 +17,29 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.schedule.gc.gc_checker import GcChecker from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor logger = logging.getLogger() class GcAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, **kwargs): super().__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): gc_checker = GcChecker() gc_checker.check_gc(self.timeline_event_dataset, rank_id=kwargs.get("rank_id"), stage=kwargs.get("stage")) gc_checker.make_record(self.result) - gc_checker.make_render(self.html_render) + gc_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + def get_priority(self): + return PriorityBackgroundColor.medium diff --git a/profiler/advisor/analyzer/schedule/gc/gc_checker.py b/profiler/advisor/analyzer/schedule/gc/gc_checker.py index 05ef28760..1fbddf655 100644 --- a/profiler/advisor/analyzer/schedule/gc/gc_checker.py +++ b/profiler/advisor/analyzer/schedule/gc/gc_checker.py @@ -15,7 +15,7 @@ import logging import os -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.cluster_analyse.common_func.file_manager import FileManager @@ -42,7 +42,7 @@ class GcChecker: self.headers = ["timestamp", "duration(us)"] self._init_rule() - def check_gc(self, event_dataset: TimelineEventDataset, rank_id=None, stage=None): + def check_gc(self, event_dataset: ScheduleAnalysisDataset, rank_id=None, stage=None): """ :Param event_dataset: dataset of timeline event """ @@ -81,9 +81,10 @@ class GcChecker: row = [self.rank_id] + row result.add_detail(sub_table_name, detail=row) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.gc_issues: return + priority = kwargs.get("priority") show_num = min(self.gc_topk_num, self.abnormal_gc_count) html_render.render_template(key="schedule", template_dir="templates", @@ -92,7 +93,8 @@ class GcChecker: solutions=self.solutions, headers=self.headers, datas=self.abnormal_gc_list[:show_num], - num=show_num) + num=show_num, + priority_background_color=priority) def _init_rule(self): gc_rule_path = os.path.join( diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py index 2786a7840..df8c22fa5 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_analyzer.py @@ -1,30 +1,32 @@ import logging -from typing import List, Dict, Any - from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.schedule.syncbn.syncbn_checker import SyncBNChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() class SyncBNAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, **kwargs): super().__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): syncbn_checker = SyncBNChecker() syncbn_checker.check_syncbn(self.timeline_event_dataset) syncbn_checker.make_record(self.result) - syncbn_checker.make_render(self.html_render) + syncbn_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + def get_priority(self): + return PriorityBackgroundColor.high \ No newline at end of file diff --git a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py index c0e10448f..e83a15491 100644 --- a/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py +++ b/profiler/advisor/analyzer/schedule/syncbn/syncbn_checker.py @@ -1,7 +1,7 @@ import logging import os -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.cluster_analyse.common_func.file_manager import FileManager @@ -20,7 +20,7 @@ class SyncBNChecker: self.max_syncbn_num = None self._init_rule() - def check_syncbn(self, event_dataset: TimelineEventDataset): + def check_syncbn(self, event_dataset: ScheduleAnalysisDataset): """ :Param event_dataset: dataset of timeline event """ @@ -43,14 +43,17 @@ class SyncBNChecker: for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.syncbn_issues: return + + priority = kwargs.get("priority") html_render.render_template(key="schedule", template_dir="templates", template_name="sync_batchnorm.html", desc=self.desc, - solutions=self.solutions) + solutions=self.solutions, + priority_background_color=priority) def _init_rule(self): syncbn_rule_path = os.path.join( diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py index d8906504c..61ec7d1fa 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_analyzer.py @@ -5,28 +5,33 @@ from typing import List, Dict, Any from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_checker import SynchronizeStreamChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.display.html.render import HTMLRender -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset logger = logging.getLogger() class SynchronizeStreamAnalyzer(BaseAnalyzer): - dataset_cls_list = [TimelineEventDataset] + dataset_cls_list = [ScheduleAnalysisDataset] def __init__(self, collection_path, **kwargs): super().__init__(collection_path, **kwargs) self.result = OptimizeResult() self.html_render = HTMLRender() - key = TimelineEventDataset.get_key() + key = ScheduleAnalysisDataset.get_key() self.timeline_event_dataset = self.get_first_data_by_key(self.dataset_list, key) - @BaseAnalyzer.check_data((TimelineEventDataset.get_key(),)) + @BaseAnalyzer.check_data((ScheduleAnalysisDataset.get_key(),)) def optimize(self, **kwargs): synchronize_stream_checker = SynchronizeStreamChecker() synchronize_stream_checker.check_synchronize(self.timeline_event_dataset, kwargs.get("profiling_with_stack")) synchronize_stream_checker.make_record(self.result) - synchronize_stream_checker.make_render(self.html_render) + synchronize_stream_checker.make_render(self.html_render, priority=self.get_priority()) return self.result + + + def get_priority(self): + return PriorityBackgroundColor.low \ No newline at end of file diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py index 83ddd80a0..7af46f766 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py @@ -1,7 +1,7 @@ import logging from profiler.advisor.common import constant as const -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.analyzer.schedule.timeline_base_checker import TimelineBaseChecker @@ -21,7 +21,7 @@ class SynchronizeStreamChecker(TimelineBaseChecker): self.solutions = [] self.max_synchronize_num = None - def check_synchronize(self, event_dataset: TimelineEventDataset, profiling_with_stack=None): + def check_synchronize(self, event_dataset: ScheduleAnalysisDataset, profiling_with_stack=None): """ :Param event_dataset: dataset of timeline event """ @@ -73,10 +73,10 @@ class SynchronizeStreamChecker(TimelineBaseChecker): for optimization in self.optimization_item: result.add(OptimizeRecord(optimization)) - def make_render(self, html_render): + def make_render(self, html_render, **kwargs): if not self.synchronize_issues: return - + priority = kwargs.get("priority") format_result_for_html = format_timeline_result(dict(self.matched_op_stacks), dump_html=True) html_render.render_template(key="schedule", template_dir="templates", @@ -86,4 +86,5 @@ class SynchronizeStreamChecker(TimelineBaseChecker): result=format_result_for_html, with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, empty_stacks=self.empty_stacks, - framework_black_list=self.framework_black_list) + framework_black_list=self.framework_black_list, + priority_background_color=priority) diff --git a/profiler/advisor/analyzer/schedule/timeline_base_checker.py b/profiler/advisor/analyzer/schedule/timeline_base_checker.py index 8bc691502..f481733d4 100644 --- a/profiler/advisor/analyzer/schedule/timeline_base_checker.py +++ b/profiler/advisor/analyzer/schedule/timeline_base_checker.py @@ -4,7 +4,7 @@ import logging from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult logger = logging.getLogger() @@ -19,19 +19,11 @@ class TimelineBaseChecker(ABC): self.empty_stacks = True self.framework_black_list = False - @abstractmethod - def make_record(self, result: OptimizeResult): - pass - - @abstractmethod - def make_render(self, html_render): - pass - - def query_stack(self, event_dataset: TimelineEventDataset = None, profiling_with_stack: str = None): + def query_stack(self, event_dataset: ScheduleAnalysisDataset = None, profiling_with_stack: str = None): if all([len(matched_index) == 0 for matched_index in self._matched_op_index.values()]): return - event_dataset = event_dataset if not profiling_with_stack else TimelineEventDataset( + event_dataset = event_dataset if not profiling_with_stack else ScheduleAnalysisDataset( collection_path=profiling_with_stack, data={}, _datasets={}, analysis_mode="fusion_ops", build_dataset=False) diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 652e10b08..72b8dd3df 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -33,4 +33,6 @@ class SupportedScopes: SYNCBN = "syncbn" SYNCHRONIZE_STREAM = "synchronize_stream" FREQ_ANALYSIS = "freq_analysis" + MEMORY = "memory" + STAGE_COMPUTE = "stage_compute" GC_ANALYSIS = "gc_analysis" diff --git a/profiler/advisor/common/async_analysis_status.py b/profiler/advisor/common/async_analysis_status.py new file mode 100644 index 000000000..f67ca235a --- /dev/null +++ b/profiler/advisor/common/async_analysis_status.py @@ -0,0 +1,7 @@ +class AsyncAnalysisStatus: + FAILED = "failed" + SUCCESS = "success" + ANALYZING = "analyzing" + + FAILED_STATUS_CODE = 400 + NON_FAILED_STATUS_CODE = 200 diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index c97cfbfd1..298e94fc1 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -33,6 +33,7 @@ TASK_TYPE = "Task Type" CPU_OP = "cpu_op" AI_CORE = "AI_CORE" AI_CPU = "AI_CPU" +MIX_AIC = "MIX_AIC" CALL_STACKS = "Call stack" INPUT_DIMS = "Input Dims" OP_SEP = "-" @@ -48,8 +49,7 @@ NO_STACK_REASON_MAP = { TIMELINE_BACKWARD_NO_STACK_CODE: "Backward broadcast, without call stacks in profiling.", TIMELINE_ACL_TO_NPU_NO_STACK_CODE: "Incoming flow is 'acl_to_npu', without call stacks in profiling." } -TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/"\ - "Samples%20of%20Fused%20Operator%20API%20Replacement.md" +TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" AFFINITY_TRAINING_API = "Affinity training api" TIMELINE_WITH_STACK_DOC_URL = "https://www.hiascend.com/document/detail/zh/canncommercial/" \ "70RC1/modeldevpt/ptmigr/AImpug_0067.html" @@ -124,20 +124,6 @@ MAX_RETRIES = 3 TIMEOUT = 3 ADVISOR_RULE_PATH = "ADVISOR_RULE_PATH" -# Copyright (c) 2024, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. CLOUD_RULE_PATH = "rules/cloud/" DEFAULT_RULE_PATH = "./rules/" @@ -156,7 +142,17 @@ COMMUNICATION_JSON = "communication.json" BOTTLENECK = "bottleneck" DATA = "data" - +ADVISOR_ANALYSIS_OUTPUT_DIR = "advisor_analysis_result" +DEFAULT_PROCESSES = 8 +CLUSTER_ANALYSIS_FILE_PATTERN = [r'profiler_info_\d+\.json', "step_trace_time.csv", "communication.json", + "communication_matrix.json"] +ANALYSIS_OUTPUT_PATH = "ANALYSIS_OUTPUT_PATH" +DEFAULT_RANK_FOR_PROFILING_ANALYSIS = 0 +PROFILER_INFO_FILE_PATTERN = r"profiler_info_(\d+)\.json" +DISABLE_STREAMINIG_READER = "DISABLE_STREAMINIG_READER" FRAMEWORK_STACK_BLACK_LIST = ["torch", "torch_npu", "megatron", "deepspeed"] DISABLE_STREAMING_READER = "DISABLE_STREAMING_READER" -MAX_FILE_SIZE = 10**10 +MAX_FILE_SIZE = 10 ** 10 +MAX_NUM_PROCESSES = 4 +DEFAULT_STEP = "-1" +STEP_RANK_SEP = "_" diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index 445d4c87e..e268b4092 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -15,6 +15,7 @@ import logging import os +import re from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.utils.utils import singleton @@ -81,9 +82,11 @@ class ClusterDataset(Dataset): @singleton class ClusterStepTraceTimeDataset(ClusterDataset): RANK = "rank" + STAGE = "stage" def __init__(self, collection_path: str, data: dict, **kwargs): self._step_dict = defaultdict() + self._stages = [] super().__init__(collection_path, data, **kwargs) def _parse(self): @@ -101,14 +104,31 @@ class ClusterStepTraceTimeDataset(ClusterDataset): step_dict = defaultdict(lambda: [0, 0, 0]) for step_bean in step_data: if step_bean.type == self.RANK: - step_dict[step_bean.index][0] += step_bean.compute - step_dict[step_bean.index][1] += step_bean.communication - step_dict[step_bean.index][2] += step_bean.free + step_rank_record = [] + step = str(step_bean.step).replace(" ", "") or str(const.DEFAULT_STEP) + rank = str(step_bean.index).replace(" ", "") + if step: + step_rank_record.append(step) + if rank: + step_rank_record.append(rank) + + step_rank_index = const.STEP_RANK_SEP.join(step_rank_record) + step_dict[step_rank_index][0] += step_bean.compute + step_dict[step_rank_index][1] += step_bean.communication + step_dict[step_rank_index][2] += step_bean.free + if step_bean.type == self.STAGE: + stage = sorted(list(map(int, re.findall(r'\d+', step_bean.stage)))) + if stage in self._stages: + continue + self._stages.append(stage) return step_dict def get_data(self): return self._step_dict + def get_stages(self): + return sorted(self._stages) + @singleton class ClusterCommunicationDataset(ClusterDataset): @@ -158,7 +178,7 @@ class ClusterCommunicationDataset(ClusterDataset): self.hccl_dict.setdefault(comm_group, defaultdict(lambda: defaultdict(list))) for step, step_dict in group_dict.items(): for op, op_dict in step_dict.items(): - self.compute_bandwidth(op_dict) + self.compute_bandwidth(step.lower().lstrip("step") or str(const.DEFAULT_STEP), op_dict) self.process_hccl_info(comm_group, step, op, op_dict) def process_hccl_info(self, group, step, op, op_dict): @@ -175,7 +195,7 @@ class ClusterCommunicationDataset(ClusterDataset): msg = "[ERROR] Cluster_communication.json has invalid structure." raise ValueError(msg) from e - def compute_bandwidth(self, op_dict: dict): + def compute_bandwidth(self, step, op_dict: dict): for rank_id, rank_dict in op_dict.items(): try: rank = int(rank_id) @@ -184,17 +204,17 @@ class ClusterCommunicationDataset(ClusterDataset): raise ValueError(msg) from e for comm_type, bw_dict in rank_dict.get(self.COMMUNICATION_BANDWIDTH_INFO, {}).items(): if comm_type == self.SDMA: - self.rank_bw_dict[rank][self.SDMA_SIZE_MB] += bw_dict.get(self.TRANSIT_SIZE) - self.rank_bw_dict[rank][self.SDMA_TIME_MS] += bw_dict.get(self.TRANSIT_TIME) + self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.SDMA_SIZE_MB] += bw_dict.get(self.TRANSIT_SIZE) + self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.SDMA_TIME_MS] += bw_dict.get(self.TRANSIT_TIME) if comm_type == self.RDMA: - self.rank_bw_dict[rank][self.RDMA_SIZE_MB] += bw_dict.get(self.TRANSIT_SIZE) - self.rank_bw_dict[rank][self.RDMA_TIME_MS] += bw_dict.get(self.TRANSIT_TIME) - - for rank, rank_dict in self.rank_bw_dict.items(): - self.rank_bw_dict[rank][self.RDMA_BANDWIDTH] = self.compute_ratio( - self.rank_bw_dict[rank][self.RDMA_SIZE_MB], self.rank_bw_dict[rank][self.RDMA_TIME_MS]) - self.rank_bw_dict[rank][self.SDMA_BANDWIDTH] = self.compute_ratio( - self.rank_bw_dict[rank][self.SDMA_SIZE_MB], self.rank_bw_dict[rank][self.SDMA_TIME_MS]) + self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.RDMA_SIZE_MB] += bw_dict.get(self.TRANSIT_SIZE) + self.rank_bw_dict[f"{step}{const.STEP_RANK_SEP}{rank}"][self.RDMA_TIME_MS] += bw_dict.get(self.TRANSIT_TIME) + + for step_rank in self.rank_bw_dict.keys(): + self.rank_bw_dict[step_rank][self.RDMA_BANDWIDTH] = self.compute_ratio( + self.rank_bw_dict[step_rank][self.RDMA_SIZE_MB], self.rank_bw_dict[step_rank][self.RDMA_TIME_MS]) + self.rank_bw_dict[step_rank][self.SDMA_BANDWIDTH] = self.compute_ratio( + self.rank_bw_dict[step_rank][self.SDMA_SIZE_MB], self.rank_bw_dict[step_rank][self.SDMA_TIME_MS]) def get_data(self): return self.rank_bw_dict diff --git a/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py b/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py index b108fc77a..8ae0e55f2 100644 --- a/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py +++ b/profiler/advisor/dataset/cluster/cluster_step_trace_time_bean.py @@ -65,3 +65,6 @@ class ClusterStepTraceTimeBean: msg = "[ERROR] Cluster step trace time.csv has invalid value in column 'Free'." raise ValueError(msg) from e + @property + def stage(self) -> int: + return self._data.get(self.INDEX) diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/advisor/dataset/timeline_event_dataset.py index c76314641..44bbc141d 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/advisor/dataset/timeline_event_dataset.py @@ -1,101 +1,47 @@ +import inspect import logging -import os -from typing import List, Any import traceback +from collections import OrderedDict import ijson from tqdm import tqdm from profiler.advisor.common import constant as const from profiler.advisor.common.timeline.event import TimelineEvent -from profiler.advisor.utils.utils import get_file_path_from_directory, check_path_valid, singleton -from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.utils.utils import get_file_path_from_directory, check_path_valid, singleton, convert_to_float +from profiler.advisor.dataset.timeline_op_collector.timeline_op_collector import ( + OpCompileCollector, + SynchronizeStreamCollector, + MemCollector, + DataloaderCollector, + SyncBNCollector, + AtenCollector, + OptimizerCollector, + FrequencyCollector, + SpecificTaskTypeOpCollector, + TorchToNpuCollector, + AclToNpuCollector, + OpStackCollector, + StepCollector, + GcCollector +) logger = logging.getLogger() -class OpCompileCollector: - def __init__(self): - self._total_op_compile_counter = 0 - self._total_op_compile_time = 0.0 +class BaseTimelineEventDataset: + PROFILER_STEP_PREFIX = "ProfilerStep" - @property - def total_time(self): - return self._total_op_compile_time - - @property - def total_count(self): - return self._total_op_compile_counter - - def is_empty(self): - return self._total_op_compile_counter == 0 - - def update(self, event: TimelineEvent): - self._total_op_compile_time += float(event.dur) - self._total_op_compile_counter += 1 - - def unset(self): - self._total_op_compile_counter = 0 - self._total_op_compile_time = 0.0 - - -class SynchronizeStreamCollector: - - def __init__(self): - self._synchronize_stream_count = 0 - self._slow_synchronize_stream = [] - self.rule = SynchronizeStreamCollector._load_rule() - - @property - def total_count(self): - return self._synchronize_stream_count - - @property - def slow_synchronize_stream(self): - return self._slow_synchronize_stream - - @staticmethod - def _load_rule(): - sync_stream_rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "rules", - "synchronize.yaml") - - sync_stream_rule = FileManager.read_yaml_file(sync_stream_rule_path) - return sync_stream_rule - - def update_sync_stream_count(self): - self._synchronize_stream_count += 1 - - def append_slow_sync_stream(self, event): - if float(event.dur) / 1000 >= self.rule.get("slow_synchronize_threshold", 10): - self._slow_synchronize_stream.append(event) - - def unset(self): - self._synchronize_stream_count = 0 - self._slow_synchronize_stream = [] - - -@singleton -class TimelineEventDataset: + collector_map = {} def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: - self._ops_with_task_type = {} - self._ops_with_stack = {} - self._ops_compile = OpCompileCollector() - self._torch_to_npu = {} - self._acl_to_npu = set() - self._aten: List[Any] = [] - self._optimizer: List[Any] = [] - self._dataloader: List[Any] = [] - self._sync_batchnorm: List[Any] = [] - self._gc: List[Any] = [] - self._synchronize_stream = SynchronizeStreamCollector() self.timeline_dir = collection_path + self.profiler_step = [] self.timeline_data_list = get_file_path_from_directory(collection_path, lambda file: file.endswith("trace_view.json")) self.dataset_len = None - self.analysis_mode = kwargs.get("analysis_mode") - self.task_type = kwargs.get("task_type") - + self.step = kwargs.get("step") + self.step_duration = None if not build_dataset: return @@ -105,59 +51,6 @@ class TimelineEventDataset: data[key] = [] data[key].append(self) - if self.analysis_mode in ["op_stack", "all"]: - self._task_op_names = list(set([event_key.split("-")[0] for event_key in self._ops_with_task_type.keys()])) - - self._post_process() - - @property - def ops_with_stack(self): - return self._ops_with_stack - - @property - def ops_compile(self): - return self._ops_compile - - @property - def torch_to_npu(self): - return self._torch_to_npu - - @property - def acl_to_npu(self): - return self._acl_to_npu - - @property - def ops_with_task_type(self): - return self._ops_with_task_type - - @property - def task_op_names(self): - return self._task_op_names - - @property - def optimizer(self): - return self._optimizer - - @property - def aten(self): - return self._aten - - @property - def dataloader(self): - return self._dataloader - - @property - def sync_batchnorm(self): - return self._sync_batchnorm - - @property - def gc_events(self): - return self._gc - - @property - def synchronize_stream(self): - return self._synchronize_stream - @classmethod def get_key(cls): """ @@ -166,6 +59,23 @@ class TimelineEventDataset: """ return cls.__module__.rsplit('.', maxsplit=1)[-1] + def get_post_process_kwargs(self, func_name): + kwargs = {} + if func_name == FrequencyCollector.__name__: + ops_with_task_type = getattr(self, "ops_with_task_type", {}).values() + kwargs["ai_core_ops"] = [op for op in ops_with_task_type if + op.get(const.TASK_TYPE) in [const.AI_CORE, const.MIX_AIC]] + return kwargs + + def add_event(self, index, event): + event["dataset_index"] = index + if not isinstance(event, TimelineEvent): + event = TimelineEvent(event) + + for _, collector in self.collector_map.items(): + collector.add_op(event) + return True + def parse(self): if len(self.timeline_data_list) == 0: @@ -173,10 +83,10 @@ class TimelineEventDataset: return False if len(self.timeline_data_list) > 1: - logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis.", + logger.warning("Found multiple trace_view.json in %s, load the file of device 0 for analysis .", self.timeline_dir) - result = self.parse_data_with_generator(self._add_event) + result = self.parse_data_with_generator(self.add_event) if not self.dataset_len: self.dataset_len = len(result) @@ -202,137 +112,100 @@ class TimelineEventDataset: timeline_data_path) return result - def _add_ops_with_task_type(self, event): - key = f"{event.name}-{event.ts}" - self._ops_with_task_type[key] = TimelineEvent( - { - const.TASK_TYPE: event.args.get(const.TASK_TYPE), - "task_id": event.args.get("Task Id"), - "tid": event.tid, - "name": event.name, - "ts": str(event.ts) - } - ) - - def _add_ops_with_stack(self, event): - self._ops_with_stack[str(event.ts)] = TimelineEvent({"name": event.name, "dataset_index": event.dataset_index}) - - def _add_torch_to_npu(self, event): - key = f"{event.ph}-{event.id}" - self._torch_to_npu[key] = TimelineEvent({"tid": event.tid, "ts": str(event.ts)}) - - def _add_acl_to_npu(self, event): - # op with task type equals to ai_cpu which derived from acl_to_npu do not have stacks - self._acl_to_npu.add(str(event.ts)) - - def _add_op_compile(self, event: TimelineEvent): - if event.name == const.OP_COMPILE_NAME or event.args.get("id") == const.OP_COMPILE_ID: - self._ops_compile.update(event) - - def _add_gc(self, event: TimelineEvent): - if event.get("cat") and event.get("cat").lower() == 'gc': - self._gc.append(event) - - def _add_optimizer(self, event: TimelineEvent): - self._optimizer.append(TimelineEvent({"name": event.name, "dataset_index": event.dataset_index})) - - def _add_aten(self, event: TimelineEvent): - self._aten.append(TimelineEvent({ - "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur - })) - - def _add_dataloader(self, event: TimelineEvent): - if "dataloader" in event.name.lower(): - self._dataloader.append(TimelineEvent({ - "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur, - "stack": event.args.get("Call stack") - })) - - def _add_sync_batchnorm(self, event: TimelineEvent): - if event.name.lower() == "syncbatchnorm": - self._sync_batchnorm.append(TimelineEvent({ - "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur - })) - - def _add_synchronize(self, event: TimelineEvent): - if event.name.startswith(const.SYNC_STREAM): - self._synchronize.append(TimelineEvent({ - "name": event.name, "ts": event.ts, "dur": event.dur - })) - - def _add_specific_operator(self, event): - # for analysis of operator aclOpCompile, enable jit_compILE=False - self._add_op_compile(event) - # for analysis of slow dataloader.__next__ - self._add_dataloader(event) - # for analysis of syncBatchNorm operator, prompt users to replace source code of torch_npu's syncbn - self._add_sync_batchnorm(event) - # for analysis of GcAnalyzer - self._add_gc(event) - - def _add_event(self, index, event): - event["dataset_index"] = index - if not isinstance(event, TimelineEvent): - event = TimelineEvent(event) - - self._add_specific_operator(event) - - if self.analysis_mode == "fusion_ops": - self._add_event_for_fusion_ops(event) - elif self.analysis_mode == "op_stack": - self._add_event_for_op_stack(event) + def _get_target_ops_by_step(self, op_list): + target_ops = [] + if not self.profiler_step: + return op_list + if not self.step or f"ProfilerStep#{self.step}" not in [event.name for event in self.profiler_step]: + target_ops = op_list + if self.profiler_step: + self.step_duration = convert_to_float(self.profiler_step[-1].dur) else: - self._add_event_for_fusion_ops(event) - self._add_event_for_op_stack(event) - return True - - def _add_event_for_fusion_ops(self, event): - if event.name.lower().startswith(f"{const.ATEN}{const.ATEN_SEP}") or event.name.lower().startswith( - f"{const.NPU}{const.ATEN_SEP}"): - self._add_aten(event) - return - - # 检查cann层同步操作，根据时间窗口索引到host侧的aten算子并给出堆栈 - if event.name.startswith(const.SYNC_STREAM): - self._add_aten(event) + for step_event in self.profiler_step: + if step_event.name != f"ProfilerStep#{self.step}": + continue + self.step_duration = convert_to_float(step_event.dur) + for op_event in op_list: + if step_event.ts_include(op_event): + target_ops.append(op_event) + target_ops.sort(key=lambda x: convert_to_float(x.ts)) + return target_ops + + def _collector_post_process(self): + # 按step过滤collector中的算子，并将过滤后的算子设置为当前dataset的property，与原始TimelineEventDataset的property保持一致 + for collector_name, collector in self.collector_map.items(): + logger.debug("Start post process for operator collector: %s", collector_name) + if collector.require_filter_by_step: + logger.debug("Operator Collector %s requires filter ops by step %s", collector_name, self.step) + target_op_list = self._get_target_ops_by_step(collector.op_list) + else: + logger.debug("Operator Collector %s use operators of all step for analysis", collector_name) + target_op_list = collector.op_list - if event.name.startswith(f"{const.OPTIMIZER}.{const.OPTIMIZER_STEP}{const.OPTIMIZER_SEP}"): - self._add_optimizer(event) - return + logger.debug("Source number of ops is %s, number of ops after filtered by rank is %s", + len(collector.op_list), len(target_op_list)) - def _add_event_for_op_stack(self, event): - if event.name.lower() == const.TORCH_TO_NPU: - self._add_torch_to_npu(event) - return + collector_kwargs = self.get_post_process_kwargs(collector_name) + collector.post_process(target_op_list, **collector_kwargs) + for property_name, property_value in collector.attribute_to_dataset.items(): + setattr(self, property_name, property_value) - if event.args.get(const.CALL_STACKS): - self._add_ops_with_stack(event) - return - if event.args.get(const.TASK_TYPE) and event.args.get(const.TASK_TYPE) in [const.AI_CORE, const.AI_CPU]: - self._add_ops_with_task_type(event) - return +@singleton +class ScheduleAnalysisDataset(BaseTimelineEventDataset): + collector_map = OrderedDict( + StepCollector=StepCollector(), + MemCollector=MemCollector(), + OpCompileCollector=OpCompileCollector(), + SynchronizeStreamCollector=SynchronizeStreamCollector(), + DataloaderCollector=DataloaderCollector(), + SyncBNCollector=SyncBNCollector(), + AtenCollector=AtenCollector(), + OptimizerCollector=OptimizerCollector(), + GcCollector=GcCollector() + ) - if event.name and event.ts and event.name == const.ACL_TO_NPU: - self._add_acl_to_npu(event) - return + def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: + super().__init__(collection_path, data, build_dataset, **kwargs) + self.aten = None + self.synchronize_stream = None + self._collector_post_process() + self._post_process() def _post_process(self): # eliminate sub aten operator of the first level aten operator by 'ts' and 'dur', # keep the first level aten operator contiguous formated_atens = [] - for event in sorted(self._aten, key=lambda x: x.get("ts", -1)): + if not hasattr(self, "aten") or not hasattr(self, "synchronize_stream"): + return + + for event in sorted(self.aten, key=lambda x: x.get("ts", -1)): if event.name.startswith(const.ATEN): if not formated_atens or not formated_atens[-1].ts_include(event): formated_atens.append(event) elif event.name.startswith(const.SYNC_STREAM): - self._synchronize_stream.update_sync_stream_count() - if formated_atens[-1].ts_include(event): + self.synchronize_stream.update_sync_stream_count() + if formated_atens and formated_atens[-1].ts_include(event): # 使用aten算子的索引，用于查询堆栈 event["dataset_index"] = formated_atens[-1].get("dataset_index") - self._synchronize_stream.append_slow_sync_stream(event) + self.synchronize_stream.append_slow_sync_stream(event) else: continue - self._aten = formated_atens + self.aten = formated_atens + + +class ComputationAnalysisDataset(BaseTimelineEventDataset): + collector_map = OrderedDict( + StepCollector=StepCollector(), + SpecificTaskTypeOpCollector=SpecificTaskTypeOpCollector(), + TorchToNpuCollector=TorchToNpuCollector(), + AclToNpuCollector=AclToNpuCollector(), + OpStackCollector=OpStackCollector(), + FrequencyCollector=FrequencyCollector(), + ) + + def __init__(self, collection_path, data: dict, build_dataset=True, **kwargs) -> None: + super().__init__(collection_path, data, build_dataset, **kwargs) + self._collector_post_process() diff --git a/profiler/advisor/dataset/timeline_op_collector/__init__.py b/profiler/advisor/dataset/timeline_op_collector/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py new file mode 100644 index 000000000..56e6165dd --- /dev/null +++ b/profiler/advisor/dataset/timeline_op_collector/timeline_op_collector.py @@ -0,0 +1,376 @@ +import logging +import math +import os +from abc import abstractmethod, ABCMeta + +from profiler.advisor.common import constant as const +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.advisor.utils.utils import convert_to_float +from profiler.cluster_analyse.common_func.file_manager import FileManager + +logger = logging.getLogger() + + +class BaseOpCollector(metaclass=ABCMeta): + + def __init__(self): + self.attribute_to_dataset = {} + self.op_list = [] + self.require_filter_by_step = True + + @abstractmethod + def add_op(self): + """ add timeline event into self.op_list, and then will filter event in self.op_list by specific step + """ + pass + + @abstractmethod + def post_process(self): + """ convert self.op_list to required format like dict, set and so on and then record the final object into + self.attribute_to_dataset which used to set property of timeline event dataset + """ + pass + + +class StepCollector(BaseOpCollector): + KEY_WORD = "ProfilerStep" + + def __init__(self): + super().__init__() + self.require_filter_by_step = False + + def add_op(self, event): + if event.name.startswith(self.KEY_WORD): + self.op_list.append(event) + + def post_process(self, *args, **kwargs): + self.attribute_to_dataset["profiler_step"] = self.op_list + + +class OpCompileCollector(BaseOpCollector): + def __init__(self): + super().__init__() + self._total_op_compile_counter = 0 + self._total_op_compile_time = 0.0 + + @property + def total_time(self): + return self._total_op_compile_time + + @property + def total_count(self): + return self._total_op_compile_counter + + def is_empty(self): + return self._total_op_compile_counter == 0 + + def update(self, event: TimelineEvent): + self._total_op_compile_time += float(event.dur) + self._total_op_compile_counter += 1 + + def unset(self): + self._total_op_compile_counter = 0 + self._total_op_compile_time = 0.0 + + def add_op(self, event): + if event.name == const.OP_COMPILE_NAME or event.args.get("id") == const.OP_COMPILE_ID: + self.op_list.append(event) + + def post_process(self, target_op_list, **kwargs): + for op in target_op_list: + self.update(op) + + self.attribute_to_dataset["ops_compile"] = self + + +class SynchronizeStreamCollector(BaseOpCollector): + + def __init__(self): + super().__init__() + self._synchronize_stream_count = 0 + self._slow_synchronize_stream = [] + self.rule = SynchronizeStreamCollector._load_rule() + + @property + def total_count(self): + return self._synchronize_stream_count + + @property + def slow_synchronize_stream(self): + return self._slow_synchronize_stream + + @staticmethod + def _load_rule(): + sync_stream_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "synchronize.yaml") + + sync_stream_rule = FileManager.read_yaml_file(sync_stream_rule_path) + return sync_stream_rule + + def update_sync_stream_count(self): + self._synchronize_stream_count += 1 + + def append_slow_sync_stream(self, event): + if float(event.dur) / 1000 >= self.rule.get("slow_synchronize_threshold", 10): + self._slow_synchronize_stream.append(event) + + def unset(self): + self._synchronize_stream_count = 0 + self._slow_synchronize_stream = [] + + def add_op(self, event): + return self.op_list + + def post_process(self, *args, **kwargs): + self.attribute_to_dataset["synchronize_stream"] = self + + +class MemCollector(BaseOpCollector): + MEMORY_OP_NAME = ["AscendCL@aclMallocMemInner", "AscendCL@aclrtFreePhysical"] + + def __init__(self): + super().__init__() + self.mem_op_info = {} + self.rule = self._load_rule() + + @staticmethod + def _load_rule(): + memory_rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "rules", + "memory.yaml") + + memory_rule = FileManager.read_yaml_file(memory_rule_path) + return memory_rule + + def add_op(self, event): + if event.name not in self.MEMORY_OP_NAME: + return + self.op_list.append(event) + + def post_process(self, target_op_list, **kwargs): + for op in target_op_list: + if op.name not in self.mem_op_info: + self.mem_op_info[op.name] = dict(count=0, total_dur=0) + self.mem_op_info[op.name]["count"] += 1 + self.mem_op_info[op.name]["total_dur"] += float(op.dur) + + self.attribute_to_dataset["memory_ops"] = self + + +class DataloaderCollector(BaseOpCollector): + key_word = "dataloader" + + def __init__(self): + super().__init__() + + def add_op(self, event): + if self.key_word in event.name.lower(): + self.op_list.append(TimelineEvent({ + "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur, + "stack": event.args.get("Call stack") + })) + + def post_process(self, *args, **kwargs): + self.attribute_to_dataset["dataloader"] = self.op_list + + +class SyncBNCollector(BaseOpCollector): + key_word = "syncbatchnorm" + + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name.lower() == self.key_word: + self.op_list.append(TimelineEvent({ + "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur + })) + + def post_process(self, target_op_list, **kwargs): + self.attribute_to_dataset["sync_batchnorm"] = target_op_list + + +class AtenCollector(BaseOpCollector): + + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name.lower().startswith(f"{const.ATEN}{const.ATEN_SEP}") or event.name.lower().startswith( + f"{const.NPU}{const.ATEN_SEP}"): + self._add_aten(event) + return + + # 检查cann层同步操作，根据时间窗口索引到host侧的aten算子并给出堆栈 + if event.name.startswith(const.SYNC_STREAM): + self._add_aten(event) + + def post_process(self, target_op_list, **kwargs): + self.attribute_to_dataset["aten"] = target_op_list + + def _add_aten(self, event: TimelineEvent): + self.op_list.append(TimelineEvent({ + "name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur + })) + + +class OptimizerCollector(BaseOpCollector): + + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name.startswith(f"{const.OPTIMIZER}.{const.OPTIMIZER_STEP}{const.OPTIMIZER_SEP}"): + self.op_list.append(TimelineEvent( + {"name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur})) + + def post_process(self, target_op_list, **kwargs): + self.attribute_to_dataset["optimizer"] = target_op_list + + +class FrequencyCollector(BaseOpCollector): + KEY_WORD = "AI Core Freq" + + def __init__(self): + super().__init__() + self._previous_freq_index = -1 + + @staticmethod + def get_op_frequency(ai_core_ops, ai_core_freq): + ai_core_freq.sort(key=lambda x: float(x.ts)) + op_freq_record = {} + + op_index, freq_index = 0, 0 + while op_index < len(ai_core_ops) and freq_index < len(ai_core_freq): + op_event = ai_core_ops[op_index] + op_end_time = convert_to_float(op_event.ts) + convert_to_float(op_event.dur) + op_freq_list = [] + while freq_index < len(ai_core_freq): + freq_event = ai_core_freq[freq_index] + if convert_to_float(freq_event.end) < op_end_time: + op_freq_list.append(convert_to_float(freq_event.args.MHz)) + freq_index += 1 + continue + elif convert_to_float(freq_event.ts) < op_end_time: + if op_event.name not in op_freq_record: + op_freq_record[op_event.name] = {"count": 0, "dur": 0, "freq_list": []} + op_freq_record[op_event.name]["count"] += 1 + op_freq_record[op_event.name]["dur"] += convert_to_float(op_event.dur) + op_freq_list.append(convert_to_float(freq_event.args.MHz)) + op_freq_record[op_event.name]["freq_list"].append(min(op_freq_list)) + break + else: + break + + op_index += 1 + return op_freq_record + + def add_op(self, event): + if event.name == self.KEY_WORD: + if self._previous_freq_index != -1: + self.op_list[self._previous_freq_index]["end"] = event.get("ts", float(math.inf)) + self._previous_freq_index += 1 + event.setdefault("end", float(math.inf)) + self.op_list.append(event) + + def post_process(self, target_op_list, **kwargs): + ai_core_ops = kwargs.get("ai_core_ops", []) + if not ai_core_ops: + return + ai_core_ops.sort(key=lambda x: float(x.ts)) + op_freq = FrequencyCollector.get_op_frequency(ai_core_ops, target_op_list) + self.attribute_to_dataset["op_freq"] = op_freq + + +class SpecificTaskTypeOpCollector(BaseOpCollector): + + def __init__(self, op_type_list=None): + super().__init__() + self.op_type_list = op_type_list if op_type_list else [const.AI_CPU, const.AI_CORE, const.MIX_AIC] + + def add_op(self, event): + if event.args.get(const.TASK_TYPE) and event.args.get(const.TASK_TYPE) in self.op_type_list: + self.op_list.append( + TimelineEvent( + { + const.TASK_TYPE: event.args.get(const.TASK_TYPE), + "task_id": event.args.get("Task Id"), + "tid": event.tid, + "name": event.name, + "ts": str(event.ts), + "dur": str(event.dur) + } + ) + ) + + def post_process(self, target_op_list, **kwargs): + op_map = dict() + for op in target_op_list: + key = f"{op.name}-{op.ts}" + op_map[key] = op + + self.attribute_to_dataset["ops_with_task_type"] = op_map + self.attribute_to_dataset["task_op_names"] = list( + set([event_key.split("-")[0] for event_key in op_map.keys()])) + + +class TorchToNpuCollector(BaseOpCollector): + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name.lower() == const.TORCH_TO_NPU: + self.op_list.append(TimelineEvent({"tid": event.tid, "ts": str(event.ts), "ph": event.ph, "id": event.id})) + + def post_process(self, target_op_list, **kwargs): + op_map = dict() + for op in target_op_list: + key = f"{op.ph}-{op.id}" + op_map[key] = op + + self.attribute_to_dataset["torch_to_npu"] = op_map + + +class AclToNpuCollector(BaseOpCollector): + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.name and event.ts and event.name == const.ACL_TO_NPU: + self.op_list.append(TimelineEvent({"ts": event.ts})) + + def post_process(self, target_op_list, **kwargs): + op_record = set(str(op.ts) for op in target_op_list) + self.attribute_to_dataset["acl_to_npu"] = op_record + + +class OpStackCollector(BaseOpCollector): + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.args.get(const.CALL_STACKS): + self.op_list.append( + TimelineEvent({"name": event.name, "dataset_index": event.dataset_index, "ts": event.ts})) + + def post_process(self, target_op_list, **kwargs): + op_map = dict() + for op in target_op_list: + op_map[str(op.ts)] = op + + self.attribute_to_dataset["ops_with_stack"] = op_map + + +class GcCollector(BaseOpCollector): + def __init__(self): + super().__init__() + + def add_op(self, event): + if event.cat and isinstance(event.cat, str) and event.cat.lower() == "gc": + self.op_list.append(TimelineEvent( + {"name": event.name, "dataset_index": event.dataset_index, "ts": event.ts, "dur": event.dur})) + + def post_process(self, target_op_list, **kwargs): + self.attribute_to_dataset["gc_events"] = self.op_list diff --git a/profiler/advisor/display/html/priority_background_color.py b/profiler/advisor/display/html/priority_background_color.py new file mode 100644 index 000000000..7da61a093 --- /dev/null +++ b/profiler/advisor/display/html/priority_background_color.py @@ -0,0 +1,4 @@ +class PriorityBackgroundColor: + high = "#B5495B" + medium = "#fcaf17" + low = "#65c294" diff --git a/profiler/advisor/display/html/render.py b/profiler/advisor/display/html/render.py index 3984fa8f3..0c1882f13 100644 --- a/profiler/advisor/display/html/render.py +++ b/profiler/advisor/display/html/render.py @@ -1,7 +1,7 @@ import os import logging from typing import List, Dict -from collections import defaultdict +from collections import defaultdict, OrderedDict from jinja2 import Environment, FileSystemLoader from profiler.advisor.common import constant @@ -14,31 +14,72 @@ logger = logging.getLogger() @singleton class HTMLRender: + SUPPORTED_KEYS = ["main", "overall", "comparison", "computation", "schedule", "communication", "dataloader", + "memory"] + PERFORMANCE_PROBLEM_ANALYSIS = "performance_problem_analysis" + def __init__(self): self.html = "" self.render_list = defaultdict(list) def render_html(self, template_dir: str = "templates", template_name: str = "main.html", template_header=constant.DEFAULT_TEMPLATE_HEADER): - self.html = self.render_template("main", template_dir, template_name, render_list=self.render_list, + + # 确保overall 和 comparison 在 performance problem analysis 之前 + sorted_render_htmls = OrderedDict() + for key in ["overall", "comparison"]: + if key in self.render_list: + sorted_render_htmls[key] = self.render_list.get(key) + for key, html in self.render_list.items(): + if key in sorted_render_htmls: + continue + sorted_render_htmls[key] = html + + self.html = self.render_template("main", template_dir, template_name, render_list=sorted_render_htmls, template_header=template_header) - def render_template(self, key: str, template_dir: str, template_name: str, **kwargs): + def get_rendered_html(self, key: str, template_dir: str, template_name: str, **kwargs): + if key not in self.SUPPORTED_KEYS: + error_msg = f"Error render template key {key}, optionals are {self.SUPPORTED_KEYS}" + logger.error(error_msg) + raise Exception(error_msg) + if not os.path.isabs(template_dir): template_dir = os.path.join(os.path.dirname(__file__), template_dir) env = Environment(loader=FileSystemLoader(template_dir), autoescape=True) template = env.get_template(template_name) + if "priority" not in kwargs: + kwargs["priority"] = "low priority" rendered_html = template.render(**kwargs) - self.render_list[key].append(rendered_html) + return rendered_html + + def render_template(self, key: str, template_dir: str, template_name: str, **kwargs): + rendered_html = self.get_rendered_html(key, template_dir, template_name, **kwargs) + + if not kwargs.get("add_render_list", True): + return rendered_html + + if key in ["main", "overall", "comparison"]: + if key not in self.render_list: + self.render_list[key] = [] + self.render_list[key].append(rendered_html) + else: + if self.PERFORMANCE_PROBLEM_ANALYSIS not in self.render_list: + self.render_list[self.PERFORMANCE_PROBLEM_ANALYSIS] = {} + if key not in self.render_list[self.PERFORMANCE_PROBLEM_ANALYSIS]: + self.render_list[self.PERFORMANCE_PROBLEM_ANALYSIS][key] = [] + self.render_list[self.PERFORMANCE_PROBLEM_ANALYSIS][key].append(rendered_html) + return rendered_html def save_to_file(self, save_path: str): + save_path = os.path.join(Config().work_path, save_path) if not save_path.endswith(".html"): logger.error("Skip save html file because file name must endswith `.html`, " "but got %s.", os.path.basename(save_path)) return safe_write(self.html, save_path) - logger.info("Save suggestion to %s.", os.path.join(Config().work_path, save_path)) + logger.info("Save suggestion to %s.", save_path) diff --git a/profiler/advisor/display/html/templates/affinity_api.html b/profiler/advisor/display/html/templates/affinity_api.html index 4d12c3e37..e9f3dd29c 100644 --- a/profiler/advisor/display/html/templates/affinity_api.html +++ b/profiler/advisor/display/html/templates/affinity_api.html @@ -1,11 +1,11 @@ {% if result|length > 0 %}
-

Affinity API Issues

+

Affinity API Issues

The analysis results of following affinity APIs are based on runtime env - cann-{{ cann_version }} + cann-{{ cann_version }} and - torch-{{ torch_version }} + torch-{{ torch_version }}
@@ -13,7 +13,7 @@ Suggestion: These APIs have no code stack. If parameter 'with_stack=False' was set while profiling, please refer to Ascend PyTorch Profiler to set - 'with_stack=True'. Otherwise, ignore following affinity APIs due to backward broadcast lack of stack. + 'with_stack=True'. Otherwise, ignore following affinity APIs due to backward broadcast lack of stack. {% endif %} {% for api_name, stacks in result.items() %} diff --git a/profiler/advisor/display/html/templates/ai_core_frequency.html b/profiler/advisor/display/html/templates/ai_core_frequency.html index d04514203..9e5f34cef 100644 --- a/profiler/advisor/display/html/templates/ai_core_frequency.html +++ b/profiler/advisor/display/html/templates/ai_core_frequency.html @@ -1,6 +1,6 @@ {% if data|length > 0 %}
-

AI CORE Frequency Issues

+

AI CORE Frequency Issues

Issue: {{ desc }}
diff --git a/profiler/advisor/display/html/templates/main.html b/profiler/advisor/display/html/templates/main.html index 3727125b4..61c52d1db 100644 --- a/profiler/advisor/display/html/templates/main.html +++ b/profiler/advisor/display/html/templates/main.html @@ -137,10 +137,21 @@

Performance Optimization Suggestions

+ +
+ Optimization Priority: +
+ High +
+ Medium +
+ Low +
+ {% for key, renders in render_list.items() %} - {% if key == 'operator'%} + {% if key != 'performance_problem_analysis' %}
-

computation

+

{{ key }}

{% for render in renders %} {{render|safe}} @@ -148,14 +159,25 @@
{% else %} +
-

{{ key }}

+

performance problem analysis

- {% for render in renders %} - {{render|safe}} - {% endfor %} + + + {% for sub_key, sub_renders in renders.items() %} +
+

{{ sub_key }}

+
+ {% for render in sub_renders %} + {{render|safe}} + {% endfor %} +
+
+ {% endfor %}
+ {% endif %} {% endfor %}
diff --git a/profiler/advisor/display/html/templates/memory.html b/profiler/advisor/display/html/templates/memory.html new file mode 100644 index 000000000..c35070109 --- /dev/null +++ b/profiler/advisor/display/html/templates/memory.html @@ -0,0 +1,18 @@ +
+

Memory Operator Issues

+
+ {{ desc }} + + + + + + {% for suggestion in suggestions %} + + + + {% endfor %} +
Suggestions
{{ loop.index }}. {{ suggestion|safe }}
+ +
+
diff --git a/profiler/advisor/display/html/templates/operator_ai_cpu.html b/profiler/advisor/display/html/templates/operator_ai_cpu.html index b3235a880..90f3bb93a 100644 --- a/profiler/advisor/display/html/templates/operator_ai_cpu.html +++ b/profiler/advisor/display/html/templates/operator_ai_cpu.html @@ -1,5 +1,5 @@
-

AICPU Issues

+

AICPU Issues

diff --git a/profiler/advisor/display/html/templates/operator_block_dim.html b/profiler/advisor/display/html/templates/operator_block_dim.html index 4e2c832f6..8079db30a 100644 --- a/profiler/advisor/display/html/templates/operator_block_dim.html +++ b/profiler/advisor/display/html/templates/operator_block_dim.html @@ -1,5 +1,5 @@
-

Block Dim Issues

+

Block Dim Issues

diff --git a/profiler/advisor/display/html/templates/operator_dispatch.html b/profiler/advisor/display/html/templates/operator_dispatch.html index c80508635..7aa69251e 100644 --- a/profiler/advisor/display/html/templates/operator_dispatch.html +++ b/profiler/advisor/display/html/templates/operator_dispatch.html @@ -1,6 +1,6 @@ {% if optimizers|length > 0 %}
-

Operator Dispatch Issues

+

Operator Dispatch Issues

diff --git a/profiler/advisor/display/html/templates/operator_dynamic_shape.html b/profiler/advisor/display/html/templates/operator_dynamic_shape.html index 59920b6c9..9d4ca028b 100644 --- a/profiler/advisor/display/html/templates/operator_dynamic_shape.html +++ b/profiler/advisor/display/html/templates/operator_dynamic_shape.html @@ -1,5 +1,5 @@
-

Operator Dynamic Shape Issues

+

Operator Dynamic Shape Issues

diff --git a/profiler/advisor/display/html/templates/operator_no_bound.html b/profiler/advisor/display/html/templates/operator_no_bound.html index cfbd20baa..d0b8925cb 100644 --- a/profiler/advisor/display/html/templates/operator_no_bound.html +++ b/profiler/advisor/display/html/templates/operator_no_bound.html @@ -1,5 +1,5 @@
-

Operator No Bound Issues

+

Operator No Bound Issues

diff --git a/profiler/advisor/display/html/templates/pp_stage_computation_analysis.html b/profiler/advisor/display/html/templates/pp_stage_computation_analysis.html new file mode 100644 index 000000000..577aeda9e --- /dev/null +++ b/profiler/advisor/display/html/templates/pp_stage_computation_analysis.html @@ -0,0 +1,19 @@ +{% if stages_rendered_html|length > 0 %} +
+

Pipeline Parallel Stages Issues

+
+ {% for stage_html in stages_rendered_html %} +
+

{{ stage_html['stage']|safe }}

+
+ Description: analysis for slow rank {{ stage_html['rank_id']|safe }} in current stage +

+ {% for html in stage_html['html_list'] %} + {{ html|safe }} + {% endfor %} +
+
+ {% endfor %} +
+
+{% endif %} \ No newline at end of file diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index 3fa10db64..9b15a419b 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -15,17 +15,22 @@ import os from collections import OrderedDict import sys -sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), "cluster_analyse")) -sys.path.insert(0, os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), "compare_tools")) + +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "cluster_analyse")) +sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))), + "compare_tools")) from profiler.advisor.utils.utils import Timer -from profiler.advisor.analyzer.computation.profiling_analyzer import AicpuAnalyzer, BlockDimAnalyzer, DynamicShapeAnalyzer, OperatorBoundAnalyzer +from profiler.advisor.analyzer.computation.profiling_analyzer import AicpuAnalyzer, BlockDimAnalyzer, \ + DynamicShapeAnalyzer, OperatorBoundAnalyzer from profiler.advisor.analyzer.schedule.fusion_ops.fusion_ops_analyzer import TimelineFusionOpsAnalyzer from profiler.advisor.analyzer.graph_fusion.graph_fusion_analyzer import FusionOPAnalyzer from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.analyzer.cluster.slow_rank_analyser import SlowRankAnalyzer -from profiler.advisor.analyzer.cluster.slow_link_analyser import SlowLinkAnalyzer -from profiler.advisor.analyzer.cluster.Communication_retransmission_analyzer import RDMARetransmissionAnalyzer +from profiler.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer +from profiler.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer +from profiler.advisor.analyzer.communication.retransmission.communication_retransmission_analyzer import \ + RDMARetransmissionAnalyzer from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer from profiler.advisor.analyzer.overall.environment_variable_analyzer import EnvironmentVariabelAnalyzer from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer @@ -33,20 +38,29 @@ from profiler.advisor.analyzer.schedule.syncbn.syncbn_analyzer import SyncBNAnal from profiler.advisor.analyzer.schedule.synchronize_stream.synchronize_stream_analyzer import SynchronizeStreamAnalyzer from profiler.advisor.analyzer.dataloader.dataloader_analyzer import DataloaderAnalyzer from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_analyzer import AICoreFreqAnalyzer -from profiler.advisor.analyzer.communication.packet_analyzer import PacketAnalyzer +from profiler.advisor.analyzer.memory.memory_analyzer import MemoryAnalyzer +from profiler.advisor.analyzer.communication.packet.packet_analyzer import PacketAnalyzer from profiler.advisor.analyzer.schedule.gc.gc_analyzer import GcAnalyzer class Interface: + SCHEDULE = "schedule" + COMPUTATION = "computation" + COMMUNICATION = "communication" + OVERALL = "overall" + CLUSTER = "cluster" + MEMORY = "memory" + supported_analyzer = { - "schedule": OrderedDict({ + SCHEDULE: OrderedDict({ SupportedScopes.SYNCBN: SyncBNAnalyzer, SupportedScopes.TIMELINE_OP_DISPATCH: OpDispatchAnalyzer, SupportedScopes.SYNCHRONIZE_STREAM: SynchronizeStreamAnalyzer, SupportedScopes.TIMELINE_FUSION_OPS: TimelineFusionOpsAnalyzer, + SupportedScopes.DATALOADER: DataloaderAnalyzer, SupportedScopes.GC_ANALYSIS: GcAnalyzer }), - "computation": OrderedDict({ + COMPUTATION: OrderedDict({ SupportedScopes.DYNAMIC_SHAPE_ANALYSIS: DynamicShapeAnalyzer, SupportedScopes.AICPU_ANALYSIS: AicpuAnalyzer, SupportedScopes.OPERATOR_NO_BOUND_ANALYSIS: OperatorBoundAnalyzer, @@ -54,19 +68,15 @@ class Interface: SupportedScopes.GRAPH: FusionOPAnalyzer, SupportedScopes.FREQ_ANALYSIS: AICoreFreqAnalyzer }), - "communication": OrderedDict({ - SupportedScopes.PACKET: PacketAnalyzer - }), - "overall": OrderedDict({ - SupportedScopes.ENVIRONMENT_VARIABLE_ANALYSIS: EnvironmentVariabelAnalyzer, - SupportedScopes.OVER_ALL: OverallSummaryAnalyzer, - }), - "dataloader": OrderedDict({SupportedScopes.DATALOADER: DataloaderAnalyzer}), - "cluster": OrderedDict({ - SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION: RDMARetransmissionAnalyzer, + COMMUNICATION: OrderedDict({SupportedScopes.PACKET: PacketAnalyzer, + SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION: RDMARetransmissionAnalyzer}), + OVERALL: OrderedDict({SupportedScopes.OVER_ALL: OverallSummaryAnalyzer, + SupportedScopes.ENVIRONMENT_VARIABLE_ANALYSIS: EnvironmentVariabelAnalyzer}), + CLUSTER: OrderedDict({ SupportedScopes.SLOW_RANK: SlowRankAnalyzer, SupportedScopes.SLOW_LINK: SlowLinkAnalyzer - }) + }), + MEMORY: OrderedDict({SupportedScopes.MEMORY: MemoryAnalyzer}) } all_dimension = list(supported_analyzer.keys()) @@ -82,6 +92,12 @@ class Interface: def get_analyzer(dimension, scope): return Interface.supported_analyzer.get(dimension).get(scope) + @staticmethod + def add_analyzer(dimension, scope, analyzer_class): + if dimension not in Interface.supported_analyzer: + Interface.supported_analyzer[dimension] = OrderedDict() + Interface.supported_analyzer[dimension][scope] = analyzer_class + def get_result(self: any, dimension: str, scope: str, render_html=False, output_dict=True, **kwargs): """ :Param mode: affinity apis, ai cpu and so on. diff --git a/profiler/advisor/result/result.py b/profiler/advisor/result/result.py index 0d0602ee5..249b88539 100644 --- a/profiler/advisor/result/result.py +++ b/profiler/advisor/result/result.py @@ -14,6 +14,9 @@ from profiler.advisor.config.config import Config class ResultWriter: + + MAX_SHEET_NAME_LENGTH = 31 + def __init__(self, result_path=None): self.result_path = result_path self.workbook = xlsxwriter.Workbook(result_path) @@ -44,6 +47,9 @@ class ResultWriter: }) def add_data(self, sheet_name, headers, data_list): + if len(sheet_name) > self.MAX_SHEET_NAME_LENGTH: + sheet_name = sheet_name[:self.MAX_SHEET_NAME_LENGTH] + sheet = self.workbook.add_worksheet(sheet_name) if headers: diff --git a/profiler/advisor/rules/dataloader.yaml b/profiler/advisor/rules/dataloader.yaml index a84abcfdf..75530a4aa 100644 --- a/profiler/advisor/rules/dataloader.yaml +++ b/profiler/advisor/rules/dataloader.yaml @@ -1,5 +1,4 @@ -# unit is milliseconds -dataloader_duration_threshold: 10 +dataloader_duration_threshold: 10000 # us problem: "Found slow dataloader, cost {dataloader_duration} milliseconds for one step while profiling, normally less than {dataloader_duration_threshold} milliseconds." solutions: - "Please check the disk I/O of your data directory. If you are training model in ModelArts, please move data to '/cache' or mount a more efficient cloud disk for better I/O." diff --git a/profiler/advisor/rules/memory.yaml b/profiler/advisor/rules/memory.yaml new file mode 100644 index 000000000..4a55386a7 --- /dev/null +++ b/profiler/advisor/rules/memory.yaml @@ -0,0 +1,7 @@ +problem: "Found {memory_op_num} {memory_op_name}, cost {memory_op_dur} milliseconds, which will lead to large amount of free time." +max_total_duration: 100000 #us +solutions: + - AscendCL@aclMallocMemInner: + desc: "Please set env by command 'export PYTORCH_NPU_ALLOC_CONF=expandable_segments:True' and then start your training job" + - AscendCL@aclrtFreePhysical: + desc: "Please reduce your batch size for training job" \ No newline at end of file diff --git a/profiler/advisor/rules/timeline_fusion_ops.yaml b/profiler/advisor/rules/timeline_fusion_ops.yaml index 8207465dc..46e02fef7 100644 --- a/profiler/advisor/rules/timeline_fusion_ops.yaml +++ b/profiler/advisor/rules/timeline_fusion_ops.yaml @@ -4,16 +4,11 @@ operator_rules: aten: add: - torch_npu.npu_confusion_transpose: ["(permute|transpose)-(contiguous){0,1}-(reshape|view)", - "(reshape|view)-(contiguous){0,1}-(permute|transpose)"] - torch_npu.fast_gelu: [gelu] - torch_npu.npu_linear: [linear] - torch_npu.npu_mish: [mish] - torch_npu.contrib.module.Mish: [mish] + torch_npu.npu_confusion_transpose: [ "(permute|transpose)-(contiguous){0,1}-(reshape|view)", + "(reshape|view)-(contiguous){0,1}-(permute|transpose)" ] + torch_npu.fast_gelu: [ gelu ] torch_npu.npu_scaled_masked_softmax: [ "softmax-(mul){0,1}-(masked_fill_|add)" ] - torch_npu.npu_silu: [ silu, mul-sigmoid, sigmoid-mul ] - torch_npu.contrib.module.SiLU: [ silu, mul-sigmoid, sigmoid-mul ] - optimizer.clip_grad_norm_fused_: [add-reciprocal-mul] + optimizer.clip_grad_norm_fused_: [ add-reciprocal-mul ] Optimizer: add: torch_npu.optim.NpuFusedAdamW: [AdamW.step] @@ -56,4 +51,7 @@ operator_rules: aten: add: - torch_npu.npu_geglu: ["(slice|chunk)-gelu-mul", "(slice|chunk)-mul-gelu"] \ No newline at end of file + torch_npu.npu_geglu: [ "(slice|chunk)-gelu-mul", "(slice|chunk)-mul-gelu" ] + torch_npu.npu_group_norm_silu: [ "group_norm-silu" ] + torch.addmm: [ "mul-mul-add" ] + torch_npu.npu_add_layer_norm: [ "add-layer_norm" ] \ No newline at end of file diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index d0a132167..8e6a8cba3 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -108,7 +108,7 @@ def singleton(cls): for base_class in inspect.getmro(cls)[::-1]: # 获取类的所有成员 members = inspect.getmembers(base_class) - + # 过滤出函数对象 function_objs = [member[1] for member in members if inspect.isfunction(member[1]) or inspect.ismethod(member[1])] for function_obj in function_objs: @@ -213,26 +213,26 @@ def format_timeline_result(result: dict, dump_html=False): class ParallelJob: - def __init__(self, src_func, ops_api_list, job_name=None): + def __init__(self, src_func, job_params, job_name=None): if not callable(src_func): raise TypeError(f"src_func should be callable") - if not isinstance(ops_api_list, (list, tuple)): - raise TypeError(f"ops_api_list should be list or tuple") + if not isinstance(job_params, (list, tuple)): + raise TypeError(f"job_params should be list or tuple") self.src_func = src_func - self.ops_api_list = ops_api_list + self.job_params = job_params self.job_name = job_name def start(self, n_proccesses): - job_queue = mp.Queue(len(self.ops_api_list)) + job_queue = mp.Queue(len(self.job_params)) completed_queue = mp.Queue() - for i in range(len(self.ops_api_list)): + for i in range(len(self.job_params)): job_queue.put(i) processes = [] - listen = mp.Process(target=self.listener, args=(completed_queue, len(self.ops_api_list),)) + listen = mp.Process(target=self.listener, args=(completed_queue, len(self.job_params),)) listen.start() for i in range(n_proccesses): @@ -261,7 +261,12 @@ class ParallelJob: token = job_queue.get(timeout=1) except queue.Empty: continue - self.src_func(*self.ops_api_list[token]) + if isinstance(self.job_params[token], (list, tuple)): + self.src_func(*self.job_params[token]) + elif isinstance(self.job_params[token], dict): + self.src_func(**self.job_params[token]) + else: + self.src_func(self.job_params[token]) completed_queue.put(token) @@ -614,11 +619,17 @@ def convert_to_float(num): try: return float(num) except (ValueError, FloatingPointError): - logger.error(f"Can not convert %ss to float", num) - pass + logger.error(f"Can not convert %s to float", num) return 0 +def safe_index(array, value, return_index_if_error=None): + if value in array: + return array.index(value) + + return return_index_if_error + + def convert_to_int(data: any) -> int: try: int_value = int(data) diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index 71736fe30..2b5f63ed7 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -2,54 +2,19 @@ import click import sys import os import logging -from pathlib import Path sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "compare_tools")) sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "cluster_analyse")) +from profiler.advisor.analyzer.analyzer_controller import AnalyzerController from profiler.advisor.utils.tools import CONTEXT_SETTINGS, ClickAliasedGroup from profiler.advisor.common import constant from profiler.advisor.utils.utils import debug_option from profiler.advisor.interface.interface import Interface -from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor logger = logging.getLogger() -def _analyze(dimensions, **kwargs): - result_list = [] - job_list = [] - if not Path(kwargs.get("profiling_path")).exists(): - print(f"[ERROR] Profiling_path is not existed. Invalid profiling path: {kwargs.get('profiling_path')}") - return - - def is_cluster(): - profiling_path = kwargs.get("profiling_path") - path_list = [os.path.join(profiling_path, dir_name) for dir_name in os.listdir(profiling_path)] - ascend_pt_dirs = [path for path in path_list if os.path.isdir(path) and path.endswith("ascend_pt")] - data_processor = PytorchDataPreprocessor(ascend_pt_dirs) - data_map = data_processor.get_data_map() - return len(data_map) > 1 - - is_cluster = is_cluster() - - for dimension in dimensions: - if not is_cluster and dimension == "cluster": - continue - for scope in Interface.get_scope(dimension): - interface = Interface(**kwargs) - job_list.append((dimension, scope, interface)) - - for i, (dimension, scope, interface) in enumerate(job_list[::-1]): - result_list.append( - interface.get_result(dimension, scope, render_html=i == len(job_list) - 1, output_dict=False, **kwargs)) - - for result in result_list[::-1]: - if result and hasattr(result, "show"): - result.show() - break - - @click.group(name="analyze", cls=ClickAliasedGroup) def analyze_cli(**kwargs): """Analyze profiling datasets and give performance optimization suggestion.""" @@ -84,10 +49,7 @@ def analyze_cli(**kwargs): help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @debug_option def analyze_all(**kwargs) -> None: - try: - _analyze(Interface.all_dimension, **kwargs) - except RuntimeError as e: - print(f"[ERROR] {e}") + AnalyzerController().do_analysis(Interface.all_dimension, **kwargs) @analyze_cli.command(context_settings=CONTEXT_SETTINGS, @@ -106,7 +68,7 @@ def analyze_all(**kwargs) -> None: help='The runtime torch version, which can be detected by exec command "pip show torch"') @debug_option def analyze_schedule(**kwargs) -> None: - _analyze(["schedule"], **kwargs) + AnalyzerController().do_analysis([Interface.SCHEDULE], **kwargs) @analyze_cli.command(context_settings=CONTEXT_SETTINGS, @@ -132,4 +94,4 @@ def analyze_schedule(**kwargs) -> None: help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @debug_option def analyze_computation(**kwargs) -> None: - _analyze(["computation"], **kwargs) + AnalyzerController().do_analysis([Interface.COMPUTATION], **kwargs) diff --git a/profiler/test/ut/advisor/analyzer_controller/test_analyzer_controller.py b/profiler/test/ut/advisor/analyzer_controller/test_analyzer_controller.py new file mode 100644 index 000000000..b1429b3ad --- /dev/null +++ b/profiler/test/ut/advisor/analyzer_controller/test_analyzer_controller.py @@ -0,0 +1,189 @@ +import unittest +import os +import sys +import yaml +from unittest import mock + +from profiler.advisor.analyzer.analyzer_controller import AnalyzerController +from profiler.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer +from profiler.advisor.interface.interface import Interface +from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env + +mock_analysis_job_list = [("mock_dimension", "mock_scope", Interface(profiling_path=os.path.realpath(__file__)), {})] +mock_global_step_rank = {"maximum": {"rank_id": 0, "step": 1}, "minimum": {"rank_id": 1, "step": 1}} +mock_profiling_path = os.path.realpath(__file__) +mock_benchmark_profiling_path = os.path.realpath(__file__) +step = "mock_step" +benchmark_step = "mock_step" +mock_cluster_statistics_data = {"header": [], "data": []} +mock_cluster_local_data_map = {mock_profiling_path: {0: "rank0_profiling_path", 1: "rank1_profiling_path"}} + + +class TestAnalyzerController(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + self.mock_slow_rank_analyzer = mock.Mock() + self.mock_slow_rank_analyzer.get_stage_step_rank.return_value = mock_global_step_rank + self.mock_slow_rank_analyzer.get_global_step_rank.return_value = mock_global_step_rank + self.mock_slow_rank_analyzer.format_datas.return_value = mock_cluster_statistics_data + + self.mock_slow_link_analyzer = mock.Mock() + self.mock_slow_link_analyzer.get_global_step_rank.return_value = mock_global_step_rank + self.mock_slow_link_analyzer.format_datas.return_value = mock_cluster_statistics_data + + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._single_profiling_comparison", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.overall", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.memory_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.computation_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.schedule_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.communication_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._get_profiling_path_by_rank", + lambda *args: None) + def test_single_rank_analysis(self): + analyzer_controller = AnalyzerController() + job_list = analyzer_controller.single_rank_analysis("mock_profiling_path", "mock_benchmark_profiling_path") + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), 4) + + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.cluster_schedule_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.cluster_communication_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.cluster_computation_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.cluster_memory_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._cluster_profiling_comparison", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.overall", + lambda *args: mock_analysis_job_list) + def test_cluster_analysis(self): + analyzer_controller = AnalyzerController() + job_list = analyzer_controller.cluster_analysis("mock_profiling_path", "mock_benchmark_profiling_path") + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), 5) + + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.cluster_schedule_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.cluster_communication_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.cluster_computation_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.cluster_memory_analysis", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._cluster_profiling_comparison", + lambda *args: mock_analysis_job_list) + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController.overall", + lambda *args: mock_analysis_job_list) + def test_cluster_analysis(self): + analyzer_controller = AnalyzerController() + job_list = analyzer_controller.cluster_analysis("mock_profiling_path", "mock_benchmark_profiling_path") + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), 5) + + def test_schedule_analysis(self): + analyzer_controller = AnalyzerController() + job_list = analyzer_controller.schedule_analysis(mock_profiling_path, + mock_benchmark_profiling_path, + step, + benchmark_step) + job_num = len(Interface.get_scope(Interface.SCHEDULE)) + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), job_num) + + def test_computation_analysis(self): + analyzer_controller = AnalyzerController() + job_list = analyzer_controller.computation_analysis(mock_profiling_path, + mock_benchmark_profiling_path, + step, + benchmark_step) + job_num = len(Interface.get_scope(Interface.COMPUTATION)) + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), job_num) + + def test_memory_analysis(self): + analyzer_controller = AnalyzerController() + job_list = analyzer_controller.memory_analysis(mock_profiling_path, + mock_benchmark_profiling_path, + step, + benchmark_step) + job_num = len(Interface.get_scope(Interface.MEMORY)) + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), job_num) + + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._get_profiling_path_by_rank", + lambda *args: mock_profiling_path) + def test_cluster_schedule_analysis(self): + analyzer_controller = AnalyzerController() + analyzer_controller.slow_rank_analyzer = self.mock_slow_rank_analyzer + job_list = analyzer_controller.cluster_schedule_analysis(mock_profiling_path) + job_num = len(Interface.get_scope(Interface.SCHEDULE)) + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), job_num) + + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._get_profiling_path_by_rank", + lambda *args: mock_profiling_path) + def test_cluster_communication_analysis(self): + analyzer_controller = AnalyzerController() + analyzer_controller.slow_link_analyzer = self.mock_slow_link_analyzer + job_list = analyzer_controller.cluster_communication_analysis(mock_profiling_path) + self.assertTrue(isinstance(job_list, list)) and self.assertFalse(job_list) + + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._get_profiling_path_by_rank", + lambda *args: mock_profiling_path) + def test_cluster_computation_analysis_for_stage(self): + analyzer_controller = AnalyzerController() + analyzer_controller.slow_rank_analyzer = self.mock_slow_rank_analyzer + job_list = analyzer_controller.cluster_computation_analysis(mock_profiling_path) + self.assertTrue(isinstance(job_list, list)) and self.assertFalse(job_list) + + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._get_profiling_path_by_rank", + lambda *args: mock_profiling_path) + def test_cluster_computation_analysis_for_global(self): + analyzer_controller = AnalyzerController() + self.mock_slow_rank_analyzer.get_stage_step_rank.return_value = {} + analyzer_controller.slow_rank_analyzer = self.mock_slow_rank_analyzer + job_list = analyzer_controller.cluster_computation_analysis(mock_profiling_path) + job_num = len(Interface.get_scope(Interface.COMPUTATION)) + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), job_num) + + @mock.patch("profiler.advisor.analyzer.analyzer_controller.AnalyzerController._get_profiling_path_by_rank", + lambda *args: mock_profiling_path) + def test_cluster_memory_analysis(self): + analyzer_controller = AnalyzerController() + analyzer_controller.slow_rank_analyzer = self.mock_slow_rank_analyzer + job_list = analyzer_controller.cluster_memory_analysis(mock_profiling_path) + job_num = len(Interface.get_scope(Interface.MEMORY)) + self.assertTrue(isinstance(job_list, list)) and self.assertEqual(len(job_list), job_num) + + def test_get_target_profiling_path_for_local(self): + analyzer_controller = AnalyzerController() + result = analyzer_controller._get_target_profiling_path_for_local(mock_profiling_path, 0) + self.assertEqual(result, mock_profiling_path) + + analyzer_controller.cluster_local_data_map = mock_cluster_local_data_map + result = analyzer_controller._get_target_profiling_path_for_local(mock_profiling_path, 0) + self.assertEqual(result, "rank0_profiling_path") + + result = analyzer_controller._get_target_profiling_path_for_local(mock_profiling_path, 1) + self.assertEqual(result, "rank1_profiling_path") + + result = analyzer_controller._get_target_profiling_path_for_local(mock_profiling_path, 2) + self.assertEqual(result, "rank0_profiling_path") + + +if __name__ == '__main__': + tester = TestAnalyzerController() + tester.test_single_rank_analysis() + tester.test_cluster_analysis() + tester.test_schedule_analysis() + tester.test_computation_analysis() + tester.test_memory_analysis() + tester.test_cluster_schedule_analysis() + tester.test_cluster_communication_analysis() + tester.test_cluster_computation_analysis_for_stage() + tester.test_cluster_computation_analysis_for_global() + tester.test_cluster_memory_analysis() + tester.test_get_target_profiling_path_for_local() diff --git a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py b/profiler/test/ut/advisor/communication_advice/test_packet_advice.py index a8fd4549e..9459ccfbe 100644 --- a/profiler/test/ut/advisor/communication_advice/test_packet_advice.py +++ b/profiler/test/ut/advisor/communication_advice/test_packet_advice.py @@ -167,7 +167,7 @@ class TestPacketAdvice(unittest.TestCase): def test_run_should_run_success_when_ascend_pt_contain_communication_json(self): self.create_communicaton_json() interface = Interface(profiling_path=self.TMP_DIR) - dimension = "communication" + dimension = Interface.COMMUNICATION scope = SupportedScopes.PACKET result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) self.assertEqual(2, len(result.data.get("Packet Analysis", []))) diff --git a/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py b/profiler/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py similarity index 99% rename from profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py rename to profiler/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py index eb383a659..35a04d415 100644 --- a/profiler/test/ut/advisor/cluster_advice/test_rdma_retransmission_advice.py +++ b/profiler/test/ut/advisor/communication_advice/test_rdma_retransmission_advice.py @@ -162,7 +162,7 @@ class TestRdmaAdvice(unittest.TestCase): def test_run_should_run_success_when_contain_cluster_communication_json(self): self.create_communicaton_json() interface = Interface(profiling_path=self.TMP_DIR) - dimension = "cluster" + dimension = Interface.COMMUNICATION scope = SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) self.assertEqual(2, len(result.data.get("Comm Retransmission Analysis", []))) diff --git a/profiler/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py b/profiler/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py new file mode 100644 index 000000000..a8be1bf19 --- /dev/null +++ b/profiler/test/ut/advisor/compute_advice/test_pp_stage_computation_analyzer.py @@ -0,0 +1,55 @@ +import unittest +import copy +import os + +from profiler.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer +from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +mock_profiling_path = os.path.realpath(__file__) + +class TestPPStageComputationAnalyzer(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + self.rank_num = 8 + rank_result_data = { + "problems": { + "data": [["ai cpu issues", "ai cpu desc", "ai cpu suggestion"], + ["frequency issues", "frequency desc"]] + }, + "ai cpu issues": { + "headers": ["op name", "duration", "count"], + "data": [["index", 1000, 10]] + }, + "frequency issues": { + "headers": ["op name", "duration", "freq reduction"], + "data": [["matmul", 1000, 0.5]] + } + } + self.mocked_multiprocess_data = {} + for i in range(self.rank_num): + self.mocked_multiprocess_data[f"rank {i}"] = copy.deepcopy(rank_result_data) + + def test_merge_multiprocess_result(self): + pp_stage_computation_analyzer = PPStageComputationAnalyzer(mock_profiling_path) + pp_stage_computation_analyzer._merge_multiprocess_result() + result = pp_stage_computation_analyzer.result + self.assertFalse(result.data) + + pp_stage_computation_analyzer._multiprocess_result = copy.deepcopy(self.mocked_multiprocess_data) + pp_stage_computation_analyzer._merge_multiprocess_result() + data = dict(pp_stage_computation_analyzer.result.data) + + problems = data.get("problems", {}).get("data", []) + self.assertEqual(len(problems), self.rank_num) + for i in range(self.rank_num): + self.assertTrue(f"rank {i} ai cpu issues" in data) + self.assertTrue(f"rank {i} frequency issues" in data) + + +if __name__ == '__main__': + tester = TestPPStageComputationAnalyzer() + tester.test_merge_multiprocess_result() diff --git a/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py b/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py deleted file mode 100644 index f18311ab1..000000000 --- a/profiler/test/ut/advisor/schedule_advice/test_gc_advice.py +++ /dev/null @@ -1,116 +0,0 @@ -import os -import shutil -import stat -import json - -import unittest -from profiler.advisor.interface.interface import Interface -from profiler.advisor.common.analyzer_scopes import SupportedScopes -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset - - -class TestGcAdvice(unittest.TestCase): - TMP_DIR = "./ascend_pt" - OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" - interface = None - - def tearDown(self): - if os.path.exists(TestGcAdvice.TMP_DIR): - shutil.rmtree(TestGcAdvice.TMP_DIR) - self.clear_htmls() - TimelineEventDataset.reset_all_instances() - - def setUp(self): - if os.path.exists(TestGcAdvice.TMP_DIR): - shutil.rmtree(TestGcAdvice.TMP_DIR) - if not os.path.exists(TestGcAdvice.TMP_DIR): - os.makedirs(TestGcAdvice.TMP_DIR) - if not os.path.exists(TestGcAdvice.OUTPUT_DIR): - os.makedirs(TestGcAdvice.OUTPUT_DIR) - self.clear_htmls() - - @classmethod - def clear_htmls(cls): - current_path = os.path.dirname(os.path.abspath(__file__)) - for filename in os.listdir(current_path): - # 检查文件是否以“att”开头 - if filename.startswith("mstt"): - # 构建文件的完整路径 - file_path = os.path.join(current_path, filename) - # 删除文件 - os.remove(file_path) - - @classmethod - def create_trace_view_with_gc_events(cls): - # Python pid - py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} - # Python GC pid - py_gc_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 2, "args": {"name": "Python GC"}} - # ascend pid - ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} - # ascend pid - cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} - # ascend hardware ops - ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - gc_event1 = {"ph": "X", "name": "GC", "ts": "1699529622103750", "dur": 1500, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - gc_event2 = {"ph": "X", "name": "GC", "ts": "1699529623104750", "dur": 50, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - gc_event3 = {"ph": "X", "name": "GC", "ts": "1699529623105750", "dur": 50000, "tid": 3, "pid": 4, "cat": "GC", - "args": {}} - # flow event - flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} - flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} - - raw_data = [ - py_pid_data, py_gc_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, gc_event1, gc_event2, - gc_event3, flow_event_s, flow_event_e - ] - with os.fdopen(os.open(f"{TestGcAdvice.OUTPUT_DIR}/trace_view.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - @classmethod - def create_trace_view_without_gc_events(cls): - # Python pid - py_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 1, "args": {"name": "Python"}} - # ascend pid - ascend_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 4, "args": {"name": "Ascend Hardware"}} - # ascend pid - cann_pid_data = {"ph": "M", "name": "process_name", "tid": 0, "pid": 5, "args": {"name": "CANN"}} - # ascend hardware ops - ah_event1 = {"ph": "X", "name": "Slice1", "ts": "1699529623106750", "dur": 100, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - ah_event2 = {"ph": "X", "name": "Slice2", "ts": "1699529623106888", "dur": 80, "tid": 3, "pid": 4, - "args": {"Task Type": "AI_CORE"}} - # flow event - flow_event_s = {"ph": "s", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "200", "args": {}} - flow_event_e = {"ph": "f", "name": "link1", "id": 1, "tid": 3, "pid": 1, "ts": "1699529623106750", "args": {}} - - raw_data = [ - py_pid_data, ascend_pid_data, cann_pid_data, ah_event1, ah_event2, flow_event_s, flow_event_e - ] - with os.fdopen(os.open(f"{TestGcAdvice.OUTPUT_DIR}/trace_view.json", - os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: - fp.write(json.dumps(raw_data)) - - def test_run_should_run_success_when_trace_view_contain_gc_events(self): - self.create_trace_view_with_gc_events() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "schedule" - scope = SupportedScopes.GC_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(2, len(result.data.get("GcAnalysis", dict).get("data", []))) - result.clear() - - def test_run_should_run_success_when_trace_view_not_contain_gc_events(self): - self.create_trace_view_without_gc_events() - interface = Interface(profiling_path=self.TMP_DIR) - dimension = "schedule" - scope = SupportedScopes.GC_ANALYSIS - result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) - self.assertEqual(0, len(result.data.get("GcAnalysis", []))) - result.clear() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py b/profiler/test/ut/advisor/timeline_advice/test_dataloader_checker.py similarity index 92% rename from profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py rename to profiler/test/ut/advisor/timeline_advice/test_dataloader_checker.py index 3d8e22b7c..f2bd26957 100644 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_dataloader_checker.py +++ b/profiler/test/ut/advisor/timeline_advice/test_dataloader_checker.py @@ -15,14 +15,14 @@ class TestDataloaderChecker(unittest.TestCase): def setUp(self) -> None: rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))), "advisor", "rules", "dataloader.yaml") with open(rule_path, "rb") as file: self.rule = yaml.safe_load(file) def test_no_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) * 1000 + dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=True) checker = DataloaderChecker() @@ -30,20 +30,20 @@ class TestDataloaderChecker(unittest.TestCase): self.assertFalse(checker.dataloader_issues) def test_no_slow_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) * 1000 + dataloader_duration = (self.rule.get("dataloader_duration_threshold") - 1) dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=False) checker = DataloaderChecker() checker.check_slow_dataloader(dataset) self.assertFalse(checker.dataloader_issues) def test_found_slow_dataloader(self): - dataloader_duration = (self.rule.get("dataloader_duration_threshold") + 1) * 1000 + dataloader_duration = (self.rule.get("dataloader_duration_threshold") + 1) dataset = self._get_mock_dataset(dataloader_duration, is_empty_dataset=False) checker = DataloaderChecker() checker.check_slow_dataloader(dataset) self.assertTrue(checker.dataloader_issues) - desc = self.rule.get("problem").format(dataloader_duration=dataloader_duration / 1000, + desc = self.rule.get("problem").format(dataloader_duration=float(dataloader_duration), dataloader_duration_threshold=self.rule.get( "dataloader_duration_threshold")) diff --git a/profiler/test/ut/advisor/timeline_advice/test_memory_op_checker.py b/profiler/test/ut/advisor/timeline_advice/test_memory_op_checker.py new file mode 100644 index 000000000..588874ef3 --- /dev/null +++ b/profiler/test/ut/advisor/timeline_advice/test_memory_op_checker.py @@ -0,0 +1,62 @@ +import unittest +import os +import sys +import yaml + +from profiler.advisor.analyzer.memory.memory_checker import MemoryOpsChecker +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +class TestMemOpChecker(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))), + "advisor", "rules", "memory.yaml") + + with open(rule_path, "rb") as file: + self.rule = yaml.safe_load(file) + + def test_no_mem_op(self): + dataset = self._get_mock_dataset(1, is_empty_dataset=True) + + checker = MemoryOpsChecker() + checker.check_memory_ops(dataset) + self.assertFalse(checker.memory_issues) + + def test_mem_op_not_reach_threshold(self): + dataset = self._get_mock_dataset(1, is_empty_dataset=False) + + checker = MemoryOpsChecker() + checker.check_memory_ops(dataset) + self.assertFalse(checker.memory_issues) + + def test_mem_op_reach_threshold(self): + dataset = self._get_mock_dataset(1, 1000000, is_empty_dataset=False) + + checker = MemoryOpsChecker() + checker.check_memory_ops(dataset) + self.assertTrue(checker.memory_issues) + + def _get_mock_dataset(self, mem_op_num, mem_op_total_dur=10000, is_empty_dataset=False): + dataset = TimelineEvent() + if is_empty_dataset: + return dataset + + mem_op_info = TimelineEvent() + for i in range(mem_op_num): + mem_op_info[f"mock_mem_op_{i}"] = TimelineEvent({"total_dur": mem_op_total_dur, "count": 10}) + + dataset["memory_ops"] = TimelineEvent({"mem_op_info": mem_op_info, "rule": TimelineEvent(self.rule)}) + return dataset + + +if __name__ == '__main__': + tester = TestMemOpChecker() + tester.test_no_mem_op() + tester.test_mem_op_not_reach_threshold() + tester.test_mem_op_reach_threshold() \ No newline at end of file diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py b/profiler/test/ut/advisor/timeline_advice/test_syncbn_checker.py similarity index 95% rename from profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py rename to profiler/test/ut/advisor/timeline_advice/test_syncbn_checker.py index d1df810a0..ecd4ee6cc 100644 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_syncbn_checker.py +++ b/profiler/test/ut/advisor/timeline_advice/test_syncbn_checker.py @@ -15,7 +15,7 @@ class TestSyncBNChecker(unittest.TestCase): def setUp(self) -> None: rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))), "advisor", "rules", "sync_batchnorm.yaml") with open(rule_path, "rb") as file: diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py b/profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py similarity index 95% rename from profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py rename to profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py index 360363ce3..a06a5044a 100644 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_synchronize_stream.py +++ b/profiler/test/ut/advisor/timeline_advice/test_synchronize_stream.py @@ -15,7 +15,7 @@ class TestSynchronizeChecker(unittest.TestCase): def setUp(self) -> None: rule_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname( - os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))))), + os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))), "advisor", "rules", "synchronize.yaml") with open(rule_path, "rb") as file: diff --git a/profiler/test/ut/advisor/timeline_advice/test_timeline_op_collector.py b/profiler/test/ut/advisor/timeline_advice/test_timeline_op_collector.py new file mode 100644 index 000000000..3e1add689 --- /dev/null +++ b/profiler/test/ut/advisor/timeline_advice/test_timeline_op_collector.py @@ -0,0 +1,144 @@ +import unittest +import os +import sys +import yaml + +from profiler.advisor.dataset.timeline_op_collector.timeline_op_collector import ( + OpCompileCollector, + SynchronizeStreamCollector, + MemCollector, + DataloaderCollector, + SyncBNCollector, + AtenCollector, + OptimizerCollector, + FrequencyCollector, + SpecificTaskTypeOpCollector, + TorchToNpuCollector, + AclToNpuCollector, + OpStackCollector, + StepCollector +) +from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +class TestTimelineOpCollector(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + self.mock_step_event = TimelineEvent(dict(name="ProfilerStep#1", ts=1, dur=1000)) + self.mock_op_compile_event = TimelineEvent(dict(name="AscendCL@aclopCompileAndExecute", ts=2, dur=1)) + self.mock_sync_stream_event = TimelineEvent(dict(name="AscendCL@aclrtSynchronizeStream", dur=1000000000)) + self.mock_mem_op_event = TimelineEvent(dict(name="AscendCL@aclMallocMemInner", dur=10)) + self.mock_dataloader_event = TimelineEvent(dict(name="dataloader")) + self.mock_sync_bn_event = TimelineEvent(dict(name="syncbatchnorm")) + self.mock_aten_event = TimelineEvent(dict(name="aten::conv3d")) + self.mock_optimizer_event = TimelineEvent(dict(name="Optimizer.step#")) + self.mock_AI_CPU_event = TimelineEvent( + {"name": "index", "args": TimelineEvent({"Task Type": "AI_CPU"}), "ts": 1}) + self.mock_torch_to_npu_event = TimelineEvent(dict(name="torch_to_npu", tid=1, ts=1, ph=1, id=1)) + self.mock_acl_to_npu_event = TimelineEvent(dict(name="acl_to_npu", ts=1)) + self.mock_op_stack_event = TimelineEvent( + {"name": "aten::conv3d", "dataset_index": 1, "ts": 1, "args": TimelineEvent({"Call stack": "mock_stack"})}) + + def test_step_collector(self): + step_collector = StepCollector() + step_collector.add_op(self.mock_step_event) + step_collector.post_process() + self.assertEqual(step_collector.attribute_to_dataset.get("profiler_step"), [self.mock_step_event]) + + def test_op_compile_collector(self): + op_compile_collector = OpCompileCollector() + op_compile_collector.add_op(self.mock_op_compile_event) + op_compile_collector.post_process(op_compile_collector.op_list) + self.assertEqual(op_compile_collector.attribute_to_dataset.get("ops_compile"), op_compile_collector) + self.assertEqual(op_compile_collector.total_time, 1) + self.assertEqual(op_compile_collector.total_count, 1) + + def test_sync_stream_collector(self): + sync_stream_collector = SynchronizeStreamCollector() + sync_stream_collector.post_process() + self.assertEqual(sync_stream_collector.attribute_to_dataset.get("synchronize_stream"), sync_stream_collector) + + sync_stream_collector.append_slow_sync_stream(self.mock_sync_stream_event) + self.assertEqual(len(sync_stream_collector._slow_synchronize_stream), 1) + + def test_mem_op_collector(self): + mem_op_collector = MemCollector() + mem_op_collector.add_op(self.mock_mem_op_event) + mem_op_collector.post_process(mem_op_collector.op_list) + self.assertEqual(mem_op_collector.attribute_to_dataset.get("memory_ops"), mem_op_collector) + self.assertEqual(mem_op_collector.mem_op_info.get("AscendCL@aclMallocMemInner"), {"count": 1, "total_dur": 10}) + + def test_dataloader_collector(self): + dataloader_collector = DataloaderCollector() + dataloader_collector.add_op(self.mock_dataloader_event) + dataloader_collector.post_process() + self.assertEqual(len(dataloader_collector.attribute_to_dataset.get("dataloader")), 1) + + def test_sync_bn_collector(self): + sync_bn_collector = SyncBNCollector() + sync_bn_collector.add_op(self.mock_sync_bn_event) + sync_bn_collector.post_process(sync_bn_collector.op_list) + self.assertEqual(len(sync_bn_collector.attribute_to_dataset.get("sync_batchnorm")), 1) + + def test_aten_collector(self): + aten_collector = AtenCollector() + aten_collector.add_op(self.mock_aten_event) + aten_collector.add_op(self.mock_sync_stream_event) + aten_collector.post_process(aten_collector.op_list) + self.assertEqual(len(aten_collector.attribute_to_dataset.get("aten")), 2) + + def test_optimizer_collector(self): + optimizer_collector = OptimizerCollector() + optimizer_collector.add_op(self.mock_optimizer_event) + optimizer_collector.post_process(optimizer_collector.op_list) + self.assertEqual(len(optimizer_collector.attribute_to_dataset.get("optimizer")), 1) + + def test_specific_task_type_op_collector(self): + specific_task_type_op_collector = SpecificTaskTypeOpCollector() + specific_task_type_op_collector.add_op(self.mock_AI_CPU_event) + specific_task_type_op_collector.post_process(specific_task_type_op_collector.op_list) + key = f"{self.mock_AI_CPU_event.name}-{self.mock_AI_CPU_event.ts}" + self.assertTrue( + specific_task_type_op_collector.attribute_to_dataset.get("ops_with_task_type", {}).get(key)) + self.assertTrue(specific_task_type_op_collector.attribute_to_dataset.get("task_op_names"), [key]) + + def test_torch_to_npu_collector(self): + torch_to_npu_collector = TorchToNpuCollector() + torch_to_npu_collector.add_op(self.mock_torch_to_npu_event) + torch_to_npu_collector.post_process(torch_to_npu_collector.op_list) + key = f"{self.mock_torch_to_npu_event.ph}-{self.mock_torch_to_npu_event.id}" + self.assertTrue("1-1" in torch_to_npu_collector.attribute_to_dataset.get("torch_to_npu")) + + def test_acl_to_npu_collector(self): + acl_to_npu_collector = AclToNpuCollector() + acl_to_npu_collector.add_op(self.mock_acl_to_npu_event) + acl_to_npu_collector.post_process(acl_to_npu_collector.op_list) + self.assertEqual(acl_to_npu_collector.attribute_to_dataset.get("acl_to_npu"), + set([str(self.mock_acl_to_npu_event.ts)])) + + def test_op_stack_collector(self): + op_stack_collector = OpStackCollector() + op_stack_collector.add_op(self.mock_op_stack_event) + op_stack_collector.post_process(op_stack_collector.op_list) + self.assertTrue( + str(self.mock_op_stack_event.ts) in op_stack_collector.attribute_to_dataset.get("ops_with_stack")) + + +if __name__ == '__main__': + tester = TestTimelineOpCollector() + tester.test_step_collector() + tester.test_op_compile_collector() + tester.test_sync_stream_collector() + tester.test_mem_op_collector() + tester.test_dataloader_collector() + tester.test_sync_bn_collector() + tester.test_aten_collector() + tester.test_optimizer_collector() + tester.test_specific_task_type_op_collector() + tester.test_torch_to_npu_collector() + tester.test_acl_to_npu_collector() + tester.test_op_stack_collector() diff --git a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py b/profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py similarity index 85% rename from profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py rename to profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py index 9060bfb8d..3ebdcddad 100644 --- a/profiler/test/ut/advisor/advisor_backend/timeline_advice/test_timeline_op_compile_checker.py +++ b/profiler/test/ut/advisor/timeline_advice/test_timeline_op_compile_checker.py @@ -2,12 +2,13 @@ import unittest import os import sys -work_path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))))) +work_path = os.path.dirname(os.path.dirname( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__)))))))) sys.path.insert(0, work_path) from unittest.mock import patch from profiler.advisor.analyzer.schedule import dispatch from profiler.advisor.analyzer.schedule.dispatch.timeline_op_dispatch_analyzer import OpDispatchAnalyzer -from profiler.advisor.dataset.timeline_event_dataset import TimelineEventDataset +from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.display.html.render import HTMLRender from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env @@ -32,13 +33,13 @@ class TestOperatorDispatchAnalyzer(unittest.TestCase): kwargs = {"analysis_mode": "timeline"} data_root_dir = os.path.dirname(os.path.realpath(__file__)) op_dispatch = OpDispatchAnalyzer(data_root_dir, **kwargs) - event_dataset = op_dispatch.get_first_data_by_key(op_dispatch.dataset_list, TimelineEventDataset.get_key()) + event_dataset = op_dispatch.get_first_data_by_key(op_dispatch.dataset_list, ScheduleAnalysisDataset.get_key()) op_dispatch.get_op_compile_info(event_dataset) html_render = HTMLRender() op_dispatch.make_render(html_render) self.assertTrue(len(html_render.render_list) >= 1) - + if __name__ == '__main__': tester = TestOperatorDispatchAnalyzer() diff --git a/profiler/test/ut/advisor/timeline_advice/trace_view.json b/profiler/test/ut/advisor/timeline_advice/trace_view.json new file mode 100644 index 000000000..f622e9ea3 --- /dev/null +++ b/profiler/test/ut/advisor/timeline_advice/trace_view.json @@ -0,0 +1 @@ +[{"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511420306.491", "dur": 13.08, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511420439.502", "dur": 4.0, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty_strided", "pid": 437675, "tid": 437675, "ts": "1704161511420514.763", "dur": 30.05, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::copy_", "pid": 437675, "tid": 437675, "ts": "1704161511420569.933", "dur": 89.811, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_to_copy", "pid": 437675, "tid": 437675, "ts": "1704161511420489.413", "dur": 171.191, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 437675, "ts": "1704161511420476.853", "dur": 184.121, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "detach_", "pid": 437675, "tid": 437675, "ts": "1704161511420689.695", "dur": 3.89, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::detach_", "pid": 437675, "tid": 437675, "ts": "1704161511420687.045", "dur": 6.88, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511420741.285", "dur": 3.4, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty_strided", "pid": 437675, "tid": 437675, "ts": "1704161511420751.095", "dur": 11.94, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::copy_", "pid": 437675, "tid": 437675, "ts": "1704161511420764.115", "dur": 35.781, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_to_copy", "pid": 437675, "tid": 437675, "ts": "1704161511420748.265", "dur": 52.351, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 437675, "ts": "1704161511420746.855", "dur": 53.991, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "detach_", "pid": 437675, "tid": 437675, "ts": "1704161511420804.436", "dur": 1.49, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::detach_", "pid": 437675, "tid": 437675, "ts": "1704161511420803.416", "dur": 2.74, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511420990.368", "dur": 3.69, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::random_", "pid": 437675, "tid": 437675, "ts": "1704161511421012.868", "dur": 25.08, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511421076.319", "dur": 3.13, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511421045.538", "dur": 34.401, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511421130.789", "dur": 2.55, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511421931.457", "dur": 2.51, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::resize_", "pid": 437675, "tid": 437675, "ts": "1704161511421944.087", "dur": 4.58, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511421982.858", "dur": 3.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::slice", "pid": 437675, "tid": 437675, "ts": "1704161511421954.837", "dur": 33.121, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::narrow", "pid": 437675, "tid": 437675, "ts": "1704161511421951.317", "dur": 36.881, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_cat", "pid": 437675, "tid": 437675, "ts": "1704161511421926.977", "dur": 223.222, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::cat", "pid": 437675, "tid": 437675, "ts": "1704161511421921.707", "dur": 229.902, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 437675, "ts": "1704161511422154.589", "dur": 4.97, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::stack", "pid": 437675, "tid": 437675, "ts": "1704161511421907.547", "dur": 256.452, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422191.510", "dur": 1.72, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422189.670", "dur": 4.52, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422195.410", "dur": 0.91, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422194.740", "dur": 1.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422198.150", "dur": 0.66, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422197.140", "dur": 1.91, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422200.180", "dur": 1.22, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422199.450", "dur": 2.18, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422202.670", "dur": 0.92, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422202.030", "dur": 1.76, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422204.780", "dur": 0.95, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422204.160", "dur": 10.11, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422215.720", "dur": 1.04, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422214.890", "dur": 2.16, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422218.100", "dur": 0.89, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422217.510", "dur": 1.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422220.320", "dur": 1.09, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422219.650", "dur": 1.98, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422222.590", "dur": 0.8, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422221.990", "dur": 1.62, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422224.730", "dur": 0.94, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422224.100", "dur": 1.77, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422226.800", "dur": 0.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422226.250", "dur": 1.51, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422228.670", "dur": 0.64, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422228.110", "dur": 1.44, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422230.590", "dur": 1.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422229.940", "dur": 1.88, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422232.870", "dur": 0.9, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422232.180", "dur": 1.8, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422234.930", "dur": 0.9, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422234.360", "dur": 1.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422237.050", "dur": 0.93, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422236.400", "dur": 1.8, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422239.170", "dur": 1.23, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422238.580", "dur": 2.03, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422241.620", "dur": 0.96, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422240.970", "dur": 1.82, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422243.760", "dur": 0.85, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422243.170", "dur": 1.64, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422245.800", "dur": 0.98, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422245.180", "dur": 1.81, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422248.170", "dur": 0.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422247.380", "dur": 1.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422250.200", "dur": 0.97, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422249.620", "dur": 1.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422252.520", "dur": 0.85, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422251.730", "dur": 1.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422254.560", "dur": 1.12, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422253.940", "dur": 1.98, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422256.930", "dur": 0.97, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422256.300", "dur": 1.79, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422259.060", "dur": 0.88, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422258.450", "dur": 1.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422261.140", "dur": 0.95, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422260.530", "dur": 1.76, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422263.320", "dur": 0.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422262.680", "dur": 1.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422265.460", "dur": 0.95, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422264.900", "dur": 1.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422268.020", "dur": 0.97, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422266.990", "dur": 2.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422270.200", "dur": 0.88, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422269.580", "dur": 1.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422272.280", "dur": 1.27, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422271.670", "dur": 2.1, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422274.810", "dur": 0.89, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422274.140", "dur": 1.77, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422276.920", "dur": 0.841, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422276.300", "dur": 1.691, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422279.021", "dur": 1.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422278.351", "dur": 2.09, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422281.421", "dur": 0.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422280.801", "dur": 1.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422283.441", "dur": 0.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422282.841", "dur": 1.79, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422285.681", "dur": 0.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422285.021", "dur": 1.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422287.701", "dur": 0.91, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422287.111", "dur": 1.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422289.841", "dur": 1.17, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422289.191", "dur": 2.03, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422292.171", "dur": 1.14, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422291.601", "dur": 1.92, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422294.811", "dur": 0.93, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422293.911", "dur": 2.05, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422296.921", "dur": 0.64, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422296.311", "dur": 1.45, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422298.821", "dur": 0.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422298.151", "dur": 1.57, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422300.701", "dur": 0.64, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422300.081", "dur": 1.46, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422302.531", "dur": 0.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422301.911", "dur": 1.54, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422304.441", "dur": 1.08, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422303.821", "dur": 1.9, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422306.731", "dur": 1.19, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422306.111", "dur": 2.05, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422309.191", "dur": 0.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422308.561", "dur": 1.54, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422311.071", "dur": 0.63, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422310.481", "dur": 1.45, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422313.331", "dur": 1.19, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422312.311", "dur": 2.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422315.761", "dur": 0.93, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422315.121", "dur": 1.78, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422317.901", "dur": 0.85, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422317.291", "dur": 1.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422319.991", "dur": 0.76, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422319.331", "dur": 1.64, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422321.911", "dur": 0.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422321.341", "dur": 1.61, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422324.021", "dur": 0.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422323.311", "dur": 1.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422326.061", "dur": 0.88, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422325.451", "dur": 1.72, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422328.221", "dur": 0.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422327.561", "dur": 1.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422330.271", "dur": 0.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422329.661", "dur": 1.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422332.421", "dur": 0.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422331.701", "dur": 1.69, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422334.611", "dur": 1.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422333.761", "dur": 2.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422336.691", "dur": 0.91, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422336.221", "dur": 1.59, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422338.841", "dur": 0.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::unsqueeze", "pid": 437675, "tid": 437675, "ts": "1704161511422338.191", "dur": 1.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511422345.381", "dur": 2.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::resize_", "pid": 437675, "tid": 437675, "ts": "1704161511422354.111", "dur": 1.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422359.891", "dur": 1.12, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::slice", "pid": 437675, "tid": 437675, "ts": "1704161511422358.641", "dur": 2.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::narrow", "pid": 437675, "tid": 437675, "ts": "1704161511422356.901", "dur": 4.76, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_cat", "pid": 437675, "tid": 437675, "ts": "1704161511422342.831", "dur": 31.48, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::cat", "pid": 437675, "tid": 437675, "ts": "1704161511422341.061", "dur": 33.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::stack", "pid": 437675, "tid": 437675, "ts": "1704161511422185.480", "dur": 197.022, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "enumerate(DataLoader)#_SingleProcessDataLoaderIter.__next__", "pid": 437675, "tid": 437675, "ts": "1704161511421128.989", "dur": 1302.013, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422488.043", "dur": 15.06, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::copy_", "pid": 437675, "tid": 437675, "ts": "1704161511422504.663", "dur": 114.901, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_to_copy", "pid": 437675, "tid": 437675, "ts": "1704161511422484.433", "dur": 136.051, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 437675, "ts": "1704161511422480.413", "dur": 140.541, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty_strided", "pid": 437675, "tid": 437675, "ts": "1704161511422659.624", "dur": 10.04, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::copy_", "pid": 437675, "tid": 437675, "ts": "1704161511422670.674", "dur": 33.731, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_to_copy", "pid": 437675, "tid": 437675, "ts": "1704161511422657.094", "dur": 48.481, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 437675, "ts": "1704161511422655.244", "dur": 50.611, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511422740.895", "dur": 2.8, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422855.116", "dur": 16.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422798.066", "dur": 76.61, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422892.817", "dur": 2.56, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422885.757", "dur": 10.62, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422907.207", "dur": 1.82, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422903.067", "dur": 6.77, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422920.807", "dur": 1.94, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422915.777", "dur": 7.59, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422933.197", "dur": 2.56, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422929.067", "dur": 7.65, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422945.917", "dur": 2.85, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422942.277", "dur": 7.08, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422958.477", "dur": 1.88, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422954.877", "dur": 5.96, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422970.767", "dur": 3.91, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422966.207", "dur": 9.07, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422984.268", "dur": 2.08, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422980.658", "dur": 6.27, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceZero", "pid": 437675, "tid": 437675, "ts": "1704161511422996.918", "dur": 2.39, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::zero_", "pid": 437675, "tid": 437675, "ts": "1704161511422992.538", "dur": 7.27, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "Optimizer.zero_grad#Adam.zero_grad", "pid": 437675, "tid": 437675, "ts": "1704161511422737.645", "dur": 268.043, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "Conv2D", "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "dur": 9.18, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_convolution", "pid": 437675, "tid": 437675, "ts": "1704161511423143.989", "dur": 73.581, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::convolution", "pid": 437675, "tid": 437675, "ts": "1704161511423098.389", "dur": 123.431, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::conv2d", "pid": 437675, "tid": 437675, "ts": "1704161511423070.928", "dur": 151.712, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnRelu", "pid": 437675, "tid": 437675, "ts": "1704161511423257.470", "dur": 2.45, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::relu", "pid": 437675, "tid": 437675, "ts": "1704161511423234.760", "dur": 31.78, "cat": "cpu_op", "args": {"Sequence number": 18, "Fwd thread id": 0}}, {"ph": "X", "name": "MaxPoolWithArgmaxV1", "pid": 437675, "tid": 437675, "ts": "1704161511423396.302", "dur": 3.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::max_pool2d_with_indices", "pid": 437675, "tid": 437675, "ts": "1704161511423348.621", "dur": 55.691, "cat": "cpu_op", "args": {"Sequence number": 19, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::max_pool2d", "pid": 437675, "tid": 437675, "ts": "1704161511423335.201", "dur": 69.641, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "Conv2D", "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "dur": 2.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_convolution", "pid": 437675, "tid": 437675, "ts": "1704161511423441.942", "dur": 30.45, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::convolution", "pid": 437675, "tid": 437675, "ts": "1704161511423433.722", "dur": 40.22, "cat": "cpu_op", "args": {"Sequence number": 20, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::conv2d", "pid": 437675, "tid": 437675, "ts": "1704161511423431.762", "dur": 42.62, "cat": "cpu_op", "args": {"Sequence number": 20, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnRelu", "pid": 437675, "tid": 437675, "ts": "1704161511423495.183", "dur": 2.09, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::relu", "pid": 437675, "tid": 437675, "ts": "1704161511423480.703", "dur": 20.89, "cat": "cpu_op", "args": {"Sequence number": 21, "Fwd thread id": 0}}, {"ph": "X", "name": "MaxPoolWithArgmaxV1", "pid": 437675, "tid": 437675, "ts": "1704161511423553.623", "dur": 2.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::max_pool2d_with_indices", "pid": 437675, "tid": 437675, "ts": "1704161511423526.163", "dur": 32.86, "cat": "cpu_op", "args": {"Sequence number": 22, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::max_pool2d", "pid": 437675, "tid": 437675, "ts": "1704161511423524.283", "dur": 35.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 437675, "ts": "1704161511423572.793", "dur": 16.891, "cat": "cpu_op", "args": {"Sequence number": 23, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511423624.774", "dur": 4.49, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 437675, "ts": "1704161511423622.474", "dur": 7.47, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 437675, "ts": "1704161511423615.164", "dur": 17.46, "cat": "cpu_op", "args": {"Sequence number": 24, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnAddmm", "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "dur": 2.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addmm", "pid": 437675, "tid": 437675, "ts": "1704161511423634.704", "dur": 39.8, "cat": "cpu_op", "args": {"Sequence number": 25, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::linear", "pid": 437675, "tid": 437675, "ts": "1704161511423612.274", "dur": 62.66, "cat": "cpu_op", "args": {"Sequence number": 24, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnRelu", "pid": 437675, "tid": 437675, "ts": "1704161511423691.065", "dur": 2.2, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::relu", "pid": 437675, "tid": 437675, "ts": "1704161511423680.365", "dur": 16.36, "cat": "cpu_op", "args": {"Sequence number": 26, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511423719.815", "dur": 2.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 437675, "ts": "1704161511423718.335", "dur": 4.15, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 437675, "ts": "1704161511423715.015", "dur": 9.53, "cat": "cpu_op", "args": {"Sequence number": 27, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnAddmm", "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "dur": 2.05, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addmm", "pid": 437675, "tid": 437675, "ts": "1704161511423725.355", "dur": 21.78, "cat": "cpu_op", "args": {"Sequence number": 28, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::linear", "pid": 437675, "tid": 437675, "ts": "1704161511423713.815", "dur": 33.64, "cat": "cpu_op", "args": {"Sequence number": 27, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnRelu", "pid": 437675, "tid": 437675, "ts": "1704161511423760.635", "dur": 2.37, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::relu", "pid": 437675, "tid": 437675, "ts": "1704161511423751.965", "dur": 13.89, "cat": "cpu_op", "args": {"Sequence number": 29, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 437675, "ts": "1704161511423787.926", "dur": 1.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 437675, "ts": "1704161511423786.666", "dur": 3.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 437675, "ts": "1704161511423783.516", "dur": 8.91, "cat": "cpu_op", "args": {"Sequence number": 30, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnAddmm", "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "dur": 2.05, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addmm", "pid": 437675, "tid": 437675, "ts": "1704161511423793.646", "dur": 20.37, "cat": "cpu_op", "args": {"Sequence number": 31, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::linear", "pid": 437675, "tid": 437675, "ts": "1704161511423782.416", "dur": 31.89, "cat": "cpu_op", "args": {"Sequence number": 30, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnLogSoftmax", "pid": 437675, "tid": 437675, "ts": "1704161511426449.252", "dur": 9.33, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_log_softmax", "pid": 437675, "tid": 437675, "ts": "1704161511426415.582", "dur": 49.78, "cat": "cpu_op", "args": {"Sequence number": 32, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::log_softmax", "pid": 437675, "tid": 437675, "ts": "1704161511426394.022", "dur": 72.14, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceOne", "pid": 437675, "tid": 437675, "ts": "1704161511426532.533", "dur": 6.39, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::ones", "pid": 437675, "tid": 437675, "ts": "1704161511426519.463", "dur": 20.13, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnNLLLoss", "pid": 437675, "tid": 437675, "ts": "1704161511426550.453", "dur": 2.32, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::nll_loss_forward", "pid": 437675, "tid": 437675, "ts": "1704161511426502.433", "dur": 55.71, "cat": "cpu_op", "args": {"Sequence number": 33, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::nll_loss", "pid": 437675, "tid": 437675, "ts": "1704161511426491.313", "dur": 67.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::nll_loss_nd", "pid": 437675, "tid": 437675, "ts": "1704161511426488.323", "dur": 70.85, "cat": "cpu_op", "args": {"Sequence number": 33, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::cross_entropy_loss", "pid": 437675, "tid": 437675, "ts": "1704161511426369.121", "dur": 190.602, "cat": "cpu_op", "args": {"Sequence number": 32, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceOne", "pid": 437675, "tid": 437675, "ts": "1704161511426617.294", "dur": 7.18, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::ones_like", "pid": 437675, "tid": 437675, "ts": "1704161511426604.814", "dur": 20.82, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceOne", "pid": 437675, "tid": 439228, "ts": "1704161511426816.966", "dur": 12.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::ones", "pid": 437675, "tid": 439228, "ts": "1704161511426785.246", "dur": 45.8, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnNLLLossBackward", "pid": 437675, "tid": 439228, "ts": "1704161511426846.736", "dur": 6.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::nll_loss_backward", "pid": 437675, "tid": 439228, "ts": "1704161511426756.445", "dur": 102.401, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "NllLossBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511426719.085", "dur": 140.901, "cat": "cpu_op", "args": {"Sequence number": 33, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: NllLossBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511426712.505", "dur": 157.371, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnLogSoftmaxBackward", "pid": 437675, "tid": 439228, "ts": "1704161511426924.287", "dur": 8.34, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_log_softmax_backward_data", "pid": 437675, "tid": 439228, "ts": "1704161511426905.107", "dur": 29.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "LogSoftmaxBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511426883.167", "dur": 52.65, "cat": "cpu_op", "args": {"Sequence number": 32, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: LogSoftmaxBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511426881.907", "dur": 60.15, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511426964.327", "dur": 5.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511426961.747", "dur": 8.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511426956.237", "dur": 17.78, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnMm", "pid": 437675, "tid": 439228, "ts": "1704161511426997.638", "dur": 7.2, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mm", "pid": 437675, "tid": 439228, "ts": "1704161511426976.137", "dur": 29.781, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427012.258", "dur": 2.9, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427011.048", "dur": 4.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427009.298", "dur": 7.94, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnMm", "pid": 437675, "tid": 439228, "ts": "1704161511427030.778", "dur": 2.46, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mm", "pid": 437675, "tid": 439228, "ts": "1704161511427018.108", "dur": 15.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427037.468", "dur": 1.23, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427035.728", "dur": 3.29, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427034.588", "dur": 5.28, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "AddmmBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511426950.567", "dur": 90.181, "cat": "cpu_op", "args": {"Sequence number": 31, "Fwd thread id": 1}}, {"ph": "X", "name": "aclnnReduceSum", "pid": 437675, "tid": 439228, "ts": "1704161511427082.058", "dur": 3.501, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sum", "pid": 437675, "tid": 439228, "ts": "1704161511427066.188", "dur": 20.631, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 439228, "ts": "1704161511427088.509", "dur": 7.29, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "autograd::engine::evaluate_function: AddmmBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511426948.817", "dur": 159.982, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511427259.190", "dur": 7.57, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511427151.909", "dur": 118.171, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427126.039", "dur": 148.611, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427123.179", "dur": 152.681, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427292.441", "dur": 2.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427290.691", "dur": 5.1, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427287.481", "dur": 10.42, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "TBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427285.141", "dur": 13.09, "cat": "cpu_op", "args": {"Sequence number": 30, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427283.891", "dur": 17.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511427351.201", "dur": 8.27, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511427313.851", "dur": 46.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427310.581", "dur": 54.27, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427308.501", "dur": 57.55, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnThresholdBackward", "pid": 437675, "tid": 439228, "ts": "1704161511427415.592", "dur": 9.4, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::threshold_backward", "pid": 437675, "tid": 439228, "ts": "1704161511427395.782", "dur": 31.02, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ReluBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427373.241", "dur": 54.731, "cat": "cpu_op", "args": {"Sequence number": 29, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: ReluBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427371.891", "dur": 62.291, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427449.542", "dur": 2.31, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427447.412", "dur": 5.15, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427444.652", "dur": 9.82, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnMm", "pid": 437675, "tid": 439228, "ts": "1704161511427471.612", "dur": 7.59, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mm", "pid": 437675, "tid": 439228, "ts": "1704161511427455.412", "dur": 24.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427484.723", "dur": 2.46, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427483.373", "dur": 4.52, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427481.962", "dur": 7.201, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnMm", "pid": 437675, "tid": 439228, "ts": "1704161511427501.723", "dur": 2.6, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mm", "pid": 437675, "tid": 439228, "ts": "1704161511427489.863", "dur": 14.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427508.723", "dur": 1.37, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427507.033", "dur": 3.38, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427505.693", "dur": 5.55, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "AddmmBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427440.652", "dur": 71.391, "cat": "cpu_op", "args": {"Sequence number": 28, "Fwd thread id": 1}}, {"ph": "X", "name": "aclnnReduceSum", "pid": 437675, "tid": 439228, "ts": "1704161511427522.133", "dur": 3.0, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sum", "pid": 437675, "tid": 439228, "ts": "1704161511427513.273", "dur": 12.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 439228, "ts": "1704161511427527.023", "dur": 6.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "autograd::engine::evaluate_function: AddmmBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427439.312", "dur": 104.601, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511427593.064", "dur": 7.41, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511427556.403", "dur": 45.311, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427554.133", "dur": 50.761, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427551.563", "dur": 54.281, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427618.334", "dur": 2.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427616.474", "dur": 4.45, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427614.614", "dur": 8.34, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "TBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427613.274", "dur": 9.97, "cat": "cpu_op", "args": {"Sequence number": 27, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427612.124", "dur": 14.39, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511427667.914", "dur": 8.03, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511427635.604", "dur": 41.44, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427633.654", "dur": 47.02, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427631.964", "dur": 49.66, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnThresholdBackward", "pid": 437675, "tid": 439228, "ts": "1704161511427704.725", "dur": 2.66, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::threshold_backward", "pid": 437675, "tid": 439228, "ts": "1704161511427691.945", "dur": 16.42, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ReluBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427687.485", "dur": 21.74, "cat": "cpu_op", "args": {"Sequence number": 26, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: ReluBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427686.325", "dur": 28.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427726.845", "dur": 2.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427725.485", "dur": 4.41, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427723.555", "dur": 7.96, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnMm", "pid": 437675, "tid": 439228, "ts": "1704161511427746.585", "dur": 7.37, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mm", "pid": 437675, "tid": 439228, "ts": "1704161511427732.395", "dur": 22.28, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427758.545", "dur": 1.87, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427757.545", "dur": 3.44, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427756.275", "dur": 5.78, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnMm", "pid": 437675, "tid": 439228, "ts": "1704161511427861.856", "dur": 7.4, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mm", "pid": 437675, "tid": 439228, "ts": "1704161511427762.755", "dur": 107.641, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511427876.236", "dur": 2.04, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511427873.866", "dur": 5.02, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427871.806", "dur": 8.62, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "AddmmBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427720.065", "dur": 161.751, "cat": "cpu_op", "args": {"Sequence number": 25, "Fwd thread id": 1}}, {"ph": "X", "name": "aclnnReduceSum", "pid": 437675, "tid": 439228, "ts": "1704161511427894.447", "dur": 2.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sum", "pid": 437675, "tid": 439228, "ts": "1704161511427883.337", "dur": 14.82, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 439228, "ts": "1704161511427899.567", "dur": 5.47, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "autograd::engine::evaluate_function: AddmmBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427719.175", "dur": 198.842, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511427972.097", "dur": 9.4, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511427934.447", "dur": 48.55, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427932.037", "dur": 54.381, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511427929.467", "dur": 58.111, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::as_strided", "pid": 437675, "tid": 439228, "ts": "1704161511428002.138", "dur": 1.92, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::transpose", "pid": 437675, "tid": 439228, "ts": "1704161511428000.428", "dur": 4.14, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::t", "pid": 437675, "tid": 439228, "ts": "1704161511427998.238", "dur": 7.98, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "TBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427996.438", "dur": 10.14, "cat": "cpu_op", "args": {"Sequence number": 24, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: TBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511427994.928", "dur": 14.9, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511428050.588", "dur": 7.87, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511428019.038", "dur": 40.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428017.138", "dur": 46.09, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428015.518", "dur": 48.69, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 439228, "ts": "1704161511428076.568", "dur": 6.49, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::reshape", "pid": 437675, "tid": 439228, "ts": "1704161511428073.448", "dur": 9.991, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ViewBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428070.708", "dur": 13.441, "cat": "cpu_op", "args": {"Sequence number": 23, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: ViewBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428069.748", "dur": 16.931, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "format_contiguousV2", "pid": 437675, "tid": 439228, "ts": "1704161511428141.569", "dur": 4.77, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "MaxPoolGradWithArgmaxV1", "pid": 437675, "tid": 439228, "ts": "1704161511428164.169", "dur": 7.33, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::max_pool2d_with_indices_backward", "pid": 437675, "tid": 439228, "ts": "1704161511428113.029", "dur": 62.16, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "MaxPool2DWithIndicesBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428092.079", "dur": 83.84, "cat": "cpu_op", "args": {"Sequence number": 22, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: MaxPool2DWithIndicesBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428090.999", "dur": 96.351, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnThresholdBackward", "pid": 437675, "tid": 439228, "ts": "1704161511428213.150", "dur": 2.63, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::threshold_backward", "pid": 437675, "tid": 439228, "ts": "1704161511428198.560", "dur": 18.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ReluBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428193.900", "dur": 23.43, "cat": "cpu_op", "args": {"Sequence number": 21, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: ReluBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428192.750", "dur": 29.51, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 439228, "ts": "1704161511428283.661", "dur": 6.44, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty_strided", "pid": 437675, "tid": 439228, "ts": "1704161511428297.951", "dur": 3.41, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::copy_", "pid": 437675, "tid": 439228, "ts": "1704161511428302.681", "dur": 21.46, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_to_copy", "pid": 437675, "tid": 439228, "ts": "1704161511428295.221", "dur": 29.32, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 439228, "ts": "1704161511428292.131", "dur": 32.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "Conv2DBackpropInput", "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "dur": 3.15, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 439228, "ts": "1704161511428357.691", "dur": 2.48, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty_strided", "pid": 437675, "tid": 439228, "ts": "1704161511428363.201", "dur": 1.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::copy_", "pid": 437675, "tid": 439228, "ts": "1704161511428365.621", "dur": 4.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_to_copy", "pid": 437675, "tid": 439228, "ts": "1704161511428361.881", "dur": 8.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 439228, "ts": "1704161511428360.961", "dur": 9.56, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "Conv2DBackpropFilter", "pid": 437675, "tid": 439228, "ts": "1704161511428382.811", "dur": 2.731, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 439228, "ts": "1704161511428389.642", "dur": 3.26, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "contiguous_h_memRepoint", "pid": 437675, "tid": 439228, "ts": "1704161511428418.612", "dur": 2.65, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "format_contiguousV2", "pid": 437675, "tid": 439228, "ts": "1704161511428400.652", "dur": 21.52, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 439228, "ts": "1704161511428425.572", "dur": 1.76, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ReduceSum", "pid": 437675, "tid": 439228, "ts": "1704161511428434.892", "dur": 2.06, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceCopy", "pid": 437675, "tid": 439228, "ts": "1704161511428525.263", "dur": 2.91, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::clone", "pid": 437675, "tid": 439228, "ts": "1704161511428478.562", "dur": 50.851, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::npu_dtype_cast", "pid": 437675, "tid": 439228, "ts": "1704161511428473.402", "dur": 57.331, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::convolution_backward", "pid": 437675, "tid": 439228, "ts": "1704161511428248.750", "dur": 287.123, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ConvolutionBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428228.320", "dur": 308.573, "cat": "cpu_op", "args": {"Sequence number": 20, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: ConvolutionBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428226.910", "dur": 321.963, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511428602.694", "dur": 7.19, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511428567.123", "dur": 44.391, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428563.893", "dur": 50.721, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428561.693", "dur": 53.811, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511428655.254", "dur": 2.35, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511428628.904", "dur": 29.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428626.724", "dur": 34.35, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428624.854", "dur": 36.87, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "MaxPoolGradWithArgmaxV1", "pid": 437675, "tid": 439228, "ts": "1704161511428703.785", "dur": 2.63, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::max_pool2d_with_indices_backward", "pid": 437675, "tid": 439228, "ts": "1704161511428671.624", "dur": 36.141, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "MaxPool2DWithIndicesBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428668.254", "dur": 40.011, "cat": "cpu_op", "args": {"Sequence number": 19, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: MaxPool2DWithIndicesBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428667.204", "dur": 52.221, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnThresholdBackward", "pid": 437675, "tid": 439228, "ts": "1704161511428743.655", "dur": 2.22, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::threshold_backward", "pid": 437675, "tid": 439228, "ts": "1704161511428730.715", "dur": 15.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ReluBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428726.245", "dur": 21.17, "cat": "cpu_op", "args": {"Sequence number": 18, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: ReluBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428724.535", "dur": 27.23, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 439228, "ts": "1704161511428781.445", "dur": 4.761, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty_strided", "pid": 437675, "tid": 439228, "ts": "1704161511428790.906", "dur": 2.74, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::copy_", "pid": 437675, "tid": 439228, "ts": "1704161511428794.766", "dur": 9.25, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_to_copy", "pid": 437675, "tid": 439228, "ts": "1704161511428789.006", "dur": 15.36, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 439228, "ts": "1704161511428787.566", "dur": 17.17, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "Conv2DBackpropFilter", "pid": 437675, "tid": 439228, "ts": "1704161511428819.646", "dur": 2.52, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 439228, "ts": "1704161511428824.346", "dur": 3.54, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "contiguous_h_memRepoint", "pid": 437675, "tid": 439228, "ts": "1704161511428841.666", "dur": 2.15, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "format_contiguousV2", "pid": 437675, "tid": 439228, "ts": "1704161511428831.896", "dur": 12.62, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 439228, "ts": "1704161511428848.066", "dur": 2.22, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ReduceSum", "pid": 437675, "tid": 439228, "ts": "1704161511428855.646", "dur": 2.08, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceCopy", "pid": 437675, "tid": 439228, "ts": "1704161511428891.997", "dur": 2.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::clone", "pid": 437675, "tid": 439228, "ts": "1704161511428862.486", "dur": 33.391, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::npu_dtype_cast", "pid": 437675, "tid": 439228, "ts": "1704161511428860.526", "dur": 35.981, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::convolution_backward", "pid": 437675, "tid": 439228, "ts": "1704161511428759.645", "dur": 141.002, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "ConvolutionBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428757.055", "dur": 144.382, "cat": "cpu_op", "args": {"Sequence number": 17, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: ConvolutionBackward0", "pid": 437675, "tid": 439228, "ts": "1704161511428755.985", "dur": 151.492, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511428953.477", "dur": 2.42, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511428921.497", "dur": 35.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428918.797", "dur": 41.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428916.747", "dur": 44.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 439228, "ts": "1704161511428995.778", "dur": 2.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 439228, "ts": "1704161511428971.757", "dur": 28.161, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428969.937", "dur": 32.301, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 1}}, {"ph": "X", "name": "autograd::engine::evaluate_function: torch::autograd::AccumulateGrad", "pid": 437675, "tid": 439228, "ts": "1704161511428968.057", "dur": 34.961, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511429058.878", "dur": 4.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429178.589", "dur": 1.6, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429176.179", "dur": 4.59, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511429190.490", "dur": 8.74, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511429165.879", "dur": 34.861, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511429220.330", "dur": 2.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511429211.280", "dur": 12.76, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429233.240", "dur": 0.42, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429232.390", "dur": 1.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511429237.520", "dur": 2.28, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511429230.170", "dur": 10.31, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511429259.120", "dur": 1.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511429249.450", "dur": 13.41, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511429284.691", "dur": 1.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511429267.090", "dur": 20.361, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429314.501", "dur": 0.46, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429313.271", "dur": 2.19, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511429396.712", "dur": 3.39, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511429301.561", "dur": 100.651, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429420.402", "dur": 0.49, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429419.312", "dur": 2.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511429427.092", "dur": 2.05, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511429415.522", "dur": 14.92, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511429503.263", "dur": 3.24, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511429437.712", "dur": 70.711, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429524.353", "dur": 0.95, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429523.523", "dur": 2.24, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511429530.753", "dur": 5.81, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511429520.073", "dur": 17.31, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511429551.083", "dur": 2.36, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511429544.703", "dur": 9.46, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429562.543", "dur": 0.44, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429561.463", "dur": 1.93, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511429566.633", "dur": 2.03, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511429559.373", "dur": 9.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511429580.403", "dur": 2.28, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511429574.843", "dur": 8.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511429597.614", "dur": 2.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511429585.954", "dur": 14.48, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429617.274", "dur": 0.69, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429616.034", "dur": 2.41, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511429657.214", "dur": 2.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511429608.354", "dur": 52.95, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429677.804", "dur": 0.74, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429676.884", "dur": 2.12, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511429683.444", "dur": 2.331, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511429673.574", "dur": 12.991, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511429739.045", "dur": 2.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511429695.985", "dur": 46.77, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429759.365", "dur": 0.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429758.425", "dur": 1.92, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511429764.755", "dur": 6.94, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511429755.035", "dur": 17.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511429783.905", "dur": 1.681, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511429778.045", "dur": 8.381, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429801.536", "dur": 1.91, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429800.766", "dur": 3.33, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511429807.856", "dur": 6.63, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511429798.686", "dur": 16.37, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511429826.966", "dur": 2.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511429820.676", "dur": 9.24, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511429844.736", "dur": 1.48, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511429832.366", "dur": 14.58, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429863.306", "dur": 1.09, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429862.376", "dur": 2.38, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511429904.687", "dur": 3.13, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511429854.746", "dur": 54.321, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511429925.047", "dur": 0.47, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511429924.137", "dur": 2.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511429930.557", "dur": 1.82, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511429921.117", "dur": 12.11, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511429986.048", "dur": 2.36, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511429941.267", "dur": 48.181, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430003.358", "dur": 0.39, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430002.518", "dur": 1.59, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430008.578", "dur": 2.17, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511429998.998", "dur": 12.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511430021.828", "dur": 9.61, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430016.448", "dur": 15.8, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430039.698", "dur": 0.66, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430038.968", "dur": 1.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430043.898", "dur": 2.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430037.088", "dur": 9.53, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511430057.468", "dur": 1.89, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511430052.218", "dur": 7.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430073.098", "dur": 1.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430062.238", "dur": 13.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430091.089", "dur": 0.59, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430090.099", "dur": 2.05, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511430126.759", "dur": 2.58, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511430083.038", "dur": 47.361, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430143.819", "dur": 0.72, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430142.829", "dur": 2.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511430149.279", "dur": 1.77, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430139.819", "dur": 12.0, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511430198.880", "dur": 2.28, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511430159.969", "dur": 42.051, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430225.760", "dur": 0.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430224.810", "dur": 2.08, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430231.680", "dur": 1.92, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430220.910", "dur": 13.36, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511430245.440", "dur": 5.75, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430239.840", "dur": 12.13, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430260.040", "dur": 0.52, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430259.120", "dur": 1.8, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430263.740", "dur": 2.32, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430256.850", "dur": 9.79, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511430277.350", "dur": 1.81, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511430271.710", "dur": 8.1, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430293.391", "dur": 1.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430281.940", "dur": 13.881, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430311.011", "dur": 0.53, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430309.891", "dur": 2.02, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511430348.241", "dur": 3.13, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511430303.031", "dur": 49.29, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430366.901", "dur": 0.97, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430365.751", "dur": 2.48, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511430372.141", "dur": 2.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430362.511", "dur": 12.44, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511430424.952", "dur": 2.92, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511430382.441", "dur": 46.401, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430442.232", "dur": 0.89, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430441.222", "dur": 2.31, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430447.902", "dur": 1.96, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430437.972", "dur": 12.49, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511430461.112", "dur": 5.48, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430455.542", "dur": 11.79, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430475.102", "dur": 0.57, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430474.332", "dur": 1.64, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430478.862", "dur": 1.87, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430471.942", "dur": 9.24, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511430491.463", "dur": 2.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511430486.163", "dur": 7.97, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430507.133", "dur": 2.04, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430496.143", "dur": 13.79, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430524.673", "dur": 0.53, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430523.523", "dur": 2.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511430557.473", "dur": 2.52, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511430516.653", "dur": 44.49, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430574.693", "dur": 0.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430573.913", "dur": 1.66, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511430579.823", "dur": 2.03, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430571.193", "dur": 11.39, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511430628.544", "dur": 2.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511430590.294", "dur": 41.9, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430644.944", "dur": 0.72, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430644.094", "dur": 1.98, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430650.654", "dur": 1.65, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430641.194", "dur": 11.95, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511430663.114", "dur": 6.04, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430657.884", "dur": 12.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430677.944", "dur": 0.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430676.774", "dur": 2.42, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430682.284", "dur": 2.88, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430674.484", "dur": 11.18, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511430696.425", "dur": 2.6, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511430690.745", "dur": 9.0, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430712.515", "dur": 1.62, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430701.955", "dur": 12.94, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430729.755", "dur": 0.56, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430728.985", "dur": 1.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511430763.255", "dur": 2.59, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511430722.245", "dur": 44.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430779.425", "dur": 0.47, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430778.525", "dur": 1.95, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511430784.465", "dur": 1.861, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430775.985", "dur": 11.071, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511430835.566", "dur": 2.6, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511430794.066", "dur": 45.02, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430851.976", "dur": 0.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430851.126", "dur": 1.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430856.916", "dur": 1.72, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430848.136", "dur": 11.1, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511430868.846", "dur": 1.63, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430863.906", "dur": 7.18, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430878.056", "dur": 0.46, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430877.316", "dur": 1.53, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511430882.316", "dur": 5.551, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511430875.526", "dur": 12.971, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511430898.757", "dur": 1.65, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511430893.317", "dur": 7.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430912.737", "dur": 1.64, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511430903.177", "dur": 12.05, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430929.797", "dur": 0.56, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430928.917", "dur": 1.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511430962.007", "dur": 2.31, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511430922.497", "dur": 43.02, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511430978.377", "dur": 0.51, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511430977.487", "dur": 1.77, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511430983.637", "dur": 1.73, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511430974.827", "dur": 11.211, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511431030.818", "dur": 2.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511430993.418", "dur": 40.85, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431047.738", "dur": 0.47, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431046.838", "dur": 1.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511431052.738", "dur": 1.82, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511431044.128", "dur": 11.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511431065.368", "dur": 2.1, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511431060.238", "dur": 7.87, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431075.138", "dur": 0.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431074.398", "dur": 1.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511431078.608", "dur": 8.451, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511431072.118", "dur": 15.581, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511431099.609", "dur": 1.99, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511431094.069", "dur": 8.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511431115.269", "dur": 1.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511431104.729", "dur": 12.98, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431132.169", "dur": 0.47, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431131.339", "dur": 1.67, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511431168.769", "dur": 2.7, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511431124.759", "dur": 47.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431186.730", "dur": 0.42, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431185.929", "dur": 1.601, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511431191.670", "dur": 1.88, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511431183.009", "dur": 11.321, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511431243.120", "dur": 2.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511431201.610", "dur": 45.06, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431261.090", "dur": 0.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431260.080", "dur": 1.89, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511431266.000", "dur": 1.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511431256.970", "dur": 11.44, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511431279.690", "dur": 1.94, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511431273.680", "dur": 8.55, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431289.131", "dur": 0.37, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431288.381", "dur": 1.43, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceMuls", "pid": 437675, "tid": 437675, "ts": "1704161511431292.811", "dur": 4.58, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::mul_", "pid": 437675, "tid": 437675, "ts": "1704161511431286.421", "dur": 11.5, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcmul", "pid": 437675, "tid": 437675, "ts": "1704161511431308.501", "dur": 1.9, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcmul_", "pid": 437675, "tid": 437675, "ts": "1704161511431303.281", "dur": 7.84, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnSqrt", "pid": 437675, "tid": 437675, "ts": "1704161511431323.381", "dur": 2.04, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sqrt", "pid": 437675, "tid": 437675, "ts": "1704161511431313.341", "dur": 12.78, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431341.171", "dur": 0.51, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431340.241", "dur": 1.83, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511431374.301", "dur": 2.16, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511431333.161", "dur": 44.25, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431390.592", "dur": 0.44, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431389.582", "dur": 1.79, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdds", "pid": 437675, "tid": 437675, "ts": "1704161511431395.582", "dur": 1.78, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511431386.792", "dur": 11.29, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAddcdiv", "pid": 437675, "tid": 437675, "ts": "1704161511431442.142", "dur": 2.21, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::addcdiv_", "pid": 437675, "tid": 437675, "ts": "1704161511431405.392", "dur": 39.71, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "Optimizer.step#Adam.step", "pid": 437675, "tid": 437675, "ts": "1704161511429056.868", "dur": 2410.824, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::empty", "pid": 437675, "tid": 437675, "ts": "1704161511431481.472", "dur": 3.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "enumerate(DataLoader)#_SingleProcessDataLoaderIter.__next__", "pid": 437675, "tid": 437675, "ts": "1704161511431479.532", "dur": 16.571, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 437675, "ts": "1704161511431514.063", "dur": 3.19, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 437675, "ts": "1704161511431520.293", "dur": 0.65, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511431534.493", "dur": 2.2, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511431524.593", "dur": 13.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::view", "pid": 437675, "tid": 437675, "ts": "1704161511431552.563", "dur": 3.68, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::reshape", "pid": 437675, "tid": 437675, "ts": "1704161511431550.593", "dur": 5.95, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnArgMax", "pid": 437675, "tid": 437675, "ts": "1704161511431573.203", "dur": 1.98, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::argmax", "pid": 437675, "tid": 437675, "ts": "1704161511431546.143", "dur": 30.74, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnEqTensor", "pid": 437675, "tid": 437675, "ts": "1704161511431594.914", "dur": 5.89, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::eq", "pid": 437675, "tid": 437675, "ts": "1704161511431582.683", "dur": 20.181, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnReduceSum", "pid": 437675, "tid": 437675, "ts": "1704161511431704.105", "dur": 2.47, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::sum", "pid": 437675, "tid": 437675, "ts": "1704161511431609.514", "dur": 99.141, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnCast", "pid": 437675, "tid": 437675, "ts": "1704161511431757.055", "dur": 2.33, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::npu_dtype_cast", "pid": 437675, "tid": 437675, "ts": "1704161511431718.405", "dur": 44.07, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::to", "pid": 437675, "tid": 437675, "ts": "1704161511431715.145", "dur": 48.14, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnInplaceAdd", "pid": 437675, "tid": 437675, "ts": "1704161511431778.495", "dur": 2.02, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::add_", "pid": 437675, "tid": 437675, "ts": "1704161511431770.325", "dur": 11.01, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431814.126", "dur": 1.2, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431812.476", "dur": 3.45, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511431820.886", "dur": 1.86, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511431803.586", "dur": 20.05, "cat": "cpu_op", "args": {"Sequence number": 34, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431836.616", "dur": 0.64, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431835.426", "dur": 2.26, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aclnnDivs", "pid": 437675, "tid": 437675, "ts": "1704161511431840.796", "dur": 1.88, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::div", "pid": 437675, "tid": 437675, "ts": "1704161511431829.806", "dur": 13.38, "cat": "cpu_op", "args": {"Sequence number": 34, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511431855.466", "dur": 2399.584, "cat": "cpu_op", "args": {"Sequence number": 34, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511431854.516", "dur": 2401.574, "cat": "cpu_op", "args": {"Sequence number": 34, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::_local_scalar_dense", "pid": 437675, "tid": 437675, "ts": "1704161511434270.200", "dur": 24.761, "cat": "cpu_op", "args": {"Sequence number": 34, "Fwd thread id": 0}}, {"ph": "X", "name": "aten::item", "pid": 437675, "tid": 437675, "ts": "1704161511434267.580", "dur": 28.101, "cat": "cpu_op", "args": {"Sequence number": 34, "Fwd thread id": 0}}, {"ph": "X", "name": "ProfilerStep#1", "pid": 437675, "tid": 437675, "ts": "1704161511420289.011", "dur": 14091.97, "cat": "cpu_op", "args": {"Sequence number": -1, "Fwd thread id": 0}}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422856.936", "dur": 13.33, "cat": "enqueue", "args": {"correlation_id": 148}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 148, "pid": 437675, "tid": 437675, "ts": "1704161511422856.936", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422893.127", "dur": 1.61, "cat": "enqueue", "args": {"correlation_id": 149}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 149, "pid": 437675, "tid": 437675, "ts": "1704161511422893.127", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422907.637", "dur": 0.98, "cat": "enqueue", "args": {"correlation_id": 150}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 150, "pid": 437675, "tid": 437675, "ts": "1704161511422907.637", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422921.047", "dur": 1.34, "cat": "enqueue", "args": {"correlation_id": 151}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 151, "pid": 437675, "tid": 437675, "ts": "1704161511422921.047", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422933.527", "dur": 1.58, "cat": "enqueue", "args": {"correlation_id": 152}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 152, "pid": 437675, "tid": 437675, "ts": "1704161511422933.527", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422946.317", "dur": 1.83, "cat": "enqueue", "args": {"correlation_id": 153}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 153, "pid": 437675, "tid": 437675, "ts": "1704161511422946.317", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422958.767", "dur": 1.26, "cat": "enqueue", "args": {"correlation_id": 154}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 154, "pid": 437675, "tid": 437675, "ts": "1704161511422958.767", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422972.797", "dur": 1.51, "cat": "enqueue", "args": {"correlation_id": 155}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 155, "pid": 437675, "tid": 437675, "ts": "1704161511422972.797", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422984.548", "dur": 1.25, "cat": "enqueue", "args": {"correlation_id": 156}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 156, "pid": 437675, "tid": 437675, "ts": "1704161511422984.548", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511422997.278", "dur": 1.43, "cat": "enqueue", "args": {"correlation_id": 157}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 157, "pid": 437675, "tid": 437675, "ts": "1704161511422997.278", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423205.670", "dur": 7.69, "cat": "enqueue", "args": {"correlation_id": 158}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 158, "pid": 437675, "tid": 437675, "ts": "1704161511423205.670", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423257.830", "dur": 1.55, "cat": "enqueue", "args": {"correlation_id": 159}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 159, "pid": 437675, "tid": 437675, "ts": "1704161511423257.830", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423396.942", "dur": 1.71, "cat": "enqueue", "args": {"correlation_id": 160}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 160, "pid": 437675, "tid": 437675, "ts": "1704161511423396.942", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423468.642", "dur": 1.59, "cat": "enqueue", "args": {"correlation_id": 161}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 161, "pid": 437675, "tid": 437675, "ts": "1704161511423468.642", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423495.553", "dur": 1.2, "cat": "enqueue", "args": {"correlation_id": 162}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 162, "pid": 437675, "tid": 437675, "ts": "1704161511423495.553", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423553.983", "dur": 1.89, "cat": "enqueue", "args": {"correlation_id": 163}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 163, "pid": 437675, "tid": 437675, "ts": "1704161511423553.983", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423669.864", "dur": 1.79, "cat": "enqueue", "args": {"correlation_id": 164}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 164, "pid": 437675, "tid": 437675, "ts": "1704161511423669.864", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423691.515", "dur": 1.19, "cat": "enqueue", "args": {"correlation_id": 165}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 165, "pid": 437675, "tid": 437675, "ts": "1704161511423691.515", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423744.035", "dur": 1.14, "cat": "enqueue", "args": {"correlation_id": 166}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 166, "pid": 437675, "tid": 437675, "ts": "1704161511423744.035", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423761.025", "dur": 1.19, "cat": "enqueue", "args": {"correlation_id": 167}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 167, "pid": 437675, "tid": 437675, "ts": "1704161511423761.025", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511423811.076", "dur": 1.23, "cat": "enqueue", "args": {"correlation_id": 168}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 168, "pid": 437675, "tid": 437675, "ts": "1704161511423811.076", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511426449.692", "dur": 7.98, "cat": "enqueue", "args": {"correlation_id": 169}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 169, "pid": 437675, "tid": 437675, "ts": "1704161511426449.692", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511426532.863", "dur": 5.38, "cat": "enqueue", "args": {"correlation_id": 170}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 170, "pid": 437675, "tid": 437675, "ts": "1704161511426532.863", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511426550.633", "dur": 1.38, "cat": "enqueue", "args": {"correlation_id": 171}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 171, "pid": 437675, "tid": 437675, "ts": "1704161511426550.633", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511426617.844", "dur": 6.05, "cat": "enqueue", "args": {"correlation_id": 172}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 172, "pid": 437675, "tid": 437675, "ts": "1704161511426617.844", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511426818.186", "dur": 10.19, "cat": "enqueue", "args": {"correlation_id": 173}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 173, "pid": 437675, "tid": 439228, "ts": "1704161511426818.186", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511426847.736", "dur": 5.02, "cat": "enqueue", "args": {"correlation_id": 174}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 174, "pid": 437675, "tid": 439228, "ts": "1704161511426847.736", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511426925.357", "dur": 6.37, "cat": "enqueue", "args": {"correlation_id": 175}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 175, "pid": 437675, "tid": 439228, "ts": "1704161511426925.357", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511426998.368", "dur": 5.55, "cat": "enqueue", "args": {"correlation_id": 176}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 176, "pid": 437675, "tid": 439228, "ts": "1704161511426998.368", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427031.358", "dur": 1.35, "cat": "enqueue", "args": {"correlation_id": 177}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 177, "pid": 437675, "tid": 439228, "ts": "1704161511427031.358", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427082.879", "dur": 1.76, "cat": "enqueue", "args": {"correlation_id": 178}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 178, "pid": 437675, "tid": 439228, "ts": "1704161511427082.879", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427260.020", "dur": 5.98, "cat": "enqueue", "args": {"correlation_id": 179}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 179, "pid": 437675, "tid": 439228, "ts": "1704161511427260.020", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427352.651", "dur": 6.15, "cat": "enqueue", "args": {"correlation_id": 180}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 180, "pid": 437675, "tid": 439228, "ts": "1704161511427352.651", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427416.502", "dur": 6.99, "cat": "enqueue", "args": {"correlation_id": 181}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 181, "pid": 437675, "tid": 439228, "ts": "1704161511427416.502", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427472.242", "dur": 5.74, "cat": "enqueue", "args": {"correlation_id": 182}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 182, "pid": 437675, "tid": 439228, "ts": "1704161511427472.242", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427502.463", "dur": 1.22, "cat": "enqueue", "args": {"correlation_id": 183}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 183, "pid": 437675, "tid": 439228, "ts": "1704161511427502.463", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427522.873", "dur": 1.62, "cat": "enqueue", "args": {"correlation_id": 184}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 184, "pid": 437675, "tid": 439228, "ts": "1704161511427522.873", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427593.574", "dur": 6.17, "cat": "enqueue", "args": {"correlation_id": 185}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 185, "pid": 437675, "tid": 439228, "ts": "1704161511427593.574", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427669.004", "dur": 6.27, "cat": "enqueue", "args": {"correlation_id": 186}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 186, "pid": 437675, "tid": 439228, "ts": "1704161511427669.004", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427705.165", "dur": 1.73, "cat": "enqueue", "args": {"correlation_id": 187}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 187, "pid": 437675, "tid": 439228, "ts": "1704161511427705.165", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427746.995", "dur": 5.9, "cat": "enqueue", "args": {"correlation_id": 188}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 188, "pid": 437675, "tid": 439228, "ts": "1704161511427746.995", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427862.546", "dur": 5.94, "cat": "enqueue", "args": {"correlation_id": 189}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 189, "pid": 437675, "tid": 439228, "ts": "1704161511427862.546", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427894.867", "dur": 1.89, "cat": "enqueue", "args": {"correlation_id": 190}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 190, "pid": 437675, "tid": 439228, "ts": "1704161511427894.867", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511427972.787", "dur": 8.0, "cat": "enqueue", "args": {"correlation_id": 191}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 191, "pid": 437675, "tid": 439228, "ts": "1704161511427972.787", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428051.258", "dur": 6.46, "cat": "enqueue", "args": {"correlation_id": 192}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 192, "pid": 437675, "tid": 439228, "ts": "1704161511428051.258", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428164.819", "dur": 5.93, "cat": "enqueue", "args": {"correlation_id": 193}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 193, "pid": 437675, "tid": 439228, "ts": "1704161511428164.819", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428213.600", "dur": 1.64, "cat": "enqueue", "args": {"correlation_id": 194}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 194, "pid": 437675, "tid": 439228, "ts": "1704161511428213.600", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428347.691", "dur": 1.82, "cat": "enqueue", "args": {"correlation_id": 195}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 195, "pid": 437675, "tid": 439228, "ts": "1704161511428347.691", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428383.221", "dur": 1.761, "cat": "enqueue", "args": {"correlation_id": 196}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 196, "pid": 437675, "tid": 439228, "ts": "1704161511428383.221", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428435.312", "dur": 1.24, "cat": "enqueue", "args": {"correlation_id": 197}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 197, "pid": 437675, "tid": 439228, "ts": "1704161511428435.312", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428525.943", "dur": 1.61, "cat": "enqueue", "args": {"correlation_id": 198}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 198, "pid": 437675, "tid": 439228, "ts": "1704161511428525.943", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428603.434", "dur": 5.75, "cat": "enqueue", "args": {"correlation_id": 199}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 199, "pid": 437675, "tid": 439228, "ts": "1704161511428603.434", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428655.854", "dur": 1.32, "cat": "enqueue", "args": {"correlation_id": 200}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 200, "pid": 437675, "tid": 439228, "ts": "1704161511428655.854", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428704.395", "dur": 1.47, "cat": "enqueue", "args": {"correlation_id": 201}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 201, "pid": 437675, "tid": 439228, "ts": "1704161511428704.395", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428744.025", "dur": 1.32, "cat": "enqueue", "args": {"correlation_id": 202}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 202, "pid": 437675, "tid": 439228, "ts": "1704161511428744.025", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428820.236", "dur": 1.41, "cat": "enqueue", "args": {"correlation_id": 203}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 203, "pid": 437675, "tid": 439228, "ts": "1704161511428820.236", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428856.076", "dur": 1.21, "cat": "enqueue", "args": {"correlation_id": 204}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 204, "pid": 437675, "tid": 439228, "ts": "1704161511428856.076", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428892.847", "dur": 1.48, "cat": "enqueue", "args": {"correlation_id": 205}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 205, "pid": 437675, "tid": 439228, "ts": "1704161511428892.847", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428954.057", "dur": 1.37, "cat": "enqueue", "args": {"correlation_id": 206}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 206, "pid": 437675, "tid": 439228, "ts": "1704161511428954.057", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 439228, "ts": "1704161511428996.318", "dur": 1.77, "cat": "enqueue", "args": {"correlation_id": 207}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 207, "pid": 437675, "tid": 439228, "ts": "1704161511428996.318", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429190.970", "dur": 7.08, "cat": "enqueue", "args": {"correlation_id": 208}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 208, "pid": 437675, "tid": 437675, "ts": "1704161511429190.970", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429221.150", "dur": 1.19, "cat": "enqueue", "args": {"correlation_id": 209}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 209, "pid": 437675, "tid": 437675, "ts": "1704161511429221.150", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429237.710", "dur": 1.3, "cat": "enqueue", "args": {"correlation_id": 210}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 210, "pid": 437675, "tid": 437675, "ts": "1704161511429237.710", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429259.320", "dur": 1.03, "cat": "enqueue", "args": {"correlation_id": 211}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 211, "pid": 437675, "tid": 437675, "ts": "1704161511429259.320", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429285.011", "dur": 1.06, "cat": "enqueue", "args": {"correlation_id": 212}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 212, "pid": 437675, "tid": 437675, "ts": "1704161511429285.011", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429397.162", "dur": 1.72, "cat": "enqueue", "args": {"correlation_id": 213}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 213, "pid": 437675, "tid": 437675, "ts": "1704161511429397.162", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429427.412", "dur": 1.21, "cat": "enqueue", "args": {"correlation_id": 214}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 214, "pid": 437675, "tid": 437675, "ts": "1704161511429427.412", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429503.913", "dur": 1.76, "cat": "enqueue", "args": {"correlation_id": 215}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 215, "pid": 437675, "tid": 437675, "ts": "1704161511429503.913", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429531.253", "dur": 4.75, "cat": "enqueue", "args": {"correlation_id": 216}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 216, "pid": 437675, "tid": 437675, "ts": "1704161511429531.253", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429551.483", "dur": 1.05, "cat": "enqueue", "args": {"correlation_id": 217}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 217, "pid": 437675, "tid": 437675, "ts": "1704161511429551.483", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429567.203", "dur": 0.99, "cat": "enqueue", "args": {"correlation_id": 218}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 218, "pid": 437675, "tid": 437675, "ts": "1704161511429567.203", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429580.823", "dur": 1.05, "cat": "enqueue", "args": {"correlation_id": 219}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 219, "pid": 437675, "tid": 437675, "ts": "1704161511429580.823", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429598.114", "dur": 1.04, "cat": "enqueue", "args": {"correlation_id": 220}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 220, "pid": 437675, "tid": 437675, "ts": "1704161511429598.114", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429657.774", "dur": 1.83, "cat": "enqueue", "args": {"correlation_id": 221}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 221, "pid": 437675, "tid": 437675, "ts": "1704161511429657.774", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429684.024", "dur": 1.131, "cat": "enqueue", "args": {"correlation_id": 222}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 222, "pid": 437675, "tid": 437675, "ts": "1704161511429684.024", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429739.385", "dur": 1.36, "cat": "enqueue", "args": {"correlation_id": 223}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 223, "pid": 437675, "tid": 437675, "ts": "1704161511429739.385", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429764.995", "dur": 6.07, "cat": "enqueue", "args": {"correlation_id": 224}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 224, "pid": 437675, "tid": 437675, "ts": "1704161511429764.995", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429784.135", "dur": 1.041, "cat": "enqueue", "args": {"correlation_id": 225}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 225, "pid": 437675, "tid": 437675, "ts": "1704161511429784.135", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429812.196", "dur": 1.28, "cat": "enqueue", "args": {"correlation_id": 226}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 226, "pid": 437675, "tid": 437675, "ts": "1704161511429812.196", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429827.206", "dur": 1.05, "cat": "enqueue", "args": {"correlation_id": 227}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 227, "pid": 437675, "tid": 437675, "ts": "1704161511429827.206", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429844.996", "dur": 0.9, "cat": "enqueue", "args": {"correlation_id": 228}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 228, "pid": 437675, "tid": 437675, "ts": "1704161511429844.996", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429905.447", "dur": 1.53, "cat": "enqueue", "args": {"correlation_id": 229}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 229, "pid": 437675, "tid": 437675, "ts": "1704161511429905.447", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429930.777", "dur": 1.14, "cat": "enqueue", "args": {"correlation_id": 230}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 230, "pid": 437675, "tid": 437675, "ts": "1704161511429930.777", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511429986.378", "dur": 1.51, "cat": "enqueue", "args": {"correlation_id": 231}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 231, "pid": 437675, "tid": 437675, "ts": "1704161511429986.378", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430008.848", "dur": 1.53, "cat": "enqueue", "args": {"correlation_id": 232}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 232, "pid": 437675, "tid": 437675, "ts": "1704161511430008.848", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430022.048", "dur": 8.89, "cat": "enqueue", "args": {"correlation_id": 233}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 233, "pid": 437675, "tid": 437675, "ts": "1704161511430022.048", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430044.098", "dur": 1.54, "cat": "enqueue", "args": {"correlation_id": 234}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 234, "pid": 437675, "tid": 437675, "ts": "1704161511430044.098", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430057.688", "dur": 0.97, "cat": "enqueue", "args": {"correlation_id": 235}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 235, "pid": 437675, "tid": 437675, "ts": "1704161511430057.688", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430073.458", "dur": 0.93, "cat": "enqueue", "args": {"correlation_id": 236}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 236, "pid": 437675, "tid": 437675, "ts": "1704161511430073.458", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430127.299", "dur": 1.32, "cat": "enqueue", "args": {"correlation_id": 237}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 237, "pid": 437675, "tid": 437675, "ts": "1704161511430127.299", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430149.599", "dur": 0.99, "cat": "enqueue", "args": {"correlation_id": 238}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 238, "pid": 437675, "tid": 437675, "ts": "1704161511430149.599", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430199.360", "dur": 1.34, "cat": "enqueue", "args": {"correlation_id": 239}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 239, "pid": 437675, "tid": 437675, "ts": "1704161511430199.360", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430232.000", "dur": 1.01, "cat": "enqueue", "args": {"correlation_id": 240}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 240, "pid": 437675, "tid": 437675, "ts": "1704161511430232.000", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430245.820", "dur": 4.85, "cat": "enqueue", "args": {"correlation_id": 241}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 241, "pid": 437675, "tid": 437675, "ts": "1704161511430245.820", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430264.080", "dur": 1.03, "cat": "enqueue", "args": {"correlation_id": 242}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 242, "pid": 437675, "tid": 437675, "ts": "1704161511430264.080", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430277.760", "dur": 0.99, "cat": "enqueue", "args": {"correlation_id": 243}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 243, "pid": 437675, "tid": 437675, "ts": "1704161511430277.760", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430293.741", "dur": 0.99, "cat": "enqueue", "args": {"correlation_id": 244}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 244, "pid": 437675, "tid": 437675, "ts": "1704161511430293.741", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430349.201", "dur": 1.5, "cat": "enqueue", "args": {"correlation_id": 245}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 245, "pid": 437675, "tid": 437675, "ts": "1704161511430349.201", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430372.591", "dur": 1.11, "cat": "enqueue", "args": {"correlation_id": 246}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 246, "pid": 437675, "tid": 437675, "ts": "1704161511430372.591", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430425.692", "dur": 1.65, "cat": "enqueue", "args": {"correlation_id": 247}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 247, "pid": 437675, "tid": 437675, "ts": "1704161511430425.692", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430448.362", "dur": 1.07, "cat": "enqueue", "args": {"correlation_id": 248}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 248, "pid": 437675, "tid": 437675, "ts": "1704161511430448.362", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430461.752", "dur": 4.38, "cat": "enqueue", "args": {"correlation_id": 249}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 249, "pid": 437675, "tid": 437675, "ts": "1704161511430461.752", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430479.222", "dur": 1.11, "cat": "enqueue", "args": {"correlation_id": 250}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 250, "pid": 437675, "tid": 437675, "ts": "1704161511430479.222", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430491.803", "dur": 1.18, "cat": "enqueue", "args": {"correlation_id": 251}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 251, "pid": 437675, "tid": 437675, "ts": "1704161511430491.803", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430507.353", "dur": 1.19, "cat": "enqueue", "args": {"correlation_id": 252}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 252, "pid": 437675, "tid": 437675, "ts": "1704161511430507.353", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430557.873", "dur": 1.57, "cat": "enqueue", "args": {"correlation_id": 253}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 253, "pid": 437675, "tid": 437675, "ts": "1704161511430557.873", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430580.193", "dur": 1.16, "cat": "enqueue", "args": {"correlation_id": 254}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 254, "pid": 437675, "tid": 437675, "ts": "1704161511430580.193", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430628.964", "dur": 1.45, "cat": "enqueue", "args": {"correlation_id": 255}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 255, "pid": 437675, "tid": 437675, "ts": "1704161511430628.964", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430650.884", "dur": 0.98, "cat": "enqueue", "args": {"correlation_id": 256}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 256, "pid": 437675, "tid": 437675, "ts": "1704161511430650.884", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430663.964", "dur": 4.52, "cat": "enqueue", "args": {"correlation_id": 257}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 257, "pid": 437675, "tid": 437675, "ts": "1704161511430663.964", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430683.034", "dur": 1.32, "cat": "enqueue", "args": {"correlation_id": 258}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 258, "pid": 437675, "tid": 437675, "ts": "1704161511430683.034", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430696.885", "dur": 1.12, "cat": "enqueue", "args": {"correlation_id": 259}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 259, "pid": 437675, "tid": 437675, "ts": "1704161511430696.885", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430712.765", "dur": 0.98, "cat": "enqueue", "args": {"correlation_id": 260}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 260, "pid": 437675, "tid": 437675, "ts": "1704161511430712.765", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430763.735", "dur": 1.42, "cat": "enqueue", "args": {"correlation_id": 261}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 261, "pid": 437675, "tid": 437675, "ts": "1704161511430763.735", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430784.735", "dur": 1.171, "cat": "enqueue", "args": {"correlation_id": 262}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 262, "pid": 437675, "tid": 437675, "ts": "1704161511430784.735", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430836.066", "dur": 1.6, "cat": "enqueue", "args": {"correlation_id": 263}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 263, "pid": 437675, "tid": 437675, "ts": "1704161511430836.066", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430857.146", "dur": 0.99, "cat": "enqueue", "args": {"correlation_id": 264}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 264, "pid": 437675, "tid": 437675, "ts": "1704161511430857.146", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430869.096", "dur": 1.0, "cat": "enqueue", "args": {"correlation_id": 265}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 265, "pid": 437675, "tid": 437675, "ts": "1704161511430869.096", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430882.656", "dur": 4.731, "cat": "enqueue", "args": {"correlation_id": 266}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 266, "pid": 437675, "tid": 437675, "ts": "1704161511430882.656", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430899.047", "dur": 0.88, "cat": "enqueue", "args": {"correlation_id": 267}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 267, "pid": 437675, "tid": 437675, "ts": "1704161511430899.047", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430912.987", "dur": 0.99, "cat": "enqueue", "args": {"correlation_id": 268}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 268, "pid": 437675, "tid": 437675, "ts": "1704161511430912.987", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430962.387", "dur": 1.47, "cat": "enqueue", "args": {"correlation_id": 269}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 269, "pid": 437675, "tid": 437675, "ts": "1704161511430962.387", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511430983.937", "dur": 0.99, "cat": "enqueue", "args": {"correlation_id": 270}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 270, "pid": 437675, "tid": 437675, "ts": "1704161511430983.937", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431031.308", "dur": 1.4, "cat": "enqueue", "args": {"correlation_id": 271}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 271, "pid": 437675, "tid": 437675, "ts": "1704161511431031.308", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431053.078", "dur": 1.06, "cat": "enqueue", "args": {"correlation_id": 272}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 272, "pid": 437675, "tid": 437675, "ts": "1704161511431053.078", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431065.678", "dur": 1.38, "cat": "enqueue", "args": {"correlation_id": 273}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 273, "pid": 437675, "tid": 437675, "ts": "1704161511431065.678", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431078.968", "dur": 7.471, "cat": "enqueue", "args": {"correlation_id": 274}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 274, "pid": 437675, "tid": 437675, "ts": "1704161511431078.968", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431099.939", "dur": 1.28, "cat": "enqueue", "args": {"correlation_id": 275}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 275, "pid": 437675, "tid": 437675, "ts": "1704161511431099.939", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431115.559", "dur": 1.03, "cat": "enqueue", "args": {"correlation_id": 276}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 276, "pid": 437675, "tid": 437675, "ts": "1704161511431115.559", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431169.109", "dur": 1.91, "cat": "enqueue", "args": {"correlation_id": 277}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 277, "pid": 437675, "tid": 437675, "ts": "1704161511431169.109", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431192.010", "dur": 1.1, "cat": "enqueue", "args": {"correlation_id": 278}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 278, "pid": 437675, "tid": 437675, "ts": "1704161511431192.010", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431243.530", "dur": 1.79, "cat": "enqueue", "args": {"correlation_id": 279}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 279, "pid": 437675, "tid": 437675, "ts": "1704161511431243.530", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431266.240", "dur": 1.03, "cat": "enqueue", "args": {"correlation_id": 280}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 280, "pid": 437675, "tid": 437675, "ts": "1704161511431266.240", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431280.000", "dur": 1.25, "cat": "enqueue", "args": {"correlation_id": 281}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 281, "pid": 437675, "tid": 437675, "ts": "1704161511431280.000", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431293.161", "dur": 3.68, "cat": "enqueue", "args": {"correlation_id": 282}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 282, "pid": 437675, "tid": 437675, "ts": "1704161511431293.161", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431308.821", "dur": 1.1, "cat": "enqueue", "args": {"correlation_id": 283}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 283, "pid": 437675, "tid": 437675, "ts": "1704161511431308.821", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431323.731", "dur": 1.34, "cat": "enqueue", "args": {"correlation_id": 284}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 284, "pid": 437675, "tid": 437675, "ts": "1704161511431323.731", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431374.651", "dur": 1.35, "cat": "enqueue", "args": {"correlation_id": 285}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 285, "pid": 437675, "tid": 437675, "ts": "1704161511431374.651", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431395.862", "dur": 1.1, "cat": "enqueue", "args": {"correlation_id": 286}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 286, "pid": 437675, "tid": 437675, "ts": "1704161511431395.862", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431442.512", "dur": 1.42, "cat": "enqueue", "args": {"correlation_id": 287}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 287, "pid": 437675, "tid": 437675, "ts": "1704161511431442.512", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431534.863", "dur": 1.38, "cat": "enqueue", "args": {"correlation_id": 288}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 288, "pid": 437675, "tid": 437675, "ts": "1704161511431534.863", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431573.543", "dur": 1.2, "cat": "enqueue", "args": {"correlation_id": 289}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 289, "pid": 437675, "tid": 437675, "ts": "1704161511431573.543", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431595.214", "dur": 5.08, "cat": "enqueue", "args": {"correlation_id": 290}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 290, "pid": 437675, "tid": 437675, "ts": "1704161511431595.214", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431704.525", "dur": 1.47, "cat": "enqueue", "args": {"correlation_id": 291}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 291, "pid": 437675, "tid": 437675, "ts": "1704161511431704.525", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431757.595", "dur": 1.34, "cat": "enqueue", "args": {"correlation_id": 292}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 292, "pid": 437675, "tid": 437675, "ts": "1704161511431757.595", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431778.895", "dur": 1.09, "cat": "enqueue", "args": {"correlation_id": 293}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 293, "pid": 437675, "tid": 437675, "ts": "1704161511431778.895", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431821.186", "dur": 1.12, "cat": "enqueue", "args": {"correlation_id": 294}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 294, "pid": 437675, "tid": 437675, "ts": "1704161511431821.186", "cat": "async_task_queue"}, {"ph": "X", "name": "Enqueue", "pid": 437675, "tid": 437675, "ts": "1704161511431841.146", "dur": 1.07, "cat": "enqueue", "args": {"correlation_id": 295}}, {"ph": "s", "bp": "e", "name": "enqueue_to_dequeue", "id": 295, "pid": 437675, "tid": 437675, "ts": "1704161511431841.146", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422882.957", "dur": 36.27, "cat": "dequeue", "args": {"correlation_id": 148}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 148, "pid": 437675, "tid": 439084, "ts": "1704161511422882.957", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422923.227", "dur": 10.78, "cat": "dequeue", "args": {"correlation_id": 149}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 149, "pid": 437675, "tid": 439084, "ts": "1704161511422923.227", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422935.077", "dur": 9.5, "cat": "dequeue", "args": {"correlation_id": 150}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 150, "pid": 437675, "tid": 439084, "ts": "1704161511422935.077", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422946.277", "dur": 8.98, "cat": "dequeue", "args": {"correlation_id": 151}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 151, "pid": 437675, "tid": 439084, "ts": "1704161511422946.277", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422957.017", "dur": 9.59, "cat": "dequeue", "args": {"correlation_id": 152}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 152, "pid": 437675, "tid": 439084, "ts": "1704161511422957.017", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422968.347", "dur": 7.82, "cat": "dequeue", "args": {"correlation_id": 153}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 153, "pid": 437675, "tid": 439084, "ts": "1704161511422968.347", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422978.278", "dur": 8.36, "cat": "dequeue", "args": {"correlation_id": 154}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 154, "pid": 437675, "tid": 439084, "ts": "1704161511422978.278", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422988.558", "dur": 7.97, "cat": "dequeue", "args": {"correlation_id": 155}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 155, "pid": 437675, "tid": 439084, "ts": "1704161511422988.558", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511422997.808", "dur": 7.88, "cat": "dequeue", "args": {"correlation_id": 156}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 156, "pid": 437675, "tid": 439084, "ts": "1704161511422997.808", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceZero", "pid": 437675, "tid": 439084, "ts": "1704161511423007.428", "dur": 7.29, "cat": "dequeue", "args": {"correlation_id": 157}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 157, "pid": 437675, "tid": 439084, "ts": "1704161511423007.428", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@Conv2D", "pid": 437675, "tid": 439084, "ts": "1704161511423217.990", "dur": 1107.771, "cat": "dequeue", "args": {"correlation_id": 158}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 158, "pid": 437675, "tid": 439084, "ts": "1704161511423217.990", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnRelu", "pid": 437675, "tid": 439084, "ts": "1704161511424331.571", "dur": 21.21, "cat": "dequeue", "args": {"correlation_id": 159}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 159, "pid": 437675, "tid": 439084, "ts": "1704161511424331.571", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@MaxPoolWithArgmaxV1", "pid": 437675, "tid": 439084, "ts": "1704161511424355.421", "dur": 614.326, "cat": "dequeue", "args": {"correlation_id": 160}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 160, "pid": 437675, "tid": 439084, "ts": "1704161511424355.421", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@Conv2D", "pid": 437675, "tid": 439084, "ts": "1704161511424975.557", "dur": 919.1, "cat": "dequeue", "args": {"correlation_id": 161}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 161, "pid": 437675, "tid": 439084, "ts": "1704161511424975.557", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnRelu", "pid": 437675, "tid": 439084, "ts": "1704161511425900.197", "dur": 15.6, "cat": "dequeue", "args": {"correlation_id": 162}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 162, "pid": 437675, "tid": 439084, "ts": "1704161511425900.197", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@MaxPoolWithArgmaxV1", "pid": 437675, "tid": 439084, "ts": "1704161511425917.937", "dur": 136.671, "cat": "dequeue", "args": {"correlation_id": 163}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 163, "pid": 437675, "tid": 439084, "ts": "1704161511425917.937", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnAddmm", "pid": 437675, "tid": 439084, "ts": "1704161511426057.978", "dur": 60.801, "cat": "dequeue", "args": {"correlation_id": 164}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 164, "pid": 437675, "tid": 439084, "ts": "1704161511426057.978", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnRelu", "pid": 437675, "tid": 439084, "ts": "1704161511426120.939", "dur": 8.08, "cat": "dequeue", "args": {"correlation_id": 165}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 165, "pid": 437675, "tid": 439084, "ts": "1704161511426120.939", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnAddmm", "pid": 437675, "tid": 439084, "ts": "1704161511426130.339", "dur": 57.371, "cat": "dequeue", "args": {"correlation_id": 166}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 166, "pid": 437675, "tid": 439084, "ts": "1704161511426130.339", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnRelu", "pid": 437675, "tid": 439084, "ts": "1704161511426189.740", "dur": 7.7, "cat": "dequeue", "args": {"correlation_id": 167}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 167, "pid": 437675, "tid": 439084, "ts": "1704161511426189.740", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnAddmm", "pid": 437675, "tid": 439084, "ts": "1704161511426199.100", "dur": 49.96, "cat": "dequeue", "args": {"correlation_id": 168}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 168, "pid": 437675, "tid": 439084, "ts": "1704161511426199.100", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnLogSoftmax", "pid": 437675, "tid": 439084, "ts": "1704161511426463.522", "dur": 15.78, "cat": "dequeue", "args": {"correlation_id": 169}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 169, "pid": 437675, "tid": 439084, "ts": "1704161511426463.522", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceOne", "pid": 437675, "tid": 439084, "ts": "1704161511426542.153", "dur": 11.75, "cat": "dequeue", "args": {"correlation_id": 170}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 170, "pid": 437675, "tid": 439084, "ts": "1704161511426542.153", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnNLLLoss", "pid": 437675, "tid": 439084, "ts": "1704161511426556.003", "dur": 24.07, "cat": "dequeue", "args": {"correlation_id": 171}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 171, "pid": 437675, "tid": 439084, "ts": "1704161511426556.003", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceOne", "pid": 437675, "tid": 439084, "ts": "1704161511426627.744", "dur": 9.55, "cat": "dequeue", "args": {"correlation_id": 172}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 172, "pid": 437675, "tid": 439084, "ts": "1704161511426627.744", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceOne", "pid": 437675, "tid": 439084, "ts": "1704161511426831.926", "dur": 8.96, "cat": "dequeue", "args": {"correlation_id": 173}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 173, "pid": 437675, "tid": 439084, "ts": "1704161511426831.926", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnNLLLossBackward", "pid": 437675, "tid": 439084, "ts": "1704161511426856.316", "dur": 16.66, "cat": "dequeue", "args": {"correlation_id": 174}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 174, "pid": 437675, "tid": 439084, "ts": "1704161511426856.316", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnLogSoftmaxBackward", "pid": 437675, "tid": 439084, "ts": "1704161511426935.117", "dur": 12.12, "cat": "dequeue", "args": {"correlation_id": 175}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 175, "pid": 437675, "tid": 439084, "ts": "1704161511426935.117", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnMm", "pid": 437675, "tid": 439084, "ts": "1704161511427007.198", "dur": 37.96, "cat": "dequeue", "args": {"correlation_id": 176}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 176, "pid": 437675, "tid": 439084, "ts": "1704161511427007.198", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnMm", "pid": 437675, "tid": 439084, "ts": "1704161511427047.458", "dur": 36.251, "cat": "dequeue", "args": {"correlation_id": 177}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 177, "pid": 437675, "tid": 439084, "ts": "1704161511427047.458", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnReduceSum", "pid": 437675, "tid": 439084, "ts": "1704161511427085.799", "dur": 14.39, "cat": "dequeue", "args": {"correlation_id": 178}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 178, "pid": 437675, "tid": 439084, "ts": "1704161511427085.799", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511427269.460", "dur": 72.671, "cat": "dequeue", "args": {"correlation_id": 179}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 179, "pid": 437675, "tid": 439084, "ts": "1704161511427269.460", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511427362.431", "dur": 36.601, "cat": "dequeue", "args": {"correlation_id": 180}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 180, "pid": 437675, "tid": 439084, "ts": "1704161511427362.431", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnThresholdBackward", "pid": 437675, "tid": 439084, "ts": "1704161511427427.912", "dur": 14.58, "cat": "dequeue", "args": {"correlation_id": 181}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 181, "pid": 437675, "tid": 439084, "ts": "1704161511427427.912", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnMm", "pid": 437675, "tid": 439084, "ts": "1704161511427481.542", "dur": 35.051, "cat": "dequeue", "args": {"correlation_id": 182}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 182, "pid": 437675, "tid": 439084, "ts": "1704161511427481.542", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnMm", "pid": 437675, "tid": 439084, "ts": "1704161511427518.693", "dur": 32.5, "cat": "dequeue", "args": {"correlation_id": 183}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 183, "pid": 437675, "tid": 439084, "ts": "1704161511427518.693", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnReduceSum", "pid": 437675, "tid": 439084, "ts": "1704161511427553.183", "dur": 10.63, "cat": "dequeue", "args": {"correlation_id": 184}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 184, "pid": 437675, "tid": 439084, "ts": "1704161511427553.183", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511427602.924", "dur": 34.05, "cat": "dequeue", "args": {"correlation_id": 185}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 185, "pid": 437675, "tid": 439084, "ts": "1704161511427602.924", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511427679.604", "dur": 30.371, "cat": "dequeue", "args": {"correlation_id": 186}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 186, "pid": 437675, "tid": 439084, "ts": "1704161511427679.604", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnThresholdBackward", "pid": 437675, "tid": 439084, "ts": "1704161511427712.375", "dur": 8.47, "cat": "dequeue", "args": {"correlation_id": 187}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 187, "pid": 437675, "tid": 439084, "ts": "1704161511427712.375", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnMm", "pid": 437675, "tid": 439084, "ts": "1704161511427756.445", "dur": 34.011, "cat": "dequeue", "args": {"correlation_id": 188}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 188, "pid": 437675, "tid": 439084, "ts": "1704161511427756.445", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnMm", "pid": 437675, "tid": 439084, "ts": "1704161511427871.846", "dur": 32.871, "cat": "dequeue", "args": {"correlation_id": 189}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 189, "pid": 437675, "tid": 439084, "ts": "1704161511427871.846", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnReduceSum", "pid": 437675, "tid": 439084, "ts": "1704161511427907.237", "dur": 10.66, "cat": "dequeue", "args": {"correlation_id": 190}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 190, "pid": 437675, "tid": 439084, "ts": "1704161511427907.237", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511427984.308", "dur": 31.5, "cat": "dequeue", "args": {"correlation_id": 191}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 191, "pid": 437675, "tid": 439084, "ts": "1704161511427984.308", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511428060.948", "dur": 29.231, "cat": "dequeue", "args": {"correlation_id": 192}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 192, "pid": 437675, "tid": 439084, "ts": "1704161511428060.948", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@MaxPoolGradWithArgmaxV1", "pid": 437675, "tid": 439084, "ts": "1704161511428174.169", "dur": 816.009, "cat": "dequeue", "args": {"correlation_id": 193}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 193, "pid": 437675, "tid": 439084, "ts": "1704161511428174.169", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnThresholdBackward", "pid": 437675, "tid": 439084, "ts": "1704161511428995.858", "dur": 20.71, "cat": "dequeue", "args": {"correlation_id": 194}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 194, "pid": 437675, "tid": 439084, "ts": "1704161511428995.858", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@Conv2DBackpropInput", "pid": 437675, "tid": 439084, "ts": "1704161511429019.438", "dur": 901.779, "cat": "dequeue", "args": {"correlation_id": 195}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 195, "pid": 437675, "tid": 439084, "ts": "1704161511429019.438", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@Conv2DBackpropFilter", "pid": 437675, "tid": 439084, "ts": "1704161511429932.567", "dur": 830.638, "cat": "dequeue", "args": {"correlation_id": 196}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 196, "pid": 437675, "tid": 439084, "ts": "1704161511429932.567", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@ReduceSum", "pid": 437675, "tid": 439084, "ts": "1704161511430770.935", "dur": 321.184, "cat": "dequeue", "args": {"correlation_id": 197}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 197, "pid": 437675, "tid": 439084, "ts": "1704161511430770.935", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceCopy", "pid": 437675, "tid": 439084, "ts": "1704161511431099.719", "dur": 32.82, "cat": "dequeue", "args": {"correlation_id": 198}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 198, "pid": 437675, "tid": 439084, "ts": "1704161511431099.719", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511431135.549", "dur": 62.001, "cat": "dequeue", "args": {"correlation_id": 199}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 199, "pid": 437675, "tid": 439084, "ts": "1704161511431135.549", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511431200.040", "dur": 32.9, "cat": "dequeue", "args": {"correlation_id": 200}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 200, "pid": 437675, "tid": 439084, "ts": "1704161511431200.040", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@MaxPoolGradWithArgmaxV1", "pid": 437675, "tid": 439084, "ts": "1704161511431235.210", "dur": 794.548, "cat": "dequeue", "args": {"correlation_id": 201}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 201, "pid": 437675, "tid": 439084, "ts": "1704161511431235.210", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnThresholdBackward", "pid": 437675, "tid": 439084, "ts": "1704161511432035.618", "dur": 15.23, "cat": "dequeue", "args": {"correlation_id": 202}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 202, "pid": 437675, "tid": 439084, "ts": "1704161511432035.618", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@Conv2DBackpropFilter", "pid": 437675, "tid": 439084, "ts": "1704161511432052.848", "dur": 174.352, "cat": "dequeue", "args": {"correlation_id": 203}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 203, "pid": 437675, "tid": 439084, "ts": "1704161511432052.848", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@ReduceSum", "pid": 437675, "tid": 439084, "ts": "1704161511432232.750", "dur": 65.311, "cat": "dequeue", "args": {"correlation_id": 204}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 204, "pid": 437675, "tid": 439084, "ts": "1704161511432232.750", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceCopy", "pid": 437675, "tid": 439084, "ts": "1704161511432302.831", "dur": 18.13, "cat": "dequeue", "args": {"correlation_id": 205}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 205, "pid": 437675, "tid": 439084, "ts": "1704161511432302.831", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511432323.421", "dur": 53.45, "cat": "dequeue", "args": {"correlation_id": 206}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 206, "pid": 437675, "tid": 439084, "ts": "1704161511432323.421", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511432380.571", "dur": 33.811, "cat": "dequeue", "args": {"correlation_id": 207}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 207, "pid": 437675, "tid": 439084, "ts": "1704161511432380.571", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432416.262", "dur": 15.33, "cat": "dequeue", "args": {"correlation_id": 208}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 208, "pid": 437675, "tid": 439084, "ts": "1704161511432416.262", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511432433.172", "dur": 8.55, "cat": "dequeue", "args": {"correlation_id": 209}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 209, "pid": 437675, "tid": 439084, "ts": "1704161511432433.172", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432443.062", "dur": 8.59, "cat": "dequeue", "args": {"correlation_id": 210}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 210, "pid": 437675, "tid": 439084, "ts": "1704161511432443.062", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511432453.422", "dur": 11.51, "cat": "dequeue", "args": {"correlation_id": 211}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 211, "pid": 437675, "tid": 439084, "ts": "1704161511432453.422", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511432468.282", "dur": 10.31, "cat": "dequeue", "args": {"correlation_id": 212}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 212, "pid": 437675, "tid": 439084, "ts": "1704161511432468.282", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511432480.822", "dur": 42.771, "cat": "dequeue", "args": {"correlation_id": 213}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 213, "pid": 437675, "tid": 439084, "ts": "1704161511432480.822", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511432525.823", "dur": 11.54, "cat": "dequeue", "args": {"correlation_id": 214}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 214, "pid": 437675, "tid": 439084, "ts": "1704161511432525.823", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511432538.703", "dur": 45.57, "cat": "dequeue", "args": {"correlation_id": 215}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 215, "pid": 437675, "tid": 439084, "ts": "1704161511432538.703", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432586.643", "dur": 10.111, "cat": "dequeue", "args": {"correlation_id": 216}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 216, "pid": 437675, "tid": 439084, "ts": "1704161511432586.643", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511432597.924", "dur": 8.58, "cat": "dequeue", "args": {"correlation_id": 217}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 217, "pid": 437675, "tid": 439084, "ts": "1704161511432597.924", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432608.044", "dur": 8.25, "cat": "dequeue", "args": {"correlation_id": 218}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 218, "pid": 437675, "tid": 439084, "ts": "1704161511432608.044", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511432617.454", "dur": 8.76, "cat": "dequeue", "args": {"correlation_id": 219}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 219, "pid": 437675, "tid": 439084, "ts": "1704161511432617.454", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511432627.424", "dur": 8.48, "cat": "dequeue", "args": {"correlation_id": 220}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 220, "pid": 437675, "tid": 439084, "ts": "1704161511432627.424", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511432637.564", "dur": 28.93, "cat": "dequeue", "args": {"correlation_id": 221}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 221, "pid": 437675, "tid": 439084, "ts": "1704161511432637.564", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511432668.274", "dur": 8.65, "cat": "dequeue", "args": {"correlation_id": 222}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 222, "pid": 437675, "tid": 439084, "ts": "1704161511432668.274", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511432678.164", "dur": 30.831, "cat": "dequeue", "args": {"correlation_id": 223}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 223, "pid": 437675, "tid": 439084, "ts": "1704161511432678.164", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432710.885", "dur": 9.14, "cat": "dequeue", "args": {"correlation_id": 224}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 224, "pid": 437675, "tid": 439084, "ts": "1704161511432710.885", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511432721.335", "dur": 8.6, "cat": "dequeue", "args": {"correlation_id": 225}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 225, "pid": 437675, "tid": 439084, "ts": "1704161511432721.335", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432731.175", "dur": 7.51, "cat": "dequeue", "args": {"correlation_id": 226}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 226, "pid": 437675, "tid": 439084, "ts": "1704161511432731.175", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511432739.885", "dur": 8.47, "cat": "dequeue", "args": {"correlation_id": 227}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 227, "pid": 437675, "tid": 439084, "ts": "1704161511432739.885", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511432750.015", "dur": 7.16, "cat": "dequeue", "args": {"correlation_id": 228}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 228, "pid": 437675, "tid": 439084, "ts": "1704161511432750.015", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511432758.715", "dur": 27.39, "cat": "dequeue", "args": {"correlation_id": 229}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 229, "pid": 437675, "tid": 439084, "ts": "1704161511432758.715", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511432787.806", "dur": 9.75, "cat": "dequeue", "args": {"correlation_id": 230}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 230, "pid": 437675, "tid": 439084, "ts": "1704161511432787.806", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511432798.776", "dur": 30.37, "cat": "dequeue", "args": {"correlation_id": 231}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 231, "pid": 437675, "tid": 439084, "ts": "1704161511432798.776", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432831.116", "dur": 9.36, "cat": "dequeue", "args": {"correlation_id": 232}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 232, "pid": 437675, "tid": 439084, "ts": "1704161511432831.116", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511432841.746", "dur": 8.12, "cat": "dequeue", "args": {"correlation_id": 233}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 233, "pid": 437675, "tid": 439084, "ts": "1704161511432841.746", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432851.576", "dur": 7.67, "cat": "dequeue", "args": {"correlation_id": 234}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 234, "pid": 437675, "tid": 439084, "ts": "1704161511432851.576", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511432860.416", "dur": 8.02, "cat": "dequeue", "args": {"correlation_id": 235}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 235, "pid": 437675, "tid": 439084, "ts": "1704161511432860.416", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511432870.106", "dur": 19.701, "cat": "dequeue", "args": {"correlation_id": 236}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 236, "pid": 437675, "tid": 439084, "ts": "1704161511432870.106", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511432891.347", "dur": 28.97, "cat": "dequeue", "args": {"correlation_id": 237}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 237, "pid": 437675, "tid": 439084, "ts": "1704161511432891.347", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511432922.427", "dur": 8.92, "cat": "dequeue", "args": {"correlation_id": 238}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 238, "pid": 437675, "tid": 439084, "ts": "1704161511432922.427", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511432933.367", "dur": 29.67, "cat": "dequeue", "args": {"correlation_id": 239}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 239, "pid": 437675, "tid": 439084, "ts": "1704161511432933.367", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432964.797", "dur": 8.73, "cat": "dequeue", "args": {"correlation_id": 240}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 240, "pid": 437675, "tid": 439084, "ts": "1704161511432964.797", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511432974.767", "dur": 7.07, "cat": "dequeue", "args": {"correlation_id": 241}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 241, "pid": 437675, "tid": 439084, "ts": "1704161511432974.767", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511432984.307", "dur": 7.431, "cat": "dequeue", "args": {"correlation_id": 242}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 242, "pid": 437675, "tid": 439084, "ts": "1704161511432984.307", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511432992.968", "dur": 8.17, "cat": "dequeue", "args": {"correlation_id": 243}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 243, "pid": 437675, "tid": 439084, "ts": "1704161511432992.968", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511433002.668", "dur": 6.99, "cat": "dequeue", "args": {"correlation_id": 244}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 244, "pid": 437675, "tid": 439084, "ts": "1704161511433002.668", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511433010.778", "dur": 26.12, "cat": "dequeue", "args": {"correlation_id": 245}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 245, "pid": 437675, "tid": 439084, "ts": "1704161511433010.778", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511433038.848", "dur": 8.6, "cat": "dequeue", "args": {"correlation_id": 246}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 246, "pid": 437675, "tid": 439084, "ts": "1704161511433038.848", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511433049.118", "dur": 29.69, "cat": "dequeue", "args": {"correlation_id": 247}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 247, "pid": 437675, "tid": 439084, "ts": "1704161511433049.118", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433080.468", "dur": 8.661, "cat": "dequeue", "args": {"correlation_id": 248}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 248, "pid": 437675, "tid": 439084, "ts": "1704161511433080.468", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511433090.569", "dur": 7.33, "cat": "dequeue", "args": {"correlation_id": 249}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 249, "pid": 437675, "tid": 439084, "ts": "1704161511433090.569", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433099.589", "dur": 7.78, "cat": "dequeue", "args": {"correlation_id": 250}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 250, "pid": 437675, "tid": 439084, "ts": "1704161511433099.589", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511433109.439", "dur": 8.09, "cat": "dequeue", "args": {"correlation_id": 251}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 251, "pid": 437675, "tid": 439084, "ts": "1704161511433109.439", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511433118.819", "dur": 6.81, "cat": "dequeue", "args": {"correlation_id": 252}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 252, "pid": 437675, "tid": 439084, "ts": "1704161511433118.819", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511433126.789", "dur": 25.53, "cat": "dequeue", "args": {"correlation_id": 253}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 253, "pid": 437675, "tid": 439084, "ts": "1704161511433126.789", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511433154.069", "dur": 8.92, "cat": "dequeue", "args": {"correlation_id": 254}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 254, "pid": 437675, "tid": 439084, "ts": "1704161511433154.069", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511433164.759", "dur": 29.461, "cat": "dequeue", "args": {"correlation_id": 255}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 255, "pid": 437675, "tid": 439084, "ts": "1704161511433164.759", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433195.970", "dur": 14.23, "cat": "dequeue", "args": {"correlation_id": 256}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 256, "pid": 437675, "tid": 439084, "ts": "1704161511433195.970", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511433211.800", "dur": 9.11, "cat": "dequeue", "args": {"correlation_id": 257}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 257, "pid": 437675, "tid": 439084, "ts": "1704161511433211.800", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433222.670", "dur": 7.81, "cat": "dequeue", "args": {"correlation_id": 258}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 258, "pid": 437675, "tid": 439084, "ts": "1704161511433222.670", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511433231.710", "dur": 8.33, "cat": "dequeue", "args": {"correlation_id": 259}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 259, "pid": 437675, "tid": 439084, "ts": "1704161511433231.710", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511433241.360", "dur": 6.86, "cat": "dequeue", "args": {"correlation_id": 260}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 260, "pid": 437675, "tid": 439084, "ts": "1704161511433241.360", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511433249.470", "dur": 27.93, "cat": "dequeue", "args": {"correlation_id": 261}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 261, "pid": 437675, "tid": 439084, "ts": "1704161511433249.470", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511433279.480", "dur": 8.61, "cat": "dequeue", "args": {"correlation_id": 262}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 262, "pid": 437675, "tid": 439084, "ts": "1704161511433279.480", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511433289.291", "dur": 29.4, "cat": "dequeue", "args": {"correlation_id": 263}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 263, "pid": 437675, "tid": 439084, "ts": "1704161511433289.291", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433320.371", "dur": 8.61, "cat": "dequeue", "args": {"correlation_id": 264}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 264, "pid": 437675, "tid": 439084, "ts": "1704161511433320.371", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511433330.691", "dur": 7.92, "cat": "dequeue", "args": {"correlation_id": 265}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 265, "pid": 437675, "tid": 439084, "ts": "1704161511433330.691", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433340.121", "dur": 7.42, "cat": "dequeue", "args": {"correlation_id": 266}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 266, "pid": 437675, "tid": 439084, "ts": "1704161511433340.121", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511433348.821", "dur": 8.01, "cat": "dequeue", "args": {"correlation_id": 267}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 267, "pid": 437675, "tid": 439084, "ts": "1704161511433348.821", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511433358.211", "dur": 6.85, "cat": "dequeue", "args": {"correlation_id": 268}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 268, "pid": 437675, "tid": 439084, "ts": "1704161511433358.211", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511433366.231", "dur": 26.091, "cat": "dequeue", "args": {"correlation_id": 269}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 269, "pid": 437675, "tid": 439084, "ts": "1704161511433366.231", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511433394.652", "dur": 8.34, "cat": "dequeue", "args": {"correlation_id": 270}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 270, "pid": 437675, "tid": 439084, "ts": "1704161511433394.652", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511433404.682", "dur": 29.04, "cat": "dequeue", "args": {"correlation_id": 271}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 271, "pid": 437675, "tid": 439084, "ts": "1704161511433404.682", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433435.592", "dur": 8.88, "cat": "dequeue", "args": {"correlation_id": 272}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 272, "pid": 437675, "tid": 439084, "ts": "1704161511433435.592", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511433446.452", "dur": 7.79, "cat": "dequeue", "args": {"correlation_id": 273}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 273, "pid": 437675, "tid": 439084, "ts": "1704161511433446.452", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433455.512", "dur": 7.04, "cat": "dequeue", "args": {"correlation_id": 274}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 274, "pid": 437675, "tid": 439084, "ts": "1704161511433455.512", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511433468.282", "dur": 8.75, "cat": "dequeue", "args": {"correlation_id": 275}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 275, "pid": 437675, "tid": 439084, "ts": "1704161511433468.282", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511433478.682", "dur": 7.36, "cat": "dequeue", "args": {"correlation_id": 276}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 276, "pid": 437675, "tid": 439084, "ts": "1704161511433478.682", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511433487.272", "dur": 28.061, "cat": "dequeue", "args": {"correlation_id": 277}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 277, "pid": 437675, "tid": 439084, "ts": "1704161511433487.272", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511433517.313", "dur": 8.68, "cat": "dequeue", "args": {"correlation_id": 278}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 278, "pid": 437675, "tid": 439084, "ts": "1704161511433517.313", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511433527.423", "dur": 29.78, "cat": "dequeue", "args": {"correlation_id": 279}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 279, "pid": 437675, "tid": 439084, "ts": "1704161511433527.423", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433558.703", "dur": 9.01, "cat": "dequeue", "args": {"correlation_id": 280}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 280, "pid": 437675, "tid": 439084, "ts": "1704161511433558.703", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511433569.443", "dur": 7.68, "cat": "dequeue", "args": {"correlation_id": 281}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 281, "pid": 437675, "tid": 439084, "ts": "1704161511433569.443", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceMuls", "pid": 437675, "tid": 439084, "ts": "1704161511433578.383", "dur": 6.93, "cat": "dequeue", "args": {"correlation_id": 282}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 282, "pid": 437675, "tid": 439084, "ts": "1704161511433578.383", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcmul", "pid": 437675, "tid": 439084, "ts": "1704161511433586.543", "dur": 7.471, "cat": "dequeue", "args": {"correlation_id": 283}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 283, "pid": 437675, "tid": 439084, "ts": "1704161511433586.543", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnSqrt", "pid": 437675, "tid": 439084, "ts": "1704161511433598.074", "dur": 7.12, "cat": "dequeue", "args": {"correlation_id": 284}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 284, "pid": 437675, "tid": 439084, "ts": "1704161511433598.074", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511433606.634", "dur": 29.06, "cat": "dequeue", "args": {"correlation_id": 285}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 285, "pid": 437675, "tid": 439084, "ts": "1704161511433606.634", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdds", "pid": 437675, "tid": 439084, "ts": "1704161511433637.924", "dur": 8.22, "cat": "dequeue", "args": {"correlation_id": 286}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 286, "pid": 437675, "tid": 439084, "ts": "1704161511433637.924", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAddcdiv", "pid": 437675, "tid": 439084, "ts": "1704161511433647.444", "dur": 28.28, "cat": "dequeue", "args": {"correlation_id": 287}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 287, "pid": 437675, "tid": 439084, "ts": "1704161511433647.444", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511433677.384", "dur": 8.04, "cat": "dequeue", "args": {"correlation_id": 288}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 288, "pid": 437675, "tid": 439084, "ts": "1704161511433677.384", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnArgMax", "pid": 437675, "tid": 439084, "ts": "1704161511433687.094", "dur": 15.151, "cat": "dequeue", "args": {"correlation_id": 289}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 289, "pid": 437675, "tid": 439084, "ts": "1704161511433687.094", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnEqTensor", "pid": 437675, "tid": 439084, "ts": "1704161511433703.595", "dur": 10.72, "cat": "dequeue", "args": {"correlation_id": 290}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 290, "pid": 437675, "tid": 439084, "ts": "1704161511433703.595", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnReduceSum", "pid": 437675, "tid": 439084, "ts": "1704161511433715.545", "dur": 121.881, "cat": "dequeue", "args": {"correlation_id": 291}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 291, "pid": 437675, "tid": 439084, "ts": "1704161511433715.545", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnCast", "pid": 437675, "tid": 439084, "ts": "1704161511433847.346", "dur": 17.8, "cat": "dequeue", "args": {"correlation_id": 292}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 292, "pid": 437675, "tid": 439084, "ts": "1704161511433847.346", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnInplaceAdd", "pid": 437675, "tid": 439084, "ts": "1704161511433867.346", "dur": 9.33, "cat": "dequeue", "args": {"correlation_id": 293}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 293, "pid": 437675, "tid": 439084, "ts": "1704161511433867.346", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511433878.276", "dur": 8.13, "cat": "dequeue", "args": {"correlation_id": 294}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 294, "pid": 437675, "tid": 439084, "ts": "1704161511433878.276", "cat": "async_task_queue"}, {"ph": "X", "name": "Dequeue@aclnnDivs", "pid": 437675, "tid": 439084, "ts": "1704161511433887.776", "dur": 6.751, "cat": "dequeue", "args": {"correlation_id": 295}}, {"ph": "f", "bp": "e", "name": "enqueue_to_dequeue", "id": 295, "pid": 437675, "tid": 439084, "ts": "1704161511433887.776", "cat": "async_task_queue"}, {"ph": "M", "name": "process_name", "pid": 437675, "tid": 0, "args": {"name": "Python"}}, {"ph": "M", "name": "process_labels", "pid": 437675, "tid": 0, "args": {"labels": "CPU"}}, {"ph": "M", "name": "process_sort_index", "pid": 437675, "tid": 0, "args": {"sort_index": 0}}, {"ph": "M", "name": "thread_name", "pid": 437675, "tid": 437675, "args": {"name": "Thread 437675"}}, {"ph": "M", "name": "thread_sort_index", "pid": 437675, "tid": 437675, "args": {"sort_index": 437675}}, {"ph": "M", "name": "thread_name", "pid": 437675, "tid": 439228, "args": {"name": "Thread 439228"}}, {"ph": "M", "name": "thread_sort_index", "pid": 437675, "tid": 439228, "args": {"sort_index": 439228}}, {"ph": "M", "name": "thread_name", "pid": 437675, "tid": 439084, "args": {"name": "Thread 439084"}}, {"ph": "M", "name": "thread_sort_index", "pid": 437675, "tid": 439084, "args": {"sort_index": 439229}}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 17, "pid": 437675, "tid": 437675, "ts": "1704161511423098.389", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 17, "pid": 437675, "tid": 439228, "ts": "1704161511428757.055", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 18, "pid": 437675, "tid": 437675, "ts": "1704161511423234.760", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 18, "pid": 437675, "tid": 439228, "ts": "1704161511428726.245", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 19, "pid": 437675, "tid": 437675, "ts": "1704161511423348.621", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 19, "pid": 437675, "tid": 439228, "ts": "1704161511428668.254", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 20, "pid": 437675, "tid": 437675, "ts": "1704161511423433.722", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 20, "pid": 437675, "tid": 439228, "ts": "1704161511428228.320", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 21, "pid": 437675, "tid": 437675, "ts": "1704161511423480.703", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 21, "pid": 437675, "tid": 439228, "ts": "1704161511428193.900", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 22, "pid": 437675, "tid": 437675, "ts": "1704161511423526.163", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 22, "pid": 437675, "tid": 439228, "ts": "1704161511428092.079", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 23, "pid": 437675, "tid": 437675, "ts": "1704161511423572.793", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 23, "pid": 437675, "tid": 439228, "ts": "1704161511428070.708", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 24, "pid": 437675, "tid": 437675, "ts": "1704161511423615.164", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 24, "pid": 437675, "tid": 439228, "ts": "1704161511427996.438", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 25, "pid": 437675, "tid": 437675, "ts": "1704161511423634.704", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 25, "pid": 437675, "tid": 439228, "ts": "1704161511427720.065", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 26, "pid": 437675, "tid": 437675, "ts": "1704161511423680.365", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 26, "pid": 437675, "tid": 439228, "ts": "1704161511427687.485", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 27, "pid": 437675, "tid": 437675, "ts": "1704161511423715.015", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 27, "pid": 437675, "tid": 439228, "ts": "1704161511427613.274", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 28, "pid": 437675, "tid": 437675, "ts": "1704161511423725.355", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 28, "pid": 437675, "tid": 439228, "ts": "1704161511427440.652", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 29, "pid": 437675, "tid": 437675, "ts": "1704161511423751.965", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 29, "pid": 437675, "tid": 439228, "ts": "1704161511427373.241", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 30, "pid": 437675, "tid": 437675, "ts": "1704161511423783.516", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 30, "pid": 437675, "tid": 439228, "ts": "1704161511427285.141", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 31, "pid": 437675, "tid": 437675, "ts": "1704161511423793.646", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 31, "pid": 437675, "tid": 439228, "ts": "1704161511426950.567", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 32, "pid": 437675, "tid": 437675, "ts": "1704161511426415.582", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 32, "pid": 437675, "tid": 439228, "ts": "1704161511426883.167", "cat": "fwdbwd"}, {"ph": "s", "bp": "e", "name": "fwdbwd", "id": 33, "pid": 437675, "tid": 437675, "ts": "1704161511426502.433", "cat": "fwdbwd"}, {"ph": "f", "bp": "e", "name": "fwdbwd", "id": 33, "pid": 437675, "tid": 439228, "ts": "1704161511426719.085", "cat": "fwdbwd"},{"name": "process_name", "pid": 800, "tid": 0, "args": {"name": "Ascend Hardware"}, "ph": "M"}, {"name": "thread_name", "pid": 800, "tid": 3, "args": {"name": "Stream 3"}, "ph": "M"}, {"name": "thread_sort_index", "pid": 800, "tid": 3, "args": {"sort_index": 3}, "ph": "M"}, {"name": "process_labels", "pid": 800, "tid": 0, "args": {"labels": "NPU"}, "ph": "M"}, {"name": "process_sort_index", "pid": 800, "tid": 0, "args": {"sort_index": 8}, "ph": "M"}, {"name": "process_name", "pid": 43767502, "tid": 0, "args": {"name": "CANN"}, "ph": "M"}, {"name": "thread_name", "pid": 43767502, "tid": 439228, "args": {"name": "Thread 439228"}, "ph": "M"}, {"name": "thread_name", "pid": 43767502, "tid": 437675, "args": {"name": "Thread 437675"}, "ph": "M"}, {"name": "thread_name", "pid": 43767502, "tid": 439084, "args": {"name": "Thread 439084"}, "ph": "M"}, {"name": "thread_name", "pid": 43767502, "tid": 439085, "args": {"name": "Thread 439085"}, "ph": "M"}, {"name": "thread_sort_index", "pid": 43767502, "tid": 439228, "args": {"sort_index": 439228}, "ph": "M"}, {"name": "thread_sort_index", "pid": 43767502, "tid": 437675, "args": {"sort_index": 437675}, "ph": "M"}, {"name": "thread_sort_index", "pid": 43767502, "tid": 439084, "args": {"sort_index": 439084}, "ph": "M"}, {"name": "thread_sort_index", "pid": 43767502, "tid": 439085, "args": {"sort_index": 439085}, "ph": "M"}, {"name": "process_labels", "pid": 43767502, "tid": 0, "args": {"labels": "CPU"}, "ph": "M"}, {"name": "process_sort_index", "pid": 43767502, "tid": 0, "args": {"sort_index": 2}, "ph": "M"}, {"name": "process_name", "pid": 4376751100, "tid": 0, "args": {"name": "Overlap Analysis"}, "ph": "M"}, {"name": "thread_name", "pid": 4376751100, "tid": 0, "args": {"name": "Computing"}, "ph": "M"}, {"name": "thread_name", "pid": 4376751100, "tid": 1, "args": {"name": "Communication"}, "ph": "M"}, {"name": "thread_name", "pid": 4376751100, "tid": 2, "args": {"name": "Communication(Not Overlapped)"}, "ph": "M"}, {"name": "thread_name", "pid": 4376751100, "tid": 3, "args": {"name": "Free"}, "ph": "M"}, {"name": "process_labels", "pid": 4376751100, "tid": 0, "args": {"labels": "NPU"}, "ph": "M"}, {"name": "process_sort_index", "pid": 4376751100, "tid": 0, "args": {"sort_index": 11}, "ph": "M"}, {"name": "AscendCL@aclrtSynchronizeStream", "pid": 43767502, "tid": 437675, "ts": "1704161511420589.043", "dur": 12.77012757357446, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtSynchronizeStream", "item_id": "0", "connection_id": 0}, "ph": "X"}, {"name": "AscendCL@aclrtMemcpy", "pid": 43767502, "tid": 437675, "ts": "1704161511420605.703", "dur": 52.40052348122958, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtMemcpy", "item_id": "0", "connection_id": 1}, "ph": "X"}, {"name": "AscendCL@aclrtSynchronizeStream", "pid": 43767502, "tid": 437675, "ts": "1704161511420768.565", "dur": 3.72003716317126, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtSynchronizeStream", "item_id": "0", "connection_id": 2}, "ph": "X"}, {"name": "AscendCL@aclrtMemcpy", "pid": 43767502, "tid": 437675, "ts": "1704161511420773.545", "dur": 24.910248853386044, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtMemcpy", "item_id": "0", "connection_id": 3}, "ph": "X"}, {"name": "AscendCL@aclrtSynchronizeStream", "pid": 43767502, "tid": 437675, "ts": "1704161511422511.373", "dur": 6.250062438123757, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtSynchronizeStream", "item_id": "0", "connection_id": 4}, "ph": "X"}, {"name": "AscendCL@aclrtMemcpy", "pid": 43767502, "tid": 437675, "ts": "1704161511422519.282", "dur": 98.43098332552343, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtMemcpy", "item_id": "0", "connection_id": 5}, "ph": "X"}, {"name": "AscendCL@aclrtSynchronizeStream", "pid": 43767502, "tid": 437675, "ts": "1704161511422674.694", "dur": 3.680036763567268, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtSynchronizeStream", "item_id": "0", "connection_id": 6}, "ph": "X"}, {"name": "AscendCL@aclrtMemcpy", "pid": 43767502, "tid": 437675, "ts": "1704161511422679.264", "dur": 24.1302410611082, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtMemcpy", "item_id": "0", "connection_id": 7}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422896.956", "dur": 21.100210791105802, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 9}, "ph": "X"}, {"name": "HostToDevice38654705663", "ph": "s", "cat": "HostToDevice", "id": "38654705663", "pid": 43767502, "tid": 439084, "ts": "1704161511422898.906"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422904.204", "dur": 1.35, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 279, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 8}, "ph": "X"}, {"name": "HostToDevice38654705663", "ph": "f", "id": "38654705663", "ts": "1704161511422904.204", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422904.204", "dur": 1.35, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422905.554", "dur": 11.71, "ph": "X"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422917.264", "dur": 1.2695, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 280, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 10}, "ph": "X"}, {"name": "HostToDevice47244640255", "ph": "f", "id": "47244640255", "ts": "1704161511422917.264", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422917.264", "dur": 1.2695, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422918.533", "dur": 31.51, "ph": "X"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422925.227", "dur": 8.020080120600404, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 11}, "ph": "X"}, {"name": "HostToDevice47244640255", "ph": "s", "cat": "HostToDevice", "id": "47244640255", "pid": 43767502, "tid": 439084, "ts": "1704161511422926.436"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422936.867", "dur": 6.910069031589626, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 13}, "ph": "X"}, {"name": "HostToDevice55834574847", "ph": "s", "cat": "HostToDevice", "id": "55834574847", "pid": 43767502, "tid": 439084, "ts": "1704161511422937.387"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422947.936", "dur": 6.510065035549705, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 15}, "ph": "X"}, {"name": "HostToDevice64424509439", "ph": "s", "cat": "HostToDevice", "id": "64424509439", "pid": 43767502, "tid": 439084, "ts": "1704161511422948.477"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422950.043", "dur": 2.52, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 281, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 12}, "ph": "X"}, {"name": "HostToDevice55834574847", "ph": "f", "id": "55834574847", "ts": "1704161511422950.043", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422950.043", "dur": 2.52, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422952.563", "dur": 0.11, "ph": "X"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422952.673", "dur": 1.26, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 282, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 14}, "ph": "X"}, {"name": "HostToDevice64424509439", "ph": "f", "id": "64424509439", "ts": "1704161511422952.673", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422952.673", "dur": 1.26, "ph": "X"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422954.043", "dur": 4.21, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 283, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 16}, "ph": "X"}, {"name": "HostToDevice73014444031", "ph": "f", "id": "73014444031", "ts": "1704161511422954.043", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422954.043", "dur": 4.21, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422953.933", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422957.787", "dur": 7.850078422283438, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 17}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422958.253", "dur": 4.0, "ph": "X"}, {"name": "HostToDevice73014444031", "ph": "s", "cat": "HostToDevice", "id": "73014444031", "pid": 43767502, "tid": 439084, "ts": "1704161511422958.727"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422962.253", "dur": 1.34, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 284, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 18}, "ph": "X"}, {"name": "HostToDevice81604378623", "ph": "f", "id": "81604378623", "ts": "1704161511422962.253", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422962.253", "dur": 1.34, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422963.593", "dur": 4.5395, "ph": "X"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422968.132", "dur": 2.15, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 285, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 20}, "ph": "X"}, {"name": "HostToDevice90194313215", "ph": "f", "id": "90194313215", "ts": "1704161511422968.132", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422968.132", "dur": 2.15, "ph": "X"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422969.347", "dur": 6.140061339212778, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 19}, "ph": "X"}, {"name": "HostToDevice81604378623", "ph": "s", "cat": "HostToDevice", "id": "81604378623", "pid": 43767502, "tid": 439084, "ts": "1704161511422969.747"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422970.282", "dur": 12.75, "ph": "X"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422979.117", "dur": 6.730067233371662, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 21}, "ph": "X"}, {"name": "HostToDevice90194313215", "ph": "s", "cat": "HostToDevice", "id": "90194313215", "pid": 43767502, "tid": 439084, "ts": "1704161511422979.827"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422983.032", "dur": 1.25, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 286, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 22}, "ph": "X"}, {"name": "HostToDevice98784247807", "ph": "f", "id": "98784247807", "ts": "1704161511422983.032", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422983.032", "dur": 1.25, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422984.282", "dur": 0.11, "ph": "X"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422984.392", "dur": 1.32, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 287, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 24}, "ph": "X"}, {"name": "HostToDevice107374182399", "ph": "f", "id": "107374182399", "ts": "1704161511422984.392", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422984.392", "dur": 1.32, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422985.712", "dur": 5.32, "ph": "X"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422989.447", "dur": 6.380063736836731, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 23}, "ph": "X"}, {"name": "HostToDevice98784247807", "ph": "s", "cat": "HostToDevice", "id": "98784247807", "pid": 43767502, "tid": 439084, "ts": "1704161511422989.787"}, {"name": "aclnnInplaceZero_ZerosLikeAiCore_ZerosLike", "pid": 800, "tid": 3, "ts": "1704161511422991.032", "dur": 1.25, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 288, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 26}, "ph": "X"}, {"name": "HostToDevice115964116991", "ph": "f", "id": "115964116991", "ts": "1704161511422991.032", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511422991.032", "dur": 1.25, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511422992.282", "dur": 1093.429, "ph": "X"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511422998.767", "dur": 6.2000619386187665, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 25}, "ph": "X"}, {"name": "HostToDevice107374182399", "ph": "s", "cat": "HostToDevice", "id": "107374182399", "pid": 43767502, "tid": 439084, "ts": "1704161511422999.217"}, {"name": "AscendCL@aclnnInplaceZero", "pid": 43767502, "tid": 439084, "ts": "1704161511423008.148", "dur": 5.890058841687829, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceZero", "item_id": "0", "connection_id": 27}, "ph": "X"}, {"name": "HostToDevice115964116991", "ph": "s", "cat": "HostToDevice", "id": "115964116991", "pid": 43767502, "tid": 439084, "ts": "1704161511423008.527"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423176.929", "dur": 1.890018881288624, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 28}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423184.389", "dur": 0.6700066933668668, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 29}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423186.669", "dur": 0.2800027972279443, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 30}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423187.659", "dur": 0.1900018981189622, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 31}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423188.659", "dur": 0.24000239762395226, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 32}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423189.249", "dur": 0.15000149851497016, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 33}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423191.109", "dur": 0.25000249752495024, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 34}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423192.149", "dur": 0.21000209792095823, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 35}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 437675, "ts": "1704161511423193.189", "dur": 0.3600035964359284, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 36}, "ph": "X"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511423223.879", "dur": 1099.9709887101772, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 70}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511423227.410", "dur": 27.990279622893432, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 37}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423378.411", "dur": 0.5900058941588827, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 38}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423382.711", "dur": 0.24000239762395226, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 39}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423383.801", "dur": 0.2800027972279443, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 40}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423384.691", "dur": 0.1900018981189622, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 41}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423386.501", "dur": 0.2600025974259483, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 42}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423387.301", "dur": 0.14000139861397215, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 43}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 437675, "ts": "1704161511423388.111", "dur": 0.34000339663393236, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 44}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423453.302", "dur": 0.5700056943568866, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 45}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423455.412", "dur": 0.22000219782195624, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 46}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423456.602", "dur": 0.44000439564391247, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 47}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423457.652", "dur": 0.16000159841596817, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 48}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423458.592", "dur": 0.3600035964359284, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 49}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423459.312", "dur": 0.23000229772295425, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 50}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423460.212", "dur": 0.40000399603992043, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 51}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423460.882", "dur": 0.14000139861397215, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 52}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 437675, "ts": "1704161511423461.552", "dur": 0.15000149851497016, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 53}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423541.533", "dur": 0.5400053946538926, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 54}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423543.733", "dur": 0.25000249752495024, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 55}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423544.433", "dur": 0.3800037962379244, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 56}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423545.133", "dur": 0.23000229772295425, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 57}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 437675, "ts": "1704161511423546.133", "dur": 0.41000409594091847, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 58}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 437675, "ts": "1704161511423546.923", "dur": 0.15000149851497016, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 59}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 437675, "ts": "1704161511423547.473", "dur": 0.2700026973269463, "args": {"Thread Id": 437675, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 60}, "ph": "X"}, {"name": "HostToDevice266287972351", "ph": "s", "cat": "HostToDevice", "id": "266287972351", "pid": 43767502, "tid": 439084, "ts": "1704161511424082.978"}, {"name": "trans_Cast_0", "pid": 800, "tid": 3, "ts": "1704161511424085.712", "dur": 4.28, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 289, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 61}, "ph": "X"}, {"name": "HostToDevice266287972351", "ph": "f", "id": "266287972351", "ts": "1704161511424085.712", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424085.712", "dur": 4.28, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424089.992", "dur": 47.4395, "ph": "X"}, {"name": "trans_TransData_1", "pid": 800, "tid": 3, "ts": "1704161511424137.431", "dur": 9.96, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 290, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 62}, "ph": "X"}, {"name": "HostToDevice270582939647", "ph": "f", "id": "270582939647", "ts": "1704161511424137.431", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424137.431", "dur": 9.96, "ph": "X"}, {"name": "HostToDevice270582939647", "ph": "s", "cat": "HostToDevice", "id": "270582939647", "pid": 43767502, "tid": 439084, "ts": "1704161511424145.799"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424147.391", "dur": 11.61, "ph": "X"}, {"name": "trans_Cast_2", "pid": 800, "tid": 3, "ts": "1704161511424159.001", "dur": 1.44, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 291, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 63}, "ph": "X"}, {"name": "HostToDevice274877906943", "ph": "f", "id": "274877906943", "ts": "1704161511424159.001", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424159.001", "dur": 1.44, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424160.441", "dur": 36.7895, "ph": "X"}, {"name": "HostToDevice274877906943", "ph": "s", "cat": "HostToDevice", "id": "274877906943", "pid": 43767502, "tid": 439084, "ts": "1704161511424176.029"}, {"name": "trans_TransData_3_MemSet", "pid": 800, "tid": 3, "ts": "1704161511424197.230", "dur": 8.25, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 292, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 64}, "ph": "X"}, {"name": "HostToDevice279172874239", "ph": "f", "id": "279172874239", "ts": "1704161511424197.230", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424197.230", "dur": 8.25, "ph": "X"}, {"name": "HostToDevice279172874239", "ph": "s", "cat": "HostToDevice", "id": "279172874239", "pid": 43767502, "tid": 439084, "ts": "1704161511424204.480"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424205.480", "dur": 10.29, "ph": "X"}, {"name": "HostToDevice283467841535", "ph": "s", "cat": "HostToDevice", "id": "283467841535", "pid": 43767502, "tid": 439084, "ts": "1704161511424214.730"}, {"name": "trans_TransData_3", "pid": 800, "tid": 3, "ts": "1704161511424215.770", "dur": 5.3495, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 293, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 65}, "ph": "X"}, {"name": "HostToDevice283467841535", "ph": "f", "id": "283467841535", "ts": "1704161511424215.770", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424215.770", "dur": 5.3495, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424221.120", "dur": 2.81, "ph": "X"}, {"name": "trans_Cast_4", "pid": 800, "tid": 3, "ts": "1704161511424223.930", "dur": 1.21, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 294, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 66}, "ph": "X"}, {"name": "HostToDevice287762808831", "ph": "f", "id": "287762808831", "ts": "1704161511424223.930", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424223.930", "dur": 1.21, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424225.140", "dur": 32.73, "ph": "X"}, {"name": "HostToDevice287762808831", "ph": "s", "cat": "HostToDevice", "id": "287762808831", "pid": 43767502, "tid": 439084, "ts": "1704161511424231.850"}, {"name": "Conv2D1", "pid": 800, "tid": 3, "ts": "1704161511424257.870", "dur": 22.5995, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 295, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 67}, "ph": "X"}, {"name": "HostToDevice292057776127", "ph": "f", "id": "292057776127", "ts": "1704161511424257.870", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424257.870", "dur": 22.5995, "ph": "X"}, {"name": "HostToDevice292057776127", "ph": "s", "cat": "HostToDevice", "id": "292057776127", "pid": 43767502, "tid": 439084, "ts": "1704161511424266.080"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424280.470", "dur": 5.79, "ph": "X"}, {"name": "trans_TransData_5", "pid": 800, "tid": 3, "ts": "1704161511424286.260", "dur": 7.84, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 296, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 68}, "ph": "X"}, {"name": "HostToDevice296352743423", "ph": "f", "id": "296352743423", "ts": "1704161511424286.260", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424286.260", "dur": 7.84, "ph": "X"}, {"name": "HostToDevice296352743423", "ph": "s", "cat": "HostToDevice", "id": "296352743423", "pid": 43767502, "tid": 439084, "ts": "1704161511424292.700"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424294.100", "dur": 0.11, "ph": "X"}, {"name": "trans_Cast_6", "pid": 800, "tid": 3, "ts": "1704161511424294.210", "dur": 5.92, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 297, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 69}, "ph": "X"}, {"name": "HostToDevice300647710719", "ph": "f", "id": "300647710719", "ts": "1704161511424294.210", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424294.210", "dur": 5.92, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424300.130", "dur": 33.2995, "ph": "X"}, {"name": "HostToDevice300647710719", "ph": "s", "cat": "HostToDevice", "id": "300647710719", "pid": 43767502, "tid": 439084, "ts": "1704161511424310.840"}, {"name": "aclnnRelu_Relu_Relu", "pid": 800, "tid": 3, "ts": "1704161511424333.429", "dur": 7.11, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 298, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 71}, "ph": "X"}, {"name": "HostToDevice309237645311", "ph": "f", "id": "309237645311", "ts": "1704161511424333.429", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424333.429", "dur": 7.11, "ph": "X"}, {"name": "AscendCL@aclnnRelu", "pid": 43767502, "tid": 439084, "ts": "1704161511424340.500", "dur": 11.270112588424757, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnRelu", "item_id": "0", "connection_id": 72}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424340.539", "dur": 497.405, "ph": "X"}, {"name": "HostToDevice309237645311", "ph": "s", "cat": "HostToDevice", "id": "309237645311", "pid": 43767502, "tid": 439084, "ts": "1704161511424341.750"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511424357.161", "dur": 610.6161000548395, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 84}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511424358.671", "dur": 16.85016833318165, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 73}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511424376.381", "dur": 13.220132069119371, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 74}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511424406.401", "dur": 4.660046553865073, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 75}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511424412.062", "dur": 2.7500274727744527, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 76}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511424415.691", "dur": 2.900028971289423, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 77}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511424420.361", "dur": 2.3500234767345325, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 78}, "ph": "X"}, {"name": "trans_Cast_7", "pid": 800, "tid": 3, "ts": "1704161511424837.944", "dur": 6.02, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 299, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 79}, "ph": "X"}, {"name": "HostToDevice343597383679", "ph": "f", "id": "343597383679", "ts": "1704161511424837.944", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424837.944", "dur": 6.02, "ph": "X"}, {"name": "HostToDevice343597383679", "ph": "s", "cat": "HostToDevice", "id": "343597383679", "pid": 43767502, "tid": 439084, "ts": "1704161511424841.526"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424843.964", "dur": 27.4495, "ph": "X"}, {"name": "trans_TransData_8", "pid": 800, "tid": 3, "ts": "1704161511424871.414", "dur": 9.32, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 300, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 80}, "ph": "X"}, {"name": "HostToDevice347892350975", "ph": "f", "id": "347892350975", "ts": "1704161511424871.414", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424871.414", "dur": 9.32, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424880.734", "dur": 27.2, "ph": "X"}, {"name": "HostToDevice347892350975", "ph": "s", "cat": "HostToDevice", "id": "347892350975", "pid": 43767502, "tid": 439084, "ts": "1704161511424881.266"}, {"name": "MaxPoolWithArgmaxV12", "pid": 800, "tid": 3, "ts": "1704161511424907.934", "dur": 14.4395, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 301, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 81}, "ph": "X"}, {"name": "HostToDevice352187318271", "ph": "f", "id": "352187318271", "ts": "1704161511424907.934", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424907.934", "dur": 14.4395, "ph": "X"}, {"name": "HostToDevice352187318271", "ph": "s", "cat": "HostToDevice", "id": "352187318271", "pid": 43767502, "tid": 439084, "ts": "1704161511424917.196"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424922.373", "dur": 7.05, "ph": "X"}, {"name": "trans_TransData_9", "pid": 800, "tid": 3, "ts": "1704161511424929.423", "dur": 10.43, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 302, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 82}, "ph": "X"}, {"name": "HostToDevice356482285567", "ph": "f", "id": "356482285567", "ts": "1704161511424929.423", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424929.423", "dur": 10.43, "ph": "X"}, {"name": "HostToDevice356482285567", "ph": "s", "cat": "HostToDevice", "id": "356482285567", "pid": 43767502, "tid": 439084, "ts": "1704161511424938.727"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424939.853", "dur": 6.15, "ph": "X"}, {"name": "trans_Cast_10", "pid": 800, "tid": 3, "ts": "1704161511424946.003", "dur": 4.68, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 303, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 83}, "ph": "X"}, {"name": "HostToDevice360777252863", "ph": "f", "id": "360777252863", "ts": "1704161511424946.003", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511424946.003", "dur": 4.68, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511424950.683", "dur": 746.5625, "ph": "X"}, {"name": "HostToDevice360777252863", "ph": "s", "cat": "HostToDevice", "id": "360777252863", "pid": 43767502, "tid": 439084, "ts": "1704161511424955.157"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511424977.097", "dur": 915.5591464358729, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 99}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511424978.807", "dur": 14.310142958328154, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 85}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511425015.387", "dur": 6.270062637925753, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 86}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511425023.618", "dur": 1.830018281882636, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 87}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511425026.627", "dur": 0.7800077922778449, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 88}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511425028.507", "dur": 0.5700056943568866, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 89}, "ph": "X"}, {"name": "trans_Cast_13", "pid": 800, "tid": 3, "ts": "1704161511425697.246", "dur": 4.03, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 304, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 90}, "ph": "X"}, {"name": "HostToDevice390842023935", "ph": "f", "id": "390842023935", "ts": "1704161511425697.246", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425697.246", "dur": 4.03, "ph": "X"}, {"name": "HostToDevice390842023935", "ph": "s", "cat": "HostToDevice", "id": "390842023935", "pid": 43767502, "tid": 439084, "ts": "1704161511425699.794"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425701.276", "dur": 32.4595, "ph": "X"}, {"name": "trans_TransData_14", "pid": 800, "tid": 3, "ts": "1704161511425733.735", "dur": 8.53, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 305, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 91}, "ph": "X"}, {"name": "HostToDevice395136991231", "ph": "f", "id": "395136991231", "ts": "1704161511425733.735", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425733.735", "dur": 8.53, "ph": "X"}, {"name": "HostToDevice395136991231", "ph": "s", "cat": "HostToDevice", "id": "395136991231", "pid": 43767502, "tid": 439084, "ts": "1704161511425742.195"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425742.265", "dur": 17.81, "ph": "X"}, {"name": "trans_Cast_15", "pid": 800, "tid": 3, "ts": "1704161511425760.075", "dur": 2.32, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 306, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 92}, "ph": "X"}, {"name": "HostToDevice399431958527", "ph": "f", "id": "399431958527", "ts": "1704161511425760.075", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425760.075", "dur": 2.32, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425762.395", "dur": 15.1695, "ph": "X"}, {"name": "HostToDevice399431958527", "ph": "s", "cat": "HostToDevice", "id": "399431958527", "pid": 43767502, "tid": 439084, "ts": "1704161511425768.945"}, {"name": "trans_TransData_16_MemSet", "pid": 800, "tid": 3, "ts": "1704161511425777.564", "dur": 8.97, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 307, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 93}, "ph": "X"}, {"name": "HostToDevice403726925823", "ph": "f", "id": "403726925823", "ts": "1704161511425777.564", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425777.564", "dur": 8.97, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425786.534", "dur": 0.11, "ph": "X"}, {"name": "trans_TransData_16", "pid": 800, "tid": 3, "ts": "1704161511425786.644", "dur": 6.82, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 308, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 94}, "ph": "X"}, {"name": "HostToDevice408021893119", "ph": "f", "id": "408021893119", "ts": "1704161511425786.644", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425786.644", "dur": 6.82, "ph": "X"}, {"name": "HostToDevice403726925823", "ph": "s", "cat": "HostToDevice", "id": "403726925823", "pid": 43767502, "tid": 439084, "ts": "1704161511425791.615"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425793.464", "dur": 9.58, "ph": "X"}, {"name": "HostToDevice408021893119", "ph": "s", "cat": "HostToDevice", "id": "408021893119", "pid": 43767502, "tid": 439084, "ts": "1704161511425800.125"}, {"name": "trans_Cast_17", "pid": 800, "tid": 3, "ts": "1704161511425803.044", "dur": 1.29, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 309, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 95}, "ph": "X"}, {"name": "HostToDevice412316860415", "ph": "f", "id": "412316860415", "ts": "1704161511425803.044", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425803.044", "dur": 1.29, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425804.334", "dur": 26.4595, "ph": "X"}, {"name": "HostToDevice412316860415", "ph": "s", "cat": "HostToDevice", "id": "412316860415", "pid": 43767502, "tid": 439084, "ts": "1704161511425816.335"}, {"name": "Conv2D3", "pid": 800, "tid": 3, "ts": "1704161511425830.794", "dur": 17.94, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 310, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 96}, "ph": "X"}, {"name": "HostToDevice416611827711", "ph": "f", "id": "416611827711", "ts": "1704161511425830.794", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425830.794", "dur": 17.94, "ph": "X"}, {"name": "HostToDevice416611827711", "ph": "s", "cat": "HostToDevice", "id": "416611827711", "pid": 43767502, "tid": 439084, "ts": "1704161511425841.256"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425848.734", "dur": 4.69, "ph": "X"}, {"name": "trans_TransData_18", "pid": 800, "tid": 3, "ts": "1704161511425853.424", "dur": 11.99, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 311, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 97}, "ph": "X"}, {"name": "HostToDevice420906795007", "ph": "f", "id": "420906795007", "ts": "1704161511425853.424", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425853.424", "dur": 11.99, "ph": "X"}, {"name": "HostToDevice420906795007", "ph": "s", "cat": "HostToDevice", "id": "420906795007", "pid": 43767502, "tid": 439084, "ts": "1704161511425865.476"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425865.414", "dur": 4.11, "ph": "X"}, {"name": "trans_Cast_19", "pid": 800, "tid": 3, "ts": "1704161511425869.524", "dur": 4.4295, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 312, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 98}, "ph": "X"}, {"name": "HostToDevice425201762303", "ph": "f", "id": "425201762303", "ts": "1704161511425869.524", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425869.524", "dur": 4.4295, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425873.954", "dur": 21.76, "ph": "X"}, {"name": "HostToDevice425201762303", "ph": "s", "cat": "HostToDevice", "id": "425201762303", "pid": 43767502, "tid": 439084, "ts": "1704161511425881.906"}, {"name": "aclnnRelu_Relu_Relu", "pid": 800, "tid": 3, "ts": "1704161511425895.714", "dur": 5.16, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 313, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 100}, "ph": "X"}, {"name": "HostToDevice433791696895", "ph": "f", "id": "433791696895", "ts": "1704161511425895.714", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425895.714", "dur": 5.16, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425900.874", "dur": 55.0895, "ph": "X"}, {"name": "AscendCL@aclnnRelu", "pid": 43767502, "tid": 439084, "ts": "1704161511425904.466", "dur": 10.340103297631945, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnRelu", "item_id": "0", "connection_id": 101}, "ph": "X"}, {"name": "HostToDevice433791696895", "ph": "s", "cat": "HostToDevice", "id": "433791696895", "pid": 43767502, "tid": 439084, "ts": "1704161511425905.746"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511425919.136", "dur": 134.2713413707003, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 113}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511425920.727", "dur": 11.390113787236734, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 102}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511425941.097", "dur": 5.780057742776851, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 103}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511425949.037", "dur": 1.6500164836646718, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 104}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511425951.186", "dur": 1.0700106894067871, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 105}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511425952.497", "dur": 0.9700096903968071, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 106}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511425954.087", "dur": 0.7300072927728548, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 107}, "ph": "X"}, {"name": "HostToDevice468151435263", "ph": "s", "cat": "HostToDevice", "id": "468151435263", "pid": 43767502, "tid": 439084, "ts": "1704161511425955.377"}, {"name": "trans_Cast_7", "pid": 800, "tid": 3, "ts": "1704161511425955.963", "dur": 4.32, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 314, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 108}, "ph": "X"}, {"name": "HostToDevice468151435263", "ph": "f", "id": "468151435263", "ts": "1704161511425955.963", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425955.963", "dur": 4.32, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425960.283", "dur": 15.0295, "ph": "X"}, {"name": "trans_TransData_8", "pid": 800, "tid": 3, "ts": "1704161511425975.312", "dur": 14.0, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 315, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 109}, "ph": "X"}, {"name": "HostToDevice472446402559", "ph": "f", "id": "472446402559", "ts": "1704161511425975.312", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425975.312", "dur": 14.0, "ph": "X"}, {"name": "HostToDevice472446402559", "ph": "s", "cat": "HostToDevice", "id": "472446402559", "pid": 43767502, "tid": 439084, "ts": "1704161511425981.047"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511425989.312", "dur": 7.74, "ph": "X"}, {"name": "MaxPoolWithArgmaxV12", "pid": 800, "tid": 3, "ts": "1704161511425997.052", "dur": 14.11, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 316, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 110}, "ph": "X"}, {"name": "HostToDevice476741369855", "ph": "f", "id": "476741369855", "ts": "1704161511425997.052", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511425997.052", "dur": 14.11, "ph": "X"}, {"name": "HostToDevice476741369855", "ph": "s", "cat": "HostToDevice", "id": "476741369855", "pid": 43767502, "tid": 439084, "ts": "1704161511426007.747"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426011.162", "dur": 8.52, "ph": "X"}, {"name": "trans_TransData_9", "pid": 800, "tid": 3, "ts": "1704161511426019.682", "dur": 13.7495, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 317, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 111}, "ph": "X"}, {"name": "HostToDevice481036337151", "ph": "f", "id": "481036337151", "ts": "1704161511426019.682", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426019.682", "dur": 13.7495, "ph": "X"}, {"name": "HostToDevice481036337151", "ph": "s", "cat": "HostToDevice", "id": "481036337151", "pid": 43767502, "tid": 439084, "ts": "1704161511426028.018"}, {"name": "trans_Cast_10", "pid": 800, "tid": 3, "ts": "1704161511426033.542", "dur": 3.97, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 318, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 112}, "ph": "X"}, {"name": "HostToDevice485331304447", "ph": "f", "id": "485331304447", "ts": "1704161511426033.542", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426033.542", "dur": 3.97, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426033.432", "dur": 0.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426037.512", "dur": 15.19, "ph": "X"}, {"name": "HostToDevice485331304447", "ph": "s", "cat": "HostToDevice", "id": "485331304447", "pid": 43767502, "tid": 439084, "ts": "1704161511426042.627"}, {"name": "aclnnAddmm_Muls_Muls", "pid": 800, "tid": 3, "ts": "1704161511426052.702", "dur": 1.58, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 319, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 115}, "ph": "X"}, {"name": "HostToDevice498216206335", "ph": "f", "id": "498216206335", "ts": "1704161511426052.702", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426052.702", "dur": 1.58, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426054.282", "dur": 8.36, "ph": "X"}, {"name": "aclnnAddmm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511426062.642", "dur": 4.5, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 320, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 119}, "ph": "X"}, {"name": "HostToDevice515396075519", "ph": "f", "id": "515396075519", "ts": "1704161511426062.642", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426062.642", "dur": 4.5, "ph": "X"}, {"name": "AscendCL@aclnnAddmm", "pid": 43767502, "tid": 439084, "ts": "1704161511426063.628", "dur": 54.260542062815205, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddmm", "item_id": "0", "connection_id": 127}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511426065.408", "dur": 5.490054845647908, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 114}, "ph": "X"}, {"name": "HostToDevice498216206335", "ph": "s", "cat": "HostToDevice", "id": "498216206335", "pid": 43767502, "tid": 439084, "ts": "1704161511426066.738"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426067.142", "dur": 6.4695, "ph": "X"}, {"name": "aclnnAddmm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511426073.612", "dur": 4.62, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 321, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 120}, "ph": "X"}, {"name": "HostToDevice519691042815", "ph": "f", "id": "519691042815", "ts": "1704161511426073.612", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426073.612", "dur": 4.62, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511426074.388", "dur": 1.3300132868327355, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 116}, "ph": "X"}, {"name": "HostToDevice515396075519", "ph": "s", "cat": "HostToDevice", "id": "515396075519", "pid": 43767502, "tid": 439084, "ts": "1704161511426076.248"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511426077.468", "dur": 0.8100080919808389, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 117}, "ph": "X"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426078.342", "dur": 5.24, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 322, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 121}, "ph": "X"}, {"name": "HostToDevice523986010111", "ph": "f", "id": "523986010111", "ts": "1704161511426078.342", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426078.342", "dur": 5.24, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426078.232", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511426078.698", "dur": 0.6700066933668668, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 118}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426083.582", "dur": 0.11, "ph": "X"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426083.692", "dur": 6.36, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 323, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 122}, "ph": "X"}, {"name": "HostToDevice528280977407", "ph": "f", "id": "528280977407", "ts": "1704161511426083.692", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426083.692", "dur": 6.36, "ph": "X"}, {"name": "HostToDevice519691042815", "ph": "s", "cat": "HostToDevice", "id": "519691042815", "pid": 43767502, "tid": 439084, "ts": "1704161511426083.888"}, {"name": "HostToDevice523986010111", "ph": "s", "cat": "HostToDevice", "id": "523986010111", "pid": 43767502, "tid": 439084, "ts": "1704161511426089.768"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426090.052", "dur": 0.12, "ph": "X"}, {"name": "aclnnAddmm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511426090.172", "dur": 7.02, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 324, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 123}, "ph": "X"}, {"name": "HostToDevice532575944703", "ph": "f", "id": "532575944703", "ts": "1704161511426090.172", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426090.172", "dur": 7.02, "ph": "X"}, {"name": "HostToDevice528280977407", "ph": "s", "cat": "HostToDevice", "id": "528280977407", "pid": 43767502, "tid": 439084, "ts": "1704161511426094.468"}, {"name": "aclnnAddmm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511426097.302", "dur": 8.41, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 325, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 124}, "ph": "X"}, {"name": "HostToDevice536870911999", "ph": "f", "id": "536870911999", "ts": "1704161511426097.302", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426097.302", "dur": 8.41, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426097.192", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice532575944703", "ph": "s", "cat": "HostToDevice", "id": "532575944703", "pid": 43767502, "tid": 439084, "ts": "1704161511426098.808"}, {"name": "HostToDevice536870911999", "ph": "s", "cat": "HostToDevice", "id": "536870911999", "pid": 43767502, "tid": 439084, "ts": "1704161511426102.858"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426105.822", "dur": 4.13, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 326, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 125}, "ph": "X"}, {"name": "HostToDevice541165879295", "ph": "f", "id": "541165879295", "ts": "1704161511426105.822", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426105.822", "dur": 4.13, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426105.712", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice541165879295", "ph": "s", "cat": "HostToDevice", "id": "541165879295", "pid": 43767502, "tid": 439084, "ts": "1704161511426108.118"}, {"name": "aclnnAddmm_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511426110.062", "dur": 7.78, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 327, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 126}, "ph": "X"}, {"name": "HostToDevice545460846591", "ph": "f", "id": "545460846591", "ts": "1704161511426110.062", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426110.062", "dur": 7.78, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426109.952", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice545460846591", "ph": "s", "cat": "HostToDevice", "id": "545460846591", "pid": 43767502, "tid": 439084, "ts": "1704161511426112.828"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426117.842", "dur": 0.12, "ph": "X"}, {"name": "aclnnRelu_Relu_Relu", "pid": 800, "tid": 3, "ts": "1704161511426117.962", "dur": 2.06, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 328, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 128}, "ph": "X"}, {"name": "HostToDevice554050781183", "ph": "f", "id": "554050781183", "ts": "1704161511426117.962", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426117.962", "dur": 2.06, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426120.022", "dur": 0.11, "ph": "X"}, {"name": "aclnnAddmm_Muls_Muls", "pid": 800, "tid": 3, "ts": "1704161511426120.132", "dur": 1.3795, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 329, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 130}, "ph": "X"}, {"name": "HostToDevice562640715775", "ph": "f", "id": "562640715775", "ts": "1704161511426120.132", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426120.132", "dur": 1.3795, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426121.511", "dur": 10.73, "ph": "X"}, {"name": "AscendCL@aclnnRelu", "pid": 43767502, "tid": 439084, "ts": "1704161511426122.849", "dur": 5.490054845647908, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnRelu", "item_id": "0", "connection_id": 129}, "ph": "X"}, {"name": "HostToDevice554050781183", "ph": "s", "cat": "HostToDevice", "id": "554050781183", "pid": 43767502, "tid": 439084, "ts": "1704161511426123.438"}, {"name": "AscendCL@aclnnAddmm", "pid": 43767502, "tid": 439084, "ts": "1704161511426131.478", "dur": 55.46055405093497, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddmm", "item_id": "0", "connection_id": 139}, "ph": "X"}, {"name": "aclnnAddmm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511426132.241", "dur": 2.04, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 330, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 131}, "ph": "X"}, {"name": "HostToDevice566935683071", "ph": "f", "id": "566935683071", "ts": "1704161511426132.241", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426132.241", "dur": 2.04, "ph": "X"}, {"name": "HostToDevice562640715775", "ph": "s", "cat": "HostToDevice", "id": "562640715775", "pid": 43767502, "tid": 439084, "ts": "1704161511426133.278"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426134.281", "dur": 4.39, "ph": "X"}, {"name": "HostToDevice566935683071", "ph": "s", "cat": "HostToDevice", "id": "566935683071", "pid": 43767502, "tid": 439084, "ts": "1704161511426138.549"}, {"name": "aclnnAddmm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511426138.671", "dur": 2.21, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 331, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 132}, "ph": "X"}, {"name": "HostToDevice571230650367", "ph": "f", "id": "571230650367", "ts": "1704161511426138.671", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426138.671", "dur": 2.21, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426140.881", "dur": 5.46, "ph": "X"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426146.341", "dur": 8.39, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 332, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 133}, "ph": "X"}, {"name": "HostToDevice575525617663", "ph": "f", "id": "575525617663", "ts": "1704161511426146.341", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426146.341", "dur": 8.39, "ph": "X"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426154.841", "dur": 7.62, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 333, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 134}, "ph": "X"}, {"name": "HostToDevice579820584959", "ph": "f", "id": "579820584959", "ts": "1704161511426154.841", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426154.841", "dur": 7.62, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426154.731", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice571230650367", "ph": "s", "cat": "HostToDevice", "id": "571230650367", "pid": 43767502, "tid": 439084, "ts": "1704161511426155.799"}, {"name": "HostToDevice575525617663", "ph": "s", "cat": "HostToDevice", "id": "575525617663", "pid": 43767502, "tid": 439084, "ts": "1704161511426159.929"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426162.461", "dur": 3.12, "ph": "X"}, {"name": "HostToDevice579820584959", "ph": "s", "cat": "HostToDevice", "id": "579820584959", "pid": 43767502, "tid": 439084, "ts": "1704161511426164.829"}, {"name": "aclnnAddmm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511426165.581", "dur": 6.3295, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 334, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 135}, "ph": "X"}, {"name": "HostToDevice584115552255", "ph": "f", "id": "584115552255", "ts": "1704161511426165.581", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426165.581", "dur": 6.3295, "ph": "X"}, {"name": "HostToDevice584115552255", "ph": "s", "cat": "HostToDevice", "id": "584115552255", "pid": 43767502, "tid": 439084, "ts": "1704161511426169.089"}, {"name": "aclnnAddmm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511426172.020", "dur": 2.74, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 335, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 136}, "ph": "X"}, {"name": "HostToDevice588410519551", "ph": "f", "id": "588410519551", "ts": "1704161511426172.020", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426172.020", "dur": 2.74, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426171.910", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice588410519551", "ph": "s", "cat": "HostToDevice", "id": "588410519551", "pid": 43767502, "tid": 439084, "ts": "1704161511426172.929"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426174.760", "dur": 6.76, "ph": "X"}, {"name": "HostToDevice592705486847", "ph": "s", "cat": "HostToDevice", "id": "592705486847", "pid": 43767502, "tid": 439084, "ts": "1704161511426176.969"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426181.520", "dur": 8.12, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 336, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 137}, "ph": "X"}, {"name": "HostToDevice592705486847", "ph": "f", "id": "592705486847", "ts": "1704161511426181.520", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426181.520", "dur": 8.12, "ph": "X"}, {"name": "HostToDevice597000454143", "ph": "s", "cat": "HostToDevice", "id": "597000454143", "pid": 43767502, "tid": 439084, "ts": "1704161511426181.789"}, {"name": "aclnnAddmm_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511426189.750", "dur": 8.96, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 337, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 138}, "ph": "X"}, {"name": "HostToDevice597000454143", "ph": "f", "id": "597000454143", "ts": "1704161511426189.750", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426189.750", "dur": 8.96, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426189.640", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclnnRelu", "pid": 43767502, "tid": 439084, "ts": "1704161511426191.109", "dur": 5.650056444063876, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnRelu", "item_id": "0", "connection_id": 141}, "ph": "X"}, {"name": "HostToDevice605590388735", "ph": "s", "cat": "HostToDevice", "id": "605590388735", "pid": 43767502, "tid": 439084, "ts": "1704161511426191.629"}, {"name": "aclnnRelu_Relu_Relu", "pid": 800, "tid": 3, "ts": "1704161511426198.830", "dur": 1.91, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 338, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 140}, "ph": "X"}, {"name": "HostToDevice605590388735", "ph": "f", "id": "605590388735", "ts": "1704161511426198.830", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426198.830", "dur": 1.91, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426198.710", "dur": 0.12, "ph": "X"}, {"name": "AscendCL@aclnnAddmm", "pid": 43767502, "tid": 439084, "ts": "1704161511426199.989", "dur": 48.33048282152339, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddmm", "item_id": "0", "connection_id": 151}, "ph": "X"}, {"name": "aclnnAddmm_Muls_Muls", "pid": 800, "tid": 3, "ts": "1704161511426200.860", "dur": 1.41, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 339, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 142}, "ph": "X"}, {"name": "HostToDevice614180323327", "ph": "f", "id": "614180323327", "ts": "1704161511426200.860", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426200.860", "dur": 1.41, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426200.740", "dur": 0.12, "ph": "X"}, {"name": "HostToDevice614180323327", "ph": "s", "cat": "HostToDevice", "id": "614180323327", "pid": 43767502, "tid": 439084, "ts": "1704161511426201.849"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426202.270", "dur": 0.11, "ph": "X"}, {"name": "aclnnAddmm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511426202.380", "dur": 1.9, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 340, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 143}, "ph": "X"}, {"name": "HostToDevice618475290623", "ph": "f", "id": "618475290623", "ts": "1704161511426202.380", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426202.380", "dur": 1.9, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426204.280", "dur": 19.2595, "ph": "X"}, {"name": "HostToDevice618475290623", "ph": "s", "cat": "HostToDevice", "id": "618475290623", "pid": 43767502, "tid": 439084, "ts": "1704161511426215.529"}, {"name": "HostToDevice622770257919", "ph": "s", "cat": "HostToDevice", "id": "622770257919", "pid": 43767502, "tid": 439084, "ts": "1704161511426220.350"}, {"name": "aclnnAddmm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511426223.540", "dur": 1.49, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 341, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 144}, "ph": "X"}, {"name": "HostToDevice622770257919", "ph": "f", "id": "622770257919", "ts": "1704161511426223.540", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426223.540", "dur": 1.49, "ph": "X"}, {"name": "HostToDevice627065225215", "ph": "s", "cat": "HostToDevice", "id": "627065225215", "pid": 43767502, "tid": 439084, "ts": "1704161511426225.069"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426225.030", "dur": 0.11, "ph": "X"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426225.140", "dur": 7.41, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 342, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 145}, "ph": "X"}, {"name": "HostToDevice627065225215", "ph": "f", "id": "627065225215", "ts": "1704161511426225.140", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426225.140", "dur": 7.41, "ph": "X"}, {"name": "HostToDevice631360192511", "ph": "s", "cat": "HostToDevice", "id": "631360192511", "pid": 43767502, "tid": 439084, "ts": "1704161511426229.029"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426232.550", "dur": 0.11, "ph": "X"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426232.660", "dur": 6.87, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 343, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 146}, "ph": "X"}, {"name": "HostToDevice631360192511", "ph": "f", "id": "631360192511", "ts": "1704161511426232.660", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426232.660", "dur": 6.87, "ph": "X"}, {"name": "HostToDevice635655159807", "ph": "s", "cat": "HostToDevice", "id": "635655159807", "pid": 43767502, "tid": 439084, "ts": "1704161511426233.030"}, {"name": "HostToDevice639950127103", "ph": "s", "cat": "HostToDevice", "id": "639950127103", "pid": 43767502, "tid": 439084, "ts": "1704161511426236.550"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426239.530", "dur": 0.12, "ph": "X"}, {"name": "aclnnAddmm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511426239.650", "dur": 6.23, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 344, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 147}, "ph": "X"}, {"name": "HostToDevice635655159807", "ph": "f", "id": "635655159807", "ts": "1704161511426239.650", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426239.650", "dur": 6.23, "ph": "X"}, {"name": "HostToDevice644245094399", "ph": "s", "cat": "HostToDevice", "id": "644245094399", "pid": 43767502, "tid": 439084, "ts": "1704161511426240.280"}, {"name": "HostToDevice648540061695", "ph": "s", "cat": "HostToDevice", "id": "648540061695", "pid": 43767502, "tid": 439084, "ts": "1704161511426244.110"}, {"name": "aclnnAddmm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511426245.990", "dur": 2.35, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 345, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 148}, "ph": "X"}, {"name": "HostToDevice639950127103", "ph": "f", "id": "639950127103", "ts": "1704161511426245.990", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426245.990", "dur": 2.35, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426245.880", "dur": 0.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426248.340", "dur": 0.11, "ph": "X"}, {"name": "aclnnAddmm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511426248.450", "dur": 6.04, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 346, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 149}, "ph": "X"}, {"name": "HostToDevice644245094399", "ph": "f", "id": "644245094399", "ts": "1704161511426248.450", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426248.450", "dur": 6.04, "ph": "X"}, {"name": "aclnnAddmm_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511426254.600", "dur": 9.14, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 347, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 150}, "ph": "X"}, {"name": "HostToDevice648540061695", "ph": "f", "id": "648540061695", "ts": "1704161511426254.600", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426254.600", "dur": 9.14, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426254.490", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclrtSynchronizeDevice", "pid": 43767502, "tid": 437675, "ts": "1704161511426263.130", "dur": 51.7505169876647, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtSynchronizeDevice", "item_id": "0", "connection_id": 152}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426263.740", "dur": 199.598, "ph": "X"}, {"name": "aclnnLogSoftmax_LogSoftmaxAiCore_LogSoftmaxV2", "pid": 800, "tid": 3, "ts": "1704161511426463.338", "dur": 18.6395, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 348, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 153}, "ph": "X"}, {"name": "HostToDevice661424963583", "ph": "f", "id": "661424963583", "ts": "1704161511426463.338", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426463.338", "dur": 18.6395, "ph": "X"}, {"name": "AscendCL@aclnnLogSoftmax", "pid": 43767502, "tid": 439084, "ts": "1704161511426468.822", "dur": 9.71009700386907, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnLogSoftmax", "item_id": "0", "connection_id": 154}, "ph": "X"}, {"name": "HostToDevice661424963583", "ph": "s", "cat": "HostToDevice", "id": "661424963583", "pid": 43767502, "tid": 439084, "ts": "1704161511426469.572"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426481.978", "dur": 55.7195, "ph": "X"}, {"name": "aclnnInplaceOne_OnesLikeAiCore_OnesLike", "pid": 800, "tid": 3, "ts": "1704161511426537.697", "dur": 1.49, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 349, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 155}, "ph": "X"}, {"name": "HostToDevice670014898175", "ph": "f", "id": "670014898175", "ts": "1704161511426537.697", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426537.697", "dur": 1.49, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426539.187", "dur": 8.48, "ph": "X"}, {"name": "AscendCL@aclnnInplaceOne", "pid": 43767502, "tid": 439084, "ts": "1704161511426545.723", "dur": 7.4000739267385285, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceOne", "item_id": "0", "connection_id": 156}, "ph": "X"}, {"name": "HostToDevice670014898175", "ph": "s", "cat": "HostToDevice", "id": "670014898175", "pid": 43767502, "tid": 439084, "ts": "1704161511426546.733"}, {"name": "aclnnNLLLoss_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511426547.667", "dur": 2.89, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 350, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 157}, "ph": "X"}, {"name": "HostToDevice678604832767", "ph": "f", "id": "678604832767", "ts": "1704161511426547.667", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426547.667", "dur": 2.89, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426550.557", "dur": 0.19, "ph": "X"}, {"name": "aclnnNLLLoss_NLLLossAiCore_MemSet", "pid": 800, "tid": 3, "ts": "1704161511426550.747", "dur": 8.82, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 351, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 158}, "ph": "X"}, {"name": "HostToDevice682899800063", "ph": "f", "id": "682899800063", "ts": "1704161511426550.747", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426550.747", "dur": 8.82, "ph": "X"}, {"name": "AscendCL@aclnnNLLLoss", "pid": 43767502, "tid": 439084, "ts": "1704161511426558.643", "dur": 20.77020749437287, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnNLLLoss", "item_id": "0", "connection_id": 161}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426559.567", "dur": 4.92, "ph": "X"}, {"name": "HostToDevice678604832767", "ph": "s", "cat": "HostToDevice", "id": "678604832767", "pid": 43767502, "tid": 439084, "ts": "1704161511426560.393"}, {"name": "aclnnNLLLoss_NLLLossAiCore_NLLLoss", "pid": 800, "tid": 3, "ts": "1704161511426564.487", "dur": 8.5595, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 352, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 159}, "ph": "X"}, {"name": "HostToDevice687194767359", "ph": "f", "id": "687194767359", "ts": "1704161511426564.487", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426564.487", "dur": 8.5595, "ph": "X"}, {"name": "HostToDevice682899800063", "ph": "s", "cat": "HostToDevice", "id": "682899800063", "pid": 43767502, "tid": 439084, "ts": "1704161511426565.793"}, {"name": "HostToDevice687194767359", "ph": "s", "cat": "HostToDevice", "id": "687194767359", "pid": 43767502, "tid": 439084, "ts": "1704161511426569.433"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426573.046", "dur": 0.19, "ph": "X"}, {"name": "aclnnNLLLoss_DivAiCore_Div", "pid": 800, "tid": 3, "ts": "1704161511426573.236", "dur": 1.37, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 353, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 160}, "ph": "X"}, {"name": "HostToDevice691489734655", "ph": "f", "id": "691489734655", "ts": "1704161511426573.236", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426573.236", "dur": 1.37, "ph": "X"}, {"name": "HostToDevice691489734655", "ph": "s", "cat": "HostToDevice", "id": "691489734655", "pid": 43767502, "tid": 439084, "ts": "1704161511426574.193"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426574.606", "dur": 44.41, "ph": "X"}, {"name": "aclnnInplaceOne_OnesLikeAiCore_OnesLike", "pid": 800, "tid": 3, "ts": "1704161511426619.016", "dur": 1.21, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 354, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 162}, "ph": "X"}, {"name": "HostToDevice700079669247", "ph": "f", "id": "700079669247", "ts": "1704161511426619.016", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426619.016", "dur": 1.21, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426620.226", "dur": 204.3275, "ph": "X"}, {"name": "AscendCL@aclnnInplaceOne", "pid": 43767502, "tid": 439084, "ts": "1704161511426629.164", "dur": 7.470074626045514, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceOne", "item_id": "0", "connection_id": 163}, "ph": "X"}, {"name": "HostToDevice700079669247", "ph": "s", "cat": "HostToDevice", "id": "700079669247", "pid": 43767502, "tid": 439084, "ts": "1704161511426629.863"}, {"name": "aclnnInplaceOne_OnesLikeAiCore_OnesLike", "pid": 800, "tid": 3, "ts": "1704161511426824.554", "dur": 1.21, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 355, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 164}, "ph": "X"}, {"name": "HostToDevice708669603839", "ph": "f", "id": "708669603839", "ts": "1704161511426824.554", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426824.554", "dur": 1.21, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426825.764", "dur": 35.64, "ph": "X"}, {"name": "AscendCL@aclnnInplaceOne", "pid": 43767502, "tid": 439084, "ts": "1704161511426833.366", "dur": 6.780067732876651, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceOne", "item_id": "0", "connection_id": 165}, "ph": "X"}, {"name": "HostToDevice708669603839", "ph": "s", "cat": "HostToDevice", "id": "708669603839", "pid": 43767502, "tid": 439084, "ts": "1704161511426833.936"}, {"name": "AscendCL@aclnnNLLLossBackward", "pid": 43767502, "tid": 439084, "ts": "1704161511426859.756", "dur": 12.0601204806036, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnNLLLossBackward", "item_id": "0", "connection_id": 168}, "ph": "X"}, {"name": "HostToDevice717259538431", "ph": "s", "cat": "HostToDevice", "id": "717259538431", "pid": 43767502, "tid": 439084, "ts": "1704161511426861.036"}, {"name": "aclnnNLLLossBackward_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511426861.404", "dur": 2.69, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 356, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 166}, "ph": "X"}, {"name": "HostToDevice717259538431", "ph": "f", "id": "717259538431", "ts": "1704161511426861.404", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426861.404", "dur": 2.69, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426864.094", "dur": 5.63, "ph": "X"}, {"name": "HostToDevice721554505727", "ph": "s", "cat": "HostToDevice", "id": "721554505727", "pid": 43767502, "tid": 439084, "ts": "1704161511426866.186"}, {"name": "aclnnNLLLossBackward_NLLLossGradAiCore_NLLLossGrad", "pid": 800, "tid": 3, "ts": "1704161511426869.724", "dur": 8.1095, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 357, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 167}, "ph": "X"}, {"name": "HostToDevice721554505727", "ph": "f", "id": "721554505727", "ts": "1704161511426869.724", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426869.724", "dur": 8.1095, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426877.834", "dur": 58.4095, "ph": "X"}, {"name": "aclnnLogSoftmaxBackward_LogSoftmaxGrad_LogSoftmaxGrad", "pid": 800, "tid": 3, "ts": "1704161511426936.243", "dur": 12.21, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 358, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 169}, "ph": "X"}, {"name": "HostToDevice730144440319", "ph": "f", "id": "730144440319", "ts": "1704161511426936.243", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511426936.243", "dur": 12.21, "ph": "X"}, {"name": "AscendCL@aclnnLogSoftmaxBackward", "pid": 43767502, "tid": 439084, "ts": "1704161511426938.956", "dur": 7.39007382683753, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnLogSoftmaxBackward", "item_id": "0", "connection_id": 170}, "ph": "X"}, {"name": "HostToDevice730144440319", "ph": "s", "cat": "HostToDevice", "id": "730144440319", "pid": 43767502, "tid": 439084, "ts": "1704161511426939.677"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511426948.453", "dur": 56.3695, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427004.822", "dur": 1.57, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 359, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 171}, "ph": "X"}, {"name": "HostToDevice738734374911", "ph": "f", "id": "738734374911", "ts": "1704161511427004.822", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427004.822", "dur": 1.57, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427006.582", "dur": 1.49, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 360, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 172}, "ph": "X"}, {"name": "HostToDevice743029342207", "ph": "f", "id": "743029342207", "ts": "1704161511427006.582", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427006.582", "dur": 1.49, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427006.392", "dur": 0.19, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427008.072", "dur": 9.49, "ph": "X"}, {"name": "AscendCL@aclnnMm", "pid": 43767502, "tid": 439084, "ts": "1704161511427010.708", "dur": 33.6803364665613, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnMm", "item_id": "0", "connection_id": 178}, "ph": "X"}, {"name": "HostToDevice738734374911", "ph": "s", "cat": "HostToDevice", "id": "738734374911", "pid": 43767502, "tid": 439084, "ts": "1704161511427012.237"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427017.562", "dur": 5.6195, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 361, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 173}, "ph": "X"}, {"name": "HostToDevice747324309503", "ph": "f", "id": "747324309503", "ts": "1704161511427017.562", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427017.562", "dur": 5.6195, "ph": "X"}, {"name": "HostToDevice743029342207", "ph": "s", "cat": "HostToDevice", "id": "743029342207", "pid": 43767502, "tid": 439084, "ts": "1704161511427017.938"}, {"name": "HostToDevice747324309503", "ph": "s", "cat": "HostToDevice", "id": "747324309503", "pid": 43767502, "tid": 439084, "ts": "1704161511427022.737"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427023.362", "dur": 7.66, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 362, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 174}, "ph": "X"}, {"name": "HostToDevice751619276799", "ph": "f", "id": "751619276799", "ts": "1704161511427023.362", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427023.362", "dur": 7.66, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427023.182", "dur": 0.18, "ph": "X"}, {"name": "HostToDevice751619276799", "ph": "s", "cat": "HostToDevice", "id": "751619276799", "pid": 43767502, "tid": 439084, "ts": "1704161511427027.658"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427031.022", "dur": 0.18, "ph": "X"}, {"name": "aclnnMm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427031.202", "dur": 6.19, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 363, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 175}, "ph": "X"}, {"name": "HostToDevice755914244095", "ph": "f", "id": "755914244095", "ts": "1704161511427031.202", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427031.202", "dur": 6.19, "ph": "X"}, {"name": "HostToDevice755914244095", "ph": "s", "cat": "HostToDevice", "id": "755914244095", "pid": 43767502, "tid": 439084, "ts": "1704161511427031.737"}, {"name": "HostToDevice760209211391", "ph": "s", "cat": "HostToDevice", "id": "760209211391", "pid": 43767502, "tid": 439084, "ts": "1704161511427035.107"}, {"name": "aclnnMm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511427037.582", "dur": 2.7, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 364, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 176}, "ph": "X"}, {"name": "HostToDevice760209211391", "ph": "f", "id": "760209211391", "ts": "1704161511427037.582", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427037.582", "dur": 2.7, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427037.392", "dur": 0.19, "ph": "X"}, {"name": "HostToDevice764504178687", "ph": "s", "cat": "HostToDevice", "id": "764504178687", "pid": 43767502, "tid": 439084, "ts": "1704161511427040.078"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427040.282", "dur": 0.18, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427040.462", "dur": 7.94, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 365, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 177}, "ph": "X"}, {"name": "HostToDevice764504178687", "ph": "f", "id": "764504178687", "ts": "1704161511427040.462", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427040.462", "dur": 7.94, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427048.582", "dur": 1.42, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 366, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 179}, "ph": "X"}, {"name": "HostToDevice773094113279", "ph": "f", "id": "773094113279", "ts": "1704161511427048.582", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427048.582", "dur": 1.42, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427048.402", "dur": 0.18, "ph": "X"}, {"name": "AscendCL@aclnnMm", "pid": 43767502, "tid": 439084, "ts": "1704161511427049.118", "dur": 33.79033756547228, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnMm", "item_id": "0", "connection_id": 186}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427050.002", "dur": 0.19, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427050.192", "dur": 2.0, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 367, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 180}, "ph": "X"}, {"name": "HostToDevice777389080575", "ph": "f", "id": "777389080575", "ts": "1704161511427050.192", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "HostToDevice773094113279", "ph": "s", "cat": "HostToDevice", "id": "773094113279", "pid": 43767502, "tid": 439084, "ts": "1704161511427050.148"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427050.192", "dur": 2.0, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427052.192", "dur": 3.27, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427055.462", "dur": 5.25, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 368, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 181}, "ph": "X"}, {"name": "HostToDevice781684047871", "ph": "f", "id": "781684047871", "ts": "1704161511427055.462", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427055.462", "dur": 5.25, "ph": "X"}, {"name": "HostToDevice777389080575", "ph": "s", "cat": "HostToDevice", "id": "777389080575", "pid": 43767502, "tid": 439084, "ts": "1704161511427055.828"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427060.822", "dur": 7.84, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 369, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 182}, "ph": "X"}, {"name": "HostToDevice785979015167", "ph": "f", "id": "785979015167", "ts": "1704161511427060.822", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427060.822", "dur": 7.84, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427060.712", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice781684047871", "ph": "s", "cat": "HostToDevice", "id": "781684047871", "pid": 43767502, "tid": 439084, "ts": "1704161511427061.098"}, {"name": "HostToDevice785979015167", "ph": "s", "cat": "HostToDevice", "id": "785979015167", "pid": 43767502, "tid": 439084, "ts": "1704161511427066.298"}, {"name": "aclnnMm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427068.772", "dur": 6.6195, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 370, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 183}, "ph": "X"}, {"name": "HostToDevice790273982463", "ph": "f", "id": "790273982463", "ts": "1704161511427068.772", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427068.772", "dur": 6.6195, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427068.662", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice790273982463", "ph": "s", "cat": "HostToDevice", "id": "790273982463", "pid": 43767502, "tid": 439084, "ts": "1704161511427070.558"}, {"name": "HostToDevice794568949759", "ph": "s", "cat": "HostToDevice", "id": "794568949759", "pid": 43767502, "tid": 439084, "ts": "1704161511427073.708"}, {"name": "aclnnMm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511427075.512", "dur": 2.08, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 371, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 184}, "ph": "X"}, {"name": "HostToDevice794568949759", "ph": "f", "id": "794568949759", "ts": "1704161511427075.512", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427075.512", "dur": 2.08, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427075.392", "dur": 0.12, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427077.592", "dur": 0.12, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427077.712", "dur": 7.61, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 372, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 185}, "ph": "X"}, {"name": "HostToDevice798863917055", "ph": "f", "id": "798863917055", "ts": "1704161511427077.712", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427077.712", "dur": 7.61, "ph": "X"}, {"name": "HostToDevice798863917055", "ph": "s", "cat": "HostToDevice", "id": "798863917055", "pid": 43767502, "tid": 439084, "ts": "1704161511427077.988"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427085.322", "dur": 0.11, "ph": "X"}, {"name": "aclnnReduceSum_ReduceSumOpAiCore_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427085.432", "dur": 6.81, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 373, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 187}, "ph": "X"}, {"name": "HostToDevice807453851647", "ph": "f", "id": "807453851647", "ts": "1704161511427085.432", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427085.432", "dur": 6.81, "ph": "X"}, {"name": "AscendCL@aclnnReduceSum", "pid": 43767502, "tid": 439084, "ts": "1704161511427088.988", "dur": 10.540105295651903, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnReduceSum", "item_id": "0", "connection_id": 189}, "ph": "X"}, {"name": "HostToDevice807453851647", "ph": "s", "cat": "HostToDevice", "id": "807453851647", "pid": 43767502, "tid": 439084, "ts": "1704161511427089.908"}, {"name": "aclnnReduceSum_ReduceSumOpAiCore_ReduceSum", "pid": 800, "tid": 3, "ts": "1704161511427092.352", "dur": 11.05, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 374, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 188}, "ph": "X"}, {"name": "HostToDevice811748818943", "ph": "f", "id": "811748818943", "ts": "1704161511427092.352", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427092.352", "dur": 11.05, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427092.242", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice811748818943", "ph": "s", "cat": "HostToDevice", "id": "811748818943", "pid": 43767502, "tid": 439084, "ts": "1704161511427093.908"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427103.402", "dur": 200.038, "ph": "X"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511427192.119", "dur": 61.02060959588986, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 190}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511427273.780", "dur": 66.04065974619087, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 193}, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511427288.240", "dur": 16.470164536943724, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 191}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511427303.440", "dur": 1.47, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 375, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 192}, "ph": "X"}, {"name": "HostToDevice828928688127", "ph": "f", "id": "828928688127", "ts": "1704161511427303.440", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427303.440", "dur": 1.47, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427304.910", "dur": 66.5995, "ph": "X"}, {"name": "HostToDevice828928688127", "ph": "s", "cat": "HostToDevice", "id": "828928688127", "pid": 43767502, "tid": 439084, "ts": "1704161511427309.360"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511427327.340", "dur": 19.640196205560095, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 194}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511427364.681", "dur": 32.330322979926564, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 197}, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511427370.261", "dur": 7.470074626045514, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 195}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511427371.509", "dur": 1.5295, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 376, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 196}, "ph": "X"}, {"name": "HostToDevice846108557311", "ph": "f", "id": "846108557311", "ts": "1704161511427371.509", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427371.509", "dur": 1.5295, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427373.038", "dur": 58.6495, "ph": "X"}, {"name": "HostToDevice846108557311", "ph": "s", "cat": "HostToDevice", "id": "846108557311", "pid": 43767502, "tid": 439084, "ts": "1704161511427379.931"}, {"name": "aclnnThresholdBackward_ReluGrad_ReluGrad", "pid": 800, "tid": 3, "ts": "1704161511427431.688", "dur": 2.3, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 377, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 198}, "ph": "X"}, {"name": "HostToDevice854698491903", "ph": "f", "id": "854698491903", "ts": "1704161511427431.688", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427431.688", "dur": 2.3, "ph": "X"}, {"name": "AscendCL@aclnnThresholdBackward", "pid": 43767502, "tid": 439084, "ts": "1704161511427432.622", "dur": 9.020090110700206, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnThresholdBackward", "item_id": "0", "connection_id": 199}, "ph": "X"}, {"name": "HostToDevice854698491903", "ph": "s", "cat": "HostToDevice", "id": "854698491903", "pid": 43767502, "tid": 439084, "ts": "1704161511427433.822"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427433.988", "dur": 45.7195, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427479.708", "dur": 1.96, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 378, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 200}, "ph": "X"}, {"name": "HostToDevice863288426495", "ph": "f", "id": "863288426495", "ts": "1704161511427479.708", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427479.708", "dur": 1.96, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427481.828", "dur": 2.25, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 379, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 201}, "ph": "X"}, {"name": "HostToDevice867583393791", "ph": "f", "id": "867583393791", "ts": "1704161511427481.828", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427481.828", "dur": 2.25, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427481.668", "dur": 0.16, "ph": "X"}, {"name": "AscendCL@aclnnMm", "pid": 43767502, "tid": 439084, "ts": "1704161511427483.372", "dur": 32.220321881015586, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnMm", "item_id": "0", "connection_id": 207}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427484.078", "dur": 8.13, "ph": "X"}, {"name": "HostToDevice863288426495", "ph": "s", "cat": "HostToDevice", "id": "863288426495", "pid": 43767502, "tid": 439084, "ts": "1704161511427485.252"}, {"name": "HostToDevice867583393791", "ph": "s", "cat": "HostToDevice", "id": "867583393791", "pid": 43767502, "tid": 439084, "ts": "1704161511427491.112"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427492.208", "dur": 7.37, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 380, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 202}, "ph": "X"}, {"name": "HostToDevice871878361087", "ph": "f", "id": "871878361087", "ts": "1704161511427492.208", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427492.208", "dur": 7.37, "ph": "X"}, {"name": "HostToDevice871878361087", "ph": "s", "cat": "HostToDevice", "id": "871878361087", "pid": 43767502, "tid": 439084, "ts": "1704161511427495.072"}, {"name": "HostToDevice876173328383", "ph": "s", "cat": "HostToDevice", "id": "876173328383", "pid": 43767502, "tid": 439084, "ts": "1704161511427499.422"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427499.578", "dur": 0.15, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427499.728", "dur": 7.9, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 381, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 203}, "ph": "X"}, {"name": "HostToDevice876173328383", "ph": "f", "id": "876173328383", "ts": "1704161511427499.728", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427499.728", "dur": 7.9, "ph": "X"}, {"name": "HostToDevice880468295679", "ph": "s", "cat": "HostToDevice", "id": "880468295679", "pid": 43767502, "tid": 439084, "ts": "1704161511427503.082"}, {"name": "HostToDevice884763262975", "ph": "s", "cat": "HostToDevice", "id": "884763262975", "pid": 43767502, "tid": 439084, "ts": "1704161511427506.962"}, {"name": "aclnnMm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427507.778", "dur": 6.22, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 382, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 204}, "ph": "X"}, {"name": "HostToDevice880468295679", "ph": "f", "id": "880468295679", "ts": "1704161511427507.778", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427507.778", "dur": 6.22, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427507.628", "dur": 0.15, "ph": "X"}, {"name": "HostToDevice889058230271", "ph": "s", "cat": "HostToDevice", "id": "889058230271", "pid": 43767502, "tid": 439084, "ts": "1704161511427511.052"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427513.998", "dur": 0.15, "ph": "X"}, {"name": "aclnnMm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511427514.148", "dur": 2.82, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 383, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 205}, "ph": "X"}, {"name": "HostToDevice884763262975", "ph": "f", "id": "884763262975", "ts": "1704161511427514.148", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427514.148", "dur": 2.82, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427517.078", "dur": 4.15, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 384, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 206}, "ph": "X"}, {"name": "HostToDevice889058230271", "ph": "f", "id": "889058230271", "ts": "1704161511427517.078", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427517.078", "dur": 4.15, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427516.968", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclnnMm", "pid": 43767502, "tid": 439084, "ts": "1704161511427519.732", "dur": 30.78030749527188, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnMm", "item_id": "0", "connection_id": 215}, "ph": "X"}, {"name": "HostToDevice897648164863", "ph": "s", "cat": "HostToDevice", "id": "897648164863", "pid": 43767502, "tid": 439084, "ts": "1704161511427520.692"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427521.338", "dur": 1.8195, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 385, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 208}, "ph": "X"}, {"name": "HostToDevice897648164863", "ph": "f", "id": "897648164863", "ts": "1704161511427521.338", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427521.338", "dur": 1.8195, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427521.228", "dur": 0.11, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427523.267", "dur": 2.08, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 386, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 209}, "ph": "X"}, {"name": "HostToDevice901943132159", "ph": "f", "id": "901943132159", "ts": "1704161511427523.267", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427523.267", "dur": 2.08, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427523.157", "dur": 0.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427525.347", "dur": 0.11, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427525.457", "dur": 7.41, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 387, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 210}, "ph": "X"}, {"name": "HostToDevice906238099455", "ph": "f", "id": "906238099455", "ts": "1704161511427525.457", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427525.457", "dur": 7.41, "ph": "X"}, {"name": "HostToDevice901943132159", "ph": "s", "cat": "HostToDevice", "id": "901943132159", "pid": 43767502, "tid": 439084, "ts": "1704161511427526.272"}, {"name": "HostToDevice906238099455", "ph": "s", "cat": "HostToDevice", "id": "906238099455", "pid": 43767502, "tid": 439084, "ts": "1704161511427530.252"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427532.867", "dur": 0.11, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427532.977", "dur": 7.25, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 388, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 211}, "ph": "X"}, {"name": "HostToDevice910533066751", "ph": "f", "id": "910533066751", "ts": "1704161511427532.977", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427532.977", "dur": 7.25, "ph": "X"}, {"name": "HostToDevice910533066751", "ph": "s", "cat": "HostToDevice", "id": "910533066751", "pid": 43767502, "tid": 439084, "ts": "1704161511427534.482"}, {"name": "HostToDevice914828034047", "ph": "s", "cat": "HostToDevice", "id": "914828034047", "pid": 43767502, "tid": 439084, "ts": "1704161511427538.232"}, {"name": "aclnnMm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427540.347", "dur": 6.53, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 389, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 212}, "ph": "X"}, {"name": "HostToDevice914828034047", "ph": "f", "id": "914828034047", "ts": "1704161511427540.347", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427540.347", "dur": 6.53, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427540.227", "dur": 0.12, "ph": "X"}, {"name": "HostToDevice919123001343", "ph": "s", "cat": "HostToDevice", "id": "919123001343", "pid": 43767502, "tid": 439084, "ts": "1704161511427541.812"}, {"name": "HostToDevice923417968639", "ph": "s", "cat": "HostToDevice", "id": "923417968639", "pid": 43767502, "tid": 439084, "ts": "1704161511427546.513"}, {"name": "aclnnMm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511427546.987", "dur": 2.53, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 390, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 213}, "ph": "X"}, {"name": "HostToDevice919123001343", "ph": "f", "id": "919123001343", "ts": "1704161511427546.987", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427546.987", "dur": 2.53, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427546.877", "dur": 0.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427549.517", "dur": 0.11, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427549.627", "dur": 4.13, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 391, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 214}, "ph": "X"}, {"name": "HostToDevice923417968639", "ph": "f", "id": "923417968639", "ts": "1704161511427549.627", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427549.627", "dur": 4.13, "ph": "X"}, {"name": "aclnnReduceSum_ReduceSumOpAiCore_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427553.867", "dur": 6.81, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 392, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 216}, "ph": "X"}, {"name": "HostToDevice932007903231", "ph": "f", "id": "932007903231", "ts": "1704161511427553.867", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427553.867", "dur": 6.81, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427553.757", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclnnReduceSum", "pid": 43767502, "tid": 439084, "ts": "1704161511427554.443", "dur": 8.5500854153533, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnReduceSum", "item_id": "0", "connection_id": 218}, "ph": "X"}, {"name": "HostToDevice932007903231", "ph": "s", "cat": "HostToDevice", "id": "932007903231", "pid": 43767502, "tid": 439084, "ts": "1704161511427554.953"}, {"name": "HostToDevice936302870527", "ph": "s", "cat": "HostToDevice", "id": "936302870527", "pid": 43767502, "tid": 439084, "ts": "1704161511427558.863"}, {"name": "aclnnReduceSum_ReduceSumOpAiCore_ReduceSum", "pid": 800, "tid": 3, "ts": "1704161511427560.787", "dur": 11.8195, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 393, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 217}, "ph": "X"}, {"name": "HostToDevice936302870527", "ph": "f", "id": "936302870527", "ts": "1704161511427560.787", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427560.787", "dur": 11.8195, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427560.677", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511427568.703", "dur": 19.980199602194023, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 219}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427572.606", "dur": 40.25, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511427604.683", "dur": 30.740307095667887, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 222}, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511427610.213", "dur": 6.900068931688628, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 220}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511427612.856", "dur": 1.39, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 394, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 221}, "ph": "X"}, {"name": "HostToDevice953482739711", "ph": "f", "id": "953482739711", "ts": "1704161511427612.856", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427612.856", "dur": 1.39, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427614.246", "dur": 69.229, "ph": "X"}, {"name": "HostToDevice953482739711", "ph": "s", "cat": "HostToDevice", "id": "953482739711", "pid": 43767502, "tid": 439084, "ts": "1704161511427619.394"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511427646.074", "dur": 17.870178523083446, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 223}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511427681.664", "dur": 26.54026513724872, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 226}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511427683.476", "dur": 2.57, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 395, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 225}, "ph": "X"}, {"name": "HostToDevice970662608895", "ph": "f", "id": "970662608895", "ts": "1704161511427683.476", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427683.476", "dur": 2.57, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427686.046", "dur": 19.13, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511427686.134", "dur": 5.400053946538926, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 224}, "ph": "X"}, {"name": "HostToDevice970662608895", "ph": "s", "cat": "HostToDevice", "id": "970662608895", "pid": 43767502, "tid": 439084, "ts": "1704161511427693.494"}, {"name": "aclnnThresholdBackward_ReluGrad_ReluGrad", "pid": 800, "tid": 3, "ts": "1704161511427705.176", "dur": 2.72, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 396, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 227}, "ph": "X"}, {"name": "HostToDevice979252543487", "ph": "f", "id": "979252543487", "ts": "1704161511427705.176", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427705.176", "dur": 2.72, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427707.896", "dur": 50.1595, "ph": "X"}, {"name": "AscendCL@aclnnThresholdBackward", "pid": 43767502, "tid": 439084, "ts": "1704161511427713.924", "dur": 6.050060440103796, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnThresholdBackward", "item_id": "0", "connection_id": 228}, "ph": "X"}, {"name": "HostToDevice979252543487", "ph": "s", "cat": "HostToDevice", "id": "979252543487", "pid": 43767502, "tid": 439084, "ts": "1704161511427714.424"}, {"name": "AscendCL@aclnnMm", "pid": 43767502, "tid": 439084, "ts": "1704161511427757.805", "dur": 31.850318184678667, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnMm", "item_id": "0", "connection_id": 236}, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427758.055", "dur": 2.29, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 397, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 229}, "ph": "X"}, {"name": "HostToDevice987842478079", "ph": "f", "id": "987842478079", "ts": "1704161511427758.055", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427758.055", "dur": 2.29, "ph": "X"}, {"name": "HostToDevice987842478079", "ph": "s", "cat": "HostToDevice", "id": "987842478079", "pid": 43767502, "tid": 439084, "ts": "1704161511427759.035"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427760.345", "dur": 0.16, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427760.505", "dur": 4.89, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 398, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 230}, "ph": "X"}, {"name": "HostToDevice992137445375", "ph": "f", "id": "992137445375", "ts": "1704161511427760.505", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427760.505", "dur": 4.89, "ph": "X"}, {"name": "HostToDevice992137445375", "ph": "s", "cat": "HostToDevice", "id": "992137445375", "pid": 43767502, "tid": 439084, "ts": "1704161511427764.535"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427765.395", "dur": 9.6895, "ph": "X"}, {"name": "HostToDevice996432412671", "ph": "s", "cat": "HostToDevice", "id": "996432412671", "pid": 43767502, "tid": 439084, "ts": "1704161511427768.645"}, {"name": "HostToDevice1000727379967", "ph": "s", "cat": "HostToDevice", "id": "1000727379967", "pid": 43767502, "tid": 439084, "ts": "1704161511427772.625"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427775.084", "dur": 7.39, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 399, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 231}, "ph": "X"}, {"name": "HostToDevice996432412671", "ph": "f", "id": "996432412671", "ts": "1704161511427775.084", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427775.084", "dur": 7.39, "ph": "X"}, {"name": "AscendCL@aclMallocMemInner", "pid": 43767502, "tid": 439228, "ts": "1704161511427775.935", "dur": 72.83072757896852, "args": {"Thread Id": 439228, "Mode": "ACL_RTS", "level": "acl", "id": "aclMallocMemInner", "item_id": "0", "connection_id": 237}, "ph": "X"}, {"name": "HostToDevice1005022347263", "ph": "s", "cat": "HostToDevice", "id": "1005022347263", "pid": 43767502, "tid": 439084, "ts": "1704161511427777.025"}, {"name": "HostToDevice1009317314559", "ph": "s", "cat": "HostToDevice", "id": "1009317314559", "pid": 43767502, "tid": 439084, "ts": "1704161511427780.625"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427782.584", "dur": 6.33, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 400, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 232}, "ph": "X"}, {"name": "HostToDevice1000727379967", "ph": "f", "id": "1000727379967", "ts": "1704161511427782.584", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427782.584", "dur": 6.33, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427782.474", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice1013612281855", "ph": "s", "cat": "HostToDevice", "id": "1013612281855", "pid": 43767502, "tid": 439084, "ts": "1704161511427784.865"}, {"name": "aclnnMm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427789.024", "dur": 6.32, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 401, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 233}, "ph": "X"}, {"name": "HostToDevice1005022347263", "ph": "f", "id": "1005022347263", "ts": "1704161511427789.024", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427789.024", "dur": 6.32, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427788.914", "dur": 0.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427795.344", "dur": 0.11, "ph": "X"}, {"name": "aclnnMm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511427795.454", "dur": 5.55, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 402, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 234}, "ph": "X"}, {"name": "HostToDevice1009317314559", "ph": "f", "id": "1009317314559", "ts": "1704161511427795.454", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427795.454", "dur": 5.55, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427801.114", "dur": 6.09, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 403, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 235}, "ph": "X"}, {"name": "HostToDevice1013612281855", "ph": "f", "id": "1013612281855", "ts": "1704161511427801.114", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427801.114", "dur": 6.09, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427801.004", "dur": 0.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427807.204", "dur": 61.3095, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427868.514", "dur": 2.23, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 404, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 238}, "ph": "X"}, {"name": "HostToDevice1026497183743", "ph": "f", "id": "1026497183743", "ts": "1704161511427868.514", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427868.514", "dur": 2.23, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427870.744", "dur": 0.16, "ph": "X"}, {"name": "aclnnMm_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511427870.904", "dur": 4.7095, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 405, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 239}, "ph": "X"}, {"name": "HostToDevice1030792151039", "ph": "f", "id": "1030792151039", "ts": "1704161511427870.904", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427870.904", "dur": 4.7095, "ph": "X"}, {"name": "AscendCL@aclnnMm", "pid": 43767502, "tid": 439084, "ts": "1704161511427873.146", "dur": 30.810307794974868, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnMm", "item_id": "0", "connection_id": 245}, "ph": "X"}, {"name": "HostToDevice1026497183743", "ph": "s", "cat": "HostToDevice", "id": "1026497183743", "pid": 43767502, "tid": 439084, "ts": "1704161511427874.756"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427875.614", "dur": 8.65, "ph": "X"}, {"name": "HostToDevice1030792151039", "ph": "s", "cat": "HostToDevice", "id": "1030792151039", "pid": 43767502, "tid": 439084, "ts": "1704161511427880.236"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427884.264", "dur": 7.55, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 406, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 240}, "ph": "X"}, {"name": "HostToDevice1035087118335", "ph": "f", "id": "1035087118335", "ts": "1704161511427884.264", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "HostToDevice1035087118335", "ph": "s", "cat": "HostToDevice", "id": "1035087118335", "pid": 43767502, "tid": 439084, "ts": "1704161511427884.296"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427884.264", "dur": 7.55, "ph": "X"}, {"name": "HostToDevice1039382085631", "ph": "s", "cat": "HostToDevice", "id": "1039382085631", "pid": 43767502, "tid": 439084, "ts": "1704161511427888.546"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427891.814", "dur": 0.11, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427891.924", "dur": 5.45, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 407, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 241}, "ph": "X"}, {"name": "HostToDevice1039382085631", "ph": "f", "id": "1039382085631", "ts": "1704161511427891.924", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427891.924", "dur": 5.45, "ph": "X"}, {"name": "HostToDevice1043677052927", "ph": "s", "cat": "HostToDevice", "id": "1043677052927", "pid": 43767502, "tid": 439084, "ts": "1704161511427892.126"}, {"name": "HostToDevice1047972020223", "ph": "s", "cat": "HostToDevice", "id": "1047972020223", "pid": 43767502, "tid": 439084, "ts": "1704161511427895.426"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427897.374", "dur": 0.12, "ph": "X"}, {"name": "aclnnMm_MatMulCommon_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427897.494", "dur": 6.1, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 408, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 242}, "ph": "X"}, {"name": "HostToDevice1043677052927", "ph": "f", "id": "1043677052927", "ts": "1704161511427897.494", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427897.494", "dur": 6.1, "ph": "X"}, {"name": "HostToDevice1052266987519", "ph": "s", "cat": "HostToDevice", "id": "1052266987519", "pid": 43767502, "tid": 439084, "ts": "1704161511427899.886"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427903.594", "dur": 0.11, "ph": "X"}, {"name": "aclnnMm_MatMulCommon_MatMulV2", "pid": 800, "tid": 3, "ts": "1704161511427903.704", "dur": 4.78, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 409, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 243}, "ph": "X"}, {"name": "HostToDevice1047972020223", "ph": "f", "id": "1047972020223", "ts": "1704161511427903.704", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427903.704", "dur": 4.78, "ph": "X"}, {"name": "aclnnMm_TransData_TransData", "pid": 800, "tid": 3, "ts": "1704161511427908.594", "dur": 9.02, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 410, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 244}, "ph": "X"}, {"name": "HostToDevice1052266987519", "ph": "f", "id": "1052266987519", "ts": "1704161511427908.594", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "AscendCL@aclnnReduceSum", "pid": 43767502, "tid": 439084, "ts": "1704161511427908.476", "dur": 8.660086514264277, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnReduceSum", "item_id": "0", "connection_id": 248}, "ph": "X"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427908.594", "dur": 9.02, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427908.484", "dur": 0.11, "ph": "X"}, {"name": "HostToDevice1060856922111", "ph": "s", "cat": "HostToDevice", "id": "1060856922111", "pid": 43767502, "tid": 439084, "ts": "1704161511427908.986"}, {"name": "HostToDevice1065151889407", "ph": "s", "cat": "HostToDevice", "id": "1065151889407", "pid": 43767502, "tid": 439084, "ts": "1704161511427912.766"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427917.614", "dur": 0.11, "ph": "X"}, {"name": "aclnnReduceSum_ReduceSumOpAiCore_MemSet", "pid": 800, "tid": 3, "ts": "1704161511427917.724", "dur": 6.3495, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 411, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 246}, "ph": "X"}, {"name": "HostToDevice1060856922111", "ph": "f", "id": "1060856922111", "ts": "1704161511427917.724", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427917.724", "dur": 6.3495, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427924.073", "dur": 0.11, "ph": "X"}, {"name": "aclnnReduceSum_ReduceSumOpAiCore_ReduceSum", "pid": 800, "tid": 3, "ts": "1704161511427924.183", "dur": 11.24, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 412, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 247}, "ph": "X"}, {"name": "HostToDevice1065151889407", "ph": "f", "id": "1065151889407", "ts": "1704161511427924.183", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427924.183", "dur": 11.24, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427935.423", "dur": 54.6095, "ph": "X"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511427947.477", "dur": 20.73020709476888, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 249}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511427986.077", "dur": 28.140281121408403, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 252}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511427990.032", "dur": 1.56, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 413, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 251}, "ph": "X"}, {"name": "HostToDevice1082331758591", "ph": "f", "id": "1082331758591", "ts": "1704161511427990.032", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511427990.032", "dur": 1.56, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511427991.087", "dur": 5.870058641885833, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 250}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511427991.592", "dur": 72.6195, "ph": "X"}, {"name": "HostToDevice1082331758591", "ph": "s", "cat": "HostToDevice", "id": "1082331758591", "pid": 43767502, "tid": 439084, "ts": "1704161511427999.137"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511428029.438", "dur": 17.0901707308056, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 253}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511428062.508", "dur": 26.1502612411098, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 256}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511428064.212", "dur": 4.9, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 414, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 255}, "ph": "X"}, {"name": "HostToDevice1099511627775", "ph": "f", "id": "1099511627775", "ts": "1704161511428064.212", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428064.212", "dur": 4.9, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511428067.368", "dur": 5.110051049409984, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 254}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511428069.112", "dur": 738.4625, "ph": "X"}, {"name": "HostToDevice1099511627775", "ph": "s", "cat": "HostToDevice", "id": "1099511627775", "pid": 43767502, "tid": 439084, "ts": "1704161511428074.598"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428133.548", "dur": 1.4300142858427156, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 257}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428138.449", "dur": 0.2700026973269463, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 258}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428146.938", "dur": 0.47000469534690653, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 259}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428147.989", "dur": 0.13000129871297414, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 260}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428149.339", "dur": 0.42000419584191645, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 261}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428150.369", "dur": 0.15000149851497016, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 262}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428151.489", "dur": 0.42000419584191645, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 263}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428152.309", "dur": 0.16000159841596817, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 264}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 439228, "ts": "1704161511428153.329", "dur": 1.0200101899017973, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 265}, "ph": "X"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511428176.399", "dur": 811.2981048680675, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 329}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511428178.259", "dur": 14.520145056249111, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 266}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428326.880", "dur": 0.8000079920798409, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 267}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428331.090", "dur": 0.2600025974259483, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 268}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428333.050", "dur": 0.42000419584191645, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 269}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428334.570", "dur": 0.16000159841596817, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 270}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428335.380", "dur": 0.39000389613892245, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 271}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428336.340", "dur": 0.14000139861397215, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 272}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428337.101", "dur": 0.39000389613892245, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 273}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428337.871", "dur": 0.12000119881197613, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 274}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 439228, "ts": "1704161511428339.040", "dur": 0.1900018981189622, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 275}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428354.520", "dur": 0.7400073926738528, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 276}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428356.051", "dur": 0.1800017982179642, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 277}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428371.151", "dur": 0.5100050949508986, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 278}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428372.141", "dur": 0.14000139861397215, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 279}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428373.191", "dur": 0.41000409594091847, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 280}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428374.271", "dur": 0.15000149851497016, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 281}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428374.781", "dur": 0.44000439564391247, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 282}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428375.641", "dur": 0.11000109891097812, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 283}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 439228, "ts": "1704161511428376.511", "dur": 0.13000129871297414, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 284}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428422.522", "dur": 0.5100050949508986, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 285}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428423.662", "dur": 0.1900018981189622, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 286}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428427.522", "dur": 0.44000439564391247, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 287}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428428.432", "dur": 0.15000149851497016, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 288}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428429.042", "dur": 0.3800037962379244, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 289}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428430.242", "dur": 0.17000169831696618, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 290}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 439228, "ts": "1704161511428432.012", "dur": 0.23000229772295425, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 291}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceCopyGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511428501.992", "dur": 18.310182918727357, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceCopyGetWorkspaceSize", "item_id": "0", "connection_id": 292}, "ph": "X"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511428579.803", "dur": 19.42019400773814, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 293}, "ph": "X"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511428638.814", "dur": 13.5701355656543, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 294}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428685.224", "dur": 1.2700126874267474, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 295}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428688.284", "dur": 0.25000249752495024, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 296}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428689.494", "dur": 0.7200071928718568, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 297}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428690.744", "dur": 0.17000169831696618, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 298}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428692.334", "dur": 0.4800047952479045, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 299}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428693.404", "dur": 0.14000139861397215, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 300}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428694.284", "dur": 0.40000399603992043, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 301}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428695.264", "dur": 0.14000139861397215, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 302}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 439228, "ts": "1704161511428695.914", "dur": 0.15000149851497016, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 303}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428776.685", "dur": 0.8200081918818369, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 304}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428779.025", "dur": 0.1800017982179642, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 305}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428805.935", "dur": 0.6300062937628746, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 306}, "ph": "X"}, {"name": "trans_Cast_20", "pid": 800, "tid": 3, "ts": "1704161511428807.574", "dur": 4.11, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 415, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 313}, "ph": "X"}, {"name": "HostToDevice1348619730943", "ph": "f", "id": "1348619730943", "ts": "1704161511428807.574", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428807.475", "dur": 0.15000149851497016, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 307}, "ph": "X"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428807.574", "dur": 4.11, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428808.675", "dur": 0.4800047952479045, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 308}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428809.795", "dur": 0.15000149851497016, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 309}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428810.535", "dur": 0.6000059940598806, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 310}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428811.475", "dur": 0.14000139861397215, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 311}, "ph": "X"}, {"name": "HostToDevice1348619730943", "ph": "s", "cat": "HostToDevice", "id": "1348619730943", "pid": 43767502, "tid": 439084, "ts": "1704161511428811.675"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511428811.684", "dur": 32.7395, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 439228, "ts": "1704161511428812.166", "dur": 0.13000129871297414, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 312}, "ph": "X"}, {"name": "trans_TransData_21", "pid": 800, "tid": 3, "ts": "1704161511428844.424", "dur": 14.48, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 416, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 321}, "ph": "X"}, {"name": "HostToDevice1382979469311", "ph": "f", "id": "1382979469311", "ts": "1704161511428844.424", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428844.424", "dur": 14.48, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428844.806", "dur": 0.5900058941588827, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 314}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428846.565", "dur": 0.15000149851497016, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 315}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428850.496", "dur": 0.44000439564391247, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 316}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428851.426", "dur": 0.14000139861397215, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 317}, "ph": "X"}, {"name": "AscendCL@aclCreateTensorDesc", "pid": 43767502, "tid": 439228, "ts": "1704161511428851.966", "dur": 0.39000389613892245, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateTensorDesc", "item_id": "0", "connection_id": 318}, "ph": "X"}, {"name": "AscendCL@aclCreateDataBuffer", "pid": 43767502, "tid": 439228, "ts": "1704161511428852.706", "dur": 0.14000139861397215, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclCreateDataBuffer", "item_id": "0", "connection_id": 319}, "ph": "X"}, {"name": "AscendCL@aclopCreateAttr", "pid": 43767502, "tid": 439228, "ts": "1704161511428853.246", "dur": 0.4600045954459085, "args": {"Thread Id": 439228, "Mode": "ACL_OP", "level": "acl", "id": "aclopCreateAttr", "item_id": "0", "connection_id": 320}, "ph": "X"}, {"name": "HostToDevice1382979469311", "ph": "s", "cat": "HostToDevice", "id": "1382979469311", "pid": 43767502, "tid": 439084, "ts": "1704161511428854.676"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511428858.904", "dur": 12.76, "ph": "X"}, {"name": "trans_Cast_22", "pid": 800, "tid": 3, "ts": "1704161511428871.664", "dur": 4.0595, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 417, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 322}, "ph": "X"}, {"name": "HostToDevice1387274436607", "ph": "f", "id": "1387274436607", "ts": "1704161511428871.664", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428871.664", "dur": 4.0595, "ph": "X"}, {"name": "AscendCL@aclnnInplaceCopyGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511428874.666", "dur": 13.540135265951307, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceCopyGetWorkspaceSize", "item_id": "0", "connection_id": 323}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511428875.724", "dur": 12.97, "ph": "X"}, {"name": "HostToDevice1387274436607", "ph": "s", "cat": "HostToDevice", "id": "1387274436607", "pid": 43767502, "tid": 439084, "ts": "1704161511428880.716"}, {"name": "trans_TransData_23", "pid": 800, "tid": 3, "ts": "1704161511428888.694", "dur": 13.56, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 418, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 324}, "ph": "X"}, {"name": "HostToDevice1395864371199", "ph": "f", "id": "1395864371199", "ts": "1704161511428888.694", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428888.694", "dur": 13.56, "ph": "X"}, {"name": "HostToDevice1395864371199", "ph": "s", "cat": "HostToDevice", "id": "1395864371199", "pid": 43767502, "tid": 439084, "ts": "1704161511428901.746"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511428902.254", "dur": 37.9895, "ph": "X"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511428931.977", "dur": 17.80017782377646, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 326}, "ph": "X"}, {"name": "HostToDevice1400159338495", "ph": "s", "cat": "HostToDevice", "id": "1400159338495", "pid": 43767502, "tid": 439084, "ts": "1704161511428936.876"}, {"name": "MaxPoolGradWithArgmaxV14", "pid": 800, "tid": 3, "ts": "1704161511428940.243", "dur": 29.47, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 419, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 325}, "ph": "X"}, {"name": "HostToDevice1400159338495", "ph": "f", "id": "1400159338495", "ts": "1704161511428940.243", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428940.243", "dur": 29.47, "ph": "X"}, {"name": "HostToDevice1408749273087", "ph": "s", "cat": "HostToDevice", "id": "1408749273087", "pid": 43767502, "tid": 439084, "ts": "1704161511428960.637"}, {"name": "trans_TransData_24", "pid": 800, "tid": 3, "ts": "1704161511428969.833", "dur": 11.8695, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 420, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 327}, "ph": "X"}, {"name": "HostToDevice1408749273087", "ph": "f", "id": "1408749273087", "ts": "1704161511428969.833", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428969.833", "dur": 11.8695, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511428969.713", "dur": 0.12, "ph": "X"}, {"name": "HostToDevice1413044240383", "ph": "s", "cat": "HostToDevice", "id": "1413044240383", "pid": 43767502, "tid": 439084, "ts": "1704161511428977.147"}, {"name": "AscendCL@aclnnAddGetWorkspaceSize", "pid": 43767502, "tid": 439228, "ts": "1704161511428980.067", "dur": 12.380123677435538, "args": {"Thread Id": 439228, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddGetWorkspaceSize", "item_id": "0", "connection_id": 330}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511428981.702", "dur": 0.19, "ph": "X"}, {"name": "trans_Cast_25", "pid": 800, "tid": 3, "ts": "1704161511428981.892", "dur": 4.8, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 421, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 328}, "ph": "X"}, {"name": "HostToDevice1413044240383", "ph": "f", "id": "1413044240383", "ts": "1704161511428981.892", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428981.892", "dur": 4.8, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511428986.692", "dur": 10.77, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511428995.497", "dur": 7.84007832238244, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 331}, "ph": "X"}, {"name": "aclnnThresholdBackward_ReluGrad_ReluGrad", "pid": 800, "tid": 3, "ts": "1704161511428997.462", "dur": 6.06, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 422, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 332}, "ph": "X"}, {"name": "HostToDevice1430224109567", "ph": "f", "id": "1430224109567", "ts": "1704161511428997.462", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511428997.462", "dur": 6.06, "ph": "X"}, {"name": "AscendCL@aclnnThresholdBackward", "pid": 43767502, "tid": 439084, "ts": "1704161511429000.177", "dur": 13.700136864367275, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnThresholdBackward", "item_id": "0", "connection_id": 335}, "ph": "X"}, {"name": "HostToDevice1430224109567", "ph": "s", "cat": "HostToDevice", "id": "1430224109567", "pid": 43767502, "tid": 439084, "ts": "1704161511429001.247"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429003.522", "dur": 723.8325, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511429005.307", "dur": 2.8000279722794432, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 333}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511429010.867", "dur": 1.1300112888127751, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 334}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511429012.617", "dur": 1.3300132868327355, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 336}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511429018.467", "dur": 0.8800087912878249, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 337}, "ph": "X"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511429020.697", "dur": 898.1689727080374, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 352}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511429022.177", "dur": 15.19015174961598, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 338}, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511429338.161", "dur": 53.51053457024035, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 339}, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511429456.792", "dur": 42.2304218819146, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 340}, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511429627.794", "dur": 25.590255646653908, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 341}, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511429707.764", "dur": 27.690276625863493, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 342}, "ph": "X"}, {"name": "trans_Cast_26", "pid": 800, "tid": 3, "ts": "1704161511429727.355", "dur": 2.26, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 423, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 343}, "ph": "X"}, {"name": "HostToDevice1477468749823", "ph": "f", "id": "1477468749823", "ts": "1704161511429727.355", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511429727.355", "dur": 2.26, "ph": "X"}, {"name": "HostToDevice1477468749823", "ph": "s", "cat": "HostToDevice", "id": "1477468749823", "pid": 43767502, "tid": 439084, "ts": "1704161511429727.564"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429729.615", "dur": 25.98, "ph": "X"}, {"name": "trans_TransData_27_MemSet", "pid": 800, "tid": 3, "ts": "1704161511429755.595", "dur": 9.77, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 424, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 344}, "ph": "X"}, {"name": "HostToDevice1481763717119", "ph": "f", "id": "1481763717119", "ts": "1704161511429755.595", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511429755.595", "dur": 9.77, "ph": "X"}, {"name": "HostToDevice1481763717119", "ph": "s", "cat": "HostToDevice", "id": "1481763717119", "pid": 43767502, "tid": 439084, "ts": "1704161511429762.925"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429765.365", "dur": 0.11, "ph": "X"}, {"name": "trans_TransData_27", "pid": 800, "tid": 3, "ts": "1704161511429765.475", "dur": 6.28, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 425, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 345}, "ph": "X"}, {"name": "HostToDevice1486058684415", "ph": "f", "id": "1486058684415", "ts": "1704161511429765.475", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511429765.475", "dur": 6.28, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429771.755", "dur": 6.3195, "ph": "X"}, {"name": "HostToDevice1486058684415", "ph": "s", "cat": "HostToDevice", "id": "1486058684415", "pid": 43767502, "tid": 439084, "ts": "1704161511429772.385"}, {"name": "trans_Cast_28", "pid": 800, "tid": 3, "ts": "1704161511429778.074", "dur": 4.77, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 426, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 346}, "ph": "X"}, {"name": "HostToDevice1490353651711", "ph": "f", "id": "1490353651711", "ts": "1704161511429778.074", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511429778.074", "dur": 4.77, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429782.844", "dur": 33.2, "ph": "X"}, {"name": "HostToDevice1490353651711", "ph": "s", "cat": "HostToDevice", "id": "1490353651711", "pid": 43767502, "tid": 439084, "ts": "1704161511429796.665"}, {"name": "trans_TransData_29", "pid": 800, "tid": 3, "ts": "1704161511429816.044", "dur": 13.7395, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 427, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 347}, "ph": "X"}, {"name": "HostToDevice1494648619007", "ph": "f", "id": "1494648619007", "ts": "1704161511429816.044", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511429816.044", "dur": 13.7395, "ph": "X"}, {"name": "HostToDevice1494648619007", "ph": "s", "cat": "HostToDevice", "id": "1494648619007", "pid": 43767502, "tid": 439084, "ts": "1704161511429826.896"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429829.784", "dur": 26.46, "ph": "X"}, {"name": "Conv2DBackpropInput5", "pid": 800, "tid": 3, "ts": "1704161511429856.244", "dur": 23.6695, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 428, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 348}, "ph": "X"}, {"name": "HostToDevice1498943586303", "ph": "f", "id": "1498943586303", "ts": "1704161511429856.244", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511429856.244", "dur": 23.6695, "ph": "X"}, {"name": "HostToDevice1498943586303", "ph": "s", "cat": "HostToDevice", "id": "1498943586303", "pid": 43767502, "tid": 439084, "ts": "1704161511429866.016"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511429873.516", "dur": 26.880268533882653, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 350}, "ph": "X"}, {"name": "trans_TransData_30", "pid": 800, "tid": 3, "ts": "1704161511429880.104", "dur": 10.79, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 429, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 349}, "ph": "X"}, {"name": "HostToDevice1503238553599", "ph": "f", "id": "1503238553599", "ts": "1704161511429880.104", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511429880.104", "dur": 10.79, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429879.914", "dur": 0.19, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429890.894", "dur": 7.58, "ph": "X"}, {"name": "HostToDevice1503238553599", "ph": "s", "cat": "HostToDevice", "id": "1503238553599", "pid": 43767502, "tid": 439084, "ts": "1704161511429891.336"}, {"name": "trans_Cast_31", "pid": 800, "tid": 3, "ts": "1704161511429898.474", "dur": 4.81, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 430, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 351}, "ph": "X"}, {"name": "HostToDevice1511828488191", "ph": "f", "id": "1511828488191", "ts": "1704161511429898.474", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511429898.474", "dur": 4.81, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511429903.284", "dur": 685.463, "ph": "X"}, {"name": "HostToDevice1511828488191", "ph": "s", "cat": "HostToDevice", "id": "1511828488191", "pid": 43767502, "tid": 439084, "ts": "1704161511429907.486"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511429934.466", "dur": 825.9382511231287, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 374}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511429936.376", "dur": 14.830148153180051, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 357}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511429936.736", "dur": 6.020060140400802, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 353}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511429944.467", "dur": 1.66001658356567, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 354}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511429946.577", "dur": 2.0800207794075862, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 355}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511429948.867", "dur": 1.830018281882636, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 356}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511429952.217", "dur": 0.6000059940598806, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 358}, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511429952.387", "dur": 29.320292909726167, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 359}, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430100.448", "dur": 22.42022397803754, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 360}, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430170.659", "dur": 24.730247055168082, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 361}, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430319.690", "dur": 24.740247155069078, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 362}, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430392.901", "dur": 28.830288014577267, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 363}, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430533.042", "dur": 20.820207993877858, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 364}, "ph": "X"}, {"name": "trans_Cast_32", "pid": 800, "tid": 3, "ts": "1704161511430588.746", "dur": 4.68, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 431, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 365}, "ph": "X"}, {"name": "HostToDevice1571958030335", "ph": "f", "id": "1571958030335", "ts": "1704161511430588.746", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511430588.746", "dur": 4.68, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511430593.426", "dur": 28.77, "ph": "X"}, {"name": "HostToDevice1571958030335", "ph": "s", "cat": "HostToDevice", "id": "1571958030335", "pid": 43767502, "tid": 439084, "ts": "1704161511430594.283"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430600.623", "dur": 24.220241960217184, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 366}, "ph": "X"}, {"name": "trans_TransData_33", "pid": 800, "tid": 3, "ts": "1704161511430622.196", "dur": 9.3995, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 432, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 367}, "ph": "X"}, {"name": "HostToDevice1580547964927", "ph": "f", "id": "1580547964927", "ts": "1704161511430622.196", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511430622.196", "dur": 9.3995, "ph": "X"}, {"name": "HostToDevice1580547964927", "ph": "s", "cat": "HostToDevice", "id": "1580547964927", "pid": 43767502, "tid": 439084, "ts": "1704161511430630.843"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511430631.596", "dur": 12.93, "ph": "X"}, {"name": "trans_Cast_34", "pid": 800, "tid": 3, "ts": "1704161511430644.526", "dur": 4.06, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 433, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 368}, "ph": "X"}, {"name": "HostToDevice1584842932223", "ph": "f", "id": "1584842932223", "ts": "1704161511430644.526", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511430644.526", "dur": 4.06, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511430648.586", "dur": 14.14, "ph": "X"}, {"name": "HostToDevice1584842932223", "ph": "s", "cat": "HostToDevice", "id": "1584842932223", "pid": 43767502, "tid": 439084, "ts": "1704161511430656.704"}, {"name": "trans_TransData_35", "pid": 800, "tid": 3, "ts": "1704161511430662.726", "dur": 13.7795, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 434, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 369}, "ph": "X"}, {"name": "HostToDevice1589137899519", "ph": "f", "id": "1589137899519", "ts": "1704161511430662.726", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511430662.726", "dur": 13.7795, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511430676.506", "dur": 28.48, "ph": "X"}, {"name": "HostToDevice1589137899519", "ph": "s", "cat": "HostToDevice", "id": "1589137899519", "pid": 43767502, "tid": 439084, "ts": "1704161511430677.454"}, {"name": "Conv2DBackpropFilter6_MemSet", "pid": 800, "tid": 3, "ts": "1704161511430704.986", "dur": 8.29, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 435, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 370}, "ph": "X"}, {"name": "HostToDevice1593432866815", "ph": "f", "id": "1593432866815", "ts": "1704161511430704.986", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511430704.986", "dur": 8.29, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511430713.276", "dur": 0.2, "ph": "X"}, {"name": "Conv2DBackpropFilter6", "pid": 800, "tid": 3, "ts": "1704161511430713.476", "dur": 15.0695, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 436, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 371}, "ph": "X"}, {"name": "HostToDevice1597727834111", "ph": "f", "id": "1597727834111", "ts": "1704161511430713.476", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511430713.476", "dur": 15.0695, "ph": "X"}, {"name": "HostToDevice1593432866815", "ph": "s", "cat": "HostToDevice", "id": "1593432866815", "pid": 43767502, "tid": 439084, "ts": "1704161511430715.814"}, {"name": "HostToDevice1597727834111", "ph": "s", "cat": "HostToDevice", "id": "1597727834111", "pid": 43767502, "tid": 439084, "ts": "1704161511430727.975"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511430728.545", "dur": 10.69, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430738.035", "dur": 21.550215286650715, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 373}, "ph": "X"}, {"name": "trans_TransData_36", "pid": 800, "tid": 3, "ts": "1704161511430739.235", "dur": 11.42, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 437, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 372}, "ph": "X"}, {"name": "HostToDevice1602022801407", "ph": "f", "id": "1602022801407", "ts": "1704161511430739.235", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511430739.235", "dur": 11.42, "ph": "X"}, {"name": "HostToDevice1602022801407", "ph": "s", "cat": "HostToDevice", "id": "1602022801407", "pid": 43767502, "tid": 439084, "ts": "1704161511430749.075"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511430750.655", "dur": 301.677, "ph": "X"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511430773.035", "dur": 317.3531703581719, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 386}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511430774.535", "dur": 8.70008691386827, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 375}, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430803.885", "dur": 27.110270831605607, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 381}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511430808.775", "dur": 7.130071229411582, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 376}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511430817.885", "dur": 1.86001858158563, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 377}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511430820.445", "dur": 1.2300122878227553, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 378}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511430822.505", "dur": 0.8700086913868269, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 379}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511430824.365", "dur": 0.6100060939608787, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 380}, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511430938.186", "dur": 20.490204697144925, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 382}, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511431003.617", "dur": 23.540235166949316, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 383}, "ph": "X"}, {"name": "ReduceSum7_MemSet", "pid": 800, "tid": 3, "ts": "1704161511431052.332", "dur": 7.96, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 438, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 384}, "ph": "X"}, {"name": "HostToDevice1653562408959", "ph": "f", "id": "1653562408959", "ts": "1704161511431052.332", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431052.332", "dur": 7.96, "ph": "X"}, {"name": "HostToDevice1653562408959", "ph": "s", "cat": "HostToDevice", "id": "1653562408959", "pid": 43767502, "tid": 439084, "ts": "1704161511431058.928"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431060.292", "dur": 10.97, "ph": "X"}, {"name": "ReduceSum7", "pid": 800, "tid": 3, "ts": "1704161511431071.262", "dur": 19.3195, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 439, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 385}, "ph": "X"}, {"name": "HostToDevice1657857376255", "ph": "f", "id": "1657857376255", "ts": "1704161511431071.262", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431071.262", "dur": 19.3195, "ph": "X"}, {"name": "HostToDevice1657857376255", "ph": "s", "cat": "HostToDevice", "id": "1657857376255", "pid": 43767502, "tid": 439084, "ts": "1704161511431077.838"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431090.582", "dur": 77.1695, "ph": "X"}, {"name": "AscendCL@aclnnInplaceCopy", "pid": 43767502, "tid": 439084, "ts": "1704161511431108.718", "dur": 20.640206195659893, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceCopy", "item_id": "0", "connection_id": 387}, "ph": "X"}, {"name": "MEMCPY_ASYNC", "pid": 800, "tid": 3, "ts": "1704161511431121.892", "dur": 1.11, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 440, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": -1}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511431138.119", "dur": 57.58057522994655, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 395}, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511431141.639", "dur": 22.200221780215585, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 392}, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511431147.878", "dur": 16.160161440012786, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 393}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511431150.119", "dur": 1.720017182971658, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 388}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511431154.039", "dur": 1.5600155845556898, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 389}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511431156.019", "dur": 0.7900078921788428, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 390}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511431157.159", "dur": 0.42000419584191645, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 391}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511431167.751", "dur": 2.83, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 441, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 394}, "ph": "X"}, {"name": "HostToDevice1696512081919", "ph": "f", "id": "1696512081919", "ts": "1704161511431167.751", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431167.751", "dur": 2.83, "ph": "X"}, {"name": "HostToDevice1696512081919", "ph": "s", "cat": "HostToDevice", "id": "1696512081919", "pid": 43767502, "tid": 439084, "ts": "1704161511431168.359"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431170.581", "dur": 37.7295, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511431202.059", "dur": 29.400293708934154, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 398}, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511431207.049", "dur": 6.260062538024755, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 396}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511431208.310", "dur": 1.42, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 442, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 397}, "ph": "X"}, {"name": "HostToDevice1709396983807", "ph": "f", "id": "1709396983807", "ts": "1704161511431208.310", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431208.310", "dur": 1.42, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431209.730", "dur": 648.9735, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511431211.750", "dur": 27.55027522724952, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 399}, "ph": "X"}, {"name": "HostToDevice1709396983807", "ph": "s", "cat": "HostToDevice", "id": "1709396983807", "pid": 43767502, "tid": 439084, "ts": "1704161511431215.129"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511431236.740", "dur": 790.5478975734967, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 412}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511431238.000", "dur": 14.48014465664512, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 400}, "ph": "X"}, {"name": "AscendCL@aclnnDivsGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511431349.821", "dur": 20.870208493382847, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivsGetWorkspaceSize", "item_id": "0", "connection_id": 401}, "ph": "X"}, {"name": "AscendCL@aclnnAddcdivGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511431415.551", "dur": 23.07023047160241, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnAddcdivGetWorkspaceSize", "item_id": "0", "connection_id": 402}, "ph": "X"}, {"name": "AscendCL@aclnnReduceSumGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511431631.564", "dur": 62.8906282773765, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnReduceSumGetWorkspaceSize", "item_id": "0", "connection_id": 403}, "ph": "X"}, {"name": "AscendCL@aclnnCastGetWorkspaceSize", "pid": 43767502, "tid": 437675, "ts": "1704161511431736.575", "dur": 16.070160540903803, "args": {"Thread Id": 437675, "Mode": "ACL_NN", "level": "acl", "id": "aclnnCastGetWorkspaceSize", "item_id": "0", "connection_id": 404}, "ph": "X"}, {"name": "trans_Cast_37", "pid": 800, "tid": 3, "ts": "1704161511431858.704", "dur": 5.43, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 443, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 405}, "ph": "X"}, {"name": "HostToDevice1743756722175", "ph": "f", "id": "1743756722175", "ts": "1704161511431858.704", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431858.704", "dur": 5.43, "ph": "X"}, {"name": "HostToDevice1743756722175", "ph": "s", "cat": "HostToDevice", "id": "1743756722175", "pid": 43767502, "tid": 439084, "ts": "1704161511431862.766"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431864.134", "dur": 27.5995, "ph": "X"}, {"name": "trans_TransData_38", "pid": 800, "tid": 3, "ts": "1704161511431891.734", "dur": 8.44, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 444, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 406}, "ph": "X"}, {"name": "HostToDevice1748051689471", "ph": "f", "id": "1748051689471", "ts": "1704161511431891.734", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431891.734", "dur": 8.44, "ph": "X"}, {"name": "HostToDevice1748051689471", "ph": "s", "cat": "HostToDevice", "id": "1748051689471", "pid": 43767502, "tid": 439084, "ts": "1704161511431900.226"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431900.174", "dur": 14.11, "ph": "X"}, {"name": "trans_Cast_39", "pid": 800, "tid": 3, "ts": "1704161511431914.284", "dur": 4.11, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 445, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 407}, "ph": "X"}, {"name": "HostToDevice1752346656767", "ph": "f", "id": "1752346656767", "ts": "1704161511431914.284", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431914.284", "dur": 4.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431918.394", "dur": 18.8995, "ph": "X"}, {"name": "HostToDevice1752346656767", "ph": "s", "cat": "HostToDevice", "id": "1752346656767", "pid": 43767502, "tid": 439084, "ts": "1704161511431926.356"}, {"name": "trans_TransData_40", "pid": 800, "tid": 3, "ts": "1704161511431937.293", "dur": 8.9, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 446, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 408}, "ph": "X"}, {"name": "HostToDevice1756641624063", "ph": "f", "id": "1756641624063", "ts": "1704161511431937.293", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431937.293", "dur": 8.9, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431946.193", "dur": 15.29, "ph": "X"}, {"name": "HostToDevice1756641624063", "ph": "s", "cat": "HostToDevice", "id": "1756641624063", "pid": 43767502, "tid": 439084, "ts": "1704161511431947.836"}, {"name": "MaxPoolGradWithArgmaxV18", "pid": 800, "tid": 3, "ts": "1704161511431961.483", "dur": 24.7395, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 447, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 409}, "ph": "X"}, {"name": "HostToDevice1760936591359", "ph": "f", "id": "1760936591359", "ts": "1704161511431961.483", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431961.483", "dur": 24.7395, "ph": "X"}, {"name": "HostToDevice1760936591359", "ph": "s", "cat": "HostToDevice", "id": "1760936591359", "pid": 43767502, "tid": 439084, "ts": "1704161511431976.447"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431986.222", "dur": 0.29, "ph": "X"}, {"name": "trans_TransData_41", "pid": 800, "tid": 3, "ts": "1704161511431986.512", "dur": 7.71, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 448, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 410}, "ph": "X"}, {"name": "HostToDevice1765231558655", "ph": "f", "id": "1765231558655", "ts": "1704161511431986.512", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511431986.512", "dur": 7.71, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511431994.222", "dur": 7.44, "ph": "X"}, {"name": "HostToDevice1765231558655", "ph": "s", "cat": "HostToDevice", "id": "1765231558655", "pid": 43767502, "tid": 439084, "ts": "1704161511431999.017"}, {"name": "trans_Cast_42", "pid": 800, "tid": 3, "ts": "1704161511432001.662", "dur": 5.71, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 449, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 411}, "ph": "X"}, {"name": "HostToDevice1769526525951", "ph": "f", "id": "1769526525951", "ts": "1704161511432001.662", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432001.662", "dur": 5.71, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432007.372", "dur": 25.1095, "ph": "X"}, {"name": "HostToDevice1769526525951", "ph": "s", "cat": "HostToDevice", "id": "1769526525951", "pid": 43767502, "tid": 439084, "ts": "1704161511432015.887"}, {"name": "aclnnThresholdBackward_ReluGrad_ReluGrad", "pid": 800, "tid": 3, "ts": "1704161511432032.482", "dur": 11.59, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 450, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 413}, "ph": "X"}, {"name": "HostToDevice1778116460543", "ph": "f", "id": "1778116460543", "ts": "1704161511432032.482", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432032.482", "dur": 11.59, "ph": "X"}, {"name": "AscendCL@aclnnThresholdBackward", "pid": 43767502, "tid": 439084, "ts": "1704161511432039.808", "dur": 10.080100700205994, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnThresholdBackward", "item_id": "0", "connection_id": 414}, "ph": "X"}, {"name": "HostToDevice1778116460543", "ph": "s", "cat": "HostToDevice", "id": "1778116460543", "pid": 43767502, "tid": 439084, "ts": "1704161511432040.698"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432044.072", "dur": 39.2995, "ph": "X"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511432055.558", "dur": 170.31170141389714, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 428}, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511432056.938", "dur": 12.01011998109861, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 415}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511432076.118", "dur": 8.230082218521362, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 416}, "ph": "X"}, {"name": "trans_Cast_32", "pid": 800, "tid": 3, "ts": "1704161511432083.372", "dur": 4.7, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 451, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 421}, "ph": "X"}, {"name": "HostToDevice1812476198911", "ph": "f", "id": "1812476198911", "ts": "1704161511432083.372", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432083.372", "dur": 4.7, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432086.128", "dur": 1.9100190810906204, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 417}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432088.072", "dur": 21.32, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432088.258", "dur": 1.6300162838626757, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 418}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432090.318", "dur": 1.0800107893077853, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 419}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432092.108", "dur": 1.0900108892087832, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 420}, "ph": "X"}, {"name": "HostToDevice1812476198911", "ph": "s", "cat": "HostToDevice", "id": "1812476198911", "pid": 43767502, "tid": 439084, "ts": "1704161511432092.728"}, {"name": "trans_TransData_33", "pid": 800, "tid": 3, "ts": "1704161511432109.392", "dur": 9.53, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 452, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 422}, "ph": "X"}, {"name": "HostToDevice1816771166207", "ph": "f", "id": "1816771166207", "ts": "1704161511432109.392", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432109.392", "dur": 9.53, "ph": "X"}, {"name": "HostToDevice1816771166207", "ph": "s", "cat": "HostToDevice", "id": "1816771166207", "pid": 43767502, "tid": 439084, "ts": "1704161511432116.988"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432118.922", "dur": 7.64, "ph": "X"}, {"name": "trans_Cast_34", "pid": 800, "tid": 3, "ts": "1704161511432126.562", "dur": 5.7795, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 453, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 423}, "ph": "X"}, {"name": "HostToDevice1821066133503", "ph": "f", "id": "1821066133503", "ts": "1704161511432126.562", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432126.562", "dur": 5.7795, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432132.341", "dur": 12.78, "ph": "X"}, {"name": "HostToDevice1821066133503", "ph": "s", "cat": "HostToDevice", "id": "1821066133503", "pid": 43767502, "tid": 439084, "ts": "1704161511432138.228"}, {"name": "trans_TransData_35", "pid": 800, "tid": 3, "ts": "1704161511432145.121", "dur": 8.87, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 454, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 424}, "ph": "X"}, {"name": "HostToDevice1825361100799", "ph": "f", "id": "1825361100799", "ts": "1704161511432145.121", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432145.121", "dur": 8.87, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432153.991", "dur": 25.4195, "ph": "X"}, {"name": "HostToDevice1825361100799", "ph": "s", "cat": "HostToDevice", "id": "1825361100799", "pid": 43767502, "tid": 439084, "ts": "1704161511432156.669"}, {"name": "Conv2DBackpropFilter6_MemSet", "pid": 800, "tid": 3, "ts": "1704161511432179.410", "dur": 9.84, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 455, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 425}, "ph": "X"}, {"name": "HostToDevice1829656068095", "ph": "f", "id": "1829656068095", "ts": "1704161511432179.410", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432179.410", "dur": 9.84, "ph": "X"}, {"name": "HostToDevice1829656068095", "ph": "s", "cat": "HostToDevice", "id": "1829656068095", "pid": 43767502, "tid": 439084, "ts": "1704161511432188.079"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432189.250", "dur": 0.15, "ph": "X"}, {"name": "Conv2DBackpropFilter6", "pid": 800, "tid": 3, "ts": "1704161511432189.400", "dur": 35.18, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 456, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 426}, "ph": "X"}, {"name": "HostToDevice1833951035391", "ph": "f", "id": "1833951035391", "ts": "1704161511432189.400", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432189.400", "dur": 35.18, "ph": "X"}, {"name": "HostToDevice1833951035391", "ph": "s", "cat": "HostToDevice", "id": "1833951035391", "pid": 43767502, "tid": 439084, "ts": "1704161511432199.149"}, {"name": "HostToDevice1838246002687", "ph": "s", "cat": "HostToDevice", "id": "1838246002687", "pid": 43767502, "tid": 439084, "ts": "1704161511432216.440"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432224.580", "dur": 0.15, "ph": "X"}, {"name": "trans_TransData_36", "pid": 800, "tid": 3, "ts": "1704161511432224.730", "dur": 9.4295, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 457, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 427}, "ph": "X"}, {"name": "HostToDevice1838246002687", "ph": "f", "id": "1838246002687", "ts": "1704161511432224.730", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432224.730", "dur": 9.4295, "ph": "X"}, {"name": "AscendCL@aclopCompileAndExecute", "pid": 43767502, "tid": 439084, "ts": "1704161511432234.100", "dur": 63.06062997569346, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "aclopCompileAndExecute", "item_id": "0", "connection_id": 437}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432234.160", "dur": 26.77, "ph": "X"}, {"name": "AscendCL@opCompile", "pid": 43767502, "tid": 439084, "ts": "1704161511432235.290", "dur": 5.850058442083837, "args": {"Thread Id": 439084, "Mode": "ACL_OP", "level": "acl", "id": "opCompile", "item_id": "0", "connection_id": 429}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511432257.490", "dur": 9.010090010799207, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 430}, "ph": "X"}, {"name": "ReduceSum7_MemSet", "pid": 800, "tid": 3, "ts": "1704161511432260.930", "dur": 9.23, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 458, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 433}, "ph": "X"}, {"name": "HostToDevice1864015806463", "ph": "f", "id": "1864015806463", "ts": "1704161511432260.930", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432260.930", "dur": 9.23, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432268.150", "dur": 2.930029270992417, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 431}, "ph": "X"}, {"name": "HostToDevice1864015806463", "ph": "s", "cat": "HostToDevice", "id": "1864015806463", "pid": 43767502, "tid": 439084, "ts": "1704161511432269.360"}, {"name": "ReduceSum7", "pid": 800, "tid": 3, "ts": "1704161511432270.310", "dur": 14.4895, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 459, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 436}, "ph": "X"}, {"name": "HostToDevice1876900708351", "ph": "f", "id": "1876900708351", "ts": "1704161511432270.310", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432270.310", "dur": 14.4895, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432270.160", "dur": 0.15, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432271.510", "dur": 1.6000159841596817, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 432}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432274.190", "dur": 1.6000159841596817, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 434}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432279.580", "dur": 0.4300042957429145, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 435}, "ph": "X"}, {"name": "HostToDevice1876900708351", "ph": "s", "cat": "HostToDevice", "id": "1876900708351", "pid": 43767502, "tid": 439084, "ts": "1704161511432284.190"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432284.800", "dur": 69.8395, "ph": "X"}, {"name": "MEMCPY_ASYNC", "pid": 800, "tid": 3, "ts": "1704161511432297.430", "dur": 1.08, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 460, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": -1}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceCopy", "pid": 43767502, "tid": 439084, "ts": "1704161511432306.670", "dur": 12.000119881197614, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceCopy", "item_id": "0", "connection_id": 438}, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511432325.940", "dur": 49.130490813603224, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 445}, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432334.270", "dur": 13.850138362882246, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 443}, "ph": "X"}, {"name": "AscendCL@aclopDestroyAttr", "pid": 43767502, "tid": 439085, "ts": "1704161511432336.340", "dur": 1.890018881288624, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclopDestroyAttr", "item_id": "0", "connection_id": 439}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432340.310", "dur": 1.3000129871297414, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 440}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432342.331", "dur": 1.4300142858427156, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 441}, "ph": "X"}, {"name": "AscendCL@aclDestroyTensorDesc", "pid": 43767502, "tid": 439085, "ts": "1704161511432344.681", "dur": 0.39000389613892245, "args": {"Thread Id": 439085, "Mode": "ACL_OP", "level": "acl", "id": "aclDestroyTensorDesc", "item_id": "0", "connection_id": 442}, "ph": "X"}, {"name": "HostToDevice1911260446719", "ph": "s", "cat": "HostToDevice", "id": "1911260446719", "pid": 43767502, "tid": 439084, "ts": "1704161511432352.371"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511432354.639", "dur": 1.61, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 461, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 444}, "ph": "X"}, {"name": "HostToDevice1911260446719", "ph": "f", "id": "1911260446719", "ts": "1704161511432354.639", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432354.639", "dur": 1.61, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432356.249", "dur": 30.3995, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511432382.661", "dur": 30.36030329942996, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 448}, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511432386.648", "dur": 1.4, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 462, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 447}, "ph": "X"}, {"name": "HostToDevice1924145348607", "ph": "f", "id": "1924145348607", "ts": "1704161511432386.648", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432386.648", "dur": 1.4, "ph": "X"}, {"name": "AscendCL@Add_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432387.551", "dur": 6.740067333272659, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Add_Tiling", "item_id": "0", "connection_id": 446}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432388.048", "dur": 25.71, "ph": "X"}, {"name": "HostToDevice1924145348607", "ph": "s", "cat": "HostToDevice", "id": "1924145348607", "pid": 43767502, "tid": 439084, "ts": "1704161511432396.041"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432413.758", "dur": 1.59, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 463, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 449}, "ph": "X"}, {"name": "HostToDevice1932735283199", "ph": "f", "id": "1932735283199", "ts": "1704161511432413.758", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432413.758", "dur": 1.59, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432415.348", "dur": 6.92, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432422.011", "dur": 8.500084915848308, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 450}, "ph": "X"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511432422.268", "dur": 1.63, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 464, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 451}, "ph": "X"}, {"name": "HostToDevice1941325217791", "ph": "f", "id": "1941325217791", "ts": "1704161511432422.268", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432422.268", "dur": 1.63, "ph": "X"}, {"name": "HostToDevice1932735283199", "ph": "s", "cat": "HostToDevice", "id": "1932735283199", "pid": 43767502, "tid": 439084, "ts": "1704161511432422.761"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432423.898", "dur": 10.6495, "ph": "X"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432434.548", "dur": 1.59, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 465, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 453}, "ph": "X"}, {"name": "HostToDevice1949915152383", "ph": "f", "id": "1949915152383", "ts": "1704161511432434.548", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432434.548", "dur": 1.59, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511432434.691", "dur": 6.360063537034735, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 452}, "ph": "X"}, {"name": "HostToDevice1941325217791", "ph": "s", "cat": "HostToDevice", "id": "1941325217791", "pid": 43767502, "tid": 439084, "ts": "1704161511432435.122"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432436.138", "dur": 8.4, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432444.372", "dur": 6.510065035549705, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 454}, "ph": "X"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511432444.538", "dur": 4.41, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 466, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 455}, "ph": "X"}, {"name": "HostToDevice1958505086975", "ph": "f", "id": "1958505086975", "ts": "1704161511432444.538", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432444.538", "dur": 4.41, "ph": "X"}, {"name": "HostToDevice1949915152383", "ph": "s", "cat": "HostToDevice", "id": "1949915152383", "pid": 43767502, "tid": 439084, "ts": "1704161511432445.191"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432448.948", "dur": 6.36, "ph": "X"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511432455.308", "dur": 1.7, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 467, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 457}, "ph": "X"}, {"name": "HostToDevice1967095021567", "ph": "f", "id": "1967095021567", "ts": "1704161511432455.308", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432455.308", "dur": 1.7, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511432457.072", "dur": 6.9900698307976095, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 456}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432457.008", "dur": 39.9695, "ph": "X"}, {"name": "HostToDevice1958505086975", "ph": "s", "cat": "HostToDevice", "id": "1958505086975", "pid": 43767502, "tid": 439084, "ts": "1704161511432457.782"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511432471.662", "dur": 6.140061339212778, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 458}, "ph": "X"}, {"name": "HostToDevice1967095021567", "ph": "s", "cat": "HostToDevice", "id": "1967095021567", "pid": 43767502, "tid": 439084, "ts": "1704161511432472.222"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511432485.042", "dur": 37.10037063270262, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 461}, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432493.122", "dur": 8.080080720006393, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 459}, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511432496.978", "dur": 1.87, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 468, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 460}, "ph": "X"}, {"name": "HostToDevice1979979923455", "ph": "f", "id": "1979979923455", "ts": "1704161511432496.978", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432496.978", "dur": 1.87, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432498.848", "dur": 16.15, "ph": "X"}, {"name": "HostToDevice1979979923455", "ph": "s", "cat": "HostToDevice", "id": "1979979923455", "pid": 43767502, "tid": 439084, "ts": "1704161511432503.672"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511432514.998", "dur": 1.73, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 469, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 462}, "ph": "X"}, {"name": "HostToDevice1988569858047", "ph": "f", "id": "1988569858047", "ts": "1704161511432514.998", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432514.998", "dur": 1.73, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432516.728", "dur": 38.8295, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511432529.373", "dur": 7.170071629015574, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 463}, "ph": "X"}, {"name": "HostToDevice1988569858047", "ph": "s", "cat": "HostToDevice", "id": "1988569858047", "pid": 43767502, "tid": 439084, "ts": "1704161511432529.962"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511432542.152", "dur": 40.230401901715, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 466}, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432551.852", "dur": 9.10009090990819, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 464}, "ph": "X"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511432555.557", "dur": 4.03, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 470, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 465}, "ph": "X"}, {"name": "HostToDevice2001454759935", "ph": "f", "id": "2001454759935", "ts": "1704161511432555.557", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432555.557", "dur": 4.03, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432559.587", "dur": 19.5795, "ph": "X"}, {"name": "HostToDevice2001454759935", "ph": "s", "cat": "HostToDevice", "id": "2001454759935", "pid": 43767502, "tid": 439084, "ts": "1704161511432563.503"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432579.166", "dur": 1.51, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 471, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 467}, "ph": "X"}, {"name": "HostToDevice2010044694527", "ph": "f", "id": "2010044694527", "ts": "1704161511432579.166", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432579.166", "dur": 1.51, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432580.676", "dur": 6.74, "ph": "X"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511432587.416", "dur": 1.47, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 472, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 469}, "ph": "X"}, {"name": "HostToDevice2018634629119", "ph": "f", "id": "2018634629119", "ts": "1704161511432587.416", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432587.416", "dur": 1.47, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432588.743", "dur": 7.24007232832256, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 468}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432588.886", "dur": 8.7, "ph": "X"}, {"name": "HostToDevice2010044694527", "ph": "s", "cat": "HostToDevice", "id": "2010044694527", "pid": 43767502, "tid": 439084, "ts": "1704161511432589.223"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432597.586", "dur": 1.48, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 473, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 471}, "ph": "X"}, {"name": "HostToDevice2027224563711", "ph": "f", "id": "2027224563711", "ts": "1704161511432597.586", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432597.586", "dur": 1.48, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432599.066", "dur": 7.29, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511432599.183", "dur": 6.650066434163677, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 470}, "ph": "X"}, {"name": "HostToDevice2018634629119", "ph": "s", "cat": "HostToDevice", "id": "2018634629119", "pid": 43767502, "tid": 439084, "ts": "1704161511432599.713"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511432606.356", "dur": 3.87, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 474, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 473}, "ph": "X"}, {"name": "HostToDevice2035814498303", "ph": "f", "id": "2035814498303", "ts": "1704161511432606.356", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432606.356", "dur": 3.87, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432609.163", "dur": 6.340063337232739, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 472}, "ph": "X"}, {"name": "HostToDevice2027224563711", "ph": "s", "cat": "HostToDevice", "id": "2027224563711", "pid": 43767502, "tid": 439084, "ts": "1704161511432609.573"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432610.226", "dur": 4.76, "ph": "X"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511432614.986", "dur": 1.4, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 475, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 475}, "ph": "X"}, {"name": "HostToDevice2044404432895", "ph": "f", "id": "2044404432895", "ts": "1704161511432614.986", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432614.986", "dur": 1.4, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432616.386", "dur": 24.6695, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511432618.983", "dur": 6.310063037529745, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 474}, "ph": "X"}, {"name": "HostToDevice2035814498303", "ph": "s", "cat": "HostToDevice", "id": "2035814498303", "pid": 43767502, "tid": 439084, "ts": "1704161511432619.544"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511432629.343", "dur": 5.760057542974854, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 476}, "ph": "X"}, {"name": "HostToDevice2044404432895", "ph": "s", "cat": "HostToDevice", "id": "2044404432895", "pid": 43767502, "tid": 439084, "ts": "1704161511432629.903"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511432638.704", "dur": 26.270262439921776, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 479}, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511432641.056", "dur": 1.46, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 476, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 478}, "ph": "X"}, {"name": "HostToDevice2057289334783", "ph": "f", "id": "2057289334783", "ts": "1704161511432641.056", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432641.056", "dur": 1.46, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432642.516", "dur": 12.48, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432643.404", "dur": 5.370053646835932, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 477}, "ph": "X"}, {"name": "HostToDevice2057289334783", "ph": "s", "cat": "HostToDevice", "id": "2057289334783", "pid": 43767502, "tid": 439084, "ts": "1704161511432650.344"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511432654.996", "dur": 1.49, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 477, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 480}, "ph": "X"}, {"name": "HostToDevice2065879269375", "ph": "f", "id": "2065879269375", "ts": "1704161511432654.996", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432654.996", "dur": 1.49, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432656.486", "dur": 26.4495, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511432669.504", "dur": 6.620066134460683, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 481}, "ph": "X"}, {"name": "HostToDevice2065879269375", "ph": "s", "cat": "HostToDevice", "id": "2065879269375", "pid": 43767502, "tid": 439084, "ts": "1704161511432669.944"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511432679.574", "dur": 27.85027822427946, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 484}, "ph": "X"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511432682.936", "dur": 3.47, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 478, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 483}, "ph": "X"}, {"name": "HostToDevice2078764171263", "ph": "f", "id": "2078764171263", "ts": "1704161511432682.936", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432682.936", "dur": 3.47, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432684.574", "dur": 5.740057343172858, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 482}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432686.406", "dur": 13.41, "ph": "X"}, {"name": "HostToDevice2078764171263", "ph": "s", "cat": "HostToDevice", "id": "2078764171263", "pid": 43767502, "tid": 439084, "ts": "1704161511432692.384"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432699.816", "dur": 2.96, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 479, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 485}, "ph": "X"}, {"name": "HostToDevice2087354105855", "ph": "f", "id": "2087354105855", "ts": "1704161511432699.816", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432699.816", "dur": 2.96, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432702.776", "dur": 12.42, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432712.334", "dur": 6.9900698307976095, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 486}, "ph": "X"}, {"name": "HostToDevice2087354105855", "ph": "s", "cat": "HostToDevice", "id": "2087354105855", "pid": 43767502, "tid": 439084, "ts": "1704161511432713.154"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511432715.196", "dur": 2.95, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 480, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 487}, "ph": "X"}, {"name": "HostToDevice2095944040447", "ph": "f", "id": "2095944040447", "ts": "1704161511432715.196", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432715.196", "dur": 2.95, "ph": "X"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432718.256", "dur": 3.49, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 481, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 489}, "ph": "X"}, {"name": "HostToDevice2104533975039", "ph": "f", "id": "2104533975039", "ts": "1704161511432718.256", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432718.256", "dur": 3.49, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432718.146", "dur": 0.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432721.746", "dur": 12.2195, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511432722.354", "dur": 6.6800667338666715, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 488}, "ph": "X"}, {"name": "HostToDevice2095944040447", "ph": "s", "cat": "HostToDevice", "id": "2095944040447", "pid": 43767502, "tid": 439084, "ts": "1704161511432722.884"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432732.094", "dur": 5.73005724327186, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 490}, "ph": "X"}, {"name": "HostToDevice2104533975039", "ph": "s", "cat": "HostToDevice", "id": "2104533975039", "pid": 43767502, "tid": 439084, "ts": "1704161511432732.555"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511432733.965", "dur": 6.45, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 482, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 491}, "ph": "X"}, {"name": "HostToDevice2113123909631", "ph": "f", "id": "2113123909631", "ts": "1704161511432733.965", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432733.965", "dur": 6.45, "ph": "X"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511432740.525", "dur": 2.91, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 483, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 493}, "ph": "X"}, {"name": "HostToDevice2121713844223", "ph": "f", "id": "2121713844223", "ts": "1704161511432740.525", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432740.525", "dur": 2.91, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432740.415", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511432741.285", "dur": 6.160061539014775, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 492}, "ph": "X"}, {"name": "HostToDevice2113123909631", "ph": "s", "cat": "HostToDevice", "id": "2113123909631", "pid": 43767502, "tid": 439084, "ts": "1704161511432741.885"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432743.435", "dur": 24.18, "ph": "X"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511432751.084", "dur": 5.450054446043916, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 494}, "ph": "X"}, {"name": "HostToDevice2121713844223", "ph": "s", "cat": "HostToDevice", "id": "2121713844223", "pid": 43767502, "tid": 439084, "ts": "1704161511432751.435"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511432759.735", "dur": 24.910248853386044, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 497}, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432763.755", "dur": 5.350053447033935, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 495}, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511432767.615", "dur": 3.67, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 484, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 496}, "ph": "X"}, {"name": "HostToDevice2134598746111", "ph": "f", "id": "2134598746111", "ts": "1704161511432767.615", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432767.615", "dur": 3.67, "ph": "X"}, {"name": "HostToDevice2134598746111", "ph": "s", "cat": "HostToDevice", "id": "2134598746111", "pid": 43767502, "tid": 439084, "ts": "1704161511432770.755"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432771.285", "dur": 5.9795, "ph": "X"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511432777.264", "dur": 3.59, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 485, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 498}, "ph": "X"}, {"name": "HostToDevice2143188680703", "ph": "f", "id": "2143188680703", "ts": "1704161511432777.264", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432777.264", "dur": 3.59, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432780.854", "dur": 16.23, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511432789.215", "dur": 7.44007432634252, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 499}, "ph": "X"}, {"name": "HostToDevice2143188680703", "ph": "s", "cat": "HostToDevice", "id": "2143188680703", "pid": 43767502, "tid": 439084, "ts": "1704161511432789.805"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511432797.084", "dur": 5.41, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 486, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 501}, "ph": "X"}, {"name": "HostToDevice2156073582591", "ph": "f", "id": "2156073582591", "ts": "1704161511432797.084", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432797.084", "dur": 5.41, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511432800.145", "dur": 27.330273029427566, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 502}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432802.494", "dur": 14.19, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432804.605", "dur": 5.710057043469864, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 500}, "ph": "X"}, {"name": "HostToDevice2156073582591", "ph": "s", "cat": "HostToDevice", "id": "2156073582591", "pid": 43767502, "tid": 439084, "ts": "1704161511432812.045"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432816.684", "dur": 1.43, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 487, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 503}, "ph": "X"}, {"name": "HostToDevice2164663517183", "ph": "f", "id": "2164663517183", "ts": "1704161511432816.684", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432816.684", "dur": 1.43, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432818.114", "dur": 18.0095, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432832.396", "dur": 7.180071728916571, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 504}, "ph": "X"}, {"name": "HostToDevice2164663517183", "ph": "s", "cat": "HostToDevice", "id": "2164663517183", "pid": 43767502, "tid": 439084, "ts": "1704161511432832.706"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511432836.124", "dur": 1.43, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 488, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 505}, "ph": "X"}, {"name": "HostToDevice2173253451775", "ph": "f", "id": "2173253451775", "ts": "1704161511432836.124", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432836.124", "dur": 1.43, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432837.554", "dur": 0.12, "ph": "X"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432837.674", "dur": 1.34, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 489, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 507}, "ph": "X"}, {"name": "HostToDevice2181843386367", "ph": "f", "id": "2181843386367", "ts": "1704161511432837.674", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432837.674", "dur": 1.34, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432839.014", "dur": 11.15, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511432842.866", "dur": 6.220062138420763, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 506}, "ph": "X"}, {"name": "HostToDevice2173253451775", "ph": "s", "cat": "HostToDevice", "id": "2173253451775", "pid": 43767502, "tid": 439084, "ts": "1704161511432843.206"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511432850.164", "dur": 3.55, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 490, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 509}, "ph": "X"}, {"name": "HostToDevice2190433320959", "ph": "f", "id": "2190433320959", "ts": "1704161511432850.164", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432850.164", "dur": 3.55, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432852.536", "dur": 5.660056543964875, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 508}, "ph": "X"}, {"name": "HostToDevice2181843386367", "ph": "s", "cat": "HostToDevice", "id": "2181843386367", "pid": 43767502, "tid": 439084, "ts": "1704161511432852.966"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432853.714", "dur": 13.2, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511432861.446", "dur": 6.170061638915772, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 510}, "ph": "X"}, {"name": "HostToDevice2190433320959", "ph": "s", "cat": "HostToDevice", "id": "2190433320959", "pid": 43767502, "tid": 439084, "ts": "1704161511432861.956"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511432866.914", "dur": 1.33, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 491, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 511}, "ph": "X"}, {"name": "HostToDevice2199023255551", "ph": "f", "id": "2199023255551", "ts": "1704161511432866.914", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432866.914", "dur": 1.33, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432868.244", "dur": 26.0795, "ph": "X"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511432871.026", "dur": 17.930179122489434, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 512}, "ph": "X"}, {"name": "HostToDevice2199023255551", "ph": "s", "cat": "HostToDevice", "id": "2199023255551", "pid": 43767502, "tid": 439084, "ts": "1704161511432871.696"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511432892.606", "dur": 26.270262439921776, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 515}, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511432894.324", "dur": 1.51, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 492, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 514}, "ph": "X"}, {"name": "HostToDevice2211908157439", "ph": "f", "id": "2211908157439", "ts": "1704161511432894.324", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432894.324", "dur": 1.51, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432895.834", "dur": 15.04, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432896.976", "dur": 5.440054346142918, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 513}, "ph": "X"}, {"name": "HostToDevice2211908157439", "ph": "s", "cat": "HostToDevice", "id": "2211908157439", "pid": 43767502, "tid": 439084, "ts": "1704161511432904.086"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511432910.874", "dur": 1.39, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 493, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 516}, "ph": "X"}, {"name": "HostToDevice2220498092031", "ph": "f", "id": "2220498092031", "ts": "1704161511432910.874", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432910.874", "dur": 1.39, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432912.264", "dur": 24.0095, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511432923.727", "dur": 6.660066534064675, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 517}, "ph": "X"}, {"name": "HostToDevice2220498092031", "ph": "s", "cat": "HostToDevice", "id": "2220498092031", "pid": 43767502, "tid": 439084, "ts": "1704161511432924.376"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511432934.727", "dur": 26.710266835565687, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 520}, "ph": "X"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511432936.273", "dur": 3.4, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 494, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 519}, "ph": "X"}, {"name": "HostToDevice2233382993919", "ph": "f", "id": "2233382993919", "ts": "1704161511432936.273", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432936.273", "dur": 3.4, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511432939.436", "dur": 5.250052448023956, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 518}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432939.673", "dur": 11.91, "ph": "X"}, {"name": "HostToDevice2233382993919", "ph": "s", "cat": "HostToDevice", "id": "2233382993919", "pid": 43767502, "tid": 439084, "ts": "1704161511432946.407"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432951.583", "dur": 6.22, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 495, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 521}, "ph": "X"}, {"name": "HostToDevice2241972928511", "ph": "f", "id": "2241972928511", "ts": "1704161511432951.583", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432951.583", "dur": 6.22, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432957.803", "dur": 7.15, "ph": "X"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511432964.953", "dur": 5.26, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 496, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 523}, "ph": "X"}, {"name": "HostToDevice2250562863103", "ph": "f", "id": "2250562863103", "ts": "1704161511432964.953", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432964.953", "dur": 5.26, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432965.897", "dur": 6.8000679326786475, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 522}, "ph": "X"}, {"name": "HostToDevice2241972928511", "ph": "s", "cat": "HostToDevice", "id": "2241972928511", "pid": 43767502, "tid": 439084, "ts": "1704161511432966.387"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511432970.373", "dur": 5.94, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 497, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 525}, "ph": "X"}, {"name": "HostToDevice2259152797695", "ph": "f", "id": "2259152797695", "ts": "1704161511432970.373", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432970.373", "dur": 5.94, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432970.213", "dur": 0.16, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511432975.617", "dur": 5.470054645845912, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 524}, "ph": "X"}, {"name": "HostToDevice2250562863103", "ph": "s", "cat": "HostToDevice", "id": "2250562863103", "pid": 43767502, "tid": 439084, "ts": "1704161511432976.057"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432976.313", "dur": 7.7395, "ph": "X"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511432984.052", "dur": 10.05, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 498, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 527}, "ph": "X"}, {"name": "HostToDevice2267742732287", "ph": "f", "id": "2267742732287", "ts": "1704161511432984.052", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432984.052", "dur": 10.05, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511432985.217", "dur": 5.790057842677848, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 526}, "ph": "X"}, {"name": "HostToDevice2259152797695", "ph": "s", "cat": "HostToDevice", "id": "2259152797695", "pid": 43767502, "tid": 439084, "ts": "1704161511432985.647"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511432993.937", "dur": 6.450064436143718, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 528}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432994.102", "dur": 0.11, "ph": "X"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511432994.212", "dur": 5.24, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 499, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 529}, "ph": "X"}, {"name": "HostToDevice2276332666879", "ph": "f", "id": "2276332666879", "ts": "1704161511432994.212", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511432994.212", "dur": 5.24, "ph": "X"}, {"name": "HostToDevice2267742732287", "ph": "s", "cat": "HostToDevice", "id": "2267742732287", "pid": 43767502, "tid": 439084, "ts": "1704161511432994.787"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511432999.452", "dur": 12.54, "ph": "X"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511433003.517", "dur": 5.400053946538926, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 530}, "ph": "X"}, {"name": "HostToDevice2276332666879", "ph": "s", "cat": "HostToDevice", "id": "2276332666879", "pid": 43767502, "tid": 439084, "ts": "1704161511433003.857"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511433011.747", "dur": 23.72023696516728, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 533}, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511433011.992", "dur": 6.86, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 500, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 532}, "ph": "X"}, {"name": "HostToDevice2289217568767", "ph": "f", "id": "2289217568767", "ts": "1704161511433011.992", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433011.992", "dur": 6.86, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433015.637", "dur": 4.790047852578047, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 531}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433018.852", "dur": 6.4, "ph": "X"}, {"name": "HostToDevice2289217568767", "ph": "s", "cat": "HostToDevice", "id": "2289217568767", "pid": 43767502, "tid": 439084, "ts": "1704161511433021.998"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511433025.252", "dur": 6.5495, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 501, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 534}, "ph": "X"}, {"name": "HostToDevice2297807503359", "ph": "f", "id": "2297807503359", "ts": "1704161511433025.252", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433025.252", "dur": 6.5495, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433031.802", "dur": 13.27, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511433039.978", "dur": 6.550065435153697, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 535}, "ph": "X"}, {"name": "HostToDevice2297807503359", "ph": "s", "cat": "HostToDevice", "id": "2297807503359", "pid": 43767502, "tid": 439084, "ts": "1704161511433040.627"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511433045.072", "dur": 8.44, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 502, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 537}, "ph": "X"}, {"name": "HostToDevice2310692405247", "ph": "f", "id": "2310692405247", "ts": "1704161511433045.072", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433045.072", "dur": 8.44, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511433050.428", "dur": 26.6502662361597, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 538}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433053.512", "dur": 26.7895, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433054.728", "dur": 5.260052547924953, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 536}, "ph": "X"}, {"name": "HostToDevice2310692405247", "ph": "s", "cat": "HostToDevice", "id": "2310692405247", "pid": 43767502, "tid": 439084, "ts": "1704161511433061.928"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433080.302", "dur": 1.6, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 503, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 539}, "ph": "X"}, {"name": "HostToDevice2319282339839", "ph": "f", "id": "2319282339839", "ts": "1704161511433080.302", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433080.302", "dur": 1.6, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433081.688", "dur": 6.530065235351701, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 540}, "ph": "X"}, {"name": "HostToDevice2319282339839", "ph": "s", "cat": "HostToDevice", "id": "2319282339839", "pid": 43767502, "tid": 439084, "ts": "1704161511433082.108"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433081.902", "dur": 11.03, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511433091.558", "dur": 5.610056044459884, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 542}, "ph": "X"}, {"name": "HostToDevice2327872274431", "ph": "s", "cat": "HostToDevice", "id": "2327872274431", "pid": 43767502, "tid": 439084, "ts": "1704161511433092.128"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511433092.932", "dur": 1.59, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 504, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 541}, "ph": "X"}, {"name": "HostToDevice2327872274431", "ph": "f", "id": "2327872274431", "ts": "1704161511433092.932", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433092.932", "dur": 1.59, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433094.522", "dur": 0.12, "ph": "X"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433094.642", "dur": 1.57, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 505, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 543}, "ph": "X"}, {"name": "HostToDevice2336462209023", "ph": "f", "id": "2336462209023", "ts": "1704161511433094.642", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433094.642", "dur": 1.57, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433096.212", "dur": 25.53, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433101.018", "dur": 5.590055844657888, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 544}, "ph": "X"}, {"name": "HostToDevice2336462209023", "ph": "s", "cat": "HostToDevice", "id": "2336462209023", "pid": 43767502, "tid": 439084, "ts": "1704161511433101.498"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511433110.648", "dur": 6.0800607398067905, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 546}, "ph": "X"}, {"name": "HostToDevice2345052143615", "ph": "s", "cat": "HostToDevice", "id": "2345052143615", "pid": 43767502, "tid": 439084, "ts": "1704161511433111.138"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511433119.938", "dur": 5.030050250202, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 548}, "ph": "X"}, {"name": "HostToDevice2353642078207", "ph": "s", "cat": "HostToDevice", "id": "2353642078207", "pid": 43767502, "tid": 439084, "ts": "1704161511433120.438"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511433121.742", "dur": 4.02, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 506, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 545}, "ph": "X"}, {"name": "HostToDevice2345052143615", "ph": "f", "id": "2345052143615", "ts": "1704161511433121.742", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433121.742", "dur": 4.02, "ph": "X"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511433125.872", "dur": 1.58, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 507, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 547}, "ph": "X"}, {"name": "HostToDevice2353642078207", "ph": "f", "id": "2353642078207", "ts": "1704161511433125.872", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433125.872", "dur": 1.58, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433125.762", "dur": 0.11, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433127.452", "dur": 5.2195, "ph": "X"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511433127.738", "dur": 23.170231470612393, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 551}, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433131.579", "dur": 4.780047752677049, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 549}, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511433132.671", "dur": 1.64, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 508, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 550}, "ph": "X"}, {"name": "HostToDevice2366526980095", "ph": "f", "id": "2366526980095", "ts": "1704161511433132.671", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433132.671", "dur": 1.64, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433134.311", "dur": 5.06, "ph": "X"}, {"name": "HostToDevice2366526980095", "ph": "s", "cat": "HostToDevice", "id": "2366526980095", "pid": 43767502, "tid": 439084, "ts": "1704161511433138.008"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511433139.371", "dur": 1.58, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 509, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 552}, "ph": "X"}, {"name": "HostToDevice2375116914687", "ph": "f", "id": "2375116914687", "ts": "1704161511433139.371", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433139.371", "dur": 1.58, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433140.951", "dur": 20.81, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511433155.559", "dur": 6.64006633426268, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 553}, "ph": "X"}, {"name": "HostToDevice2375116914687", "ph": "s", "cat": "HostToDevice", "id": "2375116914687", "pid": 43767502, "tid": 439084, "ts": "1704161511433155.998"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511433161.761", "dur": 3.94, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 510, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 555}, "ph": "X"}, {"name": "HostToDevice2388001816575", "ph": "f", "id": "2388001816575", "ts": "1704161511433161.761", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433161.761", "dur": 3.94, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433165.701", "dur": 18.2895, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511433166.169", "dur": 26.35026323912976, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 556}, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433170.589", "dur": 5.200051948518966, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 554}, "ph": "X"}, {"name": "HostToDevice2388001816575", "ph": "s", "cat": "HostToDevice", "id": "2388001816575", "pid": 43767502, "tid": 439084, "ts": "1704161511433177.299"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433183.990", "dur": 2.48, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 511, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 557}, "ph": "X"}, {"name": "HostToDevice2396591751167", "ph": "f", "id": "2396591751167", "ts": "1704161511433183.990", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433183.990", "dur": 2.48, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433186.470", "dur": 34.51, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433197.159", "dur": 12.300122878227555, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 558}, "ph": "X"}, {"name": "HostToDevice2396591751167", "ph": "s", "cat": "HostToDevice", "id": "2396591751167", "pid": 43767502, "tid": 439084, "ts": "1704161511433197.529"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511433213.029", "dur": 7.19007182881757, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 560}, "ph": "X"}, {"name": "HostToDevice2405181685759", "ph": "s", "cat": "HostToDevice", "id": "2405181685759", "pid": 43767502, "tid": 439084, "ts": "1704161511433213.509"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511433220.980", "dur": 2.53, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 513, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 559}, "ph": "X"}, {"name": "HostToDevice2405181685759", "ph": "f", "id": "2405181685759", "ts": "1704161511433220.980", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433220.980", "dur": 2.53, "ph": "X"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433223.620", "dur": 2.87, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 514, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 561}, "ph": "X"}, {"name": "HostToDevice2413771620351", "ph": "f", "id": "2413771620351", "ts": "1704161511433223.620", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433223.620", "dur": 2.87, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433223.510", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433223.799", "dur": 5.790057842677848, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 562}, "ph": "X"}, {"name": "HostToDevice2413771620351", "ph": "s", "cat": "HostToDevice", "id": "2413771620351", "pid": 43767502, "tid": 439084, "ts": "1704161511433224.460"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433226.490", "dur": 7.2395, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511433233.049", "dur": 6.2000619386187665, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 564}, "ph": "X"}, {"name": "HostToDevice2422361554943", "ph": "s", "cat": "HostToDevice", "id": "2422361554943", "pid": 43767502, "tid": 439084, "ts": "1704161511433233.549"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511433233.730", "dur": 5.71, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 515, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 563}, "ph": "X"}, {"name": "HostToDevice2422361554943", "ph": "f", "id": "2422361554943", "ts": "1704161511433233.730", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433233.730", "dur": 5.71, "ph": "X"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511433239.550", "dur": 2.61, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 516, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 565}, "ph": "X"}, {"name": "HostToDevice2430951489535", "ph": "f", "id": "2430951489535", "ts": "1704161511433239.550", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433239.550", "dur": 2.61, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433239.440", "dur": 0.11, "ph": "X"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511433242.339", "dur": 5.18005174871697, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 566}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433242.160", "dur": 5.95, "ph": "X"}, {"name": "HostToDevice2430951489535", "ph": "s", "cat": "HostToDevice", "id": "2430951489535", "pid": 43767502, "tid": 439084, "ts": "1704161511433242.800"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511433248.110", "dur": 3.03, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 517, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 568}, "ph": "X"}, {"name": "HostToDevice2443836391423", "ph": "f", "id": "2443836391423", "ts": "1704161511433248.110", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433248.110", "dur": 3.03, "ph": "X"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511433250.669", "dur": 25.30025274952497, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 569}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433251.140", "dur": 12.71, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433254.920", "dur": 5.510055045449905, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 567}, "ph": "X"}, {"name": "HostToDevice2443836391423", "ph": "s", "cat": "HostToDevice", "id": "2443836391423", "pid": 43767502, "tid": 439084, "ts": "1704161511433262.100"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511433263.850", "dur": 2.88, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 518, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 570}, "ph": "X"}, {"name": "HostToDevice2452426326015", "ph": "f", "id": "2452426326015", "ts": "1704161511433263.850", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433263.850", "dur": 2.88, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433266.730", "dur": 25.3095, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511433280.750", "dur": 6.480064735846711, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 571}, "ph": "X"}, {"name": "HostToDevice2452426326015", "ph": "s", "cat": "HostToDevice", "id": "2452426326015", "pid": 43767502, "tid": 439084, "ts": "1704161511433281.280"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511433290.490", "dur": 26.680266535862692, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 574}, "ph": "X"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511433292.040", "dur": 5.13, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 519, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 573}, "ph": "X"}, {"name": "HostToDevice2465311227903", "ph": "f", "id": "2465311227903", "ts": "1704161511433292.040", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433292.040", "dur": 5.13, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433294.770", "dur": 5.310053047429943, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 572}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433297.170", "dur": 13.21, "ph": "X"}, {"name": "HostToDevice2465311227903", "ph": "s", "cat": "HostToDevice", "id": "2465311227903", "pid": 43767502, "tid": 439084, "ts": "1704161511433301.850"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433310.380", "dur": 1.42, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 520, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 575}, "ph": "X"}, {"name": "HostToDevice2473901162495", "ph": "f", "id": "2473901162495", "ts": "1704161511433310.380", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433310.380", "dur": 1.42, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433311.800", "dur": 12.84, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433321.660", "dur": 6.550065435153697, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 576}, "ph": "X"}, {"name": "HostToDevice2473901162495", "ph": "s", "cat": "HostToDevice", "id": "2473901162495", "pid": 43767502, "tid": 439084, "ts": "1704161511433322.130"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511433324.640", "dur": 1.48, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 521, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 577}, "ph": "X"}, {"name": "HostToDevice2482491097087", "ph": "f", "id": "2482491097087", "ts": "1704161511433324.640", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433324.640", "dur": 1.48, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433326.120", "dur": 0.11, "ph": "X"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433326.230", "dur": 1.39, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 522, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 579}, "ph": "X"}, {"name": "HostToDevice2491081031679", "ph": "f", "id": "2491081031679", "ts": "1704161511433326.230", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433326.230", "dur": 1.39, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433327.620", "dur": 7.3895, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511433331.750", "dur": 6.0800607398067905, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 578}, "ph": "X"}, {"name": "HostToDevice2482491097087", "ph": "s", "cat": "HostToDevice", "id": "2482491097087", "pid": 43767502, "tid": 439084, "ts": "1704161511433332.371"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511433335.009", "dur": 3.65, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 523, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 581}, "ph": "X"}, {"name": "HostToDevice2499670966271", "ph": "f", "id": "2499670966271", "ts": "1704161511433335.009", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433335.009", "dur": 3.65, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433338.659", "dur": 13.56, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433341.101", "dur": 5.73005724327186, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 580}, "ph": "X"}, {"name": "HostToDevice2491081031679", "ph": "s", "cat": "HostToDevice", "id": "2491081031679", "pid": 43767502, "tid": 439084, "ts": "1704161511433341.721"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511433349.980", "dur": 6.190061838717769, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 582}, "ph": "X"}, {"name": "HostToDevice2499670966271", "ph": "s", "cat": "HostToDevice", "id": "2499670966271", "pid": 43767502, "tid": 439084, "ts": "1704161511433350.871"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511433352.219", "dur": 1.41, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 524, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 583}, "ph": "X"}, {"name": "HostToDevice2508260900863", "ph": "f", "id": "2508260900863", "ts": "1704161511433352.219", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433352.219", "dur": 1.41, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433353.629", "dur": 8.66, "ph": "X"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511433359.201", "dur": 5.150051449013976, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 584}, "ph": "X"}, {"name": "HostToDevice2508260900863", "ph": "s", "cat": "HostToDevice", "id": "2508260900863", "pid": 43767502, "tid": 439084, "ts": "1704161511433359.661"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511433362.289", "dur": 1.48, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 525, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 586}, "ph": "X"}, {"name": "HostToDevice2521145802751", "ph": "f", "id": "2521145802751", "ts": "1704161511433362.289", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433362.289", "dur": 1.48, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433363.769", "dur": 20.7295, "ph": "X"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511433367.111", "dur": 23.68023656556329, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 587}, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433371.031", "dur": 4.770047652776051, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 585}, "ph": "X"}, {"name": "HostToDevice2521145802751", "ph": "s", "cat": "HostToDevice", "id": "2521145802751", "pid": 43767502, "tid": 439084, "ts": "1704161511433377.291"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511433384.498", "dur": 1.38, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 526, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 588}, "ph": "X"}, {"name": "HostToDevice2529735737343", "ph": "f", "id": "2529735737343", "ts": "1704161511433384.498", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433384.498", "dur": 1.38, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433385.878", "dur": 16.82, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511433395.881", "dur": 6.410064036539725, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 589}, "ph": "X"}, {"name": "HostToDevice2529735737343", "ph": "s", "cat": "HostToDevice", "id": "2529735737343", "pid": 43767502, "tid": 439084, "ts": "1704161511433396.381"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511433402.698", "dur": 3.45, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 527, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 591}, "ph": "X"}, {"name": "HostToDevice2542620639231", "ph": "f", "id": "2542620639231", "ts": "1704161511433402.698", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433402.698", "dur": 3.45, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511433405.991", "dur": 26.300262739624767, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 592}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433406.148", "dur": 15.76, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433410.411", "dur": 5.4300542462419195, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 590}, "ph": "X"}, {"name": "HostToDevice2542620639231", "ph": "s", "cat": "HostToDevice", "id": "2542620639231", "pid": 43767502, "tid": 439084, "ts": "1704161511433417.361"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433421.908", "dur": 1.53, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 528, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 593}, "ph": "X"}, {"name": "HostToDevice2551210573823", "ph": "f", "id": "2551210573823", "ts": "1704161511433421.908", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433421.908", "dur": 1.53, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433423.438", "dur": 17.9795, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433437.102", "dur": 6.4900648357477095, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 594}, "ph": "X"}, {"name": "HostToDevice2551210573823", "ph": "s", "cat": "HostToDevice", "id": "2551210573823", "pid": 43767502, "tid": 439084, "ts": "1704161511433437.532"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511433441.418", "dur": 1.55, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 529, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 595}, "ph": "X"}, {"name": "HostToDevice2559800508415", "ph": "f", "id": "2559800508415", "ts": "1704161511433441.418", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433441.418", "dur": 1.55, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433442.968", "dur": 0.28, "ph": "X"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433443.248", "dur": 1.53, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 530, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 597}, "ph": "X"}, {"name": "HostToDevice2568390443007", "ph": "f", "id": "2568390443007", "ts": "1704161511433443.248", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433443.248", "dur": 1.53, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433444.778", "dur": 9.63, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511433447.702", "dur": 5.820058142380843, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 596}, "ph": "X"}, {"name": "HostToDevice2559800508415", "ph": "s", "cat": "HostToDevice", "id": "2559800508415", "pid": 43767502, "tid": 439084, "ts": "1704161511433448.242"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511433454.408", "dur": 3.92, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 531, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 599}, "ph": "X"}, {"name": "HostToDevice2576980377599", "ph": "f", "id": "2576980377599", "ts": "1704161511433454.408", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433454.408", "dur": 3.92, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433456.632", "dur": 5.160051548914973, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 598}, "ph": "X"}, {"name": "HostToDevice2568390443007", "ph": "s", "cat": "HostToDevice", "id": "2568390443007", "pid": 43767502, "tid": 439084, "ts": "1704161511433456.972"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433458.328", "dur": 7.01, "ph": "X"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511433465.338", "dur": 1.63, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 532, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 601}, "ph": "X"}, {"name": "HostToDevice2585570312191", "ph": "f", "id": "2585570312191", "ts": "1704161511433465.338", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433465.338", "dur": 1.63, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433466.968", "dur": 15.7595, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511433469.622", "dur": 6.590065834757689, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 600}, "ph": "X"}, {"name": "HostToDevice2576980377599", "ph": "s", "cat": "HostToDevice", "id": "2576980377599", "pid": 43767502, "tid": 439084, "ts": "1704161511433470.042"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511433479.902", "dur": 5.390053846637928, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 602}, "ph": "X"}, {"name": "HostToDevice2585570312191", "ph": "s", "cat": "HostToDevice", "id": "2585570312191", "pid": 43767502, "tid": 439084, "ts": "1704161511433480.702"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511433482.728", "dur": 1.77, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 533, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 604}, "ph": "X"}, {"name": "HostToDevice2598455214079", "ph": "f", "id": "2598455214079", "ts": "1704161511433482.728", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433482.728", "dur": 1.77, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433484.498", "dur": 20.93, "ph": "X"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511433488.382", "dur": 25.320252949326964, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 605}, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433492.322", "dur": 5.030050250202, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 603}, "ph": "X"}, {"name": "HostToDevice2598455214079", "ph": "s", "cat": "HostToDevice", "id": "2598455214079", "pid": 43767502, "tid": 439084, "ts": "1704161511433498.942"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511433505.428", "dur": 1.53, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 534, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 606}, "ph": "X"}, {"name": "HostToDevice2607045148671", "ph": "f", "id": "2607045148671", "ts": "1704161511433505.428", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433505.428", "dur": 1.53, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433506.958", "dur": 18.09, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511433518.652", "dur": 6.220062138420763, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 607}, "ph": "X"}, {"name": "HostToDevice2607045148671", "ph": "s", "cat": "HostToDevice", "id": "2607045148671", "pid": 43767502, "tid": 439084, "ts": "1704161511433519.022"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511433525.048", "dur": 3.7595, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 535, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 609}, "ph": "X"}, {"name": "HostToDevice2619930050559", "ph": "f", "id": "2619930050559", "ts": "1704161511433525.048", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433525.048", "dur": 3.7595, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511433528.692", "dur": 26.700266735664687, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 610}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433528.807", "dur": 17.73, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433533.072", "dur": 5.33005324723194, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 608}, "ph": "X"}, {"name": "HostToDevice2619930050559", "ph": "s", "cat": "HostToDevice", "id": "2619930050559", "pid": 43767502, "tid": 439084, "ts": "1704161511433539.973"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433546.537", "dur": 1.44, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 536, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 611}, "ph": "X"}, {"name": "HostToDevice2628519985151", "ph": "f", "id": "2628519985151", "ts": "1704161511433546.537", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433546.537", "dur": 1.44, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433547.977", "dur": 6.52, "ph": "X"}, {"name": "aclnnAdd_AxpyAiCore_Axpy", "pid": 800, "tid": 3, "ts": "1704161511433554.497", "dur": 1.49, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 537, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 613}, "ph": "X"}, {"name": "HostToDevice2637109919743", "ph": "f", "id": "2637109919743", "ts": "1704161511433554.497", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433554.497", "dur": 1.49, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433555.987", "dur": 8.98, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433560.413", "dur": 6.540065335252699, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 612}, "ph": "X"}, {"name": "HostToDevice2628519985151", "ph": "s", "cat": "HostToDevice", "id": "2628519985151", "pid": 43767502, "tid": 439084, "ts": "1704161511433560.883"}, {"name": "aclnnInplaceMuls_MulAiCore_Mul", "pid": 800, "tid": 3, "ts": "1704161511433564.967", "dur": 1.45, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 538, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 615}, "ph": "X"}, {"name": "HostToDevice2645699854335", "ph": "f", "id": "2645699854335", "ts": "1704161511433564.967", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433564.967", "dur": 1.45, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433566.417", "dur": 7.8, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511433570.623", "dur": 5.700056943568867, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 614}, "ph": "X"}, {"name": "HostToDevice2637109919743", "ph": "s", "cat": "HostToDevice", "id": "2637109919743", "pid": 43767502, "tid": 439084, "ts": "1704161511433571.063"}, {"name": "aclnnAddcmul_AddcmulAiCore_Addcmul", "pid": 800, "tid": 3, "ts": "1704161511433574.217", "dur": 3.62, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 539, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 617}, "ph": "X"}, {"name": "HostToDevice2654289788927", "ph": "f", "id": "2654289788927", "ts": "1704161511433574.217", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433574.217", "dur": 3.62, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433577.837", "dur": 4.8895, "ph": "X"}, {"name": "AscendCL@aclnnInplaceMuls", "pid": 43767502, "tid": 439084, "ts": "1704161511433579.363", "dur": 5.040050350102997, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceMuls", "item_id": "0", "connection_id": 616}, "ph": "X"}, {"name": "HostToDevice2645699854335", "ph": "s", "cat": "HostToDevice", "id": "2645699854335", "pid": 43767502, "tid": 439084, "ts": "1704161511433579.873"}, {"name": "aclnnSqrt_SqrtAiCore_Sqrt", "pid": 800, "tid": 3, "ts": "1704161511433582.726", "dur": 1.39, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 540, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 619}, "ph": "X"}, {"name": "HostToDevice2662879723519", "ph": "f", "id": "2662879723519", "ts": "1704161511433582.726", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433582.726", "dur": 1.39, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433584.116", "dur": 25.17, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAddcmul", "pid": 43767502, "tid": 439084, "ts": "1704161511433587.783", "dur": 5.4300542462419195, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcmul", "item_id": "0", "connection_id": 618}, "ph": "X"}, {"name": "HostToDevice2654289788927", "ph": "s", "cat": "HostToDevice", "id": "2654289788927", "pid": 43767502, "tid": 439084, "ts": "1704161511433588.083"}, {"name": "AscendCL@aclnnSqrt", "pid": 43767502, "tid": 439084, "ts": "1704161511433599.233", "dur": 5.2400523481229575, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnSqrt", "item_id": "0", "connection_id": 620}, "ph": "X"}, {"name": "HostToDevice2662879723519", "ph": "s", "cat": "HostToDevice", "id": "2662879723519", "pid": 43767502, "tid": 439084, "ts": "1704161511433599.563"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511433608.033", "dur": 26.100260741604806, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 623}, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511433609.286", "dur": 1.65, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 541, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 622}, "ph": "X"}, {"name": "HostToDevice2675764625407", "ph": "f", "id": "2675764625407", "ts": "1704161511433609.286", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433609.286", "dur": 1.65, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433610.936", "dur": 12.36, "ph": "X"}, {"name": "AscendCL@RealDiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433612.794", "dur": 4.750047452974055, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "RealDiv_Tiling", "item_id": "0", "connection_id": 621}, "ph": "X"}, {"name": "HostToDevice2675764625407", "ph": "s", "cat": "HostToDevice", "id": "2675764625407", "pid": 43767502, "tid": 439084, "ts": "1704161511433619.093"}, {"name": "aclnnAdds_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511433623.296", "dur": 1.45, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 542, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 624}, "ph": "X"}, {"name": "HostToDevice2684354559999", "ph": "f", "id": "2684354559999", "ts": "1704161511433623.296", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433623.296", "dur": 1.45, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433624.746", "dur": 25.1195, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdds", "pid": 43767502, "tid": 439084, "ts": "1704161511433639.284", "dur": 6.110061039509785, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdds", "item_id": "0", "connection_id": 625}, "ph": "X"}, {"name": "HostToDevice2684354559999", "ph": "s", "cat": "HostToDevice", "id": "2684354559999", "pid": 43767502, "tid": 439084, "ts": "1704161511433639.713"}, {"name": "AscendCL@aclnnInplaceAddcdiv", "pid": 43767502, "tid": 439084, "ts": "1704161511433648.634", "dur": 25.6402561461589, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAddcdiv", "item_id": "0", "connection_id": 628}, "ph": "X"}, {"name": "aclnnAddcdiv_AddcdivAiCore_Addcdiv", "pid": 800, "tid": 3, "ts": "1704161511433649.866", "dur": 3.43, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 543, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 627}, "ph": "X"}, {"name": "HostToDevice2697239461887", "ph": "f", "id": "2697239461887", "ts": "1704161511433649.866", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433649.866", "dur": 3.43, "ph": "X"}, {"name": "AscendCL@Addcdiv_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433652.894", "dur": 5.610056044459884, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Addcdiv_Tiling", "item_id": "0", "connection_id": 626}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433653.296", "dur": 12.88, "ph": "X"}, {"name": "HostToDevice2697239461887", "ph": "s", "cat": "HostToDevice", "id": "2697239461887", "pid": 43767502, "tid": 439084, "ts": "1704161511433660.104"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511433666.176", "dur": 1.46, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 544, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 629}, "ph": "X"}, {"name": "HostToDevice2705829396479", "ph": "f", "id": "2705829396479", "ts": "1704161511433666.176", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433666.176", "dur": 1.46, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433667.636", "dur": 19.1495, "ph": "X"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511433678.514", "dur": 6.220062138420763, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 630}, "ph": "X"}, {"name": "HostToDevice2705829396479", "ph": "s", "cat": "HostToDevice", "id": "2705829396479", "pid": 43767502, "tid": 439084, "ts": "1704161511433678.994"}, {"name": "aclnnArgMax_ArgMaxV2AiCore_ArgMaxV2", "pid": 800, "tid": 3, "ts": "1704161511433686.786", "dur": 4.7, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 545, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 631}, "ph": "X"}, {"name": "HostToDevice2714419331071", "ph": "f", "id": "2714419331071", "ts": "1704161511433686.786", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433686.786", "dur": 4.7, "ph": "X"}, {"name": "AscendCL@aclnnArgMax", "pid": 43767502, "tid": 439084, "ts": "1704161511433691.454", "dur": 10.030100200701005, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnArgMax", "item_id": "0", "connection_id": 633}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433691.486", "dur": 0.16, "ph": "X"}, {"name": "aclnnArgMax_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511433691.646", "dur": 5.99, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 546, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 632}, "ph": "X"}, {"name": "HostToDevice2718714298367", "ph": "f", "id": "2718714298367", "ts": "1704161511433691.646", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433691.646", "dur": 5.99, "ph": "X"}, {"name": "HostToDevice2714419331071", "ph": "s", "cat": "HostToDevice", "id": "2714419331071", "pid": 43767502, "tid": 439084, "ts": "1704161511433692.034"}, {"name": "HostToDevice2718714298367", "ph": "s", "cat": "HostToDevice", "id": "2718714298367", "pid": 43767502, "tid": 439084, "ts": "1704161511433697.304"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433697.636", "dur": 3.56, "ph": "X"}, {"name": "aclnnEqTensor_EqualAiCore_Equal", "pid": 800, "tid": 3, "ts": "1704161511433701.196", "dur": 3.42, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 547, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 634}, "ph": "X"}, {"name": "HostToDevice2727304232959", "ph": "f", "id": "2727304232959", "ts": "1704161511433701.196", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433701.196", "dur": 3.42, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433704.616", "dur": 20.98, "ph": "X"}, {"name": "AscendCL@aclnnEqTensor", "pid": 43767502, "tid": 439084, "ts": "1704161511433706.914", "dur": 6.6300662343616805, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnEqTensor", "item_id": "0", "connection_id": 635}, "ph": "X"}, {"name": "HostToDevice2727304232959", "ph": "s", "cat": "HostToDevice", "id": "2727304232959", "pid": 43767502, "tid": 439084, "ts": "1704161511433707.594"}, {"name": "AscendCL@aclnnReduceSum", "pid": 43767502, "tid": 439084, "ts": "1704161511433717.564", "dur": 117.74117623435058, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnReduceSum", "item_id": "0", "connection_id": 642}, "ph": "X"}, {"name": "aclnnReduceSum_CastAiCore_Cast", "pid": 800, "tid": 3, "ts": "1704161511433725.596", "dur": 1.63, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 548, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 637}, "ph": "X"}, {"name": "HostToDevice2740189134847", "ph": "f", "id": "2740189134847", "ts": "1704161511433725.596", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433725.596", "dur": 1.63, "ph": "X"}, {"name": "AscendCL@Cast_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433727.185", "dur": 5.300052947528946, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "Cast_Tiling", "item_id": "0", "connection_id": 636}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433727.226", "dur": 60.229, "ph": "X"}, {"name": "HostToDevice2740189134847", "ph": "s", "cat": "HostToDevice", "id": "2740189134847", "pid": 43767502, "tid": 439084, "ts": "1704161511433735.634"}, {"name": "AscendCL@ReduceSum_Tiling", "pid": 43767502, "tid": 439084, "ts": "1704161511433754.325", "dur": 19.35019330843115, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "ReduceSum_Tiling", "item_id": "0", "connection_id": 638}, "ph": "X"}, {"name": "aclnnReduceSum_ReduceSumOpAiCore_MemSet", "pid": 800, "tid": 3, "ts": "1704161511433787.454", "dur": 6.65, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 549, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 639}, "ph": "X"}, {"name": "HostToDevice2748779069439", "ph": "f", "id": "2748779069439", "ts": "1704161511433787.454", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433787.454", "dur": 6.65, "ph": "X"}, {"name": "HostToDevice2748779069439", "ph": "s", "cat": "HostToDevice", "id": "2748779069439", "pid": 43767502, "tid": 439084, "ts": "1704161511433790.805"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433794.104", "dur": 0.16, "ph": "X"}, {"name": "aclnnReduceSum_ReduceSumOpAiCore_ReduceSum", "pid": 800, "tid": 3, "ts": "1704161511433794.264", "dur": 2.17, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 550, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 640}, "ph": "X"}, {"name": "HostToDevice2753074036735", "ph": "f", "id": "2753074036735", "ts": "1704161511433794.264", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433794.264", "dur": 2.17, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511433796.434", "dur": 19.67, "ph": "X"}, {"name": "HostToDevice2753074036735", "ph": "s", "cat": "HostToDevice", "id": "2753074036735", "pid": 43767502, "tid": 439084, "ts": "1704161511433802.405"}, {"name": "HostToDevice2757369004031", "ph": "s", "cat": "HostToDevice", "id": "2757369004031", "pid": 43767502, "tid": 439084, "ts": "1704161511433815.135"}, {"name": "aclnnReduceSum_CastAiCpu_Cast", "pid": 800, "tid": 3, "ts": "1704161511433816.104", "dur": 188.14802169799805, "args": {"Model Id": 4294967295, "Task Type": "AI_CPU", "Stream Id": 3, "Task Id": 551, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 641}, "ph": "X"}, {"name": "HostToDevice2757369004031", "ph": "f", "id": "2757369004031", "ts": "1704161511433816.104", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511433816.104", "dur": 188.148, "ph": "X"}, {"name": "AscendCL@aclnnCast", "pid": 43767502, "tid": 439084, "ts": "1704161511433852.085", "dur": 11.62011608495969, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnCast", "item_id": "0", "connection_id": 644}, "ph": "X"}, {"name": "HostToDevice2765958938623", "ph": "s", "cat": "HostToDevice", "id": "2765958938623", "pid": 43767502, "tid": 439084, "ts": "1704161511433854.696"}, {"name": "AscendCL@aclnnInplaceAdd", "pid": 43767502, "tid": 439084, "ts": "1704161511433868.746", "dur": 7.150071429213577, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnInplaceAdd", "item_id": "0", "connection_id": 646}, "ph": "X"}, {"name": "HostToDevice2774548873215", "ph": "s", "cat": "HostToDevice", "id": "2774548873215", "pid": 43767502, "tid": 439084, "ts": "1704161511433869.086"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511433879.846", "dur": 5.7700576428758525, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 648}, "ph": "X"}, {"name": "HostToDevice2783138807807", "ph": "s", "cat": "HostToDevice", "id": "2783138807807", "pid": 43767502, "tid": 439084, "ts": "1704161511433880.276"}, {"name": "AscendCL@aclnnDivs", "pid": 43767502, "tid": 439084, "ts": "1704161511433888.746", "dur": 5.010050050400003, "args": {"Thread Id": 439084, "Mode": "ACL_NN", "level": "acl", "id": "aclnnDivs", "item_id": "0", "connection_id": 650}, "ph": "X"}, {"name": "HostToDevice2791728742399", "ph": "s", "cat": "HostToDevice", "id": "2791728742399", "pid": 43767502, "tid": 439084, "ts": "1704161511433888.986"}, {"name": "AscendCL@aclrtSynchronizeStreamWithTimeout", "pid": 43767502, "tid": 437675, "ts": "1704161511433911.036", "dur": 309.6830937341064, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtSynchronizeStreamWithTimeout", "item_id": "0", "connection_id": 651}, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511434004.252", "dur": 14.72, "ph": "X"}, {"name": "aclnnCast_CastAiCpu_Cast", "pid": 800, "tid": 3, "ts": "1704161511434018.972", "dur": 139.4186019897461, "args": {"Model Id": 4294967295, "Task Type": "AI_CPU", "Stream Id": 3, "Task Id": 552, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 643}, "ph": "X"}, {"name": "HostToDevice2765958938623", "ph": "f", "id": "2765958938623", "ts": "1704161511434018.972", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511434018.972", "dur": 139.4185, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511434158.391", "dur": 11.23, "ph": "X"}, {"name": "aclnnAdd_AddAiCore_Add", "pid": 800, "tid": 3, "ts": "1704161511434169.621", "dur": 1.4, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 553, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 645}, "ph": "X"}, {"name": "HostToDevice2774548873215", "ph": "f", "id": "2774548873215", "ts": "1704161511434169.621", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511434169.621", "dur": 1.4, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511434171.021", "dur": 0.16, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511434171.181", "dur": 1.34, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 554, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 647}, "ph": "X"}, {"name": "HostToDevice2783138807807", "ph": "f", "id": "2783138807807", "ts": "1704161511434171.181", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511434171.181", "dur": 1.34, "ph": "X"}, {"name": "Free", "pid": 4376751100, "tid": 3, "ts": "1704161511434172.521", "dur": 0.18, "ph": "X"}, {"name": "aclnnDivs_RealDivAiCore_RealDiv", "pid": 800, "tid": 3, "ts": "1704161511434172.701", "dur": 1.42, "args": {"Model Id": 4294967295, "Task Type": "Other", "Stream Id": 3, "Task Id": 555, "Batch Id": 0, "Subtask Id": 4294967295, "connection_id": 649}, "ph": "X"}, {"name": "HostToDevice2791728742399", "ph": "f", "id": "2791728742399", "ts": "1704161511434172.701", "cat": "HostToDevice", "pid": 800, "tid": 3, "bp": "e"}, {"name": "Computing", "pid": 4376751100, "tid": 0, "ts": "1704161511434172.701", "dur": 1.42, "ph": "X"}, {"name": "AscendCL@aclrtMemcpy", "pid": 43767502, "tid": 437675, "ts": "1704161511434222.419", "dur": 30.57030539735092, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtMemcpy", "item_id": "0", "connection_id": 652}, "ph": "X"}, {"name": "AscendCL@aclrtSynchronizeStreamWithTimeout", "pid": 43767502, "tid": 437675, "ts": "1704161511434273.420", "dur": 3.47003466564631, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtSynchronizeStreamWithTimeout", "item_id": "0", "connection_id": 653}, "ph": "X"}, {"name": "AscendCL@aclrtMemcpy", "pid": 43767502, "tid": 437675, "ts": "1704161511434277.340", "dur": 16.570165535953706, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtMemcpy", "item_id": "0", "connection_id": 654}, "ph": "X"}, {"name": "AscendCL@aclrtSynchronizeDevice", "pid": 43767502, "tid": 437675, "ts": "1704161511434416.011", "dur": 6.430064236341721, "args": {"Thread Id": 437675, "Mode": "ACL_RTS", "level": "acl", "id": "aclrtSynchronizeDevice", "item_id": "0", "connection_id": 655}, "ph": "X"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422904204, "pid": 437675, "tid": 437675, "ts": "1704161511422855.116", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422904204, "pid": 800, "tid": 3, "ts": "1704161511422904.204", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422917264, "pid": 437675, "tid": 437675, "ts": "1704161511422892.817", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422917264, "pid": 800, "tid": 3, "ts": "1704161511422917.264", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422950043, "pid": 437675, "tid": 437675, "ts": "1704161511422907.207", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422950043, "pid": 800, "tid": 3, "ts": "1704161511422950.043", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422952673, "pid": 437675, "tid": 437675, "ts": "1704161511422920.807", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422952673, "pid": 800, "tid": 3, "ts": "1704161511422952.673", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422954043, "pid": 437675, "tid": 437675, "ts": "1704161511422933.197", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422954043, "pid": 800, "tid": 3, "ts": "1704161511422954.043", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422962253, "pid": 437675, "tid": 437675, "ts": "1704161511422945.917", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422962253, "pid": 800, "tid": 3, "ts": "1704161511422962.253", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422968132, "pid": 437675, "tid": 437675, "ts": "1704161511422958.477", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422968132, "pid": 800, "tid": 3, "ts": "1704161511422968.132", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422983032, "pid": 437675, "tid": 437675, "ts": "1704161511422970.767", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422983032, "pid": 800, "tid": 3, "ts": "1704161511422983.032", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422984392, "pid": 437675, "tid": 437675, "ts": "1704161511422984.268", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422984392, "pid": 800, "tid": 3, "ts": "1704161511422984.392", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511422991032, "pid": 437675, "tid": 437675, "ts": "1704161511422996.918", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511422991032, "pid": 800, "tid": 3, "ts": "1704161511422991.032", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424085712, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424085712, "pid": 800, "tid": 3, "ts": "1704161511424085.712", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424137431, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424137431, "pid": 800, "tid": 3, "ts": "1704161511424137.431", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424159001, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424159001, "pid": 800, "tid": 3, "ts": "1704161511424159.001", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424197230, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424197230, "pid": 800, "tid": 3, "ts": "1704161511424197.230", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424215770, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424215770, "pid": 800, "tid": 3, "ts": "1704161511424215.770", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424223930, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424223930, "pid": 800, "tid": 3, "ts": "1704161511424223.930", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424257870, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424257870, "pid": 800, "tid": 3, "ts": "1704161511424257.870", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424286260, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424286260, "pid": 800, "tid": 3, "ts": "1704161511424286.260", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424294210, "pid": 437675, "tid": 437675, "ts": "1704161511423205.030", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424294210, "pid": 800, "tid": 3, "ts": "1704161511424294.210", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424333429, "pid": 437675, "tid": 437675, "ts": "1704161511423257.470", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424333429, "pid": 800, "tid": 3, "ts": "1704161511424333.429", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424837944, "pid": 437675, "tid": 437675, "ts": "1704161511423396.302", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424837944, "pid": 800, "tid": 3, "ts": "1704161511424837.944", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424871414, "pid": 437675, "tid": 437675, "ts": "1704161511423396.302", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424871414, "pid": 800, "tid": 3, "ts": "1704161511424871.414", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424907934, "pid": 437675, "tid": 437675, "ts": "1704161511423396.302", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424907934, "pid": 800, "tid": 3, "ts": "1704161511424907.934", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424929423, "pid": 437675, "tid": 437675, "ts": "1704161511423396.302", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424929423, "pid": 800, "tid": 3, "ts": "1704161511424929.423", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511424946003, "pid": 437675, "tid": 437675, "ts": "1704161511423396.302", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511424946003, "pid": 800, "tid": 3, "ts": "1704161511424946.003", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425697246, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425697246, "pid": 800, "tid": 3, "ts": "1704161511425697.246", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425733735, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425733735, "pid": 800, "tid": 3, "ts": "1704161511425733.735", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425760075, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425760075, "pid": 800, "tid": 3, "ts": "1704161511425760.075", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425777564, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425777564, "pid": 800, "tid": 3, "ts": "1704161511425777.564", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425786644, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425786644, "pid": 800, "tid": 3, "ts": "1704161511425786.644", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425803044, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425803044, "pid": 800, "tid": 3, "ts": "1704161511425803.044", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425830794, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425830794, "pid": 800, "tid": 3, "ts": "1704161511425830.794", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425853424, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425853424, "pid": 800, "tid": 3, "ts": "1704161511425853.424", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425869524, "pid": 437675, "tid": 437675, "ts": "1704161511423468.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425869524, "pid": 800, "tid": 3, "ts": "1704161511425869.524", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425895714, "pid": 437675, "tid": 437675, "ts": "1704161511423495.183", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425895714, "pid": 800, "tid": 3, "ts": "1704161511425895.714", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425955963, "pid": 437675, "tid": 437675, "ts": "1704161511423553.623", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425955963, "pid": 800, "tid": 3, "ts": "1704161511425955.963", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425975312, "pid": 437675, "tid": 437675, "ts": "1704161511423553.623", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425975312, "pid": 800, "tid": 3, "ts": "1704161511425975.312", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511425997052, "pid": 437675, "tid": 437675, "ts": "1704161511423553.623", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511425997052, "pid": 800, "tid": 3, "ts": "1704161511425997.052", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426019682, "pid": 437675, "tid": 437675, "ts": "1704161511423553.623", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426019682, "pid": 800, "tid": 3, "ts": "1704161511426019.682", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426033542, "pid": 437675, "tid": 437675, "ts": "1704161511423553.623", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426033542, "pid": 800, "tid": 3, "ts": "1704161511426033.542", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426052702, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426052702, "pid": 800, "tid": 3, "ts": "1704161511426052.702", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426062642, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426062642, "pid": 800, "tid": 3, "ts": "1704161511426062.642", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426073612, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426073612, "pid": 800, "tid": 3, "ts": "1704161511426073.612", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426078342, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426078342, "pid": 800, "tid": 3, "ts": "1704161511426078.342", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426083692, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426083692, "pid": 800, "tid": 3, "ts": "1704161511426083.692", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426090172, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426090172, "pid": 800, "tid": 3, "ts": "1704161511426090.172", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426097302, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426097302, "pid": 800, "tid": 3, "ts": "1704161511426097.302", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426105822, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426105822, "pid": 800, "tid": 3, "ts": "1704161511426105.822", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426110062, "pid": 437675, "tid": 437675, "ts": "1704161511423669.464", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426110062, "pid": 800, "tid": 3, "ts": "1704161511426110.062", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426117962, "pid": 437675, "tid": 437675, "ts": "1704161511423691.065", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426117962, "pid": 800, "tid": 3, "ts": "1704161511426117.962", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426120132, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426120132, "pid": 800, "tid": 3, "ts": "1704161511426120.132", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426132241, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426132241, "pid": 800, "tid": 3, "ts": "1704161511426132.241", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426138671, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426138671, "pid": 800, "tid": 3, "ts": "1704161511426138.671", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426146341, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426146341, "pid": 800, "tid": 3, "ts": "1704161511426146.341", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426154841, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426154841, "pid": 800, "tid": 3, "ts": "1704161511426154.841", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426165581, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426165581, "pid": 800, "tid": 3, "ts": "1704161511426165.581", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426172020, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426172020, "pid": 800, "tid": 3, "ts": "1704161511426172.020", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426181520, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426181520, "pid": 800, "tid": 3, "ts": "1704161511426181.520", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426189750, "pid": 437675, "tid": 437675, "ts": "1704161511423743.715", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426189750, "pid": 800, "tid": 3, "ts": "1704161511426189.750", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426198830, "pid": 437675, "tid": 437675, "ts": "1704161511423760.635", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426198830, "pid": 800, "tid": 3, "ts": "1704161511426198.830", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426200860, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426200860, "pid": 800, "tid": 3, "ts": "1704161511426200.860", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426202380, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426202380, "pid": 800, "tid": 3, "ts": "1704161511426202.380", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426223540, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426223540, "pid": 800, "tid": 3, "ts": "1704161511426223.540", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426225140, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426225140, "pid": 800, "tid": 3, "ts": "1704161511426225.140", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426232660, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426232660, "pid": 800, "tid": 3, "ts": "1704161511426232.660", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426239650, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426239650, "pid": 800, "tid": 3, "ts": "1704161511426239.650", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426245990, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426245990, "pid": 800, "tid": 3, "ts": "1704161511426245.990", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426248450, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426248450, "pid": 800, "tid": 3, "ts": "1704161511426248.450", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426254600, "pid": 437675, "tid": 437675, "ts": "1704161511423810.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426254600, "pid": 800, "tid": 3, "ts": "1704161511426254.600", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426463338, "pid": 437675, "tid": 437675, "ts": "1704161511426449.252", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426463338, "pid": 800, "tid": 3, "ts": "1704161511426463.338", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426537697, "pid": 437675, "tid": 437675, "ts": "1704161511426532.533", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426537697, "pid": 800, "tid": 3, "ts": "1704161511426537.697", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426547667, "pid": 437675, "tid": 437675, "ts": "1704161511426550.453", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426547667, "pid": 800, "tid": 3, "ts": "1704161511426547.667", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426550747, "pid": 437675, "tid": 437675, "ts": "1704161511426550.453", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426550747, "pid": 800, "tid": 3, "ts": "1704161511426550.747", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426564487, "pid": 437675, "tid": 437675, "ts": "1704161511426550.453", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426564487, "pid": 800, "tid": 3, "ts": "1704161511426564.487", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426573236, "pid": 437675, "tid": 437675, "ts": "1704161511426550.453", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426573236, "pid": 800, "tid": 3, "ts": "1704161511426573.236", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426619016, "pid": 437675, "tid": 437675, "ts": "1704161511426617.294", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426619016, "pid": 800, "tid": 3, "ts": "1704161511426619.016", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426824554, "pid": 437675, "tid": 439228, "ts": "1704161511426816.966", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426824554, "pid": 800, "tid": 3, "ts": "1704161511426824.554", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426861404, "pid": 437675, "tid": 439228, "ts": "1704161511426846.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426861404, "pid": 800, "tid": 3, "ts": "1704161511426861.404", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426869724, "pid": 437675, "tid": 439228, "ts": "1704161511426846.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426869724, "pid": 800, "tid": 3, "ts": "1704161511426869.724", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511426936243, "pid": 437675, "tid": 439228, "ts": "1704161511426924.287", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511426936243, "pid": 800, "tid": 3, "ts": "1704161511426936.243", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427004822, "pid": 437675, "tid": 439228, "ts": "1704161511426997.638", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427004822, "pid": 800, "tid": 3, "ts": "1704161511427004.822", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427006582, "pid": 437675, "tid": 439228, "ts": "1704161511426997.638", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427006582, "pid": 800, "tid": 3, "ts": "1704161511427006.582", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427017562, "pid": 437675, "tid": 439228, "ts": "1704161511426997.638", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427017562, "pid": 800, "tid": 3, "ts": "1704161511427017.562", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427023362, "pid": 437675, "tid": 439228, "ts": "1704161511426997.638", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427023362, "pid": 800, "tid": 3, "ts": "1704161511427023.362", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427031202, "pid": 437675, "tid": 439228, "ts": "1704161511426997.638", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427031202, "pid": 800, "tid": 3, "ts": "1704161511427031.202", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427037582, "pid": 437675, "tid": 439228, "ts": "1704161511426997.638", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427037582, "pid": 800, "tid": 3, "ts": "1704161511427037.582", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427040462, "pid": 437675, "tid": 439228, "ts": "1704161511426997.638", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427040462, "pid": 800, "tid": 3, "ts": "1704161511427040.462", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427048582, "pid": 437675, "tid": 439228, "ts": "1704161511427030.778", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427048582, "pid": 800, "tid": 3, "ts": "1704161511427048.582", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427050192, "pid": 437675, "tid": 439228, "ts": "1704161511427030.778", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427050192, "pid": 800, "tid": 3, "ts": "1704161511427050.192", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427055462, "pid": 437675, "tid": 439228, "ts": "1704161511427030.778", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427055462, "pid": 800, "tid": 3, "ts": "1704161511427055.462", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427060822, "pid": 437675, "tid": 439228, "ts": "1704161511427030.778", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427060822, "pid": 800, "tid": 3, "ts": "1704161511427060.822", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427068772, "pid": 437675, "tid": 439228, "ts": "1704161511427030.778", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427068772, "pid": 800, "tid": 3, "ts": "1704161511427068.772", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427075512, "pid": 437675, "tid": 439228, "ts": "1704161511427030.778", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427075512, "pid": 800, "tid": 3, "ts": "1704161511427075.512", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427077712, "pid": 437675, "tid": 439228, "ts": "1704161511427030.778", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427077712, "pid": 800, "tid": 3, "ts": "1704161511427077.712", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427085432, "pid": 437675, "tid": 439228, "ts": "1704161511427082.058", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427085432, "pid": 800, "tid": 3, "ts": "1704161511427085.432", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427092352, "pid": 437675, "tid": 439228, "ts": "1704161511427082.058", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427092352, "pid": 800, "tid": 3, "ts": "1704161511427092.352", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427303440, "pid": 437675, "tid": 439228, "ts": "1704161511427259.190", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427303440, "pid": 800, "tid": 3, "ts": "1704161511427303.440", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427371509, "pid": 437675, "tid": 439228, "ts": "1704161511427351.201", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427371509, "pid": 800, "tid": 3, "ts": "1704161511427371.509", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427431688, "pid": 437675, "tid": 439228, "ts": "1704161511427415.592", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427431688, "pid": 800, "tid": 3, "ts": "1704161511427431.688", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427479708, "pid": 437675, "tid": 439228, "ts": "1704161511427471.612", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427479708, "pid": 800, "tid": 3, "ts": "1704161511427479.708", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427481828, "pid": 437675, "tid": 439228, "ts": "1704161511427471.612", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427481828, "pid": 800, "tid": 3, "ts": "1704161511427481.828", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427492208, "pid": 437675, "tid": 439228, "ts": "1704161511427471.612", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427492208, "pid": 800, "tid": 3, "ts": "1704161511427492.208", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427499728, "pid": 437675, "tid": 439228, "ts": "1704161511427471.612", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427499728, "pid": 800, "tid": 3, "ts": "1704161511427499.728", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427507778, "pid": 437675, "tid": 439228, "ts": "1704161511427471.612", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427507778, "pid": 800, "tid": 3, "ts": "1704161511427507.778", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427514148, "pid": 437675, "tid": 439228, "ts": "1704161511427471.612", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427514148, "pid": 800, "tid": 3, "ts": "1704161511427514.148", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427517078, "pid": 437675, "tid": 439228, "ts": "1704161511427471.612", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427517078, "pid": 800, "tid": 3, "ts": "1704161511427517.078", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427521338, "pid": 437675, "tid": 439228, "ts": "1704161511427501.723", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427521338, "pid": 800, "tid": 3, "ts": "1704161511427521.338", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427523267, "pid": 437675, "tid": 439228, "ts": "1704161511427501.723", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427523267, "pid": 800, "tid": 3, "ts": "1704161511427523.267", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427525457, "pid": 437675, "tid": 439228, "ts": "1704161511427501.723", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427525457, "pid": 800, "tid": 3, "ts": "1704161511427525.457", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427532977, "pid": 437675, "tid": 439228, "ts": "1704161511427501.723", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427532977, "pid": 800, "tid": 3, "ts": "1704161511427532.977", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427540347, "pid": 437675, "tid": 439228, "ts": "1704161511427501.723", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427540347, "pid": 800, "tid": 3, "ts": "1704161511427540.347", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427546987, "pid": 437675, "tid": 439228, "ts": "1704161511427501.723", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427546987, "pid": 800, "tid": 3, "ts": "1704161511427546.987", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427549627, "pid": 437675, "tid": 439228, "ts": "1704161511427501.723", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427549627, "pid": 800, "tid": 3, "ts": "1704161511427549.627", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427553867, "pid": 437675, "tid": 439228, "ts": "1704161511427522.133", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427553867, "pid": 800, "tid": 3, "ts": "1704161511427553.867", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427560787, "pid": 437675, "tid": 439228, "ts": "1704161511427522.133", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427560787, "pid": 800, "tid": 3, "ts": "1704161511427560.787", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427612856, "pid": 437675, "tid": 439228, "ts": "1704161511427593.064", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427612856, "pid": 800, "tid": 3, "ts": "1704161511427612.856", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427683476, "pid": 437675, "tid": 439228, "ts": "1704161511427667.914", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427683476, "pid": 800, "tid": 3, "ts": "1704161511427683.476", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427705176, "pid": 437675, "tid": 439228, "ts": "1704161511427704.725", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427705176, "pid": 800, "tid": 3, "ts": "1704161511427705.176", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427758055, "pid": 437675, "tid": 439228, "ts": "1704161511427746.585", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427758055, "pid": 800, "tid": 3, "ts": "1704161511427758.055", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427760505, "pid": 437675, "tid": 439228, "ts": "1704161511427746.585", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427760505, "pid": 800, "tid": 3, "ts": "1704161511427760.505", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427775084, "pid": 437675, "tid": 439228, "ts": "1704161511427746.585", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427775084, "pid": 800, "tid": 3, "ts": "1704161511427775.084", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427782584, "pid": 437675, "tid": 439228, "ts": "1704161511427746.585", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427782584, "pid": 800, "tid": 3, "ts": "1704161511427782.584", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427789024, "pid": 437675, "tid": 439228, "ts": "1704161511427746.585", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427789024, "pid": 800, "tid": 3, "ts": "1704161511427789.024", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427795454, "pid": 437675, "tid": 439228, "ts": "1704161511427746.585", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427795454, "pid": 800, "tid": 3, "ts": "1704161511427795.454", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427801114, "pid": 437675, "tid": 439228, "ts": "1704161511427746.585", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427801114, "pid": 800, "tid": 3, "ts": "1704161511427801.114", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427868514, "pid": 437675, "tid": 439228, "ts": "1704161511427861.856", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427868514, "pid": 800, "tid": 3, "ts": "1704161511427868.514", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427870904, "pid": 437675, "tid": 439228, "ts": "1704161511427861.856", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427870904, "pid": 800, "tid": 3, "ts": "1704161511427870.904", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427884264, "pid": 437675, "tid": 439228, "ts": "1704161511427861.856", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427884264, "pid": 800, "tid": 3, "ts": "1704161511427884.264", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427891924, "pid": 437675, "tid": 439228, "ts": "1704161511427861.856", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427891924, "pid": 800, "tid": 3, "ts": "1704161511427891.924", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427897494, "pid": 437675, "tid": 439228, "ts": "1704161511427861.856", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427897494, "pid": 800, "tid": 3, "ts": "1704161511427897.494", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427903704, "pid": 437675, "tid": 439228, "ts": "1704161511427861.856", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427903704, "pid": 800, "tid": 3, "ts": "1704161511427903.704", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427908594, "pid": 437675, "tid": 439228, "ts": "1704161511427861.856", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427908594, "pid": 800, "tid": 3, "ts": "1704161511427908.594", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427917724, "pid": 437675, "tid": 439228, "ts": "1704161511427894.447", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427917724, "pid": 800, "tid": 3, "ts": "1704161511427917.724", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427924183, "pid": 437675, "tid": 439228, "ts": "1704161511427894.447", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427924183, "pid": 800, "tid": 3, "ts": "1704161511427924.183", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511427990032, "pid": 437675, "tid": 439228, "ts": "1704161511427972.097", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511427990032, "pid": 800, "tid": 3, "ts": "1704161511427990.032", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428064212, "pid": 437675, "tid": 439228, "ts": "1704161511428050.588", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428064212, "pid": 800, "tid": 3, "ts": "1704161511428064.212", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428807574, "pid": 437675, "tid": 439228, "ts": "1704161511428164.169", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428807574, "pid": 800, "tid": 3, "ts": "1704161511428807.574", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428844424, "pid": 437675, "tid": 439228, "ts": "1704161511428164.169", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428844424, "pid": 800, "tid": 3, "ts": "1704161511428844.424", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428871664, "pid": 437675, "tid": 439228, "ts": "1704161511428164.169", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428871664, "pid": 800, "tid": 3, "ts": "1704161511428871.664", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428888694, "pid": 437675, "tid": 439228, "ts": "1704161511428164.169", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428888694, "pid": 800, "tid": 3, "ts": "1704161511428888.694", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428940243, "pid": 437675, "tid": 439228, "ts": "1704161511428164.169", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428940243, "pid": 800, "tid": 3, "ts": "1704161511428940.243", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428969833, "pid": 437675, "tid": 439228, "ts": "1704161511428164.169", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428969833, "pid": 800, "tid": 3, "ts": "1704161511428969.833", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428981892, "pid": 437675, "tid": 439228, "ts": "1704161511428164.169", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428981892, "pid": 800, "tid": 3, "ts": "1704161511428981.892", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511428997462, "pid": 437675, "tid": 439228, "ts": "1704161511428213.150", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511428997462, "pid": 800, "tid": 3, "ts": "1704161511428997.462", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511429727355, "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511429727355, "pid": 800, "tid": 3, "ts": "1704161511429727.355", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511429755595, "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511429755595, "pid": 800, "tid": 3, "ts": "1704161511429755.595", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511429765475, "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511429765475, "pid": 800, "tid": 3, "ts": "1704161511429765.475", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511429778074, "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511429778074, "pid": 800, "tid": 3, "ts": "1704161511429778.074", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511429816044, "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511429816044, "pid": 800, "tid": 3, "ts": "1704161511429816.044", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511429856244, "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511429856244, "pid": 800, "tid": 3, "ts": "1704161511429856.244", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511429880104, "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511429880104, "pid": 800, "tid": 3, "ts": "1704161511429880.104", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511429898474, "pid": 437675, "tid": 439228, "ts": "1704161511428347.071", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511429898474, "pid": 800, "tid": 3, "ts": "1704161511429898.474", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511430588746, "pid": 437675, "tid": 439228, "ts": "1704161511428382.811", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511430588746, "pid": 800, "tid": 3, "ts": "1704161511430588.746", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511430622196, "pid": 437675, "tid": 439228, "ts": "1704161511428382.811", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511430622196, "pid": 800, "tid": 3, "ts": "1704161511430622.196", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511430644526, "pid": 437675, "tid": 439228, "ts": "1704161511428382.811", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511430644526, "pid": 800, "tid": 3, "ts": "1704161511430644.526", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511430662726, "pid": 437675, "tid": 439228, "ts": "1704161511428382.811", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511430662726, "pid": 800, "tid": 3, "ts": "1704161511430662.726", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511430704986, "pid": 437675, "tid": 439228, "ts": "1704161511428382.811", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511430704986, "pid": 800, "tid": 3, "ts": "1704161511430704.986", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511430713476, "pid": 437675, "tid": 439228, "ts": "1704161511428382.811", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511430713476, "pid": 800, "tid": 3, "ts": "1704161511430713.476", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511430739235, "pid": 437675, "tid": 439228, "ts": "1704161511428382.811", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511430739235, "pid": 800, "tid": 3, "ts": "1704161511430739.235", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431052332, "pid": 437675, "tid": 439228, "ts": "1704161511428434.892", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431052332, "pid": 800, "tid": 3, "ts": "1704161511431052.332", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431071262, "pid": 437675, "tid": 439228, "ts": "1704161511428434.892", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431071262, "pid": 800, "tid": 3, "ts": "1704161511431071.262", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431167751, "pid": 437675, "tid": 439228, "ts": "1704161511428602.694", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431167751, "pid": 800, "tid": 3, "ts": "1704161511431167.751", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431208310, "pid": 437675, "tid": 439228, "ts": "1704161511428655.254", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431208310, "pid": 800, "tid": 3, "ts": "1704161511431208.310", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431858704, "pid": 437675, "tid": 439228, "ts": "1704161511428703.785", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431858704, "pid": 800, "tid": 3, "ts": "1704161511431858.704", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431891734, "pid": 437675, "tid": 439228, "ts": "1704161511428703.785", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431891734, "pid": 800, "tid": 3, "ts": "1704161511431891.734", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431914284, "pid": 437675, "tid": 439228, "ts": "1704161511428703.785", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431914284, "pid": 800, "tid": 3, "ts": "1704161511431914.284", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431937293, "pid": 437675, "tid": 439228, "ts": "1704161511428703.785", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431937293, "pid": 800, "tid": 3, "ts": "1704161511431937.293", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431961483, "pid": 437675, "tid": 439228, "ts": "1704161511428703.785", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431961483, "pid": 800, "tid": 3, "ts": "1704161511431961.483", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511431986512, "pid": 437675, "tid": 439228, "ts": "1704161511428703.785", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511431986512, "pid": 800, "tid": 3, "ts": "1704161511431986.512", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432001662, "pid": 437675, "tid": 439228, "ts": "1704161511428703.785", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432001662, "pid": 800, "tid": 3, "ts": "1704161511432001.662", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432032482, "pid": 437675, "tid": 439228, "ts": "1704161511428743.655", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432032482, "pid": 800, "tid": 3, "ts": "1704161511432032.482", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432083372, "pid": 437675, "tid": 439228, "ts": "1704161511428819.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432083372, "pid": 800, "tid": 3, "ts": "1704161511432083.372", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432109392, "pid": 437675, "tid": 439228, "ts": "1704161511428819.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432109392, "pid": 800, "tid": 3, "ts": "1704161511432109.392", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432126562, "pid": 437675, "tid": 439228, "ts": "1704161511428819.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432126562, "pid": 800, "tid": 3, "ts": "1704161511432126.562", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432145121, "pid": 437675, "tid": 439228, "ts": "1704161511428819.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432145121, "pid": 800, "tid": 3, "ts": "1704161511432145.121", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432179410, "pid": 437675, "tid": 439228, "ts": "1704161511428819.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432179410, "pid": 800, "tid": 3, "ts": "1704161511432179.410", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432189400, "pid": 437675, "tid": 439228, "ts": "1704161511428819.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432189400, "pid": 800, "tid": 3, "ts": "1704161511432189.400", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432224730, "pid": 437675, "tid": 439228, "ts": "1704161511428819.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432224730, "pid": 800, "tid": 3, "ts": "1704161511432224.730", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432260930, "pid": 437675, "tid": 439228, "ts": "1704161511428855.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432260930, "pid": 800, "tid": 3, "ts": "1704161511432260.930", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432270310, "pid": 437675, "tid": 439228, "ts": "1704161511428855.646", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432270310, "pid": 800, "tid": 3, "ts": "1704161511432270.310", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432354639, "pid": 437675, "tid": 439228, "ts": "1704161511428953.477", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432354639, "pid": 800, "tid": 3, "ts": "1704161511432354.639", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432386648, "pid": 437675, "tid": 439228, "ts": "1704161511428995.778", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432386648, "pid": 800, "tid": 3, "ts": "1704161511432386.648", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432413758, "pid": 437675, "tid": 437675, "ts": "1704161511429190.490", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432413758, "pid": 800, "tid": 3, "ts": "1704161511432413.758", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432422268, "pid": 437675, "tid": 437675, "ts": "1704161511429220.330", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432422268, "pid": 800, "tid": 3, "ts": "1704161511432422.268", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432434548, "pid": 437675, "tid": 437675, "ts": "1704161511429237.520", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432434548, "pid": 800, "tid": 3, "ts": "1704161511432434.548", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432444538, "pid": 437675, "tid": 437675, "ts": "1704161511429259.120", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432444538, "pid": 800, "tid": 3, "ts": "1704161511432444.538", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432455308, "pid": 437675, "tid": 437675, "ts": "1704161511429284.691", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432455308, "pid": 800, "tid": 3, "ts": "1704161511432455.308", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432496978, "pid": 437675, "tid": 437675, "ts": "1704161511429396.712", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432496978, "pid": 800, "tid": 3, "ts": "1704161511432496.978", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432514998, "pid": 437675, "tid": 437675, "ts": "1704161511429427.092", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432514998, "pid": 800, "tid": 3, "ts": "1704161511432514.998", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432555557, "pid": 437675, "tid": 437675, "ts": "1704161511429503.263", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432555557, "pid": 800, "tid": 3, "ts": "1704161511432555.557", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432579166, "pid": 437675, "tid": 437675, "ts": "1704161511429530.753", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432579166, "pid": 800, "tid": 3, "ts": "1704161511432579.166", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432587416, "pid": 437675, "tid": 437675, "ts": "1704161511429551.083", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432587416, "pid": 800, "tid": 3, "ts": "1704161511432587.416", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432597586, "pid": 437675, "tid": 437675, "ts": "1704161511429566.633", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432597586, "pid": 800, "tid": 3, "ts": "1704161511432597.586", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432606356, "pid": 437675, "tid": 437675, "ts": "1704161511429580.403", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432606356, "pid": 800, "tid": 3, "ts": "1704161511432606.356", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432614986, "pid": 437675, "tid": 437675, "ts": "1704161511429597.614", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432614986, "pid": 800, "tid": 3, "ts": "1704161511432614.986", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432641056, "pid": 437675, "tid": 437675, "ts": "1704161511429657.214", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432641056, "pid": 800, "tid": 3, "ts": "1704161511432641.056", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432654996, "pid": 437675, "tid": 437675, "ts": "1704161511429683.444", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432654996, "pid": 800, "tid": 3, "ts": "1704161511432654.996", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432682936, "pid": 437675, "tid": 437675, "ts": "1704161511429739.045", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432682936, "pid": 800, "tid": 3, "ts": "1704161511432682.936", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432699816, "pid": 437675, "tid": 437675, "ts": "1704161511429764.755", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432699816, "pid": 800, "tid": 3, "ts": "1704161511432699.816", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432715196, "pid": 437675, "tid": 437675, "ts": "1704161511429783.905", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432715196, "pid": 800, "tid": 3, "ts": "1704161511432715.196", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432718256, "pid": 437675, "tid": 437675, "ts": "1704161511429807.856", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432718256, "pid": 800, "tid": 3, "ts": "1704161511432718.256", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432733965, "pid": 437675, "tid": 437675, "ts": "1704161511429826.966", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432733965, "pid": 800, "tid": 3, "ts": "1704161511432733.965", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432740525, "pid": 437675, "tid": 437675, "ts": "1704161511429844.736", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432740525, "pid": 800, "tid": 3, "ts": "1704161511432740.525", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432767615, "pid": 437675, "tid": 437675, "ts": "1704161511429904.687", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432767615, "pid": 800, "tid": 3, "ts": "1704161511432767.615", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432777264, "pid": 437675, "tid": 437675, "ts": "1704161511429930.557", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432777264, "pid": 800, "tid": 3, "ts": "1704161511432777.264", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432797084, "pid": 437675, "tid": 437675, "ts": "1704161511429986.048", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432797084, "pid": 800, "tid": 3, "ts": "1704161511432797.084", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432816684, "pid": 437675, "tid": 437675, "ts": "1704161511430008.578", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432816684, "pid": 800, "tid": 3, "ts": "1704161511432816.684", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432836124, "pid": 437675, "tid": 437675, "ts": "1704161511430021.828", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432836124, "pid": 800, "tid": 3, "ts": "1704161511432836.124", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432837674, "pid": 437675, "tid": 437675, "ts": "1704161511430043.898", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432837674, "pid": 800, "tid": 3, "ts": "1704161511432837.674", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432850164, "pid": 437675, "tid": 437675, "ts": "1704161511430057.468", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432850164, "pid": 800, "tid": 3, "ts": "1704161511432850.164", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432866914, "pid": 437675, "tid": 437675, "ts": "1704161511430073.098", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432866914, "pid": 800, "tid": 3, "ts": "1704161511432866.914", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432894324, "pid": 437675, "tid": 437675, "ts": "1704161511430126.759", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432894324, "pid": 800, "tid": 3, "ts": "1704161511432894.324", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432910874, "pid": 437675, "tid": 437675, "ts": "1704161511430149.279", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432910874, "pid": 800, "tid": 3, "ts": "1704161511432910.874", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432936273, "pid": 437675, "tid": 437675, "ts": "1704161511430198.880", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432936273, "pid": 800, "tid": 3, "ts": "1704161511432936.273", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432951583, "pid": 437675, "tid": 437675, "ts": "1704161511430231.680", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432951583, "pid": 800, "tid": 3, "ts": "1704161511432951.583", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432964953, "pid": 437675, "tid": 437675, "ts": "1704161511430245.440", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432964953, "pid": 800, "tid": 3, "ts": "1704161511432964.953", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432970373, "pid": 437675, "tid": 437675, "ts": "1704161511430263.740", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432970373, "pid": 800, "tid": 3, "ts": "1704161511432970.373", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432984052, "pid": 437675, "tid": 437675, "ts": "1704161511430277.350", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432984052, "pid": 800, "tid": 3, "ts": "1704161511432984.052", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511432994212, "pid": 437675, "tid": 437675, "ts": "1704161511430293.391", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511432994212, "pid": 800, "tid": 3, "ts": "1704161511432994.212", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433011992, "pid": 437675, "tid": 437675, "ts": "1704161511430348.241", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433011992, "pid": 800, "tid": 3, "ts": "1704161511433011.992", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433025252, "pid": 437675, "tid": 437675, "ts": "1704161511430372.141", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433025252, "pid": 800, "tid": 3, "ts": "1704161511433025.252", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433045072, "pid": 437675, "tid": 437675, "ts": "1704161511430424.952", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433045072, "pid": 800, "tid": 3, "ts": "1704161511433045.072", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433080302, "pid": 437675, "tid": 437675, "ts": "1704161511430447.902", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433080302, "pid": 800, "tid": 3, "ts": "1704161511433080.302", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433092932, "pid": 437675, "tid": 437675, "ts": "1704161511430461.112", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433092932, "pid": 800, "tid": 3, "ts": "1704161511433092.932", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433094642, "pid": 437675, "tid": 437675, "ts": "1704161511430478.862", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433094642, "pid": 800, "tid": 3, "ts": "1704161511433094.642", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433121742, "pid": 437675, "tid": 437675, "ts": "1704161511430491.463", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433121742, "pid": 800, "tid": 3, "ts": "1704161511433121.742", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433125872, "pid": 437675, "tid": 437675, "ts": "1704161511430507.133", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433125872, "pid": 800, "tid": 3, "ts": "1704161511433125.872", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433132671, "pid": 437675, "tid": 437675, "ts": "1704161511430557.473", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433132671, "pid": 800, "tid": 3, "ts": "1704161511433132.671", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433139371, "pid": 437675, "tid": 437675, "ts": "1704161511430579.823", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433139371, "pid": 800, "tid": 3, "ts": "1704161511433139.371", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433161761, "pid": 437675, "tid": 437675, "ts": "1704161511430628.544", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433161761, "pid": 800, "tid": 3, "ts": "1704161511433161.761", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433183990, "pid": 437675, "tid": 437675, "ts": "1704161511430650.654", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433183990, "pid": 800, "tid": 3, "ts": "1704161511433183.990", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433220980, "pid": 437675, "tid": 437675, "ts": "1704161511430663.114", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433220980, "pid": 800, "tid": 3, "ts": "1704161511433220.980", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433223620, "pid": 437675, "tid": 437675, "ts": "1704161511430682.284", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433223620, "pid": 800, "tid": 3, "ts": "1704161511433223.620", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433233730, "pid": 437675, "tid": 437675, "ts": "1704161511430696.425", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433233730, "pid": 800, "tid": 3, "ts": "1704161511433233.730", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433239550, "pid": 437675, "tid": 437675, "ts": "1704161511430712.515", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433239550, "pid": 800, "tid": 3, "ts": "1704161511433239.550", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433248110, "pid": 437675, "tid": 437675, "ts": "1704161511430763.255", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433248110, "pid": 800, "tid": 3, "ts": "1704161511433248.110", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433263850, "pid": 437675, "tid": 437675, "ts": "1704161511430784.465", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433263850, "pid": 800, "tid": 3, "ts": "1704161511433263.850", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433292040, "pid": 437675, "tid": 437675, "ts": "1704161511430835.566", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433292040, "pid": 800, "tid": 3, "ts": "1704161511433292.040", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433310380, "pid": 437675, "tid": 437675, "ts": "1704161511430856.916", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433310380, "pid": 800, "tid": 3, "ts": "1704161511433310.380", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433324640, "pid": 437675, "tid": 437675, "ts": "1704161511430868.846", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433324640, "pid": 800, "tid": 3, "ts": "1704161511433324.640", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433326230, "pid": 437675, "tid": 437675, "ts": "1704161511430882.316", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433326230, "pid": 800, "tid": 3, "ts": "1704161511433326.230", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433335009, "pid": 437675, "tid": 437675, "ts": "1704161511430898.757", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433335009, "pid": 800, "tid": 3, "ts": "1704161511433335.009", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433352219, "pid": 437675, "tid": 437675, "ts": "1704161511430912.737", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433352219, "pid": 800, "tid": 3, "ts": "1704161511433352.219", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433362289, "pid": 437675, "tid": 437675, "ts": "1704161511430962.007", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433362289, "pid": 800, "tid": 3, "ts": "1704161511433362.289", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433384498, "pid": 437675, "tid": 437675, "ts": "1704161511430983.637", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433384498, "pid": 800, "tid": 3, "ts": "1704161511433384.498", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433402698, "pid": 437675, "tid": 437675, "ts": "1704161511431030.818", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433402698, "pid": 800, "tid": 3, "ts": "1704161511433402.698", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433421908, "pid": 437675, "tid": 437675, "ts": "1704161511431052.738", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433421908, "pid": 800, "tid": 3, "ts": "1704161511433421.908", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433441418, "pid": 437675, "tid": 437675, "ts": "1704161511431065.368", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433441418, "pid": 800, "tid": 3, "ts": "1704161511433441.418", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433443248, "pid": 437675, "tid": 437675, "ts": "1704161511431078.608", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433443248, "pid": 800, "tid": 3, "ts": "1704161511433443.248", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433454408, "pid": 437675, "tid": 437675, "ts": "1704161511431099.609", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433454408, "pid": 800, "tid": 3, "ts": "1704161511433454.408", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433465338, "pid": 437675, "tid": 437675, "ts": "1704161511431115.269", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433465338, "pid": 800, "tid": 3, "ts": "1704161511433465.338", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433482728, "pid": 437675, "tid": 437675, "ts": "1704161511431168.769", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433482728, "pid": 800, "tid": 3, "ts": "1704161511433482.728", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433505428, "pid": 437675, "tid": 437675, "ts": "1704161511431191.670", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433505428, "pid": 800, "tid": 3, "ts": "1704161511433505.428", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433525048, "pid": 437675, "tid": 437675, "ts": "1704161511431243.120", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433525048, "pid": 800, "tid": 3, "ts": "1704161511433525.048", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433546537, "pid": 437675, "tid": 437675, "ts": "1704161511431266.000", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433546537, "pid": 800, "tid": 3, "ts": "1704161511433546.537", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433554497, "pid": 437675, "tid": 437675, "ts": "1704161511431279.690", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433554497, "pid": 800, "tid": 3, "ts": "1704161511433554.497", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433564967, "pid": 437675, "tid": 437675, "ts": "1704161511431292.811", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433564967, "pid": 800, "tid": 3, "ts": "1704161511433564.967", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433574217, "pid": 437675, "tid": 437675, "ts": "1704161511431308.501", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433574217, "pid": 800, "tid": 3, "ts": "1704161511433574.217", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433582726, "pid": 437675, "tid": 437675, "ts": "1704161511431323.381", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433582726, "pid": 800, "tid": 3, "ts": "1704161511433582.726", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433609286, "pid": 437675, "tid": 437675, "ts": "1704161511431374.301", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433609286, "pid": 800, "tid": 3, "ts": "1704161511433609.286", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433623296, "pid": 437675, "tid": 437675, "ts": "1704161511431395.582", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433623296, "pid": 800, "tid": 3, "ts": "1704161511433623.296", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433649866, "pid": 437675, "tid": 437675, "ts": "1704161511431442.142", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433649866, "pid": 800, "tid": 3, "ts": "1704161511433649.866", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433666176, "pid": 437675, "tid": 437675, "ts": "1704161511431534.493", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433666176, "pid": 800, "tid": 3, "ts": "1704161511433666.176", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433686786, "pid": 437675, "tid": 437675, "ts": "1704161511431573.203", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433686786, "pid": 800, "tid": 3, "ts": "1704161511433686.786", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433691646, "pid": 437675, "tid": 437675, "ts": "1704161511431573.203", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433691646, "pid": 800, "tid": 3, "ts": "1704161511433691.646", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433701196, "pid": 437675, "tid": 437675, "ts": "1704161511431594.914", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433701196, "pid": 800, "tid": 3, "ts": "1704161511433701.196", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433725596, "pid": 437675, "tid": 437675, "ts": "1704161511431704.105", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433725596, "pid": 800, "tid": 3, "ts": "1704161511433725.596", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433787454, "pid": 437675, "tid": 437675, "ts": "1704161511431704.105", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433787454, "pid": 800, "tid": 3, "ts": "1704161511433787.454", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433794264, "pid": 437675, "tid": 437675, "ts": "1704161511431704.105", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433794264, "pid": 800, "tid": 3, "ts": "1704161511433794.264", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511433816104, "pid": 437675, "tid": 437675, "ts": "1704161511431704.105", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511433816104, "pid": 800, "tid": 3, "ts": "1704161511433816.104", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511434018972, "pid": 437675, "tid": 437675, "ts": "1704161511431757.055", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511434018972, "pid": 800, "tid": 3, "ts": "1704161511434018.972", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511434169621, "pid": 437675, "tid": 437675, "ts": "1704161511431778.495", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511434169621, "pid": 800, "tid": 3, "ts": "1704161511434169.621", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511434171181, "pid": 437675, "tid": 437675, "ts": "1704161511431820.886", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511434171181, "pid": 800, "tid": 3, "ts": "1704161511434171.181", "cat": "async_npu"}, {"ph": "s", "bp": "e", "name": "torch_to_npu", "id": 1704161511434172701, "pid": 437675, "tid": 437675, "ts": "1704161511431840.796", "cat": "async_npu"}, {"ph": "f", "bp": "e", "name": "torch_to_npu", "id": 1704161511434172701, "pid": 800, "tid": 3, "ts": "1704161511434172.701", "cat": "async_npu"}] \ No newline at end of file -- Gitee From f09a99220dd46137911d999b608561bb04d825b7 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Thu, 22 Aug 2024 00:26:19 +0800 Subject: [PATCH 433/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=86=85=E7=BD=AE?= =?UTF-8?q?=E6=98=A0=E5=B0=84=E8=A1=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/compare/ms_compare.py | 52 +-- .../mindspore/compare/ms_to_pt_api.yaml | 399 ++++++++++++++++++ debug/accuracy_tools/msprobe/msprobe.py | 2 + 3 files changed, 428 insertions(+), 25 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/compare/ms_to_pt_api.yaml diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index c4cae74c9..7c3bd6815 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -16,7 +16,7 @@ class MSComparator(Comparator): self.api_mapping = api_mapping self.cross_frame = cell_mapping is not None or api_mapping is not None self.cell_mapping_dict = self.load_mapping_file(self.cell_mapping) - self.api_mapping_dict = self.load_mapping_file(self.api_mapping) + self.api_mapping_dict = {} if api_mapping is not None: self.ms_to_pt_mapping = self.load_internal_api() @@ -32,34 +32,34 @@ class MSComparator(Comparator): mapping_dict = {} return mapping_dict - def process_cell_mapping(self, a_op_name): - a_op_name = [op_name.replace("Cell", "Module", 1) for op_name in a_op_name] + def process_cell_mapping(self, npu_op_name): + npu_op_name = [op_name.replace("Cell", "Module", 1) for op_name in npu_op_name] if self.cell_mapping_dict: - for index, op_name in enumerate(a_op_name): + for index, op_name in enumerate(npu_op_name): # get cell name & class name from op_name # Cell.fc1.Dense.forward.0.input.0 cell_name = op_name.split(Const.SEP, 1)[-1].rsplit(Const.SEP, 4)[0] if cell_name in self.cell_mapping_dict: - a_op_name[index] = op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) - return a_op_name + npu_op_name[index] = op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) + return npu_op_name def check_op(self, npu_dict, bench_dict, fuzzy_match): - a_op_name = npu_dict["op_name"].copy() - b_op_name = bench_dict["op_name"].copy() + npu_op_name = npu_dict["op_name"].copy() + bench_op_name = bench_dict["op_name"].copy() if self.api_mapping is not None: - a_op_name = self.process_api_mapping(a_op_name, b_op_name) + npu_op_name = self.process_api_mapping(npu_op_name, bench_op_name) if self.cell_mapping is not None: - a_op_name = self.process_cell_mapping(a_op_name) + npu_op_name = self.process_cell_mapping(npu_op_name) struct_match = check_struct_match(npu_dict, bench_dict, cross_frame=self.cross_frame) if not fuzzy_match: - return a_op_name == b_op_name and struct_match + return npu_op_name == bench_op_name and struct_match is_match = True try: - is_match = fuzzy_check_op(a_op_name, b_op_name) + is_match = fuzzy_check_op(npu_op_name, bench_op_name) except Exception as err: - logger.warning("%s and %s can not fuzzy match." % (a_op_name, b_op_name)) + logger.warning("%s and %s can not fuzzy match." % (npu_op_name, bench_op_name)) is_match = False return is_match and struct_match @@ -76,23 +76,25 @@ class MSComparator(Comparator): data_value = load_npy(data_path) return data_value - def api_replace(self, a_op_name, target, para): - for idx, _ in enumerate(a_op_name): - a_op_name[idx] = a_op_name[idx].replace(target, para) - return a_op_name + def api_replace(self, npu_op_name, target, para): + for idx, _ in enumerate(npu_op_name): + npu_op_name[idx] = npu_op_name[idx].replace(target, para) + return npu_op_name - def process_api_mapping(self, a_op_name, b_op_name): + def process_api_mapping(self, npu_op_name, bench_op_name): # get api name & class name from op_name # Functional.addcmul.0.forward.input.0 - ms_api_name = a_op_name[0].rsplit(Const.SEP, 4)[0] - pt_api_name = b_op_name[0].rsplit(Const.SEP, 4)[0] + ms_api_name = npu_op_name[0].rsplit(Const.SEP, 4)[0] + pt_api_name = bench_op_name[0].rsplit(Const.SEP, 4)[0] class_name = ms_api_name.split(Const.SEP)[0] if class_name == "Mint": - return self.api_replace(a_op_name, "Mint", "Torch") - if class_name == "MintFunctional": - return self.api_replace(a_op_name, "MintFunctional", "Functional") - if self.ms_to_pt_mapping.get(ms_api_name) == pt_api_name: - return self.api_replace(a_op_name, ms_api_name, pt_api_name) + return self.api_replace(npu_op_name, "Mint", "Torch") + elif class_name == "MintFunctional": + return self.api_replace(npu_op_name, "MintFunctional", "Functional") + elif self.ms_to_pt_mapping.get(ms_api_name) == pt_api_name: + return self.api_replace(npu_op_name, ms_api_name, pt_api_name) + else: + return npu_op_name def ms_compare(input_param, output_path, **kwargs): diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_to_pt_api.yaml b/debug/accuracy_tools/msprobe/mindspore/compare/ms_to_pt_api.yaml new file mode 100644 index 000000000..9c37ac9be --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_to_pt_api.yaml @@ -0,0 +1,399 @@ +Functional.abs: Torch.abs +Functional.absolute: Torch.absolute +Functional.acos: Torch.acos +Functional.acosh: Torch.acosh +Functional.adaptive_avg_pool2d: Functional.adaptive_avg_pool2d +Functional.adaptive_avg_pool3d: Functional.adaptive_avg_pool3d +Functional.adaptive_max_pool2d: Functional.adaptive_max_pool2d +Functional.add: Torch.add +Functional.addbmm: Torch.addbmm +Functional.addcdiv: Torch.addcdiv +Functional.addcmul: Torch.addcmul +Functional.addmm: Torch.addmm +Functional.addmv: Torch.addmv +Functional.affine_grid: Functional.affine_grid +Functional.angle: Torch.angle +Functional.arange: Torch.arange +Functional.arccos: Torch.arccos +Functional.arccosh: Torch.arccosh +Functional.arcsinh: Torch.arcsinh +Functional.arctan: Torch.arctan +Functional.arctanh: Torch.arctanh +Functional.argmax: Torch.argmax +Functional.asin: Torch.asin +Functional.asinh: Torch.asinh +Functional.atan: Torch.atan +Functional.atan2: Torch.atan2 +Functional.atanh: Torch.atanh +Functional.baddbmm: Torch.baddbmm +Functional.bartlett_window: Torch.bartlett_window +Functional.bidense: Functional.bilinear +Functional.bincount: Torch.bincount +Functional.bitwise_and: Torch.bitwise_and +Functional.bitwise_or: Torch.bitwise_or +Functional.bitwise_xor: Torch.bitwise_xor +Functional.blackman_window: Torch.blackman_window +Functional.ceil: Torch.ceil +Functional.cholesky_solve: Torch.cholesky_solve +Functional.clamp: Torch.clamp +Functional.clip: Torch.clip +Functional.column_stack: Torch.column_stack +Functional.conj: Torch.conj +Functional.conv1d: Functional.conv1d +Functional.conv2d: Functional.conv2d +Functional.conv3d: Functional.conv3d +Functional.cosh: Torch.cosh +Functional.cosine_embedding_loss: Functional.cosine_embedding_loss +Functional.cosine_similarity: Functional.cosine_similarity +Functional.cross: Torch.cross +Functional.cross_entropy: Functional.cross_entropy +Functional.ctc_loss: Functional.ctc_loss +Functional.cumprod: Torch.cumprod +Functional.dense: Functional.linear +Functional.diag_embed: Torch.diag_embed +Functional.diagflat: Torch.diagflat +Functional.diagonal: Torch.diagonal +Functional.digamma: Torch.digamma +Functional.dist: Torch.dist +Functional.div: Torch.div +Functional.divide: Torch.divide +Functional.einsum: Torch.einsum +Functional.equal: Torch.eq +Functional.erf: Torch.erf +Functional.erfc: Torch.erfc +Functional.exp: Torch.exp +Functional.exp2: Torch.exp2 +Functional.expm1: Torch.expm1 +Functional.eye: Torch.eye +Functional.flip: Torch.flip +Functional.fliplr: Torch.fliplr +Functional.flipud: Torch.flipud +Functional.floor_divide: Torch.floor_divide +Functional.fmax: Torch.fmax +Functional.fmod: Torch.fmod +Functional.full: Torch.full +Functional.full_like: Torch.full_like +Functional.gcd: Torch.gcd +Functional.geqrf: Torch.geqrf +Functional.greater: Torch.greater +Functional.greater_equal: Torch.greater_equal +Functional.gumbel_softmax: Functional.gumbel_softmax +Functional.hamming_window: Torch.hamming_window +Functional.hann_window: Torch.hann_window +Functional.hardsigmoid: Functional.hardsigmoid +Functional.hardtanh: Functional.hardtanh +Functional.heaviside: Torch.heaviside +Functional.hinge_embedding_loss: Functional.hinge_embedding_loss +Functional.histc: Torch.histc +Functional.hsplit: Torch.hsplit +Functional.hstack: Torch.hstack +Functional.hypot: Torch.hypot +Functional.i0: Torch.i0 +Functional.igamma: Torch.igamma +Functional.igammac: Torch.igammac +Functional.imag: Torch.imag +Functional.inner: Torch.inner +Functional.inverse: Torch.inverse +Functional.is_nonzero: Torch.is_nonzero +Functional.is_tensor: Torch.is_tensor +Functional.isinf: Torch.isinf +Functional.isneginf: Torch.isneginf +Functional.isposinf: Torch.isposinf +Functional.isreal: Torch.isreal +Functional.kaiser_window: Torch.kaiser_window +Functional.kron: Torch.kron +Functional.l1_loss: Functional.l1_loss +Functional.lcm: Torch.lcm +Functional.lerp: Torch.lerp +Functional.log1p: Torch.log1p +Functional.logaddexp: Torch.logaddexp +Functional.logaddexp2: Torch.logaddexp2 +Functional.logdet: Torch.logdet +Functional.logical_and: Torch.logical_and +Functional.logical_not: Torch.logical_not +Functional.logical_or: Torch.logical_or +Functional.logical_xor: Torch.logical_xor +Functional.logit: Torch.logit +Functional.logspace: Torch.logspace +Functional.lt: Torch.lt +Functional.lu_solve: Torch.lu_solve +Functional.lu_unpack: Torch.lu_unpack +Functional.margin_ranking_loss: Functional.margin_ranking_loss +Functional.masked_select: Torch.masked_select +Functional.matmul: Torch.matmul +Functional.max_pool3d: Functional.max_pool3d +Functional.max_unpool1d: Functional.max_unpool1d +Functional.max_unpool2d: Functional.max_unpool2d +Functional.max_unpool3d: Functional.max_unpool3d +Functional.mm: Torch.mm +Functional.moveaxis: Torch.moveaxis +Functional.movedim: Torch.movedim +Functional.mse_loss: Functional.mse_loss +Functional.msort: Torch.msort +Functional.mul: Torch.mul +Functional.multi_margin_loss: Functional.multi_margin_loss +Functional.multilabel_margin_loss: Functional.multilabel_margin_loss +Functional.multilabel_soft_margin_loss: Functional.multilabel_soft_margin_loss +Functional.multiply: Torch.multiply +Functional.nan_to_num: Torch.nan_to_num +Functional.nansum: Torch.nansum +Functional.neg: Torch.neg +Functional.negative: Torch.negative +Functional.nextafter: Torch.nextafter +Functional.nll_loss: Functional.nll_loss +Functional.nonzero: Torch.nonzero +Functional.not_equal: Torch.not_equal +Functional.numel: Torch.numel +Functional.ones_like: Torch.ones_like +Functional.orgqr: Torch.orgqr +Functional.outer: Torch.outer +Functional.pdist: Functional.pdist +Functional.permute: Torch.permute +Functional.pixel_shuffle: Functional.pixel_shuffle +Functional.pixel_unshuffle: Functional.pixel_unshuffle +Functional.polar: Torch.polar +Functional.pow: Torch.pow +Functional.rand: Torch.rand +Functional.rand_like: Torch.rand_like +Functional.randn: Torch.randn +Functional.randn_like: Torch.randn_like +Functional.ravel: Torch.ravel +Functional.real: Torch.real +Functional.reciprocal: Torch.reciprocal +Functional.relu: Functional.relu +Functional.remainder: Torch.remainder +Functional.reshape: Torch.reshape +Functional.roll: Torch.roll +Functional.rot90: Torch.rot90 +Functional.round: Torch.round +Functional.row_stack: Torch.row_stack +Functional.rsqrt: Torch.rsqrt +Functional.searchsorted: Torch.searchsorted +Functional.sgn: Torch.sgn +Functional.sigmoid: Functional.sigmoid +Functional.sign: Torch.sign +Functional.signbit: Torch.signbit +Functional.sin: Torch.sin +Functional.sinc: Torch.sinc +Functional.sinh: Torch.sinh +Functional.slogdet: Torch.slogdet +Functional.smooth_l1_loss: Functional.smooth_l1_loss +Functional.soft_margin_loss: Functional.soft_margin_loss +Functional.square: Torch.square +Functional.subtract: Torch.subtract +Functional.sum: Torch.sum +Functional.swapaxes: Torch.transpose +Functional.swapdims: Torch.swapdims +Functional.t: Torch.t +Functional.tan: Torch.tan +Functional.tanh: Functional.tanh +Functional.tanhshrink: Functional.tanhshrink +Functional.topk: Torch.topk +Functional.trace: Torch.trace +Functional.trapz: Torch.trapz +Functional.tril: Torch.tril +Functional.tril_indices: Torch.tril_indices +Functional.triu_indices: Torch.triu_indices +Functional.true_divide: Torch.true_divide +Functional.trunc: Torch.trunc +Functional.unbind: Torch.unbind +Functional.unfold: Functional.unfold +Functional.unsqueeze: Torch.unsqueeze +Functional.vander: Torch.vander +Functional.view_as_real: Torch.view_as_real +Functional.vsplit: Torch.vsplit +Functional.where: Torch.where +Functional.xlogy: Torch.xlogy +Functional.zeros_like: Torch.zeros_like +Tensor.H: Tensor.H +Tensor.T: Tensor.T +Tensor.abs: Tensor.abs +Tensor.absolute: Tensor.absolute +Tensor.acos: Tensor.acos +Tensor.acosh: Tensor.acosh +Tensor.add: Tensor.add +Tensor.addbmm: Tensor.addbmm +Tensor.addcdiv: Tensor.addcdiv +Tensor.addcmul: Tensor.addcmul +Tensor.addmm: Tensor.addmm +Tensor.addr: Tensor.addr +Tensor.angle: Tensor.angle +Tensor.arccos: Tensor.arccos +Tensor.arccosh: Tensor.arccosh +Tensor.arcsin: Tensor.arcsin +Tensor.arcsinh: Tensor.arcsinh +Tensor.arctan: Tensor.arctan +Tensor.arctanh: Tensor.arctanh +Tensor.asin: Tensor.asin +Tensor.asinh: Tensor.asinh +Tensor.asnumpy: Tensor.numpy +Tensor.atan: Tensor.atan +Tensor.atan2: Tensor.atan2 +Tensor.atanh: Tensor.atanh +Tensor.baddbmm: Tensor.baddbmm +Tensor.bincount: Tensor.bincount +Tensor.bitwise_and: Tensor.bitwise_and +Tensor.bitwise_or: Tensor.bitwise_or +Tensor.bitwise_xor: Tensor.bitwise_xor +Tensor.cauchy: Tensor.cauchy_ +Tensor.ceil: Tensor.ceil +Tensor.cholesky: Tensor.cholesky +Tensor.cholesky_solve: Tensor.cholesky_solve +Tensor.chunk: Tensor.chunk +Tensor.clamp: Tensor.clamp +Tensor.clip: Tensor.clip +Tensor.conj: Tensor.conj +Tensor.contiguous: Tensor.contiguous +Tensor.copysign: Tensor.copysign +Tensor.cos: Tensor.cos +Tensor.cosh: Tensor.cosh +Tensor.cross: Tensor.cross +Tensor.cumprod: Tensor.cumprod +Tensor.cumsum: Tensor.cumsum +Tensor.deg2rad: Tensor.deg2rad +Tensor.diagflat: Tensor.diagflat +Tensor.diagonal: Tensor.diagonal +Tensor.digamma: Tensor.digamma +Tensor.div: Tensor.div +Tensor.divide: Tensor.divide +Tensor.equal: Tensor.eq +Tensor.erf: Tensor.erf +Tensor.erfc: Tensor.erfc +Tensor.erfinv: Tensor.erfinv +Tensor.exp: Tensor.exp +Tensor.expm1: Tensor.expm1 +Tensor.fill_diagonal: Tensor.fill_diagonal_ +Tensor.fliplr: Tensor.fliplr +Tensor.flipud: Tensor.flipud +Tensor.float: Tensor.float +Tensor.floor: Tensor.floor +Tensor.fmax: Tensor.fmax +Tensor.fmod: Tensor.fmod +Tensor.gather_elements: Tensor.gather +Tensor.geqrf: Tensor.geqrf +Tensor.greater: Tensor.greater +Tensor.greater_equal: Tensor.greater_equal +Tensor.half: Tensor.half +Tensor.hardshrink: Tensor.hardshrink +Tensor.heaviside: Tensor.heaviside +Tensor.histc: Tensor.histc +Tensor.hypot: Tensor.hypot +Tensor.i0: Tensor.i0 +Tensor.igamma: Tensor.igamma +Tensor.igammac: Tensor.igammac +Tensor.imag: Tensor.imag +Tensor.index_add: Tensor.index_add +Tensor.index_fill: Tensor.index_fill +Tensor.index_put: Tensor.index_put +Tensor.int: Tensor.int +Tensor.inverse: Tensor.inverse +Tensor.is_complex: Tensor.is_complex +Tensor.is_contiguous: Tensor.is_contiguous +Tensor.is_floating_point: Tensor.is_floating_point +Tensor.is_signed: Tensor.is_signed +Tensor.isfinite: Tensor.isfinite +Tensor.isinf: Tensor.isinf +Tensor.isnan: Tensor.isnan +Tensor.isneginf: Tensor.isneginf +Tensor.isposinf: Tensor.isposinf +Tensor.isreal: Tensor.isreal +Tensor.lcm: Tensor.lcm +Tensor.ldexp: Tensor.ldexp +Tensor.le: Tensor.le +Tensor.lerp: Tensor.lerp +Tensor.less: Tensor.less +Tensor.less_equal: Tensor.less_equal +Tensor.log: Tensor.log +Tensor.log10: Tensor.log10 +Tensor.log1p: Tensor.log1p +Tensor.log2: Tensor.log2 +Tensor.log_normal: Tensor.log_normal +Tensor.logaddexp: Tensor.logaddexp +Tensor.logaddexp2: Tensor.logaddexp2 +Tensor.logcumsumexp: Tensor.logcumsumexp +Tensor.logdet: Tensor.logdet +Tensor.logical_and: Tensor.logical_and +Tensor.logical_not: Tensor.logical_not +Tensor.logical_or: Tensor.logical_or +Tensor.logical_xor: Tensor.logical_xor +Tensor.logit: Tensor.logit +Tensor.long: Tensor.long +Tensor.lt: Tensor.lt +Tensor.lu_solve: Tensor.lu_solve +Tensor.mH: Tensor.mH +Tensor.mT: Tensor.mT +Tensor.masked_fill: Tensor.masked_fill +Tensor.masked_select: Tensor.masked_select +Tensor.matmul: Tensor.matmul +Tensor.maximum: Tensor.maximum +Tensor.minimum: Tensor.minimum +Tensor.mm: Tensor.mm +Tensor.movedim: Tensor.movedim +Tensor.msort: Tensor.msort +Tensor.mul: Tensor.mul +Tensor.multiply: Tensor.multiply +Tensor.mvlgamma: Tensor.mvlgamma +Tensor.nan_to_num: Tensor.nan_to_num +Tensor.nanmedian: Tensor.nanmedian +Tensor.nansum: Tensor.nansum +Tensor.ndim: Tensor.ndim +Tensor.ndimension: Tensor.ndimension +Tensor.ne: Tensor.ne +Tensor.neg: Tensor.neg +Tensor.negative: Tensor.negative +Tensor.nelement: Tensor.nelement +Tensor.new_ones: Tensor.new_ones +Tensor.new_zeros: Tensor.new_zeros +Tensor.nextafter: Tensor.nextafter +Tensor.nonzero: Tensor.nonzero +Tensor.not_equal: Tensor.not_equal +Tensor.numel: Tensor.numel +Tensor.orgqr: Tensor.orgqr +Tensor.ormqr: Tensor.ormqr +Tensor.outer: Tensor.outer +Tensor.permute: Tensor.permute +Tensor.rad2deg: Tensor.rad2deg +Tensor.ravel: Tensor.ravel +Tensor.real: Tensor.real +Tensor.reciprocal: Tensor.reciprocal +Tensor.remainder: Tensor.remainder +Tensor.repeat_interleave: Tensor.repeat_interleave +Tensor.reshape: Tensor.reshape +Tensor.reshape_as: Tensor.reshape_as +Tensor.roll: Tensor.roll +Tensor.rot90: Tensor.rot90 +Tensor.round: Tensor.round +Tensor.rsqrt: Tensor.rsqrt +Tensor.sgn: Tensor.sgn +Tensor.shape: Tensor.shape +Tensor.short: Tensor.short +Tensor.sigmoid: Tensor.sigmoid +Tensor.sign: Tensor.sign +Tensor.signbit: Tensor.signbit +Tensor.sin: Tensor.sin +Tensor.sinc: Tensor.sinc +Tensor.sinh: Tensor.sinh +Tensor.slogdet: Tensor.slogdet +Tensor.sort: Tensor.sort +Tensor.sqrt: Tensor.sqrt +Tensor.square: Tensor.square +Tensor.squeeze: Tensor.squeeze +Tensor.storage_offset: Tensor.storage_offset +Tensor.stride: Tensor.stride +Tensor.subtract: Tensor.subtract +Tensor.sum_to_size: Tensor.sum_to_size +Tensor.swapaxes: Tensor.transpose +Tensor.swapdims: Tensor.swapdims +Tensor.t: Tensor.t +Tensor.tan: Tensor.tan +Tensor.tanh: Tensor.tanh +Tensor.topk: Tensor.topk +Tensor.trace: Tensor.trace +Tensor.tril: Tensor.tril +Tensor.true_divide: Tensor.true_divide +Tensor.trunc: Tensor.trunc +Tensor.unbind: Tensor.unbind +Tensor.unsqueeze: Tensor.unsqueeze +Tensor.view: Tensor.view +Tensor.view_as: Tensor.view_as +Tensor.where: Tensor.where diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index 657cc7c24..cad9129ff 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -92,6 +92,8 @@ def main(): logger.error("MindSpore does not exist, please install MindSpore library") raise Exception("MindSpore does not exist, please install MindSpore library") if sys.argv[3] == "compare": + if isinstance(args.api_mapping, str): + logger.warning("User defined mapping tables are not supported in the current version") compare_cli(args) if __name__ == "__main__": -- Gitee From fa9e9e66431af320be48a1de71e6d5d685edc616 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 10:42:08 +0800 Subject: [PATCH 434/791] delete const.py --- .../mindspore/api_accuracy_checker/const.py | 27 ------------------- 1 file changed, 27 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py deleted file mode 100644 index cc6943a05..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/const.py +++ /dev/null @@ -1,27 +0,0 @@ -class MsCompareConst: - # api_info field - MINT = "Mint" - MINT_FUNCTIONAL = "MintFunctional" - - TASK_FIELD = "task" - STATISTICS_TASK = "statistics" - TENSOR_TASK = "tensor" - DUMP_DATA_DIR_FIELD = "dump_data_dir" - DATA_FIELD = "data" - - #detail_csv - DETAIL_CSV_API_NAME = "API Name" - DETAIL_CSV_BENCH_DTYPE = "Bench Dtype" - DETAIL_CSV_TESTED_DTYPE = "Tested Dtype" - DETAIL_CSV_SHAPE = "Shape" - DETAIL_CSV_PASS_STATUS = "Status" - DETAIL_CSV_MESSAGE = "Message" - DETAIL_CSV_FILE_NAME = "accuracy_checking_details" - - #result_csv - RESULT_CSV_FORWARD_TEST_SUCCESS = "Forward Test Success" - RESULT_CSV_BACKWARD_TEST_SUCCESS = "Backward Test Success" - RESULT_CSV_FILE_NAME = "accuracy_checking_result" - - EPSILON = 1e-8 - -- Gitee From f50737430a08215dad5b51899cce842e5a583a73 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 22 Aug 2024 10:46:04 +0800 Subject: [PATCH 435/791] add dump data missing judgment in compare --- .../accuracy_tools/msprobe/core/compare/acc_compare.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index f963272ae..33b9ed805 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -140,6 +140,16 @@ class Comparator: if both_empty or no_change: continue + # dump data missing judgment + if not npu_ops_queue and bench_ops_queue: + logger.info("The dump data of npu may be missing. For data integrity, compare the size of dump.json " + "of npu and bench is suggested.") + break + if npu_ops_queue and not bench_ops_queue: + logger.info("The dump data of bench may be missing. For data integrity, compare the size of dump.json " + "of npu and bench is suggested.") + break + n_match_point, b_match_point = self.match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) if n_match_point == -1 and b_match_point == -1: continue -- Gitee From 57864bdccade7e20eea217f2ab4aeb6ea2b7e468 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 10:49:36 +0800 Subject: [PATCH 436/791] fix torch.linalg --- .../msprobe/core/common/const.py | 4 ++++ .../compare/api_precision_compare.py | 16 +++++++++++++--- .../api_accuracy_checker/compare/compare.py | 9 ++++++++- .../api_accuracy_checker/run_ut/run_ut.py | 18 ++++++++++++++++-- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index b6cf5088d..bbee41b57 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -23,6 +23,10 @@ class Const: WHITE_LIST = 'white_list' BLACK_LIST = 'black_list' DUMP_TENSOR_DATA = 'dump_tensor_data' + THREE_SEGMENT = 3 + FOUR_SEGMENT = 4 + SIX_SEGMENT = 6 + SEVEN_SEGMENT = 7 # dump mode ALL = "all" diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 3de8774d1..8779bc200 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -19,7 +19,7 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import get_validated_res from msprobe.core.common.file_check import FileChecker, change_mode, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException -from msprobe.core.common.const import CompareConst, FileCheckConst +from msprobe.core.common.const import Const, CompareConst, FileCheckConst CompareConfig = namedtuple('CompareConfig', ['npu_csv_path', 'gpu_csv_path', 'result_csv_path', 'details_csv_path']) BenchmarkInf_Nan_Consistency = namedtuple('BenchmarkInf_Nan_Consistency', ['small_value_inf_nan_consistency', @@ -298,6 +298,16 @@ def analyse_csv(npu_data, gpu_data, config): full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status] _, api_name, _, direction_status, _, _ = full_api_name_with_direction_status.split(".") + api_parts = full_api_name_with_direction_status.split(Const.SEP) + api_parts_length = len(api_parts) + if api_parts_length == Const.SIX_SEGMENT: + api_type, api_name, api_order, direction_status, _, _ = full_api_name_with_direction_status.split(Const.SEP) + full_api_name = Const.SEP.join([api_type, api_name, api_order]) + elif api_parts_length == Const.SEVEN_SEGMENT: + api_type, prefix, api_name, api_order, direction_status, _, _ = full_api_name_with_direction_status.split(Const.SEP) + full_api_name = Const.SEP.join([api_type, prefix, api_name, api_order]) + else: + raise CompareException(CompareException.INVALID_DATA_ERROR, "This type of API has not been adapted.") if row_gpu.empty: logger.warning(f'This API : {full_api_name_with_direction_status} does not exist in the GPU data.') continue @@ -331,7 +341,7 @@ def analyse_csv(npu_data, gpu_data, config): new_status = record_benchmark_compare_result(compare_column, bs) write_detail_csv(compare_column.to_column_value(), config.details_csv_path) - if last_api_name is not None and api_name != last_api_name: + if last_api_name is not None and full_api_name != last_api_name: if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) @@ -346,7 +356,7 @@ def analyse_csv(npu_data, gpu_data, config): message = '' is_supported = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in API_PRECISION_COMPARE_UNSUPPORT_LIST - last_api_name = api_name + last_api_name = full_api_name last_api_dtype = row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] if not is_supported: diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 18387b88e..4d26e81bd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -156,7 +156,14 @@ class Comparator: self.write_detail_csv(args) def compare_output(self, full_api_name, data_info): - _, api_name, _ = full_api_name.split(Const.SEP) + api_parts = full_api_name.split(Const.SEP) + api_parts_length = len(api_parts) + if api_parts_length == Const.THREE_SEGMENT: + _, api_name, _ = api_parts + elif api_parts_length == Const.FOUR_SEGMENT: + _, _, api_name, _ = api_parts + else: + raise ValueError(f"API name {full_api_name} has not been adapted.") bench_output, device_output = data_info.bench_output, data_info.device_output bench_grad, device_grad = data_info.bench_grad, data_info.device_grad backward_message = data_info.backward_message diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index a03b9a1f8..f01da5d12 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -218,12 +218,27 @@ def run_ut(config): def run_api_offline(config, compare, api_name_set): + err_column = CompareColumn() for _, (api_full_name, api_info_dict) in enumerate(tqdm(config.forward_content.items(), **tqdm_params)): if api_full_name in api_name_set: continue if is_unsupported_api(api_full_name): continue - [_, api_name, _] = api_full_name.split(Const.SEP) + try: + api_parts = api_full_name.split(Const.SEP) + api_parts_length = len(api_parts) + if api_parts_length == Const.THREE_SEGMENT: + _, api_name, _ = api_parts + elif api_parts_length == Const.FOUR_SEGMENT: + _, _, api_name, _ = api_parts + else: + raise ValueError(f"API name {api_full_name} has not been adapted.") + except ValueError as err: + logger.error(f"Run {api_full_name} UT Error: %s" % str(err)) + fwd_compare_alg_results = err_column.to_column_value(CompareConst.SKIP, str(err)) + result_info = (api_full_name, CompareConst.SKIP, CompareConst.SKIP, [fwd_compare_alg_results], None, 0) + compare.record_results(result_info) + continue try: if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue @@ -237,7 +252,6 @@ def run_api_offline(config, compare, api_name_set): f"'int32_to_int64' list in accuracy_tools/api_accuracy_check/common/utils.py file.") else: logger.error(f"Run {api_full_name} UT Error: %s" % str(err)) - err_column = CompareColumn() fwd_compare_alg_results = err_column.to_column_value(CompareConst.SKIP, str(err)) result_info = (api_full_name, CompareConst.SKIP, CompareConst.SKIP, [fwd_compare_alg_results], None, 0) compare.record_results(result_info) -- Gitee From 0158afc11f806a2098ca5c2ede1a002c42460123 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 22 Aug 2024 11:00:04 +0800 Subject: [PATCH 437/791] clean code --- .../api_accuracy_checker/tensor_transport_layer/client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py index ced71179c..368b0229d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py @@ -140,7 +140,8 @@ class TCPClient: try: self.send_queue.put(send_data, block=True, timeout=self.QUEUE_PENDING_TIME) except Exception as e: - logger.debug(f"send_queue put send_data timeout, {str(e)}") + logger.error(f"send_queue put send_data timeout, rank: {send_data.rank}, step: {send_data.step}," + f"sequence_number: {send_data.sequence_number}, {str(e)}") def _send_data(self, data: TCPDataItem): self.tcp_manager.send_wrapped_data(data.raw_data, -- Gitee From 9db5ab8b997b65a896396b3e7a8b1d33ff3545dd Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 22 Aug 2024 11:01:17 +0800 Subject: [PATCH 438/791] add APIs in NPU and Bench models unconsistent judgment --- .../msprobe/core/compare/acc_compare.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 33b9ed805..ed732f891 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -140,14 +140,9 @@ class Comparator: if both_empty or no_change: continue - # dump data missing judgment - if not npu_ops_queue and bench_ops_queue: - logger.info("The dump data of npu may be missing. For data integrity, compare the size of dump.json " - "of npu and bench is suggested.") - break - if npu_ops_queue and not bench_ops_queue: - logger.info("The dump data of bench may be missing. For data integrity, compare the size of dump.json " - "of npu and bench is suggested.") + # APIs in NPU and Bench models unconsistent judgment + if (not npu_ops_queue and bench_ops_queue) or (npu_ops_queue and not bench_ops_queue): + logger.info("Please check whether the number and calls of APIs in NPU and Bench models are consistent.") break n_match_point, b_match_point = self.match_op(npu_ops_queue, bench_ops_queue, fuzzy_match) -- Gitee From f1e722f67b637a436f004e8ea5e11fec2093d2fe Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 11:24:35 +0800 Subject: [PATCH 439/791] fix torch.linalg --- .../api_accuracy_checker/common/utils.py | 12 ++++++++++++ .../api_accuracy_checker/compare/compare.py | 10 +++------- .../api_accuracy_checker/run_ut/run_ut.py | 19 +++++++------------ 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index fdd37337d..fbc1217d1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -163,3 +163,15 @@ class UtDataProcessor: self._save_recursive(api_name, value) else: self.index += 1 + + +def get_segment_name(api_full_name): + api_parts = api_full_name.split(Const.SEP) + api_parts_length = len(api_parts) + if api_parts_length == Const.THREE_SEGMENT: + api_type, api_name, _ = api_parts + elif api_parts_length == Const.FOUR_SEGMENT: + api_type, _, api_name, _ = api_parts + else: + api_name = None + return api_type, api_name diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 4d26e81bd..23c4b2644 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -15,6 +15,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareC from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \ DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, absolute_standard_api, binary_standard_api, \ ulp_standard_api, thousandth_standard_api, apis_threshold +from msprobe.pytorch.api_accuracy_checker.common.utils import get_segment_name from msprobe.pytorch.common.log import logger @@ -156,13 +157,8 @@ class Comparator: self.write_detail_csv(args) def compare_output(self, full_api_name, data_info): - api_parts = full_api_name.split(Const.SEP) - api_parts_length = len(api_parts) - if api_parts_length == Const.THREE_SEGMENT: - _, api_name, _ = api_parts - elif api_parts_length == Const.FOUR_SEGMENT: - _, _, api_name, _ = api_parts - else: + _, api_name = get_segment_name(full_api_name) + if not api_name: raise ValueError(f"API name {full_api_name} has not been adapted.") bench_output, device_output = data_info.bench_output, data_info.device_output bench_grad, device_grad = data_info.bench_grad, data_info.device_grad diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index f01da5d12..af84f02c6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -21,7 +21,7 @@ from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Message, hf_32_standard_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ - initialize_save_path, UtDataProcessor + initialize_save_path, UtDataProcessor, get_segment_name from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn from msprobe.pytorch.hook_module.wrap_tensor import TensorOPTemplate @@ -225,13 +225,8 @@ def run_api_offline(config, compare, api_name_set): if is_unsupported_api(api_full_name): continue try: - api_parts = api_full_name.split(Const.SEP) - api_parts_length = len(api_parts) - if api_parts_length == Const.THREE_SEGMENT: - _, api_name, _ = api_parts - elif api_parts_length == Const.FOUR_SEGMENT: - _, _, api_name, _ = api_parts - else: + _, api_name = get_segment_name(api_full_name) + if not api_name: raise ValueError(f"API name {api_full_name} has not been adapted.") except ValueError as err: logger.error(f"Run {api_full_name} UT Error: %s" % str(err)) @@ -283,7 +278,7 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - [_, api_name, _] = api_full_name.split(Const.SEP) + api_type, api_name = get_segment_name(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue dispatcher.update_consume_queue(api_data) @@ -304,7 +299,7 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - [_, api_name, _] = api_full_name.split(Const.SEP) + _, api_name = get_segment_name(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue dispatcher.update_consume_queue(api_data) @@ -351,7 +346,7 @@ def do_save_error_data(api_full_name, data_info, error_data_path, is_fwd_success def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict): in_fwd_data_list = [] backward_message = '' - [api_type, api_name, _] = api_full_name.split(Const.SEP) + api_type, api_name = get_segment_name(api_full_name) args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path) in_fwd_data_list.append(args) in_fwd_data_list.append(kwargs) @@ -396,7 +391,7 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict def run_torch_api_online(api_full_name, api_data, backward_content): in_fwd_data_list = [] - [api_type, api_name, _] = api_full_name.split(Const.SEP) + api_type, api_name = get_segment_name(api_full_name) args, kwargs, out = api_data.args, api_data.kwargs, api_data.result in_fwd_data_list.append(args) in_fwd_data_list.append(kwargs) -- Gitee From fa06b26bed2182b8778164a1f110eb5a6d2cf788 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 11:25:29 +0800 Subject: [PATCH 440/791] fix torch.linalg --- .../api_accuracy_checker/compare/api_precision_compare.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 8779bc200..5bf97397b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -297,7 +297,6 @@ def analyse_csv(npu_data, gpu_data, config): compare_column = ApiPrecisionOutputColumn() full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status] - _, api_name, _, direction_status, _, _ = full_api_name_with_direction_status.split(".") api_parts = full_api_name_with_direction_status.split(Const.SEP) api_parts_length = len(api_parts) if api_parts_length == Const.SIX_SEGMENT: -- Gitee From 7522bfdf756d9b85b6c31be2f7e6b7d991b39dde Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 11:35:43 +0800 Subject: [PATCH 441/791] fix --- .../pytorch/api_accuracy_checker/common/utils.py | 6 ++++-- .../pytorch/api_accuracy_checker/run_ut/run_ut.py | 14 +++++++++----- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index fbc1217d1..4498b74c8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -166,12 +166,14 @@ class UtDataProcessor: def get_segment_name(api_full_name): + api_type = None + prefix = None api_parts = api_full_name.split(Const.SEP) api_parts_length = len(api_parts) if api_parts_length == Const.THREE_SEGMENT: api_type, api_name, _ = api_parts elif api_parts_length == Const.FOUR_SEGMENT: - api_type, _, api_name, _ = api_parts + api_type, prefix, api_name, _ = api_parts else: api_name = None - return api_type, api_name + return api_type, prefix, api_name diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index af84f02c6..cfe984bc2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -225,7 +225,7 @@ def run_api_offline(config, compare, api_name_set): if is_unsupported_api(api_full_name): continue try: - _, api_name = get_segment_name(api_full_name) + _, _, api_name = get_segment_name(api_full_name) if not api_name: raise ValueError(f"API name {api_full_name} has not been adapted.") except ValueError as err: @@ -278,7 +278,7 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - api_type, api_name = get_segment_name(api_full_name) + _, _, api_name = get_segment_name(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue dispatcher.update_consume_queue(api_data) @@ -299,7 +299,7 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - _, api_name = get_segment_name(api_full_name) + _, _, api_name = get_segment_name(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue dispatcher.update_consume_queue(api_data) @@ -346,7 +346,9 @@ def do_save_error_data(api_full_name, data_info, error_data_path, is_fwd_success def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict): in_fwd_data_list = [] backward_message = '' - api_type, api_name = get_segment_name(api_full_name) + api_type, prefix, api_name = get_segment_name(api_full_name) + if prefix: + api_name = prefix + Const.SEP + api_name args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path) in_fwd_data_list.append(args) in_fwd_data_list.append(kwargs) @@ -391,7 +393,9 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict def run_torch_api_online(api_full_name, api_data, backward_content): in_fwd_data_list = [] - api_type, api_name = get_segment_name(api_full_name) + api_type, prefix, api_name = get_segment_name(api_full_name) + if prefix: + api_name = prefix + Const.SEP + api_name args, kwargs, out = api_data.args, api_data.kwargs, api_data.result in_fwd_data_list.append(args) in_fwd_data_list.append(kwargs) -- Gitee From 1d6593a602bf72082b90dc48a25dd8d5bbf6abb4 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 14:14:41 +0800 Subject: [PATCH 442/791] fix bug --- .../msprobe/pytorch/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 23c4b2644..eab4f531a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -157,7 +157,7 @@ class Comparator: self.write_detail_csv(args) def compare_output(self, full_api_name, data_info): - _, api_name = get_segment_name(full_api_name) + _, _, api_name = get_segment_name(full_api_name) if not api_name: raise ValueError(f"API name {full_api_name} has not been adapted.") bench_output, device_output = data_info.bench_output, data_info.device_output -- Gitee From a9fe87a434a4452e87334e50fd7b857ca0bb45fb Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 22 Aug 2024 14:40:40 +0800 Subject: [PATCH 443/791] add APIs in NPU and Bench models unconsistent judgment --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index ed732f891..197f582dc 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -141,7 +141,10 @@ class Comparator: continue # APIs in NPU and Bench models unconsistent judgment - if (not npu_ops_queue and bench_ops_queue) or (npu_ops_queue and not bench_ops_queue): + if not npu_ops_queue and bench_ops_queue: + logger.info("Please check whether the number and calls of APIs in NPU and Bench models are consistent.") + break + elif npu_ops_queue and not bench_ops_queue: logger.info("Please check whether the number and calls of APIs in NPU and Bench models are consistent.") break -- Gitee From fc5e0c49505c4c1c26b288c2c1c5bd99035576f5 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Thu, 22 Aug 2024 14:51:58 +0800 Subject: [PATCH 444/791] fix pre hook arg nums --- debug/accuracy_tools/msprobe/mindspore/cell_processor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/cell_processor.py b/debug/accuracy_tools/msprobe/mindspore/cell_processor.py index ef943b954..370eb657b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/cell_processor.py +++ b/debug/accuracy_tools/msprobe/mindspore/cell_processor.py @@ -21,13 +21,13 @@ class CellProcessor: return CellProcessor.cell_count[cell_name] def node_hook(self, name_prefix, start_or_stop, **kwargs): - def begin_hook(cell, input, output=None): + def begin_hook(cell, input): index = self.set_cell_count(name_prefix) cell.mindstudio_reserved_name = full_name = name_prefix + Const.SEP + str(index) if self.scope: self.scope.begin_module(full_name) - def end_hook(cell, input, output=None): + def end_hook(cell, input, output): if self.scope: self.scope.end_module(cell.mindstudio_reserved_name) -- Gitee From 77372eb04844875242034b5958fce2b4efefa5ba Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 14:52:03 +0800 Subject: [PATCH 445/791] fix bug --- .../api_accuracy_checker/common/utils.py | 20 +++++++++++++++- .../compare/api_precision_compare.py | 23 +++++++++++-------- .../api_accuracy_checker/run_ut/run_ut.py | 12 +++++----- 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 4498b74c8..761e31b87 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -165,7 +165,7 @@ class UtDataProcessor: self.index += 1 -def get_segment_name(api_full_name): +def extract_basic_api_segments(api_full_name): api_type = None prefix = None api_parts = api_full_name.split(Const.SEP) @@ -177,3 +177,21 @@ def get_segment_name(api_full_name): else: api_name = None return api_type, prefix, api_name + + +def extract_detailed_api_segments(full_api_name_with_direction_status): + api_type = None + prefix = None + api_name = None + api_parts = full_api_name_with_direction_status.split(Const.SEP) + api_parts_length = len(api_parts) + if api_parts_length == Const.SIX_SEGMENT: + api_type, api_name, api_order, direction_status, _, _ = api_parts + full_api_name = Const.SEP.join([api_type, api_name, api_order]) + elif api_parts_length == Const.SEVEN_SEGMENT: + api_type, prefix, api_name, api_order, direction_status, _, _ = api_parts + full_api_name = Const.SEP.join([api_type, prefix, api_name, api_order]) + else: + full_api_name = None + return api_name, full_api_name, direction_status + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 5bf97397b..76a037644 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -16,6 +16,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECI check_inf_or_nan from msprobe.pytorch.api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path +from msprobe.pytorch.api_accuracy_checker.common.utils import extract_detailed_api_segments from msprobe.core.common.file_check import FileChecker, change_mode, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException @@ -297,16 +298,18 @@ def analyse_csv(npu_data, gpu_data, config): compare_column = ApiPrecisionOutputColumn() full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status] - api_parts = full_api_name_with_direction_status.split(Const.SEP) - api_parts_length = len(api_parts) - if api_parts_length == Const.SIX_SEGMENT: - api_type, api_name, api_order, direction_status, _, _ = full_api_name_with_direction_status.split(Const.SEP) - full_api_name = Const.SEP.join([api_type, api_name, api_order]) - elif api_parts_length == Const.SEVEN_SEGMENT: - api_type, prefix, api_name, api_order, direction_status, _, _ = full_api_name_with_direction_status.split(Const.SEP) - full_api_name = Const.SEP.join([api_type, prefix, api_name, api_order]) - else: - raise CompareException(CompareException.INVALID_DATA_ERROR, "This type of API has not been adapted.") + try: + api_name, full_api_name, direction_status = extract_detailed_api_segments(full_api_name_with_direction_status) + if not full_api_name: + raise CompareException(CompareException.INVALID_DATA_ERROR, "This type of API has not been adapted.") + except Exception as err: + logger.error(f"Compare Error: %s" % str(err)) + compare_column.api_name = full_api_name_with_direction_status + compare_column.compare_result = CompareConst.SKIP + compare_column.compare_message = str(err) + write_detail_csv(compare_column.to_column_value(), config.details_csv_path) + write_csv([[full_api_name_with_direction_status, "skip", "skip", str(err)]], config.result_csv_path) + continue if row_gpu.empty: logger.warning(f'This API : {full_api_name_with_direction_status} does not exist in the GPU data.') continue diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index cfe984bc2..1eb8a025d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -21,7 +21,7 @@ from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Message, hf_32_standard_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ - initialize_save_path, UtDataProcessor, get_segment_name + initialize_save_path, UtDataProcessor, extract_basic_api_segments from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn from msprobe.pytorch.hook_module.wrap_tensor import TensorOPTemplate @@ -225,7 +225,7 @@ def run_api_offline(config, compare, api_name_set): if is_unsupported_api(api_full_name): continue try: - _, _, api_name = get_segment_name(api_full_name) + _, _, api_name = extract_basic_api_segments(api_full_name) if not api_name: raise ValueError(f"API name {api_full_name} has not been adapted.") except ValueError as err: @@ -278,7 +278,7 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - _, _, api_name = get_segment_name(api_full_name) + _, _, api_name = extract_basic_api_segments(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue dispatcher.update_consume_queue(api_data) @@ -299,7 +299,7 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - _, _, api_name = get_segment_name(api_full_name) + _, _, api_name = extract_basic_api_segments(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue dispatcher.update_consume_queue(api_data) @@ -346,7 +346,7 @@ def do_save_error_data(api_full_name, data_info, error_data_path, is_fwd_success def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict): in_fwd_data_list = [] backward_message = '' - api_type, prefix, api_name = get_segment_name(api_full_name) + api_type, prefix, api_name = extract_basic_api_segments(api_full_name) if prefix: api_name = prefix + Const.SEP + api_name args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path) @@ -393,7 +393,7 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict def run_torch_api_online(api_full_name, api_data, backward_content): in_fwd_data_list = [] - api_type, prefix, api_name = get_segment_name(api_full_name) + api_type, prefix, api_name = extract_basic_api_segments(api_full_name) if prefix: api_name = prefix + Const.SEP + api_name args, kwargs, out = api_data.args, api_data.kwargs, api_data.result -- Gitee From f4ca7ef923b1a51c3bc8a755e643a185643a93b5 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 14:53:47 +0800 Subject: [PATCH 446/791] fix bug --- .../msprobe/pytorch/api_accuracy_checker/compare/compare.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index eab4f531a..2cc022d72 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -15,7 +15,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareC from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import check_dtype_comparable, \ DETAIL_TEST_ROWS, precision_configs, BENCHMARK_COMPARE_SUPPORT_LIST, absolute_standard_api, binary_standard_api, \ ulp_standard_api, thousandth_standard_api, apis_threshold -from msprobe.pytorch.api_accuracy_checker.common.utils import get_segment_name +from msprobe.pytorch.api_accuracy_checker.common.utils import extract_basic_api_segments from msprobe.pytorch.common.log import logger @@ -157,7 +157,7 @@ class Comparator: self.write_detail_csv(args) def compare_output(self, full_api_name, data_info): - _, _, api_name = get_segment_name(full_api_name) + _, _, api_name = extract_basic_api_segments(full_api_name) if not api_name: raise ValueError(f"API name {full_api_name} has not been adapted.") bench_output, device_output = data_info.bench_output, data_info.device_output -- Gitee From 238b2d4b5bda270c7cc5f554371eea7eae2764eb Mon Sep 17 00:00:00 2001 From: jijiarong Date: Thu, 22 Aug 2024 14:54:22 +0800 Subject: [PATCH 447/791] code clean --- .../msprobe/core/common/const.py | 19 +++ .../core/compare/multiprocessing_compute.py | 4 +- .../msprobe/core/compare/npy_compare.py | 63 ++++----- .../mindspore/compare/ms_graph_compare.py | 120 +++++++++--------- 4 files changed, 104 insertions(+), 102 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 9f72b93ad..426083917 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -215,6 +215,19 @@ class CompareConst: MAX_RELATIVE_OUT_RED = 0.5 MAX_RELATIVE_OUT_YELLOW = 0.1 MAX_RELATIVE_IN_YELLOW = 0.01 + MS_GRAPH_BASE = { + NPU_NAME: None, BENCH_NAME: None, NPU_DTYPE: None, BENCH_DTYPE: None, NPU_SHAPE: None, BENCH_SHAPE: None, + NPU_MAX: None, NPU_MIN: None, NPU_MEAN: None, NPU_NORM: None, BENCH_MAX: None, BENCH_MIN: None, + BENCH_MEAN: None, BENCH_NORM: None, ACCURACY: '', ERROR_MESSAGE: '' + } + MS_GRAPH_NPY = { + COSINE: None, MAX_ABS_ERR: None, MAX_RELATIVE_ERR: None, ONE_THOUSANDTH_ERR_RATIO: None, + FIVE_THOUSANDTHS_ERR_RATIO: None + } + MS_GRAPH_STATISTIC = { + MAX_DIFF: None, MIN_DIFF: None, MEAN_DIFF: None, NORM_DIFF: None, MAX_RELATIVE_ERR: None, + MIN_RELATIVE_ERR: None, MEAN_RELATIVE_ERR: None, NORM_RELATIVE_ERR: None + } class FileCheckConst: @@ -268,3 +281,9 @@ class MsgConst: Class for log messages const """ SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] + + +class GraphMode: + NPY_MODE = "NPY_MODE" + STATISTIC_MODE = "STATISTIC_MODE" + ERROR_MODE = "ERROR_MODE" diff --git a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py index c39176fe2..7d36062b2 100644 --- a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py @@ -42,7 +42,7 @@ def _handle_multi_process(func, input_parma, result_df, lock): def _ms_graph_handle_multi_process(func, result_df, mode): - process_num = int((multiprocessing.cpu_count() + 1) / 2) + process_num = int((multiprocessing.cpu_count() + 1) // 2) df_chunk_size = len(result_df) // process_num if df_chunk_size > 0: df_chunks = [result_df.iloc[i:i + df_chunk_size] for i in range(0, len(result_df), df_chunk_size)] @@ -59,7 +59,7 @@ def _ms_graph_handle_multi_process(func, result_df, mode): except OSError as e: logger.error("pool terminate failed") - for _, df_chunk in enumerate(df_chunks): + for df_chunk in df_chunks: result = pool.apply_async(func, args=(df_chunk, mode), error_callback=err_call) results.append(result) final_results = [r.get() for r in results] diff --git a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py index 39bb87826..82acecd83 100644 --- a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py @@ -77,56 +77,45 @@ def get_error_message(n_value, b_value, npu_op_name, error_flag, error_file=None def npy_data_check(n_value, b_value): - error_flag = False error_message = "" if n_value is None or b_value is None: - error_flag = True - error_message += "Dump file not found." + error_message += "Dump file not found.\n" - if not error_flag: - if n_value.size == 0 or b_value.size == 0: - error_flag = True - error_message += "This is empty data, can not compare." + # 检查 n_value 和 b_value 是否为空 + if not error_message and (n_value.size == 0 or b_value.size == 0): + error_message += "This is empty data, can not compare.\n" - if not error_flag: + if not error_message: if not n_value.shape or not b_value.shape: - error_flag = True - error_message += "This is type of scalar data, can not compare." + error_message += "This is type of scalar data, can not compare.\n" if n_value.shape != b_value.shape: - error_flag = True - error_message += "Shape of NPU and bench Tensor do not match." + error_message += "Shape of NPU and bench Tensor do not match.\n" if n_value.dtype != b_value.dtype: - error_flag = True - error_message += "Dtype of NPU and bench Tensor do not match. Skipped." + error_message += "Dtype of NPU and bench Tensor do not match. Skipped.\n" - if not error_flag: - n_value, b_value = handle_inf_nan(n_value, b_value) # 判断是否有nan/inf数据 - if n_value is CompareConst.NAN or b_value is CompareConst.NAN: - error_flag = True - error_message += "The position of inf or nan in NPU and bench Tensor do not match." + if not error_message: + n_value, b_value = handle_inf_nan(n_value, b_value) # 判断是否有 nan/inf 数据 + if CompareConst.NAN in (n_value, b_value): + error_message += "The position of inf or nan in NPU and bench Tensor do not match.\n" - return error_flag, error_message + return bool(error_message), error_message def statistics_data_check(result_dict): - error_flag = False error_message = "" - if result_dict['NPU Name'] is None or result_dict['NPU Name'] is None: - error_flag = True - error_message += "Dump file not found." - - if not error_flag: - if not result_dict['NPU Tensor Shape'] or not result_dict['Bench Tensor Shape']: - error_flag = True - error_message = "This is type of scalar data, can not compare." - if result_dict['NPU Tensor Shape'] != result_dict['Bench Tensor Shape']: - error_flag = True - error_message += "This is type of scalar data, can not compare." - if result_dict['Bench Dtype'] != result_dict['Bench Dtype']: - error_flag = True - error_message += "Dtype of NPU and bench Tensor do not match. Skipped." - - return error_flag, error_message + + if result_dict.get(CompareConst.NPU_NAME) is None or result_dict.get(CompareConst.BENCH_NAME) is None: + error_message += "Dump file not found.\n" + + if not result_dict.get(CompareConst.NPU_SHAPE) or not result_dict.get(CompareConst.BENCH_SHAPE): + error_message += "This is type of scalar data, can not compare.\n" + elif result_dict.get(CompareConst.NPU_SHAPE) != result_dict.get(CompareConst.BENCH_SHAPE): + error_message += "Tensor shapes do not match.\n" + + if result_dict.get(CompareConst.NPU_DTYPE) != result_dict.get(CompareConst.BENCH_DTYPE): + error_message += "Dtype of NPU and bench Tensor do not match. Skipped.\n" + + return bool(error_message), error_message class TensorComparisonBasic(abc.ABC): diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index 16d09403b..259377f42 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -2,35 +2,25 @@ import csv import glob import os import sys +import copy import numpy as np import pandas as pd -from msprobe.core.common.const import CompareConst +from msprobe.core.common.const import CompareConst, GraphMode from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.file_check import create_directory from msprobe.core.common.log import logger from msprobe.core.common.utils import add_time_with_xlsx, CompareException from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply - +from msprobe.core.common.file_check import FileOpen class row_data: def __init__(self, mode): - self.basic_data = { - CompareConst.NPU_NAME: None, CompareConst.BENCH_NAME: None, CompareConst.NPU_DTYPE: None, CompareConst.BENCH_DTYPE: None, CompareConst.NPU_SHAPE: None, - CompareConst.BENCH_SHAPE: None, CompareConst.NPU_MAX: None, CompareConst.NPU_MIN: None, CompareConst.NPU_MEAN: None, CompareConst.NPU_NORM: None, - CompareConst.BENCH_MAX: None, CompareConst.BENCH_MIN: None, CompareConst.BENCH_MEAN: None, CompareConst.BENCH_NORM: None, - CompareConst.ACCURACY: '', CompareConst.ERROR_MESSAGE: '' - } - self.npy_data = { - CompareConst.COSINE: None, CompareConst.MAX_ABS_ERR: None, CompareConst.MAX_RELATIVE_ERR: None, CompareConst.ONE_THOUSANDTH_ERR_RATIO: None, - CompareConst.FIVE_THOUSANDTHS_ERR_RATIO: None - } - self.statistic_data = { - CompareConst.MAX_DIFF: None, CompareConst.MIN_DIFF: None, CompareConst.MEAN_DIFF: None, CompareConst.NORM_DIFF: None, CompareConst.MAX_RELATIVE_ERR: None, - CompareConst.MIN_RELATIVE_ERR: None, CompareConst.MEAN_RELATIVE_ERR: None, CompareConst.NORM_RELATIVE_ERR: None - } - if mode == 'NPY_MODE': + self.basic_data = copy.deepcopy(CompareConst.MS_GRAPH_BASE) + self.npy_data = copy.deepcopy(CompareConst.MS_GRAPH_NPY) + self.statistic_data = copy.deepcopy(CompareConst.MS_GRAPH_STATISTIC) + if mode == GraphMode.NPY_MODE: self.data = {**self.basic_data, **self.npy_data} else: self.data = {**self.basic_data, **self.statistic_data} @@ -47,19 +37,47 @@ def generate_step(npu_path, rank_id): data_path = os.path.join(rank_path, path) for graph_path in os.listdir(data_path): step_set.update([int(i) for i in os.listdir(os.path.join(data_path, graph_path))]) - return sorted(list(step_set)) + return sorted(step_set) def generate_path_by_rank_step(base_path, rank_id, step_id): path_with_rank_id = os.path.join(base_path, f"rank_{rank_id}") for path in os.listdir(path_with_rank_id): if path not in ["execution_order", "graphs"]: - # TODO: graph id path need remove + return os.path.join(path_with_rank_id, path, "*", str(step_id)) logger.error(f"Data_path {path_with_rank_id} is not exist.") return '' +def statistic_data_read(statistic_file_list, statistic_file_path): + data_list = [] + statistic_data_list = [] + for statistic_file in statistic_file_list: + with open(statistic_file, "r") as f: + csv_reader = csv.reader(f, delimiter=",") + header = next(csv_reader) + header_index = {'Data Type': None, 'Shape': None, 'Max Value': None, 'Min Value': None, + 'Avg Value': None, 'L2Norm Value': None} + for key in header_index.keys(): + for index, value in enumerate(header): + if key == value: + header_index[key] = index + for key in header_index.keys(): + if header_index[key] is None: + logger.error(f"Data_path {statistic_file_path} has no key {key}") + raise FileCheckException(f"Data_path {statistic_file_path} has no key {key}") + statistic_data_list.extend([row for row in csv_reader]) + + for data in statistic_data_list: + compare_key = f"{data[1]}.{data[2]}.{data[3]}.{data[5]}" + timestamp = int(data[4]) + data_list.append( + [statistic_file_path, compare_key, timestamp, data[header_index['Data Type']], + data[header_index['Shape']], data[header_index['Max Value']], data[header_index['Min Value']], + data[header_index['Avg Value']], data[header_index['L2Norm Value']]]) + + def generate_data_name(data_path): data_list = [] @@ -71,19 +89,20 @@ def generate_data_name(data_path): statistic_file_list = glob.glob(statistic_path) npy_file_list = glob.glob(npy_path) - mapping_exist = True if mapping_file_list else False - statistic_exist = True if statistic_file_list else False - npy_exist = True if npy_file_list else False + mapping_exist = bool(mapping_file_list) + statistic_exist = bool(statistic_file_list) + npy_exist = bool(npy_file_list) + + mapping_dict = [] + if mapping_exist: + for mapping_file in mapping_file_list: + with FileOpen(mapping_file, "r") as f: + csv_reader = csv.reader(f, delimiter=",") + header = next(csv_reader) + for row in csv_reader: + mapping_dict[row[0]] = row[1] if npy_exist: - mapping_dict = [] - if mapping_exist: - for mapping_file in mapping_file_list: - with open(mapping_file, "r") as f: - csv_reader = csv.reader(f, delimiter=",") - header = next(csv_reader) - for row in csv_reader: - mapping_dict[row[0]] = row[1] for data in npy_file_list: if data in mapping_dict: split_list = mapping_dict[data].split(".") @@ -94,37 +113,14 @@ def generate_data_name(data_path): data_list.append([os.path.join(data_path, data), compare_key, timestamp]) elif statistic_exist: - statistic_data_list = [] - for statistic_file in statistic_file_list: - with open(statistic_file, "r") as f: - csv_reader = csv.reader(f, delimiter=",") - header = next(csv_reader) - header_index = {'Data Type': None, 'Shape': None, 'Max Value': None, 'Min Value': None, - 'Avg Value': None, 'L2Norm Value': None} - for key in header_index.keys(): - for index, value in enumerate(header): - if key == value: - header_index[key] = index - for key in header_index.keys(): - if header_index[key] is None: - logger.error(f"Data_path {data_path} has no key {key}") - raise FileCheckException(f"Data_path {data_path} has no key {key}") - statistic_data_list.extend([row for row in csv_reader]) - - for data in statistic_data_list: - compare_key = f"{data[1]}.{data[2]}.{data[3]}.{data[5]}" - timestamp = int(data[4]) - data_list.append( - [os.path.join(data_path, statistic_path), compare_key, timestamp, data[header_index['Data Type']], - data[header_index['Shape']], data[header_index['Max Value']], data[header_index['Min Value']], - data[header_index['Avg Value']], data[header_index['L2Norm Value']]]) + data_list = statistic_data_read(statistic_file_list, os.path.join(data_path, statistic_path)) if npy_exist: - mode = "NPY_MODE" + mode = GraphMode.NPY_MODE elif statistic_exist: - mode = "STATISTIC_MODE" + mode = GraphMode.STATISTIC_MODE else: - mode = "ERROR_MODE" + mode = GraphMode.ERROR_MODE logger.error(f"Error mode.") return mode, data_list @@ -151,7 +147,7 @@ class GraphMSComparator: def compare_ops(compare_result_db, mode): def npy_mode_compute(row): - result_dict = row_data('NPY_MODE')() + result_dict = row_data(GraphMode.NPY_MODE)() n_value = None b_value = None @@ -213,7 +209,6 @@ class GraphMSComparator: error_flag, error_message = statistics_data_check(result_dict) result_dict[CompareConst.ERROR_MESSAGE] += error_message if not error_flag: - # TODO: check Algorithms result_dict[CompareConst.MAX_DIFF] = np.abs( result_dict[CompareConst.NPU_MAX] - result_dict[CompareConst.BENCH_MAX]) result_dict[CompareConst.MIN_DIFF] = np.abs( @@ -245,7 +240,7 @@ class GraphMSComparator: return pd.Series(result_dict) - if mode == "NPY_MODE": + if mode == GraphMode.NPY_MODE: compare_result_db = compare_result_db.apply(npy_mode_compute, axis=1) else: compare_result_db = compare_result_db.apply(statistic_mode_compute, axis=1) @@ -273,7 +268,6 @@ class GraphMSComparator: compare_result_df = self._do_multi_process(compare_result_df, mode) compare_result_name = add_time_with_xlsx(f"compare_result_{str(rank_id)}_{str(rank_id)}") compare_result_path = os.path.join(os.path.realpath(self.output_path), f"{compare_result_name}") - # TODO:重新排布 compare_result_df.to_excel(compare_result_path, index=False) logger.info(f"Compare rank: {rank_id} step: {step_id} finish. Compare result: {compare_result_path}.") @@ -289,7 +283,7 @@ class GraphMSComparator: match_mode, match_data_list = generate_data_name(bench_data_path) if npu_mode == "ERROR_MODE" or match_mode == "ERROR_MODE": - logger.error(f"Data_path {npu_data_path} or {bench_data_path} is not exist.") + logger.warning(f"Data_path {npu_data_path} or {bench_data_path} is not exist.") return [], '' if npu_mode != match_mode: logger.error(f"NPU mode {npu_mode} not equal to MATCH mode {match_mode}.") @@ -320,10 +314,10 @@ class GraphMSComparator: def _do_multi_process(self, result_df, mode): try: result_df = _ms_graph_handle_multi_process(self.compare_ops, result_df, mode) - return result_df except ValueError as e: logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e + return result_df def ms_graph_compare(inputs, outputs): -- Gitee From 5e87927dfcc8751481d40499b254dfaf7eb2e2e5 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 14:57:09 +0800 Subject: [PATCH 448/791] ut --- .../api_accuracy_checker.py | 45 ++++++++++++---- .../files/api_info_statistics.json | 35 ++++++++++++ .../files/api_info_tensor.json | 35 ++++++++++++ .../test_api_accuracy_checker.py | 54 +++++++++++++++++++ 4 files changed, 158 insertions(+), 11 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index d2db4f10c..818b30c3e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -78,6 +78,9 @@ class ApiAccuracyChecker: api_input_aggregation: ApiInputAggregation forward_or_backward: str: Union["forward", "backward"] + Return: + output_list: List[tuple(str, str, BasicInfoAndStatus, dict{str: CompareResult})] + Description: get mindspore api output, run torch api and get output. compare output. @@ -92,6 +95,7 @@ class ApiAccuracyChecker: bench_outputs = api_runner(api_input_aggregation, api_name_str, forward_or_backward, Const.PT_FRAMEWORK) # compare output + output_list = [] for i, (bench_out, tested_out) in enumerate(zip(bench_outputs, tested_outputs)): api_name_with_slot = Const.SEP.join([api_name_str, forward_or_backward, Const.OUTPUT, str(i)]) bench_dtype = bench_out.get_dtype() @@ -111,9 +115,10 @@ class ApiAccuracyChecker: status = CompareConst.ERROR err_msg = compare_result_dict.get(CompareConst.COSINE).err_msg + \ compare_result_dict.get(CompareConst.MAX_ABS_ERR).err_msg - self.record(api_name_str, forward_or_backward, - BasicInfoAndStatus(api_name_with_slot, bench_dtype, tested_dtype, shape, status, err_msg), - compare_result_dict) + basic_info_status = \ + BasicInfoAndStatus(api_name_with_slot, bench_dtype, tested_dtype, shape, status, err_msg) + output_list.append(tuple([api_name_str, forward_or_backward, basic_info_status, compare_result_dict])) + return output_list def run_and_compare(self): for api_name_str, api_info in self.api_infos.items(): @@ -123,20 +128,38 @@ class ApiAccuracyChecker: forward_inputs = api_info.get_compute_element_list(Const.FORWARD, Const.INPUT) kwargs = api_info.get_kwargs() forward_inputs_aggregation = ApiInputAggregation(forward_inputs, kwargs, None) - self.run_and_compare_helper(api_info, api_name_str, forward_inputs_aggregation, Const.FORWARD) + forward_output_list = None + try: + forward_output_list = \ + self.run_and_compare_helper(api_info, api_name_str, forward_inputs_aggregation, Const.FORWARD) + except Exception as e: + logger.warning(f"exception occurs when running and comparing {api_name_str} forward api" + f"detailed exception information: {e}") + self.record(forward_output_list) if not api_info.check_backward_info(): logger.warning(f"api: {api_name_str} is lack of backward infomation, skip backward check") continue gradient_inputs = api_info.get_compute_element_list(Const.BACKWARD, Const.INPUT) backward_inputs_aggregation = ApiInputAggregation(forward_inputs, kwargs, gradient_inputs) - self.run_and_compare_helper(api_info, api_name_str, backward_inputs_aggregation, Const.BACKWARD) - - def record(self, api_real_name, forward_or_backward, basic_info, compare_result_dict): - key = tuple([api_real_name, forward_or_backward]) - if key not in self.results: - self.results[key] = [] - self.results[key].append(tuple([basic_info, compare_result_dict])) + backward_output_list = None + try: + backward_output_list = \ + self.run_and_compare_helper(api_info, api_name_str, backward_inputs_aggregation, Const.BACKWARD) + except Exception as e: + logger.warning(f"exception occurs when running and comparing {api_name_str} backward api" + f"detailed exception information: {e}") + self.record(backward_output_list) + + def record(self, output_list): + if output_list is None: + return + for output in output_list: + api_real_name, forward_or_backward, basic_info, compare_result_dict = output + key = tuple([api_real_name, forward_or_backward]) + if key not in self.results: + self.results[key] = [] + self.results[key].append(tuple([basic_info, compare_result_dict])) def to_detail_csv(self, csv_dir): diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json new file mode 100644 index 000000000..886418888 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json @@ -0,0 +1,35 @@ +{ + "task": "statistics", + "dump_data_dir": "", + "data":{ + "MintFunctional.relu.0.forward": { + "input_args": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + } + ], + "input_kwargs": {}, + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + } + ] + } + } +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json new file mode 100644 index 000000000..886418888 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json @@ -0,0 +1,35 @@ +{ + "task": "statistics", + "dump_data_dir": "", + "data":{ + "MintFunctional.relu.0.forward": { + "input_args": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + } + ], + "input_kwargs": {}, + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + } + ] + } + } +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py new file mode 100644 index 000000000..da2c60de9 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py @@ -0,0 +1,54 @@ +import unittest +import sys +import logging +import os +import json + +from msprobe.mindspore.api_accuracy_checker.api_accuracy_checker import ApiAccuracyChecker + +logging.basicConfig(stream = sys.stdout, level = logging.INFO, format = '[%(levelname)s] %(message)s') +logger = logging.getLogger(__name__) + +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + +def delete_files_with_prefix(directory, prefix): + for file_or_folder in os.listdir(directory): + full_path = os.path.join(directory, file_or_folder) + if os.path.isfile(full_path) and file_or_folder.startswith(prefix): + os.remove(full_path) + print(f"已删除文件: {full_path}") + +def modify_tensor_api_info_json(json_file_path): + with open(json_file_path, 'r') as json_file: + data = json.load(json_file) + if 'dump_data_dir' in data: + data['dump_data_dir'] = os.path.join(directory, "files") + with open(json_file_path, 'w') as json_file: + json.dump(data, json_file, indent=4) + + +class TestApiAccuracyChecker(unittest.TestCase): + + def test_statistics_mode(self): + api_info_statistics_path = os.path.join(directory, "files", "api_info_statistics.json") + result_directory = os.path.join(directory, "files") + api_accuracy_checker = ApiAccuracyChecker() + api_accuracy_checker.parse(api_info_statistics_path) + api_accuracy_checker.run_and_compare() + api_accuracy_checker.to_detail_csv() + api_accuracy_checker.to_result_csv(result_directory) + delete_files_with_prefix(result_directory, "accuracy_checking") + + def test_tensor_mode(self): + api_info_tensor_path = os.path.join(directory, "files", "api_info_tensor.json") + modify_tensor_api_info_json(api_info_tensor_path) + result_directory = os.path.join(directory, "files") + api_accuracy_checker = ApiAccuracyChecker() + api_accuracy_checker.parse(api_info_tensor_path) + api_accuracy_checker.run_and_compare() + api_accuracy_checker.to_detail_csv() + api_accuracy_checker.to_result_csv(result_directory) + delete_files_with_prefix(result_directory, "accuracy_checking") + + -- Gitee From 8d7319dc340e3bd7e9c2e8f47303c740a7e6b4e9 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 15:03:31 +0800 Subject: [PATCH 449/791] bugfix --- .../api_accuracy_checker/test_api_accuracy_checker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py index da2c60de9..b1b9a6fe3 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py @@ -36,7 +36,7 @@ class TestApiAccuracyChecker(unittest.TestCase): api_accuracy_checker = ApiAccuracyChecker() api_accuracy_checker.parse(api_info_statistics_path) api_accuracy_checker.run_and_compare() - api_accuracy_checker.to_detail_csv() + api_accuracy_checker.to_detail_csv(result_directory) api_accuracy_checker.to_result_csv(result_directory) delete_files_with_prefix(result_directory, "accuracy_checking") @@ -47,8 +47,9 @@ class TestApiAccuracyChecker(unittest.TestCase): api_accuracy_checker = ApiAccuracyChecker() api_accuracy_checker.parse(api_info_tensor_path) api_accuracy_checker.run_and_compare() - api_accuracy_checker.to_detail_csv() + api_accuracy_checker.to_detail_csv(result_directory) api_accuracy_checker.to_result_csv(result_directory) delete_files_with_prefix(result_directory, "accuracy_checking") - +if __name__ == '__main__': + unittest.main() \ No newline at end of file -- Gitee From e2bfb02ff929facb737e6348ac589ef70552e916 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 15:04:41 +0800 Subject: [PATCH 450/791] fix bug --- .../msprobe/pytorch/api_accuracy_checker/common/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 761e31b87..67e2ab2a7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -183,6 +183,7 @@ def extract_detailed_api_segments(full_api_name_with_direction_status): api_type = None prefix = None api_name = None + direction_status = None api_parts = full_api_name_with_direction_status.split(Const.SEP) api_parts_length = len(api_parts) if api_parts_length == Const.SIX_SEGMENT: -- Gitee From 872fd88d5b222354042a460964fc766baf8b7a18 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 15:11:44 +0800 Subject: [PATCH 451/791] fix compare part --- .../base_compare_algorithm.py | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index 6b48f60a4..1489da0ac 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -22,20 +22,19 @@ class BaseCompareAlgorithm(ABC): self.err_msg_mapping = { CompareConst.COSINE: { CompareConst.PASS: "", - CompareConst.ERROR: f"cosine similarity is less than threshold: {CompareConst.COSINE_THRESHOLD}", - CompareConst.SKIP: "two inputs are not valid for computing cosine similarity, skip comparing", + CompareConst.ERROR: f"cosine similarity is less than threshold: {CompareConst.COSINE_THRESHOLD} ", + CompareConst.SKIP: "two inputs are not valid for computing cosine similarity, skip comparing ", }, CompareConst.MAX_ABS_ERR: { CompareConst.PASS: "", CompareConst.ERROR: "max absolute difference is greater than " \ - f"threshold: {CompareConst.THOUSAND_RATIO_THRESHOLD}", - CompareConst.SKIP: "two inputs are not valid for computing max absolute difference, skip comparing", + f"threshold: {CompareConst.THOUSAND_RATIO_THRESHOLD} ", + CompareConst.SKIP: "two inputs are not valid for computing max absolute difference, skip comparing ", }, CompareConst.MAX_RELATIVE_ERR: { CompareConst.PASS: "", - CompareConst.ERROR: "max relative difference is greater than " \ - f"threshold: {CompareConst.THOUSAND_RATIO_THRESHOLD}", - CompareConst.SKIP: "two inputs are not valid for computing max relative difference, skip comparing", + CompareConst.ERROR: "", + CompareConst.SKIP: "", }, } @@ -138,7 +137,7 @@ class CosineSimilarityCompareAlgorithm(BaseCompareAlgorithm): return cosine_similarity def check_pass(self, compare_value): - if compare_value > CompareConst.COSINE_THRESHOLD: + if compare_value > CompareConst.COS_THRESHOLD: return CompareConst.PASS else: return CompareConst.ERROR @@ -160,7 +159,7 @@ class MaxAbsoluteDiffCompareAlgorithm(BaseCompareAlgorithm): return max_absolute_diff def check_pass(self, compare_value): - if compare_value < CompareConst.THOUSAND_RATIO_THRESHOLD: + if compare_value < CompareConst.MAX_ABS_ERR_THRESHOLD: return CompareConst.PASS else: return CompareConst.ERROR @@ -179,12 +178,12 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): tested_ndarray = self.convert_to_np_float64_ndarray(tested_compute_element.get_parameter()) abs_diff = np.abs(bench_ndarray - tested_ndarray) - bench_ndarray_nonzero = np.abs(bench_ndarray) + (bench_ndarray == 0) * MsCompareConst.EPSILON # prevent division by 0 + bench_ndarray_nonzero = np.abs(bench_ndarray) + (bench_ndarray == 0) * MsCompareConst.EPSILON max_relative_diff = np.max(abs_diff / bench_ndarray_nonzero) return max_relative_diff def check_pass(self, compare_value): - if compare_value < CompareConst.THOUSAND_RATIO_THRESHOLD: + if compare_value < CompareConst.MAX_ABS_ERR_THRESHOLD: return CompareConst.PASS else: return CompareConst.ERROR -- Gitee From 20019af0f62981b33125fc59622b566d4e7b9f40 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 15:13:52 +0800 Subject: [PATCH 452/791] fix test case --- .../api_accuracy_checker/test_api_accuracy_checker.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py index b1b9a6fe3..fb2b63461 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py @@ -19,11 +19,11 @@ def delete_files_with_prefix(directory, prefix): os.remove(full_path) print(f"已删除文件: {full_path}") -def modify_tensor_api_info_json(json_file_path): +def modify_tensor_api_info_json(json_file_path, modified_dump_data_dir): with open(json_file_path, 'r') as json_file: data = json.load(json_file) if 'dump_data_dir' in data: - data['dump_data_dir'] = os.path.join(directory, "files") + data['dump_data_dir'] = modified_dump_data_dir with open(json_file_path, 'w') as json_file: json.dump(data, json_file, indent=4) @@ -42,13 +42,14 @@ class TestApiAccuracyChecker(unittest.TestCase): def test_tensor_mode(self): api_info_tensor_path = os.path.join(directory, "files", "api_info_tensor.json") - modify_tensor_api_info_json(api_info_tensor_path) result_directory = os.path.join(directory, "files") + modify_tensor_api_info_json(api_info_tensor_path, result_directory) api_accuracy_checker = ApiAccuracyChecker() api_accuracy_checker.parse(api_info_tensor_path) api_accuracy_checker.run_and_compare() api_accuracy_checker.to_detail_csv(result_directory) api_accuracy_checker.to_result_csv(result_directory) + modify_tensor_api_info_json(api_info_tensor_path, "") delete_files_with_prefix(result_directory, "accuracy_checking") if __name__ == '__main__': -- Gitee From 2e3edb7c7a1d0b9b6e1fdc6ab544de5fed33a099 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 15:17:14 +0800 Subject: [PATCH 453/791] format --- .../api_accuracy_checker/files/api_info_statistics.json | 4 ++-- .../api_accuracy_checker/files/api_info_tensor.json | 4 ++-- .../api_accuracy_checker/test_compare_algorithm.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json index 886418888..b3ba49e4a 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json @@ -1,7 +1,7 @@ { "task": "statistics", "dump_data_dir": "", - "data":{ + "data": { "MintFunctional.relu.0.forward": { "input_args": [ { @@ -18,7 +18,7 @@ ], "input_kwargs": {}, "output": [ - { + { "type": "mindspore.Tensor", "dtype": "Float32", "shape": [ diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json index 886418888..b3ba49e4a 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json @@ -1,7 +1,7 @@ { "task": "statistics", "dump_data_dir": "", - "data":{ + "data": { "MintFunctional.relu.0.forward": { "input_args": [ { @@ -18,7 +18,7 @@ ], "input_kwargs": {}, "output": [ - { + { "type": "mindspore.Tensor", "dtype": "Float32", "shape": [ diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py index 70f9401eb..eeb871ad3 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compare_algorithm.py @@ -29,7 +29,7 @@ class TestCompareAlgorithms(unittest.TestCase): def test_cosine_similarity(self): compare_result = compare_algorithms[CompareConst.COSINE](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) self.assertAlmostEqual(compare_result.compare_value, 0.9997375534689601, places=5) - self.assertEqual(compare_result.pass_status, CompareConst.ERROR) + self.assertEqual(compare_result.pass_status, CompareConst.PASS) def test_max_absolute_difference(self): compare_result = compare_algorithms[CompareConst.MAX_ABS_ERR](self.mock_torchtensor_compute_element, self.mock_mstensor_compute_element) -- Gitee From 371b74244a006f37d9153e49322ba2e42b33b378 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 15:21:27 +0800 Subject: [PATCH 454/791] staticmethod --- .../api_accuracy_checker.py | 75 ++++++++++--------- 1 file changed, 38 insertions(+), 37 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 818b30c3e..72458fb21 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -34,43 +34,8 @@ class ApiAccuracyChecker: self.api_infos = dict() self.results = dict() - def parse(self, api_info_path): - with FileOpen(api_info_path, "r") as f: - api_info_dict = json.load(f) - - # init global context - task = check_and_get_from_json_dict(api_info_dict, MsCompareConst.TASK_FIELD, - "task field in api_info.json",accepted_type=str, - accepted_value=(MsCompareConst.STATISTICS_TASK, - MsCompareConst.TENSOR_TASK)) - is_constructed = task == MsCompareConst.STATISTICS_TASK - if not is_constructed: - dump_data_dir = check_and_get_from_json_dict(api_info_dict, MsCompareConst.DUMP_DATA_DIR_FIELD, - "dump_data_dir field in api_info.json", accepted_type=str) - else: - dump_data_dir = "" - global_context.init(is_constructed, dump_data_dir) - - api_info_data = check_and_get_from_json_dict(api_info_dict, MsCompareConst.DATA_FIELD, - "data field in api_info.json", accepted_type=dict) - for api_name, api_info in api_info_data.items(): - is_mint = api_name.split(Const.SEP)[0] in \ - (MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL) - if not is_mint: - continue - forbackward_str = api_name.split(Const.SEP)[-1] - if forbackward_str not in (Const.FORWARD, Const.BACKWARD): - logger.warning(f"api: {api_name} is not recognized as forward api or backward api, skip this.") - api_name = Const.SEP.join(api_name.split(Const.SEP)[:-1]) # www.xxx.yyy.zzz --> www.xxx.yyy - if api_name not in self.api_infos: - self.api_infos[api_name] = ApiInfo(api_name) - - if forbackward_str == Const.FORWARD: - self.api_infos[api_name].load_forward_info(api_info) - else: - self.api_infos[api_name].load_backward_info(api_info) - - def run_and_compare_helper(self, api_info, api_name_str, api_input_aggregation, forward_or_backward): + @staticmethod + def run_and_compare_helper(api_info, api_name_str, api_input_aggregation, forward_or_backward): ''' Args: api_info: ApiInfo @@ -120,6 +85,42 @@ class ApiAccuracyChecker: output_list.append(tuple([api_name_str, forward_or_backward, basic_info_status, compare_result_dict])) return output_list + def parse(self, api_info_path): + with FileOpen(api_info_path, "r") as f: + api_info_dict = json.load(f) + + # init global context + task = check_and_get_from_json_dict(api_info_dict, MsCompareConst.TASK_FIELD, + "task field in api_info.json",accepted_type=str, + accepted_value=(MsCompareConst.STATISTICS_TASK, + MsCompareConst.TENSOR_TASK)) + is_constructed = task == MsCompareConst.STATISTICS_TASK + if not is_constructed: + dump_data_dir = check_and_get_from_json_dict(api_info_dict, MsCompareConst.DUMP_DATA_DIR_FIELD, + "dump_data_dir field in api_info.json", accepted_type=str) + else: + dump_data_dir = "" + global_context.init(is_constructed, dump_data_dir) + + api_info_data = check_and_get_from_json_dict(api_info_dict, MsCompareConst.DATA_FIELD, + "data field in api_info.json", accepted_type=dict) + for api_name, api_info in api_info_data.items(): + is_mint = api_name.split(Const.SEP)[0] in \ + (MsCompareConst.MINT, MsCompareConst.MINT_FUNCTIONAL) + if not is_mint: + continue + forbackward_str = api_name.split(Const.SEP)[-1] + if forbackward_str not in (Const.FORWARD, Const.BACKWARD): + logger.warning(f"api: {api_name} is not recognized as forward api or backward api, skip this.") + api_name = Const.SEP.join(api_name.split(Const.SEP)[:-1]) # www.xxx.yyy.zzz --> www.xxx.yyy + if api_name not in self.api_infos: + self.api_infos[api_name] = ApiInfo(api_name) + + if forbackward_str == Const.FORWARD: + self.api_infos[api_name].load_forward_info(api_info) + else: + self.api_infos[api_name].load_backward_info(api_info) + def run_and_compare(self): for api_name_str, api_info in self.api_infos.items(): if not api_info.check_forward_info(): -- Gitee From 950336935899ec184259ec42cdccc894f87e8313 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 15:39:27 +0800 Subject: [PATCH 455/791] relative max err threshold --- debug/accuracy_tools/msprobe/core/common/const.py | 1 + .../mindspore/api_accuracy_checker/base_compare_algorithm.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index c021aaf42..09eb32851 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -188,6 +188,7 @@ class CompareConst: # accuracy standards COS_THRESHOLD = 0.99 MAX_ABS_ERR_THRESHOLD = 0.001 + MAX_RELATIVE_ERR_THRESHOLD = 0.001 COS_MAX_THRESHOLD = 0.9 MAX_ABS_ERR_MAX_THRESHOLD = 1 ACCURACY_CHECK_YES = "Yes" diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index 1489da0ac..78dc0b6bc 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -183,7 +183,7 @@ class MaxRelativeDiffCompareAlgorithm(BaseCompareAlgorithm): return max_relative_diff def check_pass(self, compare_value): - if compare_value < CompareConst.MAX_ABS_ERR_THRESHOLD: + if compare_value < CompareConst.MAX_RELATIVE_ERR_THRESHOLD: return CompareConst.PASS else: return CompareConst.ERROR -- Gitee From a6fd892214a59b44215772d3a0eb6e4d89211789 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 15:46:19 +0800 Subject: [PATCH 456/791] err msg fix --- .../mindspore/api_accuracy_checker/base_compare_algorithm.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index 78dc0b6bc..34d762304 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -22,13 +22,13 @@ class BaseCompareAlgorithm(ABC): self.err_msg_mapping = { CompareConst.COSINE: { CompareConst.PASS: "", - CompareConst.ERROR: f"cosine similarity is less than threshold: {CompareConst.COSINE_THRESHOLD} ", + CompareConst.ERROR: f"cosine similarity is less than threshold: {CompareConst.COS_THRESHOLD} ", CompareConst.SKIP: "two inputs are not valid for computing cosine similarity, skip comparing ", }, CompareConst.MAX_ABS_ERR: { CompareConst.PASS: "", CompareConst.ERROR: "max absolute difference is greater than " \ - f"threshold: {CompareConst.THOUSAND_RATIO_THRESHOLD} ", + f"threshold: {CompareConst.MAX_ABS_ERR_THRESHOLD} ", CompareConst.SKIP: "two inputs are not valid for computing max absolute difference, skip comparing ", }, CompareConst.MAX_RELATIVE_ERR: { -- Gitee From 09f4f9df4fb8090339eb7aa18c04f73c851a64ee Mon Sep 17 00:00:00 2001 From: Henry Date: Thu, 22 Aug 2024 15:25:53 +0800 Subject: [PATCH 457/791] fix jit name --- .../accuracy_tools/msprobe/mindspore/dump/jit_dump.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py index c2dc7f5a0..0b2c28ed6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py @@ -7,10 +7,12 @@ from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutput def dump_jit(name, in_feat, out_feat, is_forward): pid = os.getpid() - ori_args = str(type(name)) - index = ori_args.find("__main__.") - if index!= -1: - result = ori_args[(index + len("__main__.")):-2] + ori_args = str(name) + index = ori_args.find("<") + if index != 0 and index != -1: + result = ori_args[0:index] + else: + result = "JitFunction" if is_forward: name_template = "Jit." + result + ".forward" else: -- Gitee From d5472d7bbdc0ef2f0263563298ce9cf733d49b90 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 15:54:55 +0800 Subject: [PATCH 458/791] fix random seed --- .../msprobe/mindspore/api_accuracy_checker/compute_element.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 534b1ae51..89198f61b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -144,6 +144,7 @@ class ComputeElement: def _construct_ndarray(self, shape, maximum, minimum, np_dtype): shape = tuple(shape) + np.random.seed(42) if np_dtype == np.bool_: ndarray = np.random.rand(*shape) > 0.5 else: -- Gitee From f400bb63382de1e5ad5ebdc8ad8228cc70cfad2c Mon Sep 17 00:00:00 2001 From: jiangchangting1 Date: Thu, 22 Aug 2024 08:01:42 +0000 Subject: [PATCH 459/791] update debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py. Signed-off-by: jiangchangting1 --- .../api_accuracy_checker/compare/api_precision_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index db9e8ee8f..b131dede7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -15,7 +15,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECI BINARY_COMPARE_UNSUPPORT_LIST, ULP_COMPARE_SUPPORT_LIST, convert_str_to_float, CompareMessage, is_inf_or_nan, \ check_inf_or_nan from msprobe.pytorch.api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn -from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import get_validated_result_csv_path +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validated_result_csv_path from msprobe.pytorch.api_accuracy_checker.common.utils import extract_detailed_api_segments from msprobe.core.common.file_check import FileChecker, change_mode, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger -- Gitee From b55d347e08a9edb803ce48e05463a400ef12760b Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 22 Aug 2024 16:06:29 +0800 Subject: [PATCH 460/791] add APIs in NPU and Bench models unconsistent judgment --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 197f582dc..3de6bd354 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -141,10 +141,7 @@ class Comparator: continue # APIs in NPU and Bench models unconsistent judgment - if not npu_ops_queue and bench_ops_queue: - logger.info("Please check whether the number and calls of APIs in NPU and Bench models are consistent.") - break - elif npu_ops_queue and not bench_ops_queue: + if bool(npu_ops_queue) ^ bool(bench_ops_queue): logger.info("Please check whether the number and calls of APIs in NPU and Bench models are consistent.") break -- Gitee From ce4ab10fd42393f96346fd6b97ddfc897cc0754d Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Thu, 22 Aug 2024 15:55:43 +0800 Subject: [PATCH 461/791] =?UTF-8?q?=E3=80=90=E4=BC=98=E5=8C=96=E3=80=91tor?= =?UTF-8?q?ch.save=E6=8A=A5=E9=94=99=E6=97=B6=E6=8F=90=E7=A4=BA=E7=A3=81?= =?UTF-8?q?=E7=9B=98=E7=A9=BA=E9=97=B4=E4=B8=8D=E8=B6=B3=E6=88=96=E8=80=85?= =?UTF-8?q?=E6=B2=A1=E6=9C=89=E6=96=87=E4=BB=B6=E5=86=99=E5=85=A5=E6=9D=83?= =?UTF-8?q?=E9=99=90=E7=AD=89=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index a8e05402f..69b7d20ef 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -25,6 +25,7 @@ import torch.distributed as dist import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError +from msprobe.core.common.log import logger as common_logger from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, CompareException from msprobe.core.common.file_check import FileCheckConst, change_mode, FileOpen @@ -279,6 +280,9 @@ def save_pt(tensor, filepath): try: torch.save(tensor, filepath) except Exception as e: + common_logger.error("Save pt file failed, please check according possible error causes: " + "1. out of disk space or disk error, " + "2. no permission to write files, etc.") raise RuntimeError(f"save pt file {filepath} failed") from e change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) -- Gitee From 6377cb8be4b2924d38b75cb3f0f1843492cd9933 Mon Sep 17 00:00:00 2001 From: jijiarong Date: Thu, 22 Aug 2024 16:38:26 +0800 Subject: [PATCH 462/791] path bugfix --- .../msprobe/core/common/const.py | 1 + .../msprobe/core/compare/compare_cli.py | 27 ++++--- .../mindspore/compare/distributed_compare.py | 12 ++- .../mindspore/compare/ms_graph_compare.py | 79 ++++++++----------- 4 files changed, 63 insertions(+), 56 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 426083917..7fb8d759c 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -143,6 +143,7 @@ class CompareConst: NPU_MD5 = "NPU MD5" BENCH_MD5 = "BENCH MD5" RESULT = "Result" + MAGNITUDE = 0.5 COMPARE_RESULT_HEADER = [ NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR, diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index ac96a65de..6afed0a1f 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -5,31 +5,40 @@ from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger - def compare_cli(args): with FileOpen(args.input_path, "r") as file: input_param = json.load(file) npu_path = input_param.get("npu_path", None) bench_path = input_param.get("bench_path", None) - frame_name =args.framework - if frame_name ==Const.PT_FRAMEWORK: + frame_name = args.framework + auto_analyze = not args.compare_only + if frame_name == Const.PT_FRAMEWORK: from msprobe.pytorch.compare.pt_compare import compare from msprobe.pytorch.compare.distributed_compare import compare_distributed else: from msprobe.mindspore.compare.ms_compare import ms_compare - from msprobe.mindspore.compare.distributed_compare import ms_compare_distributed + from msprobe.mindspore.compare.distributed_compare import ms_compare_distributed, ms_graph_compare if check_file_type(npu_path) == FileCheckConst.FILE and check_file_type(bench_path) == FileCheckConst.FILE: input_param["npu_json_path"] = input_param.pop("npu_path") input_param["bench_json_path"] = input_param.pop("bench_path") input_param["stack_json_path"] = input_param.pop("stack_path") if frame_name == Const.PT_FRAMEWORK: - compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, - fuzzy_match=args.fuzzy_match) + compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=auto_analyze, + fuzzy_match=args.fuzzy_match) else: - ms_compare(input_param, args.output_path, stack_mode=args.stack_mode, auto_analyze=args.auto_analyze, - fuzzy_match=args.fuzzy_match) + kwargs = { + "stack_mode": args.stack_mode, + "auto_analyze": auto_analyze, + "fuzzy_match": args.fuzzy_match, + "cell_mapping": args.cell_mapping, + "api_mapping": args.api_mapping, + } + + ms_compare(input_param, args.output_path, **kwargs) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: - kwargs = {"stack_mode": args.stack_mode, "auto_analyze": args.auto_analyze, "fuzzy_match": args.fuzzy_match} + kwargs = {"stack_mode": args.stack_mode, "auto_analyze": auto_analyze, "fuzzy_match": args.fuzzy_match} + if input_param.get("rank_id") is None: + ms_graph_compare(input_param, args.output_path) if frame_name == Const.PT_FRAMEWORK: compare_distributed(npu_path, bench_path, args.output_path, **kwargs) else: diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 9f5dcb9d4..763a5651f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -22,7 +22,7 @@ from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import MSComparator from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json - +from msprobe.mindspore.compare.ms_graph_compare import GraphMSComparator def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): if kwargs.get('suffix'): @@ -63,3 +63,13 @@ def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): ms_comparator = MSComparator() ms_comparator.compare_core(dump_result_param, output_path, suffix=f'_{nr}-{br}', summary_compare=summary_compare, md5_compare=md5_compare, **kwargs) + + +def ms_graph_compare(inputs, outputs): + try: + create_directory(outputs) + except (CompareException, FileCheckException) as error: + logger.error('Compare failed. Please check the arguments and do it again!') + return + msComparator = GraphMSComparator(inputs, outputs) + msComparator.compare_core() diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index 259377f42..96195ba42 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -32,6 +32,8 @@ class row_data: def generate_step(npu_path, rank_id): step_set = set() rank_path = os.path.join(npu_path, f"rank_{rank_id}") + if not os.path.exists(rank_path): + return for path in os.listdir(rank_path): if path not in ["execution_order", "graphs"]: data_path = os.path.join(rank_path, path) @@ -42,6 +44,8 @@ def generate_step(npu_path, rank_id): def generate_path_by_rank_step(base_path, rank_id, step_id): path_with_rank_id = os.path.join(base_path, f"rank_{rank_id}") + if not os.path.exists(path_with_rank_id): + return '' for path in os.listdir(path_with_rank_id): if path not in ["execution_order", "graphs"]: @@ -76,6 +80,7 @@ def statistic_data_read(statistic_file_list, statistic_file_path): [statistic_file_path, compare_key, timestamp, data[header_index['Data Type']], data[header_index['Shape']], data[header_index['Max Value']], data[header_index['Min Value']], data[header_index['Avg Value']], data[header_index['L2Norm Value']]]) + return data_list def generate_data_name(data_path): @@ -132,6 +137,8 @@ def read_npy_data(data_path): data_value = data_value.astype(np.float32) except FileNotFoundError as e: data_value = None + except EOFError: + data_value = None return data_value @@ -151,25 +158,19 @@ class GraphMSComparator: n_value = None b_value = None - if os.path.exists(row[CompareConst.NPU_NAME]): - n_value = read_npy_data(row[CompareConst.NPU_NAME]) - result_dict[CompareConst.NPU_NAME] = row[CompareConst.NPU_NAME] - result_dict[CompareConst.NPU_DTYPE] = n_value.dtype - result_dict[CompareConst.NPU_SHAPE] = n_value.shape - result_dict[CompareConst.NPU_MAX] = np.max(n_value) - result_dict[CompareConst.NPU_MIN] = np.min(n_value) - result_dict[CompareConst.NPU_MEAN] = np.mean(n_value) - result_dict[CompareConst.NPU_NORM] = np.linalg.norm(n_value) - - if os.path.exists(row[CompareConst.BENCH_NAME]): - b_value = read_npy_data(row[CompareConst.BENCH_NAME]) - result_dict[CompareConst.BENCH_NAME] = row[CompareConst.BENCH_NAME] - result_dict[CompareConst.BENCH_DTYPE] = b_value.dtype - result_dict[CompareConst.BENCH_SHAPE] = b_value.shape - result_dict[CompareConst.BENCH_MAX] = np.max(b_value) - result_dict[CompareConst.BENCH_MIN] = np.min(b_value) - result_dict[CompareConst.BENCH_MEAN] = np.mean(b_value) - result_dict[CompareConst.BENCH_NORM] = np.linalg.norm(b_value) + def process_npy_file(file_path, name_prefix, result): + if os.path.exists(file_path): + data = read_npy_data(file_path) + result[f'{name_prefix}_NAME'] = file_path + result[f'{name_prefix}_DTYPE'] = data.dtype + result[f'{name_prefix}_SHAPE'] = data.shape + result[f'{name_prefix}_MAX'] = np.max(data) + result[f'{name_prefix}_MIN'] = np.min(data) + result[f'{name_prefix}_MEAN'] = np.mean(data) + result[f'{name_prefix}_NORM'] = np.linalg.norm(data) + + process_npy_file(row[CompareConst.NPU_NAME], 'NPU', result_dict) + process_npy_file(row[CompareConst.BENCH_NAME], 'BENCH', result_dict) error_flag, error_message = npy_data_check(n_value, b_value) result_dict[CompareConst.ERROR_MESSAGE] = error_message @@ -190,22 +191,18 @@ class GraphMSComparator: def statistic_mode_compute(row): result_dict = row_data('STATISTIC')() - result_dict[CompareConst.NPU_NAME] = row[CompareConst.NPU_NAME] - result_dict[CompareConst.NPU_DTYPE] = row[CompareConst.NPU_DTYPE] - result_dict[CompareConst.NPU_SHAPE] = row[CompareConst.NPU_SHAPE] - result_dict[CompareConst.NPU_MAX] = np.float32(row[CompareConst.NPU_MAX]) - result_dict[CompareConst.NPU_MIN] = np.float32(row[CompareConst.NPU_MIN]) - result_dict[CompareConst.NPU_MEAN] = np.float32(row[CompareConst.NPU_MEAN]) - result_dict[CompareConst.NPU_NORM] = np.float32(row[CompareConst.NPU_NORM]) - - result_dict[CompareConst.BENCH_NAME] = row[CompareConst.BENCH_NAME] - result_dict[CompareConst.BENCH_DTYPE] = row[CompareConst.BENCH_DTYPE] - result_dict[CompareConst.BENCH_SHAPE] = row[CompareConst.BENCH_SHAPE] - result_dict[CompareConst.BENCH_MAX] = np.float32(row[CompareConst.BENCH_MAX]) - result_dict[CompareConst.BENCH_MIN] = np.float32(row[CompareConst.BENCH_MIN]) - result_dict[CompareConst.BENCH_MEAN] = np.float32(row[CompareConst.BENCH_MEAN]) - result_dict[CompareConst.BENCH_NORM] = np.float32(row[CompareConst.BENCH_NORM]) - + def update_result_dict(result, rows, prefix): + result[f'{prefix}_NAME'] = rows[f'{prefix}_NAME'] + result[f'{prefix}_DTYPE'] = rows[f'{prefix}_DTYPE'] + result[f'{prefix}_SHAPE'] = rows[f'{prefix}_SHAPE'] + result[f'{prefix}_MAX'] = np.float32(rows[f'{prefix}_MAX']) + result[f'{prefix}_MIN'] = np.float32(rows[f'{prefix}_MIN']) + result[f'{prefix}_MEAN'] = np.float32(rows[f'{prefix}_MEAN']) + result[f'{prefix}_NORM'] = np.float32(rows[f'{prefix}_NORM']) + + # 使用示例 + update_result_dict(result_dict, row, 'NPU') + update_result_dict(result_dict, row, 'BENCH') error_flag, error_message = statistics_data_check(result_dict) result_dict[CompareConst.ERROR_MESSAGE] += error_message if not error_flag: @@ -233,7 +230,7 @@ class GraphMSComparator: result_dict[CompareConst.NORM_RELATIVE_ERR] * 100) + "%" magnitude_diff = result_dict[CompareConst.MAX_DIFF] / ( max(result_dict[CompareConst.NPU_MAX], result_dict[CompareConst.BENCH_MAX]) + 1e-10) - if magnitude_diff > 0.5: + if magnitude_diff > CompareConst.MAGNITUDE: result_dict[CompareConst.ACCURACY] = 'No' else: result_dict[CompareConst.ACCURACY] = 'Yes' @@ -318,13 +315,3 @@ class GraphMSComparator: logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e return result_df - - -def ms_graph_compare(inputs, outputs): - try: - create_directory(outputs) - except (CompareException, FileCheckException) as error: - logger.error('Compare failed. Please check the arguments and do it again!') - return - msComparator = GraphMSComparator(inputs, outputs) - msComparator.compare_core() -- Gitee From 1dd06d9974aee0507c6beb9b50e175e818ce7b6d Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 16:41:55 +0800 Subject: [PATCH 463/791] write csv --- debug/accuracy_tools/msprobe/core/common/utils.py | 12 +++--------- .../api_accuracy_checker/api_accuracy_checker.py | 6 +++--- 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 0a3440ba0..1593efc38 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -523,20 +523,14 @@ def convert_tuple(data): return data if isinstance(data, tuple) else (data, ) -def write_csv(data, filepath): +def write_csv(data, filepath, mode="a+"): exist = os.path.exists(filepath) - with FileOpen(filepath, 'a+', encoding='utf-8-sig') as f: + with FileOpen(filepath, mode, encoding='utf-8-sig') as f: writer = csv.writer(f) writer.writerows(data) if not exist: change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) -def write_new_csv(data, filepath): - with FileOpen(filepath, 'w', encoding='utf-8-sig') as f: - writer = csv.writer(f) - writer.writerows(data) - change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) - def load_npy(filepath): check_file_or_directory_path(filepath) @@ -593,7 +587,7 @@ def get_file_content_bytes(file): with FileOpen(file, 'rb') as file_handle: return file_handle.read() - + def load_yaml(yaml_path): path_checker = FileChecker(yaml_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.YAML_SUFFIX) checked_path = path_checker.common_check() diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 72458fb21..92b4fb9c3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -2,7 +2,7 @@ import json import os from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import write_new_csv, add_time_as_suffix +from msprobe.core.common.utils import write_csv, add_time_as_suffix from msprobe.core.common.const import Const, CompareConst, MsCompareConst from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo @@ -194,7 +194,7 @@ class ApiAccuracyChecker: detail_csv.append(csv_row) file_name = os.path.join(csv_dir, add_time_as_suffix(MsCompareConst.DETAIL_CSV_FILE_NAME)) - write_new_csv(detail_csv, file_name) + write_csv(detail_csv, file_name, mode="w") def to_result_csv(self, csv_dir): @@ -243,4 +243,4 @@ class ApiAccuracyChecker: result_csv.append(row) file_name = os.path.join(csv_dir, add_time_as_suffix(MsCompareConst.RESULT_CSV_FILE_NAME)) - write_new_csv(result_csv, file_name) \ No newline at end of file + write_csv(result_csv, file_name, mode="w") \ No newline at end of file -- Gitee From 6a6743e9e15ff1cde7f441c2baf241c4b7e83cc9 Mon Sep 17 00:00:00 2001 From: jijiarong Date: Thu, 22 Aug 2024 16:50:19 +0800 Subject: [PATCH 464/791] fix rank id --- debug/accuracy_tools/msprobe/core/compare/compare_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index 6afed0a1f..bf5ac7335 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -37,7 +37,7 @@ def compare_cli(args): ms_compare(input_param, args.output_path, **kwargs) elif check_file_type(npu_path) == FileCheckConst.DIR and check_file_type(bench_path) == FileCheckConst.DIR: kwargs = {"stack_mode": args.stack_mode, "auto_analyze": auto_analyze, "fuzzy_match": args.fuzzy_match} - if input_param.get("rank_id") is None: + if input_param.get("rank_id") is not None: ms_graph_compare(input_param, args.output_path) if frame_name == Const.PT_FRAMEWORK: compare_distributed(npu_path, bench_path, args.output_path, **kwargs) -- Gitee From 17ced75d23e6bdeeea5021dd46287f7d8329d9ea Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 17:34:31 +0800 Subject: [PATCH 465/791] update branch --- debug/accuracy_tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index c394420e9..b273f0f3d 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -14,7 +14,7 @@ import setuptools -__version__ = '1.0.2' +__version__ = '1.0.3' INSTALL_REQUIRED = [ "wheel", -- Gitee From 072241fea7f9b32af3fdf021adacd7c779b7deab Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 22 Aug 2024 17:34:58 +0800 Subject: [PATCH 466/791] readme --- .../mindspore/doc/api_accuracy_checker.md | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md b/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md new file mode 100644 index 000000000..7c86fc272 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md @@ -0,0 +1,76 @@ +# **MindSpore动态图精度预检工具** + +## 简介 + +MindSpore动态图精度预检工具通过扫描昇腾NPU上用户训练MindSpore模型中所有Mint API，输出精度情况的诊断和分析。工具以模型中所有的Mint API前反向的dump结果为输入，构造相应的API单元测试，将NPU输出与标杆（CPU高精度）比对，计算对应的精度指标，从而找出NPU中存在精度问题的Mint API。 + + +**真实数据/随机生成模式**：精度预检工具支持随机生成模式和真实数据模式，即在预检dump时可以选择由工具构造随机数进行输入获得dump数据或选择获取真实输入数据进行预检dump操作；随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + +**工具支持Mindspore版本**：2.4。 + +**工具特性** + +- 落盘数据小。 +- 支持随机生成模式和真实数据模式。 +- 单API测试，排除整网中的累计误差问题。 + +## 预检流程 + +精度预检操作流程如下： + +1. 在NPU和GPU环境下分别安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 +2. 在NPU训练脚本内添加msprobe工具dump接口PrecisionDebugger，采集待预检数据。详见《[精度数据采集](./dump.md)》，注意需要配置level="L1"。 +3. 执行预检操作，查看预检结果文件，分析预检不达标的API。 + +## 预检操作 + + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json +``` + +| 参数名称 | 说明 | 是否必选 | +| ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | +| -api_info或--api_info_file | 指定API信息文件dump.json。 | 是 | +| -o或--out_path | 指定预检结果存盘路径，默认“./”。 | 否 | + +预检执行结果包括`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`两个文件。`accuracy_checking_result_{timestamp}.csv`是API粒度的，标明每个API是否通过测试。建议用户先查看`accuracy_checking_result_{timestamp}.csv`文件，对于其中没有通过测试的或者特定感兴趣的API，根据其API Name字段在`accuracy_checking_details_{timestamp}.csv`中查询其各个输出的达标情况以及比较指标。详细介绍请参见“**预检结果**”。 + +## 预检结果 + +精度预检生成的`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`文件内容详情如下： + +`accuracy_checking_details_{timestamp}.csv` + +| 字段 | 含义 | +| ------------------- | ------------------------------------------------------------ | +| API Name | API名称。 | +| Bench Dtype | 标杆数据的API数据类型。 | +| Tested Dtype | 被检验数据的API数据类型。 | +| Shape | API的Shape信息。 | +| Cosine | 被检验数据与标杆数据的余弦相似度。 | +| MaxAbsErr | 被检验数据与标杆数据的最大绝对误差。 | +| MaxRelativeErr | 被检验数据与标杆数据的最大相对误差。 | +| Status | API预检通过状态，pass表示通过测试，error表示未通过。 | +| message | 提示信息。 | + +`accuracy_checking_result_{timestamp}.csv` + +| 字段 | 含义 | +| --------------------- | ------------------------------------------------------------ | +| API Name | API名称。 | +| Forward Test Success | 前向API是否通过测试，pass为通过，error为错误。 | +| Backward Test Success | 反向API是否通过测试，pass为通过，error为错误，如果是空白的话代表该API没有反向输出。 | +| Message | 提示信息。 | + +Forward Test Success和Backward Test Success是否通过测试是由`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差判定结果决定的。具体规则详见“**API预检指标**”。 +需要注意的是`accuracy_checking_details_{timestamp}.csv`中可能存在一个API的前向（反向）有多个输出，那么每个输出记录一行，而在`accuracy_checking_result_{timestamp}.csv`中的结果需要该API的所有结果均为pass才能标记为pass，只要存在一个error则标记error。 + + +## API预检指标 + +API预检指标是通过对`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差的数值进行判断，得出该API是否符合精度标准的参考指标。详细规则如下： + + - 余弦相似度大于0.99，并且最大绝对误差小于0.0001，标记“pass”，否则标记为“error”。 + -- Gitee From 2ecdcc6871e5c67979f292baa9f7e47730ae23e0 Mon Sep 17 00:00:00 2001 From: jijiarong Date: Thu, 22 Aug 2024 16:57:26 +0800 Subject: [PATCH 467/791] bugfix return [] --- .../msprobe/core/compare/compare_cli.py | 1 + .../msprobe/core/compare/npy_compare.py | 15 ++++-- .../mindspore/compare/ms_graph_compare.py | 48 +++++++++---------- 3 files changed, 37 insertions(+), 27 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index bf5ac7335..2ae4a09d5 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -39,6 +39,7 @@ def compare_cli(args): kwargs = {"stack_mode": args.stack_mode, "auto_analyze": auto_analyze, "fuzzy_match": args.fuzzy_match} if input_param.get("rank_id") is not None: ms_graph_compare(input_param, args.output_path) + return if frame_name == Const.PT_FRAMEWORK: compare_distributed(npu_path, bench_path, args.output_path, **kwargs) else: diff --git a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py index 82acecd83..3ddd007c7 100644 --- a/debug/accuracy_tools/msprobe/core/compare/npy_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/npy_compare.py @@ -80,6 +80,8 @@ def npy_data_check(n_value, b_value): error_message = "" if n_value is None or b_value is None: error_message += "Dump file not found.\n" + if n_value == "" or b_value == "": + error_message += "Dump file not found.\n" # 检查 n_value 和 b_value 是否为空 if not error_message and (n_value.size == 0 or b_value.size == 0): @@ -97,8 +99,11 @@ def npy_data_check(n_value, b_value): n_value, b_value = handle_inf_nan(n_value, b_value) # 判断是否有 nan/inf 数据 if CompareConst.NAN in (n_value, b_value): error_message += "The position of inf or nan in NPU and bench Tensor do not match.\n" - - return bool(error_message), error_message + if error_message == "": + error_flag = False + else: + error_flag = True + return error_flag, error_message def statistics_data_check(result_dict): @@ -115,7 +120,11 @@ def statistics_data_check(result_dict): if result_dict.get(CompareConst.NPU_DTYPE) != result_dict.get(CompareConst.BENCH_DTYPE): error_message += "Dtype of NPU and bench Tensor do not match. Skipped.\n" - return bool(error_message), error_message + if error_message == "": + error_flag = False + else: + error_flag = True + return error_flag, error_message class TensorComparisonBasic(abc.ABC): diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index 96195ba42..bb83c0fb8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -33,7 +33,7 @@ def generate_step(npu_path, rank_id): step_set = set() rank_path = os.path.join(npu_path, f"rank_{rank_id}") if not os.path.exists(rank_path): - return + return [] for path in os.listdir(rank_path): if path not in ["execution_order", "graphs"]: data_path = os.path.join(rank_path, path) @@ -147,30 +147,30 @@ class GraphMSComparator: self.output_path = output_path self.base_npu_path = input_param.get('npu_path', None) self.base_bench_path = input_param.get('bench_path', None) - self.rank_list = input_param.get('rank_list', []) - self.step_list = input_param.get('step_list', []) + self.rank_list = input_param.get('rank_id', []) + self.step_list = input_param.get('step_id', []) @staticmethod def compare_ops(compare_result_db, mode): def npy_mode_compute(row): result_dict = row_data(GraphMode.NPY_MODE)() - n_value = None - b_value = None def process_npy_file(file_path, name_prefix, result): if os.path.exists(file_path): data = read_npy_data(file_path) - result[f'{name_prefix}_NAME'] = file_path - result[f'{name_prefix}_DTYPE'] = data.dtype - result[f'{name_prefix}_SHAPE'] = data.shape - result[f'{name_prefix}_MAX'] = np.max(data) - result[f'{name_prefix}_MIN'] = np.min(data) - result[f'{name_prefix}_MEAN'] = np.mean(data) - result[f'{name_prefix}_NORM'] = np.linalg.norm(data) - - process_npy_file(row[CompareConst.NPU_NAME], 'NPU', result_dict) - process_npy_file(row[CompareConst.BENCH_NAME], 'BENCH', result_dict) + result[f'{name_prefix} Name'] = file_path + result[f'{name_prefix} Dtype'] = data.dtype + result[f'{name_prefix} Tensor Shape'] = data.shape + result[f'{name_prefix} max'] = np.max(data) + result[f'{name_prefix} min'] = np.min(data) + result[f'{name_prefix} mean'] = np.mean(data) + result[f'{name_prefix} l2norm'] = np.linalg.norm(data) + return data + return "" + + n_value = process_npy_file(row[CompareConst.NPU_NAME], 'NPU', result_dict) + b_value = process_npy_file(row[CompareConst.BENCH_NAME], 'Bench', result_dict) error_flag, error_message = npy_data_check(n_value, b_value) result_dict[CompareConst.ERROR_MESSAGE] = error_message @@ -192,17 +192,17 @@ class GraphMSComparator: result_dict = row_data('STATISTIC')() def update_result_dict(result, rows, prefix): - result[f'{prefix}_NAME'] = rows[f'{prefix}_NAME'] - result[f'{prefix}_DTYPE'] = rows[f'{prefix}_DTYPE'] - result[f'{prefix}_SHAPE'] = rows[f'{prefix}_SHAPE'] - result[f'{prefix}_MAX'] = np.float32(rows[f'{prefix}_MAX']) - result[f'{prefix}_MIN'] = np.float32(rows[f'{prefix}_MIN']) - result[f'{prefix}_MEAN'] = np.float32(rows[f'{prefix}_MEAN']) - result[f'{prefix}_NORM'] = np.float32(rows[f'{prefix}_NORM']) + result[f'{prefix} Name'] = rows[f'{prefix} Name'] + result[f'{prefix} Dtype'] = rows[f'{prefix} Dtype'] + result[f'{prefix} Tensor Shape'] = rows[f'{prefix} Tensor Shape'] + result[f'{prefix} max'] = np.float32(rows[f'{prefix} max']) + result[f'{prefix} min'] = np.float32(rows[f'{prefix} min']) + result[f'{prefix} mean'] = np.float32(rows[f'{prefix} mean']) + result[f'{prefix} l2norm'] = np.float32(rows[f'{prefix} l2norm']) # 使用示例 update_result_dict(result_dict, row, 'NPU') - update_result_dict(result_dict, row, 'BENCH') + update_result_dict(result_dict, row, 'Bench') error_flag, error_message = statistics_data_check(result_dict) result_dict[CompareConst.ERROR_MESSAGE] += error_message if not error_flag: @@ -263,7 +263,7 @@ class GraphMSComparator: if is_empty or not mode: continue compare_result_df = self._do_multi_process(compare_result_df, mode) - compare_result_name = add_time_with_xlsx(f"compare_result_{str(rank_id)}_{str(rank_id)}") + compare_result_name = add_time_with_xlsx(f"compare_result_{str(rank_id)}_{str(step_id)}") compare_result_path = os.path.join(os.path.realpath(self.output_path), f"{compare_result_name}") compare_result_df.to_excel(compare_result_path, index=False) logger.info(f"Compare rank: {rank_id} step: {step_id} finish. Compare result: {compare_result_path}.") -- Gitee From 639885c63babe11e555ca4db998f72383dcdf45b Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 22 Aug 2024 19:12:18 +0800 Subject: [PATCH 468/791] fix --- .../compare/api_precision_compare.py | 14 ++++++-------- .../pytorch/api_accuracy_checker/run_ut/run_ut.py | 12 +++++------- 2 files changed, 11 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 76a037644..2eba050b4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -298,17 +298,15 @@ def analyse_csv(npu_data, gpu_data, config): compare_column = ApiPrecisionOutputColumn() full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] row_gpu = gpu_data[gpu_data[ApiPrecisionCompareColumn.API_NAME] == full_api_name_with_direction_status] - try: - api_name, full_api_name, direction_status = extract_detailed_api_segments(full_api_name_with_direction_status) - if not full_api_name: - raise CompareException(CompareException.INVALID_DATA_ERROR, "This type of API has not been adapted.") - except Exception as err: - logger.error(f"Compare Error: %s" % str(err)) + api_name, full_api_name, direction_status = extract_detailed_api_segments(full_api_name_with_direction_status) + if not full_api_name: + err_message = f"The API name {full_api_name_with_direction_status} is invalid." + logger.error(err_message) compare_column.api_name = full_api_name_with_direction_status compare_column.compare_result = CompareConst.SKIP - compare_column.compare_message = str(err) + compare_column.compare_message = err_message write_detail_csv(compare_column.to_column_value(), config.details_csv_path) - write_csv([[full_api_name_with_direction_status, "skip", "skip", str(err)]], config.result_csv_path) + write_csv([[full_api_name_with_direction_status, "skip", "skip", err_message]], config.result_csv_path) continue if row_gpu.empty: logger.warning(f'This API : {full_api_name_with_direction_status} does not exist in the GPU data.') diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 1eb8a025d..5f4517e1e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -224,13 +224,11 @@ def run_api_offline(config, compare, api_name_set): continue if is_unsupported_api(api_full_name): continue - try: - _, _, api_name = extract_basic_api_segments(api_full_name) - if not api_name: - raise ValueError(f"API name {api_full_name} has not been adapted.") - except ValueError as err: - logger.error(f"Run {api_full_name} UT Error: %s" % str(err)) - fwd_compare_alg_results = err_column.to_column_value(CompareConst.SKIP, str(err)) + _, _, api_name = extract_basic_api_segments(api_full_name) + if not api_name: + err_message = f"API {api_full_name} not support for run ut. SKIP." + logger.error(err_message) + fwd_compare_alg_results = err_column.to_column_value(CompareConst.SKIP, err_message) result_info = (api_full_name, CompareConst.SKIP, CompareConst.SKIP, [fwd_compare_alg_results], None, 0) compare.record_results(result_info) continue -- Gitee From 5897693c2c947f0fc4955c153b1fdd369736bc4f Mon Sep 17 00:00:00 2001 From: wuyuhan Date: Thu, 22 Aug 2024 21:53:39 +0800 Subject: [PATCH 469/791] =?UTF-8?q?=E9=9B=86=E6=88=90compare=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=E8=BF=9B=E8=A1=8C=E6=AF=94=E5=AF=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../advisor/analyzer/analyzer_controller.py | 65 +++++++--- .../advisor/analyzer/comparison/__init__.py | 0 .../comparison/comparison_analyzer.py | 35 +++++ .../analyzer/comparison/comparison_checker.py | 122 ++++++++++++++++++ profiler/advisor/common/analyzer_scopes.py | 1 + .../display/html/templates/comparison.html | 25 ++++ profiler/advisor/interface/interface.py | 5 +- profiler/advisor/result/result.py | 2 +- 8 files changed, 237 insertions(+), 18 deletions(-) create mode 100644 profiler/advisor/analyzer/comparison/__init__.py create mode 100644 profiler/advisor/analyzer/comparison/comparison_analyzer.py create mode 100644 profiler/advisor/analyzer/comparison/comparison_checker.py create mode 100644 profiler/advisor/display/html/templates/comparison.html diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py index e8704542e..7f071978f 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -21,6 +21,7 @@ from profiler.advisor.utils.utils import Timer, safe_index, safe_division from profiler.advisor.interface.interface import Interface from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from profiler.prof_common.path_manager import PathManager +from profiler.compare_tools.compare_backend.utils.constant import Constant as CompareConstant logger = logging.getLogger() @@ -127,10 +128,17 @@ class AnalyzerController: if benchmark_profiling_path: # kernel/api 比对 - job_list += self._single_profiling_comparison(profiling_path, benchmark_profiling_path) + compare_profiling_list = [ + dict(profiling_path=profiling_path, benchmark_profiling_path=benchmark_profiling_path, + compare_mode=CompareConstant.KERNEL_COMPARE), + dict(profiling_path=profiling_path, benchmark_profiling_path=benchmark_profiling_path, + compare_mode=CompareConstant.API_COMPARE) + ] + + job_list += self._profiling_comparison(compare_profiling_list) else: - # 单卡性能拆解 self.overall(profiling_path) + return job_list def cluster_analysis(self, profiling_path, benchmark_profiling_path=None): @@ -144,11 +152,11 @@ class AnalyzerController: logger.info("Start cluster %s analysis", dimension) job_list += getattr(self, dimension_analysis_func_name)(profiling_path) + self.overall(profiling_path) + if benchmark_profiling_path: # 两个集群profiling比对分析 job_list += self._cluster_profiling_comparison(profiling_path, benchmark_profiling_path) - else: - self.overall(profiling_path) return job_list def overall(self, profiling_path): @@ -393,6 +401,9 @@ class AnalyzerController: return self._is_cluster = self._is_cluster_profiling(profiling_path) + if benchmark_profiling_path: + _ = self._is_cluster_profiling(benchmark_profiling_path) + if not self._is_cluster: job_list = self.single_rank_analysis(profiling_path, benchmark_profiling_path) else: @@ -436,11 +447,21 @@ class AnalyzerController: job_list = [] return job_list - def _single_profiling_comparison(self, profiling_path, benchmark_profiling_path, step=None, - benchmark_step=None): - # TODO 基于compare tools 对比计算下发 - kwargs = copy.deepcopy(self.kwargs) - return [] + def _profiling_comparison(self, compare_profiling_list): + job_list = [] + + for index, _kwargs in enumerate(compare_profiling_list): + kwargs = copy.deepcopy(self.kwargs) + kwargs.update(_kwargs) + compare_profiling_list[index] = kwargs + + compare_kwargs = {"profiling_path": kwargs.get("profiling_path"), + "compare_profiling_list": compare_profiling_list} + + interface = Interface(**compare_kwargs) + job_list.append((Interface.COMPARISON, SupportedScopes.COMPARISON, interface, compare_kwargs)) + + return job_list def _cluster_profiling_comparison(self, profiling_path, benchmark_profiling_path): # 从计算、下发和通信三个维度对集群profiling数据进行对比 @@ -472,8 +493,10 @@ class AnalyzerController: if isinstance(target_cluster_analyzer, SlowRankAnalyzer): comparison_dims = [SlowRankAnalyzer.COMPUTE, SlowRankAnalyzer.FREE] + comparison_modes = [CompareConstant.KERNEL_COMPARE, CompareConstant.API_COMPARE] elif isinstance(target_cluster_analyzer, SlowLinkAnalyzer): comparison_dims = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + comparison_modes = [None, None] else: return job_list @@ -486,7 +509,8 @@ class AnalyzerController: "The product of ranks and steps of Benchmark profiling is not equals to target profiling, skip cluster comparison.") return job_list - for dimension in comparison_dims: + compare_profiling_list = [] + for dimension, compare_mode in zip(comparison_dims, comparison_modes): step, benchmark_step, rank_id_for_comparison = AnalyzerController._get_step_rank_for_cluster_statistic_diff( target_data, benchmark_data, @@ -494,18 +518,27 @@ class AnalyzerController: dimension, get_max=get_max ) + rank_profiling_path = self._get_profiling_path_by_rank(profiling_path, rank_id_for_comparison) rank_benchmark_profiling_path = self._get_profiling_path_by_rank( benchmark_profiling_path, rank_id_for_comparison ) - job_list += self._single_profiling_comparison( - rank_profiling_path, - rank_benchmark_profiling_path, - step, - benchmark_step + if rank_id_for_comparison is None: + # rank id为空则无法获取对应rank的profiling路径，无法进行比较 + continue + + compare_profiling_list.append( + dict(profiling_path=rank_profiling_path, benchmark_profiling_path=rank_benchmark_profiling_path, + step=str(step) if step else step, benchmark_step=str(benchmark_step) if step else step, + rank=rank_id_for_comparison, benchmark_rank=rank_id_for_comparison, compare_mode=compare_mode) ) + + if not compare_profiling_list: + return job_list + + job_list += self._profiling_comparison(compare_profiling_list) return job_list def _is_cluster_profiling(self, profiling_path): @@ -554,4 +587,4 @@ class AnalyzerController: xlsx_path = os.path.join(Config().work_path, f"mstt_advisor_{Timer().strftime}.xlsx") result_files = {"html": html_path, "xlsx": xlsx_path} self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, - status=AsyncAnalysisStatus.SUCCESS, result_files=result_files) \ No newline at end of file + status=AsyncAnalysisStatus.SUCCESS, result_files=result_files) diff --git a/profiler/advisor/analyzer/comparison/__init__.py b/profiler/advisor/analyzer/comparison/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/comparison/comparison_analyzer.py b/profiler/advisor/analyzer/comparison/comparison_analyzer.py new file mode 100644 index 000000000..0c79863df --- /dev/null +++ b/profiler/advisor/analyzer/comparison/comparison_analyzer.py @@ -0,0 +1,35 @@ +import logging + +from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.analyzer.comparison.comparison_checker import ComparisonChecker + +logger = logging.getLogger() + + +class ComparisonAnalyzer(BaseAnalyzer): + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + self.result = OptimizeResult() + self.html_render = HTMLRender() + + def optimize(self, compare_profiling_list, **kwargs): + for compare_profiling_path in compare_profiling_list: + self._optimize(**compare_profiling_path) + return self.result + + def get_priority(self): + pass + + def _optimize(self, profiling_path, benchmark_profiling_path, **kwargs): + comparison_checker = ComparisonChecker(profiling_path, + benchmark_profiling_path, + step=kwargs.get("step"), + benchmark_step=kwargs.get("benchmark_step"), + rank=kwargs.get("rank"), + benchmark_rank=kwargs.get("benchmark_rank")) + comparison_checker.compare(kwargs.get("compare_mode")) + comparison_checker.make_record(self.result) + comparison_checker.make_render(self.html_render) diff --git a/profiler/advisor/analyzer/comparison/comparison_checker.py b/profiler/advisor/analyzer/comparison/comparison_checker.py new file mode 100644 index 000000000..ed98f448b --- /dev/null +++ b/profiler/advisor/analyzer/comparison/comparison_checker.py @@ -0,0 +1,122 @@ +import logging + +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.advisor.utils.utils import safe_index, convert_to_float +from profiler.compare_tools.compare_backend.utils.constant import Constant as CompareConstant +from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface + +logger = logging.getLogger() + + +class ComparisonChecker: + BENCHMARK_PREFIX = "Benchmark " + SHOW_TOPK = 10 + DIFF_AVG_RATIO = "Diff Avg Ratio" + COMPARE_MODE_TO_DESC = {CompareConstant.KERNEL_COMPARE: "Kernel compare", + CompareConstant.API_COMPARE: "Api compare"} + + def __init__(self, profiling_path, benchmark_profiling_path, step=None, benchmark_step=None, rank=None, + benchmark_rank=None): + + self.profiling_path = profiling_path + self.benchmark_profiling_path = benchmark_profiling_path + self.step = step + self.benchmark_step = benchmark_step + self.rank = rank + self.benchmark_rank = benchmark_rank + self.compare_mode = None + self.format_result = {} + self.desc = None + self.suggestion = None + + def compare(self, compare_mode): + """ + :Param event_dataset: dataset of timeline event + """ + if compare_mode is None: + return + self.compare_mode = compare_mode + compare_interface = ComparisonInterface(self.profiling_path, self.benchmark_profiling_path, self.step, + self.benchmark_step) + result = compare_interface.compare(self.compare_mode) + data = result.get(self.compare_mode, {}) + headers = data.get("headers", {}) + rows = data.get("rows", []) + format_headers = [] + + for schema in headers: + name = schema.get("name", "null") + if name not in format_headers: + format_headers.append(name) + else: + format_headers.append(f"{self.BENCHMARK_PREFIX} {name}") + + if not rows: + return + + self.format_result[self.compare_mode] = {"headers": format_headers, "rows": rows} + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + if not self.format_result: + return + + sheet_name = self._get_sheet_name() + self.desc = sheet_name + optimization_item = OptimizeItem(sheet_name, self.desc, []) + result.add(OptimizeRecord(optimization_item)) + + result.add_detail(sheet_name, headers=self.format_result.get(self.compare_mode, {}).get("headers")) + + for row in self.format_result.get(self.compare_mode, {}).get("rows"): + result.add_detail(sheet_name, detail=row) + + def make_render(self, html_render, **kwargs): + if not self.format_result: + return + + headers = self.format_result.get(self.compare_mode, {}).get("headers", []) + diff_avg_index = safe_index(headers, self.DIFF_AVG_RATIO) + rows = self.format_result.get(self.compare_mode, {}).get("rows", []) + sorted_rows = sorted(rows, + key=lambda x: convert_to_float(x[diff_avg_index]) if diff_avg_index < len(x) else -1.0, + reverse=True) + + topk_rows = [] + if sorted_rows: + topk_rows = sorted_rows[:self.SHOW_TOPK] + + if not headers or not topk_rows: + return + + html_desc = self.desc + f". Only show {self.SHOW_TOPK} rows here, see mstt_advisor*.xlsx for details" + + html_render.render_template(key="comparison", + template_dir="templates", + template_name="comparison.html", + sheet_name=self._get_sheet_name(), + desc=html_desc, + headers=headers, + rows=topk_rows) + + def _get_sheet_name(self): + + sheet_name = "" + if self.rank is not None: + sheet_name += f"Rank{self.rank}" + if self.step is not None: + sheet_name += f" Step{self.step}" + if sheet_name: + sheet_name += " and " + if self.benchmark_rank is not None: + sheet_name += f"Rank{self.benchmark_rank}" + if self.benchmark_step is not None: + sheet_name += f" Step{self.benchmark_step}" + if not sheet_name: + sheet_name = "Target and Benchmark" + + sheet_name = f"{self.COMPARE_MODE_TO_DESC.get(self.compare_mode, '')} of {sheet_name}" + return sheet_name diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 72b8dd3df..852eee906 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -36,3 +36,4 @@ class SupportedScopes: MEMORY = "memory" STAGE_COMPUTE = "stage_compute" GC_ANALYSIS = "gc_analysis" + COMPARISON = "comparison" diff --git a/profiler/advisor/display/html/templates/comparison.html b/profiler/advisor/display/html/templates/comparison.html new file mode 100644 index 000000000..5963e7530 --- /dev/null +++ b/profiler/advisor/display/html/templates/comparison.html @@ -0,0 +1,25 @@ +{% if rows|length > 0 %} +
+

{{ sheet_name }}

+
+ Issue: {{ desc }} +

+
+ + {% for header in headers %} + + {% endfor %} + + + {% for row in rows %} + + {% for element in row %} + + {% endfor %} + + {% endfor %} +
{{ header }}
{{ element|safe }}
+ +
+
+{% endif %} \ No newline at end of file diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index 9b15a419b..b8c8f15ad 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -41,6 +41,7 @@ from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_analyzer im from profiler.advisor.analyzer.memory.memory_analyzer import MemoryAnalyzer from profiler.advisor.analyzer.communication.packet.packet_analyzer import PacketAnalyzer from profiler.advisor.analyzer.schedule.gc.gc_analyzer import GcAnalyzer +from profiler.advisor.analyzer.comparison.comparison_analyzer import ComparisonAnalyzer class Interface: @@ -50,6 +51,7 @@ class Interface: OVERALL = "overall" CLUSTER = "cluster" MEMORY = "memory" + COMPARISON = "comparison" supported_analyzer = { SCHEDULE: OrderedDict({ @@ -76,7 +78,8 @@ class Interface: SupportedScopes.SLOW_RANK: SlowRankAnalyzer, SupportedScopes.SLOW_LINK: SlowLinkAnalyzer }), - MEMORY: OrderedDict({SupportedScopes.MEMORY: MemoryAnalyzer}) + MEMORY: OrderedDict({SupportedScopes.MEMORY: MemoryAnalyzer}), + COMPARISON: OrderedDict({SupportedScopes.COMPARISON: ComparisonAnalyzer}) } all_dimension = list(supported_analyzer.keys()) diff --git a/profiler/advisor/result/result.py b/profiler/advisor/result/result.py index 249b88539..19522ff7b 100644 --- a/profiler/advisor/result/result.py +++ b/profiler/advisor/result/result.py @@ -19,7 +19,7 @@ class ResultWriter: def __init__(self, result_path=None): self.result_path = result_path - self.workbook = xlsxwriter.Workbook(result_path) + self.workbook = xlsxwriter.Workbook(result_path, {"nan_inf_to_errors": True}) self.header_format = None self.data_cell_format = None -- Gitee From b11305f4e8820f84f605903b9f288e506ab29ab2 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 23 Aug 2024 09:26:46 +0800 Subject: [PATCH 470/791] add ut --- .../api_accuracy_checker/common/utils.py | 24 +++++++++++-- .../api_accuracy_checker/run_ut/run_ut.py | 14 +++----- .../common/test_common_utils.py | 35 +++++++++++++++++++ 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 67e2ab2a7..834a63d23 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -166,20 +166,39 @@ class UtDataProcessor: def extract_basic_api_segments(api_full_name): + """ + Function Description: + Extract the name of the API. + Parameter: + api_full_name: Full name of the API. Example: torch.matmul.0, torch.linalg.inv.0 + Return: + api_type: Type of api. Example: torch, tensor, etc. + api_name: Name of api. Example: matmul, linalg.inv, etc. + """ api_type = None - prefix = None api_parts = api_full_name.split(Const.SEP) api_parts_length = len(api_parts) if api_parts_length == Const.THREE_SEGMENT: api_type, api_name, _ = api_parts elif api_parts_length == Const.FOUR_SEGMENT: api_type, prefix, api_name, _ = api_parts + api_name = prefix + Const.SEP + api_name else: api_name = None - return api_type, prefix, api_name + return api_type, api_name def extract_detailed_api_segments(full_api_name_with_direction_status): + """ + Function Description: + Extract the name of the API. + Parameter: + full_api_name_with_direction_status: Full name of the API. Example: torch.matmul.0.forward.output.0 + Return: + api_name: Name of api. Example: matmul, mul, etc. + full_api_name: Full name of api. Example: torch.matmul.0 + direction_status: Direction status of api. Example: forward, backward, etc. + """ api_type = None prefix = None api_name = None @@ -192,6 +211,7 @@ def extract_detailed_api_segments(full_api_name_with_direction_status): elif api_parts_length == Const.SEVEN_SEGMENT: api_type, prefix, api_name, api_order, direction_status, _, _ = api_parts full_api_name = Const.SEP.join([api_type, prefix, api_name, api_order]) + api_name = prefix + Const.SEP + api_name else: full_api_name = None return api_name, full_api_name, direction_status diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 72c619f48..adc1515ea 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -200,7 +200,7 @@ def run_api_offline(config, compare, api_name_set): continue if is_unsupported_api(api_full_name): continue - _, _, api_name = extract_basic_api_segments(api_full_name) + _, api_name = extract_basic_api_segments(api_full_name) if not api_name: err_message = f"API {api_full_name} not support for run ut. SKIP." logger.error(err_message) @@ -252,7 +252,7 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - _, _, api_name = extract_basic_api_segments(api_full_name) + _, api_name = extract_basic_api_segments(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue dispatcher.update_consume_queue(api_data) @@ -273,7 +273,7 @@ def run_api_online(config, compare): if not isinstance(api_data, ApiData): continue api_full_name = api_data.name - _, _, api_name = extract_basic_api_segments(api_full_name) + _, api_name = extract_basic_api_segments(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue dispatcher.update_consume_queue(api_data) @@ -320,9 +320,7 @@ def do_save_error_data(api_full_name, data_info, error_data_path, is_fwd_success def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict): in_fwd_data_list = [] backward_message = '' - api_type, prefix, api_name = extract_basic_api_segments(api_full_name) - if prefix: - api_name = prefix + Const.SEP + api_name + api_type, api_name = extract_basic_api_segments(api_full_name) args, kwargs, need_grad = get_api_info(api_info_dict, api_name, real_data_path) in_fwd_data_list.append(args) in_fwd_data_list.append(kwargs) @@ -367,9 +365,7 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict def run_torch_api_online(api_full_name, api_data, backward_content): in_fwd_data_list = [] - api_type, prefix, api_name = extract_basic_api_segments(api_full_name) - if prefix: - api_name = prefix + Const.SEP + api_name + api_type, api_name = extract_basic_api_segments(api_full_name) args, kwargs, out = api_data.args, api_data.kwargs, api_data.result in_fwd_data_list.append(args) in_fwd_data_list.append(kwargs) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py index 00e3e9333..cd292b15e 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py @@ -48,3 +48,38 @@ class TestUtils(unittest.TestCase): convert_type, processed_api_info = api_info_preprocess(api_name, api_info.copy()) self.assertEqual(convert_type, 'int32_to_int64') self.assertEqual(processed_api_info, api_info) + + def test_extract_basic_api_segments(self): + api_full_name = 'torch.matmul.0' + api_type, api_name = extract_basic_api_segments(api_full_name) + self.assertEqual(api_type, 'torch') + self.assertEqual(api_name, 'matmul') + + api_full_name = 'torch.linalg.vector_norm.0' + api_type, api_name = extract_basic_api_segments(api_full_name) + self.assertEqual(api_type, 'torch') + self.assertEqual(api_name, 'linalg.vector_norm') + + api_full_name = 'torch.nn.linalg.vector_norm.0' + api_type, api_name = extract_basic_api_segments(api_full_name) + self.assertEqual(api_type, None) + self.assertEqual(api_name, None) + + def test_extract_detailed_api_segments(self): + api_full_name = 'torch.matmul.0.forward.output.0' + api_name, full_api_name, direction_status = extract_detailed_api_segments(api_full_name) + self.assertEqual(api_name, 'matmul') + self.assertEqual(full_api_name, 'torch.matmul.0') + self.assertEqual(direction_status, 'forward') + + api_full_name = 'torch.linalg.vector_norm.0.backward.output.0' + api_name, full_api_name, direction_status = extract_detailed_api_segments(api_full_name) + self.assertEqual(api_name, 'linalg.vector_norm') + self.assertEqual(full_api_name, 'torch.linalg.vector_norm.0') + self.assertEqual(direction_status, 'backward') + + api_full_name = 'torch.nn.functional.linear.0.input.0.1' + api_name, full_api_name, direction_status = extract_detailed_api_segments(api_full_name) + self.assertEqual(api_name, None) + self.assertEqual(full_api_name, None) + self.assertEqual(direction_status, None) -- Gitee From 9b803a05df3f68ce0ea12d8cd1659360e79c4889 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 23 Aug 2024 09:30:56 +0800 Subject: [PATCH 471/791] fix bug --- .../msprobe/pytorch/api_accuracy_checker/compare/compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index b4deeea19..71674a8e8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -167,7 +167,7 @@ class Comparator: """Get compare result and write to result and detail csv. is_online: bool, default False. True: called by online api precision compare, only compare without write to csv. """ - _, _, api_name = extract_basic_api_segments(full_api_name) + _, api_name = extract_basic_api_segments(full_api_name) if not api_name: raise ValueError(f"API name {full_api_name} has not been adapted.") bench_output, device_output = data_info.bench_output, data_info.device_output -- Gitee From 1e281ce08ca9bf72ba50e8b91275d6e0d9e2aac0 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 23 Aug 2024 11:13:52 +0800 Subject: [PATCH 472/791] code review --- .../msprobe/mindspore/doc/api_accuracy_checker.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md b/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md index 7c86fc272..7bb454b90 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md @@ -27,13 +27,13 @@ MindSpore动态图精度预检工具通过扫描昇腾NPU上用户训练MindSpor ```bash -msprobe -f pytorch run_ut -api_info ./dump.json +msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result ``` -| 参数名称 | 说明 | 是否必选 | -| ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | -| -api_info或--api_info_file | 指定API信息文件dump.json。 | 是 | -| -o或--out_path | 指定预检结果存盘路径，默认“./”。 | 否 | +| 参数名称 | 说明 |参数类型 | 是否必选 | +| ---------------------------- | --------------------------------------|---------------------- | ---------------------------------- | +| -api_info或--api_info_file | 指定API信息文件dump.json。 | str | 是 | +| -o或--out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | 预检执行结果包括`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`两个文件。`accuracy_checking_result_{timestamp}.csv`是API粒度的，标明每个API是否通过测试。建议用户先查看`accuracy_checking_result_{timestamp}.csv`文件，对于其中没有通过测试的或者特定感兴趣的API，根据其API Name字段在`accuracy_checking_details_{timestamp}.csv`中查询其各个输出的达标情况以及比较指标。详细介绍请参见“**预检结果**”。 -- Gitee From 511785869c89c268ff353bca4710c9ca5c0a4d7c Mon Sep 17 00:00:00 2001 From: CSNIU Date: Fri, 23 Aug 2024 12:19:52 +0800 Subject: [PATCH 473/791] =?UTF-8?q?MindSpore=E5=8A=A8=E6=80=81=E5=9B=BE?= =?UTF-8?q?=E5=90=8C=E6=A1=86=E6=9E=B6api=E7=B2=BE=E5=BA=A6=E6=AF=94?= =?UTF-8?q?=E5=AF=B9ut=E8=A1=A5=E5=85=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/core_ut/compare/test_acc_compare.py | 259 ++++++++++++++++++ 1 file changed, 259 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py new file mode 100644 index 000000000..b7eeca759 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py @@ -0,0 +1,259 @@ +# coding=utf-8 +import unittest +import pandas as pd +from msprobe.core.compare.check import check_graph_mode +from msprobe.core.compare.utils import merge_tensor, read_op, get_accuracy, rename_api +from msprobe.core.compare.highlight import find_error_rows,find_compare_result_error_rows + +npu_dict = {'op_name': ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.1', + 'Functional.conv2d.0.forward.input.2', 'Functional.conv2d.0.forward.output'], + 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), + ('torch.float32', [16])], + 'output_struct': [('torch.float32', [1, 16, 28, 28])], + 'summary': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], + [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], + [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], + [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} + +bench_dict = {'op_name': ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.1', + 'Functional.conv2d.0.forward.input.2', 'Functional.conv2d.0.forward.output'], + 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), + ('torch.float32', [16])], + 'output_struct': [('torch.float32', [1, 16, 28, 28])], + 'summary': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], + [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], + [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], + [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} + +tensor_list = [ + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, + 'Min': -0.331031858921051,'Mean': -0.030964046716690063, 'Norm': 2.2533628940582275, 'requires_grad': True, + 'full_op_name': 'Tensor.add_.0.forward_input.0'}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_.0.forward_input.1'}, + {'full_op_name': 'Tensor.add_.0.forward_input.alpha.0', 'dtype': "", "shape": '[]', 'md5': None, + 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_.0.forward_output.0'} +] + +result_op_dict = {'op_name': ['Tensor.add_.0.forward_input.0', 'Tensor.add_.0.forward_input.1', + 'Tensor.add_.0.forward_input.alpha.0', 'Tensor.add_.0.forward_output.0'], + 'input_struct': [('torch.float32', [16, 1, 3, 3]), ('torch.float32', [16, 1, 3, 3]), + ("", '[]')], + 'output_struct': [('torch.float32', [16, 1, 3, 3])], + 'summary': [[0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275], + [0.003992878366261721, -0.008102823048830032, -0.0002002553956117481, 0.02844562754034996], + [-0.1, -0.1, -0.1, -0.1], + [0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275]], + 'stack_info': []} + +o_result = [ + ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.0', 'torch.float32', 'torch.float32', + [1, 1, 28, 28], [1, 1, 28, 28], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 3.029174327850342, -2.926689624786377, + -0.06619918346405029, 3.029174327850342, -2.926689624786377, -0.06619918346405029, '', '', 'None'], + ['Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.1', 'torch.float32', 'torch.float32', + [16, 1, 5, 5], [16, 1, 5, 5], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 0.19919930398464203, -0.19974489510059357, + 0.006269412115216255, 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, '', '', 'None'], + ['Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_input.2', 'torch.float32', 'torch.float32', + [16], [16], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, + 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, '', '', 'None'], + ['Functional_conv2d_0_forward_output', 'Functional_conv2d_0_forward_output', 'torch.float32', 'torch.float32', + [1, 16, 28, 28], [1, 16, 28, 28], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 2.1166646480560303, -2.190781354904175, + -0.003579073818400502, 2.1166646480560303, -2.190781354904175, -0.003579073818400502, '', '', 'None']] + +npu_dict_aten = {'op_name': ['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', + 'Aten__native_batch_norm_legit_functional.default_0_forward_input.1', + 'Aten__native_batch_norm_legit_functional.default_0_forward_input.2', + 'Aten__native_batch_norm_legit_functional.default_0_forward_input.3', + 'Aten__native_batch_norm_legit_functional.default_0_forward_input.4', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.0', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.1', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.2', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.3', + 'Aten__native_batch_norm_legit_functional.default_0_forward_output.4'], + 'input_struct': [('torch.float16', [256, 256, 14, 14]), ('torch.float32', [256]), + ('torch.float32', [256]), ('torch.float32', [256]), ('torch.float32', [256])], + 'output_struct': [('torch.float16', [256, 256, 14, 14]), ('torch.float32', [256]), + ('torch.float32', [256]), ('torch.float32', [256]), ('torch.float32', [256])], + 'summary': [[139.625, -127.5625, -0.0103607177734375], + [2.5276029109954834, -2.1788690090179443, -0.0008259844034910202], + [2.472219944000244, -2.845968723297119, -0.008756577968597412], + [2.763145923614502, -3.398397922515869, -0.052132632583379745], + [2.673110008239746, -3.149275064468384, 0.01613386906683445], + [13.5546875, -10.640625, -0.008758544921875], + [0.30550330877304077, -0.24485322833061218, -0.010361209511756897], + [623.9192504882812, 432.96826171875, 520.2276611328125], + [2.4797861576080322, -3.055997371673584, -0.04795549064874649], + [61.7945556640625, 42.59713363647461, 52.03831481933594]]} + +bench_dict_functional = { + 'op_name': ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.1', + 'Functional_batch_norm_0_forward_input.2', 'Functional_batch_norm_0_forward_input.3', + 'Functional_batch_norm_0_forward_input.4', 'Functional_batch_norm_0_forward_output'], + 'input_struct': [('torch.float32', [256, 256, 14, 14]), ('torch.float32', [256]), ('torch.float32', [256]), + ('torch.float32', [256]), ('torch.float32', [256])], + 'output_struct': [('torch.float32', [256, 256, 14, 14])], + 'summary': [[3.061628818511963, -3.22507381439209, 3.634914173744619e-05], + [0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06], + [0.9338104128837585, 0.9277191162109375, 0.930335283279419], + [1.0, 1.0, 1.0], [0.0, 0.0, 0.0], + [5.397906303405762, -5.796811580657959, 2.5283952709287405e-10]] +} + +aten_result = [ + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', + 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 136.56337118148804, -124.33742618560791, + -0.010397066915174946, ' ', '4460.480981749501%', '3855.335826136584%', '28603.33536971545%', ' ', 139.625, + -127.5625, -0.0103607177734375, 3.061628818511963, -3.22507381439209, 3.634914173744619e-05, 'Warning', + 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.1', 'Functional_batch_norm_0_forward_input.1', + 'torch.float32', 'torch.float32', [256], [256], 2.527024927258026, -2.1782388387364335, -0.0008296193100250093, + ' ', '437213.84590749856%', '345658.76916858414%', '22823.676544842117%', ' ', 2.5276029109954834, + -2.1788690090179443, -0.0008259844034910202, 0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06, + 'Warning', 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.2', 'Functional_batch_norm_0_forward_input.2', + 'torch.float32', 'torch.float32', [256], [256], 1.5384095311164856, -3.7736878395080566, -0.9390918612480164, ' ', + '164.74538192025793%', '406.7705163736246%', '100.94122819224167%', ' ', 2.472219944000244, -2.845968723297119, + -0.008756577968597412, 0.9338104128837585, 0.9277191162109375, 0.930335283279419, 'Warning', + 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.3', 'Functional_batch_norm_0_forward_input.3', + 'torch.float32', 'torch.float32', [256], [256], 1.763145923614502, -4.398397922515869, -1.0521326325833797, ' ', + '176.3145923614502%', '439.8397922515869%', '105.21326325833797%', ' ', 2.763145923614502, -3.398397922515869, + -0.052132632583379745, 1.0, 1.0, 1.0, 'Warning', 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_input.4', 'Functional_batch_norm_0_forward_input.4', + 'torch.float32', 'torch.float32', [256], [256], 2.673110008239746, -3.149275064468384, 0.01613386906683445, ' ', + 'N/A', 'N/A', 'N/A', ' ', 2.673110008239746, -3.149275064468384, 0.01613386906683445, 0.0, 0.0, 0.0, 'Warning', + 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.0', 'Functional_batch_norm_0_forward_output', + 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 8.156781196594238, -4.843813419342041, + -0.008758545174714527, ' ', '151.11009228611078%', '83.55995967687207%', '3464072756.115108%', ' ', 13.5546875, + -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Warning', + 'Need double check api accuracy.', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', + ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', + 'Yes', '', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', + ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', + 'Yes', '', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.3', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', + ' ', ' ', ' ', ' ', ' ', 2.4797861576080322, -3.055997371673584, -0.04795549064874649, 'Nan', 'Nan', 'Nan', + 'Yes', '', 'None'], + ['Aten__native_batch_norm_legit_functional.default_0_forward_output.4', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', + ' ', ' ', ' ', ' ', ' ', 61.7945556640625, 42.59713363647461, 52.03831481933594, 'Nan', 'Nan', 'Nan', + 'Yes', '', 'None']] + +highlight_dict = {'red_rows': [], 'yellow_rows': []} + +num_0, num_1, num_2, num_3 = 0, 1, 2, 3 +summary_line_input = ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.01, 0, 0, 0, 1, 1, 1, 1, 1.01, 1, 1, 1, + 'Yes', ''] +summary_line_1 = ['Functional_batch_norm_0_forward_output.0', 'Functional_batch_norm_0_forward_output.0', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 10, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, + 'Warning', ''] +summary_line_2 = ['Functional_batch_norm_0_forward_output.1', 'Functional_batch_norm_0_forward_output.1', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.02, 0, 0, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, + 'Warning', ''] +summary_line_3 = ['Functional_batch_norm_0_forward_output.2', 'Functional_batch_norm_0_forward_output.2', + 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, + 'Warning', ''] +line_input = ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 1, 1, 1, 0.95, 1, 1, 1, 1, 1, 1.01, 1, 1, 1, + 'Yes', ''] +line_1 = ['Functional_batch_norm_0_forward_output.0', 'Functional_batch_norm_0_forward_output.0', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 1, 1, 0.59, 1, 'nan', 0, 1, 1, 19, 1, 1, 1, + 'Warning', ''] +line_2 = ['Functional_batch_norm_0_forward_output.1', 'Functional_batch_norm_0_forward_output.1', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.9, 1, 1, 0.8, 1, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, + 'Warning', ''] +line_3 = ['Functional_batch_norm_0_forward_output.2', 'Functional_batch_norm_0_forward_output.2', 'torch.float16', + 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 1.1e+10, 1, 0.85, 1, 9, 0.12, 0, 1, 1, 0.1, 1, + 1, 1, 'Warning', ''] + +op_data = { + 'input_args': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051,'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, + 'Norm': 0.02844562754034996, 'requires_grad': False}], + 'input_kwargs': {'alpha': {'type': 'float', 'value': -0.1}}, + 'output': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051,'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True}]} + +op_name = "Tensor.add_0.0.forward" + +op_result = [ + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.input.0'}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, + 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.forward.input.1'}, + {'full_op_name': 'Tensor.add_0.0.forward.input.alpha.0', 'dtype': "", 'shape': '[]', 'md5': None, + 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, + {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], + 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, + 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.output.0'}] + + +class TestUtilsMethods(unittest.TestCase): + + def test_check_graph_mode(self): + op1 = "Aten" + op2 = "torch" + self.assertTrue(check_graph_mode(op1, op2)) + self.assertTrue(check_graph_mode(op2, op1)) + self.assertFalse(check_graph_mode(op1, op1)) + self.assertFalse(check_graph_mode(op2, op2)) + + def test_merge_tensor(self): + op_dict = merge_tensor(tensor_list, True, False) + self.assertEqual(op_dict, result_op_dict) + + def test_read_op(self): + result = read_op(op_data, op_name) + self.assertEqual(result, op_result) + + def test_get_accuracy(self): + result = [] + get_accuracy(result, npu_dict, bench_dict, highlight_dict) + self.assertEqual(result, o_result) + + def test_get_accuracy_graph_mode(self): + result = [] + get_accuracy(result, npu_dict_aten, bench_dict_functional, highlight_dict) + self.assertEqual(result, aten_result) + + def test_find_error_rows(self): + summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] + highlight_dict = {'red_rows': [], 'yellow_rows': []} + find_error_rows(summary_result, 0, 1, highlight_dict, summary_compare=True) + self.assertEqual(highlight_dict, {'red_rows': [], 'yellow_rows': []}) + + def test_find_compare_result_error_rows(self): + result = [line_input, line_1, line_2, line_3] + result_df = pd.DataFrame(result) + highlight_dict = {'red_rows': [], 'yellow_rows': []} + find_compare_result_error_rows(result_df, highlight_dict, False, False) + self.assertEqual(highlight_dict, {'red_rows': [num_1, num_3], 'yellow_rows': [num_2]}) + + def test_rename_api(self): + test_name_1 = "Distributed.broadcast.0.forward.input.0" + expect_name_1 = "Distributed.broadcast.input.0" + actual_name_1 = rename_api(test_name_1, "forward") + self.assertEqual(actual_name_1, expect_name_1) + + test_name_2 = "Torch.sum.0.backward.output.0" + expect_name_2 = "Torch.sum.output.0" + actual_name_2 = rename_api(test_name_2, "backward") + self.assertEqual(actual_name_2, expect_name_2) + \ No newline at end of file -- Gitee From e02ce1b30a14ce943ffee4638493737ecfb0ae19 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Fri, 23 Aug 2024 12:34:11 +0800 Subject: [PATCH 474/791] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=90=84=E8=87=AA?= =?UTF-8?q?=E7=9A=84ut?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mindspore_ut/compare/test_ms_compare.py | 32 +++ .../pytorch_ut/compare/test_acc_compare.py | 272 ------------------ .../pytorch_ut/compare/test_pt_compare.py | 20 ++ 3 files changed, 52 insertions(+), 272 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py delete mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py create mode 100644 debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py new file mode 100644 index 000000000..9ff0c7bfc --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py @@ -0,0 +1,32 @@ +# coding=utf-8 +import unittest +from msprobe.mindspore.compare.ms_compare import MSComparator + +npu_dict = {'op_name': ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.1', + 'Functional.conv2d.0.forward.input.2', 'Functional.conv2d.0.forward.output'], + 'input_struct': [('Float32', [1, 1, 28, 28]), ('Float32', [16, 1, 5, 5]), + ('Float32', [16])], + 'output_struct': [('Float32', [1, 16, 28, 28])], + 'summary': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], + [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], + [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], + [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} + +bench_dict = {'op_name': ['Functional.conv2d.0.forward.input.0', 'Functional.conv2d.0.forward.input.1', + 'Functional.conv2d.0.forward.input.2', 'Functional.conv2d.0.forward.output'], + 'input_struct': [('Float32', [1, 1, 28, 28]), ('Float32', [16, 1, 5, 5]), + ('Float32', [16])], + 'output_struct': [('Float32', [1, 16, 28, 28])], + 'summary': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], + [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], + [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], + [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} + +class TestUtilsMethods(unittest.TestCase): + + def test_check_op_ms(self): + fuzzy_match = False + ms_comparator = MSComparator() + result = ms_comparator.check_op(npu_dict, bench_dict, fuzzy_match) + self.assertEqual(result, True) + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py deleted file mode 100644 index ad8e9453e..000000000 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_acc_compare.py +++ /dev/null @@ -1,272 +0,0 @@ -# coding=utf-8 -import unittest -import pandas as pd -from msprobe.core.compare.check import check_graph_mode -from msprobe.core.compare.utils import merge_tensor, read_op, get_accuracy, rename_api -from msprobe.core.compare.highlight import find_error_rows,find_compare_result_error_rows -from msprobe.pytorch.compare.pt_compare import PTComparator - -npu_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', - 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], - 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), - ('torch.float32', [16])], - 'output_struct': [('torch.float32', [1, 16, 28, 28])], - 'summary': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], - [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], - [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], - [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} - -bench_dict = {'op_name': ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.1', - 'Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_output'], - 'input_struct': [('torch.float32', [1, 1, 28, 28]), ('torch.float32', [16, 1, 5, 5]), - ('torch.float32', [16])], - 'output_struct': [('torch.float32', [1, 16, 28, 28])], - 'summary': [[3.029174327850342, -2.926689624786377, -0.06619918346405029], - [0.19919930398464203, -0.19974489510059357, 0.006269412115216255], - [0.19734230637550354, -0.18177609145641327, 0.007903944700956345], - [2.1166646480560303, -2.190781354904175, -0.003579073818400502]], 'stack_info': []} - -tensor_list = [ - {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], 'Max': 0.33033010363578796, - 'Min': -0.331031858921051,'Mean': -0.030964046716690063, 'Norm': 2.2533628940582275, 'requires_grad': True, - 'full_op_name': 'Tensor.add_.0.forward_input.0'}, - {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], - 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, - 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_.0.forward_input.1'}, - {'full_op_name': 'Tensor.add_.0.forward_input.alpha.0', 'dtype': "", "shape": '[]', 'md5': None, - 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, - {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], - 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_.0.forward_output.0'} -] - -result_op_dict = {'op_name': ['Tensor.add_.0.forward_input.0', 'Tensor.add_.0.forward_input.1', - 'Tensor.add_.0.forward_input.alpha.0', 'Tensor.add_.0.forward_output.0'], - 'input_struct': [('torch.float32', [16, 1, 3, 3]), ('torch.float32', [16, 1, 3, 3]), - ("", '[]')], - 'output_struct': [('torch.float32', [16, 1, 3, 3])], - 'summary': [[0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275], - [0.003992878366261721, -0.008102823048830032, -0.0002002553956117481, 0.02844562754034996], - [-0.1, -0.1, -0.1, -0.1], - [0.33033010363578796, -0.331031858921051, -0.030964046716690063, 2.2533628940582275]], - 'stack_info': []} - -o_result = [ - ['Functional_conv2d_0_forward_input.0', 'Functional_conv2d_0_forward_input.0', 'torch.float32', 'torch.float32', - [1, 1, 28, 28], [1, 1, 28, 28], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 3.029174327850342, -2.926689624786377, - -0.06619918346405029, 3.029174327850342, -2.926689624786377, -0.06619918346405029, '', '', 'None'], - ['Functional_conv2d_0_forward_input.1', 'Functional_conv2d_0_forward_input.1', 'torch.float32', 'torch.float32', - [16, 1, 5, 5], [16, 1, 5, 5], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 0.19919930398464203, -0.19974489510059357, - 0.006269412115216255, 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, '', '', 'None'], - ['Functional_conv2d_0_forward_input.2', 'Functional_conv2d_0_forward_input.2', 'torch.float32', 'torch.float32', - [16], [16], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, - 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, '', '', 'None'], - ['Functional_conv2d_0_forward_output', 'Functional_conv2d_0_forward_output', 'torch.float32', 'torch.float32', - [1, 16, 28, 28], [1, 16, 28, 28], 0.0, 0.0, 0.0, ' ', '0.0%', '0.0%', '0.0%', ' ', 2.1166646480560303, -2.190781354904175, - -0.003579073818400502, 2.1166646480560303, -2.190781354904175, -0.003579073818400502, '', '', 'None']] - -npu_dict_aten = {'op_name': ['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', - 'Aten__native_batch_norm_legit_functional.default_0_forward_input.1', - 'Aten__native_batch_norm_legit_functional.default_0_forward_input.2', - 'Aten__native_batch_norm_legit_functional.default_0_forward_input.3', - 'Aten__native_batch_norm_legit_functional.default_0_forward_input.4', - 'Aten__native_batch_norm_legit_functional.default_0_forward_output.0', - 'Aten__native_batch_norm_legit_functional.default_0_forward_output.1', - 'Aten__native_batch_norm_legit_functional.default_0_forward_output.2', - 'Aten__native_batch_norm_legit_functional.default_0_forward_output.3', - 'Aten__native_batch_norm_legit_functional.default_0_forward_output.4'], - 'input_struct': [('torch.float16', [256, 256, 14, 14]), ('torch.float32', [256]), - ('torch.float32', [256]), ('torch.float32', [256]), ('torch.float32', [256])], - 'output_struct': [('torch.float16', [256, 256, 14, 14]), ('torch.float32', [256]), - ('torch.float32', [256]), ('torch.float32', [256]), ('torch.float32', [256])], - 'summary': [[139.625, -127.5625, -0.0103607177734375], - [2.5276029109954834, -2.1788690090179443, -0.0008259844034910202], - [2.472219944000244, -2.845968723297119, -0.008756577968597412], - [2.763145923614502, -3.398397922515869, -0.052132632583379745], - [2.673110008239746, -3.149275064468384, 0.01613386906683445], - [13.5546875, -10.640625, -0.008758544921875], - [0.30550330877304077, -0.24485322833061218, -0.010361209511756897], - [623.9192504882812, 432.96826171875, 520.2276611328125], - [2.4797861576080322, -3.055997371673584, -0.04795549064874649], - [61.7945556640625, 42.59713363647461, 52.03831481933594]]} - -bench_dict_functional = { - 'op_name': ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.1', - 'Functional_batch_norm_0_forward_input.2', 'Functional_batch_norm_0_forward_input.3', - 'Functional_batch_norm_0_forward_input.4', 'Functional_batch_norm_0_forward_output'], - 'input_struct': [('torch.float32', [256, 256, 14, 14]), ('torch.float32', [256]), ('torch.float32', [256]), - ('torch.float32', [256]), ('torch.float32', [256])], - 'output_struct': [('torch.float32', [256, 256, 14, 14])], - 'summary': [[3.061628818511963, -3.22507381439209, 3.634914173744619e-05], - [0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06], - [0.9338104128837585, 0.9277191162109375, 0.930335283279419], - [1.0, 1.0, 1.0], [0.0, 0.0, 0.0], - [5.397906303405762, -5.796811580657959, 2.5283952709287405e-10]] -} - -aten_result = [ - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', - 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 136.56337118148804, -124.33742618560791, - -0.010397066915174946, ' ', '4460.480981749501%', '3855.335826136584%', '28603.33536971545%', ' ', 139.625, - -127.5625, -0.0103607177734375, 3.061628818511963, -3.22507381439209, 3.634914173744619e-05, 'Warning', - 'Need double check api accuracy.', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.1', 'Functional_batch_norm_0_forward_input.1', - 'torch.float32', 'torch.float32', [256], [256], 2.527024927258026, -2.1782388387364335, -0.0008296193100250093, - ' ', '437213.84590749856%', '345658.76916858414%', '22823.676544842117%', ' ', 2.5276029109954834, - -2.1788690090179443, -0.0008259844034910202, 0.0005779837374575436, -0.0006301702815108001, 3.634906533989124e-06, - 'Warning', 'Need double check api accuracy.', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.2', 'Functional_batch_norm_0_forward_input.2', - 'torch.float32', 'torch.float32', [256], [256], 1.5384095311164856, -3.7736878395080566, -0.9390918612480164, ' ', - '164.74538192025793%', '406.7705163736246%', '100.94122819224167%', ' ', 2.472219944000244, -2.845968723297119, - -0.008756577968597412, 0.9338104128837585, 0.9277191162109375, 0.930335283279419, 'Warning', - 'Need double check api accuracy.', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.3', 'Functional_batch_norm_0_forward_input.3', - 'torch.float32', 'torch.float32', [256], [256], 1.763145923614502, -4.398397922515869, -1.0521326325833797, ' ', - '176.3145923614502%', '439.8397922515869%', '105.21326325833797%', ' ', 2.763145923614502, -3.398397922515869, - -0.052132632583379745, 1.0, 1.0, 1.0, 'Warning', 'Need double check api accuracy.', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_input.4', 'Functional_batch_norm_0_forward_input.4', - 'torch.float32', 'torch.float32', [256], [256], 2.673110008239746, -3.149275064468384, 0.01613386906683445, ' ', - 'N/A', 'N/A', 'N/A', ' ', 2.673110008239746, -3.149275064468384, 0.01613386906683445, 0.0, 0.0, 0.0, 'Warning', - 'Need double check api accuracy.', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.0', 'Functional_batch_norm_0_forward_output', - 'torch.float16', 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 8.156781196594238, -4.843813419342041, - -0.008758545174714527, ' ', '151.11009228611078%', '83.55995967687207%', '3464072756.115108%', ' ', 13.5546875, - -10.640625, -0.008758544921875, 5.397906303405762, -5.796811580657959, 2.5283952709287405e-10, 'Warning', - 'Need double check api accuracy.', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.1', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', - ' ', ' ', ' ', ' ', ' ', 0.30550330877304077, -0.24485322833061218, -0.010361209511756897, 'Nan', 'Nan', 'Nan', - 'Yes', '', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.2', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', - ' ', ' ', ' ', ' ', ' ', 623.9192504882812, 432.96826171875, 520.2276611328125, 'Nan', 'Nan', 'Nan', - 'Yes', '', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.3', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', - ' ', ' ', ' ', ' ', ' ', 2.4797861576080322, -3.055997371673584, -0.04795549064874649, 'Nan', 'Nan', 'Nan', - 'Yes', '', 'None'], - ['Aten__native_batch_norm_legit_functional.default_0_forward_output.4', 'Nan', 'torch.float32', 'Nan', [256], 'Nan', - ' ', ' ', ' ', ' ', ' ', 61.7945556640625, 42.59713363647461, 52.03831481933594, 'Nan', 'Nan', 'Nan', - 'Yes', '', 'None']] - -highlight_dict = {'red_rows': [], 'yellow_rows': []} - -num_0, num_1, num_2, num_3 = 0, 1, 2, 3 -summary_line_input = ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.01, 0, 0, 0, 1, 1, 1, 1, 1.01, 1, 1, 1, - 'Yes', ''] -summary_line_1 = ['Functional_batch_norm_0_forward_output.0', 'Functional_batch_norm_0_forward_output.0', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 10, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, - 'Warning', ''] -summary_line_2 = ['Functional_batch_norm_0_forward_output.1', 'Functional_batch_norm_0_forward_output.1', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.02, 0, 0, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, - 'Warning', ''] -summary_line_3 = ['Functional_batch_norm_0_forward_output.2', 'Functional_batch_norm_0_forward_output.2', - 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0, 0, 0, 0, 2, 0, 1, 1, 1, 1, 1, 1, - 'Warning', ''] -line_input = ['Functional_batch_norm_0_forward_input.0', 'Functional_batch_norm_0_forward_input.0', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 1, 1, 1, 0.95, 1, 1, 1, 1, 1, 1.01, 1, 1, 1, - 'Yes', ''] -line_1 = ['Functional_batch_norm_0_forward_output.0', 'Functional_batch_norm_0_forward_output.0', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 1, 1, 0.59, 1, 'nan', 0, 1, 1, 19, 1, 1, 1, - 'Warning', ''] -line_2 = ['Functional_batch_norm_0_forward_output.1', 'Functional_batch_norm_0_forward_output.1', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.9, 1, 1, 0.8, 1, 0, 0.12, 0, 1, 1, 0.1, 1, 1, 1, - 'Warning', ''] -line_3 = ['Functional_batch_norm_0_forward_output.2', 'Functional_batch_norm_0_forward_output.2', 'torch.float16', - 'torch.float32', [256, 256, 14, 14], [256, 256, 14, 14], 0.8, 1.1e+10, 1, 0.85, 1, 9, 0.12, 0, 1, 1, 0.1, 1, - 1, 1, 'Warning', ''] - -op_data = { - 'input_args': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], - 'Max': 0.33033010363578796, 'Min': -0.331031858921051,'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True}, - {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], - 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, - 'Norm': 0.02844562754034996, 'requires_grad': False}], - 'input_kwargs': {'alpha': {'type': 'float', 'value': -0.1}}, - 'output': [{'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], - 'Max': 0.33033010363578796, 'Min': -0.331031858921051,'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True}]} - -op_name = "Tensor.add_0.0.forward" - -op_result = [ - {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], - 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.input.0'}, - {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], - 'Max': 0.003992878366261721, 'Min': -0.008102823048830032, 'Mean': -0.0002002553956117481, - 'Norm': 0.02844562754034996, 'requires_grad': False, 'full_op_name': 'Tensor.add_0.0.forward.input.1'}, - {'full_op_name': 'Tensor.add_0.0.forward.input.alpha.0', 'dtype': "", 'shape': '[]', 'md5': None, - 'Max': -0.1, 'Min': -0.1, 'Mean': -0.1, 'Norm': -0.1, 'data_name': '-1'}, - {'type': 'torch.Tensor', 'dtype': 'torch.float32', 'shape': [16, 1, 3, 3], - 'Max': 0.33033010363578796, 'Min': -0.331031858921051, 'Mean': -0.030964046716690063, - 'Norm': 2.2533628940582275, 'requires_grad': True, 'full_op_name': 'Tensor.add_0.0.forward.output.0'}] - - -class TestUtilsMethods(unittest.TestCase): - - def test_check_graph_mode(self): - op1 = "Aten" - op2 = "torch" - self.assertTrue(check_graph_mode(op1, op2)) - self.assertTrue(check_graph_mode(op2, op1)) - self.assertFalse(check_graph_mode(op1, op1)) - self.assertFalse(check_graph_mode(op2, op2)) - - def test_check_op(self): - fuzzy_match = False - ptComparator=PTComparator() - result = ptComparator.check_op(npu_dict, bench_dict, fuzzy_match) - self.assertEqual(result, True) - - def test_merge_tensor(self): - op_dict = merge_tensor(tensor_list, True, False) - self.assertEqual(op_dict, result_op_dict) - - def test_read_op(self): - result = read_op(op_data, op_name) - self.assertEqual(result, op_result) - - def test_match_op(self): - fuzzy_match = False - ptComparator=PTComparator() - a, b = ptComparator.match_op([npu_dict], [bench_dict], fuzzy_match) - self.assertEqual(a, 0) - self.assertEqual(b, 0) - - def test_get_accuracy(self): - result = [] - get_accuracy(result, npu_dict, bench_dict, highlight_dict) - self.assertEqual(result, o_result) - - def test_get_accuracy_graph_mode(self): - result = [] - get_accuracy(result, npu_dict_aten, bench_dict_functional, highlight_dict) - self.assertEqual(result, aten_result) - - def test_find_error_rows(self): - summary_result = [summary_line_input, summary_line_1, summary_line_2, summary_line_3] - highlight_dict = {'red_rows': [], 'yellow_rows': []} - find_error_rows(summary_result, 0, 1, highlight_dict, summary_compare=True) - self.assertEqual(highlight_dict, {'red_rows': [], 'yellow_rows': []}) - - def test_find_compare_result_error_rows(self): - result = [line_input, line_1, line_2, line_3] - result_df = pd.DataFrame(result) - highlight_dict = {'red_rows': [], 'yellow_rows': []} - find_compare_result_error_rows(result_df, highlight_dict, False, False) - self.assertEqual(highlight_dict, {'red_rows': [num_1, num_3], 'yellow_rows': [num_2]}) - - def test_rename_api(self): - test_name_1 = "Distributed.broadcast.0.forward.input.0" - expect_name_1 = "Distributed.broadcast.input.0" - actual_name_1 = rename_api(test_name_1, "forward") - self.assertEqual(actual_name_1, expect_name_1) - - test_name_2 = "Torch.sum.0.backward.output.0" - expect_name_2 = "Torch.sum.output.0" - actual_name_2 = rename_api(test_name_2, "backward") - self.assertEqual(actual_name_2, expect_name_2) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py new file mode 100644 index 000000000..fefa8d11b --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py @@ -0,0 +1,20 @@ +# coding=utf-8 +import unittest +from msprobe.pytorch.compare.pt_compare import PTComparator +from msprobe.test.core_ut.compare.test_acc_compare import npu_dict, bench_dict + +class TestUtilsMethods(unittest.TestCase): + + def test_check_op(self): + fuzzy_match = False + pt_comparator = PTComparator() + result = pt_comparator.check_op(npu_dict, bench_dict, fuzzy_match) + self.assertEqual(result, True) + + def test_match_op(self): + fuzzy_match = False + pt_comparator = PTComparator() + a, b = pt_comparator.match_op([npu_dict], [bench_dict], fuzzy_match) + self.assertEqual(a, 0) + self.assertEqual(b, 0) + -- Gitee From b7e2cd9e75288ffb6abfaa15af4b1a3c2e6045a1 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Thu, 22 Aug 2024 20:03:57 +0800 Subject: [PATCH 475/791] =?UTF-8?q?PROF=E7=9B=AE=E5=BD=95=E8=BE=93?= =?UTF-8?q?=E5=85=A5=E4=B8=8Bmem=20anlysis=20bug=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../communication/packet/packet_analyzer.py | 2 ++ .../advisor/analyzer/memory/memory_analyzer.py | 3 +-- profiler/advisor/common/timeline/event.py | 16 +++++++++++++--- .../communication/communication_dataset.py | 2 ++ 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/profiler/advisor/analyzer/communication/packet/packet_analyzer.py b/profiler/advisor/analyzer/communication/packet/packet_analyzer.py index e77ea7780..444b643a1 100644 --- a/profiler/advisor/analyzer/communication/packet/packet_analyzer.py +++ b/profiler/advisor/analyzer/communication/packet/packet_analyzer.py @@ -39,6 +39,8 @@ class PacketAnalyzer(BaseCommunicationAnalyzer): @BaseCommunicationAnalyzer.check_data((CommunicationDataset.get_key(),)) def optimize(self, **kwargs): add_render_list = kwargs.get("add_render_list", True) + if not hasattr(self.dataset, "hccl_dict"): + return self.result packet_checker = PacketChecker(**kwargs) packet_checker.check_packet(self.dataset) if not packet_checker.packet_issues: diff --git a/profiler/advisor/analyzer/memory/memory_analyzer.py b/profiler/advisor/analyzer/memory/memory_analyzer.py index cd7b0a242..097b6e179 100644 --- a/profiler/advisor/analyzer/memory/memory_analyzer.py +++ b/profiler/advisor/analyzer/memory/memory_analyzer.py @@ -30,9 +30,8 @@ class MemoryAnalyzer(BaseAnalyzer): def get_priority(self, max_mem_op_dur): step_duration = getattr(self.dataset, "step_duration", None) - ratio = self.get_priority_by_time_ratio(max_mem_op_dur, step_duration) - if step_duration is None: return PriorityBackgroundColor.low + ratio = self.get_priority_by_time_ratio(max_mem_op_dur, step_duration) return ratio diff --git a/profiler/advisor/common/timeline/event.py b/profiler/advisor/common/timeline/event.py index e24d983a0..6bd159b56 100644 --- a/profiler/advisor/common/timeline/event.py +++ b/profiler/advisor/common/timeline/event.py @@ -1,4 +1,6 @@ from decimal import Decimal + + class AdvisorDict(dict): def __getstate__(self): return self.__dict__ @@ -19,6 +21,14 @@ class AdvisorDict(dict): class TimelineEvent(AdvisorDict): def ts_include(self, event): - return Decimal(self.ts) <= Decimal(event.ts) and Decimal(self.ts) + Decimal(self.dur) >= Decimal( - event.ts) + Decimal( - event.dur) \ No newline at end of file + self_ts = self.ts + event_ts = event.ts + + if not self_ts or not event_ts: + return False + + self_dur = self.dur if self.dur else 0.0 + event_dur = event.dur if event.dur else 0.0 + + return Decimal(self_ts) <= Decimal(event_ts) and Decimal(self_ts) + Decimal(self_dur) >= Decimal( + event_ts) + Decimal(event_dur) diff --git a/profiler/advisor/dataset/communication/communication_dataset.py b/profiler/advisor/dataset/communication/communication_dataset.py index 0a8c9e4a1..d813c90b6 100644 --- a/profiler/advisor/dataset/communication/communication_dataset.py +++ b/profiler/advisor/dataset/communication/communication_dataset.py @@ -30,6 +30,8 @@ class CommunicationDataset: def __init__(self, collection_path, data: dict, **kwargs) -> None: self.timeline_dir = collection_path + if not self.timeline_dir.endswith("ascend_pt"): + return self.timeline_data_list = self.get_file_path_from_directory(self.timeline_dir, lambda file: file.endswith(const.COMMUNICATION_JSON)) self.hccl_dict = defaultdict(list) -- Gitee From f62faec8925e25226ef841844d632d4a9fcec9c2 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Fri, 23 Aug 2024 14:44:10 +0800 Subject: [PATCH 476/791] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=96=99=E6=9B=B4?= =?UTF-8?q?=E6=96=B0=E3=80=91compare=E8=B5=84=E6=96=99=E9=94=99=E5=88=AB?= =?UTF-8?q?=E5=AD=97=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/mindspore/doc/compare.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md index 01b4da70b..d73c9e9f8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md @@ -55,4 +55,4 @@ msprobe精度比对工具主要通过对同一个模型，在两个不同的Mind | npu_path | 配置NPU环境下的dump.json文件（单卡场景）。数据类型：str。 | 是 | | bench_path | 配置CPU、GPU或NPU环境下的dump.json文件（单卡场景）。数据类型：str。 | 是 | | stack_path | 配置NPU dump目录下的stack.json文件。数据类型：str。 | 是| -| is_print_compare_log | 配置是否开启日志打屏。可取值True或False，默认为True。数据类型：bool | 否 | \ No newline at end of file +| is_print_compare_log | 配置是否开启日志打屏。可取值true或false，默认为true。数据类型：bool | 否 | \ No newline at end of file -- Gitee From 5c38e03187ce9053608259fdf7a01631b57a9135 Mon Sep 17 00:00:00 2001 From: yangxinxian <947098055@qq.com> Date: Fri, 23 Aug 2024 14:43:31 +0800 Subject: [PATCH 477/791] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=B5=84=E6=96=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/mindspore/doc/dump.md | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md index 6db079ef1..1582b265d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md @@ -201,6 +201,7 @@ npy文件保存的前缀和MindSpore对应关系如下： | -------------- | ---------------------------- | | Tensor | mindspore.Tensor | | Functional | mindspore.ops | +| Primitive | mindspore.ops.Primitive | | Mint | mindspore.mint | | MintFunctional | mindspore.mint.nn.functional | | Jit | mindspore.jit | -- Gitee From cfa07d317e790a29550705ab93b11bce5f345b5d Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Fri, 23 Aug 2024 14:56:33 +0800 Subject: [PATCH 478/791] =?UTF-8?q?[msprobe]=E8=BF=AD=E4=BB=A3=E4=B8=89?= =?UTF-8?q?=E5=87=BA=E5=8C=85=E9=93=BE=E6=8E=A5=E6=9B=B4=E6=96=B0=E8=B5=84?= =?UTF-8?q?=E6=96=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index f8da02e0a..f5e457efa 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -36,6 +36,7 @@ Successfully installed mindstudio_probe-{version} | 版本 | 发布日期 | 支持PyTorch版本 | 支持MindSpore版本 | 下载链接 | 校验码 | | ----- | ---------- | ---------------- | ----------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | + | 1.0.3 | 2024-08-23 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.3-py3-none-any.whl) | 7060cc141a5b98ef770cd9220995d299393f32a61938261e632c7e8b5160bef2 | | 1.0.2 | 2024-08-09 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.2-py3-none-any.whl) | e4a980e5d98c426ce5ce9842520d9bc031d3b3de621c74b3d59414cc6e238e0e | | 1.0.1 | 2024-07-25 | 2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | | 1.0 | 2024-07-09 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/1.0/ascend_training_accuracy_tools-1.0-py3-none-any.whl) | 5016dfe886c5d340ec6f60a959673355855f313c91f100680da814efb49f8e81 | -- Gitee From 275ae3075e470244d7f5512eb28a44e47237ece9 Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Fri, 23 Aug 2024 14:52:49 +0800 Subject: [PATCH 479/791] bugfix: advisor call interface of compare tools with step args continuously --- .../compare_backend/comparison_generator.py | 1 + .../compare_backend/utils/args_manager.py | 17 +++++++++++++++++ .../compare_interface/comparison_interface.py | 11 ----------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/profiler/compare_tools/compare_backend/comparison_generator.py b/profiler/compare_tools/compare_backend/comparison_generator.py index 356917215..4f422824c 100644 --- a/profiler/compare_tools/compare_backend/comparison_generator.py +++ b/profiler/compare_tools/compare_backend/comparison_generator.py @@ -53,6 +53,7 @@ class ComparisonGenerator: def run_interface(self, compare_type: str) -> dict: try: self._args_manager.init() + self._args_manager.set_compare_type(compare_type) self.load_data() interface = self.INTERFACE_DICT.get(compare_type) if interface: diff --git a/profiler/compare_tools/compare_backend/utils/args_manager.py b/profiler/compare_tools/compare_backend/utils/args_manager.py index f8b8b5047..e8b621ed4 100644 --- a/profiler/compare_tools/compare_backend/utils/args_manager.py +++ b/profiler/compare_tools/compare_backend/utils/args_manager.py @@ -167,3 +167,20 @@ class ArgsManager: if self._args.output_path: self.check_output_path(PathManager.get_realpath(self._args.output_path)) + + def set_compare_type(self, compare_type: str): + self._args.enable_profiling_compare = False + self._args.enable_operator_compare = False + self._args.enable_api_compare = False + self._args.enable_kernel_compare = False + if compare_type == Constant.OVERALL_COMPARE: + self._args.enable_profiling_compare = True + elif compare_type == Constant.OPERATOR_COMPARE: + self._args.enable_operator_compare = True + elif compare_type == Constant.API_COMPARE: + self._args.enable_api_compare = True + elif compare_type == Constant.KERNEL_COMPARE: + self._args.enable_kernel_compare = True + else: + msg = f"Invalid compare_type: {compare_type}, please check it." + raise RuntimeError(msg) \ No newline at end of file diff --git a/profiler/compare_tools/compare_interface/comparison_interface.py b/profiler/compare_tools/compare_interface/comparison_interface.py index 9f593e19e..936e5a7e8 100644 --- a/profiler/compare_tools/compare_interface/comparison_interface.py +++ b/profiler/compare_tools/compare_interface/comparison_interface.py @@ -22,17 +22,6 @@ class ComparisonInterface: comparison_step=comparison_step) def compare(self, compare_type: str) -> dict: - if compare_type == Constant.OVERALL_COMPARE: - self._args.enable_profiling_compare = True - elif compare_type == Constant.KERNEL_COMPARE: - self._args.enable_kernel_compare = True - elif compare_type == Constant.API_COMPARE: - self._args.enable_api_compare = True - elif compare_type == Constant.OPERATOR_COMPARE: - self._args.enable_operator_compare = True - else: - print('[ERROR] Invalid compare_type value: {compare_type} which not supported.') - return {} return ComparisonGenerator(self._args).run_interface(compare_type) def disaggregate_perf(self, compare_type: str) -> dict: -- Gitee From 2ffc6bbad99b7a0e7345adbf41c01890b2c9fb9c Mon Sep 17 00:00:00 2001 From: TAJh <2559659915@qq.com> Date: Fri, 23 Aug 2024 07:43:13 +0000 Subject: [PATCH 480/791] add reviewer Signed-off-by: TAJh <2559659915@qq.com> --- debug/OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/OWNERS b/debug/OWNERS index 09121722c..2eec05195 100644 --- a/debug/OWNERS +++ b/debug/OWNERS @@ -9,3 +9,4 @@ reviewers: - lv-kaimeng - litian_drinksnow - binghamhuang +- TAJh -- Gitee From 17c1ed053aaaade4b2f9e2841f7cc3f8478a375f Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 23 Aug 2024 17:30:19 +0800 Subject: [PATCH 481/791] update readme --- .../pytorch/doc/api_accuracy_checker.md | 4 +-- .../doc/api_accuracy_checker_online.md | 27 ++++++++++++++----- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md index bb0f09a03..7e32fb7e9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md @@ -175,7 +175,7 @@ config.yaml文件可以通过配置参数来控制dump和run_ut操作的白名 | --------------------- | ------------------------------------------------------------ | | API name | API名称。 | | Forward Test Success | 前向API是否通过测试，pass为通过，warning为待观察，error为错误。 | -| Backward Test Success | 反向API是否通过测试，pass为通过，warning为待观察，error为错误，如果是空白的话代表该API没有反向输出。 | +| Backward Test Success | 反向API是否通过测试，pass为通过，warning为待观察，error为错误，如果是空白的话代表该API没有反向输出，SKIP表示该API的某个参数的反向不要计算梯度。 | | Message | 提示信息。 | 该结果为中间结果，仅作为参考，建议完成“**预检结果比对**”后查看比对结果。该结果后续将会删除。 @@ -304,7 +304,7 @@ Forward Test Success和Backward Test Success是否通过测试是由`api_precisi | ULP误差判定结果 | ULP误差判定结果。
当NPU或GPU数据类型是float16或bfloat16时，以下两条标准满足其一标记为pass，否则标记为error：
NPU ULP误差大于阈值占比小于0.001；
NPU ULP误差大于阈值占比小于GPU ULP误差大于阈值占比。
当NPU或GPU数据类型是float32时，以下三条标准满足其一标记为pass，否则标记为error：
NPU ULP误差平均值小于64；
NPU ULP误差大于阈值占比小于0.05；
NPU ULP误差大于阈值占比小于GPU ULP误差大于阈值占比。 | | 双千指标 | 双千精度指标。是指NPU的Tensor中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅conv1d和conv2d使用该指标。双千指标法指标。 | | 双千指标判定结果 | 双千指标判定结果。双千指标大于0.999标记为pass，否则标记为error。 | -| 比对结果 | 综合所有指标的最终结果。如果比对指标中有error，则标记为error；有warning，则标记为warning；否则标记为pass。 | +| 比对结果 | 综合所有指标的最终结果。如果比对指标中有error，则标记为error；有warning，则标记为warning；否则标记为pass；skip表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | | 比对算法 | API使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和ULP误差比对法中的一种。 | | Message | 提示信息。当前提示该API比对结果为error或warning时对应不符合标准的指标。 | diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md index 5050689a3..71a8ba486 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md @@ -2,6 +2,7 @@ 为了应对大模型场景下，通过离线预检方式dump API输入输出数据导致的存储资源紧张问题，提供在线精度预检功能。本功能实现在执行NPU训练操作的过程中，通过TCP/IP协议在NPU Host与GPU Host设备间建立连接，将NPU上对应API的输入数据在GPU设备上运行，将两份输出数据进行比对，得到预检比对结果，从而减少数据dump的步骤，降低存储资源的占用。 +针对偏差较大的算子，两方比对（NPUvsGPU）的方法缺少裁判进行裁定。 参考离线预检，在线预检场景同时支持两方比对和三方比对方式，按照api的精度标准要求，选择比对两方比对和三方比对。 ## 在线精度预检流程 @@ -32,10 +33,11 @@ Host与GPU Host设备间建立连接，将NPU上对应API的输入数据在GPU | black_list | 预检的API黑名单，list[str]类型。参数示例：white_list=["conv1d", "conv2d"]。默认未配置黑名单，即预检全量API数据。 | 否 | | error_data_path | 配置保存精度未达标的API输入输出数据路径，str类型。在线预检模式下该参数不生效。 | 否 | | is_online | 在线预检模式开关，bool类型，可取值True（开启）、False（关闭），默认关闭。 | 是 | -| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效。 | 否 | +| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host、port和tls_path不生效。 | 否 | | host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机地址127.0.0.1或本机局域网IP。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | | port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机可用端口。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | | rank_list | 指定在线预检的Rank ID，默认值为[0]，list[int]类型，应配置为大于等于0的整数，且须根据实际卡的Rank ID配置，若所配置的值大于实际训练所运行的卡的Rank ID，则在线预检输出数据为空。GPU和NPU须配置一致。 | 是 | +| tls_path | 在线预检模式局域网场景SSL证书路径，该路径下包含私钥文件server.key和公钥文件server.crt，str类型，默认值为空，表示明文传输api数据，否则采用TLS1.2加密传输，加密传输时安全性较高，传输速率较低 | 否 | #### NPU侧在线预检配置说明 @@ -51,13 +53,23 @@ Host与GPU Host设备间建立连接，将NPU上对应API的输入数据在GPU | scope | dump范围，list[str]类型，默认未配置（list也未配置时师表dump所有api的额数据），配置方式参考[《config配置文件说明》](./../../config/README.md) | 否 | | list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有api的数据），配置方式参考[《config配置文件说明》](./../../config/README.md) | 否 | | online_run_ut | 在线预检模式开关，bool类型，可取值True（开启）、False（关闭），默认关闭。 | 是 | -| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效。 | 否 | +| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host、port和tls_path不生效。 | 否 | | host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的局域网IP地址。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | -| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的端口号。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | +| port | 在线预检模式局域网场景SSL证书路径，该路径下包含私钥文件client.key和公钥文件client.crt，str类型，默认值为空，表示明文传输api数据，否则采用TLS1.2加密传输，加密传输时安全性较高，传输速率较低 | 否 | #### 局域网场景配置示例 -若复制下列示例，请删除注释后使用。 +若采用TLS1.2协议加密传输api数据，需配置SSL证书，可参考如下生成自签名证书方法，仅供调试使用，生产环境请申请正式证书。 +```shell +创建私钥文件server.key +openssl genrsa -out server.key 2048 + +创建签名请求文件server.csr，如无要求，全部回车 +openssl req -new -key server.key -out server.csr + +自签名，生成1年期公钥文件server.crt +openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt +``` GPU侧： @@ -182,6 +194,7 @@ debugger.step() bash train.sh ``` -训练脚本执行完毕后，在GPU侧dump_path目录下生成比对结果文件，详细介绍请参见《[精度预检工具](./api_accuracy_checker.md)》中的” -**预检结果**“。 - +训练脚本执行完毕后，在GPU侧dump_path目录下生成比对结果文件， +`accuracy_checking_result_{timestamp}_rank{rank_id}.csv`和`accuracy_checking_details_{timestamp}_rank{rank_id}.csv`记录两方比对结果 +`api_precision_compare_result_{timestamp}_rank{rank_id}.csv`和`api_precision_compare_details_{timestamp}_rank{rank_id}.csv`记录三方比对结果。 +详细介绍请参见《[精度预检工具](./api_accuracy_checker.md)》中的“**预检结果**”。 \ No newline at end of file -- Gitee From 8dc7836dc14cbece342d1ea82d0254291e109bbb Mon Sep 17 00:00:00 2001 From: Henry Date: Fri, 23 Aug 2024 15:50:48 +0800 Subject: [PATCH 482/791] fix jit name --- debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py index 0b2c28ed6..82faf70b6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py @@ -1,4 +1,5 @@ import os +from mindspore import Tensor from mindspore.common.api import _MindsporeFunctionExecutor from mindspore._c_expression import PyNativeExecutor_ from msprobe.mindspore.dump.hook_cell.api_registry import api_register @@ -33,7 +34,10 @@ class JitDump(_MindsporeFunctionExecutor): def __call__(self, *args, **kwargs): api_register.api_set_ori_func() out = super().__call__(*args, **kwargs) - dump_jit(args[0], args[1], out, True) + if isinstance(args[0], Tensor): + dump_jit({}, args, out, True) + else: + dump_jit(args[0], args[1:], out, True) JitDump.jit_enable = True api_register.api_set_hook_func() return out @@ -51,6 +55,6 @@ class JitDump(_MindsporeFunctionExecutor): api_register.api_set_ori_func() output = self._executor.grad(grad, obj, weights, grad_position, *args, *(kwargs.values())) if JitDump.jit_enable: - dump_jit(obj, args, output, False) + dump_jit(obj, args, None, False) api_register.api_set_hook_func() return output \ No newline at end of file -- Gitee From 383afe1941b5c8d8d7447dc09e8df68effec440b Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 24 Aug 2024 18:12:52 +0800 Subject: [PATCH 483/791] rolling print log --- .../accuracy_tools/msprobe/core/common/const.py | 2 ++ debug/accuracy_tools/msprobe/core/common/log.py | 17 +++++++---------- .../msprobe/core/data_dump/data_collector.py | 4 ++-- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 4998f925e..496243640 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,6 +309,8 @@ class MsgConst: """ Class for log messages const """ + SPACE_LENGTH = 20 + LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index ed0776291..0e5129e75 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -7,9 +7,6 @@ from msprobe.core.common.const import MsgConst class BaseLogger: def __init__(self): - self.warning_level = "WARNING" - self.error_level = "ERROR" - self.info_level = "INFO" self.rank = None @staticmethod @@ -19,29 +16,29 @@ class BaseLogger: full_msg = f"{current_time} ({pid}) [{level}] {msg}" print(full_msg, end=end) sys.stdout.flush() - + def get_rank(self): return self.rank def filter_special_chars(func): @wraps(func) - def func_level(self, msg): + def func_level(self, msg, **kwargs): for char in MsgConst.SPECIAL_CHAR: msg = msg.replace(char, '_') - return func(self, msg) + return func(self, msg, **kwargs) return func_level @filter_special_chars - def info(self, msg): - self._print_log(self.info_level, msg) + def info(self, msg, **kwargs): + self._print_log(MsgConst.LEVEL[0], msg, **kwargs) @filter_special_chars def error(self, msg): - self._print_log(self.error_level, msg) + self._print_log(MsgConst.LEVEL[2], msg) @filter_special_chars def warning(self, msg): - self._print_log(self.warning_level, msg) + self._print_log(MsgConst.LEVEL[1], msg) def on_rank_0(self, func): def func_rank_0(*args, **kwargs): diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 3a9459947..d399ce15f 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -3,7 +3,7 @@ import os from msprobe.core.data_dump.scope import build_scope, ListScope from msprobe.core.data_dump.json_writer import DataWriter from msprobe.core.common.log import logger -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, MsgConst from msprobe.core.data_dump.data_processor.factory import DataProcessorFactory @@ -130,7 +130,7 @@ class DataCollector: if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) - logger.info(msg) + logger.info(msg+' '*MsgConst.SPACE_LENGTH, end='\r') if use_buffer: self.data_writer.flush_data_when_buffer_is_full() else: -- Gitee From 8d69077b1fc7394b22f6bb94cb73569591a1ec51 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Wed, 21 Aug 2024 15:24:42 +0800 Subject: [PATCH 484/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E5=9C=A8=E9=83=A8=E5=88=86=E5=9C=BA=E6=99=AF=E6=BA=A2?= =?UTF-8?q?=E5=87=BA=E6=A0=87=E8=AF=86=E8=A2=AB=E8=A6=86=E7=9B=96=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E6=BA=A2=E5=87=BA=E6=9C=AA=E8=83=BD=E6=A3=80=E6=B5=8B?= =?UTF-8?q?=E6=88=90=E5=8A=9F=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_processor/pytorch_processor.py | 45 +++---- .../data_processor/test_pytorch_processor.py | 113 ++++++++++++------ 2 files changed, 101 insertions(+), 57 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 1e529d2b4..e725d305c 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -14,7 +14,6 @@ from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, Module from msprobe.pytorch.free_benchmark import FreeBenchmarkCheck, UnequalRow from msprobe.pytorch.common.utils import save_pt - try: import torch_npu is_gpu = False @@ -189,6 +188,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): self.real_overflow_nums = 0 self.overflow_nums = config.overflow_nums self.forward_inplace_inputs = None + self.is_support_inf_nan = self.is_support_inf_nan() @property def is_terminated(self): @@ -199,6 +199,10 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): return True return False + @staticmethod + def is_support_inf_nan(): + return is_gpu or (hasattr(torch_npu._C, '_npu_is_support_inf_nan') and torch_npu._C._npu_is_support_inf_nan()) + @staticmethod def overflow_debug_mode_enable(): overflow_mode = os.getenv(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) @@ -219,16 +223,18 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False api_info_struct = super().analyze_forward(name, module, module_input_output) - self.maybe_save_overflow_data_and_check_overflow_times() + self.handle_overflow() return api_info_struct if self.has_overflow else None def analyze_backward(self, name, module, module_input_output: ModuleBackwardInputsOutputs): self.has_overflow = False api_info_struct = super().analyze_backward(name, module, module_input_output) - self.maybe_save_overflow_data_and_check_overflow_times() + self.handle_overflow() return api_info_struct if self.has_overflow else None - def maybe_save_overflow_data_and_check_overflow_times(self): + def handle_overflow(self): + if not self.is_support_inf_nan: + self._analyze_maybe_overflow_flag() if self.has_overflow: for file_path, tensor in self.cached_tensors_and_file_paths.items(): save_pt(tensor, file_path) @@ -253,22 +259,20 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): else: torch_npu._C._clear_overflow_npu() + def _analyze_maybe_overflow_flag(self): + try: + self.has_overflow = self.check_overflow_npu() + if self.has_overflow: + self.clear_overflow_npu() + except Exception as e: + logger.error(f"Overflow check failed, the current environment may be abnormal.") + raise RuntimeError(f"overflow check failed") from e + def _analyze_maybe_overflow_tensor(self, tensor_json): - if is_gpu or (hasattr(torch_npu._C, '_npu_is_support_inf_nan') and torch_npu._C._npu_is_support_inf_nan()): - if tensor_json['Max'] is None: - return - if np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']): - self.has_overflow = True - if np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']): - self.has_overflow = True - else: - try: - self.has_overflow = self.check_overflow_npu() - if self.has_overflow: - self.clear_overflow_npu() - except Exception as e: - logger.error(f"Overflow check failed, the current environment may be abnormal.") - raise RuntimeError(f"overflow check failed") from e + if tensor_json['Max'] is None or tensor_json['Min'] is None: + return + self.has_overflow = np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']) or \ + np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']) def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) @@ -277,8 +281,9 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): else: logger.warning(f'The file path {file_path} length exceeds limit.') single_arg = super()._analyze_tensor(tensor, suffix) - self._analyze_maybe_overflow_tensor(single_arg) single_arg.update({"data_name": dump_data_name}) + if not self.has_overflow and self.is_support_inf_nan: + self._analyze_maybe_overflow_tensor(single_arg) return single_arg diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index 89c1d9e84..104c5e469 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -209,15 +209,16 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): sys.modules['torch_npu.npu'] = Mock() sys.modules['torch_npu._C'] = Mock() - @patch('torch.save') - def test_maybe_save_overflow_data_and_check_overflow_times(self, mock_save): - self.processor.has_overflow = True - self.processor.real_overflow_nums = 0 - self.processor.cached_tensors_and_file_paths = {'dummy_path': torch.tensor([1.0, 2.0, 3.0])} - self.processor.maybe_save_overflow_data_and_check_overflow_times() - mock_save.assert_called_once() - self.processor.maybe_save_overflow_data_and_check_overflow_times() - self.assertEqual(self.processor.real_overflow_nums, 2) + def test_is_terminated(self): + self.processor.overflow_nums = -1 + self.assertFalse(self.processor.is_terminated) + + self.processor.real_overflow_nums = 2 + self.processor.overflow_nums = 2 + self.assertTrue(self.processor.is_terminated) + + self.processor.real_overflow_nums = 1 + self.assertFalse(self.processor.is_terminated) @patch('os.getenv', return_value=Const.ENV_ENABLE) def test_overflow_debug_mode_enable(self, mock_getenv): @@ -225,50 +226,88 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.assertTrue(result) mock_getenv.assert_called_once_with(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) - @patch('msprobe.core.data_dump.data_processor.pytorch_processor.is_gpu', return_value=True) - def test_analyze_maybe_overflow_tensor(self, _): - tensor_json = {'Max': float('inf'), 'Min': 1.0} - self.processor._analyze_maybe_overflow_tensor(tensor_json) - self.assertTrue(self.processor.has_overflow) + def test_analyze_pre_forward_inplace(self): + module_input_output = ModuleForwardInputsOutputs(args=(1, 2), kwargs=None, output=None) + self.processor.analyze_pre_forward_inplace(None, module_input_output) + self.assertTrue(self.processor.forward_inplace_inputs == module_input_output) + self.assertFalse(self.processor.forward_inplace_inputs is module_input_output) - @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) - @patch.object(BaseDataProcessor, 'get_save_file_path', return_value=['test_api_name', 'test_api_name.0.forward.input.pt']) - def test_analyze_tensor(self, mock_path_len_exceeds_limit, _): - tensor = torch.tensor([1.0, 2.0, 3.0]) - suffix = 'suffix' - expected = {'Max': 3.0, 'Min': 1.0, 'data_name': 'test_api_name'} - with patch.object(PytorchDataProcessor, '_analyze_tensor', return_value={'Max': 3.0, 'Min': 1.0}) as mock_super_analyze_tensor: - result = self.processor._analyze_tensor(tensor, suffix) - mock_super_analyze_tensor.assert_called_once_with(tensor, suffix) - mock_path_len_exceeds_limit.assert_called_once() - self.assertEqual(expected, result) + def test_analyze_forward_inplace(self): + with patch.object(OverflowCheckDataProcessor, "analyze_forward"): + module_input_output = ModuleForwardInputsOutputs(args=(1, 2), kwargs={"k1": 3, "k2": 4}, output=None) + self.processor.forward_inplace_inputs = module_input_output + self.processor.analyze_forward_inplace("name", module_input_output) + self.assertIsNone(self.processor.forward_inplace_inputs) + self.assertEqual(module_input_output.output, (1, 2, 3, 4)) - def test_analyze_backward(self): + def test_analyze_forward(self): def func(_): self.processor.has_overflow = True - with patch.object(BaseDataProcessor, "analyze_backward", return_value={"min", 0}): - with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times"): - api_info = self.processor.analyze_backward("name", "module", "module_input_output") + + with patch.object(BaseDataProcessor, "analyze_forward", return_value={"min", 0}): + with patch.object(OverflowCheckDataProcessor, "handle_overflow"): + api_info = self.processor.analyze_forward("name", "module", "module_input_output") self.assertFalse(self.processor.has_overflow) self.assertIsNone(api_info) - with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times", new=func): - api_info = self.processor.analyze_backward("name", "module", "module_input_output") + + with patch.object(OverflowCheckDataProcessor, "handle_overflow", new=func): + api_info = self.processor.analyze_forward("name", "module", "module_input_output") self.assertTrue(self.processor.has_overflow) self.assertEqual(api_info, {"min", 0}) - def test_analyze_forward(self): + def test_analyze_backward(self): def func(_): self.processor.has_overflow = True - with patch.object(BaseDataProcessor, "analyze_forward", return_value={"min", 0}): - with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times"): - api_info = self.processor.analyze_forward("name", "module", "module_input_output") + + with patch.object(BaseDataProcessor, "analyze_backward", return_value={"min", 0}): + with patch.object(OverflowCheckDataProcessor, "handle_overflow"): + api_info = self.processor.analyze_backward("name", "module", "module_input_output") self.assertFalse(self.processor.has_overflow) self.assertIsNone(api_info) - with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times", new=func): - api_info = self.processor.analyze_forward("name", "module", "module_input_output") + + with patch.object(OverflowCheckDataProcessor, "handle_overflow", new=func): + api_info = self.processor.analyze_backward("name", "module", "module_input_output") self.assertTrue(self.processor.has_overflow) self.assertEqual(api_info, {"min", 0}) + @patch('torch.save') + def test_handle_overflow(self, mock_save): + self.processor.has_overflow = True + self.processor.real_overflow_nums = 0 + self.processor.is_support_inf_nan = True + self.processor.cached_tensors_and_file_paths = {'dump_path': torch.tensor([1.0, 2.0, 3.0])} + self.processor.handle_overflow() + mock_save.assert_called_once() + self.processor.handle_overflow() + self.assertEqual(self.processor.real_overflow_nums, 2) + + @patch('msprobe.core.data_dump.data_processor.pytorch_processor.is_gpu', return_value=True) + def test_analyze_maybe_overflow_tensor(self, _): + tensor_json = {'Max': None, 'Min': None} + self.processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertFalse(self.processor.has_overflow) + + tensor_json = {'Max': float('inf'), 'Min': 1.0} + self.processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.processor.has_overflow) + + tensor_json = {'Max': 1.0, 'Min': float('inf')} + self.processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.processor.has_overflow) + + @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) + @patch.object(BaseDataProcessor, 'get_save_file_path', + return_value=['test_api_name', 'test_api_name.0.forward.input.pt']) + def test_analyze_tensor(self, mock_path_len_exceeds_limit, _): + tensor = torch.tensor([1.0, 2.0, 3.0]) + suffix = 'suffix' + expected = {'Max': 3.0, 'Min': 1.0, 'data_name': 'test_api_name'} + with patch.object(PytorchDataProcessor, '_analyze_tensor', + return_value={'Max': 3.0, 'Min': 1.0}) as mock_super_analyze_tensor: + result = self.processor._analyze_tensor(tensor, suffix) + mock_super_analyze_tensor.assert_called_once_with(tensor, suffix) + mock_path_len_exceeds_limit.assert_called_once() + self.assertEqual(expected, result) class TestFreeBenchmarkDataProcessor(unittest.TestCase): -- Gitee From 91156d818b2d1f257950f247d476ee936e648f90 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 26 Aug 2024 11:21:51 +0800 Subject: [PATCH 485/791] update readme --- .../msprobe/pytorch/doc/api_accuracy_checker.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md index 7e32fb7e9..db855e573 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md @@ -267,8 +267,8 @@ msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_de | 字段 | 含义 | | --------------------- | ------------------------------------------------------------ | | API name | API名称。 | -| Forward Test Success | 前向API是否通过测试，pass为通过，warning为待观察，error为错误，skip表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | -| Backward Test Success | 反向API是否通过测试，pass为通过，warning为待观察，error为错误，如果是空白的话代表该API没有反向输出，skip表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | +| Forward Test Success | 前向API是否通过测试，pass为通过，warning为待观察，error为错误，SKIP表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | +| Backward Test Success | 反向API是否通过测试，pass为通过，warning为待观察，error为错误，如果是空白的话代表该API没有反向输出，SKIP表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | | Message | 提示信息。 | Forward Test Success和Backward Test Success是否通过测试是由`api_precision_compare_details_{timestamp}.csv`中的各个指标判定结果决定的。需要注意的是`api_precision_compare_details_{timestamp}.csv`中可能存在一个API的前向（反向）有多个输出，那么每个输出记录一行，而在`api_precision_compare_result_{timestamp}.csv`中的结果需要该API的所有结果均为pass才能标记为pass，只要存在一个error则标记error，仅存在warning和pass且不存在error标记warning。 @@ -304,7 +304,7 @@ Forward Test Success和Backward Test Success是否通过测试是由`api_precisi | ULP误差判定结果 | ULP误差判定结果。
当NPU或GPU数据类型是float16或bfloat16时，以下两条标准满足其一标记为pass，否则标记为error：
NPU ULP误差大于阈值占比小于0.001；
NPU ULP误差大于阈值占比小于GPU ULP误差大于阈值占比。
当NPU或GPU数据类型是float32时，以下三条标准满足其一标记为pass，否则标记为error：
NPU ULP误差平均值小于64；
NPU ULP误差大于阈值占比小于0.05；
NPU ULP误差大于阈值占比小于GPU ULP误差大于阈值占比。 | | 双千指标 | 双千精度指标。是指NPU的Tensor中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅conv1d和conv2d使用该指标。双千指标法指标。 | | 双千指标判定结果 | 双千指标判定结果。双千指标大于0.999标记为pass，否则标记为error。 | -| 比对结果 | 综合所有指标的最终结果。如果比对指标中有error，则标记为error；有warning，则标记为warning；否则标记为pass；skip表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | +| 比对结果 | 综合所有指标的最终结果。如果比对指标中有error，则标记为error；有warning，则标记为warning；否则标记为pass；SKIP表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | | 比对算法 | API使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和ULP误差比对法中的一种。 | | Message | 提示信息。当前提示该API比对结果为error或warning时对应不符合标准的指标。 | -- Gitee From 93873f8a5b792e7ebb96c8f8fe5617995ce0b71d Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 26 Aug 2024 11:47:39 +0800 Subject: [PATCH 486/791] codeclean --- .../api_accuracy_checker/compare/api_precision_compare.py | 8 ++++---- .../pytorch/api_accuracy_checker/compare/compare.py | 2 +- .../tensor_transport_layer/device_dispatch.py | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 7cc1d6b5e..d5e20791f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -357,8 +357,8 @@ def analyse_csv(npu_data, gpu_data, config): if last_api_name is not None and api_name != last_api_name: if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message - write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) - print_test_success(api_full_name, "skip", "skip") + write_csv([[last_api_name, CompareConst.SKIP, CompareConst.SKIP, message]], config.result_csv_path) + print_test_success(api_full_name, CompareConst.SKIP, CompareConst.SKIP) forward_status, backward_status = [], [] message = '' else: @@ -388,8 +388,8 @@ def analyse_csv(npu_data, gpu_data, config): if last_api_name is not None: if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message - write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) - print_test_success(last_api_full_name, "skip", "skip") + write_csv([[last_api_name, CompareConst.SKIP, CompareConst.SKIP, message]], config.result_csv_path) + print_test_success(last_api_full_name, CompareConst.SKIP, CompareConst.SKIP) else: forward_result = get_api_checker_result(forward_status) backward_result = get_api_checker_result(backward_status) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 9c9723ed2..2a459c22c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -138,7 +138,7 @@ class Comparator: try: name = test_result[0] df_row = list(test_result[:INDEX_TEST_RESULT_GROUP]) - if test_result[1] == "SKIP": + if test_result[1] == CompareConst.SKIP: df_row.append(test_result[INDEX_TEST_RESULT_GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE]) if self.stack_info: stack_info = "\n".join(self.stack_info[name]) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 6d544dc60..3535b0880 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -5,7 +5,7 @@ import pandas as pd import torch import torch.multiprocessing as mp -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, CompareConst from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import online_api_precision_compare from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import DETAIL_TEST_ROWS, thousandth_standard_api, \ binary_standard_api, absolute_standard_api @@ -104,7 +104,7 @@ def online_precision_compare(api_data, device, common_config, api_precision_csv_ else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") - compare.write_summary_csv((api_full_name, "SKIP", "SKIP", [[str(err)]], api_data.rank)) + compare.write_summary_csv((api_full_name, CompareConst.SKIP, CompareConst.SKIP, [[str(err)]], api_data.rank)) finally: torch.cuda.empty_cache() @@ -133,7 +133,7 @@ def online_compare(api_data, device, common_config): else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") - compare.write_summary_csv((api_full_name, "SKIP", "SKIP", [[str(err)]], api_data.rank)) + compare.write_summary_csv((api_full_name, CompareConst.SKIP, CompareConst.SKIP, [[str(err)]], api_data.rank)) finally: torch.cuda.empty_cache() -- Gitee From 19fa836b5b2a42145dca7c10acb73296000d5459 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Mon, 26 Aug 2024 15:18:04 +0800 Subject: [PATCH 487/791] codeclean --- .../pytorch/doc/api_accuracy_checker_online.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md index 71a8ba486..9cb722272 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md @@ -37,7 +37,7 @@ Host与GPU Host设备间建立连接，将NPU上对应API的输入数据在GPU | host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机地址127.0.0.1或本机局域网IP。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | | port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机可用端口。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | | rank_list | 指定在线预检的Rank ID，默认值为[0]，list[int]类型，应配置为大于等于0的整数，且须根据实际卡的Rank ID配置，若所配置的值大于实际训练所运行的卡的Rank ID，则在线预检输出数据为空。GPU和NPU须配置一致。 | 是 | -| tls_path | 在线预检模式局域网场景SSL证书路径，该路径下包含私钥文件server.key和公钥文件server.crt，str类型，默认值为空，表示明文传输api数据，否则采用TLS1.2加密传输，加密传输时安全性较高，传输速率较低 | 否 | +| tls_path | 在线预检模式局域网场景SSL证书路径，该路径下包含私钥文件server.key和公钥文件server.crt，str类型，默认值为空，表示明文传输api数据，否则采用TLS1.2加密传输，加密传输时安全性较高，传输速率较低。 | 否 | #### NPU侧在线预检配置说明 @@ -55,19 +55,20 @@ Host与GPU Host设备间建立连接，将NPU上对应API的输入数据在GPU | online_run_ut | 在线预检模式开关，bool类型，可取值True（开启）、False（关闭），默认关闭。 | 是 | | nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host、port和tls_path不生效。 | 否 | | host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的局域网IP地址。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | -| port | 在线预检模式局域网场景SSL证书路径，该路径下包含私钥文件client.key和公钥文件client.crt，str类型，默认值为空，表示明文传输api数据，否则采用TLS1.2加密传输，加密传输时安全性较高，传输速率较低 | 否 | +| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的端口号。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | +| tls_path | 在线预检模式局域网场景SSL证书路径，该路径下包含私钥文件client.key和公钥文件client.crt，str类型，默认值为空，表示明文传输api数据，否则采用TLS1.2加密传输，加密传输时安全性较高，传输速率较低。 | 否 | #### 局域网场景配置示例 若采用TLS1.2协议加密传输api数据，需配置SSL证书，可参考如下生成自签名证书方法，仅供调试使用，生产环境请申请正式证书。 ```shell -创建私钥文件server.key +# 创建私钥文件server.key openssl genrsa -out server.key 2048 -创建签名请求文件server.csr，如无要求，全部回车 +# 创建签名请求文件server.csr openssl req -new -key server.key -out server.csr -自签名，生成1年期公钥文件server.crt +# 自签名, 生成1年期公钥文件server.crt openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt ``` -- Gitee From 17e97ec9694bf05e1473f401d6ce97c31fadac76 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Fri, 23 Aug 2024 17:01:07 +0800 Subject: [PATCH 488/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Dslice=E5=85=83=E7=B4=A0=E4=B8=BAnumpy.int64=E7=AD=89?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B=E6=97=B6=E6=8A=A5=E9=94=99=E7=9A=84=E9=97=AE?= =?UTF-8?q?=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/data_dump/data_processor/base.py | 31 ++++++++-- .../data_processor/mindspore_processor.py | 18 +----- .../data_processor/pytorch_processor.py | 18 +----- .../data_dump/data_processor/test_base.py | 17 ++++++ .../data_processor/test_pytorch_processor.py | 6 +- .../core_ut/data_dump/test_json_writer.py | 57 ++++++++----------- .../test_compare_online_dispatch.py | 50 ++++++++-------- 7 files changed, 101 insertions(+), 96 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index f595aedda..da43b097c 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -70,7 +70,7 @@ class TensorStatInfo: class BaseDataProcessor: _recursive_key_stack = [] special_type = (np.integer, np.floating, np.bool_, np.complexfloating, np.str_, np.byte, np.unicode_, - bool, int, float, str, slice) + bool, int, float, str, slice, type(Ellipsis)) def __init__(self, config, data_writer): self.data_writer = data_writer @@ -136,6 +136,29 @@ class BaseDataProcessor: return builtin_type(arg), type(arg).__name__ return arg, '' + @staticmethod + def _analyze_builtin(arg): + single_arg = {} + if isinstance(arg, slice): + # The slice parameter may be of the tensor, numpy or other types. + # It needs to be converted to the Python value type before JSON serialization + single_arg.update({"type": "slice"}) + values = [] + for value in [arg.start, arg.stop, arg.step]: + if value is not None: + try: + value = int(value) + except ValueError: + logger.warning(f"The data type {type(value)} cannot be converted to int type.") + value = None + values.append(value) + single_arg.update({"value": values}) + else: + single_arg.update({"type": type(arg).__name__}) + # When arg is Ellipsis(...) type, it needs to be converted to str("...") type + single_arg.update({"value": arg if arg is not Ellipsis else "..."}) + return single_arg + @staticmethod def _analyze_numpy(value, numpy_type): return {"type": numpy_type, "value": value} @@ -157,12 +180,12 @@ class BaseDataProcessor: cls._recursive_key_stack.pop() return type(args)(result_list) elif isinstance(args, dict): - resutl_dict = {} + result_dict = {} for k, arg in args.items(): cls._recursive_key_stack.append(str(k)) - resutl_dict[k] = cls.recursive_apply_transform(arg, transform) + result_dict[k] = cls.recursive_apply_transform(arg, transform) cls._recursive_key_stack.pop() - return resutl_dict + return result_dict elif args is not None: logger.warning(f"Data type {type(args)} is not supported.") return None diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index c822322b7..391482a60 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -50,22 +50,6 @@ class MindsporeDataProcessor(BaseDataProcessor): def analyze_dtype_in_kwargs(element): return {"type": "mindspore.dtype", "value": str(element)} - @staticmethod - def _analyze_builtin(arg): - single_arg = {} - if isinstance(arg, slice): - single_arg.update({"type": "slice"}) - # slice参数中可能存在tensor类型，json序列化，需要转换为python数值类型 - values = [ - value if not isinstance(value, ms.Tensor) else value.item() - for value in [arg.start, arg.stop, arg.step] - ] - single_arg.update({"value": values}) - else: - single_arg.update({"type": type(arg).__name__}) - single_arg.update({"value": arg}) - return single_arg - @classmethod def get_special_types(cls): return super().get_special_types() + cls.mindspore_special_type @@ -107,7 +91,7 @@ class MindsporeDataProcessor(BaseDataProcessor): if isinstance(element, ms.Tensor): return self._analyze_tensor(element, Const.SEP.join(suffix_stack)) - if isinstance(element, (bool, int, float, str, slice)): + if isinstance(element, (bool, int, float, str, slice, type(Ellipsis))): return self._analyze_builtin(element) return {} diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 1e529d2b4..a88a163b5 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -102,22 +102,6 @@ class PytorchDataProcessor(BaseDataProcessor): return torch._C._VariableFunctionsClass.max(data_no_nan).item() if operator == 'max' else \ torch._C._VariableFunctionsClass.min(data_no_nan).item() - @staticmethod - def _analyze_builtin(arg): - single_arg = {} - if isinstance(arg, slice): - single_arg.update({"type": "slice"}) - # slice参数中可能存在tensor类型，json序列化，需要转换为python数值类型 - values = [ - value if not isinstance(value, torch.Tensor) else value.item() - for value in [arg.start, arg.stop, arg.step] - ] - single_arg.update({"value": values}) - else: - single_arg.update({"type": type(arg).__name__}) - single_arg.update({"value": arg}) - return single_arg - @staticmethod def _analyze_torch_size(arg): return {"type": "torch.Size", "value": list(arg)} @@ -136,7 +120,7 @@ class PytorchDataProcessor(BaseDataProcessor): return self._analyze_numpy(converted_numpy, numpy_type) if isinstance(element, torch.Tensor): return self._analyze_tensor(element, Const.SEP.join(suffix_stack)) - if isinstance(element, (bool, int, float, str, slice)): + if isinstance(element, (bool, int, float, str, slice, type(Ellipsis))): return self._analyze_builtin(element) return {} diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py index 886d4b124..76182b251 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py @@ -91,6 +91,23 @@ class TestBaseDataProcessor(unittest.TestCase): self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(np.str_('test')), ('test', 'str_')) self.assertEqual(BaseDataProcessor._convert_numpy_to_builtin(5), (5, '')) + def test_analyze_builtin(self): + result = self.processor._analyze_builtin(slice(1, 10, 2)) + expected = {'type': 'slice', 'value': [1, 10, 2]} + self.assertEqual(result, expected) + + result = self.processor._analyze_builtin(slice(1, np.int64(10), np.int64(2))) + expected = {'type': 'slice', 'value': [1, 10, 2]} + self.assertEqual(result, expected) + + result = self.processor._analyze_builtin(...) + expected = {'type': 'ellipsis', 'value': "..."} + self.assertEqual(result, expected) + + result = self.processor._analyze_builtin(1) + expected = {'type': 'int', 'value': 1} + self.assertEqual(result, expected) + def test_analyze_numpy(self): result = BaseDataProcessor._analyze_numpy(5, 'int32') self.assertEqual(result, {'type': 'int32', 'value': 5}) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index 89c1d9e84..e31cbd31f 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -117,10 +117,14 @@ class TestPytorchDataProcessor(unittest.TestCase): self.assertTrue(np.isinf(result_min)) def test_analyze_builtin(self): - result = self.processor._analyze_builtin(slice(1, 10, 2)) + result = self.processor._analyze_builtin(slice(1, torch.tensor(10, dtype=torch.int32), np.int64(2))) expected = {'type': 'slice', 'value': [1, 10, 2]} self.assertEqual(result, expected) + result = self.processor._analyze_builtin(slice(torch.tensor([1, 2], dtype=torch.int32), None, None)) + expected = {'type': 'slice', 'value': [None, None, None]} + self.assertEqual(result, expected) + def test_analyze_torch_size(self): size = torch.Size([3, 4, 5]) result = self.processor._analyze_torch_size(size) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py index cfb1b3d55..fcbd05797 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py @@ -1,12 +1,13 @@ +import csv +import json +import os import unittest -from msprobe.core.data_dump.json_writer import DataWriter +from pathlib import Path -import os -import csv -from msprobe.core.common.file_check import FileOpen from msprobe.core.common import utils -from pathlib import Path -import json +from msprobe.core.common.file_check import FileOpen +from msprobe.core.data_dump.json_writer import DataWriter + class TestDataWriter(unittest.TestCase): def test_write_data_to_csv(self): @@ -16,7 +17,7 @@ class TestDataWriter(unittest.TestCase): if os.path.exists(file_path): utils.remove_path(file_path) - data = {"A":"1", "B":"2", "C":"3"} + data = {"A": "1", "B": "2", "C": "3"} result = data.values() header = data.keys() DataWriter.write_data_to_csv(result, header, file_path) @@ -25,10 +26,7 @@ class TestDataWriter(unittest.TestCase): column_first = [row for row in reader][0] self.assertEqual(data, column_first) - - - - data = {"A":"4", "B":"5", "C":"6"} + data = {"A": "4", "B": "5", "C": "6"} result = data.values() header = data.keys() DataWriter.write_data_to_csv(result, header, file_path) @@ -58,7 +56,7 @@ class TestDataWriter(unittest.TestCase): test.initialize_json_file() - with open(dump_file_path) as f: + with FileOpen(dump_file_path, "r") as f: load_data = json.load(f) result = {"dump_data_dir": dump_tensor_data_dir, "data": {}} self.assertEqual(result, load_data) @@ -77,7 +75,7 @@ class TestDataWriter(unittest.TestCase): def test_update_dump_paths(self): test = DataWriter() - self.assertTrue(test.dump_file_path == None) + self.assertIsNone(test.dump_file_path) cur_path = os.path.dirname(os.path.realpath(__file__)) test_path = os.path.join(cur_path, "test1.json") @@ -90,7 +88,7 @@ class TestDataWriter(unittest.TestCase): self.assertTrue(test.free_benchmark_file_path == test_path) def test_update_data(self): - data = {"A":"1", "B":"2", "C":{"D":"2"}} + data = {"A": "1", "B": "2", "C": {"D": "2"}} test = DataWriter() test.cache_data["data"]["test_1"] = True test.cache_data["data"]["test_2"] = False @@ -98,58 +96,54 @@ class TestDataWriter(unittest.TestCase): test.update_data(data) self.assertEqual(test.cache_data["data"]["A"], "1") - new_data = {"C":{"F":3}} + new_data = {"C": {"F": 3}} test.update_data(new_data) self.assertEqual(test.cache_data["data"]["C"]["F"], 3) - def test_flush_data_when_buffer_is_full_and_test_write_data_json(self): - data = {"A":"1", "B":"2", "data":{}} + data = {"A": "1", "B": "2", "data": {}} test = DataWriter() test.buffer_size = 1 - test.cache_data["data"] = {"A":"1", "B":"2", "C":"3"} + test.cache_data["data"] = {"A": "1", "B": "2", "C": "3"} self.assertTrue(len(test.cache_data["data"]) >= test.buffer_size) cur_path = os.path.dirname(os.path.realpath(__file__)) dump_tensor_data_dir = os.path.join(cur_path, "dump_tensor_data.json") dump_file_path = os.path.join(cur_path, "dump_file.json") - stack_file_path = os.path.join(cur_path, "stack_file.json") - construct_file_path = os.path.join(cur_path, "construct_file.json") test.dump_file_path = dump_file_path test.dump_tensor_data_dir = dump_tensor_data_dir - with open(dump_file_path, "w") as f: + with FileOpen(dump_file_path, "w") as f: dump_data = json.dumps(data) f.write(dump_data) test.flush_data_when_buffer_is_full() - with open(dump_file_path, "r") as f: + with FileOpen(dump_file_path, "r") as f: new_data = json.load(f) - - data.update({"data": {"A":"1", "B":"2", "C":"3"}}) + + data.update({"data": {"A": "1", "B": "2", "C": "3"}}) self.assertEqual(new_data, data) self.assertTrue(test.cache_data["data"] == {}) os.remove(dump_file_path) - def test_update_stack(self): - data = {"A":"1", "B":"2", "data":{}} + data = {"A": "1", "B": "2", "data": {}} test = DataWriter() test.update_stack(data) self.assertEqual(test.cache_stack, data) def test_update_construct(self): - data = {"A":"1", "B":"2", "data":{}} + data = {"A": "1", "B": "2", "data": {}} test = DataWriter() test.update_construct(data) self.assertEqual(test.cache_construct, data) def test_write_stack_info_json(self): test = DataWriter() - data = {"A":"1", "B":"2", "data":{}} + data = {"A": "1", "B": "2", "data": {}} test.cache_stack = data cur_path = os.path.dirname(os.path.realpath(__file__)) @@ -157,17 +151,16 @@ class TestDataWriter(unittest.TestCase): test.write_stack_info_json(file_path) - with open(file_path, "r") as f: + with FileOpen(file_path, "r") as f: load_result = json.load(f) try: self.assertEqual(load_result, data) finally: os.remove(file_path) - def test_write_construct_info_json(self): test = DataWriter() - data = {"A":"1", "B":"2", "data":{}} + data = {"A": "1", "B": "2", "data": {}} test.cache_construct = data cur_path = os.path.dirname(os.path.realpath(__file__)) @@ -175,7 +168,7 @@ class TestDataWriter(unittest.TestCase): test.write_construct_info_json(file_path) - with open(file_path, "r") as f: + with FileOpen(file_path, "r") as f: load_result = json.load(f) try: self.assertEqual(load_result, data) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py index 4db0981b2..6cbdc4db0 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py @@ -12,19 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -import unittest -import torch -from msprobe.pytorch.online_dispatch.compare import get_json_contents, Saver - import json -import csv import os -import logging -import threading +import unittest from pathlib import Path -import pandas as pd from unittest.mock import Mock, patch + +import pandas as pd +from msprobe.core.common.file_check import FileOpen from msprobe.core.common.utils import CompareException +from msprobe.pytorch.online_dispatch.compare import get_json_contents, Saver + class TestCompare(unittest.TestCase): def setUp(self): @@ -40,26 +38,27 @@ class TestCompare(unittest.TestCase): os.remove(self.list_json_path) def test_get_json_contents_when_get_json(self): - data = {"one":1} - with open(self.dict_json_path,'w') as f: + data = {"one": 1} + with FileOpen(self.dict_json_path, 'w') as f: json.dump(data, f) - self.assertEqual(get_json_contents(self.dict_json_path),data) + self.assertEqual(get_json_contents(self.dict_json_path), data) @patch('msprobe.core.common.log.BaseLogger.error') - def test_get_json_contents_when_get_list(self,mock_error): - data = [1,2] - with open(self.list_json_path,'w') as f: + def test_get_json_contents_when_get_list(self, mock_error): + data = [1, 2] + with FileOpen(self.list_json_path, 'w') as f: json.dump(data, f) with self.assertRaises(CompareException) as context: get_json_contents(self.list_json_path) self.assertEqual(context.exception.code, CompareException.INVALID_FILE_ERROR) mock_error.assert_called_once_with('Json file %s, content is not a dictionary!' % self.list_json_path) + class TestSaver(unittest.TestCase): def setUp(self): self.save_path = "./saver_save.csv" self.detail_save_path = "./saver_detail.csv" - self.saver = Saver(self.save_path,self.detail_save_path,False) + self.saver = Saver(self.save_path, self.detail_save_path, False) Path(self.save_path).touch() Path(self.detail_save_path).touch() @@ -71,14 +70,16 @@ class TestSaver(unittest.TestCase): def test_write_csv_title(self): self.saver.write_csv_title() - mock_data_save = {self.saver.COLUMN_API_NAME:{}, - self.saver.COLUMN_FORWARD_SUCCESS:{}, - self.saver.COLUMN_BACKWARD_SUCCESS:{}, - "Message":{}} - mock_data_detail = {'Npu Name': {}, 'Bench Dtype': {}, 'NPU Dtype': {}, 'Shape': {}, 'error_balance': {}, 'max_abs_diff': {}, 'max_abs_idx': {}, 'max_rel_diff': {}, 'max_rel_idx': {}, 'eb_thd': {}, 'error_thd': {}, 'Status': {}, 'Message': {}} + mock_data_save = {self.saver.COLUMN_API_NAME: {}, + self.saver.COLUMN_FORWARD_SUCCESS: {}, + self.saver.COLUMN_BACKWARD_SUCCESS: {}, + "Message": {}} + mock_data_detail = {'Npu Name': {}, 'Bench Dtype': {}, 'NPU Dtype': {}, 'Shape': {}, 'error_balance': {}, + 'max_abs_diff': {}, 'max_abs_idx': {}, 'max_rel_diff': {}, 'max_rel_idx': {}, 'eb_thd': {}, + 'error_thd': {}, 'Status': {}, 'Message': {}} self.assertEqual(pd.read_csv(self.save_path).to_dict(), mock_data_save) self.assertEqual(pd.read_csv(self.detail_save_path).to_dict(), mock_data_detail) - + def test_write_summary_csv(self): mock_test_result = Mock() mock_test_result.api_name = "api_name" @@ -86,8 +87,8 @@ class TestSaver(unittest.TestCase): mock_test_result.is_bwd_success = "NOSKIP" mock_test_result.fwd_compare_alg_results = "result" self.saver.write_summary_csv(mock_test_result) - mock_data_save = {"api_name":{}, "SKIP":{}, "NOSKIP":{}, "result":{}} - self.assertTrue(pd.read_csv(self.save_path).to_dict()==mock_data_save) + mock_data_save = {"api_name": {}, "SKIP": {}, "NOSKIP": {}, "result": {}} + self.assertTrue(pd.read_csv(self.save_path).to_dict() == mock_data_save) def test_write_detail_csv(self): mock_test_result = Mock() @@ -97,5 +98,4 @@ class TestSaver(unittest.TestCase): self.saver.write_detail_csv(mock_test_result) mock_data_detail = {'api_name.forward.output.0': {0: 'api_name.backward.output.0'}, 'f': {0: 'b'}} - self.assertTrue(pd.read_csv(self.detail_save_path).to_dict()==mock_data_detail) - + self.assertTrue(pd.read_csv(self.detail_save_path).to_dict() == mock_data_detail) -- Gitee From 9320747e2436c4c27b941130da0684e4c2792ae5 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 26 Aug 2024 16:00:41 +0800 Subject: [PATCH 489/791] =?UTF-8?q?compare=E5=91=BD=E4=BB=A4=E8=A1=8C?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=A2=9E=E5=8A=A0=E7=89=88=E6=9C=AC=E6=8F=90?= =?UTF-8?q?=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/README.md | 2 +- debug/accuracy_tools/msprobe/mindspore/doc/compare.md | 2 +- .../msprobe/pytorch/doc/{ptdbg_ascend_compare.md => compare.md} | 2 ++ .../accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) rename debug/accuracy_tools/msprobe/pytorch/doc/{ptdbg_ascend_compare.md => compare.md} (99%) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index f5e457efa..04a306f24 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -166,7 +166,7 @@ Required-by: 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位训练场景下的精度问题。 - PyTorch场景：详见[PyTorch_精度比对工具](./pytorch/doc/ptdbg_ascend_compare.md)。 + PyTorch场景：详见[PyTorch_精度比对工具](./pytorch/doc/compare.md)。 MindSpore场景：详见[MindSpore_精度比对工具](./mindspore/doc/compare.md)。 diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md index 01b4da70b..968568b86 100644 --- a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md +++ b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md @@ -31,7 +31,7 @@ msprobe精度比对工具主要通过对同一个模型，在两个不同的Mind | -c或--compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | -4. 查看比对结果，请详见PyTorch目录下的《[精度比对工具](../../pytorch/doc/ptdbg_ascend_compare.md)》的“比对结果分析”章节。 +4. 查看比对结果，请详见PyTorch目录下的《[精度比对工具](../../pytorch/doc/compare.md)》的“比对结果分析”章节。 ### 比对文件 diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md b/debug/accuracy_tools/msprobe/pytorch/doc/compare.md similarity index 99% rename from debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md rename to debug/accuracy_tools/msprobe/pytorch/doc/compare.md index ffff1f44f..6dfc7df80 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/compare.md @@ -24,6 +24,8 @@ 请先参见《[精度数据采集](./dump.md)》完成CPU或GPU与NPU的精度数据dump。 +版本提示：命令行方式从msprobe 1.0.2版本开始支持。 + ### 操作步骤 1. 创建比对文件，文件内容及示例请参见“**比对文件**”。 diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md index 019451454..83095e259 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md @@ -64,5 +64,5 @@ msprobe的精度比对工具，用来进行PyTorch整网API粒度的数据dump 3. 比对结果分析。 - 详见《[CPU或GPU与NPU精度数据比对](./ptdbg_ascend_compare.md)》。 + 详见《[CPU或GPU与NPU精度数据比对](./compare.md)》。 -- Gitee From 009d5af7cc044f4c187a30f646ff0f6bb42a9110 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 26 Aug 2024 16:31:15 +0800 Subject: [PATCH 490/791] =?UTF-8?q?compare=E5=91=BD=E4=BB=A4=E8=A1=8C?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E5=A2=9E=E5=8A=A0=E7=89=88=E6=9C=AC=E6=8F=90?= =?UTF-8?q?=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/pytorch/doc/compare.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/compare.md b/debug/accuracy_tools/msprobe/pytorch/doc/compare.md index 6dfc7df80..346149207 100644 --- a/debug/accuracy_tools/msprobe/pytorch/doc/compare.md +++ b/debug/accuracy_tools/msprobe/pytorch/doc/compare.md @@ -24,7 +24,7 @@ 请先参见《[精度数据采集](./dump.md)》完成CPU或GPU与NPU的精度数据dump。 -版本提示：命令行方式从msprobe 1.0.2版本开始支持。 +**版本提示**：命令行方式从msprobe 1.0.2版本开始支持。 ### 操作步骤 -- Gitee From db94e60acb873bf43da76d2f317dfce9c428a572 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 26 Aug 2024 16:38:22 +0800 Subject: [PATCH 491/791] fix bug --- .../api_accuracy_checker/compare/api_precision_compare.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index c01356409..624e8ba38 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -328,7 +328,8 @@ def analyse_csv(npu_data, gpu_data, config): compare_column.compare_result = CompareConst.SKIP compare_column.compare_message = err_message write_detail_csv(compare_column.to_column_value(), config.details_csv_path) - write_csv([[full_api_name_with_direction_status, "skip", "skip", err_message]], config.result_csv_path) + write_csv([[full_api_name_with_direction_status, CompareConst.SKIP, CompareConst.SKIP, err_message]], + config.result_csv_path) continue if row_gpu.empty: logger.warning(f'This API : {full_api_name_with_direction_status} does not exist in the GPU data.') @@ -351,7 +352,7 @@ def analyse_csv(npu_data, gpu_data, config): new_status = record_thousandth_threshold_result(compare_column, row_npu) elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or \ api_name in binary_standard_api: - new_status = record_binary_consistency_result(api_name, compare_column, row_npu) + new_status = record_binary_consistency_result(api_name, compare_column, row_npu) elif api_name in absolute_standard_api: new_status = record_absolute_threshold_result(compare_column, row_npu) elif api_name in ulp_standard_api and \ -- Gitee From f55d4bbf166da10f5f3804b798acd5fd57379582 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Mon, 26 Aug 2024 16:35:28 +0800 Subject: [PATCH 492/791] =?UTF-8?q?mstt=20advisor=20wheel=E5=8C=85?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 17 +++++++++-------- profiler/version.txt | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/profiler/README.md b/profiler/README.md index 411fe2a43..008cb9bdd 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -105,14 +105,15 @@ Successfully installed msprof-analyze-{version} 请通过下表链接下载profiler工具whl包。 - | profiler版本 | 发布日期 | 下载链接 | 校验码 | - | ------------ | ---------- | ------------------------------------------------------------ | ------------------------------------------------------------ | - | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | - | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | - | 1.1.2 | 2024-07-12 | [msprof_analyze-1.1.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.2/msprof_analyze-1.1.2-py3-none-any.whl) | af62125b1f9348bf491364e03af712fc6d0282ccee3fb07458bc9bbef82dacc6 | - | 1.1.1 | 2024-06-20 | [msprof_analyze-1.1.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.1/msprof_analyze-1.1.1-py3-none-any.whl) | 76aad967a3823151421153d368d4d2f8e5cfbcb356033575e0b8ec5acea8e5e4 | - | 1.1.0 | 2024-05-28 | [msprof_analyze-1.1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.0/msprof_analyze-1.1.0-py3-none-any.whl) | b339f70e7d1e45e81f289332ca64990a744d0e7ce6fdd84a8d82e814fa400698 | - | 1.0 | 2024-05-10 | [msprof_analyze-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.0/msprof_analyze-1.0-py3-none-any.whl) | 95b2f41c8c8e8afe4887b738c8cababcb4f412e1874483b6adae4a025fcbb7d4 | + | profiler版本 | 发布日期 | 下载链接 | 校验码 | + |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ------------------------------------------------------------ | + | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | + | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | + | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | + | 1.1.2 | 2024-07-12 | [msprof_analyze-1.1.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.2/msprof_analyze-1.1.2-py3-none-any.whl) | af62125b1f9348bf491364e03af712fc6d0282ccee3fb07458bc9bbef82dacc6 | + | 1.1.1 | 2024-06-20 | [msprof_analyze-1.1.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.1/msprof_analyze-1.1.1-py3-none-any.whl) | 76aad967a3823151421153d368d4d2f8e5cfbcb356033575e0b8ec5acea8e5e4 | + | 1.1.0 | 2024-05-28 | [msprof_analyze-1.1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.0/msprof_analyze-1.1.0-py3-none-any.whl) | b339f70e7d1e45e81f289332ca64990a744d0e7ce6fdd84a8d82e814fa400698 | + | 1.0 | 2024-05-10 | [msprof_analyze-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.0/msprof_analyze-1.0-py3-none-any.whl) | 95b2f41c8c8e8afe4887b738c8cababcb4f412e1874483b6adae4a025fcbb7d4 | diff --git a/profiler/version.txt b/profiler/version.txt index cb174d58a..d2d61a7e8 100644 --- a/profiler/version.txt +++ b/profiler/version.txt @@ -1 +1 @@ -1.2.1 \ No newline at end of file +1.2.2 \ No newline at end of file -- Gitee From 345e096e67a423245875f0913dfcd6759e176efd Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 26 Aug 2024 17:13:50 +0800 Subject: [PATCH 493/791] fix --- .../msprobe/pytorch/api_accuracy_checker/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 834a63d23..193721d23 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -182,7 +182,7 @@ def extract_basic_api_segments(api_full_name): api_type, api_name, _ = api_parts elif api_parts_length == Const.FOUR_SEGMENT: api_type, prefix, api_name, _ = api_parts - api_name = prefix + Const.SEP + api_name + api_name = Const.SEP.join([prefix, api_name]) else: api_name = None return api_type, api_name @@ -211,7 +211,7 @@ def extract_detailed_api_segments(full_api_name_with_direction_status): elif api_parts_length == Const.SEVEN_SEGMENT: api_type, prefix, api_name, api_order, direction_status, _, _ = api_parts full_api_name = Const.SEP.join([api_type, prefix, api_name, api_order]) - api_name = prefix + Const.SEP + api_name + api_name = Const.SEP.join([prefix, api_name]) else: full_api_name = None return api_name, full_api_name, direction_status -- Gitee From 377fb2184ae023f0013c4a21a5973933118d1b07 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 17:14:16 +0800 Subject: [PATCH 494/791] renew_manual --- README.md | 109 ++++------- debug/accuracy_tools/README.md | 21 --- debug/accuracy_tools/msprobe/README.md | 252 ++++++------------------- 3 files changed, 99 insertions(+), 283 deletions(-) delete mode 100644 debug/accuracy_tools/README.md diff --git a/README.md b/README.md index 601deec66..6a9b221a1 100644 --- a/README.md +++ b/README.md @@ -1,107 +1,80 @@ -# 变更通知 +# 🚨 重要通知 +**1. Ascend Training Tools 更名为 MindStudio Training Tools (mstt)。** +**2. 本代码仓 URL 变更为 [https://gitee.com/ascend/mstt](https://gitee.com/ascend/mstt)，原 URL 仍然可用（2024.07.04 ）。** -原Ascend Training Tools工具更名为MindStudio Training Tools，MindStudio训练工具链。变更计划如下： +**3. 不再维护：[api_accuracy_checker](./debug/accuracy_tools/api_accuracy_checker/) （2024.09.30下线）和[ ptdbg_ascend](./debug/accuracy_tools/ptdbg_ascend/) +（2024.09.30下线）** -1. 2024.06.25本代码仓名称变更为mstt。 -2. 2024.07.04 URL变更为[https://gitee.com/ascend/mstt](https://gitee.com/ascend/mstt)，原始URL仍然可用，但建议使用新URL。 +**相关目录 mstt/debug/accuracy_tools/api_accuracy_checker 和 mstt/debug/accuracy_tools/ptdbg_ascend 将于 2024.09.30 删除。新版本的预检和 ptdbg 已经合到 mstt/debug/accuracy_tools/msprobe 目录下。** -# MindStudio Training Tools +--- -MindStudio Training Tools，MindStudio训练工具链。针对训练&大模型场景，提供端到端命令行&可视化调试调优工具，帮助用户快速提高模型开发效率。 +# 🧰 MindStudio Training Tools -## 模型训练迁移全流程 -![输入图片说明](debug/resources/model_training_migration_process.png) +![Build Status](https://img.shields.io/badge/build-passing-brightgreen) +![Commit Activity](https://img.shields.io/badge/commit%20activity-high-red) +![License: Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue) -## 使用说明 - -### [分析迁移工具](https://gitee.com/ascend/mstt/wikis/工具介绍/分析迁移工具/分析迁移工具介绍) +## [分析迁移工具](https://gitee.com/ascend/mstt/wikis/工具介绍/分析迁移工具/分析迁移工具介绍) 1. [脚本分析工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E5%88%86%E6%9E%90%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - 脚本分析工具提供分析脚本，帮助用户在执行迁移操作前，分析基于GPU平台的PyTorch训练脚本中算子、三方库套件、亲和API分析以及动态shape的支持情况。 + 脚本分析工具可以帮助用户在执行迁移操作前，分析基于 GPU 平台的 PyTorch 训练脚本中算子、三方库套件、API 亲和性以及动态 shape 的支持情况。 2. [（推荐）自动迁移工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E8%87%AA%E5%8A%A8%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - 自动迁移只需在训练脚本中导入库代码即可完成模型脚本迁移，使用方式较简单，且修改内容最少。 + 自动迁移工具只需在训练脚本中导入库代码即可完成模型脚本的迁移，使用方式简单，且修改内容少。 3. [脚本迁移工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E8%84%9A%E6%9C%AC%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - 脚本迁移工具提供后端命令行用于将GPU上训练的PyTorch脚本迁移至NPU上，得到新的训练脚本用于训练。 + 脚本迁移工具通过后端命令行，将 GPU 上训练的 PyTorch 脚本迁移至 NPU 上，得到新的训练脚本用于训练。 4. [训推一体权重转换工具](https://gitee.com/Ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E8%AE%AD%E6%8E%A8%E4%B8%80%E4%BD%93%E6%9D%83%E9%87%8D%E8%BD%AC%E6%8D%A2%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - 训推一体权重转换工具，支持在GPU和NPU上训练好的模型转成加速推理支持的格式。 - -### [精度工具](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools) - -1. [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe)。 - - MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 - -2. [api_accuracy_checker（Ascend模型精度预检工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/api_accuracy_checker) - - 2024.09.30下线 - - 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 + 训推一体权重转换工具，支持在 GPU 和 NPU 上训练好的模型转成加速推理支持的格式。 -3. [ptdbg_ascend（PyTorch精度工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend) +## [精度工具](./debug/accuracy_tools/) - 2024.09.30下线 +[MindStudio Probe（msprobe，MindStudio 精度调试工具）](./debug/accuracy_tools/msprobe)。 - 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位PyTorch训练场景下的精度问题。 +## [性能工具](./profiler) -### [性能工具](https://gitee.com/ascend/mstt/tree/master/profiler) +1. [compare_tools（性能比对工具）](./profiler/compare_tools) -1. [compare_tools（性能比对工具）](https://gitee.com/ascend/mstt/tree/master/profiler/compare_tools) + 提供 NPU 与 GPU 性能拆解功能以及算子、通信、内存性能的比对功能。 - 提供NPU与GPU性能拆解功能以及算子、通信、内存性能的比对功能。 +2. [cluster_analyse（集群分析工具）](./profiler/cluster_analyse) -2. [cluster_analyse（集群分析工具）](https://gitee.com/ascend/mstt/tree/master/profiler/cluster_analyse) + 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合 MindStudio Insight 的集群分析功能使用。 - 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合MindStudio Insight的集群分析功能使用。 +3. [advisor](./profiler/advisor) -3. [advisor](https://gitee.com/ascend/mstt/blob/master/profiler/advisor) + 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyThon 场景性能数据进行分析，并输出性能调优建议。 - 将Ascend PyTorch Profiler或者msprof采集的PyThon场景性能数据进行分析，并输出性能调优建议。 +## [Tensorboard](./plugins/tensorboard-plugins/tb_plugin) +Tensorboard 支持 NPU 性能数据可视化插件 PyTorch Profiler TensorBoard NPU Plugin。 -### [Tensorboard](https://gitee.com/ascend/mstt/tree/master/plugins/tensorboard-plugins/tb_plugin) - -Tensorboard支持NPU性能数据可视化插件PyTorch Profiler TensorBoard NPU Plugin。 - -支持将Ascend平台采集、解析的Pytorch Profiling数据可视化呈现，也兼容GPU数据采集、解析可视化。 +支持将 Ascend 平台采集、解析的 PyTorch Profiling 数据可视化呈现，也兼容 GPU 数据采集、解析可视化。 ## 分支维护策略 -MindStudio Training Tools工具版本分支的维护阶段如下： - -| **状态** | **时间** | **说明** | -| ------------------- | -------- | ------------------------------------------------ | -| 计划 | 1~3 个月 | 计划特性 | -| 开发 | 3个月 | 开发特性 | -| 维护 | 6~12个月 | 合入所有已解决的问题并发布版本 | -| 无维护 | 0~3 个月 | 合入所有已解决的问题，无专职维护人员，无版本发布 | -| 生命周期终止（EOL） | N/A | 分支不再接受任何修改 | - -## 现有分支的维护状态 - -MindStudio Training Tools分支版本号命名规则如下： - -mstt仓每年发布4个版本，每个版本都将对应一个分支；以v6.0为例，其将对应v6.0.RC1、v6.0.RC2、v6.0.RC3以及v6.0.0四个版本，在仓库中将存在与之对应的分支。 - -| **分支** | **状态** | **发布日期** | **后续状态** | **EOL日期** | -| ------------- | -------- | ------------ | ------------------------ | ----------- | -| **v6.0.0** | 维护 | 2023/12/12 | 预计2024/12/12起无维护 | | +1. MindStudio Training Tools 工具版本分支的维护阶段如下： -## 参与贡献 + | **状态** | **时间** | **说明** | + | ------------------- | -------- | ------------------------------------------------ | + | 计划 | 1—3 个月 | 计划特性 | + | 开发 | 3个月 | 开发特性 | + | 维护 | 6—12个月 | 合入所有已解决的问题并发布版本 | + | 无维护 | 0—3 个月 | 合入所有已解决的问题，无专职维护人员，无版本发布 | + | 生命周期终止（EOL） | N/A | 分支不再接受任何修改 | -1. Fork 本仓库 -2. 新建 xxx 分支 -3. 提交代码 -4. 新建 Pull Request +2. MindStudio Training Tools 分支版本号命名规则如下： -## 版本过渡提示 + mstt 仓每年发布 4 个版本，每个版本都将对应一个分支；以 v6.0 为例，其将对应 v6.0.RC1、v6.0.RC2、v6.0.RC3 以及 v6.0.0 四个版本，在仓库中将存在与之对应的分支。 -当前版本预检和ptdbg维护到2024/09/30，准备于2024/09/30下线，相关目录mstt/debug/accuracy_tools/api_accuracy_checker和mstt/debug/accuracy_tools/ptdbg_ascend将于2024/09/30删除。新版本的预检和ptdbg已经合到mstt/debug/accuracy_tools/atat目录下。 + | **分支** | **状态** | **发布日期** | **后续状态** | **EOL日期** | + | ------------- | -------- | ------------ | ------------------------ | ----------- | + | **v6.0.0** | 维护 | 2023.12.12 | 预计 2024.12.12 起无维护 | | diff --git a/debug/accuracy_tools/README.md b/debug/accuracy_tools/README.md deleted file mode 100644 index e28c1d34a..000000000 --- a/debug/accuracy_tools/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# 精度工具 - -MindStudio Training Tools工具针对模型训练精度问题设计推出了一系列精度工具，包括模型精度预检工具和PyTorch精度工具的精度比对功能、溢出检测功能、通信精度检测等功能。这些工具有各自侧重的场景，用于辅助用户定位模型精度问题。 - -### 子功能介绍 - -NPU上训练的网络存在精度问题，精度指标（loss或者具体的评价指标）与标杆相差较多。对于该场景的问题，可以使用**MindStudio Probe工具**的**精度预检工具**和**精度比对工具**进行定位。 - -| 工具名称 | 说明 | -| ------------------------------------------------------------ | ------------------------------------------------------------ | -| [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe) | MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 | -| [api_accuracy_checker（Ascend模型精度预检工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/api_accuracy_checker) | 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 | -| [ptdbg_ascend（PyTorch精度工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend) | 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位PyTorch训练场景下的精度问题。 | - -### 场景介绍 - -**精度预检工具**会对全网每一个API根据其实际训练中的shape、dtype和数值范围生成随机的输入，对比它与标杆的输出差异，并指出输出差异过大不符合精度标准的API。该工具检查单API精度问题准确率超过80%，对比一般dump比对方法减少落盘数据量99%以上。 - -**精度比对功能**可以对NPU整网API数据进行与CPU或GPU标杆数据的精度比对，从而检测精度问题。具体来说，dump统计量、分段dump、模块化dump，通讯算子dump等功能可以用较轻的数据量实现不同侧重的精度比对，从而定位精度问题。 - -**溢出检测功能**是在判断训练网络可能存在溢出现象时，例如某个step的loss突然变成inf nan，或者混精场景下loss_scale不断减小，可以通过ptdbg_ascend的精度检测工具检测API的溢出情况。 \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index f5e457efa..83340fc6a 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -1,238 +1,102 @@ -# MindStudio精度调试工具 +# 📖 msprobe 使用手册 -MindStudio精度调试工具（MindStudio Probe），简称msprobe，是MindStudio Training Tools工具链下精度调试部分的工具包。主要包括精度预检和精度比对等子工具，当前适配场景包括PyTorch和MindSpore。 +![version](https://img.shields.io/badge/version-1.0.3-blueviolet) +![python](https://img.shields.io/badge/python-3.8|3.9|3.10-blue) +![platform](https://img.shields.io/badge/platform-Linux-yellow) -## 工具安装 +[**msprobe**](./) 是 MindStudio Training Tools 工具链下精度调试部分的工具包。主要包括精度预检、溢出检测和精度比对等功能，目前适配 [PyTorch](https://pytorch.org/) 和 [MindSpore](https://www.mindspore.cn/) 框架。这些子工具侧重不同的训练场景，可以定位模型训练中的精度问题。 -精度工具合一软件包名称：`mindstudio_probe-{version}-py3-none-any.whl` +为方便使用，本工具提供了一个统一、简易的程序接口，**PrecisionDebugger**，通过以下示例模板和 **config.json** 可轻松使用各种功能。 -### pip安装 - ```shell - pip install mindstudio-probe - ``` -使用`pip install mindstudio-probe==版本号`可安装指定版本的包。 +```python +from msprobe.pytorch import PrecisionDebugger +# from msprobe.mindspore import PrecisionDebugger -pip命令会自动安装最新的包及其配套依赖。 - -提示如下信息则表示安装成功。 - -```bash -Successfully installed mindstudio_probe-{version} +debugger = PrecisionDebugger(config_path='./config.json') +... +debugger.start() # 一般在训练循环开头启动工具 +... # 循环体 +debugger.stop() # 一般在训练循环末尾结束工具 +debugger.step() # 在训练循环的最后需要重置工具，非循环场景不需要 ``` -### 下载whl包安装 -1. 使用pip命令安装依赖： - - 1. 根据实际环境安装torch或mindspore - - 2. 安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、einops、matplotlib、pyOpenSSL、twisted - - - 若环境中已安装部分依赖，不需要重复安装。 - -2. whl包获取。 - - 请通过下表链接下载工具whl包。 - - | 版本 | 发布日期 | 支持PyTorch版本 | 支持MindSpore版本 | 下载链接 | 校验码 | - | ----- | ---------- | ---------------- | ----------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | - | 1.0.3 | 2024-08-23 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.3-py3-none-any.whl) | 7060cc141a5b98ef770cd9220995d299393f32a61938261e632c7e8b5160bef2 | - | 1.0.2 | 2024-08-09 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.2-py3-none-any.whl) | e4a980e5d98c426ce5ce9842520d9bc031d3b3de621c74b3d59414cc6e238e0e | - | 1.0.1 | 2024-07-25 | 2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | - | 1.0 | 2024-07-09 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/1.0/ascend_training_accuracy_tools-1.0-py3-none-any.whl) | 5016dfe886c5d340ec6f60a959673355855f313c91f100680da814efb49f8e81 | - | 0.0.3 | 2024-06-11 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-0.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.3-py3-none-any.whl) | f46d9714704859e2d67861a65bbb3c76b0a250cf6e238b978b5b959ab1fe125a | - | 0.0.2 | 2024-05-23 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-0.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.2-py3-none-any.whl) | 2e35809bde559e9c4d2f16a02ccde779ed9e436bb65fded0b7ebaf6ac2c88d93 | - | 0.0.1 | 2024-03-15 | 2.0/2.1 | - | [ascend_training_accuracy_tools-0.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.1-py3-none-any.whl) | 5801510d4e827e4859bc9a5aca021e4d30c2ea42d60a4c8ad0c2baab1b7782c9 | - -3. whl包校验。 - - 1. 根据以上下载链接下载whl包到Linux安装环境。 +除了在训练脚本中调用接口函数，还可以通过命令行使用 **msprobe** 的其他功能，具体的使用规则和 **config.json** 的配置要求详见以下章节。 - 2. 进入whl包所在目录，执行如下命令。 +## ⚙️ [安装](./docs/01.installation.md) - ```bash - sha256sum {name}.whl - ``` +## 🛠️ config.json [介绍](./docs/02.config_introduction.md) 和 [示例](./docs/03.config_examples.md) - {name}为whl包名称。 +## 🧰 主要功能 - 若回显呈现对应版本whl包一致的**校验码**，则表示下载了正确的ptdbg_ascend精度工具whl安装包。示例如下： +### 1 数据采集 - ```bash - sha256sum ascend_training_accuracy_tools-0.0.1-py3-none-any.whl - 5801510d4e827e4859bc9a5aca021e4d30c2ea42d60a4c8ad0c2baab1b7782c9 *ascend_training_accuracy_tools-0.0.1-py3-none-any.whl - ``` +msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API 执行精度数据 dump 操作，对应 config.json 中的 task 为 statistics 或 tensor。 -4. 执行如下命令进行安装。 +[PyTorch 场景下的数据采集](./docs/05.data_dump_PyTorch.md) - ```bash - pip3 install ./mindstudio_probe-{version}-py3-none-any.whl - ``` - - 若为覆盖安装，请在命令行末尾增加“--force-reinstall”参数强制安装，例如： - - ```bash - pip3 install ./mindstudio_probe-{version}-py3-none-any.whl --force-reinstall - ``` - - 提示如下信息则表示安装成功。 - - ```bash - Successfully installed mindstudio_probe-{version} - ``` - -### 从源码安装 -1. 克隆或者下载项目源代码 - - ```shell - git clone https://gitee.com/ascend/mstt.git - cd debug/accuracy_tools - ``` - -2. 安装setuptools和wheel - - ```shell - pip install setuptools wheel - ``` - -3. 安装msprobe - - ```shell - python setup.py bdist_wheel - cd dist - pip install mindstudio_probe*.whl - ``` - 提示出现如下信息则表示源码安装成功。 - ```shell - Successfully installed ... mindstudio_probe-{version} ... - ``` - -### 查看msprobe工具信息 - -执行如下命令查看msprobe工具信息。 - -```bash -pip show mindstudio-probe -``` - -输出结果如下示例： - -```bash -Name: mindstudio-probe -Version: 1.0 -Summary: This is a pytorch precision comparison tools -Home-page: -Author: -Author-email: -License: -Location: /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages -Requires: numpy, openpyxl, pandas, pyyaml, rich, tqdm, wheel -Required-by: -``` +[MindSpore 场景下的数据采集](./docs/06.data_dump_MindSpore.md) -关键字段含义： +### 2 精度预检 -- Name：工具名称。 -- Version：工具版本号。 -- Summary：工具概述。 -- Location：工具安装路径。 -- Requires：工具依赖。 +精度预检旨在昇腾 NPU 上扫描训练模型中的所有 API 进行 API 复现，给出精度情况的诊断和分析。对应 config.json 中的 task 为 run_ut。 -## 工具使用 +PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)和[在线预检](./docs/08.accuracy_checker_online_PyTorch.md) -安装msprobe工具后，可以按照如下思路选择合适的子工具进行精度调试： +MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.md) -1. 判断框架场景。 +### 3 精度比对 - 当前支持PyTorch和MindSpore场景。 +该功能进行 PyTorch 整网 API 粒度的数据 dump、精度比对，进而定位训练场景下的精度问题。 -2. 执行数据采集。 +[PyTorch 场景的精度比对](./docs/11.accuracy_compare_PyTorch.md) - 工具通过在训练脚本中添加PrecisionDebugger接口的方式对API执行精度数据dump操作。 +[MindSpore 场景的精度比对](./docs/12.accuracy_compare_MindSpore.md) - PyTorch场景：详见[PyTorch_精度数据采集](./pytorch/doc/dump.md)。 +### 4 溢出检测与解析 - MindSpore场景：详见[MindSpore_精度数据采集](./mindspore/doc/dump.md)。 +溢出检测与解析是在执行精度数据 dump 时，判断是否存在输入正常但输出存在溢出的 API，从而判断是否为正常溢出。对应 config.json 中的 overflow_check。 -3. 执行精度预检。 +[PyTorch 场景的溢出检测与解析](./docs/13.overflow_check_PyTorch.md)（更新中） - 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 +[MindSpore 场景的溢出检测与解析](./docs/14.overflow_check_MindSpore.md)（更新中） - PyTorch场景：详见[PyTorch_精度预检工具](./pytorch/doc/api_accuracy_checker.md)。 +### 5 数据解析 - MindSpore场景：暂不支持。 +该功能用于比对前后两次 NPU ACL 层级 dump 数据的一致性。 -4. 执行精度比对。 +[PyTorch 场景的数据解析](./docs/15.data_parse_PyTorch.md) - 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位训练场景下的精度问题。 +### 6 无标杆比对 - PyTorch场景：详见[PyTorch_精度比对工具](./pytorch/doc/ptdbg_ascend_compare.md)。 +[PyTorch 场景的无标杆比对](./docs/17.free_benchmarking_PyTorch.md)（更新中） - MindSpore场景：详见[MindSpore_精度比对工具](./mindspore/doc/compare.md)。 +[MindSpore 场景的无标杆比对](./docs/18.overflow_check_MindSpore.md)（更新中） -5. 执行溢出解析。 +### 7 梯度状态监测 - 溢出解析是在执行精度数据dump时，配置了溢出检测dump，那么对于输入正常但输出存在溢出的API，可以判断是否为正常溢出。 +本功能用于采集梯度数据并进行梯度相似度比对，可以精准定位出现问题的 step。 - PyTorch场景：详见[PyTorch_溢出解析工具](./pytorch/doc/run_overflow_check.md)。 +[兼容 PyTorch 和 MindSpore 框架的梯度监测](./docs/19.grad_probe.md) - MindSpore场景：暂不支持。 +## 🌟 新版本特性 -6. 执行数据解析。 +【精度预检】 +- 落盘数据小。 +- 支持随机生成模式和真实数据模式。 +- 单 API 测试，排除整网中的累计误差问题。 - 用于比对前后两次NPU ACL层级dump数据的一致性。 +【梯度检测】 +- 使用便捷，无需在训练流程里插入代码。 +- 可以精准定位问题出现的 step。 - PyTorch场景：详见[PyTorch_数据解析工具](./pytorch/doc/parse_tool.md)。 +## 📑 补充材料 - MindSpore场景：暂不支持。 +[msprobe 精度工具标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md) -6. 执行梯度采集和比对。 - - 用于采集梯度数据并进行梯度相似度比对。可以精准定位问题出现的step。 - - 详见[梯度状态监测工具](./doc/grad_probe/grad_probe.md)。 - - - -上述流程中的工具均为msprobe工具的子工具，使用相同的命令行，格式如下： - -精度预检工具 - -```bash -msprobe -f run_ut [-h] -``` - -```bash -msprobe -f multi_run_ut [-h] -``` - -```bash -msprobe -f api_precision_compare [-h] -``` - -精度比对工具 - -```bash -msprobe -f compare [-h] -``` - -溢出解析工具 - -```bash -msprobe -f run_overflow_check [-h] -``` - -数据解析工具 - -```bash -msprobe -f parse [-h] -``` +[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md) -| 参数 | 说明 | -| ---- | ------------------------------------------------------------ | -| -f | 框架，请按所使用框架配置，当前支持配置为：pytorch、mindspore。 | -| -h | 帮助信息。 | +## ❓ FAQ -## 贡献 +[FAQ for PyTorch](./docs/FAQ_PyTorch.md) -push代码前，请务必保证已经完成了基础功能测试和网络测试。 +FAQ for MindSpore -- Gitee From 031ae790e814bf05f6307864363052f68990118c Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 17:16:41 +0800 Subject: [PATCH 495/791] renew_manual --- .../msprobe/{config => }/config.json | 0 debug/accuracy_tools/msprobe/config/README.md | 539 ------------------ .../msprobe/doc/grad_probe/grad_probe.md | 207 ------- .../msprobe/docs/01.installation.md | 91 +++ .../msprobe/docs/02.config_introduction.md | 165 ++++++ .../msprobe/docs/03.config_examples.md | 247 ++++++++ .../msprobe/docs/04.acl_config_examples.md | 76 +++ .../msprobe/docs/05.data_dump_Pytorch.md | 199 +++++++ .../msprobe/docs/06.data_dump_MindSpore.md | 157 +++++ .../docs/07.accuracy_checker_PyTorch.md | 278 +++++++++ .../msprobe/docs/img/BLOOM-7B_1.png | Bin 0 -> 3534 bytes .../msprobe/docs/img/BLOOM-7B_2.png | Bin 0 -> 5251 bytes .../msprobe/docs/img/BLOOM-7B_3.png | Bin 0 -> 5095 bytes .../msprobe/docs/img/BLOOM-7B_4.png | Bin 0 -> 4727 bytes .../msprobe/docs/img/GPT-3_1.png | Bin 0 -> 7898 bytes .../msprobe/docs/img/GPT-3_2.png | Bin 0 -> 6716 bytes .../msprobe/docs/img/GPT-3_3.png | Bin 0 -> 8732 bytes .../msprobe/docs/img/GPT-3_4.png | Bin 0 -> 6609 bytes .../msprobe/docs/img/GPT-3_5.png | Bin 0 -> 4676 bytes .../msprobe/docs/img/GPT-3_6.png | Bin 0 -> 7658 bytes .../msprobe/docs/img/GPT-3_7.png | Bin 0 -> 8139 bytes .../msprobe/docs/img/GPT-3_8.png | Bin 0 -> 6084 bytes .../msprobe/docs/img/YOLOV5S_1.png | Bin 0 -> 7019 bytes .../msprobe/docs/img/YOLOV5S_2.png | Bin 0 -> 5796 bytes .../docs/img/accuracy_checking_details.png | Bin 0 -> 39062 bytes .../docs/img/accuracy_checking_result.png | Bin 0 -> 6022 bytes .../img/api_precision_compare_details.png | Bin 0 -> 25476 bytes .../docs/img/api_precision_compare_result.png | Bin 0 -> 4538 bytes .../msprobe/docs/img/auto_analyze_log.png | Bin 0 -> 65144 bytes .../msprobe/docs/img/compare_result_pkl.png | Bin 0 -> 36133 bytes .../docs/img/compare_result_pkl_md5.png.png | Bin 0 -> 20464 bytes .../msprobe/docs/img/cpu_info.png | Bin 0 -> 33797 bytes .../{config => docs}/img/free_benchmark.png | Bin .../img/grad_probe_image-1.png} | Bin .../img/grad_probe_image-2.png} | Bin .../img/grad_probe_image-3.png} | Bin .../img/grad_probe_image-4.png} | Bin .../img/grad_probe_image.png} | Bin .../msprobe/docs/img/module_compare.png | Bin 0 -> 56655 bytes 39 files changed, 1213 insertions(+), 746 deletions(-) rename debug/accuracy_tools/msprobe/{config => }/config.json (100%) delete mode 100644 debug/accuracy_tools/msprobe/config/README.md delete mode 100644 debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md create mode 100644 debug/accuracy_tools/msprobe/docs/01.installation.md create mode 100644 debug/accuracy_tools/msprobe/docs/02.config_introduction.md create mode 100644 debug/accuracy_tools/msprobe/docs/03.config_examples.md create mode 100644 debug/accuracy_tools/msprobe/docs/04.acl_config_examples.md create mode 100644 debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md create mode 100644 debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md create mode 100644 debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md create mode 100644 debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_1.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_2.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_3.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_4.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_1.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_2.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_3.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_4.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_5.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_6.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_7.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_8.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/YOLOV5S_1.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/YOLOV5S_2.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/accuracy_checking_details.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/accuracy_checking_result.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/api_precision_compare_details.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/api_precision_compare_result.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/auto_analyze_log.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/compare_result_pkl.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/compare_result_pkl_md5.png.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/cpu_info.png rename debug/accuracy_tools/msprobe/{config => docs}/img/free_benchmark.png (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image-1.png => docs/img/grad_probe_image-1.png} (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image-2.png => docs/img/grad_probe_image-2.png} (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image-3.png => docs/img/grad_probe_image-3.png} (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image-4.png => docs/img/grad_probe_image-4.png} (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image.png => docs/img/grad_probe_image.png} (100%) create mode 100644 debug/accuracy_tools/msprobe/docs/img/module_compare.png diff --git a/debug/accuracy_tools/msprobe/config/config.json b/debug/accuracy_tools/msprobe/config.json similarity index 100% rename from debug/accuracy_tools/msprobe/config/config.json rename to debug/accuracy_tools/msprobe/config.json diff --git a/debug/accuracy_tools/msprobe/config/README.md b/debug/accuracy_tools/msprobe/config/README.md deleted file mode 100644 index 7720ddbdf..000000000 --- a/debug/accuracy_tools/msprobe/config/README.md +++ /dev/null @@ -1,539 +0,0 @@ -# 配置文件说明 - -当前配置文件主要为PrecisionDebugger接口执行dump或无标杆比对操作时调用的配置，当PrecisionDebugger接口未指定该配置文件时，使用该文件的默认配置。配置文件详见[config.json](./config.json)。 - -当在环境上安装msprobe工具后，config.json文件位置可通过如下方式查找： - -查找msprobe工具安装路径。 - -``` -pip show mindstudio-probe -``` - -输出结果如下示例： - -``` -Name: mindstudio-probe -Version: 1.0 -Summary: This is a pytorch precision comparison tools -Home-page: -Author: -Author-email: -License: -Location: /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages -Requires: numpy, openpyxl, pandas, pyyaml, rich, tqdm, wheel -Required-by: -``` - -Location字段为msprobe工具的安装路径，那么config.json文件位置为/home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages/msprobe/config - -## 参数说明 - -### **通用配置参数** - -| 参数名 | 说明 | 是否必选 | -| ----------------- | ------------------------------------------------------------ | -------- | -| task | dump的任务类型，str类型。可取值：
"free_benchmark"（无标杆比对）。
"statistics"（仅dump张量的统计信息，默认值）。
"tensor"（dump张量的统计信息和完整张量数据，MindSpore静态图场景仅dump完整张量数据）。
"overflow_check"（溢出检测）。
"run_ut"（精度预检配置）。
配置示例："task": "tensor"。
根据task参数取值的不同，可以配置不同场景参数，详见：“**task配置为free_benchmark**”，“**task配置为statistics**”，“**task配置为tensor**”，“**task配置为overflow_check**”，“**task配置为run_ut**”。 | 否 | -| dump_path | 设置dump数据目录路径，str类型。配置示例："dump_path": "./dump_path"。MindSpore静态图场景仅支持绝对路径。 | 是 | -| rank | 指定对某张卡上的数据进行dump，list[int]类型，默认未配置（表示dump所有卡的数据），应配置为大于等于0的整数，且须配置实际可用的Rank ID。配置示例："rank": [1]。
对于PyTorch场景，Rank ID从0开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的Rank ID，则dump数据为空，比如当前环境Rank ID为0到7，实际训练运行0到3卡，此时若配置Rank ID为4或不存在的10等其他值，此时dump数据为空。
对于MindSpore场景，所有节点的Rank ID均从0开始计数，最大取值为每个节点可用卡总数-1，config.json配置一次rank参数对所有节点同时生效。 | 否 | -| step | 指定dump某个step的数据，list[int]类型。默认未配置，表示dump所有step数据。dump特定step时，须指定为训练脚本中存在的step。step为list格式，可配置逐个step，例如："step": [0,1,2]。 | 否 | -| level | dump级别，str类型，根据不同级别dump不同数据。可取值：
"L0"（dump module模块级精度数据，仅PyTorch场景支持，使用背景详见“**模块级精度数据dump说明**”）。
"L1"（dump API级精度数据，默认值，仅MindSpore动态图和PyTorch场景支持）。
"L2"（dump kernel级精度数据，仅MindSpore静态图和PyTorch场景支持，且PyTorch场景须配置acl_config参数）。
"mix"（dump module模块级和API级精度数据，即"L0"+"L1"，仅PyTorch场景支持）。
配置示例："level": "L1"。MindSpore动态图场景仅支持"L1"。 | 否 | -| acl_config | kernel dump的配置文件，str类型。level取"L2"时，该参数必选；level为其他值时，该参数不选。参数示例：acl_config='./acl_config.json'。acl_config.json配置文件详细介绍请参见“**acl_config.json配置文件说明**”。 | 否 | -| seed | 随机种子数，int类型，默认值为：1234，仅PyTorch场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见“**固定随机数范围**”。配置示例："seed": 1234。 | 否 | -| is_deterministic | 确定性计算模式，bool类型，仅PyTorch场景支持。可取值true（开启）或false（关闭），默认关闭。配置示例："is_deterministic": true。
即使在相同的硬件和输入下，API多次执行的结果也可能不同，开启确定性计算是为了保证在相同的硬件和输入下，API多次执行的结果相同。
确定性计算会导致API执行性能降低，建议在发现模型多次执行结果不同的情况下开启。
rnn类算子、ReduceSum、ReduceMean等算子可能与确定性计算存在冲突，若开启确定性计算后多次执行的结果不相同，则考虑存在这些算子。 | 否 | -| enable_dataloader | 自动控制开关，bool类型，仅PyTorch场景支持。可取值true（开启）或false（关闭），默认为false。配置为True后自动识别step参数指定的迭代，并在该迭代执行完成后退出训练，此时start、stop和step函数可不配置，开启该开关要求训练脚本是通过torch.utils.data.dataloader方式加载数据。仅支持PyTorch单卡训练使用，分布式训练场景下存在数据dump不全问题，**下个版本即将废弃该功能**。 | 否 | - -### task配置为free_benchmark - -仅PyTorch场景与MindSpore动态图场景支持，且"level"需为"L1"。 - -task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通过对当前模型API的输入添加扰动因子，二次执行，将得到的输出与未添加扰动因子前的输出进行比对，从而**得出该模型中可能因迁移等变化导致精度降低的API**。 - -无标杆比对优势在于省去了从CPU/GPU环境获取标杆数据的步骤，也省去了在NPU环境执行dump的操作，降低了精度比对的操作难度。 - -建议配置白名单（配置scope或list）控制少量API进行无标杆比对，一次对过多API执行无标杆比对可能导致显存溢出或性能膨胀。 - -| 参数名 | 说明 | 是否必选 | -| ------------ | ------------------------------------------------------------ | -------- | -| scope | 自定义检测API列表（仅PyTorch场景支持），list[str]类型，默认值为空列表，当list也为空列表时，表示检测所有API。需要在[]内配置具体API名（在工具dump结果中查看）。配置示例："scope": ["Torch.matmul.0", "Tensor.pow.4"]。与list参数不能同时配置。 | 否 | -| list | 自定义检测API类型或API名称，list[str]类型，默认值为空列表，表示检测所有API（PyTorch场景下还需scope也为空列表）。包含如下配置方法：
PyTorch场景下，指定某一类API，对某一类的API进行无标杆比对。配置示例："list": ["relu"]。
MindSpore场景下，指定API名称，对列表中的API进行检测。配置示例："list": ["mindspore.mint.div", "mindspore.ops.bmm", "mindspore.Tensor.\_\_add\_\_"]。
与scope参数不能同时配置。 | 否 | -| fuzz_device | 标杆设备，str类型。可取值：
"npu"：无标杆，通过添加扰动因子进行比对，默认值。
"cpu"：以CPU为标杆，pert_mode须配置为"to_cpu"（仅PyTorch场景支持）。
配置示例："fuzz_device": "npu"。 | 否 | -| pert_mode | 无标杆扰动因子，str类型。可取值：
"improve_precision"：对输入做升精度，默认值。
"add_noise"：对输入增加噪声。
"no_change"：不加扰动直接二次执行。
"bit_noise"：输入的末位比特翻转。
"change_value"：输入的张量首尾值调换（仅PyTorch场景支持）。
"to_cpu"：在CPU等价执行（仅PyTorch场景支持）。
配置示例："pert_mode": "improve_precision"。 | 否 | -| handler_type | 处理类型，可取值："check"（进行无标杆比对检查，默认值）、"fix"（将扰动后的API输出结果覆盖原始API输出结果，尝试将Loss曲线恢复正常，该模式下不支持预热if_preheat与反向过程）。配置示例："handler_type": "fix"。 | 否 | -| fuzz_level | 无标杆数据dump级别，即选择比对结果文件应输出的表头属性，当前仅支持取值为："L1"（默认值）。输出结果详见“**无标杆比对数据存盘格式**”。 | 否 | -| fuzz_stage | 前反向，选择对API前向或反向进行无标杆比对，可取值："forward"（前向，默认值）、"backward"（反向，仅PyTorch场景支持）。配置示例："fuzz_stage": "forward"。 | 否 | -| if_preheat | 预热功能（仅PyTorch场景支持），开启功能后工具可以根据每次迭代的输出调整精度算法的阈值，从而更准确找出存在精度问题的API，bool类型。可取值true（开启）或false（关闭），默认关闭。配置示例："if_preheat": "true"。"handler_type": "fix"不支持预热。 | 否 | -| preheat_step | 开启预热的迭代数量（仅PyTorch场景支持），int类型，默认值为15。须配置"if_preheat": "true"。 | 否 | -| max_sample | 每个算子预热的采样次数的最大阈值（仅PyTorch场景支持），int类型，默认值为20。须配置"if_preheat": "true"。 | 否 | - -#### 无标杆比对数据存盘格式 - -无标杆比对在dump_path目录下输出结果文件free_benchmark.csv，如下示例： - -![free_benchmark](./img/free_benchmark.png) - -| 字段 | 说明 | -| ------------ | ------------------------------------------------------------ | -| rank | Rank ID，int类型。 | -| pert_mode | 扰动因子的类型，string类型。 | -| stage | 前向或反向，string类型。 | -| step | 迭代数，int类型。 | -| api_name | API名称，string类型。 | -| max_rel | 输出对比最大相对误差，float类型。 | -| dtype | 输入的dtype，string类型。 | -| shape | 输入的shape，tuple类型。 | -| output_index | 如果输出为列表或元组，其中一个元素检测不一致，则会有该元素的index，否则为空，int类型。 | - -### task配置为statistics - -| 参数名 | 说明 | 是否必选 | -| ------------ | ------------------------------------------------------------ | -------- | -| scope | PyTorch和MindSpore动态图场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。MindSpore动态图场景当前仅支持配置为API名。 | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch和MindSpore动态图场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch和MindSpore动态图场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
MindSpore静态图场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | -| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore静态图场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | -| summary_mode | 控制dump文件输出的模式，str类型，仅PyTorch和MindSpore动态图场景支持，可取值md5（dump输出包含md5值以及API统计信息的dump.json文件，用于验证数据的完整性）、statistics（dump仅输出包含API统计信息的dump.json文件，默认值）。配置示例："summary_mode": "md5"。 | 否 | - -### task配置为tensor - -MindSpore静态图场景仅dump完整张量数据。 - -| 参数名 | 说明 | 是否必选 | -| -------------- | ------------------------------------------------------------ | -------- | -| scope | PyTorch和MindSpore动态图场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。 | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch和MindSpore动态图场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch和MindSpore动态图场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
PyTorch和MindSpore动态图场景配置kernel_api，dump前向和反向API的kernel_api级别数据，其中dump反向API时需要配置**backward_input**参数。前向API配置示例："list": ["Tensor.permute.1.forward"]；反API配置示例："list": ["Tensor.permute.1.forward"], "backward.input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
MindSpore静态图场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | -| backward_input | 该输入文件为首次运行训练dump得到反向API输入的dump文件，str类型，仅PyTorch场景支持，默认未配置。例如若需要dump Functional.conv2d.1 API的反向过程的输入输出，则需要在dump目录下查找命名包含Functional.conv2d.1、backward和input字段的dump文件。配置示例："backward_input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"] | 否 | -| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore静态图场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | -| file_format | MindSpore静态图场景真实tensor数据的保存格式，str类型，可取值"bin"（dump的tensor文件为二进制格式，"level": "L1"时不支持）、"npy"（dump的tensor文件后缀为.npy，默认值）。 | 否 | -| online_run_ut | 在线预检模式开关，bool类型，可取值true（开启）、false（关闭），默认未配置，表示关闭。配置为true表示开启在线预检。 | 否 | -| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。仅在online_run_ut字段配置为true时生效，未配置该参数后host和port不生效。 | 否 | -| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的局域网IP地址。仅在online_run_ut字段配置为true时生效，局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | -| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的端口号。仅在online_run_ut字段配置为true时生效，局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | - -说明： online_run_ut、nfs_path、host、port参数仅在线预检场景NPU环境生效，详细说明请参见[《在线精度预检》](../pytorch/doc/api_accuracy_checker_online.md)。 - -### task配置为overflow_check - -MindSpore静态图场景的jit_level为O0/O1时，不支持该功能，须配置jit_level为O2。请参见[mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig)配置jit_config。 - -| 参数名 | 说明 | 是否必选 | -| ------------- | ------------------------------------------------------------ | -------- | -| overflow_nums | 控制溢出次数，int类型，仅MindSpore动态图和PyTorch场景支持，表示第N次溢出时，停止训练，过程中检测到溢出API对应kernel数据均dump。配置示例："overflow_nums": 3。默认为1，即检测到1次溢出，训练停止，配置为-1时，表示持续检测溢出直到训练结束。 | 否 | -| check_mode | MindSpore静态图场景kernel级别的溢出检测，str类型，可取值"aicore"（开启AI Core的溢出检测）、"atomic"（开启Atomic的溢出检测）、"all"（开启AI Core和Atomic的溢出检测，默认值）。配置示例"check_mode": "aicore"。 | 否 | - -### task配置为run_ut - -仅PyTorch场景支持。 - -| 参数名称 | 说明 | 是否必选 | -|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------| -| white_list | API dump白名单，仅对指定的API进行dump。配置示例："white_list": ["conv1d", "conv2d"]。默认未配置白名单，即dump全量API数据。 | 否 | -| black_list | API dump黑名单，被指定的API不进行dump。配置示例："black_list": ["conv1d", "conv2d"]。默认未配置黑名单，即dump全量API数据。 | 否 | -| error_data_path | 配置保存精度未达标的API输入输出数据路径，默认为当前路径。配置示例"error_data_path": "./"。 | 否 | -| is_online | 在线预检模式开关，bool类型，可取值true（开启）、false（关闭），默认关闭。 | 否 | -| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效，仅在is_online字段配置为true时生效。 | 否 | -| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机地址127.0.0.1或本机局域网IP。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。仅在is_online字段配置为true时生效。 | 否 | -| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机可用端口。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。仅在is_online字段配置为true时生效。 | 否 | -| rank_list | 指定在线预检的Rank ID，默认值为[0]，list[int]类型，应配置为大于等于0的整数，且须根据实际卡的Rank ID配置，若所配置的值大于实际训练所运行的卡的Rank ID，则在线预检输出数据为空。GPU和NPU须配置一致。仅在is_online字段配置为true时生效。 | 否 | - -说明： -
（1）white_list和black_list同时配置时，二者配置的API名单若无交集，则白名单生效，若API名单存在交集，则白名单排除的部分以及交集的API不进行dump。 -
（2）is_online、nfs_path、host、port、rank_list等字段仅在线预检场景GPU机器生效，详细说明见[《在线精度预检》](../pytorch/doc/api_accuracy_checker_online.md) - -## 配置示例 - -以下示例包含当前支持的所有场景可配置的完整参数。 - -### PyTorch场景task配置为free_benchmark - -```json -{ - "task": "free_benchmark", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "free_benchmark": { - "scope": [], - "list": ["conv2d"], - "fuzz_device": "npu", - "pert_mode": "improve_precision", - "handler_type": "check", - "fuzz_level": "L1", - "fuzz_stage": "forward", - "if_preheat": false, - "preheat_step": 15, - "max_sample": 20 - } -} -``` - -### PyTorch场景task配置为statistics - -```json -{ - "task": "statistics", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "statistics": { - "scope": [], - "list": [], - "data_mode": ["all"], - "summary_mode": "statistics" - } -} -``` - -### PyTorch场景task配置为tensor - -```json -{ - "task": "tensor", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "tensor": { - "scope": [], - "list":[], - "data_mode": ["all"], - "backward_input": "" - } -} -``` - -### PyTorch场景task配置为overflow_check - -```json -{ - "task": "overflow_check", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "overflow_check": { - "overflow_nums": 1 - } -} -``` - -### PyTorch场景task配置为run_ut - -```json -{ - "task": "run_ut", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "run_ut": { - "white_list": [], - "black_list": [], - "error_data_path": "./" - } -} -``` - -### MindSpore静态图场景task配置为statistics - -```json -{ - "task": "statistics", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L2", - - "statistics": { - "list": [], - "data_mode": ["all"], - "summary_mode": "statistics" - } -} -``` - -### MindSpore静态图场景task配置为tensor - -```json -{ - "task": "tensor", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L2", - - "tensor": { - "list":[], - "data_mode": ["all"], - "backward_input": "" - } -} -``` - -### MindSpore静态图场景task配置为overflow_check - -```json -{ - "task": "overflow_check", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L2", - - "overflow_check": { - "check_mode": "all" - } -} -``` - -### MindSpore动态图场景task配置为free_benchmark - -```json -{ - "task": "free_benchmark", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - - "free_benchmark": { - "list": ["mindspore.ops.add"], - "fuzz_device": "npu", - "pert_mode": "improve_precision", - "handler_type": "check", - "fuzz_level": "L1", - "fuzz_stage": "forward" - } -} -``` - -### MindSpore动态图场景task配置为statistics - -```json -{ - "task": "statistics", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - - "statistics": { - "scope": [], - "list": [], - "data_mode": ["all"], - "summary_mode": "statistics" - } -} -``` - -### MindSpore动态图场景task配置为tensor - -```json -{ - "task": "tensor", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - - "tensor": { - "scope": [], - "list":[], - "data_mode": ["all"], - } -} -``` - -### MindSpore动态图场景task配置为overflow_check - -```json -{ - "task": "overflow_check", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - - "overflow_check": { - "overflow_nums": 1 - } -} -``` - -## 附录 - -### 模块级精度数据dump说明 - -仅PyTorch场景支持。 - -大模型场景下，通常不是简单的利用自动迁移能力实现GPU到NPU的训练脚本迁移，而是会对NPU网络进行一系列针对性的适配，因此，常常会造成迁移后的NPU模型存在部分子结构不能与GPU原始模型完全对应。模型结构不一致导致API调用类型及数量不一致，若直接按照API粒度进行精度数据dump和比对，则无法完全比对所有的API。 - -本节介绍的功能是对模型中的大粒度模块进行数据dump，使其比对时，对于无法以API粒度比对的模块可以直接以模块粒度进行比对。 - -模块指的是继承自nn.Module类模块，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump数据时以模块为粒度进行dump。 - -### acl_config.json配置文件说明 - -#### [config.json](./config.json)配置示例 - -当PyTorch场景level取"L2"时，须配置acl_config参数，并指定acl_config.json文件（用于指定L2 kernel级dump的配置），此时config.json文件配置示例如下： - -- 前向kernel dump配置示例： - - "scope"配置为前向API名称，仅支持配置一个API。 - - ```json - { - "task": "tensor", - "dump_path": "/home/data_dump", - "level": "L2", - "rank": [0], - "step": [0], - "is_deterministic": false, - "tensor": { - "scope": ["Tensor.__mul__.10.forward"], - "list":[], - "data_mode": ["all"], - "backward_input": [""], - "file_format": "npy" - }, - "acl_config": "acl_config.json" - } - ``` - -- 反向kernel dump配置示例： - - 执行反向kernel dump前需要先使用工具dump该API的反向输入，保存pt文件，在"backward_input"参数中传入该pt文件路径。 - - "scope"配置为反向API名称，仅支持配置一个API。 - - ```json - { - "task": "tensor", - "dump_path": "/home/data_dump", - "level": "L2", - "rank": [0], - "step": [0], - "is_deterministic": false, - "tensor": { - "scope": ["Tensor.__mul__.10.backward"], - "list":[], - "data_mode": ["all"], - "backward_input": ["Tensor.__mul__.10.backward.input.0.pt"], - "file_format": "npy" - }, - "acl_config": "acl_config.json" - } - ``` - -#### acl_config.json配置示例 - -acl_config.json文件须自行创建，配置示例如下： - -``` -{ - "dump": - { - "dump_list":[], - "dump_path":"./dump/output", - "dump_mode":"all", - "dump_op_switch":"on" - } -} -``` - -**acl_config.json参数说明** - -| 字段名 | 说明 | -| -------------- | ------------------------------------------------------------ | -| dump_list | 待dump数据的API模型。为空，无需配置。 | -| dump_path | dump数据文件存储到运行环境的目录，主要配置的是kernel级数据的存放路径。支持配置绝对路径或相对路径。dump_path须为已存在目录。 | -| dump_mode | dump数据模式，配置如下： output：dump API的输出数据。默认值。 input：dump API的输入数据。 all：dump API的输入、输出数据。 | -| dump_op_switch | 单API模型dump数据开关，配置如下：
off：关闭单API模型dump，默认值。
on：开启单API模型dump。 | - -**dump目录说明** - -配置acl_config.json后，采集的kernel级数据会在{dump_path}/{time}/{deviceid}/{model_id}目录下生成，例如“/home/HwHiAiUser/output/20200808163566/0/0” - -``` -├── 20230131172437 -│ └── 1 -│ ├── 0 -│ │ ├── Add.Add.45.0.1675157077183551 -│ │ ├── Cast.trans_Cast_0.31.0.1675157077159449 -│ │ ├── Cast.trans_Cast_5.43.0.1675157077180129 -│ │ ├── MatMul.MatMul.39.0.1675157077172961 -│ │ ├── Mul.Mul.29.0.1675157077155731 -│ │ ├── NPUAllocFloatStatus.NPUAllocFloatStatus.24.0.1675157077145262 -│ │ ├── TransData.trans_TransData_1.33.0.1675157077162791 -│ │ └── TransData.trans_TransData_4.41.0.1675157077176648 -│ ├── 1701737061 -│ │ └── Cast.trans_Cast_2.35.0.1675157077166214 -│ ├── 25 -│ │ └── NPUClearFloatStatus.NPUClearFloatStatus.26.0.1675157077150342 -│ └── 68 -│ └── TransData.trans_TransData_3.37.0.1675157077169473 -``` - -### 固定随机数范围 - -仅PyTorch场景支持。 - -seed_all函数可固定随机数的范围如下表。 - -| API | 固定随机数 | -| ---------------------------------------- | --------------------------- | -| os.environ['PYTHONHASHSEED'] = str(seed) | 禁止Python中的hash随机化 | -| random.seed(seed) | 设置random随机生成器的种子 | -| np.random.seed(seed) | 设置numpy中随机生成器的种子 | -| torch.manual_seed(seed) | 设置当前CPU的随机种子 | -| torch.cuda.manual_seed(seed) | 设置当前GPU的随机种子 | -| torch.cuda.manual_seed_all(seed) | 设置所有GPU的随机种子 | -| torch_npu.npu.manual_seed(seed) | 设置当前NPU的随机种子 | -| torch_npu.npu.manual_seed_all(seed) | 设置所有NPU的随机种子 | -| torch.backends.cudnn.enable=False | 关闭cuDNN | -| torch.backends.cudnn.benchmark=False | cuDNN确定性地选择算法 | -| torch.backends.cudnn.deterministic=True | cuDNN仅使用确定性的卷积算法 | - -需要保证CPU或GPU以及NPU的模型输入完全一致，dump数据的比对才有意义，seed_all并不能保证模型输入完全一致，如下表所示场景需要保证输入的一致性。 - -| 场景 | 固定方法 | -| --------------- | ------------- | -| 数据集的shuffle | 关闭shuffle。 | -| dropout | 关闭dropout。 | - -关闭shuffle示例： - -```Python -train_loader = torch.utils.data.DataLoader( - train_dataset, - batch_size = batch_size, - shuffle = False, - num_workers = num_workers -) -``` - -关闭dropout： - -在使用from msprobe.pytorch import PrecisionDebugger后，工具会自动将torch.nn.functional.dropout、torch.nn.functional.dropout2d、torch.nn.functional.dropout3d、torch.nn.Dropout、torch.nn.Dropout2d、torch.nn.Dropout3d的接口参数p置为0。 diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md b/debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md deleted file mode 100644 index fcbd2f123..000000000 --- a/debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md +++ /dev/null @@ -1,207 +0,0 @@ -# Ascend模型梯度状态监测工具 - -梯度状态监测工具提供了两种能力： - -- 将模型权重的梯度数据导出。这种功能可以将模型权重的梯度值以统计量的形式采集出来，用以分析问题。 -- 将两份梯度数据进行相似度对比。在有标杆问题中，可以确认训练过程中精度问题出现的step，以及抓取反向过程中的问题。 - -工具支持PyTorch版本：2.0/2.1/2.2；支持MindSpore版本：r2.3。 - -## 工具特性 - -- 使用便捷，无需在训练流程里插入代码 -- 可以精准定位问题出现的step - -## 使用方式 - -### 梯度数据导出 - -1. 创建配置文件config.json，样例如下： - - ```json - { - "task": "grad_probe", - "dump_path": "./dump_path", - "rank": [], - "step": [], - "grad_probe": { - "grad_level": "L1", - "param_list": [], - "bounds": [-1, 0, 1] - } - } - ``` - > step指的是优化器被调用的次数（并非模型跑的step，某些step，例如loss为nan时，不会调用优化器） - - **参数说明** - - | 参数 | 说明 | 输入类型 | 是否必选 | - |--------------------------------|-----------------------------------|-----------------|----------| - | task | 填为"grad_probe"。 | str | 是 | - | grad_level | 输出级别。决定导出数据的详细程度，级别越大导出数据越详细。可取值：L0, L1, L2|str | 是 | - | param_list | 权重名称列表，表示需要监控的权重。列表为空就表示监控所有权重。 | List[str] | 是 | - | rank | rank id列表，在多卡场景下，表示需要导出梯度数据的进程的rank id。列表为空就表示导出所有rank的数据。（MindSpore静态图模式下，当前暂不支持指定rank功能） | List[int] | 是 | - | step | step列表，表示需要导出数据的step列表。列表为空就表示导出所有step的数据。（MindSpore静态图模式下，当前暂不支持指定step功能） | List[int] | 是 | - | bounds | 区间列表，用来划分区间以统计数值的分布。需要保证由数据小到大排列。可以使用默认值[-1, 0, 1] | List[float] | 是 | - | dump_path | 输出目录。如果不存在就会创建一个新目录。 | str | 是 | - - **不同级别的level的导出数据** - - - | 级别 | 特征数据表头 | 是否有方向数据 | - | ---- | ------------------------------------------------------------ | -------------- | - | L0 | ("param_name", "MD5", "max", "min", "norm", "shape") | 否 | - | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | - | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | - - intervals就是根据值分布bounds划分出的区间。 - MindSpore静态图模式下，L0级别中暂不支持"MD5" - - **方向数据解释** - - 因为模型的参数往往非常大，所以存储真实数据是不可接受的，这里折衷一下，只存储梯度数据的正负号（一个布尔值），也就是方向。 - - **bounds和值分布解释** - - + 值分布：梯度数据落在各个区间的元素个数占总元素个数的比例。 - + bounds：一个列表，用来划分出区间以统计值分布。例如传入bounds = [-10, 0, 10]，此时有一个 grad_value: Tensor = [9.3 , 5.4, -1.0, -12.3]，依据 bounds 划分出 (-inf, -10]、(-10, 0]、(0, 10]、(10, inf) 四个区间，然后统计grad_value里的数据落在每个区间内的个数，得到 1、1、2、0。如下图所示： - ![Alt text](img/image-1.png) - -2. 插入代码。示例代码如下： - -- PyTorch框架：模型构造完成后，传入config.json的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将`模型`作为参数传入。 -```python -from msprobe.pytorch import PrecisionDebugger -debugger = PrecisionDebugger("config_json_path") -debugger.monitor(model) -``` -- MindSpore框架：优化器构造完成后，传入config.json的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将`优化器`作为参数传入。 -```python -from msprobe.mindspore import PrecisionDebugger -debugger = PrecisionDebugger("config_json_path") -debugger.monitor(optimizer) -``` - -3. 结束监控（MindSpore静态图模式下需要） - - 在训练结束之后，调用stop接口 - -```python -gm.stop() -``` - -### 输出结果 -**输出目录结构**（以level配置L2为例） - -```bash -{dump_path} - ├── rank{rank_id} - │ ├── grad_summary_{step}.csv - │ ├── step{step} - │ │ ├── {param_name}.npy -``` -+ {timestamp}：梯度工具导出数据的时候会在output_path下生成一个时间戳目录，然后在这个时间戳目录下输出结果。 -+ rank_{rank_id}：在分布式场景下，会记录卡的rank_id。非分布式场景下，如果是CPU则记录进程号，如果是CPU或GPU则记录卡号 -+ grad_summary_{step}.csv：会分step记录每一步的梯度数据统计值。 -+ step_{step}：这个目录下会存放该step的梯度的方向数据。 -+ {param_name}.pt(npy)：模型参数的梯度方向数据，PyTorch保存的是pt文件，MindSpore是npy文件。 - -**grad_summary_{step}.csv** - -样例如下： - -![Alt text](img/image.png) - -| 字段 | 含义 | -| --------------------- | ------------------------------------------------------------| -| Param_name | 模型参数名称。 | -| MD5 | 梯度数据的MD5值。 | -| (-inf, -0.01]...[0.01, inf) | 梯度值落在区间内的元素个数占总元素的比例。 | -| =0 | 梯度为0的元素个数占总元素的比例。 | -| Max | 最大值。 | -| Min | 最小值。 | -| Norm | L2norm值。 | -| Shape | 形状。 | - -### 梯度相似度比对 - -会根据所导出的权重，分step比对梯度相似度，输出每个权重的梯度相似度和总的梯度相似度。单个权重的梯度相似度为两份方向数据的重合度，总的梯度相似度为每个权重的梯度相似度按元素个数加权。 - -#### 前提条件 - -- 相同配置下，以Level为L1或L2分别采集npu和gpu环境下的梯度数据。 -- 将两份梯度数据传到同一环境下。 - -#### 使用方式 - - -新建如下Python脚本，传入npu和gpu的dump_path以及输出目录，比对结果输出目录不存在的话会新建： - -```python -from msprobe import * -GradComparator.compare_distributed("配置文件里写的dump_path", - "配置文件里写的dump_path", - "比对结果输出目录") -``` - - -### 比对结果 - -**输出目录结构** - -如下为多卡比对结果，单卡则没有rank_{rank_id}这一级目录。 - -```bash -比对结果输出目录 - ├── rank{rank_id} - │ ├── similarities.csv - │ └── similarities_picture - │ ├── {param_name}.png - │ └── summary_similarities.png -``` - -**问题界定** - -原则：对于任意权重，第0步的梯度相似度低于0.97，或者某一步的梯度相似度下降超过0.03，认为这一步存在精度问题。例子如下： - -- 第0步相似度低于0.97 - -![Alt text](img/image-3.png) - -- 第3步相似度下降超过0.03 - -![Alt text](img/image-4.png) - -- 正常情况 - -![Alt text](img/image-2.png) - -这个原则是一个经验性的指标，并不是严格的标注，还需要结合实际情况具体分析。 - -## 公开接口 - -**接口说明** - -```python -PrecisionDebugger.monitor(module) -``` - -| 参数 | 说明 | 是否必选 | -| ----- | -------------------- | -------- | -| module |Pytorch框架下传入模型，必须是torch.nn.Module；MindSpore框架下传入优化器。 | 是 | - - -**接口说明** - -```python -GradComparator.compare_distributed(dump_path1, dump_path2, output_path) -``` - -| 参数 | 说明 | 是否必选 | -| ----- | -------------------- | -------- | -| dump_path1 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path。 | 是 | -| dump_path2 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path，与dump_path1可以互换。 | 是 | -| output_path |输出结果目录，不存在会新建。 | 是 | - - -# FAQ diff --git a/debug/accuracy_tools/msprobe/docs/01.installation.md b/debug/accuracy_tools/msprobe/docs/01.installation.md new file mode 100644 index 000000000..7463ee2a5 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/01.installation.md @@ -0,0 +1,91 @@ +# 安装 + +推荐使用 [miniconda](https://docs.anaconda.com/miniconda/) 管理环境依赖。 + +```bash +conda create -n msprobe python=3.8 +conda activate msprobe +``` + +## 1. 从 PyPI 安装 +```shell +pip install mindstudio-probe[==版本号] +``` + +## 2. 下载 whl 包安装 + +|版本|发布日期|支持 PyTorch 版本|支持 MindSpore 版本|下载链接|校验码| +|:--:|:--:|:--:|:--:|:--:|:--:| +| 1.0.3 | 2024.08.23 | 1.11/2.0/2.1/2.2 | 2.3.1/2.4 | [mindstudio_probe-1.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.3-py3-none-any.whl) | 7060cc141a5b98ef770cd9220995d299393f32a61938261e632c7e8b5160bef2 | +| 1.0.2 | 2024.08.09 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.2-py3-none-any.whl) | e4a980e5d98c426ce5ce9842520d9bc031d3b3de621c74b3d59414cc6e238e0e | +| 1.0.1 | 2024.07.25 | 2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | +| 0.0.3 | 2024.06.11 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/1.0/ascend_training_accuracy_tools-1.0-py3-none-any.whl) | 5016dfe886c5d340ec6f60a959673355855f313c91f100680da814efb49f8e81 | +| 0.0.2 | 2024.05.23 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-0.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.2-py3-none-any.whl) | 2e35809bde559e9c4d2f16a02ccde779ed9e436bb65fded0b7ebaf6ac2c88d93 | +| 0.0.1 | 2024.03.15 | 2.0/2.1 | - | [ascend_training_accuracy_tools-0.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.1-py3-none-any.whl) | 5801510d4e827e4859bc9a5aca021e4d30c2ea42d60a4c8ad0c2baab1b7782c9 | + +```bash +sha256sum {name}.whl # 验证whl包，若校验码一致，则whl包在下载中没有受损 +``` + +```bash +pip install ./mindstudio_probe-{version}-py3-none-any.whl # 安装whl包 +``` + +若覆盖安装，请在命令行末尾添加`--force-reinstall`参数。 + +## 3. 从源码安装 + +```shell +git clone https://gitee.com/ascend/mstt.git +cd debug/accuracy_tools + +pip install setuptools wheel + +python setup.py bdist_wheel +pip install ./dist/mindstudio_probe*.whl +``` + +# 查看 msprobe 工具信息 + +```bash +pip show mindstudio-probe +``` + +示例如下： + +```bash +Name: mindstudio-probe +Version: 1.0.* +Summary: Pytorch Ascend Probe Utils +Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe +Author: Ascend Team +Author-email: pmail_mindstudio@huawei.com +License: Apache License 2.0 +Location: /home/***/miniconda3/envs/***/lib/python3.8/site-packages/mindstudio_probe-1.0.0-py3.8.egg +Requires: einops, matplotlib, numpy, openpyxl, pandas, pyOpenSSL, pyyaml, rich, tqdm, twisted, wheel +Required-by: +``` + +# Ascend 生态链接 + +## 1 安装 CANN 包 + +1.1 根据 CPU 架构和 NPU 型号选择 toolkit 和 kernal 包，可以参考 [CANN 软件安装指南](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F700%2Fenvdeployment%2Finstg%2Finstg_0001.html)和[昇腾社区](https://www.hiascend.cn/developer/download/community/result?module=cann)。 + +1.2 运行示例 +```bash +Ascend-cann-toolkit_x.x.x_linux-aarch64.run --full --install-path={cann_path} +Ascend-cann-kernels_x.x.x_linux.run --install --install-path={cann_path} +``` + +1.3 配置环境变量 +```bash +source {cann_path}/ascend-toolkit/set_env.sh +``` +## 2 安装 PyTorch_NPU + +链接：[https://gitee.com/ascend/pytorch](https://gitee.com/ascend/pytorch)。 + +## 3 安装 ModelLink + +链接：[https://gitee.com/ascend/ModelLink](https://gitee.com/ascend/ModelLink)。 diff --git a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md new file mode 100644 index 000000000..4f7c503fb --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md @@ -0,0 +1,165 @@ +# 配置文件介绍 + +- 当调用 **PrecisionDebugger** 接口执行 dump 或其他操作时，需要使用 [config.json](../config.json) 文件；当未指定 config.json 时，将使用默认配置。 +- msprobe 成功安装后，config.json 一般位于如下目录： +``` +/home/***/miniconda3/envs/***/lib/python3.8/site-packages/msprobe/ +``` + +## 1 参数介绍 + +### 1.1 通用配置 + +| 参数名 | 解释 | 是否必选 | +| ----------------- | ---------------- | -------- | +| task | dump 的任务类型，str 类型。可取值，
"statistics"：仅 dump API 统计信息，默认值；
"tensor"：dump API 统计信息和完全复刻整网的 API 运行情况的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持。
"overflow_check"：溢出检测，仅 PyTorch 和 MindSpore 静态图场景支持；
"free_benchmark"：无标杆比对，仅 PyTorch 场景支持；
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)。
**配置示例**："task": "tensor"。 | 否 | +| dump_path | 设置 dump 数据目录路径，str 类型。
**配置示例**："dump_path": "./dump_path"。MindSpore 场景仅支持绝对路径。 | 是 | +| rank | 指定对某张卡上的数据进行 dump，list[int] 类型，默认未配置（表示 dump 所有卡的数据），应配置为 ≥0 的整数，且须配置实际可用的 Rank ID。
对于 PyTorch 场景，Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，此时 dump 数据为空。
对于 MindSpore 场景，所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
**配置示例**："rank": [1]。 | 否 | +| step | 指定 dump 某个 step 的数据，list[int] 类型。默认未配置，表示 dump 所有 step 数据。dump 特定 step 时，须指定为训练脚本中存在的 step。step 为 list 格式，可配置逐个 step。
**配置示例**："step": [0,1,2]。 | 否 | +| level | dump 级别，str 类型，根据不同级别 dump 不同数据。可取值，
"L0"：dump module 模块级精度数据，仅 PyTorch 场景支持，使用背景详见**\*1**；
"L1"：dump API 级精度数据，默认值；
"L2"：dump kernel 级精度数据，须配置 acl_config 参数；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 场景支持。
**配置示例**："level": "L1"。 | 否 | +| acl_config | kernel dump 的配置文件，str 类型。level 取"L2"时，该参数必要；level 为其他值时，该参数不选。
**配置示例**：acl_config="./acl_config.json"。具体配置见[ acl_config 示例](./04.acl_config_examples.md)。 | 否 | +| seed | 随机种子数，int 类型，默认值为：1234，仅 PyTorch 场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见**\*2**。
**配置示例**："seed": 1234。 | 否 | +| is_deterministic | 确定性计算模式，bool 类型，仅 PyTorch 场景支持。可取值 true（开启）或 false（关闭），默认关闭。
即使在相同的硬件和输入下，API 多次执行的结果也可能不同，开启确定性计算是为了保证在相同的硬件和输入下，API 多次执行的结果相同。
确定性计算会导致 API 执行性能降低，建议在发现模型多次执行结果不同的情况下开启。
rnn 类算子、ReduceSum、ReduceMean 等算子可能与确定性计算存在冲突，若开启确定性计算后多次执行的结果不相同，则考虑存在这些算子。
**配置示例**："is_deterministic": true。
| 否 | +| enable_dataloader | 自动控制开关，bool 类型，仅 PyTorch 场景支持。可取值 true（开启）或 false（关闭），默认为 false。配置为 True 后自动识别 step 参数指定的迭代，并在该迭代执行完成后退出训练，此时 start、stop 和 step 函数可不配置，开启该开关要求训练脚本是通过 torch.utils.data.dataloader 方式加载数据。仅支持 PyTorch 单卡训练使用，分布式训练场景下存在数据 dump 不全问题，**下个版本即将废弃该功能**。 | 否 | + +**\***： +1. 模块级精度数据 dump 说明（仅 PyTorch 场景支持） + + 大模型场景下，通常不是简单的利用自动迁移能力实现从 GPU 到 NPU 的训练脚本迁移，而是会对 NPU 网络进行一系列针对性的适配，因此，常常会造成迁移后的 NPU 模型存在部分子结构不能与 GPU 原始模型完全对应。模型结构不一致导致 API 调用类型及数量不一致，若直接按照 API 粒度进行精度数据 dump 和比对，则无法完全比对所有的 API。 + + 本小节介绍的功能是对模型中的大粒度模块进行数据 dump，使其比对时，对于无法以 API 粒度比对的模块可以直接以模块粒度进行比对。 + + 模块指的是继承自 nn.Module 类模块，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump 数据时以模块为粒度进行 dump。 + +2. 固定随机数范围（仅 PyTorch 场景支持） + + seed_all 函数可固定随机数的范围如下表。 + + | API | 固定随机数 | + | ---------------------------------------- | --------------------------- | + | os.environ['PYTHONHASHSEED'] = str(seed) | 禁止 Python 中的 hash 随机化 | + | random.seed(seed) | 设置 random 随机生成器的种子 | + | np.random.seed(seed) | 设置 numpy 中随机生成器的种子 | + | torch.manual_seed(seed) | 设置当前 CPU 的随机种子 | + | torch.cuda.manual_seed(seed) | 设置当前 GPU 的随机种子 | + | torch.cuda.manual_seed_all(seed) | 设置所有 GPU 的随机种子 | + | torch_npu.npu.manual_seed(seed) | 设置当前 NPU 的随机种子 | + | torch_npu.npu.manual_seed_all(seed) | 设置所有 NPU 的随机种子 | + | torch.backends.cudnn.enable=False | 关闭 cuDNN | + | torch.backends.cudnn.benchmark=False | cuDNN 确定性地选择算法 | + | torch.backends.cudnn.deterministic=True | cuDNN 仅使用确定性的卷积算法 | + + 需要保证 CPU 或 GPU 以及 NPU 的模型输入完全一致，dump 数据的比对才有意义，seed_all 并不能保证模型输入完全一致，如下表所示场景需要保证输入的一致性。 + + | 场景 | 固定方法 | + | --------------- | ------------- | + | 数据集的 shuffle | 关闭 shuffle。 | + | dropout | 关闭 dropout。 | + + 关闭 shuffle 示例： + + ```python + train_loader = torch.utils.data.DataLoader( + train_dataset, + batch_size = batch_size, + shuffle = False, + num_workers = num_workers + ) + ``` + + 关闭 dropout： + + 在使用 `from msprobe.pytorch import PrecisionDebugger` 后，工具会自动将 `torch.nn.functional.dropout`、`torch.nn.functional.dropout2d`、`torch.nn.functional.dropout3d`、`torch.nn.Dropout`、`torch.nn.Dropout2d`、`torch.nn.Dropout3d` 的接口参数 p 置为0。 + +### 1.2 task 配置为 statistics + +| 参数名 | 解释 | 是否必选 | +| ------------ | ------------------- | -------- | +| scope | PyTorch 和 MindSpore 动态图场景 dump 范围，list[str] 类型，默认未配置（list 也未配置时表示 dump 所有 API 的数据）。该参数在 [ ] 内配置两个模块名或API名，用于锁定区间，dump 该范围内的数据。
**配置示例**："scope": ["MyModuleOP1", "MyModuleOP2"]。与 level 参数取值相关，level 为 L0 和 mix 级别时，可配置模块名；level 为 L1 级别时，可配置 API 名。MindSpore 动态图场景当前仅支持配置为 API 名。 | 否 | +| list | 自定义 dump 范围，list[str] 类型，默认未配置（scope 也未配置时表示 dump 所有 API 的数据）。包含如下配置方法：
PyTorch 和 MindSpore 动态图场景配置具体的 API 全称，dump 该 API 数据。
**配置示例**："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch 和 MindSpore 动态图场景指定某一类 API，dump 某一类的 API 级别输入输出数据。
**配置示例**："list": ["relu"]。
MindSpore 静态图场景配置 kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合“name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，“name-regex(Default/.+)”可匹配算子名称以“Default/”开头的所有算子）。 | 否 | +| data_mode | dump 数据过滤，str 类型。可取值，
PyTorch 场景支持"all"、"forward"、"backward"、"input"和"output"，表示仅保存 dump 的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的 dump 文件。
MindSpore 场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。
**配置示例**："data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有 dump 的数据。除了 all 参数只能单独配置外，其他参数可以自由组合。 | 否 | +| summary_mode | 控制 dump 文件输出的模式，str 类型，仅 PyTorch 场景支持，可取值，
md5：dump 输出包含 md5 值以及 API 统计信息的 dump.json 文件，用于验证数据的完整性；
statistics：dump 仅输出包含 API 统计信息的 dump.json 文件，默认值。
**配置示例**："summary_mode": "md5"。 | 否 | + +### 1.3 task 配置为 tensor + +| 参数名 | 解释 | 是否必选 | +| -------------- | ---------------------- | -------- | +| scope | 与[ 1.2 task 配置为 statistics ](#12-task-配置为-statistics)中的解释相同。 | 否 | +| list | 与[ 1.2 task 配置为 statistics ](#12-task-配置为-statistics)中的解释相同。另外，
PyTorch 和 MindSpore 动态图场景配置 kernel_api，dump 前向和反向 API 的kernel_api 级别数据，其中 dump 反向 API 时需要配置 **backward_input** 参数。
**前向 API 配置示例**："list": ["Tensor.permute.1.forward"]；
**反向 API 配置示例**："list": ["Tensor.permute.1.forward"], "backward.input": ["./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
| 否 | +| backward_input | 该输入文件为首次运行训练 dump 得到反向 API 输入的 dump 文件，str 类型，仅 PyTorch 场景支持，默认未配置。例如若需要 dump Functional.conv2d.1 API 的反向过程的输入输出，则需要在 dump 目录下查找命名包含 Functional.conv2d.1、backward 和 input 字段的 dump 文件。
**配置示例**："backward_input": ["./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"] | 否 | +| data_mode | 与[ 1.2 task 配置为 statistics ](#12-task-配置为-statistics)中的解释相同 | 否 | +| file_format | MindSpore 场景真实 tensor 数据的保存格式，str 类型，可取值，
"bin"：dump 的 tensor 文件为二进制格式，"level": "L1" 时不支持；
"npy"：dump 的 tensor 文件后缀为 .npy，默认值。 | 否 | +| online_run_ut* | 在线预检模式开关，bool 类型，可取值 true（开启）、false（关闭），默认未配置，表示关闭。配置为 true 表示开启在线预预检。| 否 | +| nfs_path* | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。仅在 online_run_ut 字段配置为 true 时生效，未配置该参数后 host 和 port 不生效。 | 否 | +| host* | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的局域网 IP 地址。仅在 online_run_ut 字段配置为 true 时生效，局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | +| port* | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的端口号。仅在 online_run_ut 字段配置为 true 时生效，局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。| 否 | + +**\***：online_run_ut、nfs_path、host、port 等字段仅在线预检场景 NPU 机器生效。 + +### 1.4 task 配置为 run_ut + +仅 PyTorch 场景支持。 + +| 参数名称 | 解释 | 是否必选 | +| --------------- | ------------------------ | ------------ | +| white_list**\*1** | API dump 白名单，仅对指定的 API 进行 dump。
**配置示例**："white_list": ["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | +| black_list**\*1** | API dump 黑名单，被指定的 API 不进行 dump。
**配置示例**："black_list": ["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | +| error_data_path | 配置保存精度未达标的 API 输入输出数据路径，默认为当前路径。
**配置示例**："error_data_path": "./"。 | 否 | +| is_online**\*2** | 在线预检模式开关，bool 类型，可取值 true（开启）、false（关闭），默认关闭。 | 否 | +| nfs_path**\*2** | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。配置该参数后 host 和 port 不生效，仅在 is_online 字段配置为 true 时生效。 | 否 | +| host**\*2** | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机地址 127.0.0.1 或本机局域网 IP。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。仅在 is_online 字段配置为 true 时生效。 | 否 | +| port**\*2** | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机可用端口。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。仅在 is_online 字段配置为 true 时生效。| 否 | +| rank_list**\*2** | 指定在线预检的 Rank ID，默认值为[0]，list[int] 类型，应配置为大于等于 0 的整数，且须根据实际卡的 Rank ID 配置，若所配置的值大于实际训练所运行的卡的 Rank ID，则在线预检输出数据为空。GPU 和 NPU 须配置一致。仅在 is_online 字段配置为 true 时生效。 | 否 | + +**\***： +1. white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 +2. is_online、nfs_path、host、port、rank_list 等字段仅在线预检场景 GPU 机器生效。 + +### 1.5 task 配置为 overflow_check + +| 参数名 | 解释 | 是否必选 | +| ------------- | ---------------------- | -------- | +| overflow_nums | 控制溢出次数，int 类型，仅 PyTorch 场景支持，表示第 N 次溢出时，停止训练，过程中检测到溢出 API 对应的 kernel 数据均 dump。
**配置示例**："overflow_nums": 3。默认为1，即检测到1次溢出，训练停止，配置为-1时，表示持续检测溢出直到训练结束。 | 否 | +| check_mode | MindSpore 场景 kernel 级别的溢出检测，str 类型，可取值：
"aicore"：开启 AI Core 的溢出检测；
"atomic"：开启 Atomic 的溢出检测；
"all"：开启 AI Core 和 Atomic 的溢出检测，默认值。
**配置示例**："check_mode": "aicore"。 | 否 | + +### 1.6 task 配置为 free_benchmark + +仅 PyTorch 场景与 MindSpore 动态图场景支持，且"level"为"L1"。 + +- task 配置为 free_benchmark 时，开启**无标杆比对**，在 NPU 环境下通过对当前模型 API 的输入添加扰动因子，二次执行，将得到的输出与未添加扰动因子前的输出进行比对，从而**得出该模型中可能存在因迁移等变化导致精度降低的 API**。 + +- 无标杆比对优势在于省去了从 CPU/GPU 环境获取标杆数据的步骤，也省去了在 NPU 环境执行 dump 的操作，降低了精度比对的操作难度。 + +- 建议配置白名单（配置 scope 或 list）控制少量 API 进行无标杆比对，一次对过多 API 执行无标杆比对可能导致显存溢出或性能膨胀。 + +| 参数名 | 解释 | 是否必选 | +| ------------ | -------------- | -------- | +| scope | 自定义检测 API 列表（仅 PyTorch 场景支持），list[str] 类型，默认值为空列表，当 list 也为空列表时，表示检测所有 API。需要在 [ ] 内配置具体 API 名（在 dump 的结果中查看）。
**配置示例**："scope": ["Torch.matmul.0", "Tensor.pow.4"]。
与 list 参数不能同时配置。 | 否 | +| list | 自定义检测 API 类型或 API 名称，list[str] 类型，默认值为空列表，表示检测所有 API（PyTorch 场景下还需 scope 也为空列表）。包含如下配置方法：
PyTorch 场景下，指定某一类 API，对某一类的 API 进行无标杆比对。
**配置示例**："list": ["relu"]。
MindSpore 场景下，指定 API 名称，对列表中的 API 进行检测。
**配置示例**："list": ["mindspore.mint.div", "mindspore.ops.bmm", "mindspore.Tensor.\_\_add\_\_"]。
与 scope 参数不能同时配置。 | 否 | +| fuzz_device | 标杆设备，str 类型。可取值，
"npu"：无标杆，通过添加扰动因子进行比对，默认值；
"cpu"：以 CPU 为标杆，pert_mode 须配置为"to_cpu"（仅 PyTorch 场景支持）。
**配置示例**："fuzz_device": "cpu"。 | 否 | +| pert_mode | 无标杆扰动因子，str 类型。可取值，
"improve_precision"：对输入做升精度，默认值；
"add_noise"：对输入增加噪声；
"no_change"：不加扰动直接二次执行；
"bit_noise"：输入的末位比特翻转；
"change_value"：输入的张量首尾值调换（仅 PyTorch 场景支持）；
"to_cpu"：在 CPU 等价执行（仅 PyTorch 场景支持）。
**配置示例**："pert_mode": "improve_precision"。 | 否 | +| handler_type | 处理类型，可取值，
"check"：进行无标杆比对检查，默认值；
"fix"：将扰动后的 API 输出结果覆盖原始 API 输出结果，尝试将 Loss 曲线恢复正常，该模式下不支持预热 if_preheat。
**配置示例**："handler_type": "fix"。 | 否 | +| fuzz_level | 无标杆数据 dump 级别，即选择比对结果文件应输出的表头属性，当前仅支持取值为："L1"。输出结果详见**\*1**。 | 否 | +| fuzz_stage | 前反向，选择对 API 前向或反向进行无标杆比对，可取值：
"forward"：前向，默认值；
"backward"：反向。
**配置示例**："fuzz_stage": "backward"。 | 否 | +| if_preheat | 预热功能（仅 PyTorch 场景支持），开启功能后工具可以根据每次迭代的输出调整精度算法的阈值，从而更准确地找出存在精度问题的 API，bool 类型。可取值：
true（开启）或 false（关闭），默认关闭。
**配置示例**："if_preheat": "true"。"handler_type": "fix"不支持预热。 | 否 | +| preheat_step | 开启预热的迭代数量（仅 PyTorch 场景支持），int 类型，默认值为 15。须配置 "if_preheat": "true"。 | 否 | +| max_sample | 每个算子预热的采样次数的最大阈值（仅 PyTorch 场景支持），int 类型，默认值为 20。须配置 "if_preheat": "true"。 | 否 | + +**\***： +1. 无标杆比对数据存盘格式 + + 无标杆比对在 dump_path 目录下输出结果文件 free_benchmark.csv，如下示例： + + ![free_benchmark](./img/free_benchmark.png) + + | 字段 | 说明 | + | ------------ | --------------------- | + | rank | Rank ID，int 类型。 | + | pert_mode | 扰动因子的类型，string 类型。 | + | stage | 前向或反向，string 类型。 | + | step | 迭代数，int 类型。 | + | api_name | API 名称，string 类型。 | + | max_rel | 输出对比最大相对误差，float 类型。 | + | dtype | 输入的 dtype，string 类型。 | + | shape | 输入的 shape，tuple 类型。 | + | output_index | 如果输出为列表或元组，其中一个元素检测不一致，则会有该元素的 index，否则为空，int 类型。 | diff --git a/debug/accuracy_tools/msprobe/docs/03.config_examples.md b/debug/accuracy_tools/msprobe/docs/03.config_examples.md new file mode 100644 index 000000000..f51c84054 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/03.config_examples.md @@ -0,0 +1,247 @@ +# 配置示例 + +以下示例包含当前支持的所有场景可配置的完整参数。 + +## 1 PyTorch 场景 + +### 1.1 task 配置为 statistics + +```json +{ + "task": "statistics", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + "seed": 1234, + "is_deterministic": false, + "enable_dataloader": false, + + "statistics": { + "scope": [], + "list": [], + "data_mode": ["all"], + "summary_mode": "statistics" + } +} +``` + +### 1.2 task 配置为 tensor + +```json +{ + "task": "tensor", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + "seed": 1234, + "is_deterministic": false, + "enable_dataloader": false, + + "tensor": { + "scope": [], + "list":[], + "data_mode": ["all"], + "backward_input": "" + } +} +``` + +### 1.3 task 配置为 run_ut + +```json +{ + "task": "run_ut", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + "seed": 1234, + "is_deterministic": false, + "enable_dataloader": false, + + "run_ut": { + "white_list": [], + "black_list": [], + "error_data_path": "./" + } +} +``` + +### 1.4 task 配置为 overflow_check + +```json +{ + "task": "overflow_check", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + "seed": 1234, + "is_deterministic": false, + "enable_dataloader": false, + + "overflow_check": { + "overflow_nums": 1 + } +} +``` + +### 1.5 task 配置为 free_benchmark + +```json +{ + "task": "free_benchmark", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + "seed": 1234, + "is_deterministic": false, + "enable_dataloader": false, + + "free_benchmark": { + "scope": [], + "list": ["conv2d"], + "fuzz_device": "npu", + "pert_mode": "improve_precision", + "handler_type": "check", + "fuzz_level": "L1", + "fuzz_stage": "forward", + "if_preheat": false, + "preheat_step": 15, + "max_sample": 20 + } +} +``` + +## 2 MindSpore 静态图场景 + +### 2.1 task 配置为 statistics + +```json +{ + "task": "statistics", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L2", + + "statistics": { + "list": [], + "data_mode": ["all"], + "summary_mode": "statistics" + } +} +``` + +### 2.2 task 配置为 tensor + +```json +{ + "task": "tensor", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L2", + + "tensor": { + "list":[], + "data_mode": ["all"], + "backward_input": "" + } +} +``` + +### 2.3 task 配置为 overflow_check + +```json +{ + "task": "overflow_check", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L2", + + "overflow_check": { + "check_mode": "all" + } +} +``` + +## 3 MindSpore 动态图场景 + +### 3.1 task 配置为 statistics + +```json +{ + "task": "statistics", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + + "statistics": { + "scope": [], + "list": [], + "data_mode": ["all"], + "summary_mode": "statistics" + } +} +``` + +### 3.2 task 配置为 tensor + +```json +{ + "task": "tensor", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + + "tensor": { + "scope": [], + "list":[], + "data_mode": ["all"], + } +} +``` + +### 3.3 task 配置为 overflow_check + +```json +{ + "task": "overflow_check", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + + "overflow_check": { + "overflow_nums": 1 + } +} +``` + +### 3.4 task 配置为 free_benchmark + +```json +{ + "task": "free_benchmark", + "dump_path": "/home/data_dump", + "rank": [], + "step": [], + "level": "L1", + + "free_benchmark": { + "list": ["mindspore.ops.add"], + "fuzz_device": "npu", + "pert_mode": "improve_precision", + "handler_type": "check", + "fuzz_level": "L1", + "fuzz_stage": "forward" + } +} +``` diff --git a/debug/accuracy_tools/msprobe/docs/04.acl_config_examples.md b/debug/accuracy_tools/msprobe/docs/04.acl_config_examples.md new file mode 100644 index 000000000..4095423bd --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/04.acl_config_examples.md @@ -0,0 +1,76 @@ +# acl_config.json 配置文件说明 + +当 PyTorch 场景 level 取"L2"时，须配置 acl_config 参数，并指定 acl_config.json 文件（用于指定 L2 kernel 级 dump 的配置），此时 **config.json** 文件配置示例如下： + +## 1 前向 kernel dump 配置示例 + +"scope"配置为前向 API 名称，仅支持配置一个 API。 + +```json +{ + "task": "tensor", + "dump_path": "/home/data_dump", + "level": "L2", + "rank": [0], + "step": [0], + "is_deterministic": false, + "tensor": { + "scope": ["Tensor.__mul__.10.forward"], + "list":[], + "data_mode": ["all"], + "backward_input": [""], + "file_format": "npy" + }, + "acl_config": "acl_config.json" +} +``` + +## 2 反向 kernel dump 配置示例 + +执行反向 kernel dump 前需要先使用工具 dump 该 API 的反向输入，保存 pt 文件，"backward_input"参数中传入该 pt 文件路径。 + +"scope"配置为反向API名称，仅支持配置一个API。 + +```json +{ + "task": "tensor", + "dump_path": "/home/data_dump", + "level": "L2", + "rank": [0], + "step": [0], + "is_deterministic": false, + "tensor": { + "scope": ["Tensor.__mul__.10.backward"], + "list":[], + "data_mode": ["all"], + "backward_input": ["Tensor.__mul__.10.backward.input.0.pt"], + "file_format": "npy" + }, + "acl_config": "acl_config.json" +} +``` + +## 3 acl_config.json 配置示例 + +该文件须自行创建，配置示例如下： + +```json +{ + "dump": + { + "dump_list":[], + "dump_path":"./dump/output", + "dump_mode":"all", + "dump_op_switch":"on" + } +} +``` + +**acl_config.json 参数说明** + +| 字段名 | 解释 | +| -------------- | ------------------------------------------------------------ | +| dump_list | 待 dump 数据的 API 模型。为空，无需配置。 | +| dump_path | dump 数据文件存储到运行环境的目录，主要配置的是 kernel 级数据的存放路径。支持配置绝对路径或相对路径。dump_path 须为已存在目录。 | +| dump_mode | dump 数据模式，可取值，
output：dump API 的输出数据，默认值；
input：dump API 的输入数据；
all：dump API 的输入、输出数据。 | +| dump_op_switch | 单 API 模型 dump 数据开关，可取值，
off：关闭单API模型dump，默认值；
on：开启单API模型dump。 | diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md new file mode 100644 index 000000000..3f2b2a88b --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md @@ -0,0 +1,199 @@ +# PyTorch 场景下的精度数据采集 + +msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 + +本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/pytorch/hook_module/support_wrap_ops.yaml 文件内手动修改，如下示例： + +```yaml +functional: # functional为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API + - conv1d + - conv2d + - conv3d +``` + +## 1 接口介绍 + +### 1.1 PrecisionDebugger + +**功能说明**：通过加载 dump 配置文件的方式来确定 dump 操作的详细配置。 + +**原型**： + +```Python +PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, model=None, step=None) +``` + +**\***： +1. config_path：指定 dump 配置文件路径；model：指定具体的 torch.nn.Module，默认未配置，level 配置为"L0"或"mix"时必须配置该参数。其他参数均在 [config.json](../config.json) 文件中可配，详细配置可见 [config.json 介绍](./02.config_introduction.md)。 +2. 此接口的参数均不是必要，且优先级高于 [config.json](../config.json) 文件中的配置，但可配置的参数相比 config.json 较少。 + +### 1.2 start + +**功能说明**：启动精度数据采集。在模型初始化之后的位置添加。需要与 stop 函数一起添加在 for 循环内。 + +**原型**： + +```Python +debugger.start() +``` + +### 1.3 stop + +**功能说明**：停止精度数据采集。在 **start** 函数之后的任意位置添加。若需要 dump 反向数据，则需要添加在反向计算代码（如，loss.backward）之后。使用示例可参见 [2.1 采集 model 的精度数据](#21-采集-model-的精度数据)和 [2.2 采集完整的前反向数据](#22-采集完整的前反向数据)。 + +**原型**： + +```Python +debugger.stop() +``` + +### 1.4 forward_backward_dump_end + +**功能说明**：停止精度数据采集。用于 dump 指定代码的前反向数据。在 **start** 函数之后，反向计算代码（如，loss.backward）之前的任意位置添加，可以采集 **start** 函数和该函数之间的前反向数据，可以通过调整 **start** 函数与该函数的位置，来指定需要 dump 的代码块。要求 **stop** 函数添加在反向计算代码（如，loss.backward）之后，此时该函数与 **stop** 函数之间的代码不会被 dump。使用示例可参见 [2.3 采集指定代码块的前反向数据](#23-采集指定代码块的前反向数据) + +**原型**： + +```Python +forward_backward_dump_end() +``` + +### 1.5 step + +**功能说明**：更新 dump 参数。在最后一个 **stop** 函数后或一个 step 结束的位置添加。需要与 **start** 函数一起添加在 for 循环内。 + +**原型**： + +```Python +debugger.step() +``` + +## 2 示例代码 + +### 2.1 采集 model 的精度数据 + +这个示例定义了一个 nn.Module 类型的简单网络，在进行数据采集时使用原型函数 PrecisionDebugger 传入 config_path 参数和 model 参数。 + +```python +# 根据需要import包 +import torch +import torch.nn as nn +import torch_npu # 需安装 torch_npu +import torch.nn.functional as F +from msprobe.pytorch import PrecisionDebugger + + +torch.npu.set_device("npu:0") +# 定义网络 +class ModuleOP(nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear_1 = nn.Linear(in_features=8,out_features=4) + self.linear_2 = nn.Linear(in_features=4,out_features=2) + + def forward(self,x): + x1 = self.linear_1(x) + x2 = self.linear_2(x1) + r1 = F.relu(x2) + return r1 + +if __name__ == "__main__": + module = ModuleOP() + # 注册工具 + debugger = PrecisionDebugger('./config.json', model=module) + debugger.start() + x = torch.randn(10,8) + out = module(x) + loss = out.sum() + loss.backward() + debugger.stop() + +``` +### 2.2 采集完整的前反向数据 + +```Python +from msprobe.pytorch import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") +# 模型、损失函数的定义及初始化等操作 +# ... +# 数据集迭代的位置一般为模型训练开始的位置 +for data, label in data_loader: + debugger.start() # 开启数据dump + # 如下是模型每个step执行的逻辑 + output = model(data) + #... + loss.backward() + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` + +### 2.3 采集指定代码块的前反向数据 + +```Python +from msprobe.pytorch import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") + +# 模型、损失函数的定义及初始化等操作 +# ... +# 数据集迭代的位置一般为模型训练开始的位置 +for data, label in data_loader: + debugger.start() # 开启数据dump + # 如下是模型每个step执行的逻辑 + output = model(data) + debugger.forward_backward_dump_end() # 插入该函数到start函数之后，只dump start函数到该函数之间代码的前反向数据，本函数到stop函数之间的数据则不dump + #... + loss.backward() + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` + + + +## 3 dump 结果文件介绍 + +训练结束后，工具将 dump 的数据保存在 dump_path 参数指定的目录下。目录结构示例如下： + +```Python +├── dump_path +│ ├── step0 +│ | ├── rank0 +│ | │ ├── dump_tensor_data +| | | | ├── Tensor.permute.1.forward.pt +| | | | ├── MyModule.0.forward.input.pt # 开启模块级精度数据dump时存在模块级的dump数据文件 +| | | | ... +| | | | └── Fcuntion.linear.5.backward.output.pt +│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） +│ | | ├── stack.json # 算子调用栈信息 +│ | | └── construct.json # 分层分级结构 +│ | ├── rank1 +| | | ├── dump_tensor_data +| | | | └── ... +│ | | ├── dump.json +│ | | ├── stack.json +| | | └── construct.json +│ | ├── ... +│ | | +| | └── rank7 +│ ├── step1 +│ | ├── ... +│ ├── step2 +``` + +dump 过程中，pt 文件在对应算子或者模块被执行后就会落盘，而 json 文件则需要在正常执行 PrecisionDebugger.stop() 后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关 npy 文件，可能会导致 json 文件中数据丢失。 + +其中 rank 为设备上各卡的 ID，每张卡上 dump 的数据会生成对应 dump 目录。非分布式场景下没有rank ID，目录名称为rank。 + +pt 文件保存的前缀和 PyTorch 对应关系如下： + +| 前缀 | Torch模块 | +| ----------- | ------------------- | +| Tensor | torch.Tensor | +| Torch | torch | +| Functional | torch.nn.functional | +| NPU | NPU亲和算子 | +| VF | torch._VF | +| Aten | torch.ops.aten | +| Distributed | torch.distributed | diff --git a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md new file mode 100644 index 000000000..cad25d984 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md @@ -0,0 +1,157 @@ +# MindSpore 场景下的精度数据采集 + +msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 + +本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml 文件内手动修改，如下示例： + +```yaml +ops: # ops为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d +``` + +## 1 接口介绍 + +### 1.1 PrecisionDebugger + +**功能说明**：通过加载 dump 配置文件的方式来确定 dump 操作的详细配置。 + +**原型**： + +```Python +PrecisionDebugger(config_path=None) +``` + +**\***： +1. config_path：指定 dump 配置文件路径，string 类型。参数示例："./config.json"。未配置该路径时，默认使用 [config.json](../config.json) 文件的默认配置。 + +### 1.2 start + +**功能说明**：启动精度数据采集。在模型初始化之后的位置添加。需要与 stop 函数一起添加在 for 循环内。 + +**原型**： + +```Python +debugger.start(model=None) +``` +**\***： +1. model：指具体的 mindspore.nn.Cell，默认未配置，L1 级别下传入 model 可以采集 primitive op。 + +### 1.3 stop + +**功能说明**：停止数据采集。在 **start** 函数之后的任意位置添加。需要与 start 函数一起添加在 for 循环内。若需要 dump 反向数据，则需要添加在反向计算代码之后。**仅 MindSpore 动态图场景支持。** + +**原型**： + +```Python +debugger.stop() +``` + +### 1.4 step + +**功能说明**：在最后一个 **stop** 函数后或一个 step 结束的位置添加。**仅 MindSpore 动态图场景支持。** + +**原型**： + +```Python +debugger.step() +``` + +## 2 示例代码 + +### 2.1 MindSpore 静态图场景 + +```Python +from msprobe.mindspore import PrecisionDebugger +debugger = PrecisionDebugger(config_path="./config.json") +# 请勿将以上初始化流程插入到循环代码中 +# 下面代码也可以用PrecisionDebugger.start() +debugger.start() +... +``` + +### 2.2 MindSpore 动态图场景 + +```Python +import mindspore as ms +from msprobe.mindspore import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json") + +# 模型、损失函数的定义以及初始化等操作 +# ... + +# 数据集迭代的地方往往是模型开始训练的地方 +for data, label in data_loader: + debugger.start() # 开启数据dump + net = Model() + # 如下是模型每个step执行的逻辑 + grad_net = ms.grad(net)(data) + # ... + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` + +## 3 dump 结果文件介绍 + +### 3.1 MindSpore 静态图场景 + +训练结束后，工具将 dump 的数据保存在 dump_path 参数指定的目录下。 + +- jit_level 为O0/O1时： + + dump 结果目录请参见 MindSpore 官网中的[同步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%90%8C%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)。 + +- jit_level 为O2时： + + dump 结果目录请参见 MindSpore 官网中的[异步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%BC%82%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)。 + +jit_level 请参见 [mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig) 配置 jit_config。 + +### 3.2 MindSpore 动态图场景 + +训练结束后，工具将 dump 的数据保存在dump_path参数指定的目录下。 + +dump结果目录结构示例如下： + +```bash +├── dump_path +│ ├── step0 +│ | ├── rank0 +│ | │ ├── dump_tensor_data +| | | | ├── MintFunctional.relu.0.backward.input.0.npy +| | | | ├── Mint.abs.0.forward.input.0.npy +| | | | ├── Functional.split.0.forward.input.0.npy +| | | | ... +| | | | └── Tensor.__add__.0.forward.output.0.npy +│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） +│ | | ├── stack.json # 算子调用栈信息 +│ | | └── construct.json # 分层分级结构，level为L1时，construct.json内容为空 +│ | ├── rank1 +| | | ├── dump_tensor_data +| | | | └── ... +│ | | ├── dump.json +│ | | ├── stack.json +| | | └── construct.json +│ | ├── ... +│ | | +| | └── rank7 +│ ├── step1 +│ | ├── ... +│ ├── step2 +``` + +dump 过程中，npy 文件在对应算子或者模块被执行后就会落盘，而 json 文件则需要在正常执行 PrecisionDebugger.stop() 后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关 npy 文件，可能会导致 json 文件中数据丢失。 + +其中 rank 为设备上各卡的 ID，每张卡上 dump 的数据会生成对应 dump 目录。非分布式场景下没有 rank ID，目录名称为 rank。 + +npy 文件保存的前缀和 MindSpore 对应关系如下： + +| 前缀 | MindSpore 模块 | +| -------------- | ---------------------------- | +| Tensor | mindspore.Tensor | +| Functional | mindspore.ops | +| Mint | mindspore.mint | +| MintFunctional | mindspore.mint.nn.functional | diff --git a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md new file mode 100644 index 000000000..72fb01e2e --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md @@ -0,0 +1,278 @@ +# PyTorch 场景的离线精度预检 + +## 1 简介 + +**PyTorch 离线精度预检**\*1通过扫描昇腾 NPU 上用户训练模型中的所有 API，输出模型精度的诊断和分析结果。具体而言，该工具通过采集模型中所有 API 的前反向信息，构造相应的单元测试，将 NPU 输出与标杆（CPU 高精度）比对，从而计算对应的精度指标，该过程已封装为 run_ut.py 函数；将 NPU 环境下采集的预检数据拷贝至 GPU 环境，同样执行 run_ut.py；最后通过**新精度标准比对法**\*2将 NPU 和 GPU 的预检结果进行比对，从而找出 NPU 中存在精度问题的 API。同时，本工具支持随机生成模式和真实数据模式\*3。 + +**\***： + +1. 支持 PyTorch 版本：1.11/2.0/2.1/2.2； +2. 依据新精度标准，对不同的API采取不同的比对算法（包括绝对阈值法，标杆比对法、二进制一致法、ULP误差比对法和双千指标法），最终给定预检判定结果； +3. 在预检 dump 时可以选择由工具构造随机数获得 dump 数据或选择真实输入的数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + +## 2 离线预检流程 + +1. 在 NPU 和 GPU 环境下分别安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 +2. 在 NPU 训练脚本内添加 msprobe 工具 dump 接口 PrecisionDebugger，采集待预检数据。注意需要配置 level="L1"。 +3. 将 NPU 环境下 dump 的预检数据拷贝至 GPU 环境。 +4. 在 NPU 和 GPU 环境下分别执行 run_ut.py，生成的结果最终用于 api_precision_compare.py 函数的输入。详见 [3 离线预检操作指导](#3-离线预检操作指导)。 +5. 将 NPU 和 GPU 执行 run_ut.py 生成的 `accuracy_checking_details_{timestamp}.csv` 结果文件拷贝至同一环境下。 +6. 运行 api_precision_compare.py，输出结果为预检操作的最终结果。详见 [5 预检结果比对](#5-预检结果比对)章节。 + +## 3 离线预检操作指导 + +完成预检数据采集后，仅仅获取了 API 的输入数据，为了得到 NPU vs. CPU 高精度（标杆）的预检比对结果和 GPU vs. CPU 高精度（标杆）的预检比对结果，还需要进行 run_ut 操作。 + +run_ut 预检操作包括以下两种方式： + +- 使用 run_ut.py 执行预检：适用于数据量较小的单卡场景。 +- 使用 multi_run_ut.py 执行多线程预检：适用于数据量较大的大模型场景。 + +### 3.1 使用 run_ut.py 执行预检 + +将 API 信息输入到 run_ut 模块进行精度检测并比对，运行如下命令： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json +``` + + | 参数名称 | 解释 | 是否必选 | + | ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | + | -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | + | -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | + | -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | + | | | | + | -j 或 --jit_compile | 开启 jit 编译。 | 否 | + | -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0。 | 否 | + | -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | + | -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | + | -config 或 --config_path | 指定离线预检操作过程中额外配置（包括黑名单、白名单等）的 [config.json](../config.json) 文件，默认未配置。config.json 文件的配置可参考[配置文件介绍](./02.config_introduction.md)。 | 否 | + +run_ut 执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见[ 4 预检结果](#4-预检结果)。 + +如果需要保存比对不达标的输入和输出数据，可以在 run_ut 执行命令结尾添加 `-save_error_data`，例如： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data +``` + +数据默认会存盘到 './ut_error_data{timestamp}' 路径下（相对于启动 run_ut 的路径），如有需要，用户可以通过 error_data_path 参数来配置保存路径，error_data_path 参数在 [config.json](../config.json) 文件或 [config.yaml](../pytorch/api_accuracy_checker/config.yaml) 文件配置，config.json 文件需要在 run_ut 操作时通过 -config 参数指定，config.yaml 文件详见\*1。 + +**\***： + +1. config.yaml 文件说明 + + config.yaml 文件可以通过配置参数来控制 dump 和 run_ut 操作的白名单、黑名单等功能。操作步骤如下： + + - config.yaml 文件通常位于类似 /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml 的路径中。 + + - 进入 config.yaml 文件 + + ```bash + vim /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml + ``` + + - 修改 config.yaml 文件参数。 + + ```yaml + white_list: [] + black_list: [] + error_data_path: './' + precision: 14 + ``` + + | 参数名称 | 解释 | 是否必选 | + | ----------- | -----------| -------- | + | white_list | API dump 白名单，仅对指定的 API 进行 dump。
**参数示例**：white_list=["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | + | black_list | API dump 黑名单，被指定的 API 不进行 dump。
**参数示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | + | error_data_path | 配置保存精度未达标的 API 输入输出数据路径。
**参数示例**："error_data_path": "./"。默认为当前路径。 | 否 | + | precision | 浮点数表示位数，默认取小数点后14位。 | 否 | + + 说明：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 + +2. API 预检黑名单和白名单 + + run_ut 过程支持 API 预检黑名单和白名单，通过如下文件配置 black_list（黑名单）或white_list（白名单）参数来指定不需要或需要预检的API名称： + + - 配置 [config.json](../config.json) 文件，该文件需要在 run_ut 操作时通过 -config 参数指定。 + + - config.json 文件的优先级高于 config.yaml 文件，即执行 config.json 文件时，config.yaml 文件的配置不生效。 + +### 3.2 使用 multi_run_ut.py 执行多线程预检 + +multi_run_ut.py 脚本，可以并行执行多个 run_ut 操作，从而减少预检耗时。示例如下： + +```bash +msprobe -f pytorch multi_run_ut -api_info ./dump.json -n 32 -d 0 1 2 3 +``` + +| 参数名称 | 解释 | 是否必选 | +| ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | +| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | +| -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | +| -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | +| -j 或 --jit_compile | 开启 jit 编译。 | 否 | +| -n | 同时执行 run_ut 线程的数量，默认为 8，最大支持 64，但每个 Device 最大支持 8 个线程。当指定多个线程和多个 Device 时，线程数在每张卡上均分。 | 否 | +| -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0，支持同时指定 0~7，共 8 个 Device。 | 否 | +| -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | +| -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | + +### 3.3 断点续检 + +断点续检操作通过如下命令执行： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_checking_result_{timestamp}.csv +``` + +精度预检 run_ut 过程中，若因环境、数据量过大等原因导致预检进程中断，那么当用户解决这些问题后，重新执行 run_ut 操作，可以通过断点续检操作继续前面未完成的预检，会在 -csv_path 指定的 `accuracy_checking_result_{timestamp}.csv` 文件以及对应的 `accuracy_checking_details_{timestamp}.csv` 文件中继续写入后续的结果，不会重新创建结果文件。 + +须指定为上次预检中断的 `accuracy_checking_result_{timestamp}.csv` 文件。请勿修改 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件名，包括时间戳，否则断点续检会因无法识别到文件名而失败。 + +## 4 预检结果 + +精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件示例如下： + +可以通过先查看 `accuracy_checking_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `accuracy_checking_details_{timestamp}.csv` 文件的 API 详细达标情况，API 达标情况介绍请参见\*1。 + +`accuracy_checking_result_{timestamp}.csv` + +![accuracy_checking_result](img/accuracy_checking_result.png) + +| 字段 | 含义 | +| --------------------- | ------------------------- | +| API name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出。 | +| Message | 提示信息。 | + +该结果为中间结果，仅作为参考，建议完成 [5 预检结果比对](#5-预检结果比对)后查看比对结果。该结果后续将会删除。 + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差、双百双千双万指标判定结果决定的。 + +需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 waring 和 pass 且不存在 error 则标记 waring。 + +`accuracy_checking_details_{timestamp}.csv` + +![accuracy_checking_details](img/accuracy_checking_details.png) + +| 字段 | 含义 | +| ------------------- | ------------------------------------------------------------ | +| API name | NPU 或 GPU下的 API 名称。 | +| Bench Dtype | 标杆数据的 API 数据类型。 | +| DEVICE Dtype | NPU 或 GPU 数据的 API 数据类型。 | +| Shape | API 的 Shape信息。 | +| 余弦相似度 | NPU 或 GPU 数据与标杆数据的余弦相似度。 | +| 最大绝对误差 | NPU 或 GPU 数据与标杆数据的最大绝对误差。 | +| 双百指标 | 双百精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于百分之一的个数占总元素个数的比例。测试通过标准为相对误差大于百分之一的个数占总元素个数的比例小于百分之一。 | +| 双千指标 | 双千精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。 | +| 双万指标 | 双万精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于万分之一的个数占总元素个数的比例。测试通过标准为相对误差大于万分之一的个数占总元素个数的比例小于万分之一。 | +| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。 | +| 误差均衡性 | NPU 或 GPU 数据与标杆数据精度差的上下浮动情况。 | +| 均方根误差 | NPU 或 GPU 数据与标杆数据的均方根误差。 | +| 小值域错误占比 | NPU 或 GPU Tensor 中与标杆的绝对误差大于错误阈值的小值在小值域（小值的总数量）中的占比。判断为小值以及绝对误差的错误阈值参见\*2。 | +| 相对误差最大值 | NPU 或 GPU 数据与标杆数据相对误差的最大值。 | +| 相对误差平均值 | NPU 或 GPU 数据与标杆数据相对误差的平均值。 | +| inf/nan 错误率 | NPU 与标杆 inf/nan 计算不一致的元素个数占总元素的个数比例。 | +| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。 | +| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。 | +| ULP 误差最大值 | NPU 或 GPU 数据与标杆数据 ULP 误差的最大值（取绝对值后）。 | +| ULP 误差平均值 | NPU 或 GPU 数据与标杆数据ULP误差的平均值（取绝对值后）。 | +| ULP 误差大于阈值占比 | NPU 或 GPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 或 GPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 或 GPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。 | +| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过，warning 表示测试未通过双千或双万精度指标，SKIP 表示该 API 的某个参数的反向不要计算梯度，所以没有任何计算过程，其他信息均为空。 | +| message | 提示信息。 | + +**\***： + +1. API 预检指标： + + API 预检指标是通过对 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差双百、双千、双万精度指标的数值进行判断，得出该 API 是否符合精度标准的参考指标。 + + API预检通过测试，则在`accuracy_checking_details_{timestamp}.csv`文件中的 Status 列标记 pass，否则标记 error 或 warning，详细规则如下： + + - 余弦相似度 > 0.99：≤ 0.99 为不达标，标记 error，> 0.99 达标，进行下一步； + - 最大绝对误差 ＜ 0.001：＜ 0.001 达标，标记 pass，≥ 0.001 为不达标，进行下一步； + - 双百、双千、双万精度指标： + + 对于 float16 和 bfloat16 数据：双百指标不通过，标记 error；双百指标通过，双千指标不通过，标记 warning；双百、双千指标均通过，标记 pass。 + + 对于 float32 和 float64 数据：双千指标不通过，标记 error；双千指标通过，双万指标不通过，标记 warning；双千、双万指标均通过，标记 pass。 + - 在 `accuracy_checking_result_{timestamp}.csv` 中以 Forward Test Success 和 Backward Test Success 字段统计该算子前向反向输出的测试结果，对于标记 pass 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE 表示测试通过，对于标记 error 或 warning 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 FALSE 表示测试不通过。由于一个算子可能有多个前向或反向的输入或输出，那么该类算子的输入或输出中必须全为 pass，才能在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE，只要有一个输入或输出标记 error 或 warning，那么在`accuracy_checking_result_{timestamp}.csv`中标记 FALSE。 + +2. 小值域阈值： + + 判定为小值的阈值： + + - torch.float32：e-6 + - torch.float16：e-3 + - torch.bfloat16：e-3 + + 小值域的绝对误差阈值： + + - torch.float32：e-9 + - torch.float16：e-5 + - torch.bfloat16：e-5 + +## 5 预检结果比对 + +需要同时获取 NPU 和 GPU 环境下 run_ut 操作的预检结果 `accuracy_checking_details_{timestamp}.csv` 文件。执行如下命令进行 NPU 和 GPU 预检结果的比对： + +```bash +msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_details_{timestamp}.csv -gpu /home/xxx/gpu/accuracy_checking_details_{timestamp}.csv -o /home/xxx/ +``` + +| 参数名称 | 说明 | 是否必选 | +| -------------------- | ------------- | -------- | +| -npu 或 --npu_csv_path | NPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | +| -gpu 或 --gpu_csv_path | GPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | +| -o 或 --out_path | 指定 api_precision_compare.py 执行结果存盘路径，默认为当前目录。 | 否 | + +执行完成后输出 `api_precision_compare_result_{timestamp}.csv` 和 `api_precision_compare_details_{timestamp}.csv` 文件。文件示例如下： + +可以通过先查看 `api_precision_compare_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `api_precision_compare_details_{timestamp}.csv` 文件的 API 详细达标情况。 + +`api_precision_compare_result_{timestamp}.csv` + +![api_precision_compare_result](img/api_precision_compare_result.png) + +| 字段 | 含义 | +| --------------------- | ------------------------------------------------------------ | +| API name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | +| Message | 提示信息。 | + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `api_precision_compare_details_{timestamp}.csv` 中的各个指标判定结果决定的。需要注意的是 `api_precision_compare_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `api_precision_compare_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 warning 和 pass 且不存在 error 标记 warning。 + +`api_precision_compare_details_{timestamp}.csv` + +![api_precision_compare_details](img/api_precision_compare_details.png) + +| 字段 | 含义 | +| ------------------------ | ------------------------------------------------------------ | +| API name | NPU 或 GPU 下的 API 名称。 | +| 小值域错误比值 | NPU 与 CPU 的小值域的错误比率 / GPU 与 CPU 的小值域的错误比率。标杆比对法指标。 | +| 小值域错误判定结果 | 小值域错误比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | +| 均方根误差比值 | NPU 与 CPU 的均方根误差 / GPU 与 CPU 的均方根误差。标杆比对法指标。 | +| 均方根误差判定结果 | 均方根误差比值小于等于 1 标记为 pass，1~2 之间标记为 waring，大于 2 标记为 error。 | +| 相对误差最大值比值 | NPU 与 CPU 的相对误差最大值 / GPU 与 CPU 的相对误差最大值。标杆比对法指标。 | +| 相对误差最大值判定结果 | 相对误差最大值比值小于等于 1 标记为 pass，1 ~ 10 之间标记为 waring，大于 10 标记为 error。 | +| 相对误差平均值比值 | NPU 与 CPU 的相对误差的平均值 / GPU 与 CPU 的相对误差的平均值。标杆比对法指标。 | +| 相对误差平均值判定结果 | 相对误差平均值比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | +| 误差均衡性比值 | NPU 与 CPU 的误差均衡性 / GPU 与 CPU 的误差均衡性。标杆比对法指标。 | +| 误差均衡性判定结果 | 误差均衡性比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。该字段暂不参与 api_precision_compare_result 的结果判定。 | +| inf / nan 错误率 | NPU 与标杆 inf / nan 计算不一致的元素个数占总元素的个数比例。绝对阈值法指标。 | +| inf / nan 判定结果 | inf / nan 错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。绝对阈值法指标。 | +| 相对误差判定结果 | 相对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。绝对阈值法指标。 | +| 绝对误差判定结果 | 绝对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。二进制一致法指标。 | +| 二进制一致错误率判定结果 | 二进制一致错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| ULP 误差平均值 | NPU 数据与标杆数据 ULP 误差的平均值（取绝对值后）。ULP 误差比对法指标。 | +| ULP 误差大于阈值占比 | NPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。ULP 误差比对法指标。 | +| ULP 误差大于阈值占比比值 | NPU 与 CPU 的 ULP 误差大于阈值占比 / GPU 与 CPU 的 ULP 误差大于阈值占比。ULP 误差比对法指标。 | +| ULP 误差判定结果 | ULP 误差判定结果。
当 NPU 或 GPU 数据类型是 float16 或 bfloat16 时，以下两条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差大于阈值占比小于 0.001；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。
当 NPU 或 GPU 数据类型是 float32 时，以下三条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差平均值小于 64；
NPU ULP 误差大于阈值占比小于 0.05；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。 | +| 双千指标 | 双千精度指标。是指 NPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅 conv1d 和 conv2d 使用该指标。双千指标法指标。 | +| 双千指标判定结果 | 双千指标判定结果。双千指标大于 0.999 标记为 pass，否则标记为 error。 | +| 比对结果 | 综合所有指标的最终结果。如果比对指标中有 error，则标记为 error；有 warning，则标记为 warning；否则标记为 pass。 | +| 比对算法 | API 使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和 ULP 误差比对法中的一种。 | +| Message | 提示信息。当前提示该 API 比对结果为 error 或 warning 时对应不符合标准的指标。 | diff --git a/debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_1.png b/debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_1.png new file mode 100644 index 0000000000000000000000000000000000000000..3853626d6fab127915425238c76f16836a4cef0b GIT binary patch literal 3534 zcmV;<4KebGP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D4P!|}K~#8N?VUYu z+{_WjW`p&**ujn=?saK~~lp=Nhcw!827RfOK4C-O2mnCiY9uQo1Ejo{+b(=cqGxIj3e zGU%2Ha%dvkr$I~1X1mqWWG)b+(sn;kWho0=-KL-!xjaz5?8$k5C83HxB!~jbn4Ub= zL(J{tvlLVaKa4?eu^ooKqw52&r^%~|#l}9+IQqOUTfE}ZMS(({XO3E&dby8&hN_6f z$xZz81p|CrbcL`7QYI;vee&>fT&Q;b@~?5)(6%6#UL>=Yt?PMzDh0Gjg}?XvP55f_GLPnbjwp7NRIk4)HrCM4*TT+W&iB7x%zC~Y4xaK>Ro5ydL zcRG|-Q#`bu6wz^ng6%m@zMFiz&1UE~6l?OzIyg8$lY)f957Wdipbripnk$Dt;OED)=?d0>Hij9#avU>rL>zgxr`q5yXhlmbXrC%!H`{tn~}p?JiYxog3_Mma4a(Uh34jqLfb z!gbB#j*&dyU<4D!u@q}EcFP#AYmbg;I_D>Dz&_ZN=rNEp5US9C6>pRoLkVxqfi>-}jqglfKcL_R*Ecx#hg5+41hd zzEuTkHmNLyO;;O}i7cF~2fiqE=Tyl~?F{DD-KVX$gdK+kW8%WGJN)M;CS0A3GskDg zFe&idFOv~Rw@uEAz>E$HSWN~yYIBR(oMv81@~Xo%KxdX8jdLLlM-3gmjgYdxY2Y_y zXbC?T8;)Wvgl1TCsE$;DL<)_wV&U3xb@s`=jW@%LaI|+x^3*|h04F)*Hipz}*zWs7t7WC2Q8IZVsjdE!m$s}0lR)fw;EXziBkNYqlDmYjSo5Jbzrl$CrhGU0a zX))&%!m;LCj)o(~4>xOaM(GpKZX81yH;Dq7*1*KDJ4f%Rdhk;y)fMA(Z=3f; z^n2LDwM3sv!LA)oGgfh1m2f2M(l6@+hoe3QW-I(D3jSb+A++UeM#GXppP6|4MkRzJ zf1Kij`;w)nY}1ZmZA_-&!!e{Hi6VC0mdUPopwBV}8lO}LNBfQHg>v|O2DuySvk}6c zHsIyJhg$hkVN5tiXhTuE#%CT@6pDQ*)dfAyK5etUXn_W*gAN9&i)_})WJ<@(7aZCz z+ZLaGuU34@6WN~>j-PHmf70UB{YMVO+pqV$@2S|dV^|xLX)Fv!h-={pY=)CeCOSmy zF0O4o;<_hK-GZF)rzGga`oM-`sOx<8`f0!HP)45#s@0`kNqbcAc7WH92C6omaa*p5 zP;exDr@GuOvJ{DXnhJ*PHd|t*ebn+O@9!RHns$@*3gPI-WEy!mj%6T}IlQ8Ym>1%1 zvL+2NWy|o&*`aeu$(*Ea0U*{`MJFQ`JGt!eSqVBxnTkLnQoi0rgh$HpDGB|uL+Sh! zo3*-fv7trCLtm_Z6przDQZbsWeH3Ab`Wr2Z7o6wM&~TiMyd%P+a=N%vxP-ZN_i5`bVaH+qm@Ey)M&kg4a4bCETAT67 zqS4TB#G1h03SdD%63(+v16BsQ6jHhS$if00vm#d4R zwKkTU#*UO}8IEJT1Qs2?9>oC#!{{Tl4MW zrNVBXc58K)+Dyi(=ojXnw9r zF*$aGmKTAs$Rmb?Tm8AK%*|P;|x(WIkx7F z$eNt(_}trX^ldg#ulcEvl6Mz&Z1UQv??AmsRRtA^XdBW+s8X8cOSmEpCX~;_PW&BRKRB=wwmdCVA`tI?Ch|g*%>G6cCEiV*h)=r|uXmG>XICy=h>1zaiu7P3aB4MmL>g_s9ajAMPiQFFuaz0}FRcR&?WT1+1wl)slU1`JAz%EyC+g(ugLcfgFxWDs2X8bJ6 zL;3S-E+NiMj&uF=~&A+vXlp+^kb66fz#3$AtlpKQi~T!8uQ+foZl3p{Eaf; zJkI*4?y`RA5$OwdvC(K8i)7^paOs){+AvHuv!la-ic_h1sBW}p=t(zRN8x2mQ z(Rj1itS_W*HvC%j)T5@bOUJdpqwe3GW+*p+pOODAizWq7cim&}-*{pVt?B~R|THw{h zDx@~%K{xFu%aWYv|LNhzs!(!_`QhT?XIjko<$wS2TiX7f_lO_<_N(u|{>%UV_^001Hg0ssI3odsUI00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D6c|ZFK~#8N?VZ1i zCEIbwd$p^lMF=N)4@>rdgfW{)FpF%2G$6(>(8<`?pb6w;urqVnK%s&`5=iM~!!amP zp!hNitIWYureH2!KlBG!Lg6*zz?p=fzEyR8)cMu7`~KLSyHg)#`<||@Q>RXy)2C1W z-0sNO6pCF>|7Q4`9UU!}DJ zFK%x{4pvKe1-xTSOnG4N0Q%jEaVt!=`W7f-2RL@;BzgJI0csQ36&6ykY$|Uxm6)-1+El^X8~kRBtb5x zSUwld7hSra%dt^P7vy9t>HuYj({dijUg^?(+O-|)xUE44P`Y6ul#4$^LtNpll9m%o z5(@pC&TK=8pjx8hvlNt$5PI%WhoNF*av>JGTrpL;Re zbh6>Rdh38@^h)L&1z}25p5kb9?W#4c0?n##<1O5KhJxnNIwN|7zY;^;Zh@m}-(${* zDXlE!2n#cGPO+FFlyc;C%jFy)cbYs=U8=Mok|1ZITac4k-pZxnWFusWY~=-+BF4{1S6Nr}*;4ZcLw)R#!PL?5Gy?V!J&Xk;S8MB3T0Us# z>WRNRwhuy$_*XN=JBRCb$RU^L!gZ4gHVk7Te30ci1dWRRXe|=M*c16^hrv_kBG&DU z{qeBocCTN5{C?a@KXmMr!Z3^nK{nXwCxmLu_$m|Q9~g)a=JO?|US@`jPAhtKF{htX zk-c2KqN~eTuCPqxM|82yo$V-{B&~Hy*l%6#1P1d~SIkk$7Vehf0&lMf$ATDLPj>Hw8capQn(w{`ynhZ=wX^_h+LMmA2Orr zSW=X=F3;u778k+Bw)>{=D?6So%1Z1gTJnjU()H*-9PH#y1>rlQ?v$>>`3CCeP^N%V zBGX}H01{;IitB~GPxK4wVQ%mO+F%&$luoYS80>t*x9Y`dBUfce#+V=5!-zNjHd3ASmM7J-)-b;^0p1y=K{Z$^5TxVa~IXXIj@PK6!K>{9HSonaUcf;iaG38~D+{Yb2jv~nzp zI)-5w2ZbN(=z=<3gz%s2A!N2G4=@rn9m6n;gTPK>4a0a4+@rycZnAiE+As`bOFH0^ zhG7`*FFPIVvJb|#*<{w%4}QBuSf{hiY_XMhLz}f&HBHr?V91{P>FJ9KmD(rY3mAAx zqt$k!+jIhKb3)wq;luk*tLx)&(IOWO$N9FFSe z__+cB0#I^a~Ler zen^a}V?kL~)sBY$r;iO8+wPm92cv4o8%b+W+xK8M<5#aZ#SJ6z(rFMHl@jtl&G1*i zL`Rl@Uu;Oy(4n!Bzf=XWwh1Jil?7DVJ!ClGThIaMNmQ!CJxwmg?19he%{UL^||CjkcMAqFstn_FWz2bt4?{X{A<2q_uBF zSKnNM|UA>xdnCO#;g+D(HmM=eKWA92Ioc57^hB zn&tdV>Mmi_agY|lhT+;*2Rk%(NWgL!PX=fXcA+z*9V=<1(5Tpr#d{>mPUEV?NZRQz znSOY*MGV3&uSGfIc2upC=?9$*Q3I`dQCbqb^9b4{kok;CNw_k3(5FCgi(;^Am+O$c zZj(xQ!XTC8#UG3N1-*ugz0die3H~ruJ29%^>pm&8^FkKmru$)xC(Nkcb!#&_{+cG#NMEP*NDN74Uy!*H1}{jooN+s<*3oGu4tCfS z(r#Fya%0eSy+|&t;%9c@h*8Z;KmPNXx-PU@^!~LoyHJ)?2NVtInnwE|4Iw)-d$MMr zXDW8IKN>9;$0V&|)v9|g_kBj&Ec<}SH5>S#!RRj@>3&G7z96w5!XD%Rn?fQ8L8znH zRAXFH$5ua&P3-LqfJv51@%x_gIp1<-_w2Ir;%YS@8a>#_FbbPA#;L(>zmmsp(A3yG>b#i0jaduDg~` zfOm|kY*bdOoHgZ7ZQqt}8}t)*CL0E+`%5K;SscnKcFC@;*=e>I?2HFbYiFK50QETU z<4eZ$za==veH1{IH97CTnI4Bam|>mER{#!Ru&bcsfPp^T8yLU_JHs%H4M_((`V*NN7pS967 zJ9-<8PH7!1Cv0YC7{Z2v>EVE`NK8qk}MBc6RU48t%C z!?-ux*NGjyQ?k=Lz7K7p_O9wXwcNMt=emAH?l`7;)z4<@z6A-WfSva2e!dWwbF1V0 z$(Xv|?R-Do6T#0t^|wQbgK_9I_jls~@$Bl2e2(hkmOlGevge5%#=F`u+RuFN4Bz0% zhy$fM)Dpp3OWvS8p!YNH3%#GLoJlrBFPr)VRXBI1VW%l4YIa5tYq_TbScA>W9j70) zvcS*=gs<9%YQ%muNa@wodbJ=s10G;AZ-t+uO52rNo{{khOx2&r*y>TQ7}N`X?mA$O zJ;z}LR58VvI>tFdeGpbqfD2qihsHz4&f0Ml>+7{`FSxvE9Z>hop1SW_0}Whv9~FBS z^=kUlV5FoA(Hql?|;CC~+pKd3}FU`kQ}WhV83cdpj@qL)5fq#EUG}L z3b)#8xga@WhBSz=tD6aoE0;^qYP`bwYuZKIrvPy0Bmq{~S+}*-Kdb!;wCg3ca4SqH ziI7i>QV{*JfKYKsLPzA#lsA*08-9n5x13uO>ovt6q7|POHGQ!!+d*hDu5YDZvHpa1 zZ|RdfC3?~Hvr%rabHtZ$g8J4}TAv7#2Px5?+J4F&*Wxy! zUc+@5h7P6f+uiQZeR%D~+3nrgB`1mg-q4(tKMsl5Bzj)>pK>z&OU3W?4Rf+&@ zRt1>~TsnU)Ufn!Dbz6>3Uz|jLUOInB|IHc6-<(}Z-EU4@mP!5#2z}Zf)V4Ixp(SZA z;`C6xq(#dPG1MP)ky(!@{8__D{Yt-6BHf=i?T5njiCx)iQej+c$r-Gs6%5t$TTn?! z4BBD(88WBlm{eZN38>&T_Pr;8)? zY-{e<#EupGmI)I(pxP)QZM#;^rhS@0XPL5+($$K71g4uAm=!N#%WGQ3q?V-tze$n;D5g~S_3bH=p+dAnX&EkG&BNeyQDq1;ek zNTOqXKjWtpFuc((PUL3eo{$@GYn(qSMWkyr8u~-^Wx1EQUy|4C2w2UkHB~>;2y0C{ z4gDqq_IDSD2BI~fn|VsUMHWdR7S-RuvHtY*SZfQgae z=fW_SQ?YzS-{sqZ6Omm^?7${hp2Af}U$LwOG_;A)w?fqOc!~Lz98f6z2450)2z{wm z#Ia0I^et9Pl`x=BantxRs&+t2-m*iW_~N{VtCoriVbY|v_6qu5Q{_nFsr{c+F4Zb- z@oTJKX%|qnpIQN=<%($aLsAj3>Q^MM*Gur*f^DZ7C|pajWX^{;D0kBpq&W%A(sGlm zn3OU85VXxHj=BZe&{xsKzUkzlTwkL5Ww~zmSZL;|Frd$b!d2FqpM@P!WxPIm!+nUu z5Xeo%D&>Lk5S02L>@RVc^i=f;@4hF*z!SHIDyQhZv@D?7jw+&lKOKr&qh1YvHY41H z^tWOibgfPwYx&SS;#Ta-cuRDMt?SU7*d0CdPSX9^tF;q5uz~+mDf(d+fY2R>`UAm+ zwr_oy8Eb?*66Kf@YV(tg@psjIqd2M)yZu8|^L5+0Y7d&N_w)_fce(GXUdG#@)xJ~Z zI|MT$HEVcF2QHh9-9XY~+JJYuJ??!!QiPFbrdW*2J!-Cw}Jo`XycK^tUI}b%las)P29zrRV3Oj~j7=~dO#(pR!b~0lVWI`5hqUx88 zinqq-MGq|*uwxj8VHk#C?1Apl&)P3fq*vPL|3l!pS^5D?p0-u4-YHT$hG7_nVHn0f zsNRY-48t%C!!QnyhG7`S`v6_BdmtQtc+oxj+3|$(b6)vE*R=|E zJaWI@xLX-lR&Z?fmG8C^e}B8|y3cjw(4)_N)Zy>xe|3+4+CBcslgB?1m7%g41`${o zd@R$>lB-8XsZSRcc0|KYE=Y2@$E_)qD7>ECI2^b1;k z^@f&Q*728Vw>|!gf8ynT(DE;zV438t)Co9;*d z-+l0(-Q!<%AN*hU_>+%57Lyqx`UdO_GJ2;Q(+?w~`^H!DzVS00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D6MRWTK~#8N?VU?* z+ej0~a}ckyp7m_92*6FYUi$z-j*!K70U^t|h2Q{lfH{B#3DZXm+tP^5BAC`+9^m9 zFx&r5FXM$>g_}a-=Pc%aRS@2q-auzaZ)l$+o}W{iplp}8F8ZlbzA|I}-|ddi2mHgy z^MT58!G!yxPGA(Q3`Bwdkx-Q7hz{>w!s+2ay)9Q!xV!s)_vPoeCz|{sv`cpM5`PNM z`KX$iX^#WiDm1Zf(J2X8qO9)!y0I5vwj|%290h}-vYhhSYcXN9IvTFqexy3_6EO1Tch)H_gJp#Cu1>AYT6@xq`AB; z;FY7j+*&l2G%=$?-7kF%^?_cg?qIc|J~&_Z%ZHx}*z%5L zT5uP7O6Mi-cA~l)ZT4~%nws|QR%jn*2ors%5=6haUp)249~GNyo`7wx$mb<|xGAbJ zumPL=JA31fy%lMdr$k={wnb85+VPwEo$1IC$oEZr89JDn(~kCQh`mi9<*F^<$~$px z#y8Ue2X~i=Ss{^0yQqb-uZtv3d|!xLj>cxEE}iZt`S7fXMfDN)@wi8FaEsz?T3M|5 zx;E~c37~O#AIxVkO%sxgTGfZPzAd#mJ7K;){l@XHe}3Zl?di{FD(l-2BeLmp80Dmr z+gK`XW*sm0Z1O=73Dl%)SVo#@W{x5BdqK-b=SJqU?QZ+EP0Z<@v}TlIyQQ^qX}~&gca2I=;*l zli$kr6yX!(<083?o;3NzPrB5rOFwpGf#ul_ohE(j^^=p|+LCa9ww2MY@jggL$iokN zl4Cb~TVhbjR^eI71_PJY_*uwLm?GbP<_hpC^dr;BYz%yv72Tc_ChF)DGyM522Vy0_Iy zh4HcMJ|aYJU5bfnWhv0DTGsJ}N^TRQw3*9Z?%Co)LZ*b>Rsp+RG+@W_R)Hz6B}Vd0 zq=sPGKe3;Tv2H6noNpJEPF?t}_G3GP_m56~-OO}AH#`5cJy&mrZ0`3wws`m3ak-p$ zc)XI^JeqAbg$g;gFE(?Gz%2wc=ZVP9;Qce6`QI)7{`;T(Q_*1H7?Xc6_=tQ0 zT6{4}U!`ukx7@imWDzHHUUE+x)!k~dldIU2-m!b8z1`aj(-elk_r{4jTel-;eLS-B=Of2s|6NIE;F_r=epVgNzkt`z_O?lK z=E?8IjnT-+=9)g*5|_clE6msdp7eaSB$wlpSrlVS)0QvK&c|{L%-ZRbC%?X(;^uhz z?IccL&IY+~zq1%9uA;u3=zDM3CXv1YMn8(>L*&3Fw%4ij-Q*W% z6n(Qb?fqhrec&Y0`#XLo$5*q>Ze7;o6g07@J}zQ>?D#K(l{u8MxL33lPkxQd`@lZS z`U2q3gj&r|$!#T@J}nL3|9-%6qG|u-=eK{pqkK=tg1$|k!zd?}+zn1|)9Yk;iPvA~ zLqbOsT`k9FW;R!)!8|@S#=9L?PJh6ric$L7`_{CT>=g46U5gUe3dTTeX9hp{Qx?jE z7DiJr@Bs&LrfnE-sERn6&l~+zpzo;SXm!0o+xeYNtsm_CsfVJtxQv1h8o^v@>q-Rl> z!`F8H`(N}`-%Mna%P2?=%eI0X+QgW(sb$ZsJ_oj`eV$6LV`A;}IZdD?ay}C9AX6($+oBS|=s^#jdG4oUHW!p_H@acxPYon18Z<>%+SyU!1gvh(*Re`M3= zFv>|Omk@+(v(#~I6*&1K9}0H#2b`!-j!#^mDol9`o#h}IEXJpg5tnn-Dn{w+I>y{q zwqu=6@ZE%q5l9&ZP(@mZ?97lS|62Uh{#&Ng{fk*!_a$w;AIilm=cJS`$4|4RY;9{B z>-s(R?Ofw$p`GoMe=z714es?5QoiYgflS|3N-P@JuC3K|FU@rqrWvk1`M0d%r|`E! zOSps&w}iVF9eiZ+0E24+2Yk-fF5U`82(GCf8&VmND9` zZ|PgpR-E?~jJ3GC2Y&Le*(D6}(p_Y{*~UlP(lr|~V50fPbq zMaSiK(jaRl9?B?C(>y3u&S;^M=UuU|>2#dDV2ck1PmkR8wDVu^V8~9&TZ5ncnr9|8 zv_tz`U{1!^v}F$aPi=&@E*3krAKMwcd-CMR4wHsw8gM+^rQ8mwm8_q5HUT1eG%>uq zV7(6xT=kO{v|li%f;PQ0r7bScM}6}mBsqO;>aB9h-bY~w}Uw3>%@2JfCU z`O~eYjM3tmpKmqOXVKg7x8hvHC~YR5yXEE^Y^Ib0lx47Hn@L9$_)dlGm#p=n#6EDQ zKM{q>Tn71r=<@kF%Yjb*GS4~2X~fb0LEZ8~ z_A=BbDB#+ez)mP9dUh6&qm1Ld1l!wPrk}N{#Lxw#uRDf%($!0BFN5+2e`~v z;peK>7RL2*-uA}7w3lcU&FYz-=j!*0tMyk11?E^1A2263dmwK0jkF!)7WoRaI zd1&R#ajIZeE<_`)aHfd7pJ=BKraiqXhpdXPYiGuO*N#B7qOj?rT@s5f5ia1UzmD`r z0VroEgPgm*8Dm+y+=5r*6V^Y`FWx>SfCncDaKg^&)o6Tn#}$a| z-!@=@OA$IEhbB)p(zTx_*4JKKm7hL!EBXBJ_>KR}!~N6muMf$FZ2WfV*W+Jb^h(Sc zKRG|0ame~B#z~}4@{;UD=x5Nb*{a4>#I^F^sqE)dZs4TLnPMbWR%e(Wfc>kbw11;O z9+YH%dh?n+?&WPkT(y@tr#?aciv6+sI4c%ghe)nQITUkfgFhSTZv3s`Qkc&oIfK*E zaf>8fK_^LJ)((Tjx-@4>d9NmLi6(kl$rB^mNR&ojfVh&bxnat*c4ny}3ga#H?96uz zOX{COc}7%{6M4otri;tl$$qIMWB`>>jV7IxSfSL*XKUq*Wy5GCH89~6qPSSj3wmn} z*I22v6cG4d`B`eM;rrL8Ki__Q|6J|swDJG`+k_8~_rK_`zr?Kc)8t8_KRIt5J4Pj$ z*T-p{;!?xTLOX$e$c+87=Aps00PQ+#5Ur~PY9%$_x71B@?YA81+{oP4HcXrFG#s!CN$QvkY^a%C{bTSu;Erk^D4XV(n|7nion@KyJWE)u6v>9_V9NitB@%x5$5 zIIb!R&`N4bgO$FuYw<#o9h;^dKQ3T}qn~(S*q!>uK#qhc?DyJz0eJMkx7YjCtNu5# z@#EB618emw4Lb|%1o|N}_Rp7x0MVM!t-R<_)Lx*kbQ=xVPiV(>ZL7$1^dah52=kUp zppL6!9~4y*qMD?pQdunIT%vm3T48*dKO0%N@q;Z-uJ)26Ib8LChb}Sx)tP&NQkakA zz`yEAEpZC{s~!>OHWNPU)LXhCpByO zNz>IR^6C9w)h7sf^1-g*lStEBX8*zt7K)_^UU5u*Of)<@i&^^e4;xILSd-70--5 zV~xA2JwF>eqRJ$FjGje#?%4QaR(H3nTxlJ9uk+&4ajmug*7AG-e_mtb9}EVA!C>%8yz7m> z|5oPhQM>}Wz@6r;@~`|x8h#ev4&CC=XJpE!9u^Bb<~+Y_^{y`g3i&L}j8wWwpbWc8N%_ z9?we85&BZCeD$=sL4oy3oeY#nE0O;v;_CSm#e~}l-}dQtDIgpR`&;}O3`Fq=#yBBb-Szc5JHs?MUpTu>^J7vX1Sg4FW^Wk3HD!n+|;3cGYcvFjx(3{FmT=`g1TC3AM*eJ002ov JPDHLkV1i#&ZLPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D5)4U1K~#8N?VUYt zeOkr2%skI);WNXBcym25K?aO?qFnrcYt?*2-t%d*eksGAcw<`p(RC5 z;{DGLS(_RShr=N?rYYO|?ciJh4C^#-`EWWNr}-2PBI9w6k0VRCkTl2R>2%`5VM1jr zk4HKOHc#`3%UElM!y_J&z8H*9Ej9(tkSHgb6`vV!0X&3?U>_>NKbC_8H2Dxtu>>A} zy#4v{_WJOQR-mlq;qmd6%Ybfbjtw8lz2l7i1wi@X?Ul=b&(@3JVZ8_**8K@Fg>1nI z<8&N#9?|6*t_8V<)%4?uG7$d>R1V6FC~%=R#eY&31)(@(SXoXA-3-lv*_7(}5S4yqLCN9Y&We^zu|V zlTq{6EXQP1m~^F&j`Pj)?@-#4vKRFXUNTdg_-ns&9r#Rjt^=Q`&J_@*=}_)a5aSV6 z8C>91wq9%VJDB5foN0p^{U#RG3{D6D9%b5_nl93w&;ob)l6-3J_Y6e zVn4H!+Z3SnP!h&tyBCH9fMZMn3BdcO!g7pZtHp@a;>Tj_xYvcwwZqdCXAuUSLmvZu zIwm<3c(CD-hbPG?^dp~ze`Jm&BG=Ot&~3pZ)S_D;A)sP;8d4d6H@{#fubbB*ln#v( zJkQ~jP}pIimcS!stLVXHdcSNwIwm>>ooU(c41quJNVkvxdL#S z%JPLTz9pBb<9sAMP2)O7KRV8r!C&;9D`3}YI^uiVC2Uz@4-0PKp3`~H(%!1O(Pk@F zrm0~nwn_lmZes@A^lS7!Tf(*M1gx=JJffP8ib3&TkJb))F;BfwjJx(qtdWNikKf3H{POh|j<<3=m|fRo|ykjsXoj+8>RU2Y*5yOtG^1UdUz6}#OQ zbym#dkmAWB3n%)$Z{|~&Hc|k~l9xJc$)_^k^qkOT0EU(eIDuOE^hj$U=s7&sVQRFXeI0Xq)vMV7X?451P{) zQmCGXco-XE`x17M18_s>MOdz#|(&2Fb z_J_aAR_D{3#}`t`Xa-gj63rvtrM#zxz$TPYPD072P5Txy7I_>}JOK#8+3q*#D!;kf27Nf<2a5%%BNv19pC2Y9P0VhaUAC? zdVR|8x$un=ZX(9DsxCW@<2V++fBHA`-~awk{({JHoQvW4)NveV2fhKJxCcvbs&+nA zZZp=OOshDfv#H&HRd=h+Myza8joZY-RBUJuL%aL>10|i?&DPjm;zl{zToboT_;5{B zWBrjV`8?J;X#TLx1}r=64-u~B`^#-8^{Dgdx$db?&z|Traq_V{fad+620)h;`dS+3 z$Bc)Gw`ysZcEBGz$>fI#GC{}%N+&C8of`6cWA5wLZOBP~JkIpvAoDDrjm>D}2r;$9 z&*B=RU#V?H`}~K!87pV4%w=kR2$c+Nt{ID!KIG6evXa8-2%fZjmSE=dkXY2mnyM{b z?p}BKSaHa>CEM>Hjr<5+Iv7-Y1f)M=|9;`$2 z9&c)Ks?E#cs=&)=z;6vDn@AWo&X2VmkX)&C3VoGN!*4h-;RC?Kr1Nj`f#&N8j_I=v zbT-*+)+J2}LDiR_4=3SL9FI^BPG}u2vwK-vc0LXL(ms&SoU#G%XMkFbt1`PD8g_Dj zG)W%);WD%vVrBKckgMxm+tfNqo}={_`Va{n$Fuo7OlX@Gxxp01(8t3_mE$k_6@BC} z)xK42MLXGb2SW%Ys%4C|ZO26A(;xKVQ~bKr_ZKXGe+vIk4ax^D=VbGz!S@&XKAvix%Q9UJ2BP1*Jn!#JI)Uw|v(km4EQjMIShEE@TgFLNnyBtuv&Hs#Ye;qtkg z%0Qeq&`o%tKF*~M!M$ZT7U{Usj|KeFBs@x8Z7}9atyAi&e5#3a!s-{5p%AW5MqH}v za)HBD%A-C73I-eEgPD$G(w3C7w&JN&h)5pH>dr@QYvSJ`#MibIu1I!0H0`9a#?qlO z6gXCST_aYKYq~Z=S<}^rLd%B)&S&~kjiBcwV%G(9K2-`Sp6~`FIpKPB#0lsUNElL}n-zK!` z!kPlhXd!#eO#14D2To`mE+-#iTe=QE3t?!q4>SEZqvA9LFIG?C@gr@KdxJeB#h zXWt@@MIHwoPt=KDy0vM)-^Xv7j>j-AG`^u3lc#&=+Bvd(x)!~2`ju&1X^UA~_dRW0 zA91cTsqeXbIXZJKWouj8SU0}5Z+p#?4R$!SEnIMYEvk6x*FS@u_D`4bu6*kJ)3Nf! z`W#39p8&4E@T(({MX5rywYu)5Ss#Z>6Behy_2*L<I~UNp;zA9LH&rtxtJ* zs_(H{Wyf(Gr$Fb^!xZna0$6I59mjE;CRsjp9LI4su&e! z^g@WA7n*O^5(no(?0#Ke3ugXt{z2S4{|-OW^7fveGtBt+#h0g$wOcO+!x-oEDgF=X z1u5OD1p^}1EV+UGvNZ6Q9g+E{e~~#mk-RkZ5$tg3jA6%=BP}}%NWHZCdkMC;yG%c6 z6@kGM5WeOdYLVtum)fhf{jxwi9S`6%odrJ^wYHaAk%jRQ*6OchY4>neboIo~y#p4w za2ytZ9aAjnV_I3Xldu856J$2H_QZ*zi~G6TVJ zysRb9&C?`O>$A)#p0}?;r4I}fej@^8MgN+1I`*4(7_4P= z8&Aq9wx|N3I&6*CjK8CbF@rLYV%Ih!ENhoq&}wu*`e(*P+s6RVJrx&i4p7eR;Rph*`S>dJF()j+j1sb6dSL9F;#wDj3Np9f^f zsK1SQMfxko-O@*SPV&O^(`Z+0eX%%b3j67h8aV56LI=0B+99h}(LS3?+sj$>ASe0L zx=zXCR@@f!tMn3%aZDh7+3Q_%6NN3-4xv04rLKrW8~j-VE(mznD*o}$#~q&T-~Ond zm>lkY(IG7$?w(({*2B-Io1g#;w?D%kST1iuz0_>$0xxJDgFiohz23@Ci++E)Oa8nB ze^CGTJ<5OIKXcz-&&C7i8YW&Cdc3Ns-0N3wK3MZpH(AVx%AYeV)Njl? zCDPZ|)_KV6yF4{^Pn(xZE=)(3oPfhTEE1G>213Jyt{no>1zq>yP{`%2ngBL@#(wlf zu$DW4N}mC7AzibqPthGTdUP;omuE`&bc}fcL z%!=hHphh+$A91UuX&>J`6qSmW^23IH0!*6~(_z;>k=seM5pOW&WrVk{`7&^n3j!d6a}_l(+0Ka9DI}t$v{)N2VP^-=t%Izc?7M zR!6t+qD5AFfqtObsF@Ez+qNsIKhe+!QOklEw^Rbkel_d^S(ON@MyaS27Be}Ou%5P- z8DHkldSEryTo&*uO|R8@%hL@Di?TzjDHb%DKY#A=|MnzA-dmpTwlGSZ&B15q1Cz~y zcvuLx045fOpBjVL7HpR8_Jn#nIFb0gpU&)8HvW-&|9-STggL9KZpH zCt^!9jr3+ai&LACnTvcuPS=nJ# zBI?6z_zH380kyGNrQFGfK&=nR^^($Lq^XZY_ahR6NIENB&e2zC-9U33RYc=;+KZdf zFTTzVGD$2!3cAzXR^k)llUpwC158%G3UNO)DV!_S*mvoc8$GiJrE>90P z(Mi1H|9p0N3T*IyE!B0=n-Beb1>*OQKA2f*1U+KqRAO4|rOm|8X~aR;cN#q#v(_`Kz*<2a7vIF92~`0~_o9LI4S$N56Em#4Tb9X~5EjPx5v zRGiZ1e)QI~C3hUhaU92SoOQI8r}UYLDSlSA{ZUzK8S2YZ$8j9TaUACzn9I|&!h{89 z{T%`NVZFvz2Q0bcIF92uj^iw#wLA?g(r|BolI>7+51*HzXXK9KIF92ujIF92uUkF^DI*#Kwj^jAaSK-^Y{{ycbVg2okTI>J-002ovPDHLk FV1hKEJbnNG literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/GPT-3_1.png b/debug/accuracy_tools/msprobe/docs/img/GPT-3_1.png new file mode 100644 index 0000000000000000000000000000000000000000..d633249c2ccd9b365c825374fe5472f33dd812f4 GIT binary patch literal 7898 zcmYkBcQo5y_{Z&8qoRYFrA>`eHHw(ETSVIgl^wYHpq@ZB!`}b0! zpzoY1DEOdyn(C&ZcH1S&>GMMzF((3s+haW`Je~FrEMt1Fqj}CLzGwq>uh1lcs@OYw zTyr|9Q)9Y%e%Mp+JJ^wfgU$XJ3E%RpC?MKD9(N2jJK`@gFfN8H(ptpHOeG?plVO?P zY1GUPcZ^4@sY{P-PEE_lySAFDsDDWq+-te*G<6JOEE}?+h?ts_a&gcJzT2p#`oXZC zGxxfgMc? zFyl}?q=mNzDBzptW^<*6blbrbicOI~~n6WaF$lI44Fu?~$3VGI*F5p%)f@ zAlu$3)$I9cyfxxAll9l_r<2NL>F<})@05B3g$?8O7y~JV5(Xwug5Nj1}4OefBF2mO~Qim=M_*9FQP@NCVC?MRqE)niBqD? zymRu%D{HFnFng~iDgPTKxA}k0YZ+5yzv=8rX-S#V^U>t^?H%gjayssgPt#Ak*4Q(XM?)aW1<+@hQ}nHG;=(j-wK`@v{BNii`h1C=c1X6K zw&p-S#IVK%e3omS)iZdgPpv9TG5U1!Cbx8?HSdUXB7l^%V4tC?HxzE1d;;Oa^}5G` z^;I0pS3{jOhw_Py_P8h)^@ymmv0s*G!Ifr(3 z9hSVO33A`KGw2EdGikTj&^I(7SdP(nCiNkPu>~4T!DDD9tKhCFktF8M!kxQJ zp@}tg_3tbZaoJiQd&mD`WE1L*DH^zhl=iSxG3#JAN3Z)@X_H}rTyIf56LZPNMUlq- z=D)88hL3$jeY*?f@A6cH#g$Pc0fOC?(7i^BKFd7&YP+!j^!QcG+vT7BGu~%_K!-LB zCzvEU(IsR|v6IrRG7`;{E0|ifKquGxCi1pH+(Gu!KuR9zwP!TicNCS{cW0S(?Ik@k z(;rc^Fd~YN4-$DPIFuIf1(&imD7X8Dd+4xNp8uUp95#U!xRv6^#JZhE#!m<_Pdq8J zQM>kanTDLK$i>|h;n4Kir65><64Vkw8F~CBS6xSzacnf8h3Nr3@}nnYZM%_93uBorJ9&WEx#z+>_Uq1!L2yraC8 z#bcau&1M>Sk+kksTUZI4Co*MoZzx>;1&f^^w}Yd%^|1&M^|Eb947f$Q7YOzDdzWf3 zh!9QLzyFL2?_zjkyR>vmux<9L%b0WYAR$jHYEWwFN`);RD|tG{CWhdQj_-qQoQeFI z2t+lpzM6bXc{F~KJOQvxob*UU=!Bo_UuaKW&8BvR*ZjUkjy&XR%ikfH$|R;0G~64yg0L3Rz;5Pw4Vpt|X{5LwC*@BdMD+j4LfSR8EcB0THEt^YOH zGh9CA))$>f?Da}t0d>r4FK&be@+^U=4dDzojw*QPyf_kO^+&?sNgGn7u{(Z{T6)RW zC9*hl2TWS2nKJ7(`wx~e{1ATlMc&&xR;p8nCaKqA58pVfjmFF}=$;Iw9Hra>!@s(6 z>vXdXmH7PF!=r!AYh~PW7%&3ozi@uu>I58kVi^y z1$;oA<1Q!gz%lm(iR_1jrR-NjQVrd0v%kHLQtlg@znlrMLN-+I|K74M1PruXezZ>d z-c0R2=5%Z_cmFLWSzC!|I-}$?&_a(htcl8S;f(kx8IpNDMpUj0#!1tw<)6M8;kPJS zJ+-9wJ7Mg3yZ@N7a^R*>`oNWGL+~0!%1FnMMIVOyh@Ga_<>n#aeL=3>!%-#t=J97t zG}e-wjLCxlQ5HTEYvOD4V*Y+V_IEY#7H@jz6V`x#NQ*GG%mhToQi;L`ELh-!#okF! z!GON>_S(r`(;N3ViBupvJc&e#+n3_X=}9}PD^5|wJRQ9%&AQ89rO(scGD*_HcQWyX zB2XYy6u-2yd>a01zE@_A8kv8qi~7n^K0+-W3k+$tnr>H0a`J|e3p-mR2kU^HkZdEz zjNOXfqRj6VwW#s{!&;S{y0y|8Uz>eP5LHb_0W!Hf98N+yAJ+A2U3w}LYE(=Aw>768ZhND0t-?(1AkzKGG2X7{&moTelDAfxUv-=%m&Sjp zJh^Vg8-6>^4s~+Dns5p(ZQrZ7u`cHd@6|3a^cma6iq#K`xrj$g;5S}-(eIW99>qc4 zy{)Q>gbhhSVK$NSd{AuP+cd7|1!_4Fv>`c!G~In&b@4qU*Kd*cf?%{4CufnNL+FD& z-qpDrH3!L%vS=#_eb1{az*#ivwW4V}YgKaOGb=UVG`#8D4#b+N^2MzJ4>U7$D_V7p z7#x!hzXv;>k@gP7kTAB@D(Gol`JW2xzJgA{v1k}O247XzrsaD93 z;wWuC4E~w$U{#qdq_8#f$2hn}rHWs*VD=pnF3LSTsGr~LBW+>#dPMBQtIa1;YYe=g z=cba)X&*EU%Caa~4U7uS-#UVGM3|i7Qkl>Y%e#$=UQ>dwXFtM)|Os=#2=DIHZp3hZ7+&^4)Osp z2{ns!GvuW2f00M(8pB`#wokY@8A(yOkZsCp^Yg8S1) zdil*KH}I7_lFFc$iT$x>dfWQzZHA~^Y@;wA?eR#j($^LYW#MO9Yp4F^C(;+WysnY( zfKA^bIspTiKNaWY{Zqo(J^-!OwHZMY0+vK*;>N*KN6KeUIKOpvubG##&+swD2G4cd zEeCstMTAk7{A5;3+H7;9kEM(LqSDLKbHEemd41<(E04`2hQ!(fgC6`! z{#jU)VXjG+p6qm#o}50ZJ*^X0%B!^othlmCa~hHb&s7wwhJMiV3Q$e-keXDm52I^f z%45e=&-vGt1QWq^wcSyCAzq>UP9+E7e@h~qXs&{^&CLtxu9x%r9|Z4q&I<7U-0lDoOmU%A5%k(!?g1*vGW&EKs@FN|bKiXT9JS@laj zzNl2?lZZmMXq-LhP)73lf={%n8fNNC*tp>(3+somP)OO-)mK~Z4*}EcWS@`IeRSZ7 z&EiO7l5EZvjk+J#&!ZeZ8i@0n3qvA-)<;1J$0lL>br{2d2{}z*}~OVpDkd1Vs`7)C8`_v5R!&0~pO^LLjuHp9Il&m#O2OVk z^<01}+;f&vX7$N=c5CW}#v87TxFaWoR^3{|edx~PhMolOJQjeencI|h6zDIQ2&5eW zzG(3BhYf)F1|y@Ov!n!_JF)1nNd@`n1BbqfGjEHaRFtpX3~iE-p~5fD()C$N=R8$7 zA%8M9V~tT|5|<@`D9ZUu498-`@xU;_;mf<&@2?MwHeVb(j9ZJ}N|`twhcWR20$X>3 zQz1!N;51;^UTAHTc^^b9Mxzd zrMyZE?V>x}m_P@XdRNs;qq7erj?4my;S6z0Hx(vH&2pbZSD0PKxL8#Opd6CVMGkvO z3L2xJj?%RHrq-|Ps{;ZIX#GOOZRA>X!s$7;^5P7hxVEaai8Yg#*;&b0ql5AkBYuom ziTu)vzV5hxBAZU(w5WDe&ShDoUxK=N-m;mTi+dIv8l=qA7klRM|9lb(>^t|+BBMy0 zo&FALQ2gw(nHzGkY|ECfutyw zT(;GPry}U^9sl6@WYkQALL9OcedJD?JIxfDH0sLAi4m zbA0iyuD3=U_WVsdAoN!ne+q&tk|A{lo0RUbggqZ8f%x3KP`K`Bco{O2w>T*uf*uaX zGtf0h`L8{4Cy50YmxrM696g_$R6vaLUhm9 zZBQOHtu2TTU#hrVx(H-bd(gku35F*qMu$Zt?dlPf*>~0zqRCGgjP{%q=f5OwOu4!2 znDQ>0!d3%%B)5(u{FUqjD|UxtDs%W{3VB*0!>e6?9kor#8%?y}rFNvk_Ch0M)+Bcy z9WrkzkmbKv?jNhQyv>1`!{e_{kB-T zUv}tQal{1EC7S9%(-QBZg?TM7IQ{$nJDL$Sq|1-qyqe~#(rw5UXyCqE$1Zd?aSa7N z5QDsA3n5J|7G;El@ar)hfKNjBt=3l44~z@lB@w0kvlksVjx< z`OK8{Q`&mfDKn-y!}`_-#RoRzVILbiOWMjG_gf+B<6py(MaZ*Ly&IJsvUMXRQ=R#TgEtS?IsHb}HMU7nHwWA=eCcNufEHu2|M^R; zJ{0$5vv*)+C3Y52Iq_~I#0Gk}iK6H@w3F}a-@Y4ZV!)$F*g`oe#bq2+n0VVYCW?_(Q}~`^J#gu*i!tUV4$6H?|Q{NN0h(%MLPl6c~s(R~r*L zs$V#>iB?}44w$bW#Cs3)e$x)?ijs={^pp=Wl4^-iLYnSB4|q-c$SPfgbencoV}3gQ zFxf~ZyEPPyhsm4IvJ>~)wwsU6e{UmN;Y`KxUyAO%E_$gTi`IXvA=ev=xMt02r7idM zWj@NewzCQEA?<%=Y8HSYhgvqQ+X^2Qk8{a~hA>P6`*`nqa5LXs#xLO2BsEu}-($pE z>jRd^3*^0_UC3v!mx9CbLG?QoDsJ8mwKrCn5zTh5Jo9s^CeqDM8O@4xUNskLW6413 z_SEW+2UKQKGTARPzbxe?ZKuxgC!2K1k1S=Dk}o5jm9v7FqO^qc7){6C)_9BR@)2$j zO5^>zEy>OKUT$?!_9OC*u+M!i? z5!p64Peakk!#gB#e&v2vsb#FY{^*JawOAzrLO`@P{hmL;{kgby^)ZD1f=tFZmw)=Z zMkp*2Ice-!d{f-HphBtpCuA$*{Wl{C(Om*02^wA40ZFjlBD5tFoiLXvo1HSmpKB1-jarKTRJb59id zqD7EWO#eMa^aJ3h~4$>o4$UhV2VS)_q<6>PK<27Wxpc{GO%s zNKrSpBXsC*bBv2{cROX!y`7tr$Yhhp(w{ipPy_p>GUf32ebmuWIq?p37T7jyaV46% zaYht+IXXM!cgH^ahj6j2qJ^Jd#H1H^@yEmQ&7iTcaGzq?J9ods29Io!rDO2Ui(DjIv?DvkMenB$hV15Hm~@#2a12oL70WJY9{q5`L@kL+JSld z{W43sV(;sN+cRH;?@cIPbeFWk5$P%~Zdnm+%~kK-J4BmhEBK~BEUJUMIYMqSUYKZpz)D1GJR0!4LtMlCTbHE z_{!EYUZn=!GvdPEWY}xg3e_5u#G*fI72s6o_KVvQB_E3|fOck1y4~gq5 z-jMf8$LlS0OUh<+F8VhIfCdjl?Dwv8`KPJ?{5n5xIOPl?zb&uNZ36#uH0&t}B%OC%p151x+8eyzuKXF--r4BiE6Ui_yeOdfdBfBA!^AckT=FQPWhxeU~>>$14 z!cm(UtoWTO>^pm2HSV&{0#QTg+!5kdDx`OCvMD|wtGK)JJ;p5#JR#{jv37m^J>L&Eo9-e=Kk>Yh?#OkyBITaVS^8opY~r=gZH#pfSx7 zGz}W0m4N{K>{9MZ@06$Xx5i5(_rXh;jXbA&by4Nz z`qwtLAYQVB&@tFX_0}M@dxs-lLNTtrmc^j|Q2@lcMo66{ve$CbucT|J-lppSo&LW` zuI8~;@!4D0q$+tNgSqA0sQtp@tg|U7Dp%kQr#frZHA@}eN6+OyxL##PA3WYeKe<8= z%3`T3K$vZYa}D|(`m zPqs^VGe&YWyGx3`iBi51wa>ir#UIoHfBpY|o5j*g|D=Jp>WO9=Xp}fshIfQ&@N3>> zp4^E3@7SWwp6ZkbmapcnIh!*0yn0#ee2gIrQTSyb7uUu|P>5UNTMl5mJe%Zjz@=;9 z%U|LT`%&=XK2GwX&prH@wn2e@AYM<{=2O)>eXm9U7OLJ|5qzo-FeiGhj%F)2DVlLZoc`O_py)7Sm%>yU>((9@hN*a4nIrU%7 zJQb{WIs4bwq;dO+h`zv*fF1FFN5rN?g+h_I%We@XzQzB*a|i0;MC*C(6>-V@rbI10 zHbp7?dU1}!XwE`9i?e-UmUEG_@DHOm m33+S(2hnj$0vOC6`XxcK=6za`y1)0)6na{QnspEDBj z*@flh=kNcXH_o|p&U0tZGjpH0GxPbLd7`y7l_^M>NO5p*C{$Gxba8M9Rc@a-iScl7 za0|BIuirYj9=ghMIA6z@w{IN+TUiZR9Gu!jvMVdXTc5;L#mEB(hqCvdf;;F^`UVGw zaZ*)5_LYzMUbfV`pEA^49!bvGV=u~S1po8|krHj|VsOX|zCyyoM^QAf1S*>wQbF`6 zi>_zFNgSw5B8jZu@{jhuo?DFr1~Ql^B752SVv_YS_H{VteH#l|N zEu}jOD?Ll5550?Oc`f~OUsnU!%lj~3zt*?HZ>A_gJi$P?L{1ewpql9^23 z%SiVSR0XC;aKzSH-}79FPgeulb6NFO;6|a@r(AFufyX2L<^x;&Zeh87Dixki0GX=5 zR2&PGMPaELPT#!X^26!*Ut-^HsFV0864o$urwgmpjZf2`RnOOXUpf@U$z zK8sRgk-e1u9dtC+#NB*yO@>D#_+M5}U`syX7kh*E17_BQ4$EeVoN&|L40N>br=CC9 zUQ@^^&ZW%BQqX9K+eLt9CzFLojGnB+xVSaOM!mawNDtV_n?miP(-xVwMUCq-n1N{? z(QSU~?ueH&X;keQLTRJxq#lo*1Qg5k^^Z%63)(f-w`g{^bmxxu7x1`_QWgmjruj2g zqxwJ)C0`El^N`d6S3+#S7{6R8fe zhNxj5c_=JI$eK(|`EAv=uLeAa*BGc>UA_1tKIZ?2dWpUp7hnTX+_YiIdMcQ~0aYCF zSpmb+g$!Kv2Xgq0;JzQ0Gml!sQd$Z_lV=Th$>C0naMxr!-_0d*1k3XnnB4{GJSkPnWg|J$AriKbBo*@oL<2Q0<4ibb`kfUOs{AnPI-rAs zFDzjd2?Pzl&jJvAKermwdBPxTAZ3|g(oL}Fr=7Bf-ZA@i#m7kud0fCLWIzohvEL3_ zpLS?5{=N1Yk@-nFxedbl^&}4{;Yh6DNWRzyuVUuo%4nu%iF}e3?qlGlhjD%A{9!OY zC`C}S&{$zKKu}*7ma-)&PDt%oJxW-jhLKZ@Y`J8i@hyF;8e2QOocaw8kV4k>xE>}&e9%e?qYi}8XNa9y9n0<} ziLQZ1lkNbHzDxVDioP~aJK@P%lgGnyR?l@i4u|ep20of}FS{48BWkiLy0`{68Y_~& zpjTC)E~Z%+l?!Ng=F2v*HaJSVhyeu!gshb=$a!j8F{T9<5_7UNW^Kp5;~GK%S>cvRd$69#%~TuB`7elpc_HKqmavSkdF~-^*E6`Wabh}&$0E}QK=~LVjmYi(z6W?fy-92>mJ~E>AvD3EWCCYogtz&hx{D~qn z^Prmyj`+OLQAc&)vZvlykH&Rg6m~9-Cma+i#drTF*K?}F%#_iOS>B#YP6;J?5@=+3 ziBPsFv!p_m zma+0f!do(%4xZZO0-JcIcA9m4zW2`C`{MYF3Be*Wa^aJ3Jm`^Rx@!JH<6MZB#Qoz% zg1Z?J?4%@#*RU^AjJ^TAhPnHk+~;SZC$kFw1_m!8JhJLv9mf`w_)TDPq* zT%h%m>ReuI;lBQO`u@fwHMm|!aMxxzzcNTBYR*0QmzMG~ceYKcXTv>VSWe%j33TNk z*k1kqhrs>BMtt6t22%`)%FXbZ{L=OMsz@W^_ZL1z^I0SaDiw2F~@RvPwN|%p2=l$OYNe07N;c3O+%9b zXFQKBUS7EY*9bKfs4Rh%;jVT!vYjGybHH~6KW{R!S?!kNe@0wryynKtPfG_H&uE=L zCxd^ocqjDkg?p@h@@kU}{EY={<-vf)MeTEJ{E>BOHS$O*<^&wYb4D-?uJ~y#JI#+R z9rG&;w`^hSW0hf_XvXl_u}K}3Fv>%^UdJW)_Kr=x2`n~b8Oo8yps&zGLJaS|t~X)Qg_JoJeOd1f=S+4Zn()~smL*j6ynukpk>VMm_g%94!dzW3d>H?>=VQH9X}aaoGSh>1;#o{|TBy_;Q35fTqS*zD`=W7aH(o)w2#O zRq)NyiQEKjEYoV%SbDHk$E1Fb==OsWgDeK#-%di-drI?x!B6Y^e_A(20VkO=j`yx> z8MR0{t)KJtS0y3xdOxiXIFhq;e5lm@*b+#hs%0VBBA&I?%s^{ov_3dh}cf+h?EqCyA-2bY;{GPkWPkQ9n`VH*@Fw} z3^6GbziYEsGhSVd^7gz+<7lV1C%*rw0`LIba(~cuRNSe*HPmb)QwhW8budXEvDGki zJThlqMK?j0HACL+0*?w!jAf5wpy%M8rKHnAPZ&GLwh8Tl>t=(ECJ+KeF2ar>X=#At zPRm>tiVo)YdJzeqG$vEryt*h0pOzhX)J47yocp8nXKq`RjUq1k9c6Fm3JtHv`BGEU z?mV*AfAw@}9*JgN3H86ArDg5RW7bZQ-U;O{^jap=^xE7nF5;3NFx1pa{!n_#Wu_mT7+VO~&adoBa?2Z? zu7$61IXX@dYqE)S#3i7d_IOwXW}Viwg1GVsquucuuW^a7O#>k)e0#HZqnZGbd1%KC z;AXPoxD#}Pd7f!@)lI(xx%>!c1cRTD0~w$7o8qv&CsQ?G#ao3gmU})w#|F9AuQot!ko?ANkQWQMS)lXM57Tb2h?%x z6ADVghoDmjfJ2u@=L$k|M8+u)hhLX}KB~x2fAlPk^)v4HK%YS(S7`wg*a}sfrgZV} zsb`28w+r*bVK&Y65FRF)X%eegcet)hNTua!HE%VYqk(Ei#!qyI3_HJD*UB$Kz-QN) zveQg2->#RpBXQ-niL~!2Jw!(N>O!hax60H;)F$Tk(}X*JRg8otC}HmCk94HY&la|U z1EOBG#%XaJC^OT)up&Zazb;;$kPyzl0CvTFp9R99inkUNEl0LoFx){d&S2C@OgDLoZtWACV% zQL%tyBPFubZa2Cd_8;CNnbYoyK)8`n$UbkYDe9kD^LH?v*V+frD@T&dG&!dZEkvFD z)*FSfya+&GyV6i3TLV8%<#X~dqdU!Xo+2%|5XZ~j!?`;UlyFRS@4cSuj%iT2IrW3yq`)Ea#ME8z_lA5<=()v4}{#|d@ z#FvLnuDn%)mSj;TDtr$wkjfJRYq6+EyoW&Dl)UQqdtvuv=!uH?#P2{)ao^t`SomCX z6NbC8spaK#Kmep)d?FT6)i~oVhTObA{l1{?;kx?0$l$O?mm;b_p^wdAHwF>&VktHp z;dv>@3!?aoo1R7cyIzD&BB*uY%$(@GjBFGl&T19svE;^mR`n*XlPCQ@!l`4pIt^&MGLMeg?tC#z13Zwp z#aV*K)vq_eWMcKr4LX?>@NXh3@4(nGw||at>AQlfy~kgu{pvnVUy)*aCAwP=h$a8N z6BagXUbV2-dwEK8OXRi^={ej>4mcU%sF;Ya1>=eEhD|e=2ilOf--B*ku&Zj*o2=wdh?sY=Jwb{^LG7q|G z1~u0lgt3X!%vxBW=nqx`pZ^|T+|MY6T&-WJzVzfS&Kw6UJG0nwT#2D&iBnka5@aF- zdR|wyu{@I$^o6D~=Lnq-m0j2#LG#JVWR|l7GfrYpf>8lCE90Bc(+iAR(pfi|zj&*9 zzzNaIQrgKMHq{Wb&(F@)+rJC$9W?fxl>87>4@kwh#$#)q>1It*uaoVk|6*vpK{xur zYtZ}OfUK<`X7W;$Shn-5+vp<1;2bdB;W!e$G;dCf15<7(8T(2^nkrQcLi%U1#>k)L zsKh++%Ma*gk+EpN{2szSPyb$(RSaTuT;<0CVB*PTf1cdA4q9TffE(8rb=IQ**6*if6&>FjZ z-cBF7pShtg_HFWbi7ETq*E!bXrmSFwyn3%e&q{^X_(oZDg(!9-DHJUr9Bi5&dzRGr z>2~_ORK27wQK@K7q9&s+i<_dk@>BUGD@sQ-sD`Ur&I9AYP5tK_zTZqzuN~4;Ud1kY z@1P-30<=vUe>f~(gwSN=<>G92+QZ`VgQkF8`~@+bp!?TPR5hT6YBMC6*BVW;k`kC< z`!1T`-{q${ZtK>rG0kD|q>w%F86bZK??`o%ngUVq&p|nzyUbDkUJw{#Suat#P|_<0 zG0hanaLDn(l^pt`TfAp1p`o1ZLrVOGoHUkH6VI&P4%|UT1t$vTPQAerrMIBPxHSQz z?$W6rhL6rxpKmm9O2OQwDi7?S1Ho#19ePYRflKg&#fnX+!Lw9_)>u%3s#1FqO+TCa+gqp3)71g zdiGJ?8%E-GXMmGQmSG-O?S)=zwfWC{aOBo5pmnHHnFd>Fl^a;J!l&cK`ZH=t06Dt+ zCfvS)i^mc{$3C6lpL_hWsII2dAANMp8@qQ>(v&c}qrd!MoBAyL8V93 z^B!3r1ukqCXpd+^&Z$;(s#rr>iix%1MNi9im@y6?dQo6X`MmqV__3`P} z_}z}v^u~RT*?1-58g@3K9dAwiCG{-+OgLVohUsJz5eX+&HW?eF3UfN_{7G`h zkNG;A5$E0Bx2#R97^AmpUYo?2(pF>9FYaF#?RO3AQl%tinC7Z3LIg&#qA^3Q;RD9@ z=+C-QOFIP?^@wFn=?&wp+Ab76!?3vOfN06P$Hw6(ACu{GajT6ETCi_Rq z^Fr%7R9M@7!wP~ZCQ8R2fl@V}@7FTbO*bGWT;Mj)rlVmUuAs&;vHjcqMXYaq)KUbe zo??&?n%bEzzfGoCzTcpERb9tb3|XKt!{A3@@Kvwl#$rWlTaI@74|}xDy_Q!GU*620{`Ot!=4|h%Dz8;*~3b| zysd=J4R24bw3y@hItmWp+b)UHqUpceqyKa)I>PkKz0dA-eUR&`7J>uh>&LF_R`R7A zw$pGRuENMG3rIn2jVw*_+>@&LFj#uyoOZ9)GNV|SdUUPa{7$ujYOxT_mp(LA|MDB2 zv{LLWax{?JS(a3!k466xlKj40ai3wHlQSdG#+LqEr@+`E6EWn77JxF0(TFirST)vt zkBMQ)f7e-DRuk!-56&swGQcS^ZdV#q#{cH)GW%ouoGjU#pPO!^nE>V^5S1y_u?l{OV*56xG0 zUcX!E37km&o@`(zI-;~c-MpOwtXgfhO16a4x+uY zNomM09Zh|_*A?BDO6H$?R%)x3FoYnkEv*ET{t4X|vH-J4@9SLcR`0Z+tNTNq*RPn) z;m0cRb){dO9hZ=wme-?e9r&-<=EQy|X9PVM^c>FBw61jmxmB#ktTU zU4@UU9sdOF99C|Tfi9HXF8KPMuZa7`y98Xp6K}yjGTcWc5 zmJO&?2Cm&GQHA2K!dsM8mD^{mC_CbfE#$~lFG?aHRGt>w>*c;!b1AEH)Et2aG&I2PY(h{O(|{EE;~yt~S> zF3=Z46X#Wcw-HRTiG~tI=^ehVqDLmQwK_XPJu=COzoWY!4oV(0t=U-Ze(-0uq?g=r zKnjlu@0QPRWS^QO0z8~n-5j6^RUOi=Fw@-Fip%|1JAzr-{%5~W`lFwCftVdTi$JiP zF4=|mfHS84DQq09vjlYrIj9LH?d!b|n!dvGKA(v71_=GD;{Fvs5~_Fjk>7$> z#616Mp}&>3D_uRTIretXWfZ=psbNIrnp9T&9|N9G++nP9|3@+X$G9KxBRQrI&YL>l z=F%U|zh|`^m%0@Be5(QfW6VOA1t^#S)u{jyu@q=AC-vVA;$zE~g*la*nHT*JmIaaV zGJ;4j&r-*Q^3gxi?_aZpqD=mmyBT&A-u^#bX9>Gmt&qsN@_y~Sy)E-8=q&lhw#cP{ zu}u_0q;9FJZ-NH5vL0dulSA)vOR*6T9A6yVltZnW{`N5TPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>DA+bqBK~#8N?VUZ2 z8@UpI-$9((nSmEMc)@P4KnGqpbL;~M8#&%92oB$zLhu5cP4W(40egYD!2mwOy!s)F zRs8E_ORbS+>Vqw@i^Zy9k=^WWwKV^F=?gn5v&aH@)jF+S2(t99YFPfz?0$`7#&|F}(1C1q(VRU&~mkifB#HyN(wDz1fB5+t+^yoxf{ z0-_C?TF$V9b+|0A%;z6JfB*55%cO-s`AmJmO%RhD*SBDz5)$j=kQO4fnInKV0WJ$t zORvfPzod>i95Hb#I9NMZLe{&5TxVZS0S?-&Gb)^$y9xgRN4mn`GVu6#U`dosN1DuB z&$Y4%KrL+X=CnUj0h*%(lvy*>@@vq^K$++h6NiBk+OxTaYD2$3A2+6;zzw5)-g@~A zug#Fw&MvZU?_r>A*8CJ}K-v66gSQI~^AvWo9#5a{L&?&TZ^@1Af}4h09lK{r(1vbD zO}F%)DKjmG!|6A`otf7#xmm#P@ER61I3GFe;EbEZZ$9!H0p(0CwR$IKJ3tu^3e6q}l{;VA>>C53&Vumu?WQoXyQUa8AU~9Ki zaVwoF@4~x@Bm*Q_e}QKk7p)V=h$QGy>s0jPE0ZL~4kAM@5=LwSx(tXW%PT$FRobT7 zm@y&KL6ikRTbB5x6`g=hxzUR=quS1IVWECi$=KwCLzaV@^mkt`~s zN72DDS%xzIlQN@))(-M`Ovlc{5qrwDu?L9Q&t6$a6m=D#bpUvaRr$e)?E|17VrLAV6ofg?1)3R2|`+jxpUfrJl z`|tn3zp&xFCU|9b9Ovf2oq6_VwtU8%!SdGPyb(8cYG`j@oSn+pwLSj+<9jG=z=q>8 znnSphsGihpJQf*p?bBj5W4VR7_+e+}v+UgwPu2KyTYTlG66`1IA0K}CO0Rcs-hRbb z0(R!73HE2K{rLX={{2UO(u8XPJdd4r^!O}T4zHMQ437M&TJjjW8J~>MIy;CD+VYhu zScVu6IC3HK{rQ57z-L)dCV+ zBRMjdF*5SL6b=R(yg!@+cF@5ey*`Y$<5@cN6`-J#9A(+O$~bobYN1B-jf{w zn!per_Q6ycq44>~cU+mjn8i88$!)K1T(l0n!Wc;gLZ6ici2C@-Wh<}PVZ!Y|rO!Jd zqecM>VqfW`Y(y_;QtQOp;8)@Yl>L-*w!qBB58%#R43dG4M(aWM+fcC#2QRu^H^k|7 zmh`cuGmsyO%Ifd*ydPq~?;sV+=NcD6%ema_OV@9+m121eeMa?{z;?7Xx@P9$Gog=9 zXT~14gl2#F$XC*^vk`d2?J}FO+`?t!cb*H}nPGC^ByLUR0yt6M?eQuKE*hrYo?qDl z&oo_&=NS29N;hIzzBm*IN@P0Ac0!c zARUnaViR7uY~>X@02KHn3eS#+mA;wSIU%OUK{6P}OIyj=El5lka!z*C1`VK&ng!3m;kVY7HRA?ZK?DrMpVeBLOQ zK}R$RY|C=p3d`!k1Bh*dCS7A-8LL=BNq}e*U%9O1RXYq`7wvHef-=;CzUdz?qNUE;l^b4m=(HG?<6Lb1W)+$G%X=gjJk-TG#UKBA&1}9hJC5Txj=gZl4Le`o*ziobjn2agIOcF+JMHDB)hw(1X4l>Lru?qX;fZ-Ti5=e5>##63>H8A> zQ~H~=4cAHhWtrAB*<2s1^dH{!{bdOQ`^(4}9UdR`A-6gfdCX*g;C|PtIb)dcw&5ZF z^a8-2Pt)>6@mE-6r^?aX`&nWDY(VTbSJ98pc^9jK32?ki#sG|4Y+M`hGuBl~Ilc4 z`1a+y{IX0UGeVoe7op3(zcAV_ZbxNnC*=yuT-Z;AeRars0zntcoYV%Y%F* zUR{{M4t}z|i<31PY40OC9ur&D4wR~M!?uOX!C=2_hbY<-m21&f`~hkk`ag$@gL zYRmd}ac%zk%LD5h=s>_s_~;5gF-D!>nECWpmEo9?bW(rYZLvi;kbvZ&Z3h7*8jLhZ zI%XMN+Trt<_hcutS@j#+qG97pfz1dg8^06}Yg))u$Jaw9Wd9o8pFO#c$`5?u(8=gE|BG zH+>KIXUvY{IF55g@WMPD58+&b*MEMU_DB4Ii>dsn$2Gv2Lnw{2Fv#0aWGB!Br1T8IJ z6n{bAPJfKTttVn(JfqCUuVbh%nE$iH>^ROA$Q5(En?3F2GCUrX_06A^r0e}1+6R<< zq*4wC8s)vw78f4*G}NLFc6B?v&3(6iQ_N_DKEQwl67OyhDF)WJHmiQMe%427iW1!~ z?H4vBo|YXfb0ND8NF`aAt9Ag1xo1C?H9dVURN@gZKl+R(O_fO2HYwmaOT2|;;tzxv zfFd?PIqJ@_&=x;XB8|44Mq8rtnN}=*&+|8(6X~Kx0|8kIal8u$JvOmFi!{< z*YkuwlimC}Q;F7J_;sn<^_|xr#H)vAGyd%s(#7@}q!Dk>6N>@aPHB@MP)e`YQSh z8@|8qsFlB(v^gpd>%f?IkbX3Cj8AmWv)ZrTv1@);=J4b^oWjfwusF8R_a*wL^fzxS zPixnI`HNYzlg};8E64g2PM#+i^R@0TBV&~Qk`z-Jiy~&Uf7ykZeqf{WP%Zvxa4TOD zf5{>{RqUd;-cEq#XZxFl_3W@yj$|lj3aTzu~v^#n=Ol_RA7( zd%!=~4qnja=JALeCOZAXPSB_Na&nv8HsaBCpnyqwyr7zJ7-Pn6E$KyM<{E$2_3h+CHM^lJeWMGe1UP zXMz{{nT@s`-B$M(OFI9NR{VN?n!cnM45Tuc{}{7SF3fV2i5m48I@X}XiLhU8{pF7>(m@IfuZ?JMRGSVYH6sIpF$>2IKD8>VR*ctM?2HS8cE zy)=-3$U^~*OmIwwWKNx&6RJs|K3;{dKexquuy5x4*qV0 ztZmUqb04Y4-?Y=18@4Sha>0Jv4wJVfbp9i)_yg27^aar|7}n%p3c{%iGl)kP=CAkf z@k+Q{n7L3a%*aPsk6R0~ngj96!h8;u>G}>=!8w-4( z5ra^n^KWc6{3iX(mk{NM@vCg=33RD=SZ7|}_-(|KP5Km@*4J@r9iyLw1k8aT(fV4R zFnw>t!=RUGw}aZl@mPHHwIGKD@4kG;o7nMIc4c8!eJt}ysIv9z`%AvRwCqqB3Ns6{ z*jX}fjgIcGW=%W-)P{q3q-mc6RJjFe`-mE&YW#(r`rNR6fwu9ap=HOkReq7$!i-eo zug`N6gJwMc$ycJYFq=zwT?U{`iNI5pAR-<EI_DeJ2 z4V|zCJDE+`D*ftPH`-T1-=eX8Q#^VCZ68mP!(>&rcQX5emg`%2Np9@`Q0Cb59M*B_ z`2&c)?mOGZ5+OR7Tu8ZU$3yU8tN8(Qy^n41R`y@?w=3EUv+pm=dG?pq80r1U&H*eT zXhH9H&3P;KqYf+R{%S_!vEVr(m#}TpNpi|tNMkvd!iI`Jjgj@+vZKd{ZA$zAmucAv zt4}|kpr7@Z=a_}{!-k4K*@=B2=OzZtX#RD~0x!%+Zhj&3H#^JVE=L@&EvcQ-oWio0 z=}fIM@aN|m;yKqij@sE~{-j*B?el)Oh4&uY^w;A1i>0Nn<>TYBzQ5zX>THpRQ`p-6 zxzO=6V#9^maU92SoGT$L%&+;ow$(cc*AL$+S;&gdQhGP)`n1^qc4A`&W93qlgc^<`Wy}qM|9Y1wq;3qQKAk$Ejxhp z{fLqz4>5z0X%EL?e&~a0!{<6!p29N`#%9AW>+|Q1nXiCdn6+O(a+v(u@dCs6xB@*~ zkbDt(VGedO9Tl^-uXU2#qV*TX6N#sQ{05i>kFfIrh|$+|Xl0 zec|(@jvd`L`6cX$5z~ynKF@Uw7WTzCXCqt&{Z{@ZX2)@^5W6r#;b@*BLy3AuZY|;X z3*e6^0Lng6EAt^qd#|?*M;Y}XyPytsbvs1mqy&jElMy(;;Q|QRjJBwv`}WPRH0 zdB5db+4``x4m>mj=qe#zb@3}7GgO(cZ-S2U-fRn1a2(^3fi4=DmK`8e*@IV#IUX^2 zf#mC$1L>wKmCR<{Z_uw^y#=Biv3^w!jo*|jdmzrPkka^yolHl?oa|d2<=3F(OXErG z4w8SYgSMcFNCE7FwU7=SKj^0m>@0pBqdfka9WzF19Q7EHacIIG8g@+EtiO611e)p~- z^KcqdT;D7e9#cTd5+6B1G9g9tv+wcDV`+xE;h#bdx1>nM5 zeKMtod@u&b7XP!4Cx9{UApK|tu11$G;H&-WE2o;@l{q{)52rA*FfWcR^nHo`DgDjc z$}b7yXUxbtAbOGwpe?!yn(v4;|CAqwh*g zV_&Da*f{?B?e`zw`ERW|m%!JTOS0X51UoeE*_$HqZ!~Ruy|__uK0NElUCD9m#5_Ff z{MQ%E{Z!6<<8l`8P`W_jF-%R^K<$9TnW(eQzqNWfokQmFn6wra4tKe%=m?BTZ{Vx4o7@n z3|Kn*L8Hn^7tGH!46XUL88aQ-Iv(X7i?heulev6vQn=i;0mNIMUU zfB-g}|1;g2y z-hRECo*q76G03O8uWzyZ@%`Q1dy<0u!~NZR*7P@2sUCrBMpy%6kQTCp)s&)8gGwSplAPwbRP^Gk#lh00``3KGEVh@4V7=1c zh;13G@twrbh}Bb^qC_P5wdbd%Px6L+Wf2-kha2?vBV~UkKdpF}T0b-UN*`;=d`7Y! z5`FvBM8e061XakV`Y9uzVyCNLBl*>N8_$mds#7hT3*q)2-YFAvCr9mglt)l90WIf` z-+uh_pD$dV{{7!?;UXC?n1BBM-#^~r2FQPY{_h`DF-sO%cYgZ%l7Lo9gwgbmGd3fv z0WwGnS;A^cQK&&Bks(R0=T$$>S{y8`^w-)C=I8rmvAs+L>xewqtxE6ip~=tf#&mjx}rq+VR)z)b-o6?VmS48OA~8EwEsk|)DVrcL#x08_ZcshQej1~3q1=HG03yL7VAHyY>ZzGAF#8EGT#-(9 zbP-#*v<|$#dzhY{KE;2m&#bmDa^UUV{qz8r*-xJy?&H;aVEBSLOCJHlih~@f=2=)M zH{((pUtq{Df_0?TFj!K|2Vx|i1hi77di@28Zb&4t5ITcz7Pix|--%Pa81mqdmi>-j zvL1;#nZ6*A!~!ZNQ$;foM33Pe<(AnvpDjN%M(vE+2bSzd+Rmsx0GqlN=;?QY&YDH) z_AO!(mP`CN2}R*2nuXF1(9thx^=mXg2B_v*2!)@}hD99j+(JZ$eoX7)sRP%>KR<_m z>eJ(2zy0>tKX7^c`Nu!M{TXd!pD=&<{`s$Obn)CL3yWAX--Ly7)89tthx{T~M_P-r ziGX6o+DJSJXr)Z``U@1@kVs-7bOzrnY^P(t6Q_7F?eL*6L1yoF? zie{tnh&)QddrLn7BIB7@91jDVnUUEznOS*@)%K$#ypQ@tG7|;$3lGbT->jM#T1U`; zh3&Weu861A|0TyU;}<&wug3n0c!vBYK$KgmGrEp1iAQMVp+&1-v+^hj?=Ag6R$XV+ z1v5!Ooq{tvOiPYnJk}IzLt(0Y!i=BaKHbO9ikZPrnAIo1q;D>mOZ)8$Qv zM}yPi@))f>+)+qNs3n?YoDY3xg)>*X6W@@tI7*eBy4!KT1Xjw zs5k(T)>ol%4QqpEx!0EDCLXfgigTd9x&5Hg?N^0NUtsHD1JR*XtcsmjCpv72wQoK@ zqL1>LeR3Op74cUfi?#k{@ke<>zZnlv%Ufn&Rg(*=%rq(@wrPs@)2OeVi^!w5Un;~{Y<8u=h_wfrZ4jQr*uToubj@~R7Fh^dHzWZX|1CwG9y z25N#%Q{L%s&F#2f0xN;iT%G#3kHr(Qtt}jJAQ3J&> zHYFFo(N=gzd97awP|O34HWYn3x-$_Bw2Z%Pr>@_op^xHF%>Ay*PYrB*FiBF!?cmoW zl}_f_ON8R**r=$~V~mQ?btKrACUHNS$^YVbkgAx3+_sv3Mw+U~(W|nDn*;HXtnRno z-$>0^$#DJ~@o>MOhBlxk>Ld%SV2st^C{WGVSFQ6rP|Oq;NWyzJKT&sv<=aUlDqjlxTM7>%z$@9w?Ji_yXT-!X}KAxML!V>gJ`2j1}eat!sH#xoqzP{W#+jeDq ze$?Z+@I1TN_+GAGOM7!0^Wpgzi|6`}kIzdxKg;th7tD^c9++@|{XPQA3a;vaw4R9{ ziog!}{hkWflb`W@FyC1!_ zbz!W(*Ea6YHUG!(3uXtvKcI5+$H!X%jtn=w8cF;cEl9VXpH~achvyx+>o|^`mI?XTa)4SbR!7pl^{~)$+d4Y5diJ!}Xl~OnJ^G4pwY%r|R^M<4?pL=~v~V+L&K3(#d-P)?1SHXguheDEzBHE6;@KBvAP16a^uGSgZT1%}5`P z`@J$%jdMkRp-t&`Nxw9#A2t-d{zgD-B*|;)Z)RVbpqzdH4|4!zfO0|l(3~yh`7HNe zjgF^f2jv!>=TMi&-;Af!@3D1$YGn4a%}73y6^u&VjJ1;X3LPf4v?M#V{ixaB;dp2p zbD8f)&Hb^qAE7)LrLIUo`03{SSXL<&|fZMEcGGS%yEG!{09LK4s1tl5OMn6}tZBTlXi8$zbUSfOn7`&E+Z z3q>TczeG_x9FNGOB)l6pxm5y0MZh4-gnc`uex@N?O*;+!CLR0d%})i^>gmqgF&bIa zSefTDfsV7gAc_Q4-$!`GL*6UykEAj^ivsE|_sSRmjR39~Hx|FsH>ANYA|D zhcLG8@?_^05%XZKc{H^4C>_#}E2*`Uos|T}{US651LrhiN;MjdClYO8P3uGrsOA~t zHbQmgX8NmU391CCil-mv^8RL)LVc%SRbq?@Zr$l`<{}}s>?6u5JE^~7Bt{tCNj~>T zDidR<#?!Nd)bd=5zp%rqMAWzA+oon8I9q-?Q0i=iY1UTkRP-?>ajR=2REwUSnIWK> zf0c>2ANBe>7Eem0rM^FAMx-gq#faHZmWK0_`JEZxm?q`A!f5{M@lcH(mmuurp#`e3 znN8-uG;tiqaU5ry>=WkO79*){?>2Dn;@GRqd zfqo6Kbv&c?Z#_TP7XP);%8%#GaU92SPUA&fFguRpIL-z6`|tl%|AlJDaU92SUJCk= z?Bnr3Jp1$2%-g$%PxoKL#s-x3xAy34gnxwM+%ED+vvWD7Juh|#{@FgmH1Y3&;a_!T zNtNt6S^kA;$8j9Tab61gg1LXmF(IM6is}=`D2(5Z#>*ac=6LHD%x~i=>nWQ zzN5@^bnAGOdzysDW7vXeh(CUP_G_5S_qTN)VNri)!aYBi&(D1wMI6EF|}003fjHD!GO05|!bXTy5{0AK;g z518)-tk?Rgih$Zlru};X$5BB`0RU)7A-uE3y_fMl)J$Ik03`i?FRWqrk9Gh6y@R^4 zf`OkUqEI$@^(k5CYqHjSXqW+y{$mvYHKXh0l5}XA%b`WDTCQAF8eqTWnv@J1gByg6 zE!~Gc-L$=;JFxziG7+@0JntdJM8XqS&T+lq0+ncw&`8?LX&g!%u*Q1X*Xbqod-J>@ zu&X^dC@`KR2pE1T?QOe?5PQHag^2^UIhcQd@jN#`KIH-Q-F3b!{E1 zQHr?j9rkD*Nr$40!GH`mE6^_rUSR1inYw9gu)Wsr`^F==LE}Bua&9ZN6a5dKgsKzR zA5cXEiK}n^`o**7i{`1ElNmUYy~KYaIqLD-GPlpaz0Oj*wOB)554RVGY)-im)GzRe zg7`_+`WV^=hGoi<#g>JNu=12X%}F&qBm#>6L` zoGx%2DgC0PInkZL;b&;(Ii1Tu1Fl8J9LRh5D4&UZAs?@m$6>9&!o41rCrzn`a6azI z1q6xg%SV)JA+;OC-dqj}43@fuv$_u`?ik{+U()I%vy|$`Hbv)I1w|Xx^EhdI%%iNv zuagJ*cm$|+*M9ht6?LDPYZ2qIwSNFu^ydy@|`9uC@sLR?=9z zmJB=-_sKWWnigI_iIJ_;q2Q5&REMR-nVb*QGoAA>0? zptgWID-*-v1f=9?tERMDmox!0&0&o>VeSZ;*^!lNn;W8&VgfP`_4qojMt!20xj> zndKXf7(`I+$#(|()dFR$;>WW*-)33%3M_=5k6J88;)_rdC%22P|y`6INa zAKSFs@i3cMWMSdY%M6l`{Ixi2oE1Zp#xz&m-^S;dPf-g@Uhfsoy=YdI1=NoBb3JaQ z#@GAfqZ01EreBThyXnjJ$=g(L@^Nye*U9oQ8?ExCI{~qcQyc~B{;cNvGhJc@{#7sA z72^N62kaX7dE4t{Cu@PaU`BEsE`L4KU9_x;x9p7<4GE1$wbV~W#eN}6-U^dZiq{hTLbVi6^?$b@ zBTAxMVE%9)tri7F`=8~?$#Y-#~m&Lzo|FBJ6L@A_1wDsCwm_dTE`6im6YlQv!7jUtgbK}Yw;R8rC@wE&& z{}5P9_loAA7}*6s`w8uJgUr?ia%|javTzE16^zXLO?#X?ho!i7C0LrPoU^RQ8ugL- zcZ@O1uMsCHU`Vd^v~|#n=cH+<5n)mq0GWq^=415jn<+5t2dv4gvmB>e#7kUERzGxN zdMRu%ExbKD;p8y6`}Q6C;G7)nM;egFDr2`GOEtdpOTbtPlbCt(WPLn?*KP&%g~EYf zh%k^Z`vc``6;qlAtmL7MwPyi@_(q{islS@B_oE9qSbGSpOELd0P>nJW4o zVAkVGIfm6u?g6{D)}IDmJwGA2fG)UQt*$gIxjjUM;Z|K%iws*xZLC@n`y+^i7D;Y{ zq<*uY*BvD9sv5qX!`D-a8aKWPKNTTSZ5X=cXSC~5FD-tLAV&3&0h6j-uizAQH59a# zxI5=${FzeAFyTn@Ta_Pvl-{zJ76z)#f`1y6FUr7ji6>la*iG;AF)x0cz_tzKM=;?n zg}$IDjpWcUbYj_3$i7lk8uIvD+Co(t3t=LRIEYPWB^~%Q$-B74CM4Oo%Y8{sB+b_XET_MrnuG*$Do&AN0Z8Z9NzownYAm4DnsTRr0!E&Q%Fk8#t z&}nA+RL1qIo11GKxoK_%r`==EB~-Km+u+ve-F~ zQsqu%l>>slMsaC!4pf)io;Ba9Wm~n+UGIVFgo#J5xatNSsBX_qajzb#7S{GZT9pPr zdEBK{WSCgG>N++ytdr1T4P~`r`Z}+fcp`noO6Y};{>2{1WQ*XFg7xV=cdShz=V*a2 zO)b;81oI;_nf5Zi%U0IDkkc60q7##)v~8!GEyi~XK~Yyw^`y@EN9>Y_aQMEA`WtR^ zr<)GcE0S`ucjp}+98CU!;GJ8fDeeTTUvt*|v;b$U`>R@v;ctcK!m=dGzaZ^bccAu; z+wYsEUKJG?cl@eJEg11;bxc?yI5bXz5&)Ffk$--#J_#agElGVe5!m9oNjGW z^K=@0n)Ev*8ca*S+po!IaABsVMVE?wQRFE1l}l@fGnjhSnB z&$W}b8a$=qi#oq9aPC(uF;YZSdMf^DgvA`6`02`Wdt*QO3+|Cv*8&lWTi8J0iCPx$B|*2-9W9 zaR;hxQYg4+?Li;s3;&vPq~6N4xw<+r945WlLH`7bm7|!)C{`?+`7}nI>*!eXfIU&I ztJ9>os%@wAi9Pi8ol!yD&)bf^<7&QnbsSe;IX`w|((-XEjakmVGxRj(g3XEVhn^-4 z%^BBdd(yn4T&+g>8TtMmT1__eL~FFH_*oF=%ktyzU7Rs8+0;JzZ7etCw@P1t%q2;r zZKr6wbaoXVzQL~quHB)ui^GjKQi-e1ozG&etBr?V90{62Vk9W6F)zF$r)In6s(an85>hR@<09dYB|l`wlgxZTlKtKezJ^| z$AWgaAjyN_toiCuvEe{99CY*EE2wri#ytmpvO8z|S;;HtH>b1u^Kss*6K0BrPU7So z!>gFxvb1>To=Y4e02H5t_Z6vBvDy;r1yH^=e~;cRc+vmhV(K!GLd=0$IM3Vox57>0 z{OlUdb;nkKsU+1vzdg2%@*;w4>le>ql6(d^U6dbHgx(0=eHUu@y9@2PDeJPWavTo2 z9h2@qQ)o37YPzAE@fJgwff|xIQ@YB$Rv-}Q}qButJJ)LS+u)?Rb zZUy+A50pv{bO77zAbQ=&r+*qf`4%p5Yf>0|^K296u6CcgANn5HYZnv)En~(EupzfN?g8eU9izYGG*Tjx=cLPwhy?KNZ`xBcpx6XdY zE%%6GjyQMGab6ah%?}{AZv_eEws=137F8y4PiMk$-c=h;fA61&$T!46d2^DFZ9IjK zKF`rydn&)-!5ijijx(Wz>L`LpTd`@?oXC`Px|oiH_YPG?l08}{*=(F8rwP93&r7kP zbL3Y1%Jh0x$8GVDIyf`MA%#iKxpM|I8y3zdodo8N>6H9c5&T%~_Tzz$4HxonB--3j z-K_cE1hByh2xW3D$b#oktZO8sybJ6sRH=(qwmge(ZEkTXw|_{Ikh^uO9VRuDG^ReA zsIHqKUaFYU{&8iOudh0v&$6S0imOH%I|?b-(v5|4Qczfds@pu&Fbds`g7x4v)i#W{ ztU8F}R?z&X7+olvMT89(Pkys833kjux5(sQpvM-p&Lr{OuI-~S?ovq}o9-T)f%Y{v z=g22r<%zk5n|uV0S+yyOf#yNiS1V;=2Tkz}XLm)b>Ug$J(ta_yXoA#{#NFDuFPb`=Qp*%E^$&!s_Tt z&Pm8@CjzhTbE1?_MW*wJPEF3>{7kE;bYf|``Y@y`&6b#aotw;;TP$pp?OU2p>cql@ z{)1u@DXkk$iJ9K8y@<-Vm}wj8&_A|t19PG&JqW2oCwPhag^2pER%4sup%EGNLf7GO z!a8nF@j+-osee>oJo>8HEWU($Gj@b43FM5QYvufNln7FTt3v&8V7J>xbAz<^Jd=6V zy(J-{57}RgdSByp4Ec4gV_kKsc0Ab;TQXz(=S=C0G#zxwc=;D%iB~ruoG_aBZ|7gU z?vwgw>o%RMku@FdL>&YZ4dxQLR~fh(d74)5(?2;IlflZO?nqtrl7V%QN;oBef7_l# z80&H=*|EJ|an`#y{*?9L{E zI&vgaa?Fzt#qM%?CEl$4ffbLT+iqGbEJr52SON1Od+oxuH&h?rjQdHQ!vX{*#bHl{ zCx6AA;EMhZdY{stw(T#JYcBl>OyB8Af+lU2N-u+pKJ+KwVS&IaxV1uBu1Ia^t1V=RYXKOOv&(+zHzY&i*%67{xjsS6IcM|gc7?UAh82IM_wK93 zyDjMm?dBZLsgM%AwoE7be$&^xz8&~RYXFVvFQk0%hExwtw<}4i2pPe0tAt-TPc2z^ z>ySzcs8(GHaQ`?ihIY~+>ZdWCpbopVNPuDm|BlmMR^C&_Q+4VWnB*$S!{M%AO(1EH zNl3j*$;a+5*sKZkxdzxBFh?OjL{kB~Ww4b{++CZzn-d2yyKEB^|4wu35S`)%CM zRHk3OT6x%t*P^^uKEM@<<*Dxx9sXe8vUEu#o&fF#XiEH;erGV`n8~^QKHHV-)3(-y zad81^M=aGKZ0l2EvFEvsNBoyBNdz@8rbALAEvoaS+d=k-XGgYW+#V!X7Auy{oS(M@ zr64Q~i>~0L3nlW*rAfh3$Vy!rG@@HHV$y)=VJSEdm{~C*)!u=% za-GY#w2M+MgK{?OgC}jjWja-~fGQC{1gV=X-ziEBjNv`Sam>u4U}88KW9!K`nh$K?| zqd^ugbJ5XdiuLv8#eUoNLX1Hz`FY%p{vhBs?#>@`u~ln!9$CfP7;8UZp{0}N7$G|2 zVyu3)ly89bqEX`Q*A4EF2P_oj)AaB44jd_#2jIFyI_3pzQ3irg=RcX6B>76KRm;b2 z#?ikrarV?#EOi{Dmyv(&3ggW4o^oXL|D=bD4r#|?&)t{qK?TQs=kEL}@A}j8)$$ZU zg@LH*vg*S6^ES?AsOnPPyLnj|d5V#&Hmd{zM>x$&eQK$Z0oay6z18fqxcDu(N|8AC ze}juqe7VH9s{>G0x*5z+x1vl?5|)LKwe3cO`(Qkm6qiMY8D<>MN~QS3SADHSCDUCZ z`4^H)>SpV~?m=G4#P15l7@Ho#6g`cC&fCgnDd*|@Ek8DEygGIY361A=)a|?@m#Klq zgTprh*jZliFDZ}?4S+co&5?i^_`CG)_~ru8BK;$dqfSZrGP?&0RlEaxCz3x<6b}h7 zftO*W0IIv|BqNTlpz5(zyRBciX0n>=Pi{`0sW6qB!gu|hHKf{uNtQBKO)cx3XEfu{Eiflzsy6W zP7Uy@DPR?+xwL7vl8vU}d@>7=R#ryVuaAn*FNBIdfaGnk!5FM>vzwQIF}k=8+iz1<=Y+ zGfV!2AiW8ga#|JaCnGai@~7<0@$wfBjNuXw7XDP5-B1E%QemviIjbxL_zXR4l`Cp4 z!q5iRL7ho&#xTt|^(gDA_1bhzu&y6`RfDSS0}9_u(DPVbyv^`|a#TOdSfrb58LXVV z#XvT`>qS4h16kQ)#=j1ejeGu6+~FrOkw{iB^rM5d$z9BK1{JdYQ})BR<2Gy+R9P8| z6x-N$)@g#e-yJEch+;>}o9UP^d0=r;GI0`y%EJ1Ec+qd21D&ng*!lkLobT5GeNRZd z*55Z92yZC`(j;5H>&0H+(H9!--3Gi|fREafKTRGp`2_e(m?$nyQK#r-Xm4B7^TVS=FkSTUkCsAw{RXUqwPc(m!hyh#zj0<60bJzo%5}9M`tca_75jx z>SNnT4ax8MhP^oN>-6S>5Xj2?j1HLfdndPGrt)8ib0n6Ydk_Pftoh3VJc3){n_`-N zGX(L*^p&ZGb43lLe^@KIiU*1R>{&P!QJMti232Es{rzPLj*;V)e9iV`>%O8y>S7%zWA z_35p2?*f&yjHuExaWoqR4D8bUrT{FQ$#aep7Nv+Uc}k})5m`2a{C0rYhk!Wse4-kV zb+<~=jgW$R>PN%|wxS7%7+pJ(zD`E;L$XJfI;`NNxtLQd$hlFbOU?CQ#wrVi3>xS4bf#lOpYUBKkKJa=*YHu(0H);Tx_vf7v(Z``86yzw1W@a^r5|?7@!^=xcN? zx`?^JRomcJGUe9ZMU1Ar2{J2Q`}V|@Wjpgr25=L{_Wo1g$3gL!#J`zyA`cd-FjzON XtS<6`^!fMSFaUKGZRJ`;tMLB;gdD?T literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/GPT-3_5.png b/debug/accuracy_tools/msprobe/docs/img/GPT-3_5.png new file mode 100644 index 0000000000000000000000000000000000000000..5bf1c26d3d8734b14e003dcf1b26361ed300217d GIT binary patch literal 4676 zcmV-K61(k*P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D5!p#ZK~#8N?VM|j zWJMK#hZu-i-H;#;#ffw<8D2Z?u!zE_S;@H9#bH@Cv_A}ClX16Qrnh%yGt*=-S>hiB z6Koenz#uB|k0g?IS5|)!9|=*^>15TvN|1mtam*44Ka2_SoI3S5Rk!MP-=6NC2j|Of zcimH`PMtbcx2o=AdTd-69}lC@%EmLDTobFEmQKYe&dt-GOVLtM;c&-7FWwI%=tSS9 zilP8U;UA2SEsx9#h(v*fV0E?Ksz-)Ci8}2zogI)+f+Fhr8v(OAW#G9)UMC7yv+1BL z9Ck|1yWS{h$3n2>NuCI+tE=j>(@}{njLZcR0zrY}z;1%*8$M~GZNWs)#u+>{O@h4M z?&t%T&QJPI0yZgO>%wBK3??5)^W3bZEjxL=4f|};BsrzEwlBgD3_O8BfPI9)f(2fEQU+ZhSP3}+pA=RnS6yDGT==id zk;KaRAOLt>@Cl`0^Z+~;60q5@X#n;dV23odypER)lpAYB9g1MulqEo7YwXPq!X;cz zWwnH-6($`7K-Uh@Y}?HjOXq88!%-*V2}#Ds8&&B0-KST$p|;ZwDmmvW?+ z1DXZ~3S6)0RE$(^TpnvoF>AMjLF`6Kr?~AQ)Omx3Z00@bYfn(V1B8vt7%UGo)#Hq# z((P5Y{6wg?T6U7bJIs*osOh%sqf=pnFoxY}W`0VmK@cuu7R(J-8kFd4VG}ZdOGF$H z=2O^=3}6rlg0KY`&|FaW9(NVc&$^NI@Zv`|owcl2sj{*L4uzh_(gq~Md!q^W`)XHs z|1V&mv7))4PS<&uYU4w1b<5XF1NK=VCbGJ^KCD|E`UL3i7f*u@oT*tIuR}o0Z32`I z>H=80PR;@$xEjL(EVhsn?Tf$%3!jg8y-HyHkGF9+{x~_0kM=V+c6q%ie}Xt$ z-?%?;Wc^g=wgXXpx))M!>uY1H6UZ&xx0F9wmcf9BmIeC^Mf<|{!J*{}UN3xl{>R!lM1QC{79Z*_Yv01_P5BeX+4#o&fg|fD zVA~D^`qXwBpEhDvCy-mXZw3CqhaGzszg_V-aAr<6;1iz;*?qym)BNQzAZ3&Hb!vR3 zkoyIr1mAS@m#_6@ApXFrVDO(G@Og@n)K8JHGi(AsU98s0pVE}|Q_Tgq$U{4^V_+^J za1LQ|on@YH_d{+CDF(JWIwc6g@S%M{us+~6SP{;JaM*355rOu5dh0@Sl})?-jMMQF zf*_0p+85xfa(2ykujl!4wPmo^=cT|orPWfWH(cnz^}I^^b3q&I>-i{C7j1$dj1akB zAPB?d?Ac%2e_FI5k-0z+1VrWnK@bp`3j{$x1~V7*_@KDIn}YQoUP11nvzD8Ysj_l| zb2#)mmNq0=mK=)xv}^aNDV>Tl*jU-Qpxci&*yoJkfqMRDU$+A2WeoOW6Hhbv`IEJ< z%B-dAX6gw-hL12jIleX3)~`%39WTvw@+iOiJZ}rjM&wGr-tMR;FP%=$&s& zjgROTFtZk)JDd+$3qDriSiMuMa-Gd+S$fY@3BtB$eaM6{QEx*z$lx|f-$Vl>+NO(b zsxN#W9R6Iu>jkX;sWy`R!TRImJU+%x-ZccTm-It_f;cnZxSab5=(YoazHWP6Ib?oh z$?5oVhVw1$555-+E#)(kf|n)bd>j~!>4Wf@;ViKG)a;WDNLTUGBW{!Tb!vR3p}D}U z@V;HoZ_{SzZur1jAQ5XDdSFoAXdNV{ZxZ-N9q?((a)WkXz;r_W#J<2xD5-X-G$eHF z2ZLF64GsF63nNE(y?E^T4+X-BX~a_5^bcYY$K?EuqP7Ihn869p;C@!HGfmV&mJvMu`ADO@9!+)_}MF|%^M zx^kX&0W)XLRn?y-*?mFv4E~*xBV|*iAwhlZ*TrIzz@H1CB$P072~UG*`liN5_62H# z1fQ7A)5aTg&6}$-<0Vkn=Qc{X95LrP*Wqcz+ZUKLMuhO=;(Xw-BRQB-x~Z~o9%33T zc=N8)H-m%DNp6$$$$u2+G^#2aNW3q?el-|5;@2DaA8q6C{6V=FA1Y<+JG^@1{`hg` zzHvGC)3qH4^r`J?e8L`zymdN(oZ);+`vcV=PQgY-_f}t&E!(?((Kr|!>+cVrr`AGwJ8S9pV(j_03Gn^wB_{pM1wwz| zPr}hvZk~e!5s~&bgo$MmHCXW<3-Hp+D690-W;noQEku zowrU7IdH+z*B>cb%FYFNwIc|^sDQaZ-zstEp1vKz<=&6Rlts-c9e%Av>2|x*Zo%w> zKfT;);auY?k-u3MgwcZg0{oSGb{@h>NeL%c=Wy17!#?gB>UktT7eE>kbkEc2C~QIy zgwZf&&(Ds29AaL!@#Ok^_MhVRNcp)S_E1LyjeVRVe{IYJR;zku+xAPA#kW6TADutnJ{KWYTXTp$R7AP9m+ zwgrM92!bGJHqy4Bj!zotq2ge`ccb1(FDk58d&}1QPU{z8Q#0^+Nf;_4>#yCXW>?fN z3-reZ9#=!y7C>ublYUmG7n;?ZM$u@vFsXN)9oAY(o*=#aWb_B9NT*G&QW0K;^D?QA zV1q+v5_XbuBxa`vTq{Sr-1c8@Z~9Iv2n=RG_@ZNI4J5|x5J||4+es~v##Q^po^ zZ~ctOY5wA9GhxrOfY@cSgRdZKDH@7JEF}lrdh3>36&Nr}8Ts{`@SO2lo}9Fq4_OP} zkpx-LzNno7`%yatmP$1nGi?@GWPxA}PW4v@f1?j$25G>>E^fxsTfM3VuBHx@ewTjX z_AvlzaH0SXl(RZFwaFSb|vB;gY@+Qqo}jK!zct|jD@bs+uHPY(JxtW8;25k#$)|t~&yLC>Q zGJc{wc3!9A$yq~V0lqEBoAHJ2C?!|GA#WsWwaJ0OMu|c@3OEg+@t!$_FOQW3R9K7c z8&52$(^ZqxCqNuXQ+A5&=j{{{v?$qLU(X)#j!}~EoA8g2K++}7a15t4i(A)z2_$F$ z3PUlRB`Bp7d_8xRlKTX5IE(~8S~y|9Yr6taKVPHuB;Q^`VAj^41?Q%%@Zjm{8T`%SG$sACawE~-#qk94l9G&S zTc8w7PKj*+5CqM_&eR8%K#A~$_NX`x zc>;Y)&J!t!N7N3GlBet-P(7R+en+dO`J$ZA>(W?%S$(6) zN_g?){+Fs3Tjg@hk`slWwnx4~V+Pr@$XYEN-#*Ac>#^q40^? zkM=vSHD62KPO*Lr6E{AtO2-EZsFDzjWTOL2y?%c=(_#OXW8$QHQs8b6l-?EqW&Bie z!}gm&p}z~_X^X6lA-r;!0tc!MNq@Z~CsZllnYk9Nz?3kL7u|+XApIcf& zZ}~5j8;Sl->1)E~34$O9!WM(J1+W16%RFBWaW(Xo#{zw&Yab%4;V(i;q8!TIEl^^e z(i5QHtuR($2KZ_>8|WKSRnf*`yTkZl1Yy)6&~K@fy@0%OD#U%PAU zg74{r9FJT*2D#sV^Xb1_{8ud+yIB2yY4)V_$h2G{(jbtIx3;zrF8s&prCH zqwQxJC;t88pD+LM6W3pRP3Os5j(_s?%ineESC2gZher=wcin+K|9VE58hiaa&p!3T z0}uY>wg0{K%g2BF$VK~Ke&1vFzVOD*Js-aQ&wFop@$nB;AHVRD8!E5fekbW)^~$@U z#$Q}>9K8DJ;zL(F_4dmTT=b)*2j9H*(t`)Sb^6G&SHAq+J72k^@%NU||FqTr?-O(X z*mmj4J+IcL&%gKm2j2S4H7~rG{qCD{7u@>9(+}?;{V%=s{K-4czVz5-U;X`m4m@}C z?BCw=@DJ{J@q>phfBgejUv%k}UzoV?8#jOcpMN_2(7mK@saE%X0!;_P9 z%|>IsGQGU(mR*N#+qS%1gNho9OO3{D`zKG#ADO?tG9AeGO`cf1uL(t}ckd6f_D-It z9R>%NYljY1b~Kv{i<8Rf#hD7UWns_c{NcsQjw7}C#_izhw(X#|I5WAp)O=viWNmqI zuC{l1sWyMK(wuB8)Q%pl?3ipc=9cbXSe!ppTV6iAFyE{ko~$XCk8NLu7<{H$nXc@( zKdbE8RrzA&qq}xZfBe>|>FU1EZri?1$5apmK^PfhWB&)$Jn`V8rpp=t0000McZBy>}MTqOaaN8(s9?JF9n= zH_!9?o%enJm~-aNJ$LRobLM1r+xke0|PUW zxGmzZfeBHU`-o9C{ATyBfd&4c@&N+_5%u!k6#K7@>maWS!N4Hs`gdR=?F+wPV9@v~ z$b8UrGeTwHhZ}&Peo;u^8YQb^tfP;~KQHSE1Yea-yBs8YSjGk_NWb>~Vjuc-_H}o! z#m0~42epWBhqyu;4f1hCljzr}0)@ZK#7Y>mIG&LG%KOMhostzFN-ocND@Tzc^zn(J z^vFodRmR;?%eym6%V}#TDb(r%sZrsv^ull{`*$I|w-&UA%RN*qYvW~|I;2swea zB8BYN(P|Ps(x0edhCG1nJ&|ZD50{HPjD#mbo@a24Kwn77+U^6BQ?k;4ZXe}Dj0_Ko zKw1&}Ji~_-P%z*h??=+g2JLy*y?W7#EG*w%)pR~2 zIXmB30{fhdGGJ4agcE-pIO9JPnjNHE^8Cqu+)vNq&Efu4(dkjn*;Q{<%QGftH~(DG z%Eb-}VcpP*Mcww)6;4Gq`t4|;2nqF9^mJ>Lj4m=Ey98w^4|--<^4F23I&+sV~+-h7q)kGyW}p>uyCPcWbYOq^pM>t{ZdWX?8JM$$;z{s9Q6bv&M zg}Q8sy%bs5fD-MF9`~+Bf!1bxoG#ePMV>})y%~yQkWpz!?LJ?4!z{y4&Jg1mVNxcP z+2J05axTQs%&V4E3S2Aa#x~v_4Qs&G5cAx0FSep5$70tyHG9(2*iG!x4ZeJ4YsM3k zrpE@zVVoM&t%daidGgFzNJC}D{7)2TKVXJ#aItQWGUWGgvKtO7>WRmvhSFde*Ql(& zQ_7W@+Ox#`AZj~OIh$f>Uu{XimE+fzOh=-g18a!?3b61n9HGq@Ths$B{*GP(GKT+@117qiw7 zte5b{FsVRWU(S|kKtJC6EzYs68(o~kOr|{3Et72~F)W5}$HB@SRJOU334+t) z$HR>s00HH<-f118!i-!*B4IX?9*u6eisra=ch{l%fU~*O_3DP-T{Lt->2did@sTTr z1Z!kJ6DUQM4tiS_9o>5f?3`5Qq?GCf`gbLnp>%1}w>Hp9y=(HF$t|+G#_k8y7vi=@ z7;q_ah_l>GV|uuihYffCZHj@BBSrJdWe0b0dFkevx}ud8v~mnurcSQI;CrogthYSx z{doqGS3hRUGu1aPGJHfdTGp3;(S*bDgi5c#(d(3LkzVcAfN~ML0-sg#I&;UWUSJvi z&na_IEyT_+_8?`EJe6<0@C_FN4Pz)6=XbJEs%Q^q7F9}#SeRMmvsC;Hwsw9woUbta z0c7u554sCp$sPJ6Kew?tZUnoXe4rQ6t^R3@j}5rwq6=BbiVtt@iR}BHOGVBG;BKT! zynxR3TRPTGQ6_+_iqP!0+P(fOOPh$D=Gjtl8DOCp$AC~$k}kaj^z6L31=_pbAGXSSBGzvqZ>)d+Za$5mlxQ)gY>%y75KdYy)nyYQ;Sy z5#Ux(S)3riop>N`vxQ3)*yLD=H8U1gH*3b&44BpF>>J4?hRL{p;W|nG!I=?0VN=T{ zb;U}|tLeZsi^&LzIMt?a$^RZNAD=bv!yY4lu9||F7#8|gDzvDo88y<+ZQrO1`#qLF z7Mkd;rmFJ~86-F~^NGFzX_#Il_Bh&Dc7LaOiv6Q76WZynKz}JZ{e1UEQY?VrvZ=m! zCQ-^qIs(WvVkp=sSBTuw+@62mqL| z@#mDtk}BR%zYN_6zG{nxRinajw=W#llD`xPG0me+pvN|z!`MCf5d`$`!q2ydG%pYN zX5;rZLYn0Qzh;a^+Wg*=RV0Og^#o2KgV!Vz`J%oCIq-V!6heZ`tE$S2dy{_v=kd_vHTAZ~ z@@*`AwQ8E0k=92e-A}eD#Vs~qeJjl6TiZ}pA@k}!k_d&#s z;*Lt>sTyc-75Skq$xF6v`UQ<%N`0NVc!LPZ87*ETKQs-^H?-tQxIZpuO1}G3j)@hY zFK>LOttNmdJ8CUCB|}pUR*L9nkT{Hs8^A>JjQ8cS!{qI)xo2L1mqWAU;>(g~4X0EQ zR0EtSIXc!c=$~5`x8EysXp%bN%698r zpEgl|YJ&@A<}1`c$>htAB}EF&O^75a)=bCsglDZkuKse zRY&mI-zsvE>BodHmT&SdY>bQe;P{p~z#$6le4b15 z8Mj^}i~r|$zt~nKNGxnpit%9$Ra8%lUH1Rb>g^7@F~rC-X_GheujbJj>&`IKOl!m$ z9IWhF@$^(V8%h0j{*}xaRLWpv`QkVFdW_Yz^`nQ%YWTnFGG;$=o7`bC3^6*o;a@G* z>^lyFF_)a);C97R4|vw@okD=i-n$0=%k6a+Ii9PgW>DkM)oE|ch{%t1tQ$#>|`@&mI4WmVO?n&MZcbgNaT z?^#P=IKL84xBwiKh^r_y z=)4_pZuw1fzu8)}4 zg#DP$p%m^2_mWLgLTZa@BlAvJOU&=NcBpk|&saf;XG7ChQkwNh5Trepjw>9DG$PZ03l@eS>KuG-ox zH-Hsf;<2Uj1%WH;{JRry_{LM&dZ*|wsSjZ@xr*B%+=B$R$o6lk{mhQllRd3LlIGeW z`E<`Fjl344UqWM6LIb%OtT8XWNY0G7Hv7LwR#6$HoL_{Xl2Vwa0~WEvc25FXF{p%N z-~Frars=#XBeHF3ys=9DuGCC*SZDp&^-1MnlhfbFgrE(UCFG|$Y?Z`rk|N>he>V5J zbZC-B>3)vu8z)hpv66#JqkPlhA*aUs;Ex>{W4H2$uTJ)9&kF)6icpsZV4jNA&=TeJ z7}PbiFQg}ZES9b0#qf>KT0T^ECm?9JsFwXOqTA zH&;*5U)lIoLIroDHjegn)&J1QjyQ^xog5p4;6$))JF>%!@QGvb~iRc2kVlh zCQdqyRpQ|ZY=EIHv7i)hI%W`GZgZ`Qgzvoz;Vo79K#13I2cFq0h^dil!9-tAtLKqi z-!B1$=zTnzLb2Yt&o<=SZ_2sa;O7BekNzs~m~xx;U$qXxyvLIt^0f1Z?(>M(@0Zgr z`X_pP@F-NRcymRBM297$i40O4B9P%<6(bD>oA^|;-tCGoXZnpgUne$->ikMhqztKa zzZSD z4UOx27V3`voa1ABEB*2R%6d03kx`258C^5UztYY|SgK6ZI{)>EY|Vu>uG-Psbq6QU z=Gb}IQ@b5-1@vHc20JD#&)-`$(AeR4) z_?n%da5(X;ae0VykM%D_82C6kRp0;169&gxc}|W0ArhyJg_Y}01P|ox3q8ooux!Ho zsC4Znl}8)LuR+ySHWx#mYZt0E!3=yDjPN94^dGA}n~-rcSuQ18LOUgvhzpGVNhx=4RB$3rtQ)WiWL`w%G|+ zGPRZ*WkN<}cYmIbs%f3CCowfJz&|p<>Z2Ie}Q@iijZbDs}fDY%jZ9f=4-xf}@h2nlRjux#{n&jvDTxTRy@u~CByk}UpCFeng zt+=h5Yc>%k8`+YI?B(;d5bui~A6HWb{uYI_i{M7;r^|n}HW7mkM+$bIMVNj-KkH9U zN7d8ldJcQRDhiSJ0&Y+7P&?czs@^ZR&c5&gS`M~RZ`qt<`c?tbaw*>ZmLm$O8)5`g z*u!!~8?^BL^DduDeI7Gg4!(WryH%GH6t(^J818ubglwLHEo+=7I<)m@?jh;LL#owg zfs5J)b$!7>(pvjLNQ;@{^Ob4-E*-*&>9y_Zf?eJ)zPLiLj~FvcT>1I+k8@pGL^+ok z4rui`;&$u)hzOo6^|3Hh^iB~zOuGK9Z!&t<4qhd4#m*p0w!Z5-?a>Ydp&2#tH$Rq^Q9c`GWX#vZ2;I>3qz35u%8_=srjIlOI@ zOst2S0!+~&j(0yJqzTF~p#KD69f{G0gJo}K!^%QMe|{`vg3OG#3Gnm8DyNL)HTKkB zlz~S3h=}I+Z6!TC;CFQe!0$X(BUxVW@(2$lp+S$uPxyQ~`>naTkNQUw%zW)?HbV)$ za)f-GeXgpnN9Rx*xxOB*0oF*tPOi1R_Yc7_-}9z|L>A~2OVXTr1Gy$n`u%S1`i%~v zfFF5C%eSAIv)3u#oWvh_0-0GT3LA}nmv|%&qKMY%+K`uaQH0i+xwPa=(G~&4j#mpWdZ$ z2AQ=l-e>>{m0J>JuxwLmuvs*NI^7a%!&IWpEQ_LtaBW~)ynWstx9(GGBsnD+m6tIl z-c2*?#ZfESl3~u{M{CYJY0Ngu$dv8~+7=6iocp{le+b4twaJ$lZLS4;V$j)n(y||E zF%HkPV2rtbr4uCfE*bu*f6mc~lh4;PJ`J!qpelYo>3PR^9*?A)X-!COvlw7@VtKMr zfrVb0DLa0ZY3UzNypB-#nXrE-t^Jp{LlrPOu}PZm1`oIevo%H0^L z(^iifUq?vkFv{?3i2a)4_{N3j?Zbo&EgFxlO-?Kz)Nkzq&c{d9UW%&a2 zD<9vaw@x~X@_P?wANjHuC%QFE#lk~tUlLWoHwB_GwPWAql%fYWY?k%D%;&82QGq8I zR|sOO>;)Qo=w*P*^ixBa*@iE5pe_fZ1&oO@cj-}4kHqLJoD>oJ2Xe@eGlcJob31bof&R#abhlYiLc%Q2OL~<|` z6as_LQ6zs@5ei%DCdEC(XLZ(IC1p9qlJKGJ=XX?fBvb$E9pI)3W{YoF$7c7*Xg~fG z5^rYu*dq+Q9HS_?i?o}@#kLq3Y3==aGH%h=3+1va8F?1?5^=QF*KBLjYwDdRVXpZax%ORRqd-ic-R?=1B#*gjmQUWL@oQ?>83(H>vhQxPZ zP93H9p@e+rsUl{HOK?c#OF#ig&$Cou?d0$84?hwzE4F+(S!`n)REi!8s_4=_mPzlK-F}EyC z7%lneCLXEVGmlDSX8T-phN`;v_ku;&bZwATf{}@>3y=&5*y~Rd&pjVKge#YO{Pfik zD}S#;ZrXbGd-H8s`!{~o7K;Bd^1qUdZ8nj;serV3S9xAT=7>fg!XkHQ9%kxJc=s=} z|0~WI5U0=E1|}}ykRB9qi>aLbVgq+vLC(@)Sf7N`<$rAdpCZnJc!u8{z$iKBB7OK{ z7Vi(;+^F#mNd@(v5Gu$M$ zcH&R;)9&Y`EhGmfqW*uc=?|%a{~j3s6M>5ry}jap^7_66=Spjpu;{HupsN8kXt47F z!@S~*R-^)-0M)rszKm6|YHojn6a+{)pi!$J-f?Hv#s;0)6Bq*~$sr z6&^J0pX5x|b;x|oN$$s4u1C4<`?(gh1BJC#FF8tzKIXlE7k0umS64a2)dRbrSBk*z@!+lbZExERPkt@9f0&36Iq_j)v0M8vLm%J$ty(y{* z+ZT7R6r%%s5sGL9AOo3S7-o^&ioX?^kao6F;IpFRWv%zDSN2b4n#dET;gf z-h2;J+pbipcrBc=zlzjyGb5LL_tID7WmwhmI{`tZ{7dte%d%1@Qrg}U{UueL*F#zn zmg{>_!cxj|gI0b@H{dQJ9Z!2QA21%zn}A8JdP@m$1dOYV+PC8HfOy}S;F#afIYy8eGx(P%Zx2=$iNS_`VK4jZB@5>#x zNTqqLOou0wd2K~{W=b|w+4!iYS%dqjYto(rWB`YLC!I@Sr{SFYUFw`XmN-M&D6dHpT+=-sG3%foNS4MhXlA zTtD;EyxWWC2L9X+D8BWlnx=_{w+CDo-^*>XrjU{jqL<%QDtr??{<$WG1ML z@t{oS|5S%G-!y2$G<_SCbn?brK9lZySU{Oid*dyF$0xJSV_#dR211>Odmzr+K*7-_ z5ft&%{$@hSbW&LN!O73seQvgM*U>Knqa52xE%h~qU65c>S8B28keCjHs>GeIw_9{%1Ny#U^+Lg`ksr_C)ms?#ghT5d{YThi7t0BB(}~FJEqs zps(S+w-c_$u8+$`C*@K%bBq+?x4p3KNX<6rL8^6`JId2hP2zH^r{$Tj6!d8Aj=%uA z2U-qzd3Xg!cT+DtrrCS%t7;EP-Tqd6xW#{Tcz8D=@wj)(`RIm9E%hU#G=Anu*tK}% z*R=HXoogq`KDY_Ia#$BtzWgG1K*u&YMSNS8iOP)eHVVz-+Gweo>~ zMw5vpjsu>38?6hly&IVPs~wNE);53!U;-$z4WwpS2|`t@w^Z2S5ZDHK+I^RH?XWh+ zdk1u)B*+d9Hdz!C3GW4)m{D7FOw(LLFkB9P7#5q%MBK$Y*Qt zT}lIJ$m*%WmJ~Z-s0wJ16DrxS^qEQ08I|B46s}l}H;xO6X~=ehV4pnZU+cQoYy`Oklp)__~|q^o+OQHfFFfc=!R3pdW0^FX>D7T2*xt zzSw2-;p`?Z0*v5JodF+n)jagfUPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>DA7DvDK~#8N?VSy* zTt{_)kJnD(7$5cxNDLi64R!=@okgJe5Jw73TG;<8FY1a{k4LAXg& z9x01yB?w4`CJic8f)*z-NUcDP)KVejjY3dLMaxesTDA05rj3w*($MoWGxyBBGk5RZ zx4Z8r^U2_Yh^bOEwXaq?tQLvfyIKX|gF~ z9EKR=+@>JPZkH=N$;u-zbC+OmhjZ!l=_gHTkw|G_T!701^LQfb&6^Fy{JaP|Sl}{e z>Q0Y^w!p7F(d_iP{XP{KG@DB`v={ZAg^*4VRKr)&L`S)d$p{NNB?V1wnK1=}@>%TU zQN9V?K6TB+r$a5einI&6Y#Sv9SUJtvaeZDMxJ!g9k`+5=m*^Qxre}|*DuUBdW|^CM4lS2g$+ktwNRZ2o9N~khK(MZ+ zqMaDI^d7TTl$WRupX0dW$n zYwV~NtJ^`a?0W6w+Jd|qpV^w&5u4JwmdmS_2t_;iMz4(Ql~>JyGiRkBSM`XeiH5pu z9nYMfu0?G*W=i>|rBXI!aV1S}bPEet6y}xp z;97*&5}h7B?`e1Xy{;+C(u5`E)h$_*HT5qf~$*mSUd&_ zcgcz@sC8{W-quZLPMMMr(F}NkE6w1z4CMq`$H+>W<_b=O|76#UnZflj8m3o zR1bHcb$*o{^LP=2dR8LqWqc)x7-G4o_32g0GC@URS{Z>mX(_TV7r2CJzJXlWoad| zgHq;yK4W z+(%c7<72w1_R4(AESo8d`g)S#@qtH=$h+NMr%T6*ys{4N?4c$dVCYA?JKsmsT;}Hw zA~CNb5jfS^=BX_#v!i5F@8v5k+Drbh+#brTiK1?=&5rRdQnq`kmg}B)W$Ie{S&KF7 zP}%r{?y){%>t=S0j%R4sBXL&SmwCAiDvQ5KkdNoBG?Ed;+C65rk05h8F!K7nA5F(qr;@N zY+{*#Q?|`V4ZUJ+!|Y8<3X#RIxhIFt{02$;MIsguc@}GPJRx404F->EIT;07w76RG z+V($oBBz_8<;2p5vaC!`e6^_<=TmgbW;*b(nTWztM4Tc1AtbmO(# zrC93=eOwg^F|bZ}EPVenP;N^3c3EQlCpYnb zp^szn$H{ekjGx70Qnib_u^jS8+cA*3*B9+4Vf$`iS*G8{>4ov)d&*iz%YCv~#r(n6 zrEI(7$odtHgJR2}QwPqR-FJvoFwfp}=InvTi`h18Q;)UwQ?N0K#nZE?>Z*C-F^ms_ z(Qnh|Ed5-OK~J1WC6L8(SE+ha`WPEu>5J`q=3B8Xdl0F4RhW~w!DXt2t>J6TXPakj z2t`lgrOSvIqC&sR9iu0SIUPi|A!Vb(h)Pv`yWvoF|HPQ4}1DoDU8oXLP%IGuFp3`eSqk>+5!7 z{>0*}_3_R5i75ZIekK&$F0GH`HW9z?L4VraUcb}n_PY5bT_tWs4)99WtY;6Mavz8m zVm7HP1^SqC(9cTo)H;5gGluX%LwnVj6?2o=FUpZ(VijWJsQQ>G6#MAfQg-zG*K?8; zOqo7iAZjt6hrDml<5RcUSgM#<3`O0#mGnC_A!29m(Xtf$SxEdfV;PIOs924~{ut1H z#>$mgz&x2`tm?KE(;tV=HqW-zNm!RhNa&olQ1rQ_3ghr$Sz*rMi9}+8xb>{Q?)I*lfoxX4Efcs|VR*!4Cpu<0fMf0a{oxCYXeTRJ(pThIG8GaW704CRACByuf)WyF0X z$DwWJ(APMKvTyn**BW@`K^vhyVp;oEXv^3^k-HIw*RS~6<))>sxZ(QC<*3hzL}G-L zMmAaRys@MM$O0>re8BBx0zCyam0#j)|uNYb-b!l?U7aGJlnBPve7dC zvW0%;&atK&qX?_%vcexfzO`U3?)XLoJ(fYVoNtwy3o)&u%UY+%JUTvXPAsb%l9PU? zN3YzA3h`%DZ0n(Y2(>Gc+6?W_C>U_uR`$*Gy^8)q`s>wg<1rD7`7rrt)Oj}GTcPh{ zv(E7{OusxLxngRHeymO3oMiZ(q5(HK4R|_L`dFmcR@Gz2&$B_FQ9^kZ570h+RgO{> z>zM;JE4L}J64bWw8as@ytb(;E&#DLC-Zwq?cZQ6eZkOVe?ZZfsrm?B}CKkH%)icSV zTF28@I*9q*B!k@S!+FEwBe#5Yzk0oXuZz8->Y1NHsIC~Nr}o9Zh<*p{VW8>jdbp`= z@;z&bRV=oVvG9G!@bj#0S0qw5*2l5<ldydk zbqDq)`G@8E9B{jIQ=O+D7J z`5BL=wyEptd7>G{2f^y|(*-YAD9ndv#J3!|D%qy=F+RT89PoTAv}NrKp8sfve$JCW zjur74El8eq6U*vROm&$Wma67aPmdbP>)_r_*EM0wMu%XZ5U0;Z2z|^TPL8-7s&#Vu zPA7+H$ZJ>m%;SxPU|&LYkq#b8^j_$TDNvx=sDpuOdp7g-5mm(y9*aU-20LE~uU*0@ zNBcMyf5JM}&%C~c~!ZDqcB`pQotGZNPCxIbV-C>>9p5%0T-Rq!ELuC6F&?ghC0 zrh9HT#hH`Wa)v$d_(|S&L~QD@E{u^WREejjqvNI-#s>#yqO5J5Hw96HbH zTwxoEj<5FF_up&FIh>nUNU<)SSB(9*&%@+d6vy%`qI0=KQ*;PF?s#r7d+a#L)ALNu zm>jBge0@ac*{rUMuhq}@i{)65z8Ga8T$eV@f-yIToW2(*?^DT2AC+h$UpF=uIUgKW zPI&Dqo6$aw(I3$ntnalO@h28%Phb0~bv*q{$h6%qt*_-7Uf+ZMv^#Vm)$8{=?75;s z+zMGZ zV%Fg-wsP6vG@9w;%2Y%o)yf|)AhORq>A2{A={aHj6q~WS5HFhL_Nl};ot)f}#VYDg^yX~1UvaV@=!)jqXsob#ESMbU#Xp-5(RezWR!0=F zzWdbm!N=X-^Kylf%lHQ5h)F?yuF&Hy{+JEb$7~MxzUj8K9qf`R75a!snHx7*(f8Sa zqRP*~H1gS1>FScj#$p)i)~%+`=R@x-H7=HdKMRS!W;|n2mn>G}u|ImWpRsZ^hMy;s zj8)yXV*2Cpd8j;_NF)-8YT&BWBpA(712r=tf{k%T+e7|lkD?)dm+k6|_w~5C*-JRz9<)Y)V7^ZuA z%gI$v&^EPBCil_$6@9RnUmc{q<}=m#``n(*0=_}_8-`P+k5e74Z2!7&u8;PfVCGw) zEo%n{j&JI7rsW^?=Rcj_?(%~{Q$_=Qc|K6;b0U!#A@%aCe)?9GkR4aXg5p+?i&n~XUUFTOVa0uaYdG`E3r^qOOiy$tufVQZ)Vfi zQEhoBc_kk_Iz7{I`sfb5q9W6wDvW;(p2-nwu&uq)eZ<{a)m$Gj=ICR-k6Bxp9dC4a zjjZ~JNEr=a6=@;eH%KHVgUPdtjLii?mg#(Xb#z9n%CKz>M)$U_s~a>EtOJ{YO8kLaWmO-Kq8SCEu+b^Ro9P;l`j_3Zme#~edA%6Ymp^P z`jPJhQaR76`*`Vh5s8qT^gBJduZj*{mEHQH^-hL{%Xgnwe=HFNT&GyL>zGE(e#Z_!`T=yiY$P40#P<0|;Jh7CeMGAC z%p)j6#71npEog_~^Dtv)mhXt$ESvd@#JZ+}%7YttWx8O{7xs0KqU__!z%i$kUU z4sdt$Og=tK1nl@iCfkq$6^m!V+^8nPsO;1w)-KI^3cG!;3>qAHatN}dm$Q|JbD*O0 zGnTIw$i&8r4k}YHsFQjv7dU1Tlge=}cC>6=;izoCh9glb^eZNh<*N$B!(YLljDD}{ zmb0X7S6bwoyo(Jnr`wFliDJ7>fo+r3)b^wF*}3o=j|{kbdky7J40KUeX3^nNtxt24%P*(N2=r=@5=PU zepYQ4u{M>M^^N`TxcK&sH+)*dG8eT`X;<6HU4W%mY}}xr*=?5Qrk7G6t-{7 zbH8G72;lcJ!&ndW*W%Z-yXl?9SV`0~Kb;bj$(MS3Li#L^*jE;@g6nXESbuZ=4~S39 zA0gDwoPDKFdy4V2&J=PA8j0RcnzoxF(i2FiZGrT<>w$u6>~;~P)laB z%JFMt+$5PYtxI={aU0yPqLb7@ApghnWF((cOZe_?;t+%V@sYm`#)NcZ=E|GN+%#DA ztePp6+p*-d90hX)2HB^@wI|^UM!@6jTP^52(6m;K*%@HJV5giyw`2VsD7O;xPI4cw z*Ol9~ustOjpkJ_$3eHb1kOo#^yB=bv#6AXH*U{~|Ry^{~5i$EE?Xq=?)ye5QoF}$G zikYcGH7Bqwns<`>4D@r+;uE5-ox1jsV1G;O)YV6DQ}0E}^h=!1RkMWc`#6($a-m!O zB+8$*i*}F_{UWh`)r$`{iD096YNPue*mAGD?cv%DgDcmw2h2Zp=XK`fzQ>=IKlZ=x z9&00aclOY!1CO8NrEP?|emc38nD+tw%IzK$2i)}42m2!ol*bs_zLl8wy6qA<69IBo z0}RtH5mSt;D?*OcI)2H3kF*l=e?Y%N{>1uJD^H4_eLB`X`vd$5P`aFDDH`M^(ZNh3 zCMU=V-;`)$0axVWIKHtUXUEjTF|%uX>8j~dOfGh6+@0ks1K&r6-Elfug_ynr_p7)^ z?Zo7coE?E`slFOeny(!ner~uB2d8f}U5!kiYv}7hWz`aI*IM#R(QzRH_OqG>GQdi* zN3^w{nVjugi)>p|JUQcxdIa1-a`ufzSsX&-^c`Z|Y!KSDfK_M5?sGa~Nwk-Je4IYX zL-x5@+g8E<0BkYT-<Gf=G(0uhbUB;e&aaPjjt;hUbILn7WQ9!#vtBx`Z@(@RUM?&$ocUI`}h@D!A%UPVy%VO?2KgTq^mP;HGV}j;YngNMKB9U16=;f?j*2FG% za`JZ9T#e;FlB_S&7Uxtt=4D*f#V3WWRL}XjVwbbjF-4U=M(0%#nU^_CMBOA*H;HnL zY#b{wxL>261!OTc18T%Mkw_#Gi6xF(&bImcQ+Vpmol%2wZ=s}4!=8MyWQ*u-H?mB0 z-tlSPd}+3eWrWY^tdG_0c|Nob(p`+{liVlsBI;+PU1v+_moce0Xma+mO>SVI%NenG zHGuMI7vAej!^>atz$f!3`cO3#w+j8rdx_gLCKx~dCabMLpXYb&T79utbcp1}D0js< zRKcG_B9TZWmJhiP8_-EKbY}DA<^*WZmbg>)suLrhKIzn|Ptjw;J3VJxxf69!ZYAd3 zcF5pcC32E-ecaTo6TXJqirR_jM=7yC(l``QEueP3+lYL@u_*U2R&4WOO94t84BN<8 zAgzAa{-Sk`fC<08+ls|)V848OOhA;UQoC}w;x{+YpF|>&NFp4VA-APGBZ_h>G4J;LZt$(rW-d5&^>Z3-D{3dAAEm_p=;9DTv`TbyUWD*!SI{>E z8>O3-L03>aZkIZc+Z)hFQOH6xZdnP*_bXr@@v2xv)k>X8&Y~yh64lezJmb~loJb@R ziNq2F&tq;mOJ_WVJHN%`@z2h>r=hgVoqNX(os^|f@1eYY0$Y6eZcppxo#o+&?7?$> zi9VA(!Op6N{4CfJRmSSm#{pEoqJ-=tql!ZrJXJgo)uZuXcu1`Ek&@4J2K5Q+zQx7B zlI?X}_Q@t~v01Wj?Wu885p_QXSQ58Lzast=`y^;T1LG*Tsw8(tybWb3)Zs)Tkw_$# z6}h{!FKl9Ki@Nd=f@>DFRthX|#=!*owJ40cUv=%T)Ho*+i9{kXS<-Sgkw_#G%azlo zpA7ypyAz4T3xlP!oDDx;T>uli`_=ArBHtsb#{%sv*4@5zS2D4J8F^kxRHNONY=~*G zEd04utovtnClZMl27o*V=WUyUpHIQPP58qm%r@bjm&0Xqa5cat*mD;g*$GGYV(~ZM z0I=Z`Z~ezLdp>sc_kaJ1zrO`w6~Hgzzjyt|55G0rB3rPADmAvk>U97Y9EXjYn($kj z;h#3cN9N#H=iufh{8|%kX~L=|G@9_~Cj4m=9%{nFO*qzsA5Xy_Pr*fNVM7ysxe5EH zVDUd`0pm5lZQ70>|cvY>q+x`(Ppsyi;lwusB_6?SdA9| z>)|I&cw!Dt&%qzf!TcQj98JxUsl(W>{cD>Xg@>@})(?<@nbxYA*31>3Y<-lrnAHFm z^S=$pVeQ>b_`OYV{w7%6ge%v>6PwBMd+;Cjon2!6O95UEaOH70|Fumxy$PP5gG)ES z+c(1{Q}D_u`1TZ4IiK_!uINEpoCBBQ9M}wp=V%7v97YcT*uBeE!w$fIhmOJ#YzoUT zOY$(z^DF9jjxi+9kHdvQS3hW7{X~hM=d9ko9ainxNo4{j&r9!y#_w*z!GX7a?q#s* zTP%MCz^gg!Y~}JcfIR@O0~%0D4FEU5noD8L6|iOR^`g2vo&Sm!3gOV`6x6C`6`r@q{T%h$py z>@+y_$X~%DkC0zH4c@SBE6xN;RM*ISaPfTQaX2>@&#N}WRaqD@hL-@m7PAw^@cB*V z=*EcQp?#G(v{x;SLmP(%`?Vwv?K_T1MRe&^c)EFg6TaDmpG-lw3H>>EstGfgoHxYM*&^ZwgNegdZto+UPH`k#;PI_JeZ?t5_Jn@V>iX_9$$>1CCuwg^HUuUI?={z`7mV8arXfUf8o2 zW@iD9;ayk5o*l4b56m`TmJ2&~!tUL$3)Qy6H9O#%U9kIoux=KPZil;Wgrj%B>s|u0 zPI3&6qK8Le-8wji#%E#2PO5^hyYb%<*u4k#u7aJ{VADUp5N+;;Jx5^Y5mLY`vUex! zT?e}xa052IorF7f!S-3IwR;C_KLUH5&!~xR@1R|~W?}Ch(%ea-!cz(p(~CI2Jcj?S zd-T@p9z8I#5vDidCbSv{QP_I_s_72z-3l{byS4Rvqjf%R9=!g8t(hOKo&LhP(=FNi zSnJ+fum8|_tt(1Qt)9jSfO}6IocY>cwEpj+=?BQvbz2X%E;!u!%1fqE3pY<3Y~6Tj z=0{iBz0i95shNM@ESt97Uy^pGA2`^$_we-}d=1wn%dHO|Ze4fP^g-Ht-PT824;^kz zZ_hTrdbsuGU1sx9{C9Br0dexN>8}B8LSq_Fm+;ukhJ(0y-LtE07f5I3_&L)ZYN>TI z=}bTLA_-3GPN{pr^hWOK-LiS1m}=cCn-`hA56b2S-h|etu*;8K_w2>fcwFM%Ka;(e l80F6bMqnB)oo*cj_1*I41CG;9Pf+zw?385tP zE*&92=rtkX>3#Qq|2ebg?Ci|9JA2OVe7o~ePe+yNHp^`i5)vwPH6;TQlIxLw`dl}! zk&uvHzndoYMJun0ZOH%Vw>;Fq-XtW{U4I)X!o3_q zLc$1ES5kQK+6tR1ep(IgZGwr8-* zu6{w6jrdtt|BU-*)~4zRss)kO)y133Y4>e1pR~*+q2fPv+TOY)WReJNnR{+Xon+Xs zL>JMp*D)@@NJ?(7+Nh@1+e|GG6_6GEl3gudB`VnYjd+8hj)i;hNTGJAsVa3+C#e>P z9QDEN2_Ucw84}Z!YfLf4O1I*3qVraVAA_!lN5>5TuR}V*@ZB6B`tqR_hheY5;T#kcy)b-z zICeC%e4rtDJ?~i~^Rnh-iMl6qeVY3jz(g#|)+u8-kfyerDJR9mqUu?A){2=Pw@0q9 z^ZsKF+E$D4rS5-tR>*Ze?N>#=^EhA{P-H1t`Yip8xtvWXYgSIK za?I`CM6YRKPk}KL3aDmRkC8}i)J{?YT-W$8f0H}?b3}KBP^tl-$?Buh=tBhbmBMO* z_MD02a7*Ws9^EK_B$vurAt=0>u}qmS^@D{#`nlS1KBuFRSPQ(ZZD9MC!_oei0OEP7 z^;XIGO=-&9vUCOUy+xvyHH!+Me6lF?le3!82$3`PL##)f@cF$Dt2yi}4ouNZNlod1 zq@}4RzB2LNPusI(PysSNu~e#X-?0GfjGe|sP?9_EAP{@AM(igdfcxM=W!DOX?M>*N z&TGj#ETR5RZLYIToe)OGIaCq%%IBR+Q2(OyN2PU~V7f)PZ6oVpbndUTwO2zq+_o%U zv)qO)w*w;33Oo}JTb|W&)K0f>3$>(h1<=suj_UOB*V>Lv3bpOVz4OiBN}uv5*@1j+(uaAV@Z?++^{!zdX&egb^XujAYR`q zQ=da0ivw>}9i4AuQh{kta2A74c3Vwoe^6|f`Er-83d8&9E>_djED&6F4IeZP9py$| zzT+-{=_|RKsl0IFxWCDi()bCy{wfNQmu8DPoSNd$_Hv6=kibG;b4oLN-A$fs3zp>{ zX^)f&Px;J_uYBS66AYwnwf*!0SZC+;z&Vn<7xiTHLCW(7fYuUe3TP~nU*a=yyuG^q zK{pK(nPhEz{rIF>nJmiyM>^64WpK5dP(K{8XSL9^@%rh)wRP2GV~SNQSyX1I`u`W^bRd=SRBni@Nztx4) zQvnV#`^R($H1o~&R8BwJP5*uLQ?`w^eI`~F_Rr2%;0_Jx_^}lLi^-wg(D4@8pr=lx zHNPw>;6UWJbo}bPB-B{+tG^+$3NBt}&k}MD<-uK}?VCguQ;`vRWq(R3Zux`cfV45j z@!=R&liscAriqGzTHK*mx>#xpHxI!RVJ&1JnC9lD?J{K(;*`ptJi)eu=38nXcdU0e zJLAS7#-#={rQ(}xfdT^gCTE!$FT{h(w|K3?gpOiNx1TFJ>P!z)%{sL4Z=&w99eSXB z(#Gpcg`gNF=3^9X0p^H7cyCN9RI8!siml&i2%vB5fEZ4xU90~x^QeG7a{;TgnJY36 zb!AaCQy^ATN`S(!uEq{?tpHTRkzFaOl} zASd%j2TQYnA79b|qc40%u9LrdLw!vGBB_b*gy_$@>Zg{h@n`a->#e)7qgQmzdCtLG z6s8LYWxV_sX&6?}M@Vrs=r9CIT9ksMCw$E|bv+C0vQ?I(ceWnd5#K^V)}!MxRKjPQ z_IVzqbz@zCO0RR(?B^~NIOLA!Q-1f96W-_L(km+b@@oQa5FPE6qhg3;TW!Z{3_uC2 zH)~@u;auDQIk1D!J$|{bMt5;5S)FmP8j{KWe)Q=Hr{|RlNMUDA_vLB6n?)jmqXct1 z;^7m2iD5J3>*5EI7CFsRcc8`%6%*H-PcyRgl*I!%etB2l(5U3gjzs^UF6`88;NT-j z$I(LJhsvNO!8NbW0}WZ_tmJ2b6LmB;+EVs&cb&HFRqgf-T}}@*vI~^@3@zcIOgasP zrr8Wt^5KH=4@yYm!`F??*MMA#a9(#Sx8<FxYb423_d-Af$klHXedCbe&zJaP=>PT{jMt^n|Eu6{Q|KH53E={f?K zBDp*DBKbW;_bUryI{Zw1tHgz*)XQ*? z&Oa8<_Ir&yMdq~_Nd<6af39zy$Dgqr{vRMqpFbV9?38MVW((8?=I7y#IQ#S|k4==YA{kgI9I$HuJe1w=y9<%?;o!eJ=*U^XYE4 z+SwnE$Aps4mztb;cfWJ3+3X6l{|2||INYr_Og@2K@%l*R&A*$n?a^>>4_nQi*81ip zyfMc^S~1hq6QA>TkLDLLq_XS7aj2?wBE;*foygOL-c5*em7&N!y*jnIZ|Tvb*GQC| zSKTE(Pgakz9yq=p4WjU<1M$3c>$TW4hvVr8c3aS+)4YSX%lQ?~fxgzQ(c*L8;hb|8 zB0|PEy|HZiC1Le-YNrUN2!bE)@9TBt@l^c{X$eWe+)(_}AKd~^2S$nByr-n<17rqc zLBj^R?e_yDHR!>}UctyEtADx`1sU>^l16851=J|PRvNwV^3}X^f+t?AyHAg8qe@L; z*)pQhOC-l{-n<;Up}&^M!8dEnGo>p_Qs#1%>}=Vu{6nDePQDGD-cG@I{Zza`y58BExur3a zK4sJ1j53?Rv18Km9Bv9REL&DcmJTa`}XUebqC{4ngg|9eX_6;i(!Zp>F#BY$lc%lX$k~Nm3?1=C)ANQp1yLuy-9v|UV8^3W-R$UZo zjJrJ$L}aj6Ui2bdwz~GJ83N**K&xW1nNLOa{pqWA$0`riE%23}$I9hW(z9)y=Kf2L z7QpL}8k!?^n+*z5KVIN67@B6wtyxb+elj(T=C7Qb1x`MF#7y>g?7y8<>vjN7h@L7!&wdS z4-RLoymJD%$9T>^n}`PG(&0RlA^I;3FLmT27Jlx{7Ug{81YsC(`cWU-xnM>em5fO4 zAK#dhDY63A_2&7v>5g6V;o9ak@z}@0N<+=O!5_+V>epwxtk}_;30s5Pb6!Ej;}w^a z0|&>4s~r=%iX3?jxP0mB0l%_|cfMq>Bd~6B)5E^TqNqN?R{B4tfnl}22$9b9h4 z8}ERon{emsKmg}yHl9^Zc~+)i%BnP8y`aWUU|_nQFn=m#DE^woze+nwg0;;qRhK^|vHcML_IrD=vh+}N>Ydb0 zY|z@R1hq9AZuW~c(xQwOe#k&+1bZf74E1SH1pZU^=EScmMK2e&{neBpUQ_-yPMe-zD`?H*oQtRTKGMN_jmYByZ}6hL z%JzL415(fHp?E8H${En=8YU{2>5haA>Kgj%VGMuugbvdA-frc%m^_M-wZbc$uWKpI zIG?tNXq|e;&UYssMoJD{BImwi96N+~4ml{8yB~OQxL>YG_V8Qxy<9iD3_W%mN(y%N#jS4lEMV zGm3uJB^4whXz^R_AH*LdSjP<6;NED?Kl6p(J^e3n0zjeCDH5tS&8|ND~G1l9M;8@zkdHp zFl#&a---W~3&|Q6P08KO1k24Id`!*w+DSl`v`m+D{7}d6mt;PRo&<~wh2#t2ywp}h-*<*HV9i>KtuuzxeW0ehH2ZBf^NC&P z$yEK90Ec@;urlfluLWtyC(dqxQE0Z#mCJ6OG|E@GtcLz!47S#zcpyCRWbONfBPn5D zcz?rbGvT17odHPh$fCnYCOGN`xQCAq^d(^GjIRdSzH=AdrSqc{|7YE@H|`FH94gRD z5rrY8=Vpq^D0kW}$Ze^kzjF{=x-(?Ww1=G0nLmg#ClviITsSgB-k{r=794|zl3a*_ z(Wo54C^eZ)td_#=9d>T~C&8Q)ail9-IAu=43#2?S^>{8+NRaV*w#p#xRmU%(){v7( z;i1Ie#S)L#&E(YcPO65!{Bjn48@gNOJN#@^PXh94&g96V){2c~?Xy<-iAo4r0Lmk4 z0UWDUn0Fbgp_ObM6l{hQ*qfHRpTuGO<#}!U`zx^n%mruD7I<0;=TjQ*m|xBz<_c?6 zI^%m-8T~r4c(_iiV)0=rbhQ}>{KE2%N$m20gq2Ug^zC@X-cjb$vKg8=;+gX{WBfBd zBwx>ZN_gI~9klGRWQ%mQ`!@Y1ex`YBa&N39Y*v~9LY_e;!UdVSG z@v!m{pl^?taJ0ONPB8J^APV+LMWpH)PkhczgQ>5BH|W=1;X8}+v%68A(Sl$JVBqs|9>#Jr zuv2(oe`$rY+*?I{gUqUoMqSo!t)I*5;x_L z-VJ8LLQ0$TP-5*%)Z(QB`SN4I-% z>jzYr&5_dMB)$h$$IwJy=VeIuKAvIMSQtvDl0D$iiKUc5Z$e7&m&`G-&C=SM-D2#Pb7ALF~60E~hIr)DeCI zw|+x2pmI(~sVlU{Rs%HD`5^)jX!cb`ZRpxUubGZt-bvzYVyp_hF@knJom zJs<1#Tla|BA-`Fy`p*>#@(EUq$`clU7AZShXBW-OFobNr3X#QeO+uKX^aRs3@sQx~ z8XE%`I!$Ia^c354g7-OAha(&2CpnEux=kI7(MF(6!JL~thqK9(KfQ6D_+XNkX zle~0vK1WK-KqEuIpSox&=$=jh!yb$;<`qxe_x^zh+UIrEhN3vS7eZ}Nmb>7cFq1me z8$^_JAXZLf<80&6HxD1&?CkdJCZg=uE@S*gkL=?5*5_tP#3Zqe!vfHfoueE72!HWV zdNb$_q7F_ubb_Xj3-Z~bL`82bhntHcT6rUaIW%GI=RBlTH9Wu!b#pBXQF!*xsX6O` zbs_HBx0u(>xQQ@+;5L{QrrF)-hS+0K-4H&TJUEb`2!VFDIAiLU%o_%V_h6#UB3+vY z`km853D*|?CD{Ln(?Qb##4rF1WoZtyO?sO75m57gtST)yyO*9LZDqAOVgWHk0>p>$&X-~yIqg?&@ ff6&y%nM;5)SICE&pMbxWcoKDG9iPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D8vsc}K~#8N?VUf5 z9JzA9SGf2APAK?f5by%U`T&EAHa_6a*-HkB>|6K}=QajXyk9{B z+!t8*3oM|(*#%zLKay1>tGn6rU&|k)8FqECSS(hv+1)dyqszWOKy5c0oR=0b#pNzTTK&L#!5 zODYBl>oBpb3;Zw8p2e#2G)>C8Q*j0@4x)<{>xV=uyC_pW5whin@Y7rp#YlR+_Rr14j_sC3~OpP z8HZCenw71fU>d^HIC6y>ifnT}yCmFX-NjzPTx#c^gG*o&UfF3|@gq`-pY$)&MmE{2@S(rSk+Gn97r zW%-0)h=*a~XnOPV4kqOik(ZYwhh(*Ky@d%;4k!nhT&2{?yv599Bn|PpX#wZCxSH~6 zWbA}R{mSME&H>2;c!RXyg%c$)XR>~!H%VfVym@|`HV87OEz_%USMSAHnG1l35Gm5R zL+Y{!rv12=-a>hkrz~|v_9jAe+^)7pP}v5TEEhV+3Qk!YTKf={y3Df=pKKB<&0&4N zO_y1?!}M3%q_W^zV2Wtyy4-7uZP>xlpt7eSGXfXn4Qu>hqLWaO_IS<1qc|EC7>O=C-i5odz{|bsUb5ST~gEz$VSOH|hXb zb}XPjTasXw$OSa*<+7eLkYUZow6q9RTSTNqEDmd#nUYS}OA_>fG4@G^i53`?i-odB zorALEvSaV?XmfUUI!#=`m53C5^gSu<)ak|Qm?kauCeZ=hNA zXL0ZL#qLv3W?@c`=&+G{@GE2koVelCji*bxXuJ9G?hWKCy>gl5*UxXS zpNiy;avJX`|Ckax+JuXp^h*Bec3BQB##?COgVKQ#$lDKm(51(GVVu8`z z;Kl|VIZG3eMfM2B9%BuL0~$l#|xJPbGR3fw+_17*ozNLiAj zgr{V>FCmZ}=pS1K_*FckcJ*G+PyTUTr9)Dw%X_&y;xlJ3mE$^9jZ@N0D_yciZ>e3a z!(K^?1Y6b;)DIbqO@~1cPn2c^CSdsY{{ zN@1ZgE2S2p7RmLvfDZ2!&8QPbHhL0?_RHL0xE5Q6*DNLx$l%cj#lvdH;+NtWaDw2xgjX0vnA<@TX96kHJkOde;d0a&u>fwMf?6}Nrfd{|A#^*m z$Zf%^u7xYGVOYc23U+~USbI@xT8H;?U6wG~zUYA4uxM6VEM#kCk2(iMG8lqQ!?4i{ z#40>mm?&X&!lMNR-isqg>rvKZ375%$Yk~K2uD}K@((pfCErha$TLewq5n9}=B@4M0 z>&S{0jt!M()a9hZdyOZAwl6v%gQK3;BevW*D0{dctTRiOHGHLs*Pd!}jc_e*Z!4Y; zbI^6?_~Ys9;zOs2vv{I$7QiY)N}ad!HIwqcqB|b!^a|7*UaZZG>M)b508iEW_<|g}Cv=tq$A1}7G>~EUFi#=eK0H;qgW^znT7fMBWo622JnoDUC5tp-o1Ocdx@X?x%m;lGm0gcN>v#Y z9)??)g`<9$j#;kjvOX1&?2|(b&l(JRxrI((hv~C0;KNb0CadZ@$4Y9 z%rK*6eb!;j;tU(S%gW;|_5r0uLKA5%tm|q&6J9uq$9 zEUYgZ%Kv01^o1HXK4_NbNsWGAxYOx$P6^J!Ft3N@@IYrXSsL+4W4%eI%3a{|#%zk^ z8Bo1~2T>S7;Lt1O%sBw`UsaZNX=&{Bg&4x%EQZ~N?ka!ioS2BUYD?E5uxsX z%?_e{&{#jSG8D%0q|>=%&f|`RK1%hpW##h;M`;U7+ggUVrmo}r)@H}2Zs)i$na0bL zPUn*8_k{(0`y7{DorA$2t?6_+z!f0HN6+{}%+9`mGu*b{N9c3{G7Ap~OO}J5A$zBO zzGn)pf7ep+(pVT;x~00qiGgk_ANh)Xk7u&EIce|L&*beF`n;OQBAXMC&4&0K!iS4u zyW``irN^CS!iPPL5zoT$8%lH0vjcs!PJC^hOr!qHiX*f9NiK4Cnk9qJatpim$$`m( zmOiuyRMcRXlxDzIw#7|gmIE{Iz|Ndy@L68%@fzc&s<}OLw#SSRrp~{&8w&z~y#EnDl?Yv#{rLISa4Wkn&spx(N0>94)CV zgG(QWhM}Up@|;9Xrf*=WdnaG7;4*(%?mCt2EUdnrk?OvAehcN>=lBHGlZ zf^zLY&#BGA=pn*AXY0BQM7N+rG;;E7)Hti&zPrP>=eIC@N?Aj4 zK+VF1zLX!{sBb^JYvJISpnWEESw{`tVt<9AF6XpL(1aSEx*aZYczHyVJSjkPVmynp zxN7Bgu0&w4rKE@lp|x4~NrYUdDz?9|;Mvi!%KI45K7+TgcU{C2I*UH554wf5aQ(Y; z+8;ZnbzXn@`Yiks`P(ml;M<2!&uBB^*~Bsz)`K^WnLOr(wVBE`o{1$s1m@%h*vaA? zu}Q9`h0qXK*4O!yeazcR4%d$tTNiv6`@S8bs7si$2;=!LP+jQ@p?%OS8KYPz*m)M_ zPjSwkg<)33x5_L`g<=+F;^gbi!YUgmjkM2%jckgV?Qgv<0Nt#f$J7Yb?GSMm*6o@M zaAwRqa ze@5EPcu)5toF;acc8JmTn*R(R{ow^K`;S_;>$3tD9 z8wgaY-;sFp5*4mrh8Z^K>si>`ks$r}V?g|S6tWSawHw1f)&<8bbs}9^m@?#A(ntc=Sh{ibhb`VgaQAwkveFjt*O`O?PDj7LQ**LTUa=yaKaBS-FO=Kye`>T#j&rXV! z>7yn3to4CRu2Y_mZMbp;gDkP2zFg6pF|G6LXS^f9KjP!f!k$fgbg3&%HI7D(wPS81 zn^s4)WoY!l662ZWMwGMt7-uEN)}%yN+yFLE9A9yWvOdNf$oeR4nVhd5FW5(byGgTH zNaMPltZo()+>Ff-p8xik)qT-EXi#Tao|OFg=btzWgO%!9Zw~amsnW~iAtQH4Ak0$Q z6G!lQ7xv!Yc6b&z$0+Hs^UAYW;SpCm!lyZA{F>+uL)+gM+MjXXK3_e7S-8{b zbULTuSGpqs3+CE2?S83UrS6Jcxy(KX9>Rwvju7Wje;_dQqHTq8?=y)(=&^>&MjwZ?08+R`=RcoS9jDEGXO3 zw+v!89<72 zno7W&n%Pw!1@RLo{0G#Oq(+MV7+|d~egKj*d^=8EqB0A6{#Wb}+6RsG)0Vm5=Sin? zx#Y9(hVJ`ua^nI7l6~qoxhK(U9A0H}*&bb@!&N{cg-jkUX6?Eu>1erTfe#w|+zRXq z@rjejCN#wM=7dY3d^&GaAbIjN;j` zbyK@U<&^CCukAovdMymEyDk(q*kqfts$-auF@HL<8IubqkZ zbsy*8`POR3!?Uw6{s+U$lTPPyImcO8(>@FEC$3?2CLcRT=6w`ewbfkr(41;EblTll ze0CY5PUjopTxQ|U8G-kCYY)9Vd8cy$z_n9P-%h8~>E!gKaHrGhbUK|*=aCYZ!mG{w zcDo^p9}qkZPjBwt-SRK89fZ>UpZeSG`n(+k;X=Tg!x`gx%Ic`S#Y4ZBQPCKKb1aX7Q% z=$D6wxD*}_?%cn`=mIU2O`vcqYws+wm%bFn-%Va%ts15yu=Dt~GSSgP`=guJnK%3R zZkdnrp2+sHxum7x#Ef{FknoW?%QsM6(2Hq*+B@pRnFB39vplTUlfOqrk)SS79BNL0 z+=`(Mib8mw&7$xv;jEX!UYrXi6{DZa4wOlbHeS$!tQpy+bZ&8hV#YC<9p`#S`AFLM ztA*M28ys<&(JxOvWB&2*!@q++M`w3On$yq%4jk|}YVMxCD_pIJ7Q;yd^M~KL9UgKW z4^NJizC{V-%@#kjzk#=?Qd^i_|M>9m<2C**j;Gf*!^7Q6`Ewm8A8xMc&vCqYeslAj zq#%EJdqaOU6vQ`Fm&!zaDU5&Lga6hCw?#X=cCT;W-G#P5|HfEbvJ}oI?^XIEKlGOX z!af?6a0Q`G?R9x(6_nDzP>phY8{0K4ppx!{LtU-Sp8(WSBJ?I2sm@7xoB>jUDc;Dr zF4`&cxnjf9H>zrb92LU%NLxJK^2OR*q_zz9DaKJ<&VF8;PUX|w~w;? z#ij7qx4->Fe_ICRU*G@s6YX$m^AqJ*!go@TJ+!Uu6y35G$MxAK*${j`>vrn;KED0^ z#v#Krpf=bLGh^N`W|W^(_=>Pc8BEk#l1(^3;^~5borMvWQIlv&TIcSTahvW}rU76J zbI&i-Ze{)nuEF~jcB6=B_h+PCDC3TLo`7j>#)CW%y?OC(I`4aN@ZmONzocVc+?0KG zMfT9Pw!`X*%;GHOEQO~UGOpL;3~bsaf_kcE3{2j&g|+5zL(9|3qFH!;b2rGJ&c}bO z&n&hd<-pUM+u@G>I{)2mSeFaUOW`bh1Pn9|d!QQU$V|Bzm)iIOP3asMQ~@=VNv} zf|6ElwKGc}q^twy4F0?XeZUN|6yFxvU(#P#^?rRX|I{7dS07)0{sR`i@4x)<`sZLH zyDR+R)BDdKi$A%jX!SG7gGAmZ`jzdTiUVv~^=W^YNx8>j`#~b_t?eS2i3*w33QO8W zd_~w(Hd%874h~o*(R`906JR4#M*|ja-t&K=Up0TcKFvAbnRc;5@M`QY@TaNW1c-7= zb(-Q2G_N;maq#f^H7O4gdH3{%Z1-FiErm%6>f|m2FKKvk1Ur4TgcF8vG7G}ck%B=Pf`ms$v7VRDvBdl zUZuHm76+-aQ*XAtT>^K;!)KZ@ACNYG0=CxZp;B#ts5k`5Za&LS*vEfbTXS@cd z#&c^9S@#gXU2N6%$*0mG4eqXvZ`nqxKd4{k)#y07pqh=dR2B2ZQw5rQ*xLa5!>0~) zGspt+Bn}%!I074G`#A98EQ+OQ@5beC>r%MY&p2MNpCs~5xvlL; zT_WfzM)hfLUZd`(Z>O&B10$@9FTto*DrJ3fwB z1*=oZjT`CL`&ID=sftO+b*tmZ#Hz}3Y*pFA%?5u+R&Tf7-$2b+Npt)if4E=u`B^kr z!5FK-UZ9$>uWILUAfL%EkjVRJe1dK>{3q}MuwWx~94MM0`5!Nf$M39&se8738*x4r z=RdwnS&$MZnlLXBKX2;COz+Q`g*uD~5AaFZ%h-E`_mUiaC88){<7!S;hI9p?b{w z-A4B|jK(N)fwTy zCx<1Zh535zEHR_kbZV3@d#|Ak(0YLMp?X*&)IP$jyYA_u+>*R-{dC%mwv_%D$BKg{ zv!6Dpfk~GWCa*H9Bh=DyDxYngbX}*riyl~#vGM_{cwCFyjDF>PK+j=JAb#`>;80;$ z1@ky&y4_Xj5X!|UbtO8~?vLNDlP6H0apG@jj(^o3q;`ew#$kkH=CH?67Uxc<)9D-? zeODNOi8()Y(aD2m@;GCwQ$pN((&^luPs}OGJ9UvfNaP*mH9L_^GOiDsc`&U;+;Z(W z`Yt2(_Zo)+)dFbe-3IuJp{3j)RzmZ2OBH+vVNcCMUIAOeFUWc}awE2W-16c!*)MC4 zsEBfAwaZI1+uRa=6qD^HKx8e~K@VEuZ%OS|JV3lSxL7-JOX3zNt%NwJ~PLzvfTP1H_GkhqV*8#^Ox!Ad&a+_`qh~uTH1a>70P8C%^Mj7$;_h%+cz> zW&VIKe%KKAgrBcF9eCPdXVL>rKtDAuA9PkuUCsl|SVc3UPo&_bsx?0gJFH3seSGw) z`sGry59})rV^9hi57lGrYIsnr^?{MkuvYbv;J${G6CtLt%a)YPWwU52bV7}zifH`t z#*w%={c`-N_sRNM8Am2oMR_!0Hk74V{*1IcktZQcs_6=?@vr+sd-S-7a4ZfUsK#av zava4fold9IIW^f`;m3`4erlEP5jfwfb=Hhi_&)en&(nwQSF8PVi}R)Xe{Oj3>2d3H zI-PR@m%{J<{p#w;?+?FTJv=->;oX1yC%*pv;$NQp8|?l5Uvcy5>aVEzwo^4 zB*=dce~;xF|0M4||M=fuuYUPI*5vX(K7Ardr_<>io7aEG`H*L%Q0^ptd3b<-Q23R% z1K)l3-S_zZ{pYJEPyg});`a0}e}&hs&vgB7L!WkbzB#V0{vSG$6e~bftjqua002ov JPDHLkV1k;ABQ^j4 literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/YOLOV5S_2.png b/debug/accuracy_tools/msprobe/docs/img/YOLOV5S_2.png new file mode 100644 index 0000000000000000000000000000000000000000..61357f3feb95b5f70d6317dbfef7b516c0492449 GIT binary patch literal 5796 zcmXw71yCHyvPD7&u8V9IC%6*`?iws1SdhhC7k3FP9#{gwT>=D$1%eaY-QC^Y`P}>O ztLmwlp6WT>)iu>~dcr;_%VMDe(Gd_3u;k^WR1pwfk-w~I-yk6%AWqqWnO+n`CskPq zgwi3A05>9WQAt*1aLRQj^VBH#VVhc)o&G3%TGN0PFy96hRLZC6eR&3*$0H4@x7j772B+%!MH ze1k6TbrJ{IOON=6J)vJf${Hgaf{9RzSo$u2HZmG*Ep31$x*Jvj!tv=$xD(BRI0~QK z4?2ZbLh~TvM69of`X8uh9s(u$knB11e<@AmG|5u~R-k09ScqxlL6MCS)`%$L;CKU` z-Zy!lGnj{}Tt0tV*)L-kZ8@c|b)v^chb0gYQ}~|T8-*|y&`vuM7Iy6ltV?}CE6sR= z++oH1y|t8)HxCM2n!m0oKiOOzo`)$WLK{h^barY_G*e;x`P)#Xl`{9M*X5iBNn9cf zY{wGW{u_?MHv^bfjJ1@HG$d6^L8Qic!w~c{o2+pkv8vs6XAIf36G)i*rJBc6~g zGk1d(ehf?;cc>F$BdE3CchAJ&M``p%KecnZOThHU*CYUj5I?!|rME<30Ug`oBfmcx z-mJfBG!mcZNnpPIZ3V+o4aqG)wDbrDc>ijoQrb46#NYn#HP)63;x`p5Z~#c?@tWm| z!?CL9;7LGYKXVROFcHHfqEx7Cw~RtQfbKce>9Q+3z0!((mr95H3uUQ@#{YR#~sm}VX+ZWZ7Y8UHopnn~r2rRdHln^`2|$xv2i3C%77bw0hv3m^aK z=;hl+08D!O&JK|)Y6`$eBY|wJ@H49@@SzXC`_}qz0HOj7iLM9-h`yrZTeO+X<($mm z?{F~NTjqo~l?owEKXe;&M(ZKVtn&h_^x&(Csi{uwwO2L`Zq0qMGD?tU!9dH$lw-As zhPJGZ=HV~=LnQn9v?S&OMW^)?`QXZu`=)O((~xsrJR3}t0Prpa5U5hnR`4e-2bpNb zQq3Uws!cymc4{?FQC75z`s2K9`(=!Pdkn(4Q8H_LTcoQ@hGa4~mS9CAHipsIz~<=O zd%WzgJ019F8d}>$n{|4O2^MB-pYI0Zkzr=0a2WwO@jhT&JcWm!y@iXVJE&5uyqy}U zm~=JyOqw{H)79gx4guq=R)g$Z4s|Q1sc2>>AJb9UCl;Ab*8I=uTj(&|rNt4f)&>QlcbUK2l%zgR zfIaMGu-wP~Mo?8?k<3P$w%{a7d_5ZM`22VCqcy3XQ6mhREXWJII90z0e9sg#JjV5_ z5Illhvtc#&cAN?+m*9$y3}HMNs>b$UDay|(i^8=6pl~1Nw2u!B!sq1gQ26$h{^*^2 zq`F~-bD~LrxJUV?X}Pl_3(E`HfZ)oVFz5KuQM{Q%J!wIJ2; zNX?2pD_$eX>>0GEU(f^9J~daAbA7U@gw8v8P|Jt4k868w_2I!KVJZQ9k}}DwrM+Fm z6-Wl_8(xvS4bc4cMXE06m_v-;)p@;CW6}2HDab+4n+J11q9*=rngG+umMn&X-N>Ht z&q#Z?;yFgrZs4zZ)ZgxAs_~bu`617_0#g&V^keFxF_^0uK8n*r&5?5>A`>hlDzy53 z>dWCNQIp&22K+D_QI=(36||A6x3hnVBAY0Ovsj$8DPL-RCtAN}g!}LM$;HKOSaShq z_OQTp`}+8{EWWc@1n4iTuLAxr4q+o5%Tk5XvY!bugcEh68B%2y^HdV<5hTAgqLG_N zVI$R*b|yGt%f zCMeA$m#3KzYw+!bt>9r}Q2HT~?X!+6?rxye{nuQo(S7>lXdJ&Y&HPTA)1zfkV&dhK zcHk+;CiX8l(7GmCP-#bxopzmbwu0RNdq^ICeKG1Stc@ z@bvDeQzBpCK1){9j@@hGGE6p>|3GKTcVIuZ#$Ms_uqJK1%LG)Ep-3l$+9{%_vc418 zd91@1wa&L*F#FA{)8R2dnN?a~DC0hrg8yZg9mJzEqG&Kc6m$MZZ1u%1m9PbW^JR{V zJ#w`F@b>aHTzHhi514w4*Bme@%s4?e#dnhY7Z^GUx(}e!)!BJDgYWy2&D`|27bncQOl>xwgS>Wp zKU1{I7I9w~H;x#1Lmy_UI%7*}@KpFR`Yn7|9YLXWt~2J%c2kX>Q$jmE5(3i;0u5H8 zb<~3-u4ZX-Q7&h5Ox-9PPat*zO)uU#B&b`AnWy4J6NfTN`x5N;L9ULakr>5`Y{ zR`=ok$vWwk4PG-=i*}B&H4t;fNr0)>nWQ%hHS`YX(GK}2Bt%K&X*iFK*Vd3{J`rd- zhjjMg#?66RX_tCvH-BKS5gdvG(A$>o5@Yw|-mj&4vsHeqV;{^Y@%F9HQ4bP5dq3kW?}$@okUu>TkiT*~y)suO4*fd zCZTS+d&T!-S!{L98ej|G&}#1mCF;(uglgyroQ z+%x{a*>NmeD`Qh%!J}yJZFVOdfe7tpRnT+&L@-kdQ!uP5>O8Kp?>)VR??=a0tDFf` zPhMl~vGHxG%|L@joEZ=CsnEBub+oNaql)j+WXOM2GS4V*52^2TgqtP6e6u#)yz`uA z_}^#S&RPsZU?Otx41qzkh7615QKyWy0Ig!yV3p;NXdFKCp!SPteGZk@fxp9H-<$&s zmlIi6gq$15c8v5+sshezhi9n=i==`6g+B>qFpX>>&-^UF;%byKfY{JWVP-dfQ(g0? z3uo$={2He)_!F{i7n(>{Kt?(1wrbLr3-x}_owkQ1G3G!4y&R&&NsF2@+{BQ_T&fN@7zapw;hrbRW-l%wLWDU7oD zqKN-Hs-mRIUixb<_JO=SKby(h3P}1bi=co#TY`nNWxt%mr_?nk-DNk2I1E)J2f3&v`@w-6uSv#cIq^)> z^n^0@3vA;W)V#4Rb7otCYXqYHQ|1`m5YV=ZSQW~WyPR9liBLfdU79=b#LF{&+t6c; z(Y)5>7I0Hptr#*n1)c_zbwg4UloMIUfJkO2joZo@)d`l(csOOve;i&8VBlS|S}Yor zuqKZCs&&0U@Dn@Kw6fnGj&&PE-Vuk)P3pB$72ijM0pGq!=U{{Wv`jVeS?;k_E}9K56voqpIo7ZwS0wN$*n@xI83 z+W{pRRtxCWc5-*Ma;NcO@0TxmFYec$3-REf%2U3%>bB`_y5lbXX?7V>cQ-73b}sk0 zn%D7{AAhDSy%sgq`H73416w4D1-#|`IDGwlsSolX3c=Tw#w;Sa z*7G(Jbvht;oO`N#2VOj`eJ?Of@OzWuW4tP;0G5!SPq`w@*|4;Mn1t&7tro-Nz)w5^~p^Qt(S;WY0a2 zL<9{*j=ZRxNA5txQe2*`N3*Pm>UG+#{c3M^u?P(ucWl3_^^=?HHI4w9=p4!Qhs$a5 z{2m34X-Mcg|3!XJ2t`Iz67P&>8?4KzUb{h}qr~wg>h&c5JpJ7Vx8ru< zknZc+)oaa|lzEnx{$qC4FhmIYYw-FHw)M zED40HDkAV%psMj%wtW$GMur zswXg}K#^Z=lL{Ckxzq{4rTFU7bC8~~%lc7F0PACRaN!lh^4j)Hh0=`aZ@{jkYt4im z5vha%#b)vWbT|LAU8Rc|1V(suP6y!FbCh{q3!f7=#lW;~{D$C#Ng$v?o>y7F6F~Pi zc3r#1i`$N1PkjneI72|cymO8!4mC&Z=f;S)-+|+IU!+@7ny2L1(+r;I6vz&25ab2| zK+;jtK_L{S*nX?@&}JX+oAXoVcQLDrEpU1I@#E&_HQfwfUjk7#;l@YO(oJg*g79n; zRY2zNF9MQyrc$BWhYb|ny4n+`BW;bQy=XHN(z#+C!noL3V_Mn^OMbqM3ieBu{SO#-TKJ;h2 z3Jqd(ma&+)JCqR9cF(;G*OgYo`{MUrhy$fHCNsXa0ORPU(Csp&bX04AzX4FtH-c2H zzDun{MQk-|Iqpi32m5KxqIDjLBA^x4iJ^c5`$)wQ>jeE1;!q={6BhN#z6kCYR>WPg@N(|Pu)T=YA67ez?R+? z`bYX7H`k*#6F-D4ma1{L2krg3-l3vDdtC?QBA7f}qn1z`FpkAtGq!P}Pq-#~szt5_ zeB)D$=wLomO(4p}y=G~Kw|;r{<~AG~laT95AWBN{r24FIr*;~aO(HR4)x~ez6VY4( zPyQ;#SET#R%~hRHS8hh`{dtwup$d{zn_5}(sMZWD_bG9NA45=@QBEbJ7X9sy;3nQ` zqx=Qs06^yasXuG=fjSrKC!LZ9+f^+=V0ru$wB)O6T<|Dc?-ob#&KTvu?M?Gy>wT_@ zvKvrW(56d#`rjn7-piA}TP53M6~we5noKUzx5#B3kIJ$it7y>$VQLlS=(vwcY^G;Y zlw&}Jf1<%=T`Q0}mx6u!Sq!>Z^ZhHqSoi*Y_=b}VN=#7cclm@{u~TuUXiwz~W(9}b zeN4#N^;W`k-7T9c{m31WfzX=R8czt_Kl$|aHmE9|25V7-t^vO3(qF_ARjE`KIYc4> z(hHPhzVV|*(>qTyY%o*1ljQ4ZBTS)^v+(~o-hyU_i>Tv?+FiJ(4Q*`uQ0{GMd)M#Q zYvU7M4d$U3cvF6yDVyf)%{#@BB3)FG{<65!Jw$lj^9JZhHR(EcDA=O=%4SdiUF{Ju zabJaYu1#2)Bd>Rf!~UuZ?56>JEpSIjuI~RwZ;{J>q&QIBPB7he{QazjES;&e2Uj+4P@epgb066*J*Q&!&TFUj z<*SgZ9#^zBFo@;T_d$>;D33(}CelwRiVJAH2b>)wDgUDPrrhhEhd=Ca)?(xSEZigX zQ5|CZ<(AHyQpQQRoUhFkPZWbjW+l(@h7&!X^;TEQ`9X+c20By^ETbJ7?wqYXX2k98 zZQ&)E;!9@M$K~_G+b*F`mId#hZp>UJTR$(XJ&Zkyeki_4Ww7h~z;@NB+ydf`Co+%F z)&8en`&SOa*jZn?q<`hBKO9anb?XG%Gsc07yCu-~o?>e>k{-GtzVL#7EVk{lT&5-1 z0O6B}2dggax^@47@RcQBjZrUq%ow9lQ50mR%dPx}KUDONjkRMt%iS`D2)o=W@jQT` zoDfs#ZkOUemk=<@6OY~h6r$FIe|f_{uq^&yZOkxAcQ(7y^=PgB2J3(^HtL=Z#;q()Q(m7*ddH7X)vfB=yaAc=xYRJuwpQRza6 z^q!~)VbP+L1QJMu009C-2uVo4aqn}^+GpSY-)Aky9ruoV$ITd&Fi7%#^PTgVb3XHV zzh{^2t+#Df*t}-Vnr#G$i5ch%A#Yav&y&#bBFQ<{}NS?^S{%6K&YCqD+854QUJLhJXuS|hGugeLVI@i- zT=ha(5162z)NWk(#(aG@k7#(U!Ef!3t%hkQavyv)j=~?ZJo)I!Ztzv{DptakV8lcT zABbMXk}-C`mf2V%(f1-%%1T={JbXYF%pbP{{Ga{^z~dznK2<`KM8cw?3KkG8;;<9F zNV3v@|JTuq29FSTgpf-`pd~zjq&WM3{~){7E!uUiT5?zjFUlsse4pspzxhL>y0bEIqng7vb=;qQ$&WXkeC{6NuK3vOQJm7B`IP@$M~ZgvP>}zQ z<@LJ3e_vR$Fv`M{OBS((N^r?}3)On!LQvIfx<;-b%Fd@af*O#-Q&qVe?i;sObdeRH_=wOi8lBupxEnUy(x#bSE|KXf2)HHyH1EJteN1{L(faCd@~ zMY0chMa3b`)z2Nc=K4z}yG;nv_KH8qM|_i~HQ-5;TH7i$Cd1WX&*hyDWFm z$70R!<=$m9x;EDrky6cy*d{tgt+kMB%K?&eFaZ?}{(RmeWDIB7S}4?LiA4Fnk#12B zcAoC=TJ>Lcq8jJG9p`guW!pV2gK@m6Sf+R_)H$>^K~)HGm0kNBwzU5{W_i5!O2>ki z=&UFAdu>Qx&BwF+oNF9hpA2U~X4lg9RgJNDSuFffU~Omd_Q~`6)j@Rfw6oky9&4=r zD8TO=c2e-1>Nw_Q28*gu9Y&jXN;ZRm2#ey50#&cbqFcG6?cpt&syp2?`gI!1kpVdb znit5e?`t>zZzk3u*@ABa!yQc`V-|lGXpPxCxsd>_s?)_8n)$h}j@&?ZnUz$hSvD?+ zk9vfjgK@S#Z=ozfnmF<7`MG3H@4~Pv_en&-Z8D9%h3)=DW-*&Mw6Wcw{GKDm$?Ig; zu9iv%ZZCSNW!$-7+;I4W=KkJcY%X?sl%a|la*AK^YGolc4M!_@T6}$ylGlDy@Te=- z*Da?}R;wt>`U*Vols6agrMG*CIcXvA7Nv8ELwd{Ex(&(j)j6h-9Wxrr z$hw9vcQ25Q?w%Oq@jfA|D|blj=00AIT!l!MQP^+FjdbA3h&AF0%m-F4EKilE722lP z-HJM=Qs$-A{ODd{T9q_LuqkDlN4L59?6!VO+!9pb_4Uvm#uVc^x{Ab3# zG?ai3bRYC?xupd6c%$SP)VJK`tgt&F;xpfP#RL9Q5rEhV{ftL#9^_dC`m6@^I*iU& zl-@vvZ3+K=*%5{u8MjauaPi%r!(?v4!%g$6@4hnOC!~> zp_e#>>76n$%`r?teO)IKbw~BppvJ-`L6L(y7ZZZJ;MwHSWe6sZ5%sC1^BEh&AR06k1_h(kT{2gzOa;m&eL5`Ok3F9}eL{9Q8 zg2fj8*|i2U%w7iDWVYDqNMwVimZ(R--AUmugB+$zmPMzq8!@+p$>c(0kJYrm>_bdR0($t5?K^zg1zx_%@2avD&EC z3&l(Udx^jXu98(6)Da2oE!a&;k8R_|tw-;a+kVR!<+UVOnIY#NT-h`s47e3gINK%8 zj5`BL$u3g>0>UoIXXfTs<@2t!65r{&^SI(|#O39xLoRnpT}xt=pkKgxQl zn0K;I#U=TeF3FdAqZk$B%5sV=M2;Tbur?{Rn zvK@-`zSPyz76J3iCq?+BRMSX6lpO6{6Lf)qIYnqGwY$cfQVk{u&*kxY16r37Zgfx@ zRcD_xIrt}VCa12c?mpjSQN|cPd;-jkz6J+I@g6 zD%(|7BMoRhTs*~BqDP9>S7HK;`GxTQM!_OTd<)=X6N0!a;fDu6c&z!zh5-8`w9$y% zXD8cEDGBC1&cQv#OXD9TnK@}BJaYof=3p?WrbByaz6q_YW1ULt9a}-mCfkUEZys(s z7P9CShq-4qsJqWS%!`@QoBAMN@+${YkvG+bG$&V4#$X@ak0s@2foBF_ERa36{Q>&Q zl~H85{)#sh)9bbVS}m-pI3@Rmx|xB7x`jFC6zo~^3iK}<)o{Nxdw#HX6|i7Z zU*gwP4jb)dG}s8gSiPz3Om#x(RZ-iQwv4#~&jY70_58$P&H$vmCy;a>H2b=Vu4)ED)r{U-iU8(kYqw$F^|nqgc;YpuFC+5h$yvx0sM5LCYFUM^)+kw( zaj(uuVR{0KNi*yt9t^?O2(+^4r79>ml@=r`&Frshi%_gG;e3|cK}i-8{j&Dp*8s5H zkeqS~{@8YhaG+*aS4Y^Sd1pdoj6Xmh;0m&tE4!;%7C}{BV^xz^~8*@I${cO=m_UiKk+MtUo#zK%#*fM0c`C`34 zU|a)4{~l><-JjNcZwDJngmtzQLSz#kL8<+MPEu0|a(`-SA|Cg49=@HK4b(1RdyC+( zF|u0_+=;A+!*8!IM1J2vo2SFJXVqgDibX944%+kZ&=7y1yeY~@7lgkRA$}MaUt8Ot zsno-j#o`@$;yA9O`sJmxaq21fX2Xhug~%>%+%d`%oz89rpFW<}EBF&Q{FT?i;_T)m zq#}67-{yi~d-iY2JQ|>FL~+|9Ae=EQOt7l=+&_1`S+S+21oufrE_UMl;k{$x^T%}8 zizWa%t32pP!f6LY^{MHR2rH}=Bt`pph2GX{eSsgJLaHLtNL8VQ{sDsF0F(NHAs=%{ zu14>T>wYO1sORYV2d1$R4`j>IWXCzLJ!FH^7e_K&NNQD0bHQ*CMr(5%&F*4vO5aq;_rxi7z3ullVx z2hsNVK2aQQRBv-Hz0o{g&aw8u0g1Zj$1)Vjq+wUttERvZQblWm$TvHChnMaG!{S(R zU1BPOu&fUQr16oAklD-t;WVk7ZADp3G?S$Ex$&Ph_4y-NO7!dX&G_c>J9v`#S*pJN z7r(PcxbqI8)x`SROSk>^;gzG%0WFL4mN9&E3AZUJ3pk}&=!-`opu(2ye#TpovX&hQ zKm-6m9h^I?VD9i(5w?!*U3|359(1ckbb0kzFz#fwTy}1`$EB=0+b8NL3#jR+5z7$O zA^nB8pxpsJxS*r6lwc)tS}k}Uy(B?uA*0OiB_Z_&PteZ+(zy_Qhcxc_cD}e5*0iWO zXH3w6Bk5M$r1yJ6Zt^z{XC49d-`W6_b#pE$4vcd;UlM2=P& zG*i^O+xd>X?%59SbRgEwDVpIR$T@HPn`D`HtEP2q7u8|580;`=lun#U`JPNl?l>(^ zJ4|-8ko5p*pZdUnz1Pvp8=0nMGY&Wxs}P-U8Elf@!Z*_*Zy5`|rKh(~3aZO2ui!n< zJ#lAjF>tj^xZWI5?^uw=sYA02Nw|5@?-sV-%9B^kC0Tu9P;uHqxvySuo8DPtXmevu zNhIQ|^Pss~l)8i`q*x&(7n|dLD=R1m**Dtg@pW|vhn_)^R+urF>T5eWFRQQIwF?>8 zIY{4)HR`V))~TN;+YSPxRA6Un=H3tC+#vb$*?HXF-6sRZNqf4Vnt(&x8t!wP zLSEW<#~VEq$vm4$dXhdvesg?m-NzZEAJBtxF7Hq=iOL?ooW#WJOQnJ08XCKz3L=td zN5ZIcW$>Q%z8>Eumu~vU`>J#cQz`^tAbQ~Y*3MhvKUC?3N;-7CAuawt;pHQ2h4^ATXjla!x z0a{6v+4UFWqgR3N5HE)53g9yZk|Ku9fNZ(*)OA3CUtq~m5XdpUao*x?5bQatL(ube zVLlwbtEvyMc(mgr4BNi0OL(-Ivuf;yf3c6aAahFFQv;i!F`HvSP6&F%vV)O0jzsOb zSNJ$Sw?=bu{H@;d((#(GE4X3eyE~zaEqAwuWF}gv*79}jsVBk|ACl9M0SQWp%vJ}* z*}esiH@#)tqF|I_jwXJ^+s2QGbAsXtNC#FWxjxAv&RTgHZ>J1ZG_6}6;rG<#tE?-d z)Ca=-tF9Nb*~YAtgqP4UKRVPO@UZO2Q9wdzbV%j)Q=* zKpNxju~~<%j>;> zypjhz8a(v&{p*G`Y_^AP){x!RAbXR%p^$>1zAS<|(R^O9t|7UX7m!*Vte@k4&2w{I zpGOr1(+T1XH3g=snsh~2VZ8XYaUWiIXEx%7yI5T--Bf{DQG}eRwJ)^sATVUu-FD)v zwfmi`BTL`zVUSND@0Jbu4FO>kqZ3d`E^-U?Bcm=x5u zL);Wws?|%8rM6=_%hVvo=Qq0Yw(W+id))7?OaQ-a&Muza_}pa>;8sr?&aiGqKY`B# z6m={at+XeH)+RYeo#%mQ#bux)=!S}T-Ze}Yf=jt)%&%@gEVo`8-pA^yLv4tJr7*O_zkRxEtQZ1jw}0*n z^(ohL_4Y(zy17F*R#d&P$1mN*vB0-s#DsF&q0v~{KG$Zvl6>;ER;7*JK!Zo;AMoI} zbfaXB0#J&r6^TQbWwj70$EsTDoi#_R{!O zJ;CV8T_9&U#vUS^^gh_ZO4{Y6rIaWE7AqG2KBgfK^(F$eV06QZ4XrP~LWEetfPQ>y z(;|%oNcfud)u+Z3vI~mBqcO}b_#9nsY{gohp?a9_zmy!|=iGJDLdn+ZE^WWhSK!$u zXkwyes*Rj%G=F7DavlDeEYkS~x{O1wR8JS`6U7$*KBZ)!7}K=z!Gb(&nqGixHuT1o zv6sxW@Yj%ViY&2nk^T{}Oj$X>ar0_*tvJen6mJ<+nM03Wv6JWpH^-9O8!pn#o-|GO zsoR;`t4V6%pd{o8wtGJ&uVT`?L5JRFmiS!cE1pF9n_x;E+~8@t`F4?kQZgu226=lmSyLShpsWLm`6trJy_Rt zczrj^>sML1?@?D~_I`L}IAt((-F8&mQdN>*1@WI?KHS@wQo4{mkuiiv%CCjwemrij%b*f%OE|>U1R)H z;li}31KiTQlT?>3a{)UO1%Geis|Lc>Ivo-ZIFKgPB z@{VT+B7+t$<4af>n3-zjALVTn%#}{$xnwK4v_izjfZ_|ZgJ8r|>Aka-7R+U{e*743 zWU1FUD*F4#4&qAn3qwL#_`Yh7T72KxFx5P6WL>QwFmMO4dR4BUM)8q6yvKh(`i}Zd zr)!~m1|VbWO-R%x@KNS)%w_zu+K}Q(z_L7N3i9TXX=47H^2G#fMSw?iW@=MZt%DrA zX_>)Ri!$iChtlT_14|*N?)4>|GAc|ms&bUmnIUZk!OS;idA&6-`shDSSLXUE>z}7Y zsxG5gCVb;sbO^Q>xtFDIxep!Xf3LSx>Ypt>8|vu2*DYGR^#FLAK}xpho6y8Silb7J zcwX{M>Y<59A?0NLKGqWG2)OW58FW&3RPHNL8x+*Y_7{b~_tp@rQmFHqeaQau^1@o= zUT9>xXI%vVM85{Ro;&bTX{#gMz`E+bd|fZ~jx+DBDUCQU_BfLr zSAWMrJKQ_YtAhIyyjvqUiIo66|919Vru*k?voWy#UK;Km6W4-Bm`OSlVggpejZq)Z z!(Fd(ydpDkC_=syR-(r5jRHr%)pbV2uh6e8 zxN5vT^!1QVK|z?JY8^wLa)KSydkirk%NkalJ`*Xt!zy7}^d#Xa+LV*d0qN!O&mO(q z=Gb~^u2SA|`KCp^yG+;0*C!$tSDM*tI+!NdHiuKclQOIOr&ADZc;~0fIEIr8{`EbC zWq~pym!1NzQ1#L}G4kwXq-ZG&Ucpt${uD%h=;uzY9BsQv# zz5GHY&jsQcJhKC2cHjc#`Md%nbV{eAXQJg=Ol>U~McoWwme5Z-W2f~)Yjd_hNQ)FL zWxqUu9n3WmS)-?NQzxr7N7FpTdPo%tINv&9aUj>tuTl@M6OAoRTsrMP|8nVGJbR|v zMrF~J@zutBV+3+!9eMGMe^-9MrUnl;L{pP3uROjxHRLsb+saj=9!Fa%Ku)~d$sTlw z%dmz;rn5jRU$Pu|OXn)Z<)=owiiZ|E!$|eg0<{m#R0Mzmh(g=XtqaRN`C0|wO-)*Oiut|c<`tj{atncEvPfNNy-LYDxeO}#QH^d? zi+cpTqD*!jo_)KcJeQg)XVgdgnDJSEEv~$2!vU4c@gXD70neiwufTPF9q?s;s`*7P zIK^%-tU@-u+Jk!v5O7&=UQ{L?kmpJKqc`CwMjE(sGMA*PU z(wmk58q-nVl>RB~i>F_{#Z&gesfw|;`=qoaeP&LK{=d~PYh zUB>98I-Bid^8HH?1>JL1CMwM#-cxqiit<6k?7Bd|@gh}L+pw+o*L_UPnFCz?6#9^Q z>X%m3uU6qu=_$0LC6jbmkh1c{-Adfw2}2UabOa$5~2q zB%>Ij!fNCW&7t92KeyP^3)*`1Q;6X8KL$4B_s4aeyZDKM&qr(sk7Jk|39R^J--X9*vlu|`fQ0r@oKxb}GE3YQ zqJq34fzB;Ca$!dH{aH?fdO6MmD@phW$r#&O)JUx5S!z06VDX zY}lhK)mQI^jcfO}6Yh0it|tupCN?(QB#2pY97?c^wCC_dg5+{!Ld(X_kjgWaR0;;wXt)BTVPnWKN#gamGL~d z&4H8-MnUp*l&wjMVu`g z_>v$ROuus1U@?*p2lviq-twEI`*_u1^;J1WWTnp;x#UsfUUKHs<8zK~f@pYOpD0tH zBibr*ihuS-_C>Wi^A@I#Uzttw0vgrTvTy6#$c>zJgR|VN^Q8GW+VRz(HhCrU) z7Yt8uEOG~4gkzBCWK!Ck93#ZN`gQ`tqQ{wiRF#w}L-(pwsU^M-9QM#~R#H_bv`$92 zZz)!44l(4vRX+EYegF16`BttEfgJg`&n~{Q!MX%@wmuD4d!Qqte5pE2-|#g4n_CG=+g10?dk z$Nneqc^o~wH&2jHzHKWjjjeazGhWsknSCCeTRc$)5kyYC3b+>~q>L3$y1fD#JeEd| zJVRedK$D@0UU79k-XZj;0a)kfY!vdBJP$ZuK-9nk_rj|TO33OSc_WwUrwl7DxZ$_D(RWXMHntLUnLI|e zY(~5w;omZw>zrWO{AOYNF?dA=*|Zo4Vl>6rQD=*V@*&(P!#S5i7xYW_-07z9wONQ= zS2N1CKk?_wTBG8im@g7fK;#WcdKa}lPw_qS^CyQU-UTk_L{Qr?&yrXV##Ys|0Xv*h zj;$OkJlx!DIOmz{wr(@pbdB))Si>XxkSxib!PBFiMY4xBv?a82w)x#|*jb?pUz7)U zAOR|4idzO6m);z1%xGmBSuDtQUz+GJlcalkP~se86)S}w&ykwMU45_)JK=Y?mz7Cz z+3#~^nlbilnP`%O*)ilSc=9^S*3U6+K znW-Kfe44uGV>uY4Z5IdwjIFvWDwgJnOzI3fB-;Q$HNtr2$zmKLFa?&OUEdM&C^2wo z`BZ%vF)Q+UhGZP8_j9b7oVsumR$DVZ_ zo(IVB+a)!~7O;4O(*LMk`JpPwKzwzOTUCiBi1tdp1p@GkG7Z691f@=N}*2xuga7XZ0iyVAJVcDefzI2R!cN5dlhD} zh$gRwpq8fZ?0hYZGOtEMa+9Wb)~Aoa?F9ieB8F17#wpW52`ITf-0FRPhmHtgYnmnx zj=amxP@bgI0h-H@rsJw-Ge6!SBv0?fO|vwHP4nd3TD%nm6Os*QzlUA$Os}mYUSp&C zv_%zDjp*|0f{D>j7nI5bQ3Jk%s(q>rb1gP#e^RXdlSQNq^0%Q4-5KF+N2Bk1 zf0(1O^*46san)w_u2`0dFUy~bd)P0skim3B;rwVVLeMP$@(R}MXs2j1L4Ot+_VdlpK5Uv~A6v~#iGdvrs> zToO2iD|QkqxSKt+C;lHm_DF8+KP~KklG1GkG)E_U?*|>?$O3ESAHBPK=Lo>)my1K$ zh%4d;W`+KPg}l0(fvTKXjz{^4Fu*UMmT_7ilpv2x>2_|+g>J$X1v zpc`$;4QHX0S7u`@wJob>war)%?)n1Psrkuknd>5>lTrIUUqScZtm71U33J{0sAeQM(VNx9s_pK;)SpsBf_IjUa3(6?4HwV;)q!qHTh zI^4wEwXxs61`0#L8&){$U-4f%UrDM`kY^q_QK`8p^RUvn?~9qp`uuX;8N)#%ha0c3 z=bg}Z$)I=3j_*RSAn4*z?aF)9x|e#U%0B}MGxXxGGNjd`HX{>1lc(|cHv)eDEXtC% z`Tt2>?7tE0{xiy0S9ChzYAri=HFWC#$(Q_H7W^+#(?6@={l7@``{!1wBZ|W0amfrC zEuS2~wI)?}u-FX=zF@Q<-@Y+KSiBT923VZMexTDrnp1X-;C+lNNNu;Rfor7Y-w!>r z)x$quFlCVNbQ*`>0EpdBrZY0(*g|UhxizyITXZ;B>N_eiNs_W5Q= zB|8hC?5gW99w!P>Ol0G5^+qw90<4TnTgb_Wq`9wo4xOWnM=_j>7|Y>=#5CVL7h_nm ze+ka@d$5_smgAH7H`?G zl(R^@N558Yahib4lELszdVqG>PYtcs=zyG6Pi7B9g<5tU4-f60t1LhZU>B3&s+eEv zy=p5iB@zogurPQ%DYcUH!~z*)cCL{{WE=9;JM0&=rUn$yAD!r}(pU-KdGn9t_dgXI zMc*L@HjQjZp8jxCb+^JVR$~;d?i4flfP<=BMOQ=^YuTxu|W$CO-wsc9lhpTF-ZSoBJ(FOh0 z;vqIV*>XKb{rxa_iJifn@2y*)X`iINReI><{Q;$`DSLx$S+m;IAIX`%nQ|AE6NM0( z)u1T+hc0C*+nUe@t*KC3Z}d>@#5E5EfxX}$f%JtAK@8Xw!LeU%jXl!obv5R}-}&J3ZhE}RoXh>y z?)NMnAp~{GDZRs@lODArxFF!{B{`qEqQ-|-^5>v=6(n5%S2NBnf33p zc2+$?(RN*%CRGjtHWs*A9`A%itUKv2V2@ojAq+)qG>r~Hy#@~Y9DUuuA1d;h3;ygl zuZfk(ASKfWBxXjaa_jYKA`?RD{yUv6Ex*L=oMqky?Z9NirbM|P10TFN$Hy=Ol5nca z8C@;ZmFmr_TDo^(O+kz8+7j>TiRRAPQ>@IrJ+lT}GHlF6r#!Z$rI>=l;q6t&!)+V` zcnu@>+o~0PXwW4zENv^hRIj|b3x8t?F%(6pk$e@qc zJ(PcRy)<~JC)R{h;L#|4@PXHicF^WA%_L>CSUNKe32wnSl&gX(z)nq6b8H=-**q=B zp3P&)%th8|aHGJD0CQ7#iOBfdZEI0|hlC?sy$=*~c(n=td{)F!FuSPL-+CH>Jk8xW zOF4pgA{;^N&3GB0?8ATU=$W(%TRl1#9@&!TMQK}V%ySx1hP0@YP9m=V&hp%dxhtnN z1!?f<&Z`{seVfajEwgfCm&`1#bL2NBi?#h?j}sV8li?mDwkbUPcXE9@_8)tpo;Fi@&@V|V;jPatxo_~nO-js%$?NJ5s*TjT4f9PV*`)- z9VVV47l&_%b1HlqkA-ZZ)Eq0Owf)juHhTr_r@yE&V}MU+7ZZTCPt1D+bFc@M9;0^( zs9X`FU_Z&2aI0)vP>Uq%k&ycq*&KIAA$BSETk~9UI}$|@XZ-wLdAc0Scvv{v*jia|x_Xpw%!*To$t%#kMU zc@P*}1FQka|HkwQN;QGk-YPWrd>E~t0vUoz@}rZ#x&WbTcp>+`^bW?1;|C+DlgRLC z)lZyS#-=#Ku$>aP`W$@=yz0TOcFc7TE)lSI#XRw?J(s>#4_wZzZ09g6if+9hdeUoH z1RQ9uYu3~1Aevb5YdE>#%TWIQ7 zm-@z{vF!=cihl|dNOmOno}wq?)`!0}l#wcTaOdIt2y{Z@BSKqcw6}~GgbasCq+JOMy=jdv{U?#H9;>Tuhb*2(kc{#M%9$54-4=kxeU^iHKvQgJA`>kz z9x2{+GV%Jd${S;+YMV<_eL0-!A#twg#MgTa!YYnt-4$CWB=Kr@{c(t_dZjwbty~a$ zN*P5j0pYLHevO>YoX1Zv6Ofa|Pc=KSy<84`4#wK->>-CexCxpxv-AbBbXgx*t`6pV zPA;##<3h=B#(e1Dv0@WY%Yv15;?a(!S>buX#^nfN%p3N%2SASjsEbETOU_b`1=E{f z<-eyr_B~C7#tm_RseokT*aP&+ZhV&vBNOA{Lb!C*jblD+vk?Dbes0?(ym0?bn1vq% zLy6r>PL16*j<|H8=HE1bR>h!S#Lhf3vCEmkh*-K-ktkPwKD5I6Of(HlL%cZ-pS!Cf`& zJ`!}Uo9`Fv7#r;OD-m%rN|4%*)I;I0I9^AH&iw}nZs(0CR2zeBOe=%7El!Vgn3+Lf zTkMmNC|>hyv4bfh#)`Cytz*{0uM3Asx8QUs{B-R?0(B?VeUUgrk9=X zTF@F)cD7%|2Ep9is+3DtT82#WLk=AEK-?53d=qZ#bEj_XOF|~_nx`}q_2z`;L~bul zzP9Wi0Otq~jfc)x*RlqTBbk}L_Jbz}7L~9+ye?yG1n@DPoTaY}xC|#XlG1*>;O!q- zVKX3`Yk(4vz3@}3JU;$E;*WoGM^)vf_Fi#Kc7g8Ss@1f0TAYZfMh zi$Yx4Lwhk(HlkCJK@)t#V7Sn)m%0tmTi52gC#MOcnT2!o*o#=#&-S^SFWIJGxFt;nm9ikpUd}F3p6* znnkdifUx{bB|LOaS7AZg=|G26gNZtRb5 z%<4_^x=8#s=~+u$Y{M$li~=jXYl3)(aPhSFWJVm*VVKXGx;snv`BTbaX~3;C%=&ib z--Eo`Nt<>ARD$Y}C1IX8<1w2_13=Wh$Pv35Q;=I3`0fX4G`G+y;fUAOu5dfD#7XZw^9-PeESmRptRp@Qz~p+4PFD$(6hI#nn@oP7 zv9-uAaU_1sH=BseC4Ms}7VEiFF(=`M(|Q75cWSJ#V+GuZk-@(QByKototQ10=GDsllPLxANIQEX zc@a7pF5BOP@TCqsAQB#o<$?7k_)Q{PPVs-B73|ce5xqd)TNhQISTy~n3kULcP_G~k zclSAvcYoPB(bnC7?Jlq|dNEqf{E#S9^u@WxZ{~UmPYLA{$oep3jZ<(&-4UPIvuiN1Yz|>Pup4 zXQX8=@jM0mj=R(z8op0|(xqGrWn%47gh9VHTt7R?3+Hj6Z&j9Jn{g2O!fD2rw0^qbKzK-0#I}?l)yIDorhg#v zzf7Nc?H zJ4fwVIG52H~%XwXZmIsSL$GXDbr-pGaSx;6gWR!{5pXTl1r8q~$j8#Th%mIMc zKL8uCgy~Q~+GPa=4Zx$HS}&5o8fC6_GDiw%E<;{1f~ zTQg9)fAap0qxW)wmGVyoXL#9A!~?KmhjUr*7u_z~ysKGSPAS~qBLar78W=u1uYVg5 z*F;l_NtUSmWxlrGVeby@ry|JrdFdq8fSRIgY12b1n_~7hOP~f{2%En+A*z>b&D*$h z#7crK5Z^R3Z9#3b>>P^nl3Z668&{Yxe+6cC`i4F|4BD=g z5Bc+(A1@hgrLPF0lRffw4k~u%{o)5a3K&KjZD4K=+xb0i|HPyHcdGNOR188|Sq~Kb zJy*P9Qcq@kP3lRT{=Yo<{`&vR9R3&R{r?6Hhs!E23bR+WB_zpVs`$lX`F)TK02LSP# z)>DveOxTX&0ydl1g0+x}5&WDT=CM>Dmr$9gZ^O%pdA@UvouEUgvFVwmZOWdX3Ex8G zre6q_ZJTWSz@`Mt&WD`Z#0`~L`J2M1mVnvEr*7}K+V{_i`Jc(Rf0vJ?O=fiRL1}!s zSd1yAgd~!u)^Ajs()k$-B{^`z_JO&?d{L~5;#%CewtF?qLuDZG{0DtX=;%_t^0qTw zEd#r$O{%Qf zc`S!|C;s7WmEG|-Z!h)V(f)7)aj*=0rpxGOaAbt0iFg^r3n@Q{s+SWiaI$F6miC}{ zO#iJvX@TF6{Dk&LBFoMx=gP#-Q0VDOF-)EE!dCwC&Y0xjw4W(dW;H37J82M(5bMvoK-bjphAO;HpZp~a5_s(Z{UJr$bn`^)#T+Q|z$_SY45%gmd9-2UoM zf{gVf_2XYx9Cl*Qp4h}|L9|#{yj$Re!?xojO453jmM0bz4?E}m`5Sj-{D#ES8Srf6 z5Or~?njfBrK&#wGSV&s}Y_))=kE{$ABcy2pAs(0b-Uoz4-Um=|bVg8f>+xDSx_e6z z16jM;&l2-XWSB`Ye4gp;8*a2Y2V5_v58Mw33RP*wJK$Cr;JJquD374tG z%oRl*KuZaej>(0+wM)P~TjPKQia=QhYjB=YFdnqM|`)(gwIMsDIiB7vQ?{$KjWtPr{nzgT zyb-%V{tALdL7_%J8}jZ!)tKzTVPqL?E*wzK)x#%TrwBT_dg|A!F}aOzD?d!3EvzZ^ zxKXG`N(sfy`2H9NXsH=IcsGBY2jdG~S@O*j9q7_4O#Vw(NOF+G3&asUvK?DGd?;hY zIzyEJB7RJ;(x#`(po`Dl2Y5&FKU^50tirbUMOm=wt+2liJcT$?0t0%(qZYeNIx6?X z%(4#C4#SiI7V2vZ3+G6pp{Pq$X6oUma>?nnhVW&vOF>z5G8xWUkz*UZClz6k)&_;e z{ftn6j!Ohp!I{p#HczrDZZ37t1Pj8dL2K{nGZ-@M)$ zD2$TEpjm#D;yka(h-7HGdeN44&n$XE)+;_grReFc&*<(omur0FcXzQ&(!m$ftoN85 z=vulyl;2r=1(!ICHQM;9|2AS>ng8?Sf@R|;L+Ndz_jfYskMyChRkK1gos+qo8=1T( zMryc^$6n+*Zktn>KYc2!f_Yv_oNorXUaPh$mAC9G?c%8a{=cO!{-vP0+wCB_OH^9D zw_fj8HiT<~1gIwYP@$Ju#g2kfn*47_-Ehacu>0J}c|Th(Ycq8p=lJ{}c!;O=nro)} zp4Aii1@z^kzq4j=t}SmO@st8kWr>^0XcvMqvBnOy64AVX%vMd{51 z-`|;qf!uQ$u~z>_d*2z=WY(^&D8h^=U`LRk%!r7HfYKqNB34k5CM67rLFrrR0t6;AP^E!AxMjY5PHx6k%SOQAR+yGn0e=&x6L`PoVCt(zHgoUgO!zM z@B7*N-gmjKYg^tc>){P+8<>)IC~JU=U+6bvKWeW2KrGz;ExXGW!G+E>@+2`iKDP8S zOdSp^mzzJ4zUVi_$LemqwJKqAQTbZblPiRAv)vuIj$p+rJ)-%fOc#TLN>-3Ze+oTj z$m)q_@!^hY7x5(iscQWe$rq*uayWd?wY8*B{+rv61=vVf8ED?ewDl&((wvV9L~pp_Abpw^OIm;UUt}WxVE4a#<7bS`p{!frH#opMMDZmB zq8hP?Cr2j7Nr3TJ$FlR~&F=miy%;^En^L7}^lkndT4gG)8%(xk=U(1PTNO0WVzJ^b z9h=(yh;WnVB+~Clw$cuK=H+`%&KpEm5ZDNhiW%9@X8XW9S}nh9Z~e>I`Q#$b=?u;< z2%ut_K6Kl(09;?Ifo!`tmHi|}|H-dK;=7O)914cgw+*~3T4JNkA>Q5Nt2gxhzMR=_ zSX@Ojt0|@QSgGW_*feQzx2b>|i?Gqd2)Y+12a1rXagvYh31SB9`!lm--D|rvzI;}A zNW#6{T}8q6WkjlsrO*1j^}A;CY~>{Px*4SbRXyruQf)XqnHpX&sZY1r6IpsE)O=?a z=}A{}6$UO{JHgn&mH9&JkGtjvEW(Q)pX46|f80?Yj-U1hmuB2s>z+Noe=N1GP8z6h z>FROailBi2DlLeD;^Y`^+>rX_`Csd}PtzMst&xtxM&U{Ml=v4f`{voW_*-ZH-9xhS zYZf>QgbrpL+d4}Hm+zZFvBz6Y7&_khv@*mK&luW; zfLUG#B1piwR)Qz2kG-RP@$ToY4>3Q=K0P^w8IJ^sBz=LCO3|n!34n2yYr-=0}m^@140?KetrMpQPdlZg)s_7$0w&Q#I zpzD`vF1Y}UFidiDX_1xqFt)ipBuq*e^@lWPgy~=x+nflIT~KsgF|C;SfxNv zRh6fH-S_@xtb}G)e`oL}zL2qy64Xb;O}{Rl@6rcWTP$3c>NBul-1pfq2-#E7tUAf7 zPv}s%gJ~+O@I>8gpHbhfnS5P09qt;jvsmR-cWn14dvZziG^_}~UcOcg4qPvGXATAl z2Kk4go@1PZ3$+aFxxPYh>PXiSwK6@`vPKFB7jK0;0=Z8DI1jJrb2)zMnS-T{jpsye z??ZkptiDYV>Wm&rSaD$~!W}rlu#^7saw|c>m*RGt75flM-&|DhcNV8t+j#VRj~+Wq z_DX#Axn#fQm?0O0gba{!u`|}mnI*!vGGlh1&gkBCF@~O&#Sw9jI+lb_6mLwb58w*#Bx75FMtV~{z#i$GgH`#EmM zrx}3Vvj4t|W>NIxO98`lj$=|LMU0JKUS3Qo31c-^tUwK~Shm@`AMxzJut2doXFw)+$4cu+f|DB?mV^6Y7IcbWqwHNV!RQlJU5%&+4=RgXc&w(I^c$KfCIuK(7Z>qrI2?yU zp~C8hK<6nNz0M0p_dKcrr_hilztwG)17hjbnwE8T3pq=F#xSHXgDn|IJ7uBp%aSDc zEg<(0Nvl@r3?8Y=r!sgj6yJ7Aen{;i;5m$}7t$Y7?w1bfoMjy+;j86UoneewVFCgD zp{m%v*UWAh}Nu9MfeEOwMKg_JIT7_qfMzDYTfNKf`7=Q;YY>} zf~r{}!NLxYp5d#6A;v3L!P;V%=i3!d0L?YZVUj8&(MqP>P$gyx*BUguvVfMy8?ZIG zTOJHmJMvv|^B9=@nmj*<7LENQ)5SuYEoHGtVsSpK3@l!u2IeoeQbN1idqg33*9lpE{K4U-A60k zoQN1^AHYqk=_c#K7HV?JC!Mnf8`dtaKM&Mu&+>ZDh_Ek|xKYsRG8 z2w)z7;2sH^SQl)QH`zD0+m%^Na-~2ZWuBZl1X|V(Ihh`cjUwK`Ht@svcLbN0y+MCL zI6dNsr}#0F&8Vks;^W|wD;SS8l@3#;hH+g6jB0UQoqwdwX#sI1oj@D{1XZ5UJTU$P zpAuMuLBT!zlATrJ3Aq}#-+>b^IguaV)aFhBX>dIc4-BQLkc+U}yQbf03Esd>%J)!6 zafG58erPu=eg{zQy>X|eqC^S_rSYe+RTiW?&L;4Hlysk(d2UQ(Qf@*bnh`sMLT29Z zGZ(VgIOIs8AdE6)2WvBz{TSBHK|=-*G%(^q5F(_Tv)8j-h~HyZ#gq53C_U3`VY##~ zXj?!aTs*`J*8E!^T*U9PHQhd?*lGLmvu7lKk|fs3L};TfLk2Vi8|n2eO~MsTR%whb z={vXnq1Z<0*b$B31CtChxHM>V6G@fiQ1h-~G|<8>x54)TnkV%}U!DkSi#j6IvdaxL zeqwc;<`|%jjXMqOuDIV`HPE(64iMPBO-Uv7e;{P0?O65(^i+^>J*J7&|KQMFs5>Ku?y znq1cp_^8laWHF_*y@VmvoSJ~PkdHNlE5*F7_G8+YIwm@c0^m%$Py|oie4jzFlY}{4 znfws3&bg;paCbEdDlHP zt;Ad2!^P4Yi~$f-G$U^~R_b}GgLIq}!&wJolN6p5%i(2SlyEd8e*E=GaEb^?sXyjQ zlCr{`f)SzVD0aEFWOOz~%FY9MVbY^Z!%49Sw*B80!9C~R$s&J38hxNSj#aN6l6piN z@$;uZT;!LTw7njYnAZUEAs&ATXWA1`tMolHNF;j-2hX_0w;HzN?e^A zN-A~&$B$oj>XTtB%re5O0848#*7M+Wr~y0LH=ay;D#XVU7?~KXvAD>%zNuVDEr!%z z9w-f_aPC9IA|?k*@rq>|l~|Ihn#k2@7%6PnN!&=c?oZ(vltba-dR{Q-@7q3kjC!I{ z7BaCUV;(%&lyd#^{(Vc6KF1s}KX@xu8J5}kNOcwLYK?gP{Wrv35jS`2&aJr~l0xtA zyn__<-}4TXp8aRM14f_`t8;4$%0y5E`DL~NsT%1!^t`^bz}n95>((s*tK`9?N?K_~ zzOL$2G^&Ox!KR!@q|Xx!9^(Si2eh_(1;3r3JN6LMcc=5E>VUoab1)eMIJPRI%+2Ge zyv@WJFj?rl$!-xMB&Sxcr-w(gtovsMwi`U`zy-tt;A6-= zM0|$OvQHk*`T3?qkh5p1q?4lsQd|L*zZIF@!iQH%HFsVtl>Y-}-c#gBT-zhvUyuQy z{J_<_&iHU|<-pnh$o1vJXfAsw#=J=6lW zv{q_b3)3&QrGM(ZK5t(o(WT%gAy{=NiYyxnhU`Hc=?@U1<~%Kppsmi1-Nb;XP_l^j zl_BvbhDt&n!}t3xZpQ(Zg-YPS-|2m0)3&e+<$1_^2=l3-yY_b@Y(xM8k{8UFvt=?) z%il=09h%4}BbChmCr|i>io?EeJL)Fp}ub-C5+vf zVh%#)c!ZmN7a~_$dj7OS7Y~(*4IB>zj?Y>yEwFykdhhyMTFiBYz^RRMR}TVUC~n_sq#a~i!HT9I#6+wmEO;2Y<} z(QWv2zH{kZ6vCKhhN$W!ZpM{SiltcK$Rbt&TYegVR!`66vz!BDp~6k9%rOvooaF*{ z)|_Rs3fzVB*m=6(C1$mhWCp%?mqkD5X+h$rqs|v#+kQs+OTSwuGwUZ=l1%<-^#b4# zuwE0ak|gP-ek=rpC~=^-h`6m{UWDXV1s566W?}6GR22Z(<--Yu9zGjB4PI>cnVUT2 z$`$VSuRbm`>tBC5ked9{Nlqoq0c_o$(>R=X4UT<-VzgeQYDID^Yq$i?(b$}mCTXn>n;@J^T zR*h<=wB%i%U6FER#u%%vhvN8(qth{b519)hqAXN+zKZk3JGn+hRsd} z;W^*ctx@ylutca*aI2-DsGneP-=~aop7A({qSyJHP~kL-a+kCSb{$kioCu7Kol^)iM8N0+I@jcFd9F{g4j0p zper>T?m3-5DQqya6x;sL@{isVT8%@L|X%WnA%{A^7=?bSig;;A9tbU9S&B>6I%BEdHg z(1L>FA^uqTDu5u!-v38Pla1&p>(-6O*iBAXFScJbs@56v(rjF4;)DCjd|I!18k?u6v2EHKtGwjSUN zl?tA7B7ELaZY_75KQ-Rdi;6d+9<-l4n07Bv?J&6EF2G`NB|q_UL$uOVUPLx{r`&25 zf8C#`I}v5pu}SVw37C^Kt@hY+JkhvPAo(f5h-@FBVD8zCJ8=;U#W3j;>i2h}vcx1dxvBPNGnoOCGmp{zid%~dJnC|vq zq2?(?dKu=-(~DS?BHCt7`AcKzS1(J_XP%3we(D;{hQv>!0oPz*WvW|_4;jPch8Abs z9;L9T{s|%JvuI46EJQe9430nq?grRK<`kUmA59cMP+>0QVr^+OMVBnT*|Vn_zG^U1l>`79)B)Wi`1v6tHaPk=WhEW#S4$MWMuJv_3LSjeVCemM}J` z!|bMx35E~6-GCOK8$N>L%vL@;re#W|aK>A+?ajg}ECI$g!D*~98T-j2I7~7;4J4_E zYbiIfcnO9fZSe(+8#ryula^sqM8pn30tN%;#p9)V{q7z3rFf*X>^mSRs^_J4F1rK- z?)RRwI%#P^Qt3C|AgabE8|WoejJh|nNRT?)D;}Z>7I=d>dea;f zh&%WaD$$j&iSiCE^3}_Xc)AVZ{m3(xzp1i*+I-Xa7;CHlwAtwTm|+TIwz?yI?yc$>eSLLkZ;Bw{klP`^Kt^3$S$`~d?tg0{6 z&GSG0W3a>gLwOvrP;6g5R?U;Lt>920l2&bhC^<9bn>3EvIw@S%1x9!*Xvv8;)WOS% zg+0X0uxS{6mihsQp`K%xuyXCo4^fpAdANX0owhvOa~$abaRvUN*t)Q1fN2>VYJCyAcTTk%)z8`b%osn^;Mss8rHRlfOtGb&s3)C~xeW&sFP})b!_0^? zW1dvn?zNDH()VDx*SU18s#*KuC;3gpE$>gCTGr1bT(|RE3n9l+hDS7cHQ>+3-&y&& zE-N3eIQkj;>YQyfUiUUWDHDeF^l^$UyHX3Uk!*^shMRvOX3VE$KgD12hhREonOy*@KS>=KUNbU5 zbc#}0icdO-g+~>Bv4rj(vy}~7g{Q}O84o{UP4JjYkgj$lYkaYjRYo>ahx7WfgI20Z@r#oOw_m=jcF~hUfsW=gK8!B_;R>i| zEiRr6pBtL;%zUzT_ZihZcoVfKLzwV(Aj$K6Nvtm*=AxnCu^$1{(R6}We!p|c83FlW ziyqEJGtBY9@~Ra=*xJ6Xs)uN=k51!-pQtUWd`7*>>y9;3TRb%E-!8FlK4+V1)4z~+ zhPaDi*sAWW&)umk^vQ2UeO|!yc$E^MBh;mt!6EC6ay~8K7ye&@0RK;iLC7Edij)<{ zz!PQ$*fIcfo07zU+x3J3B5C=0d545=g{bic^ywZtqLo`)KjnE2gPQEkE-FPyVi)3S z!g)MG)GnjmtaDiaz@I-|>kRY<(dx6UOz()NBawl1asG0tthPD>mS*3= z&^n#AD5&3j90ce^Rv#X2Ux3ihxDSHAls>%rIl$H1v>QOcOR54kT~!H!usqbn_hzNP zlPb8-g^?PUtX|mTBnNhdiQ=!p-$$mkH&TSu+Xy=l&THuwGR>Lzord^9n~<{OU=1j} zaO4=WLpmEI{{_g9xikOF&=9=vlG!YzW%JsZm;P!5Kx`^pjC~&NyelmK96R()FEak8 zXn?z+tMG8>URJDHWvTmnB}1ObDEW5kt1*DNCA$U1re!XqI~t62uw#L1jE+~eV3#~6ekXKZNIU5Ke2 z7ciurDZXdltm90Ag?-djJ=NhQ{}Eu zW)L6%IChXmt5uA#*%Bn?`pEefos11vCHV_+(H8+>7&Z)}Z~^0>wiA&tLR%9i26$3^ z?uF6e>&qNvR%g!F!W*fN`D0~_wToMnw{#?uD9XpqHMChEr8C*lY-~qkIkD7hy8A_d z@C^=_<_`;)W?rZB(sc}+Ui}d7iuY2bN}O^1P^bfbqDjG!yfNtGfbeKy4a0W!uy&+mwS6Tw+Rb%v`!n{cu1 zts6UH?Z)XoM3l+bd<@CLF30*&O*+{QLix-Jbg6MDTI+F;9h86h#V4~nLzNVFUrtSd zvWl~?7>m@!Tj^z_XIMAUC**bF!mZ)KeI2t`E@f*nO1n!^5?&0%r!Ay2{Yh^)x-{wlsSn$OROnh3I>*O&3QwGIaCG&>3lfJO?Kv1fD1Z7YFVC7-hxq24$ax zT{mN|s&d8I+I?P=oN4&&GMFHhLo4>)wS2xl@yYhy?(gkfW3h_M=-4`a+*HYd@J1@E zCtPlr6cXDX;H{Y+Gxq9tB;~OX6+B(hs3-jP4#&_K5Eu7G^Tr7<7&uH z->#c|;Q5o4I<5r$Cp{;}@SCot*_^sN)0z#t<@nHBr2^l(J-~6VAx(8m4gj}r&BgV{ zTcr>))-L#YPo)QRo@hvJjC`;iz7nihlmrMGVmcIG^#_J8eg2H(`m)5^zcS5+bcf&! z-()})GnQ^vawDN(C4+dI0C@u1zS-=X5rF__cA3MjUE-GZxwy_4>xk=G4p+7%g-|T* z*Y7PzKE+V*^U+qne%GXon?7@ke92<05AN_K&#`21Gi~P;tM~V-S9uTYco;%9;gDGf zl(`}!y8exOE;-w-_t%BcHbD>l7-RjS7>Bez^s>_MOTZzQ7lI87IU$SY1Mf&uAX0T2 zt+*`Sj&5K%uSVQ+Ms`_&qWEfy8VqS80x)@(<@~W@C&QMm!o+H0FD5HrbY*)!%Kv1- zjDBQNNvIQC89Z5SV7PLV7o11UX`wA7Yy97w?~hKvABD|dp?~fG zhI>Ah63uz^O;I)vHO>L4Ots?$+swy8>*UR%aekcNkB(@Zm~0Dlq7v*lxPe{)P&??; zA98dCvj?#O%QjgEu4>Yoy%r|2MC@SuHN&;|Y@#?YAtjp=T_EpI*m>}_~c+8urFykPr zBdN!qQPYz`L_dLjh8XneOrAM;bN!9KYTk~U0|EJ9Ei#P?0q&ip%*0Dk*Z%yfjh%Fc z52d~Ov*yzRRV$fZ49}E5q_tunvBcTw*P9Vc=KR%6w2w(!?W^3Rx@uL$r4|C&4taMn z;#`ODhQq88WQ}Z#yH5!1=ihbiB);G=7y(*C`O}*6_;$oFe=xyWjccS7H!Mwqn*+_O za8}l8AcDuJ?8|{1UW$?^Te*alPW zdmXs09V}T)&yrhRqsg1Kj!$&vh^?qOFGydr*px2v^y^@!Bc{g>O`)QtJl>6FQ>J?n zm2znG9b%k&l)v*8aMwpF8&G2XtxBB-VL`{19ED>h=Y`1rST}#w=Fyj`P;40`%3d&&Tau37S+RKS8Ah+ee;**fdK7njE`f^k|~5UN%gxe zaW*SHUaxh&KdgGQJk8ig`~9R9Vu~I1l~zGGDw+{Y_=#+}Z5xfp&DY;zYA5SyUiUiB zY4$-~ z)s)!vfHsr!(#-)8JCtS$%$9k>e5rkJ7J0^Ic8v{a9Vka?1;K|?l1v+4ckX;xpf)f+ z_F^Zg(A zF+)z3J*wpx`o?y~xX^Sw(URY}Gk7|rwkYBfk@#b*9a5$tzMU2J%(#3qYgxA?=kqhx z=V_f$3yY0jG)v==rUt7DtxWZ9B2*pDd|9K7B^cDY2Xwt^VVRBBxg8`T9q-2N+O2oO z0JXGAu_L$$lJ9Ypmf#z|_zBODmoHb`cVfI46GV0b-=g~8{_$T;QDq-d@%I`3U03D; zxScU`Ee;1(zecVbk7fs}#dkU!r3|XVKh^+9QO(ua1`b zp+|Q8_>MPZs&@I&Q>Fb;3V7}zKd%wOIZ^kGTn^1H+_Jo1=#*t=cqZEytRy*bZ(Y`o z@+$fIi0($d&+ogj4L4hPQGOD!;i4qu?=)V2|H3hLv9Q3-7`gKX1NGHA;X&M1Fz_SL zPHiv@m{5d)-hl7rN1npA4I`CPZ*MWiHwBZcJH$5tH0@mTFm!Txbj=N$81xI)I|d=O zq%4&|%(!D^{geh$IdW{ZpW*RI5N}|E1lxgEVet;9@ zZ3^$Lpu_9w53Y-#UY8yt`osLaM27?M>q;B9h~r-}{lzWy!#Lu zkGY3>fpWAffp1zPjr(=!VP^s@Ify3W*_3;mcxO*i9!9a2d&7Ke)URqS+iSJ!@RYCP zkom}#+nY|((UhIIau@PqWBK$4*_@5N*)pWc^=;4=JLa7Z%CZAKf+^LyrV-oBB5f6) zdV`bDfll#;)(+`&>g@zMKNc%0PD&M-cOA`GIPsqxl6inq%HVE=1H5Dof;LxkmOk%kW8>W9S?{DLG;1+r%Efo@=#fUg$`e|* zsc+G{p%_>-S|7z|_jG`CP*5y7cCIODi?U^Q`~%{TNP$Lt>Tz@H&yJ~9lNJo0`N^-n zdaS6c{G-N)L?6B3%EeWCMy@$+>;S4R9ilY%B6334D0+!g1G=H$J}Ap{*%@Pw&QLy9;2tr8U6}IUHS8EQ5jX%ul+g4aZ--+zTC^E7M$8j{F&b<^ocL ze|edRY7BQcf+)|6{8Kkm%a}otTcSQ_BX(+2G_h>m%i@rRv;cToBK0DkFyaRPhk4W)&NfVS5+-Qm7r)s zPxiBecYBju?*He}#qSFp=sL5>dp+LOHKFVe2@eAU+dY33(CxoQPk;B=e?74OpYOu> zO#tP;tNU?m;MZKL-ppbmCRS5cI3?nozo)FI`Q!_(^A!J-vEgf^Dlv zg+6*+%#eEZUs@0;^1w)l;kJ+U!Yb=$=hV0Y#m&h;q#qs6#*SacPb?s-a}o7@uQg8A z?Og*dhF4FQ*mFF;o_88hX>wSE)A*&eHRvN$eNKE#2!$-)2#X==wO3wGXTN)RHsmtG zR|b1bCcNGJeQ$aL=@)Qq9G<`8>3lsa{(?c`h<0Uab7S)FJcfrr_2N1BCVyX0KuechMOwj)Hsk zRxY;dz}`?NUZ3IyMui~TWi{h~HcCH<--q;>=>59i_dzpTXVj2|;uGxv3==vmgfh|9t)Z%c1@U~^@|d7jQG-Io@Tz%3jd z&^;UO@(8)-H?pSyon&Y07SB$_&Fk&o*x+=1uW_5Im!yR9X#&ShYrLnt40QRq@Ve6Y zcNKQ(FWVIGhR=+vRQ46c`9UTZlEDL^oYvJgVsxdhYQ&v}GMnBPDmG)Kj1RY{kys+M zb4|mGSQfKJ549{fu`;GBJVb^t@*;alKlCcW(5!)$;KL^|F(s7~+mdd&J~ug99n|Fp znOw-FP%fhLOQqj)>Exu$A&$@2^9HXRVt?T3^rJ^c+&j}I-#u(V>x?3dP7bQB+_&|! zqyq5)sZV(k-tTpXrm`=ot9~I(kPG0}y`9IFrZGvH9$Uv2`L9oiw^@f&D*pkDJ9$vL z=NT;1N4Ja&y*Z`mTOaAD-lJ-{?+KC^g z1r+Gg*PV8URyUN3>m{Q1fp<@%COLbi^oxtE^%UYM6QwUWRVrpv+*REON3{xy9@Eyf zq{N*@oE8Ad)`46=c639wR@Nn_qfiG?B9_?uW&yp@A~sYx#~~9tAbRDwHq1)<>fo&F zz3$k6k!Q>28f9~~l!rLhO!X3%yN*khOLn7n9H-_u_-queSv%z7n+yRF^EOA4YDy;^C)aSH#*Zur`akIROXY8Aa^6{wDix1=%pgx5aU^|8b{KE$K z6$|mj$PS8H_P|YTwu%Y-M$AKQk1ReNiLUn@RbEJG4*Ut4kQ^>8IiPf{`tv%7yU4+g z&T^`(R;Ra<#BM&aMRoPfKLNu8%fuYgPGyZ1m!kgsBW8}@%hXejtjPG2nD~#7;*a2A h;h_Oli;APlmxx}$FELhHt1bdQhaH>_lQX1(iI zZ&6WEgDKCp^t6;uL-C~=%I$>DEnO|DlJERWlm(5Wrhz6ERcS2K!Gn{OHG}8%yFOG@ zEG@^^i4KoKdnzg}KE11&w*#!!(k=W}KAsLERLP!+?LZz~m5twe(=>bY%-}@d{jM70)OU8zMH(ZT92)HDZkr6VfrI;{6Ah$T%skvub z)CtXA`09TrJa&v}7st-+j0+?`y?C@|wR~C++CbBw)3RS%UozT`yWGB%vOJAZ#ZS$mWL2_#Df+ylY_sVY~I}7eO^798M#F5KZXr~!k-gKTrvTo`A2z&)pZ^+ftN3)QDUDzrz=~ciDgV4A z0`YcgcVgz10m0L@6>V-r#>17&T-OeXi`4ySq-qk0DsTtV;na6|Y$g<%El z;$enzc%_f&4cuO^71n^#E+z>_;J5HHNJ0vqRUZlhb$@$nfU>LCJV$o%Ru4-J)2u^ z8^l*+-~9}d9;27dt0d-VGt#*XEb7SwxQUL;81zlN_R!@GUTEg%zvo$Y%UOauD{I7( zE_qwk7~))7Vy38jZ!d*^v9e0TdEdvFP5OL**vn@?T$X*YC&uJ^@%*?znld`y&_d{8 zZvAxa(h>~e9%W3UB-7silwau^iYMm0Rk9b6P#`1Hg)zV91JvXT4b5yV;J1-=2E$1 ztb-|<JUxDGWt&2Cjos%hw46B zk;OpvQH?wKcpSVXn&Zd$gx$vX~M2G5!voTaSaP3SRuNs+M6 zxTQo#_&)zu%s!1}^!XeS3_~LA{QET}MOKFypg0hca=YR)!{d0)x4$y^o;x72&1>g= znR&nwqV_%T^I0@)zG6Oo@BT*FY5Jh%Ez^2#jW2}WI8#^L-Fv_@aJ`Xs51b~Mlk4+3 zb+(K<OH^(J5a?vk1TJk?uEZE!)yE@a9$GYi>2&Xv86 zSxiw8lft>+047 z>4QCNz^zN~F2u^T)P}mHZLQq9KxHR%>4O{#Z$~X-&X~!)kL8>mf#LoV--vlm<&_0f zdofVt)#cFfm>;@U&y|yv#Z6pH4@*qG$|MPNw~ff|Xr*-b_Mi(Va~-d*5?j*S z-nh!)UCMsK%eUaQioFf>d{L>>)sXqo)It0gSx~?G0RKMWh{RwH_F~u$U_5j3av&GY1u5>9Y`ODjZ+nSO_ zm3flZv6{H?w+iLWLLF!;X)!@B{yS^67^>omk(yE_#U1A?N}?kjKQh~rxoGD{*LZtF zv$XG6tzyMK_U?oSFjOadKRvx^aWmN7_b^N+bovqW{v7=!u)ShcMBFdRse7k`@5E_O z)HkJj2lD%7IN9_Gs|0NPYzn1g6-0V(uu%AjJ|XVH|Ad~7<6hD1x^w)tLSqE&b4fZe z@oGkw*pDpo=Wo;I?@xpH+11^*L)2D;@=3<9+Pb@|u_Opt{MHUYGliW?;1HCMGR+vqC;eASBA4PvV8d-DyNKxTO76T z20pM@oi-;fbp~s!y&B3!uw}U0d}^`J_=)#=mJ-ofTfX85rE7nav^BsL>lCuv9n)=u zf2fPz;9VeW8M~R&7k`e%3>FH^cM%Vmdw;>Q2%fW6T@{-VvTGGwn9GP4FtfK7p96&|_s2IYJAYH0HlhOHnIw zZ=|xMven(GolV`HulNo)b7hQbk2gLG6fsxCrtEuZpjQWa#nb6A_?d&1cor-> z&Txwi2Xmb6ayjDCUKUc`3mbV)`hxBYA=y?p_mRd&I}_NPl7}d3BSp;~h5XVU^1{xm zDUXHrmv!vNI*ylnIu0?)MXu7YJ&oO|L%cxx1gYpMa6NaMz83H`7ZNGH?;25Mmos3L z=rNFUIKR@aw2la(6D&5P@ekegx8wU@w3>}|1D(xeTKjCBBi-{E-&65XLsc_f;lu6- zdCUqJ6Uz}O_8TXyhc@)py#K9)>~GA{{`dA)v6}lS!2Oy36iC?YSLN!Q>Ym-(?Jr17vmv6;@kowCJlUb`VBjz86GW3FQag%B)e#p4ZkjV@7J!5kJ z1R?X28=|nQuC}|()N)?5mw>%v$<1`7?2q{fVA_e&wHTGyD%_AMmc!gzxtCNp@+L<2g8;g=nfB z4-!tQ0`pv}aiUTPMVtSyYgor}ge?f%GnsTQSl0WRehLM&`;SuM3=v4ny3_bYz_8hc<~wN?J1Qv)l0>*(fyyyeQwWNo)QwJ`I5cz zX3On$c8lZ8*WbbL`^^3jKh^DawS$%A5+1|Bs57KT6DTv^eGd+^KmD# z&pf9OPRyCN2Ym-d?>RJ`Gl=0!Zu)`yr)JFgS%Uw+!*@vk-hWI@lE#PD!4758|1xwV zZK|;`fQfT+s^vfyHp>>DYE%@R+bRd1(7oPLD&wT8n;cYe~GGt)d~E1hQ#fk4hG)FYdiavYL} z8|g7d{`S{$(gCBs{7kldQo1-35|KnmYEsj3U}N!0>!25x3!lC|H!?R`AKy*(1a6ZBhY3tmTnBkQ#ry5pl|KUoT(2xB1#FAg3J`C^bdLG=DR_|U|8|QvS z80usbR=~%-%9QAT^1Cu*;l+!(hoL$Uo4QG>JO1L20u_;m?nCvPCI_jL{oKDf)>aj( zHF!h0TRALjWKJM?5QbDAc(#>;PEx~OOkUBosNmw*; zro(_)2)2B+)>O_@jtC?W@&M)uGdd;T2YzSR~=Im^g!vGPXBzmE-;FWYr{iCpn*XY^GS;}`Q+ukpaI z+CHdkvcnZ&00j`SDTD;)rS$gJ;w9;g^s-x1n#41c%gnf#GR?6qUrc5HGJBmXre|e- ztqeIP#O~&$s7_0UrF}HIaHI=_NLmaDF>D@PEbPotW^u&i9|^X)m7~x&dnpwF*7-4P zU0W^yN@`g1x6r(h7NB)%;aq%9c_jZgm?3*H` zjM#gcsI5cdJdRe|z#dm6#zt0Ry{z}6(CNPtV*YCCE-sp`Pd3}t&CkeMG#Z*ZbAJP0 zG9)S8;&lbBKqgcPd0Vd=Yo%{wPnn|P$~S-zRl-mRkCXJI>i9`fpE?f1PWs=HLCi6v zJu*_kioi)1>=ldj+4o#^pl@Yr;uMsVK#SZkr?0qDRoFSh+CG=MU5FwE6~z=C84@Sg z=gcI0M|0``cSssV>JIw@cG44g%2u;t7I(#dK=g-(Qz%V$a4V@3^^yHUAK$NC>D|_j zFsVRqGu`FsZB$VpgsMR>#=m>5yD_ef0kkPN*^jk^1|~)li#7{D;_XGU*7kSq16oQ& znKS{SeT{K*0doxd971kg%C6(n@wzANeJO`q$D;55O>`6jfei}c)+Z)n^kHE|2Z&FZ z^Ka)=|YfeMxBwS;^4@qx{U%yCvEz9$n$^Vc9q+Sz>aL^lR3 zW!y!MJo$S<$HL9+C@ddbXOqdD#){vRdG&l`I3bbLnp-w!S9BX@nK98)FrHxlE~Gxm z1cDUmy|NZLOZKQJ`%u4cY%~sd*b7;hn;paO6|liZElVRmu|Hlp>h)Gnd6!vZH--s! zhQTS=9($%gW34NJYqM_4DNhMA7-n#xVf@=7ZbKbD|&u&iCzW&V{rx%uE{h=;J*HZ$acRI`^q4Lr8nAH3u7`qgIfazz3Az%$9iA*3=h zgv*1il+1b$Tb@m->PLR_+^qPMGG*{|IflyR;_mmrpMU+=SY)8S1$0{nBCF&UKE z5Ro9WoU?UP*Cq6OPgmuTAa8O+WkkSQN9{25evMr!L|kxdV$kx7-<3eKKT`ic&z6V; zT$Yq?^|glL?PyP*(hxw5j7U!Dx1YOeev#754=DeM?~lrUg7D(>9>4fQY&iR{_2%BN0MP@eWi5Gvv`|FHZTI9FW zz692~1YSJ#N3zhxnkfd*QG9!tUMI(YY2%OE{I7P9o0>YJ0~y+`dMwCHQ_jw*^sX6R JEzz=h`fm=D=EML1 literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/api_precision_compare_details.png b/debug/accuracy_tools/msprobe/docs/img/api_precision_compare_details.png new file mode 100644 index 0000000000000000000000000000000000000000..b31ba52350989488df1245f4840886f2a7db4944 GIT binary patch literal 25476 zcmd?R2T+sk)-Eh6qM{;#6hUI8C{?8iB!U7KihxQdG(n_S38W~9NGMTJKtQ5cupmW+ z&?7g#*S%8C z+gNVhyleBCHEXt>Iep50%^ERt;op-tZV>(+kac6H@DGtldrPx5C0&YB!Y|hQn_8Q$ zSyPS@=Xr?=zupvf+BtH~nr%%#e~I8jU;3_D<0F6Ol<9?7kGV&#m9J}L%pI%Mw~y0V z7n<)4fyAkur#FrnChQ}vw+Pjsso%w0`75hB3?o+gEXujB9LlOdfAyP6J)Y=2#P2??X z`|eUfO{a9s02cHi*ZkucuKYokOa60!<=(&aFYO2AALiKEeK1{U(~f(Wp|~oqjFeo_ zM$*WmU#o`+s@23%b-uyozwk`Ip4_bsg)H2farzb>7B>taaJoCFf|}v^Qs00MpSa<9 z`mTCB4Pu3usK(C#aom@xkI_GPDxiA_*(el3T=05zcz6R+Xow6HyJE|R1EQj#+g)XcKOGmooyOEH#+ShdHpq5H@RH;TSQ3;IwXz2Nm6G)b zYf@73?3FWI`-T+wcYRo27gYvjS8f_?<}#9QU;8NDVI`)Rr1c`q=Q6NmqfX(n4DAlB za>U^3zzwq@gEUJ2cRKI!ZIbeA%mRHNmTi&cqM0TJrRlre?e#dxW|>jz9yJjd%S*er zl;S3=EBSXar{3FQ3ZF{97x0<>jKG`K2LeANqSLT(^KBA@`g`d9bTPcf(#$#w0y>Zb z0U>}bOdS&mP4ak|a-S8yBhKTya@d1+<0ic_W#}*Q{S%{AyaOIkqfdlOaNVqq@HhA~ zq@Dcx+J;M(dUpUEXUJ+algL*W*&}l-vZGd>r94kM_Rv5n29h2n$WY3IjYLmVzypn> z>h~g?5yOgMTj(S5N&50$&q!0b_)SKBiQ&!x3oA2S1PWzFYZhg=r_ZwMCcgtXr=R_ zHKyH`l=*a7&e-CdO2;SBA{N_tBo$dy{e@fcnZBRQwuaCX?2%7C9ksGaKuFGfOr4Ed z4Gh|kpMN!OYYb=1sgFUA_Q>WdWk*gJFspS1e`ND?_2mp%ZQIMljlHswyy&!%T~fT@ zQimA^=b-Yn?szC^B5TR|K?+BUEFp%Eraw27BpBio1)uo7skQqCY+J#Z0;tvJuDvda%Cqf z(;S;p&Y)d|lRE*jt8wZ*?+muu0UG1f&qw-D-Bw0cJ}8OP#kkHZbd3p^|F9b6*aLv= zwAjqWkSqI26QL;Df*fn`@Rhq-_|l17#e7+|RwM+J?si`oz{ck< zrqM=+(hA*cftw=@*ozfJa~R8~P1aX~4=AKU>fjx9C-vd8Jehoa_5PRw#-q(y_$sZ& z$C~$^JGKWVLZMz6K1$!)iW&1Ma`%l9r&=HIzNQxXZ5NDm#$W#Kc|M6Ms%7)Z)K5j^ z$nEm_dV~9Y<@k9Y?T~XDtoDqHpGMAI^TK6mwI9(LU5gZ-=?ksmX#y!dvJVBH>@q;r zVP04qbR&3uUT}x#L1Vw#@#@{+J5BufzJ800VJX&LgHQu-K7LZxmV~hdb2qUr+FOJi%Ie3RbnrH}u=E%jNLfeNDBJJn!G9BTc z0iK^}nfuF$kCDA$^QpAPo1Y;@iGWnXCLZ9-IDku=IMuGGyBICpkfZI?(g*R5EFV)) znJopd+J7g#^t<7V%*_55KlmGzWW7NMft=<9ew3TG>XZ1fv4m6IvW_z8jK9iOIv9t2 zw%_WZ8zoM zcRX7!_z~7Ud|RpLSwP9>)oK|`h*1IS4j_@Rm*OaVZmucz9RHOD*c8gF*~_D zPIX%lp-%5Kzn{zV1AWt;U_2{)xyEtPK?AuWdtgdj$~FK}l~&&`65@VM^7Ex8Us9Xr zP}UX&tqwuLcgJQQP}{*wkEO_~ekSqd9(~R#&15QrEt{+s&U-F_-LB#mFx^G6)|2)ceoF^6<2=WmSh+RN7iMhrid`+y8yAXehwJ9bs8f)*Wo>7T&< zmL3e*;%AiIFs zotWnVSP|3^pO4JkNNO?mA~+s6xPXr%GAz!1IGgS%1pSEu)v+w=knTU*JFJZ|GQm^Gxf9r6LjjsdQO` zkiuMW>rK*SDGKl6G6ek`J+Wro_oI#^mhus=4KhO?@n`7yMYz{pP{zNm_4IBAZ9}zH zvDj`+MJavaGfj$3p&lrN33wi}>JIG=mjuL{N;kiMU{IPV>zSondDjP%<0H+z-alLQ z!)M<>S0?U?hw3n~Lc3w}lutH3Pqf;;-}9!NX=wPiYa#juaD}C7j0SI6xmsb3&#Ith z8OSEgaNiNwE>qW%4KG()beTJeRh|P7Sv~7K3kAkL|C_z*_vO%R$fqxlV?1c6LKDonSaGuvz3S zaUw5*80C4K$n|o0Z%`fS2Oj~NBCp?KY)=bkcU)VPoIft5L^`HOGWdi^9;fXF-QEi= zmA($$>S0vQm7{Yt@b6}`ecBS2vxz!>Hfn;os(XO3`G_EjHtiYbsQ#=ye9XVGm^D8$ z3x&7n!gnXmKsR4&9NIOTnu!Xq&_hrSDzzf9R_|#>vAah;hVv>Xn`+?X9^%1h&4r@7 z4%L8I)n&XEUQ1+;Ul10~#?eP+PR^2Ad`I;NJH`FpGVZ3A*8zT>{Q&qSZQE}X?Tg9B zwr4}??I|NI##)&x;SALb=jnBeeI4BsIpO)memg+Pk+TJ_HT1u5ckcm_`I2BztQDz^ zVR_T7&9+<>c_v`@mhsdxf&|>wtJt51!3gv8^c8To=c%4rkvijy(a3cQp>W#@u6Wq?bd^*J)q$DDLFr^z3GtX9&AUt>y7&VP!r( zz=HZGt}_lfhmVH1c{!J*SF@8=H>~S>#cVWcY@o?m+%Cr(Sejru!iNs|?0xnHb-X&2 zOpW|7k!5D*{V;JDNVia_f!~TuPSJsf2dOG1F8H@c*Gb~7bnz~C7%|GAEPD^p_Ojqz z5gA6xmlV7-+ejEZ*hHDLNj1nJ-_pADlLENEix!`Qeh$x=E|SR|GGxeup1zeq!JmYG|0D=a1vGE3^Vn=UFQ(&@ z|IIsScwMqT46@#=&>sW75$!i{?xOcS4^Fr|3HWrli=gLGbpqQr zQ%Bh_#!#R5YQGA#d%QGM6N-Ytt}{&$!{Li=8>w(v&Zdov_)KoN&)#WNy$7K^9H4!e z`LO5X(gq`E=y|F#b@qO33hJXQx4vNzGw1M~CuqQdJ3mI0v z6McBB+2|P@3T5c_Xk4sGLa3%c5s37I8c zM-0(6<7LvPwjvoQr>c8sGrO-_JND?)#vTDWp0#UNee%`VR(U-(el-E6ET1OdE&c|w zFC`<1e^Bbuhpn3VfyD*(pwzZIxMaC|)Nd=ESz6_nejPooh#PYz!1rIOncTBXXQiOq zm?Py6Q@S-V3o-3<$mj4rR9w*wwBR;yiO zc>B*d&w)6YjW*<_ZCeRGiVN`4R0H_qCe9phA@4#!tbnXBmB(TEJ;u59EjlDlxb%d7 zHClxLXxhnl(wiUv)5K#N%O_$#v4ICm3>w)l9CsEL2ua&NP|JBnV1&C(3&CzQ-oLC9k=n+vL5^zVBaeq%Y+Pz*C=7C~-d zC2EfXan)D)#PdBnWE`}n#WPamS{^2%JwCNPi< zJ4(AAFn?zpEINRH$ltR;ALPG>l?ZQPT(g={x;+j?9gH-tyLFqDs)u)mYVHQBFddp{ z0lAFBi~B(ZeHE{lj@JS~Mvmp{)SG6KAcDc0hWaD%DLcYMQGP`^TG=P#k~aaFNs#)> z^ZE;pL&eST|cvcH;JkG@&eE@Zd?ZUCwD2f{{sl7sLH>mb$>qVBp!1bwN zKxQZBh9jfDDjj>n)iuPEWfWG|Q=#leP75$v=2%`o%xOaw#(a^zwJD@NHKdC0-_#6$NQ&^kj8xoj_WCRpXNH~C_LA<|wu+?9!Ep&4e|oDNm%Xq}|= zxqJ6)ix7f2Ed8pFf>Dg^SFvVeyeyhpZOeF?r0uwGyMLV55w0HH6Yxz|YNyKn=G#N_ z8eJn@Z<7sLQfZ@t{T=O5Qpxop50@FTq!EKnZTJu&QSn@1BC85ukSxgeK?ihIA=Z zwmQ4(X+O3;z}wh_XMNpQvBEGwAXhRi7Sx93%jvfdT|GT?X*frtPe*aSi5HBKC@9UC z*?plaU3mU1CAw)APgF{*xPLacjRx!mDV;mqf*7w@TlEeQge`VxJHv5S5`tVE!AHpg zJo6i8U?mK{>007kpTwz<8>mp!>OF&%IzUyvQyoan*kvOX9gSLr?{I7^9Qc; ziaqO(2@�wBEDIy7l>WoR#gYU119%<{nyaB05HydmF3G<5j@+HT0uCH1zCA`G$wI z)Bf*=K0=RZeFTYqyx;ej@qPe{Qt9XpgXG{9kD|LM5$;T%GucL%Z`|D#)L>`Yy4MU; znGpWiOXF_Z!S^PZO51FEE35uI#pGF4y>-MC!!(_xvhE152`(pU?4wn zBR|*)DU!3?yGc$eA1tZk)l(sL`AbhqzsKz%aedweNjPx`BxK`k;}gHYyjv$u?QoOE zR=0Y)v!*1AXP8NOX-t5xMRB87u+rG0itkS2w6fBbQ@h-y??unaY1^*gf6^t^F-$iN znjFVLPRpxgeN`}h^a-!<$%^+{j-SzXu&S_WiaaO8(eWWEK92pRX(AoE6KtkmgmM35 zm6s2O#jfnq2nThDcQo^KFE)YOJepnXtrzyDWoC?tsK)>Fa=> z(U?s>pJ{!s6>3WWO(Vb#O2C%~;q|HGLph-I&{Ai(pWaFeZTvwLB-rG7d0vbf74H*( z3vz_KU86Z8W|*^nY62ZAvG{%;=F!PD3~Chccg6)(>}W`EvQY&yPKHe#z`dT)HP974U`GXysLQ zAaOe$|J_aRoXX9I!ENV}PZH==v3q4n{f{7xm<`h&9sEYqahl3exGW%q-EUjFA2F-q zKVV)Q*JX84lov9pJ-?PJX2WmD)&SoX#*@*k)%%&o!*eEHwu6)Lm%eXl_znn07hX$_GBG<(9{sp zt+{^Xie}5uuHrEUWUB-}S~J+Hy|N0pM}H`~@Waq?^8zhfFL2oVxGt3cL{Rf-qO(t2 zau09+rIIm~LrBd{t_PGJJLxN4P>Z>9k%g3^#eEb!{7<|dFgF@_Xuh7#sw4L_69j7 zWQ4G)lDL&?S?g~&MhMUz(5wm$<7s2;hr}O@-LQ&ta?ELjI2rxV;2#F(L3>Dt++6psn6R;z$E+>5`9hM2GhcW`tU; z-=vDc3}6DcZ-hSHK$3jo3rdw*GA9~hlD}S7_90b;NSo$IloA7JGuGd6Rh0{N>9dzn ze%0~lORE>}u&VC>GMnJ}8pNRkiuy7M(O)LkIJES+pgC2$7yGhP>vhk#AEm-QUP%2u2V$=O6k|M8 z@gB%fo3N0ylU1wV&Rul`Cn zio8q)N2RTiiszDSq0rI}*tP|+QIGBj>J`yack$HV;&5Vk5RsT3khvGM&>(oob#oe) z$x9f*Ekn|!@H$jq_yym#;UWFaErypd)yX@mY(fpS@;!Ka9PfbGMQbB2)|Co~KE~`Y zd1SRYZ9>;MCO+{;Er*lpxlNxD8PG8|E)Cx%4B`dp;4kq<(h7?A;}N$~yl(>2MyoLf z(IKp(XXh&n&Ve$_{=ItoOQ}qo#KXT_7TiGwV!k&uQOr3_)%O6d(&owEzW(b^SO1cA z_l7%PkN!U(;>OwD>duTFV-vLP(!M0y8M9=Hy5`^(ZS^w#Q+T`oUx}I#$~Ok1xJyFy zccOI11ZRqwt2FZ-HMJ03n@tqRsv?8}xA2YcG*?(E*z#0mrj?HNDwKO7@icZ`!{!l* zpLQ=^qpJt?^v=H&!T*%XX*2MGhU=VTh(Z$&6qm1SjUfi-da&?Knx!`iuQ;ts7ix#Y zom4DH+J<&nq~!5&P;9b`G3pGb_XOO)aWQkJIX&eT7W?cXNHY8Yinyfza8+>lGk)eb z#dX?75COkbqrtjT^m3yVwV`@qfkuA8hR-t^fS3=MdCnp~@lx^UpR2M5wc<${0jgJD zv`@s?hVZyLBL!$Tou%}+Qy13j_QxtiYroDdC_z(?9UPRQD&LL_o|4POKbFS~l!%*m z_vL7rU@wzZ(_zgM$dAzaIW8Bt3oKMO)nv>;EB8NnXtQS=ml4;Ig;>wOA4N_Lz*XH@ zd{5=1lbY`!dOlx?5y_q+xL#T_5V`P8=IA{>HC9}~`=UT7i1#j0XT%Bd`FYCX3z(|w z&VKFLA3BalcK66ECyXB4oHat2iYs8ZOz|)(8ZR4gz0F~Wewe!G1*vn>MYm zbG$3>A;QXP<{hNInO4c)o!BBPWczE>kd+&z=r2`Q#4R#bw@BeO!sKv~q2bd*^ZxgT z0>iW5ib?;%A|W(3gmMQw4Ah2+Y1wnNtU>Z>meYfuo`?^;J$QHbrZGP!-;;Nv9J=4$ zr$zVVYoTsD-V-?5D*hTENDKM=lxJ4ON}UHGN;FIjDe4arigRpDjil_`}nAx9^jgy;$Z z*xX7I$p7xf;*_8l>p#ymlQ5eP+&^m

-t9=o=#R&id3lro~s!$j(8Z_%@&eQD7In z(&4&LH*$%}SaK%Uv@VeR*8B_wP%CE})B$^s)o*dTv>q2e3pwHw9KzN&r*Hi%r-%vhKn}vZl<1r?QZ2< zZx!3Z+-bwB%&A#VI^N)l;BT&lq|vhH+SgUFeC14>E&qPl7FHnr3ARfMOHbBFlYy5X zpO6|F(~#Y-=gr&+C$%TSYH%cPYyRVx(t@!G#*O$+fuQ*C{qK}3TNcDLWt$a{xT}+i*QhJAwb?thf*W!Lex0l zYD7J3Za0?Hz7zYQ+J@m;rHSEb4&v2`R~j0x(P)hDgCJbtWCT4o@7z~2Ge7h0XOEGz z?BG~4+PZyi*h-BouIabq@EfXvMV;S{@oPXlm_d*!;q7)90%`2DKT;T9u&p*Yf`IXW zp1HEqT)$-(7gASv=e@7ngEBQIX(_~b3m=_8t`IZpPVw>oH0 zDAVWCMt5Cez>0YY&fKmByN)x>gPN6$(b_4}?ovaNLA`Q&d_|=09`CA*ApA=-$&Qr`Q!C=nv=>c;In#-CWDWw z4bH(d)Vb`2a73#_5wKx9;?;?u#mK8WyM%=J2W^}OFNvs6aVt<+D*^jQFOC{h0EE+LUwLWimy+) z44~-G3qyASpFQ`qZQ+?jE618r?4e2v{~Ss7x=*<+g6lENc>YAGd^T)3U&tDzx@L?PI^S6dNarj;#!lA2;~!j;(0!>IXlU0#2%9hZxR~|Z@&GgS% zf?gj>jB2lRnnw*7z^B%3$AN(s|G~Oyh|Vs0Cz0UC7x#?ZpU=D1UWwbqRimW34-Bpj z!pnwA&(%fWf-M+B{UAK08Z?qZpe#mFfxb2ixC;$Fozbi6UiCQt6L zjmeLG^wl=7!=KZrYsJ59DQWviQ4ke{R4~X>;0oecQngx>D_YpeQz&To5(9 zhK`j?N1VF0TYGBF8B7Hs&!<9QIyp_}uVD~KDcwP(g_E}OhJvG3()*XZqzU!7SCO@7 zyNFVp^Gdk%zxLTR%=>FDvbhgAdsL&T>oWHNo6wCL5r2>g#_4#(H6jo@nD{rt|Adpv^8$Jn}h)vEyR zT<;C<)GS6sZDy2f@ez%ge#FgdYGnyb`qMc{EFYhC&b)hhft=~J>-`>S;@gM5|G^wZNI$wHSaZ2>tnzUY*U}a6E zF225@j&;d%PVX*8Eb!V|vA`hLzVIry^oU{u%MVNpU$L%TlEHW=RN9JF4|)IYk@}~L zGIIq|0*2{!m{*(Wkaxh^D_i;!6qygC`9R)_$;}z*g;$-}84xRaBmqxdjs<3;$9`d( z|6!+Pxw&1Gll8y;%%xdVq=VO-G3kwmYFK0-qFA24*WR?Ewv*2Pv~^~-O=b=Y!S#sQ z(ckolTHX$RsXF+W)#kg@79vEFVNc|L1E&9UwC-=#``Np3eUGg8>PPVWtG@hBXYmvF zOL{B(G{C}+$iv<#2ov4y%I~^qHSu!aB7lK0t@B*U3nG<5Sz47F zmrz;GELb>($e?m+C?TbLmTkKHqt)Z#SEDibrImAH*RGW@pr?XMA2^aw&gw`(Gsduc zK#Y*&AXpYpR=4JJ_1DAoGrN^tr5F=#_ogmm08~zkh`|+XD0&UEyXw^@z{(NX7(`2 z;05^BLEP6P> zNK59WYMh2Wh*NDHIUUgF9H>hts3AON!b$r81C@q#Ob0m$)8N-`;9P3#5arD#T~^+X z`Ta4wSbO~pL2L`(m91nmyQpiy>M0sPN}3Cr2dR`$cOrBemS-{{x1Z47=Q@%IO~ll~ z-VdLfn-$;(6}Szw(==>%db2h z{rh=5jUOfzc(Bfnf1=D(g5Kwmzv?|>k%m_o>(0SQN_GNaCrjtVm=_uVe#C0Xm;gpF z>8!7DWp4-c2lQ|t?29-TZosz}L%`9Md#UsV3DN4y8l;7KZu%zX4eyC}Z;dVmLz8GK zu3Exwj~`?Ro`p)ihDFc@-lUk|JV>;?DlGEnH*>TnPSVRezdpZrRf2qW`}zk~g|YB~ z7iF*$s8d`Nj?VNjtY0!W;iK@h3X^M!aHWVvSi{}S zbjFYIP~=MH_f3^&?x`gq_mfKKdKnV4j)9%N0ILnSvk@m2GUp~`6Mu7W^)6<@-v??O zlSYc(T7nz(Ni3vNg=_Mi+x=@>^+5`G<+x`CfUG*Fi0NJz!mZ=htuF>`y|})$_Vgps z-y8MChF_!S^p%H_6;j!^SklnL?@!6NI<<hNkP1L=R#_FIss{ItH&%}WIn#MXcICZ)6Ulc z8PM%`<33Bkfeg5g+|TNB+wyPU<+zJ3Q{AzhTx+{Pr3&JztmClta0+6}uiTn~mKxf< z!NfAZ#<`GQd@X2-u;->+HG(v(GdP`%!*heMI?qBMgl|DoyZg#I8VcutOH#p>-q7rW zl|ORk7b-w>`ryxP)9@74_Hzl-1+MV;U08oOI)Sn;L{v8v%)?)ESD~&?`308Wf~T&L zN=e5uP&YqO1Q4!64SVxjA#LRQv)J!d`rF0686H-;i#0{e8@3(lx1F!R3r|J_)ICdP z=i)NnjMH?6|BOZwK@gwG zZt^fU-I;qU6gZlFc6T`cuqfd{Od$R(JvqlX7)0ul?E_zvELb?c3_OTm5_RW34x4R( zoEFA!rHXG{cTqaJ@QX-qkQhBS-tH@qNusilPcyHZ8fsyT&R}qd9Db1SIbRd!^c6va z0g=c>O=>OydQDI1tQ_&0l$*hJ{7M(AfIEy}n^ho)N(m2i;qI0CA2kdN3=`Zua z@mEb;D?3Lj0PEgb%fDOc$!Dy<@;{h?O<7*8@lc+aRsT!hJ9PqayYPOjhwpnP6~+TfadaW7bG7a_EnoNxTwhxmV8 zpZepjO}$FfWZQ8e;>=O~U7Yyy;?Mtex2^F~zL0?wt|D_K)kHm);uq zxZCG_X)T1uO8zuer~Yy01)A8x`Ba*Y8e&(ggpkL5SEvf?{=v%rFs-sLV~n0FP-ajN z=A78yjZ57E9!Bb*B-UF$h%X~>XJp84+y~um4sfBLeU1>Gv(C7d18Mut&q2m*#W^F- z;CE0M*PioVt+G4k3WnGSvU@`fO~r8w?5UFEPSdS=1)C_E(cD@w%0BCqN?oc(T4b;v zpj&f7C$<**&RrH+*nqGTpEiC@>kY5HyxY=P5I7S|p|HByqrd#bE8RuX%!Ed8-R9N9 z7o_>3e_xjBs)YyL@Qmva72&zj=AubZA0|C_Qq~y=Q);(XY=hZzdmo=E4!j}ULwUh{ z=pE+t2?A5&6i0YP(54qvs`dTk8oT*wUQo7@kOa$tKy&<3|#C_4)E^X+ntI zx5uPl@fq zheKN;OYfu2D^Z|3GJ3uEj6dmkXnx3xMeA-BvT%>)o=|5z+_Nx3F$e%R@NO4mm{C)O zyHZFBiD#M}d$I>YH&%YcnIR;V#%9AU8sQOFJZ@E5d9=|B*!c9s>yno{D72yB2Jj){(hQz}EY$F)GY%n{?GJd+de!6Bevb@q+#MS^Fb&lGuHt4j zJ#(T<>=e+mtTb#CgZp~0zmGn=(oHbssc|9B*SQbJelA@apkv-N1=k)wGl3T(p+(c9(Q&p+IVXL-i?*1HPA}tvA0Gwgk6SqJVoo zEeDdks{t3@Qe|(CY5|`9t`>peLEWTbb=x?YQ{m_+!!QnT$}GxO_aiwK$M@|`o&=We zXF6mw!Cr|XIz=BpyM9mW4GuX@hf!l|1$O{8!O(%4PjfOT6*Qs3O~d{lT>L)5BQr%j z?rx>Fx@YPuKhCghb>jCyDn*f09E#;ADB+v$tu!|D9Jei2g^_3R#oKGHmh>*A4k1$u zqYz??eLSe?|4}yH6%N!uj^>5aw|>cyKwCcXb9Za7y&526x&~=c(l5D#(+K|*|G_kp z{$PGtCJP3oRDs5l9RT3l*b95~eA>ozciT!HSTJEgOhc&XO>Wy##*@}vrb0YItVH7!eYL7xkx{FTBQ`ze2U=Xp*nUZa`!!Oh5@Ehk)ir$l-*vcBC(w{ zeo2=fT*3Ya0ve>yE9rX?N!FvV4?qt_+5# zP*2cD7ACcIZ>fs*RS2i7ljzY9<|xRI%6=|>LK?x5`|WB-D~i&37bEdCvV&0t za9gq1HyDA=ktWWX^v&LifS2kqdyy4UrI_ed!}aY0e=HFK6)jI@t|&8T)q zC}r-{6Bje<-0#NAKm2;8vYZpDQ8Di-hSk%gek ze_xn}5K&mbamsGklk@jvW!Hrv-e1aDACCr~UdXIO`Z)nu_lY}$==%*XP2(vf_r7vg zAV2&ZUL8;WbKwC{ua2AMmG~z1JIFkDoxdwLYr=$aD)@)7MtCRc^> zzyl^q*u)UZr95Gy(|;LDRac^m>Nn9Ssays3y}xWf|La5#HBMvT3-2e|!OsV8X&kdD z#V?u#2HGvrbAKG~oR5dsS1O7p^?8CQLYHg@x2q;`d`#iZ1cbY?7$dP_#8{T&p8EB! zzJ((7K+a(@!u~_;uk;_0mqF|a5#aq<@>3uf#VPt?cK?5$7ZQ0JEfm8(+KzOYTXUa$ z5VF64IxO6~<>$^*d4;Ao7D=Lbj_V_)oWuF`=IQ$(^-jKm0m{pqG`K(%Wj*U(eD^9e z{AUAhx6>T)`L7tf+=vnlkUQ|tAn*&`|273BpUUr`e99LWT@FRw;t6SljGcALZkwO6 zk7@+fjQa6%-z7At9opi!$S}1#CQw6zwv=4%V2hwcYJk{`S4;laM=7s`V-MCWaXSlG znLTw2_IrClm>{(6>3~_5rHgbraq6!}wHW<5Ii{CID@Swc-$6ppQ6zdzPo(Z({ z3N-wmV_Xh!r#NH~0ANHI>bq*`&fMsZ#fl#WMs7+^WfJ&Mtg4GTbdZsp``KkpC)L8( zuii!R5OQ;qu&laE;~v)x6H^ugQjzd4)gqvgHzdYpW)Xsycl_wbKJOQKP`(Z?&g;gv zZA%9?>Z^I(@yEsohtJryXHL+0?@uaWvCG?)W2^*?d0kPD{xSuS-@eiqKk?n9SZpD? z3HJ)X1u`Y{Z4y~c6Z{x9JhuT{Qtzm;FcL4^rGFlkzxb9j{K*w1MgW6FS+;4iG}!bJbj| zV@W+h*y(E#$!{DfO|0}5+5e)6gg&X2?q!n+F2~31(*ybl77~Cpab+7FF0f%NF69-r z96puI{yGIR3nJm?^V3lmV5aqH}KI&_!q(_+l2;)Tk;~EKTmT= zsp@eFH>14=I=6;wsaeY0IC5%ka~^xZg_qdSes#q6(RNuk1$0*#NH`6|se!}q%aGL6 zD*o%$Q#gm_LEe2=A+GG4-71{3ax5MjByfe6dbpGIcY*(TjF8Q=C6Sf0KqI(GKLR{Q z%}}J1K;VUq7###jO2!x8#dVbCT@-#5^X{ITD1LIL{fOKOXNYp- zudsl`WrsM*ARLWkEiBQNa~HCuvRKobD#jl@q2vDa6a32jXYeQ(iUSwOWv9NWWB!2H z9s5{i@=kRH3Y@habqQ0-b~$^aI0xBJB<8%#{1Wsc&#Rw?-ze3t7pwBD7n9lubdu#g?N@NFQTT_9wz{@KWD52|UvH^aX;u13Cd z%`5fl_=t`k&wo~mDJw31eqOqQwpNJ>%XtJP&h<9OS<-tI+nc{51pWNlmywDuN;!m) z7Znd8-r^=KdXnBh=KSPKLuJi%JZBHw@GjI)y|p^2sfDaL<4xYSPJcxV+1bagX`pBc z!YQ+uu7QTkIUSuRlZN2+ib?$nercf|J($A>kps(C3-k)1Hr73y__y`upReNoP8j})xdRyYj=6iadN>id61I)7 z3hbiS^MNf4RCw6GHDUjw4C#WZFz==J*(q9y=T=(dyf0snPLHj7an*W^QPy%A}&_3RS(+Q!8HV zOelhVGI`70Pj>8=AbmFfe)uUJC#-`ZLREB?A9D0z)Fo*caKaX5EX3PHw?-3oG@xsY z18#xHgksa^hkO+O(gZ%6F%{LI8gNY%!ZCz?;e_P-OX3{iP>Gh-)Mu5-N2~+y#)O2j zrk;c%D}V9v1O@YMv+gfjW=Tg3*`}yX>(EvoKU#zCdiFIs*3GHN3bH7pHi&n`-5?N| z5whW0nFr5kKp`U(D=#gyo(Bn;vzpVlI`!v9zB6k;+9Q1{0gsg?N=LK^5PZcrcM@Lq zsNZC=Wp$JBYV-Jaa!o5X6BYg?iI(xt74M(R#h)Oz8QxMa5iUK>)8VfkEeI0(zE4xL zMdR(xq2RYq+^m`#b`Zr4^dY;w+(USS$6Gq#PmhxE)M)*7V*`ojBzvcVZx=ftB{D$3 zA>lHAp^ZPy<|q)fXD^R+x_p6jMACE7T`WRi-OY}LgW(i7ciAsyh8@BW7!XlbVesC; zgpLBudtMCdc$yse0XCW6Y_9p^l*xCeyYC_1B^s17ti-wIc~gCX?RSofrWmWwT(EUM zOx!(oUhdzIHnf@c|H|ENY1aW*62%Qa2OWO7wG94*a<1`fv0SLIDI3ihVLDy&2MW^KGi=d9b~}D@pS*Upqu-An+!6GeYHsVvIi~}+Ork*jHnU1m z)YbI`h|!3!Q1>y)_idQJ5Ts{ti=RPy&Z8$;L!Zh})^D9gT8OG4y9&;qYB^r1ofdjc zGpVau^PO=E15Pv5Ia*k0EIPx)nc=zKo#N6zVo4rX6USGJ!PQKHA51$NuJHlXIk zNGZdKAw2X)hOOJS`M0S&BaWQdUk=FKafz!NEWV63RfSRdPCe=6>B<@h5AWSEMGPlA zP*iAddBe&{m}49_gF!@^L%J?HNtmh&lrBbPI224?-qHug>ZqT+A{Y^RY?F|MY*KOH zC8tX%Ojn^5EY+7iZOzRRlYoab!bT5}!NDB=|K_!~;SDA~MdJUP{qWDO3EB?n~QEcHoaq`Av{UhK|i$F8Jvm z_(`a2nOhE7c_=4qn76GwJCR!WEx3FL@IIW^Bf9se*-7)>NM|0i-10|8^3B2WGgb>< z_?|}P@n6zcrD6ps?Tz)(cM*w3LuRrCMi&46T3qIe{4%iF?iswmme^i#U5+R=novZ) zr0OJitvY>muWG`J&Ks>nzsQAJk!{I%NbitwNJh^=~YyD$EwKu zM~9|F{=isy#CG^t%N-fxVQLk6>iFTXDP!SXS6eDCBx8a1 z09ohj}P-~S{3CIqzNP>h! zpmb;iqQyrXmPAodN7fQS*+PVnvV$l@6P6G{kS$<_OnV;rAxc7U% zd*5@;ch2`pq9&_Cv+ACQ;V51&WQQM_4wk{hZ{NJ%AAkqj$4gN+L|kg>sGX8?|4Wvs zr;aVig}K#yI{I1fkqX$EZF&t=+E$+h#_3Z@kYJLel+eN|s#Gc}z4sAto^=LaZFaYO zAg`W$-7%vZ%yy|E?FF%II0^kSM7n?=cn9yprxSYC^9C?vCXh+{ZA1D3OI6qC6Q=){Jua0e6 z%b-*5__I%2o;if>4n(gW&=-WJ~Ox`ct zm3G7z*(pFPUosCOI|;4&oCHrX_c&SQljS=-*SxN1djtJnCt((mo&!Gqd7eD?m>9!1 zK&=FC3w(0j8jK81uKGG4(5#DU;4heqqL|9^{51a>k_3O=T3Z84OMuMW-a6H|LW?t_ zx9Xw^j={!)D$r+27-f0ofUU=~mXpBKitL94PGWT=mxbU-x5}RH6pEN zO*g-xr~u6wx81{wg?nI1TM|T06|hA9Id?{uAb=~no8zac*7>AYoFg+@_J|~P7ADUL zB(S^&rnl6*|MEsbKRAUX5c92zy7&li?*sPY(plD5&Z@HrfFH(Lue|-bs`ZHF)8IUAaJaD*$y2e`iBp0|g zk+t^{3*{;aZbBZ`u_ktxF~Sl1sf(s-s~?x9#rSAgyIYD@U9Tj;U#23xSO-59b~iyh zS@pI7(i~@szA6uzp>*%meP671w#Y8}h9JFy%!Ai*yxBH=4M2d=F3$Dzlsf`> zQoZM}qP$rDSAJ%>rUPq84~Fz$pTdq78`3%Dor~vJz<|5&#K9-Pds;ms=w00=W;gIA zVzjwbo6@7pg{H)6yC zz&}LF$(_AkX=h$WlHJsuT?uco%QA#Rc;II$n+yVQRwg>+Om&K^ZyoXpXPvy6k6KNI zy7Vqsrj3}^?z)S#rmxB$60<^x{raIvX8nwIuTGg>C8$?|h!aqr8gpC?rM6G&V3FKs z=wBbAd;k1 z+hw%C4-laQmyPpwaS|-q>c=>@+eZzX=OF{ya2q%am0`KIC*rhgP5LKN8s}!n42XL; zM9ZJvWuGelb7ASS^ft7k)BMgh;jpC5^AR*)E+Jyw`8l}aelHsgF!1RPE1Tdt)Ep?zk zJZ}VyJeZl3_WYW8ec-_s$(W0BXk|UKSs9aE{^;Dg`Fto*;G~re(O{X)y4{{(G+tIo z&J@aR4?Z3+Yz-QhmOxH%AoKFO(4S?LHK|yRrwEbx$BTa_uNpSr0I?T*%F4sjoXqAm zRkLKm!bpccd)0=Ik{9m&ujN%h;XM!pfB}tal=(eWlsYTZ`k&gYHQAt;CtjWi{6klqB05F=ovx1b;pFd{)Dh(x3p zkuJedf)J1{z4y?*An)h5-gVbq>)v(mkF(C1nRDhm^UR*T_snzCYX%&w0;~W4faA(# zz3Tt~odoR}&cbw1BcyFiy8->L8|VV?M4>5Kfx!iA37-{7rK9?>10f3`* z2N$r>8|MfB95cD12Zr5)&!tn)qY_;D>e9?_?g>SoEk&J*l3~C7B0!Oz3b6v~3t36@pA1Iy({Q;=I*r8fBL89$=uR_T7p+O0Cljawh5 z7S^s0m@jVI;Dh`lDSrEwUNdw%tx2IJo#-Aj{qyoG!Id`O{Puqq@L^*>NZpNU>EcAW z+sQH>gUa7Ke{T2bQ+NElZy(j>r0%eKM3)r?TK_CsAJkTP&7ayiBacIs&8zxoy4`HTw$+u+;u-KcUGK*IF&y4SEA#``U%KNEaZF3%=e#+ ze#LoGX!-nWm&^jqy!;Fx_Ba?5Yv;CVwNOqOxf{5v+-OpV6SdEM$z)XH`V)j5+5e$@ zA3^+ut+{v1XTSW~GDQWteIsZ4`S-e`7m2xzbEDMzzQeKix^O#XBCA2{?qwy>8&($E z15<_y@y(`$o0DzKLElu~C~7{i_}tU6wmz-%y~CwSx;RwGo$^y`y97VmR@-AyE!dS{ zu`;1uHBVCOUa^_5!*+B%G0h9^indw2oIXYs-^aF}xP&^{Kt;hYVfm!W47!d5yjD4A zyKPO4i+Ox&{k}W8+1Um5)$xwL=kP3}WONSVX>`PX`Mo5xA^|3Tmsp&Jmbu&KM2dK_ z+$zE&=ydoS4_Dq_L>{JKU{T9crakD#O4>Tbr`ia`I~LgS}% zU*$Y3mzLalUn+f~Dy^k(ID;um{FNo#y6P?Ub$xhkuFEN(JhIwGR4FqVxEok}K?fPG z@}k%YXJjpyUp7H+L^hc8XQsm31QaPCw<|44Cq^pQSUt&+;kUgzTuOhYI=wUH=Q$QE zP5hy9!3oawg7?k&rjUKz5dBLFO^;5tyuuj8n5z&yFbbPIc^QkJEka95XB8h;DN37W z5DGY?W(gxvF`d82?Z^9N+&yjZa;M#Xq|Qth9TToBq4@A|mKG%ejpm=-y(&9>qami3 zQ`cocK499K4W5}(IX|#89C&PiA$p$5jO@!aF&Vh}A^;ja?nKq8j3HDD$qc-``Ifa5 z+I+3d{)E-hRK`z~^NKZ^;CWVb237-u>KdcFsERRDo66+C7Ixfk9-JynxGVl_$dj<( z#w}@$pVUg`74_~86S|}nwZHv-Li05?d3;gw{bNbsHp1c+J#u_fm1i8zz2T#{oNY)-`xyG*{Xd6LVYG)y4yv$#deN+d@?sG{H-X81-tDDmQ`SdLx zk~Gncnvo=t%L%{YMrR%b%u=;*?V|89da%H)BUVSTF_waami1Y+a6{F+EtADK=#(4w zW;MaMZm<{8@B4kiFdNlQizlD-gBL%Wo7L8``0Ku=;6Xr{5HWQNa3PWRLkG8`C#o9b zzB30(R)~U0oNftG57m!S z`zPIC4Xyf7UVhhH3mmbZ$uqV>V-w{+`7Sv43rgLHZo_#z1~rH46dMb^|Eg@eW|6-; zu90;-e)eMS^^^O=k34B-)?at*jl?lIPe#pkC#!p5OBM|@>|u{~W}06Y>dn68eUzEI zQ}Q}~w>Z|_SeB49XP(2u9Dv00Jq@s4U(-STWx3)W7Q^=o1{SpVSG|+3;ZoggE%PSX3&+4hNeVsWb~?r zb_c)c@#lJH;xEge%fQvh>OBOUJru86T8TxEpRq5z7mQ`4bpaOnBL4mKx1$0#WhG?U z>5f4{*KGs8VjhF3Bw#|EIwZv>Gjn} znV6}YOi>@kWESOa-BEowhIm(AY__poi#W&Sg}umAMSU4~jE{r7A{)%UW0vnd_`Hc@Y!P-I6zDxa^;A=Hl}hb^0J{P-qv90IyRZ zkJk!?gA$nWUNe0C*7aI%+WfnTgA(#rO?aPmyl5U>R(Kp)mMwF3!8QL&%Hi+Fu@L=l zS3($4Q<5*wX~h1jGyfncnTl+-1uoFj+=(qLNx7POf&N3kjI>~*J6v*hWo3GzyqTUf z?=U`O3)PGn`%GYUIWRH@ucBTS3zzH>?JXmM2)jek6A`aAIm??zEu&g`t~1&9HsTW( zjFv(hZl5jo%IK6xJ8~Gj9CjqSzMk=Bi1sTc{Mr3J>%9vIb-)IsUO#_OnQ|@Ttw*Er zjk0uxigK2(0rO|R^dwflLW&3=(^mIMwYv9i=lPsFfE7glLKJJw#D8O@w&EK^;x8XS zH5Ig^$N1zgItG5={|cJ_v~U$(u(B0Ll^75Dod1!pD}^p``_`*UIN~uKw}n!?pQ<8; z_z@y_|4fy91;rm-ckr6&dGC@rT^@r02R`sQG(K{x9ohSK_in2k_oUKHj$$27Uc_(7 z8?D-GO;@t)@1^d->)6QXe7w zYWxgjv4D(!d}T%^cR}7kL88g@x#M-Nc3cl>t21J0x-lhU?O3UaFBc9;6m8S~$WasF z1oObeUyU5>z-@0x6g-x0MfMu!dym&y9)4tK4;v$PbDZ$f`yo9Ro5~f%s!=lv$;Og6 z5MqRqon;;qDPba0IN)Hxci}Afc4h~vc3CpawHGCmp{`ZbDL2AHbFgXX;do1-@xZ^} zOQWAJCV+LAj+fTVocq{5N&&DWLwiv#l_oDp7AsC`i{>pn+o`e-aoh5N^D`WYA2+$rBDUp1T+B$`1^ zUYU5`_+3ZH!1{yU3faAisR~ei>23%JBy+)A!wcFFTlLqrK{LIt=k_hto_62eW{}@p z!ycYCq2S+CVaFwl@9eKSEl6V2dOr2^Pu|O1xh|j8V%F=THb}+?yc7D z{5Z?^i{;%M-;2+$@e{@~*E1+BD~-NtlrcOTuPrLAQ=` zJVNfiv(ifYy&r-?*C9?DDs=<;OHD|7# zGuFz@y`?U6RwfVqV!r5l*U5}J0ZDiVhMkoxZrb{^GmIu8vJx#>?YJiE8s56SrJV9a z3t=Z}mz+c!SW~06sem&*GCZfqW;D&pV1}Q$H5`gyBik;e?uC((%>7%}8BB10QZNq2 zo(V6eG)Q>>&Ds0-c~n|P!}V=YK!jv94bh4=jiVb1hF+K;vDt@zg5di=RkwO<@gc^t1kA9@m;Acx2P;TkY#PQ{Yd!!x63s@N7=leR%kL&l(l6d^Q?P zQ+XPLDmaWhXJx;{z#>ccf7sh1NH~7&CrmzEf6)C8Bk)hQ6@`cq{fmgkg>81n&XFVh zZ*Hne7h^t{L`T?Jz2s&V7z-_#UN9qmzCL+=y7M;1T`~Pe3-M{u-`F7HZYOs=Q`sI4 zlT81%&|~t|DTSNHR*;j=mgRquX9uRDdWLg|iEC3H1HFW8JDX}659cr*PTd!rN+YIi z^L!9jFg9z&-WE25cwcL;%~+L{mjrW=6Su(XNC{tKJ4{(a9wq=9r*m7_GWi07B7`xc zQ9yFm9r@a5f74nxdz;`Y`O*`iSt@D*oZ0d*I~1)cLG)Wv4X$QIL(Ae>L%))jU1w4* z=)VHa6KkIm8+Es`5Vih*SA^DaYV2TsXjV4pbmJw)Lm^dzh%tO%AScfWqagkXf+vM% zeQ$jp=pF!DV0?jGw6OO>8#1duOfs>ND8*I;@!0P0h8<=eW#Ch!(;`g`{)Sh!J51L{ zY{4o03dz$C`;l>>Y`~c80Lu(hw21r@Acen-6HM! z$LDqu7b)$nTh}BX-pXis8}BSUG}MAGoZXB9$vk;G?*e)XNg2-7OV9m~ z4p7Oe@t?a+tNSn^Vq==-4j#XpVBv4}5gKcFWt%gTah#&26M*NTtr6 zm+Js_NAg=o6gkn#U}!W+>}!lia$G=14IAN8XmF%Wd3Ew&Eq+{9K1K-R;ki3RJ+b)v zGlN>NZ=h{UZyBuUS9a;paq(l>?>Db}0|RfYR4!y6z!GJmQoMjmfyX-eK5H(oucb)x zK_u=GDDz$aRg>g*ICbzY1tn*?a(g48ZGbXvWDj@Hs`RPG`bp&TwCT6>Owd5lkGzdNM? zMhd5p&)R+@Ddj&E=^|(j)o`6Ssj!%kaYYZQ6?8J&1Gpd~pEC{`1}!oqD`V9}G3(0F zI2Zkqf@Q#wOQznlQ0P{Y=WwIJrz?mNg+BDD!$EcW{)<3w^QWY`yF>=Up!uFbzP%Sq z+s@j)PCa~qKfoKLowc}Ej8aE@G1H#XaSUq00fSCy0}U_KcK9H!0L)_qU_nmD0(3_k z`@Aebtp0NK!^VZj2YW7pJkS;2sFRdj_0}_onrX_Ue7`YSuzzKf-!m&&$jz6%5zn=? zWU^W3*Xbi>(33NH-p{j_9N$Ph-j@{lx(IOW8GyZtqYGGn25{ne9P%eJpkp-oUP<7XHhC&K zuor!wbBe|;_cgmaLw#AwyRMI#fCB%#jirDQ&tii|s%?MnHX(i;WpqJLRao~hpd&{` zc$nxt_==!jolwlfPJd^|qL+Ng*9ZXaYr?wGeUE3CQv%_h{gyXzbP~_d9#f!|xvw$H zc|c@uB!T4g2LB}83TU*HWuYksH76G8g1P}mQG0T<9E~@>(k07P-hugfeB-{kr`BV> zKj%i95S@8(!mY zH-Q|Tlnd6OLSeX>)cuba&p=_2fxs-cg@fTbR?)|h68l0-}6@VM3jDkdi~*2 z>~IfeKAXC7U_Fj4!+O`rWJ;c(&pQ+M3 z&dy$vTQ;Z?^s6kfC4)GofMWTn$kU62biIo$nWwQqqJATQT>(a&wlD4T@S$<@YTwLmCd#jn6T6ZjnOPs2G7PSh5>`7OC z{ZVCRD|sHVM)cZm&eW2Oi<$pG@k-FP81}QJH0rSD7He;G_U;)(ceqg759z(r?qufe zBh}nCqwtl6X)pXpPe@&@M;kL9^CXgSeWXGlhC#>ai*mVf;MKD-2g5S%7pLy}c8Fmc zQr00g7`l0ay(wA37Pj$p{B+k8m1%y6;6&XTH^6avN2PYxFm=-*!0}b(f=g{hgzo;0 zm3m8z$ALEm-d$CPCl72-%B)bSC%3a{%Z>C^6>j#Aj)XBDtb(>7jZPg=`fk$QZZB#< z2iVO~^!gh{#wu(>EUee9CUf-U0G)-uA*-(kyC|$)ZYXOD-nd;~jd+r>75vzrQfO(! zQpA`QJl>iDw3M^CZ@UvhFzc=LUO#?h_J=bY_}DG*(}b#74LsKTn>Ko1B~LszFZYM( z{P>|WJ>!Qr2gZv_DKXdtUf~|KJSzAf0WDX6G>pf7A~{)mCd}glmN?z(esxB?&)#UD z88ZCQL}`23&I72xy`@z?9~8{Nwkd7njxR_b#lsb3F^s-vR#7%FQpZzADyV> z>alo6I}-Q9a0a<#)E#kds&ZNLS07Z$3F;0xaM9f7oEe?Y>4sHHORD>YLn%X)PArWB z-DvMm9kcE(4Yl9Y2K%*BbEk;C_tz4va=G|RSC+bQht$vWdjykUiuk}`G>nIulkZ1n z@rHvfX`Tup&7&fx)@~So$7K$d$ z?%bUu9QKAs4=+{|!k4Nug%PSIKuduq47%%zd=!4nY=YxI2H5OGDs`nnt3hf$;01MC z!rs>$eRG~1WtKnNmTQVI`L4yk54j7Y3 zS&lQ#zj0R$iz_i){nj9JS#T^T+Z@nhOnP>}WDt8X<13|bA@*Gb36I5b`@%g=stA1* z)pkskES|oj&(R_*CVEt8$-Nsqd51?^Ny+7R=U_;(5@n&=j1JgdN zSlrVc%rdu`8qW@O!wEo!6IT^Tb zx$$KyoMK@M9b^m{K?*C7mV-Ic5l`O&c{`l#C`6<+#I6i=ZmNyj{`!#KM6G|!omcOi zDtsjN7La`|eafaz+0Eqn?7HX^>di1!6`yJtB5c}OrM%uw!Yh7gdp3$jn967a$7Qj& z#B1g963nWZ9ab&OhB)kgM@}C+e_saoFhZit*!KF_bIp>dt_780AQyb6s#?@D9eUrK zrn>$RLg_iCHJ02Nu_u-3XUozl$zZFyKL{+hMhH(PRP^|LxOcDqZ)52vk%aG``RLM1 z`bMRO>Y}p}9P1Ll26QQAp38`B(UKw=>c2b)RG^7abz7^BJ>$-+skMg@>2mQL2>R!G z9a4(~{Ic3!4JUT7$reIUWtAb6T`PTtCz6d-b~-;{OZJ}lpU?-~cnV-t3|wMX|5!16 z-u%;E>8n(hCi-ZxI%`?CKCGChSWr&xrPK8`y@tx{c+*f1dG&!#NvJw1Pkc@i4DM#Vq@>Jgp}Nfvd)8fQ+CAQXw^_uCyuB`#eZh4? zsE5$!vo5G(773nd!ahL30@*b3mI1yD_{2zS6cui!$>P&iIl!qOq9H zw2KE^?7`uwbh4ToVRZEks?F1$HYFz}lVP5Y$uVo31S7U>G_i)BqXZ7^d)e%$8k=%K zFp?+v-Usn#_!QcdfSZihFUfbL20r_BW(?oME99MaA~C2aZaIgx?7FSk5`z9STbx9cJovo64vgZ(|Gj#36`$(+=PhO#2uS@tWG-)w ztjBw*z>Nx=FMW^SLGrW!Z!-P0L?1p!PFvqGzkEBWAnA|f;su?w~@(Z7uk?`$~tc6^*a?(ulX{3!q-WLS0Ly4F&gWN6l4i!zK) zS8;UlSMb@q!$wX!uDnr+{N%eqCFFiLh&N`tz4JkX?QIR%dWkToY?FVUv~`M-YY@Ij zLS|LSzPAf4)3=?nA{OOS!$#?{lV*B17K;vfU~NeRt2{3L(l}D`JWJFq4n(a?hm9(&HG$f{wva*g*Mc<9u2t_IU`eR0tN+a^Fqw$lFtem1nVYs@n zp~1s^Yo2u$XmW{lqx<(8J84Uj^!P#U`gBj4aXa9uB|{U}^V3Ms;V*sO^muHS zDj8-{wYI9s589E+-vnw zN?5{{QKCP3IwK%2^6F>{>~uK7roc|7-^aXUr2Ku>tqNfMB(`XUwB*C*39y~_O|@29 zhvQ0RnB@wB!w}p;^VA0u;`N?Z2s+!QaE3T?a*-WIze{LEoR4{`KIw{S=qtE6V=e(* zr6j0oHlTjJOHp&Lw*U){>x10oP;NKHpA1JZYr&$|-R+Bu@BNI}xft4_`(m2xRN4xK zCV?f^@5`spvxQshCLz~q2>)*$$4m*~)P}v`IX1ff`ngHTo;JvnF^K14X3PyDescYs z;~Ukc?{-+54cx|H2~~4WXv_x|TJ8K4TNMrX8 zQUa}Y8Kp9vV_j8qGihH3*U0<#m6dQXjB%$^DtnCI*5HB3EYtK$kGA~gb0X^;JJwM1 z1!QIZD75y&NPTZYv0Wa=v3xU{8Z1tlS-h>Fpin)0`=TtY7R?e6p?W2E^`13`du`AQ zgv5Gzy8f}F@dCaktG@nZVbRcPEp;P)Gy)3GQz2_N zPvR_z)Gtj!F}bOu=QdV9t@jLC&QF?KaNGJZHlH?Y;OE)l)6U>>zPUlTcPS~<^k-f( zNBuD6dsW^pmW}6*P>R3j-Fg!67x{~oQ4dwuGXM@ITMrCsV%xqU6jdViAyLrlDSLw~ zSoX=9$5yzxL^PYNIhmY(1cL&bRR*-&URCwPwg?1_w({Ixf%tfjT#B?`VkV?;n53@g zqwYV)7dnv0epSH$saMhShnB=1_c9zo*-XMvBKpW)t_EV;mHD7pAG_fKI-Nj>@0#W! zj+@}?%LjrNeIIBEX2JFG-fbD%hmo%wuD+5F%YBd9w`+cFORCF-UM=`B*v1pq&%v2< zw!<%i(Tb&&NFJl_)k|Mo6iNUKWh$MRrO<6HCf|=BS{XBQ=~H^uzmqOVqcmcO>r>sf z*$(Y~-LpcR)p{h}Xn9rii#2O#T1ttf#C3^_+GDPO${|el;t2ohyNL7Z)+dW8i1Yg_ z=;WXn9G8$fvo*+Vz|O_J>bk(fnTFQ3_R+Ed;XZ1F?O6@i%dSZf-R1hw6<9* zl}GC>+~@_~j7x&$d*qRt(1A+gK|wjN6Qs_O(ccE;uvIZCsc7i}Vtb3%&lJ?~?;?zZ z1NtJvbEyWMhb3G*`w>3e<6l#W^^Sod;ZyLW(K%~O`6RP&Z4O~oOjhoqofNr5GW6p` z)4>yO?LyQD{rk?Eo)jalu+Pt>8X_L5JhY!lTRa$>P}NVfakR}X`ha<$%;!-8KN5Dc zyjb#C9}Qe*M3ZV#SEYn8Mxif9GFSUcdqJPE+f$Z^BPlARxx~*%2Bg*UVI#Bs!X=Ya zYim*t^>(z!xGG>KP4Vc*!?!7*!cvlY(UTx$eKF{db7ImxTzq*9^qr_LFHz4$@zjak zs#s{zn6VavuG4fgn9zZfjZHjg(oK6iuZdEBJ5ndD0|RUOq~4Y$6xWR8nPw3ZJZJ;$ zcMJxud8y5hlvV<2G9>%{&=JvpQJGUm zjJ^NZUtpxa8C-u$cc8KrYsETUuB?mBdhc`dFikXGH3N|f^&uvg-@bbgYh^7ChpL2$ z!4Hb~2(U?bZ@L)5%5+8Kc%CI$GLqT#F<)6f6}Rwj7`HPux+L~k*K%(Gm(;=L^s!5pZMnB9YFUO9y*ZLfm$bB|g1l0X<>JfkYGFvE&6XumIvx=3%Aap zBkPVyZ%xm)TmM=V>{dPgDTSlbv-B7<+{27dV{*ta^NBIQYm4fwdrVy7LL! ztba_u=z*_IbdVTeazqCS(4TdeLP>Ou&cl;O^=&ZaHT5~6r4$0&E!4mOYs8yEM|NbV zf8l#2lP}Q|b=~Eo#{tUz%j%A#Vx!Zd(+T4X_TQSwOD{SFTWZ7ykB4^`{<;Hdt9e`d zLfowSqn3KW2!S!OSn~{=b=$A_wOHeIr@)K!*%KNAHIWq3&1}e-@CHk@)hWNL#B`XZ zq&`-r`H?PRW17XU+Le>O%qOD2E*?Np%fSF}wgL>4gtrc4O=ntk8IcJ9_va#1Y45?m z7rkvt{^2duj6__XE8F>37A>M9Hx7lzpRV`Zi3vCpP`f#_3Akup+CQXB@@`5bZ@>Do z?zB*&eNiD|=$35x08s-t4f+)mg%{L1a;O}ZPU6B#CJU<9Z@@{qAYsIHV{luPl0iYp zc4X1c!X@d=#r&R+2a3gSb@#I|A1J^wd=-#(@AT#e2yj8i*RubTukGyRC{Rid=AZ1| ze@~97SEQvk^z5aIb~RsVBxujxAAsde$<5KLA&zC5Qwy{67RtdkzGA4qT&F&fA@Xsa z5-V==3-joQU+0Cis!!fTtxpSQpy?gCYK^=e6{NqxGVAP5ZbQ7c^6D_%Y?QA*!3JO@ zp`VnHOP>H<27VV5W%5^3i^LGmqv^C!akNyN#i?{pLc4b|Q0i23H|yE_`NQv?S4n&h zHF>&9{$pv2H{6eSS1O~tI9@Sb+n+l1SX)EjISkfLDBW?UZw+JCn_)sHOz<4W@l{{L z0ter$i2=`*>s}+i``c$9B@Lo4tW7Tt=ny@lL-35zG#h(@G=%yUN4y~VfSPeufkcalkd#stSb3Qm>dH-dY|ECTs(Xj| z9tWO|pq(alsh3RLI%a2&HA*ZwkKDxEHlk*eM5+V=j6Uto)P5_81AOm$?-UWk zxT4&PpeE#0qhqa=_)*w*3x=)Rn-mC7lOPSdyt_JTgzVETU$?WHo`jTmc&&&CO{p)H zF0V~ql#VBd+78$eq_|DoJxR7^HL}X^YjSY({j_9b)L^FU<=~K zi-_24zs0aB{h?|9{DLczyP8L``R9bMp~Ik74GF}^nBtsk`=UnI{J-AmSuhtIZj-)<`m{pp)v7h<^f`tM zf7uyH+VT%F0KMf4xB9Ijh2Rbjm5R+RDwK_a2nOI{ege zF_}D6Xyj`Om$SR>KFMe*Ir@91Vz*wqc_&oht&^|H`eU)Xz`S^YBPXXZS3q|p$LURC zR%zOII|XB|1|Klex-hS4`Z!)E4Ow8;x1->F7}rRQ5mWs$3nvrh}U2kll!6y6~m!{uIwELu=kLG34ghSbwvsuZh=Z7gZY&e$MHaQH3 zvE2iQ4xF#rAK#JvD%3Jqv@GdQE!e_Wd0L(0P9ut+bZzE_IX8ct|N8L2@z)!bNsr=w zc>XoThb_V$d16h?DM}tKU7g@C#x{DA4}YfMl$3J9Nz)_SI)g7)Pi{7Pz0|gk-`yH` z;YGNmUs&FJE-e=+s)KP4e)zDeKn!#1z=~JhP+zgANn4o1eAZ6*dkhmmg8?l~qs{8NDZ zTD<)0WHg2+$@a$Hw?8WR`uBUci%raR_oE113Qc)*pLp`jMZqUubF}YgBT4*at!~{d z#zgXw>doI#h^cF!7xSK36)9Z#+Rbx)rWc9W(bME{WH2-3aZKgt<#z08wo*bgGK2Kz zr7nZH9gp%u_2x&OQ|Ql+K&2dA5C`ifk%$mWU}U-}%G~9`{etp&eFu$Xq(Q5GG`E8d zB#p<>mrXPhah>4vyj}upm|Rzs@RHk6vVF$F#ZCtip9iaJ`Ma7gD5O6>fAy8yQP^W! zxT9lZeaq2hPa`!y!HXZ6slYmA%@NLsC-jPk^XAAv^lIKM)P(KBP z892psF}cxZ(e-yWk(t{?X9f9&KCQP0#06}($wV&|+LV5xi3IQRitI>QtxNhnDho1+ zq8Cs6V@pGz5M?G?LH-O^ud1R8i- zU=LWTh&|G|SBe)we7|m#nSOZB04iM?4RI9I{o^!(ygb{TRZf#isp^yB2rCar@w~v-#_-nQ@sTd8@DWnV}2_Yy+kiFRh1Zw_Wb<2HAH9pZhW_f zfic3agq~VbFlKXw)X>qj%WG2EQI95iFqQ9YZTVhgHRm(P6Xi(SyyOE&E=3Az&w|q+ zRKQ}9_@&<@QE9;AwocZH{krZbhYyxV8|IHBJ&p-??y&KjePuk~me^^zdbsb_`B6_^ z#qLkdvaly%f`X50V_}Mf#pByKR?bKB2WO2S)&k~M+Y#?*7DvTfAquAndW&*j4e%9Q z9erfMB6rNc>N!=&;oH!qyBjL5Z1qSn%$5UuQ!Hvrv*W<4aJ?$W%kJuwc2e0#&jH=( zL5oF6@R_0%R+qfLVi@U}+cm`tuex{k_Mu{ws(~Kgm0|+&MRFaXhwoZbA3d@f+W$Qr zz-~S1-Eo=D!QpW{d<&qS##+M@w->dKJ!0N32b8srnriD<>I? zUWxMnl_y&;zuKPBH`=x2NYn^xM|Ds!q2&BjfY@%`MWb5;JsVlZ&#q$ddnC^A`*%%* z!S~A6S){c6QzfTx=8D7D>0agk%3MJ%XO35{pvn|SA;ss{-&5{7 zm8p|=q>U7LSTF&8-(Y=dI+*GWI*aI%0(%5LXIk%_$a$5MDF>Q{dJph~F^Eo{2N|y5WLcvb>?=+4?R0rXIZZM)+!!KOg``61ql0?< zDzmR`@_1VDHOc8?S)-)PIKhp_Idx%8zLG5p-7aVq@=73%I_s5T)3$y#4ET5M2J6Xa zLUQjW)vLnqEg-8LI5X^1Q?G$9l`gD2cB9(64# zu4b+4vQz^04S3;|5_#_NJVRMU7-NIXZI^(_54-^3Ram)6u$z)Auji3PT$7^?!l^RD zShRLgU&BZpuNG%_A9gQzL#uNuejkoMCsj=beqNk3$6fAbX5s~4&Iq0?Ap2j}Cf*Io z)RP)&n$L}fhf3UD0oQCMj**M;zw)E!h0pAkduQIWfOlcvfpC%IF^#z4@W}0M_YUHX5$J~@Yc*Q|&x1h501FZJ+$+CY)I@(V z*3^B7*mC$g%ger?j$BwvJ5x}b9|x@Ntd_BWSx2 z@|MjDsQ?5qYmrS9@xhw{H>;VhIw1wIOcS*T7tb91#?gG&rtxT#IL*mpRc47HG8Vod z7^C0Nn5-RnsjwmW#mDCh!jLaJm{MTcQs;TiBf{QW9i3s>&52!Ya(ang#VGwj_`n&K zrtIY0)B!iW23e+Q{nGfPY|SboyKr&*-CWEMmZc3HtfdWF64A%MN2hGR0GK;waHIwM zvB6eX3}iWO4*db*>L$tc{y$BAPV?Hc00YdaE5~y7YjvF;Q6Z9J#fN zV7}z~i{Ec*`tM3=xildORW9DZez8Il8i((GE5Wck(vT}OsmJ9f@14sTZs(hN0Y4Iv zKm_{OJuSW0v@C~GWGAVxzmOd!0NiJyqIKOR%+%WYgp~Qq39C_a#i3WMDs-~i8eoa- zeo427{>~KD&$-R7hpgqrP;Seg#B-RX!~*-~XvevH@e&y9U_#>7FX>iHnv!e(M95=d z<{0Wqe&sZZ;jTZ=UeWdc>T!=!S4!&mk(#hSbQTp$-qa@Z%W6Njdx&Z`{3fMv1XHNl zHpB0`9*Y~Pnwbu;-uY}r0S3w|q{h$(q}C9mCeoHtvzHXSxOxF5V~dM#$iigpvQC&t z5wS_Q6ps?rwkDOIoKY9Mz6Tb(7Ltg;9a#xF$UY#w*E@NB$-#EMKjMeL=|7;dvUd{v z{_v=P!wQIo@4gt^iRO{Z^PRs}JJh4b;&|eot>9C1wWJETblyt}Q{rO0Re-aq@Td>s zs4x30v*WyT6kB-^Jla=is}=@Bq~nt%!0Tt032w7l`y7Jd>EcQK0p(&zcxj%Q_zPQIM zG;vSxbflZOf1&5x4AMkBP?>EsSL1KZS6{YOQIE&Os0$sK32Ge(2i{vEyHS1JQvHRc5nuEcJf4pr(66r!38-Tj9RrY> zBF?ZNK)t&4IPNh6|6BiLz`(AYl`=uOE?zbhs?X6G)?%0G8h+GS!kX%kuF;mSrmwFM z{{8!ATi6rXPr0XAL#?rk!>>*SR0A4gyxRM!E~AqPeylzpsk+#E;TXmB?*1ZbiLs!L z^pzpG#bZrIaI*G!^reZ4lrRsEkg!NyfD}ePduG*s6TxE?At*oz#PZtEx;NK&qTbb z^B5hJH56=)t3Q||%kK9@;%+b99Ov3eL@_=;cHO>=`%^5#tOABEw0Jk+OJpjHEp~J0 zp(VuOEjyLekkS2OQ_E3&{cVFb7qF~Xv2sPG44(6zF}VI*`}FlRK0Xe89HbvEV@}O) z<=r7n^=z`PlBy7UCDef*3UI8$2XnauT>6Qw^7o9-b7*ei?!z}HFVukekm`X=9tFBL ztlvI%qfQ=mGTitb&m7K%x85ZmiU-Y<`i4X?9wGtmKk#0yN(>Uz^01)re$JYSGmbzE z7@YS;U|f*&6NwA+2_3w-V10XsI)?fCGkd!%^Q)8(s0!7* zgF)@-)0pMcK+b+jrt{eK=BG?=W7CL`K6`3`Yr{t3?2C);f@-FpT~B(dIT9So+q*!( zsy{Da)Xf#zQb(({I_+nI$)WSs3z3=p^}ylnsS=+9)F2gWE0Uxu^KVMHw^rT{Z{KB- z_!)5QXFu~-aoZv$8gyJ6QSu!Rv2 zl}y?1*QN#a?!o6-<78MeIU`QSqEU~)%baygxk2?%GSS(_55FhbH+@lfB9dg2O30h) z8dLIxmWE@r`Hf9y^kgf$7>hEH>fLltc*V%a%D{9WMJ*{kWwF!MeD!UE3NCM#{$an2jZ*TS)r7zUft_6JF$AZ*_&;H|}-ve0xt5i)_A!HE zf=Kk1#m5K8P6C%&7c_iUa(Q!ercQh&r#5f7#6ZH^{lZ&>`k`c=AKzuj#ozirr1QPFXZq0V07y(1L2j0l!=R0y_?mx+53mE2TC%9dzm^8|R3 zGpv!tDtG+C7b!O)5#G3U`ZWXkGR3{&vvo4p(Rq^wk*u{~tLyHuxiYqKWKoY3PDPWN zaxMI7U1Owx-V{*)?knC=`dPf#F-3+-21)h%0hKh*i4y~`?NaRr#KZZzHDmf9t+c~; zL>O%M_MvvhL!)bgHc3QWf&sw1^N+nmtj8RZw(*O_QNd%8NfQ6oN@IHSUMr9Xhp0gv z88d8!6aQ?P>_WY&&EHZZKwu~oRen$=Pu(@yo4L*lS!k)my~2jd4;=A+ocm0y+gtIh zeL1}H;bnhH*7<|oi>sU{;DJ+Sz~0^AAf>Y{wVs5_!>qIHWko4W=iZ8FtV(ySA7443 zK4?cZNmZfm5VW5Ya=o27fCq1zeVDn9`r|zPVRU4uBd^=bTiW;TA^vom*~{mTm!gE`tgAu0WYYG5_Z*)PC-yY-5VrKp@O&b#>e zO~s48x!&EHm!zvOzJ$9s$=V?6A$!YJ5^+={i6ehu;)yHFdF1-zd zxt5cUmvu+O7w{AD_*(_)T(}i4)Jm$?4|>UyIKR4}FEicTF_SKb*3Xl4R2wG#|06bC z&3=hJrVxbe_jrq76mx!-#!FPE16V%4tnL+dF?Ev2b2J%>An7UMkNx(|O^7sgDEu>t(h~)&rR&@yOPsJKa$8n=06)D~X}8T==C^#VNicFa3r;`dWQs%TSv0&GvN#X| z9meEo%;^7?Xz3ZY+h77qBDhYGc+C?ue%GCn)rfkYQiRXiqZ^0N91O3ReO{u;`jFB0wWb0;Q~HNgF?s<7FdUlmNldFXO;Q1qvNcahk}}lP?1&IGc^1z!mcpFF zhMR-+RQp>_QSFrnZ{LYl_x``LpE=n7k^Qt@uovapjubwaki}Q4iv6BZ_VlX!DftXE zh<#o^lE3rzfW_08f_FCE=W5u2PHQ5)AyDw<8cuq6Wm)lg^#yi%sILbj=hk4EMkKaB z%fuNv^m&6FJP`>G4oZu7lm=aJ(F`8*@fBphp~E?5db2+wW!M#)O4PIgT>tbI6qt;R zsPR@6z57_A<7)ei)`*u0)%8%d&|tfgZ?a|nLnh+wYPJO9Qeu`;Cis;)(?UJ1-1@z7 z&8aH2;a@fVZ{$owifpCd5BKGu(~nxS)J&X@DdcN%D^{YTcU0c9nk=zS`(-Fk>=$(r zw~e=d>iqj@fZB|1c-ux%kw!dkVg5NynoZkusCoFxoT{yd1Yr|8Y1H5JpZMsW(z=!m zbPL zar6ZG!w~6*S@hxYmFbDHSZx>z-IclA9eeQEodt1&yZz9@KP4znIDqqliGL{T))uT< z@)<=Wv-})KG(_wiXI#Z;;QS!M#8~XPWz6<|bc#}q(sMAGkq?`W3*+!mQJbMe%hEO$ zSQ}!X=Hkf8R(;4`E=-cf-Zs>Ld@fHJ*njSilCb<;r|wK@&>tIc_&`+cZ$d(fN~4bp zzN|jC3NM2=OKvsd*}%5*xS=MGMN4fq!jshdu~2#I zg@-3riDni50KLk#u%0~Hgm>~sKWiD$+*bp2DvYQI+A{xBx189~6jbf}4i!l0FjElQ z113?A_^jd1G~C2OHY$9E_WM?pj(h>uQjy8Ad44y9RZ{unlAClFv6LyTN)A4(Uw@5c z{?JRH*3aw+Qa1BNSpM#5f-b+8VNZ}6ab7m3{*-bXA%hw`YvlI^Np)On8V!CzJOPiH2oS+g)m%dLb}NxKmAZMzGfNV;kGSpi2AHgCTMH*P3y=MPxh<;ZOGVy}%R*TZfS%s1LIROT!c4TultsqUGJ@w=aM zwjDw7ieL8~Q#mL*mzU*bL)&N}Mw-5dg*9^n?&a+~AY|8wopL$dzbJ6*;H{O;g1rx; zsk_+3&0~LG&HJMilSi^f&OWmZjArJ|xLd)_Mvw~aH&2%In^Hc=MJcI&ds49Ri=WX+ zUx2dxlFRPC-JlgSQUOVM2)nE2>>GzWPE{!G+TF7mVfv$)~30f3u&)%*9&mspu>|M=m5W$e#+N6VeANJ zYSp3cgS$D@EQN^+n9tEm9b>?sJP?nPc{6*D59B^Rs_y5h4tP`s>%I5wamQ110hp?| zU;OeOcw9+hLCh!Y+w!!bvdo8z5?`#gwh}!3d-8O<=A7KLUz_K42+O4ogW;ykdDH2P z+j{l5&Nvki%MLXzxL(TqY>RCr@^mZh zmfc;Sk05iX#~_x7kmRE+{k?f&=yv$8-Iq28{TY}V%L<2DLy4}NRvfESTVtq7zeVz3 zD|AB4G}N+sWs1^!WiRUoMkM%xAw%t_o?8`TZK8sXN(Xjcb-E2FVxU164i(XW{UP`s z>c068sA}!@X0=vj zhE4lT-{)DaRDirLWerl2OZ%?PLuhI0pUD*Eq@TS z$RheSJS4U2XBb%Nab?FNDtz;GkZgYiU4F0^OKwdJ>PDXSERVtc#{NezwceMobo%0f z0|knT!B=%Sk}?z38yWV~E7ktI(cuDj_=FxaQtUgW?;YE^%r%2MRjv=lpEYeCJWQNFk%< zAwBvsLj6G%C7?|%R;xuBCZ08c>ditVR%;QMs+^ur?ecbd_W;P9+$&6|(F_;6+- z+~ES;`w9v*+)wLSAREd#a)TzVoMazdqArnG|JSI8nE@L%nLgW3Sas}I_UuSM55I!i znf2}$B|I!tLcZ7cD3B=-A&fX_>n?F>A51SVTTclMQ3b>J(yU49PtUVk{i1uf|8fxO zH(hLM9o>shAaV(B3^8E4R};XWG$Q=H$cx+TtEPB?n!zWDj^~&hzLh$SiR@_>q)P9U zy>1qX@Oq+X^YU(fZ|7jrz<6Y$b~AE)i9ZTHDMw)icf%)+=js-oImzlc_H`~prf zT2VH(or}36PG35xGK${;4~s$>B#4D^=5@SWKNNLLP`EuKj;cssL-Ra_$HY`V2?Ei*at}D@k z)xy3G{(L?{f0j}|U5L^|CjOI2RhH&u|5_1)JdcH;LC!lAp@q157mTAuwVN2UCE>I_ z{})-7j8@iB_nVVN*sDywf~L!t9}L=;-yS`F$$U8}h>c6SYG4UHC$75;EI-^X2}A{V zh6uq6(gPe3i;EB|-^aCO#@J+IIhWhr-bgg;1P5pnv&?h(v%m;?_osI>tfxaMUyWx* zBFkZ+{Zl$$YehEjwJmHkT?H#HnyG@&u-=b-(J@|I!lo07kc6%0S7uH6Wz_nX@YSg* z2sSDp8_j{C{c-v0@f4IDIlZetAn!GdmmF)rdvKv?&2)IMXDhypcG5gOMfu;@)?Rdn zM^oTdbnR8>@VFiL%1s)sieJMRaQiGy;WDQDy+oX<@4yp)|Iz5>K`s57kp5CksgEOt z?Dw{&lie<+lzU`lrm~uMXzTJPuu7K(wL%l~lkYVXeE$fvPQr{w;X|pIKvWX&CMmE~ zQrV)m`g!$lE~&i zK+mN!16oI?9fH&vX9=e5PQ;2d=y0mCdQ*PX;*a+d8k}G<;zF#M(^@DMkT|qk7NoMi z_KNL?Z&@+Q%OJXDk`g8qf-vx}%=QzudFj<_P^x5Yx$57(90IxN+skZ#>Y4dli+lYD z7JqK}W+vDFm}!1-rV1dctD#=hJKRa+H2QpC5M8Jhc(Q-o)oFxZr@g93%?bDx>O1DW z%Z;M-GI~timenV;g7Aa%8=uT!e`x9? zmytok`nyE;7IKrHZMi9&^UjQuxTo`AiGM?=Rsn4ToLDh1-MNJ9j9N_+;sr4ag#ERW zlbp$I3-4n858!LPRK8ICT42-R^kWjATt=0b1$=|-wbl8Id#Xw-xLG}O3k#RBTp5gL zNuTa%G|TQ120iBQHYOylg) z*WCktm6an{y;e_c?T^8NWxR0F`9&KZQP|a*$F>?iDAuWlox>qd#nMV1OZdF(ae3M8 zL8qRB5z9&`t-C!TqqE%#LNECi=NZk_E!CE_QT+pKP3amtp}+6n%xjo+L1|EJL>cQR930hGo+ zemGPpKdf)R8ny>V!`OJZAK6SQUD6!I&sJYhiPM9YN{v4$Do)XI{+}YE=!Vx^4f4t? ztH z%uih%7RQuo6GoeBy;y@bzsXSz)9{LMd-sSbN1?FZ4@zAek2{-cKWM+1`=Dwj=mVav zTJbJg4+6{4Q-4UE2~psmYs2L9!`~j^h$$G12bVzpN$us*ruwML*ruThyM=$!r=6e@ zyMgxW_X@xTNdx2NQil&wZKZaWh6R3qu7GQhQ9v&dUrS#kUt8Z1nVJr`mcS9}VqFBc8C z=1p8fITy_n0fCHut?3+`>}au0Jiy+1)h1cz=yby&1WBKkYU1^o#V_6WksxMyML(Qv z;Y~aiAYs^+99DqhS6OgPJJ+UMoqE;x!U@P85amSIt~@4vo^f=H=ImxxLUBi*2+G}4W9cf$;b zl(dL+cX!v&AUTplNe$gKFhlN9pZezS+53O&{eQz7jsphPz1DrN>pHLVbMV63Z8vvT zh_Mci|8b&Xv;Hj#u+Q6T4*Dl%6a$b#xMUAeFWa(Fir36hdLQdOo$6lC44Z#dc{ zycaxj6ok}O?3V8QM2BYlm-2GD>H@5N{SB?uQKt%JWpFOlDX49P1dcjn9XCQXW8U zkM+@R7i>J1Dz%!$oRs%~H{tA4Xo&}}-pnC!@L8${nwm$r6|AVvFo_=16;^WFFxQ1&E{pz@>A3pnWblo zwS9*WNmLH$*~7b_V(InBGrc&17GeVt0*8n$$BlwpEQV{nAM<)e(Or-Hl{Ncn0Ur0% z96{w3z{GpK0KfO29^>q_Ga%&jy0Q19w=cKz%)^B6wQ3{eJnq(Fbuq==-y^&?CaRO0 zU^Oz!YdJA&OXr1Steoe=p{lCS#ZHXZ#XVCa-~>CXqOmWKUCC@|YG*91vi6>F`iu_a zt-kr)Sp#mgJtd#gFG@Isy$&`k6cSTx@F{=LF{WMNV*`N-I~#Mjoxv<8-cx{p2MSv% z#@?YtCqX_9q}AnaXA9%yFMKkt)CCX*vAa}TdJ(J@kjNp7AA*LhsI^8r7Nf=S7Z??$3AN#_k;uS9 z8sJYH!#Is$9@#0S2t~|rupu#Lo?k|vN$|b?0}_e=K$o^f%g0InA16h#LDb6-5!0p5 z4<@w-N5Zqtd2CE~-p(_-ezxi)N;9I;UdZOisfuMRWx-4e$7i@dp2T9oNmwcuz*Q}< zHB)L`BB43efZ8zmJ$}r7+Fv?eJNR4o$y$08b3HCyjfx$EXp&Of@X zBR9w*7=!R31!sai3)jI;W6qM(?)?LzX+GAlHMU3g7wVj%v+W&>rVj{MQY0OBmlw0s z*|Vq?{l;BYG+|KTw>H#rFq@?-Ps00b)E|P_RQDHx?XYtr?3-X(r`3vfIM{62_O}*a z`;J-G@x6vUkMrUz?wE4$LC~y~r#cbMYjKov%wj4L+V8pA(lo6+3XCec{Fu{i$G1*G zN+xaRNGK5{?Ri(a4X{h?wZC)jjtD%xzcm3DF02NtleOr!k8A6provDI5K(}4TsF@~ zFFb4ngSXslbh$q?_(3Ke)3;h1!e#>FQ#v?Id%kV3-Bs!5Ms`MLW0c^b?n;6Mq1!D5 z6%%c+m_AmEu~al{g3YindZ(_`Z;_PRLvqilPx;p%wbneC`O7uL-mY-o>K#pHisk;Q z{@ajUb5(1!+^A9Wd)Vog2lTtI{p1ynoZIih7bV-?HkY%l!u`rch{wKfGzg^nNK5u` z7m7N}bTG!TytQsovR(C=q1E>m1Vi)`NBHiXrtOEju=mYlcFiS5b~}-bJ|ujDhab*l zwhFZ1C9o-gq^(0<^si3$yjLN@d>YZ;5 z&}LCs@fMQVqHZ1>pWl6DeC7adX1z7k>DUD2kZdrce+p>9ypHP{z{Gq0%36pfGsX^@ zq@RPMc zn7Yui8~?%9lc|bV&q@r{ys_jgQL_+KB_J5Nf4@5~NBCYM&KObj0y!t){AcMZ`Mhe7 z;7o(}Z~|cB1ken}u*SxjlhLXEVE#bgPhNG8-X^FO{5TBXA9W_xHD^YhNwX$kR{U+y4f>3TmDH<& zexZbj>iL^)99q$d6UUmC?{y&@qm)8A3)qy1D?B$uOuVRngwh_Iw$g1w8J$aKVfLA2 z>FBR+QUCGP)fFV+gkxlps@bnj!F4tu0H1=wx<qe1uJX=ls06D54Fz~dpq8<@JsAj%eN^r$g*@c_59(Rwy31FZxaa!46% z7kA_EEBKVMoLU3R3y0u@IrB}CG%6|GTUtVhxJ(LpIZbmMXs6&8G1tLok3IReB34h! z1dBhkvAo~kPhV$#R=%e|xRPAr{y&g@Q$I<+i3dn4kSQ4tp&*gP*2!?rsol&ZjL$sU zWAvdEGIsB%7qb#@w29T_lNMJm`e`VoGlURoX5~WF-$dV5Wwd8%s)@-oC>W}DU-s3C zmT#uq)Z#;UW}2JOH+q2@qms@)|k zF1>-=AMJ!$y3KHJNhu%qAUf=O$wLE+Do=Ohw)6+}`NT^vSmredXQsouaa~kxwJ(V& zj0@O008uWjanP7UdD-9aX6OrwL1WV~kRe~fP?7mCw(Eegys*HcVRG-Bw_l|yDB-mw zud*}I*RSTSdd-okY@WQsG}`o%bM2mF`1BiQEz_g=d41HE+E_PXV^>wx{1@hZQ;@T2 zzONN!NfNTUf`cN@SwSZh~RSOn%6^L2A80=A);-TUU@&^D2 zAmSqpo^TqAa$S9Rwm@gXr2h3HemK-_Xsxx}L0rPk?&_1qC7U^O?=yKK;jvI|lg{p! z=8Ek27;2yFo(7U~Xp0T!gNhI4OA-%COTF%GA%Qie-qtHQ|A;3dmVM=UhFKiNJi+Tt z-4D5*y>HJv?`0?aB-VXw*-vDBS$D8k)Vnq}6IA1o&C$%@3&1B7U5spw-N_fL$`g{; zHo9=9n&B;Jt{E~#QHCt3RV%3zz~PA&?O{jncLk6u607gl@A`@v)$j9>pI;bST`aJ% z77Uy@BnA{lpE!(8%UkTqbL6O@iVYy%>1ySCXOq>Z{A_|SnBNCMuc;xbB{fnCcN8|`Miblt+BFZN~9_qN3_ELYN0(x&HzbW*E{h z-juV)GM&0~z0cJhoq@J@v=^bIWouLt#QHHmLZF}FQv9myc}hYIVFD?cdCZ&`e%BDC zXESax#0q>Zghl6d4sA*tsg)3;u2TGikXw6&Ge+CgRyr?BrZE5M(z(nT{6krv-;g=48WuN^d}CG;^5s&g_h!I#dK>;0Rw z4ela3C}s)4`xQlNfwH)fY=Sm7*?<+u{p)Z=`%LVyGsqG+TBNSbl@^0l+B1&H`p)R9 z9f2MB5hnlbv#k)T-ko#4%XxB-_T4j_2*^vOt#^5}`eE2(!KFS+p{{AJTvU4O-xRg_ zvMVUTq5DG7#YbX?$Z2fEMU>ApSz@yX&K1Wzjyx&tMsdq`Mg@#iPM#^K*e-4cI`#unjKHbc`F`k9 zEoXWu^KJnqqm0vL#Q+9bWQISAkatEh4EZr%-<(h0`P$1wRk8SvGdj8ZaQh%kyR`zT zbeC*106eE&idzAzEN%Fb$TSleIblbz-Q4r?bf{5-H6wJVw0UUgC@g0l9%-I-@=r*d z-81b!s5GiSsI+Ru4HEa6Qp6&;e6Y)vKT&N;2lrSZUURY@$opxap1VIp3rkdgKO=R& z(ztC?&uLtA^=$s&%tvbUMpb*-H}}~F1HuO)vdE;9=mhU`o|$|b`d=(+;E>}(*Ez2E zrdmF{F;qL;?Jz$YyM_0gs_cfP`-z@J9_RX9jVr8_(B6gWHO^1gI=&CM9wu|)+cw8W zD~T_xRi0VNBM%as0tJE)%A#7-t>}^aYgsIu-ZvDN26$y_U?`#!YJ^x#^#VfK-o}l>rS_ddoX|4 zML&{-{RViN$Yt8Bv`{66&D397T^k6|6A_3HX}1@BV*UJ?S<#kvht~TJ{C8mb?_url%uv;)nzK#g3XUc{%GT9RBp7{P+?T;cboc$z)Z(Y7tq~ z{1Vl}!S=6&xZZW)$vvO7I&8ixR(@qHY@0^9ctGr81kVI^iCyEzLP>93Z1R-zMTY1+ zTwu3&CUmjO&wqqlVb~-18Nzx0eq&r$B(EIhV8%#aZ*W5Uxcfs+%X?m?wFg%dmlWo* z{2IRVb&nH9PVM$(!ZzMnSH+Z~Ha{s;wXp_BA^yD`Rw!<_2y3ZaeNTEVx01HNd7$fc z{LM)lK)!H9an9p)#p)A9RL(se&xRTQC0@>K<|JJ}J2R%?;m}jlZ!b+)q-hzls0MR3 zHB(XvFg_TpKAa&KL71Y32rS*k73QhXIXpSyM_dMmusCf(E49R zq=rpp2-Rt8yxQHUn(&PM1*lnG?y?6rRJs1K21DS5$_zuI1M$I-E(r3JZ-yr+Nw{5& z>R@$3i90-5I7{u^iANw^DVMFz3b5DQ?CWNNACMIwT@#a?W>ETtkQ&Fl@K8qKFSne(v15awo1YwQ2bU#BD7;h2xA0&doHi9lT z66;^+5rp2oe74p~N6(_Oe6=(xPhP6{ly}~CWy>zr((JHytmi(L(#_5KTzJ$yZx`;j z+)jDD%b+?&WVw{TVa2=f`&T!}DaM@p* zTwRt@ET0d)YwLVnl|A|IIkJfK!&@8q7IQhP3e!eu=`RzNIZ~C_4%c~m3+RPn`A^?u z2z!*Q1#~M85kt&XZRqLEm%r3RNHJoEha1;X`-fQy>gxen4?HI-O=jq~dt{!s!wR%F zrdA9pMr9VWQqrr2UxD;hQPU8m$=q5Kff`?yy0aH{w*o~(2m@Vv5;yf2;*lKCS??g* z8@^!{LFw_UKH7^vo+{mhmp&e{^xW1Bu9Ck{v63oERB;cr6WOp&K=M?Q>>SLp9t#8= zw=;4D{0@GN+>|3B+ER|dO0M_Y@ACzI{xr{$HV&MM+MkM~BWaYRSJoT%0LyJ->Ru4z ziK|syrc!-Ua2dDL*sJQ6Hx_ytfW&{??Cg^J!C+-lPE88-EZhqmThvEL{=p!QDBM3j zlNU&{`f%rS*PMu&$pzhRW<1Y$taRY1*4Nk+bFA8wrCAv{la;r+mZaN?z!VH?Jqn{U(y%d5iO)Yt`ZvFDB^PLKTf4zdOSg zAJ4YmcRE+R33`|Cmh3gJ=R-h(Nso)jU-_(~GHY?`q8ZHaM}DlmFl-FDnAco?!dTzM zwdEJG#AI9qyGojK&Gl|(^qSvhGeD!0nI_J}T)dbk8SjDPbC~l%nSYaI zDpI|BD&g)faC@dmlQVU4cvpEPX;m@TH6~R0-HU~i^SzZ_KDB?KoJckQBiCaIAb~x^ zmp|I#KRBB%IhP@NLwc&Tkq?Q}~1sbC+d zc77m9lE$uhfVwfQgsEi*r7s*Pt@C!fi4mc<8FyLo7mlf^AROMUPKkw_@U^%Y^V-X) zS0d(8mpF*m?wdY;A{OBrg^zoJ>lT{OH}=Uwj+|H4e~!}AlW+TrNkYR;M2U&E;y<8e zwtf#>M`dS?W@84r0NxS#lqaUmBX)vazHxzGT@V|*^vBfx(m2I{B_23S%2g`CZS6M; zqLyMWnHTpTL1xL`IPa*d3SQhORDEG|rVc1@l#kLgOmUwcNzVqGK};>p3L)KK;QFx& zaEIUPQ#mkyi%-i;Vc9dhIFS}}zv&Tb^XnVBHkIT#($ z(-1LqpJdf8JVV2|l*SmPl?YISvnr|f+cyVpu_O8TufInoeRut=*m!r>Q@1z-#i4ly zvN~n%F2tvbDm4M3GbXCkqr4|(OMjgh7)QmcS(T$45m+Z#v2g+y3~v97bv2g9)L`VC zeBwwgP4*BW{fnk=ZxlaQ%J5<=)5#Nx9}cD9d;GI!-$=BDf^|@3%t|{-s=<$zN!x{%6z*J3^raKp;!vd6jN?7iZiK~2 z=_T#8c4NfSd+uFNJ3$|laTOsW>IlIGzlB3IAmYsO;v9CHv_tIx(afUGw+Ma1BG5jc;vIr-!=}zY z)mThfJK~bWY$rI6oMROqr!8Me=5q5io6!yyv{?9{?zwTm@CR<8r7O}IlE75aGSvW1Tv7oH;pKp&27ANls^tX0n`H9^{%|3fUIdycbb% zB@2W6#Km?#2YcnIB@CJ7J+_C@X5H^C?x(R+(NmkvtaqMLXQn#;$h0w@Jd>_;yPSsk za6{!zB}YS!qZU3aqwuTA1jRd_w}0jg`2=$c!VmwDOz6&4RmEE=`XS7)pw?Vq|# ze;tA^I}rBBPb@wVq&&jIU-#|7-+en2H0DSFuU{FVzAELIvz+{*ZBp;8hC(|H`-!JM z4dm`M_wuEZr}LXpM*v+}s7KlWq`TPcy31zz> zU`l#-z+T+>Pz6&xJ#DFbbgbWnH4?q2ccH)LwjyP_O&&#<#SCKQ@NV45Pc}J@s}pyC zumSe_XFBZIo5!M3Ai1$NeWRd3Vgn$Bcn=Bb&SN;HENqyjtj6b3>Kk+s zUV&F{4MI6=77#jJ?t->rY%9kqLy{dP@kK3QJ99&a4Q;By>EU#LK~gNT)j96le4d_mY^dF(yVuHLLg|eE83qhLv@xEHSnJyV^M6L!f7M!z9o-bo3R4c8ZK}^%TjQ~&Vn~zLU1J+ z!ulA+Miq`&KK9&GeF%wF((#JSD%P{haK9V*z-u!;G@)036!PqxMJC+I)>GD=n&0CE zu6xka6^)sh1~ajAOhDe3aH(@7Hi4uS;wgd7K=CYpci=(4t-i41 zEh<~d7K6MK6@HJo>GXCH?f>m>Sfrz;6cHw7m57- z>2M2>Zl%9(&$#1X*tGBm_A{p<(@VJlR->p{{cD4MyXsTa6f)7w=wvAYkwPdJcy6cC0MQD5qHpU?zF4yzFho)9q{_!=U}&VympCeq*Z|1hsk< zVqZJtP{1-J7ToP7f+amhO^y5A|(c z1o>^Duw1o?B*-h{5p>S1Nj7*2m7Qs*k6#3rV|#n;CD~?Q{>$@6VoEA^IJjY(BVK}x zqv`43$r#k?T-e^sm?$*hje)45^m_F5oBa%A6^dW;j!Z5P>DF9k4e6nHr|D0a?o`@q zhJ_+?C)O&2g9i`mSsW=^kg01^vn;-oCzqd8Yfu8bQvfH9nxl7Gxpz5_^&;tJ2z}9E zYfnEVmylioz$6cp2k^qhMIaiK9P;xr!J5gaBMR-$2o+$5&Wuj_KjBZFu}!P=he}#r@XYGD0gv!76T2h{Evpbnx~yA4FlR1zvv`sJt7OR zORy`~N*Y!9hg{W`PH8~3H9)ZbeVa%19nEt=pFTTtF56;cQsof7hReJmKQhD=+2JU( za;4E$%vOA!XrH|E?*w|*)+mAA%ijfh8H_C~Up_gphr}$hh@NlQ;Y65J3C4E9y1pB` zM!Z;a*Wx+J<}V3B?fT*gklYgrTHI{Koy9OM$%h#7nrG|*F4LpK&mKv`ySj@CA!>~b zy71gLLq$ENK%*^M-Y;zrL=h^VQ#?)zUhBvf(OGIiiI}Xu>zOk}Ty4@at*yZK^iUk6 zqNsj3uC1Q!#f1)CFJw`%p2*8f5aGe4q-6R3(4#_^_m>&e4g4 zZ(hA4r4W43%U))1q3%xG4S%JgGq1P@2X?z6&<^Md+9*}Iv4j<4YZNzb!ryX|cD;A;!1!I@3A`5N*lj-iX z)yS+M6RgqJn}4d2&4e@l1&aQ1EeCU*O=#^1`$)qDb?#I)tvXZQwL-5?2R9$l#z z(aSH?Ch}&uJ|g={4D^NUWI-c;vD8hT-FasH=qFla%Ul>XA7_R0&uo#woqxj?dFpnJ z6X_B8qcnBb?+XxJ_sI%AdA>-I77%zf>CXDZf;F=3JCHg$=5Qw@NCaB#WNDL0)8lDjZo&JE7l#0j1nbTFemEvkm7pDX%LrC z05(f+s37IP*xxaGbFZWuAkLz!G__~BLZS&XwVN1K*OnRM3#kZ|x58 zo>@7)(`vT;N3mPXh_H&-Gi;0Vr8v%elonv_8s!3{fk*GezjxX-5m)~+gL6+6z%?yZ z4Cn?P9CtMuqQr{wAQX6%hLX%}%}Z%w$ToQ9zG2gXGFlFq%71Nd)L8Hgr?8w~&J^hK zdr==U)tw)&Ls#O3aCZ8(gVdIgwe}R8;iXWUe45GuHAQpg*c9s-#Q;=#tZcx1vsE-V zdUg7@i#VK&>u`&&wJSJB%$$F^fCK4wr}ZrYQ4T z7<+7BSjYIl*;O_?OuYH~_}+%g@R2>LhL`>y(4@_}_;ryhxJxSC&o#xnetCJB%CTda zwZH+VzOw9k$$;-_Up02;`u&Ytnp{HKr2TZIY;(sXV~Ij)&j;c~l+(`cE9CZAA|Bb) zTXTB(>s9ERhMP|$P8^LZ8WZ(}^04^vaye-14Sm|@ne}jdiW@(JSbjq}0eY-ud4WB~ z_NC7FDCih9@oI4)E^F8|qCV^BTCP9O!)W?}z79Qr#bsFmz0e$+oF)86^>fSk10nFMtjZT81y(aHj~X)K%Ges3F=1yWDeo}{g+?yUt0iaL0`cCnLo zL~Sr+)Wl4=y09TtEjmd5XC_XCVVU+CmydhYZ12vSK%W8&+HOd)te+{G3XV%%;m+fK zP?nt_Y++5E`~ ztq?5APuhAV{U4lVZU5pd3xB<3|KF@|dShwLf4~4Q_R`;LaTM;fhD;(O5nL(n&Lw)j z?rpWI7GuF+CM+yp!vn;t!G3baU5E5;x`@wVdMgD2?|F9M3S?+5K2dQjZFnGO3YWR7 z+lYGoycd}{)zqhI&8UOixxTdf@vi-s9k-~6kv@7z^GCz6C#!)vtD+ZE;a-Dr;rTv8 zbvOs@>Ql3UT@}DC_KSYJO7}JM(N~QpcEyE&=9NlRhEwHhfl{h+w%nJ@H!PTEU~C?o zSfii8=v&;)*7*6)Rxdw&@KoS748W+-1^9}G(GIl4 zk1UC6U!v`wj=fdUEIzsHzHp>|BDZrb9D125FxTco#%iMY%&pagc zTn=fLSKk4kILIexs;+|@_HaMipQnX&hYHp6GW z*=1yA4%P%A9#K5G(w_MyAuYy9%iR7AlP!+V^*KwXKz{FJ*EI+cJ(c$9yK)kBy>%mXrk zdwQOCa^II*ZxH_ih6cM!(5k;_|Dh+F-hQpMbtC@2=*hf6Xw?=M|Hk0;3y}#VIS=!n zTGl?H1@_4-naCL*c9!2J9&%iOId2bfB!=m@`h+H-U(n>RnEuds)hT{%@;~K#t~fF$ zXeFqiB|jH0z3!VCSN)Fq$W)xIohKEor;mhUu?;DrcG%V6gJMio=&|E@Fd?B31b2Ea z7oCSmo^n6zgGb4MQV`o}KOLZ2XLq>+?^KgmyA1xSc)I0ZWoaYP9`kx|sZk{P4-too zpBk_&aVxwQr=~{PjuR4}Cw7Es$^RNCSFMRBv#}3(w`GH-WTVseVRCVDjmi*)ksA5d z`I(pIsJ=C5gD16mL^n0De<<&~du!OqL^i~v&XC#j;U*^4mW&J3M-xV-EOp<$vVFfm z(qty*Q7|7tJVmD3Rovc-5bI{$aEk3vgT0=0y3uNVgz*Tjk-R_$YZvTP!A4#{Sp2{*$pMnq#T7%&b>U5!NQB?>t^>DUCu zs$~YY0g9QZkZOq(DhkTY$@b6b7EiD(n-eRq?9nY1Sz}AwyN28fp1(pN6UWY9<$=lD z!nSL%*93zbTq%auXVo%zM9|dmPb)18y>ZyOo;GMxs)G}ce(@6@ueB?bzuX6X(JYMhh>QlVts_XtFo_OoEa|k$qz^+kbhXBQtMgV3x(d zcPkr*OYt=0H|C}xm$gbk=aaO4u~Q^k%Bb7!HJm0?=7P_A^3(SVcFvx`+f+zOBEbt;IW9FhJ8;T==QjT)8>-*=h*5D zCx+)SmqJRF()PCf`Ws&~7p`}te|!#mL&YDtsyQHPKh?g+;sebgp0+9PO@}1-PtR^L z*nEn~LI4(eQvs%SZ*L_EAwG(%ZPl;l6(WD8Q_{|s?5xEIyIjJ>YL;uNF8Nz`ip}2D z1Y+sCUIz4-){XKCTB4|{!BTgw?0k5fU?o*=1uYfL5Ot92bwvt(^~lQaVXef((8udmD`Xc;gmUPd!tnsusDW292T{!Xbx35+@Y+!ejL^Nu5*W;ff zoJMtr#^R`QSAh3+*{IgyRrKQmKJc=y+33UdT;7^FOI4xTP3^uHi;!E(vAzSSCC<*W}M9#VJHvb z$QOX7St&p@wsc8m{g7(^`&4++>>)K7u8Go=`3nf;{MS1#5{3NB(rwna!r+^a`2^1k zr0b_DK*Z)aYHf3#*p7QmiO*>ICa*XVBQ!8^C#$EIs+IzX; zUb*r)r9{;ofqR#mL8#~R!@9U@xmj%^Tc}pAy}c&O`R=>mmlV zTpoDWM5dW@mA|tRpg*XuiV!mw&~uc*`+-}N?l0;&wkMc4LYgW_i;`_~Pq9rmGUs98_OX(KdDN|?Dfo@}^^?zT z%4Mn^cI*vxu^S#WhB+q3;XA34z( z#eoRJ%Ag0~VL-;>a3_Am;|&rOqjF#^`=R<{O#EPr+eCs_e3rO*2Ybr6_iXU@RwVFb zd*JOi1?S6CVQT<ZnUa+eSd@ zhpSnJVl}i6by`EqJ9OW%NRbmTIboRqOX#Gg6bTMRIn;NCF*=k(32M@}|o!CG`| zCfJz__IT!9q^!mDe#gy?ll8Rm7d4`D3Rd5*f3`eP&n;qZ)Lq^vXODl`b*92V@FVfE zYi()m;yihWTuzR9u`t5Taf_b3M~-@=5HubAf8BXR#aBz?)b|J&(z2XfTa%CcEH%Na^{M-#n~S2^P32poBihT}h#Re(%Nh?Iaz=Ex%Jm+K`S z5{x(saIwcYbi2t_I!H@O(&Scw5DNb_?y?WAZ^VBY~!IB$(aqts=t(r)-FR!OghLj+JT+wUB$6fg_>DHRgK zv)lT5OVHDlF}KkRsEajtpu{H0vRnJk3`hKReNK5YVV}~(!CFfRZF$*Reg+d0I(96; zamtFlEW13?1TF;IE6QY9;OV1mS=+^gt)Bv+hIhgelS-dchMq~MdO(p67BIv#G`_^&8P$#xbE&7e z>EmS`HZ-v#Vp*rdp3Em{0)H$lfwL=ZYImtUv?N-?7&+Wgs@c%ai<*_j`*0{6%qvbX ze!=)6N0r2{afX9qm69aZ(rp#uy$J9&pSil2Q3(ZG%xe5_`5pFE&Kpb&oFh`_!JUlW z7icYKr86H7cRb4M^F#gl_m@lEuOhsS;CtR}r+Z1_WnH!vC^P->K+R?9uPq<7Wj7dJ zPKKr@Bz5SO(Kh=r!d~B-qo%IoAX56q@+ntMPo+98jI~CQ3Ne0ND|k+3E~v-n|9Y|Z za6uSv1lrt_asl?*SN-q3|KpBV<}T0uo4sx~1Qnrq_F~0#P+*DJkCH(N!rmdRN9}s2 z`y-EZVBg8YQ2hY<-ofX!h(02jjXyH_2wPrSe*5;|b^~|8H7?CU!JRkNKls1;j2=6_ zW7QvHP@Y`;N$D9)jU+AAV-r4!vAG`IjE^wno3zuxCA!7>;;4egUz`8vW3P%0psgFr zFOqsB*p^7qNy40pX+i>+SH=_i*Lhpal$c_154>%`y@|FZ`brM*D;KtF?BQ|gQ&w*9 zJGzK-TBF9n1u(`MzL{z8uf;Tybwk&7ekx zIu@jb8>OTyelTTdH64Fbq=$9d%KX#9Bt3S1m?>=fdzQ(X9j`m>o~^71m7p(OT_49@_q4tj{EWgp}h?^j3eC% zmWASnvgv102n&i#lfM{RE;yXc^P-snmJ2umr`t7M4R2Hq2)}gei~@% zX6_iXk@&qaeFKQT`FDD4WHJ~|!+)9vcJw-I$sg0xXW#qVqJg-+oCQ*r=r_=Z4^DKHYrqtHD$YEBe~7?Aky^AUd6IOm1R@oNaGq~Qz8Xx7poao0;$*&gI{4w{0hIf zq4Ge#x1qd>cSfZS{SE$DGduq_Pzb{v8%OyIGy{vc?|vm@gYPzOn|yv&(TLSkDU`Pt zkTZL&MzTY{g8IlRy} zNu*{gkfLY#=ja?>nK|VK<}Y^Z8z;;5$eoMVMy1*~UWS&_1hT5j8ZLaam$~bbv82dx z;&p|Xgeqz0=X5H_Mt7*23r=KO&nZF*R zh*-12tCS{yV$L2c@Xsnv99*^CN{}h7Eq6Yz%UH7;oO-}+OCTMnV{jTElC)P7%Ar7h zH#lDU_QtkklbJmxMwh_HeU=VPa&lqyWjcPcU^;7;o_T3`Ysu&h+aFB6Y|*J0F2z7j zf(b(cS{w)8EkJf~k#FGF5*qhh=MpUrabb8HnWH}q`Bjs-E%5w>z#NcvF z_HbPi@`F6Cu0lT*rHz?M*l4uB^b3B#Vg>!+y(dLKVr68>&aMk?pzR5iujzGu85ZB0nbBN>1A^9pB`o>#mG}xTj)N zdC2woK4ax;>YcShfmAuwVxmk#lb4p_FCM(cee~RVZZe&a#-xM$Oz0WkR?K$&y>1qywY6AX(S|^ z=_Aa96XnZwmMLs@zw~vD^1J zNAToD;V~JqBf-r=Q`5}CDhr9;8`VsJGjL)j`tQoBi#m^gFOMvtf4ky;fgS(`=+Jd? zsXOZP?_F+0G}SX=bj^@qjBat=HnrtXWSu%WlfHn{kYSG`lw)aiu>nd<;2}bW=3m1m zn%7Aj&qs@Nn-hq--H{@*3Ej=! zp%A70Wed;bAP68Vz;d83ie^^)t-RRj{XG6HU)hi8%Ox`Tzf>}VepWK^PXG8VsGCDn z=tt4>kKLNuZ{74?imLxS$@qU{;s2M~=U;y5uMbh;%0q6}d6#+TRBp?G+m zT{8;a5 zzc+}#KY;wWrYrQS3uol>qw72T$fFf(W~p~a7*zJ_KmjPZGl$TR1#s#&Q++wB__d@3 z{557arY04m?$-*S{&%B@lr(;2A1>d-2q1~=aV4A1W8q18(OO}up8-FgH3sj^Jt}~UdlC5=(~;BE@WOY=|9oW;?Tb1xyLfld}I?7 zN@sX(Sz37n%-nDV8Ik5S_ZxHBS_w)vr$-((VSVA$#)L8--c0f2UM-mF2?$1OI%$L_ z*%T{#h|ircdU5weWh8qqb``I^#D#B)pi)5Rsd^XHbQg~%ZoMULTBp6R(kMG4`a_C8 z!hWOpC7lyxZAqM1r-G7TE+~6--fJjfQ+39_7n64W-L3CC-`wSvhR&Zyoz`UgXx@P? zvT4+?hhuht>}>L%0AMd?1d$)2R=B!btKg3=#v2HBJ`Y~e@HiEus19p_p*hthpqW4l zz(w@;E~poH3zBd)n_85I5w1Uryrwh^(7CtQbj;j4eo5i^`^cdS@XJ>PM?Y_Y${43D zX0zuY{`vR)v0GwW@IfhF+vjs00LI+V*8aB<8OLz&ayQ%l8Hbh*QPH^G;*Wduy51az0L3d;vQ4oMqW%%_rfwjTm<{aD775 z__Wo0Ov(%K-Q8^!71R(8dYWqe_^}I12#s?Re}Y(cw4w0-kMoI?E|30YJ~4ml|M`4k zIn(veQ&Ch(&h)*${ysWe>~{{oX(3*(1>_*Q4<^0myY~;>TtMX!w>gC8@YgEMGEzbX ztNO`&DQF0V`jYjH$pwl8i4%3JQzj0ZIUC*w4MaFldtN^nnn=(k|5gQEyMkZWo>K0= z)n04AVr$$KZlBhlTLmF|uJ71M2YJ>&^G7MHdr2&9 zxD5rpyvGFRxH*c`Gj%=5j5sTgbO+Wq@DCeN=YnVF-_S^u6E@BV8s?qpV#{BdUz;$T zIPNY4Zh`0#1{kSSlAGrLi?p{4i)-ojeOE#Z2#{bMJOsA}f(1{o;O_1Yjk^R05G1&3 zaHnyX;L^Bz2X}{Vnx^5h*1O-c_r7SVT*2M2=^Jb3T^tEj2bwd$|Q%yE)@LR@r@fS_IKPNByd$&5o?& zoT_8Udp1z@%BK|j{3ME57N#GHukhtbq#Tv zyb!A;ERv8w@a6p6L;ZNSg2xTwBt^Hq0Bq>+50rp@GN#3eZY3rwK){>|7~Mb5S?%MO zuAI-|$7X07X{#oq!;Ga73BP9aWv9SfpBK74R)(Xi4pr3@IRCt-R@g|A*oZ+gfIB+U zO`LqkNf8PD_a>|b`eDr3906;@I!(UxV;a$wPEme-8D5=YI|_|+nWX%jSbe4ikQ%x1 zqCyo>Z*ky$O~&+iG8<=cd2L+`?(VA(XcCRia$m7Lm%I=5Sy)(t?$SEctir;S9A#(o zt4$6ENadwz_Wy7MTCQn`%~EE9D0PH5@w+rGX{^7w3sq zlApZ2jHa`C!qB8zV6pOUR+nng0Q*<=ik-A?{vP7b@7b`XsjO-Ek8Jtq$E7LDj17&a ztnhxXG)c|mlWqsP)ygAUej&=xtv>Y+3>Ba-we>V~(Y5dcT<-s}BDl~hfRlXZiq{Yv zMk!$(A&}nG^J#Cb^6Pv)&iS!3V}_Pz?a8AL&D3X7N|Kpn+-|(##adms8K~lXh8CO0 zC?J~rW;>=QXA|u$THNL99XbzXw~Zf9nj@oXe3pwDM|O1O(Op?u`koBM_b&6W@=yO|V77hOBiRx3ZuRd`_9Di#4;v721$ftB zG1Hb6eRS93RA*>i$mV!MTx>P_hm0rr*PLur23XW&AC|*uu!<0BA>q0ikumkMc`*Rg z?<<>a`LU!}^T}hU2FU03Of~B}OIR1`LMlr49b?bh56Gt?S^1=wZ;#^!CY|fqM8xcj zKMM{SSSWEAcg@~NfT`#1pBj+0NJ6pSWV!`V_ahV&ogyPHhH@0%(1d>8A&x9gbVzOf z)?4y3pHwza1i-bTm#ko|%-EdBqPyR$iwu9j6Yy)r1h^>W30hb{`%Wozg4A04wu)K& zoI$%?FHejwD-N4IXO(w?5tM3M%9_h`QUp&9$v1~ueXXoh`}$en)t)Gfol&Q}lCuv5 zqqk-=9Zqc-{3q?{{5%yF*SYV^JzO3S+VC7CB*Pu!NOBzTg%8i$ZS9y{zDl?Ks`urI zS_>V1+^(|UEi(HeGhO5DanUO7hm>HOxhjctb=I!3t3OpgHYb%z4Oo`E%Kz}gp%8b| zhgx$ffm*uK;$fzIbtv7So#|BD=p!fEZmZpy(e$3*`tS%nU6AuawnR5!FAqoMZo-nj z(ZbAa)`B(OazCYK=j(~d^io0Zs6!$9!w0jVEL-h4?7C%5f%7eR({k@5++nwO?VyJj)E5^*$Kr*f za}ON&5(PbVG{?eGY03$M;;H^hCpZZ(ouo01_y4TAk{>^o4l$>Dh&ko9vTx#ov}V1~ z+|l9R{>o@6r3atzmnDr4yWo&PPc7&`=ab|p8HxoBh#@|*+Q8jmK{wO)O z!bMyj_q3Jn7kR2c+I>-pR0?9cBXc|shU8;bVMwh%GDWvY?4;SBsKdna<*L~LLSr0J z;S;BAvv4{{KMTUaILNus>+gZ;K&~3_-8!y`K_`;AW(SN7wnRRw9yzK&LHgNrTyJi1 zAR}O>*CPhDhIt2r`>*Gb0tnX{gUx8a{$?aIU=Xc|UhgV0(-J_w7S`DgU7}QnJA-b) zt^ugyRqED{%21|6f*F2~Ka`C&LwzLC4H9Qp-SQ_YM}mTm#lRwEp8Tr1jKo{6f)r{1 zRmuu0{a#}Nn>s4?^FNch>pDby&YU`#eh!!^t1_%K^-|nt&mr^?oP`>jn1PzHiqatE~-OQR6653?F7Ray681R2~_Zm z&#gxPm`Y5?^Z{9j8FAXXAfS%XL)Ol5tLG${#PtAi`qBBOcKUfROq+mFN)#>$e(y#Q zpQarkI`YCk47QJONE)8Z{=PhqYp*fT7xAJYs!9yWMh7L|Y8h?*5U%JuF?3-#_`fPjIdT(;bXNhoKlG6d$B@4q>bPXHBIM|wz~&q+FooX%Gc&@wP7PlFQtfPi2M+z-^W8o?%CDD1XB#`+|v9`o%GRE5yU;&(+RCP~H&@+fz-2N(7oa{r# z!Cvjp@xAa^fd2ssdE972Jx;}*>)C0wWP4AihswBMLJJoYt8cAKhfBK26x^Fxug-e8 z>4JKQ7Q4T?<{5NX4jW z60|e*Y|OObbl7z()BNp4wf(3rfZVF5qa2Z_^p@wKeZ^9}!p#0S*bGpr|N3r-K+ zU2mVnF=2K*FpmpAwCQ_=AWe>30I!W^F07iAUE=hz{eF{xPBvx`x-}}W^R-Hsmb-5B zS&BbtX<+tp-!Ogl=)XMUoHqXWbqMKwv+l=D3iV7AbmVy~08cH}Jm~<#U!)pk;XfQy zkfUuccD$meWRX4+X9&OGGH0dT(|DVSf8ra26W^NRJkII~3l=_arC99`)FO8QkZo#q z1pL5>sKep7;0h@?UKHc3*p(V>_`qy^`y=94+@&tLr}_JuQaTh2DAIpXfhETQaBuIW z(D%qB8^GAZP8qO+qMi%4bETjcrbWnFXt5Wp-j8&ekzwYFrtnQZPiJyXNK&0(qZZMa z*8bYAKjUd(A~?FRxi_&pOhfxg;j@aKgy#d)8jc1mBGg#Xy9PT4E@6~joy^CgC_53+ z7B^SKVzfUi4b(u01ICiMvadR>1V2Bm3mE+(hecsuxT?Lj-7L+iZ1EcW`d8v`AE)n? zVfX__V8O9xRr_E^WkIl;ZS|o6I|(blVkuNVa$Zz)x)06vVe6ClE!@U30bJYPze3RM zD3ZFCuuvQ3G+Czz25lYFm}W{-LNksFZrKX_Ny1KENIN?Z4h`iLx9FYBkuW4i*9-4% z?R7peP17SP0~c0=hf4qiR>~bOF7o#;JcqfmJQf%JtnbUZY88x)Yif^4KYrq66C#4y z*FgUa1Rp>U zlUbMU?q-pfMcieJqL-3GD|zrsP1g64Vr}oPbiVit`bJ`SUMX(43$1AA^g*^-fU>kQ z_GMF)tiudmP6;9^oGXAwaPUR6i@x{ysjaZXcVCK*k!&dI`?Dyd){ZyJizeKt9CWcy z>drf=K(A?xOGRr|!oEiKx z+%w~I&5loLtK`j%Cdag^Qd5l)W*Xvp%2eC)4p(b7{=Ng?JXau>r;}Az$_#6bI1s}fN z-wdo>ttcdMC;^v#p^``28vEFC4SKxA;r2#GH|{~~rmWIi9|><2M?4lTKP`669=Ovl zp_Y@ONE5&7nP~y!k+T4N#aW=1Mjf^R)qNaqD<`5UYfgfUwR`O4ct)%SI&u1ADD!tK zH?#bX)kIa*5(l`;0AxAfV9ICsVCC%M;`elW#PG$^J-;p!F9{MQsoB#0@A2%%zSKZc z13sL1Bs><)T7)mDV`C#X;_Wvod@Fq3ETKM|g}ktQ*PhLt+XVnwpyo}v>bR__2#=~7 z=Q9s-B3^iY&P+}&O1@w?HTW@YY>Wso^e`-=HV7_8h5c|Sx;T=$xb30k;xx^A-#(N) z_$s?P+W^8BCY$n8&g!F-xJI+Gp=xJRKfX87m?R!R&VtEK57*7DKTo=KIlb8C+~_>b zVP?_btPc_3>(2redht#WN-5dkq*v#?&_L4E9|*IhwF4;g7GY3+R1`1Dv3)77eqFe- z;*@q(dAwRELOi8kCv3Gx{$-rXbGr9$;FY$`<<1zoG3DaAi;E~za{&AM6ua< z$!0^VR&!dB?0K%xe(QS<-GA*9c%B}r^q-JQc~oA z6R!Np$-;2ZKCMdUx3VT5#ydj6UIlXP?)>}1I*Pcn?ywgJSBFMYoU1PGGM~IXr{L`$ zSmN^NvaoUW^h}71XCt-)c_}?g&4A^sn92 z6}6uTJ-NroJ){=ct)Q7~j}*<$o;o%w>X|w=d6T!r|7^(WS$QSa_VSNx!~nLeIICX16#7vAh|ABBwb9F<$WwEh1Fm? zG>HlJ3rc#J>rrW#R!`Hptt;9u0r%_6eN`h+Yf)Xz?e7~!U%n5S$6-; zcZ2PR+yF1Ksg2XSzRQp_V4*tDm`hjptITSGcfz0=PzmC+7YEBr%sl2AI4f6`&evy# zXZBB|w~f<;yx4Xgn8TG@m4`hR1C&vWVqCF*`P-C!jEYOpkWiDu;&QS+N|2b58b>3o zpFOJ=2(@T%byaORKLlh7!Z1w3I>zA`Jyfa?8Qa z^-cdcxb%q;`}g8u!fb1xpK||ex-eRUK&M460~oOmG9EDxk7dQMEIKw+f1<4k3Ef;# zvy$>zm)Oi=cW_?(8IL{diwL_)q$8-XUMcLaU7r*8P&=|fcKGNe#7rUUCoBoXHa6Y` z9`QHjy7jY~vNx{a2gt$fg@m*MtyoQ&Sh7R7X{#NMKYnQ2GWn_7vG1(q?@T%tY*3AI zhz)y-ha1(=HIl|17Uo3ei)Mi*v>0_;LtEb7c3x$lSNf$gtT0R>U2DXE^lMdZ2zDY> zmI{@Y5`0LMyxY2TK`gMriS9)@#CS3i9&hhaRT``v8g4 z^^0F{lEj1gnF}I_MoD{0DU>knQLg?ppFLMAg&&El$H*Jvj6}&eN+V<3qZ4r!b;cei z&zHvFCP$b~m>^>iN>x7J|3GMz8d~ZgpFfr~4!1$T%sJ^l_jfGTvK3Wya*& zMZW9KQ(T{8#|!6aLbP25$8sHEz-bGfPs{0tcBsHgDeYyr9;wJdh`&Q&rATDl_D|c%aGt3?m_r+x*Y0#?Udm*iJWm0;Gj^w ze5L*T+(~3wQKnWZotAMHyh5u@`bwwi8}Ge(o%^5{rnDid#X~m&Ub}fH(o6%>pI_M* zU%B%zM}EE}un{n4HWuGd6(Cz~^4Ow&E(l;uj8OKyG8(~Q5N{S_s%NTbAGt2uxXN)x z;n=9^uQUI2D^ye%OR7ghw|IDCE`NP|zIi|0^9pv7TUd(r6-3ddr^Y$QIV2=i;+Hs= zN7ASj)+_J6Ep0|nG4;&HoZ4wRj;$>N90T=V{fn$YB?Ukc*eHjoVY}*mS33V4cwnt# zTE?0AxIt6}7VgP~2{I)|4BHW6Zu%B~a{YePqV!tvtu&HHg?&ywVJY$A)@?z-tgCIY ziEGGrS*m%ZK}D;E`E~LVGmHX~*SyR4A3nZQGyNs2?>qR*;ae7&U8a9RF27Yn#zLQO zMi61vGq&Za*Rl-~6-iNwiHq+S@Or3{)0Gs%%T!jcn@#mYq}IcjwLWI25LI$!KZN`H zp4G4A##|ulb%#F`_5CDb4`Y^$q?&<-geT~=wF;&(a=fq@9-ep43?GWU)M|4ekNpDV z%v^>bDaZ?EwjRs)4l()vhJ)m&wJS};zF{&Za;WTF&GF7~M+aFg?MZkV>cx!py`U=; z>xEor%BWTWl%+nlt|5P@_<~m$B&<64%7ZjL2XK9y;8n+}`}D|ZX-iw*SLmtVOPPav z2AqpYIIA^E4e%fqoCe!4<5ItLnKH*n?Xft;uhQcNziLJ&Od;XlW^o*33>jM2mgG7D z#9d_?qxBa-qa&Ip90XB7vBuO8{XC9*L~BR%w{L8VkV+KUSg$$IQk!=?*wdQTF`u zsbf`%tTQ6si=JZ|iyT}Z0(yHrK3I5(o}&x-f#~*Ky<`L``ZC5r1N<()2*4;6<3#Mi zOlDi{=IdE6*t}A#%sl}}WMjA1!S89cv-x^nbMIud(AH-ogR46GCj&@A%Au1KtxC(fSy^p!gcATVa z6;CGO>jLav-|4l$H%lNx0R)77NjOR;M4Pt9>&IF6+Tbq|vVz-0$o%_cux2+ab3m{* zLe4#YdEn9J|C#vJH&MAHfB@AU`;cSqNU+Z~!~_l5eIKcz(*S)DeGG2y=izMnNfHeo zYo+Qj=$+Npk!iAs<)*y%=7WOGfD4=qIy#qwoyU1Jb^t(~k?V3-C^dDSV!X@qz= zQDY{74Hxg|pB~HbdbLCmI6_4PAAf)4@ zPlDN)6wND>^?m(;q_X{4K&Wem3G2_=%o(*Y(Gr zZO1lkksB*`(X3vKJF8p!3yqxIvxgOIm(PH$w)18Eq_=bHpa&){{=eFwHMalpJ(Lw^ zp3ygu%*n~gi&T96G=Oqut$GDDm?_3T?_~cF+J-X2M(Lm%>`Ka0mbb&*2*+IVFZNpR zF7vLkg5%3X(mr?RDO78<2MOECugPb6J(->Wj6#qzgZ?S z_X~LF@0)UZ>(s$l>5mBWllj}@3nU5dKvey?{meXr@yz_gq9FR|)mUy%U~n1*KVn)+l`xM0?=awY_`M>>*TI^_k2)RHOKgDp9~<{iAAuk$V<8{koGBFRe>VPy z=NstwufG4~2meRo)2>fckB{;1tAF1!%y^4g|Nn0M%r^ETSo_~U`Bwn&@#>vLuSmuJ zy83swksQW3>i-IO|Cf9HMAeG_GRgm?@!4kNKhN^t`wMiG`1@691bB6uiipx0(e2Bx z{3VI^^cCiL-(+`%ftg)UMTH%xrdq-mZ73gSXqY(to^Qd>6>aXzf9~+)@F=SPEse4J1RXZH0xH3_K994S{p}Jp( zUe>m1)H-cTs{^BZ5AKwJGd3;&U8t?qsc~5*OILcV>hm2ja~p80vW(J> zkq~n0jpyj?JVSH#_shG}%dH*5kTeC2b@{9Zh=yv4E}q$TEk{}2bLo~36TKwWf zJTj!(ui0@>Vtcc#pt)m4pP(o@I{(na!GEzgm8le>>>aB_4WOZK1jJu6E#lRkg!;J0 zvh2F~{f3z|zt%sj>*p}$wMp*76Bt5wO`ofbs)k>JL-$7GjzuG;d}zv7Eow}sGt05d z&BmF)HK$W*Frleuxa7X!{G`}tOGUeu(=Lh{%Cd6E&mHP-;pbOLDMph6X}5W87p-=> zN9>GT)AlGXVf`cN?T*f~mC6rwko<#R%Qx3Jg^nA207ULrN?bgR`wWBfOy-9^MZTdm z7{9*Y(tDk9!Z7qNGR@k>FYa#(7KbRXmxY z^yo{sv<|1Q4NRPN)F3i1)OFPNMq3Ug>}FEl)baPV9C(K@tv922=%41uHn#sZQ@^#Y z7W&c1k9=WZ(&CS`ZyLI6Zy(5MCK-P=-rsPS$PpHgaEV;#4l9r^56-OjuZUXAAqB3Z z<2kKVpOqtNgu{5pnJ~=w?VXwfuGwED@;lo%r~f<;Xj&|drFKSkk>JSGK_*dOz=97h z!Hp-^K&-SeZfn-;-QFyPn*&|F&d9`T3aC2s@qkF;hrsNL zi`nZECyGU%IDgGLK|P!AdU`TmyFCpp+0QnAYQ3#@n+B~v2(`Q_%B4C&T)eVMrYRgX z7!CZUt*i{3E^HlRG{h3V?UEJ%QtV7;8pNtj1U#&8F=x+8Xe%4}Zz_6qR_<7bJJ(9e zNyUyAIqZmFu65nVIH#ZyC8o5@Jj%+}o%HHrF3uZ4IE(Il^>n9M30b02b{vI3NU@%1 z6C(E15qZRvpAG6CDEm2rX3->&|I}@o=VJ2|*jK;x&KpinTmOqJyGgA)T+#7F6&s88 z#Vy0I#dtAx&(T~Nmux9 zwdg8_rX&@*ttSyD<%q$c-sVHCKeOVsd1ybad~b=#630oC#Q9>pA9oonE%Aj@YI1Wq z;01TK#3_A~7RNo?+zVyw)RG#qOqL#d^Ec|L*%w{A171f5vg$%+u6CBDNasg+yxC7J zL&zmco5s$qZOe)3P!tEr>weaA-GT2@9W)+IeZ9AN6l`3YHgc3=|Gu(AkNEOsJ1pam z%J1D%pYC}n3&|no{864H0?3YGrgJ!^Yx}*1*2B47-<03VccicA5~A_>koQ|CYEq(* zllpS|(WNIa>xTFTCt5JgiYSMY_|NZ(+~cdGDZ;B?w*&P@?fopN^B7) zol1-H1_L;}b~+Gx@#wI)-EcxXg?VBljd|R+O5VOkx4^_W!JG)Zv!=V}4?Z?Hbjs6` z_qWitj>D%7wHsj$hnVh)cnNILVfCRNyQA&Y7H09DP5_XPM;ff;@&|m{tm4M+Lbu`n z4>tpjxFreyKY%mQZ@joZNM9arORJrHE1#ZXP;K6gU67s(#XXt|$Z+t~+JMTa?W@DTn=H zkC`REmp5?m+vKtCw328REX&JFMh5b}vk;`FZEUPcs={S;v~bHF8m}zH*YvZZmDTXE z2ApEDk+PD6JC%5<=!M+h8kZy-gb~L-9k^0FF1cacit{C4tmk;_4Ke1B+_d=o&*DHP zwJr0jMI~}BGU6phC$FVYn6CMk<5_V~RI;wUs_zxm`q6%{oHsTdHue6J7vEyRsK%=B z>Kk3VR@DQXO4a!6RH?#@kIK>2m)CIK$-SO7L~Zq?p0V@;WmEsk+40?Spql``f}uO{ zvcYm!&X#Iiv`_b3wDKOt54vG@!!@cmD-OD+etT&N3grAr1~a3+)BnIbtZXEco#_2< z|2vZN`-#iaGd3Q3TrP$dUxw|&z#TAc{`xx#x?~uurK_GJjj7VJDFj-++Mq?^8tBcO zto}7U{3}=0?&)c)F28N_$R}wCoyjqvr^aFMAnYU|W<+;+U-W=}a|RGPXtmehFC0=p z`R)g)+B#9ocluCLY%&Uu4JzJoL%%>eGrFjhwuhqs4&(s;FF+1J?=RWE%N8wS=%8do5AaiM7zo>a{TJ}yc}f5din#Y`+@1B{CA!xdUi?u9%e}p{0;e@!h67PvD&5@SH)O zwpN*m=9tMn`h07(gR_3bcryvP(B}92(!6<6)gyMw9Mx8@W5&E2<$by{CU>aSP9H)< zo|&+#M_zXw!skHM>37F1kX|m;gnWC_d`wID_2tGwj6$V=<#hOGWGPC)|3Rh0`I0dg zRi`8Ptkz+ac+PVAfW{*+t&s=JNg0`VXs+aL%FheD{E@EM8(X^_lsEObk~l#ek4ZkP z>2s`fz`>{`KJRYxL-CWv?@n+K#isf^Y1H8DsRmf>O{3ubr^Z(OElW4BpiN`nqmq8= zcqrGtZ6G)v*47hZE7I-R|M8^hRA0-Fq}g*(pqqACNkj4T>@T{H!&@s641SW7j5s2I zfekmr_W0m+C+|`@F^I;7plA>P)wvPwjlU4X7)8PUS8#~@dORHoTOIzC>9aU8~beVZQ zMSmh^O5Q2izC>(e!&IIRmmq=d4^tuDr-#9qwo)oV{B1yx3JzC6h0zp#zy+tHptEq?Zc99)m@O zufNWewgE2l==^$>TO%@B6nO+b8$U}}zSNx2VCcs2jYq}gUTEopeCfj!H4MkYAw}X+ zNJUx;BaO|VAI0q*Y^d{+!mv9$11GjJZP0tlb@DggR5S>-XMHVZFSJrma-#BDcTe*> zG6etcZpPsu`B|c;aj#puAB^+p!=)QQz1@((uj0n4}ckAvW zKRA>~d*3keVVp5@MR%JY`~70m#?C9d3cHpsg$2u@Q$C>+dJ1|$ckV!W^G`w5qBYjL z#@5&{Bc*5hW>w%)i)}p)F#|zm39g%bC)4Lz%S*;($+qS7Zuz1r)IOl_#yjuc?jCJ4A9f|)96rY{L#dw)QkA9Z zx+mc1`v&{_CCGjqH1F_tf8FymNDVj=cK-gj9dL7e`q2HA7s%WO!i2cQE~SOIxY_$= z&Kvj#u{!hbRetJgn6DSMnbsktXJhr7wjO@{RJCAhS3_A-F=(`IKj`3BpV#fEKpijf zK(24w!)^MJY%Pub31_qTo+C|Y?XI>XI=ZNEv9=&@wg5&5e>egv)UY)p-er zR5LWmx}DML2%pypeC`+2J& z!COj;`u_t?MVbbVb@ME!3G{l2BVfuOr`vXJY4c2h^~jXM{B%D|)Ri}}1vrv_OFX0z zk?UslLHaJ>3w1NpDRIx*p14iY<_kU&t?#zJzXJ6dEPx~=!rQ4HyrgCoynK-=A66de zO0h>$M}G#FX6uV)ss^Z>6fK;lO$q%Q&w?BufhgjcAoj(N&LII zN?T@-&pD|(2lR|#&mv)1%gUQXLgCjSap4TP(Q@*0T02LJi%$cA*J|NsqEK7m$9>CY z4iC#?EKmy}YfT}s#e5|FKE_T5{-8$LELQEdZxSS1hb<~N@6?}M&V_1K562oF z*A>3_t)tsx$fS=R0%pb?5_?To&mWKnMw)k2SsoYYSw2UUJ2R(SAf4!3Oj!F1O?ECy z!tA-%^5(C#bPv9WZs+iLAVwQKQQL6d3d1~=JXr3l0t9g*VEcAfzbx)$3vtU=FPgft zl$jX}Qe$q1$FiT}p$e)|_tYyLNe-=iKz+_C*Y7k-p}Yv#`+4yOnq%W@2WFdbTFMg3 zQ9jWIy?3$G5?UpRGYdXbgajpLVhmuUNor>wu#6u8rF@f-Xz zq8N7w?bG(ZEVN*Kcc;0W94j{hA4uan-AvcR-L-Va zq-yg|2N2vF42f4(C*$ZR#MHrU zlt0!EO7jm{fFF4Lz06nnGdIC`hEydn|p3xWdrl-c>Dfn4^2xzt8M&b_02G><`|0?T1%8qj!NJW8v15c zJwgOi8k<}$C@jrjTse11_63L1HSiMQ^LsFt(h2`r?e%s?C53-%<38wAp<=W--+RH z45gz1f~5#_RA7ypy*6?$79yXi`!`-D)a)aAm-dl)#QMH=tVP94U04ylY)>^EmmapzMzS$JehHs;{b~S_ zbH{7>J(zd%$1ip;MdVPNpXRhiRE}Zu@o$IZ{2xd!1t7Hsq0htOmk9bt*y}ZXU|}El zuDf`9+M!^Nx>X4Ejc0A8ujJC{WmZiz+^t}G8z5ioC(6lXV4{h{Z8s)=a*==D+lBJ+ zZTV|9l7<=QGGTR{tJ{qhUCEv^Lr@$h*-=^>bvI)=6+!G8Kd+={%qp#v0A}}Q|I!)0 zm+x>&;*+w{m*MR;6V@C{4WCzfP=;IDsB>(=Mw%(u1hPUAv% zN5#mo6UPbPCdi_ZFj{LL7SBh}p4yfd5qXo6-Rjo$KHEjNEQQdY;xJ5}Ol!)sR5rC@ zJ8q{D9*|raD=a;l#zc`0-sN6;@r(ELgFmBkpGuoKAGWD}uI>;0Wp#^GmR{iJ`>aQR zT7Ct&E-tGiZxalZ^_C~z)mcTyOrKd+DxrN#xbdZ3U}8$ANbQHZpZp+|TL0DV1czip~CM}Xw z0NG*vtkEs6W>*8QVkPYC#KgyoD!I0U)CaBdVN_@nfsP5gDzzb$YrOd z%GvT~-TY^KGM!xVxJu=DFQoqkj)Fxggwgf5xA&I#`sMI)fiXTJNS}IMtOITfqPSe) z=-ikzBiFs(z-lPqJlL|eO1MSCyDodPdA~NL`IunR&xycFiV)zov0|SF|5Wm~y83TZ zJsn#rcBJpeC%~3C7Vq|rc8DLRacgh*Sr?ldrpmtQ;j{(vi}DX z&2!?zu*^Rs8kIm4mvTo`oGLc?CpkZ;#m)A|I*s{3tjSpPD6bEkTu`NI+Oxu!i+^Ir zY*_Czk6%Vro%SRnE|JfltvL!qDu)Um{u4y2lme3yiIuc#;%~xoH9NHv@SF28R4Ceh z<8^ukLhny_+Rn_-DzROuk9n;+14D~Zo#}Uo7>xjHI zk<8aoxJBxZXeD!poe(l&Tns`;BoPHzJ;FEC5VlJXJ~`mYC$%Dnql}zlm+fM-`OAe% z|E*~MBe`*uDKFlvlMES5z(^99h9M8~4?UJj)JS}=l*B7!Uj;~qhwm$v#Arv*mnOVQ zc}tHY9?4OzYX^`VweO)~&ZVo-OM53>tbwMr|9ppq94mfukUshH6J_K4mx~sR0_$Py zv;H|&`l{@nC2#U=x=x>PVF6V3N~@uhg6X}c(Y{YTvXQkDK5%zXQsfC+-1RtV-t{n- zDL?8PAnf6uETf2DHgl&TP-9#XTs^;V`_;mkUBk!I)+TV~a++J?W7ugvePBZkRbqo7 zA7g?Pmc%NEJ+o$i#GwHKlAXQlu5g$2(75{nrFWtEYocFv`t%y|b5e*oVjC&5IiIR- zP=JWS!$1;=J8_H8V>=4mtz_R!Z$c@1xkH1HPW|*Xey@_s!OHmeKdszcQ5SYsqVxPo zJ;^C8HE(S7L79ImAs zI>bqznTYWWyrMPuK;WFT>EVn4hBqZqN3;a!yhVskGvDc8-aNx)o$xHhyAX0rI1IR*UxVdT8L9~n85h5;*zkSxdRdV`)$lIL=m1c@w7 zo{06$Noh|rl36wDHXolex3b)F5ewveRmYC@H>vwywE=NH&Hh81O6 z5YB=qFz$Roja*yhcX?U}Kb57>u*gsfit$^M|J4d6q0Y2wQ9tghmY z>1Qn*cp45(;wiigtE{Xhv4l~2_Pqg6D|zsR)l8B`B{BuXR*T7!uLr40U)f;iZI>fbmeHe?1QMYs7EcWxy*K z6qbYL7(aRPl|i9UdkO(J(BJ9WEk-lWTgqKtH4$Y7-CDu;C3iHas z!ULaTOu1iz+mjviTwaz1Roa)#eT3Oswc`5ir5}g(zu5F5`L){>xrEVE$V&(l;P!(r zn`&UVB`|Bp_CW)l_+fpIq`b`J6XiXgslj^W!;!=4 zcnfN|B|?hN6a8?H|7LBcKoAi!0Ny$|U6YoCKbOx>xwTH{_CYFfFaV>5^msF*D~M=V z(ZRtu7d>J1RphIp=7-qj~)N)3>c71<5Aele5pwD)Cjq(+G_df|An6^E3RZ`JH6MD;>MzW|A zUA6C#etXiqB{#tdsN5=y)Ab0GN%m6lbK(nLU4lMZ&}Q;|tr`Bl>iw)lvCwCDNghm% zy{$x*Uv#*w7B$X(wuEUu5PqD#2(j(@uzo}_b;?udy-caq${}ZCanXEcXmI5HG3ioH zKDSpWX|#rHBppv$?9TcGN*XHa$ykCq?p6JJg>42aHxCvbr4RZF-%#pskI;QIZCH^o z(QBa6C>FmFy3gxS`iQq5O1ucSsEGCsL)F+$SC|A{%0QXf4pZNRDrKXvsdl1pSMD}TWbbn4>DWME9Gq<$=F9Ls6&B$b_(;IX!3l-xBA!c+OM zv3)y6!?=K1LY=C2>f2j#Pn52luv6B<_ao7rxr@#nXIVZmsz$itd;eT$OpH|Vu<|D# zfzXyOCV~aGT-81jmuin;Ef?>WgI{&7vqE1yVxOYV!f`6;0Y@PN(w)N2hu^GnB%sw1K;S6w!CC zIQnwVl)D(MXXvrZlikfDsXJo6A>~lJ0fuQJSP~4_uyieqEAI5l^rU;gE^AdmySc-Z z+7``NiSE9NKB_#l$As!x81KVu0=<>&Z{UID{Kv$J##mWOMp^0 zI-w`0Vx}DWX6e1;?1k23Z9A$|0|iH;;6l4To1fRx`h9A`scFSoOc!g!N3d{y7v6DF zbi#>Jz1}V`Hfk|P!gDre-qsvO;^3#rNbg^y_q2&$hDC?bdPPzEWbk!T581-)DowXg z^1m{X$KQWMFYG>rGE*<*{Gci^6ZD3-&F&1r=NcfM%qb*&s>>qSw2f~+Lb6t<=osEz zxrqU;yK+R+*vb_rXF zIa#Dbf!j=)+`^9%lX6s6rHMo_W^z=z`8e7;I7ixxMg4G^yKY>~CQEXq=~YwVj%XaA zPrYV>UyGactOd{w$irk#C}H^;Q!Ebx76q`~o}-gfw8VK0tB%5Y$mC;k^XuhW%7kl* zVWmflvZt{vwGd}{1SF?M)I26-?riGcaal;0Z*M+-Hj}v{>6R&VpEYMZw%fSLqTm9s zxx{kG=X0eyJ^CbCAFvMH^IN%svEQO(&sxKmLX|rUNoC;4(%{-A;k_@pDklID5!A06 z{FhO?N+5gTJTjbZgD!7%flI&$l~)MAUCpZBO_v7#6%I|}p=e1)Vm%Mqz4~--K_Ip{ zvlavLr+|DmnB6A&cJg@f;$QJuQ{nt7pU8kU7Z){vaxTXEkB*dKj_hJzzp`5x3l7m; zS|#~|2eSy46Q^KCEg7A9Zf0NnK`%sidjikJF|e>^QzQST%9i+(S}B_2EqDK)+Ri$x ztz}=>r!6f|pjdG$?ocRF2vVRFFGY&GySo-BP>Q=lad+1gDDLhKMS}zh0TOQ5`^ets z>pu71=X-Ac%&e8kGc(UxGxK|Y?`VvG?xGriLs`De>j$L$jGyZuD?vnx6s{0&VF_(GO+ra~JIbsBha40G{n)f!BT(Tvn@x&-m4>U?dK)8gPi)Yu;W1$uH3_~jmT{3Rav=P&;cX~ugz@0}}xJVjcSem323 z5b?+VbLCRi*RA48LumWoqmWt<$(W~>0OZnAYlWgIuIZ1vK91JO4PJpqW-ohz3k85? zemeKp`w>r+7RO2f#IiEMN53~n2S(&vj>?P1r&G9exWt^O04djvCvQ@)-l%*QQO#eC&$qFEYKRDVcr&Y2$L*y2ixEefmhnpT=W8O*F48vGFFACyvY7i{XXHJ%5^LC?FS%8_J~{$RsScc5BL zZ?=pHJ)+Va0o{HA^*Ru>&X|Xr{&1LK3#E69!QDp$oyiQ3cXTnwQ)w52@$;b4YAvHi z{rLcOGPS`BRUPBiyb@A8XG*jdi$#0!YSF7i`iSZD(3hbc&B5_*Rc@+CZ+Y9lG;L6=DW(-7A&C^(Q3UQX_zf*Td=j6S8L}j2XH3+}QBp zzvGrBD-w2w)Q8_0R*Ns`AlMw5KAz{XhtFYydeCQ{R%5gE~$=U5toQ#JFQ zb_5>+(idV~<)ufna_fndf_E)uv21*{JeP!Sp6>0Wo}OAzpM57{!z}u=d>(##)goih z%Ga~f7A(bT)=O`LS3;$n28b662zi7q4lUL~)xnBX?XX!{cFqNsn~rdAaxZ3%lm?Iv zzGC$dK_mbTh%-uqx|LxIva8=AyvT~TG9@y~ga_$nmOzpqR32 zcksH^3i2M-oGP~$P!a7JPPiN!9&V?nKj09!|0t%XG%g>w`%h3rqy7^63f_N zSsb=k1TSYRzD@>@D0p&Lw*?h?{j6k}@VlQmnvJJ(#g|)DKlAQx4=9SeWcqkw{dUez zpwvNGS1^RzZP+E(LEV$xU{3<4&SP{8$M`Z(*rPPpft4iGFbTI{N$** z#5!(mO57^7;*>wy2*;G46mlj%^=U4}TgBwJ1ounIP|KxqO534_elOU^IzH;_L@eqN z$LKUOL{0`QlpQOj0A19BB z7<@D^Y48s#eF2+oi8<%uV)_L~tQdNG8EX4JSe&Zv>M%!ByJ>{ifbd;aqGH9{gwf;| z@^W?e0hm77p&dkM6-|!;{lG0X<3&{ zX6gG_RQka1n{xfw!`Q+RZxQpMELQfI@e)p6`aIWsHrIbCnUdZvMoXN_CoF|#?A+ol^YjfT0 zWeqUXUV9ntwv|Ub`oBBtLwqRUrDL-{oF!RV1t!V{Q2hsa6>Qkpb?&GSE9r zQ`_-1)j`%>MVWsi=(M+K`GQKK#uFsnBG`>9?-QwMtI?IhZK?J<@8@I~y1KOJe&W|M z=XKy=qf-~>uJ0p>*Ytd5Uo%35jrnQ><{Tq zrxy_z(5-}q6#Yr*fr91iO;mE=av)lB;c-Sy&G7~#vaEwi+q7U`@91`!U2Sb|D7%H+ z$U6+pdV=)mdIh%hoZGMZ+={8LLD@qabKty0l5M8L$_G5V1gJ|hG#$LoA)~pGB)@{O zs~toQSLs_jIkDIxODlK?KL)?*IKpetr3DQm#r3PV)O|M;aEH)n@c*oYM&HcUy6mC& zNQN@PicuxoS(cn|mA(S?7VTm14rO8U9~4Pb5kn^}%^p`c z!Ie`k;+@mo#%d`oX4sPf0l|4a)uN=637sZ##@UGnl}_CQs5hu=T`}64k1oLi-eylz8Xg{hGVfCS=#CMy zV562ol=vlJG)}*$SSr8>ps$v~m4(J@8m!MqCGfc@78aD3O{^Zz^S&#lq=4S#y-in) z?V9Y87afL7Q{cxqRi0-u=`n3uY>E0I%ozM)iam$@Hi8OuVoJ{5ubAA^__@k}n?jg!<1x%S`V$D+JnXZWqj8xP#JRuhDc>HxUQvi!I-<5&!F+U5R8`SV%Ly(0P zJsxCVJzJ@qb=uK0@2TcsUGYOjED;UK$mS! zb@wBYbf|d7aSP|@!GqV%{Q(L4?KKwAQd!7{oZ}-XporbH8*(6KAlk>e1ndkf#avz2 zD^h1B;(R(tubZb>vgKQrTzf+4106!Ug^rG%rBc|^%`B!dyv5*dnpZ|ArzHFqo0b#I zaH>88V)nOFw1nVc7%!JO+V}DO=$Fao#|?y9EEKZE0pLuagT=N&FT)VDU8?5`=TO0^+3;GoanaBvcK)WO0awyD z)%M-P3P&cIi%tPZG7Xq{&}UgK8T{@?Yg-#eL9Yk_G$k%<`9Jk2@Wt0YKDjCOc=t4g z@_Ij-v$%tvHBuWTL|=~7{qFb{2J2FWvTSACB(9g3mjAU5t>^(zL;0`Jy|cOupI4aH z5j|B1`x+2V2bi=9Nd?H&%LGWB0(wg!G(NJV9ccf@q5 z2}zC{yiCm8o(Fo%mRl)_!9Gw7Tz{PWfcyjJRaN|u5>eMm#;!<6rZQr;tB%_7)<##b z{#cG=A`_8mdO$2CSWJPlnO{Q4s|`7Ln2>R-=|OJ04uvKSx3CIi^7!Xdrl)JxS#M&r z4}re5Y&>7|Soq4**6sIDn|Wf&(WJLG7rTS?K-4}io|R9+esDouiw|b&V#0mVCvply zRDd#RTJGu)+qd=j54E01?c`;vrSeNv#7VxIFVL z6SIuYZLUzm1%HMI0S@ldl9UGP%mhb+r4}Tne{EcY5B=~8AG6;>gF~jfDz79E0!CZ` z=$zP4(Q5x{J-ImKRryPja%rV1&Sgj?PWoBTS7LmfF8ogQ9U7WRA&GZgXWNI-T z!61?HP%D{*5$D$lb*8aV67S2@P5bK6s2vs<$XW%iX4*Y0erE@fF;jZ6`-|gp7B8dX z(q%4_1zUkT4`oxj(c4rb|C$D@aAOqf)EDIymk{0vFUz#25)3_$%{-mi+366D(`)m3 z)Y)ZcExVE9dKsPOoKc$txZ0BK!kVv(b-X0Cw$c9#0p{m*3bQn` zXML6=eSv}9Qk#$kJ4Gu4rTOt}u8~P}yi3fe&Ydm%d_V^bxGq&5q2|ZeRcBjVi8`?7 z37E>s1ozMry<|<`>k`WU_A5k@U7adFtJEuhOwL-IR#jt=p#+rMTpM;-?*Mh%eS||h z%ZP~A_iRmwuNTn_l$zp@tS@=KcX&Z#fzLy0D#^&vmQz(q#vn0|ND7xPx*zD$i>VF> zYBdxgoO8{10jy3a_Mh~smM=ekAv_hl2Wzq~O~xk(lu+U?7{z=G*)?2}PuakXt3?d- zqc?(V%1I|jl)0!D|5GAe;XWXVFhIJNZ+YA z-lLUS=Bzsgt17MzB&;)@B#KIzotKf53f)Vca<2qYG#mU*%8i5;lDSVl^n6CFkY~|M z8q+D~WKE#3RMcZxlbdpuARpPv;3KwXe!%Ww97)j8)EB}wT>^Jccu(jZOcf64!sS9L zH5i1UAqz9cX{n#UOx@woor{TAOB99$EqW+93G!X>*|CS$=Z{w090>|W2602;yjvTX z)7gM_q89)mr8DlJI`(nqpV(tA8Mw%hvx~il%C8bC1%@2^zUiS?rAvv%kzkn0M+;66 zX8!}y+h&Zxj++Zre7)Ll=CA=DXo4+#LS6MqrnXzo*LU zk^CVd)<4uYOPEk^ir65{pyyuvT3NV92+s}Ddb`sk03NClxGB}S25EmrhjHz_&N460 zZ}$C=o7L2=L*BV)FpE$iC#?xQkaH&5hE))oLylR0Q>)22j}xb`k?Y;i_cBhj=xU1P zq*ZY9ct98a4%$HnW~V=@LV)lUi9djL`h|Z7?bgOqZ5i@wN|PpRe1EQ1M?4;HaJTE!Ys4Gt$j zI6}qzw787>)0pB=zNQDb2r==E&^ZL*C644O`f$VNQ4$hrs&5z*GdWvS7kuIy8mitP zOVn2rMv)C@U_^96=5kg|=t{>$jt-D_Aj|&BXL|3JJ7-m?6W zlLecSmhu~y37eJq^l%Rgq?(J&eKTT77-X>Zd_u-!56>n2V7}4l^&cEK65rKqiYB0O%nXA7aerZ;}nhc_q2@LQh;ey&TSoU0{D>6|pAVz^)I zN`DR&Ex{}JQ^FNJs~CDf;#EF%42%vEd<8;oe)K*UNI=-Kt=szG6D_iR$pyzaX)#~h zqDy^?_G`kgSoMX1sebwKXN>b|fp0mxnjv_3`nASYXLoHwgX{-+z~k;T=RFRQgoAGm zk>8M=*F4BQ;gPqek!^cTnJlK+0$*R5`qG2os)NTc4xq=rd4~5TUxw!qU&cHRvt|uv z5MCANVG)>N3<19Fgfwz-D3jZ9N5$*t%}~YuC(WBbM8EN<4NMAeq0F06w)40FSjiWY zN^MbRBe?iL^Xrc<2;{gK3mkSch{fU(pY}#Em=vP%Il5R0UmKb8G1oKa-OunV@?jBq zVZ9Wn!BMX}NtquF0OgpOCBP}Z=(o#`?b0oi!u+KwZCD?3`6iZ z63*NL&m2QX?%%%kT;MeBKM;XSy9zxIJHkIf%icLqZbV4TbLAv>S6FYaRi+m>qcJbM zyb>{kutyVJe;?U}-Ae$w5o;wO4(sp*`V5yj%q)r&cd@Xm_9xU`>4QIUMOpB)TNf}XD zXr?(l43mg=Y2a0u-RRojUN5Wxu7#dGx_PSJ?v4J^qg^b@YP;DCQfbX=7ycMDFT22& z>+l{=)acvS_hR!d+=?l&Vi7t6!>kmwcWTExcoR{V*)W9d6vd-`#)-hYyM!H`h4A(i zYU=(oIT6}9=In4ljcEu)_glUF>9-rj3#djcz=tE)^d-y~|FX zVRW}JhSolG%qP)kE<_xHsOjsS(hEP}$nb!F6cVx(+bVOdcH4M!bgy}v&RJhoe~LT6 zJEhPk9AOS+`L?>=2a{efAJIOgXR_d5kDgT6rgf}$kquJ+6( zk|+HkCyn!SItjg4-Kqq6-1zV}RVyrNn27Cqp|Clh?#6yj!t{bLL>7tK5CrcfqMOxu z-tFwX+b)Q!N|x6oJ?Ljyjryc5!jqVWs}AhjS!nZS!3x9vmHQ1X`d{w1ioCh2>vdZ0 z4~yS+h~k&z9jM_ecUwEDJLgu-R?_+ zY##tKL)qUl|K|HxK^G_EY$j$`rzdMvp2KYUe`|h2_z!a%^jY5r_8W)TpuK&q1=ee6Pa2YI?kITvkMoMo&Q2iQ3jvi6)Llx}4jMo9i>G z((1Ms7!FVY`&s79^x%ReHWG6)VrI*aI_>YQp>%F*{4H(T6 zN$XO`a3*mL(>_tsHgE6XnjAzp_1FbXUz%%_pwrP~3I~|e@C%Bf=Q9&C2bhl@2|kGp z53lQ(TA8o6ouDpJRZ>}57@#*w_LWOJ+*7cX{T1KN&v@7`e_EBilPCMS)$D?{K zL&W5b)DK#+H$CLJ@mA!O`sXK`u$ui?NeHlrh~J%W$XPY&D=SGwXPFjm1SE@8~i&OCL`8D{u|7*@+<535+u{?t0QTEaM5rh}7+0 zUTuqyIzmg6AqoGKri&5ku4G$6XUMxx7-{ zgtmy3IsFPpi5e*<5rwUAzIgf1h;Ah(f^*89)C}=9CDPO*btxywm--!gd- zQiJXOwo`IHt{b9~6L_~iGP}Rrr(xOnn?Y!0e%7tB%$CH8=A z@VNb>Uu|wC)#PC$H=#(CkDR!TX@O!91XWMbm8YL**yQ?=%%Y#e^W{-xC{dkf9%%N* zrOESc2ObPVSQBF(lfTVR59)%%YanS1W(I|A-_?ls(w?560p3zXD|2A$fti@52frlHUccKGo6 zR~I(#y6_6ihicxvR9?Yl)5O5cH+F5`E-x>Aq+Ls~@KzK`)Csr3CUdc}v$M}9NClXT z3sk3CN}OH2*^&{)05>X`gduv9Am-s6;D#=R*Ki)(nk{~{Vp|cPkJV=Ai+R!V;R)1oP5d?QVqV)E--M|^1^3NlQ4=xliRrj!-FLl+p=3EIV|WZZXc`yxueaR zBCsE@6)KO&CQoYad{`8CmT#Uz!p()3>sZwh!iat9;Jq`D9{Nwy8=bGS+J7&4I{=2@ z{|6jwF~~J`L452_sy9dhVguZRBg2+`#_gSL0!&Z3^<1j3TCH~S@N`IB#|Bb13tH)f$}A=JJDjE|xEDm#J`H3A zR!Ch5qz{u@?D)$}A+ah#{zl5=pV<)D9~d=4+lr7=E$046jNhFf<-*14+jt`rW~NVX z4k!~VI5JDALZlSX)a$LV9lIV?G71m(9(&Vaev+M*CZTmVE~J;J)`LDx7>=YGYmdt* zI7q_Y@66kL1ik0BdKWG^A|E^EBw1+ob!_p7n#{-Edr4sO{dG-}hi`UkMZkQK1evPOx@r&65Jo$T=F}yj%hL|z;Kou@y+2TBbwpaU(t{F~ z96M)=r_a&{4XmLL$%@`D_X7YUAC`A?e>-iY=V(JDhTRf42B3g!RPZSouu`zU_;$o4 zQGGDR&^CeuOJPbpdZ#{fmSZn?$45b%m|7RPqBiHJ7`H6nbnC%k0vTAWFb542N^h4g zVO|)t>V6Ef;FRGh>#VTjU3{Myt|w1-B}EH=V-Mi$IF3+^Y&X|7t6UF+m!gR`=XA)X zZB-y}332l(1eQV`Td;@VQM3)NJZvGTh|)Nfnf-`34%R-qb{?Lf&F_w(#m*7|$>eKU z0OFxQtuAo#;odjI?Fz~3V9m%AtcE+oE7ktA>#gWE%F=ggy_K$9Re$@dj+^NS4`IJGeo+JpM%FKehUAa^!L!!^LhXtJ(i&`nQGhQK`I7Ev zhb=?5o_8*I+M2G*N-#yX4+;xQ9}}VI`~?fyiogr2{+{XXe%_bXdV=N zi4cgPmLSwt({~{o8-M6;GCL@K ztWXm;Nd&0B^36x|O3Nj2!f2~{5P=J3!4_g9Z0cQsR5adsX7wSduxl10`n(;6jJSNS znzL1Je`_Nj(e;MTA&5K2&k}Ycp2Q{_soAJR1RWo<5b3;RE#uzX!>Xrd`O@NkgD6!} zQ7K3KM78lC;wR>TzQ7|%qBGUfaYWY^?O+Q#Il08>>~%oMr5o2_V{R|_ew^(dAJP3- zb0qqw_fKf*H$IHxp1#tfi%I9zM@x)FB#|QD@pm@@#VfG-Js&`<&U3EDgd$94!*E!w zA2tUrv|PV)TE_FbnBAFEf8Dt`et^UMyWsD#?P&5HSOuY~yW-JfU4(CX@zMkUg=@mC zLSH}bXdoalS%zAYV+aj0kJ`{ff)G&b1qxB))8iR+pZh&c)ks&a$?>wS_n&eYU|z=}?^VH;*8-m3($qOI}d* zLbUM1l5jKSeyOrhr$aH8smpBb!r@`yTOa+$leTx;4ZY=Na8oK;lJ4dL$pAhhaY|k#fgrE>6Mn2DHJL1wpW_j|N0s` zNwv{CZ89>$dQ}Jt;MqT&ToP0M z&!vc;Es5kx?rG4rdOn2O@y{)QXjQnVwd7kG)rAEoVa8q&e?qA)S-z(JUJZ-wMV&E8 hn!%zxVYI8e2aR>c?%XamOMgRrq{QF9Ef>}E|1T!halHTl literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/compare_result_pkl.png b/debug/accuracy_tools/msprobe/docs/img/compare_result_pkl.png new file mode 100644 index 0000000000000000000000000000000000000000..863708bf6daf46985328f0dc42d48f0a5b849af5 GIT binary patch literal 36133 zcmdqJ3sh5A+CSWBr`oZowTglQsZt9CUKJ@tZppMF*lNKPLO_8ih-smUT*D_Ul1KuHe_1i&brqg#W?SH-N|9$IQ%S#|R zC+9q8@BKW#%d_*21O}{Kvvti2FTAk!gM)q_zwpAUL*VtzS5|?4GnU)WfdBrG{Bgkh zFEsFM=D`nt#J~5^doR4uNO@I%;$`sjpUxf(Pk!Nrbv@?4f8Zq4oP6Ph!uAjR-U~^O zTBQ9U$^L`l&!k3=v8=^x&8xwP>0P^j_`Lu8;g5Gzlowc^ePM&&&fwsWhMH=9j@K+KL?$%novya2~oxN8Qa@=uD{Mx+(9``F9KVXy&2Xr+HAGaYz}hKn!;b zbCgBZ3Q9iVD&Kp!7rq16Ex0XHB(1}oEb3LrhwiR?mrjV0K)aPGqOHobit`RidIIj! z)3d}s#}y^*>yxkHN!89jpe%gW%RWfc%(_9^HRD8vJNJPV%Xtr7?L?9rdfK^DsH<({ zcbu2`%=-p!{QD?wJyFUfR*umk#t{a`RuzqY))ev>_QLz4xauocgH}l}ZKywx)<5kI zRUjg&Y>Jn(4yujPy%(q>gRh=zU?u0cMXVHg;2 zXFgU@)H!K@&YXm5M;&uxIu(CO#mWd{(*^wO7Q|@e;?f_}O;f`-)2TnLzp~nL?HPC6 z378?e_ZZ=yQsKbA)p1!6LzyPsC5D(jVGHzyb^H%0_w?WC9Inj=^1)Sr_>UOQ30ZSo zYsS?x{BnxKVW|2c*P6L?*_)4eQ^lexX-A~kWy;b5}+oTn`Os>$=%-$ZB54hb_urX=bZVI(=Sp4!S=c0?Fr`8Ae{5XjipXp=pyF zXgNx{clqu^Ye37*9fq036Vix!@8N_FGBr5+*}y z^PTZ^aZ^ek@`ZkOQa9m!8g3f3O6cDe3Xxx58P-q(G$3Obg6?G?MQtc(F3NUgo=94G z4)SKLkpWY^lum-~Alji}DW*BYp%SjKKHzq27#m`s3A!T#)zgc~LY?a%jctC(W0ZoZXL@(VtcE>B(J~z&Mr#;Ri6{42X$)P zH)cdsjauOjs2_yAiy)<@pFUn!NJPo*Pefg{B91mrzg`->1JvMl*$zNCk>j3WaKD;u+~pbuLX$U;YH znO{m(%zKeKd0DP4_)uh1yJN9yhr~v+A!?hI?({OHRN$9bIPTscHQqIVp`yb>58|tj zZnG*N9uV+rwF)gyMzpcS?;b2VpVZ-cQbP_t$gGO-p7|8?M@!aw7MRIQdL%W7A7QO!jWbd;{d$XOtPvUAYW%gZ6a!S zR1Z7IE-#)~qGisSVAlu?$ zLlvJsoQvB~@V4j&s%7QXGZy1&nlt1Y`Y`;AIYw=AZzj~3`sN>FN;rOvg=ew_&2mBo zYD0L`CQg`mTX;gxv$$QkN{LK9J@P7uoe%x_>1;_4iG|Oj#~Ceot?+=|#$GDn8l}0N zva#1S6kgTf&BrSc;OeSXC5-MLlVQuoE2_18knOCdpH>gA9Avl&@55?b_eJ^AtqTsk z-*l^S<9;X8CMM*n@aYh$m@)E$2R9SiyV1kBS&y%V4=Z0&AITn0rb8jLZ{o21aaHLa zr|RaA<}s$4?RJ@OfCUdyd&_W_@;l_t>AM>i8w~15L|XW?%$UPCK@pl4wWg~AgVYEs zmNFG|0J90%nEsT||JOKGTXx zd`U=?CJPX49p;b-YDbHEKcc^jsNQCJqRTyD0wn<2-8o84JQbpRg{RciX~KOu7_a5^ z$;9XP1NBW2OKV#cBhI(B2-C93*{!*>Qo(`5!g?*nXii*p-^uvKDU=skf#@Qe7vo!7 zbgQIwKfpBK9ckJ4R3z_1pTVO05yx=zV8nPYQ_i_|N7b{=Ya{2!)i{HGe$HO86u*Vn znmW=G6Ng0qZKT*nOuB1-HVEGteW?<$I)-Qf)9j=>hOA-2A@Tdw$~w6U z&M6A4T_f9KFh=e~gMhejR1i=naYK_VgN2y-62Uffc5dCSd_v5+Oef`^o*LUqK^--= zjo?WfCi!_#@?@2ESE?Ee0+uc1fZaK=RfIHpFMFF?WxB_u4PyUP@Lthp$aT18#NFY+ z8rfYAHPAw6S8&$kF^vkuoI^_d8r#UxQ*>Q`(sG!?GFU=udHYeSpOinO-gRFyt}YkA zPq&IwVk{#zMpj*jq>s0eckq8mPgw*jAm;2xDMjyx8h8+A&aWcR*CF59Dd+t?IV>vn zfjfHrlf&ago9g{QA<= zq3gnWT4+`ct~^KO2akd z>D%LWX;%RfDO-JghfLb2K)NA{w6US|Ly#0# zN!_^2PXNGhKoUGn;VP#Fc>W$S7}j|eljLojC&oyPjZqDeO-QR4 zQGBJBJvX4zt9X&reFOJBz0wOOP=09mki37mT82Z5(=t!2lLn}r9Jqv`0)2*&m>UD> z;dLAdTm`Nl0e&~FleU7feiA}2SC+sS+j|4J#ZMz@^Y#g()l0=nZv+2qwhZxlvbsnA zcIPB9aPgRWUI7kR7~&*>AfIQLXj$HFh>zZ-2fcB1H|G>mR1WJ0hXR_t>49C94Dy$h z5OrUkuG%YLwfuIrm(-)lfK4jupDgFgjE6Beb^FKcpGE0f=*x&D*PC+t&6Li+UghQf zYf>;&P~;owbF`;K!znQPKSPdZKMEn`o}oBr`DS#B`n|iC58*Cd$xS+I3$M#8NdPtc zW9_;I;+3U{E4lK1!cPT6d)2KRA>jw;wy$8OgyY`nzn3p8$9GJQS?piDOZ0pff_Xzy9Qe{{I7McOF~pHI?S@4G=C*mj@VeKqq)4>r%+T1d zdzz%ViM*w*H9qt}**T;p)s+e5WuAOI6EV3|?wIqnsD3$YZWxk!Q{DJ<)XAj`<0B(@ z3Lcv?9FRD6c{v3P^c2?iGSPPViNwuc^@4TLbB(64kLAhv^ZM+FS!YSg{#h>i*NgziIA znKY;c6}-rnyDg;35^cqNFfVo_fXCSX@cGb#lwMq1Ip1L~bhieS(<*b-TI{O9YONe( z7)s4972Gm-h71tNTlt=k>NOKf32@UFwZv~39t=B&aKj#~l>N{i(}h;=8<>Z!WO_lC z{E6~xEaKd(8(n#pNI(iaL&N9EnMyzqy>*T6@Y?X@81T*z(HOfjm8!d6gqS3ZvTjBk z?=VQ|e+Zsu9pPmmCcbK0@=!ADwWB~e+i99 z%Caqr`NZy?rC|E*dw;6z-t|^KZDpB({*-bG`RkEdW^^XCvSk?$jkP@LiPZ8be0yV| zKud!cDgFqL7vlN3noCb^j> zHlmZW!->6vNqK0~+`jE6nE98N`oK)lq8i;XJsG%Faxn8EqRDV$KpJ09D`&vGWn0x-|^b zbk??iy_$&B_Mvhn2|TYj$}U&ni%+*Z)4}DDCTuxY)XxS*sY7Tu>B{5nhiLGmmnTz1j|Adb&JO40GwL zE$c2;;f(YL6N{WH)p|@QYDkcQHB6HVGq||xZitzn4FA+_8Y#_Iya!XPx_b@Qug%1g z!@a0@`I*zJ1@bwh^|KrL1Kwsh3#1@47$O}IT79!P)2pOfYwg^fNHFH<940t7(j^B% zq!``Fg;6ChU@;-Sg)mo3>MQO3MsjZ+A>K#J%!zR#w^ro+btS*6{_B+5T|#~csjsap zUlDCvI%V4}mmw{-A!O+tq@_9G6SLRI_{aa2c1s1!?qcalfKtBk zv1fAjtpSxQV+QT=qP$A)RGfdL;+xKmbk=7kUfYiC&J}}sJ{v22_7CUAN3LVp-`E9+ zMtkVSy55tCtK&V*3(BkJ6o!haK(*nApaA-VI|k@k0BUr1YxISIG?BETK$xStFx0b8 z>IWA~le+UyOf7}YrT#E8_LA@^*VD+Y_YUDDr> zC0+mS)e1Q;n+kF?68D>{me-t6G47^MXAHy+AGJ|2CavmrY5c<}u0AMp?0`~K21A2? z21%4!L76UP=l$k$pOQex`zT>7?S2aTytmG@JotLYiGY7BM4$bHL__okPB*r%_58gQ z_j5Wth_^y>U&DaQ{H^Tx^FyDpk8?!H;8%ZX3wQtDxqxwE#Z%O|_7^MUdCkMbcOdxZ z-+Wcj-!4V@4*j0#zu$dckb;`!jKMkMVIjCeK#%|X>+GB*E+}bA)$%Nsk0)$)z+0=Y z@-KrmI3o!O%F(53oH83Ja{7s;K)UrreSHFcmtYlnA5TcC}>eCaF7!S^q6{7BWIuu`Q*pG?ik zDxugp*{c|EN)|pv<{fphr{dM{rHKAgKD~zX0ct-Sh7hMnf=d>=J{)!M?g-8Z_sw*i z8zJ6G*y6Kg`8fYeM^GbY?V96B*Z*VrdmBEEAQs6KbCy}B{Gg@gCGD;>9o+}9oS6%E zx;a$v+2syW=y}EaICmUWh1)yAaqgwTy3p))9e8-&P$M;1=};*nOJ@LRDjnwBc8l<& zFeqA#4nxWjCy`p-9U*q&LxkTjj?EvV*+Z^e;X|<(29<+hX4gQ;0NNT}8Y@#QKT15%>X%ICTM0P|y{3`nZdVPBOkB>Dc?zY^R{* z$D}HvJKd3-8k7^Wl9P){yO*9Ag4L5tLDFpk;-=BTmP4bfV-~VL3c>rq2SYfGG}vlv z_e2ny#*z`PldiLuoV9>=SKhV~~zmyH%4WFCtvyPa{7TY>PuiGLlnn*iBU@lk6wSJB8;vX>gr_ zAEL&voYl^TA}o^5dTKSP-cS%=$LAzJRR`xu&behoBRjIFKK9QBfj3_obbkPj&^Fyk z_~i1rUsmH63@2)Qe_f(q@LgbZ*;C@vi%wRs+}JP5L$Ws$I@2T@XIrpHT#oJ^YILi+ zMrS-j#(E;n4E=&LPEmP&2RI#FXc(Z$iJVy`vM=%#*4pCf4AUEo%U$iSVM;$IwMMBI zrcl<*tkfU03^s0(>HH$Jcdyd60;Q3??}c>UQQ*x5Y4kd8m~lSIr>}zReuLLLm2JAQ zgOMJJ5S*}(M{Ka8=mpZCn*m$uILU3BxU9G$XSMRN{=JaDp#$b}kRKlRr7)RZli+b` zL9!JTTjT6kM85dn|4n>a6sN&6H~R}PPdr>g#nk9jq<`wW1+_LUlB4H@wv4UfDh~+W_C#@o zziZk^Sa?>s7JgzXb4!(-F8bK7rI1@IJ3S1z$lUmLzf{07#_U8filWa;{o49y%j>^V zKit<~Pl-7%16)*brtu^wUGt(Nn@Emt7Hq$DYW+Z`108j<&x%!qq0jfdCI;m(H!f6% zg%<+id91jKPI6&c#b*3^VM#U7o{?BdbVt0!z-{+Bwkf94#~=1q6Ug2A&vt#MDjJ%# zk{(Fy^xv`j8fCfo4gecrgW2s_vAO_5dmE2OT>0KxX)WZIEe9}{Zj4NEVDUIVSm{7nv@XkkQ`VNy z3KYG+6lxO>zWyDcnP+M;sc-6f`guoarhEWr{ROjcR!d`QeqEbS!y(eJ*z2bGF8dXz z>i@loTkfU4724IIW02763)+Z9HSj*Ph#AuLlXPmj<(s5Ry|)ta?PNjB%`^Zz@|Aa} zJVR+?`{f->P_l&R2rAl8htAkYyv(B22=ft%jJ}MAP|L@q;xyh@;#19AA!G) zcMy*5!a0Mp*4)(TYc=)fWgV_khBTs<{1BcEF)(Jry9R}Umk=er9m|v*^2pn%f{90c zHc&)&asNXAf-_I)LQ>?RB6@O*$$Um}wqIL#wg*;CQDq=6&1R(A0&-w&^Uqb>Sd{$U z&Z|LBWPZ96p9xuGi)@+cd)Q}~2f-WDlaQCYUGnDcxAZjD@a9KR2Q;}QwA&9$fYh*W zv%qs&TYYOU;*|JTkWD`#9ZdO44XYq8x_YI8wdw#Wa;Ka;P2ca5oeHlW<;u@oa&KYpjl=VoBXQLN(u~ ze`oQe(x!}* zd)xRae?u9`Qk`3x7PX{*FLdS*5YMbJ4UzbyO86Ef;}y;Mq$aPbYjpLaxrvFESNW^6 z{Ik*v#eNk)t8mDV4N+GcoS_r9a!#Or?r4ID#y?h2`6cA$KjslJJu9&2BXg*}@j50o zY=4HDs8{%0n7KSUFM7a5uHxX4q6bf9#4FBt+H-}V&VOi^szsB#FuT$jUDK4EDl7s*nn8l)wUN}K9nieJ<_2tI)6b8k%R>jKKI z2_nCg_GB%Q1V*f$v8q-Uq;rt;xE~g?-JY*&oqAMH%Ou4$2xfvzId1o#K^K$0I|uq~ zx_84O7oA7L-ct4XbnBgYA?%FTwkIwcYez1Pm4lZTbY& zaN=TuyymT{U`o8$I)ZPM5iJA>?xOS2${!UF+zP7QA9o*GkR%@XngS47gYV`IaM%{u ze>3|-&ETIGfxM6seM=MOHy0B*w=G=nBs+!4I#9tmVvP=T=tG`^(PzZoiecizC`Byr z57@8>+c5yfiaZ;GdWQn4khs7!|I{=;aoZO04e9$Ar6Hc-Q;&SeM-Fa^x$!Q5XjR_UOtrdz2iZ-XLp@%bf_u@2LG>E0bWz;mgG1_`q60(momx2N7G3O!@YReTQx|zl zXOa16P-=cB$Jb1BC=vX;r12Qhymmua*3N+8TzSMgmUZ2Gu93S{MJ{I+M%gN(Yg+dH z4F6C{;-;U=gZu^F`3H_?U&QsIH*no9wz?r4CfUgi2mAA-NjvyoIPz~naLn<@i^l++ zb;34(;NiN_B=eYj5V2b4CvKiNWP<>BS}gz7l@G8mfH<0N!L^TXNpLm>ZpD%uc{ z9SuXA`#wD}R;*|%azLODo$O|=<~$DNpoWGSVBK0-^u5_b3jNFHrR~WGJTND%K!j)S z0hISo+)AQ%>5r6k-ed|*_c!EaMvJ$}=Q#i_wue_OE4g}jPdqsLJ*Y8D<>3kdne@nFj&OV31EKc~Wm^Z1u($3oeQ&7)}~7Dq=kWG*DTH>6u7eujtyL4e<2Q zDh5RlO43ydn)-;{kDgJS=%!m$`Z#11Gt<#a4JH$QVI=GH=q$j1%Ef5LzE{EI(V_#z zN>FJTEos`~%VI}LUQ(bh-CO5oK6+oUxMn82{_6VC%Fx<4M)!1fvkgWWb?F>*+va~6 zGgiM^k*Mx88og67QIeR#Y*wDk#(hP3_yuGPS*FNvcgJyRT;w(<2UUc%H=R|(nkcB1 z(KS_iZxn9TXcE1K#q|i)j!#&k;XVa`fLOoOaMmaExS|x<+TtzPA$XwNx(47 zlXK+v5|(HSKOF(D=pk7Pul1@%u_BFYdT?e|N8 zDX*IJazlM41`wrzo06kom?2xKm<=7--Ft4zz0Gt2&oCRJ!`F3gDQI5yGtl^S2B`&G z#wT5*G&1N*8zrUUX*9nPj2nyP@Wshm&R9djf)Clt0F;txWb{&hRxA7laI`dN50EbM z8^JAGHX#MD6QXfn7A9M~4KM}p|8>y#EH;=TU3iov07%*Q$|(%&uMlG`PzMes3Ic*uQCClL1 z<6Y08W_`BXou8i0}?7*c&+crhn0LNgW z1-grRW@sEHx^m?g1kCK|U{C+KL>F1&veemYzrW-_c3};{fr2)3l?Erh#jZCCT+u^g^&XwM1#J*~&d;~fEjBAw zeQ;zwy30*@2jZZka%8sZ!D!l)<9Z>QLWA$OZ0z03&<2{yh?w}Xt1&@7^yaG!GM-!} zst<5AvpHB`sN3AL50|`_RpfXoM9uB^Q9-mQT+~|9GFyP}OL!}aW_Guow(Ygi6gp~l z)kyh|7l>zmAzKX?_cXbM{4U`tDEAQ&%cBZ_qIEiDWg3&p7+bW|j*fRj0xziKg`l*&Rb4^x~v6%pGZG#|7L2Q>-)=5J|_o+&&O8v&=pcZm2PsdG)=uX>5 zH**6sFx=BUq4Oi3i<#Cr$OP}vs*{oSlUc1Zf8NdQ03l4_(-yBk+=UA`nfN9b*x_8g zbOFb0m(OQ1vYP-fwdF}$8r|qMxJ}32qlu0fWfM293<~>Y#7bYMrmH~t!my8mRhKJ; z!-}O3QvD=Y_n{9dPL~aJL6<*4-eqoF%x#QhN4J_7rh5|ToxhVtzm&!Rqmj1icZ*~9 z9*mSXZ!N&fw1+(;DUvD~wl?#d1YvnQ@?0a=V}$6Xwr#rVY{15~x5e>k{L58ZLRc+l zN|DtyA-V-Y>N_w%3}q(XGNsoo_G^y2bwu8ZL_8Fy%rA%|QOnF*Rz%Eev zVdJJ0gEJi^$g0(!w-hzFd3r;)k%60z{Xxz&rqOC=18%f>Aq3v`%)EzI@~1$W131M0 z9fUx1;}&;7R(OUqT2&e!FWlvWSg>Av;0n_GTdP?#$4h7dUBep6eXupha8Sv84z>e~ z!>_-o!mQ1rWeSwu0 zAmq}Pv2gLkxd&4KUt0(|g85WNA~2R#P2d_oo*ck&^tHaNHXcJSG5V9O} zlaRYn>el4ug9gx$Gww-mY(hyH6TGMnKE15jO#r7EUJ(b(@Q4l6^|dmr4x0s*I8OqK zUIff!>~>U^gEjzKq6TI=tTuxp##Eic$p&!H-E?HwLu$q*?BmTbp!|GBYGi`01E|==y8r zZdbZLl2z}m{6>dlcJ_0^LRs20NtKo{QF-DjhtI5TY0J6G8wcBV1(bi0_WHjm(s^ma^j3 zYM7)xkZSu2GT&et3eS9Wk>eT#JZgpoFK!|uv+C$J=d8LBLXwhXv~tSra%?@uo?xnq zXwo3O$&phlTdT-j7m}RJ3z1{lNK;w`H3+DM z(?0)GE*v0P*JS_n+{<>JSO<(31-Jv#b#bhI$jZr{JAY>kzsKAy!09Rvni}l-3>i+F zRqy(ivG}#t+d*SC=8Q<0hctF0S#DDS2SPj}>9?=CR73g!S(Vkf7641PVR-p`U?ZpL z9GJro50}mU9bXRdq|UO0r1atAYuBJF(B{a7EKbvRS26J0xygIi%PXl|k3bRX;HLdR zK}<1N)QNq>)*vWTcvr65PZYr2I53l^C`q9x+*`TA4PlqXr;vU2q5)!IKsOo=@+EJY zaTt`3y4ivAxdo!ghgT%Y!NqhM)uK)c$nkw>oPB|PMO9K69-vU6+MWR)$0-0+P#4_B zb!#VGy4<_At^kll=}{LGjmYj)93o^b1J_Qp$ zUHR-k!>}vJ7A}Hrle=4V%VN+_v7)5`Cc>zT%{#sCf0*TfM)@S~s>uTT`PxtW{~bF1 z=a{*i|2X)H8E*E}{l9(iXUJ;f`q991(#(gSftx0F|CW{_fKFxf?I*h#17XADs|1v& zdXL$l*b@i^LT#$AL;Rj2PBhy^M04Qn1Arg;Pv%8$9>+KX?;@i}YGN8bG3gST^%+u7 z!Nqwd;OS+g>J0G=>j$Z!Sg8Z>GyTYD^kZ8Wyq&F1e5TgPD z7Pq^#plPI7b;VIS0<|@xR*rVV6w7D=F#rB5u8i@WY;xLT5UmEtcOPQtwy;JK7}bae z85gyUr36*S0F=mDfG~@-`Z6dj39F?^%CgvB0goX$JN)5EA$=8Ufm+T-G$0og4@bH& z^pzm-h%V9*U0#h=)b`9gH~$xWgfEM23XCkf3_tVb1q@dQvub3a+E8gfv9J**-X<_q zX|cc1E!$n4P! zUR-xf)SPmNb3|JEXpW$})+)os$Ee}vztBHWzqgyqByu_hS|`zc_!gf<|L`3Ibzmbj zK0P~U-GU(43~iW^KjB?FdxEpLWjb0!?labZzdP2*V!o8qitxuVvlp{VCnf zc*RIqF}FbP$ljLh1=@D~#IZAX5^JA!=q9gQ%$@7GeyApd-pfS)6GO!iih_I8mT)Us`sI6uvhP z9)eN>bjq0y(6=+6kSN%#sv;KGBgqXH)U`Y;e7<*_DRcgp_wJMU-S2lBUNG#f(G>eM zJOI>jH&4H&E?6BtwKP#4R47y!o9a{zKhKkeGyP`O`&oDz;->hjy(q4-kIEy4bp0p` zJ1F+H!>L^r00Evo491e^fVv60r7~{sPs&kd1z<4F%k-9vimtNdiDB^=$aUmYOpT2O zIJy9M2ZqZD^2=)DSa4f^paBRvtL27?sK^ahE^o-dSO5u9080Im4dUn_)}p`?-Aj;f zFh}mak?;jU**CHcf}XJH**nncT3&95lBKfNw5#n3K$c5c=HH0e{A2w7-W~jYqF}JL zf?wx+G8Op2%9QDaU5swk+6szJpC2~#1%~aHoh;#gcq0g(rQLm zXjj5sGCcT|qp^Pb>Ekyjk0*}6IlT;OFz}Z(Ltbgap1;sQv36;KahLCsMy|%MjUv>< zJ+JRNVvrpvI#HtsVA^WnXOg8Q<5Vn4!z4O@{vqOP7*-AMXHWTuq>ydBuK^NzEiee6 z@>BaW6K#Yg99KVeQoAc&3iUd&_J^`GsX=0^5>8_cXPVA>WMN&tMeSNSLj&?9@TfVa zeaZ09-90M4fln_g$dJJ?XHTcR)pBZmUZ>M3A+<-&lMd!-n2+6VU2y4J(Fq>g?UDeN z?RE8e-o?c-fjt`Kr{;BT_f9M{ZdO-e2SA^`2a@0m(U_CCJ$cTd0711Zz;=d^DBdELiMhe9UMrY5;)zD`IY~b2 zS@wI`j1!@mA#Drz>~=pWFN*dF%FG6I7_f!~$Wr~@GrOb{;rjaI>qtP@$f5Yl(?(3! z#@@ofU4SP9w%nM16iRz#x#PcECQaFwDyi#x2=MH^B2OAzqy42w3dU_9Xj72@NAA@# z4HJOH64w}QLuIb5x+s>$L!l@7CX#?IC}u;3n5Dzkj}q5%vvbTTrcr+Q6#zMl0Y(PY zpp44z*N`iXJb0hC)GbtliZpShLFyTiL3E}=F;@ZJ#RI&ZKp+P19jX=CZlkhr?S6EnLogYs`-M_<2L%1jz7Nb*cu zh62Pa4ZpL@OcvCb`wZ4O@bD+`=^&m`N^>$YCYH>@w9vXhvUbMf0UXtC;8MUs8{vug zSisV-G!4Xa$qEv3Tlhx;aw=I9_=F;v^`vP7*Zn-q2O}_i8k3If+@i#Su18kf1%~l^ zdUkgu@}Sr`yK@adkdsP6zKPgbe!SKOV?l?RgC5d!|`UAsbVUgnNl4M|01 znGOZpKo*k`)9N*f5jW><(CNnzfjaSAc0@pnb2l3F3QnOxl{bDw`f*mkOkg*|eT=0w zgSH*JWuMn++}hpIcAWHjNw69~cz}M3;!f8sG=3}^=+eP4&r+cw4zT)Z6)2ja%BGs( zaSPD|v{i7s{eg|OoKE~z5UY1q_>k~uPEyp+^!s}V8qEC<&!Gg)uvYr&E*^U-* ztSaq3e4JfF{vILR!);9sDZN|EX;E1;;88kaqFpe&1~XVu@7gD%Dr>enFe%Z2Wukeh z+_AM%#_#eJjiL+lqSs}HSM+vdGf8%@vRz#7S$_U&9d>;8OD*fQSMKFKn26a>LP;%2 zIJv|OF~#hdbXNJ~aL*@sxoWJ3C`rzu#01I4<)j4Z<(+G47qSoT%d zYW0l8GI|3hP>8?r*5JGk8TcFej-NKmo@LDs-5wnvNsR+>%|M|^r@QPn+i61*H#I|W zzV@0;lv~)j_kkC-yQJ3^leH`b`6c|kiaONO?pUoVy0^Ukb%18~yGEj3iuVl406Ya^ zIlNG1@uZ4c#{)|7aurp;zdZ0D^^c>mZRHf;tKEy3ElCIg#ze%I%LA%*bs-0wukaop zQQ9MarEW0EJIcY~B{={n$iNX{|HJdrnh3EqFeIiL&c`9gE@y$BJb6fis<44Bj!k?C zV0iFcwurZlIk)?7M^j^N_1(Ig_l*?Tf%YM02KzMyNi<7@FcOp)Ly3IDqlvQYZXZ;Y z%FQd4^e(?Qet*V6$q8z5C6SZti39Bu=`Jd-;|U9&#}maih$m0I^%C#fqDCkaMx6<@K4sI_%Be-{b?B zUf2qg%~4(ur)tt%gN8VHndu2F0UTx$a4$;EWnp!GK=8`0>I_smaq{#EL-NZ$Tk>lk zy1ZH7{V2e*OjJ&(`~}X&6OjG{a-Tfx@Mb*2M&QlFK(~*MIAJ~#w00z1s*+6)CY?oI zImbdz(>kO1g`Xfv??#Dw&zMT3vnz_G#jgM`?^_D@Q z&&{*xt-4&aZgK)sB0k}wQc@|PBo>V}Ccd%Py@(FNOI_7w_4<*0~x@d+o2C_rZ#%FK{G@%X;AI|t zcXe~M_+*AN;0-(xcVxG2%Gl3@UuPwv`rbY8DH#S}V|x}vsH1lYtJsZ8Z2~ ziq@H-?a03Kuw?lR^j@g;ScDoNdwL9$Pj^0<&44h-=tK=e%k@Y{Dj<3F>%*(@syoEA z<$W3c3iUcD1D{sGr~d#9mQP~yL_h6%zB>pI|JJgUZ|HiCWIb%#18kw@EkmCFGHVQz z6#!cMwhs8<@T8YVYqc(l^u-~EzIYa~+hUp7$xP9TT0B5U8L&^hMoSP6O=}r=JfE3W z&#O+}Ts^^_ZQitye0ok7KL^asXwnLJLLRye_jt^Q* zqJZ)DHLwFnX|L_Vx#Ls4ose{}R+EAnWXd-1x{{H_LcmyAIB+Rj7Wmjh znBCQx$mWI%<|cYf_pIyR0_Ys=y5+~E&x(ZIX-&62Uka??&Nq3}HyxJ9F3_UrKkQIy#+<>&MRhl2;c$+uD7p#uJDKG; zVnn0B1;;d&MK~~KTdoEP*gGb?3u5g`dqb0GHN`@2`fmRWYfOR|55EE&xeibiFz6dR zMKy6&irg`;{Ov)&eH+%wo@kcgYck*Db|fQLN$`dv2QN`T;gWDNF#wK39&QN&bGeF> zao0;$1`N^x0Pmda!k*6zE1VsV3io|oy+1@bG$@&MFWGzX@;8|QhNdtF4{lHDNX)bT z#6m{*J@QVRVCzlE*fbw+&#cuFG+1~#b5jqrWJ5}2w487)gxrOXC(cLvV_2Qe3*f58 z8UVqF)nU|WMREab1&F;-^w0@Du-g44(`doxNiXd_1SA!p%*YbIT3B!Iy+76HZ1&G% zD5apc=Nlfbt2NNH&b%1~XwqB9AxML1F=b`O>g8e}cQnDkkIY^-R}cmO<)UxlZM0{{2rSN7)#9&n1M78 z)jf{_+NQ7EUWz+?c7X?nsHctN6*fxAfs>v=ylPqN)r1;QaC>vjo*iGOknH8`0~wn^ ztK&J)6akb`&BE*IM|~E5MaY56#@a^}9o@0afIRSlq`WR+7-4y_`zg4>RL? zqQNzRMpsSE3f42Pbv0B+x-`bK0thJOIKdugUq*zc85vm#c5?~*9$_|IPaBy=urS?Q z@S&qVcaSE}4}NuUU6i*?fsA^_5NtMjTK{eD5#YAU2bS5;Uk5QLyZf-id2Ra@$23qd znI+8Wl_d~s;6>43;8_8_XA#9PM%Rt{{qFQWM&*5E7x(xvBJq-`LT{y+-=ZV7e< zwc!zAnV)&or&MRbQBZ?0qDA`Rr%T{XYcYU=hZSz1!@+EfWkdQ}EjGfJ4s>8g{+|)L zMkWAq#tCOG)^`WU84Bv55&?nF zelGFSV8>7%FGM=2w!etT@P^uwkl!+Ji@3!|c$L)W8F|IjH4`eyz>_%-B+d(FC7e4f zYxog*(3Q@PGdYtm=gcnZjnIRw->F39G%4ipc#atL_twlNRN-e7StrwY2G!>yX8l~g zURmu^dBZ`u!R8^MdQLIwe&tg$T=t2f5bZ|lL9Z??;)kcqpOo8XhF#lnsY zjGO0VO6piO*th^qXYCr#I-(q2qB>sg!iWYQ`eQ^H#v3+K0mmaRO)fQ2BSlA^L((eM z)L@d?(woB&!Ne1|i)N5mkOZP{HbxNG`(j>8x-K^02rh(o6RyG|0C#7hc{%G26 z=M16dPZiCuogwLpMhd7}#em4XZ5@60e{k5c6Om8(t!buipXKLm5<5?moh7{dnO}VKvZdf3;hRxk)v%EaxKJbn<_0rBW62(}lgb zONIVbl+jkxxfYMtE97PPtXwz0ACtQ#$N%wEGokyr#PGdaBY&DF|9P7l8XL&2_<24# zz~FPUgMY@bF;i@fM?UL`T{%#jGA{C#Dl5p}dl!^8`;5!0-ubq3VeoZ!xqY_bY5D=M zpNPS#>+`^s{!M9+!OiscGed59h^b>a^eRlf$MdWo6}yHCP|W3ha(Tejm_}y%$GF8J zGbYn&OC(`Ky%pg1S7sYAk@dY@XdLP&vcR_?Iot3GwCB{1TStVHs)_jmDI*g|z^S_j zYxTtqqrd{9IN-U^)=m1>bMyX1t;6?|#{?ah#Ub6bv$fO@vGIvGaNLoFL>f5tAmM;3 z7ISR=WEUsm(Kp?sFFft`*8^9*7&}obJt`DZ-r+>=(nR1;Kid?>fl{)T+E0aRhBvz3dyG;Och=Yl)jA_v0~yqpY!Jd(>rm zk&y7ISu3rLLcYp>pytFK%$UEtQ=oQ;m)sLRLvAbbaeZKyn3{CRyo0)<%EWss2Z*#Wd|U#Qk!`e=k{ zsMH4XEo%+V^;G^t3=-t>aF z>tzP8OI!<%L(#TZrVBQvws#+G4wZWwK(9ubq9y=iAjr|1S*qp&tPxLVpF$p`We$3(NjuE9-|;bCDbsf_fLy%%L8LF}Xe}xS z?j>GJ6Lu%_SbaUPFI6cTl(G*$dQ;K-voHxOx5o)W452NP31aNedtP0<>>_vjzUHe|BpW z>g^VXidP}9<;t!nJ4Bl@B6{QwUt~S+-Q=1P+RhchvrLd_qgLAw7ew1h-JPKSr@C_w zYwAw-{n@)`+FfgPEaH{RZk-k+jFwtz2@qyi5sL~&asdTWK%|8h1Pn0{!qaKh3QB4v zib}``TF43{0;XI7q*W0Kn2gE-L4hiSXhO)4kYExL&bxxW&g{1P>^aY~Pygw|#Sj+j z@_T>p=lgkCJPY?y1e%h08u6!Vrc9SH#g zHx*ZSG;umB{5x~(B2jN?t0F$`E6NqO8n^_vk}a=sVXZ^#%}8FXlu%FCauK)b%2@Kq zfTv`TCdgi=sPl(Q>>i^QH!VRrx5t$9WH5f)c>I{$+zWS z24=MH->mBqXH8PDpl2)iAno|w*}g_$ne?K$=4mab^^0YYP;XfSt&t#aD3ue_M5YxY zxX?>R2E=%sW>ruHM40vy2KMDwd}fDmap$r)Vef|!T>L|?5I(NvmnB@pC;HYphm(^9ZGyz|l ztIhgp{UbTy%gOj5Yr0HCvGRxWLI;L%YFPzV9=2}_RA1hGTC8*BnrL10+G*sPKQ*t3 z^^tg8Y{T-$#kue$dMGtDddQ=+)1T)7E~7K}Hman);9>-HdO9G<7RCch1(Sx! zN;G&Al_=u|Q#8W%p?D-x@?4rkwyTO-#o*o0src?2fDwXDe;V6Ii#?LcVe}(cy89JDn7{_TFk`x7Au#TP@Z)PlUWMUXWVuChD08 z-|?ET8-C&!aO00M5n?<6tBDi*M{REsA2vlTUOd01-xIPY_+X|x04e*q|7!);@;*V>N8;YB|;HjdRxRWBe>;Jda;1sK51zPre?))3qq-R6g9YQbk-BVBdd!JNajT1#PD)H8WXG9OaJmI z6nwff;(2GP)MTpE+2GQqiIHnzQ{pRy zZBre_1efp1IXpClk*;jWH}+~{Y$+SH8xODhAphl1z*=6PEb=IOA;3+@S~flZ9=QmhCQUJ+BH?P zNCB)$Y8mvjx5a}p1G(q&7pp&LB)rr|dkoO79Vfdz1U*MzGet|4+>;@Cv(g;fMJGF4 z<8(2+f_~0rZ5U=Q;9N8a9a8q?$s1`RGc_*<`f}uS3M2gc*5^*pBoGR-!P=uKUS{pcqN(X5eCm)F#-Zh1xy8?zcnRW%pv)3vww7MAVOuqNwEBN02#b;9d zNXs@R#Fa!@!ehHlMBiOxrE|5u@FMH!Qkh5TlS-@9nd`1&YxZ_r(YZ_Ox&GXT5$HncW_L1pDa;Jr`29Pd303p1ESv+L1QzBfi_(Gyth9>1$fNCJIR5oo_E2-=Mc#({%=Dmso(*cN?{lD+DfnKg}a|0#w<$z)wbip5T{O((DPg5OFw`0f95~#8aL|Z-qRibuXP$> zf6}D<{TnoNZ_RkfO(|bLPcA1^F{>=G%$+_JY!59K`vl#TUg45b2G&i>uTah7?&_oh zJ?pg0QbhSF^43OuvpD8H#f~Y7oF=_z^R*f*{-X0h}oJU-Q2{f;Z<}|j>ofv`3 z%{b8NTa`DrS>oEX-scmjiGd?dtSF5_DMk=clm3_s?vOwn8?I4?0=!=k-Ugo$RLMh3 z$3a@34>@i;58J!hS)Qq}w0?r;pJ&a~yQx-kNAd|c(%b?uIT+#?sSRcFPRo0X+PBad zhQi@-8BYqR8jjqI&MDe>&i&@+Lgvuythmb}Yvpnj_XA{IpJ|PCsY!AdxsufkUC@V=aGRcaielu9jF))mQc0_p^_e?N%`4UyOv!-Qz zlJpPyka7YIEk7g6?Tc`vMvjK>xF#Tk90wR0%b!NgeO@Z`9FP3v_-W1l`5F$;B;K6? zVW6i_Xih@x;wLwTOj}Y2C?T#%8BXhttL3oBE=(r?+QEs zzdC;2EjFzEllZaKm^!rm6P)M1pylHa@CJ9Nd_M!g*W|uew(vjG7D;8kKWRRQ`|tM{%-s|-Zr8ITv)9xGmCErRE{>AHAr<9L|VxCV**Q! znR-Cc5%AT%m$~|yYXM(n&F!`ACE4!z!wY{5=_QH{1X)KezQ<2n!zb3&49CayOd5|! z{;WFTZh^>0-k_jX0zM6V1aowk*1FOXOy1kjtjPioGcaj+d zuUYS~Tl1xCUXYEIuw%1|aU=K1z|O79HvUyR=6$zwm2BMYnc(Tn6HQmkc>Q&0ii97Wr%i@S!e4J-y0~=U z_YbAc0hb9F^Z=_YHRN~ddw6J7x~Tqw9seRj0Oe-qXXcn!HMfpF*v%}K!!U*u9Sj1k z^{s<#?~uA}FO%C7(m;vI7!J8rj!?Ze775xTfO=r)JzpzK-XF3FrM3mO)kpX3844M4 zxYy&4Wk*N~RDr}Ezr`H~LTHea=-`rZNPYk5@b08x(uy1z{k(Eq49x^KhjRU&V6b)f=gX9OMPlx4G4w= zyX>dVPQ9WE-|2(1N9x_3O_Z1fbs0r#@IQ%Ftd!i9sf5@loqK}+eCzTdRV;!I`UYVM3OJRfbmx6O34 zm8)(zZ^hx;@NL2}1>xTS?3LT2cz?W-8rx2}L_?{BmzXf6LJAw+<@^V3z++VRK?njS z58NsTkP)wD;HPT%02Cj!Y_{dVy%}2pSCywRsZ96!xesOgHyS{OWFL!5!{A7wM1iG%XaLh9*Nra=KpUS&sL?0 zsoTYA&f%?n2CG)Hp*1Yb0~yjXXR9#|a;%QW6B#wI+K-$mP}%Rm0D^svqwx?X7g)ov zZz};8r=j6&#H0p}rWlOPw>en>=96j{RXx88fZ#wS|Fid%$22USarHnsu6YrddZKh^ zh*)tJ4ffTp-1q?6HPj)+08Rk(o))cJHil?#bx9;?%gYy=kD33r0(72Ef&i$(NZSA; zUp|>XAp`cMacZYCbFc$Af&XH(e|E6LR_7)g#|8~-+;<==)17$&x}x^}z^Na1KQUhQ zmwTTwYxnKuEVC=NasL*uV;>T5WkD&uI4upl*vV}?u7%WwuPK{L5BalQ!9vPiW9%9W zWI?3%O2`vfb`RkaU*%^DK_)f|XJvy-6NiorVNYogBU{)0kdAFUY(${Hd^B;3*UVAN zzeg8boGYZ6#H>-QVal*A%;Q<%qCQ|t$rFv?Cm@9*F!xj|WZJMAQ1b2#76Xa|UWT}l z&$2#$pAerd0TKc{Qq@@vEyBM+EO1yn2(O3!JK6pL;HIJZpUbJ$gEy|Ou*HbegXG<* z$9-rhzWh#CGO`NXuEcy8UePO_rkBhaU|)K^6^eHkey+MzE!ir4Zw zjt?oRbPurC>c-|*I+L~|7}cvETkWOyocu`pXyI1U=w_gNLELLX!IyN)W4l-9XVwj1 znS+r9yjXueG7Q!Er^XZ26cx#W^u;_*xgWrf2+qEpg158|GaV!wkjbB168FW3xldB( zhT!g?IL(Zp6I4{oVPxa{8zb638mgsNctimI%;T~l_T?;mOP}tj^e%#!^>P!(Ji2NN zSaDO|z-(>h-V~5Av}pq2-{v&nrgK@dt#Wu7|~jp`t9PCvHJVCEI-hn_=-A zTBxuxE$0(0D|z^kbLxkZ^FeWlZ-=8_i$)w`Ud0>S2V1!5zwi zBZ+T>I0a1;vz()m?>S4)-u2tq(e&nBcYdd&s|nb&hF@=qenhx662owK1kPZ*3Dk4{ zbiFTol=U}$hR%`C#G1M4{BLa@^grLRzcM*72iB&E!|s3IU48#e>*kvUhlO7sesc`a z4kxMnQ>oIgeij59bzY7913LDc<@mnyVkp+W6i;)p~~& zG55@1HNVrTnH^xYn8ag8Hz*()&H9y@f5u{Cos7}mJ;I>{^M_)}%94B2oYrS7A#DSV z>G)Ru0!PB;`JodD1XIe#`4#0@yAK{NLsqcaZhETb8?dtmRuZW@C3mIrqmTOb1;A=Z zt9c1MTk5@UKMxA&Mb?;N7l^V~Y0iDDe&^+w`+r(|2+Y>`u(fpv(2|PbFUc?{?RZuW zY));RaXe^QH_*Pdkx|lk$1{z1W60=-3>J&Cd`2F)iIZMtD4v_y48e2?I445ng7GL6 z*VG5@!J}Od4zFIZd^ue#@;wVH`KQHn9jr68L!mt7&UVa6d<-ZtzAKDw z=tn%{8=M#XDC_m-Xq7F2%#QMvw^*|uwzd}yIlPpeZNmu!eb3Wi`YaIbC*ulM%_igk zK6oR&k9u?L7b3HOX_|UUl@guYUWfbHLqwi7C5B`cP(M}C9j{EYBEn7)9&azXWj_JM2ClM|F%a1a)%^g^Dx_;Rbxw{wG$O`_Fw7Tu-X;nAYGDw z7Q{(5ALP9` zSY_|?7fGQ>vJr1QYF)u|2th*%tK|3inB5B-n&_tNJE%*8;L{5In4oN()EMv}ETAwV zf2sc3?BFPMJn3NYpZ&11xa~%P*Hgn_3YK^+u6y+C-XH4S8(DIY{* zSIPO-&#Fhl=e(CE5!uiIg;7*n`m_rW(Vd3uWqVSX#QR?#ECG$^-ZoqXp6;;`Yui=5E%TTDVAvF#PPGIwt?=7FjmTa9H5&`^mh$}FoX@EVt_dlqZ! zwkp5@fSF?}qMx1F09~Jda!vd*qW2vb=g|evqdo1u3D{UXEa3`7;lx7JB8F*w8KmHd ztX#rj{LcW(5nd#{n2hyr1A%wz8gGzu8@Bkz@YC0IeyB6hw~S$NNNP=aQCF7lq=Eiy z9(Z?q!gS5y2VOfZUPFpSN6!ZT!N@kKiv(7I$C)aqt@^(K;C5eqxtlT-{$H`;r0RaK zXQlI;C5OlvZ|I@O(Db0}GkSyn?J0Nju}PyB()Lat0I7-K5gt66bpmn+LHtT9nZ$`G zFqK25%ek!t@=>VU#M$gB7o`V~i?4GEZR@kKu)2H$@yvs+2X+kCXe)KsNsMg3}0f47{|> z(8r#Obg)9-n>^}BB?zIEn+DJCu^UmA!v=n-5;c--jM=B4FL@4U6F=cO^nYVHUl&se z7sK_=uQM+o+Y3FP*1AHY;4`c?>kp8f&^3Rdy;P7$K(XBehW#^gf!dO-VlAQ0OY=bF z!ltBTa!;ZwpLWqiap2I|FnBe|*UPXSer1p|AR#*1nZ;NA7urq?b6p_b(R0%HPhfak zCd&K})m=C=#=ndRsK(~{g3aUx?V8fnr~r7<<^qc1NKLu2Y1b$3r8l0|$ZVE3C<@L2 z1(HiO@f?Q-M~^C(*WYP4$+##pj94sDI+~?NPEhb*4<3F|S7@c@LV2|oZ(BX!fom={ zh+=fwz4-n%TPlF}DzI+*Nh?CT$$feKhY~wjKbOA91MZ&fh&6)0K`83aa zoMa^026mg?BDF=C_E)sJ*trEWtYa8P3{f5qG$kOYFh9Sg?hd3(=YLEDj1`*RnTB>Y zJ(Oh;7`E+R{C-ruji|#~%U9ZWr?gQX!j8iDXtgo9JPPa3wx8D=m&3dvB%4$K#W%0| zNEA$=Nx8^hauY_;*oAIi&P%0NNuE(YXA8Ag+M5m&fjl{j87R3a`-5$8-zUjX)|$e$ z7oN^taKZ9_9NfSDe0WPe?b%e%pU-(AZWIpwA5M+DfMCievYYFP4yOuufq)g}#BD+2G{bp1Go~`?oC>bl9rNy` zxW`M*2R);$3{nA{4PEhteOsDgpPyUVs!(tH*x7--q2VEss`Iau0N8tf;~=z$NlZBd zSl{zlVrTQXVAa+9WsvVtNu4%hxTY7W@prv`TfCH>On6qk(z#XrB=D)yJJJ~)E&Xn4 z$^a{eeP+n=!tejDO3nFZCeXKG`7wW<`#pzi;U8$&;+!7i99|9%`kAo|c2)729%e0)pm?__Yl?-M@-b*Ft^Tx)*ydK9#Zf9N59 z5An8z&*Qj%|EgZ^Zv{m#^qs@B_vXGCT>ZS#ByVWDy7H9=&BMA6fBI4x!k3dVqIlh- z>5#u1w=yTj2$7Gcn=9EK_l8m$9nZHJj&0!o=#AyKz-$&HLm4v2>*$IppD+W5a){a`;|~>@`+7&dilq4rn9W_}ekdF776)yq`Ohez$mQgnOd>H?wdlb$_)!{ZPf6Y(M3+^;6pN1yB#n$P9i@o4ge?l z=n5a9R-?$B!P^6#O9TD@`xIi>@Swa}?72;@x#XpRo;=fSQ!@Rv89e(`gAmax+7Fs* zIE7)(SDd&IF)dErpv2zw6*tcLzbdmzz}~y(7V1EUk>OzO=oL3sttLY2nwK-CHedr= zbmkBy0qstC5YXy~esS%3d;NtL$KIPPYc9GS8}v7TX^&AsBQ~aG7uNs8(aj=-Oo+|= z3#r9595eWRr3p(r-AU;cgk{kA-ZJOJW0V~u79n;YG7Jlu$8ooWnxuwuUOUwcb2<)d zVZ(BZHXLPhefw2@Bou}!B+7QD)VU|fc*B%#6OT?(7d~7eoDDyvHQX(Py;aZTOmltg z>Hg}VX@q@ZvPRtJZNytv7|(hufySt4LF1v0Bo|0?n8!LS?EGY2he)2=90Zs}I8;-J zBj&!Vg?zIxV!D$hl1RFfs?4@*MO zqGVKdd(uKNL)%kos_K25mpliQE0ske0~R;T1@>3`qFs)kbqurtI? zRit2t;*_SV&J_^fI(b-3kTDiHl~gU)r%EDA#JPKiQ}2`EiC%R;Rpw4t@=dZV5wb5- zYL}XZ%FC#x!GZ1rjDiw0^P|RsuTqvVYNxY)=@+(xta_8wci$EIm=uO7MqwdMe#xMV z@(}Q#tmU~UG77OSsLPe_rYk06vs+$rCN$-#!Nu6Q8S2L|uHs@Mr10n}O+(W*^eo== zBzHH@!O#s6B@Fd!pk`W@rmv#YPZ0^=ZF1d1q!eU{(|Lm@gSpy%xb{xdTRfC1tMz49 z)xl%IT;PH9pCav5X0^52^ZI_;>DstfMIx?c7as+4hBfCV#tCn!laB!S6n|AR_wk=GzCxQclhv=PN-WnXM6x646Zm^1HNeVuCTjLRK2dS zxumfY(RbxjhZ{O#Xd&&ECPku`ep^m$`z?_1ij^C5Nrv@AFK-3oR&aUm;KSU%~&MMeLR&QmZ zPgrepUsBU#H9};k$kOwx`jHm9{;pIw**y7nQsbO^9{&s8AMf;YGMKmM)VmuID*N(y z>Dt0tZ(&F?wMsO|u>&_W#pLbpnu_{}bKckST$jhGylxG7 zu__5P3}@4O_D)48x5LawUyXZ5MOq&jE@pRls-t`JNd@Gq9@uc1FMUO0{%{iqDA|vs zVsK3{N3(kX;-z1s!+r65$?~hSZ-5H>y3IlH8HKHKT8YPvrPXy@HV`qCm1d7}2s3$srEd&d_h=f{!r zFS8HLt9vUE>-+Wv$tib(=WD>0dX7NaA0FJ<>w!98nqk~PXYY`3&xo!c#RN!V?|XTw zS?cB6bJN3-+H1abuOa#;0M6oko*rs){{SrX;Mc;zg9UGjiCC(R#+*iN2Kr^f#Txx% zd!QY@K9X;iQ6-G?(k)DeB; zFh@PCWmnW%VTgoEt~AesyQHO?$n1+!Ep_;$6VW~7(f^lrwp&ymXJ&K0z3C4%$E`I_ zPWXh}=B^@9MuO+$4)QvyG><%}%V-9${nNrAZpNSqr|YYvC{p{KpK0|icy;4#oaPq9 zJmHahb}InlLKEZ{4$Egxw#5S0%g{6HL&mqzr76b`NK$54C2*o*J<+93A@JH?IaRd>hRFg5hSt7mWX22iUh>W#j@)S^ z=5c2;_}Vc(m*PQNCJWgYd8w!zw>m5+(7@_4vy$8=Iv&7>wdU*M%IGm!+u;(9Rk>ehPC8RH8zukF`HOw9D}@vf*(#^;m#p$Gkc6ml>x<6M zJC2a7l7b3Zk#mB~RujHiiL~Tpm)Ay?@4-F@e4l;_)BURHB^mOEae`cltTr$&C4d-f z5*h|*I;Cc!%1YPR@r7i9Havu+qLCZds|R+YTjG1yv?vmVCRp#}3_U)hI$#BCSICOT zMRw$nbh3I{3EqjZ3+W=c_yRJ^Z2V{dBqkWa+_9qvuVRSUU8j8d<>oq6V%E|V) z)Wv3mjhybz2CIeW{}1|?vC|ySbbnr3abLbM*`qHxAbiN$6Xjoxtk-If^favSaWcBZ zT_)nOLlUA06e#)j&zZ|MC1)=0Lt4q1BQ=auSG8Ng=j)@qyodk1tpW-HWB$!6XMJH1 zS+g^3mqZWZHVA7Nj7Vj-LHIV{$)%lsF8wX-?8_uD*Dr&*lAEzB0Ow(YKS2q~$fGdh zLb|(Cn~RM;kna(}2`aQ!wzF>pan83$Vie@ejL8v`pA$Rg^v+qQc1Rt?O^v|RZXD@9 zQy;{E@BI{3@?;4-$2ufXPb7F;-yTz@V#cstfpPwv>5&als}n)%yF24=`D5#T`U>?5 zkzcw7LZ+d0xQA^)1zu>=jLnAhe-g)oO{|8Trz54(F literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/compare_result_pkl_md5.png.png b/debug/accuracy_tools/msprobe/docs/img/compare_result_pkl_md5.png.png new file mode 100644 index 0000000000000000000000000000000000000000..81ba1935e69218467b006f05dfffbe54f3f04cb4 GIT binary patch literal 20464 zcmd?RXH=8>yC!T$MPLgGNL4`)X(CNp5D^e*QWWXZqz0t71Z*^EQlvwqHz6Ppq(-Ip zA`nP`C@nw;AwUu!giO?Z{%4+7ffzf%V&OdpQk~r_ zWIfhs;&J-z7TDeae`e8Wui~_zAp%W1B*QUbhuaY0j~FrKP`zRcR{lp*$BOte- zA1&R}w)t5_vPkZnfE*~(H!0KAyD^p0D4tJV3gUKNUs0?;hKk8qWt0xxEKzFlXA)Gj~&H{ma5(%PDFUzA|wa&EFNyvZdiQuP_<-^EolIsdyn<#M%{h zp=+@lm$WEdCeWsd7p>3vFf5>Gq3b?drIsUOPbSQr!0Ns$ep9 zmJ=e#XXRvt8ty9As-SUSpB9ffnHSm~!@#{gNPZv5InCGgrgC&cVDwvdEw42{E#%AR zO_4THePt~kNq61`Z@y{ErdcHH^lG|gAHr^@%{7i%1%eN`TICn?3Y)&Ay0(-cy%|Oo zC!QcSdC8;clITt^2(HFD{`oq}yxmXgrn~yW-0y&Xfo~tKxWOGtN&sEmDb}8hVj?Af zDdwe?JU-~?JwS)vMJ!4s%P0w-?eFavyqLZTSz)T;h~>4qUWQQ7yIQ3C)IvFuao=ui z%m$4xpH45pT8_io=Vn-bD%8lsJ%WO~XcuW+j^81P!85vWaO*U&gBA`*!O1NZ-+r&$ z#-yEOBQ7uAGYCX`zUMqYRWnsJ)8rwCg^uxDG?m8nOT3u-H96>!pZRf zcV3u_ZUswTYv}4?tg}O(|I~aG^(u0w<7wXtcmipe-Y@SLj}}-J>x5-_iMHl(1dC9|55%?e%6Z3yNkxsf0KXQJ2}Qdz*E=>mV6TmTB6bbl)Bh zbgZ%KR;?E*ICtn(rA7}eNRZap+V<~8S14tdPGJ3$>)c#DKfE7@rvy*~h#dR-or;LM zN)$K7?+tv8f(xTxz0;N4uLdmpu zBLKG;F`hS4c2x`fteyr=1~?556rYb^)#__Mv~AHLJB?oTbR z*J)*!@_4mu65j$wcSU+QZ(uTEf!P~vT&R-gqiy+mJuBcMB;QEoc8tB7N!qtx<{JWL z&Z~^HuUbb1)F?H44%%xLLdL1S zw?0m#k|yZcPAw7GHne@+E^6Rvkn$~pExypN+^Z_D5&CNX%e_7{ufW2?*u+hG?1x=< z`!ee1;bKu60G--#fSrc!2Yk{2XoHvl6IQh@HR=^gTJH=k9aQR`N(dsbyT z`U)CEPrO8&{TnG;JN2-VMYLGOv{Na7`)m{EWGquGqeS!T+XRs~w&C;LF(VN@d?Kbb z8o-LxW&oOb*3|oBEcHr8>$Mzi^Psm>1K$T{`ZfGOcd#Z4_tovyi`Av)tj;|ngM!g# z5{C%HRC+B+gy2N1$m+nB3K(|&ew)gTwuynsDBy^lRCLV4_O~ialt0ZjJ_2G&1n%zj zMrLDkIhEcbxRXDIayuNe-^w%_v@|sd=q}Vc-7n*=)ue?DSDv5 zVAn$GTckR%y#8lg_nu8#^Y;$ZdX9~ZH#}&K;WeLCm(b#f~euqVQULCg!k{92r;wh`Q9=rAy+5p;< zh(B;goW>5wy2l1g_!mLKU}0O*A6na@fzEN=5x=Y;4*YF}AGddd9$_LNU-eH7+_Qb2 z^eFQ_OelR;&N=x)BLIR;-#Kll(nmaN&v>n}$$~W57eJEzv7$(Ju0Xm?iOQ)2Q!4>X z`_3E2A_>`KqKh9U#W!)=7OtZ>b+R&RRJlx09Df?;@h3|nIip3VuhNr5jueoe=(MWS z-nLx(b%I!qR*7Z&o>yeFyhxt}<^#_ouzu2DHtyou$63r4c7sWZV!Um#v9qQH;Ox?s zkCWH-^Z<=UYw*SoT}8qYWV1@+IBBLLxFLw8i!#H~$=vujknHiRg_5SNqiU-4CekJd zRm-T6unoL9TDgYFTucb4NxeZ&j(#bo-=(Ab+i=Cc{J}Z@Ted=f{k-bomgFd`^6XRf zLHp8jN`^yEBl1K(>XfQlF6Q<#Z@aMhE`xGKA(gwg6p~4fS7kVKI8GlkPd1@^msAVG z`ZeI*Z>i8mn$=g`?zmQ9t}6zJAK{$4l$C2lB?2b=lHyqR?o|uyRHld21V^eGFQT@( zgJu;PXN?lZc`1Fcl7Q!9K*ii@Sd_ocJ3*3O&;Vg9eCkgU8D*(Jr-Dhpq6di{OPy+U zNCMCX3A0hOJdN7s?j)QaW03>O_S< zd8XApToEC6FmvLn4ajBaRp_DOrL$j$Vtt=UsD`tlysb9f^q(Zr!8zlpNVyJ=%-ErHfHd?`SW4=vVoTs?(-n91+@#%>CZx!^ONBxJj#+g((1#$|%`G3HTK94z#Y*I(O z>OZo3XfZ9X85M)K@2@#Zqe)n(s-KuIV(~@t%T#R}g4ZLqi9wHyT3kqKZC^Ei$Mx>@ zQ0vYJ^r@TCXqGw^=0i4Wk->}FpA7h@^YinRr|m4kPuuZ6|78Q&r;zcG+0A4&LDuMd zR}=65=Rd2>tmsYDk@KviqrhRtHU_5o`I#SKoG$}ckLQpW;;hz5Rc7@RW3SK{()aZ%F%dOq1>muzwnOEnpr~aG_F>~uDuTQXttJ-%$vnTl?&BFYDmG0 zRM-`ZH^Dmm3=E8BcXAjoyr<9~hlc=Wo15LNK0iN*baVKud#7V>_4upai9gyVpFK{N zU0cBA?$=?fuC~eGcIW+HYXT7+i7uF1tU8hi6;3k$37Tkj53QhsaFr@UxD_zYD`~&Z z#UE2_jg8;xl5bt8NdbnZ<=nYO5F@CZRm2&9*Ka-6hBSmm?p`(DU+hMvU#nS?_4P`{ zD6r6rGBsfKUDt*}~`eB%}aa+|7L zm1@lKGFr~?89Or0w{xR;{>Ll<_A1Y5Yh?ko04|@cPB_(4&X_?Q=vYN~hqy8^V~u9Wq-7R>3W+ebK8$rhn0+$ zEh{dZ-&`K`R4E>P*0yawa6Gr9(q$X`2G+Kmrn0Z_`Dcmm<+HktiHl9W^3)3khu@8a z5;As6Y!E4OR0ZuSYD;VP;JLrl+Ln^{?`lyJeq@UC&zU@+nal%rbmw^=P*>8%&!ae9 zE6Agx8)@e@lW}=-<-FRVjk$YY&!c4CUmBZvI9fM}iCR9MkC5*^ik8G&%jqS{?suc2 zRfsN=xK*I~yxJOfjY~`uc*Y=-;S{aJ-H=S)4NJLh-C<)^EwfXx_oPzt%5{jUw0p!n zR+9%fTH~BA&fyySt>w0A;g!dpY4a?|X7%TR_j{ zLlCcaYmG|{|2lEtd}C`Semr!TJZ?^TQwMJUp;YGYp;m^@*70kc{Z6T^%0UQB-VpQL ze;5B~nsdZH@@8?Oz@~ ze8&Y;FpnUtw_cZ}ET5R|>~iBHHbGU=Wz_QX11jDuU3RauZqfV|e5#NIktl&2B?b`yI|rC9x#sY2RpLkQch z!e}92E}Y~Bo@V1ONCwwFQ&V2A`5Ew@w_E$@o=M+Wbp^$H`>13@55YGD{OX#=FybQf zfl=t*h36IEG}Zk}R~i9nc%O&J$itKOci7Cqr-8clgOv0%f?pT)*3zrcGmc7I2Q|!R z$Xg99!S2ZmpNgyE>xQL<=04ny2PG9}{tWyPV%mKPY*P@{a*;G%Qm}D}jr*B9lg-sS zj+!OMQcp2Icf$s(j_f$|u|oreG@4fQ{Iy{8Ruo@R8v-nrIZ1`4B;;Q5eiF>Chv;Z1 za=K@frj^U>*X%vw5{9%-Tlq2E2w+FE6dKef2;#@n#`YvX6d6>R-i2}goG&K<5Zk%M zleR+p4MCsqzVQo>=4L@HnLPgIg)apdhlm37AciqEh-oLoo${`1wYAB)_6hR%$54!V zndgvr(7Lo6L0do^ zhLj2$jL;@IAHNk10)(3jNa&`6#v)sUlc2 zZtdo}YNri0tTT;42lI=$fo7v{%FJi&oktM<2QbG@-IoL3C^mvs8CxTtJmWE9r#(6hxT;NC8OO7xofUo_E@3i@DE*HyLNrgy&efOr@376X5g z3Rv}e(8f6$3=!>opG~gGonyQk2)?=#XK8wnP8P0dSW&9EQYT#(5(kd2>dY$4?r;u< zx?tY49FA;+-`+qWd(jgVD>q?c!g9bU%`xt*pT#Mt2GKS2@d~Vn-PqCFCateGi2wEL zATIm#Lonr2pd7Zm zLo3)ykX>tva;N_Q`VmcGWR8}DnYIWq2yuTG8VSUbNYbm$v4qNHa?yb7pn{TQ>~>q^ zSHL8Qko36Vf}PBWf28N`HTgSFg!;0Q(IBwn0)tu+_-p8_*BPNK6Q_$0*H_G6)LkbQ zpzlV)%=i(cigL$DW_qa_EM1N$FE-M{<}CsotHoAMnNWhv0vifA_iBfERgbMJuQ5=S4Y zt?M4x-{VHSB0yP9bu8i(m2Iv3b(ulzg-h1gr`@(h+8EI%7s%f(lFaHzOrmqz^**lD zYpo3RZgcp`L-LfZS)O5;N1i{udhbE^W3Am{@_39LKY`xMJyRtvM(^9Su{;2{ba%YK z_3k~J%^_YelO95>rMlT#sd-a9V=v88M?r>LIb+X^AwW>hzDfe|eK04x1()9w*lI%2 zcY1-o=Wo5;dCF@f*~wL_ZB-C?eZz1iT2g)LP?V&J;l{l|EVaxxcZMB9hUrtnHImck zynMOu0)^2hl0QTa@G~lI-0k#CT?^#ce?UN0U6;!2YHyS~_!~XAN4!DRF9e%s$)U%Sy=Zh1d9{&FnVlsOnrWXK z6lppC=(*)4spoQrxJdWZzQWtHEn+#U65?Rj;8pwsbKO&3meCbOm72V{)m54;XNdty zg8{{Q7EZ3iXDRD>lc7q>b>6L*v8HFI{^bW2Q)iP}_#Z!PVBW)?=mv#YRagjrFC6Py zFUeIlr|CuS07k&9lWc79gjtv{Cd)5ivf%sXPu+Zt%8h5uaZ&}fs@H688=*yja_gW@ z1WXmS(K6dI_qRhhqh<@bH-0&y8@BC)_QWyF-cpi55**K!~GwXbG@C^l(ruLwXTSEdN${;r0h68Z#O7=D754gUYV6IaR+dEC*}Qi z+n^>TQ$T=%B`ItvQ1bSp~WO z3;6-WV4lJ?LGU*K2d_!wFIfy}^XD6!J1OFJ%RZ(lghB|~4D0*t9?JcNYwGh~8Aovo z-*)GW0e!|nFwTwug0z9Niqo23>YWm5S-h19^7ggNlrD)!#_FpFM_pEpCw1(!wjD)< zbfhk0z>Pzn;}0g@RwlKciN>#g>k>T8BRGvlsH&ziM37ibF*8R)RYk|5nIQ z^(^pS(4_0%uJNW`v)~EVeQzeu5^^m;ZQlV$FQ*PJo%i54FJ;oWRkg~N#>@B+3A@Ss zd{biPhYlkXO-X*N5lQ3TVzYUc2z1Z5u!-tgECN-m&_zZjYPX2Tgc1QKKMJ%;^ zdX%KzalmTN(k0{RsyK0;Xoh^pX>l$^x&KZfnT+~EHs%%H0mtnJHkSY@o49o)PqhVZ zpx)h1&1|tGlRKCWO>ZauZegn>Ew3{c4tFy~tPjL_mkOLBcv5pUHZrLJ+spflO=XhT z@!-}J=w92o-+Rvk=NA4b>Xx=~cfQu@f&9o=NoiX2MX=OeZiFByl`z~DKI(f$qhy+u z8>c9y{gI-W>e?A*objqv53Q}hhVdG`NP55H+L5gyJF9hNx-nmM(dc9Uf&{P&Xi_gd zg^i=kx?-^J2%?#8ZorvbL`6-=@i=$X`jdwgrXwH@vR zfOF>~cDA=n+CdN}iAQg{;{zG7=h62FCiy?xk?B7UCAy8=4FXx}yJPD>#X4U(97F2m ztdv~&t#|5+{z3)u)DfM#Jqwua5HO#`DO_1IFWvV^{`b1w1q6$LS)G&hyOrH@#nz{K z1v01Y7ZP@-O%SSE{J%2uL>4*R%a;iz`4cS3HF#0SCMhV-MQMDV*Fg?x?=#Mc&!$>q zrz9x_ou$~Agvg&v@W$72my4vm*OgE{#wV8aGG9Nbw$sM|uQfhpVOs$toLHN`kmvN} zbYBeB+vlQ3_AM{|E+K-N*^Un^0(|ogu#oP5-&nHF_36>EE2|~v?$@^aFs;F^A*A(~ z%pvW!leSG&!m?$U;u-&}y_pL*o&7~XZ0W@84Y$)&-ZVXJEV9-k&lkeOasc&VrzPa8 zXtDG62@0+HPIR-eY8j54tn1SnK6ho1Up1zxVPtNzNlQrZ*_cVo9Gy!x@L7LQ3g=JR z!EW}e(KC5W=b${ns*-1B~%9-IE}cgO%r1d-g#$7_acJe9Tqhk&1X{?H!D8>MT$$*->1j2leKH}zc@kyDAc$nM+v9nN| z8{}i#}B@A;C{km%pf z#`{m=q>s%9pPnmk()d91Ir68%-$Xrqgs?~;qQkihhEcSf#31aFJE5p!JYt(1;QhJg zN^FVb#7^nX-hN1zjva@XGgU#ql$yG*(uQSI9F)@mydP|d@t!Lj1h>9qvT*7D7vv_n z%O_K(p+@N~(-R6bO_+r{Tl0}k z4;Z%=*^+Xd5G|+r)t`FsH-;C@{{IQ#Mc>=6>X1K>yYuD^_zr6O6<+Q2=iBnT{F;}o z_vy_;Z9#G6H*a)glk0B?QG1pSA^y6bs2HEuG~0h6kFNp0!M-Cv(T_%I9%|zc!3I@fgs=&d-&!j2x~=c4LELh;MkXipk~Y529HSXPz5o9_~&$l%|M6y$7%5X`gMb3yl0P-IF3=- zBSyt=Evnwt%qd}~wW5mu2rbfanC`)*T!DPAo%l&EbrizF%P3*A8O+?cB~-=M4e-#J z_1XFg8N0NA8FyKtwy@HHz2wHUIB*TqYDj%eP$=eI_9}rqcJcKeTR!XCHxTu+|5JFIWQjaTrCj$a84|@?-jD;G^}07ZPLjJP23pn%}3hQy}Q=k|1Ggm zg3*S5+Si)p-wIKs^Rs>IwNH#)uCzFKY-*YZq`sbC9chD2t6P6M6DqZ9a~n$2>dh74 z>eYPgb`IABP-?eIJCL5>yl9h|&fM4UwafD^?qgduHi59cy10>n+rY>T*7PHv%9QW* zJO8CJ{C&aTa_AXf8y(Yg$9~_zo%5mfeA`qF)L+b8>3wJp7_OUqJw~dho%~Pvn&M zzMYr<*z^LOVW$rRj<5Li-Z_qHS7m{|8w@O}ezo69*KVPm;ij=Ejuop4_trxi$e*p! zAY>_*y`Acj(L;gH&-x>iLyE#~c<~1d6IL*zlpkk+=jtaYE6!vc_IM$bdg~*lqyul2 z?biY@mmAWBjd}SlmBZ%;!^^GFJPaUscABpc9rjCTXa!%K)u`pDH*3Blw)B=6_#u_( zrNn!9bC))A$bT)4WAuSs(sIqCE^w=xsSUK9NXL=$huf z&?_ZZW&QN5-h*K{&6&ruqqJ#N(5q<|(RF9nmQ@5VF&NvmYvyPE zButfT#NT}P8%2Qt8NCV8x}Q)q98+8@)U1u|Zw}rj=RMBYZz-hPL96@^D{&?3*aK+o2J(KT;tz?nZRO zUa++0cc)a!)BC$j5+UFPPSqm)SDsEp>I@wRetQ~^>EM`PQU!yaAhl)L_WSPnn-8*Q z-7_VsIKDIaqdGgnNf+Pv??%`(tz1Yk6h2_wE3ARKD@#?@NlrmNnXz-aXJQ}ir{zzL zsz#gA*T!&WU@f2uXZon0nfLC4EPxg4QhDYB*~-q8)ntPC$fJMX@97Bk7-qn<#=!Aq zP-5LYO6VfaEj_)AQg^rJBYOk<%%N(-FHo*XMkqd)T#Cq%O!f50lAf1`Km{~`me#3G zuMWm*m4*8b!75$DrN#q&l%N-DQy0%8Gj@laO=dQ0GsT17^u}>NpdnJU-Cf}&JBe$5 z+R+jF-Jpry|FZW&Iz;IR#Vc>|2@+WjZ2oK0wd-@{Phe?FYxmi2(*(CnfMq3L^uO{J zH0f-xawZrFTgUPjSo+Vzd!D2jFr3HGSsBBYlr9Y%$n9L^ME!}ydn*vl*z&rCoe56W z9Q_-VIW2m^-Koobm?_{2f7%M4NB?sxY;2730d2O9VUl-$-@-vdc5(|+ z{*s{Mr{W!FbRm0)xT~WUZr6Gf-W5bgRCpwtB5n;uEO_{wtF$tOh98!7m%bfW=+0eL zCwY#FFCn^ir+tyjp?n1ZfAOBZrAEl^)p{kxNh<6E5l_6e*F3Fg1pZSkpPo~Na?Nb| zARO)jOZ*ck+y2d&+VzmS2H^;gt5}*|+^_Yi*Y584b3j>2XT%c63(LJC-2$-Lxp#1? zAH^rQ_`p4E#yYx=r8PP5YpMo3nEx2JF>o8O>l$qp^h z4C?|D12>ISHf}LTE_mqlPx@WcFHX-MHnEo}vzcQX^y2)s5p+=*E*C{_Ty5dJ2KSDZH~RFhGn@suAb37b)Ts&d$tr+2R+S5-grRQE|VxBeQ1 zOJxCA>Q-o=xmG(9d0rN0;dqi@;QDdt{inMkLt*p_F z$~Cz-HY`}vFiWhoY?R_6_Z2tz5+VAN(GXE(e|doAHBEi!pHuM&!|%BKA$!oWhDXE0 z{M!xm-~AV2`uC_?h<^a0OK9)ipJMp`9avFcevRq>v$6L7z`NeL@2dRL+HT{a@2bcm zXqIid;x;YIF#m3nQ$}Gvnv3cRdUo{AL155a@Hx=jpGj9Vom7>kpf7)JamRyK3n5+rl*xwKt( z`2M4{cjTf~Q1&-Y&*_;{IV3Iq=2yN_cs9K3>cf<7{{>MtZbD5*8emA(LjoT2bwQQ! z--u}bbhfi#`%V}LsRbY21-Z&Y1PQy*fl7k!DLV`HJvuEk1-13IjH)THDG%stNbiVr z5Y?O$DcYhgULm2OXX8K z_3H0)9QSB0pNd_MHuW6ObJDj>>mBgPO@cssJ3>ojF3&-=qf7nX~%qpG#PNZqrp6SKg*grlWY0naFON zd9ftU*-#c9e$3Lfu&UbbW#-?Em}vqtTt^3T@Pq)aySaaI_|;taOe?3mO$ZCLEsH#^ z8pz*9<=d-EVD1RruHJu=Tk}nrj=K^9m!%Z)eIXm*Fn7x^J_gTF{lA*=C8GZ5iwriK z*0*+a`FNUDoT1yyIQctZ+T+vP6Ws+Q9#6e2yg(gM7QQ|@{GL0M#YcSCym)9s1A4Z)R8m}BTlDSL-&SXlj04W zC3V5KfPbnfl&ESSds%|fwqZ53fpI;uWt}9i zIe^7L?N2$?7nZq+ig&*S0CW-tdFj02PMp+dZa*x<9ZJ57+q@;d@U-W;M|7U z!w|@JckRIA+I^)$ar?h{VrKhC@o7(F74n1T?zwt)maK-L=MjD+vkR@|QclmwAK)7! zLuQ}$b6ASn6dD$Lm*f0!h4K4Ou8`~V{l+z2BU7Sr)gMQCkR0*Gq$%F_(0cbF>Uux) zQxTW3vHviRKf%Mo-3Y9*goZE zDmCOve%E0HZp$PKBRZ4z^PQpWa3bKzI_P?#YvcnIw+nQ0z2YxESyj+kRY{q+ zz0$jYBy5a>vW%-M3EB9XhDuZH^aUFz#q$_WoPRX*8;ttxcGD>fz|09fcb~&e5_d?` zL9KXx3_z|C-jBD)le_@OJ>V<2%gG`tcc3f0e8=L+)`J3V1@14#-AhM5XzFP5DBwNy zP06kpQxDbzxEf`9@mC5HS4{;bCYesz_SRD+-m`H5qE2I7()w|b>b;pC%?I4lXA@aM zxj~TUSLwW>g$(g9(&jj={j^t0;qYoIt5iF^*&@X_| zsgZbt5Y8t{iywBh)u~3eYM`T;@^bQK-PU~;gGB_F>c~@L%|PogXYyShG7z_jZ)vUe zD9RaE%|vU^(qwOp1246(PgLxg<)Ko|b>H`bt!@Zoz~`8kU1DuB4Ukf!o*_I5Dt=hv zRHz_fTw|p@pFLbod^vM$!<(h?$=mYxa@xiG1{lo8OvHReRYnnfK1cW+#Or|p>jW#;BQCpmwe)@0r1E%va(L|%I3n-~DQ!3_8 z2rVGj$rGo7t5MtMg;kT@9J&v{s*%R?c<%K@Le5*#}oNj|02q57wDhcoKyW zBi_>DZ+HVVd#g5Nr^4ekVxk%L>o(4kD&lm0Of90W*K_3Jd+qNFm+|djd%jNCiELm2zaCvNxoi3Nb*G?b40`6Q>iL|sU>YRI2g+S{6G{V@bd&qrtbw1!Tk{XT1` zV(mV&2l3s4gTzDSFp#-@h=?Xn{v$3%iD!1SOweJgJ5I^rNu=;cokM0Yd*L++9a1Lf zE&dz1d)LZ49L!z`R+n8KgYoV+R~m(BzRO`Zu+#95R+iJ!FPUAmQzvWEu^4%Qyh~9uSsGm8e z&$e-HM-vw#@A5rGIs>zA%Uwj$+~(X0XJy#;TXcEnilUC$m;Uk*l;zDPsM!+zLcta9 z`3f(rze3@v)KHt{*c7D?3wb-Cu!me|m0sd=0iCYp+q>ty4weY3QS6pneu3YjpX5A- z{1MTZMom4d%8DPbGDM$DYW$+Rdf-K8CBEx~!N=wWHXF-J>|H7h>0YQXf|eWtO8189 zT(G)0Gh2wtIMTB_`*9W7(i}1H!GEwOe}cxRwEpvKoGS@YRqrlGo1nIgvB!vbLf!nZ z4w)v%bn1Ar78Wj&ILFXr@GaQxo0D3@-z5Bs4CwZE`0{s;2B~Mhq>@EOtu?v_B66U+ zNK*G?PD2)=*@&gxPN#tRcumO8)a+&E;H@eD!*z-m&43-Y(9a5^y=@)c(^UFNe2~TW z%kWqJ82<4+d&6fVp}2Ar*$b6%&$`n0f=5m1LKuxvk=6bt$j(Rq{OzvIftX`O%HDGb z`5%FUE@P={=_fBmb#0~mf{TO=a!gPMYBex)k@vUzhrdkzEbQ}jx#V$U-WQ=9AjwD! zogyUVmn}2yUffiTeHg5ol_(|09eI2V`}1>U{IJ)7HfTAy@5l6rVPpYm=7BR&`zXDL zsra9CrpC>EWg91-`4_1>e`h;L6!4(t>Mv54Glov;e*G7z8#gUWx`@O_H6Ex~Dw(c9 zs$x<+2f5WV?Zbhpqk?N%d2!c=&HuLj<0`mAjCw-ZE$p%_bW!R){9MkgG z;$^W&W4X=0*x#Tq>qma^yn8mDqx*iAkFi}rndtM!NES+h#bF#qL_Fob*^V2gEYa1C zk{;CrL+dMfT9i7@GD!X0^KYC9ZItl(bRkK(|HG{xwfx6_gVM;D;uTW=uW33yO;Bzi zB#uXt@V&_+okM(G4G^Klc7y{$LPuw9_iT5?ZXfr1GT!@c#MXy_z5?TFL_ZLituqvMS=78#4J7WBekpOjyu={cSy$$^EJ22@x% ztZZ$xp_&8J#zo}KP0LY!Z;{C&r#Vg+r)orneR#h>w_M~5wxOb|V6qDr)XLQ?KhguM z9FkP5x(IxHaF8#~E?eApf$E7A`!zqLri;%pRDSNWyIXTb1iw8{^nfsR-qTiPKI?k! z0~yd0ZZ^8&>&aW(8GIfoJu!9IS7bNe9XeU;riE*dX@gy`Q%ZOZ~)Xn zIWD0`PS*k!-Oz2;Y)@lWUd`pP8d|Y#dJAa0EZDKv_&p`6<_Q~Q=vsudH#QygzpSo8 zm!{#>0#3d?ww30YpcuwY_z?r9>b@yR_R-SZlutayhrq&i*^@Xn4}08PCqjl}=ZJ z1$1GFo{wxQZRXs~79B6`Wgf3DzZ?Ca3jxaMv%>3fz|O?HefPDIu=3M5)e>+fdCALH zb2L9{^j5WGSU1!D4ek93IpwkC>orVyLW!!KFyEFeAhNG*F1U{K;;XwE0X`8JbnGtSGJNpSdMi@;0tQ_?t%?_;l7LXXn1h)tfq4-L=- z?x2sVY#WXKbBFlY0jd30x0Jk0H5|XA)GMb0Qv$?{T zZU@UNwuYQ`gPCo)RCm8C{+ofN%yNe^ywpDU`?PhQ_*tZZe(&+|LmR;0w}@frD>{Lh zEAZ#+bQbvX}qY0KDfBj>9yo3Zd18DUrMN$^%3;xoV(E@3@mfOZ^5 z<_7&2Ru+PE;AIU;5a(?+cf>F}jCMH9q?+OUUyn=tFQHQZM`_gGXGQ*Jh}f?yT#GA!lL~s#q7e52z%~hOIm2Q{l<8>JQV^Couou6P zOlMw;(`A9ylH1gv#$p2LX5QoGP|#=S9rd^vIGuOhFylBznd4?*b4x2*M=zQ42@48E zcD+);Sz9!S3M1ckQSIkLZ%TYQ-J$abVicbEgAE?(+T1RuYgDzCGoHN)*;{!#@6R3j z=}tK*JUr>jr;{%lI(qEg+YLa9lF0RH?HuCtbU-V`=hG7!N9X%EwciI98gzneS3f+r zc^SdZ5rYkAlV7x!2?7)Sw++PIx{gLqY}7n8Xl^JCd9FKizU$<_A9yL;|AU%;;S9C* zD`kRrSKcImx#^^>72e~kJeHa9ntCC!O<403M}q*w1O79OxaZX`e<|N5lL>{Ym!C2H zkk{SmTfN3!gUq3;VrGru+%*p>!<>Q(TvNN^&8(81OllZ^NS^z9hKiV&0C96R?P^w$u zlfJAr^(9)vXrb}P3FozxBplQn$E0ErW5ZM7drQG;4h7ecr%I`4dro zzI1mLaB^IeYnDH?HBAcM=9t>f<*Py^M1TLuU~Bo}d{i2Mjx3FQ*8e9j((Jx#1r-{W z8BOt2e51-FM5iCW8bnSntX`*(9?#@L-3}c;l^xs?@%YU?0)_sKeY_w{AbIW;M^aF% zdh-^fI^uz!5pbK+Os1?mT!drgdSR$Fr|ADclZF}TZuF+T<%$Qoxf*DNg=U+e#55OhjKxB5ehmc&@uRh&iG})&`#40w@WGqY(d^!}CzF4uQb_`?Jf-0p9dP25@Yk0+4Hfvi!UP!1tEA4CkCNGI$my1R9~7{ipR7}(W5#;1Ly-}1!gDdOD7{4)h%|9j}R(?-vX`f~j} zjT$D&!(Toh`PU;m{&ObI$nJ^URDyzLAPp#D5O2CKa!a1(d0C}+VRgk{n_Xz(6RF9? z?JxjavZa%uNx9{c`im%IQ}{)cp-(8_)DePnPvt#g!P^F9vN+p3!3nv9#MPPSmW#i_ zTc$MR^UdT%H{@jO%d+^=;jxI0DZ39bmF6=betP*ZOo++B$GId2*^+KhK&)>8bUT|*OsqX{_mQ?*PN&JX|)I4Vb;}y~ZYmE3T8^4L%!J8&vFH!8& zIl3^8@^Hw8c;GcAb6bLboujcaX&Z6=Ja1|)PUIv`PLzfiG~4h=KCoY*{I_L;-fLEa z=vKwjPd6sPe)J0Zsg96ZeuRwNU&wE9Zi;Wdgdv*?QYV{;Czt+%hyMGAs<>G8(4V!i zyuS45YZrPUs3O!2f3dB`qYU8^lHz$<`X9~ZpZwL~A(i4Jf3SGs{{=s70yM zf10fT{op#0I~2^e$#35o@yI3VZRpA1EnNrX9Q6DAXl~El34O+{3~E&+wxv&iTRCiW zxthmjpr+rp3N0}V*Qd5!`h6Pj{=!v$r+}R+ARhWON|E2%pA}X+h2)2B^@7W zU(H+a`rwvke>$#rLAvCT)(|x)=ReRx)`x$Ml9T>XkT2578u#7St|22?ed3)Bp(~XkW}NXQ!F{2j97T=RN0no^xXD zirc~#bJ+~^)SGl#M$j3?_ zSa-L*@0I{VBBUfml@A^lUxnBFhc7U`nK4y4s$mP!LutLPu*LH9YVkMXEk$jHP?(F` zuOe)!54Ao8^8qkTs$0LdCmd2dQb82iW{A;LdpPS`{?)J|Hs4Ya&FNbXrQypj=T6_? z3vhG8ehQA}iPzw3*C-wOjQ4?cqlMktuhD9{N&L=8Gew5QfC)W9oBl3r1=b5TOnK+F zp(vMoeJV2Kq;+q4qUQ%vSc9KgYDBM@$Lr z3~ui_c}vW>DGPt8);ba`9j#H1q-1lZj{;Q5ywgwP7F)P^G9+S>LlNaZICl?~Cvdq{ zPWG5&@=KC!Oe@iTAn;Pwx?-itkQGntsna?vvC+x$k8@0N}nBg|Un zyW&`xIHGN_*>g@jv2mi{iXsZj)(8zO^;&L|(}3+?v=WlQ>qKY){<--ocPWByx{7aC zyim>%%$6`wcknOD-59$IktZ6lTeFAYK{S&l2n*fhY)`lz4^tGmN8c07`qwU)O%WQIMoPL}x}eP|aO zI=8IZbVEWKT`)T;xa{?*l>H7ystJmq{ztC?*vmY)qHOovIgQs;(rUsnM?&AYQWbWP z;uw$wyy~9x258aeyt?_ji_+zMc2Bq}GyB*!r289t{?z<#50GYfJdSEk4K)IFrQ&_P zl@+c&4gq~&dwJ-M8q!(ZhSz)L{4coJ0sZuSfj~P6irQq;e&{%ose5ka*}@8VD5J7_ zc!y{15NDCi;w8ZpK?f!jRvj!d4y$t{JVs?*lJF%`UE+(1-b99A!!M7@LU#pR3Je+Q z^c6PfP7fvxc{@#bTrdF!$W%SVENVrrG{`b}F+(T+3nFjY9Tm*1lgC7y53-Y6Y;QigZ8p?WW_)`Reh^bv z3M)kQd#4I=RwwPF1}ZI*<=49JHe^Fttk@Me4DYZLZCOd>av?87w)DD=vJT(4{t2wh zj!E&PY*Fi*3=#|wihQyeZxJ9r?v8iC#~x(_taP|N(Z4r;omJ~wt^nQEJ~CrDVC4}5 zyG7jJrGT0#)AQ$ONRT!Knq7&nMWv_o>ac%bJ|FP%fm?0U1>GOt0#-Foncv=ArK`PB zZtKgdNeQaqaCkwB5C*NbfP-xvM+0~(?Q4RZW>V`E^os&Uz{y=^H{00JPO0|NJBuBT zw~ik%9JAwJ`83R#%}=>apvBkewD0*R)!imvzYvk7#AL;&38_m=w5f9<{Kau`CqY5_ i+{e%Bd9bDo{>N{qY?uD~foHP}$%fqH@5}PWp8W?urf}r| literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/cpu_info.png b/debug/accuracy_tools/msprobe/docs/img/cpu_info.png new file mode 100644 index 0000000000000000000000000000000000000000..744d237e975e555160ecdc391810a3681d05252a GIT binary patch literal 33797 zcmZr&2RPg9*H){F+BJ&CXi>GbL8+7)wW|w_JsPbL)E-G`)v8syR%^G4m{B7(MQtVa zt}V7$@kQV7|Nmb4U2^sMi6(h|&pGEl_qoq`68Jz{^&;hEN+Keni)#0k9ug6eLWzjZ zsZx*=p4mzpq9G#UB2rUQcr+Ij)H~RUV8u76VaEMs#zr%MV9<7fx&HaP z#TBuwQqa~i z6|^~YZgYJ}UjCN6DSlz|SN`VxgZNETBJ%%xOfXO%4rnlgAk68IWU^F^yITzJ(BJa` zKfOTI0m6<*}O zbf~}f68G1xcZWsZ+Jw%zmp;GTYS9;ct&=0fy2326Mx3Lbh`Q$AkD6Y#`(*4WjAZ}M zrciHN?tx)6sK9n%!ngQ)mv`x>4djN-{`syH8jr|a|DD9RIm2vCuw z*<-y_`wQdXRfybsU@~hvv1wz_IF4ig*Ig2k=e6GD!IUXa-o%)F%w;=((%-?nNM4o} z@eLK{J1{vMqlxrl;URh)rL7-MWApa4v6o0k#71mo!aPwdDE-U8iwD5p+jI03?jCk2 zf>IdzR@$8%!`FSmQRDGbV_hc7x~`5r_6n+Q#Ur&nzb%GD&sl!-x~+CA ztqa8Vy))C|_zW$2X4@!L>me)$0bKBt&)Y_5%8JQzAQ-lpQwu45du|T1`tX)IylbE3 zAAc*=PJ&&K{g;msoXsb%NEd0)@q^;n#w3X7i_m=y-*@ESqR<9wl zOD6UGsSl5a&}(qmXK-tELG^4(T}+OtNp{M>?!{KKVilSZn&HWrC%ZZ~MwLIUP`F%! zJDJ4x+%n1CTKkrG?5|5taR&R|!P^iLU7WTzCRd=H4>GefD8 zbDNBqF7ic&#&O9$LiJ~ayiI5rF-Rdj;;`^ZYuvU8YxDD$99QzbPu~GO6C)xy^{~H2 zUTOMBA8OYSTqmi{m7PrAYZ(4gKybUvv6X6ki#nmBRrq#q*%fue647D{lvjZoXM@oB zF1tKXK4#Q;{b%y*`=nfNDkhCPP)N62n^qhBaAZ9W=i zh}INLC^(=5JXGyDZ>BxDtwMh(dE&yUC}OV7EUN)Gym1pzu}~Ot&IQLUQuUCkc$d9& z0Y{?)1w)92`T;a|ja$bu&#x;DRDscjEu;3=^!S@8sLy>tn~Lu7m*}*H=UU@40n8qv zM)TViE^iF_t8=%&`=AZi`(075Y+UTj+6=CTLil9wbhy*fp4tr}598ik=2MOibt&cniJKvd`Fs~Q+Qm16<-sp zZMq=zyzm~~i~U+{g{T&mj~{@Y{crC_$W<)~`L;7RHRIAzY4IycQSE}Pw+advZ}UcNg2mugNno`Zyz0eU$CAgquX4YwA@3_D1O2k{4n zuhG*Kb^+yE`PE?2*bR2s?;9aWP2I7QgRIJZPZnKgXd;di)5c}ZuJuMx)BwxT6qBIxv*!!7rvmqV%A%{ zQ=Qg1N4iW_mDUe556mZE?|SY$$cO%4ZDCg)1cOmFk@ z&j~)-y6#PR%6&{aYdlF4{9%C`zRH-;YZI+gKnhxQ`@mqXq%q=A(fK5z)QwzUtu#0a zV?vtZc$9mXw4!bs;GzMicOxIR0W@rBLd_;qXrQ9m(#J9>YQ=p#MDOq)(4q9wkR!cC z*+$)L3-@UBPadcu=+8QCt-ua_K&PxnL@xi@hlLXO@h3;*y>4T@oe&}V-;Wp5oP^x;qw^w+|MD>H`I1Uqf=)-e27}?WOa92QQ ztmY*}5=IYIr@!i9c8R%cU-k%%G?l-GiCP?>n)-nG+*=}?u{$ImElLT@-KWif_;xhW z82r#9W#9H$DGO`2%qqCSb4gxsyFz)< zF?_9Xw$=u8HJbf$wdq*Zt%cDY z&O8&P6pP_hmxENE{pH^>&=iJSKQ^B*l0j>Lr&BT!b!7|zMs*mOukM?3*fEjB2z*%o zq4hB3H_LKWMBsq7wC36w-bZlYQ1REM^@=He@@L=d1Yc_!S#cK$aIdYZC6~!J#eYb8 zD(p@&!debB7iL88w>|UEu&?=5obQ^Ay;t*)w0eFi-@O-~x&R_zvB%8^b9(%cp>_b@ zshqMleDd0Iw^1V`9{9in^jU6`XbBSIePhmZZwGgr4qPq?!6F{Nw_3&T?T5{I*oBLKZ1)!3vOsju=aQ{( znqKUM&I$08EHBxPOO;lQl$hRmfakQGMX2de(uXmqKRYb#0a&SaXy=R_?(wEzr;aU; zISTwC9qlDJD#WS4teItV`(WB@;40G0l$Ua4`DQ2Z2ymo)(_#F(-y)FsE@{jYulW_R zc-EFzBEG7NQo!#W9z_#Tgp?oJ^l?z%B3(pS@uG`Ft%^mnLtzoy6DHj@^wYh<0HedG zn6@bws|hu<^wlUF+9AU7V{nIyFdV#Kmild|dn#utA;y$(G5W^zz4XNR<#5B}v8n1Q zzzY))Mx3qsK^n2$oQ96V0E%QtcJbv-|Lo(_nD&r7D~7A5D*6fIe27c<}RYh z2mbb!?6b@7-xXH1@e~S2m${?*u)|q?(O0W(D6jRknJ}}!yAvoB48Tt8ZpU#K%|L3t?kD-_yE>9YlM@MOF`ywN4u9Zxife6U2j(45OP88Gp)zl2dRHe| zP^tLr!*PJOpM?3Ihr^)sa#@_}+}zQE-(vEEhg;4W`L280-aRK9@etD8^xg-&IJ#yO zT+_WB4UNXmQ0E}82{L-AuV^n0%x>auj;pdxvf-`&uO24Et52Kw+SS@jRK0co@#iqk zYN3&tmy=T$AKFdE2vd8HA^ijmFOAWrC8>A|ICyi%{vMaL$TUO=g?~8ola%|gl~)8l zNMAXa6%9sCqzHZF-p?OXVRn?-&$`OGykFT9b_s4YUfgr^xp!f5L?dCy>&XxtW!n22 zVU<4vI5sL%kpqCP9rI!gSDVdOrYUVIrr*ioXist84vDq9wS1$6mOtZ0C5oIv!EUQb z=ce99Z1ZLB3+v8q>~eCQPs+1`br5P|@uC1Z9^m&)&_|Tg>`pXup$zh&zk`4EDvGzW zey+lMmh?@O$v|g|uvKg4A~3_ytxqaG<4|M|yih1CHqsXrlQhl{y_vDx#?r2h94r-$ zLEIO6v7WyuTKpL>;L`3$5`6{azo<7mw=yj`MRUrzYid|dIN0yE7C@A8(htwwy2NtU zhJTy-zt@18nqV^aK#7PVA>6oQ1OM5i{EvMw786#!?S+?KKCkFJpY7&}T)V?~;q*uT zVS$zJd}1#Eo#t(BGl8Zd|Id5jyadx>?0*@yq2{er~sgm8f;@e(Q)AZ3nUzsFUeo*0y?(FcVP(#~MBArEv zw3R1D)m^&91+|Sya$AdnFTa<1UoOth&M&4Ju*j;^IG$fGssiG_@6?~i`~4?r{I6U3 zV9mKGf!e0xk(|Ewdud%LQ#D0?+#iwyGxAt%IHBBbxgJGBdPe&<9n zH+f8+`D)%6bv8D#{-P?KPF3x=6)IOmp{6vEjyE^+90NG!3<`D?lP37dDm84v-)VWB z&PfE*T9Fg#%h&+l73Q${;ux-8byr_4EvAhZ>(I+GInOmq^_9WtLSORBgehzJ))@uo zLIfcqA|fesGEjdB7s_s{DGLx>r6jji6FU5+aU(t0WR2?7LThU1$?X`Sc7qJF%q2}t zuN)Y}&FG)a7$%kpUsLP6UBpGl`9i4E68>!u=ihs^oNv4%YcOI(yCvp&bE(n{H6EHV z78)YqFJ=Y6Xad4=i_0D!%P_KK$?r(jb+I&tgOo!@2*Jm(_8dP*yhm zpeMg8PqfilZ|RbXdN_W~iv>3xMy%1+vNR`&_c>$7bF6gacJlRn=dzKeTFk}T_FrrR z38|r@C>{(h4g#m9CnmF}JNg87`jh6RnTH@#HiV%NFr_PMCF2|abO8KzpAL@t-5CxK z_aT@IM>gLLP6bl6S!q0~r2Gz8w%+=jOQpG>kVh%@J%itle*vn292Dl~xSc4PEVc8s zT+Je4q$OT$6K<*1H0Sud9lFh3PZbFb-1k$91WF*zczLC-j{xSr)sgg`uv(#8Qvc@j zXY`^x9zRn1zYS-;H;haS=8n#OGg`hWktk5)${fl;WAm-=h;mN1IhOh%3wLQDI}J-e zO6j5p^k64D(_R6rp(`^KMEgY)G(EGmBEYdk=@pvH^;!|>+JAT z(%c`jNO#sOk+G99k~02f{6BxsmUwMS!u90t1BK301JD*2JN8dVD5D&?#U z2W5g$N+rFahb%S9XywS{c-tg^g&nca-e5d&>}kdaV*DFuYEWWp^uAh7r4{X2+!zk` zOfK~ReMWraK4oW>Z2M^DtsN;!4M*>if@%9{BKfId-cG}=eC(>;Vs(U?ez}84&G>tn zQ!6jKO+w2A-LMIv8t@peDDx;}wTHcJU;Q<+`fQ7i`)Ym@-+57%JS|sU<%Fe{^Rn%p zdLIEC(A|;3ZOdam%=?sptu+ z18|>T2YU!i*F;_qmWHrwYK&{5lm_L7kI6tcIsHS^2j-Y_e%2n`KJ;~`s)bLBJan)wh?QsXg7@2DdUnebpN~0?-b8zZ5x(0_6o?=&C{x*-*I*MPLf>A~J@Vm)Qqp3=;{-Ja4T*P^gWEgm9eg*=-J61iJ< zD)#XD$A4EGYwpx?D#+`?Guy6k1#}P>PiPeL%*FGI60R&ug*cH5z-Mm<`vJ=i^vvyK zF3%xtd4G`7b~K1`>GkFd@s}z?QbT5)st?D7Q&n@4x;uUzk18Bc!u5v*6TOCSKRhiG;_JfA4hVEL7XTzqD;`jB0W0; zx>)A^1hTI}i;}f9vpK)8WPZpDi@Y&`B_6nmY}H=LFT|PTM$5F!=$)VDbz4sS59uEASN)|yIrwW7n9h4R!_>CJcqs@ zKX_hb__;(uR+OkgI5vPfp>P)(b$CfUB;dLpb{E&N1~=wEzBL|P-2WWJ%WQHgqVK$^akAyNOC&?RA=^9|=vHT~zW57-3` zMX?P15}ku>y{RDgxKr%%A`26Tm?IBY8u;NNV6vLWHP!3dHi(Hnkix_x@upbbWjMHNddK83q-l5KMqPPI7;?IxJ@-&ypclVn>{KC}G2bs;(UbjlY1igIhNaL-R|pQ>+04L%S|r&>56v;KU;>gwlb)^!m?< zM_#8DH?cK2@+$%uxz77U6lf#TU;)zEy4r}&OWo%|=Rup4RA(HN@GI7ekl_P-1Y5h|@gw!OjISU5&ge@8DEwpqd6bZ9oZ!j3H6ix)@%=}uXA zQJ<0k#MWLCD6Ra%o2H*CJw7$;t4oC@nPtaR$1pESZ)m#3=qc%*{V<$9fuA)A6SsR@ zEWvh}eO%djIeR{kOf&=;F&r&8AQCJrUT0R?q?wh!6-wnnUO#v7-3JQCm`*q{(jeb2 z)@Ib-KKDSEPu+#1z3o*>GpD!8*&O+pb&Vbv{K>4zSjGm~D>wYYE&OgIyMv2D;tbYy ze7sONqpzdGMH?BvJh3!C%a`eX!<4&KF3>L>HvT|WO+cm*!g}N7A)y18)x855dRjQG z2~U2qIlwMmcZ>s^@k1Z#24US*j*Dr`aq$7&&Q+Y5w|r|f>&sa$Qfcu!;qJ$1z4cMz zdMm*4M}x@A2j?ZyM2M*E<<%An5E_S<`7>Er+L1v*%?(i&z9CVwhvKsX5K*elK0D@j z&0YvI5E*>CW{3ia#u{CH+GKRSTI9wCJFauXACfcsc+qulbNx$Gp$-6hxte9QnjlH+ zIlMj1{%}0xsiwVF!HtGUis7`F03S`9y*s2LNT|?F6ZOciQ>0o65a5tkqdP!^u+WU+a7 z>LJ|3rG>^e%hRL-+Z4t&4g3aYUCBgF9-4kK=$_;`v)rr(Lt=fK8uJeA$be zQANQ;Ra$w(hk4RjitTFk)L^r)Of1IUqt-d!bQ4cw(Ixkpw$hY7wPI>X1xdy6+czY@ z-%k=6}*ldt*g&3Yv83;jex8pSfv=8{nYZCLU>vw0H zIkbrDC-Zkm&*)Sha$c@#{vNXB+(c1;aM~^RzW2W+C%^HOHOyIhnp`$3-T(ff+EP_W zQq$P)FxM-?X%zi*zv-PRRlZI26qVPxGXbGb&QQx2AoRxg=l8~EW)aye4(((qGu8aW zs+Wxf?hV_|A34j+(<5x zZ^VNaVyT%1irTI5&3Ag&v?mFb=+5dvZul<7nHC{5qJ+F=Con?K=8HLn+}e&d`Kmc4 zTuV?~@U66@8M;-s5Z4=Z&%>tu^7q@Je*3Z@dF<_aK8wq>?*KHvZ=DWpACyG+1t5Gf zR<*uNduU8i@-5loXEH2LjL_Vh5f6>NF-25K*MPHsf=86Vwctnm4MHy*-Jnn~adwo& z$YkphG5U$rLhSA@Zuk?>*WoT-a`tMiZn90$K$iO7s@)h!0@@S^hF!N8{F;UOz_TH2+uQB=EB_jUhpDw`Yr4XBr z+w^8{OyBrkX3G{YiG519Z`c=Oh(*WF4WqH=WgBf;LT;s9ZZm?6mduI=wG+Br0|l#I zpYtzDxm@0KVyz19hE0d;SWf$|*W&)S?*FrX%!%03q@TuIRyc91i3@`i`;M=>$_RyP ze_J>i+xRF@D3p>_tHXL^o%I6gT@43rLM=Q71_^*J=kD;dAw?*u#Z`81iqVVsbbvgt znDFs!w^XHnF_Nmo3~igcRi|=Hdk75?B4QNhJp$kz5(&%)Rd2bxd>o5Ns4g-pUCX;A zr$)8+uBBD6?-%#fBmGZ7Nk^v5>F`T3Gbdd>r$7-_Z<$C=3%2vi*ciXnMac=R{p*vn zKIjWaq7hfq0^S=~hNy?_%u^u4(am8xESV%GVE7HM;#OC7i$G4=n}$jqXgl6 zu(gY?qPUq;!#i@hv3xfzwCqL{y|>!XwzAfKfc&Xuu}`ez+JoBX;eIJlpAU`66cTpp z6%_1k$2Fc2Dh**Xh^_kLuB?S;W%oB=Jj93Q#l>rK#L;b8%2y5KG#YBkg)Iq!ZIdk* zkNg*>ScCzx9N5|BYa;sco_p~9Xw{o<8*loygPGTp+37hTCRS7_ z(9&a)@ddK{U?H_*f4pJ2e`&G3T;aUK-pg$ka=%iOfgH36S;Er;s#!om0^~}XeK|gI zGvp(@NwfxobwWP(lYlS&=I+6M0fZgA;KrH$jDB725%E;pMQn%D#?Rjv$R{8i>rc$96 z8~7)OFUh=|k6W{rVkO7}ue3rbwwDf1+Jc52F(}Jpda(QN^+@e8#NbZy)Kl4CondPE zYoAxBju4sf*k)B&<9cL>luQ=inysa*1^Su;TZJ9id_o@6Qz$Ceyfi)7n1wqXYv>rC zW-1_Mi~weYQ-L?Q9qRLCG4NQ@=$Uj(a$EI{*M%%(|Up{C(#raFS_10O-i<2UhH~3ky zRaqL=`x9rWldV%a@|ibsOe0G=jUp>yAh4SD&UjU1)GA1?pCP1hBi?%l(#3QD!lX*; zlP1mnJpsv(JF3fl(%*13I#$?;uy@Ou7KI#}bo?0X##+a3p;L%U! zO7fG0YRY(YZQ72$2dv@*FL^NF8&V9an*C~7jjJhRFa7{5pm5K7`Sm~)_iSGm7V^;W z$$`lnEwR7o@rbZ)P}rRV#l1Z3TW~?}$DQvFQN?U&G#DCGzAXxxF|82>`+hwaVV6P% z+6+oLqt2kCe3~hMCX0zjjkfoQ>HwOoAALSdr1qw)8(;9Z|LJxXgMB(r1~%A&ty-0k z-PDze$Oz%b`$O)M!#9b4Z}RcOAIwXh$#*8tiE33uzqyBXZBUaNb7P>TBeP(d_<4kT zYSzly>A8)RW+LBqN#^;ez%`Q7c=*qxj{SL(@f(;#>H2yj1v&qFPr{6=kK7kIf&{() z(l-eKL}0wWAx|re3!P^rYfOoHb^aYx|9ibAPoBq(A)psTR;T{5$ZSL8U`NZZMt%-w zo;p6_0Mnv<6DkAHda})zX=z*_US2$QA`FnsGV#{^-?i3+H?`q?q{=FKY5Bd!4>x!ZNDcYy3zPT0(hx-dC;MHV_j{8K7Y2 zJSPs8oJCDgyGTrOTqe-s)`n!l++0BsDCc>C3G-swtxdtpw&$A*!%b}WZQV-odyf(K zt1rv$(0)rh_eSemzV1(%D zjMEp+6H->NtVVNePYLovy6X{N^(%sG4ijnBID)su+*V2&pYexhji2$MP+qQbO9gLOx!!>~)h@sE5wlf@oX%rQi|QNPI=HVY;5*${;w%dNPgC&M zJ}Hn1b6+h;65TXQt1dE->Sf9Oe&6=bTSz1Is^8}?$F`b&#gMUGB6w;v~e zhU1;E2}Wqtn^e>3j$*Y=RvCcVBiV%Cns2qLIW%*YukI^fcmY_5wS6075he+ZLxMic z=>WnooiNH%oST9Kz=LrD%Dnl#Dha(ke9}yE4k-i2wbWI)$aa0`cp8som!mQnVQ8eS z-!0He>B&0fSZndrUhhqNTlRpzz{xS2sxjbCp(6~ph7?9Ana*9|v#Ha1xY{H*T0rRZ zNIhga8m3#lQ!-S2Y0J1EFo9I%$;QuOp>Tds$OP2|SF@YZiohi%zH|)g8WVSk*YQy^|y@l0lgVmB+{M{+xyhhyqLhbXo=M{#{Ro>21&CN;}YL|~8oEigu z05UsO3iAkDkaFr)ikMr0bza)?8V3k^$^<8a1h=RS6_MTMZv&(fD586PwO|MM7F`B1 zvkX8;Qr4ul^a(xFhJ_Tm`y79b{2{w&a%h*?qD=vGl{lny2)Vsg+2ZN;Z>sm&2Wv4= zM4Dh$*ra@yz$zm-Kip>y42>g{r#8!6h9 z$jA%ycbh^j2z?kM>xGh)qJrKotVoa!709@YA;A{`Tv@ujf_;X!sZEJZImqukOxUY7 zKp4Z5O_n$5C8Uys`-UBu3T)TwA6$xhhUQ8FP@RTYf|Y#^SaDpn&r5SCxPy0Pf}a~O zFD*$Xt?WNxy3i>KiNsk`I`{t+({_2!6BI!8VowcD@2XU(_V(Nt1WYzcCsJmWs9Jb-8s}Fj9kUw#VnY*_sujrrqmWiF#}&gEhfb# zHIr{5m<;h zn_saOJ|;x9H$#M!ov$aYa*gBLKVJObf93pr2ohbP^49ZOWBopVGDN`L3&UUPazew| zRBWbV_^Y?Cr&k}#Z|Wt$iidT}bHHfn=S+JfZ}5>)ZZd+V-3?EQlrUo{f+!y|)X zsewXJ`LUC?Yn3W;aLp|i#J>DsXSM(S{yCRZemj9;BMi7S1=7b)d=O%?iV!(}ln2f( zS9%~2prBTM_puw_7Yj12j-oG=5($L^m5t}X48b2HLIdcUSH_3BTEX{qmTmU10Y|l} zyH|kTnRyTR^TB*;7v%rau{>FE^tDYX1*tTbVb znbtO(BJ-p4z(+AUC=bdH&X?OryC8NZ#MfBK;VQ@lAnI!F`58*hs&ccI(YqzDsQMg( zA1}mYcVz0yyf{BQ8&**lz!K)t+v_-bcMbfZW0WDf6cqFnvZ%$pwI{5N8eu3A#T4(u z;^}6@tfpzQE}Y$`lgJ1&ClHqI8k)%H;ZJsq$uqatpGCEVy1g-dl~(O~xM$5L5cdD{ z0;O9B5T@oo%GV%gKtNWr2?R-V@ww15na`B#qz^?+|4t)FcZ>GuZ_cvzhA@(JXqr`R z80SELR|>kF3X6+6`zA!>@F>Eh)wk(UTWt%(^lN=P^~?3!69$8#Eb4g=j|tuKBQcEG z{Pe9A&~U!qB$cdPVt#VvlSueylhfWOa+23Ms`vK|s6yED`Z~^4&c82 z$%3$7ZQE1g-xYFA5Ah7ZL7gpe9x-6T}LQ~~x48kmg4#(`{ zy-42{nU!z9%j#NiEZK!-p;QL9c4BrwwiEEzrjeTiv}vO%>U%cB&2yBhI!pdAM&cKp zMlbIq`pG2d>Hix_%lJrG=%JxXr9yhfokdDgKV8_*aa(D+aD?0{DrkxmqBI}OD^95v zDc*84LgjlVha|l_vVD~02jI63Sh3+8-YHpzl~mK1R(J~^&+UZRyjADJNFU#St4=pJ zQYYJ)&tqu*xUFaRGwobR8J~RI3iR=)Kl8Q*I_3wTi$=ARPUx0d;kTk7#2W4COjsi- zW={xTD&~7u4tF$*qRBi&JGb5hn~S#?NVcWQ(%;^H@>p%dHCVc+z1meQFUt0DgI@qu z+Gxq{a%nc#{l`PDKk=GB9tH$#<%5++ZHZZ+)|MvB23~2P+ zXG^n*V$*5D;Q#?hrr}mZVbv%FP|3yeu)moFl!-fkI=xTGoe&Z?1qD@0s8U3oq=W90 z`)!*ybj_$f%%tS9ogcPgH{Xk&?i88VaOU6PL{{IBD}s#wK6rXSX@y$ub;j*zJ-V{Z zbl5LA4sz|!A9^?B2}!}4>?SHP3KN@|Dv)Nd`fLixz65@FcmJ=6JWyi|gaT-%kfdyN z)z?R8o+=nmRAds~oO{#ECUWfdw6dRrPWXbVEZd;nwPDBiNgw8ux#UI!go=k;8}QE# z-T+?am+WvBiY@_+^pN91V07vlee#8c&n9&_s{r>M14XqQZ=YK-ENXZ%SsGO_E3daM zoBNL(>50H0uq<#O~># zbV}6&^;z)B3NH#Ph*Ru9So-IuBI}Q`q%C1jUz=I9$~Dv~tlPrI?Rdujs-j;jdGI)xZtsQAXdZwhZQ{3FECwGmfc`_|~qvq|04(cMny& zeA+Qicpni-EIrgxcQ^<9VI~NB)8n(VhOP$k2aNj}?T-+umz+-`Wb*A2j6b24yE^xG z;Tj#^!Ksc1ux!1Of>CtLF$y2z`#u!u;2cBaJ@#7%7V*sMR@p22LtEVhaB7SVYE(=6 zt8D=yXMA?0jh6H2(L2-nnP*mN1$D2GXVQW_FG(x|G*tLo%6Hah%!gkdI2EFU=8J~q zMKLo{{44SZ3&^9f7IZ_W=C`w4jfezB0BJtg%0xX08Y$xMN-q+bz38BJTT;Z-JRGNE zl%P`AxDrq|k_w2$y(e(iO`6qg#?`atnkXayFVURUq|!CKc61y(!{jXLerm6QQ2-O} z!G=T6UGe(M?@!jaCPHZ2T;VXAwgX>2U{8A*lklO*=iGYUIc~xU0#pCbVFP8=z08Zx zh0geCO$|)Kp0k!fbS>W}K@e6&@z8~5a|w67AHUVwyghf2*Y9FmF&p#ce6Y6E9u5q( z|3>j9a<)HjJdp|U@fgO|Ts+-$eAD!&B>w*vJ)ur*sDz~dY*E_#iRaONZD=npPQafa ze;ipa%Z+-}KW1fhEGfhBc>UgXCTTKRk|$7EAvc4yGs7Y^HDI=~cc#;%(NC4?9j#mf zYQ%fP17+G=Qg0ar-ibGzetRkK(|if*hdI63=4>j!5o79W28YZ)Q0(?YD&cWrD|sz? zt%w3#q-)!@fz}@BLLY?gPPtBgwNZKKt?Fs!2S3wWdF`_#L4egv@LLKCKH1eoP6^GT zVB-blPvP_IlHjzCAcuwL?|C#q?}5o=X{78Zf!LL&@maj4s?pdso`ixJ87jmQDNn1?9rH@`pZSM+ZNYOY@ak#_iEwdy|>438}`}V=J zJNq(YAR##|(u%T+7mqYmI*HZhYy@!Wxqk)6gYdTW)OY52cg`=*pcm*~jjK%vQ_ksc(RsPjXaa zTMn8jsC+WesH#IMJoZ<-I&NPa-E>y|BKSOwxVpzT#MS1&obT<&P*-zg=!pHs&H zuNutv6TfG$_E!vAooF-CK1!(^{ouHtztyqLqYS?;(O&9+1wNw{>tCZ@<=}up2wnIq zm!fW00)7eW{`wK}EWN{VrMRLcpb@*-XGWSZx5sR^E_~K8ILrwYCxls9e)otcd!!^J zvz$}#k`JNjUi=3A_c+pPpUZ>-I_&Arj9ir0jbD?uV+2wwNK@*a#l>Edyt#?{H@W%e z+KvC@)DwrlV!_yj>&}vDXTy*%nRxP+CqI9qA>o-neHADFEf?0ZX9ojOT0_Af{}PVG z*3=33)IkYceTi+Mfo!fH!m>+_UVl2jR9$=fucJnf*vdjp3G;~(O&vwm62qD<^OD=4 zOED>w^*5sHmVRaAhY)%l2kc(LdWBTg6dnUh<4}=F|55Va*M;GJ-HB%kQ4apphI8lr|E-Q7d3EBX=UH*L=EKjneOPy}-I;H>?Txz{M<6&fPuIRoEG ze@s4vxtl4hwwWk%(T6=$xbEdbe-8~OO*okF#!S%qCagltw5VLJ64pUNX_{Zc!H5X?O9<#cC0zEs*+OM?@ zX>C-99+PnpvAC{=_%s^TrbKgAj5TSiQ6@_eBIYA1xb_pO;9|ZfndtYtB{hddq4&1ku6=Huy)C<+0A{1@hR|`(gSOHLh(RD zY8|nG18!I)Ow`*;)vVeb0SkYGxDjroAn}|T1YJ;M5uW z*(qV&jrRo)cW-ci5;4~ry|0~dcBF;3=%dPpU9H5#SH^i5cpl4o%{qjL;>P>*o1O?Tzv0vTW4qdcvtmc zSIbBv9&ywRSVrKn$9sr_&ZNtS6)P2KoU%J9g~1YZdS~omT0=NGw0V(>3nK?t9A>1?`JTik+P z7)KbY8FaWwIA(v@Vp(pDNb-kEDSti^8v_Sk?YPgIAa@2LLHM;xRbxoRH2 zIT{$pe<(f5C;aI!Wn1VY;yCO!pqWbiAw*z${CIP2MMmaH?QhW?tlFMC{^>zp4=rY$ zu68}wB2Ye^_VmrtQ+vgkN)9Vo&mWWtT*(~l=$%NjV=kzFFS~GLs!9*@ji#I*qLy27vblb?9p_{Beo(|hd z(QViZKFZ4P+(t2)mu?>)0KU%`A*3D55xXpq>W-9jyDFNwyLq`TOyV2_~F0r=dc*!zb=oAN)+>B4pB$DAUyQ=Yuk?leMJAPZ0&E_ zos`@8+O&HvywGPmsGcVk_lzoe)K9JCq|_m>b;4qJjDsF9DMhXs28dly)N#8wYh@n5 z2rA}s5HPacc30Y~-NAiL^A)rW>C92y;Xc^j5eHm+)dzYA0S&eMaJjkurXVuNILN#w zixPN^JN;^$^myz^_CXv)9t|AhYhf?;-yG>(;oL-l1`*A|dibDKy4qL8_?}nhkq7e7 z0m=cyarG_5>?jz@J1B?2#%s6#f^6umeINKK;^UJKnUEz-&?XJl48K1iOu+>qO?bK| zT-z|)$z#fJaL6NrVKUa}j6%QKhRpI?-5fG06m3ZnnVOYLFu&S9wj0>4CnvP8N!l7} znl>abH+aZD2)yQwm%THfLdB>3;JZ{E-*{%m!LsIoll&NO_+NWf!HQ$3%?yS#y#;5^ zypMZfNLzIg{NB(5{w{=3$g(XAwD~-8OrV+%SfNiVplXELh7Y{_354&_bH+;Ad*%^^5<3n*YE{WvEX+J+$1;lj@O& z#-rI70R!YBIDF2IDyGZ_( z@Le1uGHuTvNa?n4|6pgg1poSdpqOAR!XtEk8$Wk!siC_NBVd(QPA?cBh7Sf6)iN2NbO`6QHRZBMiJQiuJofqy6yOkPUnC> zZ7vLisFr;4_sYQr>!$+}rtt~(v;PUFMWD+kKmSi!B_bDTs>vWtfMXY}zjwb#{sppV z&(3@AvJlA`^$N6Dn$aciJz7%n)w8sS%@#2~v^M^j1CV)hccBE^i-AN&qnSrB%~_~@ z?BF)46*j24U$6golhvy3(kbg7ikzr5i8bbYvRi0L%X6Gc@rL0M#qOniLSHutXVX&n zF|0P}NALo^9d-*ksBxj&r2=!!!30PSY{}7IQ73Js@k-}7Z!b7R#PwPV`l!*WxbJ=340cu#*Y+%F4r{U{i6Ij|4>`Fqt*if&NbK%SRwgcZO($ zJfF~*w?Xj449nI=lISax)(|=DSb!&l@pA(pGP<*w2g-czMwsGI|4K?^Hm4B?TV(yQ zBj`0fS*=|gM37=EVeInAZVUp3kMU05Bq2LX7Q>#OaJm<5Va)U=$Civ@pLpaqTtd;M z#NKDAvIaD@gx3sg!UY_ZAXz3Hf(vYHDi zx98{Fs^f*23$(owl5`pO!tv`7QbV1)nW+7#u?=kdi5)I#{8|sKUSm0B@0yx(qtaiUq;oLxVwhz zI>wZcw^yfv7{otdNm%{J<}An(kd4=&fs4>AmgRWP_2u|A?W*0(`mtHm%M*% zCz|vw(yHm~>v<(mnq4T7_`_2|Ka5w=vDemv?Adzwv&iy|ZkPY}>B(`QX>yJXnFlAUt~S z(`*_{8>{NdYakaOvy~sz^=yB05T&%2(>QyW`g6Q9ulfmIq#`~^uDR@Ni`%DJ%$}`Q z<#sWEA0ZnFm3k<3bA{sa1=7O#r{EY(v+e}Be3p+|wa;x5I26B;CFI_Xq?E334(60y zktJw*%ble*R`lE&NBH*~*OVTw^vLgT>^gRBI!a=unuS4*UBx3jd<%nYQXNvenQgCk ze4n9zt#HqR>d?)ax5fO=mKCN1T`_}{{r0y}Gu-^JcY5il_xJncZQ~shD{mZ5MQIrU z`q-Hpb!52~u86g?HT*roverf*-6ub1Q{5lvG<<&ooz6c*bNq`rw>kcjhKb0T33#VX z1+3yE&ue;4Bp!IS_B|mOC*zqiH!$<;86Cf8-4&PrUu>iiv%g-O3C~5LONaC&X>883 za6e2&3*A4|CX3J;K43?anDw(6U}^Z;zP7j z<3FWr-<+;eaB}1#yabJ>k)TIZF&_N+1{5+jwFHQbN_BQeZ$$?@+0^1&#ExgQ*0NNS zP%5==yI=?B18yB1rGG{$En&sa30lK95|22g2kf?@+n)H2bQH84b?CW?TITh(I%hI* zreE%(Cq_{6RY+Manvtd*EVC!ykCGf4i*WG`DCo6ZmSdL+W%G; zw3N=3O&tlIuYeTeq6T6SJ3C=KJ<{~*iyyyvx#-^f?3JXm(Llj&3l2~c-0KCgY>NNb zA&;lv(xqE&fPDcyQb0zJZM?d1T+|ERi9k1B7_TQk0~AH91X=bG!&OHI9!9kfWSehD z+3DXKen;mybzk9%YJ8=JjL(e_xCn8D#Ri-&vDHBFBLK@Rpc8=1_*U<$g|u}K+1v0) zTob$TnZJEk=!bm%$XKX~GA0Vw=WB4oyO%b89K4s3G)BOxITg1_lS8W0IQ_tMLXs>& z6&Bb-WnMxD;>V=ZvnB~>QNmXk zg+s6<(Z_=eo-XT0@4|lTH$iaJrz2oO;Y8TEo=E>e=zUZ`gcl(3fqobKnmQ$&!7Y2_eL!?1Qn5eJq1n%=R07&+nY-9G5?sYp(0_nR(yObHC4X z-_QNj{9wW_;r#)?oM;| zh4SM^(R<9eB;^8b0b8WwZ}pKM`h3W8D_G^Y+93r>E7t0)nru#*4c_=U_kDKu<>OKW z-VQcrdE72{v>udFPfROpzT#GgfM_mzY;XU9wa!938>)o%`0rle&!3r_Cc#ze;7Z3< zTHZbH{-oE>Gz?4J(%Q-z+Drg@BK90ttd)I}r!X9J{jJ07sQdfwIA%Hc#_)%LL)sVX zq2r#Io6YS5$ZE*O1!=ju9jb_?am^FYV@NWj54Mvgd$X`6x53bys-PYh=C87om#TF4 zUh_v;?kYS`oAy~F&Ida(m(gw;-TYTv8SC7QfPxnkqLJ`O$Avd2BoeiFdq?=M_0eJ0 zKSSl5&xan@ig=9r`x|D~XkIhVkO!j<-t_tl%>ViKU&`~SmK(Oh8@&I6!e?UD?qSc; z>W_T&SM7&+F{O6q0kutQR_L1kXzbDdDsVBc5&TOT=w1BN`+d*ZVYr!Ncz%8>Zk-r)P<_knSp!o9NZxawgwn-WU?Wlm_kGpWm~#O#N9@G&){2ei54a zvuVhyW)#1Ici2wpz4v#8Ia2<#pF}g$1nRJdVC0ts@>{aWx<+%`4JpYdHE#geicb{+ zApIuokUFaO#FZ~E!>WsD-SZ~#dzkWW&xmZAES$mq!<4+|w~1;9FIt8*GTb9?IH%EJ8;>TJlkyAElhN#3EJDKCDSeQKv&h z3yOzTi}{sIpvtug%y9SY5^Gv#*L)^xrcwolbC2ow5xdIJamkf;!K#)Mr!5W6o3N?h z?hs#?r7iCir%fzfTqN4luir0<_>iw*m- z;f?l8M}%U$iA9afG%xjc`Ry)=w|7^YX@ZJ1k#s%4&%HX9QRa$J*JaWeZMT^sZ_wRO z&E@K1aSY+w8L88Oi4z?ouEqI)t2X-hPyiLz?&1EZSdDGfC`4s0Az--J? z@y924$zzZp*MsFgN(Um9pFHr~7%`1WPT$}+)QLz!{k3g#zU`J={Wh=i`B%H&(~)VB zK4Q|#sn$GLnJ~_lBbfF4N{P=)ts5a%kB5ab+2KKi3T`m7ZzIw86iv7Jc5< z3?apq=#(3ggQrlKrQl6PtAHxXk2P}YcJe8B+;(6_Zs~&jA4i=BkyT=H+Sr3VsOAJLh*uH06*qkrcLt=^8TvGT`*!J2_<0@nu`2u5s zfaw~&Oy(IKlA}0N)_RcMn1`b^4$vaTnmO_={a#G5#WGTxMhtS%i7icI*e_au=beE% zb+nW8QaWY*JVlS$fWsBVuBMYlRx2Yun5Tpi5*IpyW64KZ(zDm67(UdRVXr)l zmh*8tU!acB_kN$ScaVC=38eE`NTAEa76tE?Y0+bxTyDeA)BW&GKA|Sprw<3q8#l&z zj|7#{h;x_#@BmMt#xpX^CRn0~5R;lC|`E_h1byVu6=}EnD&=?d_Ws6F?lB38j7t0z=!a zAfi=sL)UkT>5vU&RtqN7wgIc9_LjIzlA)Jl$~a74(J_m;3F4)1s5eVd5bQ z5*mRaQSf`{daG+5*N=JDMia9$ljkOP{Jk^s6z98)sSOB4XL|xeI|g%E#g=~A8bJO;S|BIrMMRrU zy>VFP7R#pfImUK7EDIg%0H6^qLT9LNeC`iWc$bkxy3*l$zPq2|0~C5)w~c^XhizkG z*hh)FOtxnDblpj>Z@zK%TH$fiOx%#smXBN*W{B@Fnr+BR>fhn#9w6!+dyHaNU)kzW z9|*X$FrFW?dyu_h@(`s4lem9=rQ3kl9sVh0#)Tdlu@!PfPG6A**vew4bujzaOa-3P zdz);b2cPZ#$tR`v0SA1FT>b(~5i`Es1NwM6;C2=8|f^!-5?TWzzgvI-& zYfZ|bWbZ1}JRoQ!5Afwjpq7Qa6EL%>v>N!LDA)2^%h9uuzPzIC|JDc7ekspu|PFXAU*Xa}_ogFv5dq|y;Z5D546JBl01qy+poE=4UZ>9B$H^4O zV=hkam{HgpNH`KD(KN~^-pKBC$<&_3zVylT5ovvU5}<^I?s?j74cR+L z3oL_p-MX^kxiRDgH$!END9GOFlh$`3WL3XUErDO2P#<0E8i2@bg4JN^bHo9cM;p(E zuim1i5=?05jdErk8MS$LP&NZ9Jg{c-kg<9RY6Cugvtk2tkN{Xjc9D2=7BFiX3=L=+ z?q~QWW(lJTD4{wr)JVf5W3s9VdLwfl$e^6?ak(2&iI-7(;ZuYHyN|l8uSL>NZG)+l zZ#Cd-bOm{eAD2#cD`hxf8@50b3O%JIfCF^SnKKlMWHsX_MY-DHi~>@9YHfRkc!oh4 zBNG=$N}Emqh7V$Svo*vKPN5Xs4ZMUI6d1LsT6@4)ngRAB8TMkx$Tf!!+pqA%cuUDqsB=1$o7WOK zQ_{mPLsgnC>VK`79eLM%vqBC8DRo37Tk*#skuAzqoD$j=r>5K)F`yx16X?{YEDR4) z+rL$7iS0Upq!w0@z$2JmI6bnw(p@Sd&ZiESl!+zJk1hSSH>d`#5>Ys($8A^5Vbh(;VQ*{R!er2hv-S`v?KQ51R z5Rz#Kym%|KvZf^@qMBf?=!5m)*PewvsBs_maA*Z2?=<{eGo#61j!;Tmk-epL*D$4< zn9=sPljB>Z{jd&FA|z;L&CENZ^2cr3C~qRMD>)vORP7Rxnuv5`h%REfsD7aou5E94 zRo4)Xi+qUT=^{k6o(8KUm5T6soyx(8PmC-JFFmDqo|G!s01><4v|&;yHS?2|>g>6?lckJEMsv;v-`sfdu^ssiJ;m!PXU(yAQ z7MO1-eFa}+IU!#7cmtbb1JnvfJX_lYR9K>(5r**T(0-ed(*Xt}AxpdndxqEs^HsL+ z&7tdz#jUH9p!>28x@e8SCA?86f^7`VSoznw-E-I?DUFKB`NzysxuNY z#lkClPBIpPh|Y;f9h@cfZVUskN&;qlXv|zmryB2A>@=PQzE~NGSP0sNgfUH(zLRYn zW@?|Vc}1*XM#$%X%Q$2Ov3Af!vn_tn>1TGT18AWSSz;j5@j|22{cdv-$9d+!ZPwz$xK<0CN zWW-q{E={D1r&MN+*D1f+Wh9MM+Lo-O3@h#C_56@CP2xaxE-R89u**SCLlaCHtarqT zyMm-2IO`BY#`2bB+Wl2uMzpGTsjV-plETUer(0u;86Egm4f)3E9V9bzae0qSa~UC? zj7sD_qRVVt)&*vn!AcJKi0xb)}s`cpMpC4*X_XqO&kXy=aXaG{L(z^aq>MFeSd38@eN z(&*g~1q?ygq<4(Kt1eebkuTU{ItYt|7_I)m=8|_=>se66)ZCxE2uvmNn(r-_ZpLWG z{ElR`(+FLG;wz|aI%7N1Py^5lZ>h&b= z=-~l0@$DBR*%$CxE2t+O@>DMqA3U=b@9R=#Ruw2xV~1I0mR5*1qp;Vo2zn3cQkNph z7G1hxPbrv$Z9DoqkG{j8=?X~SlnS63rXRKi3_3tRl?Zl1ROgZC)3kYz(w0BSOyY74 zveUxn8`kqAqkx%(C=CnwxNPpvY4z43K(4|3p$^@{vnER;FM+1ig^0@x6G~Dtqd1X` zu|U_p!;HeI_+`Aqj5XygiS9_LiCCbgl$X~{K4o~q0;DQeF~jcKRBy&(_h4vt2?8Tl z@FpR9=srVe>tRa7U>8H7x&5Hy%NtvJo)Muf9GLlV*HFUwcE;l2U^6Ct$1@r_tXkF$ zyEdn`5vKjgWn89;^Je(|tBbJLjO z=1NP((e!xgISTA$&EWjznZ}k{#~gHTsqxR&z!C2v+9>^d$bg0M95MBx=S2d$wQS(p z_9B6NM!nZ)P&Ry=gw|&HtUE8FxB(!Ib;GMLV(SWhANN+kektd3Bz-U z_asjn;SJ_;mAj!IcGNdfbPT%)@l)?QEfJ|)4|!ug&};a;@EflLYSlXzcBKkA>EeSd zcOY84XFjzly@1izU3ExW;+Y&r+Bt@4<`M5lNZ?PhCxvpu2j;Q4y4*%VjW|u}R~3vx z#~B|m=YU19)*I;XBiiMd7?uca3SkcQoyQ%05XOk6_6%i{XFgM%lU01ze^z>r&lRDB z<7bFZAypeTO2A%R&qiH`J_*rBXNmYoJ)!~p3{GWCB3HQK3mAY0V;UnyuY$1@dF5PU zg>BQO$hiXy_8;TD*&i8aksVtxy-=nzLt+0$evW@);o<=2OvRYW^YML0csoY;4FlgPA z0ADnhT%?bTetzc8MEXf+OZX$(h0f~J*m>-v~><3bohW;w5`n*YBQ&2>W~+u ziuQ)P?bFu99N01p!eWJc#=!D zi$UyqgWQa_?q}ubj~I8@D#nMQNW~?u5fmj>pYLhwSq0qrQ0xsjOATCDd$=Qdh`xV> z<84mq#To6gqEQjL_;?Qhf6du}mQo-yBc-fPEIkhFTCIf1AuY6Eg=VRnu>{)=+alN( z>4CW=>MPwMa-h)7+A#Tslk$~y_*X;=Jp_I$odBc-8-j)2jfQVlfUVcP?^2Ifmg>Q(<{9_tuI}Wj0lE%t zd!}F-BeiLLcl9Zv2c1rY<PQA@2BtIDK@RTgtX=w<4JHe#)Zu}IroApjew z7DR^eh>IX7r@EIUi+RzOP(YcR*a!yGE!J{fz>ESShLZv_E)y{dZtr>9U(9IO$d6kx zFxG|q_>x8get5}8gs`ZihzF^$WK7%VrWwivbxwNTD*b!7)FT>e%o5MeGph?9!r67G_P(#SYRmP zY0;UP(9?cvB!oJnVU$*QA3jcZPiM#iEqi-Kt3d^)+lM`m&e)|=6={!=d1eXBkGb08 zQofjiVCp_#SD8uRGB2l^(95jV6&2{61>-*pF~6vC5_G?_)bQ&)tFjFBJ<>8)yAK`@ z&=?o1Nm^(gPhJuqnvMi*2RR9@ffuX|8aonsBjS|*Dn=oqBjEy|G0$&HA@3~&gnhL| z7_Vc$9~!h@grY};OTe_}7x115u4@-2rcwX_f8IhQ*O{jrYR08$T210ixNxcZ?B%3L zlGFArcAvMeFy1@3RB*k1xEbeQelL0YS%B-GQxRMzV?<)3648Q8E%}zS`2xFkV(aZm zjq>lX3o^B8&@IjlfcS8bMfC<-R+iw2Pm8BFS^u(A*ewe2iL6KUM1D^HHp6diW?zl&2$s{p3of8-+)vrkj>e(6?htqR)Cu;}ZETmXk}?Dxf(zV-@p zN3!HJE5H7im-^6DJ*L0Ny}<;b7tO0y^*8#(VqY=gQz%VmhKh!5L#=Q#M61sY%Ie*aj9b0eMBep-+YOk<{ z!Z66yL=b!K^a%ssjI*QfQei1d`+8j|t^D$K^ID%IDkPq8#y>4{fSMG3CoBeSGmU%l zCxJG-OB6I)W)=5Z`c-nhSf6?q)2HyMahKAVTGb~B9h5hBSgY|9_1Xu2b8`T<$y(Pl z34_tg?`Ep=`;_=4%Go^PRgHo>ym7_FI-3N>iaGm!RyK0S7!dnHM_ z&2Idu(iPTp2?~T_)4;=}R*aW=9m=6rMd!k%uW)$cM@@NAlEW(R_Y2F^D4^c*j{a1BVCdojPa zDsYpqSlxi%REfRp|%lH|2HGakW21XGMQlM%`=5Q(7iGf~O%WM;6L#{UJEu?>p zMX8TH<-yutcDDZz+X~z#qI!thclo*1ZkZoGEG!&)#*)8nRVfp?)1?Beb)EQKg|y3F zL@KfXrH2*V*S`8#elB}71%)eE>2usdtt;N0K*{KRiTUfQsKw`}7mnz;VOiqBT9WrrrX?&bWD=_fY8m>@tNhi={Ge zbvHNd%szr5>=D&>L}ox9q&LzIAtcR6*-Ia+diP18#R&|Q| z?fE3Y8l;!(?@ctO-B3|hRi4o1`O5oS&3yedccW#Qogl>R)k?JM=II9!{kI4uaRt5Z z|FnkEB&D}nJ$Q|^G~Rr^Aw_OVyqa_R>YP=x8`Gk2Vld^IYRN(Svj^x)#@cavu;{_C z)1FVBiQ;xv@fKw>Bc&MeVG_cHGG{V9SV4)xlMntuOGQevlpX<%1&`v9)cYGEf}%|6 z>>*uoMk_GM`sV3nDe)wQfJLg?JR?b1K)9%h5Hf}hoLZeTEEhUp#P6B)wLD=jx$%Hn z4(=+e{RNNVBQ7CzE(YnjKEnAUXgNJ^PPNdTjuA&sSnT9atMjHh0!c;u`Vy~B8L^YY zjN)yXyjrKiGfSl?6HZU|B_Y7UaIzBxzjao>$O|@DFi%0W>w12B_{S#mv)S(~JIuAo?P>1bFT|<#X@xi zZ&dsXsg*AGo4WjOl-=OO)@OAt1%GKUR=ltrwNQ= zow;zK*kvDAN$7&28^T9U(j%i?xaTlBonkehosc3nqgFPNR8MuIjo(TZ36&~~YU)<2 zcHE=w0V}f|Z5E9c7rQrkMDRZwchY;et;70;dATgveqnmw&T|1XLlxnR*9@v(gR7(t zu$5b;7h6`OUP=#SdN1oWY;Yfw(GnE^Mb(xKS(J-?`(}1*?97GlwFAI727wA6$@#|n z+*mTIzq;g|xw2au)FaLz8y7@AiCNq#QGuP@7W$#iGz9{5>HJK38e=>6r>vW#@r!(A zv#P!?=D0X-NfeqU-N zZS(s^crF7gdYr#?thzv9pKPTy8QPi=Yw2&6FVqnQu*b>g#b{v%@(Q9dmdRz3ZoFQt^bWsRY!L;`!8&RfCWhY?&wTyN!&_fV~39*67*wX}>no&31z1 znJn>VY05J}y8Sm;-*$r4pU;~gO`MKq<^L4oYiG*(@(H{paL3PGbBy%B^@lg*+no+S z!!X&O$feAz*~$6I1|4Ljx)DjNkU>?n@?>DQ*G60pSC*?iR@=D$;9P`I$DYQ|znPUg zChuAxQTw_9O}k{VLexA$*sqE62_W&K1;IeQ9mAf0{HVNmX6JW-$!n&!(B;+(3c_}G zB95>eYEmtEcG6{_x)jiDc32?lSm2=dud(2S29mQB+9WC95C;{u-UT#mC+S#}mM%v3*(JNpq0RM9YB;DP0 z-4%7d^Qb6gJGgTXse6xe5iVG7d!XCxC&PZ`FP3Iw^MD1qr#&beX?98254SUV-qoP2(-GDTv7Lf&evR!`{2br z>A*|p^uJKjiwhA@Y~x^D)e+Q%nPf( zXre!!`qM{q{l+iIX-(gSoZ%1;#c|~&rsCyxx#3D$?q~GK&_38NoDsMCUIXI%7skYS zm@ws@?r-Q%f&Eg`QGHRSUme0z1G>>~suA67+1Nlnsj;+EV&lx1rQa~-{f1UT9VtP7 zHlTiwiU(&t;W(Oq+UQIh0Smph{cF}?7Van7Djo9?>9BHo#+UVaS?GvSI+rUuj+bDn zox$>wg@yNzX@3!lECA`@e!+L|+8&)^Atdc};OB1VJMv6%!GWHnAF0rPm#N7(X+NLk znqn0Dak}loYW9Mkc^T()%klq-GO>csjae0{iseuJq{tFl_re$PW{97QN_gc8<@Xpa z)tlWf-}&GHWxy9<`C<~hrJ>(A+3z(Y|FBS+tl|iluq1vwDEkw(iP*gtjX8(4Akv^*P9Y>Vfq22FYOpjd44@sc&_>^ z6QDU>TUO&~ZVFF0QUM=Dso%X^PU-faDpk@wGUix$RTcyMSqXe$=WTrNu`S*8&r?sb z`y>lnCIWIv-W95ok;%5*y;CVJT+8u9n>8+*jev_v6fy`w-IwD$@kloLTY+)g(Zx)I zvbrH)oERG;ot1y%y`auD5#MgRPIYY>UvK-X>9j&~Kd(Uy-sjEbu8FK|C(6YfQ}kvz z$?+&x1iR3Gw*9VM+OEK)3CC8x{$9+KDssi$&#?~P6~Bz-n+UHl$iqztfmG@0`WL}5 z(`0o|vZlM)u@+6m9Xr6M3aJN|qQ4%l(Jtnf+TojPpbA8D5bbw?#l#I$GC9@N8Bw&% z?LJ956ZKZS{~uTpecnjw-|n-bG{FfJE8?lOKsuRQ;Ex+>XHHJa)c=Y^JUThHJ`28S z$D`8oRdTsr9VvIMFbGv} z2&Xo;Fx!s2Lt=zt%El6|KUco@*jZ(+EYY<-Kh`U}3ayf^y&alYQ@7;9iWI z*S}tT@#h?Y#_U~O37SP~KYt|5HK@iCKUt@|f3mfj7qgP9+n1=Bq4}pI4w1wZ1iz+h z6)GbapfZ6>P~q7kYf~oY>eFSVS!>ke-PDPIkTdGP!$%g=j_)xLVb}Qki~p7j|4Smw z@nlA`jUP6p{GZ7FuWc6zzQgRAXx@KhJ6D)FB#Me3ARs58&{*U~Oom$Xx!)99=i`_; zQnLL=7Jts^80Zl1kf4F7y0q@oo1qR?0cX%{8Q}pGK-Uiz3O2(jK)2!3?b;kFcH1T4 z$Y`?g@14u*Q@f7)nq{#|12#=q?@01(#_(Hipf1W{$_ot1CQh$OBWbQO70!u1`F#Xq znHWEx8brxN;`^{B`}*tGzXF$cVrI5G%JXPg`P^k(I=W!69&5TDrbM!IYVyftrz&p?aqpgZpsaA(NHU&s;L8SkjSt8R_*N-yo8E_}KDkm6w7lm>K7y{HEz4XFI z^W8F0hCwPn?-V$KJn+Zx`}teAgzX=^uj=4rYd>1?gpDQ(?_qkA?WQ<6%oOM9ykj}p zgCf6OSDpuf@+`z`+v`_`Q{dIoO6-b~`Ls)OyyT!fukx!5%27^~O@o z6{?p}y4K3yl4Z8e1~NmyoQc2e{aBW6%1UE2w>)*Gysqs+so5v%Q(R zM(fQtobB@|+`ceMPdi_^ZBe`uV4p{mXt6NfIOTt-tH_Ask%#n=H?wb}EAI{PhGZo; zy^V3pG_VvabTK&l3u0*SeD)3SRH+xNy`PLwKI6Q~p3|oM`xB_LF1LT;#PEtD5ACgj z($&3FHsY2Ru9t;DoN5lx><9ccCbl7B^S%iaSil3YWcaJM$PPDva=Xcz|H@-bw{A^F{%$W6nS6Ir zG3xS-ERaxC-lG$ksRCP7OFq_e&XRAX3e&-?hDRl=aILb3zv??x>-SujbYhC=7DZHx z?)iTYy%MA$J^TWV9F#(5e`~C4uF~c>dVAeO`raLS@y-YNEeqz@u*#=8lCJN4Ugl}t z27dGzsWME571CdUo5TgG6Q20gM-0ue5MtD_J&nJ%E_cvl*HlgaFbG`V;BIs*^S{ZU zuB~-E+-!=EjuL+@U9XDNn?7Nbx)n04&1;l;1X3t8yhPJ%HZo?z@=LR`#xjAYtZIzz zL+-2=wXLpJ7CKQ)&IPNf9p0J0L=NCdj=kxNPWfcLB5zZJr{zQW-{2pnw(1OiK)xtJ zV}DpT_ZZ4Q!ed9I{+;QF{%KL4?mn3p)$^&%8PVWRBNB_&&y=GNR{>|=S2nF!_v~HU;QuYkFh`a=+Vym$!>E~D zXYFcVl(Hh8w3f{ve-+){|H-A^nng+S%FbMf<`g8)&;$Wo;ahvR5%A1sa@i_VTX zd#U{7(j3aq-{HBVQz_HSbq9Ge;|{Lq!10o-kDwuE5zOMV0l6&x!)yh`I~SR(UX_Af zeUG_yb9c4sr}qwEdo)rr$rj$QmCX3VVsz-%WMb51iFYm)A2nj!!eEP{kI(3A)lELw zGhX$Y(z9{x)AxX*)f<)T%hew<+$$6eiy+P%O3=pcsKn&eb%*e%&(k^T!QEksMxr;0 z(Y>cn6`WA@A-J%Q_SDrXF~?GaoCD-B)_9x_m$n`fJo*(~+s`K}EpCpSzd$1TyXf0_JWW|5r*>(!~^ZyO@)2{ZF^a_1nzT7Amp69AeF_M>+`6wz`4?yEvlM)wU8(|6WXc5b3h( z0C%ldD`h@i8f80rwd>NB(PJky$8K#b(%M+ z!oMd%%G`~{WuE>m_0Z!omvwb|W}aPdK{j&mQ{Xo{=#QDC?P)Z{>>QW>XW0s;A2e9*fHX0A}W3E>{|10-UV5i zkxYWgOlo@u;;i-A`KxEZRd`5jn$O-_pyy|v;_FU6Xp}v_|IJOlXYFzNS0^;OmpdFe z0$UCHbfU$ER--Jfgj~FCH%T8>pwu9#sRZxnoWuat>GUoCbNjk702gQ$Ozd)r#?Y7RLLUYMmLJ9fezgSbhTQ3BOK-J+{*e}`1t%%n3Uvdomj?s z#mVg2g}K?R&V|bDA)z?O#`V$xyHJOGP~~Mm61$YfiKx$lMRK*5G+PjjX&z|?WmcJ; zq9c4DgRBhGsKPWW{hf0t-5D(#^@4+Z1~Iy?m>vFxJ%SA;A7@ zu82h3P>31u21}gawd1wF&qAjyCC)5d14=Nr_Fra#I!giq}RE?@GYP0}8d zIZ^As0Yp+{@&UDt_$6!pJrSMp1eS?bL6y!QKM##R1v&Whz$BLP&Y8AoEzGS!q7sXN z$(EiO+9&%Y0|X_LZe6odCmy2Rt`TiR`3~3!fIyTp7Yw`70v{WEvln1~5rTn+2G!I_ z4>5T0hWvO9EtqKC{LS#TC%G6mq&l?o20nFv_-L6cyTfho-%hWQOq+HwX3L-amAGJ! zw|naeD_R>b6=j}LG$hd6l~2yy>NY6z=NCQ})q1q(2s_Z~TJNX*ek-)?M#vJ*@ZQ+e^N^*2~EUSVji2tp##DKn~PK(V3?CF5-+W zAF!G=1w<>{g`sb1CCP8D=&~3O0jf~fX*VO#sIq4lpVMB=?~IMoRgQ1_m)&l`bQH4> z3!;DT#-Z+K0Urh(aM<@PjnluJ8nT~J*ReNukZ+2ccT@EB4az!ELPsM8(z!Nc>BKLq zfV4hZJSB>wCZsc!vvb4C{Z2n$T( X%**SSE}!Dt!~8KZykbzI?;7=gKllx? literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/config/img/free_benchmark.png b/debug/accuracy_tools/msprobe/docs/img/free_benchmark.png similarity index 100% rename from debug/accuracy_tools/msprobe/config/img/free_benchmark.png rename to debug/accuracy_tools/msprobe/docs/img/free_benchmark.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-1.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image-1.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image-1.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image-1.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-2.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image-2.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image-2.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image-2.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-3.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image-3.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image-3.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image-3.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-4.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image-4.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image-4.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image-4.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image.png diff --git a/debug/accuracy_tools/msprobe/docs/img/module_compare.png b/debug/accuracy_tools/msprobe/docs/img/module_compare.png new file mode 100644 index 0000000000000000000000000000000000000000..2e1ea564eb191807034afd8aceac92b29b62a086 GIT binary patch literal 56655 zcmeFZWmHvf*FFlkQ9|ibx z&GWqH_na}_6XT3=&iU|uaHxB)z1Fy5FKpJ^Z*Zc^Z z0gdEYpm{4K5+foz_w8BboC^7`-G&i<@AKX5ifxZ!eZ8&|E#2L#ojL7eL2X!D`0^cO zG%+L;96uxo6%zPoC!W6%D@?KOzdpGAjvyeA?%$W(eo}@j27NB6_~kz@3J!HQHt|R(64_4!%!X2* zNfG?>Qyl2K&JB^1rhYs~FfDS|{eRw4D-jkO{`12qu!Q)CaK8Ef!(n2HXjFV91j=6j{Ls$~fHF^So=nl+tBP(O;p=gRLj zia~>&w&q7A^>{|rsNSGwZ@y2G3o+#)>3`QW6uA0bBuPIX{yp>p_i*hb8<>0OGG%e z^OtH{$Kc7q+(07BtaF?^{!LflJy~+wlizc{T*o~eD+Q0IOnlpv+x9!sB&hA2^8Tk^LaZtFKJ#%C#yWKpda`66oCLOHA`4 zjObWTbfTjV>@}bFW1GnbNM5tA%t!Lh7paI|9erfM^&s@wovAA`eBO@*2}Ui_Lid+} zI2YEx{DMJvQ@8QEWiN{1k4KQ?_87~9eUlI(4r8+NsC=WP(WzfwTbSvVT`mujI<)%8 zoRlWTe>v)JCjT8==#8e#(uZ2IdmPiY2bei>>2f)zt@K8k4H7gE!(+q+xS7uPP6yoco(NdIab)*HDS~w2ip4mLv7rDL{(YpQs@m$a9SGqF1QTb3U< zKLn9Y&o$18?I;xPeu072q=L!5FKtiDB^}eW%d$nx3<)-9ihKrN#&KxL0g}VGsrWbA zr8?50na>2c*HTiM zKL3@sX6Ga?@H_r=yg)H&v+A9PU~*HK|6+eafFT5A7TIbj`wm}z(y)5BM4(1Og3kgq z+ZbuL1%e-t>Ufql6OkRrxwzLhM-;CoTUV3!|D8E$xNRaw#=?WzRPI=0}3Df%Sq-a#}v;<7g2Pnnpbm({6!l=j{#+yRkuF2<8 zTZL=nojj)#KKyAoBzK_|o!^I*x|^DtKZf7DEW)GzWpV1`oKNzePFZP?~?6d3z43P(;Hn1zBT z{huSS0V9}hbRhgsvjTuwsB~e7yZx7G(_jSuN0IGp z{n;K41)pLzNNct`?_g{7QFxXKTj zP*VH#y*%(f1cQL$;IMd#W~K{^OAk=h+FP>XPRq-p$HSj7R_e9RnPXvC-JMnj>~Y5^%RTA zllGIguuA94_xJt$Tu?aLoIeRH`gNBzXu95s(0eUQ)%rL5Q4J2+(wdGMMKqSe`iXGN*+V};1-#TLv-ppkhWjg6&5w)}1mPmGiZnFwK^>Uc!%_R~S(Z{NJ{ z%@yI@z{MgcG{Zp|66{sNEJA32Rz_3%40u7E%10d^OzM<(%72Qyy@ ztH9??uCRWRnSO~*^J{PvNxgg1N^N?$zJC|x{RbJUh)YRm!Ut32XJo`=+JxjUjvgY@ zS6Eccd=rHPTYU9vD#!igepqXcULZ+_BLjXf@ohKPTLq6yn*e^Tv<3EV0lg{L4Gx)A zo)|v|`j1xkR!G2D5FQ#W_$-J)J3A7FPo|9zzu2Fl>Ym*!{Kogt{X(bV*H#;RNzB1P zT${7%l;Xn!Zj1-N_O&^lEiVn?#;e8!@Jk;*z}j%6M%dsePxRPIE<0thC~OlgZ-i>v z=+=Y^8S)lK_}IRF5qn(Bjc|ne7M*KP6L&x!MlNlHNOx@h=LlDb%{pa3;JZ?8xX%i@ zmJHb;pZi52vg}_a=*f+jMhQ-p`Lyz(?4d zsPa-aU9si!QVZM$C|xmUAYl|MKNMbTGEMEvY_T4dcs|H*b20lBXz^l|GN3!y@Mx|V zGvdtuF z1Dase6sobNuKu@X`7=vvrl3>tJM`S3~ZjReD&lgQ15)E3!5sxoy+o7$m>B+ zaToaiYLILi?%q@1zkVxX{0Q2L&B*Gt|3rAHx{I z;F#x0^^YPECjMdL=&mL1!ZrW7-%No5%n2A{4N&= zRj>Sd>HTp(!H*3T#gC&u>)a3D_2T&&FoLyHKuq@LyHF*A%B9;g?{ku8t~;YVV#KVC zCT#jgS)mIls9B!BU7lHuSHeEMxL0H~5v7|UusilK6V!ku(=|z@b#F{GFp0d3KrNCL@0rs|| z51R{z>)y&Pd-Hl$zQpX82FFb!$8R))uPaouyx+KfALO*{`MH)?dOHN$Uk)e7*^ldPl^m z@f`IRwRiK!qHf4AyCn%o7csz45mUoXKjL~X>Rq*W}#y2k+Bn3V; z_YjatSL3bJqQ(2BwPVP>u=9zNdSiAEm#i4iPeL3YF$V!E<=cYxhwH^6!a_StU{7*n zERM2}q;9!&2x7<-36G(17rqg8jE9~hSs`G5SoI8Y%@1Q?07Z!hZ*zxgqqnRW- z3)xFmqArKCwZfva!@W0**018qg1-N%?0ASC%b59=IaTS6e|&<|%w1gh$N?8_6J`I2 zZEH0hoqbp~w@3&{)6sY(IKVv%nXfxu6l z-r*e3eTh1o{&ug$ego}~o`pB{3_IOya7F8ZvCEGa-Y}T1C{8(D>*@0M83q3Y7P^5jAHmEyd93o=iTGoREE9I`yvhc0+k`vid#(P{}E$GUMz%ID;yEq%%=xxUa zG7Bh89ufyhSHJw@kb z9D6$bq3!(X*yL=+QGLp(6dwg%3Q3mx`UdQ0lN)0N)bdGg13L(U%=!8<-|}X5q-yL^g}K=<;=@8R~E8yoxDRz`6n_0MQ>2p>@UmC+OEZEC;MJB+*G={NK<^x2==BXMa>Q;h;l zo*_z;(}v1%q3D&o_cDH#8pw*G6+fcdDw3Q{8A6k1{gtZ5Y)TU{-`dR_$+7!usXrm#)QY3d8BG>Nu^B&TxrJ@sU>ONCkqPubTs+9=y&QdV1Omr{d&Css3 z{Om}aw=^{WtAFkQr#Yc?bDb+YGv)=QUcDw70rtl6AS-J|?K5e?!Hm zZ(+WH(aS1mcvu93M~{A5iOD5a5Iv$Ej7&H-a8t{nnx>|FFg| zRLWBj?kg;}m8jSJI3Pb-H8r-=Zao@Tv|bxX9HYD5%!}4(*{wIQbJ@@y+ih5=_^6si zq1^yok|;2X*S%u@jxTfFRP{M$L4d+{;&j%2PfOVIH@R&NURRC@+4H>&3W4ZHZF@9B z7{N*rYk!)2JU)nU&ktY(6OkTE1=V+7id>T{oTmLConurLIdfz#>tS%BXT0W{&o4{~ zH7PH529|Eq0!zbs`|1&4)vlMU)R6|B8GY?BEzxUsE(R_l3dnQB<&cFY2Xq-~kK)@+ zOB9$qwrCd?WLH|nZPhIF(JMDN+2G;8v?2J&9Uy74V568=KX$p+u30#j$6Fe1JEX(8 zoZIu0{li+Q%IoU!Z*f?a4L)*06ygxswPuPztS9Dcm?T}X9IDn%5n^Zol7G9eR~bds zJ8rJ_V958#vahfAZbSt5+I_A(T+Y^#WO2{dc{?6@pVVYsDGC4mVP@UwhrGR0Gok}9 zxriB;DWklnQ$}tX%^R7Xxo-4_^V{h5X z-L7Lk{^@-(x2@Z8uC29k(sAQmx7U7A!tbvJI}wb zTBx9%CAlglW|I6esXQjFPaBJ>uWy`?k6tDa*s6s-GJ<4bj>@cQZ7l;u~LoGqy%j z6ULC)E}UitfE;f!rY&m3%I0(3`H^Ejl3A`(_8d23E}7Rpb9=tc(^*GCD!}WN28V>? zu-ayH`bpz*7z@o4>MT@Dq9W;VqT)$;!jReB(xH+$HjE~bZLLRZa?{e_h&X7Lap;Y-zqE(XK!W_ILtu} zzBC#RP|6U{CinW&wf(`sPJPPIDbi!F?PRj5kD}CkIQ?OH7+q*OUDy8fh>wo&@ucQ< zPB2AD9gA&P(bjB(eeHn;Ocf2Jv_;l`b%r<~!7RA1zd6$5p+zah@?HMv7EQoh(=`&o z8HGqB|0&pN?KsTXD{>mEy{q>i~paGef|o1U+ibZ9!Q7 z>~xv9gX9JGFJgaA-|NdwZ%QlF;3(va7o{F1A4R*>wNXyj?f}Y!Mg8T!9|_XQ&pKz} zIAoNz#ZN(Coom)sPI*c%uo6qhcYd^m9~ymqbuo1yS?ROc_j_O)5|DSci!X6q^?6JP zn`sFJ+zeAaI-lRj0!t-criT#d~8NLnpSM3Q9^E z(!9HJC|v~Ye!dtZ?Y0pY+bTjT>AZ-)v_VUoI=^IKN8&^#ocS$?$ z$!&i8=&VK(S`Dm*~br-iy{$W{`0Kd_n6&J}|>`TmLRk2XmhGS{<@mBRL2UCD`=-ZgPN8%kwR?!1u`V|;LqHli@>4I5kv ztVD24x^-L_0i?mR*ucI``|42i#>v^zFq7%n&G*D@Ob^HXWnu^>{~`4a1;67fXA!BM z&}-?A%*B|eCRf|KuzQqVI7TM$g{0f_;qjq*qvU%pjiC(BhE&m;H``HyJd^VoRW{Sj zn^h(IeZ|&Q*-lEBfZLhZB;~o(Q_lQp%=txH(h$1Ye00H=KZJbeude1tFRdhIs&9XX zJmTj~D=61yaeY^Er8}Z~yc&WN#>kc>$}<I)BmRR$%jyoFtE^eRmlZGR+Bzh|k%h;mKv*C9 z4vR|st1${1wC+U+|i(AIC zb#+$De;xE2x;t0=6%Xf`7{;Z&D|K_WQ&UOJC*YWt-}>Q6>9O{ihvmZ>4TcQ$Z*UAk zKl+~TTPb)Qit)Y`HJI#?=&!Px+@$L^K$OI$IRFtjf!LXk`8yRl7 zeQN`>uF}psO&gipo(mVoM?IyK*MEX{A$=En9hE+3gC-@uS5v;_p#)$T>rLVc?qEjc z!;oyNlqRL2hTg|_>R=DtaWgL+?XPLFq@m7x^ZeEfRzbAvI@@hAqjYJ1gM8di-`EvD z61_%q*(zv|njMmg3E{&Baaq8!Lcwy3ep%&oi%q+{47#aZ-KPMZ^q%b+wF#E=ys z6Ewy&)9HISQ_{7bj^6cx#d`7=1b(vPW35}h^TA8>QP2kNpG>oFe^@tK_hxE_^y`Z9 zKv+;Kzs|e&mzC49cS+B(S_kvg+lIFHmo7LDM%ougtG?sAp4qp#D5#d{mc7+TVZrub z8C(7m&Ct$fIE&K~g(_HJSn_sL~pS?d4x`NqyowU%x z?qFX&!6VZrNuC{5s-lfx@3vS1mBd4zm<-fw8;#&Fl0eXjqUq!uByb?adA* z)>(mxNIU~Md4}z%YVo*gS=?8Lkxb9hJJ>twvkY{l*k*lLkriCJ;;S8Xn?F9i&~%o` z1DUj;S^Whlze}&&gxkUW;X2#T+u~lQ{P*n5&Ma@*hH;nf30DVFdq4DA#G; z)h^A9tj|Rs97R@ul9 z8FilD(kozlD04kcOqSY^d6YIAwJLb_%!FaER9BQTVH=5GMTm6kwXC|RkGB|IQ@n7A9*@~{ zs2^_^q03wwFG0V>L{*W1U-tKXfJ)OC$qOz8NIrlZU)_-Yqp-Ylcgwtp!}S>$YR`7&bD4>8fP z_;%vflU1Ae-Q)#f!;sfYZE*P8z8*5ZoEp$9`Js{6CW(u?}O3m+0RgS$LGlvr5qZiS%U_RP#3Z zQndB$Fy??|dl=6yAu`QgUknUYZ)-E8!wR*p{le#{e?gXFx=0B*p)dR!Y8R? z8=jR2p!{y&e$Xvo8bpU5=^O*>4@7DbLb}B`kE450JO4B$tN1B^Qa{c!#iGT@KAEpA~LcwO6i@@2z3J zPoKV0#3iGarCgY}HEsNhpGGDgxY^q`xQKdLkUlly8$ALK+XLN>I#3;@CiL2 zGbffq%o7u547aur92nl8AI-H#z|6t+?hikyb@wi@ z3WB#F$b8P7><_rXJKXH(5x0=&^QS@U6#x?D{%|^bONESim_w!a!K{DbDwJNh1!&q| z-}ze)&p;;Ps0py*5JNzF+SKOssB6inpR*BqI|WNi`CT;dsR}y^tiQbV$K|)-(J~q6 zJ0gcm+YH1r$2^1@KhwfjWHhI#wfrqmF25yR!v^@=+IQif#qn;DP^DsMI(*wo;;Po% z5+r(ufxp7&08k#sV|yll3zTy)eiESf=kPrnv>yDx22ff;>zk_+Rd>QKRHF_kmF?#x zkAUWAdn_-w`Qmqrpm9#kGb+CH2M^vqgk0i~Md;Dl>OpLx0km07qu3IJ!v{bo7{01d z4&agsRQQ(ONzCNqqM`ZE+IJW!XNedA!HSlV=WZS0`v5o{#j=_TxREEN63iPZEO0g} z*aF=jut2A~_P(V)r`?n#prF>x@V!3U9+kw7^l@0fgKPkKCB2L7dm4y9Vy-uQuVsx~ zr(bVRYO2=3((fFHE&iB57UV=mIYw(OO*oK*a+=3}k;Z4T{w4t>AlI8=N$;djGRy^V zQ~T%H0&z4lr*BWTWkt_%so>;rK0{}6;v+&-Ck9g|@Pkz{))J@q&V#iOx8wQSBeBw$BPvjw0Giu!-5ffEA)Vv z_J53H`puCy3;;l8fv<6xGLSOdwx~Oi5d65DJh(f35ndQIpMsxCJkFmWq>KiR*v~46 zZ{feYheP(e_wFE-ngFuE-ngV6GJ-6Ri9<5eQBb2N0CXHu1_SjXCbEFN2~AK2;@i&T zH9IUSzMqi~K8R;Kqd3nO1S8!+Fy~bC$h~C=*$PbW7nzU+lqEWe!ME*F7=mzZg_z%y z!8tN;l97_K28wM#Hd__3k?-D;@(L&sdd%X9YWKU;QJhoX`Eh`6y+a9HB@kKwvkxu& zev43Ep67S{$I(aKlPFN`--@vkAS5*!0grP?0`ef6+~~+AXrt~Ic+5NR5X5{htD6&0 zJlffOx{p$TpMA(K3?YFVf`q~hc;Etvw!{;NEc&Pw9!fh-_TgW8jUVWUZzJ@6uOk7x z;fLu%hytV7hLHyv=dCC~1JCrE0(!~2muOLV*%NdSFKV%kUB&!Q_y~~$NzKn8P-M4M zbv;E(Q1KgXIl?i$RS4wMfn|fIh5iMHj9#oz9_>`ZZ+JFI?gb-4S^zldv^)QoCJ-9i z2}oN6E(2$6Q{B%O;LzT#{lVVNY4i zFG`>!a#7A&3DRd4Eo?r7_4xg*dEB&}$*_nRg9Tc?pJDgDi)Cw}5Sw^aHQ!uckj_Al z#$-!bDjIXO1g*$!#T;M-sDc%68A|v;qEo;h6u4^e-3#bC!WO*ld8JXCzC zT)7aS`un4LbEInhHI6=1B+GGtdHd$7r&9r-art!46?v}tcj97v@pFZ2nYvp!-HgQ$kMhpK{g9`~j zVVLPu{}Kl=vG2g{9=tjQUX8{jlE{XljEG#6odaC4c;4q?w`g}bL$DMM5o~7%Vm01e+hv{Bfv)Gl34SJUSvuV zEtl-nL*e}i4QxK&E~UL_hoez&nU>3Uo%?}sZ~1?M>mSH{A;GizUCmf{jl1=;0?#EI zlLiNG0H-v7))_=>T(nx?>s5`*x0N4XZPX9+KVoWNmsJO+Jz)`SIZfp01(c`Heh$Mu z^i7D{D7O+uNGHH=Q4WZ7sgeB4;Frd&Gz+2sAOUo&yG%%PI&C)o&-eOn!+Mx^ZKtd0 zOX_5!DTu5qeG<>76i74Nj<=SmA)nzZuqv6$+$ACl4n&Mq{eaf3K%4`(mYjst2%kk? zmRg!RV4{3L8?|_jqF7ga%Li3mD+_0i+UPJqcw>w9KW)hCRyn&tVV ziw3Px$xozvZBuK32Y#y}pKHi&4hZO$pZK!Ekj|1Y@PmL?Z|KFc#2+WfBN*|42v%DI zIB8sEC-mU0s^8_I$h$;0)2;wi>*H;h^?a*)iRa1h#FHtPds|$F(Q0+OH7S5Fw%$Ry`y03yzTlM2&Q+p<&p%i$a}(CyB=5zA+#Oj(S`aL4s? zX-uw>!Qls|;g>N^5Q$;Y6xP9fo?olnm8Ju|sUsRO(%sVN-Pm8W7Cs^I4kI+s!?K345JF=@;Z%p88KMOHYl- zK{ub$m~V&PiO=X+`pTH41#nzlAgr6HH0ek+Eo%5o?P~&*$|am3XV0rXjv!ha1BE6$(H|}NO%rOg$wCw(sZAb}c-a$zyZtfE&f21ch!!Mel28(7iQin~t6Z;l;PoIHR!ddh-#B=tsNfp8 zE=+~vczyPmZ&mq={A|vv052d#DgjKO3=1AY^c)<5Gi^L+86*jbQ2MiCkhjd7ZXP6Q zg1-<2+jJdeVWD}xJlp@M9NO>tv`I$i#UpQ-XbSRj6eGv4^y*bW(Q9brdvy$u+`rWo z(4MBZ&|eljPgX@}xwzTCf$ao`N{AR9JI2Cymb7oxS2h&twX7zXEJL>B`}?5`8y+n2 z^SOBq*jw_Sq?E8&e}%TEmy>XrN2Ch5I}#{4IdW_ee6yAMSQJ}>`CU zt=DZ}v=IKrrgonGe*SuL;9=CM4sAvqfY8|)r(g!+F?(@3Cad-@FWx^O#j_`JS`(*a zP;L2~qxUQ9b)8gJ5H4<#G++cnZHD6cLK@=}vmaC1VHb};OB;G0QQstXn*E1Gd4rPW zDLQmUw#ik&w(*>iVjx8>8ghS#i?v@v<2Z*(Y?xiqyj{X{Z#%Tc{O`rNHV~23FVOslS_Jd9KD1Q@HlBSft?~ z6KI&{4>ngV#-paX*S(f>e2SmDYS;@h3|whr9q0!R6N>U-A&|7jl7rBbrz`ph9gDft zHrC{KK~z4Y@)kF~cs}-$Vv`5{K3!MNmq_zC9V&Bo!-}T!m%ZvzE1(G>Pn+YAS)1&U z{cP;Jaf|#-5oQe(#DTy>oz1NWKM+(Mj7Q@tArfCZPF~~3Z;tU`0Q`!c5whBG1>~8-cO%kp`y7j6%tZ(dbuIG@H@8tysEMK++i~(n4uzO z((|!)y85MQ{g0+sU*jy-qH#Vi>4Q>R+_-#J#*a%In(b730CWPdc?OE{Ky1X}x;Z@Z2q zTKgHLJ4onL(Xp! z!SgEo!{jVzWzawK*jtjrBNQNCS+JshQIW~*5%wX3cSw1gAZx8v_HygFJk7+JH4d#5m-}$FAj8o#8;Sx*-qF#9Ju){X?7T?9b5)SsG2!CR|AT?VWOKfd^Gx- zxMBHCC&K+*wbay|jij1K9szT>UAWQj=I!Wi7D^UJ<85&jY2F`kT<&-Oe(KNLV4Lz% zqW;xa+4|R3OE1I7+h79(WaCjvJ8Szb=LH__Xjq{`fC(j@cKXwGuaw=WS<3b6!%%?# zp1%$_XpcEnL_eNR&1W-O*CVvr0|%rQJUe>C2Q{7a`5?J+jFraQadF9ttB|#1;ap;0 z2R|*eM12GPj)HQY`+2&mG*$lPgh2{8nR<>Vf@&8z7qr;owNso2ANzSNs}@_k1W69x zEi-6f_wXv^$K{z-!v92PuqM#+Nx5lk+9gd{v&*<951g}@wn2CLz z+-W_3l|$*2H|g0gb{kA;6XiRsTBjdk**2`dz!>l-TQR7P+o@#a`$l{}HMsY9cJbLs ze&6}W&{Eac65Cu}VTw%ka805tlO91)7QivYVwYVpLC_{Lea)0Q)glr%3{C$0Md%K$ zCM$l^8(|p-de2Fg(+XE3SWHNHU7A$fESh{R30Wm zD=FJa--HW4{-b_=mXc}x%8vpr^RmkG)E8FWed;i zr{SYGK4i4S2I3X*z%OT7U=&)(Pl+XvHHehptu{&ea^c0gMv@j7f_ z3q8gc(X4^<>EffssYLce1m=|Y(7XL46@J#g;UqCZLq4d)`NoanKY)`xQl!Mx=G+5E z3|p1ZBUBUt-Vy)(?m&>D-z6sR-?v!)tdLEvYPFTr*4xz1SUs155gx?Sok{Z2X~eGi z39%yl{SLuZ>DcN84{G5*&DNt}Y(MK&}uwNrWVboJv#o6!h zyOfct-kHa@e0Jk=(y~wt#~a!X23Qp4>pDR<+@GY*Xx~dSdT+m0D{nZyu_jmQEmNYv z4DnAu=rd_SWxWnk8Q$nDsjwRT4#P6gcQlrk-{Q7sY`79to4{}b`ZkJIgRBlhrWzJ} z3~U$LjUJG2id~&gb-4CmegoxdHO?vcPsQr>@0q&N?Vn9GAn__YAJA>bn_D;*u{w`n zt%oW*fjYxAEAIWGWoBT)pJY(X?{`hu+kR)8@>F!#|3J9#vhC9I7^Vv|a+@P;vpQc( zGSLxq`>h2iHshgANi2S;IcWETFQiK0AaA#W4D+qrKmNuwGPykqkP;Y$=T45t$?tLi zs24^F{%&+-p^VULT2eK_HT&LIgdkU36f{O<;TbihJHd;Z->n%jfk00kTxu;L|GY6;swr zP|^XOZJ@RrTl7-WZu~#(=XR2 z2=4bsG$mUCKmvy}a}AW!qTGJ!b4E~#ir^vtnyR$d4JIPjhaT-rYudu4-9X8pqXYR8 z&e2{3z$`*^99&NQ=SRDAB7kE;eYgL)|m36X>Re83z`>O9MD&yft4 zSs26!LAyUqLFbZcOU);OkvH6-6ZqJG<@{uCVf%GLFcO4}hDvn4UuS?dD+Y4UOy|S> zDyPUHe*A!Lm2X8bT=>k&8roRM>uT*Z>^z^)eU*0?@Zx>$_Q8?W11C@K({M(fW$KMU?i?Kx`Tm$Mapzph3 zEtoA-F_~g{Tx2af)!{Qh5Cs#t*vV7jUrX3TatC~4fKWFe+eHd(l4|)aB%nW{D2|g9 z;0`Y_d+J*e}4#RrgRBdrDW|%D4_4&KuHH1n2t7_H_A)pbNw!6S# zA=}EdYPW_xcKKD_Py*;(QGpwp<(0)iY#rYDw|Rri?`^?;!$&F1@@YN?va}+pSN)K4 zxbh020Em^P&^uA+EsGul>NekS#HqW#pZ^8uj9c9?oM|;R9maA-K(|oAfLC+ES;FHs z>lb!_JFFS^Ej`J60Ff2w$_c#ax<@HBvf~RvElg8YxL}p-ix)uMe9-~znoDJV^v$+7!whg-3Nm_Tav`cs3G>ql)6NRqa8} zh-DDb>q7=a-s>(Kex43jXE_Y0M>xbGoo&2&&WV9C3=ai7+~OqWaLs{c>?N=|H-{=?JlMK0o!<2yJEtAt|#VUg+ zmgO4$LD=|R4x-W~icyh(+jPB~k(Y%V-~BblOv@NM!51*wkh!)>C^Rb30GCer$v_qi8^kq<^L zaF&q(;3k%(&Sdu}9FekMjhBzZr#SI$uu~Hb)s17996b)&d$3eEb6m&O;Rpis=_*Mg zG4P_QF^`6-3s`foNVZJz&bz?Vq*}#XY>21Hk)jT`1L|UEg&rq@Jpt)V7d#^nEWb^z zR_vT9EJPVURqVe%1Rq&y-OvemT0ej&vk8G0%SUr<=5N>2o~3%0CfppnR-A0 z4w=opee@?2Y$k6s2Y6>=_6QRJZ{!z5s&j8k#|lNDi%}*0xs|BHurdnh(vG-qP-7mM zcK0P{K3gO}?qx0fjGO)h(QW|UBKXiP(A9nL$leGv-a*-De%wLkg#&vJlSa#`pMY2^ zF__zV39}aFY;baN1E`a_QFk!*Nck`h%gZ3Po}v{lh(060+XT1{F?!3Q_9t{_;b7fa z&wG6;HF*@C6DJ)!5*0*?o|qK?^b{l)YF|KPHKhpr#2NuI-yJEODG7qwpGYJC&Erb` z52XM90Mh^e5hs%askfc|2!LP(SexgGehbjI8XfJ`3=5S0Z74TrDA}4QB(^4v{8#U$ z5=d2j*6sv!SH6A01}VdR3lMY!R0uS3uuqX~C-VlvN28M8cd}pp2Z-KuEF92X{?_yX zM1h{udXnjQ=1t|Z+7}&He>GWjD_CJFk3bd!r;`AK5f5k)-+B4R}nw%^^gCET?&}q*! zvvJ?@K{%FD0IAQq6oa1Pv#+=9FD01gf>b?46o!Am&m08>5d^~9qu3W5Z`&H3W0;hPs@?-s*rAdObGz1SI!53ql5@>urM0KG0g3em+FPu*7p3fxxr zgX&n15>We=+S%;iN8tfk$d7D=46EDNJHwXT^~N}b2C^)vfIStJ=qO9g2gTpVkIHEu1wf5cfu8b? z(tS178djn`_R0&iYav<5Io_YnDK?}lJvS;;c&dEl|$ zu;HJAmtmAt7`2oXW_l<@g2r5U!$)jC4sNyM+JGw!cw>O&QnvSWv&$Q;lRrz{iC&Dm z0}$hIVk24FuAIS;t=X=cC1T4*kB()!_DuTP*$NjX? z+(t{BjEmJ84ggS?Qf4kT0rFDDHpem6n^jL-e#i354zHdLaQrI{B|o7Z6ESLFmI5fp z4?Ar}t7GYkv6nKKp}4d%MQL`;#Tmu=5v&IF_q$$eq_Pu1qOZpjRx#`uESKv9KOL^&r_7|H;we)kHmDkO|zSVLcko8XNW zxnG;+2l5*>H+Ln$yyM2-8T82Iuwdzpc$mkann{K_k^Xa#AJvI8mf3oh_1!$mE4vkI z7sW&ozVcnz?pHl*C(Hzq!kh-tI>-04?JzkDTYWJ5UtNxMa8_ILVZ>)?KiUD0*OT_e z(0w%c>jdbI``-Lgk8KR}A2$=3+Fca`3+nbwOy+3<%>D|gm&-lRlq%a>2r6Y=cJL)$uggMt#`xya;a z{d)@G;6W*oqnsg@bK3o*50B4O9I32=x@%c{+VjUAVzEsc0~)7?Fg~9PcYPYyTdYkl zfYG)%%p{&Jy;d!L??@q7r0}WbXg1HrYbm~-k+6OjVne7u;37(M$%+_XWSo#Y%n zZ0{}Hc7?Z9%ff0Hp*LWY`R9GPA^`ZlXxlgy3#kWYWTssGV?Ic!;Vfv1m~lrpl5HjD zw_y#SEuQn(AWH?WK6&Ryc0bAF`-*BvoYO_GxhEc|k9kw&_5G9A9Gcd*!yn4xMrszSMC!DC zE7WqZY_*leQ^<Z1JX=6#yw61jf7?pZ1;gq zlGH{CjlE(m2bulA>6aUE^oqHO#JHmJ77}p`f7TxdK>7E3@M&9y3N5#^al9F%E1}$i z-|WJKxqct)y-i;q$)ZH(fyiD_uqB9ePl>esca9a#iR?J(^0JWh`XZ@%a4a=1)ICEw@>+V&FTv;CS_tqm=kyEcz>9%d52L`46vDPKUV293zz~URSfDNOx$B zu+lcY$efO0)7P+vH*Bo^Y73^7*V=TB@GT5_*W@w2$*fN^&&^w`>Gk1|V!zOgK2*u+ z*9Suu3+to5HKg{fPXr7(g1;INlI@>_?Q+4?=UYeXRXLhcf7BLLYKFu1f= zp-Q+^6kyC5(e7!Gw6dv34V;h~084kbUhMQ&zt96sqJlQ_m%k5dqyVFqHCEgX zLojMrm*_K+pu!t9br2!+k2z4ae~ria6n9FMcpS*iHS3ggUXQbtA{~T^7L>pUvG_?p z0rkt~e)+o84;wD!&N!RIUq|1mid8!rIeupmVLL9m?=OED!%tj|Ka$xyjc>>|yc}AK z+$nGP9Lt@P)VKqn<1rnEtB?C|ZSY)bFf-Gyswn%f3e}*Sgwur6lH$>nec;s3c#26}Y=*1&{nAvlFSi>=yoLB1EJ|OI(NQcd`fwETLFEPi`c<)?J0KAci`0&2eWot;8 zsiD{Ym+k67_HZGbE(eer%1B7oYr;T>Z1q7#ld@tZ1zip&lfXMcd0O zcvJ>4le|il9WL&-4rz_}U+legSXEuSH>$YQ1u7{b0@5K!gLFxYfFhkD(ny2SNOww? z2nd3Jv>+iM-K8K>Qc6gdBAk0JefNI%e)rzzI_IDBedqfAd>$urjycwxbKK)s_nS$w z9;bId6=z}oOyBLf2Z`*ga96n{`LM_{1#Ffx{|q}%aWY}<55i-Aj)=G#fmCN-G>pz& zg8jd>@FpR-bD`rqXNN(`&g{wgpnlW0;4dFC>TALW=z%R@%ukCgNogbcvgM@_a5CM}UN zs<`pKJ2g^9aSWkR%#u*-aAj@}5@6IJT=X1*03PruYm+gNu)!|=%jm2Cfc4M&+HZzi zZvKgVaFGsnO~ou5gN9-Dp!`@=<>xqq)eHENt*BJojedmJU``X{H-!b64Ho3~WG`j0 z-{n8MO+|-ZyFz`2VwRz@%(%;W{_Q=0eF4y5x|YYIa~`=c=hgm@K9qJx!$^Tf0px4F z0_mYnht8|{slWK1^JnuV*iPMRqj;PuxR7XMVWVp^cbCN@pyA=0FaMefv7Jr$`)ir{ zWWY+WrFxH5yjP#_R+x6ReNGL?eTIeBYO^YBrml`Tv{MJ>ky*cO%zvbcdlEfLZ`(?0 z%2eGpKU86~g_uif$=u6_3W!10M-1!UPua1^Tg1b(-(4axI#$@~3z`&G{ z(^7VKuNx2?NJ(k|@oocCzxLLO)7^duKu!Z4uhrUM?TCr1VO6eDOez zBwoq*Not(Gz^;fP@gaEj!OK$!Og`jzaUpjK+CtUtfM@PT8Eaq8TGZ&*U3}yD` za!$i0KRG!_7s}jIvwgS~UjU$GEu;;7K8M@-y0)~yQv|sk++Q16{)8CKhhu@HqUFZ( z7C+TW$L}f-+jl+ZxYB9)qgjQd=+;6{sQS%ItlHUg{rmuGZ!91F`dC$;^P_t4Dkeb( zq_WZ=Xe%?M2NLGhhVs-EdeCkrpyj4Wj|mv9rw|57^l)9$Y5M8ed=L?;{{|OhP{^O* zT>mxp$f4J7`VFO+HWQ-w$d#%R9CjR>RR3M|q{a6vo{X)dx3loEKGe^jwIUFPV|-L} zf7;iX)ixmORDbnFtO)!(hy{Spcz9(#My=pH>wOA0=(S0=1*I@d9^q%?;R^|+WsMVu zkIFqCp=zF_VV5=y{;wD4AHC8pc_3Z>l{#!^XQ4V$H*BXiM&@8IQ0!2G)u3K?0yK9^ zzxSpzfGa3)37uXSUQfh3nWL&}>$v<;5#9~!B|jpiJ8hI+wSoX#jN3ivzQxFYxoqXj z17tBUWHcFGLL)dXEnB6POw4qDVEt^@vG$!0#lgiv?KcW~QurLqgfE4cAM640FSGXb zEl7Z~WKB9lkFBfcRHK zi%SbK0wN(6k|yEFJ!}-!9o(lG$A^oEvM}W8$SuY4}lN6T*k@ z>A`rMj6zNE-$=k`|BTww?LYg8L7Aua`WROLHY)2J&;i774JrVZf1Y2H{8n)agOod2 z+;`_j-Yt_yTOvR`*aKK;<_~Z(OF2^tALjGhE~GuYcoF$E??pTai-m(2n|Xgu&D_v!)9GUA;cbjLRcH9@H*)*`!Hjf_{u`W@}S9=eg<~v zM=q2%$-TuhVb_n-qG-g40p%rX)DTiUcW~O4jRbCt?J~Ujjc^(?!?2PdX7yGIAkTuu z22Al?GE1*u9V)IVY2ikXQi*36`Z7Dc{4aBjsE(0q-~+Ji5wJ(S{TXv->w>%-^+3n- zDVJ3qT*aEB6ab8?98l>jV(n)=U?D*?Srn{G4j~9Ce@&LNRUSBrSi?ppG`|AO^;kd3 zFcGuFM&FzfM3&eD=JZ^c=^T1#0kpU{KowDE|7&`k>bMq<8W+hE+$Qz^`m*FQW)khu4oqKGUQDJl@f8lz;yT^EFqm@}r`QB45ceq!`1Das0_ z=#502;Lsm3zx?j=)C8nV*T-u7b`vN7xCSR0G?t>xzwljGF?aSP?aJjTuwSI)YK^<& zo!pE>QjfLaK^wqf?OagA>`6BxF6{^)PulFWtjn`>P?L1LxqlIr3Y^4@m6;C*#uw@% zmuZ^@g@~U5ao-RS0E00v$Wc^O@VmDAa{Ym6lA?1SEhFRk7@i9ZOO)4SApTjZJ50dZ z0AQmDqd>N9*8jA1`=3CeZzx2CsIR7ihf*OzFcMW~i2Qw3oJobE!e;}#(bj0qXAz+Q z0?!e&p0NaRtd6C32zAuI2hvq=lRx(s2Ldh(92Z=)w?O(}R$*P28FIt6S71>p{Ws(a zyv_@3dE0`c!XQ_A_7uUN+efBz* zJZBcn;LaevxazR|h642F;+g>ReF=XI2lD-p$vM$=)yv%*tQ5UL5m=hPzM#^o`UaYv z5s29khH1|?i74rfMz{#bHzgc3TUI>g9VanA47>&?NyLP32L##S;c8vw)8ha00k<2U zetQ%0dAaj@Z5=LUE5do8LlDkmMh4+L3gTN%pMBBHY;D7myS016@iH1 z@mf2cjvt>wJ=#r6AgwCFu-Ob5rUaM>?D-yLoRM4T;~x*KK^w8 zLLW;;B9;erR!ht&2He!(6dE&4nwV3-?`F?;`1cehm{TK930F@r997+{3sZ zU=%X8RD!Pq-iIPLf+mu|*ebkLLSQaIE7~mpzRv9_GLE;;1qM0uojtJA*o{oM82<`e|#D1uY{Zq$=~4k$#vK@`m4Z4 ze(+*VTK>Zj4eO8fzkd9n$hYhhQU2b)qr1ZJ6alxOz^l)+gX%;b$@56Tz5gfg5%Z{I zz@u^)9A^~o*~96^!H9&E!7?a$5CQS_4f zA=b#7X%1Xe5&M(Wdjhoyj6@uXyxR_3rKIpl4>}^`tft32i^1LEh9lQG696ZMVe_u# zl(2z^c?N~)fx>W}#wWLKYcpKNC&xX@NHTQkx+W$QWQ5oA@ClUX*}6+PgvieWpOi_M z?_Ya;RZqPwmf^t^pZv;< zqKb+G&xJbkiM9BDXaVe@tTHwwX&7up(t+4TpN`B0PWNM9)wH$!#EDQ4cHJ~cl)5gs zfaV{lHkEZC2cp>{?Kel+8x1837g2-XEkI8b-d^vqJJx0{q{pjttZkmPy8~!!X1Swj1Y7wKu>b+&gZ{HVU69I;irfxc{w&`M6w3K&~H%H+zoye%Th=hm%)e zxc`}>XQc&8rHrlaCH#{qX^+J(+Y$du4fEw}7CyPC6gF8GJ#wB;+}g17{nu4oH|&Iu z?=x?4sL_PhFg4mBepEtEp?;k<6w%6%NGH-m|EvTY8qGAi`PR2Et9J6&Tyd_GsIy5Y zk^Q~%WY6b?Uk|Z1?ce-ohXe~#>^i(HDuvn=%*kQMz7DYzE^QAfYozems1q?NsUkV( z3qnNyL`vlC5msRnF-T3Z>(!`=93O5CG|Xo$^FzG)2)x``YsJ+ISLk;W*(bmJckYOT z%>Uqy@P-hmR3=eJ3KsR(ci`cPT(R8AT7Ck7*981d(*vgE$!gE~{-^}zNMAi)x@n6d zwodJldT+9r*6|#RJ*K1dwndqe(iWH6Rr%{!65|9Ui1c=HhfecIhhg30*RGo0J7+aW z1bcP`*)T%j6Kym#%|i%%T9p!q-AR+guTt3d<^6z{i{(J+a~yW1LFeI!MH|J*6S4gl%y<{rfOEQc!_%O7k9%=n^fdKgj3+ECVYuoD_b=Ypqqbfs1=ce-FY>X9^!%EuzT z0)5~Gk9c~H8r+Wq$1;V;2O~8t;a#O zqjc7RG1!|)rs=O1deg`X<<4K`pdzHCx%trTZWb{v0S8xOq)m88vvOHeW^u4ho5e-B zx>6b+`ugzKkjh}i7;E{E_~n2pf)z0LgNc-o_}$@V*9aurQ~n;eVqeCQO%1Kcc6$Cb z>eut}6zFjaPjzSc?D-h*k%7Bg*GX8$!K;o1JwSY#lfjM%6(3=`L{%fq;5uhjTO z=3&ScfpeKvCz8hBYEP)-;3-+UIl0+?Z2p3k(`Wwe(Y?adnIIe9!IIl7{Eh7X z24g*vpSVazVztX2v08nNopx;uJ9q9HqbYw(<*8t0$w1t2b=JA4x8&b$kbYxT+f17i zhgf!VjYD_&EXi5zygDG&gIO3LL%h3;_NTXNS+>$=72*a4d4BUuhNzjW_vr0eHI!cW(D^<&o7&L@%6g8p#X0+K>GzcE*^= z_Q}?`^3Yh)CUbzO1NfAE`u_(WiOdAzR1hG_7I5PUuZuqnbeH<}Z6s30N3zpGXD-3* z743KVdxE_*LEF^s*EDQs(&_li-wxa!#QAVNvWr*4pqoF$ELG6SqI}6%G$%xI_R-qh zouB6uezIE~{H2r4Ns(|^T8q*yG_5SO|fgimmH(D2_*&9QA0IpxR@z$Z zi$sz6WF1YA0k*Gl?9Ymhh_|1HD&hDJn_R=%3Loo;@(e$GrUj7C^vxvUZ`h9x>+lwY z$iA{%b&V-z+*}}y0KY<=^L$DE0iz)#RrE%taRNYu`9<$*1{TWY~;rX6lfSW8sfaUbykXF$kHBqK#plqz`OKLCd-4%ps`cl{P~G*Y|j} z69*qMR7RF?0#h$&R$1$9@2fk|SXCwkXv7rSr%hg=+xbqi*YbPls|2nvyM8Ud>x&ro zn8|MxUYqgHLK~`Uqak#RKZaqOFyZ&^IQ!?$>?xkfmGgm>I$Ip4jYU3guxiQgej*}Y zICrK;a`xw=^<=j}BafjEBe`7gX%lUszo>&z%Nk$mkn(5^GX~+g1aQ;umi2jEx8Ix5 zG?)*O4d9&hp{2jC&;pzO_}ec(;{YVKa)!J3u?Jn+42OTH%NZI^eJOPgw)*`CPGiOi z(!C+mZ$TtYn@K(X3aBVKt%Ke#FLN~TV)BwAPeE6Wdn+9k8&&Vxs_5wJJ~drDU)l-Z z-iN$W`ZAhp;R`i-Km`F{UpC<;`XSwQH)*`?c4HjfMM)QJVvm3cQQyyGe`Ko+W4Rif z49cR~9zPs@e|~1--~3zW6W|;5dzzXz5Y7elKXESCP;DW^WG~dh_thCwpgBcY`vLwz zyZUujXUCM~Psi0J)d#<28yGywc@@02^Fyr;LTp}rhUB9136@Evt?>|6wI0&DG>%9y z*FTorklN-o5O@LQj=(9FbNPl<$TFARO_20~9U|R8bn;1c??+lz!4h@^uVs3%;@5lX z20~NRH)q;u@l*=bMaGplzCF@jkZ{zK3&r(_5}CAiFGxs*4A*rqfrb)p5R$O=tjgH( zm`VlrW5WV2X&QQrpGO0z3$x4!Wd?$6X0-^*X&Z7B^cPT32=H7bIa}xI6ztcFq)us9 zb7)-Owg@;>tn~0W{`KgzpLuam2ogzSsvu?SxLT6{<-?S*aCXGB8K)D|RVt*|_2#!D z#fk&{&D{fhq1ES@8$$mE*TTrkSeM$`XZcCo%0b#387es%x1_{*TyMo)I%f~L zAuXYlkM=I>JA@AJZy4N3&@GiRfBHh>!FE10``Bjgm~PClDzk{iYCFCp@sG@1xF_(~ z;OB86_ayttV+w=UN*^wZ+^Ph!zOL{V!vUnO6{wDzFK%-`tf zu=thGk5T*zrpVs7H5~$SoyyvgAI3hP53if@zf=-Yg4DVM?UJt@=g0FkXXK&zFki0bPMmX$;eVp49&qTmgh{heG zAr_O2*?zgXe=k2NzwgYmve28=!&lyu} zhMij*wlIWoB5@yT6E#i>dRpeh8TDxU7>~cJV=XIwmfn5U#-*>LyI2r-v zrA0f^OTel50-q{1#rjFCMnZlhdyu=3pdiAsaITD$!ViS*_HVkBSsuZQu`~Nj91u>UHvOvqV>|AUX z1=;q=r!FA41L%nNVL2k81c%WbiYL=g2Pz8;4!@GzJ=k_gTJi(s-=St{Q#8#Pq^-bM=asx1^|-z<1V^V%I2Q>{xCohEJ_8tu5)oPv0%fYBBj7@INT-r#mINr- zB7TS0-MV%h8f@DqZrFX5;(fAypT42KW9H$+z;Z)}I%VCCe5GVZ@v5t|)DoGXquHa= zQYge`^=z0_EXxp=iaLEND(WXQKd8Bi$+=_lsqS2T3haAsZL3hfy&Mveb1FDyD+ewr zP5@0;=Cx6c(JZ~>EJG}~Dmj;kYkp?0DsF6Q+qTCu*FDJWGYqo547)fwab#yUw+1G1 zM0y)`G|}L7oOex6|A^viXTUTLG&*X~h$Hm%r>|1QR$*5~Y9=lfkWXyC$j|Qm1x~k!4;bSS z5X&RqqLaA3bBLjF*_%S6@y+Bwgx4N-VMUa)>>qZ z?0z;v?Q%BI>h~ywiTD9jwmISd2wzV2{Y7$Q_g$)Xcd~`L;Ddc6qet=%>38kj;)_P{ zWSodK1efo@N+vP948H1X(lYYO5ZNk{r2OjEXBB$bWe+qp3FF16RxS8Vo27C>a&k5T zc)l3HJ0RW$bHBYv%NxM9%*3ZZ5GjZ0AMSxzSgWHU5RY>5SwauC!BDFL8-xPcpLnui zP;=r<4l_mW;Vo+ZO(S_lHG2Hbxy*dd7It(tJ=J2xoVyGQ<fFj`!~b0l8^xpo{*Z#A+AwC*Xv1s|(&Ou&cuMkz(TbzGke}4GhMdMPw@C%@ac6y_sif=g%eTZBC_s(p>6)d=Ic~bP{F@CQbaF9FA3^fqnd2Ud8*R~ z1W{D?ePKWs@`sb+G4xM%IM)0F$otAhA(3liE_{gN=ETWkpuhG>%9ypRkPo+7=d+t0 z61E6kT1yAO*)Abk1}5MF;$)@zH>&ObO!ohtlkJHefl%?H;w`Nq1pB*H4HPcrU>OZ@ z2@ry}?gEypN^7FAFPqa1T8!2l2_SYt@6+?6X;Ga>99yjDBgyxaaM(g*i>9{pANe{l zcMg6AM678SlK5o%gj+%M(q=w%JtE#N;!oJGDCPfHjyjnEM1<3I*Dx)L+1%nv#2K_1 ztms{_4{?F#Zx^wRf9Xpq$W;A9(llZyrPD|rSgRetW)WwZNVdM!Iv`dopn0h7oB#?E zi^|jrt1+5@JONBb?GPD;{~|0B`R9BIgjnOJL5Q{4cxrxfygO_Q11^h*jSV41o}U1& z;k0<~C<^l_khJxSI^{S?Ek#O+M1!gNBBYv~6hf-CL7)l$)#tk1SzEn`9T0=pp9H)f z$2+wI8v{TGOrxpKeyf!X;u1xeth_S_*EM_j&GeJ5iM%I zluQf4U`99_vnZfhR6z|%3G zT)6H<3w@LLZh)(C5p}sXE${>w6OlQiJGt;W<~cc}uoL8M{-06BT&sTcV#h6P#2s_@d{ZFg1tx1_AH? z1uwUMYZ_9v)4oqNctyH_(%xco4JkPmZIF;bfobnk?vJoMF*KAo9V)S`37f}D?%WBiRWygGe;g$`;>}n8D*QY291uT` z3S>59#yE|h=EFa$NvkwmMFol)T}KS4fYQM)$=pho_ZDDD zeM`gN6il3cQ#|cGyXVF^2iWT;LD9JQ`HJw9H`P$4^>DV4XZ<*L<}XW!89^+J!1jTE zE`FZ>d%HFYQxMZ9T3SMK7{}D{(VpeTDO${%cH$`wR=1Y4-yz~Ik|kfq!kWami+YeW z=_Ewqel}xrGrlU(>u0VP7m_Z)SN|1OFnDG@D8k^vAaJZHh)DnD^BY)5VUfgR9^>*o zC&Bqm^|E5(^uj5M z$+o)%l9w?7Vjv`;dY?M)WdY%Dkpx$7K6l2d`eGlN#Al=5M{7SnK?X74^#VYeSni$n z{d+Uc@M|6Rfr06mvq=&?tTYW#3t0lNv+0IC58*h35OGZX{_(g+XmTUsvqvMS9WW0+ z1-ux?h#AN$Y#kp<35^&e7J?bm<*X=Ozh#=tH2EtXzGwP-aL@?{bpq-tB)@L6O%PQ} zZwp**5-1VbHrCD=?E46;mJikL5Vv7YO=S(!>)A%xIP=(jy(khjM~;>81KsYN4_Vj( z)%%d^W>s1kqWl;&cX*BxnILJngAjDZQMy2ZoAVe8?*i<6oFD-W4|^1~*UzgP67(M0 z+R|FEDgyOhL;pT?gjJxZAjZ2j6?)8Z-5j2$IGHki#bV-JD>HvN2t2btO8B0+HavOA zyjmQ0#rN^n_kkusba$H2r`MV#P2K4t>Z5b z=lTSfua|)XVUA#ci!;|NymCh$ka819DGCk;o@gE`5n2Rq{(d1H+6^9(Sg)?OkJRjw zsT_tkH|uJx?j4dsw!N|*ykfj%COuq87YqT|iT6OtPY<;1aT_ajwmJZn>btfdP)0j= zkiiC{X#b7Vb-kKlh;X3YC?1L|oQ14($?0xvzuR$eOKADlb{T;DvYwU=yKD&bn$;X@>*jU} z(pHv$l;&nUtIkIS)EqW03%QjJ5_mT30oqT}1ErPD3$?`T#;8D6puXbx<;r6D+lLk) zEzL^Gw%rtrhsy-#3)T7O*3MnJ&P04>W43mymm54K9k}y*kSRgIKaY)D5<71`&TswY z?Nv_o&ge52LtMJe#P@w{d=Rk1M;Y%*e}uQmBA9VP&>vF0GI4On)ahu9!56h~u$ZBY z3apJafi30&kdSK?$y9tJB>GPbY0Op@A)JFJQg~Bt21oYi8C`YI%FhJEU!&g%MH6m3 z)L;wQRIVra%Y1!iWM7AnJ{E`HWlp*N?L*396P9X>HekgmdC-azTw>Ig3|R?-jwhbzwGsz5cB z>V#W;j3*|qOx68laay9jOUEO+A$Scsu0OFhl`j?^ZJsmM;J2Z$+UcWIw8jl1-mbUnCDnSE0WtjvLU$zC6Y# z)lnS=2oha1kYFI^X4gt{fnW+Eu{qkCZAG5tMC2+Tf6_F3QJ`>9PA)ZP@M*u=lwe6ul01aXV!kGfc-nLV6_mA6r)zWeJ`J%r+!wPuq*NIgZ=EFQ<$gL7XQxe zkFw%rTf?7luKVAjia+eKPKbG8+-K_uG#HC^X#uhNea*(`$(zQjojnyEgsMN!@G0A7 zSLu!Uj*X?3{Ua%iAAmhKk-?9UJ+Ce{xs|>*R}@w5%B&WDclUnfz6&H%F|e@Hcy9Lk z%}u<|)gbWel4neNwx8x160di<#8hPkM|Qw)w|rEgP46fzd?;1Gfv^!`dPH8daGCfC zzT}&*L7-J05jPr}43ouRa@HPp2^ee{U-{UcX!g`tl-|Z0zJpyEQICMEdyNB~;LhRw z-L5m%bdQdIoDpR#i)Y6T%6F{SJeo5^o2fsw_f&j=tgsUH5ru^4FLD_e zd(PS$!)DjcDrj_-p7-W^;_qIZ;O!;(ce!T4Ps+YzE13fU)-FoHqWth>m32pj{mWKs zv3J0U#5V8PC)VC;jP{6>3oatDiq>omyMJofwv87-PS@vx>s*r47@aH28xrB6NO-K& zlwQzo9auX2P^=+!9M<))UK zm5w=m`9X;1xK|gt=PV$E<;r6Rv)U2Et*?_LCc!Xfj;tZ~V$7Hi4})v`Vayh9cwx3Osyyp*? zKaCYisAPE98B}_x3%5UA1SFC{Sv5v^lB}*wc{0X+j4z>Tt7-&CcJe~q2jl(qnGSV^ zUoSUaaFc(u>7r$2(kbb+?&#cVh?|ycxX;Yn!CE0jdlyIcc-se)w8| zd^R3*ypS7zHReJieXC4}O8m&_L_z2awYp{6wezytZ?pTvANGZ|Pm=N&3M>7RD*wHl)@x!FJ3UHvh}D(|vZA$RVd;=-(U zWE?C%)tete1D>^=bi9q+IIr)k`sR&sj2Df7J2H|XIdw8YRYn{EAL76g=}rgBg1Z&F z=mErq4Ta09`AB|JXs}eC+HSMIx8@*qncF==Y0F~>G4^e3F)m%Qa8A!_9Nbp{Q;kYu z4V&New<{DRuD857n}^;@eT_$Mmqz}DW(wTy1LIrox`?QbJIh|sp{Sn!i3i3X zM^L@N!U0&blp%rY?JXE_-2H4qv=oe(Hu;X**d62>@PyPpH9rKZfmfZfOXi4_ySgOD z7{4F1bD1D!?jr0*rg4l32y-lWcEQ`B#{vhbf4S@{zK*)g?Sp67e*tq$|Es54(#X}K z|J~x9hTOQ{CF24HWH=G|5Lly60AtqYy=YoX*Z=6K4wOJ9RTIuzrHY7r_Zz`C=kETk zpkyN&%pW9jU2Cd@pAT&}IERbv7~=(+x*8X-b94jZZ(M0L_TUKm82(+u{O|K8tk6dmWG2XN}*qk>3``bdB*ii{7zwL7u&_eYf-vVnW!T6~LZlQUez1K2L zGfsD1x>g2@N(zKdBuBc>(Jp{Th!Z#i^Rcg7MlzCiYM-w!y#8Tpfm9Li5^dF}kU644 zGuy>Bz7q;7+_V9Op$RA(7C`XH4f(IiQZhej8v-VQLYW|)hwkU^&taKncvM1sglCyA zJB=u$+nPcgJ!&gh*}W<33a>@rT2fg)Hid)wV{*)gM(_4pW_;w=!biou?U+=rPwG4gbMw0$FK@T}kAVUuW*E zhIbR5mC&@FPWJ#dGBboxb{laT`rq?Uq9gc9I8B}6ZtCe}yY#{P%S@+x5eHr~A{^h> z8P4nR9v|#I#r;|Y7E_JDhhzfbTUC-9!9#Q``fWMm&`Ydb;Q!~@wMn`0=$5+B9Jkx<%0v1OSFf1* zX9;_zc%8laQTseb1&Kk>zj5Ps6^=3me%SL09eS{zfDPvHji> zF8}n2EvTp&=Ls+)P&Z`WsrFtFA@a%Uo5{q=z&3NfnlV1nTUA?ExN9@EJSR@;ll4n3 zcT2faz1_%Wt#(Uk3*y02AoVyU!N60X%D&guu`iVc^6sBHoANtPIB9G_>d2LYlZ$W_ zL0MRlSc zKe7l-PZejY9sLFb|2qLC1cYS7kVS5Hf{o6q2e zo4&ZqaQyIG3veJAR+f;)H|9gF^9G3lcO=(g17zrsdL3GpbB)o^>(CiX8#Vc1FF;t? zlfecZkIpdCqw+Ubl@rUBX0EIA{L9T%y6WCf|7J$dze0-V@*>Xa9g?tK4Z9Hnay?+1 z=}R}k%dAZIt7;w-OZL-~~#j~ttl5qDYuEJKly@BQ?7(39} z@M(|2W5Y(a63xaBOA3|Xh&-Xlr_*qj#=xjE;?LFL+g|atRWktdiIS`Or>Y%d#4rh4 z7f!Z@ncZA9mPCHe8RK@ld}esdbD1e$7u)ww8VZ1Jhc?Pp}vXG}##y86@4(#B} z2#_BX0JRKH=5A>w@>yBun&Vd^25btVN$k}S!pL{Myqz@If9d1Ub@kDunJ{9spAv#=5j^Z&A`^3~z`$e-H-@pE$`JN{=^K-~ z&x3;jvQr7}_u2>4_jCAU;S-#u!e>8uX6=cQau$vnP`+vH7bZZ?U-94s8r%6$%MEBR7uVNt{ zF|gIf0M2*I#~!+n%$yKppvK36>)oO8&%gkg0DIaXJum*x<^K1cOjs3Qb>=lV+Dzzy zzPy7*kHHk((Qp>;>X)r54SkoE!xdApf)T#3zm}Ilgly(%xjzO|K7n-wj^oK&8VR26H&W;UpV)gSDtiDE>jBls_|J)A1F^CPEg{c8 z2b5u+r~d1JLL0UaH4>=Tu3y5+$7Ju zy&bToeogLmgAV@lPwMd^h&7j5tcPGlWHcBSAQfQ&{!$bGv;zcdq-|Xgv#JWRBR&A# z@9dpE^-lC>oM>UG$oCn>O1XMbW2d~FSxXV5=5X)lz27|X7exPh5kV#!9#Ht{Ca{Oz` z22pWPSnaM&W$f7uUpf6}y2VLhQjD3ZS$(H%yC|3O*JIw7n-6tIdTwV?xc~YlD`ru@ zTT$%to&9e*VJ%GhVI&OY_%bvRv0z*W<8?ClL6RVqmA0e~nl51Ij=lQ5x{6;u5`W!4 zLU56zMd;F z8juvCU#4kJ@zsD^r5^Ygf-EBN{g-No9*8gQh8W`>NE31lyTgLA0J22w)SPIdxb61I zs|i1qr=gw2=9Ij$h&wp$U8Mrf&(X?+tqwJu0Z2%G6-e?@jlLZf%@GEgFqE-~}?kMWxG9NkpEOBZd^?Bcs+mo4di6WY5z3go;T65ws==3P>7TGhU8yRed+snSA|D~59VSQ zfb8Lfirj?;I(|T`)awK^6us*Fqm@>l%dfrq1)Yp;^~we7RX?rz+VXKg$#xW4erN~# z@(2YtLw7O5{~p?xob;!P98&Vy{qJ!72fY^J_+G;$we^{SbO6u&pyKdNB|_gVDT$ewId|ets$f=bgdB-mOtLkM(7u5bJ|1ZHx0jvV`5q7L#Jh~ZY_`5 zTx5;@d5NrOH`QBl{j$xzFrMPYA1w$U5dj*$1+ce}%^?|uGL%_s;Q7W>m@S&QTFRX8$ z`mhI^Zs98Sml{`B3^z+z$m1&tHPsbXv@Q$e4u(%h5fhCiGuq+XF|tS~>f42zcL!N; zp|eXcs1qHwZJ7{K#tKEgWoE!Cfoxl=hqL3arlV4z0mvh001|GptFgPCX<1EL!B=5E zu0HzE#z$w~KwVQ#K%hCgibC$Q;|Bs+y1$`Yxum0i{4J9>vY(4r4q{+~`TCCAj6aFJ z#_i2ssqDUp(Njy(zd8PYh3r3*wd~5*_=&UtnX;n_HtuHPbXWOs5>rz1qnQ|%X`zT+ zvOFDslOA{!<0=LD+EP<~!7-IMV%D({92Sn(({eVMrOxvUpGC2mJ$zFb`3J{~RvK5V zddG1Y!B33HMgV;WhtOsbLn%l*O1c|iDT|!a^uiBgr!U<7D?T5F}=A7 zD0Kn4X&wZ-w+@xWq`QiIM!By!`)7UQaMNDNnPoffZ8e92HTb@yZTZ!K2c+CLDTY#h zR3Sxqa=J)#m2%mZR7Ctj0u>B)jO_D@8|C1dReRBs(G+(#T7c^ZUO~T5jjIL8o=EK0 zHA3>;-z4@%NbFV##BMb}?AAp5v-%n33^2CLEofVRlzu+?SScHtfMi4G*GsIg9z)>0 z$h;2ZZYisdI(B=S`b)X)WKY~Hm#WpWR>>{b>UDdV5fR78HT1{XVpNsUp&;d7NY1dP zYWr}CR^`R=X&t(FPTJ8EykG2)H;_lzOTG}!2F<7>|LDv`}Xj+9yL+K zm5w$XZFg@z*OVNr|mtQC|r|i;)jS5Pc}kqg5YSw|s{R_n5_2hNS`!+Q(C|DIQY#?Fy~Z+?Q)NCj z;wyadd)rCV)3VxN2pOw-XIe!!U-yZI^`B$6M#y>!^?-VS`1yuUUx) zekW|+TrG@L?>tIZg%tC0sY27G&1gWTHcIOD=SWU&D)U`c!^YKP|6L^FF+|y=Sl35j z*u5?w*t4gIU@Sd-8F@N+V=gbwD9kE=spXd8QVqUOmZhWaw`(U`&nurI3f95gE?w}0 zM=#-aCYB_V&H>*A%e;lUTD;ef*mSVbHEV3 ztqkmQNnOvA={1HIZ;2J6Hr~>@6IFUYvsZK1rjy9fHp0f^YCBMz=pFBBGD^m zfW}A>S+JAkBYXzCWW+r+1gcUykOSAX_z^E(W%+s7&Rpi^fmC@d3~s>$x$39o0op)e zqjD|j`^}G_lV1zc8fpRQwg^R|MPdDe^Pt6xfA5PYY6i^~Im)1%OsuM%XU$<=bL?hP zyXH@ICk6w3-8%F1fgvGW(vZ+@Z3hvYfMTQvwldw&4}>rfytfE@&_Da7)@92TSJ^~l z64FrmdiMX8)?fJY{m&Y2twF9+W;GCn-Ozru=edM$H?X2H$It>6?WZt}K<@0d>pszx zgplBAG}j=uTM}9E=*}1?tuFJ7OK+;0VmChdh;8;IWo(Wozeo~SrXL*(Et|6p3|wq1 zdC-Cb){ni!HLv4eFTIYB7FtLQRwQhJ;Z6YwRZqk83;`%2lix!X`N(rCWorA`6!cf= z0n>6Nzs2!Cv;Y>+V};)VA`z0H19^>xUbj}x#z*xifr2Z;(D096B{Ub5dB4xy2DF{* z2lS8a4Cw3?_H=}zYD0^N*bAaRr<%gsyV4M#i0Jz!=g%q)TKve3|J;mZj?KDu;- zeZ+H7r9QMCzo#30dmuu!8^SQGpuLOfBuJ_l4Y*C6ZfP~RcRQ%ZITDbo4Q8{gq5U|X zz{0rz9R?b=Q}v+vqSJ|-YyF~M=DM?Vb-_7w5OhsFAYic>oL`O4 z0rWo^LZ=p6&4Fj=cntZSHyBfGeFzNZpeB66?eWu*+jjc>T*IT^l`(g-WRkW&zAQI; zV>~#XyBVK5gsWws^QanWO^GlYu?jcGRGgp}T{3tKn1aWlzq<*NfKcgS3=_`-C{)qo z-`IhG%50(_J?qleqS$fu>S>|X^4KvT>th|RQvU1@>;_lL0C#VEm8y;W!fOf_KD*u+ zm-;087}KD05_THpL<(A7WjEUjKKwdP zD}m%EAzI!x=m~x!l`?Nh z!IfQ^smY+KnvUMd594Y9Y6a@_G%_eZJqmMTCKmoVH_$HaC0iD8IxKXL8NR#cd~O(6 zpqP=k@(iu1`3}}G$=L_otR}B*ZxzptHfV{>-*;kSA+LAB9qVm`UPc_ORlGGOS$54D zYAL0&Tn-;o#CCa&dk5Qrb5s+F0+tj%W*Xmn_lzJ5RVm+CXK>KrZX5ALbCZCO7LMFplBMm4wXwl<951d`WRW;jPCD^7jM)xfo?vyYgC z+)8fPfG8W>bcT;9NdGu5`FTR$tz73ZU5(`s~ZRR!Z8EA*7L7Ad!`F?Bhdeao1mJW4Wpn zp8GQ$76I;q@A%u1KXC z_i=n&~1l448;HR^4+kEx}BiY}p#`K!eY2MFODTZ z_qlEqQbPoZZ|YxWT8)}N0u5-}Z7^WDY#6-TbMhOt))0;LDrz?(!7_w(Zb z1SuYt!s$O+AYwNQYn%lvXkT*rGLLS6wH&|f2yh1@n8 zazAekWD&-Zb?n7(88wa}1&y!>v&EEGl7Sw8Z}{yQtPQwQ5Z|t7xh_&TiAr6j97ebd z-rQGjIib*L^@|A*e|`zB17ZfbbAMQf)&Ijn{NJ$<$ip8wJfsWL>|mtOMfDsU1(J|_ z7f+9oE&SWV5G8I~qYejwBI68jeH&3y2}Q3zV2KM&Ib`Xvu>hA7-{ZMTBq` zdL;C300^}phGG31s>s8mA3&a)kBlaNGuky7v@Mp_R_3HA9w^Yvptn&eriQx<9evKh zWf!pt(Mq3gLCXLr?=S&hVbhYA9ZzA{GDnMPcLL-C@(}Dre;fKR!m}(04Sn+ekatVa z(nbSi<>U2x*K8qh8?y6b+C#lpyqBO7Pz8xWE6 zxI(R^Af*2c(cb<=b>{;35}NPA=|6z?GdjRo37`2sK;xe6 z=fr+`2hH&$knq~!pMKh4ut3_t+gPoi$LfV*PYCZM(klVX#PR?H(@O|IyL#Ib6tVbd zf4Ck~bdxd$C%)gi$p?8h|L1c5f1b>fl-R<*R?1AkSG@4Z#!V;E4WLQe%{4sFbzc#$ zU&{u^7O)kWHX>Tb#!!+p)bCS?)t?iP8@8OYp?)qi4UPm3Bnf{ES{mq(`JTDJg(tEd zl}xF6{x25^uaufpd+{0pxo68M8>;DN@2};iJ^+41tbAuCZOuJbtM*CBJuyxsI}jeq z`@J}f@YKzGz)_}tt~9-nVsisBSNMvdR4*!obl#TR$vhqIu!O7qcl=Er(GwsSx*5d| zIx2l;g{BT7NRdn=_RAQO6C51yA`vS=cX&*9weSY^>9%jGCZT+XUVS4&pU z`zXx)Qr(dPE;T5M608-6L8tV^-;-fs+#0YKbECML(EJrlyS%2Ub^-sr2L;UgQ{8Q% zyPRbZS_PEP@~6H-xP2ZQk7XUpt<#eK%tAq<8`;ZtO$wl3n1*f`Tw=!ueUIN90Dsf> z1$GA&Kc8{}+mi4Pwk1Zbhgve=+m$S5G!u};Dp)#(6_811s6vcNz6qMPs~iRgof_RB zhfd@%2Qv#F($=PXqm6Rv8N*a4VcDi09>kkR9X?xgDa@GHE`8t$Wl@oah) zCf$kA`=7pp9pkJA0e{+HmivDuu=d}d6}=B6U;OCR|Es+#52tcn`-auFDzhX*(UOpm zh$S*-h|+`gtNv(NRN z>)ZXAHN5M6pZ9t1`*;6_eWqUIab3@eTBbYxT1s}YBae%E6W(T?t$-Pf8s8bkd>_?G zLQ7f2#@Mx3ZWmY9UT7Sn=!7Wh8t}W*Rr3!F2Z_peR%%zDnoV@Q$d*ySiPx#~M8i;8 zT~6Q2b2AseSP9~4AP+aWYYONeWUN0kRerKBkQXKX zmV%kJvrrY16ulJIwGT|%tJ_?iqXn2V*L}o*=Vp!qC_%O1%la zu}{7F(md^xKewG#kO3uSy&=aaI_P#FPpST@i9$yu^!ME;tf+Lg=fzu}m6`s9dZQ z-_7XpJ zWwE-suReIToR9h#)+I(giE>URrSqNFb58OU<)7#8RMaHwX@^;8Bo5MXj>GvC=kokZ#Eae9M=U~g7cjsi?pvwzHoD(G!7uX= zWhps0+7<8b5k2U<6b`Flw86u}Ed}yU5Dik&yfHd;WIVx5Ha)ywJ$mZ_hpn&dsE=G7 znz*hVa|my{QaC@Ht#Na6^r7?%#fnZKWGL$*_?Ki8pN$N^&=j+g=f5W9%>i+4l(V_4%xw`qWX| z4@KfYRYGNxj;F5VDK#(H8H-Hwest6}UNEWd*dm^5T5{R-xl z(nyCFN&B?cFk4&w)gRws-zP7(-7Q&f&KJDCA9&)9dTw!b- z9>Y*}yS`u3c@Mt!YN<1@(Lc+IU-&}!5K&W)!;>b94vrn4i{29y!-FDF3MdgMAVkz^ zZ`{-h2vb~kt33Fn2k}B4`y~U#50J-x`4%OQy+2#td!|mZ=1+2l9V|B5wQS5}chk zsQGzgSy!vmCZ`{$mRS_7Mt!5~Xz5jlAIm{MN1j=}7 z6l>L6-8cDJeDgZN#)Co!n;&d-8!u_9D!3LeX?YCy9-O1-3W4_T74TiXEC+Rleg?m`4I@i_o5BGAD=isn6Xo!@N&UDRDEXE>;&!(%wxhWQ7w{&tA!D4K( zD0Pm(^9PD!m+5vW5TmK9bi^5}JMiDKP%X03s({`8i}1k2As*LXUyWXzmlN65ktI0w z;8%f9Ee=r!(^?$*`IoL|%SYVrU(aJNTbYD_IbFu?0T_sG-yAmSrWX9em@x1fk8h<|M=Y73^5{Z?*2 z_-Z)l2&us><;Hx&xszf`qa^3mqO*g9G6+MV=T(sJX*3&IQR>cRiwp zuB*kYJmVbnkC{Syi>uFlb-nvtl&w3wu~y2$Z%f=6cSP#b*lW1OS`J#cR84DVI_Sz7 zO`T3W&%AMKIB8IGbKD$f;G`>yVRZ>mY{7Yu%r21<;_aoqho91*HbOGH0s_7jia3&&A=7-kK4nk!qc3R#h3Qv+ zTq=FCAd9=hsi2Wgu0%tC6f&93X*2waPE%`lVQ3+Rb&MgI{d#EF=0nhG4qBVyu4$t_ zGGf<>?3T_+`MmvUA4h=xI%0T&6_7yr&wjh?k0?1pyM<>EtuI8E&vxL6qC0%lqD1Nj z=8Pzy113GsgG~U90!aU-r49dua}uT1E?}ZE143{~{o79Z7t|DL!NCLem4VRD3+Mk4 zKg$MqQ4++@cCEMkPR}#iL}0e_I?Mmloj3?55U=q$#r!Q6h{(l{f?i3W$*x+2%8c~M zg4tT_nAkX&+r<2|=LcFe-Cs(Je$i0%{Y!U3;pOglu-VNoHMQadS7hyn2@S#b~OVR8@f&A^`iu$5~^cqyK7CT&s z)Q(8J4b!7$*aG_8r`-lFV`#akz0Yh+bSE`ffKl@(l1$e3A!&%kOD}5MYth>!;|bBS zO}9={rh*EUE)O-~%Y|MTYjx50$Cyr@5;#m=h)Q0rg^Y88JfwSAz;_$~JK$rJ+ugra zkAmaZmnd5?a9{6Z#X~~~!jRd7+8`uB03K2g62Ba)eI1}M_sD20xSqaPw4<;!F|Das zX9`_XAvWqpj5d7XBBOwu3lk5%{YD)3n92E7WI=3Lai-|X{qgSHWF&ZTE93^{(CLTD zEUcQF!>-SQi`Wh5i<~9x`tXP8RV%+Ags<>6JEmas#F{d~S)Eo?7u>;HCAfX7|n{~A}EcBKv zXLBcRK0#(n(tvPXh6zqu+*1644wkW^! zWGI`-W!}k}Qb2G4boRlVa?aW0T0w;ilun|Kh36^K)0|IkFSk^&|6CUue~1r2LV2qP znM49~Hlc>sZ~;5a1yZ`ej&f*i(th?E_=0_xU!8xW!F#keAaR`Ffv*{Kn|`H;-NI(& z z;cpE6&M`PQf7QmOo;&5U+q3N_@3BvjbX$uE$|SI+d@B!b52K+n{rgA?jATB%`?T>Q zo%cl=>*}4;1Mfj9)(je~UjF@NJS{M8dBMTcY*;62cv=N5?89?p1td&(ePiD(V0NS* zO*(zL%3#3V}XbY_9LzY=v_f-Mest!t!5Q{N#;;UzPG%T2IDLNrt@e z<4dlf&(VJARv4OL~Ox;7v<6|2|2|`Xp%@%$XLJZ4oIS1RI4* z{=Z2U;sJQXlf8&oL4B7^)Os|?u$9~qS7`>+^N}KFic}R$*t349P=;l2OCZ~&SB(It zuj^J=7uy3A%uw=Pku-9WorPjpt&3^h0Sjzx0X^S?8*Wn4BgGx#Z#DgkB0Z8p!Jf{_fi#pE?IuOn2q34eHCeSPtz8C3|V0G ziBtJnrD0!kmwJcNOF0@Cmt>K_thTG5ac34c*!nyoccv?hB<14Q-L55-uAtB)tTRUZn3le3$d*qt%|UBdf!9d*D_5X(s9^*tM1O55&LB z_4qC#30XaO=8iksJ=Slj+J&=}cL(SXXLN6-k*+q#K63#U{n%|? zJPV!oRBtiTxbLx^1g71PmDE!hS^q#IH{HdO5+=$!H9+5-whHKMD?{>Hp3ScT!zOGg zM-uoRY5+mvuIK@db;hlP=Mjt!Vrbsk(0v)8r7jGYB(oJ33Q&K{{&ourpQSzZ<&*3Q#h&X2t`v%sHmu~7ua3WgTbs}ECp$4gg{JX)Oje=RF9v97lGn^8^#B&7K=O9 z*N*ZUpJNP(;)rJweC78%%+G!RN(nv}I(K0GszV~=EHW*b$$W$#5`e4T^o+O#ob=$T z+gkl4#%)?rhgVfM8wMNcmugvt@K+97P#H#vk9W1pT9+ z5|6F%YX3r|C7KOKQ|uQzJho3=w?0>JYdCqg@r1w9ERcwz?>=dAtx20A`5hFeQ0iKT zs@bU#*_--;(8$D3=5-R-VHasvkLNR!y9MxKzDlXEP#fv`Dc|j?k`zZF?v2e5VFv{m zhhrPU={%N)c)2+z(a=*sj0dD`7RXJ4pO+lSxzALIBx5+Svf#OPV-`r-)#VEn)0FMj zt+H$+_FOZND`;oYBhv_q7cDZdra!ilGYJFBk6q{9XtV&|;n=L_+VtAdD-B~y18bFJ zFx*XAUNTu7F=_7YpD$AB6~h~t7g&=G*f}O3<&3mZ%C;F;7b0&FbjYGFMzlrlCAtAQ zbES&__cR++l48{Chi@G_IG(DeGzmx{f*>s>I-gh|n;95WBld^|xQ;fMJC3y;K)@y- zh`L~SgPu14apf2k=GPsmy;FhA7DtVzZ^n`9#69v<4OzC}&Z@cpiSgYD-ZPCq4Ip<3 zC$H#fPKe}xw6tdwu{yabTN`K%RznZ9F8$)D?_!;WaEEUs>br6X96D&9(stMcEZ>HW z>O7ZUd+u^dLJuUE>uar6*bk~8OjTg55)`Y=imOA4@dz$S+wdA9ZLCoY7%B&MJ<(PG z*F!DbUiCZMBC=1OsopuzaKRzZv0a(&K6tVAHYY0%l!K?o=q28zd!-YZST*UK1ktqi%kf(kk@11Y81XleHh+ zEf5-6Jsb}q9)VG^j-&Tru%7jt_-y8a;uQgFP?C8m5uwltHW;~IFc(Q-Y2Lh+V!?!% zfd>-=Ra9Fq5LmxMO&DdXIUjZ7t+*w>6ei%Kzd4=zOm#3zPviHeS8s_JatfEFZK^(D zx%I+k8s=|FP=FWu_{wG#-04OGCd<5s*G471Ti+viA=tBtm70ppC+d-4o6jnTrR;Ebu=dX+Hv z4cur1otoI53_{}u)+=5ychZ5+m@&U8GobSjgp%*5-l>}tS=Hb~iYVB<_9J&KepzlMhpFhaUgT~_ zn8sqjkKuCRg;s~STFNHQVX(Jb32KQ>qQ~8!xgud#Tr@Ti(!Qlxm{GlIvNB;1)pZxL z;RuP4LSO-em>?lCvOu8eW9%vv} zW=q>{wZd8y3$k@x&Uqi3Zg>moP@Ns2B4e6rMubAr)fv6kN17F+-LuX<{Flp)wd(cS z99h+Cx%~qwPIsE4oRZMpG~1(k#qU0wi!tIy3XiLp*C9Z5c>FkSD@mc_o##@qr({h4 zldfN2xm=CH{JU6=Jlu45xCYT;Lx8->0{ANG=!;GYE}iMq4~@7v@)E2kc^_^; z*ETug7q}9*816d)Y-Szt>Nl;<=2Y8Hccv~odND;Y2Zkv{)zgn=f9o(nkH*0f+8sC0 zIlE3&H$OaLVXRgQF`s>olH%e^a>(p9RVylzW4!0CJl6vEz@mRCNUNh18}!qWU*_FM z>=KdeJS4NM1iKf)u5&r;n*iqTPK3swL3`~1WJ0E%m)#{Mz7WMh|DjDJw>(l_%%I3l z%lK0Y&&{|S-ho1qx;M4;sLvxfHou(rul+zTm*`;?UBBVYN ziP;)AVna@LH>-Q~V2r>O5yRMXUO;AUsxd9Mz^Q3TYWPhVX{{%2!&itWv4C`l2J<4F z^R(uLYW-3oDm9fZ&$b83_r~~SmF7PlHEG!F{}X8YP)*4V7z#og!faOf?2#f2a*OZv zUak7bbG=>J1Am+21hdP;lk{d}4X7T*V~q?Dgg)r-Lx)}ou+;>&V$RnGUh_b``m7*r z#7X518)OG^K(^>Bxi$(#MLsG?sm>2w{ER^)$>Cm*(lvv-t69#AiOlu{zcr{B7@%UX z@wyXq9y5W+XS>d)Y+ME&<$iFY&vXR??G})Pp;6Xle|pXi1Tqw3z+&1A_GZIbc#Pab z9&)Hgngk1MyQEhT1j38~tcdSbr;y;6c=*K7Bivk38bmZzZGmx_l?S=Wga%7f$byV% zTMly=n<1v!hbtGcFpf%wYGYA?K5xSZaQ)`5&3-wSgO!I++6M79m|1_FxGClXRw<}z z((izREO&pMFv?u!i=%Sf8gzorn}T_4e^dL*x_bu!Z0UIU_6mBzZ-3}irm8+cXMW{R|E0utk_H>5Nvm7%(`TRZ z+T(OEM%^BrN4@@G6xOk~CL~$n801?}y!rYU!#Up$&Vf$H>;FDTxN`roGijAb%%J6x z{{1RYoUdB3%M%7w6Wu-RNiq6Qjf`YT1peY1qpWFvzOYo@FR&PK`~=PI?SEbvN-q^@ zCPa8v4{+Z7#}9ph<-aFJeZBl=Gk8&b$*=k)LvU z!*A;kt-Fj6a2ylscn@vc@2^$zgH^=xOVVyIbN&6b55-_%zR Date: Mon, 26 Aug 2024 17:17:46 +0800 Subject: [PATCH 496/791] renew_manual --- .../docs/07.accuracy_checker_PyTorch.md | 278 ------------------ 1 file changed, 278 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md diff --git a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md deleted file mode 100644 index 72fb01e2e..000000000 --- a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md +++ /dev/null @@ -1,278 +0,0 @@ -# PyTorch 场景的离线精度预检 - -## 1 简介 - -**PyTorch 离线精度预检**\*1通过扫描昇腾 NPU 上用户训练模型中的所有 API，输出模型精度的诊断和分析结果。具体而言，该工具通过采集模型中所有 API 的前反向信息，构造相应的单元测试，将 NPU 输出与标杆（CPU 高精度）比对，从而计算对应的精度指标，该过程已封装为 run_ut.py 函数；将 NPU 环境下采集的预检数据拷贝至 GPU 环境，同样执行 run_ut.py；最后通过**新精度标准比对法**\*2将 NPU 和 GPU 的预检结果进行比对，从而找出 NPU 中存在精度问题的 API。同时，本工具支持随机生成模式和真实数据模式\*3。 - -**\***： - -1. 支持 PyTorch 版本：1.11/2.0/2.1/2.2； -2. 依据新精度标准，对不同的API采取不同的比对算法（包括绝对阈值法，标杆比对法、二进制一致法、ULP误差比对法和双千指标法），最终给定预检判定结果； -3. 在预检 dump 时可以选择由工具构造随机数获得 dump 数据或选择真实输入的数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 - -## 2 离线预检流程 - -1. 在 NPU 和 GPU 环境下分别安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 -2. 在 NPU 训练脚本内添加 msprobe 工具 dump 接口 PrecisionDebugger，采集待预检数据。注意需要配置 level="L1"。 -3. 将 NPU 环境下 dump 的预检数据拷贝至 GPU 环境。 -4. 在 NPU 和 GPU 环境下分别执行 run_ut.py，生成的结果最终用于 api_precision_compare.py 函数的输入。详见 [3 离线预检操作指导](#3-离线预检操作指导)。 -5. 将 NPU 和 GPU 执行 run_ut.py 生成的 `accuracy_checking_details_{timestamp}.csv` 结果文件拷贝至同一环境下。 -6. 运行 api_precision_compare.py，输出结果为预检操作的最终结果。详见 [5 预检结果比对](#5-预检结果比对)章节。 - -## 3 离线预检操作指导 - -完成预检数据采集后，仅仅获取了 API 的输入数据，为了得到 NPU vs. CPU 高精度（标杆）的预检比对结果和 GPU vs. CPU 高精度（标杆）的预检比对结果，还需要进行 run_ut 操作。 - -run_ut 预检操作包括以下两种方式： - -- 使用 run_ut.py 执行预检：适用于数据量较小的单卡场景。 -- 使用 multi_run_ut.py 执行多线程预检：适用于数据量较大的大模型场景。 - -### 3.1 使用 run_ut.py 执行预检 - -将 API 信息输入到 run_ut 模块进行精度检测并比对，运行如下命令： - -```bash -msprobe -f pytorch run_ut -api_info ./dump.json -``` - - | 参数名称 | 解释 | 是否必选 | - | ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | - | -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | - | -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | - | -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | - | | | | - | -j 或 --jit_compile | 开启 jit 编译。 | 否 | - | -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0。 | 否 | - | -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | - | -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | - | -config 或 --config_path | 指定离线预检操作过程中额外配置（包括黑名单、白名单等）的 [config.json](../config.json) 文件，默认未配置。config.json 文件的配置可参考[配置文件介绍](./02.config_introduction.md)。 | 否 | - -run_ut 执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见[ 4 预检结果](#4-预检结果)。 - -如果需要保存比对不达标的输入和输出数据，可以在 run_ut 执行命令结尾添加 `-save_error_data`，例如： - -```bash -msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data -``` - -数据默认会存盘到 './ut_error_data{timestamp}' 路径下（相对于启动 run_ut 的路径），如有需要，用户可以通过 error_data_path 参数来配置保存路径，error_data_path 参数在 [config.json](../config.json) 文件或 [config.yaml](../pytorch/api_accuracy_checker/config.yaml) 文件配置，config.json 文件需要在 run_ut 操作时通过 -config 参数指定，config.yaml 文件详见\*1。 - -**\***： - -1. config.yaml 文件说明 - - config.yaml 文件可以通过配置参数来控制 dump 和 run_ut 操作的白名单、黑名单等功能。操作步骤如下： - - - config.yaml 文件通常位于类似 /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml 的路径中。 - - - 进入 config.yaml 文件 - - ```bash - vim /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml - ``` - - - 修改 config.yaml 文件参数。 - - ```yaml - white_list: [] - black_list: [] - error_data_path: './' - precision: 14 - ``` - - | 参数名称 | 解释 | 是否必选 | - | ----------- | -----------| -------- | - | white_list | API dump 白名单，仅对指定的 API 进行 dump。
**参数示例**：white_list=["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | - | black_list | API dump 黑名单，被指定的 API 不进行 dump。
**参数示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | - | error_data_path | 配置保存精度未达标的 API 输入输出数据路径。
**参数示例**："error_data_path": "./"。默认为当前路径。 | 否 | - | precision | 浮点数表示位数，默认取小数点后14位。 | 否 | - - 说明：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 - -2. API 预检黑名单和白名单 - - run_ut 过程支持 API 预检黑名单和白名单，通过如下文件配置 black_list（黑名单）或white_list（白名单）参数来指定不需要或需要预检的API名称： - - - 配置 [config.json](../config.json) 文件，该文件需要在 run_ut 操作时通过 -config 参数指定。 - - - config.json 文件的优先级高于 config.yaml 文件，即执行 config.json 文件时，config.yaml 文件的配置不生效。 - -### 3.2 使用 multi_run_ut.py 执行多线程预检 - -multi_run_ut.py 脚本，可以并行执行多个 run_ut 操作，从而减少预检耗时。示例如下： - -```bash -msprobe -f pytorch multi_run_ut -api_info ./dump.json -n 32 -d 0 1 2 3 -``` - -| 参数名称 | 解释 | 是否必选 | -| ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | -| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | -| -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | -| -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | -| -j 或 --jit_compile | 开启 jit 编译。 | 否 | -| -n | 同时执行 run_ut 线程的数量，默认为 8，最大支持 64，但每个 Device 最大支持 8 个线程。当指定多个线程和多个 Device 时，线程数在每张卡上均分。 | 否 | -| -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0，支持同时指定 0~7，共 8 个 Device。 | 否 | -| -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | -| -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | - -### 3.3 断点续检 - -断点续检操作通过如下命令执行： - -```bash -msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_checking_result_{timestamp}.csv -``` - -精度预检 run_ut 过程中，若因环境、数据量过大等原因导致预检进程中断，那么当用户解决这些问题后，重新执行 run_ut 操作，可以通过断点续检操作继续前面未完成的预检，会在 -csv_path 指定的 `accuracy_checking_result_{timestamp}.csv` 文件以及对应的 `accuracy_checking_details_{timestamp}.csv` 文件中继续写入后续的结果，不会重新创建结果文件。 - -须指定为上次预检中断的 `accuracy_checking_result_{timestamp}.csv` 文件。请勿修改 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件名，包括时间戳，否则断点续检会因无法识别到文件名而失败。 - -## 4 预检结果 - -精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件示例如下： - -可以通过先查看 `accuracy_checking_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `accuracy_checking_details_{timestamp}.csv` 文件的 API 详细达标情况，API 达标情况介绍请参见\*1。 - -`accuracy_checking_result_{timestamp}.csv` - -![accuracy_checking_result](img/accuracy_checking_result.png) - -| 字段 | 含义 | -| --------------------- | ------------------------- | -| API name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。 | -| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出。 | -| Message | 提示信息。 | - -该结果为中间结果，仅作为参考，建议完成 [5 预检结果比对](#5-预检结果比对)后查看比对结果。该结果后续将会删除。 - -Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差、双百双千双万指标判定结果决定的。 - -需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 waring 和 pass 且不存在 error 则标记 waring。 - -`accuracy_checking_details_{timestamp}.csv` - -![accuracy_checking_details](img/accuracy_checking_details.png) - -| 字段 | 含义 | -| ------------------- | ------------------------------------------------------------ | -| API name | NPU 或 GPU下的 API 名称。 | -| Bench Dtype | 标杆数据的 API 数据类型。 | -| DEVICE Dtype | NPU 或 GPU 数据的 API 数据类型。 | -| Shape | API 的 Shape信息。 | -| 余弦相似度 | NPU 或 GPU 数据与标杆数据的余弦相似度。 | -| 最大绝对误差 | NPU 或 GPU 数据与标杆数据的最大绝对误差。 | -| 双百指标 | 双百精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于百分之一的个数占总元素个数的比例。测试通过标准为相对误差大于百分之一的个数占总元素个数的比例小于百分之一。 | -| 双千指标 | 双千精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。 | -| 双万指标 | 双万精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于万分之一的个数占总元素个数的比例。测试通过标准为相对误差大于万分之一的个数占总元素个数的比例小于万分之一。 | -| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。 | -| 误差均衡性 | NPU 或 GPU 数据与标杆数据精度差的上下浮动情况。 | -| 均方根误差 | NPU 或 GPU 数据与标杆数据的均方根误差。 | -| 小值域错误占比 | NPU 或 GPU Tensor 中与标杆的绝对误差大于错误阈值的小值在小值域（小值的总数量）中的占比。判断为小值以及绝对误差的错误阈值参见\*2。 | -| 相对误差最大值 | NPU 或 GPU 数据与标杆数据相对误差的最大值。 | -| 相对误差平均值 | NPU 或 GPU 数据与标杆数据相对误差的平均值。 | -| inf/nan 错误率 | NPU 与标杆 inf/nan 计算不一致的元素个数占总元素的个数比例。 | -| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。 | -| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。 | -| ULP 误差最大值 | NPU 或 GPU 数据与标杆数据 ULP 误差的最大值（取绝对值后）。 | -| ULP 误差平均值 | NPU 或 GPU 数据与标杆数据ULP误差的平均值（取绝对值后）。 | -| ULP 误差大于阈值占比 | NPU 或 GPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 或 GPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 或 GPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。 | -| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过，warning 表示测试未通过双千或双万精度指标，SKIP 表示该 API 的某个参数的反向不要计算梯度，所以没有任何计算过程，其他信息均为空。 | -| message | 提示信息。 | - -**\***： - -1. API 预检指标： - - API 预检指标是通过对 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差双百、双千、双万精度指标的数值进行判断，得出该 API 是否符合精度标准的参考指标。 - - API预检通过测试，则在`accuracy_checking_details_{timestamp}.csv`文件中的 Status 列标记 pass，否则标记 error 或 warning，详细规则如下： - - - 余弦相似度 > 0.99：≤ 0.99 为不达标，标记 error，> 0.99 达标，进行下一步； - - 最大绝对误差 ＜ 0.001：＜ 0.001 达标，标记 pass，≥ 0.001 为不达标，进行下一步； - - 双百、双千、双万精度指标： - + 对于 float16 和 bfloat16 数据：双百指标不通过，标记 error；双百指标通过，双千指标不通过，标记 warning；双百、双千指标均通过，标记 pass。 - + 对于 float32 和 float64 数据：双千指标不通过，标记 error；双千指标通过，双万指标不通过，标记 warning；双千、双万指标均通过，标记 pass。 - - 在 `accuracy_checking_result_{timestamp}.csv` 中以 Forward Test Success 和 Backward Test Success 字段统计该算子前向反向输出的测试结果，对于标记 pass 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE 表示测试通过，对于标记 error 或 warning 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 FALSE 表示测试不通过。由于一个算子可能有多个前向或反向的输入或输出，那么该类算子的输入或输出中必须全为 pass，才能在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE，只要有一个输入或输出标记 error 或 warning，那么在`accuracy_checking_result_{timestamp}.csv`中标记 FALSE。 - -2. 小值域阈值： - - 判定为小值的阈值： - - - torch.float32：e-6 - - torch.float16：e-3 - - torch.bfloat16：e-3 - - 小值域的绝对误差阈值： - - - torch.float32：e-9 - - torch.float16：e-5 - - torch.bfloat16：e-5 - -## 5 预检结果比对 - -需要同时获取 NPU 和 GPU 环境下 run_ut 操作的预检结果 `accuracy_checking_details_{timestamp}.csv` 文件。执行如下命令进行 NPU 和 GPU 预检结果的比对： - -```bash -msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_details_{timestamp}.csv -gpu /home/xxx/gpu/accuracy_checking_details_{timestamp}.csv -o /home/xxx/ -``` - -| 参数名称 | 说明 | 是否必选 | -| -------------------- | ------------- | -------- | -| -npu 或 --npu_csv_path | NPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | -| -gpu 或 --gpu_csv_path | GPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | -| -o 或 --out_path | 指定 api_precision_compare.py 执行结果存盘路径，默认为当前目录。 | 否 | - -执行完成后输出 `api_precision_compare_result_{timestamp}.csv` 和 `api_precision_compare_details_{timestamp}.csv` 文件。文件示例如下： - -可以通过先查看 `api_precision_compare_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `api_precision_compare_details_{timestamp}.csv` 文件的 API 详细达标情况。 - -`api_precision_compare_result_{timestamp}.csv` - -![api_precision_compare_result](img/api_precision_compare_result.png) - -| 字段 | 含义 | -| --------------------- | ------------------------------------------------------------ | -| API name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | -| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | -| Message | 提示信息。 | - -Forward Test Success 和 Backward Test Success 是否通过测试是由 `api_precision_compare_details_{timestamp}.csv` 中的各个指标判定结果决定的。需要注意的是 `api_precision_compare_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `api_precision_compare_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 warning 和 pass 且不存在 error 标记 warning。 - -`api_precision_compare_details_{timestamp}.csv` - -![api_precision_compare_details](img/api_precision_compare_details.png) - -| 字段 | 含义 | -| ------------------------ | ------------------------------------------------------------ | -| API name | NPU 或 GPU 下的 API 名称。 | -| 小值域错误比值 | NPU 与 CPU 的小值域的错误比率 / GPU 与 CPU 的小值域的错误比率。标杆比对法指标。 | -| 小值域错误判定结果 | 小值域错误比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | -| 均方根误差比值 | NPU 与 CPU 的均方根误差 / GPU 与 CPU 的均方根误差。标杆比对法指标。 | -| 均方根误差判定结果 | 均方根误差比值小于等于 1 标记为 pass，1~2 之间标记为 waring，大于 2 标记为 error。 | -| 相对误差最大值比值 | NPU 与 CPU 的相对误差最大值 / GPU 与 CPU 的相对误差最大值。标杆比对法指标。 | -| 相对误差最大值判定结果 | 相对误差最大值比值小于等于 1 标记为 pass，1 ~ 10 之间标记为 waring，大于 10 标记为 error。 | -| 相对误差平均值比值 | NPU 与 CPU 的相对误差的平均值 / GPU 与 CPU 的相对误差的平均值。标杆比对法指标。 | -| 相对误差平均值判定结果 | 相对误差平均值比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | -| 误差均衡性比值 | NPU 与 CPU 的误差均衡性 / GPU 与 CPU 的误差均衡性。标杆比对法指标。 | -| 误差均衡性判定结果 | 误差均衡性比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。该字段暂不参与 api_precision_compare_result 的结果判定。 | -| inf / nan 错误率 | NPU 与标杆 inf / nan 计算不一致的元素个数占总元素的个数比例。绝对阈值法指标。 | -| inf / nan 判定结果 | inf / nan 错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | -| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。绝对阈值法指标。 | -| 相对误差判定结果 | 相对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | -| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。绝对阈值法指标。 | -| 绝对误差判定结果 | 绝对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | -| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。二进制一致法指标。 | -| 二进制一致错误率判定结果 | 二进制一致错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | -| ULP 误差平均值 | NPU 数据与标杆数据 ULP 误差的平均值（取绝对值后）。ULP 误差比对法指标。 | -| ULP 误差大于阈值占比 | NPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。ULP 误差比对法指标。 | -| ULP 误差大于阈值占比比值 | NPU 与 CPU 的 ULP 误差大于阈值占比 / GPU 与 CPU 的 ULP 误差大于阈值占比。ULP 误差比对法指标。 | -| ULP 误差判定结果 | ULP 误差判定结果。
当 NPU 或 GPU 数据类型是 float16 或 bfloat16 时，以下两条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差大于阈值占比小于 0.001；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。
当 NPU 或 GPU 数据类型是 float32 时，以下三条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差平均值小于 64；
NPU ULP 误差大于阈值占比小于 0.05；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。 | -| 双千指标 | 双千精度指标。是指 NPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅 conv1d 和 conv2d 使用该指标。双千指标法指标。 | -| 双千指标判定结果 | 双千指标判定结果。双千指标大于 0.999 标记为 pass，否则标记为 error。 | -| 比对结果 | 综合所有指标的最终结果。如果比对指标中有 error，则标记为 error；有 warning，则标记为 warning；否则标记为 pass。 | -| 比对算法 | API 使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和 ULP 误差比对法中的一种。 | -| Message | 提示信息。当前提示该 API 比对结果为 error 或 warning 时对应不符合标准的指标。 | -- Gitee From 97980bd44264ea3422522e4ca2010122fa48e19b Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 17:19:19 +0800 Subject: [PATCH 497/791] renew_manual --- .../msprobe/docs/05.data_dump_Pytorch.md | 199 ------------------ .../msprobe/docs/06.data_dump_MindSpore.md | 157 -------------- 2 files changed, 356 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md delete mode 100644 debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md deleted file mode 100644 index 3f2b2a88b..000000000 --- a/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md +++ /dev/null @@ -1,199 +0,0 @@ -# PyTorch 场景下的精度数据采集 - -msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 - -本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/pytorch/hook_module/support_wrap_ops.yaml 文件内手动修改，如下示例： - -```yaml -functional: # functional为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API - - conv1d - - conv2d - - conv3d -``` - -## 1 接口介绍 - -### 1.1 PrecisionDebugger - -**功能说明**：通过加载 dump 配置文件的方式来确定 dump 操作的详细配置。 - -**原型**： - -```Python -PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, model=None, step=None) -``` - -**\***： -1. config_path：指定 dump 配置文件路径；model：指定具体的 torch.nn.Module，默认未配置，level 配置为"L0"或"mix"时必须配置该参数。其他参数均在 [config.json](../config.json) 文件中可配，详细配置可见 [config.json 介绍](./02.config_introduction.md)。 -2. 此接口的参数均不是必要，且优先级高于 [config.json](../config.json) 文件中的配置，但可配置的参数相比 config.json 较少。 - -### 1.2 start - -**功能说明**：启动精度数据采集。在模型初始化之后的位置添加。需要与 stop 函数一起添加在 for 循环内。 - -**原型**： - -```Python -debugger.start() -``` - -### 1.3 stop - -**功能说明**：停止精度数据采集。在 **start** 函数之后的任意位置添加。若需要 dump 反向数据，则需要添加在反向计算代码（如，loss.backward）之后。使用示例可参见 [2.1 采集 model 的精度数据](#21-采集-model-的精度数据)和 [2.2 采集完整的前反向数据](#22-采集完整的前反向数据)。 - -**原型**： - -```Python -debugger.stop() -``` - -### 1.4 forward_backward_dump_end - -**功能说明**：停止精度数据采集。用于 dump 指定代码的前反向数据。在 **start** 函数之后，反向计算代码（如，loss.backward）之前的任意位置添加，可以采集 **start** 函数和该函数之间的前反向数据，可以通过调整 **start** 函数与该函数的位置，来指定需要 dump 的代码块。要求 **stop** 函数添加在反向计算代码（如，loss.backward）之后，此时该函数与 **stop** 函数之间的代码不会被 dump。使用示例可参见 [2.3 采集指定代码块的前反向数据](#23-采集指定代码块的前反向数据) - -**原型**： - -```Python -forward_backward_dump_end() -``` - -### 1.5 step - -**功能说明**：更新 dump 参数。在最后一个 **stop** 函数后或一个 step 结束的位置添加。需要与 **start** 函数一起添加在 for 循环内。 - -**原型**： - -```Python -debugger.step() -``` - -## 2 示例代码 - -### 2.1 采集 model 的精度数据 - -这个示例定义了一个 nn.Module 类型的简单网络，在进行数据采集时使用原型函数 PrecisionDebugger 传入 config_path 参数和 model 参数。 - -```python -# 根据需要import包 -import torch -import torch.nn as nn -import torch_npu # 需安装 torch_npu -import torch.nn.functional as F -from msprobe.pytorch import PrecisionDebugger - - -torch.npu.set_device("npu:0") -# 定义网络 -class ModuleOP(nn.Module): - def __init__(self) -> None: - super().__init__() - self.linear_1 = nn.Linear(in_features=8,out_features=4) - self.linear_2 = nn.Linear(in_features=4,out_features=2) - - def forward(self,x): - x1 = self.linear_1(x) - x2 = self.linear_2(x1) - r1 = F.relu(x2) - return r1 - -if __name__ == "__main__": - module = ModuleOP() - # 注册工具 - debugger = PrecisionDebugger('./config.json', model=module) - debugger.start() - x = torch.randn(10,8) - out = module(x) - loss = out.sum() - loss.backward() - debugger.stop() - -``` -### 2.2 采集完整的前反向数据 - -```Python -from msprobe.pytorch import PrecisionDebugger - -# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 -debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") -# 模型、损失函数的定义及初始化等操作 -# ... -# 数据集迭代的位置一般为模型训练开始的位置 -for data, label in data_loader: - debugger.start() # 开启数据dump - # 如下是模型每个step执行的逻辑 - output = model(data) - #... - loss.backward() - debugger.stop() # 关闭数据dump - debugger.step() # 结束一个step的dump -``` - -### 2.3 采集指定代码块的前反向数据 - -```Python -from msprobe.pytorch import PrecisionDebugger - -# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 -debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") - -# 模型、损失函数的定义及初始化等操作 -# ... -# 数据集迭代的位置一般为模型训练开始的位置 -for data, label in data_loader: - debugger.start() # 开启数据dump - # 如下是模型每个step执行的逻辑 - output = model(data) - debugger.forward_backward_dump_end() # 插入该函数到start函数之后，只dump start函数到该函数之间代码的前反向数据，本函数到stop函数之间的数据则不dump - #... - loss.backward() - debugger.stop() # 关闭数据dump - debugger.step() # 结束一个step的dump -``` - - - -## 3 dump 结果文件介绍 - -训练结束后，工具将 dump 的数据保存在 dump_path 参数指定的目录下。目录结构示例如下： - -```Python -├── dump_path -│ ├── step0 -│ | ├── rank0 -│ | │ ├── dump_tensor_data -| | | | ├── Tensor.permute.1.forward.pt -| | | | ├── MyModule.0.forward.input.pt # 开启模块级精度数据dump时存在模块级的dump数据文件 -| | | | ... -| | | | └── Fcuntion.linear.5.backward.output.pt -│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） -│ | | ├── stack.json # 算子调用栈信息 -│ | | └── construct.json # 分层分级结构 -│ | ├── rank1 -| | | ├── dump_tensor_data -| | | | └── ... -│ | | ├── dump.json -│ | | ├── stack.json -| | | └── construct.json -│ | ├── ... -│ | | -| | └── rank7 -│ ├── step1 -│ | ├── ... -│ ├── step2 -``` - -dump 过程中，pt 文件在对应算子或者模块被执行后就会落盘，而 json 文件则需要在正常执行 PrecisionDebugger.stop() 后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关 npy 文件，可能会导致 json 文件中数据丢失。 - -其中 rank 为设备上各卡的 ID，每张卡上 dump 的数据会生成对应 dump 目录。非分布式场景下没有rank ID，目录名称为rank。 - -pt 文件保存的前缀和 PyTorch 对应关系如下： - -| 前缀 | Torch模块 | -| ----------- | ------------------- | -| Tensor | torch.Tensor | -| Torch | torch | -| Functional | torch.nn.functional | -| NPU | NPU亲和算子 | -| VF | torch._VF | -| Aten | torch.ops.aten | -| Distributed | torch.distributed | diff --git a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md deleted file mode 100644 index cad25d984..000000000 --- a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md +++ /dev/null @@ -1,157 +0,0 @@ -# MindSpore 场景下的精度数据采集 - -msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 - -本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml 文件内手动修改，如下示例： - -```yaml -ops: # ops为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API - - adaptive_avg_pool1d - - adaptive_avg_pool2d - - adaptive_avg_pool3d -``` - -## 1 接口介绍 - -### 1.1 PrecisionDebugger - -**功能说明**：通过加载 dump 配置文件的方式来确定 dump 操作的详细配置。 - -**原型**： - -```Python -PrecisionDebugger(config_path=None) -``` - -**\***： -1. config_path：指定 dump 配置文件路径，string 类型。参数示例："./config.json"。未配置该路径时，默认使用 [config.json](../config.json) 文件的默认配置。 - -### 1.2 start - -**功能说明**：启动精度数据采集。在模型初始化之后的位置添加。需要与 stop 函数一起添加在 for 循环内。 - -**原型**： - -```Python -debugger.start(model=None) -``` -**\***： -1. model：指具体的 mindspore.nn.Cell，默认未配置，L1 级别下传入 model 可以采集 primitive op。 - -### 1.3 stop - -**功能说明**：停止数据采集。在 **start** 函数之后的任意位置添加。需要与 start 函数一起添加在 for 循环内。若需要 dump 反向数据，则需要添加在反向计算代码之后。**仅 MindSpore 动态图场景支持。** - -**原型**： - -```Python -debugger.stop() -``` - -### 1.4 step - -**功能说明**：在最后一个 **stop** 函数后或一个 step 结束的位置添加。**仅 MindSpore 动态图场景支持。** - -**原型**： - -```Python -debugger.step() -``` - -## 2 示例代码 - -### 2.1 MindSpore 静态图场景 - -```Python -from msprobe.mindspore import PrecisionDebugger -debugger = PrecisionDebugger(config_path="./config.json") -# 请勿将以上初始化流程插入到循环代码中 -# 下面代码也可以用PrecisionDebugger.start() -debugger.start() -... -``` - -### 2.2 MindSpore 动态图场景 - -```Python -import mindspore as ms -from msprobe.mindspore import PrecisionDebugger - -# 请勿将PrecisionDebugger的初始化插入到循环代码中 -debugger = PrecisionDebugger(config_path="./config.json") - -# 模型、损失函数的定义以及初始化等操作 -# ... - -# 数据集迭代的地方往往是模型开始训练的地方 -for data, label in data_loader: - debugger.start() # 开启数据dump - net = Model() - # 如下是模型每个step执行的逻辑 - grad_net = ms.grad(net)(data) - # ... - debugger.stop() # 关闭数据dump - debugger.step() # 结束一个step的dump -``` - -## 3 dump 结果文件介绍 - -### 3.1 MindSpore 静态图场景 - -训练结束后，工具将 dump 的数据保存在 dump_path 参数指定的目录下。 - -- jit_level 为O0/O1时： - - dump 结果目录请参见 MindSpore 官网中的[同步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%90%8C%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)。 - -- jit_level 为O2时： - - dump 结果目录请参见 MindSpore 官网中的[异步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%BC%82%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)。 - -jit_level 请参见 [mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig) 配置 jit_config。 - -### 3.2 MindSpore 动态图场景 - -训练结束后，工具将 dump 的数据保存在dump_path参数指定的目录下。 - -dump结果目录结构示例如下： - -```bash -├── dump_path -│ ├── step0 -│ | ├── rank0 -│ | │ ├── dump_tensor_data -| | | | ├── MintFunctional.relu.0.backward.input.0.npy -| | | | ├── Mint.abs.0.forward.input.0.npy -| | | | ├── Functional.split.0.forward.input.0.npy -| | | | ... -| | | | └── Tensor.__add__.0.forward.output.0.npy -│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） -│ | | ├── stack.json # 算子调用栈信息 -│ | | └── construct.json # 分层分级结构，level为L1时，construct.json内容为空 -│ | ├── rank1 -| | | ├── dump_tensor_data -| | | | └── ... -│ | | ├── dump.json -│ | | ├── stack.json -| | | └── construct.json -│ | ├── ... -│ | | -| | └── rank7 -│ ├── step1 -│ | ├── ... -│ ├── step2 -``` - -dump 过程中，npy 文件在对应算子或者模块被执行后就会落盘，而 json 文件则需要在正常执行 PrecisionDebugger.stop() 后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关 npy 文件，可能会导致 json 文件中数据丢失。 - -其中 rank 为设备上各卡的 ID，每张卡上 dump 的数据会生成对应 dump 目录。非分布式场景下没有 rank ID，目录名称为 rank。 - -npy 文件保存的前缀和 MindSpore 对应关系如下： - -| 前缀 | MindSpore 模块 | -| -------------- | ---------------------------- | -| Tensor | mindspore.Tensor | -| Functional | mindspore.ops | -| Mint | mindspore.mint | -| MintFunctional | mindspore.mint.nn.functional | -- Gitee From 4d7a9400767db50c7fa357c796389370e8e38c21 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 17:20:07 +0800 Subject: [PATCH 498/791] renew_manual --- .../msprobe/docs/05.data_dump_Pytorch.md | 199 ++++++++++++++++++ 1 file changed, 199 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md new file mode 100644 index 000000000..3f2b2a88b --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md @@ -0,0 +1,199 @@ +# PyTorch 场景下的精度数据采集 + +msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 + +本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/pytorch/hook_module/support_wrap_ops.yaml 文件内手动修改，如下示例： + +```yaml +functional: # functional为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API + - conv1d + - conv2d + - conv3d +``` + +## 1 接口介绍 + +### 1.1 PrecisionDebugger + +**功能说明**：通过加载 dump 配置文件的方式来确定 dump 操作的详细配置。 + +**原型**： + +```Python +PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, model=None, step=None) +``` + +**\***： +1. config_path：指定 dump 配置文件路径；model：指定具体的 torch.nn.Module，默认未配置，level 配置为"L0"或"mix"时必须配置该参数。其他参数均在 [config.json](../config.json) 文件中可配，详细配置可见 [config.json 介绍](./02.config_introduction.md)。 +2. 此接口的参数均不是必要，且优先级高于 [config.json](../config.json) 文件中的配置，但可配置的参数相比 config.json 较少。 + +### 1.2 start + +**功能说明**：启动精度数据采集。在模型初始化之后的位置添加。需要与 stop 函数一起添加在 for 循环内。 + +**原型**： + +```Python +debugger.start() +``` + +### 1.3 stop + +**功能说明**：停止精度数据采集。在 **start** 函数之后的任意位置添加。若需要 dump 反向数据，则需要添加在反向计算代码（如，loss.backward）之后。使用示例可参见 [2.1 采集 model 的精度数据](#21-采集-model-的精度数据)和 [2.2 采集完整的前反向数据](#22-采集完整的前反向数据)。 + +**原型**： + +```Python +debugger.stop() +``` + +### 1.4 forward_backward_dump_end + +**功能说明**：停止精度数据采集。用于 dump 指定代码的前反向数据。在 **start** 函数之后，反向计算代码（如，loss.backward）之前的任意位置添加，可以采集 **start** 函数和该函数之间的前反向数据，可以通过调整 **start** 函数与该函数的位置，来指定需要 dump 的代码块。要求 **stop** 函数添加在反向计算代码（如，loss.backward）之后，此时该函数与 **stop** 函数之间的代码不会被 dump。使用示例可参见 [2.3 采集指定代码块的前反向数据](#23-采集指定代码块的前反向数据) + +**原型**： + +```Python +forward_backward_dump_end() +``` + +### 1.5 step + +**功能说明**：更新 dump 参数。在最后一个 **stop** 函数后或一个 step 结束的位置添加。需要与 **start** 函数一起添加在 for 循环内。 + +**原型**： + +```Python +debugger.step() +``` + +## 2 示例代码 + +### 2.1 采集 model 的精度数据 + +这个示例定义了一个 nn.Module 类型的简单网络，在进行数据采集时使用原型函数 PrecisionDebugger 传入 config_path 参数和 model 参数。 + +```python +# 根据需要import包 +import torch +import torch.nn as nn +import torch_npu # 需安装 torch_npu +import torch.nn.functional as F +from msprobe.pytorch import PrecisionDebugger + + +torch.npu.set_device("npu:0") +# 定义网络 +class ModuleOP(nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear_1 = nn.Linear(in_features=8,out_features=4) + self.linear_2 = nn.Linear(in_features=4,out_features=2) + + def forward(self,x): + x1 = self.linear_1(x) + x2 = self.linear_2(x1) + r1 = F.relu(x2) + return r1 + +if __name__ == "__main__": + module = ModuleOP() + # 注册工具 + debugger = PrecisionDebugger('./config.json', model=module) + debugger.start() + x = torch.randn(10,8) + out = module(x) + loss = out.sum() + loss.backward() + debugger.stop() + +``` +### 2.2 采集完整的前反向数据 + +```Python +from msprobe.pytorch import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") +# 模型、损失函数的定义及初始化等操作 +# ... +# 数据集迭代的位置一般为模型训练开始的位置 +for data, label in data_loader: + debugger.start() # 开启数据dump + # 如下是模型每个step执行的逻辑 + output = model(data) + #... + loss.backward() + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` + +### 2.3 采集指定代码块的前反向数据 + +```Python +from msprobe.pytorch import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") + +# 模型、损失函数的定义及初始化等操作 +# ... +# 数据集迭代的位置一般为模型训练开始的位置 +for data, label in data_loader: + debugger.start() # 开启数据dump + # 如下是模型每个step执行的逻辑 + output = model(data) + debugger.forward_backward_dump_end() # 插入该函数到start函数之后，只dump start函数到该函数之间代码的前反向数据，本函数到stop函数之间的数据则不dump + #... + loss.backward() + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` + + + +## 3 dump 结果文件介绍 + +训练结束后，工具将 dump 的数据保存在 dump_path 参数指定的目录下。目录结构示例如下： + +```Python +├── dump_path +│ ├── step0 +│ | ├── rank0 +│ | │ ├── dump_tensor_data +| | | | ├── Tensor.permute.1.forward.pt +| | | | ├── MyModule.0.forward.input.pt # 开启模块级精度数据dump时存在模块级的dump数据文件 +| | | | ... +| | | | └── Fcuntion.linear.5.backward.output.pt +│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） +│ | | ├── stack.json # 算子调用栈信息 +│ | | └── construct.json # 分层分级结构 +│ | ├── rank1 +| | | ├── dump_tensor_data +| | | | └── ... +│ | | ├── dump.json +│ | | ├── stack.json +| | | └── construct.json +│ | ├── ... +│ | | +| | └── rank7 +│ ├── step1 +│ | ├── ... +│ ├── step2 +``` + +dump 过程中，pt 文件在对应算子或者模块被执行后就会落盘，而 json 文件则需要在正常执行 PrecisionDebugger.stop() 后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关 npy 文件，可能会导致 json 文件中数据丢失。 + +其中 rank 为设备上各卡的 ID，每张卡上 dump 的数据会生成对应 dump 目录。非分布式场景下没有rank ID，目录名称为rank。 + +pt 文件保存的前缀和 PyTorch 对应关系如下： + +| 前缀 | Torch模块 | +| ----------- | ------------------- | +| Tensor | torch.Tensor | +| Torch | torch | +| Functional | torch.nn.functional | +| NPU | NPU亲和算子 | +| VF | torch._VF | +| Aten | torch.ops.aten | +| Distributed | torch.distributed | -- Gitee From 34f5dc756347a85c9eab85ef800ae3f8a614537b Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 17:26:03 +0800 Subject: [PATCH 499/791] renew_manual --- .../docs/07.accuracy_checker_PyTorch.md | 278 ++++++++++++++++++ .../mindspore/debugger/precision_debugger.py | 2 +- .../msprobe/pytorch/pt_config.py | 2 +- 3 files changed, 280 insertions(+), 2 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md diff --git a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md new file mode 100644 index 000000000..72fb01e2e --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md @@ -0,0 +1,278 @@ +# PyTorch 场景的离线精度预检 + +## 1 简介 + +**PyTorch 离线精度预检**\*1通过扫描昇腾 NPU 上用户训练模型中的所有 API，输出模型精度的诊断和分析结果。具体而言，该工具通过采集模型中所有 API 的前反向信息，构造相应的单元测试，将 NPU 输出与标杆（CPU 高精度）比对，从而计算对应的精度指标，该过程已封装为 run_ut.py 函数；将 NPU 环境下采集的预检数据拷贝至 GPU 环境，同样执行 run_ut.py；最后通过**新精度标准比对法**\*2将 NPU 和 GPU 的预检结果进行比对，从而找出 NPU 中存在精度问题的 API。同时，本工具支持随机生成模式和真实数据模式\*3。 + +**\***： + +1. 支持 PyTorch 版本：1.11/2.0/2.1/2.2； +2. 依据新精度标准，对不同的API采取不同的比对算法（包括绝对阈值法，标杆比对法、二进制一致法、ULP误差比对法和双千指标法），最终给定预检判定结果； +3. 在预检 dump 时可以选择由工具构造随机数获得 dump 数据或选择真实输入的数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + +## 2 离线预检流程 + +1. 在 NPU 和 GPU 环境下分别安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 +2. 在 NPU 训练脚本内添加 msprobe 工具 dump 接口 PrecisionDebugger，采集待预检数据。注意需要配置 level="L1"。 +3. 将 NPU 环境下 dump 的预检数据拷贝至 GPU 环境。 +4. 在 NPU 和 GPU 环境下分别执行 run_ut.py，生成的结果最终用于 api_precision_compare.py 函数的输入。详见 [3 离线预检操作指导](#3-离线预检操作指导)。 +5. 将 NPU 和 GPU 执行 run_ut.py 生成的 `accuracy_checking_details_{timestamp}.csv` 结果文件拷贝至同一环境下。 +6. 运行 api_precision_compare.py，输出结果为预检操作的最终结果。详见 [5 预检结果比对](#5-预检结果比对)章节。 + +## 3 离线预检操作指导 + +完成预检数据采集后，仅仅获取了 API 的输入数据，为了得到 NPU vs. CPU 高精度（标杆）的预检比对结果和 GPU vs. CPU 高精度（标杆）的预检比对结果，还需要进行 run_ut 操作。 + +run_ut 预检操作包括以下两种方式： + +- 使用 run_ut.py 执行预检：适用于数据量较小的单卡场景。 +- 使用 multi_run_ut.py 执行多线程预检：适用于数据量较大的大模型场景。 + +### 3.1 使用 run_ut.py 执行预检 + +将 API 信息输入到 run_ut 模块进行精度检测并比对，运行如下命令： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json +``` + + | 参数名称 | 解释 | 是否必选 | + | ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | + | -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | + | -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | + | -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | + | | | | + | -j 或 --jit_compile | 开启 jit 编译。 | 否 | + | -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0。 | 否 | + | -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | + | -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | + | -config 或 --config_path | 指定离线预检操作过程中额外配置（包括黑名单、白名单等）的 [config.json](../config.json) 文件，默认未配置。config.json 文件的配置可参考[配置文件介绍](./02.config_introduction.md)。 | 否 | + +run_ut 执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见[ 4 预检结果](#4-预检结果)。 + +如果需要保存比对不达标的输入和输出数据，可以在 run_ut 执行命令结尾添加 `-save_error_data`，例如： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data +``` + +数据默认会存盘到 './ut_error_data{timestamp}' 路径下（相对于启动 run_ut 的路径），如有需要，用户可以通过 error_data_path 参数来配置保存路径，error_data_path 参数在 [config.json](../config.json) 文件或 [config.yaml](../pytorch/api_accuracy_checker/config.yaml) 文件配置，config.json 文件需要在 run_ut 操作时通过 -config 参数指定，config.yaml 文件详见\*1。 + +**\***： + +1. config.yaml 文件说明 + + config.yaml 文件可以通过配置参数来控制 dump 和 run_ut 操作的白名单、黑名单等功能。操作步骤如下： + + - config.yaml 文件通常位于类似 /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml 的路径中。 + + - 进入 config.yaml 文件 + + ```bash + vim /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml + ``` + + - 修改 config.yaml 文件参数。 + + ```yaml + white_list: [] + black_list: [] + error_data_path: './' + precision: 14 + ``` + + | 参数名称 | 解释 | 是否必选 | + | ----------- | -----------| -------- | + | white_list | API dump 白名单，仅对指定的 API 进行 dump。
**参数示例**：white_list=["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | + | black_list | API dump 黑名单，被指定的 API 不进行 dump。
**参数示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | + | error_data_path | 配置保存精度未达标的 API 输入输出数据路径。
**参数示例**："error_data_path": "./"。默认为当前路径。 | 否 | + | precision | 浮点数表示位数，默认取小数点后14位。 | 否 | + + 说明：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 + +2. API 预检黑名单和白名单 + + run_ut 过程支持 API 预检黑名单和白名单，通过如下文件配置 black_list（黑名单）或white_list（白名单）参数来指定不需要或需要预检的API名称： + + - 配置 [config.json](../config.json) 文件，该文件需要在 run_ut 操作时通过 -config 参数指定。 + + - config.json 文件的优先级高于 config.yaml 文件，即执行 config.json 文件时，config.yaml 文件的配置不生效。 + +### 3.2 使用 multi_run_ut.py 执行多线程预检 + +multi_run_ut.py 脚本，可以并行执行多个 run_ut 操作，从而减少预检耗时。示例如下： + +```bash +msprobe -f pytorch multi_run_ut -api_info ./dump.json -n 32 -d 0 1 2 3 +``` + +| 参数名称 | 解释 | 是否必选 | +| ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | +| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | +| -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | +| -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | +| -j 或 --jit_compile | 开启 jit 编译。 | 否 | +| -n | 同时执行 run_ut 线程的数量，默认为 8，最大支持 64，但每个 Device 最大支持 8 个线程。当指定多个线程和多个 Device 时，线程数在每张卡上均分。 | 否 | +| -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0，支持同时指定 0~7，共 8 个 Device。 | 否 | +| -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | +| -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | + +### 3.3 断点续检 + +断点续检操作通过如下命令执行： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_checking_result_{timestamp}.csv +``` + +精度预检 run_ut 过程中，若因环境、数据量过大等原因导致预检进程中断，那么当用户解决这些问题后，重新执行 run_ut 操作，可以通过断点续检操作继续前面未完成的预检，会在 -csv_path 指定的 `accuracy_checking_result_{timestamp}.csv` 文件以及对应的 `accuracy_checking_details_{timestamp}.csv` 文件中继续写入后续的结果，不会重新创建结果文件。 + +须指定为上次预检中断的 `accuracy_checking_result_{timestamp}.csv` 文件。请勿修改 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件名，包括时间戳，否则断点续检会因无法识别到文件名而失败。 + +## 4 预检结果 + +精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件示例如下： + +可以通过先查看 `accuracy_checking_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `accuracy_checking_details_{timestamp}.csv` 文件的 API 详细达标情况，API 达标情况介绍请参见\*1。 + +`accuracy_checking_result_{timestamp}.csv` + +![accuracy_checking_result](img/accuracy_checking_result.png) + +| 字段 | 含义 | +| --------------------- | ------------------------- | +| API name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出。 | +| Message | 提示信息。 | + +该结果为中间结果，仅作为参考，建议完成 [5 预检结果比对](#5-预检结果比对)后查看比对结果。该结果后续将会删除。 + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差、双百双千双万指标判定结果决定的。 + +需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 waring 和 pass 且不存在 error 则标记 waring。 + +`accuracy_checking_details_{timestamp}.csv` + +![accuracy_checking_details](img/accuracy_checking_details.png) + +| 字段 | 含义 | +| ------------------- | ------------------------------------------------------------ | +| API name | NPU 或 GPU下的 API 名称。 | +| Bench Dtype | 标杆数据的 API 数据类型。 | +| DEVICE Dtype | NPU 或 GPU 数据的 API 数据类型。 | +| Shape | API 的 Shape信息。 | +| 余弦相似度 | NPU 或 GPU 数据与标杆数据的余弦相似度。 | +| 最大绝对误差 | NPU 或 GPU 数据与标杆数据的最大绝对误差。 | +| 双百指标 | 双百精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于百分之一的个数占总元素个数的比例。测试通过标准为相对误差大于百分之一的个数占总元素个数的比例小于百分之一。 | +| 双千指标 | 双千精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。 | +| 双万指标 | 双万精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于万分之一的个数占总元素个数的比例。测试通过标准为相对误差大于万分之一的个数占总元素个数的比例小于万分之一。 | +| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。 | +| 误差均衡性 | NPU 或 GPU 数据与标杆数据精度差的上下浮动情况。 | +| 均方根误差 | NPU 或 GPU 数据与标杆数据的均方根误差。 | +| 小值域错误占比 | NPU 或 GPU Tensor 中与标杆的绝对误差大于错误阈值的小值在小值域（小值的总数量）中的占比。判断为小值以及绝对误差的错误阈值参见\*2。 | +| 相对误差最大值 | NPU 或 GPU 数据与标杆数据相对误差的最大值。 | +| 相对误差平均值 | NPU 或 GPU 数据与标杆数据相对误差的平均值。 | +| inf/nan 错误率 | NPU 与标杆 inf/nan 计算不一致的元素个数占总元素的个数比例。 | +| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。 | +| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。 | +| ULP 误差最大值 | NPU 或 GPU 数据与标杆数据 ULP 误差的最大值（取绝对值后）。 | +| ULP 误差平均值 | NPU 或 GPU 数据与标杆数据ULP误差的平均值（取绝对值后）。 | +| ULP 误差大于阈值占比 | NPU 或 GPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 或 GPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 或 GPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。 | +| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过，warning 表示测试未通过双千或双万精度指标，SKIP 表示该 API 的某个参数的反向不要计算梯度，所以没有任何计算过程，其他信息均为空。 | +| message | 提示信息。 | + +**\***： + +1. API 预检指标： + + API 预检指标是通过对 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差双百、双千、双万精度指标的数值进行判断，得出该 API 是否符合精度标准的参考指标。 + + API预检通过测试，则在`accuracy_checking_details_{timestamp}.csv`文件中的 Status 列标记 pass，否则标记 error 或 warning，详细规则如下： + + - 余弦相似度 > 0.99：≤ 0.99 为不达标，标记 error，> 0.99 达标，进行下一步； + - 最大绝对误差 ＜ 0.001：＜ 0.001 达标，标记 pass，≥ 0.001 为不达标，进行下一步； + - 双百、双千、双万精度指标： + + 对于 float16 和 bfloat16 数据：双百指标不通过，标记 error；双百指标通过，双千指标不通过，标记 warning；双百、双千指标均通过，标记 pass。 + + 对于 float32 和 float64 数据：双千指标不通过，标记 error；双千指标通过，双万指标不通过，标记 warning；双千、双万指标均通过，标记 pass。 + - 在 `accuracy_checking_result_{timestamp}.csv` 中以 Forward Test Success 和 Backward Test Success 字段统计该算子前向反向输出的测试结果，对于标记 pass 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE 表示测试通过，对于标记 error 或 warning 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 FALSE 表示测试不通过。由于一个算子可能有多个前向或反向的输入或输出，那么该类算子的输入或输出中必须全为 pass，才能在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE，只要有一个输入或输出标记 error 或 warning，那么在`accuracy_checking_result_{timestamp}.csv`中标记 FALSE。 + +2. 小值域阈值： + + 判定为小值的阈值： + + - torch.float32：e-6 + - torch.float16：e-3 + - torch.bfloat16：e-3 + + 小值域的绝对误差阈值： + + - torch.float32：e-9 + - torch.float16：e-5 + - torch.bfloat16：e-5 + +## 5 预检结果比对 + +需要同时获取 NPU 和 GPU 环境下 run_ut 操作的预检结果 `accuracy_checking_details_{timestamp}.csv` 文件。执行如下命令进行 NPU 和 GPU 预检结果的比对： + +```bash +msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_details_{timestamp}.csv -gpu /home/xxx/gpu/accuracy_checking_details_{timestamp}.csv -o /home/xxx/ +``` + +| 参数名称 | 说明 | 是否必选 | +| -------------------- | ------------- | -------- | +| -npu 或 --npu_csv_path | NPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | +| -gpu 或 --gpu_csv_path | GPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | +| -o 或 --out_path | 指定 api_precision_compare.py 执行结果存盘路径，默认为当前目录。 | 否 | + +执行完成后输出 `api_precision_compare_result_{timestamp}.csv` 和 `api_precision_compare_details_{timestamp}.csv` 文件。文件示例如下： + +可以通过先查看 `api_precision_compare_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `api_precision_compare_details_{timestamp}.csv` 文件的 API 详细达标情况。 + +`api_precision_compare_result_{timestamp}.csv` + +![api_precision_compare_result](img/api_precision_compare_result.png) + +| 字段 | 含义 | +| --------------------- | ------------------------------------------------------------ | +| API name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | +| Message | 提示信息。 | + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `api_precision_compare_details_{timestamp}.csv` 中的各个指标判定结果决定的。需要注意的是 `api_precision_compare_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `api_precision_compare_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 warning 和 pass 且不存在 error 标记 warning。 + +`api_precision_compare_details_{timestamp}.csv` + +![api_precision_compare_details](img/api_precision_compare_details.png) + +| 字段 | 含义 | +| ------------------------ | ------------------------------------------------------------ | +| API name | NPU 或 GPU 下的 API 名称。 | +| 小值域错误比值 | NPU 与 CPU 的小值域的错误比率 / GPU 与 CPU 的小值域的错误比率。标杆比对法指标。 | +| 小值域错误判定结果 | 小值域错误比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | +| 均方根误差比值 | NPU 与 CPU 的均方根误差 / GPU 与 CPU 的均方根误差。标杆比对法指标。 | +| 均方根误差判定结果 | 均方根误差比值小于等于 1 标记为 pass，1~2 之间标记为 waring，大于 2 标记为 error。 | +| 相对误差最大值比值 | NPU 与 CPU 的相对误差最大值 / GPU 与 CPU 的相对误差最大值。标杆比对法指标。 | +| 相对误差最大值判定结果 | 相对误差最大值比值小于等于 1 标记为 pass，1 ~ 10 之间标记为 waring，大于 10 标记为 error。 | +| 相对误差平均值比值 | NPU 与 CPU 的相对误差的平均值 / GPU 与 CPU 的相对误差的平均值。标杆比对法指标。 | +| 相对误差平均值判定结果 | 相对误差平均值比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | +| 误差均衡性比值 | NPU 与 CPU 的误差均衡性 / GPU 与 CPU 的误差均衡性。标杆比对法指标。 | +| 误差均衡性判定结果 | 误差均衡性比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。该字段暂不参与 api_precision_compare_result 的结果判定。 | +| inf / nan 错误率 | NPU 与标杆 inf / nan 计算不一致的元素个数占总元素的个数比例。绝对阈值法指标。 | +| inf / nan 判定结果 | inf / nan 错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。绝对阈值法指标。 | +| 相对误差判定结果 | 相对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。绝对阈值法指标。 | +| 绝对误差判定结果 | 绝对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。二进制一致法指标。 | +| 二进制一致错误率判定结果 | 二进制一致错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| ULP 误差平均值 | NPU 数据与标杆数据 ULP 误差的平均值（取绝对值后）。ULP 误差比对法指标。 | +| ULP 误差大于阈值占比 | NPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。ULP 误差比对法指标。 | +| ULP 误差大于阈值占比比值 | NPU 与 CPU 的 ULP 误差大于阈值占比 / GPU 与 CPU 的 ULP 误差大于阈值占比。ULP 误差比对法指标。 | +| ULP 误差判定结果 | ULP 误差判定结果。
当 NPU 或 GPU 数据类型是 float16 或 bfloat16 时，以下两条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差大于阈值占比小于 0.001；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。
当 NPU 或 GPU 数据类型是 float32 时，以下三条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差平均值小于 64；
NPU ULP 误差大于阈值占比小于 0.05；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。 | +| 双千指标 | 双千精度指标。是指 NPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅 conv1d 和 conv2d 使用该指标。双千指标法指标。 | +| 双千指标判定结果 | 双千指标判定结果。双千指标大于 0.999 标记为 pass，否则标记为 error。 | +| 比对结果 | 综合所有指标的最终结果。如果比对指标中有 error，则标记为 error；有 warning，则标记为 warning；否则标记为 pass。 | +| 比对算法 | API 使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和 ULP 误差比对法中的一种。 | +| Message | 提示信息。当前提示该 API 比对结果为 error 或 warning 时对应不符合标准的指标。 | diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 0b3fec632..d7ff0e5a6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -31,7 +31,7 @@ class PrecisionDebugger: return self.initialized = True if not config_path: - config_path = os.path.join(os.path.dirname(__file__), "../../config/config.json") + config_path = os.path.join(os.path.dirname(__file__), "../../config.json") common_config, task_config = parse_json_config(config_path) self.task = common_config.task if self.task == Const.GRAD_PROBE: diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index f86ac1580..d1d95c3e5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -176,7 +176,7 @@ def parse_task_config(task, json_config): def parse_json_config(json_file_path, task): if not json_file_path: config_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__))) - json_file_path = os.path.join(os.path.join(config_dir, "config"), "config.json") + json_file_path = os.path.join(config_dir, "config.json") with FileOpen(json_file_path, 'r') as file: json_config = json.load(file) common_config = CommonConfig(json_config) -- Gitee From ce5f1c1afeab0fd1bacba9e349de7b4be7935818 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 17:27:34 +0800 Subject: [PATCH 500/791] renew_manual --- .../docs/07.accuracy_checker_PyTorch.md | 278 ------------------ 1 file changed, 278 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md diff --git a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md deleted file mode 100644 index 72fb01e2e..000000000 --- a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md +++ /dev/null @@ -1,278 +0,0 @@ -# PyTorch 场景的离线精度预检 - -## 1 简介 - -**PyTorch 离线精度预检**\*1通过扫描昇腾 NPU 上用户训练模型中的所有 API，输出模型精度的诊断和分析结果。具体而言，该工具通过采集模型中所有 API 的前反向信息，构造相应的单元测试，将 NPU 输出与标杆（CPU 高精度）比对，从而计算对应的精度指标，该过程已封装为 run_ut.py 函数；将 NPU 环境下采集的预检数据拷贝至 GPU 环境，同样执行 run_ut.py；最后通过**新精度标准比对法**\*2将 NPU 和 GPU 的预检结果进行比对，从而找出 NPU 中存在精度问题的 API。同时，本工具支持随机生成模式和真实数据模式\*3。 - -**\***： - -1. 支持 PyTorch 版本：1.11/2.0/2.1/2.2； -2. 依据新精度标准，对不同的API采取不同的比对算法（包括绝对阈值法，标杆比对法、二进制一致法、ULP误差比对法和双千指标法），最终给定预检判定结果； -3. 在预检 dump 时可以选择由工具构造随机数获得 dump 数据或选择真实输入的数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 - -## 2 离线预检流程 - -1. 在 NPU 和 GPU 环境下分别安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 -2. 在 NPU 训练脚本内添加 msprobe 工具 dump 接口 PrecisionDebugger，采集待预检数据。注意需要配置 level="L1"。 -3. 将 NPU 环境下 dump 的预检数据拷贝至 GPU 环境。 -4. 在 NPU 和 GPU 环境下分别执行 run_ut.py，生成的结果最终用于 api_precision_compare.py 函数的输入。详见 [3 离线预检操作指导](#3-离线预检操作指导)。 -5. 将 NPU 和 GPU 执行 run_ut.py 生成的 `accuracy_checking_details_{timestamp}.csv` 结果文件拷贝至同一环境下。 -6. 运行 api_precision_compare.py，输出结果为预检操作的最终结果。详见 [5 预检结果比对](#5-预检结果比对)章节。 - -## 3 离线预检操作指导 - -完成预检数据采集后，仅仅获取了 API 的输入数据，为了得到 NPU vs. CPU 高精度（标杆）的预检比对结果和 GPU vs. CPU 高精度（标杆）的预检比对结果，还需要进行 run_ut 操作。 - -run_ut 预检操作包括以下两种方式： - -- 使用 run_ut.py 执行预检：适用于数据量较小的单卡场景。 -- 使用 multi_run_ut.py 执行多线程预检：适用于数据量较大的大模型场景。 - -### 3.1 使用 run_ut.py 执行预检 - -将 API 信息输入到 run_ut 模块进行精度检测并比对，运行如下命令： - -```bash -msprobe -f pytorch run_ut -api_info ./dump.json -``` - - | 参数名称 | 解释 | 是否必选 | - | ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | - | -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | - | -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | - | -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | - | | | | - | -j 或 --jit_compile | 开启 jit 编译。 | 否 | - | -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0。 | 否 | - | -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | - | -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | - | -config 或 --config_path | 指定离线预检操作过程中额外配置（包括黑名单、白名单等）的 [config.json](../config.json) 文件，默认未配置。config.json 文件的配置可参考[配置文件介绍](./02.config_introduction.md)。 | 否 | - -run_ut 执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见[ 4 预检结果](#4-预检结果)。 - -如果需要保存比对不达标的输入和输出数据，可以在 run_ut 执行命令结尾添加 `-save_error_data`，例如： - -```bash -msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data -``` - -数据默认会存盘到 './ut_error_data{timestamp}' 路径下（相对于启动 run_ut 的路径），如有需要，用户可以通过 error_data_path 参数来配置保存路径，error_data_path 参数在 [config.json](../config.json) 文件或 [config.yaml](../pytorch/api_accuracy_checker/config.yaml) 文件配置，config.json 文件需要在 run_ut 操作时通过 -config 参数指定，config.yaml 文件详见\*1。 - -**\***： - -1. config.yaml 文件说明 - - config.yaml 文件可以通过配置参数来控制 dump 和 run_ut 操作的白名单、黑名单等功能。操作步骤如下： - - - config.yaml 文件通常位于类似 /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml 的路径中。 - - - 进入 config.yaml 文件 - - ```bash - vim /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml - ``` - - - 修改 config.yaml 文件参数。 - - ```yaml - white_list: [] - black_list: [] - error_data_path: './' - precision: 14 - ``` - - | 参数名称 | 解释 | 是否必选 | - | ----------- | -----------| -------- | - | white_list | API dump 白名单，仅对指定的 API 进行 dump。
**参数示例**：white_list=["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | - | black_list | API dump 黑名单，被指定的 API 不进行 dump。
**参数示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | - | error_data_path | 配置保存精度未达标的 API 输入输出数据路径。
**参数示例**："error_data_path": "./"。默认为当前路径。 | 否 | - | precision | 浮点数表示位数，默认取小数点后14位。 | 否 | - - 说明：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 - -2. API 预检黑名单和白名单 - - run_ut 过程支持 API 预检黑名单和白名单，通过如下文件配置 black_list（黑名单）或white_list（白名单）参数来指定不需要或需要预检的API名称： - - - 配置 [config.json](../config.json) 文件，该文件需要在 run_ut 操作时通过 -config 参数指定。 - - - config.json 文件的优先级高于 config.yaml 文件，即执行 config.json 文件时，config.yaml 文件的配置不生效。 - -### 3.2 使用 multi_run_ut.py 执行多线程预检 - -multi_run_ut.py 脚本，可以并行执行多个 run_ut 操作，从而减少预检耗时。示例如下： - -```bash -msprobe -f pytorch multi_run_ut -api_info ./dump.json -n 32 -d 0 1 2 3 -``` - -| 参数名称 | 解释 | 是否必选 | -| ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | -| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | -| -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | -| -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | -| -j 或 --jit_compile | 开启 jit 编译。 | 否 | -| -n | 同时执行 run_ut 线程的数量，默认为 8，最大支持 64，但每个 Device 最大支持 8 个线程。当指定多个线程和多个 Device 时，线程数在每张卡上均分。 | 否 | -| -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0，支持同时指定 0~7，共 8 个 Device。 | 否 | -| -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | -| -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | - -### 3.3 断点续检 - -断点续检操作通过如下命令执行： - -```bash -msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_checking_result_{timestamp}.csv -``` - -精度预检 run_ut 过程中，若因环境、数据量过大等原因导致预检进程中断，那么当用户解决这些问题后，重新执行 run_ut 操作，可以通过断点续检操作继续前面未完成的预检，会在 -csv_path 指定的 `accuracy_checking_result_{timestamp}.csv` 文件以及对应的 `accuracy_checking_details_{timestamp}.csv` 文件中继续写入后续的结果，不会重新创建结果文件。 - -须指定为上次预检中断的 `accuracy_checking_result_{timestamp}.csv` 文件。请勿修改 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件名，包括时间戳，否则断点续检会因无法识别到文件名而失败。 - -## 4 预检结果 - -精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件示例如下： - -可以通过先查看 `accuracy_checking_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `accuracy_checking_details_{timestamp}.csv` 文件的 API 详细达标情况，API 达标情况介绍请参见\*1。 - -`accuracy_checking_result_{timestamp}.csv` - -![accuracy_checking_result](img/accuracy_checking_result.png) - -| 字段 | 含义 | -| --------------------- | ------------------------- | -| API name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。 | -| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出。 | -| Message | 提示信息。 | - -该结果为中间结果，仅作为参考，建议完成 [5 预检结果比对](#5-预检结果比对)后查看比对结果。该结果后续将会删除。 - -Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差、双百双千双万指标判定结果决定的。 - -需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 waring 和 pass 且不存在 error 则标记 waring。 - -`accuracy_checking_details_{timestamp}.csv` - -![accuracy_checking_details](img/accuracy_checking_details.png) - -| 字段 | 含义 | -| ------------------- | ------------------------------------------------------------ | -| API name | NPU 或 GPU下的 API 名称。 | -| Bench Dtype | 标杆数据的 API 数据类型。 | -| DEVICE Dtype | NPU 或 GPU 数据的 API 数据类型。 | -| Shape | API 的 Shape信息。 | -| 余弦相似度 | NPU 或 GPU 数据与标杆数据的余弦相似度。 | -| 最大绝对误差 | NPU 或 GPU 数据与标杆数据的最大绝对误差。 | -| 双百指标 | 双百精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于百分之一的个数占总元素个数的比例。测试通过标准为相对误差大于百分之一的个数占总元素个数的比例小于百分之一。 | -| 双千指标 | 双千精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。 | -| 双万指标 | 双万精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于万分之一的个数占总元素个数的比例。测试通过标准为相对误差大于万分之一的个数占总元素个数的比例小于万分之一。 | -| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。 | -| 误差均衡性 | NPU 或 GPU 数据与标杆数据精度差的上下浮动情况。 | -| 均方根误差 | NPU 或 GPU 数据与标杆数据的均方根误差。 | -| 小值域错误占比 | NPU 或 GPU Tensor 中与标杆的绝对误差大于错误阈值的小值在小值域（小值的总数量）中的占比。判断为小值以及绝对误差的错误阈值参见\*2。 | -| 相对误差最大值 | NPU 或 GPU 数据与标杆数据相对误差的最大值。 | -| 相对误差平均值 | NPU 或 GPU 数据与标杆数据相对误差的平均值。 | -| inf/nan 错误率 | NPU 与标杆 inf/nan 计算不一致的元素个数占总元素的个数比例。 | -| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。 | -| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。 | -| ULP 误差最大值 | NPU 或 GPU 数据与标杆数据 ULP 误差的最大值（取绝对值后）。 | -| ULP 误差平均值 | NPU 或 GPU 数据与标杆数据ULP误差的平均值（取绝对值后）。 | -| ULP 误差大于阈值占比 | NPU 或 GPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 或 GPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 或 GPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。 | -| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过，warning 表示测试未通过双千或双万精度指标，SKIP 表示该 API 的某个参数的反向不要计算梯度，所以没有任何计算过程，其他信息均为空。 | -| message | 提示信息。 | - -**\***： - -1. API 预检指标： - - API 预检指标是通过对 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差双百、双千、双万精度指标的数值进行判断，得出该 API 是否符合精度标准的参考指标。 - - API预检通过测试，则在`accuracy_checking_details_{timestamp}.csv`文件中的 Status 列标记 pass，否则标记 error 或 warning，详细规则如下： - - - 余弦相似度 > 0.99：≤ 0.99 为不达标，标记 error，> 0.99 达标，进行下一步； - - 最大绝对误差 ＜ 0.001：＜ 0.001 达标，标记 pass，≥ 0.001 为不达标，进行下一步； - - 双百、双千、双万精度指标： - + 对于 float16 和 bfloat16 数据：双百指标不通过，标记 error；双百指标通过，双千指标不通过，标记 warning；双百、双千指标均通过，标记 pass。 - + 对于 float32 和 float64 数据：双千指标不通过，标记 error；双千指标通过，双万指标不通过，标记 warning；双千、双万指标均通过，标记 pass。 - - 在 `accuracy_checking_result_{timestamp}.csv` 中以 Forward Test Success 和 Backward Test Success 字段统计该算子前向反向输出的测试结果，对于标记 pass 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE 表示测试通过，对于标记 error 或 warning 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 FALSE 表示测试不通过。由于一个算子可能有多个前向或反向的输入或输出，那么该类算子的输入或输出中必须全为 pass，才能在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE，只要有一个输入或输出标记 error 或 warning，那么在`accuracy_checking_result_{timestamp}.csv`中标记 FALSE。 - -2. 小值域阈值： - - 判定为小值的阈值： - - - torch.float32：e-6 - - torch.float16：e-3 - - torch.bfloat16：e-3 - - 小值域的绝对误差阈值： - - - torch.float32：e-9 - - torch.float16：e-5 - - torch.bfloat16：e-5 - -## 5 预检结果比对 - -需要同时获取 NPU 和 GPU 环境下 run_ut 操作的预检结果 `accuracy_checking_details_{timestamp}.csv` 文件。执行如下命令进行 NPU 和 GPU 预检结果的比对： - -```bash -msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_details_{timestamp}.csv -gpu /home/xxx/gpu/accuracy_checking_details_{timestamp}.csv -o /home/xxx/ -``` - -| 参数名称 | 说明 | 是否必选 | -| -------------------- | ------------- | -------- | -| -npu 或 --npu_csv_path | NPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | -| -gpu 或 --gpu_csv_path | GPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | -| -o 或 --out_path | 指定 api_precision_compare.py 执行结果存盘路径，默认为当前目录。 | 否 | - -执行完成后输出 `api_precision_compare_result_{timestamp}.csv` 和 `api_precision_compare_details_{timestamp}.csv` 文件。文件示例如下： - -可以通过先查看 `api_precision_compare_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `api_precision_compare_details_{timestamp}.csv` 文件的 API 详细达标情况。 - -`api_precision_compare_result_{timestamp}.csv` - -![api_precision_compare_result](img/api_precision_compare_result.png) - -| 字段 | 含义 | -| --------------------- | ------------------------------------------------------------ | -| API name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | -| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | -| Message | 提示信息。 | - -Forward Test Success 和 Backward Test Success 是否通过测试是由 `api_precision_compare_details_{timestamp}.csv` 中的各个指标判定结果决定的。需要注意的是 `api_precision_compare_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `api_precision_compare_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 warning 和 pass 且不存在 error 标记 warning。 - -`api_precision_compare_details_{timestamp}.csv` - -![api_precision_compare_details](img/api_precision_compare_details.png) - -| 字段 | 含义 | -| ------------------------ | ------------------------------------------------------------ | -| API name | NPU 或 GPU 下的 API 名称。 | -| 小值域错误比值 | NPU 与 CPU 的小值域的错误比率 / GPU 与 CPU 的小值域的错误比率。标杆比对法指标。 | -| 小值域错误判定结果 | 小值域错误比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | -| 均方根误差比值 | NPU 与 CPU 的均方根误差 / GPU 与 CPU 的均方根误差。标杆比对法指标。 | -| 均方根误差判定结果 | 均方根误差比值小于等于 1 标记为 pass，1~2 之间标记为 waring，大于 2 标记为 error。 | -| 相对误差最大值比值 | NPU 与 CPU 的相对误差最大值 / GPU 与 CPU 的相对误差最大值。标杆比对法指标。 | -| 相对误差最大值判定结果 | 相对误差最大值比值小于等于 1 标记为 pass，1 ~ 10 之间标记为 waring，大于 10 标记为 error。 | -| 相对误差平均值比值 | NPU 与 CPU 的相对误差的平均值 / GPU 与 CPU 的相对误差的平均值。标杆比对法指标。 | -| 相对误差平均值判定结果 | 相对误差平均值比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | -| 误差均衡性比值 | NPU 与 CPU 的误差均衡性 / GPU 与 CPU 的误差均衡性。标杆比对法指标。 | -| 误差均衡性判定结果 | 误差均衡性比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。该字段暂不参与 api_precision_compare_result 的结果判定。 | -| inf / nan 错误率 | NPU 与标杆 inf / nan 计算不一致的元素个数占总元素的个数比例。绝对阈值法指标。 | -| inf / nan 判定结果 | inf / nan 错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | -| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。绝对阈值法指标。 | -| 相对误差判定结果 | 相对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | -| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。绝对阈值法指标。 | -| 绝对误差判定结果 | 绝对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | -| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。二进制一致法指标。 | -| 二进制一致错误率判定结果 | 二进制一致错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | -| ULP 误差平均值 | NPU 数据与标杆数据 ULP 误差的平均值（取绝对值后）。ULP 误差比对法指标。 | -| ULP 误差大于阈值占比 | NPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。ULP 误差比对法指标。 | -| ULP 误差大于阈值占比比值 | NPU 与 CPU 的 ULP 误差大于阈值占比 / GPU 与 CPU 的 ULP 误差大于阈值占比。ULP 误差比对法指标。 | -| ULP 误差判定结果 | ULP 误差判定结果。
当 NPU 或 GPU 数据类型是 float16 或 bfloat16 时，以下两条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差大于阈值占比小于 0.001；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。
当 NPU 或 GPU 数据类型是 float32 时，以下三条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差平均值小于 64；
NPU ULP 误差大于阈值占比小于 0.05；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。 | -| 双千指标 | 双千精度指标。是指 NPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅 conv1d 和 conv2d 使用该指标。双千指标法指标。 | -| 双千指标判定结果 | 双千指标判定结果。双千指标大于 0.999 标记为 pass，否则标记为 error。 | -| 比对结果 | 综合所有指标的最终结果。如果比对指标中有 error，则标记为 error；有 warning，则标记为 warning；否则标记为 pass。 | -| 比对算法 | API 使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和 ULP 误差比对法中的一种。 | -| Message | 提示信息。当前提示该 API 比对结果为 error 或 warning 时对应不符合标准的指标。 | -- Gitee From f91e303d51d40047f665c199967ac6f7d29a1afa Mon Sep 17 00:00:00 2001 From: mei-feiyao <1332490378@qq.com> Date: Mon, 26 Aug 2024 17:30:35 +0800 Subject: [PATCH 501/791] master add bind core --- README.md | 3 + profiler/affinity_cpu_bind/README.md | 42 +++++ profiler/affinity_cpu_bind/bind_core.py | 214 ++++++++++++++++++++++++ 3 files changed, 259 insertions(+) create mode 100644 profiler/affinity_cpu_bind/README.md create mode 100644 profiler/affinity_cpu_bind/bind_core.py diff --git a/README.md b/README.md index 601deec66..7a7bd20b1 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,9 @@ MindStudio Training Tools，MindStudio训练工具链。针对训练&大模型 将Ascend PyTorch Profiler或者msprof采集的PyThon场景性能数据进行分析，并输出性能调优建议。 +4. [bind_core](https://gitee.com/ascend/mstt/blob/master/profiler/affinity_cpu_bind) + + 绑核脚本，支持非侵入修改工程代码，实现一键式绑核功能 ### [Tensorboard](https://gitee.com/ascend/mstt/tree/master/plugins/tensorboard-plugins/tb_plugin) diff --git a/profiler/affinity_cpu_bind/README.md b/profiler/affinity_cpu_bind/README.md new file mode 100644 index 000000000..82ecdec63 --- /dev/null +++ b/profiler/affinity_cpu_bind/README.md @@ -0,0 +1,42 @@ +# 昇腾亲和性CPU绑核工具 + +昇腾亲和性CPU绑核工具支持用户无需修改代码，直接运行工具即可实现按CPU亲和性策略绑核，提升推理或训练性能。 + +绑核工具用户arm服务器环境，对于训练或推理任务因为CPU资源调度等出现host_bound问题时使用，可改善该问题；对于非host_bound的场景无明显改善效果。 + +## 使用须知 + +使用绑核工具前手动执行npu-smi info -t topo，出现以下类似信息，说明环境支持绑核，否则请将环境HDK包升级到Ascend HDK 23.0.RC2及以上版本。 + + NPU0 NPU1 NPU2 NPU3 NPU4 NPU5 NPU6 NPU7 NPUx CPU Affinity + NPU0 X HCCS HCCS HCCS HCCS HCCS HCCS HCCS ... xx-xx + NPU1 HCCS X HCCS HCCS HCCS HCCS HCCS HCCS ... xx-xx + NPU2 HCCS HCCS X HCCS HCCS HCCS HCCS HCCS ... xx-xx + NPU3 HCCS HCCS HCCS X HCCS HCCS HCCS HCCS ... xx-xx + NPU4 HCCS HCCS HCCS HCCS X HCCS HCCS HCCS ... xx-xx + NPU5 HCCS HCCS HCCS HCCS HCCS X HCCS HCCS ... xx-xx + NPU6 HCCS HCCS HCCS HCCS HCCS HCCS X HCCS ... xx-xx + NPU7 HCCS HCCS HCCS HCCS HCCS HCCS HCCS X ... xx-xx + NPUx ... ... ... ... ... ... ... ... ... ... + +## 使用方式 + +1.执行以下命令实施绑核： + + - 直接执行绑核命令 +```bash +python3 bind_core.py -app="inferenec/train cmd" +``` +-app或--application：配置训练或推理程序的执行命令，执行命令前后加引号。 + +该方式会自动拉起训练或推理任务，检测任务进程，并实施绑核。 + + - 手动拉起训练或推理任务后再执行绑核 +```bash +python3 bind_core.py +``` +该方式会循环查找(循环5次，每次10s，若找不到进程，则直接退出)使用到NPU的任务进程，并实施绑核。 + +2.绑核运行过程的日志会保存到当前路径的bind_core_时间戳.log。 + +3.如果推理或训练进程拉起后需要一定时间预处理，才会真正执行任务，可在执行绑核命令时设置-t/--time参数(单位秒)，绑核工具会在延迟配置的时间后，再实施绑核动作。例如：python3 bind_core.py -app="cmd" -t=10，配置后工具会在10秒后执行绑核操作。 \ No newline at end of file diff --git a/profiler/affinity_cpu_bind/bind_core.py b/profiler/affinity_cpu_bind/bind_core.py new file mode 100644 index 000000000..ca9d6f188 --- /dev/null +++ b/profiler/affinity_cpu_bind/bind_core.py @@ -0,0 +1,214 @@ +import subprocess +import argparse +import os +import time +import logging +from datetime import datetime +from datetime import timezone + + +class PathManager: + DATA_FILE_AUTHORITY = 0o640 + + @classmethod + def create_file_safety(cls, path: str): + base_name = os.path.basename(path) + msg = f"Failed to create file: {base_name}" + if os.path.islink(path): + raise RuntimeError(msg) + if os.path.exists(path): + return + try: + os.close(os.open(path, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY)) + except Exception as err: + raise RuntimeError(msg) from err + + +class BindCoreManager(): + DEFAULT_FIND_RUNNING_PID_TIMES = 5 + + def __init__(self): + self.npu_id_list = [] + self.running_pid_on_npu = {} + self.find_running_pid_times = self.DEFAULT_FIND_RUNNING_PID_TIMES + self.npu_affinity_cpu_dict = {} + self.log_file = '' + self._init_log_file() + + @staticmethod + def _launch_process(cmd: list): + logging.info('Start to execute cmd: %s', cmd) + try: + subprocess.Popen(cmd.split(), shell=False) + except subprocess.CalledProcessError as e: + raise RuntimeError(f'Failed to run cmd: {cmd}') from e + + def get_running_pid_on_npu(self) -> bool: + no_running_pids_on_npu_msg = '[INFO] Now there is no running process on all NPUs, stop bind cores' + logging.info('Begin to find running process on all NPUs') + # get running process on NPUs + for _ in range(self.find_running_pid_times): + running_pid_on_npu = {} + for npu_id in self.npu_id_list: + get_npu_pids_cmd = 'npu-smi info -t proc-mem -i {} -c 0'.format(npu_id) + get_npu_pids_process = subprocess.run(get_npu_pids_cmd.split(), shell=False, capture_output=True) + res = get_npu_pids_process.stdout.decode('utf-8').split() + pid_list = [] + for value in res: + if value.startswith('id:'): + pid = value.split(':')[1] + pid_list.append(pid) + if pid_list: + running_pid_on_npu[npu_id] = list(set(pid_list)) + + if len(running_pid_on_npu) < len(self.npu_id_list): + time.sleep(5) + continue + + if len(self.running_pid_on_npu.keys()) == len(running_pid_on_npu.keys()) and running_pid_on_npu: + self.running_pid_on_npu = running_pid_on_npu + break + + self.running_pid_on_npu = running_pid_on_npu + + # delete repeat pid + for npu_id in self.npu_id_list: + if npu_id not in self.running_pid_on_npu: + continue + pids_on_npu = self.running_pid_on_npu[npu_id] + for npu_id_with_pids, pids in self.running_pid_on_npu.items(): + if npu_id == npu_id_with_pids: + continue + pids_on_npu = list(set(pids_on_npu) - set(pids)) + self.running_pid_on_npu[npu_id] = pids_on_npu + + if_running_process = False + for npu_id, pids in self.running_pid_on_npu.items(): + if not pids: + logging.info('There is no running process on NPU %d', npu_id) + else: + logging.info('Succeed to find running process %s on NPU %d', pids, npu_id) + if_running_process = True + if not if_running_process: + print(no_running_pids_on_npu_msg) + return if_running_process + + def get_npu_info(self) -> bool: + try: + self._get_all_npu_id() + if not self._get_npu_affinity(): + return False + except subprocess.CalledProcessError: + return False + return True + + def run_bind_core(self): + if not self.running_pid_on_npu: + return + for npu, pid_list in self.running_pid_on_npu.items(): + if npu not in self.npu_affinity_cpu_dict.keys(): + logging.warning('Cannot find affinity cpu for npu: %d', npu) + continue + affinity_cpu = self.npu_affinity_cpu_dict.get(npu) + for pid in pid_list: + try: + logging.info('Begin to bind cores for process %s on NPU %d', pid, npu) + set_affinity_cpu_cmd = 'taskset -pc {} {}'.format(affinity_cpu, pid) + p = subprocess.run(set_affinity_cpu_cmd.split(), shell=False, capture_output=True) + logging.info(p.stdout.decode('utf-8')) + except subprocess.CalledProcessError: + print('[ERROR] Failed to bind process {} on NPU {} with cpu cores list {}'.format(pid, npu, affinity_cpu)) + + logging.info('Succeed to bind process %s on NPU %d with cpu cores list %s', pid, npu, affinity_cpu) + + def args_parse(self): + parser = argparse.ArgumentParser(description='This is a affinity cpu core bind script.') + parser.add_argument('-t', '--time', type=int, metavar='', help='Wait time before bind cores that you want to set. The unit is \'s\'.') + parser.add_argument('-app', '--application', metavar='', nargs='+', help='Training or inference command that you want to run.') + args = parser.parse_args() + if args.application: + application_cmd = ' '.join(args.application) + self._launch_process(application_cmd) + time.sleep(2) + # if time is set, wait for setting time before bind cores + if args.time: + time.sleep(args.time) + + def _init_log_file(self): + now_time = datetime.now(tz=timezone.utc) + time_stamp = str(now_time.year) + '_' + \ + str(now_time.month) + '_' + \ + str(now_time.day) + '_' + \ + str(now_time.hour) + '_' + \ + str(now_time.minute) + '_' + \ + str(now_time.second) + log_file_name = 'bind_core_' + time_stamp + '.log' + msg = f"Failed to create file: {log_file_name}" + try: + PathManager.create_file_safety(os.path.join(os.getcwd(), log_file_name)) + except RuntimeError as err: + raise RuntimeError(msg) from err + self.log_file = log_file_name + logging.basicConfig(filename=self.log_file, + level=logging.INFO, + filemode='w', + format='%(asctime)s-%(name)s-%(levelname)s-%(message)s') + + def _get_all_npu_id(self) -> None: + get_npu_info_cmd = 'npu-smi info -l' + get_npu_info_process = subprocess.run(get_npu_info_cmd.split(), shell=False, capture_output=True) + get_npu_id_cmd = 'grep ID' + get_npu_id_process = subprocess.run(get_npu_id_cmd.split(), shell=False, input=get_npu_info_process.stdout, capture_output=True) + res = get_npu_id_process.stdout.decode('utf-8').split() + for i in res: + if i.isdigit(): + self.npu_id_list.append(int(i)) + logging.info('NPU total id list: %s', self.npu_id_list) + + def _get_npu_affinity(self) -> bool: + cpu_num = os.cpu_count() + cpu_num_for_each_npu = cpu_num // len(self.npu_id_list) + get_npu_topo_cmd = 'npu-smi info -t topo' + p = subprocess.run(get_npu_topo_cmd.split(), shell=False, capture_output=True) + res = p.stdout.decode('utf-8').split() + if not res: + print('[ERROR] Failed to run get npu affinity info, please check if driver version support cmd npu-smi info -t topo') + return False + + index = 0 + for v in res: + if '-' in v: + affinity_cpus = [] + cpu_lists = v.split(',') + for cpu_list in cpu_lists: + cpus = cpu_list.split('-') + if len(cpus) != 2: + continue + if int(cpus[1]) - int(cpus[0]) == cpu_num_for_each_npu - 1: + cpus[1] = str(int(cpus[1]) + cpu_num_for_each_npu) + affinity_cpus.append(cpus[0] + '-' + cpus[1]) + if index < len(self.npu_id_list): + self.npu_affinity_cpu_dict[self.npu_id_list[index]] = ','.join(affinity_cpu for affinity_cpu in affinity_cpus) + index += 1 + else: + print('[ERROR] Get affinity_cpu_list for {} npus, more than real npu num: {}'.format(index + 1, len(self.npu_id_list))) + return False + + for k in self.npu_affinity_cpu_dict.keys(): + logging.info('Affinity CPU list [%s] for NPU %d', self.npu_affinity_cpu_dict[k], k) + return True + + +if __name__ == '__main__': + print('[INFO] Begin to run bind-cores script...') + bind_core_manager = BindCoreManager() + bind_core_manager.args_parse() + + if not bind_core_manager.get_npu_info(): + print('[ERROR] Failed to get current npus info') + exit() + + if not bind_core_manager.get_running_pid_on_npu(): + exit() + bind_core_manager.run_bind_core() + print('[INFO] End to run bind-cores script, the log is saved in {}'.format(bind_core_manager.log_file)) -- Gitee From 1f3f4c6eb81e1c4e44c944b666b18316e875d7b8 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 18:40:39 +0800 Subject: [PATCH 502/791] renew_manual --- README.md | 109 ++-- debug/accuracy_tools/README.md | 21 - debug/accuracy_tools/msprobe/README.md | 252 ++------ .../msprobe/{config => }/config.json | 0 debug/accuracy_tools/msprobe/config/README.md | 539 ------------------ .../msprobe/docs/06.data_dump_MindSpore.md | 157 +++++ .../docs/07.accuracy_checker_PyTorch.md | 278 +++++++++ .../08.accuracy_checker_online_Pytorch.md | 182 ++++++ .../docs/09.accuracy_checker_MindSpore.md | 70 +++ .../docs/10.accuracy_compare_PyTorch.md | 247 ++++++++ .../docs/11.accuracy_compare_MindSpore.md | 62 ++ .../msprobe/docs/12.overflow_check_PyTorch.md | 25 + .../msprobe/docs/14.data_parse_PyTorch.md | 283 +++++++++ .../grad_probe.md => docs/17.grad_probe.md} | 18 +- .../msprobe/docs/FAQ_PyTorch.md | 193 +++++++ ...robe_dump_standard_performance_baseline.md | 182 ++++++ ...marking_validation_performance_baseline.md | 151 +++++ .../msprobe/docs/img/BLOOM-7B_1.png | Bin 0 -> 3534 bytes .../msprobe/docs/img/BLOOM-7B_2.png | Bin 0 -> 5251 bytes .../msprobe/docs/img/BLOOM-7B_3.png | Bin 0 -> 5095 bytes .../msprobe/docs/img/BLOOM-7B_4.png | Bin 0 -> 4727 bytes .../msprobe/docs/img/GPT-3_1.png | Bin 0 -> 7898 bytes .../msprobe/docs/img/GPT-3_2.png | Bin 0 -> 6716 bytes .../msprobe/docs/img/GPT-3_3.png | Bin 0 -> 8732 bytes .../msprobe/docs/img/GPT-3_4.png | Bin 0 -> 6609 bytes .../msprobe/docs/img/GPT-3_5.png | Bin 0 -> 4676 bytes .../msprobe/docs/img/GPT-3_6.png | Bin 0 -> 7658 bytes .../msprobe/docs/img/GPT-3_7.png | Bin 0 -> 8139 bytes .../msprobe/docs/img/GPT-3_8.png | Bin 0 -> 6084 bytes .../msprobe/docs/img/YOLOV5S_1.png | Bin 0 -> 7019 bytes .../msprobe/docs/img/YOLOV5S_2.png | Bin 0 -> 5796 bytes .../docs/img/accuracy_checking_details.png | Bin 0 -> 39062 bytes .../docs/img/accuracy_checking_result.png | Bin 0 -> 6022 bytes .../img/api_precision_compare_details.png | Bin 0 -> 25476 bytes .../docs/img/api_precision_compare_result.png | Bin 0 -> 4538 bytes .../msprobe/docs/img/auto_analyze_log.png | Bin 0 -> 65144 bytes .../msprobe/docs/img/compare_result_pkl.png | Bin 0 -> 36133 bytes .../docs/img/compare_result_pkl_md5.png.png | Bin 0 -> 20464 bytes .../msprobe/docs/img/cpu_info.png | Bin 0 -> 33797 bytes .../{config => docs}/img/free_benchmark.png | Bin .../img/grad_probe_image-1.png} | Bin .../img/grad_probe_image-2.png} | Bin .../img/grad_probe_image-3.png} | Bin .../img/grad_probe_image-4.png} | Bin .../img/grad_probe_image.png} | Bin .../msprobe/docs/img/module_compare.png | Bin 0 -> 56655 bytes .../mindspore/debugger/precision_debugger.py | 2 +- .../msprobe/pytorch/pt_config.py | 2 +- 48 files changed, 1937 insertions(+), 836 deletions(-) delete mode 100644 debug/accuracy_tools/README.md rename debug/accuracy_tools/msprobe/{config => }/config.json (100%) delete mode 100644 debug/accuracy_tools/msprobe/config/README.md create mode 100644 debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md create mode 100644 debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md create mode 100644 debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md create mode 100644 debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md create mode 100644 debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md create mode 100644 debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md create mode 100644 debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md create mode 100644 debug/accuracy_tools/msprobe/docs/14.data_parse_PyTorch.md rename debug/accuracy_tools/msprobe/{doc/grad_probe/grad_probe.md => docs/17.grad_probe.md} (96%) create mode 100644 debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md create mode 100644 debug/accuracy_tools/msprobe/docs/S01.report_msprobe_dump_standard_performance_baseline.md create mode 100644 debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md create mode 100644 debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_1.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_2.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_3.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_4.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_1.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_2.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_3.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_4.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_5.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_6.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_7.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/GPT-3_8.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/YOLOV5S_1.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/YOLOV5S_2.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/accuracy_checking_details.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/accuracy_checking_result.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/api_precision_compare_details.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/api_precision_compare_result.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/auto_analyze_log.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/compare_result_pkl.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/compare_result_pkl_md5.png.png create mode 100644 debug/accuracy_tools/msprobe/docs/img/cpu_info.png rename debug/accuracy_tools/msprobe/{config => docs}/img/free_benchmark.png (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image-1.png => docs/img/grad_probe_image-1.png} (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image-2.png => docs/img/grad_probe_image-2.png} (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image-3.png => docs/img/grad_probe_image-3.png} (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image-4.png => docs/img/grad_probe_image-4.png} (100%) rename debug/accuracy_tools/msprobe/{doc/grad_probe/img/image.png => docs/img/grad_probe_image.png} (100%) create mode 100644 debug/accuracy_tools/msprobe/docs/img/module_compare.png diff --git a/README.md b/README.md index 601deec66..6a9b221a1 100644 --- a/README.md +++ b/README.md @@ -1,107 +1,80 @@ -# 变更通知 +# 🚨 重要通知 +**1. Ascend Training Tools 更名为 MindStudio Training Tools (mstt)。** +**2. 本代码仓 URL 变更为 [https://gitee.com/ascend/mstt](https://gitee.com/ascend/mstt)，原 URL 仍然可用（2024.07.04 ）。** -原Ascend Training Tools工具更名为MindStudio Training Tools，MindStudio训练工具链。变更计划如下： +**3. 不再维护：[api_accuracy_checker](./debug/accuracy_tools/api_accuracy_checker/) （2024.09.30下线）和[ ptdbg_ascend](./debug/accuracy_tools/ptdbg_ascend/) +（2024.09.30下线）** -1. 2024.06.25本代码仓名称变更为mstt。 -2. 2024.07.04 URL变更为[https://gitee.com/ascend/mstt](https://gitee.com/ascend/mstt)，原始URL仍然可用，但建议使用新URL。 +**相关目录 mstt/debug/accuracy_tools/api_accuracy_checker 和 mstt/debug/accuracy_tools/ptdbg_ascend 将于 2024.09.30 删除。新版本的预检和 ptdbg 已经合到 mstt/debug/accuracy_tools/msprobe 目录下。** -# MindStudio Training Tools +--- -MindStudio Training Tools，MindStudio训练工具链。针对训练&大模型场景，提供端到端命令行&可视化调试调优工具，帮助用户快速提高模型开发效率。 +# 🧰 MindStudio Training Tools -## 模型训练迁移全流程 -![输入图片说明](debug/resources/model_training_migration_process.png) +![Build Status](https://img.shields.io/badge/build-passing-brightgreen) +![Commit Activity](https://img.shields.io/badge/commit%20activity-high-red) +![License: Apache 2.0](https://img.shields.io/badge/license-Apache%202.0-blue) -## 使用说明 - -### [分析迁移工具](https://gitee.com/ascend/mstt/wikis/工具介绍/分析迁移工具/分析迁移工具介绍) +## [分析迁移工具](https://gitee.com/ascend/mstt/wikis/工具介绍/分析迁移工具/分析迁移工具介绍) 1. [脚本分析工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E5%88%86%E6%9E%90%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - 脚本分析工具提供分析脚本，帮助用户在执行迁移操作前，分析基于GPU平台的PyTorch训练脚本中算子、三方库套件、亲和API分析以及动态shape的支持情况。 + 脚本分析工具可以帮助用户在执行迁移操作前，分析基于 GPU 平台的 PyTorch 训练脚本中算子、三方库套件、API 亲和性以及动态 shape 的支持情况。 2. [（推荐）自动迁移工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E8%87%AA%E5%8A%A8%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - 自动迁移只需在训练脚本中导入库代码即可完成模型脚本迁移，使用方式较简单，且修改内容最少。 + 自动迁移工具只需在训练脚本中导入库代码即可完成模型脚本的迁移，使用方式简单，且修改内容少。 3. [脚本迁移工具](https://gitee.com/ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E8%84%9A%E6%9C%AC%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - 脚本迁移工具提供后端命令行用于将GPU上训练的PyTorch脚本迁移至NPU上，得到新的训练脚本用于训练。 + 脚本迁移工具通过后端命令行，将 GPU 上训练的 PyTorch 脚本迁移至 NPU 上，得到新的训练脚本用于训练。 4. [训推一体权重转换工具](https://gitee.com/Ascend/mstt/wikis/%E5%B7%A5%E5%85%B7%E4%BB%8B%E7%BB%8D/%E5%88%86%E6%9E%90%E8%BF%81%E7%A7%BB%E5%B7%A5%E5%85%B7/%E8%AE%AD%E6%8E%A8%E4%B8%80%E4%BD%93%E6%9D%83%E9%87%8D%E8%BD%AC%E6%8D%A2%E5%B7%A5%E5%85%B7%E4%BD%BF%E7%94%A8%E6%8C%87%E5%AF%BC) - 训推一体权重转换工具，支持在GPU和NPU上训练好的模型转成加速推理支持的格式。 - -### [精度工具](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools) - -1. [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe)。 - - MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 - -2. [api_accuracy_checker（Ascend模型精度预检工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/api_accuracy_checker) - - 2024.09.30下线 - - 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 + 训推一体权重转换工具，支持在 GPU 和 NPU 上训练好的模型转成加速推理支持的格式。 -3. [ptdbg_ascend（PyTorch精度工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend) +## [精度工具](./debug/accuracy_tools/) - 2024.09.30下线 +[MindStudio Probe（msprobe，MindStudio 精度调试工具）](./debug/accuracy_tools/msprobe)。 - 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位PyTorch训练场景下的精度问题。 +## [性能工具](./profiler) -### [性能工具](https://gitee.com/ascend/mstt/tree/master/profiler) +1. [compare_tools（性能比对工具）](./profiler/compare_tools) -1. [compare_tools（性能比对工具）](https://gitee.com/ascend/mstt/tree/master/profiler/compare_tools) + 提供 NPU 与 GPU 性能拆解功能以及算子、通信、内存性能的比对功能。 - 提供NPU与GPU性能拆解功能以及算子、通信、内存性能的比对功能。 +2. [cluster_analyse（集群分析工具）](./profiler/cluster_analyse) -2. [cluster_analyse（集群分析工具）](https://gitee.com/ascend/mstt/tree/master/profiler/cluster_analyse) + 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合 MindStudio Insight 的集群分析功能使用。 - 提供多机多卡的集群分析能力（基于通信域的通信分析和迭代耗时分析）, 当前需要配合MindStudio Insight的集群分析功能使用。 +3. [advisor](./profiler/advisor) -3. [advisor](https://gitee.com/ascend/mstt/blob/master/profiler/advisor) + 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyThon 场景性能数据进行分析，并输出性能调优建议。 - 将Ascend PyTorch Profiler或者msprof采集的PyThon场景性能数据进行分析，并输出性能调优建议。 +## [Tensorboard](./plugins/tensorboard-plugins/tb_plugin) +Tensorboard 支持 NPU 性能数据可视化插件 PyTorch Profiler TensorBoard NPU Plugin。 -### [Tensorboard](https://gitee.com/ascend/mstt/tree/master/plugins/tensorboard-plugins/tb_plugin) - -Tensorboard支持NPU性能数据可视化插件PyTorch Profiler TensorBoard NPU Plugin。 - -支持将Ascend平台采集、解析的Pytorch Profiling数据可视化呈现，也兼容GPU数据采集、解析可视化。 +支持将 Ascend 平台采集、解析的 PyTorch Profiling 数据可视化呈现，也兼容 GPU 数据采集、解析可视化。 ## 分支维护策略 -MindStudio Training Tools工具版本分支的维护阶段如下： - -| **状态** | **时间** | **说明** | -| ------------------- | -------- | ------------------------------------------------ | -| 计划 | 1~3 个月 | 计划特性 | -| 开发 | 3个月 | 开发特性 | -| 维护 | 6~12个月 | 合入所有已解决的问题并发布版本 | -| 无维护 | 0~3 个月 | 合入所有已解决的问题，无专职维护人员，无版本发布 | -| 生命周期终止（EOL） | N/A | 分支不再接受任何修改 | - -## 现有分支的维护状态 - -MindStudio Training Tools分支版本号命名规则如下： - -mstt仓每年发布4个版本，每个版本都将对应一个分支；以v6.0为例，其将对应v6.0.RC1、v6.0.RC2、v6.0.RC3以及v6.0.0四个版本，在仓库中将存在与之对应的分支。 - -| **分支** | **状态** | **发布日期** | **后续状态** | **EOL日期** | -| ------------- | -------- | ------------ | ------------------------ | ----------- | -| **v6.0.0** | 维护 | 2023/12/12 | 预计2024/12/12起无维护 | | +1. MindStudio Training Tools 工具版本分支的维护阶段如下： -## 参与贡献 + | **状态** | **时间** | **说明** | + | ------------------- | -------- | ------------------------------------------------ | + | 计划 | 1—3 个月 | 计划特性 | + | 开发 | 3个月 | 开发特性 | + | 维护 | 6—12个月 | 合入所有已解决的问题并发布版本 | + | 无维护 | 0—3 个月 | 合入所有已解决的问题，无专职维护人员，无版本发布 | + | 生命周期终止（EOL） | N/A | 分支不再接受任何修改 | -1. Fork 本仓库 -2. 新建 xxx 分支 -3. 提交代码 -4. 新建 Pull Request +2. MindStudio Training Tools 分支版本号命名规则如下： -## 版本过渡提示 + mstt 仓每年发布 4 个版本，每个版本都将对应一个分支；以 v6.0 为例，其将对应 v6.0.RC1、v6.0.RC2、v6.0.RC3 以及 v6.0.0 四个版本，在仓库中将存在与之对应的分支。 -当前版本预检和ptdbg维护到2024/09/30，准备于2024/09/30下线，相关目录mstt/debug/accuracy_tools/api_accuracy_checker和mstt/debug/accuracy_tools/ptdbg_ascend将于2024/09/30删除。新版本的预检和ptdbg已经合到mstt/debug/accuracy_tools/atat目录下。 + | **分支** | **状态** | **发布日期** | **后续状态** | **EOL日期** | + | ------------- | -------- | ------------ | ------------------------ | ----------- | + | **v6.0.0** | 维护 | 2023.12.12 | 预计 2024.12.12 起无维护 | | diff --git a/debug/accuracy_tools/README.md b/debug/accuracy_tools/README.md deleted file mode 100644 index e28c1d34a..000000000 --- a/debug/accuracy_tools/README.md +++ /dev/null @@ -1,21 +0,0 @@ -# 精度工具 - -MindStudio Training Tools工具针对模型训练精度问题设计推出了一系列精度工具，包括模型精度预检工具和PyTorch精度工具的精度比对功能、溢出检测功能、通信精度检测等功能。这些工具有各自侧重的场景，用于辅助用户定位模型精度问题。 - -### 子功能介绍 - -NPU上训练的网络存在精度问题，精度指标（loss或者具体的评价指标）与标杆相差较多。对于该场景的问题，可以使用**MindStudio Probe工具**的**精度预检工具**和**精度比对工具**进行定位。 - -| 工具名称 | 说明 | -| ------------------------------------------------------------ | ------------------------------------------------------------ | -| [MindStudio Probe（ MindStudio精度调试工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe) | MindStudio Training Tools工具链下精度调试部分的工具包，主要包括精度预检和精度比对等子工具。 | -| [api_accuracy_checker（Ascend模型精度预检工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/api_accuracy_checker) | 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 | -| [ptdbg_ascend（PyTorch精度工具）](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/ptdbg_ascend) | 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位PyTorch训练场景下的精度问题。 | - -### 场景介绍 - -**精度预检工具**会对全网每一个API根据其实际训练中的shape、dtype和数值范围生成随机的输入，对比它与标杆的输出差异，并指出输出差异过大不符合精度标准的API。该工具检查单API精度问题准确率超过80%，对比一般dump比对方法减少落盘数据量99%以上。 - -**精度比对功能**可以对NPU整网API数据进行与CPU或GPU标杆数据的精度比对，从而检测精度问题。具体来说，dump统计量、分段dump、模块化dump，通讯算子dump等功能可以用较轻的数据量实现不同侧重的精度比对，从而定位精度问题。 - -**溢出检测功能**是在判断训练网络可能存在溢出现象时，例如某个step的loss突然变成inf nan，或者混精场景下loss_scale不断减小，可以通过ptdbg_ascend的精度检测工具检测API的溢出情况。 \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index f5e457efa..0994974c8 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -1,238 +1,102 @@ -# MindStudio精度调试工具 +# 📖 msprobe 使用手册 -MindStudio精度调试工具（MindStudio Probe），简称msprobe，是MindStudio Training Tools工具链下精度调试部分的工具包。主要包括精度预检和精度比对等子工具，当前适配场景包括PyTorch和MindSpore。 +![version](https://img.shields.io/badge/version-1.0.3-blueviolet) +![python](https://img.shields.io/badge/python-3.8|3.9|3.10-blue) +![platform](https://img.shields.io/badge/platform-Linux-yellow) -## 工具安装 +[**msprobe**](./) 是 MindStudio Training Tools 工具链下精度调试部分的工具包。主要包括精度预检、溢出检测和精度比对等功能，目前适配 [PyTorch](https://pytorch.org/) 和 [MindSpore](https://www.mindspore.cn/) 框架。这些子工具侧重不同的训练场景，可以定位模型训练中的精度问题。 -精度工具合一软件包名称：`mindstudio_probe-{version}-py3-none-any.whl` +为方便使用，本工具提供了一个统一、简易的程序接口，**PrecisionDebugger**，通过以下示例模板和 **config.json** 可轻松使用各种功能。 -### pip安装 - ```shell - pip install mindstudio-probe - ``` -使用`pip install mindstudio-probe==版本号`可安装指定版本的包。 +```python +from msprobe.pytorch import PrecisionDebugger +# from msprobe.mindspore import PrecisionDebugger -pip命令会自动安装最新的包及其配套依赖。 - -提示如下信息则表示安装成功。 - -```bash -Successfully installed mindstudio_probe-{version} +debugger = PrecisionDebugger(config_path='./config.json') +... +debugger.start() # 一般在训练循环开头启动工具 +... # 循环体 +debugger.stop() # 一般在训练循环末尾结束工具 +debugger.step() # 在训练循环的最后需要重置工具，非循环场景不需要 ``` -### 下载whl包安装 -1. 使用pip命令安装依赖： - - 1. 根据实际环境安装torch或mindspore - - 2. 安装numpy、openpyxl、pandas、PyYAML、rich、tqdm、einops、matplotlib、pyOpenSSL、twisted - - - 若环境中已安装部分依赖，不需要重复安装。 - -2. whl包获取。 - - 请通过下表链接下载工具whl包。 - - | 版本 | 发布日期 | 支持PyTorch版本 | 支持MindSpore版本 | 下载链接 | 校验码 | - | ----- | ---------- | ---------------- | ----------------- | ------------------------------------------------------------ | ------------------------------------------------------------ | - | 1.0.3 | 2024-08-23 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.3-py3-none-any.whl) | 7060cc141a5b98ef770cd9220995d299393f32a61938261e632c7e8b5160bef2 | - | 1.0.2 | 2024-08-09 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.2-py3-none-any.whl) | e4a980e5d98c426ce5ce9842520d9bc031d3b3de621c74b3d59414cc6e238e0e | - | 1.0.1 | 2024-07-25 | 2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | - | 1.0 | 2024-07-09 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/1.0/ascend_training_accuracy_tools-1.0-py3-none-any.whl) | 5016dfe886c5d340ec6f60a959673355855f313c91f100680da814efb49f8e81 | - | 0.0.3 | 2024-06-11 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-0.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.3-py3-none-any.whl) | f46d9714704859e2d67861a65bbb3c76b0a250cf6e238b978b5b959ab1fe125a | - | 0.0.2 | 2024-05-23 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-0.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.2-py3-none-any.whl) | 2e35809bde559e9c4d2f16a02ccde779ed9e436bb65fded0b7ebaf6ac2c88d93 | - | 0.0.1 | 2024-03-15 | 2.0/2.1 | - | [ascend_training_accuracy_tools-0.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.1-py3-none-any.whl) | 5801510d4e827e4859bc9a5aca021e4d30c2ea42d60a4c8ad0c2baab1b7782c9 | - -3. whl包校验。 - - 1. 根据以上下载链接下载whl包到Linux安装环境。 +除了在训练脚本中调用接口函数，还可以通过命令行使用 **msprobe** 的其他功能，具体的使用规则和 **config.json** 的配置要求详见以下章节。 - 2. 进入whl包所在目录，执行如下命令。 +## ⚙️ [安装](./docs/01.installation.md) - ```bash - sha256sum {name}.whl - ``` +## 🛠️ config.json [介绍](./docs/02.config_introduction.md) 和 [示例](./docs/03.config_examples.md) - {name}为whl包名称。 +## 🧰 主要功能 - 若回显呈现对应版本whl包一致的**校验码**，则表示下载了正确的ptdbg_ascend精度工具whl安装包。示例如下： +### 1 数据采集 - ```bash - sha256sum ascend_training_accuracy_tools-0.0.1-py3-none-any.whl - 5801510d4e827e4859bc9a5aca021e4d30c2ea42d60a4c8ad0c2baab1b7782c9 *ascend_training_accuracy_tools-0.0.1-py3-none-any.whl - ``` +msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API 执行精度数据 dump 操作，对应 config.json 中的 task 为 statistics 或 tensor。 -4. 执行如下命令进行安装。 +[PyTorch 场景下的数据采集](./docs/05.data_dump_PyTorch.md) - ```bash - pip3 install ./mindstudio_probe-{version}-py3-none-any.whl - ``` - - 若为覆盖安装，请在命令行末尾增加“--force-reinstall”参数强制安装，例如： - - ```bash - pip3 install ./mindstudio_probe-{version}-py3-none-any.whl --force-reinstall - ``` - - 提示如下信息则表示安装成功。 - - ```bash - Successfully installed mindstudio_probe-{version} - ``` - -### 从源码安装 -1. 克隆或者下载项目源代码 - - ```shell - git clone https://gitee.com/ascend/mstt.git - cd debug/accuracy_tools - ``` - -2. 安装setuptools和wheel - - ```shell - pip install setuptools wheel - ``` - -3. 安装msprobe - - ```shell - python setup.py bdist_wheel - cd dist - pip install mindstudio_probe*.whl - ``` - 提示出现如下信息则表示源码安装成功。 - ```shell - Successfully installed ... mindstudio_probe-{version} ... - ``` - -### 查看msprobe工具信息 - -执行如下命令查看msprobe工具信息。 - -```bash -pip show mindstudio-probe -``` - -输出结果如下示例： - -```bash -Name: mindstudio-probe -Version: 1.0 -Summary: This is a pytorch precision comparison tools -Home-page: -Author: -Author-email: -License: -Location: /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages -Requires: numpy, openpyxl, pandas, pyyaml, rich, tqdm, wheel -Required-by: -``` +[MindSpore 场景下的数据采集](./docs/06.data_dump_MindSpore.md) -关键字段含义： +### 2 精度预检 -- Name：工具名称。 -- Version：工具版本号。 -- Summary：工具概述。 -- Location：工具安装路径。 -- Requires：工具依赖。 +精度预检旨在昇腾 NPU 上扫描训练模型中的所有 API 进行 API 复现，给出精度情况的诊断和分析。对应 config.json 中的 task 为 run_ut。 -## 工具使用 +PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)和[在线预检](./docs/08.accuracy_checker_online_PyTorch.md) -安装msprobe工具后，可以按照如下思路选择合适的子工具进行精度调试： +MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore.md) -1. 判断框架场景。 +### 3 精度比对 - 当前支持PyTorch和MindSpore场景。 +该功能进行 PyTorch 整网 API 粒度的数据 dump、精度比对，进而定位训练场景下的精度问题。 -2. 执行数据采集。 +[PyTorch 场景的精度比对](./docs/10.accuracy_compare_PyTorch.md) - 工具通过在训练脚本中添加PrecisionDebugger接口的方式对API执行精度数据dump操作。 +[MindSpore 场景的精度比对](./docs/11.accuracy_compare_MindSpore.md) - PyTorch场景：详见[PyTorch_精度数据采集](./pytorch/doc/dump.md)。 +### 4 溢出检测与解析 - MindSpore场景：详见[MindSpore_精度数据采集](./mindspore/doc/dump.md)。 +溢出检测与解析是在执行精度数据 dump 时，判断是否存在输入正常但输出存在溢出的 API，从而判断是否为正常溢出。对应 config.json 中的 overflow_check。 -3. 执行精度预检。 +[PyTorch 场景的溢出检测与解析](./docs/12.overflow_check_PyTorch.md)（更新中） - 在昇腾NPU上扫描用户训练模型中所有API，进行API复现，给出精度情况的诊断和分析。 +[MindSpore 场景的溢出检测与解析](./docs/13.overflow_check_MindSpore.md)（更新中） - PyTorch场景：详见[PyTorch_精度预检工具](./pytorch/doc/api_accuracy_checker.md)。 +### 5 数据解析 - MindSpore场景：暂不支持。 +该功能用于比对前后两次 NPU ACL 层级 dump 数据的一致性。 -4. 执行精度比对。 +[PyTorch 场景的数据解析](./docs/14.data_parse_PyTorch.md) - 进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位训练场景下的精度问题。 +### 6 无标杆比对 - PyTorch场景：详见[PyTorch_精度比对工具](./pytorch/doc/ptdbg_ascend_compare.md)。 +[PyTorch 场景的无标杆比对](./docs/15.free_benchmarking_PyTorch.md)（更新中） - MindSpore场景：详见[MindSpore_精度比对工具](./mindspore/doc/compare.md)。 +[MindSpore 场景的无标杆比对](./docs/16.overflow_check_MindSpore.md)（更新中） -5. 执行溢出解析。 +### 7 梯度状态监测 - 溢出解析是在执行精度数据dump时，配置了溢出检测dump，那么对于输入正常但输出存在溢出的API，可以判断是否为正常溢出。 +本功能用于采集梯度数据并进行梯度相似度比对，可以精准定位出现问题的 step。 - PyTorch场景：详见[PyTorch_溢出解析工具](./pytorch/doc/run_overflow_check.md)。 +[兼容 PyTorch 和 MindSpore 框架的梯度监测](./docs/17.grad_probe.md) - MindSpore场景：暂不支持。 +## 🌟 新版本特性 -6. 执行数据解析。 +【精度预检】 +- 落盘数据小。 +- 支持随机生成模式和真实数据模式。 +- 单 API 测试，排除整网中的累计误差问题。 - 用于比对前后两次NPU ACL层级dump数据的一致性。 +【梯度检测】 +- 使用便捷，无需在训练流程里插入代码。 +- 可以精准定位问题出现的 step。 - PyTorch场景：详见[PyTorch_数据解析工具](./pytorch/doc/parse_tool.md)。 +## 📑 补充材料 - MindSpore场景：暂不支持。 +[msprobe 精度工具标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md) -6. 执行梯度采集和比对。 - - 用于采集梯度数据并进行梯度相似度比对。可以精准定位问题出现的step。 - - 详见[梯度状态监测工具](./doc/grad_probe/grad_probe.md)。 - - - -上述流程中的工具均为msprobe工具的子工具，使用相同的命令行，格式如下： - -精度预检工具 - -```bash -msprobe -f run_ut [-h] -``` - -```bash -msprobe -f multi_run_ut [-h] -``` - -```bash -msprobe -f api_precision_compare [-h] -``` - -精度比对工具 - -```bash -msprobe -f compare [-h] -``` - -溢出解析工具 - -```bash -msprobe -f run_overflow_check [-h] -``` - -数据解析工具 - -```bash -msprobe -f parse [-h] -``` +[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md) -| 参数 | 说明 | -| ---- | ------------------------------------------------------------ | -| -f | 框架，请按所使用框架配置，当前支持配置为：pytorch、mindspore。 | -| -h | 帮助信息。 | +## ❓ FAQ -## 贡献 +[FAQ for PyTorch](./docs/FAQ_PyTorch.md) -push代码前，请务必保证已经完成了基础功能测试和网络测试。 +FAQ for MindSpore diff --git a/debug/accuracy_tools/msprobe/config/config.json b/debug/accuracy_tools/msprobe/config.json similarity index 100% rename from debug/accuracy_tools/msprobe/config/config.json rename to debug/accuracy_tools/msprobe/config.json diff --git a/debug/accuracy_tools/msprobe/config/README.md b/debug/accuracy_tools/msprobe/config/README.md deleted file mode 100644 index 7720ddbdf..000000000 --- a/debug/accuracy_tools/msprobe/config/README.md +++ /dev/null @@ -1,539 +0,0 @@ -# 配置文件说明 - -当前配置文件主要为PrecisionDebugger接口执行dump或无标杆比对操作时调用的配置，当PrecisionDebugger接口未指定该配置文件时，使用该文件的默认配置。配置文件详见[config.json](./config.json)。 - -当在环境上安装msprobe工具后，config.json文件位置可通过如下方式查找： - -查找msprobe工具安装路径。 - -``` -pip show mindstudio-probe -``` - -输出结果如下示例： - -``` -Name: mindstudio-probe -Version: 1.0 -Summary: This is a pytorch precision comparison tools -Home-page: -Author: -Author-email: -License: -Location: /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages -Requires: numpy, openpyxl, pandas, pyyaml, rich, tqdm, wheel -Required-by: -``` - -Location字段为msprobe工具的安装路径，那么config.json文件位置为/home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages/msprobe/config - -## 参数说明 - -### **通用配置参数** - -| 参数名 | 说明 | 是否必选 | -| ----------------- | ------------------------------------------------------------ | -------- | -| task | dump的任务类型，str类型。可取值：
"free_benchmark"（无标杆比对）。
"statistics"（仅dump张量的统计信息，默认值）。
"tensor"（dump张量的统计信息和完整张量数据，MindSpore静态图场景仅dump完整张量数据）。
"overflow_check"（溢出检测）。
"run_ut"（精度预检配置）。
配置示例："task": "tensor"。
根据task参数取值的不同，可以配置不同场景参数，详见：“**task配置为free_benchmark**”，“**task配置为statistics**”，“**task配置为tensor**”，“**task配置为overflow_check**”，“**task配置为run_ut**”。 | 否 | -| dump_path | 设置dump数据目录路径，str类型。配置示例："dump_path": "./dump_path"。MindSpore静态图场景仅支持绝对路径。 | 是 | -| rank | 指定对某张卡上的数据进行dump，list[int]类型，默认未配置（表示dump所有卡的数据），应配置为大于等于0的整数，且须配置实际可用的Rank ID。配置示例："rank": [1]。
对于PyTorch场景，Rank ID从0开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的Rank ID，则dump数据为空，比如当前环境Rank ID为0到7，实际训练运行0到3卡，此时若配置Rank ID为4或不存在的10等其他值，此时dump数据为空。
对于MindSpore场景，所有节点的Rank ID均从0开始计数，最大取值为每个节点可用卡总数-1，config.json配置一次rank参数对所有节点同时生效。 | 否 | -| step | 指定dump某个step的数据，list[int]类型。默认未配置，表示dump所有step数据。dump特定step时，须指定为训练脚本中存在的step。step为list格式，可配置逐个step，例如："step": [0,1,2]。 | 否 | -| level | dump级别，str类型，根据不同级别dump不同数据。可取值：
"L0"（dump module模块级精度数据，仅PyTorch场景支持，使用背景详见“**模块级精度数据dump说明**”）。
"L1"（dump API级精度数据，默认值，仅MindSpore动态图和PyTorch场景支持）。
"L2"（dump kernel级精度数据，仅MindSpore静态图和PyTorch场景支持，且PyTorch场景须配置acl_config参数）。
"mix"（dump module模块级和API级精度数据，即"L0"+"L1"，仅PyTorch场景支持）。
配置示例："level": "L1"。MindSpore动态图场景仅支持"L1"。 | 否 | -| acl_config | kernel dump的配置文件，str类型。level取"L2"时，该参数必选；level为其他值时，该参数不选。参数示例：acl_config='./acl_config.json'。acl_config.json配置文件详细介绍请参见“**acl_config.json配置文件说明**”。 | 否 | -| seed | 随机种子数，int类型，默认值为：1234，仅PyTorch场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见“**固定随机数范围**”。配置示例："seed": 1234。 | 否 | -| is_deterministic | 确定性计算模式，bool类型，仅PyTorch场景支持。可取值true（开启）或false（关闭），默认关闭。配置示例："is_deterministic": true。
即使在相同的硬件和输入下，API多次执行的结果也可能不同，开启确定性计算是为了保证在相同的硬件和输入下，API多次执行的结果相同。
确定性计算会导致API执行性能降低，建议在发现模型多次执行结果不同的情况下开启。
rnn类算子、ReduceSum、ReduceMean等算子可能与确定性计算存在冲突，若开启确定性计算后多次执行的结果不相同，则考虑存在这些算子。 | 否 | -| enable_dataloader | 自动控制开关，bool类型，仅PyTorch场景支持。可取值true（开启）或false（关闭），默认为false。配置为True后自动识别step参数指定的迭代，并在该迭代执行完成后退出训练，此时start、stop和step函数可不配置，开启该开关要求训练脚本是通过torch.utils.data.dataloader方式加载数据。仅支持PyTorch单卡训练使用，分布式训练场景下存在数据dump不全问题，**下个版本即将废弃该功能**。 | 否 | - -### task配置为free_benchmark - -仅PyTorch场景与MindSpore动态图场景支持，且"level"需为"L1"。 - -task配置为free_benchmark时，开启**无标杆比对**，在NPU环境下通过对当前模型API的输入添加扰动因子，二次执行，将得到的输出与未添加扰动因子前的输出进行比对，从而**得出该模型中可能因迁移等变化导致精度降低的API**。 - -无标杆比对优势在于省去了从CPU/GPU环境获取标杆数据的步骤，也省去了在NPU环境执行dump的操作，降低了精度比对的操作难度。 - -建议配置白名单（配置scope或list）控制少量API进行无标杆比对，一次对过多API执行无标杆比对可能导致显存溢出或性能膨胀。 - -| 参数名 | 说明 | 是否必选 | -| ------------ | ------------------------------------------------------------ | -------- | -| scope | 自定义检测API列表（仅PyTorch场景支持），list[str]类型，默认值为空列表，当list也为空列表时，表示检测所有API。需要在[]内配置具体API名（在工具dump结果中查看）。配置示例："scope": ["Torch.matmul.0", "Tensor.pow.4"]。与list参数不能同时配置。 | 否 | -| list | 自定义检测API类型或API名称，list[str]类型，默认值为空列表，表示检测所有API（PyTorch场景下还需scope也为空列表）。包含如下配置方法：
PyTorch场景下，指定某一类API，对某一类的API进行无标杆比对。配置示例："list": ["relu"]。
MindSpore场景下，指定API名称，对列表中的API进行检测。配置示例："list": ["mindspore.mint.div", "mindspore.ops.bmm", "mindspore.Tensor.\_\_add\_\_"]。
与scope参数不能同时配置。 | 否 | -| fuzz_device | 标杆设备，str类型。可取值：
"npu"：无标杆，通过添加扰动因子进行比对，默认值。
"cpu"：以CPU为标杆，pert_mode须配置为"to_cpu"（仅PyTorch场景支持）。
配置示例："fuzz_device": "npu"。 | 否 | -| pert_mode | 无标杆扰动因子，str类型。可取值：
"improve_precision"：对输入做升精度，默认值。
"add_noise"：对输入增加噪声。
"no_change"：不加扰动直接二次执行。
"bit_noise"：输入的末位比特翻转。
"change_value"：输入的张量首尾值调换（仅PyTorch场景支持）。
"to_cpu"：在CPU等价执行（仅PyTorch场景支持）。
配置示例："pert_mode": "improve_precision"。 | 否 | -| handler_type | 处理类型，可取值："check"（进行无标杆比对检查，默认值）、"fix"（将扰动后的API输出结果覆盖原始API输出结果，尝试将Loss曲线恢复正常，该模式下不支持预热if_preheat与反向过程）。配置示例："handler_type": "fix"。 | 否 | -| fuzz_level | 无标杆数据dump级别，即选择比对结果文件应输出的表头属性，当前仅支持取值为："L1"（默认值）。输出结果详见“**无标杆比对数据存盘格式**”。 | 否 | -| fuzz_stage | 前反向，选择对API前向或反向进行无标杆比对，可取值："forward"（前向，默认值）、"backward"（反向，仅PyTorch场景支持）。配置示例："fuzz_stage": "forward"。 | 否 | -| if_preheat | 预热功能（仅PyTorch场景支持），开启功能后工具可以根据每次迭代的输出调整精度算法的阈值，从而更准确找出存在精度问题的API，bool类型。可取值true（开启）或false（关闭），默认关闭。配置示例："if_preheat": "true"。"handler_type": "fix"不支持预热。 | 否 | -| preheat_step | 开启预热的迭代数量（仅PyTorch场景支持），int类型，默认值为15。须配置"if_preheat": "true"。 | 否 | -| max_sample | 每个算子预热的采样次数的最大阈值（仅PyTorch场景支持），int类型，默认值为20。须配置"if_preheat": "true"。 | 否 | - -#### 无标杆比对数据存盘格式 - -无标杆比对在dump_path目录下输出结果文件free_benchmark.csv，如下示例： - -![free_benchmark](./img/free_benchmark.png) - -| 字段 | 说明 | -| ------------ | ------------------------------------------------------------ | -| rank | Rank ID，int类型。 | -| pert_mode | 扰动因子的类型，string类型。 | -| stage | 前向或反向，string类型。 | -| step | 迭代数，int类型。 | -| api_name | API名称，string类型。 | -| max_rel | 输出对比最大相对误差，float类型。 | -| dtype | 输入的dtype，string类型。 | -| shape | 输入的shape，tuple类型。 | -| output_index | 如果输出为列表或元组，其中一个元素检测不一致，则会有该元素的index，否则为空，int类型。 | - -### task配置为statistics - -| 参数名 | 说明 | 是否必选 | -| ------------ | ------------------------------------------------------------ | -------- | -| scope | PyTorch和MindSpore动态图场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。MindSpore动态图场景当前仅支持配置为API名。 | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch和MindSpore动态图场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch和MindSpore动态图场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
MindSpore静态图场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | -| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore静态图场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | -| summary_mode | 控制dump文件输出的模式，str类型，仅PyTorch和MindSpore动态图场景支持，可取值md5（dump输出包含md5值以及API统计信息的dump.json文件，用于验证数据的完整性）、statistics（dump仅输出包含API统计信息的dump.json文件，默认值）。配置示例："summary_mode": "md5"。 | 否 | - -### task配置为tensor - -MindSpore静态图场景仅dump完整张量数据。 - -| 参数名 | 说明 | 是否必选 | -| -------------- | ------------------------------------------------------------ | -------- | -| scope | PyTorch和MindSpore动态图场景dump范围，list[str]类型，默认未配置（list也未配置时表示dump所有API的数据）。需要在[]内配置两个模块名或API名，用于锁定区间，dump该范围内的数据。配置示例："scope": ["MyModuleOP1", "MyModuleOP2"]。与level参数取值相关，level为L0和mix级别时，可配置模块名；level为L1级别时，可配置API名。 | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有API的数据）。包含如下配置方法：
PyTorch和MindSpore动态图场景配置具体的API全称，dump该API数据。配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch和MindSpore动态图场景指定某一类API，dump某一类的API级别输入输出数据。配置示例："list": ["relu"]。
PyTorch和MindSpore动态图场景配置kernel_api，dump前向和反向API的kernel_api级别数据，其中dump反向API时需要配置**backward_input**参数。前向API配置示例："list": ["Tensor.permute.1.forward"]；反API配置示例："list": ["Tensor.permute.1.forward"], "backward.input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
MindSpore静态图场景配置kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合”name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，”name-regex(Default/.+)”可匹配算子名称以”Default/”开头的所有算子）。 | 否 | -| backward_input | 该输入文件为首次运行训练dump得到反向API输入的dump文件，str类型，仅PyTorch场景支持，默认未配置。例如若需要dump Functional.conv2d.1 API的反向过程的输入输出，则需要在dump目录下查找命名包含Functional.conv2d.1、backward和input字段的dump文件。配置示例："backward_input": "./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"] | 否 | -| data_mode | dump数据过滤，str类型。可取值"all"、"forward"、"backward"、"input"和"output"，表示仅保存dump的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的dump文件。配置示例"data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有dump的数据。除了all参数只能单独配置外，其他参数可以自由组合。
MindSpore静态图场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。 | 否 | -| file_format | MindSpore静态图场景真实tensor数据的保存格式，str类型，可取值"bin"（dump的tensor文件为二进制格式，"level": "L1"时不支持）、"npy"（dump的tensor文件后缀为.npy，默认值）。 | 否 | -| online_run_ut | 在线预检模式开关，bool类型，可取值true（开启）、false（关闭），默认未配置，表示关闭。配置为true表示开启在线预检。 | 否 | -| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。仅在online_run_ut字段配置为true时生效，未配置该参数后host和port不生效。 | 否 | -| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的局域网IP地址。仅在online_run_ut字段配置为true时生效，局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | -| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的端口号。仅在online_run_ut字段配置为true时生效，局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | - -说明： online_run_ut、nfs_path、host、port参数仅在线预检场景NPU环境生效，详细说明请参见[《在线精度预检》](../pytorch/doc/api_accuracy_checker_online.md)。 - -### task配置为overflow_check - -MindSpore静态图场景的jit_level为O0/O1时，不支持该功能，须配置jit_level为O2。请参见[mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig)配置jit_config。 - -| 参数名 | 说明 | 是否必选 | -| ------------- | ------------------------------------------------------------ | -------- | -| overflow_nums | 控制溢出次数，int类型，仅MindSpore动态图和PyTorch场景支持，表示第N次溢出时，停止训练，过程中检测到溢出API对应kernel数据均dump。配置示例："overflow_nums": 3。默认为1，即检测到1次溢出，训练停止，配置为-1时，表示持续检测溢出直到训练结束。 | 否 | -| check_mode | MindSpore静态图场景kernel级别的溢出检测，str类型，可取值"aicore"（开启AI Core的溢出检测）、"atomic"（开启Atomic的溢出检测）、"all"（开启AI Core和Atomic的溢出检测，默认值）。配置示例"check_mode": "aicore"。 | 否 | - -### task配置为run_ut - -仅PyTorch场景支持。 - -| 参数名称 | 说明 | 是否必选 | -|-----------------|-----------------------------------------------------------------------------------------------------------------------------------------------|------| -| white_list | API dump白名单，仅对指定的API进行dump。配置示例："white_list": ["conv1d", "conv2d"]。默认未配置白名单，即dump全量API数据。 | 否 | -| black_list | API dump黑名单，被指定的API不进行dump。配置示例："black_list": ["conv1d", "conv2d"]。默认未配置黑名单，即dump全量API数据。 | 否 | -| error_data_path | 配置保存精度未达标的API输入输出数据路径，默认为当前路径。配置示例"error_data_path": "./"。 | 否 | -| is_online | 在线预检模式开关，bool类型，可取值true（开启）、false（关闭），默认关闭。 | 否 | -| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效，仅在is_online字段配置为true时生效。 | 否 | -| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机地址127.0.0.1或本机局域网IP。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。仅在is_online字段配置为true时生效。 | 否 | -| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机可用端口。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。仅在is_online字段配置为true时生效。 | 否 | -| rank_list | 指定在线预检的Rank ID，默认值为[0]，list[int]类型，应配置为大于等于0的整数，且须根据实际卡的Rank ID配置，若所配置的值大于实际训练所运行的卡的Rank ID，则在线预检输出数据为空。GPU和NPU须配置一致。仅在is_online字段配置为true时生效。 | 否 | - -说明： -
（1）white_list和black_list同时配置时，二者配置的API名单若无交集，则白名单生效，若API名单存在交集，则白名单排除的部分以及交集的API不进行dump。 -
（2）is_online、nfs_path、host、port、rank_list等字段仅在线预检场景GPU机器生效，详细说明见[《在线精度预检》](../pytorch/doc/api_accuracy_checker_online.md) - -## 配置示例 - -以下示例包含当前支持的所有场景可配置的完整参数。 - -### PyTorch场景task配置为free_benchmark - -```json -{ - "task": "free_benchmark", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "free_benchmark": { - "scope": [], - "list": ["conv2d"], - "fuzz_device": "npu", - "pert_mode": "improve_precision", - "handler_type": "check", - "fuzz_level": "L1", - "fuzz_stage": "forward", - "if_preheat": false, - "preheat_step": 15, - "max_sample": 20 - } -} -``` - -### PyTorch场景task配置为statistics - -```json -{ - "task": "statistics", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "statistics": { - "scope": [], - "list": [], - "data_mode": ["all"], - "summary_mode": "statistics" - } -} -``` - -### PyTorch场景task配置为tensor - -```json -{ - "task": "tensor", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "tensor": { - "scope": [], - "list":[], - "data_mode": ["all"], - "backward_input": "" - } -} -``` - -### PyTorch场景task配置为overflow_check - -```json -{ - "task": "overflow_check", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "overflow_check": { - "overflow_nums": 1 - } -} -``` - -### PyTorch场景task配置为run_ut - -```json -{ - "task": "run_ut", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - "enable_dataloader": false, - - "run_ut": { - "white_list": [], - "black_list": [], - "error_data_path": "./" - } -} -``` - -### MindSpore静态图场景task配置为statistics - -```json -{ - "task": "statistics", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L2", - - "statistics": { - "list": [], - "data_mode": ["all"], - "summary_mode": "statistics" - } -} -``` - -### MindSpore静态图场景task配置为tensor - -```json -{ - "task": "tensor", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L2", - - "tensor": { - "list":[], - "data_mode": ["all"], - "backward_input": "" - } -} -``` - -### MindSpore静态图场景task配置为overflow_check - -```json -{ - "task": "overflow_check", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L2", - - "overflow_check": { - "check_mode": "all" - } -} -``` - -### MindSpore动态图场景task配置为free_benchmark - -```json -{ - "task": "free_benchmark", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - - "free_benchmark": { - "list": ["mindspore.ops.add"], - "fuzz_device": "npu", - "pert_mode": "improve_precision", - "handler_type": "check", - "fuzz_level": "L1", - "fuzz_stage": "forward" - } -} -``` - -### MindSpore动态图场景task配置为statistics - -```json -{ - "task": "statistics", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - - "statistics": { - "scope": [], - "list": [], - "data_mode": ["all"], - "summary_mode": "statistics" - } -} -``` - -### MindSpore动态图场景task配置为tensor - -```json -{ - "task": "tensor", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - - "tensor": { - "scope": [], - "list":[], - "data_mode": ["all"], - } -} -``` - -### MindSpore动态图场景task配置为overflow_check - -```json -{ - "task": "overflow_check", - "dump_path": "/home/data_dump", - "rank": [], - "step": [], - "level": "L1", - - "overflow_check": { - "overflow_nums": 1 - } -} -``` - -## 附录 - -### 模块级精度数据dump说明 - -仅PyTorch场景支持。 - -大模型场景下，通常不是简单的利用自动迁移能力实现GPU到NPU的训练脚本迁移，而是会对NPU网络进行一系列针对性的适配，因此，常常会造成迁移后的NPU模型存在部分子结构不能与GPU原始模型完全对应。模型结构不一致导致API调用类型及数量不一致，若直接按照API粒度进行精度数据dump和比对，则无法完全比对所有的API。 - -本节介绍的功能是对模型中的大粒度模块进行数据dump，使其比对时，对于无法以API粒度比对的模块可以直接以模块粒度进行比对。 - -模块指的是继承自nn.Module类模块，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump数据时以模块为粒度进行dump。 - -### acl_config.json配置文件说明 - -#### [config.json](./config.json)配置示例 - -当PyTorch场景level取"L2"时，须配置acl_config参数，并指定acl_config.json文件（用于指定L2 kernel级dump的配置），此时config.json文件配置示例如下： - -- 前向kernel dump配置示例： - - "scope"配置为前向API名称，仅支持配置一个API。 - - ```json - { - "task": "tensor", - "dump_path": "/home/data_dump", - "level": "L2", - "rank": [0], - "step": [0], - "is_deterministic": false, - "tensor": { - "scope": ["Tensor.__mul__.10.forward"], - "list":[], - "data_mode": ["all"], - "backward_input": [""], - "file_format": "npy" - }, - "acl_config": "acl_config.json" - } - ``` - -- 反向kernel dump配置示例： - - 执行反向kernel dump前需要先使用工具dump该API的反向输入，保存pt文件，在"backward_input"参数中传入该pt文件路径。 - - "scope"配置为反向API名称，仅支持配置一个API。 - - ```json - { - "task": "tensor", - "dump_path": "/home/data_dump", - "level": "L2", - "rank": [0], - "step": [0], - "is_deterministic": false, - "tensor": { - "scope": ["Tensor.__mul__.10.backward"], - "list":[], - "data_mode": ["all"], - "backward_input": ["Tensor.__mul__.10.backward.input.0.pt"], - "file_format": "npy" - }, - "acl_config": "acl_config.json" - } - ``` - -#### acl_config.json配置示例 - -acl_config.json文件须自行创建，配置示例如下： - -``` -{ - "dump": - { - "dump_list":[], - "dump_path":"./dump/output", - "dump_mode":"all", - "dump_op_switch":"on" - } -} -``` - -**acl_config.json参数说明** - -| 字段名 | 说明 | -| -------------- | ------------------------------------------------------------ | -| dump_list | 待dump数据的API模型。为空，无需配置。 | -| dump_path | dump数据文件存储到运行环境的目录，主要配置的是kernel级数据的存放路径。支持配置绝对路径或相对路径。dump_path须为已存在目录。 | -| dump_mode | dump数据模式，配置如下： output：dump API的输出数据。默认值。 input：dump API的输入数据。 all：dump API的输入、输出数据。 | -| dump_op_switch | 单API模型dump数据开关，配置如下：
off：关闭单API模型dump，默认值。
on：开启单API模型dump。 | - -**dump目录说明** - -配置acl_config.json后，采集的kernel级数据会在{dump_path}/{time}/{deviceid}/{model_id}目录下生成，例如“/home/HwHiAiUser/output/20200808163566/0/0” - -``` -├── 20230131172437 -│ └── 1 -│ ├── 0 -│ │ ├── Add.Add.45.0.1675157077183551 -│ │ ├── Cast.trans_Cast_0.31.0.1675157077159449 -│ │ ├── Cast.trans_Cast_5.43.0.1675157077180129 -│ │ ├── MatMul.MatMul.39.0.1675157077172961 -│ │ ├── Mul.Mul.29.0.1675157077155731 -│ │ ├── NPUAllocFloatStatus.NPUAllocFloatStatus.24.0.1675157077145262 -│ │ ├── TransData.trans_TransData_1.33.0.1675157077162791 -│ │ └── TransData.trans_TransData_4.41.0.1675157077176648 -│ ├── 1701737061 -│ │ └── Cast.trans_Cast_2.35.0.1675157077166214 -│ ├── 25 -│ │ └── NPUClearFloatStatus.NPUClearFloatStatus.26.0.1675157077150342 -│ └── 68 -│ └── TransData.trans_TransData_3.37.0.1675157077169473 -``` - -### 固定随机数范围 - -仅PyTorch场景支持。 - -seed_all函数可固定随机数的范围如下表。 - -| API | 固定随机数 | -| ---------------------------------------- | --------------------------- | -| os.environ['PYTHONHASHSEED'] = str(seed) | 禁止Python中的hash随机化 | -| random.seed(seed) | 设置random随机生成器的种子 | -| np.random.seed(seed) | 设置numpy中随机生成器的种子 | -| torch.manual_seed(seed) | 设置当前CPU的随机种子 | -| torch.cuda.manual_seed(seed) | 设置当前GPU的随机种子 | -| torch.cuda.manual_seed_all(seed) | 设置所有GPU的随机种子 | -| torch_npu.npu.manual_seed(seed) | 设置当前NPU的随机种子 | -| torch_npu.npu.manual_seed_all(seed) | 设置所有NPU的随机种子 | -| torch.backends.cudnn.enable=False | 关闭cuDNN | -| torch.backends.cudnn.benchmark=False | cuDNN确定性地选择算法 | -| torch.backends.cudnn.deterministic=True | cuDNN仅使用确定性的卷积算法 | - -需要保证CPU或GPU以及NPU的模型输入完全一致，dump数据的比对才有意义，seed_all并不能保证模型输入完全一致，如下表所示场景需要保证输入的一致性。 - -| 场景 | 固定方法 | -| --------------- | ------------- | -| 数据集的shuffle | 关闭shuffle。 | -| dropout | 关闭dropout。 | - -关闭shuffle示例： - -```Python -train_loader = torch.utils.data.DataLoader( - train_dataset, - batch_size = batch_size, - shuffle = False, - num_workers = num_workers -) -``` - -关闭dropout： - -在使用from msprobe.pytorch import PrecisionDebugger后，工具会自动将torch.nn.functional.dropout、torch.nn.functional.dropout2d、torch.nn.functional.dropout3d、torch.nn.Dropout、torch.nn.Dropout2d、torch.nn.Dropout3d的接口参数p置为0。 diff --git a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md new file mode 100644 index 000000000..cad25d984 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md @@ -0,0 +1,157 @@ +# MindSpore 场景下的精度数据采集 + +msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 + +本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml 文件内手动修改，如下示例： + +```yaml +ops: # ops为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API + - adaptive_avg_pool1d + - adaptive_avg_pool2d + - adaptive_avg_pool3d +``` + +## 1 接口介绍 + +### 1.1 PrecisionDebugger + +**功能说明**：通过加载 dump 配置文件的方式来确定 dump 操作的详细配置。 + +**原型**： + +```Python +PrecisionDebugger(config_path=None) +``` + +**\***： +1. config_path：指定 dump 配置文件路径，string 类型。参数示例："./config.json"。未配置该路径时，默认使用 [config.json](../config.json) 文件的默认配置。 + +### 1.2 start + +**功能说明**：启动精度数据采集。在模型初始化之后的位置添加。需要与 stop 函数一起添加在 for 循环内。 + +**原型**： + +```Python +debugger.start(model=None) +``` +**\***： +1. model：指具体的 mindspore.nn.Cell，默认未配置，L1 级别下传入 model 可以采集 primitive op。 + +### 1.3 stop + +**功能说明**：停止数据采集。在 **start** 函数之后的任意位置添加。需要与 start 函数一起添加在 for 循环内。若需要 dump 反向数据，则需要添加在反向计算代码之后。**仅 MindSpore 动态图场景支持。** + +**原型**： + +```Python +debugger.stop() +``` + +### 1.4 step + +**功能说明**：在最后一个 **stop** 函数后或一个 step 结束的位置添加。**仅 MindSpore 动态图场景支持。** + +**原型**： + +```Python +debugger.step() +``` + +## 2 示例代码 + +### 2.1 MindSpore 静态图场景 + +```Python +from msprobe.mindspore import PrecisionDebugger +debugger = PrecisionDebugger(config_path="./config.json") +# 请勿将以上初始化流程插入到循环代码中 +# 下面代码也可以用PrecisionDebugger.start() +debugger.start() +... +``` + +### 2.2 MindSpore 动态图场景 + +```Python +import mindspore as ms +from msprobe.mindspore import PrecisionDebugger + +# 请勿将PrecisionDebugger的初始化插入到循环代码中 +debugger = PrecisionDebugger(config_path="./config.json") + +# 模型、损失函数的定义以及初始化等操作 +# ... + +# 数据集迭代的地方往往是模型开始训练的地方 +for data, label in data_loader: + debugger.start() # 开启数据dump + net = Model() + # 如下是模型每个step执行的逻辑 + grad_net = ms.grad(net)(data) + # ... + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` + +## 3 dump 结果文件介绍 + +### 3.1 MindSpore 静态图场景 + +训练结束后，工具将 dump 的数据保存在 dump_path 参数指定的目录下。 + +- jit_level 为O0/O1时： + + dump 结果目录请参见 MindSpore 官网中的[同步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%90%8C%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)。 + +- jit_level 为O2时： + + dump 结果目录请参见 MindSpore 官网中的[异步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%BC%82%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)。 + +jit_level 请参见 [mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig) 配置 jit_config。 + +### 3.2 MindSpore 动态图场景 + +训练结束后，工具将 dump 的数据保存在dump_path参数指定的目录下。 + +dump结果目录结构示例如下： + +```bash +├── dump_path +│ ├── step0 +│ | ├── rank0 +│ | │ ├── dump_tensor_data +| | | | ├── MintFunctional.relu.0.backward.input.0.npy +| | | | ├── Mint.abs.0.forward.input.0.npy +| | | | ├── Functional.split.0.forward.input.0.npy +| | | | ... +| | | | └── Tensor.__add__.0.forward.output.0.npy +│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） +│ | | ├── stack.json # 算子调用栈信息 +│ | | └── construct.json # 分层分级结构，level为L1时，construct.json内容为空 +│ | ├── rank1 +| | | ├── dump_tensor_data +| | | | └── ... +│ | | ├── dump.json +│ | | ├── stack.json +| | | └── construct.json +│ | ├── ... +│ | | +| | └── rank7 +│ ├── step1 +│ | ├── ... +│ ├── step2 +``` + +dump 过程中，npy 文件在对应算子或者模块被执行后就会落盘，而 json 文件则需要在正常执行 PrecisionDebugger.stop() 后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关 npy 文件，可能会导致 json 文件中数据丢失。 + +其中 rank 为设备上各卡的 ID，每张卡上 dump 的数据会生成对应 dump 目录。非分布式场景下没有 rank ID，目录名称为 rank。 + +npy 文件保存的前缀和 MindSpore 对应关系如下： + +| 前缀 | MindSpore 模块 | +| -------------- | ---------------------------- | +| Tensor | mindspore.Tensor | +| Functional | mindspore.ops | +| Mint | mindspore.mint | +| MintFunctional | mindspore.mint.nn.functional | diff --git a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md new file mode 100644 index 000000000..72fb01e2e --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md @@ -0,0 +1,278 @@ +# PyTorch 场景的离线精度预检 + +## 1 简介 + +**PyTorch 离线精度预检**\*1通过扫描昇腾 NPU 上用户训练模型中的所有 API，输出模型精度的诊断和分析结果。具体而言，该工具通过采集模型中所有 API 的前反向信息，构造相应的单元测试，将 NPU 输出与标杆（CPU 高精度）比对，从而计算对应的精度指标，该过程已封装为 run_ut.py 函数；将 NPU 环境下采集的预检数据拷贝至 GPU 环境，同样执行 run_ut.py；最后通过**新精度标准比对法**\*2将 NPU 和 GPU 的预检结果进行比对，从而找出 NPU 中存在精度问题的 API。同时，本工具支持随机生成模式和真实数据模式\*3。 + +**\***： + +1. 支持 PyTorch 版本：1.11/2.0/2.1/2.2； +2. 依据新精度标准，对不同的API采取不同的比对算法（包括绝对阈值法，标杆比对法、二进制一致法、ULP误差比对法和双千指标法），最终给定预检判定结果； +3. 在预检 dump 时可以选择由工具构造随机数获得 dump 数据或选择真实输入的数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + +## 2 离线预检流程 + +1. 在 NPU 和 GPU 环境下分别安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 +2. 在 NPU 训练脚本内添加 msprobe 工具 dump 接口 PrecisionDebugger，采集待预检数据。注意需要配置 level="L1"。 +3. 将 NPU 环境下 dump 的预检数据拷贝至 GPU 环境。 +4. 在 NPU 和 GPU 环境下分别执行 run_ut.py，生成的结果最终用于 api_precision_compare.py 函数的输入。详见 [3 离线预检操作指导](#3-离线预检操作指导)。 +5. 将 NPU 和 GPU 执行 run_ut.py 生成的 `accuracy_checking_details_{timestamp}.csv` 结果文件拷贝至同一环境下。 +6. 运行 api_precision_compare.py，输出结果为预检操作的最终结果。详见 [5 预检结果比对](#5-预检结果比对)章节。 + +## 3 离线预检操作指导 + +完成预检数据采集后，仅仅获取了 API 的输入数据，为了得到 NPU vs. CPU 高精度（标杆）的预检比对结果和 GPU vs. CPU 高精度（标杆）的预检比对结果，还需要进行 run_ut 操作。 + +run_ut 预检操作包括以下两种方式： + +- 使用 run_ut.py 执行预检：适用于数据量较小的单卡场景。 +- 使用 multi_run_ut.py 执行多线程预检：适用于数据量较大的大模型场景。 + +### 3.1 使用 run_ut.py 执行预检 + +将 API 信息输入到 run_ut 模块进行精度检测并比对，运行如下命令： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json +``` + + | 参数名称 | 解释 | 是否必选 | + | ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | + | -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | + | -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | + | -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | + | | | | + | -j 或 --jit_compile | 开启 jit 编译。 | 否 | + | -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0。 | 否 | + | -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | + | -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | + | -config 或 --config_path | 指定离线预检操作过程中额外配置（包括黑名单、白名单等）的 [config.json](../config.json) 文件，默认未配置。config.json 文件的配置可参考[配置文件介绍](./02.config_introduction.md)。 | 否 | + +run_ut 执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见[ 4 预检结果](#4-预检结果)。 + +如果需要保存比对不达标的输入和输出数据，可以在 run_ut 执行命令结尾添加 `-save_error_data`，例如： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data +``` + +数据默认会存盘到 './ut_error_data{timestamp}' 路径下（相对于启动 run_ut 的路径），如有需要，用户可以通过 error_data_path 参数来配置保存路径，error_data_path 参数在 [config.json](../config.json) 文件或 [config.yaml](../pytorch/api_accuracy_checker/config.yaml) 文件配置，config.json 文件需要在 run_ut 操作时通过 -config 参数指定，config.yaml 文件详见\*1。 + +**\***： + +1. config.yaml 文件说明 + + config.yaml 文件可以通过配置参数来控制 dump 和 run_ut 操作的白名单、黑名单等功能。操作步骤如下： + + - config.yaml 文件通常位于类似 /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml 的路径中。 + + - 进入 config.yaml 文件 + + ```bash + vim /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml + ``` + + - 修改 config.yaml 文件参数。 + + ```yaml + white_list: [] + black_list: [] + error_data_path: './' + precision: 14 + ``` + + | 参数名称 | 解释 | 是否必选 | + | ----------- | -----------| -------- | + | white_list | API dump 白名单，仅对指定的 API 进行 dump。
**参数示例**：white_list=["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | + | black_list | API dump 黑名单，被指定的 API 不进行 dump。
**参数示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | + | error_data_path | 配置保存精度未达标的 API 输入输出数据路径。
**参数示例**："error_data_path": "./"。默认为当前路径。 | 否 | + | precision | 浮点数表示位数，默认取小数点后14位。 | 否 | + + 说明：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 + +2. API 预检黑名单和白名单 + + run_ut 过程支持 API 预检黑名单和白名单，通过如下文件配置 black_list（黑名单）或white_list（白名单）参数来指定不需要或需要预检的API名称： + + - 配置 [config.json](../config.json) 文件，该文件需要在 run_ut 操作时通过 -config 参数指定。 + + - config.json 文件的优先级高于 config.yaml 文件，即执行 config.json 文件时，config.yaml 文件的配置不生效。 + +### 3.2 使用 multi_run_ut.py 执行多线程预检 + +multi_run_ut.py 脚本，可以并行执行多个 run_ut 操作，从而减少预检耗时。示例如下： + +```bash +msprobe -f pytorch multi_run_ut -api_info ./dump.json -n 32 -d 0 1 2 3 +``` + +| 参数名称 | 解释 | 是否必选 | +| ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | +| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | 是 | +| -save_error_data | 保存精度未达标的 API 输入输出数据。 | 否 | +| -o 或 --out_path | 指定 run_ut 执行结果存盘路径，默认“./”（相对于 run_ut 的路径）。 | 否 | +| -j 或 --jit_compile | 开启 jit 编译。 | 否 | +| -n | 同时执行 run_ut 线程的数量，默认为 8，最大支持 64，但每个 Device 最大支持 8 个线程。当指定多个线程和多个 Device 时，线程数在每张卡上均分。 | 否 | +| -d 或 --device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为 0，支持同时指定 0~7，共 8 个 Device。 | 否 | +| -csv_path 或 --result_csv_path | 指定本次运行中断时生成的 `accuracy_checking_result_{timestamp}.csv` 文件路径，执行 run_ut 中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的 `accuracy_checking_result_{timestamp}.csv` 文件。详见 [3.3 断点续检](#33-断点续检)。 | run_ut 操作中断后继续执行场景下必须配置 | +| -f 或 --filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的 API。适用于模型较大且重复 API 较多的场景。 | 否 | + +### 3.3 断点续检 + +断点续检操作通过如下命令执行： + +```bash +msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_checking_result_{timestamp}.csv +``` + +精度预检 run_ut 过程中，若因环境、数据量过大等原因导致预检进程中断，那么当用户解决这些问题后，重新执行 run_ut 操作，可以通过断点续检操作继续前面未完成的预检，会在 -csv_path 指定的 `accuracy_checking_result_{timestamp}.csv` 文件以及对应的 `accuracy_checking_details_{timestamp}.csv` 文件中继续写入后续的结果，不会重新创建结果文件。 + +须指定为上次预检中断的 `accuracy_checking_result_{timestamp}.csv` 文件。请勿修改 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件名，包括时间戳，否则断点续检会因无法识别到文件名而失败。 + +## 4 预检结果 + +精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件示例如下： + +可以通过先查看 `accuracy_checking_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `accuracy_checking_details_{timestamp}.csv` 文件的 API 详细达标情况，API 达标情况介绍请参见\*1。 + +`accuracy_checking_result_{timestamp}.csv` + +![accuracy_checking_result](img/accuracy_checking_result.png) + +| 字段 | 含义 | +| --------------------- | ------------------------- | +| API name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出。 | +| Message | 提示信息。 | + +该结果为中间结果，仅作为参考，建议完成 [5 预检结果比对](#5-预检结果比对)后查看比对结果。该结果后续将会删除。 + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差、双百双千双万指标判定结果决定的。 + +需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 waring 和 pass 且不存在 error 则标记 waring。 + +`accuracy_checking_details_{timestamp}.csv` + +![accuracy_checking_details](img/accuracy_checking_details.png) + +| 字段 | 含义 | +| ------------------- | ------------------------------------------------------------ | +| API name | NPU 或 GPU下的 API 名称。 | +| Bench Dtype | 标杆数据的 API 数据类型。 | +| DEVICE Dtype | NPU 或 GPU 数据的 API 数据类型。 | +| Shape | API 的 Shape信息。 | +| 余弦相似度 | NPU 或 GPU 数据与标杆数据的余弦相似度。 | +| 最大绝对误差 | NPU 或 GPU 数据与标杆数据的最大绝对误差。 | +| 双百指标 | 双百精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于百分之一的个数占总元素个数的比例。测试通过标准为相对误差大于百分之一的个数占总元素个数的比例小于百分之一。 | +| 双千指标 | 双千精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。 | +| 双万指标 | 双万精度指标。是指 NPU 或 GPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于万分之一的个数占总元素个数的比例。测试通过标准为相对误差大于万分之一的个数占总元素个数的比例小于万分之一。 | +| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。 | +| 误差均衡性 | NPU 或 GPU 数据与标杆数据精度差的上下浮动情况。 | +| 均方根误差 | NPU 或 GPU 数据与标杆数据的均方根误差。 | +| 小值域错误占比 | NPU 或 GPU Tensor 中与标杆的绝对误差大于错误阈值的小值在小值域（小值的总数量）中的占比。判断为小值以及绝对误差的错误阈值参见\*2。 | +| 相对误差最大值 | NPU 或 GPU 数据与标杆数据相对误差的最大值。 | +| 相对误差平均值 | NPU 或 GPU 数据与标杆数据相对误差的平均值。 | +| inf/nan 错误率 | NPU 与标杆 inf/nan 计算不一致的元素个数占总元素的个数比例。 | +| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。 | +| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。 | +| ULP 误差最大值 | NPU 或 GPU 数据与标杆数据 ULP 误差的最大值（取绝对值后）。 | +| ULP 误差平均值 | NPU 或 GPU 数据与标杆数据ULP误差的平均值（取绝对值后）。 | +| ULP 误差大于阈值占比 | NPU 或 GPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 或 GPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 或 GPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。 | +| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过，warning 表示测试未通过双千或双万精度指标，SKIP 表示该 API 的某个参数的反向不要计算梯度，所以没有任何计算过程，其他信息均为空。 | +| message | 提示信息。 | + +**\***： + +1. API 预检指标： + + API 预检指标是通过对 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差双百、双千、双万精度指标的数值进行判断，得出该 API 是否符合精度标准的参考指标。 + + API预检通过测试，则在`accuracy_checking_details_{timestamp}.csv`文件中的 Status 列标记 pass，否则标记 error 或 warning，详细规则如下： + + - 余弦相似度 > 0.99：≤ 0.99 为不达标，标记 error，> 0.99 达标，进行下一步； + - 最大绝对误差 ＜ 0.001：＜ 0.001 达标，标记 pass，≥ 0.001 为不达标，进行下一步； + - 双百、双千、双万精度指标： + + 对于 float16 和 bfloat16 数据：双百指标不通过，标记 error；双百指标通过，双千指标不通过，标记 warning；双百、双千指标均通过，标记 pass。 + + 对于 float32 和 float64 数据：双千指标不通过，标记 error；双千指标通过，双万指标不通过，标记 warning；双千、双万指标均通过，标记 pass。 + - 在 `accuracy_checking_result_{timestamp}.csv` 中以 Forward Test Success 和 Backward Test Success 字段统计该算子前向反向输出的测试结果，对于标记 pass 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE 表示测试通过，对于标记 error 或 warning 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 FALSE 表示测试不通过。由于一个算子可能有多个前向或反向的输入或输出，那么该类算子的输入或输出中必须全为 pass，才能在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE，只要有一个输入或输出标记 error 或 warning，那么在`accuracy_checking_result_{timestamp}.csv`中标记 FALSE。 + +2. 小值域阈值： + + 判定为小值的阈值： + + - torch.float32：e-6 + - torch.float16：e-3 + - torch.bfloat16：e-3 + + 小值域的绝对误差阈值： + + - torch.float32：e-9 + - torch.float16：e-5 + - torch.bfloat16：e-5 + +## 5 预检结果比对 + +需要同时获取 NPU 和 GPU 环境下 run_ut 操作的预检结果 `accuracy_checking_details_{timestamp}.csv` 文件。执行如下命令进行 NPU 和 GPU 预检结果的比对： + +```bash +msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_details_{timestamp}.csv -gpu /home/xxx/gpu/accuracy_checking_details_{timestamp}.csv -o /home/xxx/ +``` + +| 参数名称 | 说明 | 是否必选 | +| -------------------- | ------------- | -------- | +| -npu 或 --npu_csv_path | NPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | +| -gpu 或 --gpu_csv_path | GPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | +| -o 或 --out_path | 指定 api_precision_compare.py 执行结果存盘路径，默认为当前目录。 | 否 | + +执行完成后输出 `api_precision_compare_result_{timestamp}.csv` 和 `api_precision_compare_details_{timestamp}.csv` 文件。文件示例如下： + +可以通过先查看 `api_precision_compare_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `api_precision_compare_details_{timestamp}.csv` 文件的 API 详细达标情况。 + +`api_precision_compare_result_{timestamp}.csv` + +![api_precision_compare_result](img/api_precision_compare_result.png) + +| 字段 | 含义 | +| --------------------- | ------------------------------------------------------------ | +| API name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | +| Message | 提示信息。 | + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `api_precision_compare_details_{timestamp}.csv` 中的各个指标判定结果决定的。需要注意的是 `api_precision_compare_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `api_precision_compare_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 warning 和 pass 且不存在 error 标记 warning。 + +`api_precision_compare_details_{timestamp}.csv` + +![api_precision_compare_details](img/api_precision_compare_details.png) + +| 字段 | 含义 | +| ------------------------ | ------------------------------------------------------------ | +| API name | NPU 或 GPU 下的 API 名称。 | +| 小值域错误比值 | NPU 与 CPU 的小值域的错误比率 / GPU 与 CPU 的小值域的错误比率。标杆比对法指标。 | +| 小值域错误判定结果 | 小值域错误比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | +| 均方根误差比值 | NPU 与 CPU 的均方根误差 / GPU 与 CPU 的均方根误差。标杆比对法指标。 | +| 均方根误差判定结果 | 均方根误差比值小于等于 1 标记为 pass，1~2 之间标记为 waring，大于 2 标记为 error。 | +| 相对误差最大值比值 | NPU 与 CPU 的相对误差最大值 / GPU 与 CPU 的相对误差最大值。标杆比对法指标。 | +| 相对误差最大值判定结果 | 相对误差最大值比值小于等于 1 标记为 pass，1 ~ 10 之间标记为 waring，大于 10 标记为 error。 | +| 相对误差平均值比值 | NPU 与 CPU 的相对误差的平均值 / GPU 与 CPU 的相对误差的平均值。标杆比对法指标。 | +| 相对误差平均值判定结果 | 相对误差平均值比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。 | +| 误差均衡性比值 | NPU 与 CPU 的误差均衡性 / GPU 与 CPU 的误差均衡性。标杆比对法指标。 | +| 误差均衡性判定结果 | 误差均衡性比值小于等于 1 标记为 pass，1 ~ 2 之间标记为 waring，大于 2 标记为 error。该字段暂不参与 api_precision_compare_result 的结果判定。 | +| inf / nan 错误率 | NPU 与标杆 inf / nan 计算不一致的元素个数占总元素的个数比例。绝对阈值法指标。 | +| inf / nan 判定结果 | inf / nan 错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 相对误差错误率 | NPU 与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。绝对阈值法指标。 | +| 相对误差判定结果 | 相对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 绝对误差错误率 | NPU 与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。绝对阈值法指标。 | +| 绝对误差判定结果 | 绝对误差错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。二进制一致法指标。 | +| 二进制一致错误率判定结果 | 二进制一致错误率判定结果，等于 0 标记为 pass，其余情况标记为 error。 | +| ULP 误差平均值 | NPU 数据与标杆数据 ULP 误差的平均值（取绝对值后）。ULP 误差比对法指标。 | +| ULP 误差大于阈值占比 | NPU 数据与标杆数据的 ULP 误差（取绝对值后）大于阈值（当 NPU 数据类型为 float16 或 bfloat16 时，阈值为 1；当 NPU 数据类型为 float32 时，阈值为 32）的元素个数占总元素的个数比例。ULP 误差比对法指标。 | +| ULP 误差大于阈值占比比值 | NPU 与 CPU 的 ULP 误差大于阈值占比 / GPU 与 CPU 的 ULP 误差大于阈值占比。ULP 误差比对法指标。 | +| ULP 误差判定结果 | ULP 误差判定结果。
当 NPU 或 GPU 数据类型是 float16 或 bfloat16 时，以下两条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差大于阈值占比小于 0.001；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。
当 NPU 或 GPU 数据类型是 float32 时，以下三条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差平均值小于 64；
NPU ULP 误差大于阈值占比小于 0.05；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。 | +| 双千指标 | 双千精度指标。是指 NPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅 conv1d 和 conv2d 使用该指标。双千指标法指标。 | +| 双千指标判定结果 | 双千指标判定结果。双千指标大于 0.999 标记为 pass，否则标记为 error。 | +| 比对结果 | 综合所有指标的最终结果。如果比对指标中有 error，则标记为 error；有 warning，则标记为 warning；否则标记为 pass。 | +| 比对算法 | API 使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和 ULP 误差比对法中的一种。 | +| Message | 提示信息。当前提示该 API 比对结果为 error 或 warning 时对应不符合标准的指标。 | diff --git a/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md b/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md new file mode 100644 index 000000000..b643dc69c --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md @@ -0,0 +1,182 @@ +# PyTorch 场景的在线精度预检 + +## 1 简介 + +为了应对大模型场景下，通过离线预检方式 dump API 输入输出数据导致的存储资源紧张问题，提供在线精度预检功能。本功能实现在执行 NPU 训练操作的过程中，通过 TCP/IP 协议在 NPU +Host 与 GPU Host 设备间建立连接，将 NPU 上对应 API 的输入数据在 GPU 设备上运行，将两份输出数据进行比对，得到预检比对结果，从而减少数据 dump 的步骤，降低存储资源的占用。 + +## 2 在线精度预检流程 + +在线精度预检当前支持**局域网场景**和**共享存储场景**，请根据不同的场景选择对应的配置。 + +在线精度预检操作流程如下： + +1. 准备 GPU 和 NPU 可正常运行的训练环境，PyTorch 版本大于等于2.0，并保证两台 Host 在同一局域网内可正常通信或能通过共享存储进行通信。 +2. GPU 和 NPU Host 设备上同时安装msprobe工具，详见[ msprobe 安装](./01.installation.md)章节，其中在线预检要安装 twisted、pyOpenSSL，这些包为 Python 模块。 +3. 分别配置 GPU 侧、NPU 侧的 config.json 文件。 +4. 在 GPU 侧运行 `msprobe -f pytorch run_ut -config ./config.json`。 +5. 在 NPU 侧配置训练脚本。 +6. 在 NPU 侧执行训练。 + +## 3 在线精度预检操作指导 + +### 3.1 配置 config.json 文件 + +预检工具安装完成后，需要在 GPU 和 NPU 环境下分别配置 config.json。其中需要重点关注文件中的 is_online、is_benchmark_device、host 和 port 参数的配置，保障在线预检时 GPU 和 NPU 两台设备间的通信正常。 + +#### 3.1.1 GPU 侧在线预检配置说明 + +| 参数名称 | 说明 | 是否必选 | +|-----------------|--------------|------| +| task | 任务名称，str 类型，配置为 run_ut 表示预检任务。通过其他字段 is_online 判断离线预检、在线预检任务。 | 是 | +| white_list | 预检的 API 白名单，list[str] 类型。
**配置示例**：white_list=["conv1d", "conv2d"]。默认未配置白名单，即预检全量 API 数据。 | 否 | +| black_list | 预检的 API 黑名单，list[str] 类型。
**配置示例**：white_list=["conv1d", "conv2d"]。默认未配置黑名单，即预检全量 API 数据。 | 否 | +| error_data_path | 配置保存精度未达标的 API 输入输出数据路径，str 类型。在线预检模式下该参数不生效。 | 否 | +| is_online | 在线预检模式开关，bool 类型，可取值 True（开启）、False（关闭），默认关闭。 | 是 | +| nfs_path | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。配置该参数后 host 和 port 不生效。 | 否 | +| host | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机地址 127.0.0.1 或本机局域网 IP。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | +| port | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机可用端口。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | +| rank_list | 指定在线预检的 Rank ID，默认值为 [0]，list[int] 类型，应配置为大于等于 0 的整数，且须根据实际卡的 Rank ID 配置，若所配置的值大于实际训练所运行的卡的 Rank ID，则在线预检输出数据为空。GPU 和 NPU 须配置一致。 | 是 | + +#### 3.1.2 NPU 侧在线预检配置说明 + +| 参数名称 | 说明 | 是否必选 | +|------------------|-------------|------| +| task | 任务名称，str 类型，配置为 tensor 表示 dump API 统计信息和完全复刻整网的 API 运行情况的真实数据。通过字段 online_run_ut 判断是否使用在线预检功能。 | 是 | +| dump_path | dump 路径，str 类型，配置为合法路径即可，兼容 tensor 任务静态检查。 | 是 | +| level | dump 级别，str 类型，在线预检时配置为 L1，表示 dump API 级精度数据。在线预检可不配置，默认取值 L1。 | 是 | +| rank | 指定对某张卡上的数据进行 dump，list[int] 类型，默认未配置(表示 dump所有卡的数据)，需要与 GPU 侧配置项 rank_list 保持一致。 | 否 | +| step | 指定 dump 某个 step 的数据，list[int] 类型，默认未配置，表示 dump 所有 step 的数据。dump 特定 step 时，须指定为训练脚本中存在的 step。 | 否 | +| seed | 随机种子数，int 类型，默认值为 1234。通过固定随机数保证模型的输入或输出一致。 | 否 | +| is_deterministic | 确定性计算模式，bool 类型，可取值 true（开启）或 false（关闭），默认关闭。 | 否 | +| scope | dump 范围，list[str] 类型，默认未配置（list 也未配置时表示 dump 所有 api 的数据），配置方式参考 [config.json 配置介绍](./02.config_introduction.md)。 | 否 | +| list | dump 范围，list[str] 类型，默认未配置（scope 也未配置时表示 dump 所有 api 的数据），配置方式参考 [config.json 配置介绍](./02.config_introduction.md)。 | 否 | +| online_run_ut | 在线预检模式开关，bool 类型，可取值 True（开启）、False（关闭），默认关闭。 | 是 | +| nfs_path | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。配置该参数后 host 和 port 不生效。 | 否 | +| host | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的局域网 IP 地址。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | +| port | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的端口号。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | + +#### 3.1.3 局域网场景配置示例 + +GPU 侧： + +```json +{ + "task": "run_ut", + "run_ut": { + "white_list": [], + "black_list": [], + "error_data_path": "./", + "is_online": true, + "nfs_path": "", + "host": "127.0.0.1", + "port": 59208, + "rank_list": [0] + } +} +``` + +NPU 侧： + +```json +{ + "task": "tensor", + "dump_path": "./dump_path", + "rank": [0], + "step": [0], + "level": "L1", + "seed": 1234, + "is_deterministic": true, + "tensor": { + "scope": [], + "list": [], + "online_run_ut": true, + "nfs_path": "", + "host": "xx.xx.xx.x", + "port": 59208 + } +} +``` + +#### 3.1.4 共享存储场景配置示例 + +GPU 侧： + +```json +{ + "task": "run_ut", + "run_ut": { + "white_list": [], + "black_list": [], + "error_data_path": "./", + "is_online": true, + "nfs_path": "/nfs/xxx/data", + "host": "", + "port": -1, + "rank_list": [0] + } +} +``` + +NPU 侧： + +```json +{ + "task": "tensor", + "dump_path": "./dump_path", + "rank": [0], + "step": [0], + "level": "L1", + "seed": 1234, + "is_deterministic": true, + "tensor": { + "scope": [], + "list": [], + "online_run_ut": true, + "nfs_path": "/nfs/xxx/data", + "host": "", + "port": -1 + } +} +``` + +### 3.2 在 GPU 侧运行 run_ut + +由于 GPU 侧为通信接收端，需先于 NPU 侧执行 run_ut 操作，命令如下： + +```bash +msprobe -f pytorch run_ut -config ./config.json +``` + +GPU 侧配置好 config.json 文件后执行 run_ut 命令，此时 GPU 处于预检等待状态： + +- 局域网场景：当 NPU 侧启动训练后将预检的 API 输入和输出数据发送到 GPU 侧时，GPU 启动预检操作。 +- 共享存储场景：当 NPU 侧启动训练后将预检的 API 输入和输出数据发送到共享存储时，GPU 启动预检操作。 + +### 3.2 在 NPU 侧配置训练脚本 + +在 NPU 训练脚本中添加如下代码以获取 run_ut 操作的预检 API 输入和输出数据： + +```python +from msprobe.pytorch import PrecisionDebugger + +debugger = PrecisionDebugger("config.json") +... + +debugger.start() + +... + +debugger.stop() +debugger.step() +``` + +### 3.3 在 NPU 侧执行训练脚本 + +配置完 NPU 侧训练脚本后即可执行训练脚本，命令示例如下： + +```bash +bash train.sh +``` + +训练脚本执行完毕后，在 GPU 侧 dump_path 目录下生成比对结果文件，详细介绍请参见[离线精度预检中的 **4 预检结果**](./07.accuracy_checker_Pytorch.md#4-预检结果)。 diff --git a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md new file mode 100644 index 000000000..e7257c1b4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md @@ -0,0 +1,70 @@ +# MindSpore 动态图场景的离线精度预检 + +## 1 简介 + +**MindSpore 动态图精度预检**\*1通过扫描昇腾 NPU 上用户训练 MindSpore 模型中的所有 Mint API，输出精度情况的诊断和分析。工具以模型中所有 Mint API 前反向的 dump 结果为输入，构造相应的 API 单元测试，将 NPU 输出与标杆（CPU 高精度）比对，计算对应的精度指标，从而找出 NPU 中存在精度问题的 Mint API。本工具支持随机生成模式和真实数据模式\*2。 + +**\***： + +1. 支持 Mindspore 版本：2.4； +2. 在预检 dump 时可以选择由工具构造随机数进行输入获得 dump 数据或选择获取真实输入数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + +## 2 离线预检流程 + +操作流程如下： + +1. 在 NPU 和 GPU 环境下分别安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 +2. 在 NPU 训练脚本内添加 msprobe 工具 dump 接口 PrecisionDebugger，采集待预检数据。详见 [MindSpore 场景下的数据采集](./06.data_dump_MindSpore.md)章节，注意需要配置 level="L1"。 +3. 执行预检操作，查看预检结果文件，分析预检不达标的API。 + +## 3 离线预检操作指导 + +命令如下： +```bash +msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result +``` + +| 参数名称 | 说明 |参数类型 | 是否必选 | +| ---------------------------- | --------------------------------------|---------------------- | ---------------------------------- | +| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | str | 是 | +| -o或--out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | + +预检执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API Name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见 [4 预检结果](#4-预检结果)。 + +## 4 预检结果 + +精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件内容详情如下： + +`accuracy_checking_details_{timestamp}.csv` + +| 字段 | 含义 | +| ------------------- | ------------------------------------------------------------ | +| API Name | API 名称。 | +| Bench Dtype | 标杆数据的 API 数据类型。 | +| Tested Dtype | 被检验数据的 API 数据类型。 | +| Shape | API 的 Shape 信息。 | +| Cosine | 被检验数据与标杆数据的余弦相似度。 | +| MaxAbsErr | 被检验数据与标杆数据的最大绝对误差。 | +| MaxRelativeErr | 被检验数据与标杆数据的最大相对误差。 | +| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过。 | +| message | 提示信息。 | + +`accuracy_checking_result_{timestamp}.csv` + +| 字段 | 含义 | +| --------------------- | ----------------- | +| API Name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，error 为错误。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，error 为错误，如果是空白的话代表该 API 没有反向输出。 | +| Message | 提示信息。 | + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差判定结果决定的。具体规则详见\*1。 +需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error。 + +**\***： + +1. API预检指标： + + - API 预检指标是通过对`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差的数值进行判断，得出该API是否符合精度标准的参考指标。详细规则如下： + + - 余弦相似度大于0.99，并且最大绝对误差小于0.0001，标记“pass”，否则标记为“error”。 diff --git a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md new file mode 100644 index 000000000..85004c7e5 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md @@ -0,0 +1,247 @@ +# PyTorch 场景的精度比对 + +## 1 简介 + +- 本节主要介绍通过命令行和比对函数的方式进行 CPU 或 GPU 与 NPU 的精度数据比对，执行精度比对操作前需要先完成 CPU 或 GPU 与 NPU 的精度数据 dump，参见 [PyTorch 场景下的数据采集](./05.data_dump_Pytorch.md)章节。 + +- msprobe 使用子命令 compare 进行比对，可支持单卡和多卡场景的精度数据比对。 + +- 比对函数均通过单独创建精度比对脚本执行，可支持单卡和多卡场景的精度数据比对。 + +- 工具性能：比对数据量较小时（单份文件小于 10 GB），比对速度 0.1 GB/s；比对数据量较大时，比对速度 0.3 GB/s。 推荐环境配置：独占环境，CPU 核心数 192，固态硬盘（IO 速度参考：固态硬盘 > 500 MB/s，机械硬盘 60 ~ 170 MB/s）。用户环境性能弱于标准约束或非独占使用的比对速度酌情向下浮动。比对速度的计算方式：两份比对文件大小/比对耗时。 + +**使用场景**： + +- 同一模型，从 CPU 或 GPU 移植到 NPU 中存在精度下降问题，对比 NPU 芯片中的 API 计算数值与 CPU 或 GPU 芯片中的 API 计算数值，进行问题定位。 +- 同一模型，进行迭代（模型、框架版本升级或设备硬件升级）时存在的精度下降问题，对比相同模型在迭代前后版本的 API 计算数值，进行问题定位。 + +**注意事项**： + +- NPU 自研 API，在 CPU 或 GPU 侧若没有对应的 API，该 API 的 dump 数据不比对。 +- NPU 与 CPU 或 GPU 的计算结果误差可能会随着模型的执行不断累积，最终会出现同一个 API 因为输入的数据差异较大而无法比对的情况。 +- CPU 或 GPU 与 NPU 中两个相同的 API 会因为调用次数不同导致无法比对或比对到错误的 API，不影响整体运行，该 API 忽略。 + +**API匹配条件**： + +进行精度比对时，需要判断CPU或GPU的API与NPU的API是否相同可比对，须满足以下匹配条件： + +- 两个API的名称相同，API命名规则：`{api_type}.{api_name}.{api调用次数}.{正反向}.{输入输出}.index`，如：Functional.conv2d.1.backward.input.0。 +- 两个API的输入输出Tensor数量和各个Tensor的Shape相同。 + +通常满足以上两个条件，工具就认为是同一个API，成功进行API的匹配，后续进行相应的计算精度比对。 + +## 2 精度比对操作指导 + +### 2.1 命令行方式 + +将 CPU 或 GPU 与 NPU 的 dump 文件进行比对，支持单卡和多卡，可同时比对多卡的 dump 数据。多机场景需要每个设备单独执行比对操作。 + +请先参见 [PyTorch 场景下的数据采集](./05.data_dump_Pytorch.md)章节完成 CPU 或 GPU 与 NPU 的精度数据 dump。 + +**操作步骤**： + +1. 创建比对文件，文件内容及示例请参见\*1。 + +2. 运行命令： + + ```shell + msprobe -f pytorch compare -i ./compare.json -o ./output -s + ``` + + **参数说明**： + + | 参数名 | 说明 | 是否必选 | + |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | + | -i 或 --input_path | 指定比对文件路径。比对文件内容及示例请参见\*1。 | 是 | + | -o 或 --output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s 或 --stack_mode | 配置 stack_mode 的开关。仅当比对文件\*1配置 stack_path 需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -c 或 --compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | + | -f 或 --fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + +3. 查看比对结果，请参见 [3 精度比对结果分析](#3-精度比对结果分析)。 + +**\***： + +1. 比对文件： + + 以在当前目录创建 ./compare.json 为例。 + + - 单卡场景示例： + + ```json + { + "npu_path": "./npu_dump/dump.json", + "bench_path": "./bench_dump/dump.json", + "stack_path": "./npu_dump/stack.json", + "is_print_compare_log": true + } + ``` + + - 多卡场景示例： + + ```json + { + "npu_path": "./npu_dump/step0", + "bench_path": "./bench_dump/step0", + "is_print_compare_log": true + } + ``` + + **参数说明**： + + | 参数名 | 说明 | 是否必选 | + | -------------------- | ------------------------------------------------------------ | ------------------ | + | npu_path | 配置 NPU 环境下的 dump.json 文件（单卡场景）或真实数据目录（多卡场景）。数据类型：str。 | 是 | + | bench_path | 配置 CPU、GPU 或 NPU 环境下的 dump.json 文件（单卡场景）或真实数据目录（多卡场景）。数据类型：str。 | 是 | + | stack_path | 配置 NPU dump 目录下的 stack.json 文件。数据类型：str。 | 单卡必选，多卡不选 | + | is_print_compare_log | 配置是否开启日志打屏。可取值 True 或 False。数据类型：bool | 否 | + +### 2.2 比对函数方式 + +#### 2.2.1 compare_distributed 函数 + +此函数将 CPU 或 GPU 与 NPU 的 dump 文件进行比对，支持单卡和多卡，可同时比对多卡的 dump 数据。多机场景需要每个设备单独执行比对操作。可自动检索和匹配对应卡和进程所 dump 的数据文件，再调用 compare 进行比对。单机单卡时与 compare 函数二选一。 + +**函数原型**： + +```Python +compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs) +``` + +**参数说明**： + +| 参数名 | 说明 | 是否必选 | +| -------------- | ------------------------------------------------------------ | -------- | +| npu_dump_dir | 配置 NPU 环境下的 dump 目录。str 类型。dump 数据目录须指定到 step 级。
**配置示例**：'./npu_dump/step0'。 | 是 | +| bench_dump_dir | 配置 CPU、GPU 或 NPU 环境下的 dump 目录。str 类型。
**配置示例**：'./gpu_dump/step0'。 | 是 | +| output_path | 配置比对结果文件存盘目录。需要预先创建 output_path 目录。str 类型。
**配置示例**：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_rank{npu_ID}-rank{cpu/gpu/npu_ID}_{timestamp}.xlsx`。 | 是 | +| **kwargs | 支持 compare 的所有可选参数。 | 否 | + +**函数示例**： + +```Python +from msprobe.pytorch import * +compare_distributed('./npu_dump/step0', './gpu_dump/step0', './output') +``` + +#### 2.2.2 compare 函数 + +此函数将 CPU 或 GPU 与 NPU 的 dump 文件进行比对，仅支持单机单卡。 + +**函数原型**： + +```Python +compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False) +``` + +**参数说明**： + +| 参数名 | 说明 | 是否必选 | +| ------------ | ------------------------------------------------------------ | -------- | +| input_param | 配置 dump 数据文件及目录，dict 类型。配置参数包括：
"npu_json_path"：指定 NPU dump 目录下的 dump.json 文件。
**配置示例**："npu_json_path": "./npu_dump/dump.json"。
"bench_json_path"：指定 CPU、GPU 或 NPU dump 目录下的 dump.json 文件。
**配置示例**："bench_json_path": "./bench_dump/dump.json"。
"stack_json_path"：指定 NPU dump 目录下的 stack.json 文件。
**配置示例**："stack_json_path": "./npu_dump/stack.json"。
"is_print_compare_log"：配置是否开启日志打屏。
**配置示例**：True 或 False。 | 是 | +| output_path | 配置比对结果文件存盘目录，str 类型。
**配置示例**：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | +| stack_mode | 配置 stack_mode 的开关，bool 类型。仅当配置 stack_json_path 时需要开启。
**配置示例**：stack_mode=True，默认为 False。 | 否 | +| auto_analyze | 自动精度分析，bool 类型。开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 advisor_{timestamp}.txt 文件）。
**配置示例**：auto_analyze=False，默认为 True。 | 否 | +| fuzzy_match | 模糊匹配，bool 类型。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。
**配置示例**：fuzzy_match=True，默认为 False。 | 否 | + +**函数示例**： + +单机单卡场景下创建比对脚本，例如 compare.py，拷贝如下代码，具体参数请根据实际环境修改。 + +```Python +from msprobe.pytorch import compare +input_param={ +"npu_json_path": "./npu_dump/dump.json", +"bench_json_path": "./bench_dump/dump.json", +"stack_json_path": "./npu_dump/stack.json", +"is_print_compare_log": True +} +compare(input_param, output_path="./output", stack_mode=True) +``` + +#### 2.2.3 统计量比对 + +在使用 **compare** 或 **compare_distributed** 函数创建的比对脚本且在 [config.json](../config.json) 文件中配置 `"task": "statistics"` 时，可以进行统计量比对，此时比对 dump.json 文件中的统计信息，开启后的比对结果生成 Max diff、Min diff、Mean diff 和 L2norm diff，表示 NPU dump 数据中 API 的输入或输出与标杆数据输入或输出的最大值、最小值、平均值以及 L2 范数的差。可以通过该值判断 API 是否存在精度问题：当某个 API 的输入和输出的 Max diff、Min diff、Mean diff 和 L2norm diff 均为 0 或无限趋于 0，那么可以判断该 API 无精度问题，反之则可能存在精度问题。 + +**比对脚本示例**： + +以 compare.py 为例： + +```Python +from msprobe.pytorch import compare +input_param={ +"npu_json_path": "./npu_dump/dump.json", +"bench_json_path": "./bench_dump/dump.json", +"stack_json_path": "./npu_dump/stack.json", +"is_print_compare_log": True +} +compare(input_param, output_path="./output", stack_mode=True) +``` + +**统计量比对结果说明**： + +数据量比对同样生成 `compare_result_{timestamp}.xlsx` 和 `advisor_{timestamp}.txt` 文件。其中 `advisor_{timestamp}.txt` 主要对 `compare_result_{timestamp}.xlsx` 中可能存在精度问题（Result 为 Waring）的 API 提出定位建议；`compare_result_{timestamp}.xlsx` 主要有如下两种情况： + +- "summary_mode": "statistics" 时比对 dump.json 文件： + + ![compare_result_pkl](img/compare_result_pkl.png) + + 上图是对 dump.json 文件中 NPU 及标杆 API 的统计信息进行比对，判断可能存在精度问题的 API，文件中记录 NPU 及标杆 API 的基本信息和统计信息，其中需要关注 Result 列，包含结果：Waring（NPU 与标杆统计信息的比对中存在相对误差大于 0.5，则需要重点检查该 API）；为空（相对误差小于等于 0.5，可以不需要重点关注，但不代表不存在精度问题）；Nan（表示统计信息数据没有匹配上）。 + +- "summary_mode": "md5" 时比对 dump.json 文件： + + ![compare_result_pkl_md5.png](img/compare_result_pkl_md5.png.png) + + 上图是对 dump.json 文件中 NPU 及标杆 API 的 MD5 信息进行比对，判断 API 数据的完整性，文件中记录 NPU 及标杆 API 的基本信息和 MD5 信息，其中需要关注 Result 列，包含结果：Pass（表示 NPU 与标杆的 MD5 值一致，即 API 数据完整）；Different（表示 NPU 与标杆的 MD5 值不一致，即 API 数据不完全一致，可以通过 NPU_Stack_Info 列 API 调用栈查询该 API 的详细信息）；Nan（表示 MD5 信息数据没有匹配上）。 + +## 3 精度比对结果分析 + +PyTorch 精度比对是以 CPU 或 GPU 的计算结果为标杆，通过计算精度评价指标判断 API 在运行时是否存在精度问题。 + +- `advisor_{timestamp}.txt` 文件中给出了可能存在精度问题的 API 的专家建议，可直接打开查看。 + +- `compare_result_{timestamp}.xlsx` 文件列出了所有执行精度比对的 API 详细信息和比对结果，示例如下： + + ![compare_result](https://gitee.com/cai-weiwei1989/att_ptdbg/raw/master/debug/accuracy_tools/ptdbg_ascend/doc/img/compare_result.png) + + 可以从该结果文件中进行[判断计算精度达标情况](#31-判断计算精度达标情况)、[计算精度评价指标分析](#32-计算精度评价指标分析)以及[异常信息识别](#33-异常信息识别)等分析动作。 + +### 3.1 判断计算精度达标情况 + +精度比对结果 `compare_result_{timestamp}.xlsx` 文件中只需要通过 Accuracy Reached or Not 来判断计算精度是否达标，判断标准如下： + +1. Cosine < 0.99 且 MaxAbsError > 0.001 时，精度不达标，标记为 No。 +2. Cosine < 0.9，精度不达标，标记为 No。 +3. MaxAbsError > 1，精度不达标，标记为 No。 +4. 其余情况下记为精度达标，标记为 Yes。 + +### 3.2 计算精度评价指标分析 + +1. Cosine：通过计算两个向量的余弦值来判断其相似度，数值越接近于 1 说明计算出的两个张量越相似，实际可接受阈值为大于 0.99。在计算中可能会存在 nan，主要由于可能会出现其中一个向量为 0。 + +2. MaxAbsErr：当最大绝对误差越接近 0 表示其计算的误差越小，实际可接受阈值为小于 0.001。 + +3. MaxRelativeErr：当最大相对误差越接近 0 表示其计算的误差越小。 + + 当 dump 数据中存在 0 或 Nan 时，比对结果中最大相对误差则出现 inf 或 Nan 的情况，属于正常现象。 + +4. One Thousandth Err Ratio（双千分之一）、Five Thousandths Err Ratio（双千分之五）精度指标：是指 NPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差大于千分之一、千分之五的比例占总元素个数的比例小于千分之一、千分之五。该数据仅作为精度下降趋势的参考，并不参与计算精度是否通过的判定。 + +### 3.3 异常信息识别 + +精度比对结果 `compare_result_{timestamp}.xlsx` 文件中对于存在异常信息的API会进行高亮处理： + +- 红色可能出现的情况有： + - NPU max 或 NPU min 信息中存在 nan/inf； + - Max diff存在大于1e+10的值； + - 统计数据中output的Max diff除以max(0.01, Bench max) > 0.5； + - 真实数据中One Thousandth Err Ratio的input > 0.9同时output < 0.6。 +- 黄色可能出现的情况有： + - Max diff的input与output都大于1，同时output比input大一个数量级以上； + - 统计数据Max diff除以max(0.01, Bench max)的output > 0.1同时input < 0.01； + - 真实数据One Thousandth Err Ratio的input - output > 0.1； + - 真实数据Cosine的input - output > 0.1。 + +Shape为 [ ] 时，统计量列说明： + +当 NPU Tensor Shape 列为 [ ] 时，表示标量或 0 维张量，统计量列（NPU max、NPU min、NPU mean、NPU l2norm）展示相同的唯一元素。Bench 同理。 diff --git a/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md b/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md new file mode 100644 index 000000000..7b5f7e730 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md @@ -0,0 +1,62 @@ +# MindSpore 场景的精度比对 + +## 1 简介 + +msprobe 精度比对工具主要通过对同一个模型，在两个不同的 MindSpore 环境下，输入相同的训练数据，在分别得到 dump 数据后，对这两个 dump 数据进行全量自动对比，从而快速定位不同版本之间的精度问题。 + +## 2 精度比对操作指导 + +### 2.1 命令行方式 + +该工具目前使用方式为命令行形式，对 MindSpore 的 dump 数据仅支持单卡。 + +请先参见 [MindSpore 场景下的数据采集](./06.data_dump_MindSpore.md)章节，完成不同环境下 MindSpore 精度数据的采集。 + +**操作步骤**： + +1. 使用 MindSpore 进行 dump，得到不同框架版本的 dump 数据。 + +2. 创建比对文件，文件内容及示例请参见\*1。 + +3. 运行命令： + + ```shell + msprobe -f mindspore compare -i ./compare.json -o ./output -s + ``` + + **参数说明**： + + | 参数名 | 说明 | 是否必选 | + |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | + | -i 或 --input_path | 指定比对文件路径。比对文件内容及示例请参见\*1。 | 是 | + | -o 或 --output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | + | -s 或 --stack_mode | 配置 stack_mode 的开关。仅当比对文件\*1配置 stack_path 需要开启。默认未配置，表示关闭。 | 否 | + | -c 或 --compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | + | -f 或 --fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。默认未配置，表示关闭。 | 否 | + +4. 查看比对结果，请参见 [PyTorch 场景的精度比对——精度比对结果分析](./11.accuracy_compare_PyTorch.md#3-精度比对结果分析)章节。 + +**\***： + +1. 比对文件 + + 以在当前目录创建 ./compare.json 为例，单卡场景示例如下： + + ```json + { + "npu_path": "./npu_dump/dump.json", + "bench_path": "./bench_dump/dump.json", + "stack_path": "./npu_dump/stack.json", + "is_print_compare_log": true + } + ``` + + **参数说明**： + + | 参数名 | 说明 | 是否必选 | + | -------------------- | ------------------------------------------------------------ | ------------------ | + | npu_path | 配置 NPU 环境下的 dump.json 文件（单卡场景），str 类型。 | 是 | + | bench_path | 配置 CPU、GPU 或 NPU 环境下的 dump.json 文件（单卡场景），str 类型。 | 是 | + | stack_path | 配置 NPU dump 目录下的 stack.json 文件，str 类型。 | 是| + | is_print_compare_log | 配置是否开启日志打屏，bool 类型。可取值 True 或 False，默认为 True。 | 否 | + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md b/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md new file mode 100644 index 000000000..e633a3f24 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md @@ -0,0 +1,25 @@ +# 溢出解析工具（更新中） + +针对训练过程中的溢出检测场景（当《[精度数据采集](./dump.md)》开启溢出检测dump时），对于输入正常但输出存在溢出的API，会在训练执行目录下将溢出的API信息按照前向和反向分类，dump并保存为`dump.json`，前向过程溢出的API可通过该工具对`dump.json`进行解析，输出溢出API为正常溢出还是非正常溢出，从而帮助用户快速判断。 + +工具支持PyTorch版本：1.11/2.0/2.1/2.2。 + +操作步骤如下： + +1. 安装工具。 + + 详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 + +2. 执行溢出API解析操作。 + + ```bash + msprobe -f pytorch run_overflow_check -api_info ./dump.json + ``` + +| 参数名称 | 说明 | 是否必选 | +| -------------------------- | -------------------------------------------------- | -------- | +| -api_info或--api_info_file | 指定API信息文件dump.json。 | 是 | +| -j或--jit_compile | 开启jit编译。 | 否 | +| -d或--device | 指定Device ID，选择UT代码运行所在的卡，默认值为0。 | 否 | + +反向过程溢出的API暂不支持该功能。 diff --git a/debug/accuracy_tools/msprobe/docs/14.data_parse_PyTorch.md b/debug/accuracy_tools/msprobe/docs/14.data_parse_PyTorch.md new file mode 100644 index 000000000..68a3d1a57 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/14.data_parse_PyTorch.md @@ -0,0 +1,283 @@ +# PyTorch 场景的数据解析 + +## 1 简介 + +数据解析功能提供命令行交互式界面，提供了更多的结果展示。 + +使用场景：本工具主要用于精度比对前后两次 NPU kernel 层级 dump 数据的一致性。 + +依赖：CANN 包中的 msaccucmp 工具，需要安装 Ascend-CANN-toolkit，详见《[CANN 软件安装指南](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F700%2Fenvdeployment%2Finstg%2Finstg_0001.html)》。 + +## 2 数据解析操作指导 + +### 2.1 进入 parse 交互式界面 + +安装 msprobe 工具后，可以通过使用命令 `msprobe -f pytorch parse` 进入交互式界面，如下所示： + +```bash +msprobe -f pytorch parse +Parse >>> +``` + +可在 parse 的界面中执行 Shell 命令，以及如下场景的相关解析命令（详细介绍请参见以下章节。）： + +- 支持指定 kernel 层级算子数据比对。 +- 支持指定 kernel 层级算子数据转换及展示。 +- 支持交互式指定 pkl 文件中 API 对应 dump 数据查看。 +- 支持 API 进行可选层级比对和打印（统计级和像素级）。 + +Ctrl+C 可以退出 parse 交互式界面。不退出 parse 交互式界面若需要执行非该界面下的内置 Shell 命令，且命令与 parse 交互式界面命令冲突时，非该界面命令需要使用 run 命令，在相关命令前加上 run 前缀，如下示例： + +```bash +msprobe -f pytorch parse +Parse >>> run vim cli.py +Parse >>> vim cli.py +``` + +### 2.2 kernel 层级算子数据批量转换 + +本功能会将原有待比对 dump 数据目录下的 dump 数据按照算子名和时间戳进行梳理并分类，之后再将 dump 数据转为 npy 文件。 + +输入以下比对命令进行数据转换。 + +```bash +cad -m my_dump_path [-out output_path] [-asc msaccucmp_path] +``` + +| 参数名称 | 说明 | 是否必选 | +| -------- | ------------------------------------------------------------ | -------- | +| -m | 待转换 kernel dump 数据目录。需要指定到 kernel dump 数据的 deviceid 级目录。 | 是 | +| -out | 结果输出目录，须指定已存在的目录，默认为 ./parse_data/acl_batch_convert。未指定时保存在默认路径下，比对结束后会打印 log 提示输出结果存放路径。 | 否 | +| -asc | 指定 msaccucmp 路径，默认路径为：/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py。 | 否 | + +**示例代码**： + +```bash +# 传入待比对数据目录 +Parse >>> cad -m /home/xxx/my_dump_path/20000124003856/0 +# 转换结果打印 +...... +╭──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +# 转换前的dump文件 +│ SrcFile: /home/xxx/my_dump_path/20000124003856/0/272/TransData.trans_TransData_22.112.21.948645536672764 │ +# 转换后的npy文件 +│ - TransData.trans_TransData_22.112.21.948645536672764.output.0.npy │ +│ - TransData.trans_TransData_22.112.21.948645536672764.input.0.npy │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +...... +[INFO] The comparison result have been written to "./parse_data/acl_batch_convert". +``` + +**输出结果**： + +原 dump 数据目录： + +``` +├── /home/xxx/my_dump_path/20000124003856/0/ +│ ├── 272 +│ │ ├── {op_type}.{op_name}.{task_id}.{stream_id}.{timestamp} +│ │ ... +│ ├── 512 +│ ... +``` + +转换后： + +``` +├── ./parse_data/acl_batch_convert/{timestamp} +│ ├── {op_name1} +│ │ ├── {timestamp1} +│ │ | ├── {op_type}.{op_name}.{task_id}.{stream_id}.{timestamp}.{input/output}.{参数序号}.npy +│ │ | │ ... +│ │ ├── {timestamp2} +│ │ | ... +│ ├── {op_name2} +│ ├── ... +``` + +### 2.3 kernel 层级算子数据比对 + +本功能主要用于比对前后两次 NPU kernel 层级 dump 数据的一致性，并支持批量比对，比对时需要先将两份待比对的 NPU kernel 层级 dump 数据进行 [kernel 层级算子数据批量转换](#22-kernel-层级算子数据批量转换)，可以使两份数据更好的匹配；若直接进行 dump 数据的比对，建议只比对单个 dump 数据文件。 + +输入以下比对命令进行数据比对。 + +```bash +vc -m my_dump_path -g golden_dump_path [-out output_path] [-cmp_path msaccucmp_path] +``` + +| 参数名称 | 说明 | 是否必选 | +| --------- | ------------------------------------------------------------ | -------- | +| -m | 待比对 kernel dump 数据目录。如果比对单个算子，需要指定到 kernel dump 数据的 model_id 级目录；如果批量比对，则指定到 cad 转换后的 timestamp 级目录。 | 是 | +| -g | 标杆 kernel dump 数据目录。如果比对单个算子，需要指定到 kernel dump 数据的 model_id 级目录；如果批量比对，则指定到 cad 转换后的 timestamp 级目录。 | 是 | +| -out | 结果输出目录，须指定已存在的目录，默认为 ./parse_data/acl_batch_comapre。未指定时保存在默认路径下，比对结束后会打印 log 提示输出结果存放路径。 | 否 | +| -cmp_path | 指定 msaccucmp 路径，默认路径为：/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py | 否 | + +输出结果：`batch_compare_{timestamp}.csv` 文件。 + +**示例代码**： + +```bash +# 传入待比对数据目录以及标杆数据目录 +Parse >>> vc -m ./my_dump_path -g ./golden_data_path +[INFO]Compare result is saved in : parse_data/acl_batch_comapre/batch_compare_1707271118.csv +``` + +### 2.3 kernel 算子数据的 npy 转换 + +输入以下转换命令进行数据转换， 将 kernel 级别 dump 数据转为 npy 文件。 + +```bash +dc -n file_name/file_path [-f format] [-out output_path] +``` + +| 参数名称 | 说明 | 是否必选 | +| --------- | ------------------------------------------------------------ | -------- | +| -n | 需转换的 dump 数据文件或 dump 数据文件目录。 | 是 | +| -f | 开启 format 转换，指定该参数时需要配置 format 格式。当前内置的 Format 转换支持如下类型：
FRACTAL_NZ 转换 NCHW；
FRACTAL_NZ 转换成 NHWC；
FRACTAL_NZ 转换 ND；
HWCN 转换 FRACTAL_Z；
HWCN 转换成 NCHW；
HWCN 转换成 NHWC；
NC1HWC0 转换成 HWCN；
NC1HWC0 转换成 NCHW；
NC1HWC0 转换成 NHWC；
NCHW 转换成 FRACTAL_Z；
NCHW转换成NHWC；
NHWC转换成FRACTAL_Z；
NHWC转换成HWCN；
NHWC转换成NCHW；
NDC1HWC0转换成NCDHW。 | 否 | +| -out | 结果输出目录。 | 否 | +| -cmp_path | 指定 msaccucmp 路径，默认路径为：/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py | 否 | + +- 输出结果：npy 文件。 + +- 若指定 -out 参数需要用户传入输出路径，并且路径需要已存在。 + +- 若未指定输出目录， 则比对结束后将结果保存在默认目录 “./parse_data/convert_result” 中，比对结束后会打印 log 提示输出结果存放路径及转换结果。 + +- 输入以下命令，展示 npy 数据统计信息。 + + ```bash + pt -n file_path + ``` + + | 参数名称 | 说明 | 是否必选 | + | -------- | ------------- | -------- | + | -n | npy 文件路径。 | 是 | + + 打印统计信息：shape, dtype, max, min 和 mean。默认在 npy 文件路径下将该数据保存为 txt 文件。 + +**示例代码1**： + +```bash +# 传入需转换的dump文件目录 +Parse >>> dc -n ./dump_data/ +...... +# 转换结果 +╭──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ SrcFile: ./dump_data/ +│ - Add.fp32_vars_add_2fp32_vars_Relu_9.31.5.1636595794731103.input.0.npy │ +│ - Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.output.0.npy │ +│ - Add.fp32_vars_add_2fp32_vars_Relu_9.31.5.1636595794731103.input.1.npy │ +│ - Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.input.1.npy │ +│ - Add.fp32_vars_add_3fp32_vars_Relu_12.40.5.1636595794846124.input.1.npy │ +│ - Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.input.0.npy │ +│ - Add.fp32_vars_add_3fp32_vars_Relu_12.40.5.1636595794846124.input.0.npy │ +│ - Add.fp32_vars_add_2fp32_vars_Relu_9.31.5.1636595794731103.output.0.npy │ +│ - Add.fp32_vars_add_3fp32_vars_Relu_12.40.5.1636595794846124.output.0.npy │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` + +**示例代码2**： + +```bash +# 查看某个dump数据块的数据信息 +# 默认会将数据中的tensor保存成 txt +Parse >>> pt -n ./parse_data/dump_convert/Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.output.0.npy +...... +# 打印统计信息 +[Shape: (1, 16, 56, 56, 16)] [Dtype: float16] [Max: 452.0] [Min: -408.5] [Mean: -3.809] +Path: ./parse_data/dump_convert/Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.input.0.npy +TextFile:./parse_data/dump_convert/Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.input.0.npy.txt +``` + +### 2.4 dump.json 文件中指定 API 的 dump 数据信息查看（暂不支持） + +输入以下命令，解析并输出 dump.json 文件中指定 API 的统计信息。 + +```bash +pk -f pkl_path -n api_name +``` + +| 参数名称 | 说明 | 是否必选 | +| -------- | ----------------------- | -------- | +| -f | 指定 dump.json 文件路径。 | 是 | +| -n | 指定 API 名称。 | 是 | + +- 输出结果：打印统计信息（shape, dtype, max和min mean）。 +- 若 pkl 文件中存在相应的堆栈信息，则会打印堆栈信息。 + +**示例代码**： + +```bash +# 传入pkl文件及api名称 +Parse >>> pk -f ./torch_dump/xxx/rank0/dump.json -n Functional_conv2d_0_forward +...... +# 打印统计信息及堆栈（pkl文件不包含堆栈则不会打印堆栈） + +Statistic Info: + [Functional_conv2d_0_forward_input.0][dtype: torch.float32][shape: [2, 1, 2, 2]][max: 1.576936960220337][min: -0.9757485389709473][mean: 0.4961632490158081] + [Functional_conv2d_0_forward_input.1][dtype: torch.float32][shape: [2, 1, 2, 2]][max: 0.20064473152160645][min: -0.47102075815200806][mean: -0.20796933770179749] + [Functional_conv2d_0_forward_input.2][dtype: torch.float32][shape: [2]][max: 0.17380613088607788][min: -0.16853803396224976][mean: 0.0026340484619140625] + [Functional_conv2d_0_forward_output][dtype: torch.float32][shape: [2, 2, 1, 1]][max: 0.02364911139011383][min: -1.762906551361084][mean: -0.6710853576660156] +``` + +### 2.5 API 可选层级比对 + +输入以下命令, 进行统计级和像素级比对。 + +```bash +cn -m my_data*.npy -g gloden*.npy [-p num] [-al atol] [-rl rtol] +``` + +- 统计级比对：对 tensor 整体进行余弦值及相对误差的计算。 +- 像素级比对：对输入的两个 npy 文件进行逐元素比对。若两个 tensor 对应元素的相对误差或绝对误差大于**误差阈值**（-al 和 -rl 配置）则被标记为错误数据。 + +| 参数名称 | 说明 | 是否必选 | +| -------- | ----------------------------------------------- | -------- | +| -m | 待比对数据。 | 是 | +| -g | 标杆数据。 | 是 | +| -p | 设置比对结束后打印错误元素的个数，默认值 20。 | 否 | +| -al | 判定数据存在精度问题的绝对误差阈值，默认 0.001。 | 否 | +| -rl | 判定数据存在精度问题的相对误差阈值，默认 0.001。 | 否 | +| -s | 将 npy 文件保存成 txt 文件，用于查看，默认开启。 | 否 | + +输出结果： + +- 统计级比对结果。 +- 两个文件的统计信息（shape, dtype, max, min和mean）。 +- 错误数据打印表格。 + +**示例代码**： + +```bash +# 对比两个tensor的数据 +Parse >>> cn -m Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy -g InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy -p 10 -s -al 0.002 -rl 0.005 + Error Item Table Top Item Table +┏━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ ┏━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ +┃ Index ┃ Left ┃ Right ┃ Diff ┃ ┃ Index ┃ Left ┃ Right ┃ Diff ┃ +┡━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ ┡━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ +│ 155 │ 0.024600908 │ 0.022271132 │ 0.002329776 │ │ 0 │ -0.9206961 │ -0.9222216 │ 0.0015255213 │ +│ 247 │ 0.015752593 │ 0.017937578 │ 0.0021849852 │ │ 1 │ -0.6416973 │ -0.64051837 │ 0.0011789203 │ +│ 282 │ -0.0101207765 │ -0.007852031 │ 0.0022687456 │ │ 2 │ -0.35383835 │ -0.35433492 │ 0.0004965663 │ +│ 292 │ 0.019581757 │ 0.02240482 │ 0.0028230622 │ │ 3 │ -0.18851271 │ -0.18883198 │ 0.00031927228 │ +│ 640 │ -0.06593232 │ -0.06874806 │ 0.0028157383 │ │ 4 │ -0.43508735 │ -0.43534422 │ 0.00025686622 │ +│ 1420 │ 0.09293677 │ 0.09586689 │ 0.0029301196 │ │ 5 │ 1.4447614 │ 1.4466647 │ 0.0019032955 │ +│ 1462 │ -0.085207745 │ -0.088047795 │ 0.0028400496 │ │ 6 │ -0.3455438 │ -0.3444429 │ 0.0011008978 │ +│ 1891 │ -0.03433288 │ -0.036525503 │ 0.002192624 │ │ 7 │ -0.6560242 │ -0.6564579 │ 0.0004336834 │ +│ 2033 │ 0.06828873 │ 0.07139922 │ 0.0031104907 │ │ 8 │ -2.6964858 │ -2.6975214 │ 0.0010356903 │ +│ 2246 │ -0.06376442 │ -0.06121233 │ 0.002552092 │ │ 9 │ -0.73746175 │ -0.73650354 │ 0.00095820427 │ +└───────┴───────────────┴──────────────┴──────────────┘ └───────┴─────────────┴─────────────┴───────────────┘ +╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ Left: | +│ |- NpyFile: ./dump/temp/decode/Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy | +│ |- TxtFile: ./dump/temp/decode/Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy.txt | +│ |- NpySpec: [Shape: (32, 8, 8, 320)] [Dtype: float32] [Max: 5.846897] [Min: -8.368301] [Mean: -0.72565556] | +│ DstFile: │ +│ |- NpyFile: ./dump/cpu/InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy | +│ |- TxtFile: ./dump/cpu/InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy.txt | +│ |- NpySpec: [Shape: (32, 8, 8, 320)] [Dtype: float32] [Max: 5.8425903] [Min: -8.374472] [Mean: -0.7256237] │ +│ NumCnt: 655360 │ +│ AllClose: False │ +│ CosSim: 0.99999493 │ +│ ErrorPer: 0.023504638671875 (rl= 0.005, al= 0.002) │ +╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +``` diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md similarity index 96% rename from debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md rename to debug/accuracy_tools/msprobe/docs/17.grad_probe.md index fcbd2f123..d7856e35a 100644 --- a/debug/accuracy_tools/msprobe/doc/grad_probe/grad_probe.md +++ b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md @@ -1,4 +1,4 @@ -# Ascend模型梯度状态监测工具 +# Ascend 模型梯度状态监测工具 梯度状态监测工具提供了两种能力： @@ -7,10 +7,7 @@ 工具支持PyTorch版本：2.0/2.1/2.2；支持MindSpore版本：r2.3。 -## 工具特性 -- 使用便捷，无需在训练流程里插入代码 -- 可以精准定位问题出现的step ## 使用方式 @@ -65,7 +62,7 @@ + 值分布：梯度数据落在各个区间的元素个数占总元素个数的比例。 + bounds：一个列表，用来划分出区间以统计值分布。例如传入bounds = [-10, 0, 10]，此时有一个 grad_value: Tensor = [9.3 , 5.4, -1.0, -12.3]，依据 bounds 划分出 (-inf, -10]、(-10, 0]、(0, 10]、(10, inf) 四个区间，然后统计grad_value里的数据落在每个区间内的个数，得到 1、1、2、0。如下图所示： - ![Alt text](img/image-1.png) + ![Alt text](img/grad_probe_image-1.png) 2. 插入代码。示例代码如下： @@ -110,7 +107,7 @@ gm.stop() 样例如下： -![Alt text](img/image.png) +![Alt text](img/grad_probe_image.png) | 字段 | 含义 | | --------------------- | ------------------------------------------------------------| @@ -166,15 +163,15 @@ GradComparator.compare_distributed("配置文件里写的dump_path", - 第0步相似度低于0.97 -![Alt text](img/image-3.png) +![Alt text](img/grad_probe_image-3.png) - 第3步相似度下降超过0.03 -![Alt text](img/image-4.png) +![Alt text](img/grad_probe_image-4.png) - 正常情况 -![Alt text](img/image-2.png) +![Alt text](img/grad_probe_image-2.png) 这个原则是一个经验性的指标，并不是严格的标注，还需要结合实际情况具体分析。 @@ -202,6 +199,3 @@ GradComparator.compare_distributed(dump_path1, dump_path2, output_path) | dump_path1 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path。 | 是 | | dump_path2 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path，与dump_path1可以互换。 | 是 | | output_path |输出结果目录，不存在会新建。 | 是 | - - -# FAQ diff --git a/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md b/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md new file mode 100644 index 000000000..8d12a7292 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md @@ -0,0 +1,193 @@ +# 精度预检工具 + +1. 预检工具在dump和run_ut的过程中，是否需要同时开启或关闭jit编译（jit_compile）？ + + 答：是。 + +2. 预检工具对于type_as这类涉及数据类型转换操作的API，是否具有参考性？ + + 由于这类API在CPU侧存在精度先提升后下降的操作，因此这类API的有效性的参考价值有限。 + +3. run ut过程中出现报错：ERROR:Got unsupported ScalarType BFloat16 + + 答：请使用最新版本的工具。 + +4. Dropout算子，CPU和NPU的随机应该不一样，为什么结果比对是一致的？ + + 答：这个结果是正常的，工具对该算子有特殊处理，只判定位置为0的位置比例大约和设定p值相当。 + +5. 为什么浮点型数据bench和CPU的dtype不一致？ + + 答：对于fp16的数据，CPU会上升一个精度fp32去计算，这是和算子那边对齐的精度结论，CPU用更高精度去计算会更接近真实值。 + +6. 添加预检工具后截取操作报错：`IndexError: too many indices for tensor of dimension x` 或 `TypeError: len() of a 0-d tensor`。 + + 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- __getitem__`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 + +7. 添加预检工具后F.gelu触发ValueError报错：`activation_func must be F.gelu`等。 + + 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中functional:下的的`- gelu`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 + +8. 添加预检工具后触发AsStrided算子相关的报错，或者编译相关的报错，如：`Failed to compile Op [AsStrided]`。 + + 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- t`和`- transpose`。 + +9. Tensor 魔法函数具体对应什么操作？ + + 答： + + | Tensor魔法函数 | 具体操作 | + | --------------- | ---------------- | + | `__add__` | + | + | `__and__` | & | + | `__bool__` | 返回Tensor布尔值 | + | `__div__` | / | + | `__eq__` | == | + | `__ge__` | >= | + | `__gt__` | > | + | `__iadd__` | += | + | `__iand__` | &= | + | `__idiv__` | /= | + | `__ifloordiv__` | //= | + | `__ilshift__` | <<= | + | `__imod__` | %= | + | `__imul__` | *= | + | `__ior__` | \|= | + | `__irshift__` | >>= | + | `__isub__` | -= | + | `__ixor__` | ^= | + | `__lshift__` | << | + | `__matmul__` | 矩阵乘法 | + | `__mod__` | % | + | `__mul__` | * | + | `__nonzero__` | 同`__bool__` | + | `__or__` | \| | + | `__radd__` | +（反向） | + | `__rmul__` | *（反向） | + | `__rshift__` | >> | + | `__sub__` | - | + | `__truediv__` | 同`__div__` | + | `__xor__` | ^ | + +# 精度比对工具 + +## 工具使用 + +### dump指定融合算子 + +dump指定操作当前支持dump指定融合算子的输入输出，需要在mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml中添加，比如以下代码段调用的softmax融合算子 + +``` +def npu_forward_fused_softmax(self, input_, mask): + resl = torch_npu.npu_scaled_masked_softmax(input_, mask, self.scale, False) + return resl +``` + +如果需要dump其中调用的npu_scaled_masked_softmax算子的输入输出信息，需要在support_wrap_ops.yaml中的torch_npu: 中自行添加该融合算子即可： + +``` +- npu_scaled_masked_softmax +``` + +（npu_scaled_masked_softmax融合算子工具已支持dump，本例仅供参考） + +## 常见问题 + +### 1. 在同一个目录多次执行dump会冲突吗？ + +会，同一个目录多次dump，会覆盖上一次结果，可以使用dump_path参数修改dump目录。 + +### 2. 如何dump算子级的数据？ + +需要配置level为L2模式。 + +### 3. 工具比对发现NPU和标杆数据的API无法完全对齐？ + +torch版本和硬件差异属于正常情况。 + +## 异常情况 + +### 2. HCCL 报错： error code: EI0006 + +**故障现象** + +使用msprobe工具时，报错： error code: EI0006。 + +**故障原因** + +CANN软件版本较低导致不兼容。 + +**故障处理** + +升级新版CANN软件版本。 + +### 3. torch_npu._C._clear_overflow_npu() RuntimeError NPU error，error code is 107002 + +如果运行溢出检测功能遇到这个报错，采取以下解决方法： +如果是单卡运行，添加如下代码，0是卡号，选择自己空闲的卡号。 + +``` +torch.npu.set_device('npu:0') +``` + +如果多卡运行，请在代码中修改对应卡号，比如进程使用卡号为{rank}时可以添加如下代码： + +``` +torch.npu.set_device(f'npu:{rank}') +``` + +如果运行精度比对功能遇到这个报错，尝试安装最新版本的msprobe。 + +### 4. dump得到的VF_lstm_99_forward_input.1.0.npy、VF_lstm_99_forward_input.1.1.npy类似的数据是否正常？ + +带1.0/1.1/1.2后缀的npy是正常现象，例如当输入数据为[[tensor1, tensor2, tensor3]]会生成这样的后缀。 + +### 5. 进行compare报错：The current file contains stack information, please turn on the stack_mode + +在比对脚本中，设置stack_mode=True，例如： + +``` +from msprobe.pytorch import compare +dump_result_param={ +"npu_json_path": "./npu_dump/dump.json", +"bench_json_path": "./gpu_dump/dump.json", +"stack_json_path": "./npu_dump/stack.json", +"is_print_compare_log": True +} +compare(dump_result_param, output_path="./output", stack_mode=True) +``` + +### 6. dump指定反向API的kernel级别的数据报错：NameError：name 'torch_npu' is not defined + +- 如果是npu环境，请安装torch_npu； +- 如果是gpu环境，暂不支持dump指定API的kernel级别的数据 + +### 7. 配置dump_path后，使用工具报错：[ERROR]The file path /home/xxx/dump contains special characters + +- 请检查你设置的dump绝对路径是否包含特殊字符，确保路径名只包含大小写字母、数字、下划线、斜杠、点和短横线 +- 注意，如果执行脚本的路径为/home/abc++/，设置的dump_path="./dump"，工具实际校验的路径为绝对路径/home/abc++/dump，++为特殊字符，会引发本条报错 + +### 8. 无法dump matmul权重的反向梯度数据 + +- matmul期望的输入是二维，当输入不是二维时，会将输入通过view操作展成二维，再进行matmul运算，因此在反向求导时，backward_hook能拿到的是UnsafeViewBackward这步操作里面数据的梯度信息，取不到MmBackward这步操作里面数据的梯度信息，即权重的反向梯度数据。 +- 典型的例子有，当linear的输入不是二维，且无bias时，会调用output = input.matmul(weight.t()),因此拿不到linear层的weight的反向梯度数据。 + +### 9. dump.json文件中的某些api的dtype类型为float16，但是读取此api的npy文件显示的dtype类型为float32 + +- msprobe工具在dump数据时需要将原始数据从npu to cpu上再转换为numpy类型，npu to cpu的逻辑和gpu to cpu是保持一致的，都存在dtype可能从float16变为float32类型的情况，如果出现dtype不一致的问题，最终dump数据的dtype以pkl文件为准。 + +### 10. 使用dataloader后raise异常Exception("msprobe: exit after iteration {}". format(max(self.config.step)) + +- 正常现象，dataloader通过raise结束程序，堆栈信息可忽略。 + +### 11. 添加msprobe工具后截取操作报错：`IndexError: too many indices for tensor of dimension x` 或 `TypeError: len() of a 0-d tensor`。 + +- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- __getitem__`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 + +### 12. 添加msprobe工具后F.gelu触发ValueError报错：`activation_func must be F.gelu`等。 + +- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中functional:下的的`- gelu`，工具会跳过dump该API。如果是需要dump的关键位置api也可以考虑根据报错堆栈信息注释引发报错的类型检查。 + +### 13. 添加msprobe工具后触发AsStrided算子相关的报错，或者编译相关的报错，如：`Failed to compile Op [AsStrided]`。 + +- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- t`和`- transpose`。 diff --git a/debug/accuracy_tools/msprobe/docs/S01.report_msprobe_dump_standard_performance_baseline.md b/debug/accuracy_tools/msprobe/docs/S01.report_msprobe_dump_standard_performance_baseline.md new file mode 100644 index 000000000..b2db12147 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/S01.report_msprobe_dump_standard_performance_baseline.md @@ -0,0 +1,182 @@ +# msprobe 精度工具标准性能基线报告 + +## 1 环境信息 + +NPU：Atlas A2 训练系列产品 + +CPU： + +![输入图片说明](img/cpu_info.png) + +Torch：2.1.0 + +CANN：8.0.T2 + +除上述环境信息影响性能外，API 的数量、种类以及 Shape 都会对性能产生影响，因此本次选取指定网络进行测试，为了避免算子编译耗时的影响，所有模型运行时都开启二进制，模型中添加 torch.npu.set_compile_mode(jit_compile=False)，所有模型都 dump 第二个 step 的数据。 + +## 模型信息和性能基线 + +大模型在使用msprobe工具dump数据时，建议先简化模型层数，减少dump数据量。 + +以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 + +### 工具配置信息 + +dump全部API级别输入输出数据以及相应堆栈信息，配置如下： + +```python +debugger = PrecisionDebugger(dump_path="./dump_path", hook_name="dump") +debugger.configure_hook(mode="api_stack") +``` + +多卡指定rank0 dump，配置如下： + +```python +debugger = PrecisionDebugger(dump_path="./dump_path", hook_name="dump",rank=0) +debugger.configure_hook(mode="api_stack") +``` + +dump保存API统计信息的pkl文件，配置如下： + +```python +debugger = PrecisionDebugger(dump_path="./dump_path", hook_name="dump") +debugger.configure_hook(mode="api_stack", summary_only=True) +``` + +### YOLOV5s + +单卡 + +主要数据类型：FLOAT32 + +启动命令参数：python3 train_ptdbg.py --data ./data/coco.yaml --cfg yolov5s.yaml --weights '' --epochs 1 --batch-size 8 --device 1 + +dump保存API统计信息的pkl文件耗时：**7s** + +进行单卡dump全部API级别输入输出数据以及相应堆栈信息耗时：**11s** + +- dump存盘的API numpy文件大小：13G + + ![输入图片说明](img/YOLOV5S_1.png) + +- api numpy文件数量：3009个 + + ![输入图片说明](img/YOLOV5S_2.png) + + + + +### GPT-3 + +#### NUM_LAYER：1 + +8卡 + +主要数据类型：FLOAT16 + +启动命令参数： + +``` +python3 -m torch.distributed.launch $DISTRIBUTED_ARGS ../../pretrain_gpt_ptdbg.py --num-layers 1 --hidden-size 12288 --num-attention-heads 24 --micro-batch-size 2 --global-batch-size 2 --seq-length 1024 --max-position-embeddings 1024 --train-iters 10 --lr-decay-iters 320000 --save $CHECKPOINT_PATH --load $CHECKPOINT_PATH --data-path $DATA_PATH --tensor-model-parallel-size 8 --use-distributed-optimizer --pipeline-model-parallel-size 8 --vocab-file gpt2-vocab.json --merge-file gpt2-merges.txt --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.375e-5 --lr-decay-style cosine --min-lr 0.375e-6 --weight-decay 0.1 --clip-grad 1.0 --lr-warmup-fraction .01 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 +--recompute-granularity full --recompute-method uniform --no-gradient-accumulation-fusion --log-interval 1 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --fp16 +``` + +dump保存API统计信息的pkl文件耗时：**3.3s** + +进行8卡dump全部API级别输入输出数据以及相应堆栈信息耗时：**53s** + +- dump存盘的api numpy文件大小：145G + + ![输入图片说明](img/GPT-3_1.png) + +- API numpy文件数量：5130个 + + ![输入图片说明](img/GPT-3_2.png) + + +**经测试8卡同时写入磁盘已达到磁盘I/O上限，工具的dump速度取决于磁盘性能，本机环境多进程写入磁盘上限为3GB/秒左右，理论上保存145GB的数据需要50秒左右，如果dump的数据中包含许多的小文件，那么耗时将会更久。** + +指定rank0 dump耗时：**9s** + +- dump存盘的api numpy文件大小：19G + + ![输入图片说明](img/GPT-3_3.png) + +- api numpy文件数量：643个 + + ![输入图片说明](img/GPT-3_4.png) + + +#### NUM_LAYER：8 + +8卡 + +主要数据类型：FLOAT16 + +启动命令参数： + +``` +python3 -m torch.distributed.launch $DISTRIBUTED_ARGS ../../pretrain_gpt_ptdbg.py --num-layers 8 --hidden-size 12288 --num-attention-heads 24 --micro-batch-size 2 --global-batch-size 2 --seq-length 1024 --max-position-embeddings 1024 --train-iters 10 --lr-decay-iters 320000 --save $CHECKPOINT_PATH --load $CHECKPOINT_PATH --data-path $DATA_PATH --tensor-model-parallel-size 8 --use-distributed-optimizer --pipeline-model-parallel-size 8 --vocab-file gpt2-vocab.json --merge-file gpt2-merges.txt --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.375e-5 --lr-decay-style cosine --min-lr 0.375e-6 --weight-decay 0.1 --clip-grad 1.0 --lr-warmup-fraction .01 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --recompute-granularity full --recompute-method uniform --no-gradient-accumulation-fusion --log-interval 1 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --fp16 +``` + +dump保存API统计信息的pkl文件耗时：**6.7s** + +进行8卡dump全部API级别输入输出数据以及相应堆栈信息耗时：**323s** + +- dump存盘的API numpy文件大小：878G + + ![输入图片说明](img/GPT-3_5.png) + +- API numpy文件数量：24002个 + + ![输入图片说明](img/GPT-3_6.png) + + +指定rank0 dump耗时：**47s** + +- dump存盘的API numpy文件大小：110G + + ![输入图片说明](img/GPT-3_7.png) + +- API numpy文件数量：3002个 + + ![输入图片说明](img/GPT-3_8.png) + + +### BLOOM-7B + +8卡 + +NUM_LAYER：1 + +主要数据类型：BFLOAT16 + +启动命令参数： + +``` +python -m torch.distributed.launch $DISTRIBUTED_ARGS pretrain_llama.py --DDP-impl local --tensor-model-parallel-size 8 --pipeline-model-parallel-size 1 --sequence-parallel --num-layers 1 --hidden-size 12288 --position-embedding-type rope --normalization RMSNorm --ffn-hidden-size 11008 --num-attention-heads 24 --attention-dropout 0.0 --hidden-dropout 0.0 --init-method-std 0.01 --micro-batch-size 2 --global-batch-size 2 --seq-length 1024 --max-position-embeddings 1024 --data-path $DATA_PATH --tokenizer-name-or-path $TOKENIZER_PATH --tokenizer-not-use-fast --split 100,0,0 --distributed-backend nccl --lr 1.25e-5 --min-lr 1.25e-6 --lr-decay-style cosine --weight-decay 1e-1 --clip-grad 1.0 --initial-loss-scale 65536.0 --adam-beta1 0.9 --adam-beta2 0.95 --log-interval 1 --load ${LOAD_CHECKPOINT_PATH} --save ${SAVE_CHECKPOINT_PATH} --save-interval 10000 --eval-interval 10000 --eval-iters 0 --use-fused-rotary-pos-emb --no-masked-softmax-fusion --no-load-optim --no-load-rng --train-iters 20 --lr-warmup-fraction 0.01 --mlp-layer-fusion --use-flash-attn --use-fused-rmsnorm --bf16 +``` + +dump保存API统计信息的pkl文件耗时：**3s** + +进行8卡dump全部API级别输入输出数据以及相应堆栈信息耗时：**61s** + +- dump存盘的API numpy文件大小：160G + + ![输入图片说明](img/BLOOM-7B_1.png) + +- API numpy文件数量：4924个 + + ![输入图片说明](img/BLOOM-7B_2.png) + + +指定rank0 dump耗时：**17s** + +- dump存盘的API numpy文件大小：20G + + ![输入图片说明](img/BLOOM-7B_3.png) + +- API numpy文件数量：633个 + + ![输入图片说明](img/BLOOM-7B_4.png) + diff --git a/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md b/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md new file mode 100644 index 000000000..8862c5b76 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md @@ -0,0 +1,151 @@ +# 无标杆工具场景验证和性能基线报告 + +## 环境信息 + +NPU：Atlas A2 训练系列产品 + +CPU： + +![输入图片说明](img/cpu_info.png) + +Torch：2.1.0 + +CANN：8.0.T5 + +除上述环境信息影响性能外，API的数量、种类以及Shape都会对性能产生影响，因此本次选取不同场景网络和不同算子进行测试。 + +## 模型信息和性能基线 + +大模型在使用msprobe工具dump数据时，建议先简化模型层数，减少dump数据量。 + +以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 + + + +### LLaMA2-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink +其中，softmax算子为FLOAT32，输入输出均为2G大小，为模型最大显存开销的API。 + +在该模型下、对无标杆工具处理模式、插装范围、扰动方式组合下性能和显存基线进行覆盖。 + +性能基线报告 +其中耗时为训练10步，去除第一步耗时所得的平均每步耗时。 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.24 | 13.69 | 1 | 1 | 混精模式基线 | +| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1 | softmax本身为高精度，跳过 | +| check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | +| check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | +| check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | +| check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | +| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | +| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.38 | 1 | | +| check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | +| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | +| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | +| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | +| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | +| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | +| check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | +| check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | +| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | +| fix | 前 | ["softmax"] | to_cpu | 9.70 | 16.67 | 40.42 | 1.22 | 不支持整网、不支持反向 | +| fix | 前 | ["softmax"] | improve_precision | 0.26 | 14.67 | 1.08 | 1.07 | 不支持整网、不支持反向 | +| 预热 | 前 | [] | improve_precision | 155.07 | 24.79 | 646.13 | 1.81 | 低精度模型基线、只测预热的迭代 | +| 预热 | 反 | [] | improve_precision | 72.29 | 22.01 | 301.21 | 1.61 | 低精度模型基线、只测预热的迭代，grad_output为高精度的算子跳过 | + +### Aquila2-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.17 | 13.66 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.57 | 14.24 | 9.24 | 1.04 | | +| check | 反 | [] | add_noise | 21.05 | 14.19 | 123.82 | 1.04 | | +| fix | 前 | [] | improve_precision | 0.95 | 15.55 | 5.59 | 1.14 | | + +### Baichuan2-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.26 | 12.12 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.02 | 12.27 | 3.92 | 1.01 | | +| check | 反 | [] | add_noise | 11.15 | 12.67 | 42.88 | 1.05 | | +| fix | 前 | [] | improve_precision | 0.95 | 12.82 | 3.65 | 1.06 | | + +### Bloom-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.14 | 9.51 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.64 | 11.58 | 11.71 | 1.22 | | +| check | 反 | [] | add_noise | 17.15 | 9.51 | 122.5 | 1 | | +| fix | 前 | [] | improve_precision | 0.87 | 10.62 | 6.21 | 1.12 | | + +### Interlm-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.13 | 10.76 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.19 | 11.68 | 9.15 | 1.09 | | +| check | 反 | [] | add_noise | 11.69 | 10.89 | 89.92 | 1.01 | | +| fix | 前 | [] | improve_precision | 0.75 | 11.68 | 5.77 | 1.09 | | + +### Qwen-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.28 | 18.41 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 2.34 | 23.18 | 8.36 | 1.26 | | +| check | 反 | [] | add_noise | 22.07 | 19.47 | 78.82 | 1.06 | | +| fix | 前 | [] | improve_precision | 1.31 | 21.11 | 4.68 | 1.15 | | + +### Gemma-7B + +NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.15 | 11.06 | 1 | 1 | 混精模式基线 | +| check | 前 | [] | improve_precision | 1.49 | 13.17 | 9.93 | 1.19 | | +| check | 反 | [] | add_noise | 16.69 | 11.06 | 111.27 | 1 | | +| fix | 前 | [] | improve_precision | 0.87 | 12.25 | 5.8 | 1.11 | | + +### ResNet50-Cifar +1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch。 +主要算子为conv2d，每个step有51个, 因此对conv2d进行检测。 +CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step信息和反向信息）。 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.09 | 7.63 | 1 | 1 | 基线 | +| check | 前 | ["conv2d"] | improve_precision | 0.889 | 7.94 | 9.81 | 1.04 | | +| fix | 前 | ["conv2d"] | improve_precision | 0.328 | 7.47 | 3.64 | 0.91 | | +| fix | 前 | ["conv2d"] | to_cpu | 12.23 | 7.47 | 135.88 | 0.91 | | + +### OpenSora1.0 + +4卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch + +每张卡每个step中linear算子个数为257个，FA算子个数为83（FA算子反向无效）。 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| +| / | / | / | / | 0.99 | 17.61 | 1 | 1 | 混精模式基线 | +| check | 前 | ["linear","npu_fusion_attention"] | improve_precision | 3.88 | 17.61 | 3.92 | 1 | | +| check | 前 | ["linear","npu_fusion_attention"] | add_noise | 3.46 | 17.61 | 3.49 | 1 | | +| check | 反 | ["linear"] | improve_precision | 12.61 | 17.61 | 12.74 | 1 | | +| check | 反 | ["linear"] | add_noise | 9.8 | 17.61 | 9.90 | 1 | | +| fix | 前 | ["linear"] | to_cpu | 18.83 | 17.61 | 19.02 | 1 | | +| fix | 前 | ["linear"] | improve_precision | 2.83 | 17.61 | 2.86 | 1 | | diff --git a/debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_1.png b/debug/accuracy_tools/msprobe/docs/img/BLOOM-7B_1.png new file mode 100644 index 0000000000000000000000000000000000000000..3853626d6fab127915425238c76f16836a4cef0b GIT binary patch literal 3534 zcmV;<4KebGP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D4P!|}K~#8N?VUYu z+{_WjW`p&**ujn=?saK~~lp=Nhcw!827RfOK4C-O2mnCiY9uQo1Ejo{+b(=cqGxIj3e zGU%2Ha%dvkr$I~1X1mqWWG)b+(sn;kWho0=-KL-!xjaz5?8$k5C83HxB!~jbn4Ub= zL(J{tvlLVaKa4?eu^ooKqw52&r^%~|#l}9+IQqOUTfE}ZMS(({XO3E&dby8&hN_6f z$xZz81p|CrbcL`7QYI;vee&>fT&Q;b@~?5)(6%6#UL>=Yt?PMzDh0Gjg}?XvP55f_GLPnbjwp7NRIk4)HrCM4*TT+W&iB7x%zC~Y4xaK>Ro5ydL zcRG|-Q#`bu6wz^ng6%m@zMFiz&1UE~6l?OzIyg8$lY)f957Wdipbripnk$Dt;OED)=?d0>Hij9#avU>rL>zgxr`q5yXhlmbXrC%!H`{tn~}p?JiYxog3_Mma4a(Uh34jqLfb z!gbB#j*&dyU<4D!u@q}EcFP#AYmbg;I_D>Dz&_ZN=rNEp5US9C6>pRoLkVxqfi>-}jqglfKcL_R*Ecx#hg5+41hd zzEuTkHmNLyO;;O}i7cF~2fiqE=Tyl~?F{DD-KVX$gdK+kW8%WGJN)M;CS0A3GskDg zFe&idFOv~Rw@uEAz>E$HSWN~yYIBR(oMv81@~Xo%KxdX8jdLLlM-3gmjgYdxY2Y_y zXbC?T8;)Wvgl1TCsE$;DL<)_wV&U3xb@s`=jW@%LaI|+x^3*|h04F)*Hipz}*zWs7t7WC2Q8IZVsjdE!m$s}0lR)fw;EXziBkNYqlDmYjSo5Jbzrl$CrhGU0a zX))&%!m;LCj)o(~4>xOaM(GpKZX81yH;Dq7*1*KDJ4f%Rdhk;y)fMA(Z=3f; z^n2LDwM3sv!LA)oGgfh1m2f2M(l6@+hoe3QW-I(D3jSb+A++UeM#GXppP6|4MkRzJ zf1Kij`;w)nY}1ZmZA_-&!!e{Hi6VC0mdUPopwBV}8lO}LNBfQHg>v|O2DuySvk}6c zHsIyJhg$hkVN5tiXhTuE#%CT@6pDQ*)dfAyK5etUXn_W*gAN9&i)_})WJ<@(7aZCz z+ZLaGuU34@6WN~>j-PHmf70UB{YMVO+pqV$@2S|dV^|xLX)Fv!h-={pY=)CeCOSmy zF0O4o;<_hK-GZF)rzGga`oM-`sOx<8`f0!HP)45#s@0`kNqbcAc7WH92C6omaa*p5 zP;exDr@GuOvJ{DXnhJ*PHd|t*ebn+O@9!RHns$@*3gPI-WEy!mj%6T}IlQ8Ym>1%1 zvL+2NWy|o&*`aeu$(*Ea0U*{`MJFQ`JGt!eSqVBxnTkLnQoi0rgh$HpDGB|uL+Sh! zo3*-fv7trCLtm_Z6przDQZbsWeH3Ab`Wr2Z7o6wM&~TiMyd%P+a=N%vxP-ZN_i5`bVaH+qm@Ey)M&kg4a4bCETAT67 zqS4TB#G1h03SdD%63(+v16BsQ6jHhS$if00vm#d4R zwKkTU#*UO}8IEJT1Qs2?9>oC#!{{Tl4MW zrNVBXc58K)+Dyi(=ojXnw9r zF*$aGmKTAs$Rmb?Tm8AK%*|P;|x(WIkx7F z$eNt(_}trX^ldg#ulcEvl6Mz&Z1UQv??AmsRRtA^XdBW+s8X8cOSmEpCX~;_PW&BRKRB=wwmdCVA`tI?Ch|g*%>G6cCEiV*h)=r|uXmG>XICy=h>1zaiu7P3aB4MmL>g_s9ajAMPiQFFuaz0}FRcR&?WT1+1wl)slU1`JAz%EyC+g(ugLcfgFxWDs2X8bJ6 zL;3S-E+NiMj&uF=~&A+vXlp+^kb66fz#3$AtlpKQi~T!8uQ+foZl3p{Eaf; zJkI*4?y`RA5$OwdvC(K8i)7^paOs){+AvHuv!la-ic_h1sBW}p=t(zRN8x2mQ z(Rj1itS_W*HvC%j)T5@bOUJdpqwe3GW+*p+pOODAizWq7cim&}-*{pVt?B~R|THw{h zDx@~%K{xFu%aWYv|LNhzs!(!_`QhT?XIjko<$wS2TiX7f_lO_<_N(u|{>%UV_^001Hg0ssI3odsUI00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D6c|ZFK~#8N?VZ1i zCEIbwd$p^lMF=N)4@>rdgfW{)FpF%2G$6(>(8<`?pb6w;urqVnK%s&`5=iM~!!amP zp!hNitIWYureH2!KlBG!Lg6*zz?p=fzEyR8)cMu7`~KLSyHg)#`<||@Q>RXy)2C1W z-0sNO6pCF>|7Q4`9UU!}DJ zFK%x{4pvKe1-xTSOnG4N0Q%jEaVt!=`W7f-2RL@;BzgJI0csQ36&6ykY$|Uxm6)-1+El^X8~kRBtb5x zSUwld7hSra%dt^P7vy9t>HuYj({dijUg^?(+O-|)xUE44P`Y6ul#4$^LtNpll9m%o z5(@pC&TK=8pjx8hvlNt$5PI%WhoNF*av>JGTrpL;Re zbh6>Rdh38@^h)L&1z}25p5kb9?W#4c0?n##<1O5KhJxnNIwN|7zY;^;Zh@m}-(${* zDXlE!2n#cGPO+FFlyc;C%jFy)cbYs=U8=Mok|1ZITac4k-pZxnWFusWY~=-+BF4{1S6Nr}*;4ZcLw)R#!PL?5Gy?V!J&Xk;S8MB3T0Us# z>WRNRwhuy$_*XN=JBRCb$RU^L!gZ4gHVk7Te30ci1dWRRXe|=M*c16^hrv_kBG&DU z{qeBocCTN5{C?a@KXmMr!Z3^nK{nXwCxmLu_$m|Q9~g)a=JO?|US@`jPAhtKF{htX zk-c2KqN~eTuCPqxM|82yo$V-{B&~Hy*l%6#1P1d~SIkk$7Vehf0&lMf$ATDLPj>Hw8capQn(w{`ynhZ=wX^_h+LMmA2Orr zSW=X=F3;u778k+Bw)>{=D?6So%1Z1gTJnjU()H*-9PH#y1>rlQ?v$>>`3CCeP^N%V zBGX}H01{;IitB~GPxK4wVQ%mO+F%&$luoYS80>t*x9Y`dBUfce#+V=5!-zNjHd3ASmM7J-)-b;^0p1y=K{Z$^5TxVa~IXXIj@PK6!K>{9HSonaUcf;iaG38~D+{Yb2jv~nzp zI)-5w2ZbN(=z=<3gz%s2A!N2G4=@rn9m6n;gTPK>4a0a4+@rycZnAiE+As`bOFH0^ zhG7`*FFPIVvJb|#*<{w%4}QBuSf{hiY_XMhLz}f&HBHr?V91{P>FJ9KmD(rY3mAAx zqt$k!+jIhKb3)wq;luk*tLx)&(IOWO$N9FFSe z__+cB0#I^a~Ler zen^a}V?kL~)sBY$r;iO8+wPm92cv4o8%b+W+xK8M<5#aZ#SJ6z(rFMHl@jtl&G1*i zL`Rl@Uu;Oy(4n!Bzf=XWwh1Jil?7DVJ!ClGThIaMNmQ!CJxwmg?19he%{UL^||CjkcMAqFstn_FWz2bt4?{X{A<2q_uBF zSKnNM|UA>xdnCO#;g+D(HmM=eKWA92Ioc57^hB zn&tdV>Mmi_agY|lhT+;*2Rk%(NWgL!PX=fXcA+z*9V=<1(5Tpr#d{>mPUEV?NZRQz znSOY*MGV3&uSGfIc2upC=?9$*Q3I`dQCbqb^9b4{kok;CNw_k3(5FCgi(;^Am+O$c zZj(xQ!XTC8#UG3N1-*ugz0die3H~ruJ29%^>pm&8^FkKmru$)xC(Nkcb!#&_{+cG#NMEP*NDN74Uy!*H1}{jooN+s<*3oGu4tCfS z(r#Fya%0eSy+|&t;%9c@h*8Z;KmPNXx-PU@^!~LoyHJ)?2NVtInnwE|4Iw)-d$MMr zXDW8IKN>9;$0V&|)v9|g_kBj&Ec<}SH5>S#!RRj@>3&G7z96w5!XD%Rn?fQ8L8znH zRAXFH$5ua&P3-LqfJv51@%x_gIp1<-_w2Ir;%YS@8a>#_FbbPA#;L(>zmmsp(A3yG>b#i0jaduDg~` zfOm|kY*bdOoHgZ7ZQqt}8}t)*CL0E+`%5K;SscnKcFC@;*=e>I?2HFbYiFK50QETU z<4eZ$za==veH1{IH97CTnI4Bam|>mER{#!Ru&bcsfPp^T8yLU_JHs%H4M_((`V*NN7pS967 zJ9-<8PH7!1Cv0YC7{Z2v>EVE`NK8qk}MBc6RU48t%C z!?-ux*NGjyQ?k=Lz7K7p_O9wXwcNMt=emAH?l`7;)z4<@z6A-WfSva2e!dWwbF1V0 z$(Xv|?R-Do6T#0t^|wQbgK_9I_jls~@$Bl2e2(hkmOlGevge5%#=F`u+RuFN4Bz0% zhy$fM)Dpp3OWvS8p!YNH3%#GLoJlrBFPr)VRXBI1VW%l4YIa5tYq_TbScA>W9j70) zvcS*=gs<9%YQ%muNa@wodbJ=s10G;AZ-t+uO52rNo{{khOx2&r*y>TQ7}N`X?mA$O zJ;z}LR58VvI>tFdeGpbqfD2qihsHz4&f0Ml>+7{`FSxvE9Z>hop1SW_0}Whv9~FBS z^=kUlV5FoA(Hql?|;CC~+pKd3}FU`kQ}WhV83cdpj@qL)5fq#EUG}L z3b)#8xga@WhBSz=tD6aoE0;^qYP`bwYuZKIrvPy0Bmq{~S+}*-Kdb!;wCg3ca4SqH ziI7i>QV{*JfKYKsLPzA#lsA*08-9n5x13uO>ovt6q7|POHGQ!!+d*hDu5YDZvHpa1 zZ|RdfC3?~Hvr%rabHtZ$g8J4}TAv7#2Px5?+J4F&*Wxy! zUc+@5h7P6f+uiQZeR%D~+3nrgB`1mg-q4(tKMsl5Bzj)>pK>z&OU3W?4Rf+&@ zRt1>~TsnU)Ufn!Dbz6>3Uz|jLUOInB|IHc6-<(}Z-EU4@mP!5#2z}Zf)V4Ixp(SZA z;`C6xq(#dPG1MP)ky(!@{8__D{Yt-6BHf=i?T5njiCx)iQej+c$r-Gs6%5t$TTn?! z4BBD(88WBlm{eZN38>&T_Pr;8)? zY-{e<#EupGmI)I(pxP)QZM#;^rhS@0XPL5+($$K71g4uAm=!N#%WGQ3q?V-tze$n;D5g~S_3bH=p+dAnX&EkG&BNeyQDq1;ek zNTOqXKjWtpFuc((PUL3eo{$@GYn(qSMWkyr8u~-^Wx1EQUy|4C2w2UkHB~>;2y0C{ z4gDqq_IDSD2BI~fn|VsUMHWdR7S-RuvHtY*SZfQgae z=fW_SQ?YzS-{sqZ6Omm^?7${hp2Af}U$LwOG_;A)w?fqOc!~Lz98f6z2450)2z{wm z#Ia0I^et9Pl`x=BantxRs&+t2-m*iW_~N{VtCoriVbY|v_6qu5Q{_nFsr{c+F4Zb- z@oTJKX%|qnpIQN=<%($aLsAj3>Q^MM*Gur*f^DZ7C|pajWX^{;D0kBpq&W%A(sGlm zn3OU85VXxHj=BZe&{xsKzUkzlTwkL5Ww~zmSZL;|Frd$b!d2FqpM@P!WxPIm!+nUu z5Xeo%D&>Lk5S02L>@RVc^i=f;@4hF*z!SHIDyQhZv@D?7jw+&lKOKr&qh1YvHY41H z^tWOibgfPwYx&SS;#Ta-cuRDMt?SU7*d0CdPSX9^tF;q5uz~+mDf(d+fY2R>`UAm+ zwr_oy8Eb?*66Kf@YV(tg@psjIqd2M)yZu8|^L5+0Y7d&N_w)_fce(GXUdG#@)xJ~Z zI|MT$HEVcF2QHh9-9XY~+JJYuJ??!!QiPFbrdW*2J!-Cw}Jo`XycK^tUI}b%las)P29zrRV3Oj~j7=~dO#(pR!b~0lVWI`5hqUx88 zinqq-MGq|*uwxj8VHk#C?1Apl&)P3fq*vPL|3l!pS^5D?p0-u4-YHT$hG7_nVHn0f zsNRY-48t%C!!QnyhG7`S`v6_BdmtQtc+oxj+3|$(b6)vE*R=|E zJaWI@xLX-lR&Z?fmG8C^e}B8|y3cjw(4)_N)Zy>xe|3+4+CBcslgB?1m7%g41`${o zd@R$>lB-8XsZSRcc0|KYE=Y2@$E_)qD7>ECI2^b1;k z^@f&Q*728Vw>|!gf8ynT(DE;zV438t)Co9;*d z-+l0(-Q!<%AN*hU_>+%57Lyqx`UdO_GJ2;Q(+?w~`^H!DzVS00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D6MRWTK~#8N?VU?* z+ej0~a}ckyp7m_92*6FYUi$z-j*!K70U^t|h2Q{lfH{B#3DZXm+tP^5BAC`+9^m9 zFx&r5FXM$>g_}a-=Pc%aRS@2q-auzaZ)l$+o}W{iplp}8F8ZlbzA|I}-|ddi2mHgy z^MT58!G!yxPGA(Q3`Bwdkx-Q7hz{>w!s+2ay)9Q!xV!s)_vPoeCz|{sv`cpM5`PNM z`KX$iX^#WiDm1Zf(J2X8qO9)!y0I5vwj|%290h}-vYhhSYcXN9IvTFqexy3_6EO1Tch)H_gJp#Cu1>AYT6@xq`AB; z;FY7j+*&l2G%=$?-7kF%^?_cg?qIc|J~&_Z%ZHx}*z%5L zT5uP7O6Mi-cA~l)ZT4~%nws|QR%jn*2ors%5=6haUp)249~GNyo`7wx$mb<|xGAbJ zumPL=JA31fy%lMdr$k={wnb85+VPwEo$1IC$oEZr89JDn(~kCQh`mi9<*F^<$~$px z#y8Ue2X~i=Ss{^0yQqb-uZtv3d|!xLj>cxEE}iZt`S7fXMfDN)@wi8FaEsz?T3M|5 zx;E~c37~O#AIxVkO%sxgTGfZPzAd#mJ7K;){l@XHe}3Zl?di{FD(l-2BeLmp80Dmr z+gK`XW*sm0Z1O=73Dl%)SVo#@W{x5BdqK-b=SJqU?QZ+EP0Z<@v}TlIyQQ^qX}~&gca2I=;*l zli$kr6yX!(<083?o;3NzPrB5rOFwpGf#ul_ohE(j^^=p|+LCa9ww2MY@jggL$iokN zl4Cb~TVhbjR^eI71_PJY_*uwLm?GbP<_hpC^dr;BYz%yv72Tc_ChF)DGyM522Vy0_Iy zh4HcMJ|aYJU5bfnWhv0DTGsJ}N^TRQw3*9Z?%Co)LZ*b>Rsp+RG+@W_R)Hz6B}Vd0 zq=sPGKe3;Tv2H6noNpJEPF?t}_G3GP_m56~-OO}AH#`5cJy&mrZ0`3wws`m3ak-p$ zc)XI^JeqAbg$g;gFE(?Gz%2wc=ZVP9;Qce6`QI)7{`;T(Q_*1H7?Xc6_=tQ0 zT6{4}U!`ukx7@imWDzHHUUE+x)!k~dldIU2-m!b8z1`aj(-elk_r{4jTel-;eLS-B=Of2s|6NIE;F_r=epVgNzkt`z_O?lK z=E?8IjnT-+=9)g*5|_clE6msdp7eaSB$wlpSrlVS)0QvK&c|{L%-ZRbC%?X(;^uhz z?IccL&IY+~zq1%9uA;u3=zDM3CXv1YMn8(>L*&3Fw%4ij-Q*W% z6n(Qb?fqhrec&Y0`#XLo$5*q>Ze7;o6g07@J}zQ>?D#K(l{u8MxL33lPkxQd`@lZS z`U2q3gj&r|$!#T@J}nL3|9-%6qG|u-=eK{pqkK=tg1$|k!zd?}+zn1|)9Yk;iPvA~ zLqbOsT`k9FW;R!)!8|@S#=9L?PJh6ric$L7`_{CT>=g46U5gUe3dTTeX9hp{Qx?jE z7DiJr@Bs&LrfnE-sERn6&l~+zpzo;SXm!0o+xeYNtsm_CsfVJtxQv1h8o^v@>q-Rl> z!`F8H`(N}`-%Mna%P2?=%eI0X+QgW(sb$ZsJ_oj`eV$6LV`A;}IZdD?ay}C9AX6($+oBS|=s^#jdG4oUHW!p_H@acxPYon18Z<>%+SyU!1gvh(*Re`M3= zFv>|Omk@+(v(#~I6*&1K9}0H#2b`!-j!#^mDol9`o#h}IEXJpg5tnn-Dn{w+I>y{q zwqu=6@ZE%q5l9&ZP(@mZ?97lS|62Uh{#&Ng{fk*!_a$w;AIilm=cJS`$4|4RY;9{B z>-s(R?Ofw$p`GoMe=z714es?5QoiYgflS|3N-P@JuC3K|FU@rqrWvk1`M0d%r|`E! zOSps&w}iVF9eiZ+0E24+2Yk-fF5U`82(GCf8&VmND9` zZ|PgpR-E?~jJ3GC2Y&Le*(D6}(p_Y{*~UlP(lr|~V50fPbq zMaSiK(jaRl9?B?C(>y3u&S;^M=UuU|>2#dDV2ck1PmkR8wDVu^V8~9&TZ5ncnr9|8 zv_tz`U{1!^v}F$aPi=&@E*3krAKMwcd-CMR4wHsw8gM+^rQ8mwm8_q5HUT1eG%>uq zV7(6xT=kO{v|li%f;PQ0r7bScM}6}mBsqO;>aB9h-bY~w}Uw3>%@2JfCU z`O~eYjM3tmpKmqOXVKg7x8hvHC~YR5yXEE^Y^Ib0lx47Hn@L9$_)dlGm#p=n#6EDQ zKM{q>Tn71r=<@kF%Yjb*GS4~2X~fb0LEZ8~ z_A=BbDB#+ez)mP9dUh6&qm1Ld1l!wPrk}N{#Lxw#uRDf%($!0BFN5+2e`~v z;peK>7RL2*-uA}7w3lcU&FYz-=j!*0tMyk11?E^1A2263dmwK0jkF!)7WoRaI zd1&R#ajIZeE<_`)aHfd7pJ=BKraiqXhpdXPYiGuO*N#B7qOj?rT@s5f5ia1UzmD`r z0VroEgPgm*8Dm+y+=5r*6V^Y`FWx>SfCncDaKg^&)o6Tn#}$a| z-!@=@OA$IEhbB)p(zTx_*4JKKm7hL!EBXBJ_>KR}!~N6muMf$FZ2WfV*W+Jb^h(Sc zKRG|0ame~B#z~}4@{;UD=x5Nb*{a4>#I^F^sqE)dZs4TLnPMbWR%e(Wfc>kbw11;O z9+YH%dh?n+?&WPkT(y@tr#?aciv6+sI4c%ghe)nQITUkfgFhSTZv3s`Qkc&oIfK*E zaf>8fK_^LJ)((Tjx-@4>d9NmLi6(kl$rB^mNR&ojfVh&bxnat*c4ny}3ga#H?96uz zOX{COc}7%{6M4otri;tl$$qIMWB`>>jV7IxSfSL*XKUq*Wy5GCH89~6qPSSj3wmn} z*I22v6cG4d`B`eM;rrL8Ki__Q|6J|swDJG`+k_8~_rK_`zr?Kc)8t8_KRIt5J4Pj$ z*T-p{;!?xTLOX$e$c+87=Aps00PQ+#5Ur~PY9%$_x71B@?YA81+{oP4HcXrFG#s!CN$QvkY^a%C{bTSu;Erk^D4XV(n|7nion@KyJWE)u6v>9_V9NitB@%x5$5 zIIb!R&`N4bgO$FuYw<#o9h;^dKQ3T}qn~(S*q!>uK#qhc?DyJz0eJMkx7YjCtNu5# z@#EB618emw4Lb|%1o|N}_Rp7x0MVM!t-R<_)Lx*kbQ=xVPiV(>ZL7$1^dah52=kUp zppL6!9~4y*qMD?pQdunIT%vm3T48*dKO0%N@q;Z-uJ)26Ib8LChb}Sx)tP&NQkakA zz`yEAEpZC{s~!>OHWNPU)LXhCpByO zNz>IR^6C9w)h7sf^1-g*lStEBX8*zt7K)_^UU5u*Of)<@i&^^e4;xILSd-70--5 zV~xA2JwF>eqRJ$FjGje#?%4QaR(H3nTxlJ9uk+&4ajmug*7AG-e_mtb9}EVA!C>%8yz7m> z|5oPhQM>}Wz@6r;@~`|x8h#ev4&CC=XJpE!9u^Bb<~+Y_^{y`g3i&L}j8wWwpbWc8N%_ z9?we85&BZCeD$=sL4oy3oeY#nE0O;v;_CSm#e~}l-}dQtDIgpR`&;}O3`Fq=#yBBb-Szc5JHs?MUpTu>^J7vX1Sg4FW^Wk3HD!n+|;3cGYcvFjx(3{FmT=`g1TC3AM*eJ002ov JPDHLkV1i#&ZLPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D5)4U1K~#8N?VUYt zeOkr2%skI);WNXBcym25K?aO?qFnrcYt?*2-t%d*eksGAcw<`p(RC5 z;{DGLS(_RShr=N?rYYO|?ciJh4C^#-`EWWNr}-2PBI9w6k0VRCkTl2R>2%`5VM1jr zk4HKOHc#`3%UElM!y_J&z8H*9Ej9(tkSHgb6`vV!0X&3?U>_>NKbC_8H2Dxtu>>A} zy#4v{_WJOQR-mlq;qmd6%Ybfbjtw8lz2l7i1wi@X?Ul=b&(@3JVZ8_**8K@Fg>1nI z<8&N#9?|6*t_8V<)%4?uG7$d>R1V6FC~%=R#eY&31)(@(SXoXA-3-lv*_7(}5S4yqLCN9Y&We^zu|V zlTq{6EXQP1m~^F&j`Pj)?@-#4vKRFXUNTdg_-ns&9r#Rjt^=Q`&J_@*=}_)a5aSV6 z8C>91wq9%VJDB5foN0p^{U#RG3{D6D9%b5_nl93w&;ob)l6-3J_Y6e zVn4H!+Z3SnP!h&tyBCH9fMZMn3BdcO!g7pZtHp@a;>Tj_xYvcwwZqdCXAuUSLmvZu zIwm<3c(CD-hbPG?^dp~ze`Jm&BG=Ot&~3pZ)S_D;A)sP;8d4d6H@{#fubbB*ln#v( zJkQ~jP}pIimcS!stLVXHdcSNwIwm>>ooU(c41quJNVkvxdL#S z%JPLTz9pBb<9sAMP2)O7KRV8r!C&;9D`3}YI^uiVC2Uz@4-0PKp3`~H(%!1O(Pk@F zrm0~nwn_lmZes@A^lS7!Tf(*M1gx=JJffP8ib3&TkJb))F;BfwjJx(qtdWNikKf3H{POh|j<<3=m|fRo|ykjsXoj+8>RU2Y*5yOtG^1UdUz6}#OQ zbym#dkmAWB3n%)$Z{|~&Hc|k~l9xJc$)_^k^qkOT0EU(eIDuOE^hj$U=s7&sVQRFXeI0Xq)vMV7X?451P{) zQmCGXco-XE`x17M18_s>MOdz#|(&2Fb z_J_aAR_D{3#}`t`Xa-gj63rvtrM#zxz$TPYPD072P5Txy7I_>}JOK#8+3q*#D!;kf27Nf<2a5%%BNv19pC2Y9P0VhaUAC? zdVR|8x$un=ZX(9DsxCW@<2V++fBHA`-~awk{({JHoQvW4)NveV2fhKJxCcvbs&+nA zZZp=OOshDfv#H&HRd=h+Myza8joZY-RBUJuL%aL>10|i?&DPjm;zl{zToboT_;5{B zWBrjV`8?J;X#TLx1}r=64-u~B`^#-8^{Dgdx$db?&z|Traq_V{fad+620)h;`dS+3 z$Bc)Gw`ysZcEBGz$>fI#GC{}%N+&C8of`6cWA5wLZOBP~JkIpvAoDDrjm>D}2r;$9 z&*B=RU#V?H`}~K!87pV4%w=kR2$c+Nt{ID!KIG6evXa8-2%fZjmSE=dkXY2mnyM{b z?p}BKSaHa>CEM>Hjr<5+Iv7-Y1f)M=|9;`$2 z9&c)Ks?E#cs=&)=z;6vDn@AWo&X2VmkX)&C3VoGN!*4h-;RC?Kr1Nj`f#&N8j_I=v zbT-*+)+J2}LDiR_4=3SL9FI^BPG}u2vwK-vc0LXL(ms&SoU#G%XMkFbt1`PD8g_Dj zG)W%);WD%vVrBKckgMxm+tfNqo}={_`Va{n$Fuo7OlX@Gxxp01(8t3_mE$k_6@BC} z)xK42MLXGb2SW%Ys%4C|ZO26A(;xKVQ~bKr_ZKXGe+vIk4ax^D=VbGz!S@&XKAvix%Q9UJ2BP1*Jn!#JI)Uw|v(km4EQjMIShEE@TgFLNnyBtuv&Hs#Ye;qtkg z%0Qeq&`o%tKF*~M!M$ZT7U{Usj|KeFBs@x8Z7}9atyAi&e5#3a!s-{5p%AW5MqH}v za)HBD%A-C73I-eEgPD$G(w3C7w&JN&h)5pH>dr@QYvSJ`#MibIu1I!0H0`9a#?qlO z6gXCST_aYKYq~Z=S<}^rLd%B)&S&~kjiBcwV%G(9K2-`Sp6~`FIpKPB#0lsUNElL}n-zK!` z!kPlhXd!#eO#14D2To`mE+-#iTe=QE3t?!q4>SEZqvA9LFIG?C@gr@KdxJeB#h zXWt@@MIHwoPt=KDy0vM)-^Xv7j>j-AG`^u3lc#&=+Bvd(x)!~2`ju&1X^UA~_dRW0 zA91cTsqeXbIXZJKWouj8SU0}5Z+p#?4R$!SEnIMYEvk6x*FS@u_D`4bu6*kJ)3Nf! z`W#39p8&4E@T(({MX5rywYu)5Ss#Z>6Behy_2*L<I~UNp;zA9LH&rtxtJ* zs_(H{Wyf(Gr$Fb^!xZna0$6I59mjE;CRsjp9LI4su&e! z^g@WA7n*O^5(no(?0#Ke3ugXt{z2S4{|-OW^7fveGtBt+#h0g$wOcO+!x-oEDgF=X z1u5OD1p^}1EV+UGvNZ6Q9g+E{e~~#mk-RkZ5$tg3jA6%=BP}}%NWHZCdkMC;yG%c6 z6@kGM5WeOdYLVtum)fhf{jxwi9S`6%odrJ^wYHaAk%jRQ*6OchY4>neboIo~y#p4w za2ytZ9aAjnV_I3Xldu856J$2H_QZ*zi~G6TVJ zysRb9&C?`O>$A)#p0}?;r4I}fej@^8MgN+1I`*4(7_4P= z8&Aq9wx|N3I&6*CjK8CbF@rLYV%Ih!ENhoq&}wu*`e(*P+s6RVJrx&i4p7eR;Rph*`S>dJF()j+j1sb6dSL9F;#wDj3Np9f^f zsK1SQMfxko-O@*SPV&O^(`Z+0eX%%b3j67h8aV56LI=0B+99h}(LS3?+sj$>ASe0L zx=zXCR@@f!tMn3%aZDh7+3Q_%6NN3-4xv04rLKrW8~j-VE(mznD*o}$#~q&T-~Ond zm>lkY(IG7$?w(({*2B-Io1g#;w?D%kST1iuz0_>$0xxJDgFiohz23@Ci++E)Oa8nB ze^CGTJ<5OIKXcz-&&C7i8YW&Cdc3Ns-0N3wK3MZpH(AVx%AYeV)Njl? zCDPZ|)_KV6yF4{^Pn(xZE=)(3oPfhTEE1G>213Jyt{no>1zq>yP{`%2ngBL@#(wlf zu$DW4N}mC7AzibqPthGTdUP;omuE`&bc}fcL z%!=hHphh+$A91UuX&>J`6qSmW^23IH0!*6~(_z;>k=seM5pOW&WrVk{`7&^n3j!d6a}_l(+0Ka9DI}t$v{)N2VP^-=t%Izc?7M zR!6t+qD5AFfqtObsF@Ez+qNsIKhe+!QOklEw^Rbkel_d^S(ON@MyaS27Be}Ou%5P- z8DHkldSEryTo&*uO|R8@%hL@Di?TzjDHb%DKY#A=|MnzA-dmpTwlGSZ&B15q1Cz~y zcvuLx045fOpBjVL7HpR8_Jn#nIFb0gpU&)8HvW-&|9-STggL9KZpH zCt^!9jr3+ai&LACnTvcuPS=nJ# zBI?6z_zH380kyGNrQFGfK&=nR^^($Lq^XZY_ahR6NIENB&e2zC-9U33RYc=;+KZdf zFTTzVGD$2!3cAzXR^k)llUpwC158%G3UNO)DV!_S*mvoc8$GiJrE>90P z(Mi1H|9p0N3T*IyE!B0=n-Beb1>*OQKA2f*1U+KqRAO4|rOm|8X~aR;cN#q#v(_`Kz*<2a7vIF92~`0~_o9LI4S$N56Em#4Tb9X~5EjPx5v zRGiZ1e)QI~C3hUhaU92SoOQI8r}UYLDSlSA{ZUzK8S2YZ$8j9TaUACzn9I|&!h{89 z{T%`NVZFvz2Q0bcIF92uj^iw#wLA?g(r|BolI>7+51*HzXXK9KIF92ujIF92uUkF^DI*#Kwj^jAaSK-^Y{{ycbVg2okTI>J-002ovPDHLk FV1hKEJbnNG literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/GPT-3_1.png b/debug/accuracy_tools/msprobe/docs/img/GPT-3_1.png new file mode 100644 index 0000000000000000000000000000000000000000..d633249c2ccd9b365c825374fe5472f33dd812f4 GIT binary patch literal 7898 zcmYkBcQo5y_{Z&8qoRYFrA>`eHHw(ETSVIgl^wYHpq@ZB!`}b0! zpzoY1DEOdyn(C&ZcH1S&>GMMzF((3s+haW`Je~FrEMt1Fqj}CLzGwq>uh1lcs@OYw zTyr|9Q)9Y%e%Mp+JJ^wfgU$XJ3E%RpC?MKD9(N2jJK`@gFfN8H(ptpHOeG?plVO?P zY1GUPcZ^4@sY{P-PEE_lySAFDsDDWq+-te*G<6JOEE}?+h?ts_a&gcJzT2p#`oXZC zGxxfgMc? zFyl}?q=mNzDBzptW^<*6blbrbicOI~~n6WaF$lI44Fu?~$3VGI*F5p%)f@ zAlu$3)$I9cyfxxAll9l_r<2NL>F<})@05B3g$?8O7y~JV5(Xwug5Nj1}4OefBF2mO~Qim=M_*9FQP@NCVC?MRqE)niBqD? zymRu%D{HFnFng~iDgPTKxA}k0YZ+5yzv=8rX-S#V^U>t^?H%gjayssgPt#Ak*4Q(XM?)aW1<+@hQ}nHG;=(j-wK`@v{BNii`h1C=c1X6K zw&p-S#IVK%e3omS)iZdgPpv9TG5U1!Cbx8?HSdUXB7l^%V4tC?HxzE1d;;Oa^}5G` z^;I0pS3{jOhw_Py_P8h)^@ymmv0s*G!Ifr(3 z9hSVO33A`KGw2EdGikTj&^I(7SdP(nCiNkPu>~4T!DDD9tKhCFktF8M!kxQJ zp@}tg_3tbZaoJiQd&mD`WE1L*DH^zhl=iSxG3#JAN3Z)@X_H}rTyIf56LZPNMUlq- z=D)88hL3$jeY*?f@A6cH#g$Pc0fOC?(7i^BKFd7&YP+!j^!QcG+vT7BGu~%_K!-LB zCzvEU(IsR|v6IrRG7`;{E0|ifKquGxCi1pH+(Gu!KuR9zwP!TicNCS{cW0S(?Ik@k z(;rc^Fd~YN4-$DPIFuIf1(&imD7X8Dd+4xNp8uUp95#U!xRv6^#JZhE#!m<_Pdq8J zQM>kanTDLK$i>|h;n4Kir65><64Vkw8F~CBS6xSzacnf8h3Nr3@}nnYZM%_93uBorJ9&WEx#z+>_Uq1!L2yraC8 z#bcau&1M>Sk+kksTUZI4Co*MoZzx>;1&f^^w}Yd%^|1&M^|Eb947f$Q7YOzDdzWf3 zh!9QLzyFL2?_zjkyR>vmux<9L%b0WYAR$jHYEWwFN`);RD|tG{CWhdQj_-qQoQeFI z2t+lpzM6bXc{F~KJOQvxob*UU=!Bo_UuaKW&8BvR*ZjUkjy&XR%ikfH$|R;0G~64yg0L3Rz;5Pw4Vpt|X{5LwC*@BdMD+j4LfSR8EcB0THEt^YOH zGh9CA))$>f?Da}t0d>r4FK&be@+^U=4dDzojw*QPyf_kO^+&?sNgGn7u{(Z{T6)RW zC9*hl2TWS2nKJ7(`wx~e{1ATlMc&&xR;p8nCaKqA58pVfjmFF}=$;Iw9Hra>!@s(6 z>vXdXmH7PF!=r!AYh~PW7%&3ozi@uu>I58kVi^y z1$;oA<1Q!gz%lm(iR_1jrR-NjQVrd0v%kHLQtlg@znlrMLN-+I|K74M1PruXezZ>d z-c0R2=5%Z_cmFLWSzC!|I-}$?&_a(htcl8S;f(kx8IpNDMpUj0#!1tw<)6M8;kPJS zJ+-9wJ7Mg3yZ@N7a^R*>`oNWGL+~0!%1FnMMIVOyh@Ga_<>n#aeL=3>!%-#t=J97t zG}e-wjLCxlQ5HTEYvOD4V*Y+V_IEY#7H@jz6V`x#NQ*GG%mhToQi;L`ELh-!#okF! z!GON>_S(r`(;N3ViBupvJc&e#+n3_X=}9}PD^5|wJRQ9%&AQ89rO(scGD*_HcQWyX zB2XYy6u-2yd>a01zE@_A8kv8qi~7n^K0+-W3k+$tnr>H0a`J|e3p-mR2kU^HkZdEz zjNOXfqRj6VwW#s{!&;S{y0y|8Uz>eP5LHb_0W!Hf98N+yAJ+A2U3w}LYE(=Aw>768ZhND0t-?(1AkzKGG2X7{&moTelDAfxUv-=%m&Sjp zJh^Vg8-6>^4s~+Dns5p(ZQrZ7u`cHd@6|3a^cma6iq#K`xrj$g;5S}-(eIW99>qc4 zy{)Q>gbhhSVK$NSd{AuP+cd7|1!_4Fv>`c!G~In&b@4qU*Kd*cf?%{4CufnNL+FD& z-qpDrH3!L%vS=#_eb1{az*#ivwW4V}YgKaOGb=UVG`#8D4#b+N^2MzJ4>U7$D_V7p z7#x!hzXv;>k@gP7kTAB@D(Gol`JW2xzJgA{v1k}O247XzrsaD93 z;wWuC4E~w$U{#qdq_8#f$2hn}rHWs*VD=pnF3LSTsGr~LBW+>#dPMBQtIa1;YYe=g z=cba)X&*EU%Caa~4U7uS-#UVGM3|i7Qkl>Y%e#$=UQ>dwXFtM)|Os=#2=DIHZp3hZ7+&^4)Osp z2{ns!GvuW2f00M(8pB`#wokY@8A(yOkZsCp^Yg8S1) zdil*KH}I7_lFFc$iT$x>dfWQzZHA~^Y@;wA?eR#j($^LYW#MO9Yp4F^C(;+WysnY( zfKA^bIspTiKNaWY{Zqo(J^-!OwHZMY0+vK*;>N*KN6KeUIKOpvubG##&+swD2G4cd zEeCstMTAk7{A5;3+H7;9kEM(LqSDLKbHEemd41<(E04`2hQ!(fgC6`! z{#jU)VXjG+p6qm#o}50ZJ*^X0%B!^othlmCa~hHb&s7wwhJMiV3Q$e-keXDm52I^f z%45e=&-vGt1QWq^wcSyCAzq>UP9+E7e@h~qXs&{^&CLtxu9x%r9|Z4q&I<7U-0lDoOmU%A5%k(!?g1*vGW&EKs@FN|bKiXT9JS@laj zzNl2?lZZmMXq-LhP)73lf={%n8fNNC*tp>(3+somP)OO-)mK~Z4*}EcWS@`IeRSZ7 z&EiO7l5EZvjk+J#&!ZeZ8i@0n3qvA-)<;1J$0lL>br{2d2{}z*}~OVpDkd1Vs`7)C8`_v5R!&0~pO^LLjuHp9Il&m#O2OVk z^<01}+;f&vX7$N=c5CW}#v87TxFaWoR^3{|edx~PhMolOJQjeencI|h6zDIQ2&5eW zzG(3BhYf)F1|y@Ov!n!_JF)1nNd@`n1BbqfGjEHaRFtpX3~iE-p~5fD()C$N=R8$7 zA%8M9V~tT|5|<@`D9ZUu498-`@xU;_;mf<&@2?MwHeVb(j9ZJ}N|`twhcWR20$X>3 zQz1!N;51;^UTAHTc^^b9Mxzd zrMyZE?V>x}m_P@XdRNs;qq7erj?4my;S6z0Hx(vH&2pbZSD0PKxL8#Opd6CVMGkvO z3L2xJj?%RHrq-|Ps{;ZIX#GOOZRA>X!s$7;^5P7hxVEaai8Yg#*;&b0ql5AkBYuom ziTu)vzV5hxBAZU(w5WDe&ShDoUxK=N-m;mTi+dIv8l=qA7klRM|9lb(>^t|+BBMy0 zo&FALQ2gw(nHzGkY|ECfutyw zT(;GPry}U^9sl6@WYkQALL9OcedJD?JIxfDH0sLAi4m zbA0iyuD3=U_WVsdAoN!ne+q&tk|A{lo0RUbggqZ8f%x3KP`K`Bco{O2w>T*uf*uaX zGtf0h`L8{4Cy50YmxrM696g_$R6vaLUhm9 zZBQOHtu2TTU#hrVx(H-bd(gku35F*qMu$Zt?dlPf*>~0zqRCGgjP{%q=f5OwOu4!2 znDQ>0!d3%%B)5(u{FUqjD|UxtDs%W{3VB*0!>e6?9kor#8%?y}rFNvk_Ch0M)+Bcy z9WrkzkmbKv?jNhQyv>1`!{e_{kB-T zUv}tQal{1EC7S9%(-QBZg?TM7IQ{$nJDL$Sq|1-qyqe~#(rw5UXyCqE$1Zd?aSa7N z5QDsA3n5J|7G;El@ar)hfKNjBt=3l44~z@lB@w0kvlksVjx< z`OK8{Q`&mfDKn-y!}`_-#RoRzVILbiOWMjG_gf+B<6py(MaZ*Ly&IJsvUMXRQ=R#TgEtS?IsHb}HMU7nHwWA=eCcNufEHu2|M^R; zJ{0$5vv*)+C3Y52Iq_~I#0Gk}iK6H@w3F}a-@Y4ZV!)$F*g`oe#bq2+n0VVYCW?_(Q}~`^J#gu*i!tUV4$6H?|Q{NN0h(%MLPl6c~s(R~r*L zs$V#>iB?}44w$bW#Cs3)e$x)?ijs={^pp=Wl4^-iLYnSB4|q-c$SPfgbencoV}3gQ zFxf~ZyEPPyhsm4IvJ>~)wwsU6e{UmN;Y`KxUyAO%E_$gTi`IXvA=ev=xMt02r7idM zWj@NewzCQEA?<%=Y8HSYhgvqQ+X^2Qk8{a~hA>P6`*`nqa5LXs#xLO2BsEu}-($pE z>jRd^3*^0_UC3v!mx9CbLG?QoDsJ8mwKrCn5zTh5Jo9s^CeqDM8O@4xUNskLW6413 z_SEW+2UKQKGTARPzbxe?ZKuxgC!2K1k1S=Dk}o5jm9v7FqO^qc7){6C)_9BR@)2$j zO5^>zEy>OKUT$?!_9OC*u+M!i? z5!p64Peakk!#gB#e&v2vsb#FY{^*JawOAzrLO`@P{hmL;{kgby^)ZD1f=tFZmw)=Z zMkp*2Ice-!d{f-HphBtpCuA$*{Wl{C(Om*02^wA40ZFjlBD5tFoiLXvo1HSmpKB1-jarKTRJb59id zqD7EWO#eMa^aJ3h~4$>o4$UhV2VS)_q<6>PK<27Wxpc{GO%s zNKrSpBXsC*bBv2{cROX!y`7tr$Yhhp(w{ipPy_p>GUf32ebmuWIq?p37T7jyaV46% zaYht+IXXM!cgH^ahj6j2qJ^Jd#H1H^@yEmQ&7iTcaGzq?J9ods29Io!rDO2Ui(DjIv?DvkMenB$hV15Hm~@#2a12oL70WJY9{q5`L@kL+JSld z{W43sV(;sN+cRH;?@cIPbeFWk5$P%~Zdnm+%~kK-J4BmhEBK~BEUJUMIYMqSUYKZpz)D1GJR0!4LtMlCTbHE z_{!EYUZn=!GvdPEWY}xg3e_5u#G*fI72s6o_KVvQB_E3|fOck1y4~gq5 z-jMf8$LlS0OUh<+F8VhIfCdjl?Dwv8`KPJ?{5n5xIOPl?zb&uNZ36#uH0&t}B%OC%p151x+8eyzuKXF--r4BiE6Ui_yeOdfdBfBA!^AckT=FQPWhxeU~>>$14 z!cm(UtoWTO>^pm2HSV&{0#QTg+!5kdDx`OCvMD|wtGK)JJ;p5#JR#{jv37m^J>L&Eo9-e=Kk>Yh?#OkyBITaVS^8opY~r=gZH#pfSx7 zGz}W0m4N{K>{9MZ@06$Xx5i5(_rXh;jXbA&by4Nz z`qwtLAYQVB&@tFX_0}M@dxs-lLNTtrmc^j|Q2@lcMo66{ve$CbucT|J-lppSo&LW` zuI8~;@!4D0q$+tNgSqA0sQtp@tg|U7Dp%kQr#frZHA@}eN6+OyxL##PA3WYeKe<8= z%3`T3K$vZYa}D|(`m zPqs^VGe&YWyGx3`iBi51wa>ir#UIoHfBpY|o5j*g|D=Jp>WO9=Xp}fshIfQ&@N3>> zp4^E3@7SWwp6ZkbmapcnIh!*0yn0#ee2gIrQTSyb7uUu|P>5UNTMl5mJe%Zjz@=;9 z%U|LT`%&=XK2GwX&prH@wn2e@AYM<{=2O)>eXm9U7OLJ|5qzo-FeiGhj%F)2DVlLZoc`O_py)7Sm%>yU>((9@hN*a4nIrU%7 zJQb{WIs4bwq;dO+h`zv*fF1FFN5rN?g+h_I%We@XzQzB*a|i0;MC*C(6>-V@rbI10 zHbp7?dU1}!XwE`9i?e-UmUEG_@DHOm m33+S(2hnj$0vOC6`XxcK=6za`y1)0)6na{QnspEDBj z*@flh=kNcXH_o|p&U0tZGjpH0GxPbLd7`y7l_^M>NO5p*C{$Gxba8M9Rc@a-iScl7 za0|BIuirYj9=ghMIA6z@w{IN+TUiZR9Gu!jvMVdXTc5;L#mEB(hqCvdf;;F^`UVGw zaZ*)5_LYzMUbfV`pEA^49!bvGV=u~S1po8|krHj|VsOX|zCyyoM^QAf1S*>wQbF`6 zi>_zFNgSw5B8jZu@{jhuo?DFr1~Ql^B752SVv_YS_H{VteH#l|N zEu}jOD?Ll5550?Oc`f~OUsnU!%lj~3zt*?HZ>A_gJi$P?L{1ewpql9^23 z%SiVSR0XC;aKzSH-}79FPgeulb6NFO;6|a@r(AFufyX2L<^x;&Zeh87Dixki0GX=5 zR2&PGMPaELPT#!X^26!*Ut-^HsFV0864o$urwgmpjZf2`RnOOXUpf@U$z zK8sRgk-e1u9dtC+#NB*yO@>D#_+M5}U`syX7kh*E17_BQ4$EeVoN&|L40N>br=CC9 zUQ@^^&ZW%BQqX9K+eLt9CzFLojGnB+xVSaOM!mawNDtV_n?miP(-xVwMUCq-n1N{? z(QSU~?ueH&X;keQLTRJxq#lo*1Qg5k^^Z%63)(f-w`g{^bmxxu7x1`_QWgmjruj2g zqxwJ)C0`El^N`d6S3+#S7{6R8fe zhNxj5c_=JI$eK(|`EAv=uLeAa*BGc>UA_1tKIZ?2dWpUp7hnTX+_YiIdMcQ~0aYCF zSpmb+g$!Kv2Xgq0;JzQ0Gml!sQd$Z_lV=Th$>C0naMxr!-_0d*1k3XnnB4{GJSkPnWg|J$AriKbBo*@oL<2Q0<4ibb`kfUOs{AnPI-rAs zFDzjd2?Pzl&jJvAKermwdBPxTAZ3|g(oL}Fr=7Bf-ZA@i#m7kud0fCLWIzohvEL3_ zpLS?5{=N1Yk@-nFxedbl^&}4{;Yh6DNWRzyuVUuo%4nu%iF}e3?qlGlhjD%A{9!OY zC`C}S&{$zKKu}*7ma-)&PDt%oJxW-jhLKZ@Y`J8i@hyF;8e2QOocaw8kV4k>xE>}&e9%e?qYi}8XNa9y9n0<} ziLQZ1lkNbHzDxVDioP~aJK@P%lgGnyR?l@i4u|ep20of}FS{48BWkiLy0`{68Y_~& zpjTC)E~Z%+l?!Ng=F2v*HaJSVhyeu!gshb=$a!j8F{T9<5_7UNW^Kp5;~GK%S>cvRd$69#%~TuB`7elpc_HKqmavSkdF~-^*E6`Wabh}&$0E}QK=~LVjmYi(z6W?fy-92>mJ~E>AvD3EWCCYogtz&hx{D~qn z^Prmyj`+OLQAc&)vZvlykH&Rg6m~9-Cma+i#drTF*K?}F%#_iOS>B#YP6;J?5@=+3 ziBPsFv!p_m zma+0f!do(%4xZZO0-JcIcA9m4zW2`C`{MYF3Be*Wa^aJ3Jm`^Rx@!JH<6MZB#Qoz% zg1Z?J?4%@#*RU^AjJ^TAhPnHk+~;SZC$kFw1_m!8JhJLv9mf`w_)TDPq* zT%h%m>ReuI;lBQO`u@fwHMm|!aMxxzzcNTBYR*0QmzMG~ceYKcXTv>VSWe%j33TNk z*k1kqhrs>BMtt6t22%`)%FXbZ{L=OMsz@W^_ZL1z^I0SaDiw2F~@RvPwN|%p2=l$OYNe07N;c3O+%9b zXFQKBUS7EY*9bKfs4Rh%;jVT!vYjGybHH~6KW{R!S?!kNe@0wryynKtPfG_H&uE=L zCxd^ocqjDkg?p@h@@kU}{EY={<-vf)MeTEJ{E>BOHS$O*<^&wYb4D-?uJ~y#JI#+R z9rG&;w`^hSW0hf_XvXl_u}K}3Fv>%^UdJW)_Kr=x2`n~b8Oo8yps&zGLJaS|t~X)Qg_JoJeOd1f=S+4Zn()~smL*j6ynukpk>VMm_g%94!dzW3d>H?>=VQH9X}aaoGSh>1;#o{|TBy_;Q35fTqS*zD`=W7aH(o)w2#O zRq)NyiQEKjEYoV%SbDHk$E1Fb==OsWgDeK#-%di-drI?x!B6Y^e_A(20VkO=j`yx> z8MR0{t)KJtS0y3xdOxiXIFhq;e5lm@*b+#hs%0VBBA&I?%s^{ov_3dh}cf+h?EqCyA-2bY;{GPkWPkQ9n`VH*@Fw} z3^6GbziYEsGhSVd^7gz+<7lV1C%*rw0`LIba(~cuRNSe*HPmb)QwhW8budXEvDGki zJThlqMK?j0HACL+0*?w!jAf5wpy%M8rKHnAPZ&GLwh8Tl>t=(ECJ+KeF2ar>X=#At zPRm>tiVo)YdJzeqG$vEryt*h0pOzhX)J47yocp8nXKq`RjUq1k9c6Fm3JtHv`BGEU z?mV*AfAw@}9*JgN3H86ArDg5RW7bZQ-U;O{^jap=^xE7nF5;3NFx1pa{!n_#Wu_mT7+VO~&adoBa?2Z? zu7$61IXX@dYqE)S#3i7d_IOwXW}Viwg1GVsquucuuW^a7O#>k)e0#HZqnZGbd1%KC z;AXPoxD#}Pd7f!@)lI(xx%>!c1cRTD0~w$7o8qv&CsQ?G#ao3gmU})w#|F9AuQot!ko?ANkQWQMS)lXM57Tb2h?%x z6ADVghoDmjfJ2u@=L$k|M8+u)hhLX}KB~x2fAlPk^)v4HK%YS(S7`wg*a}sfrgZV} zsb`28w+r*bVK&Y65FRF)X%eegcet)hNTua!HE%VYqk(Ei#!qyI3_HJD*UB$Kz-QN) zveQg2->#RpBXQ-niL~!2Jw!(N>O!hax60H;)F$Tk(}X*JRg8otC}HmCk94HY&la|U z1EOBG#%XaJC^OT)up&Zazb;;$kPyzl0CvTFp9R99inkUNEl0LoFx){d&S2C@OgDLoZtWACV% zQL%tyBPFubZa2Cd_8;CNnbYoyK)8`n$UbkYDe9kD^LH?v*V+frD@T&dG&!dZEkvFD z)*FSfya+&GyV6i3TLV8%<#X~dqdU!Xo+2%|5XZ~j!?`;UlyFRS@4cSuj%iT2IrW3yq`)Ea#ME8z_lA5<=()v4}{#|d@ z#FvLnuDn%)mSj;TDtr$wkjfJRYq6+EyoW&Dl)UQqdtvuv=!uH?#P2{)ao^t`SomCX z6NbC8spaK#Kmep)d?FT6)i~oVhTObA{l1{?;kx?0$l$O?mm;b_p^wdAHwF>&VktHp z;dv>@3!?aoo1R7cyIzD&BB*uY%$(@GjBFGl&T19svE;^mR`n*XlPCQ@!l`4pIt^&MGLMeg?tC#z13Zwp z#aV*K)vq_eWMcKr4LX?>@NXh3@4(nGw||at>AQlfy~kgu{pvnVUy)*aCAwP=h$a8N z6BagXUbV2-dwEK8OXRi^={ej>4mcU%sF;Ya1>=eEhD|e=2ilOf--B*ku&Zj*o2=wdh?sY=Jwb{^LG7q|G z1~u0lgt3X!%vxBW=nqx`pZ^|T+|MY6T&-WJzVzfS&Kw6UJG0nwT#2D&iBnka5@aF- zdR|wyu{@I$^o6D~=Lnq-m0j2#LG#JVWR|l7GfrYpf>8lCE90Bc(+iAR(pfi|zj&*9 zzzNaIQrgKMHq{Wb&(F@)+rJC$9W?fxl>87>4@kwh#$#)q>1It*uaoVk|6*vpK{xur zYtZ}OfUK<`X7W;$Shn-5+vp<1;2bdB;W!e$G;dCf15<7(8T(2^nkrQcLi%U1#>k)L zsKh++%Ma*gk+EpN{2szSPyb$(RSaTuT;<0CVB*PTf1cdA4q9TffE(8rb=IQ**6*if6&>FjZ z-cBF7pShtg_HFWbi7ETq*E!bXrmSFwyn3%e&q{^X_(oZDg(!9-DHJUr9Bi5&dzRGr z>2~_ORK27wQK@K7q9&s+i<_dk@>BUGD@sQ-sD`Ur&I9AYP5tK_zTZqzuN~4;Ud1kY z@1P-30<=vUe>f~(gwSN=<>G92+QZ`VgQkF8`~@+bp!?TPR5hT6YBMC6*BVW;k`kC< z`!1T`-{q${ZtK>rG0kD|q>w%F86bZK??`o%ngUVq&p|nzyUbDkUJw{#Suat#P|_<0 zG0hanaLDn(l^pt`TfAp1p`o1ZLrVOGoHUkH6VI&P4%|UT1t$vTPQAerrMIBPxHSQz z?$W6rhL6rxpKmm9O2OQwDi7?S1Ho#19ePYRflKg&#fnX+!Lw9_)>u%3s#1FqO+TCa+gqp3)71g zdiGJ?8%E-GXMmGQmSG-O?S)=zwfWC{aOBo5pmnHHnFd>Fl^a;J!l&cK`ZH=t06Dt+ zCfvS)i^mc{$3C6lpL_hWsII2dAANMp8@qQ>(v&c}qrd!MoBAyL8V93 z^B!3r1ukqCXpd+^&Z$;(s#rr>iix%1MNi9im@y6?dQo6X`MmqV__3`P} z_}z}v^u~RT*?1-58g@3K9dAwiCG{-+OgLVohUsJz5eX+&HW?eF3UfN_{7G`h zkNG;A5$E0Bx2#R97^AmpUYo?2(pF>9FYaF#?RO3AQl%tinC7Z3LIg&#qA^3Q;RD9@ z=+C-QOFIP?^@wFn=?&wp+Ab76!?3vOfN06P$Hw6(ACu{GajT6ETCi_Rq z^Fr%7R9M@7!wP~ZCQ8R2fl@V}@7FTbO*bGWT;Mj)rlVmUuAs&;vHjcqMXYaq)KUbe zo??&?n%bEzzfGoCzTcpERb9tb3|XKt!{A3@@Kvwl#$rWlTaI@74|}xDy_Q!GU*620{`Ot!=4|h%Dz8;*~3b| zysd=J4R24bw3y@hItmWp+b)UHqUpceqyKa)I>PkKz0dA-eUR&`7J>uh>&LF_R`R7A zw$pGRuENMG3rIn2jVw*_+>@&LFj#uyoOZ9)GNV|SdUUPa{7$ujYOxT_mp(LA|MDB2 zv{LLWax{?JS(a3!k466xlKj40ai3wHlQSdG#+LqEr@+`E6EWn77JxF0(TFirST)vt zkBMQ)f7e-DRuk!-56&swGQcS^ZdV#q#{cH)GW%ouoGjU#pPO!^nE>V^5S1y_u?l{OV*56xG0 zUcX!E37km&o@`(zI-;~c-MpOwtXgfhO16a4x+uY zNomM09Zh|_*A?BDO6H$?R%)x3FoYnkEv*ET{t4X|vH-J4@9SLcR`0Z+tNTNq*RPn) z;m0cRb){dO9hZ=wme-?e9r&-<=EQy|X9PVM^c>FBw61jmxmB#ktTU zU4@UU9sdOF99C|Tfi9HXF8KPMuZa7`y98Xp6K}yjGTcWc5 zmJO&?2Cm&GQHA2K!dsM8mD^{mC_CbfE#$~lFG?aHRGt>w>*c;!b1AEH)Et2aG&I2PY(h{O(|{EE;~yt~S> zF3=Z46X#Wcw-HRTiG~tI=^ehVqDLmQwK_XPJu=COzoWY!4oV(0t=U-Ze(-0uq?g=r zKnjlu@0QPRWS^QO0z8~n-5j6^RUOi=Fw@-Fip%|1JAzr-{%5~W`lFwCftVdTi$JiP zF4=|mfHS84DQq09vjlYrIj9LH?d!b|n!dvGKA(v71_=GD;{Fvs5~_Fjk>7$> z#616Mp}&>3D_uRTIretXWfZ=psbNIrnp9T&9|N9G++nP9|3@+X$G9KxBRQrI&YL>l z=F%U|zh|`^m%0@Be5(QfW6VOA1t^#S)u{jyu@q=AC-vVA;$zE~g*la*nHT*JmIaaV zGJ;4j&r-*Q^3gxi?_aZpqD=mmyBT&A-u^#bX9>Gmt&qsN@_y~Sy)E-8=q&lhw#cP{ zu}u_0q;9FJZ-NH5vL0dulSA)vOR*6T9A6yVltZnW{`N5TPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>DA+bqBK~#8N?VUZ2 z8@UpI-$9((nSmEMc)@P4KnGqpbL;~M8#&%92oB$zLhu5cP4W(40egYD!2mwOy!s)F zRs8E_ORbS+>Vqw@i^Zy9k=^WWwKV^F=?gn5v&aH@)jF+S2(t99YFPfz?0$`7#&|F}(1C1q(VRU&~mkifB#HyN(wDz1fB5+t+^yoxf{ z0-_C?TF$V9b+|0A%;z6JfB*55%cO-s`AmJmO%RhD*SBDz5)$j=kQO4fnInKV0WJ$t zORvfPzod>i95Hb#I9NMZLe{&5TxVZS0S?-&Gb)^$y9xgRN4mn`GVu6#U`dosN1DuB z&$Y4%KrL+X=CnUj0h*%(lvy*>@@vq^K$++h6NiBk+OxTaYD2$3A2+6;zzw5)-g@~A zug#Fw&MvZU?_r>A*8CJ}K-v66gSQI~^AvWo9#5a{L&?&TZ^@1Af}4h09lK{r(1vbD zO}F%)DKjmG!|6A`otf7#xmm#P@ER61I3GFe;EbEZZ$9!H0p(0CwR$IKJ3tu^3e6q}l{;VA>>C53&Vumu?WQoXyQUa8AU~9Ki zaVwoF@4~x@Bm*Q_e}QKk7p)V=h$QGy>s0jPE0ZL~4kAM@5=LwSx(tXW%PT$FRobT7 zm@y&KL6ikRTbB5x6`g=hxzUR=quS1IVWECi$=KwCLzaV@^mkt`~s zN72DDS%xzIlQN@))(-M`Ovlc{5qrwDu?L9Q&t6$a6m=D#bpUvaRr$e)?E|17VrLAV6ofg?1)3R2|`+jxpUfrJl z`|tn3zp&xFCU|9b9Ovf2oq6_VwtU8%!SdGPyb(8cYG`j@oSn+pwLSj+<9jG=z=q>8 znnSphsGihpJQf*p?bBj5W4VR7_+e+}v+UgwPu2KyTYTlG66`1IA0K}CO0Rcs-hRbb z0(R!73HE2K{rLX={{2UO(u8XPJdd4r^!O}T4zHMQ437M&TJjjW8J~>MIy;CD+VYhu zScVu6IC3HK{rQ57z-L)dCV+ zBRMjdF*5SL6b=R(yg!@+cF@5ey*`Y$<5@cN6`-J#9A(+O$~bobYN1B-jf{w zn!per_Q6ycq44>~cU+mjn8i88$!)K1T(l0n!Wc;gLZ6ici2C@-Wh<}PVZ!Y|rO!Jd zqecM>VqfW`Y(y_;QtQOp;8)@Yl>L-*w!qBB58%#R43dG4M(aWM+fcC#2QRu^H^k|7 zmh`cuGmsyO%Ifd*ydPq~?;sV+=NcD6%ema_OV@9+m121eeMa?{z;?7Xx@P9$Gog=9 zXT~14gl2#F$XC*^vk`d2?J}FO+`?t!cb*H}nPGC^ByLUR0yt6M?eQuKE*hrYo?qDl z&oo_&=NS29N;hIzzBm*IN@P0Ac0!c zARUnaViR7uY~>X@02KHn3eS#+mA;wSIU%OUK{6P}OIyj=El5lka!z*C1`VK&ng!3m;kVY7HRA?ZK?DrMpVeBLOQ zK}R$RY|C=p3d`!k1Bh*dCS7A-8LL=BNq}e*U%9O1RXYq`7wvHef-=;CzUdz?qNUE;l^b4m=(HG?<6Lb1W)+$G%X=gjJk-TG#UKBA&1}9hJC5Txj=gZl4Le`o*ziobjn2agIOcF+JMHDB)hw(1X4l>Lru?qX;fZ-Ti5=e5>##63>H8A> zQ~H~=4cAHhWtrAB*<2s1^dH{!{bdOQ`^(4}9UdR`A-6gfdCX*g;C|PtIb)dcw&5ZF z^a8-2Pt)>6@mE-6r^?aX`&nWDY(VTbSJ98pc^9jK32?ki#sG|4Y+M`hGuBl~Ilc4 z`1a+y{IX0UGeVoe7op3(zcAV_ZbxNnC*=yuT-Z;AeRars0zntcoYV%Y%F* zUR{{M4t}z|i<31PY40OC9ur&D4wR~M!?uOX!C=2_hbY<-m21&f`~hkk`ag$@gL zYRmd}ac%zk%LD5h=s>_s_~;5gF-D!>nECWpmEo9?bW(rYZLvi;kbvZ&Z3h7*8jLhZ zI%XMN+Trt<_hcutS@j#+qG97pfz1dg8^06}Yg))u$Jaw9Wd9o8pFO#c$`5?u(8=gE|BG zH+>KIXUvY{IF55g@WMPD58+&b*MEMU_DB4Ii>dsn$2Gv2Lnw{2Fv#0aWGB!Br1T8IJ z6n{bAPJfKTttVn(JfqCUuVbh%nE$iH>^ROA$Q5(En?3F2GCUrX_06A^r0e}1+6R<< zq*4wC8s)vw78f4*G}NLFc6B?v&3(6iQ_N_DKEQwl67OyhDF)WJHmiQMe%427iW1!~ z?H4vBo|YXfb0ND8NF`aAt9Ag1xo1C?H9dVURN@gZKl+R(O_fO2HYwmaOT2|;;tzxv zfFd?PIqJ@_&=x;XB8|44Mq8rtnN}=*&+|8(6X~Kx0|8kIal8u$JvOmFi!{< z*YkuwlimC}Q;F7J_;sn<^_|xr#H)vAGyd%s(#7@}q!Dk>6N>@aPHB@MP)e`YQSh z8@|8qsFlB(v^gpd>%f?IkbX3Cj8AmWv)ZrTv1@);=J4b^oWjfwusF8R_a*wL^fzxS zPixnI`HNYzlg};8E64g2PM#+i^R@0TBV&~Qk`z-Jiy~&Uf7ykZeqf{WP%Zvxa4TOD zf5{>{RqUd;-cEq#XZxFl_3W@yj$|lj3aTzu~v^#n=Ol_RA7( zd%!=~4qnja=JALeCOZAXPSB_Na&nv8HsaBCpnyqwyr7zJ7-Pn6E$KyM<{E$2_3h+CHM^lJeWMGe1UP zXMz{{nT@s`-B$M(OFI9NR{VN?n!cnM45Tuc{}{7SF3fV2i5m48I@X}XiLhU8{pF7>(m@IfuZ?JMRGSVYH6sIpF$>2IKD8>VR*ctM?2HS8cE zy)=-3$U^~*OmIwwWKNx&6RJs|K3;{dKexquuy5x4*qV0 ztZmUqb04Y4-?Y=18@4Sha>0Jv4wJVfbp9i)_yg27^aar|7}n%p3c{%iGl)kP=CAkf z@k+Q{n7L3a%*aPsk6R0~ngj96!h8;u>G}>=!8w-4( z5ra^n^KWc6{3iX(mk{NM@vCg=33RD=SZ7|}_-(|KP5Km@*4J@r9iyLw1k8aT(fV4R zFnw>t!=RUGw}aZl@mPHHwIGKD@4kG;o7nMIc4c8!eJt}ysIv9z`%AvRwCqqB3Ns6{ z*jX}fjgIcGW=%W-)P{q3q-mc6RJjFe`-mE&YW#(r`rNR6fwu9ap=HOkReq7$!i-eo zug`N6gJwMc$ycJYFq=zwT?U{`iNI5pAR-<EI_DeJ2 z4V|zCJDE+`D*ftPH`-T1-=eX8Q#^VCZ68mP!(>&rcQX5emg`%2Np9@`Q0Cb59M*B_ z`2&c)?mOGZ5+OR7Tu8ZU$3yU8tN8(Qy^n41R`y@?w=3EUv+pm=dG?pq80r1U&H*eT zXhH9H&3P;KqYf+R{%S_!vEVr(m#}TpNpi|tNMkvd!iI`Jjgj@+vZKd{ZA$zAmucAv zt4}|kpr7@Z=a_}{!-k4K*@=B2=OzZtX#RD~0x!%+Zhj&3H#^JVE=L@&EvcQ-oWio0 z=}fIM@aN|m;yKqij@sE~{-j*B?el)Oh4&uY^w;A1i>0Nn<>TYBzQ5zX>THpRQ`p-6 zxzO=6V#9^maU92SoGT$L%&+;ow$(cc*AL$+S;&gdQhGP)`n1^qc4A`&W93qlgc^<`Wy}qM|9Y1wq;3qQKAk$Ejxhp z{fLqz4>5z0X%EL?e&~a0!{<6!p29N`#%9AW>+|Q1nXiCdn6+O(a+v(u@dCs6xB@*~ zkbDt(VGedO9Tl^-uXU2#qV*TX6N#sQ{05i>kFfIrh|$+|Xl0 zec|(@jvd`L`6cX$5z~ynKF@Uw7WTzCXCqt&{Z{@ZX2)@^5W6r#;b@*BLy3AuZY|;X z3*e6^0Lng6EAt^qd#|?*M;Y}XyPytsbvs1mqy&jElMy(;;Q|QRjJBwv`}WPRH0 zdB5db+4``x4m>mj=qe#zb@3}7GgO(cZ-S2U-fRn1a2(^3fi4=DmK`8e*@IV#IUX^2 zf#mC$1L>wKmCR<{Z_uw^y#=Biv3^w!jo*|jdmzrPkka^yolHl?oa|d2<=3F(OXErG z4w8SYgSMcFNCE7FwU7=SKj^0m>@0pBqdfka9WzF19Q7EHacIIG8g@+EtiO611e)p~- z^KcqdT;D7e9#cTd5+6B1G9g9tv+wcDV`+xE;h#bdx1>nM5 zeKMtod@u&b7XP!4Cx9{UApK|tu11$G;H&-WE2o;@l{q{)52rA*FfWcR^nHo`DgDjc z$}b7yXUxbtAbOGwpe?!yn(v4;|CAqwh*g zV_&Da*f{?B?e`zw`ERW|m%!JTOS0X51UoeE*_$HqZ!~Ruy|__uK0NElUCD9m#5_Ff z{MQ%E{Z!6<<8l`8P`W_jF-%R^K<$9TnW(eQzqNWfokQmFn6wra4tKe%=m?BTZ{Vx4o7@n z3|Kn*L8Hn^7tGH!46XUL88aQ-Iv(X7i?heulev6vQn=i;0mNIMUU zfB-g}|1;g2y z-hRECo*q76G03O8uWzyZ@%`Q1dy<0u!~NZR*7P@2sUCrBMpy%6kQTCp)s&)8gGwSplAPwbRP^Gk#lh00``3KGEVh@4V7=1c zh;13G@twrbh}Bb^qC_P5wdbd%Px6L+Wf2-kha2?vBV~UkKdpF}T0b-UN*`;=d`7Y! z5`FvBM8e061XakV`Y9uzVyCNLBl*>N8_$mds#7hT3*q)2-YFAvCr9mglt)l90WIf` z-+uh_pD$dV{{7!?;UXC?n1BBM-#^~r2FQPY{_h`DF-sO%cYgZ%l7Lo9gwgbmGd3fv z0WwGnS;A^cQK&&Bks(R0=T$$>S{y8`^w-)C=I8rmvAs+L>xewqtxE6ip~=tf#&mjx}rq+VR)z)b-o6?VmS48OA~8EwEsk|)DVrcL#x08_ZcshQej1~3q1=HG03yL7VAHyY>ZzGAF#8EGT#-(9 zbP-#*v<|$#dzhY{KE;2m&#bmDa^UUV{qz8r*-xJy?&H;aVEBSLOCJHlih~@f=2=)M zH{((pUtq{Df_0?TFj!K|2Vx|i1hi77di@28Zb&4t5ITcz7Pix|--%Pa81mqdmi>-j zvL1;#nZ6*A!~!ZNQ$;foM33Pe<(AnvpDjN%M(vE+2bSzd+Rmsx0GqlN=;?QY&YDH) z_AO!(mP`CN2}R*2nuXF1(9thx^=mXg2B_v*2!)@}hD99j+(JZ$eoX7)sRP%>KR<_m z>eJ(2zy0>tKX7^c`Nu!M{TXd!pD=&<{`s$Obn)CL3yWAX--Ly7)89tthx{T~M_P-r ziGX6o+DJSJXr)Z``U@1@kVs-7bOzrnY^P(t6Q_7F?eL*6L1yoF? zie{tnh&)QddrLn7BIB7@91jDVnUUEznOS*@)%K$#ypQ@tG7|;$3lGbT->jM#T1U`; zh3&Weu861A|0TyU;}<&wug3n0c!vBYK$KgmGrEp1iAQMVp+&1-v+^hj?=Ag6R$XV+ z1v5!Ooq{tvOiPYnJk}IzLt(0Y!i=BaKHbO9ikZPrnAIo1q;D>mOZ)8$Qv zM}yPi@))f>+)+qNs3n?YoDY3xg)>*X6W@@tI7*eBy4!KT1Xjw zs5k(T)>ol%4QqpEx!0EDCLXfgigTd9x&5Hg?N^0NUtsHD1JR*XtcsmjCpv72wQoK@ zqL1>LeR3Op74cUfi?#k{@ke<>zZnlv%Ufn&Rg(*=%rq(@wrPs@)2OeVi^!w5Un;~{Y<8u=h_wfrZ4jQr*uToubj@~R7Fh^dHzWZX|1CwG9y z25N#%Q{L%s&F#2f0xN;iT%G#3kHr(Qtt}jJAQ3J&> zHYFFo(N=gzd97awP|O34HWYn3x-$_Bw2Z%Pr>@_op^xHF%>Ay*PYrB*FiBF!?cmoW zl}_f_ON8R**r=$~V~mQ?btKrACUHNS$^YVbkgAx3+_sv3Mw+U~(W|nDn*;HXtnRno z-$>0^$#DJ~@o>MOhBlxk>Ld%SV2st^C{WGVSFQ6rP|Oq;NWyzJKT&sv<=aUlDqjlxTM7>%z$@9w?Ji_yXT-!X}KAxML!V>gJ`2j1}eat!sH#xoqzP{W#+jeDq ze$?Z+@I1TN_+GAGOM7!0^Wpgzi|6`}kIzdxKg;th7tD^c9++@|{XPQA3a;vaw4R9{ ziog!}{hkWflb`W@FyC1!_ zbz!W(*Ea6YHUG!(3uXtvKcI5+$H!X%jtn=w8cF;cEl9VXpH~achvyx+>o|^`mI?XTa)4SbR!7pl^{~)$+d4Y5diJ!}Xl~OnJ^G4pwY%r|R^M<4?pL=~v~V+L&K3(#d-P)?1SHXguheDEzBHE6;@KBvAP16a^uGSgZT1%}5`P z`@J$%jdMkRp-t&`Nxw9#A2t-d{zgD-B*|;)Z)RVbpqzdH4|4!zfO0|l(3~yh`7HNe zjgF^f2jv!>=TMi&-;Af!@3D1$YGn4a%}73y6^u&VjJ1;X3LPf4v?M#V{ixaB;dp2p zbD8f)&Hb^qAE7)LrLIUo`03{SSXL<&|fZMEcGGS%yEG!{09LK4s1tl5OMn6}tZBTlXi8$zbUSfOn7`&E+Z z3q>TczeG_x9FNGOB)l6pxm5y0MZh4-gnc`uex@N?O*;+!CLR0d%})i^>gmqgF&bIa zSefTDfsV7gAc_Q4-$!`GL*6UykEAj^ivsE|_sSRmjR39~Hx|FsH>ANYA|D zhcLG8@?_^05%XZKc{H^4C>_#}E2*`Uos|T}{US651LrhiN;MjdClYO8P3uGrsOA~t zHbQmgX8NmU391CCil-mv^8RL)LVc%SRbq?@Zr$l`<{}}s>?6u5JE^~7Bt{tCNj~>T zDidR<#?!Nd)bd=5zp%rqMAWzA+oon8I9q-?Q0i=iY1UTkRP-?>ajR=2REwUSnIWK> zf0c>2ANBe>7Eem0rM^FAMx-gq#faHZmWK0_`JEZxm?q`A!f5{M@lcH(mmuurp#`e3 znN8-uG;tiqaU5ry>=WkO79*){?>2Dn;@GRqd zfqo6Kbv&c?Z#_TP7XP);%8%#GaU92SPUA&fFguRpIL-z6`|tl%|AlJDaU92SUJCk= z?Bnr3Jp1$2%-g$%PxoKL#s-x3xAy34gnxwM+%ED+vvWD7Juh|#{@FgmH1Y3&;a_!T zNtNt6S^kA;$8j9Tab61gg1LXmF(IM6is}=`D2(5Z#>*ac=6LHD%x~i=>nWQ zzN5@^bnAGOdzysDW7vXeh(CUP_G_5S_qTN)VNri)!aYBi&(D1wMI6EF|}003fjHD!GO05|!bXTy5{0AK;g z518)-tk?Rgih$Zlru};X$5BB`0RU)7A-uE3y_fMl)J$Ik03`i?FRWqrk9Gh6y@R^4 zf`OkUqEI$@^(k5CYqHjSXqW+y{$mvYHKXh0l5}XA%b`WDTCQAF8eqTWnv@J1gByg6 zE!~Gc-L$=;JFxziG7+@0JntdJM8XqS&T+lq0+ncw&`8?LX&g!%u*Q1X*Xbqod-J>@ zu&X^dC@`KR2pE1T?QOe?5PQHag^2^UIhcQd@jN#`KIH-Q-F3b!{E1 zQHr?j9rkD*Nr$40!GH`mE6^_rUSR1inYw9gu)Wsr`^F==LE}Bua&9ZN6a5dKgsKzR zA5cXEiK}n^`o**7i{`1ElNmUYy~KYaIqLD-GPlpaz0Oj*wOB)554RVGY)-im)GzRe zg7`_+`WV^=hGoi<#g>JNu=12X%}F&qBm#>6L` zoGx%2DgC0PInkZL;b&;(Ii1Tu1Fl8J9LRh5D4&UZAs?@m$6>9&!o41rCrzn`a6azI z1q6xg%SV)JA+;OC-dqj}43@fuv$_u`?ik{+U()I%vy|$`Hbv)I1w|Xx^EhdI%%iNv zuagJ*cm$|+*M9ht6?LDPYZ2qIwSNFu^ydy@|`9uC@sLR?=9z zmJB=-_sKWWnigI_iIJ_;q2Q5&REMR-nVb*QGoAA>0? zptgWID-*-v1f=9?tERMDmox!0&0&o>VeSZ;*^!lNn;W8&VgfP`_4qojMt!20xj> zndKXf7(`I+$#(|()dFR$;>WW*-)33%3M_=5k6J88;)_rdC%22P|y`6INa zAKSFs@i3cMWMSdY%M6l`{Ixi2oE1Zp#xz&m-^S;dPf-g@Uhfsoy=YdI1=NoBb3JaQ z#@GAfqZ01EreBThyXnjJ$=g(L@^Nye*U9oQ8?ExCI{~qcQyc~B{;cNvGhJc@{#7sA z72^N62kaX7dE4t{Cu@PaU`BEsE`L4KU9_x;x9p7<4GE1$wbV~W#eN}6-U^dZiq{hTLbVi6^?$b@ zBTAxMVE%9)tri7F`=8~?$#Y-#~m&Lzo|FBJ6L@A_1wDsCwm_dTE`6im6YlQv!7jUtgbK}Yw;R8rC@wE&& z{}5P9_loAA7}*6s`w8uJgUr?ia%|javTzE16^zXLO?#X?ho!i7C0LrPoU^RQ8ugL- zcZ@O1uMsCHU`Vd^v~|#n=cH+<5n)mq0GWq^=415jn<+5t2dv4gvmB>e#7kUERzGxN zdMRu%ExbKD;p8y6`}Q6C;G7)nM;egFDr2`GOEtdpOTbtPlbCt(WPLn?*KP&%g~EYf zh%k^Z`vc``6;qlAtmL7MwPyi@_(q{islS@B_oE9qSbGSpOELd0P>nJW4o zVAkVGIfm6u?g6{D)}IDmJwGA2fG)UQt*$gIxjjUM;Z|K%iws*xZLC@n`y+^i7D;Y{ zq<*uY*BvD9sv5qX!`D-a8aKWPKNTTSZ5X=cXSC~5FD-tLAV&3&0h6j-uizAQH59a# zxI5=${FzeAFyTn@Ta_Pvl-{zJ76z)#f`1y6FUr7ji6>la*iG;AF)x0cz_tzKM=;?n zg}$IDjpWcUbYj_3$i7lk8uIvD+Co(t3t=LRIEYPWB^~%Q$-B74CM4Oo%Y8{sB+b_XET_MrnuG*$Do&AN0Z8Z9NzownYAm4DnsTRr0!E&Q%Fk8#t z&}nA+RL1qIo11GKxoK_%r`==EB~-Km+u+ve-F~ zQsqu%l>>slMsaC!4pf)io;Ba9Wm~n+UGIVFgo#J5xatNSsBX_qajzb#7S{GZT9pPr zdEBK{WSCgG>N++ytdr1T4P~`r`Z}+fcp`noO6Y};{>2{1WQ*XFg7xV=cdShz=V*a2 zO)b;81oI;_nf5Zi%U0IDkkc60q7##)v~8!GEyi~XK~Yyw^`y@EN9>Y_aQMEA`WtR^ zr<)GcE0S`ucjp}+98CU!;GJ8fDeeTTUvt*|v;b$U`>R@v;ctcK!m=dGzaZ^bccAu; z+wYsEUKJG?cl@eJEg11;bxc?yI5bXz5&)Ffk$--#J_#agElGVe5!m9oNjGW z^K=@0n)Ev*8ca*S+po!IaABsVMVE?wQRFE1l}l@fGnjhSnB z&$W}b8a$=qi#oq9aPC(uF;YZSdMf^DgvA`6`02`Wdt*QO3+|Cv*8&lWTi8J0iCPx$B|*2-9W9 zaR;hxQYg4+?Li;s3;&vPq~6N4xw<+r945WlLH`7bm7|!)C{`?+`7}nI>*!eXfIU&I ztJ9>os%@wAi9Pi8ol!yD&)bf^<7&QnbsSe;IX`w|((-XEjakmVGxRj(g3XEVhn^-4 z%^BBdd(yn4T&+g>8TtMmT1__eL~FFH_*oF=%ktyzU7Rs8+0;JzZ7etCw@P1t%q2;r zZKr6wbaoXVzQL~quHB)ui^GjKQi-e1ozG&etBr?V90{62Vk9W6F)zF$r)In6s(an85>hR@<09dYB|l`wlgxZTlKtKezJ^| z$AWgaAjyN_toiCuvEe{99CY*EE2wri#ytmpvO8z|S;;HtH>b1u^Kss*6K0BrPU7So z!>gFxvb1>To=Y4e02H5t_Z6vBvDy;r1yH^=e~;cRc+vmhV(K!GLd=0$IM3Vox57>0 z{OlUdb;nkKsU+1vzdg2%@*;w4>le>ql6(d^U6dbHgx(0=eHUu@y9@2PDeJPWavTo2 z9h2@qQ)o37YPzAE@fJgwff|xIQ@YB$Rv-}Q}qButJJ)LS+u)?Rb zZUy+A50pv{bO77zAbQ=&r+*qf`4%p5Yf>0|^K296u6CcgANn5HYZnv)En~(EupzfN?g8eU9izYGG*Tjx=cLPwhy?KNZ`xBcpx6XdY zE%%6GjyQMGab6ah%?}{AZv_eEws=137F8y4PiMk$-c=h;fA61&$T!46d2^DFZ9IjK zKF`rydn&)-!5ijijx(Wz>L`LpTd`@?oXC`Px|oiH_YPG?l08}{*=(F8rwP93&r7kP zbL3Y1%Jh0x$8GVDIyf`MA%#iKxpM|I8y3zdodo8N>6H9c5&T%~_Tzz$4HxonB--3j z-K_cE1hByh2xW3D$b#oktZO8sybJ6sRH=(qwmge(ZEkTXw|_{Ikh^uO9VRuDG^ReA zsIHqKUaFYU{&8iOudh0v&$6S0imOH%I|?b-(v5|4Qczfds@pu&Fbds`g7x4v)i#W{ ztU8F}R?z&X7+olvMT89(Pkys833kjux5(sQpvM-p&Lr{OuI-~S?ovq}o9-T)f%Y{v z=g22r<%zk5n|uV0S+yyOf#yNiS1V;=2Tkz}XLm)b>Ug$J(ta_yXoA#{#NFDuFPb`=Qp*%E^$&!s_Tt z&Pm8@CjzhTbE1?_MW*wJPEF3>{7kE;bYf|``Y@y`&6b#aotw;;TP$pp?OU2p>cql@ z{)1u@DXkk$iJ9K8y@<-Vm}wj8&_A|t19PG&JqW2oCwPhag^2pER%4sup%EGNLf7GO z!a8nF@j+-osee>oJo>8HEWU($Gj@b43FM5QYvufNln7FTt3v&8V7J>xbAz<^Jd=6V zy(J-{57}RgdSByp4Ec4gV_kKsc0Ab;TQXz(=S=C0G#zxwc=;D%iB~ruoG_aBZ|7gU z?vwgw>o%RMku@FdL>&YZ4dxQLR~fh(d74)5(?2;IlflZO?nqtrl7V%QN;oBef7_l# z80&H=*|EJ|an`#y{*?9L{E zI&vgaa?Fzt#qM%?CEl$4ffbLT+iqGbEJr52SON1Od+oxuH&h?rjQdHQ!vX{*#bHl{ zCx6AA;EMhZdY{stw(T#JYcBl>OyB8Af+lU2N-u+pKJ+KwVS&IaxV1uBu1Ia^t1V=RYXKOOv&(+zHzY&i*%67{xjsS6IcM|gc7?UAh82IM_wK93 zyDjMm?dBZLsgM%AwoE7be$&^xz8&~RYXFVvFQk0%hExwtw<}4i2pPe0tAt-TPc2z^ z>ySzcs8(GHaQ`?ihIY~+>ZdWCpbopVNPuDm|BlmMR^C&_Q+4VWnB*$S!{M%AO(1EH zNl3j*$;a+5*sKZkxdzxBFh?OjL{kB~Ww4b{++CZzn-d2yyKEB^|4wu35S`)%CM zRHk3OT6x%t*P^^uKEM@<<*Dxx9sXe8vUEu#o&fF#XiEH;erGV`n8~^QKHHV-)3(-y zad81^M=aGKZ0l2EvFEvsNBoyBNdz@8rbALAEvoaS+d=k-XGgYW+#V!X7Auy{oS(M@ zr64Q~i>~0L3nlW*rAfh3$Vy!rG@@HHV$y)=VJSEdm{~C*)!u=% za-GY#w2M+MgK{?OgC}jjWja-~fGQC{1gV=X-ziEBjNv`Sam>u4U}88KW9!K`nh$K?| zqd^ugbJ5XdiuLv8#eUoNLX1Hz`FY%p{vhBs?#>@`u~ln!9$CfP7;8UZp{0}N7$G|2 zVyu3)ly89bqEX`Q*A4EF2P_oj)AaB44jd_#2jIFyI_3pzQ3irg=RcX6B>76KRm;b2 z#?ikrarV?#EOi{Dmyv(&3ggW4o^oXL|D=bD4r#|?&)t{qK?TQs=kEL}@A}j8)$$ZU zg@LH*vg*S6^ES?AsOnPPyLnj|d5V#&Hmd{zM>x$&eQK$Z0oay6z18fqxcDu(N|8AC ze}juqe7VH9s{>G0x*5z+x1vl?5|)LKwe3cO`(Qkm6qiMY8D<>MN~QS3SADHSCDUCZ z`4^H)>SpV~?m=G4#P15l7@Ho#6g`cC&fCgnDd*|@Ek8DEygGIY361A=)a|?@m#Klq zgTprh*jZliFDZ}?4S+co&5?i^_`CG)_~ru8BK;$dqfSZrGP?&0RlEaxCz3x<6b}h7 zftO*W0IIv|BqNTlpz5(zyRBciX0n>=Pi{`0sW6qB!gu|hHKf{uNtQBKO)cx3XEfu{Eiflzsy6W zP7Uy@DPR?+xwL7vl8vU}d@>7=R#ryVuaAn*FNBIdfaGnk!5FM>vzwQIF}k=8+iz1<=Y+ zGfV!2AiW8ga#|JaCnGai@~7<0@$wfBjNuXw7XDP5-B1E%QemviIjbxL_zXR4l`Cp4 z!q5iRL7ho&#xTt|^(gDA_1bhzu&y6`RfDSS0}9_u(DPVbyv^`|a#TOdSfrb58LXVV z#XvT`>qS4h16kQ)#=j1ejeGu6+~FrOkw{iB^rM5d$z9BK1{JdYQ})BR<2Gy+R9P8| z6x-N$)@g#e-yJEch+;>}o9UP^d0=r;GI0`y%EJ1Ec+qd21D&ng*!lkLobT5GeNRZd z*55Z92yZC`(j;5H>&0H+(H9!--3Gi|fREafKTRGp`2_e(m?$nyQK#r-Xm4B7^TVS=FkSTUkCsAw{RXUqwPc(m!hyh#zj0<60bJzo%5}9M`tca_75jx z>SNnT4ax8MhP^oN>-6S>5Xj2?j1HLfdndPGrt)8ib0n6Ydk_Pftoh3VJc3){n_`-N zGX(L*^p&ZGb43lLe^@KIiU*1R>{&P!QJMti232Es{rzPLj*;V)e9iV`>%O8y>S7%zWA z_35p2?*f&yjHuExaWoqR4D8bUrT{FQ$#aep7Nv+Uc}k})5m`2a{C0rYhk!Wse4-kV zb+<~=jgW$R>PN%|wxS7%7+pJ(zD`E;L$XJfI;`NNxtLQd$hlFbOU?CQ#wrVi3>xS4bf#lOpYUBKkKJa=*YHu(0H);Tx_vf7v(Z``86yzw1W@a^r5|?7@!^=xcN? zx`?^JRomcJGUe9ZMU1Ar2{J2Q`}V|@Wjpgr25=L{_Wo1g$3gL!#J`zyA`cd-FjzON XtS<6`^!fMSFaUKGZRJ`;tMLB;gdD?T literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/GPT-3_5.png b/debug/accuracy_tools/msprobe/docs/img/GPT-3_5.png new file mode 100644 index 0000000000000000000000000000000000000000..5bf1c26d3d8734b14e003dcf1b26361ed300217d GIT binary patch literal 4676 zcmV-K61(k*P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D5!p#ZK~#8N?VM|j zWJMK#hZu-i-H;#;#ffw<8D2Z?u!zE_S;@H9#bH@Cv_A}ClX16Qrnh%yGt*=-S>hiB z6Koenz#uB|k0g?IS5|)!9|=*^>15TvN|1mtam*44Ka2_SoI3S5Rk!MP-=6NC2j|Of zcimH`PMtbcx2o=AdTd-69}lC@%EmLDTobFEmQKYe&dt-GOVLtM;c&-7FWwI%=tSS9 zilP8U;UA2SEsx9#h(v*fV0E?Ksz-)Ci8}2zogI)+f+Fhr8v(OAW#G9)UMC7yv+1BL z9Ck|1yWS{h$3n2>NuCI+tE=j>(@}{njLZcR0zrY}z;1%*8$M~GZNWs)#u+>{O@h4M z?&t%T&QJPI0yZgO>%wBK3??5)^W3bZEjxL=4f|};BsrzEwlBgD3_O8BfPI9)f(2fEQU+ZhSP3}+pA=RnS6yDGT==id zk;KaRAOLt>@Cl`0^Z+~;60q5@X#n;dV23odypER)lpAYB9g1MulqEo7YwXPq!X;cz zWwnH-6($`7K-Uh@Y}?HjOXq88!%-*V2}#Ds8&&B0-KST$p|;ZwDmmvW?+ z1DXZ~3S6)0RE$(^TpnvoF>AMjLF`6Kr?~AQ)Omx3Z00@bYfn(V1B8vt7%UGo)#Hq# z((P5Y{6wg?T6U7bJIs*osOh%sqf=pnFoxY}W`0VmK@cuu7R(J-8kFd4VG}ZdOGF$H z=2O^=3}6rlg0KY`&|FaW9(NVc&$^NI@Zv`|owcl2sj{*L4uzh_(gq~Md!q^W`)XHs z|1V&mv7))4PS<&uYU4w1b<5XF1NK=VCbGJ^KCD|E`UL3i7f*u@oT*tIuR}o0Z32`I z>H=80PR;@$xEjL(EVhsn?Tf$%3!jg8y-HyHkGF9+{x~_0kM=V+c6q%ie}Xt$ z-?%?;Wc^g=wgXXpx))M!>uY1H6UZ&xx0F9wmcf9BmIeC^Mf<|{!J*{}UN3xl{>R!lM1QC{79Z*_Yv01_P5BeX+4#o&fg|fD zVA~D^`qXwBpEhDvCy-mXZw3CqhaGzszg_V-aAr<6;1iz;*?qym)BNQzAZ3&Hb!vR3 zkoyIr1mAS@m#_6@ApXFrVDO(G@Og@n)K8JHGi(AsU98s0pVE}|Q_Tgq$U{4^V_+^J za1LQ|on@YH_d{+CDF(JWIwc6g@S%M{us+~6SP{;JaM*355rOu5dh0@Sl})?-jMMQF zf*_0p+85xfa(2ykujl!4wPmo^=cT|orPWfWH(cnz^}I^^b3q&I>-i{C7j1$dj1akB zAPB?d?Ac%2e_FI5k-0z+1VrWnK@bp`3j{$x1~V7*_@KDIn}YQoUP11nvzD8Ysj_l| zb2#)mmNq0=mK=)xv}^aNDV>Tl*jU-Qpxci&*yoJkfqMRDU$+A2WeoOW6Hhbv`IEJ< z%B-dAX6gw-hL12jIleX3)~`%39WTvw@+iOiJZ}rjM&wGr-tMR;FP%=$&s& zjgROTFtZk)JDd+$3qDriSiMuMa-Gd+S$fY@3BtB$eaM6{QEx*z$lx|f-$Vl>+NO(b zsxN#W9R6Iu>jkX;sWy`R!TRImJU+%x-ZccTm-It_f;cnZxSab5=(YoazHWP6Ib?oh z$?5oVhVw1$555-+E#)(kf|n)bd>j~!>4Wf@;ViKG)a;WDNLTUGBW{!Tb!vR3p}D}U z@V;HoZ_{SzZur1jAQ5XDdSFoAXdNV{ZxZ-N9q?((a)WkXz;r_W#J<2xD5-X-G$eHF z2ZLF64GsF63nNE(y?E^T4+X-BX~a_5^bcYY$K?EuqP7Ihn869p;C@!HGfmV&mJvMu`ADO@9!+)_}MF|%^M zx^kX&0W)XLRn?y-*?mFv4E~*xBV|*iAwhlZ*TrIzz@H1CB$P072~UG*`liN5_62H# z1fQ7A)5aTg&6}$-<0Vkn=Qc{X95LrP*Wqcz+ZUKLMuhO=;(Xw-BRQB-x~Z~o9%33T zc=N8)H-m%DNp6$$$$u2+G^#2aNW3q?el-|5;@2DaA8q6C{6V=FA1Y<+JG^@1{`hg` zzHvGC)3qH4^r`J?e8L`zymdN(oZ);+`vcV=PQgY-_f}t&E!(?((Kr|!>+cVrr`AGwJ8S9pV(j_03Gn^wB_{pM1wwz| zPr}hvZk~e!5s~&bgo$MmHCXW<3-Hp+D690-W;noQEku zowrU7IdH+z*B>cb%FYFNwIc|^sDQaZ-zstEp1vKz<=&6Rlts-c9e%Av>2|x*Zo%w> zKfT;);auY?k-u3MgwcZg0{oSGb{@h>NeL%c=Wy17!#?gB>UktT7eE>kbkEc2C~QIy zgwZf&&(Ds29AaL!@#Ok^_MhVRNcp)S_E1LyjeVRVe{IYJR;zku+xAPA#kW6TADutnJ{KWYTXTp$R7AP9m+ zwgrM92!bGJHqy4Bj!zotq2ge`ccb1(FDk58d&}1QPU{z8Q#0^+Nf;_4>#yCXW>?fN z3-reZ9#=!y7C>ublYUmG7n;?ZM$u@vFsXN)9oAY(o*=#aWb_B9NT*G&QW0K;^D?QA zV1q+v5_XbuBxa`vTq{Sr-1c8@Z~9Iv2n=RG_@ZNI4J5|x5J||4+es~v##Q^po^ zZ~ctOY5wA9GhxrOfY@cSgRdZKDH@7JEF}lrdh3>36&Nr}8Ts{`@SO2lo}9Fq4_OP} zkpx-LzNno7`%yatmP$1nGi?@GWPxA}PW4v@f1?j$25G>>E^fxsTfM3VuBHx@ewTjX z_AvlzaH0SXl(RZFwaFSb|vB;gY@+Qqo}jK!zct|jD@bs+uHPY(JxtW8;25k#$)|t~&yLC>Q zGJc{wc3!9A$yq~V0lqEBoAHJ2C?!|GA#WsWwaJ0OMu|c@3OEg+@t!$_FOQW3R9K7c z8&52$(^ZqxCqNuXQ+A5&=j{{{v?$qLU(X)#j!}~EoA8g2K++}7a15t4i(A)z2_$F$ z3PUlRB`Bp7d_8xRlKTX5IE(~8S~y|9Yr6taKVPHuB;Q^`VAj^41?Q%%@Zjm{8T`%SG$sACawE~-#qk94l9G&S zTc8w7PKj*+5CqM_&eR8%K#A~$_NX`x zc>;Y)&J!t!N7N3GlBet-P(7R+en+dO`J$ZA>(W?%S$(6) zN_g?){+Fs3Tjg@hk`slWwnx4~V+Pr@$XYEN-#*Ac>#^q40^? zkM=vSHD62KPO*Lr6E{AtO2-EZsFDzjWTOL2y?%c=(_#OXW8$QHQs8b6l-?EqW&Bie z!}gm&p}z~_X^X6lA-r;!0tc!MNq@Z~CsZllnYk9Nz?3kL7u|+XApIcf& zZ}~5j8;Sl->1)E~34$O9!WM(J1+W16%RFBWaW(Xo#{zw&Yab%4;V(i;q8!TIEl^^e z(i5QHtuR($2KZ_>8|WKSRnf*`yTkZl1Yy)6&~K@fy@0%OD#U%PAU zg74{r9FJT*2D#sV^Xb1_{8ud+yIB2yY4)V_$h2G{(jbtIx3;zrF8s&prCH zqwQxJC;t88pD+LM6W3pRP3Os5j(_s?%ineESC2gZher=wcin+K|9VE58hiaa&p!3T z0}uY>wg0{K%g2BF$VK~Ke&1vFzVOD*Js-aQ&wFop@$nB;AHVRD8!E5fekbW)^~$@U z#$Q}>9K8DJ;zL(F_4dmTT=b)*2j9H*(t`)Sb^6G&SHAq+J72k^@%NU||FqTr?-O(X z*mmj4J+IcL&%gKm2j2S4H7~rG{qCD{7u@>9(+}?;{V%=s{K-4czVz5-U;X`m4m@}C z?BCw=@DJ{J@q>phfBgejUv%k}UzoV?8#jOcpMN_2(7mK@saE%X0!;_P9 z%|>IsGQGU(mR*N#+qS%1gNho9OO3{D`zKG#ADO?tG9AeGO`cf1uL(t}ckd6f_D-It z9R>%NYljY1b~Kv{i<8Rf#hD7UWns_c{NcsQjw7}C#_izhw(X#|I5WAp)O=viWNmqI zuC{l1sWyMK(wuB8)Q%pl?3ipc=9cbXSe!ppTV6iAFyE{ko~$XCk8NLu7<{H$nXc@( zKdbE8RrzA&qq}xZfBe>|>FU1EZri?1$5apmK^PfhWB&)$Jn`V8rpp=t0000McZBy>}MTqOaaN8(s9?JF9n= zH_!9?o%enJm~-aNJ$LRobLM1r+xke0|PUW zxGmzZfeBHU`-o9C{ATyBfd&4c@&N+_5%u!k6#K7@>maWS!N4Hs`gdR=?F+wPV9@v~ z$b8UrGeTwHhZ}&Peo;u^8YQb^tfP;~KQHSE1Yea-yBs8YSjGk_NWb>~Vjuc-_H}o! z#m0~42epWBhqyu;4f1hCljzr}0)@ZK#7Y>mIG&LG%KOMhostzFN-ocND@Tzc^zn(J z^vFodRmR;?%eym6%V}#TDb(r%sZrsv^ull{`*$I|w-&UA%RN*qYvW~|I;2swea zB8BYN(P|Ps(x0edhCG1nJ&|ZD50{HPjD#mbo@a24Kwn77+U^6BQ?k;4ZXe}Dj0_Ko zKw1&}Ji~_-P%z*h??=+g2JLy*y?W7#EG*w%)pR~2 zIXmB30{fhdGGJ4agcE-pIO9JPnjNHE^8Cqu+)vNq&Efu4(dkjn*;Q{<%QGftH~(DG z%Eb-}VcpP*Mcww)6;4Gq`t4|;2nqF9^mJ>Lj4m=Ey98w^4|--<^4F23I&+sV~+-h7q)kGyW}p>uyCPcWbYOq^pM>t{ZdWX?8JM$$;z{s9Q6bv&M zg}Q8sy%bs5fD-MF9`~+Bf!1bxoG#ePMV>})y%~yQkWpz!?LJ?4!z{y4&Jg1mVNxcP z+2J05axTQs%&V4E3S2Aa#x~v_4Qs&G5cAx0FSep5$70tyHG9(2*iG!x4ZeJ4YsM3k zrpE@zVVoM&t%daidGgFzNJC}D{7)2TKVXJ#aItQWGUWGgvKtO7>WRmvhSFde*Ql(& zQ_7W@+Ox#`AZj~OIh$f>Uu{XimE+fzOh=-g18a!?3b61n9HGq@Ths$B{*GP(GKT+@117qiw7 zte5b{FsVRWU(S|kKtJC6EzYs68(o~kOr|{3Et72~F)W5}$HB@SRJOU334+t) z$HR>s00HH<-f118!i-!*B4IX?9*u6eisra=ch{l%fU~*O_3DP-T{Lt->2did@sTTr z1Z!kJ6DUQM4tiS_9o>5f?3`5Qq?GCf`gbLnp>%1}w>Hp9y=(HF$t|+G#_k8y7vi=@ z7;q_ah_l>GV|uuihYffCZHj@BBSrJdWe0b0dFkevx}ud8v~mnurcSQI;CrogthYSx z{doqGS3hRUGu1aPGJHfdTGp3;(S*bDgi5c#(d(3LkzVcAfN~ML0-sg#I&;UWUSJvi z&na_IEyT_+_8?`EJe6<0@C_FN4Pz)6=XbJEs%Q^q7F9}#SeRMmvsC;Hwsw9woUbta z0c7u554sCp$sPJ6Kew?tZUnoXe4rQ6t^R3@j}5rwq6=BbiVtt@iR}BHOGVBG;BKT! zynxR3TRPTGQ6_+_iqP!0+P(fOOPh$D=Gjtl8DOCp$AC~$k}kaj^z6L31=_pbAGXSSBGzvqZ>)d+Za$5mlxQ)gY>%y75KdYy)nyYQ;Sy z5#Ux(S)3riop>N`vxQ3)*yLD=H8U1gH*3b&44BpF>>J4?hRL{p;W|nG!I=?0VN=T{ zb;U}|tLeZsi^&LzIMt?a$^RZNAD=bv!yY4lu9||F7#8|gDzvDo88y<+ZQrO1`#qLF z7Mkd;rmFJ~86-F~^NGFzX_#Il_Bh&Dc7LaOiv6Q76WZynKz}JZ{e1UEQY?VrvZ=m! zCQ-^qIs(WvVkp=sSBTuw+@62mqL| z@#mDtk}BR%zYN_6zG{nxRinajw=W#llD`xPG0me+pvN|z!`MCf5d`$`!q2ydG%pYN zX5;rZLYn0Qzh;a^+Wg*=RV0Og^#o2KgV!Vz`J%oCIq-V!6heZ`tE$S2dy{_v=kd_vHTAZ~ z@@*`AwQ8E0k=92e-A}eD#Vs~qeJjl6TiZ}pA@k}!k_d&#s z;*Lt>sTyc-75Skq$xF6v`UQ<%N`0NVc!LPZ87*ETKQs-^H?-tQxIZpuO1}G3j)@hY zFK>LOttNmdJ8CUCB|}pUR*L9nkT{Hs8^A>JjQ8cS!{qI)xo2L1mqWAU;>(g~4X0EQ zR0EtSIXc!c=$~5`x8EysXp%bN%698r zpEgl|YJ&@A<}1`c$>htAB}EF&O^75a)=bCsglDZkuKse zRY&mI-zsvE>BodHmT&SdY>bQe;P{p~z#$6le4b15 z8Mj^}i~r|$zt~nKNGxnpit%9$Ra8%lUH1Rb>g^7@F~rC-X_GheujbJj>&`IKOl!m$ z9IWhF@$^(V8%h0j{*}xaRLWpv`QkVFdW_Yz^`nQ%YWTnFGG;$=o7`bC3^6*o;a@G* z>^lyFF_)a);C97R4|vw@okD=i-n$0=%k6a+Ii9PgW>DkM)oE|ch{%t1tQ$#>|`@&mI4WmVO?n&MZcbgNaT z?^#P=IKL84xBwiKh^r_y z=)4_pZuw1fzu8)}4 zg#DP$p%m^2_mWLgLTZa@BlAvJOU&=NcBpk|&saf;XG7ChQkwNh5Trepjw>9DG$PZ03l@eS>KuG-ox zH-Hsf;<2Uj1%WH;{JRry_{LM&dZ*|wsSjZ@xr*B%+=B$R$o6lk{mhQllRd3LlIGeW z`E<`Fjl344UqWM6LIb%OtT8XWNY0G7Hv7LwR#6$HoL_{Xl2Vwa0~WEvc25FXF{p%N z-~Frars=#XBeHF3ys=9DuGCC*SZDp&^-1MnlhfbFgrE(UCFG|$Y?Z`rk|N>he>V5J zbZC-B>3)vu8z)hpv66#JqkPlhA*aUs;Ex>{W4H2$uTJ)9&kF)6icpsZV4jNA&=TeJ z7}PbiFQg}ZES9b0#qf>KT0T^ECm?9JsFwXOqTA zH&;*5U)lIoLIroDHjegn)&J1QjyQ^xog5p4;6$))JF>%!@QGvb~iRc2kVlh zCQdqyRpQ|ZY=EIHv7i)hI%W`GZgZ`Qgzvoz;Vo79K#13I2cFq0h^dil!9-tAtLKqi z-!B1$=zTnzLb2Yt&o<=SZ_2sa;O7BekNzs~m~xx;U$qXxyvLIt^0f1Z?(>M(@0Zgr z`X_pP@F-NRcymRBM297$i40O4B9P%<6(bD>oA^|;-tCGoXZnpgUne$->ikMhqztKa zzZSD z4UOx27V3`voa1ABEB*2R%6d03kx`258C^5UztYY|SgK6ZI{)>EY|Vu>uG-Psbq6QU z=Gb}IQ@b5-1@vHc20JD#&)-`$(AeR4) z_?n%da5(X;ae0VykM%D_82C6kRp0;169&gxc}|W0ArhyJg_Y}01P|ox3q8ooux!Ho zsC4Znl}8)LuR+ySHWx#mYZt0E!3=yDjPN94^dGA}n~-rcSuQ18LOUgvhzpGVNhx=4RB$3rtQ)WiWL`w%G|+ zGPRZ*WkN<}cYmIbs%f3CCowfJz&|p<>Z2Ie}Q@iijZbDs}fDY%jZ9f=4-xf}@h2nlRjux#{n&jvDTxTRy@u~CByk}UpCFeng zt+=h5Yc>%k8`+YI?B(;d5bui~A6HWb{uYI_i{M7;r^|n}HW7mkM+$bIMVNj-KkH9U zN7d8ldJcQRDhiSJ0&Y+7P&?czs@^ZR&c5&gS`M~RZ`qt<`c?tbaw*>ZmLm$O8)5`g z*u!!~8?^BL^DduDeI7Gg4!(WryH%GH6t(^J818ubglwLHEo+=7I<)m@?jh;LL#owg zfs5J)b$!7>(pvjLNQ;@{^Ob4-E*-*&>9y_Zf?eJ)zPLiLj~FvcT>1I+k8@pGL^+ok z4rui`;&$u)hzOo6^|3Hh^iB~zOuGK9Z!&t<4qhd4#m*p0w!Z5-?a>Ydp&2#tH$Rq^Q9c`GWX#vZ2;I>3qz35u%8_=srjIlOI@ zOst2S0!+~&j(0yJqzTF~p#KD69f{G0gJo}K!^%QMe|{`vg3OG#3Gnm8DyNL)HTKkB zlz~S3h=}I+Z6!TC;CFQe!0$X(BUxVW@(2$lp+S$uPxyQ~`>naTkNQUw%zW)?HbV)$ za)f-GeXgpnN9Rx*xxOB*0oF*tPOi1R_Yc7_-}9z|L>A~2OVXTr1Gy$n`u%S1`i%~v zfFF5C%eSAIv)3u#oWvh_0-0GT3LA}nmv|%&qKMY%+K`uaQH0i+xwPa=(G~&4j#mpWdZ$ z2AQ=l-e>>{m0J>JuxwLmuvs*NI^7a%!&IWpEQ_LtaBW~)ynWstx9(GGBsnD+m6tIl z-c2*?#ZfESl3~u{M{CYJY0Ngu$dv8~+7=6iocp{le+b4twaJ$lZLS4;V$j)n(y||E zF%HkPV2rtbr4uCfE*bu*f6mc~lh4;PJ`J!qpelYo>3PR^9*?A)X-!COvlw7@VtKMr zfrVb0DLa0ZY3UzNypB-#nXrE-t^Jp{LlrPOu}PZm1`oIevo%H0^L z(^iifUq?vkFv{?3i2a)4_{N3j?Zbo&EgFxlO-?Kz)Nkzq&c{d9UW%&a2 zD<9vaw@x~X@_P?wANjHuC%QFE#lk~tUlLWoHwB_GwPWAql%fYWY?k%D%;&82QGq8I zR|sOO>;)Qo=w*P*^ixBa*@iE5pe_fZ1&oO@cj-}4kHqLJoD>oJ2Xe@eGlcJob31bof&R#abhlYiLc%Q2OL~<|` z6as_LQ6zs@5ei%DCdEC(XLZ(IC1p9qlJKGJ=XX?fBvb$E9pI)3W{YoF$7c7*Xg~fG z5^rYu*dq+Q9HS_?i?o}@#kLq3Y3==aGH%h=3+1va8F?1?5^=QF*KBLjYwDdRVXpZax%ORRqd-ic-R?=1B#*gjmQUWL@oQ?>83(H>vhQxPZ zP93H9p@e+rsUl{HOK?c#OF#ig&$Cou?d0$84?hwzE4F+(S!`n)REi!8s_4=_mPzlK-F}EyC z7%lneCLXEVGmlDSX8T-phN`;v_ku;&bZwATf{}@>3y=&5*y~Rd&pjVKge#YO{Pfik zD}S#;ZrXbGd-H8s`!{~o7K;Bd^1qUdZ8nj;serV3S9xAT=7>fg!XkHQ9%kxJc=s=} z|0~WI5U0=E1|}}ykRB9qi>aLbVgq+vLC(@)Sf7N`<$rAdpCZnJc!u8{z$iKBB7OK{ z7Vi(;+^F#mNd@(v5Gu$M$ zcH&R;)9&Y`EhGmfqW*uc=?|%a{~j3s6M>5ry}jap^7_66=Spjpu;{HupsN8kXt47F z!@S~*R-^)-0M)rszKm6|YHojn6a+{)pi!$J-f?Hv#s;0)6Bq*~$sr z6&^J0pX5x|b;x|oN$$s4u1C4<`?(gh1BJC#FF8tzKIXlE7k0umS64a2)dRbrSBk*z@!+lbZExERPkt@9f0&36Iq_j)v0M8vLm%J$ty(y{* z+ZT7R6r%%s5sGL9AOo3S7-o^&ioX?^kao6F;IpFRWv%zDSN2b4n#dET;gf z-h2;J+pbipcrBc=zlzjyGb5LL_tID7WmwhmI{`tZ{7dte%d%1@Qrg}U{UueL*F#zn zmg{>_!cxj|gI0b@H{dQJ9Z!2QA21%zn}A8JdP@m$1dOYV+PC8HfOy}S;F#afIYy8eGx(P%Zx2=$iNS_`VK4jZB@5>#x zNTqqLOou0wd2K~{W=b|w+4!iYS%dqjYto(rWB`YLC!I@Sr{SFYUFw`XmN-M&D6dHpT+=-sG3%foNS4MhXlA zTtD;EyxWWC2L9X+D8BWlnx=_{w+CDo-^*>XrjU{jqL<%QDtr??{<$WG1ML z@t{oS|5S%G-!y2$G<_SCbn?brK9lZySU{Oid*dyF$0xJSV_#dR211>Odmzr+K*7-_ z5ft&%{$@hSbW&LN!O73seQvgM*U>Knqa52xE%h~qU65c>S8B28keCjHs>GeIw_9{%1Ny#U^+Lg`ksr_C)ms?#ghT5d{YThi7t0BB(}~FJEqs zps(S+w-c_$u8+$`C*@K%bBq+?x4p3KNX<6rL8^6`JId2hP2zH^r{$Tj6!d8Aj=%uA z2U-qzd3Xg!cT+DtrrCS%t7;EP-Tqd6xW#{Tcz8D=@wj)(`RIm9E%hU#G=Anu*tK}% z*R=HXoogq`KDY_Ia#$BtzWgG1K*u&YMSNS8iOP)eHVVz-+Gweo>~ zMw5vpjsu>38?6hly&IVPs~wNE);53!U;-$z4WwpS2|`t@w^Z2S5ZDHK+I^RH?XWh+ zdk1u)B*+d9Hdz!C3GW4)m{D7FOw(LLFkB9P7#5q%MBK$Y*Qt zT}lIJ$m*%WmJ~Z-s0wJ16DrxS^qEQ08I|B46s}l}H;xO6X~=ehV4pnZU+cQoYy`Oklp)__~|q^o+OQHfFFfc=!R3pdW0^FX>D7T2*xt zzSw2-;p`?Z0*v5JodF+n)jagfUPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>DA7DvDK~#8N?VSy* zTt{_)kJnD(7$5cxNDLi64R!=@okgJe5Jw73TG;<8FY1a{k4LAXg& z9x01yB?w4`CJic8f)*z-NUcDP)KVejjY3dLMaxesTDA05rj3w*($MoWGxyBBGk5RZ zx4Z8r^U2_Yh^bOEwXaq?tQLvfyIKX|gF~ z9EKR=+@>JPZkH=N$;u-zbC+OmhjZ!l=_gHTkw|G_T!701^LQfb&6^Fy{JaP|Sl}{e z>Q0Y^w!p7F(d_iP{XP{KG@DB`v={ZAg^*4VRKr)&L`S)d$p{NNB?V1wnK1=}@>%TU zQN9V?K6TB+r$a5einI&6Y#Sv9SUJtvaeZDMxJ!g9k`+5=m*^Qxre}|*DuUBdW|^CM4lS2g$+ktwNRZ2o9N~khK(MZ+ zqMaDI^d7TTl$WRupX0dW$n zYwV~NtJ^`a?0W6w+Jd|qpV^w&5u4JwmdmS_2t_;iMz4(Ql~>JyGiRkBSM`XeiH5pu z9nYMfu0?G*W=i>|rBXI!aV1S}bPEet6y}xp z;97*&5}h7B?`e1Xy{;+C(u5`E)h$_*HT5qf~$*mSUd&_ zcgcz@sC8{W-quZLPMMMr(F}NkE6w1z4CMq`$H+>W<_b=O|76#UnZflj8m3o zR1bHcb$*o{^LP=2dR8LqWqc)x7-G4o_32g0GC@URS{Z>mX(_TV7r2CJzJXlWoad| zgHq;yK4W z+(%c7<72w1_R4(AESo8d`g)S#@qtH=$h+NMr%T6*ys{4N?4c$dVCYA?JKsmsT;}Hw zA~CNb5jfS^=BX_#v!i5F@8v5k+Drbh+#brTiK1?=&5rRdQnq`kmg}B)W$Ie{S&KF7 zP}%r{?y){%>t=S0j%R4sBXL&SmwCAiDvQ5KkdNoBG?Ed;+C65rk05h8F!K7nA5F(qr;@N zY+{*#Q?|`V4ZUJ+!|Y8<3X#RIxhIFt{02$;MIsguc@}GPJRx404F->EIT;07w76RG z+V($oBBz_8<;2p5vaC!`e6^_<=TmgbW;*b(nTWztM4Tc1AtbmO(# zrC93=eOwg^F|bZ}EPVenP;N^3c3EQlCpYnb zp^szn$H{ekjGx70Qnib_u^jS8+cA*3*B9+4Vf$`iS*G8{>4ov)d&*iz%YCv~#r(n6 zrEI(7$odtHgJR2}QwPqR-FJvoFwfp}=InvTi`h18Q;)UwQ?N0K#nZE?>Z*C-F^ms_ z(Qnh|Ed5-OK~J1WC6L8(SE+ha`WPEu>5J`q=3B8Xdl0F4RhW~w!DXt2t>J6TXPakj z2t`lgrOSvIqC&sR9iu0SIUPi|A!Vb(h)Pv`yWvoF|HPQ4}1DoDU8oXLP%IGuFp3`eSqk>+5!7 z{>0*}_3_R5i75ZIekK&$F0GH`HW9z?L4VraUcb}n_PY5bT_tWs4)99WtY;6Mavz8m zVm7HP1^SqC(9cTo)H;5gGluX%LwnVj6?2o=FUpZ(VijWJsQQ>G6#MAfQg-zG*K?8; zOqo7iAZjt6hrDml<5RcUSgM#<3`O0#mGnC_A!29m(Xtf$SxEdfV;PIOs924~{ut1H z#>$mgz&x2`tm?KE(;tV=HqW-zNm!RhNa&olQ1rQ_3ghr$Sz*rMi9}+8xb>{Q?)I*lfoxX4Efcs|VR*!4Cpu<0fMf0a{oxCYXeTRJ(pThIG8GaW704CRACByuf)WyF0X z$DwWJ(APMKvTyn**BW@`K^vhyVp;oEXv^3^k-HIw*RS~6<))>sxZ(QC<*3hzL}G-L zMmAaRys@MM$O0>re8BBx0zCyam0#j)|uNYb-b!l?U7aGJlnBPve7dC zvW0%;&atK&qX?_%vcexfzO`U3?)XLoJ(fYVoNtwy3o)&u%UY+%JUTvXPAsb%l9PU? zN3YzA3h`%DZ0n(Y2(>Gc+6?W_C>U_uR`$*Gy^8)q`s>wg<1rD7`7rrt)Oj}GTcPh{ zv(E7{OusxLxngRHeymO3oMiZ(q5(HK4R|_L`dFmcR@Gz2&$B_FQ9^kZ570h+RgO{> z>zM;JE4L}J64bWw8as@ytb(;E&#DLC-Zwq?cZQ6eZkOVe?ZZfsrm?B}CKkH%)icSV zTF28@I*9q*B!k@S!+FEwBe#5Yzk0oXuZz8->Y1NHsIC~Nr}o9Zh<*p{VW8>jdbp`= z@;z&bRV=oVvG9G!@bj#0S0qw5*2l5<ldydk zbqDq)`G@8E9B{jIQ=O+D7J z`5BL=wyEptd7>G{2f^y|(*-YAD9ndv#J3!|D%qy=F+RT89PoTAv}NrKp8sfve$JCW zjur74El8eq6U*vROm&$Wma67aPmdbP>)_r_*EM0wMu%XZ5U0;Z2z|^TPL8-7s&#Vu zPA7+H$ZJ>m%;SxPU|&LYkq#b8^j_$TDNvx=sDpuOdp7g-5mm(y9*aU-20LE~uU*0@ zNBcMyf5JM}&%C~c~!ZDqcB`pQotGZNPCxIbV-C>>9p5%0T-Rq!ELuC6F&?ghC0 zrh9HT#hH`Wa)v$d_(|S&L~QD@E{u^WREejjqvNI-#s>#yqO5J5Hw96HbH zTwxoEj<5FF_up&FIh>nUNU<)SSB(9*&%@+d6vy%`qI0=KQ*;PF?s#r7d+a#L)ALNu zm>jBge0@ac*{rUMuhq}@i{)65z8Ga8T$eV@f-yIToW2(*?^DT2AC+h$UpF=uIUgKW zPI&Dqo6$aw(I3$ntnalO@h28%Phb0~bv*q{$h6%qt*_-7Uf+ZMv^#Vm)$8{=?75;s z+zMGZ zV%Fg-wsP6vG@9w;%2Y%o)yf|)AhORq>A2{A={aHj6q~WS5HFhL_Nl};ot)f}#VYDg^yX~1UvaV@=!)jqXsob#ESMbU#Xp-5(RezWR!0=F zzWdbm!N=X-^Kylf%lHQ5h)F?yuF&Hy{+JEb$7~MxzUj8K9qf`R75a!snHx7*(f8Sa zqRP*~H1gS1>FScj#$p)i)~%+`=R@x-H7=HdKMRS!W;|n2mn>G}u|ImWpRsZ^hMy;s zj8)yXV*2Cpd8j;_NF)-8YT&BWBpA(712r=tf{k%T+e7|lkD?)dm+k6|_w~5C*-JRz9<)Y)V7^ZuA z%gI$v&^EPBCil_$6@9RnUmc{q<}=m#``n(*0=_}_8-`P+k5e74Z2!7&u8;PfVCGw) zEo%n{j&JI7rsW^?=Rcj_?(%~{Q$_=Qc|K6;b0U!#A@%aCe)?9GkR4aXg5p+?i&n~XUUFTOVa0uaYdG`E3r^qOOiy$tufVQZ)Vfi zQEhoBc_kk_Iz7{I`sfb5q9W6wDvW;(p2-nwu&uq)eZ<{a)m$Gj=ICR-k6Bxp9dC4a zjjZ~JNEr=a6=@;eH%KHVgUPdtjLii?mg#(Xb#z9n%CKz>M)$U_s~a>EtOJ{YO8kLaWmO-Kq8SCEu+b^Ro9P;l`j_3Zme#~edA%6Ymp^P z`jPJhQaR76`*`Vh5s8qT^gBJduZj*{mEHQH^-hL{%Xgnwe=HFNT&GyL>zGE(e#Z_!`T=yiY$P40#P<0|;Jh7CeMGAC z%p)j6#71npEog_~^Dtv)mhXt$ESvd@#JZ+}%7YttWx8O{7xs0KqU__!z%i$kUU z4sdt$Og=tK1nl@iCfkq$6^m!V+^8nPsO;1w)-KI^3cG!;3>qAHatN}dm$Q|JbD*O0 zGnTIw$i&8r4k}YHsFQjv7dU1Tlge=}cC>6=;izoCh9glb^eZNh<*N$B!(YLljDD}{ zmb0X7S6bwoyo(Jnr`wFliDJ7>fo+r3)b^wF*}3o=j|{kbdky7J40KUeX3^nNtxt24%P*(N2=r=@5=PU zepYQ4u{M>M^^N`TxcK&sH+)*dG8eT`X;<6HU4W%mY}}xr*=?5Qrk7G6t-{7 zbH8G72;lcJ!&ndW*W%Z-yXl?9SV`0~Kb;bj$(MS3Li#L^*jE;@g6nXESbuZ=4~S39 zA0gDwoPDKFdy4V2&J=PA8j0RcnzoxF(i2FiZGrT<>w$u6>~;~P)laB z%JFMt+$5PYtxI={aU0yPqLb7@ApghnWF((cOZe_?;t+%V@sYm`#)NcZ=E|GN+%#DA ztePp6+p*-d90hX)2HB^@wI|^UM!@6jTP^52(6m;K*%@HJV5giyw`2VsD7O;xPI4cw z*Ol9~ustOjpkJ_$3eHb1kOo#^yB=bv#6AXH*U{~|Ry^{~5i$EE?Xq=?)ye5QoF}$G zikYcGH7Bqwns<`>4D@r+;uE5-ox1jsV1G;O)YV6DQ}0E}^h=!1RkMWc`#6($a-m!O zB+8$*i*}F_{UWh`)r$`{iD096YNPue*mAGD?cv%DgDcmw2h2Zp=XK`fzQ>=IKlZ=x z9&00aclOY!1CO8NrEP?|emc38nD+tw%IzK$2i)}42m2!ol*bs_zLl8wy6qA<69IBo z0}RtH5mSt;D?*OcI)2H3kF*l=e?Y%N{>1uJD^H4_eLB`X`vd$5P`aFDDH`M^(ZNh3 zCMU=V-;`)$0axVWIKHtUXUEjTF|%uX>8j~dOfGh6+@0ks1K&r6-Elfug_ynr_p7)^ z?Zo7coE?E`slFOeny(!ner~uB2d8f}U5!kiYv}7hWz`aI*IM#R(QzRH_OqG>GQdi* zN3^w{nVjugi)>p|JUQcxdIa1-a`ufzSsX&-^c`Z|Y!KSDfK_M5?sGa~Nwk-Je4IYX zL-x5@+g8E<0BkYT-<Gf=G(0uhbUB;e&aaPjjt;hUbILn7WQ9!#vtBx`Z@(@RUM?&$ocUI`}h@D!A%UPVy%VO?2KgTq^mP;HGV}j;YngNMKB9U16=;f?j*2FG% za`JZ9T#e;FlB_S&7Uxtt=4D*f#V3WWRL}XjVwbbjF-4U=M(0%#nU^_CMBOA*H;HnL zY#b{wxL>261!OTc18T%Mkw_#Gi6xF(&bImcQ+Vpmol%2wZ=s}4!=8MyWQ*u-H?mB0 z-tlSPd}+3eWrWY^tdG_0c|Nob(p`+{liVlsBI;+PU1v+_moce0Xma+mO>SVI%NenG zHGuMI7vAej!^>atz$f!3`cO3#w+j8rdx_gLCKx~dCabMLpXYb&T79utbcp1}D0js< zRKcG_B9TZWmJhiP8_-EKbY}DA<^*WZmbg>)suLrhKIzn|Ptjw;J3VJxxf69!ZYAd3 zcF5pcC32E-ecaTo6TXJqirR_jM=7yC(l``QEueP3+lYL@u_*U2R&4WOO94t84BN<8 zAgzAa{-Sk`fC<08+ls|)V848OOhA;UQoC}w;x{+YpF|>&NFp4VA-APGBZ_h>G4J;LZt$(rW-d5&^>Z3-D{3dAAEm_p=;9DTv`TbyUWD*!SI{>E z8>O3-L03>aZkIZc+Z)hFQOH6xZdnP*_bXr@@v2xv)k>X8&Y~yh64lezJmb~loJb@R ziNq2F&tq;mOJ_WVJHN%`@z2h>r=hgVoqNX(os^|f@1eYY0$Y6eZcppxo#o+&?7?$> zi9VA(!Op6N{4CfJRmSSm#{pEoqJ-=tql!ZrJXJgo)uZuXcu1`Ek&@4J2K5Q+zQx7B zlI?X}_Q@t~v01Wj?Wu885p_QXSQ58Lzast=`y^;T1LG*Tsw8(tybWb3)Zs)Tkw_$# z6}h{!FKl9Ki@Nd=f@>DFRthX|#=!*owJ40cUv=%T)Ho*+i9{kXS<-Sgkw_#G%azlo zpA7ypyAz4T3xlP!oDDx;T>uli`_=ArBHtsb#{%sv*4@5zS2D4J8F^kxRHNONY=~*G zEd04utovtnClZMl27o*V=WUyUpHIQPP58qm%r@bjm&0Xqa5cat*mD;g*$GGYV(~ZM z0I=Z`Z~ezLdp>sc_kaJ1zrO`w6~Hgzzjyt|55G0rB3rPADmAvk>U97Y9EXjYn($kj z;h#3cN9N#H=iufh{8|%kX~L=|G@9_~Cj4m=9%{nFO*qzsA5Xy_Pr*fNVM7ysxe5EH zVDUd`0pm5lZQ70>|cvY>q+x`(Ppsyi;lwusB_6?SdA9| z>)|I&cw!Dt&%qzf!TcQj98JxUsl(W>{cD>Xg@>@})(?<@nbxYA*31>3Y<-lrnAHFm z^S=$pVeQ>b_`OYV{w7%6ge%v>6PwBMd+;Cjon2!6O95UEaOH70|Fumxy$PP5gG)ES z+c(1{Q}D_u`1TZ4IiK_!uINEpoCBBQ9M}wp=V%7v97YcT*uBeE!w$fIhmOJ#YzoUT zOY$(z^DF9jjxi+9kHdvQS3hW7{X~hM=d9ko9ainxNo4{j&r9!y#_w*z!GX7a?q#s* zTP%MCz^gg!Y~}JcfIR@O0~%0D4FEU5noD8L6|iOR^`g2vo&Sm!3gOV`6x6C`6`r@q{T%h$py z>@+y_$X~%DkC0zH4c@SBE6xN;RM*ISaPfTQaX2>@&#N}WRaqD@hL-@m7PAw^@cB*V z=*EcQp?#G(v{x;SLmP(%`?Vwv?K_T1MRe&^c)EFg6TaDmpG-lw3H>>EstGfgoHxYM*&^ZwgNegdZto+UPH`k#;PI_JeZ?t5_Jn@V>iX_9$$>1CCuwg^HUuUI?={z`7mV8arXfUf8o2 zW@iD9;ayk5o*l4b56m`TmJ2&~!tUL$3)Qy6H9O#%U9kIoux=KPZil;Wgrj%B>s|u0 zPI3&6qK8Le-8wji#%E#2PO5^hyYb%<*u4k#u7aJ{VADUp5N+;;Jx5^Y5mLY`vUex! zT?e}xa052IorF7f!S-3IwR;C_KLUH5&!~xR@1R|~W?}Ch(%ea-!cz(p(~CI2Jcj?S zd-T@p9z8I#5vDidCbSv{QP_I_s_72z-3l{byS4Rvqjf%R9=!g8t(hOKo&LhP(=FNi zSnJ+fum8|_tt(1Qt)9jSfO}6IocY>cwEpj+=?BQvbz2X%E;!u!%1fqE3pY<3Y~6Tj z=0{iBz0i95shNM@ESt97Uy^pGA2`^$_we-}d=1wn%dHO|Ze4fP^g-Ht-PT824;^kz zZ_hTrdbsuGU1sx9{C9Br0dexN>8}B8LSq_Fm+;ukhJ(0y-LtE07f5I3_&L)ZYN>TI z=}bTLA_-3GPN{pr^hWOK-LiS1m}=cCn-`hA56b2S-h|etu*;8K_w2>fcwFM%Ka;(e l80F6bMqnB)oo*cj_1*I41CG;9Pf+zw?385tP zE*&92=rtkX>3#Qq|2ebg?Ci|9JA2OVe7o~ePe+yNHp^`i5)vwPH6;TQlIxLw`dl}! zk&uvHzndoYMJun0ZOH%Vw>;Fq-XtW{U4I)X!o3_q zLc$1ES5kQK+6tR1ep(IgZGwr8-* zu6{w6jrdtt|BU-*)~4zRss)kO)y133Y4>e1pR~*+q2fPv+TOY)WReJNnR{+Xon+Xs zL>JMp*D)@@NJ?(7+Nh@1+e|GG6_6GEl3gudB`VnYjd+8hj)i;hNTGJAsVa3+C#e>P z9QDEN2_Ucw84}Z!YfLf4O1I*3qVraVAA_!lN5>5TuR}V*@ZB6B`tqR_hheY5;T#kcy)b-z zICeC%e4rtDJ?~i~^Rnh-iMl6qeVY3jz(g#|)+u8-kfyerDJR9mqUu?A){2=Pw@0q9 z^ZsKF+E$D4rS5-tR>*Ze?N>#=^EhA{P-H1t`Yip8xtvWXYgSIK za?I`CM6YRKPk}KL3aDmRkC8}i)J{?YT-W$8f0H}?b3}KBP^tl-$?Buh=tBhbmBMO* z_MD02a7*Ws9^EK_B$vurAt=0>u}qmS^@D{#`nlS1KBuFRSPQ(ZZD9MC!_oei0OEP7 z^;XIGO=-&9vUCOUy+xvyHH!+Me6lF?le3!82$3`PL##)f@cF$Dt2yi}4ouNZNlod1 zq@}4RzB2LNPusI(PysSNu~e#X-?0GfjGe|sP?9_EAP{@AM(igdfcxM=W!DOX?M>*N z&TGj#ETR5RZLYIToe)OGIaCq%%IBR+Q2(OyN2PU~V7f)PZ6oVpbndUTwO2zq+_o%U zv)qO)w*w;33Oo}JTb|W&)K0f>3$>(h1<=suj_UOB*V>Lv3bpOVz4OiBN}uv5*@1j+(uaAV@Z?++^{!zdX&egb^XujAYR`q zQ=da0ivw>}9i4AuQh{kta2A74c3Vwoe^6|f`Er-83d8&9E>_djED&6F4IeZP9py$| zzT+-{=_|RKsl0IFxWCDi()bCy{wfNQmu8DPoSNd$_Hv6=kibG;b4oLN-A$fs3zp>{ zX^)f&Px;J_uYBS66AYwnwf*!0SZC+;z&Vn<7xiTHLCW(7fYuUe3TP~nU*a=yyuG^q zK{pK(nPhEz{rIF>nJmiyM>^64WpK5dP(K{8XSL9^@%rh)wRP2GV~SNQSyX1I`u`W^bRd=SRBni@Nztx4) zQvnV#`^R($H1o~&R8BwJP5*uLQ?`w^eI`~F_Rr2%;0_Jx_^}lLi^-wg(D4@8pr=lx zHNPw>;6UWJbo}bPB-B{+tG^+$3NBt}&k}MD<-uK}?VCguQ;`vRWq(R3Zux`cfV45j z@!=R&liscAriqGzTHK*mx>#xpHxI!RVJ&1JnC9lD?J{K(;*`ptJi)eu=38nXcdU0e zJLAS7#-#={rQ(}xfdT^gCTE!$FT{h(w|K3?gpOiNx1TFJ>P!z)%{sL4Z=&w99eSXB z(#Gpcg`gNF=3^9X0p^H7cyCN9RI8!siml&i2%vB5fEZ4xU90~x^QeG7a{;TgnJY36 zb!AaCQy^ATN`S(!uEq{?tpHTRkzFaOl} zASd%j2TQYnA79b|qc40%u9LrdLw!vGBB_b*gy_$@>Zg{h@n`a->#e)7qgQmzdCtLG z6s8LYWxV_sX&6?}M@Vrs=r9CIT9ksMCw$E|bv+C0vQ?I(ceWnd5#K^V)}!MxRKjPQ z_IVzqbz@zCO0RR(?B^~NIOLA!Q-1f96W-_L(km+b@@oQa5FPE6qhg3;TW!Z{3_uC2 zH)~@u;auDQIk1D!J$|{bMt5;5S)FmP8j{KWe)Q=Hr{|RlNMUDA_vLB6n?)jmqXct1 z;^7m2iD5J3>*5EI7CFsRcc8`%6%*H-PcyRgl*I!%etB2l(5U3gjzs^UF6`88;NT-j z$I(LJhsvNO!8NbW0}WZ_tmJ2b6LmB;+EVs&cb&HFRqgf-T}}@*vI~^@3@zcIOgasP zrr8Wt^5KH=4@yYm!`F??*MMA#a9(#Sx8<FxYb423_d-Af$klHXedCbe&zJaP=>PT{jMt^n|Eu6{Q|KH53E={f?K zBDp*DBKbW;_bUryI{Zw1tHgz*)XQ*? z&Oa8<_Ir&yMdq~_Nd<6af39zy$Dgqr{vRMqpFbV9?38MVW((8?=I7y#IQ#S|k4==YA{kgI9I$HuJe1w=y9<%?;o!eJ=*U^XYE4 z+SwnE$Aps4mztb;cfWJ3+3X6l{|2||INYr_Og@2K@%l*R&A*$n?a^>>4_nQi*81ip zyfMc^S~1hq6QA>TkLDLLq_XS7aj2?wBE;*foygOL-c5*em7&N!y*jnIZ|Tvb*GQC| zSKTE(Pgakz9yq=p4WjU<1M$3c>$TW4hvVr8c3aS+)4YSX%lQ?~fxgzQ(c*L8;hb|8 zB0|PEy|HZiC1Le-YNrUN2!bE)@9TBt@l^c{X$eWe+)(_}AKd~^2S$nByr-n<17rqc zLBj^R?e_yDHR!>}UctyEtADx`1sU>^l16851=J|PRvNwV^3}X^f+t?AyHAg8qe@L; z*)pQhOC-l{-n<;Up}&^M!8dEnGo>p_Qs#1%>}=Vu{6nDePQDGD-cG@I{Zza`y58BExur3a zK4sJ1j53?Rv18Km9Bv9REL&DcmJTa`}XUebqC{4ngg|9eX_6;i(!Zp>F#BY$lc%lX$k~Nm3?1=C)ANQp1yLuy-9v|UV8^3W-R$UZo zjJrJ$L}aj6Ui2bdwz~GJ83N**K&xW1nNLOa{pqWA$0`riE%23}$I9hW(z9)y=Kf2L z7QpL}8k!?^n+*z5KVIN67@B6wtyxb+elj(T=C7Qb1x`MF#7y>g?7y8<>vjN7h@L7!&wdS z4-RLoymJD%$9T>^n}`PG(&0RlA^I;3FLmT27Jlx{7Ug{81YsC(`cWU-xnM>em5fO4 zAK#dhDY63A_2&7v>5g6V;o9ak@z}@0N<+=O!5_+V>epwxtk}_;30s5Pb6!Ej;}w^a z0|&>4s~r=%iX3?jxP0mB0l%_|cfMq>Bd~6B)5E^TqNqN?R{B4tfnl}22$9b9h4 z8}ERon{emsKmg}yHl9^Zc~+)i%BnP8y`aWUU|_nQFn=m#DE^woze+nwg0;;qRhK^|vHcML_IrD=vh+}N>Ydb0 zY|z@R1hq9AZuW~c(xQwOe#k&+1bZf74E1SH1pZU^=EScmMK2e&{neBpUQ_-yPMe-zD`?H*oQtRTKGMN_jmYByZ}6hL z%JzL415(fHp?E8H${En=8YU{2>5haA>Kgj%VGMuugbvdA-frc%m^_M-wZbc$uWKpI zIG?tNXq|e;&UYssMoJD{BImwi96N+~4ml{8yB~OQxL>YG_V8Qxy<9iD3_W%mN(y%N#jS4lEMV zGm3uJB^4whXz^R_AH*LdSjP<6;NED?Kl6p(J^e3n0zjeCDH5tS&8|ND~G1l9M;8@zkdHp zFl#&a---W~3&|Q6P08KO1k24Id`!*w+DSl`v`m+D{7}d6mt;PRo&<~wh2#t2ywp}h-*<*HV9i>KtuuzxeW0ehH2ZBf^NC&P z$yEK90Ec@;urlfluLWtyC(dqxQE0Z#mCJ6OG|E@GtcLz!47S#zcpyCRWbONfBPn5D zcz?rbGvT17odHPh$fCnYCOGN`xQCAq^d(^GjIRdSzH=AdrSqc{|7YE@H|`FH94gRD z5rrY8=Vpq^D0kW}$Ze^kzjF{=x-(?Ww1=G0nLmg#ClviITsSgB-k{r=794|zl3a*_ z(Wo54C^eZ)td_#=9d>T~C&8Q)ail9-IAu=43#2?S^>{8+NRaV*w#p#xRmU%(){v7( z;i1Ie#S)L#&E(YcPO65!{Bjn48@gNOJN#@^PXh94&g96V){2c~?Xy<-iAo4r0Lmk4 z0UWDUn0Fbgp_ObM6l{hQ*qfHRpTuGO<#}!U`zx^n%mruD7I<0;=TjQ*m|xBz<_c?6 zI^%m-8T~r4c(_iiV)0=rbhQ}>{KE2%N$m20gq2Ug^zC@X-cjb$vKg8=;+gX{WBfBd zBwx>ZN_gI~9klGRWQ%mQ`!@Y1ex`YBa&N39Y*v~9LY_e;!UdVSG z@v!m{pl^?taJ0ONPB8J^APV+LMWpH)PkhczgQ>5BH|W=1;X8}+v%68A(Sl$JVBqs|9>#Jr zuv2(oe`$rY+*?I{gUqUoMqSo!t)I*5;x_L z-VJ8LLQ0$TP-5*%)Z(QB`SN4I-% z>jzYr&5_dMB)$h$$IwJy=VeIuKAvIMSQtvDl0D$iiKUc5Z$e7&m&`G-&C=SM-D2#Pb7ALF~60E~hIr)DeCI zw|+x2pmI(~sVlU{Rs%HD`5^)jX!cb`ZRpxUubGZt-bvzYVyp_hF@knJom zJs<1#Tla|BA-`Fy`p*>#@(EUq$`clU7AZShXBW-OFobNr3X#QeO+uKX^aRs3@sQx~ z8XE%`I!$Ia^c354g7-OAha(&2CpnEux=kI7(MF(6!JL~thqK9(KfQ6D_+XNkX zle~0vK1WK-KqEuIpSox&=$=jh!yb$;<`qxe_x^zh+UIrEhN3vS7eZ}Nmb>7cFq1me z8$^_JAXZLf<80&6HxD1&?CkdJCZg=uE@S*gkL=?5*5_tP#3Zqe!vfHfoueE72!HWV zdNb$_q7F_ubb_Xj3-Z~bL`82bhntHcT6rUaIW%GI=RBlTH9Wu!b#pBXQF!*xsX6O` zbs_HBx0u(>xQQ@+;5L{QrrF)-hS+0K-4H&TJUEb`2!VFDIAiLU%o_%V_h6#UB3+vY z`km853D*|?CD{Ln(?Qb##4rF1WoZtyO?sO75m57gtST)yyO*9LZDqAOVgWHk0>p>$&X-~yIqg?&@ ff6&y%nM;5)SICE&pMbxWcoKDG9iPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D8vsc}K~#8N?VUf5 z9JzA9SGf2APAK?f5by%U`T&EAHa_6a*-HkB>|6K}=QajXyk9{B z+!t8*3oM|(*#%zLKay1>tGn6rU&|k)8FqECSS(hv+1)dyqszWOKy5c0oR=0b#pNzTTK&L#!5 zODYBl>oBpb3;Zw8p2e#2G)>C8Q*j0@4x)<{>xV=uyC_pW5whin@Y7rp#YlR+_Rr14j_sC3~OpP z8HZCenw71fU>d^HIC6y>ifnT}yCmFX-NjzPTx#c^gG*o&UfF3|@gq`-pY$)&MmE{2@S(rSk+Gn97r zW%-0)h=*a~XnOPV4kqOik(ZYwhh(*Ky@d%;4k!nhT&2{?yv599Bn|PpX#wZCxSH~6 zWbA}R{mSME&H>2;c!RXyg%c$)XR>~!H%VfVym@|`HV87OEz_%USMSAHnG1l35Gm5R zL+Y{!rv12=-a>hkrz~|v_9jAe+^)7pP}v5TEEhV+3Qk!YTKf={y3Df=pKKB<&0&4N zO_y1?!}M3%q_W^zV2Wtyy4-7uZP>xlpt7eSGXfXn4Qu>hqLWaO_IS<1qc|EC7>O=C-i5odz{|bsUb5ST~gEz$VSOH|hXb zb}XPjTasXw$OSa*<+7eLkYUZow6q9RTSTNqEDmd#nUYS}OA_>fG4@G^i53`?i-odB zorALEvSaV?XmfUUI!#=`m53C5^gSu<)ak|Qm?kauCeZ=hNA zXL0ZL#qLv3W?@c`=&+G{@GE2koVelCji*bxXuJ9G?hWKCy>gl5*UxXS zpNiy;avJX`|Ckax+JuXp^h*Bec3BQB##?COgVKQ#$lDKm(51(GVVu8`z z;Kl|VIZG3eMfM2B9%BuL0~$l#|xJPbGR3fw+_17*ozNLiAj zgr{V>FCmZ}=pS1K_*FckcJ*G+PyTUTr9)Dw%X_&y;xlJ3mE$^9jZ@N0D_yciZ>e3a z!(K^?1Y6b;)DIbqO@~1cPn2c^CSdsY{{ zN@1ZgE2S2p7RmLvfDZ2!&8QPbHhL0?_RHL0xE5Q6*DNLx$l%cj#lvdH;+NtWaDw2xgjX0vnA<@TX96kHJkOde;d0a&u>fwMf?6}Nrfd{|A#^*m z$Zf%^u7xYGVOYc23U+~USbI@xT8H;?U6wG~zUYA4uxM6VEM#kCk2(iMG8lqQ!?4i{ z#40>mm?&X&!lMNR-isqg>rvKZ375%$Yk~K2uD}K@((pfCErha$TLewq5n9}=B@4M0 z>&S{0jt!M()a9hZdyOZAwl6v%gQK3;BevW*D0{dctTRiOHGHLs*Pd!}jc_e*Z!4Y; zbI^6?_~Ys9;zOs2vv{I$7QiY)N}ad!HIwqcqB|b!^a|7*UaZZG>M)b508iEW_<|g}Cv=tq$A1}7G>~EUFi#=eK0H;qgW^znT7fMBWo622JnoDUC5tp-o1Ocdx@X?x%m;lGm0gcN>v#Y z9)??)g`<9$j#;kjvOX1&?2|(b&l(JRxrI((hv~C0;KNb0CadZ@$4Y9 z%rK*6eb!;j;tU(S%gW;|_5r0uLKA5%tm|q&6J9uq$9 zEUYgZ%Kv01^o1HXK4_NbNsWGAxYOx$P6^J!Ft3N@@IYrXSsL+4W4%eI%3a{|#%zk^ z8Bo1~2T>S7;Lt1O%sBw`UsaZNX=&{Bg&4x%EQZ~N?ka!ioS2BUYD?E5uxsX z%?_e{&{#jSG8D%0q|>=%&f|`RK1%hpW##h;M`;U7+ggUVrmo}r)@H}2Zs)i$na0bL zPUn*8_k{(0`y7{DorA$2t?6_+z!f0HN6+{}%+9`mGu*b{N9c3{G7Ap~OO}J5A$zBO zzGn)pf7ep+(pVT;x~00qiGgk_ANh)Xk7u&EIce|L&*beF`n;OQBAXMC&4&0K!iS4u zyW``irN^CS!iPPL5zoT$8%lH0vjcs!PJC^hOr!qHiX*f9NiK4Cnk9qJatpim$$`m( zmOiuyRMcRXlxDzIw#7|gmIE{Iz|Ndy@L68%@fzc&s<}OLw#SSRrp~{&8w&z~y#EnDl?Yv#{rLISa4Wkn&spx(N0>94)CV zgG(QWhM}Up@|;9Xrf*=WdnaG7;4*(%?mCt2EUdnrk?OvAehcN>=lBHGlZ zf^zLY&#BGA=pn*AXY0BQM7N+rG;;E7)Hti&zPrP>=eIC@N?Aj4 zK+VF1zLX!{sBb^JYvJISpnWEESw{`tVt<9AF6XpL(1aSEx*aZYczHyVJSjkPVmynp zxN7Bgu0&w4rKE@lp|x4~NrYUdDz?9|;Mvi!%KI45K7+TgcU{C2I*UH554wf5aQ(Y; z+8;ZnbzXn@`Yiks`P(ml;M<2!&uBB^*~Bsz)`K^WnLOr(wVBE`o{1$s1m@%h*vaA? zu}Q9`h0qXK*4O!yeazcR4%d$tTNiv6`@S8bs7si$2;=!LP+jQ@p?%OS8KYPz*m)M_ zPjSwkg<)33x5_L`g<=+F;^gbi!YUgmjkM2%jckgV?Qgv<0Nt#f$J7Yb?GSMm*6o@M zaAwRqa ze@5EPcu)5toF;acc8JmTn*R(R{ow^K`;S_;>$3tD9 z8wgaY-;sFp5*4mrh8Z^K>si>`ks$r}V?g|S6tWSawHw1f)&<8bbs}9^m@?#A(ntc=Sh{ibhb`VgaQAwkveFjt*O`O?PDj7LQ**LTUa=yaKaBS-FO=Kye`>T#j&rXV! z>7yn3to4CRu2Y_mZMbp;gDkP2zFg6pF|G6LXS^f9KjP!f!k$fgbg3&%HI7D(wPS81 zn^s4)WoY!l662ZWMwGMt7-uEN)}%yN+yFLE9A9yWvOdNf$oeR4nVhd5FW5(byGgTH zNaMPltZo()+>Ff-p8xik)qT-EXi#Tao|OFg=btzWgO%!9Zw~amsnW~iAtQH4Ak0$Q z6G!lQ7xv!Yc6b&z$0+Hs^UAYW;SpCm!lyZA{F>+uL)+gM+MjXXK3_e7S-8{b zbULTuSGpqs3+CE2?S83UrS6Jcxy(KX9>Rwvju7Wje;_dQqHTq8?=y)(=&^>&MjwZ?08+R`=RcoS9jDEGXO3 zw+v!89<72 zno7W&n%Pw!1@RLo{0G#Oq(+MV7+|d~egKj*d^=8EqB0A6{#Wb}+6RsG)0Vm5=Sin? zx#Y9(hVJ`ua^nI7l6~qoxhK(U9A0H}*&bb@!&N{cg-jkUX6?Eu>1erTfe#w|+zRXq z@rjejCN#wM=7dY3d^&GaAbIjN;j` zbyK@U<&^CCukAovdMymEyDk(q*kqfts$-auF@HL<8IubqkZ zbsy*8`POR3!?Uw6{s+U$lTPPyImcO8(>@FEC$3?2CLcRT=6w`ewbfkr(41;EblTll ze0CY5PUjopTxQ|U8G-kCYY)9Vd8cy$z_n9P-%h8~>E!gKaHrGhbUK|*=aCYZ!mG{w zcDo^p9}qkZPjBwt-SRK89fZ>UpZeSG`n(+k;X=Tg!x`gx%Ic`S#Y4ZBQPCKKb1aX7Q% z=$D6wxD*}_?%cn`=mIU2O`vcqYws+wm%bFn-%Va%ts15yu=Dt~GSSgP`=guJnK%3R zZkdnrp2+sHxum7x#Ef{FknoW?%QsM6(2Hq*+B@pRnFB39vplTUlfOqrk)SS79BNL0 z+=`(Mib8mw&7$xv;jEX!UYrXi6{DZa4wOlbHeS$!tQpy+bZ&8hV#YC<9p`#S`AFLM ztA*M28ys<&(JxOvWB&2*!@q++M`w3On$yq%4jk|}YVMxCD_pIJ7Q;yd^M~KL9UgKW z4^NJizC{V-%@#kjzk#=?Qd^i_|M>9m<2C**j;Gf*!^7Q6`Ewm8A8xMc&vCqYeslAj zq#%EJdqaOU6vQ`Fm&!zaDU5&Lga6hCw?#X=cCT;W-G#P5|HfEbvJ}oI?^XIEKlGOX z!af?6a0Q`G?R9x(6_nDzP>phY8{0K4ppx!{LtU-Sp8(WSBJ?I2sm@7xoB>jUDc;Dr zF4`&cxnjf9H>zrb92LU%NLxJK^2OR*q_zz9DaKJ<&VF8;PUX|w~w;? z#ij7qx4->Fe_ICRU*G@s6YX$m^AqJ*!go@TJ+!Uu6y35G$MxAK*${j`>vrn;KED0^ z#v#Krpf=bLGh^N`W|W^(_=>Pc8BEk#l1(^3;^~5borMvWQIlv&TIcSTahvW}rU76J zbI&i-Ze{)nuEF~jcB6=B_h+PCDC3TLo`7j>#)CW%y?OC(I`4aN@ZmONzocVc+?0KG zMfT9Pw!`X*%;GHOEQO~UGOpL;3~bsaf_kcE3{2j&g|+5zL(9|3qFH!;b2rGJ&c}bO z&n&hd<-pUM+u@G>I{)2mSeFaUOW`bh1Pn9|d!QQU$V|Bzm)iIOP3asMQ~@=VNv} zf|6ElwKGc}q^twy4F0?XeZUN|6yFxvU(#P#^?rRX|I{7dS07)0{sR`i@4x)<`sZLH zyDR+R)BDdKi$A%jX!SG7gGAmZ`jzdTiUVv~^=W^YNx8>j`#~b_t?eS2i3*w33QO8W zd_~w(Hd%874h~o*(R`906JR4#M*|ja-t&K=Up0TcKFvAbnRc;5@M`QY@TaNW1c-7= zb(-Q2G_N;maq#f^H7O4gdH3{%Z1-FiErm%6>f|m2FKKvk1Ur4TgcF8vG7G}ck%B=Pf`ms$v7VRDvBdl zUZuHm76+-aQ*XAtT>^K;!)KZ@ACNYG0=CxZp;B#ts5k`5Za&LS*vEfbTXS@cd z#&c^9S@#gXU2N6%$*0mG4eqXvZ`nqxKd4{k)#y07pqh=dR2B2ZQw5rQ*xLa5!>0~) zGspt+Bn}%!I074G`#A98EQ+OQ@5beC>r%MY&p2MNpCs~5xvlL; zT_WfzM)hfLUZd`(Z>O&B10$@9FTto*DrJ3fwB z1*=oZjT`CL`&ID=sftO+b*tmZ#Hz}3Y*pFA%?5u+R&Tf7-$2b+Npt)if4E=u`B^kr z!5FK-UZ9$>uWILUAfL%EkjVRJe1dK>{3q}MuwWx~94MM0`5!Nf$M39&se8738*x4r z=RdwnS&$MZnlLXBKX2;COz+Q`g*uD~5AaFZ%h-E`_mUiaC88){<7!S;hI9p?b{w z-A4B|jK(N)fwTy zCx<1Zh535zEHR_kbZV3@d#|Ak(0YLMp?X*&)IP$jyYA_u+>*R-{dC%mwv_%D$BKg{ zv!6Dpfk~GWCa*H9Bh=DyDxYngbX}*riyl~#vGM_{cwCFyjDF>PK+j=JAb#`>;80;$ z1@ky&y4_Xj5X!|UbtO8~?vLNDlP6H0apG@jj(^o3q;`ew#$kkH=CH?67Uxc<)9D-? zeODNOi8()Y(aD2m@;GCwQ$pN((&^luPs}OGJ9UvfNaP*mH9L_^GOiDsc`&U;+;Z(W z`Yt2(_Zo)+)dFbe-3IuJp{3j)RzmZ2OBH+vVNcCMUIAOeFUWc}awE2W-16c!*)MC4 zsEBfAwaZI1+uRa=6qD^HKx8e~K@VEuZ%OS|JV3lSxL7-JOX3zNt%NwJ~PLzvfTP1H_GkhqV*8#^Ox!Ad&a+_`qh~uTH1a>70P8C%^Mj7$;_h%+cz> zW&VIKe%KKAgrBcF9eCPdXVL>rKtDAuA9PkuUCsl|SVc3UPo&_bsx?0gJFH3seSGw) z`sGry59})rV^9hi57lGrYIsnr^?{MkuvYbv;J${G6CtLt%a)YPWwU52bV7}zifH`t z#*w%={c`-N_sRNM8Am2oMR_!0Hk74V{*1IcktZQcs_6=?@vr+sd-S-7a4ZfUsK#av zava4fold9IIW^f`;m3`4erlEP5jfwfb=Hhi_&)en&(nwQSF8PVi}R)Xe{Oj3>2d3H zI-PR@m%{J<{p#w;?+?FTJv=->;oX1yC%*pv;$NQp8|?l5Uvcy5>aVEzwo^4 zB*=dce~;xF|0M4||M=fuuYUPI*5vX(K7Ardr_<>io7aEG`H*L%Q0^ptd3b<-Q23R% z1K)l3-S_zZ{pYJEPyg});`a0}e}&hs&vgB7L!WkbzB#V0{vSG$6e~bftjqua002ov JPDHLkV1k;ABQ^j4 literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/YOLOV5S_2.png b/debug/accuracy_tools/msprobe/docs/img/YOLOV5S_2.png new file mode 100644 index 0000000000000000000000000000000000000000..61357f3feb95b5f70d6317dbfef7b516c0492449 GIT binary patch literal 5796 zcmXw71yCHyvPD7&u8V9IC%6*`?iws1SdhhC7k3FP9#{gwT>=D$1%eaY-QC^Y`P}>O ztLmwlp6WT>)iu>~dcr;_%VMDe(Gd_3u;k^WR1pwfk-w~I-yk6%AWqqWnO+n`CskPq zgwi3A05>9WQAt*1aLRQj^VBH#VVhc)o&G3%TGN0PFy96hRLZC6eR&3*$0H4@x7j772B+%!MH ze1k6TbrJ{IOON=6J)vJf${Hgaf{9RzSo$u2HZmG*Ep31$x*Jvj!tv=$xD(BRI0~QK z4?2ZbLh~TvM69of`X8uh9s(u$knB11e<@AmG|5u~R-k09ScqxlL6MCS)`%$L;CKU` z-Zy!lGnj{}Tt0tV*)L-kZ8@c|b)v^chb0gYQ}~|T8-*|y&`vuM7Iy6ltV?}CE6sR= z++oH1y|t8)HxCM2n!m0oKiOOzo`)$WLK{h^barY_G*e;x`P)#Xl`{9M*X5iBNn9cf zY{wGW{u_?MHv^bfjJ1@HG$d6^L8Qic!w~c{o2+pkv8vs6XAIf36G)i*rJBc6~g zGk1d(ehf?;cc>F$BdE3CchAJ&M``p%KecnZOThHU*CYUj5I?!|rME<30Ug`oBfmcx z-mJfBG!mcZNnpPIZ3V+o4aqG)wDbrDc>ijoQrb46#NYn#HP)63;x`p5Z~#c?@tWm| z!?CL9;7LGYKXVROFcHHfqEx7Cw~RtQfbKce>9Q+3z0!((mr95H3uUQ@#{YR#~sm}VX+ZWZ7Y8UHopnn~r2rRdHln^`2|$xv2i3C%77bw0hv3m^aK z=;hl+08D!O&JK|)Y6`$eBY|wJ@H49@@SzXC`_}qz0HOj7iLM9-h`yrZTeO+X<($mm z?{F~NTjqo~l?owEKXe;&M(ZKVtn&h_^x&(Csi{uwwO2L`Zq0qMGD?tU!9dH$lw-As zhPJGZ=HV~=LnQn9v?S&OMW^)?`QXZu`=)O((~xsrJR3}t0Prpa5U5hnR`4e-2bpNb zQq3Uws!cymc4{?FQC75z`s2K9`(=!Pdkn(4Q8H_LTcoQ@hGa4~mS9CAHipsIz~<=O zd%WzgJ019F8d}>$n{|4O2^MB-pYI0Zkzr=0a2WwO@jhT&JcWm!y@iXVJE&5uyqy}U zm~=JyOqw{H)79gx4guq=R)g$Z4s|Q1sc2>>AJb9UCl;Ab*8I=uTj(&|rNt4f)&>QlcbUK2l%zgR zfIaMGu-wP~Mo?8?k<3P$w%{a7d_5ZM`22VCqcy3XQ6mhREXWJII90z0e9sg#JjV5_ z5Illhvtc#&cAN?+m*9$y3}HMNs>b$UDay|(i^8=6pl~1Nw2u!B!sq1gQ26$h{^*^2 zq`F~-bD~LrxJUV?X}Pl_3(E`HfZ)oVFz5KuQM{Q%J!wIJ2; zNX?2pD_$eX>>0GEU(f^9J~daAbA7U@gw8v8P|Jt4k868w_2I!KVJZQ9k}}DwrM+Fm z6-Wl_8(xvS4bc4cMXE06m_v-;)p@;CW6}2HDab+4n+J11q9*=rngG+umMn&X-N>Ht z&q#Z?;yFgrZs4zZ)ZgxAs_~bu`617_0#g&V^keFxF_^0uK8n*r&5?5>A`>hlDzy53 z>dWCNQIp&22K+D_QI=(36||A6x3hnVBAY0Ovsj$8DPL-RCtAN}g!}LM$;HKOSaShq z_OQTp`}+8{EWWc@1n4iTuLAxr4q+o5%Tk5XvY!bugcEh68B%2y^HdV<5hTAgqLG_N zVI$R*b|yGt%f zCMeA$m#3KzYw+!bt>9r}Q2HT~?X!+6?rxye{nuQo(S7>lXdJ&Y&HPTA)1zfkV&dhK zcHk+;CiX8l(7GmCP-#bxopzmbwu0RNdq^ICeKG1Stc@ z@bvDeQzBpCK1){9j@@hGGE6p>|3GKTcVIuZ#$Ms_uqJK1%LG)Ep-3l$+9{%_vc418 zd91@1wa&L*F#FA{)8R2dnN?a~DC0hrg8yZg9mJzEqG&Kc6m$MZZ1u%1m9PbW^JR{V zJ#w`F@b>aHTzHhi514w4*Bme@%s4?e#dnhY7Z^GUx(}e!)!BJDgYWy2&D`|27bncQOl>xwgS>Wp zKU1{I7I9w~H;x#1Lmy_UI%7*}@KpFR`Yn7|9YLXWt~2J%c2kX>Q$jmE5(3i;0u5H8 zb<~3-u4ZX-Q7&h5Ox-9PPat*zO)uU#B&b`AnWy4J6NfTN`x5N;L9ULakr>5`Y{ zR`=ok$vWwk4PG-=i*}B&H4t;fNr0)>nWQ%hHS`YX(GK}2Bt%K&X*iFK*Vd3{J`rd- zhjjMg#?66RX_tCvH-BKS5gdvG(A$>o5@Yw|-mj&4vsHeqV;{^Y@%F9HQ4bP5dq3kW?}$@okUu>TkiT*~y)suO4*fd zCZTS+d&T!-S!{L98ej|G&}#1mCF;(uglgyroQ z+%x{a*>NmeD`Qh%!J}yJZFVOdfe7tpRnT+&L@-kdQ!uP5>O8Kp?>)VR??=a0tDFf` zPhMl~vGHxG%|L@joEZ=CsnEBub+oNaql)j+WXOM2GS4V*52^2TgqtP6e6u#)yz`uA z_}^#S&RPsZU?Otx41qzkh7615QKyWy0Ig!yV3p;NXdFKCp!SPteGZk@fxp9H-<$&s zmlIi6gq$15c8v5+sshezhi9n=i==`6g+B>qFpX>>&-^UF;%byKfY{JWVP-dfQ(g0? z3uo$={2He)_!F{i7n(>{Kt?(1wrbLr3-x}_owkQ1G3G!4y&R&&NsF2@+{BQ_T&fN@7zapw;hrbRW-l%wLWDU7oD zqKN-Hs-mRIUixb<_JO=SKby(h3P}1bi=co#TY`nNWxt%mr_?nk-DNk2I1E)J2f3&v`@w-6uSv#cIq^)> z^n^0@3vA;W)V#4Rb7otCYXqYHQ|1`m5YV=ZSQW~WyPR9liBLfdU79=b#LF{&+t6c; z(Y)5>7I0Hptr#*n1)c_zbwg4UloMIUfJkO2joZo@)d`l(csOOve;i&8VBlS|S}Yor zuqKZCs&&0U@Dn@Kw6fnGj&&PE-Vuk)P3pB$72ijM0pGq!=U{{Wv`jVeS?;k_E}9K56voqpIo7ZwS0wN$*n@xI83 z+W{pRRtxCWc5-*Ma;NcO@0TxmFYec$3-REf%2U3%>bB`_y5lbXX?7V>cQ-73b}sk0 zn%D7{AAhDSy%sgq`H73416w4D1-#|`IDGwlsSolX3c=Tw#w;Sa z*7G(Jbvht;oO`N#2VOj`eJ?Of@OzWuW4tP;0G5!SPq`w@*|4;Mn1t&7tro-Nz)w5^~p^Qt(S;WY0a2 zL<9{*j=ZRxNA5txQe2*`N3*Pm>UG+#{c3M^u?P(ucWl3_^^=?HHI4w9=p4!Qhs$a5 z{2m34X-Mcg|3!XJ2t`Iz67P&>8?4KzUb{h}qr~wg>h&c5JpJ7Vx8ru< zknZc+)oaa|lzEnx{$qC4FhmIYYw-FHw)M zED40HDkAV%psMj%wtW$GMur zswXg}K#^Z=lL{Ckxzq{4rTFU7bC8~~%lc7F0PACRaN!lh^4j)Hh0=`aZ@{jkYt4im z5vha%#b)vWbT|LAU8Rc|1V(suP6y!FbCh{q3!f7=#lW;~{D$C#Ng$v?o>y7F6F~Pi zc3r#1i`$N1PkjneI72|cymO8!4mC&Z=f;S)-+|+IU!+@7ny2L1(+r;I6vz&25ab2| zK+;jtK_L{S*nX?@&}JX+oAXoVcQLDrEpU1I@#E&_HQfwfUjk7#;l@YO(oJg*g79n; zRY2zNF9MQyrc$BWhYb|ny4n+`BW;bQy=XHN(z#+C!noL3V_Mn^OMbqM3ieBu{SO#-TKJ;h2 z3Jqd(ma&+)JCqR9cF(;G*OgYo`{MUrhy$fHCNsXa0ORPU(Csp&bX04AzX4FtH-c2H zzDun{MQk-|Iqpi32m5KxqIDjLBA^x4iJ^c5`$)wQ>jeE1;!q={6BhN#z6kCYR>WPg@N(|Pu)T=YA67ez?R+? z`bYX7H`k*#6F-D4ma1{L2krg3-l3vDdtC?QBA7f}qn1z`FpkAtGq!P}Pq-#~szt5_ zeB)D$=wLomO(4p}y=G~Kw|;r{<~AG~laT95AWBN{r24FIr*;~aO(HR4)x~ez6VY4( zPyQ;#SET#R%~hRHS8hh`{dtwup$d{zn_5}(sMZWD_bG9NA45=@QBEbJ7X9sy;3nQ` zqx=Qs06^yasXuG=fjSrKC!LZ9+f^+=V0ru$wB)O6T<|Dc?-ob#&KTvu?M?Gy>wT_@ zvKvrW(56d#`rjn7-piA}TP53M6~we5noKUzx5#B3kIJ$it7y>$VQLlS=(vwcY^G;Y zlw&}Jf1<%=T`Q0}mx6u!Sq!>Z^ZhHqSoi*Y_=b}VN=#7cclm@{u~TuUXiwz~W(9}b zeN4#N^;W`k-7T9c{m31WfzX=R8czt_Kl$|aHmE9|25V7-t^vO3(qF_ARjE`KIYc4> z(hHPhzVV|*(>qTyY%o*1ljQ4ZBTS)^v+(~o-hyU_i>Tv?+FiJ(4Q*`uQ0{GMd)M#Q zYvU7M4d$U3cvF6yDVyf)%{#@BB3)FG{<65!Jw$lj^9JZhHR(EcDA=O=%4SdiUF{Ju zabJaYu1#2)Bd>Rf!~UuZ?56>JEpSIjuI~RwZ;{J>q&QIBPB7he{QazjES;&e2Uj+4P@epgb066*J*Q&!&TFUj z<*SgZ9#^zBFo@;T_d$>;D33(}CelwRiVJAH2b>)wDgUDPrrhhEhd=Ca)?(xSEZigX zQ5|CZ<(AHyQpQQRoUhFkPZWbjW+l(@h7&!X^;TEQ`9X+c20By^ETbJ7?wqYXX2k98 zZQ&)E;!9@M$K~_G+b*F`mId#hZp>UJTR$(XJ&Zkyeki_4Ww7h~z;@NB+ydf`Co+%F z)&8en`&SOa*jZn?q<`hBKO9anb?XG%Gsc07yCu-~o?>e>k{-GtzVL#7EVk{lT&5-1 z0O6B}2dggax^@47@RcQBjZrUq%ow9lQ50mR%dPx}KUDONjkRMt%iS`D2)o=W@jQT` zoDfs#ZkOUemk=<@6OY~h6r$FIe|f_{uq^&yZOkxAcQ(7y^=PgB2J3(^HtL=Z#;q()Q(m7*ddH7X)vfB=yaAc=xYRJuwpQRza6 z^q!~)VbP+L1QJMu009C-2uVo4aqn}^+GpSY-)Aky9ruoV$ITd&Fi7%#^PTgVb3XHV zzh{^2t+#Df*t}-Vnr#G$i5ch%A#Yav&y&#bBFQ<{}NS?^S{%6K&YCqD+854QUJLhJXuS|hGugeLVI@i- zT=ha(5162z)NWk(#(aG@k7#(U!Ef!3t%hkQavyv)j=~?ZJo)I!Ztzv{DptakV8lcT zABbMXk}-C`mf2V%(f1-%%1T={JbXYF%pbP{{Ga{^z~dznK2<`KM8cw?3KkG8;;<9F zNV3v@|JTuq29FSTgpf-`pd~zjq&WM3{~){7E!uUiT5?zjFUlsse4pspzxhL>y0bEIqng7vb=;qQ$&WXkeC{6NuK3vOQJm7B`IP@$M~ZgvP>}zQ z<@LJ3e_vR$Fv`M{OBS((N^r?}3)On!LQvIfx<;-b%Fd@af*O#-Q&qVe?i;sObdeRH_=wOi8lBupxEnUy(x#bSE|KXf2)HHyH1EJteN1{L(faCd@~ zMY0chMa3b`)z2Nc=K4z}yG;nv_KH8qM|_i~HQ-5;TH7i$Cd1WX&*hyDWFm z$70R!<=$m9x;EDrky6cy*d{tgt+kMB%K?&eFaZ?}{(RmeWDIB7S}4?LiA4Fnk#12B zcAoC=TJ>Lcq8jJG9p`guW!pV2gK@m6Sf+R_)H$>^K~)HGm0kNBwzU5{W_i5!O2>ki z=&UFAdu>Qx&BwF+oNF9hpA2U~X4lg9RgJNDSuFffU~Omd_Q~`6)j@Rfw6oky9&4=r zD8TO=c2e-1>Nw_Q28*gu9Y&jXN;ZRm2#ey50#&cbqFcG6?cpt&syp2?`gI!1kpVdb znit5e?`t>zZzk3u*@ABa!yQc`V-|lGXpPxCxsd>_s?)_8n)$h}j@&?ZnUz$hSvD?+ zk9vfjgK@S#Z=ozfnmF<7`MG3H@4~Pv_en&-Z8D9%h3)=DW-*&Mw6Wcw{GKDm$?Ig; zu9iv%ZZCSNW!$-7+;I4W=KkJcY%X?sl%a|la*AK^YGolc4M!_@T6}$ylGlDy@Te=- z*Da?}R;wt>`U*Vols6agrMG*CIcXvA7Nv8ELwd{Ex(&(j)j6h-9Wxrr z$hw9vcQ25Q?w%Oq@jfA|D|blj=00AIT!l!MQP^+FjdbA3h&AF0%m-F4EKilE722lP z-HJM=Qs$-A{ODd{T9q_LuqkDlN4L59?6!VO+!9pb_4Uvm#uVc^x{Ab3# zG?ai3bRYC?xupd6c%$SP)VJK`tgt&F;xpfP#RL9Q5rEhV{ftL#9^_dC`m6@^I*iU& zl-@vvZ3+K=*%5{u8MjauaPi%r!(?v4!%g$6@4hnOC!~> zp_e#>>76n$%`r?teO)IKbw~BppvJ-`L6L(y7ZZZJ;MwHSWe6sZ5%sC1^BEh&AR06k1_h(kT{2gzOa;m&eL5`Ok3F9}eL{9Q8 zg2fj8*|i2U%w7iDWVYDqNMwVimZ(R--AUmugB+$zmPMzq8!@+p$>c(0kJYrm>_bdR0($t5?K^zg1zx_%@2avD&EC z3&l(Udx^jXu98(6)Da2oE!a&;k8R_|tw-;a+kVR!<+UVOnIY#NT-h`s47e3gINK%8 zj5`BL$u3g>0>UoIXXfTs<@2t!65r{&^SI(|#O39xLoRnpT}xt=pkKgxQl zn0K;I#U=TeF3FdAqZk$B%5sV=M2;Tbur?{Rn zvK@-`zSPyz76J3iCq?+BRMSX6lpO6{6Lf)qIYnqGwY$cfQVk{u&*kxY16r37Zgfx@ zRcD_xIrt}VCa12c?mpjSQN|cPd;-jkz6J+I@g6 zD%(|7BMoRhTs*~BqDP9>S7HK;`GxTQM!_OTd<)=X6N0!a;fDu6c&z!zh5-8`w9$y% zXD8cEDGBC1&cQv#OXD9TnK@}BJaYof=3p?WrbByaz6q_YW1ULt9a}-mCfkUEZys(s z7P9CShq-4qsJqWS%!`@QoBAMN@+${YkvG+bG$&V4#$X@ak0s@2foBF_ERa36{Q>&Q zl~H85{)#sh)9bbVS}m-pI3@Rmx|xB7x`jFC6zo~^3iK}<)o{Nxdw#HX6|i7Z zU*gwP4jb)dG}s8gSiPz3Om#x(RZ-iQwv4#~&jY70_58$P&H$vmCy;a>H2b=Vu4)ED)r{U-iU8(kYqw$F^|nqgc;YpuFC+5h$yvx0sM5LCYFUM^)+kw( zaj(uuVR{0KNi*yt9t^?O2(+^4r79>ml@=r`&Frshi%_gG;e3|cK}i-8{j&Dp*8s5H zkeqS~{@8YhaG+*aS4Y^Sd1pdoj6Xmh;0m&tE4!;%7C}{BV^xz^~8*@I${cO=m_UiKk+MtUo#zK%#*fM0c`C`34 zU|a)4{~l><-JjNcZwDJngmtzQLSz#kL8<+MPEu0|a(`-SA|Cg49=@HK4b(1RdyC+( zF|u0_+=;A+!*8!IM1J2vo2SFJXVqgDibX944%+kZ&=7y1yeY~@7lgkRA$}MaUt8Ot zsno-j#o`@$;yA9O`sJmxaq21fX2Xhug~%>%+%d`%oz89rpFW<}EBF&Q{FT?i;_T)m zq#}67-{yi~d-iY2JQ|>FL~+|9Ae=EQOt7l=+&_1`S+S+21oufrE_UMl;k{$x^T%}8 zizWa%t32pP!f6LY^{MHR2rH}=Bt`pph2GX{eSsgJLaHLtNL8VQ{sDsF0F(NHAs=%{ zu14>T>wYO1sORYV2d1$R4`j>IWXCzLJ!FH^7e_K&NNQD0bHQ*CMr(5%&F*4vO5aq;_rxi7z3ullVx z2hsNVK2aQQRBv-Hz0o{g&aw8u0g1Zj$1)Vjq+wUttERvZQblWm$TvHChnMaG!{S(R zU1BPOu&fUQr16oAklD-t;WVk7ZADp3G?S$Ex$&Ph_4y-NO7!dX&G_c>J9v`#S*pJN z7r(PcxbqI8)x`SROSk>^;gzG%0WFL4mN9&E3AZUJ3pk}&=!-`opu(2ye#TpovX&hQ zKm-6m9h^I?VD9i(5w?!*U3|359(1ckbb0kzFz#fwTy}1`$EB=0+b8NL3#jR+5z7$O zA^nB8pxpsJxS*r6lwc)tS}k}Uy(B?uA*0OiB_Z_&PteZ+(zy_Qhcxc_cD}e5*0iWO zXH3w6Bk5M$r1yJ6Zt^z{XC49d-`W6_b#pE$4vcd;UlM2=P& zG*i^O+xd>X?%59SbRgEwDVpIR$T@HPn`D`HtEP2q7u8|580;`=lun#U`JPNl?l>(^ zJ4|-8ko5p*pZdUnz1Pvp8=0nMGY&Wxs}P-U8Elf@!Z*_*Zy5`|rKh(~3aZO2ui!n< zJ#lAjF>tj^xZWI5?^uw=sYA02Nw|5@?-sV-%9B^kC0Tu9P;uHqxvySuo8DPtXmevu zNhIQ|^Pss~l)8i`q*x&(7n|dLD=R1m**Dtg@pW|vhn_)^R+urF>T5eWFRQQIwF?>8 zIY{4)HR`V))~TN;+YSPxRA6Un=H3tC+#vb$*?HXF-6sRZNqf4Vnt(&x8t!wP zLSEW<#~VEq$vm4$dXhdvesg?m-NzZEAJBtxF7Hq=iOL?ooW#WJOQnJ08XCKz3L=td zN5ZIcW$>Q%z8>Eumu~vU`>J#cQz`^tAbQ~Y*3MhvKUC?3N;-7CAuawt;pHQ2h4^ATXjla!x z0a{6v+4UFWqgR3N5HE)53g9yZk|Ku9fNZ(*)OA3CUtq~m5XdpUao*x?5bQatL(ube zVLlwbtEvyMc(mgr4BNi0OL(-Ivuf;yf3c6aAahFFQv;i!F`HvSP6&F%vV)O0jzsOb zSNJ$Sw?=bu{H@;d((#(GE4X3eyE~zaEqAwuWF}gv*79}jsVBk|ACl9M0SQWp%vJ}* z*}esiH@#)tqF|I_jwXJ^+s2QGbAsXtNC#FWxjxAv&RTgHZ>J1ZG_6}6;rG<#tE?-d z)Ca=-tF9Nb*~YAtgqP4UKRVPO@UZO2Q9wdzbV%j)Q=* zKpNxju~~<%j>;> zypjhz8a(v&{p*G`Y_^AP){x!RAbXR%p^$>1zAS<|(R^O9t|7UX7m!*Vte@k4&2w{I zpGOr1(+T1XH3g=snsh~2VZ8XYaUWiIXEx%7yI5T--Bf{DQG}eRwJ)^sATVUu-FD)v zwfmi`BTL`zVUSND@0Jbu4FO>kqZ3d`E^-U?Bcm=x5u zL);Wws?|%8rM6=_%hVvo=Qq0Yw(W+id))7?OaQ-a&Muza_}pa>;8sr?&aiGqKY`B# z6m={at+XeH)+RYeo#%mQ#bux)=!S}T-Ze}Yf=jt)%&%@gEVo`8-pA^yLv4tJr7*O_zkRxEtQZ1jw}0*n z^(ohL_4Y(zy17F*R#d&P$1mN*vB0-s#DsF&q0v~{KG$Zvl6>;ER;7*JK!Zo;AMoI} zbfaXB0#J&r6^TQbWwj70$EsTDoi#_R{!O zJ;CV8T_9&U#vUS^^gh_ZO4{Y6rIaWE7AqG2KBgfK^(F$eV06QZ4XrP~LWEetfPQ>y z(;|%oNcfud)u+Z3vI~mBqcO}b_#9nsY{gohp?a9_zmy!|=iGJDLdn+ZE^WWhSK!$u zXkwyes*Rj%G=F7DavlDeEYkS~x{O1wR8JS`6U7$*KBZ)!7}K=z!Gb(&nqGixHuT1o zv6sxW@Yj%ViY&2nk^T{}Oj$X>ar0_*tvJen6mJ<+nM03Wv6JWpH^-9O8!pn#o-|GO zsoR;`t4V6%pd{o8wtGJ&uVT`?L5JRFmiS!cE1pF9n_x;E+~8@t`F4?kQZgu226=lmSyLShpsWLm`6trJy_Rt zczrj^>sML1?@?D~_I`L}IAt((-F8&mQdN>*1@WI?KHS@wQo4{mkuiiv%CCjwemrij%b*f%OE|>U1R)H z;li}31KiTQlT?>3a{)UO1%Geis|Lc>Ivo-ZIFKgPB z@{VT+B7+t$<4af>n3-zjALVTn%#}{$xnwK4v_izjfZ_|ZgJ8r|>Aka-7R+U{e*743 zWU1FUD*F4#4&qAn3qwL#_`Yh7T72KxFx5P6WL>QwFmMO4dR4BUM)8q6yvKh(`i}Zd zr)!~m1|VbWO-R%x@KNS)%w_zu+K}Q(z_L7N3i9TXX=47H^2G#fMSw?iW@=MZt%DrA zX_>)Ri!$iChtlT_14|*N?)4>|GAc|ms&bUmnIUZk!OS;idA&6-`shDSSLXUE>z}7Y zsxG5gCVb;sbO^Q>xtFDIxep!Xf3LSx>Ypt>8|vu2*DYGR^#FLAK}xpho6y8Silb7J zcwX{M>Y<59A?0NLKGqWG2)OW58FW&3RPHNL8x+*Y_7{b~_tp@rQmFHqeaQau^1@o= zUT9>xXI%vVM85{Ro;&bTX{#gMz`E+bd|fZ~jx+DBDUCQU_BfLr zSAWMrJKQ_YtAhIyyjvqUiIo66|919Vru*k?voWy#UK;Km6W4-Bm`OSlVggpejZq)Z z!(Fd(ydpDkC_=syR-(r5jRHr%)pbV2uh6e8 zxN5vT^!1QVK|z?JY8^wLa)KSydkirk%NkalJ`*Xt!zy7}^d#Xa+LV*d0qN!O&mO(q z=Gb~^u2SA|`KCp^yG+;0*C!$tSDM*tI+!NdHiuKclQOIOr&ADZc;~0fIEIr8{`EbC zWq~pym!1NzQ1#L}G4kwXq-ZG&Ucpt${uD%h=;uzY9BsQv# zz5GHY&jsQcJhKC2cHjc#`Md%nbV{eAXQJg=Ol>U~McoWwme5Z-W2f~)Yjd_hNQ)FL zWxqUu9n3WmS)-?NQzxr7N7FpTdPo%tINv&9aUj>tuTl@M6OAoRTsrMP|8nVGJbR|v zMrF~J@zutBV+3+!9eMGMe^-9MrUnl;L{pP3uROjxHRLsb+saj=9!Fa%Ku)~d$sTlw z%dmz;rn5jRU$Pu|OXn)Z<)=owiiZ|E!$|eg0<{m#R0Mzmh(g=XtqaRN`C0|wO-)*Oiut|c<`tj{atncEvPfNNy-LYDxeO}#QH^d? zi+cpTqD*!jo_)KcJeQg)XVgdgnDJSEEv~$2!vU4c@gXD70neiwufTPF9q?s;s`*7P zIK^%-tU@-u+Jk!v5O7&=UQ{L?kmpJKqc`CwMjE(sGMA*PU z(wmk58q-nVl>RB~i>F_{#Z&gesfw|;`=qoaeP&LK{=d~PYh zUB>98I-Bid^8HH?1>JL1CMwM#-cxqiit<6k?7Bd|@gh}L+pw+o*L_UPnFCz?6#9^Q z>X%m3uU6qu=_$0LC6jbmkh1c{-Adfw2}2UabOa$5~2q zB%>Ij!fNCW&7t92KeyP^3)*`1Q;6X8KL$4B_s4aeyZDKM&qr(sk7Jk|39R^J--X9*vlu|`fQ0r@oKxb}GE3YQ zqJq34fzB;Ca$!dH{aH?fdO6MmD@phW$r#&O)JUx5S!z06VDX zY}lhK)mQI^jcfO}6Yh0it|tupCN?(QB#2pY97?c^wCC_dg5+{!Ld(X_kjgWaR0;;wXt)BTVPnWKN#gamGL~d z&4H8-MnUp*l&wjMVu`g z_>v$ROuus1U@?*p2lviq-twEI`*_u1^;J1WWTnp;x#UsfUUKHs<8zK~f@pYOpD0tH zBibr*ihuS-_C>Wi^A@I#Uzttw0vgrTvTy6#$c>zJgR|VN^Q8GW+VRz(HhCrU) z7Yt8uEOG~4gkzBCWK!Ck93#ZN`gQ`tqQ{wiRF#w}L-(pwsU^M-9QM#~R#H_bv`$92 zZz)!44l(4vRX+EYegF16`BttEfgJg`&n~{Q!MX%@wmuD4d!Qqte5pE2-|#g4n_CG=+g10?dk z$Nneqc^o~wH&2jHzHKWjjjeazGhWsknSCCeTRc$)5kyYC3b+>~q>L3$y1fD#JeEd| zJVRedK$D@0UU79k-XZj;0a)kfY!vdBJP$ZuK-9nk_rj|TO33OSc_WwUrwl7DxZ$_D(RWXMHntLUnLI|e zY(~5w;omZw>zrWO{AOYNF?dA=*|Zo4Vl>6rQD=*V@*&(P!#S5i7xYW_-07z9wONQ= zS2N1CKk?_wTBG8im@g7fK;#WcdKa}lPw_qS^CyQU-UTk_L{Qr?&yrXV##Ys|0Xv*h zj;$OkJlx!DIOmz{wr(@pbdB))Si>XxkSxib!PBFiMY4xBv?a82w)x#|*jb?pUz7)U zAOR|4idzO6m);z1%xGmBSuDtQUz+GJlcalkP~se86)S}w&ykwMU45_)JK=Y?mz7Cz z+3#~^nlbilnP`%O*)ilSc=9^S*3U6+K znW-Kfe44uGV>uY4Z5IdwjIFvWDwgJnOzI3fB-;Q$HNtr2$zmKLFa?&OUEdM&C^2wo z`BZ%vF)Q+UhGZP8_j9b7oVsumR$DVZ_ zo(IVB+a)!~7O;4O(*LMk`JpPwKzwzOTUCiBi1tdp1p@GkG7Z691f@=N}*2xuga7XZ0iyVAJVcDefzI2R!cN5dlhD} zh$gRwpq8fZ?0hYZGOtEMa+9Wb)~Aoa?F9ieB8F17#wpW52`ITf-0FRPhmHtgYnmnx zj=amxP@bgI0h-H@rsJw-Ge6!SBv0?fO|vwHP4nd3TD%nm6Os*QzlUA$Os}mYUSp&C zv_%zDjp*|0f{D>j7nI5bQ3Jk%s(q>rb1gP#e^RXdlSQNq^0%Q4-5KF+N2Bk1 zf0(1O^*46san)w_u2`0dFUy~bd)P0skim3B;rwVVLeMP$@(R}MXs2j1L4Ot+_VdlpK5Uv~A6v~#iGdvrs> zToO2iD|QkqxSKt+C;lHm_DF8+KP~KklG1GkG)E_U?*|>?$O3ESAHBPK=Lo>)my1K$ zh%4d;W`+KPg}l0(fvTKXjz{^4Fu*UMmT_7ilpv2x>2_|+g>J$X1v zpc`$;4QHX0S7u`@wJob>war)%?)n1Psrkuknd>5>lTrIUUqScZtm71U33J{0sAeQM(VNx9s_pK;)SpsBf_IjUa3(6?4HwV;)q!qHTh zI^4wEwXxs61`0#L8&){$U-4f%UrDM`kY^q_QK`8p^RUvn?~9qp`uuX;8N)#%ha0c3 z=bg}Z$)I=3j_*RSAn4*z?aF)9x|e#U%0B}MGxXxGGNjd`HX{>1lc(|cHv)eDEXtC% z`Tt2>?7tE0{xiy0S9ChzYAri=HFWC#$(Q_H7W^+#(?6@={l7@``{!1wBZ|W0amfrC zEuS2~wI)?}u-FX=zF@Q<-@Y+KSiBT923VZMexTDrnp1X-;C+lNNNu;Rfor7Y-w!>r z)x$quFlCVNbQ*`>0EpdBrZY0(*g|UhxizyITXZ;B>N_eiNs_W5Q= zB|8hC?5gW99w!P>Ol0G5^+qw90<4TnTgb_Wq`9wo4xOWnM=_j>7|Y>=#5CVL7h_nm ze+ka@d$5_smgAH7H`?G zl(R^@N558Yahib4lELszdVqG>PYtcs=zyG6Pi7B9g<5tU4-f60t1LhZU>B3&s+eEv zy=p5iB@zogurPQ%DYcUH!~z*)cCL{{WE=9;JM0&=rUn$yAD!r}(pU-KdGn9t_dgXI zMc*L@HjQjZp8jxCb+^JVR$~;d?i4flfP<=BMOQ=^YuTxu|W$CO-wsc9lhpTF-ZSoBJ(FOh0 z;vqIV*>XKb{rxa_iJifn@2y*)X`iINReI><{Q;$`DSLx$S+m;IAIX`%nQ|AE6NM0( z)u1T+hc0C*+nUe@t*KC3Z}d>@#5E5EfxX}$f%JtAK@8Xw!LeU%jXl!obv5R}-}&J3ZhE}RoXh>y z?)NMnAp~{GDZRs@lODArxFF!{B{`qEqQ-|-^5>v=6(n5%S2NBnf33p zc2+$?(RN*%CRGjtHWs*A9`A%itUKv2V2@ojAq+)qG>r~Hy#@~Y9DUuuA1d;h3;ygl zuZfk(ASKfWBxXjaa_jYKA`?RD{yUv6Ex*L=oMqky?Z9NirbM|P10TFN$Hy=Ol5nca z8C@;ZmFmr_TDo^(O+kz8+7j>TiRRAPQ>@IrJ+lT}GHlF6r#!Z$rI>=l;q6t&!)+V` zcnu@>+o~0PXwW4zENv^hRIj|b3x8t?F%(6pk$e@qc zJ(PcRy)<~JC)R{h;L#|4@PXHicF^WA%_L>CSUNKe32wnSl&gX(z)nq6b8H=-**q=B zp3P&)%th8|aHGJD0CQ7#iOBfdZEI0|hlC?sy$=*~c(n=td{)F!FuSPL-+CH>Jk8xW zOF4pgA{;^N&3GB0?8ATU=$W(%TRl1#9@&!TMQK}V%ySx1hP0@YP9m=V&hp%dxhtnN z1!?f<&Z`{seVfajEwgfCm&`1#bL2NBi?#h?j}sV8li?mDwkbUPcXE9@_8)tpo;Fi@&@V|V;jPatxo_~nO-js%$?NJ5s*TjT4f9PV*`)- z9VVV47l&_%b1HlqkA-ZZ)Eq0Owf)juHhTr_r@yE&V}MU+7ZZTCPt1D+bFc@M9;0^( zs9X`FU_Z&2aI0)vP>Uq%k&ycq*&KIAA$BSETk~9UI}$|@XZ-wLdAc0Scvv{v*jia|x_Xpw%!*To$t%#kMU zc@P*}1FQka|HkwQN;QGk-YPWrd>E~t0vUoz@}rZ#x&WbTcp>+`^bW?1;|C+DlgRLC z)lZyS#-=#Ku$>aP`W$@=yz0TOcFc7TE)lSI#XRw?J(s>#4_wZzZ09g6if+9hdeUoH z1RQ9uYu3~1Aevb5YdE>#%TWIQ7 zm-@z{vF!=cihl|dNOmOno}wq?)`!0}l#wcTaOdIt2y{Z@BSKqcw6}~GgbasCq+JOMy=jdv{U?#H9;>Tuhb*2(kc{#M%9$54-4=kxeU^iHKvQgJA`>kz z9x2{+GV%Jd${S;+YMV<_eL0-!A#twg#MgTa!YYnt-4$CWB=Kr@{c(t_dZjwbty~a$ zN*P5j0pYLHevO>YoX1Zv6Ofa|Pc=KSy<84`4#wK->>-CexCxpxv-AbBbXgx*t`6pV zPA;##<3h=B#(e1Dv0@WY%Yv15;?a(!S>buX#^nfN%p3N%2SASjsEbETOU_b`1=E{f z<-eyr_B~C7#tm_RseokT*aP&+ZhV&vBNOA{Lb!C*jblD+vk?Dbes0?(ym0?bn1vq% zLy6r>PL16*j<|H8=HE1bR>h!S#Lhf3vCEmkh*-K-ktkPwKD5I6Of(HlL%cZ-pS!Cf`& zJ`!}Uo9`Fv7#r;OD-m%rN|4%*)I;I0I9^AH&iw}nZs(0CR2zeBOe=%7El!Vgn3+Lf zTkMmNC|>hyv4bfh#)`Cytz*{0uM3Asx8QUs{B-R?0(B?VeUUgrk9=X zTF@F)cD7%|2Ep9is+3DtT82#WLk=AEK-?53d=qZ#bEj_XOF|~_nx`}q_2z`;L~bul zzP9Wi0Otq~jfc)x*RlqTBbk}L_Jbz}7L~9+ye?yG1n@DPoTaY}xC|#XlG1*>;O!q- zVKX3`Yk(4vz3@}3JU;$E;*WoGM^)vf_Fi#Kc7g8Ss@1f0TAYZfMh zi$Yx4Lwhk(HlkCJK@)t#V7Sn)m%0tmTi52gC#MOcnT2!o*o#=#&-S^SFWIJGxFt;nm9ikpUd}F3p6* znnkdifUx{bB|LOaS7AZg=|G26gNZtRb5 z%<4_^x=8#s=~+u$Y{M$li~=jXYl3)(aPhSFWJVm*VVKXGx;snv`BTbaX~3;C%=&ib z--Eo`Nt<>ARD$Y}C1IX8<1w2_13=Wh$Pv35Q;=I3`0fX4G`G+y;fUAOu5dfD#7XZw^9-PeESmRptRp@Qz~p+4PFD$(6hI#nn@oP7 zv9-uAaU_1sH=BseC4Ms}7VEiFF(=`M(|Q75cWSJ#V+GuZk-@(QByKototQ10=GDsllPLxANIQEX zc@a7pF5BOP@TCqsAQB#o<$?7k_)Q{PPVs-B73|ce5xqd)TNhQISTy~n3kULcP_G~k zclSAvcYoPB(bnC7?Jlq|dNEqf{E#S9^u@WxZ{~UmPYLA{$oep3jZ<(&-4UPIvuiN1Yz|>Pup4 zXQX8=@jM0mj=R(z8op0|(xqGrWn%47gh9VHTt7R?3+Hj6Z&j9Jn{g2O!fD2rw0^qbKzK-0#I}?l)yIDorhg#v zzf7Nc?H zJ4fwVIG52H~%XwXZmIsSL$GXDbr-pGaSx;6gWR!{5pXTl1r8q~$j8#Th%mIMc zKL8uCgy~Q~+GPa=4Zx$HS}&5o8fC6_GDiw%E<;{1f~ zTQg9)fAap0qxW)wmGVyoXL#9A!~?KmhjUr*7u_z~ysKGSPAS~qBLar78W=u1uYVg5 z*F;l_NtUSmWxlrGVeby@ry|JrdFdq8fSRIgY12b1n_~7hOP~f{2%En+A*z>b&D*$h z#7crK5Z^R3Z9#3b>>P^nl3Z668&{Yxe+6cC`i4F|4BD=g z5Bc+(A1@hgrLPF0lRffw4k~u%{o)5a3K&KjZD4K=+xb0i|HPyHcdGNOR188|Sq~Kb zJy*P9Qcq@kP3lRT{=Yo<{`&vR9R3&R{r?6Hhs!E23bR+WB_zpVs`$lX`F)TK02LSP# z)>DveOxTX&0ydl1g0+x}5&WDT=CM>Dmr$9gZ^O%pdA@UvouEUgvFVwmZOWdX3Ex8G zre6q_ZJTWSz@`Mt&WD`Z#0`~L`J2M1mVnvEr*7}K+V{_i`Jc(Rf0vJ?O=fiRL1}!s zSd1yAgd~!u)^Ajs()k$-B{^`z_JO&?d{L~5;#%CewtF?qLuDZG{0DtX=;%_t^0qTw zEd#r$O{%Qf zc`S!|C;s7WmEG|-Z!h)V(f)7)aj*=0rpxGOaAbt0iFg^r3n@Q{s+SWiaI$F6miC}{ zO#iJvX@TF6{Dk&LBFoMx=gP#-Q0VDOF-)EE!dCwC&Y0xjw4W(dW;H37J82M(5bMvoK-bjphAO;HpZp~a5_s(Z{UJr$bn`^)#T+Q|z$_SY45%gmd9-2UoM zf{gVf_2XYx9Cl*Qp4h}|L9|#{yj$Re!?xojO453jmM0bz4?E}m`5Sj-{D#ES8Srf6 z5Or~?njfBrK&#wGSV&s}Y_))=kE{$ABcy2pAs(0b-Uoz4-Um=|bVg8f>+xDSx_e6z z16jM;&l2-XWSB`Ye4gp;8*a2Y2V5_v58Mw33RP*wJK$Cr;JJquD374tG z%oRl*KuZaej>(0+wM)P~TjPKQia=QhYjB=YFdnqM|`)(gwIMsDIiB7vQ?{$KjWtPr{nzgT zyb-%V{tALdL7_%J8}jZ!)tKzTVPqL?E*wzK)x#%TrwBT_dg|A!F}aOzD?d!3EvzZ^ zxKXG`N(sfy`2H9NXsH=IcsGBY2jdG~S@O*j9q7_4O#Vw(NOF+G3&asUvK?DGd?;hY zIzyEJB7RJ;(x#`(po`Dl2Y5&FKU^50tirbUMOm=wt+2liJcT$?0t0%(qZYeNIx6?X z%(4#C4#SiI7V2vZ3+G6pp{Pq$X6oUma>?nnhVW&vOF>z5G8xWUkz*UZClz6k)&_;e z{ftn6j!Ohp!I{p#HczrDZZ37t1Pj8dL2K{nGZ-@M)$ zD2$TEpjm#D;yka(h-7HGdeN44&n$XE)+;_grReFc&*<(omur0FcXzQ&(!m$ftoN85 z=vulyl;2r=1(!ICHQM;9|2AS>ng8?Sf@R|;L+Ndz_jfYskMyChRkK1gos+qo8=1T( zMryc^$6n+*Zktn>KYc2!f_Yv_oNorXUaPh$mAC9G?c%8a{=cO!{-vP0+wCB_OH^9D zw_fj8HiT<~1gIwYP@$Ju#g2kfn*47_-Ehacu>0J}c|Th(Ycq8p=lJ{}c!;O=nro)} zp4Aii1@z^kzq4j=t}SmO@st8kWr>^0XcvMqvBnOy64AVX%vMd{51 z-`|;qf!uQ$u~z>_d*2z=WY(^&D8h^=U`LRk%!r7HfYKqNB34k5CM67rLFrrR0t6;AP^E!AxMjY5PHx6k%SOQAR+yGn0e=&x6L`PoVCt(zHgoUgO!zM z@B7*N-gmjKYg^tc>){P+8<>)IC~JU=U+6bvKWeW2KrGz;ExXGW!G+E>@+2`iKDP8S zOdSp^mzzJ4zUVi_$LemqwJKqAQTbZblPiRAv)vuIj$p+rJ)-%fOc#TLN>-3Ze+oTj z$m)q_@!^hY7x5(iscQWe$rq*uayWd?wY8*B{+rv61=vVf8ED?ewDl&((wvV9L~pp_Abpw^OIm;UUt}WxVE4a#<7bS`p{!frH#opMMDZmB zq8hP?Cr2j7Nr3TJ$FlR~&F=miy%;^En^L7}^lkndT4gG)8%(xk=U(1PTNO0WVzJ^b z9h=(yh;WnVB+~Clw$cuK=H+`%&KpEm5ZDNhiW%9@X8XW9S}nh9Z~e>I`Q#$b=?u;< z2%ut_K6Kl(09;?Ifo!`tmHi|}|H-dK;=7O)914cgw+*~3T4JNkA>Q5Nt2gxhzMR=_ zSX@Ojt0|@QSgGW_*feQzx2b>|i?Gqd2)Y+12a1rXagvYh31SB9`!lm--D|rvzI;}A zNW#6{T}8q6WkjlsrO*1j^}A;CY~>{Px*4SbRXyruQf)XqnHpX&sZY1r6IpsE)O=?a z=}A{}6$UO{JHgn&mH9&JkGtjvEW(Q)pX46|f80?Yj-U1hmuB2s>z+Noe=N1GP8z6h z>FROailBi2DlLeD;^Y`^+>rX_`Csd}PtzMst&xtxM&U{Ml=v4f`{voW_*-ZH-9xhS zYZf>QgbrpL+d4}Hm+zZFvBz6Y7&_khv@*mK&luW; zfLUG#B1piwR)Qz2kG-RP@$ToY4>3Q=K0P^w8IJ^sBz=LCO3|n!34n2yYr-=0}m^@140?KetrMpQPdlZg)s_7$0w&Q#I zpzD`vF1Y}UFidiDX_1xqFt)ipBuq*e^@lWPgy~=x+nflIT~KsgF|C;SfxNv zRh6fH-S_@xtb}G)e`oL}zL2qy64Xb;O}{Rl@6rcWTP$3c>NBul-1pfq2-#E7tUAf7 zPv}s%gJ~+O@I>8gpHbhfnS5P09qt;jvsmR-cWn14dvZziG^_}~UcOcg4qPvGXATAl z2Kk4go@1PZ3$+aFxxPYh>PXiSwK6@`vPKFB7jK0;0=Z8DI1jJrb2)zMnS-T{jpsye z??ZkptiDYV>Wm&rSaD$~!W}rlu#^7saw|c>m*RGt75flM-&|DhcNV8t+j#VRj~+Wq z_DX#Axn#fQm?0O0gba{!u`|}mnI*!vGGlh1&gkBCF@~O&#Sw9jI+lb_6mLwb58w*#Bx75FMtV~{z#i$GgH`#EmM zrx}3Vvj4t|W>NIxO98`lj$=|LMU0JKUS3Qo31c-^tUwK~Shm@`AMxzJut2doXFw)+$4cu+f|DB?mV^6Y7IcbWqwHNV!RQlJU5%&+4=RgXc&w(I^c$KfCIuK(7Z>qrI2?yU zp~C8hK<6nNz0M0p_dKcrr_hilztwG)17hjbnwE8T3pq=F#xSHXgDn|IJ7uBp%aSDc zEg<(0Nvl@r3?8Y=r!sgj6yJ7Aen{;i;5m$}7t$Y7?w1bfoMjy+;j86UoneewVFCgD zp{m%v*UWAh}Nu9MfeEOwMKg_JIT7_qfMzDYTfNKf`7=Q;YY>} zf~r{}!NLxYp5d#6A;v3L!P;V%=i3!d0L?YZVUj8&(MqP>P$gyx*BUguvVfMy8?ZIG zTOJHmJMvv|^B9=@nmj*<7LENQ)5SuYEoHGtVsSpK3@l!u2IeoeQbN1idqg33*9lpE{K4U-A60k zoQN1^AHYqk=_c#K7HV?JC!Mnf8`dtaKM&Mu&+>ZDh_Ek|xKYsRG8 z2w)z7;2sH^SQl)QH`zD0+m%^Na-~2ZWuBZl1X|V(Ihh`cjUwK`Ht@svcLbN0y+MCL zI6dNsr}#0F&8Vks;^W|wD;SS8l@3#;hH+g6jB0UQoqwdwX#sI1oj@D{1XZ5UJTU$P zpAuMuLBT!zlATrJ3Aq}#-+>b^IguaV)aFhBX>dIc4-BQLkc+U}yQbf03Esd>%J)!6 zafG58erPu=eg{zQy>X|eqC^S_rSYe+RTiW?&L;4Hlysk(d2UQ(Qf@*bnh`sMLT29Z zGZ(VgIOIs8AdE6)2WvBz{TSBHK|=-*G%(^q5F(_Tv)8j-h~HyZ#gq53C_U3`VY##~ zXj?!aTs*`J*8E!^T*U9PHQhd?*lGLmvu7lKk|fs3L};TfLk2Vi8|n2eO~MsTR%whb z={vXnq1Z<0*b$B31CtChxHM>V6G@fiQ1h-~G|<8>x54)TnkV%}U!DkSi#j6IvdaxL zeqwc;<`|%jjXMqOuDIV`HPE(64iMPBO-Uv7e;{P0?O65(^i+^>J*J7&|KQMFs5>Ku?y znq1cp_^8laWHF_*y@VmvoSJ~PkdHNlE5*F7_G8+YIwm@c0^m%$Py|oie4jzFlY}{4 znfws3&bg;paCbEdDlHP zt;Ad2!^P4Yi~$f-G$U^~R_b}GgLIq}!&wJolN6p5%i(2SlyEd8e*E=GaEb^?sXyjQ zlCr{`f)SzVD0aEFWOOz~%FY9MVbY^Z!%49Sw*B80!9C~R$s&J38hxNSj#aN6l6piN z@$;uZT;!LTw7njYnAZUEAs&ATXWA1`tMolHNF;j-2hX_0w;HzN?e^A zN-A~&$B$oj>XTtB%re5O0848#*7M+Wr~y0LH=ay;D#XVU7?~KXvAD>%zNuVDEr!%z z9w-f_aPC9IA|?k*@rq>|l~|Ihn#k2@7%6PnN!&=c?oZ(vltba-dR{Q-@7q3kjC!I{ z7BaCUV;(%&lyd#^{(Vc6KF1s}KX@xu8J5}kNOcwLYK?gP{Wrv35jS`2&aJr~l0xtA zyn__<-}4TXp8aRM14f_`t8;4$%0y5E`DL~NsT%1!^t`^bz}n95>((s*tK`9?N?K_~ zzOL$2G^&Ox!KR!@q|Xx!9^(Si2eh_(1;3r3JN6LMcc=5E>VUoab1)eMIJPRI%+2Ge zyv@WJFj?rl$!-xMB&Sxcr-w(gtovsMwi`U`zy-tt;A6-= zM0|$OvQHk*`T3?qkh5p1q?4lsQd|L*zZIF@!iQH%HFsVtl>Y-}-c#gBT-zhvUyuQy z{J_<_&iHU|<-pnh$o1vJXfAsw#=J=6lW zv{q_b3)3&QrGM(ZK5t(o(WT%gAy{=NiYyxnhU`Hc=?@U1<~%Kppsmi1-Nb;XP_l^j zl_BvbhDt&n!}t3xZpQ(Zg-YPS-|2m0)3&e+<$1_^2=l3-yY_b@Y(xM8k{8UFvt=?) z%il=09h%4}BbChmCr|i>io?EeJL)Fp}ub-C5+vf zVh%#)c!ZmN7a~_$dj7OS7Y~(*4IB>zj?Y>yEwFykdhhyMTFiBYz^RRMR}TVUC~n_sq#a~i!HT9I#6+wmEO;2Y<} z(QWv2zH{kZ6vCKhhN$W!ZpM{SiltcK$Rbt&TYegVR!`66vz!BDp~6k9%rOvooaF*{ z)|_Rs3fzVB*m=6(C1$mhWCp%?mqkD5X+h$rqs|v#+kQs+OTSwuGwUZ=l1%<-^#b4# zuwE0ak|gP-ek=rpC~=^-h`6m{UWDXV1s566W?}6GR22Z(<--Yu9zGjB4PI>cnVUT2 z$`$VSuRbm`>tBC5ked9{Nlqoq0c_o$(>R=X4UT<-VzgeQYDID^Yq$i?(b$}mCTXn>n;@J^T zR*h<=wB%i%U6FER#u%%vhvN8(qth{b519)hqAXN+zKZk3JGn+hRsd} z;W^*ctx@ylutca*aI2-DsGneP-=~aop7A({qSyJHP~kL-a+kCSb{$kioCu7Kol^)iM8N0+I@jcFd9F{g4j0p zper>T?m3-5DQqya6x;sL@{isVT8%@L|X%WnA%{A^7=?bSig;;A9tbU9S&B>6I%BEdHg z(1L>FA^uqTDu5u!-v38Pla1&p>(-6O*iBAXFScJbs@56v(rjF4;)DCjd|I!18k?u6v2EHKtGwjSUN zl?tA7B7ELaZY_75KQ-Rdi;6d+9<-l4n07Bv?J&6EF2G`NB|q_UL$uOVUPLx{r`&25 zf8C#`I}v5pu}SVw37C^Kt@hY+JkhvPAo(f5h-@FBVD8zCJ8=;U#W3j;>i2h}vcx1dxvBPNGnoOCGmp{zid%~dJnC|vq zq2?(?dKu=-(~DS?BHCt7`AcKzS1(J_XP%3we(D;{hQv>!0oPz*WvW|_4;jPch8Abs z9;L9T{s|%JvuI46EJQe9430nq?grRK<`kUmA59cMP+>0QVr^+OMVBnT*|Vn_zG^U1l>`79)B)Wi`1v6tHaPk=WhEW#S4$MWMuJv_3LSjeVCemM}J` z!|bMx35E~6-GCOK8$N>L%vL@;re#W|aK>A+?ajg}ECI$g!D*~98T-j2I7~7;4J4_E zYbiIfcnO9fZSe(+8#ryula^sqM8pn30tN%;#p9)V{q7z3rFf*X>^mSRs^_J4F1rK- z?)RRwI%#P^Qt3C|AgabE8|WoejJh|nNRT?)D;}Z>7I=d>dea;f zh&%WaD$$j&iSiCE^3}_Xc)AVZ{m3(xzp1i*+I-Xa7;CHlwAtwTm|+TIwz?yI?yc$>eSLLkZ;Bw{klP`^Kt^3$S$`~d?tg0{6 z&GSG0W3a>gLwOvrP;6g5R?U;Lt>920l2&bhC^<9bn>3EvIw@S%1x9!*Xvv8;)WOS% zg+0X0uxS{6mihsQp`K%xuyXCo4^fpAdANX0owhvOa~$abaRvUN*t)Q1fN2>VYJCyAcTTk%)z8`b%osn^;Mss8rHRlfOtGb&s3)C~xeW&sFP})b!_0^? zW1dvn?zNDH()VDx*SU18s#*KuC;3gpE$>gCTGr1bT(|RE3n9l+hDS7cHQ>+3-&y&& zE-N3eIQkj;>YQyfUiUUWDHDeF^l^$UyHX3Uk!*^shMRvOX3VE$KgD12hhREonOy*@KS>=KUNbU5 zbc#}0icdO-g+~>Bv4rj(vy}~7g{Q}O84o{UP4JjYkgj$lYkaYjRYo>ahx7WfgI20Z@r#oOw_m=jcF~hUfsW=gK8!B_;R>i| zEiRr6pBtL;%zUzT_ZihZcoVfKLzwV(Aj$K6Nvtm*=AxnCu^$1{(R6}We!p|c83FlW ziyqEJGtBY9@~Ra=*xJ6Xs)uN=k51!-pQtUWd`7*>>y9;3TRb%E-!8FlK4+V1)4z~+ zhPaDi*sAWW&)umk^vQ2UeO|!yc$E^MBh;mt!6EC6ay~8K7ye&@0RK;iLC7Edij)<{ zz!PQ$*fIcfo07zU+x3J3B5C=0d545=g{bic^ywZtqLo`)KjnE2gPQEkE-FPyVi)3S z!g)MG)GnjmtaDiaz@I-|>kRY<(dx6UOz()NBawl1asG0tthPD>mS*3= z&^n#AD5&3j90ce^Rv#X2Ux3ihxDSHAls>%rIl$H1v>QOcOR54kT~!H!usqbn_hzNP zlPb8-g^?PUtX|mTBnNhdiQ=!p-$$mkH&TSu+Xy=l&THuwGR>Lzord^9n~<{OU=1j} zaO4=WLpmEI{{_g9xikOF&=9=vlG!YzW%JsZm;P!5Kx`^pjC~&NyelmK96R()FEak8 zXn?z+tMG8>URJDHWvTmnB}1ObDEW5kt1*DNCA$U1re!XqI~t62uw#L1jE+~eV3#~6ekXKZNIU5Ke2 z7ciurDZXdltm90Ag?-djJ=NhQ{}Eu zW)L6%IChXmt5uA#*%Bn?`pEefos11vCHV_+(H8+>7&Z)}Z~^0>wiA&tLR%9i26$3^ z?uF6e>&qNvR%g!F!W*fN`D0~_wToMnw{#?uD9XpqHMChEr8C*lY-~qkIkD7hy8A_d z@C^=_<_`;)W?rZB(sc}+Ui}d7iuY2bN}O^1P^bfbqDjG!yfNtGfbeKy4a0W!uy&+mwS6Tw+Rb%v`!n{cu1 zts6UH?Z)XoM3l+bd<@CLF30*&O*+{QLix-Jbg6MDTI+F;9h86h#V4~nLzNVFUrtSd zvWl~?7>m@!Tj^z_XIMAUC**bF!mZ)KeI2t`E@f*nO1n!^5?&0%r!Ay2{Yh^)x-{wlsSn$OROnh3I>*O&3QwGIaCG&>3lfJO?Kv1fD1Z7YFVC7-hxq24$ax zT{mN|s&d8I+I?P=oN4&&GMFHhLo4>)wS2xl@yYhy?(gkfW3h_M=-4`a+*HYd@J1@E zCtPlr6cXDX;H{Y+Gxq9tB;~OX6+B(hs3-jP4#&_K5Eu7G^Tr7<7&uH z->#c|;Q5o4I<5r$Cp{;}@SCot*_^sN)0z#t<@nHBr2^l(J-~6VAx(8m4gj}r&BgV{ zTcr>))-L#YPo)QRo@hvJjC`;iz7nihlmrMGVmcIG^#_J8eg2H(`m)5^zcS5+bcf&! z-()})GnQ^vawDN(C4+dI0C@u1zS-=X5rF__cA3MjUE-GZxwy_4>xk=G4p+7%g-|T* z*Y7PzKE+V*^U+qne%GXon?7@ke92<05AN_K&#`21Gi~P;tM~V-S9uTYco;%9;gDGf zl(`}!y8exOE;-w-_t%BcHbD>l7-RjS7>Bez^s>_MOTZzQ7lI87IU$SY1Mf&uAX0T2 zt+*`Sj&5K%uSVQ+Ms`_&qWEfy8VqS80x)@(<@~W@C&QMm!o+H0FD5HrbY*)!%Kv1- zjDBQNNvIQC89Z5SV7PLV7o11UX`wA7Yy97w?~hKvABD|dp?~fG zhI>Ah63uz^O;I)vHO>L4Ots?$+swy8>*UR%aekcNkB(@Zm~0Dlq7v*lxPe{)P&??; zA98dCvj?#O%QjgEu4>Yoy%r|2MC@SuHN&;|Y@#?YAtjp=T_EpI*m>}_~c+8urFykPr zBdN!qQPYz`L_dLjh8XneOrAM;bN!9KYTk~U0|EJ9Ei#P?0q&ip%*0Dk*Z%yfjh%Fc z52d~Ov*yzRRV$fZ49}E5q_tunvBcTw*P9Vc=KR%6w2w(!?W^3Rx@uL$r4|C&4taMn z;#`ODhQq88WQ}Z#yH5!1=ihbiB);G=7y(*C`O}*6_;$oFe=xyWjccS7H!Mwqn*+_O za8}l8AcDuJ?8|{1UW$?^Te*alPW zdmXs09V}T)&yrhRqsg1Kj!$&vh^?qOFGydr*px2v^y^@!Bc{g>O`)QtJl>6FQ>J?n zm2znG9b%k&l)v*8aMwpF8&G2XtxBB-VL`{19ED>h=Y`1rST}#w=Fyj`P;40`%3d&&Tau37S+RKS8Ah+ee;**fdK7njE`f^k|~5UN%gxe zaW*SHUaxh&KdgGQJk8ig`~9R9Vu~I1l~zGGDw+{Y_=#+}Z5xfp&DY;zYA5SyUiUiB zY4$-~ z)s)!vfHsr!(#-)8JCtS$%$9k>e5rkJ7J0^Ic8v{a9Vka?1;K|?l1v+4ckX;xpf)f+ z_F^Zg(A zF+)z3J*wpx`o?y~xX^Sw(URY}Gk7|rwkYBfk@#b*9a5$tzMU2J%(#3qYgxA?=kqhx z=V_f$3yY0jG)v==rUt7DtxWZ9B2*pDd|9K7B^cDY2Xwt^VVRBBxg8`T9q-2N+O2oO z0JXGAu_L$$lJ9Ypmf#z|_zBODmoHb`cVfI46GV0b-=g~8{_$T;QDq-d@%I`3U03D; zxScU`Ee;1(zecVbk7fs}#dkU!r3|XVKh^+9QO(ua1`b zp+|Q8_>MPZs&@I&Q>Fb;3V7}zKd%wOIZ^kGTn^1H+_Jo1=#*t=cqZEytRy*bZ(Y`o z@+$fIi0($d&+ogj4L4hPQGOD!;i4qu?=)V2|H3hLv9Q3-7`gKX1NGHA;X&M1Fz_SL zPHiv@m{5d)-hl7rN1npA4I`CPZ*MWiHwBZcJH$5tH0@mTFm!Txbj=N$81xI)I|d=O zq%4&|%(!D^{geh$IdW{ZpW*RI5N}|E1lxgEVet;9@ zZ3^$Lpu_9w53Y-#UY8yt`osLaM27?M>q;B9h~r-}{lzWy!#Lu zkGY3>fpWAffp1zPjr(=!VP^s@Ify3W*_3;mcxO*i9!9a2d&7Ke)URqS+iSJ!@RYCP zkom}#+nY|((UhIIau@PqWBK$4*_@5N*)pWc^=;4=JLa7Z%CZAKf+^LyrV-oBB5f6) zdV`bDfll#;)(+`&>g@zMKNc%0PD&M-cOA`GIPsqxl6inq%HVE=1H5Dof;LxkmOk%kW8>W9S?{DLG;1+r%Efo@=#fUg$`e|* zsc+G{p%_>-S|7z|_jG`CP*5y7cCIODi?U^Q`~%{TNP$Lt>Tz@H&yJ~9lNJo0`N^-n zdaS6c{G-N)L?6B3%EeWCMy@$+>;S4R9ilY%B6334D0+!g1G=H$J}Ap{*%@Pw&QLy9;2tr8U6}IUHS8EQ5jX%ul+g4aZ--+zTC^E7M$8j{F&b<^ocL ze|edRY7BQcf+)|6{8Kkm%a}otTcSQ_BX(+2G_h>m%i@rRv;cToBK0DkFyaRPhk4W)&NfVS5+-Qm7r)s zPxiBecYBju?*He}#qSFp=sL5>dp+LOHKFVe2@eAU+dY33(CxoQPk;B=e?74OpYOu> zO#tP;tNU?m;MZKL-ppbmCRS5cI3?nozo)FI`Q!_(^A!J-vEgf^Dlv zg+6*+%#eEZUs@0;^1w)l;kJ+U!Yb=$=hV0Y#m&h;q#qs6#*SacPb?s-a}o7@uQg8A z?Og*dhF4FQ*mFF;o_88hX>wSE)A*&eHRvN$eNKE#2!$-)2#X==wO3wGXTN)RHsmtG zR|b1bCcNGJeQ$aL=@)Qq9G<`8>3lsa{(?c`h<0Uab7S)FJcfrr_2N1BCVyX0KuechMOwj)Hsk zRxY;dz}`?NUZ3IyMui~TWi{h~HcCH<--q;>=>59i_dzpTXVj2|;uGxv3==vmgfh|9t)Z%c1@U~^@|d7jQG-Io@Tz%3jd z&^;UO@(8)-H?pSyon&Y07SB$_&Fk&o*x+=1uW_5Im!yR9X#&ShYrLnt40QRq@Ve6Y zcNKQ(FWVIGhR=+vRQ46c`9UTZlEDL^oYvJgVsxdhYQ&v}GMnBPDmG)Kj1RY{kys+M zb4|mGSQfKJ549{fu`;GBJVb^t@*;alKlCcW(5!)$;KL^|F(s7~+mdd&J~ug99n|Fp znOw-FP%fhLOQqj)>Exu$A&$@2^9HXRVt?T3^rJ^c+&j}I-#u(V>x?3dP7bQB+_&|! zqyq5)sZV(k-tTpXrm`=ot9~I(kPG0}y`9IFrZGvH9$Uv2`L9oiw^@f&D*pkDJ9$vL z=NT;1N4Ja&y*Z`mTOaAD-lJ-{?+KC^g z1r+Gg*PV8URyUN3>m{Q1fp<@%COLbi^oxtE^%UYM6QwUWRVrpv+*REON3{xy9@Eyf zq{N*@oE8Ad)`46=c639wR@Nn_qfiG?B9_?uW&yp@A~sYx#~~9tAbRDwHq1)<>fo&F zz3$k6k!Q>28f9~~l!rLhO!X3%yN*khOLn7n9H-_u_-queSv%z7n+yRF^EOA4YDy;^C)aSH#*Zur`akIROXY8Aa^6{wDix1=%pgx5aU^|8b{KE$K z6$|mj$PS8H_P|YTwu%Y-M$AKQk1ReNiLUn@RbEJG4*Ut4kQ^>8IiPf{`tv%7yU4+g z&T^`(R;Ra<#BM&aMRoPfKLNu8%fuYgPGyZ1m!kgsBW8}@%hXejtjPG2nD~#7;*a2A h;h_Oli;APlmxx}$FELhHt1bdQhaH>_lQX1(iI zZ&6WEgDKCp^t6;uL-C~=%I$>DEnO|DlJERWlm(5Wrhz6ERcS2K!Gn{OHG}8%yFOG@ zEG@^^i4KoKdnzg}KE11&w*#!!(k=W}KAsLERLP!+?LZz~m5twe(=>bY%-}@d{jM70)OU8zMH(ZT92)HDZkr6VfrI;{6Ah$T%skvub z)CtXA`09TrJa&v}7st-+j0+?`y?C@|wR~C++CbBw)3RS%UozT`yWGB%vOJAZ#ZS$mWL2_#Df+ylY_sVY~I}7eO^798M#F5KZXr~!k-gKTrvTo`A2z&)pZ^+ftN3)QDUDzrz=~ciDgV4A z0`YcgcVgz10m0L@6>V-r#>17&T-OeXi`4ySq-qk0DsTtV;na6|Y$g<%El z;$enzc%_f&4cuO^71n^#E+z>_;J5HHNJ0vqRUZlhb$@$nfU>LCJV$o%Ru4-J)2u^ z8^l*+-~9}d9;27dt0d-VGt#*XEb7SwxQUL;81zlN_R!@GUTEg%zvo$Y%UOauD{I7( zE_qwk7~))7Vy38jZ!d*^v9e0TdEdvFP5OL**vn@?T$X*YC&uJ^@%*?znld`y&_d{8 zZvAxa(h>~e9%W3UB-7silwau^iYMm0Rk9b6P#`1Hg)zV91JvXT4b5yV;J1-=2E$1 ztb-|<JUxDGWt&2Cjos%hw46B zk;OpvQH?wKcpSVXn&Zd$gx$vX~M2G5!voTaSaP3SRuNs+M6 zxTQo#_&)zu%s!1}^!XeS3_~LA{QET}MOKFypg0hca=YR)!{d0)x4$y^o;x72&1>g= znR&nwqV_%T^I0@)zG6Oo@BT*FY5Jh%Ez^2#jW2}WI8#^L-Fv_@aJ`Xs51b~Mlk4+3 zb+(K<OH^(J5a?vk1TJk?uEZE!)yE@a9$GYi>2&Xv86 zSxiw8lft>+047 z>4QCNz^zN~F2u^T)P}mHZLQq9KxHR%>4O{#Z$~X-&X~!)kL8>mf#LoV--vlm<&_0f zdofVt)#cFfm>;@U&y|yv#Z6pH4@*qG$|MPNw~ff|Xr*-b_Mi(Va~-d*5?j*S z-nh!)UCMsK%eUaQioFf>d{L>>)sXqo)It0gSx~?G0RKMWh{RwH_F~u$U_5j3av&GY1u5>9Y`ODjZ+nSO_ zm3flZv6{H?w+iLWLLF!;X)!@B{yS^67^>omk(yE_#U1A?N}?kjKQh~rxoGD{*LZtF zv$XG6tzyMK_U?oSFjOadKRvx^aWmN7_b^N+bovqW{v7=!u)ShcMBFdRse7k`@5E_O z)HkJj2lD%7IN9_Gs|0NPYzn1g6-0V(uu%AjJ|XVH|Ad~7<6hD1x^w)tLSqE&b4fZe z@oGkw*pDpo=Wo;I?@xpH+11^*L)2D;@=3<9+Pb@|u_Opt{MHUYGliW?;1HCMGR+vqC;eASBA4PvV8d-DyNKxTO76T z20pM@oi-;fbp~s!y&B3!uw}U0d}^`J_=)#=mJ-ofTfX85rE7nav^BsL>lCuv9n)=u zf2fPz;9VeW8M~R&7k`e%3>FH^cM%Vmdw;>Q2%fW6T@{-VvTGGwn9GP4FtfK7p96&|_s2IYJAYH0HlhOHnIw zZ=|xMven(GolV`HulNo)b7hQbk2gLG6fsxCrtEuZpjQWa#nb6A_?d&1cor-> z&Txwi2Xmb6ayjDCUKUc`3mbV)`hxBYA=y?p_mRd&I}_NPl7}d3BSp;~h5XVU^1{xm zDUXHrmv!vNI*ylnIu0?)MXu7YJ&oO|L%cxx1gYpMa6NaMz83H`7ZNGH?;25Mmos3L z=rNFUIKR@aw2la(6D&5P@ekegx8wU@w3>}|1D(xeTKjCBBi-{E-&65XLsc_f;lu6- zdCUqJ6Uz}O_8TXyhc@)py#K9)>~GA{{`dA)v6}lS!2Oy36iC?YSLN!Q>Ym-(?Jr17vmv6;@kowCJlUb`VBjz86GW3FQag%B)e#p4ZkjV@7J!5kJ z1R?X28=|nQuC}|()N)?5mw>%v$<1`7?2q{fVA_e&wHTGyD%_AMmc!gzxtCNp@+L<2g8;g=nfB z4-!tQ0`pv}aiUTPMVtSyYgor}ge?f%GnsTQSl0WRehLM&`;SuM3=v4ny3_bYz_8hc<~wN?J1Qv)l0>*(fyyyeQwWNo)QwJ`I5cz zX3On$c8lZ8*WbbL`^^3jKh^DawS$%A5+1|Bs57KT6DTv^eGd+^KmD# z&pf9OPRyCN2Ym-d?>RJ`Gl=0!Zu)`yr)JFgS%Uw+!*@vk-hWI@lE#PD!4758|1xwV zZK|;`fQfT+s^vfyHp>>DYE%@R+bRd1(7oPLD&wT8n;cYe~GGt)d~E1hQ#fk4hG)FYdiavYL} z8|g7d{`S{$(gCBs{7kldQo1-35|KnmYEsj3U}N!0>!25x3!lC|H!?R`AKy*(1a6ZBhY3tmTnBkQ#ry5pl|KUoT(2xB1#FAg3J`C^bdLG=DR_|U|8|QvS z80usbR=~%-%9QAT^1Cu*;l+!(hoL$Uo4QG>JO1L20u_;m?nCvPCI_jL{oKDf)>aj( zHF!h0TRALjWKJM?5QbDAc(#>;PEx~OOkUBosNmw*; zro(_)2)2B+)>O_@jtC?W@&M)uGdd;T2YzSR~=Im^g!vGPXBzmE-;FWYr{iCpn*XY^GS;}`Q+ukpaI z+CHdkvcnZ&00j`SDTD;)rS$gJ;w9;g^s-x1n#41c%gnf#GR?6qUrc5HGJBmXre|e- ztqeIP#O~&$s7_0UrF}HIaHI=_NLmaDF>D@PEbPotW^u&i9|^X)m7~x&dnpwF*7-4P zU0W^yN@`g1x6r(h7NB)%;aq%9c_jZgm?3*H` zjM#gcsI5cdJdRe|z#dm6#zt0Ry{z}6(CNPtV*YCCE-sp`Pd3}t&CkeMG#Z*ZbAJP0 zG9)S8;&lbBKqgcPd0Vd=Yo%{wPnn|P$~S-zRl-mRkCXJI>i9`fpE?f1PWs=HLCi6v zJu*_kioi)1>=ldj+4o#^pl@Yr;uMsVK#SZkr?0qDRoFSh+CG=MU5FwE6~z=C84@Sg z=gcI0M|0``cSssV>JIw@cG44g%2u;t7I(#dK=g-(Qz%V$a4V@3^^yHUAK$NC>D|_j zFsVRqGu`FsZB$VpgsMR>#=m>5yD_ef0kkPN*^jk^1|~)li#7{D;_XGU*7kSq16oQ& znKS{SeT{K*0doxd971kg%C6(n@wzANeJO`q$D;55O>`6jfei}c)+Z)n^kHE|2Z&FZ z^Ka)=|YfeMxBwS;^4@qx{U%yCvEz9$n$^Vc9q+Sz>aL^lR3 zW!y!MJo$S<$HL9+C@ddbXOqdD#){vRdG&l`I3bbLnp-w!S9BX@nK98)FrHxlE~Gxm z1cDUmy|NZLOZKQJ`%u4cY%~sd*b7;hn;paO6|liZElVRmu|Hlp>h)Gnd6!vZH--s! zhQTS=9($%gW34NJYqM_4DNhMA7-n#xVf@=7ZbKbD|&u&iCzW&V{rx%uE{h=;J*HZ$acRI`^q4Lr8nAH3u7`qgIfazz3Az%$9iA*3=h zgv*1il+1b$Tb@m->PLR_+^qPMGG*{|IflyR;_mmrpMU+=SY)8S1$0{nBCF&UKE z5Ro9WoU?UP*Cq6OPgmuTAa8O+WkkSQN9{25evMr!L|kxdV$kx7-<3eKKT`ic&z6V; zT$Yq?^|glL?PyP*(hxw5j7U!Dx1YOeev#754=DeM?~lrUg7D(>9>4fQY&iR{_2%BN0MP@eWi5Gvv`|FHZTI9FW zz692~1YSJ#N3zhxnkfd*QG9!tUMI(YY2%OE{I7P9o0>YJ0~y+`dMwCHQ_jw*^sX6R JEzz=h`fm=D=EML1 literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/api_precision_compare_details.png b/debug/accuracy_tools/msprobe/docs/img/api_precision_compare_details.png new file mode 100644 index 0000000000000000000000000000000000000000..b31ba52350989488df1245f4840886f2a7db4944 GIT binary patch literal 25476 zcmd?R2T+sk)-Eh6qM{;#6hUI8C{?8iB!U7KihxQdG(n_S38W~9NGMTJKtQ5cupmW+ z&?7g#*S%8C z+gNVhyleBCHEXt>Iep50%^ERt;op-tZV>(+kac6H@DGtldrPx5C0&YB!Y|hQn_8Q$ zSyPS@=Xr?=zupvf+BtH~nr%%#e~I8jU;3_D<0F6Ol<9?7kGV&#m9J}L%pI%Mw~y0V z7n<)4fyAkur#FrnChQ}vw+Pjsso%w0`75hB3?o+gEXujB9LlOdfAyP6J)Y=2#P2??X z`|eUfO{a9s02cHi*ZkucuKYokOa60!<=(&aFYO2AALiKEeK1{U(~f(Wp|~oqjFeo_ zM$*WmU#o`+s@23%b-uyozwk`Ip4_bsg)H2farzb>7B>taaJoCFf|}v^Qs00MpSa<9 z`mTCB4Pu3usK(C#aom@xkI_GPDxiA_*(el3T=05zcz6R+Xow6HyJE|R1EQj#+g)XcKOGmooyOEH#+ShdHpq5H@RH;TSQ3;IwXz2Nm6G)b zYf@73?3FWI`-T+wcYRo27gYvjS8f_?<}#9QU;8NDVI`)Rr1c`q=Q6NmqfX(n4DAlB za>U^3zzwq@gEUJ2cRKI!ZIbeA%mRHNmTi&cqM0TJrRlre?e#dxW|>jz9yJjd%S*er zl;S3=EBSXar{3FQ3ZF{97x0<>jKG`K2LeANqSLT(^KBA@`g`d9bTPcf(#$#w0y>Zb z0U>}bOdS&mP4ak|a-S8yBhKTya@d1+<0ic_W#}*Q{S%{AyaOIkqfdlOaNVqq@HhA~ zq@Dcx+J;M(dUpUEXUJ+algL*W*&}l-vZGd>r94kM_Rv5n29h2n$WY3IjYLmVzypn> z>h~g?5yOgMTj(S5N&50$&q!0b_)SKBiQ&!x3oA2S1PWzFYZhg=r_ZwMCcgtXr=R_ zHKyH`l=*a7&e-CdO2;SBA{N_tBo$dy{e@fcnZBRQwuaCX?2%7C9ksGaKuFGfOr4Ed z4Gh|kpMN!OYYb=1sgFUA_Q>WdWk*gJFspS1e`ND?_2mp%ZQIMljlHswyy&!%T~fT@ zQimA^=b-Yn?szC^B5TR|K?+BUEFp%Eraw27BpBio1)uo7skQqCY+J#Z0;tvJuDvda%Cqf z(;S;p&Y)d|lRE*jt8wZ*?+muu0UG1f&qw-D-Bw0cJ}8OP#kkHZbd3p^|F9b6*aLv= zwAjqWkSqI26QL;Df*fn`@Rhq-_|l17#e7+|RwM+J?si`oz{ck< zrqM=+(hA*cftw=@*ozfJa~R8~P1aX~4=AKU>fjx9C-vd8Jehoa_5PRw#-q(y_$sZ& z$C~$^JGKWVLZMz6K1$!)iW&1Ma`%l9r&=HIzNQxXZ5NDm#$W#Kc|M6Ms%7)Z)K5j^ z$nEm_dV~9Y<@k9Y?T~XDtoDqHpGMAI^TK6mwI9(LU5gZ-=?ksmX#y!dvJVBH>@q;r zVP04qbR&3uUT}x#L1Vw#@#@{+J5BufzJ800VJX&LgHQu-K7LZxmV~hdb2qUr+FOJi%Ie3RbnrH}u=E%jNLfeNDBJJn!G9BTc z0iK^}nfuF$kCDA$^QpAPo1Y;@iGWnXCLZ9-IDku=IMuGGyBICpkfZI?(g*R5EFV)) znJopd+J7g#^t<7V%*_55KlmGzWW7NMft=<9ew3TG>XZ1fv4m6IvW_z8jK9iOIv9t2 zw%_WZ8zoM zcRX7!_z~7Ud|RpLSwP9>)oK|`h*1IS4j_@Rm*OaVZmucz9RHOD*c8gF*~_D zPIX%lp-%5Kzn{zV1AWt;U_2{)xyEtPK?AuWdtgdj$~FK}l~&&`65@VM^7Ex8Us9Xr zP}UX&tqwuLcgJQQP}{*wkEO_~ekSqd9(~R#&15QrEt{+s&U-F_-LB#mFx^G6)|2)ceoF^6<2=WmSh+RN7iMhrid`+y8yAXehwJ9bs8f)*Wo>7T&< zmL3e*;%AiIFs zotWnVSP|3^pO4JkNNO?mA~+s6xPXr%GAz!1IGgS%1pSEu)v+w=knTU*JFJZ|GQm^Gxf9r6LjjsdQO` zkiuMW>rK*SDGKl6G6ek`J+Wro_oI#^mhus=4KhO?@n`7yMYz{pP{zNm_4IBAZ9}zH zvDj`+MJavaGfj$3p&lrN33wi}>JIG=mjuL{N;kiMU{IPV>zSondDjP%<0H+z-alLQ z!)M<>S0?U?hw3n~Lc3w}lutH3Pqf;;-}9!NX=wPiYa#juaD}C7j0SI6xmsb3&#Ith z8OSEgaNiNwE>qW%4KG()beTJeRh|P7Sv~7K3kAkL|C_z*_vO%R$fqxlV?1c6LKDonSaGuvz3S zaUw5*80C4K$n|o0Z%`fS2Oj~NBCp?KY)=bkcU)VPoIft5L^`HOGWdi^9;fXF-QEi= zmA($$>S0vQm7{Yt@b6}`ecBS2vxz!>Hfn;os(XO3`G_EjHtiYbsQ#=ye9XVGm^D8$ z3x&7n!gnXmKsR4&9NIOTnu!Xq&_hrSDzzf9R_|#>vAah;hVv>Xn`+?X9^%1h&4r@7 z4%L8I)n&XEUQ1+;Ul10~#?eP+PR^2Ad`I;NJH`FpGVZ3A*8zT>{Q&qSZQE}X?Tg9B zwr4}??I|NI##)&x;SALb=jnBeeI4BsIpO)memg+Pk+TJ_HT1u5ckcm_`I2BztQDz^ zVR_T7&9+<>c_v`@mhsdxf&|>wtJt51!3gv8^c8To=c%4rkvijy(a3cQp>W#@u6Wq?bd^*J)q$DDLFr^z3GtX9&AUt>y7&VP!r( zz=HZGt}_lfhmVH1c{!J*SF@8=H>~S>#cVWcY@o?m+%Cr(Sejru!iNs|?0xnHb-X&2 zOpW|7k!5D*{V;JDNVia_f!~TuPSJsf2dOG1F8H@c*Gb~7bnz~C7%|GAEPD^p_Ojqz z5gA6xmlV7-+ejEZ*hHDLNj1nJ-_pADlLENEix!`Qeh$x=E|SR|GGxeup1zeq!JmYG|0D=a1vGE3^Vn=UFQ(&@ z|IIsScwMqT46@#=&>sW75$!i{?xOcS4^Fr|3HWrli=gLGbpqQr zQ%Bh_#!#R5YQGA#d%QGM6N-Ytt}{&$!{Li=8>w(v&Zdov_)KoN&)#WNy$7K^9H4!e z`LO5X(gq`E=y|F#b@qO33hJXQx4vNzGw1M~CuqQdJ3mI0v z6McBB+2|P@3T5c_Xk4sGLa3%c5s37I8c zM-0(6<7LvPwjvoQr>c8sGrO-_JND?)#vTDWp0#UNee%`VR(U-(el-E6ET1OdE&c|w zFC`<1e^Bbuhpn3VfyD*(pwzZIxMaC|)Nd=ESz6_nejPooh#PYz!1rIOncTBXXQiOq zm?Py6Q@S-V3o-3<$mj4rR9w*wwBR;yiO zc>B*d&w)6YjW*<_ZCeRGiVN`4R0H_qCe9phA@4#!tbnXBmB(TEJ;u59EjlDlxb%d7 zHClxLXxhnl(wiUv)5K#N%O_$#v4ICm3>w)l9CsEL2ua&NP|JBnV1&C(3&CzQ-oLC9k=n+vL5^zVBaeq%Y+Pz*C=7C~-d zC2EfXan)D)#PdBnWE`}n#WPamS{^2%JwCNPi< zJ4(AAFn?zpEINRH$ltR;ALPG>l?ZQPT(g={x;+j?9gH-tyLFqDs)u)mYVHQBFddp{ z0lAFBi~B(ZeHE{lj@JS~Mvmp{)SG6KAcDc0hWaD%DLcYMQGP`^TG=P#k~aaFNs#)> z^ZE;pL&eST|cvcH;JkG@&eE@Zd?ZUCwD2f{{sl7sLH>mb$>qVBp!1bwN zKxQZBh9jfDDjj>n)iuPEWfWG|Q=#leP75$v=2%`o%xOaw#(a^zwJD@NHKdC0-_#6$NQ&^kj8xoj_WCRpXNH~C_LA<|wu+?9!Ep&4e|oDNm%Xq}|= zxqJ6)ix7f2Ed8pFf>Dg^SFvVeyeyhpZOeF?r0uwGyMLV55w0HH6Yxz|YNyKn=G#N_ z8eJn@Z<7sLQfZ@t{T=O5Qpxop50@FTq!EKnZTJu&QSn@1BC85ukSxgeK?ihIA=Z zwmQ4(X+O3;z}wh_XMNpQvBEGwAXhRi7Sx93%jvfdT|GT?X*frtPe*aSi5HBKC@9UC z*?plaU3mU1CAw)APgF{*xPLacjRx!mDV;mqf*7w@TlEeQge`VxJHv5S5`tVE!AHpg zJo6i8U?mK{>007kpTwz<8>mp!>OF&%IzUyvQyoan*kvOX9gSLr?{I7^9Qc; ziaqO(2@�wBEDIy7l>WoR#gYU119%<{nyaB05HydmF3G<5j@+HT0uCH1zCA`G$wI z)Bf*=K0=RZeFTYqyx;ej@qPe{Qt9XpgXG{9kD|LM5$;T%GucL%Z`|D#)L>`Yy4MU; znGpWiOXF_Z!S^PZO51FEE35uI#pGF4y>-MC!!(_xvhE152`(pU?4wn zBR|*)DU!3?yGc$eA1tZk)l(sL`AbhqzsKz%aedweNjPx`BxK`k;}gHYyjv$u?QoOE zR=0Y)v!*1AXP8NOX-t5xMRB87u+rG0itkS2w6fBbQ@h-y??unaY1^*gf6^t^F-$iN znjFVLPRpxgeN`}h^a-!<$%^+{j-SzXu&S_WiaaO8(eWWEK92pRX(AoE6KtkmgmM35 zm6s2O#jfnq2nThDcQo^KFE)YOJepnXtrzyDWoC?tsK)>Fa=> z(U?s>pJ{!s6>3WWO(Vb#O2C%~;q|HGLph-I&{Ai(pWaFeZTvwLB-rG7d0vbf74H*( z3vz_KU86Z8W|*^nY62ZAvG{%;=F!PD3~Chccg6)(>}W`EvQY&yPKHe#z`dT)HP974U`GXysLQ zAaOe$|J_aRoXX9I!ENV}PZH==v3q4n{f{7xm<`h&9sEYqahl3exGW%q-EUjFA2F-q zKVV)Q*JX84lov9pJ-?PJX2WmD)&SoX#*@*k)%%&o!*eEHwu6)Lm%eXl_znn07hX$_GBG<(9{sp zt+{^Xie}5uuHrEUWUB-}S~J+Hy|N0pM}H`~@Waq?^8zhfFL2oVxGt3cL{Rf-qO(t2 zau09+rIIm~LrBd{t_PGJJLxN4P>Z>9k%g3^#eEb!{7<|dFgF@_Xuh7#sw4L_69j7 zWQ4G)lDL&?S?g~&MhMUz(5wm$<7s2;hr}O@-LQ&ta?ELjI2rxV;2#F(L3>Dt++6psn6R;z$E+>5`9hM2GhcW`tU; z-=vDc3}6DcZ-hSHK$3jo3rdw*GA9~hlD}S7_90b;NSo$IloA7JGuGd6Rh0{N>9dzn ze%0~lORE>}u&VC>GMnJ}8pNRkiuy7M(O)LkIJES+pgC2$7yGhP>vhk#AEm-QUP%2u2V$=O6k|M8 z@gB%fo3N0ylU1wV&Rul`Cn zio8q)N2RTiiszDSq0rI}*tP|+QIGBj>J`yack$HV;&5Vk5RsT3khvGM&>(oob#oe) z$x9f*Ekn|!@H$jq_yym#;UWFaErypd)yX@mY(fpS@;!Ka9PfbGMQbB2)|Co~KE~`Y zd1SRYZ9>;MCO+{;Er*lpxlNxD8PG8|E)Cx%4B`dp;4kq<(h7?A;}N$~yl(>2MyoLf z(IKp(XXh&n&Ve$_{=ItoOQ}qo#KXT_7TiGwV!k&uQOr3_)%O6d(&owEzW(b^SO1cA z_l7%PkN!U(;>OwD>duTFV-vLP(!M0y8M9=Hy5`^(ZS^w#Q+T`oUx}I#$~Ok1xJyFy zccOI11ZRqwt2FZ-HMJ03n@tqRsv?8}xA2YcG*?(E*z#0mrj?HNDwKO7@icZ`!{!l* zpLQ=^qpJt?^v=H&!T*%XX*2MGhU=VTh(Z$&6qm1SjUfi-da&?Knx!`iuQ;ts7ix#Y zom4DH+J<&nq~!5&P;9b`G3pGb_XOO)aWQkJIX&eT7W?cXNHY8Yinyfza8+>lGk)eb z#dX?75COkbqrtjT^m3yVwV`@qfkuA8hR-t^fS3=MdCnp~@lx^UpR2M5wc<${0jgJD zv`@s?hVZyLBL!$Tou%}+Qy13j_QxtiYroDdC_z(?9UPRQD&LL_o|4POKbFS~l!%*m z_vL7rU@wzZ(_zgM$dAzaIW8Bt3oKMO)nv>;EB8NnXtQS=ml4;Ig;>wOA4N_Lz*XH@ zd{5=1lbY`!dOlx?5y_q+xL#T_5V`P8=IA{>HC9}~`=UT7i1#j0XT%Bd`FYCX3z(|w z&VKFLA3BalcK66ECyXB4oHat2iYs8ZOz|)(8ZR4gz0F~Wewe!G1*vn>MYm zbG$3>A;QXP<{hNInO4c)o!BBPWczE>kd+&z=r2`Q#4R#bw@BeO!sKv~q2bd*^ZxgT z0>iW5ib?;%A|W(3gmMQw4Ah2+Y1wnNtU>Z>meYfuo`?^;J$QHbrZGP!-;;Nv9J=4$ zr$zVVYoTsD-V-?5D*hTENDKM=lxJ4ON}UHGN;FIjDe4arigRpDjil_`}nAx9^jgy;$Z z*xX7I$p7xf;*_8l>p#ymlQ5eP+&^m

-t9=o=#R&id3lro~s!$j(8Z_%@&eQD7In z(&4&LH*$%}SaK%Uv@VeR*8B_wP%CE})B$^s)o*dTv>q2e3pwHw9KzN&r*Hi%r-%vhKn}vZl<1r?QZ2< zZx!3Z+-bwB%&A#VI^N)l;BT&lq|vhH+SgUFeC14>E&qPl7FHnr3ARfMOHbBFlYy5X zpO6|F(~#Y-=gr&+C$%TSYH%cPYyRVx(t@!G#*O$+fuQ*C{qK}3TNcDLWt$a{xT}+i*QhJAwb?thf*W!Lex0l zYD7J3Za0?Hz7zYQ+J@m;rHSEb4&v2`R~j0x(P)hDgCJbtWCT4o@7z~2Ge7h0XOEGz z?BG~4+PZyi*h-BouIabq@EfXvMV;S{@oPXlm_d*!;q7)90%`2DKT;T9u&p*Yf`IXW zp1HEqT)$-(7gASv=e@7ngEBQIX(_~b3m=_8t`IZpPVw>oH0 zDAVWCMt5Cez>0YY&fKmByN)x>gPN6$(b_4}?ovaNLA`Q&d_|=09`CA*ApA=-$&Qr`Q!C=nv=>c;In#-CWDWw z4bH(d)Vb`2a73#_5wKx9;?;?u#mK8WyM%=J2W^}OFNvs6aVt<+D*^jQFOC{h0EE+LUwLWimy+) z44~-G3qyASpFQ`qZQ+?jE618r?4e2v{~Ss7x=*<+g6lENc>YAGd^T)3U&tDzx@L?PI^S6dNarj;#!lA2;~!j;(0!>IXlU0#2%9hZxR~|Z@&GgS% zf?gj>jB2lRnnw*7z^B%3$AN(s|G~Oyh|Vs0Cz0UC7x#?ZpU=D1UWwbqRimW34-Bpj z!pnwA&(%fWf-M+B{UAK08Z?qZpe#mFfxb2ixC;$Fozbi6UiCQt6L zjmeLG^wl=7!=KZrYsJ59DQWviQ4ke{R4~X>;0oecQngx>D_YpeQz&To5(9 zhK`j?N1VF0TYGBF8B7Hs&!<9QIyp_}uVD~KDcwP(g_E}OhJvG3()*XZqzU!7SCO@7 zyNFVp^Gdk%zxLTR%=>FDvbhgAdsL&T>oWHNo6wCL5r2>g#_4#(H6jo@nD{rt|Adpv^8$Jn}h)vEyR zT<;C<)GS6sZDy2f@ez%ge#FgdYGnyb`qMc{EFYhC&b)hhft=~J>-`>S;@gM5|G^wZNI$wHSaZ2>tnzUY*U}a6E zF225@j&;d%PVX*8Eb!V|vA`hLzVIry^oU{u%MVNpU$L%TlEHW=RN9JF4|)IYk@}~L zGIIq|0*2{!m{*(Wkaxh^D_i;!6qygC`9R)_$;}z*g;$-}84xRaBmqxdjs<3;$9`d( z|6!+Pxw&1Gll8y;%%xdVq=VO-G3kwmYFK0-qFA24*WR?Ewv*2Pv~^~-O=b=Y!S#sQ z(ckolTHX$RsXF+W)#kg@79vEFVNc|L1E&9UwC-=#``Np3eUGg8>PPVWtG@hBXYmvF zOL{B(G{C}+$iv<#2ov4y%I~^qHSu!aB7lK0t@B*U3nG<5Sz47F zmrz;GELb>($e?m+C?TbLmTkKHqt)Z#SEDibrImAH*RGW@pr?XMA2^aw&gw`(Gsduc zK#Y*&AXpYpR=4JJ_1DAoGrN^tr5F=#_ogmm08~zkh`|+XD0&UEyXw^@z{(NX7(`2 z;05^BLEP6P> zNK59WYMh2Wh*NDHIUUgF9H>hts3AON!b$r81C@q#Ob0m$)8N-`;9P3#5arD#T~^+X z`Ta4wSbO~pL2L`(m91nmyQpiy>M0sPN}3Cr2dR`$cOrBemS-{{x1Z47=Q@%IO~ll~ z-VdLfn-$;(6}Szw(==>%db2h z{rh=5jUOfzc(Bfnf1=D(g5Kwmzv?|>k%m_o>(0SQN_GNaCrjtVm=_uVe#C0Xm;gpF z>8!7DWp4-c2lQ|t?29-TZosz}L%`9Md#UsV3DN4y8l;7KZu%zX4eyC}Z;dVmLz8GK zu3Exwj~`?Ro`p)ihDFc@-lUk|JV>;?DlGEnH*>TnPSVRezdpZrRf2qW`}zk~g|YB~ z7iF*$s8d`Nj?VNjtY0!W;iK@h3X^M!aHWVvSi{}S zbjFYIP~=MH_f3^&?x`gq_mfKKdKnV4j)9%N0ILnSvk@m2GUp~`6Mu7W^)6<@-v??O zlSYc(T7nz(Ni3vNg=_Mi+x=@>^+5`G<+x`CfUG*Fi0NJz!mZ=htuF>`y|})$_Vgps z-y8MChF_!S^p%H_6;j!^SklnL?@!6NI<<hNkP1L=R#_FIss{ItH&%}WIn#MXcICZ)6Ulc z8PM%`<33Bkfeg5g+|TNB+wyPU<+zJ3Q{AzhTx+{Pr3&JztmClta0+6}uiTn~mKxf< z!NfAZ#<`GQd@X2-u;->+HG(v(GdP`%!*heMI?qBMgl|DoyZg#I8VcutOH#p>-q7rW zl|ORk7b-w>`ryxP)9@74_Hzl-1+MV;U08oOI)Sn;L{v8v%)?)ESD~&?`308Wf~T&L zN=e5uP&YqO1Q4!64SVxjA#LRQv)J!d`rF0686H-;i#0{e8@3(lx1F!R3r|J_)ICdP z=i)NnjMH?6|BOZwK@gwG zZt^fU-I;qU6gZlFc6T`cuqfd{Od$R(JvqlX7)0ul?E_zvELb?c3_OTm5_RW34x4R( zoEFA!rHXG{cTqaJ@QX-qkQhBS-tH@qNusilPcyHZ8fsyT&R}qd9Db1SIbRd!^c6va z0g=c>O=>OydQDI1tQ_&0l$*hJ{7M(AfIEy}n^ho)N(m2i;qI0CA2kdN3=`Zua z@mEb;D?3Lj0PEgb%fDOc$!Dy<@;{h?O<7*8@lc+aRsT!hJ9PqayYPOjhwpnP6~+TfadaW7bG7a_EnoNxTwhxmV8 zpZepjO}$FfWZQ8e;>=O~U7Yyy;?Mtex2^F~zL0?wt|D_K)kHm);uq zxZCG_X)T1uO8zuer~Yy01)A8x`Ba*Y8e&(ggpkL5SEvf?{=v%rFs-sLV~n0FP-ajN z=A78yjZ57E9!Bb*B-UF$h%X~>XJp84+y~um4sfBLeU1>Gv(C7d18Mut&q2m*#W^F- z;CE0M*PioVt+G4k3WnGSvU@`fO~r8w?5UFEPSdS=1)C_E(cD@w%0BCqN?oc(T4b;v zpj&f7C$<**&RrH+*nqGTpEiC@>kY5HyxY=P5I7S|p|HByqrd#bE8RuX%!Ed8-R9N9 z7o_>3e_xjBs)YyL@Qmva72&zj=AubZA0|C_Qq~y=Q);(XY=hZzdmo=E4!j}ULwUh{ z=pE+t2?A5&6i0YP(54qvs`dTk8oT*wUQo7@kOa$tKy&<3|#C_4)E^X+ntI zx5uPl@fq zheKN;OYfu2D^Z|3GJ3uEj6dmkXnx3xMeA-BvT%>)o=|5z+_Nx3F$e%R@NO4mm{C)O zyHZFBiD#M}d$I>YH&%YcnIR;V#%9AU8sQOFJZ@E5d9=|B*!c9s>yno{D72yB2Jj){(hQz}EY$F)GY%n{?GJd+de!6Bevb@q+#MS^Fb&lGuHt4j zJ#(T<>=e+mtTb#CgZp~0zmGn=(oHbssc|9B*SQbJelA@apkv-N1=k)wGl3T(p+(c9(Q&p+IVXL-i?*1HPA}tvA0Gwgk6SqJVoo zEeDdks{t3@Qe|(CY5|`9t`>peLEWTbb=x?YQ{m_+!!QnT$}GxO_aiwK$M@|`o&=We zXF6mw!Cr|XIz=BpyM9mW4GuX@hf!l|1$O{8!O(%4PjfOT6*Qs3O~d{lT>L)5BQr%j z?rx>Fx@YPuKhCghb>jCyDn*f09E#;ADB+v$tu!|D9Jei2g^_3R#oKGHmh>*A4k1$u zqYz??eLSe?|4}yH6%N!uj^>5aw|>cyKwCcXb9Za7y&526x&~=c(l5D#(+K|*|G_kp z{$PGtCJP3oRDs5l9RT3l*b95~eA>ozciT!HSTJEgOhc&XO>Wy##*@}vrb0YItVH7!eYL7xkx{FTBQ`ze2U=Xp*nUZa`!!Oh5@Ehk)ir$l-*vcBC(w{ zeo2=fT*3Ya0ve>yE9rX?N!FvV4?qt_+5# zP*2cD7ACcIZ>fs*RS2i7ljzY9<|xRI%6=|>LK?x5`|WB-D~i&37bEdCvV&0t za9gq1HyDA=ktWWX^v&LifS2kqdyy4UrI_ed!}aY0e=HFK6)jI@t|&8T)q zC}r-{6Bje<-0#NAKm2;8vYZpDQ8Di-hSk%gek ze_xn}5K&mbamsGklk@jvW!Hrv-e1aDACCr~UdXIO`Z)nu_lY}$==%*XP2(vf_r7vg zAV2&ZUL8;WbKwC{ua2AMmG~z1JIFkDoxdwLYr=$aD)@)7MtCRc^> zzyl^q*u)UZr95Gy(|;LDRac^m>Nn9Ssays3y}xWf|La5#HBMvT3-2e|!OsV8X&kdD z#V?u#2HGvrbAKG~oR5dsS1O7p^?8CQLYHg@x2q;`d`#iZ1cbY?7$dP_#8{T&p8EB! zzJ((7K+a(@!u~_;uk;_0mqF|a5#aq<@>3uf#VPt?cK?5$7ZQ0JEfm8(+KzOYTXUa$ z5VF64IxO6~<>$^*d4;Ao7D=Lbj_V_)oWuF`=IQ$(^-jKm0m{pqG`K(%Wj*U(eD^9e z{AUAhx6>T)`L7tf+=vnlkUQ|tAn*&`|273BpUUr`e99LWT@FRw;t6SljGcALZkwO6 zk7@+fjQa6%-z7At9opi!$S}1#CQw6zwv=4%V2hwcYJk{`S4;laM=7s`V-MCWaXSlG znLTw2_IrClm>{(6>3~_5rHgbraq6!}wHW<5Ii{CID@Swc-$6ppQ6zdzPo(Z({ z3N-wmV_Xh!r#NH~0ANHI>bq*`&fMsZ#fl#WMs7+^WfJ&Mtg4GTbdZsp``KkpC)L8( zuii!R5OQ;qu&laE;~v)x6H^ugQjzd4)gqvgHzdYpW)Xsycl_wbKJOQKP`(Z?&g;gv zZA%9?>Z^I(@yEsohtJryXHL+0?@uaWvCG?)W2^*?d0kPD{xSuS-@eiqKk?n9SZpD? z3HJ)X1u`Y{Z4y~c6Z{x9JhuT{Qtzm;FcL4^rGFlkzxb9j{K*w1MgW6FS+;4iG}!bJbj| zV@W+h*y(E#$!{DfO|0}5+5e)6gg&X2?q!n+F2~31(*ybl77~Cpab+7FF0f%NF69-r z96puI{yGIR3nJm?^V3lmV5aqH}KI&_!q(_+l2;)Tk;~EKTmT= zsp@eFH>14=I=6;wsaeY0IC5%ka~^xZg_qdSes#q6(RNuk1$0*#NH`6|se!}q%aGL6 zD*o%$Q#gm_LEe2=A+GG4-71{3ax5MjByfe6dbpGIcY*(TjF8Q=C6Sf0KqI(GKLR{Q z%}}J1K;VUq7###jO2!x8#dVbCT@-#5^X{ITD1LIL{fOKOXNYp- zudsl`WrsM*ARLWkEiBQNa~HCuvRKobD#jl@q2vDa6a32jXYeQ(iUSwOWv9NWWB!2H z9s5{i@=kRH3Y@habqQ0-b~$^aI0xBJB<8%#{1Wsc&#Rw?-ze3t7pwBD7n9lubdu#g?N@NFQTT_9wz{@KWD52|UvH^aX;u13Cd z%`5fl_=t`k&wo~mDJw31eqOqQwpNJ>%XtJP&h<9OS<-tI+nc{51pWNlmywDuN;!m) z7Znd8-r^=KdXnBh=KSPKLuJi%JZBHw@GjI)y|p^2sfDaL<4xYSPJcxV+1bagX`pBc z!YQ+uu7QTkIUSuRlZN2+ib?$nercf|J($A>kps(C3-k)1Hr73y__y`upReNoP8j})xdRyYj=6iadN>id61I)7 z3hbiS^MNf4RCw6GHDUjw4C#WZFz==J*(q9y=T=(dyf0snPLHj7an*W^QPy%A}&_3RS(+Q!8HV zOelhVGI`70Pj>8=AbmFfe)uUJC#-`ZLREB?A9D0z)Fo*caKaX5EX3PHw?-3oG@xsY z18#xHgksa^hkO+O(gZ%6F%{LI8gNY%!ZCz?;e_P-OX3{iP>Gh-)Mu5-N2~+y#)O2j zrk;c%D}V9v1O@YMv+gfjW=Tg3*`}yX>(EvoKU#zCdiFIs*3GHN3bH7pHi&n`-5?N| z5whW0nFr5kKp`U(D=#gyo(Bn;vzpVlI`!v9zB6k;+9Q1{0gsg?N=LK^5PZcrcM@Lq zsNZC=Wp$JBYV-Jaa!o5X6BYg?iI(xt74M(R#h)Oz8QxMa5iUK>)8VfkEeI0(zE4xL zMdR(xq2RYq+^m`#b`Zr4^dY;w+(USS$6Gq#PmhxE)M)*7V*`ojBzvcVZx=ftB{D$3 zA>lHAp^ZPy<|q)fXD^R+x_p6jMACE7T`WRi-OY}LgW(i7ciAsyh8@BW7!XlbVesC; zgpLBudtMCdc$yse0XCW6Y_9p^l*xCeyYC_1B^s17ti-wIc~gCX?RSofrWmWwT(EUM zOx!(oUhdzIHnf@c|H|ENY1aW*62%Qa2OWO7wG94*a<1`fv0SLIDI3ihVLDy&2MW^KGi=d9b~}D@pS*Upqu-An+!6GeYHsVvIi~}+Ork*jHnU1m z)YbI`h|!3!Q1>y)_idQJ5Ts{ti=RPy&Z8$;L!Zh})^D9gT8OG4y9&;qYB^r1ofdjc zGpVau^PO=E15Pv5Ia*k0EIPx)nc=zKo#N6zVo4rX6USGJ!PQKHA51$NuJHlXIk zNGZdKAw2X)hOOJS`M0S&BaWQdUk=FKafz!NEWV63RfSRdPCe=6>B<@h5AWSEMGPlA zP*iAddBe&{m}49_gF!@^L%J?HNtmh&lrBbPI224?-qHug>ZqT+A{Y^RY?F|MY*KOH zC8tX%Ojn^5EY+7iZOzRRlYoab!bT5}!NDB=|K_!~;SDA~MdJUP{qWDO3EB?n~QEcHoaq`Av{UhK|i$F8Jvm z_(`a2nOhE7c_=4qn76GwJCR!WEx3FL@IIW^Bf9se*-7)>NM|0i-10|8^3B2WGgb>< z_?|}P@n6zcrD6ps?Tz)(cM*w3LuRrCMi&46T3qIe{4%iF?iswmme^i#U5+R=novZ) zr0OJitvY>muWG`J&Ks>nzsQAJk!{I%NbitwNJh^=~YyD$EwKu zM~9|F{=isy#CG^t%N-fxVQLk6>iFTXDP!SXS6eDCBx8a1 z09ohj}P-~S{3CIqzNP>h! zpmb;iqQyrXmPAodN7fQS*+PVnvV$l@6P6G{kS$<_OnV;rAxc7U% zd*5@;ch2`pq9&_Cv+ACQ;V51&WQQM_4wk{hZ{NJ%AAkqj$4gN+L|kg>sGX8?|4Wvs zr;aVig}K#yI{I1fkqX$EZF&t=+E$+h#_3Z@kYJLel+eN|s#Gc}z4sAto^=LaZFaYO zAg`W$-7%vZ%yy|E?FF%II0^kSM7n?=cn9yprxSYC^9C?vCXh+{ZA1D3OI6qC6Q=){Jua0e6 z%b-*5__I%2o;if>4n(gW&=-WJ~Ox`ct zm3G7z*(pFPUosCOI|;4&oCHrX_c&SQljS=-*SxN1djtJnCt((mo&!Gqd7eD?m>9!1 zK&=FC3w(0j8jK81uKGG4(5#DU;4heqqL|9^{51a>k_3O=T3Z84OMuMW-a6H|LW?t_ zx9Xw^j={!)D$r+27-f0ofUU=~mXpBKitL94PGWT=mxbU-x5}RH6pEN zO*g-xr~u6wx81{wg?nI1TM|T06|hA9Id?{uAb=~no8zac*7>AYoFg+@_J|~P7ADUL zB(S^&rnl6*|MEsbKRAUX5c92zy7&li?*sPY(plD5&Z@HrfFH(Lue|-bs`ZHF)8IUAaJaD*$y2e`iBp0|g zk+t^{3*{;aZbBZ`u_ktxF~Sl1sf(s-s~?x9#rSAgyIYD@U9Tj;U#23xSO-59b~iyh zS@pI7(i~@szA6uzp>*%meP671w#Y8}h9JFy%!Ai*yxBH=4M2d=F3$Dzlsf`> zQoZM}qP$rDSAJ%>rUPq84~Fz$pTdq78`3%Dor~vJz<|5&#K9-Pds;ms=w00=W;gIA zVzjwbo6@7pg{H)6yC zz&}LF$(_AkX=h$WlHJsuT?uco%QA#Rc;II$n+yVQRwg>+Om&K^ZyoXpXPvy6k6KNI zy7Vqsrj3}^?z)S#rmxB$60<^x{raIvX8nwIuTGg>C8$?|h!aqr8gpC?rM6G&V3FKs z=wBbAd;k1 z+hw%C4-laQmyPpwaS|-q>c=>@+eZzX=OF{ya2q%am0`KIC*rhgP5LKN8s}!n42XL; zM9ZJvWuGelb7ASS^ft7k)BMgh;jpC5^AR*)E+Jyw`8l}aelHsgF!1RPE1Tdt)Ep?zk zJZ}VyJeZl3_WYW8ec-_s$(W0BXk|UKSs9aE{^;Dg`Fto*;G~re(O{X)y4{{(G+tIo z&J@aR4?Z3+Yz-QhmOxH%AoKFO(4S?LHK|yRrwEbx$BTa_uNpSr0I?T*%F4sjoXqAm zRkLKm!bpccd)0=Ik{9m&ujN%h;XM!pfB}tal=(eWlsYTZ`k&gYHQAt;CtjWi{6klqB05F=ovx1b;pFd{)Dh(x3p zkuJedf)J1{z4y?*An)h5-gVbq>)v(mkF(C1nRDhm^UR*T_snzCYX%&w0;~W4faA(# zz3Tt~odoR}&cbw1BcyFiy8->L8|VV?M4>5Kfx!iA37-{7rK9?>10f3`* z2N$r>8|MfB95cD12Zr5)&!tn)qY_;D>e9?_?g>SoEk&J*l3~C7B0!Oz3b6v~3t36@pA1Iy({Q;=I*r8fBL89$=uR_T7p+O0Cljawh5 z7S^s0m@jVI;Dh`lDSrEwUNdw%tx2IJo#-Aj{qyoG!Id`O{Puqq@L^*>NZpNU>EcAW z+sQH>gUa7Ke{T2bQ+NElZy(j>r0%eKM3)r?TK_CsAJkTP&7ayiBacIs&8zxoy4`HTw$+u+;u-KcUGK*IF&y4SEA#``U%KNEaZF3%=e#+ ze#LoGX!-nWm&^jqy!;Fx_Ba?5Yv;CVwNOqOxf{5v+-OpV6SdEM$z)XH`V)j5+5e$@ zA3^+ut+{v1XTSW~GDQWteIsZ4`S-e`7m2xzbEDMzzQeKix^O#XBCA2{?qwy>8&($E z15<_y@y(`$o0DzKLElu~C~7{i_}tU6wmz-%y~CwSx;RwGo$^y`y97VmR@-AyE!dS{ zu`;1uHBVCOUa^_5!*+B%G0h9^indw2oIXYs-^aF}xP&^{Kt;hYVfm!W47!d5yjD4A zyKPO4i+Ox&{k}W8+1Um5)$xwL=kP3}WONSVX>`PX`Mo5xA^|3Tmsp&Jmbu&KM2dK_ z+$zE&=ydoS4_Dq_L>{JKU{T9crakD#O4>Tbr`ia`I~LgS}% zU*$Y3mzLalUn+f~Dy^k(ID;um{FNo#y6P?Ub$xhkuFEN(JhIwGR4FqVxEok}K?fPG z@}k%YXJjpyUp7H+L^hc8XQsm31QaPCw<|44Cq^pQSUt&+;kUgzTuOhYI=wUH=Q$QE zP5hy9!3oawg7?k&rjUKz5dBLFO^;5tyuuj8n5z&yFbbPIc^QkJEka95XB8h;DN37W z5DGY?W(gxvF`d82?Z^9N+&yjZa;M#Xq|Qth9TToBq4@A|mKG%ejpm=-y(&9>qami3 zQ`cocK499K4W5}(IX|#89C&PiA$p$5jO@!aF&Vh}A^;ja?nKq8j3HDD$qc-``Ifa5 z+I+3d{)E-hRK`z~^NKZ^;CWVb237-u>KdcFsERRDo66+C7Ixfk9-JynxGVl_$dj<( z#w}@$pVUg`74_~86S|}nwZHv-Li05?d3;gw{bNbsHp1c+J#u_fm1i8zz2T#{oNY)-`xyG*{Xd6LVYG)y4yv$#deN+d@?sG{H-X81-tDDmQ`SdLx zk~Gncnvo=t%L%{YMrR%b%u=;*?V|89da%H)BUVSTF_waami1Y+a6{F+EtADK=#(4w zW;MaMZm<{8@B4kiFdNlQizlD-gBL%Wo7L8``0Ku=;6Xr{5HWQNa3PWRLkG8`C#o9b zzB30(R)~U0oNftG57m!S z`zPIC4Xyf7UVhhH3mmbZ$uqV>V-w{+`7Sv43rgLHZo_#z1~rH46dMb^|Eg@eW|6-; zu90;-e)eMS^^^O=k34B-)?at*jl?lIPe#pkC#!p5OBM|@>|u{~W}06Y>dn68eUzEI zQ}Q}~w>Z|_SeB49XP(2u9Dv00Jq@s4U(-STWx3)W7Q^=o1{SpVSG|+3;ZoggE%PSX3&+4hNeVsWb~?r zb_c)c@#lJH;xEge%fQvh>OBOUJru86T8TxEpRq5z7mQ`4bpaOnBL4mKx1$0#WhG?U z>5f4{*KGs8VjhF3Bw#|EIwZv>Gjn} znV6}YOi>@kWESOa-BEowhIm(AY__poi#W&Sg}umAMSU4~jE{r7A{)%UW0vnd_`Hc@Y!P-I6zDxa^;A=Hl}hb^0J{P-qv90IyRZ zkJk!?gA$nWUNe0C*7aI%+WfnTgA(#rO?aPmyl5U>R(Kp)mMwF3!8QL&%Hi+Fu@L=l zS3($4Q<5*wX~h1jGyfncnTl+-1uoFj+=(qLNx7POf&N3kjI>~*J6v*hWo3GzyqTUf z?=U`O3)PGn`%GYUIWRH@ucBTS3zzH>?JXmM2)jek6A`aAIm??zEu&g`t~1&9HsTW( zjFv(hZl5jo%IK6xJ8~Gj9CjqSzMk=Bi1sTc{Mr3J>%9vIb-)IsUO#_OnQ|@Ttw*Er zjk0uxigK2(0rO|R^dwflLW&3=(^mIMwYv9i=lPsFfE7glLKJJw#D8O@w&EK^;x8XS zH5Ig^$N1zgItG5={|cJ_v~U$(u(B0Ll^75Dod1!pD}^p``_`*UIN~uKw}n!?pQ<8; z_z@y_|4fy91;rm-ckr6&dGC@rT^@r02R`sQG(K{x9ohSK_in2k_oUKHj$$27Uc_(7 z8?D-GO;@t)@1^d->)6QXe7w zYWxgjv4D(!d}T%^cR}7kL88g@x#M-Nc3cl>t21J0x-lhU?O3UaFBc9;6m8S~$WasF z1oObeUyU5>z-@0x6g-x0MfMu!dym&y9)4tK4;v$PbDZ$f`yo9Ro5~f%s!=lv$;Og6 z5MqRqon;;qDPba0IN)Hxci}Afc4h~vc3CpawHGCmp{`ZbDL2AHbFgXX;do1-@xZ^} zOQWAJCV+LAj+fTVocq{5N&&DWLwiv#l_oDp7AsC`i{>pn+o`e-aoh5N^D`WYA2+$rBDUp1T+B$`1^ zUYU5`_+3ZH!1{yU3faAisR~ei>23%JBy+)A!wcFFTlLqrK{LIt=k_hto_62eW{}@p z!ycYCq2S+CVaFwl@9eKSEl6V2dOr2^Pu|O1xh|j8V%F=THb}+?yc7D z{5Z?^i{;%M-;2+$@e{@~*E1+BD~-NtlrcOTuPrLAQ=` zJVNfiv(ifYy&r-?*C9?DDs=<;OHD|7# zGuFz@y`?U6RwfVqV!r5l*U5}J0ZDiVhMkoxZrb{^GmIu8vJx#>?YJiE8s56SrJV9a z3t=Z}mz+c!SW~06sem&*GCZfqW;D&pV1}Q$H5`gyBik;e?uC((%>7%}8BB10QZNq2 zo(V6eG)Q>>&Ds0-c~n|P!}V=YK!jv94bh4=jiVb1hF+K;vDt@zg5di=RkwO<@gc^t1kA9@m;Acx2P;TkY#PQ{Yd!!x63s@N7=leR%kL&l(l6d^Q?P zQ+XPLDmaWhXJx;{z#>ccf7sh1NH~7&CrmzEf6)C8Bk)hQ6@`cq{fmgkg>81n&XFVh zZ*Hne7h^t{L`T?Jz2s&V7z-_#UN9qmzCL+=y7M;1T`~Pe3-M{u-`F7HZYOs=Q`sI4 zlT81%&|~t|DTSNHR*;j=mgRquX9uRDdWLg|iEC3H1HFW8JDX}659cr*PTd!rN+YIi z^L!9jFg9z&-WE25cwcL;%~+L{mjrW=6Su(XNC{tKJ4{(a9wq=9r*m7_GWi07B7`xc zQ9yFm9r@a5f74nxdz;`Y`O*`iSt@D*oZ0d*I~1)cLG)Wv4X$QIL(Ae>L%))jU1w4* z=)VHa6KkIm8+Es`5Vih*SA^DaYV2TsXjV4pbmJw)Lm^dzh%tO%AScfWqagkXf+vM% zeQ$jp=pF!DV0?jGw6OO>8#1duOfs>ND8*I;@!0P0h8<=eW#Ch!(;`g`{)Sh!J51L{ zY{4o03dz$C`;l>>Y`~c80Lu(hw21r@Acen-6HM! z$LDqu7b)$nTh}BX-pXis8}BSUG}MAGoZXB9$vk;G?*e)XNg2-7OV9m~ z4p7Oe@t?a+tNSn^Vq==-4j#XpVBv4}5gKcFWt%gTah#&26M*NTtr6 zm+Js_NAg=o6gkn#U}!W+>}!lia$G=14IAN8XmF%Wd3Ew&Eq+{9K1K-R;ki3RJ+b)v zGlN>NZ=h{UZyBuUS9a;paq(l>?>Db}0|RfYR4!y6z!GJmQoMjmfyX-eK5H(oucb)x zK_u=GDDz$aRg>g*ICbzY1tn*?a(g48ZGbXvWDj@Hs`RPG`bp&TwCT6>Owd5lkGzdNM? zMhd5p&)R+@Ddj&E=^|(j)o`6Ssj!%kaYYZQ6?8J&1Gpd~pEC{`1}!oqD`V9}G3(0F zI2Zkqf@Q#wOQznlQ0P{Y=WwIJrz?mNg+BDD!$EcW{)<3w^QWY`yF>=Up!uFbzP%Sq z+s@j)PCa~qKfoKLowc}Ej8aE@G1H#XaSUq00fSCy0}U_KcK9H!0L)_qU_nmD0(3_k z`@Aebtp0NK!^VZj2YW7pJkS;2sFRdj_0}_onrX_Ue7`YSuzzKf-!m&&$jz6%5zn=? zWU^W3*Xbi>(33NH-p{j_9N$Ph-j@{lx(IOW8GyZtqYGGn25{ne9P%eJpkp-oUP<7XHhC&K zuor!wbBe|;_cgmaLw#AwyRMI#fCB%#jirDQ&tii|s%?MnHX(i;WpqJLRao~hpd&{` zc$nxt_==!jolwlfPJd^|qL+Ng*9ZXaYr?wGeUE3CQv%_h{gyXzbP~_d9#f!|xvw$H zc|c@uB!T4g2LB}83TU*HWuYksH76G8g1P}mQG0T<9E~@>(k07P-hugfeB-{kr`BV> zKj%i95S@8(!mY zH-Q|Tlnd6OLSeX>)cuba&p=_2fxs-cg@fTbR?)|h68l0-}6@VM3jDkdi~*2 z>~IfeKAXC7U_Fj4!+O`rWJ;c(&pQ+M3 z&dy$vTQ;Z?^s6kfC4)GofMWTn$kU62biIo$nWwQqqJATQT>(a&wlD4T@S$<@YTwLmCd#jn6T6ZjnOPs2G7PSh5>`7OC z{ZVCRD|sHVM)cZm&eW2Oi<$pG@k-FP81}QJH0rSD7He;G_U;)(ceqg759z(r?qufe zBh}nCqwtl6X)pXpPe@&@M;kL9^CXgSeWXGlhC#>ai*mVf;MKD-2g5S%7pLy}c8Fmc zQr00g7`l0ay(wA37Pj$p{B+k8m1%y6;6&XTH^6avN2PYxFm=-*!0}b(f=g{hgzo;0 zm3m8z$ALEm-d$CPCl72-%B)bSC%3a{%Z>C^6>j#Aj)XBDtb(>7jZPg=`fk$QZZB#< z2iVO~^!gh{#wu(>EUee9CUf-U0G)-uA*-(kyC|$)ZYXOD-nd;~jd+r>75vzrQfO(! zQpA`QJl>iDw3M^CZ@UvhFzc=LUO#?h_J=bY_}DG*(}b#74LsKTn>Ko1B~LszFZYM( z{P>|WJ>!Qr2gZv_DKXdtUf~|KJSzAf0WDX6G>pf7A~{)mCd}glmN?z(esxB?&)#UD z88ZCQL}`23&I72xy`@z?9~8{Nwkd7njxR_b#lsb3F^s-vR#7%FQpZzADyV> z>alo6I}-Q9a0a<#)E#kds&ZNLS07Z$3F;0xaM9f7oEe?Y>4sHHORD>YLn%X)PArWB z-DvMm9kcE(4Yl9Y2K%*BbEk;C_tz4va=G|RSC+bQht$vWdjykUiuk}`G>nIulkZ1n z@rHvfX`Tup&7&fx)@~So$7K$d$ z?%bUu9QKAs4=+{|!k4Nug%PSIKuduq47%%zd=!4nY=YxI2H5OGDs`nnt3hf$;01MC z!rs>$eRG~1WtKnNmTQVI`L4yk54j7Y3 zS&lQ#zj0R$iz_i){nj9JS#T^T+Z@nhOnP>}WDt8X<13|bA@*Gb36I5b`@%g=stA1* z)pkskES|oj&(R_*CVEt8$-Nsqd51?^Ny+7R=U_;(5@n&=j1JgdN zSlrVc%rdu`8qW@O!wEo!6IT^Tb zx$$KyoMK@M9b^m{K?*C7mV-Ic5l`O&c{`l#C`6<+#I6i=ZmNyj{`!#KM6G|!omcOi zDtsjN7La`|eafaz+0Eqn?7HX^>di1!6`yJtB5c}OrM%uw!Yh7gdp3$jn967a$7Qj& z#B1g963nWZ9ab&OhB)kgM@}C+e_saoFhZit*!KF_bIp>dt_780AQyb6s#?@D9eUrK zrn>$RLg_iCHJ02Nu_u-3XUozl$zZFyKL{+hMhH(PRP^|LxOcDqZ)52vk%aG``RLM1 z`bMRO>Y}p}9P1Ll26QQAp38`B(UKw=>c2b)RG^7abz7^BJ>$-+skMg@>2mQL2>R!G z9a4(~{Ic3!4JUT7$reIUWtAb6T`PTtCz6d-b~-;{OZJ}lpU?-~cnV-t3|wMX|5!16 z-u%;E>8n(hCi-ZxI%`?CKCGChSWr&xrPK8`y@tx{c+*f1dG&!#NvJw1Pkc@i4DM#Vq@>Jgp}Nfvd)8fQ+CAQXw^_uCyuB`#eZh4? zsE5$!vo5G(773nd!ahL30@*b3mI1yD_{2zS6cui!$>P&iIl!qOq9H zw2KE^?7`uwbh4ToVRZEks?F1$HYFz}lVP5Y$uVo31S7U>G_i)BqXZ7^d)e%$8k=%K zFp?+v-Usn#_!QcdfSZihFUfbL20r_BW(?oME99MaA~C2aZaIgx?7FSk5`z9STbx9cJovo64vgZ(|Gj#36`$(+=PhO#2uS@tWG-)w ztjBw*z>Nx=FMW^SLGrW!Z!-P0L?1p!PFvqGzkEBWAnA|f;su?w~@(Z7uk?`$~tc6^*a?(ulX{3!q-WLS0Ly4F&gWN6l4i!zK) zS8;UlSMb@q!$wX!uDnr+{N%eqCFFiLh&N`tz4JkX?QIR%dWkToY?FVUv~`M-YY@Ij zLS|LSzPAf4)3=?nA{OOS!$#?{lV*B17K;vfU~NeRt2{3L(l}D`JWJFq4n(a?hm9(&HG$f{wva*g*Mc<9u2t_IU`eR0tN+a^Fqw$lFtem1nVYs@n zp~1s^Yo2u$XmW{lqx<(8J84Uj^!P#U`gBj4aXa9uB|{U}^V3Ms;V*sO^muHS zDj8-{wYI9s589E+-vnw zN?5{{QKCP3IwK%2^6F>{>~uK7roc|7-^aXUr2Ku>tqNfMB(`XUwB*C*39y~_O|@29 zhvQ0RnB@wB!w}p;^VA0u;`N?Z2s+!QaE3T?a*-WIze{LEoR4{`KIw{S=qtE6V=e(* zr6j0oHlTjJOHp&Lw*U){>x10oP;NKHpA1JZYr&$|-R+Bu@BNI}xft4_`(m2xRN4xK zCV?f^@5`spvxQshCLz~q2>)*$$4m*~)P}v`IX1ff`ngHTo;JvnF^K14X3PyDescYs z;~Ukc?{-+54cx|H2~~4WXv_x|TJ8K4TNMrX8 zQUa}Y8Kp9vV_j8qGihH3*U0<#m6dQXjB%$^DtnCI*5HB3EYtK$kGA~gb0X^;JJwM1 z1!QIZD75y&NPTZYv0Wa=v3xU{8Z1tlS-h>Fpin)0`=TtY7R?e6p?W2E^`13`du`AQ zgv5Gzy8f}F@dCaktG@nZVbRcPEp;P)Gy)3GQz2_N zPvR_z)Gtj!F}bOu=QdV9t@jLC&QF?KaNGJZHlH?Y;OE)l)6U>>zPUlTcPS~<^k-f( zNBuD6dsW^pmW}6*P>R3j-Fg!67x{~oQ4dwuGXM@ITMrCsV%xqU6jdViAyLrlDSLw~ zSoX=9$5yzxL^PYNIhmY(1cL&bRR*-&URCwPwg?1_w({Ixf%tfjT#B?`VkV?;n53@g zqwYV)7dnv0epSH$saMhShnB=1_c9zo*-XMvBKpW)t_EV;mHD7pAG_fKI-Nj>@0#W! zj+@}?%LjrNeIIBEX2JFG-fbD%hmo%wuD+5F%YBd9w`+cFORCF-UM=`B*v1pq&%v2< zw!<%i(Tb&&NFJl_)k|Mo6iNUKWh$MRrO<6HCf|=BS{XBQ=~H^uzmqOVqcmcO>r>sf z*$(Y~-LpcR)p{h}Xn9rii#2O#T1ttf#C3^_+GDPO${|el;t2ohyNL7Z)+dW8i1Yg_ z=;WXn9G8$fvo*+Vz|O_J>bk(fnTFQ3_R+Ed;XZ1F?O6@i%dSZf-R1hw6<9* zl}GC>+~@_~j7x&$d*qRt(1A+gK|wjN6Qs_O(ccE;uvIZCsc7i}Vtb3%&lJ?~?;?zZ z1NtJvbEyWMhb3G*`w>3e<6l#W^^Sod;ZyLW(K%~O`6RP&Z4O~oOjhoqofNr5GW6p` z)4>yO?LyQD{rk?Eo)jalu+Pt>8X_L5JhY!lTRa$>P}NVfakR}X`ha<$%;!-8KN5Dc zyjb#C9}Qe*M3ZV#SEYn8Mxif9GFSUcdqJPE+f$Z^BPlARxx~*%2Bg*UVI#Bs!X=Ya zYim*t^>(z!xGG>KP4Vc*!?!7*!cvlY(UTx$eKF{db7ImxTzq*9^qr_LFHz4$@zjak zs#s{zn6VavuG4fgn9zZfjZHjg(oK6iuZdEBJ5ndD0|RUOq~4Y$6xWR8nPw3ZJZJ;$ zcMJxud8y5hlvV<2G9>%{&=JvpQJGUm zjJ^NZUtpxa8C-u$cc8KrYsETUuB?mBdhc`dFikXGH3N|f^&uvg-@bbgYh^7ChpL2$ z!4Hb~2(U?bZ@L)5%5+8Kc%CI$GLqT#F<)6f6}Rwj7`HPux+L~k*K%(Gm(;=L^s!5pZMnB9YFUO9y*ZLfm$bB|g1l0X<>JfkYGFvE&6XumIvx=3%Aap zBkPVyZ%xm)TmM=V>{dPgDTSlbv-B7<+{27dV{*ta^NBIQYm4fwdrVy7LL! ztba_u=z*_IbdVTeazqCS(4TdeLP>Ou&cl;O^=&ZaHT5~6r4$0&E!4mOYs8yEM|NbV zf8l#2lP}Q|b=~Eo#{tUz%j%A#Vx!Zd(+T4X_TQSwOD{SFTWZ7ykB4^`{<;Hdt9e`d zLfowSqn3KW2!S!OSn~{=b=$A_wOHeIr@)K!*%KNAHIWq3&1}e-@CHk@)hWNL#B`XZ zq&`-r`H?PRW17XU+Le>O%qOD2E*?Np%fSF}wgL>4gtrc4O=ntk8IcJ9_va#1Y45?m z7rkvt{^2duj6__XE8F>37A>M9Hx7lzpRV`Zi3vCpP`f#_3Akup+CQXB@@`5bZ@>Do z?zB*&eNiD|=$35x08s-t4f+)mg%{L1a;O}ZPU6B#CJU<9Z@@{qAYsIHV{luPl0iYp zc4X1c!X@d=#r&R+2a3gSb@#I|A1J^wd=-#(@AT#e2yj8i*RubTukGyRC{Rid=AZ1| ze@~97SEQvk^z5aIb~RsVBxujxAAsde$<5KLA&zC5Qwy{67RtdkzGA4qT&F&fA@Xsa z5-V==3-joQU+0Cis!!fTtxpSQpy?gCYK^=e6{NqxGVAP5ZbQ7c^6D_%Y?QA*!3JO@ zp`VnHOP>H<27VV5W%5^3i^LGmqv^C!akNyN#i?{pLc4b|Q0i23H|yE_`NQv?S4n&h zHF>&9{$pv2H{6eSS1O~tI9@Sb+n+l1SX)EjISkfLDBW?UZw+JCn_)sHOz<4W@l{{L z0ter$i2=`*>s}+i``c$9B@Lo4tW7Tt=ny@lL-35zG#h(@G=%yUN4y~VfSPeufkcalkd#stSb3Qm>dH-dY|ECTs(Xj| z9tWO|pq(alsh3RLI%a2&HA*ZwkKDxEHlk*eM5+V=j6Uto)P5_81AOm$?-UWk zxT4&PpeE#0qhqa=_)*w*3x=)Rn-mC7lOPSdyt_JTgzVETU$?WHo`jTmc&&&CO{p)H zF0V~ql#VBd+78$eq_|DoJxR7^HL}X^YjSY({j_9b)L^FU<=~K zi-_24zs0aB{h?|9{DLczyP8L``R9bMp~Ik74GF}^nBtsk`=UnI{J-AmSuhtIZj-)<`m{pp)v7h<^f`tM zf7uyH+VT%F0KMf4xB9Ijh2Rbjm5R+RDwK_a2nOI{ege zF_}D6Xyj`Om$SR>KFMe*Ir@91Vz*wqc_&oht&^|H`eU)Xz`S^YBPXXZS3q|p$LURC zR%zOII|XB|1|Klex-hS4`Z!)E4Ow8;x1->F7}rRQ5mWs$3nvrh}U2kll!6y6~m!{uIwELu=kLG34ghSbwvsuZh=Z7gZY&e$MHaQH3 zvE2iQ4xF#rAK#JvD%3Jqv@GdQE!e_Wd0L(0P9ut+bZzE_IX8ct|N8L2@z)!bNsr=w zc>XoThb_V$d16h?DM}tKU7g@C#x{DA4}YfMl$3J9Nz)_SI)g7)Pi{7Pz0|gk-`yH` z;YGNmUs&FJE-e=+s)KP4e)zDeKn!#1z=~JhP+zgANn4o1eAZ6*dkhmmg8?l~qs{8NDZ zTD<)0WHg2+$@a$Hw?8WR`uBUci%raR_oE113Qc)*pLp`jMZqUubF}YgBT4*at!~{d z#zgXw>doI#h^cF!7xSK36)9Z#+Rbx)rWc9W(bME{WH2-3aZKgt<#z08wo*bgGK2Kz zr7nZH9gp%u_2x&OQ|Ql+K&2dA5C`ifk%$mWU}U-}%G~9`{etp&eFu$Xq(Q5GG`E8d zB#p<>mrXPhah>4vyj}upm|Rzs@RHk6vVF$F#ZCtip9iaJ`Ma7gD5O6>fAy8yQP^W! zxT9lZeaq2hPa`!y!HXZ6slYmA%@NLsC-jPk^XAAv^lIKM)P(KBP z892psF}cxZ(e-yWk(t{?X9f9&KCQP0#06}($wV&|+LV5xi3IQRitI>QtxNhnDho1+ zq8Cs6V@pGz5M?G?LH-O^ud1R8i- zU=LWTh&|G|SBe)we7|m#nSOZB04iM?4RI9I{o^!(ygb{TRZf#isp^yB2rCar@w~v-#_-nQ@sTd8@DWnV}2_Yy+kiFRh1Zw_Wb<2HAH9pZhW_f zfic3agq~VbFlKXw)X>qj%WG2EQI95iFqQ9YZTVhgHRm(P6Xi(SyyOE&E=3Az&w|q+ zRKQ}9_@&<@QE9;AwocZH{krZbhYyxV8|IHBJ&p-??y&KjePuk~me^^zdbsb_`B6_^ z#qLkdvaly%f`X50V_}Mf#pByKR?bKB2WO2S)&k~M+Y#?*7DvTfAquAndW&*j4e%9Q z9erfMB6rNc>N!=&;oH!qyBjL5Z1qSn%$5UuQ!Hvrv*W<4aJ?$W%kJuwc2e0#&jH=( zL5oF6@R_0%R+qfLVi@U}+cm`tuex{k_Mu{ws(~Kgm0|+&MRFaXhwoZbA3d@f+W$Qr zz-~S1-Eo=D!QpW{d<&qS##+M@w->dKJ!0N32b8srnriD<>I? zUWxMnl_y&;zuKPBH`=x2NYn^xM|Ds!q2&BjfY@%`MWb5;JsVlZ&#q$ddnC^A`*%%* z!S~A6S){c6QzfTx=8D7D>0agk%3MJ%XO35{pvn|SA;ss{-&5{7 zm8p|=q>U7LSTF&8-(Y=dI+*GWI*aI%0(%5LXIk%_$a$5MDF>Q{dJph~F^Eo{2N|y5WLcvb>?=+4?R0rXIZZM)+!!KOg``61ql0?< zDzmR`@_1VDHOc8?S)-)PIKhp_Idx%8zLG5p-7aVq@=73%I_s5T)3$y#4ET5M2J6Xa zLUQjW)vLnqEg-8LI5X^1Q?G$9l`gD2cB9(64# zu4b+4vQz^04S3;|5_#_NJVRMU7-NIXZI^(_54-^3Ram)6u$z)Auji3PT$7^?!l^RD zShRLgU&BZpuNG%_A9gQzL#uNuejkoMCsj=beqNk3$6fAbX5s~4&Iq0?Ap2j}Cf*Io z)RP)&n$L}fhf3UD0oQCMj**M;zw)E!h0pAkduQIWfOlcvfpC%IF^#z4@W}0M_YUHX5$J~@Yc*Q|&x1h501FZJ+$+CY)I@(V z*3^B7*mC$g%ger?j$BwvJ5x}b9|x@Ntd_BWSx2 z@|MjDsQ?5qYmrS9@xhw{H>;VhIw1wIOcS*T7tb91#?gG&rtxT#IL*mpRc47HG8Vod z7^C0Nn5-RnsjwmW#mDCh!jLaJm{MTcQs;TiBf{QW9i3s>&52!Ya(ang#VGwj_`n&K zrtIY0)B!iW23e+Q{nGfPY|SboyKr&*-CWEMmZc3HtfdWF64A%MN2hGR0GK;waHIwM zvB6eX3}iWO4*db*>L$tc{y$BAPV?Hc00YdaE5~y7YjvF;Q6Z9J#fN zV7}z~i{Ec*`tM3=xildORW9DZez8Il8i((GE5Wck(vT}OsmJ9f@14sTZs(hN0Y4Iv zKm_{OJuSW0v@C~GWGAVxzmOd!0NiJyqIKOR%+%WYgp~Qq39C_a#i3WMDs-~i8eoa- zeo427{>~KD&$-R7hpgqrP;Seg#B-RX!~*-~XvevH@e&y9U_#>7FX>iHnv!e(M95=d z<{0Wqe&sZZ;jTZ=UeWdc>T!=!S4!&mk(#hSbQTp$-qa@Z%W6Njdx&Z`{3fMv1XHNl zHpB0`9*Y~Pnwbu;-uY}r0S3w|q{h$(q}C9mCeoHtvzHXSxOxF5V~dM#$iigpvQC&t z5wS_Q6ps?rwkDOIoKY9Mz6Tb(7Ltg;9a#xF$UY#w*E@NB$-#EMKjMeL=|7;dvUd{v z{_v=P!wQIo@4gt^iRO{Z^PRs}JJh4b;&|eot>9C1wWJETblyt}Q{rO0Re-aq@Td>s zs4x30v*WyT6kB-^Jla=is}=@Bq~nt%!0Tt032w7l`y7Jd>EcQK0p(&zcxj%Q_zPQIM zG;vSxbflZOf1&5x4AMkBP?>EsSL1KZS6{YOQIE&Os0$sK32Ge(2i{vEyHS1JQvHRc5nuEcJf4pr(66r!38-Tj9RrY> zBF?ZNK)t&4IPNh6|6BiLz`(AYl`=uOE?zbhs?X6G)?%0G8h+GS!kX%kuF;mSrmwFM z{{8!ATi6rXPr0XAL#?rk!>>*SR0A4gyxRM!E~AqPeylzpsk+#E;TXmB?*1ZbiLs!L z^pzpG#bZrIaI*G!^reZ4lrRsEkg!NyfD}ePduG*s6TxE?At*oz#PZtEx;NK&qTbb z^B5hJH56=)t3Q||%kK9@;%+b99Ov3eL@_=;cHO>=`%^5#tOABEw0Jk+OJpjHEp~J0 zp(VuOEjyLekkS2OQ_E3&{cVFb7qF~Xv2sPG44(6zF}VI*`}FlRK0Xe89HbvEV@}O) z<=r7n^=z`PlBy7UCDef*3UI8$2XnauT>6Qw^7o9-b7*ei?!z}HFVukekm`X=9tFBL ztlvI%qfQ=mGTitb&m7K%x85ZmiU-Y<`i4X?9wGtmKk#0yN(>Uz^01)re$JYSGmbzE z7@YS;U|f*&6NwA+2_3w-V10XsI)?fCGkd!%^Q)8(s0!7* zgF)@-)0pMcK+b+jrt{eK=BG?=W7CL`K6`3`Yr{t3?2C);f@-FpT~B(dIT9So+q*!( zsy{Da)Xf#zQb(({I_+nI$)WSs3z3=p^}ylnsS=+9)F2gWE0Uxu^KVMHw^rT{Z{KB- z_!)5QXFu~-aoZv$8gyJ6QSu!Rv2 zl}y?1*QN#a?!o6-<78MeIU`QSqEU~)%baygxk2?%GSS(_55FhbH+@lfB9dg2O30h) z8dLIxmWE@r`Hf9y^kgf$7>hEH>fLltc*V%a%D{9WMJ*{kWwF!MeD!UE3NCM#{$an2jZ*TS)r7zUft_6JF$AZ*_&;H|}-ve0xt5i)_A!HE zf=Kk1#m5K8P6C%&7c_iUa(Q!ercQh&r#5f7#6ZH^{lZ&>`k`c=AKzuj#ozirr1QPFXZq0V07y(1L2j0l!=R0y_?mx+53mE2TC%9dzm^8|R3 zGpv!tDtG+C7b!O)5#G3U`ZWXkGR3{&vvo4p(Rq^wk*u{~tLyHuxiYqKWKoY3PDPWN zaxMI7U1Owx-V{*)?knC=`dPf#F-3+-21)h%0hKh*i4y~`?NaRr#KZZzHDmf9t+c~; zL>O%M_MvvhL!)bgHc3QWf&sw1^N+nmtj8RZw(*O_QNd%8NfQ6oN@IHSUMr9Xhp0gv z88d8!6aQ?P>_WY&&EHZZKwu~oRen$=Pu(@yo4L*lS!k)my~2jd4;=A+ocm0y+gtIh zeL1}H;bnhH*7<|oi>sU{;DJ+Sz~0^AAf>Y{wVs5_!>qIHWko4W=iZ8FtV(ySA7443 zK4?cZNmZfm5VW5Ya=o27fCq1zeVDn9`r|zPVRU4uBd^=bTiW;TA^vom*~{mTm!gE`tgAu0WYYG5_Z*)PC-yY-5VrKp@O&b#>e zO~s48x!&EHm!zvOzJ$9s$=V?6A$!YJ5^+={i6ehu;)yHFdF1-zd zxt5cUmvu+O7w{AD_*(_)T(}i4)Jm$?4|>UyIKR4}FEicTF_SKb*3Xl4R2wG#|06bC z&3=hJrVxbe_jrq76mx!-#!FPE16V%4tnL+dF?Ev2b2J%>An7UMkNx(|O^7sgDEu>t(h~)&rR&@yOPsJKa$8n=06)D~X}8T==C^#VNicFa3r;`dWQs%TSv0&GvN#X| z9meEo%;^7?Xz3ZY+h77qBDhYGc+C?ue%GCn)rfkYQiRXiqZ^0N91O3ReO{u;`jFB0wWb0;Q~HNgF?s<7FdUlmNldFXO;Q1qvNcahk}}lP?1&IGc^1z!mcpFF zhMR-+RQp>_QSFrnZ{LYl_x``LpE=n7k^Qt@uovapjubwaki}Q4iv6BZ_VlX!DftXE zh<#o^lE3rzfW_08f_FCE=W5u2PHQ5)AyDw<8cuq6Wm)lg^#yi%sILbj=hk4EMkKaB z%fuNv^m&6FJP`>G4oZu7lm=aJ(F`8*@fBphp~E?5db2+wW!M#)O4PIgT>tbI6qt;R zsPR@6z57_A<7)ei)`*u0)%8%d&|tfgZ?a|nLnh+wYPJO9Qeu`;Cis;)(?UJ1-1@z7 z&8aH2;a@fVZ{$owifpCd5BKGu(~nxS)J&X@DdcN%D^{YTcU0c9nk=zS`(-Fk>=$(r zw~e=d>iqj@fZB|1c-ux%kw!dkVg5NynoZkusCoFxoT{yd1Yr|8Y1H5JpZMsW(z=!m zbPL zar6ZG!w~6*S@hxYmFbDHSZx>z-IclA9eeQEodt1&yZz9@KP4znIDqqliGL{T))uT< z@)<=Wv-})KG(_wiXI#Z;;QS!M#8~XPWz6<|bc#}q(sMAGkq?`W3*+!mQJbMe%hEO$ zSQ}!X=Hkf8R(;4`E=-cf-Zs>Ld@fHJ*njSilCb<;r|wK@&>tIc_&`+cZ$d(fN~4bp zzN|jC3NM2=OKvsd*}%5*xS=MGMN4fq!jshdu~2#I zg@-3riDni50KLk#u%0~Hgm>~sKWiD$+*bp2DvYQI+A{xBx189~6jbf}4i!l0FjElQ z113?A_^jd1G~C2OHY$9E_WM?pj(h>uQjy8Ad44y9RZ{unlAClFv6LyTN)A4(Uw@5c z{?JRH*3aw+Qa1BNSpM#5f-b+8VNZ}6ab7m3{*-bXA%hw`YvlI^Np)On8V!CzJOPiH2oS+g)m%dLb}NxKmAZMzGfNV;kGSpi2AHgCTMH*P3y=MPxh<;ZOGVy}%R*TZfS%s1LIROT!c4TultsqUGJ@w=aM zwjDw7ieL8~Q#mL*mzU*bL)&N}Mw-5dg*9^n?&a+~AY|8wopL$dzbJ6*;H{O;g1rx; zsk_+3&0~LG&HJMilSi^f&OWmZjArJ|xLd)_Mvw~aH&2%In^Hc=MJcI&ds49Ri=WX+ zUx2dxlFRPC-JlgSQUOVM2)nE2>>GzWPE{!G+TF7mVfv$)~30f3u&)%*9&mspu>|M=m5W$e#+N6VeANJ zYSp3cgS$D@EQN^+n9tEm9b>?sJP?nPc{6*D59B^Rs_y5h4tP`s>%I5wamQ110hp?| zU;OeOcw9+hLCh!Y+w!!bvdo8z5?`#gwh}!3d-8O<=A7KLUz_K42+O4ogW;ykdDH2P z+j{l5&Nvki%MLXzxL(TqY>RCr@^mZh zmfc;Sk05iX#~_x7kmRE+{k?f&=yv$8-Iq28{TY}V%L<2DLy4}NRvfESTVtq7zeVz3 zD|AB4G}N+sWs1^!WiRUoMkM%xAw%t_o?8`TZK8sXN(Xjcb-E2FVxU164i(XW{UP`s z>c068sA}!@X0=vj zhE4lT-{)DaRDirLWerl2OZ%?PLuhI0pUD*Eq@TS z$RheSJS4U2XBb%Nab?FNDtz;GkZgYiU4F0^OKwdJ>PDXSERVtc#{NezwceMobo%0f z0|knT!B=%Sk}?z38yWV~E7ktI(cuDj_=FxaQtUgW?;YE^%r%2MRjv=lpEYeCJWQNFk%< zAwBvsLj6G%C7?|%R;xuBCZ08c>ditVR%;QMs+^ur?ecbd_W;P9+$&6|(F_;6+- z+~ES;`w9v*+)wLSAREd#a)TzVoMazdqArnG|JSI8nE@L%nLgW3Sas}I_UuSM55I!i znf2}$B|I!tLcZ7cD3B=-A&fX_>n?F>A51SVTTclMQ3b>J(yU49PtUVk{i1uf|8fxO zH(hLM9o>shAaV(B3^8E4R};XWG$Q=H$cx+TtEPB?n!zWDj^~&hzLh$SiR@_>q)P9U zy>1qX@Oq+X^YU(fZ|7jrz<6Y$b~AE)i9ZTHDMw)icf%)+=js-oImzlc_H`~prf zT2VH(or}36PG35xGK${;4~s$>B#4D^=5@SWKNNLLP`EuKj;cssL-Ra_$HY`V2?Ei*at}D@k z)xy3G{(L?{f0j}|U5L^|CjOI2RhH&u|5_1)JdcH;LC!lAp@q157mTAuwVN2UCE>I_ z{})-7j8@iB_nVVN*sDywf~L!t9}L=;-yS`F$$U8}h>c6SYG4UHC$75;EI-^X2}A{V zh6uq6(gPe3i;EB|-^aCO#@J+IIhWhr-bgg;1P5pnv&?h(v%m;?_osI>tfxaMUyWx* zBFkZ+{Zl$$YehEjwJmHkT?H#HnyG@&u-=b-(J@|I!lo07kc6%0S7uH6Wz_nX@YSg* z2sSDp8_j{C{c-v0@f4IDIlZetAn!GdmmF)rdvKv?&2)IMXDhypcG5gOMfu;@)?Rdn zM^oTdbnR8>@VFiL%1s)sieJMRaQiGy;WDQDy+oX<@4yp)|Iz5>K`s57kp5CksgEOt z?Dw{&lie<+lzU`lrm~uMXzTJPuu7K(wL%l~lkYVXeE$fvPQr{w;X|pIKvWX&CMmE~ zQrV)m`g!$lE~&i zK+mN!16oI?9fH&vX9=e5PQ;2d=y0mCdQ*PX;*a+d8k}G<;zF#M(^@DMkT|qk7NoMi z_KNL?Z&@+Q%OJXDk`g8qf-vx}%=QzudFj<_P^x5Yx$57(90IxN+skZ#>Y4dli+lYD z7JqK}W+vDFm}!1-rV1dctD#=hJKRa+H2QpC5M8Jhc(Q-o)oFxZr@g93%?bDx>O1DW z%Z;M-GI~timenV;g7Aa%8=uT!e`x9? zmytok`nyE;7IKrHZMi9&^UjQuxTo`AiGM?=Rsn4ToLDh1-MNJ9j9N_+;sr4ag#ERW zlbp$I3-4n858!LPRK8ICT42-R^kWjATt=0b1$=|-wbl8Id#Xw-xLG}O3k#RBTp5gL zNuTa%G|TQ120iBQHYOylg) z*WCktm6an{y;e_c?T^8NWxR0F`9&KZQP|a*$F>?iDAuWlox>qd#nMV1OZdF(ae3M8 zL8qRB5z9&`t-C!TqqE%#LNECi=NZk_E!CE_QT+pKP3amtp}+6n%xjo+L1|EJL>cQR930hGo+ zemGPpKdf)R8ny>V!`OJZAK6SQUD6!I&sJYhiPM9YN{v4$Do)XI{+}YE=!Vx^4f4t? ztH z%uih%7RQuo6GoeBy;y@bzsXSz)9{LMd-sSbN1?FZ4@zAek2{-cKWM+1`=Dwj=mVav zTJbJg4+6{4Q-4UE2~psmYs2L9!`~j^h$$G12bVzpN$us*ruwML*ruThyM=$!r=6e@ zyMgxW_X@xTNdx2NQil&wZKZaWh6R3qu7GQhQ9v&dUrS#kUt8Z1nVJr`mcS9}VqFBc8C z=1p8fITy_n0fCHut?3+`>}au0Jiy+1)h1cz=yby&1WBKkYU1^o#V_6WksxMyML(Qv z;Y~aiAYs^+99DqhS6OgPJJ+UMoqE;x!U@P85amSIt~@4vo^f=H=ImxxLUBi*2+G}4W9cf$;b zl(dL+cX!v&AUTplNe$gKFhlN9pZezS+53O&{eQz7jsphPz1DrN>pHLVbMV63Z8vvT zh_Mci|8b&Xv;Hj#u+Q6T4*Dl%6a$b#xMUAeFWa(Fir36hdLQdOo$6lC44Z#dc{ zycaxj6ok}O?3V8QM2BYlm-2GD>H@5N{SB?uQKt%JWpFOlDX49P1dcjn9XCQXW8U zkM+@R7i>J1Dz%!$oRs%~H{tA4Xo&}}-pnC!@L8${nwm$r6|AVvFo_=16;^WFFxQ1&E{pz@>A3pnWblo zwS9*WNmLH$*~7b_V(InBGrc&17GeVt0*8n$$BlwpEQV{nAM<)e(Or-Hl{Ncn0Ur0% z96{w3z{GpK0KfO29^>q_Ga%&jy0Q19w=cKz%)^B6wQ3{eJnq(Fbuq==-y^&?CaRO0 zU^Oz!YdJA&OXr1Steoe=p{lCS#ZHXZ#XVCa-~>CXqOmWKUCC@|YG*91vi6>F`iu_a zt-kr)Sp#mgJtd#gFG@Isy$&`k6cSTx@F{=LF{WMNV*`N-I~#Mjoxv<8-cx{p2MSv% z#@?YtCqX_9q}AnaXA9%yFMKkt)CCX*vAa}TdJ(J@kjNp7AA*LhsI^8r7Nf=S7Z??$3AN#_k;uS9 z8sJYH!#Is$9@#0S2t~|rupu#Lo?k|vN$|b?0}_e=K$o^f%g0InA16h#LDb6-5!0p5 z4<@w-N5Zqtd2CE~-p(_-ezxi)N;9I;UdZOisfuMRWx-4e$7i@dp2T9oNmwcuz*Q}< zHB)L`BB43efZ8zmJ$}r7+Fv?eJNR4o$y$08b3HCyjfx$EXp&Of@X zBR9w*7=!R31!sai3)jI;W6qM(?)?LzX+GAlHMU3g7wVj%v+W&>rVj{MQY0OBmlw0s z*|Vq?{l;BYG+|KTw>H#rFq@?-Ps00b)E|P_RQDHx?XYtr?3-X(r`3vfIM{62_O}*a z`;J-G@x6vUkMrUz?wE4$LC~y~r#cbMYjKov%wj4L+V8pA(lo6+3XCec{Fu{i$G1*G zN+xaRNGK5{?Ri(a4X{h?wZC)jjtD%xzcm3DF02NtleOr!k8A6provDI5K(}4TsF@~ zFFb4ngSXslbh$q?_(3Ke)3;h1!e#>FQ#v?Id%kV3-Bs!5Ms`MLW0c^b?n;6Mq1!D5 z6%%c+m_AmEu~al{g3YindZ(_`Z;_PRLvqilPx;p%wbneC`O7uL-mY-o>K#pHisk;Q z{@ajUb5(1!+^A9Wd)Vog2lTtI{p1ynoZIih7bV-?HkY%l!u`rch{wKfGzg^nNK5u` z7m7N}bTG!TytQsovR(C=q1E>m1Vi)`NBHiXrtOEju=mYlcFiS5b~}-bJ|ujDhab*l zwhFZ1C9o-gq^(0<^si3$yjLN@d>YZ;5 z&}LCs@fMQVqHZ1>pWl6DeC7adX1z7k>DUD2kZdrce+p>9ypHP{z{Gq0%36pfGsX^@ zq@RPMc zn7Yui8~?%9lc|bV&q@r{ys_jgQL_+KB_J5Nf4@5~NBCYM&KObj0y!t){AcMZ`Mhe7 z;7o(}Z~|cB1ken}u*SxjlhLXEVE#bgPhNG8-X^FO{5TBXA9W_xHD^YhNwX$kR{U+y4f>3TmDH<& zexZbj>iL^)99q$d6UUmC?{y&@qm)8A3)qy1D?B$uOuVRngwh_Iw$g1w8J$aKVfLA2 z>FBR+QUCGP)fFV+gkxlps@bnj!F4tu0H1=wx<qe1uJX=ls06D54Fz~dpq8<@JsAj%eN^r$g*@c_59(Rwy31FZxaa!46% z7kA_EEBKVMoLU3R3y0u@IrB}CG%6|GTUtVhxJ(LpIZbmMXs6&8G1tLok3IReB34h! z1dBhkvAo~kPhV$#R=%e|xRPAr{y&g@Q$I<+i3dn4kSQ4tp&*gP*2!?rsol&ZjL$sU zWAvdEGIsB%7qb#@w29T_lNMJm`e`VoGlURoX5~WF-$dV5Wwd8%s)@-oC>W}DU-s3C zmT#uq)Z#;UW}2JOH+q2@qms@)|k zF1>-=AMJ!$y3KHJNhu%qAUf=O$wLE+Do=Ohw)6+}`NT^vSmredXQsouaa~kxwJ(V& zj0@O008uWjanP7UdD-9aX6OrwL1WV~kRe~fP?7mCw(Eegys*HcVRG-Bw_l|yDB-mw zud*}I*RSTSdd-okY@WQsG}`o%bM2mF`1BiQEz_g=d41HE+E_PXV^>wx{1@hZQ;@T2 zzONN!NfNTUf`cN@SwSZh~RSOn%6^L2A80=A);-TUU@&^D2 zAmSqpo^TqAa$S9Rwm@gXr2h3HemK-_Xsxx}L0rPk?&_1qC7U^O?=yKK;jvI|lg{p! z=8Ek27;2yFo(7U~Xp0T!gNhI4OA-%COTF%GA%Qie-qtHQ|A;3dmVM=UhFKiNJi+Tt z-4D5*y>HJv?`0?aB-VXw*-vDBS$D8k)Vnq}6IA1o&C$%@3&1B7U5spw-N_fL$`g{; zHo9=9n&B;Jt{E~#QHCt3RV%3zz~PA&?O{jncLk6u607gl@A`@v)$j9>pI;bST`aJ% z77Uy@BnA{lpE!(8%UkTqbL6O@iVYy%>1ySCXOq>Z{A_|SnBNCMuc;xbB{fnCcN8|`Miblt+BFZN~9_qN3_ELYN0(x&HzbW*E{h z-juV)GM&0~z0cJhoq@J@v=^bIWouLt#QHHmLZF}FQv9myc}hYIVFD?cdCZ&`e%BDC zXESax#0q>Zghl6d4sA*tsg)3;u2TGikXw6&Ge+CgRyr?BrZE5M(z(nT{6krv-;g=48WuN^d}CG;^5s&g_h!I#dK>;0Rw z4ela3C}s)4`xQlNfwH)fY=Sm7*?<+u{p)Z=`%LVyGsqG+TBNSbl@^0l+B1&H`p)R9 z9f2MB5hnlbv#k)T-ko#4%XxB-_T4j_2*^vOt#^5}`eE2(!KFS+p{{AJTvU4O-xRg_ zvMVUTq5DG7#YbX?$Z2fEMU>ApSz@yX&K1Wzjyx&tMsdq`Mg@#iPM#^K*e-4cI`#unjKHbc`F`k9 zEoXWu^KJnqqm0vL#Q+9bWQISAkatEh4EZr%-<(h0`P$1wRk8SvGdj8ZaQh%kyR`zT zbeC*106eE&idzAzEN%Fb$TSleIblbz-Q4r?bf{5-H6wJVw0UUgC@g0l9%-I-@=r*d z-81b!s5GiSsI+Ru4HEa6Qp6&;e6Y)vKT&N;2lrSZUURY@$opxap1VIp3rkdgKO=R& z(ztC?&uLtA^=$s&%tvbUMpb*-H}}~F1HuO)vdE;9=mhU`o|$|b`d=(+;E>}(*Ez2E zrdmF{F;qL;?Jz$YyM_0gs_cfP`-z@J9_RX9jVr8_(B6gWHO^1gI=&CM9wu|)+cw8W zD~T_xRi0VNBM%as0tJE)%A#7-t>}^aYgsIu-ZvDN26$y_U?`#!YJ^x#^#VfK-o}l>rS_ddoX|4 zML&{-{RViN$Yt8Bv`{66&D397T^k6|6A_3HX}1@BV*UJ?S<#kvht~TJ{C8mb?_url%uv;)nzK#g3XUc{%GT9RBp7{P+?T;cboc$z)Z(Y7tq~ z{1Vl}!S=6&xZZW)$vvO7I&8ixR(@qHY@0^9ctGr81kVI^iCyEzLP>93Z1R-zMTY1+ zTwu3&CUmjO&wqqlVb~-18Nzx0eq&r$B(EIhV8%#aZ*W5Uxcfs+%X?m?wFg%dmlWo* z{2IRVb&nH9PVM$(!ZzMnSH+Z~Ha{s;wXp_BA^yD`Rw!<_2y3ZaeNTEVx01HNd7$fc z{LM)lK)!H9an9p)#p)A9RL(se&xRTQC0@>K<|JJ}J2R%?;m}jlZ!b+)q-hzls0MR3 zHB(XvFg_TpKAa&KL71Y32rS*k73QhXIXpSyM_dMmusCf(E49R zq=rpp2-Rt8yxQHUn(&PM1*lnG?y?6rRJs1K21DS5$_zuI1M$I-E(r3JZ-yr+Nw{5& z>R@$3i90-5I7{u^iANw^DVMFz3b5DQ?CWNNACMIwT@#a?W>ETtkQ&Fl@K8qKFSne(v15awo1YwQ2bU#BD7;h2xA0&doHi9lT z66;^+5rp2oe74p~N6(_Oe6=(xPhP6{ly}~CWy>zr((JHytmi(L(#_5KTzJ$yZx`;j z+)jDD%b+?&WVw{TVa2=f`&T!}DaM@p* zTwRt@ET0d)YwLVnl|A|IIkJfK!&@8q7IQhP3e!eu=`RzNIZ~C_4%c~m3+RPn`A^?u z2z!*Q1#~M85kt&XZRqLEm%r3RNHJoEha1;X`-fQy>gxen4?HI-O=jq~dt{!s!wR%F zrdA9pMr9VWQqrr2UxD;hQPU8m$=q5Kff`?yy0aH{w*o~(2m@Vv5;yf2;*lKCS??g* z8@^!{LFw_UKH7^vo+{mhmp&e{^xW1Bu9Ck{v63oERB;cr6WOp&K=M?Q>>SLp9t#8= zw=;4D{0@GN+>|3B+ER|dO0M_Y@ACzI{xr{$HV&MM+MkM~BWaYRSJoT%0LyJ->Ru4z ziK|syrc!-Ua2dDL*sJQ6Hx_ytfW&{??Cg^J!C+-lPE88-EZhqmThvEL{=p!QDBM3j zlNU&{`f%rS*PMu&$pzhRW<1Y$taRY1*4Nk+bFA8wrCAv{la;r+mZaN?z!VH?Jqn{U(y%d5iO)Yt`ZvFDB^PLKTf4zdOSg zAJ4YmcRE+R33`|Cmh3gJ=R-h(Nso)jU-_(~GHY?`q8ZHaM}DlmFl-FDnAco?!dTzM zwdEJG#AI9qyGojK&Gl|(^qSvhGeD!0nI_J}T)dbk8SjDPbC~l%nSYaI zDpI|BD&g)faC@dmlQVU4cvpEPX;m@TH6~R0-HU~i^SzZ_KDB?KoJckQBiCaIAb~x^ zmp|I#KRBB%IhP@NLwc&Tkq?Q}~1sbC+d zc77m9lE$uhfVwfQgsEi*r7s*Pt@C!fi4mc<8FyLo7mlf^AROMUPKkw_@U^%Y^V-X) zS0d(8mpF*m?wdY;A{OBrg^zoJ>lT{OH}=Uwj+|H4e~!}AlW+TrNkYR;M2U&E;y<8e zwtf#>M`dS?W@84r0NxS#lqaUmBX)vazHxzGT@V|*^vBfx(m2I{B_23S%2g`CZS6M; zqLyMWnHTpTL1xL`IPa*d3SQhORDEG|rVc1@l#kLgOmUwcNzVqGK};>p3L)KK;QFx& zaEIUPQ#mkyi%-i;Vc9dhIFS}}zv&Tb^XnVBHkIT#($ z(-1LqpJdf8JVV2|l*SmPl?YISvnr|f+cyVpu_O8TufInoeRut=*m!r>Q@1z-#i4ly zvN~n%F2tvbDm4M3GbXCkqr4|(OMjgh7)QmcS(T$45m+Z#v2g+y3~v97bv2g9)L`VC zeBwwgP4*BW{fnk=ZxlaQ%J5<=)5#Nx9}cD9d;GI!-$=BDf^|@3%t|{-s=<$zN!x{%6z*J3^raKp;!vd6jN?7iZiK~2 z=_T#8c4NfSd+uFNJ3$|laTOsW>IlIGzlB3IAmYsO;v9CHv_tIx(afUGw+Ma1BG5jc;vIr-!=}zY z)mThfJK~bWY$rI6oMROqr!8Me=5q5io6!yyv{?9{?zwTm@CR<8r7O}IlE75aGSvW1Tv7oH;pKp&27ANls^tX0n`H9^{%|3fUIdycbb% zB@2W6#Km?#2YcnIB@CJ7J+_C@X5H^C?x(R+(NmkvtaqMLXQn#;$h0w@Jd>_;yPSsk za6{!zB}YS!qZU3aqwuTA1jRd_w}0jg`2=$c!VmwDOz6&4RmEE=`XS7)pw?Vq|# ze;tA^I}rBBPb@wVq&&jIU-#|7-+en2H0DSFuU{FVzAELIvz+{*ZBp;8hC(|H`-!JM z4dm`M_wuEZr}LXpM*v+}s7KlWq`TPcy31zz> zU`l#-z+T+>Pz6&xJ#DFbbgbWnH4?q2ccH)LwjyP_O&&#<#SCKQ@NV45Pc}J@s}pyC zumSe_XFBZIo5!M3Ai1$NeWRd3Vgn$Bcn=Bb&SN;HENqyjtj6b3>Kk+s zUV&F{4MI6=77#jJ?t->rY%9kqLy{dP@kK3QJ99&a4Q;By>EU#LK~gNT)j96le4d_mY^dF(yVuHLLg|eE83qhLv@xEHSnJyV^M6L!f7M!z9o-bo3R4c8ZK}^%TjQ~&Vn~zLU1J+ z!ulA+Miq`&KK9&GeF%wF((#JSD%P{haK9V*z-u!;G@)036!PqxMJC+I)>GD=n&0CE zu6xka6^)sh1~ajAOhDe3aH(@7Hi4uS;wgd7K=CYpci=(4t-i41 zEh<~d7K6MK6@HJo>GXCH?f>m>Sfrz;6cHw7m57- z>2M2>Zl%9(&$#1X*tGBm_A{p<(@VJlR->p{{cD4MyXsTa6f)7w=wvAYkwPdJcy6cC0MQD5qHpU?zF4yzFho)9q{_!=U}&VympCeq*Z|1hsk< zVqZJtP{1-J7ToP7f+amhO^y5A|(c z1o>^Duw1o?B*-h{5p>S1Nj7*2m7Qs*k6#3rV|#n;CD~?Q{>$@6VoEA^IJjY(BVK}x zqv`43$r#k?T-e^sm?$*hje)45^m_F5oBa%A6^dW;j!Z5P>DF9k4e6nHr|D0a?o`@q zhJ_+?C)O&2g9i`mSsW=^kg01^vn;-oCzqd8Yfu8bQvfH9nxl7Gxpz5_^&;tJ2z}9E zYfnEVmylioz$6cp2k^qhMIaiK9P;xr!J5gaBMR-$2o+$5&Wuj_KjBZFu}!P=he}#r@XYGD0gv!76T2h{Evpbnx~yA4FlR1zvv`sJt7OR zORy`~N*Y!9hg{W`PH8~3H9)ZbeVa%19nEt=pFTTtF56;cQsof7hReJmKQhD=+2JU( za;4E$%vOA!XrH|E?*w|*)+mAA%ijfh8H_C~Up_gphr}$hh@NlQ;Y65J3C4E9y1pB` zM!Z;a*Wx+J<}V3B?fT*gklYgrTHI{Koy9OM$%h#7nrG|*F4LpK&mKv`ySj@CA!>~b zy71gLLq$ENK%*^M-Y;zrL=h^VQ#?)zUhBvf(OGIiiI}Xu>zOk}Ty4@at*yZK^iUk6 zqNsj3uC1Q!#f1)CFJw`%p2*8f5aGe4q-6R3(4#_^_m>&e4g4 zZ(hA4r4W43%U))1q3%xG4S%JgGq1P@2X?z6&<^Md+9*}Iv4j<4YZNzb!ryX|cD;A;!1!I@3A`5N*lj-iX z)yS+M6RgqJn}4d2&4e@l1&aQ1EeCU*O=#^1`$)qDb?#I)tvXZQwL-5?2R9$l#z z(aSH?Ch}&uJ|g={4D^NUWI-c;vD8hT-FasH=qFla%Ul>XA7_R0&uo#woqxj?dFpnJ z6X_B8qcnBb?+XxJ_sI%AdA>-I77%zf>CXDZf;F=3JCHg$=5Qw@NCaB#WNDL0)8lDjZo&JE7l#0j1nbTFemEvkm7pDX%LrC z05(f+s37IP*xxaGbFZWuAkLz!G__~BLZS&XwVN1K*OnRM3#kZ|x58 zo>@7)(`vT;N3mPXh_H&-Gi;0Vr8v%elonv_8s!3{fk*GezjxX-5m)~+gL6+6z%?yZ z4Cn?P9CtMuqQr{wAQX6%hLX%}%}Z%w$ToQ9zG2gXGFlFq%71Nd)L8Hgr?8w~&J^hK zdr==U)tw)&Ls#O3aCZ8(gVdIgwe}R8;iXWUe45GuHAQpg*c9s-#Q;=#tZcx1vsE-V zdUg7@i#VK&>u`&&wJSJB%$$F^fCK4wr}ZrYQ4T z7<+7BSjYIl*;O_?OuYH~_}+%g@R2>LhL`>y(4@_}_;ryhxJxSC&o#xnetCJB%CTda zwZH+VzOw9k$$;-_Up02;`u&Ytnp{HKr2TZIY;(sXV~Ij)&j;c~l+(`cE9CZAA|Bb) zTXTB(>s9ERhMP|$P8^LZ8WZ(}^04^vaye-14Sm|@ne}jdiW@(JSbjq}0eY-ud4WB~ z_NC7FDCih9@oI4)E^F8|qCV^BTCP9O!)W?}z79Qr#bsFmz0e$+oF)86^>fSk10nFMtjZT81y(aHj~X)K%Ges3F=1yWDeo}{g+?yUt0iaL0`cCnLo zL~Sr+)Wl4=y09TtEjmd5XC_XCVVU+CmydhYZ12vSK%W8&+HOd)te+{G3XV%%;m+fK zP?nt_Y++5E`~ ztq?5APuhAV{U4lVZU5pd3xB<3|KF@|dShwLf4~4Q_R`;LaTM;fhD;(O5nL(n&Lw)j z?rpWI7GuF+CM+yp!vn;t!G3baU5E5;x`@wVdMgD2?|F9M3S?+5K2dQjZFnGO3YWR7 z+lYGoycd}{)zqhI&8UOixxTdf@vi-s9k-~6kv@7z^GCz6C#!)vtD+ZE;a-Dr;rTv8 zbvOs@>Ql3UT@}DC_KSYJO7}JM(N~QpcEyE&=9NlRhEwHhfl{h+w%nJ@H!PTEU~C?o zSfii8=v&;)*7*6)Rxdw&@KoS748W+-1^9}G(GIl4 zk1UC6U!v`wj=fdUEIzsHzHp>|BDZrb9D125FxTco#%iMY%&pagc zTn=fLSKk4kILIexs;+|@_HaMipQnX&hYHp6GW z*=1yA4%P%A9#K5G(w_MyAuYy9%iR7AlP!+V^*KwXKz{FJ*EI+cJ(c$9yK)kBy>%mXrk zdwQOCa^II*ZxH_ih6cM!(5k;_|Dh+F-hQpMbtC@2=*hf6Xw?=M|Hk0;3y}#VIS=!n zTGl?H1@_4-naCL*c9!2J9&%iOId2bfB!=m@`h+H-U(n>RnEuds)hT{%@;~K#t~fF$ zXeFqiB|jH0z3!VCSN)Fq$W)xIohKEor;mhUu?;DrcG%V6gJMio=&|E@Fd?B31b2Ea z7oCSmo^n6zgGb4MQV`o}KOLZ2XLq>+?^KgmyA1xSc)I0ZWoaYP9`kx|sZk{P4-too zpBk_&aVxwQr=~{PjuR4}Cw7Es$^RNCSFMRBv#}3(w`GH-WTVseVRCVDjmi*)ksA5d z`I(pIsJ=C5gD16mL^n0De<<&~du!OqL^i~v&XC#j;U*^4mW&J3M-xV-EOp<$vVFfm z(qty*Q7|7tJVmD3Rovc-5bI{$aEk3vgT0=0y3uNVgz*Tjk-R_$YZvTP!A4#{Sp2{*$pMnq#T7%&b>U5!NQB?>t^>DUCu zs$~YY0g9QZkZOq(DhkTY$@b6b7EiD(n-eRq?9nY1Sz}AwyN28fp1(pN6UWY9<$=lD z!nSL%*93zbTq%auXVo%zM9|dmPb)18y>ZyOo;GMxs)G}ce(@6@ueB?bzuX6X(JYMhh>QlVts_XtFo_OoEa|k$qz^+kbhXBQtMgV3x(d zcPkr*OYt=0H|C}xm$gbk=aaO4u~Q^k%Bb7!HJm0?=7P_A^3(SVcFvx`+f+zOBEbt;IW9FhJ8;T==QjT)8>-*=h*5D zCx+)SmqJRF()PCf`Ws&~7p`}te|!#mL&YDtsyQHPKh?g+;sebgp0+9PO@}1-PtR^L z*nEn~LI4(eQvs%SZ*L_EAwG(%ZPl;l6(WD8Q_{|s?5xEIyIjJ>YL;uNF8Nz`ip}2D z1Y+sCUIz4-){XKCTB4|{!BTgw?0k5fU?o*=1uYfL5Ot92bwvt(^~lQaVXef((8udmD`Xc;gmUPd!tnsusDW292T{!Xbx35+@Y+!ejL^Nu5*W;ff zoJMtr#^R`QSAh3+*{IgyRrKQmKJc=y+33UdT;7^FOI4xTP3^uHi;!E(vAzSSCC<*W}M9#VJHvb z$QOX7St&p@wsc8m{g7(^`&4++>>)K7u8Go=`3nf;{MS1#5{3NB(rwna!r+^a`2^1k zr0b_DK*Z)aYHf3#*p7QmiO*>ICa*XVBQ!8^C#$EIs+IzX; zUb*r)r9{;ofqR#mL8#~R!@9U@xmj%^Tc}pAy}c&O`R=>mmlV zTpoDWM5dW@mA|tRpg*XuiV!mw&~uc*`+-}N?l0;&wkMc4LYgW_i;`_~Pq9rmGUs98_OX(KdDN|?Dfo@}^^?zT z%4Mn^cI*vxu^S#WhB+q3;XA34z( z#eoRJ%Ag0~VL-;>a3_Am;|&rOqjF#^`=R<{O#EPr+eCs_e3rO*2Ybr6_iXU@RwVFb zd*JOi1?S6CVQT<ZnUa+eSd@ zhpSnJVl}i6by`EqJ9OW%NRbmTIboRqOX#Gg6bTMRIn;NCF*=k(32M@}|o!CG`| zCfJz__IT!9q^!mDe#gy?ll8Rm7d4`D3Rd5*f3`eP&n;qZ)Lq^vXODl`b*92V@FVfE zYi()m;yihWTuzR9u`t5Taf_b3M~-@=5HubAf8BXR#aBz?)b|J&(z2XfTa%CcEH%Na^{M-#n~S2^P32poBihT}h#Re(%Nh?Iaz=Ex%Jm+K`S z5{x(saIwcYbi2t_I!H@O(&Scw5DNb_?y?WAZ^VBY~!IB$(aqts=t(r)-FR!OghLj+JT+wUB$6fg_>DHRgK zv)lT5OVHDlF}KkRsEajtpu{H0vRnJk3`hKReNK5YVV}~(!CFfRZF$*Reg+d0I(96; zamtFlEW13?1TF;IE6QY9;OV1mS=+^gt)Bv+hIhgelS-dchMq~MdO(p67BIv#G`_^&8P$#xbE&7e z>EmS`HZ-v#Vp*rdp3Em{0)H$lfwL=ZYImtUv?N-?7&+Wgs@c%ai<*_j`*0{6%qvbX ze!=)6N0r2{afX9qm69aZ(rp#uy$J9&pSil2Q3(ZG%xe5_`5pFE&Kpb&oFh`_!JUlW z7icYKr86H7cRb4M^F#gl_m@lEuOhsS;CtR}r+Z1_WnH!vC^P->K+R?9uPq<7Wj7dJ zPKKr@Bz5SO(Kh=r!d~B-qo%IoAX56q@+ntMPo+98jI~CQ3Ne0ND|k+3E~v-n|9Y|Z za6uSv1lrt_asl?*SN-q3|KpBV<}T0uo4sx~1Qnrq_F~0#P+*DJkCH(N!rmdRN9}s2 z`y-EZVBg8YQ2hY<-ofX!h(02jjXyH_2wPrSe*5;|b^~|8H7?CU!JRkNKls1;j2=6_ zW7QvHP@Y`;N$D9)jU+AAV-r4!vAG`IjE^wno3zuxCA!7>;;4egUz`8vW3P%0psgFr zFOqsB*p^7qNy40pX+i>+SH=_i*Lhpal$c_154>%`y@|FZ`brM*D;KtF?BQ|gQ&w*9 zJGzK-TBF9n1u(`MzL{z8uf;Tybwk&7ekx zIu@jb8>OTyelTTdH64Fbq=$9d%KX#9Bt3S1m?>=fdzQ(X9j`m>o~^71m7p(OT_49@_q4tj{EWgp}h?^j3eC% zmWASnvgv102n&i#lfM{RE;yXc^P-snmJ2umr`t7M4R2Hq2)}gei~@% zX6_iXk@&qaeFKQT`FDD4WHJ~|!+)9vcJw-I$sg0xXW#qVqJg-+oCQ*r=r_=Z4^DKHYrqtHD$YEBe~7?Aky^AUd6IOm1R@oNaGq~Qz8Xx7poao0;$*&gI{4w{0hIf zq4Ge#x1qd>cSfZS{SE$DGduq_Pzb{v8%OyIGy{vc?|vm@gYPzOn|yv&(TLSkDU`Pt zkTZL&MzTY{g8IlRy} zNu*{gkfLY#=ja?>nK|VK<}Y^Z8z;;5$eoMVMy1*~UWS&_1hT5j8ZLaam$~bbv82dx z;&p|Xgeqz0=X5H_Mt7*23r=KO&nZF*R zh*-12tCS{yV$L2c@Xsnv99*^CN{}h7Eq6Yz%UH7;oO-}+OCTMnV{jTElC)P7%Ar7h zH#lDU_QtkklbJmxMwh_HeU=VPa&lqyWjcPcU^;7;o_T3`Ysu&h+aFB6Y|*J0F2z7j zf(b(cS{w)8EkJf~k#FGF5*qhh=MpUrabb8HnWH}q`Bjs-E%5w>z#NcvF z_HbPi@`F6Cu0lT*rHz?M*l4uB^b3B#Vg>!+y(dLKVr68>&aMk?pzR5iujzGu85ZB0nbBN>1A^9pB`o>#mG}xTj)N zdC2woK4ax;>YcShfmAuwVxmk#lb4p_FCM(cee~RVZZe&a#-xM$Oz0WkR?K$&y>1qywY6AX(S|^ z=_Aa96XnZwmMLs@zw~vD^1J zNAToD;V~JqBf-r=Q`5}CDhr9;8`VsJGjL)j`tQoBi#m^gFOMvtf4ky;fgS(`=+Jd? zsXOZP?_F+0G}SX=bj^@qjBat=HnrtXWSu%WlfHn{kYSG`lw)aiu>nd<;2}bW=3m1m zn%7Aj&qs@Nn-hq--H{@*3Ej=! zp%A70Wed;bAP68Vz;d83ie^^)t-RRj{XG6HU)hi8%Ox`Tzf>}VepWK^PXG8VsGCDn z=tt4>kKLNuZ{74?imLxS$@qU{;s2M~=U;y5uMbh;%0q6}d6#+TRBp?G+m zT{8;a5 zzc+}#KY;wWrYrQS3uol>qw72T$fFf(W~p~a7*zJ_KmjPZGl$TR1#s#&Q++wB__d@3 z{557arY04m?$-*S{&%B@lr(;2A1>d-2q1~=aV4A1W8q18(OO}up8-FgH3sj^Jt}~UdlC5=(~;BE@WOY=|9oW;?Tb1xyLfld}I?7 zN@sX(Sz37n%-nDV8Ik5S_ZxHBS_w)vr$-((VSVA$#)L8--c0f2UM-mF2?$1OI%$L_ z*%T{#h|ircdU5weWh8qqb``I^#D#B)pi)5Rsd^XHbQg~%ZoMULTBp6R(kMG4`a_C8 z!hWOpC7lyxZAqM1r-G7TE+~6--fJjfQ+39_7n64W-L3CC-`wSvhR&Zyoz`UgXx@P? zvT4+?hhuht>}>L%0AMd?1d$)2R=B!btKg3=#v2HBJ`Y~e@HiEus19p_p*hthpqW4l zz(w@;E~poH3zBd)n_85I5w1Uryrwh^(7CtQbj;j4eo5i^`^cdS@XJ>PM?Y_Y${43D zX0zuY{`vR)v0GwW@IfhF+vjs00LI+V*8aB<8OLz&ayQ%l8Hbh*QPH^G;*Wduy51az0L3d;vQ4oMqW%%_rfwjTm<{aD775 z__Wo0Ov(%K-Q8^!71R(8dYWqe_^}I12#s?Re}Y(cw4w0-kMoI?E|30YJ~4ml|M`4k zIn(veQ&Ch(&h)*${ysWe>~{{oX(3*(1>_*Q4<^0myY~;>TtMX!w>gC8@YgEMGEzbX ztNO`&DQF0V`jYjH$pwl8i4%3JQzj0ZIUC*w4MaFldtN^nnn=(k|5gQEyMkZWo>K0= z)n04AVr$$KZlBhlTLmF|uJ71M2YJ>&^G7MHdr2&9 zxD5rpyvGFRxH*c`Gj%=5j5sTgbO+Wq@DCeN=YnVF-_S^u6E@BV8s?qpV#{BdUz;$T zIPNY4Zh`0#1{kSSlAGrLi?p{4i)-ojeOE#Z2#{bMJOsA}f(1{o;O_1Yjk^R05G1&3 zaHnyX;L^Bz2X}{Vnx^5h*1O-c_r7SVT*2M2=^Jb3T^tEj2bwd$|Q%yE)@LR@r@fS_IKPNByd$&5o?& zoT_8Udp1z@%BK|j{3ME57N#GHukhtbq#Tv zyb!A;ERv8w@a6p6L;ZNSg2xTwBt^Hq0Bq>+50rp@GN#3eZY3rwK){>|7~Mb5S?%MO zuAI-|$7X07X{#oq!;Ga73BP9aWv9SfpBK74R)(Xi4pr3@IRCt-R@g|A*oZ+gfIB+U zO`LqkNf8PD_a>|b`eDr3906;@I!(UxV;a$wPEme-8D5=YI|_|+nWX%jSbe4ikQ%x1 zqCyo>Z*ky$O~&+iG8<=cd2L+`?(VA(XcCRia$m7Lm%I=5Sy)(t?$SEctir;S9A#(o zt4$6ENadwz_Wy7MTCQn`%~EE9D0PH5@w+rGX{^7w3sq zlApZ2jHa`C!qB8zV6pOUR+nng0Q*<=ik-A?{vP7b@7b`XsjO-Ek8Jtq$E7LDj17&a ztnhxXG)c|mlWqsP)ygAUej&=xtv>Y+3>Ba-we>V~(Y5dcT<-s}BDl~hfRlXZiq{Yv zMk!$(A&}nG^J#Cb^6Pv)&iS!3V}_Pz?a8AL&D3X7N|Kpn+-|(##adms8K~lXh8CO0 zC?J~rW;>=QXA|u$THNL99XbzXw~Zf9nj@oXe3pwDM|O1O(Op?u`koBM_b&6W@=yO|V77hOBiRx3ZuRd`_9Di#4;v721$ftB zG1Hb6eRS93RA*>i$mV!MTx>P_hm0rr*PLur23XW&AC|*uu!<0BA>q0ikumkMc`*Rg z?<<>a`LU!}^T}hU2FU03Of~B}OIR1`LMlr49b?bh56Gt?S^1=wZ;#^!CY|fqM8xcj zKMM{SSSWEAcg@~NfT`#1pBj+0NJ6pSWV!`V_ahV&ogyPHhH@0%(1d>8A&x9gbVzOf z)?4y3pHwza1i-bTm#ko|%-EdBqPyR$iwu9j6Yy)r1h^>W30hb{`%Wozg4A04wu)K& zoI$%?FHejwD-N4IXO(w?5tM3M%9_h`QUp&9$v1~ueXXoh`}$en)t)Gfol&Q}lCuv5 zqqk-=9Zqc-{3q?{{5%yF*SYV^JzO3S+VC7CB*Pu!NOBzTg%8i$ZS9y{zDl?Ks`urI zS_>V1+^(|UEi(HeGhO5DanUO7hm>HOxhjctb=I!3t3OpgHYb%z4Oo`E%Kz}gp%8b| zhgx$ffm*uK;$fzIbtv7So#|BD=p!fEZmZpy(e$3*`tS%nU6AuawnR5!FAqoMZo-nj z(ZbAa)`B(OazCYK=j(~d^io0Zs6!$9!w0jVEL-h4?7C%5f%7eR({k@5++nwO?VyJj)E5^*$Kr*f za}ON&5(PbVG{?eGY03$M;;H^hCpZZ(ouo01_y4TAk{>^o4l$>Dh&ko9vTx#ov}V1~ z+|l9R{>o@6r3atzmnDr4yWo&PPc7&`=ab|p8HxoBh#@|*+Q8jmK{wO)O z!bMyj_q3Jn7kR2c+I>-pR0?9cBXc|shU8;bVMwh%GDWvY?4;SBsKdna<*L~LLSr0J z;S;BAvv4{{KMTUaILNus>+gZ;K&~3_-8!y`K_`;AW(SN7wnRRw9yzK&LHgNrTyJi1 zAR}O>*CPhDhIt2r`>*Gb0tnX{gUx8a{$?aIU=Xc|UhgV0(-J_w7S`DgU7}QnJA-b) zt^ugyRqED{%21|6f*F2~Ka`C&LwzLC4H9Qp-SQ_YM}mTm#lRwEp8Tr1jKo{6f)r{1 zRmuu0{a#}Nn>s4?^FNch>pDby&YU`#eh!!^t1_%K^-|nt&mr^?oP`>jn1PzHiqatE~-OQR6653?F7Ray681R2~_Zm z&#gxPm`Y5?^Z{9j8FAXXAfS%XL)Ol5tLG${#PtAi`qBBOcKUfROq+mFN)#>$e(y#Q zpQarkI`YCk47QJONE)8Z{=PhqYp*fT7xAJYs!9yWMh7L|Y8h?*5U%JuF?3-#_`fPjIdT(;bXNhoKlG6d$B@4q>bPXHBIM|wz~&q+FooX%Gc&@wP7PlFQtfPi2M+z-^W8o?%CDD1XB#`+|v9`o%GRE5yU;&(+RCP~H&@+fz-2N(7oa{r# z!Cvjp@xAa^fd2ssdE972Jx;}*>)C0wWP4AihswBMLJJoYt8cAKhfBK26x^Fxug-e8 z>4JKQ7Q4T?<{5NX4jW z60|e*Y|OObbl7z()BNp4wf(3rfZVF5qa2Z_^p@wKeZ^9}!p#0S*bGpr|N3r-K+ zU2mVnF=2K*FpmpAwCQ_=AWe>30I!W^F07iAUE=hz{eF{xPBvx`x-}}W^R-Hsmb-5B zS&BbtX<+tp-!Ogl=)XMUoHqXWbqMKwv+l=D3iV7AbmVy~08cH}Jm~<#U!)pk;XfQy zkfUuccD$meWRX4+X9&OGGH0dT(|DVSf8ra26W^NRJkII~3l=_arC99`)FO8QkZo#q z1pL5>sKep7;0h@?UKHc3*p(V>_`qy^`y=94+@&tLr}_JuQaTh2DAIpXfhETQaBuIW z(D%qB8^GAZP8qO+qMi%4bETjcrbWnFXt5Wp-j8&ekzwYFrtnQZPiJyXNK&0(qZZMa z*8bYAKjUd(A~?FRxi_&pOhfxg;j@aKgy#d)8jc1mBGg#Xy9PT4E@6~joy^CgC_53+ z7B^SKVzfUi4b(u01ICiMvadR>1V2Bm3mE+(hecsuxT?Lj-7L+iZ1EcW`d8v`AE)n? zVfX__V8O9xRr_E^WkIl;ZS|o6I|(blVkuNVa$Zz)x)06vVe6ClE!@U30bJYPze3RM zD3ZFCuuvQ3G+Czz25lYFm}W{-LNksFZrKX_Ny1KENIN?Z4h`iLx9FYBkuW4i*9-4% z?R7peP17SP0~c0=hf4qiR>~bOF7o#;JcqfmJQf%JtnbUZY88x)Yif^4KYrq66C#4y z*FgUa1Rp>U zlUbMU?q-pfMcieJqL-3GD|zrsP1g64Vr}oPbiVit`bJ`SUMX(43$1AA^g*^-fU>kQ z_GMF)tiudmP6;9^oGXAwaPUR6i@x{ysjaZXcVCK*k!&dI`?Dyd){ZyJizeKt9CWcy z>drf=K(A?xOGRr|!oEiKx z+%w~I&5loLtK`j%Cdag^Qd5l)W*Xvp%2eC)4p(b7{=Ng?JXau>r;}Az$_#6bI1s}fN z-wdo>ttcdMC;^v#p^``28vEFC4SKxA;r2#GH|{~~rmWIi9|><2M?4lTKP`669=Ovl zp_Y@ONE5&7nP~y!k+T4N#aW=1Mjf^R)qNaqD<`5UYfgfUwR`O4ct)%SI&u1ADD!tK zH?#bX)kIa*5(l`;0AxAfV9ICsVCC%M;`elW#PG$^J-;p!F9{MQsoB#0@A2%%zSKZc z13sL1Bs><)T7)mDV`C#X;_Wvod@Fq3ETKM|g}ktQ*PhLt+XVnwpyo}v>bR__2#=~7 z=Q9s-B3^iY&P+}&O1@w?HTW@YY>Wso^e`-=HV7_8h5c|Sx;T=$xb30k;xx^A-#(N) z_$s?P+W^8BCY$n8&g!F-xJI+Gp=xJRKfX87m?R!R&VtEK57*7DKTo=KIlb8C+~_>b zVP?_btPc_3>(2redht#WN-5dkq*v#?&_L4E9|*IhwF4;g7GY3+R1`1Dv3)77eqFe- z;*@q(dAwRELOi8kCv3Gx{$-rXbGr9$;FY$`<<1zoG3DaAi;E~za{&AM6ua< z$!0^VR&!dB?0K%xe(QS<-GA*9c%B}r^q-JQc~oA z6R!Np$-;2ZKCMdUx3VT5#ydj6UIlXP?)>}1I*Pcn?ywgJSBFMYoU1PGGM~IXr{L`$ zSmN^NvaoUW^h}71XCt-)c_}?g&4A^sn92 z6}6uTJ-NroJ){=ct)Q7~j}*<$o;o%w>X|w=d6T!r|7^(WS$QSa_VSNx!~nLeIICX16#7vAh|ABBwb9F<$WwEh1Fm? zG>HlJ3rc#J>rrW#R!`Hptt;9u0r%_6eN`h+Yf)Xz?e7~!U%n5S$6-; zcZ2PR+yF1Ksg2XSzRQp_V4*tDm`hjptITSGcfz0=PzmC+7YEBr%sl2AI4f6`&evy# zXZBB|w~f<;yx4Xgn8TG@m4`hR1C&vWVqCF*`P-C!jEYOpkWiDu;&QS+N|2b58b>3o zpFOJ=2(@T%byaORKLlh7!Z1w3I>zA`Jyfa?8Qa z^-cdcxb%q;`}g8u!fb1xpK||ex-eRUK&M460~oOmG9EDxk7dQMEIKw+f1<4k3Ef;# zvy$>zm)Oi=cW_?(8IL{diwL_)q$8-XUMcLaU7r*8P&=|fcKGNe#7rUUCoBoXHa6Y` z9`QHjy7jY~vNx{a2gt$fg@m*MtyoQ&Sh7R7X{#NMKYnQ2GWn_7vG1(q?@T%tY*3AI zhz)y-ha1(=HIl|17Uo3ei)Mi*v>0_;LtEb7c3x$lSNf$gtT0R>U2DXE^lMdZ2zDY> zmI{@Y5`0LMyxY2TK`gMriS9)@#CS3i9&hhaRT``v8g4 z^^0F{lEj1gnF}I_MoD{0DU>knQLg?ppFLMAg&&El$H*Jvj6}&eN+V<3qZ4r!b;cei z&zHvFCP$b~m>^>iN>x7J|3GMz8d~ZgpFfr~4!1$T%sJ^l_jfGTvK3Wya*& zMZW9KQ(T{8#|!6aLbP25$8sHEz-bGfPs{0tcBsHgDeYyr9;wJdh`&Q&rATDl_D|c%aGt3?m_r+x*Y0#?Udm*iJWm0;Gj^w ze5L*T+(~3wQKnWZotAMHyh5u@`bwwi8}Ge(o%^5{rnDid#X~m&Ub}fH(o6%>pI_M* zU%B%zM}EE}un{n4HWuGd6(Cz~^4Ow&E(l;uj8OKyG8(~Q5N{S_s%NTbAGt2uxXN)x z;n=9^uQUI2D^ye%OR7ghw|IDCE`NP|zIi|0^9pv7TUd(r6-3ddr^Y$QIV2=i;+Hs= zN7ASj)+_J6Ep0|nG4;&HoZ4wRj;$>N90T=V{fn$YB?Ukc*eHjoVY}*mS33V4cwnt# zTE?0AxIt6}7VgP~2{I)|4BHW6Zu%B~a{YePqV!tvtu&HHg?&ywVJY$A)@?z-tgCIY ziEGGrS*m%ZK}D;E`E~LVGmHX~*SyR4A3nZQGyNs2?>qR*;ae7&U8a9RF27Yn#zLQO zMi61vGq&Za*Rl-~6-iNwiHq+S@Or3{)0Gs%%T!jcn@#mYq}IcjwLWI25LI$!KZN`H zp4G4A##|ulb%#F`_5CDb4`Y^$q?&<-geT~=wF;&(a=fq@9-ep43?GWU)M|4ekNpDV z%v^>bDaZ?EwjRs)4l()vhJ)m&wJS};zF{&Za;WTF&GF7~M+aFg?MZkV>cx!py`U=; z>xEor%BWTWl%+nlt|5P@_<~m$B&<64%7ZjL2XK9y;8n+}`}D|ZX-iw*SLmtVOPPav z2AqpYIIA^E4e%fqoCe!4<5ItLnKH*n?Xft;uhQcNziLJ&Od;XlW^o*33>jM2mgG7D z#9d_?qxBa-qa&Ip90XB7vBuO8{XC9*L~BR%w{L8VkV+KUSg$$IQk!=?*wdQTF`u zsbf`%tTQ6si=JZ|iyT}Z0(yHrK3I5(o}&x-f#~*Ky<`L``ZC5r1N<()2*4;6<3#Mi zOlDi{=IdE6*t}A#%sl}}WMjA1!S89cv-x^nbMIud(AH-ogR46GCj&@A%Au1KtxC(fSy^p!gcATVa z6;CGO>jLav-|4l$H%lNx0R)77NjOR;M4Pt9>&IF6+Tbq|vVz-0$o%_cux2+ab3m{* zLe4#YdEn9J|C#vJH&MAHfB@AU`;cSqNU+Z~!~_l5eIKcz(*S)DeGG2y=izMnNfHeo zYo+Qj=$+Npk!iAs<)*y%=7WOGfD4=qIy#qwoyU1Jb^t(~k?V3-C^dDSV!X@qz= zQDY{74Hxg|pB~HbdbLCmI6_4PAAf)4@ zPlDN)6wND>^?m(;q_X{4K&Wem3G2_=%o(*Y(Gr zZO1lkksB*`(X3vKJF8p!3yqxIvxgOIm(PH$w)18Eq_=bHpa&){{=eFwHMalpJ(Lw^ zp3ygu%*n~gi&T96G=Oqut$GDDm?_3T?_~cF+J-X2M(Lm%>`Ka0mbb&*2*+IVFZNpR zF7vLkg5%3X(mr?RDO78<2MOECugPb6J(->Wj6#qzgZ?S z_X~LF@0)UZ>(s$l>5mBWllj}@3nU5dKvey?{meXr@yz_gq9FR|)mUy%U~n1*KVn)+l`xM0?=awY_`M>>*TI^_k2)RHOKgDp9~<{iAAuk$V<8{koGBFRe>VPy z=NstwufG4~2meRo)2>fckB{;1tAF1!%y^4g|Nn0M%r^ETSo_~U`Bwn&@#>vLuSmuJ zy83swksQW3>i-IO|Cf9HMAeG_GRgm?@!4kNKhN^t`wMiG`1@691bB6uiipx0(e2Bx z{3VI^^cCiL-(+`%ftg)UMTH%xrdq-mZ73gSXqY(to^Qd>6>aXzf9~+)@F=SPEse4J1RXZH0xH3_K994S{p}Jp( zUe>m1)H-cTs{^BZ5AKwJGd3;&U8t?qsc~5*OILcV>hm2ja~p80vW(J> zkq~n0jpyj?JVSH#_shG}%dH*5kTeC2b@{9Zh=yv4E}q$TEk{}2bLo~36TKwWf zJTj!(ui0@>Vtcc#pt)m4pP(o@I{(na!GEzgm8le>>>aB_4WOZK1jJu6E#lRkg!;J0 zvh2F~{f3z|zt%sj>*p}$wMp*76Bt5wO`ofbs)k>JL-$7GjzuG;d}zv7Eow}sGt05d z&BmF)HK$W*Frleuxa7X!{G`}tOGUeu(=Lh{%Cd6E&mHP-;pbOLDMph6X}5W87p-=> zN9>GT)AlGXVf`cN?T*f~mC6rwko<#R%Qx3Jg^nA207ULrN?bgR`wWBfOy-9^MZTdm z7{9*Y(tDk9!Z7qNGR@k>FYa#(7KbRXmxY z^yo{sv<|1Q4NRPN)F3i1)OFPNMq3Ug>}FEl)baPV9C(K@tv922=%41uHn#sZQ@^#Y z7W&c1k9=WZ(&CS`ZyLI6Zy(5MCK-P=-rsPS$PpHgaEV;#4l9r^56-OjuZUXAAqB3Z z<2kKVpOqtNgu{5pnJ~=w?VXwfuGwED@;lo%r~f<;Xj&|drFKSkk>JSGK_*dOz=97h z!Hp-^K&-SeZfn-;-QFyPn*&|F&d9`T3aC2s@qkF;hrsNL zi`nZECyGU%IDgGLK|P!AdU`TmyFCpp+0QnAYQ3#@n+B~v2(`Q_%B4C&T)eVMrYRgX z7!CZUt*i{3E^HlRG{h3V?UEJ%QtV7;8pNtj1U#&8F=x+8Xe%4}Zz_6qR_<7bJJ(9e zNyUyAIqZmFu65nVIH#ZyC8o5@Jj%+}o%HHrF3uZ4IE(Il^>n9M30b02b{vI3NU@%1 z6C(E15qZRvpAG6CDEm2rX3->&|I}@o=VJ2|*jK;x&KpinTmOqJyGgA)T+#7F6&s88 z#Vy0I#dtAx&(T~Nmux9 zwdg8_rX&@*ttSyD<%q$c-sVHCKeOVsd1ybad~b=#630oC#Q9>pA9oonE%Aj@YI1Wq z;01TK#3_A~7RNo?+zVyw)RG#qOqL#d^Ec|L*%w{A171f5vg$%+u6CBDNasg+yxC7J zL&zmco5s$qZOe)3P!tEr>weaA-GT2@9W)+IeZ9AN6l`3YHgc3=|Gu(AkNEOsJ1pam z%J1D%pYC}n3&|no{864H0?3YGrgJ!^Yx}*1*2B47-<03VccicA5~A_>koQ|CYEq(* zllpS|(WNIa>xTFTCt5JgiYSMY_|NZ(+~cdGDZ;B?w*&P@?fopN^B7) zol1-H1_L;}b~+Gx@#wI)-EcxXg?VBljd|R+O5VOkx4^_W!JG)Zv!=V}4?Z?Hbjs6` z_qWitj>D%7wHsj$hnVh)cnNILVfCRNyQA&Y7H09DP5_XPM;ff;@&|m{tm4M+Lbu`n z4>tpjxFreyKY%mQZ@joZNM9arORJrHE1#ZXP;K6gU67s(#XXt|$Z+t~+JMTa?W@DTn=H zkC`REmp5?m+vKtCw328REX&JFMh5b}vk;`FZEUPcs={S;v~bHF8m}zH*YvZZmDTXE z2ApEDk+PD6JC%5<=!M+h8kZy-gb~L-9k^0FF1cacit{C4tmk;_4Ke1B+_d=o&*DHP zwJr0jMI~}BGU6phC$FVYn6CMk<5_V~RI;wUs_zxm`q6%{oHsTdHue6J7vEyRsK%=B z>Kk3VR@DQXO4a!6RH?#@kIK>2m)CIK$-SO7L~Zq?p0V@;WmEsk+40?Spql``f}uO{ zvcYm!&X#Iiv`_b3wDKOt54vG@!!@cmD-OD+etT&N3grAr1~a3+)BnIbtZXEco#_2< z|2vZN`-#iaGd3Q3TrP$dUxw|&z#TAc{`xx#x?~uurK_GJjj7VJDFj-++Mq?^8tBcO zto}7U{3}=0?&)c)F28N_$R}wCoyjqvr^aFMAnYU|W<+;+U-W=}a|RGPXtmehFC0=p z`R)g)+B#9ocluCLY%&Uu4JzJoL%%>eGrFjhwuhqs4&(s;FF+1J?=RWE%N8wS=%8do5AaiM7zo>a{TJ}yc}f5din#Y`+@1B{CA!xdUi?u9%e}p{0;e@!h67PvD&5@SH)O zwpN*m=9tMn`h07(gR_3bcryvP(B}92(!6<6)gyMw9Mx8@W5&E2<$by{CU>aSP9H)< zo|&+#M_zXw!skHM>37F1kX|m;gnWC_d`wID_2tGwj6$V=<#hOGWGPC)|3Rh0`I0dg zRi`8Ptkz+ac+PVAfW{*+t&s=JNg0`VXs+aL%FheD{E@EM8(X^_lsEObk~l#ek4ZkP z>2s`fz`>{`KJRYxL-CWv?@n+K#isf^Y1H8DsRmf>O{3ubr^Z(OElW4BpiN`nqmq8= zcqrGtZ6G)v*47hZE7I-R|M8^hRA0-Fq}g*(pqqACNkj4T>@T{H!&@s641SW7j5s2I zfekmr_W0m+C+|`@F^I;7plA>P)wvPwjlU4X7)8PUS8#~@dORHoTOIzC>9aU8~beVZQ zMSmh^O5Q2izC>(e!&IIRmmq=d4^tuDr-#9qwo)oV{B1yx3JzC6h0zp#zy+tHptEq?Zc99)m@O zufNWewgE2l==^$>TO%@B6nO+b8$U}}zSNx2VCcs2jYq}gUTEopeCfj!H4MkYAw}X+ zNJUx;BaO|VAI0q*Y^d{+!mv9$11GjJZP0tlb@DggR5S>-XMHVZFSJrma-#BDcTe*> zG6etcZpPsu`B|c;aj#puAB^+p!=)QQz1@((uj0n4}ckAvW zKRA>~d*3keVVp5@MR%JY`~70m#?C9d3cHpsg$2u@Q$C>+dJ1|$ckV!W^G`w5qBYjL z#@5&{Bc*5hW>w%)i)}p)F#|zm39g%bC)4Lz%S*;($+qS7Zuz1r)IOl_#yjuc?jCJ4A9f|)96rY{L#dw)QkA9Z zx+mc1`v&{_CCGjqH1F_tf8FymNDVj=cK-gj9dL7e`q2HA7s%WO!i2cQE~SOIxY_$= z&Kvj#u{!hbRetJgn6DSMnbsktXJhr7wjO@{RJCAhS3_A-F=(`IKj`3BpV#fEKpijf zK(24w!)^MJY%Pub31_qTo+C|Y?XI>XI=ZNEv9=&@wg5&5e>egv)UY)p-er zR5LWmx}DML2%pypeC`+2J& z!COj;`u_t?MVbbVb@ME!3G{l2BVfuOr`vXJY4c2h^~jXM{B%D|)Ri}}1vrv_OFX0z zk?UslLHaJ>3w1NpDRIx*p14iY<_kU&t?#zJzXJ6dEPx~=!rQ4HyrgCoynK-=A66de zO0h>$M}G#FX6uV)ss^Z>6fK;lO$q%Q&w?BufhgjcAoj(N&LII zN?T@-&pD|(2lR|#&mv)1%gUQXLgCjSap4TP(Q@*0T02LJi%$cA*J|NsqEK7m$9>CY z4iC#?EKmy}YfT}s#e5|FKE_T5{-8$LELQEdZxSS1hb<~N@6?}M&V_1K562oF z*A>3_t)tsx$fS=R0%pb?5_?To&mWKnMw)k2SsoYYSw2UUJ2R(SAf4!3Oj!F1O?ECy z!tA-%^5(C#bPv9WZs+iLAVwQKQQL6d3d1~=JXr3l0t9g*VEcAfzbx)$3vtU=FPgft zl$jX}Qe$q1$FiT}p$e)|_tYyLNe-=iKz+_C*Y7k-p}Yv#`+4yOnq%W@2WFdbTFMg3 zQ9jWIy?3$G5?UpRGYdXbgajpLVhmuUNor>wu#6u8rF@f-Xz zq8N7w?bG(ZEVN*Kcc;0W94j{hA4uan-AvcR-L-Va zq-yg|2N2vF42f4(C*$ZR#MHrU zlt0!EO7jm{fFF4Lz06nnGdIC`hEydn|p3xWdrl-c>Dfn4^2xzt8M&b_02G><`|0?T1%8qj!NJW8v15c zJwgOi8k<}$C@jrjTse11_63L1HSiMQ^LsFt(h2`r?e%s?C53-%<38wAp<=W--+RH z45gz1f~5#_RA7ypy*6?$79yXi`!`-D)a)aAm-dl)#QMH=tVP94U04ylY)>^EmmapzMzS$JehHs;{b~S_ zbH{7>J(zd%$1ip;MdVPNpXRhiRE}Zu@o$IZ{2xd!1t7Hsq0htOmk9bt*y}ZXU|}El zuDf`9+M!^Nx>X4Ejc0A8ujJC{WmZiz+^t}G8z5ioC(6lXV4{h{Z8s)=a*==D+lBJ+ zZTV|9l7<=QGGTR{tJ{qhUCEv^Lr@$h*-=^>bvI)=6+!G8Kd+={%qp#v0A}}Q|I!)0 zm+x>&;*+w{m*MR;6V@C{4WCzfP=;IDsB>(=Mw%(u1hPUAv% zN5#mo6UPbPCdi_ZFj{LL7SBh}p4yfd5qXo6-Rjo$KHEjNEQQdY;xJ5}Ol!)sR5rC@ zJ8q{D9*|raD=a;l#zc`0-sN6;@r(ELgFmBkpGuoKAGWD}uI>;0Wp#^GmR{iJ`>aQR zT7Ct&E-tGiZxalZ^_C~z)mcTyOrKd+DxrN#xbdZ3U}8$ANbQHZpZp+|TL0DV1czip~CM}Xw z0NG*vtkEs6W>*8QVkPYC#KgyoD!I0U)CaBdVN_@nfsP5gDzzb$YrOd z%GvT~-TY^KGM!xVxJu=DFQoqkj)Fxggwgf5xA&I#`sMI)fiXTJNS}IMtOITfqPSe) z=-ikzBiFs(z-lPqJlL|eO1MSCyDodPdA~NL`IunR&xycFiV)zov0|SF|5Wm~y83TZ zJsn#rcBJpeC%~3C7Vq|rc8DLRacgh*Sr?ldrpmtQ;j{(vi}DX z&2!?zu*^Rs8kIm4mvTo`oGLc?CpkZ;#m)A|I*s{3tjSpPD6bEkTu`NI+Oxu!i+^Ir zY*_Czk6%Vro%SRnE|JfltvL!qDu)Um{u4y2lme3yiIuc#;%~xoH9NHv@SF28R4Ceh z<8^ukLhny_+Rn_-DzROuk9n;+14D~Zo#}Uo7>xjHI zk<8aoxJBxZXeD!poe(l&Tns`;BoPHzJ;FEC5VlJXJ~`mYC$%Dnql}zlm+fM-`OAe% z|E*~MBe`*uDKFlvlMES5z(^99h9M8~4?UJj)JS}=l*B7!Uj;~qhwm$v#Arv*mnOVQ zc}tHY9?4OzYX^`VweO)~&ZVo-OM53>tbwMr|9ppq94mfukUshH6J_K4mx~sR0_$Py zv;H|&`l{@nC2#U=x=x>PVF6V3N~@uhg6X}c(Y{YTvXQkDK5%zXQsfC+-1RtV-t{n- zDL?8PAnf6uETf2DHgl&TP-9#XTs^;V`_;mkUBk!I)+TV~a++J?W7ugvePBZkRbqo7 zA7g?Pmc%NEJ+o$i#GwHKlAXQlu5g$2(75{nrFWtEYocFv`t%y|b5e*oVjC&5IiIR- zP=JWS!$1;=J8_H8V>=4mtz_R!Z$c@1xkH1HPW|*Xey@_s!OHmeKdszcQ5SYsqVxPo zJ;^C8HE(S7L79ImAs zI>bqznTYWWyrMPuK;WFT>EVn4hBqZqN3;a!yhVskGvDc8-aNx)o$xHhyAX0rI1IR*UxVdT8L9~n85h5;*zkSxdRdV`)$lIL=m1c@w7 zo{06$Noh|rl36wDHXolex3b)F5ewveRmYC@H>vwywE=NH&Hh81O6 z5YB=qFz$Roja*yhcX?U}Kb57>u*gsfit$^M|J4d6q0Y2wQ9tghmY z>1Qn*cp45(;wiigtE{Xhv4l~2_Pqg6D|zsR)l8B`B{BuXR*T7!uLr40U)f;iZI>fbmeHe?1QMYs7EcWxy*K z6qbYL7(aRPl|i9UdkO(J(BJ9WEk-lWTgqKtH4$Y7-CDu;C3iHas z!ULaTOu1iz+mjviTwaz1Roa)#eT3Oswc`5ir5}g(zu5F5`L){>xrEVE$V&(l;P!(r zn`&UVB`|Bp_CW)l_+fpIq`b`J6XiXgslj^W!;!=4 zcnfN|B|?hN6a8?H|7LBcKoAi!0Ny$|U6YoCKbOx>xwTH{_CYFfFaV>5^msF*D~M=V z(ZRtu7d>J1RphIp=7-qj~)N)3>c71<5Aele5pwD)Cjq(+G_df|An6^E3RZ`JH6MD;>MzW|A zUA6C#etXiqB{#tdsN5=y)Ab0GN%m6lbK(nLU4lMZ&}Q;|tr`Bl>iw)lvCwCDNghm% zy{$x*Uv#*w7B$X(wuEUu5PqD#2(j(@uzo}_b;?udy-caq${}ZCanXEcXmI5HG3ioH zKDSpWX|#rHBppv$?9TcGN*XHa$ykCq?p6JJg>42aHxCvbr4RZF-%#pskI;QIZCH^o z(QBa6C>FmFy3gxS`iQq5O1ucSsEGCsL)F+$SC|A{%0QXf4pZNRDrKXvsdl1pSMD}TWbbn4>DWME9Gq<$=F9Ls6&B$b_(;IX!3l-xBA!c+OM zv3)y6!?=K1LY=C2>f2j#Pn52luv6B<_ao7rxr@#nXIVZmsz$itd;eT$OpH|Vu<|D# zfzXyOCV~aGT-81jmuin;Ef?>WgI{&7vqE1yVxOYV!f`6;0Y@PN(w)N2hu^GnB%sw1K;S6w!CC zIQnwVl)D(MXXvrZlikfDsXJo6A>~lJ0fuQJSP~4_uyieqEAI5l^rU;gE^AdmySc-Z z+7``NiSE9NKB_#l$As!x81KVu0=<>&Z{UID{Kv$J##mWOMp^0 zI-w`0Vx}DWX6e1;?1k23Z9A$|0|iH;;6l4To1fRx`h9A`scFSoOc!g!N3d{y7v6DF zbi#>Jz1}V`Hfk|P!gDre-qsvO;^3#rNbg^y_q2&$hDC?bdPPzEWbk!T581-)DowXg z^1m{X$KQWMFYG>rGE*<*{Gci^6ZD3-&F&1r=NcfM%qb*&s>>qSw2f~+Lb6t<=osEz zxrqU;yK+R+*vb_rXF zIa#Dbf!j=)+`^9%lX6s6rHMo_W^z=z`8e7;I7ixxMg4G^yKY>~CQEXq=~YwVj%XaA zPrYV>UyGactOd{w$irk#C}H^;Q!Ebx76q`~o}-gfw8VK0tB%5Y$mC;k^XuhW%7kl* zVWmflvZt{vwGd}{1SF?M)I26-?riGcaal;0Z*M+-Hj}v{>6R&VpEYMZw%fSLqTm9s zxx{kG=X0eyJ^CbCAFvMH^IN%svEQO(&sxKmLX|rUNoC;4(%{-A;k_@pDklID5!A06 z{FhO?N+5gTJTjbZgD!7%flI&$l~)MAUCpZBO_v7#6%I|}p=e1)Vm%Mqz4~--K_Ip{ zvlavLr+|DmnB6A&cJg@f;$QJuQ{nt7pU8kU7Z){vaxTXEkB*dKj_hJzzp`5x3l7m; zS|#~|2eSy46Q^KCEg7A9Zf0NnK`%sidjikJF|e>^QzQST%9i+(S}B_2EqDK)+Ri$x ztz}=>r!6f|pjdG$?ocRF2vVRFFGY&GySo-BP>Q=lad+1gDDLhKMS}zh0TOQ5`^ets z>pu71=X-Ac%&e8kGc(UxGxK|Y?`VvG?xGriLs`De>j$L$jGyZuD?vnx6s{0&VF_(GO+ra~JIbsBha40G{n)f!BT(Tvn@x&-m4>U?dK)8gPi)Yu;W1$uH3_~jmT{3Rav=P&;cX~ugz@0}}xJVjcSem323 z5b?+VbLCRi*RA48LumWoqmWt<$(W~>0OZnAYlWgIuIZ1vK91JO4PJpqW-ohz3k85? zemeKp`w>r+7RO2f#IiEMN53~n2S(&vj>?P1r&G9exWt^O04djvCvQ@)-l%*QQO#eC&$qFEYKRDVcr&Y2$L*y2ixEefmhnpT=W8O*F48vGFFACyvY7i{XXHJ%5^LC?FS%8_J~{$RsScc5BL zZ?=pHJ)+Va0o{HA^*Ru>&X|Xr{&1LK3#E69!QDp$oyiQ3cXTnwQ)w52@$;b4YAvHi z{rLcOGPS`BRUPBiyb@A8XG*jdi$#0!YSF7i`iSZD(3hbc&B5_*Rc@+CZ+Y9lG;L6=DW(-7A&C^(Q3UQX_zf*Td=j6S8L}j2XH3+}QBp zzvGrBD-w2w)Q8_0R*Ns`AlMw5KAz{XhtFYydeCQ{R%5gE~$=U5toQ#JFQ zb_5>+(idV~<)ufna_fndf_E)uv21*{JeP!Sp6>0Wo}OAzpM57{!z}u=d>(##)goih z%Ga~f7A(bT)=O`LS3;$n28b662zi7q4lUL~)xnBX?XX!{cFqNsn~rdAaxZ3%lm?Iv zzGC$dK_mbTh%-uqx|LxIva8=AyvT~TG9@y~ga_$nmOzpqR32 zcksH^3i2M-oGP~$P!a7JPPiN!9&V?nKj09!|0t%XG%g>w`%h3rqy7^63f_N zSsb=k1TSYRzD@>@D0p&Lw*?h?{j6k}@VlQmnvJJ(#g|)DKlAQx4=9SeWcqkw{dUez zpwvNGS1^RzZP+E(LEV$xU{3<4&SP{8$M`Z(*rPPpft4iGFbTI{N$** z#5!(mO57^7;*>wy2*;G46mlj%^=U4}TgBwJ1ounIP|KxqO534_elOU^IzH;_L@eqN z$LKUOL{0`QlpQOj0A19BB z7<@D^Y48s#eF2+oi8<%uV)_L~tQdNG8EX4JSe&Zv>M%!ByJ>{ifbd;aqGH9{gwf;| z@^W?e0hm77p&dkM6-|!;{lG0X<3&{ zX6gG_RQka1n{xfw!`Q+RZxQpMELQfI@e)p6`aIWsHrIbCnUdZvMoXN_CoF|#?A+ol^YjfT0 zWeqUXUV9ntwv|Ub`oBBtLwqRUrDL-{oF!RV1t!V{Q2hsa6>Qkpb?&GSE9r zQ`_-1)j`%>MVWsi=(M+K`GQKK#uFsnBG`>9?-QwMtI?IhZK?J<@8@I~y1KOJe&W|M z=XKy=qf-~>uJ0p>*Ytd5Uo%35jrnQ><{Tq zrxy_z(5-}q6#Yr*fr91iO;mE=av)lB;c-Sy&G7~#vaEwi+q7U`@91`!U2Sb|D7%H+ z$U6+pdV=)mdIh%hoZGMZ+={8LLD@qabKty0l5M8L$_G5V1gJ|hG#$LoA)~pGB)@{O zs~toQSLs_jIkDIxODlK?KL)?*IKpetr3DQm#r3PV)O|M;aEH)n@c*oYM&HcUy6mC& zNQN@PicuxoS(cn|mA(S?7VTm14rO8U9~4Pb5kn^}%^p`c z!Ie`k;+@mo#%d`oX4sPf0l|4a)uN=637sZ##@UGnl}_CQs5hu=T`}64k1oLi-eylz8Xg{hGVfCS=#CMy zV562ol=vlJG)}*$SSr8>ps$v~m4(J@8m!MqCGfc@78aD3O{^Zz^S&#lq=4S#y-in) z?V9Y87afL7Q{cxqRi0-u=`n3uY>E0I%ozM)iam$@Hi8OuVoJ{5ubAA^__@k}n?jg!<1x%S`V$D+JnXZWqj8xP#JRuhDc>HxUQvi!I-<5&!F+U5R8`SV%Ly(0P zJsxCVJzJ@qb=uK0@2TcsUGYOjED;UK$mS! zb@wBYbf|d7aSP|@!GqV%{Q(L4?KKwAQd!7{oZ}-XporbH8*(6KAlk>e1ndkf#avz2 zD^h1B;(R(tubZb>vgKQrTzf+4106!Ug^rG%rBc|^%`B!dyv5*dnpZ|ArzHFqo0b#I zaH>88V)nOFw1nVc7%!JO+V}DO=$Fao#|?y9EEKZE0pLuagT=N&FT)VDU8?5`=TO0^+3;GoanaBvcK)WO0awyD z)%M-P3P&cIi%tPZG7Xq{&}UgK8T{@?Yg-#eL9Yk_G$k%<`9Jk2@Wt0YKDjCOc=t4g z@_Ij-v$%tvHBuWTL|=~7{qFb{2J2FWvTSACB(9g3mjAU5t>^(zL;0`Jy|cOupI4aH z5j|B1`x+2V2bi=9Nd?H&%LGWB0(wg!G(NJV9ccf@q5 z2}zC{yiCm8o(Fo%mRl)_!9Gw7Tz{PWfcyjJRaN|u5>eMm#;!<6rZQr;tB%_7)<##b z{#cG=A`_8mdO$2CSWJPlnO{Q4s|`7Ln2>R-=|OJ04uvKSx3CIi^7!Xdrl)JxS#M&r z4}re5Y&>7|Soq4**6sIDn|Wf&(WJLG7rTS?K-4}io|R9+esDouiw|b&V#0mVCvply zRDd#RTJGu)+qd=j54E01?c`;vrSeNv#7VxIFVL z6SIuYZLUzm1%HMI0S@ldl9UGP%mhb+r4}Tne{EcY5B=~8AG6;>gF~jfDz79E0!CZ` z=$zP4(Q5x{J-ImKRryPja%rV1&Sgj?PWoBTS7LmfF8ogQ9U7WRA&GZgXWNI-T z!61?HP%D{*5$D$lb*8aV67S2@P5bK6s2vs<$XW%iX4*Y0erE@fF;jZ6`-|gp7B8dX z(q%4_1zUkT4`oxj(c4rb|C$D@aAOqf)EDIymk{0vFUz#25)3_$%{-mi+366D(`)m3 z)Y)ZcExVE9dKsPOoKc$txZ0BK!kVv(b-X0Cw$c9#0p{m*3bQn` zXML6=eSv}9Qk#$kJ4Gu4rTOt}u8~P}yi3fe&Ydm%d_V^bxGq&5q2|ZeRcBjVi8`?7 z37E>s1ozMry<|<`>k`WU_A5k@U7adFtJEuhOwL-IR#jt=p#+rMTpM;-?*Mh%eS||h z%ZP~A_iRmwuNTn_l$zp@tS@=KcX&Z#fzLy0D#^&vmQz(q#vn0|ND7xPx*zD$i>VF> zYBdxgoO8{10jy3a_Mh~smM=ekAv_hl2Wzq~O~xk(lu+U?7{z=G*)?2}PuakXt3?d- zqc?(V%1I|jl)0!D|5GAe;XWXVFhIJNZ+YA z-lLUS=Bzsgt17MzB&;)@B#KIzotKf53f)Vca<2qYG#mU*%8i5;lDSVl^n6CFkY~|M z8q+D~WKE#3RMcZxlbdpuARpPv;3KwXe!%Ww97)j8)EB}wT>^Jccu(jZOcf64!sS9L zH5i1UAqz9cX{n#UOx@woor{TAOB99$EqW+93G!X>*|CS$=Z{w090>|W2602;yjvTX z)7gM_q89)mr8DlJI`(nqpV(tA8Mw%hvx~il%C8bC1%@2^zUiS?rAvv%kzkn0M+;66 zX8!}y+h&Zxj++Zre7)Ll=CA=DXo4+#LS6MqrnXzo*LU zk^CVd)<4uYOPEk^ir65{pyyuvT3NV92+s}Ddb`sk03NClxGB}S25EmrhjHz_&N460 zZ}$C=o7L2=L*BV)FpE$iC#?xQkaH&5hE))oLylR0Q>)22j}xb`k?Y;i_cBhj=xU1P zq*ZY9ct98a4%$HnW~V=@LV)lUi9djL`h|Z7?bgOqZ5i@wN|PpRe1EQ1M?4;HaJTE!Ys4Gt$j zI6}qzw787>)0pB=zNQDb2r==E&^ZL*C644O`f$VNQ4$hrs&5z*GdWvS7kuIy8mitP zOVn2rMv)C@U_^96=5kg|=t{>$jt-D_Aj|&BXL|3JJ7-m?6W zlLecSmhu~y37eJq^l%Rgq?(J&eKTT77-X>Zd_u-!56>n2V7}4l^&cEK65rKqiYB0O%nXA7aerZ;}nhc_q2@LQh;ey&TSoU0{D>6|pAVz^)I zN`DR&Ex{}JQ^FNJs~CDf;#EF%42%vEd<8;oe)K*UNI=-Kt=szG6D_iR$pyzaX)#~h zqDy^?_G`kgSoMX1sebwKXN>b|fp0mxnjv_3`nASYXLoHwgX{-+z~k;T=RFRQgoAGm zk>8M=*F4BQ;gPqek!^cTnJlK+0$*R5`qG2os)NTc4xq=rd4~5TUxw!qU&cHRvt|uv z5MCANVG)>N3<19Fgfwz-D3jZ9N5$*t%}~YuC(WBbM8EN<4NMAeq0F06w)40FSjiWY zN^MbRBe?iL^Xrc<2;{gK3mkSch{fU(pY}#Em=vP%Il5R0UmKb8G1oKa-OunV@?jBq zVZ9Wn!BMX}NtquF0OgpOCBP}Z=(o#`?b0oi!u+KwZCD?3`6iZ z63*NL&m2QX?%%%kT;MeBKM;XSy9zxIJHkIf%icLqZbV4TbLAv>S6FYaRi+m>qcJbM zyb>{kutyVJe;?U}-Ae$w5o;wO4(sp*`V5yj%q)r&cd@Xm_9xU`>4QIUMOpB)TNf}XD zXr?(l43mg=Y2a0u-RRojUN5Wxu7#dGx_PSJ?v4J^qg^b@YP;DCQfbX=7ycMDFT22& z>+l{=)acvS_hR!d+=?l&Vi7t6!>kmwcWTExcoR{V*)W9d6vd-`#)-hYyM!H`h4A(i zYU=(oIT6}9=In4ljcEu)_glUF>9-rj3#djcz=tE)^d-y~|FX zVRW}JhSolG%qP)kE<_xHsOjsS(hEP}$nb!F6cVx(+bVOdcH4M!bgy}v&RJhoe~LT6 zJEhPk9AOS+`L?>=2a{efAJIOgXR_d5kDgT6rgf}$kquJ+6( zk|+HkCyn!SItjg4-Kqq6-1zV}RVyrNn27Cqp|Clh?#6yj!t{bLL>7tK5CrcfqMOxu z-tFwX+b)Q!N|x6oJ?Ljyjryc5!jqVWs}AhjS!nZS!3x9vmHQ1X`d{w1ioCh2>vdZ0 z4~yS+h~k&z9jM_ecUwEDJLgu-R?_+ zY##tKL)qUl|K|HxK^G_EY$j$`rzdMvp2KYUe`|h2_z!a%^jY5r_8W)TpuK&q1=ee6Pa2YI?kITvkMoMo&Q2iQ3jvi6)Llx}4jMo9i>G z((1Ms7!FVY`&s79^x%ReHWG6)VrI*aI_>YQp>%F*{4H(T6 zN$XO`a3*mL(>_tsHgE6XnjAzp_1FbXUz%%_pwrP~3I~|e@C%Bf=Q9&C2bhl@2|kGp z53lQ(TA8o6ouDpJRZ>}57@#*w_LWOJ+*7cX{T1KN&v@7`e_EBilPCMS)$D?{K zL&W5b)DK#+H$CLJ@mA!O`sXK`u$ui?NeHlrh~J%W$XPY&D=SGwXPFjm1SE@8~i&OCL`8D{u|7*@+<535+u{?t0QTEaM5rh}7+0 zUTuqyIzmg6AqoGKri&5ku4G$6XUMxx7-{ zgtmy3IsFPpi5e*<5rwUAzIgf1h;Ah(f^*89)C}=9CDPO*btxywm--!gd- zQiJXOwo`IHt{b9~6L_~iGP}Rrr(xOnn?Y!0e%7tB%$CH8=A z@VNb>Uu|wC)#PC$H=#(CkDR!TX@O!91XWMbm8YL**yQ?=%%Y#e^W{-xC{dkf9%%N* zrOESc2ObPVSQBF(lfTVR59)%%YanS1W(I|A-_?ls(w?560p3zXD|2A$fti@52frlHUccKGo6 zR~I(#y6_6ihicxvR9?Yl)5O5cH+F5`E-x>Aq+Ls~@KzK`)Csr3CUdc}v$M}9NClXT z3sk3CN}OH2*^&{)05>X`gduv9Am-s6;D#=R*Ki)(nk{~{Vp|cPkJV=Ai+R!V;R)1oP5d?QVqV)E--M|^1^3NlQ4=xliRrj!-FLl+p=3EIV|WZZXc`yxueaR zBCsE@6)KO&CQoYad{`8CmT#Uz!p()3>sZwh!iat9;Jq`D9{Nwy8=bGS+J7&4I{=2@ z{|6jwF~~J`L452_sy9dhVguZRBg2+`#_gSL0!&Z3^<1j3TCH~S@N`IB#|Bb13tH)f$}A=JJDjE|xEDm#J`H3A zR!Ch5qz{u@?D)$}A+ah#{zl5=pV<)D9~d=4+lr7=E$046jNhFf<-*14+jt`rW~NVX z4k!~VI5JDALZlSX)a$LV9lIV?G71m(9(&Vaev+M*CZTmVE~J;J)`LDx7>=YGYmdt* zI7q_Y@66kL1ik0BdKWG^A|E^EBw1+ob!_p7n#{-Edr4sO{dG-}hi`UkMZkQK1evPOx@r&65Jo$T=F}yj%hL|z;Kou@y+2TBbwpaU(t{F~ z96M)=r_a&{4XmLL$%@`D_X7YUAC`A?e>-iY=V(JDhTRf42B3g!RPZSouu`zU_;$o4 zQGGDR&^CeuOJPbpdZ#{fmSZn?$45b%m|7RPqBiHJ7`H6nbnC%k0vTAWFb542N^h4g zVO|)t>V6Ef;FRGh>#VTjU3{Myt|w1-B}EH=V-Mi$IF3+^Y&X|7t6UF+m!gR`=XA)X zZB-y}332l(1eQV`Td;@VQM3)NJZvGTh|)Nfnf-`34%R-qb{?Lf&F_w(#m*7|$>eKU z0OFxQtuAo#;odjI?Fz~3V9m%AtcE+oE7ktA>#gWE%F=ggy_K$9Re$@dj+^NS4`IJGeo+JpM%FKehUAa^!L!!^LhXtJ(i&`nQGhQK`I7Ev zhb=?5o_8*I+M2G*N-#yX4+;xQ9}}VI`~?fyiogr2{+{XXe%_bXdV=N zi4cgPmLSwt({~{o8-M6;GCL@K ztWXm;Nd&0B^36x|O3Nj2!f2~{5P=J3!4_g9Z0cQsR5adsX7wSduxl10`n(;6jJSNS znzL1Je`_Nj(e;MTA&5K2&k}Ycp2Q{_soAJR1RWo<5b3;RE#uzX!>Xrd`O@NkgD6!} zQ7K3KM78lC;wR>TzQ7|%qBGUfaYWY^?O+Q#Il08>>~%oMr5o2_V{R|_ew^(dAJP3- zb0qqw_fKf*H$IHxp1#tfi%I9zM@x)FB#|QD@pm@@#VfG-Js&`<&U3EDgd$94!*E!w zA2tUrv|PV)TE_FbnBAFEf8Dt`et^UMyWsD#?P&5HSOuY~yW-JfU4(CX@zMkUg=@mC zLSH}bXdoalS%zAYV+aj0kJ`{ff)G&b1qxB))8iR+pZh&c)ks&a$?>wS_n&eYU|z=}?^VH;*8-m3($qOI}d* zLbUM1l5jKSeyOrhr$aH8smpBb!r@`yTOa+$leTx;4ZY=Na8oK;lJ4dL$pAhhaY|k#fgrE>6Mn2DHJL1wpW_j|N0s` zNwv{CZ89>$dQ}Jt;MqT&ToP0M z&!vc;Es5kx?rG4rdOn2O@y{)QXjQnVwd7kG)rAEoVa8q&e?qA)S-z(JUJZ-wMV&E8 hn!%zxVYI8e2aR>c?%XamOMgRrq{QF9Ef>}E|1T!halHTl literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/compare_result_pkl.png b/debug/accuracy_tools/msprobe/docs/img/compare_result_pkl.png new file mode 100644 index 0000000000000000000000000000000000000000..863708bf6daf46985328f0dc42d48f0a5b849af5 GIT binary patch literal 36133 zcmdqJ3sh5A+CSWBr`oZowTglQsZt9CUKJ@tZppMF*lNKPLO_8ih-smUT*D_Ul1KuHe_1i&brqg#W?SH-N|9$IQ%S#|R zC+9q8@BKW#%d_*21O}{Kvvti2FTAk!gM)q_zwpAUL*VtzS5|?4GnU)WfdBrG{Bgkh zFEsFM=D`nt#J~5^doR4uNO@I%;$`sjpUxf(Pk!Nrbv@?4f8Zq4oP6Ph!uAjR-U~^O zTBQ9U$^L`l&!k3=v8=^x&8xwP>0P^j_`Lu8;g5Gzlowc^ePM&&&fwsWhMH=9j@K+KL?$%novya2~oxN8Qa@=uD{Mx+(9``F9KVXy&2Xr+HAGaYz}hKn!;b zbCgBZ3Q9iVD&Kp!7rq16Ex0XHB(1}oEb3LrhwiR?mrjV0K)aPGqOHobit`RidIIj! z)3d}s#}y^*>yxkHN!89jpe%gW%RWfc%(_9^HRD8vJNJPV%Xtr7?L?9rdfK^DsH<({ zcbu2`%=-p!{QD?wJyFUfR*umk#t{a`RuzqY))ev>_QLz4xauocgH}l}ZKywx)<5kI zRUjg&Y>Jn(4yujPy%(q>gRh=zU?u0cMXVHg;2 zXFgU@)H!K@&YXm5M;&uxIu(CO#mWd{(*^wO7Q|@e;?f_}O;f`-)2TnLzp~nL?HPC6 z378?e_ZZ=yQsKbA)p1!6LzyPsC5D(jVGHzyb^H%0_w?WC9Inj=^1)Sr_>UOQ30ZSo zYsS?x{BnxKVW|2c*P6L?*_)4eQ^lexX-A~kWy;b5}+oTn`Os>$=%-$ZB54hb_urX=bZVI(=Sp4!S=c0?Fr`8Ae{5XjipXp=pyF zXgNx{clqu^Ye37*9fq036Vix!@8N_FGBr5+*}y z^PTZ^aZ^ek@`ZkOQa9m!8g3f3O6cDe3Xxx58P-q(G$3Obg6?G?MQtc(F3NUgo=94G z4)SKLkpWY^lum-~Alji}DW*BYp%SjKKHzq27#m`s3A!T#)zgc~LY?a%jctC(W0ZoZXL@(VtcE>B(J~z&Mr#;Ri6{42X$)P zH)cdsjauOjs2_yAiy)<@pFUn!NJPo*Pefg{B91mrzg`->1JvMl*$zNCk>j3WaKD;u+~pbuLX$U;YH znO{m(%zKeKd0DP4_)uh1yJN9yhr~v+A!?hI?({OHRN$9bIPTscHQqIVp`yb>58|tj zZnG*N9uV+rwF)gyMzpcS?;b2VpVZ-cQbP_t$gGO-p7|8?M@!aw7MRIQdL%W7A7QO!jWbd;{d$XOtPvUAYW%gZ6a!S zR1Z7IE-#)~qGisSVAlu?$ zLlvJsoQvB~@V4j&s%7QXGZy1&nlt1Y`Y`;AIYw=AZzj~3`sN>FN;rOvg=ew_&2mBo zYD0L`CQg`mTX;gxv$$QkN{LK9J@P7uoe%x_>1;_4iG|Oj#~Ceot?+=|#$GDn8l}0N zva#1S6kgTf&BrSc;OeSXC5-MLlVQuoE2_18knOCdpH>gA9Avl&@55?b_eJ^AtqTsk z-*l^S<9;X8CMM*n@aYh$m@)E$2R9SiyV1kBS&y%V4=Z0&AITn0rb8jLZ{o21aaHLa zr|RaA<}s$4?RJ@OfCUdyd&_W_@;l_t>AM>i8w~15L|XW?%$UPCK@pl4wWg~AgVYEs zmNFG|0J90%nEsT||JOKGTXx zd`U=?CJPX49p;b-YDbHEKcc^jsNQCJqRTyD0wn<2-8o84JQbpRg{RciX~KOu7_a5^ z$;9XP1NBW2OKV#cBhI(B2-C93*{!*>Qo(`5!g?*nXii*p-^uvKDU=skf#@Qe7vo!7 zbgQIwKfpBK9ckJ4R3z_1pTVO05yx=zV8nPYQ_i_|N7b{=Ya{2!)i{HGe$HO86u*Vn znmW=G6Ng0qZKT*nOuB1-HVEGteW?<$I)-Qf)9j=>hOA-2A@Tdw$~w6U z&M6A4T_f9KFh=e~gMhejR1i=naYK_VgN2y-62Uffc5dCSd_v5+Oef`^o*LUqK^--= zjo?WfCi!_#@?@2ESE?Ee0+uc1fZaK=RfIHpFMFF?WxB_u4PyUP@Lthp$aT18#NFY+ z8rfYAHPAw6S8&$kF^vkuoI^_d8r#UxQ*>Q`(sG!?GFU=udHYeSpOinO-gRFyt}YkA zPq&IwVk{#zMpj*jq>s0eckq8mPgw*jAm;2xDMjyx8h8+A&aWcR*CF59Dd+t?IV>vn zfjfHrlf&ago9g{QA<= zq3gnWT4+`ct~^KO2akd z>D%LWX;%RfDO-JghfLb2K)NA{w6US|Ly#0# zN!_^2PXNGhKoUGn;VP#Fc>W$S7}j|eljLojC&oyPjZqDeO-QR4 zQGBJBJvX4zt9X&reFOJBz0wOOP=09mki37mT82Z5(=t!2lLn}r9Jqv`0)2*&m>UD> z;dLAdTm`Nl0e&~FleU7feiA}2SC+sS+j|4J#ZMz@^Y#g()l0=nZv+2qwhZxlvbsnA zcIPB9aPgRWUI7kR7~&*>AfIQLXj$HFh>zZ-2fcB1H|G>mR1WJ0hXR_t>49C94Dy$h z5OrUkuG%YLwfuIrm(-)lfK4jupDgFgjE6Beb^FKcpGE0f=*x&D*PC+t&6Li+UghQf zYf>;&P~;owbF`;K!znQPKSPdZKMEn`o}oBr`DS#B`n|iC58*Cd$xS+I3$M#8NdPtc zW9_;I;+3U{E4lK1!cPT6d)2KRA>jw;wy$8OgyY`nzn3p8$9GJQS?piDOZ0pff_Xzy9Qe{{I7McOF~pHI?S@4G=C*mj@VeKqq)4>r%+T1d zdzz%ViM*w*H9qt}**T;p)s+e5WuAOI6EV3|?wIqnsD3$YZWxk!Q{DJ<)XAj`<0B(@ z3Lcv?9FRD6c{v3P^c2?iGSPPViNwuc^@4TLbB(64kLAhv^ZM+FS!YSg{#h>i*NgziIA znKY;c6}-rnyDg;35^cqNFfVo_fXCSX@cGb#lwMq1Ip1L~bhieS(<*b-TI{O9YONe( z7)s4972Gm-h71tNTlt=k>NOKf32@UFwZv~39t=B&aKj#~l>N{i(}h;=8<>Z!WO_lC z{E6~xEaKd(8(n#pNI(iaL&N9EnMyzqy>*T6@Y?X@81T*z(HOfjm8!d6gqS3ZvTjBk z?=VQ|e+Zsu9pPmmCcbK0@=!ADwWB~e+i99 z%Caqr`NZy?rC|E*dw;6z-t|^KZDpB({*-bG`RkEdW^^XCvSk?$jkP@LiPZ8be0yV| zKud!cDgFqL7vlN3noCb^j> zHlmZW!->6vNqK0~+`jE6nE98N`oK)lq8i;XJsG%Faxn8EqRDV$KpJ09D`&vGWn0x-|^b zbk??iy_$&B_Mvhn2|TYj$}U&ni%+*Z)4}DDCTuxY)XxS*sY7Tu>B{5nhiLGmmnTz1j|Adb&JO40GwL zE$c2;;f(YL6N{WH)p|@QYDkcQHB6HVGq||xZitzn4FA+_8Y#_Iya!XPx_b@Qug%1g z!@a0@`I*zJ1@bwh^|KrL1Kwsh3#1@47$O}IT79!P)2pOfYwg^fNHFH<940t7(j^B% zq!``Fg;6ChU@;-Sg)mo3>MQO3MsjZ+A>K#J%!zR#w^ro+btS*6{_B+5T|#~csjsap zUlDCvI%V4}mmw{-A!O+tq@_9G6SLRI_{aa2c1s1!?qcalfKtBk zv1fAjtpSxQV+QT=qP$A)RGfdL;+xKmbk=7kUfYiC&J}}sJ{v22_7CUAN3LVp-`E9+ zMtkVSy55tCtK&V*3(BkJ6o!haK(*nApaA-VI|k@k0BUr1YxISIG?BETK$xStFx0b8 z>IWA~le+UyOf7}YrT#E8_LA@^*VD+Y_YUDDr> zC0+mS)e1Q;n+kF?68D>{me-t6G47^MXAHy+AGJ|2CavmrY5c<}u0AMp?0`~K21A2? z21%4!L76UP=l$k$pOQex`zT>7?S2aTytmG@JotLYiGY7BM4$bHL__okPB*r%_58gQ z_j5Wth_^y>U&DaQ{H^Tx^FyDpk8?!H;8%ZX3wQtDxqxwE#Z%O|_7^MUdCkMbcOdxZ z-+Wcj-!4V@4*j0#zu$dckb;`!jKMkMVIjCeK#%|X>+GB*E+}bA)$%Nsk0)$)z+0=Y z@-KrmI3o!O%F(53oH83Ja{7s;K)UrreSHFcmtYlnA5TcC}>eCaF7!S^q6{7BWIuu`Q*pG?ik zDxugp*{c|EN)|pv<{fphr{dM{rHKAgKD~zX0ct-Sh7hMnf=d>=J{)!M?g-8Z_sw*i z8zJ6G*y6Kg`8fYeM^GbY?V96B*Z*VrdmBEEAQs6KbCy}B{Gg@gCGD;>9o+}9oS6%E zx;a$v+2syW=y}EaICmUWh1)yAaqgwTy3p))9e8-&P$M;1=};*nOJ@LRDjnwBc8l<& zFeqA#4nxWjCy`p-9U*q&LxkTjj?EvV*+Z^e;X|<(29<+hX4gQ;0NNT}8Y@#QKT15%>X%ICTM0P|y{3`nZdVPBOkB>Dc?zY^R{* z$D}HvJKd3-8k7^Wl9P){yO*9Ag4L5tLDFpk;-=BTmP4bfV-~VL3c>rq2SYfGG}vlv z_e2ny#*z`PldiLuoV9>=SKhV~~zmyH%4WFCtvyPa{7TY>PuiGLlnn*iBU@lk6wSJB8;vX>gr_ zAEL&voYl^TA}o^5dTKSP-cS%=$LAzJRR`xu&behoBRjIFKK9QBfj3_obbkPj&^Fyk z_~i1rUsmH63@2)Qe_f(q@LgbZ*;C@vi%wRs+}JP5L$Ws$I@2T@XIrpHT#oJ^YILi+ zMrS-j#(E;n4E=&LPEmP&2RI#FXc(Z$iJVy`vM=%#*4pCf4AUEo%U$iSVM;$IwMMBI zrcl<*tkfU03^s0(>HH$Jcdyd60;Q3??}c>UQQ*x5Y4kd8m~lSIr>}zReuLLLm2JAQ zgOMJJ5S*}(M{Ka8=mpZCn*m$uILU3BxU9G$XSMRN{=JaDp#$b}kRKlRr7)RZli+b` zL9!JTTjT6kM85dn|4n>a6sN&6H~R}PPdr>g#nk9jq<`wW1+_LUlB4H@wv4UfDh~+W_C#@o zziZk^Sa?>s7JgzXb4!(-F8bK7rI1@IJ3S1z$lUmLzf{07#_U8filWa;{o49y%j>^V zKit<~Pl-7%16)*brtu^wUGt(Nn@Emt7Hq$DYW+Z`108j<&x%!qq0jfdCI;m(H!f6% zg%<+id91jKPI6&c#b*3^VM#U7o{?BdbVt0!z-{+Bwkf94#~=1q6Ug2A&vt#MDjJ%# zk{(Fy^xv`j8fCfo4gecrgW2s_vAO_5dmE2OT>0KxX)WZIEe9}{Zj4NEVDUIVSm{7nv@XkkQ`VNy z3KYG+6lxO>zWyDcnP+M;sc-6f`guoarhEWr{ROjcR!d`QeqEbS!y(eJ*z2bGF8dXz z>i@loTkfU4724IIW02763)+Z9HSj*Ph#AuLlXPmj<(s5Ry|)ta?PNjB%`^Zz@|Aa} zJVR+?`{f->P_l&R2rAl8htAkYyv(B22=ft%jJ}MAP|L@q;xyh@;#19AA!G) zcMy*5!a0Mp*4)(TYc=)fWgV_khBTs<{1BcEF)(Jry9R}Umk=er9m|v*^2pn%f{90c zHc&)&asNXAf-_I)LQ>?RB6@O*$$Um}wqIL#wg*;CQDq=6&1R(A0&-w&^Uqb>Sd{$U z&Z|LBWPZ96p9xuGi)@+cd)Q}~2f-WDlaQCYUGnDcxAZjD@a9KR2Q;}QwA&9$fYh*W zv%qs&TYYOU;*|JTkWD`#9ZdO44XYq8x_YI8wdw#Wa;Ka;P2ca5oeHlW<;u@oa&KYpjl=VoBXQLN(u~ ze`oQe(x!}* zd)xRae?u9`Qk`3x7PX{*FLdS*5YMbJ4UzbyO86Ef;}y;Mq$aPbYjpLaxrvFESNW^6 z{Ik*v#eNk)t8mDV4N+GcoS_r9a!#Or?r4ID#y?h2`6cA$KjslJJu9&2BXg*}@j50o zY=4HDs8{%0n7KSUFM7a5uHxX4q6bf9#4FBt+H-}V&VOi^szsB#FuT$jUDK4EDl7s*nn8l)wUN}K9nieJ<_2tI)6b8k%R>jKKI z2_nCg_GB%Q1V*f$v8q-Uq;rt;xE~g?-JY*&oqAMH%Ou4$2xfvzId1o#K^K$0I|uq~ zx_84O7oA7L-ct4XbnBgYA?%FTwkIwcYez1Pm4lZTbY& zaN=TuyymT{U`o8$I)ZPM5iJA>?xOS2${!UF+zP7QA9o*GkR%@XngS47gYV`IaM%{u ze>3|-&ETIGfxM6seM=MOHy0B*w=G=nBs+!4I#9tmVvP=T=tG`^(PzZoiecizC`Byr z57@8>+c5yfiaZ;GdWQn4khs7!|I{=;aoZO04e9$Ar6Hc-Q;&SeM-Fa^x$!Q5XjR_UOtrdz2iZ-XLp@%bf_u@2LG>E0bWz;mgG1_`q60(momx2N7G3O!@YReTQx|zl zXOa16P-=cB$Jb1BC=vX;r12Qhymmua*3N+8TzSMgmUZ2Gu93S{MJ{I+M%gN(Yg+dH z4F6C{;-;U=gZu^F`3H_?U&QsIH*no9wz?r4CfUgi2mAA-NjvyoIPz~naLn<@i^l++ zb;34(;NiN_B=eYj5V2b4CvKiNWP<>BS}gz7l@G8mfH<0N!L^TXNpLm>ZpD%uc{ z9SuXA`#wD}R;*|%azLODo$O|=<~$DNpoWGSVBK0-^u5_b3jNFHrR~WGJTND%K!j)S z0hISo+)AQ%>5r6k-ed|*_c!EaMvJ$}=Q#i_wue_OE4g}jPdqsLJ*Y8D<>3kdne@nFj&OV31EKc~Wm^Z1u($3oeQ&7)}~7Dq=kWG*DTH>6u7eujtyL4e<2Q zDh5RlO43ydn)-;{kDgJS=%!m$`Z#11Gt<#a4JH$QVI=GH=q$j1%Ef5LzE{EI(V_#z zN>FJTEos`~%VI}LUQ(bh-CO5oK6+oUxMn82{_6VC%Fx<4M)!1fvkgWWb?F>*+va~6 zGgiM^k*Mx88og67QIeR#Y*wDk#(hP3_yuGPS*FNvcgJyRT;w(<2UUc%H=R|(nkcB1 z(KS_iZxn9TXcE1K#q|i)j!#&k;XVa`fLOoOaMmaExS|x<+TtzPA$XwNx(47 zlXK+v5|(HSKOF(D=pk7Pul1@%u_BFYdT?e|N8 zDX*IJazlM41`wrzo06kom?2xKm<=7--Ft4zz0Gt2&oCRJ!`F3gDQI5yGtl^S2B`&G z#wT5*G&1N*8zrUUX*9nPj2nyP@Wshm&R9djf)Clt0F;txWb{&hRxA7laI`dN50EbM z8^JAGHX#MD6QXfn7A9M~4KM}p|8>y#EH;=TU3iov07%*Q$|(%&uMlG`PzMes3Ic*uQCClL1 z<6Y08W_`BXou8i0}?7*c&+crhn0LNgW z1-grRW@sEHx^m?g1kCK|U{C+KL>F1&veemYzrW-_c3};{fr2)3l?Erh#jZCCT+u^g^&XwM1#J*~&d;~fEjBAw zeQ;zwy30*@2jZZka%8sZ!D!l)<9Z>QLWA$OZ0z03&<2{yh?w}Xt1&@7^yaG!GM-!} zst<5AvpHB`sN3AL50|`_RpfXoM9uB^Q9-mQT+~|9GFyP}OL!}aW_Guow(Ygi6gp~l z)kyh|7l>zmAzKX?_cXbM{4U`tDEAQ&%cBZ_qIEiDWg3&p7+bW|j*fRj0xziKg`l*&Rb4^x~v6%pGZG#|7L2Q>-)=5J|_o+&&O8v&=pcZm2PsdG)=uX>5 zH**6sFx=BUq4Oi3i<#Cr$OP}vs*{oSlUc1Zf8NdQ03l4_(-yBk+=UA`nfN9b*x_8g zbOFb0m(OQ1vYP-fwdF}$8r|qMxJ}32qlu0fWfM293<~>Y#7bYMrmH~t!my8mRhKJ; z!-}O3QvD=Y_n{9dPL~aJL6<*4-eqoF%x#QhN4J_7rh5|ToxhVtzm&!Rqmj1icZ*~9 z9*mSXZ!N&fw1+(;DUvD~wl?#d1YvnQ@?0a=V}$6Xwr#rVY{15~x5e>k{L58ZLRc+l zN|DtyA-V-Y>N_w%3}q(XGNsoo_G^y2bwu8ZL_8Fy%rA%|QOnF*Rz%Eev zVdJJ0gEJi^$g0(!w-hzFd3r;)k%60z{Xxz&rqOC=18%f>Aq3v`%)EzI@~1$W131M0 z9fUx1;}&;7R(OUqT2&e!FWlvWSg>Av;0n_GTdP?#$4h7dUBep6eXupha8Sv84z>e~ z!>_-o!mQ1rWeSwu0 zAmq}Pv2gLkxd&4KUt0(|g85WNA~2R#P2d_oo*ck&^tHaNHXcJSG5V9O} zlaRYn>el4ug9gx$Gww-mY(hyH6TGMnKE15jO#r7EUJ(b(@Q4l6^|dmr4x0s*I8OqK zUIff!>~>U^gEjzKq6TI=tTuxp##Eic$p&!H-E?HwLu$q*?BmTbp!|GBYGi`01E|==y8r zZdbZLl2z}m{6>dlcJ_0^LRs20NtKo{QF-DjhtI5TY0J6G8wcBV1(bi0_WHjm(s^ma^j3 zYM7)xkZSu2GT&et3eS9Wk>eT#JZgpoFK!|uv+C$J=d8LBLXwhXv~tSra%?@uo?xnq zXwo3O$&phlTdT-j7m}RJ3z1{lNK;w`H3+DM z(?0)GE*v0P*JS_n+{<>JSO<(31-Jv#b#bhI$jZr{JAY>kzsKAy!09Rvni}l-3>i+F zRqy(ivG}#t+d*SC=8Q<0hctF0S#DDS2SPj}>9?=CR73g!S(Vkf7641PVR-p`U?ZpL z9GJro50}mU9bXRdq|UO0r1atAYuBJF(B{a7EKbvRS26J0xygIi%PXl|k3bRX;HLdR zK}<1N)QNq>)*vWTcvr65PZYr2I53l^C`q9x+*`TA4PlqXr;vU2q5)!IKsOo=@+EJY zaTt`3y4ivAxdo!ghgT%Y!NqhM)uK)c$nkw>oPB|PMO9K69-vU6+MWR)$0-0+P#4_B zb!#VGy4<_At^kll=}{LGjmYj)93o^b1J_Qp$ zUHR-k!>}vJ7A}Hrle=4V%VN+_v7)5`Cc>zT%{#sCf0*TfM)@S~s>uTT`PxtW{~bF1 z=a{*i|2X)H8E*E}{l9(iXUJ;f`q991(#(gSftx0F|CW{_fKFxf?I*h#17XADs|1v& zdXL$l*b@i^LT#$AL;Rj2PBhy^M04Qn1Arg;Pv%8$9>+KX?;@i}YGN8bG3gST^%+u7 z!Nqwd;OS+g>J0G=>j$Z!Sg8Z>GyTYD^kZ8Wyq&F1e5TgPD z7Pq^#plPI7b;VIS0<|@xR*rVV6w7D=F#rB5u8i@WY;xLT5UmEtcOPQtwy;JK7}bae z85gyUr36*S0F=mDfG~@-`Z6dj39F?^%CgvB0goX$JN)5EA$=8Ufm+T-G$0og4@bH& z^pzm-h%V9*U0#h=)b`9gH~$xWgfEM23XCkf3_tVb1q@dQvub3a+E8gfv9J**-X<_q zX|cc1E!$n4P! zUR-xf)SPmNb3|JEXpW$})+)os$Ee}vztBHWzqgyqByu_hS|`zc_!gf<|L`3Ibzmbj zK0P~U-GU(43~iW^KjB?FdxEpLWjb0!?labZzdP2*V!o8qitxuVvlp{VCnf zc*RIqF}FbP$ljLh1=@D~#IZAX5^JA!=q9gQ%$@7GeyApd-pfS)6GO!iih_I8mT)Us`sI6uvhP z9)eN>bjq0y(6=+6kSN%#sv;KGBgqXH)U`Y;e7<*_DRcgp_wJMU-S2lBUNG#f(G>eM zJOI>jH&4H&E?6BtwKP#4R47y!o9a{zKhKkeGyP`O`&oDz;->hjy(q4-kIEy4bp0p` zJ1F+H!>L^r00Evo491e^fVv60r7~{sPs&kd1z<4F%k-9vimtNdiDB^=$aUmYOpT2O zIJy9M2ZqZD^2=)DSa4f^paBRvtL27?sK^ahE^o-dSO5u9080Im4dUn_)}p`?-Aj;f zFh}mak?;jU**CHcf}XJH**nncT3&95lBKfNw5#n3K$c5c=HH0e{A2w7-W~jYqF}JL zf?wx+G8Op2%9QDaU5swk+6szJpC2~#1%~aHoh;#gcq0g(rQLm zXjj5sGCcT|qp^Pb>Ekyjk0*}6IlT;OFz}Z(Ltbgap1;sQv36;KahLCsMy|%MjUv>< zJ+JRNVvrpvI#HtsVA^WnXOg8Q<5Vn4!z4O@{vqOP7*-AMXHWTuq>ydBuK^NzEiee6 z@>BaW6K#Yg99KVeQoAc&3iUd&_J^`GsX=0^5>8_cXPVA>WMN&tMeSNSLj&?9@TfVa zeaZ09-90M4fln_g$dJJ?XHTcR)pBZmUZ>M3A+<-&lMd!-n2+6VU2y4J(Fq>g?UDeN z?RE8e-o?c-fjt`Kr{;BT_f9M{ZdO-e2SA^`2a@0m(U_CCJ$cTd0711Zz;=d^DBdELiMhe9UMrY5;)zD`IY~b2 zS@wI`j1!@mA#Drz>~=pWFN*dF%FG6I7_f!~$Wr~@GrOb{;rjaI>qtP@$f5Yl(?(3! z#@@ofU4SP9w%nM16iRz#x#PcECQaFwDyi#x2=MH^B2OAzqy42w3dU_9Xj72@NAA@# z4HJOH64w}QLuIb5x+s>$L!l@7CX#?IC}u;3n5Dzkj}q5%vvbTTrcr+Q6#zMl0Y(PY zpp44z*N`iXJb0hC)GbtliZpShLFyTiL3E}=F;@ZJ#RI&ZKp+P19jX=CZlkhr?S6EnLogYs`-M_<2L%1jz7Nb*cu zh62Pa4ZpL@OcvCb`wZ4O@bD+`=^&m`N^>$YCYH>@w9vXhvUbMf0UXtC;8MUs8{vug zSisV-G!4Xa$qEv3Tlhx;aw=I9_=F;v^`vP7*Zn-q2O}_i8k3If+@i#Su18kf1%~l^ zdUkgu@}Sr`yK@adkdsP6zKPgbe!SKOV?l?RgC5d!|`UAsbVUgnNl4M|01 znGOZpKo*k`)9N*f5jW><(CNnzfjaSAc0@pnb2l3F3QnOxl{bDw`f*mkOkg*|eT=0w zgSH*JWuMn++}hpIcAWHjNw69~cz}M3;!f8sG=3}^=+eP4&r+cw4zT)Z6)2ja%BGs( zaSPD|v{i7s{eg|OoKE~z5UY1q_>k~uPEyp+^!s}V8qEC<&!Gg)uvYr&E*^U-* ztSaq3e4JfF{vILR!);9sDZN|EX;E1;;88kaqFpe&1~XVu@7gD%Dr>enFe%Z2Wukeh z+_AM%#_#eJjiL+lqSs}HSM+vdGf8%@vRz#7S$_U&9d>;8OD*fQSMKFKn26a>LP;%2 zIJv|OF~#hdbXNJ~aL*@sxoWJ3C`rzu#01I4<)j4Z<(+G47qSoT%d zYW0l8GI|3hP>8?r*5JGk8TcFej-NKmo@LDs-5wnvNsR+>%|M|^r@QPn+i61*H#I|W zzV@0;lv~)j_kkC-yQJ3^leH`b`6c|kiaONO?pUoVy0^Ukb%18~yGEj3iuVl406Ya^ zIlNG1@uZ4c#{)|7aurp;zdZ0D^^c>mZRHf;tKEy3ElCIg#ze%I%LA%*bs-0wukaop zQQ9MarEW0EJIcY~B{={n$iNX{|HJdrnh3EqFeIiL&c`9gE@y$BJb6fis<44Bj!k?C zV0iFcwurZlIk)?7M^j^N_1(Ig_l*?Tf%YM02KzMyNi<7@FcOp)Ly3IDqlvQYZXZ;Y z%FQd4^e(?Qet*V6$q8z5C6SZti39Bu=`Jd-;|U9&#}maih$m0I^%C#fqDCkaMx6<@K4sI_%Be-{b?B zUf2qg%~4(ur)tt%gN8VHndu2F0UTx$a4$;EWnp!GK=8`0>I_smaq{#EL-NZ$Tk>lk zy1ZH7{V2e*OjJ&(`~}X&6OjG{a-Tfx@Mb*2M&QlFK(~*MIAJ~#w00z1s*+6)CY?oI zImbdz(>kO1g`Xfv??#Dw&zMT3vnz_G#jgM`?^_D@Q z&&{*xt-4&aZgK)sB0k}wQc@|PBo>V}Ccd%Py@(FNOI_7w_4<*0~x@d+o2C_rZ#%FK{G@%X;AI|t zcXe~M_+*AN;0-(xcVxG2%Gl3@UuPwv`rbY8DH#S}V|x}vsH1lYtJsZ8Z2~ ziq@H-?a03Kuw?lR^j@g;ScDoNdwL9$Pj^0<&44h-=tK=e%k@Y{Dj<3F>%*(@syoEA z<$W3c3iUcD1D{sGr~d#9mQP~yL_h6%zB>pI|JJgUZ|HiCWIb%#18kw@EkmCFGHVQz z6#!cMwhs8<@T8YVYqc(l^u-~EzIYa~+hUp7$xP9TT0B5U8L&^hMoSP6O=}r=JfE3W z&#O+}Ts^^_ZQitye0ok7KL^asXwnLJLLRye_jt^Q* zqJZ)DHLwFnX|L_Vx#Ls4ose{}R+EAnWXd-1x{{H_LcmyAIB+Rj7Wmjh znBCQx$mWI%<|cYf_pIyR0_Ys=y5+~E&x(ZIX-&62Uka??&Nq3}HyxJ9F3_UrKkQIy#+<>&MRhl2;c$+uD7p#uJDKG; zVnn0B1;;d&MK~~KTdoEP*gGb?3u5g`dqb0GHN`@2`fmRWYfOR|55EE&xeibiFz6dR zMKy6&irg`;{Ov)&eH+%wo@kcgYck*Db|fQLN$`dv2QN`T;gWDNF#wK39&QN&bGeF> zao0;$1`N^x0Pmda!k*6zE1VsV3io|oy+1@bG$@&MFWGzX@;8|QhNdtF4{lHDNX)bT z#6m{*J@QVRVCzlE*fbw+&#cuFG+1~#b5jqrWJ5}2w487)gxrOXC(cLvV_2Qe3*f58 z8UVqF)nU|WMREab1&F;-^w0@Du-g44(`doxNiXd_1SA!p%*YbIT3B!Iy+76HZ1&G% zD5apc=Nlfbt2NNH&b%1~XwqB9AxML1F=b`O>g8e}cQnDkkIY^-R}cmO<)UxlZM0{{2rSN7)#9&n1M78 z)jf{_+NQ7EUWz+?c7X?nsHctN6*fxAfs>v=ylPqN)r1;QaC>vjo*iGOknH8`0~wn^ ztK&J)6akb`&BE*IM|~E5MaY56#@a^}9o@0afIRSlq`WR+7-4y_`zg4>RL? zqQNzRMpsSE3f42Pbv0B+x-`bK0thJOIKdugUq*zc85vm#c5?~*9$_|IPaBy=urS?Q z@S&qVcaSE}4}NuUU6i*?fsA^_5NtMjTK{eD5#YAU2bS5;Uk5QLyZf-id2Ra@$23qd znI+8Wl_d~s;6>43;8_8_XA#9PM%Rt{{qFQWM&*5E7x(xvBJq-`LT{y+-=ZV7e< zwc!zAnV)&or&MRbQBZ?0qDA`Rr%T{XYcYU=hZSz1!@+EfWkdQ}EjGfJ4s>8g{+|)L zMkWAq#tCOG)^`WU84Bv55&?nF zelGFSV8>7%FGM=2w!etT@P^uwkl!+Ji@3!|c$L)W8F|IjH4`eyz>_%-B+d(FC7e4f zYxog*(3Q@PGdYtm=gcnZjnIRw->F39G%4ipc#atL_twlNRN-e7StrwY2G!>yX8l~g zURmu^dBZ`u!R8^MdQLIwe&tg$T=t2f5bZ|lL9Z??;)kcqpOo8XhF#lnsY zjGO0VO6piO*th^qXYCr#I-(q2qB>sg!iWYQ`eQ^H#v3+K0mmaRO)fQ2BSlA^L((eM z)L@d?(woB&!Ne1|i)N5mkOZP{HbxNG`(j>8x-K^02rh(o6RyG|0C#7hc{%G26 z=M16dPZiCuogwLpMhd7}#em4XZ5@60e{k5c6Om8(t!buipXKLm5<5?moh7{dnO}VKvZdf3;hRxk)v%EaxKJbn<_0rBW62(}lgb zONIVbl+jkxxfYMtE97PPtXwz0ACtQ#$N%wEGokyr#PGdaBY&DF|9P7l8XL&2_<24# zz~FPUgMY@bF;i@fM?UL`T{%#jGA{C#Dl5p}dl!^8`;5!0-ubq3VeoZ!xqY_bY5D=M zpNPS#>+`^s{!M9+!OiscGed59h^b>a^eRlf$MdWo6}yHCP|W3ha(Tejm_}y%$GF8J zGbYn&OC(`Ky%pg1S7sYAk@dY@XdLP&vcR_?Iot3GwCB{1TStVHs)_jmDI*g|z^S_j zYxTtqqrd{9IN-U^)=m1>bMyX1t;6?|#{?ah#Ub6bv$fO@vGIvGaNLoFL>f5tAmM;3 z7ISR=WEUsm(Kp?sFFft`*8^9*7&}obJt`DZ-r+>=(nR1;Kid?>fl{)T+E0aRhBvz3dyG;Och=Yl)jA_v0~yqpY!Jd(>rm zk&y7ISu3rLLcYp>pytFK%$UEtQ=oQ;m)sLRLvAbbaeZKyn3{CRyo0)<%EWss2Z*#Wd|U#Qk!`e=k{ zsMH4XEo%+V^;G^t3=-t>aF z>tzP8OI!<%L(#TZrVBQvws#+G4wZWwK(9ubq9y=iAjr|1S*qp&tPxLVpF$p`We$3(NjuE9-|;bCDbsf_fLy%%L8LF}Xe}xS z?j>GJ6Lu%_SbaUPFI6cTl(G*$dQ;K-voHxOx5o)W452NP31aNedtP0<>>_vjzUHe|BpW z>g^VXidP}9<;t!nJ4Bl@B6{QwUt~S+-Q=1P+RhchvrLd_qgLAw7ew1h-JPKSr@C_w zYwAw-{n@)`+FfgPEaH{RZk-k+jFwtz2@qyi5sL~&asdTWK%|8h1Pn0{!qaKh3QB4v zib}``TF43{0;XI7q*W0Kn2gE-L4hiSXhO)4kYExL&bxxW&g{1P>^aY~Pygw|#Sj+j z@_T>p=lgkCJPY?y1e%h08u6!Vrc9SH#g zHx*ZSG;umB{5x~(B2jN?t0F$`E6NqO8n^_vk}a=sVXZ^#%}8FXlu%FCauK)b%2@Kq zfTv`TCdgi=sPl(Q>>i^QH!VRrx5t$9WH5f)c>I{$+zWS z24=MH->mBqXH8PDpl2)iAno|w*}g_$ne?K$=4mab^^0YYP;XfSt&t#aD3ue_M5YxY zxX?>R2E=%sW>ruHM40vy2KMDwd}fDmap$r)Vef|!T>L|?5I(NvmnB@pC;HYphm(^9ZGyz|l ztIhgp{UbTy%gOj5Yr0HCvGRxWLI;L%YFPzV9=2}_RA1hGTC8*BnrL10+G*sPKQ*t3 z^^tg8Y{T-$#kue$dMGtDddQ=+)1T)7E~7K}Hman);9>-HdO9G<7RCch1(Sx! zN;G&Al_=u|Q#8W%p?D-x@?4rkwyTO-#o*o0src?2fDwXDe;V6Ii#?LcVe}(cy89JDn7{_TFk`x7Au#TP@Z)PlUWMUXWVuChD08 z-|?ET8-C&!aO00M5n?<6tBDi*M{REsA2vlTUOd01-xIPY_+X|x04e*q|7!);@;*V>N8;YB|;HjdRxRWBe>;Jda;1sK51zPre?))3qq-R6g9YQbk-BVBdd!JNajT1#PD)H8WXG9OaJmI z6nwff;(2GP)MTpE+2GQqiIHnzQ{pRy zZBre_1efp1IXpClk*;jWH}+~{Y$+SH8xODhAphl1z*=6PEb=IOA;3+@S~flZ9=QmhCQUJ+BH?P zNCB)$Y8mvjx5a}p1G(q&7pp&LB)rr|dkoO79Vfdz1U*MzGet|4+>;@Cv(g;fMJGF4 z<8(2+f_~0rZ5U=Q;9N8a9a8q?$s1`RGc_*<`f}uS3M2gc*5^*pBoGR-!P=uKUS{pcqN(X5eCm)F#-Zh1xy8?zcnRW%pv)3vww7MAVOuqNwEBN02#b;9d zNXs@R#Fa!@!ehHlMBiOxrE|5u@FMH!Qkh5TlS-@9nd`1&YxZ_r(YZ_Ox&GXT5$HncW_L1pDa;Jr`29Pd303p1ESv+L1QzBfi_(Gyth9>1$fNCJIR5oo_E2-=Mc#({%=Dmso(*cN?{lD+DfnKg}a|0#w<$z)wbip5T{O((DPg5OFw`0f95~#8aL|Z-qRibuXP$> zf6}D<{TnoNZ_RkfO(|bLPcA1^F{>=G%$+_JY!59K`vl#TUg45b2G&i>uTah7?&_oh zJ?pg0QbhSF^43OuvpD8H#f~Y7oF=_z^R*f*{-X0h}oJU-Q2{f;Z<}|j>ofv`3 z%{b8NTa`DrS>oEX-scmjiGd?dtSF5_DMk=clm3_s?vOwn8?I4?0=!=k-Ugo$RLMh3 z$3a@34>@i;58J!hS)Qq}w0?r;pJ&a~yQx-kNAd|c(%b?uIT+#?sSRcFPRo0X+PBad zhQi@-8BYqR8jjqI&MDe>&i&@+Lgvuythmb}Yvpnj_XA{IpJ|PCsY!AdxsufkUC@V=aGRcaielu9jF))mQc0_p^_e?N%`4UyOv!-Qz zlJpPyka7YIEk7g6?Tc`vMvjK>xF#Tk90wR0%b!NgeO@Z`9FP3v_-W1l`5F$;B;K6? zVW6i_Xih@x;wLwTOj}Y2C?T#%8BXhttL3oBE=(r?+QEs zzdC;2EjFzEllZaKm^!rm6P)M1pylHa@CJ9Nd_M!g*W|uew(vjG7D;8kKWRRQ`|tM{%-s|-Zr8ITv)9xGmCErRE{>AHAr<9L|VxCV**Q! znR-Cc5%AT%m$~|yYXM(n&F!`ACE4!z!wY{5=_QH{1X)KezQ<2n!zb3&49CayOd5|! z{;WFTZh^>0-k_jX0zM6V1aowk*1FOXOy1kjtjPioGcaj+d zuUYS~Tl1xCUXYEIuw%1|aU=K1z|O79HvUyR=6$zwm2BMYnc(Tn6HQmkc>Q&0ii97Wr%i@S!e4J-y0~=U z_YbAc0hb9F^Z=_YHRN~ddw6J7x~Tqw9seRj0Oe-qXXcn!HMfpF*v%}K!!U*u9Sj1k z^{s<#?~uA}FO%C7(m;vI7!J8rj!?Ze775xTfO=r)JzpzK-XF3FrM3mO)kpX3844M4 zxYy&4Wk*N~RDr}Ezr`H~LTHea=-`rZNPYk5@b08x(uy1z{k(Eq49x^KhjRU&V6b)f=gX9OMPlx4G4w= zyX>dVPQ9WE-|2(1N9x_3O_Z1fbs0r#@IQ%Ftd!i9sf5@loqK}+eCzTdRV;!I`UYVM3OJRfbmx6O34 zm8)(zZ^hx;@NL2}1>xTS?3LT2cz?W-8rx2}L_?{BmzXf6LJAw+<@^V3z++VRK?njS z58NsTkP)wD;HPT%02Cj!Y_{dVy%}2pSCywRsZ96!xesOgHyS{OWFL!5!{A7wM1iG%XaLh9*Nra=KpUS&sL?0 zsoTYA&f%?n2CG)Hp*1Yb0~yjXXR9#|a;%QW6B#wI+K-$mP}%Rm0D^svqwx?X7g)ov zZz};8r=j6&#H0p}rWlOPw>en>=96j{RXx88fZ#wS|Fid%$22USarHnsu6YrddZKh^ zh*)tJ4ffTp-1q?6HPj)+08Rk(o))cJHil?#bx9;?%gYy=kD33r0(72Ef&i$(NZSA; zUp|>XAp`cMacZYCbFc$Af&XH(e|E6LR_7)g#|8~-+;<==)17$&x}x^}z^Na1KQUhQ zmwTTwYxnKuEVC=NasL*uV;>T5WkD&uI4upl*vV}?u7%WwuPK{L5BalQ!9vPiW9%9W zWI?3%O2`vfb`RkaU*%^DK_)f|XJvy-6NiorVNYogBU{)0kdAFUY(${Hd^B;3*UVAN zzeg8boGYZ6#H>-QVal*A%;Q<%qCQ|t$rFv?Cm@9*F!xj|WZJMAQ1b2#76Xa|UWT}l z&$2#$pAerd0TKc{Qq@@vEyBM+EO1yn2(O3!JK6pL;HIJZpUbJ$gEy|Ou*HbegXG<* z$9-rhzWh#CGO`NXuEcy8UePO_rkBhaU|)K^6^eHkey+MzE!ir4Zw zjt?oRbPurC>c-|*I+L~|7}cvETkWOyocu`pXyI1U=w_gNLELLX!IyN)W4l-9XVwj1 znS+r9yjXueG7Q!Er^XZ26cx#W^u;_*xgWrf2+qEpg158|GaV!wkjbB168FW3xldB( zhT!g?IL(Zp6I4{oVPxa{8zb638mgsNctimI%;T~l_T?;mOP}tj^e%#!^>P!(Ji2NN zSaDO|z-(>h-V~5Av}pq2-{v&nrgK@dt#Wu7|~jp`t9PCvHJVCEI-hn_=-A zTBxuxE$0(0D|z^kbLxkZ^FeWlZ-=8_i$)w`Ud0>S2V1!5zwi zBZ+T>I0a1;vz()m?>S4)-u2tq(e&nBcYdd&s|nb&hF@=qenhx662owK1kPZ*3Dk4{ zbiFTol=U}$hR%`C#G1M4{BLa@^grLRzcM*72iB&E!|s3IU48#e>*kvUhlO7sesc`a z4kxMnQ>oIgeij59bzY7913LDc<@mnyVkp+W6i;)p~~& zG55@1HNVrTnH^xYn8ag8Hz*()&H9y@f5u{Cos7}mJ;I>{^M_)}%94B2oYrS7A#DSV z>G)Ru0!PB;`JodD1XIe#`4#0@yAK{NLsqcaZhETb8?dtmRuZW@C3mIrqmTOb1;A=Z zt9c1MTk5@UKMxA&Mb?;N7l^V~Y0iDDe&^+w`+r(|2+Y>`u(fpv(2|PbFUc?{?RZuW zY));RaXe^QH_*Pdkx|lk$1{z1W60=-3>J&Cd`2F)iIZMtD4v_y48e2?I445ng7GL6 z*VG5@!J}Od4zFIZd^ue#@;wVH`KQHn9jr68L!mt7&UVa6d<-ZtzAKDw z=tn%{8=M#XDC_m-Xq7F2%#QMvw^*|uwzd}yIlPpeZNmu!eb3Wi`YaIbC*ulM%_igk zK6oR&k9u?L7b3HOX_|UUl@guYUWfbHLqwi7C5B`cP(M}C9j{EYBEn7)9&azXWj_JM2ClM|F%a1a)%^g^Dx_;Rbxw{wG$O`_Fw7Tu-X;nAYGDw z7Q{(5ALP9` zSY_|?7fGQ>vJr1QYF)u|2th*%tK|3inB5B-n&_tNJE%*8;L{5In4oN()EMv}ETAwV zf2sc3?BFPMJn3NYpZ&11xa~%P*Hgn_3YK^+u6y+C-XH4S8(DIY{* zSIPO-&#Fhl=e(CE5!uiIg;7*n`m_rW(Vd3uWqVSX#QR?#ECG$^-ZoqXp6;;`Yui=5E%TTDVAvF#PPGIwt?=7FjmTa9H5&`^mh$}FoX@EVt_dlqZ! zwkp5@fSF?}qMx1F09~Jda!vd*qW2vb=g|evqdo1u3D{UXEa3`7;lx7JB8F*w8KmHd ztX#rj{LcW(5nd#{n2hyr1A%wz8gGzu8@Bkz@YC0IeyB6hw~S$NNNP=aQCF7lq=Eiy z9(Z?q!gS5y2VOfZUPFpSN6!ZT!N@kKiv(7I$C)aqt@^(K;C5eqxtlT-{$H`;r0RaK zXQlI;C5OlvZ|I@O(Db0}GkSyn?J0Nju}PyB()Lat0I7-K5gt66bpmn+LHtT9nZ$`G zFqK25%ek!t@=>VU#M$gB7o`V~i?4GEZR@kKu)2H$@yvs+2X+kCXe)KsNsMg3}0f47{|> z(8r#Obg)9-n>^}BB?zIEn+DJCu^UmA!v=n-5;c--jM=B4FL@4U6F=cO^nYVHUl&se z7sK_=uQM+o+Y3FP*1AHY;4`c?>kp8f&^3Rdy;P7$K(XBehW#^gf!dO-VlAQ0OY=bF z!ltBTa!;ZwpLWqiap2I|FnBe|*UPXSer1p|AR#*1nZ;NA7urq?b6p_b(R0%HPhfak zCd&K})m=C=#=ndRsK(~{g3aUx?V8fnr~r7<<^qc1NKLu2Y1b$3r8l0|$ZVE3C<@L2 z1(HiO@f?Q-M~^C(*WYP4$+##pj94sDI+~?NPEhb*4<3F|S7@c@LV2|oZ(BX!fom={ zh+=fwz4-n%TPlF}DzI+*Nh?CT$$feKhY~wjKbOA91MZ&fh&6)0K`83aa zoMa^026mg?BDF=C_E)sJ*trEWtYa8P3{f5qG$kOYFh9Sg?hd3(=YLEDj1`*RnTB>Y zJ(Oh;7`E+R{C-ruji|#~%U9ZWr?gQX!j8iDXtgo9JPPa3wx8D=m&3dvB%4$K#W%0| zNEA$=Nx8^hauY_;*oAIi&P%0NNuE(YXA8Ag+M5m&fjl{j87R3a`-5$8-zUjX)|$e$ z7oN^taKZ9_9NfSDe0WPe?b%e%pU-(AZWIpwA5M+DfMCievYYFP4yOuufq)g}#BD+2G{bp1Go~`?oC>bl9rNy` zxW`M*2R);$3{nA{4PEhteOsDgpPyUVs!(tH*x7--q2VEss`Iau0N8tf;~=z$NlZBd zSl{zlVrTQXVAa+9WsvVtNu4%hxTY7W@prv`TfCH>On6qk(z#XrB=D)yJJJ~)E&Xn4 z$^a{eeP+n=!tejDO3nFZCeXKG`7wW<`#pzi;U8$&;+!7i99|9%`kAo|c2)729%e0)pm?__Yl?-M@-b*Ft^Tx)*ydK9#Zf9N59 z5An8z&*Qj%|EgZ^Zv{m#^qs@B_vXGCT>ZS#ByVWDy7H9=&BMA6fBI4x!k3dVqIlh- z>5#u1w=yTj2$7Gcn=9EK_l8m$9nZHJj&0!o=#AyKz-$&HLm4v2>*$IppD+W5a){a`;|~>@`+7&dilq4rn9W_}ekdF776)yq`Ohez$mQgnOd>H?wdlb$_)!{ZPf6Y(M3+^;6pN1yB#n$P9i@o4ge?l z=n5a9R-?$B!P^6#O9TD@`xIi>@Swa}?72;@x#XpRo;=fSQ!@Rv89e(`gAmax+7Fs* zIE7)(SDd&IF)dErpv2zw6*tcLzbdmzz}~y(7V1EUk>OzO=oL3sttLY2nwK-CHedr= zbmkBy0qstC5YXy~esS%3d;NtL$KIPPYc9GS8}v7TX^&AsBQ~aG7uNs8(aj=-Oo+|= z3#r9595eWRr3p(r-AU;cgk{kA-ZJOJW0V~u79n;YG7Jlu$8ooWnxuwuUOUwcb2<)d zVZ(BZHXLPhefw2@Bou}!B+7QD)VU|fc*B%#6OT?(7d~7eoDDyvHQX(Py;aZTOmltg z>Hg}VX@q@ZvPRtJZNytv7|(hufySt4LF1v0Bo|0?n8!LS?EGY2he)2=90Zs}I8;-J zBj&!Vg?zIxV!D$hl1RFfs?4@*MO zqGVKdd(uKNL)%kos_K25mpliQE0ske0~R;T1@>3`qFs)kbqurtI? zRit2t;*_SV&J_^fI(b-3kTDiHl~gU)r%EDA#JPKiQ}2`EiC%R;Rpw4t@=dZV5wb5- zYL}XZ%FC#x!GZ1rjDiw0^P|RsuTqvVYNxY)=@+(xta_8wci$EIm=uO7MqwdMe#xMV z@(}Q#tmU~UG77OSsLPe_rYk06vs+$rCN$-#!Nu6Q8S2L|uHs@Mr10n}O+(W*^eo== zBzHH@!O#s6B@Fd!pk`W@rmv#YPZ0^=ZF1d1q!eU{(|Lm@gSpy%xb{xdTRfC1tMz49 z)xl%IT;PH9pCav5X0^52^ZI_;>DstfMIx?c7as+4hBfCV#tCn!laB!S6n|AR_wk=GzCxQclhv=PN-WnXM6x646Zm^1HNeVuCTjLRK2dS zxumfY(RbxjhZ{O#Xd&&ECPku`ep^m$`z?_1ij^C5Nrv@AFK-3oR&aUm;KSU%~&MMeLR&QmZ zPgrepUsBU#H9};k$kOwx`jHm9{;pIw**y7nQsbO^9{&s8AMf;YGMKmM)VmuID*N(y z>Dt0tZ(&F?wMsO|u>&_W#pLbpnu_{}bKckST$jhGylxG7 zu__5P3}@4O_D)48x5LawUyXZ5MOq&jE@pRls-t`JNd@Gq9@uc1FMUO0{%{iqDA|vs zVsK3{N3(kX;-z1s!+r65$?~hSZ-5H>y3IlH8HKHKT8YPvrPXy@HV`qCm1d7}2s3$srEd&d_h=f{!r zFS8HLt9vUE>-+Wv$tib(=WD>0dX7NaA0FJ<>w!98nqk~PXYY`3&xo!c#RN!V?|XTw zS?cB6bJN3-+H1abuOa#;0M6oko*rs){{SrX;Mc;zg9UGjiCC(R#+*iN2Kr^f#Txx% zd!QY@K9X;iQ6-G?(k)DeB; zFh@PCWmnW%VTgoEt~AesyQHO?$n1+!Ep_;$6VW~7(f^lrwp&ymXJ&K0z3C4%$E`I_ zPWXh}=B^@9MuO+$4)QvyG><%}%V-9${nNrAZpNSqr|YYvC{p{KpK0|icy;4#oaPq9 zJmHahb}InlLKEZ{4$Egxw#5S0%g{6HL&mqzr76b`NK$54C2*o*J<+93A@JH?IaRd>hRFg5hSt7mWX22iUh>W#j@)S^ z=5c2;_}Vc(m*PQNCJWgYd8w!zw>m5+(7@_4vy$8=Iv&7>wdU*M%IGm!+u;(9Rk>ehPC8RH8zukF`HOw9D}@vf*(#^;m#p$Gkc6ml>x<6M zJC2a7l7b3Zk#mB~RujHiiL~Tpm)Ay?@4-F@e4l;_)BURHB^mOEae`cltTr$&C4d-f z5*h|*I;Cc!%1YPR@r7i9Havu+qLCZds|R+YTjG1yv?vmVCRp#}3_U)hI$#BCSICOT zMRw$nbh3I{3EqjZ3+W=c_yRJ^Z2V{dBqkWa+_9qvuVRSUU8j8d<>oq6V%E|V) z)Wv3mjhybz2CIeW{}1|?vC|ySbbnr3abLbM*`qHxAbiN$6Xjoxtk-If^favSaWcBZ zT_)nOLlUA06e#)j&zZ|MC1)=0Lt4q1BQ=auSG8Ng=j)@qyodk1tpW-HWB$!6XMJH1 zS+g^3mqZWZHVA7Nj7Vj-LHIV{$)%lsF8wX-?8_uD*Dr&*lAEzB0Ow(YKS2q~$fGdh zLb|(Cn~RM;kna(}2`aQ!wzF>pan83$Vie@ejL8v`pA$Rg^v+qQc1Rt?O^v|RZXD@9 zQy;{E@BI{3@?;4-$2ufXPb7F;-yTz@V#cstfpPwv>5&als}n)%yF24=`D5#T`U>?5 zkzcw7LZ+d0xQA^)1zu>=jLnAhe-g)oO{|8Trz54(F literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/compare_result_pkl_md5.png.png b/debug/accuracy_tools/msprobe/docs/img/compare_result_pkl_md5.png.png new file mode 100644 index 0000000000000000000000000000000000000000..81ba1935e69218467b006f05dfffbe54f3f04cb4 GIT binary patch literal 20464 zcmd?RXH=8>yC!T$MPLgGNL4`)X(CNp5D^e*QWWXZqz0t71Z*^EQlvwqHz6Ppq(-Ip zA`nP`C@nw;AwUu!giO?Z{%4+7ffzf%V&OdpQk~r_ zWIfhs;&J-z7TDeae`e8Wui~_zAp%W1B*QUbhuaY0j~FrKP`zRcR{lp*$BOte- zA1&R}w)t5_vPkZnfE*~(H!0KAyD^p0D4tJV3gUKNUs0?;hKk8qWt0xxEKzFlXA)Gj~&H{ma5(%PDFUzA|wa&EFNyvZdiQuP_<-^EolIsdyn<#M%{h zp=+@lm$WEdCeWsd7p>3vFf5>Gq3b?drIsUOPbSQr!0Ns$ep9 zmJ=e#XXRvt8ty9As-SUSpB9ffnHSm~!@#{gNPZv5InCGgrgC&cVDwvdEw42{E#%AR zO_4THePt~kNq61`Z@y{ErdcHH^lG|gAHr^@%{7i%1%eN`TICn?3Y)&Ay0(-cy%|Oo zC!QcSdC8;clITt^2(HFD{`oq}yxmXgrn~yW-0y&Xfo~tKxWOGtN&sEmDb}8hVj?Af zDdwe?JU-~?JwS)vMJ!4s%P0w-?eFavyqLZTSz)T;h~>4qUWQQ7yIQ3C)IvFuao=ui z%m$4xpH45pT8_io=Vn-bD%8lsJ%WO~XcuW+j^81P!85vWaO*U&gBA`*!O1NZ-+r&$ z#-yEOBQ7uAGYCX`zUMqYRWnsJ)8rwCg^uxDG?m8nOT3u-H96>!pZRf zcV3u_ZUswTYv}4?tg}O(|I~aG^(u0w<7wXtcmipe-Y@SLj}}-J>x5-_iMHl(1dC9|55%?e%6Z3yNkxsf0KXQJ2}Qdz*E=>mV6TmTB6bbl)Bh zbgZ%KR;?E*ICtn(rA7}eNRZap+V<~8S14tdPGJ3$>)c#DKfE7@rvy*~h#dR-or;LM zN)$K7?+tv8f(xTxz0;N4uLdmpu zBLKG;F`hS4c2x`fteyr=1~?556rYb^)#__Mv~AHLJB?oTbR z*J)*!@_4mu65j$wcSU+QZ(uTEf!P~vT&R-gqiy+mJuBcMB;QEoc8tB7N!qtx<{JWL z&Z~^HuUbb1)F?H44%%xLLdL1S zw?0m#k|yZcPAw7GHne@+E^6Rvkn$~pExypN+^Z_D5&CNX%e_7{ufW2?*u+hG?1x=< z`!ee1;bKu60G--#fSrc!2Yk{2XoHvl6IQh@HR=^gTJH=k9aQR`N(dsbyT z`U)CEPrO8&{TnG;JN2-VMYLGOv{Na7`)m{EWGquGqeS!T+XRs~w&C;LF(VN@d?Kbb z8o-LxW&oOb*3|oBEcHr8>$Mzi^Psm>1K$T{`ZfGOcd#Z4_tovyi`Av)tj;|ngM!g# z5{C%HRC+B+gy2N1$m+nB3K(|&ew)gTwuynsDBy^lRCLV4_O~ialt0ZjJ_2G&1n%zj zMrLDkIhEcbxRXDIayuNe-^w%_v@|sd=q}Vc-7n*=)ue?DSDv5 zVAn$GTckR%y#8lg_nu8#^Y;$ZdX9~ZH#}&K;WeLCm(b#f~euqVQULCg!k{92r;wh`Q9=rAy+5p;< zh(B;goW>5wy2l1g_!mLKU}0O*A6na@fzEN=5x=Y;4*YF}AGddd9$_LNU-eH7+_Qb2 z^eFQ_OelR;&N=x)BLIR;-#Kll(nmaN&v>n}$$~W57eJEzv7$(Ju0Xm?iOQ)2Q!4>X z`_3E2A_>`KqKh9U#W!)=7OtZ>b+R&RRJlx09Df?;@h3|nIip3VuhNr5jueoe=(MWS z-nLx(b%I!qR*7Z&o>yeFyhxt}<^#_ouzu2DHtyou$63r4c7sWZV!Um#v9qQH;Ox?s zkCWH-^Z<=UYw*SoT}8qYWV1@+IBBLLxFLw8i!#H~$=vujknHiRg_5SNqiU-4CekJd zRm-T6unoL9TDgYFTucb4NxeZ&j(#bo-=(Ab+i=Cc{J}Z@Ted=f{k-bomgFd`^6XRf zLHp8jN`^yEBl1K(>XfQlF6Q<#Z@aMhE`xGKA(gwg6p~4fS7kVKI8GlkPd1@^msAVG z`ZeI*Z>i8mn$=g`?zmQ9t}6zJAK{$4l$C2lB?2b=lHyqR?o|uyRHld21V^eGFQT@( zgJu;PXN?lZc`1Fcl7Q!9K*ii@Sd_ocJ3*3O&;Vg9eCkgU8D*(Jr-Dhpq6di{OPy+U zNCMCX3A0hOJdN7s?j)QaW03>O_S< zd8XApToEC6FmvLn4ajBaRp_DOrL$j$Vtt=UsD`tlysb9f^q(Zr!8zlpNVyJ=%-ErHfHd?`SW4=vVoTs?(-n91+@#%>CZx!^ONBxJj#+g((1#$|%`G3HTK94z#Y*I(O z>OZo3XfZ9X85M)K@2@#Zqe)n(s-KuIV(~@t%T#R}g4ZLqi9wHyT3kqKZC^Ei$Mx>@ zQ0vYJ^r@TCXqGw^=0i4Wk->}FpA7h@^YinRr|m4kPuuZ6|78Q&r;zcG+0A4&LDuMd zR}=65=Rd2>tmsYDk@KviqrhRtHU_5o`I#SKoG$}ckLQpW;;hz5Rc7@RW3SK{()aZ%F%dOq1>muzwnOEnpr~aG_F>~uDuTQXttJ-%$vnTl?&BFYDmG0 zRM-`ZH^Dmm3=E8BcXAjoyr<9~hlc=Wo15LNK0iN*baVKud#7V>_4upai9gyVpFK{N zU0cBA?$=?fuC~eGcIW+HYXT7+i7uF1tU8hi6;3k$37Tkj53QhsaFr@UxD_zYD`~&Z z#UE2_jg8;xl5bt8NdbnZ<=nYO5F@CZRm2&9*Ka-6hBSmm?p`(DU+hMvU#nS?_4P`{ zD6r6rGBsfKUDt*}~`eB%}aa+|7L zm1@lKGFr~?89Or0w{xR;{>Ll<_A1Y5Yh?ko04|@cPB_(4&X_?Q=vYN~hqy8^V~u9Wq-7R>3W+ebK8$rhn0+$ zEh{dZ-&`K`R4E>P*0yawa6Gr9(q$X`2G+Kmrn0Z_`Dcmm<+HktiHl9W^3)3khu@8a z5;As6Y!E4OR0ZuSYD;VP;JLrl+Ln^{?`lyJeq@UC&zU@+nal%rbmw^=P*>8%&!ae9 zE6Agx8)@e@lW}=-<-FRVjk$YY&!c4CUmBZvI9fM}iCR9MkC5*^ik8G&%jqS{?suc2 zRfsN=xK*I~yxJOfjY~`uc*Y=-;S{aJ-H=S)4NJLh-C<)^EwfXx_oPzt%5{jUw0p!n zR+9%fTH~BA&fyySt>w0A;g!dpY4a?|X7%TR_j{ zLlCcaYmG|{|2lEtd}C`Semr!TJZ?^TQwMJUp;YGYp;m^@*70kc{Z6T^%0UQB-VpQL ze;5B~nsdZH@@8?Oz@~ ze8&Y;FpnUtw_cZ}ET5R|>~iBHHbGU=Wz_QX11jDuU3RauZqfV|e5#NIktl&2B?b`yI|rC9x#sY2RpLkQch z!e}92E}Y~Bo@V1ONCwwFQ&V2A`5Ew@w_E$@o=M+Wbp^$H`>13@55YGD{OX#=FybQf zfl=t*h36IEG}Zk}R~i9nc%O&J$itKOci7Cqr-8clgOv0%f?pT)*3zrcGmc7I2Q|!R z$Xg99!S2ZmpNgyE>xQL<=04ny2PG9}{tWyPV%mKPY*P@{a*;G%Qm}D}jr*B9lg-sS zj+!OMQcp2Icf$s(j_f$|u|oreG@4fQ{Iy{8Ruo@R8v-nrIZ1`4B;;Q5eiF>Chv;Z1 za=K@frj^U>*X%vw5{9%-Tlq2E2w+FE6dKef2;#@n#`YvX6d6>R-i2}goG&K<5Zk%M zleR+p4MCsqzVQo>=4L@HnLPgIg)apdhlm37AciqEh-oLoo${`1wYAB)_6hR%$54!V zndgvr(7Lo6L0do^ zhLj2$jL;@IAHNk10)(3jNa&`6#v)sUlc2 zZtdo}YNri0tTT;42lI=$fo7v{%FJi&oktM<2QbG@-IoL3C^mvs8CxTtJmWE9r#(6hxT;NC8OO7xofUo_E@3i@DE*HyLNrgy&efOr@376X5g z3Rv}e(8f6$3=!>opG~gGonyQk2)?=#XK8wnP8P0dSW&9EQYT#(5(kd2>dY$4?r;u< zx?tY49FA;+-`+qWd(jgVD>q?c!g9bU%`xt*pT#Mt2GKS2@d~Vn-PqCFCateGi2wEL zATIm#Lonr2pd7Zm zLo3)ykX>tva;N_Q`VmcGWR8}DnYIWq2yuTG8VSUbNYbm$v4qNHa?yb7pn{TQ>~>q^ zSHL8Qko36Vf}PBWf28N`HTgSFg!;0Q(IBwn0)tu+_-p8_*BPNK6Q_$0*H_G6)LkbQ zpzlV)%=i(cigL$DW_qa_EM1N$FE-M{<}CsotHoAMnNWhv0vifA_iBfERgbMJuQ5=S4Y zt?M4x-{VHSB0yP9bu8i(m2Iv3b(ulzg-h1gr`@(h+8EI%7s%f(lFaHzOrmqz^**lD zYpo3RZgcp`L-LfZS)O5;N1i{udhbE^W3Am{@_39LKY`xMJyRtvM(^9Su{;2{ba%YK z_3k~J%^_YelO95>rMlT#sd-a9V=v88M?r>LIb+X^AwW>hzDfe|eK04x1()9w*lI%2 zcY1-o=Wo5;dCF@f*~wL_ZB-C?eZz1iT2g)LP?V&J;l{l|EVaxxcZMB9hUrtnHImck zynMOu0)^2hl0QTa@G~lI-0k#CT?^#ce?UN0U6;!2YHyS~_!~XAN4!DRF9e%s$)U%Sy=Zh1d9{&FnVlsOnrWXK z6lppC=(*)4spoQrxJdWZzQWtHEn+#U65?Rj;8pwsbKO&3meCbOm72V{)m54;XNdty zg8{{Q7EZ3iXDRD>lc7q>b>6L*v8HFI{^bW2Q)iP}_#Z!PVBW)?=mv#YRagjrFC6Py zFUeIlr|CuS07k&9lWc79gjtv{Cd)5ivf%sXPu+Zt%8h5uaZ&}fs@H688=*yja_gW@ z1WXmS(K6dI_qRhhqh<@bH-0&y8@BC)_QWyF-cpi55**K!~GwXbG@C^l(ruLwXTSEdN${;r0h68Z#O7=D754gUYV6IaR+dEC*}Qi z+n^>TQ$T=%B`ItvQ1bSp~WO z3;6-WV4lJ?LGU*K2d_!wFIfy}^XD6!J1OFJ%RZ(lghB|~4D0*t9?JcNYwGh~8Aovo z-*)GW0e!|nFwTwug0z9Niqo23>YWm5S-h19^7ggNlrD)!#_FpFM_pEpCw1(!wjD)< zbfhk0z>Pzn;}0g@RwlKciN>#g>k>T8BRGvlsH&ziM37ibF*8R)RYk|5nIQ z^(^pS(4_0%uJNW`v)~EVeQzeu5^^m;ZQlV$FQ*PJo%i54FJ;oWRkg~N#>@B+3A@Ss zd{biPhYlkXO-X*N5lQ3TVzYUc2z1Z5u!-tgECN-m&_zZjYPX2Tgc1QKKMJ%;^ zdX%KzalmTN(k0{RsyK0;Xoh^pX>l$^x&KZfnT+~EHs%%H0mtnJHkSY@o49o)PqhVZ zpx)h1&1|tGlRKCWO>ZauZegn>Ew3{c4tFy~tPjL_mkOLBcv5pUHZrLJ+spflO=XhT z@!-}J=w92o-+Rvk=NA4b>Xx=~cfQu@f&9o=NoiX2MX=OeZiFByl`z~DKI(f$qhy+u z8>c9y{gI-W>e?A*objqv53Q}hhVdG`NP55H+L5gyJF9hNx-nmM(dc9Uf&{P&Xi_gd zg^i=kx?-^J2%?#8ZorvbL`6-=@i=$X`jdwgrXwH@vR zfOF>~cDA=n+CdN}iAQg{;{zG7=h62FCiy?xk?B7UCAy8=4FXx}yJPD>#X4U(97F2m ztdv~&t#|5+{z3)u)DfM#Jqwua5HO#`DO_1IFWvV^{`b1w1q6$LS)G&hyOrH@#nz{K z1v01Y7ZP@-O%SSE{J%2uL>4*R%a;iz`4cS3HF#0SCMhV-MQMDV*Fg?x?=#Mc&!$>q zrz9x_ou$~Agvg&v@W$72my4vm*OgE{#wV8aGG9Nbw$sM|uQfhpVOs$toLHN`kmvN} zbYBeB+vlQ3_AM{|E+K-N*^Un^0(|ogu#oP5-&nHF_36>EE2|~v?$@^aFs;F^A*A(~ z%pvW!leSG&!m?$U;u-&}y_pL*o&7~XZ0W@84Y$)&-ZVXJEV9-k&lkeOasc&VrzPa8 zXtDG62@0+HPIR-eY8j54tn1SnK6ho1Up1zxVPtNzNlQrZ*_cVo9Gy!x@L7LQ3g=JR z!EW}e(KC5W=b${ns*-1B~%9-IE}cgO%r1d-g#$7_acJe9Tqhk&1X{?H!D8>MT$$*->1j2leKH}zc@kyDAc$nM+v9nN| z8{}i#}B@A;C{km%pf z#`{m=q>s%9pPnmk()d91Ir68%-$Xrqgs?~;qQkihhEcSf#31aFJE5p!JYt(1;QhJg zN^FVb#7^nX-hN1zjva@XGgU#ql$yG*(uQSI9F)@mydP|d@t!Lj1h>9qvT*7D7vv_n z%O_K(p+@N~(-R6bO_+r{Tl0}k z4;Z%=*^+Xd5G|+r)t`FsH-;C@{{IQ#Mc>=6>X1K>yYuD^_zr6O6<+Q2=iBnT{F;}o z_vy_;Z9#G6H*a)glk0B?QG1pSA^y6bs2HEuG~0h6kFNp0!M-Cv(T_%I9%|zc!3I@fgs=&d-&!j2x~=c4LELh;MkXipk~Y529HSXPz5o9_~&$l%|M6y$7%5X`gMb3yl0P-IF3=- zBSyt=Evnwt%qd}~wW5mu2rbfanC`)*T!DPAo%l&EbrizF%P3*A8O+?cB~-=M4e-#J z_1XFg8N0NA8FyKtwy@HHz2wHUIB*TqYDj%eP$=eI_9}rqcJcKeTR!XCHxTu+|5JFIWQjaTrCj$a84|@?-jD;G^}07ZPLjJP23pn%}3hQy}Q=k|1Ggm zg3*S5+Si)p-wIKs^Rs>IwNH#)uCzFKY-*YZq`sbC9chD2t6P6M6DqZ9a~n$2>dh74 z>eYPgb`IABP-?eIJCL5>yl9h|&fM4UwafD^?qgduHi59cy10>n+rY>T*7PHv%9QW* zJO8CJ{C&aTa_AXf8y(Yg$9~_zo%5mfeA`qF)L+b8>3wJp7_OUqJw~dho%~Pvn&M zzMYr<*z^LOVW$rRj<5Li-Z_qHS7m{|8w@O}ezo69*KVPm;ij=Ejuop4_trxi$e*p! zAY>_*y`Acj(L;gH&-x>iLyE#~c<~1d6IL*zlpkk+=jtaYE6!vc_IM$bdg~*lqyul2 z?biY@mmAWBjd}SlmBZ%;!^^GFJPaUscABpc9rjCTXa!%K)u`pDH*3Blw)B=6_#u_( zrNn!9bC))A$bT)4WAuSs(sIqCE^w=xsSUK9NXL=$huf z&?_ZZW&QN5-h*K{&6&ruqqJ#N(5q<|(RF9nmQ@5VF&NvmYvyPE zButfT#NT}P8%2Qt8NCV8x}Q)q98+8@)U1u|Zw}rj=RMBYZz-hPL96@^D{&?3*aK+o2J(KT;tz?nZRO zUa++0cc)a!)BC$j5+UFPPSqm)SDsEp>I@wRetQ~^>EM`PQU!yaAhl)L_WSPnn-8*Q z-7_VsIKDIaqdGgnNf+Pv??%`(tz1Yk6h2_wE3ARKD@#?@NlrmNnXz-aXJQ}ir{zzL zsz#gA*T!&WU@f2uXZon0nfLC4EPxg4QhDYB*~-q8)ntPC$fJMX@97Bk7-qn<#=!Aq zP-5LYO6VfaEj_)AQg^rJBYOk<%%N(-FHo*XMkqd)T#Cq%O!f50lAf1`Km{~`me#3G zuMWm*m4*8b!75$DrN#q&l%N-DQy0%8Gj@laO=dQ0GsT17^u}>NpdnJU-Cf}&JBe$5 z+R+jF-Jpry|FZW&Iz;IR#Vc>|2@+WjZ2oK0wd-@{Phe?FYxmi2(*(CnfMq3L^uO{J zH0f-xawZrFTgUPjSo+Vzd!D2jFr3HGSsBBYlr9Y%$n9L^ME!}ydn*vl*z&rCoe56W z9Q_-VIW2m^-Koobm?_{2f7%M4NB?sxY;2730d2O9VUl-$-@-vdc5(|+ z{*s{Mr{W!FbRm0)xT~WUZr6Gf-W5bgRCpwtB5n;uEO_{wtF$tOh98!7m%bfW=+0eL zCwY#FFCn^ir+tyjp?n1ZfAOBZrAEl^)p{kxNh<6E5l_6e*F3Fg1pZSkpPo~Na?Nb| zARO)jOZ*ck+y2d&+VzmS2H^;gt5}*|+^_Yi*Y584b3j>2XT%c63(LJC-2$-Lxp#1? zAH^rQ_`p4E#yYx=r8PP5YpMo3nEx2JF>o8O>l$qp^h z4C?|D12>ISHf}LTE_mqlPx@WcFHX-MHnEo}vzcQX^y2)s5p+=*E*C{_Ty5dJ2KSDZH~RFhGn@suAb37b)Ts&d$tr+2R+S5-grRQE|VxBeQ1 zOJxCA>Q-o=xmG(9d0rN0;dqi@;QDdt{inMkLt*p_F z$~Cz-HY`}vFiWhoY?R_6_Z2tz5+VAN(GXE(e|doAHBEi!pHuM&!|%BKA$!oWhDXE0 z{M!xm-~AV2`uC_?h<^a0OK9)ipJMp`9avFcevRq>v$6L7z`NeL@2dRL+HT{a@2bcm zXqIid;x;YIF#m3nQ$}Gvnv3cRdUo{AL155a@Hx=jpGj9Vom7>kpf7)JamRyK3n5+rl*xwKt( z`2M4{cjTf~Q1&-Y&*_;{IV3Iq=2yN_cs9K3>cf<7{{>MtZbD5*8emA(LjoT2bwQQ! z--u}bbhfi#`%V}LsRbY21-Z&Y1PQy*fl7k!DLV`HJvuEk1-13IjH)THDG%stNbiVr z5Y?O$DcYhgULm2OXX8K z_3H0)9QSB0pNd_MHuW6ObJDj>>mBgPO@cssJ3>ojF3&-=qf7nX~%qpG#PNZqrp6SKg*grlWY0naFON zd9ftU*-#c9e$3Lfu&UbbW#-?Em}vqtTt^3T@Pq)aySaaI_|;taOe?3mO$ZCLEsH#^ z8pz*9<=d-EVD1RruHJu=Tk}nrj=K^9m!%Z)eIXm*Fn7x^J_gTF{lA*=C8GZ5iwriK z*0*+a`FNUDoT1yyIQctZ+T+vP6Ws+Q9#6e2yg(gM7QQ|@{GL0M#YcSCym)9s1A4Z)R8m}BTlDSL-&SXlj04W zC3V5KfPbnfl&ESSds%|fwqZ53fpI;uWt}9i zIe^7L?N2$?7nZq+ig&*S0CW-tdFj02PMp+dZa*x<9ZJ57+q@;d@U-W;M|7U z!w|@JckRIA+I^)$ar?h{VrKhC@o7(F74n1T?zwt)maK-L=MjD+vkR@|QclmwAK)7! zLuQ}$b6ASn6dD$Lm*f0!h4K4Ou8`~V{l+z2BU7Sr)gMQCkR0*Gq$%F_(0cbF>Uux) zQxTW3vHviRKf%Mo-3Y9*goZE zDmCOve%E0HZp$PKBRZ4z^PQpWa3bKzI_P?#YvcnIw+nQ0z2YxESyj+kRY{q+ zz0$jYBy5a>vW%-M3EB9XhDuZH^aUFz#q$_WoPRX*8;ttxcGD>fz|09fcb~&e5_d?` zL9KXx3_z|C-jBD)le_@OJ>V<2%gG`tcc3f0e8=L+)`J3V1@14#-AhM5XzFP5DBwNy zP06kpQxDbzxEf`9@mC5HS4{;bCYesz_SRD+-m`H5qE2I7()w|b>b;pC%?I4lXA@aM zxj~TUSLwW>g$(g9(&jj={j^t0;qYoIt5iF^*&@X_| zsgZbt5Y8t{iywBh)u~3eYM`T;@^bQK-PU~;gGB_F>c~@L%|PogXYyShG7z_jZ)vUe zD9RaE%|vU^(qwOp1246(PgLxg<)Ko|b>H`bt!@Zoz~`8kU1DuB4Ukf!o*_I5Dt=hv zRHz_fTw|p@pFLbod^vM$!<(h?$=mYxa@xiG1{lo8OvHReRYnnfK1cW+#Or|p>jW#;BQCpmwe)@0r1E%va(L|%I3n-~DQ!3_8 z2rVGj$rGo7t5MtMg;kT@9J&v{s*%R?c<%K@Le5*#}oNj|02q57wDhcoKyW zBi_>DZ+HVVd#g5Nr^4ekVxk%L>o(4kD&lm0Of90W*K_3Jd+qNFm+|djd%jNCiELm2zaCvNxoi3Nb*G?b40`6Q>iL|sU>YRI2g+S{6G{V@bd&qrtbw1!Tk{XT1` zV(mV&2l3s4gTzDSFp#-@h=?Xn{v$3%iD!1SOweJgJ5I^rNu=;cokM0Yd*L++9a1Lf zE&dz1d)LZ49L!z`R+n8KgYoV+R~m(BzRO`Zu+#95R+iJ!FPUAmQzvWEu^4%Qyh~9uSsGm8e z&$e-HM-vw#@A5rGIs>zA%Uwj$+~(X0XJy#;TXcEnilUC$m;Uk*l;zDPsM!+zLcta9 z`3f(rze3@v)KHt{*c7D?3wb-Cu!me|m0sd=0iCYp+q>ty4weY3QS6pneu3YjpX5A- z{1MTZMom4d%8DPbGDM$DYW$+Rdf-K8CBEx~!N=wWHXF-J>|H7h>0YQXf|eWtO8189 zT(G)0Gh2wtIMTB_`*9W7(i}1H!GEwOe}cxRwEpvKoGS@YRqrlGo1nIgvB!vbLf!nZ z4w)v%bn1Ar78Wj&ILFXr@GaQxo0D3@-z5Bs4CwZE`0{s;2B~Mhq>@EOtu?v_B66U+ zNK*G?PD2)=*@&gxPN#tRcumO8)a+&E;H@eD!*z-m&43-Y(9a5^y=@)c(^UFNe2~TW z%kWqJ82<4+d&6fVp}2Ar*$b6%&$`n0f=5m1LKuxvk=6bt$j(Rq{OzvIftX`O%HDGb z`5%FUE@P={=_fBmb#0~mf{TO=a!gPMYBex)k@vUzhrdkzEbQ}jx#V$U-WQ=9AjwD! zogyUVmn}2yUffiTeHg5ol_(|09eI2V`}1>U{IJ)7HfTAy@5l6rVPpYm=7BR&`zXDL zsra9CrpC>EWg91-`4_1>e`h;L6!4(t>Mv54Glov;e*G7z8#gUWx`@O_H6Ex~Dw(c9 zs$x<+2f5WV?Zbhpqk?N%d2!c=&HuLj<0`mAjCw-ZE$p%_bW!R){9MkgG z;$^W&W4X=0*x#Tq>qma^yn8mDqx*iAkFi}rndtM!NES+h#bF#qL_Fob*^V2gEYa1C zk{;CrL+dMfT9i7@GD!X0^KYC9ZItl(bRkK(|HG{xwfx6_gVM;D;uTW=uW33yO;Bzi zB#uXt@V&_+okM(G4G^Klc7y{$LPuw9_iT5?ZXfr1GT!@c#MXy_z5?TFL_ZLituqvMS=78#4J7WBekpOjyu={cSy$$^EJ22@x% ztZZ$xp_&8J#zo}KP0LY!Z;{C&r#Vg+r)orneR#h>w_M~5wxOb|V6qDr)XLQ?KhguM z9FkP5x(IxHaF8#~E?eApf$E7A`!zqLri;%pRDSNWyIXTb1iw8{^nfsR-qTiPKI?k! z0~yd0ZZ^8&>&aW(8GIfoJu!9IS7bNe9XeU;riE*dX@gy`Q%ZOZ~)Xn zIWD0`PS*k!-Oz2;Y)@lWUd`pP8d|Y#dJAa0EZDKv_&p`6<_Q~Q=vsudH#QygzpSo8 zm!{#>0#3d?ww30YpcuwY_z?r9>b@yR_R-SZlutayhrq&i*^@Xn4}08PCqjl}=ZJ z1$1GFo{wxQZRXs~79B6`Wgf3DzZ?Ca3jxaMv%>3fz|O?HefPDIu=3M5)e>+fdCALH zb2L9{^j5WGSU1!D4ek93IpwkC>orVyLW!!KFyEFeAhNG*F1U{K;;XwE0X`8JbnGtSGJNpSdMi@;0tQ_?t%?_;l7LXXn1h)tfq4-L=- z?x2sVY#WXKbBFlY0jd30x0Jk0H5|XA)GMb0Qv$?{T zZU@UNwuYQ`gPCo)RCm8C{+ofN%yNe^ywpDU`?PhQ_*tZZe(&+|LmR;0w}@frD>{Lh zEAZ#+bQbvX}qY0KDfBj>9yo3Zd18DUrMN$^%3;xoV(E@3@mfOZ^5 z<_7&2Ru+PE;AIU;5a(?+cf>F}jCMH9q?+OUUyn=tFQHQZM`_gGXGQ*Jh}f?yT#GA!lL~s#q7e52z%~hOIm2Q{l<8>JQV^Couou6P zOlMw;(`A9ylH1gv#$p2LX5QoGP|#=S9rd^vIGuOhFylBznd4?*b4x2*M=zQ42@48E zcD+);Sz9!S3M1ckQSIkLZ%TYQ-J$abVicbEgAE?(+T1RuYgDzCGoHN)*;{!#@6R3j z=}tK*JUr>jr;{%lI(qEg+YLa9lF0RH?HuCtbU-V`=hG7!N9X%EwciI98gzneS3f+r zc^SdZ5rYkAlV7x!2?7)Sw++PIx{gLqY}7n8Xl^JCd9FKizU$<_A9yL;|AU%;;S9C* zD`kRrSKcImx#^^>72e~kJeHa9ntCC!O<403M}q*w1O79OxaZX`e<|N5lL>{Ym!C2H zkk{SmTfN3!gUq3;VrGru+%*p>!<>Q(TvNN^&8(81OllZ^NS^z9hKiV&0C96R?P^w$u zlfJAr^(9)vXrb}P3FozxBplQn$E0ErW5ZM7drQG;4h7ecr%I`4dro zzI1mLaB^IeYnDH?HBAcM=9t>f<*Py^M1TLuU~Bo}d{i2Mjx3FQ*8e9j((Jx#1r-{W z8BOt2e51-FM5iCW8bnSntX`*(9?#@L-3}c;l^xs?@%YU?0)_sKeY_w{AbIW;M^aF% zdh-^fI^uz!5pbK+Os1?mT!drgdSR$Fr|ADclZF}TZuF+T<%$Qoxf*DNg=U+e#55OhjKxB5ehmc&@uRh&iG})&`#40w@WGqY(d^!}CzF4uQb_`?Jf-0p9dP25@Yk0+4Hfvi!UP!1tEA4CkCNGI$my1R9~7{ipR7}(W5#;1Ly-}1!gDdOD7{4)h%|9j}R(?-vX`f~j} zjT$D&!(Toh`PU;m{&ObI$nJ^URDyzLAPp#D5O2CKa!a1(d0C}+VRgk{n_Xz(6RF9? z?JxjavZa%uNx9{c`im%IQ}{)cp-(8_)DePnPvt#g!P^F9vN+p3!3nv9#MPPSmW#i_ zTc$MR^UdT%H{@jO%d+^=;jxI0DZ39bmF6=betP*ZOo++B$GId2*^+KhK&)>8bUT|*OsqX{_mQ?*PN&JX|)I4Vb;}y~ZYmE3T8^4L%!J8&vFH!8& zIl3^8@^Hw8c;GcAb6bLboujcaX&Z6=Ja1|)PUIv`PLzfiG~4h=KCoY*{I_L;-fLEa z=vKwjPd6sPe)J0Zsg96ZeuRwNU&wE9Zi;Wdgdv*?QYV{;Czt+%hyMGAs<>G8(4V!i zyuS45YZrPUs3O!2f3dB`qYU8^lHz$<`X9~ZpZwL~A(i4Jf3SGs{{=s70yM zf10fT{op#0I~2^e$#35o@yI3VZRpA1EnNrX9Q6DAXl~El34O+{3~E&+wxv&iTRCiW zxthmjpr+rp3N0}V*Qd5!`h6Pj{=!v$r+}R+ARhWON|E2%pA}X+h2)2B^@7W zU(H+a`rwvke>$#rLAvCT)(|x)=ReRx)`x$Ml9T>XkT2578u#7St|22?ed3)Bp(~XkW}NXQ!F{2j97T=RN0no^xXD zirc~#bJ+~^)SGl#M$j3?_ zSa-L*@0I{VBBUfml@A^lUxnBFhc7U`nK4y4s$mP!LutLPu*LH9YVkMXEk$jHP?(F` zuOe)!54Ao8^8qkTs$0LdCmd2dQb82iW{A;LdpPS`{?)J|Hs4Ya&FNbXrQypj=T6_? z3vhG8ehQA}iPzw3*C-wOjQ4?cqlMktuhD9{N&L=8Gew5QfC)W9oBl3r1=b5TOnK+F zp(vMoeJV2Kq;+q4qUQ%vSc9KgYDBM@$Lr z3~ui_c}vW>DGPt8);ba`9j#H1q-1lZj{;Q5ywgwP7F)P^G9+S>LlNaZICl?~Cvdq{ zPWG5&@=KC!Oe@iTAn;Pwx?-itkQGntsna?vvC+x$k8@0N}nBg|Un zyW&`xIHGN_*>g@jv2mi{iXsZj)(8zO^;&L|(}3+?v=WlQ>qKY){<--ocPWByx{7aC zyim>%%$6`wcknOD-59$IktZ6lTeFAYK{S&l2n*fhY)`lz4^tGmN8c07`qwU)O%WQIMoPL}x}eP|aO zI=8IZbVEWKT`)T;xa{?*l>H7ystJmq{ztC?*vmY)qHOovIgQs;(rUsnM?&AYQWbWP z;uw$wyy~9x258aeyt?_ji_+zMc2Bq}GyB*!r289t{?z<#50GYfJdSEk4K)IFrQ&_P zl@+c&4gq~&dwJ-M8q!(ZhSz)L{4coJ0sZuSfj~P6irQq;e&{%ose5ka*}@8VD5J7_ zc!y{15NDCi;w8ZpK?f!jRvj!d4y$t{JVs?*lJF%`UE+(1-b99A!!M7@LU#pR3Je+Q z^c6PfP7fvxc{@#bTrdF!$W%SVENVrrG{`b}F+(T+3nFjY9Tm*1lgC7y53-Y6Y;QigZ8p?WW_)`Reh^bv z3M)kQd#4I=RwwPF1}ZI*<=49JHe^Fttk@Me4DYZLZCOd>av?87w)DD=vJT(4{t2wh zj!E&PY*Fi*3=#|wihQyeZxJ9r?v8iC#~x(_taP|N(Z4r;omJ~wt^nQEJ~CrDVC4}5 zyG7jJrGT0#)AQ$ONRT!Knq7&nMWv_o>ac%bJ|FP%fm?0U1>GOt0#-Foncv=ArK`PB zZtKgdNeQaqaCkwB5C*NbfP-xvM+0~(?Q4RZW>V`E^os&Uz{y=^H{00JPO0|NJBuBT zw~ik%9JAwJ`83R#%}=>apvBkewD0*R)!imvzYvk7#AL;&38_m=w5f9<{Kau`CqY5_ i+{e%Bd9bDo{>N{qY?uD~foHP}$%fqH@5}PWp8W?urf}r| literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/docs/img/cpu_info.png b/debug/accuracy_tools/msprobe/docs/img/cpu_info.png new file mode 100644 index 0000000000000000000000000000000000000000..744d237e975e555160ecdc391810a3681d05252a GIT binary patch literal 33797 zcmZr&2RPg9*H){F+BJ&CXi>GbL8+7)wW|w_JsPbL)E-G`)v8syR%^G4m{B7(MQtVa zt}V7$@kQV7|Nmb4U2^sMi6(h|&pGEl_qoq`68Jz{^&;hEN+Keni)#0k9ug6eLWzjZ zsZx*=p4mzpq9G#UB2rUQcr+Ij)H~RUV8u76VaEMs#zr%MV9<7fx&HaP z#TBuwQqa~i z6|^~YZgYJ}UjCN6DSlz|SN`VxgZNETBJ%%xOfXO%4rnlgAk68IWU^F^yITzJ(BJa` zKfOTI0m6<*}O zbf~}f68G1xcZWsZ+Jw%zmp;GTYS9;ct&=0fy2326Mx3Lbh`Q$AkD6Y#`(*4WjAZ}M zrciHN?tx)6sK9n%!ngQ)mv`x>4djN-{`syH8jr|a|DD9RIm2vCuw z*<-y_`wQdXRfybsU@~hvv1wz_IF4ig*Ig2k=e6GD!IUXa-o%)F%w;=((%-?nNM4o} z@eLK{J1{vMqlxrl;URh)rL7-MWApa4v6o0k#71mo!aPwdDE-U8iwD5p+jI03?jCk2 zf>IdzR@$8%!`FSmQRDGbV_hc7x~`5r_6n+Q#Ur&nzb%GD&sl!-x~+CA ztqa8Vy))C|_zW$2X4@!L>me)$0bKBt&)Y_5%8JQzAQ-lpQwu45du|T1`tX)IylbE3 zAAc*=PJ&&K{g;msoXsb%NEd0)@q^;n#w3X7i_m=y-*@ESqR<9wl zOD6UGsSl5a&}(qmXK-tELG^4(T}+OtNp{M>?!{KKVilSZn&HWrC%ZZ~MwLIUP`F%! zJDJ4x+%n1CTKkrG?5|5taR&R|!P^iLU7WTzCRd=H4>GefD8 zbDNBqF7ic&#&O9$LiJ~ayiI5rF-Rdj;;`^ZYuvU8YxDD$99QzbPu~GO6C)xy^{~H2 zUTOMBA8OYSTqmi{m7PrAYZ(4gKybUvv6X6ki#nmBRrq#q*%fue647D{lvjZoXM@oB zF1tKXK4#Q;{b%y*`=nfNDkhCPP)N62n^qhBaAZ9W=i zh}INLC^(=5JXGyDZ>BxDtwMh(dE&yUC}OV7EUN)Gym1pzu}~Ot&IQLUQuUCkc$d9& z0Y{?)1w)92`T;a|ja$bu&#x;DRDscjEu;3=^!S@8sLy>tn~Lu7m*}*H=UU@40n8qv zM)TViE^iF_t8=%&`=AZi`(075Y+UTj+6=CTLil9wbhy*fp4tr}598ik=2MOibt&cniJKvd`Fs~Q+Qm16<-sp zZMq=zyzm~~i~U+{g{T&mj~{@Y{crC_$W<)~`L;7RHRIAzY4IycQSE}Pw+advZ}UcNg2mugNno`Zyz0eU$CAgquX4YwA@3_D1O2k{4n zuhG*Kb^+yE`PE?2*bR2s?;9aWP2I7QgRIJZPZnKgXd;di)5c}ZuJuMx)BwxT6qBIxv*!!7rvmqV%A%{ zQ=Qg1N4iW_mDUe556mZE?|SY$$cO%4ZDCg)1cOmFk@ z&j~)-y6#PR%6&{aYdlF4{9%C`zRH-;YZI+gKnhxQ`@mqXq%q=A(fK5z)QwzUtu#0a zV?vtZc$9mXw4!bs;GzMicOxIR0W@rBLd_;qXrQ9m(#J9>YQ=p#MDOq)(4q9wkR!cC z*+$)L3-@UBPadcu=+8QCt-ua_K&PxnL@xi@hlLXO@h3;*y>4T@oe&}V-;Wp5oP^x;qw^w+|MD>H`I1Uqf=)-e27}?WOa92QQ ztmY*}5=IYIr@!i9c8R%cU-k%%G?l-GiCP?>n)-nG+*=}?u{$ImElLT@-KWif_;xhW z82r#9W#9H$DGO`2%qqCSb4gxsyFz)< zF?_9Xw$=u8HJbf$wdq*Zt%cDY z&O8&P6pP_hmxENE{pH^>&=iJSKQ^B*l0j>Lr&BT!b!7|zMs*mOukM?3*fEjB2z*%o zq4hB3H_LKWMBsq7wC36w-bZlYQ1REM^@=He@@L=d1Yc_!S#cK$aIdYZC6~!J#eYb8 zD(p@&!debB7iL88w>|UEu&?=5obQ^Ay;t*)w0eFi-@O-~x&R_zvB%8^b9(%cp>_b@ zshqMleDd0Iw^1V`9{9in^jU6`XbBSIePhmZZwGgr4qPq?!6F{Nw_3&T?T5{I*oBLKZ1)!3vOsju=aQ{( znqKUM&I$08EHBxPOO;lQl$hRmfakQGMX2de(uXmqKRYb#0a&SaXy=R_?(wEzr;aU; zISTwC9qlDJD#WS4teItV`(WB@;40G0l$Ua4`DQ2Z2ymo)(_#F(-y)FsE@{jYulW_R zc-EFzBEG7NQo!#W9z_#Tgp?oJ^l?z%B3(pS@uG`Ft%^mnLtzoy6DHj@^wYh<0HedG zn6@bws|hu<^wlUF+9AU7V{nIyFdV#Kmild|dn#utA;y$(G5W^zz4XNR<#5B}v8n1Q zzzY))Mx3qsK^n2$oQ96V0E%QtcJbv-|Lo(_nD&r7D~7A5D*6fIe27c<}RYh z2mbb!?6b@7-xXH1@e~S2m${?*u)|q?(O0W(D6jRknJ}}!yAvoB48Tt8ZpU#K%|L3t?kD-_yE>9YlM@MOF`ywN4u9Zxife6U2j(45OP88Gp)zl2dRHe| zP^tLr!*PJOpM?3Ihr^)sa#@_}+}zQE-(vEEhg;4W`L280-aRK9@etD8^xg-&IJ#yO zT+_WB4UNXmQ0E}82{L-AuV^n0%x>auj;pdxvf-`&uO24Et52Kw+SS@jRK0co@#iqk zYN3&tmy=T$AKFdE2vd8HA^ijmFOAWrC8>A|ICyi%{vMaL$TUO=g?~8ola%|gl~)8l zNMAXa6%9sCqzHZF-p?OXVRn?-&$`OGykFT9b_s4YUfgr^xp!f5L?dCy>&XxtW!n22 zVU<4vI5sL%kpqCP9rI!gSDVdOrYUVIrr*ioXist84vDq9wS1$6mOtZ0C5oIv!EUQb z=ce99Z1ZLB3+v8q>~eCQPs+1`br5P|@uC1Z9^m&)&_|Tg>`pXup$zh&zk`4EDvGzW zey+lMmh?@O$v|g|uvKg4A~3_ytxqaG<4|M|yih1CHqsXrlQhl{y_vDx#?r2h94r-$ zLEIO6v7WyuTKpL>;L`3$5`6{azo<7mw=yj`MRUrzYid|dIN0yE7C@A8(htwwy2NtU zhJTy-zt@18nqV^aK#7PVA>6oQ1OM5i{EvMw786#!?S+?KKCkFJpY7&}T)V?~;q*uT zVS$zJd}1#Eo#t(BGl8Zd|Id5jyadx>?0*@yq2{er~sgm8f;@e(Q)AZ3nUzsFUeo*0y?(FcVP(#~MBArEv zw3R1D)m^&91+|Sya$AdnFTa<1UoOth&M&4Ju*j;^IG$fGssiG_@6?~i`~4?r{I6U3 zV9mKGf!e0xk(|Ewdud%LQ#D0?+#iwyGxAt%IHBBbxgJGBdPe&<9n zH+f8+`D)%6bv8D#{-P?KPF3x=6)IOmp{6vEjyE^+90NG!3<`D?lP37dDm84v-)VWB z&PfE*T9Fg#%h&+l73Q${;ux-8byr_4EvAhZ>(I+GInOmq^_9WtLSORBgehzJ))@uo zLIfcqA|fesGEjdB7s_s{DGLx>r6jji6FU5+aU(t0WR2?7LThU1$?X`Sc7qJF%q2}t zuN)Y}&FG)a7$%kpUsLP6UBpGl`9i4E68>!u=ihs^oNv4%YcOI(yCvp&bE(n{H6EHV z78)YqFJ=Y6Xad4=i_0D!%P_KK$?r(jb+I&tgOo!@2*Jm(_8dP*yhm zpeMg8PqfilZ|RbXdN_W~iv>3xMy%1+vNR`&_c>$7bF6gacJlRn=dzKeTFk}T_FrrR z38|r@C>{(h4g#m9CnmF}JNg87`jh6RnTH@#HiV%NFr_PMCF2|abO8KzpAL@t-5CxK z_aT@IM>gLLP6bl6S!q0~r2Gz8w%+=jOQpG>kVh%@J%itle*vn292Dl~xSc4PEVc8s zT+Je4q$OT$6K<*1H0Sud9lFh3PZbFb-1k$91WF*zczLC-j{xSr)sgg`uv(#8Qvc@j zXY`^x9zRn1zYS-;H;haS=8n#OGg`hWktk5)${fl;WAm-=h;mN1IhOh%3wLQDI}J-e zO6j5p^k64D(_R6rp(`^KMEgY)G(EGmBEYdk=@pvH^;!|>+JAT z(%c`jNO#sOk+G99k~02f{6BxsmUwMS!u90t1BK301JD*2JN8dVD5D&?#U z2W5g$N+rFahb%S9XywS{c-tg^g&nca-e5d&>}kdaV*DFuYEWWp^uAh7r4{X2+!zk` zOfK~ReMWraK4oW>Z2M^DtsN;!4M*>if@%9{BKfId-cG}=eC(>;Vs(U?ez}84&G>tn zQ!6jKO+w2A-LMIv8t@peDDx;}wTHcJU;Q<+`fQ7i`)Ym@-+57%JS|sU<%Fe{^Rn%p zdLIEC(A|;3ZOdam%=?sptu+ z18|>T2YU!i*F;_qmWHrwYK&{5lm_L7kI6tcIsHS^2j-Y_e%2n`KJ;~`s)bLBJan)wh?QsXg7@2DdUnebpN~0?-b8zZ5x(0_6o?=&C{x*-*I*MPLf>A~J@Vm)Qqp3=;{-Ja4T*P^gWEgm9eg*=-J61iJ< zD)#XD$A4EGYwpx?D#+`?Guy6k1#}P>PiPeL%*FGI60R&ug*cH5z-Mm<`vJ=i^vvyK zF3%xtd4G`7b~K1`>GkFd@s}z?QbT5)st?D7Q&n@4x;uUzk18Bc!u5v*6TOCSKRhiG;_JfA4hVEL7XTzqD;`jB0W0; zx>)A^1hTI}i;}f9vpK)8WPZpDi@Y&`B_6nmY}H=LFT|PTM$5F!=$)VDbz4sS59uEASN)|yIrwW7n9h4R!_>CJcqs@ zKX_hb__;(uR+OkgI5vPfp>P)(b$CfUB;dLpb{E&N1~=wEzBL|P-2WWJ%WQHgqVK$^akAyNOC&?RA=^9|=vHT~zW57-3` zMX?P15}ku>y{RDgxKr%%A`26Tm?IBY8u;NNV6vLWHP!3dHi(Hnkix_x@upbbWjMHNddK83q-l5KMqPPI7;?IxJ@-&ypclVn>{KC}G2bs;(UbjlY1igIhNaL-R|pQ>+04L%S|r&>56v;KU;>gwlb)^!m?< zM_#8DH?cK2@+$%uxz77U6lf#TU;)zEy4r}&OWo%|=Rup4RA(HN@GI7ekl_P-1Y5h|@gw!OjISU5&ge@8DEwpqd6bZ9oZ!j3H6ix)@%=}uXA zQJ<0k#MWLCD6Ra%o2H*CJw7$;t4oC@nPtaR$1pESZ)m#3=qc%*{V<$9fuA)A6SsR@ zEWvh}eO%djIeR{kOf&=;F&r&8AQCJrUT0R?q?wh!6-wnnUO#v7-3JQCm`*q{(jeb2 z)@Ib-KKDSEPu+#1z3o*>GpD!8*&O+pb&Vbv{K>4zSjGm~D>wYYE&OgIyMv2D;tbYy ze7sONqpzdGMH?BvJh3!C%a`eX!<4&KF3>L>HvT|WO+cm*!g}N7A)y18)x855dRjQG z2~U2qIlwMmcZ>s^@k1Z#24US*j*Dr`aq$7&&Q+Y5w|r|f>&sa$Qfcu!;qJ$1z4cMz zdMm*4M}x@A2j?ZyM2M*E<<%An5E_S<`7>Er+L1v*%?(i&z9CVwhvKsX5K*elK0D@j z&0YvI5E*>CW{3ia#u{CH+GKRSTI9wCJFauXACfcsc+qulbNx$Gp$-6hxte9QnjlH+ zIlMj1{%}0xsiwVF!HtGUis7`F03S`9y*s2LNT|?F6ZOciQ>0o65a5tkqdP!^u+WU+a7 z>LJ|3rG>^e%hRL-+Z4t&4g3aYUCBgF9-4kK=$_;`v)rr(Lt=fK8uJeA$be zQANQ;Ra$w(hk4RjitTFk)L^r)Of1IUqt-d!bQ4cw(Ixkpw$hY7wPI>X1xdy6+czY@ z-%k=6}*ldt*g&3Yv83;jex8pSfv=8{nYZCLU>vw0H zIkbrDC-Zkm&*)Sha$c@#{vNXB+(c1;aM~^RzW2W+C%^HOHOyIhnp`$3-T(ff+EP_W zQq$P)FxM-?X%zi*zv-PRRlZI26qVPxGXbGb&QQx2AoRxg=l8~EW)aye4(((qGu8aW zs+Wxf?hV_|A34j+(<5x zZ^VNaVyT%1irTI5&3Ag&v?mFb=+5dvZul<7nHC{5qJ+F=Con?K=8HLn+}e&d`Kmc4 zTuV?~@U66@8M;-s5Z4=Z&%>tu^7q@Je*3Z@dF<_aK8wq>?*KHvZ=DWpACyG+1t5Gf zR<*uNduU8i@-5loXEH2LjL_Vh5f6>NF-25K*MPHsf=86Vwctnm4MHy*-Jnn~adwo& z$YkphG5U$rLhSA@Zuk?>*WoT-a`tMiZn90$K$iO7s@)h!0@@S^hF!N8{F;UOz_TH2+uQB=EB_jUhpDw`Yr4XBr z+w^8{OyBrkX3G{YiG519Z`c=Oh(*WF4WqH=WgBf;LT;s9ZZm?6mduI=wG+Br0|l#I zpYtzDxm@0KVyz19hE0d;SWf$|*W&)S?*FrX%!%03q@TuIRyc91i3@`i`;M=>$_RyP ze_J>i+xRF@D3p>_tHXL^o%I6gT@43rLM=Q71_^*J=kD;dAw?*u#Z`81iqVVsbbvgt znDFs!w^XHnF_Nmo3~igcRi|=Hdk75?B4QNhJp$kz5(&%)Rd2bxd>o5Ns4g-pUCX;A zr$)8+uBBD6?-%#fBmGZ7Nk^v5>F`T3Gbdd>r$7-_Z<$C=3%2vi*ciXnMac=R{p*vn zKIjWaq7hfq0^S=~hNy?_%u^u4(am8xESV%GVE7HM;#OC7i$G4=n}$jqXgl6 zu(gY?qPUq;!#i@hv3xfzwCqL{y|>!XwzAfKfc&Xuu}`ez+JoBX;eIJlpAU`66cTpp z6%_1k$2Fc2Dh**Xh^_kLuB?S;W%oB=Jj93Q#l>rK#L;b8%2y5KG#YBkg)Iq!ZIdk* zkNg*>ScCzx9N5|BYa;sco_p~9Xw{o<8*loygPGTp+37hTCRS7_ z(9&a)@ddK{U?H_*f4pJ2e`&G3T;aUK-pg$ka=%iOfgH36S;Er;s#!om0^~}XeK|gI zGvp(@NwfxobwWP(lYlS&=I+6M0fZgA;KrH$jDB725%E;pMQn%D#?Rjv$R{8i>rc$96 z8~7)OFUh=|k6W{rVkO7}ue3rbwwDf1+Jc52F(}Jpda(QN^+@e8#NbZy)Kl4CondPE zYoAxBju4sf*k)B&<9cL>luQ=inysa*1^Su;TZJ9id_o@6Qz$Ceyfi)7n1wqXYv>rC zW-1_Mi~weYQ-L?Q9qRLCG4NQ@=$Uj(a$EI{*M%%(|Up{C(#raFS_10O-i<2UhH~3ky zRaqL=`x9rWldV%a@|ibsOe0G=jUp>yAh4SD&UjU1)GA1?pCP1hBi?%l(#3QD!lX*; zlP1mnJpsv(JF3fl(%*13I#$?;uy@Ou7KI#}bo?0X##+a3p;L%U! zO7fG0YRY(YZQ72$2dv@*FL^NF8&V9an*C~7jjJhRFa7{5pm5K7`Sm~)_iSGm7V^;W z$$`lnEwR7o@rbZ)P}rRV#l1Z3TW~?}$DQvFQN?U&G#DCGzAXxxF|82>`+hwaVV6P% z+6+oLqt2kCe3~hMCX0zjjkfoQ>HwOoAALSdr1qw)8(;9Z|LJxXgMB(r1~%A&ty-0k z-PDze$Oz%b`$O)M!#9b4Z}RcOAIwXh$#*8tiE33uzqyBXZBUaNb7P>TBeP(d_<4kT zYSzly>A8)RW+LBqN#^;ez%`Q7c=*qxj{SL(@f(;#>H2yj1v&qFPr{6=kK7kIf&{() z(l-eKL}0wWAx|re3!P^rYfOoHb^aYx|9ibAPoBq(A)psTR;T{5$ZSL8U`NZZMt%-w zo;p6_0Mnv<6DkAHda})zX=z*_US2$QA`FnsGV#{^-?i3+H?`q?q{=FKY5Bd!4>x!ZNDcYy3zPT0(hx-dC;MHV_j{8K7Y2 zJSPs8oJCDgyGTrOTqe-s)`n!l++0BsDCc>C3G-swtxdtpw&$A*!%b}WZQV-odyf(K zt1rv$(0)rh_eSemzV1(%D zjMEp+6H->NtVVNePYLovy6X{N^(%sG4ijnBID)su+*V2&pYexhji2$MP+qQbO9gLOx!!>~)h@sE5wlf@oX%rQi|QNPI=HVY;5*${;w%dNPgC&M zJ}Hn1b6+h;65TXQt1dE->Sf9Oe&6=bTSz1Is^8}?$F`b&#gMUGB6w;v~e zhU1;E2}Wqtn^e>3j$*Y=RvCcVBiV%Cns2qLIW%*YukI^fcmY_5wS6075he+ZLxMic z=>WnooiNH%oST9Kz=LrD%Dnl#Dha(ke9}yE4k-i2wbWI)$aa0`cp8som!mQnVQ8eS z-!0He>B&0fSZndrUhhqNTlRpzz{xS2sxjbCp(6~ph7?9Ana*9|v#Ha1xY{H*T0rRZ zNIhga8m3#lQ!-S2Y0J1EFo9I%$;QuOp>Tds$OP2|SF@YZiohi%zH|)g8WVSk*YQy^|y@l0lgVmB+{M{+xyhhyqLhbXo=M{#{Ro>21&CN;}YL|~8oEigu z05UsO3iAkDkaFr)ikMr0bza)?8V3k^$^<8a1h=RS6_MTMZv&(fD586PwO|MM7F`B1 zvkX8;Qr4ul^a(xFhJ_Tm`y79b{2{w&a%h*?qD=vGl{lny2)Vsg+2ZN;Z>sm&2Wv4= zM4Dh$*ra@yz$zm-Kip>y42>g{r#8!6h9 z$jA%ycbh^j2z?kM>xGh)qJrKotVoa!709@YA;A{`Tv@ujf_;X!sZEJZImqukOxUY7 zKp4Z5O_n$5C8Uys`-UBu3T)TwA6$xhhUQ8FP@RTYf|Y#^SaDpn&r5SCxPy0Pf}a~O zFD*$Xt?WNxy3i>KiNsk`I`{t+({_2!6BI!8VowcD@2XU(_V(Nt1WYzcCsJmWs9Jb-8s}Fj9kUw#VnY*_sujrrqmWiF#}&gEhfb# zHIr{5m<;h zn_saOJ|;x9H$#M!ov$aYa*gBLKVJObf93pr2ohbP^49ZOWBopVGDN`L3&UUPazew| zRBWbV_^Y?Cr&k}#Z|Wt$iidT}bHHfn=S+JfZ}5>)ZZd+V-3?EQlrUo{f+!y|)X zsewXJ`LUC?Yn3W;aLp|i#J>DsXSM(S{yCRZemj9;BMi7S1=7b)d=O%?iV!(}ln2f( zS9%~2prBTM_puw_7Yj12j-oG=5($L^m5t}X48b2HLIdcUSH_3BTEX{qmTmU10Y|l} zyH|kTnRyTR^TB*;7v%rau{>FE^tDYX1*tTbVb znbtO(BJ-p4z(+AUC=bdH&X?OryC8NZ#MfBK;VQ@lAnI!F`58*hs&ccI(YqzDsQMg( zA1}mYcVz0yyf{BQ8&**lz!K)t+v_-bcMbfZW0WDf6cqFnvZ%$pwI{5N8eu3A#T4(u z;^}6@tfpzQE}Y$`lgJ1&ClHqI8k)%H;ZJsq$uqatpGCEVy1g-dl~(O~xM$5L5cdD{ z0;O9B5T@oo%GV%gKtNWr2?R-V@ww15na`B#qz^?+|4t)FcZ>GuZ_cvzhA@(JXqr`R z80SELR|>kF3X6+6`zA!>@F>Eh)wk(UTWt%(^lN=P^~?3!69$8#Eb4g=j|tuKBQcEG z{Pe9A&~U!qB$cdPVt#VvlSueylhfWOa+23Ms`vK|s6yED`Z~^4&c82 z$%3$7ZQE1g-xYFA5Ah7ZL7gpe9x-6T}LQ~~x48kmg4#(`{ zy-42{nU!z9%j#NiEZK!-p;QL9c4BrwwiEEzrjeTiv}vO%>U%cB&2yBhI!pdAM&cKp zMlbIq`pG2d>Hix_%lJrG=%JxXr9yhfokdDgKV8_*aa(D+aD?0{DrkxmqBI}OD^95v zDc*84LgjlVha|l_vVD~02jI63Sh3+8-YHpzl~mK1R(J~^&+UZRyjADJNFU#St4=pJ zQYYJ)&tqu*xUFaRGwobR8J~RI3iR=)Kl8Q*I_3wTi$=ARPUx0d;kTk7#2W4COjsi- zW={xTD&~7u4tF$*qRBi&JGb5hn~S#?NVcWQ(%;^H@>p%dHCVc+z1meQFUt0DgI@qu z+Gxq{a%nc#{l`PDKk=GB9tH$#<%5++ZHZZ+)|MvB23~2P+ zXG^n*V$*5D;Q#?hrr}mZVbv%FP|3yeu)moFl!-fkI=xTGoe&Z?1qD@0s8U3oq=W90 z`)!*ybj_$f%%tS9ogcPgH{Xk&?i88VaOU6PL{{IBD}s#wK6rXSX@y$ub;j*zJ-V{Z zbl5LA4sz|!A9^?B2}!}4>?SHP3KN@|Dv)Nd`fLixz65@FcmJ=6JWyi|gaT-%kfdyN z)z?R8o+=nmRAds~oO{#ECUWfdw6dRrPWXbVEZd;nwPDBiNgw8ux#UI!go=k;8}QE# z-T+?am+WvBiY@_+^pN91V07vlee#8c&n9&_s{r>M14XqQZ=YK-ENXZ%SsGO_E3daM zoBNL(>50H0uq<#O~># zbV}6&^;z)B3NH#Ph*Ru9So-IuBI}Q`q%C1jUz=I9$~Dv~tlPrI?Rdujs-j;jdGI)xZtsQAXdZwhZQ{3FECwGmfc`_|~qvq|04(cMny& zeA+Qicpni-EIrgxcQ^<9VI~NB)8n(VhOP$k2aNj}?T-+umz+-`Wb*A2j6b24yE^xG z;Tj#^!Ksc1ux!1Of>CtLF$y2z`#u!u;2cBaJ@#7%7V*sMR@p22LtEVhaB7SVYE(=6 zt8D=yXMA?0jh6H2(L2-nnP*mN1$D2GXVQW_FG(x|G*tLo%6Hah%!gkdI2EFU=8J~q zMKLo{{44SZ3&^9f7IZ_W=C`w4jfezB0BJtg%0xX08Y$xMN-q+bz38BJTT;Z-JRGNE zl%P`AxDrq|k_w2$y(e(iO`6qg#?`atnkXayFVURUq|!CKc61y(!{jXLerm6QQ2-O} z!G=T6UGe(M?@!jaCPHZ2T;VXAwgX>2U{8A*lklO*=iGYUIc~xU0#pCbVFP8=z08Zx zh0geCO$|)Kp0k!fbS>W}K@e6&@z8~5a|w67AHUVwyghf2*Y9FmF&p#ce6Y6E9u5q( z|3>j9a<)HjJdp|U@fgO|Ts+-$eAD!&B>w*vJ)ur*sDz~dY*E_#iRaONZD=npPQafa ze;ipa%Z+-}KW1fhEGfhBc>UgXCTTKRk|$7EAvc4yGs7Y^HDI=~cc#;%(NC4?9j#mf zYQ%fP17+G=Qg0ar-ibGzetRkK(|if*hdI63=4>j!5o79W28YZ)Q0(?YD&cWrD|sz? zt%w3#q-)!@fz}@BLLY?gPPtBgwNZKKt?Fs!2S3wWdF`_#L4egv@LLKCKH1eoP6^GT zVB-blPvP_IlHjzCAcuwL?|C#q?}5o=X{78Zf!LL&@maj4s?pdso`ixJ87jmQDNn1?9rH@`pZSM+ZNYOY@ak#_iEwdy|>438}`}V=J zJNq(YAR##|(u%T+7mqYmI*HZhYy@!Wxqk)6gYdTW)OY52cg`=*pcm*~jjK%vQ_ksc(RsPjXaa zTMn8jsC+WesH#IMJoZ<-I&NPa-E>y|BKSOwxVpzT#MS1&obT<&P*-zg=!pHs&H zuNutv6TfG$_E!vAooF-CK1!(^{ouHtztyqLqYS?;(O&9+1wNw{>tCZ@<=}up2wnIq zm!fW00)7eW{`wK}EWN{VrMRLcpb@*-XGWSZx5sR^E_~K8ILrwYCxls9e)otcd!!^J zvz$}#k`JNjUi=3A_c+pPpUZ>-I_&Arj9ir0jbD?uV+2wwNK@*a#l>Edyt#?{H@W%e z+KvC@)DwrlV!_yj>&}vDXTy*%nRxP+CqI9qA>o-neHADFEf?0ZX9ojOT0_Af{}PVG z*3=33)IkYceTi+Mfo!fH!m>+_UVl2jR9$=fucJnf*vdjp3G;~(O&vwm62qD<^OD=4 zOED>w^*5sHmVRaAhY)%l2kc(LdWBTg6dnUh<4}=F|55Va*M;GJ-HB%kQ4apphI8lr|E-Q7d3EBX=UH*L=EKjneOPy}-I;H>?Txz{M<6&fPuIRoEG ze@s4vxtl4hwwWk%(T6=$xbEdbe-8~OO*okF#!S%qCagltw5VLJ64pUNX_{Zc!H5X?O9<#cC0zEs*+OM?@ zX>C-99+PnpvAC{=_%s^TrbKgAj5TSiQ6@_eBIYA1xb_pO;9|ZfndtYtB{hddq4&1ku6=Huy)C<+0A{1@hR|`(gSOHLh(RD zY8|nG18!I)Ow`*;)vVeb0SkYGxDjroAn}|T1YJ;M5uW z*(qV&jrRo)cW-ci5;4~ry|0~dcBF;3=%dPpU9H5#SH^i5cpl4o%{qjL;>P>*o1O?Tzv0vTW4qdcvtmc zSIbBv9&ywRSVrKn$9sr_&ZNtS6)P2KoU%J9g~1YZdS~omT0=NGw0V(>3nK?t9A>1?`JTik+P z7)KbY8FaWwIA(v@Vp(pDNb-kEDSti^8v_Sk?YPgIAa@2LLHM;xRbxoRH2 zIT{$pe<(f5C;aI!Wn1VY;yCO!pqWbiAw*z${CIP2MMmaH?QhW?tlFMC{^>zp4=rY$ zu68}wB2Ye^_VmrtQ+vgkN)9Vo&mWWtT*(~l=$%NjV=kzFFS~GLs!9*@ji#I*qLy27vblb?9p_{Beo(|hd z(QViZKFZ4P+(t2)mu?>)0KU%`A*3D55xXpq>W-9jyDFNwyLq`TOyV2_~F0r=dc*!zb=oAN)+>B4pB$DAUyQ=Yuk?leMJAPZ0&E_ zos`@8+O&HvywGPmsGcVk_lzoe)K9JCq|_m>b;4qJjDsF9DMhXs28dly)N#8wYh@n5 z2rA}s5HPacc30Y~-NAiL^A)rW>C92y;Xc^j5eHm+)dzYA0S&eMaJjkurXVuNILN#w zixPN^JN;^$^myz^_CXv)9t|AhYhf?;-yG>(;oL-l1`*A|dibDKy4qL8_?}nhkq7e7 z0m=cyarG_5>?jz@J1B?2#%s6#f^6umeINKK;^UJKnUEz-&?XJl48K1iOu+>qO?bK| zT-z|)$z#fJaL6NrVKUa}j6%QKhRpI?-5fG06m3ZnnVOYLFu&S9wj0>4CnvP8N!l7} znl>abH+aZD2)yQwm%THfLdB>3;JZ{E-*{%m!LsIoll&NO_+NWf!HQ$3%?yS#y#;5^ zypMZfNLzIg{NB(5{w{=3$g(XAwD~-8OrV+%SfNiVplXELh7Y{_354&_bH+;Ad*%^^5<3n*YE{WvEX+J+$1;lj@O& z#-rI70R!YBIDF2IDyGZ_( z@Le1uGHuTvNa?n4|6pgg1poSdpqOAR!XtEk8$Wk!siC_NBVd(QPA?cBh7Sf6)iN2NbO`6QHRZBMiJQiuJofqy6yOkPUnC> zZ7vLisFr;4_sYQr>!$+}rtt~(v;PUFMWD+kKmSi!B_bDTs>vWtfMXY}zjwb#{sppV z&(3@AvJlA`^$N6Dn$aciJz7%n)w8sS%@#2~v^M^j1CV)hccBE^i-AN&qnSrB%~_~@ z?BF)46*j24U$6golhvy3(kbg7ikzr5i8bbYvRi0L%X6Gc@rL0M#qOniLSHutXVX&n zF|0P}NALo^9d-*ksBxj&r2=!!!30PSY{}7IQ73Js@k-}7Z!b7R#PwPV`l!*WxbJ=340cu#*Y+%F4r{U{i6Ij|4>`Fqt*if&NbK%SRwgcZO($ zJfF~*w?Xj449nI=lISax)(|=DSb!&l@pA(pGP<*w2g-czMwsGI|4K?^Hm4B?TV(yQ zBj`0fS*=|gM37=EVeInAZVUp3kMU05Bq2LX7Q>#OaJm<5Va)U=$Civ@pLpaqTtd;M z#NKDAvIaD@gx3sg!UY_ZAXz3Hf(vYHDi zx98{Fs^f*23$(owl5`pO!tv`7QbV1)nW+7#u?=kdi5)I#{8|sKUSm0B@0yx(qtaiUq;oLxVwhz zI>wZcw^yfv7{otdNm%{J<}An(kd4=&fs4>AmgRWP_2u|A?W*0(`mtHm%M*% zCz|vw(yHm~>v<(mnq4T7_`_2|Ka5w=vDemv?Adzwv&iy|ZkPY}>B(`QX>yJXnFlAUt~S z(`*_{8>{NdYakaOvy~sz^=yB05T&%2(>QyW`g6Q9ulfmIq#`~^uDR@Ni`%DJ%$}`Q z<#sWEA0ZnFm3k<3bA{sa1=7O#r{EY(v+e}Be3p+|wa;x5I26B;CFI_Xq?E334(60y zktJw*%ble*R`lE&NBH*~*OVTw^vLgT>^gRBI!a=unuS4*UBx3jd<%nYQXNvenQgCk ze4n9zt#HqR>d?)ax5fO=mKCN1T`_}{{r0y}Gu-^JcY5il_xJncZQ~shD{mZ5MQIrU z`q-Hpb!52~u86g?HT*roverf*-6ub1Q{5lvG<<&ooz6c*bNq`rw>kcjhKb0T33#VX z1+3yE&ue;4Bp!IS_B|mOC*zqiH!$<;86Cf8-4&PrUu>iiv%g-O3C~5LONaC&X>883 za6e2&3*A4|CX3J;K43?anDw(6U}^Z;zP7j z<3FWr-<+;eaB}1#yabJ>k)TIZF&_N+1{5+jwFHQbN_BQeZ$$?@+0^1&#ExgQ*0NNS zP%5==yI=?B18yB1rGG{$En&sa30lK95|22g2kf?@+n)H2bQH84b?CW?TITh(I%hI* zreE%(Cq_{6RY+Manvtd*EVC!ykCGf4i*WG`DCo6ZmSdL+W%G; zw3N=3O&tlIuYeTeq6T6SJ3C=KJ<{~*iyyyvx#-^f?3JXm(Llj&3l2~c-0KCgY>NNb zA&;lv(xqE&fPDcyQb0zJZM?d1T+|ERi9k1B7_TQk0~AH91X=bG!&OHI9!9kfWSehD z+3DXKen;mybzk9%YJ8=JjL(e_xCn8D#Ri-&vDHBFBLK@Rpc8=1_*U<$g|u}K+1v0) zTob$TnZJEk=!bm%$XKX~GA0Vw=WB4oyO%b89K4s3G)BOxITg1_lS8W0IQ_tMLXs>& z6&Bb-WnMxD;>V=ZvnB~>QNmXk zg+s6<(Z_=eo-XT0@4|lTH$iaJrz2oO;Y8TEo=E>e=zUZ`gcl(3fqobKnmQ$&!7Y2_eL!?1Qn5eJq1n%=R07&+nY-9G5?sYp(0_nR(yObHC4X z-_QNj{9wW_;r#)?oM;| zh4SM^(R<9eB;^8b0b8WwZ}pKM`h3W8D_G^Y+93r>E7t0)nru#*4c_=U_kDKu<>OKW z-VQcrdE72{v>udFPfROpzT#GgfM_mzY;XU9wa!938>)o%`0rle&!3r_Cc#ze;7Z3< zTHZbH{-oE>Gz?4J(%Q-z+Drg@BK90ttd)I}r!X9J{jJ07sQdfwIA%Hc#_)%LL)sVX zq2r#Io6YS5$ZE*O1!=ju9jb_?am^FYV@NWj54Mvgd$X`6x53bys-PYh=C87om#TF4 zUh_v;?kYS`oAy~F&Ida(m(gw;-TYTv8SC7QfPxnkqLJ`O$Avd2BoeiFdq?=M_0eJ0 zKSSl5&xan@ig=9r`x|D~XkIhVkO!j<-t_tl%>ViKU&`~SmK(Oh8@&I6!e?UD?qSc; z>W_T&SM7&+F{O6q0kutQR_L1kXzbDdDsVBc5&TOT=w1BN`+d*ZVYr!Ncz%8>Zk-r)P<_knSp!o9NZxawgwn-WU?Wlm_kGpWm~#O#N9@G&){2ei54a zvuVhyW)#1Ici2wpz4v#8Ia2<#pF}g$1nRJdVC0ts@>{aWx<+%`4JpYdHE#geicb{+ zApIuokUFaO#FZ~E!>WsD-SZ~#dzkWW&xmZAES$mq!<4+|w~1;9FIt8*GTb9?IH%EJ8;>TJlkyAElhN#3EJDKCDSeQKv&h z3yOzTi}{sIpvtug%y9SY5^Gv#*L)^xrcwolbC2ow5xdIJamkf;!K#)Mr!5W6o3N?h z?hs#?r7iCir%fzfTqN4luir0<_>iw*m- z;f?l8M}%U$iA9afG%xjc`Ry)=w|7^YX@ZJ1k#s%4&%HX9QRa$J*JaWeZMT^sZ_wRO z&E@K1aSY+w8L88Oi4z?ouEqI)t2X-hPyiLz?&1EZSdDGfC`4s0Az--J? z@y924$zzZp*MsFgN(Um9pFHr~7%`1WPT$}+)QLz!{k3g#zU`J={Wh=i`B%H&(~)VB zK4Q|#sn$GLnJ~_lBbfF4N{P=)ts5a%kB5ab+2KKi3T`m7ZzIw86iv7Jc5< z3?apq=#(3ggQrlKrQl6PtAHxXk2P}YcJe8B+;(6_Zs~&jA4i=BkyT=H+Sr3VsOAJLh*uH06*qkrcLt=^8TvGT`*!J2_<0@nu`2u5s zfaw~&Oy(IKlA}0N)_RcMn1`b^4$vaTnmO_={a#G5#WGTxMhtS%i7icI*e_au=beE% zb+nW8QaWY*JVlS$fWsBVuBMYlRx2Yun5Tpi5*IpyW64KZ(zDm67(UdRVXr)l zmh*8tU!acB_kN$ScaVC=38eE`NTAEa76tE?Y0+bxTyDeA)BW&GKA|Sprw<3q8#l&z zj|7#{h;x_#@BmMt#xpX^CRn0~5R;lC|`E_h1byVu6=}EnD&=?d_Ws6F?lB38j7t0z=!a zAfi=sL)UkT>5vU&RtqN7wgIc9_LjIzlA)Jl$~a74(J_m;3F4)1s5eVd5bQ z5*mRaQSf`{daG+5*N=JDMia9$ljkOP{Jk^s6z98)sSOB4XL|xeI|g%E#g=~A8bJO;S|BIrMMRrU zy>VFP7R#pfImUK7EDIg%0H6^qLT9LNeC`iWc$bkxy3*l$zPq2|0~C5)w~c^XhizkG z*hh)FOtxnDblpj>Z@zK%TH$fiOx%#smXBN*W{B@Fnr+BR>fhn#9w6!+dyHaNU)kzW z9|*X$FrFW?dyu_h@(`s4lem9=rQ3kl9sVh0#)Tdlu@!PfPG6A**vew4bujzaOa-3P zdz);b2cPZ#$tR`v0SA1FT>b(~5i`Es1NwM6;C2=8|f^!-5?TWzzgvI-& zYfZ|bWbZ1}JRoQ!5Afwjpq7Qa6EL%>v>N!LDA)2^%h9uuzPzIC|JDc7ekspu|PFXAU*Xa}_ogFv5dq|y;Z5D546JBl01qy+poE=4UZ>9B$H^4O zV=hkam{HgpNH`KD(KN~^-pKBC$<&_3zVylT5ovvU5}<^I?s?j74cR+L z3oL_p-MX^kxiRDgH$!END9GOFlh$`3WL3XUErDO2P#<0E8i2@bg4JN^bHo9cM;p(E zuim1i5=?05jdErk8MS$LP&NZ9Jg{c-kg<9RY6Cugvtk2tkN{Xjc9D2=7BFiX3=L=+ z?q~QWW(lJTD4{wr)JVf5W3s9VdLwfl$e^6?ak(2&iI-7(;ZuYHyN|l8uSL>NZG)+l zZ#Cd-bOm{eAD2#cD`hxf8@50b3O%JIfCF^SnKKlMWHsX_MY-DHi~>@9YHfRkc!oh4 zBNG=$N}Emqh7V$Svo*vKPN5Xs4ZMUI6d1LsT6@4)ngRAB8TMkx$Tf!!+pqA%cuUDqsB=1$o7WOK zQ_{mPLsgnC>VK`79eLM%vqBC8DRo37Tk*#skuAzqoD$j=r>5K)F`yx16X?{YEDR4) z+rL$7iS0Upq!w0@z$2JmI6bnw(p@Sd&ZiESl!+zJk1hSSH>d`#5>Ys($8A^5Vbh(;VQ*{R!er2hv-S`v?KQ51R z5Rz#Kym%|KvZf^@qMBf?=!5m)*PewvsBs_maA*Z2?=<{eGo#61j!;Tmk-epL*D$4< zn9=sPljB>Z{jd&FA|z;L&CENZ^2cr3C~qRMD>)vORP7Rxnuv5`h%REfsD7aou5E94 zRo4)Xi+qUT=^{k6o(8KUm5T6soyx(8PmC-JFFmDqo|G!s01><4v|&;yHS?2|>g>6?lckJEMsv;v-`sfdu^ssiJ;m!PXU(yAQ z7MO1-eFa}+IU!#7cmtbb1JnvfJX_lYR9K>(5r**T(0-ed(*Xt}AxpdndxqEs^HsL+ z&7tdz#jUH9p!>28x@e8SCA?86f^7`VSoznw-E-I?DUFKB`NzysxuNY z#lkClPBIpPh|Y;f9h@cfZVUskN&;qlXv|zmryB2A>@=PQzE~NGSP0sNgfUH(zLRYn zW@?|Vc}1*XM#$%X%Q$2Ov3Af!vn_tn>1TGT18AWSSz;j5@j|22{cdv-$9d+!ZPwz$xK<0CN zWW-q{E={D1r&MN+*D1f+Wh9MM+Lo-O3@h#C_56@CP2xaxE-R89u**SCLlaCHtarqT zyMm-2IO`BY#`2bB+Wl2uMzpGTsjV-plETUer(0u;86Egm4f)3E9V9bzae0qSa~UC? zj7sD_qRVVt)&*vn!AcJKi0xb)}s`cpMpC4*X_XqO&kXy=aXaG{L(z^aq>MFeSd38@eN z(&*g~1q?ygq<4(Kt1eebkuTU{ItYt|7_I)m=8|_=>se66)ZCxE2uvmNn(r-_ZpLWG z{ElR`(+FLG;wz|aI%7N1Py^5lZ>h&b= z=-~l0@$DBR*%$CxE2t+O@>DMqA3U=b@9R=#Ruw2xV~1I0mR5*1qp;Vo2zn3cQkNph z7G1hxPbrv$Z9DoqkG{j8=?X~SlnS63rXRKi3_3tRl?Zl1ROgZC)3kYz(w0BSOyY74 zveUxn8`kqAqkx%(C=CnwxNPpvY4z43K(4|3p$^@{vnER;FM+1ig^0@x6G~Dtqd1X` zu|U_p!;HeI_+`Aqj5XygiS9_LiCCbgl$X~{K4o~q0;DQeF~jcKRBy&(_h4vt2?8Tl z@FpR9=srVe>tRa7U>8H7x&5Hy%NtvJo)Muf9GLlV*HFUwcE;l2U^6Ct$1@r_tXkF$ zyEdn`5vKjgWn89;^Je(|tBbJLjO z=1NP((e!xgISTA$&EWjznZ}k{#~gHTsqxR&z!C2v+9>^d$bg0M95MBx=S2d$wQS(p z_9B6NM!nZ)P&Ry=gw|&HtUE8FxB(!Ib;GMLV(SWhANN+kektd3Bz-U z_asjn;SJ_;mAj!IcGNdfbPT%)@l)?QEfJ|)4|!ug&};a;@EflLYSlXzcBKkA>EeSd zcOY84XFjzly@1izU3ExW;+Y&r+Bt@4<`M5lNZ?PhCxvpu2j;Q4y4*%VjW|u}R~3vx z#~B|m=YU19)*I;XBiiMd7?uca3SkcQoyQ%05XOk6_6%i{XFgM%lU01ze^z>r&lRDB z<7bFZAypeTO2A%R&qiH`J_*rBXNmYoJ)!~p3{GWCB3HQK3mAY0V;UnyuY$1@dF5PU zg>BQO$hiXy_8;TD*&i8aksVtxy-=nzLt+0$evW@);o<=2OvRYW^YML0csoY;4FlgPA z0ADnhT%?bTetzc8MEXf+OZX$(h0f~J*m>-v~><3bohW;w5`n*YBQ&2>W~+u ziuQ)P?bFu99N01p!eWJc#=!D zi$UyqgWQa_?q}ubj~I8@D#nMQNW~?u5fmj>pYLhwSq0qrQ0xsjOATCDd$=Qdh`xV> z<84mq#To6gqEQjL_;?Qhf6du}mQo-yBc-fPEIkhFTCIf1AuY6Eg=VRnu>{)=+alN( z>4CW=>MPwMa-h)7+A#Tslk$~y_*X;=Jp_I$odBc-8-j)2jfQVlfUVcP?^2Ifmg>Q(<{9_tuI}Wj0lE%t zd!}F-BeiLLcl9Zv2c1rY<PQA@2BtIDK@RTgtX=w<4JHe#)Zu}IroApjew z7DR^eh>IX7r@EIUi+RzOP(YcR*a!yGE!J{fz>ESShLZv_E)y{dZtr>9U(9IO$d6kx zFxG|q_>x8get5}8gs`ZihzF^$WK7%VrWwivbxwNTD*b!7)FT>e%o5MeGph?9!r67G_P(#SYRmP zY0;UP(9?cvB!oJnVU$*QA3jcZPiM#iEqi-Kt3d^)+lM`m&e)|=6={!=d1eXBkGb08 zQofjiVCp_#SD8uRGB2l^(95jV6&2{61>-*pF~6vC5_G?_)bQ&)tFjFBJ<>8)yAK`@ z&=?o1Nm^(gPhJuqnvMi*2RR9@ffuX|8aonsBjS|*Dn=oqBjEy|G0$&HA@3~&gnhL| z7_Vc$9~!h@grY};OTe_}7x115u4@-2rcwX_f8IhQ*O{jrYR08$T210ixNxcZ?B%3L zlGFArcAvMeFy1@3RB*k1xEbeQelL0YS%B-GQxRMzV?<)3648Q8E%}zS`2xFkV(aZm zjq>lX3o^B8&@IjlfcS8bMfC<-R+iw2Pm8BFS^u(A*ewe2iL6KUM1D^HHp6diW?zl&2$s{p3of8-+)vrkj>e(6?htqR)Cu;}ZETmXk}?Dxf(zV-@p zN3!HJE5H7im-^6DJ*L0Ny}<;b7tO0y^*8#(VqY=gQz%VmhKh!5L#=Q#M61sY%Ie*aj9b0eMBep-+YOk<{ z!Z66yL=b!K^a%ssjI*QfQei1d`+8j|t^D$K^ID%IDkPq8#y>4{fSMG3CoBeSGmU%l zCxJG-OB6I)W)=5Z`c-nhSf6?q)2HyMahKAVTGb~B9h5hBSgY|9_1Xu2b8`T<$y(Pl z34_tg?`Ep=`;_=4%Go^PRgHo>ym7_FI-3N>iaGm!RyK0S7!dnHM_ z&2Idu(iPTp2?~T_)4;=}R*aW=9m=6rMd!k%uW)$cM@@NAlEW(R_Y2F^D4^c*j{a1BVCdojPa zDsYpqSlxi%REfRp|%lH|2HGakW21XGMQlM%`=5Q(7iGf~O%WM;6L#{UJEu?>p zMX8TH<-yutcDDZz+X~z#qI!thclo*1ZkZoGEG!&)#*)8nRVfp?)1?Beb)EQKg|y3F zL@KfXrH2*V*S`8#elB}71%)eE>2usdtt;N0K*{KRiTUfQsKw`}7mnz;VOiqBT9WrrrX?&bWD=_fY8m>@tNhi={Ge zbvHNd%szr5>=D&>L}ox9q&LzIAtcR6*-Ia+diP18#R&|Q| z?fE3Y8l;!(?@ctO-B3|hRi4o1`O5oS&3yedccW#Qogl>R)k?JM=II9!{kI4uaRt5Z z|FnkEB&D}nJ$Q|^G~Rr^Aw_OVyqa_R>YP=x8`Gk2Vld^IYRN(Svj^x)#@cavu;{_C z)1FVBiQ;xv@fKw>Bc&MeVG_cHGG{V9SV4)xlMntuOGQevlpX<%1&`v9)cYGEf}%|6 z>>*uoMk_GM`sV3nDe)wQfJLg?JR?b1K)9%h5Hf}hoLZeTEEhUp#P6B)wLD=jx$%Hn z4(=+e{RNNVBQ7CzE(YnjKEnAUXgNJ^PPNdTjuA&sSnT9atMjHh0!c;u`Vy~B8L^YY zjN)yXyjrKiGfSl?6HZU|B_Y7UaIzBxzjao>$O|@DFi%0W>w12B_{S#mv)S(~JIuAo?P>1bFT|<#X@xi zZ&dsXsg*AGo4WjOl-=OO)@OAt1%GKUR=ltrwNQ= zow;zK*kvDAN$7&28^T9U(j%i?xaTlBonkehosc3nqgFPNR8MuIjo(TZ36&~~YU)<2 zcHE=w0V}f|Z5E9c7rQrkMDRZwchY;et;70;dATgveqnmw&T|1XLlxnR*9@v(gR7(t zu$5b;7h6`OUP=#SdN1oWY;Yfw(GnE^Mb(xKS(J-?`(}1*?97GlwFAI727wA6$@#|n z+*mTIzq;g|xw2au)FaLz8y7@AiCNq#QGuP@7W$#iGz9{5>HJK38e=>6r>vW#@r!(A zv#P!?=D0X-NfeqU-N zZS(s^crF7gdYr#?thzv9pKPTy8QPi=Yw2&6FVqnQu*b>g#b{v%@(Q9dmdRz3ZoFQt^bWsRY!L;`!8&RfCWhY?&wTyN!&_fV~39*67*wX}>no&31z1 znJn>VY05J}y8Sm;-*$r4pU;~gO`MKq<^L4oYiG*(@(H{paL3PGbBy%B^@lg*+no+S z!!X&O$feAz*~$6I1|4Ljx)DjNkU>?n@?>DQ*G60pSC*?iR@=D$;9P`I$DYQ|znPUg zChuAxQTw_9O}k{VLexA$*sqE62_W&K1;IeQ9mAf0{HVNmX6JW-$!n&!(B;+(3c_}G zB95>eYEmtEcG6{_x)jiDc32?lSm2=dud(2S29mQB+9WC95C;{u-UT#mC+S#}mM%v3*(JNpq0RM9YB;DP0 z-4%7d^Qb6gJGgTXse6xe5iVG7d!XCxC&PZ`FP3Iw^MD1qr#&beX?98254SUV-qoP2(-GDTv7Lf&evR!`{2br z>A*|p^uJKjiwhA@Y~x^D)e+Q%nPf( zXre!!`qM{q{l+iIX-(gSoZ%1;#c|~&rsCyxx#3D$?q~GK&_38NoDsMCUIXI%7skYS zm@ws@?r-Q%f&Eg`QGHRSUme0z1G>>~suA67+1Nlnsj;+EV&lx1rQa~-{f1UT9VtP7 zHlTiwiU(&t;W(Oq+UQIh0Smph{cF}?7Van7Djo9?>9BHo#+UVaS?GvSI+rUuj+bDn zox$>wg@yNzX@3!lECA`@e!+L|+8&)^Atdc};OB1VJMv6%!GWHnAF0rPm#N7(X+NLk znqn0Dak}loYW9Mkc^T()%klq-GO>csjae0{iseuJq{tFl_re$PW{97QN_gc8<@Xpa z)tlWf-}&GHWxy9<`C<~hrJ>(A+3z(Y|FBS+tl|iluq1vwDEkw(iP*gtjX8(4Akv^*P9Y>Vfq22FYOpjd44@sc&_>^ z6QDU>TUO&~ZVFF0QUM=Dso%X^PU-faDpk@wGUix$RTcyMSqXe$=WTrNu`S*8&r?sb z`y>lnCIWIv-W95ok;%5*y;CVJT+8u9n>8+*jev_v6fy`w-IwD$@kloLTY+)g(Zx)I zvbrH)oERG;ot1y%y`auD5#MgRPIYY>UvK-X>9j&~Kd(Uy-sjEbu8FK|C(6YfQ}kvz z$?+&x1iR3Gw*9VM+OEK)3CC8x{$9+KDssi$&#?~P6~Bz-n+UHl$iqztfmG@0`WL}5 z(`0o|vZlM)u@+6m9Xr6M3aJN|qQ4%l(Jtnf+TojPpbA8D5bbw?#l#I$GC9@N8Bw&% z?LJ956ZKZS{~uTpecnjw-|n-bG{FfJE8?lOKsuRQ;Ex+>XHHJa)c=Y^JUThHJ`28S z$D`8oRdTsr9VvIMFbGv} z2&Xo;Fx!s2Lt=zt%El6|KUco@*jZ(+EYY<-Kh`U}3ayf^y&alYQ@7;9iWI z*S}tT@#h?Y#_U~O37SP~KYt|5HK@iCKUt@|f3mfj7qgP9+n1=Bq4}pI4w1wZ1iz+h z6)GbapfZ6>P~q7kYf~oY>eFSVS!>ke-PDPIkTdGP!$%g=j_)xLVb}Qki~p7j|4Smw z@nlA`jUP6p{GZ7FuWc6zzQgRAXx@KhJ6D)FB#Me3ARs58&{*U~Oom$Xx!)99=i`_; zQnLL=7Jts^80Zl1kf4F7y0q@oo1qR?0cX%{8Q}pGK-Uiz3O2(jK)2!3?b;kFcH1T4 z$Y`?g@14u*Q@f7)nq{#|12#=q?@01(#_(Hipf1W{$_ot1CQh$OBWbQO70!u1`F#Xq znHWEx8brxN;`^{B`}*tGzXF$cVrI5G%JXPg`P^k(I=W!69&5TDrbM!IYVyftrz&p?aqpgZpsaA(NHU&s;L8SkjSt8R_*N-yo8E_}KDkm6w7lm>K7y{HEz4XFI z^W8F0hCwPn?-V$KJn+Zx`}teAgzX=^uj=4rYd>1?gpDQ(?_qkA?WQ<6%oOM9ykj}p zgCf6OSDpuf@+`z`+v`_`Q{dIoO6-b~`Ls)OyyT!fukx!5%27^~O@o z6{?p}y4K3yl4Z8e1~NmyoQc2e{aBW6%1UE2w>)*Gysqs+so5v%Q(R zM(fQtobB@|+`ceMPdi_^ZBe`uV4p{mXt6NfIOTt-tH_Ask%#n=H?wb}EAI{PhGZo; zy^V3pG_VvabTK&l3u0*SeD)3SRH+xNy`PLwKI6Q~p3|oM`xB_LF1LT;#PEtD5ACgj z($&3FHsY2Ru9t;DoN5lx><9ccCbl7B^S%iaSil3YWcaJM$PPDva=Xcz|H@-bw{A^F{%$W6nS6Ir zG3xS-ERaxC-lG$ksRCP7OFq_e&XRAX3e&-?hDRl=aILb3zv??x>-SujbYhC=7DZHx z?)iTYy%MA$J^TWV9F#(5e`~C4uF~c>dVAeO`raLS@y-YNEeqz@u*#=8lCJN4Ugl}t z27dGzsWME571CdUo5TgG6Q20gM-0ue5MtD_J&nJ%E_cvl*HlgaFbG`V;BIs*^S{ZU zuB~-E+-!=EjuL+@U9XDNn?7Nbx)n04&1;l;1X3t8yhPJ%HZo?z@=LR`#xjAYtZIzz zL+-2=wXLpJ7CKQ)&IPNf9p0J0L=NCdj=kxNPWfcLB5zZJr{zQW-{2pnw(1OiK)xtJ zV}DpT_ZZ4Q!ed9I{+;QF{%KL4?mn3p)$^&%8PVWRBNB_&&y=GNR{>|=S2nF!_v~HU;QuYkFh`a=+Vym$!>E~D zXYFcVl(Hh8w3f{ve-+){|H-A^nng+S%FbMf<`g8)&;$Wo;ahvR5%A1sa@i_VTX zd#U{7(j3aq-{HBVQz_HSbq9Ge;|{Lq!10o-kDwuE5zOMV0l6&x!)yh`I~SR(UX_Af zeUG_yb9c4sr}qwEdo)rr$rj$QmCX3VVsz-%WMb51iFYm)A2nj!!eEP{kI(3A)lELw zGhX$Y(z9{x)AxX*)f<)T%hew<+$$6eiy+P%O3=pcsKn&eb%*e%&(k^T!QEksMxr;0 z(Y>cn6`WA@A-J%Q_SDrXF~?GaoCD-B)_9x_m$n`fJo*(~+s`K}EpCpSzd$1TyXf0_JWW|5r*>(!~^ZyO@)2{ZF^a_1nzT7Amp69AeF_M>+`6wz`4?yEvlM)wU8(|6WXc5b3h( z0C%ldD`h@i8f80rwd>NB(PJky$8K#b(%M+ z!oMd%%G`~{WuE>m_0Z!omvwb|W}aPdK{j&mQ{Xo{=#QDC?P)Z{>>QW>XW0s;A2e9*fHX0A}W3E>{|10-UV5i zkxYWgOlo@u;;i-A`KxEZRd`5jn$O-_pyy|v;_FU6Xp}v_|IJOlXYFzNS0^;OmpdFe z0$UCHbfU$ER--Jfgj~FCH%T8>pwu9#sRZxnoWuat>GUoCbNjk702gQ$Ozd)r#?Y7RLLUYMmLJ9fezgSbhTQ3BOK-J+{*e}`1t%%n3Uvdomj?s z#mVg2g}K?R&V|bDA)z?O#`V$xyHJOGP~~Mm61$YfiKx$lMRK*5G+PjjX&z|?WmcJ; zq9c4DgRBhGsKPWW{hf0t-5D(#^@4+Z1~Iy?m>vFxJ%SA;A7@ zu82h3P>31u21}gawd1wF&qAjyCC)5d14=Nr_Fra#I!giq}RE?@GYP0}8d zIZ^As0Yp+{@&UDt_$6!pJrSMp1eS?bL6y!QKM##R1v&Whz$BLP&Y8AoEzGS!q7sXN z$(EiO+9&%Y0|X_LZe6odCmy2Rt`TiR`3~3!fIyTp7Yw`70v{WEvln1~5rTn+2G!I_ z4>5T0hWvO9EtqKC{LS#TC%G6mq&l?o20nFv_-L6cyTfho-%hWQOq+HwX3L-amAGJ! zw|naeD_R>b6=j}LG$hd6l~2yy>NY6z=NCQ})q1q(2s_Z~TJNX*ek-)?M#vJ*@ZQ+e^N^*2~EUSVji2tp##DKn~PK(V3?CF5-+W zAF!G=1w<>{g`sb1CCP8D=&~3O0jf~fX*VO#sIq4lpVMB=?~IMoRgQ1_m)&l`bQH4> z3!;DT#-Z+K0Urh(aM<@PjnluJ8nT~J*ReNukZ+2ccT@EB4az!ELPsM8(z!Nc>BKLq zfV4hZJSB>wCZsc!vvb4C{Z2n$T( X%**SSE}!Dt!~8KZykbzI?;7=gKllx? literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/config/img/free_benchmark.png b/debug/accuracy_tools/msprobe/docs/img/free_benchmark.png similarity index 100% rename from debug/accuracy_tools/msprobe/config/img/free_benchmark.png rename to debug/accuracy_tools/msprobe/docs/img/free_benchmark.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-1.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image-1.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image-1.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image-1.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-2.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image-2.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image-2.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image-2.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-3.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image-3.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image-3.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image-3.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image-4.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image-4.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image-4.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image-4.png diff --git a/debug/accuracy_tools/msprobe/doc/grad_probe/img/image.png b/debug/accuracy_tools/msprobe/docs/img/grad_probe_image.png similarity index 100% rename from debug/accuracy_tools/msprobe/doc/grad_probe/img/image.png rename to debug/accuracy_tools/msprobe/docs/img/grad_probe_image.png diff --git a/debug/accuracy_tools/msprobe/docs/img/module_compare.png b/debug/accuracy_tools/msprobe/docs/img/module_compare.png new file mode 100644 index 0000000000000000000000000000000000000000..2e1ea564eb191807034afd8aceac92b29b62a086 GIT binary patch literal 56655 zcmeFZWmHvf*FFlkQ9|ibx z&GWqH_na}_6XT3=&iU|uaHxB)z1Fy5FKpJ^Z*Zc^Z z0gdEYpm{4K5+foz_w8BboC^7`-G&i<@AKX5ifxZ!eZ8&|E#2L#ojL7eL2X!D`0^cO zG%+L;96uxo6%zPoC!W6%D@?KOzdpGAjvyeA?%$W(eo}@j27NB6_~kz@3J!HQHt|R(64_4!%!X2* zNfG?>Qyl2K&JB^1rhYs~FfDS|{eRw4D-jkO{`12qu!Q)CaK8Ef!(n2HXjFV91j=6j{Ls$~fHF^So=nl+tBP(O;p=gRLj zia~>&w&q7A^>{|rsNSGwZ@y2G3o+#)>3`QW6uA0bBuPIX{yp>p_i*hb8<>0OGG%e z^OtH{$Kc7q+(07BtaF?^{!LflJy~+wlizc{T*o~eD+Q0IOnlpv+x9!sB&hA2^8Tk^LaZtFKJ#%C#yWKpda`66oCLOHA`4 zjObWTbfTjV>@}bFW1GnbNM5tA%t!Lh7paI|9erfM^&s@wovAA`eBO@*2}Ui_Lid+} zI2YEx{DMJvQ@8QEWiN{1k4KQ?_87~9eUlI(4r8+NsC=WP(WzfwTbSvVT`mujI<)%8 zoRlWTe>v)JCjT8==#8e#(uZ2IdmPiY2bei>>2f)zt@K8k4H7gE!(+q+xS7uPP6yoco(NdIab)*HDS~w2ip4mLv7rDL{(YpQs@m$a9SGqF1QTb3U< zKLn9Y&o$18?I;xPeu072q=L!5FKtiDB^}eW%d$nx3<)-9ihKrN#&KxL0g}VGsrWbA zr8?50na>2c*HTiM zKL3@sX6Ga?@H_r=yg)H&v+A9PU~*HK|6+eafFT5A7TIbj`wm}z(y)5BM4(1Og3kgq z+ZbuL1%e-t>Ufql6OkRrxwzLhM-;CoTUV3!|D8E$xNRaw#=?WzRPI=0}3Df%Sq-a#}v;<7g2Pnnpbm({6!l=j{#+yRkuF2<8 zTZL=nojj)#KKyAoBzK_|o!^I*x|^DtKZf7DEW)GzWpV1`oKNzePFZP?~?6d3z43P(;Hn1zBT z{huSS0V9}hbRhgsvjTuwsB~e7yZx7G(_jSuN0IGp z{n;K41)pLzNNct`?_g{7QFxXKTj zP*VH#y*%(f1cQL$;IMd#W~K{^OAk=h+FP>XPRq-p$HSj7R_e9RnPXvC-JMnj>~Y5^%RTA zllGIguuA94_xJt$Tu?aLoIeRH`gNBzXu95s(0eUQ)%rL5Q4J2+(wdGMMKqSe`iXGN*+V};1-#TLv-ppkhWjg6&5w)}1mPmGiZnFwK^>Uc!%_R~S(Z{NJ{ z%@yI@z{MgcG{Zp|66{sNEJA32Rz_3%40u7E%10d^OzM<(%72Qyy@ ztH9??uCRWRnSO~*^J{PvNxgg1N^N?$zJC|x{RbJUh)YRm!Ut32XJo`=+JxjUjvgY@ zS6Eccd=rHPTYU9vD#!igepqXcULZ+_BLjXf@ohKPTLq6yn*e^Tv<3EV0lg{L4Gx)A zo)|v|`j1xkR!G2D5FQ#W_$-J)J3A7FPo|9zzu2Fl>Ym*!{Kogt{X(bV*H#;RNzB1P zT${7%l;Xn!Zj1-N_O&^lEiVn?#;e8!@Jk;*z}j%6M%dsePxRPIE<0thC~OlgZ-i>v z=+=Y^8S)lK_}IRF5qn(Bjc|ne7M*KP6L&x!MlNlHNOx@h=LlDb%{pa3;JZ?8xX%i@ zmJHb;pZi52vg}_a=*f+jMhQ-p`Lyz(?4d zsPa-aU9si!QVZM$C|xmUAYl|MKNMbTGEMEvY_T4dcs|H*b20lBXz^l|GN3!y@Mx|V zGvdtuF z1Dase6sobNuKu@X`7=vvrl3>tJM`S3~ZjReD&lgQ15)E3!5sxoy+o7$m>B+ zaToaiYLILi?%q@1zkVxX{0Q2L&B*Gt|3rAHx{I z;F#x0^^YPECjMdL=&mL1!ZrW7-%No5%n2A{4N&= zRj>Sd>HTp(!H*3T#gC&u>)a3D_2T&&FoLyHKuq@LyHF*A%B9;g?{ku8t~;YVV#KVC zCT#jgS)mIls9B!BU7lHuSHeEMxL0H~5v7|UusilK6V!ku(=|z@b#F{GFp0d3KrNCL@0rs|| z51R{z>)y&Pd-Hl$zQpX82FFb!$8R))uPaouyx+KfALO*{`MH)?dOHN$Uk)e7*^ldPl^m z@f`IRwRiK!qHf4AyCn%o7csz45mUoXKjL~X>Rq*W}#y2k+Bn3V; z_YjatSL3bJqQ(2BwPVP>u=9zNdSiAEm#i4iPeL3YF$V!E<=cYxhwH^6!a_StU{7*n zERM2}q;9!&2x7<-36G(17rqg8jE9~hSs`G5SoI8Y%@1Q?07Z!hZ*zxgqqnRW- z3)xFmqArKCwZfva!@W0**018qg1-N%?0ASC%b59=IaTS6e|&<|%w1gh$N?8_6J`I2 zZEH0hoqbp~w@3&{)6sY(IKVv%nXfxu6l z-r*e3eTh1o{&ug$ego}~o`pB{3_IOya7F8ZvCEGa-Y}T1C{8(D>*@0M83q3Y7P^5jAHmEyd93o=iTGoREE9I`yvhc0+k`vid#(P{}E$GUMz%ID;yEq%%=xxUa zG7Bh89ufyhSHJw@kb z9D6$bq3!(X*yL=+QGLp(6dwg%3Q3mx`UdQ0lN)0N)bdGg13L(U%=!8<-|}X5q-yL^g}K=<;=@8R~E8yoxDRz`6n_0MQ>2p>@UmC+OEZEC;MJB+*G={NK<^x2==BXMa>Q;h;l zo*_z;(}v1%q3D&o_cDH#8pw*G6+fcdDw3Q{8A6k1{gtZ5Y)TU{-`dR_$+7!usXrm#)QY3d8BG>Nu^B&TxrJ@sU>ONCkqPubTs+9=y&QdV1Omr{d&Css3 z{Om}aw=^{WtAFkQr#Yc?bDb+YGv)=QUcDw70rtl6AS-J|?K5e?!Hm zZ(+WH(aS1mcvu93M~{A5iOD5a5Iv$Ej7&H-a8t{nnx>|FFg| zRLWBj?kg;}m8jSJI3Pb-H8r-=Zao@Tv|bxX9HYD5%!}4(*{wIQbJ@@y+ih5=_^6si zq1^yok|;2X*S%u@jxTfFRP{M$L4d+{;&j%2PfOVIH@R&NURRC@+4H>&3W4ZHZF@9B z7{N*rYk!)2JU)nU&ktY(6OkTE1=V+7id>T{oTmLConurLIdfz#>tS%BXT0W{&o4{~ zH7PH529|Eq0!zbs`|1&4)vlMU)R6|B8GY?BEzxUsE(R_l3dnQB<&cFY2Xq-~kK)@+ zOB9$qwrCd?WLH|nZPhIF(JMDN+2G;8v?2J&9Uy74V568=KX$p+u30#j$6Fe1JEX(8 zoZIu0{li+Q%IoU!Z*f?a4L)*06ygxswPuPztS9Dcm?T}X9IDn%5n^Zol7G9eR~bds zJ8rJ_V958#vahfAZbSt5+I_A(T+Y^#WO2{dc{?6@pVVYsDGC4mVP@UwhrGR0Gok}9 zxriB;DWklnQ$}tX%^R7Xxo-4_^V{h5X z-L7Lk{^@-(x2@Z8uC29k(sAQmx7U7A!tbvJI}wb zTBx9%CAlglW|I6esXQjFPaBJ>uWy`?k6tDa*s6s-GJ<4bj>@cQZ7l;u~LoGqy%j z6ULC)E}UitfE;f!rY&m3%I0(3`H^Ejl3A`(_8d23E}7Rpb9=tc(^*GCD!}WN28V>? zu-ayH`bpz*7z@o4>MT@Dq9W;VqT)$;!jReB(xH+$HjE~bZLLRZa?{e_h&X7Lap;Y-zqE(XK!W_ILtu} zzBC#RP|6U{CinW&wf(`sPJPPIDbi!F?PRj5kD}CkIQ?OH7+q*OUDy8fh>wo&@ucQ< zPB2AD9gA&P(bjB(eeHn;Ocf2Jv_;l`b%r<~!7RA1zd6$5p+zah@?HMv7EQoh(=`&o z8HGqB|0&pN?KsTXD{>mEy{q>i~paGef|o1U+ibZ9!Q7 z>~xv9gX9JGFJgaA-|NdwZ%QlF;3(va7o{F1A4R*>wNXyj?f}Y!Mg8T!9|_XQ&pKz} zIAoNz#ZN(Coom)sPI*c%uo6qhcYd^m9~ymqbuo1yS?ROc_j_O)5|DSci!X6q^?6JP zn`sFJ+zeAaI-lRj0!t-criT#d~8NLnpSM3Q9^E z(!9HJC|v~Ye!dtZ?Y0pY+bTjT>AZ-)v_VUoI=^IKN8&^#ocS$?$ z$!&i8=&VK(S`Dm*~br-iy{$W{`0Kd_n6&J}|>`TmLRk2XmhGS{<@mBRL2UCD`=-ZgPN8%kwR?!1u`V|;LqHli@>4I5kv ztVD24x^-L_0i?mR*ucI``|42i#>v^zFq7%n&G*D@Ob^HXWnu^>{~`4a1;67fXA!BM z&}-?A%*B|eCRf|KuzQqVI7TM$g{0f_;qjq*qvU%pjiC(BhE&m;H``HyJd^VoRW{Sj zn^h(IeZ|&Q*-lEBfZLhZB;~o(Q_lQp%=txH(h$1Ye00H=KZJbeude1tFRdhIs&9XX zJmTj~D=61yaeY^Er8}Z~yc&WN#>kc>$}<I)BmRR$%jyoFtE^eRmlZGR+Bzh|k%h;mKv*C9 z4vR|st1${1wC+U+|i(AIC zb#+$De;xE2x;t0=6%Xf`7{;Z&D|K_WQ&UOJC*YWt-}>Q6>9O{ihvmZ>4TcQ$Z*UAk zKl+~TTPb)Qit)Y`HJI#?=&!Px+@$L^K$OI$IRFtjf!LXk`8yRl7 zeQN`>uF}psO&gipo(mVoM?IyK*MEX{A$=En9hE+3gC-@uS5v;_p#)$T>rLVc?qEjc z!;oyNlqRL2hTg|_>R=DtaWgL+?XPLFq@m7x^ZeEfRzbAvI@@hAqjYJ1gM8di-`EvD z61_%q*(zv|njMmg3E{&Baaq8!Lcwy3ep%&oi%q+{47#aZ-KPMZ^q%b+wF#E=ys z6Ewy&)9HISQ_{7bj^6cx#d`7=1b(vPW35}h^TA8>QP2kNpG>oFe^@tK_hxE_^y`Z9 zKv+;Kzs|e&mzC49cS+B(S_kvg+lIFHmo7LDM%ougtG?sAp4qp#D5#d{mc7+TVZrub z8C(7m&Ct$fIE&K~g(_HJSn_sL~pS?d4x`NqyowU%x z?qFX&!6VZrNuC{5s-lfx@3vS1mBd4zm<-fw8;#&Fl0eXjqUq!uByb?adA* z)>(mxNIU~Md4}z%YVo*gS=?8Lkxb9hJJ>twvkY{l*k*lLkriCJ;;S8Xn?F9i&~%o` z1DUj;S^Whlze}&&gxkUW;X2#T+u~lQ{P*n5&Ma@*hH;nf30DVFdq4DA#G; z)h^A9tj|Rs97R@ul9 z8FilD(kozlD04kcOqSY^d6YIAwJLb_%!FaER9BQTVH=5GMTm6kwXC|RkGB|IQ@n7A9*@~{ zs2^_^q03wwFG0V>L{*W1U-tKXfJ)OC$qOz8NIrlZU)_-Yqp-Ylcgwtp!}S>$YR`7&bD4>8fP z_;%vflU1Ae-Q)#f!;sfYZE*P8z8*5ZoEp$9`Js{6CW(u?}O3m+0RgS$LGlvr5qZiS%U_RP#3Z zQndB$Fy??|dl=6yAu`QgUknUYZ)-E8!wR*p{le#{e?gXFx=0B*p)dR!Y8R? z8=jR2p!{y&e$Xvo8bpU5=^O*>4@7DbLb}B`kE450JO4B$tN1B^Qa{c!#iGT@KAEpA~LcwO6i@@2z3J zPoKV0#3iGarCgY}HEsNhpGGDgxY^q`xQKdLkUlly8$ALK+XLN>I#3;@CiL2 zGbffq%o7u547aur92nl8AI-H#z|6t+?hikyb@wi@ z3WB#F$b8P7><_rXJKXH(5x0=&^QS@U6#x?D{%|^bONESim_w!a!K{DbDwJNh1!&q| z-}ze)&p;;Ps0py*5JNzF+SKOssB6inpR*BqI|WNi`CT;dsR}y^tiQbV$K|)-(J~q6 zJ0gcm+YH1r$2^1@KhwfjWHhI#wfrqmF25yR!v^@=+IQif#qn;DP^DsMI(*wo;;Po% z5+r(ufxp7&08k#sV|yll3zTy)eiESf=kPrnv>yDx22ff;>zk_+Rd>QKRHF_kmF?#x zkAUWAdn_-w`Qmqrpm9#kGb+CH2M^vqgk0i~Md;Dl>OpLx0km07qu3IJ!v{bo7{01d z4&agsRQQ(ONzCNqqM`ZE+IJW!XNedA!HSlV=WZS0`v5o{#j=_TxREEN63iPZEO0g} z*aF=jut2A~_P(V)r`?n#prF>x@V!3U9+kw7^l@0fgKPkKCB2L7dm4y9Vy-uQuVsx~ zr(bVRYO2=3((fFHE&iB57UV=mIYw(OO*oK*a+=3}k;Z4T{w4t>AlI8=N$;djGRy^V zQ~T%H0&z4lr*BWTWkt_%so>;rK0{}6;v+&-Ck9g|@Pkz{))J@q&V#iOx8wQSBeBw$BPvjw0Giu!-5ffEA)Vv z_J53H`puCy3;;l8fv<6xGLSOdwx~Oi5d65DJh(f35ndQIpMsxCJkFmWq>KiR*v~46 zZ{feYheP(e_wFE-ngFuE-ngV6GJ-6Ri9<5eQBb2N0CXHu1_SjXCbEFN2~AK2;@i&T zH9IUSzMqi~K8R;Kqd3nO1S8!+Fy~bC$h~C=*$PbW7nzU+lqEWe!ME*F7=mzZg_z%y z!8tN;l97_K28wM#Hd__3k?-D;@(L&sdd%X9YWKU;QJhoX`Eh`6y+a9HB@kKwvkxu& zev43Ep67S{$I(aKlPFN`--@vkAS5*!0grP?0`ef6+~~+AXrt~Ic+5NR5X5{htD6&0 zJlffOx{p$TpMA(K3?YFVf`q~hc;Etvw!{;NEc&Pw9!fh-_TgW8jUVWUZzJ@6uOk7x z;fLu%hytV7hLHyv=dCC~1JCrE0(!~2muOLV*%NdSFKV%kUB&!Q_y~~$NzKn8P-M4M zbv;E(Q1KgXIl?i$RS4wMfn|fIh5iMHj9#oz9_>`ZZ+JFI?gb-4S^zldv^)QoCJ-9i z2}oN6E(2$6Q{B%O;LzT#{lVVNY4i zFG`>!a#7A&3DRd4Eo?r7_4xg*dEB&}$*_nRg9Tc?pJDgDi)Cw}5Sw^aHQ!uckj_Al z#$-!bDjIXO1g*$!#T;M-sDc%68A|v;qEo;h6u4^e-3#bC!WO*ld8JXCzC zT)7aS`un4LbEInhHI6=1B+GGtdHd$7r&9r-art!46?v}tcj97v@pFZ2nYvp!-HgQ$kMhpK{g9`~j zVVLPu{}Kl=vG2g{9=tjQUX8{jlE{XljEG#6odaC4c;4q?w`g}bL$DMM5o~7%Vm01e+hv{Bfv)Gl34SJUSvuV zEtl-nL*e}i4QxK&E~UL_hoez&nU>3Uo%?}sZ~1?M>mSH{A;GizUCmf{jl1=;0?#EI zlLiNG0H-v7))_=>T(nx?>s5`*x0N4XZPX9+KVoWNmsJO+Jz)`SIZfp01(c`Heh$Mu z^i7D{D7O+uNGHH=Q4WZ7sgeB4;Frd&Gz+2sAOUo&yG%%PI&C)o&-eOn!+Mx^ZKtd0 zOX_5!DTu5qeG<>76i74Nj<=SmA)nzZuqv6$+$ACl4n&Mq{eaf3K%4`(mYjst2%kk? zmRg!RV4{3L8?|_jqF7ga%Li3mD+_0i+UPJqcw>w9KW)hCRyn&tVV ziw3Px$xozvZBuK32Y#y}pKHi&4hZO$pZK!Ekj|1Y@PmL?Z|KFc#2+WfBN*|42v%DI zIB8sEC-mU0s^8_I$h$;0)2;wi>*H;h^?a*)iRa1h#FHtPds|$F(Q0+OH7S5Fw%$Ry`y03yzTlM2&Q+p<&p%i$a}(CyB=5zA+#Oj(S`aL4s? zX-uw>!Qls|;g>N^5Q$;Y6xP9fo?olnm8Ju|sUsRO(%sVN-Pm8W7Cs^I4kI+s!?K345JF=@;Z%p88KMOHYl- zK{ub$m~V&PiO=X+`pTH41#nzlAgr6HH0ek+Eo%5o?P~&*$|am3XV0rXjv!ha1BE6$(H|}NO%rOg$wCw(sZAb}c-a$zyZtfE&f21ch!!Mel28(7iQin~t6Z;l;PoIHR!ddh-#B=tsNfp8 zE=+~vczyPmZ&mq={A|vv052d#DgjKO3=1AY^c)<5Gi^L+86*jbQ2MiCkhjd7ZXP6Q zg1-<2+jJdeVWD}xJlp@M9NO>tv`I$i#UpQ-XbSRj6eGv4^y*bW(Q9brdvy$u+`rWo z(4MBZ&|eljPgX@}xwzTCf$ao`N{AR9JI2Cymb7oxS2h&twX7zXEJL>B`}?5`8y+n2 z^SOBq*jw_Sq?E8&e}%TEmy>XrN2Ch5I}#{4IdW_ee6yAMSQJ}>`CU zt=DZ}v=IKrrgonGe*SuL;9=CM4sAvqfY8|)r(g!+F?(@3Cad-@FWx^O#j_`JS`(*a zP;L2~qxUQ9b)8gJ5H4<#G++cnZHD6cLK@=}vmaC1VHb};OB;G0QQstXn*E1Gd4rPW zDLQmUw#ik&w(*>iVjx8>8ghS#i?v@v<2Z*(Y?xiqyj{X{Z#%Tc{O`rNHV~23FVOslS_Jd9KD1Q@HlBSft?~ z6KI&{4>ngV#-paX*S(f>e2SmDYS;@h3|whr9q0!R6N>U-A&|7jl7rBbrz`ph9gDft zHrC{KK~z4Y@)kF~cs}-$Vv`5{K3!MNmq_zC9V&Bo!-}T!m%ZvzE1(G>Pn+YAS)1&U z{cP;Jaf|#-5oQe(#DTy>oz1NWKM+(Mj7Q@tArfCZPF~~3Z;tU`0Q`!c5whBG1>~8-cO%kp`y7j6%tZ(dbuIG@H@8tysEMK++i~(n4uzO z((|!)y85MQ{g0+sU*jy-qH#Vi>4Q>R+_-#J#*a%In(b730CWPdc?OE{Ky1X}x;Z@Z2q zTKgHLJ4onL(Xp! z!SgEo!{jVzWzawK*jtjrBNQNCS+JshQIW~*5%wX3cSw1gAZx8v_HygFJk7+JH4d#5m-}$FAj8o#8;Sx*-qF#9Ju){X?7T?9b5)SsG2!CR|AT?VWOKfd^Gx- zxMBHCC&K+*wbay|jij1K9szT>UAWQj=I!Wi7D^UJ<85&jY2F`kT<&-Oe(KNLV4Lz% zqW;xa+4|R3OE1I7+h79(WaCjvJ8Szb=LH__Xjq{`fC(j@cKXwGuaw=WS<3b6!%%?# zp1%$_XpcEnL_eNR&1W-O*CVvr0|%rQJUe>C2Q{7a`5?J+jFraQadF9ttB|#1;ap;0 z2R|*eM12GPj)HQY`+2&mG*$lPgh2{8nR<>Vf@&8z7qr;owNso2ANzSNs}@_k1W69x zEi-6f_wXv^$K{z-!v92PuqM#+Nx5lk+9gd{v&*<951g}@wn2CLz z+-W_3l|$*2H|g0gb{kA;6XiRsTBjdk**2`dz!>l-TQR7P+o@#a`$l{}HMsY9cJbLs ze&6}W&{Eac65Cu}VTw%ka805tlO91)7QivYVwYVpLC_{Lea)0Q)glr%3{C$0Md%K$ zCM$l^8(|p-de2Fg(+XE3SWHNHU7A$fESh{R30Wm zD=FJa--HW4{-b_=mXc}x%8vpr^RmkG)E8FWed;i zr{SYGK4i4S2I3X*z%OT7U=&)(Pl+XvHHehptu{&ea^c0gMv@j7f_ z3q8gc(X4^<>EffssYLce1m=|Y(7XL46@J#g;UqCZLq4d)`NoanKY)`xQl!Mx=G+5E z3|p1ZBUBUt-Vy)(?m&>D-z6sR-?v!)tdLEvYPFTr*4xz1SUs155gx?Sok{Z2X~eGi z39%yl{SLuZ>DcN84{G5*&DNt}Y(MK&}uwNrWVboJv#o6!h zyOfct-kHa@e0Jk=(y~wt#~a!X23Qp4>pDR<+@GY*Xx~dSdT+m0D{nZyu_jmQEmNYv z4DnAu=rd_SWxWnk8Q$nDsjwRT4#P6gcQlrk-{Q7sY`79to4{}b`ZkJIgRBlhrWzJ} z3~U$LjUJG2id~&gb-4CmegoxdHO?vcPsQr>@0q&N?Vn9GAn__YAJA>bn_D;*u{w`n zt%oW*fjYxAEAIWGWoBT)pJY(X?{`hu+kR)8@>F!#|3J9#vhC9I7^Vv|a+@P;vpQc( zGSLxq`>h2iHshgANi2S;IcWETFQiK0AaA#W4D+qrKmNuwGPykqkP;Y$=T45t$?tLi zs24^F{%&+-p^VULT2eK_HT&LIgdkU36f{O<;TbihJHd;Z->n%jfk00kTxu;L|GY6;swr zP|^XOZJ@RrTl7-WZu~#(=XR2 z2=4bsG$mUCKmvy}a}AW!qTGJ!b4E~#ir^vtnyR$d4JIPjhaT-rYudu4-9X8pqXYR8 z&e2{3z$`*^99&NQ=SRDAB7kE;eYgL)|m36X>Re83z`>O9MD&yft4 zSs26!LAyUqLFbZcOU);OkvH6-6ZqJG<@{uCVf%GLFcO4}hDvn4UuS?dD+Y4UOy|S> zDyPUHe*A!Lm2X8bT=>k&8roRM>uT*Z>^z^)eU*0?@Zx>$_Q8?W11C@K({M(fW$KMU?i?Kx`Tm$Mapzph3 zEtoA-F_~g{Tx2af)!{Qh5Cs#t*vV7jUrX3TatC~4fKWFe+eHd(l4|)aB%nW{D2|g9 z;0`Y_d+J*e}4#RrgRBdrDW|%D4_4&KuHH1n2t7_H_A)pbNw!6S# zA=}EdYPW_xcKKD_Py*;(QGpwp<(0)iY#rYDw|Rri?`^?;!$&F1@@YN?va}+pSN)K4 zxbh020Em^P&^uA+EsGul>NekS#HqW#pZ^8uj9c9?oM|;R9maA-K(|oAfLC+ES;FHs z>lb!_JFFS^Ej`J60Ff2w$_c#ax<@HBvf~RvElg8YxL}p-ix)uMe9-~znoDJV^v$+7!whg-3Nm_Tav`cs3G>ql)6NRqa8} zh-DDb>q7=a-s>(Kex43jXE_Y0M>xbGoo&2&&WV9C3=ai7+~OqWaLs{c>?N=|H-{=?JlMK0o!<2yJEtAt|#VUg+ zmgO4$LD=|R4x-W~icyh(+jPB~k(Y%V-~BblOv@NM!51*wkh!)>C^Rb30GCer$v_qi8^kq<^L zaF&q(;3k%(&Sdu}9FekMjhBzZr#SI$uu~Hb)s17996b)&d$3eEb6m&O;Rpis=_*Mg zG4P_QF^`6-3s`foNVZJz&bz?Vq*}#XY>21Hk)jT`1L|UEg&rq@Jpt)V7d#^nEWb^z zR_vT9EJPVURqVe%1Rq&y-OvemT0ej&vk8G0%SUr<=5N>2o~3%0CfppnR-A0 z4w=opee@?2Y$k6s2Y6>=_6QRJZ{!z5s&j8k#|lNDi%}*0xs|BHurdnh(vG-qP-7mM zcK0P{K3gO}?qx0fjGO)h(QW|UBKXiP(A9nL$leGv-a*-De%wLkg#&vJlSa#`pMY2^ zF__zV39}aFY;baN1E`a_QFk!*Nck`h%gZ3Po}v{lh(060+XT1{F?!3Q_9t{_;b7fa z&wG6;HF*@C6DJ)!5*0*?o|qK?^b{l)YF|KPHKhpr#2NuI-yJEODG7qwpGYJC&Erb` z52XM90Mh^e5hs%askfc|2!LP(SexgGehbjI8XfJ`3=5S0Z74TrDA}4QB(^4v{8#U$ z5=d2j*6sv!SH6A01}VdR3lMY!R0uS3uuqX~C-VlvN28M8cd}pp2Z-KuEF92X{?_yX zM1h{udXnjQ=1t|Z+7}&He>GWjD_CJFk3bd!r;`AK5f5k)-+B4R}nw%^^gCET?&}q*! zvvJ?@K{%FD0IAQq6oa1Pv#+=9FD01gf>b?46o!Am&m08>5d^~9qu3W5Z`&H3W0;hPs@?-s*rAdObGz1SI!53ql5@>urM0KG0g3em+FPu*7p3fxxr zgX&n15>We=+S%;iN8tfk$d7D=46EDNJHwXT^~N}b2C^)vfIStJ=qO9g2gTpVkIHEu1wf5cfu8b? z(tS178djn`_R0&iYav<5Io_YnDK?}lJvS;;c&dEl|$ zu;HJAmtmAt7`2oXW_l<@g2r5U!$)jC4sNyM+JGw!cw>O&QnvSWv&$Q;lRrz{iC&Dm z0}$hIVk24FuAIS;t=X=cC1T4*kB()!_DuTP*$NjX? z+(t{BjEmJ84ggS?Qf4kT0rFDDHpem6n^jL-e#i354zHdLaQrI{B|o7Z6ESLFmI5fp z4?Ar}t7GYkv6nKKp}4d%MQL`;#Tmu=5v&IF_q$$eq_Pu1qOZpjRx#`uESKv9KOL^&r_7|H;we)kHmDkO|zSVLcko8XNW zxnG;+2l5*>H+Ln$yyM2-8T82Iuwdzpc$mkann{K_k^Xa#AJvI8mf3oh_1!$mE4vkI z7sW&ozVcnz?pHl*C(Hzq!kh-tI>-04?JzkDTYWJ5UtNxMa8_ILVZ>)?KiUD0*OT_e z(0w%c>jdbI``-Lgk8KR}A2$=3+Fca`3+nbwOy+3<%>D|gm&-lRlq%a>2r6Y=cJL)$uggMt#`xya;a z{d)@G;6W*oqnsg@bK3o*50B4O9I32=x@%c{+VjUAVzEsc0~)7?Fg~9PcYPYyTdYkl zfYG)%%p{&Jy;d!L??@q7r0}WbXg1HrYbm~-k+6OjVne7u;37(M$%+_XWSo#Y%n zZ0{}Hc7?Z9%ff0Hp*LWY`R9GPA^`ZlXxlgy3#kWYWTssGV?Ic!;Vfv1m~lrpl5HjD zw_y#SEuQn(AWH?WK6&Ryc0bAF`-*BvoYO_GxhEc|k9kw&_5G9A9Gcd*!yn4xMrszSMC!DC zE7WqZY_*leQ^<Z1JX=6#yw61jf7?pZ1;gq zlGH{CjlE(m2bulA>6aUE^oqHO#JHmJ77}p`f7TxdK>7E3@M&9y3N5#^al9F%E1}$i z-|WJKxqct)y-i;q$)ZH(fyiD_uqB9ePl>esca9a#iR?J(^0JWh`XZ@%a4a=1)ICEw@>+V&FTv;CS_tqm=kyEcz>9%d52L`46vDPKUV293zz~URSfDNOx$B zu+lcY$efO0)7P+vH*Bo^Y73^7*V=TB@GT5_*W@w2$*fN^&&^w`>Gk1|V!zOgK2*u+ z*9Suu3+to5HKg{fPXr7(g1;INlI@>_?Q+4?=UYeXRXLhcf7BLLYKFu1f= zp-Q+^6kyC5(e7!Gw6dv34V;h~084kbUhMQ&zt96sqJlQ_m%k5dqyVFqHCEgX zLojMrm*_K+pu!t9br2!+k2z4ae~ria6n9FMcpS*iHS3ggUXQbtA{~T^7L>pUvG_?p z0rkt~e)+o84;wD!&N!RIUq|1mid8!rIeupmVLL9m?=OED!%tj|Ka$xyjc>>|yc}AK z+$nGP9Lt@P)VKqn<1rnEtB?C|ZSY)bFf-Gyswn%f3e}*Sgwur6lH$>nec;s3c#26}Y=*1&{nAvlFSi>=yoLB1EJ|OI(NQcd`fwETLFEPi`c<)?J0KAci`0&2eWot;8 zsiD{Ym+k67_HZGbE(eer%1B7oYr;T>Z1q7#ld@tZ1zip&lfXMcd0O zcvJ>4le|il9WL&-4rz_}U+legSXEuSH>$YQ1u7{b0@5K!gLFxYfFhkD(ny2SNOww? z2nd3Jv>+iM-K8K>Qc6gdBAk0JefNI%e)rzzI_IDBedqfAd>$urjycwxbKK)s_nS$w z9;bId6=z}oOyBLf2Z`*ga96n{`LM_{1#Ffx{|q}%aWY}<55i-Aj)=G#fmCN-G>pz& zg8jd>@FpR-bD`rqXNN(`&g{wgpnlW0;4dFC>TALW=z%R@%ukCgNogbcvgM@_a5CM}UN zs<`pKJ2g^9aSWkR%#u*-aAj@}5@6IJT=X1*03PruYm+gNu)!|=%jm2Cfc4M&+HZzi zZvKgVaFGsnO~ou5gN9-Dp!`@=<>xqq)eHENt*BJojedmJU``X{H-!b64Ho3~WG`j0 z-{n8MO+|-ZyFz`2VwRz@%(%;W{_Q=0eF4y5x|YYIa~`=c=hgm@K9qJx!$^Tf0px4F z0_mYnht8|{slWK1^JnuV*iPMRqj;PuxR7XMVWVp^cbCN@pyA=0FaMefv7Jr$`)ir{ zWWY+WrFxH5yjP#_R+x6ReNGL?eTIeBYO^YBrml`Tv{MJ>ky*cO%zvbcdlEfLZ`(?0 z%2eGpKU86~g_uif$=u6_3W!10M-1!UPua1^Tg1b(-(4axI#$@~3z`&G{ z(^7VKuNx2?NJ(k|@oocCzxLLO)7^duKu!Z4uhrUM?TCr1VO6eDOez zBwoq*Not(Gz^;fP@gaEj!OK$!Og`jzaUpjK+CtUtfM@PT8Eaq8TGZ&*U3}yD` za!$i0KRG!_7s}jIvwgS~UjU$GEu;;7K8M@-y0)~yQv|sk++Q16{)8CKhhu@HqUFZ( z7C+TW$L}f-+jl+ZxYB9)qgjQd=+;6{sQS%ItlHUg{rmuGZ!91F`dC$;^P_t4Dkeb( zq_WZ=Xe%?M2NLGhhVs-EdeCkrpyj4Wj|mv9rw|57^l)9$Y5M8ed=L?;{{|OhP{^O* zT>mxp$f4J7`VFO+HWQ-w$d#%R9CjR>RR3M|q{a6vo{X)dx3loEKGe^jwIUFPV|-L} zf7;iX)ixmORDbnFtO)!(hy{Spcz9(#My=pH>wOA0=(S0=1*I@d9^q%?;R^|+WsMVu zkIFqCp=zF_VV5=y{;wD4AHC8pc_3Z>l{#!^XQ4V$H*BXiM&@8IQ0!2G)u3K?0yK9^ zzxSpzfGa3)37uXSUQfh3nWL&}>$v<;5#9~!B|jpiJ8hI+wSoX#jN3ivzQxFYxoqXj z17tBUWHcFGLL)dXEnB6POw4qDVEt^@vG$!0#lgiv?KcW~QurLqgfE4cAM640FSGXb zEl7Z~WKB9lkFBfcRHK zi%SbK0wN(6k|yEFJ!}-!9o(lG$A^oEvM}W8$SuY4}lN6T*k@ z>A`rMj6zNE-$=k`|BTww?LYg8L7Aua`WROLHY)2J&;i774JrVZf1Y2H{8n)agOod2 z+;`_j-Yt_yTOvR`*aKK;<_~Z(OF2^tALjGhE~GuYcoF$E??pTai-m(2n|Xgu&D_v!)9GUA;cbjLRcH9@H*)*`!Hjf_{u`W@}S9=eg<~v zM=q2%$-TuhVb_n-qG-g40p%rX)DTiUcW~O4jRbCt?J~Ujjc^(?!?2PdX7yGIAkTuu z22Al?GE1*u9V)IVY2ikXQi*36`Z7Dc{4aBjsE(0q-~+Ji5wJ(S{TXv->w>%-^+3n- zDVJ3qT*aEB6ab8?98l>jV(n)=U?D*?Srn{G4j~9Ce@&LNRUSBrSi?ppG`|AO^;kd3 zFcGuFM&FzfM3&eD=JZ^c=^T1#0kpU{KowDE|7&`k>bMq<8W+hE+$Qz^`m*FQW)khu4oqKGUQDJl@f8lz;yT^EFqm@}r`QB45ceq!`1Das0_ z=#502;Lsm3zx?j=)C8nV*T-u7b`vN7xCSR0G?t>xzwljGF?aSP?aJjTuwSI)YK^<& zo!pE>QjfLaK^wqf?OagA>`6BxF6{^)PulFWtjn`>P?L1LxqlIr3Y^4@m6;C*#uw@% zmuZ^@g@~U5ao-RS0E00v$Wc^O@VmDAa{Ym6lA?1SEhFRk7@i9ZOO)4SApTjZJ50dZ z0AQmDqd>N9*8jA1`=3CeZzx2CsIR7ihf*OzFcMW~i2Qw3oJobE!e;}#(bj0qXAz+Q z0?!e&p0NaRtd6C32zAuI2hvq=lRx(s2Ldh(92Z=)w?O(}R$*P28FIt6S71>p{Ws(a zyv_@3dE0`c!XQ_A_7uUN+efBz* zJZBcn;LaevxazR|h642F;+g>ReF=XI2lD-p$vM$=)yv%*tQ5UL5m=hPzM#^o`UaYv z5s29khH1|?i74rfMz{#bHzgc3TUI>g9VanA47>&?NyLP32L##S;c8vw)8ha00k<2U zetQ%0dAaj@Z5=LUE5do8LlDkmMh4+L3gTN%pMBBHY;D7myS016@iH1 z@mf2cjvt>wJ=#r6AgwCFu-Ob5rUaM>?D-yLoRM4T;~x*KK^w8 zLLW;;B9;erR!ht&2He!(6dE&4nwV3-?`F?;`1cehm{TK930F@r997+{3sZ zU=%X8RD!Pq-iIPLf+mu|*ebkLLSQaIE7~mpzRv9_GLE;;1qM0uojtJA*o{oM82<`e|#D1uY{Zq$=~4k$#vK@`m4Z4 ze(+*VTK>Zj4eO8fzkd9n$hYhhQU2b)qr1ZJ6alxOz^l)+gX%;b$@56Tz5gfg5%Z{I zz@u^)9A^~o*~96^!H9&E!7?a$5CQS_4f zA=b#7X%1Xe5&M(Wdjhoyj6@uXyxR_3rKIpl4>}^`tft32i^1LEh9lQG696ZMVe_u# zl(2z^c?N~)fx>W}#wWLKYcpKNC&xX@NHTQkx+W$QWQ5oA@ClUX*}6+PgvieWpOi_M z?_Ya;RZqPwmf^t^pZv;< zqKb+G&xJbkiM9BDXaVe@tTHwwX&7up(t+4TpN`B0PWNM9)wH$!#EDQ4cHJ~cl)5gs zfaV{lHkEZC2cp>{?Kel+8x1837g2-XEkI8b-d^vqJJx0{q{pjttZkmPy8~!!X1Swj1Y7wKu>b+&gZ{HVU69I;irfxc{w&`M6w3K&~H%H+zoye%Th=hm%)e zxc`}>XQc&8rHrlaCH#{qX^+J(+Y$du4fEw}7CyPC6gF8GJ#wB;+}g17{nu4oH|&Iu z?=x?4sL_PhFg4mBepEtEp?;k<6w%6%NGH-m|EvTY8qGAi`PR2Et9J6&Tyd_GsIy5Y zk^Q~%WY6b?Uk|Z1?ce-ohXe~#>^i(HDuvn=%*kQMz7DYzE^QAfYozems1q?NsUkV( z3qnNyL`vlC5msRnF-T3Z>(!`=93O5CG|Xo$^FzG)2)x``YsJ+ISLk;W*(bmJckYOT z%>Uqy@P-hmR3=eJ3KsR(ci`cPT(R8AT7Ck7*981d(*vgE$!gE~{-^}zNMAi)x@n6d zwodJldT+9r*6|#RJ*K1dwndqe(iWH6Rr%{!65|9Ui1c=HhfecIhhg30*RGo0J7+aW z1bcP`*)T%j6Kym#%|i%%T9p!q-AR+guTt3d<^6z{i{(J+a~yW1LFeI!MH|J*6S4gl%y<{rfOEQc!_%O7k9%=n^fdKgj3+ECVYuoD_b=Ypqqbfs1=ce-FY>X9^!%EuzT z0)5~Gk9c~H8r+Wq$1;V;2O~8t;a#O zqjc7RG1!|)rs=O1deg`X<<4K`pdzHCx%trTZWb{v0S8xOq)m88vvOHeW^u4ho5e-B zx>6b+`ugzKkjh}i7;E{E_~n2pf)z0LgNc-o_}$@V*9aurQ~n;eVqeCQO%1Kcc6$Cb z>eut}6zFjaPjzSc?D-h*k%7Bg*GX8$!K;o1JwSY#lfjM%6(3=`L{%fq;5uhjTO z=3&ScfpeKvCz8hBYEP)-;3-+UIl0+?Z2p3k(`Wwe(Y?adnIIe9!IIl7{Eh7X z24g*vpSVazVztX2v08nNopx;uJ9q9HqbYw(<*8t0$w1t2b=JA4x8&b$kbYxT+f17i zhgf!VjYD_&EXi5zygDG&gIO3LL%h3;_NTXNS+>$=72*a4d4BUuhNzjW_vr0eHI!cW(D^<&o7&L@%6g8p#X0+K>GzcE*^= z_Q}?`^3Yh)CUbzO1NfAE`u_(WiOdAzR1hG_7I5PUuZuqnbeH<}Z6s30N3zpGXD-3* z743KVdxE_*LEF^s*EDQs(&_li-wxa!#QAVNvWr*4pqoF$ELG6SqI}6%G$%xI_R-qh zouB6uezIE~{H2r4Ns(|^T8q*yG_5SO|fgimmH(D2_*&9QA0IpxR@z$Z zi$sz6WF1YA0k*Gl?9Ymhh_|1HD&hDJn_R=%3Loo;@(e$GrUj7C^vxvUZ`h9x>+lwY z$iA{%b&V-z+*}}y0KY<=^L$DE0iz)#RrE%taRNYu`9<$*1{TWY~;rX6lfSW8sfaUbykXF$kHBqK#plqz`OKLCd-4%ps`cl{P~G*Y|j} z69*qMR7RF?0#h$&R$1$9@2fk|SXCwkXv7rSr%hg=+xbqi*YbPls|2nvyM8Ud>x&ro zn8|MxUYqgHLK~`Uqak#RKZaqOFyZ&^IQ!?$>?xkfmGgm>I$Ip4jYU3guxiQgej*}Y zICrK;a`xw=^<=j}BafjEBe`7gX%lUszo>&z%Nk$mkn(5^GX~+g1aQ;umi2jEx8Ix5 zG?)*O4d9&hp{2jC&;pzO_}ec(;{YVKa)!J3u?Jn+42OTH%NZI^eJOPgw)*`CPGiOi z(!C+mZ$TtYn@K(X3aBVKt%Ke#FLN~TV)BwAPeE6Wdn+9k8&&Vxs_5wJJ~drDU)l-Z z-iN$W`ZAhp;R`i-Km`F{UpC<;`XSwQH)*`?c4HjfMM)QJVvm3cQQyyGe`Ko+W4Rif z49cR~9zPs@e|~1--~3zW6W|;5dzzXz5Y7elKXESCP;DW^WG~dh_thCwpgBcY`vLwz zyZUujXUCM~Psi0J)d#<28yGywc@@02^Fyr;LTp}rhUB9136@Evt?>|6wI0&DG>%9y z*FTorklN-o5O@LQj=(9FbNPl<$TFARO_20~9U|R8bn;1c??+lz!4h@^uVs3%;@5lX z20~NRH)q;u@l*=bMaGplzCF@jkZ{zK3&r(_5}CAiFGxs*4A*rqfrb)p5R$O=tjgH( zm`VlrW5WV2X&QQrpGO0z3$x4!Wd?$6X0-^*X&Z7B^cPT32=H7bIa}xI6ztcFq)us9 zb7)-Owg@;>tn~0W{`KgzpLuam2ogzSsvu?SxLT6{<-?S*aCXGB8K)D|RVt*|_2#!D z#fk&{&D{fhq1ES@8$$mE*TTrkSeM$`XZcCo%0b#387es%x1_{*TyMo)I%f~L zAuXYlkM=I>JA@AJZy4N3&@GiRfBHh>!FE10``Bjgm~PClDzk{iYCFCp@sG@1xF_(~ z;OB86_ayttV+w=UN*^wZ+^Ph!zOL{V!vUnO6{wDzFK%-`tf zu=thGk5T*zrpVs7H5~$SoyyvgAI3hP53if@zf=-Yg4DVM?UJt@=g0FkXXK&zFki0bPMmX$;eVp49&qTmgh{heG zAr_O2*?zgXe=k2NzwgYmve28=!&lyu} zhMij*wlIWoB5@yT6E#i>dRpeh8TDxU7>~cJV=XIwmfn5U#-*>LyI2r-v zrA0f^OTel50-q{1#rjFCMnZlhdyu=3pdiAsaITD$!ViS*_HVkBSsuZQu`~Nj91u>UHvOvqV>|AUX z1=;q=r!FA41L%nNVL2k81c%WbiYL=g2Pz8;4!@GzJ=k_gTJi(s-=St{Q#8#Pq^-bM=asx1^|-z<1V^V%I2Q>{xCohEJ_8tu5)oPv0%fYBBj7@INT-r#mINr- zB7TS0-MV%h8f@DqZrFX5;(fAypT42KW9H$+z;Z)}I%VCCe5GVZ@v5t|)DoGXquHa= zQYge`^=z0_EXxp=iaLEND(WXQKd8Bi$+=_lsqS2T3haAsZL3hfy&Mveb1FDyD+ewr zP5@0;=Cx6c(JZ~>EJG}~Dmj;kYkp?0DsF6Q+qTCu*FDJWGYqo547)fwab#yUw+1G1 zM0y)`G|}L7oOex6|A^viXTUTLG&*X~h$Hm%r>|1QR$*5~Y9=lfkWXyC$j|Qm1x~k!4;bSS z5X&RqqLaA3bBLjF*_%S6@y+Bwgx4N-VMUa)>>qZ z?0z;v?Q%BI>h~ywiTD9jwmISd2wzV2{Y7$Q_g$)Xcd~`L;Ddc6qet=%>38kj;)_P{ zWSodK1efo@N+vP948H1X(lYYO5ZNk{r2OjEXBB$bWe+qp3FF16RxS8Vo27C>a&k5T zc)l3HJ0RW$bHBYv%NxM9%*3ZZ5GjZ0AMSxzSgWHU5RY>5SwauC!BDFL8-xPcpLnui zP;=r<4l_mW;Vo+ZO(S_lHG2Hbxy*dd7It(tJ=J2xoVyGQ<fFj`!~b0l8^xpo{*Z#A+AwC*Xv1s|(&Ou&cuMkz(TbzGke}4GhMdMPw@C%@ac6y_sif=g%eTZBC_s(p>6)d=Ic~bP{F@CQbaF9FA3^fqnd2Ud8*R~ z1W{D?ePKWs@`sb+G4xM%IM)0F$otAhA(3liE_{gN=ETWkpuhG>%9ypRkPo+7=d+t0 z61E6kT1yAO*)Abk1}5MF;$)@zH>&ObO!ohtlkJHefl%?H;w`Nq1pB*H4HPcrU>OZ@ z2@ry}?gEypN^7FAFPqa1T8!2l2_SYt@6+?6X;Ga>99yjDBgyxaaM(g*i>9{pANe{l zcMg6AM678SlK5o%gj+%M(q=w%JtE#N;!oJGDCPfHjyjnEM1<3I*Dx)L+1%nv#2K_1 ztms{_4{?F#Zx^wRf9Xpq$W;A9(llZyrPD|rSgRetW)WwZNVdM!Iv`dopn0h7oB#?E zi^|jrt1+5@JONBb?GPD;{~|0B`R9BIgjnOJL5Q{4cxrxfygO_Q11^h*jSV41o}U1& z;k0<~C<^l_khJxSI^{S?Ek#O+M1!gNBBYv~6hf-CL7)l$)#tk1SzEn`9T0=pp9H)f z$2+wI8v{TGOrxpKeyf!X;u1xeth_S_*EM_j&GeJ5iM%I zluQf4U`99_vnZfhR6z|%3G zT)6H<3w@LLZh)(C5p}sXE${>w6OlQiJGt;W<~cc}uoL8M{-06BT&sTcV#h6P#2s_@d{ZFg1tx1_AH? z1uwUMYZ_9v)4oqNctyH_(%xco4JkPmZIF;bfobnk?vJoMF*KAo9V)S`37f}D?%WBiRWygGe;g$`;>}n8D*QY291uT` z3S>59#yE|h=EFa$NvkwmMFol)T}KS4fYQM)$=pho_ZDDD zeM`gN6il3cQ#|cGyXVF^2iWT;LD9JQ`HJw9H`P$4^>DV4XZ<*L<}XW!89^+J!1jTE zE`FZ>d%HFYQxMZ9T3SMK7{}D{(VpeTDO${%cH$`wR=1Y4-yz~Ik|kfq!kWami+YeW z=_Ewqel}xrGrlU(>u0VP7m_Z)SN|1OFnDG@D8k^vAaJZHh)DnD^BY)5VUfgR9^>*o zC&Bqm^|E5(^uj5M z$+o)%l9w?7Vjv`;dY?M)WdY%Dkpx$7K6l2d`eGlN#Al=5M{7SnK?X74^#VYeSni$n z{d+Uc@M|6Rfr06mvq=&?tTYW#3t0lNv+0IC58*h35OGZX{_(g+XmTUsvqvMS9WW0+ z1-ux?h#AN$Y#kp<35^&e7J?bm<*X=Ozh#=tH2EtXzGwP-aL@?{bpq-tB)@L6O%PQ} zZwp**5-1VbHrCD=?E46;mJikL5Vv7YO=S(!>)A%xIP=(jy(khjM~;>81KsYN4_Vj( z)%%d^W>s1kqWl;&cX*BxnILJngAjDZQMy2ZoAVe8?*i<6oFD-W4|^1~*UzgP67(M0 z+R|FEDgyOhL;pT?gjJxZAjZ2j6?)8Z-5j2$IGHki#bV-JD>HvN2t2btO8B0+HavOA zyjmQ0#rN^n_kkusba$H2r`MV#P2K4t>Z5b z=lTSfua|)XVUA#ci!;|NymCh$ka819DGCk;o@gE`5n2Rq{(d1H+6^9(Sg)?OkJRjw zsT_tkH|uJx?j4dsw!N|*ykfj%COuq87YqT|iT6OtPY<;1aT_ajwmJZn>btfdP)0j= zkiiC{X#b7Vb-kKlh;X3YC?1L|oQ14($?0xvzuR$eOKADlb{T;DvYwU=yKD&bn$;X@>*jU} z(pHv$l;&nUtIkIS)EqW03%QjJ5_mT30oqT}1ErPD3$?`T#;8D6puXbx<;r6D+lLk) zEzL^Gw%rtrhsy-#3)T7O*3MnJ&P04>W43mymm54K9k}y*kSRgIKaY)D5<71`&TswY z?Nv_o&ge52LtMJe#P@w{d=Rk1M;Y%*e}uQmBA9VP&>vF0GI4On)ahu9!56h~u$ZBY z3apJafi30&kdSK?$y9tJB>GPbY0Op@A)JFJQg~Bt21oYi8C`YI%FhJEU!&g%MH6m3 z)L;wQRIVra%Y1!iWM7AnJ{E`HWlp*N?L*396P9X>HekgmdC-azTw>Ig3|R?-jwhbzwGsz5cB z>V#W;j3*|qOx68laay9jOUEO+A$Scsu0OFhl`j?^ZJsmM;J2Z$+UcWIw8jl1-mbUnCDnSE0WtjvLU$zC6Y# z)lnS=2oha1kYFI^X4gt{fnW+Eu{qkCZAG5tMC2+Tf6_F3QJ`>9PA)ZP@M*u=lwe6ul01aXV!kGfc-nLV6_mA6r)zWeJ`J%r+!wPuq*NIgZ=EFQ<$gL7XQxe zkFw%rTf?7luKVAjia+eKPKbG8+-K_uG#HC^X#uhNea*(`$(zQjojnyEgsMN!@G0A7 zSLu!Uj*X?3{Ua%iAAmhKk-?9UJ+Ce{xs|>*R}@w5%B&WDclUnfz6&H%F|e@Hcy9Lk z%}u<|)gbWel4neNwx8x160di<#8hPkM|Qw)w|rEgP46fzd?;1Gfv^!`dPH8daGCfC zzT}&*L7-J05jPr}43ouRa@HPp2^ee{U-{UcX!g`tl-|Z0zJpyEQICMEdyNB~;LhRw z-L5m%bdQdIoDpR#i)Y6T%6F{SJeo5^o2fsw_f&j=tgsUH5ru^4FLD_e zd(PS$!)DjcDrj_-p7-W^;_qIZ;O!;(ce!T4Ps+YzE13fU)-FoHqWth>m32pj{mWKs zv3J0U#5V8PC)VC;jP{6>3oatDiq>omyMJofwv87-PS@vx>s*r47@aH28xrB6NO-K& zlwQzo9auX2P^=+!9M<))UK zm5w=m`9X;1xK|gt=PV$E<;r6Rv)U2Et*?_LCc!Xfj;tZ~V$7Hi4})v`Vayh9cwx3Osyyp*? zKaCYisAPE98B}_x3%5UA1SFC{Sv5v^lB}*wc{0X+j4z>Tt7-&CcJe~q2jl(qnGSV^ zUoSUaaFc(u>7r$2(kbb+?&#cVh?|ycxX;Yn!CE0jdlyIcc-se)w8| zd^R3*ypS7zHReJieXC4}O8m&_L_z2awYp{6wezytZ?pTvANGZ|Pm=N&3M>7RD*wHl)@x!FJ3UHvh}D(|vZA$RVd;=-(U zWE?C%)tete1D>^=bi9q+IIr)k`sR&sj2Df7J2H|XIdw8YRYn{EAL76g=}rgBg1Z&F z=mErq4Ta09`AB|JXs}eC+HSMIx8@*qncF==Y0F~>G4^e3F)m%Qa8A!_9Nbp{Q;kYu z4V&New<{DRuD857n}^;@eT_$Mmqz}DW(wTy1LIrox`?QbJIh|sp{Sn!i3i3X zM^L@N!U0&blp%rY?JXE_-2H4qv=oe(Hu;X**d62>@PyPpH9rKZfmfZfOXi4_ySgOD z7{4F1bD1D!?jr0*rg4l32y-lWcEQ`B#{vhbf4S@{zK*)g?Sp67e*tq$|Es54(#X}K z|J~x9hTOQ{CF24HWH=G|5Lly60AtqYy=YoX*Z=6K4wOJ9RTIuzrHY7r_Zz`C=kETk zpkyN&%pW9jU2Cd@pAT&}IERbv7~=(+x*8X-b94jZZ(M0L_TUKm82(+u{O|K8tk6dmWG2XN}*qk>3``bdB*ii{7zwL7u&_eYf-vVnW!T6~LZlQUez1K2L zGfsD1x>g2@N(zKdBuBc>(Jp{Th!Z#i^Rcg7MlzCiYM-w!y#8Tpfm9Li5^dF}kU644 zGuy>Bz7q;7+_V9Op$RA(7C`XH4f(IiQZhej8v-VQLYW|)hwkU^&taKncvM1sglCyA zJB=u$+nPcgJ!&gh*}W<33a>@rT2fg)Hid)wV{*)gM(_4pW_;w=!biou?U+=rPwG4gbMw0$FK@T}kAVUuW*E zhIbR5mC&@FPWJ#dGBboxb{laT`rq?Uq9gc9I8B}6ZtCe}yY#{P%S@+x5eHr~A{^h> z8P4nR9v|#I#r;|Y7E_JDhhzfbTUC-9!9#Q``fWMm&`Ydb;Q!~@wMn`0=$5+B9Jkx<%0v1OSFf1* zX9;_zc%8laQTseb1&Kk>zj5Ps6^=3me%SL09eS{zfDPvHji> zF8}n2EvTp&=Ls+)P&Z`WsrFtFA@a%Uo5{q=z&3NfnlV1nTUA?ExN9@EJSR@;ll4n3 zcT2faz1_%Wt#(Uk3*y02AoVyU!N60X%D&guu`iVc^6sBHoANtPIB9G_>d2LYlZ$W_ zL0MRlSc zKe7l-PZejY9sLFb|2qLC1cYS7kVS5Hf{o6q2e zo4&ZqaQyIG3veJAR+f;)H|9gF^9G3lcO=(g17zrsdL3GpbB)o^>(CiX8#Vc1FF;t? zlfecZkIpdCqw+Ubl@rUBX0EIA{L9T%y6WCf|7J$dze0-V@*>Xa9g?tK4Z9Hnay?+1 z=}R}k%dAZIt7;w-OZL-~~#j~ttl5qDYuEJKly@BQ?7(39} z@M(|2W5Y(a63xaBOA3|Xh&-Xlr_*qj#=xjE;?LFL+g|atRWktdiIS`Or>Y%d#4rh4 z7f!Z@ncZA9mPCHe8RK@ld}esdbD1e$7u)ww8VZ1Jhc?Pp}vXG}##y86@4(#B} z2#_BX0JRKH=5A>w@>yBun&Vd^25btVN$k}S!pL{Myqz@If9d1Ub@kDunJ{9spAv#=5j^Z&A`^3~z`$e-H-@pE$`JN{=^K-~ z&x3;jvQr7}_u2>4_jCAU;S-#u!e>8uX6=cQau$vnP`+vH7bZZ?U-94s8r%6$%MEBR7uVNt{ zF|gIf0M2*I#~!+n%$yKppvK36>)oO8&%gkg0DIaXJum*x<^K1cOjs3Qb>=lV+Dzzy zzPy7*kHHk((Qp>;>X)r54SkoE!xdApf)T#3zm}Ilgly(%xjzO|K7n-wj^oK&8VR26H&W;UpV)gSDtiDE>jBls_|J)A1F^CPEg{c8 z2b5u+r~d1JLL0UaH4>=Tu3y5+$7Ju zy&bToeogLmgAV@lPwMd^h&7j5tcPGlWHcBSAQfQ&{!$bGv;zcdq-|Xgv#JWRBR&A# z@9dpE^-lC>oM>UG$oCn>O1XMbW2d~FSxXV5=5X)lz27|X7exPh5kV#!9#Ht{Ca{Oz` z22pWPSnaM&W$f7uUpf6}y2VLhQjD3ZS$(H%yC|3O*JIw7n-6tIdTwV?xc~YlD`ru@ zTT$%to&9e*VJ%GhVI&OY_%bvRv0z*W<8?ClL6RVqmA0e~nl51Ij=lQ5x{6;u5`W!4 zLU56zMd;F z8juvCU#4kJ@zsD^r5^Ygf-EBN{g-No9*8gQh8W`>NE31lyTgLA0J22w)SPIdxb61I zs|i1qr=gw2=9Ij$h&wp$U8Mrf&(X?+tqwJu0Z2%G6-e?@jlLZf%@GEgFqE-~}?kMWxG9NkpEOBZd^?Bcs+mo4di6WY5z3go;T65ws==3P>7TGhU8yRed+snSA|D~59VSQ zfb8Lfirj?;I(|T`)awK^6us*Fqm@>l%dfrq1)Yp;^~we7RX?rz+VXKg$#xW4erN~# z@(2YtLw7O5{~p?xob;!P98&Vy{qJ!72fY^J_+G;$we^{SbO6u&pyKdNB|_gVDT$ewId|ets$f=bgdB-mOtLkM(7u5bJ|1ZHx0jvV`5q7L#Jh~ZY_`5 zTx5;@d5NrOH`QBl{j$xzFrMPYA1w$U5dj*$1+ce}%^?|uGL%_s;Q7W>m@S&QTFRX8$ z`mhI^Zs98Sml{`B3^z+z$m1&tHPsbXv@Q$e4u(%h5fhCiGuq+XF|tS~>f42zcL!N; zp|eXcs1qHwZJ7{K#tKEgWoE!Cfoxl=hqL3arlV4z0mvh001|GptFgPCX<1EL!B=5E zu0HzE#z$w~KwVQ#K%hCgibC$Q;|Bs+y1$`Yxum0i{4J9>vY(4r4q{+~`TCCAj6aFJ z#_i2ssqDUp(Njy(zd8PYh3r3*wd~5*_=&UtnX;n_HtuHPbXWOs5>rz1qnQ|%X`zT+ zvOFDslOA{!<0=LD+EP<~!7-IMV%D({92Sn(({eVMrOxvUpGC2mJ$zFb`3J{~RvK5V zddG1Y!B33HMgV;WhtOsbLn%l*O1c|iDT|!a^uiBgr!U<7D?T5F}=A7 zD0Kn4X&wZ-w+@xWq`QiIM!By!`)7UQaMNDNnPoffZ8e92HTb@yZTZ!K2c+CLDTY#h zR3Sxqa=J)#m2%mZR7Ctj0u>B)jO_D@8|C1dReRBs(G+(#T7c^ZUO~T5jjIL8o=EK0 zHA3>;-z4@%NbFV##BMb}?AAp5v-%n33^2CLEofVRlzu+?SScHtfMi4G*GsIg9z)>0 z$h;2ZZYisdI(B=S`b)X)WKY~Hm#WpWR>>{b>UDdV5fR78HT1{XVpNsUp&;d7NY1dP zYWr}CR^`R=X&t(FPTJ8EykG2)H;_lzOTG}!2F<7>|LDv`}Xj+9yL+K zm5w$XZFg@z*OVNr|mtQC|r|i;)jS5Pc}kqg5YSw|s{R_n5_2hNS`!+Q(C|DIQY#?Fy~Z+?Q)NCj z;wyadd)rCV)3VxN2pOw-XIe!!U-yZI^`B$6M#y>!^?-VS`1yuUUx) zekW|+TrG@L?>tIZg%tC0sY27G&1gWTHcIOD=SWU&D)U`c!^YKP|6L^FF+|y=Sl35j z*u5?w*t4gIU@Sd-8F@N+V=gbwD9kE=spXd8QVqUOmZhWaw`(U`&nurI3f95gE?w}0 zM=#-aCYB_V&H>*A%e;lUTD;ef*mSVbHEV3 ztqkmQNnOvA={1HIZ;2J6Hr~>@6IFUYvsZK1rjy9fHp0f^YCBMz=pFBBGD^m zfW}A>S+JAkBYXzCWW+r+1gcUykOSAX_z^E(W%+s7&Rpi^fmC@d3~s>$x$39o0op)e zqjD|j`^}G_lV1zc8fpRQwg^R|MPdDe^Pt6xfA5PYY6i^~Im)1%OsuM%XU$<=bL?hP zyXH@ICk6w3-8%F1fgvGW(vZ+@Z3hvYfMTQvwldw&4}>rfytfE@&_Da7)@92TSJ^~l z64FrmdiMX8)?fJY{m&Y2twF9+W;GCn-Ozru=edM$H?X2H$It>6?WZt}K<@0d>pszx zgplBAG}j=uTM}9E=*}1?tuFJ7OK+;0VmChdh;8;IWo(Wozeo~SrXL*(Et|6p3|wq1 zdC-Cb){ni!HLv4eFTIYB7FtLQRwQhJ;Z6YwRZqk83;`%2lix!X`N(rCWorA`6!cf= z0n>6Nzs2!Cv;Y>+V};)VA`z0H19^>xUbj}x#z*xifr2Z;(D096B{Ub5dB4xy2DF{* z2lS8a4Cw3?_H=}zYD0^N*bAaRr<%gsyV4M#i0Jz!=g%q)TKve3|J;mZj?KDu;- zeZ+H7r9QMCzo#30dmuu!8^SQGpuLOfBuJ_l4Y*C6ZfP~RcRQ%ZITDbo4Q8{gq5U|X zz{0rz9R?b=Q}v+vqSJ|-YyF~M=DM?Vb-_7w5OhsFAYic>oL`O4 z0rWo^LZ=p6&4Fj=cntZSHyBfGeFzNZpeB66?eWu*+jjc>T*IT^l`(g-WRkW&zAQI; zV>~#XyBVK5gsWws^QanWO^GlYu?jcGRGgp}T{3tKn1aWlzq<*NfKcgS3=_`-C{)qo z-`IhG%50(_J?qleqS$fu>S>|X^4KvT>th|RQvU1@>;_lL0C#VEm8y;W!fOf_KD*u+ zm-;087}KD05_THpL<(A7WjEUjKKwdP zD}m%EAzI!x=m~x!l`?Nh z!IfQ^smY+KnvUMd594Y9Y6a@_G%_eZJqmMTCKmoVH_$HaC0iD8IxKXL8NR#cd~O(6 zpqP=k@(iu1`3}}G$=L_otR}B*ZxzptHfV{>-*;kSA+LAB9qVm`UPc_ORlGGOS$54D zYAL0&Tn-;o#CCa&dk5Qrb5s+F0+tj%W*Xmn_lzJ5RVm+CXK>KrZX5ALbCZCO7LMFplBMm4wXwl<951d`WRW;jPCD^7jM)xfo?vyYgC z+)8fPfG8W>bcT;9NdGu5`FTR$tz73ZU5(`s~ZRR!Z8EA*7L7Ad!`F?Bhdeao1mJW4Wpn zp8GQ$76I;q@A%u1KXC z_i=n&~1l448;HR^4+kEx}BiY}p#`K!eY2MFODTZ z_qlEqQbPoZZ|YxWT8)}N0u5-}Z7^WDY#6-TbMhOt))0;LDrz?(!7_w(Zb z1SuYt!s$O+AYwNQYn%lvXkT*rGLLS6wH&|f2yh1@n8 zazAekWD&-Zb?n7(88wa}1&y!>v&EEGl7Sw8Z}{yQtPQwQ5Z|t7xh_&TiAr6j97ebd z-rQGjIib*L^@|A*e|`zB17ZfbbAMQf)&Ijn{NJ$<$ip8wJfsWL>|mtOMfDsU1(J|_ z7f+9oE&SWV5G8I~qYejwBI68jeH&3y2}Q3zV2KM&Ib`Xvu>hA7-{ZMTBq` zdL;C300^}phGG31s>s8mA3&a)kBlaNGuky7v@Mp_R_3HA9w^Yvptn&eriQx<9evKh zWf!pt(Mq3gLCXLr?=S&hVbhYA9ZzA{GDnMPcLL-C@(}Dre;fKR!m}(04Sn+ekatVa z(nbSi<>U2x*K8qh8?y6b+C#lpyqBO7Pz8xWE6 zxI(R^Af*2c(cb<=b>{;35}NPA=|6z?GdjRo37`2sK;xe6 z=fr+`2hH&$knq~!pMKh4ut3_t+gPoi$LfV*PYCZM(klVX#PR?H(@O|IyL#Ib6tVbd zf4Ck~bdxd$C%)gi$p?8h|L1c5f1b>fl-R<*R?1AkSG@4Z#!V;E4WLQe%{4sFbzc#$ zU&{u^7O)kWHX>Tb#!!+p)bCS?)t?iP8@8OYp?)qi4UPm3Bnf{ES{mq(`JTDJg(tEd zl}xF6{x25^uaufpd+{0pxo68M8>;DN@2};iJ^+41tbAuCZOuJbtM*CBJuyxsI}jeq z`@J}f@YKzGz)_}tt~9-nVsisBSNMvdR4*!obl#TR$vhqIu!O7qcl=Er(GwsSx*5d| zIx2l;g{BT7NRdn=_RAQO6C51yA`vS=cX&*9weSY^>9%jGCZT+XUVS4&pU z`zXx)Qr(dPE;T5M608-6L8tV^-;-fs+#0YKbECML(EJrlyS%2Ub^-sr2L;UgQ{8Q% zyPRbZS_PEP@~6H-xP2ZQk7XUpt<#eK%tAq<8`;ZtO$wl3n1*f`Tw=!ueUIN90Dsf> z1$GA&Kc8{}+mi4Pwk1Zbhgve=+m$S5G!u};Dp)#(6_811s6vcNz6qMPs~iRgof_RB zhfd@%2Qv#F($=PXqm6Rv8N*a4VcDi09>kkR9X?xgDa@GHE`8t$Wl@oah) zCf$kA`=7pp9pkJA0e{+HmivDuu=d}d6}=B6U;OCR|Es+#52tcn`-auFDzhX*(UOpm zh$S*-h|+`gtNv(NRN z>)ZXAHN5M6pZ9t1`*;6_eWqUIab3@eTBbYxT1s}YBae%E6W(T?t$-Pf8s8bkd>_?G zLQ7f2#@Mx3ZWmY9UT7Sn=!7Wh8t}W*Rr3!F2Z_peR%%zDnoV@Q$d*ySiPx#~M8i;8 zT~6Q2b2AseSP9~4AP+aWYYONeWUN0kRerKBkQXKX zmV%kJvrrY16ulJIwGT|%tJ_?iqXn2V*L}o*=Vp!qC_%O1%la zu}{7F(md^xKewG#kO3uSy&=aaI_P#FPpST@i9$yu^!ME;tf+Lg=fzu}m6`s9dZQ z-_7XpJ zWwE-suReIToR9h#)+I(giE>URrSqNFb58OU<)7#8RMaHwX@^;8Bo5MXj>GvC=kokZ#Eae9M=U~g7cjsi?pvwzHoD(G!7uX= zWhps0+7<8b5k2U<6b`Flw86u}Ed}yU5Dik&yfHd;WIVx5Ha)ywJ$mZ_hpn&dsE=G7 znz*hVa|my{QaC@Ht#Na6^r7?%#fnZKWGL$*_?Ki8pN$N^&=j+g=f5W9%>i+4l(V_4%xw`qWX| z4@KfYRYGNxj;F5VDK#(H8H-Hwest6}UNEWd*dm^5T5{R-xl z(nyCFN&B?cFk4&w)gRws-zP7(-7Q&f&KJDCA9&)9dTw!b- z9>Y*}yS`u3c@Mt!YN<1@(Lc+IU-&}!5K&W)!;>b94vrn4i{29y!-FDF3MdgMAVkz^ zZ`{-h2vb~kt33Fn2k}B4`y~U#50J-x`4%OQy+2#td!|mZ=1+2l9V|B5wQS5}chk zsQGzgSy!vmCZ`{$mRS_7Mt!5~Xz5jlAIm{MN1j=}7 z6l>L6-8cDJeDgZN#)Co!n;&d-8!u_9D!3LeX?YCy9-O1-3W4_T74TiXEC+Rleg?m`4I@i_o5BGAD=isn6Xo!@N&UDRDEXE>;&!(%wxhWQ7w{&tA!D4K( zD0Pm(^9PD!m+5vW5TmK9bi^5}JMiDKP%X03s({`8i}1k2As*LXUyWXzmlN65ktI0w z;8%f9Ee=r!(^?$*`IoL|%SYVrU(aJNTbYD_IbFu?0T_sG-yAmSrWX9em@x1fk8h<|M=Y73^5{Z?*2 z_-Z)l2&us><;Hx&xszf`qa^3mqO*g9G6+MV=T(sJX*3&IQR>cRiwp zuB*kYJmVbnkC{Syi>uFlb-nvtl&w3wu~y2$Z%f=6cSP#b*lW1OS`J#cR84DVI_Sz7 zO`T3W&%AMKIB8IGbKD$f;G`>yVRZ>mY{7Yu%r21<;_aoqho91*HbOGH0s_7jia3&&A=7-kK4nk!qc3R#h3Qv+ zTq=FCAd9=hsi2Wgu0%tC6f&93X*2waPE%`lVQ3+Rb&MgI{d#EF=0nhG4qBVyu4$t_ zGGf<>?3T_+`MmvUA4h=xI%0T&6_7yr&wjh?k0?1pyM<>EtuI8E&vxL6qC0%lqD1Nj z=8Pzy113GsgG~U90!aU-r49dua}uT1E?}ZE143{~{o79Z7t|DL!NCLem4VRD3+Mk4 zKg$MqQ4++@cCEMkPR}#iL}0e_I?Mmloj3?55U=q$#r!Q6h{(l{f?i3W$*x+2%8c~M zg4tT_nAkX&+r<2|=LcFe-Cs(Je$i0%{Y!U3;pOglu-VNoHMQadS7hyn2@S#b~OVR8@f&A^`iu$5~^cqyK7CT&s z)Q(8J4b!7$*aG_8r`-lFV`#akz0Yh+bSE`ffKl@(l1$e3A!&%kOD}5MYth>!;|bBS zO}9={rh*EUE)O-~%Y|MTYjx50$Cyr@5;#m=h)Q0rg^Y88JfwSAz;_$~JK$rJ+ugra zkAmaZmnd5?a9{6Z#X~~~!jRd7+8`uB03K2g62Ba)eI1}M_sD20xSqaPw4<;!F|Das zX9`_XAvWqpj5d7XBBOwu3lk5%{YD)3n92E7WI=3Lai-|X{qgSHWF&ZTE93^{(CLTD zEUcQF!>-SQi`Wh5i<~9x`tXP8RV%+Ags<>6JEmas#F{d~S)Eo?7u>;HCAfX7|n{~A}EcBKv zXLBcRK0#(n(tvPXh6zqu+*1644wkW^! zWGI`-W!}k}Qb2G4boRlVa?aW0T0w;ilun|Kh36^K)0|IkFSk^&|6CUue~1r2LV2qP znM49~Hlc>sZ~;5a1yZ`ej&f*i(th?E_=0_xU!8xW!F#keAaR`Ffv*{Kn|`H;-NI(& z z;cpE6&M`PQf7QmOo;&5U+q3N_@3BvjbX$uE$|SI+d@B!b52K+n{rgA?jATB%`?T>Q zo%cl=>*}4;1Mfj9)(je~UjF@NJS{M8dBMTcY*;62cv=N5?89?p1td&(ePiD(V0NS* zO*(zL%3#3V}XbY_9LzY=v_f-Mest!t!5Q{N#;;UzPG%T2IDLNrt@e z<4dlf&(VJARv4OL~Ox;7v<6|2|2|`Xp%@%$XLJZ4oIS1RI4* z{=Z2U;sJQXlf8&oL4B7^)Os|?u$9~qS7`>+^N}KFic}R$*t349P=;l2OCZ~&SB(It zuj^J=7uy3A%uw=Pku-9WorPjpt&3^h0Sjzx0X^S?8*Wn4BgGx#Z#DgkB0Z8p!Jf{_fi#pE?IuOn2q34eHCeSPtz8C3|V0G ziBtJnrD0!kmwJcNOF0@Cmt>K_thTG5ac34c*!nyoccv?hB<14Q-L55-uAtB)tTRUZn3le3$d*qt%|UBdf!9d*D_5X(s9^*tM1O55&LB z_4qC#30XaO=8iksJ=Slj+J&=}cL(SXXLN6-k*+q#K63#U{n%|? zJPV!oRBtiTxbLx^1g71PmDE!hS^q#IH{HdO5+=$!H9+5-whHKMD?{>Hp3ScT!zOGg zM-uoRY5+mvuIK@db;hlP=Mjt!Vrbsk(0v)8r7jGYB(oJ33Q&K{{&ourpQSzZ<&*3Q#h&X2t`v%sHmu~7ua3WgTbs}ECp$4gg{JX)Oje=RF9v97lGn^8^#B&7K=O9 z*N*ZUpJNP(;)rJweC78%%+G!RN(nv}I(K0GszV~=EHW*b$$W$#5`e4T^o+O#ob=$T z+gkl4#%)?rhgVfM8wMNcmugvt@K+97P#H#vk9W1pT9+ z5|6F%YX3r|C7KOKQ|uQzJho3=w?0>JYdCqg@r1w9ERcwz?>=dAtx20A`5hFeQ0iKT zs@bU#*_--;(8$D3=5-R-VHasvkLNR!y9MxKzDlXEP#fv`Dc|j?k`zZF?v2e5VFv{m zhhrPU={%N)c)2+z(a=*sj0dD`7RXJ4pO+lSxzALIBx5+Svf#OPV-`r-)#VEn)0FMj zt+H$+_FOZND`;oYBhv_q7cDZdra!ilGYJFBk6q{9XtV&|;n=L_+VtAdD-B~y18bFJ zFx*XAUNTu7F=_7YpD$AB6~h~t7g&=G*f}O3<&3mZ%C;F;7b0&FbjYGFMzlrlCAtAQ zbES&__cR++l48{Chi@G_IG(DeGzmx{f*>s>I-gh|n;95WBld^|xQ;fMJC3y;K)@y- zh`L~SgPu14apf2k=GPsmy;FhA7DtVzZ^n`9#69v<4OzC}&Z@cpiSgYD-ZPCq4Ip<3 zC$H#fPKe}xw6tdwu{yabTN`K%RznZ9F8$)D?_!;WaEEUs>br6X96D&9(stMcEZ>HW z>O7ZUd+u^dLJuUE>uar6*bk~8OjTg55)`Y=imOA4@dz$S+wdA9ZLCoY7%B&MJ<(PG z*F!DbUiCZMBC=1OsopuzaKRzZv0a(&K6tVAHYY0%l!K?o=q28zd!-YZST*UK1ktqi%kf(kk@11Y81XleHh+ zEf5-6Jsb}q9)VG^j-&Tru%7jt_-y8a;uQgFP?C8m5uwltHW;~IFc(Q-Y2Lh+V!?!% zfd>-=Ra9Fq5LmxMO&DdXIUjZ7t+*w>6ei%Kzd4=zOm#3zPviHeS8s_JatfEFZK^(D zx%I+k8s=|FP=FWu_{wG#-04OGCd<5s*G471Ti+viA=tBtm70ppC+d-4o6jnTrR;Ebu=dX+Hv z4cur1otoI53_{}u)+=5ychZ5+m@&U8GobSjgp%*5-l>}tS=Hb~iYVB<_9J&KepzlMhpFhaUgT~_ zn8sqjkKuCRg;s~STFNHQVX(Jb32KQ>qQ~8!xgud#Tr@Ti(!Qlxm{GlIvNB;1)pZxL z;RuP4LSO-em>?lCvOu8eW9%vv} zW=q>{wZd8y3$k@x&Uqi3Zg>moP@Ns2B4e6rMubAr)fv6kN17F+-LuX<{Flp)wd(cS z99h+Cx%~qwPIsE4oRZMpG~1(k#qU0wi!tIy3XiLp*C9Z5c>FkSD@mc_o##@qr({h4 zldfN2xm=CH{JU6=Jlu45xCYT;Lx8->0{ANG=!;GYE}iMq4~@7v@)E2kc^_^; z*ETug7q}9*816d)Y-Szt>Nl;<=2Y8Hccv~odND;Y2Zkv{)zgn=f9o(nkH*0f+8sC0 zIlE3&H$OaLVXRgQF`s>olH%e^a>(p9RVylzW4!0CJl6vEz@mRCNUNh18}!qWU*_FM z>=KdeJS4NM1iKf)u5&r;n*iqTPK3swL3`~1WJ0E%m)#{Mz7WMh|DjDJw>(l_%%I3l z%lK0Y&&{|S-ho1qx;M4;sLvxfHou(rul+zTm*`;?UBBVYN ziP;)AVna@LH>-Q~V2r>O5yRMXUO;AUsxd9Mz^Q3TYWPhVX{{%2!&itWv4C`l2J<4F z^R(uLYW-3oDm9fZ&$b83_r~~SmF7PlHEG!F{}X8YP)*4V7z#og!faOf?2#f2a*OZv zUak7bbG=>J1Am+21hdP;lk{d}4X7T*V~q?Dgg)r-Lx)}ou+;>&V$RnGUh_b``m7*r z#7X518)OG^K(^>Bxi$(#MLsG?sm>2w{ER^)$>Cm*(lvv-t69#AiOlu{zcr{B7@%UX z@wyXq9y5W+XS>d)Y+ME&<$iFY&vXR??G})Pp;6Xle|pXi1Tqw3z+&1A_GZIbc#Pab z9&)Hgngk1MyQEhT1j38~tcdSbr;y;6c=*K7Bivk38bmZzZGmx_l?S=Wga%7f$byV% zTMly=n<1v!hbtGcFpf%wYGYA?K5xSZaQ)`5&3-wSgO!I++6M79m|1_FxGClXRw<}z z((izREO&pMFv?u!i=%Sf8gzorn}T_4e^dL*x_bu!Z0UIU_6mBzZ-3}irm8+cXMW{R|E0utk_H>5Nvm7%(`TRZ z+T(OEM%^BrN4@@G6xOk~CL~$n801?}y!rYU!#Up$&Vf$H>;FDTxN`roGijAb%%J6x z{{1RYoUdB3%M%7w6Wu-RNiq6Qjf`YT1peY1qpWFvzOYo@FR&PK`~=PI?SEbvN-q^@ zCPa8v4{+Z7#}9ph<-aFJeZBl=Gk8&b$*=k)LvU z!*A;kt-Fj6a2ylscn@vc@2^$zgH^=xOVVyIbN&6b55-_%zR Date: Mon, 26 Aug 2024 18:41:54 +0800 Subject: [PATCH 503/791] renew_manual --- .../mindspore/doc/api_accuracy_checker.md | 76 ---- .../msprobe/mindspore/doc/compare.md | 58 --- .../msprobe/mindspore/doc/dump.md | 218 ---------- .../accuracy_tools/msprobe/pytorch/doc/FAQ.md | 193 --------- .../pytorch/doc/api_accuracy_checker.md | 313 -------------- .../doc/api_accuracy_checker_online.md | 187 --------- .../msprobe/pytorch/doc/dump.md | 260 ------------ .../msprobe/pytorch/doc/img/BLOOM-7B_1.png | Bin 3534 -> 0 bytes .../msprobe/pytorch/doc/img/BLOOM-7B_2.png | Bin 5251 -> 0 bytes .../msprobe/pytorch/doc/img/BLOOM-7B_3.png | Bin 5095 -> 0 bytes .../msprobe/pytorch/doc/img/BLOOM-7B_4.png | Bin 4727 -> 0 bytes .../msprobe/pytorch/doc/img/GPT-3_1.png | Bin 7898 -> 0 bytes .../msprobe/pytorch/doc/img/GPT-3_2.png | Bin 6716 -> 0 bytes .../msprobe/pytorch/doc/img/GPT-3_3.png | Bin 8732 -> 0 bytes .../msprobe/pytorch/doc/img/GPT-3_4.png | Bin 6609 -> 0 bytes .../msprobe/pytorch/doc/img/GPT-3_5.png | Bin 4676 -> 0 bytes .../msprobe/pytorch/doc/img/GPT-3_6.png | Bin 7658 -> 0 bytes .../msprobe/pytorch/doc/img/GPT-3_7.png | Bin 8139 -> 0 bytes .../msprobe/pytorch/doc/img/GPT-3_8.png | Bin 6084 -> 0 bytes .../msprobe/pytorch/doc/img/YOLOV5S_1.png | Bin 7019 -> 0 bytes .../msprobe/pytorch/doc/img/YOLOV5S_2.png | Bin 5796 -> 0 bytes .../doc/img/accuracy_checking_details.png | Bin 39062 -> 0 bytes .../doc/img/accuracy_checking_result.png | Bin 6022 -> 0 bytes .../doc/img/api_precision_compare_details.png | Bin 25476 -> 0 bytes .../doc/img/api_precision_compare_result.png | Bin 4538 -> 0 bytes .../pytorch/doc/img/auto_analyze_log.png | Bin 65144 -> 0 bytes .../pytorch/doc/img/compare_result_pkl.png | Bin 36133 -> 0 bytes .../doc/img/compare_result_pkl_md5.png.png | Bin 20464 -> 0 bytes .../msprobe/pytorch/doc/img/cpu_info.png | Bin 33797 -> 0 bytes .../pytorch/doc/img/module_compare.png | Bin 56655 -> 0 bytes ...72\347\272\277\346\212\245\345\221\212.md" | 182 --------- .../msprobe/pytorch/doc/parse_tool.md | 286 ------------- .../pytorch/doc/ptdbg_ascend_compare.md | 243 ----------- .../pytorch/doc/ptdbg_ascend_overview.md | 68 ---- .../pytorch/doc/ptdbg_ascend_quickstart.md | 381 ------------------ .../msprobe/pytorch/doc/run_overflow_check.md | 25 -- ...76\345\272\246\346\257\224\345\257\271.md" | 90 ----- ...72\347\272\277\346\212\245\345\221\212.md" | 151 ------- 38 files changed, 2731 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md delete mode 100644 debug/accuracy_tools/msprobe/mindspore/doc/compare.md delete mode 100644 debug/accuracy_tools/msprobe/mindspore/doc/dump.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/FAQ.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/dump.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/BLOOM-7B_1.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/BLOOM-7B_2.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/BLOOM-7B_3.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/BLOOM-7B_4.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_1.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_2.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_3.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_4.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_5.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_6.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_7.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_8.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/YOLOV5S_1.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/YOLOV5S_2.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/accuracy_checking_details.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/accuracy_checking_result.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/api_precision_compare_details.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/api_precision_compare_result.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/auto_analyze_log.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/compare_result_pkl.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/compare_result_pkl_md5.png.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/cpu_info.png delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/img/module_compare.png delete mode 100644 "debug/accuracy_tools/msprobe/pytorch/doc/msprobe\347\262\276\345\272\246\345\267\245\345\205\267\346\225\260\346\215\256dump\346\240\207\345\207\206\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/parse_tool.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_quickstart.md delete mode 100644 debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md delete mode 100644 "debug/accuracy_tools/msprobe/pytorch/doc/\345\234\250\347\272\277\347\262\276\345\272\246\346\257\224\345\257\271.md" delete mode 100644 "debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\234\272\346\231\257\351\252\214\350\257\201\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md b/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md deleted file mode 100644 index 7bb454b90..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/doc/api_accuracy_checker.md +++ /dev/null @@ -1,76 +0,0 @@ -# **MindSpore动态图精度预检工具** - -## 简介 - -MindSpore动态图精度预检工具通过扫描昇腾NPU上用户训练MindSpore模型中所有Mint API，输出精度情况的诊断和分析。工具以模型中所有的Mint API前反向的dump结果为输入，构造相应的API单元测试，将NPU输出与标杆（CPU高精度）比对，计算对应的精度指标，从而找出NPU中存在精度问题的Mint API。 - - -**真实数据/随机生成模式**：精度预检工具支持随机生成模式和真实数据模式，即在预检dump时可以选择由工具构造随机数进行输入获得dump数据或选择获取真实输入数据进行预检dump操作；随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 - -**工具支持Mindspore版本**：2.4。 - -**工具特性** - -- 落盘数据小。 -- 支持随机生成模式和真实数据模式。 -- 单API测试，排除整网中的累计误差问题。 - -## 预检流程 - -精度预检操作流程如下： - -1. 在NPU和GPU环境下分别安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 -2. 在NPU训练脚本内添加msprobe工具dump接口PrecisionDebugger，采集待预检数据。详见《[精度数据采集](./dump.md)》，注意需要配置level="L1"。 -3. 执行预检操作，查看预检结果文件，分析预检不达标的API。 - -## 预检操作 - - -```bash -msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result -``` - -| 参数名称 | 说明 |参数类型 | 是否必选 | -| ---------------------------- | --------------------------------------|---------------------- | ---------------------------------- | -| -api_info或--api_info_file | 指定API信息文件dump.json。 | str | 是 | -| -o或--out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | - -预检执行结果包括`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`两个文件。`accuracy_checking_result_{timestamp}.csv`是API粒度的，标明每个API是否通过测试。建议用户先查看`accuracy_checking_result_{timestamp}.csv`文件，对于其中没有通过测试的或者特定感兴趣的API，根据其API Name字段在`accuracy_checking_details_{timestamp}.csv`中查询其各个输出的达标情况以及比较指标。详细介绍请参见“**预检结果**”。 - -## 预检结果 - -精度预检生成的`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`文件内容详情如下： - -`accuracy_checking_details_{timestamp}.csv` - -| 字段 | 含义 | -| ------------------- | ------------------------------------------------------------ | -| API Name | API名称。 | -| Bench Dtype | 标杆数据的API数据类型。 | -| Tested Dtype | 被检验数据的API数据类型。 | -| Shape | API的Shape信息。 | -| Cosine | 被检验数据与标杆数据的余弦相似度。 | -| MaxAbsErr | 被检验数据与标杆数据的最大绝对误差。 | -| MaxRelativeErr | 被检验数据与标杆数据的最大相对误差。 | -| Status | API预检通过状态，pass表示通过测试，error表示未通过。 | -| message | 提示信息。 | - -`accuracy_checking_result_{timestamp}.csv` - -| 字段 | 含义 | -| --------------------- | ------------------------------------------------------------ | -| API Name | API名称。 | -| Forward Test Success | 前向API是否通过测试，pass为通过，error为错误。 | -| Backward Test Success | 反向API是否通过测试，pass为通过，error为错误，如果是空白的话代表该API没有反向输出。 | -| Message | 提示信息。 | - -Forward Test Success和Backward Test Success是否通过测试是由`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差判定结果决定的。具体规则详见“**API预检指标**”。 -需要注意的是`accuracy_checking_details_{timestamp}.csv`中可能存在一个API的前向（反向）有多个输出，那么每个输出记录一行，而在`accuracy_checking_result_{timestamp}.csv`中的结果需要该API的所有结果均为pass才能标记为pass，只要存在一个error则标记error。 - - -## API预检指标 - -API预检指标是通过对`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差的数值进行判断，得出该API是否符合精度标准的参考指标。详细规则如下： - - - 余弦相似度大于0.99，并且最大绝对误差小于0.0001，标记“pass”，否则标记为“error”。 - diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md b/debug/accuracy_tools/msprobe/mindspore/doc/compare.md deleted file mode 100644 index 01b4da70b..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/doc/compare.md +++ /dev/null @@ -1,58 +0,0 @@ -# 精度比对工具 -msprobe精度比对工具主要通过对同一个模型，在两个不同的MindSpore环境下，输入相同的训练数据，在分别得到dump数据后，对这两个dump数据进行全量自动对比，从而快速定位不同版本之间的精度问题。 - -执行精度比对操作需要安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 - -## 命令行方式比对 - -精度比对工具目前使用方式为命令行形式，对MindSpore的dump数据仅支持单卡。 - -请先参见《[精度数据采集](./dump.md)》完成不同环境下MindSpore精度数据的采集。 - -### 操作步骤 - -1. 使用MindSpore进行dump，得到不同框架版本的dump数据。 - -2. 创建比对文件，文件内容及示例请参见“**比对文件**”。 - -3. 执行如下示例命令进行比对： - - ```shell - msprobe -f mindspore compare -i ./compare.json -o ./output -s - ``` - - **完整参数说明** - - | 参数名 | 说明 | 是否必选 | - |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | - | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -c或--compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | - | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - -4. 查看比对结果，请详见PyTorch目录下的《[精度比对工具](../../pytorch/doc/ptdbg_ascend_compare.md)》的“比对结果分析”章节。 - -### 比对文件 - -以在当前目录创建./compare.json为例，单卡场景示例如下： - - - ```json - { - "npu_path": "./npu_dump/dump.json", - "bench_path": "./bench_dump/dump.json", - "stack_path": "./npu_dump/stack.json", - "is_print_compare_log": true - } - ``` - - -**参数说明** - -| 参数名 | 说明 | 是否必选 | -| -------------------- | ------------------------------------------------------------ | ------------------ | -| npu_path | 配置NPU环境下的dump.json文件（单卡场景）。数据类型：str。 | 是 | -| bench_path | 配置CPU、GPU或NPU环境下的dump.json文件（单卡场景）。数据类型：str。 | 是 | -| stack_path | 配置NPU dump目录下的stack.json文件。数据类型：str。 | 是| -| is_print_compare_log | 配置是否开启日志打屏。可取值True或False，默认为True。数据类型：bool | 否 | \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md b/debug/accuracy_tools/msprobe/mindspore/doc/dump.md deleted file mode 100644 index 1582b265d..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/doc/dump.md +++ /dev/null @@ -1,218 +0,0 @@ -# **精度数据采集** - -msprobe工具主要通过在训练脚本内添加dump接口并启动训练的方式来采集精度数据。 - -执行dump操作需要安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 - -## dump接口介绍 - -### PrecisionDebugger - -**功能说明** - -通过加载dump配置文件的方式来确定dump操作的详细配置。 - -PrecisionDebugger可以在from msprobe.mindspore import PrecisionDebugger之后的位置添加。详细使用可参考“**示例代码**”。 - -**原型** - -```Python -PrecisionDebugger(config_path=None) -``` - -**参数说明** - -| 参数名 | 说明 | 是否必选 | -| ----------- | ------------------------------------------------------------ | -------- | -| config_path | 指定dump配置文件路径，String类型。参数示例："./config.json"。未配置该路径时，默认使用[config.json](../../config)文件的默认配置。config.json文件可以配置更多参数，若需要进行更多场景的精度数据dump，建议配置[config.json](../../config/config.json)文件。config.json文件的配置可参考《[配置文件说明](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/config/README.md)》。 | 否 | - -### start函数 - -**功能说明** - -启动函数。 - -在模型初始化之后的位置添加。需要与stop函数一起添加在for循环内。 - -**原型** - -```Python -debugger.start(model = None) -``` - -该函数为类函数，可以使用debugger.start(model = None)也可以使用PrecisionDebugger.start(model = None) - - -**参数说明** - -| 参数名 | 说明 | 是否必选 | -| ----------- |---------------------------------------------------------------------------------------| -------- | -| model | 指具体的mindspore.nn.Cell，默认未配置，L1级别下传入model可以使能对primitive op的dump，否则无法dump primitive op。 | 否 | - - -### stop函数 - -**功能说明** - -dump停止函数。 - -在**start**函数之后的任意位置添加。需要与start函数一起添加在for循环内。若需要dump反向数据，则需要添加在反向计算代码之后。 - -仅MindSpore动态图场景支持。 - -**原型** - -```Python -debugger.stop() -``` - -该函数为类函数，可以使用debugger.stop()也可以使用PrecisionDebugger.stop()。 - -### step函数 - -**功能说明** - -结束标识。 - -在最后一个**stop**函数后或一个step结束的位置添加。 - -仅MindSpore动态图场景支持。 - -**原型** - -```Python -debugger.step() -``` - -该函数为类函数，可以使用debugger.step()也可以使用PrecisionDebugger.step()。 - -## 示例代码 - -### MindSpore静态图场景 - -```Python -from msprobe.mindspore import PrecisionDebugger -debugger = PrecisionDebugger(config_path="./config.json") -# 请勿将以上初始化流程插入到循环代码中 -# 下面代码也可以用PrecisionDebugger.start() -debugger.start() -... -``` - -### MindSpore动态图场景 - -当使用模型使用for循环时，在每个迭代的开始插入debugger.start()，在每个迭代的结束插入debugger.stop()与debugger.step()： - -```Python -import mindspore as ms -from msprobe.mindspore import PrecisionDebugger - -# 请勿将PrecisionDebugger的初始化插入到循环代码中 -debugger = PrecisionDebugger(config_path="./config.json") - -# 模型、损失函数的定义以及初始化等操作 -# ... - -# 数据集迭代的地方往往是模型开始训练的地方 -for data, label in data_loader: - debugger.start() # 开启数据dump - net = Model() - # 如下是模型每个step执行的逻辑 - grad_net = ms.grad(net)(data) - # ... - debugger.stop() # 关闭数据dump - debugger.step() # 结束一个step的dump -``` - -当使用模型的train方法而非for循环时，可以通过在callbacks参数中传入MsprobeStep(debugger)： - -```Python -from msprobe.mindspore.common.utils import MsprobeStep -from msprobe.mindspore import PrecisionDebugger - -# 初始化PrecisionDebugger -debugger = PrecisionDebugger(config_path="./config.json") - -# 自动在每个step开始时调用start()，在每个step结束时调用stop()和step()。 -# 这意味着您无需手动在循环内添加start、stop和step函数，框架会自动完成数据的dump操作。 -trainer.train(1, dataset_train, callbacks=[loss_monior, MsprobeStep(debugger)]) - -``` - -## dump结果文件介绍 - -### MindSpore静态图场景 - -训练结束后，工具将dump的数据保存在dump_path参数指定的目录下。 - -- jit_level为O0/O1时 - - dump结果目录请参见MindSpore官网中的《[同步Dump数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%90%8C%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)》。 - -- jit_level为O2时 - - dump结果目录请参见MindSpore官网中的《[异步Dump数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%BC%82%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)》。 - -jit_level请参见[mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig)配置jit_config。 - -### MindSpore动态图场景 - -训练结束后，工具将dump的数据保存在dump_path参数指定的目录下。 - -dump结果目录结构示例如下： - -```bash -├── dump_path -│ ├── step0 -│ | ├── rank0 -│ | │ ├── dump_tensor_data -| | | | ├── MintFunctional.relu.0.backward.input.0.npy -| | | | ├── Mint.abs.0.forward.input.0.npy -| | | | ├── Functional.split.0.forward.input.0.npy -| | | | ├── Tensor.__add__.0.forward.output.0.npy -| | | | ... -| | | | └── Jit.AlexNet.0.forward.input.0.npy -│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） -│ | | ├── stack.json # 算子调用栈信息 -│ | | └── construct.json # 分层分级结构，level为L1时，construct.json内容为空 -│ | ├── rank1 -| | | ├── dump_tensor_data -| | | | └── ... -│ | | ├── dump.json -│ | | ├── stack.json -| | | └── construct.json -│ | ├── ... -│ | | -| | └── rank7 -│ ├── step1 -│ | ├── ... -│ ├── step2 -``` - -dump过程中，npy文件在对应算子或者模块被执行后就会落盘，而json文件则需要在正常执行PrecisionDebugger.stop()后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关npy文件，可能会导致json文件中数据丢失。 - -其中rank为设备上各卡的ID，每张卡上dump的数据会生成对应dump目录。非分布式场景下没有rank ID，目录名称为rank。 - -动态图场景下使能PSJit或PIJit，装饰特定Cell或function，被装饰的部分会全部/部分使能静态图流程。PSJit场景下config.json文件配置level为L1时，被PSJit装饰的部分也作为API被dump到对应目录；若配置level为L2时，则只会dump用户网络中静态图流程下的相关kernel。PIJit场景开启dump工具后，会被还原为动态图，按API粒度进行dump。 - -npy文件保存的前缀和MindSpore对应关系如下： - -| 前缀 | MindSpore模块 | -| -------------- | ---------------------------- | -| Tensor | mindspore.Tensor | -| Functional | mindspore.ops | -| Primitive | mindspore.ops.Primitive | -| Mint | mindspore.mint | -| MintFunctional | mindspore.mint.nn.functional | -| Jit | mindspore.jit | - -## 工具支持的API列表 - -msprobe工具维护固定的API支持列表，若需要删除或增加dump的API，可以在msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml文件内手动修改，如下示例： - -```bash -ops: # ops为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API - - adaptive_avg_pool1d - - adaptive_avg_pool2d - - adaptive_avg_pool3d -``` diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/FAQ.md b/debug/accuracy_tools/msprobe/pytorch/doc/FAQ.md deleted file mode 100644 index 8d12a7292..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/doc/FAQ.md +++ /dev/null @@ -1,193 +0,0 @@ -# 精度预检工具 - -1. 预检工具在dump和run_ut的过程中，是否需要同时开启或关闭jit编译（jit_compile）？ - - 答：是。 - -2. 预检工具对于type_as这类涉及数据类型转换操作的API，是否具有参考性？ - - 由于这类API在CPU侧存在精度先提升后下降的操作，因此这类API的有效性的参考价值有限。 - -3. run ut过程中出现报错：ERROR:Got unsupported ScalarType BFloat16 - - 答：请使用最新版本的工具。 - -4. Dropout算子，CPU和NPU的随机应该不一样，为什么结果比对是一致的？ - - 答：这个结果是正常的，工具对该算子有特殊处理，只判定位置为0的位置比例大约和设定p值相当。 - -5. 为什么浮点型数据bench和CPU的dtype不一致？ - - 答：对于fp16的数据，CPU会上升一个精度fp32去计算，这是和算子那边对齐的精度结论，CPU用更高精度去计算会更接近真实值。 - -6. 添加预检工具后截取操作报错：`IndexError: too many indices for tensor of dimension x` 或 `TypeError: len() of a 0-d tensor`。 - - 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- __getitem__`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 - -7. 添加预检工具后F.gelu触发ValueError报错：`activation_func must be F.gelu`等。 - - 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中functional:下的的`- gelu`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 - -8. 添加预检工具后触发AsStrided算子相关的报错，或者编译相关的报错，如：`Failed to compile Op [AsStrided]`。 - - 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- t`和`- transpose`。 - -9. Tensor 魔法函数具体对应什么操作？ - - 答： - - | Tensor魔法函数 | 具体操作 | - | --------------- | ---------------- | - | `__add__` | + | - | `__and__` | & | - | `__bool__` | 返回Tensor布尔值 | - | `__div__` | / | - | `__eq__` | == | - | `__ge__` | >= | - | `__gt__` | > | - | `__iadd__` | += | - | `__iand__` | &= | - | `__idiv__` | /= | - | `__ifloordiv__` | //= | - | `__ilshift__` | <<= | - | `__imod__` | %= | - | `__imul__` | *= | - | `__ior__` | \|= | - | `__irshift__` | >>= | - | `__isub__` | -= | - | `__ixor__` | ^= | - | `__lshift__` | << | - | `__matmul__` | 矩阵乘法 | - | `__mod__` | % | - | `__mul__` | * | - | `__nonzero__` | 同`__bool__` | - | `__or__` | \| | - | `__radd__` | +（反向） | - | `__rmul__` | *（反向） | - | `__rshift__` | >> | - | `__sub__` | - | - | `__truediv__` | 同`__div__` | - | `__xor__` | ^ | - -# 精度比对工具 - -## 工具使用 - -### dump指定融合算子 - -dump指定操作当前支持dump指定融合算子的输入输出，需要在mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml中添加，比如以下代码段调用的softmax融合算子 - -``` -def npu_forward_fused_softmax(self, input_, mask): - resl = torch_npu.npu_scaled_masked_softmax(input_, mask, self.scale, False) - return resl -``` - -如果需要dump其中调用的npu_scaled_masked_softmax算子的输入输出信息，需要在support_wrap_ops.yaml中的torch_npu: 中自行添加该融合算子即可： - -``` -- npu_scaled_masked_softmax -``` - -（npu_scaled_masked_softmax融合算子工具已支持dump，本例仅供参考） - -## 常见问题 - -### 1. 在同一个目录多次执行dump会冲突吗？ - -会，同一个目录多次dump，会覆盖上一次结果，可以使用dump_path参数修改dump目录。 - -### 2. 如何dump算子级的数据？ - -需要配置level为L2模式。 - -### 3. 工具比对发现NPU和标杆数据的API无法完全对齐？ - -torch版本和硬件差异属于正常情况。 - -## 异常情况 - -### 2. HCCL 报错： error code: EI0006 - -**故障现象** - -使用msprobe工具时，报错： error code: EI0006。 - -**故障原因** - -CANN软件版本较低导致不兼容。 - -**故障处理** - -升级新版CANN软件版本。 - -### 3. torch_npu._C._clear_overflow_npu() RuntimeError NPU error，error code is 107002 - -如果运行溢出检测功能遇到这个报错，采取以下解决方法： -如果是单卡运行，添加如下代码，0是卡号，选择自己空闲的卡号。 - -``` -torch.npu.set_device('npu:0') -``` - -如果多卡运行，请在代码中修改对应卡号，比如进程使用卡号为{rank}时可以添加如下代码： - -``` -torch.npu.set_device(f'npu:{rank}') -``` - -如果运行精度比对功能遇到这个报错，尝试安装最新版本的msprobe。 - -### 4. dump得到的VF_lstm_99_forward_input.1.0.npy、VF_lstm_99_forward_input.1.1.npy类似的数据是否正常？ - -带1.0/1.1/1.2后缀的npy是正常现象，例如当输入数据为[[tensor1, tensor2, tensor3]]会生成这样的后缀。 - -### 5. 进行compare报错：The current file contains stack information, please turn on the stack_mode - -在比对脚本中，设置stack_mode=True，例如： - -``` -from msprobe.pytorch import compare -dump_result_param={ -"npu_json_path": "./npu_dump/dump.json", -"bench_json_path": "./gpu_dump/dump.json", -"stack_json_path": "./npu_dump/stack.json", -"is_print_compare_log": True -} -compare(dump_result_param, output_path="./output", stack_mode=True) -``` - -### 6. dump指定反向API的kernel级别的数据报错：NameError：name 'torch_npu' is not defined - -- 如果是npu环境，请安装torch_npu； -- 如果是gpu环境，暂不支持dump指定API的kernel级别的数据 - -### 7. 配置dump_path后，使用工具报错：[ERROR]The file path /home/xxx/dump contains special characters - -- 请检查你设置的dump绝对路径是否包含特殊字符，确保路径名只包含大小写字母、数字、下划线、斜杠、点和短横线 -- 注意，如果执行脚本的路径为/home/abc++/，设置的dump_path="./dump"，工具实际校验的路径为绝对路径/home/abc++/dump，++为特殊字符，会引发本条报错 - -### 8. 无法dump matmul权重的反向梯度数据 - -- matmul期望的输入是二维，当输入不是二维时，会将输入通过view操作展成二维，再进行matmul运算，因此在反向求导时，backward_hook能拿到的是UnsafeViewBackward这步操作里面数据的梯度信息，取不到MmBackward这步操作里面数据的梯度信息，即权重的反向梯度数据。 -- 典型的例子有，当linear的输入不是二维，且无bias时，会调用output = input.matmul(weight.t()),因此拿不到linear层的weight的反向梯度数据。 - -### 9. dump.json文件中的某些api的dtype类型为float16，但是读取此api的npy文件显示的dtype类型为float32 - -- msprobe工具在dump数据时需要将原始数据从npu to cpu上再转换为numpy类型，npu to cpu的逻辑和gpu to cpu是保持一致的，都存在dtype可能从float16变为float32类型的情况，如果出现dtype不一致的问题，最终dump数据的dtype以pkl文件为准。 - -### 10. 使用dataloader后raise异常Exception("msprobe: exit after iteration {}". format(max(self.config.step)) - -- 正常现象，dataloader通过raise结束程序，堆栈信息可忽略。 - -### 11. 添加msprobe工具后截取操作报错：`IndexError: too many indices for tensor of dimension x` 或 `TypeError: len() of a 0-d tensor`。 - -- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- __getitem__`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 - -### 12. 添加msprobe工具后F.gelu触发ValueError报错：`activation_func must be F.gelu`等。 - -- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中functional:下的的`- gelu`，工具会跳过dump该API。如果是需要dump的关键位置api也可以考虑根据报错堆栈信息注释引发报错的类型检查。 - -### 13. 添加msprobe工具后触发AsStrided算子相关的报错，或者编译相关的报错，如：`Failed to compile Op [AsStrided]`。 - -- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- t`和`- transpose`。 diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md deleted file mode 100644 index bb0f09a03..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker.md +++ /dev/null @@ -1,313 +0,0 @@ -# **精度预检工具** - -## 简介 - -精度预检工具通过扫描昇腾NPU上用户训练模型中所有API，输出精度情况的诊断和分析。工具通过dump模型中所有的API前反向信息；构造相应的API单元测试，将NPU输出与标杆（CPU高精度）比对，从而计算对应的精度指标，该过程称为run_ut；将NPU环境下dump的预检数据拷贝至GPU环境，同样执行run_ut；最后通过**新精度标准比对法**将NPU和GPU的预检结果进行比对，从而找出NPU中存在精度问题的API。 - -**新精度标准比对法**：依据新精度标准，对不同的API采取不同的比对算法进行比对（包括绝对阈值法，标杆比对法、二进制一致法、ULP误差比对法和双千指标法），最终给定预检判定结果。 - -**真实数据模式**：精度预检工具支持随机生成模式和真实数据模式，即在预检dump时可以选择由工具构造随机数进行输入获得dump数据或选择获取真实输入数据进行预检dump操作；随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 - -**工具支持PyTorch版本**：1.11/2.0/2.1/2.2。 - -**工具特性** - -- 落盘数据小。 -- 支持随机生成模式和真实数据模式。 -- 单API测试，排除整网中的累计误差问题。 - -## 预检流程 - -精度预检操作流程如下： - -1. 在NPU和GPU环境下分别安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 -2. 在NPU训练脚本内添加msprobe工具dump接口PrecisionDebugger，采集待预检数据。详见《[精度数据采集](./dump.md)》，注意需要配置level="L1"。 -3. 将NPU环境下dump的预检数据拷贝至GPU环境。 -4. 在NPU和GPU环境下分别执行run_ut，生成结果用于最终api_precision_compare操作的输入。详见“**run_ut预检操作**”。 -5. 将NPU和GPU执行run_ut生成的`accuracy_checking_details_{timestamp}.csv`结果文件拷贝至同一环境下。 -6. 运行api_precision_compare.py，输出结果为预检操作的最终结果。详见“**预检结果比对**”。 - -## 预检操作 - -### run_ut预检操作 - -完成待预检数据采集后，仅仅获取了API的输入数据，为了得到NPU vs CPU高精度（标杆）的预检比对结果和GPU vs CPU高精度（标杆）的预检比对结果，还需要进行run_ut操作。 - -run_ut预检操作包括如下场景： - -- 使用run_ut.py执行预检：run_ut.py适用于数据量较小的单卡场景。 -- 使用multi_run_ut.py执行多线程预检：multi_run_ut.py适用于数据量较大的大模型场景。 - -#### 使用run_ut.py执行预检 - -1. 将API信息输入给run_ut模块运行精度检测并比对，运行如下命令： - - ```bash - msprobe -f pytorch run_ut -api_info ./dump.json - ``` - - | 参数名称 | 说明 | 是否必选 | - | ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | - | -api_info或--api_info_file | 指定API信息文件dump.json。 | 是 | - | -save_error_data | 保存精度未达标的API输入输出数据。 | 否 | - | -o或--out_path | 指定run_ut执行结果存盘路径，默认“./”（相对于run_ut的路径）。 | 否 | - | | | | - | -j或--jit_compile | 开启jit编译。 | 否 | - | -d或--device | 指定Device ID，选择UT代码运行所在的卡，默认值为0。 | 否 | - | -csv_path或--result_csv_path | 指定本次运行中断时生成的`accuracy_checking_result_{timestamp}.csv`文件路径，执行run_ut中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的`accuracy_checking_result_{timestamp}.csv`文件。详见“**断点续检**”。 | run_ut操作中断后继续执行场景下必选 | - | -f或--filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的API。适用于模型较大且重复API较多的场景。 | 否 | - | -config或--config_path | 指定预检操作过程中的额外配置（包括黑名单、白名单等）的[config.json](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe/config)文件，默认未配置。config.json文件的配置可参考《[配置文件说明](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/config/README.md#pytorch场景task配置为run_ut)》。 | 否 | - - run_ut执行结果包括`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`两个文件。`accuracy_checking_result_{timestamp}.csv`是API粒度的，标明每个API是否通过测试。建议用户先查看`accuracy_checking_result_{timestamp}.csv`文件，对于其中没有通过测试的或者特定感兴趣的API，根据其API name字段在`accuracy_checking_details_{timestamp}.csv`中查询其各个输出的达标情况以及比较指标。详细介绍请参见“**预检结果**”。 - -2. （可选）如果需要保存比对不达标的输入和输出数据，可以在run_ut执行命令结尾添加-save_error_data，例如： - - ```bash - msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data - ``` - - 数据默认会存盘到'./ut_error_data{timestamp}'路径下（相对于启动run_ut的路径），有需要的话，用户可以通过error_data_path参数来配置保存路径，error_data_path参数在[config.json](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe/config)文件或config.yaml文件配置，config.json文件需要在run_ut操作时通过-config参数指定，config.yaml文件详见“**config.yaml文件说明**”。 - -#### 使用multi_run_ut.py执行多线程预检 - -multi_run_ut.py脚本，可以并行执行多个run_ut操作，从而降低预检耗时。 - -命令示例如下： - -```bash -msprobe -f pytorch multi_run_ut -api_info ./dump.json -n 32 -d 0 1 2 3 -``` - -| 参数名称 | 说明 | 是否必选 | -| ---------------------------- | ------------------------------------------------------------ | ---------------------------------- | -| -api_info或--api_info_file | 指定API信息文件dump.json。 | 是 | -| -save_error_data | 保存精度未达标的API输入输出数据。 | 否 | -| -o或--out_path | 指定run_ut执行结果存盘路径，默认“./”（相对于run_ut的路径）。 | 否 | -| -j或--jit_compile | 开启jit编译。 | 否 | -| -n | 同时执行run_ut线程的数量，默认为8，最大支持64，但每个Device最大支持8个线程，当指定多个线程和多个Device时，则线程数在每张卡上均分。 | 否 | -| -d或--device | 指定Device ID，选择UT代码运行所在的卡，默认值为0，支持同时指定0~7，共8个Device。 | 否 | -| -csv_path或--result_csv_path | 指定本次运行中断时生成的`accuracy_checking_result_{timestamp}.csv`文件路径，执行run_ut中断时，若想从中断处继续执行，配置此参数即可。需要指定为上次中断的`accuracy_checking_result_{timestamp}.csv`文件。详见“**断点续检**”。 | run_ut操作中断后继续执行场景下必选 | -| -f或--filter_api | 过滤模型中除最大值和最小值以外其他参数和结构相同的API。适用于模型较大且重复API较多的场景。 | 否 | - -#### 断点续检 - -精度预检run_ut过程中，若因环境、数据量过大等原因导致预检进程中断，那么当用户解决这些问题后，重新执行run_ut操作，可以通过断点续检操作继续前面未完成的预检，会在-csv_path指定的`accuracy_checking_result_{timestamp}.csv`文件以及对应的`accuracy_checking_details_{timestamp}.csv`文件中继续写入后续的结果，不会重新创建结果文件。 - -须指定为上次预检中断的`accuracy_checking_result_{timestamp}.csv`文件。请勿修改`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`文件名，包括时间戳，否则断点续检会因无法识别到文件名而失败。 - -断点续检操作通过如下命令执行： - -```bash -msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_checking_result_{timestamp}.csv -``` - -#### API预检黑名单和白名单 - -run_ut过程支持API预检黑名单和白名单，通过如下文件配置black_list（黑名单）或white_list（白名单）参数来指定不需要或需要预检的API名称： - -- 配置[config.json](https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe/config)文件，config.json文件需要在run_ut操作时通过-config参数指定。 -- 配置config.yaml文件，详见“**config.yaml文件说明**”。 - -config.json文件的优先级高于config.yaml文件，即执行config.json文件时，config.yaml文件的配置不生效。 - -### config.yaml文件说明 - -config.yaml文件可以通过配置参数来控制dump和run_ut操作的白名单、黑名单等功能。操作步骤如下： - -1. 查找msprobe工具安装路径。 - - ```bash - pip show mindstudio-probe - ``` - - 输出结果如下示例： - - ```bash - Name: mindstudio-probe - Version: 1.0 - Summary: This is a pytorch precision comparison tools - Home-page: - Author: - Author-email: - License: - Location: /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages - Requires: numpy, openpyxl, pandas, pyyaml, rich, tqdm, wheel - Required-by: - ``` - - Location字段为msprobe工具的安装路径，那么config.yaml文件位置为/home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml - -2. 进入config.yaml文件 - - ```bash - vi /home/xx/anaconda3/envs/pt21py38/lib/python3.8/site-packages/msprobe/pytorch/api_accuracy_checker/config.yaml - ``` - -3. 修改config.yaml文件参数。 - - ```yaml - white_list: [] - black_list: [] - error_data_path: './' - precision: 14 - ``` - - | 参数名称 | 说明 | 是否必选 | - | --------------- | ------------------------------------------------------------ | -------- | - | white_list | API dump白名单，仅对指定的API进行dump。参数示例：white_list=["conv1d", "conv2d"]。默认未配置白名单，即dump全量API数据。 | 否 | - | black_list | API dump黑名单，被指定的API不进行dump。参数示例：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即dump全量API数据。 | 否 | - | error_data_path | 配置保存精度未达标的API输入输出数据路径。参数示例"error_data_path": "./"。默认为当前路径。 | 否 | - | precision | 浮点数表示位数，默认取小数点后14位。 | 否 | - - 说明：white_list和black_list同时配置时，二者配置的API名单若无交集，则白名单生效，若API名单存在交集，则白名单排除的部分以及交集的API不进行dump。 - -## 预检结果 - -精度预检生成的`accuracy_checking_result_{timestamp}.csv`和`accuracy_checking_details_{timestamp}.csv`文件示例如下： - -可以通过先查看`accuracy_checking_result_{timestamp}.csv`文件的Forward Test Success和Backward Test Success，判断是否存在未通过测试的API，再查看`accuracy_checking_details_{timestamp}.csv`文件的API详细达标情况，API达标情况介绍请参见“**API预检指标**”。 - -`accuracy_checking_result_{timestamp}.csv` - -![accuracy_checking_result](img/accuracy_checking_result.png) - -| 字段 | 含义 | -| --------------------- | ------------------------------------------------------------ | -| API name | API名称。 | -| Forward Test Success | 前向API是否通过测试，pass为通过，warning为待观察，error为错误。 | -| Backward Test Success | 反向API是否通过测试，pass为通过，warning为待观察，error为错误，如果是空白的话代表该API没有反向输出。 | -| Message | 提示信息。 | - -该结果为中间结果，仅作为参考，建议完成“**预检结果比对**”后查看比对结果。该结果后续将会删除。 - -Forward Test Success和Backward Test Success是否通过测试是由`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差、双百双千双万指标判定结果决定的。 - -需要注意的是`accuracy_checking_details_{timestamp}.csv`中可能存在一个API的前向（反向）有多个输出，那么每个输出记录一行，而在`accuracy_checking_result_{timestamp}.csv`中的结果需要该API的所有结果均为pass才能标记为pass，只要存在一个error则标记error，仅存在waring和pass且不存在error标记waring。 - -`accuracy_checking_details_{timestamp}.csv` - -![accuracy_checking_details](img/accuracy_checking_details.png) - -| 字段 | 含义 | -| ------------------- | ------------------------------------------------------------ | -| API name | NPU或GPU下的API名称。 | -| Bench Dtype | 标杆数据的API数据类型。 | -| DEVICE Dtype | NPU或GPU数据的API数据类型。 | -| Shape | API的Shape信息。 | -| 余弦相似度 | NPU或GPU数据与标杆数据的余弦相似度。 | -| 最大绝对误差 | NPU或GPU数据与标杆数据的最大绝对误差。 | -| 双百指标 | 双百精度指标。是指NPU或GPU的Tensor中的元素逐个与对应的标杆数据对比，相对误差小于百分之一的个数占总元素个数的比例。测试通过标准为相对误差大于百分之一的个数占总元素个数的比例小于百分之一。 | -| 双千指标 | 双千精度指标。是指NPU或GPU的Tensor中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。 | -| 双万指标 | 双万精度指标。是指NPU或GPU的Tensor中的元素逐个与对应的标杆数据对比，相对误差小于万分之一的个数占总元素个数的比例。测试通过标准为相对误差大于万分之一的个数占总元素个数的比例小于万分之一。 | -| 二进制一致错误率 | NPU或GPU数据中每个Tensor精度不一致的数值的数量与Tensor中数值数量的比值。只有数据是builtin类型（bool、int、float、str）、torch.bool和torch的int类型或者在新精度标准中使用二进制一致算法进行比对的API才会展示。 | -| 误差均衡性 | NPU或GPU数据与标杆数据精度差的上下浮动情况。 | -| 均方根误差 | NPU或GPU数据与标杆数据的均方根误差。 | -| 小值域错误占比 | NPU或GPU Tensor中与标杆的绝对误差大于错误阈值的小值在小值域（小值的总数量）中的占比。判断为小值以及绝对误差的错误阈值见“**小值域阈值**”。 | -| 相对误差最大值 | NPU或GPU数据与标杆数据相对误差的最大值。 | -| 相对误差平均值 | NPU或GPU数据与标杆数据相对误差的平均值。 | -| inf/nan错误率 | NPU与标杆inf/nan计算不一致的元素个数占总元素的个数比例。 | -| 相对误差错误率 | NPU与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。 | -| 绝对误差错误率 | NPU与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。 | -| ULP误差最大值 | NPU或GPU数据与标杆数据ULP误差的最大值（取绝对值后）。 | -| ULP误差平均值 | NPU或GPU数据与标杆数据ULP误差的平均值（取绝对值后）。 | -| ULP误差大于阈值占比 | NPU或GPU数据与标杆数据的ULP误差（取绝对值后）大于阈值（当NPU或GPU数据类型为float16或bfloat16时，阈值为1；当NPU或GPU数据类型为float32时，阈值为32）的元素个数占总元素的个数比例。 | -| Status | API预检通过状态，pass表示通过测试，error表示未通过，warning表示测试未通过双千或双万精度指标，SKIP表示该API的某个参数的反向不要计算梯度，所以没有任何计算过程，其他信息均为空。 | -| message | 提示信息。 | - -### 小值域阈值 - -判定为小值的阈值为： - -- torch.float32：e-6 -- torch.float16：e-3 -- torch.bfloat16：e-3 - -小值域的绝对误差阈值为： - -- torch.float32：e-9 -- torch.float16：e-5 -- torch.bfloat16：e-5 - -### API预检指标 - -API预检指标是通过对`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差双百、双千、双万精度指标的数值进行判断，得出该API是否符合精度标准的参考指标。 - -API预检通过测试，则在`accuracy_checking_details_{timestamp}.csv`文件中的“Status”列标记“pass”，否则标记“error”或“warning”，详细规则如下： - -1. 余弦相似度 > 0.99：≤ 0.99为不达标，标记“error”，> 0.99达标，进行下一步； -2. 最大绝对误差 ＜ 0.001：＜ 0.001达标，标记“pass”，≥ 0.001为不达标，进行下一步； -3. 双百、双千、双万精度指标： - - 对于float16和bfloat16数据：双百指标不通过，标记“error”；双百指标通过，双千指标不通过，标记“warning”；双百、双千指标均通过，标记“pass”。 - - 对于float32和float64数据：双千指标不通过，标记“error”；双千指标通过，双万指标不通过，标记“warning”；双千、双万指标均通过，标记“pass”。 - -4. 在`accuracy_checking_result_{timestamp}.csv`中以“Forward Test Success”和“Backward Test Success”字段统计该算子前向反向输出的测试结果，对于标记“pass”的算子，则在`accuracy_checking_result_{timestamp}.csv`中标记“TRUE”表示测试通过，对于标记“error”或“warning”的算子，则在`accuracy_checking_result_{timestamp}.csv`中标记“FALSE”表示测试不通过。由于一个算子可能有多个前向或反向的输入或输出，那么该类算子的输入或输出中必须全为“pass”，才能在`accuracy_checking_result_{timestamp}.csv`中标记“TRUE”，只要有一个输入或输出标记“error”或“warning”，那么在`accuracy_checking_result_{timestamp}.csv`中标记“FALSE”。 - -## 预检结果比对 - -需要同时获取NPU和GPU环境下run_ut操作的预检结果`accuracy_checking_details_{timestamp}.csv`文件。执行如下命令进行NPU和GPU预检结果的比对： - -```bash -msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_details_{timestamp}.csv -gpu /home/xxx/gpu/accuracy_checking_details_{timestamp}.csv -o /home/xxx/ -``` - -| 参数名称 | 说明 | 是否必选 | -| -------------------- | ------------------------------------------------------------ | -------- | -| -npu或--npu_csv_path | NPU预检结果`accuracy_checking_details_{timestamp}.csv`文件路径。默认从当前目录下识别该文件。 | 否 | -| -gpu或--gpu_csv_path | GPU预检结果`accuracy_checking_details_{timestamp}.csv`文件路径。默认从当前目录下识别该文件。 | 否 | -| -o或--out_path | 指定api_precision_compare.py执行结果存盘路径，默认为当前目录。 | 否 | - -执行完成后输出`api_precision_compare_result_{timestamp}.csv`和`api_precision_compare_details_{timestamp}.csv`文件。文件示例如下： - -可以通过先查看`api_precision_compare_result_{timestamp}.csv`文件的Forward Test Success和Backward Test Success，判断是否存在未通过测试的API，再查看`api_precision_compare_details_{timestamp}.csv`文件的API详细达标情况。 - -`api_precision_compare_result_{timestamp}.csv` - -![api_precision_compare_result](img/api_precision_compare_result.png) - -| 字段 | 含义 | -| --------------------- | ------------------------------------------------------------ | -| API name | API名称。 | -| Forward Test Success | 前向API是否通过测试，pass为通过，warning为待观察，error为错误，skip表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | -| Backward Test Success | 反向API是否通过测试，pass为通过，warning为待观察，error为错误，如果是空白的话代表该API没有反向输出，skip表示该API的数据类型不支持使用新精度标准进行比对，如float64。 | -| Message | 提示信息。 | - -Forward Test Success和Backward Test Success是否通过测试是由`api_precision_compare_details_{timestamp}.csv`中的各个指标判定结果决定的。需要注意的是`api_precision_compare_details_{timestamp}.csv`中可能存在一个API的前向（反向）有多个输出，那么每个输出记录一行，而在`api_precision_compare_result_{timestamp}.csv`中的结果需要该API的所有结果均为pass才能标记为pass，只要存在一个error则标记error，仅存在warning和pass且不存在error标记warning。 - -`api_precision_compare_details_{timestamp}.csv` - -![api_precision_compare_details](img/api_precision_compare_details.png) - -| 字段 | 含义 | -| ------------------------ | ------------------------------------------------------------ | -| API name | NPU或GPU下的API名称。 | -| 小值域错误比值 | NPU与CPU的小值域的错误比率/GPU与CPU的小值域的错误比率。标杆比对法指标。 | -| 小值域错误判定结果 | 小值域错误比值小于等于1标记为pass，1~2之间标记为waring，大于2标记为error。 | -| 均方根误差比值 | NPU与CPU的均方根误差/GPU与CPU的均方根误差。标杆比对法指标。 | -| 均方根误差判定结果 | 均方根误差比值小于等于1标记为pass，1~2之间标记为waring，大于2标记为error。 | -| 相对误差最大值比值 | NPU与CPU的相对误差最大值/GPU与CPU的相对误差最大值。标杆比对法指标。 | -| 相对误差最大值判定结果 | 相对误差最大值比值小于等于1标记为pass，1~10之间标记为waring，大于10标记为error。 | -| 相对误差平均值比值 | NPU与CPU的相对误差的平均值/GPU与CPU的相对误差的平均值。标杆比对法指标。 | -| 相对误差平均值判定结果 | 相对误差平均值比值小于等于1标记为pass，1~2之间标记为waring，大于2标记为error。 | -| 误差均衡性比值 | NPU与CPU的误差均衡性/GPU与CPU的误差均衡性。标杆比对法指标。 | -| 误差均衡性判定结果 | 误差均衡性比值小于等于1标记为pass，1~2之间标记为waring，大于2标记为error。该字段暂不参与api_precision_compare_result的结果判定。 | -| inf/nan错误率 | NPU与标杆inf/nan计算不一致的元素个数占总元素的个数比例。绝对阈值法指标。 | -| inf/nan判定结果 | inf/nan错误率判定结果，等于0标记为pass，其余情况标记为error。 | -| 相对误差错误率 | NPU与标杆的正常值计算相对误差，其大于错误阈值的元素个数占正常值元素个数的比例。绝对阈值法指标。 | -| 相对误差判定结果 | 相对误差错误率判定结果，等于0标记为pass，其余情况标记为error。 | -| 绝对误差错误率 | NPU与标杆的小值计算绝对误差，其大于错误阈值的元素个数占小值元素个数的比例。绝对阈值法指标。 | -| 绝对误差判定结果 | 绝对误差错误率判定结果，等于0标记为pass，其余情况标记为error。 | -| 二进制一致错误率 | NPU或GPU数据中每个Tensor精度不一致的数值的数量与Tensor中数值数量的比值。只有数据是builtin类型（bool、int、float、str）、torch.bool和torch的int类型或者在新精度标准中使用二进制一致算法进行比对的API才会展示。二进制一致法指标。 | -| 二进制一致错误率判定结果 | 二进制一致错误率判定结果，等于0标记为pass，其余情况标记为error。 | -| ULP误差平均值 | NPU数据与标杆数据ULP误差的平均值（取绝对值后）。ULP误差比对法指标。 | -| ULP误差大于阈值占比 | NPU数据与标杆数据的ULP误差（取绝对值后）大于阈值（当NPU数据类型为float16或bfloat16时，阈值为1；当NPU数据类型为float32时，阈值为32）的元素个数占总元素的个数比例。ULP误差比对法指标。 | -| ULP误差大于阈值占比比值 | NPU与CPU的ULP误差大于阈值占比/GPU与CPU的ULP误差大于阈值占比。ULP误差比对法指标。 | -| ULP误差判定结果 | ULP误差判定结果。
当NPU或GPU数据类型是float16或bfloat16时，以下两条标准满足其一标记为pass，否则标记为error：
NPU ULP误差大于阈值占比小于0.001；
NPU ULP误差大于阈值占比小于GPU ULP误差大于阈值占比。
当NPU或GPU数据类型是float32时，以下三条标准满足其一标记为pass，否则标记为error：
NPU ULP误差平均值小于64；
NPU ULP误差大于阈值占比小于0.05；
NPU ULP误差大于阈值占比小于GPU ULP误差大于阈值占比。 | -| 双千指标 | 双千精度指标。是指NPU的Tensor中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅conv1d和conv2d使用该指标。双千指标法指标。 | -| 双千指标判定结果 | 双千指标判定结果。双千指标大于0.999标记为pass，否则标记为error。 | -| 比对结果 | 综合所有指标的最终结果。如果比对指标中有error，则标记为error；有warning，则标记为warning；否则标记为pass。 | -| 比对算法 | API使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和ULP误差比对法中的一种。 | -| Message | 提示信息。当前提示该API比对结果为error或warning时对应不符合标准的指标。 | - -# FAQ - -[FAQ](./FAQ.md) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md b/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md deleted file mode 100644 index 5050689a3..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/doc/api_accuracy_checker_online.md +++ /dev/null @@ -1,187 +0,0 @@ -# 在线精度预检 - -为了应对大模型场景下，通过离线预检方式dump API输入输出数据导致的存储资源紧张问题，提供在线精度预检功能。本功能实现在执行NPU训练操作的过程中，通过TCP/IP协议在NPU -Host与GPU Host设备间建立连接，将NPU上对应API的输入数据在GPU设备上运行，将两份输出数据进行比对，得到预检比对结果，从而减少数据dump的步骤，降低存储资源的占用。 - -## 在线精度预检流程 - -在线精度预检当前支持**局域网场景**和**共享存储场景**，请根据不同的场景选择对应的配置。 - -在线精度预检操作流程如下： - -1. 准备GPU和NPU可正常运行的训练环境，PyTorch版本大于等于2.0，并保证两台Host在同一局域网内可正常通信或能通过共享存储进行通信。 -2. GPU和NPU Host设备上同时安装msprobe工具，详见《[MindStudio精度调试工具](./../../README.md) - 》，其中在线预检要多安装twisted、pyOpenSSL依赖，该依赖为Python模块。 -3. 分别配置GPU侧、NPU侧的config.json文件。 -4. 在GPU侧运行msprobe -f pytorch run_ut -config ./config.json。 -5. 在NPU侧配置训练脚本。 -6. 在NPU侧执行训练。 - -## 在线精度预检操作指导 - -### 配置config.json文件 - -安装完成预检工具后，需要分别在GPU和NPU环境下分别配置config.json。其中需要重点关注文件中的is_online、is_benchmark_device、host和port参数的配置，保障在线预检时GPU和NPU两台设备间的通信正常。 - -#### GPU侧在线预检配置说明 - -| 参数名称 | 说明 | 是否必选 | -|-----------------|-----------------------------------------------------------------------------------------------------------------------|------| -| task | 任务名称，str类型，配置为run_ut表示预检任务。通过其他字段is_online判断离线预检、在线预检任务。 | 是 | -| white_list | 预检的API白名单，list[str]类型。参数示例：white_list=["conv1d", "conv2d"]。默认未配置白名单，即预检全量API数据。 | 否 | -| black_list | 预检的API黑名单，list[str]类型。参数示例：white_list=["conv1d", "conv2d"]。默认未配置黑名单，即预检全量API数据。 | 否 | -| error_data_path | 配置保存精度未达标的API输入输出数据路径，str类型。在线预检模式下该参数不生效。 | 否 | -| is_online | 在线预检模式开关，bool类型，可取值True（开启）、False（关闭），默认关闭。 | 是 | -| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效。 | 否 | -| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机地址127.0.0.1或本机局域网IP。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | -| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，GPU侧配置为本机可用端口。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | -| rank_list | 指定在线预检的Rank ID，默认值为[0]，list[int]类型，应配置为大于等于0的整数，且须根据实际卡的Rank ID配置，若所配置的值大于实际训练所运行的卡的Rank ID，则在线预检输出数据为空。GPU和NPU须配置一致。 | 是 | - -#### NPU侧在线预检配置说明 - -| 参数名称 | 说明 | 是否必选 | -|------------------|--------------------------------------------------------------------------------------------------------|------| -| task | 任务名称，str类型，配置为tensor表示dump API统计信息和完全复刻整网的API运行情况的真实数据。通过字段onlin_run_ut判断是否使用在线预检功能。 | 是 | -| dump_path | dump路径，str类型，配置为合法路径即可，兼容tensor任务静态检查 | 是 | -| level | dump级别，str类型，在线预检时配置为L1，表示dump API级精度数据。在线预检可不配置，默认取值L1。 | 是 | -| rank | 指定对某张卡上的数据进行dump，list[int]类型，默认未配置(表示dump所有卡的数据)，需要与GPU侧配置项rank_list保持一致。 | 否 | -| step | 指定dump某个step的数据，list[int]类型，默认未配置，表示dump所有step的数据。dump特定step时，须指定为训练脚本中存在的step。 | 否 | -| seed | 随机种子数，int类型，默认值为1234。通过固定随机数保证模型的输入或输出一致。 | 否 | -| is_deterministic | 确定性计算模式，bool类型，可取值true（开启）或false（关闭），默认关闭。 | 否 | -| scope | dump范围，list[str]类型，默认未配置（list也未配置时师表dump所有api的额数据），配置方式参考[《config配置文件说明》](./../../config/README.md) | 否 | -| list | 自定义dump范围，list[str]类型，默认未配置（scope也未配置时表示dump所有api的数据），配置方式参考[《config配置文件说明》](./../../config/README.md) | 否 | -| online_run_ut | 在线预检模式开关，bool类型，可取值True（开启）、False（关闭），默认关闭。 | 是 | -| nfs_path | 在线预检模式共享存储目录路径，str类型，用于GPU设备和NPU设备间进行通信。配置该参数后host和port不生效。 | 否 | -| host | 在线预检模式局域网场景信息接收端IP，str类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的局域网IP地址。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | -| port | 在线预检模式局域网场景信息接收端端口号，int类型，用于GPU设备和NPU设备间进行通信，NPU侧须配置为GPU侧的端口号。局域网场景时，不能配置nfs_path参数，否则局域网场景不生效。 | 否 | - -#### 局域网场景配置示例 - -若复制下列示例，请删除注释后使用。 - -GPU侧： - -```json -{ - "task": "run_ut", - "run_ut": { - "white_list": [], - "black_list": [], - "error_data_path": "./", - "is_online": true, - "nfs_path": "", - "host": "127.0.0.1", - "port": 59208, - "rank_list": [0] - } -} -``` - -NPU侧： - -```json -{ - "task": "tensor", - "dump_path": "./dump_path", - "rank": [0], - "step": [0], - "level": "L1", - "seed": 1234, - "is_deterministic": true, - "tensor": { - "scope": [], - "list": [], - "online_run_ut": true, - "nfs_path": "", - "host": "xx.xx.xx.x", - "port": 59208 - } -} -``` - -#### 共享存储场景配置示例 - -若复制下列示例，请删除注释后使用。 - -GPU侧： - -```json -{ - "task": "run_ut", - "run_ut": { - "white_list": [], - "black_list": [], - "error_data_path": "./", - "is_online": true, - "nfs_path": "/nfs/xxx/data", - "host": "", - "port": -1, - "rank_list": [0] - } -} -``` - -NPU侧： - -```json -{ - "task": "tensor", - "dump_path": "./dump_path", - "rank": [0], - "step": [0], - "level": "L1", - "seed": 1234, - "is_deterministic": true, - "tensor": { - "scope": [], - "list": [], - "online_run_ut": true, - "nfs_path": "/nfs/xxx/data", - "host": "", - "port": -1 - } -} -``` - -### 在GPU侧运行run_ut - -由于GPU侧为通信接收端，需先于NPU侧执行run_ut操作，命令如下： - -```bash -msprobe -f pytorch run_ut -config ./config.json -``` - -GPU侧配置好config.json文件后执行run_ut命令，此时GPU处于预检等待状态： - -- 局域网场景：当NPU侧启动训练后将预检的API输入和输出数据发送到GPU侧时，GPU启动预检操作。 -- 共享存储场景：当NPU侧启动训练后将预检的API输入和输出数据发送到共享存储时，GPU启动预检操作。 - -### 在NPU侧配置训练脚本 - -在NPU训练脚本中添加如下代码以获取run_ut操作的预检API输入和输出数据： - -```python -from msprobe.pytorch import PrecisionDebugger - -debugger = PrecisionDebugger("config.json") -... - -debugger.start() - -... - -debugger.stop() -debugger.step() -``` - -### 在NPU侧执行训练脚本 - -配置完NPU侧训练脚本后即可执行训练脚本，命令示例如下： - -```bash -bash train.sh -``` - -训练脚本执行完毕后，在GPU侧dump_path目录下生成比对结果文件，详细介绍请参见《[精度预检工具](./api_accuracy_checker.md)》中的” -**预检结果**“。 - diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md b/debug/accuracy_tools/msprobe/pytorch/doc/dump.md deleted file mode 100644 index 31d6303ba..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/doc/dump.md +++ /dev/null @@ -1,260 +0,0 @@ -# **精度数据采集** - -msprobe工具主要通过在训练脚本内添加dump接口并启动训练的方式来采集精度数据。 - -执行dump操作需要安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 - -## dump接口介绍 - -### PrecisionDebugger - -**功能说明** - -通过加载dump配置文件的方式来确定dump操作的详细配置。 - -PrecisionDebugger接口可以在from msprobe.pytorch import PrecisionDebugger之后的位置添加。详细使用可参考“**示例代码**”或“**model配置代码示例**”。 - -**原型** - -```Python -PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, model=None, step=None) -``` - -说明：上述参数除config_path和model外，其他参数均在[config.json](../../config)文件中可配，此处的参数优先级高于[config.json](../../config)文件中的配置，而config.json文件可以配置更多参数，若需要进行更多场景的精度数据dump，建议配置[config.json](../../config)文件。config.json文件的配置可参考《[配置文件说明](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/config/README.md)》。 - -**参数说明** - -| 参数名 | 说明 | 是否必选 | -| ----------- | ------------------------------------------------------------ | -------- | -| config_path | 指定dump配置文件路径，String类型。参数示例："./config.json"。未配置该路径时，默认使用[config.json](../../config)文件的默认配置。 | 否 | -| task | dump的任务类型，String类型。可取值"statistics"（仅dump API统计信息）、"tensor"（dump API统计信息和完全复刻整网的API运行情况的真实数据）、"overflow_check"（溢出检测），默认未配置，取"statistics"，参数示例：task="tensor"。 | 否 | -| dump_path | 设置dump数据目录路径，String类型。参数示例：dump_path="./dump_path"。 | 否 | -| level | dump级别，根据不同级别dump不同数据，String类型。可取值：
"L0"：dump module模块级精度数据，仅PyTorch场景支持”。
"L1"：dump API级精度数据，默认值。
"L2"：dump kernel级精度数据。
"mix"：dump module模块级和API级精度数据。
配置示例：level="L1"。 | 否 | -| model | 指定具体的torch.nn.Module，默认未配置，level配置为"L0"或"mix"时必须配置该参数。配置示例参见“**model配置代码示例**”。 | 否 | -| step | 指定dump某个step的数据，list[int]类型。默认未配置，表示dump所有step数据。dump特定step时，须指定为训练脚本中存在的step。step为list格式，可配置逐个step，例如：step=[0,1,2]。 | 否 | - -#### model配置代码示例 - -示例中定义了一个nn.Module类型的简单网络，在进行数据dump时使用原型函数PrecisionDebugger并传入config_path参数和model参数，其中model参数传入数据的类型为torch.nn.Module类型或torch.nn.Module子类型。 - -```python -#根据需要import包 -import os -import torch -import torch.nn as nn -import torch_npu -import torch.nn.functional as F -from msprobe.pytorch import PrecisionDebugger - -torch.npu.set_device("npu:0") -#定义一个简单的网络 -class ModuleOP(nn.Module): - def __init__(self) -> None: - super().__init__() - self.linear_1 = nn.Linear(in_features=8,out_features=4) - self.linear_2 = nn.Linear(in_features=4,out_features=2) - - def forward(self,x): - x1 = self.linear_1(x) - x2 = self.linear_2(x1) - r1 = F.relu(x2) - return r1 - -if __name__ == "__main__" - module = ModuleOP() - - #注册工具 - debugger = PrecisionDebugger('./config.json',model=module) - debugger.start() - x = torch.randn(10,8) - out = module(x) - loss = out.sum() - loss.backward() - debugger.stop() -``` - -### start函数 - -**功能说明** - -dump启动函数。 - -在模型初始化之后的位置添加。需要与stop函数一起添加在for循环内。 - -**原型** - -```Python -debugger.start() -``` - -该函数为类函数，可以使用debugger.start()也可以使用PrecisionDebugger.start()。 - -### stop函数 - -**功能说明** - -dump停止函数。 - -在**start**函数之后的任意位置添加。若需要dump反向数据，则需要添加在反向计算代码（如loss.backward）之后。 - -**原型** - -```Python -debugger.stop() -``` - -该函数为类函数，可以使用debugger.stop()也可以使用PrecisionDebugger.stop()。 - -### forward_backward_dump_end函数 - -**功能说明** - -dump停止函数。用于dump指定代码的前反向数据。 - -在**start**函数之后，反向计算代码（如loss.backward）之前的任意位置添加，可以dump **start**函数和该函数之间的前反向数据，可以通过调整**start**函数与该函数的位置，来指定需要dump的代码块。 - -要求**stop**函数添加在反向计算代码（如loss.backward）之后，此时该函数与**stop**函数之间的代码不会被dump。 - -使用示例参见“**示例代码 > 扩展示例**”。 - -**原型** - -```Python -forward_backward_dump_end() -``` - -该函数为类函数，可以使用debugger.forward_backward_dump_end()也可以使用PrecisionDebugger.forward_backward_dump_end()。 - -### step函数 - -**功能说明** - -结束标识。 - -在最后一个**stop**函数后或一个step结束的位置添加。需要与start函数一起添加在for循环内。 - -**原型** - -```Python -debugger.step() -``` - -该函数为类函数，可以使用debugger.step()也可以使用PrecisionDebugger.step()。 - -## 示例代码 - -### 基础操作 - -如下示例可dump完整代码的前反向数据。 - -```Python -from msprobe.pytorch import PrecisionDebugger - -# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 -debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") - -# 模型、损失函数的定义及初始化等操作 -# ... - -# 数据集迭代的位置一般为模型训练开始的位置 -for data, label in data_loader: - debugger.start() # 开启数据dump - - # 如下是模型每个step执行的逻辑 - output = model(data) - #... - loss.backward() - - debugger.stop() # 关闭数据dump - debugger.step() # 结束一个step的dump -``` - -### 扩展示例 - -如下示例dump指定代码块前反向数据。 - -```Python -from msprobe.pytorch import PrecisionDebugger - -# 请勿将PrecisionDebugger的初始化流程插入到循环代码中 -debugger = PrecisionDebugger(config_path="./config.json", dump_path="./dump_path") - -# 模型、损失函数的定义及初始化等操作 -# ... - -# 数据集迭代的位置一般为模型训练开始的位置 -for data, label in data_loader: - debugger.start() # 开启数据dump - - # 如下是模型每个step执行的逻辑 - output = model(data) - debugger.forward_backward_dump_end() # 插入该函数到start函数之后，只dump start函数到该函数之间代码的前反向数据，本函数到stop函数之间的数据则不dump - #... - loss.backward() - - debugger.stop() # 关闭数据dump - debugger.step() # 结束一个step的dump -``` - -## dump结果文件介绍 - -训练结束后，工具将dump的数据保存在dump_path参数指定的目录下。 - -dump结果目录结构示例如下： - -```Python -├── dump_path -│ ├── step0 -│ | ├── rank0 -│ | │ ├── dump_tensor_data -| | | | ├── Tensor.permute.1.forward.pt -| | | | ├── MyModule.0.forward.input.pt # 开启模块级精度数据dump时存在模块级的dump数据文件 -| | | | ... -| | | | └── Fcuntion.linear.5.backward.output.pt -│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） -│ | | ├── stack.json # 算子调用栈信息 -│ | | └── construct.json # 分层分级结构 -│ | ├── rank1 -| | | ├── dump_tensor_data -| | | | └── ... -│ | | ├── dump.json -│ | | ├── stack.json -| | | └── construct.json -│ | ├── ... -│ | | -| | └── rank7 -│ ├── step1 -│ | ├── ... -│ ├── step2 -``` - -dump过程中，pt文件在对应算子或者模块被执行后就会落盘，而json文件则需要在正常执行PrecisionDebugger.stop()后才会写入完整数据，异常的程序终止会保存终止前被执行算子的相关npy文件，可能会导致json文件中数据丢失。 - -其中rank为设备上各卡的ID，每张卡上dump的数据会生成对应dump目录。非分布式场景下没有rank ID，目录名称为rank。 - -pt文件保存的前缀和PyTorch对应关系如下： - -| 前缀 | Torch模块 | -| ----------- | ------------------- | -| Tensor | torch.Tensor | -| Torch | torch | -| Functional | torch.nn.functional | -| NPU | NPU亲和算子 | -| VF | torch._VF | -| Aten | torch.ops.aten | -| Distributed | torch.distributed | - -## 工具支持的API列表 - -msprobe工具维护固定的API支持列表，若需要删除或增加dump的API，可以在msprobe/pytorch/hook_module/support_wrap_ops.yaml文件内手动修改，如下示例： - -```Python -functional: # functional为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API - - conv1d - - conv2d - - conv3d -``` - -# FAQ - -[FAQ](./FAQ.md) diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/BLOOM-7B_1.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/BLOOM-7B_1.png deleted file mode 100644 index 3853626d6fab127915425238c76f16836a4cef0b..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 3534 zcmV;<4KebGP)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D4P!|}K~#8N?VUYu z+{_WjW`p&**ujn=?saK~~lp=Nhcw!827RfOK4C-O2mnCiY9uQo1Ejo{+b(=cqGxIj3e zGU%2Ha%dvkr$I~1X1mqWWG)b+(sn;kWho0=-KL-!xjaz5?8$k5C83HxB!~jbn4Ub= zL(J{tvlLVaKa4?eu^ooKqw52&r^%~|#l}9+IQqOUTfE}ZMS(({XO3E&dby8&hN_6f z$xZz81p|CrbcL`7QYI;vee&>fT&Q;b@~?5)(6%6#UL>=Yt?PMzDh0Gjg}?XvP55f_GLPnbjwp7NRIk4)HrCM4*TT+W&iB7x%zC~Y4xaK>Ro5ydL zcRG|-Q#`bu6wz^ng6%m@zMFiz&1UE~6l?OzIyg8$lY)f957Wdipbripnk$Dt;OED)=?d0>Hij9#avU>rL>zgxr`q5yXhlmbXrC%!H`{tn~}p?JiYxog3_Mma4a(Uh34jqLfb z!gbB#j*&dyU<4D!u@q}EcFP#AYmbg;I_D>Dz&_ZN=rNEp5US9C6>pRoLkVxqfi>-}jqglfKcL_R*Ecx#hg5+41hd zzEuTkHmNLyO;;O}i7cF~2fiqE=Tyl~?F{DD-KVX$gdK+kW8%WGJN)M;CS0A3GskDg zFe&idFOv~Rw@uEAz>E$HSWN~yYIBR(oMv81@~Xo%KxdX8jdLLlM-3gmjgYdxY2Y_y zXbC?T8;)Wvgl1TCsE$;DL<)_wV&U3xb@s`=jW@%LaI|+x^3*|h04F)*Hipz}*zWs7t7WC2Q8IZVsjdE!m$s}0lR)fw;EXziBkNYqlDmYjSo5Jbzrl$CrhGU0a zX))&%!m;LCj)o(~4>xOaM(GpKZX81yH;Dq7*1*KDJ4f%Rdhk;y)fMA(Z=3f; z^n2LDwM3sv!LA)oGgfh1m2f2M(l6@+hoe3QW-I(D3jSb+A++UeM#GXppP6|4MkRzJ zf1Kij`;w)nY}1ZmZA_-&!!e{Hi6VC0mdUPopwBV}8lO}LNBfQHg>v|O2DuySvk}6c zHsIyJhg$hkVN5tiXhTuE#%CT@6pDQ*)dfAyK5etUXn_W*gAN9&i)_})WJ<@(7aZCz z+ZLaGuU34@6WN~>j-PHmf70UB{YMVO+pqV$@2S|dV^|xLX)Fv!h-={pY=)CeCOSmy zF0O4o;<_hK-GZF)rzGga`oM-`sOx<8`f0!HP)45#s@0`kNqbcAc7WH92C6omaa*p5 zP;exDr@GuOvJ{DXnhJ*PHd|t*ebn+O@9!RHns$@*3gPI-WEy!mj%6T}IlQ8Ym>1%1 zvL+2NWy|o&*`aeu$(*Ea0U*{`MJFQ`JGt!eSqVBxnTkLnQoi0rgh$HpDGB|uL+Sh! zo3*-fv7trCLtm_Z6przDQZbsWeH3Ab`Wr2Z7o6wM&~TiMyd%P+a=N%vxP-ZN_i5`bVaH+qm@Ey)M&kg4a4bCETAT67 zqS4TB#G1h03SdD%63(+v16BsQ6jHhS$if00vm#d4R zwKkTU#*UO}8IEJT1Qs2?9>oC#!{{Tl4MW zrNVBXc58K)+Dyi(=ojXnw9r zF*$aGmKTAs$Rmb?Tm8AK%*|P;|x(WIkx7F z$eNt(_}trX^ldg#ulcEvl6Mz&Z1UQv??AmsRRtA^XdBW+s8X8cOSmEpCX~;_PW&BRKRB=wwmdCVA`tI?Ch|g*%>G6cCEiV*h)=r|uXmG>XICy=h>1zaiu7P3aB4MmL>g_s9ajAMPiQFFuaz0}FRcR&?WT1+1wl)slU1`JAz%EyC+g(ugLcfgFxWDs2X8bJ6 zL;3S-E+NiMj&uF=~&A+vXlp+^kb66fz#3$AtlpKQi~T!8uQ+foZl3p{Eaf; zJkI*4?y`RA5$OwdvC(K8i)7^paOs){+AvHuv!la-ic_h1sBW}p=t(zRN8x2mQ z(Rj1itS_W*HvC%j)T5@bOUJdpqwe3GW+*p+pOODAizWq7cim&}-*{pVt?B~R|THw{h zDx@~%K{xFu%aWYv|LNhzs!(!_`QhT?XIjko<$wS2TiX7f_lO_<_N(u|{>%UV_^001Hg0ssI3odsUI00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D6c|ZFK~#8N?VZ1i zCEIbwd$p^lMF=N)4@>rdgfW{)FpF%2G$6(>(8<`?pb6w;urqVnK%s&`5=iM~!!amP zp!hNitIWYureH2!KlBG!Lg6*zz?p=fzEyR8)cMu7`~KLSyHg)#`<||@Q>RXy)2C1W z-0sNO6pCF>|7Q4`9UU!}DJ zFK%x{4pvKe1-xTSOnG4N0Q%jEaVt!=`W7f-2RL@;BzgJI0csQ36&6ykY$|Uxm6)-1+El^X8~kRBtb5x zSUwld7hSra%dt^P7vy9t>HuYj({dijUg^?(+O-|)xUE44P`Y6ul#4$^LtNpll9m%o z5(@pC&TK=8pjx8hvlNt$5PI%WhoNF*av>JGTrpL;Re zbh6>Rdh38@^h)L&1z}25p5kb9?W#4c0?n##<1O5KhJxnNIwN|7zY;^;Zh@m}-(${* zDXlE!2n#cGPO+FFlyc;C%jFy)cbYs=U8=Mok|1ZITac4k-pZxnWFusWY~=-+BF4{1S6Nr}*;4ZcLw)R#!PL?5Gy?V!J&Xk;S8MB3T0Us# z>WRNRwhuy$_*XN=JBRCb$RU^L!gZ4gHVk7Te30ci1dWRRXe|=M*c16^hrv_kBG&DU z{qeBocCTN5{C?a@KXmMr!Z3^nK{nXwCxmLu_$m|Q9~g)a=JO?|US@`jPAhtKF{htX zk-c2KqN~eTuCPqxM|82yo$V-{B&~Hy*l%6#1P1d~SIkk$7Vehf0&lMf$ATDLPj>Hw8capQn(w{`ynhZ=wX^_h+LMmA2Orr zSW=X=F3;u778k+Bw)>{=D?6So%1Z1gTJnjU()H*-9PH#y1>rlQ?v$>>`3CCeP^N%V zBGX}H01{;IitB~GPxK4wVQ%mO+F%&$luoYS80>t*x9Y`dBUfce#+V=5!-zNjHd3ASmM7J-)-b;^0p1y=K{Z$^5TxVa~IXXIj@PK6!K>{9HSonaUcf;iaG38~D+{Yb2jv~nzp zI)-5w2ZbN(=z=<3gz%s2A!N2G4=@rn9m6n;gTPK>4a0a4+@rycZnAiE+As`bOFH0^ zhG7`*FFPIVvJb|#*<{w%4}QBuSf{hiY_XMhLz}f&HBHr?V91{P>FJ9KmD(rY3mAAx zqt$k!+jIhKb3)wq;luk*tLx)&(IOWO$N9FFSe z__+cB0#I^a~Ler zen^a}V?kL~)sBY$r;iO8+wPm92cv4o8%b+W+xK8M<5#aZ#SJ6z(rFMHl@jtl&G1*i zL`Rl@Uu;Oy(4n!Bzf=XWwh1Jil?7DVJ!ClGThIaMNmQ!CJxwmg?19he%{UL^||CjkcMAqFstn_FWz2bt4?{X{A<2q_uBF zSKnNM|UA>xdnCO#;g+D(HmM=eKWA92Ioc57^hB zn&tdV>Mmi_agY|lhT+;*2Rk%(NWgL!PX=fXcA+z*9V=<1(5Tpr#d{>mPUEV?NZRQz znSOY*MGV3&uSGfIc2upC=?9$*Q3I`dQCbqb^9b4{kok;CNw_k3(5FCgi(;^Am+O$c zZj(xQ!XTC8#UG3N1-*ugz0die3H~ruJ29%^>pm&8^FkKmru$)xC(Nkcb!#&_{+cG#NMEP*NDN74Uy!*H1}{jooN+s<*3oGu4tCfS z(r#Fya%0eSy+|&t;%9c@h*8Z;KmPNXx-PU@^!~LoyHJ)?2NVtInnwE|4Iw)-d$MMr zXDW8IKN>9;$0V&|)v9|g_kBj&Ec<}SH5>S#!RRj@>3&G7z96w5!XD%Rn?fQ8L8znH zRAXFH$5ua&P3-LqfJv51@%x_gIp1<-_w2Ir;%YS@8a>#_FbbPA#;L(>zmmsp(A3yG>b#i0jaduDg~` zfOm|kY*bdOoHgZ7ZQqt}8}t)*CL0E+`%5K;SscnKcFC@;*=e>I?2HFbYiFK50QETU z<4eZ$za==veH1{IH97CTnI4Bam|>mER{#!Ru&bcsfPp^T8yLU_JHs%H4M_((`V*NN7pS967 zJ9-<8PH7!1Cv0YC7{Z2v>EVE`NK8qk}MBc6RU48t%C z!?-ux*NGjyQ?k=Lz7K7p_O9wXwcNMt=emAH?l`7;)z4<@z6A-WfSva2e!dWwbF1V0 z$(Xv|?R-Do6T#0t^|wQbgK_9I_jls~@$Bl2e2(hkmOlGevge5%#=F`u+RuFN4Bz0% zhy$fM)Dpp3OWvS8p!YNH3%#GLoJlrBFPr)VRXBI1VW%l4YIa5tYq_TbScA>W9j70) zvcS*=gs<9%YQ%muNa@wodbJ=s10G;AZ-t+uO52rNo{{khOx2&r*y>TQ7}N`X?mA$O zJ;z}LR58VvI>tFdeGpbqfD2qihsHz4&f0Ml>+7{`FSxvE9Z>hop1SW_0}Whv9~FBS z^=kUlV5FoA(Hql?|;CC~+pKd3}FU`kQ}WhV83cdpj@qL)5fq#EUG}L z3b)#8xga@WhBSz=tD6aoE0;^qYP`bwYuZKIrvPy0Bmq{~S+}*-Kdb!;wCg3ca4SqH ziI7i>QV{*JfKYKsLPzA#lsA*08-9n5x13uO>ovt6q7|POHGQ!!+d*hDu5YDZvHpa1 zZ|RdfC3?~Hvr%rabHtZ$g8J4}TAv7#2Px5?+J4F&*Wxy! zUc+@5h7P6f+uiQZeR%D~+3nrgB`1mg-q4(tKMsl5Bzj)>pK>z&OU3W?4Rf+&@ zRt1>~TsnU)Ufn!Dbz6>3Uz|jLUOInB|IHc6-<(}Z-EU4@mP!5#2z}Zf)V4Ixp(SZA z;`C6xq(#dPG1MP)ky(!@{8__D{Yt-6BHf=i?T5njiCx)iQej+c$r-Gs6%5t$TTn?! z4BBD(88WBlm{eZN38>&T_Pr;8)? zY-{e<#EupGmI)I(pxP)QZM#;^rhS@0XPL5+($$K71g4uAm=!N#%WGQ3q?V-tze$n;D5g~S_3bH=p+dAnX&EkG&BNeyQDq1;ek zNTOqXKjWtpFuc((PUL3eo{$@GYn(qSMWkyr8u~-^Wx1EQUy|4C2w2UkHB~>;2y0C{ z4gDqq_IDSD2BI~fn|VsUMHWdR7S-RuvHtY*SZfQgae z=fW_SQ?YzS-{sqZ6Omm^?7${hp2Af}U$LwOG_;A)w?fqOc!~Lz98f6z2450)2z{wm z#Ia0I^et9Pl`x=BantxRs&+t2-m*iW_~N{VtCoriVbY|v_6qu5Q{_nFsr{c+F4Zb- z@oTJKX%|qnpIQN=<%($aLsAj3>Q^MM*Gur*f^DZ7C|pajWX^{;D0kBpq&W%A(sGlm zn3OU85VXxHj=BZe&{xsKzUkzlTwkL5Ww~zmSZL;|Frd$b!d2FqpM@P!WxPIm!+nUu z5Xeo%D&>Lk5S02L>@RVc^i=f;@4hF*z!SHIDyQhZv@D?7jw+&lKOKr&qh1YvHY41H z^tWOibgfPwYx&SS;#Ta-cuRDMt?SU7*d0CdPSX9^tF;q5uz~+mDf(d+fY2R>`UAm+ zwr_oy8Eb?*66Kf@YV(tg@psjIqd2M)yZu8|^L5+0Y7d&N_w)_fce(GXUdG#@)xJ~Z zI|MT$HEVcF2QHh9-9XY~+JJYuJ??!!QiPFbrdW*2J!-Cw}Jo`XycK^tUI}b%las)P29zrRV3Oj~j7=~dO#(pR!b~0lVWI`5hqUx88 zinqq-MGq|*uwxj8VHk#C?1Apl&)P3fq*vPL|3l!pS^5D?p0-u4-YHT$hG7_nVHn0f zsNRY-48t%C!!QnyhG7`S`v6_BdmtQtc+oxj+3|$(b6)vE*R=|E zJaWI@xLX-lR&Z?fmG8C^e}B8|y3cjw(4)_N)Zy>xe|3+4+CBcslgB?1m7%g41`${o zd@R$>lB-8XsZSRcc0|KYE=Y2@$E_)qD7>ECI2^b1;k z^@f&Q*728Vw>|!gf8ynT(DE;zV438t)Co9;*d z-+l0(-Q!<%AN*hU_>+%57Lyqx`UdO_GJ2;Q(+?w~`^H!DzVS00001b5ch_0Itp) z=>Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D6MRWTK~#8N?VU?* z+ej0~a}ckyp7m_92*6FYUi$z-j*!K70U^t|h2Q{lfH{B#3DZXm+tP^5BAC`+9^m9 zFx&r5FXM$>g_}a-=Pc%aRS@2q-auzaZ)l$+o}W{iplp}8F8ZlbzA|I}-|ddi2mHgy z^MT58!G!yxPGA(Q3`Bwdkx-Q7hz{>w!s+2ay)9Q!xV!s)_vPoeCz|{sv`cpM5`PNM z`KX$iX^#WiDm1Zf(J2X8qO9)!y0I5vwj|%290h}-vYhhSYcXN9IvTFqexy3_6EO1Tch)H_gJp#Cu1>AYT6@xq`AB; z;FY7j+*&l2G%=$?-7kF%^?_cg?qIc|J~&_Z%ZHx}*z%5L zT5uP7O6Mi-cA~l)ZT4~%nws|QR%jn*2ors%5=6haUp)249~GNyo`7wx$mb<|xGAbJ zumPL=JA31fy%lMdr$k={wnb85+VPwEo$1IC$oEZr89JDn(~kCQh`mi9<*F^<$~$px z#y8Ue2X~i=Ss{^0yQqb-uZtv3d|!xLj>cxEE}iZt`S7fXMfDN)@wi8FaEsz?T3M|5 zx;E~c37~O#AIxVkO%sxgTGfZPzAd#mJ7K;){l@XHe}3Zl?di{FD(l-2BeLmp80Dmr z+gK`XW*sm0Z1O=73Dl%)SVo#@W{x5BdqK-b=SJqU?QZ+EP0Z<@v}TlIyQQ^qX}~&gca2I=;*l zli$kr6yX!(<083?o;3NzPrB5rOFwpGf#ul_ohE(j^^=p|+LCa9ww2MY@jggL$iokN zl4Cb~TVhbjR^eI71_PJY_*uwLm?GbP<_hpC^dr;BYz%yv72Tc_ChF)DGyM522Vy0_Iy zh4HcMJ|aYJU5bfnWhv0DTGsJ}N^TRQw3*9Z?%Co)LZ*b>Rsp+RG+@W_R)Hz6B}Vd0 zq=sPGKe3;Tv2H6noNpJEPF?t}_G3GP_m56~-OO}AH#`5cJy&mrZ0`3wws`m3ak-p$ zc)XI^JeqAbg$g;gFE(?Gz%2wc=ZVP9;Qce6`QI)7{`;T(Q_*1H7?Xc6_=tQ0 zT6{4}U!`ukx7@imWDzHHUUE+x)!k~dldIU2-m!b8z1`aj(-elk_r{4jTel-;eLS-B=Of2s|6NIE;F_r=epVgNzkt`z_O?lK z=E?8IjnT-+=9)g*5|_clE6msdp7eaSB$wlpSrlVS)0QvK&c|{L%-ZRbC%?X(;^uhz z?IccL&IY+~zq1%9uA;u3=zDM3CXv1YMn8(>L*&3Fw%4ij-Q*W% z6n(Qb?fqhrec&Y0`#XLo$5*q>Ze7;o6g07@J}zQ>?D#K(l{u8MxL33lPkxQd`@lZS z`U2q3gj&r|$!#T@J}nL3|9-%6qG|u-=eK{pqkK=tg1$|k!zd?}+zn1|)9Yk;iPvA~ zLqbOsT`k9FW;R!)!8|@S#=9L?PJh6ric$L7`_{CT>=g46U5gUe3dTTeX9hp{Qx?jE z7DiJr@Bs&LrfnE-sERn6&l~+zpzo;SXm!0o+xeYNtsm_CsfVJtxQv1h8o^v@>q-Rl> z!`F8H`(N}`-%Mna%P2?=%eI0X+QgW(sb$ZsJ_oj`eV$6LV`A;}IZdD?ay}C9AX6($+oBS|=s^#jdG4oUHW!p_H@acxPYon18Z<>%+SyU!1gvh(*Re`M3= zFv>|Omk@+(v(#~I6*&1K9}0H#2b`!-j!#^mDol9`o#h}IEXJpg5tnn-Dn{w+I>y{q zwqu=6@ZE%q5l9&ZP(@mZ?97lS|62Uh{#&Ng{fk*!_a$w;AIilm=cJS`$4|4RY;9{B z>-s(R?Ofw$p`GoMe=z714es?5QoiYgflS|3N-P@JuC3K|FU@rqrWvk1`M0d%r|`E! zOSps&w}iVF9eiZ+0E24+2Yk-fF5U`82(GCf8&VmND9` zZ|PgpR-E?~jJ3GC2Y&Le*(D6}(p_Y{*~UlP(lr|~V50fPbq zMaSiK(jaRl9?B?C(>y3u&S;^M=UuU|>2#dDV2ck1PmkR8wDVu^V8~9&TZ5ncnr9|8 zv_tz`U{1!^v}F$aPi=&@E*3krAKMwcd-CMR4wHsw8gM+^rQ8mwm8_q5HUT1eG%>uq zV7(6xT=kO{v|li%f;PQ0r7bScM}6}mBsqO;>aB9h-bY~w}Uw3>%@2JfCU z`O~eYjM3tmpKmqOXVKg7x8hvHC~YR5yXEE^Y^Ib0lx47Hn@L9$_)dlGm#p=n#6EDQ zKM{q>Tn71r=<@kF%Yjb*GS4~2X~fb0LEZ8~ z_A=BbDB#+ez)mP9dUh6&qm1Ld1l!wPrk}N{#Lxw#uRDf%($!0BFN5+2e`~v z;peK>7RL2*-uA}7w3lcU&FYz-=j!*0tMyk11?E^1A2263dmwK0jkF!)7WoRaI zd1&R#ajIZeE<_`)aHfd7pJ=BKraiqXhpdXPYiGuO*N#B7qOj?rT@s5f5ia1UzmD`r z0VroEgPgm*8Dm+y+=5r*6V^Y`FWx>SfCncDaKg^&)o6Tn#}$a| z-!@=@OA$IEhbB)p(zTx_*4JKKm7hL!EBXBJ_>KR}!~N6muMf$FZ2WfV*W+Jb^h(Sc zKRG|0ame~B#z~}4@{;UD=x5Nb*{a4>#I^F^sqE)dZs4TLnPMbWR%e(Wfc>kbw11;O z9+YH%dh?n+?&WPkT(y@tr#?aciv6+sI4c%ghe)nQITUkfgFhSTZv3s`Qkc&oIfK*E zaf>8fK_^LJ)((Tjx-@4>d9NmLi6(kl$rB^mNR&ojfVh&bxnat*c4ny}3ga#H?96uz zOX{COc}7%{6M4otri;tl$$qIMWB`>>jV7IxSfSL*XKUq*Wy5GCH89~6qPSSj3wmn} z*I22v6cG4d`B`eM;rrL8Ki__Q|6J|swDJG`+k_8~_rK_`zr?Kc)8t8_KRIt5J4Pj$ z*T-p{;!?xTLOX$e$c+87=Aps00PQ+#5Ur~PY9%$_x71B@?YA81+{oP4HcXrFG#s!CN$QvkY^a%C{bTSu;Erk^D4XV(n|7nion@KyJWE)u6v>9_V9NitB@%x5$5 zIIb!R&`N4bgO$FuYw<#o9h;^dKQ3T}qn~(S*q!>uK#qhc?DyJz0eJMkx7YjCtNu5# z@#EB618emw4Lb|%1o|N}_Rp7x0MVM!t-R<_)Lx*kbQ=xVPiV(>ZL7$1^dah52=kUp zppL6!9~4y*qMD?pQdunIT%vm3T48*dKO0%N@q;Z-uJ)26Ib8LChb}Sx)tP&NQkakA zz`yEAEpZC{s~!>OHWNPU)LXhCpByO zNz>IR^6C9w)h7sf^1-g*lStEBX8*zt7K)_^UU5u*Of)<@i&^^e4;xILSd-70--5 zV~xA2JwF>eqRJ$FjGje#?%4QaR(H3nTxlJ9uk+&4ajmug*7AG-e_mtb9}EVA!C>%8yz7m> z|5oPhQM>}Wz@6r;@~`|x8h#ev4&CC=XJpE!9u^Bb<~+Y_^{y`g3i&L}j8wWwpbWc8N%_ z9?we85&BZCeD$=sL4oy3oeY#nE0O;v;_CSm#e~}l-}dQtDIgpR`&;}O3`Fq=#yBBb-Szc5JHs?MUpTu>^J7vX1Sg4FW^Wk3HD!n+|;3cGYcvFjx(3{FmT=`g1TC3AM*eJ002ov JPDHLkV1i#&ZLPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D5)4U1K~#8N?VUYt zeOkr2%skI);WNXBcym25K?aO?qFnrcYt?*2-t%d*eksGAcw<`p(RC5 z;{DGLS(_RShr=N?rYYO|?ciJh4C^#-`EWWNr}-2PBI9w6k0VRCkTl2R>2%`5VM1jr zk4HKOHc#`3%UElM!y_J&z8H*9Ej9(tkSHgb6`vV!0X&3?U>_>NKbC_8H2Dxtu>>A} zy#4v{_WJOQR-mlq;qmd6%Ybfbjtw8lz2l7i1wi@X?Ul=b&(@3JVZ8_**8K@Fg>1nI z<8&N#9?|6*t_8V<)%4?uG7$d>R1V6FC~%=R#eY&31)(@(SXoXA-3-lv*_7(}5S4yqLCN9Y&We^zu|V zlTq{6EXQP1m~^F&j`Pj)?@-#4vKRFXUNTdg_-ns&9r#Rjt^=Q`&J_@*=}_)a5aSV6 z8C>91wq9%VJDB5foN0p^{U#RG3{D6D9%b5_nl93w&;ob)l6-3J_Y6e zVn4H!+Z3SnP!h&tyBCH9fMZMn3BdcO!g7pZtHp@a;>Tj_xYvcwwZqdCXAuUSLmvZu zIwm<3c(CD-hbPG?^dp~ze`Jm&BG=Ot&~3pZ)S_D;A)sP;8d4d6H@{#fubbB*ln#v( zJkQ~jP}pIimcS!stLVXHdcSNwIwm>>ooU(c41quJNVkvxdL#S z%JPLTz9pBb<9sAMP2)O7KRV8r!C&;9D`3}YI^uiVC2Uz@4-0PKp3`~H(%!1O(Pk@F zrm0~nwn_lmZes@A^lS7!Tf(*M1gx=JJffP8ib3&TkJb))F;BfwjJx(qtdWNikKf3H{POh|j<<3=m|fRo|ykjsXoj+8>RU2Y*5yOtG^1UdUz6}#OQ zbym#dkmAWB3n%)$Z{|~&Hc|k~l9xJc$)_^k^qkOT0EU(eIDuOE^hj$U=s7&sVQRFXeI0Xq)vMV7X?451P{) zQmCGXco-XE`x17M18_s>MOdz#|(&2Fb z_J_aAR_D{3#}`t`Xa-gj63rvtrM#zxz$TPYPD072P5Txy7I_>}JOK#8+3q*#D!;kf27Nf<2a5%%BNv19pC2Y9P0VhaUAC? zdVR|8x$un=ZX(9DsxCW@<2V++fBHA`-~awk{({JHoQvW4)NveV2fhKJxCcvbs&+nA zZZp=OOshDfv#H&HRd=h+Myza8joZY-RBUJuL%aL>10|i?&DPjm;zl{zToboT_;5{B zWBrjV`8?J;X#TLx1}r=64-u~B`^#-8^{Dgdx$db?&z|Traq_V{fad+620)h;`dS+3 z$Bc)Gw`ysZcEBGz$>fI#GC{}%N+&C8of`6cWA5wLZOBP~JkIpvAoDDrjm>D}2r;$9 z&*B=RU#V?H`}~K!87pV4%w=kR2$c+Nt{ID!KIG6evXa8-2%fZjmSE=dkXY2mnyM{b z?p}BKSaHa>CEM>Hjr<5+Iv7-Y1f)M=|9;`$2 z9&c)Ks?E#cs=&)=z;6vDn@AWo&X2VmkX)&C3VoGN!*4h-;RC?Kr1Nj`f#&N8j_I=v zbT-*+)+J2}LDiR_4=3SL9FI^BPG}u2vwK-vc0LXL(ms&SoU#G%XMkFbt1`PD8g_Dj zG)W%);WD%vVrBKckgMxm+tfNqo}={_`Va{n$Fuo7OlX@Gxxp01(8t3_mE$k_6@BC} z)xK42MLXGb2SW%Ys%4C|ZO26A(;xKVQ~bKr_ZKXGe+vIk4ax^D=VbGz!S@&XKAvix%Q9UJ2BP1*Jn!#JI)Uw|v(km4EQjMIShEE@TgFLNnyBtuv&Hs#Ye;qtkg z%0Qeq&`o%tKF*~M!M$ZT7U{Usj|KeFBs@x8Z7}9atyAi&e5#3a!s-{5p%AW5MqH}v za)HBD%A-C73I-eEgPD$G(w3C7w&JN&h)5pH>dr@QYvSJ`#MibIu1I!0H0`9a#?qlO z6gXCST_aYKYq~Z=S<}^rLd%B)&S&~kjiBcwV%G(9K2-`Sp6~`FIpKPB#0lsUNElL}n-zK!` z!kPlhXd!#eO#14D2To`mE+-#iTe=QE3t?!q4>SEZqvA9LFIG?C@gr@KdxJeB#h zXWt@@MIHwoPt=KDy0vM)-^Xv7j>j-AG`^u3lc#&=+Bvd(x)!~2`ju&1X^UA~_dRW0 zA91cTsqeXbIXZJKWouj8SU0}5Z+p#?4R$!SEnIMYEvk6x*FS@u_D`4bu6*kJ)3Nf! z`W#39p8&4E@T(({MX5rywYu)5Ss#Z>6Behy_2*L<I~UNp;zA9LH&rtxtJ* zs_(H{Wyf(Gr$Fb^!xZna0$6I59mjE;CRsjp9LI4su&e! z^g@WA7n*O^5(no(?0#Ke3ugXt{z2S4{|-OW^7fveGtBt+#h0g$wOcO+!x-oEDgF=X z1u5OD1p^}1EV+UGvNZ6Q9g+E{e~~#mk-RkZ5$tg3jA6%=BP}}%NWHZCdkMC;yG%c6 z6@kGM5WeOdYLVtum)fhf{jxwi9S`6%odrJ^wYHaAk%jRQ*6OchY4>neboIo~y#p4w za2ytZ9aAjnV_I3Xldu856J$2H_QZ*zi~G6TVJ zysRb9&C?`O>$A)#p0}?;r4I}fej@^8MgN+1I`*4(7_4P= z8&Aq9wx|N3I&6*CjK8CbF@rLYV%Ih!ENhoq&}wu*`e(*P+s6RVJrx&i4p7eR;Rph*`S>dJF()j+j1sb6dSL9F;#wDj3Np9f^f zsK1SQMfxko-O@*SPV&O^(`Z+0eX%%b3j67h8aV56LI=0B+99h}(LS3?+sj$>ASe0L zx=zXCR@@f!tMn3%aZDh7+3Q_%6NN3-4xv04rLKrW8~j-VE(mznD*o}$#~q&T-~Ond zm>lkY(IG7$?w(({*2B-Io1g#;w?D%kST1iuz0_>$0xxJDgFiohz23@Ci++E)Oa8nB ze^CGTJ<5OIKXcz-&&C7i8YW&Cdc3Ns-0N3wK3MZpH(AVx%AYeV)Njl? zCDPZ|)_KV6yF4{^Pn(xZE=)(3oPfhTEE1G>213Jyt{no>1zq>yP{`%2ngBL@#(wlf zu$DW4N}mC7AzibqPthGTdUP;omuE`&bc}fcL z%!=hHphh+$A91UuX&>J`6qSmW^23IH0!*6~(_z;>k=seM5pOW&WrVk{`7&^n3j!d6a}_l(+0Ka9DI}t$v{)N2VP^-=t%Izc?7M zR!6t+qD5AFfqtObsF@Ez+qNsIKhe+!QOklEw^Rbkel_d^S(ON@MyaS27Be}Ou%5P- z8DHkldSEryTo&*uO|R8@%hL@Di?TzjDHb%DKY#A=|MnzA-dmpTwlGSZ&B15q1Cz~y zcvuLx045fOpBjVL7HpR8_Jn#nIFb0gpU&)8HvW-&|9-STggL9KZpH zCt^!9jr3+ai&LACnTvcuPS=nJ# zBI?6z_zH380kyGNrQFGfK&=nR^^($Lq^XZY_ahR6NIENB&e2zC-9U33RYc=;+KZdf zFTTzVGD$2!3cAzXR^k)llUpwC158%G3UNO)DV!_S*mvoc8$GiJrE>90P z(Mi1H|9p0N3T*IyE!B0=n-Beb1>*OQKA2f*1U+KqRAO4|rOm|8X~aR;cN#q#v(_`Kz*<2a7vIF92~`0~_o9LI4S$N56Em#4Tb9X~5EjPx5v zRGiZ1e)QI~C3hUhaU92SoOQI8r}UYLDSlSA{ZUzK8S2YZ$8j9TaUACzn9I|&!h{89 z{T%`NVZFvz2Q0bcIF92uj^iw#wLA?g(r|BolI>7+51*HzXXK9KIF92ujIF92uUkF^DI*#Kwj^jAaSK-^Y{{ycbVg2okTI>J-002ovPDHLk FV1hKEJbnNG diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_1.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_1.png deleted file mode 100644 index d633249c2ccd9b365c825374fe5472f33dd812f4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7898 zcmYkBcQo5y_{Z&8qoRYFrA>`eHHw(ETSVIgl^wYHpq@ZB!`}b0! zpzoY1DEOdyn(C&ZcH1S&>GMMzF((3s+haW`Je~FrEMt1Fqj}CLzGwq>uh1lcs@OYw zTyr|9Q)9Y%e%Mp+JJ^wfgU$XJ3E%RpC?MKD9(N2jJK`@gFfN8H(ptpHOeG?plVO?P zY1GUPcZ^4@sY{P-PEE_lySAFDsDDWq+-te*G<6JOEE}?+h?ts_a&gcJzT2p#`oXZC zGxxfgMc? zFyl}?q=mNzDBzptW^<*6blbrbicOI~~n6WaF$lI44Fu?~$3VGI*F5p%)f@ zAlu$3)$I9cyfxxAll9l_r<2NL>F<})@05B3g$?8O7y~JV5(Xwug5Nj1}4OefBF2mO~Qim=M_*9FQP@NCVC?MRqE)niBqD? zymRu%D{HFnFng~iDgPTKxA}k0YZ+5yzv=8rX-S#V^U>t^?H%gjayssgPt#Ak*4Q(XM?)aW1<+@hQ}nHG;=(j-wK`@v{BNii`h1C=c1X6K zw&p-S#IVK%e3omS)iZdgPpv9TG5U1!Cbx8?HSdUXB7l^%V4tC?HxzE1d;;Oa^}5G` z^;I0pS3{jOhw_Py_P8h)^@ymmv0s*G!Ifr(3 z9hSVO33A`KGw2EdGikTj&^I(7SdP(nCiNkPu>~4T!DDD9tKhCFktF8M!kxQJ zp@}tg_3tbZaoJiQd&mD`WE1L*DH^zhl=iSxG3#JAN3Z)@X_H}rTyIf56LZPNMUlq- z=D)88hL3$jeY*?f@A6cH#g$Pc0fOC?(7i^BKFd7&YP+!j^!QcG+vT7BGu~%_K!-LB zCzvEU(IsR|v6IrRG7`;{E0|ifKquGxCi1pH+(Gu!KuR9zwP!TicNCS{cW0S(?Ik@k z(;rc^Fd~YN4-$DPIFuIf1(&imD7X8Dd+4xNp8uUp95#U!xRv6^#JZhE#!m<_Pdq8J zQM>kanTDLK$i>|h;n4Kir65><64Vkw8F~CBS6xSzacnf8h3Nr3@}nnYZM%_93uBorJ9&WEx#z+>_Uq1!L2yraC8 z#bcau&1M>Sk+kksTUZI4Co*MoZzx>;1&f^^w}Yd%^|1&M^|Eb947f$Q7YOzDdzWf3 zh!9QLzyFL2?_zjkyR>vmux<9L%b0WYAR$jHYEWwFN`);RD|tG{CWhdQj_-qQoQeFI z2t+lpzM6bXc{F~KJOQvxob*UU=!Bo_UuaKW&8BvR*ZjUkjy&XR%ikfH$|R;0G~64yg0L3Rz;5Pw4Vpt|X{5LwC*@BdMD+j4LfSR8EcB0THEt^YOH zGh9CA))$>f?Da}t0d>r4FK&be@+^U=4dDzojw*QPyf_kO^+&?sNgGn7u{(Z{T6)RW zC9*hl2TWS2nKJ7(`wx~e{1ATlMc&&xR;p8nCaKqA58pVfjmFF}=$;Iw9Hra>!@s(6 z>vXdXmH7PF!=r!AYh~PW7%&3ozi@uu>I58kVi^y z1$;oA<1Q!gz%lm(iR_1jrR-NjQVrd0v%kHLQtlg@znlrMLN-+I|K74M1PruXezZ>d z-c0R2=5%Z_cmFLWSzC!|I-}$?&_a(htcl8S;f(kx8IpNDMpUj0#!1tw<)6M8;kPJS zJ+-9wJ7Mg3yZ@N7a^R*>`oNWGL+~0!%1FnMMIVOyh@Ga_<>n#aeL=3>!%-#t=J97t zG}e-wjLCxlQ5HTEYvOD4V*Y+V_IEY#7H@jz6V`x#NQ*GG%mhToQi;L`ELh-!#okF! z!GON>_S(r`(;N3ViBupvJc&e#+n3_X=}9}PD^5|wJRQ9%&AQ89rO(scGD*_HcQWyX zB2XYy6u-2yd>a01zE@_A8kv8qi~7n^K0+-W3k+$tnr>H0a`J|e3p-mR2kU^HkZdEz zjNOXfqRj6VwW#s{!&;S{y0y|8Uz>eP5LHb_0W!Hf98N+yAJ+A2U3w}LYE(=Aw>768ZhND0t-?(1AkzKGG2X7{&moTelDAfxUv-=%m&Sjp zJh^Vg8-6>^4s~+Dns5p(ZQrZ7u`cHd@6|3a^cma6iq#K`xrj$g;5S}-(eIW99>qc4 zy{)Q>gbhhSVK$NSd{AuP+cd7|1!_4Fv>`c!G~In&b@4qU*Kd*cf?%{4CufnNL+FD& z-qpDrH3!L%vS=#_eb1{az*#ivwW4V}YgKaOGb=UVG`#8D4#b+N^2MzJ4>U7$D_V7p z7#x!hzXv;>k@gP7kTAB@D(Gol`JW2xzJgA{v1k}O247XzrsaD93 z;wWuC4E~w$U{#qdq_8#f$2hn}rHWs*VD=pnF3LSTsGr~LBW+>#dPMBQtIa1;YYe=g z=cba)X&*EU%Caa~4U7uS-#UVGM3|i7Qkl>Y%e#$=UQ>dwXFtM)|Os=#2=DIHZp3hZ7+&^4)Osp z2{ns!GvuW2f00M(8pB`#wokY@8A(yOkZsCp^Yg8S1) zdil*KH}I7_lFFc$iT$x>dfWQzZHA~^Y@;wA?eR#j($^LYW#MO9Yp4F^C(;+WysnY( zfKA^bIspTiKNaWY{Zqo(J^-!OwHZMY0+vK*;>N*KN6KeUIKOpvubG##&+swD2G4cd zEeCstMTAk7{A5;3+H7;9kEM(LqSDLKbHEemd41<(E04`2hQ!(fgC6`! z{#jU)VXjG+p6qm#o}50ZJ*^X0%B!^othlmCa~hHb&s7wwhJMiV3Q$e-keXDm52I^f z%45e=&-vGt1QWq^wcSyCAzq>UP9+E7e@h~qXs&{^&CLtxu9x%r9|Z4q&I<7U-0lDoOmU%A5%k(!?g1*vGW&EKs@FN|bKiXT9JS@laj zzNl2?lZZmMXq-LhP)73lf={%n8fNNC*tp>(3+somP)OO-)mK~Z4*}EcWS@`IeRSZ7 z&EiO7l5EZvjk+J#&!ZeZ8i@0n3qvA-)<;1J$0lL>br{2d2{}z*}~OVpDkd1Vs`7)C8`_v5R!&0~pO^LLjuHp9Il&m#O2OVk z^<01}+;f&vX7$N=c5CW}#v87TxFaWoR^3{|edx~PhMolOJQjeencI|h6zDIQ2&5eW zzG(3BhYf)F1|y@Ov!n!_JF)1nNd@`n1BbqfGjEHaRFtpX3~iE-p~5fD()C$N=R8$7 zA%8M9V~tT|5|<@`D9ZUu498-`@xU;_;mf<&@2?MwHeVb(j9ZJ}N|`twhcWR20$X>3 zQz1!N;51;^UTAHTc^^b9Mxzd zrMyZE?V>x}m_P@XdRNs;qq7erj?4my;S6z0Hx(vH&2pbZSD0PKxL8#Opd6CVMGkvO z3L2xJj?%RHrq-|Ps{;ZIX#GOOZRA>X!s$7;^5P7hxVEaai8Yg#*;&b0ql5AkBYuom ziTu)vzV5hxBAZU(w5WDe&ShDoUxK=N-m;mTi+dIv8l=qA7klRM|9lb(>^t|+BBMy0 zo&FALQ2gw(nHzGkY|ECfutyw zT(;GPry}U^9sl6@WYkQALL9OcedJD?JIxfDH0sLAi4m zbA0iyuD3=U_WVsdAoN!ne+q&tk|A{lo0RUbggqZ8f%x3KP`K`Bco{O2w>T*uf*uaX zGtf0h`L8{4Cy50YmxrM696g_$R6vaLUhm9 zZBQOHtu2TTU#hrVx(H-bd(gku35F*qMu$Zt?dlPf*>~0zqRCGgjP{%q=f5OwOu4!2 znDQ>0!d3%%B)5(u{FUqjD|UxtDs%W{3VB*0!>e6?9kor#8%?y}rFNvk_Ch0M)+Bcy z9WrkzkmbKv?jNhQyv>1`!{e_{kB-T zUv}tQal{1EC7S9%(-QBZg?TM7IQ{$nJDL$Sq|1-qyqe~#(rw5UXyCqE$1Zd?aSa7N z5QDsA3n5J|7G;El@ar)hfKNjBt=3l44~z@lB@w0kvlksVjx< z`OK8{Q`&mfDKn-y!}`_-#RoRzVILbiOWMjG_gf+B<6py(MaZ*Ly&IJsvUMXRQ=R#TgEtS?IsHb}HMU7nHwWA=eCcNufEHu2|M^R; zJ{0$5vv*)+C3Y52Iq_~I#0Gk}iK6H@w3F}a-@Y4ZV!)$F*g`oe#bq2+n0VVYCW?_(Q}~`^J#gu*i!tUV4$6H?|Q{NN0h(%MLPl6c~s(R~r*L zs$V#>iB?}44w$bW#Cs3)e$x)?ijs={^pp=Wl4^-iLYnSB4|q-c$SPfgbencoV}3gQ zFxf~ZyEPPyhsm4IvJ>~)wwsU6e{UmN;Y`KxUyAO%E_$gTi`IXvA=ev=xMt02r7idM zWj@NewzCQEA?<%=Y8HSYhgvqQ+X^2Qk8{a~hA>P6`*`nqa5LXs#xLO2BsEu}-($pE z>jRd^3*^0_UC3v!mx9CbLG?QoDsJ8mwKrCn5zTh5Jo9s^CeqDM8O@4xUNskLW6413 z_SEW+2UKQKGTARPzbxe?ZKuxgC!2K1k1S=Dk}o5jm9v7FqO^qc7){6C)_9BR@)2$j zO5^>zEy>OKUT$?!_9OC*u+M!i? z5!p64Peakk!#gB#e&v2vsb#FY{^*JawOAzrLO`@P{hmL;{kgby^)ZD1f=tFZmw)=Z zMkp*2Ice-!d{f-HphBtpCuA$*{Wl{C(Om*02^wA40ZFjlBD5tFoiLXvo1HSmpKB1-jarKTRJb59id zqD7EWO#eMa^aJ3h~4$>o4$UhV2VS)_q<6>PK<27Wxpc{GO%s zNKrSpBXsC*bBv2{cROX!y`7tr$Yhhp(w{ipPy_p>GUf32ebmuWIq?p37T7jyaV46% zaYht+IXXM!cgH^ahj6j2qJ^Jd#H1H^@yEmQ&7iTcaGzq?J9ods29Io!rDO2Ui(DjIv?DvkMenB$hV15Hm~@#2a12oL70WJY9{q5`L@kL+JSld z{W43sV(;sN+cRH;?@cIPbeFWk5$P%~Zdnm+%~kK-J4BmhEBK~BEUJUMIYMqSUYKZpz)D1GJR0!4LtMlCTbHE z_{!EYUZn=!GvdPEWY}xg3e_5u#G*fI72s6o_KVvQB_E3|fOck1y4~gq5 z-jMf8$LlS0OUh<+F8VhIfCdjl?Dwv8`KPJ?{5n5xIOPl?zb&uNZ36#uH0&t}B%OC%p151x+8eyzuKXF--r4BiE6Ui_yeOdfdBfBA!^AckT=FQPWhxeU~>>$14 z!cm(UtoWTO>^pm2HSV&{0#QTg+!5kdDx`OCvMD|wtGK)JJ;p5#JR#{jv37m^J>L&Eo9-e=Kk>Yh?#OkyBITaVS^8opY~r=gZH#pfSx7 zGz}W0m4N{K>{9MZ@06$Xx5i5(_rXh;jXbA&by4Nz z`qwtLAYQVB&@tFX_0}M@dxs-lLNTtrmc^j|Q2@lcMo66{ve$CbucT|J-lppSo&LW` zuI8~;@!4D0q$+tNgSqA0sQtp@tg|U7Dp%kQr#frZHA@}eN6+OyxL##PA3WYeKe<8= z%3`T3K$vZYa}D|(`m zPqs^VGe&YWyGx3`iBi51wa>ir#UIoHfBpY|o5j*g|D=Jp>WO9=Xp}fshIfQ&@N3>> zp4^E3@7SWwp6ZkbmapcnIh!*0yn0#ee2gIrQTSyb7uUu|P>5UNTMl5mJe%Zjz@=;9 z%U|LT`%&=XK2GwX&prH@wn2e@AYM<{=2O)>eXm9U7OLJ|5qzo-FeiGhj%F)2DVlLZoc`O_py)7Sm%>yU>((9@hN*a4nIrU%7 zJQb{WIs4bwq;dO+h`zv*fF1FFN5rN?g+h_I%We@XzQzB*a|i0;MC*C(6>-V@rbI10 zHbp7?dU1}!XwE`9i?e-UmUEG_@DHOm m33+S(2hnj$0vOC6`XxcK=6za`y1)0)6na{QnspEDBj z*@flh=kNcXH_o|p&U0tZGjpH0GxPbLd7`y7l_^M>NO5p*C{$Gxba8M9Rc@a-iScl7 za0|BIuirYj9=ghMIA6z@w{IN+TUiZR9Gu!jvMVdXTc5;L#mEB(hqCvdf;;F^`UVGw zaZ*)5_LYzMUbfV`pEA^49!bvGV=u~S1po8|krHj|VsOX|zCyyoM^QAf1S*>wQbF`6 zi>_zFNgSw5B8jZu@{jhuo?DFr1~Ql^B752SVv_YS_H{VteH#l|N zEu}jOD?Ll5550?Oc`f~OUsnU!%lj~3zt*?HZ>A_gJi$P?L{1ewpql9^23 z%SiVSR0XC;aKzSH-}79FPgeulb6NFO;6|a@r(AFufyX2L<^x;&Zeh87Dixki0GX=5 zR2&PGMPaELPT#!X^26!*Ut-^HsFV0864o$urwgmpjZf2`RnOOXUpf@U$z zK8sRgk-e1u9dtC+#NB*yO@>D#_+M5}U`syX7kh*E17_BQ4$EeVoN&|L40N>br=CC9 zUQ@^^&ZW%BQqX9K+eLt9CzFLojGnB+xVSaOM!mawNDtV_n?miP(-xVwMUCq-n1N{? z(QSU~?ueH&X;keQLTRJxq#lo*1Qg5k^^Z%63)(f-w`g{^bmxxu7x1`_QWgmjruj2g zqxwJ)C0`El^N`d6S3+#S7{6R8fe zhNxj5c_=JI$eK(|`EAv=uLeAa*BGc>UA_1tKIZ?2dWpUp7hnTX+_YiIdMcQ~0aYCF zSpmb+g$!Kv2Xgq0;JzQ0Gml!sQd$Z_lV=Th$>C0naMxr!-_0d*1k3XnnB4{GJSkPnWg|J$AriKbBo*@oL<2Q0<4ibb`kfUOs{AnPI-rAs zFDzjd2?Pzl&jJvAKermwdBPxTAZ3|g(oL}Fr=7Bf-ZA@i#m7kud0fCLWIzohvEL3_ zpLS?5{=N1Yk@-nFxedbl^&}4{;Yh6DNWRzyuVUuo%4nu%iF}e3?qlGlhjD%A{9!OY zC`C}S&{$zKKu}*7ma-)&PDt%oJxW-jhLKZ@Y`J8i@hyF;8e2QOocaw8kV4k>xE>}&e9%e?qYi}8XNa9y9n0<} ziLQZ1lkNbHzDxVDioP~aJK@P%lgGnyR?l@i4u|ep20of}FS{48BWkiLy0`{68Y_~& zpjTC)E~Z%+l?!Ng=F2v*HaJSVhyeu!gshb=$a!j8F{T9<5_7UNW^Kp5;~GK%S>cvRd$69#%~TuB`7elpc_HKqmavSkdF~-^*E6`Wabh}&$0E}QK=~LVjmYi(z6W?fy-92>mJ~E>AvD3EWCCYogtz&hx{D~qn z^Prmyj`+OLQAc&)vZvlykH&Rg6m~9-Cma+i#drTF*K?}F%#_iOS>B#YP6;J?5@=+3 ziBPsFv!p_m zma+0f!do(%4xZZO0-JcIcA9m4zW2`C`{MYF3Be*Wa^aJ3Jm`^Rx@!JH<6MZB#Qoz% zg1Z?J?4%@#*RU^AjJ^TAhPnHk+~;SZC$kFw1_m!8JhJLv9mf`w_)TDPq* zT%h%m>ReuI;lBQO`u@fwHMm|!aMxxzzcNTBYR*0QmzMG~ceYKcXTv>VSWe%j33TNk z*k1kqhrs>BMtt6t22%`)%FXbZ{L=OMsz@W^_ZL1z^I0SaDiw2F~@RvPwN|%p2=l$OYNe07N;c3O+%9b zXFQKBUS7EY*9bKfs4Rh%;jVT!vYjGybHH~6KW{R!S?!kNe@0wryynKtPfG_H&uE=L zCxd^ocqjDkg?p@h@@kU}{EY={<-vf)MeTEJ{E>BOHS$O*<^&wYb4D-?uJ~y#JI#+R z9rG&;w`^hSW0hf_XvXl_u}K}3Fv>%^UdJW)_Kr=x2`n~b8Oo8yps&zGLJaS|t~X)Qg_JoJeOd1f=S+4Zn()~smL*j6ynukpk>VMm_g%94!dzW3d>H?>=VQH9X}aaoGSh>1;#o{|TBy_;Q35fTqS*zD`=W7aH(o)w2#O zRq)NyiQEKjEYoV%SbDHk$E1Fb==OsWgDeK#-%di-drI?x!B6Y^e_A(20VkO=j`yx> z8MR0{t)KJtS0y3xdOxiXIFhq;e5lm@*b+#hs%0VBBA&I?%s^{ov_3dh}cf+h?EqCyA-2bY;{GPkWPkQ9n`VH*@Fw} z3^6GbziYEsGhSVd^7gz+<7lV1C%*rw0`LIba(~cuRNSe*HPmb)QwhW8budXEvDGki zJThlqMK?j0HACL+0*?w!jAf5wpy%M8rKHnAPZ&GLwh8Tl>t=(ECJ+KeF2ar>X=#At zPRm>tiVo)YdJzeqG$vEryt*h0pOzhX)J47yocp8nXKq`RjUq1k9c6Fm3JtHv`BGEU z?mV*AfAw@}9*JgN3H86ArDg5RW7bZQ-U;O{^jap=^xE7nF5;3NFx1pa{!n_#Wu_mT7+VO~&adoBa?2Z? zu7$61IXX@dYqE)S#3i7d_IOwXW}Viwg1GVsquucuuW^a7O#>k)e0#HZqnZGbd1%KC z;AXPoxD#}Pd7f!@)lI(xx%>!c1cRTD0~w$7o8qv&CsQ?G#ao3gmU})w#|F9AuQot!ko?ANkQWQMS)lXM57Tb2h?%x z6ADVghoDmjfJ2u@=L$k|M8+u)hhLX}KB~x2fAlPk^)v4HK%YS(S7`wg*a}sfrgZV} zsb`28w+r*bVK&Y65FRF)X%eegcet)hNTua!HE%VYqk(Ei#!qyI3_HJD*UB$Kz-QN) zveQg2->#RpBXQ-niL~!2Jw!(N>O!hax60H;)F$Tk(}X*JRg8otC}HmCk94HY&la|U z1EOBG#%XaJC^OT)up&Zazb;;$kPyzl0CvTFp9R99inkUNEl0LoFx){d&S2C@OgDLoZtWACV% zQL%tyBPFubZa2Cd_8;CNnbYoyK)8`n$UbkYDe9kD^LH?v*V+frD@T&dG&!dZEkvFD z)*FSfya+&GyV6i3TLV8%<#X~dqdU!Xo+2%|5XZ~j!?`;UlyFRS@4cSuj%iT2IrW3yq`)Ea#ME8z_lA5<=()v4}{#|d@ z#FvLnuDn%)mSj;TDtr$wkjfJRYq6+EyoW&Dl)UQqdtvuv=!uH?#P2{)ao^t`SomCX z6NbC8spaK#Kmep)d?FT6)i~oVhTObA{l1{?;kx?0$l$O?mm;b_p^wdAHwF>&VktHp z;dv>@3!?aoo1R7cyIzD&BB*uY%$(@GjBFGl&T19svE;^mR`n*XlPCQ@!l`4pIt^&MGLMeg?tC#z13Zwp z#aV*K)vq_eWMcKr4LX?>@NXh3@4(nGw||at>AQlfy~kgu{pvnVUy)*aCAwP=h$a8N z6BagXUbV2-dwEK8OXRi^={ej>4mcU%sF;Ya1>=eEhD|e=2ilOf--B*ku&Zj*o2=wdh?sY=Jwb{^LG7q|G z1~u0lgt3X!%vxBW=nqx`pZ^|T+|MY6T&-WJzVzfS&Kw6UJG0nwT#2D&iBnka5@aF- zdR|wyu{@I$^o6D~=Lnq-m0j2#LG#JVWR|l7GfrYpf>8lCE90Bc(+iAR(pfi|zj&*9 zzzNaIQrgKMHq{Wb&(F@)+rJC$9W?fxl>87>4@kwh#$#)q>1It*uaoVk|6*vpK{xur zYtZ}OfUK<`X7W;$Shn-5+vp<1;2bdB;W!e$G;dCf15<7(8T(2^nkrQcLi%U1#>k)L zsKh++%Ma*gk+EpN{2szSPyb$(RSaTuT;<0CVB*PTf1cdA4q9TffE(8rb=IQ**6*if6&>FjZ z-cBF7pShtg_HFWbi7ETq*E!bXrmSFwyn3%e&q{^X_(oZDg(!9-DHJUr9Bi5&dzRGr z>2~_ORK27wQK@K7q9&s+i<_dk@>BUGD@sQ-sD`Ur&I9AYP5tK_zTZqzuN~4;Ud1kY z@1P-30<=vUe>f~(gwSN=<>G92+QZ`VgQkF8`~@+bp!?TPR5hT6YBMC6*BVW;k`kC< z`!1T`-{q${ZtK>rG0kD|q>w%F86bZK??`o%ngUVq&p|nzyUbDkUJw{#Suat#P|_<0 zG0hanaLDn(l^pt`TfAp1p`o1ZLrVOGoHUkH6VI&P4%|UT1t$vTPQAerrMIBPxHSQz z?$W6rhL6rxpKmm9O2OQwDi7?S1Ho#19ePYRflKg&#fnX+!Lw9_)>u%3s#1FqO+TCa+gqp3)71g zdiGJ?8%E-GXMmGQmSG-O?S)=zwfWC{aOBo5pmnHHnFd>Fl^a;J!l&cK`ZH=t06Dt+ zCfvS)i^mc{$3C6lpL_hWsII2dAANMp8@qQ>(v&c}qrd!MoBAyL8V93 z^B!3r1ukqCXpd+^&Z$;(s#rr>iix%1MNi9im@y6?dQo6X`MmqV__3`P} z_}z}v^u~RT*?1-58g@3K9dAwiCG{-+OgLVohUsJz5eX+&HW?eF3UfN_{7G`h zkNG;A5$E0Bx2#R97^AmpUYo?2(pF>9FYaF#?RO3AQl%tinC7Z3LIg&#qA^3Q;RD9@ z=+C-QOFIP?^@wFn=?&wp+Ab76!?3vOfN06P$Hw6(ACu{GajT6ETCi_Rq z^Fr%7R9M@7!wP~ZCQ8R2fl@V}@7FTbO*bGWT;Mj)rlVmUuAs&;vHjcqMXYaq)KUbe zo??&?n%bEzzfGoCzTcpERb9tb3|XKt!{A3@@Kvwl#$rWlTaI@74|}xDy_Q!GU*620{`Ot!=4|h%Dz8;*~3b| zysd=J4R24bw3y@hItmWp+b)UHqUpceqyKa)I>PkKz0dA-eUR&`7J>uh>&LF_R`R7A zw$pGRuENMG3rIn2jVw*_+>@&LFj#uyoOZ9)GNV|SdUUPa{7$ujYOxT_mp(LA|MDB2 zv{LLWax{?JS(a3!k466xlKj40ai3wHlQSdG#+LqEr@+`E6EWn77JxF0(TFirST)vt zkBMQ)f7e-DRuk!-56&swGQcS^ZdV#q#{cH)GW%ouoGjU#pPO!^nE>V^5S1y_u?l{OV*56xG0 zUcX!E37km&o@`(zI-;~c-MpOwtXgfhO16a4x+uY zNomM09Zh|_*A?BDO6H$?R%)x3FoYnkEv*ET{t4X|vH-J4@9SLcR`0Z+tNTNq*RPn) z;m0cRb){dO9hZ=wme-?e9r&-<=EQy|X9PVM^c>FBw61jmxmB#ktTU zU4@UU9sdOF99C|Tfi9HXF8KPMuZa7`y98Xp6K}yjGTcWc5 zmJO&?2Cm&GQHA2K!dsM8mD^{mC_CbfE#$~lFG?aHRGt>w>*c;!b1AEH)Et2aG&I2PY(h{O(|{EE;~yt~S> zF3=Z46X#Wcw-HRTiG~tI=^ehVqDLmQwK_XPJu=COzoWY!4oV(0t=U-Ze(-0uq?g=r zKnjlu@0QPRWS^QO0z8~n-5j6^RUOi=Fw@-Fip%|1JAzr-{%5~W`lFwCftVdTi$JiP zF4=|mfHS84DQq09vjlYrIj9LH?d!b|n!dvGKA(v71_=GD;{Fvs5~_Fjk>7$> z#616Mp}&>3D_uRTIretXWfZ=psbNIrnp9T&9|N9G++nP9|3@+X$G9KxBRQrI&YL>l z=F%U|zh|`^m%0@Be5(QfW6VOA1t^#S)u{jyu@q=AC-vVA;$zE~g*la*nHT*JmIaaV zGJ;4j&r-*Q^3gxi?_aZpqD=mmyBT&A-u^#bX9>Gmt&qsN@_y~Sy)E-8=q&lhw#cP{ zu}u_0q;9FJZ-NH5vL0dulSA)vOR*6T9A6yVltZnW{`N5TPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>DA+bqBK~#8N?VUZ2 z8@UpI-$9((nSmEMc)@P4KnGqpbL;~M8#&%92oB$zLhu5cP4W(40egYD!2mwOy!s)F zRs8E_ORbS+>Vqw@i^Zy9k=^WWwKV^F=?gn5v&aH@)jF+S2(t99YFPfz?0$`7#&|F}(1C1q(VRU&~mkifB#HyN(wDz1fB5+t+^yoxf{ z0-_C?TF$V9b+|0A%;z6JfB*55%cO-s`AmJmO%RhD*SBDz5)$j=kQO4fnInKV0WJ$t zORvfPzod>i95Hb#I9NMZLe{&5TxVZS0S?-&Gb)^$y9xgRN4mn`GVu6#U`dosN1DuB z&$Y4%KrL+X=CnUj0h*%(lvy*>@@vq^K$++h6NiBk+OxTaYD2$3A2+6;zzw5)-g@~A zug#Fw&MvZU?_r>A*8CJ}K-v66gSQI~^AvWo9#5a{L&?&TZ^@1Af}4h09lK{r(1vbD zO}F%)DKjmG!|6A`otf7#xmm#P@ER61I3GFe;EbEZZ$9!H0p(0CwR$IKJ3tu^3e6q}l{;VA>>C53&Vumu?WQoXyQUa8AU~9Ki zaVwoF@4~x@Bm*Q_e}QKk7p)V=h$QGy>s0jPE0ZL~4kAM@5=LwSx(tXW%PT$FRobT7 zm@y&KL6ikRTbB5x6`g=hxzUR=quS1IVWECi$=KwCLzaV@^mkt`~s zN72DDS%xzIlQN@))(-M`Ovlc{5qrwDu?L9Q&t6$a6m=D#bpUvaRr$e)?E|17VrLAV6ofg?1)3R2|`+jxpUfrJl z`|tn3zp&xFCU|9b9Ovf2oq6_VwtU8%!SdGPyb(8cYG`j@oSn+pwLSj+<9jG=z=q>8 znnSphsGihpJQf*p?bBj5W4VR7_+e+}v+UgwPu2KyTYTlG66`1IA0K}CO0Rcs-hRbb z0(R!73HE2K{rLX={{2UO(u8XPJdd4r^!O}T4zHMQ437M&TJjjW8J~>MIy;CD+VYhu zScVu6IC3HK{rQ57z-L)dCV+ zBRMjdF*5SL6b=R(yg!@+cF@5ey*`Y$<5@cN6`-J#9A(+O$~bobYN1B-jf{w zn!per_Q6ycq44>~cU+mjn8i88$!)K1T(l0n!Wc;gLZ6ici2C@-Wh<}PVZ!Y|rO!Jd zqecM>VqfW`Y(y_;QtQOp;8)@Yl>L-*w!qBB58%#R43dG4M(aWM+fcC#2QRu^H^k|7 zmh`cuGmsyO%Ifd*ydPq~?;sV+=NcD6%ema_OV@9+m121eeMa?{z;?7Xx@P9$Gog=9 zXT~14gl2#F$XC*^vk`d2?J}FO+`?t!cb*H}nPGC^ByLUR0yt6M?eQuKE*hrYo?qDl z&oo_&=NS29N;hIzzBm*IN@P0Ac0!c zARUnaViR7uY~>X@02KHn3eS#+mA;wSIU%OUK{6P}OIyj=El5lka!z*C1`VK&ng!3m;kVY7HRA?ZK?DrMpVeBLOQ zK}R$RY|C=p3d`!k1Bh*dCS7A-8LL=BNq}e*U%9O1RXYq`7wvHef-=;CzUdz?qNUE;l^b4m=(HG?<6Lb1W)+$G%X=gjJk-TG#UKBA&1}9hJC5Txj=gZl4Le`o*ziobjn2agIOcF+JMHDB)hw(1X4l>Lru?qX;fZ-Ti5=e5>##63>H8A> zQ~H~=4cAHhWtrAB*<2s1^dH{!{bdOQ`^(4}9UdR`A-6gfdCX*g;C|PtIb)dcw&5ZF z^a8-2Pt)>6@mE-6r^?aX`&nWDY(VTbSJ98pc^9jK32?ki#sG|4Y+M`hGuBl~Ilc4 z`1a+y{IX0UGeVoe7op3(zcAV_ZbxNnC*=yuT-Z;AeRars0zntcoYV%Y%F* zUR{{M4t}z|i<31PY40OC9ur&D4wR~M!?uOX!C=2_hbY<-m21&f`~hkk`ag$@gL zYRmd}ac%zk%LD5h=s>_s_~;5gF-D!>nECWpmEo9?bW(rYZLvi;kbvZ&Z3h7*8jLhZ zI%XMN+Trt<_hcutS@j#+qG97pfz1dg8^06}Yg))u$Jaw9Wd9o8pFO#c$`5?u(8=gE|BG zH+>KIXUvY{IF55g@WMPD58+&b*MEMU_DB4Ii>dsn$2Gv2Lnw{2Fv#0aWGB!Br1T8IJ z6n{bAPJfKTttVn(JfqCUuVbh%nE$iH>^ROA$Q5(En?3F2GCUrX_06A^r0e}1+6R<< zq*4wC8s)vw78f4*G}NLFc6B?v&3(6iQ_N_DKEQwl67OyhDF)WJHmiQMe%427iW1!~ z?H4vBo|YXfb0ND8NF`aAt9Ag1xo1C?H9dVURN@gZKl+R(O_fO2HYwmaOT2|;;tzxv zfFd?PIqJ@_&=x;XB8|44Mq8rtnN}=*&+|8(6X~Kx0|8kIal8u$JvOmFi!{< z*YkuwlimC}Q;F7J_;sn<^_|xr#H)vAGyd%s(#7@}q!Dk>6N>@aPHB@MP)e`YQSh z8@|8qsFlB(v^gpd>%f?IkbX3Cj8AmWv)ZrTv1@);=J4b^oWjfwusF8R_a*wL^fzxS zPixnI`HNYzlg};8E64g2PM#+i^R@0TBV&~Qk`z-Jiy~&Uf7ykZeqf{WP%Zvxa4TOD zf5{>{RqUd;-cEq#XZxFl_3W@yj$|lj3aTzu~v^#n=Ol_RA7( zd%!=~4qnja=JALeCOZAXPSB_Na&nv8HsaBCpnyqwyr7zJ7-Pn6E$KyM<{E$2_3h+CHM^lJeWMGe1UP zXMz{{nT@s`-B$M(OFI9NR{VN?n!cnM45Tuc{}{7SF3fV2i5m48I@X}XiLhU8{pF7>(m@IfuZ?JMRGSVYH6sIpF$>2IKD8>VR*ctM?2HS8cE zy)=-3$U^~*OmIwwWKNx&6RJs|K3;{dKexquuy5x4*qV0 ztZmUqb04Y4-?Y=18@4Sha>0Jv4wJVfbp9i)_yg27^aar|7}n%p3c{%iGl)kP=CAkf z@k+Q{n7L3a%*aPsk6R0~ngj96!h8;u>G}>=!8w-4( z5ra^n^KWc6{3iX(mk{NM@vCg=33RD=SZ7|}_-(|KP5Km@*4J@r9iyLw1k8aT(fV4R zFnw>t!=RUGw}aZl@mPHHwIGKD@4kG;o7nMIc4c8!eJt}ysIv9z`%AvRwCqqB3Ns6{ z*jX}fjgIcGW=%W-)P{q3q-mc6RJjFe`-mE&YW#(r`rNR6fwu9ap=HOkReq7$!i-eo zug`N6gJwMc$ycJYFq=zwT?U{`iNI5pAR-<EI_DeJ2 z4V|zCJDE+`D*ftPH`-T1-=eX8Q#^VCZ68mP!(>&rcQX5emg`%2Np9@`Q0Cb59M*B_ z`2&c)?mOGZ5+OR7Tu8ZU$3yU8tN8(Qy^n41R`y@?w=3EUv+pm=dG?pq80r1U&H*eT zXhH9H&3P;KqYf+R{%S_!vEVr(m#}TpNpi|tNMkvd!iI`Jjgj@+vZKd{ZA$zAmucAv zt4}|kpr7@Z=a_}{!-k4K*@=B2=OzZtX#RD~0x!%+Zhj&3H#^JVE=L@&EvcQ-oWio0 z=}fIM@aN|m;yKqij@sE~{-j*B?el)Oh4&uY^w;A1i>0Nn<>TYBzQ5zX>THpRQ`p-6 zxzO=6V#9^maU92SoGT$L%&+;ow$(cc*AL$+S;&gdQhGP)`n1^qc4A`&W93qlgc^<`Wy}qM|9Y1wq;3qQKAk$Ejxhp z{fLqz4>5z0X%EL?e&~a0!{<6!p29N`#%9AW>+|Q1nXiCdn6+O(a+v(u@dCs6xB@*~ zkbDt(VGedO9Tl^-uXU2#qV*TX6N#sQ{05i>kFfIrh|$+|Xl0 zec|(@jvd`L`6cX$5z~ynKF@Uw7WTzCXCqt&{Z{@ZX2)@^5W6r#;b@*BLy3AuZY|;X z3*e6^0Lng6EAt^qd#|?*M;Y}XyPytsbvs1mqy&jElMy(;;Q|QRjJBwv`}WPRH0 zdB5db+4``x4m>mj=qe#zb@3}7GgO(cZ-S2U-fRn1a2(^3fi4=DmK`8e*@IV#IUX^2 zf#mC$1L>wKmCR<{Z_uw^y#=Biv3^w!jo*|jdmzrPkka^yolHl?oa|d2<=3F(OXErG z4w8SYgSMcFNCE7FwU7=SKj^0m>@0pBqdfka9WzF19Q7EHacIIG8g@+EtiO611e)p~- z^KcqdT;D7e9#cTd5+6B1G9g9tv+wcDV`+xE;h#bdx1>nM5 zeKMtod@u&b7XP!4Cx9{UApK|tu11$G;H&-WE2o;@l{q{)52rA*FfWcR^nHo`DgDjc z$}b7yXUxbtAbOGwpe?!yn(v4;|CAqwh*g zV_&Da*f{?B?e`zw`ERW|m%!JTOS0X51UoeE*_$HqZ!~Ruy|__uK0NElUCD9m#5_Ff z{MQ%E{Z!6<<8l`8P`W_jF-%R^K<$9TnW(eQzqNWfokQmFn6wra4tKe%=m?BTZ{Vx4o7@n z3|Kn*L8Hn^7tGH!46XUL88aQ-Iv(X7i?heulev6vQn=i;0mNIMUU zfB-g}|1;g2y z-hRECo*q76G03O8uWzyZ@%`Q1dy<0u!~NZR*7P@2sUCrBMpy%6kQTCp)s&)8gGwSplAPwbRP^Gk#lh00``3KGEVh@4V7=1c zh;13G@twrbh}Bb^qC_P5wdbd%Px6L+Wf2-kha2?vBV~UkKdpF}T0b-UN*`;=d`7Y! z5`FvBM8e061XakV`Y9uzVyCNLBl*>N8_$mds#7hT3*q)2-YFAvCr9mglt)l90WIf` z-+uh_pD$dV{{7!?;UXC?n1BBM-#^~r2FQPY{_h`DF-sO%cYgZ%l7Lo9gwgbmGd3fv z0WwGnS;A^cQK&&Bks(R0=T$$>S{y8`^w-)C=I8rmvAs+L>xewqtxE6ip~=tf#&mjx}rq+VR)z)b-o6?VmS48OA~8EwEsk|)DVrcL#x08_ZcshQej1~3q1=HG03yL7VAHyY>ZzGAF#8EGT#-(9 zbP-#*v<|$#dzhY{KE;2m&#bmDa^UUV{qz8r*-xJy?&H;aVEBSLOCJHlih~@f=2=)M zH{((pUtq{Df_0?TFj!K|2Vx|i1hi77di@28Zb&4t5ITcz7Pix|--%Pa81mqdmi>-j zvL1;#nZ6*A!~!ZNQ$;foM33Pe<(AnvpDjN%M(vE+2bSzd+Rmsx0GqlN=;?QY&YDH) z_AO!(mP`CN2}R*2nuXF1(9thx^=mXg2B_v*2!)@}hD99j+(JZ$eoX7)sRP%>KR<_m z>eJ(2zy0>tKX7^c`Nu!M{TXd!pD=&<{`s$Obn)CL3yWAX--Ly7)89tthx{T~M_P-r ziGX6o+DJSJXr)Z``U@1@kVs-7bOzrnY^P(t6Q_7F?eL*6L1yoF? zie{tnh&)QddrLn7BIB7@91jDVnUUEznOS*@)%K$#ypQ@tG7|;$3lGbT->jM#T1U`; zh3&Weu861A|0TyU;}<&wug3n0c!vBYK$KgmGrEp1iAQMVp+&1-v+^hj?=Ag6R$XV+ z1v5!Ooq{tvOiPYnJk}IzLt(0Y!i=BaKHbO9ikZPrnAIo1q;D>mOZ)8$Qv zM}yPi@))f>+)+qNs3n?YoDY3xg)>*X6W@@tI7*eBy4!KT1Xjw zs5k(T)>ol%4QqpEx!0EDCLXfgigTd9x&5Hg?N^0NUtsHD1JR*XtcsmjCpv72wQoK@ zqL1>LeR3Op74cUfi?#k{@ke<>zZnlv%Ufn&Rg(*=%rq(@wrPs@)2OeVi^!w5Un;~{Y<8u=h_wfrZ4jQr*uToubj@~R7Fh^dHzWZX|1CwG9y z25N#%Q{L%s&F#2f0xN;iT%G#3kHr(Qtt}jJAQ3J&> zHYFFo(N=gzd97awP|O34HWYn3x-$_Bw2Z%Pr>@_op^xHF%>Ay*PYrB*FiBF!?cmoW zl}_f_ON8R**r=$~V~mQ?btKrACUHNS$^YVbkgAx3+_sv3Mw+U~(W|nDn*;HXtnRno z-$>0^$#DJ~@o>MOhBlxk>Ld%SV2st^C{WGVSFQ6rP|Oq;NWyzJKT&sv<=aUlDqjlxTM7>%z$@9w?Ji_yXT-!X}KAxML!V>gJ`2j1}eat!sH#xoqzP{W#+jeDq ze$?Z+@I1TN_+GAGOM7!0^Wpgzi|6`}kIzdxKg;th7tD^c9++@|{XPQA3a;vaw4R9{ ziog!}{hkWflb`W@FyC1!_ zbz!W(*Ea6YHUG!(3uXtvKcI5+$H!X%jtn=w8cF;cEl9VXpH~achvyx+>o|^`mI?XTa)4SbR!7pl^{~)$+d4Y5diJ!}Xl~OnJ^G4pwY%r|R^M<4?pL=~v~V+L&K3(#d-P)?1SHXguheDEzBHE6;@KBvAP16a^uGSgZT1%}5`P z`@J$%jdMkRp-t&`Nxw9#A2t-d{zgD-B*|;)Z)RVbpqzdH4|4!zfO0|l(3~yh`7HNe zjgF^f2jv!>=TMi&-;Af!@3D1$YGn4a%}73y6^u&VjJ1;X3LPf4v?M#V{ixaB;dp2p zbD8f)&Hb^qAE7)LrLIUo`03{SSXL<&|fZMEcGGS%yEG!{09LK4s1tl5OMn6}tZBTlXi8$zbUSfOn7`&E+Z z3q>TczeG_x9FNGOB)l6pxm5y0MZh4-gnc`uex@N?O*;+!CLR0d%})i^>gmqgF&bIa zSefTDfsV7gAc_Q4-$!`GL*6UykEAj^ivsE|_sSRmjR39~Hx|FsH>ANYA|D zhcLG8@?_^05%XZKc{H^4C>_#}E2*`Uos|T}{US651LrhiN;MjdClYO8P3uGrsOA~t zHbQmgX8NmU391CCil-mv^8RL)LVc%SRbq?@Zr$l`<{}}s>?6u5JE^~7Bt{tCNj~>T zDidR<#?!Nd)bd=5zp%rqMAWzA+oon8I9q-?Q0i=iY1UTkRP-?>ajR=2REwUSnIWK> zf0c>2ANBe>7Eem0rM^FAMx-gq#faHZmWK0_`JEZxm?q`A!f5{M@lcH(mmuurp#`e3 znN8-uG;tiqaU5ry>=WkO79*){?>2Dn;@GRqd zfqo6Kbv&c?Z#_TP7XP);%8%#GaU92SPUA&fFguRpIL-z6`|tl%|AlJDaU92SUJCk= z?Bnr3Jp1$2%-g$%PxoKL#s-x3xAy34gnxwM+%ED+vvWD7Juh|#{@FgmH1Y3&;a_!T zNtNt6S^kA;$8j9Tab61gg1LXmF(IM6is}=`D2(5Z#>*ac=6LHD%x~i=>nWQ zzN5@^bnAGOdzysDW7vXeh(CUP_G_5S_qTN)VNri)!aYBi&(D1wMI6EF|}003fjHD!GO05|!bXTy5{0AK;g z518)-tk?Rgih$Zlru};X$5BB`0RU)7A-uE3y_fMl)J$Ik03`i?FRWqrk9Gh6y@R^4 zf`OkUqEI$@^(k5CYqHjSXqW+y{$mvYHKXh0l5}XA%b`WDTCQAF8eqTWnv@J1gByg6 zE!~Gc-L$=;JFxziG7+@0JntdJM8XqS&T+lq0+ncw&`8?LX&g!%u*Q1X*Xbqod-J>@ zu&X^dC@`KR2pE1T?QOe?5PQHag^2^UIhcQd@jN#`KIH-Q-F3b!{E1 zQHr?j9rkD*Nr$40!GH`mE6^_rUSR1inYw9gu)Wsr`^F==LE}Bua&9ZN6a5dKgsKzR zA5cXEiK}n^`o**7i{`1ElNmUYy~KYaIqLD-GPlpaz0Oj*wOB)554RVGY)-im)GzRe zg7`_+`WV^=hGoi<#g>JNu=12X%}F&qBm#>6L` zoGx%2DgC0PInkZL;b&;(Ii1Tu1Fl8J9LRh5D4&UZAs?@m$6>9&!o41rCrzn`a6azI z1q6xg%SV)JA+;OC-dqj}43@fuv$_u`?ik{+U()I%vy|$`Hbv)I1w|Xx^EhdI%%iNv zuagJ*cm$|+*M9ht6?LDPYZ2qIwSNFu^ydy@|`9uC@sLR?=9z zmJB=-_sKWWnigI_iIJ_;q2Q5&REMR-nVb*QGoAA>0? zptgWID-*-v1f=9?tERMDmox!0&0&o>VeSZ;*^!lNn;W8&VgfP`_4qojMt!20xj> zndKXf7(`I+$#(|()dFR$;>WW*-)33%3M_=5k6J88;)_rdC%22P|y`6INa zAKSFs@i3cMWMSdY%M6l`{Ixi2oE1Zp#xz&m-^S;dPf-g@Uhfsoy=YdI1=NoBb3JaQ z#@GAfqZ01EreBThyXnjJ$=g(L@^Nye*U9oQ8?ExCI{~qcQyc~B{;cNvGhJc@{#7sA z72^N62kaX7dE4t{Cu@PaU`BEsE`L4KU9_x;x9p7<4GE1$wbV~W#eN}6-U^dZiq{hTLbVi6^?$b@ zBTAxMVE%9)tri7F`=8~?$#Y-#~m&Lzo|FBJ6L@A_1wDsCwm_dTE`6im6YlQv!7jUtgbK}Yw;R8rC@wE&& z{}5P9_loAA7}*6s`w8uJgUr?ia%|javTzE16^zXLO?#X?ho!i7C0LrPoU^RQ8ugL- zcZ@O1uMsCHU`Vd^v~|#n=cH+<5n)mq0GWq^=415jn<+5t2dv4gvmB>e#7kUERzGxN zdMRu%ExbKD;p8y6`}Q6C;G7)nM;egFDr2`GOEtdpOTbtPlbCt(WPLn?*KP&%g~EYf zh%k^Z`vc``6;qlAtmL7MwPyi@_(q{islS@B_oE9qSbGSpOELd0P>nJW4o zVAkVGIfm6u?g6{D)}IDmJwGA2fG)UQt*$gIxjjUM;Z|K%iws*xZLC@n`y+^i7D;Y{ zq<*uY*BvD9sv5qX!`D-a8aKWPKNTTSZ5X=cXSC~5FD-tLAV&3&0h6j-uizAQH59a# zxI5=${FzeAFyTn@Ta_Pvl-{zJ76z)#f`1y6FUr7ji6>la*iG;AF)x0cz_tzKM=;?n zg}$IDjpWcUbYj_3$i7lk8uIvD+Co(t3t=LRIEYPWB^~%Q$-B74CM4Oo%Y8{sB+b_XET_MrnuG*$Do&AN0Z8Z9NzownYAm4DnsTRr0!E&Q%Fk8#t z&}nA+RL1qIo11GKxoK_%r`==EB~-Km+u+ve-F~ zQsqu%l>>slMsaC!4pf)io;Ba9Wm~n+UGIVFgo#J5xatNSsBX_qajzb#7S{GZT9pPr zdEBK{WSCgG>N++ytdr1T4P~`r`Z}+fcp`noO6Y};{>2{1WQ*XFg7xV=cdShz=V*a2 zO)b;81oI;_nf5Zi%U0IDkkc60q7##)v~8!GEyi~XK~Yyw^`y@EN9>Y_aQMEA`WtR^ zr<)GcE0S`ucjp}+98CU!;GJ8fDeeTTUvt*|v;b$U`>R@v;ctcK!m=dGzaZ^bccAu; z+wYsEUKJG?cl@eJEg11;bxc?yI5bXz5&)Ffk$--#J_#agElGVe5!m9oNjGW z^K=@0n)Ev*8ca*S+po!IaABsVMVE?wQRFE1l}l@fGnjhSnB z&$W}b8a$=qi#oq9aPC(uF;YZSdMf^DgvA`6`02`Wdt*QO3+|Cv*8&lWTi8J0iCPx$B|*2-9W9 zaR;hxQYg4+?Li;s3;&vPq~6N4xw<+r945WlLH`7bm7|!)C{`?+`7}nI>*!eXfIU&I ztJ9>os%@wAi9Pi8ol!yD&)bf^<7&QnbsSe;IX`w|((-XEjakmVGxRj(g3XEVhn^-4 z%^BBdd(yn4T&+g>8TtMmT1__eL~FFH_*oF=%ktyzU7Rs8+0;JzZ7etCw@P1t%q2;r zZKr6wbaoXVzQL~quHB)ui^GjKQi-e1ozG&etBr?V90{62Vk9W6F)zF$r)In6s(an85>hR@<09dYB|l`wlgxZTlKtKezJ^| z$AWgaAjyN_toiCuvEe{99CY*EE2wri#ytmpvO8z|S;;HtH>b1u^Kss*6K0BrPU7So z!>gFxvb1>To=Y4e02H5t_Z6vBvDy;r1yH^=e~;cRc+vmhV(K!GLd=0$IM3Vox57>0 z{OlUdb;nkKsU+1vzdg2%@*;w4>le>ql6(d^U6dbHgx(0=eHUu@y9@2PDeJPWavTo2 z9h2@qQ)o37YPzAE@fJgwff|xIQ@YB$Rv-}Q}qButJJ)LS+u)?Rb zZUy+A50pv{bO77zAbQ=&r+*qf`4%p5Yf>0|^K296u6CcgANn5HYZnv)En~(EupzfN?g8eU9izYGG*Tjx=cLPwhy?KNZ`xBcpx6XdY zE%%6GjyQMGab6ah%?}{AZv_eEws=137F8y4PiMk$-c=h;fA61&$T!46d2^DFZ9IjK zKF`rydn&)-!5ijijx(Wz>L`LpTd`@?oXC`Px|oiH_YPG?l08}{*=(F8rwP93&r7kP zbL3Y1%Jh0x$8GVDIyf`MA%#iKxpM|I8y3zdodo8N>6H9c5&T%~_Tzz$4HxonB--3j z-K_cE1hByh2xW3D$b#oktZO8sybJ6sRH=(qwmge(ZEkTXw|_{Ikh^uO9VRuDG^ReA zsIHqKUaFYU{&8iOudh0v&$6S0imOH%I|?b-(v5|4Qczfds@pu&Fbds`g7x4v)i#W{ ztU8F}R?z&X7+olvMT89(Pkys833kjux5(sQpvM-p&Lr{OuI-~S?ovq}o9-T)f%Y{v z=g22r<%zk5n|uV0S+yyOf#yNiS1V;=2Tkz}XLm)b>Ug$J(ta_yXoA#{#NFDuFPb`=Qp*%E^$&!s_Tt z&Pm8@CjzhTbE1?_MW*wJPEF3>{7kE;bYf|``Y@y`&6b#aotw;;TP$pp?OU2p>cql@ z{)1u@DXkk$iJ9K8y@<-Vm}wj8&_A|t19PG&JqW2oCwPhag^2pER%4sup%EGNLf7GO z!a8nF@j+-osee>oJo>8HEWU($Gj@b43FM5QYvufNln7FTt3v&8V7J>xbAz<^Jd=6V zy(J-{57}RgdSByp4Ec4gV_kKsc0Ab;TQXz(=S=C0G#zxwc=;D%iB~ruoG_aBZ|7gU z?vwgw>o%RMku@FdL>&YZ4dxQLR~fh(d74)5(?2;IlflZO?nqtrl7V%QN;oBef7_l# z80&H=*|EJ|an`#y{*?9L{E zI&vgaa?Fzt#qM%?CEl$4ffbLT+iqGbEJr52SON1Od+oxuH&h?rjQdHQ!vX{*#bHl{ zCx6AA;EMhZdY{stw(T#JYcBl>OyB8Af+lU2N-u+pKJ+KwVS&IaxV1uBu1Ia^t1V=RYXKOOv&(+zHzY&i*%67{xjsS6IcM|gc7?UAh82IM_wK93 zyDjMm?dBZLsgM%AwoE7be$&^xz8&~RYXFVvFQk0%hExwtw<}4i2pPe0tAt-TPc2z^ z>ySzcs8(GHaQ`?ihIY~+>ZdWCpbopVNPuDm|BlmMR^C&_Q+4VWnB*$S!{M%AO(1EH zNl3j*$;a+5*sKZkxdzxBFh?OjL{kB~Ww4b{++CZzn-d2yyKEB^|4wu35S`)%CM zRHk3OT6x%t*P^^uKEM@<<*Dxx9sXe8vUEu#o&fF#XiEH;erGV`n8~^QKHHV-)3(-y zad81^M=aGKZ0l2EvFEvsNBoyBNdz@8rbALAEvoaS+d=k-XGgYW+#V!X7Auy{oS(M@ zr64Q~i>~0L3nlW*rAfh3$Vy!rG@@HHV$y)=VJSEdm{~C*)!u=% za-GY#w2M+MgK{?OgC}jjWja-~fGQC{1gV=X-ziEBjNv`Sam>u4U}88KW9!K`nh$K?| zqd^ugbJ5XdiuLv8#eUoNLX1Hz`FY%p{vhBs?#>@`u~ln!9$CfP7;8UZp{0}N7$G|2 zVyu3)ly89bqEX`Q*A4EF2P_oj)AaB44jd_#2jIFyI_3pzQ3irg=RcX6B>76KRm;b2 z#?ikrarV?#EOi{Dmyv(&3ggW4o^oXL|D=bD4r#|?&)t{qK?TQs=kEL}@A}j8)$$ZU zg@LH*vg*S6^ES?AsOnPPyLnj|d5V#&Hmd{zM>x$&eQK$Z0oay6z18fqxcDu(N|8AC ze}juqe7VH9s{>G0x*5z+x1vl?5|)LKwe3cO`(Qkm6qiMY8D<>MN~QS3SADHSCDUCZ z`4^H)>SpV~?m=G4#P15l7@Ho#6g`cC&fCgnDd*|@Ek8DEygGIY361A=)a|?@m#Klq zgTprh*jZliFDZ}?4S+co&5?i^_`CG)_~ru8BK;$dqfSZrGP?&0RlEaxCz3x<6b}h7 zftO*W0IIv|BqNTlpz5(zyRBciX0n>=Pi{`0sW6qB!gu|hHKf{uNtQBKO)cx3XEfu{Eiflzsy6W zP7Uy@DPR?+xwL7vl8vU}d@>7=R#ryVuaAn*FNBIdfaGnk!5FM>vzwQIF}k=8+iz1<=Y+ zGfV!2AiW8ga#|JaCnGai@~7<0@$wfBjNuXw7XDP5-B1E%QemviIjbxL_zXR4l`Cp4 z!q5iRL7ho&#xTt|^(gDA_1bhzu&y6`RfDSS0}9_u(DPVbyv^`|a#TOdSfrb58LXVV z#XvT`>qS4h16kQ)#=j1ejeGu6+~FrOkw{iB^rM5d$z9BK1{JdYQ})BR<2Gy+R9P8| z6x-N$)@g#e-yJEch+;>}o9UP^d0=r;GI0`y%EJ1Ec+qd21D&ng*!lkLobT5GeNRZd z*55Z92yZC`(j;5H>&0H+(H9!--3Gi|fREafKTRGp`2_e(m?$nyQK#r-Xm4B7^TVS=FkSTUkCsAw{RXUqwPc(m!hyh#zj0<60bJzo%5}9M`tca_75jx z>SNnT4ax8MhP^oN>-6S>5Xj2?j1HLfdndPGrt)8ib0n6Ydk_Pftoh3VJc3){n_`-N zGX(L*^p&ZGb43lLe^@KIiU*1R>{&P!QJMti232Es{rzPLj*;V)e9iV`>%O8y>S7%zWA z_35p2?*f&yjHuExaWoqR4D8bUrT{FQ$#aep7Nv+Uc}k})5m`2a{C0rYhk!Wse4-kV zb+<~=jgW$R>PN%|wxS7%7+pJ(zD`E;L$XJfI;`NNxtLQd$hlFbOU?CQ#wrVi3>xS4bf#lOpYUBKkKJa=*YHu(0H);Tx_vf7v(Z``86yzw1W@a^r5|?7@!^=xcN? zx`?^JRomcJGUe9ZMU1Ar2{J2Q`}V|@Wjpgr25=L{_Wo1g$3gL!#J`zyA`cd-FjzON XtS<6`^!fMSFaUKGZRJ`;tMLB;gdD?T diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_5.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/GPT-3_5.png deleted file mode 100644 index 5bf1c26d3d8734b14e003dcf1b26361ed300217d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 4676 zcmV-K61(k*P)Px#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D5!p#ZK~#8N?VM|j zWJMK#hZu-i-H;#;#ffw<8D2Z?u!zE_S;@H9#bH@Cv_A}ClX16Qrnh%yGt*=-S>hiB z6Koenz#uB|k0g?IS5|)!9|=*^>15TvN|1mtam*44Ka2_SoI3S5Rk!MP-=6NC2j|Of zcimH`PMtbcx2o=AdTd-69}lC@%EmLDTobFEmQKYe&dt-GOVLtM;c&-7FWwI%=tSS9 zilP8U;UA2SEsx9#h(v*fV0E?Ksz-)Ci8}2zogI)+f+Fhr8v(OAW#G9)UMC7yv+1BL z9Ck|1yWS{h$3n2>NuCI+tE=j>(@}{njLZcR0zrY}z;1%*8$M~GZNWs)#u+>{O@h4M z?&t%T&QJPI0yZgO>%wBK3??5)^W3bZEjxL=4f|};BsrzEwlBgD3_O8BfPI9)f(2fEQU+ZhSP3}+pA=RnS6yDGT==id zk;KaRAOLt>@Cl`0^Z+~;60q5@X#n;dV23odypER)lpAYB9g1MulqEo7YwXPq!X;cz zWwnH-6($`7K-Uh@Y}?HjOXq88!%-*V2}#Ds8&&B0-KST$p|;ZwDmmvW?+ z1DXZ~3S6)0RE$(^TpnvoF>AMjLF`6Kr?~AQ)Omx3Z00@bYfn(V1B8vt7%UGo)#Hq# z((P5Y{6wg?T6U7bJIs*osOh%sqf=pnFoxY}W`0VmK@cuu7R(J-8kFd4VG}ZdOGF$H z=2O^=3}6rlg0KY`&|FaW9(NVc&$^NI@Zv`|owcl2sj{*L4uzh_(gq~Md!q^W`)XHs z|1V&mv7))4PS<&uYU4w1b<5XF1NK=VCbGJ^KCD|E`UL3i7f*u@oT*tIuR}o0Z32`I z>H=80PR;@$xEjL(EVhsn?Tf$%3!jg8y-HyHkGF9+{x~_0kM=V+c6q%ie}Xt$ z-?%?;Wc^g=wgXXpx))M!>uY1H6UZ&xx0F9wmcf9BmIeC^Mf<|{!J*{}UN3xl{>R!lM1QC{79Z*_Yv01_P5BeX+4#o&fg|fD zVA~D^`qXwBpEhDvCy-mXZw3CqhaGzszg_V-aAr<6;1iz;*?qym)BNQzAZ3&Hb!vR3 zkoyIr1mAS@m#_6@ApXFrVDO(G@Og@n)K8JHGi(AsU98s0pVE}|Q_Tgq$U{4^V_+^J za1LQ|on@YH_d{+CDF(JWIwc6g@S%M{us+~6SP{;JaM*355rOu5dh0@Sl})?-jMMQF zf*_0p+85xfa(2ykujl!4wPmo^=cT|orPWfWH(cnz^}I^^b3q&I>-i{C7j1$dj1akB zAPB?d?Ac%2e_FI5k-0z+1VrWnK@bp`3j{$x1~V7*_@KDIn}YQoUP11nvzD8Ysj_l| zb2#)mmNq0=mK=)xv}^aNDV>Tl*jU-Qpxci&*yoJkfqMRDU$+A2WeoOW6Hhbv`IEJ< z%B-dAX6gw-hL12jIleX3)~`%39WTvw@+iOiJZ}rjM&wGr-tMR;FP%=$&s& zjgROTFtZk)JDd+$3qDriSiMuMa-Gd+S$fY@3BtB$eaM6{QEx*z$lx|f-$Vl>+NO(b zsxN#W9R6Iu>jkX;sWy`R!TRImJU+%x-ZccTm-It_f;cnZxSab5=(YoazHWP6Ib?oh z$?5oVhVw1$555-+E#)(kf|n)bd>j~!>4Wf@;ViKG)a;WDNLTUGBW{!Tb!vR3p}D}U z@V;HoZ_{SzZur1jAQ5XDdSFoAXdNV{ZxZ-N9q?((a)WkXz;r_W#J<2xD5-X-G$eHF z2ZLF64GsF63nNE(y?E^T4+X-BX~a_5^bcYY$K?EuqP7Ihn869p;C@!HGfmV&mJvMu`ADO@9!+)_}MF|%^M zx^kX&0W)XLRn?y-*?mFv4E~*xBV|*iAwhlZ*TrIzz@H1CB$P072~UG*`liN5_62H# z1fQ7A)5aTg&6}$-<0Vkn=Qc{X95LrP*Wqcz+ZUKLMuhO=;(Xw-BRQB-x~Z~o9%33T zc=N8)H-m%DNp6$$$$u2+G^#2aNW3q?el-|5;@2DaA8q6C{6V=FA1Y<+JG^@1{`hg` zzHvGC)3qH4^r`J?e8L`zymdN(oZ);+`vcV=PQgY-_f}t&E!(?((Kr|!>+cVrr`AGwJ8S9pV(j_03Gn^wB_{pM1wwz| zPr}hvZk~e!5s~&bgo$MmHCXW<3-Hp+D690-W;noQEku zowrU7IdH+z*B>cb%FYFNwIc|^sDQaZ-zstEp1vKz<=&6Rlts-c9e%Av>2|x*Zo%w> zKfT;);auY?k-u3MgwcZg0{oSGb{@h>NeL%c=Wy17!#?gB>UktT7eE>kbkEc2C~QIy zgwZf&&(Ds29AaL!@#Ok^_MhVRNcp)S_E1LyjeVRVe{IYJR;zku+xAPA#kW6TADutnJ{KWYTXTp$R7AP9m+ zwgrM92!bGJHqy4Bj!zotq2ge`ccb1(FDk58d&}1QPU{z8Q#0^+Nf;_4>#yCXW>?fN z3-reZ9#=!y7C>ublYUmG7n;?ZM$u@vFsXN)9oAY(o*=#aWb_B9NT*G&QW0K;^D?QA zV1q+v5_XbuBxa`vTq{Sr-1c8@Z~9Iv2n=RG_@ZNI4J5|x5J||4+es~v##Q^po^ zZ~ctOY5wA9GhxrOfY@cSgRdZKDH@7JEF}lrdh3>36&Nr}8Ts{`@SO2lo}9Fq4_OP} zkpx-LzNno7`%yatmP$1nGi?@GWPxA}PW4v@f1?j$25G>>E^fxsTfM3VuBHx@ewTjX z_AvlzaH0SXl(RZFwaFSb|vB;gY@+Qqo}jK!zct|jD@bs+uHPY(JxtW8;25k#$)|t~&yLC>Q zGJc{wc3!9A$yq~V0lqEBoAHJ2C?!|GA#WsWwaJ0OMu|c@3OEg+@t!$_FOQW3R9K7c z8&52$(^ZqxCqNuXQ+A5&=j{{{v?$qLU(X)#j!}~EoA8g2K++}7a15t4i(A)z2_$F$ z3PUlRB`Bp7d_8xRlKTX5IE(~8S~y|9Yr6taKVPHuB;Q^`VAj^41?Q%%@Zjm{8T`%SG$sACawE~-#qk94l9G&S zTc8w7PKj*+5CqM_&eR8%K#A~$_NX`x zc>;Y)&J!t!N7N3GlBet-P(7R+en+dO`J$ZA>(W?%S$(6) zN_g?){+Fs3Tjg@hk`slWwnx4~V+Pr@$XYEN-#*Ac>#^q40^? zkM=vSHD62KPO*Lr6E{AtO2-EZsFDzjWTOL2y?%c=(_#OXW8$QHQs8b6l-?EqW&Bie z!}gm&p}z~_X^X6lA-r;!0tc!MNq@Z~CsZllnYk9Nz?3kL7u|+XApIcf& zZ}~5j8;Sl->1)E~34$O9!WM(J1+W16%RFBWaW(Xo#{zw&Yab%4;V(i;q8!TIEl^^e z(i5QHtuR($2KZ_>8|WKSRnf*`yTkZl1Yy)6&~K@fy@0%OD#U%PAU zg74{r9FJT*2D#sV^Xb1_{8ud+yIB2yY4)V_$h2G{(jbtIx3;zrF8s&prCH zqwQxJC;t88pD+LM6W3pRP3Os5j(_s?%ineESC2gZher=wcin+K|9VE58hiaa&p!3T z0}uY>wg0{K%g2BF$VK~Ke&1vFzVOD*Js-aQ&wFop@$nB;AHVRD8!E5fekbW)^~$@U z#$Q}>9K8DJ;zL(F_4dmTT=b)*2j9H*(t`)Sb^6G&SHAq+J72k^@%NU||FqTr?-O(X z*mmj4J+IcL&%gKm2j2S4H7~rG{qCD{7u@>9(+}?;{V%=s{K-4czVz5-U;X`m4m@}C z?BCw=@DJ{J@q>phfBgejUv%k}UzoV?8#jOcpMN_2(7mK@saE%X0!;_P9 z%|>IsGQGU(mR*N#+qS%1gNho9OO3{D`zKG#ADO?tG9AeGO`cf1uL(t}ckd6f_D-It z9R>%NYljY1b~Kv{i<8Rf#hD7UWns_c{NcsQjw7}C#_izhw(X#|I5WAp)O=viWNmqI zuC{l1sWyMK(wuB8)Q%pl?3ipc=9cbXSe!ppTV6iAFyE{ko~$XCk8NLu7<{H$nXc@( zKdbE8RrzA&qq}xZfBe>|>FU1EZri?1$5apmK^PfhWB&)$Jn`V8rpp=t0000McZBy>}MTqOaaN8(s9?JF9n= zH_!9?o%enJm~-aNJ$LRobLM1r+xke0|PUW zxGmzZfeBHU`-o9C{ATyBfd&4c@&N+_5%u!k6#K7@>maWS!N4Hs`gdR=?F+wPV9@v~ z$b8UrGeTwHhZ}&Peo;u^8YQb^tfP;~KQHSE1Yea-yBs8YSjGk_NWb>~Vjuc-_H}o! z#m0~42epWBhqyu;4f1hCljzr}0)@ZK#7Y>mIG&LG%KOMhostzFN-ocND@Tzc^zn(J z^vFodRmR;?%eym6%V}#TDb(r%sZrsv^ull{`*$I|w-&UA%RN*qYvW~|I;2swea zB8BYN(P|Ps(x0edhCG1nJ&|ZD50{HPjD#mbo@a24Kwn77+U^6BQ?k;4ZXe}Dj0_Ko zKw1&}Ji~_-P%z*h??=+g2JLy*y?W7#EG*w%)pR~2 zIXmB30{fhdGGJ4agcE-pIO9JPnjNHE^8Cqu+)vNq&Efu4(dkjn*;Q{<%QGftH~(DG z%Eb-}VcpP*Mcww)6;4Gq`t4|;2nqF9^mJ>Lj4m=Ey98w^4|--<^4F23I&+sV~+-h7q)kGyW}p>uyCPcWbYOq^pM>t{ZdWX?8JM$$;z{s9Q6bv&M zg}Q8sy%bs5fD-MF9`~+Bf!1bxoG#ePMV>})y%~yQkWpz!?LJ?4!z{y4&Jg1mVNxcP z+2J05axTQs%&V4E3S2Aa#x~v_4Qs&G5cAx0FSep5$70tyHG9(2*iG!x4ZeJ4YsM3k zrpE@zVVoM&t%daidGgFzNJC}D{7)2TKVXJ#aItQWGUWGgvKtO7>WRmvhSFde*Ql(& zQ_7W@+Ox#`AZj~OIh$f>Uu{XimE+fzOh=-g18a!?3b61n9HGq@Ths$B{*GP(GKT+@117qiw7 zte5b{FsVRWU(S|kKtJC6EzYs68(o~kOr|{3Et72~F)W5}$HB@SRJOU334+t) z$HR>s00HH<-f118!i-!*B4IX?9*u6eisra=ch{l%fU~*O_3DP-T{Lt->2did@sTTr z1Z!kJ6DUQM4tiS_9o>5f?3`5Qq?GCf`gbLnp>%1}w>Hp9y=(HF$t|+G#_k8y7vi=@ z7;q_ah_l>GV|uuihYffCZHj@BBSrJdWe0b0dFkevx}ud8v~mnurcSQI;CrogthYSx z{doqGS3hRUGu1aPGJHfdTGp3;(S*bDgi5c#(d(3LkzVcAfN~ML0-sg#I&;UWUSJvi z&na_IEyT_+_8?`EJe6<0@C_FN4Pz)6=XbJEs%Q^q7F9}#SeRMmvsC;Hwsw9woUbta z0c7u554sCp$sPJ6Kew?tZUnoXe4rQ6t^R3@j}5rwq6=BbiVtt@iR}BHOGVBG;BKT! zynxR3TRPTGQ6_+_iqP!0+P(fOOPh$D=Gjtl8DOCp$AC~$k}kaj^z6L31=_pbAGXSSBGzvqZ>)d+Za$5mlxQ)gY>%y75KdYy)nyYQ;Sy z5#Ux(S)3riop>N`vxQ3)*yLD=H8U1gH*3b&44BpF>>J4?hRL{p;W|nG!I=?0VN=T{ zb;U}|tLeZsi^&LzIMt?a$^RZNAD=bv!yY4lu9||F7#8|gDzvDo88y<+ZQrO1`#qLF z7Mkd;rmFJ~86-F~^NGFzX_#Il_Bh&Dc7LaOiv6Q76WZynKz}JZ{e1UEQY?VrvZ=m! zCQ-^qIs(WvVkp=sSBTuw+@62mqL| z@#mDtk}BR%zYN_6zG{nxRinajw=W#llD`xPG0me+pvN|z!`MCf5d`$`!q2ydG%pYN zX5;rZLYn0Qzh;a^+Wg*=RV0Og^#o2KgV!Vz`J%oCIq-V!6heZ`tE$S2dy{_v=kd_vHTAZ~ z@@*`AwQ8E0k=92e-A}eD#Vs~qeJjl6TiZ}pA@k}!k_d&#s z;*Lt>sTyc-75Skq$xF6v`UQ<%N`0NVc!LPZ87*ETKQs-^H?-tQxIZpuO1}G3j)@hY zFK>LOttNmdJ8CUCB|}pUR*L9nkT{Hs8^A>JjQ8cS!{qI)xo2L1mqWAU;>(g~4X0EQ zR0EtSIXc!c=$~5`x8EysXp%bN%698r zpEgl|YJ&@A<}1`c$>htAB}EF&O^75a)=bCsglDZkuKse zRY&mI-zsvE>BodHmT&SdY>bQe;P{p~z#$6le4b15 z8Mj^}i~r|$zt~nKNGxnpit%9$Ra8%lUH1Rb>g^7@F~rC-X_GheujbJj>&`IKOl!m$ z9IWhF@$^(V8%h0j{*}xaRLWpv`QkVFdW_Yz^`nQ%YWTnFGG;$=o7`bC3^6*o;a@G* z>^lyFF_)a);C97R4|vw@okD=i-n$0=%k6a+Ii9PgW>DkM)oE|ch{%t1tQ$#>|`@&mI4WmVO?n&MZcbgNaT z?^#P=IKL84xBwiKh^r_y z=)4_pZuw1fzu8)}4 zg#DP$p%m^2_mWLgLTZa@BlAvJOU&=NcBpk|&saf;XG7ChQkwNh5Trepjw>9DG$PZ03l@eS>KuG-ox zH-Hsf;<2Uj1%WH;{JRry_{LM&dZ*|wsSjZ@xr*B%+=B$R$o6lk{mhQllRd3LlIGeW z`E<`Fjl344UqWM6LIb%OtT8XWNY0G7Hv7LwR#6$HoL_{Xl2Vwa0~WEvc25FXF{p%N z-~Frars=#XBeHF3ys=9DuGCC*SZDp&^-1MnlhfbFgrE(UCFG|$Y?Z`rk|N>he>V5J zbZC-B>3)vu8z)hpv66#JqkPlhA*aUs;Ex>{W4H2$uTJ)9&kF)6icpsZV4jNA&=TeJ z7}PbiFQg}ZES9b0#qf>KT0T^ECm?9JsFwXOqTA zH&;*5U)lIoLIroDHjegn)&J1QjyQ^xog5p4;6$))JF>%!@QGvb~iRc2kVlh zCQdqyRpQ|ZY=EIHv7i)hI%W`GZgZ`Qgzvoz;Vo79K#13I2cFq0h^dil!9-tAtLKqi z-!B1$=zTnzLb2Yt&o<=SZ_2sa;O7BekNzs~m~xx;U$qXxyvLIt^0f1Z?(>M(@0Zgr z`X_pP@F-NRcymRBM297$i40O4B9P%<6(bD>oA^|;-tCGoXZnpgUne$->ikMhqztKa zzZSD z4UOx27V3`voa1ABEB*2R%6d03kx`258C^5UztYY|SgK6ZI{)>EY|Vu>uG-Psbq6QU z=Gb}IQ@b5-1@vHc20JD#&)-`$(AeR4) z_?n%da5(X;ae0VykM%D_82C6kRp0;169&gxc}|W0ArhyJg_Y}01P|ox3q8ooux!Ho zsC4Znl}8)LuR+ySHWx#mYZt0E!3=yDjPN94^dGA}n~-rcSuQ18LOUgvhzpGVNhx=4RB$3rtQ)WiWL`w%G|+ zGPRZ*WkN<}cYmIbs%f3CCowfJz&|p<>Z2Ie}Q@iijZbDs}fDY%jZ9f=4-xf}@h2nlRjux#{n&jvDTxTRy@u~CByk}UpCFeng zt+=h5Yc>%k8`+YI?B(;d5bui~A6HWb{uYI_i{M7;r^|n}HW7mkM+$bIMVNj-KkH9U zN7d8ldJcQRDhiSJ0&Y+7P&?czs@^ZR&c5&gS`M~RZ`qt<`c?tbaw*>ZmLm$O8)5`g z*u!!~8?^BL^DduDeI7Gg4!(WryH%GH6t(^J818ubglwLHEo+=7I<)m@?jh;LL#owg zfs5J)b$!7>(pvjLNQ;@{^Ob4-E*-*&>9y_Zf?eJ)zPLiLj~FvcT>1I+k8@pGL^+ok z4rui`;&$u)hzOo6^|3Hh^iB~zOuGK9Z!&t<4qhd4#m*p0w!Z5-?a>Ydp&2#tH$Rq^Q9c`GWX#vZ2;I>3qz35u%8_=srjIlOI@ zOst2S0!+~&j(0yJqzTF~p#KD69f{G0gJo}K!^%QMe|{`vg3OG#3Gnm8DyNL)HTKkB zlz~S3h=}I+Z6!TC;CFQe!0$X(BUxVW@(2$lp+S$uPxyQ~`>naTkNQUw%zW)?HbV)$ za)f-GeXgpnN9Rx*xxOB*0oF*tPOi1R_Yc7_-}9z|L>A~2OVXTr1Gy$n`u%S1`i%~v zfFF5C%eSAIv)3u#oWvh_0-0GT3LA}nmv|%&qKMY%+K`uaQH0i+xwPa=(G~&4j#mpWdZ$ z2AQ=l-e>>{m0J>JuxwLmuvs*NI^7a%!&IWpEQ_LtaBW~)ynWstx9(GGBsnD+m6tIl z-c2*?#ZfESl3~u{M{CYJY0Ngu$dv8~+7=6iocp{le+b4twaJ$lZLS4;V$j)n(y||E zF%HkPV2rtbr4uCfE*bu*f6mc~lh4;PJ`J!qpelYo>3PR^9*?A)X-!COvlw7@VtKMr zfrVb0DLa0ZY3UzNypB-#nXrE-t^Jp{LlrPOu}PZm1`oIevo%H0^L z(^iifUq?vkFv{?3i2a)4_{N3j?Zbo&EgFxlO-?Kz)Nkzq&c{d9UW%&a2 zD<9vaw@x~X@_P?wANjHuC%QFE#lk~tUlLWoHwB_GwPWAql%fYWY?k%D%;&82QGq8I zR|sOO>;)Qo=w*P*^ixBa*@iE5pe_fZ1&oO@cj-}4kHqLJoD>oJ2Xe@eGlcJob31bof&R#abhlYiLc%Q2OL~<|` z6as_LQ6zs@5ei%DCdEC(XLZ(IC1p9qlJKGJ=XX?fBvb$E9pI)3W{YoF$7c7*Xg~fG z5^rYu*dq+Q9HS_?i?o}@#kLq3Y3==aGH%h=3+1va8F?1?5^=QF*KBLjYwDdRVXpZax%ORRqd-ic-R?=1B#*gjmQUWL@oQ?>83(H>vhQxPZ zP93H9p@e+rsUl{HOK?c#OF#ig&$Cou?d0$84?hwzE4F+(S!`n)REi!8s_4=_mPzlK-F}EyC z7%lneCLXEVGmlDSX8T-phN`;v_ku;&bZwATf{}@>3y=&5*y~Rd&pjVKge#YO{Pfik zD}S#;ZrXbGd-H8s`!{~o7K;Bd^1qUdZ8nj;serV3S9xAT=7>fg!XkHQ9%kxJc=s=} z|0~WI5U0=E1|}}ykRB9qi>aLbVgq+vLC(@)Sf7N`<$rAdpCZnJc!u8{z$iKBB7OK{ z7Vi(;+^F#mNd@(v5Gu$M$ zcH&R;)9&Y`EhGmfqW*uc=?|%a{~j3s6M>5ry}jap^7_66=Spjpu;{HupsN8kXt47F z!@S~*R-^)-0M)rszKm6|YHojn6a+{)pi!$J-f?Hv#s;0)6Bq*~$sr z6&^J0pX5x|b;x|oN$$s4u1C4<`?(gh1BJC#FF8tzKIXlE7k0umS64a2)dRbrSBk*z@!+lbZExERPkt@9f0&36Iq_j)v0M8vLm%J$ty(y{* z+ZT7R6r%%s5sGL9AOo3S7-o^&ioX?^kao6F;IpFRWv%zDSN2b4n#dET;gf z-h2;J+pbipcrBc=zlzjyGb5LL_tID7WmwhmI{`tZ{7dte%d%1@Qrg}U{UueL*F#zn zmg{>_!cxj|gI0b@H{dQJ9Z!2QA21%zn}A8JdP@m$1dOYV+PC8HfOy}S;F#afIYy8eGx(P%Zx2=$iNS_`VK4jZB@5>#x zNTqqLOou0wd2K~{W=b|w+4!iYS%dqjYto(rWB`YLC!I@Sr{SFYUFw`XmN-M&D6dHpT+=-sG3%foNS4MhXlA zTtD;EyxWWC2L9X+D8BWlnx=_{w+CDo-^*>XrjU{jqL<%QDtr??{<$WG1ML z@t{oS|5S%G-!y2$G<_SCbn?brK9lZySU{Oid*dyF$0xJSV_#dR211>Odmzr+K*7-_ z5ft&%{$@hSbW&LN!O73seQvgM*U>Knqa52xE%h~qU65c>S8B28keCjHs>GeIw_9{%1Ny#U^+Lg`ksr_C)ms?#ghT5d{YThi7t0BB(}~FJEqs zps(S+w-c_$u8+$`C*@K%bBq+?x4p3KNX<6rL8^6`JId2hP2zH^r{$Tj6!d8Aj=%uA z2U-qzd3Xg!cT+DtrrCS%t7;EP-Tqd6xW#{Tcz8D=@wj)(`RIm9E%hU#G=Anu*tK}% z*R=HXoogq`KDY_Ia#$BtzWgG1K*u&YMSNS8iOP)eHVVz-+Gweo>~ zMw5vpjsu>38?6hly&IVPs~wNE);53!U;-$z4WwpS2|`t@w^Z2S5ZDHK+I^RH?XWh+ zdk1u)B*+d9Hdz!C3GW4)m{D7FOw(LLFkB9P7#5q%MBK$Y*Qt zT}lIJ$m*%WmJ~Z-s0wJ16DrxS^qEQ08I|B46s}l}H;xO6X~=ehV4pnZU+cQoYy`Oklp)__~|q^o+OQHfFFfc=!R3pdW0^FX>D7T2*xt zzSw2-;p`?Z0*v5JodF+n)jagfUPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>DA7DvDK~#8N?VSy* zTt{_)kJnD(7$5cxNDLi64R!=@okgJe5Jw73TG;<8FY1a{k4LAXg& z9x01yB?w4`CJic8f)*z-NUcDP)KVejjY3dLMaxesTDA05rj3w*($MoWGxyBBGk5RZ zx4Z8r^U2_Yh^bOEwXaq?tQLvfyIKX|gF~ z9EKR=+@>JPZkH=N$;u-zbC+OmhjZ!l=_gHTkw|G_T!701^LQfb&6^Fy{JaP|Sl}{e z>Q0Y^w!p7F(d_iP{XP{KG@DB`v={ZAg^*4VRKr)&L`S)d$p{NNB?V1wnK1=}@>%TU zQN9V?K6TB+r$a5einI&6Y#Sv9SUJtvaeZDMxJ!g9k`+5=m*^Qxre}|*DuUBdW|^CM4lS2g$+ktwNRZ2o9N~khK(MZ+ zqMaDI^d7TTl$WRupX0dW$n zYwV~NtJ^`a?0W6w+Jd|qpV^w&5u4JwmdmS_2t_;iMz4(Ql~>JyGiRkBSM`XeiH5pu z9nYMfu0?G*W=i>|rBXI!aV1S}bPEet6y}xp z;97*&5}h7B?`e1Xy{;+C(u5`E)h$_*HT5qf~$*mSUd&_ zcgcz@sC8{W-quZLPMMMr(F}NkE6w1z4CMq`$H+>W<_b=O|76#UnZflj8m3o zR1bHcb$*o{^LP=2dR8LqWqc)x7-G4o_32g0GC@URS{Z>mX(_TV7r2CJzJXlWoad| zgHq;yK4W z+(%c7<72w1_R4(AESo8d`g)S#@qtH=$h+NMr%T6*ys{4N?4c$dVCYA?JKsmsT;}Hw zA~CNb5jfS^=BX_#v!i5F@8v5k+Drbh+#brTiK1?=&5rRdQnq`kmg}B)W$Ie{S&KF7 zP}%r{?y){%>t=S0j%R4sBXL&SmwCAiDvQ5KkdNoBG?Ed;+C65rk05h8F!K7nA5F(qr;@N zY+{*#Q?|`V4ZUJ+!|Y8<3X#RIxhIFt{02$;MIsguc@}GPJRx404F->EIT;07w76RG z+V($oBBz_8<;2p5vaC!`e6^_<=TmgbW;*b(nTWztM4Tc1AtbmO(# zrC93=eOwg^F|bZ}EPVenP;N^3c3EQlCpYnb zp^szn$H{ekjGx70Qnib_u^jS8+cA*3*B9+4Vf$`iS*G8{>4ov)d&*iz%YCv~#r(n6 zrEI(7$odtHgJR2}QwPqR-FJvoFwfp}=InvTi`h18Q;)UwQ?N0K#nZE?>Z*C-F^ms_ z(Qnh|Ed5-OK~J1WC6L8(SE+ha`WPEu>5J`q=3B8Xdl0F4RhW~w!DXt2t>J6TXPakj z2t`lgrOSvIqC&sR9iu0SIUPi|A!Vb(h)Pv`yWvoF|HPQ4}1DoDU8oXLP%IGuFp3`eSqk>+5!7 z{>0*}_3_R5i75ZIekK&$F0GH`HW9z?L4VraUcb}n_PY5bT_tWs4)99WtY;6Mavz8m zVm7HP1^SqC(9cTo)H;5gGluX%LwnVj6?2o=FUpZ(VijWJsQQ>G6#MAfQg-zG*K?8; zOqo7iAZjt6hrDml<5RcUSgM#<3`O0#mGnC_A!29m(Xtf$SxEdfV;PIOs924~{ut1H z#>$mgz&x2`tm?KE(;tV=HqW-zNm!RhNa&olQ1rQ_3ghr$Sz*rMi9}+8xb>{Q?)I*lfoxX4Efcs|VR*!4Cpu<0fMf0a{oxCYXeTRJ(pThIG8GaW704CRACByuf)WyF0X z$DwWJ(APMKvTyn**BW@`K^vhyVp;oEXv^3^k-HIw*RS~6<))>sxZ(QC<*3hzL}G-L zMmAaRys@MM$O0>re8BBx0zCyam0#j)|uNYb-b!l?U7aGJlnBPve7dC zvW0%;&atK&qX?_%vcexfzO`U3?)XLoJ(fYVoNtwy3o)&u%UY+%JUTvXPAsb%l9PU? zN3YzA3h`%DZ0n(Y2(>Gc+6?W_C>U_uR`$*Gy^8)q`s>wg<1rD7`7rrt)Oj}GTcPh{ zv(E7{OusxLxngRHeymO3oMiZ(q5(HK4R|_L`dFmcR@Gz2&$B_FQ9^kZ570h+RgO{> z>zM;JE4L}J64bWw8as@ytb(;E&#DLC-Zwq?cZQ6eZkOVe?ZZfsrm?B}CKkH%)icSV zTF28@I*9q*B!k@S!+FEwBe#5Yzk0oXuZz8->Y1NHsIC~Nr}o9Zh<*p{VW8>jdbp`= z@;z&bRV=oVvG9G!@bj#0S0qw5*2l5<ldydk zbqDq)`G@8E9B{jIQ=O+D7J z`5BL=wyEptd7>G{2f^y|(*-YAD9ndv#J3!|D%qy=F+RT89PoTAv}NrKp8sfve$JCW zjur74El8eq6U*vROm&$Wma67aPmdbP>)_r_*EM0wMu%XZ5U0;Z2z|^TPL8-7s&#Vu zPA7+H$ZJ>m%;SxPU|&LYkq#b8^j_$TDNvx=sDpuOdp7g-5mm(y9*aU-20LE~uU*0@ zNBcMyf5JM}&%C~c~!ZDqcB`pQotGZNPCxIbV-C>>9p5%0T-Rq!ELuC6F&?ghC0 zrh9HT#hH`Wa)v$d_(|S&L~QD@E{u^WREejjqvNI-#s>#yqO5J5Hw96HbH zTwxoEj<5FF_up&FIh>nUNU<)SSB(9*&%@+d6vy%`qI0=KQ*;PF?s#r7d+a#L)ALNu zm>jBge0@ac*{rUMuhq}@i{)65z8Ga8T$eV@f-yIToW2(*?^DT2AC+h$UpF=uIUgKW zPI&Dqo6$aw(I3$ntnalO@h28%Phb0~bv*q{$h6%qt*_-7Uf+ZMv^#Vm)$8{=?75;s z+zMGZ zV%Fg-wsP6vG@9w;%2Y%o)yf|)AhORq>A2{A={aHj6q~WS5HFhL_Nl};ot)f}#VYDg^yX~1UvaV@=!)jqXsob#ESMbU#Xp-5(RezWR!0=F zzWdbm!N=X-^Kylf%lHQ5h)F?yuF&Hy{+JEb$7~MxzUj8K9qf`R75a!snHx7*(f8Sa zqRP*~H1gS1>FScj#$p)i)~%+`=R@x-H7=HdKMRS!W;|n2mn>G}u|ImWpRsZ^hMy;s zj8)yXV*2Cpd8j;_NF)-8YT&BWBpA(712r=tf{k%T+e7|lkD?)dm+k6|_w~5C*-JRz9<)Y)V7^ZuA z%gI$v&^EPBCil_$6@9RnUmc{q<}=m#``n(*0=_}_8-`P+k5e74Z2!7&u8;PfVCGw) zEo%n{j&JI7rsW^?=Rcj_?(%~{Q$_=Qc|K6;b0U!#A@%aCe)?9GkR4aXg5p+?i&n~XUUFTOVa0uaYdG`E3r^qOOiy$tufVQZ)Vfi zQEhoBc_kk_Iz7{I`sfb5q9W6wDvW;(p2-nwu&uq)eZ<{a)m$Gj=ICR-k6Bxp9dC4a zjjZ~JNEr=a6=@;eH%KHVgUPdtjLii?mg#(Xb#z9n%CKz>M)$U_s~a>EtOJ{YO8kLaWmO-Kq8SCEu+b^Ro9P;l`j_3Zme#~edA%6Ymp^P z`jPJhQaR76`*`Vh5s8qT^gBJduZj*{mEHQH^-hL{%Xgnwe=HFNT&GyL>zGE(e#Z_!`T=yiY$P40#P<0|;Jh7CeMGAC z%p)j6#71npEog_~^Dtv)mhXt$ESvd@#JZ+}%7YttWx8O{7xs0KqU__!z%i$kUU z4sdt$Og=tK1nl@iCfkq$6^m!V+^8nPsO;1w)-KI^3cG!;3>qAHatN}dm$Q|JbD*O0 zGnTIw$i&8r4k}YHsFQjv7dU1Tlge=}cC>6=;izoCh9glb^eZNh<*N$B!(YLljDD}{ zmb0X7S6bwoyo(Jnr`wFliDJ7>fo+r3)b^wF*}3o=j|{kbdky7J40KUeX3^nNtxt24%P*(N2=r=@5=PU zepYQ4u{M>M^^N`TxcK&sH+)*dG8eT`X;<6HU4W%mY}}xr*=?5Qrk7G6t-{7 zbH8G72;lcJ!&ndW*W%Z-yXl?9SV`0~Kb;bj$(MS3Li#L^*jE;@g6nXESbuZ=4~S39 zA0gDwoPDKFdy4V2&J=PA8j0RcnzoxF(i2FiZGrT<>w$u6>~;~P)laB z%JFMt+$5PYtxI={aU0yPqLb7@ApghnWF((cOZe_?;t+%V@sYm`#)NcZ=E|GN+%#DA ztePp6+p*-d90hX)2HB^@wI|^UM!@6jTP^52(6m;K*%@HJV5giyw`2VsD7O;xPI4cw z*Ol9~ustOjpkJ_$3eHb1kOo#^yB=bv#6AXH*U{~|Ry^{~5i$EE?Xq=?)ye5QoF}$G zikYcGH7Bqwns<`>4D@r+;uE5-ox1jsV1G;O)YV6DQ}0E}^h=!1RkMWc`#6($a-m!O zB+8$*i*}F_{UWh`)r$`{iD096YNPue*mAGD?cv%DgDcmw2h2Zp=XK`fzQ>=IKlZ=x z9&00aclOY!1CO8NrEP?|emc38nD+tw%IzK$2i)}42m2!ol*bs_zLl8wy6qA<69IBo z0}RtH5mSt;D?*OcI)2H3kF*l=e?Y%N{>1uJD^H4_eLB`X`vd$5P`aFDDH`M^(ZNh3 zCMU=V-;`)$0axVWIKHtUXUEjTF|%uX>8j~dOfGh6+@0ks1K&r6-Elfug_ynr_p7)^ z?Zo7coE?E`slFOeny(!ner~uB2d8f}U5!kiYv}7hWz`aI*IM#R(QzRH_OqG>GQdi* zN3^w{nVjugi)>p|JUQcxdIa1-a`ufzSsX&-^c`Z|Y!KSDfK_M5?sGa~Nwk-Je4IYX zL-x5@+g8E<0BkYT-<Gf=G(0uhbUB;e&aaPjjt;hUbILn7WQ9!#vtBx`Z@(@RUM?&$ocUI`}h@D!A%UPVy%VO?2KgTq^mP;HGV}j;YngNMKB9U16=;f?j*2FG% za`JZ9T#e;FlB_S&7Uxtt=4D*f#V3WWRL}XjVwbbjF-4U=M(0%#nU^_CMBOA*H;HnL zY#b{wxL>261!OTc18T%Mkw_#Gi6xF(&bImcQ+Vpmol%2wZ=s}4!=8MyWQ*u-H?mB0 z-tlSPd}+3eWrWY^tdG_0c|Nob(p`+{liVlsBI;+PU1v+_moce0Xma+mO>SVI%NenG zHGuMI7vAej!^>atz$f!3`cO3#w+j8rdx_gLCKx~dCabMLpXYb&T79utbcp1}D0js< zRKcG_B9TZWmJhiP8_-EKbY}DA<^*WZmbg>)suLrhKIzn|Ptjw;J3VJxxf69!ZYAd3 zcF5pcC32E-ecaTo6TXJqirR_jM=7yC(l``QEueP3+lYL@u_*U2R&4WOO94t84BN<8 zAgzAa{-Sk`fC<08+ls|)V848OOhA;UQoC}w;x{+YpF|>&NFp4VA-APGBZ_h>G4J;LZt$(rW-d5&^>Z3-D{3dAAEm_p=;9DTv`TbyUWD*!SI{>E z8>O3-L03>aZkIZc+Z)hFQOH6xZdnP*_bXr@@v2xv)k>X8&Y~yh64lezJmb~loJb@R ziNq2F&tq;mOJ_WVJHN%`@z2h>r=hgVoqNX(os^|f@1eYY0$Y6eZcppxo#o+&?7?$> zi9VA(!Op6N{4CfJRmSSm#{pEoqJ-=tql!ZrJXJgo)uZuXcu1`Ek&@4J2K5Q+zQx7B zlI?X}_Q@t~v01Wj?Wu885p_QXSQ58Lzast=`y^;T1LG*Tsw8(tybWb3)Zs)Tkw_$# z6}h{!FKl9Ki@Nd=f@>DFRthX|#=!*owJ40cUv=%T)Ho*+i9{kXS<-Sgkw_#G%azlo zpA7ypyAz4T3xlP!oDDx;T>uli`_=ArBHtsb#{%sv*4@5zS2D4J8F^kxRHNONY=~*G zEd04utovtnClZMl27o*V=WUyUpHIQPP58qm%r@bjm&0Xqa5cat*mD;g*$GGYV(~ZM z0I=Z`Z~ezLdp>sc_kaJ1zrO`w6~Hgzzjyt|55G0rB3rPADmAvk>U97Y9EXjYn($kj z;h#3cN9N#H=iufh{8|%kX~L=|G@9_~Cj4m=9%{nFO*qzsA5Xy_Pr*fNVM7ysxe5EH zVDUd`0pm5lZQ70>|cvY>q+x`(Ppsyi;lwusB_6?SdA9| z>)|I&cw!Dt&%qzf!TcQj98JxUsl(W>{cD>Xg@>@})(?<@nbxYA*31>3Y<-lrnAHFm z^S=$pVeQ>b_`OYV{w7%6ge%v>6PwBMd+;Cjon2!6O95UEaOH70|Fumxy$PP5gG)ES z+c(1{Q}D_u`1TZ4IiK_!uINEpoCBBQ9M}wp=V%7v97YcT*uBeE!w$fIhmOJ#YzoUT zOY$(z^DF9jjxi+9kHdvQS3hW7{X~hM=d9ko9ainxNo4{j&r9!y#_w*z!GX7a?q#s* zTP%MCz^gg!Y~}JcfIR@O0~%0D4FEU5noD8L6|iOR^`g2vo&Sm!3gOV`6x6C`6`r@q{T%h$py z>@+y_$X~%DkC0zH4c@SBE6xN;RM*ISaPfTQaX2>@&#N}WRaqD@hL-@m7PAw^@cB*V z=*EcQp?#G(v{x;SLmP(%`?Vwv?K_T1MRe&^c)EFg6TaDmpG-lw3H>>EstGfgoHxYM*&^ZwgNegdZto+UPH`k#;PI_JeZ?t5_Jn@V>iX_9$$>1CCuwg^HUuUI?={z`7mV8arXfUf8o2 zW@iD9;ayk5o*l4b56m`TmJ2&~!tUL$3)Qy6H9O#%U9kIoux=KPZil;Wgrj%B>s|u0 zPI3&6qK8Le-8wji#%E#2PO5^hyYb%<*u4k#u7aJ{VADUp5N+;;Jx5^Y5mLY`vUex! zT?e}xa052IorF7f!S-3IwR;C_KLUH5&!~xR@1R|~W?}Ch(%ea-!cz(p(~CI2Jcj?S zd-T@p9z8I#5vDidCbSv{QP_I_s_72z-3l{byS4Rvqjf%R9=!g8t(hOKo&LhP(=FNi zSnJ+fum8|_tt(1Qt)9jSfO}6IocY>cwEpj+=?BQvbz2X%E;!u!%1fqE3pY<3Y~6Tj z=0{iBz0i95shNM@ESt97Uy^pGA2`^$_we-}d=1wn%dHO|Ze4fP^g-Ht-PT824;^kz zZ_hTrdbsuGU1sx9{C9Br0dexN>8}B8LSq_Fm+;ukhJ(0y-LtE07f5I3_&L)ZYN>TI z=}bTLA_-3GPN{pr^hWOK-LiS1m}=cCn-`hA56b2S-h|etu*;8K_w2>fcwFM%Ka;(e l80F6bMqnB)oo*cj_1*I41CG;9Pf+zw?385tP zE*&92=rtkX>3#Qq|2ebg?Ci|9JA2OVe7o~ePe+yNHp^`i5)vwPH6;TQlIxLw`dl}! zk&uvHzndoYMJun0ZOH%Vw>;Fq-XtW{U4I)X!o3_q zLc$1ES5kQK+6tR1ep(IgZGwr8-* zu6{w6jrdtt|BU-*)~4zRss)kO)y133Y4>e1pR~*+q2fPv+TOY)WReJNnR{+Xon+Xs zL>JMp*D)@@NJ?(7+Nh@1+e|GG6_6GEl3gudB`VnYjd+8hj)i;hNTGJAsVa3+C#e>P z9QDEN2_Ucw84}Z!YfLf4O1I*3qVraVAA_!lN5>5TuR}V*@ZB6B`tqR_hheY5;T#kcy)b-z zICeC%e4rtDJ?~i~^Rnh-iMl6qeVY3jz(g#|)+u8-kfyerDJR9mqUu?A){2=Pw@0q9 z^ZsKF+E$D4rS5-tR>*Ze?N>#=^EhA{P-H1t`Yip8xtvWXYgSIK za?I`CM6YRKPk}KL3aDmRkC8}i)J{?YT-W$8f0H}?b3}KBP^tl-$?Buh=tBhbmBMO* z_MD02a7*Ws9^EK_B$vurAt=0>u}qmS^@D{#`nlS1KBuFRSPQ(ZZD9MC!_oei0OEP7 z^;XIGO=-&9vUCOUy+xvyHH!+Me6lF?le3!82$3`PL##)f@cF$Dt2yi}4ouNZNlod1 zq@}4RzB2LNPusI(PysSNu~e#X-?0GfjGe|sP?9_EAP{@AM(igdfcxM=W!DOX?M>*N z&TGj#ETR5RZLYIToe)OGIaCq%%IBR+Q2(OyN2PU~V7f)PZ6oVpbndUTwO2zq+_o%U zv)qO)w*w;33Oo}JTb|W&)K0f>3$>(h1<=suj_UOB*V>Lv3bpOVz4OiBN}uv5*@1j+(uaAV@Z?++^{!zdX&egb^XujAYR`q zQ=da0ivw>}9i4AuQh{kta2A74c3Vwoe^6|f`Er-83d8&9E>_djED&6F4IeZP9py$| zzT+-{=_|RKsl0IFxWCDi()bCy{wfNQmu8DPoSNd$_Hv6=kibG;b4oLN-A$fs3zp>{ zX^)f&Px;J_uYBS66AYwnwf*!0SZC+;z&Vn<7xiTHLCW(7fYuUe3TP~nU*a=yyuG^q zK{pK(nPhEz{rIF>nJmiyM>^64WpK5dP(K{8XSL9^@%rh)wRP2GV~SNQSyX1I`u`W^bRd=SRBni@Nztx4) zQvnV#`^R($H1o~&R8BwJP5*uLQ?`w^eI`~F_Rr2%;0_Jx_^}lLi^-wg(D4@8pr=lx zHNPw>;6UWJbo}bPB-B{+tG^+$3NBt}&k}MD<-uK}?VCguQ;`vRWq(R3Zux`cfV45j z@!=R&liscAriqGzTHK*mx>#xpHxI!RVJ&1JnC9lD?J{K(;*`ptJi)eu=38nXcdU0e zJLAS7#-#={rQ(}xfdT^gCTE!$FT{h(w|K3?gpOiNx1TFJ>P!z)%{sL4Z=&w99eSXB z(#Gpcg`gNF=3^9X0p^H7cyCN9RI8!siml&i2%vB5fEZ4xU90~x^QeG7a{;TgnJY36 zb!AaCQy^ATN`S(!uEq{?tpHTRkzFaOl} zASd%j2TQYnA79b|qc40%u9LrdLw!vGBB_b*gy_$@>Zg{h@n`a->#e)7qgQmzdCtLG z6s8LYWxV_sX&6?}M@Vrs=r9CIT9ksMCw$E|bv+C0vQ?I(ceWnd5#K^V)}!MxRKjPQ z_IVzqbz@zCO0RR(?B^~NIOLA!Q-1f96W-_L(km+b@@oQa5FPE6qhg3;TW!Z{3_uC2 zH)~@u;auDQIk1D!J$|{bMt5;5S)FmP8j{KWe)Q=Hr{|RlNMUDA_vLB6n?)jmqXct1 z;^7m2iD5J3>*5EI7CFsRcc8`%6%*H-PcyRgl*I!%etB2l(5U3gjzs^UF6`88;NT-j z$I(LJhsvNO!8NbW0}WZ_tmJ2b6LmB;+EVs&cb&HFRqgf-T}}@*vI~^@3@zcIOgasP zrr8Wt^5KH=4@yYm!`F??*MMA#a9(#Sx8<FxYb423_d-Af$klHXedCbe&zJaP=>PT{jMt^n|Eu6{Q|KH53E={f?K zBDp*DBKbW;_bUryI{Zw1tHgz*)XQ*? z&Oa8<_Ir&yMdq~_Nd<6af39zy$Dgqr{vRMqpFbV9?38MVW((8?=I7y#IQ#S|k4==YA{kgI9I$HuJe1w=y9<%?;o!eJ=*U^XYE4 z+SwnE$Aps4mztb;cfWJ3+3X6l{|2||INYr_Og@2K@%l*R&A*$n?a^>>4_nQi*81ip zyfMc^S~1hq6QA>TkLDLLq_XS7aj2?wBE;*foygOL-c5*em7&N!y*jnIZ|Tvb*GQC| zSKTE(Pgakz9yq=p4WjU<1M$3c>$TW4hvVr8c3aS+)4YSX%lQ?~fxgzQ(c*L8;hb|8 zB0|PEy|HZiC1Le-YNrUN2!bE)@9TBt@l^c{X$eWe+)(_}AKd~^2S$nByr-n<17rqc zLBj^R?e_yDHR!>}UctyEtADx`1sU>^l16851=J|PRvNwV^3}X^f+t?AyHAg8qe@L; z*)pQhOC-l{-n<;Up}&^M!8dEnGo>p_Qs#1%>}=Vu{6nDePQDGD-cG@I{Zza`y58BExur3a zK4sJ1j53?Rv18Km9Bv9REL&DcmJTa`}XUebqC{4ngg|9eX_6;i(!Zp>F#BY$lc%lX$k~Nm3?1=C)ANQp1yLuy-9v|UV8^3W-R$UZo zjJrJ$L}aj6Ui2bdwz~GJ83N**K&xW1nNLOa{pqWA$0`riE%23}$I9hW(z9)y=Kf2L z7QpL}8k!?^n+*z5KVIN67@B6wtyxb+elj(T=C7Qb1x`MF#7y>g?7y8<>vjN7h@L7!&wdS z4-RLoymJD%$9T>^n}`PG(&0RlA^I;3FLmT27Jlx{7Ug{81YsC(`cWU-xnM>em5fO4 zAK#dhDY63A_2&7v>5g6V;o9ak@z}@0N<+=O!5_+V>epwxtk}_;30s5Pb6!Ej;}w^a z0|&>4s~r=%iX3?jxP0mB0l%_|cfMq>Bd~6B)5E^TqNqN?R{B4tfnl}22$9b9h4 z8}ERon{emsKmg}yHl9^Zc~+)i%BnP8y`aWUU|_nQFn=m#DE^woze+nwg0;;qRhK^|vHcML_IrD=vh+}N>Ydb0 zY|z@R1hq9AZuW~c(xQwOe#k&+1bZf74E1SH1pZU^=EScmMK2e&{neBpUQ_-yPMe-zD`?H*oQtRTKGMN_jmYByZ}6hL z%JzL415(fHp?E8H${En=8YU{2>5haA>Kgj%VGMuugbvdA-frc%m^_M-wZbc$uWKpI zIG?tNXq|e;&UYssMoJD{BImwi96N+~4ml{8yB~OQxL>YG_V8Qxy<9iD3_W%mN(y%N#jS4lEMV zGm3uJB^4whXz^R_AH*LdSjP<6;NED?Kl6p(J^e3n0zjeCDH5tS&8|ND~G1l9M;8@zkdHp zFl#&a---W~3&|Q6P08KO1k24Id`!*w+DSl`v`m+D{7}d6mt;PRo&<~wh2#t2ywp}h-*<*HV9i>KtuuzxeW0ehH2ZBf^NC&P z$yEK90Ec@;urlfluLWtyC(dqxQE0Z#mCJ6OG|E@GtcLz!47S#zcpyCRWbONfBPn5D zcz?rbGvT17odHPh$fCnYCOGN`xQCAq^d(^GjIRdSzH=AdrSqc{|7YE@H|`FH94gRD z5rrY8=Vpq^D0kW}$Ze^kzjF{=x-(?Ww1=G0nLmg#ClviITsSgB-k{r=794|zl3a*_ z(Wo54C^eZ)td_#=9d>T~C&8Q)ail9-IAu=43#2?S^>{8+NRaV*w#p#xRmU%(){v7( z;i1Ie#S)L#&E(YcPO65!{Bjn48@gNOJN#@^PXh94&g96V){2c~?Xy<-iAo4r0Lmk4 z0UWDUn0Fbgp_ObM6l{hQ*qfHRpTuGO<#}!U`zx^n%mruD7I<0;=TjQ*m|xBz<_c?6 zI^%m-8T~r4c(_iiV)0=rbhQ}>{KE2%N$m20gq2Ug^zC@X-cjb$vKg8=;+gX{WBfBd zBwx>ZN_gI~9klGRWQ%mQ`!@Y1ex`YBa&N39Y*v~9LY_e;!UdVSG z@v!m{pl^?taJ0ONPB8J^APV+LMWpH)PkhczgQ>5BH|W=1;X8}+v%68A(Sl$JVBqs|9>#Jr zuv2(oe`$rY+*?I{gUqUoMqSo!t)I*5;x_L z-VJ8LLQ0$TP-5*%)Z(QB`SN4I-% z>jzYr&5_dMB)$h$$IwJy=VeIuKAvIMSQtvDl0D$iiKUc5Z$e7&m&`G-&C=SM-D2#Pb7ALF~60E~hIr)DeCI zw|+x2pmI(~sVlU{Rs%HD`5^)jX!cb`ZRpxUubGZt-bvzYVyp_hF@knJom zJs<1#Tla|BA-`Fy`p*>#@(EUq$`clU7AZShXBW-OFobNr3X#QeO+uKX^aRs3@sQx~ z8XE%`I!$Ia^c354g7-OAha(&2CpnEux=kI7(MF(6!JL~thqK9(KfQ6D_+XNkX zle~0vK1WK-KqEuIpSox&=$=jh!yb$;<`qxe_x^zh+UIrEhN3vS7eZ}Nmb>7cFq1me z8$^_JAXZLf<80&6HxD1&?CkdJCZg=uE@S*gkL=?5*5_tP#3Zqe!vfHfoueE72!HWV zdNb$_q7F_ubb_Xj3-Z~bL`82bhntHcT6rUaIW%GI=RBlTH9Wu!b#pBXQF!*xsX6O` zbs_HBx0u(>xQQ@+;5L{QrrF)-hS+0K-4H&TJUEb`2!VFDIAiLU%o_%V_h6#UB3+vY z`km853D*|?CD{Ln(?Qb##4rF1WoZtyO?sO75m57gtST)yyO*9LZDqAOVgWHk0>p>$&X-~yIqg?&@ ff6&y%nM;5)SICE&pMbxWcoKDG9iPx#1ZP1_K>z@;j|==^1poj532;bRa{vGi!vFvd!vV){sAK>D8vsc}K~#8N?VUf5 z9JzA9SGf2APAK?f5by%U`T&EAHa_6a*-HkB>|6K}=QajXyk9{B z+!t8*3oM|(*#%zLKay1>tGn6rU&|k)8FqECSS(hv+1)dyqszWOKy5c0oR=0b#pNzTTK&L#!5 zODYBl>oBpb3;Zw8p2e#2G)>C8Q*j0@4x)<{>xV=uyC_pW5whin@Y7rp#YlR+_Rr14j_sC3~OpP z8HZCenw71fU>d^HIC6y>ifnT}yCmFX-NjzPTx#c^gG*o&UfF3|@gq`-pY$)&MmE{2@S(rSk+Gn97r zW%-0)h=*a~XnOPV4kqOik(ZYwhh(*Ky@d%;4k!nhT&2{?yv599Bn|PpX#wZCxSH~6 zWbA}R{mSME&H>2;c!RXyg%c$)XR>~!H%VfVym@|`HV87OEz_%USMSAHnG1l35Gm5R zL+Y{!rv12=-a>hkrz~|v_9jAe+^)7pP}v5TEEhV+3Qk!YTKf={y3Df=pKKB<&0&4N zO_y1?!}M3%q_W^zV2Wtyy4-7uZP>xlpt7eSGXfXn4Qu>hqLWaO_IS<1qc|EC7>O=C-i5odz{|bsUb5ST~gEz$VSOH|hXb zb}XPjTasXw$OSa*<+7eLkYUZow6q9RTSTNqEDmd#nUYS}OA_>fG4@G^i53`?i-odB zorALEvSaV?XmfUUI!#=`m53C5^gSu<)ak|Qm?kauCeZ=hNA zXL0ZL#qLv3W?@c`=&+G{@GE2koVelCji*bxXuJ9G?hWKCy>gl5*UxXS zpNiy;avJX`|Ckax+JuXp^h*Bec3BQB##?COgVKQ#$lDKm(51(GVVu8`z z;Kl|VIZG3eMfM2B9%BuL0~$l#|xJPbGR3fw+_17*ozNLiAj zgr{V>FCmZ}=pS1K_*FckcJ*G+PyTUTr9)Dw%X_&y;xlJ3mE$^9jZ@N0D_yciZ>e3a z!(K^?1Y6b;)DIbqO@~1cPn2c^CSdsY{{ zN@1ZgE2S2p7RmLvfDZ2!&8QPbHhL0?_RHL0xE5Q6*DNLx$l%cj#lvdH;+NtWaDw2xgjX0vnA<@TX96kHJkOde;d0a&u>fwMf?6}Nrfd{|A#^*m z$Zf%^u7xYGVOYc23U+~USbI@xT8H;?U6wG~zUYA4uxM6VEM#kCk2(iMG8lqQ!?4i{ z#40>mm?&X&!lMNR-isqg>rvKZ375%$Yk~K2uD}K@((pfCErha$TLewq5n9}=B@4M0 z>&S{0jt!M()a9hZdyOZAwl6v%gQK3;BevW*D0{dctTRiOHGHLs*Pd!}jc_e*Z!4Y; zbI^6?_~Ys9;zOs2vv{I$7QiY)N}ad!HIwqcqB|b!^a|7*UaZZG>M)b508iEW_<|g}Cv=tq$A1}7G>~EUFi#=eK0H;qgW^znT7fMBWo622JnoDUC5tp-o1Ocdx@X?x%m;lGm0gcN>v#Y z9)??)g`<9$j#;kjvOX1&?2|(b&l(JRxrI((hv~C0;KNb0CadZ@$4Y9 z%rK*6eb!;j;tU(S%gW;|_5r0uLKA5%tm|q&6J9uq$9 zEUYgZ%Kv01^o1HXK4_NbNsWGAxYOx$P6^J!Ft3N@@IYrXSsL+4W4%eI%3a{|#%zk^ z8Bo1~2T>S7;Lt1O%sBw`UsaZNX=&{Bg&4x%EQZ~N?ka!ioS2BUYD?E5uxsX z%?_e{&{#jSG8D%0q|>=%&f|`RK1%hpW##h;M`;U7+ggUVrmo}r)@H}2Zs)i$na0bL zPUn*8_k{(0`y7{DorA$2t?6_+z!f0HN6+{}%+9`mGu*b{N9c3{G7Ap~OO}J5A$zBO zzGn)pf7ep+(pVT;x~00qiGgk_ANh)Xk7u&EIce|L&*beF`n;OQBAXMC&4&0K!iS4u zyW``irN^CS!iPPL5zoT$8%lH0vjcs!PJC^hOr!qHiX*f9NiK4Cnk9qJatpim$$`m( zmOiuyRMcRXlxDzIw#7|gmIE{Iz|Ndy@L68%@fzc&s<}OLw#SSRrp~{&8w&z~y#EnDl?Yv#{rLISa4Wkn&spx(N0>94)CV zgG(QWhM}Up@|;9Xrf*=WdnaG7;4*(%?mCt2EUdnrk?OvAehcN>=lBHGlZ zf^zLY&#BGA=pn*AXY0BQM7N+rG;;E7)Hti&zPrP>=eIC@N?Aj4 zK+VF1zLX!{sBb^JYvJISpnWEESw{`tVt<9AF6XpL(1aSEx*aZYczHyVJSjkPVmynp zxN7Bgu0&w4rKE@lp|x4~NrYUdDz?9|;Mvi!%KI45K7+TgcU{C2I*UH554wf5aQ(Y; z+8;ZnbzXn@`Yiks`P(ml;M<2!&uBB^*~Bsz)`K^WnLOr(wVBE`o{1$s1m@%h*vaA? zu}Q9`h0qXK*4O!yeazcR4%d$tTNiv6`@S8bs7si$2;=!LP+jQ@p?%OS8KYPz*m)M_ zPjSwkg<)33x5_L`g<=+F;^gbi!YUgmjkM2%jckgV?Qgv<0Nt#f$J7Yb?GSMm*6o@M zaAwRqa ze@5EPcu)5toF;acc8JmTn*R(R{ow^K`;S_;>$3tD9 z8wgaY-;sFp5*4mrh8Z^K>si>`ks$r}V?g|S6tWSawHw1f)&<8bbs}9^m@?#A(ntc=Sh{ibhb`VgaQAwkveFjt*O`O?PDj7LQ**LTUa=yaKaBS-FO=Kye`>T#j&rXV! z>7yn3to4CRu2Y_mZMbp;gDkP2zFg6pF|G6LXS^f9KjP!f!k$fgbg3&%HI7D(wPS81 zn^s4)WoY!l662ZWMwGMt7-uEN)}%yN+yFLE9A9yWvOdNf$oeR4nVhd5FW5(byGgTH zNaMPltZo()+>Ff-p8xik)qT-EXi#Tao|OFg=btzWgO%!9Zw~amsnW~iAtQH4Ak0$Q z6G!lQ7xv!Yc6b&z$0+Hs^UAYW;SpCm!lyZA{F>+uL)+gM+MjXXK3_e7S-8{b zbULTuSGpqs3+CE2?S83UrS6Jcxy(KX9>Rwvju7Wje;_dQqHTq8?=y)(=&^>&MjwZ?08+R`=RcoS9jDEGXO3 zw+v!89<72 zno7W&n%Pw!1@RLo{0G#Oq(+MV7+|d~egKj*d^=8EqB0A6{#Wb}+6RsG)0Vm5=Sin? zx#Y9(hVJ`ua^nI7l6~qoxhK(U9A0H}*&bb@!&N{cg-jkUX6?Eu>1erTfe#w|+zRXq z@rjejCN#wM=7dY3d^&GaAbIjN;j` zbyK@U<&^CCukAovdMymEyDk(q*kqfts$-auF@HL<8IubqkZ zbsy*8`POR3!?Uw6{s+U$lTPPyImcO8(>@FEC$3?2CLcRT=6w`ewbfkr(41;EblTll ze0CY5PUjopTxQ|U8G-kCYY)9Vd8cy$z_n9P-%h8~>E!gKaHrGhbUK|*=aCYZ!mG{w zcDo^p9}qkZPjBwt-SRK89fZ>UpZeSG`n(+k;X=Tg!x`gx%Ic`S#Y4ZBQPCKKb1aX7Q% z=$D6wxD*}_?%cn`=mIU2O`vcqYws+wm%bFn-%Va%ts15yu=Dt~GSSgP`=guJnK%3R zZkdnrp2+sHxum7x#Ef{FknoW?%QsM6(2Hq*+B@pRnFB39vplTUlfOqrk)SS79BNL0 z+=`(Mib8mw&7$xv;jEX!UYrXi6{DZa4wOlbHeS$!tQpy+bZ&8hV#YC<9p`#S`AFLM ztA*M28ys<&(JxOvWB&2*!@q++M`w3On$yq%4jk|}YVMxCD_pIJ7Q;yd^M~KL9UgKW z4^NJizC{V-%@#kjzk#=?Qd^i_|M>9m<2C**j;Gf*!^7Q6`Ewm8A8xMc&vCqYeslAj zq#%EJdqaOU6vQ`Fm&!zaDU5&Lga6hCw?#X=cCT;W-G#P5|HfEbvJ}oI?^XIEKlGOX z!af?6a0Q`G?R9x(6_nDzP>phY8{0K4ppx!{LtU-Sp8(WSBJ?I2sm@7xoB>jUDc;Dr zF4`&cxnjf9H>zrb92LU%NLxJK^2OR*q_zz9DaKJ<&VF8;PUX|w~w;? z#ij7qx4->Fe_ICRU*G@s6YX$m^AqJ*!go@TJ+!Uu6y35G$MxAK*${j`>vrn;KED0^ z#v#Krpf=bLGh^N`W|W^(_=>Pc8BEk#l1(^3;^~5borMvWQIlv&TIcSTahvW}rU76J zbI&i-Ze{)nuEF~jcB6=B_h+PCDC3TLo`7j>#)CW%y?OC(I`4aN@ZmONzocVc+?0KG zMfT9Pw!`X*%;GHOEQO~UGOpL;3~bsaf_kcE3{2j&g|+5zL(9|3qFH!;b2rGJ&c}bO z&n&hd<-pUM+u@G>I{)2mSeFaUOW`bh1Pn9|d!QQU$V|Bzm)iIOP3asMQ~@=VNv} zf|6ElwKGc}q^twy4F0?XeZUN|6yFxvU(#P#^?rRX|I{7dS07)0{sR`i@4x)<`sZLH zyDR+R)BDdKi$A%jX!SG7gGAmZ`jzdTiUVv~^=W^YNx8>j`#~b_t?eS2i3*w33QO8W zd_~w(Hd%874h~o*(R`906JR4#M*|ja-t&K=Up0TcKFvAbnRc;5@M`QY@TaNW1c-7= zb(-Q2G_N;maq#f^H7O4gdH3{%Z1-FiErm%6>f|m2FKKvk1Ur4TgcF8vG7G}ck%B=Pf`ms$v7VRDvBdl zUZuHm76+-aQ*XAtT>^K;!)KZ@ACNYG0=CxZp;B#ts5k`5Za&LS*vEfbTXS@cd z#&c^9S@#gXU2N6%$*0mG4eqXvZ`nqxKd4{k)#y07pqh=dR2B2ZQw5rQ*xLa5!>0~) zGspt+Bn}%!I074G`#A98EQ+OQ@5beC>r%MY&p2MNpCs~5xvlL; zT_WfzM)hfLUZd`(Z>O&B10$@9FTto*DrJ3fwB z1*=oZjT`CL`&ID=sftO+b*tmZ#Hz}3Y*pFA%?5u+R&Tf7-$2b+Npt)if4E=u`B^kr z!5FK-UZ9$>uWILUAfL%EkjVRJe1dK>{3q}MuwWx~94MM0`5!Nf$M39&se8738*x4r z=RdwnS&$MZnlLXBKX2;COz+Q`g*uD~5AaFZ%h-E`_mUiaC88){<7!S;hI9p?b{w z-A4B|jK(N)fwTy zCx<1Zh535zEHR_kbZV3@d#|Ak(0YLMp?X*&)IP$jyYA_u+>*R-{dC%mwv_%D$BKg{ zv!6Dpfk~GWCa*H9Bh=DyDxYngbX}*riyl~#vGM_{cwCFyjDF>PK+j=JAb#`>;80;$ z1@ky&y4_Xj5X!|UbtO8~?vLNDlP6H0apG@jj(^o3q;`ew#$kkH=CH?67Uxc<)9D-? zeODNOi8()Y(aD2m@;GCwQ$pN((&^luPs}OGJ9UvfNaP*mH9L_^GOiDsc`&U;+;Z(W z`Yt2(_Zo)+)dFbe-3IuJp{3j)RzmZ2OBH+vVNcCMUIAOeFUWc}awE2W-16c!*)MC4 zsEBfAwaZI1+uRa=6qD^HKx8e~K@VEuZ%OS|JV3lSxL7-JOX3zNt%NwJ~PLzvfTP1H_GkhqV*8#^Ox!Ad&a+_`qh~uTH1a>70P8C%^Mj7$;_h%+cz> zW&VIKe%KKAgrBcF9eCPdXVL>rKtDAuA9PkuUCsl|SVc3UPo&_bsx?0gJFH3seSGw) z`sGry59})rV^9hi57lGrYIsnr^?{MkuvYbv;J${G6CtLt%a)YPWwU52bV7}zifH`t z#*w%={c`-N_sRNM8Am2oMR_!0Hk74V{*1IcktZQcs_6=?@vr+sd-S-7a4ZfUsK#av zava4fold9IIW^f`;m3`4erlEP5jfwfb=Hhi_&)en&(nwQSF8PVi}R)Xe{Oj3>2d3H zI-PR@m%{J<{p#w;?+?FTJv=->;oX1yC%*pv;$NQp8|?l5Uvcy5>aVEzwo^4 zB*=dce~;xF|0M4||M=fuuYUPI*5vX(K7Ardr_<>io7aEG`H*L%Q0^ptd3b<-Q23R% z1K)l3-S_zZ{pYJEPyg});`a0}e}&hs&vgB7L!WkbzB#V0{vSG$6e~bftjqua002ov JPDHLkV1k;ABQ^j4 diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/YOLOV5S_2.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/YOLOV5S_2.png deleted file mode 100644 index 61357f3feb95b5f70d6317dbfef7b516c0492449..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5796 zcmXw71yCHyvPD7&u8V9IC%6*`?iws1SdhhC7k3FP9#{gwT>=D$1%eaY-QC^Y`P}>O ztLmwlp6WT>)iu>~dcr;_%VMDe(Gd_3u;k^WR1pwfk-w~I-yk6%AWqqWnO+n`CskPq zgwi3A05>9WQAt*1aLRQj^VBH#VVhc)o&G3%TGN0PFy96hRLZC6eR&3*$0H4@x7j772B+%!MH ze1k6TbrJ{IOON=6J)vJf${Hgaf{9RzSo$u2HZmG*Ep31$x*Jvj!tv=$xD(BRI0~QK z4?2ZbLh~TvM69of`X8uh9s(u$knB11e<@AmG|5u~R-k09ScqxlL6MCS)`%$L;CKU` z-Zy!lGnj{}Tt0tV*)L-kZ8@c|b)v^chb0gYQ}~|T8-*|y&`vuM7Iy6ltV?}CE6sR= z++oH1y|t8)HxCM2n!m0oKiOOzo`)$WLK{h^barY_G*e;x`P)#Xl`{9M*X5iBNn9cf zY{wGW{u_?MHv^bfjJ1@HG$d6^L8Qic!w~c{o2+pkv8vs6XAIf36G)i*rJBc6~g zGk1d(ehf?;cc>F$BdE3CchAJ&M``p%KecnZOThHU*CYUj5I?!|rME<30Ug`oBfmcx z-mJfBG!mcZNnpPIZ3V+o4aqG)wDbrDc>ijoQrb46#NYn#HP)63;x`p5Z~#c?@tWm| z!?CL9;7LGYKXVROFcHHfqEx7Cw~RtQfbKce>9Q+3z0!((mr95H3uUQ@#{YR#~sm}VX+ZWZ7Y8UHopnn~r2rRdHln^`2|$xv2i3C%77bw0hv3m^aK z=;hl+08D!O&JK|)Y6`$eBY|wJ@H49@@SzXC`_}qz0HOj7iLM9-h`yrZTeO+X<($mm z?{F~NTjqo~l?owEKXe;&M(ZKVtn&h_^x&(Csi{uwwO2L`Zq0qMGD?tU!9dH$lw-As zhPJGZ=HV~=LnQn9v?S&OMW^)?`QXZu`=)O((~xsrJR3}t0Prpa5U5hnR`4e-2bpNb zQq3Uws!cymc4{?FQC75z`s2K9`(=!Pdkn(4Q8H_LTcoQ@hGa4~mS9CAHipsIz~<=O zd%WzgJ019F8d}>$n{|4O2^MB-pYI0Zkzr=0a2WwO@jhT&JcWm!y@iXVJE&5uyqy}U zm~=JyOqw{H)79gx4guq=R)g$Z4s|Q1sc2>>AJb9UCl;Ab*8I=uTj(&|rNt4f)&>QlcbUK2l%zgR zfIaMGu-wP~Mo?8?k<3P$w%{a7d_5ZM`22VCqcy3XQ6mhREXWJII90z0e9sg#JjV5_ z5Illhvtc#&cAN?+m*9$y3}HMNs>b$UDay|(i^8=6pl~1Nw2u!B!sq1gQ26$h{^*^2 zq`F~-bD~LrxJUV?X}Pl_3(E`HfZ)oVFz5KuQM{Q%J!wIJ2; zNX?2pD_$eX>>0GEU(f^9J~daAbA7U@gw8v8P|Jt4k868w_2I!KVJZQ9k}}DwrM+Fm z6-Wl_8(xvS4bc4cMXE06m_v-;)p@;CW6}2HDab+4n+J11q9*=rngG+umMn&X-N>Ht z&q#Z?;yFgrZs4zZ)ZgxAs_~bu`617_0#g&V^keFxF_^0uK8n*r&5?5>A`>hlDzy53 z>dWCNQIp&22K+D_QI=(36||A6x3hnVBAY0Ovsj$8DPL-RCtAN}g!}LM$;HKOSaShq z_OQTp`}+8{EWWc@1n4iTuLAxr4q+o5%Tk5XvY!bugcEh68B%2y^HdV<5hTAgqLG_N zVI$R*b|yGt%f zCMeA$m#3KzYw+!bt>9r}Q2HT~?X!+6?rxye{nuQo(S7>lXdJ&Y&HPTA)1zfkV&dhK zcHk+;CiX8l(7GmCP-#bxopzmbwu0RNdq^ICeKG1Stc@ z@bvDeQzBpCK1){9j@@hGGE6p>|3GKTcVIuZ#$Ms_uqJK1%LG)Ep-3l$+9{%_vc418 zd91@1wa&L*F#FA{)8R2dnN?a~DC0hrg8yZg9mJzEqG&Kc6m$MZZ1u%1m9PbW^JR{V zJ#w`F@b>aHTzHhi514w4*Bme@%s4?e#dnhY7Z^GUx(}e!)!BJDgYWy2&D`|27bncQOl>xwgS>Wp zKU1{I7I9w~H;x#1Lmy_UI%7*}@KpFR`Yn7|9YLXWt~2J%c2kX>Q$jmE5(3i;0u5H8 zb<~3-u4ZX-Q7&h5Ox-9PPat*zO)uU#B&b`AnWy4J6NfTN`x5N;L9ULakr>5`Y{ zR`=ok$vWwk4PG-=i*}B&H4t;fNr0)>nWQ%hHS`YX(GK}2Bt%K&X*iFK*Vd3{J`rd- zhjjMg#?66RX_tCvH-BKS5gdvG(A$>o5@Yw|-mj&4vsHeqV;{^Y@%F9HQ4bP5dq3kW?}$@okUu>TkiT*~y)suO4*fd zCZTS+d&T!-S!{L98ej|G&}#1mCF;(uglgyroQ z+%x{a*>NmeD`Qh%!J}yJZFVOdfe7tpRnT+&L@-kdQ!uP5>O8Kp?>)VR??=a0tDFf` zPhMl~vGHxG%|L@joEZ=CsnEBub+oNaql)j+WXOM2GS4V*52^2TgqtP6e6u#)yz`uA z_}^#S&RPsZU?Otx41qzkh7615QKyWy0Ig!yV3p;NXdFKCp!SPteGZk@fxp9H-<$&s zmlIi6gq$15c8v5+sshezhi9n=i==`6g+B>qFpX>>&-^UF;%byKfY{JWVP-dfQ(g0? z3uo$={2He)_!F{i7n(>{Kt?(1wrbLr3-x}_owkQ1G3G!4y&R&&NsF2@+{BQ_T&fN@7zapw;hrbRW-l%wLWDU7oD zqKN-Hs-mRIUixb<_JO=SKby(h3P}1bi=co#TY`nNWxt%mr_?nk-DNk2I1E)J2f3&v`@w-6uSv#cIq^)> z^n^0@3vA;W)V#4Rb7otCYXqYHQ|1`m5YV=ZSQW~WyPR9liBLfdU79=b#LF{&+t6c; z(Y)5>7I0Hptr#*n1)c_zbwg4UloMIUfJkO2joZo@)d`l(csOOve;i&8VBlS|S}Yor zuqKZCs&&0U@Dn@Kw6fnGj&&PE-Vuk)P3pB$72ijM0pGq!=U{{Wv`jVeS?;k_E}9K56voqpIo7ZwS0wN$*n@xI83 z+W{pRRtxCWc5-*Ma;NcO@0TxmFYec$3-REf%2U3%>bB`_y5lbXX?7V>cQ-73b}sk0 zn%D7{AAhDSy%sgq`H73416w4D1-#|`IDGwlsSolX3c=Tw#w;Sa z*7G(Jbvht;oO`N#2VOj`eJ?Of@OzWuW4tP;0G5!SPq`w@*|4;Mn1t&7tro-Nz)w5^~p^Qt(S;WY0a2 zL<9{*j=ZRxNA5txQe2*`N3*Pm>UG+#{c3M^u?P(ucWl3_^^=?HHI4w9=p4!Qhs$a5 z{2m34X-Mcg|3!XJ2t`Iz67P&>8?4KzUb{h}qr~wg>h&c5JpJ7Vx8ru< zknZc+)oaa|lzEnx{$qC4FhmIYYw-FHw)M zED40HDkAV%psMj%wtW$GMur zswXg}K#^Z=lL{Ckxzq{4rTFU7bC8~~%lc7F0PACRaN!lh^4j)Hh0=`aZ@{jkYt4im z5vha%#b)vWbT|LAU8Rc|1V(suP6y!FbCh{q3!f7=#lW;~{D$C#Ng$v?o>y7F6F~Pi zc3r#1i`$N1PkjneI72|cymO8!4mC&Z=f;S)-+|+IU!+@7ny2L1(+r;I6vz&25ab2| zK+;jtK_L{S*nX?@&}JX+oAXoVcQLDrEpU1I@#E&_HQfwfUjk7#;l@YO(oJg*g79n; zRY2zNF9MQyrc$BWhYb|ny4n+`BW;bQy=XHN(z#+C!noL3V_Mn^OMbqM3ieBu{SO#-TKJ;h2 z3Jqd(ma&+)JCqR9cF(;G*OgYo`{MUrhy$fHCNsXa0ORPU(Csp&bX04AzX4FtH-c2H zzDun{MQk-|Iqpi32m5KxqIDjLBA^x4iJ^c5`$)wQ>jeE1;!q={6BhN#z6kCYR>WPg@N(|Pu)T=YA67ez?R+? z`bYX7H`k*#6F-D4ma1{L2krg3-l3vDdtC?QBA7f}qn1z`FpkAtGq!P}Pq-#~szt5_ zeB)D$=wLomO(4p}y=G~Kw|;r{<~AG~laT95AWBN{r24FIr*;~aO(HR4)x~ez6VY4( zPyQ;#SET#R%~hRHS8hh`{dtwup$d{zn_5}(sMZWD_bG9NA45=@QBEbJ7X9sy;3nQ` zqx=Qs06^yasXuG=fjSrKC!LZ9+f^+=V0ru$wB)O6T<|Dc?-ob#&KTvu?M?Gy>wT_@ zvKvrW(56d#`rjn7-piA}TP53M6~we5noKUzx5#B3kIJ$it7y>$VQLlS=(vwcY^G;Y zlw&}Jf1<%=T`Q0}mx6u!Sq!>Z^ZhHqSoi*Y_=b}VN=#7cclm@{u~TuUXiwz~W(9}b zeN4#N^;W`k-7T9c{m31WfzX=R8czt_Kl$|aHmE9|25V7-t^vO3(qF_ARjE`KIYc4> z(hHPhzVV|*(>qTyY%o*1ljQ4ZBTS)^v+(~o-hyU_i>Tv?+FiJ(4Q*`uQ0{GMd)M#Q zYvU7M4d$U3cvF6yDVyf)%{#@BB3)FG{<65!Jw$lj^9JZhHR(EcDA=O=%4SdiUF{Ju zabJaYu1#2)Bd>Rf!~UuZ?56>JEpSIjuI~RwZ;{J>q&QIBPB7he{QazjES;&e2Uj+4P@epgb066*J*Q&!&TFUj z<*SgZ9#^zBFo@;T_d$>;D33(}CelwRiVJAH2b>)wDgUDPrrhhEhd=Ca)?(xSEZigX zQ5|CZ<(AHyQpQQRoUhFkPZWbjW+l(@h7&!X^;TEQ`9X+c20By^ETbJ7?wqYXX2k98 zZQ&)E;!9@M$K~_G+b*F`mId#hZp>UJTR$(XJ&Zkyeki_4Ww7h~z;@NB+ydf`Co+%F z)&8en`&SOa*jZn?q<`hBKO9anb?XG%Gsc07yCu-~o?>e>k{-GtzVL#7EVk{lT&5-1 z0O6B}2dggax^@47@RcQBjZrUq%ow9lQ50mR%dPx}KUDONjkRMt%iS`D2)o=W@jQT` zoDfs#ZkOUemk=<@6OY~h6r$FIe|f_{uq^&yZOkxAcQ(7y^=PgB2J3(^HtL=Z#;q()Q(m7*ddH7X)vfB=yaAc=xYRJuwpQRza6 z^q!~)VbP+L1QJMu009C-2uVo4aqn}^+GpSY-)Aky9ruoV$ITd&Fi7%#^PTgVb3XHV zzh{^2t+#Df*t}-Vnr#G$i5ch%A#Yav&y&#bBFQ<{}NS?^S{%6K&YCqD+854QUJLhJXuS|hGugeLVI@i- zT=ha(5162z)NWk(#(aG@k7#(U!Ef!3t%hkQavyv)j=~?ZJo)I!Ztzv{DptakV8lcT zABbMXk}-C`mf2V%(f1-%%1T={JbXYF%pbP{{Ga{^z~dznK2<`KM8cw?3KkG8;;<9F zNV3v@|JTuq29FSTgpf-`pd~zjq&WM3{~){7E!uUiT5?zjFUlsse4pspzxhL>y0bEIqng7vb=;qQ$&WXkeC{6NuK3vOQJm7B`IP@$M~ZgvP>}zQ z<@LJ3e_vR$Fv`M{OBS((N^r?}3)On!LQvIfx<;-b%Fd@af*O#-Q&qVe?i;sObdeRH_=wOi8lBupxEnUy(x#bSE|KXf2)HHyH1EJteN1{L(faCd@~ zMY0chMa3b`)z2Nc=K4z}yG;nv_KH8qM|_i~HQ-5;TH7i$Cd1WX&*hyDWFm z$70R!<=$m9x;EDrky6cy*d{tgt+kMB%K?&eFaZ?}{(RmeWDIB7S}4?LiA4Fnk#12B zcAoC=TJ>Lcq8jJG9p`guW!pV2gK@m6Sf+R_)H$>^K~)HGm0kNBwzU5{W_i5!O2>ki z=&UFAdu>Qx&BwF+oNF9hpA2U~X4lg9RgJNDSuFffU~Omd_Q~`6)j@Rfw6oky9&4=r zD8TO=c2e-1>Nw_Q28*gu9Y&jXN;ZRm2#ey50#&cbqFcG6?cpt&syp2?`gI!1kpVdb znit5e?`t>zZzk3u*@ABa!yQc`V-|lGXpPxCxsd>_s?)_8n)$h}j@&?ZnUz$hSvD?+ zk9vfjgK@S#Z=ozfnmF<7`MG3H@4~Pv_en&-Z8D9%h3)=DW-*&Mw6Wcw{GKDm$?Ig; zu9iv%ZZCSNW!$-7+;I4W=KkJcY%X?sl%a|la*AK^YGolc4M!_@T6}$ylGlDy@Te=- z*Da?}R;wt>`U*Vols6agrMG*CIcXvA7Nv8ELwd{Ex(&(j)j6h-9Wxrr z$hw9vcQ25Q?w%Oq@jfA|D|blj=00AIT!l!MQP^+FjdbA3h&AF0%m-F4EKilE722lP z-HJM=Qs$-A{ODd{T9q_LuqkDlN4L59?6!VO+!9pb_4Uvm#uVc^x{Ab3# zG?ai3bRYC?xupd6c%$SP)VJK`tgt&F;xpfP#RL9Q5rEhV{ftL#9^_dC`m6@^I*iU& zl-@vvZ3+K=*%5{u8MjauaPi%r!(?v4!%g$6@4hnOC!~> zp_e#>>76n$%`r?teO)IKbw~BppvJ-`L6L(y7ZZZJ;MwHSWe6sZ5%sC1^BEh&AR06k1_h(kT{2gzOa;m&eL5`Ok3F9}eL{9Q8 zg2fj8*|i2U%w7iDWVYDqNMwVimZ(R--AUmugB+$zmPMzq8!@+p$>c(0kJYrm>_bdR0($t5?K^zg1zx_%@2avD&EC z3&l(Udx^jXu98(6)Da2oE!a&;k8R_|tw-;a+kVR!<+UVOnIY#NT-h`s47e3gINK%8 zj5`BL$u3g>0>UoIXXfTs<@2t!65r{&^SI(|#O39xLoRnpT}xt=pkKgxQl zn0K;I#U=TeF3FdAqZk$B%5sV=M2;Tbur?{Rn zvK@-`zSPyz76J3iCq?+BRMSX6lpO6{6Lf)qIYnqGwY$cfQVk{u&*kxY16r37Zgfx@ zRcD_xIrt}VCa12c?mpjSQN|cPd;-jkz6J+I@g6 zD%(|7BMoRhTs*~BqDP9>S7HK;`GxTQM!_OTd<)=X6N0!a;fDu6c&z!zh5-8`w9$y% zXD8cEDGBC1&cQv#OXD9TnK@}BJaYof=3p?WrbByaz6q_YW1ULt9a}-mCfkUEZys(s z7P9CShq-4qsJqWS%!`@QoBAMN@+${YkvG+bG$&V4#$X@ak0s@2foBF_ERa36{Q>&Q zl~H85{)#sh)9bbVS}m-pI3@Rmx|xB7x`jFC6zo~^3iK}<)o{Nxdw#HX6|i7Z zU*gwP4jb)dG}s8gSiPz3Om#x(RZ-iQwv4#~&jY70_58$P&H$vmCy;a>H2b=Vu4)ED)r{U-iU8(kYqw$F^|nqgc;YpuFC+5h$yvx0sM5LCYFUM^)+kw( zaj(uuVR{0KNi*yt9t^?O2(+^4r79>ml@=r`&Frshi%_gG;e3|cK}i-8{j&Dp*8s5H zkeqS~{@8YhaG+*aS4Y^Sd1pdoj6Xmh;0m&tE4!;%7C}{BV^xz^~8*@I${cO=m_UiKk+MtUo#zK%#*fM0c`C`34 zU|a)4{~l><-JjNcZwDJngmtzQLSz#kL8<+MPEu0|a(`-SA|Cg49=@HK4b(1RdyC+( zF|u0_+=;A+!*8!IM1J2vo2SFJXVqgDibX944%+kZ&=7y1yeY~@7lgkRA$}MaUt8Ot zsno-j#o`@$;yA9O`sJmxaq21fX2Xhug~%>%+%d`%oz89rpFW<}EBF&Q{FT?i;_T)m zq#}67-{yi~d-iY2JQ|>FL~+|9Ae=EQOt7l=+&_1`S+S+21oufrE_UMl;k{$x^T%}8 zizWa%t32pP!f6LY^{MHR2rH}=Bt`pph2GX{eSsgJLaHLtNL8VQ{sDsF0F(NHAs=%{ zu14>T>wYO1sORYV2d1$R4`j>IWXCzLJ!FH^7e_K&NNQD0bHQ*CMr(5%&F*4vO5aq;_rxi7z3ullVx z2hsNVK2aQQRBv-Hz0o{g&aw8u0g1Zj$1)Vjq+wUttERvZQblWm$TvHChnMaG!{S(R zU1BPOu&fUQr16oAklD-t;WVk7ZADp3G?S$Ex$&Ph_4y-NO7!dX&G_c>J9v`#S*pJN z7r(PcxbqI8)x`SROSk>^;gzG%0WFL4mN9&E3AZUJ3pk}&=!-`opu(2ye#TpovX&hQ zKm-6m9h^I?VD9i(5w?!*U3|359(1ckbb0kzFz#fwTy}1`$EB=0+b8NL3#jR+5z7$O zA^nB8pxpsJxS*r6lwc)tS}k}Uy(B?uA*0OiB_Z_&PteZ+(zy_Qhcxc_cD}e5*0iWO zXH3w6Bk5M$r1yJ6Zt^z{XC49d-`W6_b#pE$4vcd;UlM2=P& zG*i^O+xd>X?%59SbRgEwDVpIR$T@HPn`D`HtEP2q7u8|580;`=lun#U`JPNl?l>(^ zJ4|-8ko5p*pZdUnz1Pvp8=0nMGY&Wxs}P-U8Elf@!Z*_*Zy5`|rKh(~3aZO2ui!n< zJ#lAjF>tj^xZWI5?^uw=sYA02Nw|5@?-sV-%9B^kC0Tu9P;uHqxvySuo8DPtXmevu zNhIQ|^Pss~l)8i`q*x&(7n|dLD=R1m**Dtg@pW|vhn_)^R+urF>T5eWFRQQIwF?>8 zIY{4)HR`V))~TN;+YSPxRA6Un=H3tC+#vb$*?HXF-6sRZNqf4Vnt(&x8t!wP zLSEW<#~VEq$vm4$dXhdvesg?m-NzZEAJBtxF7Hq=iOL?ooW#WJOQnJ08XCKz3L=td zN5ZIcW$>Q%z8>Eumu~vU`>J#cQz`^tAbQ~Y*3MhvKUC?3N;-7CAuawt;pHQ2h4^ATXjla!x z0a{6v+4UFWqgR3N5HE)53g9yZk|Ku9fNZ(*)OA3CUtq~m5XdpUao*x?5bQatL(ube zVLlwbtEvyMc(mgr4BNi0OL(-Ivuf;yf3c6aAahFFQv;i!F`HvSP6&F%vV)O0jzsOb zSNJ$Sw?=bu{H@;d((#(GE4X3eyE~zaEqAwuWF}gv*79}jsVBk|ACl9M0SQWp%vJ}* z*}esiH@#)tqF|I_jwXJ^+s2QGbAsXtNC#FWxjxAv&RTgHZ>J1ZG_6}6;rG<#tE?-d z)Ca=-tF9Nb*~YAtgqP4UKRVPO@UZO2Q9wdzbV%j)Q=* zKpNxju~~<%j>;> zypjhz8a(v&{p*G`Y_^AP){x!RAbXR%p^$>1zAS<|(R^O9t|7UX7m!*Vte@k4&2w{I zpGOr1(+T1XH3g=snsh~2VZ8XYaUWiIXEx%7yI5T--Bf{DQG}eRwJ)^sATVUu-FD)v zwfmi`BTL`zVUSND@0Jbu4FO>kqZ3d`E^-U?Bcm=x5u zL);Wws?|%8rM6=_%hVvo=Qq0Yw(W+id))7?OaQ-a&Muza_}pa>;8sr?&aiGqKY`B# z6m={at+XeH)+RYeo#%mQ#bux)=!S}T-Ze}Yf=jt)%&%@gEVo`8-pA^yLv4tJr7*O_zkRxEtQZ1jw}0*n z^(ohL_4Y(zy17F*R#d&P$1mN*vB0-s#DsF&q0v~{KG$Zvl6>;ER;7*JK!Zo;AMoI} zbfaXB0#J&r6^TQbWwj70$EsTDoi#_R{!O zJ;CV8T_9&U#vUS^^gh_ZO4{Y6rIaWE7AqG2KBgfK^(F$eV06QZ4XrP~LWEetfPQ>y z(;|%oNcfud)u+Z3vI~mBqcO}b_#9nsY{gohp?a9_zmy!|=iGJDLdn+ZE^WWhSK!$u zXkwyes*Rj%G=F7DavlDeEYkS~x{O1wR8JS`6U7$*KBZ)!7}K=z!Gb(&nqGixHuT1o zv6sxW@Yj%ViY&2nk^T{}Oj$X>ar0_*tvJen6mJ<+nM03Wv6JWpH^-9O8!pn#o-|GO zsoR;`t4V6%pd{o8wtGJ&uVT`?L5JRFmiS!cE1pF9n_x;E+~8@t`F4?kQZgu226=lmSyLShpsWLm`6trJy_Rt zczrj^>sML1?@?D~_I`L}IAt((-F8&mQdN>*1@WI?KHS@wQo4{mkuiiv%CCjwemrij%b*f%OE|>U1R)H z;li}31KiTQlT?>3a{)UO1%Geis|Lc>Ivo-ZIFKgPB z@{VT+B7+t$<4af>n3-zjALVTn%#}{$xnwK4v_izjfZ_|ZgJ8r|>Aka-7R+U{e*743 zWU1FUD*F4#4&qAn3qwL#_`Yh7T72KxFx5P6WL>QwFmMO4dR4BUM)8q6yvKh(`i}Zd zr)!~m1|VbWO-R%x@KNS)%w_zu+K}Q(z_L7N3i9TXX=47H^2G#fMSw?iW@=MZt%DrA zX_>)Ri!$iChtlT_14|*N?)4>|GAc|ms&bUmnIUZk!OS;idA&6-`shDSSLXUE>z}7Y zsxG5gCVb;sbO^Q>xtFDIxep!Xf3LSx>Ypt>8|vu2*DYGR^#FLAK}xpho6y8Silb7J zcwX{M>Y<59A?0NLKGqWG2)OW58FW&3RPHNL8x+*Y_7{b~_tp@rQmFHqeaQau^1@o= zUT9>xXI%vVM85{Ro;&bTX{#gMz`E+bd|fZ~jx+DBDUCQU_BfLr zSAWMrJKQ_YtAhIyyjvqUiIo66|919Vru*k?voWy#UK;Km6W4-Bm`OSlVggpejZq)Z z!(Fd(ydpDkC_=syR-(r5jRHr%)pbV2uh6e8 zxN5vT^!1QVK|z?JY8^wLa)KSydkirk%NkalJ`*Xt!zy7}^d#Xa+LV*d0qN!O&mO(q z=Gb~^u2SA|`KCp^yG+;0*C!$tSDM*tI+!NdHiuKclQOIOr&ADZc;~0fIEIr8{`EbC zWq~pym!1NzQ1#L}G4kwXq-ZG&Ucpt${uD%h=;uzY9BsQv# zz5GHY&jsQcJhKC2cHjc#`Md%nbV{eAXQJg=Ol>U~McoWwme5Z-W2f~)Yjd_hNQ)FL zWxqUu9n3WmS)-?NQzxr7N7FpTdPo%tINv&9aUj>tuTl@M6OAoRTsrMP|8nVGJbR|v zMrF~J@zutBV+3+!9eMGMe^-9MrUnl;L{pP3uROjxHRLsb+saj=9!Fa%Ku)~d$sTlw z%dmz;rn5jRU$Pu|OXn)Z<)=owiiZ|E!$|eg0<{m#R0Mzmh(g=XtqaRN`C0|wO-)*Oiut|c<`tj{atncEvPfNNy-LYDxeO}#QH^d? zi+cpTqD*!jo_)KcJeQg)XVgdgnDJSEEv~$2!vU4c@gXD70neiwufTPF9q?s;s`*7P zIK^%-tU@-u+Jk!v5O7&=UQ{L?kmpJKqc`CwMjE(sGMA*PU z(wmk58q-nVl>RB~i>F_{#Z&gesfw|;`=qoaeP&LK{=d~PYh zUB>98I-Bid^8HH?1>JL1CMwM#-cxqiit<6k?7Bd|@gh}L+pw+o*L_UPnFCz?6#9^Q z>X%m3uU6qu=_$0LC6jbmkh1c{-Adfw2}2UabOa$5~2q zB%>Ij!fNCW&7t92KeyP^3)*`1Q;6X8KL$4B_s4aeyZDKM&qr(sk7Jk|39R^J--X9*vlu|`fQ0r@oKxb}GE3YQ zqJq34fzB;Ca$!dH{aH?fdO6MmD@phW$r#&O)JUx5S!z06VDX zY}lhK)mQI^jcfO}6Yh0it|tupCN?(QB#2pY97?c^wCC_dg5+{!Ld(X_kjgWaR0;;wXt)BTVPnWKN#gamGL~d z&4H8-MnUp*l&wjMVu`g z_>v$ROuus1U@?*p2lviq-twEI`*_u1^;J1WWTnp;x#UsfUUKHs<8zK~f@pYOpD0tH zBibr*ihuS-_C>Wi^A@I#Uzttw0vgrTvTy6#$c>zJgR|VN^Q8GW+VRz(HhCrU) z7Yt8uEOG~4gkzBCWK!Ck93#ZN`gQ`tqQ{wiRF#w}L-(pwsU^M-9QM#~R#H_bv`$92 zZz)!44l(4vRX+EYegF16`BttEfgJg`&n~{Q!MX%@wmuD4d!Qqte5pE2-|#g4n_CG=+g10?dk z$Nneqc^o~wH&2jHzHKWjjjeazGhWsknSCCeTRc$)5kyYC3b+>~q>L3$y1fD#JeEd| zJVRedK$D@0UU79k-XZj;0a)kfY!vdBJP$ZuK-9nk_rj|TO33OSc_WwUrwl7DxZ$_D(RWXMHntLUnLI|e zY(~5w;omZw>zrWO{AOYNF?dA=*|Zo4Vl>6rQD=*V@*&(P!#S5i7xYW_-07z9wONQ= zS2N1CKk?_wTBG8im@g7fK;#WcdKa}lPw_qS^CyQU-UTk_L{Qr?&yrXV##Ys|0Xv*h zj;$OkJlx!DIOmz{wr(@pbdB))Si>XxkSxib!PBFiMY4xBv?a82w)x#|*jb?pUz7)U zAOR|4idzO6m);z1%xGmBSuDtQUz+GJlcalkP~se86)S}w&ykwMU45_)JK=Y?mz7Cz z+3#~^nlbilnP`%O*)ilSc=9^S*3U6+K znW-Kfe44uGV>uY4Z5IdwjIFvWDwgJnOzI3fB-;Q$HNtr2$zmKLFa?&OUEdM&C^2wo z`BZ%vF)Q+UhGZP8_j9b7oVsumR$DVZ_ zo(IVB+a)!~7O;4O(*LMk`JpPwKzwzOTUCiBi1tdp1p@GkG7Z691f@=N}*2xuga7XZ0iyVAJVcDefzI2R!cN5dlhD} zh$gRwpq8fZ?0hYZGOtEMa+9Wb)~Aoa?F9ieB8F17#wpW52`ITf-0FRPhmHtgYnmnx zj=amxP@bgI0h-H@rsJw-Ge6!SBv0?fO|vwHP4nd3TD%nm6Os*QzlUA$Os}mYUSp&C zv_%zDjp*|0f{D>j7nI5bQ3Jk%s(q>rb1gP#e^RXdlSQNq^0%Q4-5KF+N2Bk1 zf0(1O^*46san)w_u2`0dFUy~bd)P0skim3B;rwVVLeMP$@(R}MXs2j1L4Ot+_VdlpK5Uv~A6v~#iGdvrs> zToO2iD|QkqxSKt+C;lHm_DF8+KP~KklG1GkG)E_U?*|>?$O3ESAHBPK=Lo>)my1K$ zh%4d;W`+KPg}l0(fvTKXjz{^4Fu*UMmT_7ilpv2x>2_|+g>J$X1v zpc`$;4QHX0S7u`@wJob>war)%?)n1Psrkuknd>5>lTrIUUqScZtm71U33J{0sAeQM(VNx9s_pK;)SpsBf_IjUa3(6?4HwV;)q!qHTh zI^4wEwXxs61`0#L8&){$U-4f%UrDM`kY^q_QK`8p^RUvn?~9qp`uuX;8N)#%ha0c3 z=bg}Z$)I=3j_*RSAn4*z?aF)9x|e#U%0B}MGxXxGGNjd`HX{>1lc(|cHv)eDEXtC% z`Tt2>?7tE0{xiy0S9ChzYAri=HFWC#$(Q_H7W^+#(?6@={l7@``{!1wBZ|W0amfrC zEuS2~wI)?}u-FX=zF@Q<-@Y+KSiBT923VZMexTDrnp1X-;C+lNNNu;Rfor7Y-w!>r z)x$quFlCVNbQ*`>0EpdBrZY0(*g|UhxizyITXZ;B>N_eiNs_W5Q= zB|8hC?5gW99w!P>Ol0G5^+qw90<4TnTgb_Wq`9wo4xOWnM=_j>7|Y>=#5CVL7h_nm ze+ka@d$5_smgAH7H`?G zl(R^@N558Yahib4lELszdVqG>PYtcs=zyG6Pi7B9g<5tU4-f60t1LhZU>B3&s+eEv zy=p5iB@zogurPQ%DYcUH!~z*)cCL{{WE=9;JM0&=rUn$yAD!r}(pU-KdGn9t_dgXI zMc*L@HjQjZp8jxCb+^JVR$~;d?i4flfP<=BMOQ=^YuTxu|W$CO-wsc9lhpTF-ZSoBJ(FOh0 z;vqIV*>XKb{rxa_iJifn@2y*)X`iINReI><{Q;$`DSLx$S+m;IAIX`%nQ|AE6NM0( z)u1T+hc0C*+nUe@t*KC3Z}d>@#5E5EfxX}$f%JtAK@8Xw!LeU%jXl!obv5R}-}&J3ZhE}RoXh>y z?)NMnAp~{GDZRs@lODArxFF!{B{`qEqQ-|-^5>v=6(n5%S2NBnf33p zc2+$?(RN*%CRGjtHWs*A9`A%itUKv2V2@ojAq+)qG>r~Hy#@~Y9DUuuA1d;h3;ygl zuZfk(ASKfWBxXjaa_jYKA`?RD{yUv6Ex*L=oMqky?Z9NirbM|P10TFN$Hy=Ol5nca z8C@;ZmFmr_TDo^(O+kz8+7j>TiRRAPQ>@IrJ+lT}GHlF6r#!Z$rI>=l;q6t&!)+V` zcnu@>+o~0PXwW4zENv^hRIj|b3x8t?F%(6pk$e@qc zJ(PcRy)<~JC)R{h;L#|4@PXHicF^WA%_L>CSUNKe32wnSl&gX(z)nq6b8H=-**q=B zp3P&)%th8|aHGJD0CQ7#iOBfdZEI0|hlC?sy$=*~c(n=td{)F!FuSPL-+CH>Jk8xW zOF4pgA{;^N&3GB0?8ATU=$W(%TRl1#9@&!TMQK}V%ySx1hP0@YP9m=V&hp%dxhtnN z1!?f<&Z`{seVfajEwgfCm&`1#bL2NBi?#h?j}sV8li?mDwkbUPcXE9@_8)tpo;Fi@&@V|V;jPatxo_~nO-js%$?NJ5s*TjT4f9PV*`)- z9VVV47l&_%b1HlqkA-ZZ)Eq0Owf)juHhTr_r@yE&V}MU+7ZZTCPt1D+bFc@M9;0^( zs9X`FU_Z&2aI0)vP>Uq%k&ycq*&KIAA$BSETk~9UI}$|@XZ-wLdAc0Scvv{v*jia|x_Xpw%!*To$t%#kMU zc@P*}1FQka|HkwQN;QGk-YPWrd>E~t0vUoz@}rZ#x&WbTcp>+`^bW?1;|C+DlgRLC z)lZyS#-=#Ku$>aP`W$@=yz0TOcFc7TE)lSI#XRw?J(s>#4_wZzZ09g6if+9hdeUoH z1RQ9uYu3~1Aevb5YdE>#%TWIQ7 zm-@z{vF!=cihl|dNOmOno}wq?)`!0}l#wcTaOdIt2y{Z@BSKqcw6}~GgbasCq+JOMy=jdv{U?#H9;>Tuhb*2(kc{#M%9$54-4=kxeU^iHKvQgJA`>kz z9x2{+GV%Jd${S;+YMV<_eL0-!A#twg#MgTa!YYnt-4$CWB=Kr@{c(t_dZjwbty~a$ zN*P5j0pYLHevO>YoX1Zv6Ofa|Pc=KSy<84`4#wK->>-CexCxpxv-AbBbXgx*t`6pV zPA;##<3h=B#(e1Dv0@WY%Yv15;?a(!S>buX#^nfN%p3N%2SASjsEbETOU_b`1=E{f z<-eyr_B~C7#tm_RseokT*aP&+ZhV&vBNOA{Lb!C*jblD+vk?Dbes0?(ym0?bn1vq% zLy6r>PL16*j<|H8=HE1bR>h!S#Lhf3vCEmkh*-K-ktkPwKD5I6Of(HlL%cZ-pS!Cf`& zJ`!}Uo9`Fv7#r;OD-m%rN|4%*)I;I0I9^AH&iw}nZs(0CR2zeBOe=%7El!Vgn3+Lf zTkMmNC|>hyv4bfh#)`Cytz*{0uM3Asx8QUs{B-R?0(B?VeUUgrk9=X zTF@F)cD7%|2Ep9is+3DtT82#WLk=AEK-?53d=qZ#bEj_XOF|~_nx`}q_2z`;L~bul zzP9Wi0Otq~jfc)x*RlqTBbk}L_Jbz}7L~9+ye?yG1n@DPoTaY}xC|#XlG1*>;O!q- zVKX3`Yk(4vz3@}3JU;$E;*WoGM^)vf_Fi#Kc7g8Ss@1f0TAYZfMh zi$Yx4Lwhk(HlkCJK@)t#V7Sn)m%0tmTi52gC#MOcnT2!o*o#=#&-S^SFWIJGxFt;nm9ikpUd}F3p6* znnkdifUx{bB|LOaS7AZg=|G26gNZtRb5 z%<4_^x=8#s=~+u$Y{M$li~=jXYl3)(aPhSFWJVm*VVKXGx;snv`BTbaX~3;C%=&ib z--Eo`Nt<>ARD$Y}C1IX8<1w2_13=Wh$Pv35Q;=I3`0fX4G`G+y;fUAOu5dfD#7XZw^9-PeESmRptRp@Qz~p+4PFD$(6hI#nn@oP7 zv9-uAaU_1sH=BseC4Ms}7VEiFF(=`M(|Q75cWSJ#V+GuZk-@(QByKototQ10=GDsllPLxANIQEX zc@a7pF5BOP@TCqsAQB#o<$?7k_)Q{PPVs-B73|ce5xqd)TNhQISTy~n3kULcP_G~k zclSAvcYoPB(bnC7?Jlq|dNEqf{E#S9^u@WxZ{~UmPYLA{$oep3jZ<(&-4UPIvuiN1Yz|>Pup4 zXQX8=@jM0mj=R(z8op0|(xqGrWn%47gh9VHTt7R?3+Hj6Z&j9Jn{g2O!fD2rw0^qbKzK-0#I}?l)yIDorhg#v zzf7Nc?H zJ4fwVIG52H~%XwXZmIsSL$GXDbr-pGaSx;6gWR!{5pXTl1r8q~$j8#Th%mIMc zKL8uCgy~Q~+GPa=4Zx$HS}&5o8fC6_GDiw%E<;{1f~ zTQg9)fAap0qxW)wmGVyoXL#9A!~?KmhjUr*7u_z~ysKGSPAS~qBLar78W=u1uYVg5 z*F;l_NtUSmWxlrGVeby@ry|JrdFdq8fSRIgY12b1n_~7hOP~f{2%En+A*z>b&D*$h z#7crK5Z^R3Z9#3b>>P^nl3Z668&{Yxe+6cC`i4F|4BD=g z5Bc+(A1@hgrLPF0lRffw4k~u%{o)5a3K&KjZD4K=+xb0i|HPyHcdGNOR188|Sq~Kb zJy*P9Qcq@kP3lRT{=Yo<{`&vR9R3&R{r?6Hhs!E23bR+WB_zpVs`$lX`F)TK02LSP# z)>DveOxTX&0ydl1g0+x}5&WDT=CM>Dmr$9gZ^O%pdA@UvouEUgvFVwmZOWdX3Ex8G zre6q_ZJTWSz@`Mt&WD`Z#0`~L`J2M1mVnvEr*7}K+V{_i`Jc(Rf0vJ?O=fiRL1}!s zSd1yAgd~!u)^Ajs()k$-B{^`z_JO&?d{L~5;#%CewtF?qLuDZG{0DtX=;%_t^0qTw zEd#r$O{%Qf zc`S!|C;s7WmEG|-Z!h)V(f)7)aj*=0rpxGOaAbt0iFg^r3n@Q{s+SWiaI$F6miC}{ zO#iJvX@TF6{Dk&LBFoMx=gP#-Q0VDOF-)EE!dCwC&Y0xjw4W(dW;H37J82M(5bMvoK-bjphAO;HpZp~a5_s(Z{UJr$bn`^)#T+Q|z$_SY45%gmd9-2UoM zf{gVf_2XYx9Cl*Qp4h}|L9|#{yj$Re!?xojO453jmM0bz4?E}m`5Sj-{D#ES8Srf6 z5Or~?njfBrK&#wGSV&s}Y_))=kE{$ABcy2pAs(0b-Uoz4-Um=|bVg8f>+xDSx_e6z z16jM;&l2-XWSB`Ye4gp;8*a2Y2V5_v58Mw33RP*wJK$Cr;JJquD374tG z%oRl*KuZaej>(0+wM)P~TjPKQia=QhYjB=YFdnqM|`)(gwIMsDIiB7vQ?{$KjWtPr{nzgT zyb-%V{tALdL7_%J8}jZ!)tKzTVPqL?E*wzK)x#%TrwBT_dg|A!F}aOzD?d!3EvzZ^ zxKXG`N(sfy`2H9NXsH=IcsGBY2jdG~S@O*j9q7_4O#Vw(NOF+G3&asUvK?DGd?;hY zIzyEJB7RJ;(x#`(po`Dl2Y5&FKU^50tirbUMOm=wt+2liJcT$?0t0%(qZYeNIx6?X z%(4#C4#SiI7V2vZ3+G6pp{Pq$X6oUma>?nnhVW&vOF>z5G8xWUkz*UZClz6k)&_;e z{ftn6j!Ohp!I{p#HczrDZZ37t1Pj8dL2K{nGZ-@M)$ zD2$TEpjm#D;yka(h-7HGdeN44&n$XE)+;_grReFc&*<(omur0FcXzQ&(!m$ftoN85 z=vulyl;2r=1(!ICHQM;9|2AS>ng8?Sf@R|;L+Ndz_jfYskMyChRkK1gos+qo8=1T( zMryc^$6n+*Zktn>KYc2!f_Yv_oNorXUaPh$mAC9G?c%8a{=cO!{-vP0+wCB_OH^9D zw_fj8HiT<~1gIwYP@$Ju#g2kfn*47_-Ehacu>0J}c|Th(Ycq8p=lJ{}c!;O=nro)} zp4Aii1@z^kzq4j=t}SmO@st8kWr>^0XcvMqvBnOy64AVX%vMd{51 z-`|;qf!uQ$u~z>_d*2z=WY(^&D8h^=U`LRk%!r7HfYKqNB34k5CM67rLFrrR0t6;AP^E!AxMjY5PHx6k%SOQAR+yGn0e=&x6L`PoVCt(zHgoUgO!zM z@B7*N-gmjKYg^tc>){P+8<>)IC~JU=U+6bvKWeW2KrGz;ExXGW!G+E>@+2`iKDP8S zOdSp^mzzJ4zUVi_$LemqwJKqAQTbZblPiRAv)vuIj$p+rJ)-%fOc#TLN>-3Ze+oTj z$m)q_@!^hY7x5(iscQWe$rq*uayWd?wY8*B{+rv61=vVf8ED?ewDl&((wvV9L~pp_Abpw^OIm;UUt}WxVE4a#<7bS`p{!frH#opMMDZmB zq8hP?Cr2j7Nr3TJ$FlR~&F=miy%;^En^L7}^lkndT4gG)8%(xk=U(1PTNO0WVzJ^b z9h=(yh;WnVB+~Clw$cuK=H+`%&KpEm5ZDNhiW%9@X8XW9S}nh9Z~e>I`Q#$b=?u;< z2%ut_K6Kl(09;?Ifo!`tmHi|}|H-dK;=7O)914cgw+*~3T4JNkA>Q5Nt2gxhzMR=_ zSX@Ojt0|@QSgGW_*feQzx2b>|i?Gqd2)Y+12a1rXagvYh31SB9`!lm--D|rvzI;}A zNW#6{T}8q6WkjlsrO*1j^}A;CY~>{Px*4SbRXyruQf)XqnHpX&sZY1r6IpsE)O=?a z=}A{}6$UO{JHgn&mH9&JkGtjvEW(Q)pX46|f80?Yj-U1hmuB2s>z+Noe=N1GP8z6h z>FROailBi2DlLeD;^Y`^+>rX_`Csd}PtzMst&xtxM&U{Ml=v4f`{voW_*-ZH-9xhS zYZf>QgbrpL+d4}Hm+zZFvBz6Y7&_khv@*mK&luW; zfLUG#B1piwR)Qz2kG-RP@$ToY4>3Q=K0P^w8IJ^sBz=LCO3|n!34n2yYr-=0}m^@140?KetrMpQPdlZg)s_7$0w&Q#I zpzD`vF1Y}UFidiDX_1xqFt)ipBuq*e^@lWPgy~=x+nflIT~KsgF|C;SfxNv zRh6fH-S_@xtb}G)e`oL}zL2qy64Xb;O}{Rl@6rcWTP$3c>NBul-1pfq2-#E7tUAf7 zPv}s%gJ~+O@I>8gpHbhfnS5P09qt;jvsmR-cWn14dvZziG^_}~UcOcg4qPvGXATAl z2Kk4go@1PZ3$+aFxxPYh>PXiSwK6@`vPKFB7jK0;0=Z8DI1jJrb2)zMnS-T{jpsye z??ZkptiDYV>Wm&rSaD$~!W}rlu#^7saw|c>m*RGt75flM-&|DhcNV8t+j#VRj~+Wq z_DX#Axn#fQm?0O0gba{!u`|}mnI*!vGGlh1&gkBCF@~O&#Sw9jI+lb_6mLwb58w*#Bx75FMtV~{z#i$GgH`#EmM zrx}3Vvj4t|W>NIxO98`lj$=|LMU0JKUS3Qo31c-^tUwK~Shm@`AMxzJut2doXFw)+$4cu+f|DB?mV^6Y7IcbWqwHNV!RQlJU5%&+4=RgXc&w(I^c$KfCIuK(7Z>qrI2?yU zp~C8hK<6nNz0M0p_dKcrr_hilztwG)17hjbnwE8T3pq=F#xSHXgDn|IJ7uBp%aSDc zEg<(0Nvl@r3?8Y=r!sgj6yJ7Aen{;i;5m$}7t$Y7?w1bfoMjy+;j86UoneewVFCgD zp{m%v*UWAh}Nu9MfeEOwMKg_JIT7_qfMzDYTfNKf`7=Q;YY>} zf~r{}!NLxYp5d#6A;v3L!P;V%=i3!d0L?YZVUj8&(MqP>P$gyx*BUguvVfMy8?ZIG zTOJHmJMvv|^B9=@nmj*<7LENQ)5SuYEoHGtVsSpK3@l!u2IeoeQbN1idqg33*9lpE{K4U-A60k zoQN1^AHYqk=_c#K7HV?JC!Mnf8`dtaKM&Mu&+>ZDh_Ek|xKYsRG8 z2w)z7;2sH^SQl)QH`zD0+m%^Na-~2ZWuBZl1X|V(Ihh`cjUwK`Ht@svcLbN0y+MCL zI6dNsr}#0F&8Vks;^W|wD;SS8l@3#;hH+g6jB0UQoqwdwX#sI1oj@D{1XZ5UJTU$P zpAuMuLBT!zlATrJ3Aq}#-+>b^IguaV)aFhBX>dIc4-BQLkc+U}yQbf03Esd>%J)!6 zafG58erPu=eg{zQy>X|eqC^S_rSYe+RTiW?&L;4Hlysk(d2UQ(Qf@*bnh`sMLT29Z zGZ(VgIOIs8AdE6)2WvBz{TSBHK|=-*G%(^q5F(_Tv)8j-h~HyZ#gq53C_U3`VY##~ zXj?!aTs*`J*8E!^T*U9PHQhd?*lGLmvu7lKk|fs3L};TfLk2Vi8|n2eO~MsTR%whb z={vXnq1Z<0*b$B31CtChxHM>V6G@fiQ1h-~G|<8>x54)TnkV%}U!DkSi#j6IvdaxL zeqwc;<`|%jjXMqOuDIV`HPE(64iMPBO-Uv7e;{P0?O65(^i+^>J*J7&|KQMFs5>Ku?y znq1cp_^8laWHF_*y@VmvoSJ~PkdHNlE5*F7_G8+YIwm@c0^m%$Py|oie4jzFlY}{4 znfws3&bg;paCbEdDlHP zt;Ad2!^P4Yi~$f-G$U^~R_b}GgLIq}!&wJolN6p5%i(2SlyEd8e*E=GaEb^?sXyjQ zlCr{`f)SzVD0aEFWOOz~%FY9MVbY^Z!%49Sw*B80!9C~R$s&J38hxNSj#aN6l6piN z@$;uZT;!LTw7njYnAZUEAs&ATXWA1`tMolHNF;j-2hX_0w;HzN?e^A zN-A~&$B$oj>XTtB%re5O0848#*7M+Wr~y0LH=ay;D#XVU7?~KXvAD>%zNuVDEr!%z z9w-f_aPC9IA|?k*@rq>|l~|Ihn#k2@7%6PnN!&=c?oZ(vltba-dR{Q-@7q3kjC!I{ z7BaCUV;(%&lyd#^{(Vc6KF1s}KX@xu8J5}kNOcwLYK?gP{Wrv35jS`2&aJr~l0xtA zyn__<-}4TXp8aRM14f_`t8;4$%0y5E`DL~NsT%1!^t`^bz}n95>((s*tK`9?N?K_~ zzOL$2G^&Ox!KR!@q|Xx!9^(Si2eh_(1;3r3JN6LMcc=5E>VUoab1)eMIJPRI%+2Ge zyv@WJFj?rl$!-xMB&Sxcr-w(gtovsMwi`U`zy-tt;A6-= zM0|$OvQHk*`T3?qkh5p1q?4lsQd|L*zZIF@!iQH%HFsVtl>Y-}-c#gBT-zhvUyuQy z{J_<_&iHU|<-pnh$o1vJXfAsw#=J=6lW zv{q_b3)3&QrGM(ZK5t(o(WT%gAy{=NiYyxnhU`Hc=?@U1<~%Kppsmi1-Nb;XP_l^j zl_BvbhDt&n!}t3xZpQ(Zg-YPS-|2m0)3&e+<$1_^2=l3-yY_b@Y(xM8k{8UFvt=?) z%il=09h%4}BbChmCr|i>io?EeJL)Fp}ub-C5+vf zVh%#)c!ZmN7a~_$dj7OS7Y~(*4IB>zj?Y>yEwFykdhhyMTFiBYz^RRMR}TVUC~n_sq#a~i!HT9I#6+wmEO;2Y<} z(QWv2zH{kZ6vCKhhN$W!ZpM{SiltcK$Rbt&TYegVR!`66vz!BDp~6k9%rOvooaF*{ z)|_Rs3fzVB*m=6(C1$mhWCp%?mqkD5X+h$rqs|v#+kQs+OTSwuGwUZ=l1%<-^#b4# zuwE0ak|gP-ek=rpC~=^-h`6m{UWDXV1s566W?}6GR22Z(<--Yu9zGjB4PI>cnVUT2 z$`$VSuRbm`>tBC5ked9{Nlqoq0c_o$(>R=X4UT<-VzgeQYDID^Yq$i?(b$}mCTXn>n;@J^T zR*h<=wB%i%U6FER#u%%vhvN8(qth{b519)hqAXN+zKZk3JGn+hRsd} z;W^*ctx@ylutca*aI2-DsGneP-=~aop7A({qSyJHP~kL-a+kCSb{$kioCu7Kol^)iM8N0+I@jcFd9F{g4j0p zper>T?m3-5DQqya6x;sL@{isVT8%@L|X%WnA%{A^7=?bSig;;A9tbU9S&B>6I%BEdHg z(1L>FA^uqTDu5u!-v38Pla1&p>(-6O*iBAXFScJbs@56v(rjF4;)DCjd|I!18k?u6v2EHKtGwjSUN zl?tA7B7ELaZY_75KQ-Rdi;6d+9<-l4n07Bv?J&6EF2G`NB|q_UL$uOVUPLx{r`&25 zf8C#`I}v5pu}SVw37C^Kt@hY+JkhvPAo(f5h-@FBVD8zCJ8=;U#W3j;>i2h}vcx1dxvBPNGnoOCGmp{zid%~dJnC|vq zq2?(?dKu=-(~DS?BHCt7`AcKzS1(J_XP%3we(D;{hQv>!0oPz*WvW|_4;jPch8Abs z9;L9T{s|%JvuI46EJQe9430nq?grRK<`kUmA59cMP+>0QVr^+OMVBnT*|Vn_zG^U1l>`79)B)Wi`1v6tHaPk=WhEW#S4$MWMuJv_3LSjeVCemM}J` z!|bMx35E~6-GCOK8$N>L%vL@;re#W|aK>A+?ajg}ECI$g!D*~98T-j2I7~7;4J4_E zYbiIfcnO9fZSe(+8#ryula^sqM8pn30tN%;#p9)V{q7z3rFf*X>^mSRs^_J4F1rK- z?)RRwI%#P^Qt3C|AgabE8|WoejJh|nNRT?)D;}Z>7I=d>dea;f zh&%WaD$$j&iSiCE^3}_Xc)AVZ{m3(xzp1i*+I-Xa7;CHlwAtwTm|+TIwz?yI?yc$>eSLLkZ;Bw{klP`^Kt^3$S$`~d?tg0{6 z&GSG0W3a>gLwOvrP;6g5R?U;Lt>920l2&bhC^<9bn>3EvIw@S%1x9!*Xvv8;)WOS% zg+0X0uxS{6mihsQp`K%xuyXCo4^fpAdANX0owhvOa~$abaRvUN*t)Q1fN2>VYJCyAcTTk%)z8`b%osn^;Mss8rHRlfOtGb&s3)C~xeW&sFP})b!_0^? zW1dvn?zNDH()VDx*SU18s#*KuC;3gpE$>gCTGr1bT(|RE3n9l+hDS7cHQ>+3-&y&& zE-N3eIQkj;>YQyfUiUUWDHDeF^l^$UyHX3Uk!*^shMRvOX3VE$KgD12hhREonOy*@KS>=KUNbU5 zbc#}0icdO-g+~>Bv4rj(vy}~7g{Q}O84o{UP4JjYkgj$lYkaYjRYo>ahx7WfgI20Z@r#oOw_m=jcF~hUfsW=gK8!B_;R>i| zEiRr6pBtL;%zUzT_ZihZcoVfKLzwV(Aj$K6Nvtm*=AxnCu^$1{(R6}We!p|c83FlW ziyqEJGtBY9@~Ra=*xJ6Xs)uN=k51!-pQtUWd`7*>>y9;3TRb%E-!8FlK4+V1)4z~+ zhPaDi*sAWW&)umk^vQ2UeO|!yc$E^MBh;mt!6EC6ay~8K7ye&@0RK;iLC7Edij)<{ zz!PQ$*fIcfo07zU+x3J3B5C=0d545=g{bic^ywZtqLo`)KjnE2gPQEkE-FPyVi)3S z!g)MG)GnjmtaDiaz@I-|>kRY<(dx6UOz()NBawl1asG0tthPD>mS*3= z&^n#AD5&3j90ce^Rv#X2Ux3ihxDSHAls>%rIl$H1v>QOcOR54kT~!H!usqbn_hzNP zlPb8-g^?PUtX|mTBnNhdiQ=!p-$$mkH&TSu+Xy=l&THuwGR>Lzord^9n~<{OU=1j} zaO4=WLpmEI{{_g9xikOF&=9=vlG!YzW%JsZm;P!5Kx`^pjC~&NyelmK96R()FEak8 zXn?z+tMG8>URJDHWvTmnB}1ObDEW5kt1*DNCA$U1re!XqI~t62uw#L1jE+~eV3#~6ekXKZNIU5Ke2 z7ciurDZXdltm90Ag?-djJ=NhQ{}Eu zW)L6%IChXmt5uA#*%Bn?`pEefos11vCHV_+(H8+>7&Z)}Z~^0>wiA&tLR%9i26$3^ z?uF6e>&qNvR%g!F!W*fN`D0~_wToMnw{#?uD9XpqHMChEr8C*lY-~qkIkD7hy8A_d z@C^=_<_`;)W?rZB(sc}+Ui}d7iuY2bN}O^1P^bfbqDjG!yfNtGfbeKy4a0W!uy&+mwS6Tw+Rb%v`!n{cu1 zts6UH?Z)XoM3l+bd<@CLF30*&O*+{QLix-Jbg6MDTI+F;9h86h#V4~nLzNVFUrtSd zvWl~?7>m@!Tj^z_XIMAUC**bF!mZ)KeI2t`E@f*nO1n!^5?&0%r!Ay2{Yh^)x-{wlsSn$OROnh3I>*O&3QwGIaCG&>3lfJO?Kv1fD1Z7YFVC7-hxq24$ax zT{mN|s&d8I+I?P=oN4&&GMFHhLo4>)wS2xl@yYhy?(gkfW3h_M=-4`a+*HYd@J1@E zCtPlr6cXDX;H{Y+Gxq9tB;~OX6+B(hs3-jP4#&_K5Eu7G^Tr7<7&uH z->#c|;Q5o4I<5r$Cp{;}@SCot*_^sN)0z#t<@nHBr2^l(J-~6VAx(8m4gj}r&BgV{ zTcr>))-L#YPo)QRo@hvJjC`;iz7nihlmrMGVmcIG^#_J8eg2H(`m)5^zcS5+bcf&! z-()})GnQ^vawDN(C4+dI0C@u1zS-=X5rF__cA3MjUE-GZxwy_4>xk=G4p+7%g-|T* z*Y7PzKE+V*^U+qne%GXon?7@ke92<05AN_K&#`21Gi~P;tM~V-S9uTYco;%9;gDGf zl(`}!y8exOE;-w-_t%BcHbD>l7-RjS7>Bez^s>_MOTZzQ7lI87IU$SY1Mf&uAX0T2 zt+*`Sj&5K%uSVQ+Ms`_&qWEfy8VqS80x)@(<@~W@C&QMm!o+H0FD5HrbY*)!%Kv1- zjDBQNNvIQC89Z5SV7PLV7o11UX`wA7Yy97w?~hKvABD|dp?~fG zhI>Ah63uz^O;I)vHO>L4Ots?$+swy8>*UR%aekcNkB(@Zm~0Dlq7v*lxPe{)P&??; zA98dCvj?#O%QjgEu4>Yoy%r|2MC@SuHN&;|Y@#?YAtjp=T_EpI*m>}_~c+8urFykPr zBdN!qQPYz`L_dLjh8XneOrAM;bN!9KYTk~U0|EJ9Ei#P?0q&ip%*0Dk*Z%yfjh%Fc z52d~Ov*yzRRV$fZ49}E5q_tunvBcTw*P9Vc=KR%6w2w(!?W^3Rx@uL$r4|C&4taMn z;#`ODhQq88WQ}Z#yH5!1=ihbiB);G=7y(*C`O}*6_;$oFe=xyWjccS7H!Mwqn*+_O za8}l8AcDuJ?8|{1UW$?^Te*alPW zdmXs09V}T)&yrhRqsg1Kj!$&vh^?qOFGydr*px2v^y^@!Bc{g>O`)QtJl>6FQ>J?n zm2znG9b%k&l)v*8aMwpF8&G2XtxBB-VL`{19ED>h=Y`1rST}#w=Fyj`P;40`%3d&&Tau37S+RKS8Ah+ee;**fdK7njE`f^k|~5UN%gxe zaW*SHUaxh&KdgGQJk8ig`~9R9Vu~I1l~zGGDw+{Y_=#+}Z5xfp&DY;zYA5SyUiUiB zY4$-~ z)s)!vfHsr!(#-)8JCtS$%$9k>e5rkJ7J0^Ic8v{a9Vka?1;K|?l1v+4ckX;xpf)f+ z_F^Zg(A zF+)z3J*wpx`o?y~xX^Sw(URY}Gk7|rwkYBfk@#b*9a5$tzMU2J%(#3qYgxA?=kqhx z=V_f$3yY0jG)v==rUt7DtxWZ9B2*pDd|9K7B^cDY2Xwt^VVRBBxg8`T9q-2N+O2oO z0JXGAu_L$$lJ9Ypmf#z|_zBODmoHb`cVfI46GV0b-=g~8{_$T;QDq-d@%I`3U03D; zxScU`Ee;1(zecVbk7fs}#dkU!r3|XVKh^+9QO(ua1`b zp+|Q8_>MPZs&@I&Q>Fb;3V7}zKd%wOIZ^kGTn^1H+_Jo1=#*t=cqZEytRy*bZ(Y`o z@+$fIi0($d&+ogj4L4hPQGOD!;i4qu?=)V2|H3hLv9Q3-7`gKX1NGHA;X&M1Fz_SL zPHiv@m{5d)-hl7rN1npA4I`CPZ*MWiHwBZcJH$5tH0@mTFm!Txbj=N$81xI)I|d=O zq%4&|%(!D^{geh$IdW{ZpW*RI5N}|E1lxgEVet;9@ zZ3^$Lpu_9w53Y-#UY8yt`osLaM27?M>q;B9h~r-}{lzWy!#Lu zkGY3>fpWAffp1zPjr(=!VP^s@Ify3W*_3;mcxO*i9!9a2d&7Ke)URqS+iSJ!@RYCP zkom}#+nY|((UhIIau@PqWBK$4*_@5N*)pWc^=;4=JLa7Z%CZAKf+^LyrV-oBB5f6) zdV`bDfll#;)(+`&>g@zMKNc%0PD&M-cOA`GIPsqxl6inq%HVE=1H5Dof;LxkmOk%kW8>W9S?{DLG;1+r%Efo@=#fUg$`e|* zsc+G{p%_>-S|7z|_jG`CP*5y7cCIODi?U^Q`~%{TNP$Lt>Tz@H&yJ~9lNJo0`N^-n zdaS6c{G-N)L?6B3%EeWCMy@$+>;S4R9ilY%B6334D0+!g1G=H$J}Ap{*%@Pw&QLy9;2tr8U6}IUHS8EQ5jX%ul+g4aZ--+zTC^E7M$8j{F&b<^ocL ze|edRY7BQcf+)|6{8Kkm%a}otTcSQ_BX(+2G_h>m%i@rRv;cToBK0DkFyaRPhk4W)&NfVS5+-Qm7r)s zPxiBecYBju?*He}#qSFp=sL5>dp+LOHKFVe2@eAU+dY33(CxoQPk;B=e?74OpYOu> zO#tP;tNU?m;MZKL-ppbmCRS5cI3?nozo)FI`Q!_(^A!J-vEgf^Dlv zg+6*+%#eEZUs@0;^1w)l;kJ+U!Yb=$=hV0Y#m&h;q#qs6#*SacPb?s-a}o7@uQg8A z?Og*dhF4FQ*mFF;o_88hX>wSE)A*&eHRvN$eNKE#2!$-)2#X==wO3wGXTN)RHsmtG zR|b1bCcNGJeQ$aL=@)Qq9G<`8>3lsa{(?c`h<0Uab7S)FJcfrr_2N1BCVyX0KuechMOwj)Hsk zRxY;dz}`?NUZ3IyMui~TWi{h~HcCH<--q;>=>59i_dzpTXVj2|;uGxv3==vmgfh|9t)Z%c1@U~^@|d7jQG-Io@Tz%3jd z&^;UO@(8)-H?pSyon&Y07SB$_&Fk&o*x+=1uW_5Im!yR9X#&ShYrLnt40QRq@Ve6Y zcNKQ(FWVIGhR=+vRQ46c`9UTZlEDL^oYvJgVsxdhYQ&v}GMnBPDmG)Kj1RY{kys+M zb4|mGSQfKJ549{fu`;GBJVb^t@*;alKlCcW(5!)$;KL^|F(s7~+mdd&J~ug99n|Fp znOw-FP%fhLOQqj)>Exu$A&$@2^9HXRVt?T3^rJ^c+&j}I-#u(V>x?3dP7bQB+_&|! zqyq5)sZV(k-tTpXrm`=ot9~I(kPG0}y`9IFrZGvH9$Uv2`L9oiw^@f&D*pkDJ9$vL z=NT;1N4Ja&y*Z`mTOaAD-lJ-{?+KC^g z1r+Gg*PV8URyUN3>m{Q1fp<@%COLbi^oxtE^%UYM6QwUWRVrpv+*REON3{xy9@Eyf zq{N*@oE8Ad)`46=c639wR@Nn_qfiG?B9_?uW&yp@A~sYx#~~9tAbRDwHq1)<>fo&F zz3$k6k!Q>28f9~~l!rLhO!X3%yN*khOLn7n9H-_u_-queSv%z7n+yRF^EOA4YDy;^C)aSH#*Zur`akIROXY8Aa^6{wDix1=%pgx5aU^|8b{KE$K z6$|mj$PS8H_P|YTwu%Y-M$AKQk1ReNiLUn@RbEJG4*Ut4kQ^>8IiPf{`tv%7yU4+g z&T^`(R;Ra<#BM&aMRoPfKLNu8%fuYgPGyZ1m!kgsBW8}@%hXejtjPG2nD~#7;*a2A h;h_Oli;APlmxx}$FELhHt1bdQhaH>_lQX1(iI zZ&6WEgDKCp^t6;uL-C~=%I$>DEnO|DlJERWlm(5Wrhz6ERcS2K!Gn{OHG}8%yFOG@ zEG@^^i4KoKdnzg}KE11&w*#!!(k=W}KAsLERLP!+?LZz~m5twe(=>bY%-}@d{jM70)OU8zMH(ZT92)HDZkr6VfrI;{6Ah$T%skvub z)CtXA`09TrJa&v}7st-+j0+?`y?C@|wR~C++CbBw)3RS%UozT`yWGB%vOJAZ#ZS$mWL2_#Df+ylY_sVY~I}7eO^798M#F5KZXr~!k-gKTrvTo`A2z&)pZ^+ftN3)QDUDzrz=~ciDgV4A z0`YcgcVgz10m0L@6>V-r#>17&T-OeXi`4ySq-qk0DsTtV;na6|Y$g<%El z;$enzc%_f&4cuO^71n^#E+z>_;J5HHNJ0vqRUZlhb$@$nfU>LCJV$o%Ru4-J)2u^ z8^l*+-~9}d9;27dt0d-VGt#*XEb7SwxQUL;81zlN_R!@GUTEg%zvo$Y%UOauD{I7( zE_qwk7~))7Vy38jZ!d*^v9e0TdEdvFP5OL**vn@?T$X*YC&uJ^@%*?znld`y&_d{8 zZvAxa(h>~e9%W3UB-7silwau^iYMm0Rk9b6P#`1Hg)zV91JvXT4b5yV;J1-=2E$1 ztb-|<JUxDGWt&2Cjos%hw46B zk;OpvQH?wKcpSVXn&Zd$gx$vX~M2G5!voTaSaP3SRuNs+M6 zxTQo#_&)zu%s!1}^!XeS3_~LA{QET}MOKFypg0hca=YR)!{d0)x4$y^o;x72&1>g= znR&nwqV_%T^I0@)zG6Oo@BT*FY5Jh%Ez^2#jW2}WI8#^L-Fv_@aJ`Xs51b~Mlk4+3 zb+(K<OH^(J5a?vk1TJk?uEZE!)yE@a9$GYi>2&Xv86 zSxiw8lft>+047 z>4QCNz^zN~F2u^T)P}mHZLQq9KxHR%>4O{#Z$~X-&X~!)kL8>mf#LoV--vlm<&_0f zdofVt)#cFfm>;@U&y|yv#Z6pH4@*qG$|MPNw~ff|Xr*-b_Mi(Va~-d*5?j*S z-nh!)UCMsK%eUaQioFf>d{L>>)sXqo)It0gSx~?G0RKMWh{RwH_F~u$U_5j3av&GY1u5>9Y`ODjZ+nSO_ zm3flZv6{H?w+iLWLLF!;X)!@B{yS^67^>omk(yE_#U1A?N}?kjKQh~rxoGD{*LZtF zv$XG6tzyMK_U?oSFjOadKRvx^aWmN7_b^N+bovqW{v7=!u)ShcMBFdRse7k`@5E_O z)HkJj2lD%7IN9_Gs|0NPYzn1g6-0V(uu%AjJ|XVH|Ad~7<6hD1x^w)tLSqE&b4fZe z@oGkw*pDpo=Wo;I?@xpH+11^*L)2D;@=3<9+Pb@|u_Opt{MHUYGliW?;1HCMGR+vqC;eASBA4PvV8d-DyNKxTO76T z20pM@oi-;fbp~s!y&B3!uw}U0d}^`J_=)#=mJ-ofTfX85rE7nav^BsL>lCuv9n)=u zf2fPz;9VeW8M~R&7k`e%3>FH^cM%Vmdw;>Q2%fW6T@{-VvTGGwn9GP4FtfK7p96&|_s2IYJAYH0HlhOHnIw zZ=|xMven(GolV`HulNo)b7hQbk2gLG6fsxCrtEuZpjQWa#nb6A_?d&1cor-> z&Txwi2Xmb6ayjDCUKUc`3mbV)`hxBYA=y?p_mRd&I}_NPl7}d3BSp;~h5XVU^1{xm zDUXHrmv!vNI*ylnIu0?)MXu7YJ&oO|L%cxx1gYpMa6NaMz83H`7ZNGH?;25Mmos3L z=rNFUIKR@aw2la(6D&5P@ekegx8wU@w3>}|1D(xeTKjCBBi-{E-&65XLsc_f;lu6- zdCUqJ6Uz}O_8TXyhc@)py#K9)>~GA{{`dA)v6}lS!2Oy36iC?YSLN!Q>Ym-(?Jr17vmv6;@kowCJlUb`VBjz86GW3FQag%B)e#p4ZkjV@7J!5kJ z1R?X28=|nQuC}|()N)?5mw>%v$<1`7?2q{fVA_e&wHTGyD%_AMmc!gzxtCNp@+L<2g8;g=nfB z4-!tQ0`pv}aiUTPMVtSyYgor}ge?f%GnsTQSl0WRehLM&`;SuM3=v4ny3_bYz_8hc<~wN?J1Qv)l0>*(fyyyeQwWNo)QwJ`I5cz zX3On$c8lZ8*WbbL`^^3jKh^DawS$%A5+1|Bs57KT6DTv^eGd+^KmD# z&pf9OPRyCN2Ym-d?>RJ`Gl=0!Zu)`yr)JFgS%Uw+!*@vk-hWI@lE#PD!4758|1xwV zZK|;`fQfT+s^vfyHp>>DYE%@R+bRd1(7oPLD&wT8n;cYe~GGt)d~E1hQ#fk4hG)FYdiavYL} z8|g7d{`S{$(gCBs{7kldQo1-35|KnmYEsj3U}N!0>!25x3!lC|H!?R`AKy*(1a6ZBhY3tmTnBkQ#ry5pl|KUoT(2xB1#FAg3J`C^bdLG=DR_|U|8|QvS z80usbR=~%-%9QAT^1Cu*;l+!(hoL$Uo4QG>JO1L20u_;m?nCvPCI_jL{oKDf)>aj( zHF!h0TRALjWKJM?5QbDAc(#>;PEx~OOkUBosNmw*; zro(_)2)2B+)>O_@jtC?W@&M)uGdd;T2YzSR~=Im^g!vGPXBzmE-;FWYr{iCpn*XY^GS;}`Q+ukpaI z+CHdkvcnZ&00j`SDTD;)rS$gJ;w9;g^s-x1n#41c%gnf#GR?6qUrc5HGJBmXre|e- ztqeIP#O~&$s7_0UrF}HIaHI=_NLmaDF>D@PEbPotW^u&i9|^X)m7~x&dnpwF*7-4P zU0W^yN@`g1x6r(h7NB)%;aq%9c_jZgm?3*H` zjM#gcsI5cdJdRe|z#dm6#zt0Ry{z}6(CNPtV*YCCE-sp`Pd3}t&CkeMG#Z*ZbAJP0 zG9)S8;&lbBKqgcPd0Vd=Yo%{wPnn|P$~S-zRl-mRkCXJI>i9`fpE?f1PWs=HLCi6v zJu*_kioi)1>=ldj+4o#^pl@Yr;uMsVK#SZkr?0qDRoFSh+CG=MU5FwE6~z=C84@Sg z=gcI0M|0``cSssV>JIw@cG44g%2u;t7I(#dK=g-(Qz%V$a4V@3^^yHUAK$NC>D|_j zFsVRqGu`FsZB$VpgsMR>#=m>5yD_ef0kkPN*^jk^1|~)li#7{D;_XGU*7kSq16oQ& znKS{SeT{K*0doxd971kg%C6(n@wzANeJO`q$D;55O>`6jfei}c)+Z)n^kHE|2Z&FZ z^Ka)=|YfeMxBwS;^4@qx{U%yCvEz9$n$^Vc9q+Sz>aL^lR3 zW!y!MJo$S<$HL9+C@ddbXOqdD#){vRdG&l`I3bbLnp-w!S9BX@nK98)FrHxlE~Gxm z1cDUmy|NZLOZKQJ`%u4cY%~sd*b7;hn;paO6|liZElVRmu|Hlp>h)Gnd6!vZH--s! zhQTS=9($%gW34NJYqM_4DNhMA7-n#xVf@=7ZbKbD|&u&iCzW&V{rx%uE{h=;J*HZ$acRI`^q4Lr8nAH3u7`qgIfazz3Az%$9iA*3=h zgv*1il+1b$Tb@m->PLR_+^qPMGG*{|IflyR;_mmrpMU+=SY)8S1$0{nBCF&UKE z5Ro9WoU?UP*Cq6OPgmuTAa8O+WkkSQN9{25evMr!L|kxdV$kx7-<3eKKT`ic&z6V; zT$Yq?^|glL?PyP*(hxw5j7U!Dx1YOeev#754=DeM?~lrUg7D(>9>4fQY&iR{_2%BN0MP@eWi5Gvv`|FHZTI9FW zz692~1YSJ#N3zhxnkfd*QG9!tUMI(YY2%OE{I7P9o0>YJ0~y+`dMwCHQ_jw*^sX6R JEzz=h`fm=D=EML1 diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/api_precision_compare_details.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/api_precision_compare_details.png deleted file mode 100644 index b31ba52350989488df1245f4840886f2a7db4944..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 25476 zcmd?R2T+sk)-Eh6qM{;#6hUI8C{?8iB!U7KihxQdG(n_S38W~9NGMTJKtQ5cupmW+ z&?7g#*S%8C z+gNVhyleBCHEXt>Iep50%^ERt;op-tZV>(+kac6H@DGtldrPx5C0&YB!Y|hQn_8Q$ zSyPS@=Xr?=zupvf+BtH~nr%%#e~I8jU;3_D<0F6Ol<9?7kGV&#m9J}L%pI%Mw~y0V z7n<)4fyAkur#FrnChQ}vw+Pjsso%w0`75hB3?o+gEXujB9LlOdfAyP6J)Y=2#P2??X z`|eUfO{a9s02cHi*ZkucuKYokOa60!<=(&aFYO2AALiKEeK1{U(~f(Wp|~oqjFeo_ zM$*WmU#o`+s@23%b-uyozwk`Ip4_bsg)H2farzb>7B>taaJoCFf|}v^Qs00MpSa<9 z`mTCB4Pu3usK(C#aom@xkI_GPDxiA_*(el3T=05zcz6R+Xow6HyJE|R1EQj#+g)XcKOGmooyOEH#+ShdHpq5H@RH;TSQ3;IwXz2Nm6G)b zYf@73?3FWI`-T+wcYRo27gYvjS8f_?<}#9QU;8NDVI`)Rr1c`q=Q6NmqfX(n4DAlB za>U^3zzwq@gEUJ2cRKI!ZIbeA%mRHNmTi&cqM0TJrRlre?e#dxW|>jz9yJjd%S*er zl;S3=EBSXar{3FQ3ZF{97x0<>jKG`K2LeANqSLT(^KBA@`g`d9bTPcf(#$#w0y>Zb z0U>}bOdS&mP4ak|a-S8yBhKTya@d1+<0ic_W#}*Q{S%{AyaOIkqfdlOaNVqq@HhA~ zq@Dcx+J;M(dUpUEXUJ+algL*W*&}l-vZGd>r94kM_Rv5n29h2n$WY3IjYLmVzypn> z>h~g?5yOgMTj(S5N&50$&q!0b_)SKBiQ&!x3oA2S1PWzFYZhg=r_ZwMCcgtXr=R_ zHKyH`l=*a7&e-CdO2;SBA{N_tBo$dy{e@fcnZBRQwuaCX?2%7C9ksGaKuFGfOr4Ed z4Gh|kpMN!OYYb=1sgFUA_Q>WdWk*gJFspS1e`ND?_2mp%ZQIMljlHswyy&!%T~fT@ zQimA^=b-Yn?szC^B5TR|K?+BUEFp%Eraw27BpBio1)uo7skQqCY+J#Z0;tvJuDvda%Cqf z(;S;p&Y)d|lRE*jt8wZ*?+muu0UG1f&qw-D-Bw0cJ}8OP#kkHZbd3p^|F9b6*aLv= zwAjqWkSqI26QL;Df*fn`@Rhq-_|l17#e7+|RwM+J?si`oz{ck< zrqM=+(hA*cftw=@*ozfJa~R8~P1aX~4=AKU>fjx9C-vd8Jehoa_5PRw#-q(y_$sZ& z$C~$^JGKWVLZMz6K1$!)iW&1Ma`%l9r&=HIzNQxXZ5NDm#$W#Kc|M6Ms%7)Z)K5j^ z$nEm_dV~9Y<@k9Y?T~XDtoDqHpGMAI^TK6mwI9(LU5gZ-=?ksmX#y!dvJVBH>@q;r zVP04qbR&3uUT}x#L1Vw#@#@{+J5BufzJ800VJX&LgHQu-K7LZxmV~hdb2qUr+FOJi%Ie3RbnrH}u=E%jNLfeNDBJJn!G9BTc z0iK^}nfuF$kCDA$^QpAPo1Y;@iGWnXCLZ9-IDku=IMuGGyBICpkfZI?(g*R5EFV)) znJopd+J7g#^t<7V%*_55KlmGzWW7NMft=<9ew3TG>XZ1fv4m6IvW_z8jK9iOIv9t2 zw%_WZ8zoM zcRX7!_z~7Ud|RpLSwP9>)oK|`h*1IS4j_@Rm*OaVZmucz9RHOD*c8gF*~_D zPIX%lp-%5Kzn{zV1AWt;U_2{)xyEtPK?AuWdtgdj$~FK}l~&&`65@VM^7Ex8Us9Xr zP}UX&tqwuLcgJQQP}{*wkEO_~ekSqd9(~R#&15QrEt{+s&U-F_-LB#mFx^G6)|2)ceoF^6<2=WmSh+RN7iMhrid`+y8yAXehwJ9bs8f)*Wo>7T&< zmL3e*;%AiIFs zotWnVSP|3^pO4JkNNO?mA~+s6xPXr%GAz!1IGgS%1pSEu)v+w=knTU*JFJZ|GQm^Gxf9r6LjjsdQO` zkiuMW>rK*SDGKl6G6ek`J+Wro_oI#^mhus=4KhO?@n`7yMYz{pP{zNm_4IBAZ9}zH zvDj`+MJavaGfj$3p&lrN33wi}>JIG=mjuL{N;kiMU{IPV>zSondDjP%<0H+z-alLQ z!)M<>S0?U?hw3n~Lc3w}lutH3Pqf;;-}9!NX=wPiYa#juaD}C7j0SI6xmsb3&#Ith z8OSEgaNiNwE>qW%4KG()beTJeRh|P7Sv~7K3kAkL|C_z*_vO%R$fqxlV?1c6LKDonSaGuvz3S zaUw5*80C4K$n|o0Z%`fS2Oj~NBCp?KY)=bkcU)VPoIft5L^`HOGWdi^9;fXF-QEi= zmA($$>S0vQm7{Yt@b6}`ecBS2vxz!>Hfn;os(XO3`G_EjHtiYbsQ#=ye9XVGm^D8$ z3x&7n!gnXmKsR4&9NIOTnu!Xq&_hrSDzzf9R_|#>vAah;hVv>Xn`+?X9^%1h&4r@7 z4%L8I)n&XEUQ1+;Ul10~#?eP+PR^2Ad`I;NJH`FpGVZ3A*8zT>{Q&qSZQE}X?Tg9B zwr4}??I|NI##)&x;SALb=jnBeeI4BsIpO)memg+Pk+TJ_HT1u5ckcm_`I2BztQDz^ zVR_T7&9+<>c_v`@mhsdxf&|>wtJt51!3gv8^c8To=c%4rkvijy(a3cQp>W#@u6Wq?bd^*J)q$DDLFr^z3GtX9&AUt>y7&VP!r( zz=HZGt}_lfhmVH1c{!J*SF@8=H>~S>#cVWcY@o?m+%Cr(Sejru!iNs|?0xnHb-X&2 zOpW|7k!5D*{V;JDNVia_f!~TuPSJsf2dOG1F8H@c*Gb~7bnz~C7%|GAEPD^p_Ojqz z5gA6xmlV7-+ejEZ*hHDLNj1nJ-_pADlLENEix!`Qeh$x=E|SR|GGxeup1zeq!JmYG|0D=a1vGE3^Vn=UFQ(&@ z|IIsScwMqT46@#=&>sW75$!i{?xOcS4^Fr|3HWrli=gLGbpqQr zQ%Bh_#!#R5YQGA#d%QGM6N-Ytt}{&$!{Li=8>w(v&Zdov_)KoN&)#WNy$7K^9H4!e z`LO5X(gq`E=y|F#b@qO33hJXQx4vNzGw1M~CuqQdJ3mI0v z6McBB+2|P@3T5c_Xk4sGLa3%c5s37I8c zM-0(6<7LvPwjvoQr>c8sGrO-_JND?)#vTDWp0#UNee%`VR(U-(el-E6ET1OdE&c|w zFC`<1e^Bbuhpn3VfyD*(pwzZIxMaC|)Nd=ESz6_nejPooh#PYz!1rIOncTBXXQiOq zm?Py6Q@S-V3o-3<$mj4rR9w*wwBR;yiO zc>B*d&w)6YjW*<_ZCeRGiVN`4R0H_qCe9phA@4#!tbnXBmB(TEJ;u59EjlDlxb%d7 zHClxLXxhnl(wiUv)5K#N%O_$#v4ICm3>w)l9CsEL2ua&NP|JBnV1&C(3&CzQ-oLC9k=n+vL5^zVBaeq%Y+Pz*C=7C~-d zC2EfXan)D)#PdBnWE`}n#WPamS{^2%JwCNPi< zJ4(AAFn?zpEINRH$ltR;ALPG>l?ZQPT(g={x;+j?9gH-tyLFqDs)u)mYVHQBFddp{ z0lAFBi~B(ZeHE{lj@JS~Mvmp{)SG6KAcDc0hWaD%DLcYMQGP`^TG=P#k~aaFNs#)> z^ZE;pL&eST|cvcH;JkG@&eE@Zd?ZUCwD2f{{sl7sLH>mb$>qVBp!1bwN zKxQZBh9jfDDjj>n)iuPEWfWG|Q=#leP75$v=2%`o%xOaw#(a^zwJD@NHKdC0-_#6$NQ&^kj8xoj_WCRpXNH~C_LA<|wu+?9!Ep&4e|oDNm%Xq}|= zxqJ6)ix7f2Ed8pFf>Dg^SFvVeyeyhpZOeF?r0uwGyMLV55w0HH6Yxz|YNyKn=G#N_ z8eJn@Z<7sLQfZ@t{T=O5Qpxop50@FTq!EKnZTJu&QSn@1BC85ukSxgeK?ihIA=Z zwmQ4(X+O3;z}wh_XMNpQvBEGwAXhRi7Sx93%jvfdT|GT?X*frtPe*aSi5HBKC@9UC z*?plaU3mU1CAw)APgF{*xPLacjRx!mDV;mqf*7w@TlEeQge`VxJHv5S5`tVE!AHpg zJo6i8U?mK{>007kpTwz<8>mp!>OF&%IzUyvQyoan*kvOX9gSLr?{I7^9Qc; ziaqO(2@�wBEDIy7l>WoR#gYU119%<{nyaB05HydmF3G<5j@+HT0uCH1zCA`G$wI z)Bf*=K0=RZeFTYqyx;ej@qPe{Qt9XpgXG{9kD|LM5$;T%GucL%Z`|D#)L>`Yy4MU; znGpWiOXF_Z!S^PZO51FEE35uI#pGF4y>-MC!!(_xvhE152`(pU?4wn zBR|*)DU!3?yGc$eA1tZk)l(sL`AbhqzsKz%aedweNjPx`BxK`k;}gHYyjv$u?QoOE zR=0Y)v!*1AXP8NOX-t5xMRB87u+rG0itkS2w6fBbQ@h-y??unaY1^*gf6^t^F-$iN znjFVLPRpxgeN`}h^a-!<$%^+{j-SzXu&S_WiaaO8(eWWEK92pRX(AoE6KtkmgmM35 zm6s2O#jfnq2nThDcQo^KFE)YOJepnXtrzyDWoC?tsK)>Fa=> z(U?s>pJ{!s6>3WWO(Vb#O2C%~;q|HGLph-I&{Ai(pWaFeZTvwLB-rG7d0vbf74H*( z3vz_KU86Z8W|*^nY62ZAvG{%;=F!PD3~Chccg6)(>}W`EvQY&yPKHe#z`dT)HP974U`GXysLQ zAaOe$|J_aRoXX9I!ENV}PZH==v3q4n{f{7xm<`h&9sEYqahl3exGW%q-EUjFA2F-q zKVV)Q*JX84lov9pJ-?PJX2WmD)&SoX#*@*k)%%&o!*eEHwu6)Lm%eXl_znn07hX$_GBG<(9{sp zt+{^Xie}5uuHrEUWUB-}S~J+Hy|N0pM}H`~@Waq?^8zhfFL2oVxGt3cL{Rf-qO(t2 zau09+rIIm~LrBd{t_PGJJLxN4P>Z>9k%g3^#eEb!{7<|dFgF@_Xuh7#sw4L_69j7 zWQ4G)lDL&?S?g~&MhMUz(5wm$<7s2;hr}O@-LQ&ta?ELjI2rxV;2#F(L3>Dt++6psn6R;z$E+>5`9hM2GhcW`tU; z-=vDc3}6DcZ-hSHK$3jo3rdw*GA9~hlD}S7_90b;NSo$IloA7JGuGd6Rh0{N>9dzn ze%0~lORE>}u&VC>GMnJ}8pNRkiuy7M(O)LkIJES+pgC2$7yGhP>vhk#AEm-QUP%2u2V$=O6k|M8 z@gB%fo3N0ylU1wV&Rul`Cn zio8q)N2RTiiszDSq0rI}*tP|+QIGBj>J`yack$HV;&5Vk5RsT3khvGM&>(oob#oe) z$x9f*Ekn|!@H$jq_yym#;UWFaErypd)yX@mY(fpS@;!Ka9PfbGMQbB2)|Co~KE~`Y zd1SRYZ9>;MCO+{;Er*lpxlNxD8PG8|E)Cx%4B`dp;4kq<(h7?A;}N$~yl(>2MyoLf z(IKp(XXh&n&Ve$_{=ItoOQ}qo#KXT_7TiGwV!k&uQOr3_)%O6d(&owEzW(b^SO1cA z_l7%PkN!U(;>OwD>duTFV-vLP(!M0y8M9=Hy5`^(ZS^w#Q+T`oUx}I#$~Ok1xJyFy zccOI11ZRqwt2FZ-HMJ03n@tqRsv?8}xA2YcG*?(E*z#0mrj?HNDwKO7@icZ`!{!l* zpLQ=^qpJt?^v=H&!T*%XX*2MGhU=VTh(Z$&6qm1SjUfi-da&?Knx!`iuQ;ts7ix#Y zom4DH+J<&nq~!5&P;9b`G3pGb_XOO)aWQkJIX&eT7W?cXNHY8Yinyfza8+>lGk)eb z#dX?75COkbqrtjT^m3yVwV`@qfkuA8hR-t^fS3=MdCnp~@lx^UpR2M5wc<${0jgJD zv`@s?hVZyLBL!$Tou%}+Qy13j_QxtiYroDdC_z(?9UPRQD&LL_o|4POKbFS~l!%*m z_vL7rU@wzZ(_zgM$dAzaIW8Bt3oKMO)nv>;EB8NnXtQS=ml4;Ig;>wOA4N_Lz*XH@ zd{5=1lbY`!dOlx?5y_q+xL#T_5V`P8=IA{>HC9}~`=UT7i1#j0XT%Bd`FYCX3z(|w z&VKFLA3BalcK66ECyXB4oHat2iYs8ZOz|)(8ZR4gz0F~Wewe!G1*vn>MYm zbG$3>A;QXP<{hNInO4c)o!BBPWczE>kd+&z=r2`Q#4R#bw@BeO!sKv~q2bd*^ZxgT z0>iW5ib?;%A|W(3gmMQw4Ah2+Y1wnNtU>Z>meYfuo`?^;J$QHbrZGP!-;;Nv9J=4$ zr$zVVYoTsD-V-?5D*hTENDKM=lxJ4ON}UHGN;FIjDe4arigRpDjil_`}nAx9^jgy;$Z z*xX7I$p7xf;*_8l>p#ymlQ5eP+&^m

-t9=o=#R&id3lro~s!$j(8Z_%@&eQD7In z(&4&LH*$%}SaK%Uv@VeR*8B_wP%CE})B$^s)o*dTv>q2e3pwHw9KzN&r*Hi%r-%vhKn}vZl<1r?QZ2< zZx!3Z+-bwB%&A#VI^N)l;BT&lq|vhH+SgUFeC14>E&qPl7FHnr3ARfMOHbBFlYy5X zpO6|F(~#Y-=gr&+C$%TSYH%cPYyRVx(t@!G#*O$+fuQ*C{qK}3TNcDLWt$a{xT}+i*QhJAwb?thf*W!Lex0l zYD7J3Za0?Hz7zYQ+J@m;rHSEb4&v2`R~j0x(P)hDgCJbtWCT4o@7z~2Ge7h0XOEGz z?BG~4+PZyi*h-BouIabq@EfXvMV;S{@oPXlm_d*!;q7)90%`2DKT;T9u&p*Yf`IXW zp1HEqT)$-(7gASv=e@7ngEBQIX(_~b3m=_8t`IZpPVw>oH0 zDAVWCMt5Cez>0YY&fKmByN)x>gPN6$(b_4}?ovaNLA`Q&d_|=09`CA*ApA=-$&Qr`Q!C=nv=>c;In#-CWDWw z4bH(d)Vb`2a73#_5wKx9;?;?u#mK8WyM%=J2W^}OFNvs6aVt<+D*^jQFOC{h0EE+LUwLWimy+) z44~-G3qyASpFQ`qZQ+?jE618r?4e2v{~Ss7x=*<+g6lENc>YAGd^T)3U&tDzx@L?PI^S6dNarj;#!lA2;~!j;(0!>IXlU0#2%9hZxR~|Z@&GgS% zf?gj>jB2lRnnw*7z^B%3$AN(s|G~Oyh|Vs0Cz0UC7x#?ZpU=D1UWwbqRimW34-Bpj z!pnwA&(%fWf-M+B{UAK08Z?qZpe#mFfxb2ixC;$Fozbi6UiCQt6L zjmeLG^wl=7!=KZrYsJ59DQWviQ4ke{R4~X>;0oecQngx>D_YpeQz&To5(9 zhK`j?N1VF0TYGBF8B7Hs&!<9QIyp_}uVD~KDcwP(g_E}OhJvG3()*XZqzU!7SCO@7 zyNFVp^Gdk%zxLTR%=>FDvbhgAdsL&T>oWHNo6wCL5r2>g#_4#(H6jo@nD{rt|Adpv^8$Jn}h)vEyR zT<;C<)GS6sZDy2f@ez%ge#FgdYGnyb`qMc{EFYhC&b)hhft=~J>-`>S;@gM5|G^wZNI$wHSaZ2>tnzUY*U}a6E zF225@j&;d%PVX*8Eb!V|vA`hLzVIry^oU{u%MVNpU$L%TlEHW=RN9JF4|)IYk@}~L zGIIq|0*2{!m{*(Wkaxh^D_i;!6qygC`9R)_$;}z*g;$-}84xRaBmqxdjs<3;$9`d( z|6!+Pxw&1Gll8y;%%xdVq=VO-G3kwmYFK0-qFA24*WR?Ewv*2Pv~^~-O=b=Y!S#sQ z(ckolTHX$RsXF+W)#kg@79vEFVNc|L1E&9UwC-=#``Np3eUGg8>PPVWtG@hBXYmvF zOL{B(G{C}+$iv<#2ov4y%I~^qHSu!aB7lK0t@B*U3nG<5Sz47F zmrz;GELb>($e?m+C?TbLmTkKHqt)Z#SEDibrImAH*RGW@pr?XMA2^aw&gw`(Gsduc zK#Y*&AXpYpR=4JJ_1DAoGrN^tr5F=#_ogmm08~zkh`|+XD0&UEyXw^@z{(NX7(`2 z;05^BLEP6P> zNK59WYMh2Wh*NDHIUUgF9H>hts3AON!b$r81C@q#Ob0m$)8N-`;9P3#5arD#T~^+X z`Ta4wSbO~pL2L`(m91nmyQpiy>M0sPN}3Cr2dR`$cOrBemS-{{x1Z47=Q@%IO~ll~ z-VdLfn-$;(6}Szw(==>%db2h z{rh=5jUOfzc(Bfnf1=D(g5Kwmzv?|>k%m_o>(0SQN_GNaCrjtVm=_uVe#C0Xm;gpF z>8!7DWp4-c2lQ|t?29-TZosz}L%`9Md#UsV3DN4y8l;7KZu%zX4eyC}Z;dVmLz8GK zu3Exwj~`?Ro`p)ihDFc@-lUk|JV>;?DlGEnH*>TnPSVRezdpZrRf2qW`}zk~g|YB~ z7iF*$s8d`Nj?VNjtY0!W;iK@h3X^M!aHWVvSi{}S zbjFYIP~=MH_f3^&?x`gq_mfKKdKnV4j)9%N0ILnSvk@m2GUp~`6Mu7W^)6<@-v??O zlSYc(T7nz(Ni3vNg=_Mi+x=@>^+5`G<+x`CfUG*Fi0NJz!mZ=htuF>`y|})$_Vgps z-y8MChF_!S^p%H_6;j!^SklnL?@!6NI<<hNkP1L=R#_FIss{ItH&%}WIn#MXcICZ)6Ulc z8PM%`<33Bkfeg5g+|TNB+wyPU<+zJ3Q{AzhTx+{Pr3&JztmClta0+6}uiTn~mKxf< z!NfAZ#<`GQd@X2-u;->+HG(v(GdP`%!*heMI?qBMgl|DoyZg#I8VcutOH#p>-q7rW zl|ORk7b-w>`ryxP)9@74_Hzl-1+MV;U08oOI)Sn;L{v8v%)?)ESD~&?`308Wf~T&L zN=e5uP&YqO1Q4!64SVxjA#LRQv)J!d`rF0686H-;i#0{e8@3(lx1F!R3r|J_)ICdP z=i)NnjMH?6|BOZwK@gwG zZt^fU-I;qU6gZlFc6T`cuqfd{Od$R(JvqlX7)0ul?E_zvELb?c3_OTm5_RW34x4R( zoEFA!rHXG{cTqaJ@QX-qkQhBS-tH@qNusilPcyHZ8fsyT&R}qd9Db1SIbRd!^c6va z0g=c>O=>OydQDI1tQ_&0l$*hJ{7M(AfIEy}n^ho)N(m2i;qI0CA2kdN3=`Zua z@mEb;D?3Lj0PEgb%fDOc$!Dy<@;{h?O<7*8@lc+aRsT!hJ9PqayYPOjhwpnP6~+TfadaW7bG7a_EnoNxTwhxmV8 zpZepjO}$FfWZQ8e;>=O~U7Yyy;?Mtex2^F~zL0?wt|D_K)kHm);uq zxZCG_X)T1uO8zuer~Yy01)A8x`Ba*Y8e&(ggpkL5SEvf?{=v%rFs-sLV~n0FP-ajN z=A78yjZ57E9!Bb*B-UF$h%X~>XJp84+y~um4sfBLeU1>Gv(C7d18Mut&q2m*#W^F- z;CE0M*PioVt+G4k3WnGSvU@`fO~r8w?5UFEPSdS=1)C_E(cD@w%0BCqN?oc(T4b;v zpj&f7C$<**&RrH+*nqGTpEiC@>kY5HyxY=P5I7S|p|HByqrd#bE8RuX%!Ed8-R9N9 z7o_>3e_xjBs)YyL@Qmva72&zj=AubZA0|C_Qq~y=Q);(XY=hZzdmo=E4!j}ULwUh{ z=pE+t2?A5&6i0YP(54qvs`dTk8oT*wUQo7@kOa$tKy&<3|#C_4)E^X+ntI zx5uPl@fq zheKN;OYfu2D^Z|3GJ3uEj6dmkXnx3xMeA-BvT%>)o=|5z+_Nx3F$e%R@NO4mm{C)O zyHZFBiD#M}d$I>YH&%YcnIR;V#%9AU8sQOFJZ@E5d9=|B*!c9s>yno{D72yB2Jj){(hQz}EY$F)GY%n{?GJd+de!6Bevb@q+#MS^Fb&lGuHt4j zJ#(T<>=e+mtTb#CgZp~0zmGn=(oHbssc|9B*SQbJelA@apkv-N1=k)wGl3T(p+(c9(Q&p+IVXL-i?*1HPA}tvA0Gwgk6SqJVoo zEeDdks{t3@Qe|(CY5|`9t`>peLEWTbb=x?YQ{m_+!!QnT$}GxO_aiwK$M@|`o&=We zXF6mw!Cr|XIz=BpyM9mW4GuX@hf!l|1$O{8!O(%4PjfOT6*Qs3O~d{lT>L)5BQr%j z?rx>Fx@YPuKhCghb>jCyDn*f09E#;ADB+v$tu!|D9Jei2g^_3R#oKGHmh>*A4k1$u zqYz??eLSe?|4}yH6%N!uj^>5aw|>cyKwCcXb9Za7y&526x&~=c(l5D#(+K|*|G_kp z{$PGtCJP3oRDs5l9RT3l*b95~eA>ozciT!HSTJEgOhc&XO>Wy##*@}vrb0YItVH7!eYL7xkx{FTBQ`ze2U=Xp*nUZa`!!Oh5@Ehk)ir$l-*vcBC(w{ zeo2=fT*3Ya0ve>yE9rX?N!FvV4?qt_+5# zP*2cD7ACcIZ>fs*RS2i7ljzY9<|xRI%6=|>LK?x5`|WB-D~i&37bEdCvV&0t za9gq1HyDA=ktWWX^v&LifS2kqdyy4UrI_ed!}aY0e=HFK6)jI@t|&8T)q zC}r-{6Bje<-0#NAKm2;8vYZpDQ8Di-hSk%gek ze_xn}5K&mbamsGklk@jvW!Hrv-e1aDACCr~UdXIO`Z)nu_lY}$==%*XP2(vf_r7vg zAV2&ZUL8;WbKwC{ua2AMmG~z1JIFkDoxdwLYr=$aD)@)7MtCRc^> zzyl^q*u)UZr95Gy(|;LDRac^m>Nn9Ssays3y}xWf|La5#HBMvT3-2e|!OsV8X&kdD z#V?u#2HGvrbAKG~oR5dsS1O7p^?8CQLYHg@x2q;`d`#iZ1cbY?7$dP_#8{T&p8EB! zzJ((7K+a(@!u~_;uk;_0mqF|a5#aq<@>3uf#VPt?cK?5$7ZQ0JEfm8(+KzOYTXUa$ z5VF64IxO6~<>$^*d4;Ao7D=Lbj_V_)oWuF`=IQ$(^-jKm0m{pqG`K(%Wj*U(eD^9e z{AUAhx6>T)`L7tf+=vnlkUQ|tAn*&`|273BpUUr`e99LWT@FRw;t6SljGcALZkwO6 zk7@+fjQa6%-z7At9opi!$S}1#CQw6zwv=4%V2hwcYJk{`S4;laM=7s`V-MCWaXSlG znLTw2_IrClm>{(6>3~_5rHgbraq6!}wHW<5Ii{CID@Swc-$6ppQ6zdzPo(Z({ z3N-wmV_Xh!r#NH~0ANHI>bq*`&fMsZ#fl#WMs7+^WfJ&Mtg4GTbdZsp``KkpC)L8( zuii!R5OQ;qu&laE;~v)x6H^ugQjzd4)gqvgHzdYpW)Xsycl_wbKJOQKP`(Z?&g;gv zZA%9?>Z^I(@yEsohtJryXHL+0?@uaWvCG?)W2^*?d0kPD{xSuS-@eiqKk?n9SZpD? z3HJ)X1u`Y{Z4y~c6Z{x9JhuT{Qtzm;FcL4^rGFlkzxb9j{K*w1MgW6FS+;4iG}!bJbj| zV@W+h*y(E#$!{DfO|0}5+5e)6gg&X2?q!n+F2~31(*ybl77~Cpab+7FF0f%NF69-r z96puI{yGIR3nJm?^V3lmV5aqH}KI&_!q(_+l2;)Tk;~EKTmT= zsp@eFH>14=I=6;wsaeY0IC5%ka~^xZg_qdSes#q6(RNuk1$0*#NH`6|se!}q%aGL6 zD*o%$Q#gm_LEe2=A+GG4-71{3ax5MjByfe6dbpGIcY*(TjF8Q=C6Sf0KqI(GKLR{Q z%}}J1K;VUq7###jO2!x8#dVbCT@-#5^X{ITD1LIL{fOKOXNYp- zudsl`WrsM*ARLWkEiBQNa~HCuvRKobD#jl@q2vDa6a32jXYeQ(iUSwOWv9NWWB!2H z9s5{i@=kRH3Y@habqQ0-b~$^aI0xBJB<8%#{1Wsc&#Rw?-ze3t7pwBD7n9lubdu#g?N@NFQTT_9wz{@KWD52|UvH^aX;u13Cd z%`5fl_=t`k&wo~mDJw31eqOqQwpNJ>%XtJP&h<9OS<-tI+nc{51pWNlmywDuN;!m) z7Znd8-r^=KdXnBh=KSPKLuJi%JZBHw@GjI)y|p^2sfDaL<4xYSPJcxV+1bagX`pBc z!YQ+uu7QTkIUSuRlZN2+ib?$nercf|J($A>kps(C3-k)1Hr73y__y`upReNoP8j})xdRyYj=6iadN>id61I)7 z3hbiS^MNf4RCw6GHDUjw4C#WZFz==J*(q9y=T=(dyf0snPLHj7an*W^QPy%A}&_3RS(+Q!8HV zOelhVGI`70Pj>8=AbmFfe)uUJC#-`ZLREB?A9D0z)Fo*caKaX5EX3PHw?-3oG@xsY z18#xHgksa^hkO+O(gZ%6F%{LI8gNY%!ZCz?;e_P-OX3{iP>Gh-)Mu5-N2~+y#)O2j zrk;c%D}V9v1O@YMv+gfjW=Tg3*`}yX>(EvoKU#zCdiFIs*3GHN3bH7pHi&n`-5?N| z5whW0nFr5kKp`U(D=#gyo(Bn;vzpVlI`!v9zB6k;+9Q1{0gsg?N=LK^5PZcrcM@Lq zsNZC=Wp$JBYV-Jaa!o5X6BYg?iI(xt74M(R#h)Oz8QxMa5iUK>)8VfkEeI0(zE4xL zMdR(xq2RYq+^m`#b`Zr4^dY;w+(USS$6Gq#PmhxE)M)*7V*`ojBzvcVZx=ftB{D$3 zA>lHAp^ZPy<|q)fXD^R+x_p6jMACE7T`WRi-OY}LgW(i7ciAsyh8@BW7!XlbVesC; zgpLBudtMCdc$yse0XCW6Y_9p^l*xCeyYC_1B^s17ti-wIc~gCX?RSofrWmWwT(EUM zOx!(oUhdzIHnf@c|H|ENY1aW*62%Qa2OWO7wG94*a<1`fv0SLIDI3ihVLDy&2MW^KGi=d9b~}D@pS*Upqu-An+!6GeYHsVvIi~}+Ork*jHnU1m z)YbI`h|!3!Q1>y)_idQJ5Ts{ti=RPy&Z8$;L!Zh})^D9gT8OG4y9&;qYB^r1ofdjc zGpVau^PO=E15Pv5Ia*k0EIPx)nc=zKo#N6zVo4rX6USGJ!PQKHA51$NuJHlXIk zNGZdKAw2X)hOOJS`M0S&BaWQdUk=FKafz!NEWV63RfSRdPCe=6>B<@h5AWSEMGPlA zP*iAddBe&{m}49_gF!@^L%J?HNtmh&lrBbPI224?-qHug>ZqT+A{Y^RY?F|MY*KOH zC8tX%Ojn^5EY+7iZOzRRlYoab!bT5}!NDB=|K_!~;SDA~MdJUP{qWDO3EB?n~QEcHoaq`Av{UhK|i$F8Jvm z_(`a2nOhE7c_=4qn76GwJCR!WEx3FL@IIW^Bf9se*-7)>NM|0i-10|8^3B2WGgb>< z_?|}P@n6zcrD6ps?Tz)(cM*w3LuRrCMi&46T3qIe{4%iF?iswmme^i#U5+R=novZ) zr0OJitvY>muWG`J&Ks>nzsQAJk!{I%NbitwNJh^=~YyD$EwKu zM~9|F{=isy#CG^t%N-fxVQLk6>iFTXDP!SXS6eDCBx8a1 z09ohj}P-~S{3CIqzNP>h! zpmb;iqQyrXmPAodN7fQS*+PVnvV$l@6P6G{kS$<_OnV;rAxc7U% zd*5@;ch2`pq9&_Cv+ACQ;V51&WQQM_4wk{hZ{NJ%AAkqj$4gN+L|kg>sGX8?|4Wvs zr;aVig}K#yI{I1fkqX$EZF&t=+E$+h#_3Z@kYJLel+eN|s#Gc}z4sAto^=LaZFaYO zAg`W$-7%vZ%yy|E?FF%II0^kSM7n?=cn9yprxSYC^9C?vCXh+{ZA1D3OI6qC6Q=){Jua0e6 z%b-*5__I%2o;if>4n(gW&=-WJ~Ox`ct zm3G7z*(pFPUosCOI|;4&oCHrX_c&SQljS=-*SxN1djtJnCt((mo&!Gqd7eD?m>9!1 zK&=FC3w(0j8jK81uKGG4(5#DU;4heqqL|9^{51a>k_3O=T3Z84OMuMW-a6H|LW?t_ zx9Xw^j={!)D$r+27-f0ofUU=~mXpBKitL94PGWT=mxbU-x5}RH6pEN zO*g-xr~u6wx81{wg?nI1TM|T06|hA9Id?{uAb=~no8zac*7>AYoFg+@_J|~P7ADUL zB(S^&rnl6*|MEsbKRAUX5c92zy7&li?*sPY(plD5&Z@HrfFH(Lue|-bs`ZHF)8IUAaJaD*$y2e`iBp0|g zk+t^{3*{;aZbBZ`u_ktxF~Sl1sf(s-s~?x9#rSAgyIYD@U9Tj;U#23xSO-59b~iyh zS@pI7(i~@szA6uzp>*%meP671w#Y8}h9JFy%!Ai*yxBH=4M2d=F3$Dzlsf`> zQoZM}qP$rDSAJ%>rUPq84~Fz$pTdq78`3%Dor~vJz<|5&#K9-Pds;ms=w00=W;gIA zVzjwbo6@7pg{H)6yC zz&}LF$(_AkX=h$WlHJsuT?uco%QA#Rc;II$n+yVQRwg>+Om&K^ZyoXpXPvy6k6KNI zy7Vqsrj3}^?z)S#rmxB$60<^x{raIvX8nwIuTGg>C8$?|h!aqr8gpC?rM6G&V3FKs z=wBbAd;k1 z+hw%C4-laQmyPpwaS|-q>c=>@+eZzX=OF{ya2q%am0`KIC*rhgP5LKN8s}!n42XL; zM9ZJvWuGelb7ASS^ft7k)BMgh;jpC5^AR*)E+Jyw`8l}aelHsgF!1RPE1Tdt)Ep?zk zJZ}VyJeZl3_WYW8ec-_s$(W0BXk|UKSs9aE{^;Dg`Fto*;G~re(O{X)y4{{(G+tIo z&J@aR4?Z3+Yz-QhmOxH%AoKFO(4S?LHK|yRrwEbx$BTa_uNpSr0I?T*%F4sjoXqAm zRkLKm!bpccd)0=Ik{9m&ujN%h;XM!pfB}tal=(eWlsYTZ`k&gYHQAt;CtjWi{6klqB05F=ovx1b;pFd{)Dh(x3p zkuJedf)J1{z4y?*An)h5-gVbq>)v(mkF(C1nRDhm^UR*T_snzCYX%&w0;~W4faA(# zz3Tt~odoR}&cbw1BcyFiy8->L8|VV?M4>5Kfx!iA37-{7rK9?>10f3`* z2N$r>8|MfB95cD12Zr5)&!tn)qY_;D>e9?_?g>SoEk&J*l3~C7B0!Oz3b6v~3t36@pA1Iy({Q;=I*r8fBL89$=uR_T7p+O0Cljawh5 z7S^s0m@jVI;Dh`lDSrEwUNdw%tx2IJo#-Aj{qyoG!Id`O{Puqq@L^*>NZpNU>EcAW z+sQH>gUa7Ke{T2bQ+NElZy(j>r0%eKM3)r?TK_CsAJkTP&7ayiBacIs&8zxoy4`HTw$+u+;u-KcUGK*IF&y4SEA#``U%KNEaZF3%=e#+ ze#LoGX!-nWm&^jqy!;Fx_Ba?5Yv;CVwNOqOxf{5v+-OpV6SdEM$z)XH`V)j5+5e$@ zA3^+ut+{v1XTSW~GDQWteIsZ4`S-e`7m2xzbEDMzzQeKix^O#XBCA2{?qwy>8&($E z15<_y@y(`$o0DzKLElu~C~7{i_}tU6wmz-%y~CwSx;RwGo$^y`y97VmR@-AyE!dS{ zu`;1uHBVCOUa^_5!*+B%G0h9^indw2oIXYs-^aF}xP&^{Kt;hYVfm!W47!d5yjD4A zyKPO4i+Ox&{k}W8+1Um5)$xwL=kP3}WONSVX>`PX`Mo5xA^|3Tmsp&Jmbu&KM2dK_ z+$zE&=ydoS4_Dq_L>{JKU{T9crakD#O4>Tbr`ia`I~LgS}% zU*$Y3mzLalUn+f~Dy^k(ID;um{FNo#y6P?Ub$xhkuFEN(JhIwGR4FqVxEok}K?fPG z@}k%YXJjpyUp7H+L^hc8XQsm31QaPCw<|44Cq^pQSUt&+;kUgzTuOhYI=wUH=Q$QE zP5hy9!3oawg7?k&rjUKz5dBLFO^;5tyuuj8n5z&yFbbPIc^QkJEka95XB8h;DN37W z5DGY?W(gxvF`d82?Z^9N+&yjZa;M#Xq|Qth9TToBq4@A|mKG%ejpm=-y(&9>qami3 zQ`cocK499K4W5}(IX|#89C&PiA$p$5jO@!aF&Vh}A^;ja?nKq8j3HDD$qc-``Ifa5 z+I+3d{)E-hRK`z~^NKZ^;CWVb237-u>KdcFsERRDo66+C7Ixfk9-JynxGVl_$dj<( z#w}@$pVUg`74_~86S|}nwZHv-Li05?d3;gw{bNbsHp1c+J#u_fm1i8zz2T#{oNY)-`xyG*{Xd6LVYG)y4yv$#deN+d@?sG{H-X81-tDDmQ`SdLx zk~Gncnvo=t%L%{YMrR%b%u=;*?V|89da%H)BUVSTF_waami1Y+a6{F+EtADK=#(4w zW;MaMZm<{8@B4kiFdNlQizlD-gBL%Wo7L8``0Ku=;6Xr{5HWQNa3PWRLkG8`C#o9b zzB30(R)~U0oNftG57m!S z`zPIC4Xyf7UVhhH3mmbZ$uqV>V-w{+`7Sv43rgLHZo_#z1~rH46dMb^|Eg@eW|6-; zu90;-e)eMS^^^O=k34B-)?at*jl?lIPe#pkC#!p5OBM|@>|u{~W}06Y>dn68eUzEI zQ}Q}~w>Z|_SeB49XP(2u9Dv00Jq@s4U(-STWx3)W7Q^=o1{SpVSG|+3;ZoggE%PSX3&+4hNeVsWb~?r zb_c)c@#lJH;xEge%fQvh>OBOUJru86T8TxEpRq5z7mQ`4bpaOnBL4mKx1$0#WhG?U z>5f4{*KGs8VjhF3Bw#|EIwZv>Gjn} znV6}YOi>@kWESOa-BEowhIm(AY__poi#W&Sg}umAMSU4~jE{r7A{)%UW0vnd_`Hc@Y!P-I6zDxa^;A=Hl}hb^0J{P-qv90IyRZ zkJk!?gA$nWUNe0C*7aI%+WfnTgA(#rO?aPmyl5U>R(Kp)mMwF3!8QL&%Hi+Fu@L=l zS3($4Q<5*wX~h1jGyfncnTl+-1uoFj+=(qLNx7POf&N3kjI>~*J6v*hWo3GzyqTUf z?=U`O3)PGn`%GYUIWRH@ucBTS3zzH>?JXmM2)jek6A`aAIm??zEu&g`t~1&9HsTW( zjFv(hZl5jo%IK6xJ8~Gj9CjqSzMk=Bi1sTc{Mr3J>%9vIb-)IsUO#_OnQ|@Ttw*Er zjk0uxigK2(0rO|R^dwflLW&3=(^mIMwYv9i=lPsFfE7glLKJJw#D8O@w&EK^;x8XS zH5Ig^$N1zgItG5={|cJ_v~U$(u(B0Ll^75Dod1!pD}^p``_`*UIN~uKw}n!?pQ<8; z_z@y_|4fy91;rm-ckr6&dGC@rT^@r02R`sQG(K{x9ohSK_in2k_oUKHj$$27Uc_(7 z8?D-GO;@t)@1^d->)6QXe7w zYWxgjv4D(!d}T%^cR}7kL88g@x#M-Nc3cl>t21J0x-lhU?O3UaFBc9;6m8S~$WasF z1oObeUyU5>z-@0x6g-x0MfMu!dym&y9)4tK4;v$PbDZ$f`yo9Ro5~f%s!=lv$;Og6 z5MqRqon;;qDPba0IN)Hxci}Afc4h~vc3CpawHGCmp{`ZbDL2AHbFgXX;do1-@xZ^} zOQWAJCV+LAj+fTVocq{5N&&DWLwiv#l_oDp7AsC`i{>pn+o`e-aoh5N^D`WYA2+$rBDUp1T+B$`1^ zUYU5`_+3ZH!1{yU3faAisR~ei>23%JBy+)A!wcFFTlLqrK{LIt=k_hto_62eW{}@p z!ycYCq2S+CVaFwl@9eKSEl6V2dOr2^Pu|O1xh|j8V%F=THb}+?yc7D z{5Z?^i{;%M-;2+$@e{@~*E1+BD~-NtlrcOTuPrLAQ=` zJVNfiv(ifYy&r-?*C9?DDs=<;OHD|7# zGuFz@y`?U6RwfVqV!r5l*U5}J0ZDiVhMkoxZrb{^GmIu8vJx#>?YJiE8s56SrJV9a z3t=Z}mz+c!SW~06sem&*GCZfqW;D&pV1}Q$H5`gyBik;e?uC((%>7%}8BB10QZNq2 zo(V6eG)Q>>&Ds0-c~n|P!}V=YK!jv94bh4=jiVb1hF+K;vDt@zg5di=RkwO<@gc^t1kA9@m;Acx2P;TkY#PQ{Yd!!x63s@N7=leR%kL&l(l6d^Q?P zQ+XPLDmaWhXJx;{z#>ccf7sh1NH~7&CrmzEf6)C8Bk)hQ6@`cq{fmgkg>81n&XFVh zZ*Hne7h^t{L`T?Jz2s&V7z-_#UN9qmzCL+=y7M;1T`~Pe3-M{u-`F7HZYOs=Q`sI4 zlT81%&|~t|DTSNHR*;j=mgRquX9uRDdWLg|iEC3H1HFW8JDX}659cr*PTd!rN+YIi z^L!9jFg9z&-WE25cwcL;%~+L{mjrW=6Su(XNC{tKJ4{(a9wq=9r*m7_GWi07B7`xc zQ9yFm9r@a5f74nxdz;`Y`O*`iSt@D*oZ0d*I~1)cLG)Wv4X$QIL(Ae>L%))jU1w4* z=)VHa6KkIm8+Es`5Vih*SA^DaYV2TsXjV4pbmJw)Lm^dzh%tO%AScfWqagkXf+vM% zeQ$jp=pF!DV0?jGw6OO>8#1duOfs>ND8*I;@!0P0h8<=eW#Ch!(;`g`{)Sh!J51L{ zY{4o03dz$C`;l>>Y`~c80Lu(hw21r@Acen-6HM! z$LDqu7b)$nTh}BX-pXis8}BSUG}MAGoZXB9$vk;G?*e)XNg2-7OV9m~ z4p7Oe@t?a+tNSn^Vq==-4j#XpVBv4}5gKcFWt%gTah#&26M*NTtr6 zm+Js_NAg=o6gkn#U}!W+>}!lia$G=14IAN8XmF%Wd3Ew&Eq+{9K1K-R;ki3RJ+b)v zGlN>NZ=h{UZyBuUS9a;paq(l>?>Db}0|RfYR4!y6z!GJmQoMjmfyX-eK5H(oucb)x zK_u=GDDz$aRg>g*ICbzY1tn*?a(g48ZGbXvWDj@Hs`RPG`bp&TwCT6>Owd5lkGzdNM? zMhd5p&)R+@Ddj&E=^|(j)o`6Ssj!%kaYYZQ6?8J&1Gpd~pEC{`1}!oqD`V9}G3(0F zI2Zkqf@Q#wOQznlQ0P{Y=WwIJrz?mNg+BDD!$EcW{)<3w^QWY`yF>=Up!uFbzP%Sq z+s@j)PCa~qKfoKLowc}Ej8aE@G1H#XaSUq00fSCy0}U_KcK9H!0L)_qU_nmD0(3_k z`@Aebtp0NK!^VZj2YW7pJkS;2sFRdj_0}_onrX_Ue7`YSuzzKf-!m&&$jz6%5zn=? zWU^W3*Xbi>(33NH-p{j_9N$Ph-j@{lx(IOW8GyZtqYGGn25{ne9P%eJpkp-oUP<7XHhC&K zuor!wbBe|;_cgmaLw#AwyRMI#fCB%#jirDQ&tii|s%?MnHX(i;WpqJLRao~hpd&{` zc$nxt_==!jolwlfPJd^|qL+Ng*9ZXaYr?wGeUE3CQv%_h{gyXzbP~_d9#f!|xvw$H zc|c@uB!T4g2LB}83TU*HWuYksH76G8g1P}mQG0T<9E~@>(k07P-hugfeB-{kr`BV> zKj%i95S@8(!mY zH-Q|Tlnd6OLSeX>)cuba&p=_2fxs-cg@fTbR?)|h68l0-}6@VM3jDkdi~*2 z>~IfeKAXC7U_Fj4!+O`rWJ;c(&pQ+M3 z&dy$vTQ;Z?^s6kfC4)GofMWTn$kU62biIo$nWwQqqJATQT>(a&wlD4T@S$<@YTwLmCd#jn6T6ZjnOPs2G7PSh5>`7OC z{ZVCRD|sHVM)cZm&eW2Oi<$pG@k-FP81}QJH0rSD7He;G_U;)(ceqg759z(r?qufe zBh}nCqwtl6X)pXpPe@&@M;kL9^CXgSeWXGlhC#>ai*mVf;MKD-2g5S%7pLy}c8Fmc zQr00g7`l0ay(wA37Pj$p{B+k8m1%y6;6&XTH^6avN2PYxFm=-*!0}b(f=g{hgzo;0 zm3m8z$ALEm-d$CPCl72-%B)bSC%3a{%Z>C^6>j#Aj)XBDtb(>7jZPg=`fk$QZZB#< z2iVO~^!gh{#wu(>EUee9CUf-U0G)-uA*-(kyC|$)ZYXOD-nd;~jd+r>75vzrQfO(! zQpA`QJl>iDw3M^CZ@UvhFzc=LUO#?h_J=bY_}DG*(}b#74LsKTn>Ko1B~LszFZYM( z{P>|WJ>!Qr2gZv_DKXdtUf~|KJSzAf0WDX6G>pf7A~{)mCd}glmN?z(esxB?&)#UD z88ZCQL}`23&I72xy`@z?9~8{Nwkd7njxR_b#lsb3F^s-vR#7%FQpZzADyV> z>alo6I}-Q9a0a<#)E#kds&ZNLS07Z$3F;0xaM9f7oEe?Y>4sHHORD>YLn%X)PArWB z-DvMm9kcE(4Yl9Y2K%*BbEk;C_tz4va=G|RSC+bQht$vWdjykUiuk}`G>nIulkZ1n z@rHvfX`Tup&7&fx)@~So$7K$d$ z?%bUu9QKAs4=+{|!k4Nug%PSIKuduq47%%zd=!4nY=YxI2H5OGDs`nnt3hf$;01MC z!rs>$eRG~1WtKnNmTQVI`L4yk54j7Y3 zS&lQ#zj0R$iz_i){nj9JS#T^T+Z@nhOnP>}WDt8X<13|bA@*Gb36I5b`@%g=stA1* z)pkskES|oj&(R_*CVEt8$-Nsqd51?^Ny+7R=U_;(5@n&=j1JgdN zSlrVc%rdu`8qW@O!wEo!6IT^Tb zx$$KyoMK@M9b^m{K?*C7mV-Ic5l`O&c{`l#C`6<+#I6i=ZmNyj{`!#KM6G|!omcOi zDtsjN7La`|eafaz+0Eqn?7HX^>di1!6`yJtB5c}OrM%uw!Yh7gdp3$jn967a$7Qj& z#B1g963nWZ9ab&OhB)kgM@}C+e_saoFhZit*!KF_bIp>dt_780AQyb6s#?@D9eUrK zrn>$RLg_iCHJ02Nu_u-3XUozl$zZFyKL{+hMhH(PRP^|LxOcDqZ)52vk%aG``RLM1 z`bMRO>Y}p}9P1Ll26QQAp38`B(UKw=>c2b)RG^7abz7^BJ>$-+skMg@>2mQL2>R!G z9a4(~{Ic3!4JUT7$reIUWtAb6T`PTtCz6d-b~-;{OZJ}lpU?-~cnV-t3|wMX|5!16 z-u%;E>8n(hCi-ZxI%`?CKCGChSWr&xrPK8`y@tx{c+*f1dG&!#NvJw1Pkc@i4DM#Vq@>Jgp}Nfvd)8fQ+CAQXw^_uCyuB`#eZh4? zsE5$!vo5G(773nd!ahL30@*b3mI1yD_{2zS6cui!$>P&iIl!qOq9H zw2KE^?7`uwbh4ToVRZEks?F1$HYFz}lVP5Y$uVo31S7U>G_i)BqXZ7^d)e%$8k=%K zFp?+v-Usn#_!QcdfSZihFUfbL20r_BW(?oME99MaA~C2aZaIgx?7FSk5`z9STbx9cJovo64vgZ(|Gj#36`$(+=PhO#2uS@tWG-)w ztjBw*z>Nx=FMW^SLGrW!Z!-P0L?1p!PFvqGzkEBWAnA|f;su?w~@(Z7uk?`$~tc6^*a?(ulX{3!q-WLS0Ly4F&gWN6l4i!zK) zS8;UlSMb@q!$wX!uDnr+{N%eqCFFiLh&N`tz4JkX?QIR%dWkToY?FVUv~`M-YY@Ij zLS|LSzPAf4)3=?nA{OOS!$#?{lV*B17K;vfU~NeRt2{3L(l}D`JWJFq4n(a?hm9(&HG$f{wva*g*Mc<9u2t_IU`eR0tN+a^Fqw$lFtem1nVYs@n zp~1s^Yo2u$XmW{lqx<(8J84Uj^!P#U`gBj4aXa9uB|{U}^V3Ms;V*sO^muHS zDj8-{wYI9s589E+-vnw zN?5{{QKCP3IwK%2^6F>{>~uK7roc|7-^aXUr2Ku>tqNfMB(`XUwB*C*39y~_O|@29 zhvQ0RnB@wB!w}p;^VA0u;`N?Z2s+!QaE3T?a*-WIze{LEoR4{`KIw{S=qtE6V=e(* zr6j0oHlTjJOHp&Lw*U){>x10oP;NKHpA1JZYr&$|-R+Bu@BNI}xft4_`(m2xRN4xK zCV?f^@5`spvxQshCLz~q2>)*$$4m*~)P}v`IX1ff`ngHTo;JvnF^K14X3PyDescYs z;~Ukc?{-+54cx|H2~~4WXv_x|TJ8K4TNMrX8 zQUa}Y8Kp9vV_j8qGihH3*U0<#m6dQXjB%$^DtnCI*5HB3EYtK$kGA~gb0X^;JJwM1 z1!QIZD75y&NPTZYv0Wa=v3xU{8Z1tlS-h>Fpin)0`=TtY7R?e6p?W2E^`13`du`AQ zgv5Gzy8f}F@dCaktG@nZVbRcPEp;P)Gy)3GQz2_N zPvR_z)Gtj!F}bOu=QdV9t@jLC&QF?KaNGJZHlH?Y;OE)l)6U>>zPUlTcPS~<^k-f( zNBuD6dsW^pmW}6*P>R3j-Fg!67x{~oQ4dwuGXM@ITMrCsV%xqU6jdViAyLrlDSLw~ zSoX=9$5yzxL^PYNIhmY(1cL&bRR*-&URCwPwg?1_w({Ixf%tfjT#B?`VkV?;n53@g zqwYV)7dnv0epSH$saMhShnB=1_c9zo*-XMvBKpW)t_EV;mHD7pAG_fKI-Nj>@0#W! zj+@}?%LjrNeIIBEX2JFG-fbD%hmo%wuD+5F%YBd9w`+cFORCF-UM=`B*v1pq&%v2< zw!<%i(Tb&&NFJl_)k|Mo6iNUKWh$MRrO<6HCf|=BS{XBQ=~H^uzmqOVqcmcO>r>sf z*$(Y~-LpcR)p{h}Xn9rii#2O#T1ttf#C3^_+GDPO${|el;t2ohyNL7Z)+dW8i1Yg_ z=;WXn9G8$fvo*+Vz|O_J>bk(fnTFQ3_R+Ed;XZ1F?O6@i%dSZf-R1hw6<9* zl}GC>+~@_~j7x&$d*qRt(1A+gK|wjN6Qs_O(ccE;uvIZCsc7i}Vtb3%&lJ?~?;?zZ z1NtJvbEyWMhb3G*`w>3e<6l#W^^Sod;ZyLW(K%~O`6RP&Z4O~oOjhoqofNr5GW6p` z)4>yO?LyQD{rk?Eo)jalu+Pt>8X_L5JhY!lTRa$>P}NVfakR}X`ha<$%;!-8KN5Dc zyjb#C9}Qe*M3ZV#SEYn8Mxif9GFSUcdqJPE+f$Z^BPlARxx~*%2Bg*UVI#Bs!X=Ya zYim*t^>(z!xGG>KP4Vc*!?!7*!cvlY(UTx$eKF{db7ImxTzq*9^qr_LFHz4$@zjak zs#s{zn6VavuG4fgn9zZfjZHjg(oK6iuZdEBJ5ndD0|RUOq~4Y$6xWR8nPw3ZJZJ;$ zcMJxud8y5hlvV<2G9>%{&=JvpQJGUm zjJ^NZUtpxa8C-u$cc8KrYsETUuB?mBdhc`dFikXGH3N|f^&uvg-@bbgYh^7ChpL2$ z!4Hb~2(U?bZ@L)5%5+8Kc%CI$GLqT#F<)6f6}Rwj7`HPux+L~k*K%(Gm(;=L^s!5pZMnB9YFUO9y*ZLfm$bB|g1l0X<>JfkYGFvE&6XumIvx=3%Aap zBkPVyZ%xm)TmM=V>{dPgDTSlbv-B7<+{27dV{*ta^NBIQYm4fwdrVy7LL! ztba_u=z*_IbdVTeazqCS(4TdeLP>Ou&cl;O^=&ZaHT5~6r4$0&E!4mOYs8yEM|NbV zf8l#2lP}Q|b=~Eo#{tUz%j%A#Vx!Zd(+T4X_TQSwOD{SFTWZ7ykB4^`{<;Hdt9e`d zLfowSqn3KW2!S!OSn~{=b=$A_wOHeIr@)K!*%KNAHIWq3&1}e-@CHk@)hWNL#B`XZ zq&`-r`H?PRW17XU+Le>O%qOD2E*?Np%fSF}wgL>4gtrc4O=ntk8IcJ9_va#1Y45?m z7rkvt{^2duj6__XE8F>37A>M9Hx7lzpRV`Zi3vCpP`f#_3Akup+CQXB@@`5bZ@>Do z?zB*&eNiD|=$35x08s-t4f+)mg%{L1a;O}ZPU6B#CJU<9Z@@{qAYsIHV{luPl0iYp zc4X1c!X@d=#r&R+2a3gSb@#I|A1J^wd=-#(@AT#e2yj8i*RubTukGyRC{Rid=AZ1| ze@~97SEQvk^z5aIb~RsVBxujxAAsde$<5KLA&zC5Qwy{67RtdkzGA4qT&F&fA@Xsa z5-V==3-joQU+0Cis!!fTtxpSQpy?gCYK^=e6{NqxGVAP5ZbQ7c^6D_%Y?QA*!3JO@ zp`VnHOP>H<27VV5W%5^3i^LGmqv^C!akNyN#i?{pLc4b|Q0i23H|yE_`NQv?S4n&h zHF>&9{$pv2H{6eSS1O~tI9@Sb+n+l1SX)EjISkfLDBW?UZw+JCn_)sHOz<4W@l{{L z0ter$i2=`*>s}+i``c$9B@Lo4tW7Tt=ny@lL-35zG#h(@G=%yUN4y~VfSPeufkcalkd#stSb3Qm>dH-dY|ECTs(Xj| z9tWO|pq(alsh3RLI%a2&HA*ZwkKDxEHlk*eM5+V=j6Uto)P5_81AOm$?-UWk zxT4&PpeE#0qhqa=_)*w*3x=)Rn-mC7lOPSdyt_JTgzVETU$?WHo`jTmc&&&CO{p)H zF0V~ql#VBd+78$eq_|DoJxR7^HL}X^YjSY({j_9b)L^FU<=~K zi-_24zs0aB{h?|9{DLczyP8L``R9bMp~Ik74GF}^nBtsk`=UnI{J-AmSuhtIZj-)<`m{pp)v7h<^f`tM zf7uyH+VT%F0KMf4xB9Ijh2Rbjm5R+RDwK_a2nOI{ege zF_}D6Xyj`Om$SR>KFMe*Ir@91Vz*wqc_&oht&^|H`eU)Xz`S^YBPXXZS3q|p$LURC zR%zOII|XB|1|Klex-hS4`Z!)E4Ow8;x1->F7}rRQ5mWs$3nvrh}U2kll!6y6~m!{uIwELu=kLG34ghSbwvsuZh=Z7gZY&e$MHaQH3 zvE2iQ4xF#rAK#JvD%3Jqv@GdQE!e_Wd0L(0P9ut+bZzE_IX8ct|N8L2@z)!bNsr=w zc>XoThb_V$d16h?DM}tKU7g@C#x{DA4}YfMl$3J9Nz)_SI)g7)Pi{7Pz0|gk-`yH` z;YGNmUs&FJE-e=+s)KP4e)zDeKn!#1z=~JhP+zgANn4o1eAZ6*dkhmmg8?l~qs{8NDZ zTD<)0WHg2+$@a$Hw?8WR`uBUci%raR_oE113Qc)*pLp`jMZqUubF}YgBT4*at!~{d z#zgXw>doI#h^cF!7xSK36)9Z#+Rbx)rWc9W(bME{WH2-3aZKgt<#z08wo*bgGK2Kz zr7nZH9gp%u_2x&OQ|Ql+K&2dA5C`ifk%$mWU}U-}%G~9`{etp&eFu$Xq(Q5GG`E8d zB#p<>mrXPhah>4vyj}upm|Rzs@RHk6vVF$F#ZCtip9iaJ`Ma7gD5O6>fAy8yQP^W! zxT9lZeaq2hPa`!y!HXZ6slYmA%@NLsC-jPk^XAAv^lIKM)P(KBP z892psF}cxZ(e-yWk(t{?X9f9&KCQP0#06}($wV&|+LV5xi3IQRitI>QtxNhnDho1+ zq8Cs6V@pGz5M?G?LH-O^ud1R8i- zU=LWTh&|G|SBe)we7|m#nSOZB04iM?4RI9I{o^!(ygb{TRZf#isp^yB2rCar@w~v-#_-nQ@sTd8@DWnV}2_Yy+kiFRh1Zw_Wb<2HAH9pZhW_f zfic3agq~VbFlKXw)X>qj%WG2EQI95iFqQ9YZTVhgHRm(P6Xi(SyyOE&E=3Az&w|q+ zRKQ}9_@&<@QE9;AwocZH{krZbhYyxV8|IHBJ&p-??y&KjePuk~me^^zdbsb_`B6_^ z#qLkdvaly%f`X50V_}Mf#pByKR?bKB2WO2S)&k~M+Y#?*7DvTfAquAndW&*j4e%9Q z9erfMB6rNc>N!=&;oH!qyBjL5Z1qSn%$5UuQ!Hvrv*W<4aJ?$W%kJuwc2e0#&jH=( zL5oF6@R_0%R+qfLVi@U}+cm`tuex{k_Mu{ws(~Kgm0|+&MRFaXhwoZbA3d@f+W$Qr zz-~S1-Eo=D!QpW{d<&qS##+M@w->dKJ!0N32b8srnriD<>I? zUWxMnl_y&;zuKPBH`=x2NYn^xM|Ds!q2&BjfY@%`MWb5;JsVlZ&#q$ddnC^A`*%%* z!S~A6S){c6QzfTx=8D7D>0agk%3MJ%XO35{pvn|SA;ss{-&5{7 zm8p|=q>U7LSTF&8-(Y=dI+*GWI*aI%0(%5LXIk%_$a$5MDF>Q{dJph~F^Eo{2N|y5WLcvb>?=+4?R0rXIZZM)+!!KOg``61ql0?< zDzmR`@_1VDHOc8?S)-)PIKhp_Idx%8zLG5p-7aVq@=73%I_s5T)3$y#4ET5M2J6Xa zLUQjW)vLnqEg-8LI5X^1Q?G$9l`gD2cB9(64# zu4b+4vQz^04S3;|5_#_NJVRMU7-NIXZI^(_54-^3Ram)6u$z)Auji3PT$7^?!l^RD zShRLgU&BZpuNG%_A9gQzL#uNuejkoMCsj=beqNk3$6fAbX5s~4&Iq0?Ap2j}Cf*Io z)RP)&n$L}fhf3UD0oQCMj**M;zw)E!h0pAkduQIWfOlcvfpC%IF^#z4@W}0M_YUHX5$J~@Yc*Q|&x1h501FZJ+$+CY)I@(V z*3^B7*mC$g%ger?j$BwvJ5x}b9|x@Ntd_BWSx2 z@|MjDsQ?5qYmrS9@xhw{H>;VhIw1wIOcS*T7tb91#?gG&rtxT#IL*mpRc47HG8Vod z7^C0Nn5-RnsjwmW#mDCh!jLaJm{MTcQs;TiBf{QW9i3s>&52!Ya(ang#VGwj_`n&K zrtIY0)B!iW23e+Q{nGfPY|SboyKr&*-CWEMmZc3HtfdWF64A%MN2hGR0GK;waHIwM zvB6eX3}iWO4*db*>L$tc{y$BAPV?Hc00YdaE5~y7YjvF;Q6Z9J#fN zV7}z~i{Ec*`tM3=xildORW9DZez8Il8i((GE5Wck(vT}OsmJ9f@14sTZs(hN0Y4Iv zKm_{OJuSW0v@C~GWGAVxzmOd!0NiJyqIKOR%+%WYgp~Qq39C_a#i3WMDs-~i8eoa- zeo427{>~KD&$-R7hpgqrP;Seg#B-RX!~*-~XvevH@e&y9U_#>7FX>iHnv!e(M95=d z<{0Wqe&sZZ;jTZ=UeWdc>T!=!S4!&mk(#hSbQTp$-qa@Z%W6Njdx&Z`{3fMv1XHNl zHpB0`9*Y~Pnwbu;-uY}r0S3w|q{h$(q}C9mCeoHtvzHXSxOxF5V~dM#$iigpvQC&t z5wS_Q6ps?rwkDOIoKY9Mz6Tb(7Ltg;9a#xF$UY#w*E@NB$-#EMKjMeL=|7;dvUd{v z{_v=P!wQIo@4gt^iRO{Z^PRs}JJh4b;&|eot>9C1wWJETblyt}Q{rO0Re-aq@Td>s zs4x30v*WyT6kB-^Jla=is}=@Bq~nt%!0Tt032w7l`y7Jd>EcQK0p(&zcxj%Q_zPQIM zG;vSxbflZOf1&5x4AMkBP?>EsSL1KZS6{YOQIE&Os0$sK32Ge(2i{vEyHS1JQvHRc5nuEcJf4pr(66r!38-Tj9RrY> zBF?ZNK)t&4IPNh6|6BiLz`(AYl`=uOE?zbhs?X6G)?%0G8h+GS!kX%kuF;mSrmwFM z{{8!ATi6rXPr0XAL#?rk!>>*SR0A4gyxRM!E~AqPeylzpsk+#E;TXmB?*1ZbiLs!L z^pzpG#bZrIaI*G!^reZ4lrRsEkg!NyfD}ePduG*s6TxE?At*oz#PZtEx;NK&qTbb z^B5hJH56=)t3Q||%kK9@;%+b99Ov3eL@_=;cHO>=`%^5#tOABEw0Jk+OJpjHEp~J0 zp(VuOEjyLekkS2OQ_E3&{cVFb7qF~Xv2sPG44(6zF}VI*`}FlRK0Xe89HbvEV@}O) z<=r7n^=z`PlBy7UCDef*3UI8$2XnauT>6Qw^7o9-b7*ei?!z}HFVukekm`X=9tFBL ztlvI%qfQ=mGTitb&m7K%x85ZmiU-Y<`i4X?9wGtmKk#0yN(>Uz^01)re$JYSGmbzE z7@YS;U|f*&6NwA+2_3w-V10XsI)?fCGkd!%^Q)8(s0!7* zgF)@-)0pMcK+b+jrt{eK=BG?=W7CL`K6`3`Yr{t3?2C);f@-FpT~B(dIT9So+q*!( zsy{Da)Xf#zQb(({I_+nI$)WSs3z3=p^}ylnsS=+9)F2gWE0Uxu^KVMHw^rT{Z{KB- z_!)5QXFu~-aoZv$8gyJ6QSu!Rv2 zl}y?1*QN#a?!o6-<78MeIU`QSqEU~)%baygxk2?%GSS(_55FhbH+@lfB9dg2O30h) z8dLIxmWE@r`Hf9y^kgf$7>hEH>fLltc*V%a%D{9WMJ*{kWwF!MeD!UE3NCM#{$an2jZ*TS)r7zUft_6JF$AZ*_&;H|}-ve0xt5i)_A!HE zf=Kk1#m5K8P6C%&7c_iUa(Q!ercQh&r#5f7#6ZH^{lZ&>`k`c=AKzuj#ozirr1QPFXZq0V07y(1L2j0l!=R0y_?mx+53mE2TC%9dzm^8|R3 zGpv!tDtG+C7b!O)5#G3U`ZWXkGR3{&vvo4p(Rq^wk*u{~tLyHuxiYqKWKoY3PDPWN zaxMI7U1Owx-V{*)?knC=`dPf#F-3+-21)h%0hKh*i4y~`?NaRr#KZZzHDmf9t+c~; zL>O%M_MvvhL!)bgHc3QWf&sw1^N+nmtj8RZw(*O_QNd%8NfQ6oN@IHSUMr9Xhp0gv z88d8!6aQ?P>_WY&&EHZZKwu~oRen$=Pu(@yo4L*lS!k)my~2jd4;=A+ocm0y+gtIh zeL1}H;bnhH*7<|oi>sU{;DJ+Sz~0^AAf>Y{wVs5_!>qIHWko4W=iZ8FtV(ySA7443 zK4?cZNmZfm5VW5Ya=o27fCq1zeVDn9`r|zPVRU4uBd^=bTiW;TA^vom*~{mTm!gE`tgAu0WYYG5_Z*)PC-yY-5VrKp@O&b#>e zO~s48x!&EHm!zvOzJ$9s$=V?6A$!YJ5^+={i6ehu;)yHFdF1-zd zxt5cUmvu+O7w{AD_*(_)T(}i4)Jm$?4|>UyIKR4}FEicTF_SKb*3Xl4R2wG#|06bC z&3=hJrVxbe_jrq76mx!-#!FPE16V%4tnL+dF?Ev2b2J%>An7UMkNx(|O^7sgDEu>t(h~)&rR&@yOPsJKa$8n=06)D~X}8T==C^#VNicFa3r;`dWQs%TSv0&GvN#X| z9meEo%;^7?Xz3ZY+h77qBDhYGc+C?ue%GCn)rfkYQiRXiqZ^0N91O3ReO{u;`jFB0wWb0;Q~HNgF?s<7FdUlmNldFXO;Q1qvNcahk}}lP?1&IGc^1z!mcpFF zhMR-+RQp>_QSFrnZ{LYl_x``LpE=n7k^Qt@uovapjubwaki}Q4iv6BZ_VlX!DftXE zh<#o^lE3rzfW_08f_FCE=W5u2PHQ5)AyDw<8cuq6Wm)lg^#yi%sILbj=hk4EMkKaB z%fuNv^m&6FJP`>G4oZu7lm=aJ(F`8*@fBphp~E?5db2+wW!M#)O4PIgT>tbI6qt;R zsPR@6z57_A<7)ei)`*u0)%8%d&|tfgZ?a|nLnh+wYPJO9Qeu`;Cis;)(?UJ1-1@z7 z&8aH2;a@fVZ{$owifpCd5BKGu(~nxS)J&X@DdcN%D^{YTcU0c9nk=zS`(-Fk>=$(r zw~e=d>iqj@fZB|1c-ux%kw!dkVg5NynoZkusCoFxoT{yd1Yr|8Y1H5JpZMsW(z=!m zbPL zar6ZG!w~6*S@hxYmFbDHSZx>z-IclA9eeQEodt1&yZz9@KP4znIDqqliGL{T))uT< z@)<=Wv-})KG(_wiXI#Z;;QS!M#8~XPWz6<|bc#}q(sMAGkq?`W3*+!mQJbMe%hEO$ zSQ}!X=Hkf8R(;4`E=-cf-Zs>Ld@fHJ*njSilCb<;r|wK@&>tIc_&`+cZ$d(fN~4bp zzN|jC3NM2=OKvsd*}%5*xS=MGMN4fq!jshdu~2#I zg@-3riDni50KLk#u%0~Hgm>~sKWiD$+*bp2DvYQI+A{xBx189~6jbf}4i!l0FjElQ z113?A_^jd1G~C2OHY$9E_WM?pj(h>uQjy8Ad44y9RZ{unlAClFv6LyTN)A4(Uw@5c z{?JRH*3aw+Qa1BNSpM#5f-b+8VNZ}6ab7m3{*-bXA%hw`YvlI^Np)On8V!CzJOPiH2oS+g)m%dLb}NxKmAZMzGfNV;kGSpi2AHgCTMH*P3y=MPxh<;ZOGVy}%R*TZfS%s1LIROT!c4TultsqUGJ@w=aM zwjDw7ieL8~Q#mL*mzU*bL)&N}Mw-5dg*9^n?&a+~AY|8wopL$dzbJ6*;H{O;g1rx; zsk_+3&0~LG&HJMilSi^f&OWmZjArJ|xLd)_Mvw~aH&2%In^Hc=MJcI&ds49Ri=WX+ zUx2dxlFRPC-JlgSQUOVM2)nE2>>GzWPE{!G+TF7mVfv$)~30f3u&)%*9&mspu>|M=m5W$e#+N6VeANJ zYSp3cgS$D@EQN^+n9tEm9b>?sJP?nPc{6*D59B^Rs_y5h4tP`s>%I5wamQ110hp?| zU;OeOcw9+hLCh!Y+w!!bvdo8z5?`#gwh}!3d-8O<=A7KLUz_K42+O4ogW;ykdDH2P z+j{l5&Nvki%MLXzxL(TqY>RCr@^mZh zmfc;Sk05iX#~_x7kmRE+{k?f&=yv$8-Iq28{TY}V%L<2DLy4}NRvfESTVtq7zeVz3 zD|AB4G}N+sWs1^!WiRUoMkM%xAw%t_o?8`TZK8sXN(Xjcb-E2FVxU164i(XW{UP`s z>c068sA}!@X0=vj zhE4lT-{)DaRDirLWerl2OZ%?PLuhI0pUD*Eq@TS z$RheSJS4U2XBb%Nab?FNDtz;GkZgYiU4F0^OKwdJ>PDXSERVtc#{NezwceMobo%0f z0|knT!B=%Sk}?z38yWV~E7ktI(cuDj_=FxaQtUgW?;YE^%r%2MRjv=lpEYeCJWQNFk%< zAwBvsLj6G%C7?|%R;xuBCZ08c>ditVR%;QMs+^ur?ecbd_W;P9+$&6|(F_;6+- z+~ES;`w9v*+)wLSAREd#a)TzVoMazdqArnG|JSI8nE@L%nLgW3Sas}I_UuSM55I!i znf2}$B|I!tLcZ7cD3B=-A&fX_>n?F>A51SVTTclMQ3b>J(yU49PtUVk{i1uf|8fxO zH(hLM9o>shAaV(B3^8E4R};XWG$Q=H$cx+TtEPB?n!zWDj^~&hzLh$SiR@_>q)P9U zy>1qX@Oq+X^YU(fZ|7jrz<6Y$b~AE)i9ZTHDMw)icf%)+=js-oImzlc_H`~prf zT2VH(or}36PG35xGK${;4~s$>B#4D^=5@SWKNNLLP`EuKj;cssL-Ra_$HY`V2?Ei*at}D@k z)xy3G{(L?{f0j}|U5L^|CjOI2RhH&u|5_1)JdcH;LC!lAp@q157mTAuwVN2UCE>I_ z{})-7j8@iB_nVVN*sDywf~L!t9}L=;-yS`F$$U8}h>c6SYG4UHC$75;EI-^X2}A{V zh6uq6(gPe3i;EB|-^aCO#@J+IIhWhr-bgg;1P5pnv&?h(v%m;?_osI>tfxaMUyWx* zBFkZ+{Zl$$YehEjwJmHkT?H#HnyG@&u-=b-(J@|I!lo07kc6%0S7uH6Wz_nX@YSg* z2sSDp8_j{C{c-v0@f4IDIlZetAn!GdmmF)rdvKv?&2)IMXDhypcG5gOMfu;@)?Rdn zM^oTdbnR8>@VFiL%1s)sieJMRaQiGy;WDQDy+oX<@4yp)|Iz5>K`s57kp5CksgEOt z?Dw{&lie<+lzU`lrm~uMXzTJPuu7K(wL%l~lkYVXeE$fvPQr{w;X|pIKvWX&CMmE~ zQrV)m`g!$lE~&i zK+mN!16oI?9fH&vX9=e5PQ;2d=y0mCdQ*PX;*a+d8k}G<;zF#M(^@DMkT|qk7NoMi z_KNL?Z&@+Q%OJXDk`g8qf-vx}%=QzudFj<_P^x5Yx$57(90IxN+skZ#>Y4dli+lYD z7JqK}W+vDFm}!1-rV1dctD#=hJKRa+H2QpC5M8Jhc(Q-o)oFxZr@g93%?bDx>O1DW z%Z;M-GI~timenV;g7Aa%8=uT!e`x9? zmytok`nyE;7IKrHZMi9&^UjQuxTo`AiGM?=Rsn4ToLDh1-MNJ9j9N_+;sr4ag#ERW zlbp$I3-4n858!LPRK8ICT42-R^kWjATt=0b1$=|-wbl8Id#Xw-xLG}O3k#RBTp5gL zNuTa%G|TQ120iBQHYOylg) z*WCktm6an{y;e_c?T^8NWxR0F`9&KZQP|a*$F>?iDAuWlox>qd#nMV1OZdF(ae3M8 zL8qRB5z9&`t-C!TqqE%#LNECi=NZk_E!CE_QT+pKP3amtp}+6n%xjo+L1|EJL>cQR930hGo+ zemGPpKdf)R8ny>V!`OJZAK6SQUD6!I&sJYhiPM9YN{v4$Do)XI{+}YE=!Vx^4f4t? ztH z%uih%7RQuo6GoeBy;y@bzsXSz)9{LMd-sSbN1?FZ4@zAek2{-cKWM+1`=Dwj=mVav zTJbJg4+6{4Q-4UE2~psmYs2L9!`~j^h$$G12bVzpN$us*ruwML*ruThyM=$!r=6e@ zyMgxW_X@xTNdx2NQil&wZKZaWh6R3qu7GQhQ9v&dUrS#kUt8Z1nVJr`mcS9}VqFBc8C z=1p8fITy_n0fCHut?3+`>}au0Jiy+1)h1cz=yby&1WBKkYU1^o#V_6WksxMyML(Qv z;Y~aiAYs^+99DqhS6OgPJJ+UMoqE;x!U@P85amSIt~@4vo^f=H=ImxxLUBi*2+G}4W9cf$;b zl(dL+cX!v&AUTplNe$gKFhlN9pZezS+53O&{eQz7jsphPz1DrN>pHLVbMV63Z8vvT zh_Mci|8b&Xv;Hj#u+Q6T4*Dl%6a$b#xMUAeFWa(Fir36hdLQdOo$6lC44Z#dc{ zycaxj6ok}O?3V8QM2BYlm-2GD>H@5N{SB?uQKt%JWpFOlDX49P1dcjn9XCQXW8U zkM+@R7i>J1Dz%!$oRs%~H{tA4Xo&}}-pnC!@L8${nwm$r6|AVvFo_=16;^WFFxQ1&E{pz@>A3pnWblo zwS9*WNmLH$*~7b_V(InBGrc&17GeVt0*8n$$BlwpEQV{nAM<)e(Or-Hl{Ncn0Ur0% z96{w3z{GpK0KfO29^>q_Ga%&jy0Q19w=cKz%)^B6wQ3{eJnq(Fbuq==-y^&?CaRO0 zU^Oz!YdJA&OXr1Steoe=p{lCS#ZHXZ#XVCa-~>CXqOmWKUCC@|YG*91vi6>F`iu_a zt-kr)Sp#mgJtd#gFG@Isy$&`k6cSTx@F{=LF{WMNV*`N-I~#Mjoxv<8-cx{p2MSv% z#@?YtCqX_9q}AnaXA9%yFMKkt)CCX*vAa}TdJ(J@kjNp7AA*LhsI^8r7Nf=S7Z??$3AN#_k;uS9 z8sJYH!#Is$9@#0S2t~|rupu#Lo?k|vN$|b?0}_e=K$o^f%g0InA16h#LDb6-5!0p5 z4<@w-N5Zqtd2CE~-p(_-ezxi)N;9I;UdZOisfuMRWx-4e$7i@dp2T9oNmwcuz*Q}< zHB)L`BB43efZ8zmJ$}r7+Fv?eJNR4o$y$08b3HCyjfx$EXp&Of@X zBR9w*7=!R31!sai3)jI;W6qM(?)?LzX+GAlHMU3g7wVj%v+W&>rVj{MQY0OBmlw0s z*|Vq?{l;BYG+|KTw>H#rFq@?-Ps00b)E|P_RQDHx?XYtr?3-X(r`3vfIM{62_O}*a z`;J-G@x6vUkMrUz?wE4$LC~y~r#cbMYjKov%wj4L+V8pA(lo6+3XCec{Fu{i$G1*G zN+xaRNGK5{?Ri(a4X{h?wZC)jjtD%xzcm3DF02NtleOr!k8A6provDI5K(}4TsF@~ zFFb4ngSXslbh$q?_(3Ke)3;h1!e#>FQ#v?Id%kV3-Bs!5Ms`MLW0c^b?n;6Mq1!D5 z6%%c+m_AmEu~al{g3YindZ(_`Z;_PRLvqilPx;p%wbneC`O7uL-mY-o>K#pHisk;Q z{@ajUb5(1!+^A9Wd)Vog2lTtI{p1ynoZIih7bV-?HkY%l!u`rch{wKfGzg^nNK5u` z7m7N}bTG!TytQsovR(C=q1E>m1Vi)`NBHiXrtOEju=mYlcFiS5b~}-bJ|ujDhab*l zwhFZ1C9o-gq^(0<^si3$yjLN@d>YZ;5 z&}LCs@fMQVqHZ1>pWl6DeC7adX1z7k>DUD2kZdrce+p>9ypHP{z{Gq0%36pfGsX^@ zq@RPMc zn7Yui8~?%9lc|bV&q@r{ys_jgQL_+KB_J5Nf4@5~NBCYM&KObj0y!t){AcMZ`Mhe7 z;7o(}Z~|cB1ken}u*SxjlhLXEVE#bgPhNG8-X^FO{5TBXA9W_xHD^YhNwX$kR{U+y4f>3TmDH<& zexZbj>iL^)99q$d6UUmC?{y&@qm)8A3)qy1D?B$uOuVRngwh_Iw$g1w8J$aKVfLA2 z>FBR+QUCGP)fFV+gkxlps@bnj!F4tu0H1=wx<qe1uJX=ls06D54Fz~dpq8<@JsAj%eN^r$g*@c_59(Rwy31FZxaa!46% z7kA_EEBKVMoLU3R3y0u@IrB}CG%6|GTUtVhxJ(LpIZbmMXs6&8G1tLok3IReB34h! z1dBhkvAo~kPhV$#R=%e|xRPAr{y&g@Q$I<+i3dn4kSQ4tp&*gP*2!?rsol&ZjL$sU zWAvdEGIsB%7qb#@w29T_lNMJm`e`VoGlURoX5~WF-$dV5Wwd8%s)@-oC>W}DU-s3C zmT#uq)Z#;UW}2JOH+q2@qms@)|k zF1>-=AMJ!$y3KHJNhu%qAUf=O$wLE+Do=Ohw)6+}`NT^vSmredXQsouaa~kxwJ(V& zj0@O008uWjanP7UdD-9aX6OrwL1WV~kRe~fP?7mCw(Eegys*HcVRG-Bw_l|yDB-mw zud*}I*RSTSdd-okY@WQsG}`o%bM2mF`1BiQEz_g=d41HE+E_PXV^>wx{1@hZQ;@T2 zzONN!NfNTUf`cN@SwSZh~RSOn%6^L2A80=A);-TUU@&^D2 zAmSqpo^TqAa$S9Rwm@gXr2h3HemK-_Xsxx}L0rPk?&_1qC7U^O?=yKK;jvI|lg{p! z=8Ek27;2yFo(7U~Xp0T!gNhI4OA-%COTF%GA%Qie-qtHQ|A;3dmVM=UhFKiNJi+Tt z-4D5*y>HJv?`0?aB-VXw*-vDBS$D8k)Vnq}6IA1o&C$%@3&1B7U5spw-N_fL$`g{; zHo9=9n&B;Jt{E~#QHCt3RV%3zz~PA&?O{jncLk6u607gl@A`@v)$j9>pI;bST`aJ% z77Uy@BnA{lpE!(8%UkTqbL6O@iVYy%>1ySCXOq>Z{A_|SnBNCMuc;xbB{fnCcN8|`Miblt+BFZN~9_qN3_ELYN0(x&HzbW*E{h z-juV)GM&0~z0cJhoq@J@v=^bIWouLt#QHHmLZF}FQv9myc}hYIVFD?cdCZ&`e%BDC zXESax#0q>Zghl6d4sA*tsg)3;u2TGikXw6&Ge+CgRyr?BrZE5M(z(nT{6krv-;g=48WuN^d}CG;^5s&g_h!I#dK>;0Rw z4ela3C}s)4`xQlNfwH)fY=Sm7*?<+u{p)Z=`%LVyGsqG+TBNSbl@^0l+B1&H`p)R9 z9f2MB5hnlbv#k)T-ko#4%XxB-_T4j_2*^vOt#^5}`eE2(!KFS+p{{AJTvU4O-xRg_ zvMVUTq5DG7#YbX?$Z2fEMU>ApSz@yX&K1Wzjyx&tMsdq`Mg@#iPM#^K*e-4cI`#unjKHbc`F`k9 zEoXWu^KJnqqm0vL#Q+9bWQISAkatEh4EZr%-<(h0`P$1wRk8SvGdj8ZaQh%kyR`zT zbeC*106eE&idzAzEN%Fb$TSleIblbz-Q4r?bf{5-H6wJVw0UUgC@g0l9%-I-@=r*d z-81b!s5GiSsI+Ru4HEa6Qp6&;e6Y)vKT&N;2lrSZUURY@$opxap1VIp3rkdgKO=R& z(ztC?&uLtA^=$s&%tvbUMpb*-H}}~F1HuO)vdE;9=mhU`o|$|b`d=(+;E>}(*Ez2E zrdmF{F;qL;?Jz$YyM_0gs_cfP`-z@J9_RX9jVr8_(B6gWHO^1gI=&CM9wu|)+cw8W zD~T_xRi0VNBM%as0tJE)%A#7-t>}^aYgsIu-ZvDN26$y_U?`#!YJ^x#^#VfK-o}l>rS_ddoX|4 zML&{-{RViN$Yt8Bv`{66&D397T^k6|6A_3HX}1@BV*UJ?S<#kvht~TJ{C8mb?_url%uv;)nzK#g3XUc{%GT9RBp7{P+?T;cboc$z)Z(Y7tq~ z{1Vl}!S=6&xZZW)$vvO7I&8ixR(@qHY@0^9ctGr81kVI^iCyEzLP>93Z1R-zMTY1+ zTwu3&CUmjO&wqqlVb~-18Nzx0eq&r$B(EIhV8%#aZ*W5Uxcfs+%X?m?wFg%dmlWo* z{2IRVb&nH9PVM$(!ZzMnSH+Z~Ha{s;wXp_BA^yD`Rw!<_2y3ZaeNTEVx01HNd7$fc z{LM)lK)!H9an9p)#p)A9RL(se&xRTQC0@>K<|JJ}J2R%?;m}jlZ!b+)q-hzls0MR3 zHB(XvFg_TpKAa&KL71Y32rS*k73QhXIXpSyM_dMmusCf(E49R zq=rpp2-Rt8yxQHUn(&PM1*lnG?y?6rRJs1K21DS5$_zuI1M$I-E(r3JZ-yr+Nw{5& z>R@$3i90-5I7{u^iANw^DVMFz3b5DQ?CWNNACMIwT@#a?W>ETtkQ&Fl@K8qKFSne(v15awo1YwQ2bU#BD7;h2xA0&doHi9lT z66;^+5rp2oe74p~N6(_Oe6=(xPhP6{ly}~CWy>zr((JHytmi(L(#_5KTzJ$yZx`;j z+)jDD%b+?&WVw{TVa2=f`&T!}DaM@p* zTwRt@ET0d)YwLVnl|A|IIkJfK!&@8q7IQhP3e!eu=`RzNIZ~C_4%c~m3+RPn`A^?u z2z!*Q1#~M85kt&XZRqLEm%r3RNHJoEha1;X`-fQy>gxen4?HI-O=jq~dt{!s!wR%F zrdA9pMr9VWQqrr2UxD;hQPU8m$=q5Kff`?yy0aH{w*o~(2m@Vv5;yf2;*lKCS??g* z8@^!{LFw_UKH7^vo+{mhmp&e{^xW1Bu9Ck{v63oERB;cr6WOp&K=M?Q>>SLp9t#8= zw=;4D{0@GN+>|3B+ER|dO0M_Y@ACzI{xr{$HV&MM+MkM~BWaYRSJoT%0LyJ->Ru4z ziK|syrc!-Ua2dDL*sJQ6Hx_ytfW&{??Cg^J!C+-lPE88-EZhqmThvEL{=p!QDBM3j zlNU&{`f%rS*PMu&$pzhRW<1Y$taRY1*4Nk+bFA8wrCAv{la;r+mZaN?z!VH?Jqn{U(y%d5iO)Yt`ZvFDB^PLKTf4zdOSg zAJ4YmcRE+R33`|Cmh3gJ=R-h(Nso)jU-_(~GHY?`q8ZHaM}DlmFl-FDnAco?!dTzM zwdEJG#AI9qyGojK&Gl|(^qSvhGeD!0nI_J}T)dbk8SjDPbC~l%nSYaI zDpI|BD&g)faC@dmlQVU4cvpEPX;m@TH6~R0-HU~i^SzZ_KDB?KoJckQBiCaIAb~x^ zmp|I#KRBB%IhP@NLwc&Tkq?Q}~1sbC+d zc77m9lE$uhfVwfQgsEi*r7s*Pt@C!fi4mc<8FyLo7mlf^AROMUPKkw_@U^%Y^V-X) zS0d(8mpF*m?wdY;A{OBrg^zoJ>lT{OH}=Uwj+|H4e~!}AlW+TrNkYR;M2U&E;y<8e zwtf#>M`dS?W@84r0NxS#lqaUmBX)vazHxzGT@V|*^vBfx(m2I{B_23S%2g`CZS6M; zqLyMWnHTpTL1xL`IPa*d3SQhORDEG|rVc1@l#kLgOmUwcNzVqGK};>p3L)KK;QFx& zaEIUPQ#mkyi%-i;Vc9dhIFS}}zv&Tb^XnVBHkIT#($ z(-1LqpJdf8JVV2|l*SmPl?YISvnr|f+cyVpu_O8TufInoeRut=*m!r>Q@1z-#i4ly zvN~n%F2tvbDm4M3GbXCkqr4|(OMjgh7)QmcS(T$45m+Z#v2g+y3~v97bv2g9)L`VC zeBwwgP4*BW{fnk=ZxlaQ%J5<=)5#Nx9}cD9d;GI!-$=BDf^|@3%t|{-s=<$zN!x{%6z*J3^raKp;!vd6jN?7iZiK~2 z=_T#8c4NfSd+uFNJ3$|laTOsW>IlIGzlB3IAmYsO;v9CHv_tIx(afUGw+Ma1BG5jc;vIr-!=}zY z)mThfJK~bWY$rI6oMROqr!8Me=5q5io6!yyv{?9{?zwTm@CR<8r7O}IlE75aGSvW1Tv7oH;pKp&27ANls^tX0n`H9^{%|3fUIdycbb% zB@2W6#Km?#2YcnIB@CJ7J+_C@X5H^C?x(R+(NmkvtaqMLXQn#;$h0w@Jd>_;yPSsk za6{!zB}YS!qZU3aqwuTA1jRd_w}0jg`2=$c!VmwDOz6&4RmEE=`XS7)pw?Vq|# ze;tA^I}rBBPb@wVq&&jIU-#|7-+en2H0DSFuU{FVzAELIvz+{*ZBp;8hC(|H`-!JM z4dm`M_wuEZr}LXpM*v+}s7KlWq`TPcy31zz> zU`l#-z+T+>Pz6&xJ#DFbbgbWnH4?q2ccH)LwjyP_O&&#<#SCKQ@NV45Pc}J@s}pyC zumSe_XFBZIo5!M3Ai1$NeWRd3Vgn$Bcn=Bb&SN;HENqyjtj6b3>Kk+s zUV&F{4MI6=77#jJ?t->rY%9kqLy{dP@kK3QJ99&a4Q;By>EU#LK~gNT)j96le4d_mY^dF(yVuHLLg|eE83qhLv@xEHSnJyV^M6L!f7M!z9o-bo3R4c8ZK}^%TjQ~&Vn~zLU1J+ z!ulA+Miq`&KK9&GeF%wF((#JSD%P{haK9V*z-u!;G@)036!PqxMJC+I)>GD=n&0CE zu6xka6^)sh1~ajAOhDe3aH(@7Hi4uS;wgd7K=CYpci=(4t-i41 zEh<~d7K6MK6@HJo>GXCH?f>m>Sfrz;6cHw7m57- z>2M2>Zl%9(&$#1X*tGBm_A{p<(@VJlR->p{{cD4MyXsTa6f)7w=wvAYkwPdJcy6cC0MQD5qHpU?zF4yzFho)9q{_!=U}&VympCeq*Z|1hsk< zVqZJtP{1-J7ToP7f+amhO^y5A|(c z1o>^Duw1o?B*-h{5p>S1Nj7*2m7Qs*k6#3rV|#n;CD~?Q{>$@6VoEA^IJjY(BVK}x zqv`43$r#k?T-e^sm?$*hje)45^m_F5oBa%A6^dW;j!Z5P>DF9k4e6nHr|D0a?o`@q zhJ_+?C)O&2g9i`mSsW=^kg01^vn;-oCzqd8Yfu8bQvfH9nxl7Gxpz5_^&;tJ2z}9E zYfnEVmylioz$6cp2k^qhMIaiK9P;xr!J5gaBMR-$2o+$5&Wuj_KjBZFu}!P=he}#r@XYGD0gv!76T2h{Evpbnx~yA4FlR1zvv`sJt7OR zORy`~N*Y!9hg{W`PH8~3H9)ZbeVa%19nEt=pFTTtF56;cQsof7hReJmKQhD=+2JU( za;4E$%vOA!XrH|E?*w|*)+mAA%ijfh8H_C~Up_gphr}$hh@NlQ;Y65J3C4E9y1pB` zM!Z;a*Wx+J<}V3B?fT*gklYgrTHI{Koy9OM$%h#7nrG|*F4LpK&mKv`ySj@CA!>~b zy71gLLq$ENK%*^M-Y;zrL=h^VQ#?)zUhBvf(OGIiiI}Xu>zOk}Ty4@at*yZK^iUk6 zqNsj3uC1Q!#f1)CFJw`%p2*8f5aGe4q-6R3(4#_^_m>&e4g4 zZ(hA4r4W43%U))1q3%xG4S%JgGq1P@2X?z6&<^Md+9*}Iv4j<4YZNzb!ryX|cD;A;!1!I@3A`5N*lj-iX z)yS+M6RgqJn}4d2&4e@l1&aQ1EeCU*O=#^1`$)qDb?#I)tvXZQwL-5?2R9$l#z z(aSH?Ch}&uJ|g={4D^NUWI-c;vD8hT-FasH=qFla%Ul>XA7_R0&uo#woqxj?dFpnJ z6X_B8qcnBb?+XxJ_sI%AdA>-I77%zf>CXDZf;F=3JCHg$=5Qw@NCaB#WNDL0)8lDjZo&JE7l#0j1nbTFemEvkm7pDX%LrC z05(f+s37IP*xxaGbFZWuAkLz!G__~BLZS&XwVN1K*OnRM3#kZ|x58 zo>@7)(`vT;N3mPXh_H&-Gi;0Vr8v%elonv_8s!3{fk*GezjxX-5m)~+gL6+6z%?yZ z4Cn?P9CtMuqQr{wAQX6%hLX%}%}Z%w$ToQ9zG2gXGFlFq%71Nd)L8Hgr?8w~&J^hK zdr==U)tw)&Ls#O3aCZ8(gVdIgwe}R8;iXWUe45GuHAQpg*c9s-#Q;=#tZcx1vsE-V zdUg7@i#VK&>u`&&wJSJB%$$F^fCK4wr}ZrYQ4T z7<+7BSjYIl*;O_?OuYH~_}+%g@R2>LhL`>y(4@_}_;ryhxJxSC&o#xnetCJB%CTda zwZH+VzOw9k$$;-_Up02;`u&Ytnp{HKr2TZIY;(sXV~Ij)&j;c~l+(`cE9CZAA|Bb) zTXTB(>s9ERhMP|$P8^LZ8WZ(}^04^vaye-14Sm|@ne}jdiW@(JSbjq}0eY-ud4WB~ z_NC7FDCih9@oI4)E^F8|qCV^BTCP9O!)W?}z79Qr#bsFmz0e$+oF)86^>fSk10nFMtjZT81y(aHj~X)K%Ges3F=1yWDeo}{g+?yUt0iaL0`cCnLo zL~Sr+)Wl4=y09TtEjmd5XC_XCVVU+CmydhYZ12vSK%W8&+HOd)te+{G3XV%%;m+fK zP?nt_Y++5E`~ ztq?5APuhAV{U4lVZU5pd3xB<3|KF@|dShwLf4~4Q_R`;LaTM;fhD;(O5nL(n&Lw)j z?rpWI7GuF+CM+yp!vn;t!G3baU5E5;x`@wVdMgD2?|F9M3S?+5K2dQjZFnGO3YWR7 z+lYGoycd}{)zqhI&8UOixxTdf@vi-s9k-~6kv@7z^GCz6C#!)vtD+ZE;a-Dr;rTv8 zbvOs@>Ql3UT@}DC_KSYJO7}JM(N~QpcEyE&=9NlRhEwHhfl{h+w%nJ@H!PTEU~C?o zSfii8=v&;)*7*6)Rxdw&@KoS748W+-1^9}G(GIl4 zk1UC6U!v`wj=fdUEIzsHzHp>|BDZrb9D125FxTco#%iMY%&pagc zTn=fLSKk4kILIexs;+|@_HaMipQnX&hYHp6GW z*=1yA4%P%A9#K5G(w_MyAuYy9%iR7AlP!+V^*KwXKz{FJ*EI+cJ(c$9yK)kBy>%mXrk zdwQOCa^II*ZxH_ih6cM!(5k;_|Dh+F-hQpMbtC@2=*hf6Xw?=M|Hk0;3y}#VIS=!n zTGl?H1@_4-naCL*c9!2J9&%iOId2bfB!=m@`h+H-U(n>RnEuds)hT{%@;~K#t~fF$ zXeFqiB|jH0z3!VCSN)Fq$W)xIohKEor;mhUu?;DrcG%V6gJMio=&|E@Fd?B31b2Ea z7oCSmo^n6zgGb4MQV`o}KOLZ2XLq>+?^KgmyA1xSc)I0ZWoaYP9`kx|sZk{P4-too zpBk_&aVxwQr=~{PjuR4}Cw7Es$^RNCSFMRBv#}3(w`GH-WTVseVRCVDjmi*)ksA5d z`I(pIsJ=C5gD16mL^n0De<<&~du!OqL^i~v&XC#j;U*^4mW&J3M-xV-EOp<$vVFfm z(qty*Q7|7tJVmD3Rovc-5bI{$aEk3vgT0=0y3uNVgz*Tjk-R_$YZvTP!A4#{Sp2{*$pMnq#T7%&b>U5!NQB?>t^>DUCu zs$~YY0g9QZkZOq(DhkTY$@b6b7EiD(n-eRq?9nY1Sz}AwyN28fp1(pN6UWY9<$=lD z!nSL%*93zbTq%auXVo%zM9|dmPb)18y>ZyOo;GMxs)G}ce(@6@ueB?bzuX6X(JYMhh>QlVts_XtFo_OoEa|k$qz^+kbhXBQtMgV3x(d zcPkr*OYt=0H|C}xm$gbk=aaO4u~Q^k%Bb7!HJm0?=7P_A^3(SVcFvx`+f+zOBEbt;IW9FhJ8;T==QjT)8>-*=h*5D zCx+)SmqJRF()PCf`Ws&~7p`}te|!#mL&YDtsyQHPKh?g+;sebgp0+9PO@}1-PtR^L z*nEn~LI4(eQvs%SZ*L_EAwG(%ZPl;l6(WD8Q_{|s?5xEIyIjJ>YL;uNF8Nz`ip}2D z1Y+sCUIz4-){XKCTB4|{!BTgw?0k5fU?o*=1uYfL5Ot92bwvt(^~lQaVXef((8udmD`Xc;gmUPd!tnsusDW292T{!Xbx35+@Y+!ejL^Nu5*W;ff zoJMtr#^R`QSAh3+*{IgyRrKQmKJc=y+33UdT;7^FOI4xTP3^uHi;!E(vAzSSCC<*W}M9#VJHvb z$QOX7St&p@wsc8m{g7(^`&4++>>)K7u8Go=`3nf;{MS1#5{3NB(rwna!r+^a`2^1k zr0b_DK*Z)aYHf3#*p7QmiO*>ICa*XVBQ!8^C#$EIs+IzX; zUb*r)r9{;ofqR#mL8#~R!@9U@xmj%^Tc}pAy}c&O`R=>mmlV zTpoDWM5dW@mA|tRpg*XuiV!mw&~uc*`+-}N?l0;&wkMc4LYgW_i;`_~Pq9rmGUs98_OX(KdDN|?Dfo@}^^?zT z%4Mn^cI*vxu^S#WhB+q3;XA34z( z#eoRJ%Ag0~VL-;>a3_Am;|&rOqjF#^`=R<{O#EPr+eCs_e3rO*2Ybr6_iXU@RwVFb zd*JOi1?S6CVQT<ZnUa+eSd@ zhpSnJVl}i6by`EqJ9OW%NRbmTIboRqOX#Gg6bTMRIn;NCF*=k(32M@}|o!CG`| zCfJz__IT!9q^!mDe#gy?ll8Rm7d4`D3Rd5*f3`eP&n;qZ)Lq^vXODl`b*92V@FVfE zYi()m;yihWTuzR9u`t5Taf_b3M~-@=5HubAf8BXR#aBz?)b|J&(z2XfTa%CcEH%Na^{M-#n~S2^P32poBihT}h#Re(%Nh?Iaz=Ex%Jm+K`S z5{x(saIwcYbi2t_I!H@O(&Scw5DNb_?y?WAZ^VBY~!IB$(aqts=t(r)-FR!OghLj+JT+wUB$6fg_>DHRgK zv)lT5OVHDlF}KkRsEajtpu{H0vRnJk3`hKReNK5YVV}~(!CFfRZF$*Reg+d0I(96; zamtFlEW13?1TF;IE6QY9;OV1mS=+^gt)Bv+hIhgelS-dchMq~MdO(p67BIv#G`_^&8P$#xbE&7e z>EmS`HZ-v#Vp*rdp3Em{0)H$lfwL=ZYImtUv?N-?7&+Wgs@c%ai<*_j`*0{6%qvbX ze!=)6N0r2{afX9qm69aZ(rp#uy$J9&pSil2Q3(ZG%xe5_`5pFE&Kpb&oFh`_!JUlW z7icYKr86H7cRb4M^F#gl_m@lEuOhsS;CtR}r+Z1_WnH!vC^P->K+R?9uPq<7Wj7dJ zPKKr@Bz5SO(Kh=r!d~B-qo%IoAX56q@+ntMPo+98jI~CQ3Ne0ND|k+3E~v-n|9Y|Z za6uSv1lrt_asl?*SN-q3|KpBV<}T0uo4sx~1Qnrq_F~0#P+*DJkCH(N!rmdRN9}s2 z`y-EZVBg8YQ2hY<-ofX!h(02jjXyH_2wPrSe*5;|b^~|8H7?CU!JRkNKls1;j2=6_ zW7QvHP@Y`;N$D9)jU+AAV-r4!vAG`IjE^wno3zuxCA!7>;;4egUz`8vW3P%0psgFr zFOqsB*p^7qNy40pX+i>+SH=_i*Lhpal$c_154>%`y@|FZ`brM*D;KtF?BQ|gQ&w*9 zJGzK-TBF9n1u(`MzL{z8uf;Tybwk&7ekx zIu@jb8>OTyelTTdH64Fbq=$9d%KX#9Bt3S1m?>=fdzQ(X9j`m>o~^71m7p(OT_49@_q4tj{EWgp}h?^j3eC% zmWASnvgv102n&i#lfM{RE;yXc^P-snmJ2umr`t7M4R2Hq2)}gei~@% zX6_iXk@&qaeFKQT`FDD4WHJ~|!+)9vcJw-I$sg0xXW#qVqJg-+oCQ*r=r_=Z4^DKHYrqtHD$YEBe~7?Aky^AUd6IOm1R@oNaGq~Qz8Xx7poao0;$*&gI{4w{0hIf zq4Ge#x1qd>cSfZS{SE$DGduq_Pzb{v8%OyIGy{vc?|vm@gYPzOn|yv&(TLSkDU`Pt zkTZL&MzTY{g8IlRy} zNu*{gkfLY#=ja?>nK|VK<}Y^Z8z;;5$eoMVMy1*~UWS&_1hT5j8ZLaam$~bbv82dx z;&p|Xgeqz0=X5H_Mt7*23r=KO&nZF*R zh*-12tCS{yV$L2c@Xsnv99*^CN{}h7Eq6Yz%UH7;oO-}+OCTMnV{jTElC)P7%Ar7h zH#lDU_QtkklbJmxMwh_HeU=VPa&lqyWjcPcU^;7;o_T3`Ysu&h+aFB6Y|*J0F2z7j zf(b(cS{w)8EkJf~k#FGF5*qhh=MpUrabb8HnWH}q`Bjs-E%5w>z#NcvF z_HbPi@`F6Cu0lT*rHz?M*l4uB^b3B#Vg>!+y(dLKVr68>&aMk?pzR5iujzGu85ZB0nbBN>1A^9pB`o>#mG}xTj)N zdC2woK4ax;>YcShfmAuwVxmk#lb4p_FCM(cee~RVZZe&a#-xM$Oz0WkR?K$&y>1qywY6AX(S|^ z=_Aa96XnZwmMLs@zw~vD^1J zNAToD;V~JqBf-r=Q`5}CDhr9;8`VsJGjL)j`tQoBi#m^gFOMvtf4ky;fgS(`=+Jd? zsXOZP?_F+0G}SX=bj^@qjBat=HnrtXWSu%WlfHn{kYSG`lw)aiu>nd<;2}bW=3m1m zn%7Aj&qs@Nn-hq--H{@*3Ej=! zp%A70Wed;bAP68Vz;d83ie^^)t-RRj{XG6HU)hi8%Ox`Tzf>}VepWK^PXG8VsGCDn z=tt4>kKLNuZ{74?imLxS$@qU{;s2M~=U;y5uMbh;%0q6}d6#+TRBp?G+m zT{8;a5 zzc+}#KY;wWrYrQS3uol>qw72T$fFf(W~p~a7*zJ_KmjPZGl$TR1#s#&Q++wB__d@3 z{557arY04m?$-*S{&%B@lr(;2A1>d-2q1~=aV4A1W8q18(OO}up8-FgH3sj^Jt}~UdlC5=(~;BE@WOY=|9oW;?Tb1xyLfld}I?7 zN@sX(Sz37n%-nDV8Ik5S_ZxHBS_w)vr$-((VSVA$#)L8--c0f2UM-mF2?$1OI%$L_ z*%T{#h|ircdU5weWh8qqb``I^#D#B)pi)5Rsd^XHbQg~%ZoMULTBp6R(kMG4`a_C8 z!hWOpC7lyxZAqM1r-G7TE+~6--fJjfQ+39_7n64W-L3CC-`wSvhR&Zyoz`UgXx@P? zvT4+?hhuht>}>L%0AMd?1d$)2R=B!btKg3=#v2HBJ`Y~e@HiEus19p_p*hthpqW4l zz(w@;E~poH3zBd)n_85I5w1Uryrwh^(7CtQbj;j4eo5i^`^cdS@XJ>PM?Y_Y${43D zX0zuY{`vR)v0GwW@IfhF+vjs00LI+V*8aB<8OLz&ayQ%l8Hbh*QPH^G;*Wduy51az0L3d;vQ4oMqW%%_rfwjTm<{aD775 z__Wo0Ov(%K-Q8^!71R(8dYWqe_^}I12#s?Re}Y(cw4w0-kMoI?E|30YJ~4ml|M`4k zIn(veQ&Ch(&h)*${ysWe>~{{oX(3*(1>_*Q4<^0myY~;>TtMX!w>gC8@YgEMGEzbX ztNO`&DQF0V`jYjH$pwl8i4%3JQzj0ZIUC*w4MaFldtN^nnn=(k|5gQEyMkZWo>K0= z)n04AVr$$KZlBhlTLmF|uJ71M2YJ>&^G7MHdr2&9 zxD5rpyvGFRxH*c`Gj%=5j5sTgbO+Wq@DCeN=YnVF-_S^u6E@BV8s?qpV#{BdUz;$T zIPNY4Zh`0#1{kSSlAGrLi?p{4i)-ojeOE#Z2#{bMJOsA}f(1{o;O_1Yjk^R05G1&3 zaHnyX;L^Bz2X}{Vnx^5h*1O-c_r7SVT*2M2=^Jb3T^tEj2bwd$|Q%yE)@LR@r@fS_IKPNByd$&5o?& zoT_8Udp1z@%BK|j{3ME57N#GHukhtbq#Tv zyb!A;ERv8w@a6p6L;ZNSg2xTwBt^Hq0Bq>+50rp@GN#3eZY3rwK){>|7~Mb5S?%MO zuAI-|$7X07X{#oq!;Ga73BP9aWv9SfpBK74R)(Xi4pr3@IRCt-R@g|A*oZ+gfIB+U zO`LqkNf8PD_a>|b`eDr3906;@I!(UxV;a$wPEme-8D5=YI|_|+nWX%jSbe4ikQ%x1 zqCyo>Z*ky$O~&+iG8<=cd2L+`?(VA(XcCRia$m7Lm%I=5Sy)(t?$SEctir;S9A#(o zt4$6ENadwz_Wy7MTCQn`%~EE9D0PH5@w+rGX{^7w3sq zlApZ2jHa`C!qB8zV6pOUR+nng0Q*<=ik-A?{vP7b@7b`XsjO-Ek8Jtq$E7LDj17&a ztnhxXG)c|mlWqsP)ygAUej&=xtv>Y+3>Ba-we>V~(Y5dcT<-s}BDl~hfRlXZiq{Yv zMk!$(A&}nG^J#Cb^6Pv)&iS!3V}_Pz?a8AL&D3X7N|Kpn+-|(##adms8K~lXh8CO0 zC?J~rW;>=QXA|u$THNL99XbzXw~Zf9nj@oXe3pwDM|O1O(Op?u`koBM_b&6W@=yO|V77hOBiRx3ZuRd`_9Di#4;v721$ftB zG1Hb6eRS93RA*>i$mV!MTx>P_hm0rr*PLur23XW&AC|*uu!<0BA>q0ikumkMc`*Rg z?<<>a`LU!}^T}hU2FU03Of~B}OIR1`LMlr49b?bh56Gt?S^1=wZ;#^!CY|fqM8xcj zKMM{SSSWEAcg@~NfT`#1pBj+0NJ6pSWV!`V_ahV&ogyPHhH@0%(1d>8A&x9gbVzOf z)?4y3pHwza1i-bTm#ko|%-EdBqPyR$iwu9j6Yy)r1h^>W30hb{`%Wozg4A04wu)K& zoI$%?FHejwD-N4IXO(w?5tM3M%9_h`QUp&9$v1~ueXXoh`}$en)t)Gfol&Q}lCuv5 zqqk-=9Zqc-{3q?{{5%yF*SYV^JzO3S+VC7CB*Pu!NOBzTg%8i$ZS9y{zDl?Ks`urI zS_>V1+^(|UEi(HeGhO5DanUO7hm>HOxhjctb=I!3t3OpgHYb%z4Oo`E%Kz}gp%8b| zhgx$ffm*uK;$fzIbtv7So#|BD=p!fEZmZpy(e$3*`tS%nU6AuawnR5!FAqoMZo-nj z(ZbAa)`B(OazCYK=j(~d^io0Zs6!$9!w0jVEL-h4?7C%5f%7eR({k@5++nwO?VyJj)E5^*$Kr*f za}ON&5(PbVG{?eGY03$M;;H^hCpZZ(ouo01_y4TAk{>^o4l$>Dh&ko9vTx#ov}V1~ z+|l9R{>o@6r3atzmnDr4yWo&PPc7&`=ab|p8HxoBh#@|*+Q8jmK{wO)O z!bMyj_q3Jn7kR2c+I>-pR0?9cBXc|shU8;bVMwh%GDWvY?4;SBsKdna<*L~LLSr0J z;S;BAvv4{{KMTUaILNus>+gZ;K&~3_-8!y`K_`;AW(SN7wnRRw9yzK&LHgNrTyJi1 zAR}O>*CPhDhIt2r`>*Gb0tnX{gUx8a{$?aIU=Xc|UhgV0(-J_w7S`DgU7}QnJA-b) zt^ugyRqED{%21|6f*F2~Ka`C&LwzLC4H9Qp-SQ_YM}mTm#lRwEp8Tr1jKo{6f)r{1 zRmuu0{a#}Nn>s4?^FNch>pDby&YU`#eh!!^t1_%K^-|nt&mr^?oP`>jn1PzHiqatE~-OQR6653?F7Ray681R2~_Zm z&#gxPm`Y5?^Z{9j8FAXXAfS%XL)Ol5tLG${#PtAi`qBBOcKUfROq+mFN)#>$e(y#Q zpQarkI`YCk47QJONE)8Z{=PhqYp*fT7xAJYs!9yWMh7L|Y8h?*5U%JuF?3-#_`fPjIdT(;bXNhoKlG6d$B@4q>bPXHBIM|wz~&q+FooX%Gc&@wP7PlFQtfPi2M+z-^W8o?%CDD1XB#`+|v9`o%GRE5yU;&(+RCP~H&@+fz-2N(7oa{r# z!Cvjp@xAa^fd2ssdE972Jx;}*>)C0wWP4AihswBMLJJoYt8cAKhfBK26x^Fxug-e8 z>4JKQ7Q4T?<{5NX4jW z60|e*Y|OObbl7z()BNp4wf(3rfZVF5qa2Z_^p@wKeZ^9}!p#0S*bGpr|N3r-K+ zU2mVnF=2K*FpmpAwCQ_=AWe>30I!W^F07iAUE=hz{eF{xPBvx`x-}}W^R-Hsmb-5B zS&BbtX<+tp-!Ogl=)XMUoHqXWbqMKwv+l=D3iV7AbmVy~08cH}Jm~<#U!)pk;XfQy zkfUuccD$meWRX4+X9&OGGH0dT(|DVSf8ra26W^NRJkII~3l=_arC99`)FO8QkZo#q z1pL5>sKep7;0h@?UKHc3*p(V>_`qy^`y=94+@&tLr}_JuQaTh2DAIpXfhETQaBuIW z(D%qB8^GAZP8qO+qMi%4bETjcrbWnFXt5Wp-j8&ekzwYFrtnQZPiJyXNK&0(qZZMa z*8bYAKjUd(A~?FRxi_&pOhfxg;j@aKgy#d)8jc1mBGg#Xy9PT4E@6~joy^CgC_53+ z7B^SKVzfUi4b(u01ICiMvadR>1V2Bm3mE+(hecsuxT?Lj-7L+iZ1EcW`d8v`AE)n? zVfX__V8O9xRr_E^WkIl;ZS|o6I|(blVkuNVa$Zz)x)06vVe6ClE!@U30bJYPze3RM zD3ZFCuuvQ3G+Czz25lYFm}W{-LNksFZrKX_Ny1KENIN?Z4h`iLx9FYBkuW4i*9-4% z?R7peP17SP0~c0=hf4qiR>~bOF7o#;JcqfmJQf%JtnbUZY88x)Yif^4KYrq66C#4y z*FgUa1Rp>U zlUbMU?q-pfMcieJqL-3GD|zrsP1g64Vr}oPbiVit`bJ`SUMX(43$1AA^g*^-fU>kQ z_GMF)tiudmP6;9^oGXAwaPUR6i@x{ysjaZXcVCK*k!&dI`?Dyd){ZyJizeKt9CWcy z>drf=K(A?xOGRr|!oEiKx z+%w~I&5loLtK`j%Cdag^Qd5l)W*Xvp%2eC)4p(b7{=Ng?JXau>r;}Az$_#6bI1s}fN z-wdo>ttcdMC;^v#p^``28vEFC4SKxA;r2#GH|{~~rmWIi9|><2M?4lTKP`669=Ovl zp_Y@ONE5&7nP~y!k+T4N#aW=1Mjf^R)qNaqD<`5UYfgfUwR`O4ct)%SI&u1ADD!tK zH?#bX)kIa*5(l`;0AxAfV9ICsVCC%M;`elW#PG$^J-;p!F9{MQsoB#0@A2%%zSKZc z13sL1Bs><)T7)mDV`C#X;_Wvod@Fq3ETKM|g}ktQ*PhLt+XVnwpyo}v>bR__2#=~7 z=Q9s-B3^iY&P+}&O1@w?HTW@YY>Wso^e`-=HV7_8h5c|Sx;T=$xb30k;xx^A-#(N) z_$s?P+W^8BCY$n8&g!F-xJI+Gp=xJRKfX87m?R!R&VtEK57*7DKTo=KIlb8C+~_>b zVP?_btPc_3>(2redht#WN-5dkq*v#?&_L4E9|*IhwF4;g7GY3+R1`1Dv3)77eqFe- z;*@q(dAwRELOi8kCv3Gx{$-rXbGr9$;FY$`<<1zoG3DaAi;E~za{&AM6ua< z$!0^VR&!dB?0K%xe(QS<-GA*9c%B}r^q-JQc~oA z6R!Np$-;2ZKCMdUx3VT5#ydj6UIlXP?)>}1I*Pcn?ywgJSBFMYoU1PGGM~IXr{L`$ zSmN^NvaoUW^h}71XCt-)c_}?g&4A^sn92 z6}6uTJ-NroJ){=ct)Q7~j}*<$o;o%w>X|w=d6T!r|7^(WS$QSa_VSNx!~nLeIICX16#7vAh|ABBwb9F<$WwEh1Fm? zG>HlJ3rc#J>rrW#R!`Hptt;9u0r%_6eN`h+Yf)Xz?e7~!U%n5S$6-; zcZ2PR+yF1Ksg2XSzRQp_V4*tDm`hjptITSGcfz0=PzmC+7YEBr%sl2AI4f6`&evy# zXZBB|w~f<;yx4Xgn8TG@m4`hR1C&vWVqCF*`P-C!jEYOpkWiDu;&QS+N|2b58b>3o zpFOJ=2(@T%byaORKLlh7!Z1w3I>zA`Jyfa?8Qa z^-cdcxb%q;`}g8u!fb1xpK||ex-eRUK&M460~oOmG9EDxk7dQMEIKw+f1<4k3Ef;# zvy$>zm)Oi=cW_?(8IL{diwL_)q$8-XUMcLaU7r*8P&=|fcKGNe#7rUUCoBoXHa6Y` z9`QHjy7jY~vNx{a2gt$fg@m*MtyoQ&Sh7R7X{#NMKYnQ2GWn_7vG1(q?@T%tY*3AI zhz)y-ha1(=HIl|17Uo3ei)Mi*v>0_;LtEb7c3x$lSNf$gtT0R>U2DXE^lMdZ2zDY> zmI{@Y5`0LMyxY2TK`gMriS9)@#CS3i9&hhaRT``v8g4 z^^0F{lEj1gnF}I_MoD{0DU>knQLg?ppFLMAg&&El$H*Jvj6}&eN+V<3qZ4r!b;cei z&zHvFCP$b~m>^>iN>x7J|3GMz8d~ZgpFfr~4!1$T%sJ^l_jfGTvK3Wya*& zMZW9KQ(T{8#|!6aLbP25$8sHEz-bGfPs{0tcBsHgDeYyr9;wJdh`&Q&rATDl_D|c%aGt3?m_r+x*Y0#?Udm*iJWm0;Gj^w ze5L*T+(~3wQKnWZotAMHyh5u@`bwwi8}Ge(o%^5{rnDid#X~m&Ub}fH(o6%>pI_M* zU%B%zM}EE}un{n4HWuGd6(Cz~^4Ow&E(l;uj8OKyG8(~Q5N{S_s%NTbAGt2uxXN)x z;n=9^uQUI2D^ye%OR7ghw|IDCE`NP|zIi|0^9pv7TUd(r6-3ddr^Y$QIV2=i;+Hs= zN7ASj)+_J6Ep0|nG4;&HoZ4wRj;$>N90T=V{fn$YB?Ukc*eHjoVY}*mS33V4cwnt# zTE?0AxIt6}7VgP~2{I)|4BHW6Zu%B~a{YePqV!tvtu&HHg?&ywVJY$A)@?z-tgCIY ziEGGrS*m%ZK}D;E`E~LVGmHX~*SyR4A3nZQGyNs2?>qR*;ae7&U8a9RF27Yn#zLQO zMi61vGq&Za*Rl-~6-iNwiHq+S@Or3{)0Gs%%T!jcn@#mYq}IcjwLWI25LI$!KZN`H zp4G4A##|ulb%#F`_5CDb4`Y^$q?&<-geT~=wF;&(a=fq@9-ep43?GWU)M|4ekNpDV z%v^>bDaZ?EwjRs)4l()vhJ)m&wJS};zF{&Za;WTF&GF7~M+aFg?MZkV>cx!py`U=; z>xEor%BWTWl%+nlt|5P@_<~m$B&<64%7ZjL2XK9y;8n+}`}D|ZX-iw*SLmtVOPPav z2AqpYIIA^E4e%fqoCe!4<5ItLnKH*n?Xft;uhQcNziLJ&Od;XlW^o*33>jM2mgG7D z#9d_?qxBa-qa&Ip90XB7vBuO8{XC9*L~BR%w{L8VkV+KUSg$$IQk!=?*wdQTF`u zsbf`%tTQ6si=JZ|iyT}Z0(yHrK3I5(o}&x-f#~*Ky<`L``ZC5r1N<()2*4;6<3#Mi zOlDi{=IdE6*t}A#%sl}}WMjA1!S89cv-x^nbMIud(AH-ogR46GCj&@A%Au1KtxC(fSy^p!gcATVa z6;CGO>jLav-|4l$H%lNx0R)77NjOR;M4Pt9>&IF6+Tbq|vVz-0$o%_cux2+ab3m{* zLe4#YdEn9J|C#vJH&MAHfB@AU`;cSqNU+Z~!~_l5eIKcz(*S)DeGG2y=izMnNfHeo zYo+Qj=$+Npk!iAs<)*y%=7WOGfD4=qIy#qwoyU1Jb^t(~k?V3-C^dDSV!X@qz= zQDY{74Hxg|pB~HbdbLCmI6_4PAAf)4@ zPlDN)6wND>^?m(;q_X{4K&Wem3G2_=%o(*Y(Gr zZO1lkksB*`(X3vKJF8p!3yqxIvxgOIm(PH$w)18Eq_=bHpa&){{=eFwHMalpJ(Lw^ zp3ygu%*n~gi&T96G=Oqut$GDDm?_3T?_~cF+J-X2M(Lm%>`Ka0mbb&*2*+IVFZNpR zF7vLkg5%3X(mr?RDO78<2MOECugPb6J(->Wj6#qzgZ?S z_X~LF@0)UZ>(s$l>5mBWllj}@3nU5dKvey?{meXr@yz_gq9FR|)mUy%U~n1*KVn)+l`xM0?=awY_`M>>*TI^_k2)RHOKgDp9~<{iAAuk$V<8{koGBFRe>VPy z=NstwufG4~2meRo)2>fckB{;1tAF1!%y^4g|Nn0M%r^ETSo_~U`Bwn&@#>vLuSmuJ zy83swksQW3>i-IO|Cf9HMAeG_GRgm?@!4kNKhN^t`wMiG`1@691bB6uiipx0(e2Bx z{3VI^^cCiL-(+`%ftg)UMTH%xrdq-mZ73gSXqY(to^Qd>6>aXzf9~+)@F=SPEse4J1RXZH0xH3_K994S{p}Jp( zUe>m1)H-cTs{^BZ5AKwJGd3;&U8t?qsc~5*OILcV>hm2ja~p80vW(J> zkq~n0jpyj?JVSH#_shG}%dH*5kTeC2b@{9Zh=yv4E}q$TEk{}2bLo~36TKwWf zJTj!(ui0@>Vtcc#pt)m4pP(o@I{(na!GEzgm8le>>>aB_4WOZK1jJu6E#lRkg!;J0 zvh2F~{f3z|zt%sj>*p}$wMp*76Bt5wO`ofbs)k>JL-$7GjzuG;d}zv7Eow}sGt05d z&BmF)HK$W*Frleuxa7X!{G`}tOGUeu(=Lh{%Cd6E&mHP-;pbOLDMph6X}5W87p-=> zN9>GT)AlGXVf`cN?T*f~mC6rwko<#R%Qx3Jg^nA207ULrN?bgR`wWBfOy-9^MZTdm z7{9*Y(tDk9!Z7qNGR@k>FYa#(7KbRXmxY z^yo{sv<|1Q4NRPN)F3i1)OFPNMq3Ug>}FEl)baPV9C(K@tv922=%41uHn#sZQ@^#Y z7W&c1k9=WZ(&CS`ZyLI6Zy(5MCK-P=-rsPS$PpHgaEV;#4l9r^56-OjuZUXAAqB3Z z<2kKVpOqtNgu{5pnJ~=w?VXwfuGwED@;lo%r~f<;Xj&|drFKSkk>JSGK_*dOz=97h z!Hp-^K&-SeZfn-;-QFyPn*&|F&d9`T3aC2s@qkF;hrsNL zi`nZECyGU%IDgGLK|P!AdU`TmyFCpp+0QnAYQ3#@n+B~v2(`Q_%B4C&T)eVMrYRgX z7!CZUt*i{3E^HlRG{h3V?UEJ%QtV7;8pNtj1U#&8F=x+8Xe%4}Zz_6qR_<7bJJ(9e zNyUyAIqZmFu65nVIH#ZyC8o5@Jj%+}o%HHrF3uZ4IE(Il^>n9M30b02b{vI3NU@%1 z6C(E15qZRvpAG6CDEm2rX3->&|I}@o=VJ2|*jK;x&KpinTmOqJyGgA)T+#7F6&s88 z#Vy0I#dtAx&(T~Nmux9 zwdg8_rX&@*ttSyD<%q$c-sVHCKeOVsd1ybad~b=#630oC#Q9>pA9oonE%Aj@YI1Wq z;01TK#3_A~7RNo?+zVyw)RG#qOqL#d^Ec|L*%w{A171f5vg$%+u6CBDNasg+yxC7J zL&zmco5s$qZOe)3P!tEr>weaA-GT2@9W)+IeZ9AN6l`3YHgc3=|Gu(AkNEOsJ1pam z%J1D%pYC}n3&|no{864H0?3YGrgJ!^Yx}*1*2B47-<03VccicA5~A_>koQ|CYEq(* zllpS|(WNIa>xTFTCt5JgiYSMY_|NZ(+~cdGDZ;B?w*&P@?fopN^B7) zol1-H1_L;}b~+Gx@#wI)-EcxXg?VBljd|R+O5VOkx4^_W!JG)Zv!=V}4?Z?Hbjs6` z_qWitj>D%7wHsj$hnVh)cnNILVfCRNyQA&Y7H09DP5_XPM;ff;@&|m{tm4M+Lbu`n z4>tpjxFreyKY%mQZ@joZNM9arORJrHE1#ZXP;K6gU67s(#XXt|$Z+t~+JMTa?W@DTn=H zkC`REmp5?m+vKtCw328REX&JFMh5b}vk;`FZEUPcs={S;v~bHF8m}zH*YvZZmDTXE z2ApEDk+PD6JC%5<=!M+h8kZy-gb~L-9k^0FF1cacit{C4tmk;_4Ke1B+_d=o&*DHP zwJr0jMI~}BGU6phC$FVYn6CMk<5_V~RI;wUs_zxm`q6%{oHsTdHue6J7vEyRsK%=B z>Kk3VR@DQXO4a!6RH?#@kIK>2m)CIK$-SO7L~Zq?p0V@;WmEsk+40?Spql``f}uO{ zvcYm!&X#Iiv`_b3wDKOt54vG@!!@cmD-OD+etT&N3grAr1~a3+)BnIbtZXEco#_2< z|2vZN`-#iaGd3Q3TrP$dUxw|&z#TAc{`xx#x?~uurK_GJjj7VJDFj-++Mq?^8tBcO zto}7U{3}=0?&)c)F28N_$R}wCoyjqvr^aFMAnYU|W<+;+U-W=}a|RGPXtmehFC0=p z`R)g)+B#9ocluCLY%&Uu4JzJoL%%>eGrFjhwuhqs4&(s;FF+1J?=RWE%N8wS=%8do5AaiM7zo>a{TJ}yc}f5din#Y`+@1B{CA!xdUi?u9%e}p{0;e@!h67PvD&5@SH)O zwpN*m=9tMn`h07(gR_3bcryvP(B}92(!6<6)gyMw9Mx8@W5&E2<$by{CU>aSP9H)< zo|&+#M_zXw!skHM>37F1kX|m;gnWC_d`wID_2tGwj6$V=<#hOGWGPC)|3Rh0`I0dg zRi`8Ptkz+ac+PVAfW{*+t&s=JNg0`VXs+aL%FheD{E@EM8(X^_lsEObk~l#ek4ZkP z>2s`fz`>{`KJRYxL-CWv?@n+K#isf^Y1H8DsRmf>O{3ubr^Z(OElW4BpiN`nqmq8= zcqrGtZ6G)v*47hZE7I-R|M8^hRA0-Fq}g*(pqqACNkj4T>@T{H!&@s641SW7j5s2I zfekmr_W0m+C+|`@F^I;7plA>P)wvPwjlU4X7)8PUS8#~@dORHoTOIzC>9aU8~beVZQ zMSmh^O5Q2izC>(e!&IIRmmq=d4^tuDr-#9qwo)oV{B1yx3JzC6h0zp#zy+tHptEq?Zc99)m@O zufNWewgE2l==^$>TO%@B6nO+b8$U}}zSNx2VCcs2jYq}gUTEopeCfj!H4MkYAw}X+ zNJUx;BaO|VAI0q*Y^d{+!mv9$11GjJZP0tlb@DggR5S>-XMHVZFSJrma-#BDcTe*> zG6etcZpPsu`B|c;aj#puAB^+p!=)QQz1@((uj0n4}ckAvW zKRA>~d*3keVVp5@MR%JY`~70m#?C9d3cHpsg$2u@Q$C>+dJ1|$ckV!W^G`w5qBYjL z#@5&{Bc*5hW>w%)i)}p)F#|zm39g%bC)4Lz%S*;($+qS7Zuz1r)IOl_#yjuc?jCJ4A9f|)96rY{L#dw)QkA9Z zx+mc1`v&{_CCGjqH1F_tf8FymNDVj=cK-gj9dL7e`q2HA7s%WO!i2cQE~SOIxY_$= z&Kvj#u{!hbRetJgn6DSMnbsktXJhr7wjO@{RJCAhS3_A-F=(`IKj`3BpV#fEKpijf zK(24w!)^MJY%Pub31_qTo+C|Y?XI>XI=ZNEv9=&@wg5&5e>egv)UY)p-er zR5LWmx}DML2%pypeC`+2J& z!COj;`u_t?MVbbVb@ME!3G{l2BVfuOr`vXJY4c2h^~jXM{B%D|)Ri}}1vrv_OFX0z zk?UslLHaJ>3w1NpDRIx*p14iY<_kU&t?#zJzXJ6dEPx~=!rQ4HyrgCoynK-=A66de zO0h>$M}G#FX6uV)ss^Z>6fK;lO$q%Q&w?BufhgjcAoj(N&LII zN?T@-&pD|(2lR|#&mv)1%gUQXLgCjSap4TP(Q@*0T02LJi%$cA*J|NsqEK7m$9>CY z4iC#?EKmy}YfT}s#e5|FKE_T5{-8$LELQEdZxSS1hb<~N@6?}M&V_1K562oF z*A>3_t)tsx$fS=R0%pb?5_?To&mWKnMw)k2SsoYYSw2UUJ2R(SAf4!3Oj!F1O?ECy z!tA-%^5(C#bPv9WZs+iLAVwQKQQL6d3d1~=JXr3l0t9g*VEcAfzbx)$3vtU=FPgft zl$jX}Qe$q1$FiT}p$e)|_tYyLNe-=iKz+_C*Y7k-p}Yv#`+4yOnq%W@2WFdbTFMg3 zQ9jWIy?3$G5?UpRGYdXbgajpLVhmuUNor>wu#6u8rF@f-Xz zq8N7w?bG(ZEVN*Kcc;0W94j{hA4uan-AvcR-L-Va zq-yg|2N2vF42f4(C*$ZR#MHrU zlt0!EO7jm{fFF4Lz06nnGdIC`hEydn|p3xWdrl-c>Dfn4^2xzt8M&b_02G><`|0?T1%8qj!NJW8v15c zJwgOi8k<}$C@jrjTse11_63L1HSiMQ^LsFt(h2`r?e%s?C53-%<38wAp<=W--+RH z45gz1f~5#_RA7ypy*6?$79yXi`!`-D)a)aAm-dl)#QMH=tVP94U04ylY)>^EmmapzMzS$JehHs;{b~S_ zbH{7>J(zd%$1ip;MdVPNpXRhiRE}Zu@o$IZ{2xd!1t7Hsq0htOmk9bt*y}ZXU|}El zuDf`9+M!^Nx>X4Ejc0A8ujJC{WmZiz+^t}G8z5ioC(6lXV4{h{Z8s)=a*==D+lBJ+ zZTV|9l7<=QGGTR{tJ{qhUCEv^Lr@$h*-=^>bvI)=6+!G8Kd+={%qp#v0A}}Q|I!)0 zm+x>&;*+w{m*MR;6V@C{4WCzfP=;IDsB>(=Mw%(u1hPUAv% zN5#mo6UPbPCdi_ZFj{LL7SBh}p4yfd5qXo6-Rjo$KHEjNEQQdY;xJ5}Ol!)sR5rC@ zJ8q{D9*|raD=a;l#zc`0-sN6;@r(ELgFmBkpGuoKAGWD}uI>;0Wp#^GmR{iJ`>aQR zT7Ct&E-tGiZxalZ^_C~z)mcTyOrKd+DxrN#xbdZ3U}8$ANbQHZpZp+|TL0DV1czip~CM}Xw z0NG*vtkEs6W>*8QVkPYC#KgyoD!I0U)CaBdVN_@nfsP5gDzzb$YrOd z%GvT~-TY^KGM!xVxJu=DFQoqkj)Fxggwgf5xA&I#`sMI)fiXTJNS}IMtOITfqPSe) z=-ikzBiFs(z-lPqJlL|eO1MSCyDodPdA~NL`IunR&xycFiV)zov0|SF|5Wm~y83TZ zJsn#rcBJpeC%~3C7Vq|rc8DLRacgh*Sr?ldrpmtQ;j{(vi}DX z&2!?zu*^Rs8kIm4mvTo`oGLc?CpkZ;#m)A|I*s{3tjSpPD6bEkTu`NI+Oxu!i+^Ir zY*_Czk6%Vro%SRnE|JfltvL!qDu)Um{u4y2lme3yiIuc#;%~xoH9NHv@SF28R4Ceh z<8^ukLhny_+Rn_-DzROuk9n;+14D~Zo#}Uo7>xjHI zk<8aoxJBxZXeD!poe(l&Tns`;BoPHzJ;FEC5VlJXJ~`mYC$%Dnql}zlm+fM-`OAe% z|E*~MBe`*uDKFlvlMES5z(^99h9M8~4?UJj)JS}=l*B7!Uj;~qhwm$v#Arv*mnOVQ zc}tHY9?4OzYX^`VweO)~&ZVo-OM53>tbwMr|9ppq94mfukUshH6J_K4mx~sR0_$Py zv;H|&`l{@nC2#U=x=x>PVF6V3N~@uhg6X}c(Y{YTvXQkDK5%zXQsfC+-1RtV-t{n- zDL?8PAnf6uETf2DHgl&TP-9#XTs^;V`_;mkUBk!I)+TV~a++J?W7ugvePBZkRbqo7 zA7g?Pmc%NEJ+o$i#GwHKlAXQlu5g$2(75{nrFWtEYocFv`t%y|b5e*oVjC&5IiIR- zP=JWS!$1;=J8_H8V>=4mtz_R!Z$c@1xkH1HPW|*Xey@_s!OHmeKdszcQ5SYsqVxPo zJ;^C8HE(S7L79ImAs zI>bqznTYWWyrMPuK;WFT>EVn4hBqZqN3;a!yhVskGvDc8-aNx)o$xHhyAX0rI1IR*UxVdT8L9~n85h5;*zkSxdRdV`)$lIL=m1c@w7 zo{06$Noh|rl36wDHXolex3b)F5ewveRmYC@H>vwywE=NH&Hh81O6 z5YB=qFz$Roja*yhcX?U}Kb57>u*gsfit$^M|J4d6q0Y2wQ9tghmY z>1Qn*cp45(;wiigtE{Xhv4l~2_Pqg6D|zsR)l8B`B{BuXR*T7!uLr40U)f;iZI>fbmeHe?1QMYs7EcWxy*K z6qbYL7(aRPl|i9UdkO(J(BJ9WEk-lWTgqKtH4$Y7-CDu;C3iHas z!ULaTOu1iz+mjviTwaz1Roa)#eT3Oswc`5ir5}g(zu5F5`L){>xrEVE$V&(l;P!(r zn`&UVB`|Bp_CW)l_+fpIq`b`J6XiXgslj^W!;!=4 zcnfN|B|?hN6a8?H|7LBcKoAi!0Ny$|U6YoCKbOx>xwTH{_CYFfFaV>5^msF*D~M=V z(ZRtu7d>J1RphIp=7-qj~)N)3>c71<5Aele5pwD)Cjq(+G_df|An6^E3RZ`JH6MD;>MzW|A zUA6C#etXiqB{#tdsN5=y)Ab0GN%m6lbK(nLU4lMZ&}Q;|tr`Bl>iw)lvCwCDNghm% zy{$x*Uv#*w7B$X(wuEUu5PqD#2(j(@uzo}_b;?udy-caq${}ZCanXEcXmI5HG3ioH zKDSpWX|#rHBppv$?9TcGN*XHa$ykCq?p6JJg>42aHxCvbr4RZF-%#pskI;QIZCH^o z(QBa6C>FmFy3gxS`iQq5O1ucSsEGCsL)F+$SC|A{%0QXf4pZNRDrKXvsdl1pSMD}TWbbn4>DWME9Gq<$=F9Ls6&B$b_(;IX!3l-xBA!c+OM zv3)y6!?=K1LY=C2>f2j#Pn52luv6B<_ao7rxr@#nXIVZmsz$itd;eT$OpH|Vu<|D# zfzXyOCV~aGT-81jmuin;Ef?>WgI{&7vqE1yVxOYV!f`6;0Y@PN(w)N2hu^GnB%sw1K;S6w!CC zIQnwVl)D(MXXvrZlikfDsXJo6A>~lJ0fuQJSP~4_uyieqEAI5l^rU;gE^AdmySc-Z z+7``NiSE9NKB_#l$As!x81KVu0=<>&Z{UID{Kv$J##mWOMp^0 zI-w`0Vx}DWX6e1;?1k23Z9A$|0|iH;;6l4To1fRx`h9A`scFSoOc!g!N3d{y7v6DF zbi#>Jz1}V`Hfk|P!gDre-qsvO;^3#rNbg^y_q2&$hDC?bdPPzEWbk!T581-)DowXg z^1m{X$KQWMFYG>rGE*<*{Gci^6ZD3-&F&1r=NcfM%qb*&s>>qSw2f~+Lb6t<=osEz zxrqU;yK+R+*vb_rXF zIa#Dbf!j=)+`^9%lX6s6rHMo_W^z=z`8e7;I7ixxMg4G^yKY>~CQEXq=~YwVj%XaA zPrYV>UyGactOd{w$irk#C}H^;Q!Ebx76q`~o}-gfw8VK0tB%5Y$mC;k^XuhW%7kl* zVWmflvZt{vwGd}{1SF?M)I26-?riGcaal;0Z*M+-Hj}v{>6R&VpEYMZw%fSLqTm9s zxx{kG=X0eyJ^CbCAFvMH^IN%svEQO(&sxKmLX|rUNoC;4(%{-A;k_@pDklID5!A06 z{FhO?N+5gTJTjbZgD!7%flI&$l~)MAUCpZBO_v7#6%I|}p=e1)Vm%Mqz4~--K_Ip{ zvlavLr+|DmnB6A&cJg@f;$QJuQ{nt7pU8kU7Z){vaxTXEkB*dKj_hJzzp`5x3l7m; zS|#~|2eSy46Q^KCEg7A9Zf0NnK`%sidjikJF|e>^QzQST%9i+(S}B_2EqDK)+Ri$x ztz}=>r!6f|pjdG$?ocRF2vVRFFGY&GySo-BP>Q=lad+1gDDLhKMS}zh0TOQ5`^ets z>pu71=X-Ac%&e8kGc(UxGxK|Y?`VvG?xGriLs`De>j$L$jGyZuD?vnx6s{0&VF_(GO+ra~JIbsBha40G{n)f!BT(Tvn@x&-m4>U?dK)8gPi)Yu;W1$uH3_~jmT{3Rav=P&;cX~ugz@0}}xJVjcSem323 z5b?+VbLCRi*RA48LumWoqmWt<$(W~>0OZnAYlWgIuIZ1vK91JO4PJpqW-ohz3k85? zemeKp`w>r+7RO2f#IiEMN53~n2S(&vj>?P1r&G9exWt^O04djvCvQ@)-l%*QQO#eC&$qFEYKRDVcr&Y2$L*y2ixEefmhnpT=W8O*F48vGFFACyvY7i{XXHJ%5^LC?FS%8_J~{$RsScc5BL zZ?=pHJ)+Va0o{HA^*Ru>&X|Xr{&1LK3#E69!QDp$oyiQ3cXTnwQ)w52@$;b4YAvHi z{rLcOGPS`BRUPBiyb@A8XG*jdi$#0!YSF7i`iSZD(3hbc&B5_*Rc@+CZ+Y9lG;L6=DW(-7A&C^(Q3UQX_zf*Td=j6S8L}j2XH3+}QBp zzvGrBD-w2w)Q8_0R*Ns`AlMw5KAz{XhtFYydeCQ{R%5gE~$=U5toQ#JFQ zb_5>+(idV~<)ufna_fndf_E)uv21*{JeP!Sp6>0Wo}OAzpM57{!z}u=d>(##)goih z%Ga~f7A(bT)=O`LS3;$n28b662zi7q4lUL~)xnBX?XX!{cFqNsn~rdAaxZ3%lm?Iv zzGC$dK_mbTh%-uqx|LxIva8=AyvT~TG9@y~ga_$nmOzpqR32 zcksH^3i2M-oGP~$P!a7JPPiN!9&V?nKj09!|0t%XG%g>w`%h3rqy7^63f_N zSsb=k1TSYRzD@>@D0p&Lw*?h?{j6k}@VlQmnvJJ(#g|)DKlAQx4=9SeWcqkw{dUez zpwvNGS1^RzZP+E(LEV$xU{3<4&SP{8$M`Z(*rPPpft4iGFbTI{N$** z#5!(mO57^7;*>wy2*;G46mlj%^=U4}TgBwJ1ounIP|KxqO534_elOU^IzH;_L@eqN z$LKUOL{0`QlpQOj0A19BB z7<@D^Y48s#eF2+oi8<%uV)_L~tQdNG8EX4JSe&Zv>M%!ByJ>{ifbd;aqGH9{gwf;| z@^W?e0hm77p&dkM6-|!;{lG0X<3&{ zX6gG_RQka1n{xfw!`Q+RZxQpMELQfI@e)p6`aIWsHrIbCnUdZvMoXN_CoF|#?A+ol^YjfT0 zWeqUXUV9ntwv|Ub`oBBtLwqRUrDL-{oF!RV1t!V{Q2hsa6>Qkpb?&GSE9r zQ`_-1)j`%>MVWsi=(M+K`GQKK#uFsnBG`>9?-QwMtI?IhZK?J<@8@I~y1KOJe&W|M z=XKy=qf-~>uJ0p>*Ytd5Uo%35jrnQ><{Tq zrxy_z(5-}q6#Yr*fr91iO;mE=av)lB;c-Sy&G7~#vaEwi+q7U`@91`!U2Sb|D7%H+ z$U6+pdV=)mdIh%hoZGMZ+={8LLD@qabKty0l5M8L$_G5V1gJ|hG#$LoA)~pGB)@{O zs~toQSLs_jIkDIxODlK?KL)?*IKpetr3DQm#r3PV)O|M;aEH)n@c*oYM&HcUy6mC& zNQN@PicuxoS(cn|mA(S?7VTm14rO8U9~4Pb5kn^}%^p`c z!Ie`k;+@mo#%d`oX4sPf0l|4a)uN=637sZ##@UGnl}_CQs5hu=T`}64k1oLi-eylz8Xg{hGVfCS=#CMy zV562ol=vlJG)}*$SSr8>ps$v~m4(J@8m!MqCGfc@78aD3O{^Zz^S&#lq=4S#y-in) z?V9Y87afL7Q{cxqRi0-u=`n3uY>E0I%ozM)iam$@Hi8OuVoJ{5ubAA^__@k}n?jg!<1x%S`V$D+JnXZWqj8xP#JRuhDc>HxUQvi!I-<5&!F+U5R8`SV%Ly(0P zJsxCVJzJ@qb=uK0@2TcsUGYOjED;UK$mS! zb@wBYbf|d7aSP|@!GqV%{Q(L4?KKwAQd!7{oZ}-XporbH8*(6KAlk>e1ndkf#avz2 zD^h1B;(R(tubZb>vgKQrTzf+4106!Ug^rG%rBc|^%`B!dyv5*dnpZ|ArzHFqo0b#I zaH>88V)nOFw1nVc7%!JO+V}DO=$Fao#|?y9EEKZE0pLuagT=N&FT)VDU8?5`=TO0^+3;GoanaBvcK)WO0awyD z)%M-P3P&cIi%tPZG7Xq{&}UgK8T{@?Yg-#eL9Yk_G$k%<`9Jk2@Wt0YKDjCOc=t4g z@_Ij-v$%tvHBuWTL|=~7{qFb{2J2FWvTSACB(9g3mjAU5t>^(zL;0`Jy|cOupI4aH z5j|B1`x+2V2bi=9Nd?H&%LGWB0(wg!G(NJV9ccf@q5 z2}zC{yiCm8o(Fo%mRl)_!9Gw7Tz{PWfcyjJRaN|u5>eMm#;!<6rZQr;tB%_7)<##b z{#cG=A`_8mdO$2CSWJPlnO{Q4s|`7Ln2>R-=|OJ04uvKSx3CIi^7!Xdrl)JxS#M&r z4}re5Y&>7|Soq4**6sIDn|Wf&(WJLG7rTS?K-4}io|R9+esDouiw|b&V#0mVCvply zRDd#RTJGu)+qd=j54E01?c`;vrSeNv#7VxIFVL z6SIuYZLUzm1%HMI0S@ldl9UGP%mhb+r4}Tne{EcY5B=~8AG6;>gF~jfDz79E0!CZ` z=$zP4(Q5x{J-ImKRryPja%rV1&Sgj?PWoBTS7LmfF8ogQ9U7WRA&GZgXWNI-T z!61?HP%D{*5$D$lb*8aV67S2@P5bK6s2vs<$XW%iX4*Y0erE@fF;jZ6`-|gp7B8dX z(q%4_1zUkT4`oxj(c4rb|C$D@aAOqf)EDIymk{0vFUz#25)3_$%{-mi+366D(`)m3 z)Y)ZcExVE9dKsPOoKc$txZ0BK!kVv(b-X0Cw$c9#0p{m*3bQn` zXML6=eSv}9Qk#$kJ4Gu4rTOt}u8~P}yi3fe&Ydm%d_V^bxGq&5q2|ZeRcBjVi8`?7 z37E>s1ozMry<|<`>k`WU_A5k@U7adFtJEuhOwL-IR#jt=p#+rMTpM;-?*Mh%eS||h z%ZP~A_iRmwuNTn_l$zp@tS@=KcX&Z#fzLy0D#^&vmQz(q#vn0|ND7xPx*zD$i>VF> zYBdxgoO8{10jy3a_Mh~smM=ekAv_hl2Wzq~O~xk(lu+U?7{z=G*)?2}PuakXt3?d- zqc?(V%1I|jl)0!D|5GAe;XWXVFhIJNZ+YA z-lLUS=Bzsgt17MzB&;)@B#KIzotKf53f)Vca<2qYG#mU*%8i5;lDSVl^n6CFkY~|M z8q+D~WKE#3RMcZxlbdpuARpPv;3KwXe!%Ww97)j8)EB}wT>^Jccu(jZOcf64!sS9L zH5i1UAqz9cX{n#UOx@woor{TAOB99$EqW+93G!X>*|CS$=Z{w090>|W2602;yjvTX z)7gM_q89)mr8DlJI`(nqpV(tA8Mw%hvx~il%C8bC1%@2^zUiS?rAvv%kzkn0M+;66 zX8!}y+h&Zxj++Zre7)Ll=CA=DXo4+#LS6MqrnXzo*LU zk^CVd)<4uYOPEk^ir65{pyyuvT3NV92+s}Ddb`sk03NClxGB}S25EmrhjHz_&N460 zZ}$C=o7L2=L*BV)FpE$iC#?xQkaH&5hE))oLylR0Q>)22j}xb`k?Y;i_cBhj=xU1P zq*ZY9ct98a4%$HnW~V=@LV)lUi9djL`h|Z7?bgOqZ5i@wN|PpRe1EQ1M?4;HaJTE!Ys4Gt$j zI6}qzw787>)0pB=zNQDb2r==E&^ZL*C644O`f$VNQ4$hrs&5z*GdWvS7kuIy8mitP zOVn2rMv)C@U_^96=5kg|=t{>$jt-D_Aj|&BXL|3JJ7-m?6W zlLecSmhu~y37eJq^l%Rgq?(J&eKTT77-X>Zd_u-!56>n2V7}4l^&cEK65rKqiYB0O%nXA7aerZ;}nhc_q2@LQh;ey&TSoU0{D>6|pAVz^)I zN`DR&Ex{}JQ^FNJs~CDf;#EF%42%vEd<8;oe)K*UNI=-Kt=szG6D_iR$pyzaX)#~h zqDy^?_G`kgSoMX1sebwKXN>b|fp0mxnjv_3`nASYXLoHwgX{-+z~k;T=RFRQgoAGm zk>8M=*F4BQ;gPqek!^cTnJlK+0$*R5`qG2os)NTc4xq=rd4~5TUxw!qU&cHRvt|uv z5MCANVG)>N3<19Fgfwz-D3jZ9N5$*t%}~YuC(WBbM8EN<4NMAeq0F06w)40FSjiWY zN^MbRBe?iL^Xrc<2;{gK3mkSch{fU(pY}#Em=vP%Il5R0UmKb8G1oKa-OunV@?jBq zVZ9Wn!BMX}NtquF0OgpOCBP}Z=(o#`?b0oi!u+KwZCD?3`6iZ z63*NL&m2QX?%%%kT;MeBKM;XSy9zxIJHkIf%icLqZbV4TbLAv>S6FYaRi+m>qcJbM zyb>{kutyVJe;?U}-Ae$w5o;wO4(sp*`V5yj%q)r&cd@Xm_9xU`>4QIUMOpB)TNf}XD zXr?(l43mg=Y2a0u-RRojUN5Wxu7#dGx_PSJ?v4J^qg^b@YP;DCQfbX=7ycMDFT22& z>+l{=)acvS_hR!d+=?l&Vi7t6!>kmwcWTExcoR{V*)W9d6vd-`#)-hYyM!H`h4A(i zYU=(oIT6}9=In4ljcEu)_glUF>9-rj3#djcz=tE)^d-y~|FX zVRW}JhSolG%qP)kE<_xHsOjsS(hEP}$nb!F6cVx(+bVOdcH4M!bgy}v&RJhoe~LT6 zJEhPk9AOS+`L?>=2a{efAJIOgXR_d5kDgT6rgf}$kquJ+6( zk|+HkCyn!SItjg4-Kqq6-1zV}RVyrNn27Cqp|Clh?#6yj!t{bLL>7tK5CrcfqMOxu z-tFwX+b)Q!N|x6oJ?Ljyjryc5!jqVWs}AhjS!nZS!3x9vmHQ1X`d{w1ioCh2>vdZ0 z4~yS+h~k&z9jM_ecUwEDJLgu-R?_+ zY##tKL)qUl|K|HxK^G_EY$j$`rzdMvp2KYUe`|h2_z!a%^jY5r_8W)TpuK&q1=ee6Pa2YI?kITvkMoMo&Q2iQ3jvi6)Llx}4jMo9i>G z((1Ms7!FVY`&s79^x%ReHWG6)VrI*aI_>YQp>%F*{4H(T6 zN$XO`a3*mL(>_tsHgE6XnjAzp_1FbXUz%%_pwrP~3I~|e@C%Bf=Q9&C2bhl@2|kGp z53lQ(TA8o6ouDpJRZ>}57@#*w_LWOJ+*7cX{T1KN&v@7`e_EBilPCMS)$D?{K zL&W5b)DK#+H$CLJ@mA!O`sXK`u$ui?NeHlrh~J%W$XPY&D=SGwXPFjm1SE@8~i&OCL`8D{u|7*@+<535+u{?t0QTEaM5rh}7+0 zUTuqyIzmg6AqoGKri&5ku4G$6XUMxx7-{ zgtmy3IsFPpi5e*<5rwUAzIgf1h;Ah(f^*89)C}=9CDPO*btxywm--!gd- zQiJXOwo`IHt{b9~6L_~iGP}Rrr(xOnn?Y!0e%7tB%$CH8=A z@VNb>Uu|wC)#PC$H=#(CkDR!TX@O!91XWMbm8YL**yQ?=%%Y#e^W{-xC{dkf9%%N* zrOESc2ObPVSQBF(lfTVR59)%%YanS1W(I|A-_?ls(w?560p3zXD|2A$fti@52frlHUccKGo6 zR~I(#y6_6ihicxvR9?Yl)5O5cH+F5`E-x>Aq+Ls~@KzK`)Csr3CUdc}v$M}9NClXT z3sk3CN}OH2*^&{)05>X`gduv9Am-s6;D#=R*Ki)(nk{~{Vp|cPkJV=Ai+R!V;R)1oP5d?QVqV)E--M|^1^3NlQ4=xliRrj!-FLl+p=3EIV|WZZXc`yxueaR zBCsE@6)KO&CQoYad{`8CmT#Uz!p()3>sZwh!iat9;Jq`D9{Nwy8=bGS+J7&4I{=2@ z{|6jwF~~J`L452_sy9dhVguZRBg2+`#_gSL0!&Z3^<1j3TCH~S@N`IB#|Bb13tH)f$}A=JJDjE|xEDm#J`H3A zR!Ch5qz{u@?D)$}A+ah#{zl5=pV<)D9~d=4+lr7=E$046jNhFf<-*14+jt`rW~NVX z4k!~VI5JDALZlSX)a$LV9lIV?G71m(9(&Vaev+M*CZTmVE~J;J)`LDx7>=YGYmdt* zI7q_Y@66kL1ik0BdKWG^A|E^EBw1+ob!_p7n#{-Edr4sO{dG-}hi`UkMZkQK1evPOx@r&65Jo$T=F}yj%hL|z;Kou@y+2TBbwpaU(t{F~ z96M)=r_a&{4XmLL$%@`D_X7YUAC`A?e>-iY=V(JDhTRf42B3g!RPZSouu`zU_;$o4 zQGGDR&^CeuOJPbpdZ#{fmSZn?$45b%m|7RPqBiHJ7`H6nbnC%k0vTAWFb542N^h4g zVO|)t>V6Ef;FRGh>#VTjU3{Myt|w1-B}EH=V-Mi$IF3+^Y&X|7t6UF+m!gR`=XA)X zZB-y}332l(1eQV`Td;@VQM3)NJZvGTh|)Nfnf-`34%R-qb{?Lf&F_w(#m*7|$>eKU z0OFxQtuAo#;odjI?Fz~3V9m%AtcE+oE7ktA>#gWE%F=ggy_K$9Re$@dj+^NS4`IJGeo+JpM%FKehUAa^!L!!^LhXtJ(i&`nQGhQK`I7Ev zhb=?5o_8*I+M2G*N-#yX4+;xQ9}}VI`~?fyiogr2{+{XXe%_bXdV=N zi4cgPmLSwt({~{o8-M6;GCL@K ztWXm;Nd&0B^36x|O3Nj2!f2~{5P=J3!4_g9Z0cQsR5adsX7wSduxl10`n(;6jJSNS znzL1Je`_Nj(e;MTA&5K2&k}Ycp2Q{_soAJR1RWo<5b3;RE#uzX!>Xrd`O@NkgD6!} zQ7K3KM78lC;wR>TzQ7|%qBGUfaYWY^?O+Q#Il08>>~%oMr5o2_V{R|_ew^(dAJP3- zb0qqw_fKf*H$IHxp1#tfi%I9zM@x)FB#|QD@pm@@#VfG-Js&`<&U3EDgd$94!*E!w zA2tUrv|PV)TE_FbnBAFEf8Dt`et^UMyWsD#?P&5HSOuY~yW-JfU4(CX@zMkUg=@mC zLSH}bXdoalS%zAYV+aj0kJ`{ff)G&b1qxB))8iR+pZh&c)ks&a$?>wS_n&eYU|z=}?^VH;*8-m3($qOI}d* zLbUM1l5jKSeyOrhr$aH8smpBb!r@`yTOa+$leTx;4ZY=Na8oK;lJ4dL$pAhhaY|k#fgrE>6Mn2DHJL1wpW_j|N0s` zNwv{CZ89>$dQ}Jt;MqT&ToP0M z&!vc;Es5kx?rG4rdOn2O@y{)QXjQnVwd7kG)rAEoVa8q&e?qA)S-z(JUJZ-wMV&E8 hn!%zxVYI8e2aR>c?%XamOMgRrq{QF9Ef>}E|1T!halHTl diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/compare_result_pkl.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/compare_result_pkl.png deleted file mode 100644 index 863708bf6daf46985328f0dc42d48f0a5b849af5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36133 zcmdqJ3sh5A+CSWBr`oZowTglQsZt9CUKJ@tZppMF*lNKPLO_8ih-smUT*D_Ul1KuHe_1i&brqg#W?SH-N|9$IQ%S#|R zC+9q8@BKW#%d_*21O}{Kvvti2FTAk!gM)q_zwpAUL*VtzS5|?4GnU)WfdBrG{Bgkh zFEsFM=D`nt#J~5^doR4uNO@I%;$`sjpUxf(Pk!Nrbv@?4f8Zq4oP6Ph!uAjR-U~^O zTBQ9U$^L`l&!k3=v8=^x&8xwP>0P^j_`Lu8;g5Gzlowc^ePM&&&fwsWhMH=9j@K+KL?$%novya2~oxN8Qa@=uD{Mx+(9``F9KVXy&2Xr+HAGaYz}hKn!;b zbCgBZ3Q9iVD&Kp!7rq16Ex0XHB(1}oEb3LrhwiR?mrjV0K)aPGqOHobit`RidIIj! z)3d}s#}y^*>yxkHN!89jpe%gW%RWfc%(_9^HRD8vJNJPV%Xtr7?L?9rdfK^DsH<({ zcbu2`%=-p!{QD?wJyFUfR*umk#t{a`RuzqY))ev>_QLz4xauocgH}l}ZKywx)<5kI zRUjg&Y>Jn(4yujPy%(q>gRh=zU?u0cMXVHg;2 zXFgU@)H!K@&YXm5M;&uxIu(CO#mWd{(*^wO7Q|@e;?f_}O;f`-)2TnLzp~nL?HPC6 z378?e_ZZ=yQsKbA)p1!6LzyPsC5D(jVGHzyb^H%0_w?WC9Inj=^1)Sr_>UOQ30ZSo zYsS?x{BnxKVW|2c*P6L?*_)4eQ^lexX-A~kWy;b5}+oTn`Os>$=%-$ZB54hb_urX=bZVI(=Sp4!S=c0?Fr`8Ae{5XjipXp=pyF zXgNx{clqu^Ye37*9fq036Vix!@8N_FGBr5+*}y z^PTZ^aZ^ek@`ZkOQa9m!8g3f3O6cDe3Xxx58P-q(G$3Obg6?G?MQtc(F3NUgo=94G z4)SKLkpWY^lum-~Alji}DW*BYp%SjKKHzq27#m`s3A!T#)zgc~LY?a%jctC(W0ZoZXL@(VtcE>B(J~z&Mr#;Ri6{42X$)P zH)cdsjauOjs2_yAiy)<@pFUn!NJPo*Pefg{B91mrzg`->1JvMl*$zNCk>j3WaKD;u+~pbuLX$U;YH znO{m(%zKeKd0DP4_)uh1yJN9yhr~v+A!?hI?({OHRN$9bIPTscHQqIVp`yb>58|tj zZnG*N9uV+rwF)gyMzpcS?;b2VpVZ-cQbP_t$gGO-p7|8?M@!aw7MRIQdL%W7A7QO!jWbd;{d$XOtPvUAYW%gZ6a!S zR1Z7IE-#)~qGisSVAlu?$ zLlvJsoQvB~@V4j&s%7QXGZy1&nlt1Y`Y`;AIYw=AZzj~3`sN>FN;rOvg=ew_&2mBo zYD0L`CQg`mTX;gxv$$QkN{LK9J@P7uoe%x_>1;_4iG|Oj#~Ceot?+=|#$GDn8l}0N zva#1S6kgTf&BrSc;OeSXC5-MLlVQuoE2_18knOCdpH>gA9Avl&@55?b_eJ^AtqTsk z-*l^S<9;X8CMM*n@aYh$m@)E$2R9SiyV1kBS&y%V4=Z0&AITn0rb8jLZ{o21aaHLa zr|RaA<}s$4?RJ@OfCUdyd&_W_@;l_t>AM>i8w~15L|XW?%$UPCK@pl4wWg~AgVYEs zmNFG|0J90%nEsT||JOKGTXx zd`U=?CJPX49p;b-YDbHEKcc^jsNQCJqRTyD0wn<2-8o84JQbpRg{RciX~KOu7_a5^ z$;9XP1NBW2OKV#cBhI(B2-C93*{!*>Qo(`5!g?*nXii*p-^uvKDU=skf#@Qe7vo!7 zbgQIwKfpBK9ckJ4R3z_1pTVO05yx=zV8nPYQ_i_|N7b{=Ya{2!)i{HGe$HO86u*Vn znmW=G6Ng0qZKT*nOuB1-HVEGteW?<$I)-Qf)9j=>hOA-2A@Tdw$~w6U z&M6A4T_f9KFh=e~gMhejR1i=naYK_VgN2y-62Uffc5dCSd_v5+Oef`^o*LUqK^--= zjo?WfCi!_#@?@2ESE?Ee0+uc1fZaK=RfIHpFMFF?WxB_u4PyUP@Lthp$aT18#NFY+ z8rfYAHPAw6S8&$kF^vkuoI^_d8r#UxQ*>Q`(sG!?GFU=udHYeSpOinO-gRFyt}YkA zPq&IwVk{#zMpj*jq>s0eckq8mPgw*jAm;2xDMjyx8h8+A&aWcR*CF59Dd+t?IV>vn zfjfHrlf&ago9g{QA<= zq3gnWT4+`ct~^KO2akd z>D%LWX;%RfDO-JghfLb2K)NA{w6US|Ly#0# zN!_^2PXNGhKoUGn;VP#Fc>W$S7}j|eljLojC&oyPjZqDeO-QR4 zQGBJBJvX4zt9X&reFOJBz0wOOP=09mki37mT82Z5(=t!2lLn}r9Jqv`0)2*&m>UD> z;dLAdTm`Nl0e&~FleU7feiA}2SC+sS+j|4J#ZMz@^Y#g()l0=nZv+2qwhZxlvbsnA zcIPB9aPgRWUI7kR7~&*>AfIQLXj$HFh>zZ-2fcB1H|G>mR1WJ0hXR_t>49C94Dy$h z5OrUkuG%YLwfuIrm(-)lfK4jupDgFgjE6Beb^FKcpGE0f=*x&D*PC+t&6Li+UghQf zYf>;&P~;owbF`;K!znQPKSPdZKMEn`o}oBr`DS#B`n|iC58*Cd$xS+I3$M#8NdPtc zW9_;I;+3U{E4lK1!cPT6d)2KRA>jw;wy$8OgyY`nzn3p8$9GJQS?piDOZ0pff_Xzy9Qe{{I7McOF~pHI?S@4G=C*mj@VeKqq)4>r%+T1d zdzz%ViM*w*H9qt}**T;p)s+e5WuAOI6EV3|?wIqnsD3$YZWxk!Q{DJ<)XAj`<0B(@ z3Lcv?9FRD6c{v3P^c2?iGSPPViNwuc^@4TLbB(64kLAhv^ZM+FS!YSg{#h>i*NgziIA znKY;c6}-rnyDg;35^cqNFfVo_fXCSX@cGb#lwMq1Ip1L~bhieS(<*b-TI{O9YONe( z7)s4972Gm-h71tNTlt=k>NOKf32@UFwZv~39t=B&aKj#~l>N{i(}h;=8<>Z!WO_lC z{E6~xEaKd(8(n#pNI(iaL&N9EnMyzqy>*T6@Y?X@81T*z(HOfjm8!d6gqS3ZvTjBk z?=VQ|e+Zsu9pPmmCcbK0@=!ADwWB~e+i99 z%Caqr`NZy?rC|E*dw;6z-t|^KZDpB({*-bG`RkEdW^^XCvSk?$jkP@LiPZ8be0yV| zKud!cDgFqL7vlN3noCb^j> zHlmZW!->6vNqK0~+`jE6nE98N`oK)lq8i;XJsG%Faxn8EqRDV$KpJ09D`&vGWn0x-|^b zbk??iy_$&B_Mvhn2|TYj$}U&ni%+*Z)4}DDCTuxY)XxS*sY7Tu>B{5nhiLGmmnTz1j|Adb&JO40GwL zE$c2;;f(YL6N{WH)p|@QYDkcQHB6HVGq||xZitzn4FA+_8Y#_Iya!XPx_b@Qug%1g z!@a0@`I*zJ1@bwh^|KrL1Kwsh3#1@47$O}IT79!P)2pOfYwg^fNHFH<940t7(j^B% zq!``Fg;6ChU@;-Sg)mo3>MQO3MsjZ+A>K#J%!zR#w^ro+btS*6{_B+5T|#~csjsap zUlDCvI%V4}mmw{-A!O+tq@_9G6SLRI_{aa2c1s1!?qcalfKtBk zv1fAjtpSxQV+QT=qP$A)RGfdL;+xKmbk=7kUfYiC&J}}sJ{v22_7CUAN3LVp-`E9+ zMtkVSy55tCtK&V*3(BkJ6o!haK(*nApaA-VI|k@k0BUr1YxISIG?BETK$xStFx0b8 z>IWA~le+UyOf7}YrT#E8_LA@^*VD+Y_YUDDr> zC0+mS)e1Q;n+kF?68D>{me-t6G47^MXAHy+AGJ|2CavmrY5c<}u0AMp?0`~K21A2? z21%4!L76UP=l$k$pOQex`zT>7?S2aTytmG@JotLYiGY7BM4$bHL__okPB*r%_58gQ z_j5Wth_^y>U&DaQ{H^Tx^FyDpk8?!H;8%ZX3wQtDxqxwE#Z%O|_7^MUdCkMbcOdxZ z-+Wcj-!4V@4*j0#zu$dckb;`!jKMkMVIjCeK#%|X>+GB*E+}bA)$%Nsk0)$)z+0=Y z@-KrmI3o!O%F(53oH83Ja{7s;K)UrreSHFcmtYlnA5TcC}>eCaF7!S^q6{7BWIuu`Q*pG?ik zDxugp*{c|EN)|pv<{fphr{dM{rHKAgKD~zX0ct-Sh7hMnf=d>=J{)!M?g-8Z_sw*i z8zJ6G*y6Kg`8fYeM^GbY?V96B*Z*VrdmBEEAQs6KbCy}B{Gg@gCGD;>9o+}9oS6%E zx;a$v+2syW=y}EaICmUWh1)yAaqgwTy3p))9e8-&P$M;1=};*nOJ@LRDjnwBc8l<& zFeqA#4nxWjCy`p-9U*q&LxkTjj?EvV*+Z^e;X|<(29<+hX4gQ;0NNT}8Y@#QKT15%>X%ICTM0P|y{3`nZdVPBOkB>Dc?zY^R{* z$D}HvJKd3-8k7^Wl9P){yO*9Ag4L5tLDFpk;-=BTmP4bfV-~VL3c>rq2SYfGG}vlv z_e2ny#*z`PldiLuoV9>=SKhV~~zmyH%4WFCtvyPa{7TY>PuiGLlnn*iBU@lk6wSJB8;vX>gr_ zAEL&voYl^TA}o^5dTKSP-cS%=$LAzJRR`xu&behoBRjIFKK9QBfj3_obbkPj&^Fyk z_~i1rUsmH63@2)Qe_f(q@LgbZ*;C@vi%wRs+}JP5L$Ws$I@2T@XIrpHT#oJ^YILi+ zMrS-j#(E;n4E=&LPEmP&2RI#FXc(Z$iJVy`vM=%#*4pCf4AUEo%U$iSVM;$IwMMBI zrcl<*tkfU03^s0(>HH$Jcdyd60;Q3??}c>UQQ*x5Y4kd8m~lSIr>}zReuLLLm2JAQ zgOMJJ5S*}(M{Ka8=mpZCn*m$uILU3BxU9G$XSMRN{=JaDp#$b}kRKlRr7)RZli+b` zL9!JTTjT6kM85dn|4n>a6sN&6H~R}PPdr>g#nk9jq<`wW1+_LUlB4H@wv4UfDh~+W_C#@o zziZk^Sa?>s7JgzXb4!(-F8bK7rI1@IJ3S1z$lUmLzf{07#_U8filWa;{o49y%j>^V zKit<~Pl-7%16)*brtu^wUGt(Nn@Emt7Hq$DYW+Z`108j<&x%!qq0jfdCI;m(H!f6% zg%<+id91jKPI6&c#b*3^VM#U7o{?BdbVt0!z-{+Bwkf94#~=1q6Ug2A&vt#MDjJ%# zk{(Fy^xv`j8fCfo4gecrgW2s_vAO_5dmE2OT>0KxX)WZIEe9}{Zj4NEVDUIVSm{7nv@XkkQ`VNy z3KYG+6lxO>zWyDcnP+M;sc-6f`guoarhEWr{ROjcR!d`QeqEbS!y(eJ*z2bGF8dXz z>i@loTkfU4724IIW02763)+Z9HSj*Ph#AuLlXPmj<(s5Ry|)ta?PNjB%`^Zz@|Aa} zJVR+?`{f->P_l&R2rAl8htAkYyv(B22=ft%jJ}MAP|L@q;xyh@;#19AA!G) zcMy*5!a0Mp*4)(TYc=)fWgV_khBTs<{1BcEF)(Jry9R}Umk=er9m|v*^2pn%f{90c zHc&)&asNXAf-_I)LQ>?RB6@O*$$Um}wqIL#wg*;CQDq=6&1R(A0&-w&^Uqb>Sd{$U z&Z|LBWPZ96p9xuGi)@+cd)Q}~2f-WDlaQCYUGnDcxAZjD@a9KR2Q;}QwA&9$fYh*W zv%qs&TYYOU;*|JTkWD`#9ZdO44XYq8x_YI8wdw#Wa;Ka;P2ca5oeHlW<;u@oa&KYpjl=VoBXQLN(u~ ze`oQe(x!}* zd)xRae?u9`Qk`3x7PX{*FLdS*5YMbJ4UzbyO86Ef;}y;Mq$aPbYjpLaxrvFESNW^6 z{Ik*v#eNk)t8mDV4N+GcoS_r9a!#Or?r4ID#y?h2`6cA$KjslJJu9&2BXg*}@j50o zY=4HDs8{%0n7KSUFM7a5uHxX4q6bf9#4FBt+H-}V&VOi^szsB#FuT$jUDK4EDl7s*nn8l)wUN}K9nieJ<_2tI)6b8k%R>jKKI z2_nCg_GB%Q1V*f$v8q-Uq;rt;xE~g?-JY*&oqAMH%Ou4$2xfvzId1o#K^K$0I|uq~ zx_84O7oA7L-ct4XbnBgYA?%FTwkIwcYez1Pm4lZTbY& zaN=TuyymT{U`o8$I)ZPM5iJA>?xOS2${!UF+zP7QA9o*GkR%@XngS47gYV`IaM%{u ze>3|-&ETIGfxM6seM=MOHy0B*w=G=nBs+!4I#9tmVvP=T=tG`^(PzZoiecizC`Byr z57@8>+c5yfiaZ;GdWQn4khs7!|I{=;aoZO04e9$Ar6Hc-Q;&SeM-Fa^x$!Q5XjR_UOtrdz2iZ-XLp@%bf_u@2LG>E0bWz;mgG1_`q60(momx2N7G3O!@YReTQx|zl zXOa16P-=cB$Jb1BC=vX;r12Qhymmua*3N+8TzSMgmUZ2Gu93S{MJ{I+M%gN(Yg+dH z4F6C{;-;U=gZu^F`3H_?U&QsIH*no9wz?r4CfUgi2mAA-NjvyoIPz~naLn<@i^l++ zb;34(;NiN_B=eYj5V2b4CvKiNWP<>BS}gz7l@G8mfH<0N!L^TXNpLm>ZpD%uc{ z9SuXA`#wD}R;*|%azLODo$O|=<~$DNpoWGSVBK0-^u5_b3jNFHrR~WGJTND%K!j)S z0hISo+)AQ%>5r6k-ed|*_c!EaMvJ$}=Q#i_wue_OE4g}jPdqsLJ*Y8D<>3kdne@nFj&OV31EKc~Wm^Z1u($3oeQ&7)}~7Dq=kWG*DTH>6u7eujtyL4e<2Q zDh5RlO43ydn)-;{kDgJS=%!m$`Z#11Gt<#a4JH$QVI=GH=q$j1%Ef5LzE{EI(V_#z zN>FJTEos`~%VI}LUQ(bh-CO5oK6+oUxMn82{_6VC%Fx<4M)!1fvkgWWb?F>*+va~6 zGgiM^k*Mx88og67QIeR#Y*wDk#(hP3_yuGPS*FNvcgJyRT;w(<2UUc%H=R|(nkcB1 z(KS_iZxn9TXcE1K#q|i)j!#&k;XVa`fLOoOaMmaExS|x<+TtzPA$XwNx(47 zlXK+v5|(HSKOF(D=pk7Pul1@%u_BFYdT?e|N8 zDX*IJazlM41`wrzo06kom?2xKm<=7--Ft4zz0Gt2&oCRJ!`F3gDQI5yGtl^S2B`&G z#wT5*G&1N*8zrUUX*9nPj2nyP@Wshm&R9djf)Clt0F;txWb{&hRxA7laI`dN50EbM z8^JAGHX#MD6QXfn7A9M~4KM}p|8>y#EH;=TU3iov07%*Q$|(%&uMlG`PzMes3Ic*uQCClL1 z<6Y08W_`BXou8i0}?7*c&+crhn0LNgW z1-grRW@sEHx^m?g1kCK|U{C+KL>F1&veemYzrW-_c3};{fr2)3l?Erh#jZCCT+u^g^&XwM1#J*~&d;~fEjBAw zeQ;zwy30*@2jZZka%8sZ!D!l)<9Z>QLWA$OZ0z03&<2{yh?w}Xt1&@7^yaG!GM-!} zst<5AvpHB`sN3AL50|`_RpfXoM9uB^Q9-mQT+~|9GFyP}OL!}aW_Guow(Ygi6gp~l z)kyh|7l>zmAzKX?_cXbM{4U`tDEAQ&%cBZ_qIEiDWg3&p7+bW|j*fRj0xziKg`l*&Rb4^x~v6%pGZG#|7L2Q>-)=5J|_o+&&O8v&=pcZm2PsdG)=uX>5 zH**6sFx=BUq4Oi3i<#Cr$OP}vs*{oSlUc1Zf8NdQ03l4_(-yBk+=UA`nfN9b*x_8g zbOFb0m(OQ1vYP-fwdF}$8r|qMxJ}32qlu0fWfM293<~>Y#7bYMrmH~t!my8mRhKJ; z!-}O3QvD=Y_n{9dPL~aJL6<*4-eqoF%x#QhN4J_7rh5|ToxhVtzm&!Rqmj1icZ*~9 z9*mSXZ!N&fw1+(;DUvD~wl?#d1YvnQ@?0a=V}$6Xwr#rVY{15~x5e>k{L58ZLRc+l zN|DtyA-V-Y>N_w%3}q(XGNsoo_G^y2bwu8ZL_8Fy%rA%|QOnF*Rz%Eev zVdJJ0gEJi^$g0(!w-hzFd3r;)k%60z{Xxz&rqOC=18%f>Aq3v`%)EzI@~1$W131M0 z9fUx1;}&;7R(OUqT2&e!FWlvWSg>Av;0n_GTdP?#$4h7dUBep6eXupha8Sv84z>e~ z!>_-o!mQ1rWeSwu0 zAmq}Pv2gLkxd&4KUt0(|g85WNA~2R#P2d_oo*ck&^tHaNHXcJSG5V9O} zlaRYn>el4ug9gx$Gww-mY(hyH6TGMnKE15jO#r7EUJ(b(@Q4l6^|dmr4x0s*I8OqK zUIff!>~>U^gEjzKq6TI=tTuxp##Eic$p&!H-E?HwLu$q*?BmTbp!|GBYGi`01E|==y8r zZdbZLl2z}m{6>dlcJ_0^LRs20NtKo{QF-DjhtI5TY0J6G8wcBV1(bi0_WHjm(s^ma^j3 zYM7)xkZSu2GT&et3eS9Wk>eT#JZgpoFK!|uv+C$J=d8LBLXwhXv~tSra%?@uo?xnq zXwo3O$&phlTdT-j7m}RJ3z1{lNK;w`H3+DM z(?0)GE*v0P*JS_n+{<>JSO<(31-Jv#b#bhI$jZr{JAY>kzsKAy!09Rvni}l-3>i+F zRqy(ivG}#t+d*SC=8Q<0hctF0S#DDS2SPj}>9?=CR73g!S(Vkf7641PVR-p`U?ZpL z9GJro50}mU9bXRdq|UO0r1atAYuBJF(B{a7EKbvRS26J0xygIi%PXl|k3bRX;HLdR zK}<1N)QNq>)*vWTcvr65PZYr2I53l^C`q9x+*`TA4PlqXr;vU2q5)!IKsOo=@+EJY zaTt`3y4ivAxdo!ghgT%Y!NqhM)uK)c$nkw>oPB|PMO9K69-vU6+MWR)$0-0+P#4_B zb!#VGy4<_At^kll=}{LGjmYj)93o^b1J_Qp$ zUHR-k!>}vJ7A}Hrle=4V%VN+_v7)5`Cc>zT%{#sCf0*TfM)@S~s>uTT`PxtW{~bF1 z=a{*i|2X)H8E*E}{l9(iXUJ;f`q991(#(gSftx0F|CW{_fKFxf?I*h#17XADs|1v& zdXL$l*b@i^LT#$AL;Rj2PBhy^M04Qn1Arg;Pv%8$9>+KX?;@i}YGN8bG3gST^%+u7 z!Nqwd;OS+g>J0G=>j$Z!Sg8Z>GyTYD^kZ8Wyq&F1e5TgPD z7Pq^#plPI7b;VIS0<|@xR*rVV6w7D=F#rB5u8i@WY;xLT5UmEtcOPQtwy;JK7}bae z85gyUr36*S0F=mDfG~@-`Z6dj39F?^%CgvB0goX$JN)5EA$=8Ufm+T-G$0og4@bH& z^pzm-h%V9*U0#h=)b`9gH~$xWgfEM23XCkf3_tVb1q@dQvub3a+E8gfv9J**-X<_q zX|cc1E!$n4P! zUR-xf)SPmNb3|JEXpW$})+)os$Ee}vztBHWzqgyqByu_hS|`zc_!gf<|L`3Ibzmbj zK0P~U-GU(43~iW^KjB?FdxEpLWjb0!?labZzdP2*V!o8qitxuVvlp{VCnf zc*RIqF}FbP$ljLh1=@D~#IZAX5^JA!=q9gQ%$@7GeyApd-pfS)6GO!iih_I8mT)Us`sI6uvhP z9)eN>bjq0y(6=+6kSN%#sv;KGBgqXH)U`Y;e7<*_DRcgp_wJMU-S2lBUNG#f(G>eM zJOI>jH&4H&E?6BtwKP#4R47y!o9a{zKhKkeGyP`O`&oDz;->hjy(q4-kIEy4bp0p` zJ1F+H!>L^r00Evo491e^fVv60r7~{sPs&kd1z<4F%k-9vimtNdiDB^=$aUmYOpT2O zIJy9M2ZqZD^2=)DSa4f^paBRvtL27?sK^ahE^o-dSO5u9080Im4dUn_)}p`?-Aj;f zFh}mak?;jU**CHcf}XJH**nncT3&95lBKfNw5#n3K$c5c=HH0e{A2w7-W~jYqF}JL zf?wx+G8Op2%9QDaU5swk+6szJpC2~#1%~aHoh;#gcq0g(rQLm zXjj5sGCcT|qp^Pb>Ekyjk0*}6IlT;OFz}Z(Ltbgap1;sQv36;KahLCsMy|%MjUv>< zJ+JRNVvrpvI#HtsVA^WnXOg8Q<5Vn4!z4O@{vqOP7*-AMXHWTuq>ydBuK^NzEiee6 z@>BaW6K#Yg99KVeQoAc&3iUd&_J^`GsX=0^5>8_cXPVA>WMN&tMeSNSLj&?9@TfVa zeaZ09-90M4fln_g$dJJ?XHTcR)pBZmUZ>M3A+<-&lMd!-n2+6VU2y4J(Fq>g?UDeN z?RE8e-o?c-fjt`Kr{;BT_f9M{ZdO-e2SA^`2a@0m(U_CCJ$cTd0711Zz;=d^DBdELiMhe9UMrY5;)zD`IY~b2 zS@wI`j1!@mA#Drz>~=pWFN*dF%FG6I7_f!~$Wr~@GrOb{;rjaI>qtP@$f5Yl(?(3! z#@@ofU4SP9w%nM16iRz#x#PcECQaFwDyi#x2=MH^B2OAzqy42w3dU_9Xj72@NAA@# z4HJOH64w}QLuIb5x+s>$L!l@7CX#?IC}u;3n5Dzkj}q5%vvbTTrcr+Q6#zMl0Y(PY zpp44z*N`iXJb0hC)GbtliZpShLFyTiL3E}=F;@ZJ#RI&ZKp+P19jX=CZlkhr?S6EnLogYs`-M_<2L%1jz7Nb*cu zh62Pa4ZpL@OcvCb`wZ4O@bD+`=^&m`N^>$YCYH>@w9vXhvUbMf0UXtC;8MUs8{vug zSisV-G!4Xa$qEv3Tlhx;aw=I9_=F;v^`vP7*Zn-q2O}_i8k3If+@i#Su18kf1%~l^ zdUkgu@}Sr`yK@adkdsP6zKPgbe!SKOV?l?RgC5d!|`UAsbVUgnNl4M|01 znGOZpKo*k`)9N*f5jW><(CNnzfjaSAc0@pnb2l3F3QnOxl{bDw`f*mkOkg*|eT=0w zgSH*JWuMn++}hpIcAWHjNw69~cz}M3;!f8sG=3}^=+eP4&r+cw4zT)Z6)2ja%BGs( zaSPD|v{i7s{eg|OoKE~z5UY1q_>k~uPEyp+^!s}V8qEC<&!Gg)uvYr&E*^U-* ztSaq3e4JfF{vILR!);9sDZN|EX;E1;;88kaqFpe&1~XVu@7gD%Dr>enFe%Z2Wukeh z+_AM%#_#eJjiL+lqSs}HSM+vdGf8%@vRz#7S$_U&9d>;8OD*fQSMKFKn26a>LP;%2 zIJv|OF~#hdbXNJ~aL*@sxoWJ3C`rzu#01I4<)j4Z<(+G47qSoT%d zYW0l8GI|3hP>8?r*5JGk8TcFej-NKmo@LDs-5wnvNsR+>%|M|^r@QPn+i61*H#I|W zzV@0;lv~)j_kkC-yQJ3^leH`b`6c|kiaONO?pUoVy0^Ukb%18~yGEj3iuVl406Ya^ zIlNG1@uZ4c#{)|7aurp;zdZ0D^^c>mZRHf;tKEy3ElCIg#ze%I%LA%*bs-0wukaop zQQ9MarEW0EJIcY~B{={n$iNX{|HJdrnh3EqFeIiL&c`9gE@y$BJb6fis<44Bj!k?C zV0iFcwurZlIk)?7M^j^N_1(Ig_l*?Tf%YM02KzMyNi<7@FcOp)Ly3IDqlvQYZXZ;Y z%FQd4^e(?Qet*V6$q8z5C6SZti39Bu=`Jd-;|U9&#}maih$m0I^%C#fqDCkaMx6<@K4sI_%Be-{b?B zUf2qg%~4(ur)tt%gN8VHndu2F0UTx$a4$;EWnp!GK=8`0>I_smaq{#EL-NZ$Tk>lk zy1ZH7{V2e*OjJ&(`~}X&6OjG{a-Tfx@Mb*2M&QlFK(~*MIAJ~#w00z1s*+6)CY?oI zImbdz(>kO1g`Xfv??#Dw&zMT3vnz_G#jgM`?^_D@Q z&&{*xt-4&aZgK)sB0k}wQc@|PBo>V}Ccd%Py@(FNOI_7w_4<*0~x@d+o2C_rZ#%FK{G@%X;AI|t zcXe~M_+*AN;0-(xcVxG2%Gl3@UuPwv`rbY8DH#S}V|x}vsH1lYtJsZ8Z2~ ziq@H-?a03Kuw?lR^j@g;ScDoNdwL9$Pj^0<&44h-=tK=e%k@Y{Dj<3F>%*(@syoEA z<$W3c3iUcD1D{sGr~d#9mQP~yL_h6%zB>pI|JJgUZ|HiCWIb%#18kw@EkmCFGHVQz z6#!cMwhs8<@T8YVYqc(l^u-~EzIYa~+hUp7$xP9TT0B5U8L&^hMoSP6O=}r=JfE3W z&#O+}Ts^^_ZQitye0ok7KL^asXwnLJLLRye_jt^Q* zqJZ)DHLwFnX|L_Vx#Ls4ose{}R+EAnWXd-1x{{H_LcmyAIB+Rj7Wmjh znBCQx$mWI%<|cYf_pIyR0_Ys=y5+~E&x(ZIX-&62Uka??&Nq3}HyxJ9F3_UrKkQIy#+<>&MRhl2;c$+uD7p#uJDKG; zVnn0B1;;d&MK~~KTdoEP*gGb?3u5g`dqb0GHN`@2`fmRWYfOR|55EE&xeibiFz6dR zMKy6&irg`;{Ov)&eH+%wo@kcgYck*Db|fQLN$`dv2QN`T;gWDNF#wK39&QN&bGeF> zao0;$1`N^x0Pmda!k*6zE1VsV3io|oy+1@bG$@&MFWGzX@;8|QhNdtF4{lHDNX)bT z#6m{*J@QVRVCzlE*fbw+&#cuFG+1~#b5jqrWJ5}2w487)gxrOXC(cLvV_2Qe3*f58 z8UVqF)nU|WMREab1&F;-^w0@Du-g44(`doxNiXd_1SA!p%*YbIT3B!Iy+76HZ1&G% zD5apc=Nlfbt2NNH&b%1~XwqB9AxML1F=b`O>g8e}cQnDkkIY^-R}cmO<)UxlZM0{{2rSN7)#9&n1M78 z)jf{_+NQ7EUWz+?c7X?nsHctN6*fxAfs>v=ylPqN)r1;QaC>vjo*iGOknH8`0~wn^ ztK&J)6akb`&BE*IM|~E5MaY56#@a^}9o@0afIRSlq`WR+7-4y_`zg4>RL? zqQNzRMpsSE3f42Pbv0B+x-`bK0thJOIKdugUq*zc85vm#c5?~*9$_|IPaBy=urS?Q z@S&qVcaSE}4}NuUU6i*?fsA^_5NtMjTK{eD5#YAU2bS5;Uk5QLyZf-id2Ra@$23qd znI+8Wl_d~s;6>43;8_8_XA#9PM%Rt{{qFQWM&*5E7x(xvBJq-`LT{y+-=ZV7e< zwc!zAnV)&or&MRbQBZ?0qDA`Rr%T{XYcYU=hZSz1!@+EfWkdQ}EjGfJ4s>8g{+|)L zMkWAq#tCOG)^`WU84Bv55&?nF zelGFSV8>7%FGM=2w!etT@P^uwkl!+Ji@3!|c$L)W8F|IjH4`eyz>_%-B+d(FC7e4f zYxog*(3Q@PGdYtm=gcnZjnIRw->F39G%4ipc#atL_twlNRN-e7StrwY2G!>yX8l~g zURmu^dBZ`u!R8^MdQLIwe&tg$T=t2f5bZ|lL9Z??;)kcqpOo8XhF#lnsY zjGO0VO6piO*th^qXYCr#I-(q2qB>sg!iWYQ`eQ^H#v3+K0mmaRO)fQ2BSlA^L((eM z)L@d?(woB&!Ne1|i)N5mkOZP{HbxNG`(j>8x-K^02rh(o6RyG|0C#7hc{%G26 z=M16dPZiCuogwLpMhd7}#em4XZ5@60e{k5c6Om8(t!buipXKLm5<5?moh7{dnO}VKvZdf3;hRxk)v%EaxKJbn<_0rBW62(}lgb zONIVbl+jkxxfYMtE97PPtXwz0ACtQ#$N%wEGokyr#PGdaBY&DF|9P7l8XL&2_<24# zz~FPUgMY@bF;i@fM?UL`T{%#jGA{C#Dl5p}dl!^8`;5!0-ubq3VeoZ!xqY_bY5D=M zpNPS#>+`^s{!M9+!OiscGed59h^b>a^eRlf$MdWo6}yHCP|W3ha(Tejm_}y%$GF8J zGbYn&OC(`Ky%pg1S7sYAk@dY@XdLP&vcR_?Iot3GwCB{1TStVHs)_jmDI*g|z^S_j zYxTtqqrd{9IN-U^)=m1>bMyX1t;6?|#{?ah#Ub6bv$fO@vGIvGaNLoFL>f5tAmM;3 z7ISR=WEUsm(Kp?sFFft`*8^9*7&}obJt`DZ-r+>=(nR1;Kid?>fl{)T+E0aRhBvz3dyG;Och=Yl)jA_v0~yqpY!Jd(>rm zk&y7ISu3rLLcYp>pytFK%$UEtQ=oQ;m)sLRLvAbbaeZKyn3{CRyo0)<%EWss2Z*#Wd|U#Qk!`e=k{ zsMH4XEo%+V^;G^t3=-t>aF z>tzP8OI!<%L(#TZrVBQvws#+G4wZWwK(9ubq9y=iAjr|1S*qp&tPxLVpF$p`We$3(NjuE9-|;bCDbsf_fLy%%L8LF}Xe}xS z?j>GJ6Lu%_SbaUPFI6cTl(G*$dQ;K-voHxOx5o)W452NP31aNedtP0<>>_vjzUHe|BpW z>g^VXidP}9<;t!nJ4Bl@B6{QwUt~S+-Q=1P+RhchvrLd_qgLAw7ew1h-JPKSr@C_w zYwAw-{n@)`+FfgPEaH{RZk-k+jFwtz2@qyi5sL~&asdTWK%|8h1Pn0{!qaKh3QB4v zib}``TF43{0;XI7q*W0Kn2gE-L4hiSXhO)4kYExL&bxxW&g{1P>^aY~Pygw|#Sj+j z@_T>p=lgkCJPY?y1e%h08u6!Vrc9SH#g zHx*ZSG;umB{5x~(B2jN?t0F$`E6NqO8n^_vk}a=sVXZ^#%}8FXlu%FCauK)b%2@Kq zfTv`TCdgi=sPl(Q>>i^QH!VRrx5t$9WH5f)c>I{$+zWS z24=MH->mBqXH8PDpl2)iAno|w*}g_$ne?K$=4mab^^0YYP;XfSt&t#aD3ue_M5YxY zxX?>R2E=%sW>ruHM40vy2KMDwd}fDmap$r)Vef|!T>L|?5I(NvmnB@pC;HYphm(^9ZGyz|l ztIhgp{UbTy%gOj5Yr0HCvGRxWLI;L%YFPzV9=2}_RA1hGTC8*BnrL10+G*sPKQ*t3 z^^tg8Y{T-$#kue$dMGtDddQ=+)1T)7E~7K}Hman);9>-HdO9G<7RCch1(Sx! zN;G&Al_=u|Q#8W%p?D-x@?4rkwyTO-#o*o0src?2fDwXDe;V6Ii#?LcVe}(cy89JDn7{_TFk`x7Au#TP@Z)PlUWMUXWVuChD08 z-|?ET8-C&!aO00M5n?<6tBDi*M{REsA2vlTUOd01-xIPY_+X|x04e*q|7!);@;*V>N8;YB|;HjdRxRWBe>;Jda;1sK51zPre?))3qq-R6g9YQbk-BVBdd!JNajT1#PD)H8WXG9OaJmI z6nwff;(2GP)MTpE+2GQqiIHnzQ{pRy zZBre_1efp1IXpClk*;jWH}+~{Y$+SH8xODhAphl1z*=6PEb=IOA;3+@S~flZ9=QmhCQUJ+BH?P zNCB)$Y8mvjx5a}p1G(q&7pp&LB)rr|dkoO79Vfdz1U*MzGet|4+>;@Cv(g;fMJGF4 z<8(2+f_~0rZ5U=Q;9N8a9a8q?$s1`RGc_*<`f}uS3M2gc*5^*pBoGR-!P=uKUS{pcqN(X5eCm)F#-Zh1xy8?zcnRW%pv)3vww7MAVOuqNwEBN02#b;9d zNXs@R#Fa!@!ehHlMBiOxrE|5u@FMH!Qkh5TlS-@9nd`1&YxZ_r(YZ_Ox&GXT5$HncW_L1pDa;Jr`29Pd303p1ESv+L1QzBfi_(Gyth9>1$fNCJIR5oo_E2-=Mc#({%=Dmso(*cN?{lD+DfnKg}a|0#w<$z)wbip5T{O((DPg5OFw`0f95~#8aL|Z-qRibuXP$> zf6}D<{TnoNZ_RkfO(|bLPcA1^F{>=G%$+_JY!59K`vl#TUg45b2G&i>uTah7?&_oh zJ?pg0QbhSF^43OuvpD8H#f~Y7oF=_z^R*f*{-X0h}oJU-Q2{f;Z<}|j>ofv`3 z%{b8NTa`DrS>oEX-scmjiGd?dtSF5_DMk=clm3_s?vOwn8?I4?0=!=k-Ugo$RLMh3 z$3a@34>@i;58J!hS)Qq}w0?r;pJ&a~yQx-kNAd|c(%b?uIT+#?sSRcFPRo0X+PBad zhQi@-8BYqR8jjqI&MDe>&i&@+Lgvuythmb}Yvpnj_XA{IpJ|PCsY!AdxsufkUC@V=aGRcaielu9jF))mQc0_p^_e?N%`4UyOv!-Qz zlJpPyka7YIEk7g6?Tc`vMvjK>xF#Tk90wR0%b!NgeO@Z`9FP3v_-W1l`5F$;B;K6? zVW6i_Xih@x;wLwTOj}Y2C?T#%8BXhttL3oBE=(r?+QEs zzdC;2EjFzEllZaKm^!rm6P)M1pylHa@CJ9Nd_M!g*W|uew(vjG7D;8kKWRRQ`|tM{%-s|-Zr8ITv)9xGmCErRE{>AHAr<9L|VxCV**Q! znR-Cc5%AT%m$~|yYXM(n&F!`ACE4!z!wY{5=_QH{1X)KezQ<2n!zb3&49CayOd5|! z{;WFTZh^>0-k_jX0zM6V1aowk*1FOXOy1kjtjPioGcaj+d zuUYS~Tl1xCUXYEIuw%1|aU=K1z|O79HvUyR=6$zwm2BMYnc(Tn6HQmkc>Q&0ii97Wr%i@S!e4J-y0~=U z_YbAc0hb9F^Z=_YHRN~ddw6J7x~Tqw9seRj0Oe-qXXcn!HMfpF*v%}K!!U*u9Sj1k z^{s<#?~uA}FO%C7(m;vI7!J8rj!?Ze775xTfO=r)JzpzK-XF3FrM3mO)kpX3844M4 zxYy&4Wk*N~RDr}Ezr`H~LTHea=-`rZNPYk5@b08x(uy1z{k(Eq49x^KhjRU&V6b)f=gX9OMPlx4G4w= zyX>dVPQ9WE-|2(1N9x_3O_Z1fbs0r#@IQ%Ftd!i9sf5@loqK}+eCzTdRV;!I`UYVM3OJRfbmx6O34 zm8)(zZ^hx;@NL2}1>xTS?3LT2cz?W-8rx2}L_?{BmzXf6LJAw+<@^V3z++VRK?njS z58NsTkP)wD;HPT%02Cj!Y_{dVy%}2pSCywRsZ96!xesOgHyS{OWFL!5!{A7wM1iG%XaLh9*Nra=KpUS&sL?0 zsoTYA&f%?n2CG)Hp*1Yb0~yjXXR9#|a;%QW6B#wI+K-$mP}%Rm0D^svqwx?X7g)ov zZz};8r=j6&#H0p}rWlOPw>en>=96j{RXx88fZ#wS|Fid%$22USarHnsu6YrddZKh^ zh*)tJ4ffTp-1q?6HPj)+08Rk(o))cJHil?#bx9;?%gYy=kD33r0(72Ef&i$(NZSA; zUp|>XAp`cMacZYCbFc$Af&XH(e|E6LR_7)g#|8~-+;<==)17$&x}x^}z^Na1KQUhQ zmwTTwYxnKuEVC=NasL*uV;>T5WkD&uI4upl*vV}?u7%WwuPK{L5BalQ!9vPiW9%9W zWI?3%O2`vfb`RkaU*%^DK_)f|XJvy-6NiorVNYogBU{)0kdAFUY(${Hd^B;3*UVAN zzeg8boGYZ6#H>-QVal*A%;Q<%qCQ|t$rFv?Cm@9*F!xj|WZJMAQ1b2#76Xa|UWT}l z&$2#$pAerd0TKc{Qq@@vEyBM+EO1yn2(O3!JK6pL;HIJZpUbJ$gEy|Ou*HbegXG<* z$9-rhzWh#CGO`NXuEcy8UePO_rkBhaU|)K^6^eHkey+MzE!ir4Zw zjt?oRbPurC>c-|*I+L~|7}cvETkWOyocu`pXyI1U=w_gNLELLX!IyN)W4l-9XVwj1 znS+r9yjXueG7Q!Er^XZ26cx#W^u;_*xgWrf2+qEpg158|GaV!wkjbB168FW3xldB( zhT!g?IL(Zp6I4{oVPxa{8zb638mgsNctimI%;T~l_T?;mOP}tj^e%#!^>P!(Ji2NN zSaDO|z-(>h-V~5Av}pq2-{v&nrgK@dt#Wu7|~jp`t9PCvHJVCEI-hn_=-A zTBxuxE$0(0D|z^kbLxkZ^FeWlZ-=8_i$)w`Ud0>S2V1!5zwi zBZ+T>I0a1;vz()m?>S4)-u2tq(e&nBcYdd&s|nb&hF@=qenhx662owK1kPZ*3Dk4{ zbiFTol=U}$hR%`C#G1M4{BLa@^grLRzcM*72iB&E!|s3IU48#e>*kvUhlO7sesc`a z4kxMnQ>oIgeij59bzY7913LDc<@mnyVkp+W6i;)p~~& zG55@1HNVrTnH^xYn8ag8Hz*()&H9y@f5u{Cos7}mJ;I>{^M_)}%94B2oYrS7A#DSV z>G)Ru0!PB;`JodD1XIe#`4#0@yAK{NLsqcaZhETb8?dtmRuZW@C3mIrqmTOb1;A=Z zt9c1MTk5@UKMxA&Mb?;N7l^V~Y0iDDe&^+w`+r(|2+Y>`u(fpv(2|PbFUc?{?RZuW zY));RaXe^QH_*Pdkx|lk$1{z1W60=-3>J&Cd`2F)iIZMtD4v_y48e2?I445ng7GL6 z*VG5@!J}Od4zFIZd^ue#@;wVH`KQHn9jr68L!mt7&UVa6d<-ZtzAKDw z=tn%{8=M#XDC_m-Xq7F2%#QMvw^*|uwzd}yIlPpeZNmu!eb3Wi`YaIbC*ulM%_igk zK6oR&k9u?L7b3HOX_|UUl@guYUWfbHLqwi7C5B`cP(M}C9j{EYBEn7)9&azXWj_JM2ClM|F%a1a)%^g^Dx_;Rbxw{wG$O`_Fw7Tu-X;nAYGDw z7Q{(5ALP9` zSY_|?7fGQ>vJr1QYF)u|2th*%tK|3inB5B-n&_tNJE%*8;L{5In4oN()EMv}ETAwV zf2sc3?BFPMJn3NYpZ&11xa~%P*Hgn_3YK^+u6y+C-XH4S8(DIY{* zSIPO-&#Fhl=e(CE5!uiIg;7*n`m_rW(Vd3uWqVSX#QR?#ECG$^-ZoqXp6;;`Yui=5E%TTDVAvF#PPGIwt?=7FjmTa9H5&`^mh$}FoX@EVt_dlqZ! zwkp5@fSF?}qMx1F09~Jda!vd*qW2vb=g|evqdo1u3D{UXEa3`7;lx7JB8F*w8KmHd ztX#rj{LcW(5nd#{n2hyr1A%wz8gGzu8@Bkz@YC0IeyB6hw~S$NNNP=aQCF7lq=Eiy z9(Z?q!gS5y2VOfZUPFpSN6!ZT!N@kKiv(7I$C)aqt@^(K;C5eqxtlT-{$H`;r0RaK zXQlI;C5OlvZ|I@O(Db0}GkSyn?J0Nju}PyB()Lat0I7-K5gt66bpmn+LHtT9nZ$`G zFqK25%ek!t@=>VU#M$gB7o`V~i?4GEZR@kKu)2H$@yvs+2X+kCXe)KsNsMg3}0f47{|> z(8r#Obg)9-n>^}BB?zIEn+DJCu^UmA!v=n-5;c--jM=B4FL@4U6F=cO^nYVHUl&se z7sK_=uQM+o+Y3FP*1AHY;4`c?>kp8f&^3Rdy;P7$K(XBehW#^gf!dO-VlAQ0OY=bF z!ltBTa!;ZwpLWqiap2I|FnBe|*UPXSer1p|AR#*1nZ;NA7urq?b6p_b(R0%HPhfak zCd&K})m=C=#=ndRsK(~{g3aUx?V8fnr~r7<<^qc1NKLu2Y1b$3r8l0|$ZVE3C<@L2 z1(HiO@f?Q-M~^C(*WYP4$+##pj94sDI+~?NPEhb*4<3F|S7@c@LV2|oZ(BX!fom={ zh+=fwz4-n%TPlF}DzI+*Nh?CT$$feKhY~wjKbOA91MZ&fh&6)0K`83aa zoMa^026mg?BDF=C_E)sJ*trEWtYa8P3{f5qG$kOYFh9Sg?hd3(=YLEDj1`*RnTB>Y zJ(Oh;7`E+R{C-ruji|#~%U9ZWr?gQX!j8iDXtgo9JPPa3wx8D=m&3dvB%4$K#W%0| zNEA$=Nx8^hauY_;*oAIi&P%0NNuE(YXA8Ag+M5m&fjl{j87R3a`-5$8-zUjX)|$e$ z7oN^taKZ9_9NfSDe0WPe?b%e%pU-(AZWIpwA5M+DfMCievYYFP4yOuufq)g}#BD+2G{bp1Go~`?oC>bl9rNy` zxW`M*2R);$3{nA{4PEhteOsDgpPyUVs!(tH*x7--q2VEss`Iau0N8tf;~=z$NlZBd zSl{zlVrTQXVAa+9WsvVtNu4%hxTY7W@prv`TfCH>On6qk(z#XrB=D)yJJJ~)E&Xn4 z$^a{eeP+n=!tejDO3nFZCeXKG`7wW<`#pzi;U8$&;+!7i99|9%`kAo|c2)729%e0)pm?__Yl?-M@-b*Ft^Tx)*ydK9#Zf9N59 z5An8z&*Qj%|EgZ^Zv{m#^qs@B_vXGCT>ZS#ByVWDy7H9=&BMA6fBI4x!k3dVqIlh- z>5#u1w=yTj2$7Gcn=9EK_l8m$9nZHJj&0!o=#AyKz-$&HLm4v2>*$IppD+W5a){a`;|~>@`+7&dilq4rn9W_}ekdF776)yq`Ohez$mQgnOd>H?wdlb$_)!{ZPf6Y(M3+^;6pN1yB#n$P9i@o4ge?l z=n5a9R-?$B!P^6#O9TD@`xIi>@Swa}?72;@x#XpRo;=fSQ!@Rv89e(`gAmax+7Fs* zIE7)(SDd&IF)dErpv2zw6*tcLzbdmzz}~y(7V1EUk>OzO=oL3sttLY2nwK-CHedr= zbmkBy0qstC5YXy~esS%3d;NtL$KIPPYc9GS8}v7TX^&AsBQ~aG7uNs8(aj=-Oo+|= z3#r9595eWRr3p(r-AU;cgk{kA-ZJOJW0V~u79n;YG7Jlu$8ooWnxuwuUOUwcb2<)d zVZ(BZHXLPhefw2@Bou}!B+7QD)VU|fc*B%#6OT?(7d~7eoDDyvHQX(Py;aZTOmltg z>Hg}VX@q@ZvPRtJZNytv7|(hufySt4LF1v0Bo|0?n8!LS?EGY2he)2=90Zs}I8;-J zBj&!Vg?zIxV!D$hl1RFfs?4@*MO zqGVKdd(uKNL)%kos_K25mpliQE0ske0~R;T1@>3`qFs)kbqurtI? zRit2t;*_SV&J_^fI(b-3kTDiHl~gU)r%EDA#JPKiQ}2`EiC%R;Rpw4t@=dZV5wb5- zYL}XZ%FC#x!GZ1rjDiw0^P|RsuTqvVYNxY)=@+(xta_8wci$EIm=uO7MqwdMe#xMV z@(}Q#tmU~UG77OSsLPe_rYk06vs+$rCN$-#!Nu6Q8S2L|uHs@Mr10n}O+(W*^eo== zBzHH@!O#s6B@Fd!pk`W@rmv#YPZ0^=ZF1d1q!eU{(|Lm@gSpy%xb{xdTRfC1tMz49 z)xl%IT;PH9pCav5X0^52^ZI_;>DstfMIx?c7as+4hBfCV#tCn!laB!S6n|AR_wk=GzCxQclhv=PN-WnXM6x646Zm^1HNeVuCTjLRK2dS zxumfY(RbxjhZ{O#Xd&&ECPku`ep^m$`z?_1ij^C5Nrv@AFK-3oR&aUm;KSU%~&MMeLR&QmZ zPgrepUsBU#H9};k$kOwx`jHm9{;pIw**y7nQsbO^9{&s8AMf;YGMKmM)VmuID*N(y z>Dt0tZ(&F?wMsO|u>&_W#pLbpnu_{}bKckST$jhGylxG7 zu__5P3}@4O_D)48x5LawUyXZ5MOq&jE@pRls-t`JNd@Gq9@uc1FMUO0{%{iqDA|vs zVsK3{N3(kX;-z1s!+r65$?~hSZ-5H>y3IlH8HKHKT8YPvrPXy@HV`qCm1d7}2s3$srEd&d_h=f{!r zFS8HLt9vUE>-+Wv$tib(=WD>0dX7NaA0FJ<>w!98nqk~PXYY`3&xo!c#RN!V?|XTw zS?cB6bJN3-+H1abuOa#;0M6oko*rs){{SrX;Mc;zg9UGjiCC(R#+*iN2Kr^f#Txx% zd!QY@K9X;iQ6-G?(k)DeB; zFh@PCWmnW%VTgoEt~AesyQHO?$n1+!Ep_;$6VW~7(f^lrwp&ymXJ&K0z3C4%$E`I_ zPWXh}=B^@9MuO+$4)QvyG><%}%V-9${nNrAZpNSqr|YYvC{p{KpK0|icy;4#oaPq9 zJmHahb}InlLKEZ{4$Egxw#5S0%g{6HL&mqzr76b`NK$54C2*o*J<+93A@JH?IaRd>hRFg5hSt7mWX22iUh>W#j@)S^ z=5c2;_}Vc(m*PQNCJWgYd8w!zw>m5+(7@_4vy$8=Iv&7>wdU*M%IGm!+u;(9Rk>ehPC8RH8zukF`HOw9D}@vf*(#^;m#p$Gkc6ml>x<6M zJC2a7l7b3Zk#mB~RujHiiL~Tpm)Ay?@4-F@e4l;_)BURHB^mOEae`cltTr$&C4d-f z5*h|*I;Cc!%1YPR@r7i9Havu+qLCZds|R+YTjG1yv?vmVCRp#}3_U)hI$#BCSICOT zMRw$nbh3I{3EqjZ3+W=c_yRJ^Z2V{dBqkWa+_9qvuVRSUU8j8d<>oq6V%E|V) z)Wv3mjhybz2CIeW{}1|?vC|ySbbnr3abLbM*`qHxAbiN$6Xjoxtk-If^favSaWcBZ zT_)nOLlUA06e#)j&zZ|MC1)=0Lt4q1BQ=auSG8Ng=j)@qyodk1tpW-HWB$!6XMJH1 zS+g^3mqZWZHVA7Nj7Vj-LHIV{$)%lsF8wX-?8_uD*Dr&*lAEzB0Ow(YKS2q~$fGdh zLb|(Cn~RM;kna(}2`aQ!wzF>pan83$Vie@ejL8v`pA$Rg^v+qQc1Rt?O^v|RZXD@9 zQy;{E@BI{3@?;4-$2ufXPb7F;-yTz@V#cstfpPwv>5&als}n)%yF24=`D5#T`U>?5 zkzcw7LZ+d0xQA^)1zu>=jLnAhe-g)oO{|8Trz54(F diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/compare_result_pkl_md5.png.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/compare_result_pkl_md5.png.png deleted file mode 100644 index 81ba1935e69218467b006f05dfffbe54f3f04cb4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20464 zcmd?RXH=8>yC!T$MPLgGNL4`)X(CNp5D^e*QWWXZqz0t71Z*^EQlvwqHz6Ppq(-Ip zA`nP`C@nw;AwUu!giO?Z{%4+7ffzf%V&OdpQk~r_ zWIfhs;&J-z7TDeae`e8Wui~_zAp%W1B*QUbhuaY0j~FrKP`zRcR{lp*$BOte- zA1&R}w)t5_vPkZnfE*~(H!0KAyD^p0D4tJV3gUKNUs0?;hKk8qWt0xxEKzFlXA)Gj~&H{ma5(%PDFUzA|wa&EFNyvZdiQuP_<-^EolIsdyn<#M%{h zp=+@lm$WEdCeWsd7p>3vFf5>Gq3b?drIsUOPbSQr!0Ns$ep9 zmJ=e#XXRvt8ty9As-SUSpB9ffnHSm~!@#{gNPZv5InCGgrgC&cVDwvdEw42{E#%AR zO_4THePt~kNq61`Z@y{ErdcHH^lG|gAHr^@%{7i%1%eN`TICn?3Y)&Ay0(-cy%|Oo zC!QcSdC8;clITt^2(HFD{`oq}yxmXgrn~yW-0y&Xfo~tKxWOGtN&sEmDb}8hVj?Af zDdwe?JU-~?JwS)vMJ!4s%P0w-?eFavyqLZTSz)T;h~>4qUWQQ7yIQ3C)IvFuao=ui z%m$4xpH45pT8_io=Vn-bD%8lsJ%WO~XcuW+j^81P!85vWaO*U&gBA`*!O1NZ-+r&$ z#-yEOBQ7uAGYCX`zUMqYRWnsJ)8rwCg^uxDG?m8nOT3u-H96>!pZRf zcV3u_ZUswTYv}4?tg}O(|I~aG^(u0w<7wXtcmipe-Y@SLj}}-J>x5-_iMHl(1dC9|55%?e%6Z3yNkxsf0KXQJ2}Qdz*E=>mV6TmTB6bbl)Bh zbgZ%KR;?E*ICtn(rA7}eNRZap+V<~8S14tdPGJ3$>)c#DKfE7@rvy*~h#dR-or;LM zN)$K7?+tv8f(xTxz0;N4uLdmpu zBLKG;F`hS4c2x`fteyr=1~?556rYb^)#__Mv~AHLJB?oTbR z*J)*!@_4mu65j$wcSU+QZ(uTEf!P~vT&R-gqiy+mJuBcMB;QEoc8tB7N!qtx<{JWL z&Z~^HuUbb1)F?H44%%xLLdL1S zw?0m#k|yZcPAw7GHne@+E^6Rvkn$~pExypN+^Z_D5&CNX%e_7{ufW2?*u+hG?1x=< z`!ee1;bKu60G--#fSrc!2Yk{2XoHvl6IQh@HR=^gTJH=k9aQR`N(dsbyT z`U)CEPrO8&{TnG;JN2-VMYLGOv{Na7`)m{EWGquGqeS!T+XRs~w&C;LF(VN@d?Kbb z8o-LxW&oOb*3|oBEcHr8>$Mzi^Psm>1K$T{`ZfGOcd#Z4_tovyi`Av)tj;|ngM!g# z5{C%HRC+B+gy2N1$m+nB3K(|&ew)gTwuynsDBy^lRCLV4_O~ialt0ZjJ_2G&1n%zj zMrLDkIhEcbxRXDIayuNe-^w%_v@|sd=q}Vc-7n*=)ue?DSDv5 zVAn$GTckR%y#8lg_nu8#^Y;$ZdX9~ZH#}&K;WeLCm(b#f~euqVQULCg!k{92r;wh`Q9=rAy+5p;< zh(B;goW>5wy2l1g_!mLKU}0O*A6na@fzEN=5x=Y;4*YF}AGddd9$_LNU-eH7+_Qb2 z^eFQ_OelR;&N=x)BLIR;-#Kll(nmaN&v>n}$$~W57eJEzv7$(Ju0Xm?iOQ)2Q!4>X z`_3E2A_>`KqKh9U#W!)=7OtZ>b+R&RRJlx09Df?;@h3|nIip3VuhNr5jueoe=(MWS z-nLx(b%I!qR*7Z&o>yeFyhxt}<^#_ouzu2DHtyou$63r4c7sWZV!Um#v9qQH;Ox?s zkCWH-^Z<=UYw*SoT}8qYWV1@+IBBLLxFLw8i!#H~$=vujknHiRg_5SNqiU-4CekJd zRm-T6unoL9TDgYFTucb4NxeZ&j(#bo-=(Ab+i=Cc{J}Z@Ted=f{k-bomgFd`^6XRf zLHp8jN`^yEBl1K(>XfQlF6Q<#Z@aMhE`xGKA(gwg6p~4fS7kVKI8GlkPd1@^msAVG z`ZeI*Z>i8mn$=g`?zmQ9t}6zJAK{$4l$C2lB?2b=lHyqR?o|uyRHld21V^eGFQT@( zgJu;PXN?lZc`1Fcl7Q!9K*ii@Sd_ocJ3*3O&;Vg9eCkgU8D*(Jr-Dhpq6di{OPy+U zNCMCX3A0hOJdN7s?j)QaW03>O_S< zd8XApToEC6FmvLn4ajBaRp_DOrL$j$Vtt=UsD`tlysb9f^q(Zr!8zlpNVyJ=%-ErHfHd?`SW4=vVoTs?(-n91+@#%>CZx!^ONBxJj#+g((1#$|%`G3HTK94z#Y*I(O z>OZo3XfZ9X85M)K@2@#Zqe)n(s-KuIV(~@t%T#R}g4ZLqi9wHyT3kqKZC^Ei$Mx>@ zQ0vYJ^r@TCXqGw^=0i4Wk->}FpA7h@^YinRr|m4kPuuZ6|78Q&r;zcG+0A4&LDuMd zR}=65=Rd2>tmsYDk@KviqrhRtHU_5o`I#SKoG$}ckLQpW;;hz5Rc7@RW3SK{()aZ%F%dOq1>muzwnOEnpr~aG_F>~uDuTQXttJ-%$vnTl?&BFYDmG0 zRM-`ZH^Dmm3=E8BcXAjoyr<9~hlc=Wo15LNK0iN*baVKud#7V>_4upai9gyVpFK{N zU0cBA?$=?fuC~eGcIW+HYXT7+i7uF1tU8hi6;3k$37Tkj53QhsaFr@UxD_zYD`~&Z z#UE2_jg8;xl5bt8NdbnZ<=nYO5F@CZRm2&9*Ka-6hBSmm?p`(DU+hMvU#nS?_4P`{ zD6r6rGBsfKUDt*}~`eB%}aa+|7L zm1@lKGFr~?89Or0w{xR;{>Ll<_A1Y5Yh?ko04|@cPB_(4&X_?Q=vYN~hqy8^V~u9Wq-7R>3W+ebK8$rhn0+$ zEh{dZ-&`K`R4E>P*0yawa6Gr9(q$X`2G+Kmrn0Z_`Dcmm<+HktiHl9W^3)3khu@8a z5;As6Y!E4OR0ZuSYD;VP;JLrl+Ln^{?`lyJeq@UC&zU@+nal%rbmw^=P*>8%&!ae9 zE6Agx8)@e@lW}=-<-FRVjk$YY&!c4CUmBZvI9fM}iCR9MkC5*^ik8G&%jqS{?suc2 zRfsN=xK*I~yxJOfjY~`uc*Y=-;S{aJ-H=S)4NJLh-C<)^EwfXx_oPzt%5{jUw0p!n zR+9%fTH~BA&fyySt>w0A;g!dpY4a?|X7%TR_j{ zLlCcaYmG|{|2lEtd}C`Semr!TJZ?^TQwMJUp;YGYp;m^@*70kc{Z6T^%0UQB-VpQL ze;5B~nsdZH@@8?Oz@~ ze8&Y;FpnUtw_cZ}ET5R|>~iBHHbGU=Wz_QX11jDuU3RauZqfV|e5#NIktl&2B?b`yI|rC9x#sY2RpLkQch z!e}92E}Y~Bo@V1ONCwwFQ&V2A`5Ew@w_E$@o=M+Wbp^$H`>13@55YGD{OX#=FybQf zfl=t*h36IEG}Zk}R~i9nc%O&J$itKOci7Cqr-8clgOv0%f?pT)*3zrcGmc7I2Q|!R z$Xg99!S2ZmpNgyE>xQL<=04ny2PG9}{tWyPV%mKPY*P@{a*;G%Qm}D}jr*B9lg-sS zj+!OMQcp2Icf$s(j_f$|u|oreG@4fQ{Iy{8Ruo@R8v-nrIZ1`4B;;Q5eiF>Chv;Z1 za=K@frj^U>*X%vw5{9%-Tlq2E2w+FE6dKef2;#@n#`YvX6d6>R-i2}goG&K<5Zk%M zleR+p4MCsqzVQo>=4L@HnLPgIg)apdhlm37AciqEh-oLoo${`1wYAB)_6hR%$54!V zndgvr(7Lo6L0do^ zhLj2$jL;@IAHNk10)(3jNa&`6#v)sUlc2 zZtdo}YNri0tTT;42lI=$fo7v{%FJi&oktM<2QbG@-IoL3C^mvs8CxTtJmWE9r#(6hxT;NC8OO7xofUo_E@3i@DE*HyLNrgy&efOr@376X5g z3Rv}e(8f6$3=!>opG~gGonyQk2)?=#XK8wnP8P0dSW&9EQYT#(5(kd2>dY$4?r;u< zx?tY49FA;+-`+qWd(jgVD>q?c!g9bU%`xt*pT#Mt2GKS2@d~Vn-PqCFCateGi2wEL zATIm#Lonr2pd7Zm zLo3)ykX>tva;N_Q`VmcGWR8}DnYIWq2yuTG8VSUbNYbm$v4qNHa?yb7pn{TQ>~>q^ zSHL8Qko36Vf}PBWf28N`HTgSFg!;0Q(IBwn0)tu+_-p8_*BPNK6Q_$0*H_G6)LkbQ zpzlV)%=i(cigL$DW_qa_EM1N$FE-M{<}CsotHoAMnNWhv0vifA_iBfERgbMJuQ5=S4Y zt?M4x-{VHSB0yP9bu8i(m2Iv3b(ulzg-h1gr`@(h+8EI%7s%f(lFaHzOrmqz^**lD zYpo3RZgcp`L-LfZS)O5;N1i{udhbE^W3Am{@_39LKY`xMJyRtvM(^9Su{;2{ba%YK z_3k~J%^_YelO95>rMlT#sd-a9V=v88M?r>LIb+X^AwW>hzDfe|eK04x1()9w*lI%2 zcY1-o=Wo5;dCF@f*~wL_ZB-C?eZz1iT2g)LP?V&J;l{l|EVaxxcZMB9hUrtnHImck zynMOu0)^2hl0QTa@G~lI-0k#CT?^#ce?UN0U6;!2YHyS~_!~XAN4!DRF9e%s$)U%Sy=Zh1d9{&FnVlsOnrWXK z6lppC=(*)4spoQrxJdWZzQWtHEn+#U65?Rj;8pwsbKO&3meCbOm72V{)m54;XNdty zg8{{Q7EZ3iXDRD>lc7q>b>6L*v8HFI{^bW2Q)iP}_#Z!PVBW)?=mv#YRagjrFC6Py zFUeIlr|CuS07k&9lWc79gjtv{Cd)5ivf%sXPu+Zt%8h5uaZ&}fs@H688=*yja_gW@ z1WXmS(K6dI_qRhhqh<@bH-0&y8@BC)_QWyF-cpi55**K!~GwXbG@C^l(ruLwXTSEdN${;r0h68Z#O7=D754gUYV6IaR+dEC*}Qi z+n^>TQ$T=%B`ItvQ1bSp~WO z3;6-WV4lJ?LGU*K2d_!wFIfy}^XD6!J1OFJ%RZ(lghB|~4D0*t9?JcNYwGh~8Aovo z-*)GW0e!|nFwTwug0z9Niqo23>YWm5S-h19^7ggNlrD)!#_FpFM_pEpCw1(!wjD)< zbfhk0z>Pzn;}0g@RwlKciN>#g>k>T8BRGvlsH&ziM37ibF*8R)RYk|5nIQ z^(^pS(4_0%uJNW`v)~EVeQzeu5^^m;ZQlV$FQ*PJo%i54FJ;oWRkg~N#>@B+3A@Ss zd{biPhYlkXO-X*N5lQ3TVzYUc2z1Z5u!-tgECN-m&_zZjYPX2Tgc1QKKMJ%;^ zdX%KzalmTN(k0{RsyK0;Xoh^pX>l$^x&KZfnT+~EHs%%H0mtnJHkSY@o49o)PqhVZ zpx)h1&1|tGlRKCWO>ZauZegn>Ew3{c4tFy~tPjL_mkOLBcv5pUHZrLJ+spflO=XhT z@!-}J=w92o-+Rvk=NA4b>Xx=~cfQu@f&9o=NoiX2MX=OeZiFByl`z~DKI(f$qhy+u z8>c9y{gI-W>e?A*objqv53Q}hhVdG`NP55H+L5gyJF9hNx-nmM(dc9Uf&{P&Xi_gd zg^i=kx?-^J2%?#8ZorvbL`6-=@i=$X`jdwgrXwH@vR zfOF>~cDA=n+CdN}iAQg{;{zG7=h62FCiy?xk?B7UCAy8=4FXx}yJPD>#X4U(97F2m ztdv~&t#|5+{z3)u)DfM#Jqwua5HO#`DO_1IFWvV^{`b1w1q6$LS)G&hyOrH@#nz{K z1v01Y7ZP@-O%SSE{J%2uL>4*R%a;iz`4cS3HF#0SCMhV-MQMDV*Fg?x?=#Mc&!$>q zrz9x_ou$~Agvg&v@W$72my4vm*OgE{#wV8aGG9Nbw$sM|uQfhpVOs$toLHN`kmvN} zbYBeB+vlQ3_AM{|E+K-N*^Un^0(|ogu#oP5-&nHF_36>EE2|~v?$@^aFs;F^A*A(~ z%pvW!leSG&!m?$U;u-&}y_pL*o&7~XZ0W@84Y$)&-ZVXJEV9-k&lkeOasc&VrzPa8 zXtDG62@0+HPIR-eY8j54tn1SnK6ho1Up1zxVPtNzNlQrZ*_cVo9Gy!x@L7LQ3g=JR z!EW}e(KC5W=b${ns*-1B~%9-IE}cgO%r1d-g#$7_acJe9Tqhk&1X{?H!D8>MT$$*->1j2leKH}zc@kyDAc$nM+v9nN| z8{}i#}B@A;C{km%pf z#`{m=q>s%9pPnmk()d91Ir68%-$Xrqgs?~;qQkihhEcSf#31aFJE5p!JYt(1;QhJg zN^FVb#7^nX-hN1zjva@XGgU#ql$yG*(uQSI9F)@mydP|d@t!Lj1h>9qvT*7D7vv_n z%O_K(p+@N~(-R6bO_+r{Tl0}k z4;Z%=*^+Xd5G|+r)t`FsH-;C@{{IQ#Mc>=6>X1K>yYuD^_zr6O6<+Q2=iBnT{F;}o z_vy_;Z9#G6H*a)glk0B?QG1pSA^y6bs2HEuG~0h6kFNp0!M-Cv(T_%I9%|zc!3I@fgs=&d-&!j2x~=c4LELh;MkXipk~Y529HSXPz5o9_~&$l%|M6y$7%5X`gMb3yl0P-IF3=- zBSyt=Evnwt%qd}~wW5mu2rbfanC`)*T!DPAo%l&EbrizF%P3*A8O+?cB~-=M4e-#J z_1XFg8N0NA8FyKtwy@HHz2wHUIB*TqYDj%eP$=eI_9}rqcJcKeTR!XCHxTu+|5JFIWQjaTrCj$a84|@?-jD;G^}07ZPLjJP23pn%}3hQy}Q=k|1Ggm zg3*S5+Si)p-wIKs^Rs>IwNH#)uCzFKY-*YZq`sbC9chD2t6P6M6DqZ9a~n$2>dh74 z>eYPgb`IABP-?eIJCL5>yl9h|&fM4UwafD^?qgduHi59cy10>n+rY>T*7PHv%9QW* zJO8CJ{C&aTa_AXf8y(Yg$9~_zo%5mfeA`qF)L+b8>3wJp7_OUqJw~dho%~Pvn&M zzMYr<*z^LOVW$rRj<5Li-Z_qHS7m{|8w@O}ezo69*KVPm;ij=Ejuop4_trxi$e*p! zAY>_*y`Acj(L;gH&-x>iLyE#~c<~1d6IL*zlpkk+=jtaYE6!vc_IM$bdg~*lqyul2 z?biY@mmAWBjd}SlmBZ%;!^^GFJPaUscABpc9rjCTXa!%K)u`pDH*3Blw)B=6_#u_( zrNn!9bC))A$bT)4WAuSs(sIqCE^w=xsSUK9NXL=$huf z&?_ZZW&QN5-h*K{&6&ruqqJ#N(5q<|(RF9nmQ@5VF&NvmYvyPE zButfT#NT}P8%2Qt8NCV8x}Q)q98+8@)U1u|Zw}rj=RMBYZz-hPL96@^D{&?3*aK+o2J(KT;tz?nZRO zUa++0cc)a!)BC$j5+UFPPSqm)SDsEp>I@wRetQ~^>EM`PQU!yaAhl)L_WSPnn-8*Q z-7_VsIKDIaqdGgnNf+Pv??%`(tz1Yk6h2_wE3ARKD@#?@NlrmNnXz-aXJQ}ir{zzL zsz#gA*T!&WU@f2uXZon0nfLC4EPxg4QhDYB*~-q8)ntPC$fJMX@97Bk7-qn<#=!Aq zP-5LYO6VfaEj_)AQg^rJBYOk<%%N(-FHo*XMkqd)T#Cq%O!f50lAf1`Km{~`me#3G zuMWm*m4*8b!75$DrN#q&l%N-DQy0%8Gj@laO=dQ0GsT17^u}>NpdnJU-Cf}&JBe$5 z+R+jF-Jpry|FZW&Iz;IR#Vc>|2@+WjZ2oK0wd-@{Phe?FYxmi2(*(CnfMq3L^uO{J zH0f-xawZrFTgUPjSo+Vzd!D2jFr3HGSsBBYlr9Y%$n9L^ME!}ydn*vl*z&rCoe56W z9Q_-VIW2m^-Koobm?_{2f7%M4NB?sxY;2730d2O9VUl-$-@-vdc5(|+ z{*s{Mr{W!FbRm0)xT~WUZr6Gf-W5bgRCpwtB5n;uEO_{wtF$tOh98!7m%bfW=+0eL zCwY#FFCn^ir+tyjp?n1ZfAOBZrAEl^)p{kxNh<6E5l_6e*F3Fg1pZSkpPo~Na?Nb| zARO)jOZ*ck+y2d&+VzmS2H^;gt5}*|+^_Yi*Y584b3j>2XT%c63(LJC-2$-Lxp#1? zAH^rQ_`p4E#yYx=r8PP5YpMo3nEx2JF>o8O>l$qp^h z4C?|D12>ISHf}LTE_mqlPx@WcFHX-MHnEo}vzcQX^y2)s5p+=*E*C{_Ty5dJ2KSDZH~RFhGn@suAb37b)Ts&d$tr+2R+S5-grRQE|VxBeQ1 zOJxCA>Q-o=xmG(9d0rN0;dqi@;QDdt{inMkLt*p_F z$~Cz-HY`}vFiWhoY?R_6_Z2tz5+VAN(GXE(e|doAHBEi!pHuM&!|%BKA$!oWhDXE0 z{M!xm-~AV2`uC_?h<^a0OK9)ipJMp`9avFcevRq>v$6L7z`NeL@2dRL+HT{a@2bcm zXqIid;x;YIF#m3nQ$}Gvnv3cRdUo{AL155a@Hx=jpGj9Vom7>kpf7)JamRyK3n5+rl*xwKt( z`2M4{cjTf~Q1&-Y&*_;{IV3Iq=2yN_cs9K3>cf<7{{>MtZbD5*8emA(LjoT2bwQQ! z--u}bbhfi#`%V}LsRbY21-Z&Y1PQy*fl7k!DLV`HJvuEk1-13IjH)THDG%stNbiVr z5Y?O$DcYhgULm2OXX8K z_3H0)9QSB0pNd_MHuW6ObJDj>>mBgPO@cssJ3>ojF3&-=qf7nX~%qpG#PNZqrp6SKg*grlWY0naFON zd9ftU*-#c9e$3Lfu&UbbW#-?Em}vqtTt^3T@Pq)aySaaI_|;taOe?3mO$ZCLEsH#^ z8pz*9<=d-EVD1RruHJu=Tk}nrj=K^9m!%Z)eIXm*Fn7x^J_gTF{lA*=C8GZ5iwriK z*0*+a`FNUDoT1yyIQctZ+T+vP6Ws+Q9#6e2yg(gM7QQ|@{GL0M#YcSCym)9s1A4Z)R8m}BTlDSL-&SXlj04W zC3V5KfPbnfl&ESSds%|fwqZ53fpI;uWt}9i zIe^7L?N2$?7nZq+ig&*S0CW-tdFj02PMp+dZa*x<9ZJ57+q@;d@U-W;M|7U z!w|@JckRIA+I^)$ar?h{VrKhC@o7(F74n1T?zwt)maK-L=MjD+vkR@|QclmwAK)7! zLuQ}$b6ASn6dD$Lm*f0!h4K4Ou8`~V{l+z2BU7Sr)gMQCkR0*Gq$%F_(0cbF>Uux) zQxTW3vHviRKf%Mo-3Y9*goZE zDmCOve%E0HZp$PKBRZ4z^PQpWa3bKzI_P?#YvcnIw+nQ0z2YxESyj+kRY{q+ zz0$jYBy5a>vW%-M3EB9XhDuZH^aUFz#q$_WoPRX*8;ttxcGD>fz|09fcb~&e5_d?` zL9KXx3_z|C-jBD)le_@OJ>V<2%gG`tcc3f0e8=L+)`J3V1@14#-AhM5XzFP5DBwNy zP06kpQxDbzxEf`9@mC5HS4{;bCYesz_SRD+-m`H5qE2I7()w|b>b;pC%?I4lXA@aM zxj~TUSLwW>g$(g9(&jj={j^t0;qYoIt5iF^*&@X_| zsgZbt5Y8t{iywBh)u~3eYM`T;@^bQK-PU~;gGB_F>c~@L%|PogXYyShG7z_jZ)vUe zD9RaE%|vU^(qwOp1246(PgLxg<)Ko|b>H`bt!@Zoz~`8kU1DuB4Ukf!o*_I5Dt=hv zRHz_fTw|p@pFLbod^vM$!<(h?$=mYxa@xiG1{lo8OvHReRYnnfK1cW+#Or|p>jW#;BQCpmwe)@0r1E%va(L|%I3n-~DQ!3_8 z2rVGj$rGo7t5MtMg;kT@9J&v{s*%R?c<%K@Le5*#}oNj|02q57wDhcoKyW zBi_>DZ+HVVd#g5Nr^4ekVxk%L>o(4kD&lm0Of90W*K_3Jd+qNFm+|djd%jNCiELm2zaCvNxoi3Nb*G?b40`6Q>iL|sU>YRI2g+S{6G{V@bd&qrtbw1!Tk{XT1` zV(mV&2l3s4gTzDSFp#-@h=?Xn{v$3%iD!1SOweJgJ5I^rNu=;cokM0Yd*L++9a1Lf zE&dz1d)LZ49L!z`R+n8KgYoV+R~m(BzRO`Zu+#95R+iJ!FPUAmQzvWEu^4%Qyh~9uSsGm8e z&$e-HM-vw#@A5rGIs>zA%Uwj$+~(X0XJy#;TXcEnilUC$m;Uk*l;zDPsM!+zLcta9 z`3f(rze3@v)KHt{*c7D?3wb-Cu!me|m0sd=0iCYp+q>ty4weY3QS6pneu3YjpX5A- z{1MTZMom4d%8DPbGDM$DYW$+Rdf-K8CBEx~!N=wWHXF-J>|H7h>0YQXf|eWtO8189 zT(G)0Gh2wtIMTB_`*9W7(i}1H!GEwOe}cxRwEpvKoGS@YRqrlGo1nIgvB!vbLf!nZ z4w)v%bn1Ar78Wj&ILFXr@GaQxo0D3@-z5Bs4CwZE`0{s;2B~Mhq>@EOtu?v_B66U+ zNK*G?PD2)=*@&gxPN#tRcumO8)a+&E;H@eD!*z-m&43-Y(9a5^y=@)c(^UFNe2~TW z%kWqJ82<4+d&6fVp}2Ar*$b6%&$`n0f=5m1LKuxvk=6bt$j(Rq{OzvIftX`O%HDGb z`5%FUE@P={=_fBmb#0~mf{TO=a!gPMYBex)k@vUzhrdkzEbQ}jx#V$U-WQ=9AjwD! zogyUVmn}2yUffiTeHg5ol_(|09eI2V`}1>U{IJ)7HfTAy@5l6rVPpYm=7BR&`zXDL zsra9CrpC>EWg91-`4_1>e`h;L6!4(t>Mv54Glov;e*G7z8#gUWx`@O_H6Ex~Dw(c9 zs$x<+2f5WV?Zbhpqk?N%d2!c=&HuLj<0`mAjCw-ZE$p%_bW!R){9MkgG z;$^W&W4X=0*x#Tq>qma^yn8mDqx*iAkFi}rndtM!NES+h#bF#qL_Fob*^V2gEYa1C zk{;CrL+dMfT9i7@GD!X0^KYC9ZItl(bRkK(|HG{xwfx6_gVM;D;uTW=uW33yO;Bzi zB#uXt@V&_+okM(G4G^Klc7y{$LPuw9_iT5?ZXfr1GT!@c#MXy_z5?TFL_ZLituqvMS=78#4J7WBekpOjyu={cSy$$^EJ22@x% ztZZ$xp_&8J#zo}KP0LY!Z;{C&r#Vg+r)orneR#h>w_M~5wxOb|V6qDr)XLQ?KhguM z9FkP5x(IxHaF8#~E?eApf$E7A`!zqLri;%pRDSNWyIXTb1iw8{^nfsR-qTiPKI?k! z0~yd0ZZ^8&>&aW(8GIfoJu!9IS7bNe9XeU;riE*dX@gy`Q%ZOZ~)Xn zIWD0`PS*k!-Oz2;Y)@lWUd`pP8d|Y#dJAa0EZDKv_&p`6<_Q~Q=vsudH#QygzpSo8 zm!{#>0#3d?ww30YpcuwY_z?r9>b@yR_R-SZlutayhrq&i*^@Xn4}08PCqjl}=ZJ z1$1GFo{wxQZRXs~79B6`Wgf3DzZ?Ca3jxaMv%>3fz|O?HefPDIu=3M5)e>+fdCALH zb2L9{^j5WGSU1!D4ek93IpwkC>orVyLW!!KFyEFeAhNG*F1U{K;;XwE0X`8JbnGtSGJNpSdMi@;0tQ_?t%?_;l7LXXn1h)tfq4-L=- z?x2sVY#WXKbBFlY0jd30x0Jk0H5|XA)GMb0Qv$?{T zZU@UNwuYQ`gPCo)RCm8C{+ofN%yNe^ywpDU`?PhQ_*tZZe(&+|LmR;0w}@frD>{Lh zEAZ#+bQbvX}qY0KDfBj>9yo3Zd18DUrMN$^%3;xoV(E@3@mfOZ^5 z<_7&2Ru+PE;AIU;5a(?+cf>F}jCMH9q?+OUUyn=tFQHQZM`_gGXGQ*Jh}f?yT#GA!lL~s#q7e52z%~hOIm2Q{l<8>JQV^Couou6P zOlMw;(`A9ylH1gv#$p2LX5QoGP|#=S9rd^vIGuOhFylBznd4?*b4x2*M=zQ42@48E zcD+);Sz9!S3M1ckQSIkLZ%TYQ-J$abVicbEgAE?(+T1RuYgDzCGoHN)*;{!#@6R3j z=}tK*JUr>jr;{%lI(qEg+YLa9lF0RH?HuCtbU-V`=hG7!N9X%EwciI98gzneS3f+r zc^SdZ5rYkAlV7x!2?7)Sw++PIx{gLqY}7n8Xl^JCd9FKizU$<_A9yL;|AU%;;S9C* zD`kRrSKcImx#^^>72e~kJeHa9ntCC!O<403M}q*w1O79OxaZX`e<|N5lL>{Ym!C2H zkk{SmTfN3!gUq3;VrGru+%*p>!<>Q(TvNN^&8(81OllZ^NS^z9hKiV&0C96R?P^w$u zlfJAr^(9)vXrb}P3FozxBplQn$E0ErW5ZM7drQG;4h7ecr%I`4dro zzI1mLaB^IeYnDH?HBAcM=9t>f<*Py^M1TLuU~Bo}d{i2Mjx3FQ*8e9j((Jx#1r-{W z8BOt2e51-FM5iCW8bnSntX`*(9?#@L-3}c;l^xs?@%YU?0)_sKeY_w{AbIW;M^aF% zdh-^fI^uz!5pbK+Os1?mT!drgdSR$Fr|ADclZF}TZuF+T<%$Qoxf*DNg=U+e#55OhjKxB5ehmc&@uRh&iG})&`#40w@WGqY(d^!}CzF4uQb_`?Jf-0p9dP25@Yk0+4Hfvi!UP!1tEA4CkCNGI$my1R9~7{ipR7}(W5#;1Ly-}1!gDdOD7{4)h%|9j}R(?-vX`f~j} zjT$D&!(Toh`PU;m{&ObI$nJ^URDyzLAPp#D5O2CKa!a1(d0C}+VRgk{n_Xz(6RF9? z?JxjavZa%uNx9{c`im%IQ}{)cp-(8_)DePnPvt#g!P^F9vN+p3!3nv9#MPPSmW#i_ zTc$MR^UdT%H{@jO%d+^=;jxI0DZ39bmF6=betP*ZOo++B$GId2*^+KhK&)>8bUT|*OsqX{_mQ?*PN&JX|)I4Vb;}y~ZYmE3T8^4L%!J8&vFH!8& zIl3^8@^Hw8c;GcAb6bLboujcaX&Z6=Ja1|)PUIv`PLzfiG~4h=KCoY*{I_L;-fLEa z=vKwjPd6sPe)J0Zsg96ZeuRwNU&wE9Zi;Wdgdv*?QYV{;Czt+%hyMGAs<>G8(4V!i zyuS45YZrPUs3O!2f3dB`qYU8^lHz$<`X9~ZpZwL~A(i4Jf3SGs{{=s70yM zf10fT{op#0I~2^e$#35o@yI3VZRpA1EnNrX9Q6DAXl~El34O+{3~E&+wxv&iTRCiW zxthmjpr+rp3N0}V*Qd5!`h6Pj{=!v$r+}R+ARhWON|E2%pA}X+h2)2B^@7W zU(H+a`rwvke>$#rLAvCT)(|x)=ReRx)`x$Ml9T>XkT2578u#7St|22?ed3)Bp(~XkW}NXQ!F{2j97T=RN0no^xXD zirc~#bJ+~^)SGl#M$j3?_ zSa-L*@0I{VBBUfml@A^lUxnBFhc7U`nK4y4s$mP!LutLPu*LH9YVkMXEk$jHP?(F` zuOe)!54Ao8^8qkTs$0LdCmd2dQb82iW{A;LdpPS`{?)J|Hs4Ya&FNbXrQypj=T6_? z3vhG8ehQA}iPzw3*C-wOjQ4?cqlMktuhD9{N&L=8Gew5QfC)W9oBl3r1=b5TOnK+F zp(vMoeJV2Kq;+q4qUQ%vSc9KgYDBM@$Lr z3~ui_c}vW>DGPt8);ba`9j#H1q-1lZj{;Q5ywgwP7F)P^G9+S>LlNaZICl?~Cvdq{ zPWG5&@=KC!Oe@iTAn;Pwx?-itkQGntsna?vvC+x$k8@0N}nBg|Un zyW&`xIHGN_*>g@jv2mi{iXsZj)(8zO^;&L|(}3+?v=WlQ>qKY){<--ocPWByx{7aC zyim>%%$6`wcknOD-59$IktZ6lTeFAYK{S&l2n*fhY)`lz4^tGmN8c07`qwU)O%WQIMoPL}x}eP|aO zI=8IZbVEWKT`)T;xa{?*l>H7ystJmq{ztC?*vmY)qHOovIgQs;(rUsnM?&AYQWbWP z;uw$wyy~9x258aeyt?_ji_+zMc2Bq}GyB*!r289t{?z<#50GYfJdSEk4K)IFrQ&_P zl@+c&4gq~&dwJ-M8q!(ZhSz)L{4coJ0sZuSfj~P6irQq;e&{%ose5ka*}@8VD5J7_ zc!y{15NDCi;w8ZpK?f!jRvj!d4y$t{JVs?*lJF%`UE+(1-b99A!!M7@LU#pR3Je+Q z^c6PfP7fvxc{@#bTrdF!$W%SVENVrrG{`b}F+(T+3nFjY9Tm*1lgC7y53-Y6Y;QigZ8p?WW_)`Reh^bv z3M)kQd#4I=RwwPF1}ZI*<=49JHe^Fttk@Me4DYZLZCOd>av?87w)DD=vJT(4{t2wh zj!E&PY*Fi*3=#|wihQyeZxJ9r?v8iC#~x(_taP|N(Z4r;omJ~wt^nQEJ~CrDVC4}5 zyG7jJrGT0#)AQ$ONRT!Knq7&nMWv_o>ac%bJ|FP%fm?0U1>GOt0#-Foncv=ArK`PB zZtKgdNeQaqaCkwB5C*NbfP-xvM+0~(?Q4RZW>V`E^os&Uz{y=^H{00JPO0|NJBuBT zw~ik%9JAwJ`83R#%}=>apvBkewD0*R)!imvzYvk7#AL;&38_m=w5f9<{Kau`CqY5_ i+{e%Bd9bDo{>N{qY?uD~foHP}$%fqH@5}PWp8W?urf}r| diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/cpu_info.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/cpu_info.png deleted file mode 100644 index 744d237e975e555160ecdc391810a3681d05252a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 33797 zcmZr&2RPg9*H){F+BJ&CXi>GbL8+7)wW|w_JsPbL)E-G`)v8syR%^G4m{B7(MQtVa zt}V7$@kQV7|Nmb4U2^sMi6(h|&pGEl_qoq`68Jz{^&;hEN+Keni)#0k9ug6eLWzjZ zsZx*=p4mzpq9G#UB2rUQcr+Ij)H~RUV8u76VaEMs#zr%MV9<7fx&HaP z#TBuwQqa~i z6|^~YZgYJ}UjCN6DSlz|SN`VxgZNETBJ%%xOfXO%4rnlgAk68IWU^F^yITzJ(BJa` zKfOTI0m6<*}O zbf~}f68G1xcZWsZ+Jw%zmp;GTYS9;ct&=0fy2326Mx3Lbh`Q$AkD6Y#`(*4WjAZ}M zrciHN?tx)6sK9n%!ngQ)mv`x>4djN-{`syH8jr|a|DD9RIm2vCuw z*<-y_`wQdXRfybsU@~hvv1wz_IF4ig*Ig2k=e6GD!IUXa-o%)F%w;=((%-?nNM4o} z@eLK{J1{vMqlxrl;URh)rL7-MWApa4v6o0k#71mo!aPwdDE-U8iwD5p+jI03?jCk2 zf>IdzR@$8%!`FSmQRDGbV_hc7x~`5r_6n+Q#Ur&nzb%GD&sl!-x~+CA ztqa8Vy))C|_zW$2X4@!L>me)$0bKBt&)Y_5%8JQzAQ-lpQwu45du|T1`tX)IylbE3 zAAc*=PJ&&K{g;msoXsb%NEd0)@q^;n#w3X7i_m=y-*@ESqR<9wl zOD6UGsSl5a&}(qmXK-tELG^4(T}+OtNp{M>?!{KKVilSZn&HWrC%ZZ~MwLIUP`F%! zJDJ4x+%n1CTKkrG?5|5taR&R|!P^iLU7WTzCRd=H4>GefD8 zbDNBqF7ic&#&O9$LiJ~ayiI5rF-Rdj;;`^ZYuvU8YxDD$99QzbPu~GO6C)xy^{~H2 zUTOMBA8OYSTqmi{m7PrAYZ(4gKybUvv6X6ki#nmBRrq#q*%fue647D{lvjZoXM@oB zF1tKXK4#Q;{b%y*`=nfNDkhCPP)N62n^qhBaAZ9W=i zh}INLC^(=5JXGyDZ>BxDtwMh(dE&yUC}OV7EUN)Gym1pzu}~Ot&IQLUQuUCkc$d9& z0Y{?)1w)92`T;a|ja$bu&#x;DRDscjEu;3=^!S@8sLy>tn~Lu7m*}*H=UU@40n8qv zM)TViE^iF_t8=%&`=AZi`(075Y+UTj+6=CTLil9wbhy*fp4tr}598ik=2MOibt&cniJKvd`Fs~Q+Qm16<-sp zZMq=zyzm~~i~U+{g{T&mj~{@Y{crC_$W<)~`L;7RHRIAzY4IycQSE}Pw+advZ}UcNg2mugNno`Zyz0eU$CAgquX4YwA@3_D1O2k{4n zuhG*Kb^+yE`PE?2*bR2s?;9aWP2I7QgRIJZPZnKgXd;di)5c}ZuJuMx)BwxT6qBIxv*!!7rvmqV%A%{ zQ=Qg1N4iW_mDUe556mZE?|SY$$cO%4ZDCg)1cOmFk@ z&j~)-y6#PR%6&{aYdlF4{9%C`zRH-;YZI+gKnhxQ`@mqXq%q=A(fK5z)QwzUtu#0a zV?vtZc$9mXw4!bs;GzMicOxIR0W@rBLd_;qXrQ9m(#J9>YQ=p#MDOq)(4q9wkR!cC z*+$)L3-@UBPadcu=+8QCt-ua_K&PxnL@xi@hlLXO@h3;*y>4T@oe&}V-;Wp5oP^x;qw^w+|MD>H`I1Uqf=)-e27}?WOa92QQ ztmY*}5=IYIr@!i9c8R%cU-k%%G?l-GiCP?>n)-nG+*=}?u{$ImElLT@-KWif_;xhW z82r#9W#9H$DGO`2%qqCSb4gxsyFz)< zF?_9Xw$=u8HJbf$wdq*Zt%cDY z&O8&P6pP_hmxENE{pH^>&=iJSKQ^B*l0j>Lr&BT!b!7|zMs*mOukM?3*fEjB2z*%o zq4hB3H_LKWMBsq7wC36w-bZlYQ1REM^@=He@@L=d1Yc_!S#cK$aIdYZC6~!J#eYb8 zD(p@&!debB7iL88w>|UEu&?=5obQ^Ay;t*)w0eFi-@O-~x&R_zvB%8^b9(%cp>_b@ zshqMleDd0Iw^1V`9{9in^jU6`XbBSIePhmZZwGgr4qPq?!6F{Nw_3&T?T5{I*oBLKZ1)!3vOsju=aQ{( znqKUM&I$08EHBxPOO;lQl$hRmfakQGMX2de(uXmqKRYb#0a&SaXy=R_?(wEzr;aU; zISTwC9qlDJD#WS4teItV`(WB@;40G0l$Ua4`DQ2Z2ymo)(_#F(-y)FsE@{jYulW_R zc-EFzBEG7NQo!#W9z_#Tgp?oJ^l?z%B3(pS@uG`Ft%^mnLtzoy6DHj@^wYh<0HedG zn6@bws|hu<^wlUF+9AU7V{nIyFdV#Kmild|dn#utA;y$(G5W^zz4XNR<#5B}v8n1Q zzzY))Mx3qsK^n2$oQ96V0E%QtcJbv-|Lo(_nD&r7D~7A5D*6fIe27c<}RYh z2mbb!?6b@7-xXH1@e~S2m${?*u)|q?(O0W(D6jRknJ}}!yAvoB48Tt8ZpU#K%|L3t?kD-_yE>9YlM@MOF`ywN4u9Zxife6U2j(45OP88Gp)zl2dRHe| zP^tLr!*PJOpM?3Ihr^)sa#@_}+}zQE-(vEEhg;4W`L280-aRK9@etD8^xg-&IJ#yO zT+_WB4UNXmQ0E}82{L-AuV^n0%x>auj;pdxvf-`&uO24Et52Kw+SS@jRK0co@#iqk zYN3&tmy=T$AKFdE2vd8HA^ijmFOAWrC8>A|ICyi%{vMaL$TUO=g?~8ola%|gl~)8l zNMAXa6%9sCqzHZF-p?OXVRn?-&$`OGykFT9b_s4YUfgr^xp!f5L?dCy>&XxtW!n22 zVU<4vI5sL%kpqCP9rI!gSDVdOrYUVIrr*ioXist84vDq9wS1$6mOtZ0C5oIv!EUQb z=ce99Z1ZLB3+v8q>~eCQPs+1`br5P|@uC1Z9^m&)&_|Tg>`pXup$zh&zk`4EDvGzW zey+lMmh?@O$v|g|uvKg4A~3_ytxqaG<4|M|yih1CHqsXrlQhl{y_vDx#?r2h94r-$ zLEIO6v7WyuTKpL>;L`3$5`6{azo<7mw=yj`MRUrzYid|dIN0yE7C@A8(htwwy2NtU zhJTy-zt@18nqV^aK#7PVA>6oQ1OM5i{EvMw786#!?S+?KKCkFJpY7&}T)V?~;q*uT zVS$zJd}1#Eo#t(BGl8Zd|Id5jyadx>?0*@yq2{er~sgm8f;@e(Q)AZ3nUzsFUeo*0y?(FcVP(#~MBArEv zw3R1D)m^&91+|Sya$AdnFTa<1UoOth&M&4Ju*j;^IG$fGssiG_@6?~i`~4?r{I6U3 zV9mKGf!e0xk(|Ewdud%LQ#D0?+#iwyGxAt%IHBBbxgJGBdPe&<9n zH+f8+`D)%6bv8D#{-P?KPF3x=6)IOmp{6vEjyE^+90NG!3<`D?lP37dDm84v-)VWB z&PfE*T9Fg#%h&+l73Q${;ux-8byr_4EvAhZ>(I+GInOmq^_9WtLSORBgehzJ))@uo zLIfcqA|fesGEjdB7s_s{DGLx>r6jji6FU5+aU(t0WR2?7LThU1$?X`Sc7qJF%q2}t zuN)Y}&FG)a7$%kpUsLP6UBpGl`9i4E68>!u=ihs^oNv4%YcOI(yCvp&bE(n{H6EHV z78)YqFJ=Y6Xad4=i_0D!%P_KK$?r(jb+I&tgOo!@2*Jm(_8dP*yhm zpeMg8PqfilZ|RbXdN_W~iv>3xMy%1+vNR`&_c>$7bF6gacJlRn=dzKeTFk}T_FrrR z38|r@C>{(h4g#m9CnmF}JNg87`jh6RnTH@#HiV%NFr_PMCF2|abO8KzpAL@t-5CxK z_aT@IM>gLLP6bl6S!q0~r2Gz8w%+=jOQpG>kVh%@J%itle*vn292Dl~xSc4PEVc8s zT+Je4q$OT$6K<*1H0Sud9lFh3PZbFb-1k$91WF*zczLC-j{xSr)sgg`uv(#8Qvc@j zXY`^x9zRn1zYS-;H;haS=8n#OGg`hWktk5)${fl;WAm-=h;mN1IhOh%3wLQDI}J-e zO6j5p^k64D(_R6rp(`^KMEgY)G(EGmBEYdk=@pvH^;!|>+JAT z(%c`jNO#sOk+G99k~02f{6BxsmUwMS!u90t1BK301JD*2JN8dVD5D&?#U z2W5g$N+rFahb%S9XywS{c-tg^g&nca-e5d&>}kdaV*DFuYEWWp^uAh7r4{X2+!zk` zOfK~ReMWraK4oW>Z2M^DtsN;!4M*>if@%9{BKfId-cG}=eC(>;Vs(U?ez}84&G>tn zQ!6jKO+w2A-LMIv8t@peDDx;}wTHcJU;Q<+`fQ7i`)Ym@-+57%JS|sU<%Fe{^Rn%p zdLIEC(A|;3ZOdam%=?sptu+ z18|>T2YU!i*F;_qmWHrwYK&{5lm_L7kI6tcIsHS^2j-Y_e%2n`KJ;~`s)bLBJan)wh?QsXg7@2DdUnebpN~0?-b8zZ5x(0_6o?=&C{x*-*I*MPLf>A~J@Vm)Qqp3=;{-Ja4T*P^gWEgm9eg*=-J61iJ< zD)#XD$A4EGYwpx?D#+`?Guy6k1#}P>PiPeL%*FGI60R&ug*cH5z-Mm<`vJ=i^vvyK zF3%xtd4G`7b~K1`>GkFd@s}z?QbT5)st?D7Q&n@4x;uUzk18Bc!u5v*6TOCSKRhiG;_JfA4hVEL7XTzqD;`jB0W0; zx>)A^1hTI}i;}f9vpK)8WPZpDi@Y&`B_6nmY}H=LFT|PTM$5F!=$)VDbz4sS59uEASN)|yIrwW7n9h4R!_>CJcqs@ zKX_hb__;(uR+OkgI5vPfp>P)(b$CfUB;dLpb{E&N1~=wEzBL|P-2WWJ%WQHgqVK$^akAyNOC&?RA=^9|=vHT~zW57-3` zMX?P15}ku>y{RDgxKr%%A`26Tm?IBY8u;NNV6vLWHP!3dHi(Hnkix_x@upbbWjMHNddK83q-l5KMqPPI7;?IxJ@-&ypclVn>{KC}G2bs;(UbjlY1igIhNaL-R|pQ>+04L%S|r&>56v;KU;>gwlb)^!m?< zM_#8DH?cK2@+$%uxz77U6lf#TU;)zEy4r}&OWo%|=Rup4RA(HN@GI7ekl_P-1Y5h|@gw!OjISU5&ge@8DEwpqd6bZ9oZ!j3H6ix)@%=}uXA zQJ<0k#MWLCD6Ra%o2H*CJw7$;t4oC@nPtaR$1pESZ)m#3=qc%*{V<$9fuA)A6SsR@ zEWvh}eO%djIeR{kOf&=;F&r&8AQCJrUT0R?q?wh!6-wnnUO#v7-3JQCm`*q{(jeb2 z)@Ib-KKDSEPu+#1z3o*>GpD!8*&O+pb&Vbv{K>4zSjGm~D>wYYE&OgIyMv2D;tbYy ze7sONqpzdGMH?BvJh3!C%a`eX!<4&KF3>L>HvT|WO+cm*!g}N7A)y18)x855dRjQG z2~U2qIlwMmcZ>s^@k1Z#24US*j*Dr`aq$7&&Q+Y5w|r|f>&sa$Qfcu!;qJ$1z4cMz zdMm*4M}x@A2j?ZyM2M*E<<%An5E_S<`7>Er+L1v*%?(i&z9CVwhvKsX5K*elK0D@j z&0YvI5E*>CW{3ia#u{CH+GKRSTI9wCJFauXACfcsc+qulbNx$Gp$-6hxte9QnjlH+ zIlMj1{%}0xsiwVF!HtGUis7`F03S`9y*s2LNT|?F6ZOciQ>0o65a5tkqdP!^u+WU+a7 z>LJ|3rG>^e%hRL-+Z4t&4g3aYUCBgF9-4kK=$_;`v)rr(Lt=fK8uJeA$be zQANQ;Ra$w(hk4RjitTFk)L^r)Of1IUqt-d!bQ4cw(Ixkpw$hY7wPI>X1xdy6+czY@ z-%k=6}*ldt*g&3Yv83;jex8pSfv=8{nYZCLU>vw0H zIkbrDC-Zkm&*)Sha$c@#{vNXB+(c1;aM~^RzW2W+C%^HOHOyIhnp`$3-T(ff+EP_W zQq$P)FxM-?X%zi*zv-PRRlZI26qVPxGXbGb&QQx2AoRxg=l8~EW)aye4(((qGu8aW zs+Wxf?hV_|A34j+(<5x zZ^VNaVyT%1irTI5&3Ag&v?mFb=+5dvZul<7nHC{5qJ+F=Con?K=8HLn+}e&d`Kmc4 zTuV?~@U66@8M;-s5Z4=Z&%>tu^7q@Je*3Z@dF<_aK8wq>?*KHvZ=DWpACyG+1t5Gf zR<*uNduU8i@-5loXEH2LjL_Vh5f6>NF-25K*MPHsf=86Vwctnm4MHy*-Jnn~adwo& z$YkphG5U$rLhSA@Zuk?>*WoT-a`tMiZn90$K$iO7s@)h!0@@S^hF!N8{F;UOz_TH2+uQB=EB_jUhpDw`Yr4XBr z+w^8{OyBrkX3G{YiG519Z`c=Oh(*WF4WqH=WgBf;LT;s9ZZm?6mduI=wG+Br0|l#I zpYtzDxm@0KVyz19hE0d;SWf$|*W&)S?*FrX%!%03q@TuIRyc91i3@`i`;M=>$_RyP ze_J>i+xRF@D3p>_tHXL^o%I6gT@43rLM=Q71_^*J=kD;dAw?*u#Z`81iqVVsbbvgt znDFs!w^XHnF_Nmo3~igcRi|=Hdk75?B4QNhJp$kz5(&%)Rd2bxd>o5Ns4g-pUCX;A zr$)8+uBBD6?-%#fBmGZ7Nk^v5>F`T3Gbdd>r$7-_Z<$C=3%2vi*ciXnMac=R{p*vn zKIjWaq7hfq0^S=~hNy?_%u^u4(am8xESV%GVE7HM;#OC7i$G4=n}$jqXgl6 zu(gY?qPUq;!#i@hv3xfzwCqL{y|>!XwzAfKfc&Xuu}`ez+JoBX;eIJlpAU`66cTpp z6%_1k$2Fc2Dh**Xh^_kLuB?S;W%oB=Jj93Q#l>rK#L;b8%2y5KG#YBkg)Iq!ZIdk* zkNg*>ScCzx9N5|BYa;sco_p~9Xw{o<8*loygPGTp+37hTCRS7_ z(9&a)@ddK{U?H_*f4pJ2e`&G3T;aUK-pg$ka=%iOfgH36S;Er;s#!om0^~}XeK|gI zGvp(@NwfxobwWP(lYlS&=I+6M0fZgA;KrH$jDB725%E;pMQn%D#?Rjv$R{8i>rc$96 z8~7)OFUh=|k6W{rVkO7}ue3rbwwDf1+Jc52F(}Jpda(QN^+@e8#NbZy)Kl4CondPE zYoAxBju4sf*k)B&<9cL>luQ=inysa*1^Su;TZJ9id_o@6Qz$Ceyfi)7n1wqXYv>rC zW-1_Mi~weYQ-L?Q9qRLCG4NQ@=$Uj(a$EI{*M%%(|Up{C(#raFS_10O-i<2UhH~3ky zRaqL=`x9rWldV%a@|ibsOe0G=jUp>yAh4SD&UjU1)GA1?pCP1hBi?%l(#3QD!lX*; zlP1mnJpsv(JF3fl(%*13I#$?;uy@Ou7KI#}bo?0X##+a3p;L%U! zO7fG0YRY(YZQ72$2dv@*FL^NF8&V9an*C~7jjJhRFa7{5pm5K7`Sm~)_iSGm7V^;W z$$`lnEwR7o@rbZ)P}rRV#l1Z3TW~?}$DQvFQN?U&G#DCGzAXxxF|82>`+hwaVV6P% z+6+oLqt2kCe3~hMCX0zjjkfoQ>HwOoAALSdr1qw)8(;9Z|LJxXgMB(r1~%A&ty-0k z-PDze$Oz%b`$O)M!#9b4Z}RcOAIwXh$#*8tiE33uzqyBXZBUaNb7P>TBeP(d_<4kT zYSzly>A8)RW+LBqN#^;ez%`Q7c=*qxj{SL(@f(;#>H2yj1v&qFPr{6=kK7kIf&{() z(l-eKL}0wWAx|re3!P^rYfOoHb^aYx|9ibAPoBq(A)psTR;T{5$ZSL8U`NZZMt%-w zo;p6_0Mnv<6DkAHda})zX=z*_US2$QA`FnsGV#{^-?i3+H?`q?q{=FKY5Bd!4>x!ZNDcYy3zPT0(hx-dC;MHV_j{8K7Y2 zJSPs8oJCDgyGTrOTqe-s)`n!l++0BsDCc>C3G-swtxdtpw&$A*!%b}WZQV-odyf(K zt1rv$(0)rh_eSemzV1(%D zjMEp+6H->NtVVNePYLovy6X{N^(%sG4ijnBID)su+*V2&pYexhji2$MP+qQbO9gLOx!!>~)h@sE5wlf@oX%rQi|QNPI=HVY;5*${;w%dNPgC&M zJ}Hn1b6+h;65TXQt1dE->Sf9Oe&6=bTSz1Is^8}?$F`b&#gMUGB6w;v~e zhU1;E2}Wqtn^e>3j$*Y=RvCcVBiV%Cns2qLIW%*YukI^fcmY_5wS6075he+ZLxMic z=>WnooiNH%oST9Kz=LrD%Dnl#Dha(ke9}yE4k-i2wbWI)$aa0`cp8som!mQnVQ8eS z-!0He>B&0fSZndrUhhqNTlRpzz{xS2sxjbCp(6~ph7?9Ana*9|v#Ha1xY{H*T0rRZ zNIhga8m3#lQ!-S2Y0J1EFo9I%$;QuOp>Tds$OP2|SF@YZiohi%zH|)g8WVSk*YQy^|y@l0lgVmB+{M{+xyhhyqLhbXo=M{#{Ro>21&CN;}YL|~8oEigu z05UsO3iAkDkaFr)ikMr0bza)?8V3k^$^<8a1h=RS6_MTMZv&(fD586PwO|MM7F`B1 zvkX8;Qr4ul^a(xFhJ_Tm`y79b{2{w&a%h*?qD=vGl{lny2)Vsg+2ZN;Z>sm&2Wv4= zM4Dh$*ra@yz$zm-Kip>y42>g{r#8!6h9 z$jA%ycbh^j2z?kM>xGh)qJrKotVoa!709@YA;A{`Tv@ujf_;X!sZEJZImqukOxUY7 zKp4Z5O_n$5C8Uys`-UBu3T)TwA6$xhhUQ8FP@RTYf|Y#^SaDpn&r5SCxPy0Pf}a~O zFD*$Xt?WNxy3i>KiNsk`I`{t+({_2!6BI!8VowcD@2XU(_V(Nt1WYzcCsJmWs9Jb-8s}Fj9kUw#VnY*_sujrrqmWiF#}&gEhfb# zHIr{5m<;h zn_saOJ|;x9H$#M!ov$aYa*gBLKVJObf93pr2ohbP^49ZOWBopVGDN`L3&UUPazew| zRBWbV_^Y?Cr&k}#Z|Wt$iidT}bHHfn=S+JfZ}5>)ZZd+V-3?EQlrUo{f+!y|)X zsewXJ`LUC?Yn3W;aLp|i#J>DsXSM(S{yCRZemj9;BMi7S1=7b)d=O%?iV!(}ln2f( zS9%~2prBTM_puw_7Yj12j-oG=5($L^m5t}X48b2HLIdcUSH_3BTEX{qmTmU10Y|l} zyH|kTnRyTR^TB*;7v%rau{>FE^tDYX1*tTbVb znbtO(BJ-p4z(+AUC=bdH&X?OryC8NZ#MfBK;VQ@lAnI!F`58*hs&ccI(YqzDsQMg( zA1}mYcVz0yyf{BQ8&**lz!K)t+v_-bcMbfZW0WDf6cqFnvZ%$pwI{5N8eu3A#T4(u z;^}6@tfpzQE}Y$`lgJ1&ClHqI8k)%H;ZJsq$uqatpGCEVy1g-dl~(O~xM$5L5cdD{ z0;O9B5T@oo%GV%gKtNWr2?R-V@ww15na`B#qz^?+|4t)FcZ>GuZ_cvzhA@(JXqr`R z80SELR|>kF3X6+6`zA!>@F>Eh)wk(UTWt%(^lN=P^~?3!69$8#Eb4g=j|tuKBQcEG z{Pe9A&~U!qB$cdPVt#VvlSueylhfWOa+23Ms`vK|s6yED`Z~^4&c82 z$%3$7ZQE1g-xYFA5Ah7ZL7gpe9x-6T}LQ~~x48kmg4#(`{ zy-42{nU!z9%j#NiEZK!-p;QL9c4BrwwiEEzrjeTiv}vO%>U%cB&2yBhI!pdAM&cKp zMlbIq`pG2d>Hix_%lJrG=%JxXr9yhfokdDgKV8_*aa(D+aD?0{DrkxmqBI}OD^95v zDc*84LgjlVha|l_vVD~02jI63Sh3+8-YHpzl~mK1R(J~^&+UZRyjADJNFU#St4=pJ zQYYJ)&tqu*xUFaRGwobR8J~RI3iR=)Kl8Q*I_3wTi$=ARPUx0d;kTk7#2W4COjsi- zW={xTD&~7u4tF$*qRBi&JGb5hn~S#?NVcWQ(%;^H@>p%dHCVc+z1meQFUt0DgI@qu z+Gxq{a%nc#{l`PDKk=GB9tH$#<%5++ZHZZ+)|MvB23~2P+ zXG^n*V$*5D;Q#?hrr}mZVbv%FP|3yeu)moFl!-fkI=xTGoe&Z?1qD@0s8U3oq=W90 z`)!*ybj_$f%%tS9ogcPgH{Xk&?i88VaOU6PL{{IBD}s#wK6rXSX@y$ub;j*zJ-V{Z zbl5LA4sz|!A9^?B2}!}4>?SHP3KN@|Dv)Nd`fLixz65@FcmJ=6JWyi|gaT-%kfdyN z)z?R8o+=nmRAds~oO{#ECUWfdw6dRrPWXbVEZd;nwPDBiNgw8ux#UI!go=k;8}QE# z-T+?am+WvBiY@_+^pN91V07vlee#8c&n9&_s{r>M14XqQZ=YK-ENXZ%SsGO_E3daM zoBNL(>50H0uq<#O~># zbV}6&^;z)B3NH#Ph*Ru9So-IuBI}Q`q%C1jUz=I9$~Dv~tlPrI?Rdujs-j;jdGI)xZtsQAXdZwhZQ{3FECwGmfc`_|~qvq|04(cMny& zeA+Qicpni-EIrgxcQ^<9VI~NB)8n(VhOP$k2aNj}?T-+umz+-`Wb*A2j6b24yE^xG z;Tj#^!Ksc1ux!1Of>CtLF$y2z`#u!u;2cBaJ@#7%7V*sMR@p22LtEVhaB7SVYE(=6 zt8D=yXMA?0jh6H2(L2-nnP*mN1$D2GXVQW_FG(x|G*tLo%6Hah%!gkdI2EFU=8J~q zMKLo{{44SZ3&^9f7IZ_W=C`w4jfezB0BJtg%0xX08Y$xMN-q+bz38BJTT;Z-JRGNE zl%P`AxDrq|k_w2$y(e(iO`6qg#?`atnkXayFVURUq|!CKc61y(!{jXLerm6QQ2-O} z!G=T6UGe(M?@!jaCPHZ2T;VXAwgX>2U{8A*lklO*=iGYUIc~xU0#pCbVFP8=z08Zx zh0geCO$|)Kp0k!fbS>W}K@e6&@z8~5a|w67AHUVwyghf2*Y9FmF&p#ce6Y6E9u5q( z|3>j9a<)HjJdp|U@fgO|Ts+-$eAD!&B>w*vJ)ur*sDz~dY*E_#iRaONZD=npPQafa ze;ipa%Z+-}KW1fhEGfhBc>UgXCTTKRk|$7EAvc4yGs7Y^HDI=~cc#;%(NC4?9j#mf zYQ%fP17+G=Qg0ar-ibGzetRkK(|if*hdI63=4>j!5o79W28YZ)Q0(?YD&cWrD|sz? zt%w3#q-)!@fz}@BLLY?gPPtBgwNZKKt?Fs!2S3wWdF`_#L4egv@LLKCKH1eoP6^GT zVB-blPvP_IlHjzCAcuwL?|C#q?}5o=X{78Zf!LL&@maj4s?pdso`ixJ87jmQDNn1?9rH@`pZSM+ZNYOY@ak#_iEwdy|>438}`}V=J zJNq(YAR##|(u%T+7mqYmI*HZhYy@!Wxqk)6gYdTW)OY52cg`=*pcm*~jjK%vQ_ksc(RsPjXaa zTMn8jsC+WesH#IMJoZ<-I&NPa-E>y|BKSOwxVpzT#MS1&obT<&P*-zg=!pHs&H zuNutv6TfG$_E!vAooF-CK1!(^{ouHtztyqLqYS?;(O&9+1wNw{>tCZ@<=}up2wnIq zm!fW00)7eW{`wK}EWN{VrMRLcpb@*-XGWSZx5sR^E_~K8ILrwYCxls9e)otcd!!^J zvz$}#k`JNjUi=3A_c+pPpUZ>-I_&Arj9ir0jbD?uV+2wwNK@*a#l>Edyt#?{H@W%e z+KvC@)DwrlV!_yj>&}vDXTy*%nRxP+CqI9qA>o-neHADFEf?0ZX9ojOT0_Af{}PVG z*3=33)IkYceTi+Mfo!fH!m>+_UVl2jR9$=fucJnf*vdjp3G;~(O&vwm62qD<^OD=4 zOED>w^*5sHmVRaAhY)%l2kc(LdWBTg6dnUh<4}=F|55Va*M;GJ-HB%kQ4apphI8lr|E-Q7d3EBX=UH*L=EKjneOPy}-I;H>?Txz{M<6&fPuIRoEG ze@s4vxtl4hwwWk%(T6=$xbEdbe-8~OO*okF#!S%qCagltw5VLJ64pUNX_{Zc!H5X?O9<#cC0zEs*+OM?@ zX>C-99+PnpvAC{=_%s^TrbKgAj5TSiQ6@_eBIYA1xb_pO;9|ZfndtYtB{hddq4&1ku6=Huy)C<+0A{1@hR|`(gSOHLh(RD zY8|nG18!I)Ow`*;)vVeb0SkYGxDjroAn}|T1YJ;M5uW z*(qV&jrRo)cW-ci5;4~ry|0~dcBF;3=%dPpU9H5#SH^i5cpl4o%{qjL;>P>*o1O?Tzv0vTW4qdcvtmc zSIbBv9&ywRSVrKn$9sr_&ZNtS6)P2KoU%J9g~1YZdS~omT0=NGw0V(>3nK?t9A>1?`JTik+P z7)KbY8FaWwIA(v@Vp(pDNb-kEDSti^8v_Sk?YPgIAa@2LLHM;xRbxoRH2 zIT{$pe<(f5C;aI!Wn1VY;yCO!pqWbiAw*z${CIP2MMmaH?QhW?tlFMC{^>zp4=rY$ zu68}wB2Ye^_VmrtQ+vgkN)9Vo&mWWtT*(~l=$%NjV=kzFFS~GLs!9*@ji#I*qLy27vblb?9p_{Beo(|hd z(QViZKFZ4P+(t2)mu?>)0KU%`A*3D55xXpq>W-9jyDFNwyLq`TOyV2_~F0r=dc*!zb=oAN)+>B4pB$DAUyQ=Yuk?leMJAPZ0&E_ zos`@8+O&HvywGPmsGcVk_lzoe)K9JCq|_m>b;4qJjDsF9DMhXs28dly)N#8wYh@n5 z2rA}s5HPacc30Y~-NAiL^A)rW>C92y;Xc^j5eHm+)dzYA0S&eMaJjkurXVuNILN#w zixPN^JN;^$^myz^_CXv)9t|AhYhf?;-yG>(;oL-l1`*A|dibDKy4qL8_?}nhkq7e7 z0m=cyarG_5>?jz@J1B?2#%s6#f^6umeINKK;^UJKnUEz-&?XJl48K1iOu+>qO?bK| zT-z|)$z#fJaL6NrVKUa}j6%QKhRpI?-5fG06m3ZnnVOYLFu&S9wj0>4CnvP8N!l7} znl>abH+aZD2)yQwm%THfLdB>3;JZ{E-*{%m!LsIoll&NO_+NWf!HQ$3%?yS#y#;5^ zypMZfNLzIg{NB(5{w{=3$g(XAwD~-8OrV+%SfNiVplXELh7Y{_354&_bH+;Ad*%^^5<3n*YE{WvEX+J+$1;lj@O& z#-rI70R!YBIDF2IDyGZ_( z@Le1uGHuTvNa?n4|6pgg1poSdpqOAR!XtEk8$Wk!siC_NBVd(QPA?cBh7Sf6)iN2NbO`6QHRZBMiJQiuJofqy6yOkPUnC> zZ7vLisFr;4_sYQr>!$+}rtt~(v;PUFMWD+kKmSi!B_bDTs>vWtfMXY}zjwb#{sppV z&(3@AvJlA`^$N6Dn$aciJz7%n)w8sS%@#2~v^M^j1CV)hccBE^i-AN&qnSrB%~_~@ z?BF)46*j24U$6golhvy3(kbg7ikzr5i8bbYvRi0L%X6Gc@rL0M#qOniLSHutXVX&n zF|0P}NALo^9d-*ksBxj&r2=!!!30PSY{}7IQ73Js@k-}7Z!b7R#PwPV`l!*WxbJ=340cu#*Y+%F4r{U{i6Ij|4>`Fqt*if&NbK%SRwgcZO($ zJfF~*w?Xj449nI=lISax)(|=DSb!&l@pA(pGP<*w2g-czMwsGI|4K?^Hm4B?TV(yQ zBj`0fS*=|gM37=EVeInAZVUp3kMU05Bq2LX7Q>#OaJm<5Va)U=$Civ@pLpaqTtd;M z#NKDAvIaD@gx3sg!UY_ZAXz3Hf(vYHDi zx98{Fs^f*23$(owl5`pO!tv`7QbV1)nW+7#u?=kdi5)I#{8|sKUSm0B@0yx(qtaiUq;oLxVwhz zI>wZcw^yfv7{otdNm%{J<}An(kd4=&fs4>AmgRWP_2u|A?W*0(`mtHm%M*% zCz|vw(yHm~>v<(mnq4T7_`_2|Ka5w=vDemv?Adzwv&iy|ZkPY}>B(`QX>yJXnFlAUt~S z(`*_{8>{NdYakaOvy~sz^=yB05T&%2(>QyW`g6Q9ulfmIq#`~^uDR@Ni`%DJ%$}`Q z<#sWEA0ZnFm3k<3bA{sa1=7O#r{EY(v+e}Be3p+|wa;x5I26B;CFI_Xq?E334(60y zktJw*%ble*R`lE&NBH*~*OVTw^vLgT>^gRBI!a=unuS4*UBx3jd<%nYQXNvenQgCk ze4n9zt#HqR>d?)ax5fO=mKCN1T`_}{{r0y}Gu-^JcY5il_xJncZQ~shD{mZ5MQIrU z`q-Hpb!52~u86g?HT*roverf*-6ub1Q{5lvG<<&ooz6c*bNq`rw>kcjhKb0T33#VX z1+3yE&ue;4Bp!IS_B|mOC*zqiH!$<;86Cf8-4&PrUu>iiv%g-O3C~5LONaC&X>883 za6e2&3*A4|CX3J;K43?anDw(6U}^Z;zP7j z<3FWr-<+;eaB}1#yabJ>k)TIZF&_N+1{5+jwFHQbN_BQeZ$$?@+0^1&#ExgQ*0NNS zP%5==yI=?B18yB1rGG{$En&sa30lK95|22g2kf?@+n)H2bQH84b?CW?TITh(I%hI* zreE%(Cq_{6RY+Manvtd*EVC!ykCGf4i*WG`DCo6ZmSdL+W%G; zw3N=3O&tlIuYeTeq6T6SJ3C=KJ<{~*iyyyvx#-^f?3JXm(Llj&3l2~c-0KCgY>NNb zA&;lv(xqE&fPDcyQb0zJZM?d1T+|ERi9k1B7_TQk0~AH91X=bG!&OHI9!9kfWSehD z+3DXKen;mybzk9%YJ8=JjL(e_xCn8D#Ri-&vDHBFBLK@Rpc8=1_*U<$g|u}K+1v0) zTob$TnZJEk=!bm%$XKX~GA0Vw=WB4oyO%b89K4s3G)BOxITg1_lS8W0IQ_tMLXs>& z6&Bb-WnMxD;>V=ZvnB~>QNmXk zg+s6<(Z_=eo-XT0@4|lTH$iaJrz2oO;Y8TEo=E>e=zUZ`gcl(3fqobKnmQ$&!7Y2_eL!?1Qn5eJq1n%=R07&+nY-9G5?sYp(0_nR(yObHC4X z-_QNj{9wW_;r#)?oM;| zh4SM^(R<9eB;^8b0b8WwZ}pKM`h3W8D_G^Y+93r>E7t0)nru#*4c_=U_kDKu<>OKW z-VQcrdE72{v>udFPfROpzT#GgfM_mzY;XU9wa!938>)o%`0rle&!3r_Cc#ze;7Z3< zTHZbH{-oE>Gz?4J(%Q-z+Drg@BK90ttd)I}r!X9J{jJ07sQdfwIA%Hc#_)%LL)sVX zq2r#Io6YS5$ZE*O1!=ju9jb_?am^FYV@NWj54Mvgd$X`6x53bys-PYh=C87om#TF4 zUh_v;?kYS`oAy~F&Ida(m(gw;-TYTv8SC7QfPxnkqLJ`O$Avd2BoeiFdq?=M_0eJ0 zKSSl5&xan@ig=9r`x|D~XkIhVkO!j<-t_tl%>ViKU&`~SmK(Oh8@&I6!e?UD?qSc; z>W_T&SM7&+F{O6q0kutQR_L1kXzbDdDsVBc5&TOT=w1BN`+d*ZVYr!Ncz%8>Zk-r)P<_knSp!o9NZxawgwn-WU?Wlm_kGpWm~#O#N9@G&){2ei54a zvuVhyW)#1Ici2wpz4v#8Ia2<#pF}g$1nRJdVC0ts@>{aWx<+%`4JpYdHE#geicb{+ zApIuokUFaO#FZ~E!>WsD-SZ~#dzkWW&xmZAES$mq!<4+|w~1;9FIt8*GTb9?IH%EJ8;>TJlkyAElhN#3EJDKCDSeQKv&h z3yOzTi}{sIpvtug%y9SY5^Gv#*L)^xrcwolbC2ow5xdIJamkf;!K#)Mr!5W6o3N?h z?hs#?r7iCir%fzfTqN4luir0<_>iw*m- z;f?l8M}%U$iA9afG%xjc`Ry)=w|7^YX@ZJ1k#s%4&%HX9QRa$J*JaWeZMT^sZ_wRO z&E@K1aSY+w8L88Oi4z?ouEqI)t2X-hPyiLz?&1EZSdDGfC`4s0Az--J? z@y924$zzZp*MsFgN(Um9pFHr~7%`1WPT$}+)QLz!{k3g#zU`J={Wh=i`B%H&(~)VB zK4Q|#sn$GLnJ~_lBbfF4N{P=)ts5a%kB5ab+2KKi3T`m7ZzIw86iv7Jc5< z3?apq=#(3ggQrlKrQl6PtAHxXk2P}YcJe8B+;(6_Zs~&jA4i=BkyT=H+Sr3VsOAJLh*uH06*qkrcLt=^8TvGT`*!J2_<0@nu`2u5s zfaw~&Oy(IKlA}0N)_RcMn1`b^4$vaTnmO_={a#G5#WGTxMhtS%i7icI*e_au=beE% zb+nW8QaWY*JVlS$fWsBVuBMYlRx2Yun5Tpi5*IpyW64KZ(zDm67(UdRVXr)l zmh*8tU!acB_kN$ScaVC=38eE`NTAEa76tE?Y0+bxTyDeA)BW&GKA|Sprw<3q8#l&z zj|7#{h;x_#@BmMt#xpX^CRn0~5R;lC|`E_h1byVu6=}EnD&=?d_Ws6F?lB38j7t0z=!a zAfi=sL)UkT>5vU&RtqN7wgIc9_LjIzlA)Jl$~a74(J_m;3F4)1s5eVd5bQ z5*mRaQSf`{daG+5*N=JDMia9$ljkOP{Jk^s6z98)sSOB4XL|xeI|g%E#g=~A8bJO;S|BIrMMRrU zy>VFP7R#pfImUK7EDIg%0H6^qLT9LNeC`iWc$bkxy3*l$zPq2|0~C5)w~c^XhizkG z*hh)FOtxnDblpj>Z@zK%TH$fiOx%#smXBN*W{B@Fnr+BR>fhn#9w6!+dyHaNU)kzW z9|*X$FrFW?dyu_h@(`s4lem9=rQ3kl9sVh0#)Tdlu@!PfPG6A**vew4bujzaOa-3P zdz);b2cPZ#$tR`v0SA1FT>b(~5i`Es1NwM6;C2=8|f^!-5?TWzzgvI-& zYfZ|bWbZ1}JRoQ!5Afwjpq7Qa6EL%>v>N!LDA)2^%h9uuzPzIC|JDc7ekspu|PFXAU*Xa}_ogFv5dq|y;Z5D546JBl01qy+poE=4UZ>9B$H^4O zV=hkam{HgpNH`KD(KN~^-pKBC$<&_3zVylT5ovvU5}<^I?s?j74cR+L z3oL_p-MX^kxiRDgH$!END9GOFlh$`3WL3XUErDO2P#<0E8i2@bg4JN^bHo9cM;p(E zuim1i5=?05jdErk8MS$LP&NZ9Jg{c-kg<9RY6Cugvtk2tkN{Xjc9D2=7BFiX3=L=+ z?q~QWW(lJTD4{wr)JVf5W3s9VdLwfl$e^6?ak(2&iI-7(;ZuYHyN|l8uSL>NZG)+l zZ#Cd-bOm{eAD2#cD`hxf8@50b3O%JIfCF^SnKKlMWHsX_MY-DHi~>@9YHfRkc!oh4 zBNG=$N}Emqh7V$Svo*vKPN5Xs4ZMUI6d1LsT6@4)ngRAB8TMkx$Tf!!+pqA%cuUDqsB=1$o7WOK zQ_{mPLsgnC>VK`79eLM%vqBC8DRo37Tk*#skuAzqoD$j=r>5K)F`yx16X?{YEDR4) z+rL$7iS0Upq!w0@z$2JmI6bnw(p@Sd&ZiESl!+zJk1hSSH>d`#5>Ys($8A^5Vbh(;VQ*{R!er2hv-S`v?KQ51R z5Rz#Kym%|KvZf^@qMBf?=!5m)*PewvsBs_maA*Z2?=<{eGo#61j!;Tmk-epL*D$4< zn9=sPljB>Z{jd&FA|z;L&CENZ^2cr3C~qRMD>)vORP7Rxnuv5`h%REfsD7aou5E94 zRo4)Xi+qUT=^{k6o(8KUm5T6soyx(8PmC-JFFmDqo|G!s01><4v|&;yHS?2|>g>6?lckJEMsv;v-`sfdu^ssiJ;m!PXU(yAQ z7MO1-eFa}+IU!#7cmtbb1JnvfJX_lYR9K>(5r**T(0-ed(*Xt}AxpdndxqEs^HsL+ z&7tdz#jUH9p!>28x@e8SCA?86f^7`VSoznw-E-I?DUFKB`NzysxuNY z#lkClPBIpPh|Y;f9h@cfZVUskN&;qlXv|zmryB2A>@=PQzE~NGSP0sNgfUH(zLRYn zW@?|Vc}1*XM#$%X%Q$2Ov3Af!vn_tn>1TGT18AWSSz;j5@j|22{cdv-$9d+!ZPwz$xK<0CN zWW-q{E={D1r&MN+*D1f+Wh9MM+Lo-O3@h#C_56@CP2xaxE-R89u**SCLlaCHtarqT zyMm-2IO`BY#`2bB+Wl2uMzpGTsjV-plETUer(0u;86Egm4f)3E9V9bzae0qSa~UC? zj7sD_qRVVt)&*vn!AcJKi0xb)}s`cpMpC4*X_XqO&kXy=aXaG{L(z^aq>MFeSd38@eN z(&*g~1q?ygq<4(Kt1eebkuTU{ItYt|7_I)m=8|_=>se66)ZCxE2uvmNn(r-_ZpLWG z{ElR`(+FLG;wz|aI%7N1Py^5lZ>h&b= z=-~l0@$DBR*%$CxE2t+O@>DMqA3U=b@9R=#Ruw2xV~1I0mR5*1qp;Vo2zn3cQkNph z7G1hxPbrv$Z9DoqkG{j8=?X~SlnS63rXRKi3_3tRl?Zl1ROgZC)3kYz(w0BSOyY74 zveUxn8`kqAqkx%(C=CnwxNPpvY4z43K(4|3p$^@{vnER;FM+1ig^0@x6G~Dtqd1X` zu|U_p!;HeI_+`Aqj5XygiS9_LiCCbgl$X~{K4o~q0;DQeF~jcKRBy&(_h4vt2?8Tl z@FpR9=srVe>tRa7U>8H7x&5Hy%NtvJo)Muf9GLlV*HFUwcE;l2U^6Ct$1@r_tXkF$ zyEdn`5vKjgWn89;^Je(|tBbJLjO z=1NP((e!xgISTA$&EWjznZ}k{#~gHTsqxR&z!C2v+9>^d$bg0M95MBx=S2d$wQS(p z_9B6NM!nZ)P&Ry=gw|&HtUE8FxB(!Ib;GMLV(SWhANN+kektd3Bz-U z_asjn;SJ_;mAj!IcGNdfbPT%)@l)?QEfJ|)4|!ug&};a;@EflLYSlXzcBKkA>EeSd zcOY84XFjzly@1izU3ExW;+Y&r+Bt@4<`M5lNZ?PhCxvpu2j;Q4y4*%VjW|u}R~3vx z#~B|m=YU19)*I;XBiiMd7?uca3SkcQoyQ%05XOk6_6%i{XFgM%lU01ze^z>r&lRDB z<7bFZAypeTO2A%R&qiH`J_*rBXNmYoJ)!~p3{GWCB3HQK3mAY0V;UnyuY$1@dF5PU zg>BQO$hiXy_8;TD*&i8aksVtxy-=nzLt+0$evW@);o<=2OvRYW^YML0csoY;4FlgPA z0ADnhT%?bTetzc8MEXf+OZX$(h0f~J*m>-v~><3bohW;w5`n*YBQ&2>W~+u ziuQ)P?bFu99N01p!eWJc#=!D zi$UyqgWQa_?q}ubj~I8@D#nMQNW~?u5fmj>pYLhwSq0qrQ0xsjOATCDd$=Qdh`xV> z<84mq#To6gqEQjL_;?Qhf6du}mQo-yBc-fPEIkhFTCIf1AuY6Eg=VRnu>{)=+alN( z>4CW=>MPwMa-h)7+A#Tslk$~y_*X;=Jp_I$odBc-8-j)2jfQVlfUVcP?^2Ifmg>Q(<{9_tuI}Wj0lE%t zd!}F-BeiLLcl9Zv2c1rY<PQA@2BtIDK@RTgtX=w<4JHe#)Zu}IroApjew z7DR^eh>IX7r@EIUi+RzOP(YcR*a!yGE!J{fz>ESShLZv_E)y{dZtr>9U(9IO$d6kx zFxG|q_>x8get5}8gs`ZihzF^$WK7%VrWwivbxwNTD*b!7)FT>e%o5MeGph?9!r67G_P(#SYRmP zY0;UP(9?cvB!oJnVU$*QA3jcZPiM#iEqi-Kt3d^)+lM`m&e)|=6={!=d1eXBkGb08 zQofjiVCp_#SD8uRGB2l^(95jV6&2{61>-*pF~6vC5_G?_)bQ&)tFjFBJ<>8)yAK`@ z&=?o1Nm^(gPhJuqnvMi*2RR9@ffuX|8aonsBjS|*Dn=oqBjEy|G0$&HA@3~&gnhL| z7_Vc$9~!h@grY};OTe_}7x115u4@-2rcwX_f8IhQ*O{jrYR08$T210ixNxcZ?B%3L zlGFArcAvMeFy1@3RB*k1xEbeQelL0YS%B-GQxRMzV?<)3648Q8E%}zS`2xFkV(aZm zjq>lX3o^B8&@IjlfcS8bMfC<-R+iw2Pm8BFS^u(A*ewe2iL6KUM1D^HHp6diW?zl&2$s{p3of8-+)vrkj>e(6?htqR)Cu;}ZETmXk}?Dxf(zV-@p zN3!HJE5H7im-^6DJ*L0Ny}<;b7tO0y^*8#(VqY=gQz%VmhKh!5L#=Q#M61sY%Ie*aj9b0eMBep-+YOk<{ z!Z66yL=b!K^a%ssjI*QfQei1d`+8j|t^D$K^ID%IDkPq8#y>4{fSMG3CoBeSGmU%l zCxJG-OB6I)W)=5Z`c-nhSf6?q)2HyMahKAVTGb~B9h5hBSgY|9_1Xu2b8`T<$y(Pl z34_tg?`Ep=`;_=4%Go^PRgHo>ym7_FI-3N>iaGm!RyK0S7!dnHM_ z&2Idu(iPTp2?~T_)4;=}R*aW=9m=6rMd!k%uW)$cM@@NAlEW(R_Y2F^D4^c*j{a1BVCdojPa zDsYpqSlxi%REfRp|%lH|2HGakW21XGMQlM%`=5Q(7iGf~O%WM;6L#{UJEu?>p zMX8TH<-yutcDDZz+X~z#qI!thclo*1ZkZoGEG!&)#*)8nRVfp?)1?Beb)EQKg|y3F zL@KfXrH2*V*S`8#elB}71%)eE>2usdtt;N0K*{KRiTUfQsKw`}7mnz;VOiqBT9WrrrX?&bWD=_fY8m>@tNhi={Ge zbvHNd%szr5>=D&>L}ox9q&LzIAtcR6*-Ia+diP18#R&|Q| z?fE3Y8l;!(?@ctO-B3|hRi4o1`O5oS&3yedccW#Qogl>R)k?JM=II9!{kI4uaRt5Z z|FnkEB&D}nJ$Q|^G~Rr^Aw_OVyqa_R>YP=x8`Gk2Vld^IYRN(Svj^x)#@cavu;{_C z)1FVBiQ;xv@fKw>Bc&MeVG_cHGG{V9SV4)xlMntuOGQevlpX<%1&`v9)cYGEf}%|6 z>>*uoMk_GM`sV3nDe)wQfJLg?JR?b1K)9%h5Hf}hoLZeTEEhUp#P6B)wLD=jx$%Hn z4(=+e{RNNVBQ7CzE(YnjKEnAUXgNJ^PPNdTjuA&sSnT9atMjHh0!c;u`Vy~B8L^YY zjN)yXyjrKiGfSl?6HZU|B_Y7UaIzBxzjao>$O|@DFi%0W>w12B_{S#mv)S(~JIuAo?P>1bFT|<#X@xi zZ&dsXsg*AGo4WjOl-=OO)@OAt1%GKUR=ltrwNQ= zow;zK*kvDAN$7&28^T9U(j%i?xaTlBonkehosc3nqgFPNR8MuIjo(TZ36&~~YU)<2 zcHE=w0V}f|Z5E9c7rQrkMDRZwchY;et;70;dATgveqnmw&T|1XLlxnR*9@v(gR7(t zu$5b;7h6`OUP=#SdN1oWY;Yfw(GnE^Mb(xKS(J-?`(}1*?97GlwFAI727wA6$@#|n z+*mTIzq;g|xw2au)FaLz8y7@AiCNq#QGuP@7W$#iGz9{5>HJK38e=>6r>vW#@r!(A zv#P!?=D0X-NfeqU-N zZS(s^crF7gdYr#?thzv9pKPTy8QPi=Yw2&6FVqnQu*b>g#b{v%@(Q9dmdRz3ZoFQt^bWsRY!L;`!8&RfCWhY?&wTyN!&_fV~39*67*wX}>no&31z1 znJn>VY05J}y8Sm;-*$r4pU;~gO`MKq<^L4oYiG*(@(H{paL3PGbBy%B^@lg*+no+S z!!X&O$feAz*~$6I1|4Ljx)DjNkU>?n@?>DQ*G60pSC*?iR@=D$;9P`I$DYQ|znPUg zChuAxQTw_9O}k{VLexA$*sqE62_W&K1;IeQ9mAf0{HVNmX6JW-$!n&!(B;+(3c_}G zB95>eYEmtEcG6{_x)jiDc32?lSm2=dud(2S29mQB+9WC95C;{u-UT#mC+S#}mM%v3*(JNpq0RM9YB;DP0 z-4%7d^Qb6gJGgTXse6xe5iVG7d!XCxC&PZ`FP3Iw^MD1qr#&beX?98254SUV-qoP2(-GDTv7Lf&evR!`{2br z>A*|p^uJKjiwhA@Y~x^D)e+Q%nPf( zXre!!`qM{q{l+iIX-(gSoZ%1;#c|~&rsCyxx#3D$?q~GK&_38NoDsMCUIXI%7skYS zm@ws@?r-Q%f&Eg`QGHRSUme0z1G>>~suA67+1Nlnsj;+EV&lx1rQa~-{f1UT9VtP7 zHlTiwiU(&t;W(Oq+UQIh0Smph{cF}?7Van7Djo9?>9BHo#+UVaS?GvSI+rUuj+bDn zox$>wg@yNzX@3!lECA`@e!+L|+8&)^Atdc};OB1VJMv6%!GWHnAF0rPm#N7(X+NLk znqn0Dak}loYW9Mkc^T()%klq-GO>csjae0{iseuJq{tFl_re$PW{97QN_gc8<@Xpa z)tlWf-}&GHWxy9<`C<~hrJ>(A+3z(Y|FBS+tl|iluq1vwDEkw(iP*gtjX8(4Akv^*P9Y>Vfq22FYOpjd44@sc&_>^ z6QDU>TUO&~ZVFF0QUM=Dso%X^PU-faDpk@wGUix$RTcyMSqXe$=WTrNu`S*8&r?sb z`y>lnCIWIv-W95ok;%5*y;CVJT+8u9n>8+*jev_v6fy`w-IwD$@kloLTY+)g(Zx)I zvbrH)oERG;ot1y%y`auD5#MgRPIYY>UvK-X>9j&~Kd(Uy-sjEbu8FK|C(6YfQ}kvz z$?+&x1iR3Gw*9VM+OEK)3CC8x{$9+KDssi$&#?~P6~Bz-n+UHl$iqztfmG@0`WL}5 z(`0o|vZlM)u@+6m9Xr6M3aJN|qQ4%l(Jtnf+TojPpbA8D5bbw?#l#I$GC9@N8Bw&% z?LJ956ZKZS{~uTpecnjw-|n-bG{FfJE8?lOKsuRQ;Ex+>XHHJa)c=Y^JUThHJ`28S z$D`8oRdTsr9VvIMFbGv} z2&Xo;Fx!s2Lt=zt%El6|KUco@*jZ(+EYY<-Kh`U}3ayf^y&alYQ@7;9iWI z*S}tT@#h?Y#_U~O37SP~KYt|5HK@iCKUt@|f3mfj7qgP9+n1=Bq4}pI4w1wZ1iz+h z6)GbapfZ6>P~q7kYf~oY>eFSVS!>ke-PDPIkTdGP!$%g=j_)xLVb}Qki~p7j|4Smw z@nlA`jUP6p{GZ7FuWc6zzQgRAXx@KhJ6D)FB#Me3ARs58&{*U~Oom$Xx!)99=i`_; zQnLL=7Jts^80Zl1kf4F7y0q@oo1qR?0cX%{8Q}pGK-Uiz3O2(jK)2!3?b;kFcH1T4 z$Y`?g@14u*Q@f7)nq{#|12#=q?@01(#_(Hipf1W{$_ot1CQh$OBWbQO70!u1`F#Xq znHWEx8brxN;`^{B`}*tGzXF$cVrI5G%JXPg`P^k(I=W!69&5TDrbM!IYVyftrz&p?aqpgZpsaA(NHU&s;L8SkjSt8R_*N-yo8E_}KDkm6w7lm>K7y{HEz4XFI z^W8F0hCwPn?-V$KJn+Zx`}teAgzX=^uj=4rYd>1?gpDQ(?_qkA?WQ<6%oOM9ykj}p zgCf6OSDpuf@+`z`+v`_`Q{dIoO6-b~`Ls)OyyT!fukx!5%27^~O@o z6{?p}y4K3yl4Z8e1~NmyoQc2e{aBW6%1UE2w>)*Gysqs+so5v%Q(R zM(fQtobB@|+`ceMPdi_^ZBe`uV4p{mXt6NfIOTt-tH_Ask%#n=H?wb}EAI{PhGZo; zy^V3pG_VvabTK&l3u0*SeD)3SRH+xNy`PLwKI6Q~p3|oM`xB_LF1LT;#PEtD5ACgj z($&3FHsY2Ru9t;DoN5lx><9ccCbl7B^S%iaSil3YWcaJM$PPDva=Xcz|H@-bw{A^F{%$W6nS6Ir zG3xS-ERaxC-lG$ksRCP7OFq_e&XRAX3e&-?hDRl=aILb3zv??x>-SujbYhC=7DZHx z?)iTYy%MA$J^TWV9F#(5e`~C4uF~c>dVAeO`raLS@y-YNEeqz@u*#=8lCJN4Ugl}t z27dGzsWME571CdUo5TgG6Q20gM-0ue5MtD_J&nJ%E_cvl*HlgaFbG`V;BIs*^S{ZU zuB~-E+-!=EjuL+@U9XDNn?7Nbx)n04&1;l;1X3t8yhPJ%HZo?z@=LR`#xjAYtZIzz zL+-2=wXLpJ7CKQ)&IPNf9p0J0L=NCdj=kxNPWfcLB5zZJr{zQW-{2pnw(1OiK)xtJ zV}DpT_ZZ4Q!ed9I{+;QF{%KL4?mn3p)$^&%8PVWRBNB_&&y=GNR{>|=S2nF!_v~HU;QuYkFh`a=+Vym$!>E~D zXYFcVl(Hh8w3f{ve-+){|H-A^nng+S%FbMf<`g8)&;$Wo;ahvR5%A1sa@i_VTX zd#U{7(j3aq-{HBVQz_HSbq9Ge;|{Lq!10o-kDwuE5zOMV0l6&x!)yh`I~SR(UX_Af zeUG_yb9c4sr}qwEdo)rr$rj$QmCX3VVsz-%WMb51iFYm)A2nj!!eEP{kI(3A)lELw zGhX$Y(z9{x)AxX*)f<)T%hew<+$$6eiy+P%O3=pcsKn&eb%*e%&(k^T!QEksMxr;0 z(Y>cn6`WA@A-J%Q_SDrXF~?GaoCD-B)_9x_m$n`fJo*(~+s`K}EpCpSzd$1TyXf0_JWW|5r*>(!~^ZyO@)2{ZF^a_1nzT7Amp69AeF_M>+`6wz`4?yEvlM)wU8(|6WXc5b3h( z0C%ldD`h@i8f80rwd>NB(PJky$8K#b(%M+ z!oMd%%G`~{WuE>m_0Z!omvwb|W}aPdK{j&mQ{Xo{=#QDC?P)Z{>>QW>XW0s;A2e9*fHX0A}W3E>{|10-UV5i zkxYWgOlo@u;;i-A`KxEZRd`5jn$O-_pyy|v;_FU6Xp}v_|IJOlXYFzNS0^;OmpdFe z0$UCHbfU$ER--Jfgj~FCH%T8>pwu9#sRZxnoWuat>GUoCbNjk702gQ$Ozd)r#?Y7RLLUYMmLJ9fezgSbhTQ3BOK-J+{*e}`1t%%n3Uvdomj?s z#mVg2g}K?R&V|bDA)z?O#`V$xyHJOGP~~Mm61$YfiKx$lMRK*5G+PjjX&z|?WmcJ; zq9c4DgRBhGsKPWW{hf0t-5D(#^@4+Z1~Iy?m>vFxJ%SA;A7@ zu82h3P>31u21}gawd1wF&qAjyCC)5d14=Nr_Fra#I!giq}RE?@GYP0}8d zIZ^As0Yp+{@&UDt_$6!pJrSMp1eS?bL6y!QKM##R1v&Whz$BLP&Y8AoEzGS!q7sXN z$(EiO+9&%Y0|X_LZe6odCmy2Rt`TiR`3~3!fIyTp7Yw`70v{WEvln1~5rTn+2G!I_ z4>5T0hWvO9EtqKC{LS#TC%G6mq&l?o20nFv_-L6cyTfho-%hWQOq+HwX3L-amAGJ! zw|naeD_R>b6=j}LG$hd6l~2yy>NY6z=NCQ})q1q(2s_Z~TJNX*ek-)?M#vJ*@ZQ+e^N^*2~EUSVji2tp##DKn~PK(V3?CF5-+W zAF!G=1w<>{g`sb1CCP8D=&~3O0jf~fX*VO#sIq4lpVMB=?~IMoRgQ1_m)&l`bQH4> z3!;DT#-Z+K0Urh(aM<@PjnluJ8nT~J*ReNukZ+2ccT@EB4az!ELPsM8(z!Nc>BKLq zfV4hZJSB>wCZsc!vvb4C{Z2n$T( X%**SSE}!Dt!~8KZykbzI?;7=gKllx? diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/img/module_compare.png b/debug/accuracy_tools/msprobe/pytorch/doc/img/module_compare.png deleted file mode 100644 index 2e1ea564eb191807034afd8aceac92b29b62a086..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 56655 zcmeFZWmHvf*FFlkQ9|ibx z&GWqH_na}_6XT3=&iU|uaHxB)z1Fy5FKpJ^Z*Zc^Z z0gdEYpm{4K5+foz_w8BboC^7`-G&i<@AKX5ifxZ!eZ8&|E#2L#ojL7eL2X!D`0^cO zG%+L;96uxo6%zPoC!W6%D@?KOzdpGAjvyeA?%$W(eo}@j27NB6_~kz@3J!HQHt|R(64_4!%!X2* zNfG?>Qyl2K&JB^1rhYs~FfDS|{eRw4D-jkO{`12qu!Q)CaK8Ef!(n2HXjFV91j=6j{Ls$~fHF^So=nl+tBP(O;p=gRLj zia~>&w&q7A^>{|rsNSGwZ@y2G3o+#)>3`QW6uA0bBuPIX{yp>p_i*hb8<>0OGG%e z^OtH{$Kc7q+(07BtaF?^{!LflJy~+wlizc{T*o~eD+Q0IOnlpv+x9!sB&hA2^8Tk^LaZtFKJ#%C#yWKpda`66oCLOHA`4 zjObWTbfTjV>@}bFW1GnbNM5tA%t!Lh7paI|9erfM^&s@wovAA`eBO@*2}Ui_Lid+} zI2YEx{DMJvQ@8QEWiN{1k4KQ?_87~9eUlI(4r8+NsC=WP(WzfwTbSvVT`mujI<)%8 zoRlWTe>v)JCjT8==#8e#(uZ2IdmPiY2bei>>2f)zt@K8k4H7gE!(+q+xS7uPP6yoco(NdIab)*HDS~w2ip4mLv7rDL{(YpQs@m$a9SGqF1QTb3U< zKLn9Y&o$18?I;xPeu072q=L!5FKtiDB^}eW%d$nx3<)-9ihKrN#&KxL0g}VGsrWbA zr8?50na>2c*HTiM zKL3@sX6Ga?@H_r=yg)H&v+A9PU~*HK|6+eafFT5A7TIbj`wm}z(y)5BM4(1Og3kgq z+ZbuL1%e-t>Ufql6OkRrxwzLhM-;CoTUV3!|D8E$xNRaw#=?WzRPI=0}3Df%Sq-a#}v;<7g2Pnnpbm({6!l=j{#+yRkuF2<8 zTZL=nojj)#KKyAoBzK_|o!^I*x|^DtKZf7DEW)GzWpV1`oKNzePFZP?~?6d3z43P(;Hn1zBT z{huSS0V9}hbRhgsvjTuwsB~e7yZx7G(_jSuN0IGp z{n;K41)pLzNNct`?_g{7QFxXKTj zP*VH#y*%(f1cQL$;IMd#W~K{^OAk=h+FP>XPRq-p$HSj7R_e9RnPXvC-JMnj>~Y5^%RTA zllGIguuA94_xJt$Tu?aLoIeRH`gNBzXu95s(0eUQ)%rL5Q4J2+(wdGMMKqSe`iXGN*+V};1-#TLv-ppkhWjg6&5w)}1mPmGiZnFwK^>Uc!%_R~S(Z{NJ{ z%@yI@z{MgcG{Zp|66{sNEJA32Rz_3%40u7E%10d^OzM<(%72Qyy@ ztH9??uCRWRnSO~*^J{PvNxgg1N^N?$zJC|x{RbJUh)YRm!Ut32XJo`=+JxjUjvgY@ zS6Eccd=rHPTYU9vD#!igepqXcULZ+_BLjXf@ohKPTLq6yn*e^Tv<3EV0lg{L4Gx)A zo)|v|`j1xkR!G2D5FQ#W_$-J)J3A7FPo|9zzu2Fl>Ym*!{Kogt{X(bV*H#;RNzB1P zT${7%l;Xn!Zj1-N_O&^lEiVn?#;e8!@Jk;*z}j%6M%dsePxRPIE<0thC~OlgZ-i>v z=+=Y^8S)lK_}IRF5qn(Bjc|ne7M*KP6L&x!MlNlHNOx@h=LlDb%{pa3;JZ?8xX%i@ zmJHb;pZi52vg}_a=*f+jMhQ-p`Lyz(?4d zsPa-aU9si!QVZM$C|xmUAYl|MKNMbTGEMEvY_T4dcs|H*b20lBXz^l|GN3!y@Mx|V zGvdtuF z1Dase6sobNuKu@X`7=vvrl3>tJM`S3~ZjReD&lgQ15)E3!5sxoy+o7$m>B+ zaToaiYLILi?%q@1zkVxX{0Q2L&B*Gt|3rAHx{I z;F#x0^^YPECjMdL=&mL1!ZrW7-%No5%n2A{4N&= zRj>Sd>HTp(!H*3T#gC&u>)a3D_2T&&FoLyHKuq@LyHF*A%B9;g?{ku8t~;YVV#KVC zCT#jgS)mIls9B!BU7lHuSHeEMxL0H~5v7|UusilK6V!ku(=|z@b#F{GFp0d3KrNCL@0rs|| z51R{z>)y&Pd-Hl$zQpX82FFb!$8R))uPaouyx+KfALO*{`MH)?dOHN$Uk)e7*^ldPl^m z@f`IRwRiK!qHf4AyCn%o7csz45mUoXKjL~X>Rq*W}#y2k+Bn3V; z_YjatSL3bJqQ(2BwPVP>u=9zNdSiAEm#i4iPeL3YF$V!E<=cYxhwH^6!a_StU{7*n zERM2}q;9!&2x7<-36G(17rqg8jE9~hSs`G5SoI8Y%@1Q?07Z!hZ*zxgqqnRW- z3)xFmqArKCwZfva!@W0**018qg1-N%?0ASC%b59=IaTS6e|&<|%w1gh$N?8_6J`I2 zZEH0hoqbp~w@3&{)6sY(IKVv%nXfxu6l z-r*e3eTh1o{&ug$ego}~o`pB{3_IOya7F8ZvCEGa-Y}T1C{8(D>*@0M83q3Y7P^5jAHmEyd93o=iTGoREE9I`yvhc0+k`vid#(P{}E$GUMz%ID;yEq%%=xxUa zG7Bh89ufyhSHJw@kb z9D6$bq3!(X*yL=+QGLp(6dwg%3Q3mx`UdQ0lN)0N)bdGg13L(U%=!8<-|}X5q-yL^g}K=<;=@8R~E8yoxDRz`6n_0MQ>2p>@UmC+OEZEC;MJB+*G={NK<^x2==BXMa>Q;h;l zo*_z;(}v1%q3D&o_cDH#8pw*G6+fcdDw3Q{8A6k1{gtZ5Y)TU{-`dR_$+7!usXrm#)QY3d8BG>Nu^B&TxrJ@sU>ONCkqPubTs+9=y&QdV1Omr{d&Css3 z{Om}aw=^{WtAFkQr#Yc?bDb+YGv)=QUcDw70rtl6AS-J|?K5e?!Hm zZ(+WH(aS1mcvu93M~{A5iOD5a5Iv$Ej7&H-a8t{nnx>|FFg| zRLWBj?kg;}m8jSJI3Pb-H8r-=Zao@Tv|bxX9HYD5%!}4(*{wIQbJ@@y+ih5=_^6si zq1^yok|;2X*S%u@jxTfFRP{M$L4d+{;&j%2PfOVIH@R&NURRC@+4H>&3W4ZHZF@9B z7{N*rYk!)2JU)nU&ktY(6OkTE1=V+7id>T{oTmLConurLIdfz#>tS%BXT0W{&o4{~ zH7PH529|Eq0!zbs`|1&4)vlMU)R6|B8GY?BEzxUsE(R_l3dnQB<&cFY2Xq-~kK)@+ zOB9$qwrCd?WLH|nZPhIF(JMDN+2G;8v?2J&9Uy74V568=KX$p+u30#j$6Fe1JEX(8 zoZIu0{li+Q%IoU!Z*f?a4L)*06ygxswPuPztS9Dcm?T}X9IDn%5n^Zol7G9eR~bds zJ8rJ_V958#vahfAZbSt5+I_A(T+Y^#WO2{dc{?6@pVVYsDGC4mVP@UwhrGR0Gok}9 zxriB;DWklnQ$}tX%^R7Xxo-4_^V{h5X z-L7Lk{^@-(x2@Z8uC29k(sAQmx7U7A!tbvJI}wb zTBx9%CAlglW|I6esXQjFPaBJ>uWy`?k6tDa*s6s-GJ<4bj>@cQZ7l;u~LoGqy%j z6ULC)E}UitfE;f!rY&m3%I0(3`H^Ejl3A`(_8d23E}7Rpb9=tc(^*GCD!}WN28V>? zu-ayH`bpz*7z@o4>MT@Dq9W;VqT)$;!jReB(xH+$HjE~bZLLRZa?{e_h&X7Lap;Y-zqE(XK!W_ILtu} zzBC#RP|6U{CinW&wf(`sPJPPIDbi!F?PRj5kD}CkIQ?OH7+q*OUDy8fh>wo&@ucQ< zPB2AD9gA&P(bjB(eeHn;Ocf2Jv_;l`b%r<~!7RA1zd6$5p+zah@?HMv7EQoh(=`&o z8HGqB|0&pN?KsTXD{>mEy{q>i~paGef|o1U+ibZ9!Q7 z>~xv9gX9JGFJgaA-|NdwZ%QlF;3(va7o{F1A4R*>wNXyj?f}Y!Mg8T!9|_XQ&pKz} zIAoNz#ZN(Coom)sPI*c%uo6qhcYd^m9~ymqbuo1yS?ROc_j_O)5|DSci!X6q^?6JP zn`sFJ+zeAaI-lRj0!t-criT#d~8NLnpSM3Q9^E z(!9HJC|v~Ye!dtZ?Y0pY+bTjT>AZ-)v_VUoI=^IKN8&^#ocS$?$ z$!&i8=&VK(S`Dm*~br-iy{$W{`0Kd_n6&J}|>`TmLRk2XmhGS{<@mBRL2UCD`=-ZgPN8%kwR?!1u`V|;LqHli@>4I5kv ztVD24x^-L_0i?mR*ucI``|42i#>v^zFq7%n&G*D@Ob^HXWnu^>{~`4a1;67fXA!BM z&}-?A%*B|eCRf|KuzQqVI7TM$g{0f_;qjq*qvU%pjiC(BhE&m;H``HyJd^VoRW{Sj zn^h(IeZ|&Q*-lEBfZLhZB;~o(Q_lQp%=txH(h$1Ye00H=KZJbeude1tFRdhIs&9XX zJmTj~D=61yaeY^Er8}Z~yc&WN#>kc>$}<I)BmRR$%jyoFtE^eRmlZGR+Bzh|k%h;mKv*C9 z4vR|st1${1wC+U+|i(AIC zb#+$De;xE2x;t0=6%Xf`7{;Z&D|K_WQ&UOJC*YWt-}>Q6>9O{ihvmZ>4TcQ$Z*UAk zKl+~TTPb)Qit)Y`HJI#?=&!Px+@$L^K$OI$IRFtjf!LXk`8yRl7 zeQN`>uF}psO&gipo(mVoM?IyK*MEX{A$=En9hE+3gC-@uS5v;_p#)$T>rLVc?qEjc z!;oyNlqRL2hTg|_>R=DtaWgL+?XPLFq@m7x^ZeEfRzbAvI@@hAqjYJ1gM8di-`EvD z61_%q*(zv|njMmg3E{&Baaq8!Lcwy3ep%&oi%q+{47#aZ-KPMZ^q%b+wF#E=ys z6Ewy&)9HISQ_{7bj^6cx#d`7=1b(vPW35}h^TA8>QP2kNpG>oFe^@tK_hxE_^y`Z9 zKv+;Kzs|e&mzC49cS+B(S_kvg+lIFHmo7LDM%ougtG?sAp4qp#D5#d{mc7+TVZrub z8C(7m&Ct$fIE&K~g(_HJSn_sL~pS?d4x`NqyowU%x z?qFX&!6VZrNuC{5s-lfx@3vS1mBd4zm<-fw8;#&Fl0eXjqUq!uByb?adA* z)>(mxNIU~Md4}z%YVo*gS=?8Lkxb9hJJ>twvkY{l*k*lLkriCJ;;S8Xn?F9i&~%o` z1DUj;S^Whlze}&&gxkUW;X2#T+u~lQ{P*n5&Ma@*hH;nf30DVFdq4DA#G; z)h^A9tj|Rs97R@ul9 z8FilD(kozlD04kcOqSY^d6YIAwJLb_%!FaER9BQTVH=5GMTm6kwXC|RkGB|IQ@n7A9*@~{ zs2^_^q03wwFG0V>L{*W1U-tKXfJ)OC$qOz8NIrlZU)_-Yqp-Ylcgwtp!}S>$YR`7&bD4>8fP z_;%vflU1Ae-Q)#f!;sfYZE*P8z8*5ZoEp$9`Js{6CW(u?}O3m+0RgS$LGlvr5qZiS%U_RP#3Z zQndB$Fy??|dl=6yAu`QgUknUYZ)-E8!wR*p{le#{e?gXFx=0B*p)dR!Y8R? z8=jR2p!{y&e$Xvo8bpU5=^O*>4@7DbLb}B`kE450JO4B$tN1B^Qa{c!#iGT@KAEpA~LcwO6i@@2z3J zPoKV0#3iGarCgY}HEsNhpGGDgxY^q`xQKdLkUlly8$ALK+XLN>I#3;@CiL2 zGbffq%o7u547aur92nl8AI-H#z|6t+?hikyb@wi@ z3WB#F$b8P7><_rXJKXH(5x0=&^QS@U6#x?D{%|^bONESim_w!a!K{DbDwJNh1!&q| z-}ze)&p;;Ps0py*5JNzF+SKOssB6inpR*BqI|WNi`CT;dsR}y^tiQbV$K|)-(J~q6 zJ0gcm+YH1r$2^1@KhwfjWHhI#wfrqmF25yR!v^@=+IQif#qn;DP^DsMI(*wo;;Po% z5+r(ufxp7&08k#sV|yll3zTy)eiESf=kPrnv>yDx22ff;>zk_+Rd>QKRHF_kmF?#x zkAUWAdn_-w`Qmqrpm9#kGb+CH2M^vqgk0i~Md;Dl>OpLx0km07qu3IJ!v{bo7{01d z4&agsRQQ(ONzCNqqM`ZE+IJW!XNedA!HSlV=WZS0`v5o{#j=_TxREEN63iPZEO0g} z*aF=jut2A~_P(V)r`?n#prF>x@V!3U9+kw7^l@0fgKPkKCB2L7dm4y9Vy-uQuVsx~ zr(bVRYO2=3((fFHE&iB57UV=mIYw(OO*oK*a+=3}k;Z4T{w4t>AlI8=N$;djGRy^V zQ~T%H0&z4lr*BWTWkt_%so>;rK0{}6;v+&-Ck9g|@Pkz{))J@q&V#iOx8wQSBeBw$BPvjw0Giu!-5ffEA)Vv z_J53H`puCy3;;l8fv<6xGLSOdwx~Oi5d65DJh(f35ndQIpMsxCJkFmWq>KiR*v~46 zZ{feYheP(e_wFE-ngFuE-ngV6GJ-6Ri9<5eQBb2N0CXHu1_SjXCbEFN2~AK2;@i&T zH9IUSzMqi~K8R;Kqd3nO1S8!+Fy~bC$h~C=*$PbW7nzU+lqEWe!ME*F7=mzZg_z%y z!8tN;l97_K28wM#Hd__3k?-D;@(L&sdd%X9YWKU;QJhoX`Eh`6y+a9HB@kKwvkxu& zev43Ep67S{$I(aKlPFN`--@vkAS5*!0grP?0`ef6+~~+AXrt~Ic+5NR5X5{htD6&0 zJlffOx{p$TpMA(K3?YFVf`q~hc;Etvw!{;NEc&Pw9!fh-_TgW8jUVWUZzJ@6uOk7x z;fLu%hytV7hLHyv=dCC~1JCrE0(!~2muOLV*%NdSFKV%kUB&!Q_y~~$NzKn8P-M4M zbv;E(Q1KgXIl?i$RS4wMfn|fIh5iMHj9#oz9_>`ZZ+JFI?gb-4S^zldv^)QoCJ-9i z2}oN6E(2$6Q{B%O;LzT#{lVVNY4i zFG`>!a#7A&3DRd4Eo?r7_4xg*dEB&}$*_nRg9Tc?pJDgDi)Cw}5Sw^aHQ!uckj_Al z#$-!bDjIXO1g*$!#T;M-sDc%68A|v;qEo;h6u4^e-3#bC!WO*ld8JXCzC zT)7aS`un4LbEInhHI6=1B+GGtdHd$7r&9r-art!46?v}tcj97v@pFZ2nYvp!-HgQ$kMhpK{g9`~j zVVLPu{}Kl=vG2g{9=tjQUX8{jlE{XljEG#6odaC4c;4q?w`g}bL$DMM5o~7%Vm01e+hv{Bfv)Gl34SJUSvuV zEtl-nL*e}i4QxK&E~UL_hoez&nU>3Uo%?}sZ~1?M>mSH{A;GizUCmf{jl1=;0?#EI zlLiNG0H-v7))_=>T(nx?>s5`*x0N4XZPX9+KVoWNmsJO+Jz)`SIZfp01(c`Heh$Mu z^i7D{D7O+uNGHH=Q4WZ7sgeB4;Frd&Gz+2sAOUo&yG%%PI&C)o&-eOn!+Mx^ZKtd0 zOX_5!DTu5qeG<>76i74Nj<=SmA)nzZuqv6$+$ACl4n&Mq{eaf3K%4`(mYjst2%kk? zmRg!RV4{3L8?|_jqF7ga%Li3mD+_0i+UPJqcw>w9KW)hCRyn&tVV ziw3Px$xozvZBuK32Y#y}pKHi&4hZO$pZK!Ekj|1Y@PmL?Z|KFc#2+WfBN*|42v%DI zIB8sEC-mU0s^8_I$h$;0)2;wi>*H;h^?a*)iRa1h#FHtPds|$F(Q0+OH7S5Fw%$Ry`y03yzTlM2&Q+p<&p%i$a}(CyB=5zA+#Oj(S`aL4s? zX-uw>!Qls|;g>N^5Q$;Y6xP9fo?olnm8Ju|sUsRO(%sVN-Pm8W7Cs^I4kI+s!?K345JF=@;Z%p88KMOHYl- zK{ub$m~V&PiO=X+`pTH41#nzlAgr6HH0ek+Eo%5o?P~&*$|am3XV0rXjv!ha1BE6$(H|}NO%rOg$wCw(sZAb}c-a$zyZtfE&f21ch!!Mel28(7iQin~t6Z;l;PoIHR!ddh-#B=tsNfp8 zE=+~vczyPmZ&mq={A|vv052d#DgjKO3=1AY^c)<5Gi^L+86*jbQ2MiCkhjd7ZXP6Q zg1-<2+jJdeVWD}xJlp@M9NO>tv`I$i#UpQ-XbSRj6eGv4^y*bW(Q9brdvy$u+`rWo z(4MBZ&|eljPgX@}xwzTCf$ao`N{AR9JI2Cymb7oxS2h&twX7zXEJL>B`}?5`8y+n2 z^SOBq*jw_Sq?E8&e}%TEmy>XrN2Ch5I}#{4IdW_ee6yAMSQJ}>`CU zt=DZ}v=IKrrgonGe*SuL;9=CM4sAvqfY8|)r(g!+F?(@3Cad-@FWx^O#j_`JS`(*a zP;L2~qxUQ9b)8gJ5H4<#G++cnZHD6cLK@=}vmaC1VHb};OB;G0QQstXn*E1Gd4rPW zDLQmUw#ik&w(*>iVjx8>8ghS#i?v@v<2Z*(Y?xiqyj{X{Z#%Tc{O`rNHV~23FVOslS_Jd9KD1Q@HlBSft?~ z6KI&{4>ngV#-paX*S(f>e2SmDYS;@h3|whr9q0!R6N>U-A&|7jl7rBbrz`ph9gDft zHrC{KK~z4Y@)kF~cs}-$Vv`5{K3!MNmq_zC9V&Bo!-}T!m%ZvzE1(G>Pn+YAS)1&U z{cP;Jaf|#-5oQe(#DTy>oz1NWKM+(Mj7Q@tArfCZPF~~3Z;tU`0Q`!c5whBG1>~8-cO%kp`y7j6%tZ(dbuIG@H@8tysEMK++i~(n4uzO z((|!)y85MQ{g0+sU*jy-qH#Vi>4Q>R+_-#J#*a%In(b730CWPdc?OE{Ky1X}x;Z@Z2q zTKgHLJ4onL(Xp! z!SgEo!{jVzWzawK*jtjrBNQNCS+JshQIW~*5%wX3cSw1gAZx8v_HygFJk7+JH4d#5m-}$FAj8o#8;Sx*-qF#9Ju){X?7T?9b5)SsG2!CR|AT?VWOKfd^Gx- zxMBHCC&K+*wbay|jij1K9szT>UAWQj=I!Wi7D^UJ<85&jY2F`kT<&-Oe(KNLV4Lz% zqW;xa+4|R3OE1I7+h79(WaCjvJ8Szb=LH__Xjq{`fC(j@cKXwGuaw=WS<3b6!%%?# zp1%$_XpcEnL_eNR&1W-O*CVvr0|%rQJUe>C2Q{7a`5?J+jFraQadF9ttB|#1;ap;0 z2R|*eM12GPj)HQY`+2&mG*$lPgh2{8nR<>Vf@&8z7qr;owNso2ANzSNs}@_k1W69x zEi-6f_wXv^$K{z-!v92PuqM#+Nx5lk+9gd{v&*<951g}@wn2CLz z+-W_3l|$*2H|g0gb{kA;6XiRsTBjdk**2`dz!>l-TQR7P+o@#a`$l{}HMsY9cJbLs ze&6}W&{Eac65Cu}VTw%ka805tlO91)7QivYVwYVpLC_{Lea)0Q)glr%3{C$0Md%K$ zCM$l^8(|p-de2Fg(+XE3SWHNHU7A$fESh{R30Wm zD=FJa--HW4{-b_=mXc}x%8vpr^RmkG)E8FWed;i zr{SYGK4i4S2I3X*z%OT7U=&)(Pl+XvHHehptu{&ea^c0gMv@j7f_ z3q8gc(X4^<>EffssYLce1m=|Y(7XL46@J#g;UqCZLq4d)`NoanKY)`xQl!Mx=G+5E z3|p1ZBUBUt-Vy)(?m&>D-z6sR-?v!)tdLEvYPFTr*4xz1SUs155gx?Sok{Z2X~eGi z39%yl{SLuZ>DcN84{G5*&DNt}Y(MK&}uwNrWVboJv#o6!h zyOfct-kHa@e0Jk=(y~wt#~a!X23Qp4>pDR<+@GY*Xx~dSdT+m0D{nZyu_jmQEmNYv z4DnAu=rd_SWxWnk8Q$nDsjwRT4#P6gcQlrk-{Q7sY`79to4{}b`ZkJIgRBlhrWzJ} z3~U$LjUJG2id~&gb-4CmegoxdHO?vcPsQr>@0q&N?Vn9GAn__YAJA>bn_D;*u{w`n zt%oW*fjYxAEAIWGWoBT)pJY(X?{`hu+kR)8@>F!#|3J9#vhC9I7^Vv|a+@P;vpQc( zGSLxq`>h2iHshgANi2S;IcWETFQiK0AaA#W4D+qrKmNuwGPykqkP;Y$=T45t$?tLi zs24^F{%&+-p^VULT2eK_HT&LIgdkU36f{O<;TbihJHd;Z->n%jfk00kTxu;L|GY6;swr zP|^XOZJ@RrTl7-WZu~#(=XR2 z2=4bsG$mUCKmvy}a}AW!qTGJ!b4E~#ir^vtnyR$d4JIPjhaT-rYudu4-9X8pqXYR8 z&e2{3z$`*^99&NQ=SRDAB7kE;eYgL)|m36X>Re83z`>O9MD&yft4 zSs26!LAyUqLFbZcOU);OkvH6-6ZqJG<@{uCVf%GLFcO4}hDvn4UuS?dD+Y4UOy|S> zDyPUHe*A!Lm2X8bT=>k&8roRM>uT*Z>^z^)eU*0?@Zx>$_Q8?W11C@K({M(fW$KMU?i?Kx`Tm$Mapzph3 zEtoA-F_~g{Tx2af)!{Qh5Cs#t*vV7jUrX3TatC~4fKWFe+eHd(l4|)aB%nW{D2|g9 z;0`Y_d+J*e}4#RrgRBdrDW|%D4_4&KuHH1n2t7_H_A)pbNw!6S# zA=}EdYPW_xcKKD_Py*;(QGpwp<(0)iY#rYDw|Rri?`^?;!$&F1@@YN?va}+pSN)K4 zxbh020Em^P&^uA+EsGul>NekS#HqW#pZ^8uj9c9?oM|;R9maA-K(|oAfLC+ES;FHs z>lb!_JFFS^Ej`J60Ff2w$_c#ax<@HBvf~RvElg8YxL}p-ix)uMe9-~znoDJV^v$+7!whg-3Nm_Tav`cs3G>ql)6NRqa8} zh-DDb>q7=a-s>(Kex43jXE_Y0M>xbGoo&2&&WV9C3=ai7+~OqWaLs{c>?N=|H-{=?JlMK0o!<2yJEtAt|#VUg+ zmgO4$LD=|R4x-W~icyh(+jPB~k(Y%V-~BblOv@NM!51*wkh!)>C^Rb30GCer$v_qi8^kq<^L zaF&q(;3k%(&Sdu}9FekMjhBzZr#SI$uu~Hb)s17996b)&d$3eEb6m&O;Rpis=_*Mg zG4P_QF^`6-3s`foNVZJz&bz?Vq*}#XY>21Hk)jT`1L|UEg&rq@Jpt)V7d#^nEWb^z zR_vT9EJPVURqVe%1Rq&y-OvemT0ej&vk8G0%SUr<=5N>2o~3%0CfppnR-A0 z4w=opee@?2Y$k6s2Y6>=_6QRJZ{!z5s&j8k#|lNDi%}*0xs|BHurdnh(vG-qP-7mM zcK0P{K3gO}?qx0fjGO)h(QW|UBKXiP(A9nL$leGv-a*-De%wLkg#&vJlSa#`pMY2^ zF__zV39}aFY;baN1E`a_QFk!*Nck`h%gZ3Po}v{lh(060+XT1{F?!3Q_9t{_;b7fa z&wG6;HF*@C6DJ)!5*0*?o|qK?^b{l)YF|KPHKhpr#2NuI-yJEODG7qwpGYJC&Erb` z52XM90Mh^e5hs%askfc|2!LP(SexgGehbjI8XfJ`3=5S0Z74TrDA}4QB(^4v{8#U$ z5=d2j*6sv!SH6A01}VdR3lMY!R0uS3uuqX~C-VlvN28M8cd}pp2Z-KuEF92X{?_yX zM1h{udXnjQ=1t|Z+7}&He>GWjD_CJFk3bd!r;`AK5f5k)-+B4R}nw%^^gCET?&}q*! zvvJ?@K{%FD0IAQq6oa1Pv#+=9FD01gf>b?46o!Am&m08>5d^~9qu3W5Z`&H3W0;hPs@?-s*rAdObGz1SI!53ql5@>urM0KG0g3em+FPu*7p3fxxr zgX&n15>We=+S%;iN8tfk$d7D=46EDNJHwXT^~N}b2C^)vfIStJ=qO9g2gTpVkIHEu1wf5cfu8b? z(tS178djn`_R0&iYav<5Io_YnDK?}lJvS;;c&dEl|$ zu;HJAmtmAt7`2oXW_l<@g2r5U!$)jC4sNyM+JGw!cw>O&QnvSWv&$Q;lRrz{iC&Dm z0}$hIVk24FuAIS;t=X=cC1T4*kB()!_DuTP*$NjX? z+(t{BjEmJ84ggS?Qf4kT0rFDDHpem6n^jL-e#i354zHdLaQrI{B|o7Z6ESLFmI5fp z4?Ar}t7GYkv6nKKp}4d%MQL`;#Tmu=5v&IF_q$$eq_Pu1qOZpjRx#`uESKv9KOL^&r_7|H;we)kHmDkO|zSVLcko8XNW zxnG;+2l5*>H+Ln$yyM2-8T82Iuwdzpc$mkann{K_k^Xa#AJvI8mf3oh_1!$mE4vkI z7sW&ozVcnz?pHl*C(Hzq!kh-tI>-04?JzkDTYWJ5UtNxMa8_ILVZ>)?KiUD0*OT_e z(0w%c>jdbI``-Lgk8KR}A2$=3+Fca`3+nbwOy+3<%>D|gm&-lRlq%a>2r6Y=cJL)$uggMt#`xya;a z{d)@G;6W*oqnsg@bK3o*50B4O9I32=x@%c{+VjUAVzEsc0~)7?Fg~9PcYPYyTdYkl zfYG)%%p{&Jy;d!L??@q7r0}WbXg1HrYbm~-k+6OjVne7u;37(M$%+_XWSo#Y%n zZ0{}Hc7?Z9%ff0Hp*LWY`R9GPA^`ZlXxlgy3#kWYWTssGV?Ic!;Vfv1m~lrpl5HjD zw_y#SEuQn(AWH?WK6&Ryc0bAF`-*BvoYO_GxhEc|k9kw&_5G9A9Gcd*!yn4xMrszSMC!DC zE7WqZY_*leQ^<Z1JX=6#yw61jf7?pZ1;gq zlGH{CjlE(m2bulA>6aUE^oqHO#JHmJ77}p`f7TxdK>7E3@M&9y3N5#^al9F%E1}$i z-|WJKxqct)y-i;q$)ZH(fyiD_uqB9ePl>esca9a#iR?J(^0JWh`XZ@%a4a=1)ICEw@>+V&FTv;CS_tqm=kyEcz>9%d52L`46vDPKUV293zz~URSfDNOx$B zu+lcY$efO0)7P+vH*Bo^Y73^7*V=TB@GT5_*W@w2$*fN^&&^w`>Gk1|V!zOgK2*u+ z*9Suu3+to5HKg{fPXr7(g1;INlI@>_?Q+4?=UYeXRXLhcf7BLLYKFu1f= zp-Q+^6kyC5(e7!Gw6dv34V;h~084kbUhMQ&zt96sqJlQ_m%k5dqyVFqHCEgX zLojMrm*_K+pu!t9br2!+k2z4ae~ria6n9FMcpS*iHS3ggUXQbtA{~T^7L>pUvG_?p z0rkt~e)+o84;wD!&N!RIUq|1mid8!rIeupmVLL9m?=OED!%tj|Ka$xyjc>>|yc}AK z+$nGP9Lt@P)VKqn<1rnEtB?C|ZSY)bFf-Gyswn%f3e}*Sgwur6lH$>nec;s3c#26}Y=*1&{nAvlFSi>=yoLB1EJ|OI(NQcd`fwETLFEPi`c<)?J0KAci`0&2eWot;8 zsiD{Ym+k67_HZGbE(eer%1B7oYr;T>Z1q7#ld@tZ1zip&lfXMcd0O zcvJ>4le|il9WL&-4rz_}U+legSXEuSH>$YQ1u7{b0@5K!gLFxYfFhkD(ny2SNOww? z2nd3Jv>+iM-K8K>Qc6gdBAk0JefNI%e)rzzI_IDBedqfAd>$urjycwxbKK)s_nS$w z9;bId6=z}oOyBLf2Z`*ga96n{`LM_{1#Ffx{|q}%aWY}<55i-Aj)=G#fmCN-G>pz& zg8jd>@FpR-bD`rqXNN(`&g{wgpnlW0;4dFC>TALW=z%R@%ukCgNogbcvgM@_a5CM}UN zs<`pKJ2g^9aSWkR%#u*-aAj@}5@6IJT=X1*03PruYm+gNu)!|=%jm2Cfc4M&+HZzi zZvKgVaFGsnO~ou5gN9-Dp!`@=<>xqq)eHENt*BJojedmJU``X{H-!b64Ho3~WG`j0 z-{n8MO+|-ZyFz`2VwRz@%(%;W{_Q=0eF4y5x|YYIa~`=c=hgm@K9qJx!$^Tf0px4F z0_mYnht8|{slWK1^JnuV*iPMRqj;PuxR7XMVWVp^cbCN@pyA=0FaMefv7Jr$`)ir{ zWWY+WrFxH5yjP#_R+x6ReNGL?eTIeBYO^YBrml`Tv{MJ>ky*cO%zvbcdlEfLZ`(?0 z%2eGpKU86~g_uif$=u6_3W!10M-1!UPua1^Tg1b(-(4axI#$@~3z`&G{ z(^7VKuNx2?NJ(k|@oocCzxLLO)7^duKu!Z4uhrUM?TCr1VO6eDOez zBwoq*Not(Gz^;fP@gaEj!OK$!Og`jzaUpjK+CtUtfM@PT8Eaq8TGZ&*U3}yD` za!$i0KRG!_7s}jIvwgS~UjU$GEu;;7K8M@-y0)~yQv|sk++Q16{)8CKhhu@HqUFZ( z7C+TW$L}f-+jl+ZxYB9)qgjQd=+;6{sQS%ItlHUg{rmuGZ!91F`dC$;^P_t4Dkeb( zq_WZ=Xe%?M2NLGhhVs-EdeCkrpyj4Wj|mv9rw|57^l)9$Y5M8ed=L?;{{|OhP{^O* zT>mxp$f4J7`VFO+HWQ-w$d#%R9CjR>RR3M|q{a6vo{X)dx3loEKGe^jwIUFPV|-L} zf7;iX)ixmORDbnFtO)!(hy{Spcz9(#My=pH>wOA0=(S0=1*I@d9^q%?;R^|+WsMVu zkIFqCp=zF_VV5=y{;wD4AHC8pc_3Z>l{#!^XQ4V$H*BXiM&@8IQ0!2G)u3K?0yK9^ zzxSpzfGa3)37uXSUQfh3nWL&}>$v<;5#9~!B|jpiJ8hI+wSoX#jN3ivzQxFYxoqXj z17tBUWHcFGLL)dXEnB6POw4qDVEt^@vG$!0#lgiv?KcW~QurLqgfE4cAM640FSGXb zEl7Z~WKB9lkFBfcRHK zi%SbK0wN(6k|yEFJ!}-!9o(lG$A^oEvM}W8$SuY4}lN6T*k@ z>A`rMj6zNE-$=k`|BTww?LYg8L7Aua`WROLHY)2J&;i774JrVZf1Y2H{8n)agOod2 z+;`_j-Yt_yTOvR`*aKK;<_~Z(OF2^tALjGhE~GuYcoF$E??pTai-m(2n|Xgu&D_v!)9GUA;cbjLRcH9@H*)*`!Hjf_{u`W@}S9=eg<~v zM=q2%$-TuhVb_n-qG-g40p%rX)DTiUcW~O4jRbCt?J~Ujjc^(?!?2PdX7yGIAkTuu z22Al?GE1*u9V)IVY2ikXQi*36`Z7Dc{4aBjsE(0q-~+Ji5wJ(S{TXv->w>%-^+3n- zDVJ3qT*aEB6ab8?98l>jV(n)=U?D*?Srn{G4j~9Ce@&LNRUSBrSi?ppG`|AO^;kd3 zFcGuFM&FzfM3&eD=JZ^c=^T1#0kpU{KowDE|7&`k>bMq<8W+hE+$Qz^`m*FQW)khu4oqKGUQDJl@f8lz;yT^EFqm@}r`QB45ceq!`1Das0_ z=#502;Lsm3zx?j=)C8nV*T-u7b`vN7xCSR0G?t>xzwljGF?aSP?aJjTuwSI)YK^<& zo!pE>QjfLaK^wqf?OagA>`6BxF6{^)PulFWtjn`>P?L1LxqlIr3Y^4@m6;C*#uw@% zmuZ^@g@~U5ao-RS0E00v$Wc^O@VmDAa{Ym6lA?1SEhFRk7@i9ZOO)4SApTjZJ50dZ z0AQmDqd>N9*8jA1`=3CeZzx2CsIR7ihf*OzFcMW~i2Qw3oJobE!e;}#(bj0qXAz+Q z0?!e&p0NaRtd6C32zAuI2hvq=lRx(s2Ldh(92Z=)w?O(}R$*P28FIt6S71>p{Ws(a zyv_@3dE0`c!XQ_A_7uUN+efBz* zJZBcn;LaevxazR|h642F;+g>ReF=XI2lD-p$vM$=)yv%*tQ5UL5m=hPzM#^o`UaYv z5s29khH1|?i74rfMz{#bHzgc3TUI>g9VanA47>&?NyLP32L##S;c8vw)8ha00k<2U zetQ%0dAaj@Z5=LUE5do8LlDkmMh4+L3gTN%pMBBHY;D7myS016@iH1 z@mf2cjvt>wJ=#r6AgwCFu-Ob5rUaM>?D-yLoRM4T;~x*KK^w8 zLLW;;B9;erR!ht&2He!(6dE&4nwV3-?`F?;`1cehm{TK930F@r997+{3sZ zU=%X8RD!Pq-iIPLf+mu|*ebkLLSQaIE7~mpzRv9_GLE;;1qM0uojtJA*o{oM82<`e|#D1uY{Zq$=~4k$#vK@`m4Z4 ze(+*VTK>Zj4eO8fzkd9n$hYhhQU2b)qr1ZJ6alxOz^l)+gX%;b$@56Tz5gfg5%Z{I zz@u^)9A^~o*~96^!H9&E!7?a$5CQS_4f zA=b#7X%1Xe5&M(Wdjhoyj6@uXyxR_3rKIpl4>}^`tft32i^1LEh9lQG696ZMVe_u# zl(2z^c?N~)fx>W}#wWLKYcpKNC&xX@NHTQkx+W$QWQ5oA@ClUX*}6+PgvieWpOi_M z?_Ya;RZqPwmf^t^pZv;< zqKb+G&xJbkiM9BDXaVe@tTHwwX&7up(t+4TpN`B0PWNM9)wH$!#EDQ4cHJ~cl)5gs zfaV{lHkEZC2cp>{?Kel+8x1837g2-XEkI8b-d^vqJJx0{q{pjttZkmPy8~!!X1Swj1Y7wKu>b+&gZ{HVU69I;irfxc{w&`M6w3K&~H%H+zoye%Th=hm%)e zxc`}>XQc&8rHrlaCH#{qX^+J(+Y$du4fEw}7CyPC6gF8GJ#wB;+}g17{nu4oH|&Iu z?=x?4sL_PhFg4mBepEtEp?;k<6w%6%NGH-m|EvTY8qGAi`PR2Et9J6&Tyd_GsIy5Y zk^Q~%WY6b?Uk|Z1?ce-ohXe~#>^i(HDuvn=%*kQMz7DYzE^QAfYozems1q?NsUkV( z3qnNyL`vlC5msRnF-T3Z>(!`=93O5CG|Xo$^FzG)2)x``YsJ+ISLk;W*(bmJckYOT z%>Uqy@P-hmR3=eJ3KsR(ci`cPT(R8AT7Ck7*981d(*vgE$!gE~{-^}zNMAi)x@n6d zwodJldT+9r*6|#RJ*K1dwndqe(iWH6Rr%{!65|9Ui1c=HhfecIhhg30*RGo0J7+aW z1bcP`*)T%j6Kym#%|i%%T9p!q-AR+guTt3d<^6z{i{(J+a~yW1LFeI!MH|J*6S4gl%y<{rfOEQc!_%O7k9%=n^fdKgj3+ECVYuoD_b=Ypqqbfs1=ce-FY>X9^!%EuzT z0)5~Gk9c~H8r+Wq$1;V;2O~8t;a#O zqjc7RG1!|)rs=O1deg`X<<4K`pdzHCx%trTZWb{v0S8xOq)m88vvOHeW^u4ho5e-B zx>6b+`ugzKkjh}i7;E{E_~n2pf)z0LgNc-o_}$@V*9aurQ~n;eVqeCQO%1Kcc6$Cb z>eut}6zFjaPjzSc?D-h*k%7Bg*GX8$!K;o1JwSY#lfjM%6(3=`L{%fq;5uhjTO z=3&ScfpeKvCz8hBYEP)-;3-+UIl0+?Z2p3k(`Wwe(Y?adnIIe9!IIl7{Eh7X z24g*vpSVazVztX2v08nNopx;uJ9q9HqbYw(<*8t0$w1t2b=JA4x8&b$kbYxT+f17i zhgf!VjYD_&EXi5zygDG&gIO3LL%h3;_NTXNS+>$=72*a4d4BUuhNzjW_vr0eHI!cW(D^<&o7&L@%6g8p#X0+K>GzcE*^= z_Q}?`^3Yh)CUbzO1NfAE`u_(WiOdAzR1hG_7I5PUuZuqnbeH<}Z6s30N3zpGXD-3* z743KVdxE_*LEF^s*EDQs(&_li-wxa!#QAVNvWr*4pqoF$ELG6SqI}6%G$%xI_R-qh zouB6uezIE~{H2r4Ns(|^T8q*yG_5SO|fgimmH(D2_*&9QA0IpxR@z$Z zi$sz6WF1YA0k*Gl?9Ymhh_|1HD&hDJn_R=%3Loo;@(e$GrUj7C^vxvUZ`h9x>+lwY z$iA{%b&V-z+*}}y0KY<=^L$DE0iz)#RrE%taRNYu`9<$*1{TWY~;rX6lfSW8sfaUbykXF$kHBqK#plqz`OKLCd-4%ps`cl{P~G*Y|j} z69*qMR7RF?0#h$&R$1$9@2fk|SXCwkXv7rSr%hg=+xbqi*YbPls|2nvyM8Ud>x&ro zn8|MxUYqgHLK~`Uqak#RKZaqOFyZ&^IQ!?$>?xkfmGgm>I$Ip4jYU3guxiQgej*}Y zICrK;a`xw=^<=j}BafjEBe`7gX%lUszo>&z%Nk$mkn(5^GX~+g1aQ;umi2jEx8Ix5 zG?)*O4d9&hp{2jC&;pzO_}ec(;{YVKa)!J3u?Jn+42OTH%NZI^eJOPgw)*`CPGiOi z(!C+mZ$TtYn@K(X3aBVKt%Ke#FLN~TV)BwAPeE6Wdn+9k8&&Vxs_5wJJ~drDU)l-Z z-iN$W`ZAhp;R`i-Km`F{UpC<;`XSwQH)*`?c4HjfMM)QJVvm3cQQyyGe`Ko+W4Rif z49cR~9zPs@e|~1--~3zW6W|;5dzzXz5Y7elKXESCP;DW^WG~dh_thCwpgBcY`vLwz zyZUujXUCM~Psi0J)d#<28yGywc@@02^Fyr;LTp}rhUB9136@Evt?>|6wI0&DG>%9y z*FTorklN-o5O@LQj=(9FbNPl<$TFARO_20~9U|R8bn;1c??+lz!4h@^uVs3%;@5lX z20~NRH)q;u@l*=bMaGplzCF@jkZ{zK3&r(_5}CAiFGxs*4A*rqfrb)p5R$O=tjgH( zm`VlrW5WV2X&QQrpGO0z3$x4!Wd?$6X0-^*X&Z7B^cPT32=H7bIa}xI6ztcFq)us9 zb7)-Owg@;>tn~0W{`KgzpLuam2ogzSsvu?SxLT6{<-?S*aCXGB8K)D|RVt*|_2#!D z#fk&{&D{fhq1ES@8$$mE*TTrkSeM$`XZcCo%0b#387es%x1_{*TyMo)I%f~L zAuXYlkM=I>JA@AJZy4N3&@GiRfBHh>!FE10``Bjgm~PClDzk{iYCFCp@sG@1xF_(~ z;OB86_ayttV+w=UN*^wZ+^Ph!zOL{V!vUnO6{wDzFK%-`tf zu=thGk5T*zrpVs7H5~$SoyyvgAI3hP53if@zf=-Yg4DVM?UJt@=g0FkXXK&zFki0bPMmX$;eVp49&qTmgh{heG zAr_O2*?zgXe=k2NzwgYmve28=!&lyu} zhMij*wlIWoB5@yT6E#i>dRpeh8TDxU7>~cJV=XIwmfn5U#-*>LyI2r-v zrA0f^OTel50-q{1#rjFCMnZlhdyu=3pdiAsaITD$!ViS*_HVkBSsuZQu`~Nj91u>UHvOvqV>|AUX z1=;q=r!FA41L%nNVL2k81c%WbiYL=g2Pz8;4!@GzJ=k_gTJi(s-=St{Q#8#Pq^-bM=asx1^|-z<1V^V%I2Q>{xCohEJ_8tu5)oPv0%fYBBj7@INT-r#mINr- zB7TS0-MV%h8f@DqZrFX5;(fAypT42KW9H$+z;Z)}I%VCCe5GVZ@v5t|)DoGXquHa= zQYge`^=z0_EXxp=iaLEND(WXQKd8Bi$+=_lsqS2T3haAsZL3hfy&Mveb1FDyD+ewr zP5@0;=Cx6c(JZ~>EJG}~Dmj;kYkp?0DsF6Q+qTCu*FDJWGYqo547)fwab#yUw+1G1 zM0y)`G|}L7oOex6|A^viXTUTLG&*X~h$Hm%r>|1QR$*5~Y9=lfkWXyC$j|Qm1x~k!4;bSS z5X&RqqLaA3bBLjF*_%S6@y+Bwgx4N-VMUa)>>qZ z?0z;v?Q%BI>h~ywiTD9jwmISd2wzV2{Y7$Q_g$)Xcd~`L;Ddc6qet=%>38kj;)_P{ zWSodK1efo@N+vP948H1X(lYYO5ZNk{r2OjEXBB$bWe+qp3FF16RxS8Vo27C>a&k5T zc)l3HJ0RW$bHBYv%NxM9%*3ZZ5GjZ0AMSxzSgWHU5RY>5SwauC!BDFL8-xPcpLnui zP;=r<4l_mW;Vo+ZO(S_lHG2Hbxy*dd7It(tJ=J2xoVyGQ<fFj`!~b0l8^xpo{*Z#A+AwC*Xv1s|(&Ou&cuMkz(TbzGke}4GhMdMPw@C%@ac6y_sif=g%eTZBC_s(p>6)d=Ic~bP{F@CQbaF9FA3^fqnd2Ud8*R~ z1W{D?ePKWs@`sb+G4xM%IM)0F$otAhA(3liE_{gN=ETWkpuhG>%9ypRkPo+7=d+t0 z61E6kT1yAO*)Abk1}5MF;$)@zH>&ObO!ohtlkJHefl%?H;w`Nq1pB*H4HPcrU>OZ@ z2@ry}?gEypN^7FAFPqa1T8!2l2_SYt@6+?6X;Ga>99yjDBgyxaaM(g*i>9{pANe{l zcMg6AM678SlK5o%gj+%M(q=w%JtE#N;!oJGDCPfHjyjnEM1<3I*Dx)L+1%nv#2K_1 ztms{_4{?F#Zx^wRf9Xpq$W;A9(llZyrPD|rSgRetW)WwZNVdM!Iv`dopn0h7oB#?E zi^|jrt1+5@JONBb?GPD;{~|0B`R9BIgjnOJL5Q{4cxrxfygO_Q11^h*jSV41o}U1& z;k0<~C<^l_khJxSI^{S?Ek#O+M1!gNBBYv~6hf-CL7)l$)#tk1SzEn`9T0=pp9H)f z$2+wI8v{TGOrxpKeyf!X;u1xeth_S_*EM_j&GeJ5iM%I zluQf4U`99_vnZfhR6z|%3G zT)6H<3w@LLZh)(C5p}sXE${>w6OlQiJGt;W<~cc}uoL8M{-06BT&sTcV#h6P#2s_@d{ZFg1tx1_AH? z1uwUMYZ_9v)4oqNctyH_(%xco4JkPmZIF;bfobnk?vJoMF*KAo9V)S`37f}D?%WBiRWygGe;g$`;>}n8D*QY291uT` z3S>59#yE|h=EFa$NvkwmMFol)T}KS4fYQM)$=pho_ZDDD zeM`gN6il3cQ#|cGyXVF^2iWT;LD9JQ`HJw9H`P$4^>DV4XZ<*L<}XW!89^+J!1jTE zE`FZ>d%HFYQxMZ9T3SMK7{}D{(VpeTDO${%cH$`wR=1Y4-yz~Ik|kfq!kWami+YeW z=_Ewqel}xrGrlU(>u0VP7m_Z)SN|1OFnDG@D8k^vAaJZHh)DnD^BY)5VUfgR9^>*o zC&Bqm^|E5(^uj5M z$+o)%l9w?7Vjv`;dY?M)WdY%Dkpx$7K6l2d`eGlN#Al=5M{7SnK?X74^#VYeSni$n z{d+Uc@M|6Rfr06mvq=&?tTYW#3t0lNv+0IC58*h35OGZX{_(g+XmTUsvqvMS9WW0+ z1-ux?h#AN$Y#kp<35^&e7J?bm<*X=Ozh#=tH2EtXzGwP-aL@?{bpq-tB)@L6O%PQ} zZwp**5-1VbHrCD=?E46;mJikL5Vv7YO=S(!>)A%xIP=(jy(khjM~;>81KsYN4_Vj( z)%%d^W>s1kqWl;&cX*BxnILJngAjDZQMy2ZoAVe8?*i<6oFD-W4|^1~*UzgP67(M0 z+R|FEDgyOhL;pT?gjJxZAjZ2j6?)8Z-5j2$IGHki#bV-JD>HvN2t2btO8B0+HavOA zyjmQ0#rN^n_kkusba$H2r`MV#P2K4t>Z5b z=lTSfua|)XVUA#ci!;|NymCh$ka819DGCk;o@gE`5n2Rq{(d1H+6^9(Sg)?OkJRjw zsT_tkH|uJx?j4dsw!N|*ykfj%COuq87YqT|iT6OtPY<;1aT_ajwmJZn>btfdP)0j= zkiiC{X#b7Vb-kKlh;X3YC?1L|oQ14($?0xvzuR$eOKADlb{T;DvYwU=yKD&bn$;X@>*jU} z(pHv$l;&nUtIkIS)EqW03%QjJ5_mT30oqT}1ErPD3$?`T#;8D6puXbx<;r6D+lLk) zEzL^Gw%rtrhsy-#3)T7O*3MnJ&P04>W43mymm54K9k}y*kSRgIKaY)D5<71`&TswY z?Nv_o&ge52LtMJe#P@w{d=Rk1M;Y%*e}uQmBA9VP&>vF0GI4On)ahu9!56h~u$ZBY z3apJafi30&kdSK?$y9tJB>GPbY0Op@A)JFJQg~Bt21oYi8C`YI%FhJEU!&g%MH6m3 z)L;wQRIVra%Y1!iWM7AnJ{E`HWlp*N?L*396P9X>HekgmdC-azTw>Ig3|R?-jwhbzwGsz5cB z>V#W;j3*|qOx68laay9jOUEO+A$Scsu0OFhl`j?^ZJsmM;J2Z$+UcWIw8jl1-mbUnCDnSE0WtjvLU$zC6Y# z)lnS=2oha1kYFI^X4gt{fnW+Eu{qkCZAG5tMC2+Tf6_F3QJ`>9PA)ZP@M*u=lwe6ul01aXV!kGfc-nLV6_mA6r)zWeJ`J%r+!wPuq*NIgZ=EFQ<$gL7XQxe zkFw%rTf?7luKVAjia+eKPKbG8+-K_uG#HC^X#uhNea*(`$(zQjojnyEgsMN!@G0A7 zSLu!Uj*X?3{Ua%iAAmhKk-?9UJ+Ce{xs|>*R}@w5%B&WDclUnfz6&H%F|e@Hcy9Lk z%}u<|)gbWel4neNwx8x160di<#8hPkM|Qw)w|rEgP46fzd?;1Gfv^!`dPH8daGCfC zzT}&*L7-J05jPr}43ouRa@HPp2^ee{U-{UcX!g`tl-|Z0zJpyEQICMEdyNB~;LhRw z-L5m%bdQdIoDpR#i)Y6T%6F{SJeo5^o2fsw_f&j=tgsUH5ru^4FLD_e zd(PS$!)DjcDrj_-p7-W^;_qIZ;O!;(ce!T4Ps+YzE13fU)-FoHqWth>m32pj{mWKs zv3J0U#5V8PC)VC;jP{6>3oatDiq>omyMJofwv87-PS@vx>s*r47@aH28xrB6NO-K& zlwQzo9auX2P^=+!9M<))UK zm5w=m`9X;1xK|gt=PV$E<;r6Rv)U2Et*?_LCc!Xfj;tZ~V$7Hi4})v`Vayh9cwx3Osyyp*? zKaCYisAPE98B}_x3%5UA1SFC{Sv5v^lB}*wc{0X+j4z>Tt7-&CcJe~q2jl(qnGSV^ zUoSUaaFc(u>7r$2(kbb+?&#cVh?|ycxX;Yn!CE0jdlyIcc-se)w8| zd^R3*ypS7zHReJieXC4}O8m&_L_z2awYp{6wezytZ?pTvANGZ|Pm=N&3M>7RD*wHl)@x!FJ3UHvh}D(|vZA$RVd;=-(U zWE?C%)tete1D>^=bi9q+IIr)k`sR&sj2Df7J2H|XIdw8YRYn{EAL76g=}rgBg1Z&F z=mErq4Ta09`AB|JXs}eC+HSMIx8@*qncF==Y0F~>G4^e3F)m%Qa8A!_9Nbp{Q;kYu z4V&New<{DRuD857n}^;@eT_$Mmqz}DW(wTy1LIrox`?QbJIh|sp{Sn!i3i3X zM^L@N!U0&blp%rY?JXE_-2H4qv=oe(Hu;X**d62>@PyPpH9rKZfmfZfOXi4_ySgOD z7{4F1bD1D!?jr0*rg4l32y-lWcEQ`B#{vhbf4S@{zK*)g?Sp67e*tq$|Es54(#X}K z|J~x9hTOQ{CF24HWH=G|5Lly60AtqYy=YoX*Z=6K4wOJ9RTIuzrHY7r_Zz`C=kETk zpkyN&%pW9jU2Cd@pAT&}IERbv7~=(+x*8X-b94jZZ(M0L_TUKm82(+u{O|K8tk6dmWG2XN}*qk>3``bdB*ii{7zwL7u&_eYf-vVnW!T6~LZlQUez1K2L zGfsD1x>g2@N(zKdBuBc>(Jp{Th!Z#i^Rcg7MlzCiYM-w!y#8Tpfm9Li5^dF}kU644 zGuy>Bz7q;7+_V9Op$RA(7C`XH4f(IiQZhej8v-VQLYW|)hwkU^&taKncvM1sglCyA zJB=u$+nPcgJ!&gh*}W<33a>@rT2fg)Hid)wV{*)gM(_4pW_;w=!biou?U+=rPwG4gbMw0$FK@T}kAVUuW*E zhIbR5mC&@FPWJ#dGBboxb{laT`rq?Uq9gc9I8B}6ZtCe}yY#{P%S@+x5eHr~A{^h> z8P4nR9v|#I#r;|Y7E_JDhhzfbTUC-9!9#Q``fWMm&`Ydb;Q!~@wMn`0=$5+B9Jkx<%0v1OSFf1* zX9;_zc%8laQTseb1&Kk>zj5Ps6^=3me%SL09eS{zfDPvHji> zF8}n2EvTp&=Ls+)P&Z`WsrFtFA@a%Uo5{q=z&3NfnlV1nTUA?ExN9@EJSR@;ll4n3 zcT2faz1_%Wt#(Uk3*y02AoVyU!N60X%D&guu`iVc^6sBHoANtPIB9G_>d2LYlZ$W_ zL0MRlSc zKe7l-PZejY9sLFb|2qLC1cYS7kVS5Hf{o6q2e zo4&ZqaQyIG3veJAR+f;)H|9gF^9G3lcO=(g17zrsdL3GpbB)o^>(CiX8#Vc1FF;t? zlfecZkIpdCqw+Ubl@rUBX0EIA{L9T%y6WCf|7J$dze0-V@*>Xa9g?tK4Z9Hnay?+1 z=}R}k%dAZIt7;w-OZL-~~#j~ttl5qDYuEJKly@BQ?7(39} z@M(|2W5Y(a63xaBOA3|Xh&-Xlr_*qj#=xjE;?LFL+g|atRWktdiIS`Or>Y%d#4rh4 z7f!Z@ncZA9mPCHe8RK@ld}esdbD1e$7u)ww8VZ1Jhc?Pp}vXG}##y86@4(#B} z2#_BX0JRKH=5A>w@>yBun&Vd^25btVN$k}S!pL{Myqz@If9d1Ub@kDunJ{9spAv#=5j^Z&A`^3~z`$e-H-@pE$`JN{=^K-~ z&x3;jvQr7}_u2>4_jCAU;S-#u!e>8uX6=cQau$vnP`+vH7bZZ?U-94s8r%6$%MEBR7uVNt{ zF|gIf0M2*I#~!+n%$yKppvK36>)oO8&%gkg0DIaXJum*x<^K1cOjs3Qb>=lV+Dzzy zzPy7*kHHk((Qp>;>X)r54SkoE!xdApf)T#3zm}Ilgly(%xjzO|K7n-wj^oK&8VR26H&W;UpV)gSDtiDE>jBls_|J)A1F^CPEg{c8 z2b5u+r~d1JLL0UaH4>=Tu3y5+$7Ju zy&bToeogLmgAV@lPwMd^h&7j5tcPGlWHcBSAQfQ&{!$bGv;zcdq-|Xgv#JWRBR&A# z@9dpE^-lC>oM>UG$oCn>O1XMbW2d~FSxXV5=5X)lz27|X7exPh5kV#!9#Ht{Ca{Oz` z22pWPSnaM&W$f7uUpf6}y2VLhQjD3ZS$(H%yC|3O*JIw7n-6tIdTwV?xc~YlD`ru@ zTT$%to&9e*VJ%GhVI&OY_%bvRv0z*W<8?ClL6RVqmA0e~nl51Ij=lQ5x{6;u5`W!4 zLU56zMd;F z8juvCU#4kJ@zsD^r5^Ygf-EBN{g-No9*8gQh8W`>NE31lyTgLA0J22w)SPIdxb61I zs|i1qr=gw2=9Ij$h&wp$U8Mrf&(X?+tqwJu0Z2%G6-e?@jlLZf%@GEgFqE-~}?kMWxG9NkpEOBZd^?Bcs+mo4di6WY5z3go;T65ws==3P>7TGhU8yRed+snSA|D~59VSQ zfb8Lfirj?;I(|T`)awK^6us*Fqm@>l%dfrq1)Yp;^~we7RX?rz+VXKg$#xW4erN~# z@(2YtLw7O5{~p?xob;!P98&Vy{qJ!72fY^J_+G;$we^{SbO6u&pyKdNB|_gVDT$ewId|ets$f=bgdB-mOtLkM(7u5bJ|1ZHx0jvV`5q7L#Jh~ZY_`5 zTx5;@d5NrOH`QBl{j$xzFrMPYA1w$U5dj*$1+ce}%^?|uGL%_s;Q7W>m@S&QTFRX8$ z`mhI^Zs98Sml{`B3^z+z$m1&tHPsbXv@Q$e4u(%h5fhCiGuq+XF|tS~>f42zcL!N; zp|eXcs1qHwZJ7{K#tKEgWoE!Cfoxl=hqL3arlV4z0mvh001|GptFgPCX<1EL!B=5E zu0HzE#z$w~KwVQ#K%hCgibC$Q;|Bs+y1$`Yxum0i{4J9>vY(4r4q{+~`TCCAj6aFJ z#_i2ssqDUp(Njy(zd8PYh3r3*wd~5*_=&UtnX;n_HtuHPbXWOs5>rz1qnQ|%X`zT+ zvOFDslOA{!<0=LD+EP<~!7-IMV%D({92Sn(({eVMrOxvUpGC2mJ$zFb`3J{~RvK5V zddG1Y!B33HMgV;WhtOsbLn%l*O1c|iDT|!a^uiBgr!U<7D?T5F}=A7 zD0Kn4X&wZ-w+@xWq`QiIM!By!`)7UQaMNDNnPoffZ8e92HTb@yZTZ!K2c+CLDTY#h zR3Sxqa=J)#m2%mZR7Ctj0u>B)jO_D@8|C1dReRBs(G+(#T7c^ZUO~T5jjIL8o=EK0 zHA3>;-z4@%NbFV##BMb}?AAp5v-%n33^2CLEofVRlzu+?SScHtfMi4G*GsIg9z)>0 z$h;2ZZYisdI(B=S`b)X)WKY~Hm#WpWR>>{b>UDdV5fR78HT1{XVpNsUp&;d7NY1dP zYWr}CR^`R=X&t(FPTJ8EykG2)H;_lzOTG}!2F<7>|LDv`}Xj+9yL+K zm5w$XZFg@z*OVNr|mtQC|r|i;)jS5Pc}kqg5YSw|s{R_n5_2hNS`!+Q(C|DIQY#?Fy~Z+?Q)NCj z;wyadd)rCV)3VxN2pOw-XIe!!U-yZI^`B$6M#y>!^?-VS`1yuUUx) zekW|+TrG@L?>tIZg%tC0sY27G&1gWTHcIOD=SWU&D)U`c!^YKP|6L^FF+|y=Sl35j z*u5?w*t4gIU@Sd-8F@N+V=gbwD9kE=spXd8QVqUOmZhWaw`(U`&nurI3f95gE?w}0 zM=#-aCYB_V&H>*A%e;lUTD;ef*mSVbHEV3 ztqkmQNnOvA={1HIZ;2J6Hr~>@6IFUYvsZK1rjy9fHp0f^YCBMz=pFBBGD^m zfW}A>S+JAkBYXzCWW+r+1gcUykOSAX_z^E(W%+s7&Rpi^fmC@d3~s>$x$39o0op)e zqjD|j`^}G_lV1zc8fpRQwg^R|MPdDe^Pt6xfA5PYY6i^~Im)1%OsuM%XU$<=bL?hP zyXH@ICk6w3-8%F1fgvGW(vZ+@Z3hvYfMTQvwldw&4}>rfytfE@&_Da7)@92TSJ^~l z64FrmdiMX8)?fJY{m&Y2twF9+W;GCn-Ozru=edM$H?X2H$It>6?WZt}K<@0d>pszx zgplBAG}j=uTM}9E=*}1?tuFJ7OK+;0VmChdh;8;IWo(Wozeo~SrXL*(Et|6p3|wq1 zdC-Cb){ni!HLv4eFTIYB7FtLQRwQhJ;Z6YwRZqk83;`%2lix!X`N(rCWorA`6!cf= z0n>6Nzs2!Cv;Y>+V};)VA`z0H19^>xUbj}x#z*xifr2Z;(D096B{Ub5dB4xy2DF{* z2lS8a4Cw3?_H=}zYD0^N*bAaRr<%gsyV4M#i0Jz!=g%q)TKve3|J;mZj?KDu;- zeZ+H7r9QMCzo#30dmuu!8^SQGpuLOfBuJ_l4Y*C6ZfP~RcRQ%ZITDbo4Q8{gq5U|X zz{0rz9R?b=Q}v+vqSJ|-YyF~M=DM?Vb-_7w5OhsFAYic>oL`O4 z0rWo^LZ=p6&4Fj=cntZSHyBfGeFzNZpeB66?eWu*+jjc>T*IT^l`(g-WRkW&zAQI; zV>~#XyBVK5gsWws^QanWO^GlYu?jcGRGgp}T{3tKn1aWlzq<*NfKcgS3=_`-C{)qo z-`IhG%50(_J?qleqS$fu>S>|X^4KvT>th|RQvU1@>;_lL0C#VEm8y;W!fOf_KD*u+ zm-;087}KD05_THpL<(A7WjEUjKKwdP zD}m%EAzI!x=m~x!l`?Nh z!IfQ^smY+KnvUMd594Y9Y6a@_G%_eZJqmMTCKmoVH_$HaC0iD8IxKXL8NR#cd~O(6 zpqP=k@(iu1`3}}G$=L_otR}B*ZxzptHfV{>-*;kSA+LAB9qVm`UPc_ORlGGOS$54D zYAL0&Tn-;o#CCa&dk5Qrb5s+F0+tj%W*Xmn_lzJ5RVm+CXK>KrZX5ALbCZCO7LMFplBMm4wXwl<951d`WRW;jPCD^7jM)xfo?vyYgC z+)8fPfG8W>bcT;9NdGu5`FTR$tz73ZU5(`s~ZRR!Z8EA*7L7Ad!`F?Bhdeao1mJW4Wpn zp8GQ$76I;q@A%u1KXC z_i=n&~1l448;HR^4+kEx}BiY}p#`K!eY2MFODTZ z_qlEqQbPoZZ|YxWT8)}N0u5-}Z7^WDY#6-TbMhOt))0;LDrz?(!7_w(Zb z1SuYt!s$O+AYwNQYn%lvXkT*rGLLS6wH&|f2yh1@n8 zazAekWD&-Zb?n7(88wa}1&y!>v&EEGl7Sw8Z}{yQtPQwQ5Z|t7xh_&TiAr6j97ebd z-rQGjIib*L^@|A*e|`zB17ZfbbAMQf)&Ijn{NJ$<$ip8wJfsWL>|mtOMfDsU1(J|_ z7f+9oE&SWV5G8I~qYejwBI68jeH&3y2}Q3zV2KM&Ib`Xvu>hA7-{ZMTBq` zdL;C300^}phGG31s>s8mA3&a)kBlaNGuky7v@Mp_R_3HA9w^Yvptn&eriQx<9evKh zWf!pt(Mq3gLCXLr?=S&hVbhYA9ZzA{GDnMPcLL-C@(}Dre;fKR!m}(04Sn+ekatVa z(nbSi<>U2x*K8qh8?y6b+C#lpyqBO7Pz8xWE6 zxI(R^Af*2c(cb<=b>{;35}NPA=|6z?GdjRo37`2sK;xe6 z=fr+`2hH&$knq~!pMKh4ut3_t+gPoi$LfV*PYCZM(klVX#PR?H(@O|IyL#Ib6tVbd zf4Ck~bdxd$C%)gi$p?8h|L1c5f1b>fl-R<*R?1AkSG@4Z#!V;E4WLQe%{4sFbzc#$ zU&{u^7O)kWHX>Tb#!!+p)bCS?)t?iP8@8OYp?)qi4UPm3Bnf{ES{mq(`JTDJg(tEd zl}xF6{x25^uaufpd+{0pxo68M8>;DN@2};iJ^+41tbAuCZOuJbtM*CBJuyxsI}jeq z`@J}f@YKzGz)_}tt~9-nVsisBSNMvdR4*!obl#TR$vhqIu!O7qcl=Er(GwsSx*5d| zIx2l;g{BT7NRdn=_RAQO6C51yA`vS=cX&*9weSY^>9%jGCZT+XUVS4&pU z`zXx)Qr(dPE;T5M608-6L8tV^-;-fs+#0YKbECML(EJrlyS%2Ub^-sr2L;UgQ{8Q% zyPRbZS_PEP@~6H-xP2ZQk7XUpt<#eK%tAq<8`;ZtO$wl3n1*f`Tw=!ueUIN90Dsf> z1$GA&Kc8{}+mi4Pwk1Zbhgve=+m$S5G!u};Dp)#(6_811s6vcNz6qMPs~iRgof_RB zhfd@%2Qv#F($=PXqm6Rv8N*a4VcDi09>kkR9X?xgDa@GHE`8t$Wl@oah) zCf$kA`=7pp9pkJA0e{+HmivDuu=d}d6}=B6U;OCR|Es+#52tcn`-auFDzhX*(UOpm zh$S*-h|+`gtNv(NRN z>)ZXAHN5M6pZ9t1`*;6_eWqUIab3@eTBbYxT1s}YBae%E6W(T?t$-Pf8s8bkd>_?G zLQ7f2#@Mx3ZWmY9UT7Sn=!7Wh8t}W*Rr3!F2Z_peR%%zDnoV@Q$d*ySiPx#~M8i;8 zT~6Q2b2AseSP9~4AP+aWYYONeWUN0kRerKBkQXKX zmV%kJvrrY16ulJIwGT|%tJ_?iqXn2V*L}o*=Vp!qC_%O1%la zu}{7F(md^xKewG#kO3uSy&=aaI_P#FPpST@i9$yu^!ME;tf+Lg=fzu}m6`s9dZQ z-_7XpJ zWwE-suReIToR9h#)+I(giE>URrSqNFb58OU<)7#8RMaHwX@^;8Bo5MXj>GvC=kokZ#Eae9M=U~g7cjsi?pvwzHoD(G!7uX= zWhps0+7<8b5k2U<6b`Flw86u}Ed}yU5Dik&yfHd;WIVx5Ha)ywJ$mZ_hpn&dsE=G7 znz*hVa|my{QaC@Ht#Na6^r7?%#fnZKWGL$*_?Ki8pN$N^&=j+g=f5W9%>i+4l(V_4%xw`qWX| z4@KfYRYGNxj;F5VDK#(H8H-Hwest6}UNEWd*dm^5T5{R-xl z(nyCFN&B?cFk4&w)gRws-zP7(-7Q&f&KJDCA9&)9dTw!b- z9>Y*}yS`u3c@Mt!YN<1@(Lc+IU-&}!5K&W)!;>b94vrn4i{29y!-FDF3MdgMAVkz^ zZ`{-h2vb~kt33Fn2k}B4`y~U#50J-x`4%OQy+2#td!|mZ=1+2l9V|B5wQS5}chk zsQGzgSy!vmCZ`{$mRS_7Mt!5~Xz5jlAIm{MN1j=}7 z6l>L6-8cDJeDgZN#)Co!n;&d-8!u_9D!3LeX?YCy9-O1-3W4_T74TiXEC+Rleg?m`4I@i_o5BGAD=isn6Xo!@N&UDRDEXE>;&!(%wxhWQ7w{&tA!D4K( zD0Pm(^9PD!m+5vW5TmK9bi^5}JMiDKP%X03s({`8i}1k2As*LXUyWXzmlN65ktI0w z;8%f9Ee=r!(^?$*`IoL|%SYVrU(aJNTbYD_IbFu?0T_sG-yAmSrWX9em@x1fk8h<|M=Y73^5{Z?*2 z_-Z)l2&us><;Hx&xszf`qa^3mqO*g9G6+MV=T(sJX*3&IQR>cRiwp zuB*kYJmVbnkC{Syi>uFlb-nvtl&w3wu~y2$Z%f=6cSP#b*lW1OS`J#cR84DVI_Sz7 zO`T3W&%AMKIB8IGbKD$f;G`>yVRZ>mY{7Yu%r21<;_aoqho91*HbOGH0s_7jia3&&A=7-kK4nk!qc3R#h3Qv+ zTq=FCAd9=hsi2Wgu0%tC6f&93X*2waPE%`lVQ3+Rb&MgI{d#EF=0nhG4qBVyu4$t_ zGGf<>?3T_+`MmvUA4h=xI%0T&6_7yr&wjh?k0?1pyM<>EtuI8E&vxL6qC0%lqD1Nj z=8Pzy113GsgG~U90!aU-r49dua}uT1E?}ZE143{~{o79Z7t|DL!NCLem4VRD3+Mk4 zKg$MqQ4++@cCEMkPR}#iL}0e_I?Mmloj3?55U=q$#r!Q6h{(l{f?i3W$*x+2%8c~M zg4tT_nAkX&+r<2|=LcFe-Cs(Je$i0%{Y!U3;pOglu-VNoHMQadS7hyn2@S#b~OVR8@f&A^`iu$5~^cqyK7CT&s z)Q(8J4b!7$*aG_8r`-lFV`#akz0Yh+bSE`ffKl@(l1$e3A!&%kOD}5MYth>!;|bBS zO}9={rh*EUE)O-~%Y|MTYjx50$Cyr@5;#m=h)Q0rg^Y88JfwSAz;_$~JK$rJ+ugra zkAmaZmnd5?a9{6Z#X~~~!jRd7+8`uB03K2g62Ba)eI1}M_sD20xSqaPw4<;!F|Das zX9`_XAvWqpj5d7XBBOwu3lk5%{YD)3n92E7WI=3Lai-|X{qgSHWF&ZTE93^{(CLTD zEUcQF!>-SQi`Wh5i<~9x`tXP8RV%+Ags<>6JEmas#F{d~S)Eo?7u>;HCAfX7|n{~A}EcBKv zXLBcRK0#(n(tvPXh6zqu+*1644wkW^! zWGI`-W!}k}Qb2G4boRlVa?aW0T0w;ilun|Kh36^K)0|IkFSk^&|6CUue~1r2LV2qP znM49~Hlc>sZ~;5a1yZ`ej&f*i(th?E_=0_xU!8xW!F#keAaR`Ffv*{Kn|`H;-NI(& z z;cpE6&M`PQf7QmOo;&5U+q3N_@3BvjbX$uE$|SI+d@B!b52K+n{rgA?jATB%`?T>Q zo%cl=>*}4;1Mfj9)(je~UjF@NJS{M8dBMTcY*;62cv=N5?89?p1td&(ePiD(V0NS* zO*(zL%3#3V}XbY_9LzY=v_f-Mest!t!5Q{N#;;UzPG%T2IDLNrt@e z<4dlf&(VJARv4OL~Ox;7v<6|2|2|`Xp%@%$XLJZ4oIS1RI4* z{=Z2U;sJQXlf8&oL4B7^)Os|?u$9~qS7`>+^N}KFic}R$*t349P=;l2OCZ~&SB(It zuj^J=7uy3A%uw=Pku-9WorPjpt&3^h0Sjzx0X^S?8*Wn4BgGx#Z#DgkB0Z8p!Jf{_fi#pE?IuOn2q34eHCeSPtz8C3|V0G ziBtJnrD0!kmwJcNOF0@Cmt>K_thTG5ac34c*!nyoccv?hB<14Q-L55-uAtB)tTRUZn3le3$d*qt%|UBdf!9d*D_5X(s9^*tM1O55&LB z_4qC#30XaO=8iksJ=Slj+J&=}cL(SXXLN6-k*+q#K63#U{n%|? zJPV!oRBtiTxbLx^1g71PmDE!hS^q#IH{HdO5+=$!H9+5-whHKMD?{>Hp3ScT!zOGg zM-uoRY5+mvuIK@db;hlP=Mjt!Vrbsk(0v)8r7jGYB(oJ33Q&K{{&ourpQSzZ<&*3Q#h&X2t`v%sHmu~7ua3WgTbs}ECp$4gg{JX)Oje=RF9v97lGn^8^#B&7K=O9 z*N*ZUpJNP(;)rJweC78%%+G!RN(nv}I(K0GszV~=EHW*b$$W$#5`e4T^o+O#ob=$T z+gkl4#%)?rhgVfM8wMNcmugvt@K+97P#H#vk9W1pT9+ z5|6F%YX3r|C7KOKQ|uQzJho3=w?0>JYdCqg@r1w9ERcwz?>=dAtx20A`5hFeQ0iKT zs@bU#*_--;(8$D3=5-R-VHasvkLNR!y9MxKzDlXEP#fv`Dc|j?k`zZF?v2e5VFv{m zhhrPU={%N)c)2+z(a=*sj0dD`7RXJ4pO+lSxzALIBx5+Svf#OPV-`r-)#VEn)0FMj zt+H$+_FOZND`;oYBhv_q7cDZdra!ilGYJFBk6q{9XtV&|;n=L_+VtAdD-B~y18bFJ zFx*XAUNTu7F=_7YpD$AB6~h~t7g&=G*f}O3<&3mZ%C;F;7b0&FbjYGFMzlrlCAtAQ zbES&__cR++l48{Chi@G_IG(DeGzmx{f*>s>I-gh|n;95WBld^|xQ;fMJC3y;K)@y- zh`L~SgPu14apf2k=GPsmy;FhA7DtVzZ^n`9#69v<4OzC}&Z@cpiSgYD-ZPCq4Ip<3 zC$H#fPKe}xw6tdwu{yabTN`K%RznZ9F8$)D?_!;WaEEUs>br6X96D&9(stMcEZ>HW z>O7ZUd+u^dLJuUE>uar6*bk~8OjTg55)`Y=imOA4@dz$S+wdA9ZLCoY7%B&MJ<(PG z*F!DbUiCZMBC=1OsopuzaKRzZv0a(&K6tVAHYY0%l!K?o=q28zd!-YZST*UK1ktqi%kf(kk@11Y81XleHh+ zEf5-6Jsb}q9)VG^j-&Tru%7jt_-y8a;uQgFP?C8m5uwltHW;~IFc(Q-Y2Lh+V!?!% zfd>-=Ra9Fq5LmxMO&DdXIUjZ7t+*w>6ei%Kzd4=zOm#3zPviHeS8s_JatfEFZK^(D zx%I+k8s=|FP=FWu_{wG#-04OGCd<5s*G471Ti+viA=tBtm70ppC+d-4o6jnTrR;Ebu=dX+Hv z4cur1otoI53_{}u)+=5ychZ5+m@&U8GobSjgp%*5-l>}tS=Hb~iYVB<_9J&KepzlMhpFhaUgT~_ zn8sqjkKuCRg;s~STFNHQVX(Jb32KQ>qQ~8!xgud#Tr@Ti(!Qlxm{GlIvNB;1)pZxL z;RuP4LSO-em>?lCvOu8eW9%vv} zW=q>{wZd8y3$k@x&Uqi3Zg>moP@Ns2B4e6rMubAr)fv6kN17F+-LuX<{Flp)wd(cS z99h+Cx%~qwPIsE4oRZMpG~1(k#qU0wi!tIy3XiLp*C9Z5c>FkSD@mc_o##@qr({h4 zldfN2xm=CH{JU6=Jlu45xCYT;Lx8->0{ANG=!;GYE}iMq4~@7v@)E2kc^_^; z*ETug7q}9*816d)Y-Szt>Nl;<=2Y8Hccv~odND;Y2Zkv{)zgn=f9o(nkH*0f+8sC0 zIlE3&H$OaLVXRgQF`s>olH%e^a>(p9RVylzW4!0CJl6vEz@mRCNUNh18}!qWU*_FM z>=KdeJS4NM1iKf)u5&r;n*iqTPK3swL3`~1WJ0E%m)#{Mz7WMh|DjDJw>(l_%%I3l z%lK0Y&&{|S-ho1qx;M4;sLvxfHou(rul+zTm*`;?UBBVYN ziP;)AVna@LH>-Q~V2r>O5yRMXUO;AUsxd9Mz^Q3TYWPhVX{{%2!&itWv4C`l2J<4F z^R(uLYW-3oDm9fZ&$b83_r~~SmF7PlHEG!F{}X8YP)*4V7z#og!faOf?2#f2a*OZv zUak7bbG=>J1Am+21hdP;lk{d}4X7T*V~q?Dgg)r-Lx)}ou+;>&V$RnGUh_b``m7*r z#7X518)OG^K(^>Bxi$(#MLsG?sm>2w{ER^)$>Cm*(lvv-t69#AiOlu{zcr{B7@%UX z@wyXq9y5W+XS>d)Y+ME&<$iFY&vXR??G})Pp;6Xle|pXi1Tqw3z+&1A_GZIbc#Pab z9&)Hgngk1MyQEhT1j38~tcdSbr;y;6c=*K7Bivk38bmZzZGmx_l?S=Wga%7f$byV% zTMly=n<1v!hbtGcFpf%wYGYA?K5xSZaQ)`5&3-wSgO!I++6M79m|1_FxGClXRw<}z z((izREO&pMFv?u!i=%Sf8gzorn}T_4e^dL*x_bu!Z0UIU_6mBzZ-3}irm8+cXMW{R|E0utk_H>5Nvm7%(`TRZ z+T(OEM%^BrN4@@G6xOk~CL~$n801?}y!rYU!#Up$&Vf$H>;FDTxN`roGijAb%%J6x z{{1RYoUdB3%M%7w6Wu-RNiq6Qjf`YT1peY1qpWFvzOYo@FR&PK`~=PI?SEbvN-q^@ zCPa8v4{+Z7#}9ph<-aFJeZBl=Gk8&b$*=k)LvU z!*A;kt-Fj6a2ylscn@vc@2^$zgH^=xOVVyIbN&6b55-_%zR>> -``` - -可在parse的界面中执行Shell命令，以及如下场景的相关解析命令： - -- 支持指定kernel层级算子数据比对。 -- 支持指定kernel层级算子数据转换及展示。 -- 支持交互式指定pkl文件中API对应dump数据查看。 -- 支持API进行可选层级比对和打印（统计级和像素级）。 - -Ctrl+C可以退出parse交互式界面。不退出parse交互式界面若需要执行非该界面下的内置Shell命令，且命令与parse交互式界面命令冲突时，非该界面命令需要使用run命令，在相关命令前加上run前缀，如下示例： - -```bash -msprobe -f pytorch parse -Parse >>> run vim cli.py -Parse >>> vim cli.py -``` - -以上各场景详细介绍请参见下文章节。 - -## kernel层级算子数据批量转换 - -本功能会将原有待比对dump数据目录下的dump数据按照算子名和时间戳进行梳理并分类，之后再将dump数据转为为npy文件。 - -依赖：CANN包中的msaccucmp工具，需要安装Ascend-CANN-toolkit，详见《[CANN 软件安装指南](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F700%2Fenvdeployment%2Finstg%2Finstg_0001.html)》。 - -输入以下比对命令进行数据转换。 - -```bash -cad -m my_dump_path [-out output_path] [-asc msaccucmp_path] -``` - -| 参数名称 | 说明 | 是否必选 | -| -------- | ------------------------------------------------------------ | -------- | -| -m | 待转换kernel dump数据目录。需要指定到kernel dump数据的deviceid级目录。 | 是 | -| -out | 结果输出目录，须指定已存在的目录，默认为./parse_data/acl_batch_convert。未指定时保存在默认路径下，比对结束后会打印log提示输出结果存放路径。 | 否 | -| -asc | 指定msaccucmp路径，默认路径为：/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py。 | 否 | - -**示例** - -``` -# 传入待比对数据目录 -Parse >>> cad -m /home/xxx/my_dump_path/20000124003856/0 -# 转换结果打印 -...... -╭──────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -# 转换前的dump文件 -│ SrcFile: /home/xxx/my_dump_path/20000124003856/0/272/TransData.trans_TransData_22.112.21.948645536672764 │ -# 转换后的npy文件 -│ - TransData.trans_TransData_22.112.21.948645536672764.output.0.npy │ -│ - TransData.trans_TransData_22.112.21.948645536672764.input.0.npy │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -...... -[INFO] The comparison result have been written to "./parse_data/acl_batch_convert". -``` - -输出结果： - -原dump数据目录： - -``` -├── /home/xxx/my_dump_path/20000124003856/0/ -│ ├── 272 -│ │ ├── {op_type}.{op_name}.{task_id}.{stream_id}.{timestamp} -│ │ ... -│ ├── 512 -│ ... -``` - -转换后： - -``` -├── ./parse_data/acl_batch_convert/{timestamp} -│ ├── {op_name1} -│ │ ├── {timestamp1} -│ │ | ├── {op_type}.{op_name}.{task_id}.{stream_id}.{timestamp}.{input/output}.{参数序号}.npy -│ │ | │ ... -│ │ ├── {timestamp2} -│ │ | ... -│ ├── {op_name2} -│ ├── ... -``` - -## kernel层级算子数据比对 - -本功能主要用于比对前后两次NPU kernel层级dump数据的一致性。 - -本功能支持批量比对，若需要进行批量比对，需要先将两份待比对的NPU kernel层级dump数据进行“**kernel层级算子数据批量转换**”，可以使两份数据更好的匹配；若直接进行dump数据的比对，建议只比对单个dump数据文件。 - -输入以下比对命令进行数据比对。 - -```bash -vc -m my_dump_path -g golden_dump_path [-out output_path] [-cmp_path msaccucmp_path] -``` - -| 参数名称 | 说明 | 是否必选 | -| --------- | ------------------------------------------------------------ | -------- | -| -m | 待比对kernel dump数据目录。如果比对单个算子，需要指定到kernel dump数据的model_id级目录；如果批量比对，则指定到cad转换后的timestamp级目录。 | 是 | -| -g | 标杆kernel dump数据目录。如果比对单个算子，需要指定到kernel dump数据的model_id级目录；如果批量比对，则指定到cad转换后的timestamp级目录。 | 是 | -| -out | 结果输出目录，须指定已存在的目录，默认为./parse_data/acl_batch_comapre。未指定时保存在默认路径下，比对结束后会打印log提示输出结果存放路径。 | 否 | -| -cmp_path | 指定msaccucmp路径，默认路径为：/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py | 否 | - -输出结果：batch_compare_{timestamp}.csv文件。 - -**示例** - -```bash -# 传入待比对数据目录以及标杆数据目录 -Parse >>> vc -m ./my_dump_path -g ./golden_data_path -[INFO]Compare result is saved in : parse_data/acl_batch_comapre/batch_compare_1707271118.csv -``` - -## kernel算子数据的npy转换 - -依赖：CANN包中的msaccucmp工具，需要安装Ascend-CANN-toolkit，详见《[CANN 软件安装指南](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fdocument%2Fdetail%2Fzh%2Fcanncommercial%2F700%2Fenvdeployment%2Finstg%2Finstg_0001.html)》。 - -输入以下转换命令进行数据转换， 将kernel级别dump数据转为npy文件。 - -```bash -dc -n file_name/file_path [-f format] [-out output_path] -``` - -| 参数名称 | 说明 | 是否必选 | -| --------- | ------------------------------------------------------------ | -------- | -| -n | 需转换的dump数据文件或dump数据文件目录。 | 是 | -| -f | 开启format转换，指定该参数时需要配置format格式。当前内置的Format转换支持如下类型： FRACTAL_NZ转换NCHW FRACTAL_NZ转换成NHWC FRACTAL_NZ转换ND HWCN转换FRACTAL_Z HWCN转换成NCHW HWCN转换成NHWC NC1HWC0转换成HWCN NC1HWC0转换成NCHW NC1HWC0转换成NHWC NCHW转换成FRACTAL_Z NCHW转换成NHWC NHWC转换成FRACTAL_Z NHWC转换成HWCN NHWC转换成NCHW NDC1HWC0转换成NCDHW | 否 | -| -out | 结果输出目录。 | 否 | -| -cmp_path | 指定msaccucmp路径，默认路径为：/usr/local/Ascend/ascend-toolkit/latest/tools/operator_cmp/compare/msaccucmp.py | 否 | - -- 输出结果：npy文件。 - -- 若指定-out参数需要用户传入输出路径，并且路径需要已存在。 - -- 若未指定输出目录， 则比对结束后将结果保存在默认目录 “./parse_data/convert_result”中，比对结束后会打印log提示输出结果存放路径及转换结果。 - -- 输入以下命令，展示npy数据统计信息。 - - ```bash - pt -n file_path - ``` - - | 参数名称 | 说明 | 是否必选 | - | -------- | ------------- | -------- | - | -n | npy文件路径。 | 是 | - - 打印统计信息：shape, dtype, max, min和mean。默认在npy文件路径下将该数据保存为txt文件。 - -**示例1** - -```bash -# 传入需转换的dump文件目录 -Parse >>> dc -n ./dump_data/ -...... -# 转换结果 -╭──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ SrcFile: ./dump_data/ -│ - Add.fp32_vars_add_2fp32_vars_Relu_9.31.5.1636595794731103.input.0.npy │ -│ - Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.output.0.npy │ -│ - Add.fp32_vars_add_2fp32_vars_Relu_9.31.5.1636595794731103.input.1.npy │ -│ - Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.input.1.npy │ -│ - Add.fp32_vars_add_3fp32_vars_Relu_12.40.5.1636595794846124.input.1.npy │ -│ - Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.input.0.npy │ -│ - Add.fp32_vars_add_3fp32_vars_Relu_12.40.5.1636595794846124.input.0.npy │ -│ - Add.fp32_vars_add_2fp32_vars_Relu_9.31.5.1636595794731103.output.0.npy │ -│ - Add.fp32_vars_add_3fp32_vars_Relu_12.40.5.1636595794846124.output.0.npy │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────╯ -``` - -**示例2** - -```bash -# 查看某个dump数据块的数据信息 -# 默认会将数据中的tensor保存成 txt -Parse >>> pt -n ./parse_data/dump_convert/Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.output.0.npy -...... -# 打印统计信息 -[Shape: (1, 16, 56, 56, 16)] [Dtype: float16] [Max: 452.0] [Min: -408.5] [Mean: -3.809] -Path: ./parse_data/dump_convert/Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.input.0.npy -TextFile:./parse_data/dump_convert/Add.fp32_vars_add_1fp32_vars_Relu_6.24.5.1636595794631347.input.0.npy.txt -``` - -## dump.json文件中指定API的dump数据信息查看（暂不支持） - -输入以下命令，解析并输出dump.json文件中指定API的统计信息。 - -```bash -pk -f pkl_path -n api_name -``` - -| 参数名称 | 说明 | 是否必选 | -| -------- | ----------------------- | -------- | -| -f | 指定dump.json文件路径。 | 是 | -| -n | 指定API名称。 | 是 | - -- 输出结果：打印统计信息（shape, dtype, max和min mean）。 -- 若pkl文件中存在相应的堆栈信息，则会打印堆栈信息。 - -**示例** - -```bash -# 传入pkl文件及api名称 -Parse >>> pk -f ./torch_dump/xxx/rank0/dump.json -n Functional_conv2d_0_forward -...... -# 打印统计信息及堆栈（pkl文件不包含堆栈则不会打印堆栈） - -Statistic Info: - [Functional_conv2d_0_forward_input.0][dtype: torch.float32][shape: [2, 1, 2, 2]][max: 1.576936960220337][min: -0.9757485389709473][mean: 0.4961632490158081] - [Functional_conv2d_0_forward_input.1][dtype: torch.float32][shape: [2, 1, 2, 2]][max: 0.20064473152160645][min: -0.47102075815200806][mean: -0.20796933770179749] - [Functional_conv2d_0_forward_input.2][dtype: torch.float32][shape: [2]][max: 0.17380613088607788][min: -0.16853803396224976][mean: 0.0026340484619140625] - [Functional_conv2d_0_forward_output][dtype: torch.float32][shape: [2, 2, 1, 1]][max: 0.02364911139011383][min: -1.762906551361084][mean: -0.6710853576660156] -``` - -## API可选层级比对 - -输入以下命令, 进行统计级和像素级比对。 - -```bash -cn -m my_data*.npy -g gloden*.npy [-p num] [-al atol] [-rl rtol] -``` - -- 统计级比对：对tensor整体进行余弦值及相对误差的计算。 -- 像素级比对：对输入的两个npy文件进行逐元素比对。若两个tensor对应元素的相对误差或绝对误差大于**误差阈值**（-al和-rl配置）则被标记为错误数据。 - -| 参数名称 | 说明 | 是否必选 | -| -------- | ----------------------------------------------- | -------- | -| -m | 待比对数据。 | 是 | -| -g | 标杆数据。 | 是 | -| -p | 设置比对结束后打印错误元素的个数，默认值20。 | 否 | -| -al | 判定数据存在精度问题的绝对误差阈值，默认0.001。 | 否 | -| -rl | 判定数据存在精度问题的相对误差阈值，默认0.001。 | 否 | -| -s | 将npy文件保存成txt文件，用于查看，默认开启。 | 否 | - -输出结果： - -- 统计级比对结果。 -- 两个文件的统计信息（shape, dtype, max, min和mean）。 -- 错误数据打印表格。 - -**示例** - -```bash -# 对比两个tensor的数据 -Parse >>> cn -m Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy -g InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy -p 10 -s -al 0.002 -rl 0.005 - Error Item Table Top Item Table -┏━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━┓ ┏━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┓ -┃ Index ┃ Left ┃ Right ┃ Diff ┃ ┃ Index ┃ Left ┃ Right ┃ Diff ┃ -┡━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━┩ ┡━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━┩ -│ 155 │ 0.024600908 │ 0.022271132 │ 0.002329776 │ │ 0 │ -0.9206961 │ -0.9222216 │ 0.0015255213 │ -│ 247 │ 0.015752593 │ 0.017937578 │ 0.0021849852 │ │ 1 │ -0.6416973 │ -0.64051837 │ 0.0011789203 │ -│ 282 │ -0.0101207765 │ -0.007852031 │ 0.0022687456 │ │ 2 │ -0.35383835 │ -0.35433492 │ 0.0004965663 │ -│ 292 │ 0.019581757 │ 0.02240482 │ 0.0028230622 │ │ 3 │ -0.18851271 │ -0.18883198 │ 0.00031927228 │ -│ 640 │ -0.06593232 │ -0.06874806 │ 0.0028157383 │ │ 4 │ -0.43508735 │ -0.43534422 │ 0.00025686622 │ -│ 1420 │ 0.09293677 │ 0.09586689 │ 0.0029301196 │ │ 5 │ 1.4447614 │ 1.4466647 │ 0.0019032955 │ -│ 1462 │ -0.085207745 │ -0.088047795 │ 0.0028400496 │ │ 6 │ -0.3455438 │ -0.3444429 │ 0.0011008978 │ -│ 1891 │ -0.03433288 │ -0.036525503 │ 0.002192624 │ │ 7 │ -0.6560242 │ -0.6564579 │ 0.0004336834 │ -│ 2033 │ 0.06828873 │ 0.07139922 │ 0.0031104907 │ │ 8 │ -2.6964858 │ -2.6975214 │ 0.0010356903 │ -│ 2246 │ -0.06376442 │ -0.06121233 │ 0.002552092 │ │ 9 │ -0.73746175 │ -0.73650354 │ 0.00095820427 │ -└───────┴───────────────┴──────────────┴──────────────┘ └───────┴─────────────┴─────────────┴───────────────┘ -╭───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│ Left: | -│ |- NpyFile: ./dump/temp/decode/Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy | -│ |- TxtFile: ./dump/temp/decode/Add.InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.323.1619494134703053.output.0.npy.txt | -│ |- NpySpec: [Shape: (32, 8, 8, 320)] [Dtype: float32] [Max: 5.846897] [Min: -8.368301] [Mean: -0.72565556] | -│ DstFile: │ -│ |- NpyFile: ./dump/cpu/InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy | -│ |- TxtFile: ./dump/cpu/InceptionV3_InceptionV3_Mixed_7a_Branch_0_add_3.0.1619492699305998.npy.txt | -│ |- NpySpec: [Shape: (32, 8, 8, 320)] [Dtype: float32] [Max: 5.8425903] [Min: -8.374472] [Mean: -0.7256237] │ -│ NumCnt: 655360 │ -│ AllClose: False │ -│ CosSim: 0.99999493 │ -│ ErrorPer: 0.023504638671875 (rl= 0.005, al= 0.002) │ -╰───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -``` - diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md deleted file mode 100644 index ffff1f44f..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_compare.md +++ /dev/null @@ -1,243 +0,0 @@ -# **精度比对工具** - -## 总体说明 - -- 本节主要介绍通过命令行方式和比对函数方式进行CPU或GPU与NPU的精度数据比对，执行精度比对操作前需要先完成CPU或GPU与NPU的精度数据dump，参见《[精度数据采集](./dump.md)》。 - -- 训练精度一体化工具msprobe，使用子命令compare进行比对，可支持单卡和多卡场景的精度数据比对。 - -- 比对函数均通过单独创建精度比对脚本执行，可支持单卡和多卡场景的精度数据比对。 - -- 工具性能：比对数据量较小时（参考值单份文件小于10GB），参考比对速度0.1GB/s；比对数据量较大时，参考比对速度0.3GB/s。 推荐环境配置：独占环境，CPU核心数192，固态硬盘（IO速度参考：固态硬盘 > 500MB/s，机械硬盘60 ~ 170MB/s）。 - - 用户环境性能弱于标准约束或非独占使用的比对速度酌情向下浮动。比对速度的计算方式：两份比对文件大小/比对耗时。 - -## 约束 - -- NPU自研API，在CPU或GPU若没有对应的API，该API的dump数据不比对。 -- NPU与CPU或GPU的计算结果误差可能会随着模型的执行不断累积，最终会出现同一个API因为输入的数据差异较大而无法比对的情况。 -- CPU或GPU与NPU中两个相同的API会因为调用次数不同导致无法比对或比对到错误的API，不影响整体运行，该API忽略。 - -## 命令行方式比对 - -将CPU或GPU与NPU的dump文件进行比对，支持单卡和多卡，可同时比对多卡的dump数据。多机场景需要每个设备单独执行比对操作。 - -请先参见《[精度数据采集](./dump.md)》完成CPU或GPU与NPU的精度数据dump。 - -### 操作步骤 - -1. 创建比对文件，文件内容及示例请参见“**比对文件**”。 - -2. 执行如下示例命令进行比对： - - ```shell - msprobe -f pytorch compare -i ./compare.json -o ./output -s - ``` - - **完整参数说明** - - | 参数名 | 说明 | 是否必选 | - |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i或--input_path | 指定比对文件路径。比对文件内容及示例请参见“**比对文件**”。 | 是 | - | -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s或--stack_mode | 配置stack_mode的开关。仅当**比对文件**配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - | -c或--compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | - | -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | - -3. 查看比对结果，请参见“**比对结果分析**”。 - -### 比对文件 - -以在当前目录创建./compare.json为例。 - -- 单卡场景示例 - - ```json - { - "npu_path": "./npu_dump/dump.json", - "bench_path": "./bench_dump/dump.json", - "stack_path": "./npu_dump/stack.json", - "is_print_compare_log": true - } - ``` - -- 多卡场景示例 - - ```json - { - "npu_path": "./npu_dump/step0", - "bench_path": "./bench_dump/step0", - "is_print_compare_log": true - } - ``` - -**参数说明** - -| 参数名 | 说明 | 是否必选 | -| -------------------- | ------------------------------------------------------------ | ------------------ | -| npu_path | 配置NPU环境下的dump.json文件（单卡场景）或真实数据目录（多卡场景）。数据类型：str。 | 是 | -| bench_path | 配置CPU、GPU或NPU环境下的dump.json文件（单卡场景）或真实数据目录（多卡场景）。数据类型：str。 | 是 | -| stack_path | 配置NPU dump目录下的stack.json文件。数据类型：str。 | 单卡必选，多卡不选 | -| is_print_compare_log | 配置是否开启日志打屏。可取值True或False。数据类型：bool | 否 | - -## 比对函数方式比对 - -### compare_distributed - -**功能说明** - -将CPU或GPU与NPU的dump文件进行比对，支持单卡和多卡，可同时比对多卡的dump数据。多机场景需要每个设备单独执行比对操作。可自动检索和匹配对应卡和进程所dump的数据文件，再调用compare进行比对。单机单卡时与compare函数二选一。 - -**函数原型** - -```Python -compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs) -``` - -**参数说明** - -| 参数名 | 说明 | 是否必选 | -| -------------- | ------------------------------------------------------------ | -------- | -| npu_dump_dir | 配置NPU环境下的dump目录。dump数据目录须指定到step级。参数示例：'./npu_dump/step0'。数据类型：str。 | 是 | -| bench_dump_dir | 配置CPU、GPU或NPU环境下的dump目录。参数示例：'./gpu_dump/step0'。数据类型：str。 | 是 | -| output_path | 配置比对结果文件存盘目录。需要预先创建output_path目录。参数示例：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_rank{npu_ID}-rank{cpu/gpu/npu_ID}_{timestamp}.xlsx`。数据类型：str。 | 是 | -| **kwargs | 支持compare的所有可选参数。 | 否 | - -**函数示例** - -创建比对脚本，例如compare_distributed.py，拷贝如下代码，具体参数请根据实际环境修改。 - -```Python -from msprobe.pytorch import * -compare_distributed('./npu_dump/step0', './gpu_dump/step0', './output') -``` - -dump数据目录须指定到step级。 - -### compare - -**功能说明** - -将CPU或GPU与NPU的dump文件进行比对，仅支持单机单卡。 - -**函数原型** - -```Python -compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False) -``` - -**参数说明** - -| 参数名 | 说明 | 是否必选 | -| ------------ | ------------------------------------------------------------ | -------- | -| input_param | 配置dump数据文件及目录。数据类型：dict。配置参数包括：
"npu_json_path"：指定NPU dump目录下的dump.json文件。参数示例："npu_json_path": "./npu_dump/dump.json"。必选。
"bench_json_path"：指定CPU、GPU或NPU dump目录下的dump.json文件。参数示例："bench_json_path": "./bench_dump/dump.json"。必选。
"stack_json_path"：指定NPU dump目录下的stack.json文件。参数示例："stack_json_path": "./npu_dump/stack.json"。可选。
"is_print_compare_log"：配置是否开启日志打屏。可取值True或False。可选。 | 是 | -| output_path | 配置比对结果文件存盘目录。参数示例：'./output'。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。数据类型：str。 | 是 | -| stack_mode | 配置stack_mode的开关。仅当配置"stack_json_path"需要开启。可取值True或False，参数示例：stack_mode=True，默认为False。数据类型：bool。 | 否 | -| auto_analyze | 自动精度分析，开启后工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。可取值True或False，参数示例：auto_analyze=False，默认为True。数据类型：bool。 | 否 | -| fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。可取值True或False，参数示例：fuzzy_match=True，默认为False。数据类型：bool。 | 否 | - -**函数示例** - -单机单卡场景下创建比对脚本，例如compare.py，拷贝如下代码，具体参数请根据实际环境修改。 - -```Python -from msprobe.pytorch import compare -input_param={ -"npu_json_path": "./npu_dump/dump.json", -"bench_json_path": "./bench_dump/dump.json", -"stack_json_path": "./npu_dump/stack.json", -"is_print_compare_log": True -} -compare(input_param, output_path="./output", stack_mode=True) -``` - -### 统计量比对 - -若使用**compare**或**compare_distributed**函数创建的比对脚本中，在[config.json](../../config/config.json)文件中配置"task": "statistics"方式dump时，可以进行统计量比对，此时比对dump.json文件中的统计信息，开启后的比对结果文件生成Max diff、Min diff、Mean diff和L2norm diff，表示NPU dump数据中API的输入或输出与标杆数据输入或输出的最大值、最小值、平均值以及L2范数的差。可以通过该值判断API是否存在精度问题：当某个API的输入和输出的Max diff、Min diff、Mean diff和L2norm diff均为0或无限趋于0，那么可以判断该API无精度问题，反之则可能存在精度问题。 - -**比对脚本示例** - -以compare.py为例。 - -```Python -from msprobe.pytorch import compare -input_param={ -"npu_json_path": "./npu_dump/dump.json", -"bench_json_path": "./bench_dump/dump.json", -"stack_json_path": "./npu_dump/stack.json", -"is_print_compare_log": True -} -compare(input_param, output_path="./output", stack_mode=True) -``` - -**比对结果** - -数据量比对同样生成`compare_result_{timestamp}.xlsx`和`advisor_{timestamp}.txt`文件。其中`advisor_{timestamp}.txt`主要对`compare_result_{timestamp}.xlsx`中可能存在精度问题（Result为Waring）的API提出定位建议；`compare_result_{timestamp}.xlsx`主要有如下两种情况： - -- "summary_mode": "statistics"时比对dump.json文件： - - ![compare_result_pkl](img/compare_result_pkl.png) - - 上图是对dump.json文件中NPU及标杆API的统计信息进行比对，判断可能存在精度问题的API，文件中记录NPU及标杆API的基本信息和统计信息，其中需要关注Result列，包含结果：Waring（NPU与标杆统计信息的比对中存在相对误差大于0.5，则需要重点检查该API）；为空（相对误差小于等于0.5，可以不需要重点关注，但不代表不存在精度问题）；Nan（表示统计信息数据没有匹配上）。 - -- "summary_mode": "md5"时比对dump.json文件： - - ![compare_result_pkl_md5.png](img/compare_result_pkl_md5.png.png) - - 上图是对dump.json文件中NPU及标杆API的MD5信息进行比对，判断API数据的完整性，文件中记录NPU及标杆API的基本信息和MD5信息，其中需要关注Result列，包含结果：Pass（表示NPU与标杆的MD5值一致，即API数据完整）；Different（表示NPU与标杆的MD5值不一致，即API数据不完全一致，可以通过NPU_Stack_Info列API调用栈查询该API的详细信息）；Nan（表示MD5信息数据没有匹配上）。 - -## 比对结果分析 - -PyTorch精度比对是以CPU或GPU的计算结果为标杆，通过计算精度评价指标判断API在运行时是否存在精度问题。 - -- `advisor_{timestamp}.txt`文件中给出了可能存在精度问题的API的专家建议，可直接打开查看。 - -- `compare_result_{timestamp}.xlsx`文件列出了所有执行精度比对的API详细信息和比对结果，如下示例： - - ![compare_result](https://gitee.com/cai-weiwei1989/att_ptdbg/raw/master/debug/accuracy_tools/ptdbg_ascend/doc/img/compare_result.png) - - 可以从该结果文件中进行“**判断计算精度达标情况**”、“**计算精度评价指标分析**”以及“**异常信息识别**”等分析动作。 - -### **判断计算精度达标情况** - -精度比对结果`compare_result_{timestamp}.xlsx`文件中只需要通过Accuracy Reached or Not来判断计算精度是否达标，判断标准如下： - -1. Cosine < 0.99 且 MaxAbsError > 0.001时，精度不达标，标记为“No”。 -2. Cosine < 0.9，精度不达标，标记为“No”。 -3. MaxAbsError > 1，精度不达标，标记为“No”。 -4. 其余情况下记为精度达标，标记为“Yes”。 - -### **计算精度评价指标分析** - -1. Cosine：通过计算两个向量的余弦值来判断其相似度，数值越接近于1说明计算出的两个张量越相似，实际可接受阈值为大于0.99。在计算中可能会存在nan，主要由于可能会出现其中一个向量为0。 - -2. MaxAbsErr：当最大绝对误差越接近0表示其计算的误差越小，实际可接受阈值为小于0.001。 - -3. MaxRelativeErr：当最大相对误差越接近0表示其计算的误差越小。 - - 当dump数据中存在0或Nan时，比对结果中最大相对误差则出现inf或Nan的情况，属于正常现象。 - -4. One Thousandth Err Ratio（双千分之一）、Five Thousandths Err Ratio（双千分之五）精度指标：是指NPU的Tensor中的元素逐个与对应的标杆数据对比，相对误差大于千分之一、千分之五的比例占总元素个数的比例小于千分之一、千分之五。该数据仅作为精度下降趋势的参考，并不参与计算精度是否通过的判定。 - -### **异常信息识别** - -精度比对结果`compare_result_{timestamp}.xlsx`文件中对于存在异常信息的API会进行高亮处理： - -- 红色可能出现的情况有： - - NPU max或NPU min信息中存在nan/inf - - Max diff存在大于1e+10的值 - - 统计数据中output的Max diff除以max(0.01, Bench max) > 0.5 - - 真实数据中One Thousandth Err Ratio的input > 0.9同时output < 0.6 -- 黄色可能出现的情况有： - - Max diff的input与output都大于1，同时output比input大一个数量级以上 - - 统计数据Max diff除以max(0.01, Bench max)的output > 0.1同时input < 0.01 - - 真实数据One Thousandth Err Ratio的input - output > 0.1 - - 真实数据Cosine的input - output > 0.1 - -### **Shape为[]时，统计量列说明** -当NPU Tensor Shape列为[]时，表示标量或0维张量，统计量列（NPU max、NPU min、NPU mean、NPU l2norm）展示相同的唯一元素。Bench同理。 - -# FAQ - -[FAQ](./FAQ.md) - diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md deleted file mode 100644 index 019451454..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_overview.md +++ /dev/null @@ -1,68 +0,0 @@ -# **精度比对工具** - -## 简介 - -在PyTorch训练网络，对同一模型或API调试过程中，遇到API相关的计算精度问题，定位时费时费力。 - -msprobe的精度比对工具，用来进行PyTorch整网API粒度的数据dump、精度比对和溢出检测，从而定位PyTorch训练场景下的精度问题。 - -**使用场景** - -主要的使用场景包括： - -- 同一模型，从CPU或GPU移植到NPU中存在精度下降问题，对比NPU芯片中的API计算数值与CPU或GPU芯片中的API计算数值，进行问题定位。 -- 同一模型，进行迭代（模型、框架版本升级或设备硬件升级）时存在的精度下降问题，对比相同模型在迭代前后版本的API计算数值，进行问题定位。 - -## 原理介绍 - -精度对比工具，通过在PyTorch模型中注册hook，跟踪计算图中API的前向传播与反向传播时的输入与输出，排查存在计算精度误差，进行问题的精准定位。 - -**精度比对流程** - -1. 当模型在CPU或GPU上进行正向和反向传播时，分别dump每一层的数值输入与输出。 - -2. 当模型在NPU中进行计算时，采用相同的方式dump下相应的数据。 - -3. 通过对比dump出的数值，计算余弦相似度和最大绝对误差的方式，定位和排查NPU API存在的计算精度问题。如下图所示。 - - 精度比对逻辑图 - - ![module_compare](img/module_compare.png) - -**API匹配条件** - -进行精度比对时，需要判断CPU或GPU的API与NPU的API是否相同可比对，须满足以下匹配条件： - -- 两个API的名称相同，API命名规则：`{api_type}.{api_name}.{api调用次数}.{正反向}.{输入输出}.index`，如：Functional.conv2d.1.backward.input.0。 -- 两个API的输入输出Tensor数量和各个Tensor的Shape相同。 - -通常满足以上两个条件，工具就认为是同一个API，成功进行API的匹配，后续进行相应的计算精度比对。 - -## 精度比对总体流程 - -1. 准备CPU或GPU训练工程。 - -2. 在环境下安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 - -3. 在训练脚本内添加msprobe工具dump接口PrecisionDebugger采集标杆数据。详见《[精度数据采集](./dump.md)》。 - -4. 执行训练dump数据。 - -5. 将CPU或GPU训练工程迁移为NPU训练工程。详见《[PyTorch模型迁移调优指南](https://www.hiascend.com/document/detail/zh/Pytorch/60RC1/ptmoddevg/trainingmigrguide/PT_LMTMOG_0003.html)》。 - -6. 在NPU环境下安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 - -7. 在NPU训练脚本内添加msprobe工具dump接口PrecisionDebugger采集标杆数据。详见《[精度数据采集](./dump.md)》。 - -8. NPU环境下执行训练dump数据。 - -9. 执行精度比对。 - - 1. 创建并配置精度比对脚本，例如compare.py。 - - 2. 执行CPU或GPU dump与NPU dump数据的精度比对。 - - 3. 比对结果分析。 - - 详见《[CPU或GPU与NPU精度数据比对](./ptdbg_ascend_compare.md)》。 - diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_quickstart.md b/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_quickstart.md deleted file mode 100644 index 1a581226f..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/doc/ptdbg_ascend_quickstart.md +++ /dev/null @@ -1,381 +0,0 @@ -# **精度比对工具** - -本文主要介绍msprobe的精度比对工具的快速入门和场景化示例。 - -本文介绍的操作需要安装msprobe工具，详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 - -本文介绍的操作主要是精度数据dump和精度比对，详细操作指导可参考《[精度数据采集](./dump.md)》和《[CPU或GPU与NPU精度数据比对](./ptdbg_ascend.md)》。 - -## 快速入门 - -### 单卡场景精度比对 - -**精度分析建议** - -PyTorch训练场景的精度问题分析建议参考以下思路进行精度比对和比对结果分析： - -1. 整网比对：dump整网数据并进行精度比对，初步定位异常范围。 - - 对于模型数据庞大（比如达到T级别）的场景，不推荐直接dump整网比对，整网dump可能导致磁盘不足，需要预留足够的存储空间或者分多次dump。 - -2. 缩小范围：根据Accuracy Reached or Not找出不符合精度标准的API。 - -3. 范围比对：对不符合精度标准的API重新dump详细信息。 - -4. 分析原因并优化：分析API精度不符合标准的原因并进行优化调整。 - -5. 整网比对：重新进行整网比对，判断优化后的API是否已符合精度标准以及是否出现新的精度问题。 - -6. 重复1~5步，直到不存在精度问题为止。 - -**精度分析示例** - -1. 修改dump配置文件config.json。 - - ```json - { - "task": "tensor", - "dump_path": "./npu_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - - "tensor": { - "scope": [], - "list": [], - "data_mode": ["all"], - "summary_mode": "statistics" - } - } - ``` - -2. 在训练脚本内添加msprobe工具，dump整网数据。 - - 分别dump CPU或GPU以及NPU数据，在PyTorch训练脚本插入dump接口，示例代码如下（下面以NPU为例，CPU或GPU dump基本相同）： - - ```python - from msprobe.pytorch import PrecisionDebugger - debugger = PrecisionDebugger(config_path="./config.json", dump_path="./npu_dump") - # 请勿将以上初始化流程插入到循环代码中 - - # 模型初始化 - # 下面代码也可以用PrecisionDebugger.start()和PrecisionDebugger.stop() - debugger.start() - - # 需要dump的代码片段1 - - debugger.stop() - debugger.start() - - # 需要dump的代码片段2 - - debugger.stop() - debugger.step() - ``` - -3. 比对整网数据。 - - 第1步中的NPU dump数据目录为npu_dump，假设GPU dump数据目录为gpu_dump；dump将生成dump.json、stack.json、construct.json文件以及dump数据目录。 - - 创建并配置精度比对脚本，以创建compare.py为例，示例代码如下： - - ```python - from msprobe.pytorch import compare - input_param={ - "npu_json_path": "./npu_dump/dump.json", - "bench_json_path": "./bench_dump/dump.json", - "stack_json_path": "./npu_dump/stack.json", - "is_print_compare_log": True - } - compare(input_param, output_path="./output", stack_mode=True) - ``` - - 执行比对： - - ```bash -python3 compare.py - ``` - - 在output目录下生成结果文件，包括：`compare_result_{timestamp}.xlsx`和`advisor_{timestamp}.txt` - -4. 找出存在问题的API。 - - 1. 根据`advisor_{timestamp}.txt`或打屏信息的提示，可找到存在精度问题的算子（Suspect Nodes）和专家建议（Expert Advice)。 - - ![auto_analyze_log](img/auto_analyze_log.png) - - 2. 根据第2步结果文件`compare_result_{timestamp}.xlsx`中的Accuracy Reached or No字段显示为NO的API，针对该API执行后续比对操作，分析该API存在的精度问题。 - -5. （可选）重新比对。 - - 根据第3步的dump数据重新配置compare.py并执行比对，可以对单API模型进行问题复现。 - -**注意**：部分API存在调用嵌套关系，比如functional.batch_norm实际调用torch.batch_norm，该场景会影响kernel init初始化多次，导致功能异常。 - -### 溢出检测场景 - -溢出检测是针对NPU的PyTorch API，检测是否存在溢出的情况。当前仅支持识别aicore浮点溢出。 - -溢出检测原理：针对溢出阶段，开启acl dump模式，重新对溢出阶段执行，落盘数据。 - -建议按照如下步骤操作： - -1. 修改dump配置文件config.json。 - - ```json - { - "task": "overflow_check", - "dump_path": "./npu_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - - "overflow_check": { - "overflow_nums": 3 - } - } - ``` - -2. 在NPU训练脚本内添加msprobe工具，执行溢出检测dump。 - - ```python - from msprobe.pytorch import PrecisionDebugger - debugger = PrecisionDebugger(config_path="./config.json", dump_path="./npu_dump") - # 请勿将以上初始化流程插入到循环代码中 - - # 模型初始化 - # 下面代码也可以用PrecisionDebugger.start()和PrecisionDebugger.stop() - debugger.start() - - # 需要dump的代码片段1 - - debugger.stop() - debugger.start() - - # 需要dump的代码片段2 - - debugger.stop() - debugger.step() - ``` - - 多卡使用时各卡单独计算溢出次数。 - -3. NPU环境下执行训练dump溢出数据。 - - 针对输入正常但输出存在溢出的API，会在训练执行目录下将溢出的API信息dump并保存为`dump.json`通过《[溢出解析工具](./run_overflow_check.md)》对json文件进行解析，输出溢出API为正常溢出还是非正常溢出，从而帮助用户快速判断。 - - 溢出解析工具执行命令如下： - - ```bash - msprobe -f pytorch run_overflow_check -api_info ./dump.json - ``` - - 反向过程溢出的API暂不支持精度预检功能。 - - -当重复执行溢出检测dump操作时，需要删除上一次dump目录下的溢出检测dump数据，否则将因重名而报错。 - -**注意事项** - -* （暂不支持）level为L2场景下，会增加npu的内存消耗，请谨慎开启。 -* （暂不支持）l部分API存在调用嵌套关系，比如functional.batch_norm实际调用torch.batch_norm，该场景会影响acl init初始化多次，导致level为L2功能异常。 -* 混合精度动态loss scale场景下，正常训练会有"Gradient overflow. SKipping step"日志，添加溢出检测后日志消失，可以通过设置环境变量export OVERFLOW_DEBUG_MODE_ENABLE=1，并将register_hook位置调整amp.initialize之前解决。此功能需要cann包配套支持，不支持版本执行报错EZ3003。 - -## 场景化示例 - -### 多卡场景精度比对 - -精度工具支持多卡场景的精度比对，多卡场景的dump步骤与单卡场景完全一致，请参见“**单卡场景精度比对**”章节，不同的是多卡数据精度比对时需要使用“compare_distributed”函数进行比对。 - -如下示例： - -说明：多机多卡场景需要每个节点单独执行比对操作。 - -假设NPU dump 数据目录为npu_dump，GPU dump数据目录为gpu_dump。 - -1. 创建比对脚本，例如compare_distributed.py，拷贝如下代码。 - - ```python - from msprobe.pytorch import * - compare_distributed('./npu_dump/step0', './gpu_dump/step0', './output') - ``` - - dump数据目录须指定到step级。 - -2. 执行比对： - - ```bash - python3 compare_distributed.py - ``` - -两次运行须用相同数量的卡，传入`compare_distributed`的两个文件夹下须有相同个数的rank文件夹，且不包含其他无关文件，否则将无法比对。 - -**多卡set_dump_path注意事项** - -多卡一般为多进程，须保证每个进程都正确调用PrecisionDebugger，或把PrecisionDebugger插入到import语句后，如： - -```python -from msprobe.pytorch import PrecisionDebugger -debugger = PrecisionDebugger(config_path="./config.json", dump_path="./npu_dump") -``` - -如此可保证set_dump_path在每个进程都被调用。 - -### NPU vs NPU精度比对 - -对于NPU vs NPU场景，是针对同一模型，进行迭代（模型、API版本升级或设备硬件升级）时存在的精度下降问题，对比相同模型在迭代前后版本的API计算数值，进行问题定位。 - -一般情况下迭代涉及NPU自定义算子，因此，可以仅dump NPU自定义算子进行比对。比对精度问题分析请参见“**单卡场景精度比对**”章节。 - -工具当前支持dump NPU自定义算子如下： - -| 序号 | NPU自定义算子 | -| :--- | ----------------------------------------------- | -| 1 | torch_npu.one_ | -| 2 | torch_npu.npu_sort_v2 | -| 3 | torch_npu.npu_transpose | -| 4 | torch_npu.npu_broadcast | -| 5 | torch_npu.npu_dtype_cast | -| 6 | torch_npu.empty_with_format | -| 7 | torch_npu.npu_one_hot | -| 8 | torch_npu.npu_stride_add | -| 9 | torch_npu.npu_ps_roi_pooling | -| 10 | torch_npu.npu_roi_align | -| 11 | torch_npu.npu_nms_v4 | -| 12 | torch_npu.npu_iou | -| 13 | torch_npu.npu_nms_with_mask | -| 14 | torch_npu.npu_pad | -| 15 | torch_npu.npu_bounding_box_encode | -| 16 | torch_npu.npu_bounding_box_decode | -| 17 | torch_npu.npu_batch_nms | -| 18 | torch_npu.npu_slice | -| 19 | torch_npu._npu_dropout | -| 20 | torch_npu.npu_indexing | -| 21 | torch_npu.npu_ifmr | -| 22 | torch_npu.npu_max | -| 23 | torch_npu.npu_scatter | -| 24 | torch_npu.npu_layer_norm_eval | -| 25 | torch_npu.npu_alloc_float_status | -| 26 | torch_npu.npu_confusion_transpose | -| 27 | torch_npu.npu_bmmV2 | -| 28 | torch_npu.fast_gelu | -| 29 | torch_npu.npu_sub_sample | -| 30 | torch_npu.npu_deformable_conv2d | -| 31 | torch_npu.npu_mish | -| 32 | torch_npu.npu_anchor_response_flags | -| 33 | torch_npu.npu_yolo_boxes_encode | -| 34 | torch_npu.npu_grid_assign_positive | -| 35 | torch_npu.npu_normalize_batch | -| 36 | torch_npu.npu_masked_fill_range | -| 37 | torch_npu.npu_linear | -| 38 | torch_npu.npu_bert_apply_adam | -| 39 | torch_npu.npu_giou | -| 40 | torch_npu.npu_ciou | -| 41 | torch_npu.npu_diou | -| 42 | torch_npu.npu_sign_bits_pack | -| 43 | torch_npu.npu_sign_bits_unpack | -| 44 | torch_npu.npu_flash_attention | -| 45 | torch_npu.npu_scaled_masked_softmax | -| 46 | torch_npu.npu_rotary_mul | -| 47 | torch_npu.npu_roi_align | -| 48 | torch_npu.npu_roi_alignbk | -| 49 | torch_npu.npu_ptiou | -| 50 | torch_npu.npu_fusion_attention | -| 51 | torch_npu.npu_dropout_with_add_softmax | -| 52 | torch_npu.npu_random_choice_with_mask | -| 53 | torch_npu.npu_rotated_iou | -| 54 | torch_npu.npu_conv2d | -| 55 | torch_npu.npu_conv3d | -| 56 | torch_npu.npu_softmax_cross_entropy_with_logits | -| 57 | torch_npu.npu_all_gather_base_mm | -| 58 | torch_npu.npu_swiglu | -| 59 | torch_npu.npu_rms_norm | -| 60 | torch_npu.npu_mm_reduce_scatter_base | -| 61 | torch_npu.npu_mm_all_reduce_base | -| 62 | torch_npu.npu_conv_transpose2d | -| 63 | torch_npu.npu_convolution | -| 64 | torch_npu.npu_convolution_transpose | -| 65 | torch_npu.npu_min | -| 66 | torch_npu.npu_nms_rotated | -| 67 | torch_npu.npu_reshape | -| 68 | torch_npu.npu_rotated_box_decode | -| 69 | torch_npu.npu_rotated_box_encode | -| 70 | torch_npu.npu_rotated_overlaps | -| 71 | torch_npu.npu_silu | -| 72 | torch_npu.npu_fused_attention_score | -| 73 | torch_npu.npu_multi_head_attention | -| 74 | torch_npu.npu_gru | -| 75 | torch_npu.npu_incre_flash_attention | -| 76 | torch_npu.npu_prompt_flash_attention | -| 77 | torch_npu.npu_lstm | -| 78 | torch_npu.npu_apply_adam | - -### 通信API的数据dump - -通信类API数据可以使用全量dump方式获取，若只dump通信类API数据，可以使用如下示例： - -1. 修改dump配置文件config.json。 - - ```json - { - "task": "tensor", - "dump_path": "./npu_dump", - "rank": [], - "step": [], - "level": "L1", - "seed": 1234, - "is_deterministic": false, - - "tensor": { - "scope": [], - "list": ["distributed"], - "data_mode": ["all"], - "summary_mode": "statistics" - } - } - ``` - -2. 在训练脚本内添加msprobe工具，dump整网数据。 - - ```python - from msprobe.pytorch import PrecisionDebugger - debugger = PrecisionDebugger(config_path="./config.json", dump_path="./npu_dump") - # 请勿将以上初始化流程插入到循环代码中 - - # 模型初始化 - # 下面代码也可以用PrecisionDebugger.start()和PrecisionDebugger.stop() - debugger.start() - - # 需要dump的代码片段1 - - debugger.stop() - debugger.start() - - # 需要dump的代码片段2 - - debugger.stop() - debugger.step() - ``` - -通信类API支持列表： - -| 序号 | Distributed | -| :--- | -------------------- | -| 1 | send | -| 2 | recv | -| 3 | broadcast | -| 4 | all_reduce | -| 5 | reduce | -| 6 | all_gather | -| 7 | gather | -| 8 | isend | -| 9 | irecv | -| 10 | scatter | -| 11 | reduce_scatter | -| 12 | _reduce_scatter_base | -| 13 | _all_gather_base | - diff --git a/debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md b/debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md deleted file mode 100644 index f56165b25..000000000 --- a/debug/accuracy_tools/msprobe/pytorch/doc/run_overflow_check.md +++ /dev/null @@ -1,25 +0,0 @@ -# **溢出解析工具** - -针对训练过程中的溢出检测场景（当《[精度数据采集](./dump.md)》开启溢出检测dump时），对于输入正常但输出存在溢出的API，会在训练执行目录下将溢出的API信息按照前向和反向分类，dump并保存为`dump.json`，前向过程溢出的API可通过该工具对`dump.json`进行解析，输出溢出API为正常溢出还是非正常溢出，从而帮助用户快速判断。 - -工具支持PyTorch版本：1.11/2.0/2.1/2.2。 - -操作步骤如下： - -1. 安装工具。 - - 详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 - -2. 执行溢出API解析操作。 - - ```bash - msprobe -f pytorch run_overflow_check -api_info ./dump.json - ``` - -| 参数名称 | 说明 | 是否必选 | -| -------------------------- | -------------------------------------------------- | -------- | -| -api_info或--api_info_file | 指定API信息文件dump.json。 | 是 | -| -j或--jit_compile | 开启jit编译。 | 否 | -| -d或--device | 指定Device ID，选择UT代码运行所在的卡，默认值为0。 | 否 | - -反向过程溢出的API暂不支持该功能。 diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/\345\234\250\347\272\277\347\262\276\345\272\246\346\257\224\345\257\271.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/\345\234\250\347\272\277\347\262\276\345\272\246\346\257\224\345\257\271.md" deleted file mode 100644 index 05bebaf0a..000000000 --- "a/debug/accuracy_tools/msprobe/pytorch/doc/\345\234\250\347\272\277\347\262\276\345\272\246\346\257\224\345\257\271.md" +++ /dev/null @@ -1,90 +0,0 @@ -# **PyTorch NPU在线精度比对工具使用指南** - -PyTorch NPU在线精度比对是ptdbg_ascend工具实现在PyTorch训练过程中直接完成精度比对并输出比对结果的功能。 - -在线精度比对实现的是NPU与CPU之间的精度比对。 - -## PyTorch NPU在线精度比对总体流程 - -1. 准备NPU训练工程。 - -2. 在NPU环境下安装ptdbg_ascend工具，参见《[PyTorch精度工具](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/ptdbg_ascend/README.md)》。 - -3. 在训练脚本内插入ptdbg_ascend工具在线精度比对接口。 - -4. 执行训练并获取在线精度比对NPU和CPU分别执行后的精度比对结果。 - -5. 比对结果分析。 - -## PyTorch NPU在线精度比对 -### 总体说明 -- 本节主要介绍NPU精度比对所需要的函数以及示例。 -- 在线精度比对工具通过截获PyTorch框架中部分Aten Ir及其输入输出，并将输入数据转到CPU执行，最后将NPU和CPU的执行结果进行精度比对得到比对结果。 - -### 约束 - -- Pytorch 只支持2.0及其以上版本。 -- 只支持Aten Ir级在线精度比对，所有Aten Ir可以通过dir(torch.ops.aten)查看，其中部分IR不支持在线比对：Aten Ir无对应CPU实现、NPU和CPU同AtenIR实现逻辑不一致，导致同输入不同输出。 -- 正反向不支持同时在线精度比对，不支持跨step在线精度比对。 - - -### 场景示例 -1. 在NPU训练脚本中添加在线精度比对接口，示例如下： - - ```python - from msprobe.pytorch.common.utils import seed_all - from msprobe.pytorch.online_dispatch import PtdbgDispatch - - # 在main函数开始前固定随机数 - seed_all() - - - ... - - # 在需要调试精度的正向或反向代码前设置 - # 正向示例 - with PtdbgDispatch(dump_mode="auto", dump_path="/home/dump"): - output = model_cpu(inputs) - # 反向示例 - with PtdbgDispatch(dump_mode="auto", dump_path="/home/dump"): - loss.backward() - ``` - -2. 执行训练。 - -3. 找出精度不达标的Aten IR。 - - 执行过程中会打屏Failed，Failed在比对结果csv中的Accuracy Reached or Not列标记为No，并在Dump目录下存盘精度不达标Aten IR的输入输出。 - ![图片说明](http://image.huawei.com/tiny-lts/v1/images/d83d564e337e80c7cfb557ca3600d0d4_1689x178.png@900-0-90-f.png) - -### 计算精度评价指标 - -1. Cosine < 0.99 且 MaxAbsError > 0.001时，精度不达标； -2. Cosine < 0.9，精度不达标； -3. MaxAbsError > 1，精度不达标。 - -### 在线精度比对参数设置说明 - -| 参数名称 | 说明 | 是否必选 | -| -------- |-------------------------------------------------------------------------------------------------| -------- | -| dump_mode| dump模式，可取值"all"、"list"、"auto"、"OFF"，默认值为OFF（表示不Dump数据）。 | 否 | -| api_list | dump范围，dump_mode="list"时设置，需要Dump Aten Ir API名称，默认为None，Aten Ir API名称可以通过dir(torch.ops.aten)查看。 | 否 | -| dump_path| dump文件生成的路径。 | 是 | -| tag | 传入tag字符串，成为dump文件夹名一部分，默认为None。 | 否 | -| process_num | 多进程并发数，默认为0。 | 否 | -| debug | debug信息打印，默认为False。 | 否 | -### dump数据存盘说明 -dump数据存盘目录名格式：`msprobe_tag_rankid_{timestamp}`。 - -子目录下包含1个比对结果csv文件、cpu和npudump数据目录，npu目录下包含Aten IR在NPU上的输入输出的dump数据，由于CPU的输入是直接使用NPU的输入执行，因此cpu目录下只包含执行输出的dump数据。 - -```bash -msprobe_rank4_20230911170521 -├── compare_result_rank4_20230911170521.csv -├── cpu -│   ├── native_batch_norm_backward_10_output.0.npy -│ ............ -└── npu - ├── native_batch_norm_backward_10_input.0.npy - ............ -``` diff --git "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\234\272\346\231\257\351\252\214\350\257\201\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" "b/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\234\272\346\231\257\351\252\214\350\257\201\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" deleted file mode 100644 index 8862c5b76..000000000 --- "a/debug/accuracy_tools/msprobe/pytorch/doc/\346\227\240\346\240\207\346\235\206\345\267\245\345\205\267\345\234\272\346\231\257\351\252\214\350\257\201\345\222\214\346\200\247\350\203\275\345\237\272\347\272\277\346\212\245\345\221\212.md" +++ /dev/null @@ -1,151 +0,0 @@ -# 无标杆工具场景验证和性能基线报告 - -## 环境信息 - -NPU：Atlas A2 训练系列产品 - -CPU： - -![输入图片说明](img/cpu_info.png) - -Torch：2.1.0 - -CANN：8.0.T5 - -除上述环境信息影响性能外，API的数量、种类以及Shape都会对性能产生影响，因此本次选取不同场景网络和不同算子进行测试。 - -## 模型信息和性能基线 - -大模型在使用msprobe工具dump数据时，建议先简化模型层数，减少dump数据量。 - -以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 - - - -### LLaMA2-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink -其中，softmax算子为FLOAT32，输入输出均为2G大小，为模型最大显存开销的API。 - -在该模型下、对无标杆工具处理模式、插装范围、扰动方式组合下性能和显存基线进行覆盖。 - -性能基线报告 -其中耗时为训练10步，去除第一步耗时所得的平均每步耗时。 - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.24 | 13.69 | 1 | 1 | 混精模式基线 | -| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1 | softmax本身为高精度，跳过 | -| check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | -| check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | -| check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | -| check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | -| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | -| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.38 | 1 | | -| check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | -| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | -| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | -| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | -| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | -| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | -| check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | -| check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | -| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | -| fix | 前 | ["softmax"] | to_cpu | 9.70 | 16.67 | 40.42 | 1.22 | 不支持整网、不支持反向 | -| fix | 前 | ["softmax"] | improve_precision | 0.26 | 14.67 | 1.08 | 1.07 | 不支持整网、不支持反向 | -| 预热 | 前 | [] | improve_precision | 155.07 | 24.79 | 646.13 | 1.81 | 低精度模型基线、只测预热的迭代 | -| 预热 | 反 | [] | improve_precision | 72.29 | 22.01 | 301.21 | 1.61 | 低精度模型基线、只测预热的迭代，grad_output为高精度的算子跳过 | - -### Aquila2-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.17 | 13.66 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.57 | 14.24 | 9.24 | 1.04 | | -| check | 反 | [] | add_noise | 21.05 | 14.19 | 123.82 | 1.04 | | -| fix | 前 | [] | improve_precision | 0.95 | 15.55 | 5.59 | 1.14 | | - -### Baichuan2-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.26 | 12.12 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.02 | 12.27 | 3.92 | 1.01 | | -| check | 反 | [] | add_noise | 11.15 | 12.67 | 42.88 | 1.05 | | -| fix | 前 | [] | improve_precision | 0.95 | 12.82 | 3.65 | 1.06 | | - -### Bloom-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.14 | 9.51 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.64 | 11.58 | 11.71 | 1.22 | | -| check | 反 | [] | add_noise | 17.15 | 9.51 | 122.5 | 1 | | -| fix | 前 | [] | improve_precision | 0.87 | 10.62 | 6.21 | 1.12 | | - -### Interlm-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.13 | 10.76 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.19 | 11.68 | 9.15 | 1.09 | | -| check | 反 | [] | add_noise | 11.69 | 10.89 | 89.92 | 1.01 | | -| fix | 前 | [] | improve_precision | 0.75 | 11.68 | 5.77 | 1.09 | | - -### Qwen-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.28 | 18.41 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 2.34 | 23.18 | 8.36 | 1.26 | | -| check | 反 | [] | add_noise | 22.07 | 19.47 | 78.82 | 1.06 | | -| fix | 前 | [] | improve_precision | 1.31 | 21.11 | 4.68 | 1.15 | | - -### Gemma-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.15 | 11.06 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.49 | 13.17 | 9.93 | 1.19 | | -| check | 反 | [] | add_noise | 16.69 | 11.06 | 111.27 | 1 | | -| fix | 前 | [] | improve_precision | 0.87 | 12.25 | 5.8 | 1.11 | | - -### ResNet50-Cifar -1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch。 -主要算子为conv2d，每个step有51个, 因此对conv2d进行检测。 -CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step信息和反向信息）。 - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.09 | 7.63 | 1 | 1 | 基线 | -| check | 前 | ["conv2d"] | improve_precision | 0.889 | 7.94 | 9.81 | 1.04 | | -| fix | 前 | ["conv2d"] | improve_precision | 0.328 | 7.47 | 3.64 | 0.91 | | -| fix | 前 | ["conv2d"] | to_cpu | 12.23 | 7.47 | 135.88 | 0.91 | | - -### OpenSora1.0 - -4卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch - -每张卡每个step中linear算子个数为257个，FA算子个数为83（FA算子反向无效）。 - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.99 | 17.61 | 1 | 1 | 混精模式基线 | -| check | 前 | ["linear","npu_fusion_attention"] | improve_precision | 3.88 | 17.61 | 3.92 | 1 | | -| check | 前 | ["linear","npu_fusion_attention"] | add_noise | 3.46 | 17.61 | 3.49 | 1 | | -| check | 反 | ["linear"] | improve_precision | 12.61 | 17.61 | 12.74 | 1 | | -| check | 反 | ["linear"] | add_noise | 9.8 | 17.61 | 9.90 | 1 | | -| fix | 前 | ["linear"] | to_cpu | 18.83 | 17.61 | 19.02 | 1 | | -| fix | 前 | ["linear"] | improve_precision | 2.83 | 17.61 | 2.86 | 1 | | -- Gitee From 84ed34f750cabb35ec9af80cf0714a0895f65a42 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 18:45:01 +0800 Subject: [PATCH 504/791] renew_manual --- .../docs/09.accuracy_checker_MindSpore.md | 70 ------------------- 1 file changed, 70 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md diff --git a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md deleted file mode 100644 index e7257c1b4..000000000 --- a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md +++ /dev/null @@ -1,70 +0,0 @@ -# MindSpore 动态图场景的离线精度预检 - -## 1 简介 - -**MindSpore 动态图精度预检**\*1通过扫描昇腾 NPU 上用户训练 MindSpore 模型中的所有 Mint API，输出精度情况的诊断和分析。工具以模型中所有 Mint API 前反向的 dump 结果为输入，构造相应的 API 单元测试，将 NPU 输出与标杆（CPU 高精度）比对，计算对应的精度指标，从而找出 NPU 中存在精度问题的 Mint API。本工具支持随机生成模式和真实数据模式\*2。 - -**\***： - -1. 支持 Mindspore 版本：2.4； -2. 在预检 dump 时可以选择由工具构造随机数进行输入获得 dump 数据或选择获取真实输入数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 - -## 2 离线预检流程 - -操作流程如下： - -1. 在 NPU 和 GPU 环境下分别安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 -2. 在 NPU 训练脚本内添加 msprobe 工具 dump 接口 PrecisionDebugger，采集待预检数据。详见 [MindSpore 场景下的数据采集](./06.data_dump_MindSpore.md)章节，注意需要配置 level="L1"。 -3. 执行预检操作，查看预检结果文件，分析预检不达标的API。 - -## 3 离线预检操作指导 - -命令如下： -```bash -msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result -``` - -| 参数名称 | 说明 |参数类型 | 是否必选 | -| ---------------------------- | --------------------------------------|---------------------- | ---------------------------------- | -| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | str | 是 | -| -o或--out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | - -预检执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API Name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见 [4 预检结果](#4-预检结果)。 - -## 4 预检结果 - -精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件内容详情如下： - -`accuracy_checking_details_{timestamp}.csv` - -| 字段 | 含义 | -| ------------------- | ------------------------------------------------------------ | -| API Name | API 名称。 | -| Bench Dtype | 标杆数据的 API 数据类型。 | -| Tested Dtype | 被检验数据的 API 数据类型。 | -| Shape | API 的 Shape 信息。 | -| Cosine | 被检验数据与标杆数据的余弦相似度。 | -| MaxAbsErr | 被检验数据与标杆数据的最大绝对误差。 | -| MaxRelativeErr | 被检验数据与标杆数据的最大相对误差。 | -| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过。 | -| message | 提示信息。 | - -`accuracy_checking_result_{timestamp}.csv` - -| 字段 | 含义 | -| --------------------- | ----------------- | -| API Name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，error 为错误。 | -| Backward Test Success | 反向 API 是否通过测试，pass 为通过，error 为错误，如果是空白的话代表该 API 没有反向输出。 | -| Message | 提示信息。 | - -Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差判定结果决定的。具体规则详见\*1。 -需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error。 - -**\***： - -1. API预检指标： - - - API 预检指标是通过对`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差的数值进行判断，得出该API是否符合精度标准的参考指标。详细规则如下： - - - 余弦相似度大于0.99，并且最大绝对误差小于0.0001，标记“pass”，否则标记为“error”。 -- Gitee From 264b07d566da1fa23c4fce2eb284c836d42591bd Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 18:47:20 +0800 Subject: [PATCH 505/791] renew_manual --- .../docs/09.accuracy_checker_MindSpore.md | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md diff --git a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md new file mode 100644 index 000000000..e7257c1b4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md @@ -0,0 +1,70 @@ +# MindSpore 动态图场景的离线精度预检 + +## 1 简介 + +**MindSpore 动态图精度预检**\*1通过扫描昇腾 NPU 上用户训练 MindSpore 模型中的所有 Mint API，输出精度情况的诊断和分析。工具以模型中所有 Mint API 前反向的 dump 结果为输入，构造相应的 API 单元测试，将 NPU 输出与标杆（CPU 高精度）比对，计算对应的精度指标，从而找出 NPU 中存在精度问题的 Mint API。本工具支持随机生成模式和真实数据模式\*2。 + +**\***： + +1. 支持 Mindspore 版本：2.4； +2. 在预检 dump 时可以选择由工具构造随机数进行输入获得 dump 数据或选择获取真实输入数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + +## 2 离线预检流程 + +操作流程如下： + +1. 在 NPU 和 GPU 环境下分别安装 msprobe。详见[ msprobe 安装](./01.installation.md)章节。 +2. 在 NPU 训练脚本内添加 msprobe 工具 dump 接口 PrecisionDebugger，采集待预检数据。详见 [MindSpore 场景下的数据采集](./06.data_dump_MindSpore.md)章节，注意需要配置 level="L1"。 +3. 执行预检操作，查看预检结果文件，分析预检不达标的API。 + +## 3 离线预检操作指导 + +命令如下： +```bash +msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result +``` + +| 参数名称 | 说明 |参数类型 | 是否必选 | +| ---------------------------- | --------------------------------------|---------------------- | ---------------------------------- | +| -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | str | 是 | +| -o或--out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | + +预检执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API Name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见 [4 预检结果](#4-预检结果)。 + +## 4 预检结果 + +精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件内容详情如下： + +`accuracy_checking_details_{timestamp}.csv` + +| 字段 | 含义 | +| ------------------- | ------------------------------------------------------------ | +| API Name | API 名称。 | +| Bench Dtype | 标杆数据的 API 数据类型。 | +| Tested Dtype | 被检验数据的 API 数据类型。 | +| Shape | API 的 Shape 信息。 | +| Cosine | 被检验数据与标杆数据的余弦相似度。 | +| MaxAbsErr | 被检验数据与标杆数据的最大绝对误差。 | +| MaxRelativeErr | 被检验数据与标杆数据的最大相对误差。 | +| Status | API 预检通过状态，pass 表示通过测试，error 表示未通过。 | +| message | 提示信息。 | + +`accuracy_checking_result_{timestamp}.csv` + +| 字段 | 含义 | +| --------------------- | ----------------- | +| API Name | API 名称。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，error 为错误。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，error 为错误，如果是空白的话代表该 API 没有反向输出。 | +| Message | 提示信息。 | + +Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差判定结果决定的。具体规则详见\*1。 +需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error。 + +**\***： + +1. API预检指标： + + - API 预检指标是通过对`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差的数值进行判断，得出该API是否符合精度标准的参考指标。详细规则如下： + + - 余弦相似度大于0.99，并且最大绝对误差小于0.0001，标记“pass”，否则标记为“error”。 -- Gitee From c55e88207eda78e69d29e837f2d1dcce04708c1b Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 18:53:19 +0800 Subject: [PATCH 506/791] renew_manual --- debug/accuracy_tools/setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index b273f0f3d..01e399c46 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -19,13 +19,13 @@ __version__ = '1.0.3' INSTALL_REQUIRED = [ "wheel", "einops", - "numpy", - "pandas >= 1.3.5", + "numpy < 2.0", + "pandas >= 1.3.5, < 2.1", "pyyaml", "rich", "tqdm", "openpyxl", - "pyOpenSSL", + "pyopenssl", "twisted", "matplotlib" ] -- Gitee From 0d3d3ab34155302413e7a6e3c5c8b8122e1c72a7 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 21:49:28 +0800 Subject: [PATCH 507/791] renew_manual --- .../data_processor/pytorch_processor.py | 45 +++---- .../docs/07.accuracy_checker_PyTorch.md | 10 +- .../08.accuracy_checker_online_Pytorch.md | 22 +++- .../msprobe/mindspore/dump/jit_dump.py | 8 +- .../compare/api_precision_compare.py | 8 +- .../api_accuracy_checker/compare/compare.py | 2 +- .../tensor_transport_layer/device_dispatch.py | 6 +- .../data_processor/test_pytorch_processor.py | 113 ++++++++++++------ .../tensorboard-plugins/{ OWNERS => OWNERS} | 0 9 files changed, 139 insertions(+), 75 deletions(-) rename plugins/tensorboard-plugins/{ OWNERS => OWNERS} (100%) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 1e529d2b4..e725d305c 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -14,7 +14,6 @@ from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, Module from msprobe.pytorch.free_benchmark import FreeBenchmarkCheck, UnequalRow from msprobe.pytorch.common.utils import save_pt - try: import torch_npu is_gpu = False @@ -189,6 +188,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): self.real_overflow_nums = 0 self.overflow_nums = config.overflow_nums self.forward_inplace_inputs = None + self.is_support_inf_nan = self.is_support_inf_nan() @property def is_terminated(self): @@ -199,6 +199,10 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): return True return False + @staticmethod + def is_support_inf_nan(): + return is_gpu or (hasattr(torch_npu._C, '_npu_is_support_inf_nan') and torch_npu._C._npu_is_support_inf_nan()) + @staticmethod def overflow_debug_mode_enable(): overflow_mode = os.getenv(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) @@ -219,16 +223,18 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False api_info_struct = super().analyze_forward(name, module, module_input_output) - self.maybe_save_overflow_data_and_check_overflow_times() + self.handle_overflow() return api_info_struct if self.has_overflow else None def analyze_backward(self, name, module, module_input_output: ModuleBackwardInputsOutputs): self.has_overflow = False api_info_struct = super().analyze_backward(name, module, module_input_output) - self.maybe_save_overflow_data_and_check_overflow_times() + self.handle_overflow() return api_info_struct if self.has_overflow else None - def maybe_save_overflow_data_and_check_overflow_times(self): + def handle_overflow(self): + if not self.is_support_inf_nan: + self._analyze_maybe_overflow_flag() if self.has_overflow: for file_path, tensor in self.cached_tensors_and_file_paths.items(): save_pt(tensor, file_path) @@ -253,22 +259,20 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): else: torch_npu._C._clear_overflow_npu() + def _analyze_maybe_overflow_flag(self): + try: + self.has_overflow = self.check_overflow_npu() + if self.has_overflow: + self.clear_overflow_npu() + except Exception as e: + logger.error(f"Overflow check failed, the current environment may be abnormal.") + raise RuntimeError(f"overflow check failed") from e + def _analyze_maybe_overflow_tensor(self, tensor_json): - if is_gpu or (hasattr(torch_npu._C, '_npu_is_support_inf_nan') and torch_npu._C._npu_is_support_inf_nan()): - if tensor_json['Max'] is None: - return - if np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']): - self.has_overflow = True - if np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']): - self.has_overflow = True - else: - try: - self.has_overflow = self.check_overflow_npu() - if self.has_overflow: - self.clear_overflow_npu() - except Exception as e: - logger.error(f"Overflow check failed, the current environment may be abnormal.") - raise RuntimeError(f"overflow check failed") from e + if tensor_json['Max'] is None or tensor_json['Min'] is None: + return + self.has_overflow = np.isinf(tensor_json['Max']) or np.isnan(tensor_json['Max']) or \ + np.isinf(tensor_json['Min']) or np.isnan(tensor_json['Min']) def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) @@ -277,8 +281,9 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): else: logger.warning(f'The file path {file_path} length exceeds limit.') single_arg = super()._analyze_tensor(tensor, suffix) - self._analyze_maybe_overflow_tensor(single_arg) single_arg.update({"data_name": dump_data_name}) + if not self.has_overflow and self.is_support_inf_nan: + self._analyze_maybe_overflow_tensor(single_arg) return single_arg diff --git a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md index 72fb01e2e..290eec56b 100644 --- a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md @@ -143,7 +143,7 @@ msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_ | --------------------- | ------------------------- | | API name | API 名称。 | | Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。 | -| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，SKIP 表示该 API 的某个参数的反向不计算梯度。 | | Message | 提示信息。 | 该结果为中间结果，仅作为参考，建议完成 [5 预检结果比对](#5-预检结果比对)后查看比对结果。该结果后续将会删除。 @@ -219,7 +219,7 @@ Forward Test Success 和 Backward Test Success 是否通过测试是由 `accurac msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_details_{timestamp}.csv -gpu /home/xxx/gpu/accuracy_checking_details_{timestamp}.csv -o /home/xxx/ ``` -| 参数名称 | 说明 | 是否必选 | +| 参数名称 | 说明 | 是否必选 | | -------------------- | ------------- | -------- | | -npu 或 --npu_csv_path | NPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | | -gpu 或 --gpu_csv_path | GPU 预检结果 `accuracy_checking_details_{timestamp}.csv` 文件路径。默认从当前目录下识别该文件。 | 否 | @@ -236,8 +236,8 @@ msprobe -f pytorch api_precision_compare -npu /home/xxx/npu/accuracy_checking_de | 字段 | 含义 | | --------------------- | ------------------------------------------------------------ | | API name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | -| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，skip 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，SKIP 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，SKIP 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | | Message | 提示信息。 | Forward Test Success 和 Backward Test Success 是否通过测试是由 `api_precision_compare_details_{timestamp}.csv` 中的各个指标判定结果决定的。需要注意的是 `api_precision_compare_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `api_precision_compare_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error，仅存在 warning 和 pass 且不存在 error 标记 warning。 @@ -273,6 +273,6 @@ Forward Test Success 和 Backward Test Success 是否通过测试是由 `api_pre | ULP 误差判定结果 | ULP 误差判定结果。
当 NPU 或 GPU 数据类型是 float16 或 bfloat16 时，以下两条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差大于阈值占比小于 0.001；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。
当 NPU 或 GPU 数据类型是 float32 时，以下三条标准满足其一标记为 pass，否则标记为 error：
NPU ULP 误差平均值小于 64；
NPU ULP 误差大于阈值占比小于 0.05；
NPU ULP 误差大于阈值占比小于 GPU ULP 误差大于阈值占比。 | | 双千指标 | 双千精度指标。是指 NPU 的 Tensor 中的元素逐个与对应的标杆数据对比，相对误差小于千分之一的个数占总元素个数的比例。测试通过标准为相对误差大于千分之一的个数占总元素个数的比例小于千分之一。仅 conv1d 和 conv2d 使用该指标。双千指标法指标。 | | 双千指标判定结果 | 双千指标判定结果。双千指标大于 0.999 标记为 pass，否则标记为 error。 | -| 比对结果 | 综合所有指标的最终结果。如果比对指标中有 error，则标记为 error；有 warning，则标记为 warning；否则标记为 pass。 | +| 比对结果 | 综合所有指标的最终结果。如果比对指标中有 error，则标记为 error；有 warning，则标记为 warning；否则标记为 pass；SKIP 表示该 API 的数据类型不支持使用新精度标准进行比对，如 float64。 | | 比对算法 | API 使用的比对算法，为标杆比对法、二进制一致法、绝对阈值法和 ULP 误差比对法中的一种。 | | Message | 提示信息。当前提示该 API 比对结果为 error 或 warning 时对应不符合标准的指标。 | diff --git a/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md b/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md index b643dc69c..5fc048b53 100644 --- a/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md +++ b/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md @@ -3,7 +3,7 @@ ## 1 简介 为了应对大模型场景下，通过离线预检方式 dump API 输入输出数据导致的存储资源紧张问题，提供在线精度预检功能。本功能实现在执行 NPU 训练操作的过程中，通过 TCP/IP 协议在 NPU -Host 与 GPU Host 设备间建立连接，将 NPU 上对应 API 的输入数据在 GPU 设备上运行，将两份输出数据进行比对，得到预检比对结果，从而减少数据 dump 的步骤，降低存储资源的占用。 +Host 与 GPU Host 设备间建立连接，将 NPU 上对应 API 的输入数据在 GPU 设备上运行，将两份输出数据进行比对，得到预检比对结果，从而减少数据 dump 的步骤，降低存储资源的占用。针对偏差较大的算子，两方比对（NPU vs. GPU）的方法缺少裁判进行裁定。 参考离线预检，在线预检场景同时支持两方比对和三方比对方式，按照 api 的精度标准要求，选择比对两方比对和三方比对。 ## 2 在线精度预检流程 @@ -33,10 +33,12 @@ Host 与 GPU Host 设备间建立连接，将 NPU 上对应 API 的输入数据 | black_list | 预检的 API 黑名单，list[str] 类型。
**配置示例**：white_list=["conv1d", "conv2d"]。默认未配置黑名单，即预检全量 API 数据。 | 否 | | error_data_path | 配置保存精度未达标的 API 输入输出数据路径，str 类型。在线预检模式下该参数不生效。 | 否 | | is_online | 在线预检模式开关，bool 类型，可取值 True（开启）、False（关闭），默认关闭。 | 是 | -| nfs_path | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。配置该参数后 host 和 port 不生效。 | 否 | +| nfs_path | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。配置该参数后 host、port 和 tls_path 不生效。 | 否 | | host | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机地址 127.0.0.1 或本机局域网 IP。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | | port | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机可用端口。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | | rank_list | 指定在线预检的 Rank ID，默认值为 [0]，list[int] 类型，应配置为大于等于 0 的整数，且须根据实际卡的 Rank ID 配置，若所配置的值大于实际训练所运行的卡的 Rank ID，则在线预检输出数据为空。GPU 和 NPU 须配置一致。 | 是 | +| tls_path | 在线预检模式局域网场景 SSL 证书路径，该路径下包含私钥文件 server.key 和公钥文件 server.crt，str 类型，默认值为空，表示明文传输 api 数据，否则采用 TLS1.2 加密传输，加密传输时安全性较高，传输速率较低。 | 否 | + #### 3.1.2 NPU 侧在线预检配置说明 @@ -58,6 +60,18 @@ Host 与 GPU Host 设备间建立连接，将 NPU 上对应 API 的输入数据 #### 3.1.3 局域网场景配置示例 +若采用TLS1.2协议加密传输api数据，需配置SSL证书，可参考如下生成自签名证书方法，仅供调试使用，生产环境请申请正式证书。 +```shell +# 创建私钥文件server.key +openssl genrsa -out server.key 2048 + +# 创建签名请求文件server.csr +openssl req -new -key server.key -out server.csr + +# 自签名, 生成1年期公钥文件server.crt +openssl x509 -req -days 365 -in server.csr -signkey server.key -out server.crt +``` + GPU 侧： ```json @@ -179,4 +193,6 @@ debugger.step() bash train.sh ``` -训练脚本执行完毕后，在 GPU 侧 dump_path 目录下生成比对结果文件，详细介绍请参见[离线精度预检中的 **4 预检结果**](./07.accuracy_checker_Pytorch.md#4-预检结果)。 +训练脚本执行完毕后，在GPU侧dump_path目录下生成比对结果文件， +`accuracy_checking_result_{timestamp}_rank{rank_id}.csv`和`accuracy_checking_details_{timestamp}_rank{rank_id}.csv`记录两方比对结果 +`api_precision_compare_result_{timestamp}_rank{rank_id}.csv`和`api_precision_compare_details_{timestamp}_rank{rank_id}.csv`记录三方比对结果。详细介绍请参见[离线精度预检中的 **4 预检结果**](./07.accuracy_checker_Pytorch.md#4-预检结果)。 diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py index 0b2c28ed6..82faf70b6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py @@ -1,4 +1,5 @@ import os +from mindspore import Tensor from mindspore.common.api import _MindsporeFunctionExecutor from mindspore._c_expression import PyNativeExecutor_ from msprobe.mindspore.dump.hook_cell.api_registry import api_register @@ -33,7 +34,10 @@ class JitDump(_MindsporeFunctionExecutor): def __call__(self, *args, **kwargs): api_register.api_set_ori_func() out = super().__call__(*args, **kwargs) - dump_jit(args[0], args[1], out, True) + if isinstance(args[0], Tensor): + dump_jit({}, args, out, True) + else: + dump_jit(args[0], args[1:], out, True) JitDump.jit_enable = True api_register.api_set_hook_func() return out @@ -51,6 +55,6 @@ class JitDump(_MindsporeFunctionExecutor): api_register.api_set_ori_func() output = self._executor.grad(grad, obj, weights, grad_position, *args, *(kwargs.values())) if JitDump.jit_enable: - dump_jit(obj, args, output, False) + dump_jit(obj, args, None, False) api_register.api_set_hook_func() return output \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 7cc1d6b5e..d5e20791f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -357,8 +357,8 @@ def analyse_csv(npu_data, gpu_data, config): if last_api_name is not None and api_name != last_api_name: if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message - write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) - print_test_success(api_full_name, "skip", "skip") + write_csv([[last_api_name, CompareConst.SKIP, CompareConst.SKIP, message]], config.result_csv_path) + print_test_success(api_full_name, CompareConst.SKIP, CompareConst.SKIP) forward_status, backward_status = [], [] message = '' else: @@ -388,8 +388,8 @@ def analyse_csv(npu_data, gpu_data, config): if last_api_name is not None: if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message - write_csv([[last_api_name, "skip", "skip", message]], config.result_csv_path) - print_test_success(last_api_full_name, "skip", "skip") + write_csv([[last_api_name, CompareConst.SKIP, CompareConst.SKIP, message]], config.result_csv_path) + print_test_success(last_api_full_name, CompareConst.SKIP, CompareConst.SKIP) else: forward_result = get_api_checker_result(forward_status) backward_result = get_api_checker_result(backward_status) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 9c9723ed2..2a459c22c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -138,7 +138,7 @@ class Comparator: try: name = test_result[0] df_row = list(test_result[:INDEX_TEST_RESULT_GROUP]) - if test_result[1] == "SKIP": + if test_result[1] == CompareConst.SKIP: df_row.append(test_result[INDEX_TEST_RESULT_GROUP][INDEX_FIRST_GROUP][INDEX_MESSAGE]) if self.stack_info: stack_info = "\n".join(self.stack_info[name]) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 6d544dc60..3535b0880 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -5,7 +5,7 @@ import pandas as pd import torch import torch.multiprocessing as mp -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, CompareConst from msprobe.pytorch.api_accuracy_checker.compare.api_precision_compare import online_api_precision_compare from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import DETAIL_TEST_ROWS, thousandth_standard_api, \ binary_standard_api, absolute_standard_api @@ -104,7 +104,7 @@ def online_precision_compare(api_data, device, common_config, api_precision_csv_ else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") - compare.write_summary_csv((api_full_name, "SKIP", "SKIP", [[str(err)]], api_data.rank)) + compare.write_summary_csv((api_full_name, CompareConst.SKIP, CompareConst.SKIP, [[str(err)]], api_data.rank)) finally: torch.cuda.empty_cache() @@ -133,7 +133,7 @@ def online_compare(api_data, device, common_config): else: logger.error(f"Run {api_full_name} UT Error: {str(err)}") - compare.write_summary_csv((api_full_name, "SKIP", "SKIP", [[str(err)]], api_data.rank)) + compare.write_summary_csv((api_full_name, CompareConst.SKIP, CompareConst.SKIP, [[str(err)]], api_data.rank)) finally: torch.cuda.empty_cache() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index 89c1d9e84..104c5e469 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -209,15 +209,16 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): sys.modules['torch_npu.npu'] = Mock() sys.modules['torch_npu._C'] = Mock() - @patch('torch.save') - def test_maybe_save_overflow_data_and_check_overflow_times(self, mock_save): - self.processor.has_overflow = True - self.processor.real_overflow_nums = 0 - self.processor.cached_tensors_and_file_paths = {'dummy_path': torch.tensor([1.0, 2.0, 3.0])} - self.processor.maybe_save_overflow_data_and_check_overflow_times() - mock_save.assert_called_once() - self.processor.maybe_save_overflow_data_and_check_overflow_times() - self.assertEqual(self.processor.real_overflow_nums, 2) + def test_is_terminated(self): + self.processor.overflow_nums = -1 + self.assertFalse(self.processor.is_terminated) + + self.processor.real_overflow_nums = 2 + self.processor.overflow_nums = 2 + self.assertTrue(self.processor.is_terminated) + + self.processor.real_overflow_nums = 1 + self.assertFalse(self.processor.is_terminated) @patch('os.getenv', return_value=Const.ENV_ENABLE) def test_overflow_debug_mode_enable(self, mock_getenv): @@ -225,50 +226,88 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.assertTrue(result) mock_getenv.assert_called_once_with(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) - @patch('msprobe.core.data_dump.data_processor.pytorch_processor.is_gpu', return_value=True) - def test_analyze_maybe_overflow_tensor(self, _): - tensor_json = {'Max': float('inf'), 'Min': 1.0} - self.processor._analyze_maybe_overflow_tensor(tensor_json) - self.assertTrue(self.processor.has_overflow) + def test_analyze_pre_forward_inplace(self): + module_input_output = ModuleForwardInputsOutputs(args=(1, 2), kwargs=None, output=None) + self.processor.analyze_pre_forward_inplace(None, module_input_output) + self.assertTrue(self.processor.forward_inplace_inputs == module_input_output) + self.assertFalse(self.processor.forward_inplace_inputs is module_input_output) - @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) - @patch.object(BaseDataProcessor, 'get_save_file_path', return_value=['test_api_name', 'test_api_name.0.forward.input.pt']) - def test_analyze_tensor(self, mock_path_len_exceeds_limit, _): - tensor = torch.tensor([1.0, 2.0, 3.0]) - suffix = 'suffix' - expected = {'Max': 3.0, 'Min': 1.0, 'data_name': 'test_api_name'} - with patch.object(PytorchDataProcessor, '_analyze_tensor', return_value={'Max': 3.0, 'Min': 1.0}) as mock_super_analyze_tensor: - result = self.processor._analyze_tensor(tensor, suffix) - mock_super_analyze_tensor.assert_called_once_with(tensor, suffix) - mock_path_len_exceeds_limit.assert_called_once() - self.assertEqual(expected, result) + def test_analyze_forward_inplace(self): + with patch.object(OverflowCheckDataProcessor, "analyze_forward"): + module_input_output = ModuleForwardInputsOutputs(args=(1, 2), kwargs={"k1": 3, "k2": 4}, output=None) + self.processor.forward_inplace_inputs = module_input_output + self.processor.analyze_forward_inplace("name", module_input_output) + self.assertIsNone(self.processor.forward_inplace_inputs) + self.assertEqual(module_input_output.output, (1, 2, 3, 4)) - def test_analyze_backward(self): + def test_analyze_forward(self): def func(_): self.processor.has_overflow = True - with patch.object(BaseDataProcessor, "analyze_backward", return_value={"min", 0}): - with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times"): - api_info = self.processor.analyze_backward("name", "module", "module_input_output") + + with patch.object(BaseDataProcessor, "analyze_forward", return_value={"min", 0}): + with patch.object(OverflowCheckDataProcessor, "handle_overflow"): + api_info = self.processor.analyze_forward("name", "module", "module_input_output") self.assertFalse(self.processor.has_overflow) self.assertIsNone(api_info) - with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times", new=func): - api_info = self.processor.analyze_backward("name", "module", "module_input_output") + + with patch.object(OverflowCheckDataProcessor, "handle_overflow", new=func): + api_info = self.processor.analyze_forward("name", "module", "module_input_output") self.assertTrue(self.processor.has_overflow) self.assertEqual(api_info, {"min", 0}) - def test_analyze_forward(self): + def test_analyze_backward(self): def func(_): self.processor.has_overflow = True - with patch.object(BaseDataProcessor, "analyze_forward", return_value={"min", 0}): - with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times"): - api_info = self.processor.analyze_forward("name", "module", "module_input_output") + + with patch.object(BaseDataProcessor, "analyze_backward", return_value={"min", 0}): + with patch.object(OverflowCheckDataProcessor, "handle_overflow"): + api_info = self.processor.analyze_backward("name", "module", "module_input_output") self.assertFalse(self.processor.has_overflow) self.assertIsNone(api_info) - with patch.object(OverflowCheckDataProcessor, "maybe_save_overflow_data_and_check_overflow_times", new=func): - api_info = self.processor.analyze_forward("name", "module", "module_input_output") + + with patch.object(OverflowCheckDataProcessor, "handle_overflow", new=func): + api_info = self.processor.analyze_backward("name", "module", "module_input_output") self.assertTrue(self.processor.has_overflow) self.assertEqual(api_info, {"min", 0}) + @patch('torch.save') + def test_handle_overflow(self, mock_save): + self.processor.has_overflow = True + self.processor.real_overflow_nums = 0 + self.processor.is_support_inf_nan = True + self.processor.cached_tensors_and_file_paths = {'dump_path': torch.tensor([1.0, 2.0, 3.0])} + self.processor.handle_overflow() + mock_save.assert_called_once() + self.processor.handle_overflow() + self.assertEqual(self.processor.real_overflow_nums, 2) + + @patch('msprobe.core.data_dump.data_processor.pytorch_processor.is_gpu', return_value=True) + def test_analyze_maybe_overflow_tensor(self, _): + tensor_json = {'Max': None, 'Min': None} + self.processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertFalse(self.processor.has_overflow) + + tensor_json = {'Max': float('inf'), 'Min': 1.0} + self.processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.processor.has_overflow) + + tensor_json = {'Max': 1.0, 'Min': float('inf')} + self.processor._analyze_maybe_overflow_tensor(tensor_json) + self.assertTrue(self.processor.has_overflow) + + @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) + @patch.object(BaseDataProcessor, 'get_save_file_path', + return_value=['test_api_name', 'test_api_name.0.forward.input.pt']) + def test_analyze_tensor(self, mock_path_len_exceeds_limit, _): + tensor = torch.tensor([1.0, 2.0, 3.0]) + suffix = 'suffix' + expected = {'Max': 3.0, 'Min': 1.0, 'data_name': 'test_api_name'} + with patch.object(PytorchDataProcessor, '_analyze_tensor', + return_value={'Max': 3.0, 'Min': 1.0}) as mock_super_analyze_tensor: + result = self.processor._analyze_tensor(tensor, suffix) + mock_super_analyze_tensor.assert_called_once_with(tensor, suffix) + mock_path_len_exceeds_limit.assert_called_once() + self.assertEqual(expected, result) class TestFreeBenchmarkDataProcessor(unittest.TestCase): diff --git a/plugins/tensorboard-plugins/ OWNERS b/plugins/tensorboard-plugins/OWNERS similarity index 100% rename from plugins/tensorboard-plugins/ OWNERS rename to plugins/tensorboard-plugins/OWNERS -- Gitee From cbd58e02c912eecaf7b1cd666f23ff17b8d31032 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 23:08:37 +0800 Subject: [PATCH 508/791] renew_manual --- .../msprobe/docs/17.grad_probe.md | 201 ------------------ .../msprobe/docs/FAQ_PyTorch.md | 193 ----------------- ...robe_dump_standard_performance_baseline.md | 182 ---------------- ...marking_validation_performance_baseline.md | 151 ------------- 4 files changed, 727 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/docs/17.grad_probe.md delete mode 100644 debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md delete mode 100644 debug/accuracy_tools/msprobe/docs/S01.report_msprobe_dump_standard_performance_baseline.md delete mode 100644 debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md diff --git a/debug/accuracy_tools/msprobe/docs/17.grad_probe.md b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md deleted file mode 100644 index d7856e35a..000000000 --- a/debug/accuracy_tools/msprobe/docs/17.grad_probe.md +++ /dev/null @@ -1,201 +0,0 @@ -# Ascend 模型梯度状态监测工具 - -梯度状态监测工具提供了两种能力： - -- 将模型权重的梯度数据导出。这种功能可以将模型权重的梯度值以统计量的形式采集出来，用以分析问题。 -- 将两份梯度数据进行相似度对比。在有标杆问题中，可以确认训练过程中精度问题出现的step，以及抓取反向过程中的问题。 - -工具支持PyTorch版本：2.0/2.1/2.2；支持MindSpore版本：r2.3。 - - - -## 使用方式 - -### 梯度数据导出 - -1. 创建配置文件config.json，样例如下： - - ```json - { - "task": "grad_probe", - "dump_path": "./dump_path", - "rank": [], - "step": [], - "grad_probe": { - "grad_level": "L1", - "param_list": [], - "bounds": [-1, 0, 1] - } - } - ``` - > step指的是优化器被调用的次数（并非模型跑的step，某些step，例如loss为nan时，不会调用优化器） - - **参数说明** - - | 参数 | 说明 | 输入类型 | 是否必选 | - |--------------------------------|-----------------------------------|-----------------|----------| - | task | 填为"grad_probe"。 | str | 是 | - | grad_level | 输出级别。决定导出数据的详细程度，级别越大导出数据越详细。可取值：L0, L1, L2|str | 是 | - | param_list | 权重名称列表，表示需要监控的权重。列表为空就表示监控所有权重。 | List[str] | 是 | - | rank | rank id列表，在多卡场景下，表示需要导出梯度数据的进程的rank id。列表为空就表示导出所有rank的数据。（MindSpore静态图模式下，当前暂不支持指定rank功能） | List[int] | 是 | - | step | step列表，表示需要导出数据的step列表。列表为空就表示导出所有step的数据。（MindSpore静态图模式下，当前暂不支持指定step功能） | List[int] | 是 | - | bounds | 区间列表，用来划分区间以统计数值的分布。需要保证由数据小到大排列。可以使用默认值[-1, 0, 1] | List[float] | 是 | - | dump_path | 输出目录。如果不存在就会创建一个新目录。 | str | 是 | - - **不同级别的level的导出数据** - - - | 级别 | 特征数据表头 | 是否有方向数据 | - | ---- | ------------------------------------------------------------ | -------------- | - | L0 | ("param_name", "MD5", "max", "min", "norm", "shape") | 否 | - | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | - | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | - - intervals就是根据值分布bounds划分出的区间。 - MindSpore静态图模式下，L0级别中暂不支持"MD5" - - **方向数据解释** - - 因为模型的参数往往非常大，所以存储真实数据是不可接受的，这里折衷一下，只存储梯度数据的正负号（一个布尔值），也就是方向。 - - **bounds和值分布解释** - - + 值分布：梯度数据落在各个区间的元素个数占总元素个数的比例。 - + bounds：一个列表，用来划分出区间以统计值分布。例如传入bounds = [-10, 0, 10]，此时有一个 grad_value: Tensor = [9.3 , 5.4, -1.0, -12.3]，依据 bounds 划分出 (-inf, -10]、(-10, 0]、(0, 10]、(10, inf) 四个区间，然后统计grad_value里的数据落在每个区间内的个数，得到 1、1、2、0。如下图所示： - ![Alt text](img/grad_probe_image-1.png) - -2. 插入代码。示例代码如下： - -- PyTorch框架：模型构造完成后，传入config.json的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将`模型`作为参数传入。 -```python -from msprobe.pytorch import PrecisionDebugger -debugger = PrecisionDebugger("config_json_path") -debugger.monitor(model) -``` -- MindSpore框架：优化器构造完成后，传入config.json的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将`优化器`作为参数传入。 -```python -from msprobe.mindspore import PrecisionDebugger -debugger = PrecisionDebugger("config_json_path") -debugger.monitor(optimizer) -``` - -3. 结束监控（MindSpore静态图模式下需要） - - 在训练结束之后，调用stop接口 - -```python -gm.stop() -``` - -### 输出结果 -**输出目录结构**（以level配置L2为例） - -```bash -{dump_path} - ├── rank{rank_id} - │ ├── grad_summary_{step}.csv - │ ├── step{step} - │ │ ├── {param_name}.npy -``` -+ {timestamp}：梯度工具导出数据的时候会在output_path下生成一个时间戳目录，然后在这个时间戳目录下输出结果。 -+ rank_{rank_id}：在分布式场景下，会记录卡的rank_id。非分布式场景下，如果是CPU则记录进程号，如果是CPU或GPU则记录卡号 -+ grad_summary_{step}.csv：会分step记录每一步的梯度数据统计值。 -+ step_{step}：这个目录下会存放该step的梯度的方向数据。 -+ {param_name}.pt(npy)：模型参数的梯度方向数据，PyTorch保存的是pt文件，MindSpore是npy文件。 - -**grad_summary_{step}.csv** - -样例如下： - -![Alt text](img/grad_probe_image.png) - -| 字段 | 含义 | -| --------------------- | ------------------------------------------------------------| -| Param_name | 模型参数名称。 | -| MD5 | 梯度数据的MD5值。 | -| (-inf, -0.01]...[0.01, inf) | 梯度值落在区间内的元素个数占总元素的比例。 | -| =0 | 梯度为0的元素个数占总元素的比例。 | -| Max | 最大值。 | -| Min | 最小值。 | -| Norm | L2norm值。 | -| Shape | 形状。 | - -### 梯度相似度比对 - -会根据所导出的权重，分step比对梯度相似度，输出每个权重的梯度相似度和总的梯度相似度。单个权重的梯度相似度为两份方向数据的重合度，总的梯度相似度为每个权重的梯度相似度按元素个数加权。 - -#### 前提条件 - -- 相同配置下，以Level为L1或L2分别采集npu和gpu环境下的梯度数据。 -- 将两份梯度数据传到同一环境下。 - -#### 使用方式 - - -新建如下Python脚本，传入npu和gpu的dump_path以及输出目录，比对结果输出目录不存在的话会新建： - -```python -from msprobe import * -GradComparator.compare_distributed("配置文件里写的dump_path", - "配置文件里写的dump_path", - "比对结果输出目录") -``` - - -### 比对结果 - -**输出目录结构** - -如下为多卡比对结果，单卡则没有rank_{rank_id}这一级目录。 - -```bash -比对结果输出目录 - ├── rank{rank_id} - │ ├── similarities.csv - │ └── similarities_picture - │ ├── {param_name}.png - │ └── summary_similarities.png -``` - -**问题界定** - -原则：对于任意权重，第0步的梯度相似度低于0.97，或者某一步的梯度相似度下降超过0.03，认为这一步存在精度问题。例子如下： - -- 第0步相似度低于0.97 - -![Alt text](img/grad_probe_image-3.png) - -- 第3步相似度下降超过0.03 - -![Alt text](img/grad_probe_image-4.png) - -- 正常情况 - -![Alt text](img/grad_probe_image-2.png) - -这个原则是一个经验性的指标，并不是严格的标注，还需要结合实际情况具体分析。 - -## 公开接口 - -**接口说明** - -```python -PrecisionDebugger.monitor(module) -``` - -| 参数 | 说明 | 是否必选 | -| ----- | -------------------- | -------- | -| module |Pytorch框架下传入模型，必须是torch.nn.Module；MindSpore框架下传入优化器。 | 是 | - - -**接口说明** - -```python -GradComparator.compare_distributed(dump_path1, dump_path2, output_path) -``` - -| 参数 | 说明 | 是否必选 | -| ----- | -------------------- | -------- | -| dump_path1 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path。 | 是 | -| dump_path2 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path，与dump_path1可以互换。 | 是 | -| output_path |输出结果目录，不存在会新建。 | 是 | diff --git a/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md b/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md deleted file mode 100644 index 8d12a7292..000000000 --- a/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md +++ /dev/null @@ -1,193 +0,0 @@ -# 精度预检工具 - -1. 预检工具在dump和run_ut的过程中，是否需要同时开启或关闭jit编译（jit_compile）？ - - 答：是。 - -2. 预检工具对于type_as这类涉及数据类型转换操作的API，是否具有参考性？ - - 由于这类API在CPU侧存在精度先提升后下降的操作，因此这类API的有效性的参考价值有限。 - -3. run ut过程中出现报错：ERROR:Got unsupported ScalarType BFloat16 - - 答：请使用最新版本的工具。 - -4. Dropout算子，CPU和NPU的随机应该不一样，为什么结果比对是一致的？ - - 答：这个结果是正常的，工具对该算子有特殊处理，只判定位置为0的位置比例大约和设定p值相当。 - -5. 为什么浮点型数据bench和CPU的dtype不一致？ - - 答：对于fp16的数据，CPU会上升一个精度fp32去计算，这是和算子那边对齐的精度结论，CPU用更高精度去计算会更接近真实值。 - -6. 添加预检工具后截取操作报错：`IndexError: too many indices for tensor of dimension x` 或 `TypeError: len() of a 0-d tensor`。 - - 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- __getitem__`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 - -7. 添加预检工具后F.gelu触发ValueError报错：`activation_func must be F.gelu`等。 - - 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中functional:下的的`- gelu`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 - -8. 添加预检工具后触发AsStrided算子相关的报错，或者编译相关的报错，如：`Failed to compile Op [AsStrided]`。 - - 答：注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- t`和`- transpose`。 - -9. Tensor 魔法函数具体对应什么操作？ - - 答： - - | Tensor魔法函数 | 具体操作 | - | --------------- | ---------------- | - | `__add__` | + | - | `__and__` | & | - | `__bool__` | 返回Tensor布尔值 | - | `__div__` | / | - | `__eq__` | == | - | `__ge__` | >= | - | `__gt__` | > | - | `__iadd__` | += | - | `__iand__` | &= | - | `__idiv__` | /= | - | `__ifloordiv__` | //= | - | `__ilshift__` | <<= | - | `__imod__` | %= | - | `__imul__` | *= | - | `__ior__` | \|= | - | `__irshift__` | >>= | - | `__isub__` | -= | - | `__ixor__` | ^= | - | `__lshift__` | << | - | `__matmul__` | 矩阵乘法 | - | `__mod__` | % | - | `__mul__` | * | - | `__nonzero__` | 同`__bool__` | - | `__or__` | \| | - | `__radd__` | +（反向） | - | `__rmul__` | *（反向） | - | `__rshift__` | >> | - | `__sub__` | - | - | `__truediv__` | 同`__div__` | - | `__xor__` | ^ | - -# 精度比对工具 - -## 工具使用 - -### dump指定融合算子 - -dump指定操作当前支持dump指定融合算子的输入输出，需要在mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml中添加，比如以下代码段调用的softmax融合算子 - -``` -def npu_forward_fused_softmax(self, input_, mask): - resl = torch_npu.npu_scaled_masked_softmax(input_, mask, self.scale, False) - return resl -``` - -如果需要dump其中调用的npu_scaled_masked_softmax算子的输入输出信息，需要在support_wrap_ops.yaml中的torch_npu: 中自行添加该融合算子即可： - -``` -- npu_scaled_masked_softmax -``` - -（npu_scaled_masked_softmax融合算子工具已支持dump，本例仅供参考） - -## 常见问题 - -### 1. 在同一个目录多次执行dump会冲突吗？ - -会，同一个目录多次dump，会覆盖上一次结果，可以使用dump_path参数修改dump目录。 - -### 2. 如何dump算子级的数据？ - -需要配置level为L2模式。 - -### 3. 工具比对发现NPU和标杆数据的API无法完全对齐？ - -torch版本和硬件差异属于正常情况。 - -## 异常情况 - -### 2. HCCL 报错： error code: EI0006 - -**故障现象** - -使用msprobe工具时，报错： error code: EI0006。 - -**故障原因** - -CANN软件版本较低导致不兼容。 - -**故障处理** - -升级新版CANN软件版本。 - -### 3. torch_npu._C._clear_overflow_npu() RuntimeError NPU error，error code is 107002 - -如果运行溢出检测功能遇到这个报错，采取以下解决方法： -如果是单卡运行，添加如下代码，0是卡号，选择自己空闲的卡号。 - -``` -torch.npu.set_device('npu:0') -``` - -如果多卡运行，请在代码中修改对应卡号，比如进程使用卡号为{rank}时可以添加如下代码： - -``` -torch.npu.set_device(f'npu:{rank}') -``` - -如果运行精度比对功能遇到这个报错，尝试安装最新版本的msprobe。 - -### 4. dump得到的VF_lstm_99_forward_input.1.0.npy、VF_lstm_99_forward_input.1.1.npy类似的数据是否正常？ - -带1.0/1.1/1.2后缀的npy是正常现象，例如当输入数据为[[tensor1, tensor2, tensor3]]会生成这样的后缀。 - -### 5. 进行compare报错：The current file contains stack information, please turn on the stack_mode - -在比对脚本中，设置stack_mode=True，例如： - -``` -from msprobe.pytorch import compare -dump_result_param={ -"npu_json_path": "./npu_dump/dump.json", -"bench_json_path": "./gpu_dump/dump.json", -"stack_json_path": "./npu_dump/stack.json", -"is_print_compare_log": True -} -compare(dump_result_param, output_path="./output", stack_mode=True) -``` - -### 6. dump指定反向API的kernel级别的数据报错：NameError：name 'torch_npu' is not defined - -- 如果是npu环境，请安装torch_npu； -- 如果是gpu环境，暂不支持dump指定API的kernel级别的数据 - -### 7. 配置dump_path后，使用工具报错：[ERROR]The file path /home/xxx/dump contains special characters - -- 请检查你设置的dump绝对路径是否包含特殊字符，确保路径名只包含大小写字母、数字、下划线、斜杠、点和短横线 -- 注意，如果执行脚本的路径为/home/abc++/，设置的dump_path="./dump"，工具实际校验的路径为绝对路径/home/abc++/dump，++为特殊字符，会引发本条报错 - -### 8. 无法dump matmul权重的反向梯度数据 - -- matmul期望的输入是二维，当输入不是二维时，会将输入通过view操作展成二维，再进行matmul运算，因此在反向求导时，backward_hook能拿到的是UnsafeViewBackward这步操作里面数据的梯度信息，取不到MmBackward这步操作里面数据的梯度信息，即权重的反向梯度数据。 -- 典型的例子有，当linear的输入不是二维，且无bias时，会调用output = input.matmul(weight.t()),因此拿不到linear层的weight的反向梯度数据。 - -### 9. dump.json文件中的某些api的dtype类型为float16，但是读取此api的npy文件显示的dtype类型为float32 - -- msprobe工具在dump数据时需要将原始数据从npu to cpu上再转换为numpy类型，npu to cpu的逻辑和gpu to cpu是保持一致的，都存在dtype可能从float16变为float32类型的情况，如果出现dtype不一致的问题，最终dump数据的dtype以pkl文件为准。 - -### 10. 使用dataloader后raise异常Exception("msprobe: exit after iteration {}". format(max(self.config.step)) - -- 正常现象，dataloader通过raise结束程序，堆栈信息可忽略。 - -### 11. 添加msprobe工具后截取操作报错：`IndexError: too many indices for tensor of dimension x` 或 `TypeError: len() of a 0-d tensor`。 - -- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- __getitem__`，工具会跳过dump该API。如果是需要dump的关键位置API也可以考虑根据报错堆栈信息注释引发报错的类型检查。 - -### 12. 添加msprobe工具后F.gelu触发ValueError报错：`activation_func must be F.gelu`等。 - -- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中functional:下的的`- gelu`，工具会跳过dump该API。如果是需要dump的关键位置api也可以考虑根据报错堆栈信息注释引发报错的类型检查。 - -### 13. 添加msprobe工具后触发AsStrided算子相关的报错，或者编译相关的报错，如：`Failed to compile Op [AsStrided]`。 - -- 注释工具目录mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml文件中Tensor:下的`- t`和`- transpose`。 diff --git a/debug/accuracy_tools/msprobe/docs/S01.report_msprobe_dump_standard_performance_baseline.md b/debug/accuracy_tools/msprobe/docs/S01.report_msprobe_dump_standard_performance_baseline.md deleted file mode 100644 index b2db12147..000000000 --- a/debug/accuracy_tools/msprobe/docs/S01.report_msprobe_dump_standard_performance_baseline.md +++ /dev/null @@ -1,182 +0,0 @@ -# msprobe 精度工具标准性能基线报告 - -## 1 环境信息 - -NPU：Atlas A2 训练系列产品 - -CPU： - -![输入图片说明](img/cpu_info.png) - -Torch：2.1.0 - -CANN：8.0.T2 - -除上述环境信息影响性能外，API 的数量、种类以及 Shape 都会对性能产生影响，因此本次选取指定网络进行测试，为了避免算子编译耗时的影响，所有模型运行时都开启二进制，模型中添加 torch.npu.set_compile_mode(jit_compile=False)，所有模型都 dump 第二个 step 的数据。 - -## 模型信息和性能基线 - -大模型在使用msprobe工具dump数据时，建议先简化模型层数，减少dump数据量。 - -以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 - -### 工具配置信息 - -dump全部API级别输入输出数据以及相应堆栈信息，配置如下： - -```python -debugger = PrecisionDebugger(dump_path="./dump_path", hook_name="dump") -debugger.configure_hook(mode="api_stack") -``` - -多卡指定rank0 dump，配置如下： - -```python -debugger = PrecisionDebugger(dump_path="./dump_path", hook_name="dump",rank=0) -debugger.configure_hook(mode="api_stack") -``` - -dump保存API统计信息的pkl文件，配置如下： - -```python -debugger = PrecisionDebugger(dump_path="./dump_path", hook_name="dump") -debugger.configure_hook(mode="api_stack", summary_only=True) -``` - -### YOLOV5s - -单卡 - -主要数据类型：FLOAT32 - -启动命令参数：python3 train_ptdbg.py --data ./data/coco.yaml --cfg yolov5s.yaml --weights '' --epochs 1 --batch-size 8 --device 1 - -dump保存API统计信息的pkl文件耗时：**7s** - -进行单卡dump全部API级别输入输出数据以及相应堆栈信息耗时：**11s** - -- dump存盘的API numpy文件大小：13G - - ![输入图片说明](img/YOLOV5S_1.png) - -- api numpy文件数量：3009个 - - ![输入图片说明](img/YOLOV5S_2.png) - - - - -### GPT-3 - -#### NUM_LAYER：1 - -8卡 - -主要数据类型：FLOAT16 - -启动命令参数： - -``` -python3 -m torch.distributed.launch $DISTRIBUTED_ARGS ../../pretrain_gpt_ptdbg.py --num-layers 1 --hidden-size 12288 --num-attention-heads 24 --micro-batch-size 2 --global-batch-size 2 --seq-length 1024 --max-position-embeddings 1024 --train-iters 10 --lr-decay-iters 320000 --save $CHECKPOINT_PATH --load $CHECKPOINT_PATH --data-path $DATA_PATH --tensor-model-parallel-size 8 --use-distributed-optimizer --pipeline-model-parallel-size 8 --vocab-file gpt2-vocab.json --merge-file gpt2-merges.txt --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.375e-5 --lr-decay-style cosine --min-lr 0.375e-6 --weight-decay 0.1 --clip-grad 1.0 --lr-warmup-fraction .01 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 ---recompute-granularity full --recompute-method uniform --no-gradient-accumulation-fusion --log-interval 1 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --fp16 -``` - -dump保存API统计信息的pkl文件耗时：**3.3s** - -进行8卡dump全部API级别输入输出数据以及相应堆栈信息耗时：**53s** - -- dump存盘的api numpy文件大小：145G - - ![输入图片说明](img/GPT-3_1.png) - -- API numpy文件数量：5130个 - - ![输入图片说明](img/GPT-3_2.png) - - -**经测试8卡同时写入磁盘已达到磁盘I/O上限，工具的dump速度取决于磁盘性能，本机环境多进程写入磁盘上限为3GB/秒左右，理论上保存145GB的数据需要50秒左右，如果dump的数据中包含许多的小文件，那么耗时将会更久。** - -指定rank0 dump耗时：**9s** - -- dump存盘的api numpy文件大小：19G - - ![输入图片说明](img/GPT-3_3.png) - -- api numpy文件数量：643个 - - ![输入图片说明](img/GPT-3_4.png) - - -#### NUM_LAYER：8 - -8卡 - -主要数据类型：FLOAT16 - -启动命令参数： - -``` -python3 -m torch.distributed.launch $DISTRIBUTED_ARGS ../../pretrain_gpt_ptdbg.py --num-layers 8 --hidden-size 12288 --num-attention-heads 24 --micro-batch-size 2 --global-batch-size 2 --seq-length 1024 --max-position-embeddings 1024 --train-iters 10 --lr-decay-iters 320000 --save $CHECKPOINT_PATH --load $CHECKPOINT_PATH --data-path $DATA_PATH --tensor-model-parallel-size 8 --use-distributed-optimizer --pipeline-model-parallel-size 8 --vocab-file gpt2-vocab.json --merge-file gpt2-merges.txt --data-impl mmap --split 949,50,1 --distributed-backend nccl --lr 0.375e-5 --lr-decay-style cosine --min-lr 0.375e-6 --weight-decay 0.1 --clip-grad 1.0 --lr-warmup-fraction .01 --adam-beta1 0.9 --adam-beta2 0.95 --init-method-std 0.006 --recompute-granularity full --recompute-method uniform --no-gradient-accumulation-fusion --log-interval 1 --save-interval 10000 --eval-interval 1000 --eval-iters 10 --fp16 -``` - -dump保存API统计信息的pkl文件耗时：**6.7s** - -进行8卡dump全部API级别输入输出数据以及相应堆栈信息耗时：**323s** - -- dump存盘的API numpy文件大小：878G - - ![输入图片说明](img/GPT-3_5.png) - -- API numpy文件数量：24002个 - - ![输入图片说明](img/GPT-3_6.png) - - -指定rank0 dump耗时：**47s** - -- dump存盘的API numpy文件大小：110G - - ![输入图片说明](img/GPT-3_7.png) - -- API numpy文件数量：3002个 - - ![输入图片说明](img/GPT-3_8.png) - - -### BLOOM-7B - -8卡 - -NUM_LAYER：1 - -主要数据类型：BFLOAT16 - -启动命令参数： - -``` -python -m torch.distributed.launch $DISTRIBUTED_ARGS pretrain_llama.py --DDP-impl local --tensor-model-parallel-size 8 --pipeline-model-parallel-size 1 --sequence-parallel --num-layers 1 --hidden-size 12288 --position-embedding-type rope --normalization RMSNorm --ffn-hidden-size 11008 --num-attention-heads 24 --attention-dropout 0.0 --hidden-dropout 0.0 --init-method-std 0.01 --micro-batch-size 2 --global-batch-size 2 --seq-length 1024 --max-position-embeddings 1024 --data-path $DATA_PATH --tokenizer-name-or-path $TOKENIZER_PATH --tokenizer-not-use-fast --split 100,0,0 --distributed-backend nccl --lr 1.25e-5 --min-lr 1.25e-6 --lr-decay-style cosine --weight-decay 1e-1 --clip-grad 1.0 --initial-loss-scale 65536.0 --adam-beta1 0.9 --adam-beta2 0.95 --log-interval 1 --load ${LOAD_CHECKPOINT_PATH} --save ${SAVE_CHECKPOINT_PATH} --save-interval 10000 --eval-interval 10000 --eval-iters 0 --use-fused-rotary-pos-emb --no-masked-softmax-fusion --no-load-optim --no-load-rng --train-iters 20 --lr-warmup-fraction 0.01 --mlp-layer-fusion --use-flash-attn --use-fused-rmsnorm --bf16 -``` - -dump保存API统计信息的pkl文件耗时：**3s** - -进行8卡dump全部API级别输入输出数据以及相应堆栈信息耗时：**61s** - -- dump存盘的API numpy文件大小：160G - - ![输入图片说明](img/BLOOM-7B_1.png) - -- API numpy文件数量：4924个 - - ![输入图片说明](img/BLOOM-7B_2.png) - - -指定rank0 dump耗时：**17s** - -- dump存盘的API numpy文件大小：20G - - ![输入图片说明](img/BLOOM-7B_3.png) - -- API numpy文件数量：633个 - - ![输入图片说明](img/BLOOM-7B_4.png) - diff --git a/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md b/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md deleted file mode 100644 index 8862c5b76..000000000 --- a/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md +++ /dev/null @@ -1,151 +0,0 @@ -# 无标杆工具场景验证和性能基线报告 - -## 环境信息 - -NPU：Atlas A2 训练系列产品 - -CPU： - -![输入图片说明](img/cpu_info.png) - -Torch：2.1.0 - -CANN：8.0.T5 - -除上述环境信息影响性能外，API的数量、种类以及Shape都会对性能产生影响，因此本次选取不同场景网络和不同算子进行测试。 - -## 模型信息和性能基线 - -大模型在使用msprobe工具dump数据时，建议先简化模型层数，减少dump数据量。 - -以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 - - - -### LLaMA2-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink -其中，softmax算子为FLOAT32，输入输出均为2G大小，为模型最大显存开销的API。 - -在该模型下、对无标杆工具处理模式、插装范围、扰动方式组合下性能和显存基线进行覆盖。 - -性能基线报告 -其中耗时为训练10步，去除第一步耗时所得的平均每步耗时。 - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.24 | 13.69 | 1 | 1 | 混精模式基线 | -| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1 | softmax本身为高精度，跳过 | -| check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | -| check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | -| check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | -| check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | -| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | -| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.38 | 1 | | -| check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | -| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | -| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | -| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | -| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | -| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | -| check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | -| check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | -| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | -| fix | 前 | ["softmax"] | to_cpu | 9.70 | 16.67 | 40.42 | 1.22 | 不支持整网、不支持反向 | -| fix | 前 | ["softmax"] | improve_precision | 0.26 | 14.67 | 1.08 | 1.07 | 不支持整网、不支持反向 | -| 预热 | 前 | [] | improve_precision | 155.07 | 24.79 | 646.13 | 1.81 | 低精度模型基线、只测预热的迭代 | -| 预热 | 反 | [] | improve_precision | 72.29 | 22.01 | 301.21 | 1.61 | 低精度模型基线、只测预热的迭代，grad_output为高精度的算子跳过 | - -### Aquila2-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.17 | 13.66 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.57 | 14.24 | 9.24 | 1.04 | | -| check | 反 | [] | add_noise | 21.05 | 14.19 | 123.82 | 1.04 | | -| fix | 前 | [] | improve_precision | 0.95 | 15.55 | 5.59 | 1.14 | | - -### Baichuan2-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.26 | 12.12 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.02 | 12.27 | 3.92 | 1.01 | | -| check | 反 | [] | add_noise | 11.15 | 12.67 | 42.88 | 1.05 | | -| fix | 前 | [] | improve_precision | 0.95 | 12.82 | 3.65 | 1.06 | | - -### Bloom-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.14 | 9.51 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.64 | 11.58 | 11.71 | 1.22 | | -| check | 反 | [] | add_noise | 17.15 | 9.51 | 122.5 | 1 | | -| fix | 前 | [] | improve_precision | 0.87 | 10.62 | 6.21 | 1.12 | | - -### Interlm-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.13 | 10.76 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.19 | 11.68 | 9.15 | 1.09 | | -| check | 反 | [] | add_noise | 11.69 | 10.89 | 89.92 | 1.01 | | -| fix | 前 | [] | improve_precision | 0.75 | 11.68 | 5.77 | 1.09 | | - -### Qwen-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.28 | 18.41 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 2.34 | 23.18 | 8.36 | 1.26 | | -| check | 反 | [] | add_noise | 22.07 | 19.47 | 78.82 | 1.06 | | -| fix | 前 | [] | improve_precision | 1.31 | 21.11 | 4.68 | 1.15 | | - -### Gemma-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.15 | 11.06 | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | 1.49 | 13.17 | 9.93 | 1.19 | | -| check | 反 | [] | add_noise | 16.69 | 11.06 | 111.27 | 1 | | -| fix | 前 | [] | improve_precision | 0.87 | 12.25 | 5.8 | 1.11 | | - -### ResNet50-Cifar -1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch。 -主要算子为conv2d，每个step有51个, 因此对conv2d进行检测。 -CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step信息和反向信息）。 - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.09 | 7.63 | 1 | 1 | 基线 | -| check | 前 | ["conv2d"] | improve_precision | 0.889 | 7.94 | 9.81 | 1.04 | | -| fix | 前 | ["conv2d"] | improve_precision | 0.328 | 7.47 | 3.64 | 0.91 | | -| fix | 前 | ["conv2d"] | to_cpu | 12.23 | 7.47 | 135.88 | 0.91 | | - -### OpenSora1.0 - -4卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch - -每张卡每个step中linear算子个数为257个，FA算子个数为83（FA算子反向无效）。 - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.99 | 17.61 | 1 | 1 | 混精模式基线 | -| check | 前 | ["linear","npu_fusion_attention"] | improve_precision | 3.88 | 17.61 | 3.92 | 1 | | -| check | 前 | ["linear","npu_fusion_attention"] | add_noise | 3.46 | 17.61 | 3.49 | 1 | | -| check | 反 | ["linear"] | improve_precision | 12.61 | 17.61 | 12.74 | 1 | | -| check | 反 | ["linear"] | add_noise | 9.8 | 17.61 | 9.90 | 1 | | -| fix | 前 | ["linear"] | to_cpu | 18.83 | 17.61 | 19.02 | 1 | | -| fix | 前 | ["linear"] | improve_precision | 2.83 | 17.61 | 2.86 | 1 | | -- Gitee From c92660890d6246aaab1f532a43cb3e954592136e Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 26 Aug 2024 23:23:05 +0800 Subject: [PATCH 509/791] renew_manual --- debug/accuracy_tools/setup.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index 01e399c46..b273f0f3d 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -19,13 +19,13 @@ __version__ = '1.0.3' INSTALL_REQUIRED = [ "wheel", "einops", - "numpy < 2.0", - "pandas >= 1.3.5, < 2.1", + "numpy", + "pandas >= 1.3.5", "pyyaml", "rich", "tqdm", "openpyxl", - "pyopenssl", + "pyOpenSSL", "twisted", "matplotlib" ] -- Gitee From a54f9c784a47da599f8c2fabf767dfea14d5510f Mon Sep 17 00:00:00 2001 From: zhaolei Date: Tue, 27 Aug 2024 10:47:47 +0800 Subject: [PATCH 510/791] =?UTF-8?q?mstt=20advisor=20wheel=E5=8C=85=20?= =?UTF-8?q?=E9=93=BE=E6=8E=A5=E4=BF=AE=E8=A1=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/README.md b/profiler/README.md index 008cb9bdd..5eae3c49d 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -107,7 +107,7 @@ Successfully installed msprof-analyze-{version} | profiler版本 | 发布日期 | 下载链接 | 校验码 | |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ------------------------------------------------------------ | - | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | + | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.2/msprof_analyze-1.2.2-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | | 1.1.2 | 2024-07-12 | [msprof_analyze-1.1.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.1.2/msprof_analyze-1.1.2-py3-none-any.whl) | af62125b1f9348bf491364e03af712fc6d0282ccee3fb07458bc9bbef82dacc6 | -- Gitee From 78df8b1061ab45a9b81314cfcb275da6f0bd5a3a Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 27 Aug 2024 16:25:25 +0800 Subject: [PATCH 511/791] fix file safe problem --- .../api_accuracy_checker/common/utils.py | 26 +++++-------------- .../run_ut/data_generate.py | 5 ++-- .../run_ut/multi_run_ut.py | 5 ++-- .../msprobe/pytorch/common/utils.py | 10 +++++++ 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index fdd37337d..e151d6693 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -27,9 +27,10 @@ else: IS_GPU = False from msprobe.pytorch.common.log import logger -from msprobe.core.common.file_check import FileChecker, FileOpen, change_mode, create_directory -from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.utils import CompareException +from msprobe.pytorch.common.utils import save_pt +from msprobe.core.common.file_check import create_directory +from msprobe.core.common.const import Const +from msprobe.core.common.utils import CompareException, make_dump_path_if_not_exists class DumpException(CompareException): @@ -100,24 +101,10 @@ def cross_entropy_process(api_info_dict): def initialize_save_path(save_path, dir_name): data_path = os.path.join(save_path, dir_name) - if os.path.exists(data_path): - logger.warning(f"{data_path} already exists, it will be overwritten") - else: - os.mkdir(data_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) - data_path_checker = FileChecker(data_path, FileCheckConst.DIR) - data_path_checker.common_check() + make_dump_path_if_not_exists(data_path) return data_path -def write_pt(file_path, tensor): - if os.path.exists(file_path): - raise ValueError(f"File {file_path} already exists") - torch.save(tensor, file_path) - full_path = os.path.realpath(file_path) - change_mode(full_path, FileCheckConst.DATA_FILE_AUTHORITY) - return full_path - - def get_real_data_path(file_path): targets = ['forward_real_data', 'backward_real_data', 'ut_error_data\d+'] pattern = re.compile(r'({})'.format('|'.join(targets))) @@ -151,7 +138,8 @@ class UtDataProcessor: api_args = api_name + Const.SEP + str(self.index) create_directory(self.save_path) file_path = os.path.join(self.save_path, f'{api_args}.pt') - write_pt(file_path, element.contiguous().cpu().detach()) + tensor = element.contiguous().cpu().detach() + save_pt(tensor, file_path) self.index += 1 elif element is None or isinstance(element, (bool, int, float, str, slice)): self.index += 1 diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index 6f2b4801f..14f8c838b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -25,6 +25,7 @@ from msprobe.pytorch.api_accuracy_checker.common.utils import check_object_type, CompareException from msprobe.core.common.file_check import FileChecker from msprobe.pytorch.common.log import logger +from msprobe.pytorch.common.utils import load_pt, load_npy from msprobe.core.common.const import Const, FileCheckConst TORCH_TYPE = ["torch.device", "torch.dtype"] @@ -94,9 +95,9 @@ def gen_real_tensor(data_path, convert_type): error_info = f"The file: {data_path} is not a pt or numpy file." raise CompareException(CompareException.INVALID_FILE_ERROR, error_info) if data_path.endswith('.pt'): - data = torch.load(data_path, map_location=torch.device('cpu')) + data = load_pt(data_path, to_cpu=True) else: - data_np = numpy.load(data_path) + data_np = load_npy(data_path) data = torch.from_numpy(data_np) if convert_type: ori_dtype = Const.CONVERT.get(convert_type)[0] diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index c83fb5fb8..8334abef8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -14,9 +14,10 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validat get_validated_details_csv_path from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.common import parse_json_info_forward_backward +from msprobe.pytorch.common.log import logger from msprobe.core.common.file_check import FileChecker, check_file_suffix, check_link, FileOpen, \ check_path_before_create, create_directory -from msprobe.pytorch.common.log import logger +from msprobe.core.common.utils import remove_path from msprobe.core.common.const import FileCheckConst @@ -136,7 +137,7 @@ def run_parallel_ut(config): for file in config.api_files: check_link(file) try: - os.remove(file) + remove_path(file) except FileNotFoundError: logger.warning(f"File not found and could not be deleted: {file}") diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 69b7d20ef..7a7b5e126 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -287,6 +287,16 @@ def save_pt(tensor, filepath): change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) +def load_npy(npy_path): + npy_path = os.path.realpath(npy_path) + check_file_or_directory_path(npy_path) + try: + data_np = np.load(npy_path) + except Exception as e: + raise RuntimeError(f"load npy file {npy_path} failed") from e + return data_np + + def _create_logger(level=logging.INFO): logger_ = logging.getLogger() logger_.setLevel(level) -- Gitee From 469acd22c39b3dae56505d519a17eaf9973333e9 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Tue, 27 Aug 2024 16:32:04 +0800 Subject: [PATCH 512/791] Dynamic import ssl --- .../msprobe/pytorch/api_accuracy_checker/common/utils.py | 4 ++++ .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 4 ++-- .../api_accuracy_checker/tensor_transport_layer/attl.py | 5 +---- .../api_accuracy_checker/tensor_transport_layer/client.py | 5 +++-- .../api_accuracy_checker/tensor_transport_layer/server.py | 5 +++-- debug/accuracy_tools/msprobe/pytorch/service.py | 3 ++- 6 files changed, 15 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index fdd37337d..00e689982 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -16,6 +16,7 @@ """ import os import re +from collections import namedtuple import torch @@ -31,6 +32,9 @@ from msprobe.core.common.file_check import FileChecker, FileOpen, change_mode, c from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.utils import CompareException +ApiData = namedtuple('ApiData', ['name', 'args', 'kwargs', 'result', 'step', 'rank'], + defaults=['unknown', None, None, None, 0, 0]) + class DumpException(CompareException): pass diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 8930e853a..b0703331c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -21,7 +21,7 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Me get_validated_result_csv_path, get_validated_details_csv_path, exec_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ - initialize_save_path, UtDataProcessor + initialize_save_path, UtDataProcessor, ApiData from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig @@ -32,7 +32,7 @@ from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import get_json_contents from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.const import Const, FileCheckConst, CompareConst -from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTL, ATTLConfig, ApiData, move2device_exec +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTL, ATTLConfig, move2device_exec from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.device_dispatch import ConsumerDispatcher diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index 9ff0ad703..ecf8527e5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -5,20 +5,17 @@ import re from pathlib import Path from multiprocessing import Queue from typing import Optional, Union, Dict, Any -from collections import namedtuple from dataclasses import dataclass import torch +from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.client import TCPClient from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server import TCPServer from msprobe.pytorch.common.utils import logger from msprobe.pytorch.common.utils import save_pt from msprobe.core.common.utils import remove_path - -ApiData = namedtuple('ApiData', ['name', 'args', 'kwargs', 'result', 'step', 'rank'], - defaults=['unknown', None, None, None, 0, 0]) BufferType = Union[ApiData, Dict[str, Any], str] # Union[Tensor, Tuple[Optional[Tensor]]] diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py index 368b0229d..9d5090c5e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/client.py @@ -9,8 +9,7 @@ from queue import Queue from threading import Thread from typing import Union -from OpenSSL import SSL -from twisted.internet import ssl, reactor, protocol, endpoints +from twisted.internet import reactor, protocol, endpoints from twisted.protocols.basic import FileSender from msprobe.pytorch.common.utils import logger @@ -90,6 +89,8 @@ class TCPClient: self.factory = MessageClientFactory() self.factory.protocol = cur_protocol if self.tls_path: + from OpenSSL import SSL + from twisted.internet import ssl client_key = os.path.join(self.tls_path, "client.key") client_crt = os.path.join(self.tls_path, "client.crt") client_context_factory = ssl.DefaultOpenSSLContextFactory(client_key, client_crt, SSL.TLSv1_2_METHOD) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py index 521f8d37f..93040d07f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/server.py @@ -5,8 +5,7 @@ import time import io from threading import Thread -from OpenSSL import SSL -from twisted.internet import ssl, reactor, protocol, endpoints +from twisted.internet import reactor, protocol, endpoints from msprobe.pytorch.common.utils import logger from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.ssl_config import cipher_list @@ -29,6 +28,8 @@ class TCPServer: self.factory.protocol = self.build_protocol if self.tls_path: + from OpenSSL import SSL + from twisted.internet import ssl server_key = os.path.join(self.tls_path, "server.key") server_crt = os.path.join(self.tls_path, "server.crt") server_context_factory = ssl.DefaultOpenSSLContextFactory(server_key, server_crt, SSL.TLSv1_2_METHOD) diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 980c7d840..350ae199f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -17,7 +17,7 @@ from msprobe.pytorch.hook_module import remove_dropout from msprobe.pytorch.hook_module.api_registry import api_register from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.module_processer import ModuleProcesser -from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTLConfig, ATTL, ApiData +from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' HookFn = namedtuple('hookFn', ['pre_hook', 'forward_hook', 'backward_hook', 'forward_hook_torch_version_below_2']) @@ -223,6 +223,7 @@ class Service: def attl_init(self): if self.config.online_run_ut: + from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTLConfig, ATTL attl_config = ATTLConfig(is_benchmark_device=False, connect_ip=self.config.host, connect_port=self.config.port, -- Gitee From 3314461d747cb3db081faf7092e1b342dfe428cf Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 17:34:18 +0800 Subject: [PATCH 513/791] renew_manual --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6a9b221a1..44d7d2854 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ 3. [advisor](./profiler/advisor) - 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyThon 场景性能数据进行分析，并输出性能调优建议。 + 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyTorch 场景性能数据进行分析，并输出性能调优建议。 ## [Tensorboard](./plugins/tensorboard-plugins/tb_plugin) -- Gitee From 681226f5390e83cc7d358d51cb2b0ee4817988f8 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 19:23:50 +0800 Subject: [PATCH 514/791] renew_manual --- debug/accuracy_tools/msprobe/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 0994974c8..9720b9074 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -76,7 +76,7 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. 本功能用于采集梯度数据并进行梯度相似度比对，可以精准定位出现问题的 step。 -[兼容 PyTorch 和 MindSpore 框架的梯度监测](./docs/17.grad_probe.md) +[兼容 PyTorch 和 MindSpore 框架的梯度监测](./docs/17.grad_probe.md)（更新中） ## 🌟 新版本特性 @@ -91,12 +91,12 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. ## 📑 补充材料 -[msprobe 精度工具标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md) +[msprobe 精度工具标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md)（更新中） -[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md) +[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md)（更新中） ## ❓ FAQ -[FAQ for PyTorch](./docs/FAQ_PyTorch.md) +[FAQ for PyTorch](./docs/FAQ_PyTorch.md)（更新中） FAQ for MindSpore -- Gitee From 69ca20f526677fe33c37958fb44aa978b98c207b Mon Sep 17 00:00:00 2001 From: qianggee Date: Tue, 27 Aug 2024 11:24:39 +0000 Subject: [PATCH 515/791] remove personal info --- .../kj600/kj600/unittest/test_anomaly_inform.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/kj600/kj600/unittest/test_anomaly_inform.py b/debug/accuracy_tools/kj600/kj600/unittest/test_anomaly_inform.py index 1ad76b919..d1b73d7e8 100644 --- a/debug/accuracy_tools/kj600/kj600/unittest/test_anomaly_inform.py +++ b/debug/accuracy_tools/kj600/kj600/unittest/test_anomaly_inform.py @@ -13,8 +13,8 @@ class TestAnomalyInform(unittest.TestCase): anomaly_inform.run(exception_message, job_id) def test_email_inform(self): - inform_args = {"inform": {"recipient": "email", "send_email_address": "xueyuqing@huawei.com", "receive_email_address": "xueyuqing@huawei.com", - "send_email_username": "x30021831", "send_email_password": "********", + inform_args = {"inform": {"recipient": "email", "send_email_address": "test@huawei.com", "receive_email_address": "test@huawei.com", + "send_email_username": "foo", "send_email_password": "********", "smtp_server": "smtpscn.huawei.com", "smtp_port": "587"}} anomaly_inform = AnomalyInformFactory.create_informer(**inform_args["inform"]) exception_message = '\x1b[93m> Rule AnomalyTurbulence reports anomaly signal in language_model.encoder.layers.0.self_attention.query_key_value.weight/0/exp_avg_sq_min at step 49.\x1b[0m' -- Gitee From 7613405ef02f982412b40b48e6ff90baa6235042 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 19:42:48 +0800 Subject: [PATCH 516/791] del. space --- debug/accuracy_tools/msprobe/core/common/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 0e5129e75..3dd2ee064 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -16,7 +16,7 @@ class BaseLogger: full_msg = f"{current_time} ({pid}) [{level}] {msg}" print(full_msg, end=end) sys.stdout.flush() - + def get_rank(self): return self.rank -- Gitee From 02edecc860d2a1f016e42d45153c94b6f359e0c7 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 20:59:17 +0800 Subject: [PATCH 517/791] modify log test --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 31d56acb0..3b44fc1db 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -24,11 +24,11 @@ class TestLog(TestCase): @patch("msprobe.core.common.log.print") def test__print_log(self, mock_print): logger._print_log("level", "msg") - self.assertIn("[level] msg", mock_print.call_args[0][0]) + self.assertIn("[level] msg".strip(), mock_print.call_args[0][0].strip()) self.assertEqual("\n", mock_print.call_args[1].get("end")) logger._print_log("level", "msg", end="end") - self.assertIn("[level] msg", mock_print.call_args[0][0]) + self.assertIn("[level] msg".strip(), mock_print.call_args[0][0].strip()) self.assertEqual("end", mock_print.call_args[1].get("end")) @patch.object(BaseLogger, "_print_log") -- Gitee From 3cedd35e5a028d6b1543322f48bac34682d9ff02 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 21:36:06 +0800 Subject: [PATCH 518/791] renew --- debug/accuracy_tools/msprobe/core/common/const.py | 1 - .../accuracy_tools/msprobe/core/data_dump/data_collector.py | 4 ++-- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 496243640..723b8549e 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,6 @@ class MsgConst: """ Class for log messages const """ - SPACE_LENGTH = 20 LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index d399ce15f..061911906 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -3,7 +3,7 @@ import os from msprobe.core.data_dump.scope import build_scope, ListScope from msprobe.core.data_dump.json_writer import DataWriter from msprobe.core.common.log import logger -from msprobe.core.common.const import Const, MsgConst +from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.factory import DataProcessorFactory @@ -130,7 +130,7 @@ class DataCollector: if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) - logger.info(msg+' '*MsgConst.SPACE_LENGTH, end='\r') + logger.info(msg, end='\r') if use_buffer: self.data_writer.flush_data_when_buffer_is_full() else: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 3b44fc1db..bc120f62e 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -24,11 +24,11 @@ class TestLog(TestCase): @patch("msprobe.core.common.log.print") def test__print_log(self, mock_print): logger._print_log("level", "msg") - self.assertIn("[level] msg".strip(), mock_print.call_args[0][0].strip()) + self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("\n", mock_print.call_args[1].get("end")) - + logger._print_log("level", "msg", end="end") - self.assertIn("[level] msg".strip(), mock_print.call_args[0][0].strip()) + self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("end", mock_print.call_args[1].get("end")) @patch.object(BaseLogger, "_print_log") -- Gitee From 7f273a678896fcbb1c989f47697dd37ae238f8cf Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 21:37:58 +0800 Subject: [PATCH 519/791] renew --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index bc120f62e..31d56acb0 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -26,7 +26,7 @@ class TestLog(TestCase): logger._print_log("level", "msg") self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("\n", mock_print.call_args[1].get("end")) - + logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("end", mock_print.call_args[1].get("end")) -- Gitee From 8106e4d64be7c06cf7d9992c35bde620d526987e Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 21:52:43 +0800 Subject: [PATCH 520/791] renew --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 31d56acb0..161bc3346 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -25,11 +25,11 @@ class TestLog(TestCase): def test__print_log(self, mock_print): logger._print_log("level", "msg") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("\n", mock_print.call_args[1].get("end")) + self.assertEqual("\n", mock_print.call_args.kwargs.get("end", None)) logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("end", mock_print.call_args[1].get("end")) + self.assertEqual("end", mock_print.call_args.kwargs.get("end", None)) @patch.object(BaseLogger, "_print_log") def test_print_info_log(self, mock__print_log): -- Gitee From 4c8c6c5e3ba1a509de7f0a69a7fb5f948556aa26 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 21:59:09 +0800 Subject: [PATCH 521/791] renew --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 161bc3346..315aa558c 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -25,11 +25,11 @@ class TestLog(TestCase): def test__print_log(self, mock_print): logger._print_log("level", "msg") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("\n", mock_print.call_args.kwargs.get("end", None)) + self.assertEqual("\n", mock_print.call_args[1].get("end", None)) logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("end", mock_print.call_args.kwargs.get("end", None)) + self.assertEqual("end", mock_print.call_args[1].get("end", None)) @patch.object(BaseLogger, "_print_log") def test_print_info_log(self, mock__print_log): -- Gitee From bd34c54d0f90a35aeec335487e106e8d2c7d7443 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 22:10:43 +0800 Subject: [PATCH 522/791] renew --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 315aa558c..d769f381f 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -25,11 +25,11 @@ class TestLog(TestCase): def test__print_log(self, mock_print): logger._print_log("level", "msg") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("\n", mock_print.call_args[1].get("end", None)) + self.assertEqual("\n", mock_print.call_args[1].get("end")) logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("end", mock_print.call_args[1].get("end", None)) + self.assertEqual("\r", mock_print.call_args[1].get("end")) @patch.object(BaseLogger, "_print_log") def test_print_info_log(self, mock__print_log): -- Gitee From 5e4c085fd82d661b06b33d113506fa04940b461a Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 27 Aug 2024 22:12:38 +0800 Subject: [PATCH 523/791] renew --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index d769f381f..7a94e1faf 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -27,7 +27,7 @@ class TestLog(TestCase): self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("\n", mock_print.call_args[1].get("end")) - logger._print_log("level", "msg", end="end") + logger._print_log("level", "msg", end="\r") self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("\r", mock_print.call_args[1].get("end")) -- Gitee From 132073dcbe452138d0fe0d23b043e55e6f00918d Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 28 Aug 2024 10:05:11 +0800 Subject: [PATCH 524/791] codeclean --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index b0703331c..0ab1c02e6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -21,7 +21,7 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Me get_validated_result_csv_path, get_validated_details_csv_path, exec_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ - initialize_save_path, UtDataProcessor, ApiData + initialize_save_path, UtDataProcessor from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig -- Gitee From a8c59f79ca2915c940210671cbe2fc18088ab67b Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 28 Aug 2024 10:05:46 +0800 Subject: [PATCH 525/791] codeclean --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index f866b8637..c77078366 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -21,7 +21,7 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Me get_validated_result_csv_path, get_validated_details_csv_path, exec_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ - initialize_save_path, UtDataProcessor, extract_basic_api_segments + initialize_save_path, UtDataProcessor, extract_basic_api_segments, ApiData from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig -- Gitee From 8775a8a0f56058ec986f36b37e5523603457bc44 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 10:54:22 +0800 Subject: [PATCH 526/791] renew --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 7a94e1faf..31d56acb0 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -27,9 +27,9 @@ class TestLog(TestCase): self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("\n", mock_print.call_args[1].get("end")) - logger._print_log("level", "msg", end="\r") + logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("\r", mock_print.call_args[1].get("end")) + self.assertEqual("end", mock_print.call_args[1].get("end")) @patch.object(BaseLogger, "_print_log") def test_print_info_log(self, mock__print_log): -- Gitee From c6dfb596559665f374197a3a305bf3becf3bb281 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Wed, 28 Aug 2024 10:57:18 +0800 Subject: [PATCH 527/791] fix tuple_tensor case --- .../api_accuracy_checker/compute_element.py | 16 ++++++--- .../test_compute_element.py | 33 +++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 89198f61b..fa78f2195 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -104,12 +104,15 @@ class ComputeElement: ''' Args: get_origin: boolean - get_mindspore_tensor: boolean + tensor_platform: str, Union["mindspore", "pytorch"] Return: - parameter: Union[int, float, str, slice,tuple, torch.Tensor, mindspore.Tensor] + parameter: Union[int, float, str, slice, tuple, torch.Tensor, mindspore.Tensor] ''' - if isinstance(self.parameter, self.supported_parameter_type): + if isinstance(self.parameter, tuple): + return tuple([compute_element.get_parameter(get_origin=get_origin, tensor_platform=tensor_platform) + for compute_element in self.parameter]) + elif isinstance(self.parameter, self.supported_parameter_type): parameter_tmp = self.parameter elif isinstance(self.parameter, MstensorMetaData): mstensor_meta_data = self.parameter @@ -157,7 +160,6 @@ class ComputeElement: ''' Args: compute_element_info: Union[list, dict] - is_constructed: boolean Return: void @@ -167,7 +169,7 @@ class ComputeElement: if isinstance(compute_element_info, list): self.shape = tuple() self.dtype_str = TUPLE_TYPE_STR - self.parameter = tuple(ComputeElement(compute_element_info=sub_info).get_parameter() + self.parameter = tuple(ComputeElement(compute_element_info=sub_info) for sub_info in compute_element_info) else: type_str = check_and_get_from_json_dict(compute_element_info, "type", "type field in api_info.json", @@ -218,6 +220,10 @@ class ComputeElement: elif isinstance(parameter, torch.Tensor): self.shape = tuple(parameter.shape) self.dtype_str = torch_dtype_to_dtype_str.get(parameter.dtype) + elif isinstance(parameter, tuple): + self.shape = tuple() + self.dtype_str = TUPLE_TYPE_STR + self.parameter = tuple([ComputeElement(parameter=param) for param in parameter]) else: self.shape = tuple() self.dtype_str = \ diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py index c64bf9952..d702c1b3b 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -141,5 +141,38 @@ class TestComputeElement(unittest.TestCase): self.assertEqual(compute_element.get_shape(), tuple()) self.assertEqual(compute_element.get_dtype(), INT_TYPE_STR) + def test_init_with_compute_element_info_tuple_tensor(self): + global_context.is_constructed = False + compute_element_info = [ + [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape":[2, 3], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + }, + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape":[2, 3], + "Max": 3.0, + "Min": 1.0, + "data_name": "input.npy" + } + ] + ] + compute_element = ComputeElement(compute_element_info=compute_element_info) + torch_parameter = compute_element.get_parameter(get_origin=False, tensor_platform="pytorch") + mindspore_parameter = compute_element.get_parameter(get_origin=False, tensor_platform="mindspore") + assert isinstance(torch_parameter, tuple) + assert isinstance(mindspore_parameter, tuple) + for para in torch_parameter: + assert (para == self.torch_tensor).all() + + for para in mindspore_parameter: + assert (para == self.ms_tensor).all() + if __name__ == '__main__': unittest.main() -- Gitee From bf00133fffb8ed19dc7ea3521ec44a95778655fb Mon Sep 17 00:00:00 2001 From: gitee Date: Wed, 28 Aug 2024 11:00:13 +0800 Subject: [PATCH 528/791] fix ellipsis problem --- .../compare/api_precision_compare.py | 58 ++++++++++++------- .../run_ut/data_generate.py | 2 + 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 2774c2cd9..1364701d3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -339,30 +339,19 @@ def analyse_csv(npu_data, gpu_data, config): raise CompareException(CompareException.INVALID_DATA_ERROR, msg) row_gpu = row_gpu.iloc[0] new_status = CompareConst.SPACE - # 当前API的输出为空（例如反向过程中requires_grad=False）,跳过比对 - if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE].isspace(): + try: + new_status = get_api_status(row_npu, row_gpu, api_name, compare_column) + except Exception as err: + logger.error(f"Get api status error: {str(err)}") compare_column.api_name = full_api_name_with_direction_status compare_column.compare_result = CompareConst.SKIP - compare_column.compare_message = row_npu[ApiPrecisionCompareColumn.MESSAGE] - new_status = CompareConst.SKIP - write_detail_csv(compare_column.to_column_value(), config.details_csv_path) - else: - compare_column.api_name = full_api_name_with_direction_status - if api_name in thousandth_standard_api: - new_status = record_thousandth_threshold_result(compare_column, row_npu) - elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or \ - api_name in binary_standard_api: - new_status = record_binary_consistency_result(api_name, compare_column, row_npu) - elif api_name in absolute_standard_api: - new_status = record_absolute_threshold_result(compare_column, row_npu) - elif api_name in ulp_standard_api and \ - row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in ULP_COMPARE_SUPPORT_LIST: - us = ULPStandard(full_api_name_with_direction_status, row_npu, row_gpu) - new_status = record_ulp_compare_result(compare_column, us) - elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: - bs = BenchmarkStandard(full_api_name_with_direction_status, row_npu, row_gpu) - new_status = record_benchmark_compare_result(compare_column, bs) + compare_column.compare_message = str(err) write_detail_csv(compare_column.to_column_value(), config.details_csv_path) + write_csv([[full_api_name_with_direction_status, CompareConst.SKIP, CompareConst.SKIP, str(err)]], + config.result_csv_path) + continue + + write_detail_csv(compare_column.to_column_value(), config.details_csv_path) if last_api_name is not None and api_full_name != last_api_name: if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: @@ -407,6 +396,33 @@ def analyse_csv(npu_data, gpu_data, config): print_test_success(last_api_full_name, forward_result, backward_result) +def get_api_status(row_npu, row_gpu, api_name, compare_column): + full_api_name_with_direction_status = row_npu[ApiPrecisionCompareColumn.API_NAME] + # 当前API的输出为空（例如反向过程中requires_grad=False）,跳过比对 + if row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE].isspace(): + compare_column.api_name = full_api_name_with_direction_status + compare_column.compare_result = CompareConst.SKIP + compare_column.compare_message = row_npu[ApiPrecisionCompareColumn.MESSAGE] + new_status = CompareConst.SKIP + else: + compare_column.api_name = full_api_name_with_direction_status + if api_name in thousandth_standard_api: + new_status = record_thousandth_threshold_result(compare_column, row_npu) + elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] not in BINARY_COMPARE_UNSUPPORT_LIST or \ + api_name in binary_standard_api: + new_status = record_binary_consistency_result(api_name, compare_column, row_npu) + elif api_name in absolute_standard_api: + new_status = record_absolute_threshold_result(compare_column, row_npu) + elif api_name in ulp_standard_api and \ + row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in ULP_COMPARE_SUPPORT_LIST: + us = ULPStandard(full_api_name_with_direction_status, row_npu, row_gpu) + new_status = record_ulp_compare_result(compare_column, us) + elif row_npu[ApiPrecisionCompareColumn.DEVICE_DTYPE] in BENCHMARK_COMPARE_SUPPORT_LIST: + bs = BenchmarkStandard(full_api_name_with_direction_status, row_npu, row_gpu) + new_status = record_benchmark_compare_result(compare_column, bs) + return new_status + + def print_test_success(api_full_name, forward_result, backward_result): is_fwd_success = (forward_result == CompareConst.PASS) is_bwd_success = (backward_result == CompareConst.PASS or backward_result == CompareConst.SPACE) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index 6f2b4801f..dc94e59b1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -76,6 +76,8 @@ def gen_data(info, api_name, need_grad, convert_type, real_data_path=None): data = info.get('value') if info.get("type") == "slice": data = slice(*data) + if info.get("type") == "ellipsis": + data = ... return data -- Gitee From 372cc00227e6c079ac8024b0f191d8486b7572de Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 11:14:30 +0800 Subject: [PATCH 529/791] renew --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 31d56acb0..8d3280ce4 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -26,6 +26,10 @@ class TestLog(TestCase): logger._print_log("level", "msg") self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("\n", mock_print.call_args[1].get("end")) + + logger._print_log("level", "msg", end='\r') + self.assertIn("[level] msg", mock_print.call_args[0][0]) + self.assertEqual("\r", mock_print.call_args[1].get("end")) logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) -- Gitee From 30f11dcc2c00c274372cb972a9d1f9e23b600d14 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Wed, 28 Aug 2024 10:45:30 +0800 Subject: [PATCH 530/791] correct default jit_level --- .../msprobe/mindspore/common/const.py | 5 ++++ .../mindspore/debugger/precision_debugger.py | 23 +++++++++++++++++-- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/common/const.py b/debug/accuracy_tools/msprobe/mindspore/common/const.py index 454b48372..ede13e901 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/const.py @@ -16,6 +16,11 @@ class Const: PYNATIVE_MODE = "pynative" GRAPH_GE_MODE = "graph_ge" GRAPH_KBYK_MODE = "graph_kbyk" + JIT_LEVEL = "jit_level" + JIT_LEVEL_O0 = "O0" + JIT_LEVEL_O1 = "O1" + JIT_LEVEL_O2 = "O2" + ASCEND_910A = "ascend910" class FreeBenchmarkConst: diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 0b3fec632..d9357ae5c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -1,6 +1,7 @@ import os import mindspore as ms +from mindspore._c_expression import MSContext from msprobe.mindspore.service import Service from msprobe.mindspore.ms_config import parse_json_config @@ -44,8 +45,16 @@ class PrecisionDebugger: @staticmethod def _get_execution_mode(): + jit_level = ms.context.get_jit_config().get(MsConst.JIT_LEVEL) + jit_level = jit_level if jit_level else ms.get_context(MsConst.JIT_LEVEL) + if not jit_level: + if MSContext.get_instance().get_ascend_soc_version() == MsConst.ASCEND_910A: + jit_level = MsConst.JIT_LEVEL_O2 + else: + jit_level = MsConst.JIT_LEVEL_O0 + if ms.get_context("mode") == ms.GRAPH_MODE: - if ms.context.get_jit_config().get("jit_level") == "O2" or ms.get_context("jit_level") == "O2": + if jit_level == MsConst.JIT_LEVEL_O2: return MsConst.GRAPH_GE_MODE else: return MsConst.GRAPH_KBYK_MODE @@ -61,7 +70,7 @@ class PrecisionDebugger: return instance.config.execution_mode = instance._get_execution_mode() - if instance.config.execution_mode == MsConst.PYNATIVE_MODE and instance.config.task != Const.FREE_BENCHMARK and instance.config.level != "kernel": + if cls._need_service(): if not instance.service: instance.service = Service(instance.config) instance.service.start(target) @@ -105,3 +114,13 @@ class PrecisionDebugger: if instance.task != Const.GRAD_PROBE: return instance.gm.monitor(opt) + + @classmethod + def _need_service(cls): + instance = cls._instance + if not instance: + raise Exception("No instance of PrecisionDebugger found.") + if instance.config.execution_mode != MsConst.PYNATIVE_MODE: + return False + else: + return instance.config.task != Const.FREE_BENCHMARK and instance.config.level != MsConst.KERNEL -- Gitee From 440492c0a7c298ca2b60dc4e31ef0edc1d4d97c7 Mon Sep 17 00:00:00 2001 From: gitee Date: Wed, 28 Aug 2024 11:34:45 +0800 Subject: [PATCH 531/791] fix --- .../api_accuracy_checker/run_ut/data_generate.py | 3 ++- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 10 ---------- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index 14f8c838b..36fe2df46 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -25,7 +25,8 @@ from msprobe.pytorch.api_accuracy_checker.common.utils import check_object_type, CompareException from msprobe.core.common.file_check import FileChecker from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import load_pt, load_npy +from msprobe.pytorch.common.utils import load_pt +from msprobe.core.common.utils import load_npy from msprobe.core.common.const import Const, FileCheckConst TORCH_TYPE = ["torch.device", "torch.dtype"] diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 7a7b5e126..69b7d20ef 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -287,16 +287,6 @@ def save_pt(tensor, filepath): change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) -def load_npy(npy_path): - npy_path = os.path.realpath(npy_path) - check_file_or_directory_path(npy_path) - try: - data_np = np.load(npy_path) - except Exception as e: - raise RuntimeError(f"load npy file {npy_path} failed") from e - return data_np - - def _create_logger(level=logging.INFO): logger_ = logging.getLogger() logger_.setLevel(level) -- Gitee From 640fffe0916d92ab30837d4ce78a85b860991932 Mon Sep 17 00:00:00 2001 From: wuyuhan Date: Tue, 27 Aug 2024 20:25:22 +0800 Subject: [PATCH 532/791] =?UTF-8?q?1.=20=E9=9B=86=E7=BE=A4=E5=AF=B9?= =?UTF-8?q?=E6=AF=94bug=20fix=EF=BC=9B2.=20=E6=97=A0=E6=A0=87=E6=9D=86?= =?UTF-8?q?=E6=98=AF=E9=9B=86=E7=BE=A4=E5=86=85=E9=83=A8=E5=BF=AB=E6=85=A2?= =?UTF-8?q?=E5=8D=A1=E5=AF=B9=E6=AF=94?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../advisor/analyzer/analyzer_controller.py | 83 +++++++----- .../analyzer/cluster/slow_link_analyser.py | 126 ------------------ .../analyzer/cluster/slow_link_analyzer.py | 8 +- .../analyzer/cluster/slow_rank_analyser.py | 112 ---------------- .../analyzer/cluster/slow_rank_analyzer.py | 14 +- .../analyzer/comparison/comparison_checker.py | 11 +- profiler/advisor/common/timeline/event.py | 4 +- profiler/advisor/result/result.py | 5 + profiler/advisor/utils/utils.py | 16 ++- 9 files changed, 90 insertions(+), 289 deletions(-) delete mode 100644 profiler/advisor/analyzer/cluster/slow_link_analyser.py delete mode 100644 profiler/advisor/analyzer/cluster/slow_rank_analyser.py diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py index 7f071978f..bed45679f 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -13,11 +13,12 @@ sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__f from profiler.advisor.analyzer.cluster.slow_rank_analyzer import SlowRankAnalyzer from profiler.advisor.analyzer.cluster.slow_link_analyzer import SlowLinkAnalyzer from profiler.advisor.analyzer.computation.pp_stage_computation_analyzer import PPStageComputationAnalyzer +from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer from profiler.advisor.config.config import Config from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.common.async_analysis_status import AsyncAnalysisStatus from profiler.advisor.dataset.cluster.cluster_dataset import ClusterDataset -from profiler.advisor.utils.utils import Timer, safe_index, safe_division +from profiler.advisor.utils.utils import Timer, safe_index_value, safe_division, safe_index from profiler.advisor.interface.interface import Interface from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from profiler.prof_common.path_manager import PathManager @@ -56,12 +57,13 @@ class AnalyzerController: logger.error("Error dimension %s for cluster statistics data, optionals are %s.", dimension, headers) return None, None, None - dimension_index = safe_index(headers, dimension) + dimension_index = safe_index_value(headers, dimension) diff_record = [] # 对比目标profiling和benchmark profiling 每张卡的计算和下发和带宽，取计算、下发、带宽差异最大的卡进行下一步分析 for target_row_data, benchmark_row_data in zip(target_cluster_statistic_data, benchmark_cluster_statistic_data): target_data = safe_index(target_row_data, dimension_index) benchmark_data = safe_index(benchmark_row_data, dimension_index) + if not isinstance(target_data, (int, float)) or not isinstance(benchmark_data, (int, float)): continue diff_record.append(target_data - benchmark_data) @@ -71,14 +73,16 @@ class AnalyzerController: return None, None, None value = max(diff_record) if get_max else min(diff_record) - value_index = safe_index(diff_record, value) + value_index = safe_index_value(diff_record, value) + + step_value_index = safe_index_value(headers, "step") + rank_id_value_index = safe_index_value(headers, "rank_id") - step_value_index = safe_index(headers, "step") - rank_id_value_index = safe_index(headers, "rank_id") step = safe_index(safe_index(target_cluster_statistic_data, value_index, []), step_value_index) benchmark_step = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), step_value_index) target_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) - benchmark_rank_id = safe_index(safe_index(target_cluster_statistic_data, value_index, []), rank_id_value_index) + benchmark_rank_id = safe_index(safe_index(benchmark_cluster_statistic_data, value_index, []), + rank_id_value_index) if target_rank_id != benchmark_rank_id: logger.error( @@ -168,11 +172,11 @@ class AnalyzerController: self.slow_rank_analyzer.optimize(template_key=Interface.OVERALL) self.slow_link_analyzer.optimize(template_key=Interface.OVERALL) else: - from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSummaryAnalyzer overall_analyzer = OverallSummaryAnalyzer(profiling_path) overall_analyzer.optimize() - def schedule_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None): + def schedule_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None, + **kwargs): # 任意单卡的下发分析 kwargs = copy.deepcopy(self.kwargs) @@ -190,7 +194,7 @@ class AnalyzerController: return job_list def computation_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, - benchmark_step=None, stage=None): + benchmark_step=None, stage=None, **kwargs): # 任意单卡的计算分析 kwargs = copy.deepcopy(self.kwargs) @@ -250,15 +254,25 @@ class AnalyzerController: job_list = [] global_step_rank = self.slow_rank_analyzer.get_global_step_rank(SlowRankAnalyzer.FREE) slow_rank_id = global_step_rank.get("maximum", {}).get("rank_id") or self.default_rank_id + fast_rank_id = global_step_rank.get("minimum", {}).get("rank_id") or self.default_rank_id slow_step = global_step_rank.get("maximum", {}).get("step") - analysis_profiling_path = self._get_profiling_path_by_rank(profiling_path, slow_rank_id) + fast_step = global_step_rank.get("minimum", {}).get("step") info_msg = f"Maximum free for rank {slow_rank_id}" if slow_step: info_msg += f" and step {slow_step}" logger.info(info_msg) - job_list += self.schedule_analysis(analysis_profiling_path, step=slow_step) + kwargs = dict(profiling_path=self._get_profiling_path_by_rank(profiling_path, slow_rank_id), + benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, fast_rank_id), + step=slow_step, benchmark_step=fast_step, + rank=slow_rank_id, benchmark_rank=fast_rank_id, + compare_mode=CompareConstant.API_COMPARE) + + job_list += self.schedule_analysis(**kwargs) + + if self.kwargs.get("benchmark_profiling_path") is None and slow_rank_id != fast_rank_id: + job_list += self._profiling_comparison([kwargs]) return job_list def cluster_communication_analysis(self, profiling_path): @@ -310,6 +324,8 @@ class AnalyzerController: for stage, step_rank_info in stage_step_rank.items(): rank_id = step_rank_info.get("maximum", {}).get("rank_id") step = step_rank_info.get("maximum", {}).get("step") + benchmark_rank_id = step_rank_info.get("minimum", {}).get("rank_id") + benchmark_step = step_rank_info.get("minimum", {}).get("step") info_msg = f"For {stage}, slow rank is {rank_id}" if step: @@ -318,16 +334,20 @@ class AnalyzerController: stages_profiling_path.append( dict( - stage=stage, - rank_id=rank_id, - step=step, - profiling_path=self._get_profiling_path_by_rank(profiling_path, rank_id) + stage=stage, rank=rank_id, step=step, benchmark_rank=benchmark_rank_id, + benchmark_step=benchmark_step, + profiling_path=self._get_profiling_path_by_rank(profiling_path, rank_id), + benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, benchmark_rank_id), + compare_mode=CompareConstant.KERNEL_COMPARE ) ) Interface.add_analyzer(Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, PPStageComputationAnalyzer) - kwargs = {"stages_profiling_path": stages_profiling_path, "profiling_path": profiling_path} + compute_analysis_kwargs = {"stages_profiling_path": stages_profiling_path, "profiling_path": profiling_path} - job_list.append((Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, Interface(**kwargs), kwargs)) + job_list.append((Interface.COMPUTATION, SupportedScopes.STAGE_COMPUTE, Interface(**compute_analysis_kwargs), + compute_analysis_kwargs)) + if self.kwargs.get("benchmark_profiling_path") is None: + job_list += self._profiling_comparison(stages_profiling_path) else: # 不区分stage，对所有卡取Min max进行分析 logger.info("Without pipeline parallel stage, Global analysis steps and ranks is %s", @@ -339,21 +359,23 @@ class AnalyzerController: fast_step = global_step_rank.get("minimum", {}).get("step") info_msg = f"Maximum computation time for rank {slow_rank_id}" - if slow_step: + if slow_step is not None: info_msg += f" and step {slow_step}, " - if fast_rank_id: + if fast_rank_id is not None: info_msg += f"minimum computation time for rank {fast_rank_id}" - if fast_step: + if fast_step is not None: info_msg += f" and step {fast_step}" logger.info(info_msg) - job_list += self.computation_analysis( - self._get_profiling_path_by_rank(profiling_path, slow_rank_id), - self._get_profiling_path_by_rank(profiling_path, fast_rank_id), - slow_step, - fast_step - ) + kwargs = dict(profiling_path=self._get_profiling_path_by_rank(profiling_path, slow_rank_id), + benchmark_profiling_path=self._get_profiling_path_by_rank(profiling_path, fast_rank_id), + step=slow_step, benchmark_step=fast_step, rank=slow_rank_id, benchmark_rank=fast_rank_id, + compare_mode=CompareConstant.KERNEL_COMPARE) + job_list += self.computation_analysis(**kwargs) + rank_id_valid = slow_rank_id is not None and fast_rank_id is not None and fast_rank_id != slow_rank_id + if self.kwargs.get("benchmark_profiling_path") is None and rank_id_valid: + job_list += self._profiling_comparison([kwargs]) return job_list def cluster_memory_analysis(self, profiling_path): @@ -402,11 +424,14 @@ class AnalyzerController: self._is_cluster = self._is_cluster_profiling(profiling_path) if benchmark_profiling_path: + # 构建benchmark profiling的map，用于根据rank获取profiling路径，否则无法进行比对 _ = self._is_cluster_profiling(benchmark_profiling_path) if not self._is_cluster: job_list = self.single_rank_analysis(profiling_path, benchmark_profiling_path) else: + self.slow_rank_analyzer = SlowRankAnalyzer(profiling_path) + self.slow_link_analyzer = SlowLinkAnalyzer(profiling_path) job_list = self.cluster_analysis(profiling_path, benchmark_profiling_path) for i, (dimension, scope, interface, kwargs) in enumerate(job_list[::-1]): @@ -495,7 +520,7 @@ class AnalyzerController: comparison_dims = [SlowRankAnalyzer.COMPUTE, SlowRankAnalyzer.FREE] comparison_modes = [CompareConstant.KERNEL_COMPARE, CompareConstant.API_COMPARE] elif isinstance(target_cluster_analyzer, SlowLinkAnalyzer): - comparison_dims = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + comparison_dims = [SlowLinkAnalyzer.SDMA_BANDWIDTH, SlowLinkAnalyzer.RDMA_BANDWIDTH] comparison_modes = [None, None] else: return job_list @@ -531,7 +556,7 @@ class AnalyzerController: compare_profiling_list.append( dict(profiling_path=rank_profiling_path, benchmark_profiling_path=rank_benchmark_profiling_path, - step=str(step) if step else step, benchmark_step=str(benchmark_step) if step else step, + step=step, benchmark_step=benchmark_step, rank=rank_id_for_comparison, benchmark_rank=rank_id_for_comparison, compare_mode=compare_mode) ) @@ -553,8 +578,6 @@ class AnalyzerController: self.default_rank_id = list(self.cluster_local_data_map[profiling_path].keys())[0] - self.slow_rank_analyzer = SlowRankAnalyzer(profiling_path) - self.slow_link_analyzer = SlowLinkAnalyzer(profiling_path) return len(self.cluster_local_data_map[profiling_path]) >= self.CLUSTER_RANK_THRESHOLD def _get_profiling_path_by_rank(self, profiling_path, rank_id=None): diff --git a/profiler/advisor/analyzer/cluster/slow_link_analyser.py b/profiler/advisor/analyzer/cluster/slow_link_analyser.py deleted file mode 100644 index 0b585cbc7..000000000 --- a/profiler/advisor/analyzer/cluster/slow_link_analyser.py +++ /dev/null @@ -1,126 +0,0 @@ -# Copyright (c) 2023, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import defaultdict -from typing import Dict, List -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common import constant -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset - - -class SlowLinkAnalyzer(BaseAnalyzer): - RDMA_TIME_MS = "RDMA time(ms)" - RDMA_SIZE_MB = "RDMA size(mb)" - SDMA_TIME_MS = "SDMA time(ms)" - SDMA_SIZE_MB = "SDMA size(mb)" - RDMA_BANDWIDTH = "RDMA bandwidth(GB/s)" - SDMA_BANDWIDTH = "SDMA bandwidth(GB/s)" - COMMUNICATION_BANDWIDTH_INFO = "Communication Bandwidth Info" - TRANSIT_TIME = "Transit Time(ms)" - TRANSIT_SIZE = "Transit Size(MB)" - SDMA = "SDMA" - RDMA = "RDMA" - SLOW_LINK_ANALYSIS = "slow_link_analysis" - dataset_cls_list = [ClusterCommunicationDataset] - - def __init__(self, collection_path, n_processes: int = 1, **kwargs): - super().__init__(collection_path, n_processes, **kwargs) - key = ClusterCommunicationDataset.get_key() - self.communication_data_class = self.get_first_data_by_key(self.dataset_list, key) - self.rank_bw_dict = self.communication_data_class.get_data() - self.result = OptimizeResult() - self.bottelneck = '' - self.suggestion = '' - self.format_datas = [] - - def optimize(self, **kwargs): - if self.rank_bw_dict is None: - print("Slow link analysis failed due to data loading failure. \ - Please check your cluster_analysis_output folder. \ - If you are not concerned about this type of data, please ignore this message.") - return self.result - self.process() - self.format_datas = self.format_details() - self.make_record() - self.make_render() - return self.result - - def process(self): - if self.rank_bw_dict: - self.produce_bottleneck(self.RDMA_BANDWIDTH) - self.produce_bottleneck(self.SDMA_BANDWIDTH) - - def produce_bottleneck(self, link_type: str): - data_list = [rank_dict.get(link_type, 0) for rank_id, rank_dict in self.rank_bw_dict.items()] - if len(data_list) > 0: - avg_bw = round(sum(data_list) / len(data_list), 3) - else: - print("The slow link (identified bottleneck) cannot provide a bottleneck \ - because the analysis data is missing bandwidth information.") - return - self.bottelneck += f'{link_type}: \n' \ - f' The average is {avg_bw}, \n' \ - f' while the maximum is {round(max(data_list), 3)}GB/s \n' \ - f' and the minimum is {round(min(data_list), 3)}GB/s. \n' \ - f' the difference is {round(max(data_list) - min(data_list), 3)}GB/s. \n' - - def format_details(self): - if not self.rank_bw_dict: - return { - "headers": [], - "data": [] - } - - details_dict = {} - headers = list({k for rank_bw_value in self.rank_bw_dict.values() for k in rank_bw_value.keys()}) - headers.sort() - data_list = [[rank_id] + [rank_bw.get(k, 0) for k in headers] for rank_id, rank_bw in self.rank_bw_dict.items()] - data_list.sort(key = lambda x: x[0]) # 按rank_id排序 - - details_dict["headers"] = ["rank_id"] + headers - details_dict["data"] = data_list - - return details_dict - - def make_record(self): - """ - make record for what and how to optimize - """ - optimization_item = OptimizeItem( - SlowLinkAnalyzer.SLOW_LINK_ANALYSIS, - self.bottelneck, - self.suggestion - ) - self.result.add(OptimizeRecord(optimization_item)) - - for i, data in enumerate(self.format_datas["data"]): - self.result.add_detail(SlowLinkAnalyzer.SLOW_LINK_ANALYSIS, self.format_datas["headers"], data) - - def make_render(self): - result_for_html = { - "Description" : self.bottelneck, - "suggestion" : self.suggestion, - "details" : [self.format_datas] - } - - self.html_render.render_template(key="cluster", - title=SlowLinkAnalyzer.SLOW_LINK_ANALYSIS, - template_dir="templates", - template_name="cluster_analysis.html", - cann_version=self.cann_version, - torch_version=self.torch_version, - result=result_for_html) \ No newline at end of file diff --git a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py b/profiler/advisor/analyzer/cluster/slow_link_analyzer.py index 438d3f55e..9653a25c1 100644 --- a/profiler/advisor/analyzer/cluster/slow_link_analyzer.py +++ b/profiler/advisor/analyzer/cluster/slow_link_analyzer.py @@ -22,7 +22,7 @@ from profiler.advisor.common import constant from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.dataset.cluster.cluster_dataset import ClusterCommunicationDataset -from profiler.advisor.utils.utils import safe_index +from profiler.advisor.utils.utils import safe_index_value logger = logging.getLogger() @@ -154,7 +154,7 @@ class SlowLinkAnalyzer(BaseAnalyzer): headers = self.format_datas.get("headers") - bindwidth_index = safe_index(headers, bindwidth_key_map.get(bindwidth_type)) + bindwidth_index = safe_index_value(headers, bindwidth_key_map.get(bindwidth_type)) if bindwidth_index is not None: data_list = [tuple_list[bindwidth_index] for tuple_list in self.format_datas.get("data", [])] @@ -167,8 +167,8 @@ class SlowLinkAnalyzer(BaseAnalyzer): max_bandwidth_index = data_list.index(max_bandwidth) min_bandwidth_index = data_list.index(min_bandwidth) - rank_id_index = safe_index(headers, "rank_id") - step_index = safe_index(headers, "step") + rank_id_index = safe_index_value(headers, "rank_id") + step_index = safe_index_value(headers, "step") if rank_id_index is None: return global_step_rank diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyser.py b/profiler/advisor/analyzer/cluster/slow_rank_analyser.py deleted file mode 100644 index f439b31f7..000000000 --- a/profiler/advisor/analyzer/cluster/slow_rank_analyser.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (c) 2023, Huawei Technologies Co., Ltd. -# All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from collections import defaultdict -from typing import Dict, List -from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer -from profiler.advisor.common import constant -from profiler.advisor.result.result import OptimizeResult -from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset - - -class SlowRankAnalyzer(BaseAnalyzer): - SLOW_RANK_ANALYSIS = "slow_rank_analysis" - RANK = "rank" - RATIO_THRESHOLD = 0.05 - BOTTLENECK_LIST = ['Computing', 'Communication', "Free"] - dataset_cls_list = [ClusterStepTraceTimeDataset] - - def __init__(self, collection_path, n_processes: int = 1, **kwargs): - super().__init__(collection_path, n_processes, **kwargs) - key = ClusterStepTraceTimeDataset.get_key() - self.step_trace_class = self.get_first_data_by_key(self.dataset_list, key) - self.step_trace_dict = self.step_trace_class.get_data() - self.result = OptimizeResult() - self.bottelneck = '' - self.suggestion = '' - self.format_datas = [] - - def optimize(self, **kwargs): - if self.step_trace_dict is None: - print("slow_rank 分析失败，原因是数据加载失败，请检查你的cluster_analysis_outpu文件夹 \ - 如不关心这类数据请忽略") - return self.result - self.process() - self.format_datas = self.format_details() - self.make_record() - self.make_render() - return self.result - - def process(self): - total_time_list = [sum(data_tuple) for rank_id, data_tuple in self.step_trace_dict.items()] - if total_time_list: - mean_total_time = sum(total_time_list) / len(total_time_list) - for i in range(len(self.BOTTLENECK_LIST)): - self.produce_bottleneck(self.step_trace_dict, i, mean_total_time) - - def produce_bottleneck(self, step_dict: dict, produce_type: int, mean_total_time: float): - data_list = [data_tuple[produce_type] for rank_id, data_tuple in step_dict.items()] - max_ratio = self.compute_max_gap_ratio(data_list, mean_total_time) - if max_ratio > self.RATIO_THRESHOLD: - self.bottelneck += f'{self.BOTTLENECK_LIST[produce_type]} \n' \ - f' has some issues in the cluster, \n' \ - f' because the max difference of {self.BOTTLENECK_LIST[produce_type]} time \n' \ - f' has reached {round(max_ratio * mean_total_time / 1000, 3)}ms. \n' - - def make_record(self): - """ - make record for what and how to optimize - """ - optimization_item = OptimizeItem( - SlowRankAnalyzer.SLOW_RANK_ANALYSIS, - self.bottelneck, - self.suggestion - ) - self.result.add(OptimizeRecord(optimization_item)) - for i, data in enumerate(self.format_datas["data"]): - self.result.add_detail(SlowRankAnalyzer.SLOW_RANK_ANALYSIS, self.format_datas["headers"], data) - - def format_details(self): - details_dict = {} - headers = ["rank_id", "compute(us)", "communication(us)", "free(us)"] - data_list = [] - for key,value in self.step_trace_dict.items(): - data_list.append([key] + value) - details_dict["headers"] = headers - details_dict["data"] = data_list - return details_dict - - def make_render(self): - result_for_html = { - "Description" : self.bottelneck, - "suggestion" : self.suggestion, - "details" : [self.format_datas] - } - - self.html_render.render_template(key="cluster", - title=SlowRankAnalyzer.SLOW_RANK_ANALYSIS, - template_dir="templates", - template_name="cluster_analysis.html", - cann_version=self.cann_version, - torch_version=self.torch_version, - result=result_for_html) - - @staticmethod - def compute_max_gap_ratio(data: list, mean: float): - if mean == 0: - return 0 - else: - return (max(data) - min(data)) / mean diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py index a1971baf9..4f5bacc49 100644 --- a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py +++ b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py @@ -20,7 +20,7 @@ from profiler.advisor.common import constant from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset -from profiler.advisor.utils.utils import safe_index +from profiler.advisor.utils.utils import safe_index_value logger = logging.getLogger() @@ -139,9 +139,9 @@ class SlowRankAnalyzer(BaseAnalyzer): headers = self.format_datas.get("headers") - dimension_index = safe_index(headers, dimension) - rank_id_index = safe_index(headers, "rank_id") - step_index = safe_index(headers, "step") + dimension_index = safe_index_value(headers, dimension) + rank_id_index = safe_index_value(headers, "rank_id") + step_index = safe_index_value(headers, "step") if dimension_index is None or rank_id_index is None: return global_step_rank @@ -172,9 +172,9 @@ class SlowRankAnalyzer(BaseAnalyzer): stage_step_rank = {} headers = self.format_datas.get("headers") - dimension_index = safe_index(headers, dimension) - rank_id_index = safe_index(headers, "rank_id") - step_index = safe_index(headers, "step") + dimension_index = safe_index_value(headers, dimension) + rank_id_index = safe_index_value(headers, "rank_id") + step_index = safe_index_value(headers, "step") if dimension_index is None or rank_id_index is None: return stage_step_rank diff --git a/profiler/advisor/analyzer/comparison/comparison_checker.py b/profiler/advisor/analyzer/comparison/comparison_checker.py index ed98f448b..ccbf20c8b 100644 --- a/profiler/advisor/analyzer/comparison/comparison_checker.py +++ b/profiler/advisor/analyzer/comparison/comparison_checker.py @@ -2,7 +2,7 @@ import logging from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord -from profiler.advisor.utils.utils import safe_index, convert_to_float +from profiler.advisor.utils.utils import safe_index_value, convert_to_float from profiler.compare_tools.compare_backend.utils.constant import Constant as CompareConstant from profiler.compare_tools.compare_interface.comparison_interface import ComparisonInterface @@ -21,8 +21,8 @@ class ComparisonChecker: self.profiling_path = profiling_path self.benchmark_profiling_path = benchmark_profiling_path - self.step = step - self.benchmark_step = benchmark_step + self.step = str(step) if step is not None else step + self.benchmark_step = str(benchmark_step) if benchmark_step is not None else benchmark_step self.rank = rank self.benchmark_rank = benchmark_rank self.compare_mode = None @@ -79,7 +79,10 @@ class ComparisonChecker: return headers = self.format_result.get(self.compare_mode, {}).get("headers", []) - diff_avg_index = safe_index(headers, self.DIFF_AVG_RATIO) + diff_avg_index = safe_index_value(headers, self.DIFF_AVG_RATIO) + if diff_avg_index is None: + logger.warning("'%s' not exsits in headers of comparison result, skip render html.", self.DIFF_AVG_RATIO) + return rows = self.format_result.get(self.compare_mode, {}).get("rows", []) sorted_rows = sorted(rows, key=lambda x: convert_to_float(x[diff_avg_index]) if diff_avg_index < len(x) else -1.0, diff --git a/profiler/advisor/common/timeline/event.py b/profiler/advisor/common/timeline/event.py index 6bd159b56..401d5efcc 100644 --- a/profiler/advisor/common/timeline/event.py +++ b/profiler/advisor/common/timeline/event.py @@ -27,8 +27,8 @@ class TimelineEvent(AdvisorDict): if not self_ts or not event_ts: return False - self_dur = self.dur if self.dur else 0.0 - event_dur = event.dur if event.dur else 0.0 + self_dur = self.dur if not isinstance(self.dur, dict) else 0.0 + event_dur = event.dur if not isinstance(event.dur, dict) else 0.0 return Decimal(self_ts) <= Decimal(event_ts) and Decimal(self_ts) + Decimal(self_dur) >= Decimal( event_ts) + Decimal(event_dur) diff --git a/profiler/advisor/result/result.py b/profiler/advisor/result/result.py index 19522ff7b..fcc623d2f 100644 --- a/profiler/advisor/result/result.py +++ b/profiler/advisor/result/result.py @@ -50,6 +50,11 @@ class ResultWriter: if len(sheet_name) > self.MAX_SHEET_NAME_LENGTH: sheet_name = sheet_name[:self.MAX_SHEET_NAME_LENGTH] + worksheet_name_list = [worksheet.name for worksheet in self.workbook.worksheets()] + if sheet_name.lower() in [name.lower() for name in worksheet_name_list]: + logger.warning("Exists worksheets %s, skip add duplicate worksheet with name '%s'", worksheet_name_list, sheet_name) + return + sheet = self.workbook.add_worksheet(sheet_name) if headers: diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index 8e6a8cba3..3cf9a35a9 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -110,7 +110,8 @@ def singleton(cls): members = inspect.getmembers(base_class) # 过滤出函数对象 - function_objs = [member[1] for member in members if inspect.isfunction(member[1]) or inspect.ismethod(member[1])] + function_objs = [member[1] for member in members if + inspect.isfunction(member[1]) or inspect.ismethod(member[1])] for function_obj in function_objs: if inspect.isfunction(function_obj) and not is_static_func(function_obj): continue @@ -410,7 +411,8 @@ def join_prof_path(root_dir: str, sub_dir: str) -> str: for root, _, _ in os.walk(root_dir, onerror=walk_error_handler): if re.match(sub_dir, os.path.basename(root)): return root - logger.debug("Fail to get profiling path %s from local path %s by regular expression matching", sub_dir, root_dir) + logger.debug("Fail to get profiling path %s from local path %s by regular expression matching", sub_dir, + root_dir) else: sub_dir = os.path.join(root_dir, sub_dir) if os.path.exists(sub_dir): @@ -585,7 +587,7 @@ def check_path_valid(path): logger.error(f"The file is not readable. path:%ss", path) return False elif os.path.getsize(path) > const.MAX_FILE_SIZE: - logger.error(f"The file size exceeds the limit. path:%ss, MAX_FILE_SIZE:%ss B",path, const.MAX_FILE_SIZE) + logger.error(f"The file size exceeds the limit. path:%ss, MAX_FILE_SIZE:%ss B", path, const.MAX_FILE_SIZE) return False return True @@ -623,13 +625,19 @@ def convert_to_float(num): return 0 -def safe_index(array, value, return_index_if_error=None): +def safe_index_value(array, value, return_index_if_error=None): if value in array: return array.index(value) return return_index_if_error +def safe_index(array, index, return_value_if_error=None): + if index < len(array): + return array[index] + return return_value_if_error + + def convert_to_int(data: any) -> int: try: int_value = int(data) -- Gitee From c88be57ea651d33ee629e6bd28e372ea175b59d7 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 13:41:21 +0800 Subject: [PATCH 533/791] renew --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 8d3280ce4..bc120f62e 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -27,10 +27,6 @@ class TestLog(TestCase): self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("\n", mock_print.call_args[1].get("end")) - logger._print_log("level", "msg", end='\r') - self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("\r", mock_print.call_args[1].get("end")) - logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("end", mock_print.call_args[1].get("end")) -- Gitee From 216715003855f6a884be9fc5589ac3d87933ef9f Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 14:27:51 +0800 Subject: [PATCH 534/791] renew --- .../docs/{05.data_dump_Pytorch.md => 05.data_dump_PyTorch.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename debug/accuracy_tools/msprobe/docs/{05.data_dump_Pytorch.md => 05.data_dump_PyTorch.md} (100%) diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md similarity index 100% rename from debug/accuracy_tools/msprobe/docs/05.data_dump_Pytorch.md rename to debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md -- Gitee From 764bf675711d703a5476c717b6c3097afc51e965 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Wed, 28 Aug 2024 14:28:31 +0800 Subject: [PATCH 535/791] fix --- .../test_compute_element.py | 41 +++---------------- 1 file changed, 6 insertions(+), 35 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py index d702c1b3b..9ad54f6d0 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -124,9 +124,12 @@ class TestComputeElement(unittest.TestCase): }, ] compute_element = ComputeElement(compute_element_info=compute_element_info) - parameter = compute_element.get_parameter(get_origin=True) - self.assertTrue((parameter[0] == self.ms_tensor).all()) - self.assertTrue((parameter[1] == self.ms_tensor).all()) + mindspore_parameter = compute_element.get_parameter(get_origin=False, tensor_platform="mindspore") + self.assertTrue((mindspore_parameter[0] == self.ms_tensor).all()) + self.assertTrue((mindspore_parameter[1] == self.ms_tensor).all()) + torch_parameter = compute_element.get_parameter(get_origin=False, tensor_platform="pytorch") + self.assertTrue((torch_parameter[0] == self.torch_tensor).all()) + self.assertTrue((torch_parameter[1] == self.torch_tensor).all()) self.assertEqual(compute_element.get_shape(), tuple()) self.assertEqual(compute_element.get_dtype(), TUPLE_TYPE_STR) @@ -141,38 +144,6 @@ class TestComputeElement(unittest.TestCase): self.assertEqual(compute_element.get_shape(), tuple()) self.assertEqual(compute_element.get_dtype(), INT_TYPE_STR) - def test_init_with_compute_element_info_tuple_tensor(self): - global_context.is_constructed = False - compute_element_info = [ - [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape":[2, 3], - "Max": 3.0, - "Min": 1.0, - "data_name": "input.npy" - }, - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape":[2, 3], - "Max": 3.0, - "Min": 1.0, - "data_name": "input.npy" - } - ] - ] - compute_element = ComputeElement(compute_element_info=compute_element_info) - torch_parameter = compute_element.get_parameter(get_origin=False, tensor_platform="pytorch") - mindspore_parameter = compute_element.get_parameter(get_origin=False, tensor_platform="mindspore") - assert isinstance(torch_parameter, tuple) - assert isinstance(mindspore_parameter, tuple) - for para in torch_parameter: - assert (para == self.torch_tensor).all() - - for para in mindspore_parameter: - assert (para == self.ms_tensor).all() if __name__ == '__main__': unittest.main() -- Gitee From 3bdc00c9958c9b44260a8f37bfccb8141f9a0ac5 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Wed, 28 Aug 2024 16:03:29 +0800 Subject: [PATCH 536/791] fix remove path --- debug/accuracy_tools/msprobe/core/data_dump/json_writer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py index 112e45171..fc560b404 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py @@ -7,6 +7,7 @@ from pathlib import Path from msprobe.core.common.file_check import change_mode, FileOpen from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst +from msprobe.core.common.utils import remove_path class DataWriter: @@ -46,12 +47,12 @@ class DataWriter: change_mode(self.dump_file_path, FileCheckConst.DATA_FILE_AUTHORITY) if os.path.exists(self.stack_file_path): - os.remove(self.stack_file_path) + remove_path(self.stack_file_path) Path(self.stack_file_path).touch() change_mode(self.stack_file_path, FileCheckConst.DATA_FILE_AUTHORITY) if os.path.exists(self.construct_file_path): - os.remove(self.construct_file_path) + remove_path(self.construct_file_path) Path(self.construct_file_path).touch() change_mode(self.construct_file_path, FileCheckConst.DATA_FILE_AUTHORITY) -- Gitee From e1f7fa43843817cda81e99191feee2fd0ec938f0 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 16:09:52 +0800 Subject: [PATCH 537/791] renew --- debug/accuracy_tools/msprobe/README.md | 3 +- .../msprobe/docs/02.config_introduction.md | 83 +++++++++---------- 2 files changed, 42 insertions(+), 44 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 83340fc6a..56e7adbfc 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -6,11 +6,10 @@ [**msprobe**](./) 是 MindStudio Training Tools 工具链下精度调试部分的工具包。主要包括精度预检、溢出检测和精度比对等功能，目前适配 [PyTorch](https://pytorch.org/) 和 [MindSpore](https://www.mindspore.cn/) 框架。这些子工具侧重不同的训练场景，可以定位模型训练中的精度问题。 -为方便使用，本工具提供了一个统一、简易的程序接口，**PrecisionDebugger**，通过以下示例模板和 **config.json** 可轻松使用各种功能。 +为方便使用，本工具提供了一个统一、简易的程序接口，**PrecisionDebugger**，以 PyTorch 框架为例，通过以下示例模板和 **config.json** 可轻松使用各种功能。 ```python from msprobe.pytorch import PrecisionDebugger -# from msprobe.mindspore import PrecisionDebugger debugger = PrecisionDebugger(config_path='./config.json') ... diff --git a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md index 4f7c503fb..fb0045c56 100644 --- a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md +++ b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md @@ -12,70 +12,69 @@ | 参数名 | 解释 | 是否必选 | | ----------------- | ---------------- | -------- | -| task | dump 的任务类型，str 类型。可取值，
"statistics"：仅 dump API 统计信息，默认值；
"tensor"：dump API 统计信息和完全复刻整网的 API 运行情况的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持。
"overflow_check"：溢出检测，仅 PyTorch 和 MindSpore 静态图场景支持；
"free_benchmark"：无标杆比对，仅 PyTorch 场景支持；
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)。
**配置示例**："task": "tensor"。 | 否 | +| task | dump 的任务类型，str 类型。可取值，
"statistics"：仅 dump API 统计信息，默认值；
"tensor"：dump API 统计信息和完全复刻整网的 API 运行情况的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持。
"overflow_check"：溢出检测，仅 PyTorch 和 MindSpore 静态图场景支持；
"free_benchmark"：无标杆比对；
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)。
**配置示例**："task": "tensor"。 | 否 | | dump_path | 设置 dump 数据目录路径，str 类型。
**配置示例**："dump_path": "./dump_path"。MindSpore 场景仅支持绝对路径。 | 是 | -| rank | 指定对某张卡上的数据进行 dump，list[int] 类型，默认未配置（表示 dump 所有卡的数据），应配置为 ≥0 的整数，且须配置实际可用的 Rank ID。
对于 PyTorch 场景，Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，此时 dump 数据为空。
对于 MindSpore 场景，所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
**配置示例**："rank": [1]。 | 否 | +| rank | 指定对某张卡上的数据进行 dump，list[int] 类型，默认未配置（表示 dump 所有卡的数据），应配置为 ≥0 的整数，且须配置实际可用的 Rank ID。

对于 PyTorch 场景，Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，dump 数据为空。

对于 MindSpore 场景，所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
**配置示例**："rank": [1]。 | 否 | | step | 指定 dump 某个 step 的数据，list[int] 类型。默认未配置，表示 dump 所有 step 数据。dump 特定 step 时，须指定为训练脚本中存在的 step。step 为 list 格式，可配置逐个 step。
**配置示例**："step": [0,1,2]。 | 否 | -| level | dump 级别，str 类型，根据不同级别 dump 不同数据。可取值，
"L0"：dump module 模块级精度数据，仅 PyTorch 场景支持，使用背景详见**\*1**；
"L1"：dump API 级精度数据，默认值；
"L2"：dump kernel 级精度数据，须配置 acl_config 参数；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 场景支持。
**配置示例**："level": "L1"。 | 否 | +| level | dump 级别，str 类型，根据不同级别 dump 不同数据。可取值，
"L0"：dump module 模块级精度数据，仅 PyTorch 场景支持，使用背景详见[模块级精度数据 dump 说明（仅 PyTorch 场景支持）](#111-模块级精度数据-dump-说明仅-pytorch-场景支持)；
"L1"：dump API 级精度数据，默认值；
"L2"：dump kernel 级精度数据，须配置 acl_config 参数；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 场景支持。
**配置示例**："level": "L1"。 | 否 | | acl_config | kernel dump 的配置文件，str 类型。level 取"L2"时，该参数必要；level 为其他值时，该参数不选。
**配置示例**：acl_config="./acl_config.json"。具体配置见[ acl_config 示例](./04.acl_config_examples.md)。 | 否 | -| seed | 随机种子数，int 类型，默认值为：1234，仅 PyTorch 场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见**\*2**。
**配置示例**："seed": 1234。 | 否 | +| seed | 随机种子数，int 类型，默认值为：1234，仅 PyTorch 场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见[固定随机数范围（仅 PyTorch 场景支持）](#112-固定随机数范围仅-pytorch-场景支持)。
**配置示例**："seed": 1234。 | 否 | | is_deterministic | 确定性计算模式，bool 类型，仅 PyTorch 场景支持。可取值 true（开启）或 false（关闭），默认关闭。
即使在相同的硬件和输入下，API 多次执行的结果也可能不同，开启确定性计算是为了保证在相同的硬件和输入下，API 多次执行的结果相同。
确定性计算会导致 API 执行性能降低，建议在发现模型多次执行结果不同的情况下开启。
rnn 类算子、ReduceSum、ReduceMean 等算子可能与确定性计算存在冲突，若开启确定性计算后多次执行的结果不相同，则考虑存在这些算子。
**配置示例**："is_deterministic": true。
| 否 | | enable_dataloader | 自动控制开关，bool 类型，仅 PyTorch 场景支持。可取值 true（开启）或 false（关闭），默认为 false。配置为 True 后自动识别 step 参数指定的迭代，并在该迭代执行完成后退出训练，此时 start、stop 和 step 函数可不配置，开启该开关要求训练脚本是通过 torch.utils.data.dataloader 方式加载数据。仅支持 PyTorch 单卡训练使用，分布式训练场景下存在数据 dump 不全问题，**下个版本即将废弃该功能**。 | 否 | -**\***： -1. 模块级精度数据 dump 说明（仅 PyTorch 场景支持） - - 大模型场景下，通常不是简单的利用自动迁移能力实现从 GPU 到 NPU 的训练脚本迁移，而是会对 NPU 网络进行一系列针对性的适配，因此，常常会造成迁移后的 NPU 模型存在部分子结构不能与 GPU 原始模型完全对应。模型结构不一致导致 API 调用类型及数量不一致，若直接按照 API 粒度进行精度数据 dump 和比对，则无法完全比对所有的 API。 +#### 1.1.1 模块级精度数据 dump 说明（仅 PyTorch 场景支持） + +大模型场景下，通常不是简单的利用自动迁移能力实现从 GPU 到 NPU 的训练脚本迁移，而是会对 NPU 网络进行一系列针对性的适配，因此，常常会造成迁移后的 NPU 模型存在部分子结构不能与 GPU 原始模型完全对应。模型结构不一致导致 API 调用类型及数量不一致，若直接按照 API 粒度进行精度数据 dump 和比对，则无法完全比对所有的 API。 - 本小节介绍的功能是对模型中的大粒度模块进行数据 dump，使其比对时，对于无法以 API 粒度比对的模块可以直接以模块粒度进行比对。 +本小节介绍的功能是对模型中的大粒度模块进行数据 dump，使其比对时，对于无法以 API 粒度比对的模块可以直接以模块粒度进行比对。 - 模块指的是继承自 nn.Module 类模块，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump 数据时以模块为粒度进行 dump。 +模块指的是继承自 nn.Module 类模块，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump 数据时以模块为粒度进行 dump。 -2. 固定随机数范围（仅 PyTorch 场景支持） +#### 1.1.2 固定随机数范围（仅 PyTorch 场景支持） - seed_all 函数可固定随机数的范围如下表。 +seed_all 函数可固定随机数的范围如下表。 - | API | 固定随机数 | - | ---------------------------------------- | --------------------------- | - | os.environ['PYTHONHASHSEED'] = str(seed) | 禁止 Python 中的 hash 随机化 | - | random.seed(seed) | 设置 random 随机生成器的种子 | - | np.random.seed(seed) | 设置 numpy 中随机生成器的种子 | - | torch.manual_seed(seed) | 设置当前 CPU 的随机种子 | - | torch.cuda.manual_seed(seed) | 设置当前 GPU 的随机种子 | - | torch.cuda.manual_seed_all(seed) | 设置所有 GPU 的随机种子 | - | torch_npu.npu.manual_seed(seed) | 设置当前 NPU 的随机种子 | - | torch_npu.npu.manual_seed_all(seed) | 设置所有 NPU 的随机种子 | - | torch.backends.cudnn.enable=False | 关闭 cuDNN | - | torch.backends.cudnn.benchmark=False | cuDNN 确定性地选择算法 | - | torch.backends.cudnn.deterministic=True | cuDNN 仅使用确定性的卷积算法 | +| API | 固定随机数 | +| ---------------------------------------- | --------------------------- | +| os.environ['PYTHONHASHSEED'] = str(seed) | 禁止 Python 中的 hash 随机化 | +| random.seed(seed) | 设置 random 随机生成器的种子 | +| np.random.seed(seed) | 设置 numpy 中随机生成器的种子 | +| torch.manual_seed(seed) | 设置当前 CPU 的随机种子 | +| torch.cuda.manual_seed(seed) | 设置当前 GPU 的随机种子 | +| torch.cuda.manual_seed_all(seed) | 设置所有 GPU 的随机种子 | +| torch_npu.npu.manual_seed(seed) | 设置当前 NPU 的随机种子 | +| torch_npu.npu.manual_seed_all(seed) | 设置所有 NPU 的随机种子 | +| torch.backends.cudnn.enable=False | 关闭 cuDNN | +| torch.backends.cudnn.benchmark=False | cuDNN 确定性地选择算法 | +| torch.backends.cudnn.deterministic=True | cuDNN 仅使用确定性的卷积算法 | - 需要保证 CPU 或 GPU 以及 NPU 的模型输入完全一致，dump 数据的比对才有意义，seed_all 并不能保证模型输入完全一致，如下表所示场景需要保证输入的一致性。 +需要保证 CPU 或 GPU 以及 NPU 的模型输入完全一致，dump 数据的比对才有意义，seed_all 并不能保证模型输入完全一致，如下表所示场景需要保证输入的一致性。 - | 场景 | 固定方法 | - | --------------- | ------------- | - | 数据集的 shuffle | 关闭 shuffle。 | - | dropout | 关闭 dropout。 | +| 场景 | 固定方法 | +| --------------- | ------------- | +| 数据集的 shuffle | 关闭 shuffle。 | +| dropout | 关闭 dropout。 | - 关闭 shuffle 示例： +关闭 shuffle 示例： - ```python - train_loader = torch.utils.data.DataLoader( - train_dataset, - batch_size = batch_size, - shuffle = False, - num_workers = num_workers - ) - ``` +```python +train_loader = torch.utils.data.DataLoader( +train_dataset, +batch_size = batch_size, +shuffle = False, +num_workers = num_workers +) +``` - 关闭 dropout： +关闭 dropout： - 在使用 `from msprobe.pytorch import PrecisionDebugger` 后，工具会自动将 `torch.nn.functional.dropout`、`torch.nn.functional.dropout2d`、`torch.nn.functional.dropout3d`、`torch.nn.Dropout`、`torch.nn.Dropout2d`、`torch.nn.Dropout3d` 的接口参数 p 置为0。 +在使用 `from msprobe.pytorch import PrecisionDebugger` 后，工具会自动将 `torch.nn.functional.dropout`、`torch.nn.functional.dropout2d`、`torch.nn.functional.dropout3d`、`torch.nn.Dropout`、`torch.nn.Dropout2d`、`torch.nn.Dropout3d` 的接口参数 p 置为0。 ### 1.2 task 配置为 statistics | 参数名 | 解释 | 是否必选 | | ------------ | ------------------- | -------- | -| scope | PyTorch 和 MindSpore 动态图场景 dump 范围，list[str] 类型，默认未配置（list 也未配置时表示 dump 所有 API 的数据）。该参数在 [ ] 内配置两个模块名或API名，用于锁定区间，dump 该范围内的数据。
**配置示例**："scope": ["MyModuleOP1", "MyModuleOP2"]。与 level 参数取值相关，level 为 L0 和 mix 级别时，可配置模块名；level 为 L1 级别时，可配置 API 名。MindSpore 动态图场景当前仅支持配置为 API 名。 | 否 | +| scope | PyTorch 和 MindSpore 动态图场景 dump 范围，list[str] 类型，默认未配置（list 也未配置时表示 dump 所有 API 的数据）。该参数在 [ ] 内配置两个模块名或API名，用于锁定区间，dump 该范围内的数据。
**配置示例**："scope": ["MyModuleOP1", "MyModuleOP2"]。与 level 参数取值相关，level 为 L0 和 mix 级别时，可配置模块名；level 为 L1 级别时，可配置 API 名。
MindSpore 动态图场景当前仅支持配置为 API 名。 | 否 | | list | 自定义 dump 范围，list[str] 类型，默认未配置（scope 也未配置时表示 dump 所有 API 的数据）。包含如下配置方法：
PyTorch 和 MindSpore 动态图场景配置具体的 API 全称，dump 该 API 数据。
**配置示例**："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch 和 MindSpore 动态图场景指定某一类 API，dump 某一类的 API 级别输入输出数据。
**配置示例**："list": ["relu"]。
MindSpore 静态图场景配置 kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合“name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，“name-regex(Default/.+)”可匹配算子名称以“Default/”开头的所有算子）。 | 否 | | data_mode | dump 数据过滤，str 类型。可取值，
PyTorch 场景支持"all"、"forward"、"backward"、"input"和"output"，表示仅保存 dump 的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的 dump 文件。
MindSpore 场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。
**配置示例**："data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有 dump 的数据。除了 all 参数只能单独配置外，其他参数可以自由组合。 | 否 | | summary_mode | 控制 dump 文件输出的模式，str 类型，仅 PyTorch 场景支持，可取值，
md5：dump 输出包含 md5 值以及 API 统计信息的 dump.json 文件，用于验证数据的完整性；
statistics：dump 仅输出包含 API 统计信息的 dump.json 文件，默认值。
**配置示例**："summary_mode": "md5"。 | 否 | -- Gitee From a52d432bf7b0c1857ea7ed3a8b94bb005b8d2471 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Wed, 28 Aug 2024 16:54:23 +0800 Subject: [PATCH 538/791] =?UTF-8?q?[grad=5Ftool]=E6=B5=8B=E8=AF=95?= =?UTF-8?q?=E6=84=8F=E8=A7=81=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/grad_tool/README.md | 101 ++++++++++++----------- 1 file changed, 54 insertions(+), 47 deletions(-) diff --git a/debug/accuracy_tools/grad_tool/README.md b/debug/accuracy_tools/grad_tool/README.md index 1d35f03e4..85407c619 100644 --- a/debug/accuracy_tools/grad_tool/README.md +++ b/debug/accuracy_tools/grad_tool/README.md @@ -9,7 +9,7 @@ ## 工具特性 -- 使用便捷，修改处少 +- 使用便捷，修改点少 - 梯度是一步训练过程中差异最大的地方，精度差异体现到梯度差异上比loss差异更明显 ## 工具安装 @@ -44,57 +44,65 @@ | 参数 | 说明 | 输入类型 | 是否必选 | |--------------------------------|-----------------------------------|-----------------|----------| - | level | 输出级别。决定导出数据的详细程度，级别越大导出数据越详细。可取值：L0, L1, L2|str | 是 | + | level | 输出级别。决定导出数据的详细程度，级别越大导出数据越详细。可取值：L0、L1、L2 |str | 是 | | param_list | 权重名称列表，表示需要监控的权重。不指定或列表为空就表示监控所有权重。 | List[str] | 否 | - | rank | rank id列表，在多卡场景下，表示需要导出梯度数据的进程的rank id。不指定或列表为空就表示导出所有rank的数据。单卡场景无需关注该参数。 （MindSpore静态图模式下，当前暂不支持指定rank功能） | List[int] | 否 | - | step | step列表，表示需要导出数据的step列表。不指定或列表为空就表示导出所有step的数据。（MindSpore静态图模式下，当前暂不支持指定step功能） | List[int] | 否 | - | bounds | 区间列表，用来划分区间以统计数值的分布。需要保证由数据小到大排列。不指定则使用默认值[-10, -1, -0.1, -0.01, -0.001, 0, 0.001, 0.01, 0.1, 1, 10] | List[float] | 否 | + | rank | Rank ID列表，在多卡场景下，表示需要导出梯度数据的进程的Rank ID。不指定或列表为空就表示导出所有Rank的数据。单卡场景无需关注该参数。（MindSpore静态图场景暂不支持指定Rank功能） | List[int] | 否 | + | step | step列表，表示需要导出数据的step列表。不指定或列表为空就表示导出所有step的数据。（MindSpore静态图场景暂不支持指定step功能） | List[int] | 否 | + | bounds | 区间列表，用来划分区间以统计数值的分布。需要保证由数据从小到大排列。不指定则使用默认值[-10.0, -1.0, -0.1, -0.01, -0.001, 0.0, 0.001, 0.01, 0.1, 1.0, 10.0] | List[float] | 否 | | output_path | 输出目录。如果不存在就会创建一个新目录。 | str | 是 | **不同级别的level的导出数据** - - + | 级别 | 特征数据表头 | 是否有方向数据 | | ---- | ------------------------------------------------------------ | -------------- | | L0 | ("param_name", "MD5", "max", "min", "norm", "shape") | 否 | - | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | + | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | intervals就是根据值分布bounds划分出的区间。 + MindSpore静态图模式下，L0级别中暂不支持"MD5" - **方向数据解释** + **方向数据说明** 因为模型的参数往往非常大，所以存储真实数据是不可接受的，这里折衷一下，只存储梯度数据的正负号（一个布尔值），也就是方向。 **bounds和值分布解释** - + 值分布：梯度数据落在各个区间的元素个数占总元素个数的比例。 - + bounds：一个列表，用来划分出区间以统计值分布。例如传入bounds = [-10, 0, 10]，此时有一个 grad_value: Tensor = [9.3 , 5.4, -1.0, -12.3]，依据 bounds 划分出 (-inf, -10]、(-10, 0]、(0, 10]、(10, inf) 四个区间，然后统计grad_value里的数据落在每个区间内的个数，得到 1、1、2、0。如下图所示： - ![Alt text](img/image-1.png) + + 值分布：梯度数据落在各个区间的元素个数占总元素个数的比例。 + + + bounds：一个列表，用来划分出区间以统计值分布。例如传入bounds = [-10, 0, 10]，此时有一个 grad_value: Tensor = [9.3 , 5.4, -1.0, -12.3]，依据bounds 划分出 (-inf, -10]、(-10, 0]、(0, 10]、(10, inf) 四个区间，然后统计grad_value里的数据落在每个区间内的个数，得到 1、1、2、0。如下图所示： + + ![Alt text](img/image-1.png) 2. 插入代码。示例代码如下： -- PyTorch框架：模型构造完成时，传入config.yaml的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将模型作为参数传入。 -```python -from grad_tool.grad_monitor import GradientMonitor -gm = GradientMonitor("config_path") -gm.monitor(model) -``` -- MindSpore框架：在训练开始前，传入config.yaml的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将优化器作为参数传入。 -```python -from grad_tool.grad_monitor import GradientMonitor -gm = GradientMonitor("config_path", framework="MindSpore") -gm.monitor(optimizer) -``` + - PyTorch框架：模型构造完成时，传入config.yaml的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将模型作为参数传入。 + + ```python + from grad_tool.grad_monitor import GradientMonitor + gm = GradientMonitor("config_path") + gm.monitor(model) + ``` + + - MindSpore框架：在训练开始前，传入config.yaml的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将优化器作为参数传入。 + + ```python + from grad_tool.grad_monitor import GradientMonitor + gm = GradientMonitor("config_path", framework="MindSpore") + gm.monitor(optimizer) + ``` + + 动态图场景下，optimizer不能在jit装饰器修饰范围内。 + 3. 结束监控（MindSpore静态图模式下需要） 在训练结束之后，调用stop接口 - -```python -gm.stop() -``` + + ```python + gm.stop() + ``` ### 输出结果 **输出目录结构**（以level配置L2为例） @@ -106,7 +114,6 @@ gm.stop() │ ├── step_{step} │ │ ├── {param_name}.pt(npy) ``` -+ {timestamp}：梯度工具导出数据的时候会在output_path下生成一个时间戳目录，然后在这个时间戳目录下输出结果。 + rank_{rank_id}：在分布式场景下，会记录卡的rank_id。非分布式场景下，如果是CPU则记录进程号，如果是CPU或GPU则记录卡号 + grad_summary_{step}.csv：会分step记录每一步的梯度数据统计值。 + step_{step}：这个目录下会存放该step的梯度的方向数据。 @@ -143,28 +150,28 @@ gm.stop() - 单卡比对 新建Python脚本，调用grad_tool.grad_comparator的GradComparator.compare函数，传入的前两个参数分别为梯度数据的rank层目录，第三个参数为输出目录。如下所示： - -```python -from grad_tool.grad_comparator import GradComparator -# 默认framework为PyTorch，MindSpore比对时framework设置为MindSpore -GradComparator.compare("需要对比的rank_id级目录", - "需要对比的rank_id级目录", - "比对结果输出目录", - framework="PyTorch") -``` + + ```python + from grad_tool.grad_comparator import GradComparator + # 默认framework为PyTorch，MindSpore比对时framework设置为MindSpore + GradComparator.compare("需要对比的rank_id级目录", + "需要对比的rank_id级目录", + "比对结果输出目录", + framework="PyTorch") + ``` - 多卡比对 新建Python脚本，调用grad_tool.grad_comparator的GradComparator.compare_distributed函数，传入的前两个参数分别为梯度数据的rank层目录，第三个参数为输出目录。如下所示： - -```python -from grad_tool.grad_comparator import GradComparator -# 默认framework为PyTorch，MindSpore比对时framework设置为MindSpore -GradComparator.compare_distributed("配置文件里写的输出目录", - "配置文件里写的输出目录", - "比对结果输出目录", - framework="PyTorch") -``` + + ```python + from grad_tool.grad_comparator import GradComparator + # 默认framework为PyTorch，MindSpore比对时framework设置为MindSpore + GradComparator.compare_distributed("配置文件里写的输出目录", + "配置文件里写的输出目录", + "比对结果输出目录", + framework="PyTorch") + ``` ### 比对结果 -- Gitee From 9df27a4f043bd0bcdb73ad1a7a045e5be72b0070 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Wed, 28 Aug 2024 17:48:48 +0800 Subject: [PATCH 539/791] review --- .../msprobe/mindspore/api_accuracy_checker/compute_element.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index fa78f2195..1ed5ce20d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -169,8 +169,8 @@ class ComputeElement: if isinstance(compute_element_info, list): self.shape = tuple() self.dtype_str = TUPLE_TYPE_STR - self.parameter = tuple(ComputeElement(compute_element_info=sub_info) - for sub_info in compute_element_info) + self.parameter = tuple([ComputeElement(compute_element_info=sub_info) + for sub_info in compute_element_info]) else: type_str = check_and_get_from_json_dict(compute_element_info, "type", "type field in api_info.json", accepted_type=str, accepted_value=api_info_type_str_to_type.keys()) -- Gitee From 5f5b38773a4b32d3f30866e52f1dae3621c3f317 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 18:48:16 +0800 Subject: [PATCH 540/791] renew manual --- .../msprobe/docs/02.config_introduction.md | 83 +++++++++---------- .../msprobe/docs/03.config_examples.md | 2 +- .../msprobe/docs/05.data_dump_PyTorch.md | 1 - .../docs/09.accuracy_checker_MindSpore.md | 18 ++-- 4 files changed, 48 insertions(+), 56 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md index fb0045c56..845bfb3d9 100644 --- a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md +++ b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md @@ -12,13 +12,13 @@ | 参数名 | 解释 | 是否必选 | | ----------------- | ---------------- | -------- | -| task | dump 的任务类型，str 类型。可取值，
"statistics"：仅 dump API 统计信息，默认值；
"tensor"：dump API 统计信息和完全复刻整网的 API 运行情况的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持。
"overflow_check"：溢出检测，仅 PyTorch 和 MindSpore 静态图场景支持；
"free_benchmark"：无标杆比对；
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)。
**配置示例**："task": "tensor"。 | 否 | +| task | dump 的任务类型，str 类型。可取值，
"statistics"：仅 dump API 统计信息，默认值；
"tensor"：dump API 统计信息和完全复刻整网的 API 运行情况的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持。
"overflow_check"：溢出检测，仅 PyTorch 和 MindSpore 静态图场景支持；
"free_benchmark"：无标杆比对；
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)。
**配置示例**："task": "tensor"。 | 否 | | dump_path | 设置 dump 数据目录路径，str 类型。
**配置示例**："dump_path": "./dump_path"。MindSpore 场景仅支持绝对路径。 | 是 | -| rank | 指定对某张卡上的数据进行 dump，list[int] 类型，默认未配置（表示 dump 所有卡的数据），应配置为 ≥0 的整数，且须配置实际可用的 Rank ID。

对于 PyTorch 场景，Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，dump 数据为空。

对于 MindSpore 场景，所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
**配置示例**："rank": [1]。 | 否 | +| rank | 指定对某张卡上的数据进行 dump，list[int] 类型，默认未配置（表示 dump 所有卡的数据），应配置为 ≥0 的整数，且须配置实际可用的 Rank ID。

PyTorch 场景：Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，dump 数据为空。
**配置示例**："rank": [1]。
MindSpore 场景：所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
**配置示例**："rank": [1]。 | 否 | | step | 指定 dump 某个 step 的数据，list[int] 类型。默认未配置，表示 dump 所有 step 数据。dump 特定 step 时，须指定为训练脚本中存在的 step。step 为 list 格式，可配置逐个 step。
**配置示例**："step": [0,1,2]。 | 否 | -| level | dump 级别，str 类型，根据不同级别 dump 不同数据。可取值，
"L0"：dump module 模块级精度数据，仅 PyTorch 场景支持，使用背景详见[模块级精度数据 dump 说明（仅 PyTorch 场景支持）](#111-模块级精度数据-dump-说明仅-pytorch-场景支持)；
"L1"：dump API 级精度数据，默认值；
"L2"：dump kernel 级精度数据，须配置 acl_config 参数；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 场景支持。
**配置示例**："level": "L1"。 | 否 | +| level | dump 级别，str 类型，根据不同级别 dump 不同数据。可取值，
"L0"：dump module 模块级精度数据，仅 PyTorch 场景支持，使用背景详见 [1.1.1 模块级精度数据 dump 说明（仅 PyTorch 场景支持）](#111-模块级精度数据-dump-说明仅-pytorch-场景支持)；
"L1"：dump API 级精度数据，默认值；
"L2"：dump kernel 级精度数据，须配置 acl_config 参数；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 场景支持。
**配置示例**："level": "L1"。 | 否 | | acl_config | kernel dump 的配置文件，str 类型。level 取"L2"时，该参数必要；level 为其他值时，该参数不选。
**配置示例**：acl_config="./acl_config.json"。具体配置见[ acl_config 示例](./04.acl_config_examples.md)。 | 否 | -| seed | 随机种子数，int 类型，默认值为：1234，仅 PyTorch 场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见[固定随机数范围（仅 PyTorch 场景支持）](#112-固定随机数范围仅-pytorch-场景支持)。
**配置示例**："seed": 1234。 | 否 | +| seed | 随机种子数，int 类型，默认值为：1234，仅 PyTorch 场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见 [1.1.2 固定随机数范围（仅 PyTorch 场景支持）](#112-固定随机数范围仅-pytorch-场景支持)。
**配置示例**："seed": 1234。 | 否 | | is_deterministic | 确定性计算模式，bool 类型，仅 PyTorch 场景支持。可取值 true（开启）或 false（关闭），默认关闭。
即使在相同的硬件和输入下，API 多次执行的结果也可能不同，开启确定性计算是为了保证在相同的硬件和输入下，API 多次执行的结果相同。
确定性计算会导致 API 执行性能降低，建议在发现模型多次执行结果不同的情况下开启。
rnn 类算子、ReduceSum、ReduceMean 等算子可能与确定性计算存在冲突，若开启确定性计算后多次执行的结果不相同，则考虑存在这些算子。
**配置示例**："is_deterministic": true。
| 否 | | enable_dataloader | 自动控制开关，bool 类型，仅 PyTorch 场景支持。可取值 true（开启）或 false（关闭），默认为 false。配置为 True 后自动识别 step 参数指定的迭代，并在该迭代执行完成后退出训练，此时 start、stop 和 step 函数可不配置，开启该开关要求训练脚本是通过 torch.utils.data.dataloader 方式加载数据。仅支持 PyTorch 单卡训练使用，分布式训练场景下存在数据 dump 不全问题，**下个版本即将废弃该功能**。 | 否 | @@ -74,9 +74,9 @@ num_workers = num_workers | 参数名 | 解释 | 是否必选 | | ------------ | ------------------- | -------- | -| scope | PyTorch 和 MindSpore 动态图场景 dump 范围，list[str] 类型，默认未配置（list 也未配置时表示 dump 所有 API 的数据）。该参数在 [ ] 内配置两个模块名或API名，用于锁定区间，dump 该范围内的数据。
**配置示例**："scope": ["MyModuleOP1", "MyModuleOP2"]。与 level 参数取值相关，level 为 L0 和 mix 级别时，可配置模块名；level 为 L1 级别时，可配置 API 名。
MindSpore 动态图场景当前仅支持配置为 API 名。 | 否 | -| list | 自定义 dump 范围，list[str] 类型，默认未配置（scope 也未配置时表示 dump 所有 API 的数据）。包含如下配置方法：
PyTorch 和 MindSpore 动态图场景配置具体的 API 全称，dump 该 API 数据。
**配置示例**："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch 和 MindSpore 动态图场景指定某一类 API，dump 某一类的 API 级别输入输出数据。
**配置示例**："list": ["relu"]。
MindSpore 静态图场景配置 kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合“name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，“name-regex(Default/.+)”可匹配算子名称以“Default/”开头的所有算子）。 | 否 | -| data_mode | dump 数据过滤，str 类型。可取值，
PyTorch 场景支持"all"、"forward"、"backward"、"input"和"output"，表示仅保存 dump 的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的 dump 文件。
MindSpore 场景仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。
**配置示例**："data_mode": ["backward"]或"data_mode": ["forward", "backward"]。默认为["all"]，即保存所有 dump 的数据。除了 all 参数只能单独配置外，其他参数可以自由组合。 | 否 | +| scope | PyTorch 和 MindSpore 动态图场景 dump 范围，list[str] 类型，默认未配置（list 也未配置时表示 dump 所有 API 的数据）。该参数在 [ ] 内配置两个模块名或 API 名，用于锁定区间，dump 该范围内的数据。
**配置示例**："scope": ["MyModuleOP1", "MyModuleOP2"]。与 level 参数取值相关，level 为 L0 和 mix 级别时，可配置模块名；level 为 L1 级别时，可配置 API 名。MindSpore 动态图场景当前仅支持配置为 API 名。 | 否 | +| list | 自定义 dump 范围，list[str] 类型，默认未配置（scope 也未配置时表示 dump 所有 API 的数据）。包含如下配置方法：
PyTorch 和 MindSpore 动态图场景配置具体的 API 全称，dump 该 API 数据。
**配置示例**："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch 和 MindSpore 动态图场景指定某一类 API，dump 某一类的 API 级别输入输出数据。
**配置示例**："list": ["relu"]。
MindSpore 静态图场景配置 kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合“name-regex(xxx)”格式时，后台则会将其作为正则表达式。例如，“name-regex(Default/.+)”可匹配算子名称以“Default/”开头的所有算子）。 | 否 | +| data_mode | dump 数据过滤，str 类型。
PyTorch 场景：支持"all"、"forward"、"backward"、"input"和"output"，表示仅保存 dump 的数据中文件名包含"forward"、"backward"、"input"和"output"的前向、反向、输入或输出的 dump 文件。
**配置示例**："data_mode": ["all"]。
MindSpore 场景：仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。
**配置示例**："data_mode": ["backward"] 或 "data_mode": ["forward", "backward"]。默认为["all"]，即保存所有 dump 的数据。除了 all 参数只能单独配置外，其他参数可以自由组合。 | 否 | | summary_mode | 控制 dump 文件输出的模式，str 类型，仅 PyTorch 场景支持，可取值，
md5：dump 输出包含 md5 值以及 API 统计信息的 dump.json 文件，用于验证数据的完整性；
statistics：dump 仅输出包含 API 统计信息的 dump.json 文件，默认值。
**配置示例**："summary_mode": "md5"。 | 否 | ### 1.3 task 配置为 tensor @@ -87,38 +87,36 @@ num_workers = num_workers | list | 与[ 1.2 task 配置为 statistics ](#12-task-配置为-statistics)中的解释相同。另外，
PyTorch 和 MindSpore 动态图场景配置 kernel_api，dump 前向和反向 API 的kernel_api 级别数据，其中 dump 反向 API 时需要配置 **backward_input** 参数。
**前向 API 配置示例**："list": ["Tensor.permute.1.forward"]；
**反向 API 配置示例**："list": ["Tensor.permute.1.forward"], "backward.input": ["./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"]。
| 否 | | backward_input | 该输入文件为首次运行训练 dump 得到反向 API 输入的 dump 文件，str 类型，仅 PyTorch 场景支持，默认未配置。例如若需要 dump Functional.conv2d.1 API 的反向过程的输入输出，则需要在 dump 目录下查找命名包含 Functional.conv2d.1、backward 和 input 字段的 dump 文件。
**配置示例**："backward_input": ["./npu_dump/step0/rank0/Functional.conv2d.1.backward.input.0.pt"] | 否 | | data_mode | 与[ 1.2 task 配置为 statistics ](#12-task-配置为-statistics)中的解释相同 | 否 | -| file_format | MindSpore 场景真实 tensor 数据的保存格式，str 类型，可取值，
"bin"：dump 的 tensor 文件为二进制格式，"level": "L1" 时不支持；
"npy"：dump 的 tensor 文件后缀为 .npy，默认值。 | 否 | -| online_run_ut* | 在线预检模式开关，bool 类型，可取值 true（开启）、false（关闭），默认未配置，表示关闭。配置为 true 表示开启在线预预检。| 否 | -| nfs_path* | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。仅在 online_run_ut 字段配置为 true 时生效，未配置该参数后 host 和 port 不生效。 | 否 | -| host* | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的局域网 IP 地址。仅在 online_run_ut 字段配置为 true 时生效，局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | -| port* | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的端口号。仅在 online_run_ut 字段配置为 true 时生效，局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。| 否 | +| file_format | MindSpore 静态图场景 tensor 数据的保存格式，str 类型，可取值，
"bin"：dump 的 tensor 文件为二进制格式，"level": "L1" 时不支持；
"npy"：dump 的 tensor 文件后缀为 .npy，默认值。 | 否 | +| online_run_uta | 在线预检模式开关，bool 类型，可取值 true（开启）、false（关闭），默认未配置，表示关闭。配置为 true 表示开启在线预预检。| 否 | +| nfs_patha | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。仅在 online_run_ut 字段配置为 true 时生效，未配置该参数后 host 和 port 不生效。 | 否 | +| hosta | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的局域网 IP 地址。仅在 online_run_ut 字段配置为 true 时生效，局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。 | 否 | +| porta | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，NPU 侧须配置为 GPU 侧的端口号。仅在 online_run_ut 字段配置为 true 时生效，局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。| 否 | -**\***：online_run_ut、nfs_path、host、port 等字段仅在线预检场景 NPU 机器生效。 +**a**：online_run_ut、nfs_path、host、port 等字段仅在线预检场景 NPU 机器生效。 ### 1.4 task 配置为 run_ut -仅 PyTorch 场景支持。 - | 参数名称 | 解释 | 是否必选 | | --------------- | ------------------------ | ------------ | -| white_list**\*1** | API dump 白名单，仅对指定的 API 进行 dump。
**配置示例**："white_list": ["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | -| black_list**\*1** | API dump 黑名单，被指定的 API 不进行 dump。
**配置示例**："black_list": ["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | +| white_lista | API dump 白名单，仅对指定的 API 进行 dump。
**配置示例**："white_list": ["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | +| black_lista | API dump 黑名单，被指定的 API 不进行 dump。
**配置示例**："black_list": ["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | | error_data_path | 配置保存精度未达标的 API 输入输出数据路径，默认为当前路径。
**配置示例**："error_data_path": "./"。 | 否 | -| is_online**\*2** | 在线预检模式开关，bool 类型，可取值 true（开启）、false（关闭），默认关闭。 | 否 | -| nfs_path**\*2** | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。配置该参数后 host 和 port 不生效，仅在 is_online 字段配置为 true 时生效。 | 否 | -| host**\*2** | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机地址 127.0.0.1 或本机局域网 IP。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。仅在 is_online 字段配置为 true 时生效。 | 否 | -| port**\*2** | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机可用端口。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。仅在 is_online 字段配置为 true 时生效。| 否 | -| rank_list**\*2** | 指定在线预检的 Rank ID，默认值为[0]，list[int] 类型，应配置为大于等于 0 的整数，且须根据实际卡的 Rank ID 配置，若所配置的值大于实际训练所运行的卡的 Rank ID，则在线预检输出数据为空。GPU 和 NPU 须配置一致。仅在 is_online 字段配置为 true 时生效。 | 否 | +| is_onlineb | 在线预检模式开关，bool 类型，可取值 true（开启）、false（关闭），默认关闭。 | 否 | +| nfs_pathb | 在线预检模式共享存储目录路径，str 类型，用于 GPU 设备和 NPU 设备间进行通信。配置该参数后 host 和 port 不生效，仅在 is_online 字段配置为 true 时生效。 | 否 | +| hostb | 在线预检模式局域网场景信息接收端 IP，str 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机地址 127.0.0.1 或本机局域网 IP。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。仅在 is_online 字段配置为 true 时生效。 | 否 | +| portb | 在线预检模式局域网场景信息接收端端口号，int 类型，用于 GPU 设备和 NPU 设备间进行通信，GPU 侧配置为本机可用端口。局域网场景时，不能配置 nfs_path 参数，否则局域网场景不生效。仅在 is_online 字段配置为 true 时生效。| 否 | +| rank_listb | 指定在线预检的 Rank ID，默认值为 [0]，list[int] 类型，应配置为大于等于 0 的整数，且须根据实际卡的 Rank ID 配置，若所配置的值大于实际训练所运行的卡的 Rank ID，则在线预检输出数据为空。GPU 和 NPU 须配置一致。仅在 is_online 字段配置为 true 时生效。 | 否 | + +**a**：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 -**\***： -1. white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 -2. is_online、nfs_path、host、port、rank_list 等字段仅在线预检场景 GPU 机器生效。 +**b**：is_online、nfs_path、host、port、rank_list 等字段仅在线预检场景 GPU 机器生效。 ### 1.5 task 配置为 overflow_check | 参数名 | 解释 | 是否必选 | | ------------- | ---------------------- | -------- | -| overflow_nums | 控制溢出次数，int 类型，仅 PyTorch 场景支持，表示第 N 次溢出时，停止训练，过程中检测到溢出 API 对应的 kernel 数据均 dump。
**配置示例**："overflow_nums": 3。默认为1，即检测到1次溢出，训练停止，配置为-1时，表示持续检测溢出直到训练结束。 | 否 | +| overflow_nums | 控制溢出次数，int 类型，仅 PyTorch 场景支持，表示第 N 次溢出时，停止训练，过程中检测到溢出 API 对应的 kernel 数据均 dump。
**配置示例**："overflow_nums": 3。默认为 1，即检测到1次溢出，训练停止，配置为 -1 时，表示持续检测溢出直到训练结束。 | 否 | | check_mode | MindSpore 场景 kernel 级别的溢出检测，str 类型，可取值：
"aicore"：开启 AI Core 的溢出检测；
"atomic"：开启 Atomic 的溢出检测；
"all"：开启 AI Core 和 Atomic 的溢出检测，默认值。
**配置示例**："check_mode": "aicore"。 | 否 | ### 1.6 task 配置为 free_benchmark @@ -134,31 +132,30 @@ num_workers = num_workers | 参数名 | 解释 | 是否必选 | | ------------ | -------------- | -------- | | scope | 自定义检测 API 列表（仅 PyTorch 场景支持），list[str] 类型，默认值为空列表，当 list 也为空列表时，表示检测所有 API。需要在 [ ] 内配置具体 API 名（在 dump 的结果中查看）。
**配置示例**："scope": ["Torch.matmul.0", "Tensor.pow.4"]。
与 list 参数不能同时配置。 | 否 | -| list | 自定义检测 API 类型或 API 名称，list[str] 类型，默认值为空列表，表示检测所有 API（PyTorch 场景下还需 scope 也为空列表）。包含如下配置方法：
PyTorch 场景下，指定某一类 API，对某一类的 API 进行无标杆比对。
**配置示例**："list": ["relu"]。
MindSpore 场景下，指定 API 名称，对列表中的 API 进行检测。
**配置示例**："list": ["mindspore.mint.div", "mindspore.ops.bmm", "mindspore.Tensor.\_\_add\_\_"]。
与 scope 参数不能同时配置。 | 否 | +| list | 自定义检测 API 类型或 API 名称，list[str] 类型，默认值为空列表，表示检测所有 API（PyTorch 场景下还需 scope 也为空列表）。
PyTorch 场景：指定某一类 API，对某一类的 API 进行无标杆比对。
**配置示例**："list": ["relu"]。
MindSpore 场景：指定 API 名称，对列表中的 API 进行检测。
**配置示例**："list": ["mindspore.mint.div", "mindspore.ops.bmm", "mindspore.Tensor.\_\_add\_\_"]。
与 scope 参数不能同时配置。 | 否 | | fuzz_device | 标杆设备，str 类型。可取值，
"npu"：无标杆，通过添加扰动因子进行比对，默认值；
"cpu"：以 CPU 为标杆，pert_mode 须配置为"to_cpu"（仅 PyTorch 场景支持）。
**配置示例**："fuzz_device": "cpu"。 | 否 | | pert_mode | 无标杆扰动因子，str 类型。可取值，
"improve_precision"：对输入做升精度，默认值；
"add_noise"：对输入增加噪声；
"no_change"：不加扰动直接二次执行；
"bit_noise"：输入的末位比特翻转；
"change_value"：输入的张量首尾值调换（仅 PyTorch 场景支持）；
"to_cpu"：在 CPU 等价执行（仅 PyTorch 场景支持）。
**配置示例**："pert_mode": "improve_precision"。 | 否 | | handler_type | 处理类型，可取值，
"check"：进行无标杆比对检查，默认值；
"fix"：将扰动后的 API 输出结果覆盖原始 API 输出结果，尝试将 Loss 曲线恢复正常，该模式下不支持预热 if_preheat。
**配置示例**："handler_type": "fix"。 | 否 | -| fuzz_level | 无标杆数据 dump 级别，即选择比对结果文件应输出的表头属性，当前仅支持取值为："L1"。输出结果详见**\*1**。 | 否 | +| fuzz_level | 无标杆数据 dump 级别，即选择比对结果文件应输出的表头属性，当前仅支持取值为："L1"。输出结果详见 [1.6.1 无标杆比对数据存盘格式]()。 | 否 | | fuzz_stage | 前反向，选择对 API 前向或反向进行无标杆比对，可取值：
"forward"：前向，默认值；
"backward"：反向。
**配置示例**："fuzz_stage": "backward"。 | 否 | | if_preheat | 预热功能（仅 PyTorch 场景支持），开启功能后工具可以根据每次迭代的输出调整精度算法的阈值，从而更准确地找出存在精度问题的 API，bool 类型。可取值：
true（开启）或 false（关闭），默认关闭。
**配置示例**："if_preheat": "true"。"handler_type": "fix"不支持预热。 | 否 | | preheat_step | 开启预热的迭代数量（仅 PyTorch 场景支持），int 类型，默认值为 15。须配置 "if_preheat": "true"。 | 否 | | max_sample | 每个算子预热的采样次数的最大阈值（仅 PyTorch 场景支持），int 类型，默认值为 20。须配置 "if_preheat": "true"。 | 否 | -**\***： -1. 无标杆比对数据存盘格式 +#### 1.6.1 无标杆比对数据存盘格式 - 无标杆比对在 dump_path 目录下输出结果文件 free_benchmark.csv，如下示例： +无标杆比对在 dump_path 目录下输出结果文件 `free_benchmark.csv`，如下示例： - ![free_benchmark](./img/free_benchmark.png) +![free_benchmark](./img/free_benchmark.png) - | 字段 | 说明 | - | ------------ | --------------------- | - | rank | Rank ID，int 类型。 | - | pert_mode | 扰动因子的类型，string 类型。 | - | stage | 前向或反向，string 类型。 | - | step | 迭代数，int 类型。 | - | api_name | API 名称，string 类型。 | - | max_rel | 输出对比最大相对误差，float 类型。 | - | dtype | 输入的 dtype，string 类型。 | - | shape | 输入的 shape，tuple 类型。 | - | output_index | 如果输出为列表或元组，其中一个元素检测不一致，则会有该元素的 index，否则为空，int 类型。 | +| 字段 | 说明 | +| ------------ | --------------------- | +| rank | Rank ID，int 类型。 | +| pert_mode | 扰动因子的类型，string 类型。 | +| stage | 前向或反向，string 类型。 | +| step | 迭代数，int 类型。 | +| api_name | API 名称，string 类型。 | +| max_rel | 输出对比最大相对误差，float 类型。 | +| dtype | 输入的 dtype，string 类型。 | +| shape | 输入的 shape，tuple 类型。 | +| output_index | 如果输出为列表或元组，其中一个元素检测不一致，则会有该元素的 index，否则为空，int 类型。 | diff --git a/debug/accuracy_tools/msprobe/docs/03.config_examples.md b/debug/accuracy_tools/msprobe/docs/03.config_examples.md index f51c84054..bfaae3f43 100644 --- a/debug/accuracy_tools/msprobe/docs/03.config_examples.md +++ b/debug/accuracy_tools/msprobe/docs/03.config_examples.md @@ -1,4 +1,4 @@ -# 配置示例 +# config.json 配置示例 以下示例包含当前支持的所有场景可配置的完整参数。 diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md index 3f2b2a88b..af61c7d31 100644 --- a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md @@ -23,7 +23,6 @@ functional: # functional为算子类别，找到对应的类别，在该类别 PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, model=None, step=None) ``` -**\***： 1. config_path：指定 dump 配置文件路径；model：指定具体的 torch.nn.Module，默认未配置，level 配置为"L0"或"mix"时必须配置该参数。其他参数均在 [config.json](../config.json) 文件中可配，详细配置可见 [config.json 介绍](./02.config_introduction.md)。 2. 此接口的参数均不是必要，且优先级高于 [config.json](../config.json) 文件中的配置，但可配置的参数相比 config.json 较少。 diff --git a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md index e7257c1b4..a89cddb67 100644 --- a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md @@ -2,12 +2,10 @@ ## 1 简介 -**MindSpore 动态图精度预检**\*1通过扫描昇腾 NPU 上用户训练 MindSpore 模型中的所有 Mint API，输出精度情况的诊断和分析。工具以模型中所有 Mint API 前反向的 dump 结果为输入，构造相应的 API 单元测试，将 NPU 输出与标杆（CPU 高精度）比对，计算对应的精度指标，从而找出 NPU 中存在精度问题的 Mint API。本工具支持随机生成模式和真实数据模式\*2。 +**MindSpore 动态图精度预检**a通过扫描昇腾 NPU 上用户训练 MindSpore 模型中的所有 Mint API，输出精度情况的诊断和分析。工具以模型中所有 Mint API 前反向的 dump 结果为输入，构造相应的 API 单元测试，将 NPU 输出与标杆（CPU 高精度）比对，计算对应的精度指标，从而找出 NPU 中存在精度问题的 Mint API。本工具支持随机生成模式和真实数据模式b。 -**\***： - -1. 支持 Mindspore 版本：2.4； -2. 在预检 dump 时可以选择由工具构造随机数进行输入获得 dump 数据或选择获取真实输入数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + a. 支持 Mindspore 版本：2.4； + b. 在预检 dump 时可以选择由工具构造随机数进行输入获得 dump 数据或选择获取真实输入数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 ## 2 离线预检流程 @@ -27,7 +25,7 @@ msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result | 参数名称 | 说明 |参数类型 | 是否必选 | | ---------------------------- | --------------------------------------|---------------------- | ---------------------------------- | | -api_info 或 --api_info_file | 指定 API 信息文件 dump.json。 | str | 是 | -| -o或--out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | +| -o 或 --out_path | 指定预检结果存盘路径，默认“./”。 | str | 否 | 预检执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 两个文件。`accuracy_checking_result_{timestamp}.csv` 属于 API 级，标明每个 API 是否通过测试。建议用户先查看 `accuracy_checking_result_{timestamp}.csv` 文件，对于其中没有通过测试的或者特定感兴趣的 API，根据其 API Name 字段在 `accuracy_checking_details_{timestamp}.csv` 中查询其各个输出的达标情况以及比较指标。详细介绍请参见 [4 预检结果](#4-预检结果)。 @@ -54,16 +52,14 @@ msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result | 字段 | 含义 | | --------------------- | ----------------- | | API Name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，error 为错误。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，error 为错误。SKIP 表示跳过该 API 的计算，跳过原因在 Message 中说明。 | | Backward Test Success | 反向 API 是否通过测试，pass 为通过，error 为错误，如果是空白的话代表该 API 没有反向输出。 | | Message | 提示信息。 | -Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差判定结果决定的。具体规则详见\*1。 +Forward Test Success 和 Backward Test Success 是否通过测试是由 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差判定结果决定的。具体规则详见 [4.1 API 预检指标](#41-api-预检指标)。 需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error。 -**\***： - -1. API预检指标： +### 4.1 API 预检指标 - API 预检指标是通过对`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差的数值进行判断，得出该API是否符合精度标准的参考指标。详细规则如下： -- Gitee From 1459c9ede5d2c8478ef0b0efc3c5c9f2150a8013 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 18:52:58 +0800 Subject: [PATCH 541/791] renew manual --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 44d7d2854..7c28869d3 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,10 @@ 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyTorch 场景性能数据进行分析，并输出性能调优建议。 +4. [bind_core](./profiler/affinity_cpu_bind) + + 绑核脚本，支持非侵入修改工程代码，实现一键式绑核功能。 + ## [Tensorboard](./plugins/tensorboard-plugins/tb_plugin) Tensorboard 支持 NPU 性能数据可视化插件 PyTorch Profiler TensorBoard NPU Plugin。 -- Gitee From 703621e3ed8020b126dc95e19a13d53d8294bd7f Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 19:03:22 +0800 Subject: [PATCH 542/791] renew manual --- debug/accuracy_tools/msprobe/README.md | 4 ++-- debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 56e7adbfc..cf1d70d6c 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -31,9 +31,9 @@ debugger.step() # 在训练循环的最后需要重置工具，非循环场景 msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API 执行精度数据 dump 操作，对应 config.json 中的 task 为 statistics 或 tensor。 -[PyTorch 场景下的数据采集](./docs/05.data_dump_PyTorch.md) +[PyTorch 场景的数据采集](./docs/05.data_dump_PyTorch.md) -[MindSpore 场景下的数据采集](./docs/06.data_dump_MindSpore.md) +[MindSpore 场景的数据采集](./docs/06.data_dump_MindSpore.md) ### 2 精度预检 diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md index af61c7d31..4b9a5e2bb 100644 --- a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md @@ -1,4 +1,4 @@ -# PyTorch 场景下的精度数据采集 +# PyTorch 场景的精度数据采集 msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 -- Gitee From ecb03ce1ad59a8979504f913bda849fdd4f62915 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 19:17:00 +0800 Subject: [PATCH 543/791] renew_manual --- .../msprobe/docs/09.accuracy_checker_MindSpore.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md index a89cddb67..4e86f7d89 100644 --- a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md @@ -52,7 +52,7 @@ msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result | 字段 | 含义 | | --------------------- | ----------------- | | API Name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，error 为错误。SKIP 表示跳过该 API 的计算，跳过原因在 Message 中说明。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，error 为错误。 | | Backward Test Success | 反向 API 是否通过测试，pass 为通过，error 为错误，如果是空白的话代表该 API 没有反向输出。 | | Message | 提示信息。 | -- Gitee From d2d15654d9d021a6300ba27bb94a471505ef7a9f Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 20:05:42 +0800 Subject: [PATCH 544/791] renew_manual --- README.md | 4 ++ debug/accuracy_tools/msprobe/README.md | 7 ++- .../msprobe/docs/06.data_dump_MindSpore.md | 4 +- .../docs/07.accuracy_checker_PyTorch.md | 36 +++++++-------- ... => 08.accuracy_checker_online_PyTorch.md} | 2 +- .../docs/10.accuracy_compare_PyTorch.md | 14 +++--- .../docs/11.accuracy_compare_MindSpore.md | 44 +++++++++---------- 7 files changed, 53 insertions(+), 58 deletions(-) rename debug/accuracy_tools/msprobe/docs/{08.accuracy_checker_online_Pytorch.md => 08.accuracy_checker_online_PyTorch.md} (99%) diff --git a/README.md b/README.md index 6a9b221a1..1e7f8ef97 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,10 @@ 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyThon 场景性能数据进行分析，并输出性能调优建议。 +4. [bind_core](./profiler/affinity_cpu_bind) + + 绑核脚本，支持非侵入修改工程代码，实现一键式绑核功能。 + ## [Tensorboard](./plugins/tensorboard-plugins/tb_plugin) Tensorboard 支持 NPU 性能数据可视化插件 PyTorch Profiler TensorBoard NPU Plugin。 diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 9720b9074..7b86123d9 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -6,11 +6,10 @@ [**msprobe**](./) 是 MindStudio Training Tools 工具链下精度调试部分的工具包。主要包括精度预检、溢出检测和精度比对等功能，目前适配 [PyTorch](https://pytorch.org/) 和 [MindSpore](https://www.mindspore.cn/) 框架。这些子工具侧重不同的训练场景，可以定位模型训练中的精度问题。 -为方便使用，本工具提供了一个统一、简易的程序接口，**PrecisionDebugger**，通过以下示例模板和 **config.json** 可轻松使用各种功能。 +为方便使用，本工具提供了一个统一、简易的程序接口，**PrecisionDebugger**，以 PyTorch 框架为例，通过以下示例模板和 **config.json** 可轻松使用各种功能。 ```python from msprobe.pytorch import PrecisionDebugger -# from msprobe.mindspore import PrecisionDebugger debugger = PrecisionDebugger(config_path='./config.json') ... @@ -32,9 +31,9 @@ debugger.step() # 在训练循环的最后需要重置工具，非循环场景 msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API 执行精度数据 dump 操作，对应 config.json 中的 task 为 statistics 或 tensor。 -[PyTorch 场景下的数据采集](./docs/05.data_dump_PyTorch.md) +[PyTorch 场景的数据采集](./docs/05.data_dump_PyTorch.md) -[MindSpore 场景下的数据采集](./docs/06.data_dump_MindSpore.md) +[MindSpore 场景的数据采集](./docs/06.data_dump_MindSpore.md) ### 2 精度预检 diff --git a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md index cad25d984..b8304ec5c 100644 --- a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md @@ -1,4 +1,4 @@ -# MindSpore 场景下的精度数据采集 +# MindSpore 场景的精度数据采集 msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 @@ -23,7 +23,6 @@ ops: # ops为算子类别，找到对应的类别，在该类别下按照下列 PrecisionDebugger(config_path=None) ``` -**\***： 1. config_path：指定 dump 配置文件路径，string 类型。参数示例："./config.json"。未配置该路径时，默认使用 [config.json](../config.json) 文件的默认配置。 ### 1.2 start @@ -35,7 +34,6 @@ PrecisionDebugger(config_path=None) ```Python debugger.start(model=None) ``` -**\***： 1. model：指具体的 mindspore.nn.Cell，默认未配置，L1 级别下传入 model 可以采集 primitive op。 ### 1.3 stop diff --git a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md index 290eec56b..b057f3a14 100644 --- a/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/07.accuracy_checker_PyTorch.md @@ -2,13 +2,11 @@ ## 1 简介 -**PyTorch 离线精度预检**\*1通过扫描昇腾 NPU 上用户训练模型中的所有 API，输出模型精度的诊断和分析结果。具体而言，该工具通过采集模型中所有 API 的前反向信息，构造相应的单元测试，将 NPU 输出与标杆（CPU 高精度）比对，从而计算对应的精度指标，该过程已封装为 run_ut.py 函数；将 NPU 环境下采集的预检数据拷贝至 GPU 环境，同样执行 run_ut.py；最后通过**新精度标准比对法**\*2将 NPU 和 GPU 的预检结果进行比对，从而找出 NPU 中存在精度问题的 API。同时，本工具支持随机生成模式和真实数据模式\*3。 +**PyTorch 离线精度预检**a通过扫描昇腾 NPU 上用户训练模型中的所有 API，输出模型精度的诊断和分析结果。具体而言，该工具通过采集模型中所有 API 的前反向信息，构造相应的单元测试，将 NPU 输出与标杆（CPU 高精度）比对，从而计算对应的精度指标，该过程已封装为 run_ut.py 函数；将 NPU 环境下采集的预检数据拷贝至 GPU 环境，同样执行 run_ut.py；最后通过**新精度标准比对法**b将 NPU 和 GPU 的预检结果进行比对，从而找出 NPU 中存在精度问题的 API。同时，本工具支持随机生成模式和真实数据模式c。 -**\***： - -1. 支持 PyTorch 版本：1.11/2.0/2.1/2.2； -2. 依据新精度标准，对不同的API采取不同的比对算法（包括绝对阈值法，标杆比对法、二进制一致法、ULP误差比对法和双千指标法），最终给定预检判定结果； -3. 在预检 dump 时可以选择由工具构造随机数获得 dump 数据或选择真实输入的数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 + a. 支持 PyTorch 版本：1.11/2.0/2.1/2.2； + b. 依据新精度标准，对不同的API采取不同的比对算法（包括绝对阈值法，标杆比对法、二进制一致法、ULP误差比对法和双千指标法），最终给定预检判定结果； + c. 在预检 dump 时可以选择由工具构造随机数获得 dump 数据或选择真实输入的数据进行预检 dump 操作。随机生成模式执行效率高，可以快速获得结果，但数据精度低，只能大致判断精度问题；真实数据模式执行效率略低于随机生成模式，但是数据精度高，可以准确判断精度问题。 ## 2 离线预检流程 @@ -56,11 +54,9 @@ run_ut 执行结果包括 `accuracy_checking_result_{timestamp}.csv` 和 `accura msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data ``` -数据默认会存盘到 './ut_error_data{timestamp}' 路径下（相对于启动 run_ut 的路径），如有需要，用户可以通过 error_data_path 参数来配置保存路径，error_data_path 参数在 [config.json](../config.json) 文件或 [config.yaml](../pytorch/api_accuracy_checker/config.yaml) 文件配置，config.json 文件需要在 run_ut 操作时通过 -config 参数指定，config.yaml 文件详见\*1。 - -**\***： +数据默认会存盘到 './ut_error_data{timestamp}' 路径下（相对于启动 run_ut 的路径），如有需要，用户可以通过 error_data_path 参数来配置保存路径，error_data_path 参数在 [config.json](../config.json) 文件或 [config.yaml](../pytorch/api_accuracy_checker/config.yaml) 文件配置，config.json 文件需要在 run_ut 操作时通过 -config 参数指定，config.yaml 。 -1. config.yaml 文件说明 +#### 3.1.1 config.yaml 文件说明 config.yaml 文件可以通过配置参数来控制 dump 和 run_ut 操作的白名单、黑名单等功能。操作步骤如下： @@ -83,14 +79,14 @@ msprobe -f pytorch run_ut -api_info ./dump.json -save_error_data | 参数名称 | 解释 | 是否必选 | | ----------- | -----------| -------- | - | white_list | API dump 白名单，仅对指定的 API 进行 dump。
**参数示例**：white_list=["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | - | black_list | API dump 黑名单，被指定的 API 不进行 dump。
**参数示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | - | error_data_path | 配置保存精度未达标的 API 输入输出数据路径。
**参数示例**："error_data_path": "./"。默认为当前路径。 | 否 | + | white_list | API dump 白名单，仅对指定的 API 进行 dump。
**配置示例**：white_list=["conv1d", "conv2d"]。默认未配置白名单，即 dump 全量 API 数据。 | 否 | + | black_list | API dump 黑名单，被指定的 API 不进行 dump。
**配置示例**：black_list=["conv1d", "conv2d"]。默认未配置黑名单，即 dump 全量 API 数据。 | 否 | + | error_data_path | 配置保存精度未达标的 API 输入输出数据路径。
**配置示例**："error_data_path": "./"。默认为当前路径。 | 否 | | precision | 浮点数表示位数，默认取小数点后14位。 | 否 | 说明：white_list 和 black_list 同时配置时，二者配置的 API 名单若无交集，则白名单生效，若 API 名单存在交集，则白名单排除的部分以及交集的 API 不进行 dump。 -2. API 预检黑名单和白名单 +#### 3.1.2 API 预检黑名单和白名单 run_ut 过程支持 API 预检黑名单和白名单，通过如下文件配置 black_list（黑名单）或white_list（白名单）参数来指定不需要或需要预检的API名称： @@ -133,7 +129,7 @@ msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_ 精度预检生成的 `accuracy_checking_result_{timestamp}.csv` 和 `accuracy_checking_details_{timestamp}.csv` 文件示例如下： -可以通过先查看 `accuracy_checking_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `accuracy_checking_details_{timestamp}.csv` 文件的 API 详细达标情况，API 达标情况介绍请参见\*1。 +可以通过先查看 `accuracy_checking_result_{timestamp}.csv` 文件的 Forward Test Success 和 Backward Test Success，判断是否存在未通过测试的 API，再查看 `accuracy_checking_details_{timestamp}.csv` 文件的 API 详细达标情况，详细介绍请参见 [4.1 API 预检指标](#41-api-预检指标)。 `accuracy_checking_result_{timestamp}.csv` @@ -142,8 +138,8 @@ msprobe -f pytorch run_ut -api_info ./dump.json -csv_path /home/xxx/ut/accuracy_ | 字段 | 含义 | | --------------------- | ------------------------- | | API name | API 名称。 | -| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。 | -| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误，如果是空白的话代表该 API 没有反向输出，SKIP 表示该 API 的某个参数的反向不计算梯度。 | +| Forward Test Success | 前向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。SKIP 表示跳过该 API 的计算，跳过原因在 Message 中说明。 | +| Backward Test Success | 反向 API 是否通过测试，pass 为通过，warning 为待观察，error 为错误。如果是空白的话代表该 API 没有反向输出，SKIP 表示该 API 的某个参数的反向不计算梯度。 | | Message | 提示信息。 | 该结果为中间结果，仅作为参考，建议完成 [5 预检结果比对](#5-预检结果比对)后查看比对结果。该结果后续将会删除。 @@ -170,7 +166,7 @@ Forward Test Success 和 Backward Test Success 是否通过测试是由 `accurac | 二进制一致错误率 | NPU 或 GPU 数据中每个 Tensor 精度不一致的数值的数量与 Tensor 中数值数量的比值。只有数据是 builtin 类型（bool、int、float、str）、torch.bool 和 torch 的 int 类型或者在新精度标准中使用二进制一致算法进行比对的 API 才会展示。 | | 误差均衡性 | NPU 或 GPU 数据与标杆数据精度差的上下浮动情况。 | | 均方根误差 | NPU 或 GPU 数据与标杆数据的均方根误差。 | -| 小值域错误占比 | NPU 或 GPU Tensor 中与标杆的绝对误差大于错误阈值的小值在小值域（小值的总数量）中的占比。判断为小值以及绝对误差的错误阈值参见\*2。 | +| 小值域错误占比 | NPU 或 GPU Tensor 中与标杆的绝对误差大于错误阈值的小值在小值域（小值的总数量）中的占比。判断为小值以及绝对误差的错误阈值参见 [4.2 小值域阈值](#42-小值域阈值)。 | | 相对误差最大值 | NPU 或 GPU 数据与标杆数据相对误差的最大值。 | | 相对误差平均值 | NPU 或 GPU 数据与标杆数据相对误差的平均值。 | | inf/nan 错误率 | NPU 与标杆 inf/nan 计算不一致的元素个数占总元素的个数比例。 | @@ -184,7 +180,7 @@ Forward Test Success 和 Backward Test Success 是否通过测试是由 `accurac **\***： -1. API 预检指标： +### 4.1 API 预检指标 API 预检指标是通过对 `accuracy_checking_details_{timestamp}.csv` 中的余弦相似度、最大绝对误差双百、双千、双万精度指标的数值进行判断，得出该 API 是否符合精度标准的参考指标。 @@ -197,7 +193,7 @@ Forward Test Success 和 Backward Test Success 是否通过测试是由 `accurac + 对于 float32 和 float64 数据：双千指标不通过，标记 error；双千指标通过，双万指标不通过，标记 warning；双千、双万指标均通过，标记 pass。 - 在 `accuracy_checking_result_{timestamp}.csv` 中以 Forward Test Success 和 Backward Test Success 字段统计该算子前向反向输出的测试结果，对于标记 pass 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE 表示测试通过，对于标记 error 或 warning 的算子，则在 `accuracy_checking_result_{timestamp}.csv` 中标记 FALSE 表示测试不通过。由于一个算子可能有多个前向或反向的输入或输出，那么该类算子的输入或输出中必须全为 pass，才能在 `accuracy_checking_result_{timestamp}.csv` 中标记 TRUE，只要有一个输入或输出标记 error 或 warning，那么在`accuracy_checking_result_{timestamp}.csv`中标记 FALSE。 -2. 小值域阈值： +### 4.2 小值域阈值 判定为小值的阈值： diff --git a/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md b/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_PyTorch.md similarity index 99% rename from debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md rename to debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_PyTorch.md index 5fc048b53..8d69912fd 100644 --- a/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_Pytorch.md +++ b/debug/accuracy_tools/msprobe/docs/08.accuracy_checker_online_PyTorch.md @@ -195,4 +195,4 @@ bash train.sh 训练脚本执行完毕后，在GPU侧dump_path目录下生成比对结果文件， `accuracy_checking_result_{timestamp}_rank{rank_id}.csv`和`accuracy_checking_details_{timestamp}_rank{rank_id}.csv`记录两方比对结果 -`api_precision_compare_result_{timestamp}_rank{rank_id}.csv`和`api_precision_compare_details_{timestamp}_rank{rank_id}.csv`记录三方比对结果。详细介绍请参见[离线精度预检中的 **4 预检结果**](./07.accuracy_checker_Pytorch.md#4-预检结果)。 +`api_precision_compare_result_{timestamp}_rank{rank_id}.csv`和`api_precision_compare_details_{timestamp}_rank{rank_id}.csv`记录三方比对结果。详细介绍请参见[离线精度预检中的 **4 预检结果**](./07.accuracy_checker_PyTorch.md#4-预检结果)。 diff --git a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md index 85004c7e5..e59dd115a 100644 --- a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md @@ -2,7 +2,7 @@ ## 1 简介 -- 本节主要介绍通过命令行和比对函数的方式进行 CPU 或 GPU 与 NPU 的精度数据比对，执行精度比对操作前需要先完成 CPU 或 GPU 与 NPU 的精度数据 dump，参见 [PyTorch 场景下的数据采集](./05.data_dump_Pytorch.md)章节。 +- 本节主要介绍通过命令行和比对函数的方式进行 CPU 或 GPU 与 NPU 的精度数据比对，执行精度比对操作前需要先完成 CPU 或 GPU 与 NPU 的精度数据 dump，参见 [PyTorch 场景下的数据采集](./05.data_dump_PyTorch.md)章节。 - msprobe 使用子命令 compare 进行比对，可支持单卡和多卡场景的精度数据比对。 @@ -36,11 +36,11 @@ 将 CPU 或 GPU 与 NPU 的 dump 文件进行比对，支持单卡和多卡，可同时比对多卡的 dump 数据。多机场景需要每个设备单独执行比对操作。 -请先参见 [PyTorch 场景下的数据采集](./05.data_dump_Pytorch.md)章节完成 CPU 或 GPU 与 NPU 的精度数据 dump。 +请先参见 [PyTorch 场景下的数据采集](./05.data_dump_PyTorch.md)章节完成 CPU 或 GPU 与 NPU 的精度数据 dump。 **操作步骤**： -1. 创建比对文件，文件内容及示例请参见\*1。 +1. 创建[比对文件](#211-比对文件)。 2. 运行命令： @@ -52,17 +52,15 @@ | 参数名 | 说明 | 是否必选 | |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i 或 --input_path | 指定比对文件路径。比对文件内容及示例请参见\*1。 | 是 | + | -i 或 --input_path | 指定[比对文件](#211-比对文件)路径。 | 是 | | -o 或 --output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s 或 --stack_mode | 配置 stack_mode 的开关。仅当比对文件\*1配置 stack_path 需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | + | -s 或 --stack_mode | 配置 stack_mode 的开关。仅当[比对文件](#211-比对文件)配置 stack_path 需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | | -c 或 --compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | | -f 或 --fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | 3. 查看比对结果，请参见 [3 精度比对结果分析](#3-精度比对结果分析)。 -**\***： - -1. 比对文件： +#### 2.1.1 比对文件 以在当前目录创建 ./compare.json 为例。 diff --git a/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md b/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md index 7b5f7e730..b0977eb6a 100644 --- a/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md @@ -16,7 +16,7 @@ msprobe 精度比对工具主要通过对同一个模型，在两个不同的 Mi 1. 使用 MindSpore 进行 dump，得到不同框架版本的 dump 数据。 -2. 创建比对文件，文件内容及示例请参见\*1。 +2. 创建[比对文件](#211-比对文件)。 3. 运行命令： @@ -28,35 +28,35 @@ msprobe 精度比对工具主要通过对同一个模型，在两个不同的 Mi | 参数名 | 说明 | 是否必选 | |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i 或 --input_path | 指定比对文件路径。比对文件内容及示例请参见\*1。 | 是 | + | -i 或 --input_path | 指定[比对文件](#211-比对文件)路径。 | 是 | | -o 或 --output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s 或 --stack_mode | 配置 stack_mode 的开关。仅当比对文件\*1配置 stack_path 需要开启。默认未配置，表示关闭。 | 否 | + | -s 或 --stack_mode | 配置 stack_mode 的开关。仅当比对文件配置 stack_path 需要开启。默认未配置，表示关闭。 | 否 | | -c 或 --compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | | -f 或 --fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。默认未配置，表示关闭。 | 否 | 4. 查看比对结果，请参见 [PyTorch 场景的精度比对——精度比对结果分析](./11.accuracy_compare_PyTorch.md#3-精度比对结果分析)章节。 -**\***： -1. 比对文件 - 以在当前目录创建 ./compare.json 为例，单卡场景示例如下： +#### 2.1.1 比对文件 - ```json - { - "npu_path": "./npu_dump/dump.json", - "bench_path": "./bench_dump/dump.json", - "stack_path": "./npu_dump/stack.json", - "is_print_compare_log": true - } - ``` +以在当前目录创建 ./compare.json 为例，单卡场景示例如下： - **参数说明**： +```json +{ +"npu_path": "./npu_dump/dump.json", +"bench_path": "./bench_dump/dump.json", +"stack_path": "./npu_dump/stack.json", +"is_print_compare_log": true +} +``` - | 参数名 | 说明 | 是否必选 | - | -------------------- | ------------------------------------------------------------ | ------------------ | - | npu_path | 配置 NPU 环境下的 dump.json 文件（单卡场景），str 类型。 | 是 | - | bench_path | 配置 CPU、GPU 或 NPU 环境下的 dump.json 文件（单卡场景），str 类型。 | 是 | - | stack_path | 配置 NPU dump 目录下的 stack.json 文件，str 类型。 | 是| - | is_print_compare_log | 配置是否开启日志打屏，bool 类型。可取值 True 或 False，默认为 True。 | 否 | - \ No newline at end of file +**参数说明**： + +| 参数名 | 说明 | 是否必选 | +| -------------------- | ------------------------------------------------------------ | ------------------ | +| npu_path | 配置 NPU 环境下的 dump.json 文件（单卡场景），str 类型。 | 是 | +| bench_path | 配置 CPU、GPU 或 NPU 环境下的 dump.json 文件（单卡场景），str 类型。 | 是 | +| stack_path | 配置 NPU dump 目录下的 stack.json 文件，str 类型。 | 是| +| is_print_compare_log | 配置是否开启日志打屏，bool 类型。可取值 True 或 False，默认为 True。 | 否 | + \ No newline at end of file -- Gitee From 850bd899cd80a21405698c0bd28764e0ab58812d Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 20:40:19 +0800 Subject: [PATCH 545/791] modify handle data test --- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 2 +- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 15a0883f5..b0f5a5cbc 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -74,7 +74,7 @@ class TestDataCollector(unittest.TestCase): self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - mock_info.assert_called_with("msg") + mock_info.assert_called_with("msg", end='\r') mock_flush.assert_called() mock_write_json.assert_not_called() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index bc120f62e..5f832d447 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -25,7 +25,7 @@ class TestLog(TestCase): def test__print_log(self, mock_print): logger._print_log("level", "msg") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("\n", mock_print.call_args[1].get("end")) + self.assertEqual("\n", mock_print.call_args[1].get("end")) logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) -- Gitee From 51f6487aca36b917b3f9e0d654b6331a3c99b702 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 23:01:20 +0800 Subject: [PATCH 546/791] add spaces --- debug/accuracy_tools/msprobe/core/common/const.py | 1 + debug/accuracy_tools/msprobe/core/data_dump/data_collector.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 723b8549e..496243640 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,6 +309,7 @@ class MsgConst: """ Class for log messages const """ + SPACE_LENGTH = 20 LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 061911906..1501affaa 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -3,7 +3,7 @@ import os from msprobe.core.data_dump.scope import build_scope, ListScope from msprobe.core.data_dump.json_writer import DataWriter from msprobe.core.common.log import logger -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, MsgConst from msprobe.core.data_dump.data_processor.factory import DataProcessorFactory @@ -130,7 +130,7 @@ class DataCollector: if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) - logger.info(msg, end='\r') + logger.info(''.join([msg, ' '*MsgConst.SPACE_LENGTH]), end='\r') if use_buffer: self.data_writer.flush_data_when_buffer_is_full() else: -- Gitee From 595e8b608e1ba7ad62b5e296b604e21ee8b8128f Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 28 Aug 2024 23:17:50 +0800 Subject: [PATCH 547/791] add spaces --- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index b0f5a5cbc..4c40372af 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -25,6 +25,7 @@ from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.data_dump.json_writer import DataWriter from msprobe.core.data_dump.data_processor.base import BaseDataProcessor from msprobe.core.data_dump.data_processor.pytorch_processor import StatisticsDataProcessor +from msprobe.core.common.const import MsgConst class TestDataCollector(unittest.TestCase): @@ -74,7 +75,7 @@ class TestDataCollector(unittest.TestCase): self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - mock_info.assert_called_with("msg", end='\r') + mock_info.assert_called_with(''.join(["msg", ' '*MsgConst.SPACE_LENGTH]), end='\r') mock_flush.assert_called() mock_write_json.assert_not_called() -- Gitee From aebb4e6823c3f5d91acd5f3633ec7aa75bca3fcf Mon Sep 17 00:00:00 2001 From: hanqing Date: Tue, 27 Aug 2024 16:01:53 +0800 Subject: [PATCH 548/791] safety issues --- .../msprobe/core/common/utils.py | 4 ++-- .../msprobe/pytorch/parse_tool/lib/compare.py | 22 +++++-------------- .../msprobe/pytorch/parse_tool/lib/utils.py | 5 ++--- .../pytorch/parse_tool/lib/visualization.py | 9 ++------ 4 files changed, 12 insertions(+), 28 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 1593efc38..b5d387563 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -532,10 +532,10 @@ def write_csv(data, filepath, mode="a+"): change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) -def load_npy(filepath): +def load_npy(filepath, enable_pickle=False): check_file_or_directory_path(filepath) try: - npy = np.load(filepath) + npy = np.load(filepath, allow_pickle=enable_pickle) except Exception as e: logger.error(f"The numpy file failed to load. Please check the path: {filepath}.") raise RuntimeError(f"Load numpy file {filepath} failed.") from e diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py index b24065915..cf97f9c6f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py @@ -22,7 +22,7 @@ from collections import namedtuple from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException -from msprobe.core.common.utils import create_directory, write_csv, save_npy_to_txt +from msprobe.core.common.utils import create_directory, write_csv, save_npy_to_txt, load_npy from msprobe.core.common.file_check import FileChecker @@ -86,17 +86,8 @@ class Compare: if left is None or right is None: raise ParseException("invalid input or output") if self.util.check_path_valid(left) and self.util.check_path_valid(right): - try: - left_data = np.load(left) - right_data = np.load(right) - except UnicodeError as e: - self.log.error("%s %s" % ("UnicodeError", str(e))) - self.log.warning("Please check the npy file") - raise ParseException(ParseException.PARSE_UNICODE_ERROR) from e - except IOError: - self.log.error("Failed to load npy %s or %s." % (left, right)) - raise ParseException(ParseException.PARSE_LOAD_NPY_ERROR) from e - + left_data = load_npy(left) + right_data = load_npy(right) # save to txt if save_txt: save_npy_to_txt(left_data, left + ".txt") @@ -160,10 +151,9 @@ class Compare: return res def compare_npy(self, file, bench_file, output_path): - if self.util.check_path_valid(file): - data = np.load(file) - if self.util.check_path_valid(bench_file): - bench_data = np.load(bench_file) + if self.util.check_path_valid(file) and self.util.check_path_valid(bench_file): + data = load_npy(file) + bench_data = load_npy(bench_file) shape, dtype = data.shape, data.dtype bench_shape, bench_dtype = bench_data.shape, bench_data.dtype filename = os.path.basename(file) diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py index 5dee65364..8f48f2784 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py @@ -32,7 +32,7 @@ from msprobe.core.common.file_check import change_mode, check_other_user_writabl check_path_executable, check_path_owner_consistent from msprobe.core.common.const import FileCheckConst from msprobe.core.common.file_check import FileOpen, FileChecker -from msprobe.core.common.utils import check_file_or_directory_path +from msprobe.core.common.utils import check_file_or_directory_path, remove_path from msprobe.pytorch.common.log import logger @@ -134,8 +134,7 @@ class Util: @staticmethod def deal_with_dir_or_file_inconsistency(self, output_path): - if os.path.exists(output_path): - os.remove(output_path) + remove_path(output_path) raise ParseException("Inconsistent directory structure or file.") @staticmethod diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py index 2a274f3ca..8ca69af85 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py @@ -21,7 +21,7 @@ from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import save_npy_to_txt +from msprobe.core.common.utils import save_npy_to_txt, load_npy class Visualization: @@ -29,12 +29,7 @@ class Visualization: self.util = Util() def print_npy_summary(self, target_file): - try: - np_data = np.load(target_file, allow_pickle=True) - except UnicodeError as e: - self.util.log.error("%s %s" % ("UnicodeError", str(e))) - self.util.log.warning("Please check the npy file") - raise ParseException(ParseException.PARSE_UNICODE_ERROR) from e + np_data = load_npy(target_file, enable_pickle=True) table = self.util.create_table('', ['Index', 'Data']) flatten_data = np_data.flatten() tablesize = 8 -- Gitee From 96206d2979de83d8e25957e03f1e9877b245ba69 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 09:30:26 +0800 Subject: [PATCH 549/791] fix --- .../msprobe/pytorch/api_accuracy_checker/common/utils.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index e151d6693..3ce4e1369 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -138,7 +138,11 @@ class UtDataProcessor: api_args = api_name + Const.SEP + str(self.index) create_directory(self.save_path) file_path = os.path.join(self.save_path, f'{api_args}.pt') - tensor = element.contiguous().cpu().detach() + try: + tensor = element.contiguous().detach().cpu() + except RuntimeError: + logger.error(f"Failed to transfer tensor to cpu for {api_args}") + raise DumpException(DumpException.SAVE_TENSOR_ERROR) save_pt(tensor, file_path) self.index += 1 elif element is None or isinstance(element, (bool, int, float, str, slice)): -- Gitee From 1ff3daab063baf62be64a6fb27a3ed9e10671338 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 09:44:38 +0800 Subject: [PATCH 550/791] fix --- .../msprobe/pytorch/api_accuracy_checker/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 3ce4e1369..af280401d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -140,9 +140,9 @@ class UtDataProcessor: file_path = os.path.join(self.save_path, f'{api_args}.pt') try: tensor = element.contiguous().detach().cpu() - except RuntimeError: + except Exception as err: logger.error(f"Failed to transfer tensor to cpu for {api_args}") - raise DumpException(DumpException.SAVE_TENSOR_ERROR) + raise DumpException(DumpException.INVALID_DATA_ERROR) from err save_pt(tensor, file_path) self.index += 1 elif element is None or isinstance(element, (bool, int, float, str, slice)): -- Gitee From 3306f3c02da3f1358cfedf9351ab97690c055445 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 10:15:11 +0800 Subject: [PATCH 551/791] support npu api --- .../msprobe/core/common/const.py | 2 + .../api_accuracy_checker/run_ut/run_ut.py | 6 +- .../run_ut/run_ut_utils.py | 14 ++-- .../pytorch/bench_functions/fast_gelu.py | 2 +- .../bench_functions/npu_fusion_attention.py | 82 +++++++++++++++++-- .../msprobe/pytorch/function_factory.py | 9 +- .../pytorch/hook_module/wrap_npu_custom.py | 10 ++- 7 files changed, 102 insertions(+), 23 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 4998f925e..6c17d9c67 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -88,6 +88,8 @@ class Const: BOOL_TYPE = [bool, np.uint8] INT_TYPE = [np.int32, np.int64] NPU = 'NPU' + CPU_LOWERCASE = 'cpu' + CUDA_LOWERCASE = 'cuda' DISTRIBUTED = 'Distributed' INPLACE_LIST = [ diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 8930e853a..9f5c97b55 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -292,7 +292,7 @@ def blacklist_and_whitelist_filter(api_name, black_list, white_list): def is_unsupported_api(api_name): split_name = api_name.split(Const.SEP)[0] - flag = split_name in [Const.NPU, Const.DISTRIBUTED] + flag = split_name == Const.DISTRIBUTED if flag: logger.info(f"{split_name} api is not supported for run ut. SKIP.") return flag @@ -331,8 +331,8 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict cpu_args, cpu_kwargs = generate_cpu_params(args, kwargs, need_backward, api_name) device_args, device_kwargs = generate_device_params(args, kwargs, need_backward, api_name) bench_grad_out, device_grad_out = None, None - out = exec_api(api_type, api_name, cpu_args, cpu_kwargs) - device_out = exec_api(api_type, api_name, device_args, device_kwargs) + out = exec_api(api_type, api_name, cpu_args, cpu_kwargs, Const.CPU_LOWERCASE) + device_out = exec_api(api_type, api_name, device_args, device_kwargs, current_device) current_path = os.path.dirname(os.path.realpath(__file__)) ut_setting_path = os.path.join(current_path, "torch_ut_setting.json") api_setting_dict = get_json_contents(ut_setting_path) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index 84db853b9..2ded677d7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -55,20 +55,16 @@ def get_validated_details_csv_path(validated_result_csv_path): return validated_details_csv_path -def exec_api(api_type, api_name, args, kwargs): +def exec_api(api_type, api_name, device, args, kwargs): if api_type == "Functional": - functional_api = FunctionalOPTemplate(api_name, str, False) - out = functional_api.forward(*args, **kwargs) + torch_api = FunctionalOPTemplate(api_name, str, False) if api_type == "Tensor": - tensor_api = TensorOPTemplate(api_name, str, False) - out = tensor_api.forward(*args, **kwargs) + torch_api = TensorOPTemplate(api_name, str, False) if api_type == "Torch": torch_api = TorchOPTemplate(api_name, str, False) - out = torch_api.forward(*args, **kwargs) if api_type == "Aten": torch_api = AtenOPTemplate(api_name, None, False) - out = torch_api.forward(*args, **kwargs) if api_type == "NPU": - torch_api = NpuOPTemplate(api_name, None, False) - out = torch_api.forward(*args, **kwargs) + torch_api = NpuOPTemplate(api_name, None, False, device) + out = torch_api.forward(*args, **kwargs) return out diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py index a1a9ca080..c324e8c6f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py @@ -1,7 +1,7 @@ import torch -def fast_gelu(input0): +def npu_fast_gelu(input0): attr = 1.702 const_0 = 0 - attr const_1 = 1 diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index 63f1fa2a3..0de00b3ce 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -1,6 +1,14 @@ import torch import numpy as np from einops import rearrange +try: + import torch_npu +except ImportError: + is_gpu = True + from flash_attn import flash_attn_func +else: + is_gpu = False + from msprobe.pytorch.common.utils import logger @@ -43,7 +51,7 @@ def softmax_grad(dp, softmax_res): def broadcast_kv(num_heads, num_kv_heads, kv_tensor, dtype): - if num_kv_heads == 0 or num_kv_heads < num_heads: + if num_kv_heads == 0 or num_kv_heads > num_heads: raise ValueError(f"num_kv_heads must be non-zero and less than num_heads.") factor = num_heads // num_kv_heads @@ -266,12 +274,32 @@ def rebuild_softmax_by_max_sum(q, k, atten_mask, pse, scale, softmax_max, softma return softmax_res +def get_head_num(*args, **kwargs): + if kwargs.get("head_num"): + head_num = kwargs.get("head_num") + elif len(args) >= 4: + head_num = args[3] + else: + raise ValueError(f"Unsupported npu_fusion_attention args {args}.") + return head_num + + +def get_input_layout(*args, **kwargs): + if kwargs.get("input_layout"): + input_layout = kwargs.get("input_layout") + elif len(args) >= 5: + input_layout = args[4] + else: + raise ValueError(f"Unsupported npu_fusion_attention args {args}.") + return input_layout + + def npu_fusion_attention_forward_patch(*args, **kwargs): # query, key, value, head_num, input_layout - if len(args) != 5: - raise ValueError(f"Unsupported npu_fusion_attention args {args}.") + head_num = get_head_num(*args, **kwargs) + input_layout = get_input_layout(*args, **kwargs) - B, S1, S2, N1, N2, D, H1, H2, DTYPE = parse_bsnd_args(args[0], args[1], args[3], args[4]) + B, S1, S2, N1, N2, D, H1, H2, DTYPE = parse_bsnd_args(args[0], args[1], head_num, input_layout) if N1 == N2 and S1 == S2: logger.debug(f"running case : BNSD = {B}_{N1}_{S1}_{D}, sparse = {kwargs.get('sparse_mode', 0)}") else: @@ -332,7 +360,8 @@ def npu_fusion_attention_backward_patch(*args, **kwargs): def npu_fusion_attention(*args, **kwargs): new_args, dims_kwargs, new_kwargs = npu_fusion_attention_forward_patch(*args, **kwargs) - query, key, value, input_layout = new_args[0], new_args[1], new_args[2], new_args[4] + query, key, value = new_args[0], new_args[1], new_args[2] + input_layout = get_input_layout(*args, **kwargs) N1 = dims_kwargs.get("N1") N2 = dims_kwargs.get("N2") S1 = dims_kwargs.get("S1") @@ -419,3 +448,46 @@ def npu_fusion_attention_grad(*args, **kwargs): dv = convert_from_bnsd(dv, input_layout) return dq.cpu(), dk.cpu(), dv.cpu() + +def gpu_fusion_attention(*args, **kwargs): + deterministic = False + new_args, dims_kwargs, new_kwargs = npu_fusion_attention_forward_patch(*args, **kwargs) + query, key, value = new_args[0], new_args[1], new_args[2] + keep_prob = new_kwargs.get("keep_prob", 1.0) + scale = new_kwargs.get("scale") + N1 = dims_kwargs.get("N1") + N2 = dims_kwargs.get("N2") + S1 = dims_kwargs.get("S1") + S2 = dims_kwargs.get("S2") + B = dims_kwargs.get("B") + pse = new_kwargs.get("pse") + sparse_mode = new_kwargs.get("sparse_mode") + pre_tockens = new_kwargs.get("pre_tockens") + next_tockens = new_kwargs.get("next_tockens") + attn_mask = new_kwargs.get("atten_mask") + atten_mask_dtype = attn_mask.dtype if new_kwargs.get("atten_mask") is not None else None + pre_tockens = 65536 if pre_tockens > 65536 else pre_tockens + next_tockens = 65536 if next_tockens > 65536 else next_tockens + if (not atten_mask_dtype) or (sparse_mode == 4 and (next_tockens != 0 or pre_tockens < S1)) \ + or (sparse_mode == 0 and pre_tockens >= S1 and next_tockens >= S2): + atten_off = True + else: + atten_off = False + causal_switch = False if atten_off else True + if sparse_mode == 4: + window_left = pre_tockens + window_right = next_tockens + else: + pre_tockens = next_tockens = 65536 + window_left = pre_tockens - S1 + S2 + window_right = next_tockens + S1 - S2 + + pttype = query.dtype + if pse is not None: + alibi_scopes = torch.rand(B, N1, dtype=torch.float32) * 0.3 + else: + alibi_scopes = None + + out = flash_attn_func(query, key, value, dropout_p=(1-keep_prob), softmax_scale=scale, causal=causal_switch, + window_size=(window_left, window_right), alibi_scopes=alibi_scopes, deterministic=deterministic) + return out.cpu(), None, None diff --git a/debug/accuracy_tools/msprobe/pytorch/function_factory.py b/debug/accuracy_tools/msprobe/pytorch/function_factory.py index c2fd8bfd0..5bdb20584 100644 --- a/debug/accuracy_tools/msprobe/pytorch/function_factory.py +++ b/debug/accuracy_tools/msprobe/pytorch/function_factory.py @@ -2,11 +2,12 @@ from msprobe.pytorch.common.utils import logger from msprobe.pytorch.bench_functions.apply_adam_w import npu_apply_adam_w from msprobe.pytorch.bench_functions.confusion_transpose import npu_confusion_transpose, \ npu_confusion_transpose_backward -from msprobe.pytorch.bench_functions.fast_gelu import fast_gelu, npu_fast_gelu_backward +from msprobe.pytorch.bench_functions.fast_gelu import npu_fast_gelu, npu_fast_gelu_backward from msprobe.pytorch.bench_functions.layer_norm_eval import npu_layer_norm_eval from msprobe.pytorch.bench_functions.linear import npu_linear, npu_linear_backward from msprobe.pytorch.bench_functions.matmul_backward import matmul_backward -from msprobe.pytorch.bench_functions.npu_fusion_attention import npu_fusion_attention, npu_fusion_attention_grad +from msprobe.pytorch.bench_functions.npu_fusion_attention import npu_fusion_attention, npu_fusion_attention_grad, \ + gpu_fusion_attention from msprobe.pytorch.bench_functions.rms_norm import npu_rms_norm, npu_rms_norm_backward from msprobe.pytorch.bench_functions.rotary_mul import npu_rotary_mul, npu_rotary_mul_backward from msprobe.pytorch.bench_functions.scaled_mask_softmax import npu_scaled_masked_softmax, \ @@ -62,8 +63,8 @@ class Register(dict): # register for npu custom bench functions npu_custom_functions = Register() npu_custom_functions([ - npu_apply_adam_w, npu_confusion_transpose, fast_gelu, npu_layer_norm_eval, npu_linear, npu_fusion_attention, - npu_rms_norm, npu_rotary_mul, npu_scaled_masked_softmax, npu_swiglu + npu_apply_adam_w, npu_confusion_transpose, npu_fast_gelu, npu_layer_norm_eval, npu_linear, npu_fusion_attention, + npu_rms_norm, npu_rotary_mul, npu_scaled_masked_softmax, npu_swiglu, gpu_fusion_attention ]) # register for npu custom backward bench functions diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index c2300470e..37e6f0945 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -36,6 +36,11 @@ else: is_gpu = False +cuda_func_mapping = { + "npu_fusion_attention" : "gpu_fusion_attention" +} + + def get_npu_ops(): if torch_without_guard_version: _npu_ops = dir(torch.ops.npu) @@ -52,10 +57,11 @@ class HOOKNpuOP(object): class NpuOPTemplate(HOOKModule): - def __init__(self, op_name, hook, need_hook=True): + def __init__(self, op_name, hook, need_hook=True, device=Const.CPU_LOWERCASE): self.op_name_ = op_name self.prefix_op_name_ = "NPU" + Const.SEP + str(op_name) + Const.SEP self.need_hook = need_hook + self.device = device if need_hook: super().__init__(hook) @@ -64,6 +70,8 @@ class NpuOPTemplate(HOOKModule): if not self.need_hook: if self.op_name_ not in npu_custom_functions: raise Exception(f'There is not bench function {self.op_name_}') + if self.device == Const.CUDA_LOWERCASE: + self.op_name_ = cuda_func_mapping.get(self.op_name_, self.op_name_) return npu_custom_functions[self.op_name_](*args, **kwargs) if torch_without_guard_version: return getattr(torch.ops.npu, str(self.op_name_))(*args, **kwargs) -- Gitee From 6f05c2068bcd541016ad73bc2909ac644504e7eb Mon Sep 17 00:00:00 2001 From: makai Date: Thu, 29 Aug 2024 11:10:23 +0800 Subject: [PATCH 552/791] del. space --- debug/accuracy_tools/msprobe/test/core_ut/test_log.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py index 5f832d447..31d56acb0 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_log.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_log.py @@ -25,8 +25,8 @@ class TestLog(TestCase): def test__print_log(self, mock_print): logger._print_log("level", "msg") self.assertIn("[level] msg", mock_print.call_args[0][0]) - self.assertEqual("\n", mock_print.call_args[1].get("end")) - + self.assertEqual("\n", mock_print.call_args[1].get("end")) + logger._print_log("level", "msg", end="end") self.assertIn("[level] msg", mock_print.call_args[0][0]) self.assertEqual("end", mock_print.call_args[1].get("end")) -- Gitee From fe065fd7a1afd2c06de09d47f4a28320dd6c6c8f Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Thu, 29 Aug 2024 14:38:14 +0800 Subject: [PATCH 553/791] Security modification --- .../tensor_transport_layer/attl.py | 30 +++++++++---------- .../msprobe/pytorch/common/utils.py | 21 +++++++++++++ 2 files changed, 35 insertions(+), 16 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index ecf8527e5..e74eebb19 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -1,8 +1,7 @@ -import io +import glob import os.path import time import re -from pathlib import Path from multiprocessing import Queue from typing import Optional, Union, Dict, Any from dataclasses import dataclass @@ -13,8 +12,8 @@ from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.client import TCPClient from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server import TCPServer from msprobe.pytorch.common.utils import logger -from msprobe.pytorch.common.utils import save_pt from msprobe.core.common.utils import remove_path +from msprobe.pytorch.common.utils import save_api_data, load_api_data, save_pt, load_pt BufferType = Union[ApiData, Dict[str, Any], str] # Union[Tensor, Tuple[Optional[Tensor]]] @@ -43,7 +42,7 @@ class ATTL: self.kill_progress = False self.check_attl_config() if self.session_config.nfs_path: - self.nfs_path = Path(self.session_config.nfs_path) + self.nfs_path = self.session_config.nfs_path elif self.session_config.is_benchmark_device: self.socket_manager = TCPServer(self.session_config.connect_port, @@ -86,9 +85,8 @@ class ATTL: buffer.kwargs.pop('device') rank = buffer.rank if hasattr(buffer, "rank") and buffer.rank is not None else 0 step = buffer.step if hasattr(buffer, "step") else 0 - io_buff = io.BytesIO() try: - torch.save(buffer, io_buff) + io_buff = save_api_data(buffer) except Exception as e: self.logger.info(f"{buffer.name} can not be saved, skip: {e}") return @@ -122,9 +120,8 @@ class ATTL: if buffer == b"KILL_CONFIRM": self.kill_progress = True return "KILL_" - buffer = io.BytesIO(buffer) try: - buffer = torch.load(buffer, map_location="cpu") + buffer = load_api_data(buffer) except Exception as e: self.logger.warning("there is something error. please check it. %s", e) if isinstance(buffer, bytes): @@ -147,21 +144,22 @@ class ATTL: self.logger.warning("there is something error in save_pt. please check it. %s", e) def download(self): + buffer = None + cur_file = None for file_type in ("start*", "*.pt", "end*"): - cur_file = next(self.nfs_path.glob(file_type), None) - if cur_file is not None: + pattern = os.path.join(self.nfs_path, file_type) + files = glob.glob(pattern) + if len(files) > 0: + cur_file = files[0] break - if cur_file is None: - return None - else: - buffer = None + if cur_file is not None: try: - buffer = torch.load(cur_file) + buffer = load_pt(cur_file) except Exception as e: self.logger.warning("there is something error. please check it. %s", e) remove_path(cur_file) - return buffer + return buffer def move2device_exec(obj, device): diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 69b7d20ef..cb44c6bbe 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ +import io import logging import os import random @@ -287,6 +288,26 @@ def save_pt(tensor, filepath): change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) +def save_api_data(api_data): + """Save data to io stream""" + try: + io_buff = io.BytesIO() + torch.save(api_data, io_buff) + except Exception as e: + raise RuntimeError(f"save api_data to io_buff failed") from e + return io_buff + + +def load_api_data(api_data_bytes): + """Load data from bytes stream""" + try: + buffer = io.BytesIO(api_data_bytes) + buffer = torch.load(buffer, map_location="cpu") + except Exception as e: + raise RuntimeError(f"load api_data from bytes failed") from e + return buffer + + def _create_logger(level=logging.INFO): logger_ = logging.getLogger() logger_.setLevel(level) -- Gitee From dc86050147b9437292e014e131bcde4d483a53d9 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 15:10:45 +0800 Subject: [PATCH 554/791] support npu api --- debug/accuracy_tools/msprobe/core/common/const.py | 1 + .../api_accuracy_checker/run_ut/run_overflow_check.py | 4 ++-- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 6 +++--- .../tensor_transport_layer/device_dispatch.py | 2 +- .../pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py | 2 +- 5 files changed, 8 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 6c17d9c67..2fdddb263 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -88,6 +88,7 @@ class Const: BOOL_TYPE = [bool, np.uint8] INT_TYPE = [np.int32, np.int64] NPU = 'NPU' + NPU_LOWERCASE = 'npu' CPU_LOWERCASE = 'cpu' CUDA_LOWERCASE = 'cuda' DISTRIBUTED = 'Distributed' diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index 98e5eaab3..1575a77df 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -82,8 +82,8 @@ def run_torch_api(api_full_name, api_info_dict, real_data_path): npu_args, npu_kwargs = generate_device_params(args, kwargs, False, api_name) if kwargs.get("device"): del kwargs["device"] - out = exec_api(api_type, api_name, args, kwargs) - npu_out = exec_api(api_type, api_name, npu_args, npu_kwargs) + out = exec_api(api_type, api_name, Const.CPU_LOWERCASE, args, kwargs) + npu_out = exec_api(api_type, api_name, Const.NPU_LOWERCASE, npu_args, npu_kwargs) if out is None and npu_out is None: logger.warning("The %s overflow is a normal overflow, out and npu_out is None." % api_full_name) return diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 9f5c97b55..76dde782e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -331,8 +331,8 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict cpu_args, cpu_kwargs = generate_cpu_params(args, kwargs, need_backward, api_name) device_args, device_kwargs = generate_device_params(args, kwargs, need_backward, api_name) bench_grad_out, device_grad_out = None, None - out = exec_api(api_type, api_name, cpu_args, cpu_kwargs, Const.CPU_LOWERCASE) - device_out = exec_api(api_type, api_name, device_args, device_kwargs, current_device) + out = exec_api(api_type, api_name, Const.CPU_LOWERCASE, cpu_args, cpu_kwargs) + device_out = exec_api(api_type, api_name, current_device, device_args, device_kwargs) current_path = os.path.dirname(os.path.realpath(__file__)) ut_setting_path = os.path.join(current_path, "torch_ut_setting.json") api_setting_dict = get_json_contents(ut_setting_path) @@ -365,7 +365,7 @@ def run_torch_api_online(api_full_name, api_data, backward_content): if kwargs.get("device"): del kwargs["device"] - device_out = exec_api(api_type, api_name, args, kwargs) + device_out = exec_api(api_type, api_name, Const.CUDA_LOWERCASE, args, kwargs) device_out = move2device_exec(device_out, "cpu") return UtDataInfo(None, None, out, device_out, None, in_fwd_data_list, None, rank=api_data.rank) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 3535b0880..0865a8817 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -75,7 +75,7 @@ def online_precision_compare(api_data, device, common_config, api_precision_csv_ try: # NPU vs CPU - cpu_out = exec_api(api_type, api_name, npu_args, npu_kwargs) + cpu_out = exec_api(api_type, api_name, Const.CPU_LOWERCASE, npu_args, npu_kwargs) npu_data_info = UtDataInfo(None, None, npu_out, cpu_out, None, [], None, rank=api_data.rank) npu_detail = compare.compare_output(api_full_name, npu_data_info, True) npu_data = pd.DataFrame(npu_detail, columns=DETAIL_TEST_ROWS[-1]) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py index 3ade53454..0f0ea155c 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py @@ -21,7 +21,7 @@ class TestRunUtMethods(unittest.TestCase): [api_type, api_name, _, _] = api_full_name.split(".") args, kwargs, need_grad = get_api_info(api_info, api_name, None) cpu_args, cpu_kwargs = generate_cpu_params(args, kwargs, True, '') - out = exec_api(api_type, api_name, cpu_args, cpu_kwargs) + out = exec_api(api_type, api_name, Const.CPU_LOWERCASE, cpu_args, cpu_kwargs) self.assertEqual(out[0].dtype, torch.float32) self.assertTrue(out[0].requires_grad) self.assertEqual(out[0].shape, torch.Size([2048, 2, 1, 128])) -- Gitee From bf853562f7109300076d9c02d5d7affd54c28d0a Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 15:23:40 +0800 Subject: [PATCH 555/791] support npu api --- .../msprobe/pytorch/bench_functions/layer_norm_eval.py | 2 +- .../msprobe/pytorch/hook_module/wrap_npu_custom.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py index f6949c079..4bc6d9a7a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py @@ -1,6 +1,6 @@ import torch -def npu_layer_norm_eval(data, normalized_shape): +def npu_layer_norm_eval(data, normalized_shape, weight=None, bias=None, eps=1e-5): result = torch.nn.functional.layer_norm(data, normalized_shape) return result.cpu() diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index 37e6f0945..26add2b4a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -72,7 +72,8 @@ class NpuOPTemplate(HOOKModule): raise Exception(f'There is not bench function {self.op_name_}') if self.device == Const.CUDA_LOWERCASE: self.op_name_ = cuda_func_mapping.get(self.op_name_, self.op_name_) - return npu_custom_functions[self.op_name_](*args, **kwargs) + if self.device in [Const.CUDA_LOWERCASE, Const.CPU_LOWERCASE]: + return npu_custom_functions[self.op_name_](*args, **kwargs) if torch_without_guard_version: return getattr(torch.ops.npu, str(self.op_name_))(*args, **kwargs) else: -- Gitee From 4506c8c9e46254494d38a02edb77357ab87454f9 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 15:56:42 +0800 Subject: [PATCH 556/791] support npu_fusion_attention backward --- .../pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json index d8df6098b..1eb8b192a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/torch_ut_setting.json @@ -1,5 +1,8 @@ { "topk": { "grad_index": 0 + }, + "npu_fusion_attention": { + "grad_index": 0 } } \ No newline at end of file -- Gitee From c16345c6eb454f5a9ef5da79ea4069b6981716f9 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Thu, 29 Aug 2024 11:08:31 +0800 Subject: [PATCH 557/791] =?UTF-8?q?mstt=20advisor=20wheel=E5=8C=85=201.?= =?UTF-8?q?=E6=9B=B4=E6=96=B0=E6=9C=89=E6=A0=87=E6=9D=86=E9=9B=86=E7=BE=A4?= =?UTF-8?q?=E5=AF=B9=E6=AF=94=E5=8A=9F=E8=83=BD=20=202.=E4=BF=AE=E5=A4=8Dc?= =?UTF-8?q?luster=20-o=E6=97=A0=E6=B3=95=E6=8C=87=E5=AE=9A=E8=B7=AF?= =?UTF-8?q?=E5=BE=84=E9=97=AE=E9=A2=98=203.=E4=BF=AE=E5=A4=8D=E5=AE=89?= =?UTF-8?q?=E8=A3=85wheel=E4=BE=9D=E8=B5=96=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 1 + profiler/cli/cluster_cli.py | 2 +- profiler/requirements/build.txt | 3 ++- profiler/version.txt | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) diff --git a/profiler/README.md b/profiler/README.md index 5eae3c49d..05460f77f 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -107,6 +107,7 @@ Successfully installed msprof-analyze-{version} | profiler版本 | 发布日期 | 下载链接 | 校验码 | |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ------------------------------------------------------------ | + | 1.2.3 | 2024-08-29 | [msprof_analyze-1.2.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.3/msprof_analyze-1.2.3-py3-none-any.whl) | 72aa827b8b09557cfb29684e13b496527d53087f6cac6803ddf9933335fa8e0c | | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.2/msprof_analyze-1.2.2-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | diff --git a/profiler/cli/cluster_cli.py b/profiler/cli/cluster_cli.py index 22760b0bf..261d70dff 100644 --- a/profiler/cli/cluster_cli.py +++ b/profiler/cli/cluster_cli.py @@ -38,5 +38,5 @@ context_settings['ignore_unknown_options'] = True help='Path of cluster analysis output') @click.argument('args', nargs=-1) def cluster_cli(profiling_path, mode, cluster_analysis_output_path, args) -> None: - required_args = ('-d', profiling_path, '-m', mode, '-cp', cluster_analysis_output_path) + required_args = ('-d', profiling_path, '-m', mode, '-o', cluster_analysis_output_path) cluster_analysis_main(required_args + args) diff --git a/profiler/requirements/build.txt b/profiler/requirements/build.txt index cacefda04..ec879a353 100644 --- a/profiler/requirements/build.txt +++ b/profiler/requirements/build.txt @@ -10,5 +10,6 @@ requests xlsxwriter sqlalchemy urllib3<2.0 -numpy +bottleneck>=1.3.6 +numpy==1.26.4 pandas \ No newline at end of file diff --git a/profiler/version.txt b/profiler/version.txt index d2d61a7e8..e2cac26c1 100644 --- a/profiler/version.txt +++ b/profiler/version.txt @@ -1 +1 @@ -1.2.2 \ No newline at end of file +1.2.3 \ No newline at end of file -- Gitee From 3b3b56cd1b6b781cc3e5320e2e0b0fd88fab20c7 Mon Sep 17 00:00:00 2001 From: qianggee Date: Thu, 29 Aug 2024 08:45:56 +0000 Subject: [PATCH 558/791] safety check when create file --- .../accuracy_tools/kj600/kj600/anomaly_detect.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/kj600/kj600/anomaly_detect.py b/debug/accuracy_tools/kj600/kj600/anomaly_detect.py index cbd7b6daa..5793acd72 100644 --- a/debug/accuracy_tools/kj600/kj600/anomaly_detect.py +++ b/debug/accuracy_tools/kj600/kj600/anomaly_detect.py @@ -1,10 +1,12 @@ +import os import statistics as st from abc import ABC from typing import List import sys from torch.utils.tensorboard import SummaryWriter from collections import defaultdict -from kj600.utils import print_info_log +from kj600.utils import print_info_log, print_error_log +from kj600.file_check import check_path_before_create, change_mode, FileCheckConst, create_directory class ScanRule(ABC): def apply(self, history, cur): @@ -61,7 +63,16 @@ class bcolors: class SummaryWriterWithAD(SummaryWriter): def __init__(self, path, ad_rules, job_id, anomaly_inform=False): - super().__init__(path) + check_path_before_create(path) + create_directory(path) + change_mode(path, FileCheckConst.DATA_DIR_AUTHORITY) + try: + super().__init__(path) + except Exception as e: + print_error_log(f'error when init summary writer at {path}: {e}') + raise e + for event in os.listdir(path): + change_mode(os.path.join(path), FileCheckConst.DATA_FILE_AUTHORITY) self.tag2scalars = defaultdict(list) self.ad_rules = ad_rules self.job_id = job_id -- Gitee From 4568e5f34482892bc64527827bf505c91c9825cf Mon Sep 17 00:00:00 2001 From: qianggee Date: Thu, 29 Aug 2024 09:03:29 +0000 Subject: [PATCH 559/791] del redundant change_mode --- debug/accuracy_tools/kj600/kj600/anomaly_detect.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/kj600/kj600/anomaly_detect.py b/debug/accuracy_tools/kj600/kj600/anomaly_detect.py index 5793acd72..fcacc823e 100644 --- a/debug/accuracy_tools/kj600/kj600/anomaly_detect.py +++ b/debug/accuracy_tools/kj600/kj600/anomaly_detect.py @@ -65,7 +65,6 @@ class SummaryWriterWithAD(SummaryWriter): def __init__(self, path, ad_rules, job_id, anomaly_inform=False): check_path_before_create(path) create_directory(path) - change_mode(path, FileCheckConst.DATA_DIR_AUTHORITY) try: super().__init__(path) except Exception as e: -- Gitee From 8623420805c689954268cf9503092fffe79ad270 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 17:04:37 +0800 Subject: [PATCH 560/791] support npu api --- .../pytorch/api_accuracy_checker/compare/compare.py | 2 +- .../msprobe/pytorch/bench_functions/apply_adam_w.py | 2 +- .../pytorch/bench_functions/confusion_transpose.py | 2 +- .../msprobe/pytorch/bench_functions/fast_gelu.py | 2 +- .../msprobe/pytorch/bench_functions/layer_norm_eval.py | 2 +- .../msprobe/pytorch/bench_functions/linear.py | 2 +- .../pytorch/bench_functions/npu_fusion_attention.py | 8 ++++---- .../msprobe/pytorch/bench_functions/rms_norm.py | 2 +- .../msprobe/pytorch/bench_functions/rotary_mul.py | 2 +- .../pytorch/bench_functions/scaled_mask_softmax.py | 2 +- .../msprobe/pytorch/bench_functions/swiglu.py | 2 +- 11 files changed, 14 insertions(+), 14 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 2a459c22c..fc4917bdf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -270,7 +270,7 @@ class Comparator: if npu_dtype == torch.bfloat16: bench_output = bench_output.to(torch.float32) device_output = device_output.to(torch.float32) - bench_output = bench_output.numpy() + bench_output = bench_output.cpu().numpy() device_output = device_output.cpu().numpy() if cpu_shape != npu_shape: return CompareConst.ERROR, compare_column, f"The shape of bench{str(cpu_shape)} " \ diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py index caf21a604..ce0a999b1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/apply_adam_w.py @@ -25,4 +25,4 @@ def npu_apply_adam_w(beta1_power, beta2_power, lr, weight_decay, if (1 - beta1_power_out) == 0: beta1_power_out -= eps var_out = var_t + torch.div(-lr * m_out, (1 - beta1_power_out)).div(denom) - return var_out.cpu(), m_out.cpu(), v_out.cpu() + return var_out, m_out, v_out diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py index 627bf11b6..bbc45deef 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/confusion_transpose.py @@ -3,7 +3,7 @@ def npu_confusion_transpose(data, perm, shape, transpose_first): output = data.permute(*perm).contiguous().view(shape) else: output = data.view(shape).permute(*perm) - return output.cpu() + return output def npu_confusion_transpose_backward(grad, perm, shape, transpose_first): diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py index c324e8c6f..d86f371a4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/fast_gelu.py @@ -19,7 +19,7 @@ def npu_fast_gelu(input0): div_down_rec = torch.reciprocal(div_down) result = div_up * div_down_rec - return result.cpu() + return result def npu_fast_gelu_backward(grad, input_x): diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py index 4bc6d9a7a..517c3121e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/layer_norm_eval.py @@ -3,4 +3,4 @@ import torch def npu_layer_norm_eval(data, normalized_shape, weight=None, bias=None, eps=1e-5): result = torch.nn.functional.layer_norm(data, normalized_shape) - return result.cpu() + return result diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py index 95db875ed..2caa25893 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/linear.py @@ -3,7 +3,7 @@ import torch def npu_linear(x, weight, bias): output = torch.nn.functional.linear(x, weight, bias) - return output.cpu() + return output def npu_linear_backward(grad, input_data, weight): diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index 0de00b3ce..96148152d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -484,10 +484,10 @@ def gpu_fusion_attention(*args, **kwargs): pttype = query.dtype if pse is not None: - alibi_scopes = torch.rand(B, N1, dtype=torch.float32) * 0.3 + alibi_slopes = torch.rand(B, N1, dtype=torch.float32) * 0.3 else: - alibi_scopes = None + alibi_slopes = None out = flash_attn_func(query, key, value, dropout_p=(1-keep_prob), softmax_scale=scale, causal=causal_switch, - window_size=(window_left, window_right), alibi_scopes=alibi_scopes, deterministic=deterministic) - return out.cpu(), None, None + window_size=(window_left, window_right), alibi_slopes=alibi_slopes, deterministic=deterministic) + return out, None, None diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py index e647312fd..617c96020 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rms_norm.py @@ -4,7 +4,7 @@ import torch def npu_rms_norm(x, gamma, epsilon=1e-5): rstd = torch.rsqrt(torch.mean(torch.pow(x, 2), axis=-1, keepdim=True) + epsilon) res = x * rstd * gamma - return res.cpu(), rstd.float().cpu() + return res, rstd.float() def npu_rms_norm_backward(grad, x, gamma, rstd): diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py index 0e0fda5f7..9eb910559 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/rotary_mul.py @@ -5,7 +5,7 @@ def npu_rotary_mul(x, r1, r2): x1, x2 = torch.chunk(x, 2, -1) x_new = torch.cat((-x2, x1), dim=-1) output = r1 * x + r2 * x_new - return output.cpu() + return output def npu_rotary_mul_backward(dy_tensor, x, r1, r2): diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py index 8717aebaf..8fc92d0e8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/scaled_mask_softmax.py @@ -9,7 +9,7 @@ def npu_scaled_masked_softmax(x, mask, scale, fixed_triu_mask): x = x - torch.max(x, dim=-1, keepdims=True)[0] x = torch.exp(x.float()) y = torch.div(x, torch.sum(x, dim=-1, keepdims=True)) - return y.to(dtype).cpu() + return y.to(dtype) def npu_scaled_masked_softmax_backward(y_grad, y, mask, scale, fixed_triu_mask): diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py index e03c975a5..e7ffd4179 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/swiglu.py @@ -14,7 +14,7 @@ def npu_swiglu(x, dim=-1): tensor_out_float = torch.nn.functional.silu(tensor_self_float).type(tensor_dtype).type( torch.float32) * tensor_other_float output_data = tensor_out_float.type(tensor_dtype) - return output_data.cpu() + return output_data def npu_swiglu_backward(grad, x, dim=-1): -- Gitee From ddca0229dc7e6b718fcee851c4d7bd414204aacd Mon Sep 17 00:00:00 2001 From: qianggee Date: Thu, 29 Aug 2024 09:05:09 +0000 Subject: [PATCH 561/791] correct file path --- debug/accuracy_tools/kj600/kj600/anomaly_detect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/kj600/kj600/anomaly_detect.py b/debug/accuracy_tools/kj600/kj600/anomaly_detect.py index fcacc823e..7d3389a85 100644 --- a/debug/accuracy_tools/kj600/kj600/anomaly_detect.py +++ b/debug/accuracy_tools/kj600/kj600/anomaly_detect.py @@ -71,7 +71,7 @@ class SummaryWriterWithAD(SummaryWriter): print_error_log(f'error when init summary writer at {path}: {e}') raise e for event in os.listdir(path): - change_mode(os.path.join(path), FileCheckConst.DATA_FILE_AUTHORITY) + change_mode(os.path.join(path, event), FileCheckConst.DATA_FILE_AUTHORITY) self.tag2scalars = defaultdict(list) self.ad_rules = ad_rules self.job_id = job_id -- Gitee From d1d33d2f61aa3b856fee10ea2fa1751f624846f8 Mon Sep 17 00:00:00 2001 From: makai Date: Thu, 29 Aug 2024 17:25:40 +0800 Subject: [PATCH 562/791] renew manual --- README.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/README.md b/README.md index a66381243..7c28869d3 100644 --- a/README.md +++ b/README.md @@ -51,11 +51,7 @@ 3. [advisor](./profiler/advisor) -<<<<<<< HEAD - 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyThon 场景性能数据进行分析，并输出性能调优建议。 -======= 将 Ascend PyTorch Profiler 或者 msprof 采集的 PyTorch 场景性能数据进行分析，并输出性能调优建议。 ->>>>>>> upstream/master 4. [bind_core](./profiler/affinity_cpu_bind) -- Gitee From 4b350443429b7085bbca03feccb5f986eaeb3ca5 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 17:27:29 +0800 Subject: [PATCH 563/791] support npu api --- debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py index dbcc9bfa3..d2370b6d0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py @@ -26,4 +26,5 @@ def get_ops(): wrap_functional = ops.get('functional') wrap_tensor = ops.get('tensor') wrap_torch = ops.get('torch') - return set(wrap_functional) | set(wrap_tensor) | set(wrap_torch) + wrap_npu_ops = ops.get('torch_npu') + return set(wrap_functional) | set(wrap_tensor) | set(wrap_torch) | set(wrap_npu_ops) -- Gitee From 1bd89b1c76f1581010f559564056a8a2725e54e6 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 19:14:33 +0800 Subject: [PATCH 564/791] support npu api --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 4 ++-- .../pytorch/api_accuracy_checker/run_ut/run_ut_utils.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 76dde782e..1188db0fa 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -18,7 +18,7 @@ import torch from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Message, hf_32_standard_api, UtDataInfo, \ - get_validated_result_csv_path, get_validated_details_csv_path, exec_api + get_validated_result_csv_path, get_validated_details_csv_path, exec_api, unsupported_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ initialize_save_path, UtDataProcessor @@ -292,7 +292,7 @@ def blacklist_and_whitelist_filter(api_name, black_list, white_list): def is_unsupported_api(api_name): split_name = api_name.split(Const.SEP)[0] - flag = split_name == Const.DISTRIBUTED + flag = split_name == Const.DISTRIBUTED or api_name in unsupported_api if flag: logger.info(f"{split_name} api is not supported for run ut. SKIP.") return flag diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index 2ded677d7..acc613274 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -10,6 +10,7 @@ from msprobe.pytorch.hook_module.wrap_tensor import TensorOPTemplate from msprobe.pytorch.hook_module.wrap_torch import TorchOPTemplate hf_32_standard_api = ["conv1d", "conv2d"] +unsupported_api = ["NPU.npu_fusion_attention.1", "NPU.npu_fusion_attention.2"] class Backward_Message: -- Gitee From ab76e0393b25f5ed8aa62e3b2656295a31fbad34 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 19:18:38 +0800 Subject: [PATCH 565/791] support npu api --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 4 ++-- .../pytorch/api_accuracy_checker/run_ut/run_ut_utils.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 1188db0fa..76dde782e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -18,7 +18,7 @@ import torch from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import Backward_Message, hf_32_standard_api, UtDataInfo, \ - get_validated_result_csv_path, get_validated_details_csv_path, exec_api, unsupported_api + get_validated_result_csv_path, get_validated_details_csv_path, exec_api from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import gen_api_params, gen_args from msprobe.pytorch.api_accuracy_checker.common.utils import api_info_preprocess, \ initialize_save_path, UtDataProcessor @@ -292,7 +292,7 @@ def blacklist_and_whitelist_filter(api_name, black_list, white_list): def is_unsupported_api(api_name): split_name = api_name.split(Const.SEP)[0] - flag = split_name == Const.DISTRIBUTED or api_name in unsupported_api + flag = split_name == Const.DISTRIBUTED if flag: logger.info(f"{split_name} api is not supported for run ut. SKIP.") return flag diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index acc613274..2ded677d7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -10,7 +10,6 @@ from msprobe.pytorch.hook_module.wrap_tensor import TensorOPTemplate from msprobe.pytorch.hook_module.wrap_torch import TorchOPTemplate hf_32_standard_api = ["conv1d", "conv2d"] -unsupported_api = ["NPU.npu_fusion_attention.1", "NPU.npu_fusion_attention.2"] class Backward_Message: -- Gitee From fb5149c91b1d12624c44991b3afb3cb60f6ac8f0 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 19:36:13 +0800 Subject: [PATCH 566/791] support npu api --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 76dde782e..40cf2c9a1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -352,6 +352,9 @@ def run_torch_api(api_full_name, real_data_path, backward_content, api_info_dict device_grad_out = run_backward(device_args, device_grad, grad_index, device_out) else: backward_message += Backward_Message.MULTIPLE_BACKWARD_MESSAGE + if api_name == "npu_fusion_attention": + out = out[0] + device_out = device_out[0] return UtDataInfo(bench_grad_out, device_grad_out, device_out, out, bench_grad, in_fwd_data_list, backward_message) -- Gitee From 2f62f7d37b8a33c6d8622864134a99291cb5ef43 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 29 Aug 2024 20:52:03 +0800 Subject: [PATCH 567/791] support npu api --- .../msprobe/core/common/const.py | 1 + .../bench_functions/npu_fusion_attention.py | 31 ++++++++++++++----- 2 files changed, 25 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 2fdddb263..4c39af0ef 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -23,6 +23,7 @@ class Const: WHITE_LIST = 'white_list' BLACK_LIST = 'black_list' DUMP_TENSOR_DATA = 'dump_tensor_data' + NONE = None # dump mode ALL = "all" diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index 96148152d..d34ee556a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -5,12 +5,18 @@ try: import torch_npu except ImportError: is_gpu = True - from flash_attn import flash_attn_func + try: + # flash_attn为gpu的fa三方库 + from flash_attn import flash_attn_func + except ImportError: + #如果为cpu的ut环境，则不做任何处理 + pass else: is_gpu = False from msprobe.pytorch.common.utils import logger +from msprobe.core.common.const import Const gtype = torch.float64 # arm host必须选择float64，x86环境选择float32即可，64也行。arm计算很慢，s=8k的场景建议使用x86 softmax_build_mode = "QKV" # "MAX_SUM" @@ -449,6 +455,19 @@ def npu_fusion_attention_grad(*args, **kwargs): return dq.cpu(), dk.cpu(), dv.cpu() + +def is_attention_off_due_to_mask(atten_mask_dtype): + return not atten_mask_dtype + + +def is_attention_off_in_sparse_mode_4(sparse_mode, next_tockens, pre_tockens, S1): + return sparse_mode == 4 and (next_tockens != 0 or pre_tockens < S1) + + +def is_attention_off_in_sparse_mode_0(sparse_mode, pre_tockens, next_tockens, S1, S2): + return sparse_mode == 0 and pre_tockens >= S1 and next_tockens >= S2 + + def gpu_fusion_attention(*args, **kwargs): deterministic = False new_args, dims_kwargs, new_kwargs = npu_fusion_attention_forward_patch(*args, **kwargs) @@ -468,11 +487,9 @@ def gpu_fusion_attention(*args, **kwargs): atten_mask_dtype = attn_mask.dtype if new_kwargs.get("atten_mask") is not None else None pre_tockens = 65536 if pre_tockens > 65536 else pre_tockens next_tockens = 65536 if next_tockens > 65536 else next_tockens - if (not atten_mask_dtype) or (sparse_mode == 4 and (next_tockens != 0 or pre_tockens < S1)) \ - or (sparse_mode == 0 and pre_tockens >= S1 and next_tockens >= S2): - atten_off = True - else: - atten_off = False + atten_off = (is_attention_off_due_to_mask(atten_mask_dtype) or + is_attention_off_in_sparse_mode_4(sparse_mode, next_tockens, pre_tockens, S1) or + is_attention_off_in_sparse_mode_0(sparse_mode, pre_tockens, next_tockens, S1, S2)) causal_switch = False if atten_off else True if sparse_mode == 4: window_left = pre_tockens @@ -490,4 +507,4 @@ def gpu_fusion_attention(*args, **kwargs): out = flash_attn_func(query, key, value, dropout_p=(1-keep_prob), softmax_scale=scale, causal=causal_switch, window_size=(window_left, window_right), alibi_slopes=alibi_slopes, deterministic=deterministic) - return out, None, None + return out, Const.NONE, Const.NONE -- Gitee From c1d9d9ebcbf3e112e6035b3df1a681f5b6aeabb0 Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 30 Aug 2024 09:26:49 +0800 Subject: [PATCH 568/791] fix bug --- .../msprobe/pytorch/bench_functions/npu_fusion_attention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index 63f1fa2a3..b3443c573 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -43,7 +43,7 @@ def softmax_grad(dp, softmax_res): def broadcast_kv(num_heads, num_kv_heads, kv_tensor, dtype): - if num_kv_heads == 0 or num_kv_heads < num_heads: + if num_kv_heads == 0 or num_kv_heads > num_heads: raise ValueError(f"num_kv_heads must be non-zero and less than num_heads.") factor = num_heads // num_kv_heads -- Gitee From ee92d892f534185a905da5608305d9478a4774cc Mon Sep 17 00:00:00 2001 From: wuyulong11 <2284273586@qq.com> Date: Fri, 30 Aug 2024 09:27:51 +0800 Subject: [PATCH 569/791] =?UTF-8?q?=E3=80=90=E4=BF=AE=E6=94=B9=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E3=80=91=E3=80=90tbplugin=E3=80=91=E4=BF=AE=E6=94=B9c?= =?UTF-8?q?leancode=E9=97=AE=E9=A2=98=E5=92=8C=E6=9B=B4=E6=96=B0=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=E6=96=87=E4=BB=B6=20=E3=80=90=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=BA=BA=E3=80=91=20wuyulong?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tb_plugin/fe/src/components/TraceView.tsx | 2 +- .../tensorboard-plugins/tb_plugin/fe/src/utils/vscode.ts | 2 +- plugins/tensorboard-plugins/tb_plugin/setup.py | 6 +++--- .../tb_plugin/torch_tb_profiler/__init__.py | 2 +- .../tb_plugin/{ => torch_tb_profiler}/config/config.ini | 0 .../tb_plugin/torch_tb_profiler/profiler/data.py | 4 +++- .../tb_plugin/torch_tb_profiler/static/trace_embedding.html | 2 +- 7 files changed, 10 insertions(+), 8 deletions(-) rename plugins/tensorboard-plugins/tb_plugin/{ => torch_tb_profiler}/config/config.ini (100%) diff --git a/plugins/tensorboard-plugins/tb_plugin/fe/src/components/TraceView.tsx b/plugins/tensorboard-plugins/tb_plugin/fe/src/components/TraceView.tsx index 8f1f36843..9c34dee8c 100644 --- a/plugins/tensorboard-plugins/tb_plugin/fe/src/components/TraceView.tsx +++ b/plugins/tensorboard-plugins/tb_plugin/fe/src/components/TraceView.tsx @@ -59,7 +59,7 @@ export const TraceView: React.FC = (props) => { traceData.then((data) => { iframeRef.current?.contentWindow?.postMessage( { msg: 'data', data }, - '*' + window.origin ) }) } diff --git a/plugins/tensorboard-plugins/tb_plugin/fe/src/utils/vscode.ts b/plugins/tensorboard-plugins/tb_plugin/fe/src/utils/vscode.ts index 62f1a9080..523fd5510 100644 --- a/plugins/tensorboard-plugins/tb_plugin/fe/src/utils/vscode.ts +++ b/plugins/tensorboard-plugins/tb_plugin/fe/src/utils/vscode.ts @@ -8,6 +8,6 @@ export function navToCode(filename: string, line: number) { filename, line }, - '*' + window.origin ) } diff --git a/plugins/tensorboard-plugins/tb_plugin/setup.py b/plugins/tensorboard-plugins/tb_plugin/setup.py index 61d29c402..10d00e6af 100644 --- a/plugins/tensorboard-plugins/tb_plugin/setup.py +++ b/plugins/tensorboard-plugins/tb_plugin/setup.py @@ -25,9 +25,9 @@ from configparser import ConfigParser import setuptools - config = ConfigParser() -config.read('./config/config.ini') +config.read('./torch_tb_profiler/config/config.ini') + def read(rel_path): here = os.path.abspath(os.path.dirname(__file__)) @@ -97,7 +97,7 @@ setuptools.setup( }, packages=setuptools.find_packages(), package_data={ - "torch_tb_profiler": ["static/**"], + "torch_tb_profiler": ["static/**", "config/**"], }, entry_points={ "tensorboard_plugins": [ diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/__init__.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/__init__.py index fd7b265cf..2fbe631c1 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/__init__.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/__init__.py @@ -4,4 +4,4 @@ # Entry point for Pytorch TensorBoard plugin package. -__version__ = '0.4.0.8' +__version__ = '0.4.0.9' diff --git a/plugins/tensorboard-plugins/tb_plugin/config/config.ini b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/config/config.ini similarity index 100% rename from plugins/tensorboard-plugins/tb_plugin/config/config.ini rename to plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/config/config.ini diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py index ac06a2640..7cd6f0232 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py @@ -22,6 +22,7 @@ import gzip import io as sysio import json import math +import os.path import re import tempfile from json.decoder import JSONDecodeError @@ -46,7 +47,8 @@ from .trace import BaseEvent, EventTypes, MemoryEvent logger = utils.get_logger() config = ConfigParser() -config.read('../../config/config.ini') +config_path = os.path.join(os.getcwd(), 'torch_tb_profiler', 'config', '../config/config.ini') +config.read(config_path) class RunProfileData(object): diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/static/trace_embedding.html b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/static/trace_embedding.html index bb84da0d0..4ef884658 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/static/trace_embedding.html +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/static/trace_embedding.html @@ -91,7 +91,7 @@ found in the LICENSE file. Polymer.dom(document.body).appendChild(viewer); if (window.parent) { - window.parent.postMessage({ msg: 'ready' }, '*') + window.parent.postMessage({ msg: 'ready' }, window.origin) } }); }()); -- Gitee From e728a1c9e1aa826442a973bf9bf944ba68f8eccb Mon Sep 17 00:00:00 2001 From: gitee Date: Fri, 30 Aug 2024 09:28:53 +0800 Subject: [PATCH 570/791] fix bug --- .../msprobe/pytorch/bench_functions/npu_fusion_attention.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index b3443c573..63f1fa2a3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -43,7 +43,7 @@ def softmax_grad(dp, softmax_res): def broadcast_kv(num_heads, num_kv_heads, kv_tensor, dtype): - if num_kv_heads == 0 or num_kv_heads > num_heads: + if num_kv_heads == 0 or num_kv_heads < num_heads: raise ValueError(f"num_kv_heads must be non-zero and less than num_heads.") factor = num_heads // num_kv_heads -- Gitee From 00cc53e251ce3f6a9d34b8b42d10d210399b49e5 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 10:43:53 +0800 Subject: [PATCH 571/791] fix space length --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 496243640..8d301d72c 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,7 @@ class MsgConst: """ Class for log messages const """ - SPACE_LENGTH = 20 + SPACE_LENGTH = 1024 LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] -- Gitee From e4791cacaa07d436546b6d3fc2f67064ff6489df Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 10:44:28 +0800 Subject: [PATCH 572/791] fix space length --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 8d301d72c..fe3a8799e 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,7 @@ class MsgConst: """ Class for log messages const """ - SPACE_LENGTH = 1024 + SPACE_LENGTH = 512 LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] -- Gitee From f2738fe71786ec1a2065a221c86ac2216819d0f5 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 30 Aug 2024 10:57:21 +0800 Subject: [PATCH 573/791] bug None type and int require grad --- .../api_accuracy_checker/api_accuracy_checker.py | 9 ++++++++- .../mindspore/api_accuracy_checker/api_runner.py | 13 ++++++++----- .../api_accuracy_checker/compute_element.py | 9 +++++++++ .../msprobe/mindspore/api_accuracy_checker/utils.py | 7 +++++++ 4 files changed, 32 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 92b4fb9c3..8ade7bd70 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -8,7 +8,8 @@ from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation from msprobe.mindspore.api_accuracy_checker.base_compare_algorithm import compare_algorithms -from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict, global_context +from msprobe.mindspore.api_accuracy_checker.utils import (check_and_get_from_json_dict, global_context, + compute_element_list_trim_none) class BasicInfoAndStatus: @@ -58,6 +59,12 @@ class ApiAccuracyChecker: else: tested_outputs = api_info.get_compute_element_list(forward_or_backward, Const.OUTPUT) bench_outputs = api_runner(api_input_aggregation, api_name_str, forward_or_backward, Const.PT_FRAMEWORK) + tested_outputs = compute_element_list_trim_none(tested_outputs) + bench_outputs = compute_element_list_trim_none(bench_outputs) + if len(tested_outputs) != len(bench_outputs): + logger.warning(f"ApiAccuracyChecker.run_and_compare_helper: api: {api_name_str}.{forward_or_backward}, " + "number of bench outputs and tested outputs is different, comparing result can be wrong. " + f"tested outputs: {len(tested_outputs)}, bench outputs: {len(bench_outputs)}") # compare output output_list = [] diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index ff754ca20..f28a27646 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -30,6 +30,8 @@ api_parent_module_mapping = { (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional } +torch_float_type = [torch.float16, torch.float32, torch.float64, torch.bfloat16] + class ApiRunner: def __call__(self, api_input_aggregation, api_name_str, forward_or_backward=Const.FORWARD, api_platform=Const.MS_FRAMEWORK): @@ -133,17 +135,18 @@ class ApiRunner: res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_tuple] else: #set requires_grad - for tensor in inputs: - if hasattr(tensor, "requires_grad"): + requires_grad_index = [] + for index, tensor in enumerate(inputs): + if isinstance(tensor, torch.Tensor) and tensor.dtype in torch_float_type: setattr(tensor, "requires_grad", True) + requires_grad_index.append(index) forward_results = api_instance(*inputs, **kwargs) forward_results = convert_to_tuple(forward_results) for forward_res, gradient_in in zip(forward_results, gradient_inputs): forward_res.backward(gradient_in) backward_result_list = [] - for tensor in inputs: - if hasattr(tensor, "grad"): - backward_result_list.append(getattr(tensor, "grad")) + for index in requires_grad_index: + backward_result_list.append(getattr(inputs[index], "grad")) res_compute_element_list = [ComputeElement(parameter=api_res) for api_res in backward_result_list] return res_compute_element_list diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 1ed5ce20d..97aee03d0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -33,6 +33,8 @@ class ComputeElement: self._init_with_parameter(parameter) elif isinstance(compute_element_info, (list, dict)): self._init_from_compute_element_info(compute_element_info) + elif compute_element_info is None: + self._init_from_null_compute_element_info() else: logger.error_log_with_exp( "ComputeElement.__init__ failed: not init with parameter or compute_element info is not (list, dict)", @@ -109,6 +111,8 @@ class ComputeElement: Return: parameter: Union[int, float, str, slice, tuple, torch.Tensor, mindspore.Tensor] ''' + if self.parameter is None: + return None if isinstance(self.parameter, tuple): return tuple([compute_element.get_parameter(get_origin=get_origin, tensor_platform=tensor_platform) for compute_element in self.parameter]) @@ -156,6 +160,11 @@ class ComputeElement: ndarray = np.random.uniform(minimum, maximum, shape).astype(np_dtype) return ndarray + def _init_from_null_compute_element_info(self): + self.parameter = None + self.shape = tuple() + self.dtype = "None" + def _init_from_compute_element_info(self, compute_element_info): ''' Args: diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py index 54abf59c2..f394b4572 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py @@ -44,6 +44,13 @@ def convert_to_tuple(input): input_list = [input] return tuple(input_list) +def compute_element_list_trim_none(compute_element_list): + trimmed_list = [] + for compute_element in compute_element_list: + if compute_element.get_parameter() is not None: + trimmed_list.append(compute_element) + return trimmed_list + class GlobalContext: def __init__(self): self.is_constructed = True -- Gitee From 0c2199311c064beeb7ea23741fc5f5276c0cbf01 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 30 Aug 2024 11:04:32 +0800 Subject: [PATCH 574/791] dependency separation --- .../msprobe/mindspore/api_accuracy_checker/cmd_parser.py | 6 ++++++ .../msprobe/mindspore/api_accuracy_checker/main.py | 7 ------- debug/accuracy_tools/msprobe/msprobe.py | 2 +- 3 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/cmd_parser.py diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/cmd_parser.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/cmd_parser.py new file mode 100644 index 000000000..c96287c0a --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/cmd_parser.py @@ -0,0 +1,6 @@ +def add_api_accuracy_checker_argument(parser): + parser.add_argument("-api_info", "--api_info_file", dest="api_info_file", type=str, required=True, + help=" The api param tool result file: generate from api param tool, " + "a json file.") + parser.add_argument("-o", "--out_path", dest="out_path", default="./", type=str, required=False, + help=" The ut task result out path.") \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/main.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/main.py index db830e76b..c33214898 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/main.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/main.py @@ -1,12 +1,5 @@ from msprobe.mindspore.api_accuracy_checker.api_accuracy_checker import ApiAccuracyChecker -def add_api_accuracy_checker_argument(parser): - parser.add_argument("-api_info", "--api_info_file", dest="api_info_file", type=str, required=True, - help=" The api param tool result file: generate from api param tool, " - "a json file.") - parser.add_argument("-o", "--out_path", dest="out_path", default="./", type=str, required=False, - help=" The ut task result out path.") - def api_checker_main(args): api_accuracy_checker = ApiAccuracyChecker() diff --git a/debug/accuracy_tools/msprobe/msprobe.py b/debug/accuracy_tools/msprobe/msprobe.py index f41dab68b..787d8759f 100644 --- a/debug/accuracy_tools/msprobe/msprobe.py +++ b/debug/accuracy_tools/msprobe/msprobe.py @@ -64,7 +64,7 @@ def main(): _api_precision_compare_parser(api_precision_compare_cmd_parser) _run_overflow_check_parser(run_overflow_check_cmd_parser) elif is_mindspore_available: - from msprobe.mindspore.api_accuracy_checker.main import add_api_accuracy_checker_argument + from msprobe.mindspore.api_accuracy_checker.cmd_parser import add_api_accuracy_checker_argument add_api_accuracy_checker_argument(run_ut_cmd_parser) if len(sys.argv) == 1: -- Gitee From b9e86bbbf2132dfd2249023d8177ce30cbbb8745 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 11:07:51 +0800 Subject: [PATCH 575/791] fix space length --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index fe3a8799e..5f6038d50 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,7 @@ class MsgConst: """ Class for log messages const """ - SPACE_LENGTH = 512 + SPACE_LENGTH = 256 LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] -- Gitee From 42a515993b0abf49f6dac2996670601547bc6242 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 30 Aug 2024 11:24:25 +0800 Subject: [PATCH 576/791] online run_ut add rank restriction --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index c77078366..dd2f70f01 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -255,7 +255,8 @@ def run_api_online(config, compare): _, api_name = extract_basic_api_segments(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue - dispatcher.update_consume_queue(api_data) + if api_data.rank in config.online_config.rank_list: + dispatcher.update_consume_queue(api_data) def shared_storage_communication_flow(): flag_num = -1 @@ -276,7 +277,8 @@ def run_api_online(config, compare): _, api_name = extract_basic_api_segments(api_full_name) if blacklist_and_whitelist_filter(api_name, config.black_list, config.white_list): continue - dispatcher.update_consume_queue(api_data) + if api_data.rank in config.online_config.rank_list: + dispatcher.update_consume_queue(api_data) if config.online_config.nfs_path: shared_storage_communication_flow() -- Gitee From fee3eb53ca9aa6c4c346b7ace24cff339018626a Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 30 Aug 2024 11:25:09 +0800 Subject: [PATCH 577/791] fix online run_ut print api_name is None --- .../api_accuracy_checker/compare/api_precision_compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index 2774c2cd9..77bc8bfde 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -368,7 +368,7 @@ def analyse_csv(npu_data, gpu_data, config): if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, CompareConst.SKIP, CompareConst.SKIP, message]], config.result_csv_path) - print_test_success(api_full_name, CompareConst.SKIP, CompareConst.SKIP) + print_test_success(last_api_name, CompareConst.SKIP, CompareConst.SKIP) forward_status, backward_status = [], [] message = '' else: @@ -376,7 +376,7 @@ def analyse_csv(npu_data, gpu_data, config): backward_result = get_api_checker_result(backward_status) message += CompareMessage.get(last_api_name, "") if forward_result == CompareConst.ERROR else "" write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path) - print_test_success(api_full_name, forward_result, backward_result) + print_test_success(last_api_name, forward_result, backward_result) forward_status, backward_status = [], [] message = '' @@ -398,13 +398,13 @@ def analyse_csv(npu_data, gpu_data, config): if last_api_dtype in API_PRECISION_COMPARE_UNSUPPORT_LIST: message = unsupported_message write_csv([[last_api_name, CompareConst.SKIP, CompareConst.SKIP, message]], config.result_csv_path) - print_test_success(last_api_full_name, CompareConst.SKIP, CompareConst.SKIP) + print_test_success(last_api_name, CompareConst.SKIP, CompareConst.SKIP) else: forward_result = get_api_checker_result(forward_status) backward_result = get_api_checker_result(backward_status) message += CompareMessage.get(last_api_name, "") if forward_result == CompareConst.ERROR else "" write_csv([[last_api_name, forward_result, backward_result, message]], config.result_csv_path) - print_test_success(last_api_full_name, forward_result, backward_result) + print_test_success(last_api_name, forward_result, backward_result) def print_test_success(api_full_name, forward_result, backward_result): -- Gitee From b1fd7c7ef57c507cd0ab9960e02015135f9e056e Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 12:27:36 +0800 Subject: [PATCH 578/791] fix space length --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- debug/accuracy_tools/msprobe/core/data_dump/data_collector.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 5f6038d50..0bd8246aa 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,7 @@ class MsgConst: """ Class for log messages const """ - SPACE_LENGTH = 256 + SPACE_LENGTH = 128 LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 1501affaa..78fced99b 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -130,7 +130,8 @@ class DataCollector: if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) - logger.info(''.join([msg, ' '*MsgConst.SPACE_LENGTH]), end='\r') + msg.ljust(MsgConst.SPACE_LENGTH) + logger.info(msg, end='\r') if use_buffer: self.data_writer.flush_data_when_buffer_is_full() else: -- Gitee From 706446fd065bda44390f8e9508934d282c6aa172 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 13:03:18 +0800 Subject: [PATCH 579/791] fix space length --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- debug/accuracy_tools/msprobe/core/data_dump/data_collector.py | 2 +- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 3 ++- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 0bd8246aa..0367f10ce 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,7 @@ class MsgConst: """ Class for log messages const """ - SPACE_LENGTH = 128 + CHAR_LENGTH = 512 LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 78fced99b..601dcb890 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -130,7 +130,7 @@ class DataCollector: if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) - msg.ljust(MsgConst.SPACE_LENGTH) + msg.ljust(MsgConst.CHAR_LENGTH) logger.info(msg, end='\r') if use_buffer: self.data_writer.flush_data_when_buffer_is_full() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 4c40372af..19105374b 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -75,7 +75,8 @@ class TestDataCollector(unittest.TestCase): self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - mock_info.assert_called_with(''.join(["msg", ' '*MsgConst.SPACE_LENGTH]), end='\r') + msg.ljust(MsgConst.CHAR_LENGTH) + mock_info.assert_called_with(msg, end='\r') mock_flush.assert_called() mock_write_json.assert_not_called() -- Gitee From d2b62f7edeaba36080472a5a3cafb79f5ef20c2d Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 15:13:13 +0800 Subject: [PATCH 580/791] add clear symbol --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- debug/accuracy_tools/msprobe/core/data_dump/data_collector.py | 3 +-- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 3 +-- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 0367f10ce..a1b6046a6 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,7 @@ class MsgConst: """ Class for log messages const """ - CHAR_LENGTH = 512 + CLEAR_SYMBOL = "\033[K" LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 601dcb890..cbda55a31 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -130,8 +130,7 @@ class DataCollector: if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) - msg.ljust(MsgConst.CHAR_LENGTH) - logger.info(msg, end='\r') + logger.info(MsgConst.CLEAR_SYMBOL+msg, end='\r') if use_buffer: self.data_writer.flush_data_when_buffer_is_full() else: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 19105374b..509405286 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -75,8 +75,7 @@ class TestDataCollector(unittest.TestCase): self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - msg.ljust(MsgConst.CHAR_LENGTH) - mock_info.assert_called_with(msg, end='\r') + mock_info.assert_called_with(MsgConst.CLEAR_SYMBOL+msg, end='\r') mock_flush.assert_called() mock_write_json.assert_not_called() -- Gitee From 88034840f7068b95bb7531332a66a3bfd53d41d0 Mon Sep 17 00:00:00 2001 From: fanhong <2532845962@qq.com> Date: Fri, 30 Aug 2024 15:23:24 +0800 Subject: [PATCH 581/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E7=B1=BB=E5=B1=9E?= =?UTF-8?q?=E6=80=A7=E4=BD=BF=E7=94=A8list=E5=8A=A8=E6=80=81=E5=AD=98?= =?UTF-8?q?=E5=82=A8=E6=95=B0=E6=8D=AE=E6=9C=AA=E9=87=8A=E6=94=BE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/analyzer/computation/operator_checker.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/profiler/advisor/analyzer/computation/operator_checker.py b/profiler/advisor/analyzer/computation/operator_checker.py index e24eae1d0..c021541ed 100644 --- a/profiler/advisor/analyzer/computation/operator_checker.py +++ b/profiler/advisor/analyzer/computation/operator_checker.py @@ -34,11 +34,11 @@ class OperatorChecker(VersionControl): MSLite_OPERATOR_TUNE_SUGGESTION = f"Optimize operator by AOE in mindspore lite framework, such as:\n" \ f"converter_lite --fmk=ONNX --optimize=ascend_oriented --saveType=MINDIR " \ f"--modelFile=$user_model.onnx --outputFile=user_model --configFile=./config.txt\n" - _tune_op_list: List[str] = [] def __init__(self, cann_version: str): self.cann_version = cann_version self._op_list: List[OpInfo] = [] + self._tune_op_list: List[str] = [] @staticmethod def get_ratio(op_info: OpInfo, attr: str) -> float: @@ -56,7 +56,7 @@ class OperatorChecker(VersionControl): :return: checker name """ return cls._PROBLEM - + def check(self, profiling_data: ProfilingDataset) -> bool: """ check if any operator need optimize @@ -282,7 +282,7 @@ class OperatorChecker(VersionControl): logger.warning(self.SKIP_CHECK_MSG, self._CHECKER, "op summary") return False return True - + def get_details(self) -> list: """ get details of operator to be optimized -- Gitee From f8ed1e633231f825ba4ec98e88c37519a4b2865e Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 30 Aug 2024 15:31:07 +0800 Subject: [PATCH 582/791] add ut --- .../api_accuracy_checker.py | 6 +- .../api_accuracy_checker/api_runner.py | 5 +- .../mindspore/api_accuracy_checker/utils.py | 16 +++- .../files/{input.npy => 2_3_input.npy} | Bin .../api_accuracy_checker/files/2_input.npy | Bin 0 -> 136 bytes .../files/api_info_statistics.json | 4 +- .../files/api_info_tensor.json | 6 +- .../files/index_select_statistics_dump.json | 88 ++++++++++++++++++ .../files/index_select_tensor_dump.json | 88 ++++++++++++++++++ .../test_api_accuracy_checker.py | 60 ++++++++++++ .../api_accuracy_checker/test_api_info.py | 4 +- .../test_compute_element.py | 8 +- 12 files changed, 266 insertions(+), 19 deletions(-) rename debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/{input.npy => 2_3_input.npy} (100%) create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/2_input.npy create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_statistics_dump.json create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_tensor_dump.json diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 8ade7bd70..bb96644c6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -9,7 +9,7 @@ from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation from msprobe.mindspore.api_accuracy_checker.base_compare_algorithm import compare_algorithms from msprobe.mindspore.api_accuracy_checker.utils import (check_and_get_from_json_dict, global_context, - compute_element_list_trim_none) + trim_output_compute_element_list) class BasicInfoAndStatus: @@ -59,8 +59,8 @@ class ApiAccuracyChecker: else: tested_outputs = api_info.get_compute_element_list(forward_or_backward, Const.OUTPUT) bench_outputs = api_runner(api_input_aggregation, api_name_str, forward_or_backward, Const.PT_FRAMEWORK) - tested_outputs = compute_element_list_trim_none(tested_outputs) - bench_outputs = compute_element_list_trim_none(bench_outputs) + tested_outputs = trim_output_compute_element_list(tested_outputs, forward_or_backward) + bench_outputs = trim_output_compute_element_list(bench_outputs, forward_or_backward) if len(tested_outputs) != len(bench_outputs): logger.warning(f"ApiAccuracyChecker.run_and_compare_helper: api: {api_name_str}.{forward_or_backward}, " "number of bench outputs and tested outputs is different, comparing result can be wrong. " diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index f28a27646..4c2f84f0a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -9,6 +9,7 @@ from msprobe.core.common.const import Const, MsCompareConst from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple +from msprobe.mindspore.api_accuracy_checker.type_mapping import float_dtype_str_list, torch_dtype_to_dtype_str class ApiInputAggregation: @@ -30,7 +31,6 @@ api_parent_module_mapping = { (MsCompareConst.MINT_FUNCTIONAL, Const.PT_FRAMEWORK): torch.nn.functional } -torch_float_type = [torch.float16, torch.float32, torch.float64, torch.bfloat16] class ApiRunner: def __call__(self, api_input_aggregation, api_name_str, forward_or_backward=Const.FORWARD, @@ -137,7 +137,8 @@ class ApiRunner: #set requires_grad requires_grad_index = [] for index, tensor in enumerate(inputs): - if isinstance(tensor, torch.Tensor) and tensor.dtype in torch_float_type: + if isinstance(tensor, torch.Tensor) and \ + torch_dtype_to_dtype_str.get(tensor.dtype) in float_dtype_str_list: setattr(tensor, "requires_grad", True) requires_grad_index.append(index) forward_results = api_instance(*inputs, **kwargs) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py index f394b4572..16a97b24f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py @@ -1,5 +1,7 @@ from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.log import logger +from msprobe.core.common.const import Const +from msprobe.mindspore.api_accuracy_checker.type_mapping import float_dtype_str_list def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_type=None, accepted_value=None): ''' @@ -44,11 +46,19 @@ def convert_to_tuple(input): input_list = [input] return tuple(input_list) -def compute_element_list_trim_none(compute_element_list): +def trim_output_compute_element_list(compute_element_list, forward_or_backward): + ''' + Args: + compute_element_list: List[ComputeElement] + forward_or_backward: str, Union["forward", "backward"] + ''' trimmed_list = [] for compute_element in compute_element_list: - if compute_element.get_parameter() is not None: - trimmed_list.append(compute_element) + if compute_element.get_parameter() is None or \ + (forward_or_backward == Const.BACKWARD and compute_element.get_dtype() not in float_dtype_str_list): + # trim case: 1. parameter is None. 2. backward output has non float parameter + continue + trimmed_list.append(compute_element) return trimmed_list class GlobalContext: diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/input.npy b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/2_3_input.npy similarity index 100% rename from debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/input.npy rename to debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/2_3_input.npy diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/2_input.npy b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/2_input.npy new file mode 100644 index 0000000000000000000000000000000000000000..5454521756ce164d6ec6e0e21e28a985a24977cd GIT binary patch literal 136 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC%^qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= aXCxM+0{I$7I+{8PwF(pfE(RcA1Y!U##v5G# literal 0 HcmV?d00001 diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json index b3ba49e4a..0d3f0fd04 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_statistics.json @@ -13,7 +13,7 @@ ], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy" + "data_name": "2_3_input.npy" } ], "input_kwargs": {}, @@ -27,7 +27,7 @@ ], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy" + "data_name": "2_3_input.npy" } ] } diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json index b3ba49e4a..e91c88b1e 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/api_info_tensor.json @@ -1,5 +1,5 @@ { - "task": "statistics", + "task": "tensor", "dump_data_dir": "", "data": { "MintFunctional.relu.0.forward": { @@ -13,7 +13,7 @@ ], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy" + "data_name": "2_3_input.npy" } ], "input_kwargs": {}, @@ -27,7 +27,7 @@ ], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy" + "data_name": "2_3_input.npy" } ] } diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_statistics_dump.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_statistics_dump.json new file mode 100644 index 000000000..738ed4a79 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_statistics_dump.json @@ -0,0 +1,88 @@ +{ + "task": "statistics", + "dump_data_dir": "", + "data": { + "Mint.index_select.0.forward": { + "input_args": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "2_3_input.npy" + }, + { + "type": "int", + "value": 0 + }, + { + "type": "mindspore.Tensor", + "dtype": "Int32", + "shape": [ + 2 + ], + "Max": 1.0, + "Min": 0.0, + "data_name": "2_input.npy" + } + ], + "input_kwargs": {}, + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "2_3_input.npy" + } + ] + }, + "Mint.index_select.0.backward": { + "input": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "2_3_input.npy" + } + ], + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "2_3_input.npy" + }, + null, + { + "type": "mindspore.Tensor", + "dtype": "Int32", + "shape": [ + 2 + ], + "Max": 1.0, + "Min": 0.0, + "data_name": "2_input.npy" + } + ] + } + } +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_tensor_dump.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_tensor_dump.json new file mode 100644 index 000000000..b635d01c1 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_tensor_dump.json @@ -0,0 +1,88 @@ +{ + "task": "tensor", + "dump_data_dir": "", + "data": { + "Mint.index_select.0.forward": { + "input_args": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "2_3_input.npy" + }, + { + "type": "int", + "value": 0 + }, + { + "type": "mindspore.Tensor", + "dtype": "Int32", + "shape": [ + 2 + ], + "Max": 1.0, + "Min": 0.0, + "data_name": "2_input.npy" + } + ], + "input_kwargs": {}, + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "2_3_input.npy" + } + ] + }, + "Mint.index_select.0.backward": { + "input": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "2_3_input.npy" + } + ], + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 2, + 3 + ], + "Max": 3.0, + "Min": 1.0, + "data_name": "2_3_input.npy" + }, + null, + { + "type": "mindspore.Tensor", + "dtype": "Int32", + "shape": [ + 2 + ], + "Max": 1.0, + "Min": 0.0, + "data_name": "2_input.npy" + } + ] + } + } +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py index fb2b63461..372f50606 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py @@ -3,6 +3,7 @@ import sys import logging import os import json +import csv from msprobe.mindspore.api_accuracy_checker.api_accuracy_checker import ApiAccuracyChecker @@ -27,6 +28,15 @@ def modify_tensor_api_info_json(json_file_path, modified_dump_data_dir): with open(json_file_path, 'w') as json_file: json.dump(data, json_file, indent=4) +def check_csv(csv_path, row_num): + with open(csv_path, 'r', encoding="utf-8") as f: + csvreader = csv.reader(f) + assert row_num == len(csvreader) + +def find_with_prefix(directory, prefix): + entries = os.listdir(directory) + target_files = [entry for entry in entries if entry.startswith(prefix) and os.path.isfile(os.path.join(directory, entry))] + return target_files class TestApiAccuracyChecker(unittest.TestCase): @@ -38,6 +48,13 @@ class TestApiAccuracyChecker(unittest.TestCase): api_accuracy_checker.run_and_compare() api_accuracy_checker.to_detail_csv(result_directory) api_accuracy_checker.to_result_csv(result_directory) + detail_csv = find_with_prefix(result_directory, "accuracy_checking_detail") + assert len(detail_csv) == 1 + check_csv(detail_csv[0], 3) + + result_csv = find_with_prefix(result_directory, "accuracy_checking_result") + assert len(result_csv) == 1 + check_csv(result_csv[0], 2) delete_files_with_prefix(result_directory, "accuracy_checking") def test_tensor_mode(self): @@ -49,6 +66,49 @@ class TestApiAccuracyChecker(unittest.TestCase): api_accuracy_checker.run_and_compare() api_accuracy_checker.to_detail_csv(result_directory) api_accuracy_checker.to_result_csv(result_directory) + detail_csv = find_with_prefix(result_directory, "accuracy_checking_detail") + assert len(detail_csv) == 1 + check_csv(detail_csv[0], 3) + + result_csv = find_with_prefix(result_directory, "accuracy_checking_result") + assert len(result_csv) == 1 + check_csv(result_csv[0], 2) + modify_tensor_api_info_json(api_info_tensor_path, "") + delete_files_with_prefix(result_directory, "accuracy_checking") + + def test_index_select_api_statistics_mode(self): + api_info_statistics_path = os.path.join(directory, "files", "index_select_statistics_dump.json") + result_directory = os.path.join(directory, "files") + api_accuracy_checker = ApiAccuracyChecker() + api_accuracy_checker.parse(api_info_statistics_path) + api_accuracy_checker.run_and_compare() + api_accuracy_checker.to_detail_csv(result_directory) + api_accuracy_checker.to_result_csv(result_directory) + detail_csv = find_with_prefix(result_directory, "accuracy_checking_detail") + assert len(detail_csv) == 1 + check_csv(detail_csv[0], 3) + + result_csv = find_with_prefix(result_directory, "accuracy_checking_result") + assert len(result_csv) == 1 + check_csv(result_csv[0], 2) + delete_files_with_prefix(result_directory, "accuracy_checking") + + def test_index_select_api_tensor_mode(self): + api_info_tensor_path = os.path.join(directory, "files", "index_select_tensor_dump.json") + result_directory = os.path.join(directory, "files") + modify_tensor_api_info_json(api_info_tensor_path, result_directory) + api_accuracy_checker = ApiAccuracyChecker() + api_accuracy_checker.parse(api_info_tensor_path) + api_accuracy_checker.run_and_compare() + api_accuracy_checker.to_detail_csv(result_directory) + api_accuracy_checker.to_result_csv(result_directory) + detail_csv = find_with_prefix(result_directory, "accuracy_checking_detail") + assert len(detail_csv) == 1 + check_csv(detail_csv[0], 3) + + result_csv = find_with_prefix(result_directory, "accuracy_checking_result") + assert len(result_csv) == 1 + check_csv(result_csv[0], 2) modify_tensor_api_info_json(api_info_tensor_path, "") delete_files_with_prefix(result_directory, "accuracy_checking") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py index d2c62bc4f..d4f1c11d7 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_info.py @@ -34,7 +34,7 @@ class TestApiInfo(unittest.TestCase): ], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy", + "data_name": "2_3_input.npy", } ], "input_kwargs": { @@ -53,7 +53,7 @@ class TestApiInfo(unittest.TestCase): ], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy", + "data_name": "2_3_input.npy", } ], } diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py index 9ad54f6d0..db42bf2fd 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_compute_element.py @@ -79,7 +79,7 @@ class TestComputeElement(unittest.TestCase): "shape":[2, 3], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy" + "data_name": "2_3_input.npy" } compute_element = ComputeElement(compute_element_info=compute_element_info) self.assertTrue((compute_element.get_parameter(get_origin=True) == self.ms_tensor).all()) @@ -94,7 +94,7 @@ class TestComputeElement(unittest.TestCase): "shape":[2, 3], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy" + "data_name": "2_3_input.npy" } compute_element = ComputeElement(compute_element_info=compute_element_info) parameter = compute_element.get_parameter(get_origin=True) @@ -112,7 +112,7 @@ class TestComputeElement(unittest.TestCase): "shape":[2, 3], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy" + "data_name": "2_3_input.npy" }, { "type": "mindspore.Tensor", @@ -120,7 +120,7 @@ class TestComputeElement(unittest.TestCase): "shape":[2, 3], "Max": 3.0, "Min": 1.0, - "data_name": "input.npy" + "data_name": "2_3_input.npy" }, ] compute_element = ComputeElement(compute_element_info=compute_element_info) -- Gitee From bc99645a4bf2dcd701eb6208690bdc223629bc06 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 30 Aug 2024 15:51:37 +0800 Subject: [PATCH 583/791] fix ut --- .../test_api_accuracy_checker.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py index 372f50606..2df1d6e6f 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py @@ -31,11 +31,11 @@ def modify_tensor_api_info_json(json_file_path, modified_dump_data_dir): def check_csv(csv_path, row_num): with open(csv_path, 'r', encoding="utf-8") as f: csvreader = csv.reader(f) - assert row_num == len(csvreader) + assert row_num == sum(1 for row in csvreader) def find_with_prefix(directory, prefix): entries = os.listdir(directory) - target_files = [entry for entry in entries if entry.startswith(prefix) and os.path.isfile(os.path.join(directory, entry))] + target_files = [os.path.join(directory, entry) for entry in entries if entry.startswith(prefix) and os.path.isfile(os.path.join(directory, entry))] return target_files class TestApiAccuracyChecker(unittest.TestCase): @@ -43,6 +43,7 @@ class TestApiAccuracyChecker(unittest.TestCase): def test_statistics_mode(self): api_info_statistics_path = os.path.join(directory, "files", "api_info_statistics.json") result_directory = os.path.join(directory, "files") + delete_files_with_prefix(result_directory, "accuracy_checking") api_accuracy_checker = ApiAccuracyChecker() api_accuracy_checker.parse(api_info_statistics_path) api_accuracy_checker.run_and_compare() @@ -50,7 +51,7 @@ class TestApiAccuracyChecker(unittest.TestCase): api_accuracy_checker.to_result_csv(result_directory) detail_csv = find_with_prefix(result_directory, "accuracy_checking_detail") assert len(detail_csv) == 1 - check_csv(detail_csv[0], 3) + check_csv(detail_csv[0], 2) result_csv = find_with_prefix(result_directory, "accuracy_checking_result") assert len(result_csv) == 1 @@ -60,6 +61,7 @@ class TestApiAccuracyChecker(unittest.TestCase): def test_tensor_mode(self): api_info_tensor_path = os.path.join(directory, "files", "api_info_tensor.json") result_directory = os.path.join(directory, "files") + delete_files_with_prefix(result_directory, "accuracy_checking") modify_tensor_api_info_json(api_info_tensor_path, result_directory) api_accuracy_checker = ApiAccuracyChecker() api_accuracy_checker.parse(api_info_tensor_path) @@ -68,7 +70,7 @@ class TestApiAccuracyChecker(unittest.TestCase): api_accuracy_checker.to_result_csv(result_directory) detail_csv = find_with_prefix(result_directory, "accuracy_checking_detail") assert len(detail_csv) == 1 - check_csv(detail_csv[0], 3) + check_csv(detail_csv[0], 2) result_csv = find_with_prefix(result_directory, "accuracy_checking_result") assert len(result_csv) == 1 @@ -79,6 +81,7 @@ class TestApiAccuracyChecker(unittest.TestCase): def test_index_select_api_statistics_mode(self): api_info_statistics_path = os.path.join(directory, "files", "index_select_statistics_dump.json") result_directory = os.path.join(directory, "files") + delete_files_with_prefix(result_directory, "accuracy_checking") api_accuracy_checker = ApiAccuracyChecker() api_accuracy_checker.parse(api_info_statistics_path) api_accuracy_checker.run_and_compare() @@ -96,6 +99,7 @@ class TestApiAccuracyChecker(unittest.TestCase): def test_index_select_api_tensor_mode(self): api_info_tensor_path = os.path.join(directory, "files", "index_select_tensor_dump.json") result_directory = os.path.join(directory, "files") + delete_files_with_prefix(result_directory, "accuracy_checking") modify_tensor_api_info_json(api_info_tensor_path, result_directory) api_accuracy_checker = ApiAccuracyChecker() api_accuracy_checker.parse(api_info_tensor_path) -- Gitee From 74b7a806c5a7828419904b6c8b0708c38d6ac7c1 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 30 Aug 2024 16:22:19 +0800 Subject: [PATCH 584/791] clean code --- .../msprobe/mindspore/api_accuracy_checker/compute_element.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 97aee03d0..9fb44fbdc 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -112,7 +112,7 @@ class ComputeElement: parameter: Union[int, float, str, slice, tuple, torch.Tensor, mindspore.Tensor] ''' if self.parameter is None: - return None + return self.parameter if isinstance(self.parameter, tuple): return tuple([compute_element.get_parameter(get_origin=get_origin, tensor_platform=tensor_platform) for compute_element in self.parameter]) -- Gitee From f7b23b178faee4579e0b200dcbdd23169da55d4e Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Fri, 16 Aug 2024 11:16:34 +0800 Subject: [PATCH 585/791] =?UTF-8?q?grad=5Fprobe=E6=B7=BB=E5=8A=A0=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/docs/17.grad_probe.md | 207 ++++++++++++++++++ .../msprobe/mindspore/grad_probe/hook.py | 1 + .../msprobe/mindspore/ms_config.py | 3 +- .../pytorch/grad_probe/grad_monitor.py | 2 + .../msprobe/pytorch/pt_config.py | 3 +- 5 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/docs/17.grad_probe.md diff --git a/debug/accuracy_tools/msprobe/docs/17.grad_probe.md b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md new file mode 100644 index 000000000..38eeb3dc6 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md @@ -0,0 +1,207 @@ +# Ascend模型梯度状态监测工具 + +梯度状态监测工具提供了两种能力： + +- 将模型权重的梯度数据导出。这种功能可以将模型权重的梯度值以统计量的形式采集出来，用以分析问题。 +- 将两份梯度数据进行相似度对比。在有标杆问题中，可以确认训练过程中精度问题出现的step，以及抓取反向过程中的问题。 + +工具支持PyTorch版本：2.0/2.1/2.2；支持MindSpore版本：r2.3。 + +## 工具特性 + +- 使用便捷，无需在训练流程里插入代码 +- 可以精准定位问题出现的step + +## 使用方式 + +### 梯度数据导出 + +1. 创建配置文件config.json，样例如下： + + ```json + { + "task": "grad_probe", + "dump_path": "./dump_path", + "rank": [], + "step": [], + "grad_probe": { + "grad_level": "L1", + "param_list": [], + "bounds": [-1, 0, 1] + } + } + ``` + > step指的是优化器被调用的次数（并非模型跑的step，某些step，例如loss为nan时，不会调用优化器） + + **参数说明** + + | 参数 | 说明 | 输入类型 | 是否必选 | + |--------------------------------|-----------------------------------|-----------------|----------| + | task | 填为"grad_probe"。 | str | 是 | + | dump_path | 输出目录。如果不存在就会创建一个新目录。 | str | 是 | + | rank | rank id列表，在多卡场景下，表示需要导出梯度数据的进程的rank id。列表为空就表示导出所有rank的数据。默认为空。（MindSpore静态图模式下，当前暂不支持指定rank功能） | List[int] | 否 | + | step | step列表，表示需要导出数据的step列表。列表为空就表示导出所有step的数据。默认为空。（MindSpore静态图模式下，当前暂不支持指定step功能） | List[int] | 否 | + | grad_level | 输出级别。决定导出数据的详细程度，级别越大导出数据越详细。可取值：L0, L1, L2。默认L1。|str | 否 | + | param_list | 权重名称列表，表示需要监控的权重。列表为空就表示监控所有权重。默认为空。 | List[str] | 否 | + | bounds | 区间列表，用来划分区间以统计数值的分布。需要保证由数据小到大排列。可以使用默认值[-1, 0, 1]。 | List[float, int] | 否 | + + **不同级别的level的导出数据** + + + | 级别 | 特征数据表头 | 是否有方向数据 | + | ---- | ------------------------------------------------------------ | -------------- | + | L0 | ("param_name", "MD5", "max", "min", "norm", "shape") | 否 | + | L1 | ("param_name", "max", "min", "norm", "shape") | 是 | + | L2 | ("param_name", *intervals, "=0", "max", "min", "norm", "shape") | 是 | + + intervals就是根据值分布bounds划分出的区间。 + MindSpore静态图模式下，L0级别中暂不支持"MD5" + + **方向数据解释** + + 因为模型的参数往往非常大，所以存储真实数据是不可接受的，这里折衷一下，只存储梯度数据的正负号（一个布尔值），也就是方向。 + + **bounds和值分布解释** + + + 值分布：梯度数据落在各个区间的元素个数占总元素个数的比例。 + + bounds：一个列表，用来划分出区间以统计值分布。例如传入bounds = [-10, 0, 10]，此时有一个 grad_value: Tensor = [9.3 , 5.4, -1.0, -12.3]，依据 bounds 划分出 (-inf, -10]、(-10, 0]、(0, 10]、(10, inf) 四个区间，然后统计grad_value里的数据落在每个区间内的个数，得到 1、1、2、0。如下图所示： + ![Alt text](./img/grad_probe_image-1.png) + +2. 插入代码。示例代码如下： + +- PyTorch框架：模型构造完成后，传入config.json的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将`模型`作为参数传入。 +```python +from msprobe.pytorch import PrecisionDebugger +debugger = PrecisionDebugger("config_json_path") +debugger.monitor(model) +``` +- MindSpore框架：优化器构造完成后，传入config.json的路径实例化一个GradientMonitor对象，然后调用gm.monitor并将`优化器`作为参数传入。 +```python +from msprobe.mindspore import PrecisionDebugger +debugger = PrecisionDebugger("config_json_path") +debugger.monitor(optimizer) +``` + +3. 结束监控（MindSpore静态图模式下需要） + + 在训练结束之后，调用stop接口 + +```python +gm.stop() +``` + +### 输出结果 +**输出目录结构**（以level配置L2为例） + +```bash +{dump_path} + ├── rank{rank_id} + │ ├── grad_summary_{step}.csv + │ ├── step{step} + │ │ ├── {param_name}.npy +``` ++ {timestamp}：梯度工具导出数据的时候会在output_path下生成一个时间戳目录，然后在这个时间戳目录下输出结果。 ++ rank_{rank_id}：在分布式场景下，会记录卡的rank_id。非分布式场景下，如果是CPU则记录进程号，如果是CPU或GPU则记录卡号 ++ grad_summary_{step}.csv：会分step记录每一步的梯度数据统计值。 ++ step_{step}：这个目录下会存放该step的梯度的方向数据。 ++ {param_name}.pt(npy)：模型参数的梯度方向数据，PyTorch保存的是pt文件，MindSpore是npy文件。 + +**grad_summary_{step}.csv** + +样例如下： + +![Alt text](./img/grad_probe_image.png) + +| 字段 | 含义 | +| --------------------- | ------------------------------------------------------------| +| Param_name | 模型参数名称。 | +| MD5 | 梯度数据的MD5值。 | +| (-inf, -0.01]...[0.01, inf) | 梯度值落在区间内的元素个数占总元素的比例。 | +| =0 | 梯度为0的元素个数占总元素的比例。 | +| Max | 最大值。 | +| Min | 最小值。 | +| Norm | L2norm值。 | +| Shape | 形状。 | + +### 梯度相似度比对 + +会根据所导出的权重，分step比对梯度相似度，输出每个权重的梯度相似度和总的梯度相似度。单个权重的梯度相似度为两份方向数据的重合度，总的梯度相似度为每个权重的梯度相似度按元素个数加权。 + +#### 前提条件 + +- 相同配置下，以Level为L1或L2分别采集npu和gpu环境下的梯度数据。 +- 将两份梯度数据传到同一环境下。 + +#### 使用方式 + + +新建如下Python脚本，传入npu和gpu的dump_path以及输出目录，比对结果输出目录不存在的话会新建： + +```python +from msprobe import * +GradComparator.compare_distributed("配置文件里写的dump_path", + "配置文件里写的dump_path", + "比对结果输出目录") +``` + + +### 比对结果 + +**输出目录结构** + +如下为多卡比对结果，单卡则没有rank_{rank_id}这一级目录。 + +```bash +比对结果输出目录 + ├── rank{rank_id} + │ ├── similarities.csv + │ └── similarities_picture + │ ├── {param_name}.png + │ └── summary_similarities.png +``` + +**问题界定** + +原则：对于任意权重，第0步的梯度相似度低于0.97，或者某一步的梯度相似度下降超过0.03，认为这一步存在精度问题。例子如下： + +- 第0步相似度低于0.97 + +![Alt text](./img/grad_probe_image-3.png) + +- 第3步相似度下降超过0.03 + +![Alt text](./img/grad_probe_image-4.png) + +- 正常情况 + +![Alt text](./img/grad_probe_image-2.png) + +这个原则是一个经验性的指标，并不是严格的标注，还需要结合实际情况具体分析。 + +## 公开接口 + +**接口说明** + +```python +PrecisionDebugger.monitor(module) +``` + +| 参数 | 说明 | 是否必选 | +| ----- | -------------------- | -------- | +| module |Pytorch框架下传入模型，必须是torch.nn.Module；MindSpore框架下传入优化器。 | 是 | + + +**接口说明** + +```python +GradComparator.compare_distributed(dump_path1, dump_path2, output_path) +``` + +| 参数 | 说明 | 数据类型 | 是否必选 | +| ----- | -------------------- | -------- | ----- | +| dump_path1 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path。 | str | 是 | +| dump_path2 |需要比对的其中一个dump目录，也就是配置文件里写的dump_path，与dump_path1可以互换。 | str | 是 | +| output_path |输出结果目录，不存在会新建。 | str | 是 | + + +# FAQ diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py index 243fb33de..008fd6fe3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py @@ -78,6 +78,7 @@ def hook_pynative_optimizer(opt, hook_input): dummy_csv_input = CsvInput(None, None, hook_input.bounds) output_lines.insert(0, GradStatCsv.get_csv_header(level_adapted, dummy_csv_input)) write_csv(output_lines, output_csv_path) + logger.info(f"write grad data to {output_csv_path}") grad_context.update_step() opt.register_forward_pre_hook(hook_fn) diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index 4109a8679..fad61ec7b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -82,7 +82,8 @@ class GradProbeConfig(BaseConfig): super().__init__(json_config) self.grad_level = json_config.get("grad_level", "L1") self.param_list = json_config.get("param_list", []) - self.bounds = json_config.get("bounds", []) + self.bounds = json_config.get("bounds", [-1, 0, 1]) + self._check_config() def _check_config(self): if self.grad_level not in level_adp.keys(): diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index 36aec34e0..756ad8a93 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -61,6 +61,7 @@ class GradientMonitor: def _hook_optimizer(self): def optimizer_pre_step_hook(optimizer, args, kargs): self._step += 1 + logger.info(f"grad_probe: optimizer step {self._step}") if not data_in_list_target(self._step, self._target_step): return output_lines = [] @@ -86,5 +87,6 @@ class GradientMonitor: header_result = GradStatCsv.generate_csv_header(self._level_adp, self._bounds) output_lines.insert(0, header_result) write_csv(output_lines, output_path) + logger.info(f"write grad data to {output_path}") if int(torch.__version__.split('.')[0]) >= 2: register_optimizer_step_pre_hook(optimizer_pre_step_hook) diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index d1d95c3e5..4b36b350b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -139,7 +139,8 @@ class GradToolConfig(BaseConfig): super().__init__(json_config) self.grad_level = json_config.get("grad_level", "L1") self.param_list = json_config.get("param_list", []) - self.bounds = json_config.get("bounds", []) + self.bounds = json_config.get("bounds", [-1, 0, 1]) + self._check_config() def _check_config(self): if self.grad_level not in level_adp.keys(): -- Gitee From 5160a4b56270fc081abd1c201a8ddba7cefa7c4b Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 30 Aug 2024 17:08:35 +0800 Subject: [PATCH 586/791] safety problem bugfix --- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 788d8d6ff..5aa0c5909 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -7,6 +7,7 @@ from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.utils import create_directory, check_configuration_param, task_dumppath_get, \ check_compare_param, FileChecker from msprobe.core.common.utils import CompareException +from msprobe.pytorch.common.utils import load_pt class PTComparator (Comparator): @@ -18,7 +19,7 @@ class PTComparator (Comparator): path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.PT_SUFFIX, False) data_path = path_checker.common_check() - data_value = torch.load(data_path, map_location=torch.device('cpu')).detach() # detach for less memory + data_value = load_pt(data_path, to_cpu=True) if data_value.dtype == torch.bfloat16: data_value = data_value.to(torch.float32) data_value = data_value.numpy() -- Gitee From b29d0a80a7ef5e61930f8555f12bfa160bf7961a Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 30 Aug 2024 17:14:29 +0800 Subject: [PATCH 587/791] safety problem bugfix --- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 5aa0c5909..413eefda0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -19,7 +19,7 @@ class PTComparator (Comparator): path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.PT_SUFFIX, False) data_path = path_checker.common_check() - data_value = load_pt(data_path, to_cpu=True) + data_value = load_pt(data_path, to_cpu=True).detach() if data_value.dtype == torch.bfloat16: data_value = data_value.to(torch.float32) data_value = data_value.numpy() -- Gitee From ad8075085eaf35d62a13061bf4a22e2900e4a4e0 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Fri, 30 Aug 2024 17:18:09 +0800 Subject: [PATCH 588/791] Add reviewer. --- debug/OWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/OWNERS b/debug/OWNERS index 2eec05195..06d108d5c 100644 --- a/debug/OWNERS +++ b/debug/OWNERS @@ -10,3 +10,4 @@ reviewers: - litian_drinksnow - binghamhuang - TAJh +- jiandaobao -- Gitee From 568f7078caa47139d51c5c96db038bc18b6ddc1f Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 17:20:47 +0800 Subject: [PATCH 589/791] add clear symbol --- debug/accuracy_tools/msprobe/core/data_dump/data_collector.py | 2 +- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index cbda55a31..ba7787489 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -130,7 +130,7 @@ class DataCollector: if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) - logger.info(MsgConst.CLEAR_SYMBOL+msg, end='\r') + logger.info(MsgConst.CLEAR_SYMBOL + msg, end='\r') if use_buffer: self.data_writer.flush_data_when_buffer_is_full() else: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 509405286..6541309a2 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -75,7 +75,7 @@ class TestDataCollector(unittest.TestCase): self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - mock_info.assert_called_with(MsgConst.CLEAR_SYMBOL+msg, end='\r') + mock_info.assert_called_with(MsgConst.CLEAR_SYMBOL + msg, end='\r') mock_flush.assert_called() mock_write_json.assert_not_called() -- Gitee From 041c9350d52b812abeaff352063d525da5f44da0 Mon Sep 17 00:00:00 2001 From: zyy Date: Fri, 30 Aug 2024 17:24:17 +0800 Subject: [PATCH 590/791] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E7=94=9F=E6=88=90?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=EF=BC=8C=E5=8F=82=E6=95=B0=E4=B8=8D=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E6=89=93=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 11c52ae5b..d92c1f041 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -67,6 +67,7 @@ class Interface: def run(self): PathManager.check_input_directory_path(self.collection_path) PathManager.check_path_owner_consistent(self.collection_path) + PathManager.make_dir_safety(self.cluster_analysis_output_path) PathManager.check_path_writeable(self.cluster_analysis_output_path) data_map, data_type = self.allocate_prof_data() if not data_map: @@ -93,7 +94,10 @@ def cluster_analysis_main(args=None): parser.add_argument('-m', '--mode', choices=COMM_FEATURE_LIST, default='all', help="different analysis mode") parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') - args_parsed, _ = parser.parse_known_args(args=args) + args_parsed, unknown = parser.parse_known_args(args=args) + if unknown: + print(f"Unknown arguments provided: {unknown}") + parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, Constant.ANALYSIS_MODE: args_parsed.mode, -- Gitee From 6f4988c4d23c2fb306837122c9361e07a40a32a0 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 30 Aug 2024 17:25:38 +0800 Subject: [PATCH 591/791] safety problem bugfix --- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 413eefda0..1eb364e7b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -19,7 +19,7 @@ class PTComparator (Comparator): path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.PT_SUFFIX, False) data_path = path_checker.common_check() - data_value = load_pt(data_path, to_cpu=True).detach() + data_value = load_pt(data_path, to_cpu=True).detach() # detach() because numpy can not process gradient information if data_value.dtype == torch.bfloat16: data_value = data_value.to(torch.float32) data_value = data_value.numpy() -- Gitee From 03ba23d8ddeee0f8e9d16a59c39ee0ac4cbd2bdb Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 18:04:26 +0800 Subject: [PATCH 592/791] add clear symbol --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index a1b6046a6..4693aedf9 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,7 @@ class MsgConst: """ Class for log messages const """ - CLEAR_SYMBOL = "\033[K" + CLEAR_SYMBOL = "KKK" #"\033[K" LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 6541309a2..575a20c73 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -86,7 +86,7 @@ class TestDataCollector(unittest.TestCase): mock_update_data.assert_not_called() mock_info.assert_not_called() mock_write_json.assert_called() - + @patch.object(DataCollector, "update_construct") @patch.object(DataWriter, "update_stack") @patch.object(BaseDataProcessor, "analyze_api_call_stack") -- Gitee From 86cf98fc97d65d8b64c29cd2be80680867c537ef Mon Sep 17 00:00:00 2001 From: zyy Date: Fri, 30 Aug 2024 18:13:12 +0800 Subject: [PATCH 593/791] =?UTF-8?q?=E5=88=A0=E9=99=A4advisor=20output?= =?UTF-8?q?=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/dataset/cluster/cluster_dataset.py | 10 ++++------ profiler/cli/analyze_cli.py | 2 -- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index e268b4092..6894436ae 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -33,14 +33,13 @@ logger = logging.getLogger() class ClusterDataset(Dataset): def __init__(self, collection_path, data: dict, **kwargs) -> None: - self.cluster_analysis_output_path = kwargs.get(Constant.CLUSTER_ANALYSIS_OUTPUT_PATH, collection_path) super().__init__(collection_path, data) def is_cluster_analysis_output_exist(self): """ check whether input path is valid """ - for filename in os.listdir(self.cluster_analysis_output_path): + for filename in os.listdir(self.collection_path): if filename == 'cluster_analysis_output': logger.info("[INFO]Cluster has been analyzed " "because of the existence of cluster analysis output directory.") @@ -53,8 +52,7 @@ class ClusterDataset(Dataset): return parameter = { Constant.COLLECTION_PATH: self.collection_path, - Constant.ANALYSIS_MODE: "all", - Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: self.cluster_analysis_output_path + Constant.ANALYSIS_MODE: "all" } print("[INFO] cluster analysis is in the process, please wait...") try: @@ -87,7 +85,7 @@ class ClusterStepTraceTimeDataset(ClusterDataset): def __init__(self, collection_path: str, data: dict, **kwargs): self._step_dict = defaultdict() self._stages = [] - super().__init__(collection_path, data, **kwargs) + super().__init__(collection_path, data) def _parse(self): self.cluster_analyze() @@ -152,7 +150,7 @@ class ClusterCommunicationDataset(ClusterDataset): self.SDMA_SIZE_MB: 0, }) self.hccl_dict = defaultdict(lambda: defaultdict(lambda: defaultdict(list))) - super().__init__(collection_path, data, **kwargs) + super().__init__(collection_path, data) @staticmethod def compute_ratio(dividend: float, divisor: float): diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index 2b5f63ed7..01758f7b9 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -29,8 +29,6 @@ def analyze_cli(**kwargs): help='Directory of profiling data') @click.option('--benchmark_profiling_path', '-bp', 'benchmark_profiling_path', type=click.Path(), help='Directory of benchmark profiling data, used for compare performance') -@click.option('--output_path', '-o', 'cluster_analysis_output_path', type=click.Path(), - help='Path of cluster analysis output') @click.option('--cann_version', '-cv', 'cann_version', type=click.Choice(constant.SUPPORTED_CANN_VERSION, case_sensitive=False), default=constant.DEFAULT_CANN_VERSION, -- Gitee From 780c805dfa7edf1300ecd53aee99968e52ebb495 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 30 Aug 2024 18:31:23 +0800 Subject: [PATCH 594/791] add clear symbol --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 4693aedf9..a1b6046a6 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -309,7 +309,7 @@ class MsgConst: """ Class for log messages const """ - CLEAR_SYMBOL = "KKK" #"\033[K" + CLEAR_SYMBOL = "\033[K" LEVEL = ["INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 575a20c73..4282d8ffd 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -75,7 +75,8 @@ class TestDataCollector(unittest.TestCase): self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - mock_info.assert_called_with(MsgConst.CLEAR_SYMBOL + msg, end='\r') + + mock_info.assert_called_with(MsgConst.CLEAR_SYMBOL + "msg", end='\r') mock_flush.assert_called() mock_write_json.assert_not_called() -- Gitee From e54c43c43af6fa9694898bf625011e625fdbd692 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 31 Aug 2024 10:36:48 +0800 Subject: [PATCH 595/791] safety problem bugfix --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index cb44c6bbe..4e03f761d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -270,8 +270,8 @@ def load_pt(pt_path, to_cpu=False): pt = torch.load(pt_path, map_location=torch.device("cpu")) else: pt = torch.load(pt_path) - except Exception as e: - raise RuntimeError(f"load pt file {pt_path} failed") from e + except Exception: + logger.error(f"load pt file {pt_path} failed") return pt -- Gitee From 9ba878db97117e4b930aa84e5dcc93fc979f04db Mon Sep 17 00:00:00 2001 From: wangqihui01 Date: Sat, 31 Aug 2024 10:38:10 +0800 Subject: [PATCH 596/791] =?UTF-8?q?=E5=8E=BB=E9=99=A4redos=E9=A3=8E?= =?UTF-8?q?=E9=99=A9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common_func_advisor/trace_view_preprocessor.py | 2 +- .../advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py b/profiler/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py index 7b9baa32d..6eced27ef 100644 --- a/profiler/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py +++ b/profiler/advisor/advisor_backend/common_func_advisor/trace_view_preprocessor.py @@ -104,7 +104,7 @@ class TraceViewPreProcessor: check whether op is hcom send or recv op """ # eg: hcom_BatchSendRecv__101_0_1 - p1 = re.compile(r'hcom_\w+SendRecv__\d+') + p1 = re.compile(r'^hcom_\w+SendRecv__\d+') # eg: hcom_send__101_0_1 p2 = re.compile(r'hcom_send__\d+') # eg: hcom_receive__101_0_1 diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index 305d23994..1bf2b0b71 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -261,7 +261,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): op_pattern_list = op_rule.split(const.OP_SEP) format_op_pattern = "" for op_pattern in op_pattern_list: - matched_res = re.search(r'\((.*?)\)', op_pattern) + matched_res = re.search(r'\((\w*?)\)', op_pattern) ops_index_range = (matched_res.start() + 1, matched_res.end() - 1) if matched_res else ( 0, len(op_pattern)) -- Gitee From 072f1e6686423244a3d6bcd27d03c59e0a1930ab Mon Sep 17 00:00:00 2001 From: zyy Date: Sat, 31 Aug 2024 11:50:27 +0800 Subject: [PATCH 597/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9other=5Ftime=E8=AE=A1?= =?UTF-8?q?=E7=AE=97=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../compare_backend/compare_bean/profiling_info.py | 9 ++++++--- .../profiling_parser/npu_profiling_parser.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py index 7985501db..891ed2f49 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py +++ b/profiler/compare_tools/compare_backend/compare_bean/profiling_info.py @@ -204,9 +204,12 @@ class ProfilingInfo: return (self.fa_time_bwd_cube + self.fa_time_bwd_vector) / Constant.MILLISECONDS_TO_SECONDS def calculate_other_time(self): - self.other_time = max( - [0, self.compute_time - self.cube_time - self.fa_time_fwd - self.fa_time_bwd - - self.pa_time - self.vec_time - self.conv_time_fwd - self.conv_time_bwd]) + self.other_time = max(0, + (self.compute_time_ms - self.fa_fwd_time - + self.fa_bwd_time - self.conv_fwd_time - + self.conv_bwd_time - self.mm_total_time - + self.vector_total_time - self.sdma_time_tensor_move - + self.other_cube_time - self.page_attention_time) / Constant.MILLISECONDS_TO_SECONDS) def calculate_schedule_time(self): self.scheduling_time = (self.e2e_time - self.compute_time - self.lccl_time - self.communication_not_overlapped) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 8f2714d9c..61e3f1b6a 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -190,9 +190,9 @@ class NPUProfilingParser(BaseProfilingParser): self.__add_sdma_time() self.__add_overlap_analysis_time() self.__add_communication_wait_time() - self._result_data.overall_metrics.calculate_other_time() self._result_data.overall_metrics.calculate_schedule_time() self._result_data.overall_metrics.trans_time_to_s() + self._result_data.overall_metrics.calculate_other_time() self._update_bandwidth() def __add_communication_wait_time(self): -- Gitee From cfc2a501d14a775bc8c5e56fa16864f71e04cc57 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 31 Aug 2024 14:15:16 +0800 Subject: [PATCH 598/791] safety problem bugfix --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 4e03f761d..6380f1aac 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -271,7 +271,7 @@ def load_pt(pt_path, to_cpu=False): else: pt = torch.load(pt_path) except Exception: - logger.error(f"load pt file {pt_path} failed") + logger.error("load pt file {} failed".format(pt_path)) return pt -- Gitee From dd184a721404313a1c421778f787cb540fb4bb8c Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 31 Aug 2024 14:39:27 +0800 Subject: [PATCH 599/791] fix --- .../api_accuracy_checker/files/2_input.npy | Bin 136 -> 0 bytes .../files/index_select_statistics_dump.json | 88 ------------------ .../files/index_select_tensor_dump.json | 88 ------------------ .../test_api_accuracy_checker.py | 39 +------- 4 files changed, 1 insertion(+), 214 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/2_input.npy delete mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_statistics_dump.json delete mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_tensor_dump.json diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/2_input.npy b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/2_input.npy deleted file mode 100644 index 5454521756ce164d6ec6e0e21e28a985a24977cd..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 136 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC%^qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= aXCxM+0{I$7I+{8PwF(pfE(RcA1Y!U##v5G# diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_statistics_dump.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_statistics_dump.json deleted file mode 100644 index 738ed4a79..000000000 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_statistics_dump.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "task": "statistics", - "dump_data_dir": "", - "data": { - "Mint.index_select.0.forward": { - "input_args": [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape": [ - 2, - 3 - ], - "Max": 3.0, - "Min": 1.0, - "data_name": "2_3_input.npy" - }, - { - "type": "int", - "value": 0 - }, - { - "type": "mindspore.Tensor", - "dtype": "Int32", - "shape": [ - 2 - ], - "Max": 1.0, - "Min": 0.0, - "data_name": "2_input.npy" - } - ], - "input_kwargs": {}, - "output": [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape": [ - 2, - 3 - ], - "Max": 3.0, - "Min": 1.0, - "data_name": "2_3_input.npy" - } - ] - }, - "Mint.index_select.0.backward": { - "input": [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape": [ - 2, - 3 - ], - "Max": 3.0, - "Min": 1.0, - "data_name": "2_3_input.npy" - } - ], - "output": [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape": [ - 2, - 3 - ], - "Max": 3.0, - "Min": 1.0, - "data_name": "2_3_input.npy" - }, - null, - { - "type": "mindspore.Tensor", - "dtype": "Int32", - "shape": [ - 2 - ], - "Max": 1.0, - "Min": 0.0, - "data_name": "2_input.npy" - } - ] - } - } -} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_tensor_dump.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_tensor_dump.json deleted file mode 100644 index b635d01c1..000000000 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/files/index_select_tensor_dump.json +++ /dev/null @@ -1,88 +0,0 @@ -{ - "task": "tensor", - "dump_data_dir": "", - "data": { - "Mint.index_select.0.forward": { - "input_args": [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape": [ - 2, - 3 - ], - "Max": 3.0, - "Min": 1.0, - "data_name": "2_3_input.npy" - }, - { - "type": "int", - "value": 0 - }, - { - "type": "mindspore.Tensor", - "dtype": "Int32", - "shape": [ - 2 - ], - "Max": 1.0, - "Min": 0.0, - "data_name": "2_input.npy" - } - ], - "input_kwargs": {}, - "output": [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape": [ - 2, - 3 - ], - "Max": 3.0, - "Min": 1.0, - "data_name": "2_3_input.npy" - } - ] - }, - "Mint.index_select.0.backward": { - "input": [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape": [ - 2, - 3 - ], - "Max": 3.0, - "Min": 1.0, - "data_name": "2_3_input.npy" - } - ], - "output": [ - { - "type": "mindspore.Tensor", - "dtype": "Float32", - "shape": [ - 2, - 3 - ], - "Max": 3.0, - "Min": 1.0, - "data_name": "2_3_input.npy" - }, - null, - { - "type": "mindspore.Tensor", - "dtype": "Int32", - "shape": [ - 2 - ], - "Max": 1.0, - "Min": 0.0, - "data_name": "2_input.npy" - } - ] - } - } -} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py index 2df1d6e6f..15ee6d375 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/api_accuracy_checker/test_api_accuracy_checker.py @@ -31,7 +31,7 @@ def modify_tensor_api_info_json(json_file_path, modified_dump_data_dir): def check_csv(csv_path, row_num): with open(csv_path, 'r', encoding="utf-8") as f: csvreader = csv.reader(f) - assert row_num == sum(1 for row in csvreader) + assert row_num == sum(1 for _ in csvreader) def find_with_prefix(directory, prefix): entries = os.listdir(directory) @@ -78,43 +78,6 @@ class TestApiAccuracyChecker(unittest.TestCase): modify_tensor_api_info_json(api_info_tensor_path, "") delete_files_with_prefix(result_directory, "accuracy_checking") - def test_index_select_api_statistics_mode(self): - api_info_statistics_path = os.path.join(directory, "files", "index_select_statistics_dump.json") - result_directory = os.path.join(directory, "files") - delete_files_with_prefix(result_directory, "accuracy_checking") - api_accuracy_checker = ApiAccuracyChecker() - api_accuracy_checker.parse(api_info_statistics_path) - api_accuracy_checker.run_and_compare() - api_accuracy_checker.to_detail_csv(result_directory) - api_accuracy_checker.to_result_csv(result_directory) - detail_csv = find_with_prefix(result_directory, "accuracy_checking_detail") - assert len(detail_csv) == 1 - check_csv(detail_csv[0], 3) - - result_csv = find_with_prefix(result_directory, "accuracy_checking_result") - assert len(result_csv) == 1 - check_csv(result_csv[0], 2) - delete_files_with_prefix(result_directory, "accuracy_checking") - - def test_index_select_api_tensor_mode(self): - api_info_tensor_path = os.path.join(directory, "files", "index_select_tensor_dump.json") - result_directory = os.path.join(directory, "files") - delete_files_with_prefix(result_directory, "accuracy_checking") - modify_tensor_api_info_json(api_info_tensor_path, result_directory) - api_accuracy_checker = ApiAccuracyChecker() - api_accuracy_checker.parse(api_info_tensor_path) - api_accuracy_checker.run_and_compare() - api_accuracy_checker.to_detail_csv(result_directory) - api_accuracy_checker.to_result_csv(result_directory) - detail_csv = find_with_prefix(result_directory, "accuracy_checking_detail") - assert len(detail_csv) == 1 - check_csv(detail_csv[0], 3) - - result_csv = find_with_prefix(result_directory, "accuracy_checking_result") - assert len(result_csv) == 1 - check_csv(result_csv[0], 2) - modify_tensor_api_info_json(api_info_tensor_path, "") - delete_files_with_prefix(result_directory, "accuracy_checking") if __name__ == '__main__': unittest.main() \ No newline at end of file -- Gitee From bd931f35661398a27808e48b599192376c9ad8b8 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Sat, 31 Aug 2024 14:54:36 +0800 Subject: [PATCH 600/791] safety problem bugfix --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 6380f1aac..aeb3a2454 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -271,7 +271,7 @@ def load_pt(pt_path, to_cpu=False): else: pt = torch.load(pt_path) except Exception: - logger.error("load pt file {} failed".format(pt_path)) + logger.error("load pt file %s failed", pt_path) return pt -- Gitee From 85f3092e5b2b87bbf47d04f39939dc389abfd34f Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Sat, 31 Aug 2024 15:43:43 +0800 Subject: [PATCH 601/791] doc --- .../msprobe/docs/09.accuracy_checker_MindSpore.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md index 4e86f7d89..a14c3a0c1 100644 --- a/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/09.accuracy_checker_MindSpore.md @@ -47,6 +47,8 @@ msprobe -f mindspore run_ut -api_info ./dump.json -o ./checker_result | Status | API 预检通过状态，pass 表示通过测试，error 表示未通过。 | | message | 提示信息。 | +注意点：torch无法对dtype为整数类型的tensor进行反向求导，mindspore则支持。反向过程的预检仅比较dtype为浮点型的输出。 + `accuracy_checking_result_{timestamp}.csv` | 字段 | 含义 | @@ -60,7 +62,7 @@ Forward Test Success 和 Backward Test Success 是否通过测试是由 `accurac 需要注意的是 `accuracy_checking_details_{timestamp}.csv` 中可能存在一个 API 的前向（反向）有多个输出，那么每个输出记录一行，而在 `accuracy_checking_result_{timestamp}.csv` 中的结果需要该 API 的所有结果均为 pass 才能标记为 pass，只要存在一个 error 则标记 error。 ### 4.1 API 预检指标 - + - API 预检指标是通过对`accuracy_checking_details_{timestamp}.csv`中的余弦相似度、最大绝对误差的数值进行判断，得出该API是否符合精度标准的参考指标。详细规则如下： - 余弦相似度大于0.99，并且最大绝对误差小于0.0001，标记“pass”，否则标记为“error”。 -- Gitee From a6a6e9b99eca2463f1e27a2dd550bbacf05b2903 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Sat, 31 Aug 2024 16:03:19 +0800 Subject: [PATCH 602/791] =?UTF-8?q?mstt=20advisor=20=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E9=97=AE=E9=A2=98=201.=E5=A2=9E=E5=8A=A0=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E8=AF=BB=E5=86=99=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/__init__.py | 3 - profiler/advisor/common/constant.py | 11 +++ profiler/advisor/common/graph/graph_parser.py | 5 +- profiler/advisor/common/profiling/ge_info.py | 9 +- .../advisor/dataset/profiling/device_info.py | 6 +- .../dataset/profiling/profiling_parser.py | 8 +- profiler/advisor/result/result.py | 13 +-- profiler/advisor/utils/file.py | 88 +++++++++++++++++++ profiler/advisor/utils/utils.py | 37 +++++--- .../common_func/file_manager.py | 12 ++- 10 files changed, 159 insertions(+), 33 deletions(-) create mode 100644 profiler/advisor/utils/file.py diff --git a/profiler/advisor/__init__.py b/profiler/advisor/__init__.py index e79018ed0..8400fd5ec 100644 --- a/profiler/advisor/__init__.py +++ b/profiler/advisor/__init__.py @@ -12,6 +12,3 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - - -from profiler.advisor.interface.interface import Interface \ No newline at end of file diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index 298e94fc1..63e975c44 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import stat + # timeline DEQUEUE = "Dequeue" DEQUEUE_SEP = "@" @@ -156,3 +159,11 @@ MAX_FILE_SIZE = 10 ** 10 MAX_NUM_PROCESSES = 4 DEFAULT_STEP = "-1" STEP_RANK_SEP = "_" + + +MAX_READ_LINE_BYTES = 8196 * 1024 +MAX_READ_FILE_BYTES = 64 * 1024 * 1024 * 1024 +MAX_READ_DB_FILE_BYTES = 8 * 1024 * 1024 * 1024 + +WRITE_MODES = stat.S_IWUSR | stat.S_IRUSR | stat.S_IRGRP +WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_TRUNC diff --git a/profiler/advisor/common/graph/graph_parser.py b/profiler/advisor/common/graph/graph_parser.py index ef4dc4d68..fe3f6bf29 100644 --- a/profiler/advisor/common/graph/graph_parser.py +++ b/profiler/advisor/common/graph/graph_parser.py @@ -6,6 +6,7 @@ from dataclasses import dataclass from typing import List, Tuple, Dict from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.utils.file import FileOpen logger = logging.getLogger() @@ -220,9 +221,9 @@ class HostGraphParser: def _parse(self, graph_file): # pylint:disable=broad-except graph_list = [] - with open(graph_file, "r", encoding="gbk") as file: + with FileOpen(graph_file, "r") as file: try: - graph_list = self._parse_line(file, graph_list) + graph_list = self._parse_line(file.file_reader, graph_list) except Exception: logger.error( "Parse line %s of file %s failed, make sure the format is correct.", self.line_no, graph_file diff --git a/profiler/advisor/common/profiling/ge_info.py b/profiler/advisor/common/profiling/ge_info.py index 4fd5846d8..5cdebe6da 100644 --- a/profiler/advisor/common/profiling/ge_info.py +++ b/profiler/advisor/common/profiling/ge_info.py @@ -6,9 +6,11 @@ import os from typing import Any, List from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError from profiler.advisor.dataset.profiling.db_manager import ConnectionManager from profiler.advisor.dataset.profiling.profiling_parser import ProfilingParser +from profiler.advisor.utils.utils import check_path_valid logger = logging.getLogger() @@ -33,9 +35,14 @@ class GeInfo(ProfilingParser): ge info """ db_path, db_file = os.path.split(profiling_db_file) + check_path_valid(db_path) if not ConnectionManager.check_db_exists(db_path, [db_file]): return False - conn = ConnectionManager(db_path, db_file) + try: + conn = ConnectionManager(db_path, db_file) + except SQLAlchemyError as e: + logger.error("Database error: %s", e) + return False if conn.check_table_exists(['TaskInfo']): with conn().connect() as sql_conn: self.op_state_info_list = sql_conn.execute(text("select op_name, op_state from TaskInfo")).fetchall() diff --git a/profiler/advisor/dataset/profiling/device_info.py b/profiler/advisor/dataset/profiling/device_info.py index 110cd0794..abb0e6000 100644 --- a/profiler/advisor/dataset/profiling/device_info.py +++ b/profiler/advisor/dataset/profiling/device_info.py @@ -6,6 +6,7 @@ import logging from profiler.advisor.config.config import Config from profiler.advisor.utils.utils import get_file_path_from_directory +from profiler.cluster_analyse.common_func.file_manager import FileManager logger = logging.getLogger() @@ -40,9 +41,8 @@ class DeviceInfoParser: if info_file.endswith("done"): return False # skip info.json.0.done try: - with open(info_file, encoding="utf-8") as file: - info = json.load(file) - except (IOError, ValueError) as error: + info = FileManager.read_json_file(info_file) + except RuntimeError as error: logger.error("Parse json info file %s failed : %s", info_file, error) return False if "DeviceInfo" not in info: diff --git a/profiler/advisor/dataset/profiling/profiling_parser.py b/profiler/advisor/dataset/profiling/profiling_parser.py index 51996617c..a2dae670e 100644 --- a/profiler/advisor/dataset/profiling/profiling_parser.py +++ b/profiler/advisor/dataset/profiling/profiling_parser.py @@ -6,6 +6,7 @@ from typing import List, Dict from profiler.advisor.dataset.profiling.info_collection import logger from profiler.advisor.utils.utils import get_file_path_from_directory, SafeOpen, format_excel_title +from profiler.cluster_analyse.common_func.file_manager import FileManager class ProfilingParser: @@ -19,7 +20,7 @@ class ProfilingParser: def __init__(self, path: str) -> None: self._path = path - self._raw_data: List[List[str]] = [] + self._raw_data: Dict = dict() self._filename = "" @staticmethod @@ -101,9 +102,8 @@ class ProfilingParser: logger.debug("Parse file %s", file) self._filename = os.path.splitext(os.path.basename(file))[0] try: - with open(file, encoding="utf-8") as json_file: - self._raw_data = json.load(json_file) - except (OSError, ValueError) as error: + self._raw_data = FileManager.read_json_file(file) + except RuntimeError as error: logger.error("Parse json file %s failed : %s", file, error) return False return True diff --git a/profiler/advisor/result/result.py b/profiler/advisor/result/result.py index fcc623d2f..0c4c00528 100644 --- a/profiler/advisor/result/result.py +++ b/profiler/advisor/result/result.py @@ -1,7 +1,6 @@ import json import os import stat -from textwrap import fill from collections import OrderedDict import click @@ -11,6 +10,8 @@ from prettytable import ALL, PrettyTable from profiler.advisor.common import constant as const from profiler.advisor.utils.utils import singleton, logger from profiler.advisor.config.config import Config +from profiler.advisor.utils.file import FdOpen, check_dir_writable +from profiler.cluster_analyse.common_func.file_manager import FileManager class ResultWriter: @@ -19,6 +20,7 @@ class ResultWriter: def __init__(self, result_path=None): self.result_path = result_path + check_dir_writable(os.path.dirname(result_path)) self.workbook = xlsxwriter.Workbook(result_path, {"nan_inf_to_errors": True}) self.header_format = None @@ -170,12 +172,11 @@ class OptimizeResult: return tune_op_dict = {"tune_ops_name": self._tune_op_list} tune_ops_file = Config().tune_ops_file + path = os.path.dirname(tune_ops_file) + file_name = os.path.basename(tune_ops_file) try: - - with os.fdopen(os.open(tune_ops_file, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, stat.S_IWUSR | stat.S_IRUSR), - 'w', encoding="utf-8") as op_tune_file: - json.dump(tune_op_dict, op_tune_file) - except OSError as error: + FileManager.create_json_file(path, tune_op_dict, file_name, True) + except RuntimeError as error: logger.error("Dump op_list to %s failed, %s", tune_ops_file, error) return logger.info("Save tune op name list to %s", tune_ops_file) diff --git a/profiler/advisor/utils/file.py b/profiler/advisor/utils/file.py new file mode 100644 index 000000000..767243bb4 --- /dev/null +++ b/profiler/advisor/utils/file.py @@ -0,0 +1,88 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import logging +from profiler.advisor.common import constant as const +from profiler.advisor.utils.utils import check_path_valid +from profiler.advisor.utils.log import get_log_level + +logger = logging.getLogger() +logger.setLevel(get_log_level()) + + +class FileOpen: + """ + open and read file + """ + + def __init__(self: any, file_path: str, mode: str = "r", max_size: int = const.MAX_READ_FILE_BYTES) -> None: + self.file_path = file_path + self.file_reader = None + self.mode = mode + self.max_size = max_size + + def __enter__(self: any) -> any: + check_path_valid(self.file_path, True, max_size=self.max_size) + self.file_reader = open(self.file_path, self.mode) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.file_reader: + self.file_reader.close() + + +class FdOpen: + """ + creat and write file + """ + + def __init__(self: any, file_path: str, flags: int = const.WRITE_FLAGS, mode: int = const.WRITE_MODES, + operate: str = "w", newline: str = None) -> None: + self.file_path = file_path + self.flags = flags + self.newline = newline + self.mode = mode + self.operate = operate + self.fd = None + self.file_open = None + + def __enter__(self: any) -> any: + file_dir = os.path.dirname(self.file_path) + check_dir_writable(file_dir) + + + self.fd = os.open(self.file_path, self.flags, self.mode) + if self.newline is None: + self.file_open = os.fdopen(self.fd, self.operate) + else: + self.file_open = os.fdopen(self.fd, self.operate, newline=self.newline) + return self.file_open + + def __exit__(self, exc_type, exc_val, exc_tb): + if self.file_open: + self.file_open.close() + elif self.fd: + os.close(self.fd) + + +def check_dir_writable(path: str, is_file: bool = False) -> None: + """ + check path is dir and writable + """ + check_path_valid(path, is_file) + if not os.access(path, os.W_OK): + raise PermissionError(f"The path \"{path}\" does not have permission to write. " + f"Please check that the path is writeable.") \ No newline at end of file diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index 3cf9a35a9..d4c8c5abb 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -579,16 +579,33 @@ def get_file_path_by_walk(root, filename): return file_path -def check_path_valid(path): - if os.path.islink(os.path.abspath(path)): - logger.error("fThe path is detected as a soft connection. path:%ss", path) - return False - elif not os.access(path, os.R_OK): - logger.error(f"The file is not readable. path:%ss", path) - return False - elif os.path.getsize(path) > const.MAX_FILE_SIZE: - logger.error(f"The file size exceeds the limit. path:%ss, MAX_FILE_SIZE:%ss B", path, const.MAX_FILE_SIZE) - return False +def check_path_valid(path: str, is_file: bool = True, max_size: int = const.MAX_READ_FILE_BYTES) -> bool: + """ + check the path is valid or not + :param path: file path + :param is_file: file or not + :param max_size: file's max size + :return: bool + """ + if path == "": + raise FileNotFoundError("The path is empty. Please enter a valid path.") + if not os.path.exists(path): + raise FileNotFoundError(f"The path \"{path}\" does not exist. Please check that the path exists.") + if is_file: + if not os.path.isfile(path): + raise FileNotFoundError(f"The path \"{path}\" is not a file. Please check the path.") + if os.path.islink(path): + raise FileNotFoundError(f"The path \"{path}\" is link. Please check the path.") + if os.path.getsize(path) > max_size: + raise OSError(f"The path \"{path}\" is too large to read. Please check the path.") + else: + if not os.path.isdir(path): + raise FileNotFoundError(f"The path \"{path}\" is not a directory. Please check the path.") + if os.path.islink(path): + raise FileNotFoundError(f"The path \"{path}\" is link. Please check the path.") + if not os.access(path, os.R_OK): + raise PermissionError(f"The path \"{path}\" does not have permission to read. " + f"Please check that the path is readable.") return True diff --git a/profiler/cluster_analyse/common_func/file_manager.py b/profiler/cluster_analyse/common_func/file_manager.py index 380192f87..26e6607fa 100644 --- a/profiler/cluster_analyse/common_func/file_manager.py +++ b/profiler/cluster_analyse/common_func/file_manager.py @@ -101,13 +101,17 @@ class FileManager: raise RuntimeError(f"Can't create file: {base_name}") from e @classmethod - def create_json_file(cls, profiler_path: str, data: dict, file_name: str) -> None: + def create_json_file(cls, profiler_path: str, data: dict, file_name: str, common_flag: bool = False) -> None: if not data: return - output_path = os.path.join(profiler_path, Constant.CLUSTER_ANALYSIS_OUTPUT) - output_file = os.path.join(output_path, file_name) + if not common_flag: + output_path = os.path.join(profiler_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + output_file = os.path.join(output_path, file_name) + PathManager.check_path_writeable(output_path) + else: + output_file = os.path.join(profiler_path, file_name) + PathManager.check_path_writeable(profiler_path) base_name = os.path.basename(output_file) - PathManager.check_path_writeable(output_path) try: with os.fdopen( os.open(output_file, os.O_WRONLY | os.O_CREAT, cls.DATA_FILE_AUTHORITY), 'w' -- Gitee From 1f91fcd12f925d9c5a0ec38d91be6507e318489b Mon Sep 17 00:00:00 2001 From: CSNIU Date: Fri, 30 Aug 2024 13:11:39 +0800 Subject: [PATCH 603/791] =?UTF-8?q?=E6=94=AF=E6=8C=81=E5=9C=BA=E6=99=AF2?= =?UTF-8?q?=E5=9C=BA=E6=99=AF3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/compare/ms_compare.py | 121 ++++++++++++++++-- 1 file changed, 109 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 7c3bd6815..ae7998506 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,8 +1,9 @@ import os.path +import copy from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, \ task_dumppath_get, load_yaml, load_npy from msprobe.core.common.file_check import create_directory -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator @@ -16,7 +17,7 @@ class MSComparator(Comparator): self.api_mapping = api_mapping self.cross_frame = cell_mapping is not None or api_mapping is not None self.cell_mapping_dict = self.load_mapping_file(self.cell_mapping) - self.api_mapping_dict = {} + self.api_mapping_dict = self.load_mapping_file(self.api_mapping) if api_mapping is not None: self.ms_to_pt_mapping = self.load_internal_api() @@ -44,15 +45,18 @@ class MSComparator(Comparator): return npu_op_name def check_op(self, npu_dict, bench_dict, fuzzy_match): - npu_op_name = npu_dict["op_name"].copy() - bench_op_name = bench_dict["op_name"].copy() - - if self.api_mapping is not None: - npu_op_name = self.process_api_mapping(npu_op_name, bench_op_name) + npu_dict_new, bench_dict_new = copy.deepcopy(npu_dict), copy.deepcopy(bench_dict) + npu_op_name, bench_op_name = npu_dict_new["op_name"], bench_dict_new["op_name"] if self.cell_mapping is not None: npu_op_name = self.process_cell_mapping(npu_op_name) - - struct_match = check_struct_match(npu_dict, bench_dict, cross_frame=self.cross_frame) + if self.api_mapping is not None: + npu_op_name = self.process_internal_api_mapping(npu_op_name, bench_op_name) + if isinstance(self.api_mapping, str): + npu_dict_new, bench_dict_new, target_dict = self.user_mapping_api_trans(npu_dict_new, bench_dict_new) + if target_dict: + bench_dict = self.reconstitution_bench_dict(npu_dict, copy.deepcopy(bench_dict_new), target_dict) + npu_op_name, bench_op_name = npu_dict_new["op_name"], bench_dict_new["op_name"] + struct_match = check_struct_match(npu_dict_new, bench_dict_new, cross_frame=self.cross_frame) if not fuzzy_match: return npu_op_name == bench_op_name and struct_match is_match = True @@ -81,9 +85,10 @@ class MSComparator(Comparator): npu_op_name[idx] = npu_op_name[idx].replace(target, para) return npu_op_name - def process_api_mapping(self, npu_op_name, bench_op_name): + def process_internal_api_mapping(self, npu_op_name, bench_op_name): # get api name & class name from op_name # Functional.addcmul.0.forward.input.0 + npu_op_name, bench_op_name = npu_op_name.copy(), bench_op_name.copy() ms_api_name = npu_op_name[0].rsplit(Const.SEP, 4)[0] pt_api_name = bench_op_name[0].rsplit(Const.SEP, 4)[0] class_name = ms_api_name.split(Const.SEP)[0] @@ -94,9 +99,101 @@ class MSComparator(Comparator): elif self.ms_to_pt_mapping.get(ms_api_name) == pt_api_name: return self.api_replace(npu_op_name, ms_api_name, pt_api_name) else: - return npu_op_name - + return npu_op_name + + def remove_element(self, op_name, struct, summary, idx): + del op_name[idx] + del struct[idx] + del summary[idx] + + def user_mapping_api_trans(self, new_npu_dict, new_bench_dict): + npu_op_name, bench_op_name = new_npu_dict["op_name"], new_bench_dict["op_name"] + npu_struct_in, bench_struct_in = new_npu_dict.get("input_struct"), new_bench_dict.get("input_struct") + npu_struct_out, bench_struct_out = new_npu_dict.get("output_struct"), new_bench_dict.get("output_struct") + npu_summary, bench_summary = new_npu_dict.get("summary"), new_bench_dict.get("summary") + npu_in_len, bench_in_len, npu_out_len, bench_out_len = len(npu_struct_in), len(bench_struct_in), len(npu_struct_out), len(bench_struct_out) + ms_api_list, pt_api_list = npu_op_name[0].split(Const.SEP), bench_op_name[0].split(Const.SEP) + ms_api_name = ms_api_list[0] + Const.SEP + ms_api_list[1] + pt_api_name = pt_api_list[0] + Const.SEP + pt_api_list[1] + target_dict = {} + for api_dict in self.api_mapping_dict: + if api_dict.get("pt_api") == pt_api_name and api_dict.get("ms_api") == ms_api_name: + ms_user_args_len, pt_user_args_len = len(api_dict.get("ms_args")), len(api_dict.get("pt_args")) + ms_user_output_len, pt_user_output_len = len(api_dict.get("ms_output")), len(api_dict.get("pt_output")) + if ms_user_args_len != pt_user_args_len or ms_user_output_len != pt_user_output_len: + break + ms_out_list = api_dict.get("ms_output") + for idx in reversed(range(npu_out_len)): + if idx not in ms_out_list: + del npu_struct_out[idx] + del npu_summary[idx + npu_in_len] + del npu_op_name[idx + npu_in_len] + pt_out_list = api_dict.get("pt_output") + for idx in reversed(range(bench_out_len)): + if idx not in pt_out_list: + del bench_struct_out[idx] + del bench_summary[idx + bench_in_len] + del bench_op_name[idx + bench_in_len] + ms_para_list = api_dict.get("ms_args") + for idx in reversed(range(npu_in_len)): + if idx not in ms_para_list: + self.remove_element(npu_op_name, npu_struct_in, npu_summary, idx) + pt_para_list = api_dict.get("pt_args") + for idx in reversed(range(bench_in_len)): + if idx not in pt_para_list: + self.remove_element(bench_op_name, bench_struct_in, bench_summary, idx) + npu_op_name = self.api_replace(npu_op_name, ms_api_name, pt_api_name) + npu_op_name = self.para_sequence_update(npu_op_name, bench_op_name) + target_dict = api_dict + break + if target_dict: + new_npu_dict.update({"op_name": npu_op_name}) + new_bench_dict.update({"op_name": bench_op_name}) + new_npu_dict.update({"input_struct": npu_struct_in}) + new_bench_dict.update({"input_struct": bench_struct_in}) + new_npu_dict.update({"output_struct": npu_struct_out}) + new_bench_dict.update({"output_struct": bench_struct_out}) + new_npu_dict.update({"summary": npu_summary}) + new_bench_dict.update({"summary": bench_summary}) + return new_npu_dict, new_bench_dict, target_dict + + def para_sequence_update(self, npu_op_name, bench_op_name): + for idx, _ in enumerate(npu_op_name): + npu_op_name[idx] = npu_op_name[idx][:-1] + bench_op_name[idx].rsplit(Const.SEP, 1)[-1][0] + return npu_op_name + def reconstitution_bench_dict(self, npu_dict, del_bench_dict, api_dict): + ms_user_args_list = api_dict.get("ms_args") + ms_user_output_list = api_dict.get("ms_output") + npu_struct_in = npu_dict.get("input_struct") + npu_struct_out = npu_dict.get("output_struct") + npu_in_len = len(npu_struct_in) + npu_out_len = len(npu_struct_out) + if npu_in_len == len(ms_user_args_list) and npu_out_len == len(ms_user_output_list): + return del_bench_dict + ms_input_args_list = [i for i in range(npu_in_len)] + input_sub_list =list(set(ms_input_args_list) - set(ms_user_args_list)) + ms_output_args_list = [i for i in range(npu_out_len)] + output_sub_list =list(set(ms_output_args_list) - set(ms_user_output_list)) + bench_op_name = del_bench_dict["op_name"] + bench_struct_in = del_bench_dict.get("input_struct") + bench_struct_out = del_bench_dict.get("output_struct") + bench_summary = del_bench_dict.get("summary") + for idx in input_sub_list: # Fill in the blank value field in the pt dictionary + bench_op_name.insert(idx, CompareConst.NAN) + bench_struct_in.insert(idx, CompareConst.NAN) + bench_summary.insert(idx, CompareConst.NAN) + for idx in output_sub_list:# Fill in the blank value field in the pt dictionary + bench_op_name.insert(npu_in_len + idx, CompareConst.NAN) + bench_struct_out.insert(idx, CompareConst.NAN) + bench_summary.insert(npu_in_len + idx, CompareConst.NAN) + del_bench_dict.update({"op_name": bench_op_name}) + del_bench_dict.update({"input_struct": bench_struct_in}) + del_bench_dict.update({"output_struct": bench_struct_out}) + del_bench_dict.update({"summary": bench_summary}) + return del_bench_dict + + def ms_compare(input_param, output_path, **kwargs): try: stack_mode = kwargs.get('stack_mode', False) -- Gitee From f41f71cdc6e886222016faf6c4c80a39dcaa589d Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Sat, 31 Aug 2024 16:12:09 +0800 Subject: [PATCH 604/791] bug_fix_for_undisclosed_interface --- profiler/affinity_cpu_bind/README.md | 1 + profiler/cluster_analyse/README.md | 9 +++++---- profiler/cluster_analyse/cluster_analysis.py | 4 ++-- profiler/compare_tools/README.md | 8 ++++---- profiler/compare_tools/performance_compare.py | 2 +- 5 files changed, 13 insertions(+), 11 deletions(-) diff --git a/profiler/affinity_cpu_bind/README.md b/profiler/affinity_cpu_bind/README.md index 82ecdec63..47ffa7be6 100644 --- a/profiler/affinity_cpu_bind/README.md +++ b/profiler/affinity_cpu_bind/README.md @@ -28,6 +28,7 @@ python3 bind_core.py -app="inferenec/train cmd" ``` -app或--application：配置训练或推理程序的执行命令，执行命令前后加引号。 +-t或--time：设置绑核之前的等待时间，单位为秒。 该方式会自动拉起训练或推理任务，检测任务进程，并实施绑核。 diff --git a/profiler/cluster_analyse/README.md b/profiler/cluster_analyse/README.md index 785056252..232654315 100644 --- a/profiler/cluster_analyse/README.md +++ b/profiler/cluster_analyse/README.md @@ -50,10 +50,11 @@ experimental_config = torch_npu.profiler._ExperimentalConfig( 参数说明： - | 参数名 | 说明 | 是否必选 | - | --------------------- | ------------------------------------------------------------ | -------- | - | --collection_path或-d | 性能数据汇集目录，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 是 | - | --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | +| 参数名 | 说明 | 是否必选 | +|---------------------|-------------------------------------------------------------------------------|------| +| --profiling_path或-d | 性能数据汇集目录。 | 是 | +| --output_path或-o | 集群数据分析结果存放的路径，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，默认为profiling_path。 | 否 | +| --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | --mode参数说明： diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 11c52ae5b..1400528f0 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -89,13 +89,13 @@ class Interface: def cluster_analysis_main(args=None): parser = argparse.ArgumentParser(description="cluster analysis module") - parser.add_argument('-d', '--collection_path', type=str, required=True, help="profiling data path") + parser.add_argument('-d', '--profiling_path', type=str, required=True, help="profiling data path") parser.add_argument('-m', '--mode', choices=COMM_FEATURE_LIST, default='all', help="different analysis mode") parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') args_parsed, _ = parser.parse_known_args(args=args) parameter = { - Constant.COLLECTION_PATH: args_parsed.collection_path, + Constant.COLLECTION_PATH: args_parsed.profiling_path, Constant.ANALYSIS_MODE: args_parsed.mode, Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: args_parsed.output_path } diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index 535bff4be..338e91c9d 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -118,9 +118,9 @@ MindSpore性能调试工具采集结果数据目录结构如下： msprof-analyze compare -d [比对性能数据文件所在路径] -bp [基准性能数据文件所在路径] --output_path=[比对结果文件存放路径] ``` - - -d（必选）：比对性能数据文件所在路径。可以指定以“ascend_pt”结尾的目录、ASCEND_PROFILER_OUTPUT目录或trace_view.json文件，指定trace_view.json无法显示算子的内存占用。 - - -bp（必选）：基准性能数据文件所在路径。基准性能数据文件若以GPU为基准，指定到以".pt.trace"结尾的json文件；若以NPU不同版本为基准，指定文件与-d一致。 - - --output_path（可选）：性能比对结果存放的路径，默认保存在当前目录。 + - --profiling_path或-d（必选）：比对性能数据文件所在路径。可以指定以“ascend_pt”结尾的目录、ASCEND_PROFILER_OUTPUT目录或trace_view.json文件，指定trace_view.json无法显示算子的内存占用。 + - --benchmark_path或-bp（必选）：基准性能数据文件所在路径。基准性能数据文件若以GPU为基准，指定到以".pt.trace"结尾的json文件；若以NPU不同版本为基准，指定文件与-d一致。 + - --output_path或-o（可选）：性能比对结果存放的路径，默认保存在当前目录。 #### 脚本方式 @@ -135,7 +135,7 @@ python performance_compare.py [基准性能数据文件所在路径] [比对性 - 基准性能数据文件所在路径（必选）：若以GPU为基准，指定到以".pt.trace"结尾的json文件；若以NPU不同版本为基准，指定文件参考**比对性能数据文件所在路径**。 - 比对性能数据文件所在路径（必选）：可以指定以“ascend_pt”结尾的目录、ASCEND_PROFILER_OUTPUT目录或trace_view.json文件，指定trace_view.json无法显示算子的内存占用。 -- --output_path（可选）：性能比对结果存放的路径，默认保存在当前目录。 +- --output_path或-o（可选）：性能比对结果存放的路径，默认保存在当前目录。 #### 通用参数说明 diff --git a/profiler/compare_tools/performance_compare.py b/profiler/compare_tools/performance_compare.py index dff87db2f..2fb700fb2 100644 --- a/profiler/compare_tools/performance_compare.py +++ b/profiler/compare_tools/performance_compare.py @@ -21,7 +21,7 @@ def main(): parser.add_argument("--enable_api_compare", default=False, action='store_true', help="开启host api性能比较") parser.add_argument("--enable_kernel_compare", default=False, action='store_true', help="开启kernel性能比较") parser.add_argument("--disable_details", default=False, action='store_true', help="不展示比对明细") - parser.add_argument("--output_path", type=str, default='', help="性能数据比对结果的存放路径") + parser.add_argument('-o', "--output_path", type=str, default='', help="性能数据比对结果的存放路径") parser.add_argument("--max_kernel_num", type=int, help="每个torch op的kernel数量限制") parser.add_argument("--op_name_map", type=ast.literal_eval, default={}, help="配置GPU与NPU等价的算子名称映射关系，以字典的形式传入") -- Gitee From eb650e1675b8a22b03ab0dd0f86fadcde8d4d141 Mon Sep 17 00:00:00 2001 From: zyy Date: Sat, 31 Aug 2024 17:21:17 +0800 Subject: [PATCH 605/791] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E7=94=9F=E6=88=90?= =?UTF-8?q?=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index d92c1f041..0aba2d711 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -95,8 +95,6 @@ def cluster_analysis_main(args=None): default='all', help="different analysis mode") parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') args_parsed, unknown = parser.parse_known_args(args=args) - if unknown: - print(f"Unknown arguments provided: {unknown}") parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, -- Gitee From 6f280233b903c8e853b5306ea87532199f2eac5e Mon Sep 17 00:00:00 2001 From: zyy Date: Sat, 31 Aug 2024 17:22:38 +0800 Subject: [PATCH 606/791] =?UTF-8?q?=E8=87=AA=E5=8A=A8=E7=94=9F=E6=88=90?= =?UTF-8?q?=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 0aba2d711..d5b376579 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -94,8 +94,7 @@ def cluster_analysis_main(args=None): parser.add_argument('-m', '--mode', choices=COMM_FEATURE_LIST, default='all', help="different analysis mode") parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') - args_parsed, unknown = parser.parse_known_args(args=args) - + args_parsed, _ = parser.parse_known_args(args=args) parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, Constant.ANALYSIS_MODE: args_parsed.mode, -- Gitee From a222d83cd5ed30b23aaaa1465b76e678b54c718c Mon Sep 17 00:00:00 2001 From: zyy Date: Sat, 31 Aug 2024 17:27:40 +0800 Subject: [PATCH 607/791] =?UTF-8?q?=E8=BE=93=E5=85=A5=E4=B8=8D=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E5=8F=82=E6=95=B0=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 11c52ae5b..c964d8c06 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -93,7 +93,9 @@ def cluster_analysis_main(args=None): parser.add_argument('-m', '--mode', choices=COMM_FEATURE_LIST, default='all', help="different analysis mode") parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') - args_parsed, _ = parser.parse_known_args(args=args) + args_parsed, unknown = parser.parse_known_args(args=args) + if unknown: + print(f"Unknown arguments provided: {unknown}") parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, Constant.ANALYSIS_MODE: args_parsed.mode, -- Gitee From ed1823d0ea2b761390a5437adb3e62917ecd02f2 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Sat, 31 Aug 2024 17:39:43 +0800 Subject: [PATCH 608/791] =?UTF-8?q?mstt=20advisor=20=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E9=97=AE=E9=A2=98=201.=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E5=87=BD=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/utils/utils.py | 163 -------------------------------- 1 file changed, 163 deletions(-) diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index d4c8c5abb..f41bba53d 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -31,18 +31,6 @@ def ignore_warning(exception: Exception = None): return exception -class ContextObject(object): - def __init__(self): - self._debug = False - - def set_debug(self, debug=False): - self._debug = debug - - @property - def debug_mode(self): - return self._debug - - def debug_option(f): return click.option('--debug', is_flag=True, @@ -271,19 +259,6 @@ class ParallelJob: completed_queue.put(token) -def mp_queue_to_list(job_queue): - queue_list = [] - while True: - try: - if job_queue.empty(): - break - token = job_queue.get(timeout=1) - queue_list.append(token) - except queue.Empty: - continue - return queue_list - - def load_parameter(parameter, default): if not os.environ.get(parameter, None): return default @@ -291,21 +266,6 @@ def load_parameter(parameter, default): return os.environ.get(parameter) -def get_supported_subclass(clazz: VersionControl.__class__, cann_version: str): - """ - Returns a list of subclasses that support the specified version, because of the __subclasses__(), - you need to import the all subclass first - :param clazz: Class name which is extends to VersionControl.__class__ - :param cann_version: The CANN software version - :return: The list of subclasses that support the specified CANN version - """ - # 获取所有支持这个cann版本的子类 - dataset_classes = clazz.__subclasses__() - sub_class_list = [cls for cls in dataset_classes if cls.is_supported(cann_version)] - logger.debug("The support subclass list is %s, cann version is %s", str(sub_class_list), cann_version) - return sub_class_list - - def to_percent(num: float) -> str: """ change float to percent format @@ -328,15 +288,6 @@ def safe_write(content, save_path): f.write(content) -def create_directory_for_file(file: str) -> None: - """ - create directory for file - """ - dirname = os.path.dirname(file) - if not os.path.exists(dirname): - os.makedirs(dirname) - - class CheckPathAccess: """ check path access permissions @@ -381,20 +332,6 @@ def get_file_path_from_directory(path, check_func): return file_list -@CheckPathAccess -def get_dir_path_from_directory(path: str, check_func: Any) -> list: - """ - get file from directory - """ - file_list = [] - for root, _, files in os.walk(path, onerror=walk_error_handler): - for filename in files: - filepath = os.path.join(root, filename) - if check_func(filename): - file_list.append(filepath) - return file_list - - def is_regex_pattern(string: str): """ Check if str is a regular expression. @@ -443,13 +380,6 @@ def format_excel_title(title: str) -> str: return kernel_details_col_name_map.get(title, title) -def format_float(num: float) -> float: - """ - format float num, round to 2 decimal places - """ - return round(num, 2) - - class SafeOpen: """ safe open to check file @@ -476,99 +406,6 @@ class SafeOpen: return True -def save_downloaded_file(response, url_path, file_save_path): - """保存响应体中的文件 - - 参数: - response: 请求后获取的响应体 - url_path: url路径 - file_save_path: 保存路径 - 返回: - final_file_path: 文件保存绝对路径 - """ - # 获取url路径中的文件名, 拼接在保存路径下 - file_save_path = os.path.normpath(file_save_path) - file_name = os.path.basename(url_path) - final_file_path = os.path.join(file_save_path, file_name) - # 若目标保存路径不存在，则自动生成 - if not os.path.exists(file_save_path): - os.makedirs(file_save_path) - if response.status_code <= 300: - logger.debug("Response status code is %s", response.status_code) - flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL - modes = stat.S_IWUSR | stat.S_IRUSR - # 若文件已存在，则移除已有的文件并保存最新的文件 - if os.path.exists(final_file_path): - os.remove(final_file_path) - # 保存文件 - with os.fdopen(os.open(final_file_path, flags, modes), mode="wb") as f: - f.write(response.content) - logger.info("Success to save content in: %s", os.path.abspath(final_file_path)) - else: - # 若响应码不为预期的数值, 显示相应告警 - logger.warning("Failed to save the response body. The response status code is %s. " - "Please check the network or try another region", response.status_code) - - -def request_with_retry(url_path, region_name=None): - """使用requests请求获取文件, 失败则进行重试, 最多请求 max_retries+1 次 - - 参数: - url_path: URL路径 - file_save_path: 云文件保存路径 - """ - logger.debug("Requesting or retrying to get file from region: %s", region_name) - - # 若从环境变量指定了保存路径，优先从环境变量中获取，若为空则使用默认的云文件保存路径constant.CLOUD_RULE_PATH - file_save_path = os.path.join(os.path.expanduser("~"), const.CLOUD_RULE_PATH) - if os.getenv(const.ADVISOR_RULE_PATH): - file_save_path = os.getenv(const.ADVISOR_RULE_PATH) - - session = requests.Session() - # 使用session发起的所有请求, 默认最多会重试 max_retries 次, 计入最初请求, 最差情况下请求 max_retries+1 次 - adapter = HTTPAdapter(max_retries=const.MAX_RETRIES) - session.mount(const.HTTP_PREFIXES, adapter) - session.mount(const.HTTPS_PREFIXES, adapter) - - logger.debug('Session try to get response') - response = None - try: - response = session.get(url_path, timeout=const.TIMEOUT) - except Exception as e: - logger.debug("Error: %s: %s", e, traceback.format_exc()) - - if response is None: - logger.warning("Fail to download file from region: %s, response is None, " - "please use the environment variable %s for more detailed information", - region_name, const.ADVISOR_LOG_LEVEL) - else: - try: - # 若响应码为400~600之间，response.raise_for_status抛出HTTPError错误, 跳过调用save_downloaded_file函数逻辑 - response.raise_for_status() - save_downloaded_file(response, url_path=url_path, file_save_path=file_save_path) - except Exception as e: - logger.warning("Error: %s: %s", e, traceback.format_exc()) - # 关闭 session, 清除所有装配器 - session.close() - - -def read_csv(file): - import csv - - raw_data = [] - logger.debug("Parse file %s", file) - with SafeOpen(file, encoding="utf-8") as csv_file: - try: - csv_content = csv.reader(csv_file) - for row in csv_content: - raw_data.append(row) - except OSError as error: - logger.error("Read csv file failed : %s", error) - return [] - - return raw_data - - def get_file_path_by_walk(root, filename): file_path = "" for root, _, files in os.walk(root, topdown=True): -- Gitee From 0714e28fb15654a3d98e8f1078e53cd69365a600 Mon Sep 17 00:00:00 2001 From: zyy Date: Sat, 31 Aug 2024 17:56:00 +0800 Subject: [PATCH 609/791] =?UTF-8?q?=E8=BE=93=E5=85=A5=E4=B8=8D=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E5=8F=82=E6=95=B0=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index c964d8c06..848df2550 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -95,7 +95,7 @@ def cluster_analysis_main(args=None): parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') args_parsed, unknown = parser.parse_known_args(args=args) if unknown: - print(f"Unknown arguments provided: {unknown}") + print(f"Unknown arguments are provided: {unknown}") parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, Constant.ANALYSIS_MODE: args_parsed.mode, -- Gitee From 9505c7039e5798637b7eaa15c767ec93ea15349c Mon Sep 17 00:00:00 2001 From: RICHARDNAN <1466684392@qq.com> Date: Sat, 31 Aug 2024 18:00:54 +0800 Subject: [PATCH 610/791] =?UTF-8?q?advisor=E6=A8=A1=E5=9D=97=E9=83=A8?= =?UTF-8?q?=E5=88=86=E6=96=87=E4=BB=B6=E6=93=8D=E4=BD=9C=E5=89=8D=E6=9C=AA?= =?UTF-8?q?=E5=AF=B9=E6=96=87=E4=BB=B6=E5=92=8C=E8=B7=AF=E5=BE=84=E8=BF=9B?= =?UTF-8?q?=E8=A1=8C=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cluster_advice/kernel_cluster_advice.py | 3 ++- profiler/advisor/dataset/profiling/profiling_parser.py | 7 ++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py b/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py index 6fa83c765..e7b334bbf 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py +++ b/profiler/advisor/advisor_backend/cluster_advice/kernel_cluster_advice.py @@ -5,7 +5,7 @@ from common_func.constant import Constant from common_func_advisor.constant import Constant as AdvisorConstant from cluster_advice.cluster_advice_base import ClusterAdviceBase from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor - +from profiler.cluster_analyse.common_func.file_manager import FileManager class KernelClusterAdvice(ClusterAdviceBase): COLUMNS_TO_GROUP = ["Name", "Input Shapes", "Input Data Types", "Output Shapes"] @@ -32,6 +32,7 @@ class KernelClusterAdvice(ClusterAdviceBase): kernel_file = os.path.join(profiling_dir_path, Constant.SINGLE_OUTPUT, Constant.KERNEL_DETAILS_CSV) if kernel_file: # 判断csv文件大小 + FileManager.check_file_size(kernel_file) PathManager.check_path_readable(kernel_file) # 读取CSV文件 df_temp = pd.read_csv(kernel_file) diff --git a/profiler/advisor/dataset/profiling/profiling_parser.py b/profiler/advisor/dataset/profiling/profiling_parser.py index 51996617c..35d326fb8 100644 --- a/profiler/advisor/dataset/profiling/profiling_parser.py +++ b/profiler/advisor/dataset/profiling/profiling_parser.py @@ -6,7 +6,7 @@ from typing import List, Dict from profiler.advisor.dataset.profiling.info_collection import logger from profiler.advisor.utils.utils import get_file_path_from_directory, SafeOpen, format_excel_title - +from profiler.cluster_analyse.common_func.file_manager import FileManager class ProfilingParser: """ @@ -77,6 +77,11 @@ class ProfilingParser: def _parse_csv(self, file, check_csv=True) -> bool: logger.debug("Parse file %s", file) + try: + FileManager.check_file_size(file) + except RuntimeError as e: + logger.error("File size check failed: %s", e) + return False self._filename = os.path.splitext(os.path.basename(file))[0] with SafeOpen(file, encoding="utf-8") as csv_file: try: -- Gitee From 213d16b5d9858f3776b5837ca72c48984013c091 Mon Sep 17 00:00:00 2001 From: hhqx Date: Sat, 31 Aug 2024 15:46:57 +0800 Subject: [PATCH 611/791] replace os.path.realpath to os.path.abspath --- profiler/advisor/advisor_backend/advice_base.py | 2 +- .../advisor/advisor_backend/advice_factory/advice_factory.py | 2 +- profiler/advisor/advisor_backend/interface.py | 4 ++-- profiler/advisor/interface/interface.py | 2 +- .../cluster_kernels_analysis/cluster_prof_Info_analysis.py | 2 +- .../compare_backend/generator/detail_performance_generator.py | 2 +- 6 files changed, 7 insertions(+), 7 deletions(-) diff --git a/profiler/advisor/advisor_backend/advice_base.py b/profiler/advisor/advisor_backend/advice_base.py index 35939bcea..97fd9ff4c 100644 --- a/profiler/advisor/advisor_backend/advice_base.py +++ b/profiler/advisor/advisor_backend/advice_base.py @@ -23,7 +23,7 @@ class AdviceBase: ADVICE = "advice" def __init__(self, collection_path: str): - self.collection_path = os.path.realpath(collection_path) + self.collection_path = os.path.abspath(collection_path) self.bottelneck = '' self.output_format_data = { self.DATA: [], diff --git a/profiler/advisor/advisor_backend/advice_factory/advice_factory.py b/profiler/advisor/advisor_backend/advice_factory/advice_factory.py index 639f4800c..1b4b0c1be 100644 --- a/profiler/advisor/advisor_backend/advice_factory/advice_factory.py +++ b/profiler/advisor/advisor_backend/advice_factory/advice_factory.py @@ -19,7 +19,7 @@ from common_func.path_manager import PathManager class AdviceFactory: def __init__(self, collection_path: str): - self.collection_path = os.path.realpath(collection_path) + self.collection_path = os.path.abspath(collection_path) @staticmethod def run_advice(self, advice: str, kwargs: dict): diff --git a/profiler/advisor/advisor_backend/interface.py b/profiler/advisor/advisor_backend/interface.py index 3e20c26d4..deb68822e 100644 --- a/profiler/advisor/advisor_backend/interface.py +++ b/profiler/advisor/advisor_backend/interface.py @@ -30,7 +30,7 @@ from advisor_backend.advice_factory.overall_advice_factory import OverallAdviceF class Interface: def __init__(self, collection_path: str): - self.collection_path = os.path.realpath(collection_path) + self.collection_path = os.path.abspath(collection_path) self._factory_controller = FactoryController(collection_path) def get_data(self: any, mode: str, advice: str, **kwargs): @@ -50,7 +50,7 @@ class FactoryController: } def __init__(self, collection_path: str): - self.collection_path = os.path.realpath(collection_path) + self.collection_path = os.path.abspath(collection_path) self.temp_input_path = None def create_advice_factory(self, mode: str, input_path: str): diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index b8c8f15ad..8138f7a1a 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -85,7 +85,7 @@ class Interface: all_dimension = list(supported_analyzer.keys()) def __init__(self, **kwargs): - self.collection_path = os.path.realpath(kwargs.get("profiling_path")) + self.collection_path = os.path.abspath(kwargs.get("profiling_path")) @staticmethod def get_scope(dimension): diff --git a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py index 27e3c229c..126dddb98 100644 --- a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py +++ b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py @@ -287,7 +287,7 @@ class DeliverableGenerator: if os.path.islink(params.get('dir')): print(f"The file: \"{params.get('dir')}\" is link. Please check the path.") return - prof_path = os.path.realpath(params.get('dir')) + prof_path = os.path.abspath(params.get('dir')) PathManager.input_path_common_check(prof_path) if params.get('type') == "all": self.analyzers = [TimeToCsvAnalyzer(chip_type, prof_path), StatisticalInfoToHtmlAnalyzer(chip_type, params.get("top_n"), prof_path)] diff --git a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py index 5bacc24ed..b8df1ae34 100644 --- a/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py +++ b/profiler/compare_tools/compare_backend/generator/detail_performance_generator.py @@ -51,7 +51,7 @@ class DetailPerformanceGenerator(BaseGenerator): return dir_path = self._args.output_path if self._args.output_path else "./" file_name = "performance_comparison_result_{}.xlsx".format(datetime.utcnow().strftime("%Y%m%d%H%M%S")) - result_file_path = os.path.realpath(os.path.join(dir_path, file_name)) + result_file_path = os.path.abspath(os.path.join(dir_path, file_name)) ExcelView(self._result_data, result_file_path, self._args).generate_view() print(f"[INFO] The comparison result file has been generated: {result_file_path}") -- Gitee From 57435f1a3dd574b13817de1d347e1e2bb5a777a6 Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Sat, 31 Aug 2024 18:19:14 +0800 Subject: [PATCH 612/791] adapt cross_entropy_process input_args --- .../msprobe/pytorch/api_accuracy_checker/common/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 3ff48f596..467c1f045 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -96,10 +96,10 @@ def cross_entropy_process(api_info_dict): Return api_info_dict: api_info_dict: Processed argument of the API. """ - if 'args' in api_info_dict and len(api_info_dict['args']) > 1 and 'Min' in api_info_dict['args'][1]: - if api_info_dict['args'][1]['Min'] <= 0: + if 'input_args' in api_info_dict and len(api_info_dict['input_args']) > 1 and 'Min' in api_info_dict['input_args'][1]: + if api_info_dict['input_args'][1]['Min'] <= 0: # The second argument in cross_entropy should be -100 or not less than 0 - api_info_dict['args'][1]['Min'] = 0 + api_info_dict['input_args'][1]['Min'] = 0 return api_info_dict -- Gitee From 8143e35018760bebd10faefc1c6103e38a23b6d0 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Sat, 31 Aug 2024 18:13:56 +0800 Subject: [PATCH 613/791] =?UTF-8?q?mstt=20advisor=20=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E9=97=AE=E9=A2=98=201.print=E6=89=93=E5=8D=B0=E6=97=A5?= =?UTF-8?q?=E5=BF=97=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cluster_advice/cluster_advice_base.py | 11 ++++--- .../dataset/cluster/cluster_dataset.py | 10 +++--- .../profiling_parser/base_profiling_parser.py | 18 +++++++---- .../profiling_parser/npu_profiling_parser.py | 32 +++++++++++-------- 4 files changed, 41 insertions(+), 30 deletions(-) diff --git a/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py b/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py index e9be46759..2cc1eebeb 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py +++ b/profiler/advisor/advisor_backend/cluster_advice/cluster_advice_base.py @@ -14,11 +14,14 @@ # limitations under the License. import os +import logging from abc import abstractmethod from common_func.constant import Constant from advice_base import AdviceBase from cluster_analysis import Interface +logger = logging.getLogger() + class ClusterAdviceBase(AdviceBase): def __init__(self, collection_path: str): @@ -37,11 +40,11 @@ class ClusterAdviceBase(AdviceBase): """ for file in os.listdir(self.collection_path): if file == 'cluster_analysis_output': - print("[INFO]Cluster has been analyzed " - "because of the existence of cluster analysis output directory.") - print("[INFO]Skip Cluster analyze backend.") + logger.info("Cluster has been analyzed " + "because of the existence of cluster analysis output directory.") + logger.info("Skip Cluster analyze backend.") return - print("[INFO] cluster analysis is in the process, please wait...") + logger.info("cluster analysis is in the process, please wait...") self.cluster_analyze() def cluster_analyze(self): diff --git a/profiler/advisor/dataset/cluster/cluster_dataset.py b/profiler/advisor/dataset/cluster/cluster_dataset.py index 6894436ae..b3611a627 100644 --- a/profiler/advisor/dataset/cluster/cluster_dataset.py +++ b/profiler/advisor/dataset/cluster/cluster_dataset.py @@ -41,9 +41,9 @@ class ClusterDataset(Dataset): """ for filename in os.listdir(self.collection_path): if filename == 'cluster_analysis_output': - logger.info("[INFO]Cluster has been analyzed " + logger.info("Cluster has been analyzed " "because of the existence of cluster analysis output directory.") - logger.info("[INFO]Skip Cluster analyze backend.") + logger.info("Skip Cluster analyze backend.") return True return False @@ -54,7 +54,7 @@ class ClusterDataset(Dataset): Constant.COLLECTION_PATH: self.collection_path, Constant.ANALYSIS_MODE: "all" } - print("[INFO] cluster analysis is in the process, please wait...") + logger.info("cluster analysis is in the process, please wait...") try: Interface(parameter).run() except Exception as e: @@ -92,7 +92,7 @@ class ClusterStepTraceTimeDataset(ClusterDataset): try: step_data = self.load_csv_data(const.CLUSTER_STEP_TIME_CSV, ClusterStepTraceTimeBean) except RuntimeError as e: - print("捕获到异常：", e) + logger.error("捕获到异常：%s", e) self._step_dict = None return False self._step_dict = self.format_data(step_data) @@ -164,7 +164,7 @@ class ClusterCommunicationDataset(ClusterDataset): try: communication_json = self.load_json_data(const.CLUSTER_COMM_JSON) except RuntimeError as e: - print("捕获到异常：", e) + logger.error("捕获到异常：%s", e) self.rank_bw_dict = None return False self.process(communication_json) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py index 44f0e7105..2e5df1982 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/base_profiling_parser.py @@ -1,5 +1,6 @@ from abc import abstractmethod, ABC from decimal import Decimal +import logging from compare_backend.compare_bean.origin_data_bean.compare_event import KernelEvent, MemoryEvent from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean @@ -8,6 +9,8 @@ from compare_backend.compare_bean.profiling_info import ProfilingInfo from compare_backend.utils.constant import Constant from compare_backend.utils.file_reader import FileReader +logger = logging.getLogger() + class ProfilingResult: @@ -313,19 +316,20 @@ class BaseProfilingParser(ABC): def _check_result_data(self): if self._enable_operator_compare or self._enable_memory_compare or self._enable_api_compare: if not self._result_data.torch_op_data: - print(f"[WARNING] Can't find any torch op in the file: {self._profiling_path}") + logger.warning("Can't find any torch op in the file: %s", self._profiling_path) if self._enable_operator_compare and not self._result_data.kernel_dict: - print(f"[WARNING] Can't find any flow event in the file: {self._profiling_path}") + logger.warning("Can't find any flow event in the file: %s", self._profiling_path) if self._enable_memory_compare and not self._result_data.memory_list: - print(f"[WARNING] Can't find any memory event in the file: {self._profiling_path}") + logger.warning("Can't find any memory event in the file: %s", self._profiling_path) if self._enable_communication_compare and not self._result_data.communication_dict: - print(f"[WARNING] Can't find any communication op in the file: {self._profiling_path}") + logger.warning("Can't find any communication op in the file: %s", self._profiling_path) if self._enable_kernel_compare and not self._result_data.kernel_details: if self._profiling_type == Constant.GPU: - print(f"[WARNING] kernel compare only support between NPU data and NPU data.") + logger.warning(f"kernel compare only support between NPU data and NPU data.") else: - print(f"[WARNING] Can't find any valid kernels in the file: {self._profiling_path}. Please " - f"make sure that the profiling data is greater than level0 and aic_metrics=PipeUtilization.") + logger.warning("Can't find any valid kernels in the file: %s. Please " + "make sure that the profiling data is greater than level0 and " + "aic_metrics=PipeUtilization.", self._profiling_path) def _read_trace_event(self): try: diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 8f2714d9c..28d25e3e1 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -1,5 +1,6 @@ import os import sys +import logging from math import ceil from compare_backend.compare_bean.origin_data_bean.kernel_details_bean import KernelDetailsBean @@ -11,6 +12,9 @@ from compare_backend.utils.constant import Constant from compare_backend.utils.file_reader import FileReader +logger = logging.getLogger() + + class NPUProfilingParser(BaseProfilingParser): FLOW_CAT = "async_npu" TORCH_OP_CAT = "cpu_op" @@ -85,9 +89,9 @@ class NPUProfilingParser(BaseProfilingParser): try: kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) except FileNotFoundError: - print("[WARNING] The file kernel_details.csv does not exist.") + logger.warning("The file kernel_details.csv does not exist.") except Exception: - print("[ERROR] Failed to read kernel_details.csv.") + logger.warning("Failed to read kernel_details.csv.") return if not kernel_details: return @@ -106,7 +110,7 @@ class NPUProfilingParser(BaseProfilingParser): " please check whether the data contains this step." raise RuntimeError(msg) else: - print("[WARNING] Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") + logger.warning("Failed to enable enable_kernel_compare, type of kernel_details.csv is null.") return self._result_data.update_kernel_details(kernels_dict) @@ -114,10 +118,10 @@ class NPUProfilingParser(BaseProfilingParser): try: memory_data = FileReader.read_csv_file(self._operator_memory_path, OperatorMemoryBean) except FileNotFoundError: - print("[WARNING] The file operator_memory.csv does not exist.") + logger.warning("The file operator_memory.csv does not exist.") return except Exception: - print("[ERROR] Failed to read operator_memory.csv.") + logger.error("Failed to read operator_memory.csv.") return if memory_data: self._dequeue_data.sort(key=lambda x: x.start_time) @@ -156,12 +160,12 @@ class NPUProfilingParser(BaseProfilingParser): try: communication_json = FileReader.read_trace_file(self._communication_path) except FileNotFoundError: - print("[WARNING] The file communication.json does not exist.") + logger.warning("The file communication.json does not exist.") except Exception: - print("[ERROR] Failed to read communication.json.") + logger.error("Failed to read communication.json.") return if not communication_json: - print("[WARNING] The communication.json file is empty.") + logger.warning("The communication.json file is empty.") return for _, group_dict in communication_json.items(): step_dict = group_dict.get("collective", {}) @@ -317,10 +321,10 @@ class NPUProfilingParser(BaseProfilingParser): try: json_data = FileReader.read_trace_file(self._info_json_path) except Exception: - print('[ERROR] Failed to read profiler_info.json.') + logger.error('Failed to read profiler_info.json.') return if not isinstance(json_data, dict) or not json_data: - print('[WARNING] Invalid profiler info.') + logger.warning('Invalid profiler info.') return level = json_data.get('config', {}).get('experimental_config', {}).get('_profiler_level', '') if self.LEVEL_0 != level: @@ -339,7 +343,7 @@ class NPUProfilingParser(BaseProfilingParser): try: kernel_details = FileReader.read_csv_file(self._kernel_detail_path, KernelDetailsBean) except Exception: - print('[ERROR] Npu kernel details csv file is not available.') + logger.error('Npu kernel details csv file is not available.') return if not kernel_details or kernel_details[0].is_hide_op_pmu(): self._result_data.overall_metrics.hide_op_details = True @@ -355,16 +359,16 @@ class NPUProfilingParser(BaseProfilingParser): try: memory_record = FileReader.read_csv_file(self._memory_record_path, MemoryRecordBean) except FileNotFoundError: - print('[WARNING] Npu memory record csv file is not available.') + logger.warning('Npu memory record csv file is not available.') except Exception: - print('[ERROR] Load memory info failed.') + logger.error('Load memory info failed.') else: memory_used = max([memory.total_reserved_mb for memory in memory_record]) / 1024 self._result_data.overall_metrics.set_memory_used(memory_used) def __add_overlap_analysis_time(self): if not self._overlap_analysis: - print('[WARNING] Failed to get overlap analysis data.') + logger.warning('Failed to get overlap analysis data.') return min_ts = sys.float_info.max max_ts = sys.float_info.min -- Gitee From 5acbf3bdacd45338daef353270fce88d6d44e239 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Sat, 31 Aug 2024 18:30:05 +0800 Subject: [PATCH 614/791] mindspore free_benchmark adds exchange_value pert_mode --- .../msprobe/mindspore/common/const.py | 63 +++++++------- .../mindspore/debugger/debugger_config.py | 4 +- .../free_benchmark/api_pynative_self_check.py | 11 +-- .../mindspore/free_benchmark/common/utils.py | 2 +- .../free_benchmark/decorator/dec_forward.py | 5 +- .../perturbation/exchange_value.py | 51 +++++++++++ .../perturbation/improve_precision.py | 3 +- .../perturbation/perturbation_factory.py | 2 + .../msprobe/mindspore/ms_config.py | 9 +- .../perturbation/test_ms_exchange_value.py | 87 +++++++++++++++++++ 10 files changed, 191 insertions(+), 46 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/exchange_value.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/perturbation/test_ms_exchange_value.py diff --git a/debug/accuracy_tools/msprobe/mindspore/common/const.py b/debug/accuracy_tools/msprobe/mindspore/common/const.py index ede13e901..ec0ee5816 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/const.py @@ -1,7 +1,7 @@ import numpy as np import mindspore as ms -from msprobe.core.common.const import Const as CoreCost +from msprobe.core.common.const import Const as CoreConst class Const: @@ -9,9 +9,9 @@ class Const: API = "api" KERNEL = "kernel" TOOL_LEVEL_DICT = { - CoreCost.LEVEL_L0: CELL, - CoreCost.LEVEL_L1: API, - CoreCost.LEVEL_L2: KERNEL + CoreConst.LEVEL_L0: CELL, + CoreConst.LEVEL_L1: API, + CoreConst.LEVEL_L2: KERNEL } PYNATIVE_MODE = "pynative" GRAPH_GE_MODE = "graph_ge" @@ -22,46 +22,47 @@ class Const: JIT_LEVEL_O2 = "O2" ASCEND_910A = "ascend910" + OPS_PREFIX = "mindspore.ops." + Tensor_PREFIX = "mindspore.Tensor." + MINT_PREFIX = "mindspore.mint." + MINT_NN_FUNC_PREFIX = "mindspore.mint.nn.functional." + COMM_PREFIX = "mindspore.communication.comm_func." + COMMUNICATION_API_LIST = [ + "mindspore.communication.comm_func.all_gather_into_tensor", + "mindspore.communication.comm_func.gather_into_tensor", + "mindspore.communication.comm_func.all_reduce", + "mindspore.communication.comm_func.reduce", + "mindspore.communication.comm_func.reduce_scatter_tensor" + ] + class FreeBenchmarkConst: - DEFAULT_DEVICE = "npu" - DEFAULT_STAGE = "forward" - DEFAULT_DUMP_LEVEL = CoreCost.LEVEL_L1 - DEFAULT_PERT_TYPE = "improve_precision" - DEFAULT_HANDLER_TYPE = "check" - FIX_HANDLER_MODE = "fix" ADD_NOISE = "add_noise" BIT_NOISE = "bit_noise" NO_CHANGE = "no_change" + EXCHANGE_VALUE = "change_value" IMPROVE_PRECISION = "improve_precision" CHECK = "check" FIX = "fix" - DEVICE_LIST = ["npu"] - STAGE_LIST = ["forward"] - DUMP_LEVEL_LIST = [CoreCost.LEVEL_L1] - PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE] + DEFAULT_DEVICE = "npu" + DEFAULT_STAGE = CoreConst.FORWARD + DEFAULT_DUMP_LEVEL = "L1" + DEFAULT_PERT_TYPE = IMPROVE_PRECISION + DEFAULT_HANDLER_TYPE = CHECK + DEVICE_LIST = [DEFAULT_DEVICE] + STAGE_LIST = [CoreConst.FORWARD] + DUMP_LEVEL_LIST = [DEFAULT_DUMP_LEVEL] + PERT_TYPE_LIST = [IMPROVE_PRECISION, ADD_NOISE, BIT_NOISE, NO_CHANGE, EXCHANGE_VALUE] HANDLER_TYPE_LIST = [CHECK, FIX] - COMMUNICATION_API_LIST = [ - "mindspore.communication.comm_func.all_gather_into_tensor", - "mindspore.communication.comm_func.gather_into_tensor", - "mindspore.communication.comm_func.all_reduce", - "mindspore.communication.comm_func.reduce", - "mindspore.communication.comm_func.reduce_scatter_tensor" - ] NO_CHANGE_ERROR_THRESHOLD = 1.0 SYMBOL_FLIPPING_RATIO = 8.0 - OPS_PREFIX = "mindspore.ops." - Tensor_PREFIX = "mindspore.Tensor." - MINT_PREFIX = "mindspore.mint." - MINT_NN_FUNC_PREFIX = "mindspore.mint.nn.functional." - COMM_PREFIX = "mindspore.communication.comm_func." API_PREFIX_DICT = { - "ops": OPS_PREFIX, - "Tensor": Tensor_PREFIX, - "mint": MINT_PREFIX, - "mint.nn.functional": MINT_NN_FUNC_PREFIX, - "communication": COMM_PREFIX + "ops": Const.OPS_PREFIX, + "Tensor": Const.Tensor_PREFIX, + "mint": Const.MINT_PREFIX, + "mint.nn.functional": Const.MINT_NN_FUNC_PREFIX, + "communication": Const.COMM_PREFIX } PERT_VALUE_DICT = { diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 24b61f3df..64d993515 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -33,12 +33,12 @@ class DebuggerConfig: if not task_config.pert_mode else task_config.pert_mode) self.handler_type = (FreeBenchmarkConst.DEFAULT_HANDLER_TYPE if not task_config.handler_type else task_config.handler_type) - if self.handler_type == FreeBenchmarkConst.FIX_HANDLER_MODE and \ + self.stage = FreeBenchmarkConst.DEFAULT_STAGE if not task_config.fuzz_stage else task_config.fuzz_stage + if self.handler_type == FreeBenchmarkConst.FIX and \ self.pert_type != FreeBenchmarkConst.DEFAULT_PERT_TYPE: raise ValueError("pert_mode must be improve_precision or empty when handler_type is fix, " f"but got {self.pert_type}.") self.dump_level = FreeBenchmarkConst.DEFAULT_DUMP_LEVEL - self.stage = FreeBenchmarkConst.DEFAULT_STAGE def check(self): if not self.dump_path: diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index 510d5044f..4185b17ed 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -7,6 +7,7 @@ from mindspore.communication import comm_func from msprobe.core.common.utils import load_yaml from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.config import Config from msprobe.core.common.file_check import check_path_length @@ -53,23 +54,23 @@ def get_supported_ops(): _all_functional_ops = [] ms_ops = dir(ms.ops) - ms_ops = [FreeBenchmarkConst.OPS_PREFIX + i for i in ms_ops] + ms_ops = [MsConst.OPS_PREFIX + i for i in ms_ops] _all_functional_ops += ms_ops ms_tensor = dir(ms.Tensor) - ms_tensor = [FreeBenchmarkConst.Tensor_PREFIX + i for i in ms_tensor] + ms_tensor = [MsConst.Tensor_PREFIX + i for i in ms_tensor] _all_functional_ops += ms_tensor ms_mint = dir(ms.mint) - ms_mint = [FreeBenchmarkConst.MINT_PREFIX + i for i in ms_mint] + ms_mint = [MsConst.MINT_PREFIX + i for i in ms_mint] _all_functional_ops += ms_mint ms_mint_nn_func = dir(ms.mint.nn.functional) - ms_mint_nn_func = [FreeBenchmarkConst.MINT_NN_FUNC_PREFIX + i for i in ms_mint_nn_func] + ms_mint_nn_func = [MsConst.MINT_NN_FUNC_PREFIX + i for i in ms_mint_nn_func] _all_functional_ops += ms_mint_nn_func ms_communication = dir(comm_func) - ms_communication = [FreeBenchmarkConst.COMM_PREFIX + i for i in ms_communication] + ms_communication = [MsConst.COMM_PREFIX + i for i in ms_communication] _all_functional_ops += ms_communication return set(supported_ops) & set(_all_functional_ops) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py index 3bb062800..693c0c989 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/common/utils.py @@ -60,7 +60,7 @@ def make_unequal_row( row.max_rel = ratio - 1 original_tensor = params.original_result fuzzed_tensor = params.fuzzed_result - if index: + if index is not None: original_tensor = original_tensor[index] fuzzed_tensor = fuzzed_tensor[index] row.output_index = index diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py index 78661d7fc..ea1f36844 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/decorator/dec_forward.py @@ -1,4 +1,5 @@ from msprobe.mindspore.free_benchmark.common.config import Config +from msprobe.mindspore.common.const import Const from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams from msprobe.mindspore.free_benchmark.handler.handler_factory import HandlerFactory @@ -23,7 +24,7 @@ class ForwardSelfChecker: return params.original_result def get_compare_data(self, params: HandlerParams): - if self.api_name not in FreeBenchmarkConst.COMMUNICATION_API_LIST: + if self.api_name not in Const.COMMUNICATION_API_LIST: return # 以下为通讯类api处理逻辑 params.fuzzed_result = params.fuzzed_value @@ -37,6 +38,6 @@ class ForwardSelfChecker: self.get_compare_data(params) handler = HandlerFactory.create(self.api_name) result = handler.handle(params) - if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST: + if self.api_name in Const.COMMUNICATION_API_LIST: result = original_result return result diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/exchange_value.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/exchange_value.py new file mode 100644 index 000000000..f5a003a41 --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/exchange_value.py @@ -0,0 +1,51 @@ +from typing import Any + +from mindspore import Tensor + +from msprobe.mindspore.common.log import logger +from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import BasePerturbation +from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams + + +class ExchangeValuePerturbation(BasePerturbation): + + def handle(self, params: HandlerParams) -> Any: + """ + 返回首尾交换后的api输出 + + """ + params.fuzzed_value = self.exchange_value(params.args[params.index]) + if not self.is_fuzzed: + logger.warning(f"{self.api_name} can not exchange value.") + return False + return self.get_fuzzed_result(params) + + def exchange_value(self, inputs) -> Any: + """ + 返回首尾交换后的api输入 + + """ + if isinstance(inputs, Tensor): + if not self.is_fuzzed and len(inputs.shape) > 0 and inputs.shape[0] > 1: + result = inputs.copy() + if len(inputs.shape) == 1: + first_element = inputs[0] + last_element = inputs[-1] + result[0] = last_element + result[-1] = first_element + else: + first_element = inputs[0][0] + last_element = inputs[-1][-1] + result[0][0] = last_element + result[-1][-1] = first_element + + self.is_fuzzed = True + return result + + if isinstance(inputs, dict): + return {k: self.exchange_value(v) for k, v in inputs.items()} + + if isinstance(inputs, (list, tuple)): + return type(inputs)([self.exchange_value(v) for v in inputs]) + + return inputs diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py index f55a96aca..01d7ea098 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/improve_precision.py @@ -7,6 +7,7 @@ from msprobe.mindspore.free_benchmark.perturbation.base_perturbation import Base from msprobe.mindspore.free_benchmark.common.handler_params import HandlerParams from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.common.log import logger +from msprobe.mindspore.common.const import Const class ImprovePrecisionPerturbation(BasePerturbation): @@ -26,7 +27,7 @@ class ImprovePrecisionPerturbation(BasePerturbation): args = self.improve_tensor_precision(params.args) kwargs = self.improve_tensor_precision(params.kwargs) fuzzed_value = args - if self.api_name in FreeBenchmarkConst.COMMUNICATION_API_LIST: + if self.api_name in Const.COMMUNICATION_API_LIST: params.fuzzed_value = fuzzed_value if not self.is_fuzzed: logger.warning(f"{self.api_name} can not improve precision.") diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py index 6c8328dc2..621f87c3b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/perturbation/perturbation_factory.py @@ -4,6 +4,7 @@ from .add_noise import AddNoisePerturbation from .bit_noise import BitNoisePerturbation from .no_change import NoChangePerturbation from .improve_precision import ImprovePrecisionPerturbation +from .exchange_value import ExchangeValuePerturbation class PerturbationFactory: @@ -16,6 +17,7 @@ class PerturbationFactory: FreeBenchmarkConst.ADD_NOISE: AddNoisePerturbation, FreeBenchmarkConst.BIT_NOISE: BitNoisePerturbation, FreeBenchmarkConst.NO_CHANGE: NoChangePerturbation, + FreeBenchmarkConst.EXCHANGE_VALUE: ExchangeValuePerturbation } @staticmethod diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index 4109a8679..77e78a095 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -65,7 +65,8 @@ class FreeBenchmarkConfig(BaseConfig): if self.fuzz_device and self.fuzz_device not in FreeBenchmarkConst.DEVICE_LIST: raise Exception("fuzz_device must be npu or empty") if self.pert_mode and self.pert_mode not in FreeBenchmarkConst.PERT_TYPE_LIST: - raise Exception("pert_mode must be improve_precision, add_noise, bit_noise, no_change or empty") + raise Exception("pert_mode must be improve_precision, add_noise, bit_noise, " + "no_change, change_value or empty") if self.handler_type and self.handler_type not in FreeBenchmarkConst.HANDLER_TYPE_LIST: raise Exception("handler_type must be check, fix or empty") if self.fuzz_level and self.fuzz_level not in FreeBenchmarkConst.DUMP_LEVEL_LIST: @@ -83,12 +84,12 @@ class GradProbeConfig(BaseConfig): self.grad_level = json_config.get("grad_level", "L1") self.param_list = json_config.get("param_list", []) self.bounds = json_config.get("bounds", []) - + def _check_config(self): if self.grad_level not in level_adp.keys(): raise Exception(f"grad_level must be one of {level_adp.keys()}") if not isinstance(self.param_list, list): - raise Exception(f"param_list must be a list") + raise Exception("param_list must be a list") check_numeral_list_ascend(self.bounds) @@ -97,7 +98,7 @@ TaskDict = { Const.STATISTICS: StatisticsConfig, Const.OVERFLOW_CHECK: OverflowCheckConfig, Const.FREE_BENCHMARK: FreeBenchmarkConfig, - Const.GRAD_PROBE: GradProbeConfig, + Const.GRAD_PROBE: GradProbeConfig } diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/perturbation/test_ms_exchange_value.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/perturbation/test_ms_exchange_value.py new file mode 100644 index 000000000..368a3b55d --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/free_benchmark/perturbation/test_ms_exchange_value.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import unittest + +from mindspore import Tensor + +from msprobe.mindspore.free_benchmark.perturbation.exchange_value import ExchangeValuePerturbation + + +class TestExchangeValuePerturbation(unittest.TestCase): + def test_exchange_value(self): + perturbation = ExchangeValuePerturbation("api_name") + + original_tensor = Tensor(1.0) + original_cache_tensor = Tensor(1.0) + target_tensor = Tensor(1.0) + final_tensor = perturbation.exchange_value(original_tensor) + self.assertTrue((original_tensor == original_cache_tensor).all()) + self.assertTrue((final_tensor == target_tensor).all()) + + perturbation.is_fuzzed = False + original_tensor = Tensor([[1.0, 2.0, 3.0]]) + original_cache_tensor = Tensor([[1.0, 2.0, 3.0]]) + target_tensor = Tensor([[1.0, 2.0, 3.0]]) + final_tensor = perturbation.exchange_value(original_tensor) + self.assertTrue((original_tensor == original_cache_tensor).all()) + self.assertTrue((final_tensor == target_tensor).all()) + + perturbation.is_fuzzed = False + original_tensor = Tensor([1.0, 2.0, 3.0]) + original_cache_tensor = Tensor([1.0, 2.0, 3.0]) + target_tensor = Tensor([3.0, 2.0, 1.0]) + final_tensor = perturbation.exchange_value(original_tensor) + self.assertTrue((original_tensor == original_cache_tensor).all()) + self.assertTrue((final_tensor == target_tensor).all()) + + perturbation.is_fuzzed = False + original_tensor = Tensor([1.0, 2.0, 3.0]) + original_cache_tensor = Tensor([1.0, 2.0, 3.0]) + original_tensors = [original_tensor, original_tensor, original_tensor] + original_cache_tensors = [original_cache_tensor, original_cache_tensor, original_cache_tensor] + target_tensor = Tensor([3.0, 2.0, 1.0]) + target_tensors = [target_tensor, original_tensor, original_tensor] + final_tensors = perturbation.exchange_value(original_tensors) + self.assertTrue((original_tensors[0] == original_cache_tensors[0]).all()) + self.assertTrue((original_tensors[1] == original_cache_tensors[1]).all()) + self.assertTrue((original_tensors[2] == original_cache_tensors[2]).all()) + self.assertTrue((final_tensors[0] == target_tensors[0]).all()) + self.assertTrue((final_tensors[1] == target_tensors[1]).all()) + self.assertTrue((final_tensors[2] == target_tensors[2]).all()) + + perturbation.is_fuzzed = False + original_tensor = Tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) + original_cache_tensor = Tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]) + target_tensor = Tensor([[6.0, 2.0, 3.0], [4.0, 5.0, 1.0]]) + final_tensor = perturbation.exchange_value(original_tensor) + self.assertTrue((original_tensor == original_cache_tensor).all()) + self.assertTrue((final_tensor == target_tensor).all()) + + perturbation.is_fuzzed = False + original_tensor = Tensor([[[1.0, 2.0], [2.0, 3.0], [3.0, 4.0]], [[4.0, 5.0], [5.0, 6.0], [6.0, 7.0]]]) + original_cache_tensor = Tensor([[[1.0, 2.0], [2.0, 3.0], [3.0, 4.0]], [[4.0, 5.0], [5.0, 6.0], [6.0, 7.0]]]) + target_tensor = Tensor([[[6.0, 7.0], [2.0, 3.0], [3.0, 4.0]], [[4.0, 5.0], [5.0, 6.0], [1.0, 2.0]]]) + final_tensor = perturbation.exchange_value(original_tensor) + self.assertTrue((original_tensor == original_cache_tensor).all()) + self.assertTrue((final_tensor == target_tensor).all()) + + perturbation.is_fuzzed = False + original_tensor = {"value": Tensor([1.0, 2.0, 3.0])} + target_tensor = {"value": Tensor([3.0, 2.0, 1.0])} + final_tensor = perturbation.exchange_value(original_tensor) + self.assertTrue((target_tensor.get("value") == final_tensor.get("value")).all()) -- Gitee From cc8e6aafd64812eeb99de052ac3b3055a96d384f Mon Sep 17 00:00:00 2001 From: zhaolei Date: Sat, 31 Aug 2024 18:30:15 +0800 Subject: [PATCH 615/791] =?UTF-8?q?mstt=20advisor=20bug=E4=BF=AE=E8=A1=A5?= =?UTF-8?q?=EF=BC=8C=E4=BF=AE=E5=A4=8Dgraph=20advizer=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py b/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py index e9dcd263d..7e31c8926 100644 --- a/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py +++ b/profiler/advisor/analyzer/graph_fusion/graph_fusion_analyzer.py @@ -45,4 +45,4 @@ class FusionOPAnalyzer(BaseAnalyzer): else: checker.find_fusion_matched_issues_with_times(graph_data, profiling_data) checker.make_record(self.result) - self.html = checker.make_render(self.html_render, add_render_list) \ No newline at end of file + self.html = checker.make_render(self.html_render) \ No newline at end of file -- Gitee From 534040e239dbf6db6bbafdd1dbf035316d56d4c4 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Sat, 31 Aug 2024 17:58:50 +0800 Subject: [PATCH 616/791] =?UTF-8?q?mstt=20advisor=20=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E9=97=AE=E9=A2=98=201.=E5=A4=96=E9=83=A8=E6=96=87=E4=BB=B6?= =?UTF-8?q?=E9=BB=98=E8=AE=A4=E5=80=BC=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../analyzer/computation/bound/block_dim_checker.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py index 5b358ebaa..54921247f 100644 --- a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py +++ b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py @@ -13,6 +13,8 @@ class BlockDimChecker(OperatorChecker): _SUGGESTION: List[str] = [] _CHECKER = "block dim" _PROBLEM = "block dim" + _aicore_num = 0 + _aiv_num = 0 _description = "some operator does not make full use of {} ai core" _ITEMS = [ "op_name", "op_type", "task_type", "task_duration", "income", "block_dim", "mix_block_dim", "input_shapes", @@ -35,9 +37,16 @@ class BlockDimChecker(OperatorChecker): logger.warning(self.SKIP_CHECK_MSG, self._CHECKER, "block dim in op summary") return False if Config().get_config("ai_core_num"): - self._aicore_num = int(Config().get_config("ai_core_num")) + try: + self._aicore_num = int(Config().get_config("ai_core_num")) + except ValueError as e: + logger.warning("get ai_core_num failed, please check info.json： %s", e) + return False if Config().get_config("aiv_num"): - self._aiv_num = int(Config().get_config("aiv_num")) + try: + self._aiv_num = int(Config().get_config("aiv_num")) + except ValueError as e: + logger.warning("get aiv_num failed, please check info.json： %s", e) self._description = self._description.format(self._aicore_num) if self._aiv_num: self._description += f" or {self._aiv_num} ai vector core" -- Gitee From 7f40250d17fe964fd0521a94b3e19b8f7668702d Mon Sep 17 00:00:00 2001 From: hhqx Date: Sat, 31 Aug 2024 18:37:32 +0800 Subject: [PATCH 617/791] check access before csv read write. --- .../advisor_backend/compute_advice/npu_fused/csv_analyzer.py | 2 ++ .../advisor/advisor_backend/compute_advice/npu_slow_advice.py | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py b/profiler/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py index c85c14d61..8ae109856 100644 --- a/profiler/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py +++ b/profiler/advisor/advisor_backend/compute_advice/npu_fused/csv_analyzer.py @@ -18,6 +18,7 @@ import multiprocessing import pandas as pd import numpy as np +from common_func.path_manager import PathManager from common_func_advisor.constant import Constant from .op_perf import OpPerfFactory @@ -27,6 +28,7 @@ class CSVAnalyzer: self._path = path def process(self): + PathManager.check_path_readable(self._path) df = pd.read_csv(self._path, dtype={"Start Time(us)": str}) # 分析是否存在可融合的算子 op_type_list = df["Type"].tolist() diff --git a/profiler/advisor/advisor_backend/compute_advice/npu_slow_advice.py b/profiler/advisor/advisor_backend/compute_advice/npu_slow_advice.py index caff1c792..48522cf55 100644 --- a/profiler/advisor/advisor_backend/compute_advice/npu_slow_advice.py +++ b/profiler/advisor/advisor_backend/compute_advice/npu_slow_advice.py @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC +import os import multiprocessing import pandas as pd +from common_func.path_manager import PathManager from compute_advice.compute_advice_base import ComputeAdviceBase from compute_advice.npu_fused.op_perf import OpPerfFactory from common_func_advisor.constant import Constant @@ -34,6 +36,7 @@ class NpuSlowAdvice(ComputeAdviceBase, ABC): @staticmethod def save_to_excel(data: pd.DataFrame, file_path: str) -> None: + PathManager.check_path_writeable(os.path.dirname(file_path)) writer = pd.ExcelWriter(file_path, engine="xlsxwriter", mode="w") data.index.name = Constant.TITLE.INDEX data.to_excel(writer, index=True, sheet_name=NpuSlowAdvice.OP_PERF_SHEET) @@ -73,6 +76,7 @@ class NpuSlowAdvice(ComputeAdviceBase, ABC): return self.data def process(self): + PathManager.check_path_readable(self.kernel_details_path) self.data = pd.read_csv(self.kernel_details_path, dtype={"Start Time(us)": str}) # 去除末尾的\t分隔符 self.data["Start Time(us)"] = self.data["Start Time(us)"].apply(lambda x: x[:-1]) -- Gitee From c1a095e2f8126dbe9c6db0ddb109cef3c0b26142 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 2 Sep 2024 14:34:56 +0800 Subject: [PATCH 618/791] fix --- debug/accuracy_tools/msprobe/core/common/const.py | 4 ++++ .../pytorch/bench_functions/npu_fusion_attention.py | 13 ++++++------- .../msprobe/pytorch/hook_module/wrap_npu_custom.py | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 4c39af0ef..366e252c7 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -210,6 +210,10 @@ class CompareConst: RED = "FFFF0000" YELLOW = "FFFF00" BLUE = "0000FF" + + # run_ut const + MAX_TOKENS = 65536 + SPECIAL_SPARSE_MOED = 4 # highlight rules const OVERFLOW_LIST = ['nan\t', 'inf\t', '-inf\t', 'nan', 'inf', '-inf'] diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index d34ee556a..5098065ee 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -16,7 +16,7 @@ else: from msprobe.pytorch.common.utils import logger -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, CompareConst gtype = torch.float64 # arm host必须选择float64，x86环境选择float32即可，64也行。arm计算很慢，s=8k的场景建议使用x86 softmax_build_mode = "QKV" # "MAX_SUM" @@ -485,21 +485,20 @@ def gpu_fusion_attention(*args, **kwargs): next_tockens = new_kwargs.get("next_tockens") attn_mask = new_kwargs.get("atten_mask") atten_mask_dtype = attn_mask.dtype if new_kwargs.get("atten_mask") is not None else None - pre_tockens = 65536 if pre_tockens > 65536 else pre_tockens - next_tockens = 65536 if next_tockens > 65536 else next_tockens + pre_tockens = min(CompareConst.MAX_TOKENS, pre_tockens) + next_tockens = min(CompareConst.MAX_TOKENS, next_tockens) atten_off = (is_attention_off_due_to_mask(atten_mask_dtype) or is_attention_off_in_sparse_mode_4(sparse_mode, next_tockens, pre_tockens, S1) or is_attention_off_in_sparse_mode_0(sparse_mode, pre_tockens, next_tockens, S1, S2)) - causal_switch = False if atten_off else True - if sparse_mode == 4: + causal_switch = not atten_off + if sparse_mode == CompareConst.SPECIAL_SPARSE_MOED: window_left = pre_tockens window_right = next_tockens else: - pre_tockens = next_tockens = 65536 + pre_tockens = next_tockens = CompareConst.MAX_TOKENS window_left = pre_tockens - S1 + S2 window_right = next_tockens + S1 - S2 - pttype = query.dtype if pse is not None: alibi_slopes = torch.rand(B, N1, dtype=torch.float32) * 0.3 else: diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index 26add2b4a..39aab712f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -73,7 +73,7 @@ class NpuOPTemplate(HOOKModule): if self.device == Const.CUDA_LOWERCASE: self.op_name_ = cuda_func_mapping.get(self.op_name_, self.op_name_) if self.device in [Const.CUDA_LOWERCASE, Const.CPU_LOWERCASE]: - return npu_custom_functions[self.op_name_](*args, **kwargs) + return npu_custom_functions.get(self.op_name_)(*args, **kwargs) if torch_without_guard_version: return getattr(torch.ops.npu, str(self.op_name_))(*args, **kwargs) else: -- Gitee From 2267bee05b83a821e3dd5ce45af3ff647d7a5a49 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 2 Sep 2024 14:40:31 +0800 Subject: [PATCH 619/791] fix --- .../msprobe/pytorch/hook_module/wrap_npu_custom.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index 39aab712f..26add2b4a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -73,7 +73,7 @@ class NpuOPTemplate(HOOKModule): if self.device == Const.CUDA_LOWERCASE: self.op_name_ = cuda_func_mapping.get(self.op_name_, self.op_name_) if self.device in [Const.CUDA_LOWERCASE, Const.CPU_LOWERCASE]: - return npu_custom_functions.get(self.op_name_)(*args, **kwargs) + return npu_custom_functions[self.op_name_](*args, **kwargs) if torch_without_guard_version: return getattr(torch.ops.npu, str(self.op_name_))(*args, **kwargs) else: -- Gitee From a47d0d86db4457beae45c81249a345a7c5d3957b Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Sat, 31 Aug 2024 15:30:36 +0800 Subject: [PATCH 620/791] fix_unpublished_url_and_email --- .../op_compile/dynamic_shape_checker.py | 3 +- .../analyzer/computation/operator_checker.py | 6 +-- .../fusion_ops/fusion_ops_analyzer.py | 7 ++-- .../synchronize_stream_checker.py | 3 +- profiler/advisor/common/constant.py | 10 ----- profiler/advisor/config/config.ini | 9 +++- profiler/advisor/config/config.py | 42 +++++++++++++++++++ profiler/config/config.ini | 4 ++ profiler/prof_common/constant.py | 2 + profiler/prof_common/path_manager.py | 13 ++++++ profiler/setup.py | 23 ++++++++-- ...347\275\221URL\350\257\264\346\230\216.md" | 20 +++++---- 12 files changed, 111 insertions(+), 31 deletions(-) create mode 100644 profiler/config/config.ini diff --git a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py b/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py index 2521b6e7e..9cd75a6e9 100644 --- a/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py +++ b/profiler/advisor/analyzer/computation/op_compile/dynamic_shape_checker.py @@ -4,6 +4,7 @@ from typing import List from profiler.advisor.analyzer.computation.operator_checker import OperatorChecker from profiler.advisor.common import constant +from profiler.advisor.config.config import Config from profiler.advisor.dataset.profiling.info_collection import OpInfo from profiler.advisor.result.item import OptimizeItem, StatisticsItem, OptimizeRecord @@ -55,7 +56,7 @@ class DynamicShapeChecker(OperatorChecker): release_suggestion = copy.deepcopy(suggestion) if release_suggestion == DynamicShapeChecker.ENABLE_COMPILED_SUGGESTION: release_suggestion += \ - f"for details please refer to link : LINK" + f"for details please refer to link : LINK" release_suggestion_list.append(release_suggestion.replace('\n', '
')) format_result = {"record": record.__dict__, "suggestion": '
'.join(release_suggestion_list)} return format_result diff --git a/profiler/advisor/analyzer/computation/operator_checker.py b/profiler/advisor/analyzer/computation/operator_checker.py index e24eae1d0..2e2f5a5c4 100644 --- a/profiler/advisor/analyzer/computation/operator_checker.py +++ b/profiler/advisor/analyzer/computation/operator_checker.py @@ -150,7 +150,7 @@ class OperatorChecker(VersionControl): logger.warning( "Skip dynamic shape check because of not containing ge_info.db file in host filefloder.\n" "To enable dynamic shape check, please try to set data_simplification=False in experimental_config.\n" - "More details please refer to link : %s", constant.ASCEND_PROFILER_URL) + "More details please refer to link : %s", Config().ascend_profiler_url) else: # CANN 8.0.RC1 之后 op_state 属性从 op_summary 文件中获取 if hasattr(profiling_database, "op_summary"): @@ -176,14 +176,14 @@ class OperatorChecker(VersionControl): release_suggestion = copy.deepcopy(suggestion) if release_suggestion == OperatorChecker.PyTorch_OPERATOR_TUNE_SUGGESTION: release_suggestion += \ - (f"for details please refer to link : LINK") + (f"for details please refer to link : LINK") elif release_suggestion == OperatorChecker.MSLite_OPERATOR_TUNE_SUGGESTION: release_suggestion += \ (f"\nThe config file for MSLite AOE usage is as follows:\n" \ f"[ascend_context]\n" \ f"aoe_mode=\"operator tuning\"\n" \ f"--tune_ops_file={Config().tune_ops_file}\n" - f"\nFor details please refer to link : LINK") + f"\nFor details please refer to link : LINK") release_suggestion_list.append(release_suggestion.replace('\n', '
')) format_result = {"record": record.__dict__, "suggestion": fill('
'.join(release_suggestion_list), width=200), diff --git a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py index 305d23994..d3e1c4669 100644 --- a/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py +++ b/profiler/advisor/analyzer/schedule/fusion_ops/fusion_ops_analyzer.py @@ -8,6 +8,7 @@ from profiler.advisor.analyzer.base_analyzer import BaseAnalyzer from profiler.advisor.common import constant as const from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.common.timeline.event import TimelineEvent +from profiler.advisor.config.config import Config from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.utils.utils import format_timeline_result @@ -155,7 +156,7 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): if self.empty_stacks: desc += ", but with no stack" suggestion = const.TIMELINE_EMPTY_STACKS_PROMPT.format( - timeline_profiling_doc_url=const.TIMELINE_WITH_STACK_DOC_URL + timeline_profiling_doc_url=Config().timeline_with_stack_doc_url ) sheet_name = "Affinity apis" @@ -188,8 +189,8 @@ class TimelineFusionOpsAnalyzer(BaseAnalyzer): cann_version=self.cann_version, torch_version=self.torch_version, empty_stacks=self.empty_stacks, - with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, - api_doc_url=const.TIMELINE_API_DOC_URL, + with_stack_doc_url=Config().timeline_with_stack_doc_url, + api_doc_url=Config().timeline_api_doc_url, result=format_result_for_html, priority_background_color=self.get_priority()) diff --git a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py index 7af46f766..9136d611d 100644 --- a/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py +++ b/profiler/advisor/analyzer/schedule/synchronize_stream/synchronize_stream_checker.py @@ -1,6 +1,7 @@ import logging from profiler.advisor.common import constant as const +from profiler.advisor.config.config import Config from profiler.advisor.dataset.timeline_event_dataset import ScheduleAnalysisDataset from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord @@ -84,7 +85,7 @@ class SynchronizeStreamChecker(TimelineBaseChecker): desc=self.desc, solutions=self.solutions, result=format_result_for_html, - with_stack_doc_url=const.TIMELINE_WITH_STACK_DOC_URL, + with_stack_doc_url=Config().timeline_with_stack_doc_url, empty_stacks=self.empty_stacks, framework_black_list=self.framework_black_list, priority_background_color=priority) diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index 298e94fc1..d607c5ba9 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -49,17 +49,7 @@ NO_STACK_REASON_MAP = { TIMELINE_BACKWARD_NO_STACK_CODE: "Backward broadcast, without call stacks in profiling.", TIMELINE_ACL_TO_NPU_NO_STACK_CODE: "Incoming flow is 'acl_to_npu', without call stacks in profiling." } -TIMELINE_API_DOC_URL = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" AFFINITY_TRAINING_API = "Affinity training api" -TIMELINE_WITH_STACK_DOC_URL = "https://www.hiascend.com/document/detail/zh/canncommercial/" \ - "70RC1/modeldevpt/ptmigr/AImpug_0067.html" -PyTorch_AOE_OPERATOR_TUNE_URL = "https://www.hiascend.com/document/detail/zh/canncommercial/" \ - "70RC1/devtools/auxiliarydevtool/aoe_16_045.html" -MSLite_Infer_AOE_OPEATOR_TUNE_URL = "https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning" -ENABLE_COMPILED_TUNE_URL = "https://www.hiascend.com/document/detail/zh/canncommercial/" \ - "70RC1/modeldevpt/ptmigr/AImpug_0059.html" - -ASCEND_PROFILER_URL = "https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html" TIMELINE_EMPTY_STACKS_PROMPT = "These APIs have no code stack. If parameter 'with_stack=False' while profiling, " \ "please refer to {timeline_profiling_doc_url} to set 'with_stack=True'. " \ "Otherwise, ignore following affinity APIs due to backward broadcast lack of stack." diff --git a/profiler/advisor/config/config.ini b/profiler/advisor/config/config.ini index 06e993160..f027b90a1 100644 --- a/profiler/advisor/config/config.ini +++ b/profiler/advisor/config/config.ini @@ -14,4 +14,11 @@ frequency_threshold = 0.05 # region : URL of different regions where can download rule yaml file cn-north-9 = cnnorth9-modelarts-sdk cn-southwest-2 = cnsouthwest2-modelarts-sdk -cn-north-7 = cnnorth7-modelarts-sdk \ No newline at end of file +cn-north-7 = cnnorth7-modelarts-sdk +[URL] +timeline_api_doc_url = "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" +timeline_with_stack_doc_url = "https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html" +pytorch_aoe_operator_tune_url = "https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html" +mslite_infer_aoe_operator_tune_url = "https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning" +enable_compiled_tune_url = "https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html" +ascend_profiler_url = "https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html" \ No newline at end of file diff --git a/profiler/advisor/config/config.py b/profiler/advisor/config/config.py index 4f36dfedf..bfa20a3d3 100644 --- a/profiler/advisor/config/config.py +++ b/profiler/advisor/config/config.py @@ -104,6 +104,48 @@ class Config: """ return float(self.config.get("THRESHOLD", "frequency_threshold")) + @property + def timeline_api_doc_url(self) -> str: + try: + return self.config.get("URL", "timeline_api_doc_url") + except Exception: + return "" + + @property + def timeline_with_stack_doc_url(self) -> str: + try: + return self.config.get("URL", "timeline_with_stack_doc_url") + except Exception: + return "" + + @property + def pytorch_aoe_operator_tune_url(self) -> str: + try: + return self.config.get("URL", "pytorch_aoe_operator_tune_url") + except Exception: + return "" + + @property + def mslite_infer_aoe_operator_tune_url(self) -> str: + try: + return self.config.get("URL", "mslite_infer_aoe_operator_tune_url") + except Exception: + return "" + + @property + def enable_compiled_tune_url(self) -> str: + try: + return self.config.get("URL", "enable_compiled_tune_url") + except Exception: + return "" + + @property + def ascend_profiler_url(self) -> str: + try: + return self.config.get("URL", "ascend_profiler_url") + except Exception: + return "" + def set_log_path(self, result_file: str, log_path: str = None): self.log_path = log_path if log_path is not None else os.path.join(self._work_path, "log") os.makedirs(self.log_path, exist_ok=True) diff --git a/profiler/config/config.ini b/profiler/config/config.ini new file mode 100644 index 000000000..9117b35ff --- /dev/null +++ b/profiler/config/config.ini @@ -0,0 +1,4 @@ +[URL] +msprof_analyze_url = "https://gitee.com/ascend/mstt/tree/master/profiler" +[EMAIL] +ms_email = "pmail_mindstudio@huawei.com" \ No newline at end of file diff --git a/profiler/prof_common/constant.py b/profiler/prof_common/constant.py index 87bc51b56..4d3d1ce63 100644 --- a/profiler/prof_common/constant.py +++ b/profiler/prof_common/constant.py @@ -12,6 +12,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + + class Constant(object): COLLECTION_PATH = "collection_path" ANALYSIS_MODE = "analysis_mode" diff --git a/profiler/prof_common/path_manager.py b/profiler/prof_common/path_manager.py index 3e41b8b50..b91f23133 100644 --- a/profiler/prof_common/path_manager.py +++ b/profiler/prof_common/path_manager.py @@ -17,6 +17,8 @@ import re import shutil import platform +from .constant import Constant + class PathManager: MAX_PATH_LENGTH = 4096 @@ -189,3 +191,14 @@ class PathManager: msg = f"Invalid input path which is a soft link." raise RuntimeError(msg) return os.path.realpath(path) + + @classmethod + def check_file_size(cls, file_path: str): + if not os.path.exists(file_path): + raise FileNotFoundError(f"The file {file_path} does not exists.") + file_size = os.path.getsize(file_path) + if file_size > Constant.MAX_FILE_SIZE_5_GB: + check_msg = input( + f"The file({file_path}) size exceeds the preset max value. Continue reading the file? [y/n]") + if check_msg.lower() != "y": + raise RuntimeError(f"[WARNING] The user choose not to read the file: {file_path}") diff --git a/profiler/setup.py b/profiler/setup.py index 4a625f6c9..00ecb353f 100644 --- a/profiler/setup.py +++ b/profiler/setup.py @@ -1,9 +1,12 @@ #!/usr/bin/python # -*- coding: utf-8 -*- -import os.path +import os +from configparser import ConfigParser from setuptools import find_packages, setup # type: ignore +from prof_common.path_manager import PathManager + extras = { "test": [ "pytest==6.2.4", @@ -23,6 +26,20 @@ tests_requires.extend(set(requires)) with open('version.txt', 'r') as f: version = f.read().strip() +config_file_path = "config/config.ini" +PathManager.check_input_file_path(config_file_path) +PathManager.check_file_size(config_file_path) +config = ConfigParser(allow_no_value=True) +config.read(config_file_path) +try: + url = config.get("URL", "msprof_analyze_url") +except Exception as e: + raise RuntimeError("The configuration file is incomplete and not configured msprof_analyze_url information.") from e +try: + author_email = config.get("EMAIL", "ms_email") +except Exception as e: + raise RuntimeError("The configuration file is incomplete and not configured ms_email information.") from e + root_path = os.path.dirname(os.path.abspath(os.path.dirname(__file__))) setup( name="msprof-analyze", @@ -31,9 +48,9 @@ setup( long_description="msprof-analyze provides statistics, analysis, and related tuning suggestions for the " "performance data collected in training and large model scenarios. The main functional modules" " include: performance comparison, performance analysis, and cluster analysis.", - url="https://gitee.com/ascend/mstt/tree/master/profiler", + url=url, author="MindStudio", - author_email="pmail_mindstudio@huawei.com", + author_email=author_email, package_dir={"": root_path}, packages=find_packages(root_path), include_package_data=False, diff --git "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" index 2db9b8786..1fe807d14 100644 --- "a/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" +++ "b/\345\205\254\347\275\221URL\350\257\264\346\230\216.md" @@ -1,11 +1,13 @@ # 公网URL说明 -| 软件类型 | 软件名 | 路径 | 类型 | 内容 | 用途说明 | -| -------- | -------------------------------------------------- | ------------------------------------ | -------- | ------------------------------------------------------------ | ----------------------- | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md"] | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html"] | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html"] | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning"] | Advisor优化手段参考示例 | -| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html"] | Advisor优化手段参考示例 | -| 文档 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/common/constant.py | 公网地址 | ["https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424"] | Advisor优化手段参考示例 | - +| 软件类型 | 软件名 | 路径 | 类型 | 内容 | 用途说明 | +|------| -------------------------------------------------- |------------------------------------------|------|------------------------------------------------------------------------------------------------------------| ----------------------- | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20Fused%20Operator%20API%20Replacement.md" | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0067.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/devtools/auxiliarydevtool/aoe_16_045.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.mindspore.cn/lite/docs/en/master/use/cloud_infer/converter_tool_ascend.html#aoe-auto-tuning | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/config/config.ini | 公网地址 | https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/AImpug_0059.html | Advisor优化手段参考示例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/config/config.ini | 公网地址 | https://gitee.com/ascend/mstt/tree/master/profiler | msprof-analyze工具地址 | +| 开源软件 | MindStudio Training Tools - msprof-analyze | /profiler/LICENSE | 公网地址 | http://www.apache.org/licenses/LICENSE-2.0 | 开源软件协议地址 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/rules/aicpu_rules.ymal | 公网地址 | "https://gitee.com/ascend/mstt/blob/master/profiler/advisor/doc/Samples%20of%20AI%20CPU%20Operator%20Replacement.md | AI CPU 算子替换样例 | +| 开源软件 | MindStudio Training Tools - msprof-analyze advisor | /profiler/advisor/rules/environment_variable_info.yaml | 公网地址 | https://support.huawei.com/enterprise/zh/doc/EDOC1100371278/5eeeed85?idPath=23710424 | 组网指南 | -- Gitee From 278ef5cd0bf1487ae020afbe2bf19c1bc83b7e87 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Sat, 31 Aug 2024 15:08:44 +0800 Subject: [PATCH 621/791] =?UTF-8?q?=E3=80=90=E4=BC=98=E5=8C=96=E3=80=91?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E5=AE=89=E5=85=A8=E9=97=AE=E9=A2=98=E6=95=B4?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/advisor/advisor_result.py | 4 +- .../msprobe/core/common/file_check.py | 5 +- .../msprobe/core/common/utils.py | 60 +++++++++---------- .../data_processor/pytorch_processor.py | 5 +- .../msprobe/core/data_dump/json_writer.py | 45 ++++---------- .../mindspore/dump/kernel_graph_dump.py | 5 +- .../mindspore/dump/kernel_kbyk_dump.py | 5 +- .../kernel_graph_overflow_check.py | 5 +- .../msprobe/mindspore/service.py | 14 ++--- .../api_accuracy_checker/common/utils.py | 4 +- .../msprobe/pytorch/compare/pt_compare.py | 6 +- .../msprobe/pytorch/parse_tool/lib/compare.py | 4 +- .../pytorch/parse_tool/lib/parse_tool.py | 2 +- .../accuracy_tools/msprobe/pytorch/service.py | 17 ++---- .../msprobe/test/core_ut/common/test_utils.py | 18 ------ .../mindspore_ut/test_kernel_graph_dump.py | 2 +- .../test_kernel_graph_overflow_check.py | 2 +- .../mindspore_ut/test_kernel_kbyk_dump.py | 2 +- .../msprobe/test/pytorch_ut/test_service.py | 6 +- 19 files changed, 76 insertions(+), 135 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py index 2bfea2eb9..d2dbbc843 100644 --- a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py @@ -20,7 +20,7 @@ import time from msprobe.core.advisor.advisor_const import AdvisorConst from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.file_check import change_mode +from msprobe.core.common.file_check import change_mode, FileOpen class AdvisorResult: @@ -38,7 +38,7 @@ class AdvisorResult: file_name = 'advisor_{}.txt'.format(time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) result_file = os.path.join(out_path, file_name) try: - with os.fdopen(os.open(result_file, Const.WRITE_FLAGS, Const.WRITE_MODES), 'w+') as output_file: + with FileOpen(result_file, 'w+') as output_file: output_file.truncate(0) message_list = [message + AdvisorConst.NEW_LINE for message in message_list] output_file.writelines(message_list) diff --git a/debug/accuracy_tools/msprobe/core/common/file_check.py b/debug/accuracy_tools/msprobe/core/common/file_check.py index c567f9454..ee56c68b5 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_check.py +++ b/debug/accuracy_tools/msprobe/core/common/file_check.py @@ -227,18 +227,21 @@ def check_path_type(file_path, file_type): def create_directory(dir_path): """ Function Description: - creating a directory with specified permissions + creating a safe directory with specified permissions Parameter: dir_path: directory path Exception Description: when invalid data throw exception """ dir_path = os.path.realpath(dir_path) + check_path_before_create(dir_path) try: os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) except OSError as ex: raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, 'Failed to create {}. Please check the path permission or disk space .{}'.format(dir_path, str(ex))) from ex + file_check = FileChecker(dir_path, FileCheckConst.DIR) + file_check.common_check() def check_path_before_create(path): diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index b5d387563..cf62e6d8b 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -15,6 +15,7 @@ # limitations under the License. """ import collections +import fcntl import os import re import shutil @@ -23,7 +24,6 @@ import time import json import csv from datetime import datetime, timezone -from pathlib import Path import yaml import numpy as np @@ -75,19 +75,6 @@ class DumpException(CompareException): pass -def make_dump_path_if_not_exists(dump_path): - if not os.path.exists(dump_path): - try: - Path(dump_path).mkdir(mode=0o750, exist_ok=True, parents=True) - except OSError as ex: - logger.error( - 'Failed to create {}.Please check the path permission or disk space .{}'.format(dump_path, str(ex))) - raise CompareException(CompareException.INVALID_PATH_ERROR) from ex - else: - if not os.path.isdir(dump_path): - logger.error('{} already exists and is not a directory.'.format(dump_path)) - - def check_mode_valid(mode, scope=None, api_list=None): if scope is None: scope = [] @@ -294,25 +281,6 @@ def get_dump_data_path(dump_dir): return dump_data_path, file_is_exist -def create_directory(dir_path): - """ - Function Description: - creating a directory with specified permissions - Parameter: - dir_path: directory path - Exception Description: - when invalid data throw exception - """ - if not os.path.exists(dir_path): - check_path_before_create(dir_path) - try: - os.makedirs(dir_path, mode=0o700) - except OSError as ex: - logger.error( - 'Failed to create {}.Please check the path permission or disk space .{}'.format(dir_path, str(ex))) - raise CompareException(CompareException.INVALID_PATH_ERROR) from ex - - def execute_command(cmd): """ Function Description: @@ -614,3 +582,29 @@ def save_workbook(workbook, file_path): logger.error(f'Save result file "{os.path.basename(file_path)}" failed') raise CompareException(CompareException.WRITE_FILE_ERROR) from e change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + + +def load_json(json_path): + try: + with FileOpen(json_path, "r") as f: + fcntl.flock(f, fcntl.LOCK_EX) + data = json.load(f) + fcntl.flock(f, fcntl.LOCK_UN) + except Exception as e: + logger.error(f'load json file "{os.path.basename(json_path)}" failed.') + raise DumpException(DumpException.WRITE_FILE_ERROR) from e + return data + + +def save_json(json_path, data, indent=None): + json_path = os.path.realpath(json_path) + check_path_before_create(json_path) + try: + with FileOpen(json_path, 'w') as f: + fcntl.flock(f, fcntl.LOCK_EX) + json.dump(data, f, indent=indent) + fcntl.flock(f, fcntl.LOCK_UN) + except Exception as e: + logger.error(f'Save json file "{os.path.basename(json_path)}" failed.') + raise DumpException(DumpException.WRITE_FILE_ERROR) from e + change_mode(json_path, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 133038255..5900fe54e 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -12,7 +12,7 @@ from msprobe.core.common.const import Const, OverflowConst, FileCheckConst from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, ModuleBackwardInputsOutputs, \ ModuleForwardInputsOutputs, TensorStatInfo from msprobe.pytorch.free_benchmark import FreeBenchmarkCheck, UnequalRow -from msprobe.pytorch.common.utils import save_pt +from msprobe.pytorch.common.utils import save_pt, load_pt try: import torch_npu @@ -361,7 +361,8 @@ class KernelDumpDataProcessor(PytorchDataProcessor): if not KernelDumpDataProcessor.forward_init_status: KernelDumpDataProcessor.forward_init_status = True output = module.forward(*module_input_output.args, **module_input_output.kwargs) - grad = torch.load(grad_path).to("npu").requires_grad_() + pt = load_pt(grad_path) + grad = pt.to("npu").requires_grad_() torch_npu.npu.init_dump() torch_npu.npu.set_dump(self.config.acl_config) torch_npu.npu.synchronize() diff --git a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py index fc560b404..5d802260c 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py @@ -1,13 +1,10 @@ import os import csv -import fcntl -import json -from pathlib import Path from msprobe.core.common.file_check import change_mode, FileOpen from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.utils import remove_path +from msprobe.core.common.utils import load_json, save_json, remove_path class DataWriter: @@ -42,19 +39,14 @@ class DataWriter: def initialize_json_file(self, **kwargs): kwargs.update({"dump_data_dir": self.dump_tensor_data_dir, Const.DATA: {}}) - with FileOpen(self.dump_file_path, 'w') as f: - json.dump(kwargs, f) - change_mode(self.dump_file_path, FileCheckConst.DATA_FILE_AUTHORITY) + save_json(self.dump_file_path, kwargs) - if os.path.exists(self.stack_file_path): - remove_path(self.stack_file_path) - Path(self.stack_file_path).touch() - change_mode(self.stack_file_path, FileCheckConst.DATA_FILE_AUTHORITY) + empty_dict = {} + remove_path(self.stack_file_path) + save_json(self.stack_file_path, empty_dict) - if os.path.exists(self.construct_file_path): - remove_path(self.construct_file_path) - Path(self.construct_file_path).touch() - change_mode(self.construct_file_path, FileCheckConst.DATA_FILE_AUTHORITY) + remove_path(self.construct_file_path) + save_json(self.construct_file_path, empty_dict) def update_dump_paths(self, dump_file_path, stack_file_path, construct_file_path, dump_data_dir, free_benchmark_file_path): @@ -83,33 +75,20 @@ class DataWriter: def write_data_json(self, file_path): logger.info(f"dump.json is at {os.path.dirname(os.path.dirname(file_path))}. ") - if Path(file_path).exists() and os.path.getsize(file_path) > 0: - with FileOpen(file_path, "r+") as f: - fcntl.flock(f, fcntl.LOCK_EX) - data_to_write = json.load(f) - fcntl.flock(f, fcntl.LOCK_UN) + if os.path.exists(file_path) and os.path.getsize(file_path) > 0: + data_to_write = load_json(file_path) else: self.init_json['data_path'] = self.dump_tensor_data_dir data_to_write = self.init_json data_to_write[Const.DATA].update(self.cache_data[Const.DATA]) - with FileOpen(file_path, 'w+') as f: - fcntl.flock(f, fcntl.LOCK_EX) - json.dump(data_to_write, f, indent=1) - fcntl.flock(f, fcntl.LOCK_UN) - + save_json(file_path, data_to_write, indent=1) self.cache_data[Const.DATA].clear() def write_stack_info_json(self, file_path): - with FileOpen(file_path, 'w+') as f: - fcntl.flock(f, fcntl.LOCK_EX) - json.dump(self.cache_stack, f, indent=1) - fcntl.flock(f, fcntl.LOCK_UN) + save_json(file_path, self.cache_stack, indent=1) def write_construct_info_json(self, file_path): - with FileOpen(file_path, 'w+') as f: - fcntl.flock(f, fcntl.LOCK_EX) - json.dump(self.cache_construct, f, indent=1) - fcntl.flock(f, fcntl.LOCK_UN) + save_json(file_path, self.cache_construct, indent=1) def write_json(self): self.write_data_json(self.dump_file_path) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py index 8320ee090..7db2b8142 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py @@ -1,9 +1,8 @@ import os import json -from msprobe.core.common.utils import make_dump_path_if_not_exists from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_check import FileOpen, create_directory class KernelGraphDump: @@ -45,7 +44,7 @@ class KernelGraphDump: if os.getenv("GRAPH_OP_RUN") == "1": raise Exception("Must run in graph mode, not kbk mode") json_path = self.dump_json["common_dump_settings"]["path"] - make_dump_path_if_not_exists(json_path) + create_directory(json_path) json_path = os.path.join(json_path, "kernel_graph_dump.json") with FileOpen(json_path, 'w') as f: json.dump(self.dump_json, f) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py index 54e57c508..cea9037f3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py @@ -1,10 +1,9 @@ import os import json -from msprobe.core.common.utils import make_dump_path_if_not_exists from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_check import FileOpen, create_directory from msprobe.core.common.const import Const @@ -54,7 +53,7 @@ class KernelKbykDump: def handle(self): json_path = self.dump_json[KernelKbykDump.COMMON_SETTINGS]["path"] - make_dump_path_if_not_exists(json_path) + create_directory(json_path) json_path = os.path.join(json_path, "kernel_kbyk_dump.json") with FileOpen(json_path, 'w') as f: json.dump(self.dump_json, f) diff --git a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py index 664060873..ae5697586 100644 --- a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py @@ -1,9 +1,8 @@ import os import json -from msprobe.core.common.utils import make_dump_path_if_not_exists from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_check import FileOpen, create_directory class KernelGraphOverflowCheck: @@ -35,7 +34,7 @@ class KernelGraphOverflowCheck: if os.getenv("GRAPH_OP_RUN") == "1": raise Exception("Must run in graph mode, not kbk mode") json_path = self.dump_json["common_dump_settings"]["path"] - make_dump_path_if_not_exists(json_path) + create_directory(json_path) json_path = os.path.join(json_path, "kernel_graph_overflow_check.json") with FileOpen(json_path, 'w') as f: json.dump(self.dump_json, f) diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 016c68782..40620cb4a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -15,7 +15,6 @@ import os import copy -from pathlib import Path import functools from collections import defaultdict @@ -33,7 +32,7 @@ except ImportError: from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope from msprobe.mindspore.common.utils import get_rank_if_initialized -from msprobe.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create +from msprobe.core.common.file_check import create_directory from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import Const from msprobe.core.common.exceptions import DistributedNotInitializedError @@ -299,19 +298,14 @@ class Service: self.data_collector.write_json() def create_dirs(self): - check_path_before_create(self.config.dump_path) - if not os.path.exists(self.config.dump_path): - Path(self.config.dump_path).mkdir(mode=0o750, exist_ok=True) - file_check = FileChecker(self.config.dump_path, FileCheckConst.DIR) - file_check.common_check() + create_directory(self.config.dump_path) self.dump_iter_dir = os.path.join(self.config.dump_path, f"step{self.current_iter}") cur_rank = self.current_rank if self.current_rank is not None else '' dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}") - if not os.path.exists(dump_dir): - Path(dump_dir).mkdir(mode=0o750, parents=True, exist_ok=True) + create_directory(dump_dir) if self.config.task in self.data_collector.tasks_need_tensor_data: dump_data_dir = os.path.join(dump_dir, "dump_tensor_data") - Path(dump_data_dir).mkdir(mode=0o750, exist_ok=True) + create_directory(dump_data_dir) else: dump_data_dir = None diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 467c1f045..d9e46bbe9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -31,7 +31,7 @@ from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.utils import save_pt from msprobe.core.common.file_check import create_directory from msprobe.core.common.const import Const -from msprobe.core.common.utils import CompareException, make_dump_path_if_not_exists +from msprobe.core.common.utils import CompareException ApiData = namedtuple('ApiData', ['name', 'args', 'kwargs', 'result', 'step', 'rank'], defaults=['unknown', None, None, None, 0, 0]) @@ -105,7 +105,7 @@ def cross_entropy_process(api_info_dict): def initialize_save_path(save_path, dir_name): data_path = os.path.join(save_path, dir_name) - make_dump_path_if_not_exists(data_path) + create_directory(data_path) return data_path diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 788d8d6ff..188b62e18 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,11 +1,11 @@ import os.path import torch from msprobe.core.common.const import FileCheckConst, Const +from msprobe.core.common.file_check import create_directory from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.common.utils import create_directory, check_configuration_param, task_dumppath_get, \ - check_compare_param, FileChecker +from msprobe.core.compare.acc_compare import Comparator +from msprobe.core.common.utils import check_configuration_param, task_dumppath_get, check_compare_param, FileChecker from msprobe.core.common.utils import CompareException diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py index cf97f9c6f..add1dea38 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py @@ -22,8 +22,8 @@ from collections import namedtuple from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException -from msprobe.core.common.utils import create_directory, write_csv, save_npy_to_txt, load_npy -from msprobe.core.common.file_check import FileChecker +from msprobe.core.common.utils import write_csv, save_npy_to_txt, load_npy +from msprobe.core.common.file_check import FileChecker, create_directory class Compare: diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py index 67c981412..879329ae5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py @@ -23,7 +23,7 @@ from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.compare import Compare from msprobe.pytorch.parse_tool.lib.visualization import Visualization from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception, ParseException -from msprobe.core.common.utils import create_directory +from msprobe.core.common.file_check import create_directory class ParseTool: def __init__(self): diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 350ae199f..22a17f36f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -1,13 +1,11 @@ import functools import os -import time -from pathlib import Path from collections import namedtuple import torch -from msprobe.core.common.const import Const, FileCheckConst +from msprobe.core.common.const import Const from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException -from msprobe.core.common.file_check import FileChecker, check_path_before_create +from msprobe.core.common.file_check import create_directory from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs from msprobe.core.data_dump.scope import BaseScope @@ -159,19 +157,14 @@ class Service: self.data_collector.write_json() def create_dirs(self): - check_path_before_create(self.config.dump_path) - if not os.path.exists(self.config.dump_path): - Path(self.config.dump_path).mkdir(mode=0o750, exist_ok=True) - file_check = FileChecker(self.config.dump_path, FileCheckConst.DIR) - file_check.common_check() + create_directory(self.config.dump_path) self.dump_iter_dir = os.path.join(self.config.dump_path, f"step{self.current_iter}") cur_rank = self.current_rank if self.current_rank is not None else '' dump_dir = os.path.join(self.dump_iter_dir, f"rank{cur_rank}") - if not os.path.exists(dump_dir): - Path(dump_dir).mkdir(mode=0o750, parents=True, exist_ok=True) + create_directory(dump_dir) if self.config.task in self.data_collector.tasks_need_tensor_data: dump_data_dir = os.path.join(dump_dir, "dump_tensor_data") - Path(dump_data_dir).mkdir(mode=0o750, exist_ok=True) + create_directory(dump_data_dir) else: dump_data_dir = None diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 5151d6f26..c923b9179 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -26,7 +26,6 @@ from msprobe.core.common.const import Const from msprobe.core.common.utils import (CompareException, check_seed_all, check_inplace_op, - make_dump_path_if_not_exists, check_mode_valid, check_switch_valid, check_dump_mode_valid, @@ -81,23 +80,6 @@ class TestUtils(TestCase): test_prefix_3 = "Torch.sum.0.backward.output.0" self.assertFalse(check_inplace_op(test_prefix_3)) - @patch.object(logger, "error") - def test_make_dump_path_if_not_exists(self, mock_error): - file_path = os.path.realpath(__file__) - dirname = os.path.dirname(file_path) + str(uuid.uuid4()) - - def test_mkdir(self, **kwargs): - raise OSError - - if not os.path.exists(dirname): - with patch("msprobe.core.common.utils.Path.mkdir", new=test_mkdir): - with self.assertRaises(CompareException) as context: - make_dump_path_if_not_exists(dirname) - self.assertEqual(context.exception.code, CompareException.INVALID_PATH_ERROR) - - make_dump_path_if_not_exists(file_path) - mock_error.assert_called_with(f"{file_path} already exists and is not a directory.") - def test_check_mode_valid(self): with self.assertRaises(ValueError) as context: check_mode_valid("all", scope="scope") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py index 1d308c18d..8bbdcd1b4 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py @@ -45,7 +45,7 @@ class TestKernelGraphDump(TestCase): self.assertEqual(dumper.dump_json["common_dump_settings"]["file_format"], "bin") self.assertEqual(dumper.dump_json["common_dump_settings"]["input_output"], 2) - with patch("msprobe.mindspore.dump.kernel_graph_dump.make_dump_path_if_not_exists"), \ + with patch("msprobe.mindspore.dump.kernel_graph_dump.create_directory"), \ patch("msprobe.mindspore.dump.kernel_graph_dump.FileOpen"), \ patch("msprobe.mindspore.dump.kernel_graph_dump.json.dump"), \ patch("msprobe.mindspore.dump.kernel_graph_dump.logger.info"): diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py index 71e78cc79..91bc79fff 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py @@ -43,7 +43,7 @@ class TestKernelGraphOverflowCheck(TestCase): self.assertEqual(checker.dump_json["common_dump_settings"]["op_debug_mode"], 2) os.environ["MS_ACL_DUMP_CFG_PATH"] = "path" - with patch("msprobe.mindspore.overflow_check.kernel_graph_overflow_check.make_dump_path_if_not_exists"), \ + with patch("msprobe.mindspore.overflow_check.kernel_graph_overflow_check.create_directory"), \ patch("msprobe.mindspore.overflow_check.kernel_graph_overflow_check.FileOpen"), \ patch("msprobe.mindspore.overflow_check.kernel_graph_overflow_check.json.dump"), \ patch("msprobe.mindspore.overflow_check.kernel_graph_overflow_check.logger.info"): diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py index d680032b8..4ce6bf397 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py @@ -42,7 +42,7 @@ class TestKernelKbykDump(TestCase): self.assertEqual(dumper.dump_json["common_dump_settings"]["iteration"], "0|2") os.environ["MS_ACL_DUMP_CFG_PATH"] = "path" - with patch("msprobe.mindspore.dump.kernel_kbyk_dump.make_dump_path_if_not_exists"), \ + with patch("msprobe.mindspore.dump.kernel_kbyk_dump.create_directory"), \ patch("msprobe.mindspore.dump.kernel_kbyk_dump.FileOpen"), \ patch("msprobe.mindspore.dump.kernel_kbyk_dump.json.dump"), \ patch("msprobe.mindspore.dump.kernel_kbyk_dump.logger.info") as mock_info: diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_service.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_service.py index c09b6abcb..8af40be6d 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_service.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_service.py @@ -51,9 +51,7 @@ class TestService(unittest.TestCase): self.assertEqual(mock_logger.call_count, 2) def test_create_dirs(self): - with patch("msprobe.pytorch.service.Path.mkdir", return_value=None), \ - patch("msprobe.core.common.file_check.FileChecker.common_check", return_value=None), \ - patch("msprobe.core.data_dump.data_collector.DataCollector.update_dump_paths", - return_value=None): + with patch("msprobe.pytorch.service.create_directory"), \ + patch("msprobe.core.data_dump.data_collector.DataCollector.update_dump_paths"): self.service.create_dirs() self.assertEqual(self.service.dump_iter_dir, "./ut_dump/step0") -- Gitee From 38d14dc5b297b59d8ebd65e8dbf0922e545b21a9 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Mon, 2 Sep 2024 15:23:11 +0800 Subject: [PATCH 622/791] =?UTF-8?q?=E5=88=A0=E9=99=A4utils.py=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E9=87=8D=E5=A4=8D=E6=96=B9=E6=B3=95check=5Fpath=5Fbef?= =?UTF-8?q?ore=5Fcreate?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/utils.py | 13 +------------ .../msprobe/core/grad_probe/grad_compare.py | 4 ++-- .../msprobe/mindspore/grad_probe/global_context.py | 3 +-- .../msprobe/mindspore/grad_probe/utils.py | 4 ++-- .../accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++-- .../msprobe/pytorch/online_dispatch/dispatch.py | 3 ++- 6 files changed, 10 insertions(+), 21 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index b5d387563..ffe2de547 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -27,7 +27,7 @@ from pathlib import Path import yaml import numpy as np -from msprobe.core.common.file_check import FileOpen, FileChecker, change_mode +from msprobe.core.common.file_check import FileOpen, FileChecker, change_mode, check_path_before_create from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.core.common.log import logger @@ -447,17 +447,6 @@ def check_file_valid(file_path): raise CompareException(CompareException.INVALID_PATH_ERROR) -def check_path_before_create(path): - if len(os.path.realpath(path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(path)) > \ - Const.FILE_NAME_LENGTH: - logger.error('The file path length exceeds limit.') - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not re.match(Const.FILE_PATTERN, os.path.realpath(path)): - logger.error('The file path {} contains special characters.'.format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - def check_inplace_op(prefix): if len(prefix) > Const.DISTRIBUTED_PREFIX_LENGTH: return False diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 22acdf2fb..6323787ba 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -5,8 +5,8 @@ from tqdm import tqdm import pandas as pd import matplotlib.pyplot as plt -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.utils import check_file_or_directory_path +from msprobe.core.common.file_check import create_directory, check_path_before_create from msprobe.core.common.log import logger from msprobe.core.common.utils import remove_path, write_csv, load_npy from msprobe.core.grad_probe.constant import GradConst diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py index 16d0bd0b8..1a5055a1e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -5,8 +5,7 @@ from typing import Dict, Union from msprobe.core.grad_probe.utils import check_str from msprobe.core.grad_probe.constant import GradConst from msprobe.core.common.log import logger -from msprobe.core.common.file_check import create_directory -from msprobe.core.common.utils import check_path_before_create +from msprobe.core.common.file_check import create_directory, check_path_before_create class GlobalContext: diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py index db0a36a02..bef542f37 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py @@ -4,8 +4,8 @@ import numpy as np import mindspore from msprobe.core.grad_probe.constant import GradConst, level_adp from msprobe.core.grad_probe.utils import check_param -from msprobe.core.common.file_check import create_directory -from msprobe.core.common.utils import check_path_before_create, change_mode, check_file_or_directory_path, save_npy +from msprobe.core.common.file_check import create_directory, check_path_before_create +from msprobe.core.common.utils import change_mode, check_file_or_directory_path, save_npy def save_grad_direction(param_name, grad, save_path): diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index cb44c6bbe..98d623e4a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -27,8 +27,8 @@ import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.log import logger as common_logger -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, CompareException -from msprobe.core.common.file_check import FileCheckConst, change_mode, FileOpen +from msprobe.core.common.utils import check_file_or_directory_path, CompareException +from msprobe.core.common.file_check import FileCheckConst, change_mode, FileOpen, check_path_before_create try: diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index f63460950..f4686f9d8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -15,7 +15,8 @@ except ImportError: else: is_npu = True -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, load_yaml +from msprobe.core.common.utils import check_file_or_directory_path, load_yaml +from msprobe.core.common.file_check import check_path_before_create from msprobe.core.common.const import Const, CompareConst from msprobe.pytorch.common.log import logger from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ -- Gitee From 91d42413770542cd851ee165c8a7da21f3f61534 Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Mon, 2 Sep 2024 10:40:54 +0800 Subject: [PATCH 623/791] =?UTF-8?q?=E5=B0=81=E8=A3=85savefig?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/grad_probe/grad_compare.py | 7 ++----- .../msprobe/core/grad_probe/utils.py | 14 +++++++++++++- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 22acdf2fb..a0ce453b0 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -10,6 +10,7 @@ from msprobe.core.common.file_check import create_directory from msprobe.core.common.log import logger from msprobe.core.common.utils import remove_path, write_csv, load_npy from msprobe.core.grad_probe.constant import GradConst +from msprobe.core.grad_probe.utils import plt_savefig class GradComparator: @@ -90,11 +91,7 @@ class GradComparator: create_directory(picture_dir) fig_save_path = os.path.join(picture_dir, f"{key}_similarities.png") - check_path_before_create(fig_save_path) - try: - plt.savefig(fig_save_path) - except Exception as e: - raise RuntimeError(f"save plt figure {fig_save_path} failed") from e + plt_savefig(fig_save_path) plt.close() result.append([key] + value) diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py index f5db74baa..466d5282a 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py @@ -1,7 +1,10 @@ import re from msprobe.core.grad_probe.constant import GradConst from msprobe.core.common.log import logger -from msprobe.core.common.utils import write_csv +from msprobe.core.common.utils import write_csv, check_path_before_create, change_mode +from msprobe.core.common.const import FileCheckConst +import matplotlib.pyplot as plt + def data_in_list_target(data, lst): return not lst or len(lst) == 0 or data in lst @@ -50,3 +53,12 @@ class ListCache(list): def set_output_file(self, output_file): self._output_file = output_file + + +def plt_savefig(fig_save_path): + check_path_before_create(fig_save_path) + try: + plt.savefig(fig_save_path) + except Exception as e: + raise RuntimeError(f"save plt figure {fig_save_path} failed") from e + change_mode(fig_save_path, FileCheckConst.DATA_FILE_AUTHORITY) -- Gitee From 2dca3dbd72680e3e8a55361689ab87fc832b83ce Mon Sep 17 00:00:00 2001 From: jijiarong Date: Mon, 2 Sep 2024 10:03:48 +0800 Subject: [PATCH 624/791] ms graph compare bugfix --- .../mindspore/compare/ms_graph_compare.py | 240 +++++++++++------- 1 file changed, 142 insertions(+), 98 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index bb83c0fb8..8c449ad58 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -1,19 +1,17 @@ +import copy import csv import glob import os -import sys -import copy import numpy as np import pandas as pd from msprobe.core.common.const import CompareConst, GraphMode -from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_check import FileOpen from msprobe.core.common.log import logger -from msprobe.core.common.utils import add_time_with_xlsx, CompareException +from msprobe.core.common.utils import add_time_with_xlsx, CompareException, load_npy from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply -from msprobe.core.common.file_check import FileOpen + class row_data: def __init__(self, mode): @@ -29,57 +27,72 @@ class row_data: return self.data -def generate_step(npu_path, rank_id): - step_set = set() - rank_path = os.path.join(npu_path, f"rank_{rank_id}") - if not os.path.exists(rank_path): - return [] - for path in os.listdir(rank_path): - if path not in ["execution_order", "graphs"]: - data_path = os.path.join(rank_path, path) - for graph_path in os.listdir(data_path): - step_set.update([int(i) for i in os.listdir(os.path.join(data_path, graph_path))]) - return sorted(step_set) +def list_lowest_level_directories(root_dir): + lowest_level_dirs = [] + + def recurse_dirs(current_dir): + for entry in os.listdir(current_dir): + full_path = os.path.join(current_dir, entry) + if os.path.isdir(full_path): + if any(os.path.isdir(os.path.join(full_path, subentry)) for subentry in os.listdir(full_path)): + recurse_dirs(full_path) + else: + lowest_level_dirs.append(full_path) + recurse_dirs(root_dir) + return lowest_level_dirs -def generate_path_by_rank_step(base_path, rank_id, step_id): - path_with_rank_id = os.path.join(base_path, f"rank_{rank_id}") - if not os.path.exists(path_with_rank_id): - return '' - for path in os.listdir(path_with_rank_id): - if path not in ["execution_order", "graphs"]: - return os.path.join(path_with_rank_id, path, "*", str(step_id)) - logger.error(f"Data_path {path_with_rank_id} is not exist.") - return '' +def generate_id(value): + try: + return int(value) + except ValueError: + return -1 + + +def npy_data_read(data_path, npy_file_list, mapping_dict): + data_list = [] + for data in npy_file_list: + if data in mapping_dict: + split_list = mapping_dict[data].split(".") + else: + split_list = data.split(".") + compare_key = f"{split_list[1]}.{split_list[2]}.{split_list[3]}.{split_list[5]}.{split_list[6]}" + timestamp = int(split_list[4]) + + data_list.append([os.path.join(data_path, data), compare_key, timestamp]) + return data_list def statistic_data_read(statistic_file_list, statistic_file_path): data_list = [] statistic_data_list = [] + header_index = {'Data Type': None, 'Shape': None, 'Max Value': None, 'Min Value': None, + 'Avg Value': None, 'L2Norm Value': None} for statistic_file in statistic_file_list: - with open(statistic_file, "r") as f: + with FileOpen(statistic_file, "r") as f: csv_reader = csv.reader(f, delimiter=",") header = next(csv_reader) - header_index = {'Data Type': None, 'Shape': None, 'Max Value': None, 'Min Value': None, - 'Avg Value': None, 'L2Norm Value': None} for key in header_index.keys(): for index, value in enumerate(header): if key == value: header_index[key] = index - for key in header_index.keys(): - if header_index[key] is None: - logger.error(f"Data_path {statistic_file_path} has no key {key}") - raise FileCheckException(f"Data_path {statistic_file_path} has no key {key}") statistic_data_list.extend([row for row in csv_reader]) + for key in header_index.keys(): + if header_index[key] is None: + logger.warning(f"Data_path {statistic_file_path} has no key {key}.") + for data in statistic_data_list: compare_key = f"{data[1]}.{data[2]}.{data[3]}.{data[5]}" timestamp = int(data[4]) - data_list.append( - [statistic_file_path, compare_key, timestamp, data[header_index['Data Type']], - data[header_index['Shape']], data[header_index['Max Value']], data[header_index['Min Value']], - data[header_index['Avg Value']], data[header_index['L2Norm Value']]]) + result_data = [statistic_file_path, compare_key, timestamp] + for key in header_index.keys(): + if header_index[key] is None: + result_data.append(np.nan) + else: + result_data.append(data[header_index[key]]) + data_list.append(result_data) return data_list @@ -108,15 +121,8 @@ def generate_data_name(data_path): mapping_dict[row[0]] = row[1] if npy_exist: - for data in npy_file_list: - if data in mapping_dict: - split_list = mapping_dict[data].split(".") - else: - split_list = data.split(".") - compare_key = f"{split_list[1]}.{split_list[2]}.{split_list[3]}.{split_list[5]}.{split_list[6]}" - timestamp = int(split_list[4]) + data_list = npy_data_read(data_path, npy_file_list, mapping_dict) - data_list.append([os.path.join(data_path, data), compare_key, timestamp]) elif statistic_exist: data_list = statistic_data_read(statistic_file_list, os.path.join(data_path, statistic_path)) @@ -130,25 +136,50 @@ def generate_data_name(data_path): return mode, data_list -def read_npy_data(data_path): - try: - data_value = np.load(data_path) - if data_value.dtype == np.float16: - data_value = data_value.astype(np.float32) - except FileNotFoundError as e: - data_value = None - except EOFError: - data_value = None - return data_value - - class GraphMSComparator: def __init__(self, input_param, output_path): self.output_path = output_path self.base_npu_path = input_param.get('npu_path', None) self.base_bench_path = input_param.get('bench_path', None) - self.rank_list = input_param.get('rank_id', []) - self.step_list = input_param.get('step_id', []) + self.rank_list = [generate_id(rank_id) for rank_id in input_param.get('rank_id', [])] + self.step_list = [generate_id(step_id) for step_id in input_param.get('step_id', [])] + # split by rank and step, generate rank step path + self.npu_rank_step_dict = self.generate_rank_step_path(self.base_npu_path) + self.bench_rank_step_dict = self.generate_rank_step_path(self.base_bench_path) + self.common_rank_step = sorted( + set(self.npu_rank_step_dict.keys()).intersection(self.bench_rank_step_dict.keys())) + + def generate_rank_step_path(self, base_path): + + def generate_rank_step_id(path_with_rank_step): + split_path = path_with_rank_step.split("/") + if "rank_" in path_with_rank_step: + # KBK mode + rank_id = generate_id(split_path[-4].split("_")[-1]) + step_id = generate_id(split_path[-1]) + else: + rank_id = generate_id(split_path[-4]) + step_id = generate_id(split_path[-1]) + return rank_id, step_id + + base_path = os.path.abspath(base_path) + lowest_level = list_lowest_level_directories(base_path) + + rank_step_path_dict = {} + for dir_path in lowest_level: + rank_id, step_id = generate_rank_step_id(dir_path) + if rank_id == -1 or step_id == -1: + continue + if self.rank_list and rank_id not in self.rank_list: + continue + if self.step_list and step_id not in self.step_list: + continue + rank_step_key = (rank_id, step_id) + if rank_step_key in rank_step_path_dict: + rank_step_path_dict[rank_step_key].append(dir_path) + else: + rank_step_path_dict[rank_step_key] = [dir_path] + return dict(sorted(rank_step_path_dict.items())) @staticmethod def compare_ops(compare_result_db, mode): @@ -158,7 +189,7 @@ class GraphMSComparator: def process_npy_file(file_path, name_prefix, result): if os.path.exists(file_path): - data = read_npy_data(file_path) + data = load_npy(file_path) result[f'{name_prefix} Name'] = file_path result[f'{name_prefix} Dtype'] = data.dtype result[f'{name_prefix} Tensor Shape'] = data.shape @@ -229,7 +260,7 @@ class GraphMSComparator: result_dict[CompareConst.NORM_RELATIVE_ERR] = str( result_dict[CompareConst.NORM_RELATIVE_ERR] * 100) + "%" magnitude_diff = result_dict[CompareConst.MAX_DIFF] / ( - max(result_dict[CompareConst.NPU_MAX], result_dict[CompareConst.BENCH_MAX]) + 1e-10) + max(result_dict[CompareConst.NPU_MAX], result_dict[CompareConst.BENCH_MAX]) + 1e-10) if magnitude_diff > CompareConst.MAGNITUDE: result_dict[CompareConst.ACCURACY] = 'No' else: @@ -246,58 +277,71 @@ class GraphMSComparator: def compare_core(self): logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") - # split by rank and step - if not self.rank_list: - self.rank_list = [int(i.split("_")[-1]) for i in os.listdir(self.base_npu_path)] - for rank_id in self.rank_list: - if not self.step_list: - self.step_list = generate_step(self.base_npu_path, rank_id) - for step_id in self.step_list: - compare_result_df, mode = self.compare_process(rank_id, step_id) - if isinstance(compare_result_df, list): - is_empty = not compare_result_df - elif isinstance(compare_result_df, pd.DataFrame): - is_empty = compare_result_df.empty - else: - is_empty = True - if is_empty or not mode: - continue - compare_result_df = self._do_multi_process(compare_result_df, mode) - compare_result_name = add_time_with_xlsx(f"compare_result_{str(rank_id)}_{str(step_id)}") - compare_result_path = os.path.join(os.path.realpath(self.output_path), f"{compare_result_name}") - compare_result_df.to_excel(compare_result_path, index=False) - logger.info(f"Compare rank: {rank_id} step: {step_id} finish. Compare result: {compare_result_path}.") + for rank_id, step_id in self.common_rank_step: + compare_result_df, mode = self.compare_process(rank_id, step_id) + if isinstance(compare_result_df, list): + is_empty = not compare_result_df + elif isinstance(compare_result_df, pd.DataFrame): + is_empty = compare_result_df.empty + else: + is_empty = True + if is_empty or not mode: + continue + compare_result_df = self._do_multi_process(compare_result_df, mode) + compare_result_name = add_time_with_xlsx(f"compare_result_{str(rank_id)}_{str(step_id)}") + compare_result_path = os.path.join(os.path.realpath(self.output_path), f"{compare_result_name}") + compare_result_df.to_excel(compare_result_path, index=False) + logger.info(f"Compare rank: {rank_id} step: {step_id} finish. Compare result: {compare_result_path}.") def compare_process(self, rank_id, step_id): # generate data_path - npu_data_path = generate_path_by_rank_step(self.base_npu_path, rank_id, step_id) - bench_data_path = generate_path_by_rank_step(self.base_bench_path, rank_id, step_id) - if not npu_data_path or not bench_data_path: + npu_data_path_list = self.npu_rank_step_dict.get((rank_id, step_id)) + bench_data_path_list = self.bench_rank_step_dict.get((rank_id, step_id)) + if not npu_data_path_list or not npu_data_path_list: return [], '' # generate file name - npu_mode, npu_data_list = generate_data_name(npu_data_path) - match_mode, match_data_list = generate_data_name(bench_data_path) - - if npu_mode == "ERROR_MODE" or match_mode == "ERROR_MODE": + npu_mode = 'ERROR_MODE' + bench_mode = 'ERROR_MODE' + npu_data_list = [] + bench_data_list = [] + for npu_data_path in npu_data_path_list: + npu_mode, data_list = generate_data_name(npu_data_path) + npu_data_list.extend(data_list) + for bench_data_path in bench_data_path_list: + bench_mode, data_list = generate_data_name(bench_data_path) + bench_data_list.extend(data_list) + + if npu_mode == "ERROR_MODE" or bench_mode == "ERROR_MODE": logger.warning(f"Data_path {npu_data_path} or {bench_data_path} is not exist.") return [], '' - if npu_mode != match_mode: - logger.error(f"NPU mode {npu_mode} not equal to MATCH mode {match_mode}.") + if npu_mode != bench_mode: + logger.error(f"NPU mode {npu_mode} not equal to MATCH mode {bench_mode}.") return [], '' if npu_mode == 'NPY_MODE': npu_data_df = pd.DataFrame(npu_data_list, columns=[CompareConst.NPU_NAME, 'Compare Key', 'TimeStamp']) - bench_data_df = pd.DataFrame(match_data_list, columns=[CompareConst.BENCH_NAME, 'Compare Key', 'TimeStamp']) + bench_data_df = pd.DataFrame(bench_data_list, columns=[CompareConst.BENCH_NAME, 'Compare Key', 'TimeStamp']) else: npu_data_df = pd.DataFrame(npu_data_list, - columns=[CompareConst.NPU_NAME, 'Compare Key', 'TimeStamp', CompareConst.NPU_DTYPE, CompareConst.NPU_SHAPE, - CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM]) - bench_data_df = pd.DataFrame(match_data_list, - columns=[CompareConst.BENCH_NAME, 'Compare Key', 'TimeStamp', CompareConst.BENCH_DTYPE, - CompareConst.BENCH_SHAPE, CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN, + columns=[CompareConst.NPU_NAME, 'Compare Key', 'TimeStamp', + CompareConst.NPU_DTYPE, CompareConst.NPU_SHAPE, + CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, + CompareConst.NPU_NORM]) + bench_data_df = pd.DataFrame(bench_data_list, + columns=[CompareConst.BENCH_NAME, 'Compare Key', 'TimeStamp', + CompareConst.BENCH_DTYPE, + CompareConst.BENCH_SHAPE, CompareConst.BENCH_MAX, + CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN, CompareConst.BENCH_NORM]) + npu_float_type = [CompareConst.NPU_MAX, CompareConst.NPU_MIN, CompareConst.NPU_MEAN, CompareConst.NPU_NORM] + npu_data_df[npu_float_type] = npu_data_df[npu_float_type].astype(np.float32) + + bench_float_type = [CompareConst.BENCH_MAX, CompareConst.BENCH_MIN, CompareConst.BENCH_MEAN, + CompareConst.BENCH_NORM] + bench_data_df[bench_float_type] = bench_data_df[bench_float_type].astype(np.float32) + npu_data_df['Local Index'] = npu_data_df.sort_values('TimeStamp').groupby('Compare Key').cumcount() bench_data_df['Local Index'] = bench_data_df.sort_values('TimeStamp').groupby('Compare Key').cumcount() @@ -305,7 +349,7 @@ class GraphMSComparator: compare_result_df[CompareConst.NPU_NAME] = compare_result_df[CompareConst.NPU_NAME].fillna('') compare_result_df[CompareConst.BENCH_NAME] = compare_result_df[CompareConst.BENCH_NAME].fillna('') - + return compare_result_df, npu_mode def _do_multi_process(self, result_df, mode): -- Gitee From 7318fc67e507975e7c239711d6f473deebac6bf5 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 2 Sep 2024 16:13:44 +0800 Subject: [PATCH 625/791] safety problem bugfix --- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++-- .../msprobe/pytorch/compare/pt_compare.py | 10 +++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index aeb3a2454..cb44c6bbe 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -270,8 +270,8 @@ def load_pt(pt_path, to_cpu=False): pt = torch.load(pt_path, map_location=torch.device("cpu")) else: pt = torch.load(pt_path) - except Exception: - logger.error("load pt file %s failed", pt_path) + except Exception as e: + raise RuntimeError(f"load pt file {pt_path} failed") from e return pt diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 1eb364e7b..8f07856bb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -19,7 +19,15 @@ class PTComparator (Comparator): path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.PT_SUFFIX, False) data_path = path_checker.common_check() - data_value = load_pt(data_path, to_cpu=True).detach() # detach() because numpy can not process gradient information + try: + data_value = load_pt(data_path, + to_cpu=True).detach() # detach because numpy can not process gradient information + except RuntimeError as e: + # 这里捕获 load_pt 中抛出的异常 + raise RuntimeError(f"Failed to load the .pt file at {data_path}.") from e + except AttributeError as e: + # 这里捕获 detach 方法抛出的异常 + raise RuntimeError(f"Failed to detach the loaded tensor.") from e if data_value.dtype == torch.bfloat16: data_value = data_value.to(torch.float32) data_value = data_value.numpy() -- Gitee From 1fbc0139ad32638a2a7e6e39a8e10bb9ba0f1a74 Mon Sep 17 00:00:00 2001 From: hhqx Date: Sat, 31 Aug 2024 16:32:00 +0800 Subject: [PATCH 626/791] Add check_path_writable to remove_path_safety() function --- profiler/cluster_analyse/common_func/path_manager.py | 1 + profiler/prof_common/path_manager.py | 1 + .../test/ut/cluster_analyse/common_func/test_path_manager.py | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/common_func/path_manager.py b/profiler/cluster_analyse/common_func/path_manager.py index 7ef7b4c34..c57092c46 100644 --- a/profiler/cluster_analyse/common_func/path_manager.py +++ b/profiler/cluster_analyse/common_func/path_manager.py @@ -158,6 +158,7 @@ class PathManager: def remove_path_safety(cls, path: str): base_name = os.path.basename(path) msg = f"Failed to remove path: {base_name}" + cls.check_path_writeable(path) if os.path.islink(path): raise RuntimeError(msg) if os.path.exists(path): diff --git a/profiler/prof_common/path_manager.py b/profiler/prof_common/path_manager.py index 3e41b8b50..006c51914 100644 --- a/profiler/prof_common/path_manager.py +++ b/profiler/prof_common/path_manager.py @@ -149,6 +149,7 @@ class PathManager: def remove_path_safety(cls, path: str): base_name = os.path.basename(path) msg = f"Failed to remove path: {base_name}" + cls.check_path_writeable(path) if os.path.islink(path): raise RuntimeError(msg) if os.path.exists(path): diff --git a/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py b/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py index 8693ed33d..66ef15cd2 100644 --- a/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py +++ b/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py @@ -71,7 +71,8 @@ class TestPathManager(unittest.TestCase): os.symlink(PATH_FILE, link_name) with pytest.raises(RuntimeError) as error: PathManager.remove_path_safety(link_name) - PathManager.remove_path_safety(path + "not_exist") + with pytest.raises(RuntimeError) as error: + PathManager.remove_path_safety(path + "not_exist") PathManager.remove_path_safety(path) os.unlink(link_name) -- Gitee From ca74cbe2983dd61f6c295ce1766a67c2dadb976c Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 2 Sep 2024 16:52:21 +0800 Subject: [PATCH 627/791] grad_probe ut --- .../mindspore_ut/grad_probe/only_optimzier.py | 39 +++++++++++ .../grad_probe/test_grad_monitor.py | 67 +++++++++++++++++++ 2 files changed, 106 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/only_optimzier.py create mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/only_optimzier.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/only_optimzier.py new file mode 100644 index 000000000..45eb1a000 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/only_optimzier.py @@ -0,0 +1,39 @@ +import mindspore +from mindspore import nn, Tensor +from mindspore.nn import SGD +import numpy as np +import os + +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + +from grad_tool.grad_monitor import GradientMonitor + + +def main(): + gm = GradientMonitor(os.path.join(directory, "config.yaml"), framework="MindSpore") + + class SimpleNet(nn.Cell): + def __init__(self): + super().__init__() + self.my_dense = nn.Dense(16, 5) + + def construct(self, x): + x = self.flatten(x) + logits = self.my_dense(x) + return logits + model = SimpleNet() + optimizer = SGD(model.trainable_params(), learning_rate=0.001) + + gm.monitor(optimizer) + + fix_gradient = tuple([Tensor(np.arange(5*16).reshape((5, 16)), dtype=mindspore.float32), + Tensor(np.arange(5).reshape(5), dtype=mindspore.float32)]) + + steps = 10 + + for _ in range(steps): + optimizer(fix_gradient) + +if __name__ == "__main__": + main() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py new file mode 100644 index 000000000..ddb29f8d4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py @@ -0,0 +1,67 @@ +import os +import numpy as np +import shutil +import yaml +from unittest import TestCase +import hashlib +from grad_tool.test.ut_ms.only_optimizer import main + + +file_path = os.path.abspath(__file__) +directory = os.path.dirname(file_path) + + +def save_dict_as_yaml(data, yaml_file_path): + # 将字典保存为YAML文件 + with open(yaml_file_path, 'w') as yaml_file: + yaml.dump(data, yaml_file, default_flow_style=False) + print(f"字典已保存为YAML文件: {yaml_file_path}") + + +def get_hash(file_path): + with open(file_path, 'rb') as file: + hash_object = hashlib.md5() + for chunk in iter(lambda: file.read(4096), b""): + hash_object.update(chunk) + return hash_object.hexdigest() + + +def check_npy(gradient_output_path): + my_dense_bias_path = os.path.join(gradient_output_path, "rank_0", "step_1", "my_dense.bias.npy") + assert os.path.isfile(my_dense_bias_path) + my_dense_bias_real = np.load(my_dense_bias_path) + my_dense_bias_target = np.arange(5).reshape(5) > 0 + + assert (my_dense_bias_real == my_dense_bias_target).all() + + my_dense_weight_path = os.path.join(gradient_output_path, "rank_0", "step_1", "my_dense.weight.npy") + assert os.path.isfile(my_dense_weight_path) + my_dense_weight_real = np.load(my_dense_weight_path) + my_dense_weight_target = np.arange(5*16).reshape((5, 16)) > 0 + + assert (my_dense_weight_real == my_dense_weight_target).all() + +def check_stat_csv(csv_path, md5_value): + real_md5_value = get_hash(csv_path) + assert real_md5_value == md5_value + + +class TestMsGradientMonitor(TestCase): + def test_gradient_monitor(self): + config_path = os.path.join(directory, "config.yaml") + gradient_output_path = os.path.join(directory, "gradient_output") + config_dict = { + "level": "L2", + "param_list": None, + "rank": None, + "step": [1], + "bounds": None, + "output_path": gradient_output_path, + } + save_dict_as_yaml(config_dict, config_path) + + main() + check_npy(gradient_output_path) + check_stat_csv(os.path.join(gradient_output_path, "rank_0", "grad_summary_1.csv"), "b00499bee5c3e3ca96aaee773a092dd7") + os.remove(config_path) + shutil.rmtree(gradient_output_path) \ No newline at end of file -- Gitee From e53c141499c07114921ca1a419881fda283c1510 Mon Sep 17 00:00:00 2001 From: zyy Date: Mon, 2 Sep 2024 17:00:36 +0800 Subject: [PATCH 628/791] =?UTF-8?q?=E8=BE=93=E5=85=A5=E4=B8=8D=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E5=8F=82=E6=95=B0=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 848df2550..e260bf197 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -15,6 +15,7 @@ import argparse import os +import logging from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor @@ -95,7 +96,8 @@ def cluster_analysis_main(args=None): parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') args_parsed, unknown = parser.parse_known_args(args=args) if unknown: - print(f"Unknown arguments are provided: {unknown}") + logger = logging.getLogger() + logger.warning(f"Unknown arguments are provided: {unknown}") parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, Constant.ANALYSIS_MODE: args_parsed.mode, -- Gitee From 75f215e0359fe7d47d6bd2053e25b16247101be2 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 2 Sep 2024 17:01:59 +0800 Subject: [PATCH 629/791] msprobe grad_probe ut --- .../grad_probe/test_grad_monitor.py | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py index ddb29f8d4..ba5ac46ba 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py @@ -4,13 +4,42 @@ import shutil import yaml from unittest import TestCase import hashlib -from grad_tool.test.ut_ms.only_optimizer import main +import mindspore +from mindspore import nn, Tensor +from mindspore.nn import SGD +from grad_tool.grad_monitor import GradientMonitor file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) +def main(): + gm = GradientMonitor(os.path.join(directory, "config.yaml"), framework="MindSpore") + + class SimpleNet(nn.Cell): + def __init__(self): + super().__init__() + self.my_dense = nn.Dense(16, 5) + + def construct(self, x): + x = self.flatten(x) + logits = self.my_dense(x) + return logits + model = SimpleNet() + optimizer = SGD(model.trainable_params(), learning_rate=0.001) + + gm.monitor(optimizer) + + fix_gradient = tuple([Tensor(np.arange(5*16).reshape((5, 16)), dtype=mindspore.float32), + Tensor(np.arange(5).reshape(5), dtype=mindspore.float32)]) + + steps = 10 + + for _ in range(steps): + optimizer(fix_gradient) + + def save_dict_as_yaml(data, yaml_file_path): # 将字典保存为YAML文件 with open(yaml_file_path, 'w') as yaml_file: -- Gitee From adaeab44328524253dae367747db47e4dc2e65ef Mon Sep 17 00:00:00 2001 From: zyy Date: Mon, 2 Sep 2024 17:04:19 +0800 Subject: [PATCH 630/791] =?UTF-8?q?=E8=BE=93=E5=85=A5=E4=B8=8D=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E5=8F=82=E6=95=B0=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index e260bf197..0054a3ccd 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -25,6 +25,8 @@ from common_func.file_manager import FileManager from common_func.path_manager import PathManager from analysis.analysis_facade import AnalysisFacade +logger = logging.getLogger() + COMM_FEATURE_LIST = ['all', 'communication_time', 'communication_matrix'] class Interface: @@ -96,7 +98,6 @@ def cluster_analysis_main(args=None): parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') args_parsed, unknown = parser.parse_known_args(args=args) if unknown: - logger = logging.getLogger() logger.warning(f"Unknown arguments are provided: {unknown}") parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, -- Gitee From 8d884db64864b564ec896f1dace1881366fc3b82 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 2 Sep 2024 17:08:07 +0800 Subject: [PATCH 631/791] add FAQ --- .../msprobe/docs/FAQ_PyTorch.md | 177 ++++++++++++++++++ debug/accuracy_tools/setup.py | 6 +- 2 files changed, 180 insertions(+), 3 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md diff --git a/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md b/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md new file mode 100644 index 000000000..b98e2d7fb --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/FAQ_PyTorch.md @@ -0,0 +1,177 @@ +# 1 精度预检工具 + +1. 预检工具在 dump 和 run_ut 的过程中，是否需要同时开启或关闭 jit 编译（jit_compile）？ + + 答：是。 + +2. 预检工具对于 type_as 这类涉及数据类型转换操作的 API，是否具有参考性？ + + 由于这类 API 在 CPU 侧存在精度先提升后下降的操作，因此这类 API 的有效性的参考价值有限。 + +3. run ut 过程中出现报错：ERROR: Got unsupported ScalarType BFloat16。 + + 答：请使用最新版本的工具。 + +4. Dropout 算子，CPU 和 NPU 的随机应该不一样，为什么结果比对是一致的？ + + 答：这个结果是正常的，工具对该算子有特殊处理，只判定位置为 0 的位置比例大约和设定 p 值相当。 + +5. 为什么浮点型数据 bench 和 CPU 的 dtype 不一致？ + + 答：对于 fp16 的数据，CPU 会上升一个精度 fp32 去计算，这是和算子那边对齐的精度结论，CPU 用更高精度去计算会更接近真实值。 + +6. 添加预检工具后截取操作报错：`IndexError: too many indices for tensor of dimension x` 或 `TypeError: len() of a 0-d tensor`。 + + 答：注释工具目录 `mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml` 文件中 Tensor: 下的 `- __getitem__`，工具会跳过采集该 API。如果是需要 dump 关键位置 API 也可以考虑根据报错堆栈信息注释引发报错的类型检查。 + +7. Tensor 魔法函数具体对应什么操作？ + + 答： + + | Tensor魔法函数 | 具体操作 | + | --------------- | ---------------- | + | `__add__` | + | + | `__and__` | & | + | `__bool__` | 返回 Tensor 布尔值 | + | `__div__` | / | + | `__eq__` | == | + | `__ge__` | >= | + | `__gt__` | > | + | `__iadd__` | += | + | `__iand__` | &= | + | `__idiv__` | /= | + | `__ifloordiv__` | //= | + | `__ilshift__` | <<= | + | `__imod__` | %= | + | `__imul__` | *= | + | `__ior__` | \|= | + | `__irshift__` | >>= | + | `__isub__` | -= | + | `__ixor__` | ^= | + | `__lshift__` | << | + | `__matmul__` | 矩阵乘法 | + | `__mod__` | % | + | `__mul__` | * | + | `__nonzero__` | 同`__bool__` | + | `__or__` | \| | + | `__radd__` | +（反向） | + | `__rmul__` | *（反向） | + | `__rshift__` | >> | + | `__sub__` | - | + | `__truediv__` | 同`__div__` | + | `__xor__` | ^ | + +# 2 精度比对工具 + +## 2.1 工具使用 + +### 2.1.1 dump 指定融合算子 + +数据采集当前支持融合算子的输入输出，需要在 `mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml` 中添加，比如以下代码段调用的 softmax 融合算子。 + +```python +def npu_forward_fused_softmax(self, input_, mask): + resl = torch_npu.npu_scaled_masked_softmax(input_, mask, self.scale, False) + return resl +``` + +如果需要 dump 其中调用的 npu_scaled_masked_softmax 算子的输入输出信息，需要在 `support_wrap_ops.yaml` 中的 `torch_npu: ` 中自行添加该融合算子： + +```yaml +- npu_scaled_masked_softmax +``` + +（npu_scaled_masked_softmax 融合算子工具已支持 dump，本例仅供参考）。 + +## 2.2 常见问题 + +1. 在同一个目录多次执行 dump 会冲突吗？ + + 答：会，同一个目录多次 dump，会覆盖上一次结果，可以使用 dump_path 参数修改 dump 目录。 + +2. 如何 dump 算子级的数据？ + + 答：需要配置 level 为 L2 模式。 + +3. 工具比对发现 NPU 和标杆数据的 API 无法完全对齐？ + + 答：torch 版本和硬件差异属于正常情况。 + +## 2.3 异常情况 + +1. HCCL 报错： error code: EI0006。 + + **故障现象**：使用 msprobe 工具时，报错：error code: EI0006。 + + **故障原因**：CANN 软件版本较低导致不兼容。 + + **故障处理**：升级新版 CANN 软件版本。 + +2. torch_npu._C._clear_overflow_npu() RuntimeError NPU error，error code is 107002。 + + 如果运行溢出检测功能遇到这个报错，采取以下解决方法： + + 如果是单卡运行，添加如下代码，0 是卡号，选择自己空闲的卡号。 + + ```python + torch.npu.set_device('npu:0') + ``` + + 如果多卡运行，请在代码中修改对应卡号，比如进程使用卡号为 {rank} 时可以添加如下代码： + + ```python + torch.npu.set_device(f'npu:{rank}') + ``` + + 如果运行精度比对功能遇到这个报错，尝试安装最新版本的 msprobe。 + +3. dump 得到的 `VF_lstm_99_forward_input.1.0.npy`、`VF_lstm_99_forward_input.1.1.npy` 类似的数据是否正常？ + + 带 1.0/1.1/1.2 后缀的 npy 是正常现象，例如，当输入数据为 [[tensor1, tensor2, tensor3]] 会生成这样的后缀。 + +4. 进行 compare 报错：The current file contains stack information, please turn on the stack_mode。 + + 在比对脚本中，设置 `stack_mode=True`，例如： + + ```python + from msprobe.pytorch import compare + dump_result_param={ + "npu_json_path": "./npu_dump/dump.json", + "bench_json_path": "./gpu_dump/dump.json", + "stack_json_path": "./npu_dump/stack.json", + "is_print_compare_log": True + } + compare(dump_result_param, output_path="./output", stack_mode=True) + ``` + +5. dump 指定反向 API 的 kernel 级别的数据报错：NameError：name 'torch_npu' is not defined。 + + 答：如果是 npu 环境，请安装 torch_npu；如果是 gpu 环境，暂不支持 dump 指定 API 的 kernel 级别的数据。 + +6. 配置 dump_path 后，使用工具报错：[ERROR] The file path /home/xxx/dump contains special characters。 + + 答：请检查你设置的 dump 绝对路径是否包含特殊字符，确保路径名只包含大小写字母、数字、下划线、斜杠、点和短横线；注意，如果执行脚本的路径为 /home/abc++/，设置的 dump_path="./dump"，工具实际校验的路径为绝对路径 /home/abc++/dump，++ 为特殊字符，会引发本条报错。 + +7. 无法 dump matmul 权重的反向梯度数据。 + + 答：matmul 期望的输入是二维，当输入不是二维时，会将输入通过 view 操作展成二维，再进行 matmul 运算，因此在反向求导时，backward_hook 能拿到的是 UnsafeViewBackward 这步操作里面数据的梯度信息，取不到 MmBackward 这步操作里面数据的梯度信息，即权重的反向梯度数据。典型的例子有，当 linear 的输入不是二维，且无 bias 时，会调用 output = input.matmul(weight.t())，因此拿不到 linear 层的 weight 的反向梯度数据。 + +8. dump.json 文件中的某些 api 的 dtype 类型为 float16，但是读取此 api 的 npy 文件显示的 dtype 类型为 float32。 + + 答：msprobe 工具在 dump 数据时需要将原始数据从 npu to cpu 上再转换为 numpy 类型，npu to cpu 的逻辑和 gpu to cpu 是保持一致的，都存在 dtype 可能从 float16 变为 float32 类型的情况，如果出现 dtype 不一致的问题，最终采集数据的 dtype 以 pkl 文件为准。 + +9. 使用 dataloader 后 raise 异常 Exception("msprobe: exit after iteration {}". format(max(self.config.step)))。 + + 答：正常现象，dataloader 通过 raise 结束程序，堆栈信息可忽略。 + +10. 添加 msprobe 工具后截取操作报错：`IndexError: too many indices for tensor of dimension x` 或 `TypeError: len() of a 0-d tensor`。 + + 答：注释工具目录 `mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml` 文件中 `Tensor: ` 下的 `- __getitem__`，工具会跳过采集该 API。如果是需要采集关键位置 API 也可以考虑根据报错堆栈信息注释引发报错的类型检查。 + +11. 添加 msprobe 工具后 F.gelu 触发 ValueError 报错：`activation_func must be F.gelu`等。 + + 答：注释工具目录 `mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml` 文件中 `functional: ` 下的 `-gelu`，工具会跳过采集该 API。如果需要采集关键位置 api 也可以考虑根据报错堆栈信息注释引发报错的类型检查。 + +12. 添加 msprobe 工具后触发与 AsStrided 算子相关、或者编译相关的报错，如：`Failed to compile Op [AsStrided]`。 + + 答：注释工具目录 `mstt/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml` 文件中 `Tensor: `下的 `-t` 和 `- transpose`。 diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index b273f0f3d..01e399c46 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -19,13 +19,13 @@ __version__ = '1.0.3' INSTALL_REQUIRED = [ "wheel", "einops", - "numpy", - "pandas >= 1.3.5", + "numpy < 2.0", + "pandas >= 1.3.5, < 2.1", "pyyaml", "rich", "tqdm", "openpyxl", - "pyOpenSSL", + "pyopenssl", "twisted", "matplotlib" ] -- Gitee From b36c474d6402e62ac72743a4e119fc1b20915d0b Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 2 Sep 2024 17:08:57 +0800 Subject: [PATCH 632/791] add FAQ --- debug/accuracy_tools/msprobe/README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 7b86123d9..802f1e857 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -57,7 +57,7 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. [PyTorch 场景的溢出检测与解析](./docs/12.overflow_check_PyTorch.md)（更新中） -[MindSpore 场景的溢出检测与解析](./docs/13.overflow_check_MindSpore.md)（更新中） +[MindSpore 场景的溢出检测与解析](./docs/13.overflow_check_MindSpore.md)（待补充） ### 5 数据解析 @@ -67,15 +67,15 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. ### 6 无标杆比对 -[PyTorch 场景的无标杆比对](./docs/15.free_benchmarking_PyTorch.md)（更新中） +[PyTorch 场景的无标杆比对](./docs/15.free_benchmarking_PyTorch.md)（待补充） -[MindSpore 场景的无标杆比对](./docs/16.overflow_check_MindSpore.md)（更新中） +[MindSpore 场景的无标杆比对](./docs/16.overflow_check_MindSpore.md)（待补充） ### 7 梯度状态监测 本功能用于采集梯度数据并进行梯度相似度比对，可以精准定位出现问题的 step。 -[兼容 PyTorch 和 MindSpore 框架的梯度监测](./docs/17.grad_probe.md)（更新中） +[兼容 PyTorch 和 MindSpore 框架的梯度监测](./docs/17.grad_probe.md) ## 🌟 新版本特性 @@ -90,12 +90,12 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. ## 📑 补充材料 -[msprobe 精度工具标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md)（更新中） +[msprobe 精度工具标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md)（待补充） -[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md)（更新中） +[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md)（待补充） ## ❓ FAQ -[FAQ for PyTorch](./docs/FAQ_PyTorch.md)（更新中） +[FAQ for PyTorch](./docs/FAQ_PyTorch.md) FAQ for MindSpore -- Gitee From 3fc0991579b90fd02f866684bffa2f11cefede3d Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 2 Sep 2024 17:11:09 +0800 Subject: [PATCH 633/791] remove only_optimizer --- .../mindspore_ut/grad_probe/only_optimzier.py | 39 ------------------- 1 file changed, 39 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/only_optimzier.py diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/only_optimzier.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/only_optimzier.py deleted file mode 100644 index 45eb1a000..000000000 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/only_optimzier.py +++ /dev/null @@ -1,39 +0,0 @@ -import mindspore -from mindspore import nn, Tensor -from mindspore.nn import SGD -import numpy as np -import os - -file_path = os.path.abspath(__file__) -directory = os.path.dirname(file_path) - -from grad_tool.grad_monitor import GradientMonitor - - -def main(): - gm = GradientMonitor(os.path.join(directory, "config.yaml"), framework="MindSpore") - - class SimpleNet(nn.Cell): - def __init__(self): - super().__init__() - self.my_dense = nn.Dense(16, 5) - - def construct(self, x): - x = self.flatten(x) - logits = self.my_dense(x) - return logits - model = SimpleNet() - optimizer = SGD(model.trainable_params(), learning_rate=0.001) - - gm.monitor(optimizer) - - fix_gradient = tuple([Tensor(np.arange(5*16).reshape((5, 16)), dtype=mindspore.float32), - Tensor(np.arange(5).reshape(5), dtype=mindspore.float32)]) - - steps = 10 - - for _ in range(steps): - optimizer(fix_gradient) - -if __name__ == "__main__": - main() -- Gitee From 16b3175177ad3f756074c28417ec1cacfb1f12d5 Mon Sep 17 00:00:00 2001 From: zyy Date: Mon, 2 Sep 2024 17:20:38 +0800 Subject: [PATCH 634/791] =?UTF-8?q?=E7=94=9F=E6=88=90=E4=BA=A4=E4=BB=98?= =?UTF-8?q?=E4=BB=B6=E5=90=8E=E8=BE=93=E5=87=BA=E4=BA=A4=E4=BB=98=E4=BB=B6?= =?UTF-8?q?=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 3add0ab99..3f3d58367 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -87,6 +87,7 @@ class Interface: comm_data_dict = CommunicationGroupGenerator(params).generate() params[Constant.COMM_DATA_DICT] = comm_data_dict AnalysisFacade(params).cluster_analyze() + print(f"[INFO] The cluster analysis result file has been generated: {self.cluster_analysis_output_path}") def cluster_analysis_main(args=None): parser = argparse.ArgumentParser(description="cluster analysis module") -- Gitee From 0646c490fb431d7b02fce38474ac20504d2e054e Mon Sep 17 00:00:00 2001 From: zyy Date: Mon, 2 Sep 2024 17:27:27 +0800 Subject: [PATCH 635/791] =?UTF-8?q?=E8=BE=93=E5=85=A5=E4=B8=8D=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E5=8F=82=E6=95=B0=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 0054a3ccd..20a7cc6b9 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -98,7 +98,7 @@ def cluster_analysis_main(args=None): parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') args_parsed, unknown = parser.parse_known_args(args=args) if unknown: - logger.warning(f"Unknown arguments are provided: {unknown}") + logger.warning("Unknown arguments are provided: %s", unknown) parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, Constant.ANALYSIS_MODE: args_parsed.mode, -- Gitee From 3659ae067739501f1bbba07823a6b67f65e5257c Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Fri, 30 Aug 2024 16:17:08 +0800 Subject: [PATCH 636/791] =?UTF-8?q?logger=E5=88=86=E6=A1=86=E6=9E=B6?= =?UTF-8?q?=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/log.py | 17 ++++++++++------- .../api_accuracy_checker.py | 2 +- .../mindspore/api_accuracy_checker/api_info.py | 2 +- .../api_accuracy_checker/api_runner.py | 2 +- .../base_compare_algorithm.py | 2 +- .../api_accuracy_checker/compute_element.py | 2 +- .../mindspore/api_accuracy_checker/utils.py | 2 +- .../mindspore/compare/distributed_compare.py | 2 +- .../msprobe/mindspore/compare/ms_compare.py | 2 +- .../mindspore/compare/ms_graph_compare.py | 2 +- .../msprobe/mindspore/dump/kernel_graph_dump.py | 2 +- .../msprobe/mindspore/dump/kernel_kbyk_dump.py | 2 +- .../mindspore/grad_probe/global_context.py | 2 +- .../mindspore/grad_probe/grad_analyzer.py | 2 +- .../msprobe/mindspore/grad_probe/hook.py | 2 +- .../kernel_graph_overflow_check.py | 2 +- .../msprobe/pytorch/common/log.py | 11 ----------- .../msprobe/pytorch/common/utils.py | 17 ++--------------- .../pytorch/compare/distributed_compare.py | 2 +- .../msprobe/pytorch/compare/pt_compare.py | 2 +- .../msprobe/pytorch/free_benchmark/__init__.py | 2 +- .../msprobe/pytorch/grad_probe/grad_monitor.py | 2 +- .../msprobe/pytorch/online_dispatch/utils.py | 2 +- 23 files changed, 32 insertions(+), 53 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 3dd2ee064..248fed365 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -9,13 +9,6 @@ class BaseLogger: def __init__(self): self.rank = None - @staticmethod - def _print_log(level, msg, end='\n'): - current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) - pid = os.getpid() - full_msg = f"{current_time} ({pid}) [{level}] {msg}" - print(full_msg, end=end) - sys.stdout.flush() def get_rank(self): return self.rank @@ -62,5 +55,15 @@ class BaseLogger: self.error(msg) raise exception + def _print_log(self, level, msg, end='\n'): + current_rank = self.get_rank() + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + pid = os.getpid() + if current_rank is not None: + full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}" + else: + full_msg = f"{current_time} ({pid}) [{level}] {msg}" + print(full_msg, end=end) + sys.stdout.flush() logger = BaseLogger() diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index bb96644c6..ec71615c1 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -4,7 +4,7 @@ import os from msprobe.core.common.file_check import FileOpen from msprobe.core.common.utils import write_csv, add_time_as_suffix from msprobe.core.common.const import Const, CompareConst, MsCompareConst -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo from msprobe.mindspore.api_accuracy_checker.api_runner import api_runner, ApiInputAggregation from msprobe.mindspore.api_accuracy_checker.base_compare_algorithm import compare_algorithms diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py index 47c33aeab..5be09346c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_info.py @@ -2,7 +2,7 @@ from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElemen from msprobe.core.common.const import Const from msprobe.mindspore.api_accuracy_checker.utils import check_and_get_from_json_dict from msprobe.core.common.exceptions import ApiAccuracyCheckerException -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger class ApiInfo: def __init__(self, api_name): diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py index 4c2f84f0a..8c05f753e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_runner.py @@ -7,7 +7,7 @@ from mindspore import ops from msprobe.mindspore.api_accuracy_checker.compute_element import ComputeElement from msprobe.core.common.const import Const, MsCompareConst from msprobe.core.common.exceptions import ApiAccuracyCheckerException -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.mindspore.api_accuracy_checker.utils import convert_to_tuple from msprobe.mindspore.api_accuracy_checker.type_mapping import float_dtype_str_list, torch_dtype_to_dtype_str diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py index 34d762304..a2b25de67 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/base_compare_algorithm.py @@ -5,7 +5,7 @@ import torch import numpy as np from msprobe.core.common.exceptions import ApiAccuracyCheckerException -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.const import CompareConst, MsCompareConst class CompareResult: diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index 9fb44fbdc..ffadce87b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -4,7 +4,7 @@ import mindspore import torch import numpy as np -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.exceptions import ApiAccuracyCheckerException from msprobe.core.common.utils import load_npy from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np_dtype, api_info_type_str_to_type, diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py index 16a97b24f..6b3593cf6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/utils.py @@ -1,7 +1,7 @@ from msprobe.core.common.exceptions import ApiAccuracyCheckerException -from msprobe.core.common.log import logger from msprobe.core.common.const import Const from msprobe.mindspore.api_accuracy_checker.type_mapping import float_dtype_str_list +from msprobe.mindspore.common.log import logger def check_and_get_from_json_dict(dict_instance, key, key_description, accepted_type=None, accepted_value=None): ''' diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 763a5651f..073969862 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -19,7 +19,7 @@ from msprobe.core.common.utils import CompareException, check_compare_param, \ check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import create_directory from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.mindspore.compare.ms_compare import MSComparator from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json from msprobe.mindspore.compare.ms_graph_compare import GraphMSComparator diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 7c3bd6815..1efaeeb25 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -3,7 +3,7 @@ from msprobe.core.common.utils import check_compare_param, CompareException, che task_dumppath_get, load_yaml, load_npy from msprobe.core.common.file_check import create_directory from msprobe.core.common.const import Const -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator from msprobe.core.compare.check import check_struct_match, fuzzy_check_op diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index bb83c0fb8..69d95ca5e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -9,7 +9,7 @@ import pandas as pd from msprobe.core.common.const import CompareConst, GraphMode from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.file_check import create_directory -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import add_time_with_xlsx, CompareException from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py index 7db2b8142..6c07cfbb8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py @@ -1,7 +1,7 @@ import os import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.file_check import FileOpen, create_directory diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py index cea9037f3..52a4c816c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py @@ -2,7 +2,7 @@ import os import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.file_check import FileOpen, create_directory from msprobe.core.common.const import Const diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py index 16d0bd0b8..1b2a11ac6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -4,7 +4,7 @@ from typing import Dict, Union from msprobe.core.grad_probe.utils import check_str from msprobe.core.grad_probe.constant import GradConst -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.file_check import create_directory from msprobe.core.common.utils import check_path_before_create diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index 2bdc11114..146f892c3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -12,7 +12,7 @@ from mindspore.common.parameter import Parameter from msprobe.core.grad_probe.utils import ListCache from msprobe.core.grad_probe.constant import GradConst -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.file_check import create_directory from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py index 008fd6fe3..1b98ee8e7 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py @@ -9,7 +9,7 @@ from mindspore.common.parameter import Parameter from mindspore.common.initializer import initializer from msprobe.core.grad_probe.constant import GradConst -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import write_csv, remove_path from msprobe.mindspore.grad_probe.global_context import grad_context diff --git a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py index ae5697586..0e41c6bea 100644 --- a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py @@ -1,7 +1,7 @@ import os import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig -from msprobe.core.common.log import logger +from msprobe.mindspore.common.log import logger from msprobe.core.common.file_check import FileOpen, create_directory diff --git a/debug/accuracy_tools/msprobe/pytorch/common/log.py b/debug/accuracy_tools/msprobe/pytorch/common/log.py index cea518fa4..e7df8fdf7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/log.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/log.py @@ -17,16 +17,5 @@ class PyTorchLogger(BaseLogger): current_rank = None return current_rank - def _print_log(self, level, msg, end='\n'): - current_rank = self.get_rank() - current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) - pid = os.getpid() - if current_rank is not None: - full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}" - else: - full_msg = f"{current_time} ({pid}) [{level}] {msg}" - print(full_msg, end=end) - sys.stdout.flush() - logger = PyTorchLogger() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index cb44c6bbe..d42a19d96 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -26,7 +26,7 @@ import torch.distributed as dist import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError -from msprobe.core.common.log import logger as common_logger +from msprobe.core.common.log import logger from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, CompareException from msprobe.core.common.file_check import FileCheckConst, change_mode, FileOpen @@ -281,7 +281,7 @@ def save_pt(tensor, filepath): try: torch.save(tensor, filepath) except Exception as e: - common_logger.error("Save pt file failed, please check according possible error causes: " + logger.error("Save pt file failed, please check according possible error causes: " "1. out of disk space or disk error, " "2. no permission to write files, etc.") raise RuntimeError(f"save pt file {filepath} failed") from e @@ -306,16 +306,3 @@ def load_api_data(api_data_bytes): except Exception as e: raise RuntimeError(f"load api_data from bytes failed") from e return buffer - - -def _create_logger(level=logging.INFO): - logger_ = logging.getLogger() - logger_.setLevel(level) - ch = logging.StreamHandler() - ch.setLevel(level) - logger_.addHandler(ch) - return logger_ - - -log_level = logging.DEBUG if os.environ.get("API_ACCURACY_CHECK_LOG_LEVEL") == "1" else logging.INFO -logger = _create_logger(log_level) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index a90213da4..bdabd0ee7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -19,7 +19,7 @@ from msprobe.core.common.utils import CompareException, check_compare_param, \ check_configuration_param, task_dumppath_get from msprobe.core.common.file_check import create_directory from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.log import logger +from msprobe.pytorch.common.log import logger from msprobe.core.common.const import Const from msprobe.pytorch.compare.pt_compare import PTComparator from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 188b62e18..17730d742 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -2,7 +2,7 @@ import os.path import torch from msprobe.core.common.const import FileCheckConst, Const from msprobe.core.common.file_check import create_directory -from msprobe.core.common.log import logger +from msprobe.pytorch.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.utils import check_configuration_param, task_dumppath_get, check_compare_param, FileChecker diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/__init__.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/__init__.py index d234898c0..f0b40ccb2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/__init__.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/__init__.py @@ -1,4 +1,4 @@ -from msprobe.core.common.log import logger +from msprobe.pytorch.common.log import logger from msprobe.core.common.exceptions import FreeBenchmarkException from msprobe.core.common.const import Const diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index 756ad8a93..b577b2fab 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -8,7 +8,7 @@ from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target from msprobe.core.grad_probe.constant import GradConst, level_adp from msprobe.core.common.file_check import create_directory -from msprobe.core.common.log import logger +from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import remove_path, write_csv, save_npy from msprobe.pytorch.common.utils import get_rank_id, print_rank_0, save_pt diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index 596ab090a..85786bf60 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -13,7 +13,7 @@ else: from msprobe.core.common.const import CompareConst, FileCheckConst from msprobe.core.common.file_check import change_mode -from msprobe.core.common.log import logger +from msprobe.pytorch.common.log import logger cpu_device = torch._C.device("cpu") COLOR_RED = '\033[31m' -- Gitee From f87d293f5665bf050af6dd71313a0c6213476b3e Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Mon, 2 Sep 2024 17:40:35 +0800 Subject: [PATCH 637/791] =?UTF-8?q?file=5Fcheck.py=E9=87=8D=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E6=88=90file=5Futils.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/advisor/advisor.py | 2 +- .../msprobe/core/advisor/advisor_result.py | 2 +- .../common/{file_check.py => file_utils.py} | 0 .../msprobe/core/common/utils.py | 2 +- .../msprobe/core/compare/acc_compare.py | 2 +- .../msprobe/core/compare/compare_cli.py | 2 +- .../data_processor/mindspore_processor.py | 2 +- .../data_processor/pytorch_processor.py | 2 +- .../msprobe/core/data_dump/json_writer.py | 2 +- .../msprobe/core/grad_probe/grad_compare.py | 2 +- .../api_accuracy_checker.py | 2 +- .../msprobe/mindspore/common/utils.py | 2 +- .../mindspore/compare/distributed_compare.py | 2 +- .../msprobe/mindspore/compare/ms_compare.py | 2 +- .../mindspore/compare/ms_graph_compare.py | 4 +- .../mindspore/debugger/debugger_config.py | 2 +- .../mindspore/dump/kernel_graph_dump.py | 2 +- .../mindspore/dump/kernel_kbyk_dump.py | 2 +- .../free_benchmark/api_pynative_self_check.py | 2 +- .../mindspore/grad_probe/global_context.py | 2 +- .../mindspore/grad_probe/grad_analyzer.py | 2 +- .../msprobe/mindspore/grad_probe/utils.py | 2 +- .../msprobe/mindspore/ms_config.py | 2 +- .../kernel_graph_overflow_check.py | 2 +- .../msprobe/mindspore/service.py | 2 +- .../api_accuracy_checker/common/utils.py | 2 +- .../compare/api_precision_compare.py | 2 +- .../run_ut/data_generate.py | 2 +- .../run_ut/multi_run_ut.py | 2 +- .../run_ut/run_overflow_check.py | 2 +- .../api_accuracy_checker/run_ut/run_ut.py | 2 +- .../run_ut/run_ut_utils.py | 2 +- .../msprobe/pytorch/common/parse_json.py | 2 +- .../msprobe/pytorch/common/utils.py | 2 +- .../pytorch/compare/distributed_compare.py | 2 +- .../pytorch/grad_probe/grad_monitor.py | 2 +- .../pytorch/online_dispatch/compare.py | 2 +- .../pytorch/online_dispatch/dispatch.py | 2 +- .../pytorch/online_dispatch/dump_compare.py | 2 +- .../msprobe/pytorch/online_dispatch/utils.py | 2 +- .../msprobe/pytorch/parse_tool/lib/compare.py | 2 +- .../msprobe/pytorch/parse_tool/lib/utils.py | 4 +- .../pytorch/parse_tool/lib/visualization.py | 2 +- .../msprobe/pytorch/pt_config.py | 2 +- .../accuracy_tools/msprobe/pytorch/service.py | 2 +- .../msprobe/test/core_ut/common/test_utils.py | 2 +- .../data_processor/test_pytorch_processor.py | 2 +- .../core_ut/data_dump/test_json_writer.py | 2 +- .../msprobe/test/core_ut/test_file_check.py | 60 +++++++++---------- .../test_compare_online_dispatch.py | 2 +- .../test_utils_online_dispatch.py | 2 +- .../msprobe/test/pytorch_ut/test_service.py | 2 +- 52 files changed, 82 insertions(+), 82 deletions(-) rename debug/accuracy_tools/msprobe/core/common/{file_check.py => file_utils.py} (100%) diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor.py b/debug/accuracy_tools/msprobe/core/advisor/advisor.py index 9824ac22a..4217121c9 100644 --- a/debug/accuracy_tools/msprobe/core/advisor/advisor.py +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor.py @@ -21,7 +21,7 @@ from msprobe.core.advisor.advisor_result import AdvisorResult from msprobe.core.advisor.advisor_const import AdvisorConst from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException -from msprobe.core.common.file_check import FileChecker +from msprobe.core.common.file_utils import FileChecker from msprobe.core.common.const import Const, CompareConst, FileCheckConst class Advisor: diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py index 2bfea2eb9..c2ccb4a0e 100644 --- a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py @@ -20,7 +20,7 @@ import time from msprobe.core.advisor.advisor_const import AdvisorConst from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.file_check import change_mode +from msprobe.core.common.file_utils import change_mode class AdvisorResult: diff --git a/debug/accuracy_tools/msprobe/core/common/file_check.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py similarity index 100% rename from debug/accuracy_tools/msprobe/core/common/file_check.py rename to debug/accuracy_tools/msprobe/core/common/file_utils.py diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index ffe2de547..6ef199c12 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -27,7 +27,7 @@ from pathlib import Path import yaml import numpy as np -from msprobe.core.common.file_check import FileOpen, FileChecker, change_mode, check_path_before_create +from msprobe.core.common.file_utils import FileOpen, FileChecker, change_mode, check_path_before_create from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 3de6bd354..492e30ba9 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -2,7 +2,7 @@ import multiprocessing import os import json import pandas as pd -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py index 2ae4a09d5..a20a17abf 100644 --- a/debug/accuracy_tools/msprobe/core/compare/compare_cli.py +++ b/debug/accuracy_tools/msprobe/core/compare/compare_cli.py @@ -1,5 +1,5 @@ import json -from msprobe.core.common.file_check import FileOpen, check_file_type +from msprobe.core.common.file_utils import FileOpen, check_file_type from msprobe.core.common.const import FileCheckConst, Const from msprobe.core.common.utils import CompareException from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 391482a60..a72096804 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -22,7 +22,7 @@ import numpy as np from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import (BaseDataProcessor, TensorStatInfo, ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs) -from msprobe.core.common.file_check import path_len_exceeds_limit +from msprobe.core.common.file_utils import path_len_exceeds_limit from msprobe.mindspore.dump.hook_cell.wrap_functional import load_ops_functions from msprobe.mindspore.common.utils import convert_bf16_to_fp32, save_tensor_as_npy from msprobe.mindspore.common.log import logger diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 133038255..3096ce49b 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -6,7 +6,7 @@ from typing import List import numpy as np import torch -from msprobe.core.common.file_check import path_len_exceeds_limit, change_mode +from msprobe.core.common.file_utils import path_len_exceeds_limit, change_mode from msprobe.core.common.log import logger from msprobe.core.common.const import Const, OverflowConst, FileCheckConst from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, ModuleBackwardInputsOutputs, \ diff --git a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py index fc560b404..c64826d31 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py @@ -4,7 +4,7 @@ import fcntl import json from pathlib import Path -from msprobe.core.common.file_check import change_mode, FileOpen +from msprobe.core.common.file_utils import change_mode, FileOpen from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.utils import remove_path diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 6323787ba..0f871d378 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -6,7 +6,7 @@ import pandas as pd import matplotlib.pyplot as plt from msprobe.core.common.utils import check_file_or_directory_path -from msprobe.core.common.file_check import create_directory, check_path_before_create +from msprobe.core.common.file_utils import create_directory, check_path_before_create from msprobe.core.common.log import logger from msprobe.core.common.utils import remove_path, write_csv, load_npy from msprobe.core.grad_probe.constant import GradConst diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index bb96644c6..7807d5c61 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -1,7 +1,7 @@ import json import os -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.utils import write_csv, add_time_as_suffix from msprobe.core.common.const import Const, CompareConst, MsCompareConst from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index dc4444e78..1de9ef959 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -16,7 +16,7 @@ import mindspore as ms from msprobe.core.common.exceptions import DistributedNotInitializedError -from msprobe.core.common.file_check import path_len_exceeds_limit +from msprobe.core.common.file_utils import path_len_exceeds_limit from msprobe.core.common.utils import save_npy from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 763a5651f..ea22e51d0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -17,7 +17,7 @@ import os from msprobe.core.common.utils import CompareException, check_compare_param, \ check_configuration_param, task_dumppath_get -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.mindspore.compare.ms_compare import MSComparator diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 7c3bd6815..79c0b219f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,7 +1,7 @@ import os.path from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, \ task_dumppath_get, load_yaml, load_npy -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.const import Const from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index bb83c0fb8..d9007c0aa 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -8,12 +8,12 @@ import numpy as np import pandas as pd from msprobe.core.common.const import CompareConst, GraphMode from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.log import logger from msprobe.core.common.utils import add_time_with_xlsx, CompareException from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen class row_data: def __init__(self, mode): diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 24b61f3df..d49fa8057 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -4,7 +4,7 @@ from pathlib import Path from msprobe.core.common.const import Const from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.common.const import FreeBenchmarkConst -from msprobe.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create +from msprobe.core.common.file_utils import FileChecker, FileCheckConst, check_path_before_create class DebuggerConfig: diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py index 8320ee090..c65671339 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py @@ -3,7 +3,7 @@ import json from msprobe.core.common.utils import make_dump_path_if_not_exists from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen class KernelGraphDump: diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py index 54e57c508..c5147015d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py @@ -4,7 +4,7 @@ import json from msprobe.core.common.utils import make_dump_path_if_not_exists from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.const import Const diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index 510d5044f..e000b44b5 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -9,7 +9,7 @@ from msprobe.core.common.utils import load_yaml from msprobe.core.common.const import Const from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.config import Config -from msprobe.core.common.file_check import check_path_length +from msprobe.core.common.file_utils import check_path_length from msprobe.mindspore.common.log import logger from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.free_benchmark.decorator.decorator_factory import decorate_forward_function diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py index 1a5055a1e..eade73770 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -5,7 +5,7 @@ from typing import Dict, Union from msprobe.core.grad_probe.utils import check_str from msprobe.core.grad_probe.constant import GradConst from msprobe.core.common.log import logger -from msprobe.core.common.file_check import create_directory, check_path_before_create +from msprobe.core.common.file_utils import create_directory, check_path_before_create class GlobalContext: diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index 2bdc11114..7e8b80d44 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -13,7 +13,7 @@ from mindspore.common.parameter import Parameter from msprobe.core.grad_probe.utils import ListCache from msprobe.core.grad_probe.constant import GradConst from msprobe.core.common.log import logger -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py index bef542f37..301b6b3f8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py @@ -4,7 +4,7 @@ import numpy as np import mindspore from msprobe.core.grad_probe.constant import GradConst, level_adp from msprobe.core.grad_probe.utils import check_param -from msprobe.core.common.file_check import create_directory, check_path_before_create +from msprobe.core.common.file_utils import create_directory, check_path_before_create from msprobe.core.common.utils import change_mode, check_file_or_directory_path, save_npy diff --git a/debug/accuracy_tools/msprobe/mindspore/ms_config.py b/debug/accuracy_tools/msprobe/mindspore/ms_config.py index fad61ec7b..7b552860f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/ms_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/ms_config.py @@ -1,7 +1,7 @@ import json from msprobe.core.common_config import CommonConfig, BaseConfig -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.const import Const from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.common.log import logger diff --git a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py index 664060873..51400b0b1 100644 --- a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py @@ -3,7 +3,7 @@ import json from msprobe.core.common.utils import make_dump_path_if_not_exists from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen class KernelGraphOverflowCheck: diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 016c68782..d2914f29b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -33,7 +33,7 @@ except ImportError: from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope from msprobe.mindspore.common.utils import get_rank_if_initialized -from msprobe.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create +from msprobe.core.common.file_utils import FileChecker, FileCheckConst, check_path_before_create from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import Const from msprobe.core.common.exceptions import DistributedNotInitializedError diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py index 467c1f045..b1d796a34 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/utils.py @@ -29,7 +29,7 @@ else: from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.utils import save_pt -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.const import Const from msprobe.core.common.utils import CompareException, make_dump_path_if_not_exists diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index d93ad463d..a38b11158 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -17,7 +17,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECI from msprobe.pytorch.api_accuracy_checker.compare.compare_column import ApiPrecisionOutputColumn from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validated_result_csv_path from msprobe.pytorch.api_accuracy_checker.common.utils import extract_detailed_api_segments -from msprobe.core.common.file_check import FileChecker, change_mode, check_path_before_create, create_directory +from msprobe.core.common.file_utils import FileChecker, change_mode, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException from msprobe.core.common.const import Const, CompareConst, FileCheckConst diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index 7812ad93b..c68c35957 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -23,7 +23,7 @@ import numpy from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import hf_32_standard_api from msprobe.pytorch.api_accuracy_checker.common.utils import check_object_type, get_full_data_path, \ CompareException -from msprobe.core.common.file_check import FileChecker +from msprobe.core.common.file_utils import FileChecker from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.utils import load_pt from msprobe.core.common.utils import load_npy diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index 8334abef8..4971a7c84 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -15,7 +15,7 @@ from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import get_validat from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.common import parse_json_info_forward_backward from msprobe.pytorch.common.log import logger -from msprobe.core.common.file_check import FileChecker, check_file_suffix, check_link, FileOpen, \ +from msprobe.core.common.file_utils import FileChecker, check_file_suffix, check_link, FileOpen, \ check_path_before_create, create_directory from msprobe.core.common.utils import remove_path from msprobe.core.common.const import FileCheckConst diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index 98e5eaab3..ed46824cf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -13,7 +13,7 @@ from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import generate_device_params, get_api_info from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import exec_api from msprobe.core.common.utils import get_json_contents -from msprobe.core.common.file_check import check_link +from msprobe.core.common.file_utils import check_link from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward from msprobe.core.common.const import Const diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index dd2f70f01..c442bf9c0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -26,7 +26,7 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare import Comparator from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareColumn from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward -from msprobe.core.common.file_check import FileOpen, FileChecker, \ +from msprobe.core.common.file_utils import FileOpen, FileChecker, \ change_mode, check_path_before_create, create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import get_json_contents diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index 84db853b9..7ac11991e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -2,7 +2,7 @@ import os import re from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.file_check import FileChecker +from msprobe.core.common.file_utils import FileChecker from msprobe.pytorch.hook_module.wrap_aten import AtenOPTemplate from msprobe.pytorch.hook_module.wrap_functional import FunctionalOPTemplate from msprobe.pytorch.hook_module.wrap_npu_custom import NpuOPTemplate diff --git a/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py b/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py index ccad90372..89edd834c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/parse_json.py @@ -1,7 +1,7 @@ import json from msprobe.core.common.exceptions import ParseJsonException -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen def parse_json_info_forward_backward(json_path): diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 98d623e4a..cef23857c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -28,7 +28,7 @@ from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.log import logger as common_logger from msprobe.core.common.utils import check_file_or_directory_path, CompareException -from msprobe.core.common.file_check import FileCheckConst, change_mode, FileOpen, check_path_before_create +from msprobe.core.common.file_utils import FileCheckConst, change_mode, FileOpen, check_path_before_create try: diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index a90213da4..7c64d3ae6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -17,7 +17,7 @@ import os from msprobe.core.common.utils import CompareException, check_compare_param, \ check_configuration_param, task_dumppath_get -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.core.common.const import Const diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index 756ad8a93..bce547668 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -7,7 +7,7 @@ if int(torch.__version__.split('.')[0]) >= 2: from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target from msprobe.core.grad_probe.constant import GradConst, level_adp -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.log import logger from msprobe.core.common.utils import remove_path, write_csv, save_npy from msprobe.pytorch.common.utils import get_rank_id, print_rank_0, save_pt diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py index 985e0f743..22c520312 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py @@ -7,7 +7,7 @@ from collections import namedtuple from rich.table import Table from rich.console import Console from msprobe.core.common.const import CompareConst, FileCheckConst -from msprobe.core.common.file_check import FileOpen, change_mode +from msprobe.core.common.file_utils import FileOpen, change_mode from .single_compare import single_benchmark_compare_wrap from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index f4686f9d8..9199fe5bb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -16,7 +16,7 @@ else: is_npu = True from msprobe.core.common.utils import check_file_or_directory_path, load_yaml -from msprobe.core.common.file_check import check_path_before_create +from msprobe.core.common.file_utils import check_path_before_create from msprobe.core.common.const import Const, CompareConst from msprobe.pytorch.common.log import logger from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py index 5b2e94e3c..6b6e671a3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py @@ -6,7 +6,7 @@ from datetime import datetime, timezone import pandas as pd import torch from msprobe.pytorch.common.log import logger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from .utils import np_save_data diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index 596ab090a..b0ba6e790 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -12,7 +12,7 @@ else: pta_cpu_device = torch.device("cpu") from msprobe.core.common.const import CompareConst, FileCheckConst -from msprobe.core.common.file_check import change_mode +from msprobe.core.common.file_utils import change_mode from msprobe.core.common.log import logger cpu_device = torch._C.device("cpu") diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py index cf97f9c6f..06de9f7fd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py @@ -23,7 +23,7 @@ from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException from msprobe.core.common.utils import create_directory, write_csv, save_npy_to_txt, load_npy -from msprobe.core.common.file_check import FileChecker +from msprobe.core.common.file_utils import FileChecker class Compare: diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py index 8f48f2784..b4e39bb48 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py @@ -28,10 +28,10 @@ from collections import namedtuple from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.file_desc import DumpDecodeFileDesc, FileDesc from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException -from msprobe.core.common.file_check import change_mode, check_other_user_writable,\ +from msprobe.core.common.file_utils import change_mode, check_other_user_writable,\ check_path_executable, check_path_owner_consistent from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.file_check import FileOpen, FileChecker +from msprobe.core.common.file_utils import FileOpen, FileChecker from msprobe.core.common.utils import check_file_or_directory_path, remove_path from msprobe.pytorch.common.log import logger diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py index 8ca69af85..680797399 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py @@ -20,7 +20,7 @@ import numpy as np from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.utils import save_npy_to_txt, load_npy diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 4b36b350b..49d5555aa 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -2,7 +2,7 @@ import json import os from msprobe.core.common_config import CommonConfig, BaseConfig -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.const import Const from msprobe.pytorch.hook_module.utils import get_ops from msprobe.core.grad_probe.constant import level_adp diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 350ae199f..51df23c9a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -7,7 +7,7 @@ from collections import namedtuple import torch from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException -from msprobe.core.common.file_check import FileChecker, check_path_before_create +from msprobe.core.common.file_utils import FileChecker, check_path_before_create from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs from msprobe.core.data_dump.scope import BaseScope diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 5151d6f26..ad34697f7 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -43,7 +43,7 @@ from msprobe.core.common.utils import (CompareException, get_dump_data_path, task_dumppath_get) -from msprobe.core.common.file_check import FileCheckConst +from msprobe.core.common.file_utils import FileCheckConst from msprobe.core.common.utils import get_json_contents, get_file_content_bytes class TestUtils(TestCase): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index 331970bf7..7d44c49e2 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -299,7 +299,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.processor._analyze_maybe_overflow_tensor(tensor_json) self.assertTrue(self.processor.has_overflow) - @patch('msprobe.core.common.file_check.path_len_exceeds_limit', return_value=False) + @patch('msprobe.core.common.file_utils.path_len_exceeds_limit', return_value=False) @patch.object(BaseDataProcessor, 'get_save_file_path', return_value=['test_api_name', 'test_api_name.0.forward.input.pt']) def test_analyze_tensor(self, mock_path_len_exceeds_limit, _): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py index fcbd05797..6b39e002b 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py @@ -5,7 +5,7 @@ import unittest from pathlib import Path from msprobe.core.common import utils -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from msprobe.core.data_dump.json_writer import DataWriter diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py b/debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py index ecdf3da9f..4517304f8 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py @@ -22,25 +22,25 @@ from unittest.mock import patch, MagicMock from msprobe.core.common.log import logger from msprobe.core.common.const import FileCheckConst from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_check import (check_link, - check_path_length, - check_path_exists, - check_path_readability, - check_path_writability, - check_path_executable, - check_other_user_writable, - check_path_owner_consistent, - check_path_pattern_vaild, - check_file_size, - check_common_file_size, - check_file_suffix, - check_path_type) +from msprobe.core.common.file_utils import (check_link, + check_path_length, + check_path_exists, + check_path_readability, + check_path_writability, + check_path_executable, + check_other_user_writable, + check_path_owner_consistent, + check_path_pattern_vaild, + check_file_size, + check_common_file_size, + check_file_suffix, + check_path_type) class TestFileCheckUtil(TestCase): @patch.object(logger, "error") def test_check_link(self, mock_logger_error): - with patch("msprobe.core.common.file_check.os.path.islink", return_value=True): + with patch("msprobe.core.common.file_utils.os.path.islink", return_value=True): with self.assertRaises(FileCheckException) as context: check_link("link_path") self.assertEqual(str(context.exception), @@ -72,7 +72,7 @@ class TestFileCheckUtil(TestCase): @patch.object(logger, "error") def test_check_path_exists(self, mock_logger_error): - with patch("msprobe.core.common.file_check.os.path.exists", return_value=False): + with patch("msprobe.core.common.file_utils.os.path.exists", return_value=False): with self.assertRaises(FileCheckException) as context: check_path_exists("file_path") self.assertEqual(str(context.exception), @@ -82,7 +82,7 @@ class TestFileCheckUtil(TestCase): @patch.object(logger, "error") def test_check_path_readability(self, mock_logger_error): path = "file_path" - with patch("msprobe.core.common.file_check.os.access", return_value=False): + with patch("msprobe.core.common.file_utils.os.access", return_value=False): with self.assertRaises(FileCheckException) as context: check_path_readability(path) self.assertEqual(str(context.exception), @@ -91,14 +91,14 @@ class TestFileCheckUtil(TestCase): mock_access = MagicMock() mock_access.return_value = True - with patch("msprobe.core.common.file_check.os.access", new=mock_access): + with patch("msprobe.core.common.file_utils.os.access", new=mock_access): check_path_readability(path) self.assertEqual(mock_access.call_args[0], (path, os.R_OK)) @patch.object(logger, "error") def test_check_path_writability(self, mock_logger_error): path = "file_path" - with patch("msprobe.core.common.file_check.os.access", return_value=False): + with patch("msprobe.core.common.file_utils.os.access", return_value=False): with self.assertRaises(FileCheckException) as context: check_path_writability(path) self.assertEqual(str(context.exception), @@ -107,14 +107,14 @@ class TestFileCheckUtil(TestCase): mock_access = MagicMock() mock_access.return_value = True - with patch("msprobe.core.common.file_check.os.access", new=mock_access): + with patch("msprobe.core.common.file_utils.os.access", new=mock_access): check_path_writability(path) self.assertEqual(mock_access.call_args[0], (path, os.W_OK)) @patch.object(logger, "error") def test_check_path_executable(self, mock_logger_error): path = "file_path" - with patch("msprobe.core.common.file_check.os.access", return_value=False): + with patch("msprobe.core.common.file_utils.os.access", return_value=False): with self.assertRaises(FileCheckException) as context: check_path_executable(path) self.assertEqual(str(context.exception), @@ -123,7 +123,7 @@ class TestFileCheckUtil(TestCase): mock_access = MagicMock() mock_access.return_value = True - with patch("msprobe.core.common.file_check.os.access", new=mock_access): + with patch("msprobe.core.common.file_utils.os.access", new=mock_access): check_path_executable(path) self.assertEqual(mock_access.call_args[0], (path, os.X_OK)) @@ -135,7 +135,7 @@ class TestFileCheckUtil(TestCase): path = "file_path" mock_stat = TestStat(0o002) - with patch("msprobe.core.common.file_check.os.stat", return_value=mock_stat): + with patch("msprobe.core.common.file_utils.os.stat", return_value=mock_stat): with self.assertRaises(FileCheckException) as context: check_other_user_writable(path) self.assertEqual(str(context.exception), @@ -147,7 +147,7 @@ class TestFileCheckUtil(TestCase): def test_check_path_owner_consistent(self, mock_logger_error): file_path = os.path.realpath(__file__) file_owner = os.stat(file_path).st_uid - with patch("msprobe.core.common.file_check.os.getuid", return_value=file_owner+1): + with patch("msprobe.core.common.file_utils.os.getuid", return_value=file_owner+1): with self.assertRaises(FileCheckException) as context: check_path_owner_consistent(file_path) self.assertEqual(str(context.exception), @@ -160,7 +160,7 @@ class TestFileCheckUtil(TestCase): path = "path" mock_re_match = MagicMock() mock_re_match.return_value = False - with patch("msprobe.core.common.file_check.re.match", new=mock_re_match): + with patch("msprobe.core.common.file_utils.re.match", new=mock_re_match): with self.assertRaises(FileCheckException) as context: check_path_pattern_vaild(path) self.assertEqual(str(context.exception), @@ -181,8 +181,8 @@ class TestFileCheckUtil(TestCase): def test_check_common_file_size(self): mock_check_file_size = MagicMock() - with patch("msprobe.core.common.file_check.os.path.isfile", return_value=True), \ - patch("msprobe.core.common.file_check.check_file_size", new=mock_check_file_size): + with patch("msprobe.core.common.file_utils.os.path.isfile", return_value=True), \ + patch("msprobe.core.common.file_utils.check_file_size", new=mock_check_file_size): for suffix, max_size in FileCheckConst.FILE_SIZE_DICT.items(): check_common_file_size(suffix) mock_check_file_size.assert_called_with(suffix, max_size) @@ -201,16 +201,16 @@ class TestFileCheckUtil(TestCase): def test_check_path_type(self, mock_logger_error): file_path = "file_path" - with patch("msprobe.core.common.file_check.os.path.isfile", return_value=False), \ - patch("msprobe.core.common.file_check.os.path.isdir", return_value=True): + with patch("msprobe.core.common.file_utils.os.path.isfile", return_value=False), \ + patch("msprobe.core.common.file_utils.os.path.isdir", return_value=True): with self.assertRaises(FileCheckException) as context: check_path_type(file_path, FileCheckConst.FILE) self.assertEqual(str(context.exception), FileCheckException.err_strs.get(FileCheckException.INVALID_FILE_ERROR)) mock_logger_error.assert_called_with(f"The {file_path} should be a file!") - with patch("msprobe.core.common.file_check.os.path.isfile", return_value=True), \ - patch("msprobe.core.common.file_check.os.path.isdir", return_value=False): + with patch("msprobe.core.common.file_utils.os.path.isfile", return_value=True), \ + patch("msprobe.core.common.file_utils.os.path.isdir", return_value=False): with self.assertRaises(FileCheckException) as context: check_path_type(file_path, FileCheckConst.DIR) self.assertEqual(str(context.exception), diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py index 6cbdc4db0..86628c774 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_compare_online_dispatch.py @@ -19,7 +19,7 @@ from pathlib import Path from unittest.mock import Mock, patch import pandas as pd -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.utils import CompareException from msprobe.pytorch.online_dispatch.compare import get_json_contents, Saver diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py index 764d844ad..a8d91151c 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py @@ -48,7 +48,7 @@ class TestUtils(unittest.TestCase): if os.path.exists(os.path.join(self.data_path, f'{self.file_name}.npy')): os.remove(os.path.join(self.data_path, f'{self.file_name}.npy')) - @patch('msprobe.core.common.file_check.change_mode') + @patch('msprobe.core.common.file_utils.change_mode') def test_np_save_data_should_error_when_input_wrong(self,mock_change_mode): np_save_data(self.data_nonumpy,self.file_name,self.data_path) mock_change_mode.assert_not_called() diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_service.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_service.py index c09b6abcb..aeaf97b71 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/test_service.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_service.py @@ -52,7 +52,7 @@ class TestService(unittest.TestCase): def test_create_dirs(self): with patch("msprobe.pytorch.service.Path.mkdir", return_value=None), \ - patch("msprobe.core.common.file_check.FileChecker.common_check", return_value=None), \ + patch("msprobe.core.common.file_utils.FileChecker.common_check", return_value=None), \ patch("msprobe.core.data_dump.data_collector.DataCollector.update_dump_paths", return_value=None): self.service.create_dirs() -- Gitee From 34ea82f671aadda1b0de8579a34df4a8302f9b3a Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Mon, 26 Aug 2024 17:29:27 +0800 Subject: [PATCH 638/791] =?UTF-8?q?[cluster=5Fanalyse]=E6=96=B0=E5=A2=9E-o?= =?UTF-8?q?=E5=8F=82=E6=95=B0=E8=B5=84=E6=96=99=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/README.md | 11 +++++------ profiler/compare_tools/README.md | 4 ++-- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/profiler/cluster_analyse/README.md b/profiler/cluster_analyse/README.md index 785056252..9b8d79c5e 100644 --- a/profiler/cluster_analyse/README.md +++ b/profiler/cluster_analyse/README.md @@ -50,11 +50,11 @@ experimental_config = torch_npu.profiler._ExperimentalConfig( 参数说明： - | 参数名 | 说明 | 是否必选 | - | --------------------- | ------------------------------------------------------------ | -------- | - | --collection_path或-d | 性能数据汇集目录，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 是 | - | --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | - + | 参数名 | 说明 | 是否必选 | + | -------------------- | ------------------------------------------------------------ | -------- | + | --profiling_path或-d | 性能数据汇集目录。未配置-o参数时，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 是 | + | --output_path或-o | 自定义输出路径，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 否 | + | --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | --mode参数说明： @@ -148,4 +148,3 @@ O列：TP Index，指集群数据按照并行策略切分后所属TP组的索引 - diff --git a/profiler/compare_tools/README.md b/profiler/compare_tools/README.md index 535bff4be..c095c64bb 100644 --- a/profiler/compare_tools/README.md +++ b/profiler/compare_tools/README.md @@ -148,8 +148,8 @@ python performance_compare.py [基准性能数据文件所在路径] [比对性 | --enable_kernel_compare | 开启kernel性能比对。仅针对NPU与NPU比对的场景。需要使用性能数据中的kernel_details.csv文件。 | 否 | | --enable_api_compare | 开启API性能比对。需要使用性能数据中的trace_view.csv文件。 | 否 | | --disable_details | 隐藏明细比对，只进行统计级比对。 | 否 | -| --base_step | 基准性能数据step ID，配置后使用基准性能数据对应step的数据进行比对。为整数，需配置实际数据存在的step ID，默认未配置，比对所有性能数据，需要与--comparsion_step同时配置。配置示例：--base_step=1。 | 否 | -| --comparsion_step | 比对性能数据step ID，配置后使用比对性能数据对应step的数据进行比对。为整数，需配置实际数据存在的step ID，默认未配置，比对所有性能数据，需要与--base_step同时配置。配置示例：--comparsion_step=1。 | 否 | +| --base_step | 基准性能数据step ID，配置后使用基准性能数据对应step的数据进行比对。为整数，需配置实际数据存在的step ID，默认未配置，比对所有性能数据，需要与--comparison_step同时配置。配置示例：--base_step=1。 | 否 | +| --comparison_step | 比对性能数据step ID，配置后使用比对性能数据对应step的数据进行比对。为整数，需配置实际数据存在的step ID，默认未配置，比对所有性能数据，需要与--base_step同时配置。配置示例：--comparison_step=1。 | 否 | 说明：以上开关均不设置的情况下，**工具默认开启所有的性能比对**，当用户设置了以上开关，则按照用户设置的开关进行性能比对，示例如下： -- Gitee From 17cd21081d812c7d03766f57153f72861ec1f317 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Mon, 2 Sep 2024 19:23:44 +0800 Subject: [PATCH 639/791] =?UTF-8?q?file=5Fcheck.py=E9=87=8D=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E6=88=90file=5Futils.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/advisor/advisor_result.py | 2 +- .../accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py | 2 +- debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py | 2 +- .../mindspore/overflow_check/kernel_graph_overflow_check.py | 2 +- debug/accuracy_tools/msprobe/mindspore/service.py | 2 +- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 2 +- debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py | 2 +- .../accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py | 2 +- debug/accuracy_tools/msprobe/pytorch/service.py | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py index d2dbbc843..350a19977 100644 --- a/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py +++ b/debug/accuracy_tools/msprobe/core/advisor/advisor_result.py @@ -20,7 +20,7 @@ import time from msprobe.core.advisor.advisor_const import AdvisorConst from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.file_check import change_mode, FileOpen +from msprobe.core.common.file_utils import change_mode, FileOpen class AdvisorResult: diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py index 7db2b8142..98fc330eb 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py @@ -2,7 +2,7 @@ import os import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen, create_directory +from msprobe.core.common.file_utils import FileOpen, create_directory class KernelGraphDump: diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py index cea9037f3..475973610 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py @@ -3,7 +3,7 @@ import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen, create_directory +from msprobe.core.common.file_utils import FileOpen, create_directory from msprobe.core.common.const import Const diff --git a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py index ae5697586..508ee8e2b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py @@ -2,7 +2,7 @@ import os import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.core.common.log import logger -from msprobe.core.common.file_check import FileOpen, create_directory +from msprobe.core.common.file_utils import FileOpen, create_directory class KernelGraphOverflowCheck: diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 40620cb4a..145623032 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -32,7 +32,7 @@ except ImportError: from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.scope import BaseScope from msprobe.mindspore.common.utils import get_rank_if_initialized -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import Const from msprobe.core.common.exceptions import DistributedNotInitializedError diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 188b62e18..7e2b67361 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,7 +1,7 @@ import os.path import torch from msprobe.core.common.const import FileCheckConst, Const -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py index add1dea38..9dd65e36a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py @@ -23,7 +23,7 @@ from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException from msprobe.core.common.utils import write_csv, save_npy_to_txt, load_npy -from msprobe.core.common.file_check import FileChecker, create_directory +from msprobe.core.common.file_utils import FileChecker, create_directory class Compare: diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py index 879329ae5..af3b5630a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/parse_tool.py @@ -23,7 +23,7 @@ from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.compare import Compare from msprobe.pytorch.parse_tool.lib.visualization import Visualization from msprobe.pytorch.parse_tool.lib.parse_exception import catch_exception, ParseException -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory class ParseTool: def __init__(self): diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 22a17f36f..7da32d0f1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -5,7 +5,7 @@ from collections import namedtuple import torch from msprobe.core.common.const import Const from msprobe.core.common.exceptions import DistributedNotInitializedError, MsprobeException -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.data_dump.data_collector import build_data_collector from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs from msprobe.core.data_dump.scope import BaseScope -- Gitee From b003c77cd9f7bcd4688f20ccf43755591618cff9 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Mon, 2 Sep 2024 19:29:11 +0800 Subject: [PATCH 640/791] =?UTF-8?q?file=5Fcheck.py=E9=87=8D=E5=91=BD?= =?UTF-8?q?=E5=90=8D=E6=88=90file=5Futils.py?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/compare/ms_graph_compare.py | 4 ++-- .../msprobe/mindspore/dump/kernel_graph_dump.py | 2 +- .../accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py | 2 +- .../msprobe/mindspore/grad_probe/global_context.py | 2 +- .../msprobe/mindspore/grad_probe/grad_analyzer.py | 2 +- .../mindspore/overflow_check/kernel_graph_overflow_check.py | 2 +- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 2 +- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 2 +- .../accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py | 2 +- debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py | 2 +- 10 files changed, 11 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index 69d95ca5e..f971130c5 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -8,12 +8,12 @@ import numpy as np import pandas as pd from msprobe.core.common.const import CompareConst, GraphMode from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.mindspore.common.log import logger from msprobe.core.common.utils import add_time_with_xlsx, CompareException from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen class row_data: def __init__(self, mode): diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py index 6c07cfbb8..00e0ac577 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_graph_dump.py @@ -2,7 +2,7 @@ import os import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_check import FileOpen, create_directory +from msprobe.core.common.file_utils import FileOpen, create_directory class KernelGraphDump: diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py index 52a4c816c..9c0a979b6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/kernel_kbyk_dump.py @@ -3,7 +3,7 @@ import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_check import FileOpen, create_directory +from msprobe.core.common.file_utils import FileOpen, create_directory from msprobe.core.common.const import Const diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py index 1b2a11ac6..d36f560f2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -5,7 +5,7 @@ from typing import Dict, Union from msprobe.core.grad_probe.utils import check_str from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.utils import check_path_before_create diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index 146f892c3..1ec71820d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -13,7 +13,7 @@ from mindspore.common.parameter import Parameter from msprobe.core.grad_probe.utils import ListCache from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext diff --git a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py index 0e41c6bea..4892e9de8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/overflow_check/kernel_graph_overflow_check.py @@ -2,7 +2,7 @@ import os import json from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_check import FileOpen, create_directory +from msprobe.core.common.file_utils import FileOpen, create_directory class KernelGraphOverflowCheck: diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index d42a19d96..69dc04fdb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -28,7 +28,7 @@ from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.log import logger from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, CompareException -from msprobe.core.common.file_check import FileCheckConst, change_mode, FileOpen +from msprobe.core.common.file_utils import FileCheckConst, change_mode, FileOpen try: diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 17730d742..e9ba8f42b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,7 +1,7 @@ import os.path import torch from msprobe.core.common.const import FileCheckConst, Const -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index b577b2fab..786ac7fca 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -7,7 +7,7 @@ if int(torch.__version__.split('.')[0]) >= 2: from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target from msprobe.core.grad_probe.constant import GradConst, level_adp -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import remove_path, write_csv, save_npy from msprobe.pytorch.common.utils import get_rank_id, print_rank_0, save_pt diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index 85786bf60..0f120c23d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -12,7 +12,7 @@ else: pta_cpu_device = torch.device("cpu") from msprobe.core.common.const import CompareConst, FileCheckConst -from msprobe.core.common.file_check import change_mode +from msprobe.core.common.file_utils import change_mode from msprobe.pytorch.common.log import logger cpu_device = torch._C.device("cpu") -- Gitee From 540ae80c967c045c5fb8a4341df426c634694b2d Mon Sep 17 00:00:00 2001 From: jijiarong Date: Mon, 2 Sep 2024 19:39:07 +0800 Subject: [PATCH 641/791] code review --- .../msprobe/mindspore/common/utils.py | 29 ++++++++++++- .../mindspore/compare/ms_graph_compare.py | 43 ++++++------------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index dc4444e78..9b64abef3 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -12,11 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ - +import os import mindspore as ms from msprobe.core.common.exceptions import DistributedNotInitializedError -from msprobe.core.common.file_check import path_len_exceeds_limit +from msprobe.core.common.file_check import path_len_exceeds_limit, check_path_exists from msprobe.core.common.utils import save_npy from msprobe.core.common.log import logger @@ -43,6 +43,31 @@ def save_tensor_as_npy(tensor, file_path): logger.warning(f'The file path {file_path} length exceeds limit.') +def convert_to_int(value): + try: + return int(value) + except ValueError: + return -1 + + +def list_lowest_level_directories(root_dir): + check_path_exists(root_dir) + lowest_level_dirs = [] + + def recurse_dirs(current_dir): + for entry in os.listdir(current_dir): + full_path = os.path.join(current_dir, entry) + if os.path.isdir(full_path): + if any(os.path.isdir(os.path.join(full_path, subentry)) for subentry in os.listdir(full_path)): + recurse_dirs(full_path) + else: + lowest_level_dirs.append(full_path) + + recurse_dirs(root_dir) + return lowest_level_dirs + + + class MsprobeStep(ms.train.Callback): def __init__(self, debugger): diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index 8c449ad58..06cb6533a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -11,6 +11,7 @@ from msprobe.core.common.log import logger from msprobe.core.common.utils import add_time_with_xlsx, CompareException, load_npy from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply +from msprobe.mindspore.common.utils import convert_to_int, list_lowest_level_directories class row_data: @@ -27,38 +28,15 @@ class row_data: return self.data -def list_lowest_level_directories(root_dir): - lowest_level_dirs = [] - - def recurse_dirs(current_dir): - for entry in os.listdir(current_dir): - full_path = os.path.join(current_dir, entry) - if os.path.isdir(full_path): - if any(os.path.isdir(os.path.join(full_path, subentry)) for subentry in os.listdir(full_path)): - recurse_dirs(full_path) - else: - lowest_level_dirs.append(full_path) - - recurse_dirs(root_dir) - return lowest_level_dirs - - -def generate_id(value): - try: - return int(value) - except ValueError: - return -1 - - def npy_data_read(data_path, npy_file_list, mapping_dict): data_list = [] for data in npy_file_list: if data in mapping_dict: - split_list = mapping_dict[data].split(".") + split_list = mapping_dict[data].split(CompareConst.SEP) else: split_list = data.split(".") compare_key = f"{split_list[1]}.{split_list[2]}.{split_list[3]}.{split_list[5]}.{split_list[6]}" - timestamp = int(split_list[4]) + timestamp = convert_to_int(split_list[4]) data_list.append([os.path.join(data_path, data), compare_key, timestamp]) return data_list @@ -141,8 +119,8 @@ class GraphMSComparator: self.output_path = output_path self.base_npu_path = input_param.get('npu_path', None) self.base_bench_path = input_param.get('bench_path', None) - self.rank_list = [generate_id(rank_id) for rank_id in input_param.get('rank_id', [])] - self.step_list = [generate_id(step_id) for step_id in input_param.get('step_id', [])] + self.rank_list = [convert_to_int(rank_id) for rank_id in input_param.get('rank_id', [])] + self.step_list = [convert_to_int(step_id) for step_id in input_param.get('step_id', [])] # split by rank and step, generate rank step path self.npu_rank_step_dict = self.generate_rank_step_path(self.base_npu_path) self.bench_rank_step_dict = self.generate_rank_step_path(self.base_bench_path) @@ -153,13 +131,16 @@ class GraphMSComparator: def generate_rank_step_id(path_with_rank_step): split_path = path_with_rank_step.split("/") + rank_id = -1 if "rank_" in path_with_rank_step: # KBK mode - rank_id = generate_id(split_path[-4].split("_")[-1]) - step_id = generate_id(split_path[-1]) + if len(split_path > 4): + rank_id = convert_to_int(split_path[-4].split("_")[-1]) + step_id = convert_to_int(split_path[-1]) else: - rank_id = generate_id(split_path[-4]) - step_id = generate_id(split_path[-1]) + if len(split_path > 4): + rank_id = convert_to_int(split_path[-4]) + step_id = convert_to_int(split_path[-1]) return rank_id, step_id base_path = os.path.abspath(base_path) -- Gitee From aeb06530f8f355f8462f5e985eadb86f105f959f Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Mon, 2 Sep 2024 20:03:12 +0800 Subject: [PATCH 642/791] =?UTF-8?q?=E5=88=A0=E9=99=A4utils.py=E4=B8=AD?= =?UTF-8?q?=E7=9A=84=E9=87=8D=E5=A4=8Dcheck=5Ffile=5Fsize=E6=96=B9?= =?UTF-8?q?=E6=B3=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/core/common/file_utils.py | 6 +++++- debug/accuracy_tools/msprobe/core/common/utils.py | 12 ------------ .../msprobe/test/core_ut/common/test_utils.py | 3 +-- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index ee56c68b5..f749e26fc 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -192,7 +192,11 @@ def check_path_pattern_vaild(path): def check_file_size(file_path, max_size): - file_size = os.path.getsize(file_path) + try: + file_size = os.path.getsize(file_path) + except OSError as os_error: + logger.error('Failed to open "%s". %s' % (file_path, str(os_error))) + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) from os_error if file_size >= max_size: logger.error(f'The size of file path {file_path} exceeds {max_size} bytes.') raise FileCheckException(FileCheckException.FILE_TOO_LARGE_ERROR) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 613e424f7..cea5eaa43 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -197,18 +197,6 @@ def check_json_file(input_param, npu_json, bench_json, stack_json): _check_json(stack_json, input_param.get("stack_json_path")) -def check_file_size(input_file, max_size): - try: - file_size = os.path.getsize(input_file) - except OSError as os_error: - logger.error('Failed to open "%s". %s' % (input_file, str(os_error))) - raise CompareException(CompareException.INVALID_FILE_ERROR) from os_error - if file_size > max_size: - logger.error('The size (%d) of %s exceeds (%d) bytes, tools not support.' - % (file_size, input_file, max_size)) - raise CompareException(CompareException.INVALID_FILE_ERROR) - - def check_file_not_exists(file_path): if os.path.exists(file_path) or os.path.islink(file_path): remove_path(file_path) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 9001b2749..d87c87dff 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -37,12 +37,11 @@ from msprobe.core.common.utils import (CompareException, is_starts_with, _check_json, check_json_file, - check_file_size, check_regex_prefix_format_valid, get_dump_data_path, task_dumppath_get) -from msprobe.core.common.file_utils import FileCheckConst +from msprobe.core.common.file_utils import FileCheckConst, check_file_size from msprobe.core.common.utils import get_json_contents, get_file_content_bytes class TestUtils(TestCase): -- Gitee From f7a5c2174361cc1af90dcd088b5860fe6c4bc304 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 2 Sep 2024 20:13:11 +0800 Subject: [PATCH 643/791] fix --- .../msprobe/pytorch/bench_functions/npu_fusion_attention.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py index 5098065ee..530a12a93 100644 --- a/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py +++ b/debug/accuracy_tools/msprobe/pytorch/bench_functions/npu_fusion_attention.py @@ -58,7 +58,7 @@ def softmax_grad(dp, softmax_res): def broadcast_kv(num_heads, num_kv_heads, kv_tensor, dtype): if num_kv_heads == 0 or num_kv_heads > num_heads: - raise ValueError(f"num_kv_heads must be non-zero and less than num_heads.") + raise ValueError(f"num_kv_heads must be non-zero and bigger than num_heads.") factor = num_heads // num_kv_heads kv_shape = kv_tensor.shape @@ -281,7 +281,7 @@ def rebuild_softmax_by_max_sum(q, k, atten_mask, pse, scale, softmax_max, softma def get_head_num(*args, **kwargs): - if kwargs.get("head_num"): + if kwargs.get("head_num", None): head_num = kwargs.get("head_num") elif len(args) >= 4: head_num = args[3] @@ -291,7 +291,7 @@ def get_head_num(*args, **kwargs): def get_input_layout(*args, **kwargs): - if kwargs.get("input_layout"): + if kwargs.get("input_layout", None): input_layout = kwargs.get("input_layout") elif len(args) >= 5: input_layout = args[4] -- Gitee From f6268c7da437f4870c51200b79ea7e3e0fbbda71 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 2 Sep 2024 20:31:23 +0800 Subject: [PATCH 644/791] fix ut import --- .../msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py index ba5ac46ba..dcc67963b 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py @@ -7,7 +7,7 @@ import hashlib import mindspore from mindspore import nn, Tensor from mindspore.nn import SGD -from grad_tool.grad_monitor import GradientMonitor +from msprobe.mindspore.grad_probe.grad_monitor import GradientMonitor file_path = os.path.abspath(__file__) -- Gitee From f1d7579bdb4e01e75bbf036cb259083d745d7469 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 2 Sep 2024 20:42:38 +0800 Subject: [PATCH 645/791] auto create dir --- .../mindspore/api_accuracy_checker/api_accuracy_checker.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index bb96644c6..9e8402018 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -1,7 +1,7 @@ import json import os -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_check import FileOpen, create_directory from msprobe.core.common.utils import write_csv, add_time_as_suffix from msprobe.core.common.const import Const, CompareConst, MsCompareConst from msprobe.core.common.log import logger @@ -201,6 +201,7 @@ class ApiAccuracyChecker: detail_csv.append(csv_row) file_name = os.path.join(csv_dir, add_time_as_suffix(MsCompareConst.DETAIL_CSV_FILE_NAME)) + create_directory(csv_dir) write_csv(detail_csv, file_name, mode="w") @@ -250,4 +251,5 @@ class ApiAccuracyChecker: result_csv.append(row) file_name = os.path.join(csv_dir, add_time_as_suffix(MsCompareConst.RESULT_CSV_FILE_NAME)) + create_directory(csv_dir) write_csv(result_csv, file_name, mode="w") \ No newline at end of file -- Gitee From 115f9134cd252694880f6b0144cd67baf4bd7602 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Mon, 2 Sep 2024 20:47:31 +0800 Subject: [PATCH 646/791] rename --- .../grad_probe/{test_grad_monitor.py => test_ms_grad_monitor.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/{test_grad_monitor.py => test_ms_grad_monitor.py} (100%) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py similarity index 100% rename from debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_grad_monitor.py rename to debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py -- Gitee From 7e30ab0520fce42136786bcd661e58455a4412db Mon Sep 17 00:00:00 2001 From: hhqx Date: Sat, 31 Aug 2024 16:21:04 +0800 Subject: [PATCH 647/791] Add divide zero check --- .../cluster_advice/slow_link_advice.py | 2 ++ .../ai_core_freq/ai_core_freq_checker.py | 2 ++ .../cluster_utils/parallel_algorithm.py | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+) diff --git a/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py b/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py index f8a625242..8d2993262 100644 --- a/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py +++ b/profiler/advisor/advisor_backend/cluster_advice/slow_link_advice.py @@ -96,6 +96,8 @@ class SlowLinkAdvice(ClusterAdviceBase): def produce_bottleneck(self, link_type: str): data_list = [rank_dict.get(link_type, 0) for rank_id, rank_dict in self.rank_bw_dict.items()] + if len(data_list) == 0: + raise ValueError("Cannot calculate avg_bw, data_list is empty!") avg_bw = round(sum(data_list) / len(data_list), 3) if avg_bw == 0: return diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py index c8a94287d..05c5cd25e 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_checker.py @@ -47,6 +47,8 @@ class AICoreFreqChecker: op_total_duration = round(op_info.get("dur", 0), 2) max_freq = convert_to_float(Config().get_config("aic_frequency")) + if max_freq == 0: + raise ValueError("max_freq cannot be zero.") decrease_freq_ratio = sum(max_freq - freq for freq in freq_list) / (max_freq * len(freq_list)) if decrease_freq_ratio >= Config().get_config("frequency_threshold"): self.ai_core_freq_issues = True diff --git a/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py b/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py index 9da829bbd..283b93ec1 100644 --- a/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py +++ b/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py @@ -13,6 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. from abc import ABC, abstractmethod +import logging + +logger = logging.getLogger() class ParallelAlgorithm(ABC): @@ -30,6 +33,22 @@ class MegatronAlgorithm(ParallelAlgorithm): context_parallel_size: int = 1, expert_model_parallel_size: int = 1, **kwargs): + # Check for zero and adjust parallel sizes to avoid division by zero + if tensor_model_parallel_size == 0: + tensor_model_parallel_size = 1 + logger.error("tensor_model_parallel_size cannot be zero.") + + if pipeline_model_parallel_size == 0: + pipeline_model_parallel_size = 1 + logger.error("pipeline_model_parallel_size cannot be zero.") + + if data_parallel_size == 0: + data_parallel_size = 1 + logger.error("data_parallel_size cannot be zero.") + + if expert_model_parallel_size == 0: + expert_model_parallel_size = 1 + logger.error("expert_model_parallel_size cannot be zero.") if data_parallel_size % expert_model_parallel_size != 0: raise RuntimeError( -- Gitee From b6890ecade3ac1c85d3f51c95f2356afc29db5e5 Mon Sep 17 00:00:00 2001 From: lijiaojiao Date: Mon, 2 Sep 2024 09:45:21 +0800 Subject: [PATCH 648/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91online=5Fdispat?= =?UTF-8?q?ch=E6=97=A0=E6=B3=95=E5=9C=A8=E7=BA=BF=E6=AF=94=E5=AF=B9torch.i?= =?UTF-8?q?nt32=E7=B1=BB=E5=9E=8B=E6=95=B0=E6=8D=AEbug=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/pytorch/online_dispatch/single_compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py index 1408b4778..83a6bafc6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py @@ -157,8 +157,8 @@ class SingleBenchmarkAccuracyCompare: max_rel_idx=max_rel_idx ) acc_result.get_result(eb_thd, error_thd) - return CompareResultInfo(acc_result, error_thd, eb_thd, None) - return None + return CompareResultInfo(acc_result, error_thd, eb_thd, None) + @classmethod @func_log_wrapper() @@ -191,7 +191,7 @@ class SingleBenchmarkAccuracyCompare: zeros = torch.zeros_like(npu_out) diff_value = torch.subtract(npu_out, bench_out) diff_abs = torch.abs(diff_value) - abs_mask_idx = torch.where(torch.abs(bench_out) < benchmark_standard.small_value, ones, zeros) + abs_mask_idx = torch.where(torch.abs(bench_out) >= benchmark_standard.small_value, ones, zeros) abs_err_idx = torch.where(diff_abs > error_thd, ones, zeros) abs_err_idx = abs_err_idx * abs_mask_idx abs_err = diff_abs[torch.where(abs_err_idx == 1)] @@ -354,7 +354,7 @@ def calc_status_details_dict(npu_out, bench_out, high_precision, summary): def calc_status_details_tensor(npu_out, bench_out, high_precision, summary): - return single_benchmark_compare(bench_out, npu_out) + return single_benchmark_compare(npu_out, bench_out) def calc_status_details_builtin(npu_out, bench_out, summary): -- Gitee From fb232d3987faab3f9017b1951880d3c4343ad5a4 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 2 Sep 2024 21:26:17 +0800 Subject: [PATCH 649/791] safety problem bugfix --- debug/accuracy_tools/msprobe/core/common/utils.py | 2 ++ debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 1593efc38..5a415a3e2 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -61,6 +61,8 @@ class CompareException(Exception): OVER_SIZE_FILE_ERROR = 18 INVALID_SUMMARY_MODE = 19 INVALID_TASK_ERROR = 20 + DETACH_ERROR = 21 + def __init__(self, code, error_info: str = ""): super(CompareException, self).__init__() diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 8f07856bb..3fbef2263 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -24,10 +24,12 @@ class PTComparator (Comparator): to_cpu=True).detach() # detach because numpy can not process gradient information except RuntimeError as e: # 这里捕获 load_pt 中抛出的异常 - raise RuntimeError(f"Failed to load the .pt file at {data_path}.") from e + logger.error(f"Failed to load the .pt file at {data_path}.") + raise CompareException(CompareException.INVALID_FILE_ERROR) from e except AttributeError as e: # 这里捕获 detach 方法抛出的异常 - raise RuntimeError(f"Failed to detach the loaded tensor.") from e + logger.error(f"Failed to detach the loaded tensor.") + raise CompareException(CompareException.DETACH_ERROR) from e if data_value.dtype == torch.bfloat16: data_value = data_value.to(torch.float32) data_value = data_value.numpy() -- Gitee From 015e3720ded08bbca10d211430a2b7c6dcd0090a Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Mon, 2 Sep 2024 17:18:42 +0800 Subject: [PATCH 650/791] mindspore dump, overflow_check md --- .../msprobe/docs/02.config_introduction.md | 10 +- .../msprobe/docs/06.data_dump_MindSpore.md | 146 ++++++++++++++---- .../docs/13.overflow_check_MindSpore.md | 20 +++ 3 files changed, 143 insertions(+), 33 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md diff --git a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md index 845bfb3d9..0b6a4a137 100644 --- a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md +++ b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md @@ -16,19 +16,21 @@ | dump_path | 设置 dump 数据目录路径，str 类型。
**配置示例**："dump_path": "./dump_path"。MindSpore 场景仅支持绝对路径。 | 是 | | rank | 指定对某张卡上的数据进行 dump，list[int] 类型，默认未配置（表示 dump 所有卡的数据），应配置为 ≥0 的整数，且须配置实际可用的 Rank ID。
PyTorch 场景：Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，dump 数据为空。
**配置示例**："rank": [1]。
MindSpore 场景：所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
**配置示例**："rank": [1]。 | 否 | | step | 指定 dump 某个 step 的数据，list[int] 类型。默认未配置，表示 dump 所有 step 数据。dump 特定 step 时，须指定为训练脚本中存在的 step。step 为 list 格式，可配置逐个 step。
**配置示例**："step": [0,1,2]。 | 否 | -| level | dump 级别，str 类型，根据不同级别 dump 不同数据。可取值，
"L0"：dump module 模块级精度数据，仅 PyTorch 场景支持，使用背景详见 [1.1.1 模块级精度数据 dump 说明（仅 PyTorch 场景支持）](#111-模块级精度数据-dump-说明仅-pytorch-场景支持)；
"L1"：dump API 级精度数据，默认值；
"L2"：dump kernel 级精度数据，须配置 acl_config 参数；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 场景支持。
**配置示例**："level": "L1"。 | 否 | +| level | dump 级别，str 类型，根据不同级别 dump 不同数据。可取值，
"L0"：dump 模块级精度数据，仅 PyTorch 与 MindSpore 动态图场景支持，使用背景详见 [1.1.1 模块级精度数据 dump 说明](#111-模块级精度数据-dump-说明)；
"L1"：dump API 级精度数据，默认值，仅 PyTorch 与 MindSpore 动态图场景支持；
"L2"：dump kernel 级精度数据，PyTorch场景下须配置 acl_config 参数；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 场景支持。
**配置示例**："level": "L1"。 | 否 | | acl_config | kernel dump 的配置文件，str 类型。level 取"L2"时，该参数必要；level 为其他值时，该参数不选。
**配置示例**：acl_config="./acl_config.json"。具体配置见[ acl_config 示例](./04.acl_config_examples.md)。 | 否 | | seed | 随机种子数，int 类型，默认值为：1234，仅 PyTorch 场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见 [1.1.2 固定随机数范围（仅 PyTorch 场景支持）](#112-固定随机数范围仅-pytorch-场景支持)。
**配置示例**："seed": 1234。 | 否 | | is_deterministic | 确定性计算模式，bool 类型，仅 PyTorch 场景支持。可取值 true（开启）或 false（关闭），默认关闭。
即使在相同的硬件和输入下，API 多次执行的结果也可能不同，开启确定性计算是为了保证在相同的硬件和输入下，API 多次执行的结果相同。
确定性计算会导致 API 执行性能降低，建议在发现模型多次执行结果不同的情况下开启。
rnn 类算子、ReduceSum、ReduceMean 等算子可能与确定性计算存在冲突，若开启确定性计算后多次执行的结果不相同，则考虑存在这些算子。
**配置示例**："is_deterministic": true。
| 否 | | enable_dataloader | 自动控制开关，bool 类型，仅 PyTorch 场景支持。可取值 true（开启）或 false（关闭），默认为 false。配置为 True 后自动识别 step 参数指定的迭代，并在该迭代执行完成后退出训练，此时 start、stop 和 step 函数可不配置，开启该开关要求训练脚本是通过 torch.utils.data.dataloader 方式加载数据。仅支持 PyTorch 单卡训练使用，分布式训练场景下存在数据 dump 不全问题，**下个版本即将废弃该功能**。 | 否 | -#### 1.1.1 模块级精度数据 dump 说明（仅 PyTorch 场景支持） +#### 1.1.1 模块级精度数据 dump 说明 + +仅 PyTorch 与 MindSpore 动态图场景支持。 大模型场景下，通常不是简单的利用自动迁移能力实现从 GPU 到 NPU 的训练脚本迁移，而是会对 NPU 网络进行一系列针对性的适配，因此，常常会造成迁移后的 NPU 模型存在部分子结构不能与 GPU 原始模型完全对应。模型结构不一致导致 API 调用类型及数量不一致，若直接按照 API 粒度进行精度数据 dump 和比对，则无法完全比对所有的 API。 本小节介绍的功能是对模型中的大粒度模块进行数据 dump，使其比对时，对于无法以 API 粒度比对的模块可以直接以模块粒度进行比对。 -模块指的是继承自 nn.Module 类模块，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump 数据时以模块为粒度进行 dump。 +模块指的是继承 nn.Module 类（PyTorch场景）或 nn.Cell 类（MindSpore场景）的子类，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump 数据时以模块为粒度进行 dump。 #### 1.1.2 固定随机数范围（仅 PyTorch 场景支持） @@ -36,7 +38,7 @@ seed_all 函数可固定随机数的范围如下表。 | API | 固定随机数 | | ---------------------------------------- | --------------------------- | -| os.environ['PYTHONHASHSEED'] = str(seed) | 禁止 Python 中的 hash 随机化 | +| os.environ['PYTHONHASHSEED'] = str(seed) | 禁止 Python 中的 hash 随机化 | | random.seed(seed) | 设置 random 随机生成器的种子 | | np.random.seed(seed) | 设置 numpy 中随机生成器的种子 | | torch.manual_seed(seed) | 设置当前 CPU 的随机种子 | diff --git a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md index b8304ec5c..b38cae641 100644 --- a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md @@ -1,8 +1,10 @@ # MindSpore 场景的精度数据采集 -msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 +msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。目前，静态图场景仅支持 kernel 级数据采集，对应 config.json 配置中的 "L2" level；动态图场景支持cell、API、kernel级数据采集，对应 config.json 配置中的 "L0"、"L1" 、"L2" level。 -本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml 文件内手动修改，如下示例： +精度数据采集功能的配置示例见[MindSpore 静态图场景下 task 配置为 statistics](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#21-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-statistics)、[MindSpore 静态图场景下 task 配置为 tensor](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#22-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-tensor)、[MindSpore 动态图场景下 task 配置为 statistics](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#31-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-statistics)、[MindSpore 动态图场景下 task 配置为 tensor](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#32-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-tensor)。 + +动态图 API 级 dump 时，本工具提供固定的 API 支持列表，仅支持对列表中的 API 进行精度数据采集。一般情况下，无需修改该列表，而是通过config.json中的scope/list字段进行 dump API 指定。若需要改变 API 支持列表，可以在 `msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml` 文件内手动修改，如下示例： ```yaml ops: # ops为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API @@ -13,7 +15,7 @@ ops: # ops为算子类别，找到对应的类别，在该类别下按照下列 ## 1 接口介绍 -### 1.1 PrecisionDebugger +### 1.1 msprobe.mindspore.PrecisionDebugger **功能说明**：通过加载 dump 配置文件的方式来确定 dump 操作的详细配置。 @@ -23,73 +25,153 @@ ops: # ops为算子类别，找到对应的类别，在该类别下按照下列 PrecisionDebugger(config_path=None) ``` -1. config_path：指定 dump 配置文件路径，string 类型。参数示例："./config.json"。未配置该路径时，默认使用 [config.json](../config.json) 文件的默认配置。 +1. config_path：指定 dump 配置文件路径，string 类型。参数示例："./config.json"。未配置该路径时，默认使用 [config.json](../config.json) 文件的默认配置，配置选项含义可见 [config.json 介绍](./02.config_introduction.md)。 -### 1.2 start +#### 1.1.1 start -**功能说明**：启动精度数据采集。在模型初始化之后的位置添加。需要与 stop 函数一起添加在 for 循环内。 +**功能说明**：启动精度数据采集。需在模型执行模式（静态图/动态图、O0/O1/O2编译等级）设置后调用。 +静态图场景下，必须在模型初始化及 mindspore.communication.init 调用前添加；动态图场景下，如果进行kernel级别（"L2" level）dump或没有使用 [Model](https://www.mindspore.cn/tutorials/zh-CN/r2.3.1/advanced/model.html) 高阶 API 进行训练，则需要与 stop 函数一起添加在 for 循环内，否则只有需要传入model参数时，才使用该接口。 **原型**： ```Python -debugger.start(model=None) +start(model=None) ``` -1. model：指具体的 mindspore.nn.Cell，默认未配置，L1 级别下传入 model 可以采集 primitive op。 -### 1.3 stop +1. model：指具体的 mindspore.nn.Cell，默认不配置。Cell级别（"L0" level）dump 时，传入 model 可以采集 model 内的所有Cell 对象数据。API级别（"L1" level）dump 时，传入 model 可以采集 model 内包含 primitive op 对象在内的所有 API 数据，若不传入 model 参数，则只采集非 primitive op 的 API 数据。 + +#### 1.1.2 stop -**功能说明**：停止数据采集。在 **start** 函数之后的任意位置添加。需要与 start 函数一起添加在 for 循环内。若需要 dump 反向数据，则需要添加在反向计算代码之后。**仅 MindSpore 动态图场景支持。** +**功能说明**：停止数据采集。在 **start** 函数之后的任意位置添加。需要与 start 函数一起添加在 for 循环内。若需要 dump 反向数据，则需要添加在反向计算代码之后。**仅未使用 Model 高阶 API 的动态图场景支持。** **原型**： ```Python -debugger.stop() +stop() ``` -### 1.4 step +#### 1.1.3 step -**功能说明**：在最后一个 **stop** 函数后或一个 step 结束的位置添加。**仅 MindSpore 动态图场景支持。** +**功能说明**：在最后一个 **stop** 函数后或一个 step 训练结束的位置添加。**仅未使用 Model 高阶 API 的动态图场景支持。** **原型**： ```Python -debugger.step() +step() ``` +### 1.2 msprobe.mindspore.common.utils.MsprobeStep + +**功能说明**：MindSpore Callback类，自动在每个step开始时调用start()接口，在每个step结束时调用stop()、step()接口。实现使用 Model 高阶 API 的动态图场景下 L0、L1 级别的精度数据采集控制，控制粒度为单个 **Step** ，而 PrecisionDebugger.start, PrecisionDebugger.stop 接口的控制粒度任意训练代码段。 + +**原型**： + +```Python +MsprobeStep(debugger) +``` + +1. debugger：PrecisionDebugger对象。 + ## 2 示例代码 ### 2.1 MindSpore 静态图场景 ```Python +import mindspore as ms +ms.set_context(mode=ms.GRAPH_MODE, device_target="Ascend") + from msprobe.mindspore import PrecisionDebugger debugger = PrecisionDebugger(config_path="./config.json") -# 请勿将以上初始化流程插入到循环代码中 -# 下面代码也可以用PrecisionDebugger.start() debugger.start() -... +# 请勿将以上初始化流程置于模型实例化或mindspore.communication.init调用后 +# ... ``` ### 2.2 MindSpore 动态图场景 +#### 2.2.1 未使用 Model 高阶 API(非 L2 级别) + ```Python import mindspore as ms -from msprobe.mindspore import PrecisionDebugger +ms.set_context(mode=ms.PYNATIVE_MODE, device_target="Ascend") -# 请勿将PrecisionDebugger的初始化插入到循环代码中 +from msprobe.mindspore import PrecisionDebugger debugger = PrecisionDebugger(config_path="./config.json") # 模型、损失函数的定义以及初始化等操作 # ... +model = Network() +# 数据集迭代的地方往往是模型开始训练的地方 +for data, label in data_loader: + debugger.start() # 进行L1级别下非primitive op采集时调用 + # debugger.start(model) # 进行L0级别或L1级别下primitive op的数据采集时调用 + # 如下是模型每个step执行的逻辑 + grad_net = ms.grad(model)(data) + # ... + debugger.stop() # 关闭数据dump + debugger.step() # 结束一个step的dump +``` + +#### 2.2.2 未使用 Model 高阶 API(L2 级别) + +```Python +import mindspore as ms +ms.set_context(mode=ms.PYNATIVE_MODE, device_target="Ascend") + +from msprobe.mindspore import PrecisionDebugger +debugger = PrecisionDebugger(config_path="./config.json") +debugger.start() +# 请勿将以上初始化流程置于模型实例化或mindspore.communication.init调用后 +# 模型、损失函数的定义以及初始化等操作 +# ... +model = Network() # 数据集迭代的地方往往是模型开始训练的地方 for data, label in data_loader: - debugger.start() # 开启数据dump - net = Model() # 如下是模型每个step执行的逻辑 - grad_net = ms.grad(net)(data) + grad_net = ms.grad(model)(data) # ... - debugger.stop() # 关闭数据dump - debugger.step() # 结束一个step的dump +``` + +#### 2.2.3 使用 Model 高阶 API(非 L2 级别) + +```Python +import mindspore as ms +from mindspore.train import Model +ms.set_context(mode=ms.PYNATIVE_MODE, device_target="Ascend") + +from msprobe.mindspore import PrecisionDebugger +from msprobe.mindspore.common.utils import MsprobeStep +debugger = PrecisionDebugger(config_path="./config.json") + +# 模型、损失函数的定义以及初始化等操作 +# ... + +model = Network() +# 只有进行L0级别下Cell对象或L1级别下primitive op的数据采集时才需要调用 +# debugger.start(model) +trainer = Model(model, loss_fn=loss_fn, optimizer=optimizer, metrics={'accuracy'}) +trainer.train(1, train_dataset, callbacks=[MsprobeStep(debugger)]) +``` + +#### 2.2.4 使用 Model 高阶 API(L2 级别) + +```Python +import mindspore as ms +from mindspore.train import Model +ms.set_context(mode=ms.PYNATIVE_MODE, device_target="Ascend") + +from msprobe.mindspore import PrecisionDebugger +debugger = PrecisionDebugger(config_path="./config.json") +debugger.start() +# 请勿将以上初始化流程置于模型实例化或mindspore.communication.init调用后 + +# 模型、损失函数的定义以及初始化等操作 +# ... + +model = Network() +trainer = Model(model, loss_fn=loss_fn, optimizer=optimizer, metrics={'accuracy'}) +trainer.train(1, train_dataset) ``` ## 3 dump 结果文件介绍 @@ -100,13 +182,13 @@ for data, label in data_loader: - jit_level 为O0/O1时： - dump 结果目录请参见 MindSpore 官网中的[同步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%90%8C%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)。 + dump 结果目录请参见 MindSpore 官网中的[同步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.1/debug/dump.html#%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95%E5%92%8C%E6%95%B0%E6%8D%AE%E6%96%87%E4%BB%B6%E4%BB%8B%E7%BB%8D)。 - jit_level 为O2时： - dump 结果目录请参见 MindSpore 官网中的[异步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.0rc2/debug/dump.html#%E5%BC%82%E6%AD%A5dump%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95)。 + dump 结果目录请参见 MindSpore 官网中的[异步 Dump 数据对象目录](https://www.mindspore.cn/tutorials/experts/zh-CN/r2.3.1/debug/dump.html#%E6%95%B0%E6%8D%AE%E5%AF%B9%E8%B1%A1%E7%9B%AE%E5%BD%95%E5%92%8C%E6%95%B0%E6%8D%AE%E6%96%87%E4%BB%B6%E4%BB%8B%E7%BB%8D-1)。 -jit_level 请参见 [mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.0/api_python/mindspore/mindspore.JitConfig.html#mindspore-jitconfig) 配置 jit_config。 +jit_level 请参见 [mindspore.set_context](https://www.mindspore.cn/docs/zh-CN/r2.3.1/api_python/mindspore/mindspore.set_context.html) 中的 jit_config 参数。 ### 3.2 MindSpore 动态图场景 @@ -123,8 +205,9 @@ dump结果目录结构示例如下： | | | | ├── Mint.abs.0.forward.input.0.npy | | | | ├── Functional.split.0.forward.input.0.npy | | | | ... -| | | | └── Tensor.__add__.0.forward.output.0.npy -│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：`{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}`）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） +| | | | └── Jit.AlexNet.0.forward.input.0.npy +| | | | └── Cell.relu.ReLU.forward.0.input.0.npy # config.json文件配置level为L0时dump的cell模块级数据，命名格式为{Cell}_{cell_name}_{class_name}_{前向反向}.{index}.{input/output}.{参数序号} +│ | | ├── dump.json # 保存前反向算子、算子的统计量信息或溢出算子信息。包含dump数据的API名称（命名格式为：{api_type}_{api_name}_{API调用次数}_{前向反向}_{input/output}.{参数序号}）、dtype、 shape、各数据的max、min、mean、L2norm统计信息以及当配置summary_mode="md5"时的md5数据。其中，“参数序号”表示该API下的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个子参数；L2norm表示L2范数（平方根） │ | | ├── stack.json # 算子调用栈信息 │ | | └── construct.json # 分层分级结构，level为L1时，construct.json内容为空 │ | ├── rank1 @@ -145,11 +228,16 @@ dump 过程中，npy 文件在对应算子或者模块被执行后就会落盘 其中 rank 为设备上各卡的 ID，每张卡上 dump 的数据会生成对应 dump 目录。非分布式场景下没有 rank ID，目录名称为 rank。 +动态图场景下使能 PSJit 或 PIJit，装饰特定 Cell 或 function，被装饰的部分会全部/部分使能**静态图**流程。PSJit 场景下 config.json 文件配置 level 为 L1 时，被 PSJit 装饰的部分也作为 API 被 dump 到对应目录；配置 level 为 L2 时，则只会 dump 用户网络中静态图流程下的相关 kernel, 其结果目录同jit_level 为 O0/O1 时的静态图 dump 相同。PIJit 场景开启 dump 工具后，会被还原为动态图，按 API 粒度进行 dump。 + npy 文件保存的前缀和 MindSpore 对应关系如下： | 前缀 | MindSpore 模块 | | -------------- | ---------------------------- | | Tensor | mindspore.Tensor | | Functional | mindspore.ops | +| Primitive | mindspore.ops.Primitive | | Mint | mindspore.mint | | MintFunctional | mindspore.mint.nn.functional | +| Jit | mindspore.jit | +| Cell | mindspore.nn.Cell | diff --git a/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md b/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md new file mode 100644 index 000000000..2d8eb0d53 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md @@ -0,0 +1,20 @@ +# MindSpore 场景的溢出检测 + +msprobe 工具提供静态图O2编译等级下的过程溢出检测与动态图场景下的结果溢出检测。其中前者检测对象为 kernel 级别，对应 config.json 配置中的 "L2" level，后者检测对象为 API 级别，对应 config.json 配置中的 "L1" level。 + +溢出检测任务的配置示例见[MindSpore 静态图场景下 task 配置为 overflow_check](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#23-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-overflow_check)、[MindSpore 动态图场景下 task 配置为 overflow_check](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#33-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-overflow_check)。 + + +## 1 接口介绍 + +溢出检测功能提供的接口与数据采集任务一致，详见[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)中的"**1 接口介绍**"章节。 + +需要注意，目前暂不支持动态图 "L1" level 下 primitive op 的溢出检测。 + +## 2 示例代码 + +溢出检测功能使用方式与数据采集任务一致，详见[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)中的"**2 示例代码**"章节。 + +## 3 溢出检测结果文件介绍 + +溢出检测结果文件目录结构与含义与数据采集任务一致，详见[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)中的"**3 dump 结果文件介绍**"章节。 -- Gitee From dca4d6030cbdeb4148994df2403aef8712f7c874 Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Tue, 27 Aug 2024 09:52:20 +0800 Subject: [PATCH 651/791] bugfix --- profiler/cluster_analyse/cluster_analysis.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 3add0ab99..8a32b14ef 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -15,6 +15,7 @@ import argparse import os +import logging from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor @@ -25,6 +26,8 @@ from common_func.path_manager import PathManager from analysis.analysis_facade import AnalysisFacade COMM_FEATURE_LIST = ['all', 'communication_time', 'communication_matrix'] +logger = logging.getLogger() + class Interface: ASCEND_PT = "ascend_pt" @@ -100,8 +103,10 @@ def cluster_analysis_main(args=None): Constant.ANALYSIS_MODE: args_parsed.mode, Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: args_parsed.output_path } - - Interface(parameter).run() + try: + Interface(parameter).run() + except Exception as e: + logger.error(e) if __name__ == "__main__": cluster_analysis_main() -- Gitee From 6b159ceecfb10e1b44e7b820fce6ae5c440f62ec Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 3 Sep 2024 10:42:44 +0800 Subject: [PATCH 652/791] fix --- .../grad_probe/test_ms_grad_monitor.py | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index dcc67963b..47fa37681 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -1,21 +1,21 @@ import os import numpy as np import shutil -import yaml +import json from unittest import TestCase import hashlib import mindspore from mindspore import nn, Tensor from mindspore.nn import SGD -from msprobe.mindspore.grad_probe.grad_monitor import GradientMonitor +from msprobe.mindspore import PrecisionDebugger file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) - +config_json_path = os.path.join(directory, "config.json") def main(): - gm = GradientMonitor(os.path.join(directory, "config.yaml"), framework="MindSpore") + debugger = PrecisionDebugger(config_json_path) class SimpleNet(nn.Cell): def __init__(self): @@ -29,7 +29,7 @@ def main(): model = SimpleNet() optimizer = SGD(model.trainable_params(), learning_rate=0.001) - gm.monitor(optimizer) + debugger.monitor(optimizer) fix_gradient = tuple([Tensor(np.arange(5*16).reshape((5, 16)), dtype=mindspore.float32), Tensor(np.arange(5).reshape(5), dtype=mindspore.float32)]) @@ -40,11 +40,10 @@ def main(): optimizer(fix_gradient) -def save_dict_as_yaml(data, yaml_file_path): - # 将字典保存为YAML文件 - with open(yaml_file_path, 'w') as yaml_file: - yaml.dump(data, yaml_file, default_flow_style=False) - print(f"字典已保存为YAML文件: {yaml_file_path}") +def save_dict_as_json(data, json_file_path): + with open(json_file_path, 'w') as f: + json.dump(data, f, ensure_ascii=False, indent=4) + print(f"字典已保存为json文件: {json_file_path}") def get_hash(file_path): @@ -56,14 +55,14 @@ def get_hash(file_path): def check_npy(gradient_output_path): - my_dense_bias_path = os.path.join(gradient_output_path, "rank_0", "step_1", "my_dense.bias.npy") + my_dense_bias_path = os.path.join(gradient_output_path, "rank0", "step1", "my_dense.bias.npy") assert os.path.isfile(my_dense_bias_path) my_dense_bias_real = np.load(my_dense_bias_path) my_dense_bias_target = np.arange(5).reshape(5) > 0 assert (my_dense_bias_real == my_dense_bias_target).all() - my_dense_weight_path = os.path.join(gradient_output_path, "rank_0", "step_1", "my_dense.weight.npy") + my_dense_weight_path = os.path.join(gradient_output_path, "rank0", "step1", "my_dense.weight.npy") assert os.path.isfile(my_dense_weight_path) my_dense_weight_real = np.load(my_dense_weight_path) my_dense_weight_target = np.arange(5*16).reshape((5, 16)) > 0 @@ -77,20 +76,21 @@ def check_stat_csv(csv_path, md5_value): class TestMsGradientMonitor(TestCase): def test_gradient_monitor(self): - config_path = os.path.join(directory, "config.yaml") gradient_output_path = os.path.join(directory, "gradient_output") config_dict = { - "level": "L2", - "param_list": None, - "rank": None, + "task": "grad_probe", + "dump_path": gradient_output_path, + "rank": [], "step": [1], - "bounds": None, - "output_path": gradient_output_path, + "grad_probe": { + "grad_level": "L2", + "param_list": [] + } } - save_dict_as_yaml(config_dict, config_path) + save_dict_as_json(config_dict, config_json_path) main() check_npy(gradient_output_path) - check_stat_csv(os.path.join(gradient_output_path, "rank_0", "grad_summary_1.csv"), "b00499bee5c3e3ca96aaee773a092dd7") - os.remove(config_path) + check_stat_csv(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv"), "874174395c56922f86118050e8c93e74") + os.remove(config_json_path) shutil.rmtree(gradient_output_path) \ No newline at end of file -- Gitee From f76944afbb2aba5dc4c7fef23a46e5d8d2681ef3 Mon Sep 17 00:00:00 2001 From: tangmengcheng <745274877@qq.com> Date: Tue, 3 Sep 2024 11:24:59 +0800 Subject: [PATCH 653/791] safe --- .../cluster_kernels_analysis/cluster_prof_Info_analysis.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py index 27e3c229c..992d59176 100644 --- a/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py +++ b/profiler/cluster_analyse/cluster_kernels_analysis/cluster_prof_Info_analysis.py @@ -54,6 +54,7 @@ class FormDataProcessor: if "mindstudio_profiler_output" in f: continue # 判断csv文件大小 + PathManager.check_file_size(f) PathManager.check_path_readable(f) # 读取CSV文件 df = pd.read_csv(f) @@ -81,6 +82,8 @@ class FormDataProcessor: def getChipType(self): file = self.files[0] + PathManager.check_file_size(file) + PathManager.check_path_readable(file) df = pd.read_csv(file) if 'aiv_time(us)' in df.columns: return "ASCEND_NEW" @@ -159,10 +162,10 @@ class OpSummaryAnalyzerBase: # 创建结果文件 self.result_dir = os.path.join(dir_path, "result") PathManager.check_path_length(self.result_dir) - if os.path.exists(self.result_dir): - shutil.rmtree(self.result_dir, onerror=self.on_rm_error) + PathManager.remove_path_safety(self.result_dir) PathManager.check_path_writeable(dir_path) PathManager.make_dir_safety(self.result_dir) + PathManager.check_path_writeable(self.result_dir) def getColumnsToGroup(self): return self.columns_to_group -- Gitee From 5cea1d207937e1ac56f632ad8a7c814acc21488d Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 3 Sep 2024 14:22:57 +0800 Subject: [PATCH 654/791] =?UTF-8?q?=E5=B0=86check=5Ffile=5Fnot=5Fexists,lo?= =?UTF-8?q?ad=5Fyaml,remove=5Fpath=E7=A7=BB=E5=8A=A8=E5=88=B0file=5Futils?= =?UTF-8?q?=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/file_utils.py | 33 +++++++++++++++++++ .../msprobe/core/common/utils.py | 31 ----------------- .../msprobe/core/compare/acc_compare.py | 5 +-- .../msprobe/core/data_dump/json_writer.py | 3 +- .../msprobe/core/grad_probe/grad_compare.py | 3 +- .../msprobe/mindspore/compare/ms_compare.py | 4 +-- .../dump/hook_cell/wrap_functional.py | 3 +- .../mindspore/dump/hook_cell/wrap_tensor.py | 3 +- .../free_benchmark/api_pynative_self_check.py | 2 +- .../mindspore/grad_probe/grad_analyzer.py | 3 +- .../msprobe/mindspore/grad_probe/hook.py | 3 +- .../api_accuracy_checker/common/config.py | 2 +- .../compare/compare_utils.py | 3 +- .../run_ut/multi_run_ut.py | 2 +- .../tensor_transport_layer/attl.py | 2 +- .../msprobe/pytorch/compare/match.py | 3 +- .../pytorch/grad_probe/grad_monitor.py | 5 +-- .../msprobe/pytorch/hook_module/utils.py | 2 +- .../msprobe/pytorch/hook_module/wrap_aten.py | 2 +- .../pytorch/hook_module/wrap_distributed.py | 2 +- .../pytorch/hook_module/wrap_functional.py | 2 +- .../pytorch/hook_module/wrap_npu_custom.py | 2 +- .../pytorch/hook_module/wrap_tensor.py | 2 +- .../msprobe/pytorch/hook_module/wrap_torch.py | 2 +- .../msprobe/pytorch/hook_module/wrap_vf.py | 2 +- .../pytorch/online_dispatch/dispatch.py | 4 +-- .../msprobe/pytorch/parse_tool/lib/utils.py | 3 +- .../core_ut/data_dump/test_json_writer.py | 7 ++-- 28 files changed, 76 insertions(+), 64 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index f749e26fc..5dd625fcf 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -16,10 +16,13 @@ """ import os import re +import shutil +import yaml from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.const import FileCheckConst +from msprobe.core.common.utils import CompareException class FileChecker: @@ -257,6 +260,11 @@ def check_path_before_create(path): 'The file path {} contains special characters.'.format(path)) +def check_file_not_exists(file_path): + if os.path.exists(file_path) or os.path.islink(file_path): + remove_path(file_path) + + def change_mode(path, mode): if not os.path.exists(path) or os.path.islink(path): return @@ -288,3 +296,28 @@ def check_file_type(path): else: logger.error('Neither a file nor a directory.') raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) + + +def load_yaml(yaml_path): + path_checker = FileChecker(yaml_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.YAML_SUFFIX) + checked_path = path_checker.common_check() + try: + with FileOpen(checked_path, "r") as f: + yaml_data = yaml.safe_load(f) + except Exception as e: + logger.error(f"The yaml file failed to load. Please check the path: {checked_path}.") + raise RuntimeError(f"Load yaml file {checked_path} failed.") from e + return yaml_data + + +def remove_path(path): + if not os.path.exists(path): + return + try: + if os.path.islink(path) or os.path.isfile(path): + os.remove(path) + else: + shutil.rmtree(path) + except PermissionError as err: + logger.error("Failed to delete {}. Please check the permission.".format(path)) + raise CompareException(CompareException.INVALID_PATH_ERROR) from err diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index cea5eaa43..3aa7db364 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -24,7 +24,6 @@ import time import json import csv from datetime import datetime, timezone -import yaml import numpy as np from msprobe.core.common.file_utils import FileOpen, FileChecker, change_mode, check_path_before_create @@ -197,11 +196,6 @@ def check_json_file(input_param, npu_json, bench_json, stack_json): _check_json(stack_json, input_param.get("stack_json_path")) -def check_file_not_exists(file_path): - if os.path.exists(file_path) or os.path.islink(file_path): - remove_path(file_path) - - def check_regex_prefix_format_valid(prefix): """ validate the format of the regex prefix @@ -223,19 +217,6 @@ def check_regex_prefix_format_valid(prefix): raise ValueError(f"prefix contains invalid characters, prefix pattern {Const.REGEX_PREFIX_PATTERN}") -def remove_path(path): - if not os.path.exists(path): - return - try: - if os.path.islink(path) or os.path.isfile(path): - os.remove(path) - else: - shutil.rmtree(path) - except PermissionError as err: - logger.error("Failed to delete {}. Please check the permission.".format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) from err - - def move_file(src_path, dst_path): check_file_or_directory_path(src_path) check_path_before_create(dst_path) @@ -533,18 +514,6 @@ def get_file_content_bytes(file): return file_handle.read() -def load_yaml(yaml_path): - path_checker = FileChecker(yaml_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.YAML_SUFFIX) - checked_path = path_checker.common_check() - try: - with FileOpen(checked_path, "r") as f: - yaml_data = yaml.safe_load(f) - except Exception as e: - logger.error(f"The yaml file failed to load. Please check the path: {checked_path}.") - raise RuntimeError(f"Load yaml file {checked_path} failed.") from e - return yaml_data - - def save_workbook(workbook, file_path): """ 保存工作簿到指定的文件路径 diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 492e30ba9..988e5ad39 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -6,10 +6,11 @@ from msprobe.core.common.file_utils import FileOpen from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger -from msprobe.core.common.utils import add_time_with_xlsx, CompareException, check_file_not_exists +from msprobe.core.common.utils import add_time_with_xlsx, CompareException +from msprobe.core.common.file_utils import check_file_not_exists from msprobe.core.compare.check import check_graph_mode, check_struct_match, fuzzy_check_op from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx -from msprobe.core.compare.utils import read_op, merge_tensor, CompareException, get_un_match_accuracy, get_accuracy +from msprobe.core.compare.utils import read_op, merge_tensor, get_un_match_accuracy, get_accuracy from msprobe.core.compare.multiprocessing_compute import _handle_multi_process, ComparisonResult, _save_cmp_result from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_type, reshape_value, get_relative_err, \ get_error_message diff --git a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py index 0f0c37f15..ad0722325 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py @@ -4,7 +4,8 @@ import csv from msprobe.core.common.file_utils import change_mode, FileOpen from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.utils import load_json, save_json, remove_path +from msprobe.core.common.utils import load_json, save_json +from msprobe.core.common.file_utils import remove_path class DataWriter: diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 0f871d378..7b36cc4eb 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -8,7 +8,8 @@ import matplotlib.pyplot as plt from msprobe.core.common.utils import check_file_or_directory_path from msprobe.core.common.file_utils import create_directory, check_path_before_create from msprobe.core.common.log import logger -from msprobe.core.common.utils import remove_path, write_csv, load_npy +from msprobe.core.common.utils import write_csv, load_npy +from msprobe.core.common.file_utils import remove_path from msprobe.core.grad_probe.constant import GradConst diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 32443f67e..6fcda39e4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,7 +1,7 @@ import os.path from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, \ - task_dumppath_get, load_yaml, load_npy -from msprobe.core.common.file_utils import create_directory + task_dumppath_get, load_npy +from msprobe.core.common.file_utils import create_directory, load_yaml from msprobe.core.common.const import Const from msprobe.mindspore.common.log import logger from msprobe.core.common.exceptions import FileCheckException diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py index 489ca55bb..27c73c4ae 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py @@ -16,7 +16,8 @@ import os import mindspore as ms from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell -from msprobe.core.common.utils import Const, load_yaml +from msprobe.core.common.utils import Const +from msprobe.core.common.file_utils import load_yaml cur_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py index 7c919a1b6..eda473dc1 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py @@ -16,7 +16,8 @@ import os import mindspore as ms from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell -from msprobe.core.common.utils import Const, load_yaml +from msprobe.core.common.utils import Const +from msprobe.core.common.file_utils import load_yaml cur_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index e000b44b5..d0795a5fd 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -5,7 +5,7 @@ import importlib import mindspore as ms from mindspore.communication import comm_func -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml from msprobe.core.common.const import Const from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.config import Config diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index 1ec71820d..aa6eb07ee 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -14,7 +14,8 @@ from msprobe.core.grad_probe.utils import ListCache from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger from msprobe.core.common.file_utils import create_directory -from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file +from msprobe.core.common.utils import check_file_or_directory_path, write_csv, move_file +from msprobe.core.common.file_utils import remove_path from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py index 1b98ee8e7..6b7328b18 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py @@ -11,7 +11,8 @@ from mindspore.common.initializer import initializer from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.utils import write_csv, remove_path +from msprobe.core.common.utils import write_csv +from msprobe.core.common.file_utils import remove_path from msprobe.mindspore.grad_probe.global_context import grad_context from msprobe.mindspore.grad_probe.grad_analyzer import grad_dump, get_rank_id from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index 14478dfef..4d46b6298 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -1,7 +1,7 @@ import os import yaml from msprobe.core.common.utils import check_file_or_directory_path -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml from msprobe.pytorch.pt_config import RunUTConfig diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py index 4c2b921e1..70ca31fc0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare_utils.py @@ -4,7 +4,8 @@ import math import torch -from msprobe.core.common.utils import CompareException, load_yaml +from msprobe.core.common.utils import CompareException +from msprobe.core.common.file_utils import load_yaml from msprobe.core.common.const import Const from msprobe.pytorch.common.log import logger diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py index 4971a7c84..42a1c66fc 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/multi_run_ut.py @@ -17,7 +17,7 @@ from msprobe.pytorch.common import parse_json_info_forward_backward from msprobe.pytorch.common.log import logger from msprobe.core.common.file_utils import FileChecker, check_file_suffix, check_link, FileOpen, \ check_path_before_create, create_directory -from msprobe.core.common.utils import remove_path +from msprobe.core.common.file_utils import remove_path from msprobe.core.common.const import FileCheckConst diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index e74eebb19..b2a71e56f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -12,7 +12,7 @@ from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.client import TCPClient from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.server import TCPServer from msprobe.pytorch.common.utils import logger -from msprobe.core.common.utils import remove_path +from msprobe.core.common.file_utils import remove_path from msprobe.pytorch.common.utils import save_api_data, load_api_data, save_pt, load_pt BufferType = Union[ApiData, Dict[str, Any], str] # Union[Tensor, Tuple[Optional[Tensor]]] diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/match.py b/debug/accuracy_tools/msprobe/pytorch/compare/match.py index ac445ad8e..2b77d0297 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/match.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/match.py @@ -1,5 +1,6 @@ import os -from msprobe.core.common.utils import CompareException, load_yaml +from msprobe.core.common.utils import CompareException +from msprobe.core.common.file_utils import load_yaml class AtenIrMapping(): diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index 786ac7fca..fb5e0c51b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -9,8 +9,9 @@ from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_lis from msprobe.core.grad_probe.constant import GradConst, level_adp from msprobe.core.common.file_utils import create_directory from msprobe.pytorch.common.log import logger -from msprobe.core.common.utils import remove_path, write_csv, save_npy -from msprobe.pytorch.common.utils import get_rank_id, print_rank_0, save_pt +from msprobe.core.common.utils import write_csv, save_npy +from msprobe.core.common.file_utils import remove_path +from msprobe.pytorch.common.utils import get_rank_id, print_rank_0 class GradientMonitor: diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py index dbcc9bfa3..4fbc7a7f8 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/utils.py @@ -16,7 +16,7 @@ """ import os -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml def get_ops(): diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py index b63bd9d02..651d63f89 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py @@ -21,7 +21,7 @@ import torch from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml from msprobe.pytorch.function_factory import npu_custom_grad_functions cur_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py index 1f720a32b..3ccda8cf9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_distributed.py @@ -22,7 +22,7 @@ import torch.distributed as dist from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml cur_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py index 95715ec1a..98bd17b27 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_functional.py @@ -22,7 +22,7 @@ from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const from msprobe.pytorch.common.log import logger -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml def remove_dropout(): diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py index c2300470e..43952c988 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_npu_custom.py @@ -21,7 +21,7 @@ import torch from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard, torch_without_guard_version from msprobe.core.common.const import Const -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml from msprobe.pytorch.function_factory import npu_custom_functions cur_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py index 90bb3c61b..df086096c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_tensor.py @@ -22,7 +22,7 @@ import torch from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard, parameter_adapter from msprobe.core.common.const import Const -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml cur_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py index 32d086a6d..034646dd7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_torch.py @@ -21,7 +21,7 @@ import torch from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard from msprobe.core.common.const import Const -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml cur_path = os.path.dirname(os.path.realpath(__file__)) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py index 022535824..43bfdbc0f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_vf.py @@ -19,7 +19,7 @@ import os import torch from msprobe.core.common.const import Const -from msprobe.core.common.utils import load_yaml +from msprobe.core.common.file_utils import load_yaml from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.common.utils import torch_device_guard diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index 9199fe5bb..994bda369 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -15,8 +15,8 @@ except ImportError: else: is_npu = True -from msprobe.core.common.utils import check_file_or_directory_path, load_yaml -from msprobe.core.common.file_utils import check_path_before_create +from msprobe.core.common.utils import check_file_or_directory_path +from msprobe.core.common.file_utils import check_path_before_create, load_yaml from msprobe.core.common.const import Const, CompareConst from msprobe.pytorch.common.log import logger from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py index b4e39bb48..484ce8921 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py @@ -32,7 +32,8 @@ from msprobe.core.common.file_utils import change_mode, check_other_user_writabl check_path_executable, check_path_owner_consistent from msprobe.core.common.const import FileCheckConst from msprobe.core.common.file_utils import FileOpen, FileChecker -from msprobe.core.common.utils import check_file_or_directory_path, remove_path +from msprobe.core.common.utils import check_file_or_directory_path +from msprobe.core.common.file_utils import remove_path from msprobe.pytorch.common.log import logger diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py index 6b39e002b..919d94dce 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_json_writer.py @@ -4,8 +4,7 @@ import os import unittest from pathlib import Path -from msprobe.core.common import utils -from msprobe.core.common.file_utils import FileOpen +from msprobe.core.common.file_utils import FileOpen, remove_path from msprobe.core.data_dump.json_writer import DataWriter @@ -15,7 +14,7 @@ class TestDataWriter(unittest.TestCase): file_path = os.path.join(cur_path, "test.csv") if os.path.exists(file_path): - utils.remove_path(file_path) + remove_path(file_path) data = {"A": "1", "B": "2", "C": "3"} result = data.values() @@ -35,7 +34,7 @@ class TestDataWriter(unittest.TestCase): column_last = [row for row in reader][-1] self.assertEqual(data, column_last) - utils.remove_path(file_path) + remove_path(file_path) def test_initialize_json_file(self): cur_path = os.path.dirname(os.path.realpath(__file__)) -- Gitee From b3349ab1f6ac2de0be0a8242137b77ec76387588 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 3 Sep 2024 14:36:04 +0800 Subject: [PATCH 655/791] refresh single --- .../msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index 47fa37681..702142cbd 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -15,6 +15,7 @@ directory = os.path.dirname(file_path) config_json_path = os.path.join(directory, "config.json") def main(): + PrecisionDebugger._instance = None debugger = PrecisionDebugger(config_json_path) class SimpleNet(nn.Cell): -- Gitee From d50b86804bd86a03f1cd5fdbfb9d842255b0d423 Mon Sep 17 00:00:00 2001 From: jijiarong Date: Tue, 3 Sep 2024 14:48:28 +0800 Subject: [PATCH 656/791] save bugfix --- .../mindspore/compare/ms_graph_compare.py | 22 +++++-------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index 69d95ca5e..d36735bf6 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -8,12 +8,12 @@ import numpy as np import pandas as pd from msprobe.core.common.const import CompareConst, GraphMode from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_check import FileOpen from msprobe.mindspore.common.log import logger -from msprobe.core.common.utils import add_time_with_xlsx, CompareException +from msprobe.core.common.utils import add_time_with_xlsx, CompareException, load_npy from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply -from msprobe.core.common.file_check import FileOpen + class row_data: def __init__(self, mode): @@ -58,7 +58,7 @@ def statistic_data_read(statistic_file_list, statistic_file_path): data_list = [] statistic_data_list = [] for statistic_file in statistic_file_list: - with open(statistic_file, "r") as f: + with FileOpen(statistic_file, "r") as f: csv_reader = csv.reader(f, delimiter=",") header = next(csv_reader) header_index = {'Data Type': None, 'Shape': None, 'Max Value': None, 'Min Value': None, @@ -130,18 +130,6 @@ def generate_data_name(data_path): return mode, data_list -def read_npy_data(data_path): - try: - data_value = np.load(data_path) - if data_value.dtype == np.float16: - data_value = data_value.astype(np.float32) - except FileNotFoundError as e: - data_value = None - except EOFError: - data_value = None - return data_value - - class GraphMSComparator: def __init__(self, input_param, output_path): self.output_path = output_path @@ -158,7 +146,7 @@ class GraphMSComparator: def process_npy_file(file_path, name_prefix, result): if os.path.exists(file_path): - data = read_npy_data(file_path) + data = load_npy(file_path) result[f'{name_prefix} Name'] = file_path result[f'{name_prefix} Dtype'] = data.dtype result[f'{name_prefix} Tensor Shape'] = data.shape -- Gitee From 546f196bb8433bdf246fb7a8658e585303c14064 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 3 Sep 2024 15:13:02 +0800 Subject: [PATCH 657/791] =?UTF-8?q?=E5=B0=86check=5Ffile=5For=5Fdirectory?= =?UTF-8?q?=5Fpath=E7=A7=BB=E5=8A=A8=E5=88=B0file=5Futils=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/file_utils.py | 17 ++++++++++++++++ .../msprobe/core/common/utils.py | 20 ++----------------- .../msprobe/core/compare/utils.py | 3 ++- .../msprobe/core/grad_probe/grad_compare.py | 3 +-- .../mindspore/grad_probe/grad_analyzer.py | 8 +++----- .../msprobe/mindspore/grad_probe/utils.py | 7 +++---- .../api_accuracy_checker/common/config.py | 4 +--- .../msprobe/pytorch/common/utils.py | 7 ++----- .../pytorch/online_dispatch/dispatch.py | 3 +-- .../msprobe/pytorch/parse_tool/lib/utils.py | 7 +------ .../msprobe/test/core_ut/common/test_utils.py | 3 +-- 11 files changed, 34 insertions(+), 48 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index 5dd625fcf..5f4fed9f1 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -265,6 +265,23 @@ def check_file_not_exists(file_path): remove_path(file_path) +def check_file_or_directory_path(path, isdir=False): + """ + Function Description: + check whether the path is valid + Parameter: + path: the path to check + isdir: the path is dir or file + Exception Description: + when invalid data throw exception + """ + if isdir: + path_checker = FileChecker(path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) + else: + path_checker = FileChecker(path, FileCheckConst.FILE, FileCheckConst.READ_ABLE) + path_checker.common_check() + + def change_mode(path, mode): if not os.path.exists(path) or os.path.islink(path): return diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 3aa7db364..74b36eb2d 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -26,7 +26,8 @@ import csv from datetime import datetime, timezone import numpy as np -from msprobe.core.common.file_utils import FileOpen, FileChecker, change_mode, check_path_before_create +from msprobe.core.common.file_utils import (FileOpen, FileChecker, + change_mode, check_path_before_create, check_file_or_directory_path) from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.core.common.log import logger @@ -161,23 +162,6 @@ def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=F raise CompareException(CompareException.INVALID_PARAM_ERROR) -def check_file_or_directory_path(path, isdir=False): - """ - Function Description: - check whether the path is valid - Parameter: - path: the path to check - isdir: the path is dir or file - Exception Description: - when invalid data throw exception - """ - if isdir: - path_checker = FileChecker(path, FileCheckConst.DIR, FileCheckConst.WRITE_ABLE) - else: - path_checker = FileChecker(path, FileCheckConst.FILE, FileCheckConst.READ_ABLE) - path_checker.common_check() - - def is_starts_with(string, prefix_list): return any(string.startswith(prefix) for prefix in prefix_list) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index ad985865c..9cf02b953 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -3,7 +3,8 @@ import os import re import numpy as np from msprobe.core.common.const import Const, CompareConst -from msprobe.core.common.utils import CompareException, check_file_or_directory_path, check_regex_prefix_format_valid, logger +from msprobe.core.common.utils import CompareException, check_regex_prefix_format_valid, logger +from msprobe.core.common.file_utils import check_file_or_directory_path def extract_json(dirname, stack_json=False): diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 7b36cc4eb..61b206a61 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -5,8 +5,7 @@ from tqdm import tqdm import pandas as pd import matplotlib.pyplot as plt -from msprobe.core.common.utils import check_file_or_directory_path -from msprobe.core.common.file_utils import create_directory, check_path_before_create +from msprobe.core.common.file_utils import create_directory, check_path_before_create, check_file_or_directory_path from msprobe.core.common.log import logger from msprobe.core.common.utils import write_csv, load_npy from msprobe.core.common.file_utils import remove_path diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index aa6eb07ee..905b2f450 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -1,21 +1,19 @@ import os import time -from typing import List, Tuple +from typing import List import multiprocessing from multiprocessing import Process import numpy as np import mindspore as ms from mindspore.communication import get_rank -from mindspore.ops import operations as P from mindspore.common.parameter import Parameter from msprobe.core.grad_probe.utils import ListCache from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_utils import create_directory -from msprobe.core.common.utils import check_file_or_directory_path, write_csv, move_file -from msprobe.core.common.file_utils import remove_path +from msprobe.core.common.file_utils import create_directory, check_file_or_directory_path, remove_path +from msprobe.core.common.utils import write_csv, move_file from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py index 301b6b3f8..e56ce51f9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py @@ -1,11 +1,10 @@ import os -import numpy as np import mindspore -from msprobe.core.grad_probe.constant import GradConst, level_adp +from msprobe.core.grad_probe.constant import level_adp from msprobe.core.grad_probe.utils import check_param -from msprobe.core.common.file_utils import create_directory, check_path_before_create -from msprobe.core.common.utils import change_mode, check_file_or_directory_path, save_npy +from msprobe.core.common.file_utils import create_directory, check_path_before_create, check_file_or_directory_path +from msprobe.core.common.utils import save_npy def save_grad_direction(param_name, grad, save_path): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index 4d46b6298..61a163a2a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -1,7 +1,5 @@ import os -import yaml -from msprobe.core.common.utils import check_file_or_directory_path -from msprobe.core.common.file_utils import load_yaml +from msprobe.core.common.file_utils import load_yaml, check_file_or_directory_path from msprobe.pytorch.pt_config import RunUTConfig diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 69dc04fdb..457032742 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -15,20 +15,17 @@ # limitations under the License. """ import io -import logging import os import random import stat -import csv -import json import torch import torch.distributed as dist import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.log import logger -from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, CompareException -from msprobe.core.common.file_utils import FileCheckConst, change_mode, FileOpen +from msprobe.core.common.utils import check_path_before_create +from msprobe.core.common.file_utils import FileCheckConst, change_mode, check_file_or_directory_path try: diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index 994bda369..8d1a0c378 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -15,8 +15,7 @@ except ImportError: else: is_npu = True -from msprobe.core.common.utils import check_file_or_directory_path -from msprobe.core.common.file_utils import check_path_before_create, load_yaml +from msprobe.core.common.file_utils import check_path_before_create, check_file_or_directory_path, load_yaml from msprobe.core.common.const import Const, CompareConst from msprobe.pytorch.common.log import logger from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py index 484ce8921..81918a624 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/utils.py @@ -14,14 +14,11 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -import logging import os -import io import re import sys import subprocess import hashlib -import csv import time import numpy as np from collections import namedtuple @@ -31,9 +28,7 @@ from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException from msprobe.core.common.file_utils import change_mode, check_other_user_writable,\ check_path_executable, check_path_owner_consistent from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.file_utils import FileOpen, FileChecker -from msprobe.core.common.utils import check_file_or_directory_path -from msprobe.core.common.file_utils import remove_path +from msprobe.core.common.file_utils import FileChecker, check_file_or_directory_path, remove_path from msprobe.pytorch.common.log import logger diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index d87c87dff..e1e4c2578 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -31,7 +31,6 @@ from msprobe.core.common.utils import (CompareException, check_dump_mode_valid, check_summary_mode_valid, check_summary_only_valid, - check_file_or_directory_path, check_compare_param, check_configuration_param, is_starts_with, @@ -41,7 +40,7 @@ from msprobe.core.common.utils import (CompareException, get_dump_data_path, task_dumppath_get) -from msprobe.core.common.file_utils import FileCheckConst, check_file_size +from msprobe.core.common.file_utils import FileCheckConst, check_file_size, check_file_or_directory_path from msprobe.core.common.utils import get_json_contents, get_file_content_bytes class TestUtils(TestCase): -- Gitee From bc584778cbd3c900b20113dd132b0dab6b995ddb Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 3 Sep 2024 15:39:42 +0800 Subject: [PATCH 658/791] assert -> assertTrue --- .../grad_probe/test_ms_grad_monitor.py | 41 +++++++++---------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index 702142cbd..117c884ca 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -55,26 +55,6 @@ def get_hash(file_path): return hash_object.hexdigest() -def check_npy(gradient_output_path): - my_dense_bias_path = os.path.join(gradient_output_path, "rank0", "step1", "my_dense.bias.npy") - assert os.path.isfile(my_dense_bias_path) - my_dense_bias_real = np.load(my_dense_bias_path) - my_dense_bias_target = np.arange(5).reshape(5) > 0 - - assert (my_dense_bias_real == my_dense_bias_target).all() - - my_dense_weight_path = os.path.join(gradient_output_path, "rank0", "step1", "my_dense.weight.npy") - assert os.path.isfile(my_dense_weight_path) - my_dense_weight_real = np.load(my_dense_weight_path) - my_dense_weight_target = np.arange(5*16).reshape((5, 16)) > 0 - - assert (my_dense_weight_real == my_dense_weight_target).all() - -def check_stat_csv(csv_path, md5_value): - real_md5_value = get_hash(csv_path) - assert real_md5_value == md5_value - - class TestMsGradientMonitor(TestCase): def test_gradient_monitor(self): gradient_output_path = os.path.join(directory, "gradient_output") @@ -91,7 +71,24 @@ class TestMsGradientMonitor(TestCase): save_dict_as_json(config_dict, config_json_path) main() - check_npy(gradient_output_path) - check_stat_csv(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv"), "874174395c56922f86118050e8c93e74") + + my_dense_bias_path = os.path.join(gradient_output_path, "rank0", "step1", "my_dense.bias.npy") + self.assertTrue(os.path.isfile(my_dense_bias_path), "bias npy file not found") + my_dense_bias_real = np.load(my_dense_bias_path) + my_dense_bias_target = np.arange(5).reshape(5) > 0 + + self.assertTrue((my_dense_bias_real == my_dense_bias_target).all(), "bias ndarray not same as target") + + my_dense_weight_path = os.path.join(gradient_output_path, "rank0", "step1", "my_dense.weight.npy") + self.assertTrue(os.path.isfile(my_dense_weight_path), "weight npy file not found") + my_dense_weight_real = np.load(my_dense_weight_path) + my_dense_weight_target = np.arange(5*16).reshape((5, 16)) > 0 + + self.assertTrue((my_dense_weight_real == my_dense_weight_target).all(), "weight ndarray not same as target") + + real_md5_value = get_hash(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv")) + target_md5_value = "874174395c56922f86118050e8c93e74" + self.assertTrue(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") + os.remove(config_json_path) shutil.rmtree(gradient_output_path) \ No newline at end of file -- Gitee From be8e4dce3a49b4210f97cb5f7e5d8f61ddede129 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 3 Sep 2024 15:41:37 +0800 Subject: [PATCH 659/791] fix --- .../test/mindspore_ut/grad_probe/test_ms_grad_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index 117c884ca..612f7a10e 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -88,7 +88,7 @@ class TestMsGradientMonitor(TestCase): real_md5_value = get_hash(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv")) target_md5_value = "874174395c56922f86118050e8c93e74" - self.assertTrue(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") + self.assertEqual(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") os.remove(config_json_path) shutil.rmtree(gradient_output_path) \ No newline at end of file -- Gitee From aa84d196f718b5f6a5c5a81bb551d4c0cd659d12 Mon Sep 17 00:00:00 2001 From: hhqx Date: Tue, 3 Sep 2024 16:07:07 +0800 Subject: [PATCH 660/791] fix log info in parallel_algorithm.py --- .../cluster_analyse/cluster_utils/parallel_algorithm.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py b/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py index 283b93ec1..9ec9c28e3 100644 --- a/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py +++ b/profiler/cluster_analyse/cluster_utils/parallel_algorithm.py @@ -36,19 +36,19 @@ class MegatronAlgorithm(ParallelAlgorithm): # Check for zero and adjust parallel sizes to avoid division by zero if tensor_model_parallel_size == 0: tensor_model_parallel_size = 1 - logger.error("tensor_model_parallel_size cannot be zero.") + logger.error("tensor_model_parallel_size cannot be 0 and has been set to 1 to continue.") if pipeline_model_parallel_size == 0: pipeline_model_parallel_size = 1 - logger.error("pipeline_model_parallel_size cannot be zero.") + logger.error("pipeline_model_parallel_size cannot be 0 and has been set to 1 to continue.") if data_parallel_size == 0: data_parallel_size = 1 - logger.error("data_parallel_size cannot be zero.") + logger.error("data_parallel_size cannot be 0 and has been set to 1 to continue.") if expert_model_parallel_size == 0: expert_model_parallel_size = 1 - logger.error("expert_model_parallel_size cannot be zero.") + logger.error("expert_model_parallel_size cannot be 0 and has been set to 1 to continue.") if data_parallel_size % expert_model_parallel_size != 0: raise RuntimeError( -- Gitee From 111ed3f5dbe664e0579f2710575f5946a89255e6 Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 3 Sep 2024 16:45:10 +0800 Subject: [PATCH 661/791] =?UTF-8?q?=E8=BE=93=E5=85=A5=E4=B8=8D=E5=AD=98?= =?UTF-8?q?=E5=9C=A8=E5=8F=82=E6=95=B0=E6=8F=90=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 20a7cc6b9..892eae4e7 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -15,7 +15,6 @@ import argparse import os -import logging from cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from cluster_data_preprocess.mindspore_data_preprocessor import MindsporeDataPreprocessor @@ -25,8 +24,6 @@ from common_func.file_manager import FileManager from common_func.path_manager import PathManager from analysis.analysis_facade import AnalysisFacade -logger = logging.getLogger() - COMM_FEATURE_LIST = ['all', 'communication_time', 'communication_matrix'] class Interface: @@ -96,9 +93,7 @@ def cluster_analysis_main(args=None): parser.add_argument('-m', '--mode', choices=COMM_FEATURE_LIST, default='all', help="different analysis mode") parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') - args_parsed, unknown = parser.parse_known_args(args=args) - if unknown: - logger.warning("Unknown arguments are provided: %s", unknown) + args_parsed = parser.parse_args(args = args) parameter = { Constant.COLLECTION_PATH: args_parsed.collection_path, Constant.ANALYSIS_MODE: args_parsed.mode, -- Gitee From b0832afddf00e24f50eb1c58580a52ebfb7f63a8 Mon Sep 17 00:00:00 2001 From: hhqx Date: Sat, 31 Aug 2024 15:50:15 +0800 Subject: [PATCH 662/791] add divide zero check to core_num --- profiler/advisor/analyzer/cluster/slow_rank_analyzer.py | 5 +++-- .../advisor/analyzer/computation/bound/block_dim_checker.py | 3 +++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py index 4f5bacc49..74cdaf4e5 100644 --- a/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py +++ b/profiler/advisor/analyzer/cluster/slow_rank_analyzer.py @@ -21,6 +21,7 @@ from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.dataset.cluster.cluster_dataset import ClusterStepTraceTimeDataset from profiler.advisor.utils.utils import safe_index_value +from profiler.advisor.utils.utils import safe_division logger = logging.getLogger() @@ -196,8 +197,8 @@ class SlowRankAnalyzer(BaseAnalyzer): tmp_rank_list.append(rank_id) tmp_time_list.append(time) - if self.compute_max_gap_ratio(tmp_time_list, sum(tmp_time_list) / len( - tmp_time_list)) < self.RATIO_THRESHOLD: + if self.compute_max_gap_ratio(tmp_time_list, safe_division(sum(tmp_time_list), len( + tmp_time_list))) < self.RATIO_THRESHOLD: continue max_time, min_time = max(tmp_time_list), min(tmp_time_list) diff --git a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py index 5b358ebaa..f1efe34b6 100644 --- a/profiler/advisor/analyzer/computation/bound/block_dim_checker.py +++ b/profiler/advisor/analyzer/computation/bound/block_dim_checker.py @@ -60,6 +60,9 @@ class BlockDimChecker(OperatorChecker): return False block_dim = int(op_info.block_dim) core_num = self.get_core_num(op_info) + if core_num == 0: + logger.error("The aicore number is zero. BlockDimChecker is skipped. Please check the info.json file.") + return False if block_dim % core_num == 0: return False if op_info.task_type == "MIX_AIC" and hasattr(op_info, "mix_block_dim") \ -- Gitee From 6696c7339064802a83b13f6e61cc386fdc05a2e4 Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 3 Sep 2024 17:26:29 +0800 Subject: [PATCH 663/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E7=AC=A6=E5=8F=B7=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 8a32b14ef..2d0d4f384 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -70,6 +70,7 @@ class Interface: def run(self): PathManager.check_input_directory_path(self.collection_path) PathManager.check_path_owner_consistent(self.collection_path) + PathManager.check_input_directory_path(self.cluster_analysis_output_path) PathManager.make_dir_safety(self.cluster_analysis_output_path) PathManager.check_path_writeable(self.cluster_analysis_output_path) data_map, data_type = self.allocate_prof_data() -- Gitee From 43ba0756f391d36f5df90948a52de9bcced099e1 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Tue, 3 Sep 2024 18:52:08 +0800 Subject: [PATCH 664/791] =?UTF-8?q?=E3=80=90feature=E3=80=91=E9=80=9A?= =?UTF-8?q?=E4=BF=A1api=20dump=E6=94=AF=E6=8C=81all=5Fgather=5Finto=5Ftens?= =?UTF-8?q?or=E5=92=8Creduce=5Fscatter=5Ftensor?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/const.py | 3 ++- .../msprobe/pytorch/hook_module/support_wrap_ops.yaml | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index b6cf5088d..c6240edd5 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -92,7 +92,8 @@ class Const: INPLACE_LIST = [ "broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter", - "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single", "all_to_all" + "_reduce_scatter_base", "_all_gather_base", "send", "recv", "irecv", "isend", "all_to_all_single", "all_to_all", + "all_gather_into_tensor", "reduce_scatter_tensor" ] CONVERT = { diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml index f68708e94..11281396b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/support_wrap_ops.yaml @@ -1874,4 +1874,6 @@ distributed: - _reduce_scatter_base - _all_gather_base - all_to_all_single - - all_to_all \ No newline at end of file + - all_to_all + - all_gather_into_tensor + - reduce_scatter_tensor \ No newline at end of file -- Gitee From 884589389fb3be9dc71b0a6c3803c57df1393062 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 3 Sep 2024 19:14:42 +0800 Subject: [PATCH 665/791] =?UTF-8?q?=E5=88=A0=E9=99=A4=E6=97=A0=E7=94=A8che?= =?UTF-8?q?ck=5Ffile=5Fvalid,get=5Fdata=5Flen=5Fby=5Fshape,=E5=B0=86get=5F?= =?UTF-8?q?json=5Fcontents,get=5Ffile=5Fcontent=5Fbytes,load=5Fnpy?= =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E5=88=B0file=5Futils=E4=B8=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/file_utils.py | 33 +++++++++- .../msprobe/core/common/utils.py | 65 +------------------ .../msprobe/core/grad_probe/grad_compare.py | 4 +- .../api_accuracy_checker/compute_element.py | 2 +- .../msprobe/mindspore/compare/ms_compare.py | 4 +- .../api_accuracy_checker/compare/compare.py | 5 +- .../run_ut/data_generate.py | 3 +- .../run_ut/run_overflow_check.py | 1 - .../api_accuracy_checker/run_ut/run_ut.py | 3 +- .../msprobe/pytorch/parse_tool/lib/compare.py | 4 +- .../pytorch/parse_tool/lib/visualization.py | 4 +- .../msprobe/test/core_ut/common/test_utils.py | 7 +- .../run_ut/test_data_generate.py | 2 +- .../run_ut/test_run_ut.py | 2 +- 14 files changed, 54 insertions(+), 85 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index 5f4fed9f1..329fe884d 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -15,14 +15,15 @@ # limitations under the License. """ import os +import json import re import shutil import yaml +import numpy as np from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.utils import CompareException class FileChecker: @@ -327,6 +328,16 @@ def load_yaml(yaml_path): return yaml_data +def load_npy(filepath, enable_pickle=False): + check_file_or_directory_path(filepath) + try: + npy = np.load(filepath, allow_pickle=enable_pickle) + except Exception as e: + logger.error(f"The numpy file failed to load. Please check the path: {filepath}.") + raise RuntimeError(f"Load numpy file {filepath} failed.") from e + return npy + + def remove_path(path): if not os.path.exists(path): return @@ -337,4 +348,22 @@ def remove_path(path): shutil.rmtree(path) except PermissionError as err: logger.error("Failed to delete {}. Please check the permission.".format(path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) from err + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) from err + + +def get_json_contents(file_path): + ops = get_file_content_bytes(file_path) + try: + json_obj = json.loads(ops) + except ValueError as error: + logger.error('Failed to load json.') + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) from error + if not isinstance(json_obj, dict): + logger.error('Json file content is not a dictionary!') + raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) + return json_obj + + +def get_file_content_bytes(file): + with FileOpen(file, 'rb') as file_handle: + return file_handle.read() diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 74b36eb2d..5d7bb21dc 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -26,8 +26,8 @@ import csv from datetime import datetime, timezone import numpy as np -from msprobe.core.common.file_utils import (FileOpen, FileChecker, - change_mode, check_path_before_create, check_file_or_directory_path) +from msprobe.core.common.file_utils import (FileOpen, change_mode, + check_path_before_create, check_file_or_directory_path) from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.core.common.log import logger @@ -271,16 +271,6 @@ def parse_value_by_comma(value): return value_list -def get_data_len_by_shape(shape): - data_len = 1 - for item in shape: - if item == -1: - logger.error("please check your input shape, one dim in shape is -1.") - return -1 - data_len = data_len * item - return data_len - - def add_time_as_suffix(name): return '{}_{}.csv'.format(name, time.strftime("%Y%m%d%H%M%S", time.localtime(time.time()))) @@ -344,30 +334,6 @@ def generate_compare_script(dump_path, pkl_file_path, dump_switch_mode): logger.info(f"Generate compare script successfully which is {compare_script_path}.") -def check_file_valid(file_path): - if os.path.islink(file_path): - logger.error('The file path {} is a soft link.'.format(file_path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if len(os.path.realpath(file_path)) > Const.DIRECTORY_LENGTH or len(os.path.basename(file_path)) > \ - Const.FILE_NAME_LENGTH: - logger.error('The file path length exceeds limit.') - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if not re.match(Const.FILE_PATTERN, os.path.realpath(file_path)): - logger.error('The file path {} contains special characters.'.format(file_path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - if os.path.isfile(file_path): - file_size = os.path.getsize(file_path) - if file_path.endswith(Const.PKL_SUFFIX) and file_size > Const.ONE_GB: - logger.error('The file {} size is greater than 1GB.'.format(file_path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - if file_path.endswith(Const.NUMPY_SUFFIX) and file_size > Const.TEN_GB: - logger.error('The file {} size is greater than 10GB.'.format(file_path)) - raise CompareException(CompareException.INVALID_PATH_ERROR) - - def check_inplace_op(prefix): if len(prefix) > Const.DISTRIBUTED_PREFIX_LENGTH: return False @@ -442,16 +408,6 @@ def write_csv(data, filepath, mode="a+"): change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) -def load_npy(filepath, enable_pickle=False): - check_file_or_directory_path(filepath) - try: - npy = np.load(filepath, allow_pickle=enable_pickle) - except Exception as e: - logger.error(f"The numpy file failed to load. Please check the path: {filepath}.") - raise RuntimeError(f"Load numpy file {filepath} failed.") from e - return npy - - def save_npy(data, filepath): filepath = os.path.realpath(filepath) check_path_before_create(filepath) @@ -480,23 +436,6 @@ def save_npy_to_txt(self, data, dst_file='', align=0): self.log.error("An unexpected error occurred: %s when savetxt to %s" % (str(e)), dst_file) change_mode(dst_file, FileCheckConst.DATA_FILE_AUTHORITY) -def get_json_contents(file_path): - ops = get_file_content_bytes(file_path) - try: - json_obj = json.loads(ops) - except ValueError as error: - logger.error('Failed to load json.') - raise CompareException(CompareException.INVALID_FILE_ERROR) from error - if not isinstance(json_obj, dict): - logger.error('Json file content is not a dictionary!') - raise CompareException(CompareException.INVALID_FILE_ERROR) - return json_obj - - -def get_file_content_bytes(file): - with FileOpen(file, 'rb') as file_handle: - return file_handle.read() - def save_workbook(workbook, file_path): """ diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 61b206a61..517360bcb 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -7,8 +7,8 @@ import matplotlib.pyplot as plt from msprobe.core.common.file_utils import create_directory, check_path_before_create, check_file_or_directory_path from msprobe.core.common.log import logger -from msprobe.core.common.utils import write_csv, load_npy -from msprobe.core.common.file_utils import remove_path +from msprobe.core.common.utils import write_csv +from msprobe.core.common.file_utils import remove_path, load_npy from msprobe.core.grad_probe.constant import GradConst diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py index ffadce87b..5dd005b7a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/compute_element.py @@ -6,7 +6,7 @@ import numpy as np from msprobe.mindspore.common.log import logger from msprobe.core.common.exceptions import ApiAccuracyCheckerException -from msprobe.core.common.utils import load_npy +from msprobe.core.common.file_utils import load_npy from msprobe.mindspore.api_accuracy_checker.type_mapping import (dtype_str_to_np_dtype, api_info_type_str_to_type, ms_dtype_to_dtype_str, torch_dtype_to_dtype_str, dtype_str_to_ms_dtype, dtype_str_to_np_dtype, diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 6fcda39e4..a41313067 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,7 +1,7 @@ import os.path from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, \ - task_dumppath_get, load_npy -from msprobe.core.common.file_utils import create_directory, load_yaml + task_dumppath_get +from msprobe.core.common.file_utils import create_directory, load_yaml, load_npy from msprobe.core.common.const import Const from msprobe.mindspore.common.log import logger from msprobe.core.common.exceptions import FileCheckException diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index 925cfcc6e..ec182f84e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -3,9 +3,10 @@ import os from collections import namedtuple import numpy as np -from msprobe.core.common.utils import write_csv, get_json_contents, CompareException +from msprobe.core.common.utils import write_csv, CompareException +from msprobe.core.common.file_utils import get_json_contents import torch -from msprobe.core.common.const import Const, CompareConst +from msprobe.core.common.const import CompareConst from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, \ get_mean_rel_err, get_rel_err, get_abs_err, get_max_abs_err, get_rel_err_ratio, cosine_sim, get_rel_err_origin, \ get_small_value_err_ratio, get_finite_and_infinite_mask, get_small_value_mask, check_inf_nan_value, \ diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index c68c35957..091a04619 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -23,10 +23,9 @@ import numpy from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import hf_32_standard_api from msprobe.pytorch.api_accuracy_checker.common.utils import check_object_type, get_full_data_path, \ CompareException -from msprobe.core.common.file_utils import FileChecker +from msprobe.core.common.file_utils import FileChecker, load_npy from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.utils import load_pt -from msprobe.core.common.utils import load_npy from msprobe.core.common.const import Const, FileCheckConst TORCH_TYPE = ["torch.device", "torch.dtype"] diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py index ed46824cf..c2379d86b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_overflow_check.py @@ -12,7 +12,6 @@ import torch from tqdm import tqdm from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import generate_device_params, get_api_info from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import exec_api -from msprobe.core.common.utils import get_json_contents from msprobe.core.common.file_utils import check_link from msprobe.pytorch.common.log import logger from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index c442bf9c0..bb8020c0c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -27,9 +27,8 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_column import CompareC from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.common.parse_json import parse_json_info_forward_backward from msprobe.core.common.file_utils import FileOpen, FileChecker, \ - change_mode, check_path_before_create, create_directory + change_mode, check_path_before_create, create_directory, get_json_contents from msprobe.pytorch.common.log import logger -from msprobe.core.common.utils import get_json_contents from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTL, ATTLConfig, move2device_exec diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py index 9dd65e36a..aeea170d4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py @@ -22,8 +22,8 @@ from collections import namedtuple from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException -from msprobe.core.common.utils import write_csv, save_npy_to_txt, load_npy -from msprobe.core.common.file_utils import FileChecker, create_directory +from msprobe.core.common.utils import write_csv, save_npy_to_txt +from msprobe.core.common.file_utils import FileChecker, create_directory, load_npy class Compare: diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py index 680797399..a0dc806fd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py @@ -20,8 +20,8 @@ import numpy as np from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException -from msprobe.core.common.file_utils import FileOpen -from msprobe.core.common.utils import save_npy_to_txt, load_npy +from msprobe.core.common.file_utils import FileOpen, load_npy +from msprobe.core.common.utils import save_npy_to_txt class Visualization: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index e1e4c2578..b8cf00c80 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -40,8 +40,11 @@ from msprobe.core.common.utils import (CompareException, get_dump_data_path, task_dumppath_get) -from msprobe.core.common.file_utils import FileCheckConst, check_file_size, check_file_or_directory_path -from msprobe.core.common.utils import get_json_contents, get_file_content_bytes +from msprobe.core.common.file_utils import (FileCheckConst, + check_file_size, + check_file_or_directory_path, + get_json_contents, + get_file_content_bytes) class TestUtils(TestCase): @patch.object(logger, "error") diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py index 8d8815d02..838293773 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_data_generate.py @@ -4,7 +4,7 @@ import unittest import copy from msprobe.pytorch.api_accuracy_checker.run_ut.data_generate import * -from msprobe.core.common.utils import get_json_contents +from msprobe.core.common.file_utils import get_json_contents base_dir = os.path.dirname(os.path.realpath(__file__)) forward_file = os.path.join(base_dir, "forward.json") diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py index 3ade53454..8cf581b72 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_run_ut.py @@ -3,7 +3,7 @@ import copy import unittest from unittest.mock import patch, DEFAULT from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut import * -from msprobe.core.common.utils import get_json_contents +from msprobe.core.common.file_utils import get_json_contents from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import UtDataInfo, exec_api base_dir = os.path.dirname(os.path.realpath(__file__)) -- Gitee From 94883ff5d7158e3624a4aa9f7d55627e2f70e5ee Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 3 Sep 2024 19:21:10 +0800 Subject: [PATCH 666/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mindspore/api_accuracy_checker/api_accuracy_checker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 5e8367d52..22d2b6173 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -1,7 +1,7 @@ import json import os -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen, create_directory from msprobe.core.common.utils import write_csv, add_time_as_suffix from msprobe.core.common.const import Const, CompareConst, MsCompareConst from msprobe.mindspore.common.log import logger -- Gitee From 0401d9d29a1e56ce4c20d20e167fc99b4ca3f9b5 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 3 Sep 2024 20:14:45 +0800 Subject: [PATCH 667/791] FileOpen --- .../test/mindspore_ut/grad_probe/test_ms_grad_monitor.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index 612f7a10e..f6f399db2 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -8,7 +8,7 @@ import mindspore from mindspore import nn, Tensor from mindspore.nn import SGD from msprobe.mindspore import PrecisionDebugger - +from msprobe.core.common.file_check import FileOpen file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) @@ -42,13 +42,13 @@ def main(): def save_dict_as_json(data, json_file_path): - with open(json_file_path, 'w') as f: + with FileOpen(json_file_path, 'w') as f: json.dump(data, f, ensure_ascii=False, indent=4) print(f"字典已保存为json文件: {json_file_path}") def get_hash(file_path): - with open(file_path, 'rb') as file: + with FileOpen(file_path, 'rb') as file: hash_object = hashlib.md5() for chunk in iter(lambda: file.read(4096), b""): hash_object.update(chunk) -- Gitee From 4c28a481f1cb43ec7ce428d928a9253547b590b2 Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 3 Sep 2024 20:17:48 +0800 Subject: [PATCH 668/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E7=AC=A6=E5=8F=B7=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 2d0d4f384..ec4ca6771 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -70,8 +70,7 @@ class Interface: def run(self): PathManager.check_input_directory_path(self.collection_path) PathManager.check_path_owner_consistent(self.collection_path) - PathManager.check_input_directory_path(self.cluster_analysis_output_path) - PathManager.make_dir_safety(self.cluster_analysis_output_path) + FileManager.create_output_dir(self.cluster_analysis_output_path) PathManager.check_path_writeable(self.cluster_analysis_output_path) data_map, data_type = self.allocate_prof_data() if not data_map: @@ -80,7 +79,7 @@ class Interface: if data_type == Constant.INVALID: print("[ERROR] The current folder contains both DB and other files. Please check.") return - FileManager.create_output_dir(self.cluster_analysis_output_path) + params = { Constant.COLLECTION_PATH: self.collection_path, Constant.DATA_MAP: data_map, -- Gitee From 8f13a20a60a42f244993f48119fe290f4b133ac4 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 3 Sep 2024 20:19:31 +0800 Subject: [PATCH 669/791] remove and np.load --- .../msprobe/mindspore/grad_probe/grad_analyzer.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index 2bdc11114..52296e1d2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -2,7 +2,7 @@ import os import time from typing import List, Tuple import multiprocessing -from multiprocessing import Process +from multiprocessing import Process import numpy as np import mindspore as ms @@ -14,7 +14,7 @@ from msprobe.core.grad_probe.utils import ListCache from msprobe.core.grad_probe.constant import GradConst from msprobe.core.common.log import logger from msprobe.core.common.file_check import create_directory -from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file +from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file, load_npy from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext @@ -77,7 +77,7 @@ class CSVGenerator(Process): self.level = GradConst.LEVEL0 self.cache_list = ListCache() self.current_step = None - self.stop_event = None + self.stop_event = None self.last_finish = False self.bounds = [-0.1, 0.0, 0.1], @@ -89,7 +89,7 @@ class CSVGenerator(Process): self.dump_dir = f"{output_path}/rank{rank_id}/Dump/" self.save_dir = f"{output_path}/rank{rank_id}/" self.current_step = None - self.stop_event = multiprocessing.Event() + self.stop_event = multiprocessing.Event() self.last_finish = False def run(self): @@ -139,7 +139,7 @@ class CSVGenerator(Process): if stat_data is None: continue if not self.check_valid(stat_data): - os.remove(file_path) + remove_path(file_path) continue step = int(stat_data[GradConst.STEP_IDX]) update_step = self.current_step is None or step != self.current_step @@ -147,7 +147,7 @@ class CSVGenerator(Process): if update_step: self.create_csv_file() self.gen_csv_line(file_path, stat_data) - os.remove(file_path) + remove_path(file_path) self.last_finish = False def check_valid(self, stat_data): @@ -170,7 +170,7 @@ class CSVGenerator(Process): max_try = 10 while max_try: try: - stat_data = np.load(file_path) + stat_data = load_npy(file_path) return stat_data except Exception as err: logger.warning(f"load numpy file failed, retry...") -- Gitee From 74b909d211298d7a08068e53a21ffd7317f34f88 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 3 Sep 2024 20:26:42 +0800 Subject: [PATCH 670/791] =?UTF-8?q?=E5=B0=86move=5Ffile,save=5Fnpy,save=5F?= =?UTF-8?q?npy=5Fto=5Ftxt,save=5Fworkbook,write=5Fcsv=E7=A7=BB=E5=8A=A8?= =?UTF-8?q?=E5=88=B0file=5Futils?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/file_utils.py | 67 ++++++++++++++++++ .../msprobe/core/common/utils.py | 68 ------------------- .../msprobe/core/compare/highlight.py | 3 +- .../msprobe/core/grad_probe/grad_compare.py | 3 +- .../msprobe/core/grad_probe/utils.py | 3 +- .../api_accuracy_checker.py | 4 +- .../msprobe/mindspore/common/utils.py | 3 +- .../mindspore/grad_probe/grad_analyzer.py | 3 +- .../msprobe/mindspore/grad_probe/hook.py | 3 +- .../msprobe/mindspore/grad_probe/utils.py | 6 +- .../compare/api_precision_compare.py | 2 +- .../api_accuracy_checker/compare/compare.py | 4 +- .../msprobe/pytorch/compare/pt_compare.py | 6 +- .../pytorch/grad_probe/grad_monitor.py | 5 +- .../msprobe/pytorch/parse_tool/lib/compare.py | 3 +- .../pytorch/parse_tool/lib/visualization.py | 3 +- .../common/test_common_utils.py | 3 +- 17 files changed, 93 insertions(+), 96 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index 329fe884d..6c7ad7025 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -14,6 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ +import csv import os import json import re @@ -338,6 +339,72 @@ def load_npy(filepath, enable_pickle=False): return npy +def move_file(src_path, dst_path): + check_file_or_directory_path(src_path) + check_path_before_create(dst_path) + try: + shutil.move(src_path, dst_path) + except Exception as e: + logger.error(f"move file {src_path} to {dst_path} failed") + raise RuntimeError(f"move file {src_path} to {dst_path} failed") from e + change_mode(dst_path, FileCheckConst.DATA_FILE_AUTHORITY) + + +def save_npy(data, filepath): + filepath = os.path.realpath(filepath) + check_path_before_create(filepath) + try: + np.save(filepath, data) + except Exception as e: + logger.error(f"The numpy file failed to save. Please check the path: {filepath}.") + raise RuntimeError(f"Save numpy file {filepath} failed.") from e + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + +def save_npy_to_txt(self, data, dst_file='', align=0): + if os.path.exists(dst_file): + self.log.info("Dst file %s exists, will not save new one.", dst_file) + return + shape = data.shape + data = data.flatten() + if align == 0: + align = 1 if len(shape) == 0 else shape[-1] + elif data.size % align != 0: + pad_array = np.zeros((align - data.size % align,)) + data = np.append(data, pad_array) + check_path_before_create(dst_file) + try: + np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g') + except Exception as e: + self.log.error("An unexpected error occurred: %s when savetxt to %s" % (str(e)), dst_file) + change_mode(dst_file, FileCheckConst.DATA_FILE_AUTHORITY) + + +def save_workbook(workbook, file_path): + """ + 保存工作簿到指定的文件路径 + workbook: 要保存的工作簿对象 + file_path: 文件保存路径 + """ + file_path = os.path.realpath(file_path) + check_path_before_create(file_path) + try: + workbook.save(file_path) + except Exception as e: + logger.error(f'Save result file "{os.path.basename(file_path)}" failed') + raise RuntimeError(f"Save result file {file_path} failed.") from e + change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) + + +def write_csv(data, filepath, mode="a+"): + exist = os.path.exists(filepath) + with FileOpen(filepath, mode, encoding='utf-8-sig') as f: + writer = csv.writer(f) + writer.writerows(data) + if not exist: + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + + def remove_path(path): if not os.path.exists(path): return diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index a3ab8de76..8efad0801 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -18,13 +18,10 @@ import collections import fcntl import os import re -import shutil import subprocess import time import json -import csv from datetime import datetime, timezone -import numpy as np from msprobe.core.common.file_utils import (FileOpen, change_mode, check_path_before_create, check_file_or_directory_path) @@ -203,17 +200,6 @@ def check_regex_prefix_format_valid(prefix): raise ValueError(f"prefix contains invalid characters, prefix pattern {Const.REGEX_PREFIX_PATTERN}") -def move_file(src_path, dst_path): - check_file_or_directory_path(src_path) - check_path_before_create(dst_path) - try: - shutil.move(src_path, dst_path) - except Exception as e: - logger.error(f"move file {src_path} to {dst_path} failed") - raise RuntimeError(f"move file {src_path} to {dst_path} failed") from e - change_mode(dst_path, FileCheckConst.DATA_FILE_AUTHORITY) - - def get_dump_data_path(dump_dir): """ Function Description: @@ -401,60 +387,6 @@ def convert_tuple(data): return data if isinstance(data, tuple) else (data, ) -def write_csv(data, filepath, mode="a+"): - exist = os.path.exists(filepath) - with FileOpen(filepath, mode, encoding='utf-8-sig') as f: - writer = csv.writer(f) - writer.writerows(data) - if not exist: - change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) - - -def save_npy(data, filepath): - filepath = os.path.realpath(filepath) - check_path_before_create(filepath) - try: - np.save(filepath, data) - except Exception as e: - logger.error(f"The numpy file failed to save. Please check the path: {filepath}.") - raise RuntimeError(f"Save numpy file {filepath} failed.") from e - change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) - -def save_npy_to_txt(self, data, dst_file='', align=0): - if os.path.exists(dst_file): - self.log.info("Dst file %s exists, will not save new one.", dst_file) - return - shape = data.shape - data = data.flatten() - if align == 0: - align = 1 if len(shape) == 0 else shape[-1] - elif data.size % align != 0: - pad_array = np.zeros((align - data.size % align,)) - data = np.append(data, pad_array) - check_path_before_create(dst_file) - try: - np.savetxt(dst_file, data.reshape((-1, align)), delimiter=' ', fmt='%g') - except Exception as e: - self.log.error("An unexpected error occurred: %s when savetxt to %s" % (str(e)), dst_file) - change_mode(dst_file, FileCheckConst.DATA_FILE_AUTHORITY) - - -def save_workbook(workbook, file_path): - """ - 保存工作簿到指定的文件路径 - workbook: 要保存的工作簿对象 - file_path: 文件保存路径 - """ - file_path = os.path.realpath(file_path) - check_path_before_create(file_path) - try: - workbook.save(file_path) - except Exception as e: - logger.error(f'Save result file "{os.path.basename(file_path)}" failed') - raise CompareException(CompareException.WRITE_FILE_ERROR) from e - change_mode(file_path, FileCheckConst.DATA_FILE_AUTHORITY) - - def load_json(json_path): try: with FileOpen(json_path, "r") as f: diff --git a/debug/accuracy_tools/msprobe/core/compare/highlight.py b/debug/accuracy_tools/msprobe/core/compare/highlight.py index 8f0a625be..600760dc4 100644 --- a/debug/accuracy_tools/msprobe/core/compare/highlight.py +++ b/debug/accuracy_tools/msprobe/core/compare/highlight.py @@ -4,7 +4,8 @@ from collections import namedtuple import numpy as np import openpyxl from openpyxl.styles import PatternFill -from msprobe.core.common.utils import get_header_index, save_workbook +from msprobe.core.common.utils import get_header_index +from msprobe.core.common.file_utils import save_workbook from msprobe.core.common.log import logger from msprobe.core.common.const import CompareConst diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py index 03170e610..0a7b89e88 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/grad_compare.py @@ -7,8 +7,7 @@ import matplotlib.pyplot as plt from msprobe.core.common.file_utils import create_directory, check_path_before_create, check_file_or_directory_path from msprobe.core.common.log import logger -from msprobe.core.common.utils import write_csv -from msprobe.core.common.file_utils import remove_path, load_npy +from msprobe.core.common.file_utils import remove_path, load_npy, write_csv from msprobe.core.grad_probe.constant import GradConst from msprobe.core.grad_probe.utils import plt_savefig diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py index 466d5282a..6aaebf110 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py @@ -1,7 +1,8 @@ import re from msprobe.core.grad_probe.constant import GradConst from msprobe.core.common.log import logger -from msprobe.core.common.utils import write_csv, check_path_before_create, change_mode +from msprobe.core.common.utils import check_path_before_create, change_mode +from msprobe.core.common.file_utils import write_csv from msprobe.core.common.const import FileCheckConst import matplotlib.pyplot as plt diff --git a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py index 22d2b6173..7277cff54 100644 --- a/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py +++ b/debug/accuracy_tools/msprobe/mindspore/api_accuracy_checker/api_accuracy_checker.py @@ -1,8 +1,8 @@ import json import os -from msprobe.core.common.file_utils import FileOpen, create_directory -from msprobe.core.common.utils import write_csv, add_time_as_suffix +from msprobe.core.common.file_utils import FileOpen, create_directory, write_csv +from msprobe.core.common.utils import add_time_as_suffix from msprobe.core.common.const import Const, CompareConst, MsCompareConst from msprobe.mindspore.common.log import logger from msprobe.mindspore.api_accuracy_checker.api_info import ApiInfo diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index 1de9ef959..56f4f6c2a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -16,8 +16,7 @@ import mindspore as ms from msprobe.core.common.exceptions import DistributedNotInitializedError -from msprobe.core.common.file_utils import path_len_exceeds_limit -from msprobe.core.common.utils import save_npy +from msprobe.core.common.file_utils import path_len_exceeds_limit, save_npy from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index 905b2f450..f30224272 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -12,8 +12,7 @@ from mindspore.common.parameter import Parameter from msprobe.core.grad_probe.utils import ListCache from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_utils import create_directory, check_file_or_directory_path, remove_path -from msprobe.core.common.utils import write_csv, move_file +from msprobe.core.common.file_utils import create_directory, check_file_or_directory_path, remove_path, move_file, write_csv from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py index 6b7328b18..dabd6f710 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py @@ -11,8 +11,7 @@ from mindspore.common.initializer import initializer from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.utils import write_csv -from msprobe.core.common.file_utils import remove_path +from msprobe.core.common.file_utils import remove_path, write_csv from msprobe.mindspore.grad_probe.global_context import grad_context from msprobe.mindspore.grad_probe.grad_analyzer import grad_dump, get_rank_id from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py index e56ce51f9..2608042fc 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/utils.py @@ -3,8 +3,10 @@ import os import mindspore from msprobe.core.grad_probe.constant import level_adp from msprobe.core.grad_probe.utils import check_param -from msprobe.core.common.file_utils import create_directory, check_path_before_create, check_file_or_directory_path -from msprobe.core.common.utils import save_npy +from msprobe.core.common.file_utils import (create_directory, + check_path_before_create, + check_file_or_directory_path, + save_npy) def save_grad_direction(param_name, grad, save_path): diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py index a38b11158..2af34924e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/api_precision_compare.py @@ -7,7 +7,7 @@ from collections import namedtuple import torch import pandas as pd -from msprobe.core.common.utils import write_csv +from msprobe.core.common.file_utils import write_csv from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import API_PRECISION_COMPARE_RESULT_FILE_NAME, \ API_PRECISION_COMPARE_DETAILS_FILE_NAME, BENCHMARK_COMPARE_SUPPORT_LIST, API_PRECISION_COMPARE_UNSUPPORT_LIST, \ diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py index f2439bb1c..08d212b1e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/compare/compare.py @@ -3,8 +3,8 @@ import os from collections import namedtuple import numpy as np -from msprobe.core.common.utils import write_csv, CompareException -from msprobe.core.common.file_utils import get_json_contents +from msprobe.core.common.utils import CompareException +from msprobe.core.common.file_utils import get_json_contents, write_csv import torch from msprobe.core.common.const import CompareConst from msprobe.pytorch.api_accuracy_checker.compare.algorithm import get_rmse, get_error_balance, get_max_rel_err, \ diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index f40efc4fb..3d9477cf7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,12 +1,12 @@ import os.path import torch -from msprobe.core.common.const import FileCheckConst, Const +from msprobe.core.common.const import FileCheckConst from msprobe.core.common.file_utils import create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator -from msprobe.core.common.utils import check_configuration_param, task_dumppath_get, check_compare_param, FileChecker -from msprobe.core.common.utils import CompareException +from msprobe.core.common.utils import check_configuration_param, task_dumppath_get, check_compare_param, CompareException +from msprobe.core.common.file_utils import FileChecker from msprobe.pytorch.common.utils import load_pt diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index fb5e0c51b..eecf51c60 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -6,11 +6,10 @@ if int(torch.__version__.split('.')[0]) >= 2: from torch.optim.optimizer import register_optimizer_step_pre_hook from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target -from msprobe.core.grad_probe.constant import GradConst, level_adp +from msprobe.core.grad_probe.constant import level_adp from msprobe.core.common.file_utils import create_directory from msprobe.pytorch.common.log import logger -from msprobe.core.common.utils import write_csv, save_npy -from msprobe.core.common.file_utils import remove_path +from msprobe.core.common.file_utils import remove_path, save_npy, write_csv from msprobe.pytorch.common.utils import get_rank_id, print_rank_0 diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py index aeea170d4..255f131fa 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/compare.py @@ -22,8 +22,7 @@ from collections import namedtuple from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException -from msprobe.core.common.utils import write_csv, save_npy_to_txt -from msprobe.core.common.file_utils import FileChecker, create_directory, load_npy +from msprobe.core.common.file_utils import FileChecker, create_directory, load_npy, save_npy_to_txt, write_csv class Compare: diff --git a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py index a0dc806fd..835e2efcc 100644 --- a/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py +++ b/debug/accuracy_tools/msprobe/pytorch/parse_tool/lib/visualization.py @@ -20,8 +20,7 @@ import numpy as np from msprobe.pytorch.parse_tool.lib.config import Const from msprobe.pytorch.parse_tool.lib.utils import Util from msprobe.pytorch.parse_tool.lib.parse_exception import ParseException -from msprobe.core.common.file_utils import FileOpen, load_npy -from msprobe.core.common.utils import save_npy_to_txt +from msprobe.core.common.file_utils import FileOpen, load_npy, save_npy_to_txt class Visualization: diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py index cd292b15e..4696c5585 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/common/test_common_utils.py @@ -5,7 +5,8 @@ import unittest from unittest.mock import patch from msprobe.pytorch.api_accuracy_checker.common.utils import * -from msprobe.core.common.utils import write_csv +from msprobe.core.common.file_utils import write_csv + class TestUtils(unittest.TestCase): -- Gitee From 949016f327cb022c8d17e5416daf6dbe0873322b Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 3 Sep 2024 20:42:36 +0800 Subject: [PATCH 671/791] =?UTF-8?q?=E5=B0=86load=5Fjson,save=5Fjson?= =?UTF-8?q?=E7=A7=BB=E5=8A=A8=E5=88=B0file=5Futils=EF=BC=8C=E4=BF=AE?= =?UTF-8?q?=E5=A4=8Ddt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/file_utils.py | 27 +++++++++++++++++++ .../msprobe/core/common/utils.py | 26 ------------------ .../msprobe/core/data_dump/json_writer.py | 3 +-- .../msprobe/test/core_ut/common/test_utils.py | 11 ++++---- 4 files changed, 34 insertions(+), 33 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index 6c7ad7025..fd3b9b019 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -15,6 +15,7 @@ # limitations under the License. """ import csv +import fcntl import os import json import re @@ -339,6 +340,32 @@ def load_npy(filepath, enable_pickle=False): return npy +def load_json(json_path): + try: + with FileOpen(json_path, "r") as f: + fcntl.flock(f, fcntl.LOCK_EX) + data = json.load(f) + fcntl.flock(f, fcntl.LOCK_UN) + except Exception as e: + logger.error(f'load json file "{os.path.basename(json_path)}" failed.') + raise RuntimeError(f"Load json file {json_path} failed.") from e + return data + + +def save_json(json_path, data, indent=None): + json_path = os.path.realpath(json_path) + check_path_before_create(json_path) + try: + with FileOpen(json_path, 'w') as f: + fcntl.flock(f, fcntl.LOCK_EX) + json.dump(data, f, indent=indent) + fcntl.flock(f, fcntl.LOCK_UN) + except Exception as e: + logger.error(f'Save json file "{os.path.basename(json_path)}" failed.') + raise RuntimeError(f"Save json file {json_path} failed.") from e + change_mode(json_path, FileCheckConst.DATA_FILE_AUTHORITY) + + def move_file(src_path, dst_path): check_file_or_directory_path(src_path) check_path_before_create(dst_path) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 8efad0801..b1b544e34 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -385,29 +385,3 @@ def get_header_index(header_name, summary_compare=False): def convert_tuple(data): return data if isinstance(data, tuple) else (data, ) - - -def load_json(json_path): - try: - with FileOpen(json_path, "r") as f: - fcntl.flock(f, fcntl.LOCK_EX) - data = json.load(f) - fcntl.flock(f, fcntl.LOCK_UN) - except Exception as e: - logger.error(f'load json file "{os.path.basename(json_path)}" failed.') - raise DumpException(DumpException.WRITE_FILE_ERROR) from e - return data - - -def save_json(json_path, data, indent=None): - json_path = os.path.realpath(json_path) - check_path_before_create(json_path) - try: - with FileOpen(json_path, 'w') as f: - fcntl.flock(f, fcntl.LOCK_EX) - json.dump(data, f, indent=indent) - fcntl.flock(f, fcntl.LOCK_UN) - except Exception as e: - logger.error(f'Save json file "{os.path.basename(json_path)}" failed.') - raise DumpException(DumpException.WRITE_FILE_ERROR) from e - change_mode(json_path, FileCheckConst.DATA_FILE_AUTHORITY) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py index ad0722325..c970cec70 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py @@ -4,8 +4,7 @@ import csv from msprobe.core.common.file_utils import change_mode, FileOpen from msprobe.core.common.log import logger from msprobe.core.common.const import Const, FileCheckConst -from msprobe.core.common.utils import load_json, save_json -from msprobe.core.common.file_utils import remove_path +from msprobe.core.common.file_utils import remove_path, load_json, save_json class DataWriter: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index b8cf00c80..304d650aa 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -41,6 +41,7 @@ from msprobe.core.common.utils import (CompareException, task_dumppath_get) from msprobe.core.common.file_utils import (FileCheckConst, + FileCheckException, check_file_size, check_file_or_directory_path, get_json_contents, @@ -156,7 +157,7 @@ class TestUtils(TestCase): file_path = os.path.realpath(__file__) dirname = os.path.dirname(file_path) - with patch("msprobe.core.common.utils.FileChecker", new=TestFileChecker): + with patch("msprobe.core.common.file_utils.FileChecker", new=TestFileChecker): check_file_or_directory_path(file_path, isdir=False) self.assertTrue(TestFileChecker.checked) self.assertEqual(TestFileChecker.file_path, file_path) @@ -164,7 +165,7 @@ class TestUtils(TestCase): self.assertEqual(TestFileChecker.ability, FileCheckConst.READ_ABLE) TestFileChecker.checked = False - with patch("msprobe.core.common.utils.FileChecker", new=TestFileChecker): + with patch("msprobe.core.common.file_utils.FileChecker", new=TestFileChecker): check_file_or_directory_path(dirname, isdir=True) self.assertTrue(TestFileChecker.checked) self.assertEqual(TestFileChecker.file_path, dirname) @@ -261,9 +262,9 @@ class TestUtils(TestCase): @patch.object(logger, "error") def test_check_file_size(self, mock_error): with patch("msprobe.core.common.utils.os.path.getsize", return_value=120): - with self.assertRaises(CompareException) as context: + with self.assertRaises(FileCheckException) as context: check_file_size("input_file", 100) - self.assertEqual(context.exception.code, CompareException.INVALID_FILE_ERROR) + self.assertEqual(context.exception.code, FileCheckException.FILE_TOO_LARGE_ERROR) mock_error.assert_called_with("The size (120) of input_file exceeds (100) bytes, tools not support.") def test_check_regex_prefix_format_valid(self): @@ -329,7 +330,7 @@ class TestUtils(TestCase): self.assertEqual(context.exception.code, CompareException.INVALID_TASK_ERROR) mock_error.assert_called_with("Compare is not required for overflow_check or free_benchmark.") - @patch('msprobe.core.common.utils.get_file_content_bytes') + @patch('msprobe.core.common.file_utils.get_file_content_bytes') def test_get_json_contents_should_raise_exception(self, mock_get_file_content_bytes): mock_get_file_content_bytes.return_value = 'not a dict' with self.assertRaises(CompareException) as ce: -- Gitee From 3568b793bc34636cc6cb5130f64d9738e8c539f6 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Tue, 3 Sep 2024 20:44:06 +0800 Subject: [PATCH 672/791] =?UTF-8?q?=E6=B8=85=E7=90=86=E6=97=A0=E7=94=A8?= =?UTF-8?q?=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/utils.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index b1b544e34..9eb3d88db 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -15,7 +15,6 @@ # limitations under the License. """ import collections -import fcntl import os import re import subprocess @@ -23,9 +22,8 @@ import time import json from datetime import datetime, timezone -from msprobe.core.common.file_utils import (FileOpen, change_mode, - check_path_before_create, check_file_or_directory_path) -from msprobe.core.common.const import Const, FileCheckConst, CompareConst +from msprobe.core.common.file_utils import (FileOpen, check_file_or_directory_path) +from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.log import logger -- Gitee From ef5611d99c73e00c4c2a46a0508577fbbbb48f95 Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 3 Sep 2024 20:51:15 +0800 Subject: [PATCH 673/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E7=AC=A6=E5=8F=B7=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/common_func/file_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/profiler/cluster_analyse/common_func/file_manager.py b/profiler/cluster_analyse/common_func/file_manager.py index 26e6607fa..46bba017a 100644 --- a/profiler/cluster_analyse/common_func/file_manager.py +++ b/profiler/cluster_analyse/common_func/file_manager.py @@ -128,7 +128,8 @@ class FileManager: if not os.path.exists(output_path): PathManager.make_dir_safety(output_path) return - PathManager.remove_path_safety(output_path) + if os.path.exists(output_path): + PathManager.remove_path_safety(output_path) PathManager.make_dir_safety(output_path) @classmethod -- Gitee From 10875cfbfd044b68fbe4a40477ffe77dfa8369a0 Mon Sep 17 00:00:00 2001 From: zyy Date: Tue, 3 Sep 2024 21:07:36 +0800 Subject: [PATCH 674/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E7=AC=A6=E5=8F=B7=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 1 + 1 file changed, 1 insertion(+) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index ec4ca6771..56d69a7a8 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -70,6 +70,7 @@ class Interface: def run(self): PathManager.check_input_directory_path(self.collection_path) PathManager.check_path_owner_consistent(self.collection_path) + PathManager.check_input_directory_path(self.cluster_analysis_output_path) FileManager.create_output_dir(self.cluster_analysis_output_path) PathManager.check_path_writeable(self.cluster_analysis_output_path) data_map, data_type = self.allocate_prof_data() -- Gitee From 619652d8cd1d1caafe64d38861333c807a4999ef Mon Sep 17 00:00:00 2001 From: lijiaojiao Date: Tue, 3 Sep 2024 18:05:08 +0800 Subject: [PATCH 675/791] =?UTF-8?q?=E3=80=90=E9=97=AE=E9=A2=98=E5=8D=95?= =?UTF-8?q?=E3=80=91=E4=BF=AE=E5=A4=8D=E4=BA=86online=5Fdispatch=E7=A7=81?= =?UTF-8?q?=E6=9C=89=E6=96=87=E4=BB=B6=E6=A8=A1=E5=9D=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/pytorch/online_dispatch/compare.py | 2 +- .../msprobe/pytorch/online_dispatch/dispatch.py | 16 ++++++++-------- .../pytorch/online_dispatch/dump_compare.py | 5 +++-- .../msprobe/pytorch/online_dispatch/utils.py | 17 +---------------- .../test_utils_online_dispatch.py | 7 +------ 5 files changed, 14 insertions(+), 33 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py index 985e0f743..5755e1ecc 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py @@ -8,7 +8,7 @@ from rich.table import Table from rich.console import Console from msprobe.core.common.const import CompareConst, FileCheckConst from msprobe.core.common.file_check import FileOpen, change_mode -from .single_compare import single_benchmark_compare_wrap +from msprobe.pytorch.online_dispatch.single_compare import single_benchmark_compare_wrap from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index f63460950..541218852 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -1,8 +1,7 @@ import os import time import json -from pathlib import Path -from multiprocessing import Manager, Pool +from multiprocessing import Pool import torch @@ -18,10 +17,11 @@ else: from msprobe.core.common.utils import check_file_or_directory_path, check_path_before_create, load_yaml from msprobe.core.common.const import Const, CompareConst from msprobe.pytorch.common.log import logger -from .dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ +from msprobe.pytorch.online_dispatch.dump_compare import dispatch_workflow, dispatch_multiprocess, error_call, TimeStatistics, \ DispatchRunParam, DisPatchDataInfo -from .utils import get_callstack, data_to_cpu, get_sys_info, DispatchException, COMPARE_LOGO -from .compare import Comparator +from msprobe.pytorch.online_dispatch.utils import get_callstack, data_to_cpu, get_sys_info, DispatchException, COMPARE_LOGO +from msprobe.pytorch.online_dispatch.compare import Comparator +from msprobe.core.common.file_check import FileOpen, create_directory current_time = time.strftime("%Y%m%d%H%M%S") @@ -59,8 +59,8 @@ class PtdbgDispatch(TorchDispatchMode): self.root_npu_path = os.path.join(self.root_path, f'npu') check_path_before_create(self.root_cpu_path) check_path_before_create(self.root_npu_path) - Path(self.root_cpu_path).mkdir(mode=0o750, parents=True, exist_ok=True) - Path(self.root_npu_path).mkdir(mode=0o750, parents=True, exist_ok=True) + create_directory(self.root_cpu_path) + create_directory(self.root_npu_path) self.result_csv_path = os.path.join(self.root_path, RESULT_FILE_NAME) self.detail_csv_path = os.path.join(self.root_path, DETAILS_FILE_NAME) @@ -94,7 +94,7 @@ class PtdbgDispatch(TorchDispatchMode): logger.error("Please check train log, An exception may have occurred!") return check_file_or_directory_path(summary_path, False) - fp_handle = open(summary_path, "r") + fp_handle = FileOpen(summary_path, "r") while True: json_line_data = fp_handle.readline() if json_line_data == '\n': diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py index 5b2e94e3c..82cb56e3d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py @@ -7,7 +7,7 @@ import pandas as pd import torch from msprobe.pytorch.common.log import logger from msprobe.core.common.file_check import FileOpen -from .utils import np_save_data +from msprobe.core.common.utils import save_npy class DispatchRunParam: @@ -87,7 +87,8 @@ def dump_data(data, prefix, dump_path): if isinstance(data, torch.Tensor) and data.is_meta: return # dump data may greater than summary_list collect - np_save_data(data, prefix, dump_path) + path = os.path.join(dump_path, f'{prefix}.npy') + save_npy(data, path) def save_temp_summary(api_index, single_api_summary, path, lock): diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index 85786bf60..52b6a637b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -11,9 +11,7 @@ except ImportError: else: pta_cpu_device = torch.device("cpu") -from msprobe.core.common.const import CompareConst, FileCheckConst -from msprobe.core.common.file_check import change_mode -from msprobe.pytorch.common.log import logger +from msprobe.core.common.const import CompareConst cpu_device = torch._C.device("cpu") COLOR_RED = '\033[31m' @@ -69,19 +67,6 @@ def get_callstack(): return callstack -def np_save_data(data, file_name, data_path): - try: - if hasattr(data, "numpy"): - data = data.numpy() - dump_path = os.path.join(data_path, f'{file_name}.npy') - np.save(dump_path, data) - change_mode(dump_path, FileCheckConst.DATA_FILE_AUTHORITY) - except Exception as e: - logger.error("save numpy failed, error: {}".format(e)) - finally: - pass - - def data_to_cpu(data, deep, data_cpu): global cpu_device list_cpu = [] diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py index 764d844ad..4448716fe 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/online_dispatch/test_utils_online_dispatch.py @@ -23,7 +23,7 @@ import logging from pathlib import Path from unittest.mock import patch, MagicMock -from msprobe.pytorch.online_dispatch.utils import COLOR_RED, COLOR_CYAN, COLOR_YELLOW, COLOR_RESET, COMPARE_LOGO, np_save_data, data_to_cpu, DispatchException +from msprobe.pytorch.online_dispatch.utils import COLOR_RED, COLOR_CYAN, COLOR_YELLOW, COLOR_RESET, COMPARE_LOGO, data_to_cpu, DispatchException cpu_device = torch._C.device("cpu") @@ -48,11 +48,6 @@ class TestUtils(unittest.TestCase): if os.path.exists(os.path.join(self.data_path, f'{self.file_name}.npy')): os.remove(os.path.join(self.data_path, f'{self.file_name}.npy')) - @patch('msprobe.core.common.file_check.change_mode') - def test_np_save_data_should_error_when_input_wrong(self,mock_change_mode): - np_save_data(self.data_nonumpy,self.file_name,self.data_path) - mock_change_mode.assert_not_called() - def test_data_to_cpu_should_return_tensor_copy_when_input_tensor(self): data = torch.tensor([1,2],device=cpu_device,dtype=torch.float16) deep=1 -- Gitee From 97a3ce9f9ed85147bad2e3d8f93cb092e916eba4 Mon Sep 17 00:00:00 2001 From: zyy Date: Wed, 4 Sep 2024 09:20:23 +0800 Subject: [PATCH 676/791] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=BE=93=E5=87=BA?= =?UTF-8?q?=E8=B7=AF=E5=BE=84=E7=AC=A6=E5=8F=B7=E6=A0=A1=E9=AA=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 56d69a7a8..1b63ed94e 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -70,9 +70,6 @@ class Interface: def run(self): PathManager.check_input_directory_path(self.collection_path) PathManager.check_path_owner_consistent(self.collection_path) - PathManager.check_input_directory_path(self.cluster_analysis_output_path) - FileManager.create_output_dir(self.cluster_analysis_output_path) - PathManager.check_path_writeable(self.cluster_analysis_output_path) data_map, data_type = self.allocate_prof_data() if not data_map: print("[WARNING] Can not get rank info or profiling data.") @@ -80,7 +77,9 @@ class Interface: if data_type == Constant.INVALID: print("[ERROR] The current folder contains both DB and other files. Please check.") return - + PathManager.check_input_directory_path(self.cluster_analysis_output_path) + FileManager.create_output_dir(self.cluster_analysis_output_path) + PathManager.check_path_writeable(self.cluster_analysis_output_path) params = { Constant.COLLECTION_PATH: self.collection_path, Constant.DATA_MAP: data_map, -- Gitee From bfe1875b7a6dc447f917d72f9c613c1617e77a62 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 4 Sep 2024 09:31:57 +0800 Subject: [PATCH 677/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Ddt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/grad_probe/utils.py | 3 +-- .../msprobe/mindspore/grad_probe/global_context.py | 3 +-- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py index 6aaebf110..36a7e5a33 100644 --- a/debug/accuracy_tools/msprobe/core/grad_probe/utils.py +++ b/debug/accuracy_tools/msprobe/core/grad_probe/utils.py @@ -1,8 +1,7 @@ import re from msprobe.core.grad_probe.constant import GradConst from msprobe.core.common.log import logger -from msprobe.core.common.utils import check_path_before_create, change_mode -from msprobe.core.common.file_utils import write_csv +from msprobe.core.common.file_utils import write_csv, check_path_before_create, change_mode from msprobe.core.common.const import FileCheckConst import matplotlib.pyplot as plt diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py index d36f560f2..a081d1fad 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -5,8 +5,7 @@ from typing import Dict, Union from msprobe.core.grad_probe.utils import check_str from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_utils import create_directory -from msprobe.core.common.utils import check_path_before_create +from msprobe.core.common.file_utils import create_directory, check_path_before_create class GlobalContext: diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 457032742..737f91ad9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -24,8 +24,8 @@ import numpy as np from functools import wraps from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.core.common.log import logger -from msprobe.core.common.utils import check_path_before_create -from msprobe.core.common.file_utils import FileCheckConst, change_mode, check_file_or_directory_path +from msprobe.core.common.file_utils import (FileCheckConst, change_mode, + check_file_or_directory_path, check_path_before_create) try: -- Gitee From c58dfc8858d1e4b3d86c155508c89b2cde44e925 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Mon, 2 Sep 2024 21:36:45 +0800 Subject: [PATCH 678/791] =?UTF-8?q?[msprobe/mindspore/compare]Mindspore?= =?UTF-8?q?=E7=9A=84=E6=AF=94=E5=AF=B9=E5=B7=A5=E5=85=B7=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E5=A4=9A=E4=B8=AA=E5=9C=BA=E6=99=AF=E4=BF=AE=E6=94=B9=E8=B5=84?= =?UTF-8?q?=E6=96=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../docs/10.accuracy_compare_PyTorch.md | 4 +- .../docs/11.accuracy_compare_MindSpore.md | 198 +++++++++++++++--- 2 files changed, 171 insertions(+), 31 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md index e59dd115a..bfd994138 100644 --- a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md @@ -52,7 +52,7 @@ | 参数名 | 说明 | 是否必选 | |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i 或 --input_path | 指定[比对文件](#211-比对文件)路径。 | 是 | + | -i 或 --input_path | 指定[比对文件](#211-比对文件)。 | 是 | | -o 或 --output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | | -s 或 --stack_mode | 配置 stack_mode 的开关。仅当[比对文件](#211-比对文件)配置 stack_path 需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | | -c 或 --compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | @@ -61,7 +61,7 @@ 3. 查看比对结果，请参见 [3 精度比对结果分析](#3-精度比对结果分析)。 #### 2.1.1 比对文件 - + 以在当前目录创建 ./compare.json 为例。 - 单卡场景示例： diff --git a/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md b/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md index b0977eb6a..4bd383e8c 100644 --- a/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/11.accuracy_compare_MindSpore.md @@ -2,61 +2,201 @@ ## 1 简介 -msprobe 精度比对工具主要通过对同一个模型，在两个不同的 MindSpore 环境下，输入相同的训练数据，在分别得到 dump 数据后，对这两个 dump 数据进行全量自动对比，从而快速定位不同版本之间的精度问题。 +msprobe精度比对工具主要用于如下场景： -## 2 精度比对操作指导 +- 通过对同一个网络模型，在两个不同版本的MindSpore静态图环境下，输入相同的训练数据，在分别得到API dump数据后，对这两个API dump数据进行全量自动比对，从而快速定位不同版本之间的精度问题。 +- 通过对同一个网络模型，在两个不同版本的MindSpore静态图环境下，输入相同的训练数据，在分别得到kernel dump数据后，对这两个kernel dump数据进行全量自动比对，从而快速定位不同版本之间的精度问题。 +- 通过对同一个网络模型，在两个不同版本的MindSpore动态图环境下，输入相同的训练数据，在分别得到cell dump数据后，对这两个cell模块进行全量自动比对，从而快速定位不同版本之间的精度问题。 +- 通过对同一个网络模型，在整网环境下分别在MindSpore动态图和PyTorch环境下获得API dump数据，以PyTorch数据作为标杆，进行自动比对，从而实现跨框架的精度对比。 +- 通过对同一个网络模型，在整网环境下分别在MindSpore动态图和PyTorch环境下获得cell dump数据，由用户指定可以比对的cell list，以PyTorch数据作为标杆，进行自动比对，从而实现跨框架的精度对比。 -### 2.1 命令行方式 +执行精度比对操作需要安装msprobe工具。详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 -该工具目前使用方式为命令行形式，对 MindSpore 的 dump 数据仅支持单卡。 +## 2 命令行方式 -请先参见 [MindSpore 场景下的数据采集](./06.data_dump_MindSpore.md)章节，完成不同环境下 MindSpore 精度数据的采集。 +精度比对工具目前使用方式为命令行形式。 -**操作步骤**： +### 2.1 命令格式说明 -1. 使用 MindSpore 进行 dump，得到不同框架版本的 dump 数据。 +```shell +msprobe -f mindspore compare -i ./compare.json -o ./output -s +``` + +**完整参数说明** + +| 参数名 | 说明 | 是否必选 | +| ------------------- | ------------------------------------------------------------ | -------- | +| -i或--input_path | 指定比对文件。比对文件内容及示例请参见[比对文件](#31-比对文件)或[比对文件（kernel）](#32-比对文件（kernel）)（比对文件（kernel）仅[不同版本下的全量kernel比对](#23-不同版本下的全量kernel比对)场景支持）。 | 是 | +| -o或--output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：
`compare_result_{timestamp}.xlsx`
`compare_result_{rank_id}_{step_id}_{timestamp}.xlsx`（仅[不同版本下的全量kernel比对](#23-不同版本下的全量kernel比对)场景支持）。 | 是 | +| -s或--stack_mode | 配置stack_mode的开关。仅当[比对文件](#31-比对文件)配置"stack_path"需要开启。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | +| -c或--compare_only | 仅比对开关。该参数默认未配置，表示关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的“Accuracy Reached or Not”列显示为No），并给出问题可能产生的原因（打屏展示并生成advisor_{timestamp}.txt文件）。配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | +| -f或--fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的API，可匹配并进行比对。通过直接配置该参数开启，默认未配置，表示关闭。 | 否 | +| -am或--api_mapping | 跨框架比对。配置该参数时表示开启跨框架API比对功能。仅[跨框架的API比对](#25-跨框架的API比对)场景支持。 | 否 | +| -cm或--cell_mapping | 跨框架比对。配置该参数时表示开启跨框架cell模块比对功能，可以指定自定义映射文件*.yaml，不指定映射文件时按照msprobe定义的默认映射关系进行比对。自定义映射文件的格式请参见[自定义映射文件](#33-自定义映射文件)。仅[跨框架的cell模块比对](#26-跨框架的cell模块比对)场景支持。 | 否 | + +### 2.2 不同版本下的全量API比对 + +1. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》完成不同环境下MindSpore静态图精度数据的采集，得到不同框架版本的API dump数据。 + +2. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 + +3. 执行如下示例命令进行比对： + + ```shell + msprobe -f mindspore compare -i ./compare.json -o ./output -s + ``` + +4. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)]》章节。 + +### 2.3 不同版本下的全量kernel比对 + +1. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》完成不同环境下MindSpore静态图精度数据的采集，得到不同框架版本的kernel dump数据。 + +2. 创建比对文件，文件内容及示例请参见[比对文件（kernel）](#32-比对文件（kernel）)。 + +3. 执行如下示例命令进行比对： -2. 创建[比对文件](#211-比对文件)。 + ```shell + msprobe -f mindspore compare -i ./compare.json -o ./output + ``` + +4. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)]》章节。 + +### 2.4 不同版本下的cell模块比对 -3. 运行命令： +1. 配置[config.json](../config.json)文件level配置为L0、task配置为tensor或statistics并指定需要dump的cell模块名。 + +2. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》完成不同环境下MindSpore动态图精度数据的采集，得到不同框架版本的cell模块dump数据。 + +3. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 + +4. 执行如下示例命令进行比对： ```shell msprobe -f mindspore compare -i ./compare.json -o ./output -s ``` - **参数说明**： +5. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)]》章节。 + +### 2.5 跨框架的API比对 - | 参数名 | 说明 | 是否必选 | - |-------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | - | -i 或 --input_path | 指定[比对文件](#211-比对文件)路径。 | 是 | - | -o 或 --output_path | 配置比对结果文件存盘目录。文件名称基于时间戳自动生成，格式为：`compare_result_{timestamp}.xlsx`。 | 是 | - | -s 或 --stack_mode | 配置 stack_mode 的开关。仅当比对文件配置 stack_path 需要开启。默认未配置，表示关闭。 | 否 | - | -c 或 --compare_only | 仅比对开关。未配置默认关闭仅比对，使用自动精度分析，工具自动针对比对结果进行分析，识别到第一个精度不达标节点（在比对结果文件中的 Accuracy Reached or Not 列显示为 No），并给出问题可能产生的原因（打屏展示并生成 `advisor_{timestamp}.txt` 文件）。该参数默认未配置，使用自动精度分析，通过配置该参数开启仅比对，关闭自动精度分析，仅输出比对结果表格。 | 否 | - | -f 或 --fuzzy_match | 模糊匹配。开启后，对于网络中同一层级且命名仅调用次数不同的 API，可匹配并进行比对。默认未配置，表示关闭。 | 否 | +1. 配置[config.json](../config.json)文件level配置为L1、task配置为tensor或statistics。 -4. 查看比对结果，请参见 [PyTorch 场景的精度比对——精度比对结果分析](./11.accuracy_compare_PyTorch.md#3-精度比对结果分析)章节。 +2. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》和《[PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)》完成不同环境下API精度数据的采集，得到两个框架的API dump数据。 +3. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 +4. 执行如下示例命令进行比对： -#### 2.1.1 比对文件 + ```shell + msprobe -f mindspore compare -i ./compare.json -o ./output -s -am + ``` + +5. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)]》章节。 + +### 2.6 跨框架的cell模块比对 -以在当前目录创建 ./compare.json 为例，单卡场景示例如下： +1. 配置[config.json](../config.json)文件level配置为L0、task配置为tensor或statistics并指定需要dump的cell模块名。 -```json +2. 参见《[MindSpore 场景的精度数据采集](./06.data_dump_MindSpore.md)》和《[PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)》完成不同环境下cell模块精度数据的采集，得到两个框架的cell模块dump数据。 + +3. 创建比对文件，文件内容及示例请参见[比对文件](#31-比对文件)。 + +4. 执行如下示例命令进行比对： + + ```shell + msprobe -f mindspore compare -i ./compare.json -o ./output -s -cm + ``` + + 或 + + ```shell + msprobe -f mindspore compare -i ./compare.json -o ./output -s -cm cell_mapping.yaml + ``` + +5. 查看比对结果，请详见PyTorch目录下的《[PyTorch 场景的精度比对-精度比对结果分析](./10.accuracy_compare_PyTorch.md#3-精度比对结果分析)]》章节。 + +## 3 附录 + +### 3.1 比对文件 + +以在当前目录创建./compare.json为例，单卡场景示例如下： + + + ```json { "npu_path": "./npu_dump/dump.json", "bench_path": "./bench_dump/dump.json", "stack_path": "./npu_dump/stack.json", "is_print_compare_log": true } + ``` + + +**参数说明** + +| 参数名 | 说明 | 是否必选 | +| -------------------- | ------------------------------------------------------------ | -------- | +| npu_path | 配置NPU环境下的dump.json文件（单卡场景）。跨框架场景指定为MindSpore的json文件。数据类型：str。 | 是 | +| bench_path | 配置CPU、GPU或NPU环境下的dump.json文件（单卡场景）。 跨框架场景指定为PyTorch的json文件。数据类型：str。 | 是 | +| stack_path | 配置NPU dump目录下的stack.json文件。数据类型：str。 | 是 | +| is_print_compare_log | 配置是否开启日志打屏。可取值true或false，默认为true。数据类型：bool | 否 | + +### 3.2 比对文件（kernel） + +仅[不同版本下的全量kernel比对](#23-不同版本下的全量kernel比对)场景支持。 + +以在当前目录创建./compare.json为例，示例如下： + +- 单卡场景： + + ```json + { + "npu_path": "./npu_dump", + "bench_path": "./bench_dump", + "rank_id": [1], + "step_id": [] + } + ``` + +- 多卡场景： + + ```json + { + "npu_path": "./npu_dump", + "bench_path": "./bench_dump", + "rank_id": [], + "step_id": [] + } + ``` + + +**参数说明** + +| 参数名 | 说明 | 是否必选 | +| ---------- | ------------------------------------------------------------ | -------- | +| npu_path | 配置NPU环境下的dump.json文件目录（单卡场景）或真实数据目录（多卡场景）。数据类型：str。 | 是 | +| bench_path | 配置CPU、GPU或NPU环境下的dump.json文件目录（单卡场景）或真实数据目录（多卡场景）。数据类型：str。 | 是 | +| rank_id | 配置比对的Rank ID。npu_path和bench_path目录下的dump文件需要存在对应Rank的数据。默认为空，表示比对所有Rank。可配置一个或多个Rank，多个Rank ID用逗号隔开，例如："rank_id": [1,2,3]。数据类型：list[int]。 | 否 | +| step_id | 配置比对的Step ID。npu_path和bench_path目录下的dump文件需要存在对应Step的数据。默认为空，表示比对所有Sank。可配置一个或多个Sank，多个Sank ID用逗号隔开，例如："step_id": [1,2,3]。数据类型：list[int]。 | 否 | + +### 3.3 自定义映射文件 + +文件名格式：\*.yaml，*为文件名，可自定义。 + +文件内容格式： + +```yaml +{cell_name}.{class_name}: {cell_name}.{class_name} ``` -**参数说明**： +左侧为MindSpore框架cell模块的{cell_name}.{class_name}，右侧为PyTorch框架cell模块的{cell_name}.{class_name}。 -| 参数名 | 说明 | 是否必选 | -| -------------------- | ------------------------------------------------------------ | ------------------ | -| npu_path | 配置 NPU 环境下的 dump.json 文件（单卡场景），str 类型。 | 是 | -| bench_path | 配置 CPU、GPU 或 NPU 环境下的 dump.json 文件（单卡场景），str 类型。 | 是 | -| stack_path | 配置 NPU dump 目录下的 stack.json 文件，str 类型。 | 是| -| is_print_compare_log | 配置是否开启日志打屏，bool 类型。可取值 True 或 False，默认为 True。 | 否 | - \ No newline at end of file +{cell_name}.{class_name}从dump cell模块级.npy文件名获取，命名格式为：`{Cell}_{cell_name}_{class_name}_{前向反向}.{index}.{input/output}.{参数序号}` + +文件内容示例： + +```yaml +fc2.Dense: fc2.Linear +conv1.Conv2d: conv3.Conv2d +``` -- Gitee From 9da4fb0951d27b02de9ca4016ca690df6edf0ddf Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 4 Sep 2024 09:42:59 +0800 Subject: [PATCH 679/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/mindspore/compare/ms_graph_compare.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index 933172aa9..821ec5928 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -8,12 +8,11 @@ import numpy as np import pandas as pd from msprobe.core.common.const import CompareConst, GraphMode from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import FileOpen, load_npy from msprobe.mindspore.common.log import logger -from msprobe.core.common.utils import add_time_with_xlsx, CompareException, load_npy +from msprobe.core.common.utils import add_time_with_xlsx, CompareException from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply -from msprobe.core.common.file_check import FileOpen class row_data: def __init__(self, mode): -- Gitee From d6ee5152c57102973fd085c6d53a0a7dfffbd16f Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 4 Sep 2024 09:48:20 +0800 Subject: [PATCH 680/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/pytorch/online_dispatch/compare.py | 2 +- .../msprobe/pytorch/online_dispatch/dump_compare.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py index 5755e1ecc..473d9819e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/compare.py @@ -7,7 +7,7 @@ from collections import namedtuple from rich.table import Table from rich.console import Console from msprobe.core.common.const import CompareConst, FileCheckConst -from msprobe.core.common.file_check import FileOpen, change_mode +from msprobe.core.common.file_utils import FileOpen, change_mode from msprobe.pytorch.online_dispatch.single_compare import single_benchmark_compare_wrap from msprobe.pytorch.common.log import logger from msprobe.core.common.utils import CompareException diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py index 82cb56e3d..64a3c1ef7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dump_compare.py @@ -3,11 +3,9 @@ import json import copy from datetime import datetime, timezone -import pandas as pd import torch from msprobe.pytorch.common.log import logger -from msprobe.core.common.file_check import FileOpen -from msprobe.core.common.utils import save_npy +from msprobe.core.common.file_utils import FileOpen, save_npy class DispatchRunParam: -- Gitee From b00904ebaa93111626b560b01912ebb6be4b18e8 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 4 Sep 2024 09:56:55 +0800 Subject: [PATCH 681/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Ddt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py | 2 +- .../test/mindspore_ut/grad_probe/test_ms_grad_monitor.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py index 9bf6c4e7d..441f2b992 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/dispatch.py @@ -21,7 +21,7 @@ from msprobe.pytorch.online_dispatch.dump_compare import dispatch_workflow, disp DispatchRunParam, DisPatchDataInfo from msprobe.pytorch.online_dispatch.utils import get_callstack, data_to_cpu, get_sys_info, DispatchException, COMPARE_LOGO from msprobe.pytorch.online_dispatch.compare import Comparator -from msprobe.core.common.file_check import FileOpen, create_directory +from msprobe.core.common.file_utils import FileOpen, create_directory current_time = time.strftime("%Y%m%d%H%M%S") diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index f6f399db2..3af0d5bbc 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -8,7 +8,7 @@ import mindspore from mindspore import nn, Tensor from mindspore.nn import SGD from msprobe.mindspore import PrecisionDebugger -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.file_utils import FileOpen file_path = os.path.abspath(__file__) directory = os.path.dirname(file_path) -- Gitee From de007488716edbed1ae0de18b74387b5eaf24436 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 4 Sep 2024 10:13:55 +0800 Subject: [PATCH 682/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Ddt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/file_utils.py | 9 ++++++--- .../msprobe/test/core_ut/common/test_utils.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index fd3b9b019..0316a47f5 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -38,6 +38,7 @@ class FileChecker: ability(str): FileCheckConst.WRITE_ABLE or FileCheckConst.READ_ABLE to set file has writability or readability file_type(str): The correct file type for file """ + def __init__(self, file_path, path_type, ability=None, file_type=None, is_script=True): self.file_path = file_path self.path_type = self._check_path_type(path_type) @@ -193,7 +194,7 @@ def check_path_owner_consistent(path): def check_path_pattern_vaild(path): if not re.match(FileCheckConst.FILE_VALID_PATTERN, path): - logger.error('The file path %s contains special characters.' %(path)) + logger.error('The file path %s contains special characters.' % (path)) raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) @@ -204,7 +205,8 @@ def check_file_size(file_path, max_size): logger.error('Failed to open "%s". %s' % (file_path, str(os_error))) raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) from os_error if file_size >= max_size: - logger.error(f'The size of file path {file_path} exceeds {max_size} bytes.') + logger.error('The size (%d) of %s exceeds (%d) bytes, tools not support.' + % (file_size, file_path, max_size)) raise FileCheckException(FileCheckException.FILE_TOO_LARGE_ERROR) @@ -249,7 +251,8 @@ def create_directory(dir_path): os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) except OSError as ex: raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, - 'Failed to create {}. Please check the path permission or disk space .{}'.format(dir_path, str(ex))) from ex + 'Failed to create {}. Please check the path permission or disk space .{}'.format( + dir_path, str(ex))) from ex file_check = FileChecker(dir_path, FileCheckConst.DIR) file_check.common_check() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 304d650aa..584c831de 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -333,9 +333,9 @@ class TestUtils(TestCase): @patch('msprobe.core.common.file_utils.get_file_content_bytes') def test_get_json_contents_should_raise_exception(self, mock_get_file_content_bytes): mock_get_file_content_bytes.return_value = 'not a dict' - with self.assertRaises(CompareException) as ce: + with self.assertRaises(FileCheckException) as ce: get_json_contents('') - self.assertEqual(ce.exception.code, CompareException.INVALID_FILE_ERROR) + self.assertEqual(ce.exception.code, FileCheckException.INVALID_FILE_ERROR) def test_get_json_contents_should_return_json_obj(self): test_dict = {"key": "value"} -- Gitee From 54987e24946f7820a4854bf2d5689422e03b122f Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 4 Sep 2024 10:25:22 +0800 Subject: [PATCH 683/791] BaseLogger add debug level --- debug/accuracy_tools/msprobe/core/common/const.py | 3 ++- debug/accuracy_tools/msprobe/core/common/log.py | 7 +++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index b9bababff..d09f3641d 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -322,7 +322,8 @@ class MsgConst: Class for log messages const """ CLEAR_SYMBOL = "\033[K" - LEVEL = ["INFO", "WARNING", "ERROR"] + LOG_LEVEL = "LOG_LEVEL" + LEVEL = ["INFO", "WARNING", "ERROR", "DEBUG"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 248fed365..414e604cc 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -4,6 +4,8 @@ import sys from functools import wraps from msprobe.core.common.const import MsgConst +LOG_LEVEL = os.environ.get(MsgConst.LOG_LEVEL, "") + class BaseLogger: def __init__(self): @@ -33,6 +35,11 @@ class BaseLogger: def warning(self, msg): self._print_log(MsgConst.LEVEL[1], msg) + @filter_special_chars + def debug(self, msg): + if LOG_LEVEL == MsgConst.LEVEL[3]: + self._print_log(MsgConst.LEVEL[1], msg) + def on_rank_0(self, func): def func_rank_0(*args, **kwargs): current_rank = self.get_rank() -- Gitee From ede6e30ecaf1feac6183850801ccdbb601c56ebf Mon Sep 17 00:00:00 2001 From: RICHARDNAN <1466684392@qq.com> Date: Wed, 4 Sep 2024 10:40:44 +0800 Subject: [PATCH 684/791] =?UTF-8?q?=E7=AE=97=E5=AD=90=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E8=A7=84=E5=88=99=E6=96=B0=E5=A2=9Eflash2attn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/compare_tools/compare_backend/utils/constant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index 51c7ce737..d1ddf68d9 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -92,7 +92,7 @@ class Constant(object): BWD_LIST = ["bwd", "backward", "back", "grad"] - CPU_OP_FA_MASK = ("flash_attention", "fusion_attention", "flashattn", "xformers_flash", "efficient_attention") + CPU_OP_FA_MASK = ("flash_attention", "fusion_attention", "flashattn", "xformers_flash", "efficient_attention", "flsh2atten") CPU_OP_CONV = "aten::conv" CPU_OP_MATMUL_MASK = ("aten::addmm", "aten::bmm", "aten::mm", "aten::matmul") KERNEL_CUBE_MASK = ("gemm", "conv", "cutlass", "wgrad") -- Gitee From 5c53f8c5ac7d43c8aaf1776c36bb7ce575837a46 Mon Sep 17 00:00:00 2001 From: jijiarong Date: Mon, 2 Sep 2024 19:59:22 +0800 Subject: [PATCH 685/791] clean code --- .../msprobe/mindspore/common/utils.py | 2 +- .../mindspore/compare/ms_graph_compare.py | 100 ++++++++++-------- 2 files changed, 59 insertions(+), 43 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index 9b64abef3..a0dd3aa2e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -46,7 +46,7 @@ def save_tensor_as_npy(tensor, file_path): def convert_to_int(value): try: return int(value) - except ValueError: + except Exception: return -1 diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index 06cb6533a..9050e0c4a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -5,10 +5,10 @@ import os import numpy as np import pandas as pd -from msprobe.core.common.const import CompareConst, GraphMode -from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.const import CompareConst, GraphMode, Const, FileCheckConst +from msprobe.core.common.file_check import FileOpen, check_path_before_create, change_mode from msprobe.core.common.log import logger -from msprobe.core.common.utils import add_time_with_xlsx, CompareException, load_npy +from msprobe.core.common.utils import add_time_with_xlsx, CompareException from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply from msprobe.mindspore.common.utils import convert_to_int, list_lowest_level_directories @@ -32,9 +32,11 @@ def npy_data_read(data_path, npy_file_list, mapping_dict): data_list = [] for data in npy_file_list: if data in mapping_dict: - split_list = mapping_dict[data].split(CompareConst.SEP) + split_list = mapping_dict[data].split(Const.SEP) else: - split_list = data.split(".") + split_list = data.split(Const.SEP) + if len(split_list) < 7: + continue compare_key = f"{split_list[1]}.{split_list[2]}.{split_list[3]}.{split_list[5]}.{split_list[6]}" timestamp = convert_to_int(split_list[4]) @@ -48,7 +50,7 @@ def statistic_data_read(statistic_file_list, statistic_file_path): header_index = {'Data Type': None, 'Shape': None, 'Max Value': None, 'Min Value': None, 'Avg Value': None, 'L2Norm Value': None} for statistic_file in statistic_file_list: - with FileOpen(statistic_file, "r") as f: + with open(statistic_file, "r") as f: csv_reader = csv.reader(f, delimiter=",") header = next(csv_reader) for key in header_index.keys(): @@ -114,6 +116,18 @@ def generate_data_name(data_path): return mode, data_list +def read_npy_data(data_path): + try: + data_value = np.load(data_path) + if data_value.dtype == np.float16: + data_value = data_value.astype(np.float32) + except FileNotFoundError as e: + data_value = None + except EOFError: + data_value = None + return data_value + + class GraphMSComparator: def __init__(self, input_param, output_path): self.output_path = output_path @@ -127,41 +141,6 @@ class GraphMSComparator: self.common_rank_step = sorted( set(self.npu_rank_step_dict.keys()).intersection(self.bench_rank_step_dict.keys())) - def generate_rank_step_path(self, base_path): - - def generate_rank_step_id(path_with_rank_step): - split_path = path_with_rank_step.split("/") - rank_id = -1 - if "rank_" in path_with_rank_step: - # KBK mode - if len(split_path > 4): - rank_id = convert_to_int(split_path[-4].split("_")[-1]) - step_id = convert_to_int(split_path[-1]) - else: - if len(split_path > 4): - rank_id = convert_to_int(split_path[-4]) - step_id = convert_to_int(split_path[-1]) - return rank_id, step_id - - base_path = os.path.abspath(base_path) - lowest_level = list_lowest_level_directories(base_path) - - rank_step_path_dict = {} - for dir_path in lowest_level: - rank_id, step_id = generate_rank_step_id(dir_path) - if rank_id == -1 or step_id == -1: - continue - if self.rank_list and rank_id not in self.rank_list: - continue - if self.step_list and step_id not in self.step_list: - continue - rank_step_key = (rank_id, step_id) - if rank_step_key in rank_step_path_dict: - rank_step_path_dict[rank_step_key].append(dir_path) - else: - rank_step_path_dict[rank_step_key] = [dir_path] - return dict(sorted(rank_step_path_dict.items())) - @staticmethod def compare_ops(compare_result_db, mode): @@ -170,7 +149,7 @@ class GraphMSComparator: def process_npy_file(file_path, name_prefix, result): if os.path.exists(file_path): - data = load_npy(file_path) + data = read_npy_data(file_path) result[f'{name_prefix} Name'] = file_path result[f'{name_prefix} Dtype'] = data.dtype result[f'{name_prefix} Tensor Shape'] = data.shape @@ -271,7 +250,9 @@ class GraphMSComparator: compare_result_df = self._do_multi_process(compare_result_df, mode) compare_result_name = add_time_with_xlsx(f"compare_result_{str(rank_id)}_{str(step_id)}") compare_result_path = os.path.join(os.path.realpath(self.output_path), f"{compare_result_name}") + check_path_before_create(compare_result_path) compare_result_df.to_excel(compare_result_path, index=False) + change_mode(compare_result_path, FileCheckConst.DATA_FILE_AUTHORITY) logger.info(f"Compare rank: {rank_id} step: {step_id} finish. Compare result: {compare_result_path}.") def compare_process(self, rank_id, step_id): @@ -333,6 +314,41 @@ class GraphMSComparator: return compare_result_df, npu_mode + def generate_rank_step_path(self, base_path): + + def generate_rank_step_id(path_with_rank_step): + split_path = path_with_rank_step.split("/") + rank_id = -1 + if "rank_" in path_with_rank_step: + # KBK mode + if len(split_path) > 4: + rank_id = convert_to_int(split_path[-4].split("_")[-1]) + step_id = convert_to_int(split_path[-1]) + else: + if len(split_path) > 4: + rank_id = convert_to_int(split_path[-4]) + step_id = convert_to_int(split_path[-1]) + return rank_id, step_id + + base_path = os.path.abspath(base_path) + lowest_level = list_lowest_level_directories(base_path) + + rank_step_path_dict = {} + for dir_path in lowest_level: + rank_id, step_id = generate_rank_step_id(dir_path) + if rank_id == -1 or step_id == -1: + continue + if self.rank_list and rank_id not in self.rank_list: + continue + if self.step_list and step_id not in self.step_list: + continue + rank_step_key = (rank_id, step_id) + if rank_step_key in rank_step_path_dict: + rank_step_path_dict[rank_step_key].append(dir_path) + else: + rank_step_path_dict[rank_step_key] = [dir_path] + return dict(sorted(rank_step_path_dict.items())) + def _do_multi_process(self, result_df, mode): try: result_df = _ms_graph_handle_multi_process(self.compare_ops, result_df, mode) -- Gitee From 41ba451e36d8c383404590eea6615734cbe51db2 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Wed, 4 Sep 2024 11:18:45 +0800 Subject: [PATCH 686/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Ddt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py b/debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py index 4517304f8..cbcd8be1e 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_file_check.py @@ -177,7 +177,8 @@ class TestFileCheckUtil(TestCase): check_file_size(file_path, max_size) self.assertEqual(str(context.exception), FileCheckException.err_strs.get(FileCheckException.FILE_TOO_LARGE_ERROR)) - mock_logger_error.assert_called_with(f"The size of file path {file_path} exceeds {max_size} bytes.") + mock_logger_error.assert_called_with(f"The size ({file_size}) of {file_path} exceeds ({max_size}) bytes, " + f"tools not support.") def test_check_common_file_size(self): mock_check_file_size = MagicMock() -- Gitee From 18f849786af935bc8ada46a284e3a874de41aae2 Mon Sep 17 00:00:00 2001 From: yangxinxian <947098055@qq.com> Date: Wed, 4 Sep 2024 14:07:42 +0800 Subject: [PATCH 687/791] =?UTF-8?q?=E5=8F=96=E6=B6=88=E4=B8=AD=E9=80=94?= =?UTF-8?q?=E9=80=80=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/mindspore/service.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 40620cb4a..42f742f99 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -60,6 +60,7 @@ class Service: self.dump_iter_dir = None self.start_call = False self.check_level_valid() + self.should_stop_service = False @staticmethod def check_model_valid(model): @@ -252,14 +253,19 @@ class Service: self.primitive_counters.clear() def start(self, model=None): - self.model = self.check_model_valid(model) self.start_call = True - logger.info("msprobe: debugger.start() is set successfully") + if self.should_stop_service: + return if self.config.step and self.current_iter > max(self.config.step): - self.stop() - raise Exception("msprobe: exit after iteration {}".format(max(self.config.step))) + api_register.api_set_ori_func() + self.should_stop_service = True + logger.info("msprobe: dump has successfully ended") + return if self.config.step and self.current_iter not in self.config.step: return + self.model = self.check_model_valid(model) + + logger.info("msprobe: debugger.start() is set successfully") if self.first_start: try: self.current_rank = get_rank_if_initialized() @@ -284,6 +290,8 @@ class Service: PIJitCaptureContext.__exit__ = self.empty def stop(self): + if self.should_stop_service: + return logger.info("msprobe: debugger.stop() is set successfully. " "Please set debugger.start() to turn on the dump switch again. ") if not self.start_call: -- Gitee From 5476a3e3dff95c5b98f182f16e159050a644cefa Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Tue, 3 Sep 2024 17:47:15 +0800 Subject: [PATCH 688/791] feature: cluster optimized --- .../analysis/analysis_facade.py | 11 +- .../analysis/comm_matrix_analysis.py | 21 +++ .../analysis/communication_analysis.py | 123 ++++++++++++++ profiler/cluster_analyse/cluster_analysis.py | 5 +- .../cluster_analyse/common_func/constant.py | 9 ++ .../common_func/tables_config.py | 47 +++++- .../communication_db_group.py | 152 ++++++++++++++---- .../communication_group_generator.py | 18 ++- .../prof_bean/communication_bandwidth_bean.py | 82 ++++++++++ .../prof_bean/communication_time_bean.py | 70 ++++++++ 10 files changed, 504 insertions(+), 34 deletions(-) create mode 100644 profiler/cluster_analyse/prof_bean/communication_bandwidth_bean.py create mode 100644 profiler/cluster_analyse/prof_bean/communication_time_bean.py diff --git a/profiler/cluster_analyse/analysis/analysis_facade.py b/profiler/cluster_analyse/analysis/analysis_facade.py index c438fbcf3..6a3fc052e 100644 --- a/profiler/cluster_analyse/analysis/analysis_facade.py +++ b/profiler/cluster_analyse/analysis/analysis_facade.py @@ -16,12 +16,17 @@ from multiprocessing import Process from analysis.communication_analysis import CommunicationAnalysis +from analysis.communication_analysis import CommunicationAnalysisOptimized from analysis.comm_matrix_analysis import CommMatrixAnalysis +from analysis.comm_matrix_analysis import CommMatrixAnalysisOptimized from analysis.step_trace_time_analysis import StepTraceTimeAnalysis from analysis.host_info_analysis import HostInfoAnalysis +from common_func.constant import Constant + class AnalysisFacade: default_module = {CommunicationAnalysis, StepTraceTimeAnalysis, CommMatrixAnalysis, HostInfoAnalysis} + simplified_module = {CommunicationAnalysisOptimized, StepTraceTimeAnalysis, CommMatrixAnalysisOptimized, HostInfoAnalysis} def __init__(self, params: dict): self.params = params @@ -29,7 +34,11 @@ class AnalysisFacade: def cluster_analyze(self): # 多个profiler用多进程处理 process_list = [] - for analysis in self.default_module: + if self.params.get(Constant.DATA_SIMPLIFICATION) and self.params.get(Constant.DATA_TYPE) == Constant.DB: + analysis_module = self.simplified_module + else: + analysis_module = self.default_module + for analysis in analysis_module: process = Process(target=analysis(self.params).run) process.start() process_list.append(process) diff --git a/profiler/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/cluster_analyse/analysis/comm_matrix_analysis.py index 5d674aa52..b8cf865d7 100644 --- a/profiler/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/cluster_analyse/analysis/comm_matrix_analysis.py @@ -104,3 +104,24 @@ class CommMatrixAnalysis(BaseAnalysis): self.compute_ratio(link_dict.get(Constant.TRANSIT_SIZE_MB, 0), link_dict.get(Constant.TRANSIT_TIME_MS, 0)) step_dict[Constant.TOTAL_OP_INFO] = total_op_info + + +class CommMatrixAnalysisOptimized(CommMatrixAnalysis): + SAVED_JSON = "cluster_communication_matrix.json" + COMMUNICATION_MATRIX_TABLE = "ClusterCommunicationMatrix" + + def __init__(self, param: dict): + super().__init__(param) + + def dump_db(self): + res_comm_matrix = self.adapter.transfer_matrix_from_json_to_db(self.comm_ops_struct) + output_path = os.path.join(self.cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) + DBManager.create_tables(result_db, self.COMMUNICATION_MATRIX_TABLE) + conn, cursor = DBManager.create_connect_db(result_db) + if res_comm_matrix: + res_matrix_value = [list(data.values())[1:] for data in res_comm_matrix] + sql = "insert into {} values ({value})".format(self.COMMUNICATION_MATRIX_TABLE, + value="?," * (len(res_matrix_value[0]) - 1) + "?") + DBManager.executemany_sql(conn, sql, res_matrix_value) + DBManager.destroy_db_connect(conn, cursor) \ No newline at end of file diff --git a/profiler/cluster_analyse/analysis/communication_analysis.py b/profiler/cluster_analyse/analysis/communication_analysis.py index 5d642cddd..78e7d2306 100644 --- a/profiler/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/cluster_analyse/analysis/communication_analysis.py @@ -1,9 +1,16 @@ import os +import logging from collections import defaultdict from analysis.base_analysis import BaseAnalysis from common_func.constant import Constant +from common_func.table_constant import TableConstant from common_func.db_manager import DBManager +from prof_bean.communication_bandwidth_bean import CommunicationBandwidthBean +from prof_bean.communication_time_bean import CommunicationTimeBean + + +logger = logging.getLogger() class CommunicationAnalysis(BaseAnalysis): @@ -101,3 +108,119 @@ class CommunicationAnalysis(BaseAnalysis): bandwidth_dict[Constant.BANDWIDTH_GB_S] = \ self.compute_ratio(bandwidth_dict.get(Constant.TRANSIT_SIZE_MB, 0), bandwidth_dict.get(Constant.TRANSIT_TIME_MS, 0)) + + +class CommunicationAnalysisOptimized(BaseAnalysis): + COMMUNICATION_BANDWIDTH_TABLE = "ClusterCommunicationBandwidth" + COMMUNICATION_TIME_TABLE = "ClusterCommunicationTime" + + def __init__(self, param: dict): + super().__init__(param) + self._communication_ops = param.get(Constant.COMM_DATA_DICT, {}).get(Constant.COMMUNICATION_OPS) + self._communication_group = param.get(Constant.COMM_DATA_DICT, {}).get(Constant.COMMUNICATION_GROUP) + self._aggregate_time = {} + self._aggregate_bandwidth = {} + self._total_transit_size = {} + self._total_transit_time = {} + self._output_time = [] + self._output_bandwidth = [] + + @staticmethod + def _execute(conn, res_data, table_name): + if res_data: + sql = "insert into {} values ({value})".format(table_name, value="?," * (len(res_data[0]) - 1) + "?") + DBManager.executemany_sql(conn, sql, res_data) + + @staticmethod + def _format_time_data(communication_data): + data_dict = {} + for single_op in communication_data: + formated_data = CommunicationTimeBean(single_op) + data_dict.setdefault(formated_data.step_id, {}).\ + setdefault(formated_data.rank_id, []).extend([formated_data]) + return data_dict + + def run(self): + if not self._communication_ops: + logger.error("[ERROR] communication data is null.") + return + self._aggregate_time = self._format_time_data(self._communication_ops[0]) + self._aggregate_bandwidth = self._format_bandwidth_data(self._communication_ops[1]) + self._compute_total_info() + self._dump_data() + + def _update_total_bandwidth_data(self, formated_data: any): + step_id = formated_data.step_id + if self._total_transit_size.get(step_id) is None: + self._total_transit_size.setdefault(step_id, 0.0) + else: + self._total_transit_size[step_id] += formated_data.transit_size + if self._total_transit_time.get(step_id) is None: + self._total_transit_time.setdefault(step_id, 0.0) + else: + self._total_transit_time[step_id] += formated_data.transit_time + + def _format_bandwidth_data(self, communication_data: dict): + data_dict = {} + for single_op in communication_data: + formated_data = CommunicationBandwidthBean(single_op) + data_dict.setdefault(formated_data.step_id, {}).\ + setdefault(formated_data.rank_id, {}).\ + setdefault(formated_data.transport_type, {}).\ + setdefault(formated_data.package_size, []).extend([formated_data]) + self._update_total_bandwidth_data(formated_data) + return data_dict + + def _dump_data(self): + output_path = os.path.join(self.cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) + DBManager.create_tables(result_db, self.COMMUNICATION_TIME_TABLE) + DBManager.create_tables(result_db, self.COMMUNICATION_BANDWIDTH_TABLE) + conn, cursor = DBManager.create_connect_db(result_db) + self._execute(conn, self._output_time, self.COMMUNICATION_TIME_TABLE) + self._execute(conn, self._output_bandwidth, self.COMMUNICATION_BANDWIDTH_TABLE) + DBManager.destroy_db_connect(conn, cursor) + + def _compute_total_info(self): + if not self._aggregate_time or not self._aggregate_bandwidth: + logger.error("[ERROR] communication data is null.") + return + for step_id, rank_dict in self._aggregate_time.items(): + for rank_id, communication_op_info in rank_dict.items(): + total_dict = { + TableConstant.RANK_ID: rank_id, + TableConstant.STEP: step_id, + TableConstant.GROUP_NAME: '', + TableConstant.HCCL_OP_NAME: Constant.TOTAL_OP_INFO + } + total_time_info = CommunicationTimeBean(total_dict) + for com_info_dict in communication_op_info: + total_time_info += com_info_dict + self._output_time.append(com_info_dict.convert_output()) + total_time_info.compute_ratio() + communication_op_info.append(total_time_info) + self._output_time.append(total_time_info.convert_output()) + for step_id, rank_dict in self._aggregate_bandwidth.items(): + total_transit_size = self._total_transit_size.get(step_id) + total_transit_time = self._total_transit_time.get(step_id) + total_bandwidth = total_transit_size / total_transit_time if total_transit_time else 0.0 + for rank_id, communication_op_info in rank_dict.items(): + for transport_type, bandwidth_info in communication_op_info.items(): + for package_size, package_info in bandwidth_info.items(): + total_dict = { + TableConstant.RANK_ID: rank_id, + TableConstant.STEP: step_id, + TableConstant.GROUP_NAME: '', + TableConstant.HCCL_OP_NAME: Constant.TOTAL_OP_INFO, + TableConstant.TRANSPORT_TYPE: transport_type, + TableConstant.TRANSIT_SIZE: total_transit_size, + TableConstant.TRANSIT_TIME: total_transit_time, + TableConstant.BANDWIDTH: total_bandwidth, + TableConstant.PACKAGE_SIZE: package_size + } + total_bandwidth_info = CommunicationBandwidthBean(total_dict) + for bandwidth_package_info in package_info: + total_bandwidth_info += bandwidth_package_info + self._output_bandwidth.append(bandwidth_package_info.convert_output()) + package_info.append(total_bandwidth_info.convert_output()) + self._output_bandwidth.append(total_bandwidth_info.convert_output()) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 8a32b14ef..db39fb883 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -86,6 +86,7 @@ class Interface: Constant.ANALYSIS_MODE: self.analysis_mode, Constant.DATA_TYPE: data_type, Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: self.cluster_analysis_output_path, + Constant.DATA_SIMPLIFICATION: self.origin_params.get(Constant.DATA_SIMPLIFICATION, False) } comm_data_dict = CommunicationGroupGenerator(params).generate() params[Constant.COMM_DATA_DICT] = comm_data_dict @@ -97,11 +98,13 @@ def cluster_analysis_main(args=None): parser.add_argument('-m', '--mode', choices=COMM_FEATURE_LIST, default='all', help="different analysis mode") parser.add_argument('-o', '--output_path', type=str, help='Path of cluster analysis output') + parser.add_argument('--data_simplification', default=False, action='store_true', help='data simplification switch for db data') args_parsed, _ = parser.parse_known_args(args=args) parameter = { Constant.COLLECTION_PATH: args_parsed.profiling_path, Constant.ANALYSIS_MODE: args_parsed.mode, - Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: args_parsed.output_path + Constant.CLUSTER_ANALYSIS_OUTPUT_PATH: args_parsed.output_path, + Constant.DATA_SIMPLIFICATION: args_parsed.data_simplification } try: Interface(parameter).run() diff --git a/profiler/cluster_analyse/common_func/constant.py b/profiler/cluster_analyse/common_func/constant.py index 54e419910..9f8245b74 100644 --- a/profiler/cluster_analyse/common_func/constant.py +++ b/profiler/cluster_analyse/common_func/constant.py @@ -110,3 +110,12 @@ class Constant(object): # metadata key DISTRIBUTED_ARGS = "distributed_args" + + # mode + ALL = "all" + COMMUNICATION_TIME = "communication_time" + COMMUNICATION_MATRIX = "communication_matrix" + + STEP = "step" + + DATA_SIMPLIFICATION = "data_simplification" \ No newline at end of file diff --git a/profiler/cluster_analyse/common_func/tables_config.py b/profiler/cluster_analyse/common_func/tables_config.py index 7122d6461..9f33aecda 100644 --- a/profiler/cluster_analyse/common_func/tables_config.py +++ b/profiler/cluster_analyse/common_func/tables_config.py @@ -72,5 +72,50 @@ class TablesConfig: ("rankId", "INTEGER, null"), ("deviceId", "INTEGER, null"), ("hostUid", "INTEGER, null") - ] + ], + "ClusterCommunicationTimeMap": [ + ("step", "TEXT, null"), + ("rank_id", "INTEGER, null"), + ("hccl_op_name", "TEXT, null"), + ("group_name", "TEXT, null"), + ("start_timestamp", "NUMERIC, null"), + ("elapsed_time", "NUMERIC, null"), + ("transit_time", "NUMERIC, null"), + ("wait_time", "NUMERIC, null"), + ("synchronization_time", "NUMERIC, null"), + ("idle_time", "NUMERIC, null"), + ("synchronization_time_ratio", "NUMERIC, null"), + ("wait_time_ratio", "NUMERIC, null") + ], + "ClusterCommunicationBandwidthMap": [ + ("step", "TEXT, null"), + ("rank_id", "INTEGER, null"), + ("hccl_op_name", "TEXT, null"), + ("group_name", "TEXT, null"), + ("band_type", "TEXT, null"), + ("transit_size", "NUMERIC, null"), + ("transit_time", "NUMERIC, null"), + ("bandwidth", "NUMERIC, null"), + ("large_packet_ratio", "NUMERIC, null"), + ("package_size", "NUMERIC, null"), + ("count", "NUMERIC, null"), + ("total_duration", "NUMERIC, null") + ], + "ClusterCommunicationMatrixMap": [ + ("step", "TEXT, null"), + ("hccl_op_name", "TEXT, null"), + ("group_name", "TEXT, null"), + ("src_rank", "TEXT, null"), + ("dst_rank", "TEXT, null"), + ("transit_size", "NUMERIC, null"), + ("transit_time", "NUMERIC, null"), + ("bandwidth", "NUMERIC, null"), + ("transport_type", "TEXT, null"), + ("op_name", "TEXT, null") + ], + "CommunicationGroupMappingMap": [ + ("type", "TEXT, null"), + ("group_name", "TEXT, null"), + ("rank_set", "TEXT, null") + ], } diff --git a/profiler/cluster_analyse/communication_group/communication_db_group.py b/profiler/cluster_analyse/communication_group/communication_db_group.py index f39bf65d9..73e627745 100644 --- a/profiler/cluster_analyse/communication_group/communication_db_group.py +++ b/profiler/cluster_analyse/communication_group/communication_db_group.py @@ -1,10 +1,49 @@ import os +import logging from common_func.db_manager import DBManager from common_func.constant import Constant from communication_group.base_communication_group import BaseCommunicationGroup +logger = logging.getLogger() + +def get_communication_data(rank_id: str, db_path: str, analysis_mode: str): + time_data = [] + bandwidth_data = [] + matrix_data = [] + if not os.path.exists(db_path): + logger.warning("rank %s db path %s does not exist.", rank_id, db_path) + return [], [], [] + conn, cursor = DBManager.create_connect_db(db_path) + time_info_sql = "select * from {0}".format(Constant.TABLE_COMM_ANALYZER_TIME) + bandwidth_info_sql = "select * from {0}".format(Constant.TABLE_COMM_ANALYZER_BANDWIDTH) + matrix_info_sql = "select * from {0}".format(Constant.TABLE_COMM_ANALYZER_MATRIX) + if (DBManager.check_tables_in_db(db_path, Constant.TABLE_COMM_ANALYZER_TIME, + Constant.TABLE_COMM_ANALYZER_BANDWIDTH) + and analysis_mode in [Constant.ALL, Constant.COMMUNICATION_TIME]): + time_data = DBManager.fetch_all_data(cursor, time_info_sql) + bandwidth_data = DBManager.fetch_all_data(cursor, bandwidth_info_sql) + if (DBManager.check_tables_in_db(db_path, Constant.TABLE_COMM_ANALYZER_MATRIX) + and analysis_mode in [Constant.ALL, Constant.COMMUNICATION_MATRIX]): + matrix_data = DBManager.fetch_all_data(cursor, matrix_info_sql) + DBManager.destroy_db_connect(conn, cursor) + return time_data, bandwidth_data, matrix_data + +def dump_group_db(dump_data: list, group_table: str, cluster_analysis_output_path:str): + if dump_data: + output_path = os.path.join(cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) + result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) + DBManager.create_tables(result_db, group_table) + conn, cursor = DBManager.create_connect_db(result_db) + sql = "insert into {} values ({value})".format(group_table, + value="?," * (len(dump_data[0]) - 1) + "?") + DBManager.executemany_sql(conn, sql, dump_data) + DBManager.destroy_db_connect(conn, cursor) + else: + logger.warning("[WARNING] The CommunicationGroup table won't be created because no data has been calculated.") + + class CommunicationDBGroup(BaseCommunicationGroup): COMMUNICATION_GROUP_TABLE = "CommunicationGroup" @@ -16,42 +55,99 @@ class CommunicationDBGroup(BaseCommunicationGroup): return -1, ({}, {}, {}) rank_id = params[0] db_path = params[1] - time_data = [] - bandwidth_data = [] - matrix_data = [] - if os.path.exists(db_path): - conn, cursor = DBManager.create_connect_db(db_path) - time_info_sql = "select * from {0}".format(Constant.TABLE_COMM_ANALYZER_TIME) - bandwidth_info_sql = "select * from {0}".format(Constant.TABLE_COMM_ANALYZER_BANDWIDTH) - matrix_info_sql = "select * from {0}".format(Constant.TABLE_COMM_ANALYZER_MATRIX) - if (DBManager.check_tables_in_db(db_path, Constant.TABLE_COMM_ANALYZER_TIME, - Constant.TABLE_COMM_ANALYZER_BANDWIDTH) - and self.analysis_mode in ["all", "communication_time"]): - time_data = DBManager.fetch_all_data(cursor, time_info_sql) - bandwidth_data = DBManager.fetch_all_data(cursor, bandwidth_info_sql) - if (DBManager.check_tables_in_db(db_path, Constant.TABLE_COMM_ANALYZER_MATRIX) - and self.analysis_mode in ["all", "communication_matrix"]): - matrix_data = DBManager.fetch_all_data(cursor, matrix_info_sql) - DBManager.destroy_db_connect(conn, cursor) + time_data, bandwidth_data, matrix_data = get_communication_data(rank_id, db_path, self.analysis_mode) comm_data = self.adapter.transfer_comm_from_db_to_json(time_data, bandwidth_data) comm_matrix_data = self.adapter.transfer_matrix_from_db_to_json(matrix_data) return rank_id, comm_data, comm_matrix_data def dump_data(self): - output_path = os.path.join(self.cluster_analysis_output_path, Constant.CLUSTER_ANALYSIS_OUTPUT) - result_db = os.path.join(output_path, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER) res = [] for data_type, data_list in self.communication_group.items(): for data in data_list: rank_set = "(" + ",".join(str(i) for i in data) + ")" data = [data_type, rank_set] res.append(data) - if res: - DBManager.create_tables(result_db, self.COMMUNICATION_GROUP_TABLE) - conn, cursor = DBManager.create_connect_db(result_db) - sql = "insert into {} values ({value})".format(self.COMMUNICATION_GROUP_TABLE, - value="?," * (len(res[0]) - 1) + "?") - DBManager.executemany_sql(conn, sql, res) - DBManager.destroy_db_connect(conn, cursor) - else: - print("[WARNING] The CommunicationGroup table won't be created because no data has been calculated.") + dump_group_db(res, self.COMMUNICATION_GROUP_TABLE, self.cluster_analysis_output_path) + + +class CommunicationDBGroupOptimized(BaseCommunicationGroup): + COMMUNICATION_GROUP_MAPPING_TABLE = "CommunicationGroupMapping" + + def __init__(self, params: dict): + super().__init__(params) + self.bandwidth_data = [] + self.matrix_ops = [] + + def read_communication_func(self, params: tuple): + if len(params) < 3: + return -1, ({}, {}, {}) + rank_id = params[0] + db_path = params[1] + time_data, bandwidth_data, matrix_data = get_communication_data(rank_id, db_path, self.analysis_mode) + comm_matrix_data = self.adapter.transfer_matrix_from_db_to_json(matrix_data) + comm_time_data = (time_data, bandwidth_data) + return rank_id, comm_time_data, comm_matrix_data + + def set_collective_group(self, rank_id: int, time_data: list): + for single_time_data in time_data: + if single_time_data.get('type') == Constant.P2P: + continue + group_name = single_time_data.get(Constant.GROUP_NAME) + if group_name: + self.collective_group_dict[group_name].add(rank_id) + + def analyze_communication_data(self): + for rank_id, comm_time_data, comm_matrix_data in self.rank_comm_dir_dict: + time_data, bandwidth_data = comm_time_data + if self.analysis_mode in [Constant.ALL, Constant.COMMUNICATION_TIME]: + if not time_data: + logger.warning("[WARNING] rank %s has error format in time data.", rank_id) + continue + self.set_collective_group(rank_id, time_data) + self.communication_ops.extend(self._merge_data_with_rank(rank_id, time_data)) + self.bandwidth_data.extend(self._merge_data_with_rank(rank_id, bandwidth_data)) + if self.analysis_mode in [Constant.ALL, Constant.COMMUNICATION_MATRIX]: + if not comm_matrix_data: + logger.warning("[WARNING] rank %s matrix data is null.", rank_id) + continue + for step_id, step_id_dict in comm_matrix_data.items(): + if not isinstance(step_id_dict, dict): + logger.warning("[WARNING] rank %s has error format in matrix data.", rank_id) + continue + self.set_p2p_link(rank_id, step_id, comm_matrix_data) + self.get_collective_ops_name(rank_id, step_id_dict.get(Constant.COLLECTIVE)) + + def generate_collective_communication_group(self): + collective_group = [] + for group_name, group in self.collective_group_dict.items(): + collective_group.append((group_name, list(group))) + self.communication_group[Constant.COLLECTIVE] = collective_group + + def collect_comm_data(self): + comm_data_dict = { + Constant.COLLECTIVE_GROUP: self.collective_group_dict, + Constant.COMMUNICATION_OPS: (self.communication_ops, self.bandwidth_data), + Constant.MATRIX_OPS: self.matrix_ops, + Constant.COMMUNICATION_GROUP: self.communication_group + } + return comm_data_dict + + def dump_data(self): + res = [] + for data_type, data_list in self.communication_group.items(): + if data_type == Constant.P2P: + for data in data_list: + rank_set = "(" + ",".join(str(i) for i in data) + ")" + res.append([data_type, "", rank_set]) + continue + for group_name, data in data_list: + rank_set = "(" + ",".join(str(i) for i in data) + ")" + res.append([data_type, group_name, rank_set]) + dump_group_db(res, self.COMMUNICATION_GROUP_MAPPING_TABLE, self.cluster_analysis_output_path) + + def _merge_data_with_rank(self, rank_id: int, data_list: list): + res = [] + for single_time_data in data_list: + single_time_data[Constant.RANK_ID] = rank_id + res.append(single_time_data) + return res diff --git a/profiler/cluster_analyse/communication_group/communication_group_generator.py b/profiler/cluster_analyse/communication_group/communication_group_generator.py index 3dca90454..f1e438977 100644 --- a/profiler/cluster_analyse/communication_group/communication_group_generator.py +++ b/profiler/cluster_analyse/communication_group/communication_group_generator.py @@ -15,18 +15,30 @@ from common_func.constant import Constant from communication_group.communication_db_group import CommunicationDBGroup +from communication_group.communication_db_group import CommunicationDBGroupOptimized from communication_group.communication_json_group import CommunicationJsonGroup +SIMPLIFIED = "SIMPLIFIED" +ORIGINAL = "ORIGINAL" + + class CommunicationGroupGenerator: GROUP_MAP = { - Constant.DB: CommunicationDBGroup, - Constant.TEXT: CommunicationJsonGroup + ORIGINAL: { + Constant.DB: CommunicationDBGroup, + Constant.TEXT: CommunicationJsonGroup + }, + SIMPLIFIED: { + Constant.DB: CommunicationDBGroupOptimized, + Constant.TEXT: CommunicationJsonGroup + } } def __init__(self, params: dict): - self.processor = self.GROUP_MAP.get(params.get(Constant.DATA_TYPE))(params) + version = SIMPLIFIED if params.get(Constant.DATA_SIMPLIFICATION) else ORIGINAL + self.processor = self.GROUP_MAP.get(version).get(params.get(Constant.DATA_TYPE))(params) def generate(self): return self.processor.generate() diff --git a/profiler/cluster_analyse/prof_bean/communication_bandwidth_bean.py b/profiler/cluster_analyse/prof_bean/communication_bandwidth_bean.py new file mode 100644 index 000000000..80438552e --- /dev/null +++ b/profiler/cluster_analyse/prof_bean/communication_bandwidth_bean.py @@ -0,0 +1,82 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from common_func.table_constant import TableConstant + + +class CommunicationBandwidthBean: + + def __init__(self, single_op: dict): + self._rank_id = single_op.get(TableConstant.RANK_ID, '') + self._step_id = single_op.get(TableConstant.STEP, '') + self._group_name = single_op.get(TableConstant.GROUP_NAME, '') + self._hccl_op_name = single_op.get(TableConstant.HCCL_OP_NAME, '') + self._transport_type = single_op.get(TableConstant.TRANSPORT_TYPE) + self._transit_size = single_op.get(TableConstant.TRANSIT_SIZE, 0.0) + self._transit_time = single_op.get(TableConstant.TRANSIT_TIME, 0.0) + self._bandwidth = single_op.get(TableConstant.BANDWIDTH, 0.0) + self._large_packet_ratio = single_op.get(TableConstant.LARGE_PACKET_RATIO, 0) + self._package_size = single_op.get(TableConstant.PACKAGE_SIZE, 0.0) + self._count = single_op.get(TableConstant.COUNT, 0.0) + self._total_duration = single_op.get(TableConstant.TOTAL_DURATION, 0.0) + + def __add__(self, other): + res = { + TableConstant.RANK_ID: self._rank_id, + TableConstant.STEP: self._step_id, + TableConstant.GROUP_NAME: self._group_name, + TableConstant.HCCL_OP_NAME: self._hccl_op_name, + TableConstant.TRANSPORT_TYPE: self._transport_type, + TableConstant.TRANSIT_SIZE: self._transit_size + other._transit_size, + TableConstant.TRANSIT_TIME: self._transit_time + other._transit_time, + TableConstant.BANDWIDTH: self._bandwidth + other._bandwidth, + TableConstant.LARGE_PACKET_RATIO: 0, + TableConstant.PACKAGE_SIZE: self._package_size, + TableConstant.COUNT: self._count + other._count, + TableConstant.TOTAL_DURATION: self._total_duration + other._total_duration + } + return CommunicationBandwidthBean(res) + + @property + def rank_id(self): + return self._rank_id + + @property + def step_id(self): + return self._step_id + + @property + def transport_type(self): + return self._transport_type + + @property + def package_size(self): + return self._package_size + + @property + def transit_time(self): + return self._transit_time + + @property + def transit_size(self): + return self._transit_size + + def convert_output(self): + return [ + self._step_id, self._rank_id, self._hccl_op_name, self._group_name, + self._transport_type, self._transit_size, self._transit_time, self._bandwidth, + self._large_packet_ratio, self._package_size, self._count, self._total_duration + ] diff --git a/profiler/cluster_analyse/prof_bean/communication_time_bean.py b/profiler/cluster_analyse/prof_bean/communication_time_bean.py new file mode 100644 index 000000000..a3ef35669 --- /dev/null +++ b/profiler/cluster_analyse/prof_bean/communication_time_bean.py @@ -0,0 +1,70 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +from common_func.table_constant import TableConstant + + +class CommunicationTimeBean: + + def __init__(self, single_op: dict): + self._rank_id = single_op.get(TableConstant.RANK_ID, '') + self._step_id = single_op.get(TableConstant.STEP, '') + self._group_name = single_op.get(TableConstant.GROUP_NAME, '') + self._hccl_op_name = single_op.get(TableConstant.HCCL_OP_NAME, '') + self._start_time = single_op.get(TableConstant.START_TIMESTAMP, 0) + self._elapsed_time = single_op.get(TableConstant.ELAPSED_TIME, 0.0) + self._transit_time = single_op.get(TableConstant.TRANSIT_TIME, 0.0) + self._wait_time = single_op.get(TableConstant.WAIT_TIME, 0.0) + self._synchronization_time = single_op.get(TableConstant.SYNCHRONIZATION_TIME, 0.0) + self._idle_time = single_op.get(TableConstant.IDLE_TIME, 0.0) + self._sync_ratio = 0 + self._wait_ratio = 0 + + def __add__(self, other): + res = { + TableConstant.RANK_ID: self._rank_id, + TableConstant.STEP: self._step_id, + TableConstant.GROUP_NAME: self._group_name, + TableConstant.HCCL_OP_NAME: self._hccl_op_name, + TableConstant.START_TIMESTAMP: 0, + TableConstant.ELAPSED_TIME: self._elapsed_time + other._elapsed_time, + TableConstant.TRANSIT_TIME: self._transit_time + other._transit_time, + TableConstant.WAIT_TIME: self._wait_time + other._wait_time, + TableConstant.SYNCHRONIZATION_TIME: self._synchronization_time + other._synchronization_time, + TableConstant.IDLE_TIME: self._idle_time + other._idle_time + } + return CommunicationTimeBean(res) + + @property + def rank_id(self): + return self._rank_id + + @property + def step_id(self): + return self._step_id + + def compute_ratio(self): + total_duration = self._elapsed_time + self._transit_time + self._wait_time +\ + self._synchronization_time +self._idle_time + self._sync_ratio = self._synchronization_time / total_duration if total_duration != 0 else 0 + self._wait_ratio = self._wait_time / total_duration if total_duration != 0 else 0 + + def convert_output(self): + return [ + self._step_id, self._rank_id, self._hccl_op_name, self._group_name, + self._start_time, self._transit_time, self._wait_time, self._synchronization_time, + self._idle_time, self._elapsed_time, self._sync_ratio, self._wait_ratio + ] -- Gitee From 46a03137000a1b39451f53575659aa3f251ee959 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Wed, 4 Sep 2024 10:22:25 +0800 Subject: [PATCH 689/791] use create_directory --- .../msprobe/core/common/file_check.py | 26 ++++++++++++++----- .../mindspore/debugger/debugger_config.py | 12 ++------- .../debugger/test_debugger_config.py | 17 ++---------- .../debugger/test_precision_debugger.py | 2 +- .../mindspore_ut/test_dump_tool_factory.py | 2 +- .../mindspore_ut/test_kernel_graph_dump.py | 2 +- .../test_kernel_graph_overflow_check.py | 2 +- .../mindspore_ut/test_kernel_kbyk_dump.py | 2 +- .../test_overflow_check_tool_factory.py | 5 ++-- .../test/mindspore_ut/test_primitive_dump.py | 4 +-- .../mindspore_ut/test_task_handler_factory.py | 2 +- 11 files changed, 34 insertions(+), 42 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_check.py b/debug/accuracy_tools/msprobe/core/common/file_check.py index ee56c68b5..e699ec0fb 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_check.py +++ b/debug/accuracy_tools/msprobe/core/common/file_check.py @@ -224,6 +224,21 @@ def check_path_type(file_path, file_type): raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) +def make_dir(dir_path): + dir_path = os.path.realpath(dir_path) + check_path_before_create(dir_path) + if os.path.isdir(dir_path): + return + try: + os.mkdir(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) + except OSError as ex: + raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, + f"Failed to create {dir_path}. " + f"Please check the path permission or disk space. {str(ex)}") from ex + file_check = FileChecker(dir_path, FileCheckConst.DIR) + file_check.common_check() + + def create_directory(dir_path): """ Function Description: @@ -235,13 +250,10 @@ def create_directory(dir_path): """ dir_path = os.path.realpath(dir_path) check_path_before_create(dir_path) - try: - os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) - except OSError as ex: - raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, - 'Failed to create {}. Please check the path permission or disk space .{}'.format(dir_path, str(ex))) from ex - file_check = FileChecker(dir_path, FileCheckConst.DIR) - file_check.common_check() + parent_dir = os.path.dirname(dir_path) + if not os.path.isdir(parent_dir): + create_directory(parent_dir) + make_dir(dir_path) def check_path_before_create(path): diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 24b61f3df..09416f0b8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -1,10 +1,9 @@ import os -from pathlib import Path from msprobe.core.common.const import Const from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.common.const import FreeBenchmarkConst -from msprobe.core.common.file_check import FileChecker, FileCheckConst, check_path_before_create +from msprobe.core.common.file_check import create_directory class DebuggerConfig: @@ -26,7 +25,7 @@ class DebuggerConfig: self.framework = Const.MS_FRAMEWORK self.summary_mode = task_config.summary_mode self.check() - self._make_dump_path_if_not_exists() + create_directory(self.dump_path) if self.task == Const.FREE_BENCHMARK: self.pert_type = (FreeBenchmarkConst.DEFAULT_PERT_TYPE @@ -65,10 +64,3 @@ class DebuggerConfig: for s in self.step: if not isinstance(s, int) or s < 0: raise ValueError(f"step element {s} must be a positive integer.") - - def _make_dump_path_if_not_exists(self): - check_path_before_create(self.dump_path) - if not os.path.exists(self.dump_path): - Path(self.dump_path).mkdir(mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) - file_check = FileChecker(self.dump_path, FileCheckConst.DIR) - file_check.common_check() diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_debugger_config.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_debugger_config.py index 4f1c7c92d..f4f18914e 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_debugger_config.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_debugger_config.py @@ -17,14 +17,14 @@ import unittest from unittest.mock import patch -from msprobe.core.common.const import Const, FileCheckConst +from msprobe.core.common.const import Const from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.core.common_config import CommonConfig, BaseConfig from msprobe.mindspore.debugger.debugger_config import DebuggerConfig class TestDebuggerConfig(unittest.TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_init(self, _): json_config = { "dump_path": "/absolute_path", @@ -55,16 +55,3 @@ class TestDebuggerConfig(unittest.TestCase): self.assertEqual(str(context.exception), "pert_mode must be improve_precision or empty when handler_type is fix, " f"but got {FreeBenchmarkConst.ADD_NOISE}.") - - @patch("msprobe.mindspore.debugger.debugger_config.os.path.exists", return_value=False) - def test__make_dump_path_if_not_exists(self, _): - json_config = {"dump_path": "/absolute_path"} - common_config = CommonConfig(json_config) - task_config = BaseConfig(json_config) - with patch("msprobe.mindspore.debugger.debugger_config.check_path_before_create") as mock_check_path, \ - patch("msprobe.mindspore.debugger.debugger_config.Path.mkdir") as mock_mkdir, \ - patch("msprobe.mindspore.debugger.debugger_config.FileChecker") as mock_checker: - DebuggerConfig(common_config, task_config) - mock_check_path.assert_called_with(json_config.get("dump_path")) - mock_mkdir.assert_called_with(mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) - mock_checker.assert_called_with(common_config.dump_path, FileCheckConst.DIR) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_precision_debugger.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_precision_debugger.py index ee9970f51..a65f85618 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_precision_debugger.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/debugger/test_precision_debugger.py @@ -27,7 +27,7 @@ from msprobe.core.common.const import Const class TestPrecisionDebugger(unittest.TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_start(self, _): class Handler: called = False diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py index 3b5282f96..973132a5a 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_dump_tool_factory.py @@ -24,7 +24,7 @@ from msprobe.mindspore.dump.dump_tool_factory import DumpToolFactory class TestDumpToolFactory(TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_create(self, _): json_config = { "task": "statistics", diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py index 8bbdcd1b4..7addb91fb 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_dump.py @@ -25,7 +25,7 @@ from msprobe.mindspore.dump.kernel_graph_dump import KernelGraphDump class TestKernelGraphDump(TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_handle(self, _): json_config = { "task": "tensor", diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py index 91bc79fff..be597efef 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_graph_overflow_check.py @@ -25,7 +25,7 @@ from msprobe.mindspore.overflow_check.kernel_graph_overflow_check import KernelG class TestKernelGraphOverflowCheck(TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_handle(self, _): json_config = { "task": "overflow_check", diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py index 4ce6bf397..b24b7e29c 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_kernel_kbyk_dump.py @@ -25,7 +25,7 @@ from msprobe.mindspore.dump.kernel_kbyk_dump import KernelKbykDump class TestKernelKbykDump(TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_handle(self, _): json_config = { "task": "statistics", diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py index 90ffbcb97..f46f171aa 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_overflow_check_tool_factory.py @@ -24,7 +24,7 @@ from msprobe.mindspore.overflow_check.overflow_check_tool_factory import Overflo class TestOverflowCheckToolFactory(TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_create(self, _): json_config = { "task": "overflow_check", @@ -48,7 +48,8 @@ class TestOverflowCheckToolFactory(TestCase): with self.assertRaises(Exception) as context: OverflowCheckToolFactory.create(config) self.assertEqual(str(context.exception), - f"Overflow check is not supported in {config.execution_mode} mode when level is {config.level}.") + f"Overflow check is not supported in {config.execution_mode} mode " + f"when level is {config.level}.") config.level = "kernel" dumper = OverflowCheckToolFactory.create(config) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py index aa6372fa3..23c0b5ad1 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py @@ -37,7 +37,7 @@ class DummyModel(nn.Cell): class TestService(unittest.TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def setUp(self, _): json_config = { "task": "statistics", @@ -97,4 +97,4 @@ class TestService(unittest.TestCase): with patch.object(self.service.data_collector, 'update_iter') as mock_update_iter: initial_iter = self.service.current_iter self.service.step() - mock_update_iter.assert_called_once_with(initial_iter + 1) \ No newline at end of file + mock_update_iter.assert_called_once_with(initial_iter + 1) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py index 6dc6322cc..752b5f916 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_task_handler_factory.py @@ -25,7 +25,7 @@ from msprobe.mindspore.common.const import Const class TestTaskHandlerFactory(TestCase): - @patch.object(DebuggerConfig, "_make_dump_path_if_not_exists") + @patch("msprobe.mindspore.debugger.debugger_config.create_directory") def test_create(self, _): class HandlerFactory: def create(self): -- Gitee From f0a05ab5a91b0e3e183cd0c5b564cc3d9511767f Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Wed, 4 Sep 2024 17:06:46 +0800 Subject: [PATCH 690/791] codecheck --- debug/accuracy_tools/msprobe/core/common/log.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 414e604cc..eaed07807 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -38,7 +38,7 @@ class BaseLogger: @filter_special_chars def debug(self, msg): if LOG_LEVEL == MsgConst.LEVEL[3]: - self._print_log(MsgConst.LEVEL[1], msg) + self._print_log(MsgConst.LEVEL[3], msg) def on_rank_0(self, func): def func_rank_0(*args, **kwargs): -- Gitee From 420084c403b34c497eb35a629b63243949772cd1 Mon Sep 17 00:00:00 2001 From: RICHARDNAN <1466684392@qq.com> Date: Wed, 4 Sep 2024 17:44:52 +0800 Subject: [PATCH 691/791] =?UTF-8?q?=E7=AE=97=E5=AD=90=E8=AF=86=E5=88=AB?= =?UTF-8?q?=E8=A7=84=E5=88=99=E6=96=B0=E5=A2=9Eflash2attn?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/compare_tools/compare_backend/utils/constant.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/compare_tools/compare_backend/utils/constant.py b/profiler/compare_tools/compare_backend/utils/constant.py index d1ddf68d9..4e41f47d8 100644 --- a/profiler/compare_tools/compare_backend/utils/constant.py +++ b/profiler/compare_tools/compare_backend/utils/constant.py @@ -92,7 +92,7 @@ class Constant(object): BWD_LIST = ["bwd", "backward", "back", "grad"] - CPU_OP_FA_MASK = ("flash_attention", "fusion_attention", "flashattn", "xformers_flash", "efficient_attention", "flsh2atten") + CPU_OP_FA_MASK = ("flash_attention", "fusion_attention", "flashattn", "xformers_flash", "efficient_attention", "flash2attn") CPU_OP_CONV = "aten::conv" CPU_OP_MATMUL_MASK = ("aten::addmm", "aten::bmm", "aten::mm", "aten::matmul") KERNEL_CUBE_MASK = ("gemm", "conv", "cutlass", "wgrad") -- Gitee From 197db84cb30de2568dd03e3f40f9280c7512fa5a Mon Sep 17 00:00:00 2001 From: zhaolei Date: Tue, 3 Sep 2024 21:32:51 +0800 Subject: [PATCH 692/791] =?UTF-8?q?=E9=80=9A=E7=AE=97=E5=B8=A6=E5=AE=BD?= =?UTF-8?q?=E6=8A=A2=E5=8D=A0=E5=88=86=E6=9E=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../advisor/analyzer/analyzer_controller.py | 37 ++- .../communication/contention/__init__.py | 0 .../bandwidth_contention_analyzer.py | 56 +++++ .../bandwidth_contention_checker.py | 175 ++++++++++++++ .../communication/packet/packet_checker.py | 6 +- profiler/advisor/common/analyzer_scopes.py | 1 + .../dataset/cluster/hccl_collection.py | 11 +- .../display/html/templates/contention.html | 41 ++++ profiler/advisor/interface/interface.py | 13 +- .../advisor/rules/bandwidth_contention.yaml | 10 + profiler/advisor/rules/packet.yaml | 4 +- .../compare_bean/api_compare_bean.py | 6 +- .../compare_bean/kernel_compare_bean.py | 9 +- .../compare_backend/utils/common_func.py | 4 +- profiler/requirements/build.txt | 2 +- .../test_bandwidth_contention_advice.py | 217 ++++++++++++++++++ 16 files changed, 559 insertions(+), 33 deletions(-) create mode 100644 profiler/advisor/analyzer/communication/contention/__init__.py create mode 100644 profiler/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py create mode 100644 profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py create mode 100644 profiler/advisor/display/html/templates/contention.html create mode 100644 profiler/advisor/rules/bandwidth_contention.yaml create mode 100644 profiler/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py index bed45679f..6cba61b19 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -29,6 +29,8 @@ logger = logging.getLogger() class AnalyzerController: CLUSTER_RANK_THRESHOLD = 2 + SDMA_SUPPORT_SCOPES = [SupportedScopes.BANDWIDTH_CONTENTION_DETECTION] + RDMA_SUPPORT_SCOPES = [SupportedScopes.PACKET] def __init__(self): self.dimensions = Interface.all_dimension @@ -230,11 +232,14 @@ class AnalyzerController: job_list.append((dimension, scope, interface, kwargs)) return job_list - def communication_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, - benchmark_step=None, bandwidth_type=None): + def communication_analysis(self, profiling_path, benchmark_profiling_path=None, **kwargs): job_list = [] supported_trans_type = [SlowLinkAnalyzer.SDMA, SlowLinkAnalyzer.RDMA] + step = kwargs.get("step", None) + benchmark_step = kwargs.get("benchmark_step", None) + bandwidth_type = kwargs.get("bandwidth_type", None) + scope = kwargs.get("scope", None) if bandwidth_type is not None and bandwidth_type not in supported_trans_type: logger.error("Error transit type %s, optionals are %s", bandwidth_type, supported_trans_type) return job_list @@ -244,7 +249,8 @@ class AnalyzerController: for bandwidth_type in bandwidth_type_list: job_list += getattr(self, f"_communication_{bandwidth_type.lower()}_analysis")(profiling_path, benchmark_profiling_path, - step, benchmark_step) + step, benchmark_step, + scope) return job_list @@ -304,7 +310,7 @@ class AnalyzerController: logger.info(info_msg) job_list += self.communication_analysis(analysis_profiling_path, step=step, - bandwidth_type=bandwidth_type) + bandwidth_type=bandwidth_type, scope=scope) return job_list @@ -447,7 +453,7 @@ class AnalyzerController: self._get_analysis_success_resp(pid, resp) def _communication_rdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, - benchmark_step=None): + benchmark_step=None, scope=None): # 小包分析 kwargs = copy.deepcopy(self.kwargs) job_list = [] @@ -457,19 +463,26 @@ class AnalyzerController: kwargs["step"] = step kwargs["benchmark_step"] = benchmark_step - for dimension in [Interface.COMMUNICATION]: - for scope in Interface.get_scope(dimension): - if scope != SupportedScopes.PACKET: - continue - interface = Interface(**kwargs) - job_list.append((dimension, scope, interface, kwargs)) + if scope in self.RDMA_SUPPORT_SCOPES: + interface = Interface(**kwargs) + job_list.append((Interface.COMMUNICATION, scope, interface, kwargs)) return job_list def _communication_sdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, - benchmark_step=None): + benchmark_step=None, scope=None): kwargs = copy.deepcopy(self.kwargs) job_list = [] + + kwargs["profiling_path"] = profiling_path + kwargs["benchmark_profiling_path"] = benchmark_profiling_path + kwargs["step"] = step + kwargs["benchmark_step"] = benchmark_step + + if scope in self.SDMA_SUPPORT_SCOPES: + interface = Interface(**kwargs) + job_list.append((Interface.COMMUNICATION, scope, interface, kwargs)) + return job_list def _profiling_comparison(self, compare_profiling_list): diff --git a/profiler/advisor/analyzer/communication/contention/__init__.py b/profiler/advisor/analyzer/communication/contention/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py b/profiler/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py new file mode 100644 index 000000000..46c8be7a4 --- /dev/null +++ b/profiler/advisor/analyzer/communication/contention/bandwidth_contention_analyzer.py @@ -0,0 +1,56 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging + +from profiler.advisor.analyzer.communication.base_communication_analyzer import BaseCommunicationAnalyzer +from profiler.advisor.analyzer.communication.contention.bandwidth_contention_checker import BandwidthContentionChecker +from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor +from profiler.advisor.display.html.render import HTMLRender +from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset +from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from profiler.advisor.result.result import OptimizeResult + +logger = logging.getLogger() + + +class BandwidthContentionAnalyzer(BaseCommunicationAnalyzer): + dataset_cls_list = [ProfilingDataset, CommunicationDataset] + requires_cluster_dataset = False + + def __init__(self, collection_path, n_processes: int = 1, **kwargs) -> None: + super().__init__(collection_path, n_processes, **kwargs) + communication_key = CommunicationDataset.get_key() + profiling_key = ProfilingDataset.get_key() + self.communication_dataset = self.get_first_data_by_key(self.dataset_list, communication_key) + self.profiling_dataset = self.get_first_data_by_key(self.dataset_list, profiling_key) + self.result = OptimizeResult() + self.html_render = HTMLRender() + self.html = None + + @BaseCommunicationAnalyzer.check_data((CommunicationDataset.get_key(),)) + def optimize(self, **kwargs): + add_render_list = kwargs.get("add_render_list", True) + bandwidth_contention_checker = BandwidthContentionChecker(**kwargs) + bandwidth_contention_checker.check_contention(self.communication_dataset, self.profiling_dataset) + if not bandwidth_contention_checker.contention_issues: + return self.result + bandwidth_contention_checker.make_record(self.result) + self.html = bandwidth_contention_checker.make_render(self.html_render, add_render_list, + priority=self.get_priority()) + return self.result + + def get_priority(self): + # 提升1% ~ 3% + return PriorityBackgroundColor.low diff --git a/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py b/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py new file mode 100644 index 000000000..13438aa1c --- /dev/null +++ b/profiler/advisor/analyzer/communication/contention/bandwidth_contention_checker.py @@ -0,0 +1,175 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import os +from typing import List +from profiler.advisor.dataset.communication.communication_dataset import CommunicationDataset +from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset +from profiler.advisor.result.result import OptimizeResult +from profiler.advisor.result.item import OptimizeItem, OptimizeRecord +from profiler.cluster_analyse.common_func.file_manager import FileManager +from profiler.advisor.utils.utils import convert_to_float +from profiler.advisor.dataset.cluster.hccl_collection import HcclInfo +from profiler.advisor.dataset.profiling.info_collection import OpInfo + +logger = logging.getLogger() + + +class SDMAOperator: + def __init__(self, hccl_info: HcclInfo): + self._ts = hccl_info.ts + self._dur = hccl_info.elapse_time + self._name = hccl_info.name + self._bandwidth = hccl_info.sdma_info.get('Bandwidth(GB/s)', 0) + + @property + def ts(self): + return self._ts + + @property + def dur(self): + return self._dur + + @property + def end(self): + return self._ts + self._dur * 1000 + + @property + def name(self): + return self._name + + @property + def bandwidth(self): + return self._bandwidth + + +class BandwidthContentionChecker: + _CHECKER = "BandwidthContentionChecker" + + def __init__(self, **kwargs): + self.contention_issues = False + self.desc = "" + self.step_id = kwargs.get("step") + self.stage = None + self.threshold = 0 + self.contention_topk = 0 + self.sdma_list: List[SDMAOperator] = [] + self.matmul_list: List[OpInfo] = [] + self.abnormal_sdma_list: List[SDMAOperator] = [] + self.suggestions = [] + self._init_rule() + self.headers = ["op name", "duration(ms)", "bandwidth(GB/s)"] + + @staticmethod + def check_sdma_operator(hccl_op: HcclInfo): + if hccl_op.sdma_info: + if hccl_op.sdma_info.get('Transit Size(MB)', 0): + return True + return False + + def export_sdma_list(self): + res = [] + for hccl_op in self.abnormal_sdma_list: + res.append([hccl_op.name, round(hccl_op.dur, 4), round(hccl_op.bandwidth, 2)]) + res.sort(key=lambda x: x[2]) + return res[:min(len(res), self.contention_topk)] + + def check_task_dict(self, profiling_dataset: ProfilingDataset) -> bool: + if not hasattr(profiling_dataset, "op_summary"): + logger.warning("Skip %s checker because of not containing %s", self._CHECKER, "op summary") + return False + if not hasattr(profiling_dataset.op_summary, "task_dict"): + logger.warning("Skip %s checker because of not containing %s", self._CHECKER, "op summary") + return False + return True + + def extract_matmul_operator(self, profiling_dataset: ProfilingDataset): + for key, value in profiling_dataset.op_summary.task_dict.items(): + if "matmul" in key.lower(): + self.matmul_list.extend(value) + self.matmul_list.sort(key=lambda x: convert_to_float(x.task_start_time)) + + def extract_SDMA_operator(self, hccl_dataset: CommunicationDataset): + for step_id, step_data in hccl_dataset.hccl_dict.items(): + if self.step_id is not None and step_id != self.step_id: + continue + for hccl_op in step_data: + if self.check_sdma_operator(hccl_op): + self.sdma_list.append(SDMAOperator(hccl_op)) + self.sdma_list.sort(key=lambda x: x.ts) + + def check_contention(self, hccl_dataset: CommunicationDataset, profiling_dataset: ProfilingDataset) -> None: + if not self.check_task_dict(profiling_dataset): + return + self.extract_matmul_operator(profiling_dataset) + self.extract_SDMA_operator(hccl_dataset) + hccl_index = 0 + matmul_index = 0 + while hccl_index < len(self.sdma_list) and matmul_index < len(self.matmul_list): + if self.sdma_list[hccl_index].end < self.matmul_list[matmul_index].get_float_attr("task_start_time"): + hccl_index += 1 + elif self.matmul_list[matmul_index].get_float_attr("task_start_time") + \ + self.matmul_list[matmul_index].get_float_attr("task_duration") < self.sdma_list[hccl_index].ts: + matmul_index += 1 + else: + if self.sdma_list[hccl_index].bandwidth < self.threshold: + self.abnormal_sdma_list.append(self.sdma_list[hccl_index]) + matmul_index += 1 + if self.abnormal_sdma_list: + self.contention_issues = True + self.desc = self.desc.format(threshold=self.threshold) + + def make_record(self, result: OptimizeResult): + """ + make record for what and how to optimize + """ + optimization_item = OptimizeItem("bandwidth contention analysis", self.desc, self.suggestions) + result.add(OptimizeRecord(optimization_item)) + + sub_table_name = "Bandwidth Contention Analysis" if not self.stage else f"Stage-{self.stage}: " \ + f"Bandwidth Contention Analysis" + result.add_detail(sub_table_name, headers=self.headers) + for hccl_op in self.abnormal_sdma_list: + result.add_detail(sub_table_name, detail=[hccl_op.name, round(hccl_op.dur, 4), round(hccl_op.bandwidth, 2)]) + + def make_render(self, html_render, add_render_list=True, **kwargs): + priority = kwargs.get("priority") + return html_render.render_template(key="communication", + template_dir="templates", + template_name="contention.html", + desc=self.desc, + solutions=self.solutions, + headers=self.headers, + data=self.export_sdma_list(), + topk=self.contention_topk, + priority_background_color=priority) + + def _init_rule(self): + contention_rule_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.realpath(__file__))))), + "rules", + "bandwidth_contention.yaml" + ) + + contention_rule = FileManager.read_yaml_file(contention_rule_path) + self.desc = contention_rule.get("problem") + self.threshold = contention_rule.get("threshold", 0) * contention_rule.get("sdma_baseline", 0) + self.contention_topk = contention_rule.get("top_num", 3) + self.solutions = contention_rule.get("solutions") + if not self.desc or not self.solutions or not isinstance(self.solutions, list): + raise RuntimeError("The configuration file of the bandwidth contention analyzer is abnormal. Please check.") + for solution in self.solutions: + for key, val in solution.items(): + self.suggestions.append(f"{key}, {val.get('desc')}") diff --git a/profiler/advisor/analyzer/communication/packet/packet_checker.py b/profiler/advisor/analyzer/communication/packet/packet_checker.py index d270667cd..6ddf17c43 100644 --- a/profiler/advisor/analyzer/communication/packet/packet_checker.py +++ b/profiler/advisor/analyzer/communication/packet/packet_checker.py @@ -61,9 +61,9 @@ class Statistic: dst_headers.extend(src_headers) datas.extend([self.count, self.abnormal_count, self.abnormal_ratio, self.abnormal_duration]) self.desc = self.desc.format( - abnormal_sdma_ratio=f"{round(self.abnormal_ratio, 4):.2%}", - min_sdma_size=self.min_size, - abnormal_sdma_time=round(self.abnormal_duration, 4)) + abnormal_ratio=f"{round(self.abnormal_ratio, 4):.2%}", + min_size=self.min_size, + abnormal_time=round(self.abnormal_duration, 4)) return True diff --git a/profiler/advisor/common/analyzer_scopes.py b/profiler/advisor/common/analyzer_scopes.py index 852eee906..a07a6d5de 100644 --- a/profiler/advisor/common/analyzer_scopes.py +++ b/profiler/advisor/common/analyzer_scopes.py @@ -22,6 +22,7 @@ class SupportedScopes: SLOW_LINK = "slow_link" COMMUNICATION_RETRANSMISSION_DETECTION = "communication_retransmission_analysis" PACKET = "packet_analysis" + BANDWIDTH_CONTENTION_DETECTION = "bandwidth_contention_analysis" OVER_ALL = "over_all" ENVIRONMENT_VARIABLE_ANALYSIS = "environment_variable_analysis" DYNAMIC_SHAPE_ANALYSIS = "dynamic_shape_analysis" diff --git a/profiler/advisor/dataset/cluster/hccl_collection.py b/profiler/advisor/dataset/cluster/hccl_collection.py index a9fa536ef..6a7156e49 100644 --- a/profiler/advisor/dataset/cluster/hccl_collection.py +++ b/profiler/advisor/dataset/cluster/hccl_collection.py @@ -26,7 +26,8 @@ class HcclInfo(): self._step = step self._rank = rank self._name = op.split("@")[0] - self._elapse_time = self.get_elapse_time(rank_dict, "Elapse Time(ms)") + self._ts = self.get_communication_time_info(rank_dict, "Start Timestamp(us)") + self._elapse_time = self.get_communication_time_info(rank_dict, "Elapse Time(ms)") self._sdma_info = self.get_communication_info(rank_dict, "SDMA") self._rdma_info = self.get_communication_info(rank_dict, "RDMA") @@ -58,15 +59,19 @@ class HcclInfo(): def elapse_time(self): return self._elapse_time + @property + def ts(self): + return self._ts + @staticmethod def get_communication_info(rank_dict: dict, name: str): communication_bandwidth_info = rank_dict.get('Communication Bandwidth Info', dict()) return communication_bandwidth_info.get(name, dict()) @staticmethod - def get_elapse_time(rank_dict: dict, name: str): + def get_communication_time_info(rank_dict: dict, name: str): communication_time_info = rank_dict.get('Communication Time Info', dict()) - return communication_time_info.get(name, "") + return communication_time_info.get(name, 0) def get_rdma_transmit_time(self): return self.rdma_info.get('Transit Time(ms)', 0) diff --git a/profiler/advisor/display/html/templates/contention.html b/profiler/advisor/display/html/templates/contention.html new file mode 100644 index 000000000..3d7fb89c5 --- /dev/null +++ b/profiler/advisor/display/html/templates/contention.html @@ -0,0 +1,41 @@ +
+

Bandwidth Contention Analysis

+
+ {{ desc }} + + + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+

+ The following table lists the {{topk}} operators with the + most severe performance deterioration. + + + {% for header in headers %} + + {% endfor %} + + + {% for row in data %} + + {% for element in row %} + + {% endfor %} + + {% endfor %} +
{{ header }}
{{ element|safe }}
+ +
+
diff --git a/profiler/advisor/interface/interface.py b/profiler/advisor/interface/interface.py index 8138f7a1a..72c24d6c8 100644 --- a/profiler/advisor/interface/interface.py +++ b/profiler/advisor/interface/interface.py @@ -22,6 +22,7 @@ sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(os. "compare_tools")) from profiler.advisor.utils.utils import Timer +from profiler.advisor.result.result import OptimizeResult from profiler.advisor.analyzer.computation.profiling_analyzer import AicpuAnalyzer, BlockDimAnalyzer, \ DynamicShapeAnalyzer, OperatorBoundAnalyzer from profiler.advisor.analyzer.schedule.fusion_ops.fusion_ops_analyzer import TimelineFusionOpsAnalyzer @@ -40,6 +41,7 @@ from profiler.advisor.analyzer.dataloader.dataloader_analyzer import DataloaderA from profiler.advisor.analyzer.computation.ai_core_freq.ai_core_freq_analyzer import AICoreFreqAnalyzer from profiler.advisor.analyzer.memory.memory_analyzer import MemoryAnalyzer from profiler.advisor.analyzer.communication.packet.packet_analyzer import PacketAnalyzer +from profiler.advisor.analyzer.communication.contention.bandwidth_contention_analyzer import BandwidthContentionAnalyzer from profiler.advisor.analyzer.schedule.gc.gc_analyzer import GcAnalyzer from profiler.advisor.analyzer.comparison.comparison_analyzer import ComparisonAnalyzer @@ -71,7 +73,8 @@ class Interface: SupportedScopes.FREQ_ANALYSIS: AICoreFreqAnalyzer }), COMMUNICATION: OrderedDict({SupportedScopes.PACKET: PacketAnalyzer, - SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION: RDMARetransmissionAnalyzer}), + SupportedScopes.COMMUNICATION_RETRANSMISSION_DETECTION: RDMARetransmissionAnalyzer, + SupportedScopes.BANDWIDTH_CONTENTION_DETECTION: BandwidthContentionAnalyzer}), OVERALL: OrderedDict({SupportedScopes.OVER_ALL: OverallSummaryAnalyzer, SupportedScopes.ENVIRONMENT_VARIABLE_ANALYSIS: EnvironmentVariabelAnalyzer}), CLUSTER: OrderedDict({ @@ -112,8 +115,12 @@ class Interface: if scope not in supported_scopes: raise ValueError(f"Error scope {scope}, supported scopes are {supported_scopes}") - analyzer = self.get_analyzer(dimension, scope)(collection_path=self.collection_path, **kwargs) - result = analyzer.optimize(**kwargs) + try: + analyzer = self.get_analyzer(dimension, scope)(collection_path=self.collection_path, **kwargs) + result = analyzer.optimize(**kwargs) + except Exception as e: + logger.error("%s is skipped when an exception is encountered. The exception is as follows: %s", scope, e) + return OptimizeResult() if not output_dict else dict(OptimizeResult().data) if render_html and result.data: if hasattr(analyzer, "html_render"): diff --git a/profiler/advisor/rules/bandwidth_contention.yaml b/profiler/advisor/rules/bandwidth_contention.yaml new file mode 100644 index 000000000..2ad30d8aa --- /dev/null +++ b/profiler/advisor/rules/bandwidth_contention.yaml @@ -0,0 +1,10 @@ +problem: "The SDMA bandwidth is lower than {threshold} when computing and communication tasks are performed concurrently. +Generally, parallel computing and communication improves the running efficiency of the model. Concurrent computing +and communication tasks may affect the communication bandwidth." +sdma_baseline: 18 #M +threshold: 0.8 +top_num: 10 +solutions: + - benchmark data comparison: + desc: "compare the performance data before and after the convergence function is enabled to evaluate whether the + impact of bandwidth preemption exceeds the convergence benefits." diff --git a/profiler/advisor/rules/packet.yaml b/profiler/advisor/rules/packet.yaml index 521822af2..d93809882 100644 --- a/profiler/advisor/rules/packet.yaml +++ b/profiler/advisor/rules/packet.yaml @@ -1,6 +1,6 @@ problem: "Excessive small communication packets may cause host delivery bottlenecks.\n" -sdma_problem: "In the SDMA communication, {abnormal_sdma_ratio} of the communication data volume is less than {min_sdma_size} MB, and the total time is {abnormal_sdma_time} ms.\n" -rdma_problem: "In the RDMA communication, {abnormal_rdma_ratio} of the communication data volume is less than {min_rdma_size} MB, and the total time is {abnormal_rdma_time} ms." +sdma_problem: "In the SDMA communication, {abnormal_ratio} of the communication data volume is less than {min_size} MB, and the total time is {abnormal_time} ms.\n" +rdma_problem: "In the RDMA communication, {abnormal_ratio} of the communication data volume is less than {min_size} MB, and the total time is {abnormal_time} ms." min_sdma_size: 16 #M min_rdma_size: 1 #M min_sdma_ratio: 0.2 diff --git a/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py index 55e08a86b..4f3340828 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/api_compare_bean.py @@ -17,9 +17,9 @@ class ApiInfo: for data in self._data_list: self.total_dur += data.api_dur self.self_time += data.api_self_time - self.total_dur /= 1000.0 - self.self_time /= 1000.0 - self.avg_dur = self.total_dur / self.number if self.number else 0.0 + self.total_dur = round(self.total_dur / 1000.0, 2) + self.self_time = round(self.self_time / 1000.0, 2) + self.avg_dur = round(self.total_dur / self.number, 2) if self.number else 0.0 class ApiCompareBean: diff --git a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py index df96addc4..630f79341 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py @@ -1,6 +1,7 @@ from compare_backend.utils.common_func import calculate_diff_ratio from compare_backend.utils.constant import Constant from compare_backend.utils.excel_config import ExcelConfig +from profiler.advisor.utils.utils import convert_to_float class KernelCompareInfo: @@ -15,10 +16,10 @@ class KernelCompareInfo: return self._kernel_type = data_list[0] self._input_shapes = data_list[1] - self._total_dur = data_list[2] + self._total_dur = round(convert_to_float(data_list[2]), 2) self._number = data_list[3] - self._max_dur = data_list[4] - self._min_dur = data_list[5] + self._max_dur = round(convert_to_float(data_list[4]), 2) + self._min_dur = round(convert_to_float(data_list[5]), 2) @property def kernel_type(self): @@ -46,7 +47,7 @@ class KernelCompareInfo: @property def avg_dur(self): - return self._total_dur / self._number if self._total_dur and self._number else 0.0 + return round(self._total_dur / self._number, 2) if self._total_dur and self._number else 0.0 class KernelCompareBean: diff --git a/profiler/compare_tools/compare_backend/utils/common_func.py b/profiler/compare_tools/compare_backend/utils/common_func.py index 1ced3c0f8..8bdfca95e 100644 --- a/profiler/compare_tools/compare_backend/utils/common_func.py +++ b/profiler/compare_tools/compare_backend/utils/common_func.py @@ -7,8 +7,8 @@ def calculate_diff_ratio(base_value: float, comparison_value: float): if not base_value and not comparison_value: ratio = 1.0 else: - ratio = float('inf') if not base_value else comparison_value / base_value - return [comparison_value - base_value, ratio] + ratio = float('inf') if not base_value else round(comparison_value / base_value, 4) + return [round(comparison_value - base_value, 2), ratio] def update_order_id(data_list: list): diff --git a/profiler/requirements/build.txt b/profiler/requirements/build.txt index ec879a353..a5763cf47 100644 --- a/profiler/requirements/build.txt +++ b/profiler/requirements/build.txt @@ -11,5 +11,5 @@ xlsxwriter sqlalchemy urllib3<2.0 bottleneck>=1.3.6 -numpy==1.26.4 +numpy<=1.26.4 pandas \ No newline at end of file diff --git a/profiler/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py b/profiler/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py new file mode 100644 index 000000000..fd1c40d8c --- /dev/null +++ b/profiler/test/ut/advisor/communication_advice/test_bandwidth_contention_advice.py @@ -0,0 +1,217 @@ +import os +import shutil +import stat +import json +import csv +import unittest +from profiler.advisor.interface.interface import Interface +from profiler.advisor.common.analyzer_scopes import SupportedScopes + + +class TestBandwidthContentionAdvice(unittest.TestCase): + TMP_DIR = "./ascend_pt" + OUTPUT_DIR = "./ascend_pt/ASCEND_PROFILER_OUTPUT" + interface = None + err_interface = None + + def tearDown(self): + if os.path.exists(TestBandwidthContentionAdvice.TMP_DIR): + shutil.rmtree(TestBandwidthContentionAdvice.TMP_DIR) + self.clear_htmls() + + def setUp(self): + if os.path.exists(TestBandwidthContentionAdvice.TMP_DIR): + shutil.rmtree(TestBandwidthContentionAdvice.TMP_DIR) + if not os.path.exists(TestBandwidthContentionAdvice.TMP_DIR): + os.makedirs(TestBandwidthContentionAdvice.TMP_DIR) + if not os.path.exists(TestBandwidthContentionAdvice.OUTPUT_DIR): + os.makedirs(TestBandwidthContentionAdvice.OUTPUT_DIR) + self.clear_htmls() + + @classmethod + def clear_htmls(cls): + current_path = os.path.dirname(os.path.abspath(__file__)) + for filename in os.listdir(current_path): + # 检查文件是否以“mstt”开头 + if filename.startswith("mstt"): + # 构建文件的完整路径 + file_path = os.path.join(current_path, filename) + # 删除文件 + os.remove(file_path) + + @classmethod + def get_communication_view(cls): + data = {"step1":{"collective" : { + "hcom_broadcast__844_1_1@13681369207305868844": { + "Communication Time Info": { + "Start Timestamp(us)": 171317.0, + "Elapse Time(ms)": 10.6086, + "Transit Time(ms)": 0.00126, + "Wait Time(ms)": 0.014939999999999998, + "Synchronization Time(ms)": 0.00714, + "Idle Time(ms)": 0.044660000000000005, + "Wait Time Ratio": 0.9222, + "Synchronization Time Ratio": 0.85 + }, + "Communication Bandwidth Info": { + "RDMA": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "HCCS": { + "Transit Size(MB)": 0.28575999999999997, + "Transit Time(ms)": 1.8620000000000001, + "Bandwidth(GB/s)": 13.3151, + "Large Packet Ratio": 0.0, + "Size Distribution": { + "0.004224": [ + 6, + 0.00736 + ], + "0.003232": [ + 1, + 0.00126 + ] + } + }, + "PCIE": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SDMA": { + "Transit Size(MB)": 0.28575999999999997, + "Transit Time(ms)": 1.8620000000000001, + "Bandwidth(GB/s)": 3.3151, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SIO": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + } + } + }, + "hcom_allReduce__844_2_1@13681369207305868844": { + "Communication Time Info": { + "Start Timestamp(us)": 188417.2, + "Elapse Time(ms)": 10.9042, + "Transit Time(ms)": 1.35236, + "Wait Time(ms)": 1.47632, + "Synchronization Time(ms)": 1.44524, + "Idle Time(ms)": 0.07551999999999981, + "Wait Time Ratio": 0.5219, + "Synchronization Time Ratio": 0.5166 + }, + "Communication Bandwidth Info": { + "RDMA": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "HCCS": { + "Transit Size(MB)": 176.16076799999996, + "Transit Time(ms)": 9.55658, + "Bandwidth(GB/s)": 18.4335, + "Large Packet Ratio": 0.0, + "Size Distribution": { + "12.582912": [ + 14, + 9.55658 + ] + } + }, + "PCIE": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SDMA": { + "Transit Size(MB)": 176.16076799999996, + "Transit Time(ms)": 9.55658, + "Bandwidth(GB/s)": 18.4335, + "Large Packet Ratio": 0, + "Size Distribution": {} + }, + "SIO": { + "Transit Size(MB)": 0, + "Transit Time(ms)": 0, + "Bandwidth(GB/s)": 0, + "Large Packet Ratio": 0, + "Size Distribution": {} + } + } + }, + }}} + return data + + @classmethod + def create_kernel_details(cls): + # create csv files + csv_header = ['Step Id', 'Model ID', 'Task ID', 'Stream ID', 'Name', 'Type', 'Accelerator Core', + 'Start Time(us)', + 'Duration(us)', 'Wait Time(us)', 'Block Dim', 'Mix Block Dim', 'Input Shapes', 'Input Data Types', + 'Input Formats', 'Output Shapes', 'Output Data Types', 'Output Formats', 'Context ID', + 'aicore_time(us)', + 'aic_total_cycles', 'aic_mac_ratio', 'aic_mac_int8_ratio', 'aic_cube_fops', + 'aic_vector_fops', + 'aiv_time(us)', 'aiv_total_cycles', 'aiv_vec_fp32_ratio', 'aiv_vec_fp16_ratio', + 'aiv_vec_int32_ratio', + 'aiv_vec_misc_ratio', 'aiv_cube_fops', 'aiv_vector_fops'] + csv_row1 = [1, 4294967295, 1265, 16, 'MatMul56', 'MatMul', 'AI_CORE', "172317\t", 21.2, 261.56, 9, + 0, + '4,1025', 'INT64', 'FORMAT_ND', '4,1025', 'INT32', 'FORMAT_ND', 'N/A', + 0, 0, 0, 0, 0, 0, + 1.77, 29508, 0, 0, 0.0062, 0, 0, 5856] + csv_row2 = [1, 4294967295, 1265, 16, 'Add2', 'Add', 'AI_VECTOR_CORE', "183317\t", 1.5, 261.56, 9, + 0, + '4,1025', 'INT64', 'FORMAT_ND', '4,1025', 'INT32', 'FORMAT_ND', 'N/A', + 0, 0, 0, 0, 0, 0, + 1.77, 29508, 0, 0, 0.0062, 0, 0, 5856] + csv_row3 = [1, 4294967295, 1265, 16, 'MatMul57', 'MatMul', 'AI_CORE', "189233\t", 3.14, 261.56, 9, 0, + '4,1025', 'INT64', 'FORMAT_ND', '4,1025', 'INT32', 'FORMAT_ND', 'N/A', + 0, 0, 0, 0, 0, 0, + 1.77, 29508, 0, 0, 0.0062, 0, 0, 5856] + csv_row4 = [1, 4294967295, 1265, 16, 'Add1', 'Add', 'AI_CORE', "189933\t", 3.14, 261.56, 9, 0, + '4,1025', 'INT64', 'FORMAT_ND', '4,1025', 'INT32', 'FORMAT_ND', 'N/A', + 2.3, 28888, 0.2, 0.1, 0.1, 0.7, + 0, 0, 0, 0, 0, 0, 0, 0] + + with os.fdopen(os.open(f"{TestBandwidthContentionAdvice.OUTPUT_DIR}/kernel_details.csv", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w', newline='') as fp: + csv_writer = csv.writer(fp) + csv_writer.writerow(csv_header) + csv_writer.writerow(csv_row1) + csv_writer.writerow(csv_row2) + csv_writer.writerow(csv_row3) + csv_writer.writerow(csv_row4) + + @classmethod + def create_communicaton_json(cls): + raw_data = cls.get_communication_view() + with os.fdopen(os.open(f"{TestBandwidthContentionAdvice.OUTPUT_DIR}/communication.json", + os.O_WRONLY | os.O_CREAT, stat.S_IWUSR | stat.S_IRUSR), 'w') as fp: + fp.write(json.dumps(raw_data)) + + def test_run_should_run_success_when_ascend_pt_contain_communication_json(self): + self.create_communicaton_json() + self.create_kernel_details() + interface = Interface(profiling_path=self.TMP_DIR) + dimension = Interface.COMMUNICATION + scope = SupportedScopes.BANDWIDTH_CONTENTION_DETECTION + result = interface.get_result(dimension, scope, render_html=1, output_dict=False, profiling_path=self.TMP_DIR) + self.assertEqual(2, len(result.data.get("Bandwidth Contention Analysis", []))) + self.assertEqual(1, len(result.data.get("Bandwidth Contention Analysis", []).get('data'))) + result.clear() -- Gitee From 8fd1c91c36d100999cec0f6be01fa30e1a62bd1d Mon Sep 17 00:00:00 2001 From: zhaolei Date: Wed, 4 Sep 2024 18:14:44 +0800 Subject: [PATCH 693/791] =?UTF-8?q?wheel=20=E5=8C=85=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/README.md b/profiler/README.md index 05460f77f..e6e894525 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -107,7 +107,7 @@ Successfully installed msprof-analyze-{version} | profiler版本 | 发布日期 | 下载链接 | 校验码 | |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ------------------------------------------------------------ | - | 1.2.3 | 2024-08-29 | [msprof_analyze-1.2.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.3/msprof_analyze-1.2.3-py3-none-any.whl) | 72aa827b8b09557cfb29684e13b496527d53087f6cac6803ddf9933335fa8e0c | + | 1.2.3 | 2024-08-29 | [msprof_analyze-1.2.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.3/msprof_analyze-1.2.3-py3-none-any.whl) | b0348d5cbf82d72cc292fc979a9f385c046e46a1edc5a0cb38cbea515b239845 | | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.2/msprof_analyze-1.2.2-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | -- Gitee From abe9c3d87baed69286f3577352326846716fb80b Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Tue, 20 Aug 2024 16:58:41 +0800 Subject: [PATCH 694/791] =?UTF-8?q?[advisor]=E8=B5=84=E6=96=99=E6=96=B0?= =?UTF-8?q?=E5=A2=9Egc=5Fanalysis=E5=8A=9F=E8=83=BD=E6=8F=8F=E8=BF=B0?= =?UTF-8?q?=E4=BB=A5=E5=8F=8A=E7=BB=93=E6=9E=9C=E7=A4=BA=E4=BE=8B=E5=85=A8?= =?UTF-8?q?=E9=87=8F=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/advisor/README.md | 126 ++++++++++++++++++++----- profiler/advisor/img/cluster.png | Bin 26371 -> 33703 bytes profiler/advisor/img/cluster_1.png | Bin 47672 -> 21351 bytes profiler/advisor/img/cluster_2.png | Bin 66908 -> 53151 bytes profiler/advisor/img/cluster_3.png | Bin 0 -> 11062 bytes profiler/advisor/img/cluster_4.png | Bin 0 -> 21548 bytes profiler/advisor/img/cluster_5.png | Bin 0 -> 16509 bytes profiler/advisor/img/communication.png | Bin 58862 -> 35142 bytes profiler/advisor/img/comparison.png | Bin 0 -> 47043 bytes profiler/advisor/img/comparison1.png | Bin 0 -> 47331 bytes profiler/advisor/img/comparison2.png | Bin 0 -> 41740 bytes profiler/advisor/img/comparison3.png | Bin 0 -> 65269 bytes profiler/advisor/img/computation_1.png | Bin 47410 -> 26627 bytes profiler/advisor/img/computation_2.png | Bin 0 -> 42333 bytes profiler/advisor/img/env_var.png | Bin 61753 -> 44030 bytes profiler/advisor/img/gc.png | Bin 0 -> 41689 bytes profiler/advisor/img/memory.png | Bin 0 -> 21547 bytes profiler/advisor/img/overall.png | Bin 49616 -> 8820 bytes profiler/advisor/img/overall_0.png | Bin 56377 -> 14520 bytes profiler/advisor/img/schedule_1.png | Bin 73378 -> 28123 bytes profiler/advisor/img/schedule_2.png | Bin 75404 -> 65231 bytes profiler/advisor/img/schedule_3.png | Bin 82264 -> 27510 bytes 22 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 profiler/advisor/img/cluster_3.png create mode 100644 profiler/advisor/img/cluster_4.png create mode 100644 profiler/advisor/img/cluster_5.png create mode 100644 profiler/advisor/img/comparison.png create mode 100644 profiler/advisor/img/comparison1.png create mode 100644 profiler/advisor/img/comparison2.png create mode 100644 profiler/advisor/img/comparison3.png create mode 100644 profiler/advisor/img/computation_2.png create mode 100644 profiler/advisor/img/gc.png create mode 100644 profiler/advisor/img/memory.png diff --git a/profiler/advisor/README.md b/profiler/advisor/README.md index 0f6a03807..85921f786 100644 --- a/profiler/advisor/README.md +++ b/profiler/advisor/README.md @@ -78,6 +78,9 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 |communication| packet_analysis |通信小包检测 | | scheduling | timeline_fusion_ops | 亲和API替换调优 | | | timeline_op_dispatch | 识别算子下发问题(路径3/路径5) | +| | gc_analysis | 识别异常垃圾回收事件。需要Ascend PyTorch Profiler采集时开启experimental_config下的gc_delect_threshold功能。 | +| memory | memory_analysis | 识别异常的内存申请释放操作 | +| comparison | comparison_analysis | 识别标杆和待比对性能数据的Kernel和API数据（无标杆场景是集群内部快慢卡的性能数据对比，有标杆场景是两个集群之间存在明显耗时差异的相同卡之间的性能数据对比） | - all @@ -123,30 +126,96 @@ msprof-analyze的advisor功能是将Ascend PyTorch Profiler或者msprof采集的 | --debug | 工具执行报错时可打开此开关，将会展示详细保存堆栈信息。 | 否 | | -h，-H
--help | 在需要查询当前命令附属子命令或相关参数时，给出帮助建议。 | 否 | -### 报告解析 +### 报告解析（无标杆） -如下图所示，工具会从集群、单卡性能拆解、调度和计算等维度进行问题诊断并给出相应的调优建议。 +无标杆是指执行msprof-analyze advisor时，未配置-bp参数，指定基准性能数据进行比对。 + +如下图所示，工具会从集群、单卡性能拆解、调度和计算等维度进行问题诊断并给出相应的调优建议。并通过红、黄、绿色块表示问题优先级，分别为High（高）、Medium（中）、Low（低）。 ![输入图片说明](./img/cluster.png) -cluster模块的分析 -1. 包含快慢卡和快慢链路分析，仅识别问题，不提供调优建议。 -2. 通信重传检测分析，识别发生重传的通信域并提供调优建议。 -如下图示例，识别到当前训练任务的通信和下发（free较多说明存在任务下发存在问题）存在问题。 +#### overall模块的分析 + +overall模块仅识别问题，不提供调优建议。 + +- 无标杆单卡场景的overall模块的Environment Variable Issues是对环境变量的设置做出推荐。 + + ![env_var.png](./img/env_var.png) + +- 无标杆单卡场景的overall模块的overall summary分析包含当前训练任务慢卡的性能拆解，按照计算、通信和下发三个维度进行耗时的统计，可以基于该分析识别到训练性能瓶颈是计算、通信还是下发问题，同样不提供调优建议。 + + ![输入图片说明](./img/overall_0.png) + + ![输入图片说明](./img/overall.png) + +- 无标杆集群场景的overall模块包含快慢卡和快慢链路分析。 + + ![cluster_1](./img/cluster_1.png) + + ![cluster_3](./img/cluster_3.png) + + ![cluster_4](./img/cluster_4.png) + + ![cluster_5](./img/cluster_5.png) + +#### comparison + +comparison模块内容如下图示例，识别标杆和待比对性能数据的Kernel和API数据，无标杆场景的comparison是集群内部快慢卡的性能数据对比。包括： + +- Kernel compare of Rank* Step* and Rank* Step*：Kernel的总耗时、平均耗时、最大耗时、最小耗时和执行次数，以及标杆的对应数据，最后计算Diff Total Ratio（标杆总耗时/总耗时）和Diff Avg Ratio（标杆平均耗时/平均耗时）。 + + Diff Total Ratio和Diff Avg Ratio大于1则表示当前环境性能更优，小于1则表示当前环境有待优化，等于1则表示当前环境与标杆环境性能接近。 + + ![comparison2](./img/comparison2.png) + +- Api compare of Rank* Step* and Rank* Step*：API的总耗时、API自身耗时（除去API调用的子API的耗时）、平均耗时和执行次数，以及标杆的对应数据，最后计算Diff Total Ratio（标杆总耗时/总耗时）、Diff Self Ratio（标杆API自身耗时/API自身耗时）、Diff Avg Ratio（标杆平均耗时/平均耗时）和Diff Calls Ratio（标杆执行次数/执行次数）。 + + Diff Total Ratio、Diff Self Ratio、Diff Avg Ratio和Diff Calls Ratio大于1则表示当前环境性能更优，小于1则表示当前环境有待优化，等于1则表示当前环境与标杆环境性能接近。 + + ![comparison3](./img/comparison3.png) + +`mstt_advisor_{timestamp}.html`文件的comparison模块内容仅展示Kernel和API的Top 10条数据，详细数据需要查看`mstt_advisor_{timestamp}.xlsx`文件。 + +#### performance problem analysis模块的分析 + +performance problem analysis模块包含如下子模块。 + +memory模块分析内存的异常申请释放操作。 + +![memory](./img/memory.png) + +communication模块从通信维度进行分析，目前支持通信小算子检测。 + +![communication](./img/communication.png) + +通信重传检测分析，识别发生重传的通信域并提供调优建议。 + +如下图所示，识别到当前训练任务存在通信重传问题，并提供调优建议。 -![cluster_1](./img/cluster_1.png) -如下图所示，识别到当前训练任务存在通信重传问题，并提供调优建议 ![cluster_2](./img/cluster_2.png) -overall模块的分析包含当前训练任务慢卡的性能拆解，按照计算、通信和下发三个维度进行耗时的统计，可以基于该分析识别到训练性能瓶颈是计算、通信还是下发问题，同样不提供调优建议。 -![输入图片说明](./img/overall_0.png) +computation模块从device计算性能维度进行分析，能够识别AI CPU、计算bound、动态Shape、AI Core算子降频分析等问题并给出相应建议。此处不再详细展开，按照报告进行调优即可。示例如下： + +![computation_1](./img/computation_1.png) + +当存在pp stage（流水线并行）时，computation会按stage分析，每个stage就是一个流水线切分，比如0\~7卡为stage-0、8\~15卡为stage-1。 + +![computation_2](./img/computation_2.png) + +schedule模块包GC Analysis、含亲和API、aclOpCompile、syncBatchNorm、SynchronizeStream等多项检测。 -![输入图片说明](./img/overall.png) +如下图示例，GC Analysis提示存在异常垃圾回收事件，用户可以通过有效的Python内存管理、使用gc_set_threshold()调整垃圾回收阈值、使用gc.disable()禁用gc等方法处理GC问题。 + +![gc](./img/gc.png) + +如下图示例，Affinity API Issues提示存在可以替换的亲和API并给出对应的堆栈，用户可以根据堆栈找到需要修改的代码，并给出修改案例（API instruction超链接）。 + +![schedule_3](./img/schedule_3.png) + +如下图示例，Synchronize Stream Issues提示存在耗时较多的同步流，并给出触发同步流的堆栈，需要根据堆栈来修改对应代码消除同步流。 -overall模块的environment_variable_analysis是对环境变量的设置做出推荐 -![env_var.png](img%2Fenv_var.png) +![schedule_2](./img/schedule_2.png) -schedule模块包含亲和API、aclOpCompile、syncBatchNorm、SynchronizeStream等多项检测。 如下图示例，Operator Dispatch Issues提示需要在运行脚本的最开头添加如下代码用于消除aclOpCompile： ```python @@ -156,20 +225,33 @@ torch_npu.npu.config.allow_internal_format = False ![输入图片说明](./img/schedule_1.png) -如下图示例，Synchronize Stream Issues提示存在耗时较多的同步流，并给出触发同步流的堆栈，需要根据堆栈来修改对应代码消除同步流。 +### 报告解析（有标杆） -![schedule_2](./img/schedule_2.png) +有标杆是指执行msprof-analyze advisor时，配置-bp参数，指定基准性能数据进行比对。 -如下图示例，Affinity API Issues提示存在可以替换的亲和API并给出对应的堆栈，用户可以根据堆栈找到需要修改的代码，并给出修改案例（API instruction超链接）。 +有标杆单卡场景：不进行overall模块的分析，performance problem analysis模块与有标杆场景下的performance problem analysis模块结果一致。 -![schedule_3](./img/schedule_3.png) +有标杆集群场景： -computation模块从device计算性能维度进行分析，能够识别AI CPU、计算bound、动态Shape、AI Core算子降频分析等问题并给出相应建议。此处不再详细展开，按照报告进行调优即可。 +- overall模块进行快慢卡和快慢链路分析，与无标杆集群场景一致，请参见**“报告解析（无标杆）** > **overall模块的分析**”。 +- 提供Environment Variable Issues，与无标杆单卡场景一致，请参见**“报告解析（无标杆）** > **overall模块的分析**”。 +- 有标杆集群场景同样提供comparison模块（无标杆场景是集群内部快慢卡的性能数据对比，有标杆场景是两个集群之间存在明显耗时差异的相同卡之间的性能数据对比）。 -![computation_1](./img/computation_1.png) +comparison模块内容如下图示例，识别标杆和待比对性能数据的Kernel和API数据，包括： -communication模块从通信维度进行分析，目前支持通信小算子检测。 -![communication](./img/communication.png) +- Kernel compare of Target and Benchmark：Kernel的总耗时、平均耗时、最大耗时、最小耗时和执行次数，以及标杆的对应数据，最后计算Diff Total Ratio（标杆总耗时/总耗时）和Diff Avg Ratio（标杆平均耗时/平均耗时）。 + + Diff Total Ratio和Diff Avg Ratio大于1则表示当前环境性能更优，小于1则表示当前环境有待优化，等于1则表示当前环境与标杆环境性能接近。 + + ![comparison](./img/comparison.png) + +- Api compare of Target and Benchmark：API的总耗时、API自身耗时（除去API调用的子API的耗时）、平均耗时和执行次数，以及标杆的对应数据，最后计算Diff Total Ratio（标杆总耗时/总耗时）、Diff Self Ratio（标杆API自身耗时/API自身耗时）、Diff Avg Ratio（标杆平均耗时/平均耗时）和Diff Calls Ratio（标杆执行次数/执行次数）。 + + Diff Total Ratio、Diff Self Ratio、Diff Avg Ratio和Diff Calls Ratio大于1则表示当前环境性能更优，小于1则表示当前环境有待优化，等于1则表示当前环境与标杆环境性能接近。 + + ![comparison1](./img/comparison1.png) + +`mstt_advisor_{timestamp}.html`文件的comparison模块内容仅展示Kernel和API的Top 10条数据，详细数据需要查看`mstt_advisor_{timestamp}.xlsx`文件。 ## 工具使用（Jupyter Notebook方式） diff --git a/profiler/advisor/img/cluster.png b/profiler/advisor/img/cluster.png index 0ac2ee57e1abd73544714fa6025d2eb7343494b1..7c3817a1d70f50d85ed15239846d795e74ec4357 100644 GIT binary patch literal 33703 zcmd43cTiL7+b+!3%@!#^r79$dqOheZQVdOrhy@9#Tbh7KRR~2wO+qY$h$x7tfHVUr zSm+QsQ6fg9B!JRFF@PjU2{j=E!dcP%d(WBq{`qFkd(IqZm^IJJv!38Zx2506e=PDA`Acipad0qiHO`aJ#+HJrAW8gjm^}5R!m_q}o5#BOnWwTFm^AKf*VJ$rFwA8Lw={Te20=ZTe?et`-4et%W%X7=7x&V;BP z=|5RSi&0H0OgkwfO7iZ_%B~v8X@qC#vbpAb^y@bj3wA53jIv$+STlAbmC=yR+=yXY z@Kmtto zol3-$&w01qWSj34lt&fsXJqS6jM!SJ3l`%e&h_rz!fx6j5M&pa#Mov z2+eT6)v1i#H$*~QOmnPiSlQ$7PC~|IPBIA`qm8yJmtvksLfteT86{h@L_{Rcst{lS z!-;xfbpaOcu@23nC7Q=5$u49#>E2kfDksnwO+D#25?&&!EiGe`4GMZX=SdrpE-^3L zZ=*#`qQ4Z8hc1V6kj8ZpJvNV!(=rK~NMpWpWI)?}R#=$crJUh}<2l@X8X=SRhlumCL>V@Op@UZ2220AM z9B?G;0XL?^&Gl?n=5pYRp|_|$EAAPemRIv!yt{<&^A?AgB(}84!S1oyHz+qTrZ|6( z8iD*>Yj6vDSx`!dk_N`hH3hq2jl^B4*@4px{Ysdpydr_TE(R94dSa@CzFbR=;B2(4 z+;zwXaRbuD-u^t%viOe-ql7(W;3gQJ?{RVAlbEXm^J@acO6{I!_amw;gL%_J-$t`CS`j(E{v-Aq8wan{ebROEcZ*D#W6UsqO~!x42Torl$Sv zJ=Ig~Q7;jXN!cc3xch5v`{%FEMx8B6(4dFfC~}v%YK!1oW=wKf zS@4GLJ$>#jG5*srGto`8kQ|wo@VQ)1?!u>IeWMNryvEYkZJ7S?nikIv9acJ@*>ZK% zJqx2(RTT4JD2w6O3f&w}8o-!!IjW~%Vya4dD9r0BT#t(kflFE7xf#&^s2hGykavwVI_< zRUTJg_t$x8<0*I)HPR#XYtO7nfbY9M_|j*h`g^Aa)k)w3yvLpudc<6$n@uG6bka+#O|>l<;E_~SRC z(rPv%DUhv9w*e(X*})?NoNV&a8g9O5L+SCxa8wP2mNA1K^AGtp;&8S%p@jV^7=F1S zqK?@&ykvRC;{t-ESB9{r7`mo58krlL$%Q}Z#C6X)J19dlJBjQ_mG;ln##!^9Awy$j zYf|7ze$?Sou7h-xqB1*ZI15j92~{>fTuNgs_kihv!&$nq4t#O2Egm~_6y8#2Lhf=H z3!OrOnM-VY&OmaOF9k9EET~_}O`FKoVz~1)@92jO*s5v&nb}V8NHY4h zh6}sBr;(KZt@&qE!y$G^iK`lvpu>KuYbG^ zk9|Gi@M_M=`&!HW%j{db+A4IXyhZu73c1^SUw^DT zBDIx_Hwet`8(yzkvMR|z@ z88YQD^Iv3U%>uqy;<*LQiTMvBVzKBV!ESDhAaY=ATDvxJX`QQaJ%Je7E+`r-*I8f5 z;*%{PNzAx2D=0W8u#ZuzKHm3Lu=^2aq_}Oanx7+vcYnQ!GL-(czXv4>-JS&{M1#M+ z97b2iVztn%=Ow{GDrh(npg<}_bD_lN8KE>QQDN-_th`vT$KFm9=Lkz=(wpH=k38-6qjv-v9h za1)%=J@SJ;!wB}R+$6=+&A)Xd__2;@fD-M)8KT6>NIV-ou+c%SlTZR3iNhr>*~(PcnGKlI1Eh_!EQvw;d%JS*GAttSrE+~p(v=z|PY%rMf)Fv`=WnP9MKqx&vbAT@e9*h%N*3#=X)Xd?(NXVyUX-thNb0;-O2OG0;Xdbx% ztJbQQ{?>fPZ|XEFv0`Kznc0iox<5N_S*g`S5hUsmwFCf^@4Vg`wcranX7B;__(+k04!#AWA%RIJ-*V5#!aDo-2K zQ?TdGqN2U~_<~gj9%tH^jZG@2vefk2z8;j!ZYf#aDgllICWW26o2N#In8QLFjwi{>DJsIlV9tdrm$fSCmUwgVkJ;{z>Ah2z`d7@uvHa^S`xkOB zd~<4jOp<|qMf71Wji05M65TX2(yh*NG3`T@C*UoXwYoS|ebQb{BRWIwM@b_WiW$9# zk=;5hk!fin%VwK%EZ$CI3|)+-PK{E%;#Zt_5~?u!f^pCxq~cd*%lfGMQs~^WCSzWP zi$^Fq!;ks>^cNerVu_ELOCCY=QLARcvi@3N+jDt0d?q_sy`lM|sB(vT&^NR6#%i`M zC^=r+T~1Kmmy)ukh7`Fa>Mh@LQlVnwabJuw0S2?6L{Cp?Uyt~?9&(eXIPS2>T(rir zk;O}bEl?`ohe&|s+&=L}FWE~47Cu-P$UU6w#q(!3+KzHBsk9vDZc#T&CGzWYuH~_( zhs}Lk7iZ9QonAQ#M`WfV7uUNmT#E(ENHdiZ$n4vm5We<&bQSM~x$fkqV9$$+0bfgx z$D|Q5M&5yC7gG$_;hY-%0AK)KURJBq1#MfaV%V*2vrsg5qMpGT`7|DG;%J^c?75^A zttuSI%h7AO^$y%q`j4x_*mKqf1Q^!!Vvl7-K;dz$`}&^f$|*FkHt20F+l}~Z1D}Ag0fpzct_7tkq(j-I`;X_MA@sB zD3vv>*fpTvMBc4m{r(A4;W}On878W~nMgSv=&z%@VQLnG?e@>z$i(|bwybZO-OrNp zbwjDHFAES1y0=*f)JE2%kmeD|ZmnPx0RS~=xb}d}yF*iQ7miS=0l&7~ZW|WOf2?@q zkT_V}J^wHLWUtgMWv|9)ssl|M=UBY$;I*`)I+_=Hzr~c9E4yji8u$9|YdsOKf=Qh= zO!uId-U>UA(*Hx;YW<&CT-ZV@WWz=+sQL1-SeDgH>f)@uJ zB$sGZc!uw@Dfnqkn5`2vEiucql^-5r+rY?(9J z7}X(tYP=frWfAil;x*hY~}mQ$AUykUo7h!T4#(POWYf~~4iT8QcYghk;C39q*svN7Y5*v)jx;}wig`y~$eSx4Z=4Ta zoo;sg@wR+R&3X-PpLxZ#bJIJYB9i{nlJGim-Q(y^$ZI(VUvlj=Mn9!$H5Y~t9WXmq z9lu|K&7XnXr?1~Fs6NS`S(ifl>Ns$Yxvy5|uYPW? zEbVyu+bkC?{p{hB&FF@)Qr?Y!J*IwlAzM6Kr}qF>n(wBVjanS4FTv%(O_k?{9%sSM z*kv}5pvxDgQhx<(f8apreG;VDuuE$%fmu`yh4PR5vtI6U!<7_sDrVa_{x~-=aM;ffS_voWujr}eC(m}z5@5b<=jzM*Wb57< zRfe5ok2|Bg2(X*$ZsN>TB#uhHui$e(T{@91r6kU#+vdQ1vlB=4yVh_-_B6)y7B%4m z;Q9~qQ7G2}^$yvD4agTN@98{u_~0I`pYz8ye~yeX9gTkUh0JY2suXw;ki=}q1%W1` zfbPr<7}%d53Tz?FummgwE?+#m=tA}^soYW=THr3m4e--uN9L@Y5eCIyzFXoO#Lm}e zJ#(zyZ}T*~wpb7OIkQcCXt}TH4tnfublF9pLQA}jiv#ELqMT-~H+az7&Z=u*CV>?< zX9XXTc(LH`+m%j^kdIN-byf(sS&3deVX@dsHGG*8^voG@Fx+(5Q|o3lhJ>MxRQkfa zT%G#6A4xM$*TwgBc-;GdQ-)Sb`^OiFY36bfJY}WyW`92o^J3yCcL8mo`xi&>lQPiP z?CDZxvEq3uJZLnhbMwO2*--D6XTU;dIByOwWWp_m3A2mi7CRwR)1d<7Dg{(>o%eBr zQ6S9?1v3HDt&-zkyM6@4&nIqdGnw6p|$N)9kP z>Gyo9-FEh#57^E%4Qa5GY(ndWbz&EBFOO4i7 zPL3x>Iw={c_&>L05k0VHy3FtCzBtdu_eoo1R{9H$K8)9nh@(7*9juGw0&Ixcl$V!g z@A+6#6ElAP0@WXSj!lvWu}S*=)vPNO*ZjVn3|H|I?K;;|tA6(BmC=UOGn_rW-|pW@ zFVHnzv{YfgjBppjl3oT|%wZ28D$I*pM8VLBMfW83(4`>KyEKf>Fzx<`pn?y>`~`2{XVYk5w!xXk7IY#ks=~jcgjGIr|ZLmGI&+|sqtBvquCgvhB^yKlfB?l z>gK{+j|ctZvtXv6fP4;-@|g(FG@hoc+&r-ZDk4&PZ#(#f&BBzNYgqJ`=644xGudht zlA1#DD|>rad$<4US1gEjo|pe@ zWxESMeOWIRu7}zr&d+*v_obgXMQWU71g}=C2tFJPngimKWre+pPxZu6vztB3W!7MKuLP$u-^c1kv7?oN)bz zKTG)Cgyfg$P2ZRY+sM-g+UnL;u;Nr7CTHeD#Ftd6}XjofjFhi`k!BIb!j0<%_$aH8ig{KeoVJh zmLiBb{@*Aw`vOUq20Mo(*Y|u48wuKgye%yH-PWnv9Q@|9pe3xJ!OLlqWZ_63#q!xj zB<(dCqn#wZ={#DDD%%WS&Dku=Sy@v@Y8WpC^k*X~Hhx63rEV2Dz72y1Wl8kcMajk& zVHEB&BwiQ5%!2(!xe9$fgg%4mX>_MQ9Ix?p7qoqNiv(D(Vo@x;IZbM?YqBk72j^lD z1o93I3D9KTj3QWgFw*h_miWRKf)s^_WK9jhEP zI5ab!t$_3Awa7rl!22H37{0tspyef-R%p8E-qpU%#k=WYI1onDC*H6RrsI^JeG?-R~hlvG!A(^Pw)qidrn;enY!m`z0A3srHkJ%e}d zELRyNy7=SAV)AGQLzr8O#${oTy;O>bhlru?6!)|{3>^L-28XzCay_>V>TAaxN*~+i3X`8rR*40K3#$u z=Wb0`HpwpG21i0cPu;M&8$lu>R~|*`DmHC%~F``)^r^+V`tITt53jrLjc#XW-h4Sl#S_GbuqhonyAfp86aZ()MxT3FHO?p9#h=pHA-bn%@6})edGmhs4Yd=n(Eih{ zwQ~6_>+4U!d3olR`1JX`pxuL8ap`usYWF^KHn5E&gT+KQ)l3nQ@6BILkoWF?xV+#- zlefvY3UlgGQHaaB|AlkMXe7vgIG+_XW29r5L3*DmDw3S9ej+cjGiiX}my|$Hoc$Wp zUW14E53g3WPy*MBR-)UggB`6MpM0)?nVyHBeIN-TN9`m2E2%+&n3vNvYrZ*foIz(?56Y zo4C)Pt&2HhaxM=C2%)Qb1b7&RpjCp zcEr^b&RqITlzRmh=XB5V|GqQ6)Lto9=S>=+&`-9Ymj}DI)Hlv2(kqtHD4dbI7i-71 zt>EGl(<{{^PA9+PscXeEA9mcQ{3or@A_P5A?%$cZxdc!~nIP}rn8?1g|6uZ#raD+`(`37==BQu(K7cQnVYp6 z)fos^#e*K+X>`ntkklwAwV3T*T*-}6^@%>1Ki&T4w%S0(iE-1xKBp%H*wWv!*Msnj zkR2=W``k5_{R%tS^Yb=)y9iPuA~o~NU}(Fa?z?jm-z-0DqA%2>D!jiQ;y&u@;t?%? z;fc5Q>fXtq%ekHNz-GblyJt_>mLcMfB0pD&S{?Y%tZa!VqTYF?eXA`Rq~^V}e<6KV z=nnQGKO5Y^$Dy$;2n8g|?g+uFm|R!1_eJsS8M&hG1l3V%teN~Cvwc4WZ4weOkyZmS zVrskDAlrxxcyp~X}#*$^<{X+FE5%WvO($Mb&fy0-k<>7DfHfy15k&S`q} z1s9gzI@t-!Eck_X>Ei>30Vqt11zpi|HZwXK?X~>+H%7?}s)FqFmazdH|BXmqSa^T< zfPMl7Ansr1S?_6@AXvJ;F8<2zUkh!_19ErhtWSPV9E@uB`TQ~9x9gEyyZ9Sa{eFpL zqcJiK{{4~WZ&P*l8`InGwU46}?Em|YqFf+s2H^V)KuU;+96#`X@bcFlXyyhhZ=O>U zBNxM)%wT*~0{kEV<@?`T;*-qROP7VE_hUy(X{v07UAVAxHDu3e);xu3Azc8^I%Foi zRcb7yhFo2*pq`Boc3I192?Snr1La`Wu1|FWW4YDhzn4HR$U$GI1C)G`s|(K~SdXNO zRwu@j&i(s@5T}T|*tK;#*xWp#7|8@m@5c`OHvxY|c{~xHoSa-Y%SiVdSZGvoX)~OY zqU3o4Z_#@f3nE(r5xdCyE&Gv*^1nJF6lqI^;f_e@dr1$L9yk#)BojDl3ioDWBXG<77*4ofWd?h)!tw{#=qjo>zc6={1B}u;bSh(JL@J)=PiL<*e@_hKa zEJpI{#~3hODX^GbATMbH$EgG=OSJu@50@`pl@-erqt4MGfxAm|OLj~7_rc>_`LCMW z7Ggqw_a}`Z4lds(Xo~l?UvNFTCvYItIP(E9Dr$7Hcd`7=`h2$Dh+|WLOXQ8Zs+rZ> z2oHp(Q--(4*vRV%$Lp-p!$S>+&HB9Qpx!=uJ>w$jBJ>r@}t+r+K@fP}uKT)jvugx89}a-a$L1^+)L`7je%T zM@?Mqm*d_#9j#@9FCbPuJ3n)*dx-hk=v6c3trGtVc|?Oz1*Z%;iA6aVnB6V_E;V@-~X{6614n$s8&7P_%u zhIg5cP&#OzyBj1+q=%FWxVEp7a7<~inDp!jwhgOVLKQKGP!43(OA{_ znPZHM+^;;W-*v^XXR9h`pLnR5}BSzOvC(VA@aMHnTBIDU>(L*CnZr*;|n zS$RO_;Lo;GpSZEiae`cnrP+4rgJi)cG^kOHF30ruGtM;4)z8Vf7I5vzOn-&0Y3Qmw zSyR}?QJ@XuogBEK_}sDfDZ9{prscI|=zT*CWd@aF&oH>KCCs23OlH{jr8 zArodgW8TAYzQ5vR$E$G%k;W^Zl7WTOaAf&8ozcbDP2D{cbZ@8}v5k&8Kk;f#%0F-A zi(KM2IBSRWw4#$tpha-NBwHPXg64PfBnx$02*Kaf1dE4 zjq=YPKE%5aXSc(j%JW1+;CCbaPO4KtaVoaO(&L^F^{yU$=W4M#CbDz{rs`CkLsN~= zGOa=#u{p}A4bZZ&0n%h9V8&2DITD&aQ6Djw(6(Jw{q;JLr~lV3ijJ)LHltCk|Cnek zWaM-z>3=D%cBJdS6!uL-zTkE&CnO$~Livl^pg1;N6TKc)&1i3oS%)!Ggcxvcn2A&Z? z+CW6|`$(XmriD|hEz3yl$W>;l+Z&uTQ2+hdkw-K}1>8c4*_kax+WdZZek5I=;SkjL zj|5~9A|17=xAF5WRk;{Jk6r9W@%!=Hh6Xk=tR6eyGdyT!_0afF%qK$)Nxe%YZr84?7OPUjb!S= zBe+hw#kYOZs=zO|TD-3UxJ(+dNRW?O75=hqM{N={OPJ*)pMEdQkeL5tF{maiqgf8Zg8q!3JVM{XLaNKzkKLlr-imy2d%-)c=>m`q*!oQ&~+4Xo)&TjC{Rg;RQ{%Fhz-0x|1dyI_Ga zHZ6@g%IEtz*899exTfh6$fCc-@zfdcEXA;{()(moGm!hEry+d&nhNl^!FQTRM5Li$ zi$$IJO!RWFvH9>SU5W8yKfZZuifgpOgY$9QK;Z4V*7Qu%5H(XitFzRj60=cLgW|pK z4W>7d89&yrIbVnD?iE&aU zDpFL+lB-tVA50CdVEpK7i3Hp6upAp=x;Q@~eB{-T>0DLo z)*_VJX!zL3e0$j^EVXh)dtuE}sR9FkSrP_yVbAAzI22KADg#~V8-`_OGZyG~D|H3% zSf=!dQsCdU1<`RJ?>-ZfwRCum^;k1>)P3Bnchx0a+7x>gwdAb8G(Q#<@P>~frs#fe z3Lueg@~WVYf;~V6#KQ%V<0ow4Zu!DF5+@z$yy4G%}AS96yt3EIXHRx;j3g zK82E-TYOMcF=h`oUH6Mb#;2xJWCTA^mZ+O%t_&HxqIG1vFH_D47k>zzWvm#;j>?vz zG>^OV;rSpdE8m&^yaS@F-ucLit2z-!G^2a=X`3*o55jfc1o5IiEf04;il0Rh|up`|z`CH#;0Pyz z=g{sdCW_&*b5q3x@<-16rD|f1`7X$SbB*=*9YalYg6eVd!d4e@97@^>A!8!XF&fpx ztZtcWi=PP#$87eFxVsL%a+)dGE)Bkp;4XeZOc|zky~}z|dERNBy5|I-IXt{b0mahq ze}JXZ2;AvIo2H^7w><1TShRDOl@Sk!9SKilyr<=q2NVJm6VqiQtax>;sl43FtWqLV zfnPWEZm(kxG^V5vjU4?n!rYyaJ$Dn*!${9sD6z3MwTBLRMnxl-_+#jsYLgtAW zDc|VM2Z*5*H}boplv{Pgx2{(rA#rJ28@Vl1Tl&ua-P^uZ(kz*^7Oq z$0`u{Gp0dp5|6u97`KW0%&{9$T~8u(Ez)E^^JouxW7P9rys~@$aBe%*nr>Ea;(YmL zr?vEqq!DxWS>uR`e6+I~{h%u~ukY{?`M}OjgDA_>74KbAgs7W(D5-A{hChWe;+Tzd?uMA0L|j4@ZR>{Yz%2#VkMQ&|2K^yY{?$IBe!EIv ziXtgA4;QrV$&%{)*_=@s+};)-1}m!w>uBGoDliK$i3#>$ky&e5Q~cb3LhIK{Sog$F zC1pFbhI#>^yWkr3JUV;I*A`ltyTr7Qp|<2~ZS}nZk+f=qW?Q364AtKw0#aiF!Nb+& z&H$%QCIIP*Kkm{ca97qOJl={Zp1hU3om(weqa?Ke`ZslC9Edh4HG}R{VxmFT;A^$~3 z=hDckAQ2_zCC&|^@(%dC&E=O@bR9MQX@t}mvIy~eDp{lvd%iWc%7Ax8iQg<;9RwOb z;Nf?%V9AlB8`V)e$oFS=2GdVMl&$!HA6C5toJ4cky++TM{ru3iwi46XxpxlzJy(8b zjJP(MPzR7`<42|Az+L69)Y9%rj9V_y<CO%JVnGgX_6* z6lri^#(SNFk@J~;C^fEh<|!B@1rQTXVO?U4FpH%YL%S1 z8=vUmc-`bBIeu(<@HGw288~<4in=Hwq{e1JaF#vFivuyIn*w%AjJrg{ZR3C?sh?yb z3<0hNDX!41w@*r}0wMm#HG4|@Ws_{mr}@)INhj;kH@^)3kh!--lEW^%7@SVE2N)V8S-(Geau6VuK>z7)N=>2e!DP7J8$i z`ue)g{2-76=BjKM=(9T_3Hn~UP|LXs@1l6qRt8Sst z1^}^Kiq!H6vBlRsG?7~~0i7yge*93HXAD27&NM*^t?%T-;iu}-?APaG2l|Vr{aOFa zP*Ng%5Q0b~{6I=rxV|AIaG=S=^oU%R^{XXRtrqg$!?-FL7jkjqHi@Lr*(xB=%)~y8 z14XI!W@4IV|CZG!DH{^cM=NJlv)%YG6V5?lEH86bqAi54UU1A7>8XuWNL@H zc2cnVaTqtZI5iG3k#EKEw$xQ&j`Y1(*bU80y`--CWksU`Q+T8Ux}_*VzsO_Koo+WhnCE>WWcRXN-b1=YZ?IZ8FdYvGFr) zs4KLvC)hIh{fBP2?gwWr@%ds1vo{ruh1&AhLbc~^`j_|ZQW(zyV=Kdo8xQvT8`tS( zrIl1f4f$i@K>K9|4zJa!FYOKR9W;5Rjb!EaC*+0_M*S~*s2ejItST+uE1Z6B)vsSv zTAy1u-$b6T+30Fs7jM~-@5|EL6_|+Petl}OpypqauWl#s>t09?);39Xc|5;-I#MlL zI=3h4-5K@=(+3DsIiCPi%us;&#H$&hY?E;5I%|G0*;eMwd0$Z&N`c>1G*!YvysFS0 z#|8epD-K?CKZ&vxNd;SAY-5TY2uJe(4?eo@Xy_5m02g{kRn1$aVQ0Z~klLo4;F(%x z>tJ<$WJ1;X;=XSHs*Hi*+gZIBoX(8ZfPhzrr!1J%xL7K!pdXB z*mRbA1+mWzqh&hE$o}cXN-CyyGJ&umi@32s26r+W@St=66KB>677V@&u3i@v!6yBJqV!k8BF&xf;Cas4qT#f0-+wi`@&T6DX z(S`tuQ7BoY5zuC7y4*d?wv$J?5YPfZ#GpN--IjQQdMN%du~Rbrdd8EZ*ge)Bt^hN- zpRiRygS$s=aL5CK^GrG@GXR8u!`s0%$7}lvFGOQIjCW=ax|!pFaNf?A^3iZ2rZ__4Xal!cV0Nc;Z@3c@cAq>^Kz(6s(TH}&6O_6#eS-dpriQyBTS zU)}{>Z);6q4EmR7yY2+;Ts3@7Agiljl?C(s(O>bofQ((0W(ZW)_pZ;53J|MCo0Frl zV3Y7zFuhtBeJ>ZuL)RCUs6r-rm-#wbr8x+@lrjIddPKa0#Bq*XdFs$?L0Z0GEEwHH z8FR)GfM-N7{&ByEXk&htwJ^5UY2bI7Fj7hD-jTEJf|v594eLuec$v+0szH>Pu&18sc@UzGokZtntyDCU!F12F;s583hreIp z?)r;dx>dXD*UJ9iBbWaFZ=o7M=oS(-BZ6-w&AOv#{mkfbKuHq$zC~}DK(Gs^}VMvt##qEQq{BOmsa_N(!8j|{T>uTq>v=r*aZ!akvhi;2K-)Sn5kA9)w5p6 zdAty}+}2qF@XXl*BjR$uRG)7b*TZ&>LQ>j1QhKrztHn9jJ=1M6jeqW~P{D+agI@!c&Ic z48s>>OsiN#2GzWbu@x#B+FXbvC2!-~$HuLOa~;p94bS4&j%h1ph*h)Alx+=rsfmEv zVBm4o-NItDgBw?nXvKeP(iGw8LE6wsLTw3FJccx_&+Ox*H!Ttf|87_kh zX}!ORBA>*2nqScH5rPDKYf%>c`fMlc`aEqv(m2Y`H!9@$OCsQ;3|%V|Tc zw$2Sd&y5p+$Z(O-6kqO7(*r~q8Qk%(>!?7f>Z17S_$OBY&hC~(SF<$ul{C&Z>wbOz z$Lx7@dXJA{*0Rjr5qC;Rg^kv@@>%9M~?v0 zP4U1ErR2M?f&<)ZYR`SAl?KpnUp`?G7W5WuGzX&gCao-fn6>BY_ydVEAaEki+Av%~ zd5>^pFB?4`b7lyUMZosaIVoX|j09G&FwI3KFesd7u`_#EW}Ae)2Na_>1d$`P9SjH7 z^eg_PjmmcT>;vq8yL_NU{~jHCNpP=^=wjEP?vi@*#TNI8RD`6oO;#a5WPF_q4y}A= zcr>qxO0bc&y z+{SaP`I*MIE^WEQR{Ht^B81){IrTZ!tfJB2&;3%{5@YebM!mX~zKHtX@&O<%eDOPR zVh}!K>PTwRhlPZS&h8hTtL`|9OL;le%Fz$5Gq2S=;w!( zbb_?nhRRFaF7+u(NL&4Jf=MILB=%vYLdxo*+?TH09KM4)fde6Cp_six5N)`=d1LGor87r)~(Xu#p#hXRQGtUDt6ZXssYb$IxL?+o}54Jp_9zy zKB#^s0i5q!qE~O@LHOq4LXn}6(txaR>db>Jxbo&BDSyeBWVD1{Xg}p~vZJigJ}7R5 zLGXga1CXsb7}p%G#DT;=1GENJop)`|xN^`5nC5e=F7dp@D{|NCyNqNrlayAl67X$V zKDTFf+n44fOTDs+Dt5kE!tF~TwL-h&C?bz$(aKh5b5SRp`v{S9b%?sy8-*7JAnfAgSxFlT=L+^&s0%@j-T%LEBE zy{kJNzY}291r9d&ylun-@EE_iUFs02YAf7$W{%t82?F``-G!J`Z#ty7`aJM$LJQ)p z7`V=~F?RxeqXpfa-4d)}f@kdsmjV-e$#c z-men1f#v(S$r^teKbO08rfc{^d9zot!X?(DOt>^t6nRjZ4T2k zp8~RK!b*es#|rKzvps_k14on+57Msyng(Z2&Oo+2Q`>1L34`jVt+JPPLcMGugS$V~ zi%LgH-y;`gAE3@b*pUxUSzEhox+;o?0H^Np?Amtlkse;&WuO+Iv^8HmdGp`b0w3aZ zl+AygYc2IS|8@u1RbLoX8kwfZZoY14Ti;4I#4nUZCj+9`gP;+4h_R#O-#%9yfv%p6 z3?JnLpUZQ-Q4)lTR8tw;k;9G>2jBSDPnef$J$P6mol-9y%S@r*CSQ@47I?m*n}eTL z&O}=VfRfbc^2E1|b6JenKve0h_0>y@vHuq=Q#z0YTD9#*6d;=sJbz!Q6WHkW)6-V7a5lX#vL}0lfqWX-T<^Z6@)+auP`?eYML3v9iE>vVSjB{;?RLA?;q{{NmEtW z_OC97zS+9|TK+fkmMkDhgKtL7Ka#Ezg5jI3rnf$0&~w6B)XUeoMZRWC?M;-nwwysAFi(RKb6NN?^Sk9Hq!E~r_~2Rd|%Axy9b1gOJb31kYh^J(_)cK*9^c$N(fcSF*k${hGJt&F!_ zD;|Bf_cLA(<4Ejm@q&@ayud^2dTv0^-iunXjw^V2+@xNLtd$qZ6ibp;Ln=xsg+s)w z3w-b3d~w3vC-IDSyYry**RA*yaiDr~MF&P)2sPz_9EvQ=&fkd?pT+9ffZvz7*MZHbEZsE^mtcf)n+reUj&#=GUdsKb@FQ)kj8JrB^=gWpt*v~4R7JIIao zP0aKY%zlk0GZtoLR#trDbd`g$dyvNFRq8g0a$F^apBEW;9$G93l1OYz44bfvyn6Fp zsp~5T>h10yDU1b~hoq`DcZX2^cc8w3bop0>C= z^;a&_w!Vmn)vnUF%g&!>Q}OD&9ZA;YCCb=r63&+!C?SKO1Cg1YL&83QHC*#E9?^of zDNFFHX-pezkWci;JD<(%U_&ul>B@TFjwC z&t!TS1H%uG=M{gUc_95G*ByCKDfNPIBMBaW2=iRH2F$w}M&Rtx`eP3c(pzSYz=@w! z?M}or`KKU&O_KH&rviP~D&ARabz?XlNL*QdyFQUe`O>F(=+8f0C8qnAk@(~LfaeDW z27NFC?GOGoF|TUtqg>>K*TA{7ft$n`5pCql?Rh>4M`6_&F2y& z6;6uJd0m8RnOG8_Jv+iO~z>0=WKtXK*M& zvFLrLI^yp#IFrQL*}fq|e-a?1WTyE73v*RZK9W^x4xsseY_d&}L4&g~%Ye`s4}3iL z_4fRGPKUk~n!~L^xsbjyNob2pik8XJjgcCI?AQDG&2ekxYREKm*|~&h_=6eKC$U>8 zD#QKGmszersxi<~G_pBYhtSRjqFhROng6u8Mqx5OzxYxB!^bu`Ubz?xhe z?eT;F8qDA;%zs|?4~rIbIQ`FU4{QIr50$03R?nhpEUSSO=!MbOjwO1(3HU=XTTX5R zlI7OKk3*|GO8CcxuFei;B?na=R)XvF4-`ihHlw4Vmv%v)D71vmPSHZ@uT(FAhauVb zJF6!@CeN>bz%cN1u2f89%NnL@$35N(@)vB$Q~_{XzcR-<2^pZ&W>J0+KyjM$Nj5kN zH$b|2LdM(&JG#T{sRzJAUy_AZk`kLUK@OE)`y?3jVv%1;eh-hTQZ6Y=?*Ti>UNko@6H(;BlurjS{Dydp$jfJu1? zU^y>DsV|1%{L4r;i2+8eaCe5}|7hIABFVWnp>Hd~OdpU~mGgK5LxPJe-&{E){-8-M8kS)8K4Gf8 z%Wb-+*e&e4BM^*$*W?fhN>-rV{EgsuXka9d(7xR@Mo0oEI^@~leY-q>QK8FPCJynXYbP?fOgo3On>^fL*gP! z^sHl7z!nE#?zi3U`iKj;3)nvF%6mBeC;s@H%_H8q)fY?o(vRofSx_bO>@3ExwPKI< z*aC2^)rG$~coJ;>aYtdWn}YY)P=C)27A=k=P1#4pS?e29@YWY>wEW>K*k&uSdYix& z#{rj~T^U!G&UShE1@MPQpMc7KR}DQV3^xDxp-|f`C270Z(kzKmW1~vt%YTI`u_}o5 z^`V--&cpWF`gcgUl+P3pqeZV>Yq&l;ib2Y?_Irr%MhFz`Sjo5b)8@!UP`*m2wg`H& zH)#@=5>6ZS?r1D%fXQRH%v0PgRzCk;K@Z#DO-+1Cjs3ysK78G5ThUBi zsOfru%nW!oKzAOw{>#9nqC;ziiVhJ?+WJUL5f>S@`kop%k)kNhwf0J2v!)Hh9RMq> zN7Q|7g!fsY<;l$ov73pB6W`so%Mg-!2hW`X5d4wS^KSS_G?x}CFUpPZ4dr*{tJ~x; z%}@`IYPeBwAzIP)1PeW2M_F=f2tZW^8?IDJL$@c~21Cr+H4$IJRJYFPNM`L7I0k0- zKZFda%7+~4RG`BOwn|)f*OXiZG>kDvc5YUhc7R;&GjHNjzyIme4xGf$jcGcaXp23V zu{9Vt5Y*DxbMsju{hN{l-`rSN135kqB9@{EJ27vg(ZeaAn0bgd0VaVqYKTv==QO&t zGvRGzgNbRwaOu`wdVp$=cO<)?H`hbHqAZMhOci73c$ms z&XhWFLS#)8k9jzw;MVJ6X%ob(>7omKO{R_TZDWYX0jsuESM753A>g20E#v%-7{);M z$6UQZHrTEmXUIYXqAwt>SGI-3K>7R?FkL|~KZR#b3s9Pz+Xyo8+v6l?sKEgC9A`#S&@IKoTE=Surt9-4 zkr@Tr45Y6NYl;2{t7=kxof_eEX{dU1y3;o`)I)p7j~3&7yNuff4VzOnpg34ym~{!v zi2gO=v>Rc5OS5sg+-WZS+tMov`c>HK@t-&z1%V<=JLo>uaO}>#YPsoWFIJVGcE}*U zJc;aAJ8g_P#EU``VaH!06zH}Y>#Z^Ow_^#Nw|kbtV>jC{JXM#7b=`VMCFHXS+Rc~m zQH2F-oEv3@5d7Pfb4KTR)0zgKWXV~fIjvryGv~DAkL*0K8N(kG)(kygcR=VH>Xmiv z$`bu%R=3%jx}cih5UY)nQCW#q3{gU)1&kY0q9>tJG%QT>lIz3-r6Aq||1#apC+Jz{ zT;kfQQOMqkvnNu%&ee8MRb5)#^^qDw4iKb+9~Vfg-SLj(Sr!nwdKwkQoaQskGYE#W ze#@(gJ5Wio7KFJ%128xXR?S#GajNY z=@;e-nYVQ3B60PFHLxjVrEevW(Gi_C%n@Gok(`b3-*5f#l@s-FaqGN8XJ_3pMgAv` zJ$+N!L5g>Ko8dw+L|y8PuvsrC>3{K2t~9L3)3Z;@eILi84bV;&(0wuqU!dCzZ^z}f z?IiCuE8|*d*I5Kr<>xFfo5k;20uYNC6^BZD1qdl!GNksIqisb!?YqNp&5%y)pEun zn&r9&)K2`)C%*8kq|G)$=>+FuH%jSjx|cN1Qj(OQ4<;MUSDvR?VfZk-Mg|orYDkP6 zX3z8d3tiFDChcLald5qrJ{H{1rr27Y*bO&Zv)ZirH7)1vh83X`6DOD40mMTaClRj` z3UA)uekisrVvAyx4%ruWfW}f&(j%iyB^QG-f;FL#O$PmDe)bf264rD!hWWk|Z^w=&~cH5NC82bAjR0^;N6m5(v#qyr{=^w$4LZty3CWGm^_k*oR;h12NN_ z8L;=*qo<(swM`ba47s9=?3N@lJ$(f_4eb`PB zm%DMfHM=xJzX6ULdmtvwtvkgOJC)YOs zr~?_x_MT)%hx0qf$+s!bxIwHRLr|-QY|ev44Hj%<4HNAOGj;o^7OHY<oK=8O|FZ2spm5bp0vp|5afx`3$oZt z_ZGteM9?^@*=|HhOkPxaV-O{A%=NJb2(vxRq7#*(+;H`ZhXs}#-~ocQR{2qrnNgB3 z5h2a*OYa@wQ>`<$U>vsUH;&3j>~kog&g5L~lAAN`e1ExQGD<{~Uv93BS^y8od4#K{ zzwV~$7)gGt)pmcNgUw!-eg>rsv|vVqs0fekH(?j+>=zwHFWyQ_H+%cqi%5HR!u7Hm zu{#TN0)QaCQBq=U+XnTYUDmeg4L0>%cgAgP^{#oVz@1^*+PRB3)4^hQl{Pa7Uunq4 zZ@w~av6Fo&t>}Qzoh}ONhQyfNQuW9<&NDL>fXV|#RAr&4yT3D?o;TJ=j{bHRouxnN z_FZE|I5y)P4dxwW*)5H4UoDk}s%nm~xPlxSgLD#p4lR_S?Tay@cIRi{ZHc$RgmUoQ zL`psus@R1NTZ(+_$=PhyS%A!qFP#w=+7##Cn+1B{ckE6;4ovovyKPeEv0qRJE?&Hm zd9qhOe>vpg#P1&lx+JAEd+MwDZc5ksfA=HxZZS=~`$CtlN$s;9mbo*l$US^7BI5l* z#rTD^ijyS*CKXW;021HPLuy=)&C-b(D17HB!0od}4~pceDOBDp(9wHRe^B49FAs7>xe*P2SgAqJN~-kHBy7@QWu_fFimNNBgzZq!*Rf373YXWaq&lC_> z8OHY|?W>UW^Z*1QlZ%6hQT*yHzQ&}*aB zyQ@_(SuNZHHqi6-4{sxn=TF2P(y3zuihhb{SsL7^)J;;cq{{rN_BM+b(xp^)KugKy7&=Ba@@>!# z^q4`32ho*~TGrkp zIgMacO>N(&u1120>CXMb;&+744`%Q==Hwuv6{^ zQ~+Wt08#|+{woYx*dgGKIQiFqM1&hHJJ|B8;6WaJvIg;D+73+@rFIBxD+{v&xxeWH zy3)?#clwY+xVm=P{A6zvJ7}Kt>;dvA7KN9hzJL`n%hVBPO^mnDm%1V(>@~x+#X=Sv z>7!lY?BKAyFB@2ifM}D_XRdQtGHe~bKI|1CI>KUL2x(sR$HFf_fM$uc@gDVn!WO+K z>#{EnL^6b$G8e*v?)u64fJu|}Z+2eac9S>~Iwttd2W;&WIQUVJH6lwNec|)ytf6ES z???n*Pz`lJkm#&udIqkg2Rs-F zVtRmM?mQQCroM=Mg4|^n^mS^SSii>pDPo#*0gqdw%`V*T3s_rz@4!RE}S}OJ=1Nm$(n- zX*EprY@&8@-P2EqO73{BHwE-Z8Dp$R26to}DZQc?UUDEE&C_hO=5UVOJm^PR8m_ys z+Ox9(!>bNX^=sWAx)OI-;AK?P4~ZWb&tu|7&ef*%N4e#-5;J=Yd$xyQ@GT|FD4Cw% z_lh;=pxXoU4XeOXAyG!%{V;jD6w9V>9(@N9`sZPa=-90u@e^3)d=f)6n18ry_!t@* za;p@NP6>y^>7H0{^aokqQ7wPJA^Tlg54Z&$ptQ6o4=Q9}xUD?v>TRwk#F+~Q^M0-s z7_KYR3pa4nJuMVvymMCty5m%!PRP>_h?<$NJ_qHWa~y9XWy6nS_2MNXy|&l$lxr^y zhSQ?G!|)`-U0#s?u=s&Hgy90){Q)`&`p5MO54lOolhJNH>9dZ~(6<6hX-OV-YC~bS zt1oAg;A?1l(bTB(wg5vk0ON+fsPgdkSG!Rz9`U?+PGZ=?VDa6#_eB($iXa8G$i>W1 zU04W77&?sqtYn4X{h+UN4kC6&Q>N)5|^6k;){!) z4J5)u(F*BuIrH7(@K85R!{($m%P0XdP8BM&-xzc!A(bpb{T!#8RX zcSk(19x{(s_^c^3Z1m!S$}zNYLu5GPgvgeYKA5qwO3H}9rAlRgqw+2_p(xCdm00b= zfd2WOpVMRpr;1*?P`#;jaE1WGStcdUi$|RwJ)D<0#IG11f+|1JHmFAj*M+WwfRQmU zSzK?UU@yul*YR=c4Y@;OP}1IKUCqyC9;BG#hc}sDxT{{V^vHG*S)Z!rpGQB53x&i< zK2Tt0>mi3JX=K-S=aN%U){nI`thO2}P`|pmrelI43JQzT!b8K#3_mXwk-61X2SXo( zRBLk{h*@@Thajr8;>b65LWGQy@?U>mjtGT>jxAoRF4l7Y^1N@hr)M#P-8Dfrcaj#Z zl^rBBI{I+htD&`tY^cat5CL6n^UKfHlD~&>+s5vk!hJBjPaj$%L=TJhIt*KR@~E%W zg*g>x>2IwSDx1qT$L)PqE@TwS<8XE#m(R&WTvH8_r$q;KZ)(GOq)o=5QY&=k(@Yy7 zQQxk?N+iOeMk@4TP_2=s6wL&a8mIwQlPCkKvXZmDyhxirat7CDo3ih?={Emx>YgYb zEQ}vls+A$1)wfY0>P~%8=@xEp?Db`;u==$7<*z{lSC4Zzr$AR3#hde_21Y_v@Qs(L zZd5vm-ZA$@rxO*uD&3A>n@BiWs?OIz1wnB!c=RzjQCmF15L`FOp}`Q0 zO-^8_G8_=)Rp_z3y#@I4DB~&txq_{5mCzqsfK+^>+&}2hBZES|A~QV)Zn}3hs~KXs zVwOf({occ4bJ&BEf||J=eHLY=OIa7wC7--GD3dM=VhR+syA!r`Rt!VOBJEXN7$lY_DnN5T{8#wBaT*nUU#Cj98Qm&U!e&& zW2HBHiasnD+FDm3Ww~jeQO^3@tqh;=4M$L?s}pCCkYd6=>GC>JSIt+Y5a*)+{k77R zav=|^hx8f_xZSeE+M8;o1_?40Ci{B`V+-{I{l+}fC2I}?Lz+DR)c52a#3(88*WHL6 zMjNapC&Mq<`Y-m6b|X`~hN~2z+TkA6OSrix4cke9Of3SCq7rY96!bUk`5fI)!ya_jAo%uv zO{?lctd?~K)(4!{Z!FhVHRL&^Rc0uqX8t_&%U~>6-+nYmAzsIRBsEy)D7Pad{9nko zZx>%;F=`FZ(1*k+t>dT#`zB6bv36a=hjfC#;)PnfUp8MpH`7da(8)KL@pITpUx@Ys z9xhY3%zb+n7d;2X_wTcUN}@wbdj!F|^Ep&pu1s?Qjw~;Z;0FZyS9{cq(xm4!_YJ7{+@tq*_DGML6WH4u_Elw_uF0~$;b&_Oox+7<7tBU4&qW&(R# zCY8wB$e;eT?2Z{7R64dRcfzj#CDa;$u{O+aXP4wOiZHQO%Ng` z5}Z!vZQ!n3c5LF)Q{)xGz=({JBvHw_^U#gCf`-Oh@J!wJj_xxl-eJv;&gk|>_eZ`m zR_d@PMyKWne}+iiz^tti)bRlx)+MXG_kQ^F2v|kEB9lfYM!rK&(=F~{OR)hjfJv+72P5@y0VZ4-}i|*!`P&$iQ@k4515$KaH`%V9P znK7BB8gvv)bR6v+Amt;16&LxWKQi{GrGWS3Z~&ohq26M-ljOyQkiO+-^pQ}Y_TNRz zOD!_X`L^RyyU93R@VH+lJ^?}1DPBqT9*$FMi#5LBpZ?H3KmAYL6Z=CYKq>yWmMvBj z5lE}@irg9d{Ft78nm1Z^Cts3%Lwn0Xg9t?|+qymhTO3c=?Ni6vApavE`I7nTA|6{@ z^>-TqK+hU@Uiv>YXLa&lgO6x0k`fmenK;yIOnA>mTJ;Oh0j6uMviqQ>Qz=upIU@eP z4R=)cyRxsTX6+dTSlzj}Fc+23<(A5k(2WHDHz=sii1ELo3!fpZ@ z>0m*$oWs?HN^-S)MHfOWJN7BrDdl!c`QH-8I$90u4xx+vx3VoC_w_X>-NTSYup?ZcOTb_Y78HMej#BIts$V@>vQVWLHHDK@)2`Mitj?`4vzLb1pGgu- z;yR7Euk=RUbb>gY*MONpA-%Ly(T0qq?mILhVK8$@e7_%il{1aTd?wOJ5qxU?gS+6} z;Y=0*3eh-mO*G2%o{{BHXn{L+DA}&}lxwJpjk;pegQ$kAIm)z`gG5SyRSh*}H}Xho zILFwgcPzj}Eukg#xXcO4XsF^d9@6JeS6FVWs#U1uHMdV+4B)i&3i#Hd<(Y?l30~at z82RTP9fT~&SjhRw+EwW@XO}XEbS@rja%Nz*xA(t^9vTv4gdTJ5B!Pl=uOTIGyii?#J) zddqWP{`MhWtEku<`hyl{#)jNrr4d(GTSQV9#>cLDtEow~eLh}P-H^0>a>(1ke>=@1 z=(cs3LrYknIn#8Gljt6gcHOqG*}`evyS<+O>Dy|rUs9#wTXP%g&ANog17NnUEUdwZ74XO#-?>x3Q@DXKh#Uc8_Zk-4FD3E@Ogl?(K)2HzZqd_jEL_b=* zXv4a>CO*P*vn6GgWg{>g!3a{ttqu3qJZK$^P-~m|DuJ9oG`6AY7Ji_!-q13X+3%gM zVavgRy9pSwLGQKS`+W0n}c z#!c!-QO>BV(66vy0x|hi9mCe&WyUYF;M8H;O&)Q7m;GB4McMhE4b;t|xubl|xL#p| z2C3P_o6Qi;SfD;H{ISvJWyrKot#DwiHOmp+$K!$Pb7)!7z~3Mp-_~hxDK$5s?ac|DH z+V*&!w=0sVR^>EjO4-zrvoIywIl{jUm!&;b^;jsR)-z`&*6c4;m<6sXwC=yklo4?5+s8BW(b|I1)Kj7FpVWcVgBHHMJHcQ{H3_ z33yH)*K5ziDq)%E`=qU~;WL3&k>5h57kWZV2da-M^0)d%?9B6!hX$vw_VoxbeiY<` z`g#Ai3+4923Es-+NADM=^a4fJCTpcQf8v_EHM@~q-dqUW_{$Bqa9hFa*UC`+^TkqH zX3H!^Zq7X%mn8wbO%h-dRdLjNn{VoVJogTA+%|zi1!GfWz6)(#OU&feuU5$@RM}jr zr!^A|9eyuU8tOok>j+g6^*F~1pu$A4Gv?D>xc7E{(}kQnS+Zesjfz9A#C5$o?J7v& z^O!!FG4%=w^Nk@8eUe5cj=I9hMe&;|vE7ueGQGlzc_JYYy9Pp4T>Zw{-7JAMCz@^K zL|YCKOIsv0w7x2GHaeD)?wbSYy(6WCVxrHHLFF!hf)n%Ch%D{gnECfZ0z~A@H25eI zgovEzDIKp3aK#5wjUGzgBd*jge1@F0_4wXawQ7P`pj;@`jtgh@CsAaENp(doH^n>~ z;f5vFX7kTh7+s9*G@c$2t^!H-4p_;B)T-=+3~kkPJ>+_sT%C@Q`D2j!+o~xKomoW$ zUVk_b)2sXr-kod0`kSJmqqx*i+0TzZU$?Hw54#V1m%f4gkdesAb{WLeDkEH^i$RMY zsJ=;Z=&@`3p`PhEID980#_FI zJ$8jxohl3~toGn7J1m~$m!qeF<7rXKU+j)q{pkXz-B>k?=6lKS>WuRX$jlq+OO7xyqB~C|Ok4AryH%|anb|zMa z`l%1o_ogX^iw)Ne1N>Yck-U5+weFNX&dE+U^8$DYDr!&Yr}tq$*OHk?p>Sh$#F2uD|&>6 zHGnf|2uPgiwV#=R-P|&$LsG?(@4TDpelbk3OYq<&4ux4*hmHL{IUdLj9vw{<+|Maq zJDC6JvuVeu;(2NHNUv+86=$_9ePjlYI3cR~gM0g)V@91=;u4PmJx>k<-t!qpteSw| zzGw1?5Y0Q@Bul*4|E)gt#Szik-<9#7A8D^WnU&b`6y6zOz!lz{DRygrjP+K~0IH?V%GxN-34~MV<$ZLk`LivQY!a8U#o?9$j&ewN+bj#A@$QcCH zx_w9-O$xr9;E?KaLK9)r|23=Hbnrv&i&A#ISy_ptaVvv!BVx4Z;K*=AiIMM#`%F0;_#Ifq@8ONLJjb{fpa zl@BLvGqza}JivlBb3wy5=~6xZwD;MnxMT z^v6?2_j0_G%OZ$XK(N`}QFUiBc;A5>SJH(^Q*4N4`@Q;y3!`}DcBy3DRWHWc68aly zO1u8l38pR(cZUhw*$<4iH{84mQD;|8o=(00z+LhbnC25J;AjKu5kaHGz!qy^W@k|a zdW<~l8)(4D(jV%}+G2Fw*n0T_Ep?5eX|Og3C28sUZwU`=T|noMN0cufftzO%V{fy_XI>w0%Q}pl(qC~FN{qIj zvd3x9nOeO4f?3w-@}<7;@O4IbRe?^xYf!7mYb;ApO{kDPFsq=2ydOd9lekNpiNo{_ z4QkZ|j8Sv}3FwUSH{R`gN1^xDI->*AG-#(0Pa$jD#T2=!s0t&c$M)FU{>XT9HX4Q` z>g1JLQ2F*FFZLR?HO=XMSsl3K7f?<4)KYwoR?#~V#>;WV8O&MztiUQ6H@OD&@Z$oO z-%*>Q^YWn@7=guMSzOyHUst7Wg?yN2*rmsC%^th0m%q8&sE|T5;dd55nOc1;<%%so zD4SwsOGIiD1J6hBSsIW!H)sMfQBLL%edHE#sPOu;O!w}O93k$$%V&9wQ!3G>A)g>Q z#2S>6IoFqOym@OTGqh8SYICI9lA4I1U<1oLBi;dly~veF7mce#K#bLlOqtnEgJjCs(-jc_x3$L<+xb)yxS^>5C19@FoyGxFy}i zU@seopZ9zqut09LcO&{t_%-^@-@d;3lD}j7OYHkqJ-2I%t{gR;=6h&<^Al}-*%*d! z(=ePc-!LXkY{GM9?sRAMUz9!EuUS2s=N(q^3(Gcs7QhxJ!F_JX?EuH4bBDzz<>fy&hnRM! z*wvmZe83t--H1u4AV%L7|1EAb(nHmJM6jvWWTn!{BP5G>5%Fb+j_t(IvW>In2cqEO zY2nb<$ak{y7mBsy=SL;D=M-yw{F8SW&X!q(rQ*?$x*M3=`kQW{d2RHR8o^Hytbw8^ zdzHjlk#gCMxY!>`z&&_;|AS`VjtPUAF;}m6bjDCG1LtY2k|FReiW$;5m%jFyhG>6p z6Qb_mfYPwJ$7FTHPI~fLbw2p%vmy~k^m%CIM6BpeArE5F%}%Axl%e_Ai+O#0Dy57n z&qMMeJFt2kuu4m0KCVF%$UxxvvKxLyB>8obb%<-hJNsbQ>J`Aiyjt_{iV%b4Vh?SG zuTQR_xpNpwy&KZcQ45~j>G##k!b4|c(beY)w?2-2)PUHCoxNH2 zvHT(pk&407g(}-fKbq1S)VT=!YLFU8=;`n+M-S{KD~BNxERY0Y0Kmj)OuXk&>0w-r zbH8qIOC{fM5R1RxY2baYZ>%b+%}dNpFudY0WT6w|HprL#ZJQN)So}erf0SEgQ^HDN zmFxF0Vdm1d_jZ_i)GOlqga@#P-+wP2C8)Who@e=o@1^Zq(pB)$}0WRmx-4v{U)m?NH1wVb8gKoq%yy^gUnzL|L|c6`P2%d zCHszc=o34FUqXP$>a7UVmZ?9Yy-jsiWakgxdKfm@FJY55#pjd~%Cg`70fP-na6J0` zuz2(p%&BKm{JOMv?r*hYqt}!;I1sW3;h$nsxJ=*=|BxYnAMrbLfGh2vJM=fCQ)$Up ztwZZ0_Hewr2)?Uuvm7d%H1lgfqCHP+#y<}wm;ydjPIy-NmRxM?Upv?o$-kE}`(Hln z{|O)VpV_hhNn1icQ@vMn5*kSO&D-k}XC)Ztwi;HU?s8$zKvYD#aEXVrkoZj`sXtxw# z0>bmo)#I8+UhI|w*hcc~1z-0AysTgrfn)d1-Z=wTeK{UvbAWj?_n0HxCg+X>jIT-E ziPQ8iqtt{rk}7(HPfcuu!V zHHWbs7hN$UM?i;JP$qgAF7?_KO0xq)mz&+wX8j;4uhJcwAN{yF(6hz7KHzg^*nv0! zlHM1mF0!q@3#;7SwSg}&sHCxG)`GiKfPy-o%cV*~Dud_Vk zaPQJksnW0_x41tN)JnNWiIu3@l<0^{%KOeqlWjy+f=?3ej>6 zdkuT%GQQX3;R@5->j*`2z~~7HtfzCv#1Hd&C^#rpx-a#VHxs^Md)%n+L0OAae;&$^ z^vaHJJ}qVyK$Sm!tN247SsgesAgECO)3ZeH4Xj0xv)pQw-5XMTWCe*QGKXP*0N9rt zRwp94$5T9XzYc+~k}9MPoXF5rft1w`B*VvWlW%$Dcs<=1Jk}}cE;^U8E8R84AjL)c zmyK_g4Tlwfs zp0<>F%llrKRDE8%c;RLs%Nlaz%;Hi`a*>B6Sv2FkLIm*kP`*%eQ5!+`zAV8VbRBqm z&^G+Nk_f0Yz^(HhykT+ zrRwfaS*%~j7M$Bqb0)#pL8BHcpI_Gy&%jo-DL?B0MiI;_fjH%ov5Q(c68lKJvTe0S z`bY%zDrxB+)w>XrC14t#A<(=Uot75iASrPp*ypR??5mqssV}y$sCbH0@(qwUPzdCT zIMt&595_xl*O_>}wo#Mt7k|JA4V0a>=~h@`zKGV{D))wU>E4t$w_=+S&6Au^sT))FwIuA;JLL?M``;1w zUHB{8fv5Ime0P@9b#?(tY!|56z1!E^u#8-dV)i%h`FV!8Hpa~MewX`5C+LoyaN6r5 zb*mC4lhLp3fVaH_*>cD3o();bP&1$KIb*fvBwqi9K!=dQ*OYD=Iv7MeL&CXm>8^^2sk#MNfZndYzSoe9_I3GM`0O{J zM&l4Hjnb{ZyLod!btLBx`(%NxM*E{)Vuusl=aqN#VG?&ge`**425E_5l$4&$(&KGS z0V!|CL@CsqMhWioZdG2qRjNB$+=m`KHn^mHI)L_6a=|xJ1VlJNk}^p*q^tOBTE@r?wB@JaR~7*QDGmvLtx@ zsHA-OZ-zTSe}~FnT0X(+CMwlhT4I|9QTa9{oeCaW;ZUsaDI@U9yInVjf8~_^>sQVH o^+)*c_ec0sm&UnuVQXj4C1QVKKwgeAXnYPM1GAsedUt;NKYut=q5uE@ literal 26371 zcmeFaXHb*x_AYEeK|nx}UM&>qNbevjN|&y5(17#~QW6pARS}3F9YK1NUV>DGgd)90 zdMEVI0`G%Ab)W5@?Cm>qzML~>h7XzK$z4{v*LAIRhd@ty7=J!9ko-Z z@UERYb^0mZ8Q`5NxMIPnQ;eq`+>z06JH0fz=y+u|UTO)GI1{R|qmlfv$kN9kX%zp) zmQVR93OBmk26u+Dx-xk!5kaA*QhwKh?a4hz?fRhCpaN1Y%vW8sIB=;tzZ9jtK1Flm z^gUcB+A&E1e@TQ)k__Gr{kN^uCGM4|+DZ@cJ@8B|3RV?~*t;hE;?(IgICvzCf4=Bp zTy+%?Z&5$tqn{7E7CqwiX{SBu%g0Yhhn_Lc-z_4yO4cWa;RpvA~DCU$ZcGFa3^vr&XvKU#vMJ|=deE_57}%z>|0lN z-0CFw996Xe_Zf5+tQ0d49x+syY>V?kDdd$dc}-4w7j5AZ$+hTvf3U(-*tpc5K5Is_ zJX8tp?cQoq-}E8N2G>r056t^q;mP})pLL}d);eFyN727~w$jTg`ln6{R?jsO5**PY z7iso(w`TqOE+4DedypakKT`%#y0jlOBc=Z?QizyzzJ^_X`>v6y5dsD^E?@kvAah0&910%{{@PHZ4*GhXWnv;q~Qg4hlB}TaUSfX zKi1X6PC};Ertq@sO)qX9{Pe@cVVMj2{$jA|C=VeK(LGjJhMt~vT=~I6bvtkhd+O#x zuWs+W@xkZWBHWUtM~r#ieL`n?<#`)?1}2xRH~JC{YKQL@O)U3y4OZL+g~q4C_WFvFJ2MVt0;euZeoKaMd25b z%7nxQ9)1-M_u_R_A{la!9x0YdsvPb5Y`C$e@wDKb$87_ILI!L31iZ3MV}xY6^ijSN zxP4a)c^9;6w(YecB&0-!VQ8NR)jja$59=Rk!AdRHK+ygSmB|Y~?B4q>7bD1R>d;Pq zN;|EA<%e$&t0c+8Bl2TH<3}!mZ&jE)wd298wI}B5MXOw2NHnOJ54AYEiH2;dG%ZZF z^QG!DShaW5s=}5A5xCtlS8v6IMnz?}5RzhY{atMySLAdSuxRHXtM?~2b6Zi?DHm() zZE_~-W)7^*Y^mfA9>|D(^zrvk95!9DTZoaUnoM=7mAfkltG)?qX^GC=+Z~Hum=wO~ zSpnJbyLO)~%H5%0~;%1g-Vt5E8zLJOGo z@OBNuBdxswnz*3Uso5c^<<{PXw1<}JnPg(Rt)Km?#t4+{*-BD4-mcQLwuhcSe)$2sc&w%h2ODnca{SA*cW9j_6o90hDv>ARetlVm2E4%8EF*HJ0sLwJ) zaiIDwVPsU_{GQ!6wQx?|lH_lX@`|Q+&ZlNqPUxr&F2%naxn9Wo9OOo)RjF}4+?I&Y z<aWZW*QUp=JGUzQ6TG!obPgM1_;0qjX^Cj4sm)`1ngd(LvYR;c#1 zxMiVFP(C@*NJBRN3LkuvK2WpL%z;j8*T-&YDy(2I6n4-S?0hM777|C}!iaCW5NC+F zyp+%15F14%D`!-i+W8?%B^U^s&os+K}TV})Rr{cvQU$|OS~WMe79+O zZ{5Kx3g7h8MhjrXTF>^wm4w7@V`e6sRw$ZWhr5GvcfI;0U^h`xeOEp$7RKi?C8~w9 z`RL;|gl8X3No#)+@8V<}Q2mPDqd25g$zV&2X}n8$Hbi?I^`zJsu38lSy@e$)hgK`f zM-K6^bkQg-5cPFiPqMsnb$Acavfd(WpUotnmZjA@sT*R;g&#uW7dZn1YCA3oUl&#B>zIZ_kd|pX)7Ji`O|rQRQc9{h;|m zp|kwCnJFi^e5<2RQu=Pw@8_g;+j!F1Jq90)>^#*DK>mvDCI)LUW}gk%Cb$bP9#yb3o04P=*B-d5ZdC20AdC=21Ore%d6^u_fl=P}llnRdJ5QHX$tu#Sbs9Q!fW873;yW1+2_g zWHe9;z4?QqYC9iyjPw|J)m;^r3I#kmuB7*@h;9derv#}=N|vz;k9K#ElhD~(w0ymD z@3O11Z=$0Le{GILVYl|nW&?J;_R!QG@ETro_(p1nzj}QO*<%h5W=Zde_3pDoXok*<3_t% zc>nn3yvDTXA#tD8VK~f5v-~v51*I+3<^!=HGQ30$S+mzZbf>AdWA!9!Jss0XSSD#r zls`p=XsKK;TL|5h8&`Vmp{(ZYm+#zlH*UcBNRB&qU~|FS)a)gdl1YrJQ-zl3=o;zU zYK@sWCx3*(4QtX_mt39V=jh}}iuL@^!iIN;>#MrIeDDk@3Ez#wC;CQa{o_9qNWM$= zdmuioX7lL17#|zlT?aKQ^S4SZCLbDf=&I9NX~U`H!DKa@fox8hQ!KHLvcnK+qCJCk z-h;yE_|I}Ni<3-;Vu3HM@|N@inOt_bz4VqIzebEbyTFpF+|+V3zlUjk!TcyVN9`#A zyVHk~wkJ{IuUlVf&TFj8A_lu6^UsTgQ{)s6<`MVh@9l>9w0zT^=w0rDy_)Q|5+z@p zCSi}$;tg!S&q%H=e?>*1KZMTqD^-ZK@s@X<4)v9|qts8W3{~`O&$PLvTVgL&NwS!* zY@tj8HTu0Zy)BDE;^tnx{#eMY;%XA{KIrTdrv8r1W@^LQ>5rJGg?7zH7L{@noE2q@ z1)FKO6c5Th`7sQO?AxbXosBHA#7iD5CB4yBCS-f=Vp1l1U05e)hPqps8a(gjcUy6L23E_hdnjy~Mb zLRsD1?7(|$;Id$v)_3)1LeD`29AjlK{&kGy9Mbd+z9E)xA?*OAT6laN5PmqFo%Vz_ zn2JqPnP58W<85tvzB^14TTeHx-7($jK`AS8ijAHRM!BapKvo(C^zh75pvGnjPMM8@ zI+?3<)sgg7_99*roq4&I_>rcwB5i%)y(*-W=bEBfvVsQZcp#Jvu1o{Zo_*!cO2}6X zGbVGXz8_^5Wpj1NAx&d%ZPh&fy_&{r4hf$wa;+%BzAM49^7t#RU3vIqmd$&z+9~As ztnb(KU8b|zS{_li^&1Ge5+%nJL>*DX=Wdy_I8sRLVue5ZlIxw_HJC%A&O9iyRjpg% za=r<2*qL&a=(X_yB|&(8!v0*SQdh|5wV1apeM>c}vAaz~3-qRbb1tk^A*S)RZ!&c@ z>otdHZVp$9Em6NN*G$bAnY0{z7eDT&mZ@P=zgy)s1^FTEK#Es@xC_-I#e$DBl1ZO% zU6e6n&OBH6IzjX0U!HJqg@kWpI1zVaW651k;h#eZeuq)ZBQ)lWW;R9 z&Da|Yhjo#{#EITru|XYp1jj*B9CfxClR{`9_(I?{qr-VkA^KsX-C}%O|C9!mp7KfF zhIezJ-J3MDOM7h|f#z46UmpZGvHaFE>qZk;ZyGFHG5GhVuU{TS@vJ(lRWDM^s3wNV~om6U-uO{ihr;|#1&}zW9_Yr zvKkZ$Z{sgZ-LQmc5(Q@SChbjjwXHq(QJA3%6p;Lvr0AFU3kuKQ;3}QF`FWS^$YAJ2 zpzeHwSf9pwkoeYAx>U2f(ll?XzJ};%uS1f>MKEKAM+$6Oh`VtQ)I>?u0Z-)z==9S%&IyEy~5pwvd`)6{LMDG18%UKi2lZLzilD$tTukMW5M_+}--_-06jgrPGMg5h!!GTa^d^fYM9GaL@w2xGLfzK0yA$gAl6j5 zw4Py}aVMD1K{bFBNAuqM@X$a*1{7Dsu)~zz&<-wel%?3vlo$(|_&v+;3Ntp?HBu^M z-5(%e*uq)Y>~OI0o#%>scCBpwURS1jxJI^#wm!Ysr*G)-UX`czh#h>$Qn-}Ys+%xc z{uapGODx2E!gV5k-o}5BmjBsIzVqjs36ykm7Q5!?nOT#wPp0!;6=k+(NR4W$G>#;$ z1a`4gNJ{Q3q-(19&)$GMW61~#Ow zRztoZr^S}gYfULbT8Mm3Kq7|#vp;3lGyJ(mmba#2xehygp;Rebqym_E{D5A3L~nFU z#tW`@8{6>V(29;PM@*xnH0zRD-;@+j>tjAO>$HxsZ`0xhO=oH}Jqjajz|9?q;~Km@ z^?q+ZJ~y??*0kF%20r^bRG%0?USjwfzFAu7;U!lSUTPqf#p!NVh^;&ynjMH@U`Vm3 zjBg$?yxAOF{HeJEWEF=7hil}Tw3J5YpyX!rmR<(ZEd|Bru3h*_u)H+DfGDMlTfb2j zKSw05$aLA4zZ)KqP)%pzSf6qExtbu2$GE)mX^D{{zNLW3CATXc@+SCmMW%^u(>MssJpG~ygB5;*pGAINcWfiK{%=8UXi3ArOECPTtdbZ>dJPot zGrN1cOc=)UA=}H*?7IstHmYw0LZy&vW8=|zhdMQ4=xAOoNQkyuB#PN|tNWY$Nc>W+ zR1K|Gi55SD@+>F>N=a94_uaSUr4PEYB&uAIFrwgVEkSGQ9$HcU(n`kR9=@grEhaCy` z+$w&cC#UNwSM}id)4)9M+mCs8-KX0yaVD^XaZ66quY5H5{j;FgUh~p>_g2JT4KEcFH5|ot#d!3MQX?MO=SpG9`R@l|H>u@5<~(6=u6@ueM7KbVpxcTf71Q>0b$tepsHOw2*{|FjTrxU1cU6;` zZzQnB?gJpW#nv`tfHqt`^9h^;(kfBYchOLpL9D=WF;;+S(&1>A3V05aXYT zBFJ?0^61~%pX**W{><;L`gkpzOxH4ms>n5uHt4dixypDdS=D2OwvvZebJCnBqbjp# z!5XC1Wzg}yjBzM;6Yri}RT$=4PEt%SS0sjS;=#SGEEBHW%v)7fm&W~V-xClFR`3mw zExsfXy-VD>Ppjymek0>rfMBGj#?jK4({9UvsSh)gHZdd%a#bWqIS*5K+7W+|>AYT+ z0dBY?{h`r$rJzyvE zIO=s)zpnGxyaDaX^0j9iH$R&duUUWtpYE7W2z~{j6w}bnr~p8^&3w247ZJbq(RzkY*P_ZwSlpqLwuH>!V}r}tHn~IdVb`U$ z>pv4*8qJ;xm3&F96uG`zBcVtw|E+um~n$_%;{?1pumBCGTGMNMaOeuSu^!A4e5 zneGM1>XX&+6*nm#apjUqgOX|A^TZiwNgAtL0}A8&k=%51Y}qK{_am3~N4yR$u*ZhS zDBsmh(H?IZQ7l+YeU~>m+$ND{xYgPcl_3+>MA%VNDzxyK#45S)2IEcIrMZd$Rl!|N zl=Vn5>-m7FeNqXw3BV)j*bV8swtco^z~Lw%-uD{i6Ok(C8J4-@TCR0!nFC}x;L7ZT zq=RvI(|qmvv)zj{JLw6YnM0h0<5yx5ofLkN=^Oe$7_oevFc>u+=WQ?V;&mjG@9ipn8E#JCcLn9yh9xH z#fxpXttL*3tIhwEilEKF!5F{~2j=-m-PD@u^CZy>a;ysLB`4kLwrFc8T58PABi)zv z?jhF7ImRfyyszt>Cy4WoQ&#i7K7b}1*=$bk%Z_SVnXg99iSB+5QVTOU z^?yuYYS!dRC!s7#>`Lchs&1OsZg5OHqP*TgFA8`0WpjE!=*9XO(o$Ebdy`Wr84Xs# z@6w?~%+)?&3Wgr9GZrG3;&lN#)=!ig)4+7!Ju>e+DiZM~Xn3&fR%*xD%dK1SZ1!uw z+zOzzmDfZ04Rcdk<!3$_GJDr|F zCX2S!P+V1XO2%9G1;gr3%r+mwmq#?rhl#bU_LSP{?pR%mfFa|Egy=o&I`W-&#}n}V z8I+?A28uLk1W}^V3=5p4h*dQPMTRVwW0l&8Ioee^=8NFE9C$t*r*79@YKn=UDEN=-PksauWUc%HU@=G*4@4`_1uD}$O1J8!9Fc7L; z{JG^*-sf0;EU)j}Jzb>Er91dfkqjQ=M%p^fBshp3f2aS0B#*MFw$S1finX{rSHq=vzJ)oEJdZRKHqva?x=m(>&UP?cA&@a z|J=r4WwvxhYS7uT9TkjgGQs+lECn)@l=a&Tjc$UZPZ$|ikYxsY<_+db*sdZjWgfW=kLEu*z;5QO|UsvPm zSDp_YA9FQT!eVsNM&lO(s4SMw7p-!gle_cFxBun+Ki&bcF@i|=ehvKFYks{Dzz2qS z756N`)CPo z5%i~&e(eFr0^mP-@XA&%v;3bYhUDpm!(Y7szg72-fedEeH2j~k_~)IvGvv@(h%6^> z!)j+IVzJ-W00wUO$yodz=J$&Nl5;i&FiCM?VF3{;W>&HdzjsZ9gPEC!mi=7lZS3s; zH3G?Fc13 zb?kd6>wKqOKCi5cg@q#ANDUv$(h7EPxVE+ps@%6`LTP|aG0`2bEo7{g?mPuORu2ij zle7_M3w&{VYb!J~EZWH&Xkf7Oc=F`6n}{NonWP!7nA_T-7M2cHhHY)bLli^8TUwMV zGoV++1Ox;`Zi4V+ewY6KoH$sHakQ^*ZL2?qTc4VWK7o=UJUouy2Wkw6XDvcR4cnAS zjCO1FzS~ZfWf^8}<{zK9pXZg$&whpF%p-x``(E!$Fz-~!5=li(dxh@{*>89Hb1U)^-pB!rV3#I+};9Vvl*6{qfq2IduC)Vg{7{B#l`fZy3{(=n1HU4X? zA(N#cz+xD6$Jtl_)gKG{eOh}{1LF(Sb7t|he*RS;`r}#epuY)LLCK-MzxVa;FN_{u zfcrql52_Z`d?A+Tc$C*_rnKvDBGPOAXdgxlmUFb@aFM~{3-oCK1$l|(Q{n(RKh!x> zV&0r!4!)L*-?1= z)A2c#fe%34vC}G7+%Jh2dTvRjhU{mqOllV5vxq7J^XUxM+MkLWezs>M^mke!`B@xPN70_2H}$!5Va&AwjpFB)4`! z=!lfAT`xg@A6p+3vBvlDCc^j@R}Hoiir@`)^a1A_s~rqj!qOwnfw>_G2>YEg@rKp= zdNv8OdWJp_v7vaB0gpbs(~E6M{GprH0nAwmci4vrKT1%oZ1X5k%Pnd4TQfGJr6mym z1Wm?ZLS+hTjTa&D3o9!%Bct21ql-b)M;fE-NBT>r?GkxqcQ(zy_Wab7tZ;fYC5Ahu z{F(`I$@^cTO|qkN_7P^*6?{FT&-Y-ZI2f{&cKkJH@;H(xp7_lh5y?y!IV=`8bML!x z6QMN&Kbg9pnSt0PItz4QJi9h#Bg~5dG8juhzg~k=|DLhL1-WYi^R{s0(a;$?+&5AI zY%_DDnHb`dRZ~2{dN0bi!4mjse(+;>GPxbK;@p}-=%Q(@%L1w%s}A0-O-~k6^2odiZT;8$z6)V3>Jp- z>cV$;8y37Wd|tnu`1-2lw~_z*3!|$S0Ktw&C8;tHg=vItfld9!##TDU9;@5u&1)I;~9vP5nMjHtFAfYawU-8-o3@UboG2^pD{r+n|BHhqIfCK$C%TO;shOVUCB; zpn7uE#wG+cyf%^b;i&{o0z^t`^mg9HXMK0RgAG)J0J}f#LtDQjNW-T8OxeG$@V8Qo z3<$(?pvhZOiR?3{11qT9#fXoFziD#YK+s<(W_{*K)5ED2Txi3U>wM9P#G()0HWQAM z+#8KwK60E`wKdrD=dlgA<{iWOs2&#%q=9tM13GqT$@5%jI5Ec zORqZ{90{!x3BjqSYx3YbScFWM4q}}1`fpoZp09MSQu#EC`^e(Rs-x0Gbq(I$5fo2T zZ=Iepm*N2PKj%UvLVI!;(s_S|m=jb!^4zm!o#zUxN5PX(NCGGM9D6fiSA9_iIXMom ztS$Oz)F6cGht=m+h^Lm7?L&yHIag|oX9IaI1Td#_*_7}!-3BbVH|}x2Q-Q?JR4vrl ze(Y{o>>qmlC#MY7I`;RPnwTRrW|qAJK2oriY7X*ceHhS@eXzBr-Kub;&{wk_JTXOR z47aN&LvJ32s%R)*U+Al$K}~e-uEFkNt~zw!p>;lFI)@HRA<%BDdeSC-cRj9)*qyjd z4rOnDYQD{FgB2Uh4kBHkE%;^~X&X%<WNFlASse61M?>wP-~DHp@R>Ky3lG!p`J8@gJw=zfRb9 z044#=lu){t3*S3&=Om1F;eT_)dW6{L{A#fIGxH}e_=_%5o6~YCM4>0@8wYO$fTI7k z4|Rp!lr|e%vU85(FMyS9U00+mHkfyObpKaqR@Fk;Vu3AaoF@2-epqt5&8bc-`l^!v z@ZW#!s9oW0E~Ngg;(vdkQ}WPcRg0i3oOB|-EzHI&|EvLFvD(AR)EbN>c3}3BwA}9S;9Yk{N4$i`_`z;iKCISr{u?k5 zFkA?(Qe`9pl#$Nd+ydZ&k!t(Y?Be3w+~w@PzGAbE1QEv>_YO8Qxctlhf55_T89;Ss;$Y zT=ROdBSa&d$GR{J>l(X5Pv9#;baXN99{YQHz*sCS%oZV3_{GxUgv5pbcv7>q)iyJ0 z={GFP*f<&*`i6bA=dMR6mt}O*8A3*T2cKt_p@(|20nEnw4r+Y9hH8S2+Gec0tG^$; z9K{ZRorUH3`L9nrvG^NMO;t=P%a~pNpnx=}_}1U=y3rguo}Bz9kB~2nDxH}HSmr%H z|Kp86pKJznc34{r{MB0a|A!IpC8-|4Bzf$Bp+4)D0bg&qCpQ@>h5QOKBj>v!%74XA znYzYn9}qEsvVp(O?@8QPf||6E_k}BJFNUvQPc5?z4-buUeG0XQlQ3Tdpfl;@B=83y z&i^2)o2p;V0J>}4=k56xm%RFZsYhen(!${zc8$F_SFf5Bap~;=8YK@-=n5ZCw*(iw z7^TDJscv&;1SnP;Sd+;ZtgoH80W~?)b~dd7ANq-AHsP=c&}n7n&_kJ%Cz(J7$_;J zX=!Qb;@?A$r;wulCUK|*k9e+Q)&s_tj2=Y`*^;OGB&hH<^h`|wA>sXl z!Asa9)2fnO%d|pF9x-7EAu$EWyu9TQzdL`s&V%m?Gl-IbhFyeS3Ja1wea4Zwb8KvE zxSOvNDJ9O&FDWP}AR@v}N8LzxUI`oYAuwc&S4Ka%h4JzNLMKp~fYi=%z>2P)Z1y^Y zXXZ;w6BN@`;02V*rKZ1HL;e(ZF?)oKWt)>LTE8k?$R`i$6^=SMKsNGi#EwR^^kE5t zb=Bm50r3%wb`O5`*Z&XU*5%;+#i0jcnku|>@~#-j@I+nB2h#F|IgBsUf`10^%0eydciG&YccCBiR zFdUBHQ}UeEK5;03bOWj$4W2+3<)7O6=XszARiI<)*B++s?`k{PS?=_}M$M=4X=t{g z=f>Q;+h@$geh0;@*uc(M)04TLqFK2nLjerec6PVin7h4mgZi$aL~p@flQ!~5e0x|i z9la^G7}No56*H*27}EzdtY=UQ9>U~;NZFhW#Mj13L)q8c94#fsR=d-eQC@w0RF=>m zTZpV?Gnu`AGUHQVjzj(w9&xy|#W2fu@britSw8#3%x-0PY*5Nf%-0s05?oyuGLcbZ6l+XaGm!xT} zDkrBH#QH{;nhDL2`D%y`?d`=r%(m59juY+R8`NTieD26b*)1Rv`cs z?0s_HN>Al|mx9lUe(ZA3%A%IFlmi=*F1dL6%XR@ownr{`a;|TpGJE?IPGF~5zLY-R zg#<9B`}Q_Tu2vZs9hkpPAoiHZ`eLQffjK&}emAUzP@!+-0b!<-3v#F)WEyyT_Vo;7 zxVa?x6s?J$+|0}cuE6g-=9K@#fsWTko>*rROeIQGqn9(%Uvps0fB7MEcibV*TvbHS z9)suJ5T9>v!$(HF^-W6_e0AnK#&ru`oP|M-pFEfM%{JOhc^mj(drK>K7uv7uHWCJ- z5~}xoKc9RkgohkYB{^t7i+i(Vx*_a|giy4!)lu3`c6$Np_GAKVCgmk5jBWd=`G|8- zxiW`1-fi!1sHR%HLV>s=N`e~`+OjQ!++kkjsQ%$t@X+cVD?WF=8!B34HRWkosDst- z=g9Tps>YNl(z|Y(Rg9o7Pn4G*J^HvUwXtVb^KgMAF83-qeWQbSV*@^~tT3Kh0eUPr zacTFFDNmC6#C)cPJl1-Jx16d`VH2S->XpYLBI4`jp*1?|HsXlE6|(l6$M4&a>*EGD z#vrO#B7^ta>f2qhg%>FChv&sBX61}Osg`ZFC?Pzy8IBfD`!GwqM29e+oGwYT77Up9 z1>-`ra%Pa-BZL)%-;cdDp~zu$ZzA9U4YG<7f7`{A^3`eQr)PPY)cv2C_)X@Y$58gI z?CS31sH7Z{8KZF3rTtHb0&tcY* z*-%wxWv&;CN>?q*v7cuhq4E4V{ zi~kHlKf1JfJ$M4HoqYBHw+x^C)ybHL3_Abg_TT@EE#8lSyEXrn!biA&H}yH$GNp~i z01o26g4!AVq_of2M(F#ST@=&r3^4xB4g7cq>iS<`@sF;A=M=2Zc(9G&e01D6Lq!U zr_%8ws}Bng^@2<`=Tu=;0V7Y%@x%c7?Vfj1VeXcKnEC;L5kQ3LQW`L96(}5#LQ+Q6 zeq26G5;?S|X=&-<;bE)NAq_bK@So=b!j1)CBHq8JWMH78uLYpv?zayB-N5x;3hBNw zGRSAv{$6r9_)ow2B~~}-GiDv{pGml`)bDN}ot*S)o#p}b@eeluzybXh_R1JC`kkZ) zu0?}OJv~#6jmxiJqin~j_eNdct*mTsZwFIwdG-zsxv#v(ibTJkz5qPpeq0UQ-d1!) z_o8a|4`&bnGzG9|@h#qlcq^9`6TiZ)uH?_37tfvr@bsJcmBaMZ)S4QJc;$@YA?%R3 zm>YofWo3}=uGhdEvl@C9Ze&Iz7oKq02Kj$0FK)kfMVB(H$G4TreNCuthrb~ zOHF4tu^XBk!T%&9y+~ahc=I1(po3PC?N=|r2`1z%Lc*#sV2O>ubzL3x`d`kMcX_HATAvHz(oPa3+U)?{t z%=d)8kE+BF8{RWJ1Nd(?(hH42#QDxA&8owB05HxgDQTwWf~|K4?=7BZcb7d7VgN6@ z-q7GnMZE%J2i$kt8x3Mrg!4!WYB`95v4L)!8O5ZUni`-VAPEL>GdK6Nva)b+&=Tbj zpdyULHYa(lv}pXw*C%`Xd%zx>fZq}D?05uzDtEH{3Mg^D9kaHIiBMzS_f{>~jv|mY z(9{q;zp_&0qGWqsAA5oRpYnk~)z$FalQY{>-p@Jzf;NxqAAi$~HNyZ9@_$hBKTQG% z`7fqeLAb%+5brb2d)-9F; z{a%taeiH8ZCwmBo`Dxw;%=xn>epPp_6YL$@FP%7%&eRW(kHP0y*ZnenqCAXvfxWDh zeaRDQFsX&mik&zgc|~1H2o+a?_~v4IFMD(!8MCNcRhnPiwjNdBYCd{cc#_TV-ZfgJE|JSjz+Nl1IK9Axy9`|jkf(!XH-qteIB-P^l| zGZ%gJrN#=3hxZc=mvVV3>RAwXTyTIt2Pc92w7JdK$pFqSr{(O6gug@yKoG!fd;50#Ws z>c8sHsalV1#&f(~@M*@@sp6);LOavy`SuvA?FnYwNSL&7o~ebrp7IyYHo)v#jL z*7!!Jy{AWkaM0O$IiS;^q#SgG?-fi@@)*tPq@t__DVlM=QX9Qf%MB#sERizeijEa;13+KUrQv2)epoD9dX9?ZR-X0oPE#00 z#Re@(V1@z*8ZWRI7*;GaX`|^R2V;rp-TB_>z&m&P5>qK>dtmuoHpa_hzR>YG?jQoK zBM%dv6uaF@B>9a0&Jzz3P|k1+ly{;rzrL?S&DJm4^`(}L7_}ID%x?FMwvP>IS29^7 zG!2TS!KVQ42k;eHbMJi47(%0#*`v-L2gZz36wb$n+Y&QB+G%<{&mF7-Cy73EB-=HA z&I=M2Lj*i%D!t-<|2^1DKSFNF@&P%TJ;eRWcE|nRTj!iUeHNBGSR3-1c}f29-vQw- zXOt#8=6yC@N@)-Jn;B&T`n%OLnL>xd$vgyUZHd5Sgkc(-Rsj>QjyBpep$?y_x1q^7 zjSNgA#%WhU_nCPM)m4|@K2ko}DvFs3c1-+kQ|)C(`fpnemLIG!D&p1R%HFtJLuGJi zxA*lz>%=RQ-YTo-J#oqNuDgL$S!9hJE5z=6{C=cL6dmX_ZJ!rYHbCpaYgSef@+%Xm zwFIBL8<g z^Wkx&D6#dXK@*sNk#O>3c&Gi`^aS(b%4PjKj(j;okiYQh9=$eIHQ+taqb_gbf!%Yhk>JpZ>Pj(4!plO%`3|+ z$g;i8>Hm0r@PS-ga_;B61SC>x`rEr)B7sublh?hWZS)%8A&OD?+*+%S<1(U#eJEiM zlm4OlZNVPKptNW=Qu~_(?v3h zSG=+_ZjKQTG#}?7Ly|&HZe(9|=Fa3M(B6?gxw9SQXd+d+Ru*_QZpIdxK7Dd*U44N= zZU~R~N-VuOmaQaZ?UthB52+q!5aLJe-%uo)=mRd#g!Fm-JgVB6wFW5Q69wDxeFHnU zdnf$Iex688-){Gz%OTdkF2Omkg=Zj5I z3LD0<<)0@%XSo9J&8rX3?#Bu73tqVv!K6SVWaV+R@oY9TjP4oMK06(aGr_k&s@JeN z1`bFTfcC~WGVhY{i|Am9lB&gjf$j5Vj9N&s6{KGNO%lWjP5SD6a!5Mm9s4&Vi2$xx z0=BG>JmfDnVe=VBiRffv7zeNCZ<;W&MF!UHV3A{i(O-49Gn+dwo@{~ANua;rY4i}E z!&tD?>AcSYmwm7mVE{K2z=!Wc#{uy4@~bB;-meqk5C4S9osk$F-H-(B03FA(uglZ1 zPu4lu_1eaA5lHd(j#K=WmS&)OUi=S?IoPY(_dL{3O!(Dpv47Ycdd^W2`@G!8^HQ`Y zs$_HnFyQDQl*-TFZ!b?jj(cwC!N~y(XQt|ym=Q9Bc-59{0i6#3O)-{`;C*-q&@;O| zapTH@V*g;HTjHKxM&FN1{s31Z45{53sVD{*77?!@thgVD2<-_X-2fSL-1aa(R}S=E z0Nntu$HxQL)-u0-tqRk2;$|0Lm5E$=QVcLl02~E~5hSwu`BoL`*68T+@-hXt-VV?S zvNa-tz02e5W#Eym310%>`F{bsGP!#|4IlK>#19~>Mw*WMUI23Bn185oVCo*l0R0#M zog^BsTvoD+tpJMy0xGaJwzfyz8{I76x))1~rDfA4eZv)*5KuK5+vr^R-!;RkF z+9>F+WGdYJ8DfW`vEdE6xN zQ3$v-A+PO+>lHO1i6ypt#U?Myi_Uo4XpMC?+ldjC=>>eF;U5*_ith|Ngq&|Iy z52*6P;g&A4^t#x(5jP(H7q0*Kj+Ib;15+rr#HIM!e>ul~Ts4b|Xt{Utu3!9bYV9DG zfp@? zi;~YB)mDdhZtse5=YCZ9poVBMQ3bksw{P*XzfKQ5N!kUBKvGn=uv?!^mk40J!eJR} z-Ub$gOxZc!hL_bxbSF0(Z_<-bxLfv*aR+c^g)w1mbwguat#jM#vArj zI1EQO7l~ozI?3P11|6Psw9!zZeuD^3mg{5`vqrQwU29z`C*@2n$cuh8)bB;YPX3RYhMBN9~DGIDw7YL zCQZRdr`OP>V1LR1AgIoI3zFf9%QgAVYMkk#WEO2f$g98Pl`H`PM33^hdJRDwZe z{y^cB;J!t-3QfWnspY`wP?|V&BCGGkaH0N|l;sT!f`&TSgk7-u&_B98mTUS=SG zi5m2*_gyN!PfJK>Twc-=Qk`bZHc%7JM+dL~R2t)?>{4q7E8O3@VvXJMsDh zb*$7)jok(x`A=qh(B$blg*MV`3W(0PfdgX_+o>>7@v-J~C9+pEr4++FxI!=VPci}N z7j>8)^wVVIGMU1-Z_rM-3*?gLd)QW)CA;>-hGnzk%T}`8g0b4_WDe2$AJkM}htDZB zUOh#xihRXYhy~z$jvh6OhOQ)VpJcl#y*^oDW8GU17StZbeJCM5Fu_3-MQjlZHFh7+ zXxN@K)^RACR1%=$SUz=$7W5XObNl41i`dYp3%KZkvc{9&xu|>C`RyHhk)e1YJ4aDJ z*{tY_a1qm7%s*Vul)@Apr(BtTVr~AgL%EJnxb_rHDcCLRdKn^h4SeCah<{*WdraEJ zeSQzmG(Sk0>&9eiu=Qnm9bKLw1^Q2}bf zIDj)}Bz2-EA?oUq!hVnscZt`Yat~}=u1bYW_N94oriQHjbbD|Mgv{pZn4do+azrI; zw&KM2uQPBa7`Q6nwrbX7vIDe}aEO+~C)OOMMggWMve#c+@%?J#{DGZX*bU1V-*wfL;{ZsqeUeBuC=l*D%jHJdL%+A}It8`DO)ZfG0`bAT6E^nW!%jO?+1 zx18*0ERi>fiui#ac>PfCHT(Me3*dL+Sc%JZ;dTxkQslai9q?)(qcvGOaRq1PJ`FZL zn#1dwwEo!h*bB??_`T~GMK`bv`$*y=An)_QUY;HiOw|X*Uqc!TaVB~vcZ(^T2xV3y zg7H%laI(>N)g=`f_+I($Bw0g7$nTI1`06FK&Eg&MxnKUh}h+TKLzm%0WEu%YUhNXoD|*2-yBbW>)16V zzak5!i8(7SB#MbCV~l@X!|C5pmo*e$RpSYk9St{>7+EuH+HXy|X9)D26iPKN zU5tGz|AE?I2>nR+QMT6dvzrDkoTwb4*r86(G*5)aG&FfFS86uwQ}K23%LkJy-1P~k zA0mTys_`DeSHCeh;0`~6Uab0dO3Cm*^adf()yQZ2oAEKxxCl(wih9%N-8eEnjx=r3 zmXjh&-L)$^P5@=o!ZO@Y@2Yw#yq5dX{^9%t?t#j(F3k=6E>we@bKziunE(h6P2xd< zdf2u5z%c^S*cTubvIPnNfrzxs57W>a{RuY@dw|XFioApEO>HsAwNTS2v&HgO?Szz| zrbIP0f~efP6Zuun##y*wHs(E)8*#&f$Cxvb6cn#N0B@9NRwNP#j1v5mX9y?6z~LV{Zsl%Ib^lnTd&POj|UnygJO(C|aMEhwS|Fu94FW5y#2WgSWW#D)60Gy>;^fdQiav_rW;}{K48MN z$1Y}VuO+x`&Gn-5OSvBnl^eGuG_0{+bHz8~1({1m&mU|Y&TJ_-?&}sg^#n4{&}}Mb zHD}svYmcVOG7?7+JeGX6+X)!V+-DVZt$QNXnEo0Xmay!#87!H&AXh~$xwR3ljMhHM zXYfc^gX;7Vgd3`Hh+C~Wnxs}#r%lkrCS|R zj-G4pW(_^u6P`YbRc@Z#D6{Pko?AFE$6=KIPK4hx;pS?Vp~uyq)!R1%@&o7hi;GqH zpgZ1RBX=L;SlD~6?rkY&Zf0o&rOXv*DLl>v=a)UGqT<>RSLmL9a%&Df{6I}gRP>{U zi>r(L@Gy5-X{kXq&mSiB+qk{SE>w>Uv02-qR+eZr*tSwrUEUDxVdkH_`7mFCV&9Wu z(Aiap1P1RYauTH@I{*Le`P_4vivxQfYjk}pE03S*we)RnGmgHoK*YoYtAjH0%gwtM zWy#DplLG23Fz{G@d1;N|;naEeW6~u0-S*~fpR1yN{PgeG_$7Peg?7I6gV@u`C#BcYOTr9Y;cSnZ$Nuc}IU+2%>8nrcQb=c~{ z92~idk;z+RhHaz$nyatA{>of@(Qnebqen9==Ik}hoLpvn=H6@*ecru!-}hY3RQJ4; zwRP6}|8wS=!p};A2OyM@(8+W*ZS_r`sI_I2=9F9Q%`1P|`>-u;R>{og;ogTUOr-eO zn(Ox8fBkh=ENLM;``StZ0I2c>&6Hu zC>Uqp+%e*?py7-Fhxmk&IoMh!pcI!klZZl1_q6Xc9P@1~Oe`sEGjY@!90D976C{}) zGEM5kb!W-}UEv9mDL4<%Q3!BI<7nBPy7Ocaj=bm4K-dE@4wnjXEEN&hk;-J$jQh?N u7OoRKjRo6v%p>`5L;;5Yeh)l||8Y*`(;cQ)MZn8#7(8A5T-G@yGywo3Sc&KW diff --git a/profiler/advisor/img/cluster_1.png b/profiler/advisor/img/cluster_1.png index bef236b22cc5fcb1645f0acf32b16586080124b8..33934cb072a358c5a83468ac2d46541e8d723d64 100644 GIT binary patch literal 21351 zcmd423sjP6+cwpJf9JkI0P%+sg5HvZG-pX=7G+j#8g;j`=3eMMil?jM)F`C4~n->WN?y03pEo%Q;4 zT|L!gPIvNU)S;7y)~##E`)=jpSGx0W>k}c{q?1F%@ap& zVXWT%>-U0gTjBR6ys&uBbtQK`BDs2kR!03Z0;4Hq#MwgzOaB^>*%UL9md*f+CDKZ2 ztA+#8GXJ47X5Bi;QS2A1-!K1Tvtadm-Sg|+E^7x?KQ`#C9ekHAAR!g~A{9rAOr^QY zuU4HZ`;WCd=IOPvpVp2}Tm0Slf6(9mi6*Lr(H;@m7`kD93NWj_48i5gm?Kf`O+)r; z_UG=;l&bFptY1BGO4vo0dxo2?9>4Ft8^F=eOkO>{Zorf6qWfUxkk>T7%KK(w&OBBG z$(c{LwNq&9k^ktVy2~RX>DP9AsYEa z#h$BX|MItCW2kWVUC%0ZT^lU;PCr%tn|$T{T>rQL|3uQY?eP;uKWjRQ98=YyRd@Ff z;ortsyA?MVnA347{*Rw%nEkvC@gIp*06uyU7f}yNx2g;NcfV2L>exl0fxu+82MXV-zS;DlSDiAGab65mIzB?y}o)2GOpfdh$97 z??D?W95>*wyC(4}=T88H@0QHJE9x1n2&hlC_F!5NCGfZq;v4nS?;0DsDw^9ncSYj; zY*JdeY9Ro0H-I$Z4sx^Q#>(d_ae<2~S6A}ZL9bf7Zw<}=dv|4b19ZK^uAJD53rYJ4 zapHJ9$AE<+3oI9!-ueKQf64CdKl76tOwIg`MarSs`)r=@?}!2zg#mUmtW!zX;;*@T zV?n6Np+)C;Us1uQUN6J|GNqLELWg5Bj_n}#gvzt#-b7G(EL^x$(d#&jK{<6I+f}XG zS~zYbH2moIfvfK165Ja^Ng{ znVj~q;=A!Da{PBJAG1t=e*W%?{4W(xMLf9|qjHx#g60D#L%x7}z|2;+t7TZ z?4z&QSD#!1*<-Uno(kkp?C5a)tE3lhVI*c~Cea2tGq?V2V^m{1`h_)tZEMkh-u1f6 zaYhBa%`91tB!2QEFEg9+x|KB|xKHTq&Hd1;xKKBa`s0afHx+XtSN$?^*m2@lnnd|> z5i5Bqnsb!4Spf|Zp-~@x7qR1o?Dg%Kuos<#t=ggCr^%0PJ}PK9r-XWVBo^|VQw7Mu-nf-%LZbi@|9>|+y32o2$=Bgnw!8N zeLX~&zgIE|{@F>#|N4Kzhpsxo|0+KG_qKej`)=F*s~oi`wF8P$jgNc^8!zdtnk9{j z6$~Yes3(ufnn$$G$5F;Q|#&NYT!C6Q7fWg%Lz zKPc2+TytQ<#9$alRdYhyi<*IAR6AAi@Y1DW(=z{8IN(qKnuiX#UO6%tZ8 ztC4%%y01A(Wi77#PKb7;iPr7|R6J3An9l3S11c|&|kN0!PkOPP#gI+va%? zLNVjgYnHyC;H`D=3pZFR#-W7&x^n@LKmX|x%lvcbod5gayZ?Sc{~eS6f8FHe@4M+* zK_O|Ls8x$oSy0#FGuUKmo zKe~dlCXOQ1BmOK$5X#;174k=2#6XDQBh77^lMJqv7pPMMB$Dz1hqcE|{8jsle zqJ3XEPl%$WT8Fh4H=B~!uWe?GEc?qf<2S0B;NuH!>h}gL%Z1Jm zWTk3yZaHk#pYbywPQ_N=Vgws#w)i`M3fpb3-17w z<{9++ckoa@a2yK)faB_^MH%zHr4NS1-iESVm--}onU@G;q145|^x|2n<{uXtMFY}pW|_+KnY_7-_NeIuP{*ix zo(KtEwP&5z$R-~w$uJvjr<^*S|7iIL_KduJk=Vi`M}_f6JNoU?F(r5|($b z;}}R}2DD?3AH9QtJywlWuFRg5f4Ii!+Gg~vqWvcYLh-A7STzhqW03Pxva|I zOHQvqH7b6OXd#1&6@hZpUOrWv3qGR3DVU~w31Cnk%15FRu4b#|ysviAt&dZY`-_!C z`HF8|G7$Dr+zzjme_7Y9EE4s4p9w8QA2pLVrhtm~Xh(eI`i!?;?VYj{T8L~{uPJP< z!DGn8q6tU(c=_I!xMTSoM*B8E=)LD2EcNnV6Wz)g(Kh{Vd1PL5(NQdjv=yl2MlZbH zzB=;tiuqeQSfm0GI}k`1KU!Y94Lyr2MO1W@1^9rhHQ&qyJxJF_eMoxoVAqhf`v?Lm zWwxMn-CM^3whg0hXfz_;nNV;e;&|~2vJIZTsQ){AT+Q;#yP5|~E{D&}^bb`eeC?Yl z?Q7-b3k)))@9-lpq7&9$H>cv<6@c|d+M-7BwB|$F_xCzm6#PyPp+PlDJVf z*gSJdTGOrU)4fkVk=Yy9mUxeoGE@4Bpzk)}4x5}29oJlbIE3HUjvC*U8vO0j(vYn; zsR0dw&(&15W=gz$u5ymmgZRjWb4wfLg*03}Mfgs59JyZZo^T~NQW%svGqbHd3Ulp= zYHY&m)x2E;a^^2PmSut!ChfOqP8M;WN3vFU97{@>oJ^iT zexD;HJYnK_3)nK?b2SwgL~ty;Mj9eLb8SN}VgaGUP;!~hYngLSSck&6UM#|b`(1#W z>F%W)S$V&4-cgD6#Kf671r5L{3!vqX`Thk6V6jub&y@OBi3DaSnCf+jpf+Ft_P}j* z1z%oFc0)HHl^Ri%4`?2loLurU)FNTk2mOdNbPDpIO0wyQ7ueg4Hv4ZRAr!+Geo@>n0bwA#J}S|u?DUS#WazXdayy$O97;!(zI&P z8rg_fr^%U6)Hoq+=6I^Yku-;t2tc2*y3lQj^c|_DiK9CTO%s_rT1^utQmq;^3@CAq zs9fw94f}X};xrxN&L-q2{Pj{s?w?rH?5;J7CLlYaOcNP9GE5W4c2t=rj_+7CU z|4)vrTgd<+Mi82@C_&VlPL#k;PHE+YrS7?;^5p%M6#bXy`!yE#KJME#{+%YhHo=`P zy|(!~!+LGPJJWh?BG-EX`ee$hblF@LD+8tIZ{NA2A3*#Sfc7yk)0q8{yOu`0EOX%2 zcv4+H)O+f=erWVO<@%we%Iph+3El9^byAx)vqsfqtz@$8;AFXY()!ugVuEV_C)*Ri zXqOLIPm#-q=broFC!YBrg56)}h~1B;>N9Wy8N8uCG6)%@Cq^#o*V?BBzYffpNPc}W zBScpEBrRUYMSnbKLfi{**<`k8=JLb8>3uvS2ln?|*aheG^{g5^{#j=Dnd!%mX2a%# zayz&$4L(V2@umO!f&2g9C3;zY3ae#Uqs9R-V^Mgg!H>}_Gj#GFw^Ybv6A)cKBu)dcU1qG#IIM^t@fei_gmM#`jkaZ zlt4R&2r8;83j4QdaUwwi>BI*T_02kEhQcZ#uDc8MdfaU26 zpXs#ITO>*s?WV~&BKlwnivmDgwKWic(ls#q&qj2}l!DcibIlVdopsh)eNOx~;3Iw zv6}pwQnH}fs7z^j83fqp;_$7q=2KFi`%R`)=S#!}z2HUXW7nbGz49FkrMr53r(0i^ zWX|z-hD7T^Edcf(DBA}1Y?Pl@a_5L<4H72E&4N_72KK39oWL-QliuHP^yT^v12JN% z9o^}(S@4@_0>~feV@{t&Z7{^GzoLDn{pNFSZgNKV=&lAci$@txBz)d8ry;^A4=;v; zG^$rz?D{YOAm|r-PR4R~>g>B400J6CocO8g7;lWcOQj@kPo8&fFw^dQ$>(j)l+qpP z6vn4y@A~xNaJAkpx+F&0IOf=6IrXLV(oevLB>}YlUD|J++ZF{tcP+=z0%)qepOCd` zif-pX76n;fCSpo+4c;Q!YpY2=3F!wr8M zry~&Kx_9ju|5x;c*~VUJWZRBIriHzv?YiyV{fwv6l0Amqjpc;PJNXo4(1QT|o4Zi8 zy3BADSMQy4O6N3qj-DmB!(xsp3FD*V@4>%Pb`YB&s zO;bNN);0M5_+m!4(p^5`yYc%qL$ir$A=De2mnqeSauoLKZ`ZCK(F(*`WLW^|a}w@4 zmMP`wd|>V0zARs>5wq|NP+@>}G0VDLrwLVe#83VzXNi%i>hqeQmuuZP!TxUpd8(7vc*V{8D5?=4f2uXbXbH>DCrZ-kAErGVLDiy2ltR0B@ct|dC6=MM| z08P)1FZZknJipnxW|D>f>n$0j39n5vOcG!3&L~ZMy)VNg>9sv0@l9)2`3Wp)RfW+B zEiK5cR8?zSGAdmqOuq0J$>on9|K!d5kukL9B;ptUFiH3?I+A1_$F85AzTlx+8eWrH zOz(WeZJexKnRIrZ+_7}%8^cdVP#yyG+NSQ<>9tMY@ziVkaL3nUmh$q;4BG||WDMd& zeFiGx8_=Xajqf(O2fuQ31Y@6OsWw8nyR7n3m!2O+fKLPoA*RTI?w3h#Y9P^)=s12`4Cu(%@ zIEhtT5VL+Jt{L_r)VcNcSdD0+pbua1;Z*i?J((XV2WT6EINHUK)Y>$4wbTA-r=#J` zcg1Zbv}A3&QUStr6g@Bw(~s~e>u7`#P(I&io~rUn&cND(Y3ly(OTh6O&Z#>46i1WW z#M1YY*-=#e_3`;r^}LP*53@>n#?CIWbrVY{O+LZS`*0)jijwQy%1sF&dh~I(Txn3BoEMri@(S@tU=W+C6a0;;qIp$@kPq;BrMDNL%cDGQvFG14uFc*sj zk(ZO2;fRe<^SSn?Wj3#A@5}2ax?#4?4?0BG&^hpwU*Oy~r6U+CW>b7@!TJHw9)V6+2##Nr*~!jCGc|l{c5#`oe2|^cTr(6Lo?N3vG(AN@dx280(jKCiwv#b70O> z(NS7f7-+o|oqAg8VDgNXlFGU`a3$Wt6xXdxKpA)(o=&=x2sa@+mf!G>n;1jvyi!{I z*{F>+1N{TX7fVk!83-sVH{|8Yf%wZ{#J^}2t~M6%xSd*3nG(iFQE_k_ql-6Fu8d7(*n4!r z^j&Ro&xa!xtmR$hMq5>TKuaZG?6e#&GA<@a+w)X5c(+qL^XhG0^0VA`qQ&1j7aR1|dNwEea%XND%J`%&g=YSh zDbLM?ACjASxE1qrP*znqx+!h1yoci#t$FL%M;W?3Q}4)(osS7z$+8crwFBMgSq=^o z?BV1qXmjehj7D6*9B&j!W!k{+3Ofwa-? zkqJ05WT&y}0;_emASOb-%r9OTltVR1i713PPYE|CGhbMj-GB_#VtS1oB==}gDZ4XI z8o;197CUwMDX$es>s3WdKMTB{lCrpNd5{D2wxaR$m9ftH!R#}a?ADL5tLwA+h~A9L z?N!S5CXa04yj$KWH28ji*pa#P592Yb#YKGLp}JUykoI33!9A9_0E zF!;6}=A_mk!CP!S6%2>Ga18Wm>l{$>mY%0c+ZtHqw1xCK2kb!gMCNnLogoI$duQ zbqKO=gm2Al-I`Z+Gp!4ez^Mh(xK~R_$FU{T!6OYXs28dot^F}21q3QA0rY*(9=?vZ z24Z=+77k=9BD6YcA3v$h;aAQgW*DwaK1(ME!GmwaPGNW?eagP`0Rx0~wRD|6pQlEO z%8FaLY0NRdP>3~yX-#9?J`y76OMaf?^^UrRWs}y#^qhe6WZx`CK>cWcV43g*of*)} z2(N!ADF{StCwFX>R{3$z^pS~$jKl=14AtWVYa|F)S9r)KwP!R%9wiB?^(nz`&q`uzjIb>2ik2{oo(=Q#LcLpW>s$&-qD95h4qIeEpX1#Z9-5CcplWbf3LIv97=ew z5_VTeinWFaj)DV1Tk#7UaFG6y@KABo#W<6{VidGMxHqks7dbN9;(DsJ=HhT{WDS7*^78>*8TtWyxAYD5tx7{@vn+l)T&sE_T>cS8Yk zYH4eoJ2fV$zE7D#-!c$LxDidsGmpHc^I18B8@#C?%I9O#y)DW;T>bfkk*{uBC8cd z{ch!uUkr&rFlpDGsV-ry1yO9w5v?+%f_P9C-cTP!2VOz=FvYN_WV7MAB~2POs_aVA z^y$0gtFvA6l6ZfGItM{_iD^EAE%=o#?{yLJWVYX1t77 zk8_@`9c?=$Ni(+T<8`(J!6Rv0Y}LgR`GVSWn8Jc&U<-~hJtU85@-Vh}c(Ast9MdH0 zIgST(@H$)K7&n!RvczU@vtmCv6&X%wzc>{${*|x$OC!kc zl9cELb4YXFd55#pKkiYE8iEq9pEB7yViU!W*jr-b&|DO|G#E21gq(PQaXHXPJlSJQ zLhW%SH*LZ22@SDfO2sYZeM9z_#u*H(V{W9|+kmC%B9CSRg)+L^mgAkA;~|RL7$rGB znoOAWwaitRm7$v+g}6$%)m!qe)V2FhwOtu08{mi>W$z5LUZ^MW=>(p{W+j9*4KI^C zN)r#XmFcE&1XUZ=lWP-8&FUPu#%X&~$ZRtl$D2QHk3m3}8VqUJYxc@U3|s;m-!u$^ z6!alVL$j0xrGfignd8L|NefKKwY(UX@e?_1V%T!z;TGXQDTCg5l5BCd0~H5)mQ>I~ z93hoS+>%y`m}74!9SEOt8{P=a7zFKZ8^hGVu5??mfOX5=3($hVD}&zs-{;Rcm!L}$ z%vu&+_ZP~=+2=1BO6u({8KN3MjzQon-OcUj>9zoMXXa$C-SY?@dMvtbWDtK+D+i+a z9=!tNzBqa?SiMEODaYZqCPF%xhRb{>yK*`Q%&sB_v@pm>hj0RzgB{HAtQ4c3@3SL1 zjD=llTtQO8gs2v35M_nkCh$0(eP8e;3Z&b39-jpTIt$4X8S zH}H8qJi2TK5xtZUo|lVwt>Vy9JC8(Chp++MAaz)#z*y4}5o$jEg}r(rTmGjFSLiQ8 zp00JB^F8RV1J#8Wj{EG~TT6)4%rlAFM@(*KlDXo#9epTZ{j{^KJ&>o`zEAC~K-TPZ1 zQ6veo9q5}SpCFzar*b3m1ktkPsIrZoq2M-tappm8C#Lm5tm%e6&83B!n!(=V%0LTa z-#F9O%oZcayI2YVEY<_lrs?3;rY+4X^^^+l&-hV zG2Q#zQ92%1gSf{i9FQIdTlK&XSL94-=6NhZB(?1>CHjuM#;Eqt*{GfLB)VY2X~PUT z;ZH_?vd0qGsuhem@qi;9t(8X!VymlmqjBe+4jxaxy*Mr1wLiz%h;J;IV|aTv=iali zpK3jmThYEh*oL}WV*9X-7@22%cQEQCyDA9aAC1|O?{S{t???L>*OyUi!u=ntY)~&{Khk(2mm?dSJNDu z?1kdl^%VYg!FPIpkpF`dHXiqyU`@}|>{@kE9`DUF-nGOIQJjrmobf+xnp7EPZtRT* z7yv5CCxhqU=g%OcH*R2%Q>wy?IKca+s&o77DBbPjX0iAeP|i`Zh6~u7r06=QsyHzgZ=~jgH|ued?0Wk5!D}-<(P(OgP%{F8f&})gf_Mf$@}~HZ?5n zUGa3U0H(?z0PXX-@Pngr7v3wZM{eUPluqIYzv7)hn!TeR`k^9}CllFk8qr~hPC2uN zp{0mV7$OX;VyI*}=p9Zoz_ho{A)I-*>YxHB9`rB)`$qywgRUoIFY7?$vITQgsA)sg z)E8OF;357=6i`ltojF($M7T>X^=EiTrlKMTyeqVGQDs)t^9GQU-FqC!wJjQN^~8W> zN293g``U6~zB$FH#Gs@gxugz1X5_uIa-`i{X@hMzP#>D~e8dU7_nF`Tv1Lx{L2kN) z*%09fHsy8@$rVUP<{l9Zi8ew>(w_Q^B*HL8;IXiNS!@F!YCwQ>dMALYbAMQxu9eo* z?;&v-EFzAKxzs)=s@NjFa7%H#jS`b^W#CLpJwra2J;RJg+1-qpae5|5OzgbNEFGMP z%A&M7RDfWZXhcYna>E#?f9h;)l$u#DJ!to!NZE_>Z3>S(=bO<%MIwYY6iT{}UGh~v4hx}*tbHeUG;cDrL4$I843FGu0#hv(gk z_o)2G9h* zfYd>m;4PG$;-pZ>#yC(Ozh3G-H|^()c3vpX)v@wHUY%*aGvxe1MriX*gcV=}{NyxRJ15$tJ_0Euz~R;lZdCsqM_z z9cG(euuWaAqXQ1;%18zK%*cf%l)ibp`pTiamfcBZxcqj$Sq`f+u7!yAZeY{Mgh9$&Nxf&Ve2%xT13|b%Y1vhCp~dQ+|ekLJ&H*F^tih zg6SX}-Nvh;jQnEJ-q!NSmJxefJesUIi7mLa;f&j&N4`r~rnDg8#+D4PfD1!E{p=8a zI`9HK3_H#JXO>->Z*g?$I8R8JbK*m}zX7&E-Z$qE>#@Y!(IJapJiRt-qy!w!@PA7? z66=`q0B7 z(>i5Se`5rN@umdl9I^s|aspcD7wg*}7|FRY*Q^SgO_g>d+5#q%qxn!3bj#t0myl^GtpaCPJf?H1|94J9#W#)n??rQB3a zG5mzP)P{8bbocup7hVM5MmeU35HS{L)3*`fqinb3mHtfcGt_3ksw+-F6pH_$h(V~w6!bS75ZO^kJ3ziPUvO<6EQHXH zzAaQf>|Z`FTdE96k!?Yg0&x*oohl-ty$iQ%wH6TWVJ1Dt`r5O=2jHPXL)51JWQ-rf zQCP zEZ{xNVWv-sArYcq0S*Z@l=5tq2YGnZqNnt$v4C2&F?ScNZipO^;tSG9Dm7O(;5-=e z7`8{9u&v-YMwvKP+N{*6MiZ2hw%Ui=Gt!Y1B!u#u-}P7$pQ=FQAVT9PF2Pq?Y_>FY z4xZmIPAZ&_DDCPTAqS47me|%JEvEO#j?`y|deG_;&*Sk2lMsVOpb^8`W#akJL3!tK zSI?qleDB1eH--$6^jzjlyE?tDQVFYJ70NH(^0M_WAAF&)`T9ypKlOQIS;Bk?!(9H` za+~co%MM<8wpSWwl(xbScWVki_pD`X6f9^gQD(>%?e8q@u=W+~tkAJD@y2|!+I`nA zd&>I$?2z}kSW!o4xLo73)8qF`PLj4Nhw{pk->;v>KhA4%jAzH8e3A!>(u#x+DFLnI zEvN&t0_L+Gi*b!P;w79d=Rae_#x)vtbaOzseN1) zTNWj~FOOPGI2b+_l<~A01s?y7{xKj0?637wAbbJ){eohg&E!;PZ~93$jw`qNmKO-= z(lVZ41-#+6=x%VLuD}Y-tA98BP{cXEVXrENXGYqX#HO7j_m zMC<|JZcLJf+bQr$nmF97%W0~;!uFWDpzWW09<}}&KAbZ?PLRSZ>E0=bg-*T2WM=NG zT*a-KB!eH+z4Rk>CmrJ>+|$>TFT= ziibD)(0jShZw>y9QO~i1+%ab8NN}guLFOJh6ztwQ9U7Q7;xd(3*OYc(f;4gqWV2pT zkuMSl260-!WgFut@S|Z~n`zspPcVi{<#P5)a-eT+1qlee zVQf=>Yx;Q{T&DbIN*-=M$+}n`?P^EO=sdA0zc0WNhubZv$)0In$sLX3?jG0wErhX1 zOr}=YDDka#_BVbovck%8GW z`tYqSsFfv9yY@w(wP$F z0Nm(v+IUm-7qdYKL=WkYR)LX{^s^%`dwdA#a5H2g4~hC9T0~~PQ?(|Fj^kI#;`dPR zS8!jH{@w~R$q>5>-c#pwm6YrcN270fwa2NcHhX0u5R9WtGsQW1>JF@-7f6d`jmWb+ zJEBf~BoYjGN;Bp+Jg?ex4g43#NnQkBtd$`BAIjmBKQ6A|X`^cAw)45W#nXMED_A`W ztM>%fe`!4^tWq04x$XD%-K6_D87Smz21CqWj|cM(AYQt5103?a&t7TJ6*<0R8Uhb| zU9dP~{d<>ndhhzKH(}{1`a&L05lzi+N)PyAP7x+U$W@d9nPs26Y?AG{@+y_8nk!2& zRCS*@C(^zb)#K7IGVvomHWo{%Rq z=!EJAhP$5%XY_639j_BfW9Hi4a<*yr+YZ3a)vq8TBIsgMUvi3HqxxaqQ@Tcp|3JJ5 zbp=fRaulo22fW!uuNqKYcd*t>?9Z6{g1tlY9CjJ_2+*vXiG&$|Y4a>n2EKJdF_QeX zk*~?6wqW42e%Xjh_R-@&tvZNM;9+W;vEU{*>k^&;c8Ekumufqe&hk3}KIefqCUk|? zZti@WQVgl18GI=Tx7)?pMUQgP!A3W*%6Yc`phuIw^n;)(IrmAUOKoxw;ua`xV@!mI zZG<-fkB!qCrIAVJ>MSr6 z=4hjB$wNOS`|N!ABUs83xWvtuj$q>TZX%C>pUHOS1}-Z9*e;Yy9^zVwb;{D*pd_c3 zLaG|ch^ZeXwUztu>SM>p21c@_?zJZ(hpSfDeWS|8;2xK~^rRHRb5}I1r01c7jWIXH z%JE(D^Qa37<{ncxycD7BbEs8aS=inRwo*9VRCcxdOnWp$-q01UJUI`rI0BXvW=T{i z^0lRGj7T0Enk_$(jZXu4tS@=!FT3V_&{_cxSgm~E1H_|nnCq3;n{Y{529nCzX1P}O zz~StQ$FYNpCYXc=Gmo8*it6gKn8f}mQ*n*W*MV-Tl$7~NE9AK5Hz|}Jvg|CToOZxu z{D{u#wa4cmgz}a)IeU;Zz9Q@HoX9=1i@Tr43*Kz7Yg z7%||Gcdd`ywQy!@68#OiML46lsfZ{>7=@oN{W-^-)CnAiVS5_uL8RXlQp3IKAO>kV zyH8-Eye}hRvgVL_68nbM7NvtD=DP~Gl3*rmI|S%hlpZ$AxELo_D(nbE%iQ?R)lz4? zQ+Eb4rOPYh75Dq?NcR1H{cPoAh)KwAh@r{*rf4{!m1OPy{29*OUlFg<1&vd$Tm~wB zw2}RsLu=QV0-S;}P^^KLjD&8mkDl!0#9paznoSH*RG#TBe@o`x2(qb4h>cDNBgiJM z(W2eW8|D~M?c}{+y}e1G79i1YE;Uj7CXRiioL%^sr5=cy!PDJ#M^ex1DV*2-lv`jo z)t8hHkx$bo4Ytjsx>{RynR<-qCmuc>b(s{96*yo7&Ji2)YC`-i70c#W@!0k}Qylah z_sL^1JVLu$RpXBxJC?+R78E5xA$l2Yfmz%I2P|WP8n6MfRd9u>MEh1gZsA79*mXY8 zhLA0VsVnFp<`!CzmZESqiF$8tmJi05Mb#3*L9~Y)Vtp(qj{Sw@V!{T0y#bw2nYa6$bWFd8>F7ko~etI&^e5TVqu5M16_g76U(B{wUcryYlwlz5C# z?!VGxdTGl3kjxX?_k7%jD}NLC9B^%8m+ei0|> zTx6Nx_`ZPr(y|6|<^cZaHh%PHesqi-Dt7X}@3g3s_gL=Y*?5RA_$4|#V) zla#b#poC8$^^-~|ZXo>;>81@0hK$$CQ4p4sR3AM<6gJDla5E0m&T3(k{dG<`FFc6k@wg8lNgz})M2;s*^Mq8)c zI??hbhRBDI{^oXI3f+DxzngTva_mTOBR87ym!a-2tJdERfNT(oi}TC1<7rGmBm-8$ z9MRNkC}ihcL3Fnxx}WNb`(e~VC$t3<0lQRb+?8Avf~^VY1v8>8GK0O{(M@#5 zz`~Z4P;2G8V#!L#c3Lj|VT`nyPbCM1;#`}jn(krq{a6bt!tB7f?BCp4Rx3DB$+WR; zSvJ6h=cf9c4)HH2)^?yq{$w$5LCQ6E5EWVTbVDSU*I#BH1{1Gzg&D<^*t`3v^>w>oSSNpFAPX4aDVG! z8SYus+pqx=C)lnu!H<+utV9ii%sz1m(_i6&r3&t~{XQ)m-A#{KACwTFa~#SNFPKwa z#!=Qa?K|Vp{w0E92ixu3dI_`7Uhl0=(dE{Ae2D-C3469S6Y?B?*U2fkyNerZZ+YT# z7BvSQe5V1i?Dz0;O{1GX9 z1@LMB8O=Ovgl!)P0>MK`z3KE1h-dPKH0~wDvwGYU^dx>3-5rqPr_=1#3If=sx9x*v zHYE~vs7Q@3$1IE%tHN3)WD1@s@NIt20Y5!QC5Z>lYyIH!*P*D_i8 zF-ZN$QvU__73Sf!it%ZJ!uYC9(D^=u|*dFiuruR&rG(Y^kCnBO;qkPFtppzuupv3+g{d1$X_TO z#*|6k@=Y>aC~$fS|=IkBa6NGm=ks6#!IJR|nZS@z?_{-kH8 zBev~GFh05A0>XN(znWcTb!tRfT&FsU+B$C-gS{06M$;Q%HL^OJ)Lu8A0rCD32Y5ah zhb>t|R2O2U?gLOD?u@Cuj?|{;ExE_+v+Q6?mq+bdQcu(7BMG%tv4kFFs>;QelvTIzGYVmZ}d_o#-EMZ+Zef<|dnH zg-JkSlq+50apT=M+MZ^o^v#LD;xn4sc}rDeeziH3$PG#Txap8A&SBHRg#l-4^S)SK z=f0(yWLW`DU*bGc#dk5JpC|6OPwVf|PsEZ12z^U+2U8jB_*b=%mXn z>`U?_s0WdM$gpU{uO`oPhS;}0*BN?WxNtwRHPlo!;P^X&QzFON5~Gc0mKXIm4KCnD z7^xtybf2AUT>%AQMPC`vKbb5geweOj*h5e8RVg=hp4hqi!FFzW8`?7JNTs1R2;A8a zsII}~*O8TbMC0zcT&7_dlu~_AZ1rtq_Z7$yt^cCbmvckHKaLlTonZ`)tmjVQ-KaUv z1F|D$Tejn7z_mEe(d05}QEOXLaPfq%XX&#f*ZL8^l97j}j!g%Gd@MaXN(}d92GXn2 zH`4qdcDBy4Sk}4z$_0*KMPvf<$fdV6$8^TPFz_TgBaGrmZbX2_s`#)mBl!_FNf{7T zR~6VWa8MqE{CdziKcySXBu%12CHRV;h z*>o?A+IYp(*>(vQT(I4>c)$LICFx0A&D^M%$G*zu*4KSitYZZ@YVr<2YR`uvCug3H z5uBq0l8MaLOYYF83!^a{?jVX3)}(--vu>2dJRXu|;}6yX2l4ioOMmV`z=brfG!%Y7 zm&WJrjY!RbJtbjDmoVt>Zb~L5F@p-X{v{e$=vn>j7_X5dm6%k=_=3Nt^s>9 zj;aq-n^LUJ}{A^l5Tp;!?Hb#&IQm!8`nH*6=fE7ig5)4`osrznl>tAF8@I5ZE@wZUYn> zMI(L=>ljRW%PAp0IS`Y3vpvgw8@~0KMNP%Rq%Oo^F3kFF7M$ip#@Z>HZ0oa@ZcHV0 zJ-i9OckL?}Ydq~wVI7U$LQ?Ak!Pj>2L|F3S&HK?gG4ZwES#hGfs z7OmWtXqQX5c%>x)MOx#EtO|6YAsQq|7Lf$VDuRZPK&>N{3Q+`#kQ0*Z;gr42?ysHMnVtQ6&Ntufo%wF>JWs5~uPf@gsi-~4 z0cbRwYHi!djaJ@EpQp!V^7R*O{B#1RhGauweUo)V8GM552&h1b&Nk{U^iQYFV!TZ| zOB8jtbcdR+44$|xl3ps;@(5X{x9B${tmGpqj>P{-`Nhjaj2J&vDM^R28^KRh$?0LZE0|Pm zP*pHzp=uA-++KEc3aBX#*AiGKg&yP%W;^{Zes85E+29TEh=Qc7NNh0MjPGcO@zG=4 z@|xczr_}Mg3K%Dg;Y=p(+xK146y8GDYp9$mG}cmmB!)Gh0@ zkL;(9QzA{je-wP5L?>dIn$shI{-NsJKEra1cl>W1DYMXt#)>@r!fQ{GxU|G)fcZU~ z4@@Ncordr3^n{kWXoe$H%G&N_KP+oh1h5n@n$-A65~|Q)LxRE|oqqJpoBXN$+sCep zwsb0hxTWcbhS>_ph4r@C7TiqVk?PYd;EMw*ihumHKaWAB!kf@`Vw$m4e=2qzO(GIL zjA!3p+>7fDPwr)69tTD03Bn3`|3zQJ(6TXH-KSsLut$`7L2z%_{Nrm1B{E!lv_`L{ zQT94#J4C~qk+dMw$hUMlDHXT#bid;A9JMuB{=kqAQR6g*oBx!RD8+SJ+&iG}CcLR> zC;g{@U1(0AHXReolRFgf;LHw;)EIKWokQXeK7(Jk_aSPUZ);`XBq3?dnzD$$0%iZ| z`1)TW5nZO_P$HK(C^0F5Gpuv4^&%fVg1{R&>Ag$kO&Y^~<2c(O&}22>x^|AmzR2ii=yxIv8884uee%mA=T(A6wv@%l#GC z!qauQMcQ#i&w(uW`Moott3cSH2&pfstW*5sLNVuI;>h{ivr@Tq+YKV22rjL$J zg0H?ZJnhml#IC>aa|5bO?!!p<7Q*7q{_ZN%UC7^28hR(>DrgaX?B!ZfE%l!J+H;2; z7$p@-!|qpKohK9e7mAjS(9h=!!Q|{i{6gnTvO-8ZD6F$l+Kd3~# z$97i?T?I2aNclJAa!7xeFR&l(t0LGu^-~cvn~0w9^Qezz7t4)xyR;b3G7Q8ZgN_S^ zF-vTGkXc1r$SlOO?6tzH^2~FZh|~qNstVu));vRYI`vK78C3H=;M4Uk; zeYPsjhI83;=b~Ge$JzZJuT0}h3l+KIExF@M7pLdZsyS1T4T8TTgCnhid)n~-Hm>)d zaDj}&e5;Z}#r=GB4M1C`t;KkhJwUZ8tx;dEeznrgP@s^h?Lmt<(~=AI_CR(^I?u@s z3AcmU&?M8>Nh~dLsDzl+$5`$Q85U1V#TKXRi1n)&o6^~;v!2ghGEa+n(Si)Cd5SkY zEjS6JzI@a-+(Jt$C)mPdF-#w?c%u-ESVf?wBVe%}u02+A?w)v=qd6}#f&wJyn6|&; zYS|ou2xJ!Ql%ZJjp literal 47672 zcmb4r1yoeu`ZhUqcXud_2uL?5QX<_UCEZ;k9YZNCEhvnDAcMfrDM)t=-7$2-eEhC@ z|My$#{@1s@cdc1xo!E2sdH1{Dc%FSujHbFW9yT>L5)u;La}|YGNJwaONJz+$n1F{P zA~{z-9yZ7xuaxDGz7EoEKkT5{%BsmCAytELZY|Is_OV=53_OsK@H_wcA;0Isr$Ivc z@$0#Q>}y}sy#@4n(k=wX;n10+T9hZm(+4v=v^Z2~lxJaqGA%!p8M9C*e3UNSQ)KkY z(|B<|#0HK}Ka3a(Bt1MiKkHq9b?*)A?G0RbUkK^ODIyd7mXXDmyEp04zh%p%DylJonGyW44`VZlMd|ZFy?QBNU4Qx=gJVLycFGlu*Z@<%Njtg9HI+WsAn5 zj=I&R@Y)wMu?d-%7#EE$VribM)rBg8n(_<@v! zhCv?p&)5ZggIo_7Ke$X|Sv5G?oeef(4&Oph38=yJ?5W5;gAAP;3moQp@tm)=@zVIx zbZ}{%ESD~CJe2V$4Vc}y1`|B{R|8}e9OQ5Ryb!#J=|32d}5Op-;a zHX47`Q|v0$P4%zrKkiy0^4q^e^}AzE>Ph8VK`tZ=tzu*m5@TuSJLm|ngyb8}NpYr2 zK0_sVcy!RpKH0-Y+Ae>rH^0OE<&C+RQUYzFMMsp?WmSQlCKRn9xK@X*>Z-5pTOc38 zbJmtGVVLIEjwi5f>rSsSVvsOYxpXrB^^D=P_l8uh?i0`Bey8U}Vb5_h)|=RiU?L+L zs|$&Jm_=eBDt8ZRuzIfZ_^vtsRbA=H$DMd~c*E(C**B)ib;Qei$!zCjfLDXo9lLRE zk0Y^m-h~iYP_1P;ygQrsYExqfp^PU=Nz7ktU_i=ggn=4~-%(|Os@ZtE%W2vo-fj^& z0wK7+6^_0;g98Wjq=CK+ZvIc*-#m&-1>rGQ#_xFMQbq&NG0J6G-}42r15MLXp9k@@ zP^I6(NhV>~7LSbh*Rh@QUHM{?TO4>NEa zYJ949iG|K$#_X-DDf&{BT@7?$5GB9+c}@6TI$lkQjd!XCPv;-zc-Q3Tq|#T~fz zp7Y-Cqjch$)b(!Nt*SBa@m&O-RdSsIs=`*sM_Lgvw}HOnU@5Tsb!x2ygv!tB-R(=$ zpq0wGYqN0^x4p|<==If!JsUGK@n$wrM-=D?%FfK}pQwbJb1Hh);c%Nl_)c4o&-`|{ zfO7P^)Va%iXc5QI5?I$M$#^qd9oHXk(3RV$3G+r{K_(GJki>i%3vY0xan+sh1NPx$Nw_LUkUEJ3x}e~exqb(Bf( z)A(V%iRPLNhH&7`x=ZIde|!^5b``V60SNC$U2 zHPg*z2s`lZ&cd2U6XnqWef^r=$qv$RJd06FSBlHj@<0McuXY)~oaeEJK$Wn6mDLsV zo)la*MlMWW!beJd(I?-xEvLD*S0@^|kaC(HR)<~QJeSx=PJ+GOhud}W1Gj<_Pd5`gaoY>;7@eA7;GXJo$8UB1O{$?IKd7GA(S#AWSm(WQq|br+7| zM6L7Qr?8dWjqJphSN1r(1 z#qYeGnc+=?q-9>MukYF8LUXycTXaH4ck|tEY?r|o$h6Y_q{@l3ZDBzH%j(H=2HF_% zBeBQhZ074L==zQJBJ>7lzLGA&$x+T)gV=RJ0yH?n^rXN!nQAlqHC|0jo32DIy|1l= zX+&2i%>Ym@UuHTxOO?l&-lg*mvTyr0#+$c&UA8cP3&Hxsj+F>e_p$iGXJmm=a1Z0? z;e5N`Y96nTn|7Z3^#?_7OT+lr-$rqj)_wL{$PY#K_!Jb)^{BtVuWQ~?O46Bj@tp<* zBGh^YhfzM3QNctyRzunZSIs|J;}NQ2Z~(@g2*{-;<-HN%hZpn{PtB&UFp;Ar{pil`J$cTHdk8Th+tEYdnv7fapf2{Y`O1`xoCeJ6JFs^DCFx; zvQ^d}PE4JMPeltF#Ie|ZW%Q~;QQ_KwaFTVx#1fv9l&;iu;ewwMTTq=aYyV`_x1wL3 z{T0=l(t;1bfRlSwHTz%3&NxlI?xVLpH)A0@{<6?$@^K~`upfdu% zB}}If+0>Q%SY=f;4wpI8b?b&rzwSjXIt85ZEsR-(o?9v%gMf%AnZTuzk+~2dYpP}5 zzL}e4zS5_%Csl=$2{rp*6__njwG~fBQx_wv^O*7ASU9b621TSO$I`X5HAh0EdEpzX zjz_d?8f9k%^$GrUkQc2|I{Fm7=+XMuCuO$t?{Z3R7*$7T>jUmt+h+&5-fJ|rk$9l1 zl}~ooy-j9DcMk+Bp4C}Z`|G;Y@8MHQCfga83##{EFfNb!)qm6JIMU8e-n;gG&AkAK z%=zqkFrb6>E|Td@WT0=k0(M}79cs8XtFp|e(dlYi$!}D9!!bcrV-46gbamt13jRPj3Z8-o@RrbMAZ9br!yJzrlPQQur^!GOh)ujbWyH5R=6lo}9dX7B-WZznCJp7-GDYF1d zBI{MlbZXcuwqb1taB0AvZat|$>l2zIttb(Jb#sYHm$}GqnJ9;s0~ckwKe$rXRij8a z24!9FLz(!W*;LF#eJB}SZA;cE)BjEreqz%K#<3lCJ!a?LGa-ST^T^Fmn~bbx11Tq~ zMCU>3g-QuK(ZX+fz$dl^Xpu~!IyOxwHpqcTGq3Ezwc9CE$acN=<@!#=?C=a{_j?AATM zJZ9U4N7J_kKGjoNTgG+F-kn<(RT2uu-y}r69YsbBDvJ!euw89;;!GCt!soWbI97yU zUv&vE;9YVo{(SVwe0Vp5ttoC!=8zc+!MWi^|EGJ2n6k#5&)@ZI6IzewE2aDCi#2ca zxuB<@+FA+0z^4HGo~)5iR%WLMlPG<8#=c8L{nHc3*XMqcnvg}M!0H?2CZp@8#xvJg zkH>dGOuL}fJ43&5FQ=E)8w&v`${!jQmFvN8vOBy9V>Ezi_Lh%@S? z2aZvL2HrTZK-M1U)q{ zpf@|Fdx+_QZb zD>xCc(USaGLR3`6un1}Jq=>y#m` z2RkfV6Go`k!)$~j=b|q!&mXUJ1?vpu(N85-b_QXX3sO~wY};{5IrOAG?NKx=5@=L* zLVM&q?@L-Kh339JjVxMX-3r?kh>l5+oh$q-5U_^U8C!e5lHv`<*CKFc+4z9^`KR8y zCQ;M7QT+;y$FBs*(xN$ye;la7nX2Qy;)!nlMuV>H$8UA1-x)VF_TDi( zVezyJOk-Hlvh^Aj-#y4Mm#D3cNyE8-kc5!ZE_^J5BAvLRzRpcdZWV4Zb%i8RE9!O` z7dG5&+y+afD3}P~n-Pf##KOjY&A`9&4a2-G`K_xVF`=6S`m4+w%%6sIYUO0)tKI|$ zi+-WhK{ggAkeU(OPN7;pBNPa)Q@bSBk~dq=?ZP5c`W3z@{FLh8<9xzvXR7!km?_a# zBeb0nFpm?}0wOu6)epFAHsm93LI?{73sUz7H?23mmnq^s(NDW;0$&hO1mVAN|J2=; zr_0BMU&tbwu#I$HYn7AUx zj#rg`e|WFFK(z8D)l2;*{F#2A*41fW)*ORy)t(_{At8l@uY=~TnQ#h=g|MhCAupa4 zus8R?Yg${lA~S1mfJU-9NsR1WDO0hzhr-X*tz`@A36su4f=L-G!XB+l#frOa;-9gn zXS)+PaV?8sRz08sJ#bT%5twt@TC|zckWj=V({KM3JCEfM#+%f^>%w-F6iV z4TaWOXcfgzID~AtSnH zB@cnHhj{6~pRAXbpjN@Lx;w#N?`079%!xJ&szYR!(|}$yh-TnrJu5LHkmQps%H!r9 zE~a@OacOjU&@M^JGmA2Dg7)P^S8SkmL$13s-lf;nj_fp^$;MU?AK#m**ie@G1h-_|=N$i`ZW1qucL zHUgWm`c9eZ-Q39v*BoBGk?-5HuX^*!!+Ow+`c8Q;LXiU7qtMoUlDE_J-N z`Jge{)f4O1+mAZQHNH=A^Nz?Qm@HOKZ1Xf^6%;!E0YmQCyH>+Fgjfa@66Ir^m270$2iMu_hGJ;O(q89X0ZR`F_xyC<5_=v>>6 zBR&%Hk%2HQxv^?!Po`Gn~auLtF#?+P(vyFf`5AQS9$I|T9r{H(jj}t@HHv?jLU!!LP`68qHE}Y{%z@E4`l3`fbTh z_2Fo-Wn&3vtKOZ(<77*BzkT-83QmX0&T~z^Z#skv($8cXxqJ;KL2xCkuw+IBAD1YP zOB!_Az{j}rl7>`I$y57{Oc6s4>-*eCA{tApMmb$x8kJucxkYVsJvN>k!@jvp510iul9%<5(klWqP}=R#WF+4#=>bQJMwA z7Sx0Vww-FM8j+mVsT%ftcU}_mq3lE@9h_w-)?4UMc+zgtI?wS@h|PLs@tnQhc$Cc! zaS*fKdJAPY+fIAxc-*zcMRE7>U8#W2tShN1JxD`3C*F|l#G*G{T29LMeROm$=pPj9 z+GGt##8TL4-tVjfPe>2h(GdA5Yhq`(_A(t+>!N75N|^zdS`?`(j$FAEe?PhVb`JtZ zJXvDoljko{NRY^M+y3caAx>QYE37)}zY-;)ik>aO?#paC`EuXlbq8`{b-&uurH#Yx z*y{7pD$)n&Gfa14>Jk*u3VjBim3|D^EmFMy`HU#w41024lmI*u@8scMberu|4Yy>? zC%H*^!gio6CDY~OZ@H8o#;I3KIqP*0O1slRpEnoTYi05?-0oEY8ZU!coD@m3F2nGn zyv?fpljV$-JI>g&@+fzMHIX~mn*^ID_`G+t$6`dCAA4r|DUu&6(ih|DNRy-L*Yx5w zQU+thoZhyzEy4Shws5g+E}X*NNGlbcbgom56qkvaNa4uB46AHA{6$aFI`$0YlUdXZ z=W9C4A>Nc-p}Kcix_)foFif8`5#QstdZCP*s1DYuKoX90s~$ES>s$N%cJz8eipW{Z zDjbk>{h6{IA$hU+QTz7RycZ+=W%*8#YCRDZNJiWLlFt)U{R6Ro=Tf^D9L%#=xrhll z+UBz={`CRJVjEcH((1$z+Qt8j90TOqSnZZq48o5d za4?CS?CQ->;tLDwRz><;k?85UdNAXLc%CTtt)*{we_qpPxMx8tjRm>d3x>e>Q zv!Z_mIjcVRj{N;vGkY-HT~xDRowD{3U1&A9|FgZ5usc@!0`=P9N2PcmnGlU2@()^( zBPC~-=GSGqR4x!2jPir2;GbHKLkDtGpbzmH^2eqI5;+TCJ4(iIH1r7eLRrROD=ilbcywY%S=qZIqC{3dVs6DW6YYX|> z-QTyXHJS!8wro?par7#Cv*+nBas#B|{1Ox$Su$GgH@dxgW;Y#xV)*_S_fEpga+GY& zY_zxuJyT}sF0qfzAs?8JWf6vtxN~3(ZnnBiE|0a81Y10GItpC#5tqrBIynvt1A6rm zf`CSW^0#DY=x9r0ECQI5hUV#dL%p`n#hlU0PCKicsAh~hcKq!>>y%>$J}c)?#Dv2YjjZ7;85S3DT(t;rqA*@_W(7KnP=#2U2r{R2fioC;c&#Gti zaa@!(tcPhd>+4ILc>hYNx}Z_jg+=SimWakkiHx%&PRcHU!cS9d=|o-%2YHN()5t>hQ0x&QQYMQVvQE7pWlt!wn=RWDmg@;dk5j^1&?l&b|S0$CL zl0Cx9f{t~JzEO@iE=gj_Y#K7d{rJ-U(D|ad%#a)70lSR2;27*)5`H>9N3s^GV)BzP zRIckqkxDYR~UA)p89g8YRA!r~*Lxq6Ypyz{Dz4WnseH zPiDV6U;=pnOwYKM<22ag+A8tFs66sz*aghf^h~2=gA(Psvh})r?Gv{0=eIRBm2pu{SApz48-|V@>{bhX^Daq zr5E-~*+*KnuhCyu=wnTk>nY3w%sB<=kG*gnx!THR43WFSfS_AAPK_9kJeZaZ1h zg}xYnvZDk`{toaoNzl)~X`-I&#o8~F)PVpi!q_y6F`Qc&H%SbpOd==nNLvT2>hh4F$EpTl5u zSxmMEw44SF#980>!~ITd3VlDxoRJgx?IsC$;n(_YsyvsjmT+sR~W0U4IPlF@1 zmGB+MRz|OiJgY@Ze*q-)<7a_#c6V=*z>~+i;5hpe@`8Pus|l_+!X)zj8(d{@`y;oT z^B~Gs>+NaJaI_$H{zpH`;#Nd^b(;V`*VLk#N?@IHF0U2*wM>uC^OA%tvkRNXq9~VL z>oqGT#eFLcc&rC9=v=;U$3%E^GXrPPv#D=Wl_@&~4LSua5h-zA(PG3|lQ&W(Rrs^k zGs8|xDrvtQTK5|Xh%&yDZu~NP=RN~8PYsiOp!OKlV|B&PRZ`V;nu{D=wsm`j5Vp4u4Uk`Ede(J=FjTGIJ9TTC~3D zGkc$;r&&MI_90KHTsuTd&MoFDm)GtLK#AvnmTh{42M#gfjoR#P`*D7qg)eH z^v4stv-E>c5HsUhio0{~(oIVtU>Yw)0>&P&$GA8T7#Y{|}t+b?c3^5*Sl}xFv`$neeptRUeDV!%Cjb1uOwOLyXXMhO_*U^A%4o< z-P6su~b$uvnx`iV`1f#7^Crqksv^4fx>L2%* z{Lem2Q6N*zMw1q)(eD~BfJR7SMX2cy?)@G97i)J<1g65|%C@H%G2HRtUq zc0_(BbeLQ{URJvd+$OIzszoKCA@~25r*SOj+Ml=hlJs@zja?c<-FFqGNk10MFydYI{W&xg~r z78!PJO(V5ys)lO_B?vDL`N+hGBn>x>J&@;E9=@8>thGt|JVx$etEM!lNwCL@zJY|^ z&rZ(-GQ%(`t@= zD22_MY+K2DXjX7ivDgs)z$|aQ>HZ(GIc9U^=nuoJ5JK?(W}g4`9RD!YS}ErL-0)wL zg7yy%@h?eZ{fFc}g5IkNQ76F3{&3pRSkiS+1mXRMeBgvb#DtC2(x0YVWF@{5Dw%@o z4A>HfFa8qT*CzMxx&D&i(I0y-o)iE6qyP1$l%9;gx$#2Qw*PpzkhLkUwITbg0h>kf z-{+bAN9s*@MRU~`8(Wl9Fz9ZPqbi%Pu?0p(b^`uh;U;RrJN{J%$M1L@IAM7;*M0Du z-jg2#Z7KJYn%h=s9${892)fX#$Q|54i7Tu0|O;y>zv|{?{3N=2E78eW^_FW7J`&<8eCk_2h;dRpsj253;W(34&T2Z?&?=XPINrkk9EcyAvL+q>5TO4hAtY3ehoE&tqQITBd&%Fm z1W+`gtd=trl>a`iD-rl&oUu3`-l6F`sxPKVtA@>Ohwj4tE+4$mzvK>|&v955K)YAx zX*UHGH-zdO@}bd_>aP|O=KK(HFR}giFy>Wy?=)U}EiUwM)rx1p4;&Rn>;CHHDJ0UEsa6P;E{`?0uVuro?=(!h3y>E)%eVv~W_MS3-_qN|=29F_bt!~c=s zmXN1gj3GB;HX`P|H}LlBO91tUp@_l+_6}5vZ={2GLr)0IHJ54)EUNXP5beuF4t)zB zKh5KW$I*eF3_c8!ougXv!vVdV^8~%|CV2^REubN0r>wQDS98$>&0gk?dH(gyAw;9^ z0Qtk&Rzxo9z>DmObJ#13>JIakC~9;;Cfi?am@nzu|4zq|yhA>-?WJ&|M~RO1LcX3B z&jyt++HTDy-QV;lz!+TQRg(Hd7PVt+XvhAkOt}ct@lV6s zJzTklw1R)nvb*O9_hDI<`?!b>tat4FI01p&g_aV&l|CV9KR*%=ZU#HZ6^$X$OU`%e zcPel&%4;&t)1&)aAqB&pkWhJ$^=#RD9CkF1xn_@AZtZC#^GV^~XAA9}qfIkA4Wpy2 zj+{Rq-a<{1ctc(MJe3xGHzbm@52awlfso=LQWk+Caqwq=w%E@3Kow{FTZyYt@*tv= zM*I)KLWFl5zq+fMgATjmVbKYY>kZTg;-o@kW!W2}VCeKYrKHCr*w3}F1u!EM6aH%R z4V8coK4Wl?T&>+Cnt1zn{)QDorVmf3Ub;Tp^DUYo6?)-8(&#LjR1RaT$a+RS)-nr- z>k9vo`+}wIq9(_Xr_$w**Ss#jpU4L)70H@wQcBT|yx9ar zeHHg&TrAn2*0TH<8C3^yZuPsE3NsL~d_{pL!ux*jYVF4u4P=m%n)C=3of{+2r5E0D zEE7?G3Ves`6u`o(=uRv8jhx;?P{@9YLB-854b|Z-*n~6fx^MXYdi>paUWym#yb1F^ zJXrYo3HwsmfFpLK9TzRa!5X9az_lY^bI9C8%jKo7C@p^3bDMuhfCxrD@+64vLo)P0 z?{jiu`a|#6F~xeNvws1~Cw^{<|BZ3xLWVrG)CvAxz(ApJ#N#LmsV&AURDxsBKZM3V zQb>^+7ztD_y)i<;AC)HdOrmuBW2OGpeKJ7W76ctK#x?`aKMB=8G7{2ncC(N3yYa)? z0F3|0MgD(O{eLa6`|oMayKaYS&gLmTv;-y8)S112hrH`w67pegJ$!8Y6ne7>)s)2u zXgg#`+nuBobUUz1tfR_Kd#8Ub5*6yHKn2n6XKY*NG_?kgQk_EG#rC%MVEi{_i9N-CPHVk3G*@dcEJ6Gi7rDU>#9{u9qpd{HS<{K&ckJ^z)FM{0Eo;GbV?Ks1O{n;Eb%ptcH0`xw|_fcW(-g~+3Z~C zFD2SXnjft7y?|y_Er>>!Ears`T15!!IM>>p3cn!b3Ll@vy!G6sk8j|%TLiOVm$)q- z&SY1W4o1*}vzTpnd}$?p2(DjVjhrp8t2lw$x-f`_cRwcd_l9=7y^1YTyfZx9k|?{f z30ix;-`U)eu4#ZfzpGOjT8Mtqc6a#LS!pI;E2XF_U_Xgt57{#1z1-NWP7;R)=ODc} zL8-pak=U;!X(8mdQdcNinQ{54Dyo^`_D5qQal_Q7!7&~qu#y8F`c4{cW&g7k&(rFhF9Hn8_V294y{Z-CQuVpUquZj78*YX(Iv++*AiQ$73JrRr=pUqDAZ`mfM>r zE&bv>_vgePUY56|Bex!_-_kc;U??6jN)3h)w9Le`@$|FoxCuGb?t}Klr{a*^S&cQC zRrETSW@E<{XDx;<%!)MN^jl-)80>>a5tUKG?YYXONnoXbQ}`kUG69Qbov~!oR5Q=@ z`5x-J8`{rwr`U(|QpTm8rS7Uf+*O=TB@a@s{|WVz#>bd8RG9Kag@v7zJsG+BK_Vjv zRVUl4v+#Yn7PkTEij#%iNP%PnzH#UoPHDa}BysDR-D4>#O$;|IVXxfS{xB62|G;WT zf$OFzBqkm1S|TrEKygER)z+y(s+>4Y_oV8g^K&525O! z_HGMB4zh@jjaXQhn=6n%GF^RD?+*)`;ezD5tfoXKzL3dP2UhP#z3I)53+>1yhejo~ z-#Ec!taC7GT4v&5398Xo+`i>wgz&(~Ersed`>XLgIx%+AnZdQGk!z&GqwB zr!nGHDbq0POb@xEOXV8uwOkVU?u$_I9bs6d5jmA~3f}Edz`=lQB;mHN# z?$PlP;=K!gU6ZFZi`k~Za zqpO4ajWN6gZtbhQQ7GG9zP|oc&5h2Ah9<1dCf~6QL0B``>8%$S=dEvs^Isi6v`Q4H za~A!Hb%FTkFV1AUXv6Bi1@X`WFjl7I@lE}U=zD@iF@tt+Rfx9Q2<8$QZO^3Y&-k(P za36y%(ZuJQH^uB~0iq1FJwDPLGCzD!k77;njLS&TvE`DO-ZC0n`iJ)*uX;c{B}%kh zG(_F)C<>q2AFZF~5e0-i5=yRiWyPl5A`GK@tIjx`7W!lvYxog8r=@MMSd%t)I=*VB z`^W{kJl6*gF%lDh#&5%pqsviVa~2}#1?ttKWyYVm2hM#f=Dm>E7lIwV`VEZ)AXjYDoZmM+slFFeRXm7lHsvX_ZF- zbOYY5$0rr)x14)4o%!?IBpxI@%43iTHQ|qxe=aN z!~@#`{?;P;g)4WeQmR^=!OupVcJwG`ppST+vg35mlv}BZInS@R>((Yns2hn4bMXP- zocuC^sl5W;DauXApyDn>xr6z`m;{6^VGwN{X5%R3(zc72RcMnS7~N1uCq5WJNS&|P zq7_ci_{?Z-5z4_ziDc3VIp<`W#$nE{NEU<2~ zy5Cvib)~QULmo1hSWo(%kj$kcl1b0hi{d(unbt%~UY#m{ktU7*qV=6K%Dl9AT-uiA@Qmv%Iik5)lGNn&0vTCb3zF3cj@ z@-M(}L}~isESEE*jt$!GUVT6y!3HlZ_O>gob=A9rM;XU-5V6Q6x~OWh`GBanwa%kdvxbMplT zm}nzvPP68h-84>JJ4+mmh=_W73E0f?aNY|O3s>=H5@GR6^61u9!U5_@B317k1(*Ym zK!vJyr`bXr`L6~RUjAn&@&1nDO&Ioq)^e`%ez%k~@!AyUEKZRFuBHnyPZAj^@0ABB zQGrCH?2}f)GDT111MKmnOg!i#vOx-gw>oG_W)Y8c44VuJF^%c*UKL}@|6F5&MLdlx zg?AxIi2A)?)p$?+S^HD#mzh+ziTAWDwl5X28_^GJ`Q$WFvDkCT-?INwlGDU@gwjO4 zaq?KXWI;!NA);WDfP>ukirL`Hnr7t~;V!|e#ZsB7>Aa_@o2}G2ICBSG%jJ{LTvw`% zzY+=B=j^yYQlMUQ2UMG3h7G`I#|@#h#L zsFe6-A0JEWFI2w%{HooYxmTzN}~2nf;@Q$M;#97 z1aHx5{aNJX!!3pb^2V3yjEJacDq-qLf{L8Rb!caEeqbiTNA0IO4Z%;Xgv657St_t} z(IVb44LiObbB>9NRD_Djq84cpM0H@LQkF%b?GREC;VC+D=p1VbRGw_#STjX~65maL zW%VBmEK%4eCdM9{G99h8Weknj5po{)**p&&KmiXTfmew8w~{3kDJxOahZEcEdUm|xPMR+)xz6inWnB-`&=6Hi7QbwYQ#app9Jrf)o|Np7nWH4l54SY zf1*qDsNa1-l;yCCr{&n@A|o^ZQpkNnS6&IHv#3emp?dZ;zo=tN{cL;wlzGoATfHq# zXXX8u4KwCSRyQWIYKZ2>Sz5h^xAY`!eLN`>a`RBkziOAQH|_brp}2=q*LtbVp&B6# z(X+EqFAN3d;Y#R(g^wy-He&W4Q2Bp{AD*v7 z;E2^5^PUx?{7J7)r+z$ClI0;VoQqwu1nQ^lF(e%0r4E1a`H?&dDyo>Plg%Q#kwQ#7 ztSc9U^{4C2BlEV+JQik&`I|7M)w{nhy#>wIauuc3tO@@g%vB+{n$HECq$koFDzB+Yc}w6}o>HIf0Io1m5jyyYnFj=eK%!R~K>Mf4y{Epy#mH5^0AbJ;Q}Avv z3QYfPdyy0dcs6)nii$sqL-lWVA$?K$LY(TaF7l2cX50B0FOMUM{)1nztI%C6FFu0+ z=&W01A#ZSM#l4YiYFr-wwkaYge__#b&XY|BEQwpFY*Z)`<@I?)@W#tU(%)nGz)N zS1%=rJ%*TQA42@L!||zk@`C;#SghNgVx8UHU-TrKqD>o!ZsRC_{DuyguSI$rk#cw` z=A9weIZ+f&0|WhlzsQ9RJ%PkA6&~M`73RMIApguh43%rsW;@d#`M~~d89BLd_z()j zr(L1{f^9`hC z>5gyPd)iZNfnZ+J6531j?d)ujIE$z38I3G3nL-gu%T;HByV&Tc(cHL!!2#Hfe3zV;o z?+$+*XJ$Y=1q^Le5|j2KsvHk!PqP|G%@WGP<_5?0Z`U86?u=p3NauyMPVKJ#?91Jn z{VD*!gT*#fF7`A8aq#aKeb%pA2^d$;!*xmL!*cpG%9&O44BvSta`)O(AF>^AdhnEL zF-REY2tvQoA5_ZkH0sN31jV3xQEdJfPP~6&x`6TA0SVd`osqq{JWAR*o=G;`U@0%; zTf;XykQY6G4hyG;92xXGl)=v43fa%m=?Mpi_Gy={f+tM~si@=oxi&3NYp;AOrk7nE z%%A&4KX;m4-XBaIcCq2$Hf+SIPWM?^iipAlMJrGFC|m3+L;3xUeJcDe#j>Dx)P*bu zOJf(XjyVQCg_Di%=8zL|+}|c~?c(e8z`b2i5m;o_bP#`LRIyTsdaix6yr!!!wE1=~ zXOTo3A|+P|zgSvpFW|37bz>1mVUlgbYZ(Q#`5pC)Yb)CBK6i;qN9w(>awJ_TWto^P4PYp<7-PXJJA zGI?k$9bwwp(^IDrY&J21j_AoB!xj6K#u7~oY&2(`95n&v)}^p|ck@U7JOS@A8PpZe zdv4q|w)`q2{ixc)!bF4Y*m$LJ-*KGVS_jo|Wi@%z?@Lc74x=tw&E!^;8RZ_i}4b0T}6WE6ZtO#zWMX%fYr(zQ3!3LIiHYgr(L4h$hI!h!oid zEXWha(^Vwmf?P{o1!QzxOWz*KbL&?jwY9Z1e;!JKyq>1J-(`0cXi^pdOpxDoq+B~v5w#?!6wZ7gB)o)M^BdMVPuCzXg{ zq#FInMto=+62u6IluD1^h^G!wHYqhNjaaY&Y84x;y0>pcX7mJ{q5bi_W@PmAiFY^W zpY?Y7-M;CTM<_|rFX@$8omX2GFg#&@-o4kdG4b$Pg}H*JW)gge#x+TZS`vjsZOqOS z6Zdq`cDP*IoF7mIe`Ege`gQAGKpilLp4?LDJ97!)d!{04*QDEB{#YkK2A(@gybWAz zP3(igq%Ie8*VO3a6N zjz~;<#C zKC-d>Vt_>snup6?Y-+#Z&pe_aN0`K0(#*2Lx)};acFYB-1L5~P-zwA`46;A2fkd5A zP|O9l3P!fZr>={6a~Au|@dQU~#0D?6=+si@ z%8s8&d*9wqc+ijZA9T@{+#3|KaM7qD@><*wrP~Qgv?do1ys}-0-jeL62~KKX;9%I0 zGy;28dYZp4bt;!6yK3{rI1Ys`|CWTll#kTjb8_F|XbEr2tELFo+iqbYL%$RPLLeDx zu+*_6zA5NVn?n~UI&xd<_AEGY?CxtT5E2;V63kI#3RHVbv6_5hhA(lM_$YU)O00<2 z>enY0gyoW~=WH5Ug-IQt0SR@I+bp_=*RO)+aG%5K@a*GBp*9`J9yM~McmG|8(=UI2DQ`)m7K3wK|Y^|?`%Az0m4AC#(b}qGm8kO>`rnteAl7+>U z)-Dydz1xwxZW>y_=&%pn=PHf?coHGsNZ7~R-XO$i@1AXF6t7k$BV*|8@l?Q?%+l83 z;^erN%`D=REAI7o6sU~3SrFtE(Tfd@6={QrH(lOG|Y6wX4(XWV~trYts~Tdh6EZwr#rU+p&a2T#4Sm z4<{LtZkIOuV$)AU3V#uv$;AAK{Ju*db0UsHK%IMGe5G8+f{{FlVO#ACw|7gt^+6?o znODl0ar9fx}Ln};F&9Y`eNGax*!YUrg#3Z4p(D|)L!zt8}%o? zY3zh8@%!?J1e*qk)p5Oi=(L-9Ka#p?aw}>7_MX%%M6NVybyhh7A34uu32 z=1$l__d{0J6Et9miEqr;H&C&suY^C?P(*%riIingwmiIy5%uJi!;{$Ricyxt4B{~I zlY5WKfe-@20%QVc^xazHx(8TUhAd}O6Z0#ril0c`MH-v_Vl;DDhcE-+&u3fk{Jx!t z;Qt>0dO(H0a+#;=f7*$ZZ{@$!+gHF(cCjuo(3IQu&h@nO=a^DiKj_A(del` zhpX|>%56RARQT8NbTDp|Cu1?~`9}Kr>2lxHWqs?C`)Cp;M0J|TJAWf+@Gsztj>|#R zv%L8y=aLPzeC_Wzn-XnPC-Tofc+ksl1bwUZ&*R_DlkFzT0QnO!>NdnIGJlNE<+r5K ziE>LgiWaS>iaMcJgARvoT1451{d|&NPq&x2VfXWYrW4B_+4v2v$LdAr=4m+_#Jbd% z{*J#GP|1J4=-vdo!^L}p`0%-WQuEa$=Sv(g5hqb`9^SNe2_nR6SvF4x8@~D|{+=Z9 z$@aW;)L)X@_HVWm9aSSdkuvCTOiY&Z zK25LAx4q}T;Zn<6EazXZf1Cbb;FE2t>6YID`1VOXmi;GSFYml&v8)|ym#%>u;=Mdh z#Te?=_PID3SO+bu7sfXu2BJux$tas66pAm%B_y>r;_HCa3GnsZ-29EoOF=u}q#&oFrY>GB_sGH*palCQnTgnDyxkD;T`wPkB;x)|}~nDy4N6KL1$U2M1* zqt;2Y!HrMGc1EARL-5hOZE)G|V%n$y7&&?z26b(MdhdR$rbm8buKp|XS?k+}CvAc6 z{jUWSl%7rt%Cc2d?M zUwbOIZ8CV<&L_u?4ElH{Jp0~9-GmVB91a(0(I?=Wwd3p1 ztykN{XBrvrKFYT0(!sSjax)GAV*iY4J&DKQ(JL@@WKRU&*@{_5(qmxf+JMvLOP9z8 zu`2bv9F9vw>!Zj`j7@$N7Um; zq(Q9fNt&M=V^8bKlRQsUSh8=v>)R!G?D3YDe}Mh;+~q%q_|y07{>wBb?tA&$?*D&1 z-nyvHBg#DX`j6On^rjl5Onkn2a;*5$>Rpf6hRK2z-+T{6^dA$aL(aSW{fcNkeV00) zIY5uEMs`KFJ}cGtvW<#3y%DWDcER9*{c$GTqpn*998C7B=-8|g26uWJBfj#s#rr?% zhQ=KRV1y{EwPFk;+oh-N{jcP&zYZks`w=A(-H74C$M|0Vb)s_Xwr_{_zyY|%jDg1G z@_mr>UqzJ@zXao8?w!^KB2m@$1i9+@(tNcQx2eDgAl>GL8@mUh;L@}#o~1bQSQx}h&{0Zlb3A5&Fha)x^hiCym1YA^5+vLCIz$m55%h7d+~a; zT1akG@cq(Nhy5ko{c$+)~M4{1{!c{=SqzGMZ&Uo{V^r=Y?~n!_JknFn7s%6t3|mqKIyan@P_r~Rj zczg>V%>ELu6|0KZ0s*%oI-fcZzM44{TaR9aDN8}T=Ddo>2*tWD=i!d_HI&GcUgVDv z+m_G7oTc9*d%2oOBRcTnoGW-wT0boN2n)a2gc8-O;nKFXNMH3GB#6$$?ZB+alER3e zzFz@NmZBp6teE-!Kx~N$K`HYYj9u^@uHSix*UMHzYV$)(>^~BF?rKoIR6aa7vmHmz zT*ZP#dr+}fDQMMv6P*ja279*ufXo@vV)jRKq0LYfc`^j6X@x^kBTT4KJP#h8T#b>_7Q&h`7Yb!fi67Q1MQHXCNMW$z`&B!Tt!RGe?jJ?J z;UB@7A`c2=PXm42W%M690jF#!QMphywfu}mE%q(@6yNULgZtL>C|xuV2T#Vpa&s2-hjBvqD&@y3VX`jD3~p!Sk|MS(5-h*V9S}C*!R^8>~~~A`H&mvHT)B# z%U2GWQ<>q4zJ!UxCgG>Z08}oP4c{!9hpZ*aA=v#0pUvMQ23-Z=ak-%pgDaEsF2)TW zi(5)4Vy<4p8;$GX-i_->lQtDT9Wx#0ZbcxKDIUvKe2*IsAES7MN>HwRhar>Z!<9Ne z3S>!vpEiGoi|2mCwkvLw5q0{@5BrP#IFE|eD#G>XJa!%a83)$Sz~-Oas8S)nx(}N0 z)95GC7$1@ag>z;=iP(4e zA`BUFqiBwF@MsK(I=2af*GA#%StC)cdR3Gzl^=RhiH%AGrVSp2AC8_y>8h0w3>!9Y z*?>paPhsY=tteBaB392CiA{$cB5nB)dF>E(UATkoU(UdBZEh6To<-l$pCfbO^2iWs z$NDu}k+DEQ(J6V1jeCDWzU-;7W&I9ZJ+TdoHvWh*mCD29vEs&!$H{ zv1!u=+`n`Lb5`y~h05jd^^B3&dBTHA%&PB+pkGSyLL|5(lN6{_~L? zh}MQ;`&aXE(Uc9PbESpO9ghVcPr|x==TWA7c?5}$)X}d$#ne^Xk*8=`q!4{WMRdY7 z_HgXjxEtGcZiY5Z5u~-9#^6!Yal@1qWpcj;Si>=A+6ZhtehXzQmVrTZ@QjvQm?+BV z6f&ZGMg@CL-9q%uqgcK33`&+Qqi*Z=;iV!!79${AanY8Ah;-_)WX@!4K6)8?cLGx7 zuY_EwOmNGFOAcC%4$PlB5B8w^s8zorDwL^&>Y{zg?70i$>rGItU|Qsf>>ZZxi1TI7KTTFV1bmp#ZId;!`_Rggz)@9>%#h$+w$L%TLY*^)U#`yPbKZ`4M$ zs+AFU{1|HV8;A_ptFM3`vrc5dE_ zdzZIi);B+*TD1~bDC+V43vQGzl}By6F7KI-2{XSIb*r4HTPC$`$$@5k_#Ie`X^|yu z5axe00pB0Ggi@l61MPP)dCW+hwWLAWTqdmk?htYm$OCQkRctzP9rDy!{Qz;Alt`n!k3Gj_{w>Cyvky_OOc5yR zF=N85lRseQ=NobPUOcLm$b-E{Pa!b&4i+rljY8$iAVshlmYat$ZtRD+Ys!w|xkdhn zHdE4yW6or~feE9>;F2pNN@lcU^T7x-cq_D?t~^J0N>7EfY{fHqTp*`Vj$52dkz| z#=><+QMzU&B%IlSZI|6Blq)^XZd-&4u1unj7ohe_WS?QrvMHGR!xbbqJQjU}YRDfL z4NFiK+}|?;-)uX9c+qB@+O!$_&zylFT|pGhl^h>U7=zVE?x0qs5^#l|$DX~XvFz*h z$XUK3;?8Wq*lF|N$Wj^wUK4HKrtk6a)+u~Edl9mfego+OH86+)<~mW|<{h|#$|7%r z%r>zPW3lGgUA!sQDeIQnt&!i78$`ct+<2_pcpByFRD$Kg0nGSp4K81KjAG?WAXH?h z3?Lo&ZY#RXnU8mB)kHyYOrF}e7H7|2$GYuDP_bMwESWSCU+%er8r8gQr8epu#*Cee zE3v6ix^Nb6f5L3W{%^)(*|t-7EY9O)@@K%^NG0524P%Fo z!~xbtt;EUm>~Lr`%t7tHH2y0YW~W$=#V&fgrp9{qEEy*IcGWKNfQM37V_Cd#^cnPSi?-u_9PXuE$rdl_FW{{iKymdEWMS7O443!uqU z33Q4r4$WI& z)r9v@>YbLjw{JJ94fz0V>eoa*Q5Q9qyZCnBX)z93ghj{W@n-3i7&dSOq74P`dTx=w zo`)DQbQG>cCqs!snc+#)-qR~i)K}e}J<#sn{m+frtrm;&cl}uuMR5)qB*($(>tCBE zN&c)3MX{Yx+O(b|w)%9tPH`r@v1W?WX!`f+^c7?4Dj)t3t;FmerDUx$N}bledSn!<0y+PMs#|v_1{1E0@G>DPryl6`HBjT9r<&thp{;gzr*HzCBc(Cil}V z{ynvPk^5&UJr^8TV)uQZbpGh5I_>Vy*A$!0rr2E~ zjaw%voj*RNPJejszH|J$1*J}>3FSv`=rFYf5v57pgIXsHz;Xb8p&pC2&W$!0l zmFoSLsjmmttDu||Nj~t^lsTqi50JIZvi&Uip}H+^-4%%uY3S3uf--g8S@GJb^nbIFBC4OVcytqG?RoJc_J~q{ zz(?Xmgz{GHM#@7`*Okbd%ALp<<@^n~|B5oLLmTC)NKc>E)sz#WJ}9nmrCZ~^N`hGC zorb;Dc^=#rwb6Q8Y0|Wv`u^~op~{DwM8$R6Mc&#K#S^J?ZO~SctG8a;q`Y3To|@0Q zruR`suM!D)@T2lh`_byMvxYQPHs5e79=la>h?ySzIA6(8si*qc$XZ2|I-}Q%S20Sn zMs3tQIDhPl`q|zYLzTIEA1N1Bj#K*2*`;h6-$Ge-SR~jf%U7f!;hxg4VSPnrVcfaZ zO37lK)#bXjtf^cQAOE~*o-$*tukH-)p&W}=6vvI7N{h}D)oC9MYOWj>1u?OC6=mFd znV*U>dvFtFt2~zO?^2qN{amc%2PJ>W2I^;!{z-n@`bq?kEvgwks{W zdXItW1K&}0-g1j2H+RE4Naotr*rx znX&1TNQA>zrya`F?(LO}3F2ks8Ku?R-PLK0YLrvfoUtj9hZZPlQ&kdsAymd$AO1wy-)ti@FK5V8m9qUa~k4A4PM?|w6e*2tSx8l#OR65OA ztK8i^Pw78*mpaeSqHVK_HpymlDz1m8m1^ZnDre-(k54L1-jjK*C@V+3udFy0p+wv` zsn+v5I~ORUKNV#ed0Kh1X;*RVh-{0vE85RPO7*gDDfh+uZ0b;weMyq6BmnfgU6}Z#EHjw@32z6+8fGE(a@e3>*)01CQ-QI%E{Yu>x$B^ zcU$FFY_!tt-ByYycJa{-<&jK}+;7(d<-O)D)$(vg+)^yoNTpZnw#p@$#JJPSyLG#X zR@1Jutyog|_Dr1e_|WG{x@?uzbMM9}ZIoF%ZYuYGT&VP#xJFD-l-}K2DVIbW;k0?r zr`td4qfFdzL0LPli?aHhSgA+UVbKoteXq5;4p+i`CI0p)rA4bDqD2$O_vKAWr?IjR z5U=#?&{fUbb1SAObHBf!d^)g;@}peH5vN2vjuz>+C}Pl{9A7?08S(ic#eVGvrFgk_ z)IQVJ>1~zCE5&(HZqJwB9}Ij?Irc~_;JTw!sZ>KbBr-GR=;z8i?~A%-5yy8|Z(nBF z_@2t@lMmGTWwVRDfAEu1zeO*#EaRf@Dd88kDxJE0ES5R1ywhf+nx_dDzEiqS`%1aK zZKg7Csq7nglp7DkcJJ*}y7us%myRr(tc+bD&(#*Cd#gcW{r8p9B`Yb1?uiPk9^2l! zA=|;nr`9NC%ePSX|K_ewlx`D$5EbsY(!TpRbzNVK?51qLX;r?R&_S90gKV?olpFVA zl(y0m-8rhX?mR-VijB)O$o6UL2Yr>{OT7K60dE#o79WpMj(yQznfT2a zF(q0V(tVJqqoP=^tW)~W+n~-jwqpzRT&0{}qr5j>91pj@Q(mvsMBS$S)Hh?(Gfzzst>Cjv`Dl6%kl}d-v zvM=iPwjJ&mrFDb1l$)}jX*s6Us?$@=yDx_fQjSN6l_zSaW^}2nbnG`m88V=sa!QY3ozYRrSk8OS9nreJBD+?5J{hRY*d@#Tq0+YbFvTL~ z^Y_c9w9cD6f?j1Yy zY5b;gU2Lpty{2kg5cl|@5+9%N@|fiEe&9chb_^iev)2H zp1pDpru3}?kIRFmT?S(F)-})?0@1K$U3H#X<*K3QyA{Q&RLGt=UY!#|pWmNK|TWds%cWSBADi<$>OLqWsU?75wz{<_1FmuS; zaNDiOkS;Tf1~+C79g10>Zo=tDx7B4FPB*Lx@|6Ywfx%+J352Jvi`?qfPHEAztPQ&# z0%4)$@J2Q?e8TPl8a1q;zOGQT7_L2VA;@zNLx+vT7uzo4?$1XNQlu{S zeAN&Mt`7d#m`5LT)NcUFY!Zu7EKNb%j{fNuRgRDQTMc45k zB2}g=;xO?=WHs_~Lht==fni~&Q7MCXl^!M22jPKurLI*RRyLarE~oc0cZb%5M)fMG zugerFhV9Xv=w-K^fVs-{1Xbi(FtiZV*diu zAw#ktT--DdV?UmW^LOv#!ksu#rviPKJ<3bUjnE+liiGY#qux{D$&eN?XSU;Nyd6^~ zPr}OM=W+b_F)^#R{KNx;2|#TUXBENEGAEygp(KU zF=BzH$wDn~X%- zJ@c?NR~NinDj94}k9wKyqszOIwdlJ@EtUz+Q643-JLM|X{Fb|Av)SRWS=DJarxWsv zz7t!&M!Y3bP1mXOcX0B=6{Jj`1+~kH{IuM{fkQWOVC58yowW}4emsE)QO?zBH&xeB zuyRp6eDDD0f7*|?I}H&#;y_r23^==G8R9ID)pedZe-~%YToc)(he0$n+qP}SI~`i7 zb+TCfHn{!sZrH_f?R1EO_mOFhk9R+FIj{HL?rjJw`VP{_3F(TVUYc9D6zM_QEE!O) zP!P=N3ZnjycE~I~&Xl_dVk2Y4cC@Hczp`8cWt-JPyGHW08Lx$fBF?E%zXkDLo+~e1 z6%CO{SLTdWP$(2gma`0kHKIWgWi0E4CE+seifPqznqhb8kUDirL|@&GG2_Qz@6{`~ zd;0-gE~nTRajXmSGSr{|Ew=33gJx};sCBhe!`8TdbstIC@Se@_nN=0z@?me*` zf4giGb;t#)tW0WM^j?yhHd|(t%ws}u#zJT?xUD#Hfy}uJ!(xfY-t9ji?)G6!o;V&S z9$m(n>ygm;+IfdF3~yB`roJv!Dk~mET48oZW6JQc`1FhYcy!|?UMo@#Kh7Q@+P2Ne zD%v)2URCG!i2rhCsfq$2z?uEu<56rRrigOgcK!@bT)qi?!c9>(Ct%5jpK$u&BiuRh zBWgD5fj}2KWRTC{Bkg;$T*drovblW;nHkZ}J=c@D;TDNYXPA^h44I*tO6icf^!*}}OvL^%5h-USfc1f^XO=!`5I3`S)j_UbS z!{+dyy00#75p8zlr5&QY#^Gf66&&2N9akPb#21q%W5MP_*mvfXsFdEe#qD+?N7jnS zB#JXb;qsz-+!q;>9H-XJ!j!QyaqD^n9wcZ{ETspHTMWc?O-7`aSE|S|)FnsZWNtJP z({C6u!F_EvY=vqew|rzpa3Pc|>c*)%;mDk?ILhV>Mv4r%@NSP*;%o|J&sP{G(I6k# zya6!}j;ZB!{p@vIzWqqO1w)dzKFC+4D;2`X9*wYa|4ngD7w41^9ojVRgtKv}5EQC| z(dfNpLf#?~5Dj!c5@ng&t`@49@QS3Nz9u>+ZNB_3(VBPt6q(~Ja4;Pi(hDo%Q zZgHM1Ri_SQ((@Fng2+1}A2i-JRC0(m&1x0xo2WbK^X5ceQRzanl|)L99Vd5hM^uDp zlSJFS_t*)XI&)KPf8{NJ=lATuo9}c%h^R%Tp!5hdy1iv7FSj=t)O5?M6J-09rEmqD zn$!nVmTp9rv;nY*^PJOnuH3S77akp3E6Q{lPTW6(8xP`;yF?-6PoslFoZAv}iF_^A zuz^~)GMB4^xVwi$KJP+o)FU;G2Tz~E*;7B^%AFMG^ky#6ZtIaLeJC`MS21kF6s+2K z9uIEZf=0a+!dp+|IaVv`cCiXov7lFL%>V8@(g#?ubKh+oTKOR+ez*b;&hA8nXr5%7 zC->3ra6CCDIGh<#w_FbKDm4n{3xVa{Wo)_m5Ia7bg2`Y0fXnB0A6gre?_^S6 zi>F1tISnXZyf#Fy5bL+BRj>My?bO*jCvfyiB=ot9p~35^M8_u|GU*jq}Eah6!;*E?|W%xzi0#MDiOTM)&mw=E;#{JZjJ!QSUefPTkiTLnWET0KqgV;F9oSx-HBYX*h!^La z@An?U-CduH`Zo{9?;S^Y!qao8!inU7$YvJX}`@cf| zMtu<`<_Roa8KnYDaQU)T-pcavpaB>#cRVr}6!pp^**|iM_EcWU;pcM3Vat!FF?jfR zG!8j{c2l;B{;v0SA9;J?6QaBl;;iUA?Q48JXF8guIDuwUMITzUQAMhk7rj1Fk4u$8 zhyESZ+4JSdh5Mo_QLjuK>Nf3(GtOiPGW+}OL`lPYhJD#7qQMR;1H+@W5T|SPvNriA zWeatR21D$qXy{!JFClZzf}*i;JvpJOonW;ut%w8Nn;nYi45`PwI4L+p!y>uF075Gc zaFghy8UjSmU5Ijt=35@~VrGXpp-4`*8!_S}BJVeo>o9AhaQ3cfO2hz3j>Wpa*ALx9 zhotv__mIr)Jz?9$b^?Obm!jPeO`+XG9LsTaxKkQ|%E0RMEJ|#`9^D zqlh$k_sJS}G z5Ytzq$bF|2iWXMpyGKLHp zhH)P(#t#dJi0yk%_+H{LdU6p$Oe#|^~Lkz=v%$RXqudGpE1qZn5qJu0^zqy{H`K{UMHWO_tLK3)t$ z#Y@q{ z?X>oYjGB+7alEOs?I(z2yFz2d>v4aL<$R+? zR;eupi_X;a&(~q;utuW%yn|&~ALSrkR!^CYE?=IxomN<^(FjVJ4VByU$IxNJ@cFuZ z7}zkUs2kpTEy<3S`mfyyyIn3Tc1R3%?T^l4&^OmGe#}^OeygZRr#RU;f5xpK9WHEJ z1G$v9W1)!r4Z)2&5%P^#JRTAE{&%w_vKxV^vY=XfZ@RwRun(Q9WkJ z4Pyok!RSdduxHb3AUalj=G!-MTwCl32r#&D;rcBxzxM+cUZM#>)r#ft?b1!6xV`sS$_Iu9CQpZZ zcdq&7Qt%)?4Czyd!H7%LVUd*{dE9#&DbWUqcF5P`^5?C?pSPk;%e2UWlMIB4stJuj zBl2F%FYA-2|FUG<;$M$wKfJwckrXiiwqCuXzLw89h`qcGlXk>m%*ZilQzO56DW^xY z6Y?1XUf#AcZEyg77M(G<9(T+=BukMII*$yr`18>Mt6fGIyxWn>$xGwqv5@%8BZHeV zZ`>}qpWb>W_Y;Z+At^JXd9U#pI(#I)-@FG6^9I50yQM|0&p(KFiE=PH?qTrA57BSp zX!IZ63LzG;?XcHz;QW3RiTDBCC$ARAuYVt8J~-8SW(r7y+Vy*2h$z?3R*HRZng$(4 zuMu@~1O|1lj{q^SG6d>y`m(4CqEjdbX_Cj+9?HONcziqp)8;~h?jzOuv2yDk)GePG zO6)z@e966kC*D_f8u70kckYTd_UFBrIBEpWoY{vtbGORsBL?b+@x~j);St+4o3;32 zSO>W3Ps7U3hKa3vMAdnEetSyRG(B?H%!$(ZUlXr`P#{MdgvW|{*p4f33()=5$JTJhre>an<%qArGv?t}7~ zQX@`mrsJ$-*u1bEhBbH#*F<`BGQcW!tJl2cVtQK)ZSXc88neKD^QL#DLVSz|X;OrG zgJoj9a{cN#P>_QvMbuMM+PrAdcdS}o8}=MP-TYyR>Xw+#mVoq?d*IxzO*sD9AS~Q{ z1Mhyk2wT1$jdAZa!1ZW@Xd2AwaVGDvb~zO_@Q|%^3w3{|e!3E?hBrci29w3WdkMCU zZI2D7BYySaZZ2PW$YVX+cMN!m^O;CfK%gjZah%E9&D6Um#C&#Fg6N9`qx8F7F;ukO zpRV1Hv8{@Wyb#VvmZ#5AEK)2Qe^)M9G(PeW$o4OdboZFAW*Vm)d zXS-oGYQ^E_T_`A3Ui26}MwIDj9Nhi`vZd0AcF!B!^?&i=rx7c3JoJ8f18XGA$ul8! z3NuQ#?5U=4!MX$JQ?H~L?A$|~NT&Mh1NN)vH*FmzO&EecUFyLk>b*|Xdu@*m5}x606-E&PnrdVD-n)isR2C&P`z{GvN01^Xj^#C~GLvh%T5vF#9YIETTR2_yW(8 zx1A40@`8CWZloxOF&|;a`p=L`WJ<0ooz&y%qs3dWq)!cT%zMuX-aex^=ZTlT(~rt6 z%14%vEH``NbAadw2Ba#4A!9|mD(c_qJ)4kT@9^FBD!Ik}%e@unB+<|+H}_(|8Y?CZ z>5sN=mxf}Ahq-7=oIkr64;FO6oE^7C-F{jQvR@*lFlIu_KI7ChZ#;Ms4FY$f=SsPr zerQv_B($RZgN;_4xE`gRyZv?Ef38!H+xR#bGZx1CV@F_^SkL|=+fgWaDDK>pUwsMI zDBKh0UiIjgbs_#1>?v|$&CKx_KKpwdT{jOK&OC(4@ND}KnkEOzR`7n|m#a*9IPb{y zdG{fWO~Dm4h!0$iaUx$Ti|SK0}(6ICa)kE)-n}Zw;zfT{aZp)XE0= zmEW{RpWTCTzUd8X=R>Xc$6@QF4j4OTB)U!g0i#F0h1iJ4q66s5GqGP9B3{Gq$2u`?p}fa#-EwuCu{t{AOMh zsM8f2rnJW7DRZ#p_!%UKZ0|9)J9^fCOLPuKVC3q9qT#=b$z#W2=8i*X^F~9lMz!G7 zZNx-It9cU}6Zxb(#h`+i4Uu-=o)@2pj@aI6?*KduDTBrt&Z9xc!5A^+1Dw5SgW`yXnyukKqFc95 zp9wr9#D-)3h^ZLTzXP_~-bD3mAsFy(2{dTXQk2OMtUYi9p?MpiQ_hoU*l`fXeexAj zRA>y>?wOc0X^MKUxkU*;o7N4`r&=paoIM*SZ+b*caABDkm<{>t0BSZbkCYjzps4K& zwCpW9G#}2u(cAIfhaic3)rtXG;T(4K9={eY(VvQm^Pb_{4l5!=xn{1>7jX+aq0h+C z_-Mui+=&+*ao-8c?XVyw-aEjsiTY_(J4d3dqM~3*QxN46j-mgg$r!)vkQi{pU`&Uu zcz4Fvm{&Iq8uVGLc1S%g2a@HkgSu(=ph>rpnDfbO-0}^QVq>BpSLqak1ApDO#>OJS zs-R%Qz6hV!5q)L4W=}!1M<`k^3T2x2!Qs*Gpr5E8vlq-nlqUrSw=IqiO&a6F>9cV7 z#tqStmP;sbhz@dq+3jCd*%TyhVH5uyJcF=+Pa6dUQo(x+J_IVkgp^IA--yw*sbyAqzXgO-GNXk2kvy?7FakNph!-)@ey(T7m6TwTP70i3_H zAoJSlZDdsrYn-T)-g0i!y#wa8Y^(-~=P%j-Po@f}8+Zn7JM_ZX5g+2nwfj%b4IZ~h zM@*brGh~_~MSD`LN+bOE%~VXCG6EYe{0yTf4y(SLh~eWGqLAoBrxm;G5qWR+#A4|} z(ODn=ISRa457}}RLyKGvbm`p}z58}Y^6K4@DbS6msF){pE7n)HWW9_OWnZ^*AAHyM zZ44MO5}$nenK=JGebA#kpV~zJ|GaHEMu~H`J*Wt(77oR_CG*5KXCQC!`pA?yH)^K3 zgEr#a{n4V8xEp1IHhpi>of7$g*Gj+?xkU^b52NSc4j5mzDTa+4hehA)P}3C?8wdGi>*$W};=1MO zAWVJ>aq`qUjGa6ME!qu6n~trJyZU>G*gP8D-j`|o0*~X(7&*8R+Ba#7VWY?3bWD0w z&7wo+0mCqM?qZQ1E7F#01?{c@=r?K>)~r0B4)h+LS%=}GFK|hh3T2BFN3UkZQTv@v z7&&e%R&F^BoiDQD7H$0NWvXD+XQS}rO$(fs*eAiHIMF|d(_}{9hDOwD-5n#xkH(T8 ze|{PW66b+f(JqNGrdlrkHr6555gqSVuZ|quy)VX(nvUDiPTbu8HKv|Ti-jw`#ZNOk zV()c}dgY^2w8w*ncR~LKO)!4eEUZ5C$UDdr>xozUir($WRC%0MQ8#5!mGxeno1#QssMT85f8U_lDhBaUaZZdF*c2UW zG{dkF6R>pM9=JuH)h|w_OBR+O`_4+HvMB93gh8XHV#b24uNl3Ze2u@-OnzCCaVR*tBJ!9#|l|L5yO7rX*WJAcIBG1D+};Wk7jB%o@)4{`pB zF6cL4Fc$1MEq2H8z2&6-ul2Q52V`d+OLj%$>OgDtlI-|L_rLKKM%v zo74v153Pe%U!1{!4@Y8ls|Fa@sT&HuI}#ZKbvU=@Q;}CA(X{zxptwGG|pGSW4ka(L8nXkTlk`#wcoNncl zp?$d}21xQU-ltEI_LiSYUM^}jsoOA$Z6r>cEbk{#aru;9xx84HeBhKuoHT59w>Q7! ze#^2-yggqOT#~veSCq?3JazKhAo1F17kOZM z8dPwJebk8c%Fj-&7>*O!`=fo4AoaEakythTPbq3Wl;2AG?p_f2<`iWp)9;^8UhN_u z0Qt1+yzs6(k0OR?IKkyp; zyU5$bzq6fqU**pz%3D|bWFC67^0?Be>r6~NR(-E!zRGVV5??_hJ7M0sZW03|QGexS zsG`l0>yh7b>$NUXK6>%5yrM~#T>uo>*0|-dW)g!LF`t;0xX)l0by;4@Dvwop<&Iom z;%TxC@~26Ddt;Elc=3e1JxK9h3M;RSk`LGvX|y}#v}gBO5uH4LnS0afO{1S$4+MER zvHV#*`R%jJQ}2GtU$yY&m%6NIZ|n}=b|ukimGk>+rrOpfZu3O@Dh?3er(WLU;c5FT zj~7XO^0#Pj)Hc-DMtLc|cKF%)CU4DpGLJ}y98}1kN>l5LsIT(s2fL^*PwIe3q+K5S zY94qeNs2fI5`UYn^0iB9+n)Hf{Oq}NnyfEwQMYB?_17zJ`FqzZZ%6X)uO#Rh&=S4dz>P_#F` z-oJe@P0bTIzfa+6cF3r=QdKi23D}L5a!r z`|G$QZ#U2w#5vy=iFJx)+`ey;<*%yfG;T4N(>`r`M13(lcg*rN{$^Q}eV;G*RrhTRWfQjAp2iWI!@kWKJm1~ZGg9(S0{_Z&U@uW;`4RlISyu_9*ueh zj@tJ4+G4Fd_lonIKONpYkmpg~2HdhA__SZYGa(gZ}fIw+}Dd6nTY-ul@6g&%}i1`z}x0u&1=5&uG*q z>eD6~dq`@2`^bHeS6FF8J#sij+wtVs^tR)P`-rW5XH-+swsjBz1q2I4x}8o$dZc%e z-aAO|NbdweMX4ekq;~>@-iwF?={1BN1c3w$NblrD#Cz`>_rCk%8{herk(0f%&)Q|p zwe~)9uAZ@N(J%Cy?E2@%><%JzZmv(^IM=Zu5`+ee3%;3K)y>_7`;>W)+1{tUTcbhfE6_j*W+iZ^Peu&0;b;7GkpW1h~Zo_WQrkx+xVU0m9d zlP`$jprE?x$g!-fE7zeO3mf3bxTSgIV{<~%jW+B}B8GpXdu#qAK@@wnk<%b^wgUfg zYJdh-bZg{B>Nkqej`n{@-(2OYqmz)3czm05RrgS!t9N+#LVH57!P*OOgVn_GVB~ty{ILaYw zLIzS_&f1MB5{=L1q#Hc=676Qb;+aPc3DF<^s!~Yd$l9b-brpW<@{=vDV1s_3)ujq} z?tS$kS%gJ)gauH?@aNGzw+;w#wCBIXs=oMjFE8n_%ahs+GPz8(==sb$HoUk86|X3} zQ-XE4_ee1~OBI1pBhu4HHic^3If)nzxaj2)<6AduQRr17SjZ*A>>#tBYMPEU?Cf`VDs+_@7?t zkgRZ<^R{plCAJ?gk(&EK#qyr z3;JwW&-l&}|3L)3D_hwV(C%Lio(-2a*@5!@dV1i;7X zv=Fx37(ZaS-S?{e!NiT*4zK&yUrpxTnubfdOundj8enk29_Y)h>&Tz0d@CG&k*y z8#DS8-$mmzBQCpxW=S#{&Swpn@VLRa600JOP3=Ix?p=~h;kCU^PjMMT!mk~OBSTb< z$%?sgLtb7!Xd1JOdqDwr?PPp3tvE1lISRgn9{_!aLa;EB|MYw+`Q^(8kQUqJFmqW5 z{0Zupb$4b+#5|55Q`4Q7BfC95S@FEE3GNilyx;C>m40cfj`WWG57Rk|CwCtR-hT1o z#pBd|yP@wgTey|-M@GkYRk}NeP&wrbCd0Fw;NW0?sIkaauk5`eY&SdYj&BQX#10WX zj^Zl(Nsh`b9j>cTd@SO zbIeMdYJ_XsadTl?Kh|qrC>FJ}{{`n0a*E)-z|X35$vX8n9{TFZ$VlJe<)uJkf8pC{ zLI*x~FvSU5D*Wa$%WhLnPc+dIw!V2P)A~8BYf%k*D$tD6@t}EGuFK)Q^XqO-iZrl1 zCbY~+~c%A0oaqajc@rbNo``OU=CThy{-+q*bT{x2i{nJQsHA1}pPiy}^8|Hjh`su?q$uTnIB-pvsUUuv>H;vgvOR1>)%YflGu5s5kV+;<|6 znv)4*k*RoP=J4)b7y$((-JY>(1>tiCHEz3~YPW8&(^7V(>zeJa04t<(-WB5I8BQ8w zdHnnePoz&fIP1LjeU3rSR@TeOlOmoOKCQ=nJb-d4Oj(PKTO%d04*Sjbh4~;3Xs??8 z%)QhvH8sSCEuB8wH|h}3)m2*aSy}@8z{#+|@$6#N$YO|pzx||PXQkw;WlW!JgV2Uc z$!T5%NVqfol4yZevV*v_p+|gF>J#N|ckNbj`Iwig#wOx?eEnT0&#C9reEW}oH8{Wm z8Hj1RsK^K`PLq#cPxY_hj(0m5XfS-Vj@-H=>WtjEVhkJ*I(ow-?BZ5ozWhYIvN);IOYr#Q;+giAXtlr5&>`* zYGz6cWy+te(~K?IoAz*nd!mb98T>4HYH}j~A_Gz_O9CmDG!}8`HH@4#OFAND%<33y z*5~$<&tij+F(Hi%(qD!&YU=bhoNTNbA(Z^my_$v(%3_vAqMBor(ZDfR z$qS^gn@}S#!oU@s(7OHlk)wzrAI!z6_LM?oYY)7%e;9u_{{p^4n+ZAFdPl}<94kr} zJ+|q#kQQPr)L#oXFsb6VjfAb$iMETxN1ElkfLSLdflnEkH(V)GIl z?uqCJo0Pp{FJ@O3k#F(y(lEl_1(^+O#P9NlvJ4KiX(bxFv~2}u_`lI=Jc(Q0ZQNxl zzCU={;yQP|9sjo#-CZ#(?&aSw%B#)hQ951giBWkQ;=djny7D}_yL$#WJTCkpTgfJOZPgKch-fArN+Z_TUkEFYU zDx9!tn}bCgK9Mb6=iCZyWKqY~t_O-hc?N{_T56=Jjeb3~ z^zRqB&Y|6O5=F=$uxQFj;#g!&HIw(wtOq+W`<#P+a`ZW`+ffB=;EqvGzIh6WCEp~& zD{Umunfo<*vPPeL#xt>Yc0(G|raCdkdgBtev9%(o7=m z70L|@c!$DVLPkzLEnJJv3nLl=M%jMI-L*EDuM(4R2|45A@uscz1>fJM4H|$L?c&55 z9ZARD?tCfjvOi`GMaAzV=vSNNJ=*=on#Nvg#*1OtD`aeTenqGxYV2j?`q7HsqtQud z;|ibuNg-okfm)LyEmZL&pk?!gA+UKr;b@nML+-TK*Xx-Cv}OAv;d*sTole2Sw+p#uhn2=UHD~qG;y6gwVT#un5owaK7=pllFewJ1;w6r3tAMljj zcqm@O&+g1<*^^5vFVZp(qu8LR-hRW+u3();bZxS7cAkk^fPh*c}<< z(DMDdVVX8SV}!_I4C86PJ71?p-pxLP7OEd4nngngjj)I*ZR@#%;APH}x@8j(hG-4% zw+TTRo)jPois+}j8*_&s_l>JZtNqM;8$H`KokNlCNWZHMX`CvZHd6)*VWrg&FT@A4 z;lh*@Gh#33lk`$0B>z}%%jN(r@?nlnhMy`^bE|Yy`pXK~>sa4KG|^5^6D#J)3+hgd zwJFm9991`4VvVs9td*z1u!2asqR9CS`awsMk9Mdx&nMA@*)JYxxV6Zk?c=Bfvg^9B z%J0s8y8c~9t9&wYMU8FngFsJN@n&&F#j~Q^3TSY}gk2(Hq)@MIN_sZbF^_H;n^T{* zvVujQmhMuT~y@aRYu^YVnUaj&mK)7mJj$+^imRAtt< z>62?{@ku&-DL0lCJ)2(n}_8vv=8n_J}DtbQ7PVJqm~Ezs?2FXdb-%35~S zBE?>oi5|KZ&82|fH0r7!9V6!wI)ZYFa#@10JF7b*)3UyDklMmNZ<4=2qzS*E&0z_;@3 zdGtMP$!Z%8;L|(5LX{e~^JIL-C!E}!6Q9^SSSu7%9>K?0i|xFn3ep2|zI5!3MA7-h z2}JGUvin*l^#X`(XK*eM30Crpz+{1e{xXnHQShvg4y)lDm~VKC0jjeOTlx^|tJUN@ ztie=T!T5V9Wp3MwWl7xy+80I}W{8yqX)i*1WKB6NQ z-bW17|9YYY^fP)I%~)6#z(2mL0?zCJHt2_>xyano!|=O252*5QP#38kE?=##r5BQC z*Dm?syq(7THCA=6OETo6Bv+2EaGCpnAuH5EI6tn!ghx}p;A2=UHE01zQi2nHQKq7p7VMXn-jV1&jm>dyiYkG?J=FVg3i_;hjT`Q3mvQ) zXm4rjZNc35dU`EVp@TL;Wtt#I2Lb)>?m13PKZT9FdlYB)zRLAjwrZJ=9K=#r_D?Tu z!9>+v-I)q7vIo0!a^ZX~8?%=0nd7QJ)h^VjTw*?AN@Cv)jocPXeYa16e7G(BV&(ie z4$;9lSNsYEoO92$0DgL#p?}W{ip)XHRo`6-d%H8A=pTs0PNfkTDZQus$H6u)3sL8 z&<0OWg%^va^V4F5d>Z$(3+IQ(nl=jYY9zEgFnP`W1ue{&ETw-c4ZwTr#m zLb1-HFUM2~^KBv(7=NB{=K6cg2GxG-U9zZ~UfT7=z4&9HS4HAaqjz)DTgIQpkxbs^ zj6<(^!p0x$WzfZB?o}TFYwOn)Gojb`?BGDBBI5*=d*h&AR6RT)Ky%GJvG-x>rf>cz ztEC%~sbZKd8sS)y!MYdqc!T2mZhc&Du2%Vj9TaB2XqaW=m|B#ASSaVyJF=vj&=a33 z9jR>Z?l1fMYcCs@%99h-c)`8cQy)FdR()CzMIHO?CU+Lp{%U-|1_B#6cPI4J&yUAX z$)a+~zwVT;dEA=}X8%zj3f{chfvKb$oTf*)MsAI>Anm-qVcbhRno86Hw^E~b5(|vS ztH@_dVR%HSNdslg(_$O+oxa2&r{=d&W{F89n^U>{_D@Nb7kwrpR^mBC25U;sVW5qH zm9uxC>&WFpFXA2+s#(Pi|A4lcUb?VE z#S&l!0`gpWM#@XHN}drxp5eutKIXIER+ijhsBn@!?RAE&nE2$%6`+S-mr>Mu-)qD} z;8a;k>TtyJ{(09#1r4>U3T1_K=c_*8kh5Js?wfyM$xcAV>Hq3Ov;AeVe&R6x61L~_ zLtjW4?Y?OBtUL+h=f5fVyR0~o6w0))WaxYz1x00XN`+plWBz>yFjRCMbp9cz@{arY z2Y{uTOBQ~|?Mvskxi#m+KbrWf(T{yGtB6Yg6!JEUA)*iUaHco|4see%tWfLHZ3o25 zKlrhENKsUTp7DZyTiGs#LUlN3Ku%~xrTf6WL|Sa?G|sztS(iVbrQn`cv(`rlPVN#S%Vb5^NJP*4LoohmjeVHOFb^3>PkJ){mgwZLbC?EO3if2Mv5B z{fY-z>JoxA=9GPzN@}6S=_XuyCBC`;wzW7jTQW4kg`sc0os_=s`X%utZm|;F-z6(1 zCF97S^wUZ>j{A=Z$2LzbT~-+qFpn#{jLL^(8mr5(?-WByO;3aWP; zY{K_-JKvBY<$eJ&Tg=60qFxpIIR^g1BAYfGP(XG+mH#!NZp_$KzA+K)`&LnseFjZUk1ftnYbi>gecvaw^?7 zpOhln7N-u_)7uo7%{y;lp<;uf@+)!ic2yLS4o69JrQG{N$*iQ0*UyJ0K((Ro!`);> zwitN|zrku4oOo%YZH$=c>Ur6o-k%&F1}wBg$x5?IvOl86+T{#c1E!{@yWe=29~ z%CwxypRb0Tk~$t#Q(c(x7sFqBqW{H=w{f%OUX1BxXZgfzaisJQkEs;u^N}9pF@lyN z%0NQxl{0e7L<(3n5L&e=*kaX0r=wY9;!WZ629-0iJ@_5H>_QqFWn<(cgYnuN^6ryd zw%26X5g5GvX)dTjx#s6`fFLLTAtV^)FU<#k;WIn19M9pYMHfu~iFjgv54P93(BXuO zWMRz>fr7dZn`#cN(p$xKYdA$1=+n(qh%&-I|Cy<~YIs>8GVLE!ROUP@V9^Zub99c` zQiJvCxS;ZTrU~=|C+E5xf5f$1b{My{|^V&h)n=yMXrVQ5rRj{QpCNFqtu zc#K=d5W_~9<`Kj?-ajC#taTB(qy2E{!W6V?92{ z6rjgRqBM8JqV3M6A-t|zc(kSvM662N{VpA--S;k5rD=QTJX$2;%Sxi%&bu2uU8_qN z9_fGx>+L=bl^(P^TcScS@@t4Y+P2GXX1@HyAT3k!MFU}(uSVTWmH2@8XYV~1Jr2sD z$@9oHh^(+5*Nlf{2gFUjUf0;DBH3soS!YCS=#!0*{QPVf6r_du+UO!l#t1Uq$2d7f zE#*mB=tr2ZZKx4{xYnkwCc6%$`{$pNDb4+Wyb(Pdbnqb2fn*guqnV69@qWd>W>BDZ zi+ub!6IO?;sei@~{bz=V+uJV=-G%ydW#i>Y`R<|#j#NgVqBUHzeh7`8rDDE7`O=hZ z7mR?T$-}e5h=Jo#k|TIrfL5BV`oOf3QHvU9=^*;YHgHU+>Jj^mNVe;!e1B2rIHM1} zG#qE1PZ=6dqYP^LnpzV4tK|0$!3i@;+})CY78mh`>cbj>O=*2=;{!UGgtj6hIdaX! zxeyObFc|%=w+s#&s8=t_4bIscuV01kEG0%x1t6b=J-n%46%4Z6Q#-;PpNALJYK!nE zXRfUWXyX&^YH}s2-J{ZAOXKDtvfLg)MIN)a{^aWyIyPfy?Q(C~s79wQU8~gNP$;;G z!>-H4Y`3Rji!PR~Rxkeyh4J_c6qu-FE)`NT&NL`MeE5muRV!%qTkK zIvf{MQyiTazV>6=a~Fd!fa80P1>%wd@qKI=N)V@m%l)6Kkn7i)Cs3V#2U^8##)L%ua`eq*sSD(ki3!HwC-N$ ztfV12jpfj8Fyz94i&*7?IHR0`SF`AkqgHzn^6`weRy9w5xC-{|WKXbX3q2_BS@dAf zc*YvvO^M(UGISD(V6Ygmr7YQlFA05uE$#*1+|=B3fle1GvSleo;k+I08uH1F*sGkz zXg|0a(_v!|EyZsKY6)eXk7R(2MkxcR`Y`Fj&zRr)>-zuQs}NiiVF|x>6ZCTN+4;%< z%DCp%+XrlAnX|&}DB@s&_{!%M<1W*2O8`Lg<^cDUvYaUJG1llIj*Q7)RL0Iue<>l^ zlAK}duz7WDjWS#ohc3O{04U9rh(Xb6FkV|5F9AT@a-Kd1$$bKnk zc!)zhqzrwz7M)>~#m+XW{IT4Nu_NjL)Wjzk5#Ll>wVoFqKvhDp%jPM=UUQ}@izmfP zKjC7sP)bt5@T`s_Px^^FSyHfr9O)dj&PA8JQS``70O~*(7Ro;*#v@Dw$o%*+YSPfMT^E+VS~A>B;!4xZgfI|LtM(XNRg4*M!(KonB~fv_vr;i_DI#dY+v3NCViAC|cNgpxw!38k{&I2R19 z>@q(b(Ivb5r~5Uzcq|?h_3&;Cz1Tz9L_#Qgkjw>?WPeaeT;inOowV)}G6sDqJt-m{ zN^bHv(Tj4T+c(1<_zfHe>!NWF@%zf0eH{AChVufy%s>4LWJYfM1(`N=Ss|V4>W&ZY zW%O$MS};t0>U2F}ltaI&*(Q(-lMa5scV5xSB~0b110h-MAT-@HxSDA(wZ6!2?z`aCpX#9 zvZ~f&Lw%}`2sk~dF2)HyF8gD@F>-0SmXG*I1mZks>sC*}5l=&=-EE6qF9OYqY9Yp0 z)QvbZ22I$ySP*6!-*k~21$w>Bb%J$PDxvVD+s9?Z2Np|nndi+d8DGu{nMIHHkY^yy zZVBi(CYI6G9}YAwz9FaB6AEt+F8ywv_VU;IDxv_&KkWpZJ>Fu5J<)M72#4SGTDE5p z@@V$GMq8G{D9eA8@Lykd(0}jH=KvQ%1a*)wmg?;Lnc9;)7vIKIg>H!d!Ixe zm*E!sq%6qc@*n?BQR9N!$DfWJ_mN|QaK)CXJxU!u8HJ&mlQnAQ*_-Sxu`+6QuuLs3o9Fezx&WukG@E0SbW*CIZt`CXmG7~SSN*hTX)G< zWvm%#oR8OlwSh@6voZWE{$jn&XTP_h#piCF=C#o7y9ETtb?6au_ob87)F25+Au9&H zqP$f?BE{gf=oF9jQrf2&Qlo%&R{gzb1K!1xrxc_ok3XGTJF}`dsF&SgOHT4=8pCY! zJIk3jmT=^7$fmSzKU#;a5(fh7P!rowLwN z;J$l1I+QJ~jxDVRHHuiVPqwS>eix_@zYZ@~B7BByEwdJjLifLc%FmTG9xbQxnZvS< z7OzecVilZ83%NyromRbal}r zVO>8P&w$aWp%!Koq%M7|88U#F$f837H}W*)p&w_h;;VY{^56PW-s`AdOL_&f@^RBx z+7g2AZ4|$KE8DqPWcFSgB2%O(v3#A3-`uBqc*ZGiAVtIgA5v+jm~T;WdCAKXJ~qMP zmm6&tu^Vwyi-jVBsoSZJvdg;sDSajN^V7R-dK#kl!@KhRpOn2l^1U`;RcV#cnDGD72LSP zP>8axv;R>xx;rDL(~V}Ct+?k5Bj6&rw#YsIT-$MFpD-!qap7*r}8twLMV0 z(?+(mQ$=#FlfUm8CqY?`QlXN@392SORKOw8(frlm=Ve3vUSh?A zJa?6l&g`8h=DaI~6~4;@Ra%IL*;l&;HJa}hNAj;{89plY<}hyV%Wf!ia1l+8VCVHs ziDH~q0t;h5EX=akPGjXoPV=67L?ylb>Cx=Dx<1-(y@bXsx_j47;cf>o={gU4}|gPzE;h{ zGU-P^FiSRYPOq>4yXoA}bZsOs)+<3>si*Wvv2F(YsWwSaDaTKTwQfsec1wKwq|i9< zFj&iXB{_6wO71z_aY9fj?Xe=e4!?qSQ*n!rh&%H1s8%%xJ?b?Y>?2U4I%!hL1u>j_bfjh1pM z)~>P^bxSp%h=jCVCv?s)-?K?N6_I0E~@>uxXr4rG0Ilg$Ki+iFu0;+Vgl9W zWlQacrQ!orhf^Ysh!2GR03gSZqhcKX0o{pLV5N%2ZKbl%lj#JaOrPNVE~{RJ*_%~5 z+PHp;zJuIq3P4cL0xz4+{7P>BHVwc=YD~wmpko~Gjbuem(Db63hhT~6V_i~)@w&y& zxVZlbjOih5BpYKidzx}$0sx@uBP=UJazejNLJ6>W{=4jb1fCU?5SP{xzoU9GORK{k zC*p2HgJ>sZ?EBnKdKUmF)U+ZXIr(JqL@(u1P_0A&+!M!3ZAyqcFlMA&P<30pWFwC< zzcTb9F@i%?lM!K52SN=FwK%-==gD?J{}KHJDb(7*i$^HRJ(uV80YeFeP#k`9D)|>4_AfTc+YR8G}`H z)Ofvq)=7Y?{`WiQMa6xR(+EbA`&Ngt_~Q87A8Xv$7Dq2$l;Z(be6y5ili8=qv zw40dk885ticF_=W+qn$m6ZCw$S^RX&Fov2bXP*`yx1Zj)Q-75hp{x})e_mqf>hBEg zTIBVLw+RLgxD2f>e^bj2D!QU)ZNGQXwI@4xq^Le6`>J`5L+|v`_VEu-kmk3Qc!jSN zO1WVM1d=y|h;8#o5*D@`Kw0^^3a#Se%1t%(P8u9~9AXqj+H*McK-I;6V;XPd@@ZHNWu~$7KY1xZj+1YrgjA`Ak{pixC9r`>l12NI2bZ98MsjJxS zKJNXI|Bc-jw%c8r<)9z#6OpbOvJGmM{rH!E8`~6LK5x{uX*NtcVz0L2F2v)sRT3Hm zoG&HTnRMTO$ZQ*%h0kkxz>?-ZnXyFr)}l&ish4wfMuQA7FLBE29=_SQ)tX32tkunQ zWwlzQ(Izy9YfQ;{P%|Sipl1J>K=+!%SV{ME;r{#$(Bd++rq&YAxZ~K@DNvdlrUT@e z;Rw&c%7Xj{B72UDDX$!O^y6S?%?fYcJqd!7KqJRKuQiXLpDisZm2gis2VvboBk(&} z2H!Wn?$N03)Pb=Nfg)phekPc}gCc$H!r{6>!^~4BA6b0Aaod$wjI(DijGcvj3YLvs zp_RsUpPbrgMGtXryw-*v?V@u^nfr${uqWDg+vG&|i7_L2sZmWk+l?I%7i1QR$&|V%cX;FpvjIWbc$iD7pq!E>#GfFmNOtLPh(q(j-)ac={wu4`Q zekIU0zF7q>)w>nCN}iaWS+(ISqh*{ZStCLbu)%sdDxzI5IHO_Ga%>$r>-m;H#lL!w z>8a}AjBmQJ!D`&7lMW(%c6TQOs$p^R0%Sd&euV6Df=1n^j=XHi&JZia z^j?!Su>zf$6aBHN=vheZq#Uy{QV;91RhQz?9ox9pqNM=TEBq=(wOTyA0&)8Kpnt-* zFlFHxgYOiSuXcbdeD+}mf-`lN)vseZ#q-yD{ZE*JZD6ae!T6Z{-uGks_5z`XG-VI; zu#vN9U?N8d^XiX~N4T>y0mcaAc6D=`1;?MvBAY5*C>fQ|PbK|>iC$X%T^l1eSWJq? zvP>fe*2K5*b+%?FGIZx)Ok zOjScTtHnA!Q^~HRCyp!fr67YZ`s`S916AhZbt*xmlA2Gq-e{ESd#iE(EOg1UDegJ* znBPcDs9^ApFFc}7zdveGuW49(+*OyK0v#P7N>LX-dhK4TK?<| zv-c|QSQDh$U~9ncE^-~wYog0*hPqp;e2%BY<%X9{@|bVxrTH%lRR$b=f;o&y zi3m(eI0M5V;Eh%VLfO@=ssdExp-!>MAn4kRt4d1-W9Ax{;ImB|!y*gCO^e2cOiAPS zx{5^+lX9PJK3k85buB;yD-}ms+Jz6qk^9hKo(Iu9IRuCSP1Cxn>b2!mmfD|uX>NO+AO|S^PU~pCa{}~< zSLRD<$P?I^7tN1hN}xYyXk}PuY)|e2tsq*Ep}{gOP{7Xw?g1GklRJ~GxxYU|!vkYa zkf(|kNgkkxhLf6q6iYhK>1Uo+q&>ABc$0^&v~{r2U326oJ}miU6ly;%;m2ZV1nIAv zRIUQH>OZ}9wK%Kd*(pC#DpYAs8E({QJL7b`+K4pFeaBbq&TrCul8OAlJ>?+oW$7xU z_cJi^HjjQlP7H z%YJL3wYZqB#C+g1(#nZXe0+7Cw!s#>RAr2L;zH-sG0W?LNUQ0HLr)c(h?@YNhn%Vg z5E9cn8u!-6Wjl_GkB1yvp-&0)-Swp0wdUu@gSrXZ#JF^cNNW-g7RQ_2`M8u&v@4CSR1D6nk zw&kep5(j?+J=Md7q6HZ`PRpfATU=uMcZSk|$}++u@(rw-LM1nXkJGg=#n~-c;bEi< zm${zARdc=GD%-i}einA~IQU3x`@K3ujwFnwO_1wr%J%u%6lc<7v(h;0By;bE5PejQ z=beK*Hmh98sxGN}Hw0)m-ZSBHX#QLph6(b4)PhMP&&*HF=1qIpZ$yJPw!$J7VEzL$ znIA@ro!hhm#*zkeBf4&hU7=T;99cv=lnL5cG*bvB>GwSOy|^$(n9;x*ldCI?xL-b| zjN?zF8El_%e&nbO)`=?e5wp@mwz+3#T5L`jTzbp5at^s{z-;*iQa*Z=VVaN+5kQnvUU(*P*#J8&us z;lQx_#`@<$g}C3>6~UoC;WoUXLsjxGV=-%hZKU_Tm=9em8oBxTTI0GRiqh28*p+iibb6m#9fbsAmnA)clBlVu*a!^c3zew&m^ zDs&ekSS}$M8nG$&UN$GJYf+(KH18oGXb)ztDGAU#ktDzcK);LP0-*nzP!&J6&=v*c z{tYP7423}IY5#XXQEgJHwh3UsGsyjvw8iw^(2YyYvG{$FzH{~^r$|6UP| aJeK@Em;|9b7Ic2bcM<83KLl|C$9Z|?4a*U~$9lI|Dh34Cm^a5=bqsV^SgPaL4;@c1@7a9r zS^2TBaCQ9p9q9rWJ!3w+?0?tN|2D+g(cc{mx@79^?aU%CC$Fe3r>cHK?a~bubp=Ir zMLECo7P~AggRJ-O=$HoEZ_uD+=gqQKeHknQ;q82yMQ;>$g2@k-+(9aYV?FUnFJDj< zJ5GhhhsnQH7L|W159he`M4nS?O~bq5N9M=p`Xr9DowYCS2BIsXd2`OI)IUEwXZ-1; zz?JFrb6u6w;Odm%*7@ThA?OlovwHK^aZLT*Bz1!*HrX`kN({BCN0*d1lvBo1wpRx1 z&NGMbtK2l=_oQ?Gea*6Z`FPZkzpryx&j|jSA7*(je8=a)-{EIZ;hl_&3+w+W%o^-PK zw(Q@vS#BblRH=-s79a*ZtbKkiKffUH0u=*H3Hs;#bCB{bumP3-FnNiDLqTx8=Nm)E z0J0gUxbVPe+5)y8`AC2{wz9kJSrgc?cqlzYk!yL##c{qC_w%j?i(%+hnk}uZ|CIxsqkcHNN0ZT zJjWlR<;VYd6b-ST8IQieWgq?Dj2X*#Ka%_JGpk41-^{c8&#lh?=Pdv6e^U9!Z-y!n zalMI>khNivMn?v8IX=|34VD`2Z-dN0g5KE*YqW=#xWJq(E7ahYAkFQFnZxgT%O-Ug zhh8F)nEZa(VKHh2Wd}%sN5AxiOS!seUQQmAO4iU;$`;QyVqi&8Z&fIaui^JVVbscJ zxs1wKh48&22Rqi=ar%7E`fzbeo(?S%h^$bnisq-<6&(}co~{uz>VO?C8Wg+Ku%R4u zs87!$+Hc1NEfpZ>OqoGW6?RNHx%=TF8HX7e&7FrM+if#Hr-AX)gQ@V?ufU{{Jz9AN zV1?niu>zM=Fg$$Eh*MrKXyf=b>vEy30eehgs%%Av*#*r8$ioiz&Eo{y9JEvEainH& zS^OY4W&Zf<`cfzAQmFv_j&|DCJ3&UL2`T?Tuyw)wAiQ^QX&(xg%(e?FeQ6=CTh0av zn8}1}6(m9^S;+W74|gaJE*y*PA=0;%)ikyYixn^&OHIWNZT?3)rZP#ma=-qt-~9iv zW_^MA;nzf%y>?o#YjF-X5f6yxE|g6|En`=YsFWwq+Ykx;&zF_9m+~rQ6&P-vJftC^ z?WMD9Pk*>%8#V#!3HELCH>yGZh?7v6v%824W=lbBEC_t=LK~^gHf(nVf=|_wpRei0 zZDA7!#ekOq*r%+lQ8>yX!(em&+aXWhv;kbKtFT{TnxW9Rxs=_TAQK~@<}T7Em;ekV z68nKi5xtFVWcl#)^WClT*WzPouV$XoCsRNM_n&Vz=wp!J%y9Q%CEIPB1_U>iBg@dQ zfHI751GN~SuBGdj#NWi_@O;TnbTjsu;1ujmWHNFQE(2ZZM@lDyq4tItZM)7)xQQt2 z@RR=GlHL88Vogv*c=(D5X_9jUH>?_;w$OWme{amOagAmGK{XE229E{`8)ss-X5mS) zNP0rJFVtt6=1Qs`gv-Pm9m)=EwLDZQPvkC;N_s!OKO*e6CX^aXseDE%3>1E*(k%xI zemfKD-paAppIEtpZDrKy)Agm}l8*(Jhs*oE1`bMkWUbhLYKyP7t-Y&e!@Ub=Cs1JGL@ntDs*=sR`pDh27gMOQ9yQeD;aWbZBk#9^<^a3L$$bGF{)>hmviB+txs%x>Vo~Ws1Ge$lU$`gjL-K zy(8dB)SC5?W547s=L`X*7LLEw_~b6I;=td}of!3GKW76~!WT*}U%%x?IfbK3kEw@I@(kmBI*ROz+gu?TUO8o<)I zNmO_4s27feb)~&t*AbW~VCI?jJ>RuuAqXXwwfttx!K~dN>L>X6K*hmHymj^z~JET`c;T}rLm94e3@qw76&cc~G zrEwrU#ih?*EsUsahRFT1ssORL5~MAQL^?Gce9^Y({Ok6k1}ZL|zbEv3;ykdIxao0) z+&fNJJc3elp_ehg3+EuFAK6l|5-7XHoc&qIYl`s|mJIMg6f8 zrvMY}2%*L1ZLa+`P#zh)9obtSa8UWK-XoN>m7NH9v37xx4519XI&oks)@^|}~k@ofQWQMjV`z00IjTWY&RDjgkuK-scz4!8X3 z!B5F@)2!m(K2TI*;3Lm`<*zts2;U(;doHeMo|vlfaztK72DATiRQU;bMvHV8yG`oh zG;jUtZ@rN+m~4VZO$x}bC%Q)1Hd&2Lww;XJSTwbf18XHS%tT?qxmg5@&vd)dJdb{} zoEv;}D-b;B?V)iv?1WmX4VIqNGig&Muhx32ts-eFE*9!Q(dLfGDT&=FPgZbz;@gyw zDOi`uPO6Vn>9*?{1!F?*Z0@gS%3TNvS#ao$ithuSW@?F_+)f#y;d82!)EBsYIW_>4BK*wwQ;61toiy7kwLJ!wzssNl=0? zq!#bYZb^%-l6a`rU_9jx<4hwVey=_5TY=PFy$>r5RNB$- zkK#=;hQc)xIRk2C2*Tk)M$Y1RZ&y13WBnDQUQ}ZYIvh@0QR43HGJ!;xT40j#+LkLP z=xwm@AEz&fFBYQIwi`ZqQgOgd;9T3Lzz1&Pe#7~?rn&sKHwy`AC-(<=cZIYjVS z*YQC);%KP-#ZPQ}H00{R?CzE?u{99a1pFA+PgRumhO1~@=Zn0zlK9K2%i1M+e1+Ts zsc-p}W~W?U0aKGg7*)2XoC^4yXAThhKS);_={s@G3nz%&y)lsaWk~zPdy}kF$JVAM zWF3^qw-UFA4vAal(AMw~9`hXayPUMhzz`CmOnU*E>a%OWAn-*(%FnkSuD`x+rkmxH z=|4v^39~VHH1g&srhd$1(6F(6U2Fjr3L2&v+qC*vf^G=4uc%;n?DYlx)?P+4dW{@! zY%Un4lA_=-lv*~whyO2$I-eDWURzKKxC3M~hx^(oW>!12Trx3gks1$k-2n8EJK!QF zz&gXMu36daLN3vkOFY^XcBWA9$Qx=W;|++_g7Doa$jE`*MYCbV(za*bEkVCc zcu!{G?mjNyplXJsL6g(TWZnKS6C4^jv&#z1P~40J?nTCkUgwV$PIwDZdbcyWys*?r zN(`xtG~qEijxUdt2zy2fgNyJGE{*+FcIMAe4O*if#rb>LKFEZ!{2LxnAl>nbUXa#wWjIYBNQ83O z%ZX?1pN#m7j4j}^eKfIm3@8BFqG| zC-eBAvXX&0u4l#{%|=ZUz@ao8J~H+PHI``2^vBwbzAZB(;t+Uvd#lr64^~v5~QWG^bePF-cIWFG%n}PUNy=*SK+@g%0_saUqDBMmTLmJskB`$G8-{VlS zuxK%p6F69odDE>Bq$P`F7%{FPTj2-} z>?4NU+F(WNNQ;R~o4AVD6(qQ)Kk$Qu-*KN7P*J6Idm5!xcAN_wuYsXg$`edCK0h1B zc;u_3Ue4~1j?rxM%p&iJlgz#gR=4GbPX4u9`iKEK+_3ZXFG7ns%vL?p&a5#uoDI4e zi{0iXMcv2t8gTv~?FMaTJKyr!s*FZj3}9;>4;&83;$CvO88IFn%uK$~z)@tKpjah^ zB~&`oGO6V5CT=o4K&DrfY*ZlMUhi;h$lVZG{SP{}#Dyr!P&5l;34j z188;|uswcoyfa(P>%;;vK^C0pi79*!)|(aWdbXT;YIJlke5*wNlwO8SvB*ZkS&LE9 z+Nfk_004O&^&XyQ+Z31;4(mD@!pNG%e1AR?bXaY9Y|?K`OY1lDN<5hE2Up*WMDoSA z+_C&%TyD#Cr^kb_vp2tuRlDz*!etgAdbUb`Tr%#oRrAK=8>7;1P|BSvV8ZP3is0xp z7d1q0p-E8p7S-XSNlz8?AM%!nq=R^NZQ31a=bdSjoF1hCSSnKOyRI7J9c9xc$%5CKdrD{ka7<3Vp{x`RQ zs+A6OA1u&{+x5^Eq{iGIzNQ5DN(JXE0!4(MS#(aI(D$}~u<_ft*mWFEbxM?Mnvt_6HuowwxLXeUEly zO@RJYMnI13$r)CO*Z&In`T8aQ?bm){Wq%rF+kRDp?^sGu8$kTB7A98(5b6k?xW-}2 z97MdGu)@=icN*m@_Fp`nN;YBWyauN_#(1?}T9o1F2>^hT)7#vnj@%N~kbV)Jxq7_x zC|_pF_|+v+Gi0$CC+|Gb>!$@u3MY|~DDE9w{!jPk2}|ykz(ny>wgx|2`n}v6*&hSJ zK%qOWTYGR(Y;aiS3}p-1J1&#YYEo(6tgqb!J&srm(RysZvMZE;Zwm+J)o#T51(n6= z7C1~^MIfmTID8kP=P%jV(k2h>ApH;T5*n+6kN7(-HOT@O$)u{dP`9 zrZos@vA6SExQcR>98av=|9K@@Ok1R5x3wz?rNYR}kPG1*oT1FF>gcUts*g(YEG@(^ zw0zSKa5eycfMpOpV7txcyPlO1Q+;JV#UMJ6y-Mg)1k@tqn)&hwj+=2GyUS2e)50N9 ze2W2do6V%2zpnRpq_9`^CO9$73K&#}?gSHA)Jt9tK-LrCsU?1f^uaY_#AdQl1;WHA z!|i&&J-Z6pliC@#LV>R?Ssd_1+1Z?3i)_6kOVuN|!*asucMWjrD;Ypix{;Rj2%QKg z>sn!9p+7?IrGWMv>@OQQhIr4meFw<& z7EYcv!8B=+;14!YdurTqBh&j&=S5*C_qgN=xRlx_iaI$pGj!pb&UVF#PN3+@NQKOj z1mkCGxO7P&A1iLE%6;I2Y~hZcxQGl=M8(oBX0ey8aVrw}sTo5vA?kz%(P7Q{hu?%a z;UR~b+Z8XH3R!$z*9BU0;q^Q<7ex4U<*|tScA61?x|Bu zYrmnMt{$5ylWjm_Js9mJPKH*ZVpei*wS>~($$Nz|yNFbT=3pf062 zm3UGU$we78B=PKwr@9^XFbmNBDiOE4r;G|P|07@n zSGH>FnTf;$@B|)`!ubWQI#ZLvImEUw^x_VZk#-?`{F?eSf%6 zuuD%04>qcTa(8d>H!(~qGCp_$L##j=bXQ2;`Bs2mzFU;(zYDHn&16?6CYfIsV#krq)=y31^SzbqpX;al&iV>%EUcP^=oWtOEwN;((b#IOAQoPnTxn zIT_lCRG;OjYMuj`tw}a9zSS{L!0x#3vMjDoT45?4F%F?cF%}XIED&Bl5mrw_UgMhL z?9CaBd!5$pEf0B^hB@VhjZ-12km>wqPa6HWsDKfA8&1wuk_SbEt2rZ7vQ0*^ukD$i{0JZBNU9~>w~TT~ zvdLhVP+0Qw%+${Q!}i*v0BEqu^1aoE2(P`Vu3~W6U_bey`%|jYl5=CvRG`BwsqAom zIqWbeFKvYn%8mN|Q=7i${H?2;J#e{klN)`EuM29UowZfOFJ9T}_U-9G$6wdKccjR2 zzq43;&5CQ13oU8Dj*p{UMekM0zJ_Y9WfdWt_mlLcJ`yQSdvK6}iP_aLV8#%)L^B+5 zNSA3YyHjznM0AFr4X)MSeQnYv4t34@7zF7@;5JkaFA;wl1U0o2s!f)1TVDV* zt~q#KpW1Jn30{bTY&S+|<|T98$HEVT@0dK8=!E}tQfYn3rEX{=c(SH~0o{mr@Xj|I z6gx(?aK1%1I;~huyQsLe#GOapiI_V^($ETh+`rwS(dONIyBldA+JoGyQhlLj3-aER2Bo+tZ3Kc~GtJAN&Z zFQeI1laCwqC80@$9$A=A9LYg$#>;lFXHhPPz^^!@YOQ}ATqG#k&HfWEE8#sERgYL& zUSvpb&+Ai^Co7@VMxvSFcU%s4PvDj3yT2O2q#WR`nfjC+p~mH!fTM7nLgO+jlIbln z!^!G27ah6U__doW*(h%SXE9j;AnqiClx6xxHmSRq^m5ESwngoO&-!pG7}x1XL|KNV z+YWI*SHGr`d8m*YxK6I>>*8d*(pQ82fL5g-yd4s!G4>)VFO5HI;0_6aT$7m)YO#fL z1nm#eRy$cvl*MxPB<7P1oazc0XZnhsvK)FkSDpLv7a`>m&6Nx+@>BZ7)7pq$wzV*< zFlVKC(1Hm>RFG65TXd+BN@~}slv>35(^yNFD&oVQ2GCyWYu6i81ohtZF03@vKn{-V z?|PcnWl@gU4nV8}HY&I=@_^XrDxoiAyCx{?Fhf6NuX0J33SME@Z&Hw@pJ<5-BfX)aC^4`aZDU*;-qZSs|1phxzurhAbt7nJ_d=NOJ<+zxskOJk z8vT#|RfjHp2YxHxbSx|?Y(G9smg(W75*4zUa7U9}8#zQxO{CiqZVL(z3Sl?#|820y z$zz(RjPCkzdwgp}^QJmK5Eeeontzlg12&QoBVsI(_^Lc3);aRF-{{zu(emccvlrq=4Lmi!~64oj^)~@s161fvbOQomNxsE*j zht^eFo%J`>PFd7<00-liEz0vKt}04v~EgLMhQXK(thO>=W|=)wG~ zCoR;%$!bTv_Z+TnF8H%WtV$$V$~dsDt8>S?G0_eSnjPOu__)fdQAQ*j?rK;MBu6cU zRA!X!u@MLlFFFi>hAn+mHO2KqL@xT9esHfG?G?<1>583Enj(#?Z+4FGzux7ffVqhA zx9bs{BSYUCLOfR{J=XAw)}w9Suep2Lm_golsN%$K&CxQ7naPLx=4mib--}JR86pSZ z@$_(*%Jnc$E9B)#&yt?c&?w!UO2Jv0|H^tBgxgTp+V|T9U`4$HXLfY-0<7qK7rRz7 z@>Y~P6uY{o;+fy-8;jn=chFOHjvCC{BH8^X4j}$s2{R4lH-=X@L-Y@8*{Po?dRzd^ zHEs*lC&ISHlE&?OCK_)Y&Cck*?>>|HC*_o~5*Ymr#^|4UC0{uRx8ZKB4ZfpX;Dtu_ z*N)YUyVn&#r+50}U-{7xGV(G{1DbCs6a+)Z+=!@?p8nLM}b~Rwp%pm@3A!gJ(i+l9o*oH zZwC-cRLUhe^62%=&8fG3G?^d9D=6a}^hgp5<6f4^($I_f#Nl!~fLRF@-S5mYUqy<1yk90G0lHYCTbjR+LEC)u8N1f9wxU7q-pk8eG5O znTXv~k^P*Mqrll(*Te7gL!Cy77;oD_soO7KP%b#p#lBJbJgFqiq4HSA@mHRmjWE8| zoNe4^sOW`~0YK{qn+F>eCse^j4&_YEC#jv^_or=i*GxEO4SAD6)wOUSCaTD$N zpj_gKng#<%*fiWI$)*lA)YLV8HO=BDB>8(PNJncR9=VZ}00{uk_JerlH8YqvDbRQ~ z^uT2O38ta^ucnj+pU5(~*^sqnX-LvlE4>m>p(WH9*;5i!oI$Z!R@o%I+CJUz_EXW- zEu>W*`B3AT{_ORMtez%5$vAn!*ryEps?F-a`6rPtErIzCBI^&T-Td`$vXev1o~n8j&6cYLqz-qZX1&YV?W8vrS1E;K14Y7WJDaM-X_ur&)KW2Y5;gO9%!lL4=&hJ;v$DC$AM;3qs!zRAvH(%OG#1{pz%vIGo68 zrY-hw3CZSqr;`<4?Fo2}{+#Pw+IhGmcD1;3u4?L!`PM*ZumT}oMCbUG#nT0g)*J@0@hf7XaYdBo9mygfox9>=wMr!{Pp7?cbW0u#sR zx9@uA9(k*7={U_>BU{jqe=oC}$X|@^gPa2uCIl9&J5NquZY3mqEw=^_euBnXx5W&! z95$sCU2VeN_s*U?J6-z}Xf>LRL)=ES#Z=1=2W=NYAC}RDr|O2(Lhk(t&^AUVGOh)1IKLN1#*%n--NML+HAMvvbPVk|cp zVdoy~-BC%f9^c+`)iyK45Gl^z-__}`#cYf}c#OXwWY2UX)>UQ^Cl;n_m*u7H3i0Zl z(o+DXAKW2G_r*kOI$;jR1 zG0K;ExE^%AQnPoIyA{eqo(4+}4bsTh!!nj6YQxRYh}@GDH~7wzt3fx;C!5%HUCO|# z6iGwDW4q~S*|j!VYKpj3A$x3f>8`Ul3H5ovDd~FtOGEIPg3O2f`>74s3%2~&HuMSK z6+y+c!4`6ybKPDpSz^2^*=At9WKWDR?(d^|;qA}aD&zh{&PvnOoWhy-c-~v7XO~Q> zBf4(O9&8WmV8!Wr_ivLPjub^`UNf$8%BgbXBw{X@OKAvomf{WMra*O+CE3Gfe&byA zLq?22l;93Y${zaOdfoHrJwLd%@#(mZl8EwT3$mhv6(Gskq%pDl z;3YmU?e^{)&oSfIoZYSGA)gOS{f6JYM0ph8XLG&E8Md{CsfZ-YZH~552ggRP-kVRa zme!=5@hV+ihcsSWHdh18NKrOl$+$8jt^viT`!3C=MvHuEQn*n+ks>*H*YtX^rQsTlBTBI-ASOUHPcQz=|x_x zq+P78#JH1Zx;hRpme_t|#CMv2t$>%zY?8I5#zjl+quWW-4VJ;p{vYt3=v%cIckiCD zQOG&3!ui=_F6k4~ZA+^kd^9Gf6^c~8<8j7hc&HrKtTwml!t|zn*bOQ#@y_@4#YkKr znEC)@B#p@SpP=bl!_9wiNk?%F%1!sLSB{1PPE#6u>_}i9c0H~M=REuNP(GU0=SiB^ z19-}#2_uWO_eueC{Kq|a>RSG+vQJzHHE~WCQ9{F$H@BsRZoJ2KqRMUbU(uYVb~Sxx z(u!KSxstV7D|7=|H|Y8!VmvT4GI(%;mvdvBbRd?7*$VKv;y6|)54CB8U2oLVs!w!( zrB3i(pYZo@tq6{`sRNeGa8FYGs4CJL7}K(YbL61SEvKufl7y^m;}Gn+_sm4iM8M!W z`?t3Gc6!z0+49e_CMAM}ib0!ycsn#_Z%v~dC-*-&Kz47`+6B_p%HWcr{w?RdIzFu^ zuQLpUlFz%9#ewu;YU2IEkqH%eUgAnM*#ZEm0N3}-(91%x(~qd-{(Yl!HyUP7c7!Ye zqV!Jpe(rQiDmv_&`ys~%85KLoov>Zk?mFvywQBuKV$&_|rw`1k?49Gjqhwqok}rNQ z(P&p5?E-O<#RwML$JYFBzcqy5hs3s@c8o+GYo1veaX){ZBk-J(6y7nYF&+z~??-hB8g@me=K6DzhU!Z~4@r2g%Ov+*h-w!&KX z{m5cQmguI6X;$yrXZPhUk6C8i{8(Q$aNpi)S2L14?xMvT*wQ2PqO}|X8tI$s(+d!6 z*C`FE7gv4D9+c6Qf9TN|a4)^S*IREW@LoCyS@uTR(opD)p>l17aun0RUMo`m3ElRs zAHCrruqx>on#FI?@$LP?#;fu(yBC-_H}~)ZkqO7rO2`wK0yH?l?f^F7OJyxGq+*|p zu?Ni6-jQ`kRUTqNa812Kcx&UYNo`o?}`2#hZcbMxvwb;ejkS9)pnPAE#m~t;M|Scf60dhDk_EjC@nKquv0u(~F(~ z6^Fah1L4U?z@v-SXo>A@vr2e>CL>{!TM|9%Mz_v`S|u(fhz<&s5d=5~YINI$fE7ud zc3E|P6JE|qmAw@6@q2*Tlm&>##)X%Dv=8-VnPC;!C)5B+Y_3zm$=@oL4Tt*N=)_xk zIE)@?WBF9AMc6%8)8PA~Gnc*DSfD4~R@T#Mo3bx~veI#)-KUQ2+-#bA@J9PyQ850` zwIBnXxBA&1+wwaMQJy|J9d+v1bB-qxp4whWn|A>Xl-_GqRLFgrw5E( zTkji|Hh6myPIbh~kxv_`p4RoQD~SDmZuh(NaLqx%Sd2QgCn9hnrzLXOs&wJ1^dSxG zBVaRxt*!R}+T6--3x0GmQrUzwH#U~$os_WgYftSV#!4}Gb;1NhzG4QE+C6*t5N?HXe%&w8sX!vvX@eIqJsQyWG-i|x zC(3#z+!FCt3{8c`o`0gu*E-73Y>d3^sm1F#*5-SB(EUM?`;rgScll#h{X^PqvBZ*n zXVSSSi!3gtHQ}<_wVNz_Bfb$6s^1zj^A|rrX#2M?M zgniAtfjqmJ>DdSio~x??a^IRCfM*+=W_3;NekzVkuo*9V4=KqRWdwlR-{@}aNl$Lj zw3?hBpqev#-XQew3y(7wj#SO}JuvfslFWCf^0wBra#KoD{wnCb@@Vz9Hr2SWFyH-N z_T6*7puKnY&bqt>o;pCQA6)oC!(}q0i&DD!)Qic8=pV2Kh=ZnuP(m87PjI}K{`)8A z#?l+W3JwR_vU&vX+&9`GBqMDC6L4{Ge;p9Lej>l`zP0MRfoOfDJbnR8C9c}Yf0WG- za@=bcYQx@JM+j(ggO9oa6>oK=cXZ)ihS=Eipsd|t(i4a&iLuO!co1_vZK&dQnkO|% zmIs7J+eBOPXG6b(*}w~5i$_Ujme^QxOT!Gjp6*s7>=nfpb;G2<95B7VuIlnux5MRO z?CTis!Dwi#f~x=ZQ@2*QxyBxpTNG=z%|Oh@Cu#Sg?joZ{y!@krQm?Hw@w~0L?ktvs z&MjqChiCC(v{RWbUUJPg2lvTsF8%V-^VK{p^RvyZk*HPP=ng~e^I~8FjC{j=e-V_& zIh}#!CQHO^j*7aV@}WU3i1-Hh#XT6I z34lFv%#5xR(of-T5T~y~+hx<38Pzo zx|ngxb3c1e2i1Xr=@=uGQNBT&^A#3#;G#$1=v}W`64bt79nV|b2KUbwd{o+Yc}6<= z!wxM?dzl=G`rg$gvyybX7AQW8Fe5}+TNZc|qu%Z6iBLc90>Di)yU~?%+2>Rm$~JxV zQxC?F;C_pWO+}j^0RpNf&g9cJh%|F~rmfn^$T@hgVTmrfC}{qEfSbT_FBMl>)0SEG zHDfA>WB5xOzx-@V%u!!7tcC0Dijo~uky=u{4^xVxcZH`FVSP^v6Yia~>419rIHJ{s zs}ZitzV%wF{(m0-?8h3_^a*cP6Ia=k!G^x~1Rrr5u_IQ$EZ6`ZgIuh3#8!9}2LCw1 zCZ=a_T4hkr{!;hp3;9ZqBpC+wRUKK`tux%<&~b*Ga@ti&*yufL73+ed%@M4P7T=6z z1FE-4DIXOvp7|RifJu#VAO2WW%bZw}+!J1XF@>UhE=uHdzo~21g&`gkVC;0Q@Ov4> zJH-!2@gEHaxM~j74>q2|9wiwGWqSsI#$Fz|wf%10ZD({l^sdG^#`(GczueD(KAr?* z6vEvV;Yn2T-|(G)tvS9qe|bCt>iKxAX(W8syH2(gk}*8f z7@0XeI@PvL{#mg4)MZ|+EJ+o(GZK(s)m9LTQorY@R=;)Oi<-Hdgzdlr4Xl*6XN!DH zEuT**wk(bFPEe1vD!T-=ex%Tw>V5M16g8#}YIw_^%i6=I$h^uvpR(VJ;$N}%`3b8$ zHfT4TJ$iVvX~Ix)TcY%{-`C<}cxEc|V3Tz`pHG!vc9C>^Mon10+{ITH3e~UGd%Zn# z!jEWdfL^id;o_dSiXof}n0-7ZWp+kI168DFG9Hkw_m|}RgflrXVR!0Zx+AEn_K+s` zOVSp$qR?imUd`5_I8IIPT6w#M;;Z~+tgF`((f3X8xKWK+L**}t=8G-z9-cyDVnb>O zR0veZ{`olYzESMYEu#TEUO%AO$Q^(oHKG9Uvn{f{MhRHabM=|CXADB^hSyD$h8pTA zO?{G+<}7MggB;O`16S(DZp;om#T6CW1^U-u4BeThl59kbbcKd!Y2@ceh+*Q?C@&nf zr}KPW2k#GRM+-E32s>S$Z9pg#*nCg$E;fYq-G?@Y@{F&0Cso^uGgX7`xOCpkLlf#N zx%|{hIMv1Kj9@~XVi?jT=DCO4vo|yr;eeW=V=XvJLoo937a`{-z`d9Oo%)vc&uSaw-YHnMcH26nbMlyr_w0}CXf^oejM=s@%kzpH zJ$Q(PFWC?lJ;+81a)d;J{>CjED^ z>wklriozl~ZcmpLj#Vbkb}Vm$&Q{zmO@`mg;zq6OSlPI{41qha_1=xr9nH?=Dudo0 zZ+|GaH`q7ZG*!R-8@9-4*<_wadp_2yvmLi7*AnI(7Fm2*XvR)_*39~TN7Qc5W#Vkk zhKTyiS@!^cx9noID6<0|p}+B!3u8@JR0U6#eQ~f?4Sr+~{A!i7lNzV~fvZWYWv%MK zBAt`KeyoEw9bhGt(4T2*$xm69P6)v?!>klhVUZV|$A+xfsWcTovx6Xq`oGD!d15xv zFL+Z+{(H;akBtzp3ZFUl?Bs^sDNB*RUQfHBC5RE<@af5?s{ck;o)49Oo~wdugnoz& zv<$2y`;EFROK@&?os``(-*pR}e%3vpay5TqKS#mO4T&A(x$oUt@PwX$OmDha6!7{@W= z_s^J7Il>+@D)%6#v;->Pgeypz|X=D2F)A!2Bc3)Pvt;F#Y5anqG+HIqku;XPHT5!BjJ!dRXq2^(k} z+9DgG*e@S@yIPypb{reCBCRb|^D}Wm@eWsUoJvbBlJU+7Ij(EA}KZ(ZVF+f1zK-9{tOGFyW{f@wquiKGxvTpT>9#2q>#2ZceD2m zQ4m)!F1F2^R&xsFWf=?D!vu2?G?oVUcA8P#zaJZ)NQFiWE7S~~B6Sc+|LsHpyTp1z7kcKA7WI2o(DqMdZ+@Kr5&iy7#Sp_3N6h`6Y$ zC*uHdC$7z|;c5Ua4c~g>)bZ7FZv;-{T^R@xLcbZaV$ZwzlyRCzvG!=DLG8F-lGJGz zmtLp+AKE;o9H4{oO1Eo8TBi{i*Dvsu$0t6i$ zv~z+n_N>(zc8cz_Myb|`vHv&=tdD9eEnPJ%|g?zp+S^C~WN zkky&^L0ENzRg21()S~)GxA>;t4AUFEsJ6LvwA$VMQSaMi|D zA81HEhV%ao*yl7cWpN^6l!eW&eDoL^)!g4|wKen!G!?OlmX-YrrH?dM622J|r_QCf zpp%d7|iW7wCVSMRyZ zDQr}7oj9^hB)t{hTr7TgGu+7oedOOOeOcv&&1G!cbtD{bF^=yk5Uv(*0 zPf1((>M!K&tMO3!ANJMV9bsZ+8K2D!XF~7*1s!b-*7C%wX05`Q(T$(>c(i}m6uP5U z`A(0|v}doDqujvYf%R5kh=y!=GS?*uMDUa9GON-EmW<4b5t#t}njI`n~d?zraX(C2|aFNruPA*p@ zTX2IwPi#z6vuKaxHCWtESmQZ3N=R72Qobg`FNv;d>aBWJhR~X4wHm`pep5}b&h<7B z093+iV5DH4)Ff!K9_Z4DZ%Emb(*rk`y<0a-RG&%;bop06rW}9e(Nv`E<#Q74CAq^8 zLwMV7$Nhf@y>3YTIIjLk{V5;!Yv(;{1qsU)&H)ratFAx#M}Fo%`#FrP{ef%sxO(|5 zDMm$G&Y3SuZZU3kC^qkxD)-XiaZlCdT7I(joX@2FDZ%KEAod)Kry-b?Z>s#tfMR+6T4?s**j7hA z`%+;THUS={5KKtShK*1=R#n1YAqQ-D*?-$e{ycy8{exePEU<0fCT^11z52|m<=gRx ztwY~oC2wEkVE|gNcPD9$=>2>XJF5)4qx^l@NBxHqI!g+#I%B^ zM59sU(}9hTFD^Z5TIX|Ae=oGVaKUv~6VD3eJG`@-N_;gkv+D=Ufi81SIvwpWVtfnN zL$;Wn@z6rFh*M!_rS4XF**x?05Pq8{|MRPrYFHNX6fY>9<67=2)tSLePuYK^DHKjA zRr!mtY|NGwhL{rz>)+e)eATxL{AC$ST>Z_V&VGBx69!iYf%Njxx>ECp%A|25E?UK-K)o;Pb40T*9TL`KGIpXt@ z8sqcV&k-ZbkTFb1JG>7k!g%nAGgZYVpYCoEuJ)%@D@ z*BPT#tKYpmdOs<7Xv#WtZ(*rure-&pJ`Y|pot!I5gnfgR%N+|GiJX~Uig}&au|-Az z=Ek2-O19wq^gM{kTHrr^IS4nMe&-J5=uj?$M06_*vo0=whiXUH2`6R_bS%{s%PYMJ zo63ug7VMDh{on08_i_C89?kQ?xs`Cw zf-uQu(U7^>H=A;o3Nng+FuhO<{2kW3QvD+>dvY}|a`yUAQmq@~k%gJfeD=JLJ%LvR0zBy~D{vKc@oKpodc3zqGcsPj0rc-F$| ztMa%dH%aMB17rV__Mkl(hhZqs6v__i}SyyaLwnx!{R?*zEc05|4(V& z`QNLU{7)*iTRz7#O=ZiWJT3kNk$+1lvtInJw~Z9T*+|Yz1?a^$a+uYVs4~uvi&l)VyrR{VlPkNhrV!j+>9UFIdZ8AiZmSD zNa2**{dD94WVmm-QlCDq0XH0-YEv)-Xh4f17ke}rJ_g16-^^|HU+>29ewT_h!Gq)?A-;Jo_oVS?AXMbD$ank2?a#Chcwy@zejBo!Ou ze$tu9Ed8YtVSG{_g1>CBH{-rD?OxTFB#Kyh$z`yKkexjiZDap+J=)OrCOzBPZu$nh z#)}qMQAN0C72Uh(NhWA+W$ecWnWw6y>p_4FQn$Q9eg#rqjkY(93Q4f47ohIwZ>7Rf zT+0#fKj(BbC)EZ;at3VMxhpqZx-Cdp*N}3>_`hYsBo&sbC=K@a#qqwqrEPg{sCaK{ zAWvgC;=!0y#Rd5Qp@Ii7!?j|#a1*HsBzR-Sp%jlV7IRB(>+?$}=nIKQ*OzEvBgVlVBFoR)@41NL-LSfp$ zS2M7PQ2K7psH0h3Li*;9)r8P6RV&T}z^o&A<1n^UnmRO9E5y%4Qbr8umfIgx5zGaA z28zblzHNC=@s>IEx(|H_E|=0^k-K_w@sz$tq3pqoNCFq#;>5ce^Vx|~n009QN^?EE zH`3wwNUD~&RGwhi>)=)PqtDcCf9VPCcAf24J7?`b-lJ1vxeOiiX*&+=`EOfC z9DE!|=_-cOv~2C~qyfcNvMMS#uf{oXXJUVw$^g@Q{APhJuK>4Qb>>FovFyTCSMRLW z%QJ9j5SLb*hs_2>?xmFva>*&xCIwkDXG$Csp;vu_fA;Cepx<%*mAYsIA3=Qmm%x^5 zG}}(0vuX|ga?+th$Id%6x1-E=_4cV|=!#WAFBdy)UpjXwN%Dn|9vyWIqgWK%hFxF$y;^HMZk5A>{`f z+#KVBb}G;8j0%y(ZNg#F1~1>Dd$Z<%NOK1MA>%pST|=f$e4i5rPcSQCBtR zU>yT?T&YznIPoMZc0L-ZnU^Y?_#qmf%3!Xx1C4hY&kC^7)h9N@C(=N% zsmz0&tkHz$ewSKR9AZIyyY992B(~7{b&g$JgoIx=+afK=v&t0vlF_s+Ldoxy1VM_( zPO{|M!H+Figq~{AMFhry_wGKvE$;*4H#(6C9F{qA-2WY^{uJTn-XVONx< zS2{FsZetGRy`S2i-mhas*Nw*nkDUyPx++p!uH~hs=bLaEbAARc6Vn4|*r1*-bwBMY zo-}8n2%DJV@4Hvo;z3qK#*wnZ3yS`$*zQ!|N;7!LGpV`Tj>GTgvlOiWzOa66|HHGZ z;d)>QQO-u~BR+}oMqoaHgr%hH=?U7x&~~<@AkV4GQ_ZrQ_jIz?%afA)FT6hB?y5y{ zi@~9~b)&QnrXJat4LG#F-m9ey%!B_GWo{`qicfCR86XsenVf4_N=gjY=__G%3obUT z_iK`uif+iVg>{F(QP>+cL)7jBFGGpSKHnF58whv+VplE3tZW%QA4q7~+&Z!GNDOg? zSq^b+mVDt=1;Ew(X4ztdepnokUUSvRBbLs_k9!YHu(wT^73y%?Z12^q?m*CWg3nCo zGK0#=VJ27;XV1c-!;1#(TA6cYfo};Slb8dFil>4#BF=~dI# zCms;YDV-Yu8wpNpwi9`1Zr0?ev5}3~+uh(NDwrX(5XiBSFOw_I^`s^)<6grh&ZMur z(zN@>92te}+eUk9QxwM+WH9c8QmY{s$7`@cUHGImz-J|JSw@DC;e(X zKLfN_(AcQ7fqKM@_jgs;lIvv$fYx70$53;274}&l$E2A2M4p4KlxAE9)+bJ^m~#_j%40kIAyb7#?z*S-D1@MrtSP9iq5C=fP%XJeoBht`&% zweX2@y%6+3=BAFMOnaGGq;Fk~sY78pcwbLJzlcS7{KqTlChfAxFZG1V;;Xk$Wc`Ir z8C6KdjQ+F2Xr-6^SnC+q=Gd>}qK?Z+4<6cxyt1%m#kWbH$JCP4HM52!G)TR7xqD!P z%v(68R$0<4CvLYHEBGX`Bni&(7;cceS=t(Ed@xP07`H$9OI1HU9~Q1N_Sw+@=i&sd zB=_%MuUwwCu)lyy7jYdC9o+dvn4;+Bw^FjTuR=5WZHssn6KYqCq5-fyNt(OgvdqkfW66R}^>+n>Y)zQL@BHI!_yL`M#lk6NM*(xr;E~R0>1ixB5 zxnefHA)OkmomjYfL$)H@bPPKYz_;zqzm+*_*NkqrKH)^Qu=`IHZ#r_QQFq3^m6bQ8 zth0i5Zk%ycTE6{Q{v(1R;8#9qW#B{?*YTCGh~X^wa_T{k8kM%N=@>6xnkv~`foAPn zIkQnOjVe)(C93m>)BHq!{+c?}I)3y-QTBtphMk&zw0(Sc<#(w-ja)VKuQi&)rCTQi zcK_4qOY!jo(LTEOB1lp3ne7Xnex*rWN1@auEHy}~hAh*HXK=|v*}Mt!e*br+Z3{=hCOlAPIGlek@G^83g_~k_uE(| z!MKidwYJ5p@@IA$kwcbFpf4(_J@V~&4HGpp_K9L=d*>`D^*=Xd`uE2N#rdaBf|(A= zeAT$9JMR@-<(yYk()*`1uB+&kJnQGU0c+bD%JO>Y{qj$j4H|*lWhPr;SeMQ!=dIz<3Y6W`j%T@( zMVB--{k=njixn!IUa&Pi5J+@x9g$N!XoOA-g^ac@J-T>}MNr*#`gI~27AI~U@g4`% zeDcWH*#>T4*2ZLb^H&MOE&BQc#j5foz{y?nfXKIQ3PPJPbQ?C#;h|3euZwL0gnaPc zdo&yT5D!i$qoks*dthe$6W4T{keiG1<7#|5W6V3pxmm?3wKpiGc+I_*62IeJ`Wc9upMYvL1au~{1$gfq<3T`U5 zVnx(vSn7qoS=8xes$PBKC<1oTX>A3$*`oSfja5hl*zw`pj?|eaTDH$hHDS{UsAm)e zy=DE~Cq`f6f;cnF%K&x)&n|y`wMDKJ!vJR08a@uL&YcbG^1bXD*EeUHHQm|Idb4$R zCuNQ7J|*ud7q?^BQgvj2LhM~q$XHrgD=C6(;OTLvH_uJr6(1c(9qnhz>{~1!Q%wCC5q8obCnutXt-X%X zEe2X36+g8dqJFNUu_7rJMNSZV4(5Q*`?7Kaj`0~ok-Xm{_DAkbeSbCnN4SXQ(VL^9UlonRu!J>>O!@P5vB*bkZX z+ckbeMrutd&Z$k>o;#ze9rEpO6H4q=J03+9QK!k3&EuVduU!xGjZd(pihR_@;>s7L zTE7UvO|G;*HsT7CDl>@<`LhA8*LYbns&{$0=@F;(695*_PtXn0MUg z*8FtAz~`C_)Leu(zB03G8P&Q{*y96VyZ52z^+!qef${wNSx5PF%W(HSO2i>c*XnVV{vqS?E^k^RCnP^!IZ7yD(I+b?!}Ie2b3g7e3uOS&94HIPnOr5KF4CkJye zp){Mng2q3{a$J{jT=7UW=w6)kql>93ZN6dO@_MLIAP3|{nAxSRm#{KhI5v2U)72`) zaw2T%ir|V8Q$YxFg~)VMmi zGND<$#BU*4hq(^n?yAq0;>u0AXM}=l=GQj^PAfq8r)ZjZJ+r0CmYQ1Z7~6|D#R`CT z{QGtHDWxzvbZ>BAOZv}KfTrWSZJeGOGVXqs+QAsZzg}|OW)P9_Ix4l#!v_9rezF~<1qj#R6%M8 zQG*~9m%(iFaz!Nw9#*q^kUmoJg40<5T!H+;mz9_M#k;ICIRgCR-e=pLMz66=A|$3k z-q3F#fx3aoUA-}LWK&`OqFE>9WyvwNp1J5~{T(ui$L{wARdw_qOsv)v#2qXv84mlM z@0xuY|G-+%>i)^ob3vJ~q-V2(i)X2E6E$-3qGMwFL?!R0ho>NIxc-F&J_Y%cJO5qc z)PHg6djI=m0?f7YuY^_3wf|#o-<(G)Ls>0u7;vrogdmae&j+F|Zak@iHg1>{q`p@p zPWzu?QN!(`9J78{!dx7|FNT4)zK@!1s$<7O9$=ybO7QTm#)eMxLGpw z6A9*92KH1nh6(x^!IM-;JRqsGqxv`IH!Sy_$e_*a-&5_IF{}^Kg6oc9HvJ!=zo$`b%(J~}r_m@DaM zs+~`X*g5P#)P5a0%C+}f5J#QfPZ5!%DhGn}Q;5s=6HNwYv9!oo6bYwt3`=_Qd$K>3 z<7dgq#z7SWNFY94_s#jCv$oSEam1I?pwwfZV zaU%MKZ~65Q+w4E5VNd8G@{%)9W%1s>H0n$N;4mqEGNVbXe_3CQPyKg^rH7Bc+cxek<$_efO2QF3+S-y+Wy zZH6PRe!cdxzjFY4TrNraU}0xss?-6hYfx$o>v8-!C~*)Po>guZ^2Nz^_ za?5x#3&2rfG%eQUT2dPiu4tdR33t~M8kk5o6j~tGgstdsJF-TGYk6d+_w#~Nv?g2E zoAKa2&5<59J0~?8fN;@KZ4c^>i`l&Y&dJ*O!p&U`r=|ht!G?#{Rg=;M0zl1KJ{GKC zP3?t=itZygzkOUG+6Pse?(J2S4f(Qt(4EQ!u;M$eM~ET(w+=6wDa4Cl>V`(QI zb`VCZU<;Y2g18goe|*1*Mu5iLBYxajxhIyR$KvPJKg37&oetfvGc9p#y~{fGQ(e_| ztRn^UhD;1{Y#Tu06D4^VR1L`3V86B>-_`mvy}02jbfcxo(ICDSsP$bUs{LoEGci#< z4N`3v>TMlo1-~@I652`8XVhrt7g@^J+u{_=2wL2F5Qv*)KHfJ(KDyXPwAHV((-8uS zHHE*}9Gt*ZA7itnIZ|q%v&wX{FBI>nfafvqI`(&e1VD}ybH1_4RZVEL3H84#gWC$U zfA$`)GTXcn^Sw05 z?}JT5I0c=BTdT2-!52KZ>tIdzTpG_BKl6Z@s)ah1Ab6GscmMVRN{mu}lR|GK4Q?gz zj>q38zgNyq%YjnlEv{)JGr|w|w3ACrup!MMGhsSH3##MSnaiMZX!xz<2*zVMwq3a< z{6UUMJ87S>3??NNZ|$EwPjU>h>|mD+vay^z0lT<=Nrt&aSe822jZg02zIpBKjnz$b z5OX}n+*@!O##7ul_mJ!n2|bologf$N9T1vn2g^IQ}}- z1l3u3b^ksOT)ti>>lA(jQ6&cc`2usuFtRz`YWj#Lh~t8~(1=NlOx>;m=lM6v#-ZqECONSCtVgfx zu2PQ?)9&lZypBmkORC`Eyl#>HO{)zx9ahV)uPD~I?Vz7c?-Y~9?fR|ab3;qTUR>{S zDuH#ASFSagRcPjA6*q_ueJPZPo|zZS>;vRp{j=stny!vD`0qN1uwayHT2J_=L_4TjYNF77w!NE+o}u{ z?PmYHPG|S@f&Nv_*Ao)j@Ah!y#`_~I|#@ojS*8a^S{Am;_! zf~SvUwAhyt@7zdW=8uqg+G=N7uHP+3P`a(7O$-3ET7@{z2@Z9{XSJ$qUHB%36J}w zVaN3XISkE}mO4xLrnKC2$31?MHSYq^!6}MD1>A##+lr999g@$Wo_Z2c40@ivg?_}X?A7(=+A6-G71;fcU#-RyeoM-nr7Bmw>g{Q zsD>&S1EwVuW&n6t*6RIo^ih2bnl_^K`#ctw(>I7F3waS0f&9 zRUMrIo5e}TyMEvC!YSP zxU1F^xd;6!I9RQZD4(tEGm@N8e9fSW`;z+O(`D6JpA$-L1%Xu|q;qo)r^?Dr$I5a6 zinqX5@V2T383V{%f|T)!tmX6x80cPrH&%j|8F~N061|1+l(MXaDO~%ES(^RD$AGMf zZrG=^8?!UN!Y=);ZnFkHqU*Wmx8=X+6yrJ_7^i6i^DXGT#c1=Re==2Sh`O8nn#&vQ z=zHIo@8fP(XuY$G5QcW(L%MvHY_Cp@LqFlYXDWbn4R7upE3FT>XhimzSwBx+Os1;Y zZa@8bTI;;Zbyn|VPtT$_tfw`7)q44e$DMY5k2Pq0#^vO9M`ASF&rG(qzPU~7n@SOe z5m%=lI0f$bY_kmKe?W8*^G;=iTeV`24;v1IDeraFa^WB-Y+UXOOU_$fiuwIQSdZXb zK#349{ph)SLe~y>&g}S_2J4>DULXpusr>ARDAWm7;Ql=QP!lqhNTue^&Eby z$?o=!p9+neRvph7x)!&iPyZEe43(q40dU@dU&@fSI};I8!cXq)eUHfruIelj}xd(sc7-F1*|eb$s9d|j(tI)~Wy zpdCC|5>7RgwD27z1jelc*4>mW()=HqHm$O?BQqMga!bUE!D=b7ZCSqU)pvlQ<3JXI zC>D~6ZBm&%Lt%;vjc(z5EBAHUIPG#}3ErfJldlZe_rAI!5Xsi4W?PZ$Gnq%;H_zHT z%rZpcGxuiNL*KPtO{S(r@z&#DDdKQwGWvKo=mFZUH6(&kU<*c(P84l;+}u+ROZ(wv z{i}}>V(0e}@^|=qbF!gOsKN;^59gaulTV#JEai5`|8#lme~y%YtLl#EpU4CRwq(tn zdli?bmQu;KYxpoBiRJfXA$;Xl6uQBAQTWvZKh(AV(#|os_YHj@&JcZA zi$C;YM!coxz2zuxH>`M9ZE!{Um-J<#(lVDXw4okbj0pj;G8Zkq2f;0EP)yX*Zp~nq zjCKlYW24eGEMVWV9W!pu(iSJ8A|)VcyPC3qcqXTn%uc9z)Mo1GI03cVC&@Y9JM3V@ z2g|fm;!=rAR9gh!3NG|%Fgfu%xGqDCg^f|~f{|?$EjzQrG#^Bywi0O<1dtpQ-B0~;(4rDItjz>ImN+Rf?{e=kmkhPLukWntsj90(hG z=rcY!GPm|+x+2FKb}ij?T-lB}@zA!gAgS_jl%v~-2MhB4rsm6wjf^rA&L@Y{a zlb<6YmL~(^NA!TK``r?tws|PgVcz-#aMEdU%vvAmmotBJ#;jRr2-!w}o|nE|H_8Ty zc^XRmT^ABO&oX_D-@>7^kE)&j+BWX6yN}E&p@)YCKrhDV*U8tfCE&Pk<+uK+i{3wV zp=MyZGF0=300d0_oVMKKk1Z|!KY!D@#tyu%6_e`ke^W@L(z<+~1r+$Yn~nK;HBz}AHI^+3mq~t zs?IM$SS|s(5L(j{m#9FYhedk;B)670nbM7<+M9S=KeOzWJ&wIlK#7wN)pDrA6w;ZM zbkzk7SS~@y2(9A#tJ+FJ0B>5hdfoQcIfxLYHV!QHKGwBk`!6zIP5}-xf&V6~IG_Hq z5_?v9H6+j_`~3S-qMFcR>XjDv(}S!yGyISFt&e&p-NuV>SI8fp_bOTNDVZF&O0G5&sE2RzVI^k8%bQv3@J|LtTe3 zjz>QSgc_NDWEINQc4vXU9fsXV782#OI~0RBT)5oApNVE@1wXa{hM`iFfoGZz<1Apj zMD3XJ^WHbo4y51)Vi$T30K!5ovM(H~Z0DuTEQ+dIBS=~nRg-70LA7U=)MvtQ9z&C$ z{$#yhS*yXX%K=FY z^ScZ1*;FGY=Av>0Um53K7(Q7|cj|iycI}k1r35N-voktpo-MA7rauLEV zF)>s{@!d z2OgYW@-i-{Ts!k1WtFainr4$rx9b-ZSy}Lz=uKU>TCY`aFmrFl97%6kW3TdI-yB-c z2&I%DH-a1|N>)=^IQQgB4H;|vP(m%)CAYQDkG4M*|2#jCR>JXu%mB&~E3@p8!WUexMK`t0i=v>mCr+mNlShi6_igJb0r(XI5FLrwt z3}bCt#KYW&a%59XHR73mSUQIVMZ!!en5@Fe4+XqlsbdG-GHss&>^~TV>)DPAJerlY zAnx9Jap=1bh_SIT#R{NrQ-2!L0@>EonzaHfOA^Y0YDOy5l+cDKtxaVlPDQot0{+#; zTSJ^{`q+*E3E3uK0)WOI$S}>WIob700QjiP=Gy7}aQzHROzfCFFr!eQ#VrZv(_-TY zIng)BaCwr)Xgb;OZavzSc$gO*>Zn@!xQ^`|n(FZ`39{JpR|m@6_bfx9>m0E8sla?u zp?1LsxDOj3L$q^4_g?v=9t+9&fG}JOv@eg@ot1UNUMXv*1_!>n99Mj^^pYV3tD8Ky z=^^oEy>irb)>65*eud#JtuRf_ODSa%1yGBR-w!vbGywJS>-%I9BOjf5^A#(ls+k$;S|>g&8|sIgA3S-iYeVj zJn+Orn;t5Hj-&*{DpU5*n!7&Xa{V0KP5u0KDWrV&dofnmoBktGBKzhO}XY1PNNR@6akg#ThBO z*kw^_dgUge&84lMw7ryC*NmeQSRS*pF{<7Tk?=64y7u^(rB`|O7r(yKFFbQ#Tt2rw ztaaS#I}G}GVlevYXjH89Hj~D+^&1a8%)447-ZR5>Be!uuS7RDjE94L8PdrtVi*=X4 zQH)PR!*kP6dpW4G3%T-`t&hiTj}xT@ZZA+~vh|WX7phC2;g?M05N4=I$0=h^M|b)8 z!{X&naz8EolAqIf1K(M@$d6~%-<_{bUkidhw8AS)lp8Hq+(*eqY~sw!Dd=N%N(un`6O3Is84^*uNKVbBGb&C=2~jj$ZkmEXWFJ& zLSujd1f##szIiSX`8Z1{k|}b{zB34%q4i^AO->i;9;o>_dVTYq@h3y^*}Tx;HYPYE$}Q2a`lvyG1tZA5c$f|uE(cj!c|V4* zokhC_?7y0Nu-70YC4Fy3cW>_^PG>tP(3q0VF#sAbFBbgX&@y2=0PB%4s5$)gtZ5bv zOGF#N3Eb@d?Fp!C=6WS~ZkOnQf88M;1QYoNCSVOdLU(s2&``xtX4M##a6K1)g^W?o z2iNwHCWkL{*aK)gZpjG<374A{G-8aWoHSxd*aEVwmhZq%=(abW#pwCT=LR`FHs}Ml z&4?~Xy{a+;8=)2jcytm!*HWs z$FsNa`K(EHx|(&pw+ryiJ>)ZP&pj&*^s&1Y|4JS7M?a&pjv~|t!5zA zSSkHR5FI^{p4HDwnhDzNYn|NMo%<#zao}rt_Cf@^3IjZ7GOP3&YdJGDz1S`DFp5!r zTv~yJy>M6H{HR&6bn0q7GF4^G?s-4I<%F^68-JO1Y9|+vpDMZ+bnkFJeJXB;_anZg>G^U&=QiBjTwy7f9CjfDe_ptob;6a7yk&H zssk1KRuR$g6Ob_^eV(2?UL2b&p9^r1@`f6r8lVjtaV|1z3G{`P7Wp)cZB{=Ys5`?@ zJH=UzAeqdtcV@4d;4(`aWxMwpoC~Okjd0U3#{7{Fd-6ZhVQuZaT>Kd^ml;*UC2paI zQVC4(64dqR^;svIhfL)6I8vckMnN$q9K1U1uwze1U7N2 zKX8`u$D?Wzg^iiFWN7ED6&wWISH9pSCd0B|`KzBxUjLb5pB{L-Y<%wFZLcTQ3%zpi z&})%03b!cOxe;4We z6}VQ7+__~vEL<@tp<@0zlLkEJrf2D#`22fJv8d)_rJ0^DclT)tFdmaP(LAj*Z~Zg) z_LpN88Hv3CRjaparM3h2)^t#R1>#ZV0qlmYXqWPx3o@0s!&HxRGLx?+o{RhaOb2x; zXnkp@Obu?;oy>ey!j5_Wi*Z8coWKaE8*M+5MaPg1if^eQi}0EBH~5TM027g=0JNn{ z=hi*#w$7_@Zwy!~O6(uK3w;?~?*Vi5%O4D@&0!aRxS8XND*dHSAAZ6*^WAiJ-=kuB zC{@2ih$Fcp2azlr;0JNaxPPDgdrjSG;exlT3}aOL&~h0$4ecPlYH8PtYJ@^pr}otT zn1Ktb^{YYp&f#tvoxOYz3uNDs z%P%vrU>uHg#iNTmNpsc;i4%6d2Y=W}OP@@i!FqgGIYIrOs0by;qjhnqdT6?B0mWu z&5g!I0jE49>kOIMCzsBS;PrrK6H?(^FUXq z?Y32ChNLGiB!n3S&1X)%x^kFc5K?KED?i~qkX$m}D-NTAYw|{_rxNp@+oMJ8vllKh zvR`;Jd=Q}oF~pmc#tT6+$Nl`n4ecz?`<-H)ows?Q(Y&FY$EP_JRRCGVy&lXidI!}j zq=t9}>#|)D9j!D*csG^xGr)j-&&@fJ=CcdcNqRkfR!Ptpz!jUm+FL50>Y4;mgi8^8 z1U^xvS}U-PrM;#NEnahO9^{jK1vh{k+_cw`*zY$)mZN2UGP&Nu2n)vY{_brbTFu~`eNZO-#Im+_OWwiu5Xpqa{RXC~aC6_4eto8-Bl*n#rG-m?A-iBnrAqy<_7m>AiG6<23E zaw`{uQq~vBs36M6S0w&BNjQq_fW;sS`m=xPGqoFI88D zvHtL+>+97k&O-ZL85oj6OziT9V~)p(sMyq!Z(6oWJOJ8n^0h_l_E*v%EKtp1}Zjg%sqFR2RYDt}8E- z!>05H7R@)KAp767P*-V#W_8Cj^<~(3}iZBh+5i3_h^%M+5 zo-w&bdcG$fo2oEy$Ib3y9X{I<3~FVfpMLv-^D`aMJKMpUKO8;Xlv2By{J~$-mHW+J zhYv=~(yTYn-w8mjbv0clIFOKsK__SmBdk1G2xh-))u66Zft8 zq40WLC2H|3+xiT*8~k3;x-CzP>YeW;j2Z$~FO)lGf{l|%fo8Q`GU?DFO9{=1H6inG33wfE!~ z!B(Y7oIZwjm6&cwP=+JFnI?5=81aKY+#ekayQA&8jK{((OwZ6QaUn2O7F;>EZryyL z@JcOo#tdFDK!hZtZX}-`0tzNj2_p&c8nd;3)PowKiscy zTpZ7m#(75kiyvCM7v(8zSEWWJ*V@Jj>p$nK3(?^K`^8bop%ahPeRib`Yln)1g998B zMxnlxrm%v;%n>zXX(wKhr@i9L@fLZ>Z*MjQ8mWfiDeuvkouSZtAAJF!mkVQXTWy0~ z`ET;w%e_|?Rua41_Ji3npz&6^Q#pLABX$@Q>oz2?x99e$Qa^{Hird zNzlVK?7^tUpYx&NYyTm&WP0&$Z}5NbVinw%xvgPwyerAwaliVYHRCOQ7`MckxYAV{ zq>nO9x7bB*fdE#X@GVaZJi;f#@@~COb;ZO+s;9?4cZNKyl6li5{S4wge3UP9anlx! zdFG~02Qt$HrI?Ox<%W9y_ z{<*io+T?iF4Dk2qn7$%tiS0{py!IoYfkWjC8u4zNDhLd|{+H?bntAbbNPkPt*5UH~ zryjrU)u&E9^L2kHxsHwFSj?f~V%#ZgkhCVvuA`a`cw35GI!63vtq&r7U+=8ab0%P7 zBng`t*m&UAHql_t@fWwV=T>O=&CI~El4<;v#hF;x)excR6S9l)GL?<|;W)Mh1~YH! z0b9_6WdfLoWHj3oZlFJmqF&gL&4Z|&6EMj-(iwa78BqUH15_NdhDx9)Ig+zQYQLBR&yU9cg*lSV- zt7QIe0vE^wHAeimQLfq%n2Zgb7vae##v5|AeT(r7tJt0hm+6-`CvTj1G78;DNwfob zIXdOIV!xM%w7Oy3XZ@t=*DhO7vvOn#H7j>zwbTv!-Mp#9v4JMj_LE?Ovjw7M{!}{W z3_T6ujvT^atHwNSO(!9EC5W>>7o?*%9bFkh=?OTVlQOc@X#o4}B#cOFa6acQ z1Ro-d$eDfj+8;W5Kq&NJXAqkl9L0gEE8O|*|9<#ll~m-D_0*R6wZSx!`K4dd{F|$i zMI^8Kj+7C?!u*GhgYe<-ApVO#RXd755N#qKzwkqKKKwM<-gyR5 z{=^x%PRhI={U$AK=Z1X!&>7|kt78)1t@Cd2odVn!zN_H#5s900ik)3R7+dH zO%pkm@F}5|rO8ikI*vh=O}pQ~M)n5X^iI3azPZMC$Q&muP8ht>(4wl=A~8l?|3Dv? z`~M!hrw}e8lCnxp*VJ}+T^xe8>8eO;zh1m=<`{MQE6aw8NA^6u>v-Li=N*zh0vhRv z&7Bv%Bpo_$zf4q|lKD!s|Q};(l0Wi4^$%^LCKQiOgrz^ zZc^cHu^WJet`_KH3C&?m%G@HHJ%~rUT^Kmudh5;5xJtfr67&Fkx!{Gm{E;Vh|DLZ< zVMXo=8J_-hBoc7Wvc$UmwHq=6G8}U7>s7WDG(0!`>Fz;pD0%U1n7uqO?r|Pz>%oDe zzWH;Gm7Swg?)+vi{}763rQ)(kj4DhEKK(8(RzDRz{6nL5yvp{H10?gax&j^7@4Qjk z3DLxTuqb0JT$U2FQL~yEG=%^2_y8ljKjwvi78ktCYa6=L)j*%Ez&h*{4ZZ%)4s~<{ZgksB%NK< zn@_Ks@Jr(n8LX2mCWX!l$peh1EMKFfX(CFGBzkf0DzCaOS;XmXUM@HKeJf9#FCw#{ zGP|a^3LTk&Q$!+TQVIlnlbf?FHkt*NLC6%{vmX@cC}3O zcVPg#(qQ4!?7oq?0H;L=V{+VNPsH=7HdM?7RA2;gM~w*}%H`|o{7>h$Yh(IKBmG*p z>=L}aOvvR7kJNN(bdXC%dF+zgjcJyG=vBKLXWsjLlT6jJtWx^ioU1AC^7l(~*i;$q zcSl~2m$P*d*dDX}2hC*opOG_sD~9d+DpEqJFas){wF#|gDz1oxsrXi#2}CC8=IJ>N z2m=Aque3RA|Nf8l>Pr7%Et!q|Pq0qjcs2~xrxN&QCqym0dX0J?-s8=F*N2iJ*wFp7 z@YCm7sYey{47cC=pZ=N;IHgfff14dZqn>ATYRwm2e2eWt#hJg~S=PovDRL{1~@D#q&a2}CWYQ4sV&6DXb)DX% z0%8 zfnaTcV#Qqp#obEqQXCT8A$X8La0r$!?eqNJ^B#GxbDi_oH-F`V?46m}vu4k_*IM_@ zGQh2EV}TL2Ak7B$c!oFjpNLT5zHHSg_-3Tpy7hJzz6Wed+7r|;zuP)Sh?P$nv|P1a zMs_2m48!jI=wq6QcIrH*{oa2&pSG3Rv3Asg(#cQ!+es1VEggHwWA`}>`&E(9asuTC z;Y;~}L$9$>`kl2|F2E8i#oSoXO79*lnkSy_d-ucZMm0(9`GB=dP0qOXinY%?9+%?Q zXd=s%M!X3OBP>+1V(!i8`8irKPU|AF<*<~PYWTqi--cFo*ID~AG8(p<79;$xWVbm8 zMVsF^w)j$N4gi;nt-wjW%F8d;^Xbm=m93KMi!|1i2ZI*7t7l*AFlhyt$&><=#nG(z z^m47+9ouP-Y;T+3z_hqLqRq{jfsZ-3ax!?tCAXTTNI#RjBZ<0aXxi3C;~&&t;~mk? zpIZ~XfkHRn+Q30pzM0c(R-MEtS4VUi)x0OW0`9m=@Rs(cz zNr7Dcv%c!Pka<8Q4OUp4ppc)tR1!W}y@#pL(ysC!#;Z)jYwhy&X9HVFWXLIQxM$gN z^Sk{4al~~ZH#SO>)1c4wdzg>Y!pgc3f4z+elOH1F!?iw58G0w z$@50LAGV##=$VmLa!*P-wXzB&D+|Ltc1=@7KE6veu35Co7HxJMuh6XC;6_Kw6Qqk$ zk0reujJp4OKIoRU2zGmtAwZiDGmba5xmy7O3OnADQm}P!5dv%32VJ}Cjc%~EyFCc$ z$8Z1|`tP4i2APOcH5VwF3>$((tj`<5ZzR11>T4H{KYj$3)Ebrpk*b;=KcSm6-U;8q z8L06S&&R^WLeYfz3zQmsHMvq#Vyy-T8p-E^Jdc~B6aoajp0v=|_+7Cp&h96{t_^Ps z-R7133ds+=q7i}MlU$u9NO4Ae~=hK6|PGHuk{S5E3y!@0A8p`C0c|PnkKS?;g)qfm?tfG&7i zCYmj2ErCQsXbS7E&&mOI69TC;3l9pC&_h`^T{Noh5g$I_pP@}{$Y}oXJmvsJY;EK# zaD%A)4UJZAxBE4ZYeWWzKzKtoNaG}v^7r*+6 z6*uoPnx*Ld{ELSWlCcnf6r4+2@evDd=Qqn`BaUA4T55F>Y!6!IP3poS;m?9`^CGZ^ zkeCK`ZKmX3117uq5Bo3MJOHidt`HS%lo$9bT=7g;w@P{5GN%jMr1Rb}niy~Y#5ezA zCA((fW_G2DN>j%t_d-c7numSi1flRy=^@iI?3JSH1y>FOE^~OCgy^2~2SH8E=AthT zcU-)k%91#WD$EokXZYgn=qDP3-E2w;6dWh@lZQHfn&^Nw?(M9Rz8e-%=VyLw&#q0} z1#Yj874%LFmQ4{VPP^6-Dz> zv8-f_$euO(MSqP;uYq;iD4~4$AW?Z{`rI8(!X(Ep&Dbf_hUJN{zTymjTT+{>DOO~E zME1QwZl4Zo!X&=tN4!=aZ8D7+D|O2p^OE>{i-L}u{N>WAwJ?TKwbh|+?NTMJ{diFH z>`_P@ih*a(RKs0_n0j#9oq9JzlTGCNCeG))4^&rbnMW|)fvk3KFY=uniz%bnDf;IC z5j9D(2DxIlu%0i{qcse#Gu3m{vH0h#Z!^C+oSa^}@T4F8 z`}7vwTV`rKJMYetyb~e?tByHBHFVxS)=uA{0sxOe)57k_z82rwkTd~y;ZjZo z{d-$n+40X!jULZ<$S4uuRIf%yJvMlpQ?$vcV*W!m=c83Rc?KW>GEY{s_&J#EyNE_L zF~+;IaG7a4(z?}7_kJuW!n!L*A&N^we&%U9RsKq@m_jiEdYq8)AR5R1!{FrU#hrYQ zkw46!LP5V2eH>#3Al$}c-?-N3%Rk5EG_}B(| zHG};D&Ds`_b!HXg7A7m2V|lAxo^M$Zd4p^>5P+bE)Wypdm39)$E+_d2LQ%S2A2n96 zbqb|^<_w1b&*03MOf?*U@=0VK`{^_rrhBf}JI(il_sH63)Zcwi`mG4dnOur)4o!Dr z_KwPm;OMX&nTqZ5RJ@B+H1)%Gckf_iZp@-DBP`1ka%~E!@l2z91gftv&ziOH1oDSF z>4Y<`HR zj*La`;bMWlwG~WJz0(vcVfFjw(J7-H1hI(9D-!{~`UxnlS}GK~Ps{M@UHqAM_>*{J zNpjo=T~F`X{hIGO2Ql;p`qdL` z&hc9B7nS0xg~Xi=TTW_O5?rWwDKNH=%%Sim3|}jT@B{T1(Kv)Vuz)k^ClPt=cod^v zF;h#=d-q%0f`qp?B%teNR54S@bcAMqk7=fYfJH;u`@wxSF^6_ZIW41}sL2T4;>x9*vgU4hGcHZ{+kY>7{L3 z(%ktiuXVBET%IoS*8Vp5XC~Rq9ORkvTD9}yI|)N+n834*@AiXdJ3gcRl{Y266$B>< z2d3TrO?Mc_8|D^L@;Y!)^P_kaj-3FHExAyad;^+BQIHCP@7}f-pL}n^&PG1zTM5Ll zl{kYO+dw5Hq2D_takGu*!zvuFee5shm$k32jpUwEKX6_&IeF#(Z8GPg+Qc1_l>R2? z-AYc(GBs;yOxXFWm9ru8Q%W%*j>n=lr)R^}IVCseJ>zbXky|YWleKlBZ3f#kd5*jI zF7ZTqZnI?OJcILLc!>N-DuC2`posz4n0$0OcUcS&i?C~XU3BWA`9T&A_1U&cvoxYV zx~gcQ*ZPto6>rV;&|K0}S)ug<6=F(eK=uLgZsN~_bT8%j>ZsLmwf2;cMk8m&6rB)m z+hUVNnxADdG;qG#Gm1>B+1rdRdj;_6aM^qkf+nbK%vwB`cX@GtRJxIV$oNS!Wl`nj zbU?k>Eu(pMzt@kXPBt|odCBW6ROsLmitMoBx^O{bZKJ4USDVfMq5el&-^)oShhxkm zxb2|}T4A?OZ_uec5*^&i8J+U0;4pPRnu~ho{kBl8Wqe+{R1O3>QrihkI#x9ob(S?w zk@M})SO0vn26r<^>L3o^^r3RQwH zsRlq)r!vbwY3<6bUis!i=m|*r~0( zM%eydLvE!q9;oe98gz=|;%^|&{Vi{Z6+4nv9LD)TPa3A%tgb-MbI^mmxjcJmjDH?s zW)M^Pkl2=r;)IpTt@<}@t{qb0n}U--9!)Q;x~R2DO7y>eGQ%im9#(D}7OA0W4gM$H zY#ZwFnQTDWTC^pWE9S6U3CQKRfCS?{R30D&=-r)qt#Hs9*9#xzVT6kl%dcL)aLh{G zAnuVC0k!Zg3jCp4xYbgz3%gkl$ttKS?y&h;a82lFJo+UOPp%>7vo0bNZV3hiK4IJJCF zs-`+kxTLrNtfnM}v&Hurwad%Sy;4sa_9DQgX%E_P8m?=^3D%OFntE}WGoGfZpN?fu z91V#I-|;+gEj9n3)_U`8T>IOlEk@^5>(G|Md_1CFDGgK92#V!@(-wIVwo@1Sj8m6M z4FHE8K56e{y~5=Ybq)l0+YG$aJCxxyZ^%8-LBR=5=|Z1nn*9DjR`rAGhEdiLa+rhT|Ih%@ zaH-MgO$o@V--&A2`{ZDgE3##y=``!rW`P=pF1~6qhPTUW&l53^P9XO-jy)s8{bog` zDejmj05al|IO` zy#-mm`;8y=tFRq=l97(5bQd%K?@A>U_84#0ZP2&QU$26z_HCl_s#U$+AB5S5GET|# zoa}{JXYF~Li(}+z@Q*uWZXV!~Zuj92k9%>|7$kX@<74Xzt2?&GckirE?)A*eW+ZN1Wwt zI*QAI{rJUSu5_cc)MRfw#6}1v6zRfxqlr#le|94#-NPQcG`fN$mGM1m6SV9Q8NEs` z!DqtE#4rEQRf&2!EUZXMZKt_8M)_;J-enY#XcUAfuem%gw7MWl1ES%w^nUk~F6ubK z)(FV3l)}sV26JxIx`l62qm>`OjxOAPIO(q%|0MZhxR@AwN&XL&dA%alGlaV6J(5br zXif;CYB#p;OTtgwc4?!rQP4O_2b4M5Dc>}Dtd%c$r$v2RZ5n@P1*T4HC^kwawOXuU z?z1>inl!;Y-|(f=NSvqekhxY0#)sjok*9I9vF9GB3*vJ$JK#C??xZ{{Rp`Pa%XX9h zwPV2`$x@xf1J=8b1af?EUm8hUtNL{up*tdW?<*8&nu=N=3d|PSsQi=)*Zri5Dbtgp zv6g(iC=lygswm&q_efg77$0-^i|aLLJiDFeq69zH4Nh-+;iuJ1MjSKzn?f4}S7TvcGIMn3L z+~}t^%0Q;~8r6UQA57(7OW( zXIL#f8OL9zXhNT5nelTKlVJd1X-#i{+ZMFB3RtK)pxP> z__0p2ElG7!w^Or%*VjGjqC|gyiT7%ODjxA-b3NL24Et4Hi%-DK$L=TBnjq?{vn@0$ z2ED=}=Rnk{LnVz@7u4v`21L?>1ZVmn%X0w!6I3t4Q!)H^N32WWhuI0@>WCwVkIZ7P z*(8MPSKZTgRFk9{iQqj67`oI1HUC^N>%C)^di2)I?F%sepQV%1;vJEwE8$(FjW~Z(e9442;7Lqk8s`8?7=)7*_jzogMkj1l)f9mtxJVtSYD`w@8LUIc~SvluCJ}4DWz~$STx8vLlj^D4 zoLB7l=oU`)kM2CM!2%nB_oR{2WAof2l$S{+XxZt~=B2m%ZmXpx6kg@Ov&T(M$b>~q zZmI&cO;E&D|ZPm(Si)Pyt@u-P@>0|xx1*!iMvIlm| zJGZOM7s=AGr@s1L?hSq=a(6F}7vpN4N-r(>a<=3MVIV7J z0whtH1#cJaB&jm5(uEO?`#3|TP+KX1BgoneFhaJkUFS`&Fj0Xx`@A{I8R!mn{9EK( ziV1K0;lApy{p8+3w90uAeyy@ODTQW3MlVatKO_|bbrQRe2*!4efmMXbr7?@BCVr)B zOUT9&T6_B?t}tM9r=yB=5$pud#Sc_{Oel>1+ELvV@G^$$qGX`P8PZ2;SJ z8OI3vz0V$;w0=-hQNiiTgX@fk%vd7SohwcSBq10fWl8kKmSPx&+zl%y*z~0GoX-at z|GAjR4@yVTg4Q7>pp{_2K+O>;{j2y#w99KEwQx9duN8Zv#l?Gi0BuK~=;hY{!q#N~ zQGTJ=pTom3!m-A0m{RYiytm7*V(yksYdfLpe$B)>O#NuvG> zY88|?TO`45=$o|ISYrtQ*RZS~npljTmB}BM+dTRk9Tf!leiDNMCeSRs9$(tZow_<7 z1Ri5mvnDEjXbcIgjdzu6MO^=i4~gKJU*cwny_4hin0jBkG%Jj-gPAMoS*OH~VBt@X zKV0=&(uJx%xhs(@llXb+k<#|zzeW>$T1-Ek(L`dMlZqo*i>ikxRVEW8NEaY{&|>KV_!hzt#w4*AT}DvR-Z|V z)GQFSD<|!w4Vx&IHYFg>41P#gtAn#gpEEDR&ULt6?C5<#gT?oC`Jl)}UiGG@6m-Nn>p6U0|HB;kd%S*FRmMTiNXQ1Sip%MA@fEXSwOOOB&oBKsq( zcGH~)qUkq4<;sc=vB;leZa5{N=R}M1M4ZASv26LB3}#GHYH4Dp>9^^`(C4Ww7x#Cy zFSIm71U`y?-^awQJ_Srv*DSLPmd$U1M5$~@?#uM1yu3Y)ztOOHi2z|s0}f0EL*V1H z*o_m9S_hdPPVjisVdz`wg%--SZ+^AD*xtC+m!@UF{7jAxF45qT>wQ4>dI|>-4xD5~ z<|pOx3D;6W`5Ap8uX7(Q7qRy9tw(i+eCD?(LBylk(}>3KAGDYEus8(%@7fJ>yBOsa z9k|ZTn_EX*1^vu*j2HF85$pATg#{X67s_PSd!Hz5 z_m=zIZO-}v1>eRccx}mc$~?tbfMjza2T`ObSQG#2X*aRx=>sfx!hZsV-1b9B&j0~0 zM6OS-*BmU>MDt8?K|IS*-zOR;dEXFc+8wQO0Kds+N;Y@vygzvhkv(1~UIFeoXtH$nFnv#b7T?%)0p3VLnuJ*5V^ z-1;4JVFiWY6OIT3lF#9Ok8Zfb_-8l%yP@C7A9n=VyQKVY$vnWYT86k8l{p%TIVjaPuib0>9*b%WKtM?AujM!D|#rE z!ELBrGjs737xDwF|K=YeMDlV>aGL?sc_t0q6a#dKj+k1?_-*gfzGF@#iD zg+elc>fm7aYOBaXHtu8pL*0{BTMCPIt;#wp)V4M(Z{deU$z!hk=}SWvoJBYRsShV|E(M03!1l46IN= zC}0{EhPXcU);8k)45jgVkIW3iR)*^(hhsH)*l2)Bo7NB9ZM+jD66_;wY@57sg-CzoP@pnOYroYx@jSO0uo-ZJey?5N}?x za5QuaIb}Hk%jOH(?3@YH0swgn0_wBXOd&h`*1iKns2zhBqC`L2hXfV0kyUqdM5Ic{ zOyPr0IR>9Lt>i0dQ)~67W_E2fn1$MPcGr`n+r3ThCu^$gT4|A1+o|Z+^;{V z_MK3_GCuS0lGqC&cWALSj9328$OJL#Ju+X|E8>~)(^Ni{4AE<7Rzmzvz~H9M=YwdP z16t;!v-8wrk1(B^B|=Egrl4I~);x78nnv`ycj&?%j^^O|L7Eu3rB97yTR`!m3d|wHq~3~* z#L~aG>;$;VZib0V@v|Y3dYzMJ!rQWiSXlrzLTL-WapP z4uVjX{e(8w7~=3u5t+ow>?DS)wI*wOpOc&|E9fkP|1b|uF|%S69;a~4KN(RcIKf>~ zy{{e|Y~Hl8RxS8`>C|W93Pjv2txso?Y!X8V1C#E{JIumr$5d0)!Nu60RGiEty=wE} z-W5tya(8)yQS1Ha15JC~PJ3j%VR`7~2d?z>=4&D;o+1Rk05>7mAH=x|W zU_6+X`FWXpr2pFydh_~b=1yjSSDO$8gFbUo+Z*AlU2Iob4kK;(N6>s^-?i`y1;Xn& zH3G9wKqRihT9CD*xGd}2)9+ykbRmWPG}G_&|XDe=b@ngeoy zE>BPB`txjp-ASefsTRu=sCNS-86s}8)tK8s#!Fqt!KU6$iVyyFhdZr~OX#E1xF=o# z4Pca_bqW#2pBZ1f5(_xA8U-OJ(-I(G?A73M=sVgfug9-Pmm*EI1}yD#LD-=g`#E z0tZ4`iNY<<^{5JhB4pdvgnxYR4CNdkH2J461)uZVgM(H~iBR4pt~if3#C{uD;O#Wm zmY;aimK)SYW*eRslO8xq?^BizM9@fnEkK!m;O$lN%$%q_R>n}WIlAN zzklXZRliQ2!{qyB?relC@XE(YuTI%FOT_EI0a~|vS1Obn)Mvmcq-XEfdv~I=XN}o( z;=TWAQu+A0@b1hQOb5(zb(s%bRP`t`Zv&lgi%#=P=-%D(W@bwD2bP*Zs+%OH4(W8w zZY!b#)z66tGR5bYQQ z?@Tt;uTaVf^u8GjLKz2~OGjKvbK({(98?6emP9_NA!v?^8;j4`eDJ^Cr#GW4DH>?7ZcaM?VT^ z4>az5D8|QMqA;S~?=&IBG2K5^!g z=rnbYn~P2lT#CPa^!eT8fi8bG>zAk8m&)-7_?s)Zm)mAQulQ2ano}xBe<;Z0O6G-9 zJ>8l57V$Q8y-CJhprZ^vT16yXY3uJYGp{!ki}HLpeJ_LE%`7_YG;_9z^_m*(6L4tdz>i15&?{BZ=gjHDE3foKmW%Y;AKoPb}H&WeFsnIuXkf#t9U ziBmV^) zN@Mz_r0sdZ?|Z2Q?_{o~psS{SvsN+&d|=>Cn}wFJUb(ObJhh7b$8J8;ZI%BhBG|ga zb$QMBT7+)vJqHmXAgH?nte@uw!I+eS`=i_-vCcmyrp`Wo@l{79hC7Ly^3)1LZ<}3w zlpkCAFo4!4I=e|@`B&<$@q76%pP~`!T@sbG!#{mP1zdOBuI2XgUs_9~YJa_T7%x#N z&IVLwh-q`Y&YKte>DsQ<@L?;z#o%|1)A=_eOT{IExLa{7spLaWr^^sgdkVul?ytGz zk~|)fiB+ZI|GraRh>y#0j;#n1!hI>9F0f8r(Xp_^qj4>k&?#+2LnJ!ZISa?cNd22m zqGuxYo!%)Bv#Vh`_#w5d+RX)$02F6O_T72SR{WA1gq=JL@1P0nqU$eR>l(a?C&RG_3s0>%0#LKb?)L3RdS#7kRChF391nz8p;$eG{Vb|!~YB`3_vlaO> zA_?Ba-BISoP9~OPI`+cHJr$z$aYI#LUMWf3US2ObC7NX~Nb}uNXT(m}al^g9QTbDK zE->gbi5lwkjIBDBbEyk%?w^pS5YGgD3uw1E4M*Kba*qvQtgN+nP`FaH=jxbJ`XF%3 zV^m@lvBXhx8s;zdD3UpTJw*DPg157)I?um{QSYQE+2)roL}tLvS@G|-NElBlUJ`vd zDcqH`<+nByv12H;r!bYGh%BV6hdjrGK$am}3VVDRQeG7QP0KM39}X2nzm&=L<4VRp z_F}Bq3N5C79Lmo4zn+zPy&jMmRbyCevN*6f4U9xxYN95gnh2bsiPumk&gKI8@ zQv*yhPsROrb^AYv9e(aCwHGD2`W+U+?Vi>{ zIH#Y}d4^l3*jq=C6EgCD0WE%V5ap-W*+!L-UL_@(8$)I(j{;w3`dudNiMTpW@zc~> z*PL7Vi)sdp?@rBLpC(vAUn0n$#Vv z;levysRdYJ#4KwNO)Qfpf4xksf9~@AT|`zaO(`6st4P!@LsUF{RL-;4G`;@rMc2Rg z!p7X|-q@YhX=4C~A3rrY=CGM%w|V~0f47AHp9wK#G31=@dyV;;!ZeK~=KxcF|Cx(N zon|>Qn0;+PjygcR);13Hm-!O|eqoT{e{T6d;$UI_r`h`d_vM}pNw579ACrL!qvoN1 zYXGsN3jY9-K02y0ohv20cYN>sTl!#2B}Ke<_pQP_R-7%F6j*)^?|)0#Z~5OC>foy! zKd(Rf7R>~Xn!AU~0u(&xec8pT$bY73PyP=mX6k)DH-|lT%jc0ZA5gxS8DO`9hAl1f z7FJslP)lA>yrtF)r-@b4GT05HQr{rHA4|jbx}O1G&}SLDBc8P?3y103)RS<-3JuE_ zO$`rRU%ZNCV9NLET9jB6SB7$TaJM|TZ7*rkNiieG+bn=)NYD(Ut@afo*CRNQSWX}Q z_9SrQF*RNwfj5x)vVBShyA__9JJC#Zblj=zr4p;^=o#qk$K}Y2egWG3dU6f>d&%<8 zKbN!$CP@ddY9I&THRwVOE&XB5V`RusBy4m#`jM*I#*ie0uPu2#D-}NRSM(YSOX15h zca;NZm%8At-!$45tU7t095$G}a%`=bM10Slnq>*8_Q^Tf@HWpCcUEIJg;R4Gc0S0O z#bw+(;V_&~PuMi@GQXi40bOmxP%8TRkI%c8Abv$oUrqbhZ2uHTS!TmrTP-b||834p$GfdHAqt(RHRoWh?+5kb(Q2`NOfQnVwSENo-jEauha@p+BAn(94-sHEH z*?t>-g=64{cXKG|_w2ck1t_)zh>;t$I{1Yuc3-lg$;xkAmi&IK3tZ51WLni^S?e`g z8{{Wxl{6sysxQRbap#R;wIV*cNl+cLkxA!LW0)iKP^xaAYc^fyOz+j{J~Nyuh9 zm4s%~RPsP3QI(qQ2>5yH36jatodZtu4){+s*YXx-oq2AbMB%K+#g2uyoCWBwv7j7p zbt;GMtwddVa#xqZ2cHofO7v@zXQ;HAY|HN6&rsZ8f5g}p*BEYUPs`#O7cJN3Yk|Gk zo^@k^^boRmYFVC9P-8L-&eY;DLE2J$ORMu}sV`)yru(#6qi8O<`MdTy?J%o0vH~{S zxy(F29-yYFw?vh{dk`}RGL%gnAR;?bs%;BF?66V^wXAS+46qaa3$`tjV_pwuBg>-- zve|T2^o2(PPo!1T&#%jtch0U=WeaPW?)I>a?#7>ft^eF@^pasFwUK=ge*q3jE)l#2 z9nr)5lIk^+p!y;f8c7UBYd*t6m$n-2tyI>RmmXn?D(eGykW{^YUUt!UJIzdy91jj4 z0HC_5&oL>ULzK@Q%PU_Klk<-nO5UDWe3*Qq;Vdpz(Un^F_yQ_16N8yarz8X=beCtB zIS6Wcdvmg-XphrrFFH8{=SG+8TI17QPwON*E;{@^PU)YroJt$G6auO8{bM)E7`yRr zD7r;vz!m(<+YMr6yThX1C026xgtR>xn!9;j`Ad3R`DNjYklT*LulE>@okKv|Qnjy$ zUO!3n4XdeOkw20N$&M99ZeMIT49=>fC;?OQwnhz9kRmC3gQ!pLT45ph!%!Q?%E3GyncggXw zCt7_irkEIPjo#;1R5PoE3G&i7Zx`d__k5NlMJuaz~Ui??k%y);oJJgO&MjpJ1=Kz0Gzp!Mf>KA>KBc= z@}*a?WNSZqtcKrsR683jeeNx2^o%@gfCeEU&XkTJgej%Ib(DYrH`_boE2eh8WJnam1)bxb%E6!-Ug^BuBf%ftw8@iE{ zs0RGSw)L|+^KBc<_l?8aZ>}oaZ`aU}jjMCy(d`WY&B_9(d0fjY=kex7)n`n_-E4{; z+0@#RE|{ay$5Ci9nln`E*MD#1X>x0{Qf;;owuju^ny=z^=W?o{n>fyd=1O?Mh#+Nb z@GHFUQ}r2jLrA}!^CO<8bAo}`jb}G6Z2dm&Y=(++p1wNsK5xmjpsPZ(VTRjlDh_|7 zVta26M*{1IVCLF!Pg=!s5Qm;GtZx)=_#>>qBpEX*oq1Yai0@g%gfN5S7q@#?Se^xa zrxT<+p~KN`rFx&G&yNnJE@fhUnVKRJR8@xp07nex!_Sgc8na9fBzWc_)`pGiZv<7; zi|2|5-S`crP4Y|`(L*oAO_lDkAB$qSyghmI5;73IKd>vOoAH{$Y9O>#v@VvUopbV7 zh6^#&ZilDjiW(P}RZNRg8zBIEg)zNXJ|n-!)GDT$@ck|!G<=AbY_1*&d1=#sJWwj= z$}F=@WT0S3>a#pj6QRouJ=`}y@Gy~KKB`W^Jhfw;!vLfBrZ~V7_ zJVr9^KKd1Ju-D4p=yk1KHDJi0n^>u@X$QyS(QeN`ZNlX?ekQLdVX(9fYkvuyV~8;qzZ^@{1z4n^-#O4wVa@Qa$whTc0C{35mxZTqk6+B41;#`vGeJ;N$OHA7ucRg$ z)I7WO9TIbXE;Cu|n~U|2F5*Y`?Y^c>p;U0|j}K<;I2rb|mzJIleWcwEG=5Ru% z4Mv6bg)o#ZUYc**&#;&8>cPvbw$aI+SUpZIEs=PKN-)b3FOuH%qk^dlTE(kZ?bcY{ zD0!!N4LY6DK&wTKEUG)+o*5?AwemD{h!SA*X>eUh1{)%W&`%!ZoBwl~G zTt&d%R@BU-@AiXvFSqCMgNgf`S{pA1496oIJ!1n0>?wL*XWI$t(b3@)KkJXN^G4oI&4YYQLm)DIF{tcy>?bMsmKKemxV5XVaW_>gMd09p*oEBu~gVdifO7x%vEO zQzGnx3U0B_yV@?|J}4c`2Aw`2+s7g{WUhRLHsv*PwNYiMwQwJ6K5cU5`NaDG%b0y$ zwAbblOl)0gUw3AYR*0UGeM&<=1Tu~LPN|}9^+nTJHcjJRP|HVDD3jGbb{qbWVREOX z*UN0i35z(P{<&8sy!xw}^)U?{lTGekUck+kqM=?gp>vOPQcC}Ny}^Es$Lilf4Dd_u z0;f$B_`w&$y* z)2pgut&*W4BD6`uKVSj5bMzr%=^Hx`eU1ibPZBS$B4jPlNd%McHLp%8+~K5Np`(@Z{g8!2kjdJ zmfp*hhZB}GEG92rR3rBMIoZSqB}pi}4lfs!?+)o(_;h077L*uH)N6lwFFG4wqS1b! zHzk~8VMQRsKWyBRf_>QHjMI_(y0>W8GU;G~wOx6NCQc21 zZn2P#d9!CVh*tp$cD^+22s(cCTv#GEN^b}hj;XtjSxTfNn0xGV7F1+&BB%S6#9L2* zj+!&`T{qv{RdE7mIUt|HthGl}MaOW-K3tt~R8W^s z)6&4aN5Y}`B7dRG7a;o^#(As$S#C2HuL$?S&*WJ7jWTv7?T z*lOTU)rtjjKrdbtAE;n^)tbJ1jMu{YBlqaJn00Hsn%ZWH$1_9Q#F8-{cytkS_0_Ng z+kH{upn@lc+Yv>Z(9Ul=#082rmSip2a?Rb}|ATj#H$0^lP}FLA@%>9vT^2Ot z+EBsnoB#PWE4FZA*^$}njsLpJSTl?ml!=p#SzKUCxDkCRXEAv5jA? zVA`7n?QQczU%}XmZk;#Q?4Ry@yo2dZD4*2z@roJGGr~v7C9d@YyZ@kMF7E%uTha04 z{~)mP_7szSG^4pO_>=ULFSsP>+VG!0OTGRtm~8pp zAA+*KEsy~Nll}7%>(7+@`%&o4|Ld2d`vUH?Hbn(+${xD^x$2#qs%*s@)8PLF3y{So literal 66908 zcmdSA2UJs8+c1j5C@KOvDvAi`s32XXNsWq#bfrp1DWM~s5JFT$R1}mJsR8LVbfiW^ zYG~2|gb<_?NRT8Dk`VHT8J%~&yS{(jb^mq0yVl)HJSQj5+56c~+fU1@dxqLPhtC}5 z;NalVxqI7~gX5qy2gh&O2M@5H$n5~_*q1|(?^^nEaPWTJ``ec&$t%dgagIah_D$1} z$u%tGLz44TrCnKBxif>antX&m3D-8SFCR$$GRIxGkH4Y4THI6jc|m>V3m2=)d2XMq zE?;UXdyzZK{qev@9_!ba3VwIrRtdN~9N*@Bg%K?#EPgKi*fz7ZX^NBc$l<`jwg?fTK&A`czw%t-_QHYhqLzo zyv@A)oc|Z_$Ny8{kJ+N^If62ZrUH5=k}FDWRRZ;BfG~OI2&3+LF{)7=FZkXjden(n zEdrcOqZL}FU(8(zx`BAJGe!{j);W=z+b~(Awk1+^E%<(*9k@o+BkrQDanRsXaR8}G zV0Yl;3I?KN;cgHp!Gieq$eexOQre8HyfznQn;yxNdciwfV(F~S6yxiYLwrEepf`B> zkX`-tU}k`sM89y{n0uXj|Mm_!N9(P0vNik+Iy4o@RT^4<4FH+nkP6#PG)0tPD38#U z3D8P=2@U9@7(k_zt;#2^H8+Ugz^4{A(Sp(jf8*dN2bWqW;w>z4>7(t*43U%d?j$XN zaYw!aG#x*suvMJgQ1YX_p@fc&8d_*G6pVy8oAK{5>ioJnM+-5teF}x3Kwoz|+qC*% zLmk>F(RncLY@Bdo=#yY(iYaS$F>L)sk10a%y|rvW%5+7;q*O+DpbV{hwnTaMqiGX@ zIG~WMyt1YbdeoP+URtKJ8LyB`nlI5`so}ylZt8GwJfSuPL2c{2vt9Oh5u?~!U0$!T zwA5-HmAmX{ujh@b0Afe49K>F@P}Z!zbaw1gVX>aBUb>Z{rG09T-W}(mXAO$b97+M% zpW)nQJm!Wr^Y97BL+fzu-pZmt1E2Bhvmez+5w8Ql_F-g&WWYvr{rraF7Qrstf0f`k zgeKgF|IWb?0-N+;)5^g@;L7*!YRnOnJ}5annaf8weqUrnot}#maeeDaPZ>fo#J$C$ zr>DV_p8xr`{YClqK{et$X*!1ER-a zsf4_p9F*Tc8`84{3ZUN*g_~yNWi+pq>8Cd8HnN%Zqs&s6jlnoXB_Xe~HQRsmB(w(r z@i1>Y=j~+&@-p+gu{!X!jE0?>0(C+_Yf7V*H8SD^yNrFBFBVWNXm3i|&0=m$jy#sPtZSUynKS!Drq63@F7=v6Ytcv`K8eIqS6d2m44?<&wk)hR zr`}A3;2bTRoJ_36bYON1E6@>`|D-r(+faJ!K8}(AomgL7>{lh~F2pNX#v6EPkgEN! z&6X3?M4G}4-1*I1CVQ1Fw`~l@F6!*bKB?;nfQBl(9-<1jH%YG#HbPPAxlNiYWmZO{ z7g&r>wU8jNxw#8|Z!G|MJWPgzimqW>%nJ0jqC6b5kiiX zlj-^Zl`w+wzV*^jQ0ljBGY^zZ?@XRe{zO2!u+_!P2PDS=mZh^lpPQ!<(SiQiaQYF(6}|(9;N<|F&d55#S@S9M7r2T6lAI>alur zv_`!qp&^RWZ6<5sLT6T0F6ZJ$O1wiVJS=+v6S618(o()hC7ZA%cbad{BtOBHQ5+Xm z;GV#Hl`^BjaF|@ac}>WzW^d=fMI)pN8931MQZoFVf6Vyqla9(2*nBZJze0iu%uE(7 z%252B-%9A5ZGui)Om|Ms=+p8BG|+jr-sBoI5H(j~S$o}o-6pM`ZwkHGu)xwsT>?t? zdKZCQYlNdFc zP9lesbw#I?l~1Z$`K%UM_jg$id(?&i`zrCh_CZ>U1L2;6aG#A=YEbF~jXu)1dRZ&v+cce<9^H)npV_L5l&(cRx z0JuRVo9{BN;~3tAJiI|$T8Xx*#jY;`@%zc-@vc?B)$v_xUQgmJ7FDTkp+r!^y4nrf zQ`DPA8I{@{u=ZO$1nn`HlQmeV>|1TTg793mS=X&e6o5{S8lX+1TYgMUnF5rQ64|Ov z08;yiAGDqGr)8)6Hfx$0e))X-o_#$Df?Q4{Hm?%?kw9jjAMJl06C>F>MbSmydI z$Rnw5;LR&?RRO-i#%}t>wK22{@hie*_PB@DlnI;3f09p`F3|Nb#$AmJ^`dk+4g2J3 zSJZ_0Vfy@=sE&E~t8v<&;s>;&dN21Jgy_wz` ziL1r_=xxc`$sh$e+1{^=S{CO$b>^3(r;)c%F3vlNR3JDwGI$eAiLy*)FSLAB%0m4I zDo+E{sTAXRxPPQM%v!X+BpgTS=0~b(zd;ElzSUb(OY3gqE*n+0aKguX>_i>s;8nXl zM^u{Xe}b6QvrlGiZ&f7d9Y7)3x9^QBg!c-EM5Oa>q)2NOv$8E0HN! zcWHf6n!ZYK|0D|c-k_qUtR4il1h6|V!^9j&(JZSF2~A`JS-kw^I^WI^B|sDpP{?QmudPAhOJ~$#n zf6=Mz0ljX=6DbF-a-lNvt4)T~z zXB^T*8P}DFVx$&qcZEws-5!!F1tX%I9Ogz&3N_?}ZMWAuJhnWL@i<%tNh3}@_Mt9QU| zTJ1{M?n7xH-+2Dx`>ntvXDz{^2P!*)^_dm6x+vofM80C34&Dm%s8J>w?kW6P7w%iX z^Jdpzcqmxe^+88E7O(&pP9n=9)%omHrKA@>7JxO3_|srd=fr@N*0FORD|4&#KzqeBKBn_=DgdUZzIv;>R(IBvTlS#xn`w+6NYA zqoWjfE^lq`ysTdKRqvQ5b+gcT z*jhH}0gDPyk^^RnxZ|O3x)y)lI>aV+MM^jVu6VWwC}?+Ic&UUI-gj z8@<*O5hINg33-eX3uL+A_y|AHCDyj7`2C5=p$ikVtu-N^yj|AzH~xsWl}Tx5%+A-> zB_TDS;?Dc<@kduG*PgHXb($>!w}Gf(NAgqr|8dKSc>@UK0@8 zEZP%A(VKAL49Jb*?K+h*U|8f4H223bW{l^VOoO<9@%#lwCq)PJ5Tcs62qmtm>=4V| zz)e4S-m$!+-y(5ReR3!n?KE0h)SIq;36=&W2f=bD3WK_tv*|YWssR>;w60ZBv}K*s zm#(Mc9t!$qFf(p%6ezpD-xfhE+5}ilAJ>gtM_0lXb`+1SRHqvshU%&C=pZ8$F5Sq}~S8h<&r-X*`a@+!Fvu|&&lcLLEi$dL~p&tPe`u3gb{nQw~M zh=-mDbj$0_NG*CKrOoygPnc;Nur+zjFc*s&E-sPkk9t)0I1AVbHh+r!im*q(mJ&hb1K9tk62`xgc5{WD{l|VWk-oA zY5qSH52T+iN4KXnBS_oxic}Ht!}`$03F<%QQrqS?HnkTY6M?XoF949;&Y#9 zWxlN{50Hai2gt?enA9D){ypuDy7S$hQr5P{UJ ziM2}^Mx?>>zGKUr1+H&@J#t3nob6&p!Uo@(vweuAS5Q;u8k{p$w=(k~IT_z%4m&cw zs5v=mNiuC+yxE+R1j*zXL+6oRldokv+^gCP%v%9oj;|vF&5~7!O;ekV9q-460#rjB zBs2i>olJytJ*YPUtBUzzmr1#>An@Zo>Bwmn(Zb@sFHq!_IpCPQjovWnYjL*8w6>}j zlUiW?!e;YvPq?pjcnGS*)T`olZ;GjUsDD2=!0LcqD7Z|erIV54xY;-E#+T2+;s(Al z_`L*@tm`DW^X=b9R;{C!yX#!mTD>#a2a3{P2g=rUM!?GZoS@-5$Y|2c=E;fUFMJy| zFYA$y*N+w!8qmG(R{)$xp^846%asEe*zv$8oQMJdmm?YluiYhp%ttWo}TO(VG#ftro*BpK* z3hxUKd>C$yX|kL|P=oUIeKs4>r8-OoQRW`3sPL4FaqIBPio%~tHs9ivM>snuu8;Y#u>O>D53s{0^ou&?U|Ql9J7|21`jPyWF_uf` zD~{8Urb-hitzIJQu{D-EW$T9|D`yi4Q;Slw~5r=JFk~%^_%bx z=TQ;6ZiEOtzTQZxca~r7X-W5v2heF_hCGaY0ea-4EvFu_(ClbwRy;iByp-p%y<**7RZ zraE9?I9B6bcN^+3zUt4KVq6rYQqwK;to+3`B*>ZgVH353|BmtBy}!Tv?WG!8JWKvJ zjwj72e@kv?e69IfG^})zLRZ~*|2oaz%96a*pp_gjIg#9uDQbXy46${Gr@q(g z$iP0bj1U%sT)4pN+IH0)IB#vyuN&C*c_9U7wm7*8RN(uubzLEXZ<6F_y9@k&_H?uf zlb+WwGTKZnV>Q5h`nPqW4HO^V;tY3V{fpKGqxp-T8`FU%L{Gh7FkU_ud^uXcGhNR#O&ls zMT@9EqWMn~ZZ3*lXUwtu&mJHIf?3nfqz9aG@#yzbwZVW`Sj)EAmkTB5ARDahgy zeGD>iv)Pin>nYW>N4jwVw0kK@C+<4S&}^LJoIdBWbMG72tx1L!%qX^dzjmJw9uvrZ zYa^RDxd`m($VfSFYYmzcq0gJ756MUCVV@Pt;pHsW$4AEyf97wdBo^{~ho`VZ;*m7T zpzXSxN2Zo8hF;6}Q4(^#rp}_oT3j|e>kKlL0X1FieXNxiwwCfnwNhJG^`fP1dQVge z!+8ZLc`Y+hrWU=~y*Zk^v58Pj$LzQ~>H+xqw%nVR&J~Jlm~2npt|L@7mLPDH*-SNCHzK9mw*mvueISqwQL0Pk|U#uD}$GC z?TVw=*$MefySf>CnA_l0Wx8Cx(`*_$sY@`vGHw^2lGs@EC-&NWrM2AFj8~l{taS-U zDumP>;h>yJT4y>usKq+=82h#>tY0Y~%+!oEwb^ti17t%RJcE~p5}@BJr@(E?3Bm5x z8%(1!@vlpJJM0umbeo!*eLbXo0%JY;zXz^PhuUM6dZO%;jy0z+R^N~{VC*k^0e0Ls)CJ+(+CN-X!B&yevi+lbE+t9qreD%xn4h1w{9PHO(-W9mYKOkDZD{ zYTI6W9QGYjDk>+Imf*-G6u*aowndmiyfDJ62*oqURU2wQu}XJ}BE887b0#-?V?XK( zklXWul(%wxoI2tE7mi2+KIii{$SxaDJg%sd%hC+y|j5y%( z^|>{vt2+8xc_$HJwW1xhP!uK7UwZrwu|-=;pw?DzT-EnXYKiMMtd9PEOPJ9L37uL(?cux6*)1+J4mPN}RC^-oVZ=2QB1 z^KD`Frqk)&xU)%5xlWEK>=n0IN0bR4r|6#t{5vBxha9r=oAW^4P8(y5NsTsi76@-2Ab0hha;9uB zO{R;>?brpidpB_!Y%vQM?}seVeFgk}BvBLLb{cBVm%QqgdAiNyyk$QL$FVctE^9S> zi!pKut$~PDR?A*6p8OjK%CJM*fJm})88tC6u$D3w+ zQ`%AEQ>=AsZ5B^&$lWPFF94wtK=YlBL%|;3^O`G+woZRIG#s7*7EOnLhs*e=Af?x7 zd`qV+*2dLOu!~1N?thgs-S=vlmtXufSA#x_4MDqJrqKtq^GZ;Py2>Fo#AJmliLt#sQ(B`S z!f1+o;!v|1d^yoMd$%0-PTb=D>xP5l#{IoQ z_uoV7{GLhtYn+JObAQt{VmUQ<>7x;;KNGt}a9qZMCBO+GEyhikYKK@K} z0qu#--uQo|4$d&$$Cli`djb9jtL<})6_oeXr*8smx!D^}2(IWdthaK%oq$ycp$6^O znI61Yzm$Qp_G!*lVhi%#xFJZW8qiQ)D{u3|FMm^B&WuR0!=1eO;;))9A|-sL25pBz zcf37r{x>|wli-JN$X54TrKxwEd}>z+lQkZ6O!DNq*V`cW{D-A}f_-kJj9L3E^t%mL zhBN7&+>1;ey1(iHq&!plk=g43!*Q|=G|1?E7ELq{Mw0(K;Z17g4f5);k-%oOKc$$4*JFE)6 zw(q}fr6`>|DJLcW$NyOK#d5)+pce?IAxf6G$9NHq)}{ZcvC9lUB_@`kpa?NfgGsLY zzxcVyaq>awwLq2p^;4Y-y|Th_M24sWbmY&U->xM;=igEha~kQ6%4z)diUS|oj$M@a zd$c>`MV77vnHmD9IcJ#_#1)&NA6CrCk{zahVc3spH9DroOL8e;-MM3_=ZvDeu`&r$ z59FH0t((|)CySj;Z-HVIF1_6xzXz{@hN=(jdWMEmwukZ{+7M`{L}-m#*u+vEuwyAs zDmDJtDV+5^qFepUd^YJDU!b85qvIM3*o`3e#R}WS@#Bcg+~jzKP+b4pz^0vcO3Tc1 z3bA3u-)9GXyq-C$N-nYE{2S-byjO;ANviPC?|3&qw21+fpDTZN@lyIQ^Ycw*CI-;8 zKSE`l6iU%p0M~D%WhqlG?0*`0(ak=t9}5Qlu#oM}F$rB?u0wrisVFOD_}hdA`y7;# zPqa*GGitf0q^~A!En5SQ@jfCYALfmWbE^0LXW2ij#;3Gf;?~o`L$)yHsOt5UHYDk- z>DtiL7hmO-Vdseivj?d~jmY$>J2Aa4ug>{*>JEqZxGOdDDPI`fo{SFZQOH)FSEHlt@?u*isWKuQ?HN>c$8}0t|U9r%GB>L;!2U0%o{SQ z{)VBvD(AB`z#*ZqK%=ETM^QrK;t!}myylWE>{QTlmt@rpV`-toBB;8XpEx*($Q z=YBnz0K@i9ji&U0l0FiMx`+o+MOV@?xP+71xP%iOW7pz|48^q}LfcBbC!qQs`Oes- z33*vfDn;XCFsa8lU+C}EpL{G=yUjhY_{zdU$ty8|ukWNIS1xerVmjF6$GFe@GBds( zi}G6rBIEoA&Q>!2WUan^HmHd{8~Vjfx!T$hgfq|rt;$JL)R*YJA2cAtI4{`z_Qmc= zDX5p7a7w6IKjdQ`Xo=q7DXsF2#@QSnU@+0yAPRlJ=3FXnHGGX$Px-1@X&)D$dQr?S zRF8Z$AEG1{Ja*wHZN>xH#7;itqLH2wu4!T(ck$ASva4tKbyrVGa9KaMKhOvHc_|6F zFs?ateb;_y^^;A&urG%BU~~;=;{QNXU4`2H3YPJX70j4YYkcIj#Td^_+v(0QV2q)g zK6KU^8&YrL`rh2s*(E*lRwJiOY52JDvW(X~U+V@K;ACw-SR<Y< z512}YHqK2nRXns-9R<=5#ID#3Z%vQ~aOqonah=^zWlD*srur6RTc>`8tIoNDvQod< zQz6gj_sV>5duV58UGWyLCa+uvs0&RsD`PR!*a8?$|K+tcCJf$^9P4kC z0YaFIYV^sTl2X}_>=q55o$u|Y)0E9Ha_2yv*YRT|ea8W5?*v8cQxf-!6tGY|Qjnd+ zH1I}TY0|K1tVPn&TP5oj3l=QVQf0m0hF0o9x*h#n(o&2FHWkP!SJ`~q8qKo_avT*g z!99SB1D8rtg;7I^9;sCElHpvcOQHAAy?>S&`^yOnLqK1>6;yJW>v(I442K*L0hd(m zbzSZa)o!Ekow=|XPLaDy{pnTfEsVVCYIz}*fV+)BgrPFq#$dDBzny(H@RR@JuEqa3 z5AA<5^H#&)EjPEq|LpUoj!w6XG!(%tO5P=wAJCuomt78j&O1R#=`#LmYpo2%S(s%Ao^RGyvHruu_$PL{BQ3+ap z2icks+nZA*5TCGj&&Jyg*8-WWgHoea1Ez}4lx!d%jDWTAt3s+lz4Xo1_=|#*bv`NY zf4Nz%wsYfPMi3#TV_G$6?L!a&Q+GNQC!j*~dW&v3zfzHvb5T&y^KM+g^5FSZG{Wbw z)WKhN_H)viwu5Z9RWFKHXsE)mGa)G{kG*#x)qrvFD#;W0Pjw1*j4eXy8JUR_j~*by zoBs)QJUbV&B6)>WJP%6+tbO3+$%;%@K;5c@N%Osz@2$Rf^noibOHy;s^#z2a-ZUngll#@e z`JZ?@AnyG;1ft$fkL0gy8~PO?HJ#d99mF8iRr$8P$g3c#Lz=8z9}1o^%J@eczzv%` za#Eg`4@zGIZ+)oy=lRt!2+7FoV9c6KosP@U53yA%y8SAFk&=baXBIu$xE;YW>@9 z|ICQ_@6VGjq@!y$LxCfd6@lFITj5;cm*n!GTB?83_A?`G@C&gz2xPZCy4FhzG_$j= zRoT=bwPuK`F26f#fB${!f^Be9FVCiB{roFa4Gr0WCgXUeX?@*>#D&kXB8kWE95KLo zN2!8i5WY_KI5oYTdGT zs;91RdI)H-|Tox2-Zr-z%%Z1DS9>q`P|V@t6IVBZNF0_Nr}I&zXYzHek!Td zklA8uvh`}tp9G6q97IaG&28YTcD{)xUm>&`Ed8l&;!C4m!+(wo24#3u&b*(XO*}hg zT;*=kO>-EaFy|i%s0Fyj;79A*i9WrJ?3Qqd?{LSdmQ(>TmRHf@RQv?lUVrp;4ZB10 z>$^1holg`ZofwODPlQ5j`#-Hrc)>Baf&O}!A|)0QwfI^DdZ}4($=$L>-bjru0t(oO z8!36vSjeS7Y0GMh!I08UWDFeXJ2{Mun~q#xcLn{VIXI3@^Gd&)WzCdDH3qEDj)fwP zsMVs#{$3u7m6!W=xlKYg*W6FzkzcHvE`H?7FsXd{I7d=?5ZQX(_xGsK~Ri zrH^y$iNm2U>V$7%(EfC>)Tq}mB;8Zf;k*u>{AC@sG*8MfKs4=Q#8P%`w+b~>mCMUH zcPlSaSr{SE#~)y1y^6ws`nzJd^}FL@;j&^dALu>sLQ~&w+^}#emW$uKA2oW)ppVTDv&J5UkW9II?WLqS&k+X$cm`n@3U^DU+a-@K~DfMK6Yper$o9CZ0eWfkgY0J>#E zP-rFIgBP(rOdfr(obfu^yBkZ@A9%AeOpcF^$^*7($m%yM$L|BnICa^d6`WqMJfF7GEjwzkGzj__s0&$MCTu3q5KGCC^FO8 zAuDnLwEXQGHd?j0Bnr3V6$)1)K^Md0h^(=H(I{7Lj)I~ov zO^yMRA4T|WJ$n{}?bS=&&wsd?_Q*xetja$YU65nMtJ<<5r&pEf?u?=;jm$@a4Bkhc z#UM()<0Q$&D{uEjX!_9=h80J@XSmV6BZY^KCRt|m6t5eYL98bq17SZ8*tPEG{6W!Z z@@)3o3nl8qOrOTQoU8>^=xk1!-IWJ6G<}0q-4f_R)>Ol%irzSi#zx%KbuW{nv&2=| z?pFpzS?HxmjEKAN3?cAtjF-N2Nthx+f%hF6nUNcnC}I1rYvbmD_m!#b%?758MX`&p{W=)nQ`c*g*%l z+Ep>(vGOEMoq;2}Q@;BN4QkM-J6A7pWxUfv;N%}bseBy^1<#5~!)o%BpOnYPpvR+l z*OaassBaXW8v3*L9{ExD==Lqz;8|>O1+O0IQJ^DoIXQ!so%MX{(h6v8MPM|29~p5?iW-;J9j*sgx}Df=pYm-j1QoDm(yq^#tbhl#hp8iO$-elQ1Pgf4 zlCYkM)Ay>SB~^R;u>6e8P_vO?%koYMV^RS`>61V5BU~I3yPTFI1&NW;2{(u5h|0>? zrq5B=Q+C32!fV=nGrHn%@FYz>{pknvN5dsYuv|iMl^<(?)U=(N3W1%97kzwNhVA8Z zeyq`0FIKkX;{0SW3a)fXbp4K>`v2b4Y?nO^f%1pP?!#F z^;4{ZBaEFisSoB=(J|oiKsmkbkOmODfgDnMHUL*=#H*|VqHGwX4(u+G=3h?h#7#3} z^tuNYb2PfS%@EFn%(g1hC{Xa$QLI17f@{g54)89qa6k?*<}~sYTUCv3fALJu?`tRO z^;AalFxkht#kkT((2+|LR`((D;lk@g%Ty<1x=kB>6h)@57gbT}^hmRMvU%=Wg24UD zc1al`o;2en$QCBBQ*xw5DvB4M!4R=Y?zr@446BitD$U~?5(D_?fRE~j_bMhJj)i1;MH%Z2`1-38-JuQzP%3KeUG@`Ja z-T0@14O&3NI)9dj7x-4oZr>ubPQ!tnI}^D)<Xk(+Ij7gRT3HL+*MdjUjqh5%NcL z*TEt5vy6?i-+t`&Sa^``KmwC?8XmGBv8eA0?cRONO8cdB(-FPk`79Vd*s5>L9FQa^ ztD%sermg{FK6Yofdpr~3KDvg)@$SDoy}6;dp3gj>)c0|)`u6SbLr1mGCTUtdmrz;y z0ZspKAMVP0K7t6nnrMPdt?F-);Wn?$J@6kM8?kwev1$PGve4@bL=l62EqisDsC8UV z=31%(Xc@PZUzTbDJs>C*Niz;O&=aEbn^xuNx7(1N`dj{ufi5Fl%DV zZlf~!YxYp1wdDB}q*|(AzUAi~f2>$AhmGQ53^&a6Pk81Qy*wPCQF* zO=;Rdo$_bBz9=c&#WdpW*NSLpDgMZAELPGJRMG%hd7p*aG$6#uQ|DD%O3K*XFDNj7xjDSXe0guOl~{&?5!ij8vl!gUPoY3ucp3 zqz3MJ#Yzo~*K#}*-lozzM^KCq+9>or;L zud-g%{F`e2z_EywFPxjPM%_sDPxCC``t)3J(fSS!=|Axj%O`R#JU-@<;i2l=e76mH z&hO*Yd?c-JVAX1J>A*VwzE%M(@yzNPY5w2b)A!B0>_72nP~c|jh{($W;ZKg9j(XYJ zdDKv$|Fm}M?zQjRm=oio%Fs2;BpA_5ouvM5H_#cuRlin(qSm(HYBW|EW>MhA{+Q#F zc4TV;N$UL7$CW+ERaSTtK{eYHzuS=AcD(Xk{+DBAT-2>&*=9#64CA-EF{RL@_CYls zk&EJTMu}AO5ZJBjTeG}Es)9?kNK#+OdyDInr*51pTt!YEhsx+o+8cIL&DM4s0vV#s zi~YN4B1whk&M|}xJZu&k8m!Ea;{=}YiV6P&C6`JUtk;K)^POe;j%}qc6Mn_v930D- zgI0Ihwy5aLT=*n7s7`TIvAB)=WC~>Er5#()&MYM~RGjgOZPHC&*F945d($TKP6gEzqN)&8LMlPy!Ds8yrzB*J>kWTw{Vgqf|5LoA_}>2! zVX8v)h0sxAHFL-FCxc4|BO3KJ6svCm7&;YF?fab;3&M!9qh@4oFMt64Rkdw9m34#YSeB5RJ;{*4mp^X|8355XdQx)*(JJ8$QFcE6o=5&B|?KEVszYbyRXlP z?V2uVKEhZ{Z_1hOMgd2+b8`lQie09?ak5;K&LYe2=1MNs;EjahUUURgx-oFu$T{8I zyoeT&w*~h$4#vk$k`y4{ZE9pKvoF}I1bU(zQ<~f79UY12k0Og&(}5rBeq|Tf#Y`i| zyhu}l4~P{bjr;|2UJ(Uv)7A!U^wU03OxhKz_HIA2!1u-_6_m17wN&H2flaxlXcU2k zdI-kl&k7R9;zk7(kWymva1261UP|x$$=I$<2J^@Lg%|6G%>ojcUr?~LYX)$l^14fN zd6nf=qi2sTD_lxLZ0d}J!Y%SXcqdO%fxw>p zX9f0I0OThy2Bb||Z>v)Cg9kvMh=4mEuC3xa*6OLZp8 zHBg51oa-~Eh`pZh1%vWtc4siSOTk^T{9m7o5%YP%wBw4Qgi}PRk7q1vj&66ZrzCD!V%d z)%G3EIMP=;5iq@yK;8kRs|Fj|2L>0{xSIql&gS<$Nds~14dWFk4ar>Do* z#U<~WoE-DFQ*=KL&YRrk*T%`R>@O!Kdiwj#+46j`P)yE*jpy&ba+rIF8TXK-ROK$T zF{D&gRjmbHWJb1V%>BmRqO*&OgsP;Tx?*Z*Xy`>9HjCNPuysJ{=%;_}=kESFsl$8R z@9!UbvX9fLY9fJ<;hP0G=i2(n1=x^3)YTt!$&!s&Lu2Eie={e?#$oW5xp|VL!+BLq zqyBAsxbw|FU0eO`02^R0euSkbC+1FP2yap-M}OtW_Ex>iJ)pNOKpf>%lbr*eNqV`A zI%rk?p*v3YajV?wEgv5Q^;Z>S54v#ikp7R~WkYs1qn_-4l^I!Z%_>gG|LuQ~nVjgW zH(F1B9TYiQebo7mAWLkoHRsnDJasza1YPg*v)hBWZe2^6`Il(kP@&{=?3YV+`jraX zd*?2fi`C07-SJ{IZe$lB^-xoGf#JfP2vc7w-(i8M#>TehCuD}Bp5-lVyuvvuwIDigDD|$`q|C6a> z17;2=2NTm2eI`D#rIk&&g1>Z_>A3sbGk&RM0%7!9wzkf%jubZh4LAkCq9goN(&_kO z1l?@rmZIgU46zHz&i^KXMLGa#w=@8^2&$Dla*iz`$;oA)BXpASvEQX14BF@Q?cw-y z*fj0&g)KL7awdDmPsQQpqkno}_F!ZVvS$jGKK2b;nvsza?%Vsj^zA|WG1{*Q*jJmW z7CK%MvN>vjC#-B}tP)oT*phRwEgtk=OaZ;Yn|g#>{YB>fLLUHG^xtOT3HKAXT^#8>=%=|V8cX{yRf6k!%-`cV+(L+P6WzWRr&)eXd!dVSOdS zRO1)FWQ>28&YA2wES=_G{92}NKTBp<6q9LlGNJEYE6&htbyaecY+( zmbaj7=PrKz1Xg4)n<6Pel>)2UBBN8XfZ4$dMrYZz_mN5$MPd6zO-?1tOq zd+_d}wOv*oCfL0(;>54SG{==T($PYBzZXfe84rcak0!_kEf|3xaHH08Fj`KNLUmS> zBBwH29bJ~c3;t1_b*L&}od;Ao+;Bdz?30#On|`=A%3_0JGxU#FbMAK+tGmfI*zk3y z1s)3XD~)!DP=w2)2TzV%YV_+&DkS90!DCl3&O_BT(uo#(qk$Yy>RXqai4JCvnKIWZzGKv>6MR}b8`kvYF z+wSa^b4^hIa{pS0&s$+-d1G=uOajm;bnd$=xsZcalPB z`4~#Sp28%B=!SHZhB##lpLL{q1&te8qMoCfJ z_n!LRE8s8ew*jL$=rO^eVIP)vn}n%_^~3AJ+#cUB{$?6XZ?l*_YPx@{Neqiw!RyKL zx@X0eyClX_WHpfo7H%>-erLXm*bnVoDxGX*T6zIeJ}VPdQ){waqDW}mx-Di}@w1c5 z`}4WGopUm)1QA8|RkG^nta^^Iv<(2=C~agb9-3($$oHv-2{95mQot=(y=fEaEdpDW zsEXkgDz-05L92d5Awy8`i;RNq24>RXlYFT6*yB?pa|vV=pwM}}*s7WNk~JjYRtSwS z4aG=dcj~e?-=aKUgwEXS6w@}dZ@k%PxF7g^X4S=YY~+x{_Ug^biFpf|*qs*hRqF3K zFCys$=Oa^sI!hbEP0M7VX@-FiU;Rc!pOF|oPE-+;<_fN|La{>X7L8;_f3O|~r$-NG zZ9$=W7}m;3p;tu(OGTBRl~SKo5nh@@9t{s`VgZ6yIjM;05;KtxOs}woJ6qpUw&EKV z7A9?Y6c4l~kG((Ip_UUEd%~lfaoWo6Lt!rXH>!Ve_~w);`{$`EN+!{Eej%|Bd)`=^ zDt92ut`B64!42iz84aH)lmTudtbB$^0OlXpl__wOTijS##=fPqLP+Jum)e;qqqvJv zcv!zFq)U3igD!i@b+T(Fs<9U3mjG?EDP?#k*Y3{67Xh4GBKP6yZHqgOknF+OjN*?N z;ju5W7R9AI=UD{cqxuxY~&R2#T`x<7GfHfDz&ZQsVHMX0;(kJTts$IJkQ9c?=xD7$( z8W-s1>Zr%exrLa*!H8^akL}9N>dgdMxVQb*CyA(Jsm&{o-}xajO1R)VFAk_R+R&cx@NB#JzczvflBH1E;VR3njGZ7?9^aq#}uMIQ07BAofK|KHY0Ir z{q3bhEcGa6tsvH5OiGuVdpdwv*Clu+u*HA=2MS5=uMSAZw1VKT1 z4S@s_iqsH#h$Mu3@wE56_dj#*%y(z*{W8oj@a#O#Ui(>lt+m(s?ccVh+t>i#a&)tC zL(N`VDO)d!Iy}71bohNTaOIUTX%LQ8Vbfo^T>rF|cWpp$zMexvTqOo~Pj%isZE!!3 z4ZiCtQ~E-n(MUkp+OzQlq(%1NQ4-0z1kkM5Q2F?DF-Mqie4u!%EMVmXO)k-u{k|vr zD9f}}Ujueh9OBtWLS8zq)pkXY)Uas583woKWn)|{|W-=MuG6$PcQ6Yp9KA~!t_KGn+T1; zuF#tX7ceXgBVi!LIzDcJwE-kBTU4o_KnMk&07m;v_C!g02|6;X0-mOfgDhei^ zoKA6kZdDT(&ztNQ$u|%MIJ;}#WSt1AkE#k@0~Zg7<9X4fN}y3=ksqsHh`&6NqP4m> zX4sZ}(q!OfM5af*wA`9;+3|teIr?)4OPhBy$!_-RbS9}cPwt5+`3w6!uPx#))d|a+ z^8yBU<6|JI2f;U(YZm%I)%FLgWe=ydZW!E=an;>AYL?fa1@IH<(26va+gaV4TbJlhHRcO;WleZOCTTw_KQMC~k@+ z%825#>dVV@g+0SWAUmY{DV1Z zQWw2+ZLuznfyu4HN0*)&UzAo~X^5my_Owp&cAUL5qJiJcP_!*8Ycr~M^cf5!gkSY0 zqBWY5Q$L_6DEB`<>qJUc=mRrw_~*0Qas@@p*`eFMnAO@yeFqk2M59I6cKIKYj#b4` zSLe|3Ob_EDgU#1YoT{f~CA4*gz*AKkL3Gn^>idf(9cE8IZtyJ=)pdtJXxQkzWB3l&>EOlM(r$%;zPBY{zUd##E2(pbEi=5U9D?yzwl zL7NsT)z9BhN7G&u0{t%S+{TfkH(1X56q&Oc1HL=4y74DEZg$yuQ8To4sMuSmD*R+zSzI5;W50s?tx8D4{aTonlz0s-rOg`nq zZkyqK>NMog8RIIO%>;hMsl`vS)%(G(%iD~|A(&K|&l?5QR9L6Uu#i2-2aqJByJjj} zV7}bu(HH@O?c7h%F8RpBv61Uwqg%e&YNL?hy*}d!tBS%<+?3%RyF#U0etJX#N4Ukk|$0a)^8Ze-?o>7eYQblT@ zyG%03ay1p1$xvI6o5mjhikYPDMWce$E*f!TU2-(dc4?!A-DEO8HJTj^;PYh;6emA}2Ep6^F>+zfo0A4dCo8%}q1aGLrYlB*}V=8a+Fj#3g zlu2p`+l^03&eLgUHF_^=rc^vUfLwR($J5YZ=IrYWX0ui?fJZuC(16kdGAjSP8jXY0 zJtbOaQxj)aJ9D<*f_ozw2KBAO*q^N&K*|j3W+hpIh^S$sDVb;G!5iYf@bgP*^Pt7WQRreUK226Y17hPgD>w{vG>*rO;r@66T-X4^S?x|W5vVSQ_vP%Qbf{m6b)f8(h= zoHj|#n{|AXGs}A@hpIie-Pj6{z9@PS#c4H=Ec-Q$j!l%vy6)JgnR8;%8JM-5nWwF^j@P-7Di;u1^Ha~q=pr(jtP_C}t9SFwF8#Q}pA~YJ- z4GuI}W?G-=3Ox5^_#bxZ5eZzMc&WEJWnZets(e(VM8@vJ2R&HGOy2(BJb18QH}4j6 z!x${@X77Ut7F#c~ef7oDJ>u|_9IHMk(OD<99u+gN2Og%F3vN|h?9EfU=d#}4*Ic@!`aB#sy7{HO`-BuPm7>9;{YCe9fT&mHLB+Ye7?KeQaCvo1$MhCnEep#{ z9UbgA%P*dGM_KTSky~(H9QuGf9zy=6FL9%4`Q0YjHhdH9SRvYb2?hVkKrD zs1Eu#u&<*-Nl4?e^@z3S1Dq$DLc0=P=2?KTyt15eBpUXO(@58D8P2JL)vL)FmV zc}4h{M#%H@@m!`J5(TcsDuXYHHkNyQG>fp@g5sjG1qa_6#XlB?0cm=#blR=ij4Jk( zA{cXIwe+b;ksgnt3p*ur(EGJKif!j+OjMq4O;?2$vbPyAHVp3Sw`a*t58nsMxehRY zo-0>L3G^=I*@*%k!^rAUaq7ts zVgtB-mtjp+-|b%bxPC2jwgQj57b=AjUF@21l z>}QXiH7m z;ehYg9QtbC$hu26=doo_dIu%^sPekoC-Gw*D8{|gSVG}gFP8nwf5Lts8zt;_e_n}= zRD2v>HTCYvxa&7z=O!elPmk7hnbx!!fuB>%@dL+l*=sa(qnEzh*k{BA4I6nh4^%(? z5pmFf&=4u#gI#J2%2*Hg5)fFhfqrdx^W7y|uv1dnAC4P0t0ZqEQQWL=*HMDDFgZam zg141rN2auFb90PL?;4E_l|Xtw#vVWDuhlPKMHkKrBd31KwYRB)XAsH03XMg@wI7>1 z3GcSE8lPc8&=pOwzn#@m_$W+z5Yf4#0m%oCW|k_7JYAnrWLcG?%LvywS)T5Uzi~ZSMIk6y6^Xv)62hoY4&P31mN&+OUK8yP4x3`qE0C_W#tx%5+NJ+? zWMlAJsplW3*j!)@n%myf!mnP)g7QxAO&2wm1=GuqxFl+}J@1@hIxE zHjC-RD?GD|-m~#`WVeX6y9l9R;StBn9j{e@m;BaNo@hhPqZGUZP5V-|?VPZ`?7=149=w20C=Ekspmy z`75)SYIWM`#%talfL|il3D?PA?B(Q;(Xj2egH7M@DK}S&kP>jCWA%&+_Ca#<2h1Ln zG^&}>Ds4wih~HOEslLgDtd>)P3M%*Q6U9QF8sbFrN`zvg*Qh-nFQnt9=#(|@-JsTNhHcZERs00& z2|flm{!+Z*tnd0)13b*3~j9nz#-MZ{WroM+8^;8~oea)KkLV6zS^C z?=c0s+rbPDOnQ`}dr~>v3{^C*h+K&`ZM7QI@t@R5X=fYw5tC8TQSMPl2iAcoILz9b zXv-+B$dY14T#XC#kD62z2E8!ng*uRFpO5gqk$_A2aSo@RX+MCs(u=QLdrF9NVLZ^1}N6IMc#5ioKN9t8R9Y(F)D2Sq!;WG z#p)!YZhcf6!BvX=9tl64e05O&JQ|Gl0hc^4A&IH^@f{|~uEjA!)|J0?Z2(tIiN6e(pZ?}Lp z1+Xnye5`W%88?cZsvVKz$rRD3ce#tD<#I3c?mJI7F1K+#rHjX6eEMY!{KMO(jxr)v zw4JUD$d(sL=cRRQj8&P+n&zs3l}*`LG)anzI%Kd9X**2bH%6>LsCY8Lq5!ubD^sQA z>mw#JQ$b9{rZQt4fW@YjJ6p+ChLt=~t-(7|JIb|h{mCVqaR1ZwcOt_%hc^03B;XO3wE$CEd9z! zr97+~yxb$&lETD0)#>0KnmCLvJ|LF&&-L+pA-C^-I2mNos&wSrC+zUo=mLnI@cP%O z@hYEb8KW2Xr60_c-A5wBCl>ULU35Do?;`MgWA_CbRC1gp_KPrR`1Gi50wo9*hu?mp zcJp7Rf{n77W}v}=gt*k0gCl+Q@y{!g?a+djSL&K#UrLj9sa?C?Mr%9oIO##wxM!%c zaEQQ2D(xdFwN^9^4-MQ5R(o>l+$ZM@%!2@ml9%IHg6U2#0Ro#rD1-7gm@Mp2Gh7S! z6mdHJM4_we-tD=4?aFdtaaP-`OdrtY^lll&7rq+0VWxF-ITgpyj0&lqX(|(mnGoMz zYoF@nZoQRnoUl#c~-soPb}+fq#MU|tG2-vQdZ-BzF8tb1l7KO-%Bjv z`(qr1$uNaZ-;YhK*~vktd=Vb5IA0b~?x?HZu6GS`_FAGutXn{?VHb0IDENXN%cIT<(OZebz3T5WQQ$gx`9($S8B>zBwR7se!8(Wa&xj` zUN5oSr`a{Dj@$NAt;wx6TMIW9MGPAVR>e{Ho|Gz&uDg^Qj)IfNuQmSnMNAl*y9Z zU>|Oi8PyfVw(@}ViWX7#z_g95A8+xc{0CYQdK#J=p(D|#7M<#>tx_B@dL%YQAY3I8 ztgRNp6k;^UdXnld{McIS4YPtfs@?HoRdI);$=Qi!)}C)UDrm-xkpAwB(evsZW)RE26{M zOI_rMJ&7KclRo$)klQcPpjw|=rEBV3`78YB>|!YHZ!HpRHVf7gV8%(kX8ruunAPLr zO;{CdADqk$U*)#S(y8ysISY1~tfui7^P8&7Bu~AB!KNMNCy2T-QzXUWEL``;oas&6 zFUwODzIoZ0J;wpr*zL=F#a7??i(AhpW@iU~{<%({@;x*S6zcSO!1evXZ$4X73ACXT8Jb!iNft*f%FavZ~ud+AjrI>*TU+Ar;cP%$XAZaR_Z4`7}>rbiLFS@SEc#_ z#>=4KQn0J6#eRH=E;3{$%5WxXRU zf%USP+2;+bAh3OiML|E>dtF1F*BW!1bdq<`2)gzTp&>~c(1qJRXmG?a4+jS{6Q{2;HR_fKDX7t8^CMxwRpqfQ=%}!yH0_&hmX@kV#?j4Sx=gf-UPZjx9&pn>2YcF z5j&A3+eUrkKCl{v&LAV4jSd%*5U8o5SOJm{(*LLdfp#1#iy9sgPGNR9!7>k(IRd0*D>%X`cHBXz^_SO%es=Ve;du&E`hP!|8=CurKI!4{PE--wodPa7vyi_wo3Sh%w3_#|Grf4 zGDhp=%Sb^%K?2PEW`p_or%eYN^@nosz+Q`OGhLA+yQN>7(WHIGIFyv$b$}I8k zO*xc)#RvSMN<%t6dZ~|Ef83y;SzZH(2)Gvoo$%eJo|{s>&&UA!UXOID8JjZJ{4Yz**R|Y9ajkhbGM!W=Q7LO%_cvT?M(wiF_><0fD*m!zS+OE)C zIhCJT@90y$>*Hy^9+SigSa@qY9f3lb?a2O1Hm%{C=?hoPc26*LXRqCmyQXQ~z~Q>e z82-=PqO^T=$13#gllQ++FYE)KL{2sS3zZ@8_!9Ra1@A=xpuBtY^vgf9oC0U^TGu#x z)#kEM?!Vp*1@n}_@vrVuKqdc&BBuT~=GVUl_WuIM_yepJe}>=jfkXG#4@P5S{&G7v z?FX`A1NhU$G;eZMXt?AP3<9hTONClsMa|FEMo73^zNHf$7v(XH2vLeXhMze&NsD?sXwbS?hKNyC;&RpM}69&pxurFj=GIEN9Ms+Tk*qIa|2)BnfR0p#h zzS$8vwr?W51^!I@*|8y2rxC@W{T5Jw{})YrpGqz%exe}%5Z(k)5!Xi6%a;3e+OEW1COrNle2!Fc7F?ibbo`36qRDt zHXBjCuAy1vPdI^}nlHhO0>z7GeGz>$34N2KW|mDIQh}(w|4q1RIcbZHUsko9BBUL_ zrSHrc?iXY7SDB51wQuEo>oE%U{1GKlbZ2*&sAF5`=vFi08bXX6yBMtzHc(~?h!KFQ zE8IXOWLYIv_s`k#NECbWrRWZX55_MkF$yS^Y0U~xwIrube{M)4wVw8t$)dJC2dG2IeF#f-?gZ1e<`d5tBD>rsY&b^d^-_0BLgoNPFuy+q ztHr7fG@@=+ba`jjo_;IPh)JWNLvbiI5));ar91XSZQx2p$Fqelqmm8{4wI&}z@&7F(unXM8vAui`c z#h#~VVDAXC0reCiO;bg70M2W;UhVBBI0acnmvwS_4$gZo;|8$u!FqhH3>@lpFhY1O zsJ@&jCRnI!%-&Qjhg{Eov;0GK302U9vn5;94ZL@NuUpl5WKzbcg`l}ikb3*mXRzt z{;p%SA66?gPU;~n zCwZgL{)DPc9aqa;m z^DX_Z^#veT2k6jz*S0f;rR;CA%ihc?(wriS5x?sZq!y8-onvT|TtaN<(}Q-}=j*b) zI`lnQfoaF`6d&>K11PYDDxx}8lK_Ex7$XW34ZUr0LU8wbHD9%R5a{;fqxyXXyS~dkom7(0xSMF;*ifU zM&9tEd2Af{`w4DV7ApnD1q16{^=fBE*iB?PM}h%2`fk?7SW{aCtH3A4`Yv5>vXzk$ z&DMXN5U3D`Ef5#Lz{N7+0%?%6vqjI_pl=ERw;Z;vH+?GFJyophEj<(Z zj^aKuKDp-VYSf8w>V=ivi8=84pf=h;7-yrV`IC>abC4 z#ov7G_pb{UK4Zqd-kB^8H0{0CXD7eG?2ihrHQY0ojY<^|V(Hi_zcK^BGV}X+$#sd) zIr^(*JCi6kM;xM9eZU^Cj2aHi37NHR5KZYitrYan%G*t-?`tHYDF zUkbt|Rp9nMGeOahg*)9-Z)2Ko-PA!_^7ehP=9$`i`1O3j^UIII`5k*$6p}@qmz@Qy za;^$#$#_p?H6(}{O1bLGTmUxt#I7)H5gpDUtLQ613?`j){7~+JFtB0p%SrW@4x<$U znh7P2DnIub1{3e*Z6;vOI;mnUl0_{G9HJrE?}9u21e^4_*DjV0Ov_GC1)R6T+B@n{ zWwV^eP?<-U`J&Fkn->J{NQ|ltwnbPS{@D2W!q;A_yULNCC1f_Msgv7y*%c|jYAZka z^kV>)p)WR`t+JN%jHekUXKdRtzX(}-`?@;+PT4(SY`m@H?89Q zN9|V}22}~%WjR)6BTLMZULt&txzXO4(ob^HY?!U{da(kT2Wnm3d_fTIv5xfOM{$h; zY|d;P^1sv=)g+1_FnNQyBc)>M_@mn%an_A4nJSxSbKR!f<7czom)9CZbP>D^2G=ez z_X#I;2nc7O$dh9H1b4Ylcm|3xbAj#4#=cR->+C@tRt&`HjDmn-S2RIW~^w=k}`GpLG(=D4oV{%{%N7J-+~`H zYbnbShShXq(k`C0LB_70CZ+^y%BMvrb(7-W#A=^Cb46}UzPpu`K|S5>{da!D64Cq? z_XUhWM9b~#Pj@$TNYUH)u%1fdo6GrbK$rJbyliXR@g2+-UGRJuN?kE!8jIdB8gwQb z?F>7hyMnUmA*)1)Z#>S@P4V!xHYm`z+A(3RvbCs3LI*O9!=pr>53$~2oz8P@m&GIG z@r;oZc||>CM#~7Y#Hi{W&0((VL%!Mc&9j<2OrQMV9-b`40=_XD4FFfax4Kft0mCNH z!=wjltGAw`ir4CUu_CaNBfnj9z)Ce$p+$-4S@|OJX)Dj;b_R(}{dCzqQ>>JEXw-Gg^ft8P{0%GAmM9<0GF2wBIfYiZKgT!HgUO{* zxu@%Ux(L;rwrOB-JhfO~%58;i>Ss%(u1}JeFIx}zS&ayg2c^%8q7_tjjN*xSOw5a( zB|F#f!CYT)sva3jBgH3GM}nDx^uWMnV+B+^K9ax`?6xx!73yQ(!2xrrbvyr~hPMBp znO8T%>U*HV-My(ag+YLxih7|QBj%$mTH~Vn25UN)9?@@ysK1PRya!LcyfVJOm;M8c zM*p0cZg6^OM;HFwskp!{`aB|m5iW)#BJSKwdcn4x)j4(8!j3W1H7mK+-rbe@2#;wnqBL=V+$SH_vT9t zxQXkb!2pj92>0{`C9Ft?u{f(zs3tDMa!4EFYNWKx-sV({q*SlJ|gWCk?jgw@wqcMVsj(j2vvQ_ zdKeitidybv%GKp*?=@QMJ7{J9mQ+FWvm;?n3&yw{GjvW@k9x0tqja;KS~QeKNHLw} z1fp5dN%b~(d@1s~hY*Uk(7kLSF`~=|3>NLWufBX|epT9*7kOBS6XQ3s`%cbQ)(wbt zjwQ!<0L*^M2GtC>f+OvS?6-Y06xZ4ZyG=o{hUpur3ben)Xp$lYmoh|L-|ji*$Tk_q zfkb}f5&3kElgTt*t}+by1;CX7{Y$_N#%=ajCC#lSuCFenyx2L4S(#>ZMWlCHFJB9| zN~YI!!tqymFgK~*jkB#phabQFs7&)jHEAQ@yYSbVO6A*MO@tJnh8}juYMN>r*<(k7 zWrFn1fj58O@#KL_#zl(r0%RE2&IA+;(UoJNzq0{gIlYdVi+)nt#Jg-fFJYjIYqnn4 z&$uJLAZgW;MoumNPhL`T$7}GHw3;h9x^m%rB}fI8e5dF}jdGf^*_~}{&aJfwk>1&0 zCZa^INAR$7C)0f{y2B*>2de3-#d+IiLU3|qbRKocJ~+o>GaJS^EG1fgjv$)nXiV*I z%gHj$=;Fdzo0#n(T@Ze#-@#!by3!g!DL(Ekj+YR`r*+X$73O>H-csw z9o7O)j4w2c^Lg4oI1J{m7k?G^*ZJ%r>6yRALFcggXsKZ z?wSgg6vq)m@dWPo?bsLR5&}RaUm1$dY@LjXxKhD7dUO_=pTBDsy5Ymf+w%?OByxE$ z=j->JmOb=-2^av!+GRKHWxxZN_EH^8g3*ZK{E^p^|Y z$rT=2N_VL+S6@J$)l^ro;~84uFV>1aPjwV&FBGN2cl6>j6V+LIMC=<_v%#Rw!k$4t z@+*se*y_b9^siOS3+Y!Mp%7K|yU~J`N!^HNG8WFtza#tqHj7U{%U3ECDTKqJq7wYm zv37_hgv3q^jiab8>D#18FFcqk^?-eV=p(WDobMbwHP+`!;;yq5sh~=C6Wl`(#)fUnHFC@9xo3sa7P_|^cwVeY;GSQ1+Akp8(em)!Z zI-

)khK+sO!FX`g#O=OyF>ZT)qk`-&iyr=hBEH02quIdWb` zFeY~*Hbw5(f|e<+GZ3e$lfmD=u<;2hRB}FonzduLv`R;LESre+t$&Qhy$Ol1nAu6e zLh?k0H+TY7ZWuE2|IW{O248ArhbtC8GZku`eRFt-0LQkTda!vLH%n|C_Tb~^Z9Cf; zOhXT?{9Mc;L>yMDPUwpfbzNf%%Hf13LWpg7iT-2noG5FiN&4haMa9^8V25Zqk{ zg1bv_2m}ch+%0%;*TI5ou;38f8FY|gnAhxm_P*zw_v+nxbzj}OU0pR()!oz6|N7Tj z-}?U5Ye`{;V9-kyjL6w=SyO zG~kTyzWoB_v%ZLSXL{23W9m+`ZXR-GBX1?dY zi4g_WTV5Y&9FItBA9il3)FQSJK4W(2kcqWABIb)0jCmnfctOm8Xz`pzxEvERX9ru+ zGD&Qy`yfK22x=m)dQ;GxCpjNE9t+t;E+WYob^QXB=aIJrUwfBtUK}^#S?9#rb&izu z9OpZ=G%E83=|oVrp4X}(yP#%wj(Hj_X7 zPC)LiTkav^vq9q?)ay%_6+unUPKeJFRBa2fgdD##E8DE4J4b`UEEW;%Qj&~Gk90bn zP~U&80lkzY&a)7Yvht)x^HOdb_Q|pv8fVMt z5b-KUOfD<-=p6IS(i)`llVYB^Vn>xy3icFAdY$JI% zDQnDy!@y1#aq%9N&BN7)>Lbv?-)gr3`WSWhXEe<9&DK(UXgV@ZuVB`AhgM)?!@k0* zWOJ*Y26PN@WWhIZe`NhP{BUflt0cMK)xKQ))Zc|aP|k*KbzpVmdb0{+5Fm8JrFIxY zH57Xw&TCE&OYB|XxvG7-ygk1pa(d4*vD9gOVWdX!_8nLPIV8R2pPuz^{(59e6Zvyp zXN8$+$m&vWQEvRDb_Gll0uH5q&-^uJ)xzDue5J=zI)rZnEPUGYhnZ3~Oo?9S1Hgu= z_5RcX2~|qN8qEZ?toZ|4-{2nK;stT-=GI@{>PAalWPlR!^CTNxrwL*j^?F>u7_X0z z@`MnKI9OU$fN3W|J!rA?WZ?b(1oU?`;wQhy_d}dR|FDUe~XcLHE`x3CM zsFe1Y@Xr2Vm2l-bBDpzjyzs5-<*fUJW<8I6FkQ>S`<0Mh;cRY}!MLUD4s1l9ZsDa- z3kyr>TWMt`9w&j>-$(D;8FZ}XAA55e89M)@ryZdsk7IHW8D<>3tj5b@NF32=Hp=AT z-PXtcG>r|y?r#tZE-wcM3+ltEc>5X!kBBDR`R5dRyXt*ei-ye(Ew;NOzT2dt8@x?A zaHo6x!u@%{0(a2tGE?b3mjlM&hFe^{{-bS7x9$Cc?jDVBtEMkdKB(HKFda6teJim2 zc~(8@sNM6MI?0%KHkWRVZrbyMhnazcubL52pLN?6JClY1P$O(OIMqvs5lCp2ly|=9 zlv)GbV8NgVvrUklDs?`KV=EtB80joIT9NBiWbmL&o*r!fcAvg39U74;I8n;@o}oB- zxzSLH`PPm1kjI?Ghq2{-V0}`5MCfd1Y{Hanl-Fqo>l$30=0-@mO(^b(f=tM}FwUje zH=A)UqyQsXG6GXZXJb|*F5GtI@fjCcFoq<8q8qLy78bq$DC2-T=1K_SuzNEv*8uI7 zS3HgNJ6QxrY%y{DGIF7`KpB&lDNojWzV7C8u=Xvl=Rg)gqw=KXc0*C!I(URd1mhMD z93fyP|C&dd)SYnbhr*V7+IqRoL1ew5a6x2s=5DpdUa{+K=jfP2-Px1VP<<=nOpV^z z--!8U;Sw{MJ978WT!1AwrgUg(W%9DC?2TG0b>Q@f_}Y(xDamW?VlzBUeTClH`WmSI zcM9WkY3u$fDjEwERT|jG0N)jTgNtDQ1M-tv))T$VwOC|bG2EKB>}iM{ItHcBvoDFj zjdau0WI+Q_y#D@s`n!00fci4n4eE{^XCheKwMvzQjO^d*6MJ|K_Zrsa$Z^v;p3SWh#=4^b`1Dj_%g4$)4UmI zH$)I0F5-nEVvUl}FzRDeF0{CW_^^YOuXb?Ck_mfxjIJrqX6D{aw2JNbIh<9%?NC<< z8Y+zqdQgMnTY$WO5PC6dm%*Y*f%PY~y<=bBkF}o-327-sooT{Di5=gCY0d#>9WW&)}wF#f66R{{mn8o?vVYLEVL)ki-S&EGf5@BXCFnY7MX_o@~2!GfdM2EgLE;X_b~IW;q=&u$SHu{%NT-0HvU<9|K?6c>Lc zf6vdv0GO?3v$!MwAmIHz|A74qP5unH_CGd1RluA5)294({GThdNCGk6KOKGXU%U8U zr0Dq1D+1WITO;X=yNEaK658a+&yh*7psd}V+Y`c7Y(Rc@yCHTZI_UV1z2&lMyh89l z-eUuDSpL8M&Hvs3sxTbdJK0MpnP#=}dDb$&Gm*e#B{^}3Qy*-sl@Mghx*FEttEHEa zS2pI}>hrfOOn}MqHREcGrS^QPkf&b1w!_o;>YRJHHEH5DV&lz8O;AaR_=2zP?qnlR zN^Q@_{aYN!07P{#7-WEs-7TshBfGHSG5L!pJ0W&uFVbnaVdJ*>ZMs?0*X0Fgo(kTRFn1a8c%#8OG#1KMTzzl)Es=Ox>A2?rlHCMf| z1~wSFCvwa!1M@mQ4mxre^QqrQe+<9oQmPQv6Gf7>pCr7Z4WtQuJXSf5(c=JOqNHZd z{kXET7t24oZr4}WqIruh7d%40rE8t9pkrBWnl64y!qqlbti@Q&f`Nuog5i%M(TtK= zE4C{7+k;ant9lKuX(-CI$VNZ$>F(LgG=Rr5`s0NR9FbMm*Ors1dnZCsz<5^uvbI2b zZ5nK=LYKMs%{R)FYfuZMh{EAty7FiX4Ja~YwcTf(6=7L##yp%78zkTPi6{CavZ@Kj zW{R3+F!;WjgCXU&@kRVQ@7h(K`sw_M)pRTc3OWOaq{=u( zr0oMSuV}@Llw*k4yyw8yt0Sbdqf@kdGBI2zOgPqf(dyGz2p_qYqi<})9Ho?l(oqfjh$=RrY@f`wD$MZx7{BkJwJ$< zVkIBgn1rKd(u0^-VK8-X{#@NM4d%2*AT^a@M56)elHEd!6y+=h2ZP;P0cbv+u~E`?umE(viDm=J;~(kPI*maY?cYapF z9AMO9OTDZFW8%$ET7pl8m(ru$S+AC@&XTf|Fd$Cyr)h(IMnOUam!~?HTx!>X{TX&j zc4kIb=|!n8m0w`l9hbW2%kb3&XoiT63d6Yjb$d`dy@F(^|zAh0l`E8vvOU2!~Uu{PWZRedSJ5I z3#Gm}{tf$Dy*loD7}{F5D?8^aSd1mbwTIYj)30K?1S7Vp@n56%u9I-t9CzPL% z2Npl4&fJEV*yqaiM=eEhzH1TkTkA!TJe70C0C6NdeX}}A9QGBLfr3r}&)=VUMp2pMi4yb3b#Y)auK3>56^-5Rsy9 z9c_DFJeovQLznzb!uMher_XmDhnJGW~~(4Z>%0uE7Hh35=VNMG9qZ)hf0VuX@%(;TpA1 z8X2WzOHLSrQ@@1P(t4@Oi>OYUZamaqu0Ta34`le#1vraKs!t#X+BqZ1I(#3~>CuZV zyVSE^t#HguKd+`jFKQAji?;8b4I@TxHAAcbqo!k`o4J739nI0>wg>3=Ge+P0AdsRw zsV0u)jZ*YuVlzh&=z#;{aQrx49b@BDqw=pT9+NpZUiaI-e`TVLBYC+g(Hs|1ybl7= zyZfnBT&Azj5*KA#_&ZK>A8H)t8k1YZPHio~M9V$&lN4QP^P~5T=Nyz!)xX-imT>&$ z-n*SoHpfjAIp>Xyb-0+ewT&K`XGFDx^be3sCE+KIe9uj3Uly2!aBQ?tdVanLp|(&z z{m#s6nm#(f553LlD6=+bS=s^guQeuu19plKgb^Vj30^b}0;I@~G%fkkt93eT*%uLW zt6PZA;0;H;5w{;pBi}ZR?FDk8UwZqW@_WxI|4LEYN-i6(i?CPUPza=}h!)yv7IHL} zjA#tZj8L&{Tv623`tf+!hnZfVtltv=U?L{oU+h<3gW^VJH+h~KRV7TO=CEQL5Z8pu z>kMRQI4hBYazk=I$D6PDY&gOrNH<-sT$mMIfGP1=Pt-a>1GUycFFVIbI`=xN=67t! zX^x)do)Zt)T*>Y*%@S;<;>(8zjF=`yYKgC2hPJQl7pRzAh6NRTHRjXd591pvUT?jBYwY+rlHpdfsIk&C6i0-!M)mTvg)c@bl+$v29Xss+m{IlSNL^ zVi?Ocs)zy3Zi@0gD~8b)mNqLo(KBc@mDCG-38o0%fJQF&Cye{b3AVL31=hA15Lk?l zsck1}dLGS)+!ts51_YdGIOw}=mxYwQEM+u=4AA_TI-kWrbk=CTTuBqUIZDtI?DK#& zxOsrHkbc{8Dd*OWT!z>|kVISYH#Nf9FbYj~hg<{M`;3(0AF;5Rh)^py5v=kZhSjW8 zurFKPySeklCO=I2wxA>973k_Y5fPTroQ_US>+k}*NXb-^^DfZ@6@-U6{sX*6{KVbQ zcy3dHy6OS6*VoQht==jJn*=+*8iX1%k)!*9zxPA0MaU3)4YZ#mdZ27uEp!^dajBOi zZRAKVDX>c8bk-)10l*%Ki--UW!> z7AmZRW0m2q#GIn&i!>3ywOc41k;C6z=Am{fbiA(FfcQe~hK$nQ{`{G<7n*<6nELBy z^v~ku2A%ZO+~}8j;FCaVxhPs+IqX~J*{LFoh*zE-{El)KyfyQU`6WuHVazGpRvh5O z2`uF>uPIL8H;C%p^5mH*-5wX*I2jA~+e`h;%e2wsF^YN)sK-3S}5Ty%E zl=_^zd((P8>{tk5V~?b?xOP^5xp|7Bi4xNt2taQw^`QonY$69Fe#EkNpo1Q$5H$ZT z*J0;MDTm~`=e!@WY3xbgbU12RCnBCsy_0Qvpq|Jym@a9hxRtCsbUKsl%1pQO39XnN zS8yV4;hIuQA{?@*zpZwdcZ5iYX8OvSp{i2~%d{Tri!LcSg)uYJjTWicL?1k4@VxP< zO)pbA%+uy-bCCCd$Y~Aw)_26fYUeWd;!8?=hw|mq1tyAGy%Et)^YFa961PL`$vtbC z$J>D|(JI3hKuwU;poCbygNs8}SPH?+yW!n|Df7M)prze(g+rU8nI+IfD`sN}=y<*5 zNaOqoaFoMe;Lh|K^~Tl<9Jn8gc^!!o)ED;A==~877yKIe|$iFc(0Ag=O!ZO<;Jq#a&AKNW3&AC2ukt1mrUE2#bC3+qGyv&m>|7V0B? zklzfd-{sv7l3E)aNf!Y6_85AGsQ!VljrWT(x|ZA03de;^07rdzpC%EoN`GxdR2G!X zcwBowDc{2wS#tr1`%L<5MvdaSJHza+bAX~2&VudqlA|eRO>v&4W0`8hcI%TvJD6Wn zEO4}DXAl!Z5EG?##NaLf*Ft9md~=bdZM(?srTR;n%Y$*`@deEsYx!?a6PdqnoFEVb zBn5_*sKcH34z|SNpnxOTNRYVJ0szF_TSU-j^xhJG-G7i|{@q{SfzSWe5dYs|XRod2 z%RX=g`FAEP;Fdn1B=6rFQ$xCdTUp`7Dd;)9{zH(=baxnG=>H+n@8~Yk?|%$~@o$g! z|ID4Ve_<H-s|y&Ztn1us=-o5jxCwMx%c z;X*6Sy~{_S5`%2bZHtJ|X5A>)^998{k;^8apMh-?a_O59p@)ee&_6I1&C-%*#P#=F z?EY2p39%e*;m-crpW*%@A1uQob8QIi)s^(qrX~AQ3`Lq1Zp&7f^<{%#crxhKI2y^& zcDq1#FKK7*d-!4%q1e7q_8vNXg63=#g~<+3;YuMj-SCjecv}-P%=Rau(GY%IIG=>K%y7u0iN;Z_MCJ{VYVAQ{FR@NiP(NDdef>Ose9X8^if){q`yvKz8f z+Y2`RBk0!~u$(?D`7`Ke=6C&2a)uJ?$>(xm{d3+3vb4pQL7f2t%j{T@j% zyp3yyBBGV%Uo9BW1k3lPNxB`OBh(ipo(*IZ7vt5R=P+hM&X*5i0kH&)hx6I~fjK|TdHf^}^GqQrE?Ge`Ru0)We$ zmD^z}Vt>m6ET{jq1k1p}z%!(#X$0vaI(@Ovfw7YB%n&=4nMpEHi!D)G8`Hb!rP><- z6xZ)~TWhIl)^f7pvP)z>b)K;dH<>~)Q2VI(YbJb#k^ zx0@}6*I`2>P_Cf(IqCcdSzKN$Y!i7VLF*zO&OSr^deKpp_JzEX6nV+s-kviUg^AP*v{iNZ0UiY^kk6{(~NlSS{Ua|G^ze(oPYq+0r6QZ5k;Nz%?v@0 zEJ$?|EN`|H3ROw9N!$K|25=jBgElj8A!dbpsW?&BOr&n>lJe79lAa`;h*Qol)1hQQY^~iM;J#rBf`pWQg`c% z;_rxn6f#8>+BHOC#6j0wzw(kpfJ(GxJQsA-*qPxeM?MCJ<-kL82Ims*g*3-t>^DA@ zA+GePzXjuq>$VmVAX@CpRg}Mt}d*bxQ{NNkz{)-D>_df5&u{d4xpeCA~ zx}0Dh-{Jt$tFhk*tx@j^J=4{@G#JJRWy1%(xAxwTzA++su0Q1h^g223;2sHr4ko=w z1DeW4*WK2z&152$*5)j!PZYd4nQWHwdqmpck8Ng{^pF$02Q&uVl6T1MeK@E>^lQ!6 zPmNLKyB_OQ%I~8)1nH6~?VHp4ewVX=DhwCiWa~Y*nW(K~({obK(1-5eny$&?7WU8j zE(spasc?uBApKHYGk>n38Qmh#N=a;^WGIM!p_s+PYb!7I<7EzrxlA&qhrnBQtYlNa z+;C|X-XxJJ)<xIOFmt>JsxCk*yCF&!z}@4DB%e2E6i;Q^n!uc0gMUothQT>>sTc4Rs7D+Ydpe zO!ObctPS(e`iyj_hs`&^{;kj-q-PC(a|Jbte(^7rWbYRdo9#n35n8%CPu`c}cSIiJT2YuaT=7uENvtQ(76YySB=XR~D);)#!v)0D=fazrA zitpmq&%Qhn%2}i{RL^~(7OTNUcF60=meYQ*FV;|!J+I#Q{n#^`*RkW$7BB+!npf{Y zjGq?eiGQJRNcA`3qKdfi*%;W2)zO88_wmM}ffPe@EXv}1D)jpEh&(?#kL6IzcLsw> z9sogN5u4$mvO4A}g8)cGn&S6UCLcQYO(%;z1;STurLB6?7`gedt*Obig*r1uJ7{Az zT4pb2OuDQ9i*);gCnJ$A7z>9DNLSjFA2f=BBkRTs2p5G~Y3K|i>TXN_0M49&ye@xn z399MS#K^hcA1?73skP6plLW^gNZZ5P> z;tgDBZ;z!v%Efot)Ejor+x>bb4E4ltu_>9pvYnaNs=>Wi&p~1f#Z8+o>Ila*Hg(KP=Pf=d5B>*huppF`8BTL!o#r^G6V)BhMfPiGnR+AarF#@w#p)aDqKi*)i#EFD zJkO`*d+WAE@Cqms69`|Z@NBCD+wfjUS6H6j^Up0EtvTncR@*| z84g*9;icd3h>Di9NJ)>~4f7Q~q<(|j&K|{Pq`uu`Pt{TT1SGwqLv7dA5V{2VDx1#F zLN=AYNawsc&iX(eG#5tnC6Z)jUcpPaW9AdL{k8uyRxMe{r!QO3-@ynnshL2U(vsSm z?+I&vL1a--?4QZJdFAYz9T>#Wkf`D|8lm{ZU7q8aaD{NlPwyf&fLs88jKmXSFBRo; z#)+q%EmY-2Z=43wYw;;PT+IM-VS1i}#1+GdXI-kKLPo1;iaV{U(H%A?!~b)Smg%RK z_Ub~eWI@1uwLng?;#350DOMysbyea^KehH!d5L6Y&@*v-o(~-|#<=v#v!Sl{{klXE zza=Q?!}j0fm9+3Gzore0fAq2?5S`#5ie}0!VSz#A zEa79%?o&gT@q?+V74~(fY7S0o|EWTsRBaj@Urx0XgvxUqnbi6s2g<>s+k?n5#y29{ z#!odCF6-|+sUCDk{V>B@Wl^Pce7KmTE%Bc?hRg3|m{H-&MkDSDWlaUZ&+U!I9*-uj zbH#h&R4tJKhW@87VWed-Kr&bv_3T!sisxaS&aL27%s?*yzju_Ste(#YPpHQYkFKb& z<{wM&=*5`&o-`_CubWe6!}7=q^#sv+VwZzzfPQCzY-wklC_oR`{&%)P<)7Gwv>v4M zySeJ&WTqPUTF%jxsY)lLq~0~t9&wD z^Lsfa^ZArI&R+X()%BFB^BLAOtAT(jSluq*`+D{q`QKVjwceAu?@BeKbnrpsqeUv& zeV>ao6h79*KH?)hT9diuD9Mt>L=~7l%OozlPi*Bfmqc0Q3J7$s?5-{Vh;cejK6yID zlh0&6_@xdWib&ByS3sxDcv5TB#7GpW1PW-rp$W6#(?Rkg%=oth&*n#fhtL;xE7B3< z8UBNa7{=d)UCY9VKFtz*2{)2VA$e2y;q(er0_r#HyaRRYY-cSO1XD??MeJC@GwX|I;^bH!w#6aSO9s<#UptPZ)_o@Lc8-WHX?tL5yeYOo)XRM#884dY!vk{ zewQ~~S@}a9&Sh>O^@bG@Pi_C!2)z&c0OmVyt(r4A^A33j`T|RQcaXUKe`6$MGZ{XF z6XAxRUlRqOR6PBc0JjhbX%mhJf!XN;*6`Jy`)mNX&F0&hQhuEAeqda~`d=7ycN2m% z!N(?+uXE6#U;mAbDCTOd5ENseh;unB{DX(k&|kRYApqM)p1;oickm2)p$06SZhVmG zjiRs8cc%m(_IJf^y=5mqU&7w}W8weqMb^|9RfQzXkFg|6~68 kf0c0fOLqS`4b+XqB%%0zog3Hs>h2<`_wr(8q6VM-7ZO_h?f?J) diff --git a/profiler/advisor/img/cluster_3.png b/profiler/advisor/img/cluster_3.png new file mode 100644 index 0000000000000000000000000000000000000000..108e9a16b8dee738574147da67de96f818496f47 GIT binary patch literal 11062 zcmd^ldpMih)-S88wRW|&)?Ug|RJ+>h#jSMYgb)c zsz{JUE7~H8xI`qVkCcd%ln`+ZNr}WI2tu6b_w4=WIcJ~ed-gu(e19Zk&iTwa-uaF> z<{ZEI8}GBr?iY6N+_zInNolw9#dBAcl)e`!%5y*LQ2YbA&o3&3GUDom-;@YlI#Y_u z_d$+sj!H@uX{zhDw=3#@3A^ZpP*U2{vh`D@g}x0?QaV9*KIeEn%2zbRNS)}9)*q!7 z*WhlNdSM$HE1-8yhsLc=X0c4-*RJtA95#0;95&IzJDX7TrwmPJqW$oFLd(QuxB)HYsq(jgypgQoS1OH{@y~xVX$Az-v<1 ztb05=cB6z9DxZ@~;i#Lvi&k6njrxuZ5Xr@n4!F1h4pFX93Hza~$LtaZ{k{(1?+@=sV(%ONeS*A2KllH}IDYmU# zz=PNy$d)WZN)1FJDuVP`Oz&v)0<5}`#ksdL6sk6F+lqf<5dv(t`bQm3a3d2|ksD>C z8yp~AJ_YSL=aL3#yJ;Ong=96FeCrdS=;Mcr!^uR|_?$j~h~tls$J8%i?{5qFd9IPUcwZ-vT{)FQ{Y^hAU^;FT| zf<+#Qze-jZU|Ks$9-y(MKxOYrC}%RwSv!uW527lwH`}4Vv&4q7*@a<%)U@I!k8DaY41Cd?6dAhUR-aE-WoXfpvGcrzbHK* zv%W3RgUA25Q2&0p%8!@x9P;J!dh#u^C|UgxDrb8*|DTlt$d)rGsl5I#l4gTM znS6G0|CZ9I?MZHAF7kmleET2zo)7MR6K{(kDl_K^O5GDXe{}mMs?9|YhCb~*fB`BO zwr`1oS9skI+NAH!Wf+_>|0cfl5Vs)@{auAZQ-U`J*#!HC3WcW2F04KMDSub7rRknx zkCI>fU4=r^Z$@r5w*ddL$uNFGOctljy3Y=yZ8u&H)QW{uwIlh=&E=;ozbctZ+Y0*nz$jZuJqtAdi_-VE|1VgJV@H)_2J`S z5#;E$gP7KclJ-g;jrM{F@2nJ!Uu=(&;7KS8QJgK>Ht0*FL#%GdNEzhlAyW4;5Q1A} zN4qab%K8`<9on^itgc6UhZ_V?PRN06W5$LA6By`lt>D65L0SJJz2(E6d)`7Z6nFfm zi?CX2Pv7~he9Y37yH%H8uW4*q-CPuK;9ON5wXOofo?z+1=X`k|^V&1F#W3)B$%mU- z%taVs9BNI(?xgx~pft$hhWy1|LYW ztgrV|)9aEMYZG#w8{a4w=xe>y0fcY@2^LUTK`x2E6!(OtT>IV!Xd?J{QY9uMVI0^N z?%=eaZP;EI{9J{%@7Qs05d5_+S-Z*x9IzOZF2kyKisaow0}{-HBwxv0M!!VaV%Qy- zP`C6iRwA8_AcjbQl!3*OQE3SFFO8|B`Bx*j@H_xTe6w*g#$%|WO7KRCV2~zd)3d0> zeSwGf)=XRy=Y-~G2&<&&M@^U>3x+}30YaKNdw<}$$g;)SiF%!1>>87pD6sLB!=(_w z?spQviZf9AWtYzOS^I|EGy=ymMsN z9F=Tw8-s=HnMBMQ+BKp=``R*IoO>3`DPo4mNqWq&bbk(%z?gT3#~)#$koFZPW)x8} zv}aMdzT(;YzOmPCmSGLK_T1v(rs7Q4-5C2CK2qAbt?$W9uK0G&U%qBZ z5_D`6Kj1-Gs%T%}a;=_vksCRScix(~crYU~PDx)c_AH`&dtEyo*uJ-^NXCjeY*QR$ z`lmJWEQ`Q=*{Y-6=K#^zU6aRTPN#;nT}oOR7wT(aQW4$BMK^dA4T?2jTbc;~v^{*_ z28py@*WXXXYKMvx2%>7z6zLGh&rg@W2T+T}4Z(I<$)s$ANwm?SXK*PVw3%Ck5fCP_ zAVnNV!Md3d{XnKNw%+%?(axRH;qy|ze+GyvpI<0sp49e&++G%wko2Wgj zJq*2Yo~jiY$}x%?&eB1=Lhq+zk(2=e#rgsZRzfxx8S58hqM9%v1|vlA?QHv7EVaLm z_TRxESJhH$?)vzt*A6VgxgYw44!1fGf!;cNRZbjA{&@G>S|iU_8sT(V(lmxtXjK+j zk2va;q0wZvecn81L25@55G(4z5oO@`S7O=43cHoe%u;Q0iTYqx6UmK z!AnhtamkrjkJ3b$-j!ILr;NkvRgQypO6>qKT)c!G$pge4SP&@FG+0Ee!Dr*ao?oHj zv!cnm@X}>rg4&-iTn}=B`oer7O(uEY72QJwUXRTeh~_r$L=8B+_k>%xTKd1*IS8>& zNgC+<)xvWY$js|=XVu?@bv8#}*_I7y^6Vv{eDQuPh5P7FQu*S>gU6^cV0#x&To-I) zo;_HbYeQ+Y-?md+?y!8?cWr54T$Uj}xjf0?&3LaBi01M&D$NldQ>cu-XCl^E1-N2G(d~a_xKHH_=gALCe)jwf(sUnotXu z`psa|$SOoV{cq+r7=Og-%K)#M=XrE8anC*-z`mo_@!)yud-@YybYFn3O}qy@?SjOc zlpc~8AqY&<1@c}YOr8jdmPa|O+C4<_&6xOl*BSKseQ>LeISnTxPCjp0l+OO{^UIJR8^MHq5SZu{__2x6?ZsZ@ND^s<& zx01Rq2FdLgVbY5vQ|TbGwvLt^ny>}GoKF7e#xnji!34s(jnB_6V2^*;wzTtS({ydS z8w6tf^uFH5?VDOQMO)y_7Wfl)l1Xo5oSb;9l6r=<=MOwYMSubRn9B;`t#&1e?@({YLDyIR7wjwY{l@J(m4WTiOaVaykId7^dD6clvOcGN*;GJC& zAqRsr5R3Q!rik;8sxwu)+s|H|yQR!5h`Dd~L%*f}&ZIM=Qz48@e7|LhCuF>0osm0M=+4>FUExV{7W~o^4RPiP*9KIO^UtFARl-<59|5(yf-+1 z7ZyYUa~BbfbYggEI_V~SHo@AmC)cd&C8+_(=vMh_KM~saj)A4lYk#&ske~-^YDrxE zaIXq1KkCrrbG5e4-EeP~o0~+wXuqob@uttQ#(2NJuO~`S_2a0U>TLD8pb|Dcb&#U( zMc2OWSx!U*)9dvIud4!Q+AKt-(bw-p%&gb3q#qX6!VV~6T?7rV)Nz-v%YiBAyJ9wY zRC?J;(?v|F#OHKyWqnfds8}MX&~8-5Am=QD8JV#a1L&KjdkdPdVTu#WrGQLa37O8O zQ$PM%Q7Ud%Z*xOeQ+@zZ^6|RK+;oFhAtq<(YK8?=b^N2@v3(`&WtpQLj|V~0-fV7vUybLuB499MUz&(&oR=i+%zQQlJ<&l`f&YOYx_WSzZCVJ) zbhCgJaDM*jFLRy0VvvZWnX%;o@4%aqQlrZziO@HH;&e29O&Go7ZQE=M>=cpNHOK36 zabo4Ikon;gq>*~tV^Kzk^4Ih-IWYP^%=$3~KKZgNTT&UMwuZ8ppO{+m4g%MYaKD{u z|J*Z_Q$6xo5OeKe!&`jb`0lSmTp~2F&9(1vYe4sRD1wrxy!%)?!FaQ$;yf=Lp0LUn zR(!O?gFb>g-4xBqe$4VxvT3FH1-(xtmLiS{ zFIWOli8_Zto*@ouvIV(b(6&5~YH#!XjF2kT&jp00T%g%Ne9c-5lR642eceZh?AA5Y zDb^-WsC<|U77r2Wlg>QO;y%GeGT8gmt3t!Cnjh#|y@K(>n?3U~R2~H!Pf3v8u{5J; z0VB~?udr(dK_rs3h}>C@^X8E<>$|GEo-$7%PN8Ps73yebx6?_Y66=^-tzMd>l}79@U@ zAEmh1ucyJ1kGMp15%0oOdnM5ZNmI`X&yT@h*kj)nxm}DhjSS%!b(J6q?(ECkRYirX z^)-Kn7L^yQp3eBGq&z|tgMhXLpf;AqtQ3ye5=7mYb)lRJ#b|B!GW{}f>70go7yFTp zP#Y;JAA6!>_cjlU1cmtow_u4Do}4g`$Flp|H2^`%#v z_~B&|>zwQ|x;J9h`O3rH9$)j_$diT;_N5HNi>8IgRJ&-d<=uUkP27l*Q#mu)0l!s- z;p|{{%D^tO#TL*0{*#)P)=am2~1L;)c1FmIU83VLEM zgc!^>0G$FHgJGN4D=McIR~(m$(>Gq)<=}^klyup4ETE6igjC-@qE$Gz^b%NJP9r2l ztt~l$gQE<@Y@(3Qn|ZVrTzO2S(bdulbJQ?r?+fe;&rM?L;d3lN>XCvyG?(&dc}(mE zcZt6c8b+CksTf@g3-dt~1;g<5q^sRT*}_!I*O=8Ew+DcRBx$;i3)8??1?f*o#?fEX z=1;hh>Dt+c!pBIxlb1Y4mo7cp7iD%&VQ-Fq!0m&eK_dCC>WC(U!qO~?=FRt1=vUw1 zXFH=ap*?}aSZ@EE2Dc0Xs6BtF2#R-Iei`7{Ze@}*Fl^ou17+t13>>~zte~r+5wjlM z&?k>M`392+i4&RygHsYPL1^PwLnsJ-agfEMrE`rs8fjrTyzZGN|2Gj%|8!XYKceEi zgTGPVN;02d&ScHDFX{h2hV5VT!v7oGqtM-rTwXj9ClfEu)3{VU54{nJ4MlrJ{Iq-v zAR!uo%ERwPq#khlhdt5Kd0PAZ_>Hl_`BN#O9*rCI#LZT#k{*tLH&ra8c5Wo^O0$!# zwM>!66(+?e-vn@%;U zYxBIX*|EMaxw5N$jN>5fzU1AsD;WkyBmg^nwYft*$vmfpMe5IOZ1k{=qY;cxTt8Ph_)Qmqs*wlv%?Q09W_3n;bFHmK|m!M2b{!eC;MX0}sG`P2$RSi+ClSSE4alfaE= z8O3k!knGFEa0y&Jby5)pAiJMS;K4P%kcC?i>MO9Lybg-rU1Nb+Md!~Rfl?F`iP}&J z`vVVJE>O--#q0iPR0sipxLum~DzMpmZRlk|X|LTt>+;M>NCjqCNVmA;>(VMrIEt*r z4+RV}gY~+kkJ>!N7=$%z|6mFLfQ;Oz`RT*_a5;SU6H^un=SyFEaRC9Pq{s8=f`OAq zEB(vBj3FNe*7+^eO(XB~8bwk+r*@CNb?cuYPw_p4w!!q=R?AP95^OA4@p zLsPZ}j7Lemwil-}c5j*Ihfl9xNA==|mKYlKbNY8V%H1$OAWrz4`w{uZBsVCLS`<6I z?54_HzbT@+Hrt_6stW-FegTt9A2Y}@oCAZ=LZ6{|vQpJNc3QQ4p|jg4xF2IMj6h*}nNHQZh!z|khFSb9LtSFMc-lwThthvaY1iK(=|m_t56a1{Jy}BkMdQiMNVtL+qomu&$qCQ zrZIH&rX8V75eGHXK%*ax#EQJ$FHGhPsVb$qVv$Rl(J~4Xv~?LK7hkG8lI5&r0Bs4e z-bo9Gvc!d7bMzP^>$XODHm@5#&4eCdXP0596ilXoe5-?9 ztM=GehRImNM*^`w5p9e}yhj+LFpqzF(OW&%rjzhE@h&IRPVzYB;6_4aiyMDr225fG^IOKwNs0YSwRn)x6mnwsknIDn|BdF2BVDx6ugq!z5W z>}+ei(j-!lA9n}gMT^Cy0^WJ2+n&e38gEOshj`lVDIdts;CnTXN0{Dd7f!Y58h4Ow zb&a^HekKjuBNd0DV2Cyb%=5X4$MBwLA#!l-D>=FC3?7%_ws6cdG?Ocf9^@>^I_vpB zNJv&RdR5`o9ANq3r8CPpjhzKoSav!7X8gFY?5?Ts;mpaK&0J>tMDv7&W}xHblG@Dg z7-SZ}B1mVksY;OtE$lTH!7VNxV)suo)MK0)r%t_#hr3aihi^a&2)9IKYEH~e*_6mEES{>szB-Hh-YV7-Jzw_It6vcNLbJn%9lBpJQWO3xS zSqF5xj*2Z716*C0+Bjx1Op%4H#X@O8rS>iTUf z_t4xxRw?-xphWVtQ?A+Kea{C59!`5PDZAb>pA7R>`Nloj2+|`y%RGur26(5lq8%b# z)Bmzc%I=z7M>|dShv(J4w2Kx+Jc+ge*`$_Io`(HsK}zo;$HoW#*^ZM`KtN{i4yI$k zXH%|t>y?e5Aa<5FUO6$3G(Z5cKMTH#Uv$HvK0GL{vi3Ah2*sXSRZ|%i4PXoy~q~=8%H5gUO+(pj0jDD@ZD#-g3{*h4*A; zul^35b8ZyVSnA*^d9hQ9FwQH=4ZmGgsbU!EB?o$ykHIs{~GRpEws=n;3VTWtN zgJk-Ekv~!no<{%jAn3a1_r;vjii_VH{{}DeI<{vgCs>!Ik;tCOc^3gz zo(>|aN-a^?Rx@TOf?%%I)0IUcW#D)$(m?Kru4jz7GjfO0PI_6|_ntI?#XjWCnOO}yN%lE#O*kyDa} ziWG23d~}etOPEsglz;QR>FXPsqP|pHypJ%mNFWdNgV7PktU)jH;WdWM;r9s|k z^`%)58}1r~W`{5slaK0*I&H9+>MaPQq8?h_YiLrG{48AYCGGTZA;*ph1;z zB?}8}m0#~heW$jybk8QOYu`Cm*T|>UQ?zpZ5ww7jl2)imn#u*F0v+A4jYmjsmsKE# zYb|So1f%QyA&iP#)ar~EftIKq*zG^qsDNnKMw@vQm zs!{27rJ??lu~!s9ouzUkUP zu7TVAn)}LsX*jN6QbO(;nFI?`b2M__xsI(&(usWM_!GG{coXo^?aP<$$DD-`8maFv zh|#ah_c&#`K9xVhj4Wt75;4A+)~|lA$o8$HEl9Ve9xa5_+N+p)T{ibb(GfL_((t%f z49ViK;9i+iuh(L~f-+)+s0{z@eQWjjuA3he^pfUt5XJOF3>n-%$7Sk0Y2|mS>6`ec zX!MBelGAKLwb4TuG0o6PDIP}+9ZewU!514zxrGX3n) zzZhX8Nh1h?x*@j(KC>Rv<}ZL7X2n0_quVV+^@s~n#`vue4V!Ly~y!y+HDfHNw6A`dDzImq-tADw%6hiMnUuMvlejPVQ`e z2>A)CV^*6zlQ`PS>)8jvD}fBCN${%18~i0>_MIsdT!w)HnDTQ>o$;rGWo_38u1EyzW+(?L%WL9t5*FWAs0VgQqvqYn$x7 Wd(^)E$@SS9=j`Nuj_}*9NB;wSL>3DG literal 0 HcmV?d00001 diff --git a/profiler/advisor/img/cluster_4.png b/profiler/advisor/img/cluster_4.png new file mode 100644 index 0000000000000000000000000000000000000000..023b027a45c94d9ae0b570723e90018c8358e882 GIT binary patch literal 21548 zcmeFZc~p|=|2N*OO`A5;V!1IhHCmaPnY+SdGdV3TrMcunjj0)~A#MoNr_5Y0`zJCyp*XV8mIj+<)w zb!XIrFGEf`pWL*m7QJot>R0OVmZ;x+pqn;nbgcjTj2u~ZZPODu(DAiS%d>42f7yN(dvcG*tDWC_y!k5V&JRCN ze^p3$w$;(>WO63#yy1_=KD!TtV@TerZi&DG(?+fGF^A{rBxq3F?O*oiKZSwoDXM6A zf1HM%?QS(Sn>O8&+@0OHa640`v2l^{zuyEf%ASzOVMHkyqjVnHknEC7ZKW1bU5lxo zt&<$J^P7>)pQt{+t3vaek&Wpr0}sH?iZ9lmI99#00-%9*fZF;IZW>RVcbv?zs<}sX z0cfIwc>vYh;*kU$OuMV5CfW&|N31()m@i&^zs#PWgktYn)r42|C&V4_Bi;Og5VJZk zNeIx6V10uHiqFTdbfS{B&FME^5FQ zaylQG^uv}9*jx?tPh<@}n_W!50Qdnv`&!%BrKOBlP3#`ab_k{Tr1IEAe_Mjk8_Hz! z9!I`oHSf$v{g}CMJ$`nHdor&B_2oYt4Bt}$u+d}w=E$L2nWSLyZ3dIhtFSmL6R?X=rk{2}7>Ai%E;Yn%XA_XJ_#c?A=m1L?iDTt@KTbsSIdpet14Pv7&G_hqquWL{pF|{gbQ}YJ z)a;+DgUa%e^$O{Td52ST%)M7|?HIsT*J5G9k+f1}yyJ4M^bVvEFyr)u3#eW;% zzy#ZEZ(;%0dMkP#I{`F7iNcupm3HnNx=>2}L}Hn{#)xTwcHY>&FWDV#h~?$1N;cQ9 z?wJE>RGyYR8#8NArUjs=dlU=sZ>J=w=h!PtV^!Hvc5n~phG=6#^h}bqV&X6wwV~cu&Se(L-)sIEA%8ks z-2rOZ9U&!S{e2yFAvhP8Lsh*?=~|vV;=0VACJWP#v_2WaVa%Y5*6+ALzU4PmMGk-{ zr!bno>o<_(wLMOPBIn7!8MO;IGXR^ud@eFRzRgWG)6MbElZ5VfBFV1f-(Vdu$OmmdPV}p8zn3TT+S5}4y-`1|&dtuFGnj;^ z<1L;aMfZ2ZK@W7=y@G(tVzL7^PZNYZ(Pm)2oVnbdx{XaP7Hd0oJiw&<5GRxyQ4XRb z?%!w^baaHgrtG%|1^lzZQ4h363OV0K&m^EF(_XyFh!WE>q864G=BKAuC)COWspQv? zZdOIP%tpm8ws^JcP%TeOlyfahKGSWqD8==7X@(Iyaq*9*L4dDW*GPMju~nqg|E%u# z@nX}HPtNDr^0>K+kH{5WKn_e>GYLttZ4ZYgrEwWovj{5$4KpG$W zW?iP6{!V1AFhoeLjlnGm5h@ZgHm)~S$?{x5D+fOg_y`^!OI#0;A77E0qPF~du%?nz zHwK%k3_q&ht+SsTWZA+0VtO|l)i9PC1b}mMI_Tj#BCeLf}ifK0mZK==f^_| zl1WHKc+nG>EY`wfEotD7iw8a$E_P*!gYEzmR}7LA7}EKB{8sG>H4?yRuY-t9o8WyS z7Th#xrBjQo=Hrca$;FpMNMT~tTpPXV!Hx}q-D)!cZk%Vqi`eejKs zA${%=fYGW2REY~PuM&IlLvwI1M@iNM`AgF9_Zu6GRU+3^3g?&lq*{HzSLrBU1l3Uz zNmSv-{<$AvJhAfr&U`6IWtOBG4{;uC;}<5b+9s`DPAWgSvBpa;rF5!qo(UHvgOZjm zEYA(OI-zfU`!U@9%(K$bVM_&cHj~lH9&!b?pbF()V5Cpj&W)GnF6oE}yCPWlJwoL2 zRk5EW3--yIK6{@BUc3CsmMnfR(_izRNr@AFGO8mv<~Lj2D7oQH<%gQ%!)Mw0)gU$b8q+pRl+YbmoL_RDZwTM2otEwD@(j772;IPQSe)jt-*QUZJ0UA- ze{v3u?N%(Tq_P)!KwD5OJ-e9cNApRZtUj?}CRN`y(U$O3x5pm@4TA~ao?}mO-Pz?% zfKa5`5zox8Cchw2P3~%;LEq}zP2`bwD(02DO1B~QVGs%ig#EiN^*{=px;Ich`GH0~ zP3?ckYZSA|PM&0ao7iC7Ix1D`6+@*lg3aH?a&_d?I<>op)C5zjw~VNxd4YcTo!Pn$ zwxSe!H|MQIWG*r(VfB&0UnYPD`e45teUxCbqGr`QVrLS-Zg2kV=WC+}*kmo*&ude~ z>FioA|0}fawh(N8hVuOKh41cZ9<4zn?5uKq$m{w5yKTF^_55Gs(_ii(#m&0 zh3}6cA%57Afxd_N`m;s9GNdsfDa^!8 z>JnU8OYS)5C8zd${VHy1e!$(o?ZI)(RcY<~2d<7lVr8=Zl(;MBygSrTzV|@L#JCOlx zJM&#huT*dDM@8@7?RuA^!|Om}xDuDUUJ8DV8d1rxn(fPV_RGl0v6*>!>Kx&mAqMg0 zj@XJ~UyDcX6l!$?<$lfajfyEn29&*AhgxYIj}K_p4nX4J*UrbN18e=9@4`A`HYrAc z8Y4sB+a(^fPn`QDerBS0zN>MrW$FueHNWB|uV*DrF1Y~v55?b}TAX}jLX{W&39FRVkh6s z5>9c*y&Id(g(%dZ0lRi^N0fOA`W~mWKgDV>@}}#(Sxme5%oU*gPEqR7c$CSCLA>@e z0O}_~Tf9J0kc739!P%vyUg$qUzgW^?hmc#v>CX`T$CiQnP4@PmVGt5+{Osdw?NJ|Q z=Hq3XoiS66lJ=&l&EybANukJCZXwgu{dnjemFiTn8qGB_#O7`3?1_hnF9oxC6;Fvve0qhvk1&q2sPMLM55)xF` z;%`bdn@xI?XUA(-IM70Q`@j%8^~J_sx0GQ8DYJPxUhS2ou6IR%Gx^*+?IdyH5qkj8 zC3uMC0nmjXv{!VuL=o&nZsJ2vmzEdi7meQ5Q(XP_fo_ys@t@aY*cSk`knH(K>o(Uu zF2;0p(#WJRBM;O;dzk!ue5_3-db@u5hG%|R;GkQWc!P5}o-^1oKV&#~OX+P5uLgx2 z@o)u@>sCT4oXp+{(MqL^*t9At?Qf)bqZApOP7ZeyDh#n<2UT{&d6}W<33Oq33$oNv zQtIL#+GQ+-?H8g%*W$xhocuR5S&CKaRDc5CCZ zZVq3+q8wx`B0nj%k!)&Y8+)bbE%fa<7&AUdB3LS3w5+rTxz{T}oAgQ5D)~%cCAl&NW!=yg zVe1ij=lz^!;$CNm!E4Lmvaa_7YOLLR;_L97CZGp}7>=5YiErC7XVtEw&TG&T>S?rQ z0gNbCi;P=PHJkK81<>5sNm`*)E@RdaFqwMyz0;x-S}omeI+zaEp~x3#T@4lp3X6CC zAb`4R_M;9e154Cq`@l45%}*)rqNdDl!?-4s)18L5>jHH(Oi^j-g-cV|TrcOMv% zn(U>r2eoAhvG(hG{T(3LY45H)9j}~F+v+Z-$A2$J#m_C2cGQQM?D-eZqA)S=03m@L zv%D~bPH}3zC4Z#O1X!SGTR%Tw4x;^it+|-u4*>EJZ53_^_LSP-$4k99(aUd!*yK{v zqx}G6y2Fm8I71KcAL!RU_g0;}(Cr%n981wBf;>Lrq}E?P#k~LS?0e?+y;)K#ta9wo zh8)Hq5lYb-aLi4qWGQRLj(wU`$)@@YJV*$>XS0HxA_IA>b+o}d?`<6G?*K6$tUMA% z5cl8NGD1?#R?g2D7TmRq6P^G+MW0RGh_kmga~W$Z1@m;)s=-8a^nq2GZ)7qnEbK15 zDKZ5fC$iih*NYeVmZ`=|3+}B0jCfI-tUWT4xjk}i){9-ur*<@gRmWEsNve;yOTRhG zhdgH+JZ9FD_B6H^7ubaWJ4)68*zyz~+rcY9>{Tt1LHFzcN$>2(91K=#FPI6?J3bL( z96V~)nzlc-w=S?muU?Q9O|G&FW+GH=kV5$ePE&)iKJySDKF)MirIh7#y?z3+JPO1Z z0OvQ-olW{rvM!N(ZG)+19cezXy$yjlJ#%3eOq5urFG1g0`5>lrkWnh=&>vYsGPaM* z8+g6B&8ha$20Noj5*t`=RbZ^1`CL{I`Mg~)eWeKe+<)7MBC=BUD)GF8L*;Uy*FSv0 zC;HT*ZeodT=4;C2i+>06>&gADvkzv8#< ztsr-+gd){{=;4Q=E5May&U`1Y!1e2m2e4#}_IjO74R=PIWWr?h+A1&TrGCBego2sj z=3pQ*du3|ziK+?~X$`rniX2%*c6v3*@?@{F}E~DGPcHnR$nI6sNs6@r*hnUKNe;DDIwA}^FX7jmOAX>KGk}TIsH$# z3ymvf$Rh;e^t_;-UwW1|ccIk8l&*%6J?zfOu?4Y}CpjWllbjytYv~SHXXM`}4|-@- zxPB7!b~PrR9r{Ga7mkvYgC49sA2YaHw-JB}0P=&n+%a<_9$uu0DoR~Xvgc=A^EFbO zs2+D`Q_yz-dBmN}W}Mj}v7i#uevS<2>j&*OF@TX0rryjiLow}rLHjv!eLE;DVNSZ9 z-oLGVWQ=fk5QtLeXBRryZ+QXxrQzxTtL(iWb>cAA%!kFESDJf$t;8c$PusmW!mz!c z6jp{@e(KdOIXGs#Kcx-gC@E4`+Zg*YAGIRcli%M$N*%v*0|PC}BBR3CQy*w5@9m{v zf<3J23Vj`UiErEtn3{SnK>rlvG7=ZFAV21&hThKDjn(6S6L8#=zs1j-1Zd})zF8-4pEo5rutn3GArx%G8k4xh zZ%y*&zy&V_Ziu|~s*z_<=B(Xn@5(FCwb$Pxmv+Rhv;S7guo@z!pe`D;XWQM$IhIYHvLd_k#iDYAaN5wmDwt5igw~hv+*US62hKgkTMV{2k0yUrPtB{-&C0F~kRUkedkzqH?c- zeezJ-qP%?Hu;!il91OBf^>os1X9arp?Z4J$+z?)og3cBvF=SNL|Vf}by1&cfgIpUXRM-2+{eJ%Xtu3|kuay$7{D?GALQN7u_q~?= zJ2#~|%{q?ctc_jCjR*iy@&+uO!K~1=d#IFno2`AKnU=%|mk%L+PXO&kE%g#Ev`2 z*Du>XB+P$W92DcvLLH1fM*O%ob{z~BmA3QUDHlZH zej27~L-gHs9H2ZCA3mSDRLkeN-gO#+eCOKs`0mO8*ZnO(Y;V7Ms&pF_d^lOWOM1SH z7_}4=A7~?8K)>7`_KO4iVu8l~(`o*!J0(&y8^M&~_63FY%XqCL`!WyE<8d2_CgR(pL9=<5wQ zG9Q&Vt#UiOLH*p`!i^kwlgD?TLgi2YJrl*7wJT7vR+h2lN6abBCMoG8#LGfCXiW^# z()p+Qw6>Chy6z~UtIJxg0A&2yDanJi1;D@29aPF%bNH|5Zkv|LU6i~Ob%SULzVR`z z+aE%Stx&sE3xGAVTnD)2|Alz}r$fB|CcXXt-j=3SC|^@h*0!%nBiHs46%5h6Qz;r7 zS;)`n`~QtNJHR9*?m~A~++e=uh^ZQS>K@&&%xNWy8n6wuGLd&$f~3W_ zA2xklL-b!jD69Gk!Zl^Kb_gYlLwV_b|8bR0r&mdf3uN{K3Wznb@xTH^Ui@lfiCYu@ z351$YU6c~^;FccfwSVng`>F86$8_p1g;F81QiJxc#H8K55lO!5d$5v}u;?or8+;$f zXT){nX#&4N$vJ@aRA%&A?`8uJG`I&1rkL*7zVXx~^1s5+*2Qo3eeyJ#|2en1r!Lh` zzE7S7BUXa`*`jHo-;OP<_GJrG0G57GdDy>_YD-vw-Z1L@n$MM@f!T}g5=lk_O zt=#)<#XpU(b(FXPkJPQ?Ppu_&)5HHGQvLa-Lb*DIng$@PCIYs97~{_W&R*Tj@rXwZ zlu%tW(wyN7i|hDK#jDQP*lPae&I*x78IqG^o!^FBT~)(q#)&7)5Ep+YORy1r$F4d` z{I`!#_2}aNfTR8zBbs{Zz^3nTycqGYd(q4+86tM-CLmPH@0ZISE?qD;+YWxRaszzY zT{S;o?7V_;-N8RY8$k9TVO7Q%&@TspE_ z)IQ%+n#pT;B|Wkmkb&7OW>7-kG4H+i-`{7|n*y7>y?p#UI3KF+S0AX1TXL2h^$^Ya zGzR*09nUSO+#vf(f%;&HWz2hT5cBrmkT8Sq+qUIxzP;?WCJ?=K!!M6zUM$ORJfJBG zIVHIt8yvJ-;T!f5hcgX5NZ3Mo%oXO7fi~i5tn5${dNf5Ce%)U~xVcbX_OPM)L4o?M zCBWx$&cI7VCu-BDj-%p<8-jXDaw19iIj@uQYv>!smz1})K`TD<{x<=qBt!yX`$u`s zAN#IC6kEo$*C^KfE9Ew{5qKrCrt(l=Php~5bJK$YBS%SjWQ7d-QRcT&?zluCBb=It0!a6;|y#~;^zI)dh|Lb*%wtl2aDw|4{q zT2-$3)a0QVn-!~er=WvBz{1xxCEg-jQI_^Xudi0!?8DzeRs8!@-&TlzE+Yb*5v>~% zy%p*RD@ zv$kFU>wWln!`h>wI;*UMyUgs;_TTJ%99W`T&&@I;SG7Dj899=#F|xMUXT|O@C0@@* zhzHu>Y?+9}+J1JGde8djg%#_CSK^N(*qa?l^NH?#7#OQ-KAH6s`MgbVk69q_SYhHY z=<_MHf7S%7tx)=|yha860cN2oo%c0!(I2jSqF`wys`t;pVBLC7)(_;PHo@Jk3)7hE zYG15u8c|$vQ1KmCX~-mfm!x+|tJ`V@)NQGaU88RO=NDzS|LNmp0f^q zV|I>N{j-BQ*d>~3`~rBn;bVWsQqbqd7hPyxpH#~JxBo?LREcs@zK#8^@Dw|lpByL~ z_chhM+`vtq_hftLwX(H9-cJ_p6r{jAIYuKeLE*wUeU_YGZF@b?kIq_1pp!7YbipE^ ztQDjQ@|zGNm&L7Ij^b6vyA(vAovhd)$(25ML)d1ZlW;l2Q?`6VsM8v*xu(DTvW#H*a8w=Nj1b;^39q4nd8bj`GkH%V^99R&yg!be*UsBSnh`s($Y*T$iy+{H>v)xpf*-t;y zsMITHQo_$*8BHq)B(IxwNB-Z*WV&G1e(|QVq1@)ig3Y-$O%mUYhpk?cpNh$mYcC&PY8+?^BTREBsAmU5vEl= z#L5oHSSbe1(X>U_9Hp|%SEru$WKz-m%&0zbZv6RuS(!1UCx;ffl2|>CSrO@XLKaz9 z4dm$<7+82AL-uUoGV7J>sa}hw-p`uRW%+FvVjhGy?=xV2nVA!w6d190K@<7XI$)*8 zr}K0*bCX^3RU98lj85feHowgjke4TzA3^P# zcsPTPl{-K3H@h5|vs`TPl&+}vY9f}zRw0TVTB1;L$`=_dh2d!X&Adz|DRBRcoH6J? zG(yFqfxl%a8kPg_5{G{MrfrJSC3yTATr?@V?%$S9wt_qdXihp@y?Nf9%ToLai1G?DSMnei0l!CLQp90 zeSN5C(b3WOSBza*?wchHtYX+xbkeu2=88gAcxUGQ-Pvt@KOPQ$abNRqj@S#U{81%( z4HLA(e;~Lu6GztSv{1L79yV;cqZ8cvr%nXsA%B)^szdL4=n56SR0{Crb3?EDiW`DG zEmwMeDNM~VV`Sc62y{K%53V-|Mn)U!vu=Zbr_$uvv|9MviTsKCi{nN3Mob$zQ^*s` zc)gy&1%?Hj?SQYkS+JLNt}W6&H#TJL2}-j7Wl9?sU9pf3!rOoGz1PjD=WuTB99Ee$ zl*41;h`s=yaW;!Is?->J>d#3&E^ljVqu>y24=A{t1Z){iM|NhREfTYP_DDZHxU2MV zeUegGy)Ys#&lX+Ou_}6rgxB-)9iICK_9afV({d*^?+O|rpwcK$SK7*A^CCw+@rW)f z{9;y(Rqyv4_IofY?22L=Ye#w_kpmp zPzG_d$-IC(nQFL_P_a5ODL+Slni>+SN`X|zKE&fpeo~rLV_uISFETn1>f{@13Si;VuEJXr zKHYYZN?SCd_Vm&&FV;EittJ?o?6lK+mq4$7n7=sw!lV{Y}+a7=rYisY1Nm|s5Gick;u?U z;b@RgkwXh10@goE$QZBfr=BXRJTOCf**iEk6^@;tD<(eMI&#kHUyC{9?s)g=y-0W% zp(XHXIOC4<<9%UwKa}MBHF3}@=Q+4}CfK*(sxt_q^P>%xbV1Y7&ntvR9NE!^tv5!! zOCeS#$2M@arivW%Lkz>*7t)p5dk_iXk30nn#Ec1=r9Pz*@rET&U*me8<|uI z!6#e}80rgTinh4(BO8P72|}r_Un=i3YQqBU;qDRL1v%J-#IS3eCZg~r=(#m6REJ*c zq8J6|;OZe%$53Nzch2>-pP4)%Zz;P|Ti!Wr9m|K<_d{V?v(pc|ZhHEXGo{c`+(LFY zxp)}2V8dB4IwW|Tj9a~!FuoiSlE0Y;qgp(+M0aDoOBohDKrZ6wQo;o5Me4R5`Dv=l zQe4B{CMxgI{n!@>`C5&sw0n5M3buRnv}c65$sI}S>8oLe{2P-3Vb@_j%CAq;;l$cQ zf$zE&a))bpgOFm#gFnTd*)n2p*!j~`w8CsKRrVfVlY4q_3fDiGZo^>QKjvYC{|OSH?02k>&ZE+ejo;88MJ&wGZ7sVcZbj z!y=KX=q>_e?K;wlbMgAQ8JZBE2k%quUc`BxGeO}=u$MXFdl>*aJc|7&h+cn$;n zMe<64anZZ9b|jq769<~Tqx2Y%O)+TRjYwR0KlKV-hGSGhhd3+p%5kGS}+Hhi*!U1T6vw!sm!X8TA5oj|`4vC9Q)shBxYDdbS z4x8ok_a`sH!)-dEL+O}ClCNRWf?s_=piihxhXvRheF2Ds@P*xn8$+T6S0dB8kI0%% zm>^m$<0AVhGsynYa}vdp!Dz@(N(*jJ*Y8pZ(?e&GZ#gjPnGp8-ey9vhWWr=GV_n1u zt1xh%%}+)A=Rx!#{x(@*yREh`J%xIE%Sg{IyviUu^1~9Zq&Mu26&A+jQgJj}CzAEU zexqs2j`#Oy`!G)-3KH#m!95=>$Ou)I0i&wIxuKw=6)ZUg|oFS=23-Vps zWbOrq`Qw~(H*RX2wyG;vX=IFgSI+=5x?&3XLv@-(mQIXDb9hPFVMycN)6hZEXj4;# z@IG>P5d4L=YF090pDFrw1*zRLWd)$a@M0%Hj^5f#AfKMoO(VYRckV>?^Y=;M&*vub zxN0tZXl~j0VYIIb4`*2NZ0t(gOd`*kOAFa$h&~ee9@sQ4+HwV>j=ay@-R% z zhsDw^PK&SfDyZEs95zGol7RFG@u+A5Sslj43+9#A>#B;o@mnpuTx;w|F9S9Q`V6EM zMmSHFNPo4uH*VMhCBGMa>T6*;oX!jd_pxpBryU-I5BmnHh)Mkv|#N* z;ami!FtR9EIx(EnA|m4o>Oa63MZOp@tpTLdV?4ZvOSLrU3h{^-lFiYQl16Eh+NKkR z4-jt}INq%V=H`eS{C+eyT(`;Q1{q3Z1cv)46OH;LFDJ6`tuXLNW)k{I512^Q6|2oh zW3&b{hZhiCMBn*Eob!UAJ5Z7K;ojSxr!78HR19;ce61&x#gqmQqMf|FI#UxSM(1eE z*UQ`KnN3BPaw{i;Cww#g!y@BUWw|$3!bzhO*Us{bFSfW2yj~8nYFlV-D(NvkC*#~) zJq@++GlIcfB?pE`-~Pg&V*;cT)#1{+QkmjaU9o} zUpi&tML^EdVsf=Wt|JaMu6ZBgd61ZRVO`olV@7y&Tg{j>tXHv;?;W@~e62zBgHH>C z=A9ZBa^m!WR7SZTW=p4?O6QCm!L%K_aV}Kv8o%Bgxt8yJ$^wzKFMUAw{1R#`^Clc~ zgMZ`=xmKA_Qgi#h1b^)M+G}-F*4Vh`uwbF4rajk+dks1~T7aqDqf}kz2HLb}z#$7ZAFqCJqswV4U2@j5IvjR zsO|H#{^8|o;ShNiH9c<#!aoxhor>wkUgnHvg?Il_@chHFL?EIM`GLJL=G?t7@$qT? z^%@#J1rp@XFoy(x-?Dj$=aNx?v1u&9Q^{1>B2E%M_PV)vbhK#@Te$*lm0E<)(ncIc zVRC$p=6F1sdNDMz>QQyM_Y_`19@fm3ge< z-RrjuO{z6U?-qbL``0-x-SXmvBJ@(tUi6W;JgYcdO?L!ppgG+#C7-cR*v z20ApYh@@lcq^VA9m2Etd0jB4)eH+C8p%v#~2tfZ55KCoNSiLBRDv(FlBYi=2dEfOe zV?a@tS{)UIda}dN@LAS1UuTjNbSS@GH@&K0LnT{@HUZPVp<#21QL`Ykb(3VRK z5sbY}NXuUIl;yr^dAr&W&OZ_Kgx7@9>`{2mLTgF8_~kZnv?u)M)$=U)Gsv4V0)J} z%bR1XS$3$>YO+hEPFs1gxOKak~zMOI+63*NLI4%jz$>hyV zX^Rw|Su7kNZ*bKroFezBY>8cEez%8g8X000;a)Q*AUK%tOJj3JKgL`ZEx zF)^ddzz*xH70M_d3^CGR9-D$KvPLiDta`I>KM5h??eZv`Ja}xsd%~K1eonIYanHo48mS2)P(D$O+NdKIc z@5;nta2Va5^rqC3Hg6En%r>rKJWO45Zi<2Gfms3cTpjv2I&@W3LKx{g=zy|=aS^%N z1q4waC4=gsUvMLmEy->PS6GjfTJV2wzpgZPeh9_^O0Wf1`Pk(48xw`3R0wZkYBeSf=)n6X$q4c+c(f ztCDZ*g4fwoee7Zr^B-`75Rm?-;LJ<3IP1wNKN+ms+m<&KwNn zM4vssWQ=k^0p6shxi(hL%jL?(?2g|-I^5_!`Xy5Sxj;*^xRwtRcWN-dq*>$0`G;}Q zVPYGAzwh`cjbAZ}Q>N<|*rHYw?IMO0KUR!`zSomZ_tOvHsAca8eHr378!vPphJj={^H_Ty9?21LkK$>T-Ui<$K9V zQ#@Wjb9>2NNQG~65%g=T;+qAu9mnhU>m(GEc9p^%T8W3D(GjbxAe|hX@vJf-44$Zp zojpQv7NnLrMEbML6UHxDVXjwz1B)dC1j~^Ui$zJ(@eyiOwKI4T(cscDhl?DaSb}gO zs;nwYD}*ihI!NovtisbiWF^rSE6C~<_B^-XmxF*UCI&TNa}zEyyjqh^r?e$0(G3ZNArTB`*ytN6NXNuBzl6GXr#<(#ia;+W2rL=_25wZ&tlaLYn8O1U1&rj8E#}Nfs?YwVO}#!#)rm ze;~35t3VqoWHFFOl|jA4q|HIpFz zljH5z%AhqfNA3I`SimsU#DwrT@koJPqo+L9q-Sn+pP&V1K+6PRjhi6Um##zED|uC| zFK*U|V|SAiucix9*R`s`4vZ>l%m-W5yN-_cBg(=@Ph8REY5x8tReE_5*g4F(Y|8i zewN!BvA7NDbVoM86Aa=@6X)C;~hX7i`X*CXT!{%^>~jY%xo9mqB)pcCjU zF8;IU!ZP1hUk0VR#$hG9n{JiGiw4m6XBCGs zs3Y1KKq|fVfOvx66*0yXP7n#%Fk1EPpqxdE3(cB))ZWZ$n(j!nG@uT>Z&Z7;gQAY- zxhL%mY5CfX@Y!skWFc{yY#fSW)^x>(<6!8X2h_+|3J*$^8HIZZ5+f)^y}qNW=fATI zr`leMa1EG0VQJLsi4j0nCB%q+8Sc-e;ZUA}$ev#L&{FL&X0gh6Hk%GgKm>6b_m2n* zjjK7_TtluYIh=7>M_Nn89f*wdqP&-T2Ol?J#G|L8PiStdO%3(gMNOM(w2M4u@SqMk z0QPeCdKHznS6Q))NVskfbf-4JQppIL$R1+GsJj(+^4q43hcoVMtM;LPlGx>tnaw;@prQiFdP z2im`ea~`)R-xRlHX!o76@+?}IrPX@tD2vR80{MLXrlt&vGsv_9s%Qzw00XgL&>fqH z2;C8N>WH&O4HbmDGqNdFBS5kr%Yzr`3>FZ%-VXhgby`=@e8Eq_j&3wVGW1Ih|0L@}b|qA(=xPb$|^~=|A67h^G#P^&oS0A!_oaFO_H#8VNN zSt5Z0ePs`B(vN#`F6XlD+w7N`XPTMCeHBWl*XH6aS)6Kus2=kDk*YgoGp7PVhz3qn z&vozWTqR&<0Vb5@*=zmJCI-f~bAui#vd~_y5GdC_>JYj;RhJ6g$9-&%Q*w{ zw7k_?+6fno3I!dI-*xcBDQYa#{L)ksgDbU^-?URUvBgk&^mJ#e%af-WURBU5elASR zIQ(RdZ~6+-8C~6Nf-Sn5Q{QN1rs`OD8UaNzUlJBNAl1w2uh zArYhw^6|D6qEiCG$|>f1z<07BLC48s z3q7VU18g?kMufGJWF3|h1uT)_=d{Z}I&c#s*ZFR5*lpPP(`EgijSyeWO_>tuiocr` zg1{;z=6#C7`FPDa{#G0$2XK`A{Vw3*&PB~%?Y(8}dJc|w{(ZgRi|9?nW3eQQgxjbW z-~Zsjdrzy6Jl>wo&|1Tu8Q)wW)cTb)r*7sd{-oeGUtUdVV^3i8(?6d(QD_@=3`UBt z$zS1u7t{}oMdDF}-#{$+X^T1APyN&X(bsu$lRr=iEAI-m+I^D)6?aJQ5YPNjS9?$| zprRT{HTJ3k<4~^Xz=Kv-7>f&x$1Fez?U9PjnneHDs5mS?x!lQ^Tum9clYKzl*A?0o z%1w55b7_mCDL87!HnXJhYi)vf^cOjoveL7&F*U;BGQDKzpS&R4x0hL383IU`IiJt; z=Q~QOm)gp`v2|r?M;a!jb^p=#MxIDWVG`+Ig4>e-cRhf zRG4T}7EX=;ZGLzDTu0rYF4^J{k(rX1Vg z(%NZnSa#BEStzYF?n&=zp@k0supR2z#V=+GeI(1Y85AK5(({6&h@a->Q-KFsDW-MQi$D`hug zON}5tC{)t^vrqW?qh9j&wi>yMnniSh*ey==z9Yi3EBIfJ8v$wbI*v6@nLYH-*}iGt zp{nll*!^QmDbkx|z--YkhvWD4%i<%*_(V({zGT!ZT6Y`oD3(GI#E>`R>qCEWH1erK9+%deL#24U#Yy|VNxhrH=`)yZA<4B_$NtO*nkH*gc|Kj(bp?Msw z+@JD7uhRllWNjiA7&QjQW2;*!YQfHBrl^0c3Hoctoqv@_J*!=w&swu@E?K*ki^)q` z_C=P7kO7EeV8l1NN!EVIXL^{Cd@I#_7V7fg_E!C{MENQO7hpRhiJ z_{-#MPgs`~plIf=YA+q{su*ZNYDWyKi(^>>jqazUFP{E=|M<$d_Tg$Q(f;NAJK^0%!Ns+ri%QSZgDP0QRChy`aAY_EVa9~Wt&Ux$Z0l^J2~w^Z|LP8jcVqrmSamV4 z*dnVxWR2!X?AIy6#YhY|c1+DY;{dvddQyR}GvJj!uX!{u)<6w|84nBEW+XNNJuQxk zBXT#C>-3Y+^X5c%1+k?6S5gSePVM7jsi$-+LI9(9%{LvI`3cJ&7nlFeMvj>z9n(ep zCFlnvJ(~kP3jv&`Yo=8$E*8L@_0F`$6;qTp#hR~k%)V_tQMb^exRi-_Z_@kfMv{9+ zZB3=or6bU*N3P9up+-d6wb;L8O_hcP1xb1?)x}cD{S}`hO>NCWdGzJ9o?l20#F?`- zM#4w;l_%Uo{C&wBPkk2B*nU;fB2&{AMt>Z|Lid-$T!*OSZ6*b%AZ1z|8Tdmk?;Pm2us0UX$=d|5=z z?73WC2{{PJ39m_RlvtGTVpVb>{sWUQXZe7zkR^zfIn4D2Jr7qoMO3@DD3rql8g&MY zRlO@X3-k`=HBj4j26!jE>jCXcn`NDUGmzUVKW3wf8tFmWY~KI% z5%a=dF3r0Wi2_%qEG(RN(mnE~|B?aCFIQC6-`;pB+&Irmw_pKQ2?f+B?fjge_(W>1 z-3?SU&?^&y#8Ei#Gl9QvA3{9A_-8**tqxo^ZfvWpkLWdcGCDB~?54T-om2AD1Fe~+ z=MsF+3d;N|i;>1)@KB1tFGobZFEzO|YVjEDEE9WD9U07=W-a6MU{Yd1k76t!T^dcW z?1qF+wbeHS%f(LICyJqd$W^8K8zcMg9VK%e>|m<@2wF!}m!=G{kYYv2FMJ1gvWUFJXFgn^;f|HY|=deF(++npa#0-FktY`BrWvr*CtbmAqf?{n ze93Pe^lk`pX)KNaREK*}dE*x81mHf(Dy_Jt;6Y$&qfGr{!CQ=<)DCG$3$&bZCmomO zsX7_$fm%5A;q_+r)Wk>%L)yV#?hrj=C@H=(&Pzn|RPP3FX-kJiUjNc&%0?d+!yJ`kn^+0*1GL2vI2KKgpE;0~c+yk!?Zs_nj| zm7q`dOPuhgMiqFPSr}VAQ^fKrwu6?Sc7ue~y)=5{R3#PiBClLrP zTd~wCzA$&J$+>!MFq_}4Eito;Q_2rL-Cz2NDPgGdhx=c}*7Pa7v{@cO{s-KlPjuN1hs>1J)`~~b#56U-R9c! zmO6YFZ@2EwPMtN)%t8F7mz(nhvo=w-{a-iaekXRld_gV+X7LO_W$5Zi-;<#ag-kt+ znS2weq@XB)KkCR7U1-_G+tZt3z?Di~y{Yr>lvo7hbXwFUAsDMq=od}QL2>E)pwQou zQ_|OUX$y_8aN}GrUDkadH#`Tz$KwQ#ly;=k(Eq2IvyW=(KI6D`=Loh!Y~3u3Qf8+x zi;{vMRzuQ?Vih7$UKLDy8R)=(gh>$cf>dZHXxvg@3M6LM8U+$4CILbc8b;7U0_u|R zA^}1HAtoWp%_~WE3F%Hx|5^XoAN%*7d!OfbpYuHT_dEA|Ki`B&W2M-qYBoz2C+M+g zl4SdVOIyJ9GLB`5lY!J<-rj#KJmNrX zZ7{DuKW(xmCo4y|+v8|dUa>SP5JOFvR?t$K+if5x*(C#grr?XS0T1~Qa50bbJLptM z%^hp(8wWAQ4ipPTvYo;q>yISSW50#;SK!b-ghx`uNp|Ry>wn7|mDmHemVMSD`)=rP z)iVLwlVQz6bNGAks2owuD@#b(@rrzmk3y6a5_9SxuX>I4Ll#@i8H25x53Y)2Jd|A3 zgkTUyQU*>+|7q-(X5X)4?h=l*TU1C^6IS!0Q*E9BF2-Q}*nB&eJZhYo;x@m~G*_}z z-?b;$LPA3`<)#|RQbb?qz-+^y@M-#@l09?!8{vNPn*5!A@4agHvLk!VbPc*9M$;M& z!HQmS!UwAm=h9M)x0h~mr>AQa^M%{qJstJ<`L9jgVg=h@|617iDT3uBw0kda@x7UO zjtT2u@@_rlGBV!_R2Z0|sc&&w^Icm@C$0M8bzM)Hu}TrgkUqnvKLk&%`LJb4Ex1J)XyEE`e=VfrX@Pt%+iedzn!^kck|0XqFrNaW3Hf!^3$DND_Y`3{% zZLQ}!5^RGf7j`?)kdQ)xNfmqH8H>Mdy=rg%oUt^YF`7 zbY1{Y*>B4J$MeR{f)Lzr-A3iaA+#jRJ6LjudL5fjt5tV^Sh0aH^OvUHHU1?O>1 zthC4;`{3{hXDaH+3+v7#g%zmZS}9#Tp2}__!@P<7@{A149*u?x{Z%&ywi}-*sP3y5 zd4zaL^3}AFzP`^!65j{O_>wUIRX)Ga_0{U2iA`Z|53Wv2S?b*cUKhkNz4x5c`BisC ze`5C7(~n35j{r7`Tg3nW>p`TgK-rgdxpbXZP*bm;RH!dDQFaO_%|uFVzm$d*1rxDE}8XNI;kW0KTbLeD6>=&(?+xvwBb!T=0$3} zH=5~?*I+h!xEJtqC!J8YD&6g*gZ{0LE6bxERFtFTDMbI=?a|Zq9^)75()CA~j*-XF zlY=cgCW|JAzRo@?P6&clQIKt4Gk~KHrdu1H!BivmO)~UeY=l=Mr=+E2g1PTmK zXMmnS(bJ(9T>Qk)ZWl>{y)ArGTr+v;od-Pjmdk|%l2u)IPnNFvek&nCu=ZbHY$|GD zoo1p_Tzdaipv&Q?mz;lrVD?%F8I*~2NhPYR&yDck1$;4qETJL11K*N>y@9KD73+LG zipXaaZNLRlvgg*(?!y5uL*NZP21;m!g%)h_EyP+*AGx z1&aW829`$F06Jyh2*+ZQ;jWOiA0(1*fNmp?KqexWl^d2A54^!YAYMs#&6Q5NLtZhv zZQW`MY#-F)2kq}qa5p|UAO(a?u0+jS_5#}S`M0ox0xQ@0>aW3l&3y89JGaQMge{DK z8<4=)l{>FcE;g0E2D=zaKK0>qSGjIqMG-yr0_Q9TKD~41tay0Q0Ikm$Qu#)eXZYSS zO&0rAvRNPdqBW%k$A|^QRh7kyE`)LU#qfVv0njHVgZ>NK^NU;Qf5c3@>+cwAO6ze$ zR;nha9kUEd*hIDyc_pr*-ZqIMb9XYVTJWPX20_5u7}fyYSn!W?PqH^)%0T%3;~|Ds zi6Q<5ZvTOLVOCZ@(_lODr)F1#U4<$;-MLm|m*$GE`9+G!A|Uyv(@5RkTX_k@ZA9hH ZfYK*C-M*r-L(2~y+<$l^VG2w zRXqk{nm=EGj{=XbSW%t3elh54&Hr`RPy58KSh4BV@^2+A>UqeD6-T~5bK>ZQqyWhv z^?~>_(&S@mx$@lp5Z|Vj$_L!74cJ|!FPi68gzkRkTo_6$>GFqw3NOAab3<+N4@tRYC>iL^$5sZpZ&x<%Q%gwKtmhfr^-+e6Z{UV5~ z7wS-Ii<>Vj6X^y7y<*DYIn)I2QAk2% zLCS_|f&7;{kPtHOMtxUC%?d@mzS@s80yW? zTuLtRd1VjjOW-|U+%bRy@(7b2#`C1lllfim?J~(3_P!mqfffiv!mK%GmOHOsMOcch zSu7+f5^j!8CG${oNHi%kYM!MoTK&1h6=F0A{Gx@cTvoFsG*T^LCZFBSw$Ev?hI?s(D!X`SN?0#ZS2iB{{c+ zs**2lT0Zy3dex_;#oVRo&u`Qp@cH2-6?RGbS*t(GKfR(Z;g{zAc}*!tEdANk|Hzy5 z|2y{kH|?NH^F~X%KR2G3l*A;>IyPEU_WnTGt5>jG{#6OpLX!+I1K-~t_&&R9x%{hA zqgf|5dBAR+hTS?Dwp{*I;bfGV-!vhg*h48T>f2gxyFr2ig7y;<7Q*ET>e#|8`?k} zoc}5Pf7mccauhz$wK-L>FpX-3H!SeTxmaqdVqU@HcTVA}Q}hwad8S7>L&YTLQg!&$ zltO9j4}a=VjLq|q|F-a6jxPr*P`P-j&X7L**p6zQ<>~rsmd-<9%F#hT*Ol{1GWIt8 z;(Ddc(Zd!5=YrGPH%f(l&z^-|EeUsOlqqTK)_OGQx)EjXQy>M%cwaF_C?UGm9YhGW z$GzA+PT$C@=hQa#)^-OLR`F8OJ8j#2_Unl8%>3%*N%%|sIEuOL)Y-hpY9iKEJxdlt zz{i=1-oI;U-@v+y(^P2-L+?U8y-i2)z(J(3pge+G<^3q~q6{CtUVb0+1J`?Z2tzH6 z4W0Fi!f9m*5w0$HBsz&*7AW!Qx8KkZ>TSOr;KNM7^)=t#$R|me{({DKH{}7SpfOb} zvSn#Z@h>C1#=BvL{jT(BTHCYAyl-dE3vmPbk>){$1BIr;R=wUE!?kIJUoeL;l0zo%n8 z3pGaKxlH-$WtNCWZi@od2F1?uQzCHvKnaoy z+5KX|Byqxefc|g8l4Yj3ww%uHId7Q`@w6*AQG{^Uz5)Bbx5*y+j}3V>9W{BHaYRdX zPgZFo-=s6sjq~m@tAWNp5Z3ixi?Q~_^VRd0vxMzfDff_}%o6)^02!cO9E^GO53m(S zuL+BbRdfl~c?At}Yf7P$Ez6y|oq$qyV1nq3!9vDprK>JrZ3Fnte8IpN_KpbrCW+Hw z#UbDjWm}Pj+g+G6M(6AV14|%?X69oz*|v4WQ{sAzv*{+Rwf*SOc5)Jb6axhE%z}ZCuA?m*aEdGeI^!H=fR`1cZ6s+A=kB}{(8TQRN$tK zq=%H+taR)!zANlt%*~C*3??j4C8#NBje9db&Tv$FaCTw544Q5j!Am6SUA?@u&%$Nx=%|F!k^>tVccyuDwnc(`G z?bCoCuQ7#S;v(}P%?o<9mj5PjniLk{`PPJR=JY`e76IEZN7HVWh6K%1JU=GVkie{Z z_5p2LwK{gIap0Z2+iz=~8zy!GuY@U!+=pP2sT6m@nLtM^L_rZYyDd){ADd%5|;JdW||vzNQehQDY$=8dF_^d@z4($RooLrMFI#sHq(3 z(&Ugn7Mp7IFlsk|t&(Ma~YNl_4701gV` zxTjS4V+VuR)86)cfpzh!~Pa}NGVJJRmT$3RS|D!JV-Zm+E?ecvh9q|i%%ZBM~%qa zi+EAmHuM8_sFyd5e%H*dEP;Q{{$K|DLZOVJxb3FF!jSui}QkUW}s%;)Q(`%`Q4S;SN*C&rP>d8`nLt9c-g&y1h#CU z5rWe)W(+?=sEfxl3h4*!!=iHY#vCqPE(6)SEZxS2C$(=Po`E*fU1KGSvv{yLkr(+~ zYm#$m+R*=!)Gx=HI@$)^MC#XV=Yj&f8f-yb?V{b}_+8_iokI=bg@zAKQO1(9<@%9R zY5VYj{408wATS8K9-;ER$2MN2ipiX=JM3P*VU*W_YO-xFmqYSMgyA~lWRI@}@lBE6 z$2c@krx2CKf%YnASJc?ya;3AFtdM$?$ENr!jijcQv%|EXw2F$c^1f=Ukj60$lykvp zRio7jaRDaZ($YGP9&|Fw2)YW7U2HHo4y6pI47_vCn6ZW5n@xTxAMnkr>?w`)qO3Vz zJ1RrtUMMJf?GGq3bCiSB)FHh5cp9Y^*KYZk2W#%38-G-R>*5bi_N9@Iy*lu7*+7C^ zDiNK4r1e^0p^3U)1cPp5Q~YGqC~UUUAp7VmRkBZ%9Wr~89@NB5vUkb~8hj(tb)AY- zspq}fYvmIaNsCf3&yCk);I$Mx<{a2w&(Xz_-W+a@EX>WWinUS|y-G$`go->p1{~FK zKn(_T9(?wmkvnNz|s)$!5F)hY+h#1BzMe%)Ly4Ik!mMAYE-I9Aa&sU|7gsG80Q38e%caQ#!lk`FZ^ zniC07c6&oypbmB_LkfOg378<)rK>qcwRo{zgeWOJ05wIN&|T9n6x)IAcfFUi zo>H7)b1;r><1CM$DZdwjy9A}N#Gp#vF%9B|C&8w?O`jR!2R`onRZcfI<2mj!)xj~3 zA)?Jp%>XvvsN>X8@s%z*WI$S~3vdO5H!0`YBjmb-slMWyYqRC%UoY&ID_8N1L!`H^ z@xRdpKkGYpI)c8mC;`d&zR&>)KqZxG78CZv=Tit6l{->)%1#%?g{+#t9PyQ5Sh-~? z?=oCm=S$;=AqdAMvYl%}@J|cYSMn__J=1!3h)d8O{sFAHD8qWbPB>lCd6IuM`y#hJ zV>}~>DjJ8y;GN2{S|u}!ANv7r6vOem>t&ItSdZ_N zl5;)6kBc&87pvGHSjIUi9;zZNNVgEx5#3OP>!{Boa2%!J5M#+UYLIkWT#RyPAC_KTV$=-HV z?ggV*L$EZp^3%{o@!>)eyqVun{jyOdwL)1|9k#pM{vPI(`&_HW*H~ZNDc*vl=aS(7 zr)GLxfT5FSD=}f<&)QM-=JEAS+j^q6Kq*c4X-9XVv`bYUgqKK=@*vNvxL(4gYvj$SiNz_V&geJ7L(Vf#t2G*yyWVn zh4=iECRew;4scm9CVKPU#`TseC39uQah~^{q7khouT3WQ&|TEh_?s)o>j$n~A4@k} zbEoh9#RquAgPhRRRQ4~?l7wUvW`+8&`QT``%8suFP&3-L-_bEovBkQj3?$H#MFgGn+j)}!hyUdZ#n zN)}@Afl-aBxW;g*V%i@NJM5=YvX8=nUD@(8tMujwftjeo-@(LOoWA1g{VTdbD^{R^C>@W(R zCIm7R@yanm7|qInKgZqDbJ5#q%M{imDTqmmv$P^;UrL6)V_o0ndA&-ftj3_IcH27D zirTR63Prfa$+&w(L;7O~hGC3byBL+;gM@JHrdn^@y@s|19azQQmPM&bsLN?edxAnv z4bfRHRAhTsM~zPSqCxhQP&f+PA^xBw=smY;24l(d+wTzjKKZ@Icq!8L#g$FBx!`Nv zP;ADSLbHWI$QbcHbw%XqktYBKQ~4Nyqn^yX)gGy@f-uv}d^)+f$jYvZYH8A75PVQ3 zjdd8X7fN(;L+1_J(nc4gq{c4z`Q6`jBfDhsG<+(8ltW-*-56Dq#M-8r`GRa86P5M* zp@qj*YfPl^ru4+%q4>%P7M!0F;mUZ6yafAcq^v1pcWq z9l0XOLRAJu?9yrd>F{}COC>x__u;l*U41sWlY}x>x5I`b%*K3n_u8RZX?WA*)c)=@ z&p}mUe7#YE?zJRmZL6o@J&}41_0h9WodX69==CW-Sg5mvO3PU1lNhr7qO-rOOVu;8?n$mm7$=yN+v z1H`R!UFR6W?2a>uz$`2wbZx2l-k^TG57hD_i&~s*OL-ZJsDbTuYm1i-80Se!L(lRW z(Yc$a+_R6^6k5Y6HyNG4;yFR2I+m)A-!Mk|wd2u612*Qo(S`Q+gj54pH)T;aC@=WM zU1+Ifqmq$pj>y`=b}_~Ci(h4mrIV!51W(f~*0r-;9Sosr?&z4~7OB6d!w^h09Ua+o z(Kjr?j2VB22!jqflLetSgU?_g)vh0r9b{>eg}RV5_K!G|;J9DcWUl80E~4X(>HO-t zUh5_s&GXGF(YrkszZpRdZQn#KQG~R^B4R4MwD$dm!WV}*x=B@SJ z+%cDyH9BlPiqGvibq0XB*iK6Io9bfp$8+A)`S{5zQy253`n^q*rqR&Z#h1>f@SSr< z)Y1`7iUUni!)9kij*POcyc~vH=_L%J!=>=e9xy&rXI>c33#MBICN6j-~ac z$vg)?UA9;P{Gji*=K1tGbH8^m3a9P0*;i@(3kGK)F!1VLAIn&cX6q&+DPjiV%{|83T*D_0Y)d7#*CcOd>2wH$_^)B|q1&N%1Tod$RGMxTw=o{L$|JF3 z27?El-^q*=a(X4ynu$;V8Dpwg7@THTG2oib1_D^VeV@o0_+aK%=amGgh|&Ptnz{dL|SFpZi;($>+kP(Zu*7^SfGuR;u}w z^eE>|U9<&J9`V?muD2OT$`{r=Kcilpk5hfyGPRFv2pxnHW2 zZ+7lt%p*2HJ`yoKbhZ!NgW8|7QX$LxQ0@27YJiH25*5GL{M)EyK{xgmP7oQM`Vpm6 zTre65W<+wzS!*)|{ss-w*z>HktkeRHC0-z*5Tmt|6OV2Sq8-V17#O)`-!>`=o32O3 zTu6e&^DcWCnTekHF+&c}oZtuf_bb};!g_Ie7I`G>polP#gTfWv`H*~YQaT-=M4XVt zr_6+{4zye`77}J6YJ?fihVSbX9~EEGD9=%`!f`|r@^EaXKAc03$;U=*e59jTcy+|S z3AGUH5bJRwzMw7{l`Dr}}5xA#cQ zf-^f>&tOs032yWW@p$xrWu93QEGtpsohPJtc{OSWxd%k;fWc1+uNmZ-$%z}EhDLn07F4MGKvUTn# zIX6|GAWia4&WWSvw9|(DfbCszJHlwbhK$`B%&U<0VD}RIo~5#8To6dPsCs^&xR@Ww z>9dZiOfb_PTR=|cPK)YIF1>O!-tLHZ@-1^>EiQUb=5b_<<-icY;}bWE zTc$O{f0vn;Y_)~LTwmn=@!DYDI+cPq#fh0a`ar<-+_QyXv*B3>n=i^Zkvy16>3bj3 zy*TqY+1nS9VlhV}fAUF!gG@pOZK;;59P1cJL0^oQ(79)e5hm3b1i8`o$UY=G32#In zjSz%V)0545*ac<{*U#yU`TRt+@ElIIbjj(cD`KAs7Qhd{)m>BMR&Yfs@`LVuWK(Jx zy&f`Nsg@qh)t;)bC8q_P4d#3@FdCxA9(tagp_(B~#Is!iQ=A%mYf9Wea7nPC)q^M{ z#xFig8kcXBNXCKkUJ*XXM6^CUJl}#}T*z^m8zS|SDEY~cY`-g;k#5`z8s{d%=)uEr z1%wn5Jy}8~97bMfZ@$$qo)PXcn9C7_;*Dw7D_lS3I5`=$3C=00{eBIVg=xq>-ClFh zOZ^Ym7|>mL3J0JM+Kl<4xFWp}G@hY0^sQgR9LPsGoMEdf>m_b>C&jq#iza|Xg;GT- z6FrGpY?Cl@_ZkJCrMWsyd3^Fl7!C)qHlXh{0%1>D$_9{zL&t-DO47v|T@Xaq+PpBj z-5#+LO$xO53BLd}-@55USY;XxWs5m0aNilD%u48fo1Ry5O0$^=|1d;FdPJPMoj3GA zaY8qyoqH*HP?w!;*+To8l4jAssxJ*pa=xYUxqyG)`&2f7yj{~76=r%C$OG}NLH^NR zWI)KNmh}fjBig9tLYa%ij-$2Uc9S)rap{rM$GA8`LQJDz1o`ML(PH+tQkr*nU5pYY zt8kJpmM(j8mRlVi*k+6YYFFi5WhO*7n%l+^5ZnhY69F>l5lqiz<(fWK{b=EoT)bZN zQBD|Wt{??E!unk7<@JGvnn`Xelk|x!viF87TH;)&4?_#8WkhasNOZZ!+lu=<``Y6; zP->X#eNA6F(BCCSFWFC%S^+LjcY)e9XI!}^jT-;yRROv1k8G61gj61tX?%T=-)LR_ z`?7QADbThxWEDDn%=3jQcq4`*Ro8Y$F}Z(Og!)|lLVsZi;p;bQTu0m((0mp>i+>D> z2>J=~`?ls#@;JyXEm&Sa-66F7!X3Uv^P0m8E7I=lQ{5>ze>}0DoKvrNg&JkN`Od#k zm{5wj%`sCT&nGzHZJc=;GvV4ldYY?2-1HP^a^EQSr*!Xj-(?4vX8K-?Ilf&@2zn?lAuRjAE3J9Mm3s?GV zhb(*7G;B2yoFHEq@zC3%-U{rC+P%LKA5Q=N-y4hIvP=Sqa#SHnz>c!L;c<{(Z*GJt zlZb+sHL~}Mw}OU8Y1$Z_!SPbzXp-5Xe_@%;$@wQge&9dy=ht?d#1n<1XS)YIGYS}g zd|~J{ZXbK7DE#Bu86@W+2?Hlj!J z!QRg^!wzK|V6kzitNY}{X@F&ACH$(VJo#@WDF~EH3wxmw)XU@@_u2aD+0FESZG|Cm zMI>CA0~DKCy-;?2nflrJfu5xc$F!GjXX*dT-Q0&f@RAE<)|}T3WP^>2EdCVvricW0 zjaq7pl6cs(d{)fg?jRgRV(dOOFIqcU^xcg5&-UFZO;dsIuW$Km8UNRKJpbDrw}wMs zVmYof+_Z--P3Izx?E9x*_kR}$?|=Jn|0Dg>NhM_xd2S$Lab}txRVq(c&I{?NT#MZR z^+MC{2DQA0yeVq>;^Cf=Txxo%h51^Kn*C&|%4+*5>f#J2*&*pysO_;peZN?xfQIM0 z(OY7lZ~pCLl>>+3{X9gb!~=5L1w?6es-c`ti^>#|poNU9@Gz-k(2&RFF7E(+@VLV{ z3TDt$U%8DF{g8cLyQ#3Dotw9-P-=KN9k*T(7692Z*JwV+ixIa=+|#vQBpXheqe)a% zOw;Sq?~3`@8&Tgp&swe3Tf}=vg%~p*8|O|ucOZ}blCnJ*?2sjjz;E_!aXg@2NC|p! z{l;3a-QT?_bKumi{0SY09sli2_r3AkLdUB^?+#=O2<@4A!t0Zr6Rq?0wjfo?T#BHN z`g$_oJnc?;pkPhp7{$_Nx{78KnA-&Hlw7eHb%$S(iUS*2gSw5A3xP9^yh|Mp5Wxh4 zVE>V9n!%8LD1#ckK6rS4E&2G{2HShBzM4g1O?l%ZsIJZMW^PQOA<3uZnn9>Xx9zcL zKZ$sacCzg0$(@?DI$`ceC)D!Qk3#O%t3d|T9>unZhZkur;9kGol^y)i$ie5f_04U$ zMgR=C>4y^fTtaA|qRDF_e-`TjoOP5?n#O5dubdjOZ;#t9dPT$?iHdZqulcQALoybS zs|mcgQ!6CRTmdTPdP6LL6kIlb@Ku7~mA)R4^^LIo=mB5g2D$Ip>ZTLA?@f&Iywd}Xo+}#RnxDzJKEh7vlTw^1b6be{pxxVHIQ4Q zZM$moptTA7^zQK=vV|p*tz>@x{t~zdGrDRQF)YDf?T#i~R0u&#)R=MJ9S>(m8*!aY8pya+2>Z2!mlb^hpUg-L6tv$yDoS7(K?@|mKBWNhdf^< zYc*;IvHzPn%l zfmFA@9v>@(hyGp`^@eXG5S(S)ZB2P4b2Qa!cfO>&T-d+5v_9yyIH|I6#94#N&Gy`h zt&hm|ye`I2BE{%_%);XKR$HirR$8Wo*41MtPXHuU|F_}Umx4y#z$dou>;*>tH-HZ=|kp zEzYMIf{#V%0(rBTmAAmx=ukC?0m8qEwGoo+nh$0-e7aI-96~l$ehP{B>W6x;JIkis zyU-78y3}?eUVpN1cX)Rs>d7(psSa;j?1OGNG&Y zG1^9NQ~wJVNbjGdes=yoypZWT3tE#Pe zay!JSt*gk&vAHL_V9{|&MxRBsNULqVI>6xrn8}Jbqd7WIe#s}NAv2tdS$+QMy$=C} zF@TBeKm7;?WHv1`Dp#j)4T0jP0EyB{BK9cR zM^j-S>q1f41zuH$H+GN|#+(T7;gCP(f){Wd&7AxTkm5zd3v?>iB^X-HT1_ASZ6>}a z#%nvs;(+o%HcwGh?Pw?}xKs{kQ`$g>fBY|jVE?;Eg#AYWhT-8aVYDkq@c$PO_*XdN z|Ju05#`8>Blo4WKj80?o-KJFP>A5cm450)CIb2J;1Y43x1R{$-fZ8)f3vq07zE98Jm}%CS+t^#Xnn0oy=p0Gb9(| z4F*$j>wW6U$bimzZkYX)RJ7R!ntUG~V|ugktQQO-fXILM2TZoQ_|mvBf}OxWyw5#& zuYri5f^SMK-~9vKG~mS>isU8`U%ik)iyHiK;kq_9Tj#f+tqaj-=o)ASW~aEqhAz2TfScgZX_T?^*>`0U?$PgI6caL< z^jQe%I-sEHYOYIdHu1${vn?5eQhuFg+a3eLg(`17?AQpWl#~21!P)3l*|Fq1Tg3&U z9UhRdON^Eoi+uxK{27Z4?MnYbW}7gAm^3`ssJ7HfYk~$#B++qnB1SaP?3>~srpwpm zZmkLDtv!7@=`K7*9Mn;v`RcQ3=l%#i1r3Ha_1a9kufc?#MninNvJE=4t=^TA4y35BSRG43B`pMwqFcB0KcXW5od-4ey5kn zIpqbbAVvnQTH_Jt0d*=Y!sd_v28t!$ha8Vkz4Le8-)U$HQH1g`FXN9#j# z4jN_pUU^U$p-pl&z9hZ_a}qS_YyeiBnlrCY3m{jgM$Rd7MZQ?xoZzDi+tJ7#!m9^} zRW3Hz<1Z62yN)pJzHZ!dAn9j#OvU4oi`*C<;E0ru?zDhw2&TfV<+qQ@`jNb|-4UMM zJsKG*@)QH;z{RpPe;@-l;axYmW{!3xaM0}E{b5;bh{80C+|>LCJbOOvPHpk|L05&} z?g$0D9$YEevaEp~YsGWldgL6(u1|En9Kcd)BA2#&b^WHsGLLpt)Q*xEG)tRtOML4GosAPy7;eJcVbJs4GrqxtBzO5~QdeczFcZRvd~(!L zAHlTBqi`EZx_I6Licj+;7Hcr2dR+p8KD3O^O1moSSrA$` zk2s5-CLKmHCJXvh?YnRM4aejoB%_fH=O&`uU^j2hTp1hQkV}GA0^3G+TIddU*a_`` zA!{No$`U-FprRtwmQk2ugc{}Ae|KEON%erH*=PV6xbgz8C+3lgP9T`a)@xSzny5EM zjCo$}af0cKA%qVw?oSBoj$2C_^~D?7qab75n)H!HU6!-cHJK|Al(hLjHrW0@3R?a5 zr#@(MGCt>2{VdTiV=5Xzz4FQHe`ZGfADbfaA2n;k*=Q0Ukgl4h(=4d@LrRGNWueL6 zxVtznmbb?AR=JSVylKgcGjt7`^*-4yTSJEZKyK_J>FqL#LM+bD!NVPra)1<-T1?9o zRLf|AfMygj!-N8HPpH~@pg?mzw`4j=l4^#fsb;1@(Ug#&rPKhPwG$R4RNZJB7}X#! z%=AOCkGH+dyhgAOUt*kMpO{TFIc_I%`F+vXgVycLq>Rb*Xz$&Rvu5D0=T+CE-qflj zDDjU6ZJ~@Qk2O_KbXnKmuhhiFZG4b7(QnB8* ztmc5rUXiWAHlhSOQkmY-?#>*$?v8e9uKl*3as%QgR-#U-w5Zveb&VpSdwd z$ZcFDLir~W3ypWCWxbp54lG1*wzgO0)@kZ?>ukpMru4+l>-CDKAOvXg=W>{8#LtdKwfY}Po#a- z5E`Cuol+6rFaWNz!?VKTqv8FArxUPSP^LD0@B05MMFta+>V~o`lc216mASg_#;J9@ z+PNd5^OK#>52$Y@a2gsW@W8X^OVu<(ZaX){`Y~LsAH_NI`P&aTYDEdMkiR8CM3==8 zd>dpgDG{O^O`^R;bNXZPi|fC&hi}RcY~CwSRdtVlh#~qmeo7%1zX97Vr7i=qeZiT| zqih9BTpUrwqrE5ad_c_}NtO42JXf1zt-%BNJi>(ykM34$a2N|sGOU|;4z0x5LxG;} zyu4shK=j*n6C;|>UidM-$4Fk5Gr5vn_T&!{6U}jR&q}`RSsBiJH()_nlnk`#?XDyvp1wkK!&-f&y-n>uXFotdQFD~Rj4#INUUGWP94h@ zUF()qX5sob9!IV%FH(6|AS#HoxhAA#(nZ!mn*E?VR!|Bh{HTG{`|5NCl?QQM`5C=-qet%0tqYg%#$R1qS>k9?BDX(z=*i(=& z2=NaMjrPb7-t>h!>mJ{|u#IEMnk0hcI~nov)Jk9buKpR)!(X_L{S%nVPG85#e4T+kxs-+d(MqD?o$fR10AJ=DP zg>Fb7l!AL(Y1}_CfpPG0W5!W(o->A#vNtsd*WA&rFkp8}2!>@&kh!gw%(XdJ7 z3Zeg%N3-aCJN{eS+>~xcR2Ysgn@B%HEBV5pRc*A2e2WBaTs?GLoWuiMI~Dw*I=`66 z{$>odvE*sYi`O^6*Lt84-}{NbieIg40K0#o(IUSGe4)|KwI4>DlN6ZSJPS*9qo33J zNy@$22y^pyrjIYPXt)1l(faO+`HV?&t)xTe(6@3OqS8VQYe}0j}bBV1~6E3Z!OH$|RQ1?nwUo#smE%g;lA-_cM)V zR$;2#rkcJxmGMNwqDcYRn=gfdVoWv3OZ-b{{0WI8Ue6-xZcAG<9Lk!3>Nj9Nq@hWl ze)xZmwf{MJte-ypncd8DX$%LY>=-KeXFvZl5@#_}e1ID9GkZ-EphKL!oXdOWNB0v| J$FBVP{{cd7N&o-= literal 0 HcmV?d00001 diff --git a/profiler/advisor/img/communication.png b/profiler/advisor/img/communication.png index ba7c753f6de93cdd483b04c16bcb41002a1432ef..d6595a0fea12d46b83dacab4556421cd60bef10b 100644 GIT binary patch literal 35142 zcmc$`30RV8+c(}bGmTSLHaVq}Ce7qhS(24Yitw~d=HBX(3YD3;XXb(m%w%R}>ZGY9 z3YBA4E~KWoLbDb}3@$gyYP?iY`wf*xn zW}H)h`5~B>k{~Sn{-RA<-FGK$L#&L>==6NM8dM$So_ss(_iy|lTQ^+HbUS(5i59r^ zI}fj%f~V(AvL|0!oObn`I`VgT+Jio(5}FkiJF_i6CgaX^`=vNIqT;j^5Ps8WU7zDqD(~uul z#=H-9NY$rgoh>@wERWwH7ye)?WGs55S{$kOpXU(jX|IlNSDv^AM}6`lD*}Zpqg91o zpruJL`V{2Lkk_Y$76N!Zot5DmqzQ7v%2Aq*LJmR{!rhtX!ka#}tJoICd10vZ=ui1%e!>1hXMj&Qsv&?LYa z7&!g&@;3IjqGB1FktN?#{kR}Uy{FTU0P zvPBL9#+MTElKJ$`d}y5oaZeRe!@|-00==gybWH<_e;(*5NvCV5Y^eoN{&E`}NvuMS zcZAY?l|f|>n`OgobftgMRMS8tf_&OgKA56a;D~P-9Jc+;pYHZqluibLf{mL0eBdEJ zZA%Upyzm$W;r0IJ-3YW|df^kB?Fh3E;f=B8w!Ftk{$MT_$bwv}M_e|%h)N0lRKYU} zfeIJ;qBeL66IA!rRBu%)if(4%-_0EJWx>n4f2*U+VQlAy-iF}7m}k5SE(e2o~wGQH$e?@FG4!q?%+c9onVVrAu}sK1duyDHPu(9aTiI5grrWm0IZK&=Rv zRhw+W580hg^cN|1jH&Nm=U)e2OcXLx`fx@=S2>4_nk3_#vi=I$##u7mZr9wCM`jEi zf6%g^%`Qo3K&V$iezEb%PoFds%X zfdJ=j8;Q`BjU76LsI(%C-0jTTP>>HoEIL5qCRKgXsOtUq>HHL~4!+qA+3gEPo6d0_ z>?kMfYe2~p3U0LYHP6#3XO#G0_)AJ4hcYS5-&P`Hx1ej0wM=5zHa8On@ebkvgxoj$ zy!?J_IArO4FFmvaoyQ|{TaPkw)z98Mt^m8MD2YL}TSA4q%az4MmID&a&Y_349eTS} z-_P4$?8{3DE%oi9}PXh8ay?(;L1FoTx4i*-uzuD%Y-Lnm;y86#-n{U;+#M;yu8$hYIV z^7UehyuiajFVjS^`t7@4cxJMIm$k6Qf=2PgpIzZOg_DTy+u@0_wgCGxnXgDsCt^c( zCRRxmU0~w`8(fR6IJ8oA3|p^ye=WiRNjj%^Iw5^9wOFUfnnKr0`JqbWEV$lUoli-T zLw{CZH$$F#H!eivqNfig+wdk(Tr-?B{0`9lxOOTO&;U^{tZ zmRUmfA-;>eFmg}cl< z3x9NfAOST=n~lvVA;d}Fmz14)M6C3iW{z=lp*Nue2_8@bSeg;DDP4RM zczVc#d2!)g-uZi}uzwEY^$?vJY5pPf-isuKqn0hM@J_%99MR*XY1+hRr*82`??`Xc z2i-a^tXYrbh$EHy*O|iK?XSXRu08Oe-AL6FGpbqr&|lm$DJ&Yq3NqEW339*bxpJpz z@FT~k@vlAVQaxN)o6tFP{sj<*+<^M4E0m&a7KVxbT1p+d%v z9@e85PqQV?qNJfE6B-a-Y4ihb5bh-RahZ#l8ttF~Nl=Z$P)xq5t|iyEydqtt-y&a2 z!(9ByAy@UL_k*1`h#XsjRM8v@Jch~A!0waQXVV3Bi_0PI+OM&A2rRw|dC)xi7j!|L z$7M&Sl0%#vX4E&8f8G$dnlOq3Sh#iYQp=Ynxq7vrjJ-+Y^_!uGM@mv>@`-4= zi&q;=;Zq4dKa|o?h>$y81-~R@8)s!t2VaLJ#ke|6>U{R(GzET@^N1P4oHlytS9|B( z^cN-!K8->j# z|4~PzsF|s{>{{oFL&V#nK{DA=?PBN|sC>Rx8!>eHA%0N!nFb_l+(dG8$W!Bpv0akz zWfkJ)3<=AEGZ!Y1QQNjn(txAwf?GK}JtI<~iIm+KTZ_YMVLaj$TIbxCur#ip%}a^<-AwQoNjO8b#>0}3N$+ah z;3b0UO792F@JG)2(4~dUQ6xMRv;SgUpa}A^#!Vcq`g4NPuKO{X)-)}w@17~*NnX8zWuSNiY_xoj^q!}WD&0<=0P)%RG9xRDcpdp>yC3= ztn~;_C2V>0JTgM?F%#sKh;M`#Vfdoxw|0$Nl^6RPwdtvInCHj^TBi_Xzg`~oe97YZ za_;xLZqzy&keeJz2eOjlm0HH&VaZA0W8b{hjxi9=MRkq4;zzA9JHhtgc+vG!qJ+Tx z7ayvsNWNYLW;g5b^_yN-iKa`0yFM;!N>}Wk5Lo?ilAW!YJ8o__(g>C|M1qc|M8E|N zk~VsAK0%8r`|Eo4J)E7HvMtxjmq2)A^*#-Azq%mHJYQpUEYr_JW;elBU`G!*)v52f z%JWs`oN}H+Ge6WC9ZH_*-;a*7M55)QwhvWTm8OgI*`h%qCj_jVJT4HY<7uM^P({w1 z6?jY-kNIA0=4L|jW&LMNcrZa5%`#5wtQPBR&-5^0n6}&XruAym&2Z^C z=SteGi98xGzm2|}dQEi1oVN1lF(ni}oayWmrcX{>;$Db>=j)R;F4oM|miC$rm}{cn zGuA(Otw-qqEBU%^57cpy5ziO39X?X0!mJJ3ydolkLkMOnrbJ=w|R6e4*T>MEuA-6Xu$9Hi2SmRCq z=AGFg@TM)$HHKGp%TrjzNn+=cBp#(k=eM-t-)Q_?Y5SCtjwkDXJa=8$u(bI{^YlTy z>xqa*5Hhcgc7_)7p|>HL_TdpnLdf(mize8BA%-OQ#1SAZs1#!vm1qST;GYW7yxkTP zhOf5j8YhDXZ9skl*~rAmWs3=gm9U|tUlZPua1i!xv{~T_RdvIu&17>-gy3uz=uBe8 zf*SI-ijesxC3TBpe{c-_4maUfl5gKvkvxOxCt!f7Aj0=fPkia28c<+i>h$?T3ique zE*##1-e>~SWW0k(s{2~iW)^xT1g|_3cN+pXek`PLDQpNm*2AUEEflW!D54vV;J4%r zw5JO}Taq+su3KTNSzeo0LYIl7p0NjbC2co8E--ox$SwmfwAOk|RSqW#PXj%*zEOUW z{-N-f?X}ZC@I#|S%`synu5P{sfi>1Y$}O7nD97LEW~|{y@+=L2x0*P$oNjkz<29LA z#?cPJUpYK;J!st&JAN+2s1h;+X*{npmNPzPev&Kh{wU;JjhrR9d;krh_#!i4|C2

@p|@@mr`TFm`=WoeNQU(^JoKK}s zcIIQ*@)mX|nomTRd4DC5t|38%$xZDUZ^anyUu1EJkD>z(Cl3GrWQ%xcYB2fUAZm*@q|4xQ!LPi%ITV8q1>=WOTWs_hm zUb#?8*TKvua}xz*wuWI9FXo#T)Hf||PeIMfSU}7oEte!#t>RI%(8@A|)ohtBT9$)9 z2wNgze9hhuX@0%xo8@y&^d`29c^-T6EB#O3lBJ6Ttau85{xPcgCAF5Iv}AHY3#(Z& zBB(@3?Aa^p@_Y2yw`xfwy7$(a1@UH!S^M`xuwP}xAl0=Z*RP~57eNHNpJ&be28|Qs zj>m?MCGdGL`3g#2RcL-x1aW;TNxQMarEOOAR|H=e_uq)^|0U!9h4R4vJc;rC93CQJ zGMB5P`lM5!gPu>OJ4lAVR6$CQW1Cbu>M__sFj;XOz#`NM74&dWs7UcH*Sw>9Wi8$W z1M0-767F)f1mGi-)A2bx9q7`9sxKt24Jzaqwcv{efYB;|e&CdXJ4Bj5%Dod`M15K53X()6r|YO@*_md@QE1F4N$=UmeJj@rJ=IFQ8d>yD zSlxsaj1i_^YFjyf?L3N)WrvXfu=+ZL28eU_&T99$tpB3F!7)7XEO&^ls{1Ozh0#N{ zS4UBj5e;MN%Z3Jk(?^g-GPn3ru{$=AQv5-jh)#qPn-1wibySg1NA*gb!WNlepii>CK|Z4ompmh*l06kL5QVyW~EYB*m~xKi;geVRA|Qj za4g^vwuQ-}e*5BX`nS=dHwYdWGhm5Q6#>I8B;k)cCRa+G0X<6D(*_ToFol$+xhUZ0TN;Qyu9*+#UVJ z;kN||MjHnq>UpKVG!dJgUwP2xHgwxTn@YbBi&5-H184<`-_R$F*z={v^Q8bIt)u4? z;t0k*4Jf5E(B6^AvGcNk>WM?o3C1~NIRnH39=J}h0aJ&xx(bXe7kfO=*FX1Oc$>qU zH9rT%w>MV^`glf+fEp%6*uky!JuFuq%Fl^EK3ghN?THsDzSLbZsm%psVr7f>y%_Vm zLD*Le=ESt3EFTL~dxtx$RR81`BC|C~YeHSa2eZBDl{R?HPB-sSwnCbIxVmOSHp%Uk zr1IKnzz{3vyZKKE8YHFjkxy2I75=?Jm!!o?zMMEiiPo{wL32B{zc4zVA-r^w_fAvb z(xQisfweXf36RmGozJ5!kZARE{GlsF$$f`TbqNq15T}CDN*F6?{D>D?j|RwLz0Ehw zA>cTUXsPoa+>jAU56wmHp>JLm^DCHmho)-5p?CHf2YzB|ctYKbnH;ZSbxxjkdTO7r z?e7Q4$!W&I!$8lDfTG?t?DsFb)2c=_C5Wo-+!132&o5LZ`{VKGiwS8*1LFFPN{2(n z&%hATxIVSd5Zb6B|DZ{uuJ3YhB`e}3o%eT`F~HV%vJvh~9A_Z?Ez$`R^jGw9{dd(& zR?7(-#@P8L=pX?qJqbLqc0vnMv5sKs8qqIr6WAr(zE410q<=6j{xgZ&I^CZlHwwYk zZ!$ZKXt}|D4=;d_zmy9qrd#IUOgAsAnc~y6vmmct5>d0X$qho=wx;0q7&+VESdjIa;ONt_SVPlYJSA$-kbkcc z+{L89eog)zQfSTf@L!)*OSn5ws7pzFNUlvS;3Zc%3jNZ!QyVe+7LMjuX7VbUVLRln&Al1y>b18$J>kloCLhct|XxBca;H*%ovVj52lI1nX)3$<&{qNe3_6S{H3+pA$jS`6v%- z!fI;134;^WUgaabhV_eJUsT-yJ|89fCb5$yTFr_PhBKx}G-AJiqdT80=Two!3*WX? z*&@*`Qbj5!8iqQwwEB-P9%`n-;RKHA^xg}c;>2s*S(PwLTPhkUvV99R{eu%t{Yy63 z!Fvi5Nfk%mR%vklgw@Ugq|R!pyD8Gh^o|Qv&x8TnLaH{&pnoG#MF44z7FHOP)LG;R zvC(8b9u(!o62|ae_IHUV<9=c!>tP*>0WNRF-}dVy=0mcw>vf;$mK8Z~iWdzZL>ehi z{avDg2*I*{oQDhu8uMN1wKp zTwR)*EruXTJR0Cpcv%EVSXPU%u}_6BCl(J?i2E$SP!?qIZw_^SFmI`Ezhi>P?~8Nn zF4rsord+C6wD`Fpy9L|ib_1@w0x)aM=|H|wgzB0y6(k5YN=k4PK-i@dNKZj(asReN z5#*3rB2Vg|V*?(xyBguwGOSqKp<_iJQbqL`kEy0h#90LKWGScGUj#v?s_u^>ExuIp zgP2Cf3_&ndG_z>=%YdmjKItludKf&cF@IXw2he=82900=qj)>#-${v@8d>{MNri5W z-dy5y|FD3ECm&wrC*ls&877~G;3f;~MK;SC>?E-78|zq)$E zu8t;l^C0BM1*fm;l&Z0JcJ#96d0S>Zt?`ba>;b(34@kaoR;Z>wkYs!{5}(bB!dx*- z6eUewS~8a2O7263pSF{lP%}ES4(h~R zGZUFLi*G0vn!jjfp^p4M1ko`6-tTlB>Q`j|&D?)3Ql;qF{>Vpzy0~X%`A2U8;@wEn zR!$J1-HH1?!`zFi!X490uIdxctH1x-KN23lq^eY;EGZYxWXG{EJCW1qfWsObq7Ev^ ze`?khJ$puhE$w&w634GWXWT?dv!m&BT`)X!bS?e}>?-?-(hnF&m*g>~;2px&KE9Ce zUx^5Z*Wa*o$*dEbBT3iC{@{KHB3RqptbfgsD|YA{)N57YPA-ETO&n&JSz;ugG+rUy z!y2D(0f5UsDCwozN(94W;j(PG2#2#5XoO5142b6V1|#IrQ`dp%#eGOJ(oC+KyJ-AH zX?SgDlP?XSPv8AtHh}&uaagvERsSD)NdNCi6rK<%l0k|KRbR8Q@MYk@znsk@tL0?G z%nAdm3a!-)wG;z4W3ZJ>Vx|pJ)e2J_{g31hvWQWD=$d2-6e&{Z!PH1VlXkyqNRnO$ z63LQ8K6S*U=u3{?G>MVnRi#|EvI(OxX$W&b%I8v+07a0B ziR6*t22>N0Pq)EhZ6Xe`*zv_k&LZmz45|!`W`@r-(k~JlW~@t5duYH)!*4lDmjR&wptA~0XCocOc6PH3ITj$MIFtLEyggGRWOLx!=~&WPwiY92)02gVF2W=dh< z=%r-PL3X0%S+W{{5e&j>#Nk?l63S~Z{0od|Dxsev9^zZajc{KL`Nd*;p?MR?GE7XT z2S${`)zaj>e9B-LN?xBRCp{FW@GI&LtgPa-9vLJCTN4q@3g6H~4+!g^kZnlXn{nBwPtcQAxaLb2=W3X)z(F5V`^8RdI#lEjVXc@;~7TQ3S z_Fp6w=lfr0aKjM?ze|v>&9_b)RsHy(?gNv75r5&Qo-b?uPntQr7R@bYJzp00^%T&r znW_D2V*CpHzm)#2u=f9}7XLrmQv0YWBNDB~(lL#TegGLRXJfNeEb@vvd|DS8{Z@@v zi{t1pCEuoCS&u?XhJ}?gfg(i)Mwwt4zw!o!C_;Um>Lrx~i&rdB`NP5njZ$Qd5VP3% z->hVSZ=_LQ5mW4!0)fT`ukr{H_rFv-E(n^VuW35b8f3nVx)q;OVGkf`DFf~KAM^YR zN6{16$~jpJFMQ5h)V%Rud=klGD;j|Nili#6k)x^)?*(X{eKzl_e!VtkeEZ*eXzzk! zHx4v6t)2cWayXcT=6M1=^dM|{iJkl3h-PJXe<=Y|D@{N0$X}krCmdu&U;;bqRCUNH z_l?wAVgV+p<*YaYNoo)+R$;UvVThg4iaE;gGPro-^9$0*_QM476P-woXKngyR&b$3 zoCGNB{S$}kVw7d{!)*2&EJ8kXUE1`uw=XWz+#X?~ah}{FZl?{*tCIh)=<>}Ny>R8^ z;ggP^`2-t`&rVF-7cG86%7N$@dWaNP0OAKB!wR3EOy}tKx5TMB{gJXH4Zu^sPug|D znVrPDGXImzo}M4^pu54q-SI0(#Azne`0tj9kr9MC1Q__dltV`}BU>7zi}r{8&Drry zNY1BRF1Ru!@MsAzcTg*?(guUsfgvY2JW;v=4avk}yULLSQ#&FmPMR-h?oD#o8GVg+ zMP(#;SLSIw8eHk8^Ul3Nn(*FJjst8TsTPyXZGb@bm8`|40G3a>lWZVM?X16PptqDyaFZ^#bZlqr% zZkH)J=d!qnIE^CbsDJ5&`O|qHy*0ubqUU1PBAU)1MG$8dI|Qem?x!4AXy*zY=Cqk@ zHlBPlr+5$LrOVir;^ZgDeTYtBm?ljOv2%@pHzQH5)oA4+_XTUHeI|#FNjTs>4sY6Q z8m10`{?aIR2TDFEr%S#IE!g9(emvEIDR91>6(Odv{f!x95grT11V98_7H%X-Me!=I zNtTVq0|ZgDhSAqzCGfTZfwa6z%oyS3|9}xu%;~j{VZzY3-8A5GcMa)yKrp^s03{1$ z`sBWZrQuMl*$5cD2r4ii1G{9G4wTV=z5M0u{deG)2y$&d4KTx|64U`e7LjvI!qQPi z=O-!s7LFwpZdYG2+0~S7e4mV#YjlYa>miG0+90I1{-#0Bo9+Ga)S^98~YJQ)SWfNtQ-g z=v}848*G0h?L!)=vv(xLxG3J`&A(W<8o?S3jvxVjTtOtdO4@tD;!N#-VX`R)!ccGI zeHv5`B#F7^bePewpZX$oVR+vzgBF-lZ2m#F=giaZLSH`kgiBiOQmNz=cGPXifvDeM zf-6_YY#@Y9p{!i{Qw3`U9kdyRPP)5sla`0Ij=DFU4j>X+8e-+MpYzv+y5#U`jgb$} zdSIkNN@y zFo?9mU|N(r3c%I~)p5yy^8qS@nvi_7Bk}ORIXFj+=ctwoQeitAvYX}h z&hL(6?o_)iz{8j2|1Nx3p{flf^kp< zWJVe_*P{oe<6i0~$X(LnCkWUQ_0uppL?td#WZlhcmeyK6^2OLmBa4kgvBzfdp5*wN z!X>v6*G+Ff^a}r&Z~0ucloi_T;7~*|P!3jz^R;HHG;GnpYv-0dN`m~d3Sotj)=sRM z73cq>W^px1gfA|Y+q@$iE`1Df$Ox|`moGKV>1~(=S+2)eUN4o3=;*G+hLO7eB61Z1 z#)8DW+Or1NGNdp^+{>Y_p4FYgY=KPqd*IZ8(L#J{=_j@(x(TvSmC`ViA4lafdv%Gl z_i<&d0YOiFBG3wngSa7FkECXsX^MN429zjC3SK*|cDZm^lw^YCO*c*RI3o$bEciI) z>T4)*sBZ-^@X|C9SyMpvWgNuI2P?Sr$a`sB1a?DT(iEMCg{9$K!45AJmjS&%rCfE&IP1@BHGIP2n8ta0NFh4Oqxu6WRPT zX^Sa~B6lIO-Ls`}1K;5rfou6!IjP{#mjc|#@_^K z^e2Q@@2_~r-8QZXLFx4TRRxn(tH2$Y9LvJ4lx9o+m_Yth zRLI#vrnW@Oa}`--E2rafznm5aArj3`6q0*~A@L>B8p4ClFPF2HFRyI1{9OaK{rrIK8FwY&zaY~`wawM?B0(OJ~&jZ{wo9lx)<{57$IBDa)91LDYw70YnWOVsRv zBg@>B3$;NcBYQC=xR?Xrw%0Ue9>Xpr>zt?cGNQ!n@ z8o)!4dlW>h4K{AWhJrXYiilLVNSKE>)^SO5IXsx^LskJsIVvfL#K?!}MMFd^aWtu> z0m(TYt<10~uz_Q?Ei?W5f#>vp5F0*On+KRqe*MQv$?wxlaP*sSi+cJRV#AoV<1qK`P+=6-J|g<> zl3|gxrjn;x!AJuhm_~FeT{4OxOEr5N8T{?Fk?U`*3{D2`2HKSLc(DqC-~{4%TL1ctos&?jrXVeYM=;cM6(5z!q@ z=s=fH4dznz@-+QNWyKM>H(CzIiaX|K5uV}y(J=h9?lV9N-%w9qO*}MaZ8L1v9jE+= zTz+sFT|+b3B*rt;FG+H}b3^|U>+AoRX+FxN+c)tMB(nDHRZv^xx4onvNb67kXFD>>G*UV+4g z%D87h58(HY_9{N;jhY+sx~5r64xmWV5`1iFS1uwzZ+AwVKrF3C2*iw>3f|bQSr?cN z6}6hm;|5@BW3EfS<0PPGj!)T^dd(jQ##+^sN`47SX@OEcm7~~0or!q=;T%DSRYzRp zT;_pC5D++4QzLq@j(Qy)aR&f-k|vD$q6o!qN}eDzJ2AbK%-Kc{O=yd!&XM1``wmNyB@~uG^%$bE?WX^s?JFior!bXHIsZnP*gb~ z?I*mjDjLzV7_g@?UIg;@*B^H)(jC#FSng%F)phe2hBTmeE&^hils^e8H)dqb=(9f9 zB51$8CT|3FuAdC9?gV{$t8*w+xzJ=_^YHEZ?m8HwPIctuR>9~j5HJ|fhg^e3n^+F! zgRhtkFm|BUfM0f~jpxjJzV)@*nl+-?VD*Obb5Gnv(t6h?hN_gXY!p zSy3i6ZA16$XLFJ6|NKTsRgDg>LeI$oy-VccKfiWt0$XdLU^20Wf|j4~2CDoF_C z5PiMhT2c4`_DI6sK=XPExs#yI2fujfgM`1zyV=V8h@%6jewQdpSsfavVV z$w>Y~v3CsoCLTef3rWLeg_mJO)h774bhVJ}buX$@GEJvayqV($I6U({<`Io|6#r8LRpn)Lx&*!EfW(Wx_COZN$T7=Ih1ZB)((1$hR{8 zvz1M;I`T5}1BEQaP}YvoROzGh@}Hw0+V$^{1E>p{m#(!!Ie$c3DZ9ocmo!+semD@nd`G%4~F{ z31FQ>+Fwfoyl{2INwrzF)xF>vDnin9qkv87|_*b z`Tb&GDA+b{A$sFI8$sySp*Pw3$HTcC+&#Bz5sjGJKz2*fOX`i1S00=Oa0kldv)4gZ zjU0+#Ox5o{b8kIkJ zq1;$h^D=SgP@V1!;YP-0nZEt-;*(rLZ~feKbMtBMwu(wd)XYt59o_uP-Ol`7e0FQ? z0QKod#!l0tD5>mG)AxTlvohN<<4)@L4mQp`#)WNrz;uV!HSpW?C4=XOpI0V(_7g6Q z-fjdXhIuoDgR?B@bo?*RlAox~4IOQMCO6Q=FvI_fqDdU>x+l&X(SJi&N0@~K>?BXc zeT0K*(xiJU8fNT%?6dsPeZIQxHp9zQ?6WKH6#l8@cg%2O%Nd4Gnt1xW9oY#}3^ls4 z*L7z_MFrE>tsbT=zj|^ft0U%NY9#_|D~>99SrcYACtWlKA9yEuPibU)Xmp)+VkPlg8W7)kvgJV?3U3W=rE-CXp3PBvmB zh3By*`_)Us@dz=G0_4u+=9Xsx(GMgI=jum~C2qf8t`>U3J+{^i)LYzFJ18MnvwZTrma_PhTg^I`5PKB-yU#wC_i2$@Jft+D2yo9N8R4VK+Kg<9kegIxZek$t!HXs3-m=zMr$NK_>CbQ7A%fe`gt7i+eE|2AG8Q#1$k#Oy9$sO)_F5haanOYXJ0?ayU3 zmn=+k(RvF}FSij^0mX+(!RZEq0+M>*vW%o!n#7q*r(5 zUK9)`uDKl6=itA&%j;s1M;^aY7X`5fUi%42NmV3LwBeL{O&59_aT&q zSv;dl72k(GZ~Xp>IAoOPyrw`K0D|p+L zpO{A?uLktZ^~3qEqV1H(uzS4$Hmz!-&IRF`tkmfJ?Sy)pE9lt5!#vF<;IZ+wnBO1Z z+qG?cU;o{xuDx{Cta;UXz1tC%tM%@VQ?T|p|2O04$iekczQuRxGA738;mDn_lydTC zW4$)CVSs`nj(~0<`iFqkdq4b>l~Hm*WfahkiN3Q-<8N z_@dgB0?mloBKLWDz}FGrC;EH*BoO^p`;o8&KrJ}Z4~^@_&)-{opZZSv=&@gask2{e z)q_ijdqNqY!kD{{Z(2az<9nOlje%*ESX$gs{&jj?n|}EMYS(OETZd^qeBJE#coS~G z;M|iX#gnYIVXsT4?_jF%e3>_!!g%xj}pFPsvGeQ9m{OL3c$t z`XS|K)=zE+9sFRiVi%&-MdKwY>hlSNr)JCRF<%jv23oVro zWTy%V@6JCu>#f~V6=|~g@Z)&h9-Q#}J8qPIfUhGG@%q=JZ0de#$M)ys z1mVktU-mJvVlzDd`hck5;f!F$?a0Njfux8BvIoSG-5(#(JBDQz=F0i<4lBNIl%nED z^X|}~s~FKuf6$OZ8i25d+3t@1`R5^dG1bUG65SuWPyF=B+q!zX73Fas+ksObrXHo5 z7zS<4`o!KG4l504{;|%c`5WWCusCqd$;Q}0MqWl-`3}!?%%ozbAodpjXqk0s4aXpZ zbxXK3KhP)$0I^U;lvleDcZ19NBPLZl&iD&jWYA8QUqcw&!ouE9l8$=OB|jNkUSo(q zf?vzaV=!2r$UO&cH|)%UWrl9eA8bU>#%phY4ZG_j9LE^b^Ghc&@2@~e1>23xKeW8@ zoJ>GTFYBzv=Q8YFhuaui%F}zKA9x4&`)$cPXCn#BxyN;;vy1b?PKi~lQ00pbE6W9w zeprXJd}k4!M$H#jKw;7)N;L`_Jku1d&3WkCdvRS&>`mES)A7Q-&G0jcG)3TTv7Dr( zv>1F~Fl3Yh*u^!M*UfRczZS9od666#fr*```q-mTRO&UXE(VIUX($9HoUPn1|yF)mDTQiHYRgg3`NZ#4l<%S8-7Do`fTD%#hf24tvJ2s zo$_ajw;{sGp&NcfK4BD z=NfXuk{s{446;{rC@7$Lf8HA7fERh#=S1TX``MsQiaqnkvJUcfv^CMO9{Qbg)f9`i zVE90epL;RSK8l*E8e9r}Hf~qmH@L)??}6FeiF(h~&wrJ1WMF)>tSj+J*dged+4g{L zjp{HL4C8K+I<8RgNu;#6Koqgs3cky?L2tx)Ume=v0SB&z-{?nFBH% zng7Rp?MwPwDH{Fx%dke;@WVJfgXjJ1CnBxo+!=3egW5*!&`wE(YB_j5gN>1?b-e)> zU4sbNoLRgB&mIZycjX+>+45VIB?x_VeL6um^5r{W&u`oJG=qOJxpDHZ! zI?|84hu2s2RJW0nkw2p^c=*4`yI0zi`o=rbu(QW6zcY_@3?6$C!o=&n^NhIUjCvQ) zx4Yfk*RhioC;Z0liw5=XZr{!qP~M{hGi@kUzrjw{#8R%8Nc#(YKl_^Ao7AI-#tP3; zagrbXjLqDJ@3*~1H_!-85?+DKOdX?ZzZr43y z1&3$3cE?@Y_5;7b2=UyA_XHDN<+_=Iods&ENaOb6-BB*N{0+H-J|yaR)iYs%cWKF? z=iM$XxD`Em-<`0xM4ie1+{e07(xpd8K0AG9Tmnb6 z1Ji|%!pug^rBdnO?ktmiAGftPU&Hui@t34mias)x&Rdd8632EvyR?Z=BT6!Ux@ceJ zo}Z_^pu9V-zYy3!{^@GvU*5e+jMdRJZH}gN_rZ1%@D5-@0JHTU);O!j)h+xewKN(f%H+olmz1- zN7xSf-C*i#raLkNkK)-PjUp++oBMGG&|4yZ*Cb>#Va!PF`hAjbcRRob_O2?^A?ac~UVUa_J~^EWx)P z6cUsAY-rb-!41^&-t0-~u8NM_G~~|#23b2m>{FT~ugo`KQbqKfUEuHY)47N(V*qiu z$nkUj=1(4v#ta=z;zti`o!%g;zR_$ox#ZRDA>UR6tFl=;2#nVgTq|pAt;ZB4)Z5VY zC>_?K^G5Qdv?9v*Qp zZ`+)@lc=Hf3m&*! zQ@J@?EnOWp)lp{6jXDGNF^Fo|H+NW>6R1ZCP$FN1&HH>fTb>QnAhjt68b~fE{hZ|@J4aHOlJYM+i`UiwqDT?{AvWA44ukjAN*Ai1y3vK5e z-$Eds9w@0AxI^7i*-1tX;nyc@>!xl!)NWV7P&>cEhXfRy z&*CUdO)0fMcJ*az+`N6J`6I~%x8GuJObfkF#_gPnv)Yn8ILJ>4cSEFe>ZXD0j9Op4 z5@9$fzcmIEQm)PP8ZF1GlCUh;#YAz=X_na?EM!||Qf4=eeVg`j%@pLy9Wr%tBx zDcOevDI9l=t{E|P#pYST`!F5-^lj|ti;2_ct=1AAO^PY9pm#=7LH-J#Q%v`M{OP35 z(6TxsrgG=v6OE%P4;rE9=Ub`{!`9xdS}XG%)Gmq1#}-K9_j<9=`+~;eZIlmRlKY5N zul)#FM-2sTAUUJa&WuVvIA@+~buKT<3~nFIzy9m!f!QIx0n)WI(QK%=D4z69={15L zy@QbM65R4&ZY(r3K`ITLl1@Dg;Uo9n>GX~WtFb1wkSc`3vrJ4&mT;?6APU!DW8Qbl zPpv(d{1P>{9Q>|Gq25k%(zSo3PKOgqTF&~zjw;9C@K;8yq3{p-rB$#=>k!K#U*PD# z#Tg*+2EM3s0vX?2XG>vd!~T4_{;VxDu4;{*Y*^T{y8O&s*4 zO=85#^2*}JVFZ|&4>la1+HVT%6BpM87Y`GgM?LG%=E`k@9=wB_7-#CZt<@jSUme=Y zxi5Y}s&wZ6n#teCPN~hO>@Skjg%dTkk3jCJK9MEWsdwBCmJY;EPg}Y%1ATe#@$b$v z-em-&v3SR=Z|~@cAG6E2)M(jFKfWdB$fMV|nfRRpk-}QzZEV+dYdQX&UWWL?zZi|l zUa+>UWxeT2s#vW&U%Q#wl!S(w<;K1QvdeoT{LfN}=lpJ3->mi-3QdvzJ-CM+bq(%K z+U7CV2q#fo8K;5;h7rvsB^sZlL-|oqR_vd=!wEwbPKG2V-Z@hx5Gt-%u9xK^v~r?D z5{X_9CH0JyMZL=O+ISB!UXSJfB5WUS9A{qHR*x4Nz6R|2x?Ph>nOEveDUFnegx4|W zxCeaG`PLN16@h^Gb-0cJ7BfXm~><6~`z36glhlAS}%I6l>(CP<8IZ?s2;p=%OsNEmqo&zpVYtQe( z(><;2~U&K`zTW<;%(pNS|o>{T@G-_2||=$Sa}YRdQn z*t7IYA_e@Z^!Va&?M&av@3;QGZJJP>aiUO50`JuJWK7FK%TU4GRD`MF#aUsge; zT@QW=yY|3V^Zcqe!DdL32@NpG)NCg5y?JYKV^$#lf?p)(413b5{w4Ts%nN$Sq%&~3 zY^8(Rop+!BqO&Qf#dw#2nNhOwm@}^ZUKTNC#P3gt0AJu9p8Y-mG$bB3%hrxD67~Ju z8~%SZ_vS%OChglOv#6ttI^%|lfQ|wxO9W+yFpeU!DWQ(l90107L0>lskh7gv7g!ACcTc=Ka@2OL#>Q~=6e^Mdoe!8Fg?(XZp zuB!t(0thquKAiW$xH|F{Usjgl)+EG7-btrY4s>k(r3)A&& za>D+xANt!&ucVv5IXk=I{Pd~x;c)j%O;sHwX>!L$`;}O=o7G+2UBPnDwrG84y=I z_3n=tOlngq8a(gc7dk+D?jhZbER~xS zV<-vO8@qzAvyS&dZ#EzKXFN1za`AHhT|tGtWEgXZ1<xw$9ugq7Hw zhP2W-KXHIn8vhO7o;xj!Y=~ohWvfW}vn)w`L-P%MH~9?D$^0e-VB?t0)L5O&rC@I! z5++&|=PfuPE!V=60|IB0KJ+*xhn?HIK>%GNwkuO=h3RThQ|4nm11jCD%510fc;3_A zthUl{vLp?5_P7Q1$dG4d%U~BZSf%v(+JtU~zD=oiJ3{l+`ad;&@~pq)-L0Jgk<$J4 z|Iz;IKhlj3U>=RvBjUeOi2gottLGb3Hc;>09&-)+N;0V?pQA)r{zHnnyKPpyoc~X2 ze@n6qr#1d*<_|HDD53u{t3@^|vA}n}et8WwyP5uv0hrXHaW`M?U)Aeqd?w34=^w31 z_4fZn1AiaFUWs2bQ1~7E<l-5At`_$1t^4{;0~Frb_z|yOr<6Fa5-i))ZP~hzD*h?*V@y_g}u6 z3ATQ}C#n9f#BS3VckM zmv&EoNlI!k^WX9IE;XC$Im}2jE4FFbU8Z%8BAqmQbJRg?Q%>oPiWkA6c#j$Hp(k;A zv#2dn(hyaqaLucEXKwnQ{rdy+cQQM=?WO^8<-LaunYWg+SWKU*-x?am{|@t4j`c(6 zw!+FM$hY(u0F^q?rZ6MmF>V3yQSf2vnzJ%cyY_}jNjd%c?<6}Z-Mnbpb^i->b~FF9 zwTJVebDcWfvo60h4LS%a_|}ea%dYiIu&OoiLs`m68!te}pzpof^=o;#9^V{Rn*T=X zh4sI`q?=8JnKb_O`^x88}n|#n|^=~cV#N)tYH**pAa{%1>{Q0|Gcx_ z`0kI3-K^8Zam5S^Gu~&da|pRlb~D#nc!%lER6Wb&L{?trl$oXon7$ybm&M9kFf$$? zILttA%|?DIVgX|(J5s*-&HverzI|CEyROYdl)0(h_NZwXIQ>peap5d<25~c2xsBd- z--{+5$YO;s&npf4WU@vy{$hP2_3NMihQ}okBH<<|{-nOOgg|~2Qu6(S%t(0FN4zpF z>-*@Ks;%Vw=0%r5_=PBWJU%zI{<-1wdyXYDxNbIT4B#ha-~2Z!%e$U#6vqTcDW2F5 z7$D^T`{NH*#%3c{Lc_lt317XkIlaZ3&6?V@V**x6;QxfPGAPQ+;#|~1>_QQ%SbvYy zAOGpt1D)(CBM-~Ri8ikf&Uvw=CRK^uO#Lvf_t{&P{-;PCv1MN3Oa@n5x z=F4+-<*WKp2e`AWk7o)yW0Fe7!;K4zmt~|D_W+r9Ts>+^kw_1 zz=tOdiw>2a(W}c#zekxGImfs_+#Nuar_2cME1Nw+Fs38IxTs*!aU^^ay-YlJ`k+r9 zzhy*X3|lYZisNIKNkmjc3uBM6xOOrkD^m@-mjsb|a1%(`;h$$Wg(>gnZ+t#)a|8&V zT-jl}wtj9d@Cco`oZBnSo{+V5u+kHop5jh#1a{cnlm^>4HA!FWP-n$disR*%Oq(?1 zH@8pDH59E05fnmluMBihWpJ=SW#&ci#u z)`VW-c8nR?*BKm@+6E1B^YQdJ)OT$HeotJ=Rp!agT<(bRH})vUYwQtQ(M|2)N8~6o z&d*s%Ehh^V@I!e?v=236XMe0F;mQrhmK-tuG6#Aw?)GHVG%J;lTs@N-4?{*lT)g2w zUgpFMG>UhzQgNTbvR#Ha?L7j`Yc1`%kF<}a2Ea3{mx`!7#h7#X%aM|mkaA((03zDZ zRn*?%jYwlLDI@Wc-2%e4HB;dU?H)C8qqsr#+P$T+FhtVA#Zw0t0kZ=YXgMSpeQsHL^0_}%GvgVs*f?wKRBNbZNY7$|e;m)7z5-&8LHJA*|)T1Ib_ z%~@4auND5OFl9fh6aOyl*T}Gz=D$&JpMDiClC5040rsuenz)+|O^0Vc z*59Xdra6g5mi4lv=tz4LX0^~8-joxksGN-XZ!f>S=3 zgN+l`M3lDRgvg^J=cfc>(p74rPuarxRb@S8S;t0g5$Q2X5Tuf3B=j%6B6%50#jb9d zL#V_PN75_qWH)Y*pl?CP^j%u}&DAWmOjMtlRy&lFMu7+rlUNcu(jVz$ zy^gp{x^J`JIG!KbvBL2Im(^K*imrJ-->jKt7zm7{;s8a^AM#5h3#bGiQOz+h~2wL z$H%&=#oKjhzY*`Gwu7_h6$1m$W&~9zh>v^I94|bUT2$LfVgLeI)^y=`+^7uvkiIX< zLMLX^%9VZN{#iLwA}MD{Ns+{6kC_RbQ?1U)EoaIO3y;sI`-t^=oYEh_)zmq-J>n_2 zHEYI>z_uG)PSB~xj@+yq+VxTTQ zhsZcz*Lpve(YO$XvgtAi3}K9^onH&viq(%p3fnb-6~pBuKhsZPCESiJxkxTQS5)4X zPt89H%^%J}zr24ynLZk6Slo+A4|VI5boj-D@;4_&UAo7{g0{829`gNo7W6T{@<<(*{ni+^Yo*E{bMbAzfbFpEb$g5n{xRuT$KrK8CFSF+G;Q0L& zX$Z+g+sX558jNcNnBr?#9&s}tT#O|L8xO%2KM9zwm*`g^y|4>hrkj5rtyaUB)*sbg zcm%t@#e6FdBl4dT>h2a-VkX@T3GMYk4V-%~iyiJ+u{aNiUeJYr7{da7&dl?LnM2r$ zx`EPwJ^_9tXYY8W+hywUJZz2%doxE6maX^Q$53Dkj1$`wN9c5;8M65~g8sHq_@>es zqHlpPGi#L?J%s!GxGrl|!{Ov|CbDR2w?^<24S8QxQ^1t#T7$gEUzMue7=Mt}-%);d z$XeyIonm?rib}XY zAJ=68gMHX|D@HzNdU%vgc-JWH^FtI((BJXG79R>L1-p_pKm{qcz1)!Cx~?c^Dz=mU z{xt;ONJp&;?(-~J)qw`B*62I)XZ=d(IcRdYXPVln@ds7Ek2(IQNAo zYX~EP^5NDTs*?}vfk+FePopi-Q#sUpSdde^2hpSdB-36YUS$~;q*fYIb2t77hv8l2 z?#hkw6-617p^2{RDtNv_?$Wq@tMy}?+{?6;=1|l@etoPJeF2%3<{RGHCui)5QsFnc zyZOyUBUd=QM6Zkdlj{+0mzJDbNh%x_2~p~AFtC-wAuNjzLTu?{YeUF{BxItAPIC9A zo!A`i-6KFB3AD|WEqJqd9J$G}t{X?DaL;r_X5&s79q4}UoD}FGGvF&+26ASjrr52{ zs}@{}Tj)T%P1P}^tydn$Myvi7b*iyZq;X_?*-8)uwTri+!xiN>8#03&dj$_cG6=CR zV|5x&TID}a4g0V`3&@^-sabr}!<#YIkEW=|fl~EO(uIZUx$ic5#ZEr~VO4P7O>7uC zw!+p2t;2d@_o>+P>IF;9s>KP!3#m}MaKl{8yRdQgR+twD#~oAWexQz0g$B3X(uZ4j zan4q@5(s=PJ>`?8=L15JQ8vA3k?Xk5y@GxgLg6tq>B^K=XVm%kp0^o~K1h1zFXxSn zyXX=WW3HfKix!UBy2lEV&Ab-bV+NMnuQpxi#)8_7^Dx5`5%%F_Q>%RS16p=`56v!^ z5Pbja*O{C1hPqbxd~(uZh0_9#&7XU#^0}7~6YYsoo2)NI)P#a4_@6t^4WWQ(lVO$5 z{s7%a!OEl9A-a%Gu;T(GPj10&?1l%N!=ac@OfNOhgeg|k){tH`>_L539Sjt!hf+M7b0;UP(MkO{7X=qP#KcXehkXi6 z8)F`m!;>NB=2~sLFIv#yXRK4Y9NE*E8#z7MEdNh#+4K5xA-L9F(iipQsBX7Z8aJ1m zkKubNl+A}-B$&0*$jU63A`IpSZ)|uxry_be8tzDtA5S7!KvmEg5+?}xXQ8A2*&V*T`yWe-cr|$y@nZ zBs>JGH4=*&^sLr@)$lmX+A~kPu8mD%XeaZZz3Dis;HW6+sTkuGOU`wk5f&JGLvDtq-wL&$03~?M_3QuX=xe-R&Ev} zkxP-rn4W0obv5+aWE*D<`F0k2)I2O;=E|xMN22DYkivBIIB2I0#c=;D=@(HEhm#hp zXzy~9`L~{;Q+l?@%;&DX3DNXpsp!!-k|{K(g<+^Zf-l0be;2EBAwltd=#fb@Pj`1h zbc&c0Fs7nl@K?9C-wMeZQ~GiuHnm64ml0NJBsU_9rOf$37r+O}nt1grzE03*a>!rW zv~BB|OlMMAFF(sgv&;qALS(ul*u1uA^);f348_EQRim9rl4lFE*6SMxuI%HBKU)Tlqk5U`-?P>00e4rL!BV4JT9tm62KYje9w8%~#0Wwlqs% z|1e`^PPdMJBsY~Hrph~GLyiiF|Qip4+SAB8x{7CFQl7t%NxNr#2zDjOJf42)vRhf zI#H5Kv#GpkKb@ph4IsA)ifFDdpW*e(E_(uEsX>k+dndYyZ!DjbL$~#S z>k|Z8dIq#zF^y^pMbkx>W3`)>dd;C@sb^1EIL0>y4r#0l5@4|+?MA(uo@2$xobIjy zEZfT1d+l>iriK8KMq^!^Vn^f?nNUk1E_dcghA~n!(CB|W(IntC&w{C4e3;x2+y7pK zTxxZ=H&8>IcGCBeDvSdTsDyaH4p_3gg8zABwe0TJ`pBozs$YGF9Zh3 z?ctA1tJ|?~Ts;XkE-{5@PxylQI_2TkbU! zuZ&=Hl>7S!ddC(uV6<;cMrCOX^s2fxX>|+zdu3*(At*8_m-IA&xg^T!cy)xI`O@-b z{B6dj_)a>-oPHzlgI3Mx`u$WkPqjG8ASgBq4`Cbx=AZYu$GVF!k|R6j&%$Yfc>3-Y zbwu~=V^tYd62mxI-g7td6_vxX^E%GbY$w9=^_)#aR@L+_PAJ^6V#aV4pfw-FeeMOA zM2)R9s^H3%XnV#iJ8Kf`nz?w-S6QQ+fWQ0!pA!Bq?%wRnl6;Ds|2<*v2-!gY0BrXrBs zM|a|N1f8OLz^q5z9OzH?fvTa|)tDYUG=Py`w%)PD8!JqVdEHQUF$U_sHW+YZCaY(%LGs~xt-fSvfl2Ma`Mdj{>h=7U31@mTRkdt2{<^ZR(f4k z>Tkk7nDZadvrz-j>TJ0E#=DfG4w%-)&0Im~=N!&7{${__gCqYS(0}8e8M1cFoHolW5B$$j^uEbt{Y@Hm!+z^Q|5j@8KltE(gR1^VvHAbj!2bf| zfe`;l79~&r|5R4^Z(ZMk@s<%UH%nrce3yKU!b{xBX6B*Xs0=L~Y4T`@j=U>2i3~rI zF6U6+CHXA9oRhIkRTm-X&Gc9vv$937Q_PM)6}|T`=#CmYYns#oo4rs!#4#ZSQclr- zP#RhLJ8d2}(&#+xQ#>aQa_|zxZQ|je_}yj3GH&Zqf9%?U8@_zQh1-fi^5|jzLCsF?rQu5HOI_f$2K~HJ5RN1;a3wDgAAS8Ku`d zEtMO1nP9t^5>xiK)=GAhY&Rc{DtrqYrh;=YKPMiy;XQA`x3%2%ONl8hxrx=`L*9CdFjSVYGs+EvS0Guk@hn+h&JR6%uuVoM{J!+&do?DmhJR8_viIBStf_POw` z&|_$TIx4jjFC?hJ-p>y~ZWp>kdH{9~$zlv))j>r|7u{_`4oF*KM|B$+%LjHvtV@Nvv;dWJJj0?f8+1Hxf zVhdw>Z-4JN?jg7ldvC>yFnJfA8FW3YJmir4a(WKQ@KiLI%F7*dm!?<<<&@m^F%{Cw z#Y=zL44TaC!3&e zeAoTD*Foa?K?Gcx3s9@^Lg^UA6vy$zGo&C!w{}doOXT7WQXrHaD`2I0&13f;b%vB> zWW?Mh**yDvGU!Q&v`9t@&H(rNi@vm)7K)m=1^yGK^Za}OO}dk{MM$E%_$0X zR7AQFw7hVke-~hcx^ps-`DSt=c24K1oi;1`uxzc4e~v8q(_`%hw%XKjZ^)@*vG;y} zncfypC6Iov;>l!u;w$oTZzjK_nEmt! zTxKE6B!pydzNtc84R7Y!>7wSqx$`YYP(tAk>(`<%E5W<2`mg9cp%~dil<1Mu)3dcB zd20(27qYjfD#Or!A!(l5B~u3dnJs*VK|5&MM4@z`>V=SerlfY^q$^{-k@2>RbOm!g z8a|a6H_#RG%Vfh!RIlhMgjr6z`M~g`>8C=uMKO9n4Us#5sF2Jws>C`NMqEXzl%hhA zd}_G#J^_A#i5jg%harLrkdPmQPw{!nco&o=L(9{PO;iD7Lmr9J!_1ueXWTsasm)E( z&kz}`t^Yj6v{Z}MJCi?f)hj@TA_uNS*Y-yjG!oLyaW0W@XC#OpX^#c^8VZSp(JzA1oquA>QOkb2vhh=D@u(@};nL^QqR7jv9`IHpNVA zXKY6g=wB_bL3v|iCVj3E+qU2{x8JHP9+s-k*hzQqfpj-egZy%WA(P|~aVh_B6_kQn5X4IkpcIW2kVXl+_QE78BUad#6%#=SRF zJDE#Z;Iufpte328B7^p@?8j|h!%Fb06FIiPe6;Ji8=KGd26Tu@!zF8CjwhrLSrCP; z;@~}-^SCKOH!Fd1($H&+!UuV$$T$tj-0`xI8u6BVsU#t4lqWk8|klj4g;c+3~bj=t5FE+ zpyh9*M1W}~IsMRJzZEkaIL^z`mbx&}(5YUyl#V%McMT{gONW^t`+3S9$==8c$`<7m zyAs*zYp_rGkD(Uu#Vk;`J0rCDStpkUeM;H1KlL!!G(TgEh+dGX_`7 zv`N4*)@!Y!L%)97a%pPECFv(}pcgi6Ij|!r$#kiPqB+kaddVC-aILD{%v}@qws_IW zHa_T->l!x2P7s~lWlSA0z{?XRWX938yMQA|X`~w9|0VHFE!h~EKd%?T5&-vY@L}|l zkE5fKEy`^Flsd%`wi!$7`iWwNC;7W22#=wOavhos1Ef$z1M(s=z5tU`2kOKRUqBcA zZdvJ6)!U)LAZ$zh&EBbdkMF7zd5)R~uE<~a|>M_d1c6Ak)inBh7B4}d$=G+1|@$w!^-Gv!W5 z__0=dI4#B?F2y`3wN%%#GahcViYOS1!1K_51)M4cbuzQ)4`27`tD9BXmIj6LCi0z8 zk6cs@ZWj@|wVJ^Z4Egf~7B{kI#rY>7FQVbbQ{8%Q7t$|OoW(CWYHk-)FuOM{a$<}K zS9ZHIUbdR*MjUQ>u3nZSWHwl(Y9%ed(yoMEN^-TjXPv_PA)~*F@n-s-k-MwbHHCc$ zfJ7bxB&toP1GodqN0tM3yBkgBUq*;KgHDm+nQp`L{@!kvd8zaIms>zImcv*-t4v1O zsXV8P@MWvLBi@~WTE_H6Z|xfI3#^bR!d`?M?dx3=RE;}TV_}^8UYoH>;!_>#IwQiG zprZ>&mk?j$^h3S=@=g4)pt|I+#A6eXvTbY8A#}#|V0a`2Cs_{L)k;8pCFq&sub`I&N66!NV?9&|qENqh z7~(9$k(9F{yb>kypC&vD>`NuO@WW86nWi`WGlg~Yo>LbJTb-p3bM9jX2E3Pdg`vk? zs;4TU=GikXKoW{AOms@r!fRb}a=@@|d& z<~)~3iT&14#5ruUl9huG`^D~>|8Xh=DsZ3U?zzhW_X+C2|Xd`pfUOapVp0v zy@skSv4$@lL<;%ghjaAgOU0`B=o0*k;+&Z+>ps&UHKi4SF=&}QL6XlQUwY29@#E}i^#!Nb>~0cL zIfz4#xUBU{0+7>p_=fezE6DslbdE4K>@9gHpT5>z=4LDz)73#yh8<7DG?rzY>Mh8( z#3lrv0_wwIWdZjQNz7v|8THi?osiPgR%c zRNk4B?Oc&**_y8#ZC#rWU3eU{aKn(u>nDE@oMF4hz#Wv(FvL*d)M$j)BZ&jqe!+C? z#@&$b#JoG(KBSGXT>Lo2iW=?+U+T4T3V*{t&wp0rx;^uuhvrAm3r%$3Dc7p9`PPYqq3gSDQhzF;~P)`#WqywC9N~10>xtDpqkM>{N%I0>7)qXmCwY zuU_Ra=988CkXTJnLaVoTgO8MxXQG@GP|uP6ZY^+0NyxZHqUatxZFu1(xOZ*&25y;s zQm==Ifb7GeDveWI9o7A6RkEwJc`BG`3q~rDY(#iVIIokg<63auo2xd^SRvO zrW&?|kRSF*U-f)C`uBHndaE8O#wr+gUm3=;s&S3Qbb_c{o30G0>DNbo4k^15C`|!% zwN`?}n@#Gj4K%svxD1#06~;q=+ZWBF%#zaXQzo{4i-*U@!okuAcTSg9YO5$B60TDg z8IZeR!5Mom3hr>a!EexCerjx23%lhAsne$7Z9iKG8lT4zYS=>yU}IT`$a%_|ApgxD zEvI)@jB~(MZ%vjFkMOh7_jwkuh!X9vsnZSb76VW7kuwgPN-fLGyG!%c!A!Si4fXUA zj_IH`FVck`U1(M)tno!X%J;V)KsG#^Uht0kdTgm-$UMW}EfOHO0~Ub`_0KbAC# zv{~mu=rq)tkD4kS-R?>J4aq1Ro`1WQa7i;YkRSh3D))5^6pAGM$alV|Gn|8c1eTwR zvNl87nPKjyV#38$i&+INj!UNZC`azI!ZqZDSAv#Yyo>_- zG6Lh$UXvZEkw<7IFe5_U_SfY^FG6h@654DVuK@@C#)KDkZVc2h7<~cQG+4ycL?Pxc zm~oW>CdAo&Q~&)saCH!XlAB!Qd4w*x93QVnNz?9KETUwdkGm}L3dZ)YgOE)(b!cI- zrHJQi===H}w_9PNS0N2y1hy+0;{=?24deYlWw4jW$pP~~$WgF=5*x5#Vy9!2v``XN zOWzn(8s!_Nk<#_717`#j=1WO%e>7>aERe0m{sCvn(r7twA8M%>P(L@MuM#jU)V#>$ z;@o26rZY}!i=C)WQc^KHhv}v8U-i5ik#noPgST^9s~@QPL8`1D3GxU%GL0dDS?5Lt zQ{TNGIh0?h5~uW?RH4~dK*;)camSQKY(cJ}@zrwOD3ZsXV$X}N6r|C&g@3I2It(Zp zXMH0zp!AjW=<~qRrl=k`CJ*{r700ygm$KgT^;5v?8enWH$Cl3qDY_}{d~8u1A?9VD zm74JS+7xiR(f=%s^S|LO|94oFlk7ia*EPt6J+Za}_gA(=)7xLI_@rVifnodK&O!Zu z)A>=rJeOG|@SF>{zM$SfUBvgmdNqevrq! zXLOTB zz4&L4s&?yBy?C2()JdA4TR)5k?lexcZUZnY_p>%X(|gZ;vDhv32jbjL4l|6B=fJD7 z%8MxK-FyJPj5?Im4Z24VLtoZoj5NSb~;C=uZYhS^LJ)uC6W**6iV1AI% z<~;!~3!5$|tfFb#_tReiJcu`4xGlh-L4QE?@jJtEFS7Q~@kDYH^xf#=ps+BR-gT6+ zI0Nbo*D(gE9gvtw$9==;<}hO z=@aH-^cO9WkX?_pvnaL9gqEyMY$#W)<8(kRHL*XE4<-lQ9CxFZ|4I*koiRR$ts7|X zBvn;jeqstrzc!~ojn~QEzc%EJG3a*E&dmRwC6555GoO>zc2P&ssazWNnX}QO*+TQh zHvOTX10ukp_d;G(;u!w(AZs`5mp51ioAAl!?gn^aCion=zRAOS-AL7#Mz7%3R$-Fc z`+UYwp0aG+EV1TlL>26J5Z3~rp9NBxiOF|fa=Y<>!sa%>y&Js^xXRvtSH}jaG5aHg z%Q=;lRLH)^$d#q(cX@yCl$x9FfhNBsqK;D3DpnC2hklCnWpfHQYlumoV%yH8U(;N3 zpnHsM3-YR8w=!jFmueHllmV5ll9a&bQcm?6Mgz?fN$qc~4sH4O=Uhs4j|!XhbhfHg zhVYjQr}0JguG8Ob@^-;qTv2Fe@q8RnNi6L<3lRCryUpz8quFbS06FZqIrAg|m(0F= zn5Q;yZ=KL&InG3p;Gfxt!E?SU|UEIgA zo?_V#bYqp@HMH?-iN-xdd{}K#kOKxC7|cDgz+veZB1e4M!+ejX=!Od-O;A>KM@Jm} zS}Oy?Q8|{YSY74PFa`XxjI<vxQBB=jFdIna9*?l1-CFzC>*?CR%{ota^9i3h*7` zm4S(79#z4Yi{n0BM0Ed9N>+h*P(g?AvmNvNq&3<_#Io=Z;+Yep!*6TF4sIZRMZ8YW zcu+ODF(ko?p04U}=~Rgi>(0!xJeBpcwnfXc((YcxDH<1O1?y2m)6~4Cx}q^6&uLG> znhg5S1il8$*HRRuRuyykaf5#V#00n(qfM>{z(!AX;@+&rV_Im`<7yrebKr6f6%6rJ zog~K-Wpmo0Vn&}Ud^QYmRd}g;qm<_lcGX1r76gle5O^>gd~n9y`Et-UMut*An_ zuSUC^hzXLUu&nzj)G)i}^|*jW#%~Irb;Xp&;50F>dPW^)jrc?FUcASG3# zS3R3KAxzLOctR6J8uF&65p)+_je&3ZD$pRdncB4=^%NmQ*k*Uaj+SSID)%%HcwIbs zEueTP7(LUl0+t!~I8KRfwSZ*HVE*jhyveDmTvwAbMMJzcWft4==B+2JTf_OmF!qEs zw_AWCB8uGX_8lYt&Tgy}=q)UVHkI^vfbh5yr#q~&*Q4Gwxg|A%MjhUhaSDymm|l=? zCAc6PGD>$T$f+Cf1pKtTDoA)m#6IJLcLv<4*fD*;@b(5?QWjI?YU;Z^BvcWf@ACgd zM7J+@CPmqdEXbx?RZV6R^A%q0jH+$?NhX)#-l#-S%acb9p<5hA9W>I-To1!8rkHqn zc(b;3#N>^-VOQz(b^Q^IG0=#nfe%X5*AR!f&5MdzlLNdQrh%f zhqtLRjbm;~K^}DDmK)~c)N}#MCN{`b8`DhLY;ZW%RU5hVC91|d5A$adzM(?r0_546 zmVx$alXiwzsSyg3<3USQ$B2)=(kKgQO26j8imWnjrERMhX!3R;f>LuaO&ExlIFh-M zaf~aV3U!049Lna)UJqxGBvV3sb!)hL?sN8^5{U zc^&bISFP}}=XvUYRj=h*WeV)jy1g$V(0;6M&B=?R7{b^#2*5Td$P|z3Z+TRuRo-=< z&NRx!BqUb|=hKzBN3qJj6Z*cX^~Mp=R;m2jQ0}?8i-;cr5G{t5uIy+Pnqxy;X}UfN z{RT97++kaNf$YEzgg8P-M#UK~f1=2Byv3h@Dpx5;{c(1Ovjup{E4Ti1f!| z)QhkSM@;oK@EfzB2w8lTbuT`d*&}lEkGW@*PxQ=DG4mXfmRvT{DoUd%aQ*6m7!^UmLob5hQ5Id(V4}lu1HZxIlV*I z*n)0t3->IhCzGWEjxr;k5U5V}^QCK-A3PY|^fy7UyDmlSch;0x3r?QO9lu)j8&X?^ zxl8J<3b4Blk0>^zv^JxyCZA#zA@8Zkm9b4T;#gj0KWj*56AwCt2mQd7c^2S#2h${? zI&b`zwf6D|M|HQf2t~#m%AgIRtRUsgMqBnZ?Pd3tO_{FOZk0|xQ>>ZiPRA+NI`4aZ zIQgdQ7o2*1f1coq!w|yyDEZc6uUpF`FHXJIXadduE%Z zPe`N2Ur@lkEv(A6dm{UY=(ZlnMPZHv2kb^(-b14)D25K&6P3$tnkoDdwI|HlUG^b= zh~wk;0srESW?~j}S%kamk>bZuD(bD+q(JkRvLex40L_H2@j3=1jxY;wQ?ytgr4rF} zU^oLn?lCYsPG}yh4A(oq_7vuGUgxq-r(kO2M6sP}eKXo$U4~nm**@Uo;W(CO!tuXo zZ3bBhw+PaUuF~WL+W{8{eY~`j5FP+`I6l@Rnk~aq+d<_>B&z^9x7fVoZMfxsN9pZZ zv1Z;>whv^qtk*IhXqboU6D&nsF{t#4ZALV97ZRAg5xbRQ@1EGhjgL;T(S_Vdhzk|2dz(A$Cp z4YiK@=ETWZH)QZTpUQmj$^Opy!h3Q1_#^b6JAX;BL2t9CE;t`AxmMNWhk6uugg$X* zJvtg~U|Tg0q=p9TCgD&%wwH!gY&*k%=<+z19zj#tOj;# zy{1lwPKijR(DRs#vt52G9%Yrv&qJi74w&p59kypTHxwH}vnDUb>773wI4p|R4olRT zYL|I3Fe{TjBlG$<03o^+5tHRX?56C}_}p4;KQNi9k@g|>$II^G7E|mo0Azxii=HAF zTx03hAJrD)c^38;Sit83#ti!if6K#W%<1;O($Mdm~d=$h;& z-Z|}FY@YP^$(}up+C!8SQPu1venGYxB+CbLCP(7U+E1^pGrmf^G;-?yx^9d)JYPOXEr*3DlsN1K_`<%uKAWV}85) G@c#hfg$LUJ literal 58862 zcmeFY2UJsA*EWjB!m)wA3LLr$ic+OZ4TuOFno^|&kP>=O2t5QV9zbc*OHg_XNbe{} zhtPY82qX|72mwL}N$$p@=k*=;|L=dlJHCJ1G48l)494EsYwfkxo^!4>pJ%SgU-$Gh z&Yt2p#lXODR!j4a0RzMDW(*8RPyT)acrt_v3;`}DJvGg|7#Ns7AO0PQ6Jp|IV7SDf zbw}06Z)$B8o@hL3D@NOB_|vwC9IcH|AC_2$jaa-Ae+<9ubnZfhh+XEy^?AYftOcre z6{_MKCrvot-j#~}vvgBQQ|gO$JHO_?)URf+O3NScI1Je@pVjlInLl~{B>GnGMx?*O z(W6`!&$Qr(ip1w|v~plc5d!ZwzuJtc`5*$!?t5`F%@_Q6KOuhk^pE?HD~Dh6 zce$wA`P+}%jNfmw{(P){uI%US^`}wCf848_e=hvQ2NLL**Ys=ak6%=_uYzh6(bA6&W;sB-KeE}xaw?)NVs9=(OLGU??X8ftsY9ZMP6vlQI1K8=x|REsSZFmenvjHYl7?cu|unt)d~lH-yMbFkJlXQLAY!jYA+v(9^;)((2c z@k6DL+_09Yq`42+oT5ZuHPk*f%^?uB7R06RmY^#Q+;8Zemo#&8{+8|UiB3HD)PXmdOTQpilWR8+WYa$9tsyF3 zAn5F&CnzK>vTmMY;5%xT^r515Ins!!S<2T^^4cie*ZRXQ){JG2m^}5)&`;L>^A!qjG@}3ow5DRB3+XVp1-WKnvW` z@b}#Pd`5`N4J-!=M^rIhtg5WFF?p(*k3zZ7b3=|F=fGOcN!mR6IG~gK2+BjfZ_C>( zRmtc}O2KBAODv@b4HfNA#8S65XD}?Kc{+XUB<+Ip9JPD(wB(+JW15MZ`AKT3X(y;&K|crs@{I%pnr$=ON)<<7p(^Cf%vC z?zPrZNb(s@>1#+bW$FxjY4L?(?O8VEIQIT3=<76U<_In5E{aBKbGEcjqJ>6w$|dOa ztsFl(ze@96KwD|A){W4=&}Qgq9p!6~)uGfZmw|yZoO1FDXs)>A0(V43N72y*kDf?+ zfN|4S_H?=5&PyYvKVr@=i1ozf`Y<`NqM07M<%p&xT^M5<({kA71*d2w=;^F~GMi z6H$A)XUbK$`Od}G<#Lo?UWX}rw5YQOjxYryX6r7$?pb|IdfHL8Ms9c0 zjQPF-?7a4qkXq~W3VR1HUkW#o2)z~!!e?DmVw`6V?Dax~IR04W)wn}sjX>#i9) zq`{PBtr33yfruFo|7ua${h1BLlioj=TCvAo+s@ycHFQSQp79O*R*zVO0K8gm}{ z)!@Q3(zO{=7hFC1W@C+#w@_Y~kV}{&n{BrUt@r0VRhCDG+jf2L*f!z@#YU zTEbFYFJ-qynC`5`G49*orvh_Per)feuOyJm>wsng%((yR;g;I~5u}%0Q;X$3^Ij%* zeolKU(rY!amtmlx>asR=HtH&~Snq@@U7C%%eBnsqC=u1|ubtq_l`FRj_V{z(`Ph1`jOva9c8%8-jpep5wfLeY8Eu~6AO+;c=3fn-8!yN+Ga@}Tx2nNa{6aZ% z2}zmNkD^|mt%YG(mmbnDcYIJR7m8TZKMfetoMd023E`V5 zewEd#ax}z*nUhn3e%jRG6P_Chog8ZURNTD{a&Xh54JZg7OtsbGpB6LbR*6nut=(C= z!urb3$3ODZm+m8#tj z)nst%f%$}}#R=i6gSDfETWz777~9qoZR*z3)eVJfnKT^J%VlPopPoWdV7c>(&?Ifr z)SlL`G`&WAO%(0l=54C(k(4?p^+lklhY?~Yf^c93yf3Re$Ch@)$&Zg6q3)+;e{ZdCZs|v76w!kdpxVd*K z8KPz#XQf4J6o)r3n`|nr4Lyg(Rt!$OF%DNy2oP*2+{m;R?cGL)w0#t>r9I(=?Q|)C z5N@Uex#Xp3Nw{jO@~}W)&<2;JobMP+w02w5F&*q#qU__2Rj&MmxCLk_kpB=|*^pf4 zqH=K1ejKV;$nTMja?mnsbG$n^x*m$>aYHEvJ;aW{m=3ICr&QAv#vr?DLI%Az@9os2 z5T6Mk(USV~>ycpPD^?fIoyqI#%|)U!4I@P+C=)tX6XIWn?O4OHMSEirPw(AbZ>v7Im+0A8>$u$FY#K5?@kS-O`KWDOYK&zWVf*RMy9*z zJW@M2zO$7X7#RP^&dOw4;?2at!7RCZ2*uhTby61hiMMjRi#cG#)r%ED*_7Hj@b33n zQQ(htL^L;2gT0|SuTg}l@?e76hUdBcorAA!Py3jeyPr5HphEF2`TN#ZX*pNWR`C5^ zcQBon1ZMU9TQxYvJCg z#!Xt3RH6mHm3z@asDbKDJSN8h?(-myo6e&s7ppw^i0WX>9)M3`pQ>Egl9Jg8o|9WR z1y;a@DNfyg=`LDdy3RO9@CP?MqZD3ejOc!@HEN;6i)ZUYzb#jey8wdO=JZ?4HDl+8)Q4zoR_5nOTS(+r=zHO|Iv3wcd;;v;VD4C$o6p-n!HeSm$LQaf5ubsREV zStH)T8*w}RrQs!=d#Y*@E-frc{uFra)`BsOauR^Hd?ydLhx(Z^c4YUX4qglZ@0z=E z#p-iA#1@1=a&mT6e_VGD-pm(Xo`T##9VBeB4~a7v$~hT{*4yilN)T*{#-B6Di8{&% zHsSJ8TQfYXH`1*WN0&3y7_Z>1E|Q=!bH=VVOUPIe?-eyEz{J-4jHuh;0*=c^)Z=Xz zM|2b#iK#~o1KSgMpv;Q&WJ~!V8^N9!uA5Soxh!X<+}DbQjoHPE120jAG6T@)QtSDz zG5sj?s`|wbTigA(GIT(Gk={Beb=XS4nXtsR*=M$CL<+Gup^vU0hWdyY-h)jf>NI&e zxhN}NOVF$5W1f?`ri9&h3cjq*QqlA3hNL9z8#+r9cGqf`oH{4X*U_T5^HqV8kMNq0 zV4^q5w|!;{7_kQTLw%%Fl=g-!eK$WSp{V<=^%P1y#~p;m-NkO94)3!eOb62+Rt6VN z==D~hx8jW*mNDt^$?H`+3d7qI#V#~V~m}Vdx&We$_3K!gK zPaYL;oO*jdpHL^}BDsVAYjUvVdY_QBrWLRa-Hb&wUCBHF<5~XCj*CCyWf(w=*v)f|e& z$~)K@e`>ZIHCCFvX6a+V@Tz~UYvwhLF}J}?LFT?_-qpDS4{CwZZh9U&%)X%dk9$Q2 z%cZG9v+~YPLCgtfR^E#I3rtHlpZDp=+?B}|C`PUA6ReZ){sdcgzD%sLmJzi>CJZv306*O*A7 zQbH=smf#QbEoK}hCf)PjidLeSb@;aCH7VTsbKX}Ab6;+&RK(|cVOO+(DJvqv_ZxhK z-d|8ZUFiJCQRg5f^%k~8hwf+<-#E2BC)!%ZxYKo%-e9wv7Dx2zVn=pbBVjN{rG{HO zh*wL_z%rhD^4J)-G_VOWj@aLW!Ss~jK`xQS1eniXXPZU;$PR#+pyh+deBMl?Vyk^T zhps@lC!%4xua8?u=M!L2q{wpXPnRr-OdXdB4V7#J@Fm{Ovs-k6Oc6M}E^q_gkh@y$ zu~1)_t9sP8s!BF1-$tdfj;?n?ZA>=1zC+4>%&Y@aujII3vC$-X0I?f$=wzadT0q%6 zxD%NPHtjL=FUs~S*A*7vQtkD&Wg~`}S*JvUJj@p>EJpZaW0A7_ zt$(q5{^#vq`3cUJ-&Po#h>3D72DtS8NrqE?&xls@+4wJjvYbYnkldQ7U-nK z8U)i~<=4W{*nRNoN#478BM(H50o$qCkL?tozxLSWSt8I#UNy`6N!EU8_;^a_ zS-km)=T|CZvI$e#63sT}svhrU*RGXROTK7p-|tNe(jt(}aGjn2dsMzsiBOOippr(D zlpc)*Dx|c5Gq%*E*>~Ro99K!)*`vD*QLkvoD&;ey?Th=%v?JHCG+=6%be|qkM~gP1 zf7uP%sd|x*ap}r9n{O?pkRV*Az=vw$8%HX7@~%8klUN~WNNmrQ*zZ$&s+macmI)3- zR9{)>M*m)tm&A%Ev;HA%;X9-)YvQ|w9En-^@P$sHtuG>#lNVm5?K2nYk;KwDhbG>h znJlnr?y95X=QtizV$!W9`-CLuIfM{(0K*)@aTNPqJo5SkDN(hN_OFB4!tRf)G2RD{ z;cX@0em(U|`vfF`M^eg=x2Coxo8JeuW9Vh1TUJ3dU5EJxt$aSiS(}D@&|D{samUXU zDQ-n^>%GIQR?0#(dz48Ro%EAuGDoW=U2hLBl-D0+g-7;{K+!tu zq4IN<(<62mRm2#E_`a|%0Wv0#Q_+7ZYW>)vAfw|1L;$yaJdFY{vV_%`PfDcXwR-w) zKWR0m$Zx#=`8aW3NWrh{KudQ>uD?%6cV*{#bAT-g!5e3s?YNztL>!MTCnS_C3V?!y z!;uf-S(F^jC!?;)_VtHFCpbDSDGv$s>_e8HC;4x$VWp&nyYzW1V(PMcfN<59Cd@=s zAgx$5x0gNk2_Wc+^0x&*2+l@dd^yT?mQzf4V=18-aS0)FefFwFbRAO4qf(oAI+E>7 zktxHUGHYAw3C@H*S>M} zg~w+FhxfSBR?328XW<1o0Uh%p)#eyXe@IRTsvC!8nXx{Ua#E~h*2b=KrKnJP{>EGp z_cV8@U#@vf_3jIt`qJfmxtu_+Nh&Ef*rMk|v2$W>!%$F{Poh?L#sL zSd&;EbWN(clt0|Jv|{iH@4n|JrP^C!4RuTKx-0skF2Ng3k==sBh=GBTbcC))?FPl) zaqwu8e*jF?AFMyT4=>e^H+KADoB4PjWR;-6M!OCQUVLLI-*Cyftn5bfCh{&2*5c7T z44F>Y83k`I$Thm&1W3wO+jRN+e7j3UwKsY^nr=d@q=ksQ7SZjnz74BSCsL!P^nI8Q zMlc82y?#ZPIn84$WFWJJr@cnwi1mzPjA^k`d>DPhRJtgX#RTH22ELx!^$_kATpehx zs0LELW+gr99hifF``4ue`8&?~>2p|vQv)4!x@wWNf-u+9pkRCD-~eAGu_U>xUP$Pl zUK;gMB584_(RK3W*inZzJZ!zB`2uf2pZbM@`f6Fx{)m%oj)b9hZ&>RXR!Rv3GS#gs z?T}zQ+P@dn&SUvmE*XnEJuaf7)}gZc5;?k*W5MdftjVk>z{fnjZ)Z#|_Iwk~;=HFG zXqKFUL68u{o^nZt_N}L?d-#Yne=53PZ)j%7?h>L^KY{c>Fl&j_XsSn9W8?-uNHXS> z$CtsN!Qk!wlbISY%(Frq&dboFzpRClT`8w4*57Z;)=x;~ywdfGYabIRXW6kPvCbhu zsj8Eb@tun-C$>%FeQ8NIy&h4-W85an2x0r8&fPPs9-UuDxf#tcQOC!@>15q{GR4Wx zkR!O}W;NK?RM7#FK(FGfhil2d%M36s}pvMtAU< z-P4ah1uUXno$nNRmn%-qYbxuqnWq;;#gf=VT8xn-Og_xrpV=R2nUQyTNN5NQa+!;r zWGj$(IF!x48_QYdW)~xs;Dg$9glp3|`xxI^*Wv z`!xmCH`8*Y(=?^b)>oQ*9|w3I7!dANytDB126_AG9EtlDAvIMhqa#;xy8-}(~ z6vlmR=G*6@RLbWfa6f46U#oIn5A^24|T)764N0)Mu{{#B+?{Fw1h5^nyoSSiDTFOTOe*a5oVC6WCkHf^*nPC%RLhaq&X)D{r-ne8sH9k}P-ru{;4U~h33p~vzVxP( z4sFT0!=9p)BVRc_x5sppmAm^y&h9>=R}Wduu(CBx$k3}>nmjh%o|^-G6v`XW{&~d1 z&WDD7RK2+Gel~CHOiv=%F~DhVZ%lb-CQpu<9Kk&@YmKwsl9eOuEdWji3dGBRp!`g= z5N&m5F7RtK->G<`>v=uc>Rkd(ee&Z7nAm42Usmu67)o*;+skfR{<=EUaSiuo*0D6t zWpOV;>9NLAe%D^oxLYPc5NH!->8*1f{&41XQq^nM0!6=|9B|N@$ANM|_p2K(`YVBR zih#R%it~G1z6YCfsNII%JzGOlvnCmbJ0<;bfG*-k?GJhZ1_R>+JnCnC*l5pRkRWBE zsn2eltAv*Y^$c*B#H{mu!!3UcP}hD;iU!0w`U7u?79FLEFEHYjxnh?)7q79=bcOFk z6G^@BARLIaL~BSu@Ou&R1$D0Xihvk=M4hK2!NODH8n@X_Q$GGlCIqV>tK0kP8O8TZ zm`V12-u+sq_D28h=xK>sSO2i3e2b|H%Bi91YYWZ0nuHYbT8&0*2!13eNJq4B<%y!a zyG}v%`3K90-t1cFynB+o#~qgw-pouUXA8PsJuK}!Yn1GfS+V_ z%YM4t)`+P~y&Q_pii8gG($^w~8^n}k_km|#gw7GeP@fTN1G$WHYBN%!St`eM92?38 zQHRD!W&lxxhAX+ebN-;iLHP`b0+e-Xe|XcV9Cc~~HwXmE4!YC~CG$!xjq{a@GN4G1 zl(BVxw>YTC)_TbN|4m?%B(?!Mqdts5+BZoC#z#i z5@=oucl>^uX%kuMF<2=_Dljwq*19i+ntj+EjfFWsxGC!k(=9SgO2&iVpLb=0?AMw? z##+M-n-3#>r{`SV(RHSP!-UV46OCw*E=LZ}QNv77ho{2~6BmqjFrY%Dd7_!V$jm1C z8M>>pHNc-5LL)DXC)j^71^$?r4bo2Xz0C7Fe9!T4M*$qn?{ONs=<#;r4Zn`24(!ls zM@~*r^|Ld5i%E9n3zt2#tvw&m0jM&4Nb1MGqkEq%j8m)BZ+WD*J-dhZf?D7*mbe1dn1u?jaPXqD?+JW zruBqz{$pSgK7V#?$a=ULo;{q=VVQ;;o|k)`IspXy?_2FwLMdC%7;3U9ND^!1TvH-Zy%Q)Et>{9VoPQSbVV8(K(Z zxPh^8TsL)hbyJ=jwh62@Qkc|jZ7;82>Z}xFb98omY-Loqot)a&a1UD z_xN%%Wp?A(&+LFNRxYltJ)F>n=&`Xex48y6cX#*p1%fqE=Co=|u!4Jvtncf-jnxI>#%h#0WZ^q|KkNhOU`qU18}@0GdKolw&_Wa- z_utFJpYv-h*Fa7J|D#k08zINiiRAxM?!^CEwf~Wp(NYQ|O-Qdn?^V{(N0DMu$#~H{G3snUX;RtZdbUt{Kn4nT83o zc7>Oa(KXJkb(YoPGP%A>f||udLGfeqbuQb_GA_Pk5a!)=b{-7H&j1 z#E)^Og_lr*UNTQKwmJE)KM7iS89H3!96mlSokL4%9{~~VU2>_~X!(kVxo;@b$hkN~ z)BI-9jiAlf()kEST|ClXI2@gtE(R}p+A1;tJ`WcQhCt>f$~2 zOz;*y*iwBgCvfByV4`9I4>ZRrQm`^}hz z@`6GA^8Rkf-BxY0N?O*LTQ;k$XKpQ>`R=z5rM7E!n^h}Dn%s7J)}n*Ue3rOAgin>B z_H5|h-#?2M=Pm3ULVCZvGS^!G;Xwm`dND@Zpeq7&YJNU-?=o5CLwMtXkNYQy%R*IBgb2=b8W43z+={J=i@EQ_3pZQqh z+qzn7^l`Si)NEJG!uz-tfx~$LWv#x~e(>FU{g^iy>ODE}{4DjP=f%_hCHgRzk_IJ# z600~!x@D?Bykdg!s(Ao`*zQH%Z^)MW8Z$pxdtC7)u54NWx10vvS$M@Ir2_il;A}7s z*8ri@Q<%>Z<^tw%u2p``SHANpKb<)WXV4;d@Mo5wp?_`mOmaLK@??_L7Vq9s6?k5+EOb z7HRf_4`^TWvgLR)vu&d6w5)7%QoFh=b`9+X-ty#-T|NJEa#w*Bx$HZF!B!ka_NbKz zbR`S#6}<7^bPvR%#(;$)`EBZi{G^=5RRsMoL{>2*ho*o(>YL%qbW{- zl{*HZ_(7gGPqWO~9c)Q60FH{&{O+96HmF(bx?rs&pwHi>am@kEBCxn@i0)Jc`Dehf z;ksf==t2Q3x)=?Po?p0|ZN7b&M0i-L9^Ps+Y`ZK+4C|5$ps%lc-_vVahU1%2v)(e4DUh~nR*6e*fwYnMQ`QW#s5VV?K&Gi#rAvtAM!U0_47sdzF6CTdllxM z02#-nJpG$!pE@3IwA3k*ruHkcteN(|oo6=-G;skPARc3Ud8dQr zf)DQT&&za65_{>@g0EN2V|X0oRd|0)G{pTnncDu^ePg_(y0S96xLBmP zxR?xT-j5sm;dVo;7W-av5;YLp8Y|-^*8AHlvEAL>Z{y;!v$FWFUcE}OPkZo>i~jFP zYWy!c=AWD0e+f1I3g?#J0r2hW_QR5sW=19^F1kNtGjZJ9!om=MUokQNio0(E)NZc~ zYqV0?(5b8|0UySn3wH|%4&~}20qmDp^%Jr97+6?Xm?x(Y+c}~3UKSSP zI;@Tr%O1(NWKD0&-z7K-dMjfKJgg6H;b#(xvrSjhp64G{cbv6BXd*CNft1`Kzt!`#! zc8CPc%#y;w!ruD!I(-8FtLzZHI(HadU0PaCFfun!9vd_HCn0s>iHI7jgM-8Rk?^f8 zl!Mytj+t;wqkU~umff}Rry=UjW@b||P&BqL<%Q6XS!Vp67R&Y4yXe}W+@z}Wr zvPQ>i;aLh!gK~xHfc(aP+G;60O*IT%s*Xd3sMl{b=D&;emiDmyC!zMbo^8hao8F(m z7t5C2W?JYhr9b76X8K3Lxzc5mCNZgs(x29{mZ;LU8+RDH|6W44rwbj+oSZ$-8^NS{ zx2cM*wZm-6!%}(y(8Hl_jf{=6A2(A28uT=*4uTij8?U?`7@A47rtgop{A~P96tIBB zXS@eCf`UST@eJ*BfGaFJfld|v@rZs2Q&Ur0fK6moR!XPHLiH>xQa1OPvIDh`LZ>RU z{;@F8Z(|lvK;>7V&D5$4^(R{-%5iU*toDCaW^Qe*9H52|Pq((V-o{t;FO-UjaT_$r zT0Rf?=QGWrjXNeWS$`YPjT<*6bC0rify@5kTBWYiva;1;>_bz@eJQeqQESI9pVofy z_m=}$oU}U)#jRIk^ro?WR(3?n{@wva-hOb>@1N}DQrAkKUYifKm+Yh0rtP-BM{>Wv zUS)7A@I@E%)IZ4n*EKg0#x7iS?SF*i|9hzLJ=7`9n1g9;aJ}wMzWQeunyD*RWE>|?e{u^I^B{H~GL}V4>L^?8T_F$U7AT-A zv!(r=b}Jfc-e<3Ri2;d_SG%y4{tw64wqCP{SOj7Fiu@g%c>)}q(=Dh=y?ecvr*Fyp zJx_+xt6uEhX2|UFYNiL<+m7k=3*ogZNjp|0+X^m5dW7K*o5KRcwUU6~hkZi&IUg1< zEPQXe-=*chto#Auow^F3`aP)-%uG8)48|xNV^3Fmj6M0xFnX~M$J@!xy>WJL5hsaj z@=rowX32OE*C0#51L7>;ALWaGGPyR1SoP3&Q=aZr)WB_ZCb{)3%OL!hGkKckW2oL1 zM7=n3<5)}4#ezl%uaq-(=ZVCzD@HV(#oF$Y4uDs+}dcLDRt6~i5I5M4Miwnv(79R4*)=^ zcK8}@k*6oDsNYLpbo~L(;;RU`o#mj63TiTwysft$jg(-J!0&&yn0=tlmQn~=P?0@tv~eI$H0b}190uj!5x=KyyX>@+3>e7pg1 zJ7fVzCAPx+lR!`~w!dDHj{^12{-Mi{3buzxSgZ`6L@2%x%0w_X$<#)ElKWa?Kgc{S zi=>oDcZr8qK#h2#FEA=bK}&RpzS)EQs9C8&HbZ;E0(HN!jUob#3KaN6nyoBKbvnC) z7GfTe+huxo4T3id?e~sYG<)^F8=Y=0ir+CK60Pk=3*z!i5I4jc%dSCP+ZJ?OXKcP+}d9(`)NSSGc~>5Ghy}A z1FmQYO_@E!8CVD_NilU7Hb$Md4QL7*B5SA{ZW5m!^} zv}|wpd-PTH{nWS78oSf@KQy=$8Wu2xTVKP_uiCzGk)lO*i}Sj^M8Z5O><2~4%a(D_ zDyT=Wq_jA1dN$EFUE!~#i z<_s`LbJj^=9t;ADt&pcqnGG~DozB8ei$tQMz&X#CeHTy4)%Fh&Zr+Uvi`DPcs5U>9 zW&K8ZDS~S|mT^jrIR)IEW~$$t)|loxlV`M{b1_H1*}FJsi`eEL)Fv+HyL_aJ6Nt!+ zZ}f&AuitGF{;A*rPDwNG(a#|fsftPvp1Q^X)n+cG6>G0eacTgf{-Ni~NWp_3bv+=p zVb1Q%Grhi%ex9`8X+=}@0{wbHHm+eP4rMoxmkPg)gN)e^nu@t0&oA&cr$U?Kx2K!y zrAnyy@hR>f<{8Fse?LzjrbiiS#;uy_b#`eOX_+9o%chusKv4zTk3P(g zI>mFdT_ca=IxK#150yE=MIr|g#|j^hJ$hz1+w9eRxc9eujm25_MCkKwF=3OI6Ef-Z z)?Pc1m$)dj-LBlFP%d8=QCfHNQi}DOsPzIV9aEdF`C}e7ye8jc{&OWSpsudkhsf2! zf`|y-5&Cxku``8J78^-hjvOknZU)e>KwMnhA$SA=lK@8A?UEOd{Ruq&AEWXAS2@Ih z8i2+ATy6tjD9bA<8cF^ko6^%2RaHQ+^N{ceW_|=}mxpsB|Di~}`2cZh##CsjeCMNMf6d4urBIBnf-HJWq!|2FHN+hbFiAbe)YcXX&YHv%*iiX~j9&+8j zon>G1^!GELMq7*?3;Aj2W||ZkYd*;uM4Hg@^OX?MPe-1s zG+sJkda{rs_oYuVl)VO}_ChUYgBQ2f2!bHt=t-Q$?SG|5R$9i`BjLNWwSLG(KW&W^ zj5u}rw6T#?=vz~o;)asdH;S{^!RT_U_L(o`Uh14(o^9e zTncY#X>kfszaOpS-dUU8cAMkU@C(fkI;?j?)H6!&&01OE@%Sq>KbgRx$a#zS8IOOM zyo2s-uh zK+C-mjPFPJDEkk;G`R%QLn;>!=js>cUFR^l2dGx%QgX+*s#Y%gFeW%2XP4qSO25nZsebOU*!M+-VY7<%U*PRu^BLP>DKDZ2F~#CKmmXAplMFoG#r*#F zfBswZu=3lJOFx5Cp)h7|i?Zv%?P1r4oF_@^Pl0t_Ya^8KYxn3^$(N?Hy=<_8X0=tFFZQ zocbBh0P4}UhxNS;>(`8}?#LZ}@Q?C)nTqvJOKZV8(Gc2@r4AE&RP?h7I8WwMje23x zb!=9geodDPTyXE^ZoTFMZr*jn;X*Ozo>1FJ<`+gL@JvD;dv1@Ipvld1lnDs;ehryL zY3D$c+ZXK1Ud}%!-2X)9ayS8ykGM7UcTQBgGQ+ zPF2h5JU4VYhb`Sa*Qz5TZf-Y`quB&F{)RAa_o~$!9nPYm$mm^27OHfL1RY8ikS#PG z8gO-V9&|S}0&~p{cK#*PNNtfQhLi5hRO!{w<@$IpJRVb^JX+t{Az4&Z*w($bxHv{b z&Sv%@9wiX=rUm2LnhcT1d(-K!189@QF2*tN`zgtIp0RL(0k61PIYifR^k#1F4*6sy z?^OQySkpyo8G|CO4deA1ZQ}sd0rCz`U?e!ptfW}HTarMr6HP+3zg%hGO{54^QVaCL zXP#VsC!t&GAMb)H@Z7BxdjAcnJcd3C(!0L;eCxoz+$Ev9ZM+Nhss-URI(AmywYBM{ z`!AbNz)4lsm-lv*0=A~@q@}*>tKG~R>M4YEv>J1eiHzGLKEzKxFFa_2zx6c;zb$S0 zay+|WuiLViw(8Qj7ppPq{h1QTNF`b9^d>$$z95sMV~nXWJq@yWS9OE*xde1>WMJM< znm+i>fI9r>2s1jtx?no}kK%V~5t!q*1v$OK3H6{N7W)0-K!{1_4BqBZZBCy>f!Hzp z&ImdvfpEzxXkpmg5-*s0xlOPnJ$nQ3!6=-yIm-v>T3E9#4yU^=Z91@8SWhOdB!eDj ziCS%yf)&4V%omK_Kh2v$_Qx`w@!YguS&vg&e^akK0|6>noBHQAs zmzx3w_^rP@9ol0E2g{lKd%ZD;9W(oEMBVsHu$tFVW2A{{Hj6M#bHOR0&hl~?%0IvB z@_y$M?WeYCbRV* zpYP}jS_6E)2hwf|HhOnb$)Ni@tbr6352UHO4xg8FxRc*~mtv-&DT}E5Q*9lopSLvH zAuV8noqogN;5%rXEGNF7lw>EgmJv~vrti8CYO5tsK_?eKqJRTz-T1(M4=Q0Wm;KLs zczv0|DwN*!a$X<=|937xC68f3O&r|yWqY#mLT2&cf`xAa0pU|P6=B62FnVV8(W9`q z$sN4tY;$ml{FTQ4P`<}8GtHR0E?+%yfB7wTI#OH9r$>3>z1KoyXnrZ?t+ucrZxsu$ zpLu@+EBrQix7zq|%Qg9m_07;qpTPQ-m==v{*Y&SUQ@e~fi|!^xwK%Xfue0$mbNXmi zrtOU&O;=K|+p#;E9Vz3>(c{cFib~DF7}Qw*!J6TnVqMTk^7{H5$8q5ZcVrwnIe}Ej z_xScY@7mByB&a}I;p&YdHr|5prf6(t3-At*86FjJGGMZR#f`2EJrj>6n?{D{pcy{+`nT#puC0Pk04#>-=5MEx(6f zUaaWI1fCy5z@uE!8j)u&2YtOq5IR{Wp}!}R5|f|YxO`W)_SOi60`D$&ydET|WhDU* zHb7)){Ts5WAU}Fv=DGc>CrE@%+5MQ{8K$mPLx*Q-N1GZ>>^SD>EmnxfbzeFz_Ko=N zR3BJTNcrOxKy9JSPSb@cqg!T8vJ%4OE`H;=5-(10f_#Pu|6br3C48N?W4!Mm6z315 zwo9&ESC)FOQ;u{R5DVT)WDRz$YiA0l?Pzv9f<%YkEx<~U0)&htTRI;O4x)}y8}mf} zD9~OA7B@FWjD|nq@8f>_r&3f*(-ohgDER-<*wx_-QH|{WF|AlAf@XU_)(K6Cdd4Bji zbAjxM-Gl;{u`|XrzgT6_^n%ILxdk(Oof$;S`3qPn9ya4uUe1E6FDmH}U*}2mqQzuc zaOSSHe|14ofp}1XBS_r1ys@CYUd(wtw?g&bX0*K8X>Uv3zDgaQYLuUgb9wh_3Ii^b z-fi6}tx_V}xhwuluaRKKSh1@iRN34d--p(n+SS_9l7532LV1>W^bZH%O;>Wrx8B7g zz`1Ooxb08!jqr)iVDYclM1oPO&3+H#+v87MT}Wiq?ly8Pa+uaIJJJ5U+I7+}Wh6Pr z=1p_89>)5U(J#Us;5}z=`#Th2?nMjEuHquxn+ST!``7YbF5^hW*9{oGpH%nTDf7yC zM$$43QeCjLd{RAvaX0$Hx{JSSHnum=(&EhyF2;E@JW>&Lwr^dY=PyiiBPY)&HANu+ z5e_&EFCKsOK+tP3FF21$$NaiErSoXOqP2N3U*T&|fRR{$8=}WTNE$LJ3Ykg>FYqS* z0ZH1$A&GmIp|n3XKSXPQ2SGFCDz0<#S-Th78SAdd2X9AJ73Gbm3RL57=`#OYw)-!h zf{==nv(~ar_4O8qbZKpEn}mz8w7zYRRB{?*FURXA&=k15E#s)Gyn0=QEp|nwS0lxT(r&*B&wd zAA)2AlBDjbsy;b&>eTf!tsy7NYO+yp+BH=z`>?1N{4f|SAK=IU`_0u$8|sP`*dgPPt0Vb_^23w)1e7Su z*LRrX-=@N#nF&NOjz0+rJ9S#x@0-n8c8NPUB%CHS8N>t0h*7lveg2Fn(>EA?tc~{f^xc@2RV+ADZIRPZ_|R`d8Lia^63A zTzd3L2s|U(X+RwU5%>>7%$jE^yDlsoapJ*6mi(Kn_K#4I58oNOfpF{#ATn}Y*vQv3 zp{}m(_g`ZV9~p(ZR8`$%P6|z`eoT~Dw3M-tFw@k_co8@HzP^5%_RDHw$Z|Yx@Zb~c z7~-9_E#mo>={vi;k|U#yNnB>>>Ozmz>7wD8N%5a$2C`cNGIC$Ru-au)~< z-S{^sU{EO$x_&+59QwDX!a=u!_DaW47P)ZB?h3FpF8<8^1-klD#IER9imJm4&eHmz z_%I<!UoF(!dMfWa{ZUts35n0 zjb*)OyLkWDTv3n{eTVh!8hc1}}D9fyT85P?~d_} zd&an9+;g6PVB}fPs&mb`)^GjhT+@FcS+_^p1OE7r>3U#oi@Wv*um9cu`1QY86FFI> z%Mpw6|3AvQw!*E0FLNxqE7JqIf;68Vx$nooPeG{qHLU!#>_UyqH>VzHvYdB_OLZ6g zZE-7bSQp~yL7K726YCPv>bGLDFY#C_s3(}g)QVIk-f~|6RiB^Z^-C?=$an3Yq95m?e838+28XGmwka#gQnpV!(lg`=8M8W$v+ZcK&Gcr*m_YlGtq-o9Zdh*{ zCNaOjH}CIOrIwdn&yzg^ywrkFaNG-{ZK$6vBZLr+^S`70y|Z@!d&p`6JAq$21Lo4A zs5jPFiRvPP5zGnsre3EPP?I|*Wt(l~^>557e#%Pa*xAamT>t!N>I3NK)Qun!oqHBH zJsX!WUnBvu#YG?O8dk%cs?kB3n7$9vQ;7?NzkBePS-0nX@b~k?J&~2Z>mSR0l5(Fn z2^^KK1rg!<*`A9>WLX!iMB&X(L6~;#t+mmt~GOO z{PVH-`Mahp)3USY@7|9h;@o9KKxI6tQ8~e9sFZtD$58-bBgR!nF-y&P4MtmwFU!}T z1I&~S@oT=3{}Nd5iW6c~**HB>S#G3|=V^2QxD&9UtnTfy!*OF5J+E*aEA+p40!IxOzZG+T97<=BU1$_seRV@J& z;Fef4^yFSvt|?T9#S~DHbR%Yy>}We=+^Fv|{7oBHPrngt2x-s??Hb-I%h-fZ3Bk&~ z!~jF7I>2m%k?Q#p?ygM2?((ufY>JFTQA{~NX{~Sb84hh3^I&w7jx7G-mXU{^((%>TvQHS z!l?ViI=J_fWoJZku~QnycA%LLxtgi?NIwYxS$a1&^-PhVve;2CcF)p;-shEQrf?Fb zL)@#R%ecz9+Yo%|@&{Qs1AlhgUHk1$DlT8eXNfm_+>JlD(v@FB1<1};vGx@`drBfn zx*UzpWXB5^QghFifmuTmT%BOugQAm!rFBRL?%6>$O{bY_vd?dYHaJOy;p#0wtNtJ< zMu>m#3(o>MumO1FKg5HLH}8tSZi^!WgIcxyuEUUeL5XIdPX@I86taQWlF zUa}f{jNqWb49}gy`#8)&Q=id%ywC21nSPJ+Q2;EI*~t1T zR=UmY>g!Y0L9RHRepGJQBhF}%k+dP|0sX!CN&8*7b*m^QE*FR421@y|()LMF{V;6T zeR2arIn7KNh;BxSuFdY5Sx0O=o6*~_@E%mXY$LTPq+Q=oCMbd@ltX|4vh)hp`Kv=C zxb})6ycQd4Yw=XfzrsrQV+B#qU5j>{w3P1H(a}@@A(Ly!{89CwajYbP;P;lE~jDne)zS-pc-|Xo_?-%I`GR< zOjl52DfiA(KO;uSn`E(qxzCo51;W^k0&`Uh57{Ehu4h_Oh3I+t1T&yD8^2qb8{Oxg zW{Vi5)HhF(&<`slB5>tIxRGV3s^~s^Y@C83KKU>kZgu`{(n`1j#|@Q+TRl8b@9?5X zy3)ZxL4#Svo@=%VG;1B(jxu6mVwGyn1F^`tmEvok>K)M^y^q=X+e~3@8l_w8Ojk-h z5A8MR0VcYx?WU4rRm9Uy6(@^Do7a3Q0tnQ<9X!aB>*zMoNF49oajbrq}^ zuho58^SaDdu=S;`0M!xjRqh^oIBY*Y;@A7JGNN~gH?bXCk)%I6l&|l4V&*@DpY+w( zUk!~;DZ46}W3EDi5Kg^b?{4>`dE>1X4#aJ)t=i;eUVszQsiHbK8J2Hm3ct=2cv3{J zG?+!onU03Jp-0QTf_{b}!0#wjQ`W-7O5FWGpSc7c+HltMvH;$#!Jo2e#BH!=MtW(*||j zP6(M62u>YPlj&@FUExZ?*}EL~zpI~%MPvWofqa!yj8z1Rok4Nu@7`*FVw*`zNAfkH zJ2dsXpbpt7LNU$w?o2QNN>}vhzC-X{LFd-oAoh3jGsoj0$o2_dyuk$+VAUA$BDAs6 zSI-G>QbyXd6EA)6h}$SP?I}(DUcZq=SmTN$H}~E5x#4cK;}kL}8_UZZ1k|fk$fJTm zE^^dY_~(`682uqSuCEXFHw^fX$0QrNuep9FPSlxh7lX(+j9%i6gaTpxMQQ!}PGzxg z%5g-Ve&qLUIBtIhj;k%uPUYz}FXUb+vR_Ut#61y|7i8e>x6@YWoc_0K&6)ANaP!$+ zu<@H%;#*V6s;SD*49hU>md-ct@O?Pa@hfJiqqFD0C(95=h-VY&a8eEKt;bW|sfgF$ zNqDbwlv2vWs$cbMTXs?juL=Me4t9Pi(D2{ezDgj~3txH|(mP);o4h@q#vTt?D4dFR zOzgFyRwbi!5vTP^ykab)d+}pwN13y&ehPdu?Biuoh@)3#Uqb7$pMmdlgoF=@9w?;4 z6Vz2+KD92$cpORtKYrbjGj1yWz^azCmM|ldVJ?e(S*bzp@`Y? zeu(Er&~=-ZMemOB`7$RvgGY}_oqyURrA)+})aGm1MNhd96CaH3e~>jQmP&R`{J!l7 zZZoe0fpmOTf*l1u(H?DhN@e_qIDEaEbA|AQ=2d2W&?9|V1$`MM87j`ue@li>L;lTXOE}Hnr=n zHDnsc%$cqR>eldp0=3H-877;TpK*PCuG80f$QJrynjRTSR@Nww8{`O&F%F~~{f$=j zMHa8rVSBTEju64w+HE)?SLm7Qm*@K8AKazs`n>i%kj3Ldbx%ALUR+Cbfcvmc)T>1* z29tX;STyt$D-UK;kvMc**cfJTdr`gZP&X*$xvI{e_0+&7*M2q`7|>L$aDO6=eguGLw}#1ySukX;n=gw0hN}5+-nV$+#97$}RL?`F2E5TGOgY+h z8Z)x9Uvr=0;H3A0_uZ&pHWQ*Cs{_PR)FAexlcp4k%pdZ%RSf+4b&(pNujF^DSFb#; z`$%ZMRjstK9o%)kfnGK_=twC)uy*O%BW{+`^n0?!y~rd~v|HgdhQ8d{0=pFrBtc2QQY2ohgrziBsij`5_$t*^kj@}i??_jfXIJ7=mZ&J% zN!gBe!)>l*s*iBlnKC;P>E*U?Etq`DI7VL?uo6AhoRfe{8gkUq;i}koed_<9svSoA zvYEV|osum(&Ys^9348dTCZ+t>2$z<(F6AB>B!H3k;AA{YqXlPj?V^mpK?8cDxG5@9 zqiej1pITrPuEqVAZw*;bY!H``veSOZ%wp|F1^4AY+K=WuqydV<9-JGFPNkM9PFvbRC{4mo9OFx$pDCh$ zKXx>!d~fT1ubf`vBJHn>bsRdTeE(Z5zXgP;-~DQQ`>B^sE$7vZYvn6H1Uh!RB^z{} z{FMQD7tTedmt;H)Fbp(o;C=KDUAHcn@?UNly#=VC-7}*I!_) z|I2ULT3K5~bjxH^w44l;XJ%%GD?D&?aCq4@D2O8B5r!PMbKm}>XgiI^bd#rKc9Xf! zf>*2O;hP)RE}uc@8z-2N>Dc@7WUN--$mqq#2o||m#oJ^y5Sibl!*TnJz~s6ATLnxy zLP~mVejX+RENy2*raS-OzvSS_6sQumo7yj5<(|Q>Wa(Nz%l6l|U|Pz{v{x?Px$?M0 zv6J@YMQ)w~ERZcHFYf^@Ep2ICoo@Uj?UfmN!GBbAoqwusOYYe_B41lc$%{+xu}L3PS*Te4jFsmPzebMA)gG?U*Nm)m*dV^4hNTceb1LW&O+7{X#@0EE^dW$D+UXYaonr_BqSY3`ot=}diCnb*SwM!AvJ8*YmE+=$U~s_ zQ_i{UCJOQH`*Sa!Gk7}U*I#s3a(Q2kUp>6>C~Rohl8jB!>3(E8gROMAA99U;A8;I7 zSa3Q0hwRFM>i*!au)L0v{e$leLjv#=akL5lC+jVP-&QV!-t%3Sd~C=_xWj+Kv2sAm zGdOk(bEWo2uGMYlzbDl>3C1fW5>i)cbH@K6QRi6y_Yqkp%U_@V;sX3*O#UAz<^NUS zEfceb$zz9jPEC{2B#+n0F)kK4pWKcE)itQq!^!lK>t1=#$^2m6{JyPsoXm`)A#-`O z33*tLZ)?_*4IZEL`?&bEb(MnveQRH!#HviGZLBj;s}?}zYzM%FLoWI6@YR(y9beQ0 zTe!vUqpQI`DmK++w?jv#C^Su*1}t+H(AdFFEdb#+aXI6A2foDE&$m1d#L+b<9A@sWW zWrzsWcpvsy7_cAI6TY$${4w0HeoZ=)x?}i>Uz@F+mg5owOWhWI}w>qz%t3qQ9<5`B>_-*J64*E0(9JUWnPYoQD&Xp+fxU{7&DM2juU3oF!;UfKM;%HDeTt{XQn~wE zO|3&Hnx-1R)XuEk$6N&MALZ1k8qhj7*OOrWpuq~}soCx^5Ddb%I!iRgFv7ifMGHi1;UTu(NIopehz31C6GAQ>^ps1`kUBfoX zg^V-sBWmQOe17Ubq_{&YWr<2-bn(%e{O$MamR$BTDGKRS>a^bzEdt%p+#Rmx<6td_W>4d$j!h~tE_g~@~~@_ zEFj4>b$!ht{_IuN@Dgkl+i~-8e_~C}ooPH-gHk-ZD>I}X<=n1sQ*>Pf!JwWzF1Uog zI##`N0N(K`IBavWP`6|3cwhr|Zp*%7e|2LZ(yO(R%au6Qs#}yb%op-TY`lnBvmxjz z%Bb}2^j=L;)F}%_%FLsq51g&fQf1AO+&E&qNL0#dQztqXPrK=s5z-4ZIEswDq3@cN z)j+#tC`1729o&SfXRrt?Fq-O2W^fkSaihD{or zxSOf2v;44&sKzW^wpSA?l>1aSsZ?1iN*1TG#wOaxyJJ-Bg8zhE`ne{iF;*lZf#+bk zX9qMf&eQHxw{^=`oMjF-}gQ;R_U%u4Nc{(|@_{zL0F)>Rd3tQ8Bt^u$9WF6S4 z&(`_VxqH3B7B-)Y#P|HO4*X*eT302L`p6C}999m_7fncPfx+c{^P zMsgbfJl}uXGLZ>A!5WJ&)n)?>MSD%dG{r=#&+n8stT(Lg4Qf@Z%mP%41Y?3qT-raC zu4!#KWgu2{rqC@>W9RqDQ8-3SEo|s{J&s#%k#B*J4n1zg*-cRI1w=Za>b+uIw7j*F zim0<2HH$kaUU;C9#)7&C>jE_97>fSk!gT+*aTAQrtnN&WfoQk74;9RV=kEnfgIQOo zRM0Plx9obceFi<)+1q&Js!>qALAkkLhn^<>NZZe>(y>pmntso?FM4CaC4j>`vvmY% zWk>acb5ziUap)j$1v9_=k}DxfvCsqxJx1$U%aA&yuwFt$$rH_}B$9=Au#+lFZ>&43 z?vDb{QlUZlcFyG(bZxOt6>Pf(V%2{zx6Kyz+n5;n>p&3^*07=qn?28fo&nPX2kGhU z9ijA6)Ssu;aYpAp(;r$Gqpy9S7oDAjA{viU;suXUFTMN7%m2C!UNDO9*0|i?pNyoiTsJcsh?YDRfXU!Rr^j0=)%0EUvS> zDaJZYGF5{QxBU(MlyqD9->_mn5}1(r*u~*Vl)5fdDy_a-WXPS0dt+YQ&rLjFG1F_r zf6eD>elFlXt;;-))hv}`+oFuol2ty|TSlCP#kc5>gY4xILp=z~oFlxJ9(#kIA=&qW z{h}URJc7OnX6__F=?;z$E}uTt2cxi#{Z6#47sbBA%Cuy##!pSGvl$t9jg34N zPHPOUn)%0xrq-sXaU7J>RGKk3n(bS$P@X1~iD~HR>QA9S4^`wVV14gXor@QJ>NjXm z2p|vsR!QOl_EEAB+1qRQ%#L!xKVie0`BF)-GH(_R2RR#N95OST5`U%(i-;yCj*q0} zPit=56ya;fBU8spZT$s#Z;DMu%p;f4p;l2DZrkL57v(wM6&p?Yd>%`*4TQ1?rEt$LUPZi|-wma#y$M)T*?l z)6T_A>&l*UNaBR9Y4$fOx2(2uM?`KPv|cH~0`xnB~LcfUNvgkL6if8@#GnV(yR_Qipc4 zF}HIbDQR&#woe9w3!P#|XDomd(ARSY6NynratqDppDadJu_W48|2$Mk-Po9CG^L zrJhjqY^dE#ClgP^zKm?$FsYJW__|59>0JzZ!}Wp_jMFVT*{dM4;!KpWb1YwInn?kO^cI5&TnfJRvi{q>k$g|$85u2{RX@(3kISDkQoybO@1r>EIt)vv7m8c#FNDbGS${h}*^V zhd@;G(bQbV#~BEt%LwGxWVLqXkWS{U9NSN97^A32$fGWEG88Zz zLR*7dTJ`A*M{$NbH3##&PGD>{(foMgKwh}Bb9Bl9Rh4F048U(nwG{4j6<1HBA}&aP zRvC|%Aft&eBe^UBe`nvVT9ymN{uN# zw6PO+D)EsS{Yk!~Yi6dq!rEYtNp>~u&a;nGN52R&4NnPQyiW3_>$b}6MwKQqK*CZ@ z-S{|Sl%gN2VSvfw<95p_+Hx2FD6PU%*Pv-d*12qVzTslZYVH;bCyZ!O?A+nBoZV$f z^Gd}a!&&QT_UQBHa%9uc+<_XvIID z@ym&GQJz>tx?RV{x^=l0GJCu3tVrc)I!aWh+p`m9)cb9r4_&0~aaQg0mPw%^%}$@q zH8b>lNK>g*kiN@VE3TaxO&{Wwe2T58JzFk+OR!aV!^*qD{}B4KX*Oyyz+1KBrAI$CKWe*6I8c=S2BA1*Bv*Ymk58 zUX@fQckz5BH!#(VbY%I;neg^2l|w5U6|^ zrA+S%wCOm%Vggv{R0o+Mg`t`pm(jmR#jL17G9ybzfu>VC>D`<83tZ|8RPBpd`;x`Q z9a()HtH9DUBTk9pm{+!mDp?_%q}EsD_zyS`u4~_`a?ogvcw2Ux4@z9&8v$my+}9%8 zyH9&L-%~aFd=Ha3=*0AaQ-QjkWKl`Ds8C=$oJKKE}#6)t$7v$vR-KXMSB>C;EC@pKpQXM)uv>_Z>;e?ldkUORn{c zjlLA&rGjPgxsE!G)P^ROO}ot$IK(N#&;opDNltQI=1ui2}@VspRhmS5#R*yIn zJk%g))`~qoWV@xt^wP!L*Wx>gt6pO$z=LdSx7p8Tz0+{%Xzt$jB&wO}55Y{?DdPl#J$-zf^zcQwAdl|zL$3zdlz6Lx8>cTeo|QplWq)6y@b%`x~a zA%lV|ehS^)atD*ZGc1F)BZzxpprGL>%M{`&+_9<=(qaL=Tn95xjY>=2 zKqc&F3J3#u<>0kKm*w(;#9s>29(Y|x`@jrYaHd4RW|~N( zcm8Z)!|md3_{<(cOsD8lv?Omlx0(NZA7|+{fU}iO5vl8>2MY&Lr;U%SY`fZNmd(%q z7W=xzHPf6tXRL*d85uv~O{AEOo#TE>d=net`+nJ3dthVP z_&Kkd;@)Nd=UQe?0Zu37r(xMUJaF+$@iMN?{#F(@%5&$4rneMF)gtRj2BEo2mKx8M zANo%uSji1%B}mU|Al#vqP3DO#0s;Ej}V9X2t1ykKy^G$QXw7Ql1 zS%o@I=FzUrhuuJz6m|+X)mu^+_PTy`Y=B?$-S~9$G-$5Kx=h`zJ1?_W^;l^#CI5)& zw>dPE95!6ge<8)-KC8G-P5KudWLrj)l>b3vH&#ZIx!T-eA*UdvP0XbuUav_z0@9|g zmR{J~!mFPdtBQ6c@2)}hErTU3@D}T;XcsZ0Mr3D?j))8*W|Ww$vHHQJG*`$!IwdFa zvVVMHX-AB$zuWOiT$CQNp?_E|?B#U8$7~NRPgjIrmB-4C3wm?rfUnr%O=8{AWw-B^ z+GWDKQoh2DwT`e7B~i!YZ~Z#s+|oKtq1MJ=EpL!}p)_?1s+4(VM%SG$nUZk650YHr z$~jx5xN6c`FN8B7h`M<_GzS(#2Ukxl6QlfCj(_{~6})tq^4Ybqb1eK2s$hX!!kP#R zB<)M+=cLD-=jR3z0eGx5n6)d5b2;O4K_>R~Y4h|5&;q#$vL9;5}qgnK}B#|(|Eo$rYPv=E!A5$el$s0AH0v0A~$vJB+z>fSNl%sSO z>HGXLHt>_xtgp54bCFi%$5lhA&W0rP1ToXpKMM+ZF0kTEs*aF(dUqO+PWSgd;WZBM zth%6b0b=znqRkxq0NHGEK9H0Yad}}&rI~ZqTGaL}@mr!MeRI(-)GQzn6#eXvbFskJ zwd=IKOm$x6TakC8JSu-c*!k_{!Df5;-16Tp(o^jt9=&V zq>obACiC)7Kb*Q5M&xveU`X2OUr8<~N3aBI^qy?HJp?KLF8*XGK0e+vG_r-cuVWzI zxYi!CgWj;8;JO+tB-r-;%J_7-?`oNG^QdMMpwH1s371+pxQuu!-RM_%>#A$JO5$(+ z;rNZg6(nryYUe;N*lezzlpFCM zQBE*$9+rWs8z8gD;FLJ|pkZNNt=Fu!^zu!sRcELudNvwketDPxII?kO9coGhH-$QP z_SwCS2>{Xz%kN<)`z0->gxCZ0##gLZLZ3L)VhjXG$7Jg`aWWjuDe7lqLWg>NPOXHu zlpu^t|Im>+`yhB00y8~LE@1ot z)6zx!j_|;rus$Qu=k&4U{+vo&!Fm^@r_fbwWXP{H!~vdqN^&Cr-cNgpZ6EO>@P<3; z)N4ngVsBhDHLQzO#H+NtWgjffzZ#a$+EGG*G*ryB5kXnYvNoyYA<@JkbDh8(p&GC# z*6X^}8Mt|T7mXg?YF+Q3I<AZiz1-o+Y_KcRZb=@OTo zV{WfF--N5E<8WC&+!$Co0Ih@E)3CFzc9?(OAtWOfgdCS~^gNZp$zZH9Inee;Geg>~ z&|;=vu6ov(taUr{3r3xdLnzxP0?%P5KYwTBw`RUDUulkPlVtdmogAHAH`11DQT<6R zv(`+;yykN%S^;03YeBOh&h1t1|7gJ^N&MN+{^D@pf>}WYVpvSV#;>8*bZ%AX#aaay zVQom%BAAEQ2`14blw|8_e3D(qGC91h<8(e@>jv#HCpnyx!R*snrNG9$ z&&c4D&+o9d#GN&ns%P&HB@fkTL>`2YVWI;; zn7@3BzBfRoF2v^un9$Q8jA@K(U+~N1>ZmfT!C1j5x|J4Qw#tdeJOfmd3iIA&VU+ZX zjyeywS)l7)FpfzYAE`Ezh`Tt_xmTN=rPJPPsI;`!OxWQLD;Jubqw$n-7zmi1F7`Cu zkGDt|@*B%USp~|nb;{anTv*MJ`bj~(2yV9z*lQ!B)?t@yQoKZP=K?mQ{G0NXLRS#>@(^(0 zZWHP?>$_v(?|L%$V|m#%X62sDYl^jpxMxv>v^S6{*d}41+dtF&=}I8FDqwA<rVR&tRadB(&!ewAK8SR>3`+LtmUaywsvkZ`ee%*^6|VrMj`O??xA8eQnWnGOFp$7l1xWWUm`?Nv>@ps7e_UR^DG`RJ+oe%C4*qo;lceVi~0GA@EZol z@D`KG#kc^5*#eh-!`W`!G~_}zT25klzVe5y*RqBPj z(kPluhpp|?!}DXm`+A&MHq}uoy{qxhWOHgEp?y9UmHW^G4~YFF6W$cXf;2sj%4G=% z0O+R@1W$iY5B!|_SU2(s>smbG+1Kh$7Fj) zQ?2Y(zDx+dilnMmH?=qsxHnNd;Q!@7+iZahaGRB*@~E$IYL*$UoSiqUib^3?vq%gH zPO=;eWUUXWYQWyXzo{aYnWr9hK#q6&H|tMFj+?pm);)fYvyQBG=8MD21Uq|E_a~@T z#XtVGi|h~jQ{Sg%GtO^6M7fEr0C=0!E7pA9(`KZ!vZgvQ4&mnG%OHTyD-y;w9X{TP zdl8*+t(6?a9m=YREYuH6zF|B7_l9#ai$QliZm;?_OghdtaQ>TO7QUJV3lm{mM7n2z zU-^tGWT(0hdr|ItvwS}tSzI8!@LIh3on*5bbneTldPWvolv2J@m{BgX^Ygws)M$CP zP5TY@Z$^&n5ve~emd6RZQu+g$iA%RqaM22bA08##{r-Z5kAH_Nm&qx8Z+PaWa|UPa ztvuSw2TS#tH8%1=9)PtY?cy@6j%@!8@n(Vk_|hs#HlCuCSHV+)<7hsze2)% zu_TApM3CQn_rCwv=k$NaLHY5wOO@Q^zlkuyf4euv{vBrJE#=>5_OJh@$*}#yH}bz_ zeQ+{S_}{ny|8p^xAx94&qV7R~7}D_yuxj8eE}7-tpY?s4xV|~El1{!K4~y8o=lMpI zv`S8sF01x%s}9i(%@jOA528Q)$;h}H^Ji!nqgQ~FS>oOh7qIh!j{jMDu6&+CJm4l` z&g2kb$1HS1HA9dWH_0M-djtV(S` z$@llDX{n(_G}q~Iu5bB%RCWiwY}HKXN~Js^f-6X8CC4y`1yLc{YwETRE&hYPiqsx4 zmvx7HXg2Xu+Ai~J*l7R{#LPPi)RTKk!W=%%Tr@c%mvwHw=Fh?Zs#?25zPhrqH0Yk8 zZ;6@WgYtoGWxydD;N(}Z@2WBkMBKgC^EOy{dG}PH9o=;bmS8FMN5aTRkMDA-A)rKJ zja=@%kgu`S2(>@>8d;*704YHTS=X?z8>cr}A-k&5G_=!6i!2XUBgCouaxpL6hJ$!l zVWy!Z=+UoFd6-)Wweh{FB`Y3_XnaC*q9EiLq~f3b$xvq+KZ1ao%8-5D-b#J`*YLmG ziDr!~d`}%d%o?uj=;AZh9mw{L-R*Gi=-fvTz=B@dO>|N4lJ!wNmjX$K-^A@MxD zAwsaydv8AhGw1x-p#2b<{h8uJhU}e~$+;?aQ=VnwX+ni~4fYMC>x)&Gwovg}Fqf*% zM)R>LqWl)$?ZOd&u?OnRDe0Fc`@hB}|WwS#8m0g7_O=fq`yI%6iil}h|qb4quJkP{3A>t|5; z%NA7yTai7LUVV6HqcZ#A7LH3x#xHO;Hr;oaq zXJ7omG8ftqBy%Rd^SAyzobFg;s)0^(HP=;BiBr%X+2nr~Zy9IE z)5NTjUH5AD$HLzBsew8=H-*n;r8pPtLYtg;;&lV+5!U$*lCZFcEKmDmI z9?SX(cOE6cgZynD89!X^TnzK&4L5zmROVB0c>l!Dd@cMl3soPC-3P^gtJgO)EASz949Y2_3d!1$rNzrxB4dFOZ+AB z-wKXG{RccUG}VE-&rYn}8b01Q)vnq9F0!Y(+Ia8Q%bD9Nh~?d>Ni|dIK5|e>-#al! zOT4SfzH5mlyxI!$MB=G2b}=mMaBS!BTL(4)|C5)sTQ=x)XB-MW>N4CcXflv=G*?sX z^jweYI=#zEdK2t_=A-CSQhl;9D0s@p2=~~TZ#hhd9x>9BmH*dmSPK@CgKM=S?J2p z+Y01V=UiF;v^l@1@I$PSo2;bAU5w~Ig&dLmiv0R3GYa}&Y|OW7|GU=btOWA&KUM-5A65*PSfCsMvnw$rBO8D)qAz3X!Xo9$GeDddBhEBbVoAO^Pzs|d>&y2 za#d+52qZ^*Lal-a*N<9kbf9;0sRSI;42@Wgjb-5jBQ}y#^$CrNOo-%qfWUP77#>ly z+Mx~ZR&5>W!0hycP;Rj{FqQ*6xutT&7mfK6({+X5Yr&ZDy?<)DD+@H%iZ8yj}`{}{x6H? zr)MrO6?c64-}i{o(6q#*M23ZZo0+ZbTFi)=4WpaqXnr5k$aC4^N1t31b^7FzYKPV; z`|0Fz(HrjJ+0K@fj+(ExWXHDamOCf*zkAtnNQ_XUJmkxyzqJ;m z{*L9XTQ!a-65S%5kl(ETnJPK=W!$LWN~m&<<|uF}sIA3v<7dBFWCs7q>A^cQb)|00 z1z^uZXoi%SIl%`(nhm%?KGZ-cVU;4;Jz+`GK;-$t(D70~3WF{ea+%P-DHHssPlbs#dU8!CVaP5<jO7+d1j=(1DLGCU^cY zkf6H38X@smXG~CU3buJt3A?9I!}2*(yiz*q#)RQZO$_>J=2BBS%ueea zIAwwC8r|y3P9!@!IkEY*H;OPIq%&n=L$1=%VhMDPDS3Q4Hn;kx$&Lm>7e=jj(xF1E zvVl#F{Q!}NpqL*?KB`Q2VJZa%F*ZQ?9Gn)F(igJzPGme7OZ6eJY!^>(R4+cw3%YDk zXF&Wht_))h;w5AaoWw|C>)q#h+lD!A(9fw4;xk=hys((YSDDT!Qx)5tDlLs;MJTX1 zeMFq(c4IG5D+y!JUJN?j*i59*UGb1yh{8OShb|mgJdn?-pOFpK9o{Wq=$RO~3clxay>K=Ipq0CCD~o4;4`4q@+&yN`2I>=hcD`z> z&l~Qw@U93w0WcL~j=p#0p=jcy9^cLfRtJuX(zBQ1X<0n=nFK|jeinq_*7PEGSiu&L{4LQo&3|e0bl2Z?jGIq zSF6@AkD)V~c4U}SSGxyD*Z3&xA{u1H$r%s4!xbZA{XqP-JXgSjQ$3I|b97MxS7~U? z(SHAI?z}UsYupn;TV3b7SaHp7l*amK?Dvd^kQap0$_cdM)Glm<7ir+axZk36@g>Y? zA29I?IGFut(wVIy&z=*UBz1AbMvx1DSm?xCQBud{Rml%a9QxlyGBQ%tUhWY+nDiE$2+1YwM)i>`UCEGm$cIj3H}fA)T@W{>DtRZlgU0 zVO*l|oFCC!_eA|j?3xI&*vi%1>#YWW)>8N&kPrUBV8X3ngU+a!Xrp2}AqKD5){99E zap6&77oJQgB5Ea|oFqEA5)#210mUd?A+I_gQmZ3u&$`cQ(-d3XQ7F{r$eUFFBI^!X z%a`;eDG@bEI}gji40VPN2#tXa$JXXa?Uf>&Pt=?y#4^*Es{MAn=RBjH&<2M&O)m@_ zXH#0+ba-TJFH6jPLAZJfzO>_WN-By(ykfuuwscr6{ul240nT6pL)_lYqkA*CW4ya&JOb5#A)IWHt(IVyCYKfpe+*2H8otFX#m93;@Y$?4^d7Gn~I_l zau>7OwckWzH*Nj%ic>)RpIwXz5z~9md%U)pwsJZrOHNbA#c8Tp)2m(p70|lYDw!6M zt&F#(fiZi$w9^b7HAs7`eg$08#Je(NMXRq6A!eFhQYVd`3zGC&)EC`!&YpU zE1kll0JVt(vymZ0cOmaBF!0nd6HuTO@M8F7YSp4f@GT+wFL(qO898ITLhzJOOgaAl0{RQIY?np43WI$gFpShpP7D~Gv*rd(|Ww3HOI4hjgR zig;h9B~fb*{6k%xDJ7CDKzm2; z!iCCVl8b~Cb8mevWvkk)49v8#=-uQ^InY)=k{zKlI%1|SBYQhx>|Q#wp={9Fe5$>& zBEfh(?G;};gqxE+57}vgbZycY>6vs&xJ8SH_wXCmE}TAL-2S zxt+o^7t{wD+?~xZa?WJy3EmUWtlpYQMEj{>iD!Emqm7Hk^JBJF50d#j{H+^Lo*B+| zZM@gVV->oce8t%nKk2vOUa>H7los&H0WtmW?>;F(7s3|wJ}Y$L3R1HfHAzO79gsdAwA1PO%|VJ z`MDTbmssE}AkiNfaa62xF_KaFs(c~SdpAIQa@e&zCV&a4U^DPT;NNWaLRY) zod720Bb5_uzLldKif)(gX-n!7;#E?cQLY_MQDFfAd9O6iR(1V4>xo`f zepwAkNSKY$3&Re~6bOhtp4KG%Fxd1kck<8UdQcR@TrOJ*gt5eX=ruOj+DzxLmO2By zP|63rYs^D%r-12c9bySnc|zb;`}UpcI+AF;{*EYIBUdXcGA==6wr`qqddelLFRsGc zQjw)8v5~>0{?%Nm;WQ)u{x)8g|KPs3ops|H-KYaI2>er^RP?2VT}2sKYo0*F{~9we z%<^Dk3W&`OD5-*cG1gRF4gCt2kBP#)(mT z)}jYQXusW{Rni`4zQHb4HFE0~dCpJ1`MJXqv$xPf7gGX9b)wQ5$0)HYs#6F2uEa;` zm%Z$MI!HrA1@X4qw&WEE-Lb?ycDsbiYR4@nS@W#`UjB`lv`F^2ZI)bDV^I}Tj96Ok zwwX#}Wog{UwH2Ec5*K`19*`cju^usm_l8C{iP;vzm)?Enxjg4|AeGRcM4|t5X4yy@ zm<86KyW!FK=leUw+rR;5{|aVSkKVcV==fN4O>x z+2Ys5rBF}<5*}Vigq-c}DxBZ*?Qryeum{d&)q4odjX#5@-*I`OQfEnxYmHxYb}*`a zBEXKF$TkU=5Oov7t59$`9ps3hx*gOEdcIyiq6=I8{r$ynS4#0ksrHR-NWzFE| z;wXZEfPzRTN|oLTEkuU_lqw*-2?$6F(n3v8>0NpWBvJx|05J&=l8|t27@hB&bDit@ z-uL~^Ip2NV`~f$2*?X_O_WG^$TYGf}j?*7$kc>D*2fOXBDO#lo0yN;dxAsr$*~Zfv z;-@@R`!I69UaTKIoX?3FO`yGwTqH zt&l_Y{y4GR>L(K_FBrPLRcX*DSf2x_bf;g64F^plWhCaJRN zOIb;%xM;8~(VH}$Sc=lsRr^p}y@y=rlN}jK@7cajFrflVXSEZbOGtc|y}K)%vVU8* zzs=4rXgftgKo81C#XtmYDKC1)P?~W<4cp&|M0% zNg*C@nfs~5^Hv^;74QUY=K9xzUE$?{6|7_10AyU|196SFqzbEGIUATvT;D8yrpUs# zrxsX<{mEippWO(Q zMIs#wC)r-cyi$J)j$48&ncCkjkoJ`7FrWRToVJRT($AVZJZqw0`*?C~b{&0fa#guy zSKTY~yV)V|2iL&hJax1e>nl_9uXektkaZPqlaXnw4P+Q}2SBNjAlR1sF1KZc-IKK^ zyXcwX=pmH-CjpCwH-H55=n*bL{lrRO(muRa{joS6!F(}wtz1=T zF-5J5_NZjLV=Jc4W5FUdZKs+F2dlqzlqxYaOjQqK?kwY?+Mh_sQ;f`Gdir~dvBo1q?j#oxkw7z$Tfi6XwO|d%ruqGaI1)K+Bcyw;;q}q%AP<#gD1oN> zPt!Z0j0#rcZ3HT+NLs{xh586(7+zqL3xC;nGSx-AQlqAB*;JurQUu?m!QMRoteiPf z<%os2`m(Y0RG-1-6)z0tNF0VeJ>MuAGWD)Iy-rBRQU;lzp~;zR5IWMSCB@yGkT~vs z#+}rx?J_W?on4=MThQ#`w7pbG-iWbu){dqH{dLxIsFyniW0uRrn`FW&Lhbhs7#+;4 zt!D3Bti9?FZ?HUv=apNZ(IUKaaC4Y2Vq^YitN$Rkq0SoyfTf^Bwe8c=85SGa|MZAs{M`x7`G z0m2i_t-v|$aEUAF<@tQk%R6O@N^sQxPQcJRWa`U2n$tl_M;XO;ecdLp(6H2O_n$_e z=A9E*xLDWfmsDYLSldt?xhd58Ab}N~o2W8!^gZr)dVVG3-WCyZ1iUed z5A&G(_DN7zV)rKX`8bgj8gs3EcwC0}PH|27QeM%%J|%Z(`wx);dJ}s6kd#I z^T?W#uwQIZuc10f`Lb-xpKF{g(Vq@fE1=H2bX;9=2jq*-!Xmj(sWC+Mg^d zB>>mj!2|2LF(q@mBfvL2mEU=?c4xAHO)}}&aZ)q*b2YF6oZ#w>1fyT|2!!6(mN*G# z{pm#0Ru}-cXm;er`>LOBXVD!L?0TGj;?lPhk;;uXmeT zM3aw_;hM*BH1XpSy+0p$KpLqnP+uF)jb#$T&9sj`5I8qDXL%05-aiMFbvf>yM|o}0}<^yUyX_ztCKz^v7*M;qPLPzwv~n~yoDW7J@0 z!{#jq;qVM>{h$8H6k4z9`}~SgPp@hEuaIA=8eX^&eTf?6KuMu?j(Gbv)Dzyz+(z2d!MK%<*$(%*%;JV>W~NS$*s*VtO&i=F4<+RU=tcL&oqc`4edb4_L&PEd}T3SCQ9 z%46p|C3ekt1aYT=?JUv-N6U5-r7cf@;3zN}XnE-Jpa=u&5iW`gJsw~ndThtO5VGuj ziCV$oLo8mhr+tnXlw8R8N^5)1 z@Mg>&%{Fd!H}370^rYQT>CP?$Ygn@3K-u=%n&^4%0AuO~QR!0j(o*DPG1P085)`N9 zmU3${7*skWVcJ|%d$6WM^wbKoziMkzrGUO2;<`8MVa4B^E~Pajuy)i-8?b%cmEX)e z*QiQi4juc}82Z=%%6~Pp!)OdWw1iAF{Mt zMhD%Lx{0%{$p_C8wXYlo=eil?)~P|t)QjBcc65Qq%%@rAFKrqMi^?aUUss`LC> z3odw^{v^G$XmiFau_uk`u;o}N|APzAygMM>4dWhADevAF2ZIw0`o+L_+T(i<6 zdu*=z){^D*AzU~jSY5pG(H9#09F}q;lDru&7r!l7L256|Np7(WAG$aPULbi;_N#VM zIjQLFjGP(>n9daO17*EyAEo>iLXq-FRJ9;PWbJZn?bJ!4V-wgNG=_M(xtB+YMnNdEvz7eQYFajY`abJ|RB6Jw;`8 za=FG4lR0CmVpNOOmB&ak7K9Re6ca=T*g)z6R(l8@6b&S?X_S{kKAHg`bfS4Sjnl=Y z`P-7)(a1s=?5Qq@13Pzm1xFtjM_E z_cE?sg`HbEj#K}J0Q!Q@3P!rCO2>plA|DjfD6O-pf_`H0?eQ=lAOg&n`L<2}8qvBS zu;K}of#;fA^-|dAz_f0c-gFb3h3C68s`Wqe#1VsQ!a1Q;p@4Ek@-GeMwOXN7mZA`S8o1#)O5NMfp@g4zp3YAgqcaO7p@#fmCSRQJTw z|B@r_h?4^ZsB%Oi9atm1b{s$f-@M=LKkK5jqQnbl$D@K_B_a`YX0K(ohhiZS!adf zQui7=KE?N%d*%h==w{VRw!T>=xhM*9B-SSyxy3CciYt0pLmf1U-jyIO?wu#Iwu=>l z3;eBDmIkY1+ZW1r2gY`Zou62QY$2`G>YlND{WN;D+8T|p8S_$(C}_4ue!nqI?sHzN zuDFJX|HIdY9XsIX6{K`0>xx|mav#URP7t6@$M~CirhZUSIGkpwfQMZ-Y&q~)V5zFg zqkK}@4-q@J-Pbikv|+E#>;IIC5Dc;nR`+?YDITDP7DZqpp64`51d`Hx|&vj)gnNnlPhI7o@l;wsErs6L_`B~>@u&F#Kas#)*{t+FGvwz|T#ao5P_-%M&4EY@QW54$LOR0M^GE z8^a5VZNPy@A{-5^NRx0xMd*~a)*`oUyxLAWNFORT#Wa1S`Gy(m^@i~ob_4fFxh5%W zkD}P0E-V$QWYq>^Szh$2_Ea`t?|StakcAwf!l_?wEj}gQVL3!HBc>jYSh+Q%tP~L9 z58bQlQP&f@16rm|QdD-4@pzaG_Z7Vq0fUV@^K}Zy(Z}dV)6Hqs#V<6Oi-@QcxyZhh z9S88@VmPB1=XS53IEYk88}q!eIBS{z;9<0t`$;^-_GK)0@*$EOxQyN4wl)@&YUhcz zYKtO)K68tvi4C-WF+$LzHJ#oI?Iy6ZQ`{4xO?q(JDA0O%TkRI-hB3K5tzCLq89((Y z#>eh5y==$gko4)L5{J?Bd`U$e5zM~rY6*)D@cmHlYPdVaJ3FWE=26wR3js6>Q@D3? z*V5|hP0LUJKBZQ9dU&(V#^3LZBXg|d0>}w*uI&E!n&MK?3}Z&z|CxTG%hjA}Mxt4w&?i^<{44VgiXl_|v z*5q+JEJS0_1FvD9?|dtbwTcQ0zm7&8JW)HC-}qI-E14Bn9FwlMG2MdYAePYt*w67F z+_hF?R`*4GQ<(VO_j+?qmBs~uxxg+X zD~kkLo^>4m;WhZCIJ}rePD&IiPO6^B{&v6N3!MI!0_tNFgg6kPXopII#=9j-4aIQb zDa4XR_@_gudX}!z_|_@I(gfX$04D3d)LNo>ALx7;flgb!K$z= z8$nR$;Z_jk)q)KCvYpvx{9#jz^wimz`&G=T@(TW(@=>i0 zyZhIK*Y@ojtBV&Mw5T17{eY(~__R{8;->f}h8g^{eOCxYU+yNdPU_E;>ci^|?iTG$ z>*jw6=eu|KCzO%AQw-?BvB=)LKC*#c?o>!N#I&RDoz=ySdnxVCXZ6f4GlHhxCf#2P ztET^SntzAwc*caXw)dk0SGs1V4j+~uTK_~jn>*1VY)&YMXn&T|ZTNGRtSw09;}Y=e zHIz#EL5Or0DqSd$zuL5C!~M>ep}?An<;B|;>nBzc==};kD^4%YFG!7a{a%mTXd0i( zm_yJ9+$onU+8N$z7reM1!}mKvYw)V}?CiuwBdwe)1Sh+oV;kj2BYh2g+WPvg{NXL{ z$luJ}gPK;6GpsHQu;l`%Iahvi=1@&`v&Kfw3&sgQO|8!6)3o-awoMLv!h{@dsf2b) z*S54z89a7*pK9AklyZm#V8&$psEzL?Eui-78dYAN@%uG8HHX? zo$BFW5sU*Ts@aA^jx16RR`NZyWB9aVducYjZL@&hz_HL@bUSNl5@G5R($Xh@x=CKe z7y+^X?%LKd15?`wT}j+za;Pt0@}d&j%pT@%7~PU`K)?>&7I@Dsrr=?Zyl%g3kkC+%k8_G*r)z1W3pL_m_~ZMvQjF+retXF(t{|ol_R5k5#>zR(T^08_)uR8hUBPy47R#@ru#AIr|Ig z5nhbx?nsg*hmmHs1{(NB)$`Pmy2+$545(5`Th@iqY0P35P$LLB1dnvxNu{PvWTuT| z)>n{dR+aRN6Q{@4-+DC#79Pl9Uua!KSt33r1&*8ORB5!PgqWM(ETv{-7bE1i+_J4f zPkkFKT3x)2(kFVHD53lfv0azWaX%ErqQ-(yE}+?s3~RA!lD~DRP;-J*y7Q1sQB##w zI#SdhJns^4e<&O}iwGrto_g+VtIR1_{HOF|D19+#orEIp5lmiO&0n*B*A87o@*NH5 z2ZzllfHC4FUk`wqtdJFb3M`Vji-1th_^H9)Yy})XxfI5@tuLNX$=vH8eo5kHrOSgEZLLeCM&TDw zG`fEEgjnlts8J`(jKW27{nTi2-SC7~XGJ9KQD9e(bf;XRKup9TGw9MRT2SZ8>1qEk zGxv%axzK=Fx>y=is)Ci^EdR_~LIGeT6H!_3PzZOi5w>$+M0Qb+-O3OABEdAx&nMvA7#Xc-x28@Y1 zxSc#wb{#}tC{%6XZSDc_9O157O)J*p(VUwca0$#V#9)w9h#rlNyTrjr4t7Q;u=7Z) zBUUpX=@j6=B#?W|2Ynpb>BLZ9nZ1zC-aYQMaa!jqV8YtzTeK%{@aSu@RSKj-I%tU0-L^JN-^7%wt;t^75N&o25}3Vv<0dt< zq*D4N=^*qvcZFO-`YOa@9N&6iM-3GlSlK{DR$-7f7KOTBpBB*(#^C{DQe0kY*4DXi zyxf*ON+FJJ_P*t9F_b%9wYINGIU_w|LN`oMP-@cs7UBy zwK|#i>RvikJ$rf4K)O3=pw*6|k6b(EDeL|Z-q@v@9)T403t>laNV;3Gh3CZihx6Gj zwqJi@eHb-r_iePW32?q`wvKl7m7w|>f|{XTTInc(TMbr8&^@Bkc~-&RURkI6$3*^9 z8QHcP?%Y4y_~IFj={;iy^J##EdB2mR)nIjkM2+9S7TASx4gGk#l~$bzSNN0xyMdzwdFU$?{2T}=_EYM&=u zfL~YQh=f#lnBFqGib`+ei$tIpc3dao`i}nR;~4E9FYO-U82JF0z919$N@naoxgXCA z|0xXezuiasRm$Q^rr+9dAHbqNKAKJfH8ma_g4@M^@M6txaHE5DL?k$l+}VF0|A%u( zzQ5%^*4X*7AGB6j@e06XTlGyEz+M6Smh6xQ`&Js}pWR=OV1{0CvQ!Ie&9|~$wcDUt zujJ*&G)rCD+UreG@FgBRGcIj0C#29+L&Bv#n)_wjuJMl(-N^TF!=2(=6owiSAlV;#Ed*16hW*R+@Y<2-E| znx=t>3J{3qQa-tAx7RHznVnrH!^a+x9Wr2h6m5JN-$A@3)>KF-3tP!|Wn7rdx9X+> z*2=8=eH|^M+1Lu8ykgiHf82$wRji%@rO#dKO!um%(Iwc$m!k)_z8E4_&$_-CXvH`# z(t1&r;GWg4wsT*mj~gtWbVPP-QOAC|fQR>d>wbw#)J#=8z67ZCL5n`Su1tVb8f~W* zwY==9sy=kZ;Rb^li?8D5=`5hkA11@#qmFwa<87vDD6=64c2YuOTlN@F-gy zd^skqdR4hKC5McRzD(L7id|h3KO&6=b$0j5Pbhhg`%${Sm93(J26F}D3+%^>p_F2h z%c=wH4H|c{86XLNI&@ZYI>Zn&U_OZayuq&C*@=c!63Kt3Ev*F}vorKg(@a`3aJc)y zP}u!`S(~VfV-~5=Wxi;^o_@3{beN=;C$ZW$8n$U+W^<^AQ<>zb-jZ6i=%QCoypKo? zHa3M{cg%M)X3@jSaQQqD4%kZFy|FPp{g@;_DbfSCj!Ci~zqt|U7@CuiH1B{j2dM*m z!k)r%sgWGuQ{s1O#K!V8jLO{y{%lU39x#58w*4qmWVbr?0B4J+muYc6AWHQIs?1{L{NXq-kdF+Df@Cl!4p28OEUu$7MI8$+X(gs%MMg*XPPLQz@L; z%f_@2*Sw+#`YUb5t#M>YYTL#z;#fC+vH4RX?aMsXUMAe}Mz`&{&GVuW&G#eRf^)X{ zF=g2Zwv*>e64~yKj4pAVjkJ$b!sNwghUmm z^WNMWRtJa6M+GKo%V(NbG#CtYLL#Hb{${+|ofp4yMe~7uvT7l^MW_J=U3W0rdi%14 zD#H;Wb1@~|*>0p|Vhsk0>06Io^k6H=(&Jz)Dn3@IUEJ^ z&cf~8o|NI)PvSSG8Fm*FVEXN86~Fh2Ss#9xs~J7{cTS=Ax+6T}?v$N~ z4s(?aS*JeJ(4BYU8<4jJ<-eSNG2!58tawDBb)abNJo-hu;mlp-kG;na*K; zcAq`tK`r8ZkKKVDwb{&L;bMdge|k9TNmk3Sf+Qd=Em_zhm}WM5#(LQ=vTt;Vcq`&o za#9_8gw6oo1mJVZ@ElVt%Y6;kbO|P<_CfaX6h|XSxivs-8ve9Ft6zOF(cL^n*US`DUPuu^5vP(l& zPbzR_CYqsFSym&$Emc6G_A@`nLHVb1sVyNYHVU}#GB=8L!`$6+Mm|Kc#4|9YeL_39 zgr0CjDe(WWX{pkj|7?y$`jKsepY&0N#|eqwfmO8fXtg5uL9?aq0vRczWWNQ?QH1&^ zH+u?;h-FsOwQS+ImjIjd9rNZ9E7!)aw*3IIb)EJ1S1+r>5z-ya?Y!!ILM56^`DKe^ z5-1zQUZEwE9ADiYsZ#5qMC%vnI-OD!p~_d;sbhe*$#s+3k>;GXvRG4b!Q1p#3_6-L=5cNKZq8V9 z)sIy|bComo(+;F5K1|v;a4?XElP-zl}fo zt$ZHE0g>Ua%fG1{#OEPF56Z1C?pM>LEjigkbUbAAG1IPmZmX;AgHy`3p=)@DGx)vz z>G&41%b;x^*P}@y;vpiCoxK9t(CmbIMh!0;Csn$Z<}Mh?6C$`(s%tR=$BQ^?N8n|B zmm^v>R3Rz>As}bf+E&<=^@2xTg|GQ!N*{97V6WWkTJXBn!RB1v-GeUc#TNly#Cfx1 z>=(B-FR2hWTQh3ja)eBpCcpZ~xvqUqni=d%qM?fFAF^Dz^4Nnvz9!KRpjXc(Tpkekw$M8c1Gqb-QTBmA{FX4>7344* zjgSu69-4vS5%^`1GVA9dikT7(Ak)tVu*Nz#TGzJ0M1lXB=g9~6aid{5Oxm+E{Q8rW zBc?Bpw8uqRzvh3{PwXj;e^`Jt4A^&A52Aa$+Z5=3!s$di52d5DmYe9kKI_hwsgb$* zc*zo}hOWqsM7UWr*M+6oC#LM zZe1Kz9V+FN4z9Z9e$IQ-cu;MUFDbyce)O-vDN`=4keg8w$&%N~)7L!f&c55xNCh&g zy84QVJpDu6=0%dBElH&d^#GfbeDdM^A*6QIcE=@NB#+m?yx}c=7dG_#Ljo!!I(X7| zwqCq}?Ut*Z$976TUYEt~?$et&Mf`t}Zn}g0SmhMtRc%=ye~5C)KX)n%D3X+V#OHX;W`j<5&~VSB;0=fRVt?*>2soYlh4FMm**>_`~j zkWkJ&aIGl;W`CAo?5BGIn{Y{Guxwe@;~9k~m#@BHF`%@ovU39#i)QO7 zKDdxt)83Nq#9`I}$I?cg>#6sxRjd&50lU7m22A5Gl2=M>v*X^+SGT38JP?OA0u;n& z%U3t7hQLsKAe+s;{DehqE1^o%C4eU%+^o_F@ZA+=VlKDBrdxkd4pHl+ zM>BM&;*8^XlCS}1g<2Sm=+D?ou8gW{uKM^~Vha{oQ!u>8iJi{RA@>*W4-H8ru;l9f zZ00(dvV9L8TV~SO>hcF_$!e zoStXcUl-yo-JWJhdtC|n_Ib@TE%f5i+pgF zn(?aXem)w=rw||EKYH%7`;(zf*1J!_0bZWqt3JnvqP#0T!0Om6WVhEaZRfk}?e4Yr z1+uVroSyc!ErAohz!R67`V*_Oj`40$hA1aNU*wJ|>!3?bYK|t?GQeR9YT6r{_LzKO zZ2M--VEjbsV{$!5-(Rcy`(Z7ogOfZvOk{jCY`7v{X)BLNZ1CHYlr7vZJwPsho0z}R%h<5>E zu-mQLAT3^wb(G>Hz!&r4G)~2284~Gd*t5UwzW`*lk>&XJe$9=$bIj`OoK4~}NT&dn zhh%HQUe8~jJH1*2hC4f7{K`|qRVyi_2bG7m)z&s?q`o=4vIEV9uSigQ`6r5Bmdv}f($0ZFoy-_Non_lSJqP6deyrCZVAKd`&6SiN+~1V4_rX|NFI%wToJ80tI(gC zJ&oKoGGpgoP!j_jUy=W2;Q51H1@QnGPLnrj;0F5yeZu@q9{(!w1-Z%V&>TQUvDPl z+XQm~NhSa_t<+vVYHT{kMVAuifdERJU#p(GmJQqG(`DwM1!epzRg4q}83{>s5E-?$ z@;cgV<$|LC-w|cpUM9PujX4{Gm#+MM3?NsQdvXrXGK0QE`aE066E^4~D$el?6h6)7 zKO)W-hbZyb>tenbu%fuQOEt4JZObJBE{a2Gqyf!=sJ$FjuoSJG zZgY|S;eG)A#jF$oBW2W@SNa{+{%m+6BBU{F@EG}R|N7Gj3AzTM^5e19<7osqQA7no zH)Sr?oSBjO3uZ15nuCc7Zm}wunzuao4c};CCXDxUbn@_I72p#WFa5T^lPG0^|=J`4A@5dXo z9Q}FX);}8py#4L=5$W~#xcRStO7XmZ@&7nT+5Y;+Z|eH*|J0+4O$t&{^}6mK&sV@V z>vQ4suMys7OvD@>3h?v}NVn%Onc<&)6!Hn;JRNzPmyeH}*4TEH%`0U5DMn6~tNN1C zjjLn7V_&>z+h2|VhG9{TGGrFuHH;Z(<;ezZ6!UMy8_ zkATVl^y$+XHnw8pvLHdtLM2m?ruKG>`*(Tyc*^_R*H3Q-u$NT)?|muM!^NXtjJ}f3 z{Qes6r88&FynDo(!1+e6_(E()z$HU$@f|&r0*;@5bep#wzgo}@U1JF@6X3jd!4UWS zuI$~ke>}T}l`merSiE}YHx@a0d52rFzux9bJN@15`<#RDrQnXT|M~;5J9q2}t46yN zve)|SuJx5uKL+dLtD8#3pX}a?IV9!G5!NNSe+c@e;+BxeC6*|x`)h2`KhO9- zG2lSYYz+*vQA(GKIf93g{$4gI1|yeWy5|pG z8Lg~Fhsb*WIXp`_n3|_H8?Bbyw5kL5@_ZMPZ`r1@zqC0?dkp15cb^32eMt#)>~NlP zh#fuVT>>%UfZM%)Dwg=cdQakEH-6L%&ykNbJA3JU+e0s;aoIcLG6!$(8|@*BksN+w z1(4ZUqynEHCNDRXmMm-_=j5~(8SB^A>RejvvgCwC+fa!@iH-v!jm7k0Um&Jf*W@_$P68x#0>RAchAl4e7rNeQwQ7Jsvg<#<8q?*zHR|Fx4gZDMvKc&MeZED1_lZ9i+sSWz$-^SyE_ZO>h^W*9A zhWaSg^p)BHZs)0sI`dzSx-T82qrkN$j)&bt`A`T0X;vCZs!$nX#$Yq8)uQ>yJHj0Z zj*075JLu(@-8(?OiPlk&TpJ#qXXSS86Kp6c3)-2vgoRX)KC;XuK-=x7-A9A~FY}ga z$XNP27i~AEyhf$;13J}3-Bqfh$14y<-HXp`Z5QGb@EqY#ZHZpPRw`TUh?+S7TVr4z)nXc|H z(Fr-*S_%{09vRZa_tvI@hDm;g#`mARSF}P>*P6*V(nHEjU6Bx9p#M8E)s#wOnKQKZ zE@+ekqe+UhwQIf2n}%z7s$*31&@}6ISu>>loUrIjcJ85v>GqtZdNpn5ZXO&Ej}3WtAsB8H`*D#+J^woIG<9Es6~G|G4x!#0Z^|y}jOx z*4+bZ+VFpPKh~}eM%Ri}h$=&=IF>prwGY;^`|Wwx)3vSPD66SUt<|CrFec|#VM)*L zlJDmCQ>I`8C*24`9)kJmT*m9c4)%FLIIUsq=ufdiJx|6Z6`dJ!DMQv^6}X{BwK|`` z09-?p@9)ohZt_ZxONf?7d#VtJc;nSqXoSfSUz0D@fZ)P66T%kJTOHE#`OQG=yjjUC zFXdgEIN$c1#@-bqGpE>s+UJ-#X`TNjw9aptzuB?lTP%$XWlwsyR+e(CD%em z_TxcAKO0aW43wI!bMky>$k@1&su=_~C!tbt9Wv6mh60|BN)|@oLOemfQx20n3F|$F zZJHI+?gmCcx{(8rljyj8cc}X5M2X{Ke}KjQ*wj2Je=2T%{F^KvYfqZcZ2E8&zmUDW zYPv+OYHg=3#F9&5xKPkt&?3)1G$bS?C#fFt#71EM*6tX8U$S%+p8%P6M+3S2q1ifU zx1pJo7@aPJ3k`Ie3U?m5ccMs?Z_2Z;wLQ!yOY~xU4IHOGjbQ{J_a996A`|=>9SxbG z37GFH?HL-k?EI~uO4Pi zO*e8RKl?SNJJzqGe<124m##4;$09vkO3_-LxBQwjc^M`9lo+u(;Jd)$T-qGIA5u4* z51j!DAqfheH5b+$x8QczR&~XjqM|hef;73TbMaFAnts#cts+%0pQSS_G6Lp}Qs*iw z&Vu3+`d>|@w7%O0@?E>^Yj zu?Obg1=i3)%tBZRtw8*}|Eb%Fc=mUWn^(j^;Dv4b+t1s)&mqdjGDG z?R{B>1DjtkC?pReWn$kS&ju}niKrVG7x80$Em5rIu?NR&J7vmP^D``nn zMA1{v=-$R;)hm)*C?K3 zaPaRKU7R79eeeH|6(;h(=#o@}@q@&wd`*;JC({?cKz|iRR6XZ3kdU{@-)A6y=sEdW z(8zscxX5-ry^f44zj~MX;ijZ$l@V60rjGvJs}&4R@XDFa!sLDMFbbjY7cEo}p(?e3 z5>I4Q%@)GFkd$IKKkQxdxPm?H6)3z2*GQhKG1X`puPItvx8K{;>zPTuGUP(%9zXaI zrB;%<94p*cKggW+>w$*|wg`bW7qUqYk_&u8@{yWuiUj>>4nDs%5aTZLTgNsau9aZN z#dY5cjKtMFD}ljbbkZ-{&5EjmG*{2V(Pvj5ubCt!_-XPR`7mwtB42KJ#-xWq{*gPjYL?7H4Y6|0$Y>r6szOPh?S5- z`LHzKLt=va(0punk%SF}Ike9sd^D}M+ZcKN@dDm^OV-|LpznQrqQ|-5FLnb$h*#Rs z`e2m0?pVrDqm3p}mMgx_kJ|3S?;@|59`-CS&%90@D~w&>^CR!_iK;;v)alh)KfIA| zbx5t_kaL~mkQSp!X+Y|k07k=4+mcW2aeUEQQuV5R;zUqp%SL#B^H#igt%5GPQS4j} z=1t9A(>#boqK3M(S+1hwEGCt$f-np(Pt%baH%9z5kcQI8k<=w6RFDy=tlb@#Ii3ZN zKnIh1%pAw_mgD}gtUV~=E}dOI_je<1u|2BgF~y`|#jA(r#}fk{=yTaX4Oa7ImDfqVEfsFGLgEznrQWQl4F`|;)Al*R?ceN7J1`@Itk9tc zdiGW@YigN9j!&h%W!)yZqV_)L59`|XUGm+ARciBMYh7&}#6v*?sjf=dxvHDxRPVGJ z3|JzCzq+U{_iFFDKn~021DLRb{iL2+>>6r}VXW+ulQ!kI=D2uwpmu%Ur9El8-a072 zGl(!)32Il!&PS*FF4xy1{B)&S!+e}_W&WUW&u2_J!`G0}ip72uh4s?s!~gCL`3~dg*mPO=f%;lQOmXX($)Rdoddh zmKrSVij~t$#I}s6WuS!@G{eLe2rjyAK0(&bl82Tsx1ek1yteL-FyArbrUqTAYPPk= z^VWWNO5J+tR`3e3v=*XcM3eOek-1iYB4hbeB8h6>Wv@ZJ&G+jqY=#8MxLrJgx^cGXO}?TNy7L;UM6I92a|m*xgq zW@TlK&CFy(MMX_aP8I`&h#y?bNwzOM-kk5~eYgJWpzr@Xp8R?P0FD2?|D^vf37|Iy zMZR}H&IzD1uU|L~oWB+Izd2)pJ#wo#hxGB*Spl_-I}s6OXV^~J ze1G^YHUOEP?y}SCvO6+)kCEOvR?D3J7qMe+GQN%))npi#y#Y=DX$to}sL9YPzVht* zdr$P-plu_}uf-ukrBn5;KZ=wv+>PNXE*nd-1Maf1iQ2#exbTfIP}6qc>({U5A8&v8 zf$$O@+Y}eVU{b}#*V)+E%Bri4T;3J_Y9j3L5HPwgT{krrV*{6&OE`IZKllgFCoK2A zzH4K%;Qg`i5wDfP%A;z#r29$O+sR3KT&6$8+S%9uNb@2u?|Inwwms*K<5IpBS1BT6 zx%t+-!J)Uzz5h||z3vFEB2!7R)c%qmMJbU^>dnjqVh_M&v{n66r%h|@M z68$Gn22-)=;}}f!-&?x@Z*2p5{&IYLeDHu3uzo8da32U4FUA5;FXOz+U%+VDh7s>B z{^LP78mD13^5BG`JRs3=kc`^{k!=U0fq%gIH(pb*)J;sgCLl^cFX$B;6E(uMe9G^8 zL!s@n-`B$D9AANo$=A~d0AT+5?cLNdKE%l)4woO!w6C2Pd3f%X$zKw`1`H@yG+rsE zUqAXy$!*bJe)PuIPL-fEfg7Zgbh@uPbMEkCPDR>kW)r}s?-yp1EzFdwS?c@02o_WsU=Kb!^t_P;QSCktt zqgMayrKqGAuzR}eT-5&(`76bo=F<7+87KbtxOo1%*?#TfpDA91DJycR8~mJC+ww-3 z7xvHlhX&~X8Pwv5ImZ%p;>0(;cmJ;{CnEXs`?2X14^4E&UjF9b_8O%)N4T`{-R?Oj z=h5V=7D0RUy4l+cqj_`%kD)z&bsV3IBA%bMHVF6UVCJa*yguRRngIIE@B-af197!( zRdP6WfcsrQVaXl(f4_9O=nXX2$x%LcS&($9T(X2!#wlSs3TkaKR(Tt`psX+8l)I$_$9=`R<2)KFxhO_9245 znL!4>^E+h65D2;y6ZK|tna+6du41k4ALK=PbjWxOtvXg!~18;BIv?VqJ~+!XpLc2&ZHx-j1uXmbQh4zY5vAEeUm*TY`$Qmnm6v+)wU0#9A3I#?Yohg6@)mc+CMko>M* zCxqVs?f>C?wzX(YaTV_-(yUDzqps|wsLr*A-2mlRj_4-mxl^aAX1}%l*jn`Y^tmH* z)o&IVypsym&w@HzZs=&2;N}N>Cktsw5S!T@2TD*KBX_p`soF22wd>uLNU(5@4_Qan zxiU-3Y)S)t6E}$;vB(;M;?Zg!JOK!~7jGeN9|ocfZ*THbTiYts!3;ZxBA-B53zgpC zF@??sY9|K?pGkfLC3UFDaQ$}Od2NdAV(UORT!KE>7c}o_bYvzMH?J(zjgQHr%hwJA6TI5UoAm%;J*7hzp~xB8Y`5G zPFCiI2rGsy(}Tal2&-^un{4dbU-lZ_StV8t+l;&`ku!8NbX#862CIb6hE7~wP*rtRF*70{g?KemL_xdz z3W?ETA&t%yfy7JS`^@XiZ7yBXLAQ0I-CZeA&6RZMPTuAvgG+#4azL3}x0xkgo;LnW zZ#+BdFpHT$%~YiY5F#ufn7y5ZdP$1s$Q)f@2))2R`(d?$y2FhcXJmQSYIw-4C3>Vn zcZTMX&8iTgPYFn8!68XOeq2WkzHV`LR0ygmCw-~fX|!QG+k)=z)qh7fiZ+G_FnFvv zq1ty7DlX|Ih?#@?m;`PM5- z?&X($=T<+TJ_RVLJ_}YqYA|0d4d**{_-xx&QB_egKXN0}^igVKPrzXKyr__{L+kw4V4o%_{w z4U`m%Q3&6%ARxs26sr5+!hm3upkN~CKvlUHrDdGQM+|xt4XSsJu4Z6h z_3?CZ47r(gcGJ}BY42W6mJh1Gc1ii2q37qOnlHatGiSc}cW=7|xBAWNWfwzdUbZ-S zPO^J*%E8s|-7fE0{kb)D?aKLoI?nuk(i>Se4XLD^s^ez5E%j&k{>hd0xv;Thw`r>~ zyiVUvOFQTK{_G_CywhKllQoY%JG1}a@0mLn+kVfzWp(e}muXx69ekrx^?s6$P5Q0Q z-RF@yjQ>*7%k1~-KZOp+dw)K7)VP1&ah0>cKJE4Vxii1==bo9i95`nE%f~)FPi$X$ zrk>?~{nzp{mpHk8Zd&ult=P==Lx9=eHQLp``S)MzetkQ8O;_MryWHt@y7y<_p2*9; zJ^l9oSAOUJsq|jHRP~Ov^1iCK@qZmtSR<{XRug+)4c4cyaQww5b0Ty3+-b{}EdzEo zmMvVUSXx>N>4GeOG9v=s?xN`|8d4LBx)XF3K1+iPIl%p;wl=oS(8S1~e&E4_1W<>k zwdvprqf{p`VDmc-wsj$HvDeZ`;lF%cJ10F(tQ+&4mXJ{FM6=ZL{+WgOR&ze5Ur&lTB*Rv9Z#bWon=8QIbb94 z2Ml}mRqKm!CBMJty)=j`RIxjF?;_XCRVAQCcu4s4g};6QCjnJGfgLy_3kwUe4FaHh z0YIQ(acKC;6{}Z!uMF`5O=`S;9W1pnpTdA!;TdfyvX@4o`uN9R`xF)q;5Gzong32XY5X6uKh+IPwEmdk2 zqR%3C0YVZnm|!3YAtWK!IkC0&@RgOw^r(B}>SMLw4=> zE;~+cW-DEJ84{xm|8lN<;#eqW-M1^N`Jc_!J`a88*0Gwe&N}T~dg|MXUq83KgE`KC}7|XdvXJpS{wRid6YOyoB|9U zz(KPiijfZfqjbrV7f1VDZQdTFMDkpLVGsOkwh)KCb-0O@cEyuKa*ec*O}3K>%Cd1NU{QV7&|{z zuh*&vWrGEvNRfX0!(m&UcK{f6PnfO5nL6si8<_UO24VJOcPhpxm$@sW$VlN`p<(X0 zCfPU|UiBS_0px1(TFmU|tOkOM#%P)CH_IBM!8S&k#!`YLhKm@jc!51gOE53Iu91m# z$S7FH$Bd?7Byfakj`Jm86h^EO#}L#h8AoYaPQJydD)1g1Z^T!d-1O{dRHO($cgHZB zObvaWXvuqsnCeCk!?To!K*X4FUJlb?G!K^xum$bjlq@9;V^+j4G!Hxz-}Xf69y^gM zLv>0+3r4gI=@w~GHih$!Bzgq)6YuPZ@CJei=ZC9AI6<-B1#vnpDkCWpW{?!r<{*Ra z(zAMJ(e9^;q3y7sqBgJ#EsA#4>shtPIby;o+ z_?!^{kZ)GsZnj;h{;s-H{k3jljS@2LVh^F=$?FEL(f1>wvAv|qc* z5TuUS7~TL9BC9SsFsV#Nvg2>sXh8r$*uc0V_zAx~;a-U4zJa4QqcJSdQb?|64(`Ps zQ^A>jH+_u^<1x*nmlvSyaR*;rlZuM8 zaL3s+NSD~$9i)b>{A;wPcH51+63S`zibRKp7fKOb`ee=H8IDiLdb0}S!6O70iVKE8 zoyTD41_NF1b9&Kcn^BtR#R6G1n-v&^@gB=EpGo#{%$oS=P-7`B9J+HkQt@U;FMCp zO@F0)N{ITA$_TNfTTO;JzB;I#3Iu;Gz`EQHK??5B7flVrjwsPZ;A6HT!{aw-{{sn_ zm0bMR0=FNtK@``q#ax%pSXvukGK2g2Dz|?YSb7Ft-&c6XmHgHD{C00}JlsSM=e^49ZVveu( zAzsB+vI27joI0JdytyM9o4~mZWjchWzwIR6%lo18NmUJWtjMQR?ZQYZAK_Yu@+z#@ z&pq#Ar#V($8R5+eUJ&Jl(<{x>ahmQAsqr*bj7Fn#=?1c;gP37(pRv*4rbnk`5|n}_ z`U6xZG(uo%jA~E@PBXw4_%;ueciM-V}wIom4-O$f1%6~6S%5ry9rAh4_oou8X1ENvh|ynxRngTcd0rJx9h?$nrOtX>8BfXU!%g#y_K zdyyz1h-Mg-*vnQQlGBhoBg~hj%@>^KS{)Q_QOlc%7f`Jd8y8?y+mzihRX%1onvCoq z-T2edzBuYsAA%b@IscEhisHi?SF$th3 zf@P7=0JV4kZ!qSY z18OlyNqqIkeM^?alr)7{D>8A1J%HEMcNs=LqtRlJLP-{8HK3;-aIvoCufl<;@Vihw z3b1{9-OO;Gde@P=y3>uSZh?ms(k3GX_*m@INb zIlh{-u60pQP`?1CBxnOp8-;JxMd9_Oe>Lb_Q%IH1v#xN`kD?9sK$%no2t;zCnksx)S=bv(R+C9A!U;>Mv1_H!z$3VuLc|jzU zbH%L?9ALrfzWP(TzO{hT9)H-}3kA>`5-{(9Y%#fp&U;{Af1`N(=S8Xefk~*TKvoJY zgb$POng(o3AmSGgEyP7QCo)srA3Qsz#EwO(~hr#QN;zM`xu)y z!PcAQ(MFSoi;pwQts{`P!5%x)R)8IH%bD&jXzI-R_K$0pEU9<`EUKX*@Ut%K8crHT zl~|u-vKjgtm8xlvK$wYAt5D!c_)JMG6Vq|{;V%-Qq6@l zScjtJo}t<*i(#nAEDDBFzW&Qxr4GLtVZ-5g!=OwGH6GFE@VcxzL@^*MWt_Hu6GOgP z&O~Gz-0=s^3*o>+!x+pwY{_En6qWy5`&L?}7-sOl<0!-jR$EQTvRIq>#lKBm+g~Ph zNy`5a3pjGfi^aeK?!)0(?#)?1XcjT43P{v+xhGO9xi0hwT4vXn_9pF!)yb-Az%<1D zSa3shmNJS^Jr18t)-0NNm59PrY8bfbC*GsEO`^qO=d1|q$S-E3OYZ!A!?f92Ihe-NAca=9zVL!3yQ-dG z(QqU~Uq!z3ptS0y#`US@vF3QJO7pvl@lH90T4Ne(oRrWuM&T^W`KrVzhb+XbinVDB z7v?+`&?Cm*oedNW3)L2S*om~2b9zC=_vQHdU?sOApo&D-Os>Skt^^P3;%M9QtbyEc zw2sJw<^Y?^&%kn}$J1wbda;+>kF=I5qg z9y7BlQ70t3LLa#g>rl32ifW?44EA>{5q%4--5vU9O%2rbNBzqgt~|X%^|huu$oQpc zV41X0X4EECxnK`tjl-h5NFS~!Tk$G^(bdNxEnm#l?1&1c9?c%+OVS%r#b|6Q_8}94 z<^4_p#58O|76$+!iwXHAYtQk2M+>B0;C#x$WS)-B#jCqyX+}mhs8;meQ`%+`}#jmgJ|& z#Z<1mttIq~05WSM|1F6c58NDM@b2x!8YaOdk{9!Prq!r0Q_{jDGy@OGsZ`tFtaTA# zg8aJMo7Jyt_A4gMaoMeg{DP3KJKK7CeXuS<66AsH$Grje%A}l6Ce5=szZ3$V!-b_* zpwDssW^SrW7X4av>0gVsqF~)!R2g;;Z|~hHe(CMdM^d3r=);H0eO1xjk6-x5rkNAU z&m>nuaRA&5`t(6z*9(fw8maQI21|p4;$&==sJfgLNZ7u^4&grOPI4Wz!*v!lU z4eeoMFi9iGNRhUfbBjBL>#2}p%EDxq$%*dRJMAwu+^N+6VN-kyH>DJd8sjifM27;> z7oInb!|5JpKHSl%duW)HTe?mU-3bYzMA4doIVW%GF!Wgl085%^3De zH~p7JIr(FIkx${vXcgGBClJT=$u9N&Me=P7MHd(=}A}CU5_m0N?;3eF*sHy z+S2(Z2Dx+fr6j6j42y&8zoNXI>G_nsl+Y!0f$x zKIikmtQNz(pnc;S{L&1)M(n*Xa6ZLQK=7a=aG){P2^RShI^z?%22jzm?xGmh@b2lY z{zJpqpq=R-!{i!R$gui5`)n zF#5+6;MVGkfQocqxn`z%%W?+7(L1_oqJdtO!+7APFO)?e5KkY_JeIE^CLS6eeeL)a z?4hWPulF*Y{k}Td9Pt@wH`~$Z*xJ?25n~W;A?VH7hU*57B`F#iNfusUY+P3f8+WxQ zVKfx1=RYrrbv3A4(0a(?QR7ieU`YNZea^c^+!ta{(vY+RxKKXL_~bs*^Ce|58)ulp zX(G`@LX-ngF9Mf8!c#L}Yg$UY-Vx<`TXufC7{Ax> z<@1yKE8Skewfcc&vO*Kyw!arx^?Z68R$jHP^m!hpHF<;ZOuhMMDR zx}v0I(=WyKYSJu!81NSfK7+0zeDA_;K*QZ78Wmz+ggPu$sj)3?md~8=o9?dx>mkND zq6*wo44CNf`Q7X>_b$9?Q!Y+-*7!E?&DTElJC%GaS@SEGJG8URGn)WvsWXA3j{txoB8^ zGdKGv_NrId;Id;|hu^(ag-<0Lpv6-4xTqW%M--b@Js;h+JCr3*?HaqqCDxvZp@uoj zn{xPH&7y62-x6Py$ZE(i5^(I}@*)&&?QqYrdx;%KCR{~qJZr)QOXMY9?fW<%mFJHB z&1!&csE}^yz=CxDd}w+T;T!577@#17W<_RflAr)-{L-I%#c>z=hmF|yFD#$Qy?DBp zZZOdXZM71T!cFN|01d41`nm>xvM;%ZTCC3HncOZE2uBcl` z9|HP3U3uihFItCV;3^Eo&`m@|iGaDsb099iSqToZbfI#J{&livcPqU?x(J?yr>z(G ztXVU(r{TT}lm@a~K8W|^3Bps=%v3kS&($8bEfI8_)L44+g|Dzth@0IiG`g$ep&|*w~EnkYdeMW6C#VNChH~ z8WV@38MKhnMNoTVp%<$X9#ne7un3oUH*Odl*mgA*v=W^>pGOa=dSRFbK`+kyP79@` z@Wxz~^xLI*4O(|AR77dmO+!yj+r- z&V?SRJG*kqDvG$pM9ta3%lpQ3L>FK^n1$visiVJj{F1!q>v&JS-%Ww{Rm)OxB@xy@ zS3gRWT%0*BEBkQfvFbIAXTw})j#?3B`{^s_H;el7;0F7f)x`V2iJs-YN&h17N@A1}!(n5oH$$;JL|(ZXIC%BR91 z_vj~p!Ra-k3%H`sO<|ho*LZ&fw7o<0mG)P!nAKqY&Dw9~?LR3x`uzPudWhwr;YEg( z)mwBwYGw8Eo*!F_aQ($+H*MK2$s2S>z;TX|jKN~zMkD3ZER$I6g!?d;J`HmI1zQrR z-??v*8B((6MKG!se`z0ZtaT;siL^`Cpq#+9EJ`EKGh>e0KdzDG>*UHu0no=@M|Y+L z+2bKe8;mWW`YY9i8xjAoW9+V2BUf8Frl@87MbR&2D3TF{HO+;X?Pa;>S5w|=sj3b$ zds;$sf!u*)OW86<&n0k$o-~SI@KJ{fcNfNuY0`S!{CL4LF-`VwWbx>A49xmA zL+#DR&ybKgMP3`<%`o;WPkF8TjFs?P7g1O~{=^M7uxfD%A$6QQGlRNBzZyaub@1n*~8wfOB7=O0NvCZn8ti1o5KIEKcNrNsA8j~-<+!1*8lDbdPN0%f&aLF#+nvuW@P29hct=3 zSVsR3Y`?*5QGuSm95=nL(KV0?%IT`F)160iU9R}VW+`?3f~W?mj>l41IfX;mS6bt+ zd%us0UgQ6vhC&@d1?r-Q*K~Q~11wo1CE(KLxoY zHFbV!xpTUuL(^tn{XoZTyD?3q>*5Gi7YVAbLg%F~@jvl6n#oQE$b65JnOYJMgzCGJ^e}jS!r1gx-u(KecAF#?L=5G9S5iQw1`eb?fr! z+atJRMTnM~-6O}UKZLK}aPlAL)68)P?FIOEu6XIXKp)Z{q7t!$@=;GGz(`Xjb55% zyg>Z8o~e8a;#@4h1uxILaib)8$-`w(=a!ov8%#utND)cBj2bT3xe4`~Y zS8An*-G|*@lfRC>(GN234>`X61k>lvvu>aCd zi|onp`8>oTRj?zgiL2bc*1WUNnm;0qqn5h28sAXk3J2Um6}o99`V|t7zuXCnG@$k* zJcY`)WylL5c=516Qx#x|{c-{v%E|X(W!d4dPBkI)E=@m_)cJ>}c5k`iruXDPKPk1; zy+P0C@Yyr{gNx99+`@H(5Mgv|ZhheP`e!$oSkg!xw(o*2OKdaFvbtmiSu?0WCO=$> z>MLA>j1pnyHXyG;u?Mm9ll);GtS9e2;}7!NIMKl~-LlxK$y(8hP#F4Wx14N&dKfIk z94kLK8K{UPFJp@<9X+B@*u|}P32!oQOaX3i!t^lI8kJhJ$vPC#+Zf}?^sKjp5bcYb zKWH_ajb8Kk=kX1Qf1HQTy4sQ~S%pN^)^(?uj@XhB%d)wjXObQk6{GU(=wmrGhG$?A z0q$?w`Ll9ZG}mg}n4uma4(pWe-uFeyCOZWKhNbHL3MS!M%yeDXodau{0Hw$xG9sRE zV`p1JNso1R!4)%^-%^v!;Z-wi*$6Qi(ks(<_i9Wda`%`z-Hf=cP|Ssx5=*&5tzay| z4xWam6+r(n-r;w;#Pov4v}TP0VnFQ7mv243LzI=H*SeKMmYiy_JX)Hv;t$nKu@6m; zu`i$c>f`b=U%C8`Df`_gznQ-9|87>`YG;UrXr)mVXaCK5GE|O?8&+nXqcjy^Qr|Xo ziPIo31DK$?(D^qsc_h@>z!>miK^M8eSdj0Z7Ztq@0C>QmAae<{-HK zfSN@z$VC>pIf_qP`ey>*#YYx0AjdaY%nU#`oJChx$^gBJ&;VE#k~M?XR`j9g;@Xr^ z1$0e_b&|*b$?QV@2WIK>0z32c^HMwW>*sZLW?6|h3y{NrR)Z7+Cr@obTR0-^Xs_LiJ9N|3b}9nEyiE&T9XKhMgk+1=3ECaXNG!ub1MR zaC$2#3RIRdEP8GOppxS0(Cg1B_BLy9AG(=N`u|x|!AAzp4Zz9p{07L`FmMCvY#6!$ zb2f~)Cl}ku!<$TlP)odqj8}t=OP4zv=Z3X&j$0?01No`f|E%F&zqVp0;J0vNC)9u8 z_D;P2LcKh^@m(v^{6~)BMg)jH;DR%LVcMI!$lI0AF!VCseS7nBsX|4-R~0pI$g!ZN6>@ zqE>pVyJiB_dUzlp%CClt@09H!2`9FNK2u>JZh6YgIo$;P@l$Eah*n>XH5im}y-rr66n4fl1xTGR>&MXf8gNwU z2J;IUDv9wa=3Y*HN?4l1jxgJtS-)G}fH**oDe^A2W9z?Vl+~*ja<{^tJwX4fv2v;J zOsgPq!ts*eXFvJP7&dmabuE#)UtX0{67#spwz;(x`-Fm)25yI?aS|K-d$1ePtClSB zJ?n`oCfYCm64P=L(%E(%^gSX!acXA)<(ea%GmBbC^rl>U$SlpNL?9KrLD?_Ot$c;9 zGGVoxX?*V0^DoXgcT*QDi2jZb=*7|3+kSPzd%01>wgmT#;zYR*jrb<*VcA_RKY~@YlW?~NzNQsyFZ_Ex5iJM?prv{8u!_SMb^O^nZy$>w zlXJM%`=i)FxppP*$U#_*&LRKk=?CEt)eY>=uuYOq)M-c}|8d&ljN6hWUtRFD<|n}Y zJ1`|>lnW`S+M7hF7CQ^^1mA1cwrUS|Be%*!1@USJ%!3{rq1)}M>4>-#&jR_(d&vt6{xzCeGakb^%cfTDY3Ifb%I0&p*U^6&?O$d zB?9ESY`;LqAB|tw9>!j4<545&lc_$_$}P=@=}E+`nx>GcX*S z$=TND%aopaa6wywnRphnDStf6=|!Ho^$UCud)+2t)IBK2Vw$y->pZftSLzKd@t3!Y zuYxXK=}Z^%Rp?n6e0J26E1&siX>6Mv1{XRz%99AG!$XM-4`P4M!~iWgaev=L#A(DX zzAgM7R>2W#&E4JGaopTX6@HniI4aZa)cq93g=UpKH;GjJsG<%NDkU}dPh=TpPNnb- zW^`*C_p|7D?P$e4w*EB##DkxtY1TApx|Ht<1|JZz!|?eM(V~INxA$hV6Lk=+0lysZ04yS=_Qm1zdR3i4u*e{RI z+|2f~N4vOS7F-0GSV&r7QW^F)KCz;vxhDvD4KZMLG3QnCLN_{ZQkMnQE#y9sevM$h8Kh)nNK(F1#~zngqw%U%ro+|=*2K2os%8zjCqL1m@vZj{YI5DrpU5wfr)Bs zWh(o?0^dwRE6H2>+@)@;)EOu$Idp8M9$Sgs%=z3@T1u66FlC(+x|!c{fKlgah_0XF zdGG-Y243C2NNgq(h^kcS&nH1Hc!rM{$+*qC6t!MHQ=@W_$RIvnnYmEIZ%C(k{qg=JkjRae*7VrRkg!T@2JBO z7lQ&Qj^Ddy;iHqNC;1j^3v;{Xoz2r)UKLz|y~s@7Q2ncC?HA06`KF7szOr2g^*ZD8 zk5@pm%Qh<}UUNS$jrxv6VHP~##q0cm@*aBAa0;~+Ig42UO+7oE<1hVKzRB5M=f8YA z3A8om3pe#$Svj}KuPbGp-;g|6&~vYIB?0s z7tKURcl^RsB5&V5vOSi*BAYby^oZExNMAF{8 zh$NKo0k~zIM#~BPepAWVi`wytTOG^%rigwe{Dh*7YXh1fj#4*h3qpH%4>4n7-PN0_ zEzWf}@*+7HKW2sZ=!i?aAr5Q4)eSlQZocYS!pFQ5JBrTnA}sj?Klc`rX)U-MAsu3W z`#NXw$Wu~p4F7d@0;&EA<@4f@fGK~6-wGNUhslY53k9* zZfYTc;)}jy^$Kd83tqy2Nn#xJh|(F3=USE^ zD0tJlQkB`z*@x!UQ_VeXOUDoe(wIDUJMZhutj*Ku=m(jE{+JK<9A#Js#%+!Xq`0>m zTpFNaN9N{xW!4~&edWZSE@46)I72%+OVNXz&Q}nBi*#$b4Q74ex4=)(9!lRC=-X6p zUuGS~P=b9lH}4EQpTzoJWJ>I*@N`+m zCwZ2G&{8x-h2x;uYS6b%NyB; zB+>>9tdvrj2n^;GF>o#<%Rb4>6>-Hc*PKO0LYN5H}R}(|K!AOoJ^QC%B9zW~W{3|OZI|6~6D2WI3_L-n2B5$=|VI19T z91o&fr_|Dj(uo;90)fN&NHpx673#Ifl|fgBz8drO+n(`C!A)^w6s&J)G~cJ-C-}(I zPPMWP&cxTvND&z-ZMHvk9O#l3VmUQrGK&2MT zw#@1~NZ3xIjT|=tzr{tJo|tW~VQ4@ z{Q8M`kEd`&A;Sla2oPU^lE$-}D1N`WB;mroq(@eC%<6bm7aLhWY->iJi4QEj zVkR4%kF4*wmr{sHSvH1Xx1JE8Uz*%R{EGNa^H6I=!v(K~1fXbkw3V_7G}uG`VXC~> zSvnzBmfb5rr{cD(WoWna_KDl2Gi+c-pxu@HONW4Q9C6ZjKZDLiZWhntI!x()vt^gm zXx3rdYpPb>gXnL%5OoB>Sa&D)MOmf8#WJyMR-(&=Y>!tUlE%=25W+S(t^)p9fKnp! zpOkp|gxO(=*cL+|ar10irqZt>psu?)g#l5H^Bo=6lUK7Bi+W%--k2{XWasJvMWBy& z|HjVXbdGzLlNfF7qOXJ}yjg5^bymE8`^KM| z0kzsl9GkFYZU-}3kayX(hI{so`Piq-7lne+(M#CNeF#2Csi0ATKCm$HKKHWJ?8^T4{e_Fq!?Gvpzl^dVL{2un)e~VyARi zfD!ZN(r4O}V6Zswg{iohbWCLJ^a^RL!24}SMefNGS`*%mNT z56`*fmLj)HcPj4?J|k=t(50LU6!Alnhp#2fi^EZ0lsxQf{oYxg=Pnx`l*9Fh$Kt2q z>(%rc5{dwYTpda<%Wat)K5C4>_flhUe4YDf#RTL*}&HN*}wHS04VsdIj-RaT0yqhZTnd!U@=3 zD^9U~WLN0sx8)Od*ybIAV^>4HSS8hn7vpbEEJ^w3?{~tB(XmU`u#AmL&V*>zYQdMZ zUHG4ZM0@)c#{ZT1%u}KBIzS0H{w_AF{X5iYL0_;=l?KS+&~xwWXD-uL|FMBzVc3oR z*PF;5(G0ZodI&vjinLRE^zCN1#Xo`cUu}+ljwb;zKe*|;|6bSI`v3i;-}m6>Of(C6 zdHIxM@ZakCv;KcS>G!=)=XD0|7blBRPXDXH|I6-REpYajRttIpVMQ_BvzgNaef!-@w^c9<*#(zqP=)g7v6%CAgV7SdZ zjQ9O1!cry)TNidy6{02I)QaQR5(D-LJ#p33Fn4IEo#CRNo;fq`-|#+xle>qu znd7y2CTPqp1Mg~L+e+_+Tsb|(_L3yY;b&pyH-s)!N)eHW#}`RxeX3pTXR~Qq-rlrC zn+Ef1-Dw!gfz}26)@uOz(SBc~TuSP<_}WJqOT1(AX!5vfNTTeiSsc?5`uRw#HM;*N zzVR}Hhu^ZJBv0G}E0}G4ySZq}=!t_PEn{v+hHj0&cJOaII|wGpy00J-bJ$(<11CW< z#IQ+KK>nT^i0`#bK19MuIvppcjMjd~i}{d5aC{&H+3o8iVKGJ9h#}9NOAv9H?9Sup zVeUOPQ?w<}Vq_aT#PRbpE-ki9?P#$C79i;o-+_TD8*&gsx3+8r<1t0e?Jg;C+OmOd z*Pf-ErkcfQIJAXyXZb`Tg0zwv?zy_(GO(iCxkXEjxAGySGufdY^;mMPe5RAxfgP)0 zwrzKz$)KslxIMI<-h9oE{+;eu>b%!5wJe;u;arcJr$tB}%+`}b- z{1yL7$$eE`g!Jqa`QVXt@M5F1y@ksxDyU=M+}g5#<@0&RyPS4*&)A#O@$6mP36ncF z?Q+x}bkT`xe(UUh8KGmt1FpJ7s9h9oW9VMqd8xIf%&sWjz~m_?nSG;t%G|bIdbQCO zG}H9EfBlbU2G0nAq%17jKKicqo$Ml!Yq%p(0pA@u|2R$3_RXh>lEjs2CV!piNrf3< z6l>xFa7{bWYVOGC+=^seW^~QJWk+bl(`a@!{7j=KbJL=mK3iU=AHyClXfRJ?gW5tA zJU{CffPrs(_Kz zVE!%k;Bqw_FLTN&ccO~-6HBQnjB#34RY)uZmhsud20 z&)JTd#a#%*(k9JpOPA#M1$}FE@AVVqDQ#!3POCSv`=H9}*h=148BFrlBz&~eR4sS~ z1Au=rwUO}L1MJ0OJ%H_myxIg%kQ*8+Zsz=kB{r( zRywtXAZ0fPd2|76`G1$z#)r-?MiaeBkB7jwJI}u;;=7V;g`(uI3c<8}jt?k^lEg#4 z3PtNPH;PUu)Mio#`W(Wm8s{2<9j{SoNQImmk0wlL_P`^oa{91iE$5;q z{2s|{ol=hDPp^Gq%<{f6^z27550_r4&`Uepv}lrFN=csXaQcc#GgY_u)QJ!N+xq*z z0HJj_{T(L%8^-0N019-=}F(2j&*^0`tmu z!sX+q06~;Yh1YTj6H+c)bB|h@C9m`{&eCqPzGOE=&;0OVT#IKI%!nOEhHPk)Oo>D*2-k8hu_X*;;6M3XYK)W=(mvObEdAs*0 z!?N{SN{u${eRmeVAKSH}+gc#&t<0(2T$i zsN}-+yeVbQ)w=@w9)wvDQ(-tX%C2X6P+ie#AB~BFrUF^@QaU6EWT4C zOaw2J$E5u>ljqn<*y$T$v1vz5!=oM&)=wCn)*B~Up}}N(m4lGoz=~PNCHFGLsd~_Z z5XC*N+~;SN?|=gi*~)13&O=%hs!@lJ@{sk2havRDu)f&A0E(IL8ou+P5BFiF!f&Yc z8ap>3Z+<`iI617{7_{=gRA@oSmq$8lh|jV=|7p$p>WNwS*}?tkGFvk z*LQ;IGKgKH_v{JH}3~8bGes?-}Lw&!zZ`(uVD-=q|pxTL5int_hTJ6puOB5`b zyU|aijmhE~HA(5HF+XPF2^TW&P$p+BZLCGfcR>TA-?4+m!!&~D$a^Crc$gI1jYBA_ zvXGYwTG#MKN5>~saLVKm4q0I^FDP?*Ci1)3L5E+a>X|$wCUll0Zj$VmCrW;GBCT$m z^hd(C9|`t1)6bK_h0K70UsorAoWCJrurb0`5wy*d5~evamiCa;TXgPPutCq~s96yA zZfr^bCd`asdxC)t{F91rBr|4h8rzn~*h8*P%7X7~0jBc!I&3DHo~=IJ3GwR79vfG* zJYa$k4kn7a(D&m5P`19-pE{b!V>u`a@zrbyIzaSk!mG+}mOr|!LxDbTHJd$?3y;`TM!>Wurn2B5^Tx?U|8W@YUcSmfDd zU!N-uQ-CSFFv3Er5&%fd5X7|`DQsdPT^a6;eOA?})@ zjD<4iHNg3&==(YDu@zMD1Mtdgs_oKmLF6&ctBLDtYrxz4+ef%YJ|8}+|0;nr)87or z`e_`HWyt{%YH);%6^=wSS>*areZ4R%aYclpbJUTq{En?7wR%J2R{Ftb7`>^b=ga^V zF)k)^iT0qb@!p}sNv1&aQKTJ&uTPug77P;6|7nEO4G^I#s*7+O0A6wG9&AUzg8 za>;Hz>}gCN(xkwJ`*`)g)XzNjyGQoLpf|tHg|=(itwH`#^P)&wSwm?_fcP`NP(esh{6FG78SDMdx)Rb8+etqk~L zstK(=o^&(OM*2xwH@fy!D@y1QwNs)x4C9$Cv%cC{OG=?FdE1dyc#c*y=Cj3p*+c^{pwsK`9xfhcC8Bk?OTu?nWp zB`ZCJ%1q})_bSjA{!4qOD7;m278p54z~W1l=xg~cFFfzzg)|;X81LSyVcd(MlxeQK%P2SP@Z*u zY*G5+%aPY`v&u7GF1)^A;2HV1nJ-YN?h!|2WqaIJzg_YI@ieEtv8$=}iQlX4f6~V` zvnB->QUVm6Cs&-JbV?A|d)KmlXw`sT37z4@(CS#PJraNCL?k>VPd>L8Kl;s^E;Zv! z@rQ|+)H@WLUZ*Yi05B814<9;2u$Pg}L|xp$@G!Rft=9QFy1=|d*}R?Uc7nD|n>oV1 z0-oEh{$1_D;BVHB7%156&A7p7By2l(%~eJ4+#P+IKT~4lFYy> zw@D2@x3wA{#;+la4Xu53?+Ch}HOUq(4C=lD&&>lM9y5h@5srb`H`9&>rmfc zXq+2gAovkAJ&WxXazXrpM5bu=(bcPtRAK%e-u`dwy?I!Z*S0@?+Mep^u|Qi}k-4o3 zPJl9nIkhSlB~XVdb5vvuBtn=1Bx$9J%A}w&1c-`)$PgI<1VU0o2?$9B9I6n zM2IAWOy5`SIrW_0d)q(n?|08Xd6MVJ^S*npwf5S3ul4z?6>zo7tp)}1)wWiy+0zmi z@zYC7u-n+)Y@fU_&y#u4J8rP9ZLh1DR;G(w$N?`}EkZWOVvKmLlIMj+qMv|dg*K*y8-xhPueY~P9%pty+=drX!I6- zeK@SXKM*PG=(`RdfO)z_+T3f^b_VWG;9J0oc6nIxmv+>34VKwmjSK_g6LN8Q#%n)5 z=;1$mTB%3^`>*a414#7;41c|9b;kfz)WMMusBm*T(dqLY2J0fFgIV=jQ#$k8&n=q4 zx(--sDebs^e|~N!n6{iOW={Yr71J1O4KwL;vtva;UCeopQg*A+9If)SR!Xz1(5X<= zX@bG!8ii@i!=1oUD#R(9bsEtnC*1B%dg@tmPjsl|9J7TiNAvuP7vddDlp@ezqrj%9 zs7RA{xJ^q@qhhw|x~4cuZSlV&M^@~aE-2LM;%aOd1zVyA8>q@$J4z3puwqJ4WBZuV;MCR8sW3g~v zaSM0KzPw{sbS@%N4oj}aGjh?S;y_ie-3W$rC-JMlfr*7^qV^);zwAUcp`VuVU2!aLPR( z>YIf`sR>>!%R%ZbtNs?7yJ;?!dCyY!3%zAB1czBSR%ay~AN|hLn$Vz-j!|3P8s^jv zb`|#T#`0^MQmlxZvQzA4O81iZPWA`&TC>|iT8oR>ZH1{3?}z-Vq3X8p9m#d$>L(v= z;g*>E&3+>DJ@&zdIn-!uCtms@fsrp**4vLrbSpfO?xU6S z6)1Ed;C%k47-6uXO>h0Q8=TdrB5wHr!&?FUyi|2OMuM36HMm$ z6xQAr8Y%3#kJ<=@x$!(7G3dy991yLEkpBa)w00UFPQixC-NM-WTWqlv#$%+^VY zZupF$W$&)mh|4fw(Y+V=My24S!ovQI$fr9f=OH=$QE1^hBcyr=$zL8*fJ9e~iVlfI z<&7T>yXBiT`?u-N`+|Kf(^c(8)ioq1kQKM%e1#0IKNvQ@-ZaY7b&sTzMt+)4JDo$$ zSs9PBE-~hnBdPS|^~Qr?wiZ1F+d9Bjvp^@Dg5N?MBV+$Y9=k}t=lgSdRNw?I& z;rm}94OmPM(aBVFeR+}J#A&6QNU&dcG$<=9DRzm_Kb}4%+}0^fMUqn>C6YXsYuU{f zF4r8az!uEDz1 z;9jOKJ(pr5c7Jo*d492UO8D)Ls#WB^oc9B7pWeB~YfaL*o@wcV?QmjUUg^x&C3|1~0-SbP zQw0yXBHucrWc4bpaM8Z7rkWcmEba4^+izzIh7pMvOYFX2QIB$~yEONJ&h~m6J1rW$ zg1mXv#`9sAM_9vRU3tn%nOdR1`_ZgQ+$1L@#&}qDL(Q0!JtChFRRSv*%5*x0OQFUc zi{vLs9o$!3W6hfba>IAHoSRAV50z<_8-kFTfU3=!`|q>xZde>CA=ReD7|GIS&T%Y) zvuh~p^h?t3IjMU3%5-&0v2!a;>t`u`W(Q>(l9Au%aXlp5I@QK4e@nFhVY} z*C}YjCj81$jUz1*3TnN!J)q}6*?*L~sb+H{d@;21FteJ5um35hTf_5ttJkfiQ-vS4+`H%Y z@WYlv6DQZTuN`_ns}GEL{w>cXrXo9hUH1n;SG$4=*BDCY!9QJSo7uEx-)LLFFO!nO zy<2h6rVy(d(V%1FsS^-uCN|^7df(*|m;>pfyGYZ44xV2$T4=}b7anzHmBeo$!ugG~ z;(iwYFC{rQiT4K>6u8x3&~2n-)YXApWqeR?BPg1;{^}bqP4TFnAFu3~ZKKn5DoN*{ zIIzh@7+Ta7y}#GCfZk9iJRrXQ05~HI+^;xXFnK)4-l+%KKU~~zAJpTP9Rn`GKZn(W zvjG2GN5|AR)-B4KXe7!}#z6JycC|(s&J1e4NjlMSjT|_)3F&pos56y}L+1{vXr==X z$MQ|HafKdWkRc8lv3)MJ#3eex?0i!_O@RBrBU#YclhZw}mLc)|%(ADaoVtfyce?1T zh!*ul6V4Ku++!{STo9R9ZaLRTPxK^0O_*->PCTtoFHdYQo9J5zT5$iuBuI6@nxMH+ zAjU9HiJ=EV^K!ej9JL)GzTt~D&$*;$=1^*!gT>iK<}mGwBMhGS@u#}=<7GV5EvGt3 zG^U~c6vlWgg^IsEcFaz;?^;K&oW z%c{o%IK+`V*QO;r8;hucqE-Td&V2>fWv;+bq8#Hk7Q>yr^+c*fJ}s+cdg(5EM7r-J}9)i3pZ=`*36XowAxO zP8TRp04W+-G`z753rR5XU5oNkwy5Au`&J_gNhM02da#A(L%>KH>Q-Z^r%G ziPy%mlv_xAN#LdOHrr3$jI?7;rMgj|YF+khStr0Uq6!S=Q}Wbj^FIYfI}%B)l_i$9 z;y*m%D7YyHl5uY`a@lHHmuf!;xO@@ZH@!(^1t%AdJA^dreJq=}PT6Ir_ZZ-RB$o=m z8xBr-1qw+1Buvv-Y_OAuXuWD$YhA2X4cpxI)`mR&iz1KorC*_+!sKAw(Rri?j-;4q zW|i_XG!k4~Y_e5~`x`Gz z-dpQd(K;4mLj92bo#=zSxEFF-j3V6#vRHxNeXv5l$~i7@Ow@rcRv_A>y`$#)pldUY zU7qI>{~^_iv2@wUiFGr`-LJ#Nd0SNhFnGJ)g?Wz^QVrssTfU3p60d8dM13Ql!zQiF z@xV6aOb;94Z=eq_L-Tz0J?XKxX3&b8>i_Z-_(3sx7%1{^I87p^CvT#&x>uJKgmvXu zk~Ef@9ohHYX}w(K>GiiPKH+5#aV)q?p#ELj{cqAvQ-sP#n4^W`JJnxz4_cxk$5a-7 zf9Gv)-}^Rf-caSu$97s$Av}LN=$qzT(FYFw&9q28&u7~KUzC@=&?`cK>mEYpNst85 z?ogo%P*Llv*LljXu|V4LU^qhRu?jn7u|x4?O$lUiT=T<_$IPeStNsnb@-`9n{Jw6& zH(e%m%$BUql`9K+kxza$bKsO4ST#J67z5duDclsgN10GMwUDZ`cC=>mA*{wSu=1CPV&4KZ;RV z+(xc}$p*depMEUG+jFHYg1^iv$}`rFp+8Z^C_-UCq}3%+;Caz=F8I*v?2K?Ud(r)y zLiaS?DYXC{zGAQ?>%oJXU-K})NGb>Ue5Rh1m#jFTQeC;As=5tX?0r`8;=vvC8}a=1 z@iw;|zW`bMQu}P;ym5J$-sASK<`+yx8Yy2Ue8BNM2gCJ6)d z(6;9?HYk0stow_?VGM{?hqEWVM>I=zV@V$KZy&m4WC3M(C4o^i^9` zW0#^!`|bNj1HMTUsuPqs?riioJ~q$p92C6BkH71sKY7=RvF3CECx4bUtf*OcWxm7t zdt@Chn@l2(55cG|2sbDfrafZuqC??6oN5u}o{z)YW4%P%@1SKBSw>P!h`;K@vpdSe zs!GO5o`kW9HS$9O)36uFr5LGx;Yr%?_%2IjP0TONn{^xWM>~iwb}&wL;ZLt4Qj6Of zMprzuT-T=zDc!{T6GO}wnxA|XoGfV*M+;@vM1-^W`TP717b(9-yunmBDs+X*&x#yc z&k_7BY$rOLmE*`Ob+#!Qy*Fp~W5XM1pS?*v(FOz=&c2?`Eo&Zm!!LE5UHnFGk@cQD zeV7B%+pPP5_miwII)C!9Ucm=*ic}@eU{wjO=4poWi~PxZdfh#9iYQ<*h5jkTb<<$^ z4MNh3z-`HvV)Rb~^%`|YLJLXwHybD2IPfh={?Z1;2LL(c4man#o!Y$` zs8~U#g1WAVpHCr~-OZG_CD6Yh3F;@O@w&$ut~AZdCV7wUAPlVfY5oew*tr+VX6H*5 z7A;66t~cg|%Jpo)d+@*KUjq}t7m|*_4r+zFp$_viS-bkv()GpY5@1L&|6{CMDXaa~tW?oF=1)$KkVP zL5cOnX%}O@CQ+WSQIl>xPt;j1CEL3iKa$f@?{lPf#L2Q8XlK0OXG89m)&6&S&gmoEOuGR?#`91RDLCAjvv{M&pVb~gO=bE7~7C9IJTir zZz`+N6nyqI!Zs3V)9GFtv#b~G1(V;kdHk3u#FH#n+&Ldly3SP~n9Koond(JHK_?@5 zWiqN{Cl9Db_rm!}9J_D?Q*cn#;Y2v$o9tdrJWXIkHWR^tPeUOb_XJ{iv&|i#4qwFg z0u_xMCR1Du^@Evq$iqsI`@@CX5D@)S?1!J4g0Qqf0C!BT!4epmyX&Y{pS=U*D?2PF znVbOPZrMj4`2HASm-oeZMp<8h+6wuoT%kottD?U?hWvn!h}3=x>Ir(406M9qpEtq{ zqEbOyhQ1Dt3Xq84IV-ZkT&8w?C$h=$#r7#LEL`aIG9Z$C(_lZwiPs3h+<8=1t?CG8 zKfS4LW+x(%4r%sD9UN(%eIYUtQjDCH?+j20QwK?dJHfhv1|WJmm)qh}4Y!C4R5=&# zRf;@cInKEs0z)xfD&&EX>Y^rok!zyFkPscAonhW4VUR}a>bI4UHgz(8;WC)ccLDNh zl=h(lnOwmukCc_}%C)@6`akrjeiNj~e< zJxi6bIVtj9*a;y_@G5C-+~ml&ogr1L^h)6cg-uOjOlx^G!q$ zE(v!7J=oPf8!Dh+GwQVWqZM3)6#JL+Nra#s6$3HgDMQUbU;8EQ?V zJ*ME%byS9$O|Ys(tNdG7^FQy*X_!1w(S6$IDGhW~Rq0rMs<@uFXC^s)6Fb+%%Ste@ z^qEw833%;JYQ*X}j^k;sU(L6yF2!h%Xue@v)wM#@#CVhZ=XbnDWH}miP`)+>zRTF> zZiHQ;*|HE;lQ|`w`rP`np>M(D=tmK=b7Af4m8!Y?Eu@(QL$GdrhY7_uEkl^JrdM{H zC>SYqId&L3M29O<(``uci*_?@x$Bm@J?UZIHV-ZC4mu|lOs?tBxG*6uK3cS7sbmUX z<$FXWY+TM{3@OemqN2r!kqk^%Wh= zkOS4FvPWhe@@%$)m4d7_t!95=RXvyPqI%JqOX~p>mh?-p!qW9De^0k#Mc&;VcI^x= zKJ1=BbAA1E4GjHNfFx418sFABEzG5}v{4XTX~y(bE#~A_Rc!Lv4t5(GsM<8Mbdq*2 zrLV&P5M<+o3yGZG*_CMhs(h})`OKtH>VGu}-#^3_to`WCmi0EL#@L!}-}rFFJ86ee z>k}Mjj$&+8@g;KIH1yJ8ZB=UO(<}6rRPIEwCe4-7<5@b1oNeZEe>%w_vf(T|49k-Xt2;@GLKRW#gK^7BCG z{-dhQoE7vQqgy_fJv$*50)Jr5w}A0Jtb&?_31>YB6vW?|M!Zz>Q#Z;w8}GmuzJA&#BiS zFRm+p;YHi-K%XS-1TpcK2H38qAJ#7R(TNqD+L7h`eKVSJyk z-HWg^eToR~yTfY*V^yhb&_ZuGe-v=0!-?*zQdT54QNv-OpvJhdq)D@|hIP+H&oy;# zL|a}s5yBQWdTv-qivKH(E;v}01WrcVgld~QL*JE;bw&6t5DsT{t`F58b960tZJ z3UE%ke_q+&QoIGglsQzZPaZc%7;Z_KxY!IgG?r?3=wu}gq(9W|oIQ0~Rgp3|ZkF#d zf9OCAHud9BiKN)466Yk^{Sz1ys*&eIB+Xc?n{TpuB~IcTiF^f5P3DB<=T$$RQ+I{! z0PU8qU-7Y;eYF_?qdkljv;5L_vrSk$--w(8bUN_7q90VPN69M}nuz2M?gFiSkW|+h zMBJ?NF&m{#iU3Jt;R_ZDf|0bVfR^trx(aMt3Yv3c6$-icS#E8KVHNueTPuV6TXeK{_e;+$E~7g?M9!a%dl@q(KnU9t*em;+ILvxHf3if` z&1#^Z>Lh}l3`#$~k0kk+4K3eBj%Yo^WdjW+()CE}gWC?yKJ2k;&S#E*~cEevzQV;w}W-GEtd!EHsBX zj<<>shUL;1o|C1g#k3e2lmCgAIzAr|np)jemP{0w_PiicsRvzukl+Bo+^&YIpt!tE zL_zNKeS-p|jJnJLFqW0{h3&xUt7WsUW@XG9cH>o=LUhdxLGc_|qKtRLsEH7y=;Con zM?rt|T>H4ee91zvGgV~jggv~bK@Ny?a^!hk(!&sw-uzmsuRU$r5w8EuQ59+FDZPfE z3AU6ZpNOr|l;5hV42-L2PBPE$e!SdqP!ycZOQkv2eTaajXUYQSpAjwkM^e5{55X;P z*k#La9qVQWM=fQ1^zWW^i|R%1mp&(X8@YmSmI^248>PM3h3WQU$d%Q>%k01cthX=m z)B@1zqA$0+@Q2cx&h_E)u{gIrH_wK9fAgB;lx3z=zOZebK@|lYRrS*kJGa?@aQ(go zBJF4=?MYZN{L!j4D11=jhT^eE95n6eXr$TCVNR1nc_Yty5mp`*aNHDmjrAk&BZo2s zJ}9HYyFwMF5GS?^IHeqBWTEM?BVbLsFB^l&I6!A&=SFQS<&%>a#ntIdWIft+@TR(} zBkH!yyL#@mpaUHYe|DYA+t8KwJc;OODbvpEv}8x!i(za_%;DGuK5dfaqZ`l^Fl{C& z{FJ(^w4dp%HFYXm!*~^)_cPG!K5)b zWi#!LhW{NpXahUx%C5fss1Vcccv;F|ziTObHwMk?9oy)LJ-?*b^DchuzyJmR@g(l< zfg~PLz;8E=tm_E;4(`7{L_>bc`*I^lfAtgo4x6r-cDb4@w4eP;So#QXC0oDi9h%JV z@brw^;eLhcro_plXNaFoZ@x!8i?UMCvqqwx_=@dZ)xk;USE-32Kgouy*I@Nu<702W z5&v~)6_(0rmyG7E3j9fWb1q7>BjDkRt|fjEY-HAWL5pMeLFaoKRut zO5?l+Y_gQ#^y-xjBITIAN5$$yj1t?nGV57udjzU!P^g2!cQLxLG%S;^yeuQ@6A2Ih z7zbVcgEnNh>KDlEFiFbo=2}f_P0Ah16Jmc=$P9T=YQo8izSNdS6eQh2TmLaOS)8#5 zhTBU42q_x>6cod(m2(k0EhbGA4l&h;11uBr1!9;W@NtSMQF-7fhD+_qbO~WSgd<4jL{W-8Na&?u_}2^95tNx zva|)w7_vzA$=wHGH*&x|UJ?=;1E!5C?+>RQp$&UJy8H(uAyENU1(CWy5~{t|Ki<53 zBJwbQV@m1IKdCyxGJ;Y{n)?NDH-T8QAB!UR+fi2q#}-j=SXKN-Q$u&N+?}4hlo^v9?%LU5Z@tv69|AiSgbDLaB*hN|r&%sVYE3Zt zTpenAKWi}t4C>t~3RTTpY#0m+EzHR0(Ej>I5^!H+qH#{}zw0m2xeYsu??JX0|KL&r zJ~Uc&9ccZ5>pecGTL*6-rN@X0u=DvKQYoD1a%~thZ}N0&Bvdt43U6^a^e2>Aq?QR5 zTLq>B6D%K8*5!3#{L&+bo}U>tawPVf^Ar~fLS3@39tJet8inIz%a+K~PRht;Z|!%V zFD6qa+(T5#g&#;ibMkfHeJ<3qHxGj(#8_;2y`bK8=D&q@-%xkzW&OH#$Xv$SP*Gis zt}mC2gaNH1uILWRZg{)pacCd?;bOUhW%qe5`+Zv`07~QmsJ+&bl$YR*WhV-qoNLvc zIr`8_hv@2qD(5f$NXp}j;Ed+Gx(h!eV?J9YI;Nua-PNJ4A(b9N*4VT_L#9m_-=(Q~ zp1KZG*8PWc@K1v7{~6jW|nFxTGw z_#fze@1Br;cTD=XmGuv1+;1-Xf7s{VB@X^)!0(FrfAC%Z+a-*{=`ukTH5Djf?(9#p z@$GnOP}eJ;QMUbvi9wkNyMS7nKhPACp6h{Fhx zs6+NFciEJmx?@x*OMzZ&i)S{wR1C%CXo?M|Mp#~H!Br7;MSP7rzu^;P{+r|SADxsz zAYGV4za~Oj;?!T+8^_%kdW}}V5s0X6=QO&crW#tc(5WA{3Z)3P*FRV}79GJ>p=;6< zGI`?(^Q15yyfC&CboFX9xf-&lX-ro^&yn`MvDkN&_SZe>|cyX;YA`E<&3Rwy4XW~mUger#^D4jtCXZ+L+& z^;$l_h<~z?YW{1k_%|liZ{PmOsTv7TwIv$V4q$Pk$-cNN58!sW6Yx z+fagWGjinOiZF;pe3jaZo9eODwQM3%!Sk)&73K%Fu{Sb3_+^UJN-(@Yn%`rDI|1Ok z;lNqTX+;x3iEH1NZ~I`p^39w7=q>}qh2S=Rj{SvkrNY?ZS2?8vkPlpA8qy$(muX}L z_kuB%KiRqyE_ZWh`V^AybkDg3O7fk0$`L2`g<(;^ixe0jq7;PnlKSOA4LQ^ z%X@*7X)~b_?3;+1v&;B?{BNLDG%=|EaNN_3=*qG`Z*Lq+6#KD{PEnzWNUj2Uv-%ds z@z5d*{K!TrpQ;qhh$EZ>B?ft2mth6@)1NWoPmvG9ujx^}=N(@R0EOm{c;ua!A9^#H zclU?i{QPe_g-{3{xiaX9cY@pJj%g!(?H~zl@8KMNK&s4PtC;Wt1@~iq_2Z!!?CCj~ z1%VbnPa2uLl%C7psK9)?$uarkF+{cD@jv}8C~KnyCJ6O-&@_5Sn<{ln^hLI+54$$4 z={-CuBfc=hD&6P9JRr6lhR7|?G{Vgre4%mL+XNOJ+J=Zq7i`p#SH{Pre{y#56OhG@ zl5257M>@s#iRoPFxti`|Ho$8Shb)?L6XCsI#rTSh$AqMA3}@k@Ig)asf}0{-aD*i2 ziXWNAJvTK?C%}*3L}hV2_f^TFxaPiacpa=DYx1JW%S>cRw6|80-fAfZZN8J6NOcq$ z=ocb6;r>TeGZ_;0A<$~aD)9y$BJ5J!;Ub>XfYv!#ADGzLu+O~-MZL#$#}28w)#`x+ zYm-Xj+l9x;!;I+Al(4b*uCMpiYDQ$p zX3F+uQ{4e>mFh8KXi-k zl%KHZ#)k9#P{{R6Z`&@Tac{(T2`Yv=1uT4pUPT=884Uw{XOGBX*#rlyO)B@r z%p{NP2uI?`0xbjB*f(wDMNhhLW55TAz z_2*ZHk;VINN;nJ6Y@eY(vxc@a>HKH!$D9h#LTu7ML@GVAfOg!W`RweQ#57R#-pSFy z_)xt>7>z+hFv27F7+Gy_zLjD{YmqWld`gzdzi; z4tyoN#DWb&=5Q2Z@-s>`(Ska`3<||n4r8hq2j-PQOXEbY;woVqQYMHT9S-XkZJ44H zfDL4fX<;w?Y+su7_R!k$IGfvE5h5y>?b_^u?El^_Ih!4WzzvXW_1|R0iZsY=q@RF7 z#J29TR3!2=9Ta}}=(Kdy?}#C%-I=&MG9SefL7UpISSP5&5huY)T=v`fZUP62shsyY!#{I<;12dsnlF13Y0SM=-z(x;gqp-D&V3lfxWbp!-AEQK}P! z)4rX@2Vh|Q!Fg@ftLASo!35?6P({M26{&l+a0HyMQjyGoaOYgwJ&WoyNhhhidE>EP zI+`MQU2=|ltK*m<`ke8n1cPm?*$Yb(Ww28aEoq79Q>IuXrakAs7!(oS5Ut|N0R+~= zE1~Dt8uHfF;oP&TY@J&4%RPhvIPB2#A6imHVf zz7sX%>Mb$M8!8jOFb33Dq7+9b5!;1@&WLkZRnAOR0{X8g>^xQ~O)QdQbP2}q_%^^Cu^7nE$z z`e`NN#+dL7f|Gw#wKweZY`eJS$z(I4#UO*agMD-7#RnxUwF_*3C~%P(d{A^8qvfMJ zNvoQ|3J_nmJ~zNVO)uX)0tIl2$5zNV` z?Wa>v696;w&w0)SGGJ1pPBtT;c8wGjBnsXDTf~>qnrGvi3FXa4rEPJt6jo7R3G?lu zJjsPD4r`M>{4jnm3sT{qIY&Gy5)2wnf(+>2I>Hv<@bv{Qb)nlOVf18xU)fHPb8~tT z<{Vs0Kz3WFzQbgc1+2C6fq_$Xjizbmem!_yC#|7Uo|*KkbDGF?BqCz3m&C^cMD2`w zTy{mt8ezuE)>Q=DsP^xJ4Vp7KuYX$7mi}%=mhX#6CX?*c+sn*o+v0OLO!nLkC~Sw* zzs#>DkTI4&HlH-K)Rb5c$6Js@9wOs_Lfx`^#n4n412UFV#o}vZ8ytUAs+?Kxe9oem z!0qT?3%k_C`fAZO@RxPgUq***O{r{}TjeF$@!BpAz$>04eF5svV=p}bb z#dkHw(b{kk<&I~eYGaJR`Hi}KweZ~A3I$-Cgim#uZ7CWb^!L;ZSqyc1<@icqez`8- zZqz?D<~Y0vZJzR+DFCd~aLJg=jy)a{&^#ZS4s2JYa|fi3ZV>BE$jDw;5=`yPAbGK- zzzWaK3Pt7wEWs$Zex!bA(pjc_8%vB8Tg-1a>eCy)6Ck$qtJ@_HT-{HNq7>kmBDXS#Roqt zb)8;M1XJ0M>$nl%-9>Fx*ZiC3BVa~*S*O6QdHN1Xcsx}^80lI_=OY^~J1Rx&p&RH` z8(bLB+T2E!#-94CcLW-bArLzXvbR|O%miDkJJdx_vN=?dRA|NeGQEEH6ttz?$NI}o zR8;#tE2MvOemKZbBzmK*!yvy+;&0Hh#+Sn9AD7nw8ULtbDn_8mG&-DUQOBgtja$+B zaOnl)8gy8U4eizl5xjhlm67TzS{}nbAdQ1Nt_Fgo&PI+9_HwEJrRlka4)-jAW}<6< zoR;I9gioBwJEVdzqRW|~CZKx_b^SUHOg73I!OR2~1Y~U}iMuG&E=)SVwms}Z@biYA z3tzyNUc{Fv)xSbqfy;*e9Ta2CDRymF>z3;-pmjxX-js{qe2@HSxn26l^a1epjouPh z*~8<$(TJOp=e)tanwu?L%fNBm-fg6#vtklxOKoWLz>&cS?U-)s{-#FlIP zfueG3f1p{bHy^Ph;(|S%`WD?j;?$p+yby!pj+}L(&>e$7D8}K)a{N?Y_#ztTI&1Y~ z^4B}_UUo-AI3k!oKhi!)WPrIE|Sc8!LZ;Tch&d);xL}$+^vUwyDu{ zlhWCzO2fM8dv*Ry=;|j6)bXX~{zR~gSE$55$i0t-zRap3DGf(3-Rn_6lI|}|uHssZ zAZj)Rrnr#RvQ_s>+~=+Nilvf9p}S^P$^hc8R=Nw{tz@j=S2i-)Qi(&(0iFG2pe7-k zu5Ahf-f|JVKX}!jy>upPA(>>vu&dw+ytR>PKcb@ypRq)7|I#LHOrPV)*O!V^3M9qM zbF}KHMNi@eQevmewQwe=$KdRry`udHzI1oUD z@_PCTKMt=V5e3uZ+M(-7&n7v4|FeTXs?9`P=F)KI5``w*?k0Rtjma2aD^HNJ-QoF3 zFEJ$9&RO>8q)QozD7857^ac3-LWtX+f4Bs6*IrJKsCMB64sWy=6NJjhDek+5-TjPb zoKw?2)lk1p^-7iP?HpLwVX zw}lD5N0>~};@p`H8nhOTS(tR_7oGgGx7}&*!c(hsNG9r~MBKU%TS|r_K^n=+`+|+O zn${Fmg;Z!V5N)sXk6>%H2JiQT0i6~Hb|_xFsRa1D8an@0NCzn+m^Af++ICNlIn;3D zV(q)&?$vZrRXp7InFLl@wM=R3N*{E(4+h%swRxNqD(|2yj32?ny5d+?(#!5(ZPt%H z{)beVt1vRiN|AH^(qV*$h-!oIy`p^gf!hkk;4d?{-!VN9 zEqdwHPvNC6Dn4+kEMj3wC0mSF*ysPGb#8debNEvM8lZ&5zolJ&7f|{ia*)3ZjrBEe<6Fkv z?+9}aU6DYJg(MpV+7Vn6>0WmeODRh4qa?l98PzLigo#Moc7Rou zGJ$5-QdY{WP9?i6xtNfiVP)PEiD_Hu!QbWr?Z)N8J8Dz^qB4PMTnS~lorrW37wXou*-dA$j;mqc8nQOQTYy|^2CiTu26Ci}lZBW0X z(^LYrrgUJ?s6GlP0ubcdS`-7(B4em)XWg1Zx0e%aMmk85X>kw{4n~+J2}MvR@U0|w zH$@Hevl-*dT{>4NtpN?S$5;|3%GSCKd+` zAk_dk1>W7k!K0V~t_mMa9)yh+Q4;Uvb4c_PwTreZ$xH8y{WTOOO!S|{mf19#>HvI)^5lb723x9_0 z3wDNvG~GrrUvXyZR<8%jDGsUc;d(0LRv}leEHFP~CqPUIa#;s^U9h1ljuYrM{x_~>2VcEJ`YzWe=~3LaXP$C1Z6v z7D|ymZ2`we#-DBtytR*EeOeacs$X;hBR@;Di(U>?WGXSi2jD(7Pn%FwMkS>?GT;G}I& zV+^>Kl)@FcCdCC0X&{VobuAKJ%pcf;C4{RCo%@$KirjyXteL$eRSrdxz;tn>_SO7ed1Mp7?XEs6-q zKd33X_Y3Ke#Ph)ukXc>pqO5A z%d&^fgGF`G2xcfZK9ataq&-Ua7&C6L?`%!K66XA%!_5OcNIDirV_+fa%y5g&BeWV# znJaTg{*^(nOn7IY#ZvbqV_dRIVb28w5v1DH?aIj;xT%3&s{{)0wDq7Z$w$Sk@%yUo zU_FpWg?!;5KCFw$FE2V#&%D{5K!vbLD{{_w;1 zyly6+PfEF4AsPho0%S*d2kQ(FYd&X|*{>8m5|yz^)!Cw#1*#G~#@hq9x4EPam;T@0aguc>Kof4{FcKMs5c=Do(MEVm+dSbK5HRT?1;kET|ukSLTFT zWi}>y9yrx-Va3@->gwVpG0+p;8B$iKQi~8t62rkwn*a|`_Syp+#nhXR?~Z$B3knySlkcuH|qr!%j5r3Rhr^cmy#!m4h~rItr+_wGWSFY`+B9M|?G$51oVGHq;JU6kQWh_xWGV(dFnrdC zwj=O@j3I&X(#tv>m0X;SRC(jf_D)=#m_8m@p=;_Gr0*P(snntzad_Zs;jj`t%F;3h z)=g87W7gxV$>Pm|jTYp{`33uw98SIxPwN%pmE!*Cz(Qcp2;J8C##GcCzwqAsFiRVb zE*y{@TVdTp+MrlIT7OsbB;A+LkQr59ry$NDW5?plCPxYV)L8VOVpp6wOsE-mTGBFY zkuX3EBO3{;Ys@|oOr6c?BR}6#(ibzo$MMOlc^Zp+&QfV-4{z@nU(@SQu>HJB;|$ul zq&;BCd~F&+kziQ&;N%k< z;Y3I6S2fE~6W2u4I=KRQsH)p8h0>8&E36pLXe7i}W2$4YnLsWn5wpIZQqeQp!K@)< zt)!VW7*tcCE_>3J4TCWu@$wMmjk|&AQ~P$u2jIvgCoMFJreJ zQ>joHBO*qe{|~C$u#kD<3+4ndww+LrbR9VE&>EqI*cs(i!{rwXjxtI!{dPvOf1&Q7 zwx_(SY0YX1Ny#>x8~&P~4Wj=dJxig1}4iZjNGcd(&eoFkNS=uUg&_vf$ zlFNRHVIKp$O^&(%%$<_w)Ra_GXJP`Z4XDbVHLDlAdS8uZ1n<^%0?3rHoS8=xSPTbij1KqJs)RH#>cUy@Hh}T zmfQHKfjLH+?XYHoK?aU4k$^dnk2pX>N`62xzw2Vj&Xa6b$v!<3pfsc#oS^s$H;Bw1 z;_-EZkryJM8nI0Z8t?~xqEdY1EIX91834@9qPz!kZbru#5wnD~m4m9vl4(}Vz9ILN zvAlz-kT|YroLS9ny(>p1%?)dhTfdy`Kn>0#c0@{gcVMXFrt1i1n_!pkm#W1`V?OL- zk#ZvPFVc=JC5oPOFC$m>5pAhE7`9jq^bpCGA#AsAGtNvt1p7HV`sImA(jB*+3)jnE zJ!gkxEfMh();_#R&8?Vbvx|oKw3CwQ-jj))Q<^Y)_-2Dsq1xzr%TkUX6U-WJYu>Ea zU?>W_1A*EX9S@O2#ylJ#qGP`%bQ4d1m8}Ej#N@RQjr0l1%cS%jo{fhYtE*ewb>r@1 zgKJ7U=)A}4=*NyljEvhKWX`xJ)Gu}R)&ZMoR$_=Lz^0ZRJ~h?6C(YEb!xKotR3poa zu?-pZ!1WaDnMGTwVC*L4v^hY{!ubf%hX_Vr!${t^me98|2At-ZvwN}>8GP~w2~$ZX z7BaSI{cnkrw6Z<4uHGk1I!Sv5upP1KIc2I$`z8AenReLms;7k;;mt`MX#C9j5?g*a zl8Rq{*5bQiDgM`LCjSlA5%5HBIh4NT=;Zj_|Iw{ze(@6Z~xyAj}C~b|D%P_y>me8iuiSqMX36@;U~XVNBK|G@b|qf{x=8V z9{*R(IR5`N;7E)de@sew<+C(+KzdY4Ij9pb_qC=^{-hXZWPCOTC%hPI#*7 z5D=w795E<}>zFj+^)9PWb-O2-c zm*INTIU5BQoo%}CUu*095nK|8#$P-277%;KI}~(NL6=r7c*GWf^j_Rj27}tFOW8|W zZbXO9I8vwTP3{UAQaR+lEGnD4<&E+9GjLGAwgU>#!yEl@ef7bG?uyhENy4%5jZAbQ zdX(%}VG>u+mW!d5AWY_mCfEFVDVbWUh9cT!khuuekU;J$574O}-8)g9?hb%f6}bvW zL8|RO$y>6*AEjlc2*{Q>AfZ->xL`2C+^07y^tkDjEX#eh3Fvx3YP3X+Vi7Hk;Gcf2 zne;nLf8)R2O+$#H#(=8ue5am$l%M#H^#J5L?JcSlrzBkkLVr-s=IXZ=;AO4SuReyvEzD#8dH8CCgBi;E< zeIoRMPB>f?I_cQoh5z0=*KD_(JM-b6w3_+tG6aU3_IR+%Zr^|>`(p$)C}Y;*L2xNn zE((sB){B$krtrt`ZmMbARM8t+*?$vF{%7?#+_`7Zs*O0bwK9rPx<)7zX#ToSXyxb% zsZ2_6Gk`26H9;1&I4%!KLR|LC<7Uq4rPCc%E`x0p4L%6<(rf%ji?RWq1B(bjv!f?| zShQ2L+#w#|VGFZSlqsudVB#Ev?!7VA9_zEHQw+7ezkPwG4H;;qTYUqWqUI1_C{`Ly zXp+lY8piAZ>q>}J!n6z4-Bf5o4>BQW_-Yrk#!bp<@y#OLxUvRxP<8CoK?(4)vx-rM zuNy+?I!tQ;sAC@p!$f)H9Ron4QzacD^4GO29!aryzV6ZgYwzmAl04V=SzB|PTD#T~ z&CJc2)ha1HMp(+qrIn$5xYSCl$V^fEKu}U(W$Koi9V`_=ZMFPFXc~Ti$k|+)AyN~i z1o7kt8Y4^mVID@T%5+i+R)Hl(>vyxm}3ww?Mu!74fj*k=$s23yeN zvcqQyW}dFlfvPm}={EE;kuES-uOL<_iG0LkfDI4VVGsrU#)=jP#>FOBbk4K;(yz(P zy+bK{q(GSk$~CEtV}9B?cB%1I77}EG2`Pc&O8(e=R~D5wE+J1j+TX;cZ*^$o*=i=Y z-`@ySv;=0~&RjS@qK@Y}30`srIIb*CQ)Yi~L_o>pXdf(dyEM?-v;PcL zd=ipsS)9k^$ve;~_$;~}^-<10RvgvIk7GMa%IOyKZCTVtZLWpR+op%~I}bXGTACp( zX}9@D8{%CX8pVRh8;l3%qIsfaN00WX@toNc?HxqTXB;g9Ag3gyZs||!-ua+oPGAQX z_}={VCbz9~#iJx9MDqJ`_FRL`&`+{NkOEFoZk45={dl>pVSrGX_8J`dx*ivNfP})@ zL-F6?TQ6WY(df_&Giv(pR;Ek777ms4#tY6B>}b2zC!##z?>+^p7WMKw1eGgTv;eY# zd)?<8!5pla906;h>20X4NjYMXaAAw3O0zB9*?nKp-1652ox_?(mhYe@J@q(sBRfCE zQvQ5mXW4sRvMBl_C}9y%W%&ER*n0Ys%;lF#wqbt?>%n ztXZ_?bV11|4#o0WC{+l%T3LEq?Ig|+2ykvuvWV)H(wD)J0YekfJ5(Tq_h{D4t=5sb z)zObl#}{w& z6TLHBrM#i!)H0pUAC@9XsSRz0yowIeoxb9TKTT{073Q);wj@go)Bk|!Dk^TXY#47Gp_dmWp zTr^KZy%1uVfOF%8gbjvK<}S#aJ-Qd1=!@^lqP|G1Z2X?iwgstE1hY4Wa>c zo%7nGz4MYji2>zXshXiu^J%207N_{h5UMU~GlYS=kkxQdO=hmWeLEbnwaz4LC6BlW zpmj)bXt2_R0VnK19lxO1Z*`Ih@#Vu%C+lZtnULdQY04x{FdIx<)CSOU3=5VnlY-K8 z7N64l)?oLapGNtQ+HlxOxJqv=z;0&(HH$RjGssl!SrH_tew}0rydCp0j3FTnCJVNC zG?Yr}oN7_ql%~0@J+$m?#9iDhPnI+9{cu!^ZmR`~3@|%Y%g3UWGEnm%TXEW);zL}~ z^wL#}b?l$@NMPNOnty$oR*pG$uL5|DmyqxITjsKQda6r#8|K&5=+i$^rA@e;)oh~4 zYFL-x9Lm-eO&W)&cUS2wOrqr?qh4RrUE8_ipcc;Vnu@2y7Z@{z#&PKxqJqT(3ebo;vbRTn+k5T6kMi+@i)8rQV1Lu{Vi@7f zjn%T2K7NKx{vdg8J(Tr3Lqxf^6*>RVx-Vu@8v4-X$@= zcHM8ID{4^mB#$_1cqb`d+Vlh0P{3W-kciNQaVMF;+-tRQ7|NGEO9ENIQ!{m@z$wNx zWL9YCk`-seYel|?YqZrGTY^#N$$H}aDb*r=d zJ0IAG>Tyv-j4CzD~2jLe8sA@EDb{}}{62C5T2CSo_6ztJ7 z7f*0e=^5pr~9azV91cUe=lc+R@Hqqpm;2RVl!@3>+?eUx-aLv@|s z+UzvhdFJeLN#_u8cDN^=q(&DK7xIRY9_H$h=KK^*E|(}rL9=sW?jffNAH|eK0X*h7 zE?r5zvMZcZBy(P_s{V4M`eoqc(Y1U%?$xx!oZK1OqvysW%5GXN4~qw9Zs}#DXEW|j zvP3D_4OVmKX@@agO7_ZIsht!mH(2oF97>{ci2QI1>(xci5N~>Al~LM0p0W>zoc`UK zFBLP@;7ct0ryRNkO*H6B_h=j&=mPT0-VmUB>@wuhkE# zCdYnA7qOd%ot74g0NUKTHpbhP%K?v*XXB!@W6j-iqv=6?XYH8t)wx7o%Ckb*=}#OF zWl7_ZYU!$|q=cjOx{*N(@AGh0_K?w7JY4bUH{PCYMK!qNnc2^a;@&8}|Jj(X02}bj z#S%E63(Zvjh%Qw+kKV9T>`Em{t?Bg{c^fn)UXaTu)lnurEP-OoU+*6H38DmyCLFYO~4g@=CREesAzi-x0R5*FNq>Q&{NW$BAWn9 zCJ|YP{$a_;C6a2oEvRh;8#kj*#ZgJUi zXY>_I(62|7QWq+=*)1JT_K7+Ho3+1DYA@%$c9V22skIfHJcDc2n)i*pfb~L*0hLeh zEZH?QI%&k+)hs`#Nh$a*epNvNRwi?c2HdOimkdThcL1YqD*8m;-X-JVW_Q3Q=?aB8 zkVWq>1?}chI-1Re^I+p|N2Yz&(!CQ5dgmQ+pyXp!P59NN$71al0O?)R-E?p_R(-s5 z@4y4WS^g2P>u!22{Rypoy-T8-{$KL@e_mAGsz?9VI(4N!Y0%WEI%WVRa z1SqEA{roFp(KWYCW-sPnnb)y!0q4ukBO9rER(!d<(m!!$`zRdnGd-dg;a~HTB zTcKIJ>c_=zDz~pX5A$z(PCk_P^eBrnv>B0?Fnu%--@l`BE0l3kTJo305+pK?j+mJJ zjD73YEg=CqJ$`0flIsp{nr$4~%u(UD5jyAZ(^-l|wCI0M-*TT27yfhJ<6%ztemN?Y z@A6OcDTzOOMhG*W`n@M)__2#4vp94nuy&5zJq4c4DK+tH z5ai_&t$fVbrM@Pdfg!>}2P0K?12=oI8A1)bO!R+0%u$TLw%?0k(}x(uNl zt8GV?gV0}XWMlnZT{22yH%K9BuGCDvJEZL1sj5GT(F)Zn3j=6T7gIB`AH^8xjb}K6)dTAfoi$|ji-(sY@l{8<8u4%&ZUN zaXEU$5jIEyN?Jpmojdv2t8VpzdI5}7L{1Ss0)Ebi;jPTkn%q6S^NYw>bd^`+>W$n* z6O8BuViQ-6Rh!UUB0V?lD2HcD{**|XMUVFZ(^<9*F^QmyZRRWDl`l5=YZAzP?bCF# zc9PQ0E?r<#OA-hLiGe1#`A3W*c$LUiDQ+?T9ZfS33c)T$w@%ADyp&=j6bE|W3@+jsD2CgJN6dqs;Ysjl44_7 z1Xfp<#_A|dRBXnAJ+qP+pfMWDV3oefi;8=LY{IXayVzMB#@?KsvTO`f#=bZ?@2C59%76c40}7$*wW z=sQ^*439+(KVV_88l!#$g9x`bLm_pNeAle(6VYhNbCj|G=#~!uwVjlOzb^13F>B=? z9;Pd=%32x{dq5l^#^8@e;lD4!%eKuD|!qVGqxiyWr0c}Hdj##YQt^8uRMOTMb zQ}_weDg=$O@+DvKuvNS)_=VUl0RpPG#Vpg-pf+bBv3~5a!X*}Ob%0FcTfbp6M<5&$ z!*Q%1{#MSnH3uD_>A#-`B11A@;dP8I@NdhlUKRfy_qvZI%RP_~8-fA6gyZ27j>iFQ z|AEtwiKUjw8g5h*;94lAv2J?z{nN1?@;2GA{N1#uf34(}2PfFr1P1w~P3IE!iHLSr zg6lCh2;0Y90(KbXu_=ju?E7bj-jm&u>g&8e_|y{C@7EoDtRyGt!@m+Z0>AvibYWX* za6?GVNA<9Eu1*90WPjR^xZsOKJt|& zB93U(Bl#$VJaw&(kS3`Go_%thT|C_#AvO7l>K@Aga4*5i#Y-^lWQ^=|&|?g8gj)=O zInbgFGJCPO`t@?0c=C{PpLr!!H~fL)#lw&%Sf5PD1iOa;>_z&&iZ13nLg$~C^&wM? zI{sIQm{`OozmZh#`96rZpbtPO-hIYmn0EaEwEOFK8I2|WV`AZexuH~~rgWr4TM$rh zc)w;*7GlsN(6G&A^1gbM&>gI+PMKHsNv*8h~B@Za%p z@+39}8iYty$-pN6yy+e2jAyuyCAhxbu0D9@7Ds+0v3!K z&(o{u4q*N8cNb~aM@>5es}4|hQZ=M|R86zr_dER}@i3^;Aw-C_5&XTa!j&QuLUb*< zvF!i4vvCv%s;>0A7k3Je78rC)T>SS=H#Z&i*?>|9{@CEGZ=FfCeo?Y;Vv} zNe0CLfCQsXe^=1U858j%1}OpOK_%)KhAF?XelhAd{-T z_aCj-zZ;-grz9UYLk-61Kjt?Rlgu1%HyrsNaKs}Dh&o!vHu1-YEgAGNXI{Q%EVw?g z#;a^%xY|Tqn=l@Z8s}O`q&j-9bI@z6R6rfVDHk4D42n~*=PDI>YvnynQZHrANMK!+ zn4MxO&J!ZoPa}qGSPIoOVvV)fRP?|OxClln&Tc0qkJ*?A%85z|IJW$51iXGn(ug>m z%6G~IpD!1VyCeEPA4dqe5z1Z2!+okxx``%T|3FRuNG8djn))Xu5B0kH&hmj|>FPYb z^xNzewdI==@U>Y$;FAi`5%%8RbWL_nLCq!kJ72kK?+NMQ9gr#VGF>QZYi%NePaz_A zM%zoqhfr$;nah+@jp@y@?TgBYe))eezzJEP`eT+$(wQ~>^(cAYE4|&1X)MfgfMe4) zGjR%`cueFtUIc{J;trUy>}b)?Y)F1i1k8XEW^;n~+B3G?gT8mKm1IF=ImySqrMBkm z?^3aRAV+*?8e|T`s`g;Ynyl*PnIfuXMsN2>4w;Ck)z#kObJn^GHu$sc-l~F5qrd4v zLUZDbABNSTC$7Y42`!3+#)lbaMO!vvo!aX>f2UOZS`$PnZD`)69BHBrYq@={4X%G2 zm5K{Bg9lJLEJYP}{2*#HpSy&Y#Be%jN6MlB&D+EEpD@yc=m_PYnh-B0nFf)*t-sMhEym?j@@FF!Sjz`7~UG+fJB(>L0c^yqQ z>!JxM*fML7q#8y_8wdDq!55vX6q3VaY2^C)T{a?ANETL37E8Maj%WO}O$T zA0_X`jFUZ1x@>Fm^OK5cjY2-FC^B7$KA|5UnuDlDYOj(|UqJTSp|@W@D$;vHbt$`o zez`bY!G+DCuXv9klpkuOkXp`fr-xo_f=P6DvLpm%yyBd~6^n`=&fCTCw8|p4t0oza z!y@2q-J1)*SgnVvm{bIG4Pu7}D^(-yh0>(TM?lF&nd?9V?jQY&xM4{kDJN{4F5-#6 zr?9_L+DQn&F~Eew&MQNcXO#)^cZ5sbWW)&d$_8*fBqp%DP~xZ_ZlmY@5EgXiue^9G zFnXz1WYX+T16*eM4)s$W`IFARzZWp}vn>%Jw(({zw>-n8#Ce;?{V8*0TLJO4jItE! zvlsEbzLrsCoEQn-EUWybWOps*(0E$`teM%`N_A7Ho}hrSj@+;5fV!e!UR=pS-cv8iGg9jEKpO*^(XOh<{u1No3M`)g^WJN)fxHB|Zt@6EHjhf$VVyB1JbM`8V!HulOJ zjw_*4zAqU=t}wW#12>XU|CEE@VS_RT+b^Z@ir1QFoq?z(^*ak?=7XH>aCW#x_1Ey{ zDCTV;?Q3fXAmj}CZb_EejIzJoyJ8CzP$!_7bUvPy09CO9%G1rBQGxiUJ0w`uGt6QC zQ`c=m>qYrxnWpJ&y0SV(c#QyJf%!77)>cFY|TE~-bj_qEY{j9>6dMKj6D$4T7gBSDulUoVI2sDLm-rrQ4t{Rhqqr_ z#ZZjHWmWZ_{$Y84d`EXAhnACih%_B?j(wXqvPk@HrPZ1CI)da&bOE|}QX8*C${zvk zLd~%Am(){lmd zI^BK=si+P7a*<(vk}5--hJzK{{Gz|Qdh>_STHdr}ysYXlNl{OI90Yf+1C&@%7!~{O z#y$984>b4`z%CdLLiC>61d@|#c<`%GEPMXqsI2y{2H%N{36TY1fK@Ps8Wx|kB*^G~ zt4GWPoSeySi)=Z|e4&NpDn1HBp^t^PSP4;O_Nox+9#J>q&}nDsFNxt=S(Pj6_lTIo zr2vZveV$s7QWFLsWjTe$SZT@WjPOeYJG<>VVb_2e$J?3f5u6dfQJrLD8flt!k66ht zMvIFnwD<|8aYyc2m*ebko@xZ|JHWy|OK084BLBVTU9SEmgdL9Q9P`(1gBgOx8WSc~ zRt51|&QO(`gavh}~{Dg@UoSX337iDBO>ffoKx@CN8nMJAi$nDd|<)RXT8VE_)N9yN9D%{=oMsz9^lwq{&A^y!E%Y z)${WmS~R4`iE3emNtb^_a5V3tS8k5l3~lB6M!k6aKs9|Loi$ng2Da_OWVjE|Ty7p( z|AJ^+6LroxU{if!b3<1vT`cZ0ZE=j=H7UO)HQI7cx+!Lv4Ox3$dk+Ri^~tGv7o8?cIVm^l8fk{hkmUjhMhtm%aSDi?P
~{B*+s(Av$3dUrzV|Zc&;8h!!?L=| zLY{BHQNV&6R$jwzu-U7s9;`q|)E464nvFz39n~;an=S`zL)}D#EZuIHz$kc1FTWVr zR2~NIp`W`Cjh~XyY6tX-4F#h&JSGTrwHO9QKlqELW*sN0ecUXcDwy_?rbbkUSj;7q z{-p`;!B?|BzQ0YN|0QRKuO}qB4!fB2yEmf1*;BCRMi0#Bm&tFFV*0OoSm~Bf?o}GS zI;7uV!&kVN-(_tCXfqt)Az}TI-tBU@fgY4YU&hN_sE>dDk)QC#oq>o9T6B|7<44uZ z9yse_E}36*^oHuFKA^85hvvNV$b`}N{-g~h1y(^{ygm9TDAyG95r4=$Yx|7JaNN5? z$yU&Kz5rD7L#$6-$&E3YC5TMC%>f@Z<5x{BLsgfw%`*>R=n>(xu3XIK>t`1)D zw>52JrSlbIy9wHeU|Q-($U9d-!PTZ|qd`+sUf3Holt}|$x5QOM&4}h?u4y{G7}n+W z($o&fF)}1p!EY1MvOkPkWBm?Oe*8uHmvXz%Wm))mE0|rK;CQ86UbPz`EZ7*DM|Pq( z8|hC%zjnJaRtMtSoIULe0_t5N{g2Iu0PPJESrPg1Fdp*r6n=)IUClmmBd{Lb$F=Z& zU1bHU^C>6?)ghff43hg?JUi7*E&kH7K3?&-{X24OvViVA&98fp>OB}6GPM?R{-RwL z?xBUEt;)M>$T71ZcJY(|)WMF6Y1U0lJVbSQD&Lb6;}~QILD2|%CzupN8pqq)pY4z~ zr;1#oO<_jorm1hYGT*ISTX;OEK3(0>YA-I+K97y@Y6%UNr&yW6h^A}kVqkOKARDu~ zZsFd<45NX-+svcJc%b_CdN8ddnt?urRZ6@C!8B(DK)4yY4YVcmZy#(}v_F=lTBVst z@(*%brn}QmoOoAAA;9}Dc^OcI^HkD&eNHkz8S!~Khi{faGq6GH zDNT+>TKFTcV9|nA0uIWoScshN(fM1Bm`++xA*aFDv6ThshJ7dT0-A7<^0+8>ndVyd zk$PDzcIlLf6u9Ur44t;d(0og?Zu7=#-t8*}ii!YUx(ZAiF>ePvh2OS>;P$~CQhYvV zBw^9J(qI3aBwypbIg~a`A|^8JDayF+(1+8GU;DBz6f9PaNmIqY{{l#v%X?DERKmg7 zNBD)o5wWOk0}*p*q8|ntoBmmGT&_8Rl#-&Fqq?r)pW^|{J!2&7EYLAC^PmIU(%>lq zW%-o$N@bO}p=|`8!Pt0;f<{X88%hCmTo}=cMcYBlK#pJpSzT&JBs&)L> z^40Y9ZxZq)eApe_)S(^1MT>fs_NKnbX{&q=LSpkpARG&4+%$&3yQde5%90(F4I!Q- z2PM`>8fll-cWG>vrijovAng0?4K7A=-zbT4K%Nn|=D>omgv9)DrcNZJmi9)oundVj zW1u$}QTgCbgETbJl9OHcxt@84E`LD!+wB~u*JKo04;GdgC7G8t|%+#+ML zBlB38bt>HTn=7p&&Dc9TjmY)X&lzMr9_-LW0=cRnP%>`dq{a~C2({`4fm{OE=g`al zS(X{Br~HJ~`hl}8x@yQY_^0@xtEeg`Vy>6KDYUj9^ar%Fqs0N*zOlV4(~RSz&pMV@ z3_b-g^6;>DfNfLC&#=9-J5kf`F>wQFiKR2p!dE;dN)@*b0;)Y@8DZOxQ!Soc zcZk(k8a5OHn`k?Z%wicuP5^lGDvnY1PxGbZ-<{le<{T|Rg)(4KArCt;jcyO@8s_l| zh^vv>kFo4F8Uc7hAh4U^KsV6e0Uw3e6;YyrZ3YxqEKy5+ekzm537xkzyG#H&g_yc<9r2APcS^sU;a`9e-Jm@<7ObZlBo-p??{^AHh3U%;tr#Qa^)iA=KZ7bujMM9QBrg z`oi(S?{M@~p0THqSk6F9%@9?auAzao$r=q__fe;$>R89GuyDyNt6D$(SUDS_8-o0k zL7P=!NfD0=8F$iI;h3n~xM=|{!7xG9Ghe+GQQwJRMgVT{6S%vyGAW5;n&!kvh|kh^>%E1)x9;X4_x zI84!dl8B<|A7<_ajMIR(j+%0~<=AQ8?~?6qVd9xb*=5sf5Md+#jwPV`gW8oA78N!9xqM!{`BC>S}p2lHJ!XStkBh&4L> z+@pR!|8w!%=QCheD8UN!9I6H}_P=6go=!m++yA|L&$DJ@F0E9=(+|7;XRDD%^R#N_ ze>NE}#QENa7{&tsqjlYXBm}k<6LrN9{ZX$?-$$MEZ=UV{#l%eJyO}-ctJKw4xA81e ze8;!9QW(SXvM85N)g=6UUzkfu5?n{cuF(u!M@c8H#g)X!pP<&OkAL1DTlROdO6SSC zq`6M4u@|ouJ-D0fEACEyT9FqHfCNyL2eTjF783&OPK4J7TxxYJO(?GzZjZkVlZX_FA()MlY^J@7DRO)pz5zvf- zYLUu}T)gGpaxM^%w!M2# ztmZApYbi7LZSS|G@M*aN6oSawvt(rYj(x$bnR1XaFO#0TIbu_(pPpL(6m7b7{YLI+ zig85DUYP%hdDJD8{Q+qR6TrOaDRr~0vIeNdlg2=xrQTy~$YI+*sTY$lmv}}DjSf(G zAo@=xa$$$rR<4ck2M$m?Rgmw>L=B1$*DXv5lclqY64CDoUw2?fxE_#?YxORm)XD5! zwJDn7e&B(b-Low0JX8md(I2?Ph30&9fEvpu9=SgoF=f8_JOBCV>t?8lk~5|Jx+tBa zXGM?67EI336sHCTDwpZ_4^@lt_0-XN7*rFo3Oy`AZEmkjDy5+E?Q#(kuKOj3U8;9o zAj8gH>}YNIYd%+@T!vljY4}zn%fKew*ctQW26{Vk1k$l;RrT>101tHLaA+J{@ZD7E(JO=i6IEqR#% z{X;xAxNWr`JJJ;xPnm0zMHXv-`KU`H*dWI0%Jv|ovH{x%4cpNZ6pK158d#g}+lunj zY{{vak`48zit{MbTM_8cI-^OCHA2Z3_<+CEm3U?jetio3w%(NR;&FK7(l^L0Ex+Zk z?nLi^qeE}W_K1OhryDmuG)@a)ZDInCeB4t}Q$O^sIYQaE=kzvJ3}$K%Iw;cy{FfNE zgM1k&RlEheGR5=J#+s!R?QkBS)}QfW!U(y;ZSA3R`XQ2|YAj99rNdfeEx)%_*^6Ip zIkT@<<3=0?NopCdt$0!H8!O13;xki2d9vbxMP+O_s=L}#Zo3xXP8>ZZis4ah-8xtcst(FJY zDT@}-x!aVU)A}qbD2QXtFi z!k2-4=hMFsjPdGI7L%Rw1-Pn^%~#NjRDD>%-wwVm9`cGr+pyg);KFk?$SGe|R=GqK zF8pRnI>*fkPhpkY4R?2_6p6=cs#1awEE z)#o^F)Fm%WoOnKYOyv6t#o61Y;43{g&q`Jk`${IF+LJ zx8(D`VQ-}wO^#TQb7Xa;tTd!;`pmmYx%8HX-|vO+WnWwUiG?q2;J?>z$6)81KB$XO zj_ifbmKYNQ?!5iZe)kPyeSKPa#VlKSMX-;u|(`JC=(X41T&}JaqQ5hQ`d0kCuqs7{fu;G z5~U72ijg{Yj#%fubFFi*YQ$aT=~q|0SOwRvQk@tL;fH*Y60A`?h<^!k)QG@g@MFs#3fc=Uk+rzzFy=x1P@wU&UDfmfzm~T;<^bVcAf`=QcLC@q?kLcI5TXn*~ zi`4rfoxb@vk53uvpcY#W_7*_bs}+w={I~!V>O+$r9b=)FEqL$_@HXkCp@Wn;;>GQR zmM^txoT5cn0Iakl)%u9N;F=mbAqH!f&aw)eXkE*xyk#=kfG@Jzd*B*{)VcOh)fL&J~WN6y#q`w=xfFC^(Ve9Pmc~@HlY? zaq0=`nfdMCC*@af>|l-_0)n2OvEbeoxjun?`2ymb-gbQo?jDxppbr7Lc_c9ibUXUF z>32Y0+-RLA{8>zA|G~S0^=NiiDl$+V(Tf@=ciYiPhDRXkl+fYVu|>v8V7of2yx$YH zT)gvyxnJigP+w#1t*nwd-wj|e>~CcCLAo}IvbTdX_^>%~EUyL7kT|2deYWe&{v*a~ z_fg%MjDe@P=x^Hbc2BH_TJigRvF?7d8Ix|t_TWbr9>Tyi@4f1t7)v6#EoY1+ksc{m zxIg>`bq;W_Sx2eG#H>|C8XVdH+emmLBd_+iq%g#UOuA`Wxrb&rVajx|!Dr#qAGnvTM=!gW>o+gsMazeP#RM=lza$l=@}7Ya6;n zhGy6}*X?Ih7wMQ8c{A_4t=+OG$$Si*mxz^*fAFfqvB)^E ztY%${@EO)_Fh?~!{_F>lQ3d>ORSN{mjBCoqTA>t2$JN~?WL=lDu`kMjAGVNq7yNQ| zfUt()Nkr@MYxsZi13TAP>gTBueO*|pqMB<{Lsfn<;hoP3vu=ibJSM(k%kd-Bn#4|c8q(((QX9tvg%H?_x80kd z7ovYXnh2f@2%#L$ zUWD5jvrW6Z0?-<=?Z=2R*1ShCHV(9gmK<93Ja6w%l6TP4 z{yYR1s;Qz-7F#mRIGPsB`|SaAAC>Q`#DzP4D)%9C=Ff=d)M!FDdr^8;ODccw)Iy1o z7N#0WuByLoV)8mEgi|bM52^5VfLhX_CC~%2x2fkmpv8mhkhR*zTLLFiNGhL|nNqjU zmcoRwi|r9?W4baGY4B^7>>-@I$t2Ez^udRk`XqqS)YN8W4Wi6bjCBPnR_@Lt>(S&V zKjc^uI?@|+M1bWaJe%!zebrb6`qV5n8(OU zYwGYCa}8iaHMfAjwkVfs`#VNXhFf8h?lEhNPaC8uJ!*GM!J442_T|T@t0o_YU6OE- zOQ$MT?47fTn@{0M{`%9*m%6A)?NTLmz=-{i#)cbAwk0Fil!HxpC0_MQv|H+%wp$RM zQV|m(BZ3a9moSo95cBS1Y=z4vyIrrBlm*})%B~MKRz`MASUsY$1chED%*SC-5YqqM+MrwRH%>!%I zR)>o9@m+@3Q^uIw59<)FS-*8=c4m1P%jOgC&1U55S0f!Mt_1D#5J6*n)&%#(Zra;F zi;paE^4348YzGtkP*@1|Chur0DNB1l%72!z;G9dU+ia6hLbEWdP2J}rHTRzUz-rlf zY<%CXl(KLWF(MA1N43 zY|up30*W!Mqcbx1y@Qoae8ssYx;mYu8yL7jT8>6RtPDrKfZPRl!aaYM1lzGM?A<81 zI*~K7UvIevLk$%#;pNoJe1C5H3wG<9g}Z;2KogCGJmT!yy-vNCOYlKF!u?=}CW4T| z`99v=O*7w5OsqyG&6e?_kx@Uk#mfDdG?^2&XVWZtxMzJh*0jFTh~<5io_fo3ZB`@6 zEaVx>-xj>C`1RPSV%tz_6>X;=>00O|Fxt}}E#nYk=L0rrwKQ`P+(xG^0@1cW-McdJk8N>}dXgb(-KCTZ3Q$U@vjf5V*W z=y?HZ=G1hd{~N#1XAA4mtqu&D973W_SNjPo}tivk*FaF@kbXH1rR@NbN zf+=Po;=F#-aQi#@Lj$#KlL`v?xLv`*N5X}a+RZS*He)7VovaF8t-5wrkaO3H zgHjp;`O<6#+p4+7Hm;9bhb{l;u=x#H5ivcHk21V=IpJB#$v0-L-#f3>%m_O!<|PZX zo3Y=G7)QrQx8!=@!N`g_&h;%S3RZhMKm}NdYK*92JujAuaY%bj+uy;P^=7D*nohrh z5;Ugv=YpLLy|u+NUk4O`>UWe&%#2BluIQb049Bve!#!3Is|dL)hZmhCB^(N_j;`N( zPq)RBx3Vdfy0BS9#0-B_&dkH*!X6#qZs?sODFJuqrEQ;e;3M$2l>^^Yy28PnkbQ&i zax-Om?!JhBY7SHQirdEW)7QUHXZ>;G?`&L|KRbMD8tXU9VXN;6(;JW4r6)dwoO(yhcQpuRyz!yDYcca9qRM$Zju60dd!@QU$w%zYU}6x-pso zK)LGbzYl|d**??fIJ=BW4znWaT!}iz16nU4PSJ_4!|OS`QZ2_1`#mf0c1wKQS0+Bv zg5VY0ryy*UD*F_;Gm}OsrXR2TULJoreo`*@N{eO~Z|8UbeT!wdwI4E~z{*?yE4y%l zqOZTtPiV8M74G_XEIF^c|E-yx#rp04SDzaE-&ehdOz`I*hCi-A3~Rt~;7)AjLd{Ff z9f_4lvTXvC_{65lm#_X9##%~_0UW~&nPH~_h6E&K6#zpWV*rTdckvN{HB7|F9Mm)J zoZM-O<_>-{m|lR9?{ts6!CBkQsH!`hQI#A|Sdqe94p4%3sa8<3jtfyed8MwX z9$kZaUFvw_7TYXW6dRV~=$5(pVnXi@LvL@bgIxPc0@Yd}Mr-o7fr}-u6Fgpcs@*N;4lM5YUT=E*;X9S?^qv9-@F2((>_P~(ay`!OXNW)%S+M&1L{qZE) zN_raENh@qOY3==E6BLz$0)9YqlPAe z0zSWfw4+F5vB0Nk>SNgN?Qg=^P&{F@c0b}}i=Io&C@^y3<(&n8ZCA}+6KCr*4iqfV_CU=)fY>m zx(UB=D-G~lIvo26{bDISx|m2FEf`r>3viPU3<8(B$X9!nSG+WjxeI!BOR`!lgJArr z=lrgm=R6XI(?hqYk)F^!?@sliT0|bQuTIcb7ij!2qZ!_t|0L$jK?ZyM@a47=WmrHu zi?oV)?Gu|qCf9aWbqsIg%IY+mzKs-`N{g2EKiS}Vi@!U%RMUR~d!C{B+m)@OKPM~0KQ!-*5~w{-1I|MN?e3tQ{Dc9`@dPB~rROW5GFwJi(lUmkjdo^UBNmeU`)3TsC$swu^2yraL^SXHN6 zL45X>?Z#}0GJ-Y*zek~!N7*-F#!gl-Lh_2b@8C3`yIThI7B@!Ksw=UU8Oap#ULJ8w zTXIwx@O7S)aiB#wPvT=^9afd?YG*X@-BCWY=(BK9R}TIC@N4dwxjAF%_(At&)?fW~ zp-9I$){I?n%GmZtht9-Ba#zOX;uCw?5Q< zf_|ZGI9U&!mQBe#M7E>kCO6W)xCkfpS=5dCy`>ouYva!M?yKJ`rwnNv$_r5k$7*e* ziOYgrk4k)M^ZhkgndZ<~k6v1NTYYNfjc77s``#(J%i0Y6yEh^P|B*`q=Xg%D46)Ue zwOvj$&&q3mu%C<{Udd7}jp}`c06BcXzOwW1#3Qat3o?=|Vny-zHoMkX{62CxDzqKF zuB*Ij_yy#SCkzHIFZZqYA@gc<>ua4Z>`CZHbzZp!o{&k)GIe6+#;AURaZ+Hnw@t%REJ_ulUYxe9^^nZzBzM_21 z_pLd?V1KUPyJnfQ@=QiX9}x;D#wp&VtL5s~KSQc6`ku`c)l>PES3nOC+1DNaHF+cL)?df6A~I1N}%+j;LQrTI2d`az{Gdu#Wqt6`w8~35P%7 z_W4+I)xEuk;xzg})J?$RI=Z@M5ibCK5T3!Wd+Krweq>w52j!YFzbF)PyziRlg{S_m z21VmO6?-`Tjm;MuZ=QmF9bR3&#I`D_L?XSZKfO_$#qS-geI&dU{b^f!fxt{r`5R-o zR9pR{Bw$6=PJ9%DNnYfbKGA#ihR2}r(p+71DbumM17NSdtOhl>9P+YTjPYN<+Kme< zuRS-NpCk1rm)QWWD+1|9SFOxW-eYZo+}EOMv1{dob}2GhyPWUdxmdKpDq4IqI8Dip zjx6-sD+;$`*7$v*2fTN081VKm34Ugm{oYxMbk$?;2^Ovx1Ghv>deH{7r#nLt!CAEx{?xMD%1_lMf^4Q_y(WrU4*E&3Mzr!xttSzC zmhwm06J!atEYU2!j2}8y?GQcGtUm_!16eAp4Zw*s{PwJSm@;ZAU)9L7)(CEuXOTAT_Y^Wfo}FcIKU%Ng0{74D6D$yioR^4#0*@sjSbNAu$wto2maCDPsQPx{H;Bn{?ZXvie8MRRoPL|sI1 zZ9<^|Q6;$@QLurvpHb-nctcNNmQ{PB!-fn5ObOagF5}DAuY(_M3wud7(q&eY#zOGY zoB8P$3wMUveT=@I7cnf$89+@b!5#Yf`XG_bmCET?n3pVwy4)-2FndF@e#g;m;(OR- zWXWN$KR2M=dCYEaefNXPTcCtBg|KTPR`_lDh^a?kQIR({GE#R1%kPQTmqCP{Gp z;axKAl5$=>^EvPmtIi9OgOJrXJyILp)3-l*kOVdQC(reZ^LyULyL1_I;$6c>aSL}| z9D`qSjaw4GELVT?6X$VJ`u=44I*8xbXT>?;XW{v7E=b1%w>ISM)PBeW+)nlw2kXrd1AYN}>mP2i{UfUKg8OipO3>Y-cUj)4R5;}5W% z_cWlXng#gm5qPX!v-7>Ns!DOJ$Yv=Xp1}FObl#TI%JcP1Gfl)^k3#TNdShfM- zl@Di;-6K8q_m~n)=m9|0J*5LTpxB-$;|1{Nn1}t*->4Y8#DQ5QA+0%4Z zM*C--pfXwS`|f$wVrT@BSRUerpNi;qjo-^Yk?O8?$*QH40qN zC(yB!uiE|F&RxqF6!s$iN<6gQ;o<(n26>z&1AW+`2)r>m^n9;(v;FxyZdqUAH$?;$ zku%TlUeBuzHLi_c=3&23vYrT#N#K@My~XIf3SOr8&UImHwBe=XDKdPgFe{pshL0o+ z`Cp^;yfAoZ>b&u<7)LzIvIt_g1?semkw|WxnM*XpMpK}bbZO4p86JM@)ez@J$@y4Q(1 zvoWTH;EwJ9N*oAIQPF%=aeOAF$s$B$Yc$XRp#ee!S1lp$y)Y&>QdI?GSy@ybiE;cm~d;AxMCzv!3YsDnnYk6nsQ&CHKo}suZemMYZXq3s`NrA zs}6fm(R}0Jpp9{%WarD60?{bE4Rr?-x>mhwGHFnAv)p|`KMxX8PoGddsmvW)78~L% z2wYcH+*_!t^~@fLD$v~)OumohL<9tzL2$d@#3L3RX3W(^_~G!?Z4`)m_p=JvkFPa; zyy~a{YEI*Xj8!9y8W==AQutQy-uWgr$!S!$dkw_>ei!A7SAwedUc_DJP;0iwdD;F- ztIdM7w`vn|*YToOQ<^4r+;zU0iwMx|D5&v+99}SOS1hZ>JicIx%xjBE5VSwBk+m0h zHgOL?_FjOPPb>CiB_H*$lr`Ps`u!j5y=PdH>DE6wGmbKf=qL&(0;4F1QbZ(luz`gp zphyXz1VTwdgwP={>Y&m^I-w{k9U>)!PzR-#&=W`~N=bweA|;fBkpH9Z*)yKK_xsL! zKAh`1=e*x?JzUStz1F?%)qm^J&vJEIP)%5iucmDg4gtzg1?a74&bdK-;|nru+%b@) zwRku8jCVbaz9>_fRC+WCTgv(MXbJ0~=m!~<5G5u`MfdoRENUc(c%8b6vzrk3eXx|t z7Qv2fqJ}@E5H)lre7iU1)(YU++c;2d`n!wx;z`EoC#!Rt@+1_zxAno=Y#6URAgf#X z19vtgufIG_N}f{5s7xL&c}8l@7i6Ipn--1@y9 z<2NKv*H7$WkJ$8hmW8exk%PxB*aALQNcY@E#_06AX&n98<Q~koOiHrA$G&Fvxzo7BwiD-_aa>yfz6Nwi;29KfrEO@e-YK5-{O{{WsEy45`l zJO^Y%JA#SOB#=GXt-u@dEdldJFe zQ~WQC`^-C$;;q`n$yJB?!0iv)T4cOR!@={%#4&Bnie4=opG^P%9eYhoL80?o zpR^`CC**1u8-Wb%gi_R@zbPN8AdG3~4UwlcWayh8Vhm#;s>^}tfzd0*N`883z zVd#1*XBiaxKY7h9f{F}&zf}~n85`OwQWg5CHG>WZP#ti|zb8;F(3c{3?yk0+_pUm7H$uNsrc6avB|xTQx@5!RK_NqDAm^83)bj_^=aG^s>y!)$3y}zQB~q z%3uF9Jpa)!L=*LSVZX{Zu?l{@=4s1)Z8!(gveow#w>|%-CHnU~1oXQB{n>hD(9+%@ z-}+gJgX%Bex_pX87hf$7M%C8FX)nOHD$Sl?y{B1H% zm;o?xaa2U6J@kWx(!J}pr##firz1#p6Qkpvp0<^p1q#BaV3;A-`{>^~?{*%oQeGF-+{qc1s?hHkeVgk)_CC0IDkI3FGn<(fNww*# zCRlpvGOLtZVv{fTSsut@teq&I+kD`d_tR**VTJ7{dt8vmUMKmUq4*0M{=r3Fb!9!f z>#+O&BR4bB!26}1uO?=I19f+I>?M{{>l93mH+&EM5tRx!e&l($N%N*|!<$Nvm#Dby zmu}g#Ps#ND?N+w7|tOqr{EL8a~cL) zlc*DEqJsg=)QckF0lm~)4a00OC}j8RWo={MXPv((53aglsaUm3*7(2{QuSZ_#vJ;7 zMcr#JMY7!YGmT0Pot4Gk6W*7)R4rbXw76ehyV$dNs!8BEk%04+>bo+Cv>vG_A;_O3 z&F8V0o&If&hamvLG6;LzgJef1J*C#J1Y>VU#b~z3Q*up!REFUM&W)K@d|~eK%TH(@ z!K}qa94FuyEMUt=6d&2Dx$L`qn5oQMmOp^U!FpRBjZ1Q>m0}f6^0G%b;u%!xgP9OX>avAg4mI~i?Qa!8-CWc5q55*~pH;+dm@Ah| z81ccXev->1t>jr&T}J%Okr!Q_L)uRh!+!4CNbQ-u+VyaQ+)q{JENTsJZ#QCUP1)W8`89ghF*s9_Et_IgZS&xEh zM!dgaps@Jt@Dp~9gpx%|p>DG8O32kBBWvk{UwkIOEbT5Xt+91bp}*8oMs`il-+EW) zQ75p6Vsaq(EgDXvV@w*aCO2HkTq6`PYX%~6$JfCvE`1f4;8;fqA?{iVXtpLP4w1cZ zm*k@`zOK`5Lr&evO)`U=Om%5kFaM<5<-=%ujoooWkm5He)=~U@MIgDcv0h1zk)&A( zdiQ6fQ&9q@p+BzC5O2L?@qOn*PSI~j?QqfSi?;FA=$(I5+vXBW%gM7OzzNR{4I zfEcWhIawww#{rzu)KLLglwEnbbt?_0(YmswjodRp@86>wDnLFI>fg21?0NW$a{_~d z<_@23T}t3B*f4 zQ#$W*^~6SG+$Y2alvaG>$m+gFus#{qO8w(ObEK*>9?NZ*lUra6DA~Xiy)+$;6`$ zqpI;enVg(5TEkKseRWPxYV^%V=*+S4D@17SKxL!q6~Su;mU0C2aonBlWNU|)T}C?C zw%2EUw~5)drnrqq&Yvz5YUb`zJLo8{s-e#^%S64ubdBYSIkMMA&wae-T#>SGH&G@{ z@NMfZX|-X>7=aOjlD8-qLKE_A1U-}{;syjOcj@|e%}w637L zIh-!03ILF@wKEx>_SaS;FV{RtuVD@$iy2)X(S}=M96k?8nd0aw_EvZwkfycRR*ZB{IHt(K%v0+?$0Q?wy^AZYlY`N7a$gWWLwMTUm zu5(k9r_GZz3)8#Lrn$hg)W8Lm7=?Gc6) zkz!PNbjTU#=S&0;IhB-#6RyTEZ*ai`k~=qhH8@`bxZo+VfotaBxLRel=CpQ-fAl^s zJpcl|;~^4|s3%IM*5u03_(0ocGuzu|&HeryIDay+RuRXFIj5UkYVD0h@tT?Dfpx~L zqiJisU_YGfRp*p>(ZHGt^2xQs0i_!b)#D$TuX56{U zz+Deu&23q&kmu+l5LdXT?IG+ni?zPR;a!q~LhRI|e>%rGJ~P=$KacJ7W>;yBu#wXb zu9?|h#C3F|-_${Z5%7f^mG^oxX^Wj)@0(mxaskC5>ifY%DZ5jKWJ8y58 zhZ2Gv+PmfQfziw=)B#C-(H-G5KLmXD{^jubSSDe*_7&EyU85^jP@^YrIS0_1$6KtV z7j<*)bba<(qq_}C`d+v$%+1%;`brejcL&r%CF8iqUW-Z7F?!th59fT?=!(T)+hU$yiw{W%McVRSVw*KWL@x& zMufVR_e+Yyf)yd)VZAn!LZLa+IWJYP2x6rbXjC?(R8( z+h<;)cGP}wW>Uxb!$*f{j)?roYPj#hbXFiAw-wUwVh@_+G#@Kj?aUOi95I;HRU7Pq zL0c}xJMh`7ccV*Tz97e7{@ukFwVSLrd-s^rg_TV&O958)dXW6 zj&9D~f3Q6#L&cjdCaGDHd4)1HZF7iq+KV*N6g)U$c!KzvcxlmRF!o9soy5939dZG% zX1lw(9ZeSI>!!K$6?O{cc0WKUIzF3Ed=cX>E5{Ki2N2+r`^b>{;;f&n?m5SlTm8A9 zKxe34nP$_ERq>H}L-K*H)Sp-!w}7~Q+>BF3AKTKV26%XN39Sr;lj02dE-MSQXFPV# zkiYxPQ^(9AbVa{Q8eKJ~E~M)lJwLcp9(VO%wJ>BtM!qg@&Di6x&kx9xlGwtQ?Wy1Qj7IO0Gqf+HelHK>iS4 z2gppYSC!A2u_+a-;ILPmGLpYg!akyneOKwvRfIM(}@wkw4wT) zh6i+pH2u1QkQS?}NT*Hz=@!}CRvZf}>@`wfd?h3ErqS3+>jIAM8q3TYtsdN2D^r5k zKstK$*yqXG@cs7*^9s-?>}~6-y9k1qSF3Rb6AxayNY(n%t$5YsyCxv77pD6I&b5lN z*cOWuJP7o8mS9~g!d1I%ZLk~559GdS8CKO2*QnQX<-X4&SZ0_u)uznlI@b3~T998} zq;E4{NFU@LH31{f(Q^M#4KjolgJrX&swW}f+VPyV9mRlrf|JXV3SpJ$xz(*8Kn0Jt zk{z;?CTf)j)YKwj;??6O)S;Z(1`pd3$=g&wLVGnXOSzLI5OoYzJi3K`_8(#@XZ=$) zuanUUCV!F93bLD&;8J(RpHiB)wzGLN8aWInqk^*tY%{%#XJ zRm$hGcl=WRQKyo*w@jgYwJ%?Hsu_Lvt$N&73+U@~Jx z<>kgT?B@NHo}7l*l{NlqEU3MRLVwcwuvvSa zUTPQh*kfW}>mD#N3k~L@yj5&i;Bukz^6(bFz=gRNMA`g+iz~DD0}L{1rp1n=7;06h zDh1Xq1AnJK+mHkm0K-uTu3M@kG=$3X(L-53DjoRv|Xxy z?2BfwE>Se|)^MIAwdB`jSHUjs5)yG?$atb=VMRH~_jI%Bxh&PedgUC4kIZL>=^C`? z)V7c4&Nk(QuJS_sM8xKFz}#+(Udid!q~N`ii=eed#DojrqmV_qBtEEiW<`r%nq7uH zAkRCQuP}x*61Ehmb+ghPp82}cQG{{b&*rl>xtVz;<@)LmLr6QJaahx04|w{{|eh7jfB1gJRFW#Eu_nbc~}M)dSLBUNAu<=2AY z6F@Q)Rd-iKrr42zDUNB@BEhNtaY9p&xL?UlK8rT2gH1|2HXamZMJL^55C$J?oI-(% zZIS$mKGmI&@_K&`Uyp*rNX+O$F3{yeo)-rkU(uUVtl60de_Yc9CO(nUHpBUZh=+CB`q10RPjpcAU6vW|JkmxZhHpZdt;LI2{xtK zc6lWJL2}x8x^)l;HMs3=JfdEflxV1>+=8ioa5iS8{-Cjkk8Wf3RL_JYS&iVPDXw!= z2h`+f^$9zIZU_$x%nl&@MyB@DPNOflQEz1J&FQkOL%;HCd^Ylgk%%ppWVj)ek>iUq%!8we842P!359H-URZ{$Q!r$tJ2&&uEDw zf(3ZGFDH)Ee=u+;7p@^n>$XE+hrE&t++BsFtxSwnPOhTzK!Rs>EIE%?h}zD3t4B)5*7??9 zqg7lK88<%rMXp)*bj0GqK2Ckl4xsn|v--F#DIt9-t(^>`ia6+aN;u%G9v=^K-+f{^ z>Pcm9>%)2OlB!e4;<`c@D!->vHvSM`#$CpdA6bY|vHf(Zz6tZGe7Bmsy&Gh>eeSIE zb z9QkLU@k3+YF&pQdwk@X`PoYw)^$u`n{Y_4-=1zGCt1eyglqyYjXcfrv8L?Mbne9dLs4zNU~6OXvd9)SoM1m{fxy(h^Wk{BrfxnwhKHUXQ!Y^spx+2gUUXeInw) z*x~dOkw2IHT7N7Avmdd5fn7taa(3KV(OiOZk|L;M6K`LIet16t@vnu~G$HdevweS} z$c5M^6pGFqs?V|Ce2TtaSLu(^qg?|35T6HRnv(P3o9RDybLS`(-&uMcB5y&+d-zsB z+OFg@h2R^u(Ur4EaOGYac`O{XpWt_mY4Lg=*8w+yb()XOMnl$7i1e9x!FS3!&U?mf z6wR0IBi8z+Ph`mwMV*Hi@cJefsO!tF1TV+DDewt5{s>`2gK$(eL zy(i=w2nCzHUZr2so@>A-Ns{+@?XyB?5u`guF8{-CXC_Y4*@s`>9vD z=Z=FSXUTHLBMG@(9Wukm4IM9Dle93S;*l5ki_>UZ_a3M_;6K^#3#h=#aSaU7La8oG zJm+B7*~a7UKvLTI6sF#%`RZMiEqwZzAmW|)xF_s!*_K*sJNaPL%J(!CObfkV;dmvw z%0+2^kA)7;YjDeNh%b!Rb3wj3aNd@pE>01R;Q2!~B@r;fPu+`Vn2w(BnHn>rBs9}H zOJRT3ZTF?onVuSKL$*^P>)m6zTlk-c9o$|Qc)>Q8_KWD3>O_EP1eFQB7SgsmJQ`VN zmb|^qb-Bdj+{~i}0NZTjadj#hT9qnw#A*&|R(r*>*lTLI=6eY@OOSre4sv!~)@7=n z+S{eldgEeODbMsI$fmdsbJM-11Li@LEicZRu#fE0+&t5-tGnatu@}RqwTBLag-wo% z6Q<`SY^F#t9OY7jL9FtPpDC(eNnd3R=^DuWlJik&4>hFztS|p`s})M&L(1Gx5Rbxe ziM+8S8Anv z2F8^u?k;S6_d57I%)vs}lqH(M5M~*xLLNBp$9=6O!>6CwGs;&=B(5tLhX|FnsO8jd zJp#h$!&#BTqP+J4h^)>znAtw78L@r@%?J2~ga+m| zqFnuh4Lv0}+Ldt?6)im;TOG%`TJdmI(@|K$J@ydilQe4n2u3>;0N3ST`qEJV^4fQ%-+1BspeKuD^j^kk4*_!O-5SD^wbbl7 z@+~M6l{?>j3z#73VJk~4bfW6|acbNCwfw^BmP$zA|5g_I@I**vX}+Ku!O>!5WONNt zzaPU*7@zcA*=j1S6Yq`7aT%zz3(AlN4R&zbVN)-JP;AX^ek4j2J7RC>cWGgkmyLp# zIdr#vxlKS-(YoE)mDMX&=R4TS#`2fgb>pz%{Vh>|MvMze+Di!;u1UVWk5?7ARm5W@ zpv{+$Aktm2@dOu!2B_5%wJ&J;oTsG56`rSG{dK%sP;_0ELKE9tWgn}2W9IH;ZHLb^ z8ynK){FmMc-_JSdWnf6=J0XV(YY|M&d#ZJNH*U0vf%%=Lz2x&!XfanWPtA!Z<)Ky4 zogFN(C^;=KH7@k=YJ7QLiCu*)D!NQT9|N>bBi3q#7J45Mw6lCmw#W(SOCi6^a-Y9K zd}KgEe<7+#ez`S3zlW1d@*P&M-4)*E0=aAv2F%@cuvPPWq6`^W2WRU<&NI+KW})B; zznyE%w#7SEI)~v+GP2G8NG9nBp1pUfPRC6kSBkxzL*y^^by?asa8z z4~?b+9%hB7r;+2)g*ugMbx_BD-kQ(#;jgMOw~E>RApaEQ7dE2gF)?xY$45IrYjWY; zjvSxsz)%UVfpJ%UKfz{{^u)J=`o0oNy*+B_fZvGRu(|KJ{w~%R&b2a{g-Opt_anH- zPan1miKx|7#u@9VVNrW@nMTp6_t~zhtY~|!KoURi(?FVilXWgSy(j6+j#hgASb1CB zwX))8yz=8&Y5{z4R`*;h_lffGSR`e(e^1GaS&R(_dO39}e5|xG#Hb#m;R)U|K zSm?Y{9=9u^3MH93%)Z(M8yR&rpj`Fgdlx{qN zyH$fx;7c|?0!v!7Q{T%_F0`w=Udn(2nZpDT4w-RgE`vnz?79Z~j`dQfx{;GpmxzK?N)*C5_zA&l+tm1Y%07nobHtP< z>bLZ^wtC|$9 z6uzOe6bEkNnikJ8kDM5{Ju~r<=dxv2lW1jfJc6|Hf`kmt+~e}z^dlANv~b4nfx4H2U(A{1dI0=jI=opLs5OY_X3!Yf=*kj3V#Ge z>|g{SMd)RLja1jWHI+kME!Ga8{6)`dYM=!xnmwUdGa`@LO%hW9Wpsd>1gv*mJOmYa z>im5PP+m<7_qnY4BEvk(O^6k(%GFNBK~8ocfrA}!13BvSMkA?C_H3)OD`By^HIgdm zW(0CGbNH%)tI!`AS%kKD1T`KGO7{M6C&>3qK*`;ss%pv1JE%89ll?z2I-xJ7AeW?e zTk)A?nC^?Mcfx>cS5IG>-8<=+DZgd*b4P!0PU>wjW~HC<#OAM)tATQen@=0=h}Fd& zJhZUvLP2MCK5Oo`aLgS!9x@^)SP0?r+}dGeNKX6tkn!Y(C^0*MH*b3E_x{n1B;G?y zfd5#D2vl5pcU|AazmaBRWen#S^u}Wv&1HgvN_2DihMS#4+$;OW8pLuUGA3vxcPgQ!WTm7ugAmf2@%Ka}C_uG+ex%GMzHlXel zQLc@~-XE94j?=>eORfs8xxPLl?W{{GQSjIHDd`?zrkGLMgkkz~i;t{a%%@z2lJYBW zXQ`$0bF4EuIb3uV6kh9ucpRzXDny}13XHQze`gJ$0~ z9+sx4m)=G=vC4&)p)W9genrd&|41d~MZ3hO{M?6hS8U&_#D04w@S>rD$q9B9Ef#z2 z!{jXo7j@4FGL}zic$InEsEE>1@r%RD)D&68gdAEKjrRyaRn%ol`6mum9%Q~)JGyZg zK}u>_u_>FCkjjpYT36-Xbb?8^ll(bw~-inv%ZH4ggFfb@w2wT>|K-Z;Tq7|D3 z>F!WbhQcXp;6cx_7D7mJWsa$9agCqWJt;epEpg%X){LR=0suZ)G&xZ_)Z5 z)Sel5oS2$|z5c;v3_)e5xY48t#3!r$^s1{QU0aOW!17C3hak~E#2RRpvp|LONAj!v z+nVhU{8X{iI7DpD;o~%fsrs=8>mXZ8x+k2Sfq!n)H3nl^e}`%IryO&0t3Ah3YdZs9 zhBodLIg6Mw!#}Rf^LcF_luSxCb`O5CS|F8-(ef-Jv4+5dbk@D#;|@WZB(a(n=lB%y z%MJsaK=nto5;LyO4ivx^pp5XDA0?fH7YfH#iVM9K+O2GP=?A)kD5HhiB6gTiBZVjM zK9AO@bZiZ!=XhpT>$}~LBd#?rb1AxQVDP}6&{8Ka#m>1k9f;f4EA6E>xDU71RZD4Q zcqX^$M-fVc@_i@(H@IcT+8U*_21%K&toug*SV4g40-8&1Fkb$%E3sH18T#V7{Cu5o zT1wTV(0P}Kxd$Y)OK*UL9ha@N56zKZD@F0*kqV==B7>`KE^f7hf_AmTdvVF*l^wL~ z9Qalm{SZa0L`l4^xI(uCj|++Muw~FkXV-u3VQj$Md>)T4T-1d34@8xT3WP_VO)IIj@3KbPslE zbkIOLif^Yby;~mIifz>XTJkM+eo1ub&=z&0KS{@_mRbTcIsozS#+{%YJvy%D(c#f* zgpa2DU{;Z<&=z&^n1gejpcJ;Ja5`SwmkTF4Yw&b&<07l_4-+z=18XDo$k>COZH4Gs zO{A$zch&2vvC0br_Gq4#4Artos8Lc?QmsRhkTc}(`}?0M@pE#?(6hpVH&hn8(tCG= z+9s!LR)iy`!nQ4zAKcJn_f$u^>@l2jhBPyYJ=Yr-o5%Y)eRi0#ss=|0V&X0H7vNs!caD@ z63dP6z6_9X*7Yb+$P^R=JW>Vbn6DNz0rYIS%=IC3Pck#R_;yF2sIEO89k^wRwWdOl zik-|g9VXu8(5v}#;>xDpdgYzI=~=hA{;Kun5W|H&-}O`!pb{-clVaN|hiJtzmr^gd z2RAK4O}VlvRK-7xhZAHV0jw#ge<5T2neZ*`7wxEyZ~$u;ps<*dejFE87Leg@@xru% z^EM?d!PX|If-Eg5-q`Q&Htw*p)mo7pQZ&dEwx7fKxuHj1wLI=`9Wu^>lPf3ivF-J_ z(HM!C*lCz)>zDJd@H{GcdS%Urx9ABQoIU(d4mf<@L_-~e>BCXIq_S|`F4(fsbuDth zhtaAuB}*o6@JxJdtyGqg%L83LqgoiPB2hYLm@ zd9^w+250*yDqmUn^)EFib4ADRU27K&R2SQKDuDAWEd}GsthUGBPS$L1*EhC1NrlW@ z6qSch9$~&p8&1F3+R?7|T%k-@?OaZa;bh%(CBPYV#pi2Tb{^*(8C?caQh`#Yt6HWy zCLTXjpcd{0pPTYQiy!=~0H6bhMB}O(=FoL#f}$xb^@x=%wWgp6F1mw6SHve+Ts`-=yGsQ8cND2TVcjt{f zRwa|{uecM(N#h7SDz)F>bXjyK?!4`#@slK(0Ha_cr9kQ=0xLnKFoU7U=yvAXcI)24 z7+c)TSEVu7GoR_&YPNjSpU6?4SeOejqDqr0E4OBSb)abkZFd0;zc5tX8=OtA$NHbG z**UcRMmIx#FbX*nC9*mFTTZ&)iBmRh;Yy`>?d{mqsF~8Gw5BO$Q^xR{o2>iXO(?Qs|ef9)xn;TBjU9t9*;Ps zOwVHfBOkF^pKDB0<>8>J`m87Q4r>8**kr)Qel{k|=9GMDkhqjOMz^17O05_HU)GDF zcrVmDr>|;OF5dUCV54>SKh}=>lIlYMchZ>Z%PAQ>os1Cg;%=?y7*CZR&9b$cNy=2KVYGX!IjR2T zUe5Ns&eWa?_kEbwj^;SQ0aJwf8F3usCW=j+@{UfGh>8`elK~juG~gl%dE=y)%)}e> zUbrVqd8uIxtE=jaZ5<}aZga-Q4c>_c`h7Wwpxq?jG()EF{^0kFU@;r4&p|mOT7U5H&4}%=p^)=qGkChW$yCY+Y27vQ zIz#D5X$#OV{H_wTBP?Sk`#`c`V5!P_SiwmTwY9MW6fSv}mySuTFZXabc`Un~0DLhk z4JiwNRI+gwD_}wCnUQIC-t8ub)h?+2YYhMI@CP|#)7kvf&rLiV z#}Ptkp!`YkYtqH6Z3r0290 ztmNq8rsl&>$>cKi;^~sDY!3ik3YrYFKwzopkEOWem0?4GPu2K z5D#x&77(oLE)92dTPe4($cuIeG9)D%QWX(LH5#fE8lojC&A5UQ3#CiR`^Hcmt#6Nc zPgXx#vW%cc(^O#`CCk6t+Q~egoN)j@Q<{C|^p6wAOzKvXjC%A#k>j~r;;{!QB9-(ggkGNNW;(YxY-=?8l$$J{M~W9n6FOD28wYXtl( z@({bqsH$?J%*=sFUMDgK-#l^uV#Vh6UtI+^gl0LorAR81Auy37m3Vj3>KZIIhb6S> z_*&pg`{RGzUkxC1+-?)vCS9+JNC=0u0pE&T(7_wpEUL-Zd-PO*o0NnOBdgR3eB)TJ z2et}91A=|CMZ4xa-!(I7R*S>)lfIkAn~$n82bGNeRx7GcL7Cewr-25ZTeAsjpMIyo zjd`O3scJVyJ$k0P`MxZ0JZdLp&>@tn*p9pwqyy|G!0;|6`m_f_pSHOhNChrwZ?DVK z-NT3i((V5>WhEBL-G}SHcfY>@_U$DyPX0>7{YHRuCt@6WloYI7{`=0{j21Xhw~X)^ zehf}S+8|#K*K(1FpvaWY%1xzTNNhtv(%dvr#yve6hXkPQN)7%V zCV|3$r|BKBjthfFP=~+!d+?2nNAQZEd#D}+{D;4X!&p_ue3tH|%HwS1#^UR%KmBd! z1rq|Y@#OC1TS;0>#Mf{i%dBnx+wTH$w8~35F8TLgsE4oeZK?l*7fRq;gSA|IZ`d0!)r|N9mPzZK>k(J-;BV6P9$)x|%l`|t{DRv0U#R65LHK_R$qHl~ zuw%IsT2sVemc<&5vwZZ*0x5NJV|*GVzYWszCD;d$af$*9#nMcd#Fw@+zaKgL+#z(X z-j9}~)gMxM168I`eS#7^6SOdZWkb|v7^V%kYZqoaz0DO1w-lDda5tQzRC;1@+m(K` zXf@Qf+(EVOa6{25-A0;Vt;NzjbvKf_ZKfO{VY%9MsL*M#+p6OTe(UD zeT18Wfi=J0M(e(`N>(905PJ7O^sU^~iG`OlI}{+o=|i+G1g>x7=CaiQT?=wm8IrCr zR6&rR@4UnaR(V`{P0AS#!en$F8$u;j<*VVh1UgVGi_$xGEm;28fef{IR#S^<_=x3Xskq;XJb9vig-5&`?-FKcpf2su> z;5LSp3x`sA1k$V-uVHc{4FqjjWt28Em)2Pa1AJRtXy7UqV_<#J95>=VemLdbt2pjd z{pUlIcIe6W>2eM*n0~U8U^EOd9(7U5_?&?OKt|V0A6prFOTUF~`;ZaLdRK-8Ovj@Jdmssw7=C zV`Ui(qTBg&Dx=TM*@tHu`_5SmfYWWR#stq6PD7@T%t(w+qtA!s{sSnlb{3hb zd!8TQ>SM3BRdMm-si_Z>AKkSVJWJii{STGy8H8NnTEp+8JL5J|$SUq&&=~ub_2~lN zBE~Ogv0iYTUSw!Yi8Vw9#k|@bfM)1*d@dMEir`z3Wy_)+Gh&IxwD<5Q;kWEwIf)c* zza~kSg{LNN-9In&X$;ai-~8xxpiW`K2Rl>68L@WCug=gKcUIHmvJ=3jUsDPPq=Kd^ zG*HwwxaE|C8{hp%9-~9^E5)$z%4A;1ENb8Yq-=x)rYYzc7Bcw#tuflF>)C4<>aTk#+5EKhfY2n;Aio)Cqlzr;eK*0MCbO^!^%ls zlym~Z4rSc~MM_M;??a3Er=4Xowj`XZRtRnp$|G=?Vy9iCi3*0HC-qxvxAW{O%NCm( zr9$FwFpNeXdGceh|TQkE-v?)RnG=!j$#syr$T*>@xvzr*G;W!UZP zs%Oro5{zLNrMNF0b_h?cJ+@MV zCqTd2y)3)7yQHi;%=LbREOBYwV^8}9yQhLl^MYe2A6d@(MGmDvC~C-f?lXnZl}v6| zhQFxeJU9uC-6wd<1fzm42D;0}=8FNP)2m71dsvR8|0$!>t|*>M^daD&sSQ{KfVQ$! zJh6x}id8YQ%<%0Vm3^ry3rKGDscHQlW>j%H4(4NoNB&k&nn~DWb4E59NK>30_(T|J zv9`y+ur6b-d<<3?Ub>O3(Up>BqmmF(N;_C9r-*;@OGVapbYXYL;8fVG;LNP*DNJLd zHR9x4yGopXdECvRjX_L&|J;!wnM?H{e*3YawZ8@S-vCZ3&?|iDqe==wPGeEAX_z2C zWIqE=z~7kpDwF2oVVP`*0&sjeC48o4T=_K@k2FUk5H+=P{msmXEuXE7KQ~1y^Ts5i zF!lDPhg^#I!sOI~39e6$3f(p6w?@wg!-MPz%n-J)XbwDtau$hNB31Dy0>pOb1x<;r z(DKi;t##u?zMR^NhKU8wd3;N%D)cOOC4nvF3=E41K(YuKfnf0>_;)~MN~PlIWxZIC zC~ok5;$XYA+*QBW&dr~5(mS8Y5xlvHf)|k0=e*(xtC~09sPdVs3`ABm`h$ zE`sNrc}jHlWsD`t5395BD**=FtqyRKHC&z9Rm@N2J{<4>UY%p1r|xv9LwH$lh&RzT zNcde<-ulOvYcEkZkcR^Hl{Z|!g$LN6;Do7}x)*Dm3fLmNeIUAx;q z3tu2lw2aFpJNTjI^S7T%&{$$jf*X~$O8t+(u*t&~Hh^?)X1n&8ba2XT=v=R%&z8>waupo*G3zlYr*afXYY6`4sqS6)ith(h(!OMRAJUB zZmHByCUJ{Xuc>6M{lbn!e|2k8pK>&gSy*)SiL zNRc>?aRZ`?e1yu@^LvVP@nA|&+U8HBDLzp+)JvVpuQd2F|AU8eF> z*=$MAjE&TqK0?BD`lq2H<#;QtY94Bn6MOb^?{yY16c;t2VHqf*qIs1(4WIPI{uuA% z6Q%1(+E#TCdW1Jhw?UlkZ*7Mcb{k7`ZscZm##|%ZWaW$CgILG(&Jj-;D*a+CuH@+u zOM4zDZ#J6gd>zD2$!q30DEuT8rSe+~&Ac+;dUdQ48n~C6DTh)z=X})SE+Gw4VZHm| zj=bCSG8-gy`THsmf)~4*?a9AQ=D+Y1apXy*8jsO-w$~Gns8Rr*4&VU0+~^$NnfJ__ zoxYGR#=0aPACd*j2_3Rnw{B>tY zRh8YI(Y^M+#q#H`OehIJF5cUW-1{~w^wTkQ^0Vk}TH)RTkdg6&TehtM&iT7q7Eu^- z6!Ywr$Y$nTTH0#6OkrPdQe-`7XP*#2P1Xs9Pcc|B?&u`|(!T?*%uH7wmN z%EGy;=4&RHbBL~+fQz}H{aRZoQ~ET9bHrt&2)rvKqyvU`Rm!dOTT4j{Be?@!SZ`Fjn)U+5;T$HR%iN`l14lU&EIV#}Jeo>HNRCvw%MLZM^2R%&flJk(ww5`u6 zjSj+HI}3bN4Qy(n$5MnpM2sDfTfc`n?J?jbtlJy1XeBm>bR5uio6q>+W%(g$%lIqL zO87GU>`U11a%T{{i6*dRv3}IcgN6+LoU-m_Nt6R$cD{7i#*~prSdxw&F{87L6dDBr zVs~jS`&7-RsCyL8jGkP-D!xgDndPtE94kZXPfrvb#;n6M)PEY4{6Da}b zt>&t9%~g~8o+D962(n+@whGtf41cTUcoQU^(fNp2E}ujSMQbY#d}N2NNEQN-yd_<+ zwYRq#_andwil=Z+bgIt^%sE}CJ6!OrX0rR9Q&iM=NKrh@QCo3n9iHRppwqGr?pU>B zGe&iF-Lj%@f_#yNXC3r`<*lWHSWyb3;67$OU^#6m zIAW>FCMcTOWM^_N17@;P`YE`tM~iNc^&}h@$-57`*=mpxwshAQcXTmU$^W6cZ|y4( zy$-F}ClwDV%&&}%-Pi=@mO>2D$k@by((%&*tqKF_;Ab$C9Fvl$n%}G7 z`(c&H3HQA>$Y-4Fop&&MCbM3`NGaEtVbT_)&{yRn{7K)rs<}m=_=QO@#gvY;eQ#+R za{&Wuev#Sw;7G;Pu6GZs(?%acI?~^f^CUY@dT`f7_@QqOEvyu>4<{OWMI9e!T_1wx zA2S~vUsAQyFat`)?uIj$;JJF&@>g$h(^Gn$@$MWla8@aZeg^>XT*TC%=3!Y|4}Q)$ z6yFV{pH+@+pAzXHa-S{5T?%6-mwCV^#p# zI0#ZK1|kL^ByGInA)<+Sw<1vV2^`kcQkI%0-R-$ed}w`}H{GU;V^#6f&7KiJGEXH> zv)9mPjQK@1-$#veAE+IRZ;1pcv2k~oT+H@(K1xN+xB*e)JV-<)S|!S!ueI!3qil*@ zOaa0@AbZxqcW@rH^4Ry5$Z?PmoqpC9_-7j3#l}hlMZHnDlst zL!(o>dg$@1BUhp26V1#_NJsch-zN)~-#9_a5 zWQtJO??;`0^&3BGnr=86zauG7_}Y`7q6Lq1Jw@6c8!qNcY%OwMeLmE2xx%zxK0oEW zc$QOndHUM1Fjk!1OjS%xOq)T(e$57<`SmigK1MJ(|LL3o=p{TEFN0aGoiaALZW-!1 zj`f&Y)f+*baJHDvg8PhK>#>&@-q2`B$03H&+Y^j~fyMIZ2*g)?E$Q^8v)bmZFNW1~ zmixoDaMg7}VBk_<*1cc)&6f7Zbn}=D+fD%7QN$as3Zyga4z`nsS!++h2ovp3oA-2rV(8W zM{ttNa}QM>x4df6ixLd~97$Vg0ZiJe>tVescRH~yo#1bkYleN(C+;0YjbH0F+?_%( zTWGvk_|B;kxE?K|l|bt+{YIL{tXO(Xfhi}C@FwJaU-)ZAElI00_r!1a2Lajp!}nI8 z$3(t}@8AzM{Pr?=9OlJzO4d4mONwm7XBnm}KLEHfx;ORWp@$zU^gHI;4|^$Q+~<%C-j=nCy#Xd`k_b zePv^jd2b7*6?=xtp$iTZ1)CrTfRNllI}ojD(6PF`kU6gN2&+SBmF^pUx(|p-uY3w> zJ4xtk?q}X^|3f3tjQvUF0YW*iSeNTBE4((L$>tjEB0;8YR?pT*1I)wYPJ!rLPT`^U z2V+q*+7#}?is$1{EVSK!sC3>(wiC*iPsHrDRR^nydVu!z;Dcc>*5*?|#(K!Fm~1!! z%LJ#f5$7#xxS5v>yYqV|oDb|Xwg5blgtO>{swA?ixxAZs&8u2X4XvwM_{NivC(9X- z85&YsNd9dJkqNvFysLw_l4I@ZJ6yNdcZul&6g4gtLVXo{zi|;SCSRNR1^83nLWWJV zvmjP`rQ#|uz1R3xGLDNr5f~rijo9#tNQvvf$gp{MaG57C*C2$Pp>6l~O#YvMdG8Tm zpxXnV!VR5awfcY4Cj3dZu?q8j_}*7ltXmUcPTIe0n_GVx3^c5rI*mLS2|^$KmtAxJ zZ^4UytU>!jFWlc%2LJB>B@wUt2pCiI>6dq~#x2i=gc9-wgZ#Mxknk++oB z?~BkF(S6=c4TY8fA@tUx zkeFl0XqUyw5jr+F!7$deE@6q^N}zgJEi^I)m>?U1)caO4MAvw{bGNGL5h`xqw=N^`KOprrqXW3Mh^Mhx!2UzTs=)ytQtbWH1Nc zOC#x$$EKcpCIwuTuN$@RWyG7&(h0uBnxcECKm$jYYRsrDd;;(SK1fw#wJoY^#F$Rc z3afwnFrDF*(lGK7Jj>eersxLGq$R`PI>!SS(E@tf8JRG9wP9PfpfZb5%NiM^xyz&_ z&f`-Q`L)^;y%mM3_S)4#Dw`E>rji+{$Qe2;BKsp;kD#aWy7O~?Uii2wGdB5@%ele3 z`#tey!RH%e>vj=OI$54Lwx^cEVDG$tx$q2s)|)nQpcdH{(&X4V(@%q!Ay0;H8^Gqa z9-9sEEqc7gxYuwu=q50f$VA}sqz zLspq1B5i9|bB-0lK|1i#*>8NMdHo>GW8~`6jDLESuctGij&+6^QAM?g$gsnyF3>;D zTf>8hfEinmnWOO&dgB#%2K){`F`e@fwjiqbF_s^-2&1m;?U;;I9%id3?=BPNy@x!x zVp?sGwSc_fGV~dcz&}wsBu${bLI13u>xwQ0sglIFw_jKUZntl4JTEVJk@{LP{AKjh z6{}1e{T%xGh?5toAmShr=rMU14CJ?~waDLA;d?3#ItuZ$>MPXq-#x?VkOJ1=XMHUwzCT89~uZ9bRt>xFa&t~P7A8`lqSf35I$=!gV-sbHM7yNa3U3 zw8;6VMn33_Z5lc4Dhc z&5$1Wm3#3$cvfX#na$|q)8ZYx_H)q#pagTNFwYq=MKOtXKFSJfXp>2r41gkmbCbl> zyKX5Z&LY5W0yVgD9Q!3&HtDP*Nx*noWOAk%R1ZVsNL<^zLJKXz=Q0wDDC}^R3XGZ0HHb$A)sz?x!1)NUQ4E3|Xigrd^QU=+t#cXo`Kiri3=A^p%J_k8WokHOiG5_e*meR9@^A9t`a8R2SJL|@?GC&$Qo-D#t z64_k8*y<$`dOw=`*xZ}Xo6@?dF_xFsV~QR$RPKS$6yTrNNxMJCAeAKRqee$G^JApY zG10-!*sN0kFA<{MT7)b zt}sfGoAri+Wy03eQZf0?3}FchI3QDgZ9*K^gkT0JY!-c>3QWSeDsj`YRYH1AQ4KSW zpnO`2XTviUOJo3;0}db|l}8AG{a`3RT`;>NulUpL8bz-{(IoG8tPfL@d=-YNLei&{n9 z2Wr$$^4&0OQCD7!6`grdHy05O8PLv_@@YBx(DJ~{tPJEMj~ENcbFuvqBH&Y6&o-hz9g zeA^U`>ErN_vb?tO@8x@PhIALIY4juxW%-A%Yj6Qvdas z#tNK&dr87*y3Rsbq%P#OiXHJZvG*L#l%ZUQrnPcNgXb|8RWbJ?T-@uVV{~Ll;x~9B zo1wF=vBpG$y#dRZhY_M$D`?!fKSFa^uYgRinc?oW1570-Mu89?2_M6{%}@=Z-Gffj z1&Q!)c`GpR{o=h`fTV!z6_Ql9KfP7_BI9hK8*tZ2W89uw^(#LfQtz@gB}QF#OFimb z^{d;V-mU2Ke&Q*QTBy=w=h?Roq*Kbak+&~Z#ynOj1ih(lee)8xQ*X*Cr)%rL;v1lr zvG6z|?$&~+nUuOR5&X)kb)$f6W*0ecKWl80LN$?Hh`!03ILt{3n{^LyR|}PL31(q6 zPDqYH(sCLT&U*4F+Pt_au`nQ1`$4bO9!so)zh<&T@`I95=K zS{$_sIN4W{BGUt&Nz2??qy{}Zg4zQu?|jErG|iqFV&9N_5F1ij0RFbc4fLNv14qG& zAgm=6rN24DZ4TWnz)ZvZnunf>+NLte9YRLfYQxrbbwL(gO_=b0XzLX!|A31Tft)Yh}qpu&e*4X7ek%`VXwQG-**I1NHz! zrCs1f-@`bCDgCy+UjZ*w+*7iF6$&J7u1HgOq(+kurB;I-`XMh`8e~bA((AMI8?9b3 zir5YkH;U$$&_n1*7lpErhX4Qo literal 0 HcmV?d00001 diff --git a/profiler/advisor/img/comparison2.png b/profiler/advisor/img/comparison2.png new file mode 100644 index 0000000000000000000000000000000000000000..30c2d8e9e5045ee33aadf48cfaa40849bc621c8a GIT binary patch literal 41740 zcmeFZ2~?9ww>D~jc5B-XbW1A;2zINWEf5*P3~@qQ5phNa6P1}Tii`YCbyVlKGtoKd5RrT(w z+O_wypIssK)Jd0(-|2j}YSpTZ$9_F>X4R^17^_x&mHX`)bxD-P`(E|stLQT>hgMbg z=*_CXd>weu?cl0aRjBpyE8nQUuZ#TEGkVpkO|2jQzM@1t^INs5@X4_w2hYWWdE+$x zOnU-fDgyzp#W9m%IS0T#5jFR;%y)carL$>tz5h?N14Qn6-MHzmM%TAl=y>0~5zP30 z!?$a6dYsokx)q){I(xb=m>Q$U$;r2Akj`Lpcc(a4Uvxk3=v*Bi-y?KP;kT`i6@K_| z7h8Jqg>cD(snk$0$tqaZikK-cky3HUarue|IRqh1gIq@lgxJ+;kw1NsM?=Lv{o~dX zU;S0^TX^-zroZl2v3o<;{&oNEKnPx;ikI=&ZCa4f@Ui+Y4OSV}BNB%~RX0JB)iHrz zT0ZVZ^Hj4Sg;9*l-$iL?9(iR3fMKpwFwew&{iVF|*@zMYQF)E8`%?1YUtjBsS1^%6 z44Y0~dDy7=rOF<4XA5xzsQni)elth&zC6^s9bo_Eer@63!+CWhi120LkvsnHRBvjq z#_&}iQm809cTB{1pXiGeK7p1D>fh4;7VUx`*Q3V%C5ug*CF4s)o3|pWsN4esU?PVd z64*$urser4;h#+rmRQmm5jhIq5W^?3Z&GNlmF^e~CYe6gn7gnK@tu5DAzJpPdKfr_ z7s$du@KJaqS(R7r!3Q#7OO|@V*;cl9MDu`8Dr>ya0IYhiX+p)pg@zoQ)@EFlT+P+TVlQLJDezW1k)qEQX~$6w=oO$Z~9CB98CY@dvB(Xg$n*%)!6` z_7*w*;+lZK4=q_KswL4>*yI5%q8LpZJEy$5K_2_H~J zfmF?r$0&zzQYWtNuZjMxa3h8XP)qwgy4HO{7ttb5rWck7){4Q}B60{x0uDN&dM|(} z-0wk%Z{d>_DNh7E^f*95yPFX#9&Keq>mWgu;bWl)m6G+62;&_T4(3DLXp%X8BjQ=(qOBLlXH-ZF;c_KS)v6jwUPn~dyu*SwR!1Z+a zyE?^((?X!2pH&$ppAkE=NV0X2b&lZ#q$E3HtPZ>d%SZgKaGf%$XU2ozB-ye>tLWJe z${NQ&DKz}PP`2Va1#BB6rc7Scn9!>$#R_Coj*i3;Q_MR95*NeMyfBk%$uJ;pQRJ+} z!n>{I-WvQikh8{L(rJecg#v&W*Fr^`I|k)$f$-Qz9;V&sI z6Zy$(D9gP{^-`+DECM;trPFD37<=(VI7#;5v>XaOPK^a>5l8e@N8qZ_6h3);Y!-1J zz%fSkBb0H>__Bt^Pgd^6@Wf`&2)S}Zc}+;A zAQ*UdYaArPX{M)qTs4=>pwbP3Glal|zAv8WHcC{ihj3i{Qx*S3_5C%zkjQMDe@o=H1J<%|exgKXuEb9{kl#l9(v5P^UNL{307mnx&KMLB0lM*7*UV*Xy4 z0d8bLN4-H|<&GN2^gwD>3f~Hnj@a&6P9#YTOnBlZVPD!*Tr(()BuiutfEl6udgP8} z#~lB$14!l4X`v#H8RTT5%PT@V1b^{OueK5*2-thlqc(r3*D7HUq5sS9SABGDe+&H4 zrTxpJAo|}=RuTTGpTDjC+a;}fci`{kU1jtAKdWy#^0zwwS5$uhWFTdANbm6+FaH9#^?VGc+cB~&5gQ9>hwqyYy7 z3$%b<_)>j`hys!v91B%_LQ0ETw&#VkakQK2)K3VGK!iCc9)P7Y% z`A%aX?aLg$>QgDniU{>OFPFRNEVx7vR6TroX5!y!d2xluRD&jtN~ErYkan}J$;#z} zZsH;<#`rqrP=q4;U+bb&0+1C_+>f)NS{=J{)hp-=!q`OiSfk7%uk)J@i4>_R8{wHzWs?p9b&Mb1fg9;CV zeo`ta1u04SsCCjg$nNL?we#C<{cmc@`axmJDS)z2I>J;XiBZLWo6l8``>=4e-Tb>!RrQ+=8Sfee+L_P0G+^>XWfzL~HOB9KSmsuWNtVj$ry+?l~)L#cK8 zvZTB~s+b8kWAZ~gpomd8H=Ceg`ez!B%z%H8cv{AS68RfrAeip7VuHQua-@v?a;cgIdaruahc^N+%U3b0!t+ zeVW_A#HFLc@wI%bx(D==JPW>)K8^TDO8tbdx6)@T1>@{Az%MeeN8N(8fmDc0L|VyQ ziI7Gwk{T7i_l)POaW5u@nb(+C227%o*bIMi;4up+A# zib(6~9#6tTJ?i+@?yqgjbIHR!d(C&wdfMX(TvrRUsiNN8F~Hf{P{#(OJwiFS0C}zG zvn`c<3`$p>3gb{kS{R(eZR9VK*18<4O_df2R7(b62&2(5oU{TKwqe3$RCa)v+}Oyz zQtr6G-$$`6SqYGpQ?1(|kCGz!$45%1pGyy+J6i%|gEgAi;g5KLK^xM&zLFLh!^G9zUm?-R)~~sYqVS z&hp^d)GrgX*DmIsZbEJocW;{je3baN5c_aKX@&@mecQAUMBx7(N&tj@-_4$M5l76( z)c8WapsR+7*}9;*tfb5rjU$K{ob<8-^CL_?FWTF%9KLk+d495Imr&lz5~J;Lr2TIU ztTD9aC{RdO($WNUl1P7?5Wn2QW0z|z9le$!lJsh$X*(rD?RpVSo`ruboQFlEg zT2ynKkKmqFLSJ6!uKF?-2RXFLLKwnL2>Jo z$bFEUSsAdN0R?6lEVN|hxYI}Qs3bpP&bEBU1pW)V61PdYj^DOd3ETdI++;qXnTpcWC zEXEc9)$t2Rmfwx9)f0Jxb8KCO50>5DSQZ6fQu4j2bw{#KTg=WCvwNB}&>GlwNhVK|j_cfmXtz&EoMtADak+CJe8z;H!pJE=%rtV#%lG6x zHh)TH>uJH9>X-cej`>+{@c7mM*?D1d zT?$iW7r!{c^<`N`tEaCvC93F1qA9vEGqlGc4zzKPC~0w_yE8{Yk-86V8jG4i1}LYv z{4HW00&N}gv;ztr4n+!Fsp7(0hm46Q#Z3!j<+HqQRDoj%32avy$cyQ|MfxBY`1bnT zFHi^FJM{ov4N0gMoITR<`dR)cwQXbBh!;UNFzA2id_grSHFwQasPF^36b%@I?FiX{ z{D*}KckqnB#`A*Ez%f(Rd$=M&o1>=!bC?g^8Ux)iMl73xpmqYdSsbZtO+%ynEG&cX zKI||aPZBBWm_eiuWwXd#(WHd^GgCv8OGNC_HflgX*~`cMB1Tia9Z7~o{Jy3PGA2pNf^H93miNruRbTcK7ZK%}L7AP9Rp zo`z>P;bh`zYO4frOub{8mqQ>;J?p9T8tloOzFU}`P&$~URmvccK^eh^c8zIb_ zcB=#%o9WN^4l<7gGG+Hc+0E>4)I5TJ>(mz#VdL%3B*HZRX$*4j_dxS3P?w3{kTTLhfAFI-PG9{)rY6n^5EY3W2VUBLjw-*ec^9s$P1l{AMvW%P3}z)F-!RsWeIR; z3g)9+eG7$&AaHbN&h&oWlsK#2n%{w(dH$-IH%El~^E~7^@ljx&tqgg<9PM6UDC}G> zz^&|2f_by-Cod>b=oIfvEFI0yq;&C#%p7D{Z4_SNu#odd9@AdUtW6}E$d-f3#qN>h z&yegl*E&W3Ob$KR2N=JZ3Xlr47CxMPk9TW&->qzAmscP2k!;2)r>_o^caxeyY7*v3 zQ!`r{04p2u*~|l|5z8*!Y0^P#!PzC$w_Wu6?D!{Z%mbzU?}_zh5B=GQ#-%m}H6IpX z%}m;ZG`i;Mqw+D~atpn40=d6ZMJWAdHK(C0U#-y2$6!Mc;$>?LZX8LL?g%M(h)xNn zcDwCuaJ;WB&Yc(&=N0U)ILuyyx=X_ya?BC#Gczqu0w+R z(z)oI1&>1;7_S`x(YDR&V)4>}a-7;{@5tP%+FZ83q_*L?-<$II7dP~FqL^U_>m;JZ zxWm{Y{iRBTpgom{VcGC8V9tA~8tQmb-&kyMdr{gT$LHH?Sy z)X8ksUb=v|m?qf8?C8f>{2=v5asy;)hEM8l9d-s!op?R3T-3oG#1mC9t7btLBsJ0x zlQ1fsSw9Y+E0|c7t4s&R^j4@HZDG{e=8|$oFP}r#i5UUym#JA*qD+hikXFzgQ{Pfs z18L6@I<_7BQY{MBCr0({RY|R;d5*JZ*L)i1oCIKeXCv9K61`tJ<1VNTP?SYu>LNYb6Dm_u6*S=P-dxjXTW1MU-8)5$tpVxFOV($Ru@(kQql*}F1ib5uRNGntx(oUAZ}N&A3J z7N~6>5*5F2gvB|ol&zhGGwt5dJA9OipFtpL4EXsSMFYgHo3Hh8(Oxx-dg2{b*S4!& z=@3iCawoYfSC=K1P}M|x8S=J8*e@~wzuygWVv)OwWUPD6BkFLmY<%f7%a01Wi!SIt zlJQ+e_Oma8^tTRn$H(1y=H)N6p>}Gl*c0+V^_yQEDq{0V%SVOGEMl1Ot@YE& zQCXPY+Uhdtih`XgI|h!!mVi2qDa#);#*Sum+GgdFegs+~m(<);g*BLm#Ky>0sP5z8 zBproUdMPyyqC4Ros((^(6V(YSn8&BHvz83L9%mcC)8};aaiN(#z(98(Vw&?bVHeB3 z6C52S?&gUK_KX8?sbUH8`^H$+&kH$?pYeWVrJ5HHC(7dEiAJibI;Ix&M{&~x`NJXE zjGSG;=1v);WPxns(*s~*4u~<6Us9axx);fXK+ z>$vGd?Ib})@X$sIs%%tjX9|a{Ke9%FaXv)$i5KQQ&-3= zbts5_AnUhk%|Wa29>!a&f176_$Wq{IUsSqwq8+?3)(meCoHcWc?@Q=+#CdQU@{Cwo zRM*su1FPep?0T)n(M;7#)+Dv{ZO0u89WnVn@~5GRCO)w@W4v4ooQ9xM#@xk6S;1Gb zPalpxiJ=F|VUR7zb^GNrU4!efbGoSN8`60-YFIFs2&K$2P%Sp`OBaNNpbM6jDdMrl z5z-LJNXMR+v|ClZ-@M-;oxN5p>r9E7n6PIF7&1c{!2IV@T@dyM>L15CIvq@SSVZm`l`xGORBXgH&9K}0-Q=yb<~tJ2r0iv zGqmF@#Sye_V<++|{L3p%HJ=MZw-=2}bfK>v#{VMQKJbkJX$IMHy_WKM+KrVMulcN^-Oora?ZwR zYU@y1n8~1TT{X8qKSX|FoZIVD<*W|bg2mk~eB|CnNQfA~Ld+UBf3}aD)BxEI1?9{* z=&7C6)k<*4LbGhY^6CWQv`XoX^LM+Y9&%SeQ?zi1$cCW`?g-&S@TCt#b~mnMD@*qy->p_fw?W6Khx zT+l{mX7uo-{Vr7TLamf+im2$5WIgR?y=+VvzRyv~71r&l@BUQ0{$OoTQ@h1C&cRxs z+?+5rLlvpBWD!=(<4z2JkzCNqZsO@pA~xG3(FV)KXueU5azR%aBZj!VwvF>${?nVv zFZHz}g2G8daNd<5r^aXrn;T8W!ww4HHYB(bk=3%$mc8xn6zT}i$EpB|;ja~|x!)vj zto!lU7;=s8j5j1PX3XXJIPyDjTJ@)(=H1o#$0M1pH80k!@u_ct9R2E4OT1|M_&z zKiQ-ItDN%N5zEiaF!7@i|MW3ds42M~sU<^|qb7(4(!P|dxjQ*X;huz7xPTg!|3Vc5 z{>~+@$13InF)k{)YEV9ll^+voAq84gno`NfU{wq*I-$d<69t!#4CR>%TQj;JmMUKj)8pvTa&nB2*Vy&=nm zpg8h(3J-;QW;C%~{Y4!x~X9f%JdIhLMDtVXD@2 z-)kZBjt!VRS9g%A#Rl-f*Ue5^uPJ^U>&99>F(0ji2tARJ47EW_>v`Da?3V3`1tO}` zhalX-#_jKhDG;b0U~cTenEiS-jtlMdxq;_ksat%Pr^W335!>ZsdK_{SRilPp_cPtz z2($WR0-23%LmRAG^)B(r2sl`N-@^2rvv64_sVZ(oo^E>USxZJg`K`l9Bb=(xVb?a2J32sVK#&^L zEg!-Y`y{&qFTO${okZ?)=dMmBZEz)PCzrSn140>%hH0dD4aoZ(c&v7RMH%4vlxx@n zmnKD}DlZDV6PSLB+bBEIJ=R%=a&ylz^il3^Xr?W11A!ExOHri6Ay-lAe6aj>3E~{N zPxQ^!aq(Ikk$}4)I>O#S@e1o-x-6sq;y2S9&0qQ4dE*br$#ECC*L&FG!hUUVH8 zmh3_PaGUfN1<_kdUaXHW0{_;1^cG}Slkbs@n|g2el59?FN6xmusNzZB$2Qx1D-( zs^Qj>_BE^k7{<7Wx_kP%@{XYSzE{_qm9SCAth-VPbfBl7@vgBj(U6aNxB!_!c*X-H z!u8uyqFCZ>sD`&o%Wn)o2-QH|W=NWqp+B)2ZM1D!{Kp$$z3getB|Z0@w6djwAOo|7 zMzgcp_+F=Dn-B)eE4S!gXMH_M%7zHgww{9nS0!-dNl+N22p-`AN7#r@slqYM%-68~ zXs&3BZN|1cw6`JmE*0&p6h7WyE*Lg-Vq`U@kN7QLyXEtMeyHT6(sOdG711v?RNK4E zSu~K51qj$qew3GJqPNfZ)x)s4DZ-4_W7d?D^5-2HX>{5|*uo|Xee27)LWpsCYgCK- zh6dpP0e0;N$2fS{Gyv?&w3=>)IX}8XAq+>b|v1dcy!lZpYBIz6M0vCLOgE?Kir~4L*6T-h8rt>eUN%T zIlBx5{wV@0D);fY__WDto^AE6R%c&_ior)u^y-`rlf#_)Bf-^~ zlMa6Ec-o&;*;D$*_%{c>q5}(>&OLLB=PxOm*x>i(cNRcT;++~gxt@{mFnL+@pO>$l z2oT?~$xJjgJxuI@Vt66?c#fHN`|0}j{iOaVhXsq=T7#YI+pFd2W|Yb1Cf_=h>X>R< z8?191<*I7LHwiD?!1B;tEgg>GY^yA~zG*Ig&Pn!a!|T8$!%QQlMNGuW!51G$`;{$Y z??6xov8Da7F_?O}Y$~~fp&nbOipuD(t6HvFGP6yOLP_sn(tlj=Oy9?uW99(9X6XIGpt;SiFv9q~Bj{dcQdnft{>(^1k=P5MH2# z;+)#qaH-Qw+8pU4aEf`8_|v`wH&)biOgiTrk||r6;gqI7Df23$zq>>hXkNnRWMu4? zy>8(o95ci)|2cgsU(M}D-U3HkDb7@J#r7%a! zyG==Qm^{kp8F23ajdQU%;&sDOnci$;0L^G~Wm%rM3-;t8Hci!>%xVP2IN~V=!VOps zt@eR?eYTwRmXMJv8T(Fn4@AE%e@Fm41k>W11rDzrXX-K|U1);=KL`yuJQ$xK=lFugM!qa{bkd#@(zA zz^0L$JzXI~YmO>M0LT)`3pxf!|DW~5j7&n7s z@`4wuMo(97f-LM(C5$G_Fc!IPl{tSkG#2l1<--J-O>~vI%s3M!k3B0_pSH*EHH`nm7*dLVL@_Kme&-;Pk6g zT@7)MEU&-{+%EQuJ~51gVa?b2+n(^jW_yaZ@d$HJlVd%*8;1n7`{OdP++wUjM%FjJ z(Z|5~%iG>l%E{KtF!&Ua#VN*a=^2M~$7h1CtJHq!-Ehx4^~p(yWk0o8ss`brGT7Nab zPO6m*2a(-v2*{*nL!QukB*%L8F84eo*DyUTbUu|vt2Al~Fe!yI9$_TzT-O-!(u?lw zaf6?*RZ1@c%gT)mfg_X>Fa23)XF~w+N5*UI45{x!NjSq|uDJSDc<@w?LmAw(svJMp=LiZESL^|-A`)C7{_`a?rPWmF~1B*OOc+*9ut2I}?sjbRql3 zA_^SnQJWU1nQpsj6&7z%9>|C}-xjQgHkJjl`7*_4g`KKP&Ca&khuGb;vgAKFaHdtW5e_lYpdmQwhQGmAYT4+ep%L&aff zl^zvbceH_nz*daR#Bbnu7lI>Xo=*@&GYhQZ;!d7Umu)6}S4er-`s0*0b(|PCV{Mr9 zQBPg6@O;RYnR{m{+pf!N_^H)r7TTO;yG&!`ds+gG64DbW4E*RXjCS8H;fI6_Rmtr) zVMomuilWTsvyw`zz%}R$&Y&v!ByuAneFHhs8Q$|FuX%tsH+*at-60&*86&MaZwCrH zQNrJw!8owp$aR4hvN)DKt{P7$6BnIq%9exIVJ~&c4frm#hz%N!#;QsjD}C&Ji@&KPhnBhwU9(}T84$TP>WBkqcSloPK!=+ zbu+A&%_zizRQcHut=Kbs$A7*)S1Tf3dEz}`WL&R__^Rn$H&bGJ^(T<&AC%-5l7DPj zw|=}YXLlHk6sD92Av!HTLv+Ld|J_>LQ7xmHw37(sN}%|b@7`q*gs>>MKU98Qo(c$d zt@M}%D;KB#pykLiSbiOb>AGZM#PA=&0n&~cMxWWXOfKrI4nQ+O5;~(bbk5AecHzF) z28u4tc9Q9lINnRCDQL)=8M~KjxHX*!KLOC=_+}V6kt0oFSuPEknXzMKNt@N%pRwUQUB9gKd&Vi9TyLvvp&sWCR)B9s6ZSP{h1R)H*V3?2 zhGLaBDI-BA3W{h~e{l~!zos*~^K=1kRw8H-^b4;wOjGnJ9r2Y?hr0!`NBIxZcdA!L zk5(H&KtaMwz^<8e@>ZoDuQchWMJxz$09@kD9$&fODMf!sS>6=T2U`&tjPGN!D^IYb9)?o^GxwS7zmQ3N@-j{VEsZJtroK z*WMgqIW(;OJ<`-GYznqpKM>teWkdtOP!5zmapDFqx3kGmGd>~%&yZLt7 z#hfzqCc_@jMXV8re;s2tp(ofjTD96g^G8Y-#d+33$m$S^+QnT_RT7`oyyl~`4jwPW zm&YKw>k|?$1vnW;p@2=fK#r&kr{ejrH&_loi`q(UepzGXVjR zZj&o*wL#c>zf3nItmz$&hCL zDlNc%!`o1bC7NgR8bNVi>@S&fdX;xY zPtgDJ)ANjfd9G}CU)i@=Qug_byz$2z+`AW@0rnZO@Af=k9%QoCe)<8z*Z=1P)Bkg4 z9REkh)&9?Azw3!tC{>k-M=P2tKK_gQ*XE>8vDlFx72})_lJH{O|g>CWR)=ne>J- z_b%4CYz)0eHd(B5R*Ih8_;gjb`L})eC_f5h$%wo2?_T@q_3AeNwvQb0Q0c^Gswn^8 zzxMwZ{r?lQK7^L5xa01Ys!5PMWz1;7N_Boj^=Lv0R}-W4vXnV4ef+sVl|kc-#5pmW zi+IQ*tcw!4^#GlAFli&HR}Rij;)_X7)a6L^QM@DKky4!E!kkdm$_62SrcKx?9Ntk{ zjL<51$|#p^K)d`|um5K{S!<}eiXXR)YRwOT&DFShpm_~hR%5}0z?1e`Y8!Orlfup{ z(pLt&3r9$2|KmP^x+#5cp0^x2yjoe~MozM3I1jN1{Z5yjqPOc^}WCoGIIu)=FWswtBDzR1Pn+KdKV58?Y zEvwp3m9LZpCT(m1rc*@%9U%>cJ6J&01|hXkwezK{ELo64j}IZgkF$ZJv)#VO`Z@JL z0p)lDpXUz0TKMEg>%u-d#qa_w z{kj3O1})E*~F-nVvVDU-nYECYlkj$M5u*Np%GRPnA zm`w+7Z@Nm}ElfceNPUgVuSlaym>!LEVYDXY80W?|hnxRv%cxfL5UJ4I}C#e6_1KRu@lA{fn^?$)wYN;SMyfv_5x>d zp4&hp@g9qz3VKsq4*k@}uB+YhBD(ll1}+oWm^qS#7pY8nr%tlvqLc+TIH2 z`i5U`#>v7w4@X`#7%^lhepywB9Oao}SQy&4%rPIls~7Urawk4aiT`fn z#V}*P1Sqyh&!fD*-lWjdY(2|2bwwvAhK8A1X%MeOaZo!k<+D~{23Ou(XfBwk(DS|G z)`F;-pZFymwR=ve;n}@LXtPz`kSp!cTA84;92IZAYVOy;4{%)v)sY zPFvso?6^4B3oB9M45EO0GXlb_U3qbJ9#LS zrgJyhaC}GQJ)4u&Zn&7oM74lPJ@*5en1L@a%qz|Bq?Vv8W4=cZSmpgr$-lU1XjeM8=}Ml@qMWw65=$0 zhaD>S-CgkKXIJ{auuQ>cs~ApJjc{R5<*{Pxx#}=hlyhsDW2^}3_ganIKaXX^>6nE( z^!cC{P>B}?Ww==*Ny)dKJ_XM8g-z8z$>de_Lv~sWTGE{;kk;3-E=q35)uD3S4J*Ro z2_?EFbcyxa*E^z~@LEDeW~AF3&8}Otbq%@C%R-gCIJ$3ugG2G6Oh0hS$Bov7$)Bo& ziY~if$VUy+lO6c-=oa7uriW9{H1T|D;LFeMzCm8=w5-lU@_&>vyf5ASCas?9Y&`r8 zxr1@_&*t_;`LZx)`l?jB#yP>*xY5F>C#LKs`*GAg<}Zj`hu;Mg-N^tG&XZqM?;G24 z0PhDE9CUxx5OjvxKQJKatedg5X*FvPZ8NL> zxbSDPqOh-2tFdRr7b;fdrDTmD@%7L|xgTRd)?u)7pnl8}V#S>1ucwIGk(XoUCZ5;K zTD-cvvARv~a`2$VfjvWqCivjs<__3Y+yPr)>v z(({wReUHD5IZpl+2Lx$VZcF2CkG+)g=Ag|OTAn5*S&4rUE)O&whNMmVP8j&YH>E22 z18Za7$P+55KV3T+19emdk{9gd1z?~ujdsA_@j38>0tn5>Ib>monAkTGvQYU$pe+kVgWE)$s1_+u0T4r0gg6UR%PmbTK5)4)dAGivCTtNgSkY< z!j!&LttZh#w1bnb2US@CMn3PKi^0wBZAKc5iEGzCUv$^HGJ8=6OhYWr{;GBw(~~bc z{qmP%o)OnpI|tm4R;`%v(^}+O87NJsoOVD+r_le&a`g9v10Rw5XY@TZjjK?#jJ~g7 zrUerW>bHualxh36_%UbZ?nyOT!XrbdexOx66U3X{1+{G0TIRVrgJK|Ibu7P$cMvq5mFG?R%v8@<(ge)tR-X=o4h%gu6b8))v&xoJ zo5bZ7Ih5_ui!s@_oc40n#O4ZCTikD{Q7`mi4n2!}NWs6|=BErFM)iq7p1(-ex;}a= z=PQ3~Keq2g`{a!Wq2J8r?<-= ztpgQLLhJ*!hwyI*d`3g7ZU4lkiO}^dUk;Tf>`|ZEbO>M$I_UwX zC~Y?3U;8_p@KkJQH~eR8`VoNvbE^$|hrYc^_J{6J$a0ZMKD2bM`0`xSgB-H^iww@t z0Zs0ZaycO-QxzE|<*bCZ)9>@Iu(*BM)YzCLeST^FVHFx2mykc=6K?Vr)W*Kd1>&57TUo_6PfN-ntw9GWrcadWy|lj`K-VdKL4>7Ee# z!`w6qf{?fk?o0~v%or3DdU>S9NM*Dx${jD%6?r<{TyBf=u<7v&t8sRo1*!rQ9Q?iO zK_x;V{rk%3=WOBg#T-t)@goIvp}8*)D2=(TgHQ?uRqNYnr$ZhR3C}kD)Lj-GM|g3V z=lJ~$#^)xInt#{{@>HB*b78Q#oEcF}uSwX)FeH6HLwO(v-mZFL@_ye7#2R1Ck37L= zzCaxZUEvjK(@n29npbwq6MyowaI3r_x7+9C3onw=I(3X*xV9v!nerCSv@D3x#qGGh zaaX1VKHS9ix*SC}q|=M(ef&+C1gD4${-x`eRvNFr$gqh8r_DmYl?VP2^|&H}{;cAl z8`~>$lu*XYeUUjjoUd}aOLrMVYSwDgEOxe^|K?x$fpX}6Cwc4ZLbsJ!tE8Wov;0WI zZw;w0!#Z0IXwyd4YrGINzn*@c&$Tgg8FH)aX_O1s-Y#%=4gNZqt`mkDa2PO}yO&R? z*2(D^94XvdzUa|TN-XF_%+_w1JAZM>`2kH_Eqt3<8mfywp=IMce)YQ0+RG z7TKas#@)49#`t!=UsVY+>kjGk6Y?uHCrQ2UMRoY8K3SP^MUWG6-ud4PwZVbm(s(pjd{Q5 zO5kvwy>XsW4iVaFq!I5d@^kB|!$!4UUa$?U!kSZtF8&95L5VqDqjVj*(up=7!fB}| zR-RZ~W7r3Fr`wEjrlCcPaW5Bj_CV-^cGutWhV;bImF8mq3ndkDpl|^t;`|2wDNGME)qGjRCufjH6{3I4*zVElGw@7 zZ`f2TK9J3~(Pz< zm8ESurn2zfe)E0nN0J@9yH>{Ssh?oqsjP-<-@T08xhb@hWX^5=nl{+}Z5mzTo4c8< z6yL6-BNZ(Er0_GS)fx)*Fl=+S4jH-sXWgjH+rERCJtvN)*ql_NPiZEs{v)MhCFOO= zcMu6GIA-I6sG^T{*fzT;CEkj_x^-MaeABsWt7FG^4P_clEv*MIk15evVuH+A#aUkV z%0W&(S@5GjM}_4kAvfgK!BDE`c=842BXe=gaWNg9(H+gSs(scC$-`M`m1}rK6?Bc+ zg0cPjXLVDy`A6-S>Nk)XF1g*VwGTNQun?Ik^1i0A_1p|Dug=QJu&%b6`@J+hMw9*^ zV5-yiXie4Ro1s5Z59R6zQhg-2%Wk0OAgkdvcc9&!WI}f0T^uD`bcp!17k}_qPSgBY z`+VQn)*g%Cuo5DTI?^U@>P)Qg%>$rR zi*i>a&k<5D&)H`Z+`PpS-*m2S71GYB#+U882Yxi(uoK;2#!@wpMKu(HHjsl6frJ$t>x!^Q}uucXJ2u6PIs7q{_LBp8jx

@PO2xzw-;t2*Qqmq{s~d@ew~^|B4~$`2ltN$BGx9|=+TDMYBe4eGcTe~O$tFY6!@;04B#UW9tgH~C|8tBl6 zUYj!$46^=Q)I8t+v#=&;UMcS8xP!%vox@FWx}3)&tf5cC>OUd%*`Lw>t-bRMYckvV z{y3hIGlR;^Fp5&GU_(R0irU93{64l5EbbqQW79Qq9TMALQNnv zA&^K*fB+#RB=3!$nem+G%z0+6>;3Xv*YS%Vf{}aOd#%0KZ~fQWbM)jLb7xd{ssYvc z@I{#c1{KGG5Tr|c~ zQB>&LtiuCa4?OT2Uj@o3E>#lMuy?1Bb9)ID{09pg)t!@Mq)lM^sg$R@BQ&Ia1=y}i zKKq49o}Ng6dk5M<8OVMuY^yM$JZ#Z(^D;u&t|ih+96gUx^TMo)L+tFqIyUK8Bk-*` zoV&mUrKFakCswmXPHhKycwzZaLmi__n=o9p1UBrBpL;~(b77-3<(}hp3G4%7{Tz4q zJ)9YA6ffIvZVZef-`R?RK~0+6^jH{QrBTW9W3k*cZC9=le(tq)agA6x{Cf^(*590Z z#%A<2K_FPD?s&hN%j?-&s%Eu zI_6OD(^5n`_?V_(RKV9=tP5Xw3keh~EKjM3TaWK``Jx(S$%K08Hokn=dbv2ed|AKd z%>HN6=kji#kauk9(zw@6aX+OQO2WKQ##*&L?T=R9X_gvCTS|4b{lKKu=a!gASk#tE zi&~fwWm;U`l?ty#etdr3sCV_HC4~Kng$tq6b}}47WAG zK9-tdssiW>&Y7Z8Lz4!kOk5 zybaoxttn=h5!+4{iNc0l>8C`xO1ySQ-FBzzCSr3Li=L5v_9XC$ET*e)#2PAAHkVc(s@|FTjlvquNyodv; zY464j8L}%~OO_Mj+l||FMz9}N#X{^YqratmQDoi-Lz@{hX>ZWbceMBDY`_ZGur8|p zz?#2Bg;fDyhRJv6YZf?XarU+un5VTqPwzP>sa*RICQYPwO*7KWLPdhuLn#m|wl$AM zJ|Qha2UV9htHcF>Jy)jKAIzr5W3&uoTuQ>gs<|Pt8wODq;R_Ifc7{Ny>u2cTb$Rpp zQ!)^FqC(h_Q14}`j`K7u#fEK>II;c>$iI`788T?^ZvmHBh3E5@4;Rfmw;3(Bxv_V2 zAUGF0pcnV#E`$6bbN>ePN%}2%kBRG_oNBvpufG)X^j-L&*>vxtDnC8oMULD8822Wr zIJBqE(3BTzX%i$gmLnb!at1`0x9}187WX3m)jZ{7Xt8Pq#*m+89q0^`bOI;Nnb@gJ z@dxwC&{CyFs~e^K=m|ta0Kx1jV4X~~XB#DJAT6wgyJ@}d>}CVJ#;C@uIke4`j+E8$ z@`&IZhC*Q>#+KCqs`yC4wXRa+G$QSyh^07kloMTAHkuSKDH?eP#$Ger_7b*>NLymY zvlj-M*=@=wgtk!y_i(B!#wtWleEKn4W~Ghz9=8#po60(!)w1aA>s8q8v}HH=FIeTB z<>@Gwao8+23Za>f5a=E4hlF!uKcRt7HNTuTps{db^L|NiZ=1xyxn1 zqFVltIA>|okQ@1Qe_9#iz<*8WqBbv*_03G$(5w( zS#n^5e@6yvVe9b{L)qj48acB8_aA{ zeARfp|7kC4EmL95Vny+iYyk*qY$Fi>>hJKE?!o+adSkmnuRcxpnBKC4_NSf6me}Kz zkP>3CGe$fr=*6k9TuAAt_xKsR8Q@3}bh+i*n|Ab+9>?{?ZzA(1ODMOdSLv{2Ac*09 z{&popP6_Fmkwjz#2wXfsVe7Jib^Nh@I$;00G_q?gZiN}r?)#&xj>zN~9{33}`j{o6 zPE}IJeVfsO0SA^?;yz2}oflu3;cTT4f}9OwlVs8dssZAl2Y*UO9iksz=J98`D^i6C zS%ko+CEJ#@`XMG6dVHUDJ-$FoS0tN2eEc2x{zscZvepMZC`d>oi!-I}baiOipwiy1 zf-l#Z+hw0L>d}rtX6@>^lX3nkdpEf#CrpAvHO=oi3j*yz@6{2jDVdk|&7`HYSG8z0 z*gBMt{J@LhCgxka)ln%gZqE1!KD~tIror|ge+qRk0q?}tk1G4|@Z?U2+63~-Q&N2d z%x?T9`2&V&C5zqx3?d-ytnr+@$zwhRMV(kbW;_Ld15}~G{e8D$L*#u*ds0e27wK@t z&N@3k1rEyfMRWY4tOv->KILo9TcNpem}^@61!PQ!U{&kwgGm7sWq-eZ40<*)!ZDQY z9?L0x>svle*puljoY*K{dfJ{BTJ>;ad?Yp`J9({0Q?+ZtgICKt1n5ABamD(c7j826 zb5UCah1!`b2(DQNZ2Naex4<~Eb?ldDc*bsX&FNiIVsVS#FY#B5dh)e<8N6tf&}^^! zxuOz8<>T~_;mO0uBL{7-ty0gkgfrO%5RY8XlkT1|l-ervPY$j1==k2c7D`GL@{0r6|xuB7eGLA~99>v#4ZCfPU3G0~d=9OAFaCZ`PDd27V48N+Y9Fy6SSjWl@|cz#rs*!; zQ&;86h>n$K{6s#;LUZM=JxUd|z%>@TW8e-y7eIC!Y~m!m2dj;Q-x>6^;kwM{>_=KN zQwZuv1-@ml!O&))zC$TkjHhPbRz5-vdosW7y2dxQw=F7hiwr(uEkiDiiX(afx|??b zxswuu&Wp-nY&+bF!HTL+CS%Vs|A{CIq-s>5YFn5Nf50{aeObG*~uod%)`0F*j^HQAK6C*TH+DpANBg zb>d+cbsKJb7n7<}BtgrA6K8GP)CPCDL+c=w4hnC}I0_Y+t=D6GQJ3`CuHrrf!#G>z z!4PZh<9BFf@$`JqWnxwWCdso}DVK4SVAuh_iygrF$+vza>OGthZpHik&Y@>+^4gYN z@htQFnVf@*%lf!2M@nO(VK3=rGNRkIsc8aMcCiG}$`E}CDDx3EXN~x0g@@?b8!F({ zn)xp^bNjUWMofrb*45cK>S-d@Vq!Azw@y;U8RWAoGTVrumVPIp93!0dj_0O8d5 zQR^80v$ld=J&eqdbaYg+Rf#WNIlZD>Mf0}@S68ZRTF|_n7m~ghk%!3-pa%!H=mP6Q z_187`3Y%SBosvFeZA|*dd&dMHu#P7)Wfr61(QqnW;pr&i zD%E{^YtwBN5o-rCC2q2?Co35lh;%|u?y|lxyoJt15m%`AM3ha^( zmvmxOR=+$hUIELCw7RKOQkJVe_%6JsNB%i6=!85%K%`f__UUkc(pxwPFkR4lDN0)l zOD0%$uK$wHte~fn5=83qpNv-V9BHZIr8V1N8Y}T{O?cNi&p6kr zWu+^{XDje`mGaeAyFX)MYnD=;h$qHXpC4#&(@k<+a~o8YA&lu(-n~lux|5(}mEi%b z`cYIZG9N}o#gA5+MR@9s9%>8u;iNpFx7u`iY7{=(*j&aTHzg|MS;e(#KBJ<%O?D35 zCJ5w@#E#`)z;W4>J#yCWJ0%^=+BehGQJ8lBA(dKGD_sj+f;1k>8tC*#Zh>n5j#vGG z!UHA>yVEp=?fagVhnxe3EmvT=9)L=)SqY1vu7&Ag|4OB+gxVqgxLy)M86RRFols`ca zKa>MbIiEyRM+#iat`+LHu{Gq6?sPBGI92s27+Q!^XC~;Db`@BX%l?_~cPiF|Ed2OG zv=?&Z&R+OdVh2QYR`p5zo%O}&hggyV)OP-`AZ)r$ku5i;f{p{i;5Ei~%QQLP_2A(p zPdz2e#Kk(r6%+1tM!l<^PTc?W{CHT3z3mT_vv;EXCAqN zxh`Ma#b2Mfba%rYT*Q@?Y8ifKYAb=pPN_+H6{vMOXXxm4e1l8o2ccGgkT>4ef`()o zSKwZUP<8F4;lZ1Y1RXe8T~e2qz$(%IQc7?GWdw?KXAUcuDozs$S;$*fI8f zQWN7TiKRwJq6O4t91;1-#?guK$eNNp))md%W>8+6BKdF$JixfUI4Z7XWBV9iKvA6Z z0KuP6e(YUjEH^w5SFCS!GS=x15t~Tw9 z_Sjmr23HYGItR?>wVD5{ z6E#b{&1ra?Hp7pCcd=^Tg{Ih@d!Ke_!y)$% z0jL)mQ94)mt7Tj|p^g_YFOtrelx2CFpf9QK?sHJ7YFVYGMU@DpmtV?F*Pr*{n0oL-S>pW&LiLOqg>Z7BN=<5#{gghz`a zzx!r#2WkZb#*x( zd-UJFxqnsD!#EP8=Ti@@g(l4}o+!y&u&fC6cpRmOa{ZadWiWq;gM{T$maO_c$Z5W* z4bt(Kc8gk<5%NN}24D4nYolA19j>?9P{-U`h!eoV80F z^*(axx{k>1dS0EPP;I@;F%c+6rhA#B5z5|`nbY);lY%DLFJvt_ge9d!chJW;#(#15 zFLXY)rL5FB-BX-?y5mX0-PvbBTRzKJJ`;`40x>}P{{{7YLMXxT;tdZE)cK#^f9*TN z7oGVDv9HQrfMiwH;VP$IbTd*sw4DZ@GAc$oTdN{24(8IWt<^kebIqfk0%1D*+a{r%w zGt{))7h^xAT?{rkbZRj-ZO)|)?r)_rn7J^dTT)hYoYVUPG*6Z9AcQCws1H`;vOY`z(>zetk4KeS%FMM_<(r#o z?}SBo(95Js2XF(d7;a|Xkat&^`S_pGpfVuCdkkg5toGl*4aiH!^lYoRK(ZF$Qx)%M zhIM<$95%$hD@)c65iprgZO!ei%~Y1xICq^eH>#$#4fqmN(LRD3bRRb`Rl9P_056BM zU=}uPWL4`!FrXYSv9?b~j{4CpWC@R5?5vR^;~WVeSK$B;Xi)`@2biAnL-Cfu$DFKJ zlE1nGnG8EJSyTC^7rk-%&CIgHKdmCGPt7Vx|B{Kn|5RhCd;M=+b=@8NM;Y-uG%C&} z9%ZImIB*OT46k5bV28%&kW=is1%>Dx%%^s+z)(rUx)@4GZyeeqTKa**er*kk{;*4F zRL#f>Q1eL{(U^tmNCIpc?f>#j`Vag={w7u1hzZ<>>UJ0LO|7A`G5$;FHvWgNA{nSr zqEO*@U)oss*ic2yOP@KTUBs)yx!4O@hV3wix$pQf^kKqr2n_tE1ht;R;B&P~GEf#s z#B?R(zv$CQsdO%(_+5n<>qvN}-<5A!K(b}V$c@g3?pw1^=;#v7QP8%fq|Mm#Vjh*| z)gxV5GMko|i#0on35&=r*2NcHGD8=S$ubeHjDiN(9+{fs`Xc6?(shjcHkw($`hssS zt^m_Z-mi@>iWOC$P5h866VQ}Zt2WO_Ou`8KPBd#Qxct}FMKa^s{S|_i5Uob6e|+B6IT9>#QyuZ^;5!tynl{;UkZ}{ zsO{;C{-ZB*r$1_Y`t%5&l!N~%(crqCAy=r(aPXlKsSrWO(c6R_ z#LFu+oD;#eA0bD1F_Z?KaW$_YH~C+vch~*bLhXW?^oiwzM2k4Y=T^%gXd?8`ahZ;D720k|fh4y1h@0Nd!3WV9U$&2qTz7vC zY859~_;JTKw=0)YdvQ~8J zw(yJ*dAQu5#k)fuD4iyo3h>Ct6=ZqnnQR*!JFeuiu+9|=mHiz~{krjf`uoeKGG>;tTfLGO!cX)ZGMy!nkT+T9yiS00SB$S?)o(5Aay``_+RXa~?SO#m>8;^xAIERp4G>T>Vi;xmTturg!7?rSgXG}b|b6E_1F%K?MM-Qw^^{wA6EI3n{dlL;G>n|n2 z0z^*!gt$W1{R3ii+R3qVv^vHLIjf~OCau`F?xb< zv#1TK$Bw+)+}`6Kxz<>@VwXB9%78)nTa&bw$o+MS_6q9=Gf%q;exSeJsA9*sboI&N@f@FXO3 zIE|)#2kdI!XyL=%?}aOOKxJG^pr_rrqXm<6XicCJd!4q4iX4A@RW3$BBRHZ80c;@(FWrCPvQ-fxvbaI2Xq5_)--piTe zUg2cnsU4PmvQIA)E3hl^Gvnp3iqL)+TI_nuF(x+<#cS1>OY0rh2t(Dd6{yX7c`opC zT;@bgdb<=Rm^3%qSG5i|M~HBB66812(y!p*+8RX8g-@=C8lVh8^tGq7Y`kaK_s4>@ z;x2{84mdHn1sib+SdD9z&j{L-vWq}u;P|ap$iV5~PO`=rS$VGJUh}y0E6cTD`{Jnb zhsIf(nm->II!Xsv-mLC7QSu&7?0@{|Lq{XuxD6yr?WSYBZd{fJ`ZLTC=;lmvPbXYR zHd%#hGXs}fvNpM8g@VVA(WU)~SElwX=FcfIu`#K#5Iiek6}&u;TqeyGqs493Ixa$K>&P8K7Fx z2?+*jH-w6=0L1Xf%-bH=HQpH;M4Iu&swH<^r;Zj1YG?FNu(}TSMRt>!I%T{poz)XR z#z~bpcC7SFniKHq^qLo0%vslxpC5fz5yhmcr+MB-G+Pm$}+{l-FS2#Zv$#BfQSZ;my!11z!~nBhfqKkxr~; zw)cwwQD$KpvfCg#_tHshpTN2$q-0pB3(L}M%^1Qx<-7WBD|Z>5Z=cRGUaL4xJ~Uul z1c*&mw~``?Gm}O)^8|a-H;WR7C<>QI5ClFWuTU#r@?;m|Ug>rRBTvtv zpcTu4nZ`RM3@&xF2qHq~tqCqP(PU5PN`-7=>H5(JTrAS50^u~bzMI2LD=fW_JYJaE z2*--hUkmDQY|ju*)153opKcRGQ5>i(JoTcIdq%HIK-Y*2rk5i1z1mErX|b@-=&`Z*~7bI zGW8UyMq-HNSZ@_Olsm80bCPcJOCiumYpbf{=f4nKG}9Or!I>_iz zRwLleY{=kNz0Dxd$I^NdR1SS4ko>c%%@=mUiQyK8oNxFdYqt{b_Wf$2zK%0;%ebk&oA19#cBfLpB+tJdBC!=(i1*wd^b@Ktrh73v2c2InanV!d0ZIDDNb}~ zKi*qQw^`Y@@LVguuOyH*hdxinEe$XaawE)oT+>PLg{Q#siZH0Hqs_5Y78)%m)E85# zPlJyPP~`Gn@5KEL!=pNN5tJs}T_|Ug2=avtD4QC@o5JIgR(qN@kdR+BSsx zpX}E^F1;0Qs4MUC>XUyG($U?jF95qz7ZJ}1r6G2xgpHpc5@O)AWfL{ASPQau?%M7> zjLn{`4y++bXqIw@U2$2gUuDHCqrJ1LM*^ zq5MNjZ5zxxRHiw_!^X~eeW!+<=RZfg;O%UpYr>ol)RAsmqvQzu;_rONxf$pNBB3*` z&!bKW9%dDOPBc4{NSU#ju4r79pp_lvP#DQIb?zeFA(gSZZU_{q4|^WVw6Mg2AP$aU0O2#lg^ zC5S%uO=@HY{Hj(X-BM=8gbqzbCct&__}Kx1NZG$>0Zm<`CjPA>=rNFIIu7Y_*E3{T zx#Y~6?zZ6UcjbU^_Blq;0?v8gUG3>-{n-kB>xoOyv*90}4 zOSTSe%snq;n5PD-t^sxhr0)8nVy_-b>Z4xLnSJ`ogC6PSx#!=1>J#(3C^~%vdtSJb zMBQWg)lukWoY+w2SRk?6!sh3{lFcSjPV)!pEG0Qdli_twMyCg)3N>5y4*a>`^Uv_} z3qALjHA;QhV-Q77*i#(-mvYi_>7T;PkC9gX>c0Yh#hrg8MYT2q3OK~S_(UJVTxw|^ z#=@?fy<>lcKjeT`S2NAan{aFKxi7!t-upKo@_!`0pC?`^qu(|#I$~u=nUM{^h`PMs z$)=+u9fh{gYtet@eHOZRg8IErX=tCKRlWHuZ}DI2mH!#v{PX0l-x=E%;_m-6Y3UCk zz<;NI{n|tQ@1WWLNvk{3e!HIZ!%vmKI2VB9(*iQK{PEnlFA6Am&8SZ!O8zhq{Z7*S ze-XHS8Wi&9b7MSbzf6WLY-LyFfFMv}6+=(DF;zAe}+#Dna->Itc)o6)w`& z6KAtWbxfEG!2E8Ma=oHKNXMs(7DhxJ!%feF84U_^qYpvSQ|L3jdYjAr z&nE;gwJY~ton%yPqx{5V1d(rX4J=mLGwqa=EeKgPYlt_yS)2VkQt_ASs*}pVV13FY zRC+WrhB{WSlO9>Ph@Se!{z;mEuazAp1TNcgA`M$x+Jv(Z{lDPTtG!pVlB~H~^Wj@a z0aZSa!;31LzZZ{bi8Sr^CWn{a_1Q*B(yW*ipLPp1T%jDZu-lwmQ7a}8<)RH^o#wzd ztQv$tUEos&y>yp+N#RnkU}-S0QMit`tgodf5ly&nyns#xG}bsa4e^|RR#{l+FK{le+XGTmf>`peoCSHKMLVY`rODt(1$W2 zg8GDiqOw{A@+%uRoL624*6y2<cCh7<-e`Hl#hQ$wAX`yVdk;+MJMad|O6$!;!3~yF=PIP?YJ-Xs;h`tR>*d5hHIk zs!^_uLF|)MMf&xEE%n0nDZPfj9gp9y#5u?uQ+Sstcss~eA*L>Nj`Vj6I4U3`9v%p{ z0KgLY-S0|GQTxoSm0EkrVnT9D|8Y=F0&a=CSZ&l*pRK1h3{a<=7W+(}+VeiL zPJIi;;3O5+jmXU-T_00pQqau&75VuAN1o~E{&x@V+7{9f&rMp30_p^ge5;t2r+BUOQ*@)?czEWTBlv&BVJ&P!c$ye`Yn%Y^6m>$VM!uSJv1^0W>@&jO6&l_k)VJeZ3ru;6XY_fsUTguOmi zW6z1~doPvLp{I0zk|)6<9sg=g+6j8Ghr7*CIytmIaNxmww`@pwkwjTdlxYi%dVoJp z|459M&xdvF=iPoDw_~|_u;A_+r-AC3qsLytfVxK`FVNwjoY=ai8POc5iI{>!hq$*_ z_oWWa`#>Nmh9c8E;ctH2%lUVD9^u0G0~9l1zMuI1tF=2%eg9-y|#y zA7-+dg$eb&*~dj=sfe+ecP1N!xQaZaS(P&0d!N zOr_PYZ6|3fzVL0|aGi6)dswCOX3HdigTQ3vQNOdHZWHw>js;$*D zd$wMunC zqT@irFK^9ts&O_I+SRfE&a8dgI5Uj8piI*q*&!ZRZPak?>4l4!5BxS^$8_j zxQ|!4kt2IIOQGgohF*LCGT6wyzQ=)qDgz=OW%n(z^=JR1wC?<0?Wcwmx6p;!93N}APS3H7wL0hb`<|wy+Xl?oH zo8ZWZ$q##p2#7 z!AUFy&)Z38vGVQfFIDTz>bL?u$>q6xt}7C{7FtQ}zhNtEfPVi|M@j9<$Fw}|rS;YL zCi9bFFN9 z#v1m(v>fyGQhm?e?PSNe@as* zhZpny?V#vill%XE?LQ~>|L1Jm- zI&42;@MLCuHc(@uA@;N)e$!479afDv*9JE3mGv%M%cii=WG(3~%-tGD07&;)ewc{Ej<(twgp*V!ZK@Zab;Z)5+%T665%>-H+j) zH6CXwhLw{hKS=K+4;(nUvoFV4N0O62y@|Os8oPQi6Y6uUhw9}mJ~s*S(W*tf*+3Y^ z{BE>oBJRT2>Y9&23&J%epGZ3@iPw2pgc6M|)Ke{haVY860mk!L8P&*R00~GoF#313 zJ8N__wBhFqHuZq61A4L|*bzl0D<30>J=CfzI)i7VBAU*cbGKISr^v2nsDT+$LsG+a z*wHPgfkdK*Sqc&RF`5#)kMAeS%rCm163L(>+Gg$ZK8Wlc-Mmh_f!yBSl_iNl2K0i% zCs*ZI-(=N_6(F_{0jzO_xz}+g^;VxO{+3)N{7dPROxj~xlMB{ASD$HprCZo4`WXG; zcbs7i{`fT4+`P7xdjNWxgE)dtM*O`NQX5L1Z0jBuU>eURTz~bsjH^%AZV9@%QN{%j z&H{o+`es1WY_Vw{2Hu&e!)i0qjF`aUc?E?SDyozoQiL24yBi>idJ2po<_jcn1N zQ3l&yy2r%;bT_|JoZ*D}uG0C2H*Avh#MunadtN^B+fyRwvH=9UK1I;d`jI<2@P zS{(;T_HFF@kA17;uioiKMYi0{Up4p`KgVrj<^!qwsmKj8O!>Q^rY0=>>7sV(iAbwt z(K{EnB({y=PI>OT;)1t=z?0xMeTyn@&pz-6R3;7fjn&F9wnf_xe=cj&nw0_wIR6pn zSDEhsm8Wg7Z3@BI!(h&CP4{i zU@4%4UvH%=>{C&fgkSue5Ir(j|Jtf!a;l$|)o}fjph>7{uQ$zo!ZBGZCj1Nz za>|P$+P}mT*2s(yl;1*_miWpf_g;I@t~?X^oz;ZI_7ci?Qyiyh&Hy_eRorWo~f{{94>yh zOqoG`8@&SPYN%Ft6(A54HPWHB#fI&>0s42dPax?u&DxXs zx;*3;7bp$}1AO*CbQqW}%7~kLp*Ns?=+A$6{UIkmM;L%k zL`DquKDVs=`vQ@+@X{ez2+}Alua<}HwQHO1Az<^qd1!)^c=~j)r|Oc#IVWE#^4sM^ zjGbuq{@sU33^Ww>E64#@qPnWf21_*M7^k_N@;=~TO(OKwD;?X4;UAz;LmCXHM$_3(CPnrf2}02{xfx*o2?jD8O4!@xRjjRQ&#=FiOVrWCEu4 zorlJ-iHm|ru|bdA(AbH0XG$7hKM#;-ea$X!PsgA89y9<6#74?kSUcz)A& z*yOHqhlwqUb*tkEWs=}%GYyYaXI=_OB2?sVi6$@=aRPG4Fdk#dJx$UCesV4J6e8Xx z1t*Af0TBBm2Y^b*rPC}(*f}2~G?2@=Z75{a*DYf{53T)#CGJ-)jh(9qO)V6&3FLG* z<_s;FU>+$=w25vM6coPwxuDUKYjtCD4M8{(bf;q`X!o*J2C}Tn)Z+BUk^neFxkX9_ z0YEf)Yc8j?!`3T!&(y7-1RkE5*hL$X^qh%<4@AEQZg()#(5M=OMltB!f{Y<4E5mHh zbD=8LS`|R1?MR#K^YUhI(n?y!7~kDGr~i0ymXt+K&UW8^)t`HQDwXCU=aN3bc29OL z8kUsu0LGgm_P&gr>%ME!>3(O7(hE!jH%$#Nz6F3#Rbzu#9t#u2cDcjPDGc4h_6*a+ z0xTlV(6iWiFkr>N*5Ow;vsSU|^N5704WUr_ft(A?M1{79u@i34`Q&CTkYq8P&`b39 z(OB>?3Dl?#s^?cUSC`P`$~5C=D#>?yDW^3!jY^&tIY@cM+dLu5{;<-_l6MYQhIlEN8*(AwE<*)I;u2(PZZU2}rCm+!ce&6j_19 z?UEp%)#zkZVYq;?o`ZF)2h;+*=1@aw1E|1TzCv5|ALS;jqixVZ3)^V{=jL+5z-bydO4bhv6^{o>m0!ZzgU9f;%O4sgze05@bW zz>UM6peLcT0HJUa(q4LfK$T3@{WyuP!@Iu78oHExk{bj_{+BPnQ_?H66hrOuxcAIV zoUCyHxKCK(R-xNTT}The5EozwHAfA%Vqe82FC8mDg`dw7&)T%m-^YjxnyaW{`pv0T zn^#LcTsThj(hhvZPDO-DPFcd?}4qW z4XY_y3O;fIOD)njY)sp0`p(h#h16# zeUa+@fnw>uT9@B1`ui-?NmU;NE@b@lNAsJFd%1d?Yq4^9Xa4xMzTY)Jlm(86%Mgie Yzb#!*xKG-6<6Ev=Fgjm)_SS>{2jv7Q#Q*>R literal 0 HcmV?d00001 diff --git a/profiler/advisor/img/comparison3.png b/profiler/advisor/img/comparison3.png new file mode 100644 index 0000000000000000000000000000000000000000..f0ba50ae2f2c89c93107ffd981e4829ecfb7885f GIT binary patch literal 65269 zcmeFZcU05a_xEcV1_c!h8WklomO({{h=K}9Ml2{Q*ik7F8Jd*PBtS@_Fo@DJR#Zv^ z6jYi>s3D2KKtxJFq@@A@0)`|&AR+yJaK@SMbAQht&${d0^?UBRH*2Z+B%d5l&e><5 zz4z;Va_@|T-BP{vdh_PZTYBR7k#qCr=`iNaTOeNaBk)SdbyN}X&-~DHcDD11+BS>< z7e9C(I(=x~yb`>=%3TMzUL17%O6a_K%j)O;%%>q=|2A))CgH@9Ll+|4#?`IG8||QSZ5VIMGRz(*Pv||JoTrEP0P*ioHQi#NsPNpHW>w&J%J#K-n zp0|+Agd*&$^o2_iLC6HrUvJEt_oDMldtzMp_scWVGVK&XleqBv3zBGI%QK5*-!EGZ zXKnoZ`7NDo#P7GgoA+-MYhJW}zx~4kL&p5SZ@+xY*7|!Z`e%G}|9*aLzMk;=%R7Jl zZ*QDpk}8IWA~gNF&U6u}xpD9JzF`a*vfuyt2I9hH(e>7}R=F`Wn7|>4+zyQ5!g>9kC?3cCT|ptAZcK74 zpu1AUFk~Kjh(n%DQO=%_h=oxUx6@9P95REFCbLqCBdv=+by&#y>Y1X}!~x=)g|g3& zokcnICLP{t1vc6=wePg-gtgE@HsHBgb_0$HQNAVlV2%GC2RD~ETjf!iD5vs6W|U^s zi#qXDznMNhX5evXMKVgZagbn3q|FF9J{Z znC>JZFzpyk-$Ge;oDeZ@LppUmNG*uLkXWC3TNH@nzsE^`#Ox4eC{zSI80?Dc+vsl`u*xvkR5Q6-7=9rtPU_=qF9_?y^iom{I}HM> z9GP_02TX`XM6r_(vU}gt+y|z__+mJ`W;OizFFWh_m zUIT`aZ&H;0#qHQ37zxM5bmB^aL>Fa&A*sJroU0axa+xvmEY1ckq5Yw)#Fj83LHftV8if(XgJd_y<%65*kW?R*N}QY=bK+ z+P8y{D*G&)2yG0{ta>^WN+W-!fItkiAR5+|{LeBBYq3=d1~wnhWTTL4JEt2O81%q+ z)G4b6*Ht-bLiE`z=#B+W5BQn5h^-Mw`z&aNxbg8IpByVmWjR|RNd{o!YI^1jf#9}- zh>L1AOX2GYz1_&GR+pjoz>gS-smzhkFTCrj&9XqH+XL4Lox3xzfBf9&(>SUBib@|B zbITozfDhIRjC`}8(MipZY|#g~$tu2q8QO{6Gzht_bJ`W3DWHs|NO?Pm@G8EW0f+gC)qxFa#TeF5 zCdW&Z)3qCAqB85v;EfN9{JbZclVT;J^2-r+*8g;PEggFd z9mWsUt`m9}L4ycjXrEyQ(xvJQym81=R$G+Wfpv)s_j3-R`{1Ru7P5}6kt@?~Sraw+ zw&>CR8dAh?HCkKC8@)0!fK4C>DdX#e|9JOnHl%j!ghLogk?Lq%X;d7j-^zC*i(pYJ zYha`bzBuWp z;fYCe9XgmjZhSr@r~$Kfb&&){>V!)FbS`Xnn~=x)rPWQ3l+Kwr9$e*!MB;8uv+}Lj zw5H9T*|edZH96{6!2ArufQ3jMa%0#es?$k9!{th%dcn{~Of*!L)wNL8s1AN6C-qVL zux&a2%wedfzEE>^vlzC1_UCpJ1D5mHW_vSAr|FTrOb+!bJTSM^K1)4XqTE>#*o05a zJRrHD+Lj)}Hz>MEaK7xrHEgkqZkn9dJQkh2-F;M+5x)?z)22$yu}Q_I{A2em7q=17 zx}wJd87t3mFh(iy#=Z z!Jkn|F{d0AeSF7dY`cnb)UXWxZ!j>jejAYxeOM>pZx1r>$-fiwzY+!9K9Be(B`<}7Z&*S^R?stG|X}B=$G>Kb<)8yd}K8aD@hzrM=kjidwFORW(nr8EeSCO#>ih){d=Q{cI`8QngVyYG`YeuN9pRa(K13g!gsY|(x-G4PTZ(jFAgM=@# zgDdVU2{;)_1cGbj4q))Vi#+J7dbs1ha;h1^cBCpksa4)^>&Q4MSEc9~x$}LP-rdQe z&g#pC4RE6rMU$f?-B)(g@81|dX&Qvn@c-F3<~?l&G1Laxnjsqe-+l1Ig@14Ozb$WY zQ_eEhp+-HYokL-58`Viz0cX{X4sPf*#jeZqUvcJgX}p0h!r@5CmwqJN`I5f>`* zXCG_JB=u7w1?}nOGehM{ul6E;Z+4YFh$V1BR^c<(N-i_&{c5XSL&46SLmN0DpzHvF zk;vIjl;g+s$a%Dsi9EkMDp{Vizn*~6%7ujtq<_O_u7&Cl>@GVIwGkIQANQGuxn*d6FE|F z8-1h2BcS6TH3~g=@M6I1h^Oq6JT}T$SM(|sLPCKbDLO4=O&NZ*`}xY2t+MV6J5gw; z!kqx-pv10)40n?8YAPZBW>v6qV8Rj_*-r@-rW`RYIV|HeM4|6AHr+7dkX7%|V=po_ zK<+|js&BrdW#Qdku6<(aN``@%+YP{(T0rsvgx+`fZNb8ImaSY(ft~T8=||U zDdUlao`7O@|?6NiY(ExMit7V#03zq_yxyBUWP9s9qgu zJ1MqFfSn(th8gXAZ`FnnW8RdE2WoiJe_|qpw(kR;1%Jj31jEwXf;1at_hzolt>*nd zZzJ|h#Yw-~PTh#8=uRi~Ijq+cevBNVtTM!LqpqEPXr8+*V!9U-w#zJJDQ7z_-3!D3 z(oxWJI@yNj&^mkHJb8PZ^!0v}^db;~;A3sa?xPi>Pn2Gl3n#5!a1!9#Y30M%a4>jG)JembPUy0Go-bUwRHr%b?LbiCVPE)6z z7AgoRB&S&Up~GD@9Qj3BZ!RiOjueiZHK)IYtxMOB_V7PDd!nqs3#(BLY?Mv%`cD?- zn1~)aPa+n-^2h{+>6vmGE9o)Yj!svp+Fk{&$H^1};oxZF#^$EXLXiVWZ$X@iPx>bp2HccgKZwVoI?o6SeY_qQ;SR@7)K` z&Zq9BmSDoiQO`HUG(?zvHBBuh85mzV+7bse7WuKYG$4`w0vbYqRt|GcNtvd7{B5)b zeQF95^Vud|@qrvRmeF5IXPE#&S)!1F=_5xnBf0}GU!D=Z;5_l7WKJuMyWNmM;854wR1f)=WgaS#Hf01l)7=PRCN-(BJ-Ozu+KutP1}(nXgH9w& zFxUQ-V7F(D^A~Ei^!#b`KwW>12P~3rBvg7Aaqv^g<(TV}Ay<_G{Kc_|nsxZV)s3{) zX%I;oRU*ov0m12OhP!pf0j)2O{ebcm!QU+Nz3_UOvF`M2CTW0XrB(jW6y7M&&qQm+ zfwcDOzP3w30qfAYy~~*5QEW!ivys$?be3*vwJ#&|kC&~q&o<-UpZJ+2;6QDzqg{YM zTGa%_!x*Tp5S2!#30Tb_REgaJIdGoY7Ak**w6nDJul2e3NEW#pLX;8S9CZ235i)z8 z&o+_`TAT0^)lNL?Q~CP6LbmHboI(dBx7S=LhPaC1NK3Otxoy>nVjFvy1@-CG=pX%t z$Ky1F250ylBkQ1IC(y{`!N!Jsq+7M2A$ftOuMViDh~4f{cl%CDoh~1*3TY)jmY+w3 zS|`~pdG(n;AKy%UZdtX^>ZWwiTJ-P){_;Ak*F^0rXHn<1%(LJV$=ffa?9wy7$XwrI zZ8k2Ys_bW!V=BkcTkME}+zZx&58&uE=;0bfGC znWbiVjaSXIV=Uh$C+#9Sg2iiQip$mMH$g?ScBrMv=O;Z8>U3fEHtRH-8=Iq-iNy%A6DTHym3P)TA@L$_N~W_5{rpTdY-<&G z@9DTW>OMUood9((?wU@1O`Wy7O*~3-%2%KFmPK~S>grG|FVHkN|JIQ5LVH^w_~4qrzrxnA4X7<9v3-@Vy}nHZNi3`TEr2Hj8BVx*T-Ej zBlkj;Ukl44I!{2R=_D%#Ez>3xOkD-}uk6X=ZL!&!-m&-@ZmwF>^ ziud7Pok(5n;seJ8?d$FFW;X~780rzQU&!W~PN>4Yg&46KcCx3kb4qnl=1y+OlNBA) zsq;~Owtb;>36%vu<|#WeCP8ep&^2a2@iOlB*|XaQrW%L9!ki%LB{_DDy3Do`rekNo zN;@@j+C~EtJHZ93~#}r zfQZ1@Cbie>a|7y}_r)vNKMd3B24|!xoXQNx9YiM>DV2(j5`1uNfo8tBd^gjI(<(+h z$tvGJjqQab4-(wn=3E?mKWYd@4n43QUq^3*e_B`Wto`_SXq|F%MQz%--`G1a9S5*; z^ObWGJ5#RQs}O7#KuVW%Ak5Q)`DRX%v5m-8O^Dv~s!`f}kLqj{;$qpdDg0^s3h48Z z$z7#%k?NNL)oFuw0+4Sx`NA>uW4dv6}4W-WNURdc_kN4y1LZ$sj7 z_KOK1G!63^X36`Ai6~Q^iq1o~BAw5i`brr;p5T-!BI<0Ta|8M)JENRb# zRp*YEppn02VWPEfjtB>8(A#V3WnHYz%Z#g05SJj3s`LTfEzC*;4sYm)7gr`8LZ@_Z zjr#4BWb~0|oHSX{*F2K*_-buvA(Os3!NY<^VoQsGIT=>pN1XF~%s@mI*o$k#Fe9A1 z2lVY0Y@;T4HKWiD^>~Oz+KMs1%12(nn2+1_#6UvrKSQEwN0@dxUXTM56 zjU6|A=Ju!Mn%SHyUn>5j2Vf+Wi@<1424wwY<~E~e*%XJXLFh9=)E$XqLfGMgnG?~`&~4e?j8K;(18~~joPfQ@%4vx`VjV=| z(1t3yF*FFwsy0^N3e36S17_oD&QUeTs6SaA3rn4f1NNI%!PsVx=Rz(Jv<6QXFuW}DclEtr)W%CBQ5r==ESh~-=HN1uOyZxTFIR;+q2QkW(m38~3&@!_X@qe^){#7VD4m3&PcXOes$lF;<% zdh%+heU4@6#G^G=h59YebY5N57l!J=cu!V&cCO+4sP{*MzP2u;p6;1oB~@I=gocS- zDc5Ymx2-Wl?$>&3BjUIJ<|X4Mm3Mq3-sd|4dzV>U9D--J*BQ@_?{PD(w6+c~I4rPv z4_#}g4Kg}Sx<`;x>TS_qf$H1rC6&*!QOG8HOw={%pD}c^F$DB*RU?Wf z;}G_Fwff=?uS?lUOvCI-_Zqhx(yRD}U^;T{7l1qbTi1%8OgBUnoXTj8KI{BIwYByx z^+!ggK^S4NcsXKF&bxs)hJSJoZ0gA`dz?u!1Pa_<@U-31-|(P)TIYpA<^SltI~r~% z4ttLoe8%vuY9n6_#9be8IDK+#LA6Ec{uQeq*rUc@-Ff1DSL8#zX0x(xC8+KRtAk^@ zz!1XvW?|J2Ngmu_^jc^s1B7jJ^~a8iIF+=YpUP>>E1Ha3!P(wpVquDu*!uwH#Ep1# zLVOaUt6h3T~{Rq_& zp6IhJS{VEo9Z_Rp?70W^mp7HZor%(VO+4@Dfss&TQ6ispC1=-BJo~Lo$u~e&896~N z5)O!$C7B<GbeJbbX)q4q+-KFsX^>LSwUcsBHg9srsZu`{E$+$4_Iw57Z zWGYgo=yDdt+O^`#prKqc3**&f!9X9qH5i9;S7y_%&`&vL{>*0S$S0zLy!rV}l1#fb z_!iYe$odex+4G;Guxd_bN1&lj=K_SnBkSzh4z1^j-bQ<&yH5tKJl=^-il-RJ!qCQb*N!ibr=B0+w2Bb91Lc<$=e9>tBHzqo zJY{QsY~BLyu9eadn~f>&=XQxQOdUymCs&N8!$PKoQTPwG`q7) z6G!@(ZT8bi(o)f#4(ingULgo9fNa4nHk6(h=&zU?+Wp&sq!I`}#&j^M4Z{lci(kV= z<~Dyx`dAK4RG(i*@X!MdY$wxW>JwRFqa>TqfiLKp|$3sP=dZrHiVpVm6KUCHAoG4mqnw6%E@STIJYn zZS^xUIvgiW;C8>uI6~i%0wZ7#ohfvBcN$nSSNSEvpMC5!CvQt zM=8k%P9P3M^kMk595Q*r*-N2vnLvT*=@Khk4<~ z*!jk-r7Jh;W6fPFtIaC8hzUvi{5zs-QdjrM(JT;C`qyMBG`c2 zzFa$VOdAY;G7b6inR=~sT5VLtcyjx^trv1dTp2Qpeq7YaB{Ane-qYo9-v|~#v(uk)gd`$cRQvFhA zVkJ=v`hV)xp1mIq6b8z2UN2H4e+>mnd~5aX;d|CyIpZTU=9-nKHTF%Twp$uAkA4l0 zPKSL9j_(IdPLgK>2ky=|&x@2WxG}cqVR;Vffpw3yH-bFExk1Pw^K8)#pon@PPxNc$ z`Hdvc*==w}4t4hlY5j-E;r`DXJ<`!G0R*-sd5l-64Ji%V5C|l!`JoB35iTc0El8r0 zKbl1f&I(!rtsH_KZ%p&SxmAvHK8|ljYqvBSGE85U*HkrQzC2wtH^$X~yJ-Y!WAS4; zNUi)&f)aeVCajkH#jvt1GZbV76kLh;b7KW(eYg=*AIJinp~xlF9MkU>C4Y~W-M(09 zbmhaRNa;sRknr5Qf>IGHE|onD0MfiZYddw$^1v#qe06vcWjP!zsvCJG0(*+0VH#VK zHXY~kVV|ru${DEpUJjteo^o(x*;zqIiLK#IRfw8sOd2{u*Jt8sW(N?|3RQ?n__z~p zq`!t+Ts5<91WcD?Y6qy-G9WJ1UpW@cxQZ76K>qIZ*;<6SL0!TanN-MgKRT81yf8V| zDfQmVAfxbwIxk0y0(r}3@(hqp9<}oF50=nkpw5(RBH}T7*u$wL%CKBBW8Vei;(AqH zG42+DF8=U}D_UlJ$MpF|lBe2qU)C-K?enx^q_w5@+@3u3E>j|MI;3)lQ!^|(KikI! zF$%%kMaIu8F>^~GzWabe{~-r)*d~xhuZZ@9n%&;i*RF1XD*tX;6>FHO)b zHh@xp-}b`-TD(T*_}>t5t^&L0?4lo14AlC6{4Ej+$J<&j&tAT016Tq2>Goe}blmuV zBh;0iV5>x^C*nsMzoU_$bnGl#LmS&h)XpH(kN<-P`bV%RYI_uP#Ih+60`8w>CLh^{3C%YEoACE zNA^Rw`n)VpE2Mt!{k!uN&A6bQVk{JVroe=PFBCUSww_QsMc0mF0oqlK&o0 zc^3pXrkMn5+)K?hX$x^_6x=T;yZ9<3+i89_O8SU5{`X8uMOU1)s8{u~ppG=FN{`)0@ z6$aV?cqKyn)ky^Gv8}oN6v1`nj<|5@XYB}AWfZ#byS40h(3tOmD*<3iRSZM$uX-u= zS!`|!GTc4jtAfc)m#N2eWg#*mg zmD2^Zq@I&GR=Iz_r(hr+K6@MByP85J0;iRUQmL~Ixq+DL09BQuaYmG~|F|`gPiKEK z_iu&|ECERd40h)qZAJeVWpmDE*>EbNnt+{otd!HS;Zp!pmCqFaCsVb`C%nm?|dGW5XZcywEC`xQ!N1 z9YDkUs^!}*11Ybo-}Cg?RCY2teiF;@XF^fD;n;C+PK~-9BOTv1+pJbX4rlt$} zGN<;BRSfWG^cc$i-M@VEt6n+nflY?n2pn`*Ce|DvWTa^Ei7(5H2<*tdbjrT+cFw`K z49-!so8L2MAL`|qtd~#Me2dBZ%41C%K{sxD8?aGkKX&4SCNtu5r1DO*6YG#z!h~EX zClJnGws8aH``6V)2BsgZ4534p<|ZyCmTXO_`qr%G&+-aC(l#R4r3zdYHyI~Q6p`yE z65qa24`eIb>dlp^drHyjoc{UAwa8`YpxCfKy2vt9X|p>z%p{D0-(0OqeVSUapB^2GK%xG=c?Cg2I3~zW?9A0 z^&ki{z&gmv_!m^VbLOeaw3#kzk%gb}FNh!*7>D&*A)CzzO+|vGdg0RjH>*^`4UHbM z+Wc;B$6IB9uj}eT>=`M00#x@F%0A8Lk|8dQYZwrYJ(Ee)Otf0FU8iHSgWdg{P!cf< zumts}{vwElLnc@;RT47Y&VDLS{iX=M1=ABkonqlln%m0pv!U^CfcQ78OR*%#rHF)& zz21~p@eAwr{_JfLK^l7$-z#9kLC)Ko*N&YzG#yvhsoTT0Ql~E01t8Fg+m-C{R9<$3 z@>ytcwUI#`^{4$IV5&tRf!6Q^2&U4USaBoGf820&bJWM<#g$HVksntx=v7EJk`T7I zvHp*XfPRzj{*Htm%~0kCz`iU$}>I^NXC zvbcA=f|sitEm3csBi_>=pEf?y1odIoifASVK%LCYpOJ;r_N!jK#LH+lRkrATX5^dJ zHlW3QydkR`>hC~VuqKMLaV_Fjbd;&Q&S1``?KX;&eks#El=TsGaIj7ZAZj7~VqIb= z;*3vbGNa}RPAYkC%_!vmjC^u2>c>KeEhnlUhFQ7hNQxf)Y)T@Cz?ue)%d+Wk+|)-h z>W!Jbg73+NkLxY8qraUoFNf|dfK~hYH_xen>nsur`5{i0^!|O)n-_k4^Mntc=~%Pz zEk@c@c#=J>nVfF`1{_2rf5f`2{AqclS5Rq7R3@1FmXRobrV{{g+KK)r%DtD5QP+-u zeIMA+Sj{lUgD&1w7S_3MT)=c5Q%0sA@^PozC=UBY`a?TWyp2-J0}tmt+^I&VUnK?5zk!jt446OI9-~mIZDM@4O-* z?B^VnD~Hz;WIUJ_={ajS8WYSG0Oy&C66vx)Wv!xQXIQynY;1?0eJQ7LV)Q}2}ebu%C%2m2E7wQ*evLrkox+$ z>|W!T1?L!T06b=3%_m2oTF>LI?Sj>nbdoGK=HV5eV(G9^hxY>vvIZF`I8;e^E5rtM zl7rn_@VJy+#5|q03uP_VO{Lt_>Eah^8(cJ<92w}7o&XO0V*2(N;(9J3aHz!>*K~Pi zHW-_<*LdlL_2&srJU1Cfq%VZ%z&9`;;|^ePwhC`dqQ4gwXt!-fXBe=IWDhqf3sYk) zLBtjCW0z<#+Y*VVx767++7ySRNnSNRv*=;jot;#cD7UEu4;}zD$i%Ka_E#=6tA2{q z$W@ZqiPDQ;s#7w?y~rzz%yP;Jw+xB?ri5CO({6k)!X)aHosvIP+z1z?D$fgE?>7Iy zU58v_vLi=dn0gX^aBW1i^)^aU($NY(2{ zv4Yy1UsamOJ}RmWBe|rgf9igAdm!EOmGs@NpI*UV_nQ9l#AONf$fELHSKB;f*y*H^ z%Kb>t`z2MI*mIKofrYZ#RzJ~I3De-P^0(cz}NvLFOeGFaqvvCGpKkALbMg}wQZd^cLWUk^zkZR z19BU~D>z|b_<=Am$6sS>BL55!2Ltdr9(A_Ll1$zk!mFW5w4J)Bw~>et$Q8tN(heeD zRPDHh4kzHS{djP^s5HKUot7p=kiC)~wi1m1QrSo``ioRvTcanuJu9zspTApUG0wt& znK64wunDqqn?Vt;(z#1{TZSzq9Ob2 zv9NgYp$p^q#AmOpFLv50?<;{kXUR0`p|w&XtQF_YHg7>(J0+I3oD|(zhHKd$c}yv( zXEL$v&Amp>cMk?3w*;^sU~JBpEYGm1dwYfZ70PwYCi&G*`i5)YFDYxdEz80B8fv3P z%Bh;mT`8fI-55yQHkH5XaPJ|~OoN((N2m1NDcpXv`GU-lLrwnO16V2x5ASi1Hs)nvP+Ox3f9KD7aneiYDjh_kMD0s>m43w<#Bq4II1hjQ zxF|`iw<$E?%_ROe7+H2hg)J+fThbHByg(xShg8-ZBlehEqG;YS+!nL#ZqOloGD0)UFuOXdz%#1Znh;5^UNYznPcBGcAi9P}OsBY-6}S`nSj4D zWLQ1-R-&4%)_tR2+lkZ_lsUvWjV;^NDv;0(mVCpCT@h2sLzubU{KHN|hLLXY{nw_u zdmfM;pyP!0VLxXWeys>Wxi4G&dP!+@3cZj;r} z1;5B|e>_I%-!}&n*D&C?A!pmf#CQs@FPh(kaiGp;_ zaK^rD04VRt9phQZZgQn7u2=n@(kxcK|piuBii{IQ)VOPKec{4hOO7eeL1 z_9O(p35pD!{J3NHpUeY3H?p8sm~d^YkI~Z7l__!G7U4efS zL>MlpP!;#9*iMg7h378D6_^JDx=Yz^O-F;HnrE9mdCe+roA&7o6!=g4u6vqI* zr6+gJj>l-atRDHPoGi&Qe$tgv2KP)bs%ifQ2a9l4t6?B;>Zqra*0C0z-S^ajGfaCt zAmf$17nd@d5WhXW^L~-`xry8qe}z_QuFAO@SdGty@G>jKZeMfFC2xf<+XJPgS(w#p zKkbX!`=msC2lg#$dRrM8+B~HC%tKhKIcCPpM7q~ zMC13I0Q>Ufv=z{rxl=l!(>WdrflWCy0;rG5>b-MKP`BehTYphYK0+Rgm$d;w!ynK3 zQdjjD{~)-$LvKXk-ApB#u-)M37OhhE#LqU0TA3VGh z_b3_CvY{$3-hidsQ(UQ^297?7{c^MRKOx|{zaSuI{sv4NZfu37@;7Ycs|BCB@A7*c z4dGNQg6<;*>7aFBT(5fH2ar%0c{OL9+l@I3Du>Q`CnyKC`DaOGnHg;5V1E-vmOHsQ zJ|kbWP`mb?qOWt(!`3aC&;`wX`ot#c6Rwex2;(3-YEDLOjZ<+-yF*8$kj^0IotQU< zP;SkIc>yhfxWf%>)*KV;#=D7!)(Oc0@E|11fbSf+QnM}Cb0h+U3j_qWuh&v#PxI5tKPf&Y2>%vxAhEN)~agD z`pH8wpSP35Str{$7j~;-8_{Xd_I2=lmc1w%4urLSP`$C z+yg^KRo?1Kb=j%3#uZbm)g7JRGO@wB)8}{ND`tR1%C6$8_W~t6k2(xnoUm!~;4;Hg z0P^$DKlFHxB1T!Oax6LUB^BngrZE$tfIi5%RaNu))=EgR%LKz-*6}UhvpUKQ6{U6L z7$?=e+N!Y7*teUhEGe>WfmQ^Lp7>=(r{#@>K)##SRDNs@`DuH}U>F#>eJAa2EL*fu z4$-L#xToI=a-FWvb!=4m+?aax5YnQ=22F%~M#WeEL!%K=~wa7V6y?A^3J^Xa%W|q&=c&-mj^t~^6=*;xYAVrhV%TUALl1AaP zZw!!LX;p7sZW_QX4vFA(=Ul_oxUOc9fe=DmYF(R}2gqSuaO)-5z>IpZSs8Wzl0t-! zBh_Z=0479`KhpZ!a;iUx%k7E?3DU%%oA9Q8=JMKNJz*0&ziQBLGv)N?lV z)MiHZ%-x5Xbv&GH>72d*Jzbw38xCYC(~XiBb7aqG=f^c1dU5C!=$Ae!XX4KLZEN>o z`^=8v2iHrdu_~k0#df%=_P%ntCUsNpP{+Z!efn@!lt*LES-kdJ(J~1TE6YQLwYI;8 zn0U@c+#a%y4}{Db;&y-E-out5`E7dYF9NnSJAm`n<%e*C=c)m%i6JPV&leYnI-dy zVa3U8+V4vWCB4!=hybwrO4u&ePgsRR019qr#GWHj>agCb{4rj{9zX`e9V;W`hu^Pu zN|m{XS9hBPl5v``#)t21rGcHz`Fy0{9+E?4O$yKwC-65sAGU*q9WiVY*o5p z1i^`xDoh1S8cDS={sZdI2lH_554Z-(?L;N@r^wKu)HDJ-I!~+9*`@1c&{YCl{Yp5G-pe@>G}vCycyQcwFP_xey6<3S~4s|lk5Jh-74eM z)7WuE+MlUtCTR~Z(rjDqC7?J6cwW550!jI(srTxqB!BzI8)#}+7Jb;9-cMh_pg9^=(`#5&5(TXnhTc7 zeWkG2ay(zYt?8@$;bB=vp>X3{HGL-_p4Pvpwic22FM1+>P`+^Gj;8P??w33V+`=Ks%_)x&B#*miA)OTRnb%%dSCdfw2F|)h#dvk#2x=>x0GA&5q9r zeR(wZLq#``vSJb!nKy~YR#uEv_!=>Y%4=>$ClyZg-^(K=-=e8KR$#BjCl3w?kyoaPCSy6;DUkgGNkzP?K;nv1P@W^M3-leB4Yii>j{Bc9XRl<^NCQ{y#|M{`-I1 z;{WRT$zQY7TOt+Z)}bRw){;VT72&Fw_Y^a%nRV{c$ks(5;rnv?@~)|ex|b$mW}q_N zn@qIw&7_9Yp>Z>Wt2q^4gC_!H7@nFP`IS?|RDq+#raa8(3+-m4^l68&ECpR~g%;)E z?V;rt3Xq|NsMbH%0zawHops&^>2S9#x>-Hx)TlH$duFZV$r>6(8*Wj2o``oy8oJ)l zoblSE$}VN*HPU#P2q;CNJg}(tP?X>Bu~_-!F~Z=FNMzOixJLR`Y9b(>+(QXi32@ z+b%6#6Kx;YmdvaEupYhQ{+j3o?#vzjw00e2u~4^K4Jx4NfuQW8PS|H;U%k#EvyVBI z+F>T42N^qTxf-;@2gm!3@08|T<5qn6i|5O$KH{={g4uN8!@M`W4l7PNNSm8}GM%(V z_n?3QyeBW|#2-kT;f&`9VMYg~ORgZt3x43W!Dw_jv*ulfwCLH|9H0_$5a3N|qTfXZ zl(NBl`%-AVoAOehS?-M5MZR2`OujqOkYBm$s-TzO|SgDL3t{Blr7C#Tle7iYxw?)20-aB+kfH>W&lcFAJ_u&;M+kAnoB@UCsxZGRr#5VAtuS@Au1#oU!dnN%ePOyJ|oV0RVyY+yc z!xkg;rlS!dPWyvTCL9<$;ciVeH;EQT$%SJM473TUDejERA5A-PElqDr5QCmZpp|3y zydRt(2DTDAV2P!I?jCC9&{NT24-9YSp1xUY_3sU-ZsvaD_rkK4Mv~>-??Gm@s%+l} z1!+A>iKF7f(+!pdiFYHY-aQ8{R|v~#N>YZ|jhvH=kl_&U-x5>&u!rjdy=}V z-m|?ml0ziNZb0<+Ont4_HN}~&_(F($?(4K&E_D*0hoAP!xIQz@Rdf#tE2M&B#R2O& zWw{-JxRPrJMkaSeP+i^L7x*aaZ`$|lYuTS^#$2;XZRt~do@MgDvujE+wMW_MAI}aS zma4-s(bVIN!+SHxr>wQ7}r*yawKaAz9^qi4qvk+>2nR`PL7o z!r^X_;7d`;T1MtOu>Y{E>kTgq`q|CL$^l!q z>P5@F%i0B7ZL0J~)}xHOM6Pp9arO28^Ldd@B)Np3+2Ku;p`Ne~4JfHM9czd2H zWk!_}Xn;3DE@jTVmw8t%3qS`lJ0>P!&ym%IhqU)b9-^U!r#ffIShpt)%rt4v)i9g8 zzbju{a&5lFNl(?$l(vRZeN`n*_?bA?iUlB>|)@Am@ zkYN#UeVo$!Ru-_kI0!sUdva~vvr~)`leA|*LGYYWK3AfAm5JEi>_ZY@`oJE^!u&_f z{z`f7dwCu&CX3m2iqIx4x!;FAtlrVn{T1^ElM=*MC=d2#hF7;x)^^UFhTlWf7uMdp z6fwzp=WscmYv8y+uuG z9l;f{hL$Fuc4SzIbp;&=zJ2Jq=jkd-L3hS~YiwW*MC6L{$IW62PJb1?h7@@tP z<`$=aXI{@t*A<#Q?Y@zC|2J@hLT(2_c83&fXxZ>E@$NOZjy>lF6jvpBjI=7`66w)1 zjKJ7{MN(V^Y=B~~^r*xRRR6|6nU=PMahFmudp%R!jcfPRW3Kwg4)D=`G7!6werXJ8 z`IW$mmO-hLTctF0RN?Ec5(t&o*fE$caB;ods`Dm6_})R*CoMG>gi4jtgO7VG6ww)W zI5ux5ll5HfG4fPzQPn9BDc@`QCw5AU~3=pSYqRpef^&ok@EuQ93P^{#ot&Ao0$z@W{r+g$Dvg{_N@f zKiGTEs3x;$e|TmrSYaHLBF(X((mO~~6dO$}2+|D%5JD%^gi%CCX(~z!5D^8D4j}k|zs zAxw&?M#{nlvf>_0;>RYy)Bt1^rWAChMC2NRj1f88cHDSnp?Wf=dhc(C-=~hAj(0EYZB~U*(Kqu=`)(& z;4qDtM4Mga-es9mDaRTUcjB8H88L?^;ORcaoOuLMgyi{zqcM!DL@IzD3D*fmP?_Y6 zlY5Z?YQ+Q=7aC6D;@j-doTq{(y)KG(67rXqxs>Mn+-ok58U;lzzUkCW+>s}co|?+K zM6^Mz!ijPHD^2@-W7Dc;IoMW|@v=`eT~--C0Ef5ElY+{kN}J2$mUSXU1kY+x(STsN zc+3L`RF`0aQ^YX)`Z{W$iZ`XF6gCVM~DDF1A~ z;WSrMu?dHwhG|%@EJ$l3IkPFHmNFYcS@K=8#q*sJ{_xUa)ac=YE=|>{s|lNOu0_P^ z-1liyK6bQ>KB32=2biM!%{|_`^y23sdFV`X$1`4JvDjc&H&LxQdvGQ=yFb8E@v-oA zVi2g)HL41B0>clw(vM^5C!=!i;JZH{IrQXY3xt(Ai%G z4~IX2#>T&cc{NE6dN4{0RgeMAVo@z0uM54c9WV37g%QZL0(D5ac|5Gyfnq81ER>~? zoWMZ)lgph)+_Jbe-6os-Xtxc7cB>f=eLO^*J);q3atSg;0XeGIceh~11v=!gr&thoe6(V_@ zdwW4@8Meb?u$$E2)>%=o)rCc_c1*5`rE~mv<_)>+W?oA)1q`fUM51*?d-|v9^e+3B zdg)A>r14YF@sL?Gf#1+*S>9|qHoYM5!5Dz)p8Wwd%6Y{{!WvEb04MH#YfQ9th(PN` z5SF9p(xSns#l)A zHi#z7>#~#J7!@?7WJYTgdJ|HtJp!$NB3z_Ta)DN03DP&lvYF`yJ_x3l*XFdoQ`}?e z7n>D4Lu55vQw_M|TJJFFZnT!BG(f-vw!&9k-POXF3Nbo_Gf5qxOmL>jAxi$HfMAT} zWS9v~OkS_)Gc2mIu8pJ6#4+zZj189gD|E(fC&ShLjs8A{*C*^H%|@%R;!XXh^M4&m z3p6gKq#NbK-tT2(=r(v)s7ayeeg3|N%`4hZH$M(E2*N%yIbW~eG49`^mZ9r%^H(iM z#4pW}jI6SZV?zaZ1ds*VjS`jDp?&D}EXR-8NL8odn-6^oRMvHM^6-K17Wly!sM4U~ zw4hzrD9GvZeb9TVzg!z9iGDGq#$-n(S8`LDWSQ@v$bx3(2hKZ{+3hvW(0Z`5pZoA! zRWPK;5%VloW;vX-<5QWdpe=GDjBP=2|9CizE`v_6l<_keDe}q@-4ZI;1bft{8j@8r`lr4ZC&jEz*aS!vTG>9J~}$Ayi8i$+BMSO-A` zqRdxOIDNv;28{gV%LUEn_BB-+`)sn2*hAQOfxA%iX04jh(L`xQZ>so;Th{ov1`tM{ z571OY)u&om_&d^4as8TYOQHo@HaL+2!sL0D>v16LCh1IH4M5j~9^IaeDqOEBdWR(i zhra3zhtd|`wqBlnccpr`nFGBH%kQ5YnpE#?ddHDQSfF!m)H$x_J7>6-my%ifrKXpI zVcLVtM)W??Be94zqE3S!V2j+FuN)fg!Oi&xaP{4n#**9=Zjds@ zY~2}jB9yckA~m-R9eKzN@;70ubSbFC!JMQ*&On|a!ZR*aShv90Hiiu|WNoYUjhreP z1DDP*|{;8DN!s$2hlepFLevATudDAYC?wZ=S+?YHkW+-7Cnkpy$3 zg-_lNuT@C&!7i1i$qmJeTayrlOrA3!PVK+Ljy~uH?V(O)4>z5ml>fpmAC4|+McK+E zph*dIFKiA+dyb2ebcBdPbw|yMkX|#pYx-2fEoC$u93t%GJI%qp0gqu9S!oIZEM7iv zLD*xTOFb!InOsGLXXU4t+=>nG`*c_Yrrz}lF>diL>^w@i1!H|$;LD?SnElq5!XZLg z5-OyQX1bnmIW*EyVEB-|I&6e5C#Btk4U`||M%_dzRIen}8ryemrt`_af2^WY^_w^< zm{<$-@(PF-eh-L|AJ*YvI={f-BZMUy11nb>*@qgJld~^9qT{(x`k1X2ry*=s^Ay*$ zjB)dR<6W3$4U zPj{kP|5CTZtPJW%%5;M!tlr}cH=qBIVVxhwE3fI&!TkoU!*Tbr=sf{iK>7L$qr>rB z$%}!HeR4J+Js79`Berd1n+3MKG~Z@-Qcz7e(ES7c(ZQEF=!HYT>Ha_iZp||BP`CA) zat0M?$m#70#K8O7;Xg^va6dJk{0(#@4>kf+5^J5EdTG+PV#cs{9rOQ;ynV45z$`TZ zz0|;udT+xX3*#S~8Ja`i9IwiS4QKGT0R3!#cw=twR(? z)qA%j@PhdS~#k31s4R?Z&zx1IiEtNIvSJ7MJ3?KIotx&Vv)Nj%@Hy5Y@@Dq!FuDrA~g znpOj`NqzVq{0MNd`KKSJ6LHT?%K_ljolhM7=gOg0g#E7GBm6egD%0?f2T)@l{sRX) zo5*~UicOpY-!Q9=4g%e~vV>v8LR?n!1H#+60jsvNTC7LC$zvzK2+d!QSoSO7F97`_ z^MC&&AAi0NWx%i7EV`&ePGVHAi8F(qUA-+Jx|X~0*6~dv>9EF}9{6^v*|SeStUG{k14MpRcam#BsJY zR87XW&zwE~L3C5U>uOYUgxrn}tXb4cAQ!ms%0>^>{jR2>!-XY(J9Yc$8b-iMFONYdJVg7FOU zYZa8WXk}o>RznJXcin2thjmBZI}~TQW-bgx@e}_f)Ssu^-#;>r z47OKZ0*P72nHiY_*h}OU{7Xy!*9sXpK>PRT8|E_~QNVlI zj+zWeGyuJ{jo&Yo7lzNy=Jn>%O^01J52;1v8Groc&^B~44B)dJcmLw*f0VKUptN>Y zJ0K%|8vU~gu#WYJwnk;_pv`9P)ab`cRq1-?j3+q=L@6 z3|H$LuAA7!835*r|I-29fhQMjz44p=))4=|I~^xT<4&5f>+T9|X~S~wPgm+3Y&{X% z>9(Ug$?zS9b#FT@u37d}x^%hkhkZvcJxN|Q+ubNLR2ywnmuhVF1KQ`ROQ*JbP%Xj+oXM$vzorH1 zso=Za5~K#=m_^sxw5t7d|4R5X#!WXkd8NZh^x!}lek$=+n$N^e)V?u`jA8B1qP_fm zRH|z4{+QD?%7WahKR9*M{BnK=T_Ut?;->lVyQS4^30hq1Rr~id&EOqzZ9Z+0Q1f~0 zi0Ed_=*N;P@O=kfr06s@uN5fYlek9@v*3n3jlZ?Ry|M8dXK1Y$}1@7pj%5y8q=@)8|Pe9 zPg%E)S~0k@B;fpe4AXN5oU=R1ru>vK7}iz5ORxCN&pY8=JeKN97kca{Qu1**+A}Aq za_ye9n3o&#A^hIY?_iaw4!G@!o?qFU|BjmyTWcsgj7Iykk)_(URG~>B0qal_*l4Trj)FhPVJ>10M z@Yk4EY4bs2_5 zp}_~8GHy!FCGesz9q1gER=i$m2^fOQ*@Pmgk#xyrC?r_beXf6Chhy7yf8~zA&~3WP z-guyoP#B;IHOje5<&>_eNeawgXogj(E-lKtj=FbAmB|gs*OJe?W&pVxTm0%NWxpm} zr)AopM|ni@47um2PqPpvi6V>(YsZg^1Wz9m_sUcyU@ zb)0snNqRqe0fNj7F+)mQoNvFv^;g@wpLy=Gp0DX;{H_HvmEkh@Hr<@2z8KYx!`Z84 zn>nw28-uXUC-kg%2JvRt8d1BE3f?W)kXT`<6z8!qQ7UKd!ByKUHQlmy}VnO*1|3Qs> z6!cyK1MG`~BWSnypw}5N-<Px+nN4A`Upsn}%yTBvSCb5W_Fuc^eWG}4 zKJ9ql8WTxjwX4{z+j6~1rCMt2`o72-`puohw z8hwO=Vt-~Tf8fTL6~%RO@lv!h=b%dkc~W-I)As-Fqt60y=I6X?^QKXQ}TyyM5D`ZP@evlxy%H|A9 zGYFZ%RL5SH4B#7dTBm4f^*HWd%Ja+hoUVC)T8ag|XHm4i_jm5>6BBLp`rd=`A|iBn zhrWL%j1}?WD?gEQDBW}2h5+iu(ZOrF;cqkK*K(cN&JQdiT2tYDwwyvAI#_n#5a%e= z%&v?6`^}ui%twv7vBOO%;xAJB6>x8)@_mDJvHJJ?nHtT;YBFu?N|d5hbN-U0n#al5 z+c_scdp!kUkUXfZ$%q*fu6W|hu2bof3|4;CEBV6fEJTgJ3C&(NWoUVHJ1K-}D^WK# zbwjyz;mu<9=nO`2M!Xtg=_AR7Z=?iWH65M~0$*aP)R{aWRPP=1axLePhH>;cV81gv zja6TWW#c8d%TIP537lalSUfj*&pDVWCER5=Gn4zZw3G!Ol-zY+e6O0(V1dZn6Y9U= zMMTsbdOEJ5Bi>5MZ^>HflJ(}(WsvGMvtqeMREddS{O}p#<-<)$WCMd|Ngap0{A4=i zjPjTK$Qs?<*#UPgAn!W8uS&uS7-%0k)$LHD#~;uIIIkgnik0Y+GQmA`6CsMf88v=|Y_93zyU#V??^b(Eg24>47;SdO)pae9%% zBBeT38Mvmb&Z3m-mqWW=u`t)Xg`|)S7t?Z2x;eOctpSgBi3%`WD$xl`5Xv4|5v{I_z?Fs);+90hQ5QNdZb>DVbYW~Y2=Mxrrdh~h^LLRGn|wy2 zG9lrQr|T>UkDGI`YN&vT6wGqS?@$@qQWmJX z*C0GrMpriDN&c%(SrX;V?S92toZabAI$>e#Q6qUaW#-uV&72dlgsf=!9B%1U&EQ#t z^xJRA&5pYmt3J};FWNAX`pWw^wCEK1i0>(}{nG+Q%hTAZoua)$PLjl@7@|08q z_tRWo?AB2hhpS812@Fm*tli1h?LQ!Aq(tcS3f|?<4M}&@o8{KLe_4~m%#swXgv+pz z1CfnrsV2Kwp1+GbVl$ud0?MWABRk&8Srl1yX=n(=*8AYcKC6??5B^I@M_VF(LF4^h zMr%D3uvMUpS;Y!?;AP} zY>6Ku#KHues5)+>S*+jIlnF+C80lh`M!$VZ5MI!~+^XNj?|v~T>qp@V-C%@;uJI3d z5lsg^ScbZIC?HzL%rIN)B0p?-8^)$tq(5AuT>!LjmO0osZko?T(kUX<8|WdsoXgS1 z2uPlb;%6P@9?NlTChn~gV9AC7oqo|}Vp|EOo|dKq)-_OM&viiWdvXKj4#{#gg+GQ=HMaVb9=DixzNLG$Jy;UY# z_V$YQ5CoZPtYV`JOI9l;>c*{KB`y(8*{HXQ-!>4boc|FM`hE)!1~%o*6CttrL)K@~ zJjF|51I^sgU9FwNP=yKl&{@NPjzu5=JkL-2V!9s1u8v|5XbDWq(aDHk62&{?%XC~E z$|q2;!klvBH12j)f8B0Q#;reeQh|(?pOu1qR+;q4`sr4XXOxS^4W@P5r z*?M^b>o&KVplfO5stzclHJ}jn{k)#WRt}qk#dQdO$*dX)A=wO;b967`u{y-A0`NTt zB0DJbibTswGD@JOe@p!}P`4vyzp>dC;$zO!kcxv^8Qr$=d3xWHcsBsqh{Q!mE7!@IrmI&X5GS}+>nVv zp#Ny~RuCEE=YW)!LPXQE99+jTDs{XBH-TmxACaNXJN(XiSk<5&)Av6-&tVW;R6;%iZPd%n6iGc%(Zq?-63OaQ+^WTCYF;anLs*f+HsOwzYI1kUXwpv=hrUNu zr=&3uWS3cI`}q(jYR%wLH9=|TH*59p&O&?7b6g+ zdQ%>_d}w1Ls4lWw6koK0;_xL>c+-5n1(^T`g8^KW)W=h7M#AZK+b;Term)=g?wb6KR!!Gxo7|M z$Z}=z3t5}svE#i3A~@*~_9nrb7xpxe+`@fI^ejQ!IQT)%7jV51@B^!YK7(ytI+Oqr_xfS10VPy{>$!L~_IBEQROJ#*Ar*;e$D=fSW zP3&Ak4n^E|7#VKz?WQ)ZGz~S$mz>|95-3wY=2*AY0e=)*YE_M`R89xfbqKz1qr(SS zzhTc#0hJam^ZGD~vJ*_t=vQS6@3WX7Krh_}#6VndR+rdx1-o^AGn3>oxS}39h-rmv zaG4E8HqBZ9?j_d0Z*W_A$cJH+$pd8bzRO^>)aT9tlkTY5z-A2OWrivR{L^_d=H=(; zVc_%4oS;2LNhbDboI@WUZ(MU>Wf!J9$W!ZNx>wWrrUTH27J8=?1={-0mi9tSjNGS& zXKA29rNh?J-i^bWQT=g80yW*Prs+-G@ELsUbVkt7pY$WSsq~@f)rSUScIYI(2co&7 zheZSiPG}uS#9eD;X*$>o z>ADb?I_?Ou($rN+=~&c^#;87Kb1}33cUIT2UySHV)~JKMzAd;2hD8hf-T05Y;k?K2 ztWyO*5!hoj1@JW&onXp_4BzE zDrW4+SD-?aq`eU_kJmgx#_BoyJgQl=KD=hRX`5_@528RPskoKo~Z$ zr)b%+&4d8B?m4_vH6!e1N?Q&*4J+qMuF#h$jy=hNAIEkkZ_CvX1>?}Q!~u@+bR2vI z5*whUiPKLAYVhOe&ra7R4y`}tQ8^fyd%l<^Yxlgy z-f6Jt)ij5su;-O)25!#0xW?G?DLBpA>rwqG$UM)Y8wYi?N7I0CnziTD6s@!_`Rh z`6K2nk#)gQCdDcAk2i46YsA*p7@qymBrH7>(!VozQAAS4CG2sA%I`PYKPJ6#EEqS- zcTOL=?d79>DouRl=Nsn$usr9&N|#u3(%j-`Q&&s~C6#^4+w>{YB6#V((U>PKKBK}a zla=P;ZCUrI+wBMiNS;2a0?@9SGYEWv65Y8#G7}I-j8Q(Mi2| zZ`(|aVaGw59^8a{c;^|4QlIV;FI?Mo58YcCk}h7du;H(GB4Ta^ZfJb-GX61Rm*kg# z-_7%O-eT4j)soL~r@f-DNE%&wW?;KjO!J&;&D|sO~tb12|q&J zkq(CkR-vu(o7Q94+BzYtY3uQC#v5U|Zo4vES1o?KeD@@#y8;s0?Rg^-j0t$T?!u32 z>VXW-zK2%RwY$%GI47J?b?*!KFdj{-Y|~%Ckm9?x{*L+)r)5P7nU>VS>N@F5p2ioM zVY5S5us*AqXw%7|jsrMo30ybSMJhW5Q-I7bO|y8EV?r$)4LhPo4D@5EMfcU1Pewi6 z))Os&$1cm9KyxEMyw{hzqD=ZD1D%*|e5u?XU$pPf^xx=%9W*RZbah=%BKwA(n1%b|GN#AHqK~wBG z`6j!=Euvf-^_+(wzEz=4Y)JD>)=-aHH+ne+qv}qci?`+Rw{MyHly$Vz9o2s=ZL~kz zY-e4fLHLxVAyBQh4*xq?dQE!~3}i$C3~(7Npa+zA1)L1aqtz@V6)~IL6?`fCX3m?p zFJaO^Ig&S-el4h$hQeHn^31NA8t@EeCpjisU_5v-q)g=|u4?%3UV3hhs>>~nvFi9w zfzr>pEG?H%w)ft!Cg)%OICUV4xl+UKguvmP_LIh9)mpc7k$P#)nrAb5N?8L5arYUb zvJ5VUrSBfyz4`atyi{Imdv;Lepo-SVY=&wSOVSVF)8a<_t?WxBK$6ouo^N{xFPdLL zQQo7E!%Ql#W&nvWE8;$VH9i-E)zy1mx^La|Zw?!3{L(gRjfttNxlG%kBh`j*_vA!+}`!=LI8zJ%Qaqc(G>yTR&K zA({x|J|fahRTY(Da1U+m^a$(Vf8&{M4=7tH`N-OkNiOE5P zuHov=c<%XIpDP)XoUUuYx@(Q52W=0aPLkr!;A@vYzka)kq}Jxg4mcxd&M0o?> zHC6@+NG&lvwi}cQF#VrZ6JH7>vg;L& zZC&lIiK&uoFnzmL%Vbp4Ixt#t=Tq^P?)Vedz^DN~S9QoIE?cIC3d$Ub8x?stZXz9j zOmTK8m=-Ita5bG;YO(tEMK#?#faZt2ocyEnm!#;Sx-B_|sQn6tS<|oZtAWcdL2`Cx z$UcCj-e*s6%H$-7PlK;RonqmA%d^=FPqz_57ocQUy|wiuz0AN z@^IGu)SCX=E4>Yxicc!9ZBEMWzU>P)aQr2n`~IiLOH1$8nmXX~gfi?xU}KD6i+E^A z-RpKdG}f{G{^({6U)oH~gFf>9LE>-2H6v4In^Z(m_2iBMX+*fedL;B54E!bnCCvkl za}$2`$%faNu4idaQ}uHjH^2g;RD~gF?}zqc%d0pcN*{;V3nPm|C6SGLr{aVm+k6ei zh}2+Ga^+>!IF>%D(Q&%TF%+WFYu8ZJ<;vf62f%t2($OgCZ84k3S zydXb?;6ot>?$YB8>aYUo@+eoXZkJ^Dqwn6|6T zoGm|0-!7r3Xock66VI+qvpFj~+NNJbyJk*3E+EX({RIL=I9w(!AuceBR-4M{hO28P zbMuImj1h)5L$eL75xOlEmH$Ne1c?~?AhH-G)3&rl?TJL$syC;}ut$7PzH`r$$VUtY zcZam_WT$_%2LGhKYDw6Y+B2&gaIZj9yeQayee09;FeA=jUrdkw$}P~|yEv`@1!x1WNu)?z;13YJ;GUB@3=&Lw;uitzWwa_!^M8HipeIpc<#^9HJZvt8r)$ z?cGpGXR&=Z`zrf-@Id`dA45-|g_WW8uW?ldoH|9)?EWO56aIz{7zS-tWB*jF@n+mY zl<6&)o~#U~(k_8p89NA##dAJB@w{72@jEm2@p_1=lG=jQe!mgeeiJBu62ps1+>$ct zE|U?Fj8gj?N4k05*&82>&rJ$Cm*kE|B!tZ|!xUGGJzk4W`fgW;RzvQ9975h3lXpjh zUUwHzoptM0g6eroK@b^ytaClJAp=$c#M(j+g=|!Pe$YCpq9Q|1VofwqJ!c;%t3epF zsq?)&8UpG%E#Fmd@?bo?RkMTYtJ^a7LKijyUbxD?-%C*3Cz#QCYKGMlX4^lJK&|;J zymgrCcK}5UZm1hikYZ8Wjx@!^)eqUDZ;3y=xY=2pt%{ID#mbV*H+^}LN3e<$z>D;U z)CoAK3Mq-*z}xh@kd&4Bxx8?l;TVLOz+YPM>=!syW7$ZzdOl#mWyl7imS;F@T_d3IVh+kkc-v3!*)>r8ja>n`$S=TiEeQS|~Xj9*9 zxSLi7P4THq#FhGo^6J>fFTpN-n76o=8i$mr7k8CSFHx85e( zg9B{RAC7n(Rc&edeSz)j#%neZD=;Xy&OT?@SukjNUsoPpHaLUEjrY?mR`qk$Z1q=I zS!q_2AAUx)fo1Q|q5>Lzr%&OgKm!M-;`RW>B4^Qwk6-FJZEEVOf#mW&d<3>M0DspL|4e*XwcGf z6o4bTvb2!|IjpXrpT={itCu{yW*)38$=c0CFr&-lS;15R8VLPqzb;yX=v-EFWSTr);L7G-L_q+ zq(!GeT}dIdjZz<0VLGC$=tB8$qa!PzUH{Y~r6y3_-)76CT*X7a1-8-l=A21@7`*T3 z>%$wba&LwG#_3XtMncoA%3{|FiX)AU`nD9*&>AxnmY0pWDeNldrx^NW-OS2gy?yN* zQv|*Jpn=YrI1pgP?j@AHuJREqLH(vS?R+|RRiB%vkLGI}+{r7cq(rDV=&fcsWIOc1 z1(y0QyfDh%Tm}5tmRp+~`x_s8Asu`e2Z zI#S0;srQCE3v7RwyB-HM)W?)RT~poY!aZ6av&;LXV*iei*u=sC*Q_}IsJ9xZW;V<> zdrCV|ypJ1VrdWjS!VO=tlfm$6j8j&c(9Al@%C?v?prTn86l&E~FjAnK*m|tVRcgOn zzBN=qHsNys8(@*M-Ga7&aQr~t*u)c!zFr9jS`c2!^-eqv*Dyh>88H!0qyR^%$648u zyaqR?3U8z~b1PdNX%f)UCyAaysOw;?qH)W=cYu(U6LOEazafrjYW$aq5T&{!4{pu_q|m6{nv*h5KCtPwoar+~I!#8TkVSR&-te z0&c8o#Dp|*(z)`GL1F0)>Gta}Nn_3*qqJQ2AlWDuNwy1z5)93ugTI{oQ(r;9f!)>P zE)3RE%;rp#FwHC)1B^1ecy**GaAZCH=9H`sM-OKZ-1dBqeeLVqYl#!>nvT8{I_^zt zOW1qI?(QRgTU+riFE8sqD5iH)@&;U!OM&9I0Hw>;siyJVsMRNea?=!lT7~gW zA-G(a?}&8H(GE)N(@{Z`F)~0$d%_Xsltg9DA&# zE^xV_NG7E<2(40u>e9fCK^gs(6~CyCI8F~inqj!XlcTk=>^SHaF$MG};27hh2)uRDd8pi@1`)Z}CaO)ENM!F-{grkaZT2-gjM*g&OhEm1| zFOP>1-|0oO6{wy>{AWfefp2TDJBSGrEcxJ{0i9<;fTSh&p#|w=D@grHv1jUbiISlG zg)*#$2*063NknCtj2^9lpwW|2cFCLVOZj!@r4%9@uL)AP*xH>%5eW7fq1QRAb2u*S z8{}~-RRB-v!8P+)DJ?YPQgC~tc-qk(m+Fn&$l8FmN?!qgzJbpKWjb9ao`m^(0t#Vr zP;O>{AI37G%!t%2u;FNKkgC>+ng75U*W+wiEZ1_8(ZIkYu`C$`(C7F~;`)A*kQZCU zDkhI&5xPcGCq$@j@{Avl2DZ-&w~DRZ#`?{4Vf@VVW)HL{`xAi*5(YV5NJZl!XE2B- zhE4LK=XhC5=H9hm7y_+&3s^JOSwQ*#s6Z@t-J8X|idlN~_KS+ld0;~F=XH-wAK+wc z$E3jYSDfBK)9a>;$wJxbG-=E#KZ0p!*u7=hq3O-R8tOz(`W6$?EIuag@BZGWQ}pw>&k=K`*XZNB+*1zfEArO8&MFR zC{3SHau2G^^{-a@Vrp;=VBU41zEAcYCe@P3!$z8bbN1-k8Lm)2IWCn(RI$;)9VzD5 zUSB;b82u`d3)mR%-o>~e?rt4}&zGWpo9_ zq*}|Uc=_Grq8On$D22f;=bJ!+Ynr-Ok=~TE7Simq0CgMzqVtETu)q@WKCdicBd$Gw zeuB^OvBD!PdPP%V^@|OjS^mc4+KA!T=u)y*;+(4L-X!mQoc%-WoMZ%YU5T)P4O)YXnEpzYPVLTsYSScG^TTe6BdA+;S7f5}>lsWv zsO-}0!XJF7c?y5q8!0f708CTbV!21t@L_yY6^xe&9pTdaBgz^Wn7-QC!T_ERO|O+` zr&~9HC#=kQG&4Q6er=yfNDHmIg>fqySH-j!#I{_mLn$8WJWU9q;WhF4&M>f;a4aLO zQ{#jr^#MCy%zv6$wbJCH`kouv0t1}d%^cs$!VgVx3z{i^1xr3vb9|jRYur$Wk_=UH zzA*FF%vmkH;7ujj`HVoxWUB!L2Ii~;IjxfbYO^{D{5U3qgTAVb*iJsE(ZFtRxcG^1 z6CILy`1PImyW`fj<;G>4XS=X3?>q6sudkzz(3v$>*v*^XPY?RVs2Vr>RMXg)4?R6A zb$7#JY|<1|sxwvVK&bF7r-ymhO~MWn32*)`8UMyYCET}&o}*&b>7~v)Ecb@^sKq3; zwp)iub_p7a)tTDd%o_g5OK)hfZ3&=6)hcQ9WrZzl70LYGMo&qXe6{d6vuHs!`3SFJ z>_~)Bk22Qr1h`HqE5`!bOdjJTnnhnaqPJF1gXrmL?^Zu-vng%FgsG#gJ@RJZS0#aW z9ZCJK!0M;E=dX_DtMCUa|@6kH&22;M{PU&RM z;-`n!1UyzNj1yjwcoGut@7oN%GfRkRqtL=SVwyU4t?V{dejpPWNdS=XIvAark7^U& zSwY)4e!r1bPu_k$BwRrRsh}*6A%QnxI-`@&-r1_r`>1xil(J^_(4Hm}3CmHGc zgg$L*C-@6nToX3G&0N~~ihclvp?T`E)glYr{)#Go3a z-qQz$LghGe&rC9}1RyXaKm*WszHHQqqec-BVE&OC&0o%p%Li)foxi65*Cp6zbK#nl z*^V7p#qBN}AKz)YUNW=av)sc)mCT?q;086}C>iOeu{Cb6GK&0(zhj{P{oAHmoPA<- z=gI04Cm?(s1O0REkX2)`Su`OE9POy>pU}q2dYG`3IPWHCGvs2;7LV8(ebn4NC^?!l zMySF`(y$XJW6jkMwsoU^^^HYA0^V5U{`6mjPyQ1Y!6V37UZD zJyNx)FX-0@u6xvL;7%6kjQ?Tf1(+?m`4%dqYZI?=S)H2&XKZs}8x%EplD; zZ}ym#=K)QDS7I=4VY;Qg{*n!NFa7#n`JTcnk{=33_5U4=!W!_bjxz-9=wMde5;i1U zVV_fLwMVbGZe>qy-k$uaKIvZo4b~uC0lc9@HT#AW&9L{FxrU$LaaFJHN&=Bh~&#|5HI> z_{97En>q1U8Z^Hf1!4`>C5VA=kD1|mDd~B=$#UO^9AX{!BCSmMs3<=_?~T_0vLoL> zOTI2b{rlKh+8!X|>KrfBnEu^w(Qf~s6~cb0o;J;gDaC|ZYFRDnXNuqWe9id=3HP-p z`Zr3WEHxwPqW5Z>>83)T+|l9HdjqN0zx%(W48rMHzr-(fEZ=iz(_NF}BZQgHH_~rV zeE+ns`l@Ad9zl=aSnC(u;6ECXzk{*g2G;)%dHDYe_+{Kz zBYyUMLb`cu0xAV~v0_KR5vH#>@RQH*37hs0c)rj4@TX?_7j5}}%l;u0pHqJSZl3<% zqODe&WH`gutEZMS19xw3Uh}uA_>B4dlXt#N<^bNvV;Fd$ok1DO)Y2Quo6FS(=9?)0 zffM}(xBMp){%sz)nS{|KEucTu=ThU(KmNvO0QUXMmw$PPG%wxlw1w~fc~&X`ePp;U zNU9+=B7gPrz9>se=VzkKdlEw=&+JG}nP7LsNbZbII8XiHb&UDkv2b1uC)E8~L#@iB z_UovEcrG~FUzWC-d+l#S@h`L}qJFp@X`eVO(d!x#-u%o}=Dc?~Wb@#(9LLpf^IKc@ zMz~5AIf~QE7~I~Lv^xTvbBC@OCcT|zWBh$2OY^E$F~XQLfsWfkL$1no z$s~H(ux9dC2-bgL&V;IsGZF!tsR^4nWCpu4dMTKV9aZu(VpsM055wdoNA-|3o94wg0bRZ{qonb_eK!G5Q!~sLfjQld?HuQkK9ehs zC)CEI{zim;MmYbocxvyHSy-M9>#I{iQC6)tbL{$PQIIKV4k0Fl1aSoh^rTSJ`nvSj ze@3Z%F_mBAdjGTiw0c!=2~{@xgV-GsCx_|&V`EcvjFW9+u2<+KY*=A*HQO!MDz8Bk zn68dd0s2b7sl5cR!+(2P{~`bdVA2qwaUx+nOylsOLn7uZej|qoc;uHJ@LGAT6kl-P zVTWY3mxb#VU5++t`8sau@-{u0Cmi_0A^WC=U6Q$FTOY(>tXqoxAWi2*S9F_uWjOg z))oKpHu<;L`JYtS|JJfXzox)&SHrQM1}FvQ-V}TIZ-4M#82JfuYdm!74=4Ix&iNZC z{J(JA*Czg7mi4yNy`=|eb+5yEqH(;qlZ|^x4E-&WwuTLTjj@y6zB4nU&b-S@tK|&g zOjfURIdM>i=L1uL|sC1MW~Ve+qX-`ETmyjJZDcITRP3$24)Yp=@t2{VC!-|Jm*j z-5kdDx6yRdt$%kR_!k|XT2-`}Xh6Rn3QPsu-m2FgTrAay>K(ScM8sdVDZlg^CXwbM zNM5mBPIYS13;>HI7^Eh`j=KF zeJH=T_~t=i4l_q#8^mSgXkO!T_~qTSPXC^(WRiNRI@uiEKIJahQB4=>!dRFSr{AX$G z2G(>N?``V`$(b47(dhCztgH6y!vU7$+(IdnM|0-IYrsC3n>l{8AqRD)f5Gm4msdo< zjV#Ma;4$IXMa((tfP(VKOKd}$z&wamy^lVf)GC|;K++$Vi7~*SobP1ENZr$;k${RQ z59CFt=Qv-OjYWGz<}qC00oYGf58t_pt?p?4xvcB+Wq@1Tpd>VlF5BhOY*N+R*w8C4 zxmu`dYW`P`l7G`)1Aa98lL*A_&~vAN*_jI2;f6`JKL?nkzuWN&g17TlsQbA6f1s%i z8Y&OE;KTD~pY@u6hk>D%#yP9Ux6?G%|IrZYUsV11Dh*`(u+iInj^4!Wt*Nd2XOWjm z9z&Z?ndIoC5UTpkA_D{mP|c_QyfOU;*9gd&r@h`uliArwU5L==_4>1P+bz#{YB9}Y z*Pz5HEU)dS12;eUyB?xduOw*L#Cq&&EU~6MDKp%#yjv{g1f)$Nrhj_A`wC3xc?4s< z^rK3Y)wAB=7>z!u8aAqVYpi=EV|Y%$)y|B5#F(kJ{+KRP8ER>fiDY<~W;1+rl@RqQ zn0~djD%E^yg0Hoz(8LYHu@&BFsG3an${pd2232@pHTBh$At~7Wnlu(0-UN!COV(T9 zThuGya}2EvfXUdViU<_P3s(&wGS0cE-Q9=JILL`AWNr~pxs;k5zguGBoOCk*#Lo;W z(csp>AQ$9qFpX5@%aXC07SuQl8<(J6QP;(Ib}b(IMnBYWZy_Cl@6s- z-uYP5ehfesfW;N2KDHE_{H`m~Zn=K$k!{WoJLbE4qW2i>x}|t-x+R8oEey+Ex$WSe z)5@XyJMyWXA{Cvm0?bG$Ed6=mMsnY9+L7p=qN=Q&pq@SUX)`1UBCBatcVW7wu!12O z9Q~tZPl10|YO=y7YCwkVB`^~Zg*$@oXK-^D^AKG1`wV-wQc|Z-Ss&N@$e>exRS%7n zc_=)n)!4j9T!Uwi*@EaNaz=4y{R`;>ejiPeX@QJek4?0yT$6~bb}goqR-JP|@SQ?B z_!qA?AqUAnH}E5Lz{)u>%G)*7(x>T01xU!GG-o6VsYo$ISVmKhtW#W!3iH<`L3j(N zeX#;fy>yN}YDbcew5(0R!sp_#>@2hB*xyQR+*0F43XWP!|Lz*d+B%hS<*MrB#*WFo z`Re+J*#NNyZi^e8=@QLpms?cirSqR?4kNjP7d;4GsUuvs2IHSoYIhm57kHl6hi~VO zmB6{z=>(1ab-Sdf5mFXlWXJ_k(OrVpTOydw}l#w;F5Tv{J$siBhI z-BLB9M~!!HN$M;<$~HKOa-f=u$mped{!J+&W4ZVi>Ne zU5#vf0}V;M%@fuFr+IABE0u~V>J*k3tTYTtO5DM(z86Z?%vy5PxItKddgj=2cF6kV zRjfhJOF@uRb5<%P!Q-@Q7j5bPYwydWn!NJAr&_fw3T?F_BG8Uivs&)Rv0pIFKE?F9Oay6el?yq*dLZ9WZH{Zz_hh ziqnpPGUu$Yya5!q(nd#1xz3|YswY=D7W8i!~Ls&xL1hV2^{J}6NueXCZhSAa|NqF52~UX9RpliUGrtj~$9 z&9|#px6u8DS224q-Axnqt>2H&8Bl2^4blB~*y;ypD{jW+kUM(*B*x7nr?$$wh9at6 zLys7jFe5Wrsc32@He(vt#ZeMDaH>Jqu*ALc>y%1Xh@|A(Nf99A6~o;Kkw7fT^wta)O6pg7=~eC zFB+Oxeqa~VZ`+Q(yhf3)+*Xd08;UF<)VdAMh;*k{+ZpjZ%HHu*vo^Z-r3jn}D{&Q( zUAQ=3>N)Vt3fPa}l2KpC28!j6N$P$c2(EuUP`QmtC;8yf)%xb7G-0@++#B>ro&;_M z>YD(c;p`2%DWj#1^P^eB>Z^o6QIj6y2O9Y`s_M?LDH4BF`Q)T@SJ-TLuFBD_rq_E% zsehhLJ`4ffC{CN4{<>lMy#Au0Xf!TOS^aVnwRVf^ZkwK=3dg2xjM1a$udtu>WIpLM zG&#g^32=!8L_%cm5v({D6YkTea4vsqA$~DD7%{Iiy2U45W3TB6fln^K zE;}MVBS5D&yzr8-{fzq#zaz%UB;->9k|lEjeH~x9SG2=QM;Y#6A-2EUhM5FpxxqcQ z#btDQ(WZ|bK#iZj?COFek8cL6a5ek!9rWrOKEpd7z!luNV)HU; zh}1K^c0XIkoK*kP|1f&HX)(0rCGU`_dXrf;1g~>#{dI)c3PBCKGe6k_*zIkV*Vv;b z%H3|8oMWcrOtj>RKcBGG4DztP(<|?IHS}(gHF(n_Y}4u&_o80W;2&HW&_9(^dtcT3 z>~2zPaV+1Rqe(K|m{H1qHS9GJDbP$+t{juxH}5GN@V^$g+i>cIuN!b|mCCY<9Q9D^ z*|>XoW@sNqAMJ4Q7pIn7M)&pg+5C@#)j3t2^iT2Wi6_T+y6r^UnQzju3Ob*rvg%32 zbW6`c-M7%Du~ECtikX&YxnZ<`XzaK&VPLM?MupqLT5(WF{b;+dS<%?>3Ac!u=o`)8 zhGD6Xwexiw|A)m#bB}O2;)Wwz*{NOIXN?-Skz2;yEgliB_hSPw2P-=*&~xV@MCYbl zY|H>@@Dg?QuZz(UI)-xqrf{*QJEQM_{3X!azkD9$0BRoffIl@)tvtO_p>J-u@^S#0 zm(0s1Au{|%P7o1W?V>ykZ!6IzLq83Fm{l?Ps#^7Wi=}Sk2PKVP>W~{CwjtqPW~DfF z7;X+E2%(owuAbL9=Nd7t*Rl1iuEChwx|vrC;zRDjpf0q6Y)rSC|Iuu8^<{WylJC7F zwb!+MqvDh*m%Xojs*nl;D^i!G%B8uXV)aLdVCc?iACE=#iy?<58McIF-Lor3w9BcsSy~ok9Y(2JLi4ci9*N6KU~04legx zIA8VrJ4$*9sBSL^KkNb=7{t7-%5Ul>xxAMNw7J_7A{Yn-slC zJ6!{WN~=K=m6g`=Q-_lL#J1zv@~Kd%!mQ+b*ajx=Ibts%H&x#@xd7RDbgoxrRuHej zXVI_izNX7M)4z_2=0{~Ifo%l8QM|uHLVtVsSC$-rX(q0~XYV;(Jsv0N(-Sy3RPXX; z)lYuLyV28}ckE}}eb$-Kk|qgjjc z;o2;prARLB-Een}{%zCjw$P1}$$yS1YO~LN2KnZW)ICnpT)e#nv0~Y2@^)XJjqU3J zWGhh^K)7fqON|V3kAn*W4#&Mai}#PA=bXUaC*o_8!iO~_2jsWt_gTLhy$+Km$l_`b zEFaBm@$>x1<(H-e1ew;Dig>JhNKh4}=j0y)_sXtOtvK`aXKxq(zDxmd{pAxz(@}f3 zst)wdWeeyB!2LO%jlc|=P#$|CQHtAGfKbfi0cRUU6wNZ@5+&WgVK5r>eIaXuDk~PHZ z9z>mG+izfl5hTF*lQz~l(E?G>Us`hoNlaZZnnQ4pCz3Mb$_6BCx z?0i|?G*Ec-5)B2^6K%rtFSS%%J58+oS3c+(;3BZ3Zz$$mK}PS3f&{q4#Yvys-xF#K z`m_)_?kv9hkg~InD)BIf9kXQb&YN}Dy>|w@e%THH$nUi8mk+Io5Qa*aq^SNFl5wf$ zD&Ywe^3Gdp2>k;`>=kJY!9c%pNx5p=mc$F^Tp%Y6>P>e0@4F`Uc(2B3N&9w7gYR@1 z2_Q^?zx1BUu0h#V@3Kv2`}H==TwHYlwEGHIpMS!oWapzW6_%WZM~2-g>6h%}m3{ax zh>~_J0Q3cA-zxyTj}#T{X_)!0za8VwHVONJ6?>ox>Q|v4{hVe0U2e7a(Pd+;dyVh_ z836k%dhD=+*`aaL4oXc#pW<@+sHQjo8Q5;~X-kQB;qlZkfwxo*6MC^@NFFK(^_sx! zUU%^Fk@;L-vtoc-N!d_>{(!tI-ZKu)i8wM?v+I^;Q=$et0l^bHa@#k&=-@x3ei_9t zyI{{k${9J4dS%|C7- zP>db>yk7c99-~<_4v=~t#$YBZTb0SPx_@Ny~6-- z4^XWW;_mBpbLckzZ-1-ldf)s0--TYln?M*;^%5}Vd)AGa13isCGz;qMXUrp> zVgXhJG1|Niv8h^;b8p!&yN5+&wxWzIxZt{ltx{`?Qe>=i{DGt-a z;*>+pP4drMQ13m2BziFMHho zv$Xxcz5V-39RB}yCus+4-d@WB{5raD(aOsW%mPLc_I;M@A7}ZJG@uS=!|&>_e>eBP zWz%m96Muh`#y@2Af(Zb(yNpX-3ekQC|NeG{f9MSU+rf9f^0xx8-)i{&$41aBX5pf0 zpiUM%H{d3b+wA|_0r2myr{DSmkvGQ!7o`BTt}axm5#VcC0(k@KH9*!r{X>8K*Vm+M z`-9nbOO?o#(Sc(o?gcC?FZxn`oNuNJsPAHNEK(jgrRpe`6Hn&QUUbGyT+k1!wSzRI zg`w%ny$jW`o&p?U`=Ke)4ppaex}4Mzw`yjHl-ltlFKr_e&qvDhmBJ;{vqf2Q{Wuz< zXHB9#_p9&7Im8J@!vGcykru&o=KZh?+|64h^l=}nxnlNUhO&%UxMf2DPf8*r0x`((0S&eWl$dsicixKfDPdz9YL&S432f)? zd&(`q$y14cgxf97!}3dD-ujKaDep5*zMA>{#|Dn z6AS7e7;Xb8S8**YL@r>w**O%dULi2&X{wgnd0*GvN&D(~j+e>C9{W~}OQu2XdvDj0 z+ad@BdBML7o%+Bw3jO0}F{6^uPo``5grzFO-&xKR7LkKi{+_=YYcuf&=5pObzqpqI zc$)DEnLBS>{K?dvpB%5%C?2Sl&-+ zDYj_RlLw^RE=vgk{Qm_GTXcO_DGU5(W;F%a^eqw&%O*Om^$rxYSf6ZSK2hiRhAx%% zzcbA(xjTP4jQB-vLuC!kQ~>+Og-_4u+X{S<*)=D?^V`0;CHF1u=;ZyXl&136dE1Bz zmRHKU*m_H2?oRvioE%Zl0o0U79oEkcxVo#eUJ{_gm)FI?iPbgN{XAcb)8ARS7B#g7 zRTe(zp!q-SD!*4;)vHpw=;6>FCe2m=POt)-iWcu`ieI%ftK@8VX$PP0>&~$4b(F67qgJ=h5;Q8D!s+;8w90t6G?|<%(wAsZn zz<^z!v3{*_!dKju8NGIoj?V3yP3a!KYxo_Xq6S7K6+D9I^WDHd0&=UGyNB5>Vjj zF=^mJF&5ZJ%OQ+Y2D)eK&dC+x!%@n)5Zchz?&2&)gOfzhtWH)1ht9^l_7g^vXuRrs zXs=I4Z9L0UCUZ&F~`dz zjqt+5Bilb_Xw07>ByG-1N)h#3Y`(-2R;~8!E{q$qm%%8HWg3Bk*I}HIXzM3MGIM1# za4Qij-rK)4(vEk&9C_D9lUdl|$BkkXf1|X3ggl$)``0 zT=yA7$HTJ5+WHicWu$S^^iXv*9AbmED-tqhtsldZekQWmqHqRM(;Dc%t`uY4751M zPc4ag9U6>BMWs2QsGI`t{hAu}W=0PW+g#-SIG{4-XjC^Ok3PLig!Xg~2)jxVI@W80 zU*8c2#wet7-{QU2%e6n}@?Hl9C)?Y1OLz(kLU}JsQP#)Z;@Xp>)QA_R_QuHe(OU=ExQiAUG44T7cNbVGa}C$)FNd^Nu-aVo1`osraMb-SF}M2f4FTp8{) z_7eC)QzYVic58qAv7r?&9poom@qJbg*qF))r(E`&*LY!8vLGU<5k_MrQs<(2Uh}+2 zWjkS_%@2Ie%`xS)N~n9^Na|^1dq(KKQ0ci25h;*g?5oQyESTqd*pw5Zj$Oq0Y)3(C zAa2_Vb>U*`%XSb+b?eXipn6o$Jjle&7_2 zciUk%Dk&3bM%YL#{a!KtB;&L8?eMnAKsW{Sieb9kBwV&X(!F%NN_Yqu(Ia8FwmI z^X0Oe++>&_yg5Xyh?NYrS{b+XKff@&+tBjnRz&MzosAD!nBqU#WrvKv2DKGX8BrgY zJ|y4GByn|cVEna0M{s7wGkxtE3a#I5hw7T=| zp1pLrDj&GFz5Fz5^hhKluqpdhr7qvzFIN-|myeVSiw_0_AzQU$U=R|FGih?$&j0!; zJe5>KbDXcT2uQWe3lT(qj;gxF$az+(mVS%ywJp4w79kP)M-gQW8US82x> zG7kyzT)mFt6E^QH^6L181e!5k-eP~Xeszh) zYxt~5Yrh7Ve)%uu!Izrn&Ye5yiT?%W{S`8EW0^JLo*k{Z6x{@wVIiVof-3gTf<@+V z&lDq8#RbPNyaO2KSQsiPtuGSRxij3k_chN^Li9C{6|_~HY9a51ibg`U?6_4Gc}JC` zt4_gPFwY0&PyvR=FyK_#*!!~;TM0$+%7|}-`Vb4q{ZZ%D9De`DF-`Sdk9!yoLhX*i6P=4Ixj|4f=pjJK4mDK}49lUC z1A#rGaRi!XaJ>zp)XWN&3Dx(%CE7vFY5(HoM-S-);%;-#dw+QRs5uAFY z=aD#_cDTlYm^3UEXU) z=5JSZ@5RDAlhRb@%*y-cBVicZuV>}zJKo$A{khS2Lw)=nsxx`ru9?>EN6J(k$@m=+ zTCfTdB`?E)fbDFZp`6OT;M6&ZWR&qj2WrVXTSuiI3_nLC!tgLmeY1`<#)}SG9uNOaI_v+XnvoCdY z`}H^v)+1%9jaxM8!2!rYW=!)n(4%hS?+;E_VviCddQn-u(KThNir&P@VFuJQI0b{U z$l@iG8QW@J5r2cqvdZf=KFPAY<)t3toINUX>+bgBM8bmHPxSJhNHToj>r!$~0d`~; zZA{}yqP$fpFAS~|Vb%2ZqMHKqa*SSkue3yekjE%)_NlfoW=1yOZ`IqGj_s|dvLm4DVlND!BxtH zZm5fvSJAYiqgTTE8tRh+hE@dh04FA2mvfImW6DWaTeYvE?U)Vi;brj8s1h-Q!F(X7 zcikd>Gq+MZTte$8CDm4-+f==d8ktwVieRZFUN6yn{f_ z>pzc`w06!;G#c*^1|G#Y zIBvEKhA-@Bh$x7ad@@)G-V<6k?6J|_!V;4sYB~^OA05d`N|~{6CaCj*JC8&)cbbEf zb1giH!8BSu@hBrYlaUqS82YSqysSBZpDu(GV(Q%7wWXQz1hFBrA;0X&yuzR`kViW< zpAL%3GW61*o-;a{rLM6EEd~@d!{$VA8aEmK)(R=W@sEgbsRieGW>r*eYNi*{dH1jb zS-*Cvj8_25I@4ya^pz4{Wy`L|N=ix=^Yee2VKTe7i-~@5>PJmyHdSy}cTQ7Pn<;e) z>!+G3@xu{i12=zjx|Wo0z*Zl>J4KYOQhRjC506T}SA=LXq3@F4w^{qUeD~d>!mTXJ z0Jq~2`0%;vO_#8Ws&$W(g#j*WAJPVAhmg6K#gk@T@t4IjMbrKsU0Tcpnsc3aI{YgA z!clbSsR{ohFdU03wF8Tq zD#lV7OMw-gow)}4we35qJN5vVn~>-Jn@Io-^pS7(d7;(OGoHO7-HX_ z(A7!_+ReVH-$C9~w8SC_f|;Qeug-m>oaR|PbJomXA3GiXCA9)NR1_I234aRNR4t{K zNW=lp!0OoAIlEY4GE=|9N3WEszlfy#Q=}xv3D)qH?BrCEK<1yV6+6*v)0T($nf{Xj z<6_FflukrkMGUc5k7f$`r^eY^6_Zkd)oU=q4yW@Dqxq*T|NAhQ{|{lIe_xkE@L;OF zIvOQc@EC+yhKVXhrBw1od^wS>z6oVFECWd>YXJ+x2w`E_LIPHOgpD;@!*66o?p|u< zyDjSODwQI_7|BM9rB`Yg#wc16d0^dN&F?s-Q2M}G-kQwha-REA>RbT387sj_n_$y- z?G&kL!(U=zLM{;!q_L4K!ni-Aa@3Ma!h|3nWZ+tUNAnyKqr)oEDjIIo?~ zyQ_3ux?6z5j{3EKJib@-)!d#wZ%no0#8!Wm%v|$Hor8Su8w!Siqf2EBuJrWUnglrn zd#4F~0#~Vg!V#9tdB+Cs2direv5ptoFrO?f&vQh+))hpB;i}l_fMotg zuPuxbhCwM?ZL6{8?K8COy|-7pfE5C@dMZLp@5d6(eq^tHm^zMI5i3v)-mTsn;G*23 z$)vwn!tVi{4{x*Y&x(;E&b4JlWj*HEFv0N-@Ut)bwO*5jR_G^3rD|fGvX@gxw~}9z zo)xox#uDJ-05wz-(V|nf8cG#m|9QAvKMYamC#FewEq7C(qFBAE#=4;mp?F~Bf^sxrZ!VPoJicU-k#EB#R z*$AjsLf<&INsAX&ZbXI7CCxhQRS>QTgQ#fgg{N5trHDJ=FlodsA@w$G?NVO;#(M<7 z`>6WwIDzwjL*3s}0`E}ucR>80b@BCB_!sZ<1MdOtzo!fqApifZT`C1&Svi-*VztN* z&tsU)%P_x{Etco8>P>1Hok*t|s}%rOFpVxtTYB1#d*T=yfv*&~Jhwtl}+B=61NZ;uvPlXT|nO7Vh!_&8L`@e#BxL#}+M&^8s; zfFK08g=KI(5A%U&H(?nZM^F&wFwOSbGR6l;fw61N5o8O|GAvxMSwo>@Hy;Nlr-Q-3Jta#Sl>1yMG=$LGYal*=GR3|L{Qz6--4_YS40pENq@f z#a=onFd$IZL*>?-yGojHj)YoBE(lqRB?$irs3>#_QYk(HsuqBRea=|i`Aq)VYc^u; z-m}aE6%aPl|7qmu#@KF-#e4`94g>|8VPYJE{?f0N?JtDW*NxE5in}8BXQ=|x2O%@H zu+x+COrK?|mjT8R>=qU&zvGbd8ma z6*YZ@R2a_&KBX!!N@XE?mq~yDRXXE8nYC-WFR>Ogq4jYJ3zJ0GQz$z2t9IEZ3@r0A zVaWy|pw+GIuyI5dD4sX?B?X%lMqpPB0@tS1?Xis>I5vg8IWI(@X!BDg9}2Swcy$a% zt^VaJ6m5B5V7}h2aKAJB7wavh2?u8z&-K`?ZPCWbBq9-FNN=*7dt}{|tI!deK5*S$ zotA?bNyudna;L~A*NOK1VY1OxcBF-1PzzLPSd^V!+d|8RTV85i%ZxZtClIKmI0JU? zfEGv2P19=Y5Z=PYOFER_Df#A9THg@jrPj1?nbW}R_Os~IUo%mTNW}rk<#SrGV`1!U z=#*VQcGDFE`ySSC{@f;S{Zg0m1mSbxgVlUyrr8(TZnEf){g>4b_2uIUs@r;&ZSl8Kjz}9?E6{<>Wnp);D`e7Y|Q6Bc&CvF6yR`VfMuxc;DYdPY}J_bNYQPc#K zQ*fwj&2|4A`85{xX;Nfs)DuUG0$&c?cwN%-vY?)UxZumWidOh$3IyuiyRE~uNNY=Ji25jpl( zm0~Ew@yJv7R&@t_ftvRok=-N1!j0}!wLS#QBX5Xg+#&2$3L+V0Cs~S%MwAMA6pRw1 z3dPfPY3r;z=+b7Qn80T0Qrm&Khq2_4jmu-vA8pZ(2qF(%Lt}ib5r$JDL_-$+Em(`Gi3d91$6hT6t* z=$y;k6@fkNXf(bMjkfGNrVH(1cOJD$QPG*tNXF8N!U7xYPi&tsk^gCJ7TrX|4r5m$Yk1vaV`YU(Zr9H1 zLS?8Kr11V=off+x1wpWJlF=y68?ErDbWvAxBHygn17$9{8oIjOBu(#82WZ1}ZBuib zp|n092Zj>hzRL*O&MV3}d~<-zAF-!3y*hM0w1tOyz&OAtYp~EIg-cch#uU!Rbx^dw zCf=2$>VPCaXv-^=HPr3j>#adEn)q?Nt7d?VF=ydLaO0kfnHHA9qKFicjgzPke%>#8MH_7OyK5D6L$x*)5Abj_Ogbioe zW`tm7)%|k+yi3h}(YwQ3h_IuiBUqzTMaCzygwk2-Si9ifLS^NQqG*h9;ktB^H$;hO zP7=)@5~QYe!kU`tQ!9vW!f+KRloPM?+NWisHl&a$?D}Yil`O+JpR<_e0zJaFi-CtX zq774n?n?uyvA;qy2>fs7K9ah(MPgA^r2=l<3j0@?XJ{N)1se&qodkR5iXS~)7jf{- z^ZJ5dPai-!OYDuC)VoaimEkW+fhv~7>gk+tQ+9l>M(xAkw%iNZ$nTtBxe?D|z*LJ@ zPwaFHJ63ZvD9MF^WV)GaN}nvI)HZCGQ8bA0AseDTfV;pT-6a2Lb|I9t;&uzP>rUB| ziUpbXNv*E}GD`saCZd%VLWoA+;9atikUm6GF9xCu6SYI23Ecx)K|&e0{5o;7G-928 zxQZ$ui)e=!y&cG4)s~KQ?MpTG7QhMW1XlP>yS&SbF5vtR>UwhW!He*Z-354*<mjXh^`i>yDL6Da;LTZ_NGG5=*&h{*^}ui(u~5SloC04 z(gGV78Sav@z*fStOhcPyL^!7_N}oA>5$p4UH2w^-uK4Xnystr@b z0z@J2yzRz~+{bh7tyvw<-uX?%T|1ATBg#?j$)&>ckBP;VN9!!xytx3TjrabwIyHAec`*bKx9NgXVnJkp5{!q)@Vu~?kdk zXRfcVn}F*P{XzydL}ZT360Yia8$=C`fbZEMP!(;qoFvgla%a-~H=g89 zAy=0CY-oic`NBT$fSR9;Ao2XPf$k%f<{)@?IcAW|uk!evqXDKZ*4HTM&(3$9HOXpGAfhW9Q zYqB099i&%0$ipP)xHSw7!-ob3t0tJt&L>BwH-Ev-d%&>bThU5G>K=(`QRozGYt<<6 z=|c4#m*o0_IVSH;;8!_cE-6put0Mw6hJ!VsG>Hap7 z|4CcF)E{0wt!iRHY%fYOL&PUZk%7H*n#QfNK2hQ99`-+wkhk)4rO5G21_ezvMQ{}qnmV{Cy6k!8&(ibbUm0s?n;Wopjj81_4D6m z^3OH2n4p|PkzTgX>#fS?g1C*6P5x1ri}IUZVeEI$mk@DM0;stieUsMt^e&inAV!Pj zT)Im>QPI!>y>&?MlylNaHn$!Y>^jg;=dqspkfLW6ti4ay_>s4x+~ZEvXP?5PQIH_3 z?u)ec-qNzl$tF>4dxcSo96r?dX=ch0oE~*neLJ1}t=_qIkBvIyuQ!82k9zIPI3)Mj zXrR?j(T8!;%UVP~Jgju8ulf-|VMiHTUzPZbZZFrX3od^oI?S)6=E{IN3RKm32H^tv zN0>-3_wnXD(#QyDM0#`!v5_@oo?4jiG=iy7mtCU_0W`a>(&ux=nX@Tu@aJ;$QyhX` z(+8SJihCs9kh?e}!CFjzs1V-ieU3YoV;IH;hNSM}t&>)nXAFa}N*~s1Ul`!2W)r98 z8biv@;_1yU9~!=9`Z7lYIi#%3AF}qwgp40}Wgbd5NoF^+*9ZDn@6Wsia;%B!k52lS zkrAcVVkmx5V)@ZS=hImq0#{M#nQ71LVg$C!dV4QKgXQWly5<2`p1|)+au$-@= z)ePzs=uH~60{rnG610EI-ziRt!s*sMv(8H1_a&@umAH8tnk)lOk|jR(QlCtFQ`G;) zc1x*Ly+s*pJWi0Fj%xmmFnEsw{q50WWcdczRbT`gqBoJz#)As8`}1vC?MnI^Z-02Y1;q$82c_WY|I~H!he;uvvYL5RWwbIBk)+cg0$(9 zR;=*#VbZrN+hqbGB#T;Z7GmX07#NIA5S}4uaaPumn{7%E z#DtZK*0~!d`InV5nG>2!vE?#gq~pPO88;iDrW*W&6NU)aaFuN_7JZubJ~g_zS}Uij z98f`Lpf&-}0b%7bN3#XCkl#MDTi29uRvP)Ue#|w;xoMD5tVE0`1tqB4m7kY`S1xnH zD_dp-Vs^iJb)CQfEU%r1S!T^?5uBwE-BN#jR1I2EbFWr8iOF|Rnoa!?>i)M>pUGM) zJfW7rRITE~P1{^r;G8vq`^!mll>3PLOSH|$cPmC2}d^%@Gk4Yd1__uhpA5{?{l L{2(xpUURR#8q>PD)Bj@zlxR&r3=Dh?A1qK$6`AT$$)MmH|FCM4Y$%O{%0_eG>Tc zgWs_;$E2jnQsq~?egwX64n64-Atj|mUj4s;67uS*l$2lAso#%XKtN|itm?hTZ<-1^ z=wagI6R!Gg1^zECZmCs}{qU6e)K8Ok&Z;B7M2pM38Ey4ErKGpyN6l}gyR_Uk6f(hFBx5*gWOGQvvM*w$*O`pQ&!1i*w-{hDp|OVeC6c_g zftADxScjC9db&kK%5p^OGjC0eh{;QcVTDq99P2G-Of4kd6pFUSiw6T5Zm*Gg8oysL zANy>Wno@h+5TA18hG-ljdaZng$e*om_}MaMe5QKCDxv4aUX|;u`QG}&Dp2Z`KRsjy z)*dg-n6i(#f>2!h!aB45M`P=x0*)=|#gVkZb9X`q{RzIL@#Q!I)i24JY{24QfR z-`cgypbjh1b%1c6%?N5d_v_!3DOPjw9Mx5l8w248|JNE&B6pGYK6hOT*18%Nd-1Cp zCFNPA9s<{JBg9)e}u%k#RO$F6iUtG7|`Tt zmjaz?<^}a6a#TnNo?yK)KUqkwkS4nKMy$z9HRZhy9~Ps#w%irhdX6A1 z>T1O!Y(FYs>1*xn`#eECSEaSS3S>h!pywv$+A$dQB_Q;$+r~YbQO@89e&aBsg}3}< z!%Wv~D9w2M(rB=pv&nD66P@)!J5-=xiN}C-sk{&S_@J)!v`*}BR8$QkE$mLx83sNe zSc%x`yN9hrx2^_9`Y?Fkg0jnRAa|YRyKjr!&%4u-lT`I~Ny?*GKMkrm;G1=e?bl`Wj?NP1g0K_H3LY zcJjw(bG!gDaj(}{Q$4ONbJOdAc%PE(nhXajO3-B&xUYdA5EH8WkERe2{)LgY}?gLqBv?#%L^Of!CgV;0WK`IPh9$L-9iwXaRD(tFal~-2EzEf!jpV;>Y0j2|0Z!b=(FrJ5 zu-NhA{5D%7ZcH+Xg^G&r*h>>!GULW{w=t->G>Qh&AKYT@I;b3nVtX*ESk<_u^4-f1 zb!?5fuItvy-XGYgj-j%U?We2y@FQE=>)}y|#l}2NOMS2sS~9sLMO=!}d{h$K#xj<} z$p@G?){pTHUDfJeNag#29^_cygLU7|v$7$13!O`mCAxSqVhLMRIaawY zA3lW^mB5B}gJ`Dcc>%{j_%O(ET}4f>4@A@F;X~S{&N$^kn2ItK63Z&+4ag=ZC|{!Z z*%eK1u2$@D9(bf>YJ%5{|D0`rKNL6{#8zspz=T{QdFE2dv6Vx_S>|KK^nvqT& z;T1IL&i=HnJf&L$)S&!8C0a1S)GGF8n9{d9sCshy49(SR{PUP&qS|bKx>oh|pRMq$ zlu$AyO@&&?lf^Ns&R~@QZ_sgVGG(dsFH1oMw)=2EcJ;9OEG=!*my2q%4d#KgoR0Wl z%zJ7gdZfX?MHzX(s*W|*B)!K#5sjwIa;|OVP{b>l*+t@3Utd$|guVvgLHONaw0Mri z;Pv38uWeBn7`p{GnqLp>@bJ#e-~F}rtb4jkS2Reh<^ArfQJmCeG>NtOyYpo^BCKz& zOVd-d8K=Ev-8mOS7`z`y0jtzEc-r6H`Iszj&Bte%uiugKO<8~P#^dkPc>dQ&x>caq z$U*=2Yw>}Ur(SN%)N5e#M>AniE7?_Er24wvD|_^4KqW2K{OzDM7k{JjI!&^SPAss& z3^I82Z2UGTdZsdzKOBW=(XTHyJzjrd?NO=XL?;LHeJi6du}{rR6^%rkp=&7_=%tfc zs}bJ{V>FW-(P6C&gg3vNE%keQN%B(@QAmDbL#ow!{8e;ttiuqK5`De3pn@ZSNLI!< zBlYY=Q#xYL3PlGOKaQ3NIj!A}{PVMjqxN`a)7reHPp*j7BPOzJDeiSUppwZ3K8Y66 zumg(Z^>XIz3=+_iVRnpV^9xJO?Nrq}_k7kC-}uL$wGpjYZzT=xM3$W&1Af6kW4R9+|oxd;FdLIAVL)9c!%pzRFlcLfew)sKLrbRmg z!Y*hmBOy;Y54~nirBY7knp6G>Zgud%o7PDqWB+8ba1DKy>q5We^baS>WY({1M7yp~ zFlNpF>G1cz7z3_F(~YhF|BaFZ2jO6Xwvp2DClXCD?>TX^gen6hQ@}q@hritUO$P4D zP{fx994uNK;dnp>s>o7>5;-i)UB&g#n6M%i)8ioxeCQGvD}J|s_hMZl6%;3yzykNK z4Z|J1+2Nr!vYM34TCx{@o9JxPz+!KI5$^-91eTIZR@i z!SOvu*9I8tZZT-s4iz!AFdHIUZ`v($P8ZuDG`bhF>M~VVQ$4B1SOK3dDpCHfBE31N zhF}H{$tJi3s;>>foqSw~RqBw4`3wj&dLMqdS*<}GQ3A|z(aJ%^z*uRO>c;Z*Z+Emw zt4Bx7nQvhvQ(>^)0-NvlrT7B_h>fQ!Iuh5CFd(hss;uV=&1<&g5H>?eL#_kNwMWb) zD95m%6!pvBvj9W*#2jTkDwk@_*af9vObDD6?%%fO_J7UxupG4HVGw_t{?gs>6H9&O z1t{CN>&^c8B1a3;l~<)5dm-k+l~B=vLJSDIL^$W7owFSr+vUO#v?*h-+f)3_$G?_} zk}60$)Et!xsXl>byK@L(PBnjY*D^9d?w7zN<0Q@i!*JNaeC}Il=SM#bE@Zj?Zp#Pz zVOWcfu**na?@`mxVEMjUf*VP%0JSQ~EF_7guVrTH{^WQilsU-5L`N%fT1Y{KxrfbH90G@G zjEAZ!UE1YtjVl($vnm9#IF~(ZGQYvR#i^Tk7%FN$w-nv`vVy!!E-hB+8oRU;3fmcM z&J*P`V=A6Q9_h`5b;#%|(Hv>p9aLtwwLlEqe*2cE=V$8cj@=4IjvZ&%l0yaMocxLo z5G)#9u6}AX$H_o}zjyQqnb8q^s=61vGuV=u6nd$g?o#OSC=mNHDc!ZD{BiWZvGHvh zlmG&e%WB%;n!e+^gMyDUcrxHe@yvol(cgrAfpX)2RpL7Ezpfn0Hq51#SsB$wo?4nZ zHL1&#WMTH?v1eMzaJu+EQK)4@*e zd%)0YFRcqPtk#gtu;KG>x(j-duS#rQkEB&OUkXKwuC>%Qu&k_JQrX$0i-lPF(weFr zlEhPgtj^77y#-$0(=gBmG3PFH2C!+W!2#+{Zl@3;k&%O@Muk^IKjGn)gYrYB@m#ln zv!*!_(3)QcBxFDLoItFJI>8LoIt~P^oKuCBzHg|z#i=G(W8Y!UHqvsTN00cH*L0Ub zVM0eP`q`uH-?#;K`(+0Sj$Xug?Q%cw+AW$Ou}0~z%mg=+ zGnYYTW24Rluk^aqS7DHGGlG8rsZQa1y{ST(>SZiXu&nOw{TI zLVN_qEzUZ4!g#UXK$o76b^`7N>QTCB)0dWPMwymb$X&vnW4vV%$5@EFRY5wF9BM2) z|0=pfKUefX%y05%2pGBo-$ra7vplq$Q`Z2AjpP=PKQC!tq^E2g8$)ZWA4fDi6za@S z(!kpZFe(K6&3M`1?pNg>R*gi&X~gh5pMb5Kn)lgE1x0@t+Obs+g4e8GXkLgzG+(yG2>I7GVi(%~v z44r{_qi9sQjZlg6X@0Zh90$jZ5RHrU7?VF zzBg}Lvz46?Tl+LUk*ZTq?4s-V<;EOi$|o$0*rEMaJwmG~18(F0@qP@=I=e$Ph$VX` zLkntptA9r#HTu{3TO12Z-{>Z8bFv$0@e;=h#DM!hzAOK*+|H=Kd!{e}2da`xw0^4# zF)M0VXzAmU3lnM12k_#?TE_ElyzDM-S6YXrMrJx+9D6BpAF-ob@J*6_zvCSa(Cnq2 zJHYYF34_+?xwu*6dgLZd44{qP`T0amO{GSFx0a|9G#dh*hD^t`1sH z_D#B*lbFO2g$2an%<#|wQM$zM_-MQn>0)-fBCXneYuw==;@Z+w81Vg|<6NKJ2i**0 zv8Sr4N%f!R$*eHe2}c8%Y5kTPJJn}eS>2A}vo)jBky^TiNe{fYK#7u}&eqAn3w9Ps zJ$4yaNBiu2ug?3pUfPFmhhq0m(x+=knTI^aTQU?L`}#k*nR4Gkv65o_LuEk)@hjT) zo6(Sy8a>LDdGjF(IPmSeLjK%6H#v@BR};Qn2C`(5ut=k|@XD7Fp64k*{5O{O;T?t| z`dpY=fA(FFEzHz{rn(_xVwPL!lYMvxa^>qjd`gRI&J*yF5@u6^wT4ge-So`dgw_I9 zg(?G1QQk5MuGj`OV8*)8b;ny0uYb&qiM|i0#Nsr&thpD>I?iCzC4{TUrp<*QzaxB) ztpxa|c+Bo#v)Vb%C%6($hab4?S#xdr_C98~@pvWi{;K4%oJg!^v3i4(N3_0HJ>=Ax z2h_cL^nQE0`0Ns=G)a-MVx+Inb`B`+Tww%C!WbMMMut=O<2Tyzb{#c)2cX%vId$6) z`!T*Quw>}v2Wq1n8lILq5~`NVNvsNsszQp7(YW^IFhp)U9&GN8jtp=8VN%(qp}?kV zpkECFoI$gf-7f^rv&)SMz}C0RjuUZAYgAYC48|reBCJ2I*@ba;no6>lZ*)O~;BOv}rs0O0vyhdGawGSE8L zlvId`FvL7rOrV^ONfR~E8ceZ(~g67~*7PE1p+IG=B>h^r8O ziRV0SWA<5Ve(c+;|KrDGm1-bgx|Yjy72p4BZ*d!;^i>Jto!9(nGV2W;0*P(d{t&8O zf|i_O+;swM3t{x(ie`JpkhxB5WnC&Oyxkyf*|2##13r}(p})C02oP^i^Fkgrbg0jc z%&*Q-COmBW>OgmjMl^&uF`=I_PM<2_%dv6gEhI;9=s7?TW7GGF()F!ZE!Clbc&hZW zbZ2v0TyX2>;2+X;#hJEEO|?ajTx)w+wK@^r+BhkRbs?Llu1s%Lg4rje0xUCpAPcW zkk>YYsR^_=m280hb6l_Th`gmPV? zA%4#Sm|nNF^LA8G`S$K#M;~2HjJ4=Syl23TPHJ_VrjI?D%lCFXx#-q}R=wV1fCM?S zll$gXCc}7@MYGIKy$9ImiKQQgcX8F9; zT8)$sk>jb>lY;v6&CenTCX1V05Ow98x^f%*w(cKEpQuTv($R+d9u z3ezv;Fb49~fuFRlXBu35m}!{nPdx{1;na00jNNm%7GDu&o_?fX!$2OF+3Osgr7lhT zNEN<_Ju{gvi{lp9G@F%$!hDX+9ra&MIaN2MOt&E$Z2lTJgy);QZu$P zfr@HBFrG@79*FVZxj)2ed5?=n1z+&Pvb_*+t^7c9OY{AkMlzL%p`VEA{_d-wzzy^L z_>?;Fd!m!3$zmTzBhW1jp%K(z=8o=)$F!}5sgRtv1J_-a;v!UTJ{M2Vva4^gu6+tM zlb6pw5}U{5)XkHP96S7K{oW907cLL_v?^~=GYwX9-q%uhgk7ff(kNK1-5hSW)WaF~ z?N=xMzKdVl#81#^4{YtRe!{8Sdb?+iPX%0Q%+BzahEDkM=RQXJ<5|x3h*yl}Db(Y( zPkCea^0yUnkI;2`!C9yf6FS~6)azjf5z*Q8&&hkFKkXgCNZ+FpBa1OEx+wyzi_-UG(Y_`t#kWw2Hk=QhI zr&mq~fmkgr^h)e2<_j66qw&72_C9S}VV<(8MAJr|Udo zalnrZfHwUtXm2H}Oa@O4N7P(Pv1`Zk}6j`QR5C}v6^E|$MA;isEV_jx04h|dd<7SI0|05@#Y=SX((pf;Xvjvb7QEd;2(|k+EY<<3))eCP z&9y083o~Ic2k-(4L`ix&OFDC&k$yRoWASpv5dIHl|Zzn8owO;50%hQ;MMcwx^}Tu3za#j0`Xq zh|4ri8B$vvQWD3H1Lv6Qa`!Muzq#yl$^@+n@lYbC&Ti*OO5liiW?n3vMyAa$K81Lw zIt*uJ(jw2BiO@q^z`bU1;fUNp^(wI^l0UZ&ba|fGjI&(nO?I>09XLe_V&LS2)MiXI zjXVmou*|uiVcK_x(Hb(GrA{0If(b*cp8h&oKXwxUYnnNo@~khBHvRJjj`3&UKi7K& zEKOJ!cng{nsxn|0>C?Ai*=oF3Zgps}esY%E;U7khnJag=#*m=OeRP6R6bdXGFgo@?>M1^sC(bn2pWqCEPLV17-Pvr;&9U+x0nUqUvTODAa; zQNgvMTb%)JV2zS%>P{?$kcy+RHfBO(!e_W#cR~kUK^baj;ao>AlgEW9D0>*oVeK4V zEr#fSoPVtqdxa$UG6Qc&)yI!$mf+|1cxOZosJ9c$VV_2)kc3?(eiftHgDu3hnVFk$ zZu2fJZo!&XPs6Cnc@7AnXYNZ!ThQ$q(%?`G=8M$61uqD|>^`SJiVjlm{s%8Mv zvhyp`I*t&%rf~~AKAB*+v&E?cn&8FSZ8A8lW>p+Jo9yEWc1efNbF$`I+Sb@JXTZC2 zu@0n`1uqO;3B7E}V*#3ht&e?;Ewg*``n+1w8QvcU>2Kj5Sq3W40pKzEu{L^Ehk;An zEmUOWJNN8{$B1z9ZZbID#zozy4!39ig`IU2ppX!?e9diVV5)dWLNmR_74qWuhnE^kk9ojC< z`FdN^SrzJsb38nirc*+5?+1r|>MI>Z$L@1?S$e)^^CtG=9j>-G+vjf)M*_#OKQFPp z=-n9nW3v%CXEiUxJ! zjX_EV;&D$itxy(6`BCWck9Cx>BrT>t+Ppd=N^O$(05eaniC>5oB1n}kN@F$C>tdbl_mSvwM{tY0@v zcDauNY>rER)HkR4UX>fC&z~KM)PpQT=h&;Q>k(6b-%cw@*zoE=qQM= ztG&fH*%5MEW0jwjC=%;7xHi;nsc1UuG}V4m_bGjg17U2isd##ZfqN5hAJ>rE-Wt5nCSw4~ z95Q;v5}792fisJPJUSK}{9c9NtzwIp6MnB+Ila6ayUO>`Y#?=@F`gGq-6%V}dwV;z z8EhCRYPv>S3XMkAtk`YqI{~03Wt$c{G1QAHG)Q@pMWDVuv$LQmZos_ztXVsO0JPFo zD7BdG-@q186F{g?PE2oy-w48H>30n!Q_{Dz3Kh=sZBvRJnHA>=t_|q|FZR$94ymW+H8kkaHx1yD z`gfU9mNdnx=Fv6-DiA}XW7rL=kvFf;I(oSvlSGuvMl~e9zX~=vT#~$1nZI7hHF>XY zI#z2)OKzJgkO0}8VWIPqL+!>0GYgB=c}20Z`kctd`_=c*n{ z`P%(!*0P$b)jzRI{GGKEO0B@Lq%8N(g&mA`m*1y9rhz@7(g4&x*B)!lvLGk_vucYP zXqyYQ6X5rpswd^Zw=l&=HGf9M%K3b+BUkAC!yWaPnv_0cpoLZXYtLMNRt+L5@bW@C zEo42PVNThbj@28ECr7jYkO2n7P%HbhEIxcMSNEud9FTY@Qgqj&%NzQCD{(tOD1=2@ z#LhH&Fd~oy8RtEy>BK+JT;nRsQ6jFV^EOVKNn7ggE`L&i1FGtA z?BS??G7)R;LN{ZXE&;DKbpYKR`I*uk7lB#sQVuW+{w`auwuvBSYsX1i&^UAaoPbqS zcm2=rDa#IX{5v3-0lG7~?inOfVaOmb00E4|)&G-Fbv?5IvZ1w8^jpXW}saI`n@C`_`N;nKifMy>_ywdkH_k1$fjDu_gcBZKYw<@ki}$B zf+ZS>?7w6d>@_o>5~&H^GD~yrkF;O;W%?A(uJZxs+XHEam=dQ-Sn+(F#B>=!Bb6R4 zy7=^3Umt%a@{**r@p~R{`nRRVW36Lsm#yKR=yHiT)XyLLBX7nAV;h8?f5w?1fme6n z9#Ft4-!l|G;fV;}l>tZ2f4I$fXfn_Li^kw zD4@ZU2=x1*57ugG1tepasWWKh@ECRJ@EhVHD(KMFHa1#ox z5Xhxw@vv7%+9X(`;<)nroOo&*2TrrGyi5JVe=;u+(l=728w)<)P#C94nER}Ed!{-u z09|i6jXGHC;5Z{+UEm{&6R!0@G=`l(SYLOnfyi5c5jelbxE?)^Q?5dObf&pkqEE^nH2Tl_=SSzCFQ%2(Um03PcH}auC~7Pfe&6A^ zLqm%vpJ`Q`jYZdRAE4h(U6i&^s zPT|ve(zhdDP)#@81&y4mv@S^n5Ar|m5t2JJhdcF+4o=f8$3EWn>R|E{(brm)h+NJ! zY(JZ4tgb2icC$19jVcizHpnj*q6P@mmlMRA_T{do(nmZPJ}L^6eVY2t#^{E~!3!JI zYn@GilpCCKWx$1^pM>=bBNHyZXdGV{l{3eH!{36edMq6m<20f9*Xa)oriI7*chJ&! zln15Pufmq|au-IMUd-rBq>7DXRQXW2y*oLMdQ(da)}aO{+znW%$Yf(-bJ8rO+(7!Ew=o)#Mr`NFT6sJJ&Q|2%lw6``pMHz1i%6@%Ob<&5u}Axy?`ll>txosLA2IU}LfsTRDpWpT z$RZ(oA^`&H7ZzFN+%r0Z9Cc8ELKg1Lh8lFshF5Uv4)lebKV*SI3|mpXQ~7?+(X~@7 zL1n+AsSjm%YozWzBV^{RMv&VWPlS0fgxkj-Nb9W{nI4u20hwx?mL|lGg{eYQc^AAl z1;1393Gfz#3+()`a#3(;lOz{gMRVcD_@I#0u z_T(cipqOKlAml`zRdEO5htrb!y#xwDi?9V2!lT?Q$AQ)axzx8+lr znNZm5uy=JL&N^cgTSKC^8UyKWXYd-M%B5ahi!d$!h!5b z^^hgk5c32#GIbI1sN;i!^w&!W)U{o&3P+>dP9Q{IaHNr|^mCj%!iX%ne4mXf z&6Udc?5~uxL$@p?G!SJ$SD&F{3<_YfM+D{PDDSM>I57yA6~eZE;0el&@9q9(_hvJl zs96Anl^dK$o_Esc4~z!v&fM1$jk-?~H^R52z>!Gb*XdM^u)>iPZ>^5*Q#(hBn;V;a~3 zrU}*B46;}Gb#eHi%;y$499nbMkyf_Ay~L@}lRuj_KuqQcJ{F)oPK0Xc?ZF36TgruT z2YM&Fb`Lz<-R3edzzgC>5$v0<^SgSHFBNdzqjt|lldhd%mzHz$gOUfh-3ipFc&zrb zE3CS|ko=EiF9`!fY8UZ}`zLz>Y^gp`{(74`Wz+att(lem8SPJD|F(QF_Lp;#Co6#x zM>3p@@iMWRKiZGgaOxF(5Q|&+w#~ZCE)@sIe!_F{wc>MH8s=MO(JRWpvAq5GE3p2? zFdKw*sNL2p4r;n*jJXyl)I>($Ca;{$!xq8M&)o;InuPnmMg-I>R`U~cQz2UyuY#Vb zU3Ne;79`k{qjoJ!V{iNtu=$;@Nim98gJlYLIQQnlS?efKfoV!=GFW&2_*B-UGEKP8A0V1Io%3OTeLSY;Lr*yWI`cKeRlG(K#8^4TK*YD)fcQ}@1mFQ-liK{Ot^ z&N;esm$IG?Ub7eY(-c6Odu z%i%uEc{zwyQ1>no*DkClJm2hbFSv#s>u+^8K={aw5XDn(k z9T(k@$bjQ7O~Unlgib2|2t|%D^CGk&0J#{0_v;cC)myXFpr*6weXidC zuV#HNeFyb)-1Im@&4Z-_GM3mdZl3$~No~U)ep{i1E&0U}=yA$h5#YZ430bN!oW;=L z5Vj)3%_0l+6SdCoaCN)^WWGC;)?_=6DT-WVJ`06KrjFW8U!}agf{tY{YNlJHy&M-U zWA+hEfpy-9{sr&g&hNizwGb^h*-u=!$H~RWJaZ0tJAzT&?FdmI=Z!^l|3Du$%^AW%+Au2u2v4@&u89EBj?q>5P382f>C=IohIx-Bh5DdzKMYl4cobhXF;cr3WDZT|!PXNJjzix^7^xHRO} zp8AR}!T|rqo&)Ho!{0a-p!z)P`I9wzV9sSmebFKwEfoWpRuz)UekR7%7ag5;C>YS#k~K2n&$$t5Z4pGnpD)M30_ zcgYnte_`CB2UVgCVmpplwy`R1iL=J!%dXH@^WS?X5W~C0Zo*v5B1YtBax1e?2_wR2 z$ysUz#h1OllvTB7XnyJD-Xm{5U0zKELZXu&x{)B$=tI-i;d|wCuvXze4^lPqRPjh1;3$;me<31ka*m?;@^Vad;Kxmq=m_~TfRMiMFUAS7TwzIQvU5S(7A$) zTx1D)t`YsPcR_;KUx#XO-quTDn)p3{t$)ym(!e?4tJ#nx0CqM~WDU_-BC&>F7g8wS zi{y&6l_~`hlOG72iW2&{2G(HQmal1!nhGFwytW$%{jHBPLs`7YW;csd-}5iU80;wN zI4#Jpb9N;U#!DnnbSd{{vYZmGq)CS>@fH72VAoJC)5BbYXb|F&uZn7I2E?$Xe~i=oq45XFQDijOF#nR&(L!8s=lZ0q{_x{T3+hM+PCS1QrFd4jnc>njhjFa$@k9GQbO4jYCFN zKdu>aolQ%C%aZeUlakt*z^S`(10>b>{Lp`&o28m@I|yN8uz2FqJ)$gDa?D?N`t8M9 zDelMqot_S&DHfyk4|}09dM?V#*59%*>?Fd&uu*2LGlj^KsXC74-~MVy=?Og)cZ4TA zwiL@+081;sA&LeZ1F zJR`DcT)LJiu?~Q5SD&#`HDw_vbPa8wi$d@F_;o@#cK4kQBfcZpY!WoC7*_o_cFriL&1S;Y_GwPi4qM7TCgXXh87?5!&90-j zf&B^}esq{lSmDIPTArSl zi46OURro+jqyn8Y7O?UVSKlsBbc2*HwoLC(^}{mPms>vjHgReUq25+zsvSxjM|_fO zyN}ktnuFc6*+zNMMrD;nQkSZa=wo|KNdapOlB<)d0&Vy*bxw)Y-m=kS(dbaR>U8z> z_$KRw4e)W#x+6=os(4erh7;z}uX^}RJ5zzl*r%DD;;iB$CyL*ZzRb>f23XZ6&0UMm z$$8b0-s89T0P!MyjJ0j!uIi)>iydpCo=CdZGQ*P)QT}U;_0N z!b&wZHKn5^I3v5q>#Hu%ZdHl^G%Vpk(y)MuKeZvqE2(SlomR&1@`IOF?g_DW7}bxcF$7!PO(kCmKL z>pB)m)!NMzm!70OQHQMD2HvL9)%14r)O;Z*W!0RC|c(%vfpm-I^$?wMut903>dL1t{=Y5my~j(&(?>TLd4(ys5Kt?e;>{ml4fV3HqmG0^_; zG;J5Hx$FwHOKZ8tFqoz3ZIUVdk?)`6jC|t#`E-86Fwo$)i=6uS)x+NE3j4n|RDF*J z`kGYfmF6m`;Y2|1Z=Lvbb<=8iwQ8Hxfmh3JT=WZ+KYND};oTQ$;cbzdi)XT*R6Oh+ z=uU9ymMT_bz~4)0ec1)P>{)E#?RMj|zd7F3yARF}k8OS>MFEmm$}Q77 zdchT0XEmn#*2_A~->y3f)pCYJa`vfWuKM(vUPpkERTVMF^Jonw& zU9?%k?m*z}%AwNw(xBRuX<27yACCs+$!OspE;w#6<~1#R?{rO!O@AmaCG;w1a8~UN z)^un~q@U2RXNnf7rJ3Fv%Pw7XH@K0%A78TCM|gR=juvfJ$u#iKrmIs^rOcE5^hqA6 z-8WPl53Y)*1^}=6|6&{I-jKyl=cs;p1vGZA7AbeWKiym<`toXvuhOl|beXCm^&t&A z5{neMe;d@%+l#mzP%L$027LARX&RqjUhI!Ne-w3raS!-un9Vrq15S)8FiIU@dDjP< zZ?5Y8Az%h|;+T{7>j&?0j$KQ8U2Q6zyVrgu(_@5)D|>Xx=uf@eYHf(VrK8W4nnyW> zd9Oz-s*JYh9c~Ql6D1B0EP4?^?M`fGxo4EBkSA3ON%f1jQHj*>(h!eJWp9&FSIpax zUP<_pnSDV|f`WUW6_wnMYi|kp;q@tNF4w+BtG)!qRCVZug<*Xbs0$*)XJ@3_=F2S< zXbs5A;|qjotEvHzbu`_S-Pu?9ICM66m>(#~(coWZxWY%WJ^1bwiW<#}(T>b04kxPl z3-2rH$3rl{*R3{E-n$a*O^a(ZJTNgSW0YU&v;EcH4fYzp`P>Y!4W%l4c_H04(?-50I$KEC(I8DC+>(tvr7g))1WnIsj)|#x?h0mwdXnAfIv&xbNe@e0jIeA$vaYh ziYT$k!RY9u9HIPBp;ChC_@tSMd}yNFc!(x;)>PkqN+D<7+wPgOq%!i5X_wNQ{IRgn zI0LQhfOvPOs)8Q$+e3iMZ4-ygI!|SGvY70q5^wF`kP)@_bxHzU7|PWr@?Awyh^^3A zSUOy2-7s%E?v^Hg@&@w!MD`|OGOFYHh||90d7Zbyg9r0WRoyFg`-tB2xG=(t%>a0cZE1mc!Z|G{fuXH~hDO>UFH?7s1VNu`M*bWO(Fg7(z>vxYJ z$0VYO_ov{+Gjr}5BI<;2HpV0+gcp<_=YfJC8erlqlQYveLy8H@!YBI^--B6~R9krX zw&-hgH`pLIpM@s}wOB{_Zn+WlLq~TawGHcLAuvOu?Vt68kf|Bxlp58yLY-F&qPllS zlY?kRKifSyX6N^+NA7{et#`gpyMF@jxc#h&kd~sc$@0|EojlfvkG*UUJnwV`q&$Y7 zq9b$F=C*j^e0Ajzdj{M-*pFrbplx!n{bEUa}?LwEHWF?CRQVHz3`w9 zi&_!0WvbL)W-yZ^Rv60qb0@`PwD9F*b=?YKGtuu~wJ0QzCR=fLQ26N9%e-T|j>!FC z@hN1Jg$BdP5dTMBK4SU{KEYyVOU?;fjr>cG8y0!DUqDY9PsV-}zn_sFjSE=_Gt_nS zMOYWJw^IoS0??P*1JaQy!hG}cgoLNjZ!VBI!7E6xgB|V#N&{EKq)ZC41N3@!x{4DY!$JY%=6nS4Tfh_5RA~;@*+3 z!D{T8fv-(9axleqGTp?Zz^FN(A&ZdlV`*bSt!lzL`Xqx%*GZ-28p<6Sk>vIVref1O z$8}`F>OZ9I^+|s2t?-d3UNQk3JN00nHLivKx{;5iA&vQ@+`0&DYJ`d2QCTKr`WOmP zXVyLI$qqA!B8uiQMV!qqT$@*Yi>xmc{fcNVLQLe2=hlxX2wWLF#7EbSVyV(`&bjzx z1txnr<({qt%BoQ%&d7Z}27X1B4ETSr|en|bPv6VJQvZ@XPQR;dVMcWgI)+P5dWs%E!k$n&%&io;A(Tb1cmY0RntGT@Sp|aA#^?BfSyi~IM1zGD~qnj>+s{wwM!t) z0f^s~WuQ3y3)E!}6vXG{9j3n!pYIf>5Vk6AnJ>ij4A(4Jh5X^9RuzOPJ+C|i(#5}? z?vahZu5^qa75^}Oul2}}S)N`wP^)FbXOH)Ogs|gTp6g!Hl5U^}*Eh+czn*!B1Hw-7 z>5mA0?VT=oJKj1xRIlxHUhn5U6WMjU0*I*CFFxh~{|O1v{(jxct|-okWzY5AAx0VE z-LUEnJ2tQ>JVrPHE^a~%5WwcGpGJ-X*h4qNm0YN-qb0Azfjg4lDA*VC_rmnZCt{?B z2Mhh!Uag{A1xNSDUbM4iPY;S$H@06O={gdp)TwIl@U_JgY3@5B3V$A2a1J(!9K|Ol z^0#r!z$bzZzN_eQ%S$4C%J~x#?B64E$SAZ7MLR?!)5r4tt!v-qBqL`memIIYYon>H zRBTf^pPz%O8p->t+!^>S;D+e3#q0INve*DVIrURslFs-_2x-0zG5t?$%hCYpA`b6* z?c@g2tLODgij~SHHHcU=BMdj>lrkr0;b?+U>+KJ4`{2$y zUFWye@_M9ExuYF00e_V{e3iR|sN)j3=5|%CcHNMTEy+lNk7us7oK<**3>@{=kSx!! zgJxS2maA)ICm74}=yp-$y8wG+HF-c)P`Fwcl9SlZ7*Rxne0OV3>W=Y{YssQZ%LQrX zS~#fndO=4(2z$vr9$osqSMgR$U7w$Zo2Xl5 zce**o#9KO=u4~*sfHc5~w-U8-Q!#6AS?i>^Udq01+qs|+x-h2d#Dw@P<1;cxSW(T)AHBxznh)vNnLUv;7r9d=iA!v%l;6rMc-nV$ zG1r&N_=!bG(?SNIdZx7 z*VDZpT}DnBBLF)8hhKUqUo%q+4oSPy@38QxW;Ko-&DT+2@PCAy_d%pqgISBS6Aaho#SvNTK z@)n|Bj%OumU-1Y*x!4Qw9>S&&rzHm`plVD>2XPNniwT)?*i}H=BZ)5w8jIa-NOi+_ zfv63Veb#qX^2hnc=%Ksw`~-ks3)PM?T+i5Ug21bg_k6X->l0!}C31a7NP`b+S3N@u zOx!GEjDqc_P|>FK}F{m)`#zeG5)I?P~1$VnCpKUz^ld?azbN_PW&6Dtq%+&3S{4 zfd@;+6Z5$kak^@J6TebS_|pY`4!80?BdibB;j4VFaok2BW)stgGhJcC5^Ak}yOXkx zp!Hc(KSDo~jdv;-;JvB{8nT5r#GEB$He6zSQ^l|3#wIIz$xZs5`NT7siN#RmK3!x5 zhA^y_0vyFEWAJD_ZYjrE<~rwgCSR$AguPN52OIxo*WBB#hZF@=D&A&&IyzY8^^KWgY-F+!V*$`j-4HHnB737D?bO_4Yce*4>(K(O z6Hl;z_7Su5L+75v1ZT8CE@%RV%AF{LGo=24+!R$>kadMNgIzt*igS~Wv|J#q#4sv2 z6shO z;AskeK;;AbEuKH9?h;!0rO!Jz`Rlofr`Hvc!&{9aawlAs}D zk)sXSa+;LiD08t(@+R3g&A%t;nS+h_L9ELXx%Guv`U6cDFB;qAUpS|jUevTIA|&<@ zVy`~&lGiB5AK+|tGpLCA6zz#)EVBS@ZnLl`tdePjG-OFlOy89OhuOGH++ z7)&M?mGrlrG0XiehRfjauztP zc9zOlckmwSK+GkE)mmHY&#WA;wr>E0BjBLXJH=hC zt3;K?_K0vhm61YN2?eFA&`Yb|*;qOf&ydj8eER93gMeK*V9yXw6K+>F?$k4|{BR>o zS$xk7|RRFR@*(=V`~!o2(^%|NgC2Vu#t2A}M|=z;8F_I*LT@xf7}!z)d6lKD90_ibJk(OYGX z(YB7a;$cnpbGCjA`S>zTaVuUi?xT%>D)09%R&pO^&u`+!+#f{QiKw?UwkO?dTk(wc zigie%Z6N?}slHd1Xo;~e<9s5|?az#Kf%kLD>OzDT zUH66=DbEDh*@(g;1^ zra;*{?Vsgus-k`yyos%EOm$xKu^c6v=Dk*kY3Hqd%_qq1^vd2B4CcXbd57d$#CJuKI$T1!U1{4rU8dM zPuGE?Tt*O;faakRaCEw`YPTS^itjEY^`~v+3-a>KF$yWifm=6XGNf8cB`f-WT1Jl$ zxs5h*cuAkKWlm`jru20QjrxELy%m!cu{Maz>nsH)n-Iz1L_gpK#IrqTFt1fv)h_m8 z%F{uEIh>Jcs%EY^D2c~xlszcnN^x=LsPo-z_RlA)hq)y)g{TPX%#(-jYaq>jbC6jN zU_b_ZC{Bh%Quo=>mmv2a7JVsQ{jtvyQg%E-OlfI3%-23C*3SQ?pfe$iifMC=HP@@o zDaN0LEKe8wru8Km`r=~q5a~+Li}+$Fbtj$kr{VV%d!xzs~^ zInKQi?lka06Sv;LXduWH9ShWSx$CTmP`LaEBJ8@D-r;XN9gEmIjL4dwC9jej`*s@^U zE;K&fz?aOnde<+`&f_Ll+a+m` zOx&YWw%@72hNnG^E1E(W6!sPP_1gEoeKI#N_hL~Lkhb{;B@JvsJFM(L%!er?#{}26 zPn*7oGY4(wTtvvI^bp)-RtH*LedvuFr$dqiy3CV)GVpz?H&7yI|GWy8-WHh{(xwaU z|4|K=FZt&W*sn2VaI@Ay^d61B*aprlOcfKux{y3Mj#))tk_l6AM~Or4os5z_LQ3Az z?qEhAi7WD2L*miGBHLX~ML12GJO0Fn_}jNK(Ju>^4{b{uMw90$p!}qp&cC6#V+{#@ zYu&YOTG_pu%f{+id#Z!5wOH3BgKUEjX0>!N4h0^-lD4C9^2!cmR-)KrBhiOLcCTpT zzUkN{aQ4h!a678Kfl8zhV;51%k=n6dO>XNLQ-wJo9EdiwpZZPhOUV=t^>cF#uYq)0 zZ^n0Z%g7YxY}F$#*@85({jvAl6#c&wL|@ltP^IJDv0ri%78xQhWdPYU5(LB&vhjPX zsznp<5xmR#%O`i6o%$rwt(S74yktOd>G3}k-cg?gNClO3t8ioHjYrnLN3Olf{X7y- z8wVPEq^Cw{t180yakKEEPXJc7<&Uw9FMDLyB*+`JK{hgN?700*`Nb?oo&mh%40>Gp zp-;Io9(lzz_9YDx0npTezqZLYUFB{V5N&CxvHr=qtR{Frpp*6n`b75?Bnb2_w&0VQ z``cF9BwySt4eOZG30QOKHv1lKjw-QAF`%lpTi++oWoJ^#{Kop=^>`~{>ka7(zB z=siAvGyhtf5eaB!e-Ph%y!R&J126xt9uvSG_l#`;n90Aow*RjW1Dz-W>!e28YKqX@ zE=*gvhhqBzc&l~C*u2gIDO)xt2sYTE8)s57@t;H@K34C~7C998q09*U(YKaX)cJD( zS&Xxz5~ehQe$j(7U&zhv(X36|vY{A+e^i(V$qYoSK`~!pj^rVdPfxQjYriInLM0CP zrQv9_^L)53)Z_{o*zRA{LZ2;P6U(t7eUGZ!De^t>?9(ZAXmF7}LA`GAB7JKNAT~p&CYHJASfOB6=Wbh3%#7l&X>Dn0DUP|kp|e)$?aIYB;YD{ z#G#P2CP?6WU6-@USIgh0aZ`*QkA7rTE-l0Oi=gqqe>mqIF!j9qeaBawGg9=>Veg;e z8LKgo!OF4+wPb%NX#EgZ_j6RuD_D_?ggX~c?XQ{CQ!EI)dLfVh?5{q%*@IW9kkIYYK7KC+5Fzqer; z5c?z4dG*>X-LPFx_{1gV@4a2@kp;Shz^@k%eg2DQM6?;iwu+UFWd>GHgmCI(^t`7r zHo+O6KZJGg&cW}~H$(i#M@x@c%f!1?CtVwpG=5<$=nKvnLR!trB~MF46MO2^a`ZHO z#Ns7Y$3jTo(Z*Ka-k<9#9jW)hI&~8;5LyE{??;J4LER~ujcUCS{LIJW5C{ybf1dTL zkP;Q_f_}{pi8*p^{>w|b#644!SD#QC%uH*elE(;Xq$C55Q+WU0D_U{Ecr?}V$oisM znkhS@ln}yoOc=&jXZ6<;mQ>n{O7*F*dYS$_pT?_93_Dv{DVH}xNd~qdLq!Z~1KYKI zIfbfYf&Lk1koruip%9F}i^qhqk<9z=4ISuKtmiK#Bg4;?%l4CYy_O#aeeHgH7-KNb zkyu`30Q7O8et7!k7w)-k(m^-wmwX;>B#dPv3z^lnwjTWL{5_2hn6{2E zmH?`Nw5)z#y(AQo1mU+$p>kOmK8^X0;9p0+AQtu&f{xF%afXkOKOZx)TKDmYv?TzN zS{KOZzWdMVhgUUc_$2NZY8 zy0m8O;1C2CjoEkjr$l7gd9RyumHTuLM`mWofGH#xb-|t>yC)D}PlwmniSO*#SY*+f z>#68Adc5Ut_izMKsY8@5WoPvvF4-)_vnA`Nm>Tf606jktP}pS$BGzKM1bi8-L+as@ znfcE3WY`Vv!~F0}mSp;DYkpxW>vT5SSFh%^-+9dRTV9pXxS`zINHs7NESk*0Pj2tP zU&@MTDl=au;tO3UMRQ8H0d6!3;O}4krdHpqh{9iPJ;qzdoVA5)UDti=2^WNseeb@( z0_IZws{FvUf8e>&w7_~ZKyBRlt$Yq)%k4due9xr5AgzJ3S?bY+FG#IW`}Ywv16bSp zv8vD-dq!%n$@A%VIWJ^KTqGUZk;korDC^pA4ngAtg~LmnJ1HAw&hf0L1L6T_bLNor zI+u;`yjwI^cHoI|q~Y&q`r&@GVaLHYUyjso_{`gQDm)b8y)D?}(TR{lJR#t&OX5o| z?k*BQRlm+`JNz!s8Dzgf?xzil4>OtOCXES3uBYSPzJG8^u)F^Xp}1g*`JCGzV>Yjw z|Gkx9jUGM_A2ceGFBL0l>K!)_?{aB+%{@W`5DO~;MK=*(7<{xMOHEI`0`PY2hk^hmg0oc+14-)gvqU YjS!B;?3lwMR+M7q+uqF|#4N^cSY=>pP808x6eP^Bu;ON7u{@CAYr zARxU3AwYltAp{a2?Zn@U@Bf^+_s*GfX71c`_Y5%ES!?fAp6A)m_p{a}*2qAQ;{^W+ z006*o@9u450N^+~0C2?lI1A$pAHT^N#vdkMW4&8|ssX`O#)qTMH}!7<0M*HCl&8lS zpZ|Js*TNS7INA04hbdL!qyPZW*mLjp&BuYZ>r}R{LXJ6CL+Ltq!;MBBs9PIk#&xN5 z-cc5fkm(Q?y><%t*S*AdUTg8Mkbc4#CxtY}F>!u%z;}wh)^Z@~Ull z-YAm3uVrE@%sau?GBJ?T68etO+~0@n>chm# zkjy`Y0J7O2;-r{ChLVhQi{(9k)x)Q43feHcx!ep>0_$&(8Gc zmi_{Y?lBwF4J7&J`4}eVy43eJe=ZP=9yPP;FScdjNU)0fQz1XVxQBXHc5BKFGRn*X zcd$P4d)j^<;V9RBSy@xG;-x=j2i3%(+8>QOv@1Qpk{bjK4&IW=nD9T}u2c#^V74N0 z`TUZyQ;NYlzvun;(Q(O$<1DYoTkX_;i)OI*o{z9yqVUN6Qv?9`a68u`pLO!s87Ah# zQ-je*8D{jKLtEs`zXd^eI}XnRb}sxc77nRV>PwENZN}sA9}v@Rvgi!+n)QP{gm3S( z!UHXlfY~5t6_93HTEJl&Waa+ioe&k`CCD6vRy5OU%=cJt04;-X6MZp2+Fn`X(l3Govx8lOaJ+@yds;rGee}pkAl=fF zy1|T}2~$7gM#mEyac0a*IZtjljHw~x;<@v9^#p_MA4-HwdC_MHc$hE^9J-(8hd1^% zoBipwnQ$li9AR3RY8krlsS}MB2@DbR-bz>)h^WL*aKiZP%*ikPMGGmgYXc#eY-QQX z;iL`x&l{X@4?CKF1=)kP{=y%PN$qDE z)JJM`e~)L5ZR_!@!o*qn3v1hYnl3IRi%blsJc(V!#&pzcVXMjOzt zZuYPz{mVDcn zajqwaP`x*zmiz-1b|>`NO+Lm|bGTV)-~QhFJ_nLqP7kP?)}mqUY|n`(mojRshi%PQ zZ*61IQ-#s#r1c{=Q0MY2@jTtlwd`_ld!G;7bM7vtZ_g-yZiVm=!!VBCon}9YNSC8* zmxgO&e?jMBWNdtHnck1gqmeaL?opRMX=t!kaL$$4r z61bSWAa9S&%O{fj>r~Ec0i1>LK!b|%vf$XhjR|lsdsrqpSxnI$QNPRTIwvPTJ9g&p+obzR1EO{UxN%#oj52GJ9M3mZ5Y zf%ndKyg02-?tdAwDE5GqK-Z-i-qsj6sf2E)mwai}+Ik&GzY|HiH%H3~i_J>2Ipb`% zP4b?*F^vehFY@AK&-y*%V*DWkXT&QDde zEU^o0=}tImVq&V_{Za~)Y|OXVdAcw8P1ws9cq8Hjeec_N+|#ox^>C1cNNi;{b0(i0 zR50onv3wYdi|z03pIHuxC-p^(^1}ZvESBFuV1%jQ&W)Vj)_mQOmLqn%Wjo@j zJ9(6p)d1`sttQS1;9BX~O*-r@gA$HLH(l4YFz@&NM)tORNSpsF7@eTJx~1zB{+PuK zviDRy?bX?Z!a(<3KW{&U6$kifJn?3Y{1MvJ|pxJHZV}gO-}n=u?f=*&?wU)qcYo{nPK-#;52OF58;FIs$!FS|rx# za4v@-6_GeY>MjpDDyy9uf~;kACh7$cC}w`RPPUsU5}aY$H6Bk% zU80|!6rQ%d>MT*eQL-#Op+M9+sMtfK&c!~F(Byu=(~6?hecR8qRvb+z1cB_`oYQdX z>YH40@1~jPjT>oAjdv&9SU`(}*$EP&da|$!){-hRA^?V*GG~5ehPHq=eYoQ8CFXquX!aw8K2nA zyKnf!?&XSH^$I6qzqu1QB$xV>Eq9JTIThOScSXT~>tqsN^$JYTc)-~`gzII|nt^AthLN_ZEsIHo)W@ z=#iK%Z?=KNWXj~@AxItX<^+!lG!UsgWS|q%K)eKus|ZIBZI)o${oPI7LA_>lmGkF&lksc zuiu@nTP$l_X6})Hb(|$PE)OD&`Q!RZT>f`o^?<)hE1CrtA%ii|3lEg$_yT4r-Hkgr z&}--g*x6+2UUc`mj921lvMGOub6@P6bg-N$BipI4#D8KWHM>>2r;(8OX|L6wBebUv z<8_j>BsA-4hW*t=k|}7=Qm7MkY&|#1C0a^w(NCzgx$V8Uy``Yt3riu(P%yr&Mj z8k8JkBAp{r`h(ulgn`f5$~Kge=tep>1-A!au`QdFU~6(IXE$ZL{aZyly1fWwArra2 zn|JV`$@?w{Sl?qp3cY--@Q+O?XMSa3u3k#on~}nz3x$?8lInZL@@Dvzs1>bz%=lk% zkgl~bk*i*u3k?G3G}!)>v{2IdNg{=i8bv8%I3z07dl;6k^dVvtj~x9K%{`qrY5k*U zaA97*+VAT{^EOrCQoV~`v=a~uBAcx@X!B(^7GlmCJ`%@e(t|uz(kTzrce8x_ztA)` z5Po=TOrngV;}&wfo5z$BiaTuMN^Gov?AuA=Fr+KbX1}fx_XA_nvlq27jV_cD0B-b3 zpt+PFkT08txk)(c=sA3B)@HmuEJ(cGjK8A+R;!>mORw95A|UZ(uobF_O#?>j-J8k5 z0^Kg3w4Wvz_JAh+Ll20>h!9uftxo3!I5{r~bIoIAI*b&mp6b!5SA|N^0LqfO)1ez- zYx`?#tP4%_HmQ@biQec@JMrFHW5wy*Il<&m91gu8qHe?Kn&H0}i;oC4j;i!VXw95X zb}vm?O>)gUQ(QJ3>{U2i$_9u@__qyJS2D?0ybk$ld-{*(yU{C`N2 z|5aV44HcH>$eS%*eHn71#Az>Bh9)TZVUDUo(ZkI5hN71{|J>_;SwI|C{NI-*{0-p| z3{6;^{YT}%9j0nsmaK6JRvjJq9ud2NL$?M1y?3lo@9BAr%=F*`GI_M|7<+cefXeTI z+vYxOhb+H%O9!8G_9M=-EBcMvL`@k^_H{{2k2_`{XTKKMlQhu|3UQyR72j+`Ck0Z{ zVF_k-pt{h1t7o6Rx|LRt6gOEA{Mvh_O`Stddh-#%Zz>eHd+0Y|sxqxtPhS zTT;x#`~wSt}6&qPCDnQ z8`;CC#Bx^lFFily`87Cs?|uW+g-4`3Ie(08k%@Vdk~D6Mh-CR!>ve1~^yLZejAjz5 z8o9F?l&}4<$eF;XG4zb*E?mk!Us3Yr3I4m2$9~es86~AU!}I?P8Lzdx*Mi0M+Y#}%_J1E^uVvvFDH{Lz_Gl_Qa~=J#rP?}D$Inye+}1`W)Uu4WBaa>Z%|L!19ba8fo&&Ft~_h_cbEVzua1`eYe(8LsW*>sX+*dGm#I{%LpzQRNWcAm&!+xN=F|U?a(E>R z(yRBP9mTpLmilK1*acrRu#{q|(sYi%I4VfDPu$C+os_+|v$2$T<;Fq=NivZH3-M&| ziwHRn%OFzHMm046*=+*bOE)Jcle&N~DICNNOi%h5;y%ZXq%O2KE?H?M=rb=-n>@#5 z999(q;Ga8t1&}jb&UPTZAmG(mnASfdr44Ix94(s&m;*6D0blhTnB2XyIt*MJDV1~V zO$q&Y^+2<;gJP2Ga+3S*NzE%L-%y4Q3=7l^U6YuLfwlg!@$M6lU^xDWi`+(sg?FDC zGSSc`%$Kr}CuplFdt_CEs3Gf=^U{>E6j#2G!nZ^{oS?xsnw-4myv4$!rG?ZEv z#QvkbPG9sCGFzZW$TIN^Z`TN)ch~#m?qZZm6gQn21HY38wp+JD9sTDL;qyBI9UZT@ z91PlWX!{csk;GOSnV!z&@bUsP)M2%o4LH!g)Za&RR_dP0(&7s9A5ifos-$;`wfpF) zn_HNkFjC$$luktVT2&hN(xHTL?%{3(P1UnG%-3B)#IT4uA6tDp?N7vQ9vNq1j;BqZ zPPE-eYMpDKBIvI2eKz?8a9M+jj&T0>JQ%}>l=B0r~?0`@6hMM>EIE**%pyRKc8y6TjiQD!-$-7g3&ZYHAmS@Wn7GLP-+#M_Q z>cX;2OA~%3t151PiK@`4A?%eQ!c(|QG1$*`Z{8?kuoj6qo*woyq29ly2NDfI^H^`I zB?XshQyaRH_v))h>{5pv{+hMp-O)i_=;{>UAR{%XGS#cJ5Qkwegqnk8>wY6tH+vZ- ze*8jiavr2vH20d9fk%%R@GjD)v5@SNepx&nGL$DlFr^0k4%BK^jGY4EO0G!l)8XJ+^Ijz z7P!*CJUy!^XxA9#>u)&Ufli0_@T4eRhYo0p8ieY|!9WODS9-e5d(O5k=kZydEUe5& zfd>%5+4HA8oLVKMVT(3$M4KjB_IiCVKCkz!;K1ufkh_ivY< z&3`e&w3O&Rt4Kxr?ZW9j{(wo9nY%9Du7;GN(n*^oLX z9#{A{nf6i9z}}!k7mqF1VrFrYj02u@dntpA7QKo=r!(RjGL4XF(bF!|>_y8tcLU2Y z)OG{2(;K)yxx*Posy08Uw@3_+?t37Bi^q)$R0gDb;9cA5K)QmyL*wWwNSGXLd)pT{ zbZ>Zp6=7T-;h>6xfXy{!LPm)P@eXvYW?9;NJ)|eSs$DWI7kuj>d0q?)Q6ayAC58>Ng3~_k^dh$QnQaVqMxF^=Z+;?1)Ba>rE zf_4L(*un`S*~gz}iY7wpN{7`{wWhi!@r0QTY#Nigfk=6Nx-aWM91!KNE7`V9z;iat z3Mx@{qaEglQu~f)l7-#a&E94O+xHz5>{mZu?OB*g{mm=rSENBmx$3a}tqDOMRuS_7 zCt@D?Q;~f_%Vw(zw`tT_<*m1(ahY~i@x^&EPHsf4Km$S20E2eGXbu(kZ=OU9Bb=`-?1H~e%3eHL35J|n@RRW`wa zLC#(EqjoV?!olq-&+{1=K7YTnB^UahwH0&-I{t-Z~?e0Ve< z1=&X-lh6=p27&b+)?mO8b&+ZR6q1>~hO7bQ!vl2V_LYBn&~bY-fQcE3^0j_UV14QM zn;%|YqR5WXNX7iX*`5gRnLPF}39xQ<%{tb5x_`W<4k4XzN33t{Q7?6|Ia;WvJBcJKA@~OIg|fiNRs^A3n4O4UYgW zP#Hv7C2xP@VDDD3?UKvE2xw@2%sGhxaO;6<*87+LMjuf&Y9Uvp!Rzb&VN6;dSjaGizUBe3gqubU>~^7`gO@ z!$2)1sgF2uuz27DPD!Ka&zOo#&BuEWtuJh*(}19gHi8>|by~^vm3V!(c6%Hy9!52A zRE~-7eUUWhnWVSbChAF9uf&^Q9<-mv2~5}F_jwRa&kS0rWpCspQs)Jh;qQ^)tBmN6 zzYDFz6X^TD>FkaT9*=C7NkZg>*b6o+$}EViO>gDwONsGUZ?$S1O zqMs@EIv_VLh-DE%d|Vo1d=7JS`_F5T@CZi#t4RkSdUgsqc9nw=wTg-5nyk~l)oL)U4 zo<&=IryDfDp#StqQg^JEBym0>Y^$pArKhM@H_=%*p8Q`Ajxxv*mL#?PEnVOX_AV1n zhy+G$l7DX6d{sf%ccBs5BG~t94i{q>MoXI5ncgNE`9H(r4ad7No9>G@R$`?Bx-gy= zJjf^fa8`w_mlf@Jw@6mQmmW5u`-nvQSeM^;^dP_v8L|Yi#8>Nz(un=l^);=2cKi1$ zsL4Keiu>bCr1wLI5cFjkcn_9iZyHEwlR(4u2m){NDwA0A_(GZOf}Pwjxin_21euu{ zP1~tQsLe%EKh`|O8Lc4x{TEuobD&{Egtn>U-7A22%pv2EU{yu^#D1vIJ6@0LF) zPwU3_aSTjKugDhZsuyvObN$}ISk4%yPV6uiM?yEZIeU0K>FE6`}#0lEZ zv5B2Q5*jWc6AL>)6?wbZqZn|n``ddEBxtJbB-|1enEL9n zG@ZPM>fAug&=`x3*aA{lq6qon463%Z7)?adfXHTUenBdq4m5*Vt}@B^%CI@df91*XMyR)7E<--5^6vk?Cub=N-t}>ZC=G zH@arbucCE1eR8`D^27c23h(1<2*>g6NR7U;he@ieZBeVb>{n=Z-dF$4nM^}9|MeIX z1C6?W_3)KkE7Qt25b2QfY&6(C%+u5;GRWaKDSG&$IQBE-U!e&&@qdrt{?FqEgA6r4 z;hYERRXKdDWtbP~XPFM@hklT@|Fwrsmcbl;Veg=FAm#OgZpdJd)E^kJvU@zTFE&*$ zT?~nbusSRxEB9snh;z2Lu7;7lLp0knENQ!=W55Ue_tYQgS?@I`oTcayr?%NBDokIJ z`MtS9D({cRZ?QdbHW2YhuY0vPPuL4g$O0Z{lDojTc+LCT19&ibF4@+YQiR+r1J&=Whl}9y@o~s*b9(BTPibRt&KVB{%3IQtJ;_Fh_jNiez-PZ-s&(xJ3_wj0?Cbm8^l|*udg0q zab=LL{D>5r^VBe@W9%c>*^l%888x$O(fk?B3 zh1h+@pqCb3Ax^gq4U2xy8zM>BBYq^QFJ>UGUd%?LWfPKhn~Ou@i;C z>^_z-boOtOWBK1vw685sBAHzYTngsrGgE%^Ic@E)hy4I>-Bi%FzxeyYiuUl1_L{sXK=_TtTgLuL< zpOkZkZtG1>ezjA^G3#*20zjX=pML6;zaXKP+#_lyvn;&#5^ zU>#jTEryyOrrw@fzmb*75#kS;p`#uAm%;*$F^9jqhLp-FYIixs`^6qEW$(>9Y}T3>#ze+Vo@Q`KwuQBtizBjQq+{XucXn5mVXZH! z^&arZ3?$b}Y3#f<-`aE_Zqtcl4lDp(krr}m7C(s)j9kLcCZ=~)ceeq3S3=MTdZ>Oj zH)q!AbonNyRU_(D>j-N$~DwW(DQ9W;XLeL9%a{_yQc+Lg3I)-Mz|SB z;F1G>tsUbkV?|esTpPn9a2p;_T3{Pvh^81F#ij7QSm>Oq%qu3Wwk&lgXrOL-AMnI` z&`ic<0KAdIRZXyc>w|6*@!r=xE3~1+^z2?+n58uVnYz7aq&l=YK-gsS#>JJRBiuK) zam1_U`OAYO_^}|dh*Z2#=cyw*iIFIBAw0+v9Z*1%XkI1lrZ4QAJ-;>fAa0tg6uh-j_%z_=Kc@L)N~GVQP!-RNT>1O#EhU zD3W12$7@#lPP24dbG1FqubfqF5ohv4+Q`Z1>Q5v8;5lQ^eqyOY95kr!dvAzepfup+ z#$@IG9l~Pmq?aV-zE|59T4Xz6qC1uF=t1w&a&5w>uZLT3=d!P*s-C~CY0*aLw(ovOuz8feGlTD^_8K&YzW`V2Jrk=u zmwJjr2=l+;ywDvbFO z#|wRTT+Gan?vpC5olRAZF9xqC82aaWfIY->#-~xDh1AOI$YI}L#oAlmkBUGxKd-a? zH5PK9hQ}BArKItOUwDS8B+k`F=BXJ6|60a4W5&ngcXe`DTU65hRh8&_Ys#?Url9N@ z*k?yIX!Exot;yiT^H#WXqjLzMF|`kCile^%9_uTQIJuOq`Njn-P2r;80%qAWLScsB zi)yU(bSF=?7j9}w3U+!pOR2BYCK_h6g)E_}>f$7yf_AWM+}w1B@PiJ(Ta~L$Lbd~A z>g0mGP+Nt1XK(jn%~B8Bi-)Ln~y6)lyc2N zklTXUr3y#n3m^9tsXw@tdr++t{9Rqar-cXnQlEP39xyY}H?$`=a+1K4;`dtjniB^L zfO?6uy?iG{VDnYLFmCby*K%_}D3xGUJeQ!8%i5xz?jPnUER#Bi8Lj=U1q*u&(yQQO<9g>bv_s7#LFOV{6zZ;g%@7i;Awq{5m+6Jmdu6cb~1PQH6k`JiYo# zM^2wq1>Zyvk7&m?*!~QaYu_es&ehk`n_DTZy61y;m4MT}Y78?~O;7qYqn;#9_gZ>Y z4hxXL`mTq$Ro2O?o*iKp8Gekrz2#M{klG2gf;KHm$(0kcz_rb#ZV3zgOaRdOEZaiY zHs!I#IAi;JiS@e>kIG{7$0>i90rU7LuT;i|-L!80zO0)*RKQG*o4(C#O{xmD0J z|2^>+C3vqQnEAW8AoZ_(dL605*$d)X#2MSmPW9sg^?6~oevShXz z1ZWK&hR8?_Ri$JItM1TUImOu&{7opzcz~%xdDYQqss+K$Qs=heay5UfS%?39v8D5> z^5U+^1UWp~HkD;u2 znevf^_7N8o!NreTJIhFEkhNuX<(a115g*!gv!hQo)sk5;uC&Q)aUwy-h8X~N&%mzC z24XMoemW)Gc<}Ll+UG1pg+{7lt)C+AWmrQ*!tW=G3}uBY^=w3b4ju1GbmS!83StOUD%AziK7J3rtd=cELP@e|P zk*}|7Vs##Fd{_4}0U#^v@F=IYRgE&bJz(Xg|wj|ALxY4V`gIm;A)1i{+$NOy@ z2}z6*QnTnTeBZoMdytGfn-h3X&A*-m7%|jKR0cx#I13IpXWgtP|`F# z1UzGhQVUKcVNY$n?fW=evu>zp_qG&Kzk8!@x5C}6&0Ui+yJwOr3V>T7_UpZE`v+d^ z;?D?_GGT!t{A~ zC6@w#z5F$cppWNPS+EgqrD8V2`_~HPH&ayi^OdRbQlw&E$KsSXs~T3DhNZ97MwwXD z>`VpjtKP#5uk8(O0696Yc<2@9MoKLm0TgVH?G4ux)<&!ITStE^&3R}>FRhZI_xsH? z{$niPnu2cK!_IyTk~QPl^xApu{_M=%pDM66_mFG0y^gwf8xlFIc}1_yDoRQv;9A6Ak@MGg%>OPW*`fq4+Qd8;GU1y*D;sv6TEieeKb<xQ|)wj^{~lxFW;Hc!$A0nebaKq*b5Eqdp34RSf<1JMFEDC#=%a z&c0c_6;iy@4gdfdUh+v!(HEyKTuzujW*F~AGyg3WU+Q!)ey_jd2MR2kfY#B&3m+Y& z_?Ru>7JVU0mvQY^G)?&K0|5CvKL^=^PAtl|Z=VNBBwL@zkgPeA>)$`BfwX4S54rYx zP5x`oKUsun;Yh21l`&mdRe}*JO$$>HFj)q)ku~@kh=fgK=xYcl=JUp@v|-YZsUHDo zkN?3qJ7X0VKy@)dMt(xeaR663fg#@>4h#Wgy$cCDztx`Ahx9)L4Di3dH|c-=j>Ui9 zAd^2kl{)q;S_8dRl!7+AnV7|3XrU!3fc?e&Nf@pCcSDT^h5@p2DtgQvU8w`ZA-6ih zFZ{(Ancv6%u&L|+TcQ;1jdz%72z?}J3AR&gz-oBA)vw8$85N+cfMj^e^ zEyK4h(|;_!Av*+_dLnRhLV9T7@7~A%)0y=PN7++2TuSl98EHKrnUv(A5XT_5=Mv({ zH`@Z~+v|jhf_&Cy6UEn@WYEMxTT|8%AnzMblH7kP5AqzS+MoA&+qxB%0h*mMZ>d+z z5wbFMrSK~uJz_bQB>9%)N}L;BUd=3!8PF=`5wSrPMd!vmG4qI(XrJBFY~3h0tzP(`y%moof@#-#4yCT3h7%hZ6TKB;QslsmDG5>26aN66aPVjk6SEBbAkQmC z+4s%^H2RS@e*29awDs7aNDZ#N4gs9M$=E2+cgzfaj&RXbYvwphphG`L^;6V2@1<8$ z4bO*gNyh*pI8Ul_pH89my&&kc2Z78u}eo_ zIw{S#kb(#~ac9zxvOYQn8zPHz32C>_uWsYpkytfRcd9k>e>-YSq zF&83iV>ojMv{HeUJU;vF>3Y_7ED8x@Q4lBca>W8Q4b%2>_@E+wzs~Us(BHNm;>J zMADBnIn5DG>@rk|;@gXRz2-YMknG2UlI{H@6W37|1m%?SY~R98yr*VS;b~5oNYq&< zG)$mg-D*a-uBL+1>QPmQqGQjmg9|X$a`?f)$IpUU$>kEFii%<7RqbmBF$hEt40MP1 z7b28sQzs(bP>*`3!=JsT8-X_GQoxe~A4Fqilrm*^1w#F${?$c?zW{dIS6n*$m; zcCGY-lNvXmZ)Cuc{e2sNr*--d9=;UGQNlFC+E$%pMic(WyQAH?*|ntt12^ZkP7 z`%|vhwoD5w8a{uke=)*pqO%dz_*B0s;dEusdvpA^+}+K| zs27fzN&xtJpoybEvZay6N&ehMg<~>b>?1x6C&2>gHWn)Nf`*J)kG2K9I+PBz9+wZAo-xofeZm*6U@B9Q99_nA;+{Xdw7V)eN_K2KeT__+ zmg-r31+RRD(u26KBi&V;i_fRGjQgplkFb+9RDR;1*E^g%5G3!;cs$m*CM>$G6ihi; zipA;vXCCYg2N?nI9zV2YWuEU*k*3%E2ULd&dXOt{2bA!_a7_;u)!nYA75TY=n#}d+LZ_p zoE@tAQ}gVvp5ZWvqGD7Ws!jN%OA7aFpH~n*Cnc8Xb(5jEw;TbGq!OY!@+`-oPK9$- z7Sfgn$>*=4BcUlU%l`Dy6LuR}f4%PRAXmD)`-5tb=z}*}W8-3?>+j~A9#Dc~aCuK* zVtRPcSgJIV6-f2`XYa`s9wz2tn|HSN5o*OfA|pl|Ewl3B!XWSJnft)`e2zD%_@$Pq`lpGl&Xl*!l86FLb9K$< z`41#x&#eX;+&tF#>|zO4@%uA&hAS!PI%Ct)sZ-00p302@iv^rb;av~|eR7dt?#1Ei zX!471Y_p8_nt}#iAoy;%N@_ybuUoX#>o`T!u+r1!UTWTtLR=Y2wTz?IT+ra)gSPqw zrhZD;-Bm#2EmnvjK9}a4J3sP?wQ7L$Udk=%Az_Cz9Ba^_U0yUYt0P~V0(5LB{w&f| zU|{*v259i%9mFa`W=+gtcJ?#@1-^xPuF$-D^{n`~RqzyS3wxJzQq0cN?X7NA5ViTe z5;r%;n@9y$xUbbk1*~E*mxv?h>UN-um1z^FF7S)vGGf=Vo14kVc`ybh*0QgO{k!j&vor3U&IaJhNLa18M>x1t7(T_^~di;%utC0tx!SVJi)+GmFx&jM%53W+Hm8P%~F6vj8t zwpDC{S8oLl4U^X`IdPAw^sf7EPG8)84SEjr(68BUlTCGuH{e*`_c1a2HMUe=F=r31 zD+{9bvNVq2yRk{Rd2Md>tG();cPuFtxTIsAA3D2f1aWyEW zKN&iuqJ|oI+P6Ekb*vvdv02hxszhL9JsyJMA-9!OR$d2MVQFhStx2l#a3@HANL>SO z1-|8nd2){Q**s{P&yVEHUXorD^})b>4Y)J>Z>TYv#^DXBj3{Z(?VnRIoUvq%J)K*V0w=H>!BY8sI$58uDh>&_$e)ya>gYP14W;}T1*{XW5Kopi-NPC zX=J-{&YZT9Wh> zY&-M9oYMWZVo2XjHgvB% zm~4iWunREjqxmByX6_*falHvKX>?MG zTw^h;deSr&nbxANaLu#_8x{!8Y|Fp40jbqYrKW1gAfFAs$z-dN6lh0`gTwr1#_AKs zFdSod)#1QmiB;b;53f4wZIE@*x*?Ai#4@KC4-xU5l(RVMG7@#Aq|({31?n;@;FdS+ zsTne>UTi(S)_xDYh$-D@8KR&POScQI7b1{EGxVM{x0Pn_^p?(plu2oR$OFfTZJ~EU zk+@_mqGx(e4e8*$O!-E_%0iHc;w4^~W>p{Q33R)qj*<24=BU9%%i@v5?=WO@isqaf z6u9+u%{#lOw>rDNC3-GVcF>w1~ldQ}+|uC6(_EZ_-FrnrV?d34cq zwz}*ix_`EM&Qq&VEjV&Qss~G z}-Om?d@zc z)FJqDLc#lr*=9Q@OC8sq2jRv@SjEC;Sa6x=_@Lv|%(Uag5@)?gFWGH)kX%~kmseZi z+`N)DJJ+6PX={dCwNx4C9k16CbI}wQh$C%;TFVR2phFwYDVBM|HJ-ava~UjrJ8G8B z!`<2NSj)|?6475YMpYWAiI7uW*VnNfK}NUnuNr|TF$Yx#FYGv6O6pEK?hCj-FZRhO zlR~C8*qMuPdSvjJNped9I&v={|5n0zc;7M^%hM4}q&3ZBuvTE5v#vT%6zu{s`Mgvm z$!27Y+`Jo5-<^B?n7&M}>>R3%5O!^M>tf{Ki*K()H5-@#Xx$C~fXgm-ti`PJq3o(4 zl@=Nk)1oThT-6d{+v?$oL9ms+`?|7ZRCK?u$l>@0S5e=gtCspLjs6=Hb2CMwlUHD0 z-*Ga%kD@e5h1skXdOtql`MlZ4_^R{-`;h&6rQkFT1y`(hP}ji}c`=!%?dDo?7&$Jc zzP7jYn{r6r{k0!En&Sa?aT&eEE-etN_nbC2o&$q&2VjClpVMA7gT0F-dqzaR5W%}o zKSa4?Q%B1}1K%F_r*`I!i~(1!SK1sEpl%c3Yp(bE-9OS_@bSuhsqOQVgEhCxr`)p= z1;@7zH9Okn?qHYY^Tf}Ka)_IRvIaox-Rsl5A|4Kta(Dtq%- zrLpX{WDBokR16fF{r5DcZufN{ zp4Q^9o9Wz;n3_! z`n|q*32DgA&k_wtnaTcQ1Yac_{?Z}k)7z{hx^W#Mg$<9{8vu4`a5i+%9RN>|E5)yLJwD4D=dr<#$_6D{H z9eA&zgjoBd_iCa@Gegz!Uf=s9F$-2)i?T{L=)u@K4mBrV@7$o@ROqcK#TXO85(9nQ zv!+&wk z`?Ief+x2YL)B!r48Ozg>AHLJD=VfgZZw}-J$VU9dyTTfToVo8M3CMWnB(1EL=H7S# zY=m>P-LZ7ne`EfEq?v5cX)btkX!zMac zqgzulgqU!%Auq9ce$4SJ`Uai7F0K28MxYvFu-Kb%|Bbo#4r`+8-i0Fw_(-(@(iIy@ zmEJ)VPH_Y<>)9=FOQEyHTM~?+2p;SW<{2k-g2xi7N=#gB-is)U#O|_ z<10nG_u34uZxFC4bcFH_`V13X=ndnrcE5C4VO*0bb!IFzQqLFI?V2LbqpKe;hPHP5 zz!t=KrLtv_p%mmP?!$cHGU86xLuF57FQ3*2V!IE=smWdz^TZpF?R#mgwH0YdW?$cn zNR739A&w+bP!lfr954EG!D;tkpREOI=k}19y+Q!w$v&vYDJcq?(6IBk%kL=I$*Gbr zX=oMNr0(V6@YeM2m=$apPv)}wbMn^_ryrG;ikX_f-rcmqQ#$69Zm8*9FnV#}Vvxl{ z+m4IxNiJC7CwbqICB#^}?R-|U3j~e3ZQn}ity6gUOd&okHYJ@+mmGAjMl8Sg68s>^ zZG=JgFo_!UvHqOU_Tiga80@XL<^^6#@m+pY-$ zk-ovJ#B788?2>}D!AtiYjpD6^b=BexaEZyS1JUB~ra_^OlJqim#tC;vwW2<^@>$fl z)!|9_HTAGL1p!xVdV1BTlx42+(ccMENENm?H_lZv*%`4UPW-PacK5m&5y!G0DsO^3 z8q6NIk*m)jk&vJ>L(d$L^k z;AK?UGjl1(CmfBS#C28-M2(kM*$aY2uY0MYXx8;0_?>U6 zAz@~OQP^F8sm*uy>_Q3)H7O}&EMS4k};oCEUdNKqERrUpEJX~ixEer{h4akm%R z-Ov@%40?iM6@R&}mH)+uEo0=-$wuj@YYeq;goB~C@%Xwodnj)Ds;6H=nBt`+kEMnC zI&Cj{M-~t#FO?Ry*D)|Yf&GxHaEO@iO$@y9$ zukESeddlifSwHtwZETzMIuM0K@2okGJ+C@$sOKJ^S{ZxLm1SNjhRO$qHz+7If|<{+ z6`VG+JaqQVBYhbYGJkAkY*q`^(+ zr7Df)_*-Ab^UzDP?OnrpF2qQo6Bk6?y0a(2 zc94#(4?%m<8c4>e&^QEMSK218Dn}lfeyv}Nc$CXfh70R-p-G_;WT?D7!pWRB zCjc?huR8KbqcP~EvmiTu-24>%m~nGw5SzJh$A~r`z6FGCLzv~D%da1mp)=FNcHC#u z7^^DcT zmZ`0|7)?~N`9%h^9dF^FySsMeQ5B{38+G=zb_#~un{!d-rUj+QrKKyQqY{u+QNd$s z$781F=&u`MqIEl`#T#3E$z40k82DqE7lJYe;orN7Fj2aC25n7<^-fGn^FwrWzBYQs zeVS5`5OnSNkzeeoK+XwgH!@IW1UU(1O$By%sOkMV*~SsWUB1lCO6x9 zp6b(>>a0|qXYr{2i*%#7PNsG|H6yUTfCAnXAc3GZGWW5c`X@VE_b#5~&~1OP?ME^t zAeEXeAhDQk{tt`3@#1;d)$-FYU*+aRMT;BYTAEl=E-kQAFO17(-ui1`A^0u`6pWoe z?b>De?kmq~JM}Xr`EmA0&~*0A{&rvR+w*umrX2`+P_O-U2^2oiP1MyT^)xUzytP}m zT1?dc-qYd^pzY@kGp*lqYHXBt3LPJBAUAJ!7VP{z}vEWb@HWWBa%C zoQAR4l*Vzvtmi)jjsS`CrY4?GKQV_Np}Oy<4gH3{==$TsTFn2s0#r=0jtK~?&6uCB zsA>VyWCHfymG!?=XFyoPQ+nt-LuYc!mHy_dZYY#LdyuBTs}2#HgDcRH%~Cubt} z*GJnP!Rpmfzb1f`Bp^@WH~+Wy?)+;~uJDXVvC{e~$=8>T9-9K%6ut+gU4Qf(q7?!O zA34?YL;~`RV!G)9n;qDPcwAhp-oIew{JG@96?ZX!8p=-~ztFL`pZqZbz@>}wk<0vQ z7D(&2IHOXZlnH>Z_sH7#tvpcfvp=k}n``o#_~>!-@9W*(i=<}<1dbX*3YoZzv)AhW z3=FLzI5<#T&xGRn?`u;sy9sOo{y>GmvUo*GjS~Ro7gcTieKu$u5G&aKd?~Mq0P@nH z>|Z%}P4eqw*Mk0C5kTqYoj%H%b)rA&i_cQ^OevB7GlBr7ioFqFOg-;bo??D{4;S-# z;bE6!Bsd}dk3vlNMY3>pyT*p~Uf$bV;`e(cvIVBEA(^3iQ|^OW{e#7UA7G(jk!fnE zUmXf4xyH>+XGwUr3_-qAnK4Es)Uiqc8f+z-0X5G3LYBX23jxL-cm82+L{++?dwnNL z%mJeATJ91&qipY!sChrVKx;&TG0l4CG2L?8z}G8PGsfPFi}^2y(~uFTT}jWu#nNY~ zA#s9_&qWgr>u*D%zNSUhDW_?mlDwBwg^t%mJAZ%0djlN#+AWA*9FJ4%Bv+P78WSL) z5&_J{#lh64!GTwjn9ER9C7Y}6jDU8aJ7W-0DvZqN)j|d#4|*?YLE}D+N-hb@P_V#3 zYwOzut}t+d6(#B{@dR6GFFd@p9f`g-%_SQ)aR zMQI|OWAF0v_b)p&p3`nx-CRrn{O0@!NCjTf%k`J*POMl--?Mb`BL$c(-vO7npQFT` zrU-QMHET-#?#9iu4rXNbb+K2CQwQz*=y=Ls--MLYU$nGuCNa~rlN2B6!$c#PU+ri) zcbZ+9Oa)%pOaE>k83%;f*>gF~;5#d`SmgJJD2k``X7TKV|2!TPPI`ebW4mm9K347@)F4X&%6R`?q;_#)I&#mq|VXvSq8i)%k$g zt|<|MD1*~~8B@?+XKVrqu}Azar2MD+>bVwctcD5)hA{(}pIU5E>Gg^BkUg-oGZVY& z2Lb}lumy?-POVa6*a#By<+QXbuBkQmdl5&G8;B9K#ah&j0dA5oiY}{xAH)g9_oII` z+mvDIf232n`TSq%9l2=)#AJU=!2gUk>aS=2uZ9nOMv(pYw5$h;Ndkq*>y&;sxnG@Z zwt90fj64PQvzDJ}e&jYd)o?Wl9BBF}Q0c`yZ`tFAFvZXJN`zHmZKs))ue#HML=j5n3>!^j%V-^#{P#(CwO25c~5dRvrS#~_oO&wB_i0U7&Z1b zZIn3P_yDvG5V+b82;9^Cgyr+Q!5#hjf*m0En-8cOwG$GT9x}q<@g(Gb)Kylv*v{7* z#c2p;%=&0gOqE5+Hs@Hsu`xB?h-I}aAaJ7iBAX41C5=4ZL;*1GWQ4DOeO{m=ng3Go zNUE`siG_&pztlY1NVJm%t>TR}vRhH(v9B-MY4GcwU!VRTG>?iyUIc;;8y8NeIyqgA(!S}1KiWBdU#YK0XI;OenuJ|64b0>sBz`{vy3q7{fjiBo zV^B?+`4B|(9zjkkpZ9NiNJj<$B};{pS{^__{Tn6g^egjgLXj)nzh(%?&jz;hOP~eR zC-<+9;D3=i@UO@8h5UlKSyKU#_&?ssM@4EbW5eG57mZRh>Wz&ta!h(C0+6Z7K0vL? z!(9OF{pBtR3OI{i^CKW8wwC|h>;I%-_8)wi|Ly)+fCTm&6f@6mBY-DS&>Jz~2P zOAl%+(x?eAO1alZG-affeGx<(@wrH0KQqwpDs4wajmZfvt|iRh7b;e2u;v+YglU5g zCDCEH5PFUa7G#$$&|05Af(2%SK(BeF*H8bDGCWO1Y1yIG*9;vO>XGl*P(FPRIFg_k1~nOz`8{5@sN(fH*UNVvpFKrnKfY zk%WFTDI*#i2Bu25dM&rl*=QF%^O@tx%Y{~#TFD}BPiGvV4%bT3gQYo!JrtK#=SrR0 z*@g{U9aqb0-JIMdP~+cw2R(7Y2VDzw`yv{jN=49fOMo-ymRq4*!E?4(xFiFqzG9RF zS-;z|$%PQ~88nn6&|?&8SUv(6O1$EisdI#}QLuM|rTBAR&cQQh8qiGu5t^~%RZwJ9 z-}4Yg>6>%ZHGdA=jd+3U+;!r7lpfVqWw#1fU(27+HC-BFD^5>+m@mNpwcD=VBs8>8 zyo&nPBwLTJ`ORnguvVg#+pJH(bmNsv$U+B&p1yVNmvN76{i2R)7jr!!M)5#$KuLQ> zxIvL9$z%+1Cz8YI2GEC)Ph7K=qj>$0OEyJy!&+e0O9rZ zxfBNw!FHnyEnoelZn=`<)|0n%xH;M`LdHM*xLZ?jwD{AyU!gmq1Y8f@)Hk(+4<7M3 zSUVhYOpdHh7#K_!@9UdmnW}EM&vqffL+(j1(vfmQ<^uXJ;go((hS8;D-#ZU_ZD1w@ z8-eGqr*$F9Vz~23#m>Z!5nUIhP#vW+j><~yi3Wdv27?BWp{}=og^P)%{rXKZyMc8P ztxBki(ynXSi(|BC<{Jm&R@KnUo|8Leg%Xl=4GYguBkr^jRR~y23ZPPw4M|c`| z<7(M2fbPA1;kR23MvY!&@5{@J#Y#m+LMdco zD70ygwuhbdyZXNNjLT`HIOKfJs;vu-qeCXG`{|`%PLyS`B0pp|Tw`{c=+#7@JoT-2 zpDf)kGM4EMqMBF>Ue!0hYco5(?+iU-N@#nR(RspccvCRgzy^W6)Hie9^KYaE_`+mS z4lDBaPw?q_PHbo<8%o-CW6vJRgj~c~jr^N9og#d0&a;Le8Q{=RhfZTvg}<~RsL_VO zkrliYc532yxNYC=#eMBAuOCuVc$>V-+3c6-7Xp1$WgPrM%gq}OmJgSR);fK()#D25 z3x#)tSF0&lJ$~|alqFGg24OXwM7zq8)IJ916GplMQ8Rn+d0l(|NO^e&6ThF5Wmeky z-eo_~Yxajeyi|>uowJxiBl`riL^Nk7&YpC1casamedR`q866yH0I9CBWBSIzOeo2# zYwnuCIWi}r59g^rs<(eQc;@L&zCO;};%Rq*C=Lv**q^vt>TalN90q0=%;%M@LG7=* zj#uJwk6)&vYI1!FOSDZK_OQ^o{^K%Shws66GXg6T9>tqzl+Vi&r*pG1U&|t;+z4vi z`Y}gJXTCqGVP=_%`g9uLb&Kin`OS=^+aJJ^rfV^Ej09A9m;?*V;Zl~_RTSO*5&u5> zLnR*G99gHcWn7_!(u(G1N23rmk(C3YrKr%jZTZ8bFa9MpgHdmQfG(&pQ#}_`fBZ>T z9Y(-(Jv&XOLD2}N6ES7$+`~MIDp$*9i186Gz^%Gm0JW-Ij>prbuqUI zwXYQPw_jAPGp=!0sBtfw`G{u@!#;Lg0{R;fa3O%_X8lWHP3qUinm+jxGcb8GZB0a& zEnhQHIl_q*{G=|`LE~$G!`mlCMBIFMs=QTqJpzy-J8N~c_d$UyGIz00uWsv`GfA6& zPJdaEsv!&0Iu!uQ&KOPRnWdpmA|%k~Z{6X_{v2#zR~S|yvIpNz^s&lPr*kwNV<$#@!LX`gtEd< z!8dFaWwMzd*$hGLxB& z^!?WxWd<1qF)9UAo+*pt-$et0|9J*GD)^6R#A02KNW_4~XY!^xo+6j*q!09j@lUUb4}K^mKgmqVz$M$HMuCRQDY0S5}DD(@r?FxO^vP$g>n( zSft}rDq^b_^@7Gqr5n~H;G@A$x0#QmPw92<47E2vOnLNEgG^t7oqjN9p)}!<+HmSo zv$s9wlI10Lb4u&Ly2C!B!+1@|z}UL$LZQgT)kBWa1z-V`755z3nAa;z`DlmJ33=`} zCpVL80^Q7nhlA}#l_$%-5$ruWUw7O#Lbjs7cB}b2L)sO477^Q3>Xi6i@v*W_XM=}j zT??;q)JpT`47$=LF^H8zeW9=2#oJ^>F1HTWh+5WBuF$91ng(%v6)i5I&X)A6=J^Yu z!|{IlhGhlfm%dclE!2xsQ7&4JpAUiF`4FgR_?79SZF9STngJ( zG65;-b-WvQOW-G^qju`ZHFUPOsK^Yy@5fjqhfS9dMCCOVD~+lpg{T#hJT15AwYwbq zpi2mO)@S$eaj$qCe<(xV2Mb-203#{DY~(Or&xSe|%Mag%k0g+Q7telpCP8$Ua(nX3 zv%c;X4wHW8PifnBiZ2{3U3RZMsWS;BwWA5i7l~MpqqjIb^zmm?o*{}`wWC8R$Belz z=%064LvM!R&b~v&9NfzoLV5N@S)bh0f#OMfW^5I!eBoq&x_hTb0@U-Z$YW6BMRoGd zZp-~Ft~4ER_15h4=y%VgwWqn90^RQAS?jvfhRrYb5%K6u+=`c<%gg8DXUr?A)?%BV zaQsCJ!=;hhkN;Xk`8@9eYmASGh>>S3>Ccfyp-e7bA#;acNk=OFBbkY00q7&wC7tvhv1A+XWOjFX<@$uP? zH%sy>LY?c&T(%C+-M`R{3kxniFb}1Y!F^d%mH0KkFd8mSr~Vf}L6Ljuonv5(G1dLN z`tu!lBNI%5uXz+1<>b!9@p1_GwPz=VJPi=U$f4o}I{*HQ?;x#tMaigg7&!i2_xYwK z;Oqy$NOF1mCJqg~y;Vq|#7!MFV`#xC-+Z%O`*PpYMqqu49Wn*31oe4R_&ND{FSH)C zv@#jf`>x*|KmLlV2Yeh1K|1$8WFxU4-7>3)?QRh0D8SkN0_Z5DmKh`kmETAFg-DZ!ROCgB%nTK zb+}4nU2Ch3Z~?G-Avk7gx3BU-6LIXxK|Tx#sqWfAOXsK*hSv zT67G5Wx@{Rg~n7-1&O1pwT6?fVE6Ad3y39j3wfcFy5Lizo#V2)x}q_4%tp2U*AZ}n zj5rO*v=sP^e!X$ibkYn!Hudy@4LOxO4i;Wrsv}LfJ51)PZ#RGfX4o10JOYIsN|(QHNGjV&Of;1c%=6d>LV8);bs zAVG$xoEgUbrW+-9>O7}moucYor5IQ*@8uybT5I=jk0eF71bp{Nwx4>qMl|0&mq&vUEcZ8FjbhR;8E@syDuBYpcNDC*BkYqntP9%Qtb% z=6yG&p7{U_g1)fjNaHAEu9bdr+7y0v5AAbJ_~KP7Tl<$|h$Jx0fw0bqes@0Vqc807oM9Mo`>+{!QMk;tq!a&6zJXR2BfB#;2iR1gjH z!>5Ii9RTis82q*w#rGxnKzW>~^-H0@9`V&73|E;&#jwf+(24>9v5!NsJW4uKe$LLP zG?BEom(`znLRWJ(&CAOjTP5srm_zgh3fwX{vE$ z`m@RhTc3<9Af3FjJqYl}uy4SlWe=$?Sf+5nZ~(nWLeXZO>hmgMIsD5xSdX#O#@w*B;?4b*nwl{{0pd0jD_IWq4WD@Qh{h}I$Y7V7h%qiP7r>n)+l_9X zPvHKcVRZKm#>b(u1^(3)GOEIVHzn@2r5dH*W^OHKx>PK{DUqomJLK)$vNT+v3xNEA zi9h)<h#VeY!3jC(WV}{8>Fw4waiw;m^_qN zQHtS&S7IMPoEK!2XdEwqKDHZ&(})^Ib*=lk^DZcbe68+(nbRL^ov~{B)kuTcmuKtv zY;BpH*joyGyx%gf59CKrWg8p4gpTIL=o{DasVgGP;-g#*C?0VG5b_R*54j-bJq7+b`ML ztpoO7dUdgaWl-&5u&6`9Xc!J=r{8vtE^r(xzN771T)F|+ym8x0g`|&08M}^blf6CI z?K{K(A0hQue%hP31W+*&I%LE=aEkbTt;A04NLrN#6IxyPcu(rc%-J|E99_yxF(V;b zY4{Po+<6qZ#y4eXKml8)=P235eZ+ODiR*ib zYi?bWoaF+e$DwAqdFGP@pki=D{T2=&eTwlnCJ2(`;@sWEVZOcboUo;@jRBTOJ`)|q zonuT)9>}w(k-*HQV-}-5-&o49RdM`vcf4?HP*hEnAE@m?SM1>N_asE-!Z&k7TpyW_? zE^jw#0u^UwLQ!pJu$rdJ4P5vbeCt<-Onn{3if;j0ZJaiyK#p-wYIF93(eHC7@T0Z- z!#g&n_Q099gun9YrlZdpYc(1h!7#WsC9UbXV#PlP_iDKnt?43gZNLz)^8GyOv^&{> zpf|TFxNO)_^f3W(YmP^w@Q+_-JyIBdm&8lotc3iD?tXO33nX}bQC6<;*M*=DDmU-2 zXqxW*>tSq5i9{AOwEX7`@}I~5#}|J6zZU`Y43GDh*b?zm_Rc8Y?1#0qJQ|6s^4U1g zcn-WZ;DmD$8$I#qO=G5|Ox;bahS%p0|EwD4tdi`v&$uLBcV7QZsqAG>`Xslcg!`bS zGoWbPlQ-iKPM;53S3PskHz_l)nDj-nZO;pGzT-M6kU314>^~f1>sqK*DQ{&7NwkFVnZvGsxQ6_QVQ(2FU-hVw`!wVfTI$WNC z0Qq5&B9jRe?-&29k5ukID<6)D{|}3v(58|RUQ?`j-2=roXS;fbf?e_LlLwN5M$9U# zadAo+S8N{SI+xdSjPFH6IR5|=t9h3D>I_Z(lM%{@hnym3HM)~Z3kR3>u+^OOXD`jC zbWsi1MB$R0BY58($k_lE(__p$dU!aCjCz}$*Y?DlfzNN*GFQ-wUEE3;#WHH+b^ZqY% zWnF{6F4mr|4Hb*rKLdwoe{Z6zf%qA6i4@4dj&_9JHv7G?$poMOo-#BxK zTewHr+K$Lpai*9eOr{^rN-y2e-LjpX-o!1=;39SvNyd3RUuYxqIO}$;#rZpD^g3Sk z-^Ta0OQhjO2tIf2PMKww`$}djf`;ucH0Kn~z~jORfQfg>E+?gp*Z_M$?)&8Zo*KY+ z2bMqii)4zl&j_EF*4#Dn?pGbS1wZQ0X~`4Cgx5XP028XCc59E4Ur?5xl;{ft5vp?? z+hLYUx@IH|Py<=+)io|W@aky)&4GF7rf|=?s$sd45v(5VfZ%9ZIBYrarO(666n=@} zn&8}{{LFsbyJWO?v3qp;lWQ2hVspKZu-Kj1-4_`@dLv>dsn+Ii#493^AR$0R4|kO^h(RZgKj=Eg94d|GnneABY*a4ptvGFMDv z!QR>4Ln2j#xr;+weHUj_19M4U#Kb^RR^OeW5}g1gmg?9Ddo)-_8DQh9ofnEUL{J=x zSWVWHA0B7Id1?<`MaCckaLz|aRtnm;*-TruWu{qGZSH8~Ad7fW*-q^PqMWHZp2|2+ z{_z`a?v>#cg}Dm`ND_`?>y-tG=^Ve&)FiwUG_9!toP<~?%{?63dS$p+ihC6FeBiLK zqoZto)}C3AS@0)RwZZyNoln6Ok4rlv}x{nFSO9* zVYTv@B;|IZZ67)PXxF@X;&nWy=dDA}L@Am<3x!vna@Yoi`DyM&!W^Kk6NmPs8mxYU zRF}}LP$h9HLHES@)43s0o;X)@Y?t{U>RNS1eV`4CvEp&=R^$P*E&NmGtIXx^-Gsqx zVM76v`v}H)uNQ4Qb1LqMr3fPgvj+0VxT`W|Dfdj{tT4h3K2WycbB%c6N?cJ#6dOYcNW}hiL z9jlMP*5vqw2u7y#2GTr>4~{L&lyS93*&G?#Z@Nt}yfUq?NuG|IsO06plf7%Jfqic# zA!jR@;xB(UNEJdH>np6Zx{}sK7*4mOCt#(cv1W~NHcVLElFM8@NulcvGtG|D#DLj? z=0+tU%A*&YZWzTHm2S;te}YV)FA&I#lE$Z|`el`w`zZOn2{niF3l**P;Mqh+@gCr% z`l(_(^hKcAbLS15{iTeFk{N0tZGWzaQBRI@!%^G@z=PBl$?r^e(a)FSBb?uY;j?t) z8HYrhil@RAOpZ$}d$NR9_nbg<4y?I`0;*p3y4xr!XMLWTzWtNdspLhOb2%W+Fs z0YA#}SZTp^lX%O^eK+5kbrs!{1ms0nv`ML6=<)hUh<&qJY6{zw`8>4d`sU^2sO+W? zf<5whQyfmJc>X>>N09-6&-3KTP56GGZ6UkClU5bMEkjiCoDX3lwnnIESF8l|)jvB6Np%96Zcc`PnL5T?uu1YVpT4(Qp-^SnNQhGSOee7I`Zsux9v=1hk9!;=kuu1bpazH2?`#Cm0(@51n;eHc-*7#=E^F!(C? zQC;vC_S(Ow@`p3j8!gK%u;UJPX_I!#v0u}iBu11ugW*C;&`|NiN)2*5x$tw8_4kYA!=*j(s3H~+h2HYu(xTc9tD?}sHt7x;!ca3}K zDc4YBNJ;i9C!X)sN6&oLVqZgZSmJmthJ>^5ewU}uxtF(`nl;@4nIC-&A-kIMxPSs) zN^-n)Ge3q;AGcQRXkyj>Lzna|tv4&KNVo{>F}gk7uBpMdyxi8RDsyWwBCXll_M=T` zV?Cc`n{;B#edbiNJEg9zEtUf5e45hR{)J82aV`Z)*@CZKM4)F~pI_Yj`|~JtsPl^cAp&oK;4oI=3+$B3C`K%|4V~!&Ea@;O}o7GQ>hZJMIR=u%D_qvDm3ROInP$ zv0pxEd3i01rT+G!%vJFMODeVH~xjHNXD4}xEqp|SvTzDJ@xArXO-G8Ww&PrI z3V*gXE@o$sfP!T8wS!YiN!BxPTR_n5kLeGc(mN%uSz?wdTTA9et$y&Kg7{3y`CGmA zcX_Uwixhz^2St>j`_)APn(>2z>d{Xs<8rUm+_iZ*6}5gMl*0%KigmsDeJIx3l&|en ze)=r&i{KthMDD8CkY*%06E!qW=q>ePJjalUwWrM@rp5Am(_?raPkI5_ZQI8~=$sE5 z)tylp^APi54A!^cbO%yJyY*$)vwv%(=VEtdb3t*7t0{ z(af@Kr_s=5+1*n^H44E#v5hAoW)`g+x<9A%iW~v)z35gdf*#v7Nf+P5*VU=7EBje1 zQ$JO4EIEzO#@doS>yy8QLR~{_Yzxc7`kdW$El;-3QOu>Ag(Ocbpu4S|FPe`}ESVX! z%yFinJXpD;+jX>Ue!XV7YPA{}s@g@v-Bb$b*0YM%_0pI!VW~{o`+k!dI5MXCXc9KE zf!T>SBcC72evx;O6y(g~yNb@8DC74jR*W*dSnc*fFWo-VcAZW5i8rC;Vp`&eY&=*4 zvouC4sEu{#&(Z2wRAX{&?vhAQpl`Z*=wY4K!C%3sMI69KM;7Kc&YOFSO;W&-vC>eq zqYWf!s@#1uQyPq4kQ@`D*e%~4m#D9LP$f3)t^nJfcO4tUY<6yMPSRC|&gP6X=G51# z!%jzLBRM0Ms*S7Wn|zPy+T1S(+ej42ebH&#pRgXNN&lMMxxm6-Ow-Id^qSP}g5`6d zYPhh39CV8E>m@o{38!TGureH@L9;){Gv9@e3};^Sa);J+!n`Wm@C4hxzR|9^#tJ4^ zT@1Rhy3_@#Ig)^hW$!2J(i3Fu*X1P76xQnDcap~$7`RWo{l`l4*@7a8dmapaJ_$XI z9S&PtAA3`YvlVWL(oQA6cbsXpG*SaV*{JkGN9N7$TWQkF=gt zXS{!c%};9EJ#B~j6x)i;WO`gUWgTDWiD>1>V{7TM?WZ3orpQo9NTyVXUKL_!!U(jQ zs~Ic<5fvwl?OH@~E53-|g9a(gO3{nmZ;1;#Q-brlT1J0UwLJ(Kx^(Ym3E%b{^4@8Uz{JX}CJ=eHmT+(JVy9#VjXo01H|-H^|O?&{erNH_nq5ndVd6a~|A@ zhqr)5m{pP%O0uKcF$%D*s=6aou6-%efK>_|>$OealQ!tXs_`)CR*gpz z+wpij<&8)xRaj1Qd+JIheYx1S^Fg1ne~Z7Jaob!4LIv_lU#C{Lp;F|92vZqm0iE<( zdQg>0{OFuq3na^}|KV79%bNWzyehObq|pVzNv(eIurpm-C0ZSd9zsb;jAs&`Vp|7? z7GU*#GZUuMq-=#dt2{OImj(oqPX_vycsbjq+sP}V?k7k~IC9 zlh+C;IrrVO&$GK8Upbp(HQlQg!g$^p!M(r>n-qX(|wxKy@`pDE<=}H3A^I8L^Kq z4XHyhk*Gmi!$Wlbf~Css!-FBcnpL7&(#%vOn}d6*2Sm_9{+x}V@Zzq-A{7~gxQ*Y9 zjR<~P>CJFfA}m;>G~--FI`Y%L(y{>wQ?;2tiEr*e+V`d#7Jo`&{wFffpAhT+260Ir z8=&UKA3!iI*I<^?BlOScwcEcV*~1euW_IFF@W5CPr1_@CT;uMOL8rPZ14UNg^}piV zQn0VT%Zo@=H?I6Dh>3i#F;&Bx=CLt$<$nldH)U5(Z#0M$d8jOTy63fexp?9LjZiW6A)jbQ**1*0}yg@psva) z9?WK%9&aYq_j9@esgsph8FIyGFl-Ehf$y>e##*LRvquyB1tf;U0|J@kn?QxqgcYk5 ze0QYpHcM^(27yoK{#W%yrhx%x1+MqD3fzD?2)%ZL8u$HarR+T-0t-A)o%awx)xR>u zz==r11A#}p2)tlR`x?KCKao6yj{nZy_^=WYW`Y7wt-9OGE@!d;)lvcgKzl}+&S4sp zybM+SkCWZ=1jip<&dTQ@;-m=+C=Pn2vf+EW@1q^~w778vNQ!*Mf1ptO1C?L6SJ83S zfXCns$Vg2pk#^aPj6e?LJw>CAKmPbndFO?t;myeffLYKLL~Iub!2V?c)h@~*3JDUH zf5xqmv)I+T{yc|4MFgZlsQ9>F03-c9kmR4!#q0Iw6=Z-S$%6p`lfXRuJ=&RrpUeWz z=Ov-~?|L^I`fO!DL6t_S!XE9q&31qxlreDu-S{i_Fy`)2f9B)&;I&2_SJRKY?(tRB5 z!o*#)X=Q?>tJG;q#fp*d#*%o#S*pIbU84TWrPO^tz;pNI-Xyz6vTnbDiq^iH7Cvz+ zb+w-vi6eA)p0Tay>F!?f5e*1Y-k@dr>-V#PCIo17Xj7-xHulN?wiM^>G9`%l-S_kV zHeS4KcK+|LBjB&6K&t#-$F;YA|1?3r{&KHxC_jOV@)Sq=ZEo)KGzA64;tVqFSWa$l z5pwA3(1o8hG&+<$+wM=s=UQ_4zlSy4&^uf+XY(WR>)lLccQ8)qI{RFZoz&$bH?o-4 zm2ko;XRCs!oBYr+U_O?>j6uDRFv6SLoCQx&;!UKF(H3}Q33prliK++=@m_`VdU=Cn z|5fx zXwPbM*HoGzwca{DX|=A}Hd6TYG(xac8pU@!+UqHLlF39x+qWU&&FR2?1nc@}4c0Jv zU;4BcSEbyCa6fk6JcC^Z@^`6&0!8GEb=qIop^pilnn8;;0`Kz=IvGqN zEnYDT@$8H$c{@a05hG)j9N7?Oh>-`uEtUHN> z-;4I7f#>4;qyh7-!l@jx$PIh3Xl9cG=F2@D_f3e}Dr@eAEvu2+XGYx(3>81@-J@B> z_li~3Jxp`jTIFJ7X6c^|hV%-S-p+~$Iuph9lk0g`KAaKXrVN)Sd@~Wn@0GN*`^hc1 z-q8-1ouxQAmAn6qj)6R?#UAG<@8qC|@Sk#OAx3Y~!VjJc2#Ay#-xoqj)tyd|Xe|X# z;i6z%jvXnl4Bvi4Lz9>VC$~ni{;wbgMQ{9?a6v1dM4x1s&`Y$_g}2~zo(ee5MD%7W zP;;N&^HCl@w8@#3h_-h!k*eHsfAt(uY%-0f6J;b#OI-gNXK}$23$0+KpAn$N@N=8| zz-gyIbk!t{6T0Dj4gPrd?utjtsj-oBcl|dLOYZIphK!>Sh@V?e5NI}&#ccLsKj8Q zn%*(TDwCO2>h3(;sZ{Oe-FpnV_a{Bb{<JIf_~@+@^WJ_P{c~Y&1S$Y1g!i zIC>*we$MsvO&6?Z$40GDiY0TeF;j}4->$z(hTkRYm^Cu-X=@z6w{ktD$gW_6iN|uG zI5iHEWMdX}EA^{>6zR@$oY^UUeG*X(pJvysi%NoQfn9uw%BIWEIY0P$@kk=RqTx&5 zdb>J9g)1NEO4ipzA>Ko+Ubs!p%FgcWSRvSV-$c5dRQc%?b8W+K5>~#BVs!V81RPOQ?DM1`;UU#mM*Q_yzGCXS# zz?7tNsHDP~Eqal&{xHcq=9{UfpP(r_B?5E{-PE{E#fIV=hNYHg=uFpsXuk1Yrgbl{ z_gP+C3mI-2aqeWY9!hnGu}&`Eh0V2KK2*APri7ij3OpL2&$%n1(qf)ENpoDcSYn2q zxgTM;j+@EEw7_EN>mE+_?#DbZyD!YuB(ZX~u1+Je3pMKEuo$WdInB175EHbjzt6Z^ zrH`qBcpi%++C9bMwffL2&gJ5EzhDYdH?I;aU#4eKn+sWaCv2cDl)$daEm~9;@6Vm; z`+%umN~MY9#M~T(i7aAs$iKUycy>}STVx_QE|Klnfsz;%z{)_4%)QFyj#}Q~L|n8y zzT)!nHJxv~YXqFIrzU2z9y7#cUs#@a4i`40GC;EmN<`%xJ$zkGoX5Rg|E9}~dQAZ- zo#uDimF~HU!I)HS4jO@TlRfW9UaO8Kr|MBai{s1OpBJtAKXVuxg1uZ22%308SY3hK z3^>d5J4A!|6-LnGyE(kgenc^IBb(U3VQcur=#M2zzG4K2s99fcS`;4(xTvcleGfj4 zELsh*E!GHnm$)-I{#EQU!NzgJM_fw7J)Mx6d->%fFx*$1;$q%vvKmYIkEyM?ub23u znfI6a)b`dDT!iaYY*_g9A72C)*%ijB(hp27v>;Rj zAIQNhu}&eQUx-I}yR1KswxiYzu1q&n%m*uY)ZUbcw+&i56-4Hy{GiV1_psaY<3;_w zkfOcVequCxbtdn~LUvBPU`%eDtUq+{7t!Y$CE_bkgTlSXuJ%UMQzFg=do`>2LKpufm@H9>ONb!NGgw1Z5? z2w87udvVt=g~NzS=qyRO&7SI{clG{+rh=6pg_E&1qC25La_#DBWzVM{e_fkw$UL?s z^hn&jvfunjf)-=F79T&uXkj;CQy(ic@&gFY&Pws|DNhSyHRwCU$S%T zX)V+*(V362q0C%~FmcmzK)0Yw(hVEnEPkZjr`hKBk+aQtS zG$o4M+|tdfSYs5+AuRqxyxVwT?sUe*{CO&@_sjFtxvNCa-h(COdpqyH`eihw>Hwqp z_5Dp3I+jvPh{LQv{6*>yiFPLV$4-R8cm@yZGj6+r=A@KoVCzdSRD8_ToBmqF@)|8n z-F3Ej_|(E$?WpsjfLBV=(xxUORN|F-kjhgVr)gK_f|4e(7|}PDWhvQ7RW28bzE91F zx)|~X(|6q!t2WhJwI8_FjIownSts;R>sa@dbc_3IZ|b^cZ*gAa3=#4ZF)_VunTgy5 z+?T)KidSv<(2{XDm!H;VrjX{RYl1xx@4vH=H&W`Y&UN4_uPR@+VXoq)rFXDI3lzMc z4Qj9|>*9S%Y$Vq)R`l*cOVi9Z$lEK3U;_ZFMqQ(Ruo;Ho#X5yv!0iDecJaF?6p11y z&bO=6tY?>^58X05J$;w=Xc150YKZTb@W3~3Dq6;qrM}a7xAx9Es;RA8^mK#+f}qk39s+ofCRIcr2m+C= zg7koN=`BbL^}w-E3{s^jMWiY%)C3Xfy$GTA7J4Tk$&25?q4&P`?j7GZzCZ387qnt z59P_w^86^HrngrhF?fxWgTw4KGU-wDlB;ppJ5uKAdzVAYjM0}CPrPUZ)Dsf+uh1I(hPkt>=io`1r&mLfTPS3u+v9K-l`yZJ?wfMEf|mBj zu(mDNMaGkmf!#8q((g%0gE^rS>vh&?HHnPRFD01893eJlG|Fh64z7_!ngd{(Rl(Q5 zH`_`)oUd~nqbFsq8Gig$R(wTY(R3_P9soVurGC^DWN&RC!%1rwG~P_Gmup@Qn=@l6B5<-}YBQO5_JT#snQH*&x< z@=9-zCduwIaJoBCf$|1{)AhxP)Kk}&;FhHNr;)JfEa{dnnCxe-us#DaXdmns7!QpQ zKCk>mx0^GwT)fncLzoLrdEn^Y7+Ubg-2~dm)gx{58quQzi$siAXh8=BGPXHrz`(;X zO?WV}DMBJsm4(Bb=h6*)$047r>+9$ZtSm@kvvyJRdF#7*LwjQULfbp4B)g-N>W%v) zB5K9e`aVqMt*;w&)rU=Y!3u`MaE3!dW@(yuHQ>x@H9C;2mhu2U)ttmZEwWtN4dxbp zef8xi%yG)B9RhHd7|1=hPw9@+`>gd8qE`LHaOI|V_(3b=6v(yo6!(mI3BpyxK(U33 z1u$G%7oGnmXc2diCz&gPhzUTLCb~11c+`t2!Ehibe5R7gvxV+=*Yy3u>%T3AlE2B( zV{tFOTKDiSYe_|eBav5Q{hNzL96#XLi%!%>=(l6uwy!-7=b^O6ewk{*s?pt-;`N6%6^Lzx>LRY^T#y#l^NQuk0* zwoTcj=peEhBHM3>vVaT)DCeaID)l{Qa`lS<*6@O_J%AO_VVAe@u()?wD;eJ1m_(XG z9R6va1XW3FL{M2RD*9DJ)qKZo{K1+em_58C`yKDzAYlLY2~`b2jwxfl_W>~@S(rSY z=f&w!Ye5eI7hsphtrwJ8>RcD0dyCT+9-JPK%%hvFINORN94A;Ltp@!OY`%0cZJIzh zhW(&8-=FcSDPEfAMh~*AInBVGyqLW?{&U`mh3UhN`}XUQ8jEcc?A7YTo&%*Zrepk3 zt14v{9H=+}FCLAj+t(I-*Tx|~^P3d1a^*h^`6Xofkb<0Kf#8@@TL*WjFDBb()4w(1i;q_Z3EBon`2my%IZX?@Fh4^c!dj%!1Be4Oq5Kd1!huI(-N5|tp2gu zQ@bku$P8l}W_#<)6t|{3eqv&!&EI>{Axte0UjzJe-Ps`+X(*p!y}#^~-^?R@2UXR2 z6m5o+lk-*Vx5@W-+W;#zu=3nkz$p*u-1??Ik_&KZA{b{+k6eIo`HM-+@0GionI~+{ zne^$-!>q#VmzH*0?-4WoP zmr~BPY=PCJ9(|l>_-J{$8ytR6TJsJgfe1d?!(d@P6j#oko(xWF!<|V{gaC%w?d?8S zT3XuOt>s>Pw%b*_o++hPagj<|ALza!2sY*bQUcr_HqNA0UaT60Pr(CvAKZ(t2`9*N6x2*lB}rG?a7g+eA3v;4c&&^3vfQY1*37FqLKdbmgu_! z{-!lMc}E<}XJRA+#UKsvloSrRU_yOYPftgU*RqYjze!$R-g@MdhwNl(b0fo7Dk2(h zj&?A6JX;^Mg}?$UF1`NHGKxN^jIGVVr3LQh_B2gCw_`;$tc-vvHFy8)sSmzr*yH#) zUc{6mr()s<8O>s*qil?;2m{@Je0UW))@+^^I#yFY?l`*N~kfZ>#aZ;t@| z3bZFM6Ud@E8P_${MEL$8c%0YX~vK{UQR{mmN(etxP#S``(O7FolUN zFZgnwDg_Ho>Q$>O>Qsg}ZBC6U*;=5oo)zy2{ZG;J1;G@3rS^I67mLtRU|w{S4LJ+V z8lP1bKRSop)VMKFJCU!i)kP)#S4q1aeh)c;1F1)C>7+bn){WuRY9LbUO5Fe@T|M3+ z;xqC0q<4q=V!Cwo-|Ap&$Od32P(-x!u(5}#jQVWA>!jX)MjTfYH+Z2LzN zj#L&+;SKaO0SA#g#gxJf$nTalx@)jI^aod3!>|1mpUgV@2AcFe_Fz}zr+`d;9eHk2 zQ@!KaKG;IxGJi=nzuTG_-0?=%&3OO>zX=)ud6Bw=cQw!(^fS>-im0gFRzUD5KRt;d zYjCo&&{2xKo&TvwW4|9BTlu}|8JWvnVt$3tjUVER#Y2`z5|ZcsW9i)A#m+c5+LBL7 zS@AwYvmjg|BSJ6m^TfpbGDOvvXG&36%?lCj6wR&}KC7S7kh!l#Jfi^8F+}}`dRFJ- zss2Ax|9^_9X9pPFeWP~NtOap9MW~=_cn*C29#bKR)_(!2KerFlf8RgYEq*K&0*gb{ zaJXRD0BzliTZBjUz+XhcxO?nJaCvVy8@hl7Me% znDw|>{N25L8k@mZIPXA4pgs1*ez1MUxOqaa?9qNw3{!=}Ot-=^m+^(c7G+{OZY8ql zwrRpv#xbp0PUxMDR?d3uBR}n1Di*SpIMUV)E}KLCR*W%`t&gh$WOEQz!VslR#N+Ti z=t~Qdi_OJjT*}JrmNg062>0|<*~GPuupt(7Zi=dQ?V-A3-?$t(1t?WFh>SELXK>2i z-8^smOBn$x3@vlG>%Vg$w$zpf`F~4>u`0Y@50rZ74$!EwMp>9VI=WR2gp8JM21lpI z{Mp-eFktd~4w^U|;~Y3{SwCWv_HLo5BfzLidvb|jC~kf9KE);!7Th`gTj9DAOfN1u zYkt_F^oPy4N}9VdGsab0KIRto2HfGQW3tHx2&?HnT4u zc2;~PeVcDYi+g>5ESV`MEziqzsi}3BY4F={=x%&7^*38vD~9rlam>iBjiwBg%|=0k zslSJcxSXOj=W! z5F4e7Oqr}*pmLO^-N=Sw+?%DN)%C@G^oZ=%kczEDsj~{%SeINN8*&?0WtIt$vvVOXx=26 zcfG!-DA}6p;);XiW_BrQxfIJJt~Y=nzx)r7e)#VO>4D<6kUE|JSB&n$tZQR?5NUS6 zbT{T5aE6qCfVy*p|I0RIH-g>|EJPK>ysJ?viKpm~r{dQr3l(QBY)1!W-S zKOnh$b5qvK^Wcjg`ovFGF_Q1KLQlQ|#zMWSThrppi)lfZ<=66T_+{tB;TE^w{~3~Q zqHOsOl8%g`&LC`qpi5t#)8VR0*9fZFnF6QI-;8BGS3J|MGAR!xo;}tYZ@vGuzWrzDKr?{88A3z~Rwj1%sqadL{)Q&&ulJtX zQ!-6jNZoui6se6DQgGQ-c6m!aF}4f^1rH!k_bOBQF?;KNLn*V!ur_xL6xX;`Zn5EX zyi3bz)Z2CYV407xEfwP_axKX1u-pl2#koFY`PfWN5%F4GIV6b4wujssSy&Svu}S!b zJwWbfJTsF{m!yxloMFn=Jl+q24JE0l_m_&bFK#Qo4IOg$5I3v&dds=!-j?JvQ`gg+ z@&a?)H*-!Z;k70viZ!C}8@D&)`Jb^}kj-0_`$&r31(>Sex{J3%2oiUuujEd6JaG(e z$W$BF6e{6U&~<9a3e!VUJP*or{YVv)h(_^Pyx=tJ70$G>$aeT0+x8BJFr092*OC3w zD&eBnKwOfmWSydr9@FV=aBF$vUVr^uUtKezKru!CDbrrpv^CQ~2DgEh`-uDUmY3+x zTdDT9$%T*K4F}Rsi6`A$YR&bCt~|T3r~JPYOBvApOvdJv($!nduI7&!jew#3#Fh8M zYD78C(|+31s8vU@!H?uwR^iFv5xMX2&Rfpg62ka^S`1slI;aKSRN+m>zOWnZuKU(PWe+XLs8YHpd<@+R*f zVjREFeBMffz02f=j7w zzkkt+<793Ah)#-4UYWVr{5xhJ9Y?z>8_8xQ0waK$Y&P})dv06QCIn)~t=+n{ZZrt8 zyKNzX&k8g|X;%H!^NF!f=C_twf^^Kii~O~Pj%2c99dO-1xZy=+=(DTnz1!Qj%7%@a zRL>t>wRv>2y9%=jdwwoUhw?qJo;fN9>>rTVw_C^M+TNFFml;P18 z=+f+v0R2178K%AsQhz1g8iSb+^!#a{S0F790nfdr5sfO^-(CnXr4yvDh&9#wgKw&% z$Hj>MbHe6GK!JJ2Rf&~b_gbLZs?Y2v1nIY~svUDePe~2A1s=Tp&VLT~6949Lqe4Ru zK`_)i-EUgeU_95%^koKbBBwVKCiXqQ0aVILQaj^AQrn*Qpn=yP+O?lLg@rr6^vL3o z7&xt3an)I1(DMF2tQq{6k;>_0Dg1KJgu3jzO1JoCa>a+Deg7JS&+*h?g$ls#qqBkf zx%WR8t(SeWf2bYy&E)`d8c^G44QJNpXzr9tg{9vtA*34?cziY+mt7P6)3US64W0D_ zH|#Q&keDEgZ+x`ysZ&zQs!*)@3E8c^i3ST9w2(uqjik%#Ye)kAKEGho3?5)oT%}o= z&}QJrRLHLDHIj>}(q!fAlD10nX4m`LY$zoj^0rxX8=SqH4kumBQtJ~`Cf|oRYtZqm-?l1 zN_G7i#GT&frAfkBbH+v8)mfX3#zQK?vwE>&t+1R^f1JKg2T!4=#z)Vj^?N#q*4n_?zGp)t5U<6G0m;DKvuqU^*e#&Q@QfdLIb^5n36JPFKCosI)&t5-vZYh&my;I1Q993p;+f%clnNH`d zar&s}C9;RF4IV!q19S?JEk18KdN7Ny!9~F%D76K)r%kHhp_$PR;g6}EM8?-H zESBfG)a+T=i{V%{Ng-CzbNzAJX6@ZqdXiL2-w4n299$!Blsj3F)E;|kBrzx@htS-C z* zML6wcP#y{IoSO{ey!AbUc)(%IfA>QO;ykGzG-@M2W_cDPzExO$he3`{_<{NwI7)HG zCP(wa3hm6{+pv^vn40NuQJ=>O4JH;vT8h)PbI5wN3ngR3K{a&*P+}JvHE$`_yyP zt13&RU3-cr;&7j9e3Xl`;89y%!(BcnE~&y+v7~9z3ovuPU`P(Rim#|?HpXxqd1Jz4 z3(B}XDYBXfX!MNbIn4&^W)bpV(FEc z+MD@~uVD&D{cr_j1;j~0b)xi*Lm{M8Y^{NJPzMQcAnO;2jRUd)>1;b-rvQ$abAkij8H z`*3>s`nc$9e2m(&HzTO1ljmamZgyKqv(vKe>}*rAk$QmuXmBfc=fRnCI5YHLgZT>c zzrp;mUF~@UK4dyLSWAjb2t4;?gLdFXD@zG0%+J_bl zx=QFJsBYq^^Ny-Vut{736Jo;}OI-JeDlK-gE2~wEe zbpyL6{ctXZ^DSh29;a4tH?#@SczQD3oC6ZN60E;)U-_ zMKOJqoS-#V-%R%J=OjWKurs%7?tzrS*a?MxL`}8V%%SbEO-hx)7Rpbj>~$>B*kE!Q zQ;|MK{Q6RMmi;}QP$zmJM>%!ki;$4K1M0@S9dU~**hO{3b{;Qzih(u}@?CB;P2V`P zqXOgq=k=A-cd&1(Xp&9vAQMXP3E7EbHaR|U`=R}*$Z~jx%P0PZMQ_@3`|MYIXb`Q5)8qV^83FI^4qs16NqM!m+}pXnUX&*9tCRQsz3r1H9j^Imf0A1QVdfjkn&)1I z7c*Q)YEv{nTx?Z0RZe;Gax#A7t>!wD8);&G-yZ39xBS_(#lBP482`y*ALZKErVcrg zr)9R+!(ZOt1pLtX!Vf%;#r+F-z6G-#Yp07Gys@bX_c<#v>yJbrdw;Vq`0V%hli)c` zmX|T~|4xMdOX>gJy4io0-8=LT4ZS9w^~d*m#6_c`q6qRY!y-a2rw1+b5P5#qZfQ1E z?`Mq~15awLU*RN?)YSBpo#`{f@;zCLa^Y{M1gwveeu#JI>?`rU-;2e?CFL=}XTIF` zG!=JQsR_sPOeY(N>4j1hp=_ACiLqO|!T$vQl}9ZnT_2%DJbeHo;(h%N))Meo?17Wq z&6^)l){Knbv;TdtM+?xT$$6=f2Fv24?{oqwz>|`mUJ04@Tjs&ZPwFv|gzK{j0a}41 zO{9z)1sIaIn5ODKfhm6tUNkyLS%KoI^LjH9e0$T&pHbANOHK>YQ9k6#k0ogjq@zq_ zj{`C~_>kkVYOFMHdP?9gA~d?64p0u-_q3tQ?Oos@h8!AjYN=Nt5utV@VjLnit8W>Ktk4af|lyj2NX3?k;Hzwu5lJbCM92*TUi$mfU&#fukU&~kh5Wp;9shV-N=iZg{l?|w83>HHE1rti~^r-Db@gwDo^ z_mw$^k+uI)eP4#S9PdfSKQ4;XSpBDqCF05nOL08-^|oo_%0v#u8wNS9m}x)1(MUbMgRZ+ diff --git a/profiler/advisor/img/computation_2.png b/profiler/advisor/img/computation_2.png new file mode 100644 index 0000000000000000000000000000000000000000..097b87291929132704e6d5ea0ff4826fe2e5738c GIT binary patch literal 42333 zcmeFZcT`hb*EfpC^N5Im8Wp8Qj~-A_KvX)!W5WWX0)n(q1XN1sokWim5rt!+mmK8? z(n1qLD2WCT6$m0FKp;5*AwUQrKzjRj@O|F#-16P~jd7p%`|h~o{*kfw+F99a&ADdz z&AH}E+O@0JN?X*n$jQkmUAlPghMe3#Kyq>$JvMIwo;*05_X@ad2)|){Mh->LmL=!3!^k18JhqMQs4OnUh9@!2`v zx#Lzozt>iO+PwExy;Apt0(zTX(`uU=Zn8Z(x@NdtP;Y&rWGufw-7d%$>QuATezT?~ zjwybfDCEdyC3pWg{+#UUxx?!h+0`J;V7u&^doFjg>}vCi&Cjx{_IB-`WY-s)K^uU} z{Yyz{vgZ4P|E`};ogV{_Fef&1!C(4Z$>WjeRH2=Q0-d$SE}<@-=ud)WT}{gTxGEde zQ(ggNe3#Lpk}OZ?J&TpsH?x@llM&xK7=cRoL8UEI;(HFXM|WTyeyYq^(eV}h8y9rO2c+$ho&%TxZ)!A(xJE2_+qj$lqu zpG~KB2qc_(I(2l)2R{Rf4j?8!v^K_%g%7_Ar>>=S#StgetQtb^Nsj*=R#1k2SY)WN zMdjvdI)edK0h=FhB4BIbYrWn@{`j#H`scR5PdOHcbMK8n6^Ru1L5r9bn+7Q!Gj=>k zLqWdAwUWC%^?7-h{i6~)Z9RM1Jj*KPjtAlPtWf zFzeiyzE_~Tk4B@G&hh3>ttPrTt2p0aaI!sBLV^`C z{ExXInl(QbZRrli7*_3qIE0Se&MhY&n zPkO0o7vq>CcZyE#2^`78ETNV!TIwqkzeQ?cR`dOe^0?L?T-}%Qs#QS3$q95#A0S2* zt2c9xZt8b;svs;*P1`D= znR(zV4ZjiXG^kk3ywH0Uvezooy^Ua_d#LiKR)!LYn_3BLIhC+RD4{-hx^|#TRb4w6 zMR4_+`bb?XSxEfwtBkzf+Z~-IhpwE(wwe5S8EN45=xjxJcgF|W2&E6)ESNSXOt}9)#~MREd~H2X%E9qK#}Uc3Fkn10;x! z)9kJBt-A;1I~toOKpmZpXLa%!eIpoC!Bu>P3}d-J2>m*G{pSFA0lY4@vZM0^HDaYb`RSmw%3nB73lh;kE?YcOZr^UV z)ngZ;xA{g=coEZP@ma!2es&1LK{Y&mT5Ah08x{Uz9SVr*+Sw?~uL)DU&ml%zcJ(7o z_i8n-W``td930`Kvd>y~bd19=N<_quF}8yI2Z<;}Fcx-`bI;}aQipNl z@14nFI@L$#>Uq0WN8gO5F$s@_4ZzE11fQdlFd^`10zVq02k*nQw!3zSBi&2EwjQnX zfC+GfaVb^>#%jyQ>}fyTfDz2Z#_e$8{sS^0t=;wC@e!YWcm8lDq)Y!(oy>F9$vAqo z%zepu_NHI8ZF{PVs{&u!iF$@zG!wKL(y>(8IW4)h~94eR#?v*&Sq zc8yHc<=*G2iiwK)>n`&CaeaYQLdUK9$@hEy7yHxyYi~aG%z*W&45CRE%^IfySYVwg zy7x#OorvaD!eYj{j~}jQcuf_M*Gm7hUF|g$j^o?Le7@rGeDb)A0ktkor)(0ad|HF~ zI-a1FJcHJvU0UD)9Vt)G{aK`KR? zDRHxT8)$g`37w9Osl*OROgn%e0^57jmIq?UP3Akbs#Q0NnAS(ic$dSsH=c8MYp?d6I+BF2pi9%P3t{Do3K;@u4JL zM@5y?D$Ty*?)7?oId66@q@*c2O~%%@+E8AkRWDD?(t}mHjb7`l5(1NaEGbIP2}t+4 zK+xtQ#WLfJq=&>~Kb4Hv?{_-czS)6OlbWds*_(xELV{9PDUD7E5X!*Q5F=%W91*qlh0Y8v%^l$+6<8q$;u*e^OAll>BRq0%spGyTMIH#P35qpW3 z(?VLXdW0nF46riOCB0`f(HlM5|FG~ngGC1P+8_V%6?PN2Cu2{2PGRLZ*qK%* z(_xvvC!gRZbDf0FX5Sx|^%t6BV$8y@nWb$sycm6x?w0rSg)ZxR>1CY9Wu^J|>)6-F zwax-$W)UMGvu;+`quqH!pYM&Ahb(66(q4T6=s6JGhwL3kOg~sg zy+*+x9z-c0l-dE#+CSmaypY|jd^N?+-IzlD#%}Ya)lnQ9=%>^_?Xu$)nW$Ms(QT;} z1W@{kcIwCqyKVQd4>J@{%dTgBWX#j$w3WNvBk|tTA~mL`~8Wd3}e0GS`t`t z2Qe#qPt1u$@G~gRWSld%yhBWS7_^NTX( z<@e0VJHLJ){mX!5(<~j~rJ-~?x{WGg`=U1*y?TFwx|oPQ*VApcSfr?u@OFIU`F{<9^>n98^=T2qJkE%R9MGiGN|I-CM`~0hx^* z>U}HMw_3)7vyL%ng|TUmC}F{&er-U8YLGk&i?F}e zBO_0?ySCSqXdY>10^r7Cb1mhYgQ{WRyo|FC?T zi?MUDJ7D{*V5?;}2H%G*0l;*Qy8K|}FeO6#+y9oJC1C|m^0^iNVPksPm32w4j>?Te z(X0=ZCZ%D%YLh7d=ybnSYw%h*B3lvj@l2&j=UaT|{07g6Maz#(jtMv~j=c6r@ICQ0 zn+tX_v-u#xRgpLm_t!41rl2M(^om9BnY|h0y;@ItLRL^AiKh^AEp8RX(}pzo&)UUM zcXie~ugF|Y#L*VU&;^sIJ5!nIYnt485;TtQ+tF$32Kn6m+LchaRb?Xf3SB-KFe09( zszfTaw}Z~-I;U7(N!qNQs%odfPQKQ271`Qsa9s7!n{R*UiwT`@2AFRG(i&@qNgf_Z zU8MyELW;m@rj6UR4i)RQ&S%>++dG%aCOA7oul+SfzkwL%&6Fgk@Ut9Irxhi%VOy$0?;%6nD z`aGCWP}5&+o@qH1ZWu24J`l28x*%jdzE2Mk9H;Tx*va><5La(X<*w)!YE(WPlT3#_ z^{@k^bM!fh>1w6m=xWE*e!%QNktO_gQ(gazIc0Q{w}sXBnW*wF?cL?(f((r zd3*n}S{U6eg6AQ0AjUn_;DcLP2pR6{A+C-yx(n#d76k7ieyEY~5MUu(Uz;5wgJv?5 zdO6|5&wj2t%&o+oJ*dP(v0=;yXN`w(4sFtpJV8JzdoRWybc_M3uwV_QHgNj~T)#wq z;o4DDBu68~6ML-eVKldx@1Lrv3uA1zK!3&;APX^YYEn5trhsTttJf69z8&{Y`1h+d zr}UbX zvG?lUY$lduIzv}qO%DilYvP~C$nXQ`D7B!scbf;OzCI0aAoh;raIJ)D!5^-@JXL0_yVn&nK?uhUgVeHtQ$wN=@z`IjSC} zlzOw>%lnn!LF%D*CJ4U+6Vg`&+W$yP?n_QPQ3TzKg0uPgb3Vwv!DsfI6Vg*O?ak&; z(E@|Yu9c1m^W$oY{~+B?^bo~rG(f7rW@=hCoa6SABb-Dk2))y8OBB&n7Ht_wGH~Ht zFCI0-UKv@PI7QuPZImCvnJ8age&*`W-VL_b(BA#HZ1I~wpBp(vSeEo=CcT8jg#FUa z+0qq1Cz&d2%&R^+8E(c1^zT*4xW-X(S6Z>sgk~UbBmLCm!#O_ry3TzS9 z3JnJ`o1VWimJC8Gm6B9L#;#xk2H7eAdyd-WZq9keyEL*QRDm7Ef+CDy(x+6{0FB*IMiPOP+@naqUK%&F!SNfiIx zdf-#5283WoJ0Ezn#A)Z>@*0qmlcF^ph>Y`cH$MS{CI52msMU&F(&Mh~V#V{#>(Omh zhNc2Y_xxMuf27O$0UR0l!Fgx*e{WRwUzAV2_4)rKfHl88gZ~l0K9CV5M+yMi#_jXA zV;k4G(L}tQoYpU%HKzWmAA5;&TGr=2qw$8jdkO1T{kQm!?89wcy={eggU!oX$1*{xwZI z(3Nk%`nX*F!Z9Z1h(_zom)a&65OogNvyT_IGqZcwbsRabj{R{vlhSgSVC^#%dQr!VN7# zVk{T!veh}L1AU-JWtkH@`&<>lq_<5#kZ%45N?ta8mgP=uRB}N{=9=FyMcqV8TNy6( zoOdvZ@;o*!Jw+|iaoO)TGgLlX{sBlowH{9JF7YqQTh&oc4p?BQ2!SxLU0#LVAI}SM z^OL>q&SH@Ps|B$7>F*as09fpEBAjh#)Ky8cD+>Inb2vAOXn9AJ3g%#Rxuh)bJG(!Z%odoUT(nv(W9@A7`Fo%R=>( zu)au`M~@;l&j4=gq{aJss&55wkiu>UAo@01 z(nW@~2||L4Opf+iUW$3<<<0+V`dTg!Q_0q*IVS$7O<@FY#iHXyl6TVLz0Hw3dTtyp?0x5fu;o4t+hH88G-$bnt%M=5^_c-B8F(!L{ zWcHp}0?YJZlK;XQFOMQakmxAiWv#=d%WL2x$CjCX6V4)VmKb~^}*yaFI z$urh(?f$Ye4c0)NZEYlF8Vh4)hrEj#L&Oa3dAnbC=Nqhwxu+<;F|wSh{xv341!*?+ zb9Uy$-o&Hv*F@i^s(R6Gx3c+S>w-Abc*?Q7_01R0V5S8iCHoYpD_4_!?<*)^x%xX)Va9;LSZrk;)H0~ZG# zArc$p*Yn1K$gC|OBw`7(jKuNI^9EE9d$8<2SPIO7buLDwM^^uBik40Fp>p#9T}IfhIVB zd#<~Y^w_a>odbb2S;{}Qjf)QaPufmim$m=HW_DN}0P{8@5nOC74EnJZq6*J-_xBes zhckcJ{GK>b2A?JVOoRVXgt*1~K*J+n(pVYGF~O4OO@9%tQjSA1lZ&uTP0dzO zPi-}c^C8{uV@HFSnbmM9z}NRtvSKc6@IkW6@S>C*ijIlrnM)bX0bd+#e&xiY@t3gx(}@FH?HFkAg?kIU?(uLt+# z)+6(-YN9!LL)W30sl>jrr^o_7v7pT%u2>@g*3cg44-YW!!w%Aa@hBaMX8g(?I7%^1 znREO8Q=uQ<&m7Zo_grK`-1U@u@p%C!t-3!)g7wv*`bE=^Yx35d)ot^Znh%V3d&4QL zt;3Fnh_`Ku^QKH)np^#(w@X;>aNW*LsPxMe_Vp}dy#Rj~DMq91f~&9>o4RS?Xmuq< z)R+y$x*3t9-i%xOd5m)!bMl67l?Bg!3%HHv1I%l-VN4nvS9?obHG9ZPCY*-cO#VmS zzM#_aD9e3_=u`WOR4xcRlA`;wi)sd)w|dHN>hF7x_@3*RZ+7?hq$R7?Zen%MWexbb z3ZH^^$h-+w-_ZWki8iZ|3uaH`rPZfB552oXyrrggbg27}FG11RKuO%8^D~RBdgH>q zZ5$W&b{HDiYHV(E+*@%s@x!Ir1ig%7uBii2`Qb*c=tej0cu)~-&ZITputE0G?jO2|V^x|s~RA=PQWK zr>q0{yOWh!_nQ*8vf(9$#v2zaj`nfrVYf5=Z&eFOi|`6xv&eU&!KHc?)5WP5C$AkI zaq_nnXMS>5#Y{gY1es6VgQ{v`5*^~`2dv1PQ$wg1e9;kueEkMz9d!SQ5j}jcq-D0k zEC}_X0{(iRUUYFo_e=Y%yLFQSi(YzZc(Bg!aOa92;d-2Alx?guno<*#SKljI4o4Q~ zLn0DK1_k}T61BdzvWj~BCew1l)J$AW!x=wBT?_d&*Y!eUO+_%>r;@9gRhNGg)@mx@ zL;-YjcGzhXdgaRo_SO3wKmBx*;PcuTXJJEVz&?Nh)z$nWw&6zN8N}x2%Ef~-ahv`w zigevzgK-NUZ`O`^~;R0e;JLk?;Z%{;#6DcT)KxH zE3~PfJ=BtagDhY^!G>|c;cJq9+biG6XP-?Rk1Ay&d!pQXPxVCi^_adPJil8V{`F=~ z$&Q>dRLM=vtsbeC44>BE3S9VeGr*jWswen+TkpT2l3R%v zXf6G=>8L@DHLNnSC8ehBo_SH&lcC|4DA}7kx6S2%zWT%I!UC)zv)tn4|(P?^{8an~94gnxmz z^bB=>EiW-~D<5e+Fj41>HU2rEni3s6t37m9CpWFMi|$JpTgD21r8m+`B<84x3npQM zyZ4h`!)YYc@95?XQrtpmIJ|{fO^8GmWDhynOQX%+m}5(CceM_Zk=b;vJ#yh-_>;-F0BGv<_F*BxhW^e6 zkkoh*VAUJO9};i=eXVj6c8ov#cjI2S?urdD&^tq5?b`=uf*7#lH4VrIjF~fUv@hPV z3sC8-oKcHBeZ3qJ^GYqrZp@C}?6&}h&kbESYp$W!|1sp)A6Yx<(MmAuzJ3R)Ter}X zTi<_X=U;1p#GWO3lv>qyU%R7e-R9<)@|RXgebfwrW;2_A1(zOr&|#sF*2O6?n^UZf z>lrcwhbG#US6-Uwb88J3?{||9mdNLhrNbr0!9>&b}BI0hNj@7ySO-8M{NT=TgDIl&6u#l9gfsw@dpR~!f} z_3d?&zn(jQYPj0GGx_o@N4{B4B$tG0#nN9N!0g^~J&`?RO*47zgBm9F1;MJ^Fsr7j z^0r+@6}U=NpyC%Jpwx%UabVAYr8c75nUw>U@uIGKdwHW zFA8YFkvG*05F*Fk#SYSsR!c>vj73^{jhUfF)ntmzlG)MDo}HvBX>bvmaLO>sN@jTN zb7FIwe=pr6AK{DFx&L|7PyWEu*`&1Arv{;e3p%Oec1JgSzg=8aI4A*N)Y^9KB_n+2 zn>S#{n!_&!*Y#}HkTU4|Fe$C7P1|PYK^fEv)cJpP$-w{P0ziVOCmOd~f>SD*gWG-v{6Ah1i;lq~F|7ONNo6)OMPMRPrYHg;JC zdF5J?((?Y}juV*6gX*_!D+X|N;2YVOa_;eFHVI3{xX(A&k@VMJZF(L4>+ZhzO~fqr zsMfz>F~#C5lJxMn0CY$bd#4>f3GH5GIyVPeg{r>Njwp_qj^ zSjj0b=>u+ZKqifdn{bc+vpikbN<dDb^{bt)bFr5cwnS^j#8nx$W$Nphv2fs&(-lI8pk zaBuK*Sd-Xm>eccH$wSqna43=_S#D`n`IY{oq~MeS=tP#`qd(}}deUYw$yY@j*(_ngw$uo!OC?jYItVbL_UM)arpjX!!NF$4T96F4o+wj_1(|tc)!ch!>ofc-7RW%vgJlIas<+KxpM% zLObXhN{GGBaU*vOdYf4&H zJV4>wCLGg|S`jbRYhyP#4b$5NA*D7@XO)0qGe^(-m#ae53=6njgiV*F?`73g;Dc-x8P&(Wz)Q1O>o zXUj@TBexAm40IRUv{9G1$;9^R!%ou3m#Q#nByy2}V|^`H;0wvC{!~W#Ji~y+fbtr~ zLT}FG4%>!SYxG#27Z9l`Rxv@Nh~cSb#jZwnws9&&e4EaT`Ys;!KV(y=*#T`cZL3J! z3TriLt4rK^MIWN!txcmZMR>#(!Fl7$6-1oHxOIPgRB4-pe8asU`Bo7DHBW>uk6aOe zw*s;os4rVU?KB8EcDS{JjqyFAGO6a_s?rS1RwDl8U|L5;C3zd_QSsEh2pnvQ(LK_! zYk;!@x9cBCZw|S|uqDfnW7?f4JIO9bSCem<3Ywf^#?4AaWc3`G^gE7iPOUgA-vQQZ^$->kIXNF-4dhYcmUm~wnz0D>ydpc>BIQjE z<#sObD>TKJvqYO|;GPyBe?^#>Bif0C-t_ckBNr_oxbI0dIBtK~ej?gRflZ zJ_HriWYikOOp$Gd4(&Q_bveEWgX{6xcIo_@}fU?htj~6 zV$YcA;bE&m<|@62XbLqRvI`=I@W$s_H`XY@$_d1}Xd;cZOFp>zY>#1WMNTGXdve;0 z8I#{|-h`->wZJH9KGmkT<;y~CT&ccIf-n%`7~7MnqmN3Xm;5o;#l)OaN(`T(A2&o) zuv0?VI#R=lKNM777}Raeg}Dy(S>wUyr~z&%>MGENL$2HZ6#JeKGYfpmH?H6_+HivJ zXWP|PaIwc3j?~6fh%>l@gj+77Hr4>u!S=lN^QJlO@NmF9jtNUDR&PzhLs;(uSBewe zV|;eKF6Ip`FLW6_=HAoPf~2W4x9UK%~F zUIk7kn(mbRZD||hkhl}Ap!$vZr>>jy1M%o=ALNTUQY%$aU#Uj?J;Wr3cY9}`qp#l9 zq%;;NF#s&;@hifo6?91MwUfWjVrd7;`k$2X%VKVisJ@F;x8+h1oAYNO?nFklI9XtZ)caF-sfgW8k zsN1Eyi!9+1qeecbY{=prsE_Kl!f>^yQ%-o39 zCgnv~TC6==PUVu-;LL|az=F2(GH{#~^A*X$3l;YBX+6;h9bYM(nq3~z|9)2TgKDS~ zOW3wEh*`moohv9r;(I#A&B>Z@Ci$?J4nTb6)!f|jmxrf*lkmC=r?*wpdS0BJd2!gt!du?gcU*_ynJT{z78icBV++qe|?$Y^)eOS{*{ zKv52M@k*(Rz6RT*r!Q{bq~VN&_`X{MD#Z8M^bU92zw70#Hv7>os|y@(tOu0XY05AD zgg^ahSY2Bi61@6txgPDI4(E2DQ|Q3@yc7Mp6;xT)XLJCrvQ~^<6P{_GdNp$wg~1#7-rmr=LIMg`qzw&i+&A!zp-l4?X#%M zdNzx(E_K&L3v6!zAvG=ZJyFSaTJEI8cMg?#68E!5jQA9FAzs>o!GZ;zyoatHs+O1$ zQ8Qr$Sor6|Y{y6NgMS&@Of+$wkbo>KG7yM9Z+@P_D-Spy@KD74KEl-d$|;o#(EAyv z?gv0g&rd`J-^)mRanuArY1Qs^J0kRK7F1u)HJoKPe25f%hwg~lxeB+3Vq zcTFhdzdF0Rt$efg0-KX6LmltU0E}LTs?0G0hw7nKv#7t|J41yX^d&-)Pc)Rmf9uEj zTpRewaWBGJiA3V3SIeoXH9@FK^rRaS1~5_&EK}s8w1-r#gm;S1R7!7K3!)GF8NXlI92IwV z6yd0syST{{h7+bNe=(U$Z(mfZE_?Q|_>X&tt>!DPHGx)_^AgY=YX^R9oV{Czro`K; zozwPE8woZaP5LDnAfKB`)F2=t-ibSJu3TRm1uV*%Imky}NnZ%-Q%M`LlUur$>s-i1 zlXpW-jjq%alPH9L)a<=vK%DM!n*8JE7d@$It8yoC$ejL#`Z(q}a1G@dATq_=Nma?o z#2ZG_T@9-q5Y6Vrl@sUrtL%3WH`d7P-RbZ1U#3z$>e==&^NZtY6m#%a93^h5#V5=X zFkU~@WbBE0rubIObUb;d2VbDQFOG$Xw~ZM#58muV(M&aYcJ&eG!*cdUnDfsu0KSeP zv8mh)_RnVBFeRKYgQ8IL7XE;b=A=5c`&+uHBSZ}+XQs2~RV;SSkP&|aUIE0iHg~0@ z8Iz=rFcOaUhMbd<Zh?KEC%IyZr`aW%(qSSqP^?(13u|}M|c;i9+L~#M0J_YUZ zWyd0bWqh9v0z%4v0K>@5$tLi>;%4fb2{c)4tjT)yi>%|<(ZThqmH+#lzF&ahtE?78bf~yu z>rScI8nT5VC3Er7xYeTNA{e4I!DW3HHkUN$<-K2dXFAjF=$S60c&&_D2I0%<;ec7K z%4W4QJ<~$c*AJQec?CdV|8;jS`%HIVwP4%uMowfY zQhPFK*#*K+#>G`RmfWX4W4s1-zCW@za)0cX4Y-n#st;-3+*b^@ zD72vbd?zze{335g9^Y9bxngoFsB(E|nu^qt2F4)v?txphNrzNBAeMQXQlN+_GK;bM zssu+Zfg&tor&uhVjEH}}`he`!^&|a-7081L3lpjxLL9tQ`71V(+C%L>L0?OTOQ+Ld zC+Sw~;pKtF7f-oKK6X}n3GOi#g(j=~5c+37vS)7dGAq0<+Tdo-VnS|=KkpOKF1Z`| z7^f08I}c0hCL|tCIA~VdcNDV6yEwM(7A~<0YP!RU_4EXFVz5tuVdfUPa@Ga#W`QiM z?%~HYmnqZql2RXo-T9X_S`rUf)bb969t8_D6`*a`s-2QN=ehf@)~R_-b$3|fwkMaG zs!r{TTt8r8CFokd<{%9Ms9wGPXLCZHOt5bj$=-W~zmS9Xag13TDxosJ%!j_uRbj8Z zf@+KuDM1rXy8AzV1r+gAa)C1v2~m>ci<(LxusAFx;W(pQ5vmLpFIb#B25%RntLDd5 zLa^hyKQuU< zX=S|M$H&4Z!D52s;{W*IPL0*IKrpng3OxxNjGlGpUDoVmyF~Hx-gI;x(pW64=>|Ww z`iSKvFWJ77slVs&dNR;mRQG+Zyz$0b$K`ay^{jj5^_ot`BY? z1!+ymECZH0^?`zyJIOhhH#DedTGe`UJ;uedgg1y%@bt<%NIMidycPq5*SZmgBms>Am?+Wf=cFJiMy zY%D+-wlP1|K!z&e3Ya|PWn`gS6rZ68nx6LfSIHPm*l#hU{>w}k)e}0$ji$Z6MlZ)g zH%frhv4NN}y+Q>YtfXt%X@JWH&d&1I%0ujeab*Ap$ITf&qYqz^@d77^x)pr3hx%&O ze=t+o9~%(1u)>1lB%S^**x`0wC~HdM7`T-4PkDSMlbaWvAI9yWO5o4OPOPTQIThj7 zbwhP9{w&VgI3N7eVGDMQ2?r*H@uXR+5G^udA)SgWwn4s5NAP?BLoV8QFj+A(#ja5B zeX3+EI>l8QbijfWVIkF}){K)rk-p=jUCDc^i7s+0IB_a4e6b-`>I3RgU434r;vX5YSm#}2-eFBB zzC>?=Z(+Tlj`IBkMBq%0box6bnQfhEZTF|uUahzU$<8^yFS0|HwF|O%XD>SokpVA{ zT}-|BE%U6?21>{rm$mG~_^L~r2W*%$Tk@%Uwxcu88n;L5F}9(u-`-AgrTG{deu4A3 zvnIhRfrWPt;Xe&|ZPw(Qqo2Tr;y(*C-W0biKP?TO&O8Yj$6ck;DZGSLhpGPy0C+0a^WlD75o&6ac+Zwk`GQV!(Rq5G!FDY5Pbrav8P(wYRoR(app@4{P z;$43>246@daO0Qo)6@sunNJYC#^0pJX;a--w3IU4?=WcdIvvw?QHY>8{tV4-SFs6r zSUgDCoM{!i_8~KmofC}%cT8iC-A|={uGsHsqtwbb0aJD%|UsjFt{>cA&Z1g{h z$^L%~^*@fG{)_*eV=;8C0lAK&fe=^g{0smykIHR)mnAp-<5O#vlxBJI_CK4l>;G3D z#ebb-D(S}!q}P+9qy6}r(a|x`TgAF~XyRdG#3g9U0k`YH%yw$=L_+RqS4`^6c(3l# zW|f(2cgOKdO^=$wxbbrtx-CI2)b{AIRaFGjuftb-gIB5hdb+21Dnq|7n=~aPq%Fn5 z+)_k_FB}qwwiN2CjelvcxCawtO|!1VK}VcER^L0IZ(9(5e4Xp@Ur*;cw|sy2z0cTS ztl;v)+fDbI!e9%U5j7#XZ;Ni5-_?!Ak)whJw71GfwDyGDiiYJnX9-_F?EErChW9je zHx1rga;yZ*Ms|=+ID<|=V^pu?j-EZ+BtPusJz?_3*sU_|#0{}N^T>o;gUh?D`0GP@ z+Cg6e1@TVCHbYnIi{53mLN(vBMP;96fNI!$j_4a@T5r=gR{`?IFpID9%JEh8L1ji% zciBc=OxUTx?0aYF*CHZ+@6hn_R#6($dZ`3b9C(k3d}vRpVxQc^{ySoS(Cz$>I=3Ja zaL)GwYQrmi@Ttt-5BiqFfI`y@2kv6nlrHv(~wwM+Me@#f>w;bpmz1L0ht!z@wIY6d-efLVx6O=P~#r4v} z%{6q^aC}=$k^4K>OC*DDQBBOl=3z5m8eiRYb$%>N;b9H(C_&*@ia0@4h33{jc*;&m z3q3p7dj+<5q_4o-7#Doo?CF_TlXLFdQE$~=ucTCx-VYf&ICUyf&tR^W9xs?zKS;|g z|9aG(*cFIm_=J=+aqsAIU%zwzjAfo+TnToG0}*eioGd6nqvleMM28!lm{{8FJ8-pL zEL#?6yn~V&o$nK5?+=fFsFur@v5EX}34dxo=RHni8m@wi#r;YNm86{DmnFb2pd}23u=kU6$lbWyE%D zKXrNjsXO|?U9}Ivrc-y>VJur;;hqa>9kRl+%CK|BEhWa)Zoz=KIShF?Jm(x4ybx73 zGoCTGsVaPM_+8>z#x>fentT40jJ2(Y8eatJ1kEfF+hF#?@$JPFp%!ZV_ll=I@BA@M zo^OM`^j3~n#A39YO4{?;gYkqjjYYkQ{*J|OZ5OZ7QN+cP8q{m61Far@;}=O}bWoH| z(66EDBNG9~-Z6V#`?hjD^X?W!Roqpr@SWml4=+|$BZ|(HDt&qPqA#q2n^6?S+p$BN z^YstLZ|wVx@eKoa?=SeC*bURMU6Bqn( zoIV>|8a_Cx4iI@O;)Q+r_0oQ9{4=zd{pE!1&0ukhMyLQX#>*kwRg6Wr04ckkUC8U8sz28jwUCxM@{xca&wSIL;s{>!Y|M%*BSlH)H3lj z+O3Zg>S|6l?!8S~8mU`*$HaH;`g1?&VLhyuhp;a(Ua5yo-7L9D(fIh(5!hK_1VyX& zOhj!e^M$#o3d^O0$!$i1Va9IyI<)&WM!1+SY5(ldlW&;e2Mic)$^H{ zWuuEiT|I&j#uIj1K=F4zasCuMEzU9W8z?Y;7Q5PAVVoERO^*y$@+>Dgk379}!Hq2N zJ6fDGX2;~bRXex)P_cWCx<~Oyd6WEk! zqaGu=F}G-v9mlLY5p+8YXBh&sY7DBO1;ji3-t@eA^2+!_z+4<1B!B<-&C%Q-4yhJY zb^wZM(RU7vZ-ZS#4_CFGBbhoAfNJl>SHOY^$W;S!b=Br?tJKnLHV$Lc%GF!Gcu%@* zbY3tr=(C-i{48lPPP!W_*z}oY6i^*T+ti7mn^-cZXDpR{m%3}$x*5a ztBH24E-1G#_n<6C&Njk2-@OO~15)RNvnBz5`xIf|{BHX&#W8;SBzjU4?d8K52_ZS$ zn=H@{`c{=NNGk9n0VnWwKaXl_FS;XnU)Pm*u6^mExGx7rJlvWEP#5&{Z0n65)GbX` zM0vku1l9}=*rS?*bV(C_m;_4gHnh7h#<2UF2kNB5$D&_lH2Um{fGv3X-_;$vF`#5m z+f=_>smj$jxz_bAAa}(N_P19E$0Z!ZrQv-P#8lPRpkJ$$Ws11B@^tQ}H>K6zy`NYr zoP_F0z~rH9B@?d#;Ag*hO|4W&kBS|XIImCfrytJ?nc zH|73uyZiV5$w!F=LRIn6PP0fzZSYLMS(I@2;7v8PCUG9`AB$ek|M;FAY`?7>0{n@A zkeVexpZjo;Wasf<-*ZYnKAHaZDm*G)re^^=eKe>uG-#Mp!Pxfj?p30fKJHu`Q_6LG zR&cdp&=9AV&gu_LQ|X-K;cM?)Kdo^}#h;vtcpE?(2yREMu!A!aO zpd~)8#6uPG4|!lu7ARDGXv;S>cvRfJ6n)n41IBg0Kv2}e>HD+rc1_zIy5A*iPNvTP zG#Jq_vGXRBKK6nw=V!1d;V0F)(6SucaT0Osg#P63SCpaWYiv%jPCYbF?W#0KyChU{ zseGcBRhj4Z+^jV`?s}ildG?+M^x31FCFHtERSP*gK1_&msf(g|y!`{?=V8#>b+_y* z#6B?|J{+ zzmmjpv6?9+JQ*CU`7G2fkWexabn;AXLW`YPHHx(KhoLK(hXhU#*SJS29!=SPYwl)LU}j{wdOP1HC+fLuy)||0Buyt z#GuI@hjp--q3bL|f_jg_o@cKsL*)}Bb)FLgG`=^#dYqCEi3lK88Q}|g9kUR}cQ1?Q z3Y9%W+}6sUFY)>%MPVRLsM$B@%Hlk&UA+H)5J-kq6)N2lOXS3c6^_lHZ@5O%1Ra!*R z`qg`kMlVjgXB#2m16OVFlV3y(JZ#EW-G-R>a@=<1s5qzxwk3RM=Rpg_+z1ULS80vb zxZ_>MBZSu0-C+f2Zu@-gsf`1#hAj>PSNrl&aq76Io(0cFQd_v(#n{xcj+j129lpRH z?MYIdh8w>L)6Y1GA65zb>sd&~`fLVs;68z}Ds8q}AE`lDxR!S9o-TwkwIUtNqoL7r zauixF{H5l*A{U3Uc8}HAjeFUy8Lsx>fUVGsb)OHXi=ZAOVL;DE%|d<{TE;u(5!4cB zgMat&ybETOd}}(mcl6enB)s-P7-%)u3xm$Vd4q+g18l8VoaaK9%tdTHCbqZu%D;+w zZ?Sc|OCJC(C-5P@aL|1{#39ketGA%7Rpw4}iU(R3^)kj3wXf>SG{J97Uj{fM6So#5 z#OMg0z$ZtS_sCvmH^UvfS68H>p_)mCS8(1hQb*UzJ6 z+Ea3)WCX$ji@rfUmb>Kx>hK$g>9ZspIXSkkwx z^8s%Lk|$HHyzHmej{&#PW=hSJdiq_s*%(2cT4UJYE3YbhFV*EkWM0(=ysG8uO&e9g zg-b5wgx*f_(50Y(0Gvo8;%5E-T;EGAXQer4Wlusi-&<{qJm+yaq{uJo!oh@Hk>C^d z6c#>o(ckHMQ*&I^`K$VF88@tkbYdPgHT)`uOp#GN%Y|+ZU&6H821+B}De^wkjSF-C z{l;yP9|zN@H3RgI%T-`+TrXvgJf?$utT~EwU5JBz7MIiy%Q6?Ck)VF1!~n|`ctBY# z;heqrv`-Z=H;gDVZ0LcgwGx*z!GJSG-Mnoc4}n4bKBVdZq}xL20Q$j1z0OOo3+Tm! zVmnJ%Rn3_DZFWp^U`CLge2&uiI;tPIX7*xkyk#L zcjvyopsru7G4W^=dVDgk3^q#`vKx9d(~Z;` zQ@)bQ_;AdjXeo#RkMlFAO)-aXC8O+Jf3|3tH*w3>!BF0E2vNl3cqp>D57Xw$GzpdG zwFZ%G{p{MrJwKJbx`6d=nw6@-A-r>@vDqhx{U4}s^KvS~Z$|efBWuElC{@9b6%sf0 z!b|0&>$+E3Xd*Yle`f!sZ4B&%K0ENwT|ARfEf44i);$L&Bi}VyaJ`PBR}0mP9N|M} z_S%Igg{(QvX?a@3KWSF*qgfRYTx(^fEqIVSc)a^h=3RsrMP;jehtBwPN+B_u+{M;3 z<5s9Gi<2c?`u3;Hu!XS){&YUtljz3E$$#>N1*I z5hrDr9od|8deE~S6CX>NWojlY3`=>3nRRMs%saw zEtR7TxIQ%{K&B4>6FAfJ018!RCvqu19%^5_i+WouW& z4&dr(TvDfWH#^9#zmt)5FB=gEdo=4ZUE@6Sy z-K#zCt&Y(z;%m!Ora}piQ>r0yNwnM5pw)2(+ES!ryX5*DW!O1kVETrg>L|T9p$uYF zVQ8x!DH7i-R^??et3jvT)UWLCQyy!$IZ2dgJuY6CVo)2Pl#k|}l6W}jOcVjuNFq_; znBzO6XR#Sqe{RQki(U!z9_*zkMvrYSmRI*}b6h&CtfBgLTBs*<@a>Ez&#b}#Ey;U6 zyK9R~lvn`$rUG-0r8TIfhO|)E5}tE6WX`!SETyLup;nbtS8r>ZOt^Un#nFu>1aPM$ z@!f5k&9xYVI-N#**tsdQ0Sw^1BiqR1kL~&yx+H8{D4apo&Ee+^bY3V<$UV8-Plftc z@HA5H`*^gg8`jz6!1nHC#ggm?6X*CuP?L%WnM;0~aqGDUuJktgSS%m~82 zA;`OToem}w7Dxvi;m5?ON5v1m!U5)wo56tr{Jh!Kxwr5!vMo#KnUPxd^pKr`sWXib zCF|{$mG_X`mYrvTu!-jY zBl^ghc^5L)Y{yTads$H(UTZN?1x~dsJUNvDzHdZOk3kI7j+Pf?To`bw5{-44>|NK$ z7U1ruTM+vxdK&ssmcw>|S#4CH+iI z;}Zj)!9uR6@Kkrs3otfu-517Ghti~m*eo>MVX5hWQSVpDVNjKKL*E16nW3_@J9VB( zy~Ft+)rsJmh^ojixk93neG%N4Q64=WG5sNn0v?%M{dS3#oD0@o;5gpWW4o-0=u=LU zv~4z|Mcf~v-S>7N6Y*n`fmmCX@Y`xSqnEC}gerIC#|Lggu)zr&CZ%9Hn%CG?z7;#8JI+@0X%;!h%>RFhzr15T$wgG zi-P;NgaCs0E!UO>%vp5-u)~D9i5<-2W5B@6fMndw9q`2GoY}7g-(L`pLFI-r&?i`? zpk3q)zbmf6!olJ5G<&N(AY1R!H~5m%Ttrlaf^gRK>n`?*4URIO_d>kh&;LsLSy-i1 zLBvt#V%D=m=`kGb(W*(>>`M>Xvbhg0GECB}RX%-LHQ}MMPcVs*f0%slFl$$f)6G4TW0{ zPs#SiM2)%KM8v#hD0V*!$Jk$>3UoP&19Q*0<_@4+m`4 zpDXwUUldvnI%j`*#dat}XuGwmv0lOND^KkVWm$$_Bo z>9cWHgKJ4OMj51hNlWj{G8fmLF|QmP?#t|yc%Ftw@8tq5+1t|?YNmQ3C8!$GJFcVT z!3 zx@A;3#JusnDw4yhJP6Z#2niIHcwoP&YeSi{HB{9(0n9f}xv_;wi&ZN14RM68d?i`R zu&>|Qh4_qdCn~OBpS`R!)JtXL#I9bs%*=W1a1=Ys8M%|OwT!T7F zEm0eHgk~&ffR%HmZR_||2ZChcfX@7U2{lbP;4BTb)tZhb7h<57SG?M3Ot<2AYIYDg@t)*Z6Ho2&pTuP_AtoY64{?2aU}vui>4$ECs?NmP0VR9sBO}n7;$-7+5zE&aZ_ok%%1l^sqbpqE8{uA^pKEVy(*XTU4c+JV2d$ ztZVCESEUBC!U4XxtW^P)!?X%fhwE=G_?=_+vPS$0w4Ux$7dioB3OnxMB059bY_b z;?C}IbQBsusQ$Bg$;hYIV%Lp?4V17O&jB#()|X$~wQc$4f5C9-w|z6Y@HYy6 zeEYVGD>s000gekuQ32>xRj=+tF4>>|4<>$E_6HvjrD7MyOM3lV?v#IK;Lq!uV;T8Q z+Nyg9HmN{~Ph}7D#l44%h?a|SySWVyh}xN?P0Q@p$@UgdVF5Amft*`?I?t?PUIFMg z1a}|^MaITb*LhFSZ?9pYaTWe@Gk}8_;7gYPv_MY7aOJlc2>;c&SUpM&CEv}JkbSsk zEDnXseMs;z*>BV{fH=ODP7eeRd8_=8&bhww2cPYJ(>>1uBm})wSj(4qm#C;}B5q+K zC0D3#qLBJe?^mXJvht5s9{D{hq*{=e%c0@SVI0$^}@g!de* z=wyDPJVviGG%EeK7|M3Wk8nQR_j7xV%?28aklFic)X6{pgFW+~*XRFS@~3^&LBzfN zv9{aUQij_&6g|_kV1GdF_OE&RW;6+^TMCD}N%QYEO->ac)W!Brd``IBHeNpRSmZ{B zSd0&ZocUeIMC+KSE<__MY^Czocz)tuDK7fy^|${W_(0$M&lygA5Xgq=Y%gL_JUeUT zoHP>&9mY!a_l%+(9)W=zUXyIiJ(RLE5nvdYi-B1bc%(VU%NxyYYg(YmUJecX!QAm+8K_WD>bsEw*KhXeKoh+oa^l1n7oV=|a2=dbpxmYajh;1<}0M_5Wqer zQfK(^E;$mWKRmu(9W?!Z$>2M@JUz{;=`_UY&&$9kjEo$rGq<7)6dy&O%tbu_U=@YZ zqzp@YA!`6Om=3@Fvj9ZXei3%(mM^}QIL0V+@Hdh`^6f<54}N{{I`ty+WVq>e&K-BL z&4OQ+q!s||nVAD^zy5R3p_19%eKyTB{ZvWMGh2agUKXReK1*s9VMC^~VWxW&?^!*n z9s0&yE3@tXs9z_{cx&Wl`BjsgkXM>Hk2HhjPm`od(cAJ?EyOXQY8;X{ihTpQz zt#T5O62t&$CewLRx6S@N&{k!pp7%;pLJI$IOt;Fj3*B5qmiSV>4f9If&Yd11Qj@0H zn2xq{&b1-v$>Fd>w@PnBBH9x^B4=CM@=^E;i=(iA1M4u_WBl=tu)nFC>;A1;M@BtfLR~Qs&r(al`xx1@(_OSz**Z3Z*VwcUe*zlppKC0_MRz!?>RQ zx)jas4-w|-bAI<)P#--r+@mR~W+M7p@e6x1C&BLQ>8Q)mSyS7ejwNagHUhtj9@~r# z>K_1&Mqy}4IAi-dCo!J&IpQenrg_58kfNXy{cQ3)eED9Mj1q))J-l#bwC1tqv&59~l zLS$Ta{Y<%#($`F_KR@gMX21?mw$M*wjY5}^(yC?E_SK0&cHrf5x0Y4F7|8Zx!lhLQ z>z8};bzJ0Kf)uV|jN5PY%MZ=ZK@vlO#&+axs3XdhbHomAC0^%^1i=#E0b29R&hETJ zHn95u2w32Q0QZnn-29q_X~z&r`@}!tL4yPsHO+YAGj&QdW$VhOYCO}&msBZYD@I$= zh}qBMlB(i@LO8?IN*|tpidQ+`DS`{;lWS1Rg1QTyQiiWm%f=GB&0v&e$U~Xb;;~{v z`D>hms|}PXd!ONB;-Z=D}S$^HBLk+KP0`rAG-DB%nzLH4UI$*=BXK9Yo)0>;tgjNsrRN ztvpX-DYNPT@*>#sGtuXRv4R>P2`pIPpMzw71*D^kZAC`<;}|D{rO5;k;Y2u z=92vq)YE%{B4Ik4oH^zJI5*@%Mn1W`)u{;I?ysbHN}F{*7J=Xw|FPbYQI{*TyBnpw zOBtypsqWyK(J<2<>xQ|PK^@sV>&sqaB6n?T5It%5uEUpk^3$5HVz#s7-ah7x{lay# zG2VUT>WoNmM1%3g;YV0*Xi0wjSe#}@9(yUvn3Z;MDNsqo+}pm&zV6(LS(?OB&1s{f znicR5pbUtc_n&X)pBdbOcG;&gFo+t+Diu6U)wElL1gk>jg~WsElrzVc7~))9-xWUq z>z&KBhG(AXYPN2Ho=Wb0uLUhHKsx%yt>grKPAm|_62T+5)dXh^h`D#Od717h@c}Sn zIxuHqP+lD|u?|ywH9DIo7uFqaxW9W^3?GLLh?)k65P7t*Z`zLxIqdDQrRSYv7VcAO zpS3xh^b|jE!GLdX4==f$p8ipD#}HC~>6>Ei{?2T5yPFGLk{#%&+gEuQVsfiWYky9> z16oo}a35YNx0~95w{6{r#G^I@|c4XsV?UXMW`4*|v(W)~@G- zUu{(ZmJ_+kVr;2(_o1VrmM3|A{?=+rEHEFtj9W^-M2nfk?wFlDL*WPoX8@s?V&X1VcSuarxq4A|@TMLrD94H;&OOSST5YSyVjs5w z=BfXs=;|^F!{5&Mgh7?XZe55j@OK>HdfLk>RSuEnbmq9p2KoRv1lclei@A34F4;C1 zt_2rD`+id2OjDl5wt54i>)Bz=rVD=|yLMbDI_vg9zHxMO{gWorV6j(DM?@nIw2I3xE4m%i)f73Py_8HQ>uh}$oPh=cC;iQq}nq?%dw`>4&b&+;f_n)2*75n8~~f zgo^@DOAVpsI^P9omPAK|;hZk#0^(slJi3d~r(K-hM@#;*!T?|(K2x8@l6hoS9p4$3 zWMGM(0de$)kyw`OR>mVoGakgUgeLIBGpB$>maaUsr19PH!%E4!wV@y&SCrB+ufP-6 zov`Uhfp7N!-v9L%*mNGBAorFAnwxuaG50}rDNv8*I0gn@Bi|dTRbu7yh>eqt&r`=R zOS0!)hl8wn48+_wjo9?^lDW5#S$pSph>V!c%_@j|>j1D0y>`)Ps+vD7_z0}|?z^${ z5*HAiZezSx;gagbJB)K|<5@sA2B2vi;qBMItxEvih6V?b2MUJYoPhP@=}Bw8`*<#6 zM+9tB?UFfAJe}KLr#sblphCci1Yd+09Op`n&I+j*JuPE&s`)_c3G^$t9Oy-lb_UBs z1*Nq5m}0pEjp6=1Mw@JtBLm<^H9FT=XS`(%o#GkS-tAGPodAmDc%Oj-zlQFd59^ov22 z#Gc1(5Ic}-t(}Z^L>g^d^bQ!Y}27av(+`lXW>VhNtPwFCMl4^xY|M|-GJyo4P4n3TVWtdun{=?zY(bLtGGq=U1ovtprppj1Sbs)Ve*v@i*BZq^>=}!8HFwg-UwcK7=^~_q2yIy)1eYj7 zu0^u$&wB+qts%<;2bg06V%p^(=vcV+K{j8&aigi#;%Th=Yd&1>`;yZNcCJnj92Cat zX2w;5@0ccpy}zuh+E7uT#75BOB!m6gKZm{_EHxW5xqQn5YZ*Lf2NKZ3w?JoWtEPza z*24IEc;|Lxsv@E#n;!SXC$i-Y?W?gQaFhd|Xro6A&I%i_NORk@iFLZ~m*CNBgtB|Y zIxz7936{lzST1^bH%`+T4<#_TF*`JQ-rcKY)M5i0a2Zu+Uz7s&6iXW#sp7_VsGM#o zPcv=ijG0`ahnGy_1GnBcYM}=lnuGJ_CCk6oFilQUbS{grMU?>z+V6mV5%g4MsWr$i zf2L8;C$;P$$Hdxp*tZj2K4+oBI*Nxd+{PxtMhFp~y3@(#?92zB-r-fa8MiiyGL?1V+--;)p%y8nosPqmA!|NDXyV zgEPKM1J)K9j#K)glBWU3F4=RAHh8ASQix4&ary;z$ki$%z=F4IITSgNSNc4!$Y!g& z_%_ZXHB(3pF%$5$3^i4?1~wh_nGG+PxQW^BqsDoG6EwqlaNuMHqi-cFt4!CIYqTZ= z_JQY!%c0DSY|~U!1aA!?q?e*nkvKyho8=#&WcPVs&6;%h7t~JY)ICPgJ>Vdey^40s zOrub+nEMm{Or+Ge%p5+?z zG0`t;X$y&%jy#=D5Sj8dUJ&l+sCaY8b|Y_<<6OKohZTd+0zw)9Jk62+zpP@_$Fklt9d4tyPuSq2FsQV zzMnY#>#V5S0-@BZNH7FASR`?E@gLgScsv#>jAo4F3z!V=i@AmCGm}`%28VvMJWXzmKl*I2A)G0u=>h^DAOQch-p- zz04N1>>HY9{pggYYU9f-QhIa^oN;k=q+fF!m{Crfp(bLAu*hDr2NvgNIdHh8GabXVa#gx<}n%3Bs@Y)(qEL|RmP0gD04 z#h%wq-CG>E793h`wmg4owtp=dXPpHvd^K7{2TPH`!-S8`i+Nrr&?97knom6H>sSWl zfK*h18QnRqZo0gPs^Lt=VKLjJ<*9Ch%SrHH6FfrX$7IzVYeY7|eG@YHdCa;gzLff= znKbV*yaj`&i5fKIgk+X2R4u%ig|hFFcA8~L*2E<&qafRFmO+6S14aZQ(Y?1;kz(U8 z65!uQcJFVDV~(O<_2;;Y(4wE`X%wdlz6AlHu01aR8+S#SSU>`&&1ih6a{8kkJ6kR; zDOg+2U_bqHf24=wtVl&a%}&I?`z}9R3BmgL&Brm)d$s_G*u8W^aOi-yH=8}Lc$(<1 z9~>b>cQR_;BX7#V6gj5;C=k1pCanjST|`Xx`M{M_t4K47^5^j%z}$Twu($u2usJwd z&C(6KpxzON{gP-$`qtK~LJCqs1BYRXh+&$IUwRdQ1wG(OX|j6TqEalthl4ix#RnQ; zkM#Cm@8S1p+6-RQ8Y8^wC!r9Fq^@hTekTas}ToOnWUnC<_4Nwgs?Mte(B5hRCt5Q{pS+^e!OQYz#xrN#T>1C z!WXKoOZI!sR*$q`R&ZlKJ#rvhvOIb!N~X^EtR-@r#X)GSHV(Fjig1mZhBr;S3==-I z!21wMSBbl0G<`Nq_wq}~;;cS702-*GYv;4sGE|=Lw?OqjaCKEU_*zsD%q5`+8l11}*q9u&6&;c6YMp1QA^@j67BVSlL0RZq>Kt zAv|N!${Ig6kM@#xHweJ{ew#W^QLmpe1pJz4Okg->1Ca;suE{YERyF+MFoem<4>_g837{DK>-nQIq%JY==KmcyO|gG)MD=e zbmrOVpJZAIHpv zur*fF64AeI`K;gu7q2LKA!*BpcWV(EJSI=GO}?Pox@_%M2+5vY5GjRE4ql>=nh+Pm zy(=OQ?Fu9NriC z2d=BPK1yBkJeQrr1(NFGX10E-RgPNXqS~;(pl|biZ?kA15c~TQomN-A>MdMSLHE_A z#aw(duqO8~(JR-g$mhz02C_Heo0?`qnAr4sXOHQN#A-7Wm|_j5F<|=wHcuxN*>>lG zj{sg0l#;7KH>=A08>S-v=wNu?$5+1*kr8)!N31q?poAA_Vd6ZzgCduS2n^{SE^bV^ zfYj98%6Rw-l8n@sGMkYGs|&ns^l)fu!Al2d!REcwDoG*yxX0LF_m4a_E}zR$gXC-( zLLLpGQ$xDhq!YkcO0I3ew0V?-m*epbJCnAbL~aMt%x@)L3XX{PpP11ghKtF_j2WQE z%bG$%`4=t{R{K_3giS z5skWOirdY}vNqxlSCK^!rn#E*uK5C-XNlc|T}GqwGr6(k8ya10@sPblB`*jWWGB9&H4^ z z^>j@|h%KrGh=uzH$bMfC+%>HDKbH^xL!##sDZ>aR!6E0L#AV>o-l6%*?cH|1xA>l!0-2J^I9x2WxNef8LkTp@J^9?;n$o0HWpnV^S7v zf9^JSg=1f~cRc>T)k^Fg#wYEI@lm(;1^(9a|CeU3pA$tnJB-)KQzUX2SA!rKM~0;X>G8yUouw$+ z)?;Mvjdk$k;0*AA_0B&UKc;2`B`l(;_W287U;U6~B@*X%GAY~PkCYJTJhAtRU^op> zzO*g?!>g*%=}P1v>drgomwGQ@+W( z7UETHM7b;vUmO`^uV3~}Eqj)S%Af0R6O+f2=@mF|+YN@+G?c+*88wrDfkFkeEuWUd zdp%W{2`Pun{rOF-DbG%5J~$@6+qyNW&MD9IkRE)m*Fr+UA!*i+fa-u+^B%xKPc!w< z$Dr|xKNr4yQV-A4{n7jdE`B~$9R*;RH2L#zD-Ly-AG6h!5l*rC_GSKgl# z{!!R<*8Y3K@aSxr7nUJ2bxah_^5$+NNE{Bp{p`z;ycTy+*sG5{HjHjjZz^I?<`(H! z`b9^A!;+#wyqjD8oYnf_h;y%3Ak}ij$5dYXiD8qRf9_2NsHUUSpOqH@YlO+t;^{S1 zWQl*z@V%wlA)bOs{0AWw+4jjt%qg8=j}NHq0IX3nW^FyNfN+^1SL6nNwt3owP-IK( zQ#VO)`qudKVrnL-sT4j#_#mlA@|SJFf3OdJ|Edf^71U8Rx{@W6CLK;VeG6fhQVgMjf>z zPJgRo|Es}Olh_tge9G7DTCUXVk66LezbqA9Bz26Z?7g)<#!_)S3d@Z@c@29Z*q%|F zax2u7tN3ecPt9x#Z)rI_w~S?B1je#?mWt#Dr#}_RSUvNYlzm>q`_zyXa{Lp~u7TDzI9R{Hb5 zT)*!^&_q~dB(3v&mQ6Djs<%+2B-_l{d@+hco3BbYuA+$#C(8yZ12MPp&9&tX>zhjW zs&FI@yW8apH`43dm(LXPbLY7&M$ZO1;xa6_5!Fu|l*(-2f{v=u%%Jzjc-V2rDeUVCD`0KWmYqBJZsE`peS` z1%VAAxZ0qP_gK^1I&;9VGuzhoEp>rx0G>+Fv(`l-8_703S$2gN!-H}0bd|RO8U%&@>MgQAlztMIf4Uo%O9l%&wP1$c#t-`gD~Ad5a)nEm2HHf&tymE%B*$X-!~HKJ>UnYJ(2zn&xYvi}SZc zOf*%sU5v`fp@KvHat|@4o=E13h7wGX>QzKR60}uzTlYd?t?C$ef6w^htioNX!+iB`GA0JqO*_&>ntTw^%zew zqZp0-O}=Zxb+&(Fdp1SQYRTHw%;~}X0qT~B_V`h^K$gvyY3jEe)a2A#TkuF1FoldJKw$(SD(mk76G#Zojw$_2Jk zGaLx)1Y-?VvU4BSsMZ^!SUH7M9G>hMYB09RbtUmaDP=+E!N!OAcpBsQI zf$$_(gCzJo7Z$Qj zE!mZwv_%^GraprQDY23I2!|ieQI?rM+VjFD#xC{$NKg*n*4|2h^N86v0bao-h0RNJ z_umat!We&K+?7xlpr{KSF%6?Cn|XUYW_z2JL#0mQ&$mP_4iOMYaZZdER2%!x4h^{C zo{SqJ58=5M%s_&2faqHYzsriQ<+Cbe7zUdcS+doaf*O=_EAei`f9Uxfv=tA_q;Koe zrJJs_mqCSDJo5M*I*Gc@NV%l+FA$@sVeCRm$`g4vh(-#UC z(w*x*T@%8;p6&z#tU-k>fNtEpal6uLXV|Rrut_}e(hd`~Lo?JS>bQj3ns_gyM#IJc zxI=X5U%}1uB4nB%P@)UKS>qP~xWg;yfAAGWaJ`)_j;sn??z!hA5vm9vr#jXh-!sZ( z`@fY%n|rynQKT!f+JMhq@%Vm=687U3zBb(wc~gg+f!RRvuX>?o!hM)Fh6eU-tTnNn zN}4itVwqB%Lu}?J_hK{h^#NbeMPbdLQd;e=hGj}X4q@-bXT0tXeFodETMMZ5a+a|V zY}PdbW+5~RvKx7~CGz_pFzf?+x<My$m-tUzS9G0PMv3p3zx2_6wItQ*-3 zl!=;z?+~D)@UEJ5<@GC+tycE^oMfPHc2*(%bUc)W2R;&$*38Z5xn0zPC{c z(Y#|o&L7}K1Bb19B#)KcWlHXI4R}}cI(TtsJ7;H@@vgeE@+-q8q&^I?`F%fs1Rd?L z3e?7fL2=lrjg&3C09SIL_gaQII7^gZlO#0v^Ora%1~i4djvdgq6tt=n&`rh0v#pT#KKl!cf6Q);)rC0(ut>lF2kPmDgJX1TQDb>1 z6qD!!l!Y9QWDW%XUGr&DWF3vt#7=f5GS4&Q?Mv3VlXb_+pMQwthgX6CTg?I6wbMe4 zJP#%aj|(Hc>=KED+K_k`I3r`Tb|g%!3tm}Cb6Y!&ZXxlVBkBYHAPkf!9wpiyb-`U8 zGC1#nPI!TZrZB3zdy+P%oJx>CevWw;NJjMll8uV-Y@}5Np)xmm;;=(kprV8(-4_nd z4UG8C&k#c()r>&@2z-<|(#o_bh0LZ-VPx$M^k~saT^_}FR#iwm_Y8dC1iW$u^2>0| zX1O6J!NGZgv5_%XJJN{qncWe9SGKKn?!FSt0g|vXZ_(73@0o)hSWd6!9o3@af!%9Dy3-@}$kQ+Ap=V$j~a9NWH3 zQZtm~m$J!-ey%w`z_B7n6TBa+r(VGf@ZvJk;gZ0C%0&|76e*yX&eD{?}gSU2}%x`#OstO(5;v zh)y~kpN-Bwv-^729Hpss2h|eL6o1b0;LpV$FMeG9aqG9|XFYjhoPCcswK7M&kd1`f z)Y6+li4Yy0Qqi{*SC3DQ9l$^&>Fzu|yf^4~i>g1u#V60!u~)0z`;gyGiQBWq#P0sT z`}6CNf?+co*Iqp1-TQCH9MeM%oZ8>Qf&12TtqN-g753*ca37@~V}nYu zt+5D!FxK6xSXYIWm4)mvY)9DCdl8zWnu zdne$va-j2j->l-bV@B!mX+Dhxc75yFgqh+XTWDjB>caff@=U8Q24msQLOHfm`8hW# zzb-0!B5jAEo5yw2wymM$14g3J;1=!}d$aK6)$2bRIf<~61w{C|5;+KE1l4uGvel?jaww=Pz#1Z+uDh*Edr+ zJ^hy#Phh~Grw=U{44p{Pb_!5)2mPfqsx#WTM0R14(Ivlzdp>Se;YzSAIX93~&O|Sa zL$(3eE4ifT=*Vtmh6Od|?e|Q?N1mH6kG3IK9}KS#kA*X1YCpx!zYnlmc^N5jE1bx9 zn2KAB+x9fKl@1-_HRJs%axsUSF0r~)K07+oeFu4A(2evU)P7WKcj{zC10yAbq~Ld1 zY}KHOZhM1>ralt>Z5~YxoW5))wD#@cjOQeW(`CLUBABVBI~Hs=-Cc<)FIjBAe7AZr zU%?t1pL|BcdHZpYC`L~IP0qc-6gNMW^qX*90f;$Ga-v6&+0din4jj}Lv-9!wU!xhV zSt)mk^!cHhLk#19L6s3ygB1uePcmkLd|A zm2T5?j7cV~aBs5Khg|e)7uC;^TYF{bk&{TiS~bfdn>}C4lfZ)0%7}?=32prT$4-J* zLU+K{@8LgtGgqj;M^+&pzcyq)s?1yU-4;gUPIW?Lmp#kj42o0R`X251a6Uan37;8+ zJ5`5wHK~^MxL5WnpT2&-rQjw%WsxbHK6P=iF<;~E$3iUOE559}wOq>Q#}A9p)QK>J zMPRpbWzXp4Ez?&C%*yPr?uBB$kx=Ap_rV!h#VcBkM+3Pkvb6DK6CUs#{20Q>cG@YnSyS}im~k>;RxgH}i=P(McJn$0 z=I@;UIE}Q(l8D zW3NZ941?-0)wz>Of$vU_4+3QR=vwf~cEQ?}Fk$m>IHt5^y}Dj#v~Z*RvRK|}^t+|| z_Vv3|(%CNd4@2?@^~`Mcy@!p0?|T@3z8FrWl+H*fXZ^Rnu01O0G!5@;XLENoZ z#xCP+b{$=BnWpH>XkHr8Y|FCND=LdXrirMCv`*uio2Fg7F#?UEHHway5eis&DI_5+ z1TSc5s0oP3a0wOW2m8nTx8Hy7`Of#9^S(i>|)S!K*(yCW0pu1Vzoq~rg#3k*UL)RN{5!Mc?b6nOjwj6M9wZ?ml{nEP7=rWY^ zQYoG{Ld6_ACEZ{d?ophC45r4yvLr~V*548zUMxw##@aBc<3&jqyi%>5e~vfB>wjg= zW*f$%jqkVO=yFjcjD8`P7cgw^RZA*9%XF2+KY{xSuP*017^o=E`=Z6 z8>ez4{37^}p{nbQ7)x*kzry|9m{1BY{^#PocOXLS8&D7;8$0PuAo61=)<3;Hk=MF= zYM+!aGTo#^6z< zRUE(B38#BmZrz9Z5l(>1(6@#T(Y{&f$_L1gHHNQCoNwTr#|WIz2a49|^Hfqh%Tob1 z#C1#?R%AgQsbjJZBr4MQMd51Ve#xbqpuPO}?8=nHXTd;hc$CeilfV9T%6pP+{gnYz z8sb9_8P&Fwe9S6)ieBc27cGBk%-nE~3#PF1aHq@)G1X6jY-3{UMFZ;*kh@4S!toQi zeX>qkC+GAD_V`uOKKj@*c?Ea5nNX3cnoAdDwv4KR*EYFD(?KXFY413f2T7m2d}=^H zJB?Z(H#Qck&S~3qGp$|Ve$|=mZs#oWO&^e7?BpDrnV%Uf>BIEN<9gcBTE0f-%X)U8 zo!g1QYg9nEoTRL{D*j#v4tRfnua++`EOhSbp9p8kFaap`j8Cu;MXCajt4OA{rF~Y= z-=;phZM_QKWO`r2w~67nImu%IZ$R`WoHZ`(K9Kw$trP=(IV&bykd1=X7frs4Q=H&pTU9`Cbg-% z?HOh zdf7SO-MvrzHU9Rw;v?Vvu<-+Kg4wuperDxT=%R{?P%A~R-n~Cu z@%Y2cUvSdfJ_(`lEzpCJ92HdfR$fzL+d}e);gvnAp>_Yw7d$sI)lm8{_Q!*e{3|!l zXpa|JUKU$8k}Ohv?jfi18VlqY$`V63TT4pp98*9xh8PuWjKwXiH!DJ4wTm}cS!*zblst*6EiUM;D*FK*v5IoPHQEMFuq z0@{e>;H9Ps4FU^esF#o0DX90E+gs&P|CJQLR{E-+f7YmL>sIyBYrEt%xf)iK^#cWwAtC!f9m69DN8+*y$ zVJNRr#GZ)O*E2FOdR{9ceM@~p3Lw894BI&U$QYPs;F%?ULhQ{rAdS~-wTHXzoVBzT z5uQ2I)MAWdHyG literal 0 HcmV?d00001 diff --git a/profiler/advisor/img/env_var.png b/profiler/advisor/img/env_var.png index a2c9b6f20e67600f09cff6f5269a464dd0010115..1009bd195a14f7429945e3341efec927d2f52a99 100644 GIT binary patch literal 44030 zcmdqJd010d+c&DUiXsvpYefZ#whAgka9|QhDzs8X4GId%5D^uTc^*O%1(8}5tO%$K z5h?Q!k$FfWqEMzp!4Rep<^X{N2q7dp$vzvj&r{F&&L8J}zweLta9v!mclKKQUiZ3( z-#x6{w?1pRNpYv*s#U8to&5FZ^Q%^^Wvp8D4SM}L=+5&Xvv%mu>VWf>=BtWnYLn2# zw{FMJ9ACAnBz~jRWi52Q!S~lo0jpLiH!T0IZt}^!wrW*D!pWbH+lD&N$XZFBOc(eP zmoThX?fo<%-TJ$nH9zdy!8+{AcyxU4i*@sF)Mshkj}+!_@6`MD?&weJonL`bky&A-Rv`6!v@-;eG2 za`)lgsg-fB{`fMp!Im%NoG=es`K0^TFSDs`V60xgbLGdsC)sODEg^-l!De&~@hk-t z4px&0z7f(Kl=uZuH1U}r^_>}#hhIjjZ~TAxKh{BsK@21kWJSDMCOn9^c8|b~7}1U6 z7cpy+6JArr@|>K*d_oa+iy0f61vCa((*d7%bgJ~S=%ENTjhDLjcZsON6QWjhq<;ci zGHAiH6SjQ&*IGE4DQhgq@ko!Dk?oEEp+y~CPZ*ow#WZnHlN5l_8GxoN1dF0} zpQcC5<3%aQ9msmN;B=swij z0TDOUJk*<9iwWaK?Y|4knLrb#Ci?S6#=md35ZVn);WyuHLCHp=c&a4o;fS=>1UqLw zW$w6i!=j!kST2g>6tkz0D5khWrG@|uO+JsO3g3_D6A4^NU^^N)MFARBnn+1u^QT0K zm87Uy6L}zZXFefV7Ce7J7ya_EpRc-t7=;&=4;ue_Z6nrZlApzBY&7`nk4X@FW zuTSR68+`D4LcM#!mxvK(&hR_qYv6UDw}_^S5eX41O4M&^3y%2=44N<$&^2tONO3&s z81B5ZR75q9)_2^}LiFSy7Ms7b=1Sq8K>4?z1# zH@+;~x9GD;87!%awkC^~wTdLYLysSr!Fw(J9McA*B}7 z6ds6;KAe$^;Zo-0F|7CobP+*vQq&+~PdRJd6>&4d`3#;RC5VMn`xG`!SU9=(itz9G zw^@9)ccCT!3p@AO1phaU<$ryWD1rnH=Kb6D6XiXqzrDsbzBGlB|sNTLaj5(7ag5ft#!LIx(~M7S&$pcC=EI36fVKuyXAVe%JPd5;?_ z9vmS|;}_gCJGD_VdS&07zN3nWWsCE`>EWD`Gvfxfcbw8 z5MpE(L~h^=byIyfcwZMG-) zfJ>rMlzgNf4t|8Myy#8lVYgA7^8jZYhaccf5P=vf2o%G*z-1Q<6i}tDJOnt5sG%+i zQ1J9`mzQ$8sgT|Z^w-hpfSN1-l)=Fgpoy>)Qw(E*a=zSqeK(c>7!p>z_p12b)Mp}b z7%`K;14R(qnA`@T@s*V=Uv+j|Nw)p7TV2)q-*dSCbs74H*Z$uyNuXPX1eDcM`(z}L z3--$1BBbV`4lBfGUZ+h2On_qqF{Xx3009;OY$B*V@#psWYGz0j;1Skp+gC)TKMUky z*dzZbHU82Q-nm6b;Zb!DvtF`||#=2Cfbo(~wjp6O#^PZe5|`Gepr zN(y^lzw+Q4rzxu9I31j(*HGmJ)S84VvMvju{6}HdozI`+mJ*0y9Z(7`vg7K*i4Mv0 z(^mbratCr`xGBuel?`9j9|O+E5RAV4B1+Zm=mK_lu3SVAg}guh_cRdl`V+_mC74=} zj|R(DWM@yM_hDN;%A8GjLOJ?b52|2v6i^66p_Zb)sM$P1K75h-&56&h?ajt96hY!H z>Lkj`05KikC3{ZBLY`xiv690u>aV-@*Ho zfC!`y$X)@wq-w(BX%dXuBlFMepH8f-g|8b!VZn#~)qS%<*jbsRS%Kw~*pfO_9)dDv zgp7NPYB19|<-9)P=@~WY5{s#LeR-i*Ofn?f%!QUyCq=D3b$zng&VX_GNs$9QwTlbE z$yY!nW`Ck8PCf&RdJVZZnb5!5VUMlKica~unhSOAPQ`z^9surDBoR7fvom(Tk2!}= zCi2o?=rO#c8OtJl-^$vOQL|z~Z>wS#GHM<@uh}SSNgxHWQ;;m=((DXRkmcWON0mY@ zY5e+5jTPPv9@>&m+?5~0By`|_a=Df{g#kecbbOF}0rK*Es*J&7jOaryJpx>ix~(8o zM$a-r6*~GXv3iB_6ZNAC7{0Tt7hp9Br~tA~nC03-f&B;SwqI990;3H)tPi=19HFhU z9@W#YK$L~V61X*h(fS7{U?Pf_Mxs1-R8@G!2E&n-vLN$vc-F#Ld6G5MWK@rc3sxo{C?wScA- zR=re!kxsQiPre}a#7SOUKFpg1uH7kZ|cNc8p2l!bJ|QMpdwWRQp^-T0)V z{N!)aQ7fMi)UT1Uzcl$F)JyA&Tc}S6Y89!8%A3Zfh*fH-eimIf3 z!r?>7b^Wpy2bbNu0vRxpRy_gF|VwIv8Y&=WUPMUy}i z6C2T`CIPwBqw?haJ>kwF?EO-et;@oqEr>Z~_x|`*ybOZVY)C;8k#{4+u`Hi`#Pu+X zSqv5ts;kJNe311G+O#&^4)Q&bEB}aPl_K5|?QxTdl{N6Y*O5>x+sGv;-I@LpK{;A( zUChal-4E#UaOW|zFsr{)cnZZ`$3l54aS>8o_DAPlBe2C>FXi48 z(TSQy(D_y07RJ=-;iHFTrBz@gSvar4o}lcKsjOed`kDS-dP#xK6qA`sf<#W{F<=%H+AM*M*pHes@gx>A z5`*03?7;qwc6TWmyS)FAoMkq4o99w)8Qan+a5M3@{#)PS4@qn8@4og>9ksSFGyIim z%yWne;PRGS_&au=%tdr|v3v)q5o56hKWwV#B9M&KsF(}mDc|Grmy)B;3r%kqP((cUyTL*p8i# z;zm$>{!r-Fw~fTQb|NWeYDg0&;CA5c8ddR<;HXYh)-`fcdfyKlygBG}O;Grz4t5n0 zLS2xkvJ5F=XOV0)zEv3tb{YaO8`4dBH|L|m#;jOA5waOxA0HEflI{h2|Co|%RqTiv z9Ab_?u^1gqKI9Q8nYsm84c?^%%$7g1QOBOkli`@QHQZl8g|pOj2*_Jm2;Y;0CwGy6 zs|y&)C~^uv%VQQ02A37(_H~MlZ2CpujAV?HNvhnA%urL95yQ(0nJO_;V~>t1j@{aG zO~=z|ql@9asUdibBc<4G2OqL+oFZm#Id^z)QLjsL(k}HKW@Bf}Bz197D6vntVz7zP z`62j~9v@AaE$7w3b)wrceU-Z+hn>B^8F)V2G!OivKS2r$r%1|0OeQDOCkV3ve<_ff zA*nCGSW(jRMj?erK>04G{@xtRkn|j%JY(J)HDS{N59Zfx!_NmPIc3go6nbsY&$Jeu zAljlE_*oqOF)@eozI_`)t%r_b0@VEbyIw+Y#SL}8&6`xDK#UN6?j=| zHPtk1p8HYc2}43CqaMqi#&em}>KQHuiG%z^H6=3zcU~4u;bX5O)a#`0G5morKIE8X z$Da1&K<4@xMHk?Qf|6R62&L5c$9mA zllEfgW_XP1;@F~?sf=Vo9wyYf*{oTf$-hk4qQD9HK0k+Ogr)@N z)PUhcd^ZkzIYE;YQ4Y*Z6kL$JqAc6N1=t9wQew+w{vs`(Iu+@0_B?#N{lena zg`KCVk(?Jxm=o`?vrT-QK4tvHV)MM_M?1M%qmXA;OBZ<%c`WCuisnmmJ;FYg-96NA zy#z%__Y^)NbQ;l(mH|u4XtF)1{iP63#U>$P%fk5sAvZ8#HX+||gS#6>CxUfik96lJ zZr|1~X_S*U?F2)HBcw=sC2DdyH5Ot6@8n$azpCMQ!9uyJp#y2DI&;d=PAb8^$8*5z z=JT;p@dMm+Az{A&+9VQAOT0C33!c5-Hhx^#$y4x&C)^Z_A@F8^Gf!`VZZ6!{oI@+X zicZcqIeB!DFJI`3CG7(*yn*P?#w0Be3iTChqAHMtrwyHsLTb0TV|yHoU~iyQ4PYLx z%>a!F-|gF_J}Q7>!mj0T_sy0J ziALqFe9oO~lQyKxa9kghoGD=WP^epO;}^U|9u1n2YaT{VH6Yeg%B344>Byz<`NTbj zmq>QQ{BMbAst{witdPbS5^Eg-^U-wkXT`qEJ*P+Z28rUwinLqMQbvX;PlRc6+%M& zZr(>n*6=7m*HZK7^z(6Z;m1}InStXspoje;=3PYOVI%J<=Vb`i_c0aMi<8znu)u9e zJ-a2|(*tjr-gOc#8?vqEtmP|dK+#1}Z|j_9PXp?1SgEyausONLrw$GtZ-o<>gK(Kd^1hBhip@pS4)8$8 z1vnGw5@Ujh%czi;J*6Z`$a5nwHh(}o3b8`1@P{|k|*S(UX+1wW+n z`ON~CK8LF)Q zog)+ZnhU8wA9Y4POb6?*e6py9Fo%%8pFIb3yjd2JRf7i71j0PJh9IXyNh_KOKMSo9O>k=*cG*^8Aa1`UMb0HTE`ERbj=5(g4(L}5Wh2a!1F029fCj666iBCuuc z7*IHCrcam`5JSK*80fqGCots>UQZU%1Hpsk9bdJoGJ~E5NUP}?fN2dq3y7$p=K(9L zQet2Qc5kHr$;0j96Mp_gro@1W+q_p`DTZxH7it590uyfP;+T2_}%q>1%|2(kJ13 zC|%>cy-)-%;!{hr!NW^(QsH9M=dJJi#G&JqS)7kJU7X6DaQ<0yp)>m**Uk(RX=3dqR>j44u8kt8m5cf6gRT9v}}BL)I*xyQHt^fh9ZPyo8yR3hg^g62+<(2gv~bA| zES`-xl}n3#Uw!(VZ}50;9i99T3Xp(yOj|MZ5?p|8=`Hzn%y2Z8?$om7EfJl^*@KjpBI~ZC=qi=K8 zrd|I%(%P&h#;RdaWx^)X3Z*rOG)~UDEP1jxL>7&#XSgiS@N83k)7%SnpY%+&M4j*} z12y(^Ac812i#bwX9wUjPE1*`PnQil`;;Mlcw5Xe&4vpVQeGgLIf8+AgSdGVL3@7lr zk>0c$!T+jTeV?HI&MgqM$7|$pF3L;yp@4*%PPE|-;wqiA_r9pq)~yV4Vg0098-7pq zSVS!E7fW~59^7a|+zC-e0IJbozSPJ#CV`TMS05EoGB1-$ebqJU$+K5Ra7XT@w%V@c zxJ95_VkoiruU$w3$bC*N@gm5;h;*- zfK9qG;AQt+2?n&Dd6>DnmfI0vJ$-nv&5Ese%@8b|ou(5iM%@UcjvGVm4Szi}1$=dF zY&Y4SaPj?v9g`g?C^|p>>BN|GAks2cl;J9i6%BpwT3ha49^Qa|@}xzFZD{dV_2~=Y zi?(>Z_|cHjEAX4U&y(v<<((;!yUH z+;O;4J?zm)d9M8aHbziF z%R@PEv=6Apr8Twov^k&UOiZ@5(x&BgN_6V*8LZqMzbJZ}vM6>mby7{g|Grd(cR%3S z%eOf1y|qn!!IWL#uC->-^X_IoXU7_V4}Zo>wAt6EwfJVsgqwpnxNy?x?KyEl@#@f!>s9B&I&oAVV9os6No zk8zGWq9eMpMZ>~UBq`G$q%IzTDc_wkJv`~+us?wIkXJGtYJt=KPOUN4Q1B5(>>W{v zUPBHX!4zgLMdi#}XlOYAL#8R?6EBSg4fp>jeJ0!MHa=IliT?V<>8a3*9F)UF`h?g~ ziT3X0!oyYlYdD90(OPY>?VTr*-6BdE2@4mKu|5DURNPN%wAHHp5EEXtHCboCu;E>G zn?}7c?r2)j*bxmQLXLk{sa}yuyJCl+8qd4nDGA%m{0>w*1)N851%9?I#nQH@bn7(B z@p)B2IouqSu%kgRkhPd1Ht#;HC7s<5zS8>Sw8CSpS{F(U4T4gC;e+(P+*IN#O^D*y)eVzsbxRRe4KFg%Ukn`G{!FK=RE9J) z>Wc~h0&$KZ7kz~CV4xa@chqoKljyiJuQo*RKnPp6|eH{`Ykrd3h%?^C3muJ3DNQK1J> z*A%bEkLXIRWlVuu`VqH!SkC(w>6Q&WZN+hJ9h0!eG+Ops`h!^wF!Y{EOCoJxu%xqk zd)=O9JYHJh(w69&yqDDxLSsNU(r!OK*7o9+-Up@rci`vaQ%axVvN+zMaf!51nH~I|?gG&k-13u7*AALqt#QJGtvp--rNO)u`F=KSQaz zD)-sCK{}tGRmvF*)g?VCSUa&Vg=;E+!fwuSeXITbv7`qh3xb3DHElVkmx(r6&&WJB z;t>wX{H0uE3q)eQI5+eNCf{V~xSh2ATwODo;QM5}XhGxZNa{XRgzQmxKjf)&O%pl< z2RGPC1XH0EP*R9IQOKn?we+MUrrF|8ym1PEsNR~7NVC)8I2`!7nZGGY5Rk;;ria1t zMlqz9k2I<5IPSzluP6Klm}07!{lAzirz0dq#bSP#-D?koZx8kRpM=NgnO}NM1~pEJ z1-R~gwg@xjH~7@{RbGC&%bZa9p#C4WD1Cq+*iTrfoITyr^ZCLdwRP3Zzw+0-g52v* z%NHx-EOV9L6#tK$M7^zATFMWkN;<4qBs|A$jN>prm%_4&p-{wrL&o{|g>kExFl;Gs zZm7CKS_q^e;1bCgKM;jLvg2ZoTJe$iRe}3;Q%}5vJ_#@)1xI-XH|)A1uRqH)5;Jc40>f+%g`VXoph8g3=fL+Y)ER z^$K^)g|Sf0hiFZnjX>v8piaHp5cWR=1p*bRSX~v=FZcr)$5v-ZX1>c;N#}21s6eeP zz6NWKHbPsO6hI#fFs~_@Za#30Mn&ETm51b@>n*b@UT@Z3Z;-PYdr_%0;V@N_7H_DK zDfF}5Y9WlTHHOFmzwwVI%DwtgKf-D5_o5%&e*{Ro zBuzlr$?^QE6O*ZX!K`<~5$UPDre+56;r$0t4>fmMy6_Xtd&l(im=~seG@fw+i%*Sk z+g;Vx2B_50hVhy~p1p=amByNvP^G0WUmT;&6Z{euD~)JYT|WO%@pU&_t2y(B9$0fK)meom%GOBwNcqtA*OY~M z`8DqOH2OPhUEL$19~H01_y*0(BThE`4$)>+ z?(8Q;)a(k{Nq21<4D6-Q%4-*=|HNd8r&B^Xcs8_M{FMsn8YegTYHj-Sn@hgkDUrE) zzFJ9IYwsvrTJNCv>D1W#i$me2imesfNe)WW2S5k)$1G;f|N4Oj1&rg4E3H8%;}Q@byH9N)9Q>ha{M>{ z-oCo;K)==o^H-<{r8q7*IC5}9)Zh>Z6PU$$RpD=< z`Yrkv2P*V?F4~=&?d1w_51Be?ozb$RtfYzcF8!1C^r!5?w&PyQq&q$* z)y0deIy`7!#q0hiocYFAS{F8q5AXJU9DPN#;jdd&mur#Mb-@ilUK8 zy;Xbv(yHjYk&ARj39q}$v59`)ZxY74&H$TflX?}D;+xZbi_XU=uf2lCT9{snjATjK z0Z&~i_Vvzu6_Qm`c%xKzI@$9JDU;Q2B@Y6uC;N7MC7RwG$FtvL?|FJzUN-e&OWXGi zueHDJ+g5j>p>%RC>vqZZ%{1osZ_6?_)%UB_?P|i)9M#}x5?gOsDBt@Uq!qmFhxq5` zc|2{TT)e%Jk~(!7bh6!H*Hh&-Aguz~Hh_gmj9hk>O}a@of2z+e+sJo*nj>`cO&I_1)TTrh^W)M_o316Vv|izr)5eP{@H(A|B7Akhj$dM^A6ab z^es_Ves2AR4HZd|l{P&yE~DGElO%RE$VcVIjI>Y0OB=_tIhn}o0Y=Xxn}wx;vMyuv zh2A2eaSAw5V)|~E?KgjAc0g#FQPX@=)Ul-m$&paP`RN;3`_f<@jwRfUl_=QH~n{2GF zSm%>jMaNCp)greo`qULW683H!e_TD$*T4O|a^wr&50y^ZR4>|(iNRJwH@$S!H_=2M~}LI+*U zJ)%B^`1W?tSJ|%k5yye~w7)i`P&nduOv;S1O9DmT-4-g&p}Sxn63$F9Gf5vGPD3miDT+ojG?H8MB+%|IF6t#D@kev@Gv`dWpsxVAiF=JY|Z9T*A82OTYuctu6 zN~MS(UFe>AdvEBLp8!2`m+s`>zD}TlZt6!lKac54%^6eq)ohTmH0~>uL>s_l&JjZj z-0x_u9ou6#G)F|}Z;&{6Rh!z9`Yl-t>N7tivw(pDcjqZlo5)Q}0fPH2Rbf|I4zF50 z9;X4FQz6@q?@5z~(iMr>nQwBObu3Rqg#7mmu1G~G(hX82G)1#?&1CDyIvvW>3h+bk&V1N$FF%`Y zy)~oHzvuYO_NC>FE%qrg*wV8gRj5DHG+*FDF-~e%?wDUD-X0N}yF$FSEvg?BibAu1 zjFf>)r%@@EjWuLjrVH+g<5CH10tL7^a=Z|`F*xju|99rXNGtOu;DbH&o>1q;5!kzZ z97fqXSLDTJcLP#qmo<(8J6M;b=moLyC7I)%g@Yg))+i&~x4}gBlvjg~Lt%S{#N`ML zQC8(Sq%gXVslB1mF@iFh#Bmp|?Xb_4&lyqQd&C@$@%vVA1XX#oamd(bedxC8a1uz17Y2()OQ=mkqw>n5F(W}GHy^myDohUa4=h%QaD^sZ$FrDB^6)dg zW5sTgntaEEnT)djxjH3vlkK8LcT#E#gPsk&?vzREl-EO9t7uwpz?_(mB^wm`PH*fG zw>R0DfGrZq>j2N8h-XdTSDKU?C)NthjtDt+RKc7dO@ee|o{5hMN>*ZS@_GQB^Lf^F z<2vs|=XRAAb&hK~v&_?lY2AntQzUcg};3A1sX{;95kdZ73 zdD<#}{%U%mzhgW7nO5juU&I|H#fE>korS(GvM&cZ7S6D2d9&SJ#$^lkA~# z7n9^gpBnGZl+q5fu?2e}(jQ-VQ{5_*9vV7rQlI*Bz$L2N890!ZH#3*Musheyk@+W~ z$5f^FVB4IdoooQ_UuE#TvXI>A>Q~)he=n$+>CNpnpdUXznU0Up?nhMCYFf#5c|L0> zls7kui_6Vuc3wpP?>KA(2b(=!l|Iy)NPBQKO+ZmD)|ttS@+6!dd&xXm!kSFo(3jTw zeQ#chOr2h*FV!*Hp`!rYg94Y zc_p_*S(CC+Tit5A=j@0^k>@1tvbc93R3;70LNt&A_3lj%)MOh}yv5#rOLl3V!p20N zaaFqJVel*Gs7!Rz!9ggrH*l)#h)JlEdAasE>)LEWc6e|De4Ep?MU{Kg8lr}Qbcuaa z#aWqdrPHKg?>WFfbTkG?gdB-=1v zatv1baHRJ}gi~?*OmH79`DU>qs=ymZKMm@pmuqp8cJ?WaSAPIK>teeI%I`c(+WlNe zoPf{){R>%#imQZS&zZjG9GON6)kP!r*fT|xi%{HzYkzhB@h>d~T?^mXhyD?LQKiE7 z=2ndF(DBe_{f+lL43l>C*PH54e^OL%a8ahc4#$z(#lN5my^MThYHi@zmc^vT(8Zl^ z*6g!YxY>HfLO7j%ptdQWm8>yzyvQ*)zQ>AxM1_ZI;ChG%lej+L_Irlm<&sEFT+p9j zv5_d8Vu8&4G<*)$j!{ZKR89=&`%brZ8CFIEc@CINZnQo;P!rnK*P*LEl2Q(MyNj7vG4d2Q6N=!r6zyFW7(KH!1NIT9e!*ajyqFvhUUavt9FL;wtCAG}QxHUaPQiaZz{O<9*EO9!yr^+xYcw1{b z)-QwAB8+{^v=be=9#5|8_&h|@u=y7OAulBXgHnPky$zfG`^2blXEyEo;!Lno+mZfTu6Lm~K#0KhI>CE2m z^}^5ZY-|YaYEe>=SL4XNv&t{5NSFNUV>zGAE3C>8zBiA-<$B$7g-ck;JJL5wFj@{f zMrQlFgpmr+X~_>sP+u@CPGsC;c5#LhSjW7qW7waQJ#*%IyvTi{h%zLM21;KRi5h$C z1&}m)-onnPQ=}1P8dlILCjQBUnpi=){Y{TaN{Dq`#iy7o<5rGqXE$ws^5vb(+5WlS z=o;Pbg!jRfZ7xOq9sL>})T84)_SuJ>X%%j{NZ$P{Nj7I(Q29E!!}CgYe;vEa*^o<0 zd~K|UBr9(tB<<>B>yc6#FId-k@A;6{#VIuI4u$-!JWiN1!~UCXTjnlDcR5!9nnGd^)F}QZtHyM&hztsmfKJj`wlj0TRqGB*o zh7?iQ*H}Sz%&sx) z5I>(f*jB#}`?{NbCLq*0f^9?Wt0I(Fx9CvNquL?xe2&}c8a$(Pzgg=+H-`$JgQT>w zv|hX_96?^JIOXhNLFkwp6I4ZLz&+LM@|UDXlZAraW^P@cCYYD{+7CFzi>Lk6tmwZa zaeiP~7H~stQL1j`X5rJo5Hi;^hT~M4`G2|~@CD(>s#^2ehT#;qmeN`w4PnoON ze`xG9V&0Q=NSU5i?)=5SV4?O0M*=mW-}Q`-{A_%?2p!$j0c2zij)W3}^AU9+DZqEE z+Uq#u34Y((U_zS`KP^-`6mTp5)I=L z*H98COzvHuCF-r;xlOXUeXJ>_M9H-ss0;FRXdN5JH!{bCYe%Z1%`Wly6~@{WlCk}i z)slTs@xTL)`+mc6hr*P0-D-?3$;MkdRa&9Tx|C~e2A++24xL;Pb-woMhJBeYzI!Er* zcOw^P%Ng|Kz%v42ozN`dq1jQ!AE|cFGGx(Wox)brI8V9cb<<*39|kf&oc!*^FlOO1 zk%vM2kYs>is#EL|1|r()w?QbKGVXoHnw;526W!&F53YcGlZKb zdKerOii{8;M$01pw%(xqxFhFg9>H>1&QEjUjj6PtjOpb_Ll*+;H&>unyR~fLhA9>O zkbel^Heriws^6;eo#!?U*)Br6QnP5;O!l0k_}1i@9jvIHOMh)wMQST9>bFcv8~589 zN2sw(+co~8O2eHPc_@j0vQ+>l^&JoOaXio&?KQ6QoYH4>04;BK|t|o5Ery99y zdh~N@yj{^3If6*z2hJ((wb9Jr@|kEe1OEMX(jF+_K`6LcRKU@3+-HS2V~3}UHrB;~ z>ps!5uCPzfK-h|NL_NstPEJ_xY-xO2)`>!WN@>R&gB>T(ci8>F!(a^PyBmea zswealDYU4$heEq?!c>`indh5+vFlYC*h!8k>!v1@b1Un{y-)ookZy45-g4uoX8~dD+rM?*suxf( z_S1HC*gvv>A;)uRW2voL_t4FXVfJz zULr=a^OWPA@XeHj@N8b@=FFxknR{5&KsTJx6J#afNYEoWuKuT`6#^rk~ z;TIm8Z#FYQ*v2>TFaNbmSl-K@?VJ@_<5CmGtYyXcp;cvHXMS&FNVcwz+2aeB(?AMj z2iD!4dSmf*lC{$sCl#qJ#?{ZwVO!wWa(P_LZP{08xPBvB)b_|hq_%qcMvAMJ?j>0u?Kp_0(vm?5_Nm2~`+sXP$S-t@NB75!GWJi$NekWk46LOGldhMOKh zt%tDeRltI~up@`bDajfNmFCUOm0T%&ANKsM=mjE-T~V%3*A3ae|J3HZHEoHJ?4woQ zX14q@^%8CH*wX>EKzhq+ky;_W!+0ljudtis1-(=EpG5!vzR0&+vkVJ_mc0Da|MQpb zJAx#Kp!k!;|H!iCf42&oluaMw*s(exW0>x2hCmS(>T1I!=S3Zz1-!61aY_wW!v7#F zbj3VgAv{ z=zl3!URE3KjF(^Bz<|o1@f#S`Yk5$=7!KG)rU!9=_YZMo32}d&_AHDN;H(4AP4oC}YiyaLYbSAK-jc^gN<-clOyv+Sn>m_zgmwJC>vPwq;lv0ENN z%{79X)n5_smJtr$ayG-qN9m^#uB7Je?xO`J>vuJv%{f8en+tbO7bP8d+R*U|(=;1Q zsn}u*8yi1-oQ)pKwl7%Ev{&i5uIUlZkf9Ox_50!c7`++(axw#Ma z$oIX;0uq|u%Kp&^kko1}%o-$Zd#h1@^Omw3HbAdt^E*qTBD$~8$iF9h6X zXGI)y9at=M*=Mw?f$40W-Ay|6TWS~4F-3@tsz$#ruk}rJMKy-Uj~t9ERf*RJo=El% zd{q}m-r+vI=;z+bERA@AI?3ywPDCv zf4%O50~>ydk>6b%Qp9(kc)n0hFijb`YK^^=OWCQYa>lD&R(CD63EdTcZyqWD8e!+8 zmVky05PkeQ#k|0yBqOe6`^)gh!RHyW=FFk5MBO_h{7{7H5nEKnA7#c)$uXr`2KQ<{ zx>{hG@Rm@yEu}Reh?60CMzn?PS~GL4m}xG&T02?w`^fZr*N04e!Mr2M45P$I?k#?h zu{EDFa`%s~1RWba^*%iN)gkC0=3r<(ck5`j*IAHO>~L=5ZyfyAHB2{_aPyxUcl?Tq z_XnQA-`Cr*W+Ge-yTm5-KHU~cf9N7O z@MoG@qw~fqBQKz`S5Jmyzxv3j?Pd?h?2)N4?IWoY)viC1jO8ZJ5{7H`33qA74eclE zI@WGlRT$LJ^ycJ*YFVj^`-R@R(GfL ze4*}{e{{iMXgQyJpkn2^uN$_<1Zk9wNSSwtH}u6$P|M)|=H8kl6XWC%aa01mwjc)) zgfJsdrK=Dz!LmZIA(NcLVhH@7#hOy5($%MM;letp)Tx$Vese7T;ee!$19Gaf0|qnu3$hQ5}k|KB|_W5WSq2b8c^G9;THIr*|QvOe&U9Jl21 zs%J-E)pcURJ%dZZEmup^%!gl|!g%XjBV7sG`o7O!f8W1g#mThh7w$C3#xx(21GbpMP3~q6fZ3(gsK@7Jovsd3_B=}$@YBzN0o1nX zcpX`D%P3ILmE)%*C~wxBR#|jC#w1sRmmHeH&Mw+kQQh2A=*4!N02_>=FkAR8;GBK@ zV70TKPDk)M8+KGs9N;KHsCF0_%9*f}Ku*PDu@!LXa_qx?#6M)Hy4BiDv1t6w0i}V+ zpFAV9VY*i1FLXVD?YYrR??OVAL0F;BkA(!!BireRKA9#OCvJROQ3z8=^l-JYPhXeV zP#%Di%-yYbc8PFzZ~$u@>N*#Pvo_DvxkbNx5>58zQt}+_*`}00@{w4a@%qN|&r2yY3>+rB z@CPD4Dy;3?uFkp>0clhNgPFW`L8Xfq{cEJ)bZ@G){>q+JyiH8qwNB-)4sPnGV4@bZ zbdq;)QE_=&*2Ud%*j$kq+1yf|cmB7kUS{A5;ZE9y?UU!NpE}wpnXA|~CC7Q3G2q*< zJ1N)3DtM#_&lL0CciA%kN9`j=>8<}4bKf1vcK-k0O{sA`Ma{a?Vbhwmj^6doXenxo zYK#)0#3r=0?yBfj?Nx2<5i5uhMJ25jH4=!8wn*K%I`{k?H7ry)v*3zRPqsS|WTU$9pGrs%vNd4GD z#y9kQh03Ys^ml%Bb0Cd4vtzZ_R;0UIt)A=V)+|r!Cxkj-CKTvk$n&gp@f+`^g2B^Z z!&Hxv>za%<(R)LwDuZ{e4(cqD+$bi<{h~Q1?(qgdngn^(u$VHcE@Zi&^>~;~{$kc~Cs3__Q7O}Mh{u_;jfWwuTq z=I)dVSAY{&5=FEVGjjF6*87Eajo~3du}9aQN)LD`G+n)|Ruk&OoXGsyt&?vkyR&@N z*(+vnGE(#KvCkDnY_cwn3o4&p9V(0A{$Lx6);Fe?qs#qo7{(4aiC!%UMsOluFm!X% zs)aF{O?SS+F64HmV=L>!RZg_VLMe`|1_cL>oQ5qLv1Luu26v1cyEtb8rS`wtVxFvjFanzqJHw(S>KQ5+5mYzkljB;e6oi z>apjrVYjk=-e@rlKA&l@$H#6I`f@V36bhv>AI_e9{CR2uzoG<+=>mNub?CV|lz|)( zpEoyHK|>ig#@^kCbSSu{i1+^qfUaAj#utMg-iEvF#tiO0c8UOO=f_R6bvVokRRpyc4vC2RmaHNJ~t^^^EmZZ+k~?DsJ7 z1yC+lgxTN@c0IMcS3<}jZ{!h-uQwS{S-#OF`#)3WnR4YSb;0{h)zpwfvDvCv1(!!^ z{cyvokT0EeebmywzG%Ow=F6t_6+DHaa`M?IxS~5f;J46 z+FO{ba}pm%N|r8NMMsKxN)YVm|C7kUI8rve<>l zyUv6yh~f-{rm=(;@m=nJrP|qr1h*||aI>4+JG)W&Q}YrP%#n_)fymyb-RYJl@k6ow zqBE)M{IOBK$EtU4T*>1+d2IWQh4r7TL_0J5L4w7R%Y}+$31Vpl?O^MF{J=u*_`$Is z_;F-}<)+C2fJ8s^>kGY_(FdXL2xxWO|2nRADc&%%&^6e?HKIK7FJI1=9CKpJA=D*o z?Ph(n;d)KM+?I59d$i`>LZ25Qo^0HFJuy_3KdkF;YPOGxn$>UXdCmg(>Xf*ZVcA2` z&yYdpy7pUiut0TZ1N&>UrDV>RzlX$0mK}GDgTopR`7Ir?vsCAbtnjI79HPLs&`CYM ztwH;XWTzi#J8G%Y)+hMHBKBg2Jb}`4Kx6{NL2xca-daMxXis|Dc)n#WzP4{z=(oOmJh!( z`-tY5C;g9C?@L<85YgstFu7O!QsOMBrskYlW4?n ztFptYF>r_h7s0SXQps@PNRAgpm#*^_x7Q#4P(Y%G?auf3`$HS#r(}Z@GM*p9eAd6} z{QZepNbo>W#7L^TstVUvuw}bCUfV|H1Oej1DU?DatmG18EVJ9%Q`~19oWdW?CB+n3 zD#N0j%lPjedPOt0$_!4HDlB;Km#=xyE*JI)Mj%rhW8OSM2`3;9?qA@%W$l@)F74`= zAPoCh`HX-9;S9IOx&}|QNDvA)DN|D)&2)llwFKNL? za0myzd@*K#VKdB+4J&R{)oOs?2)0O{l;&mH^U;$~XM3Q|gR` zX8ejc`ew;}r7_pa&?p0eGID?)?|3XZ@xop^s-NVU(*Q>K@?(*MHN4{_cm%Mu!H2ds zFGktIEquFtZM#lx%!SW_92ig*1=j$jogA{}mN?m?5bVo0-rsrRD+RRwa|n=D_8BH& zb@e*6xvbmVHVn9McREW0hJq#g^pY?_T(h~PwH2tBUjzc;bD0O0xiSG5E7GILCW$g8 zP@KKM{hlkgd-3h;eooc=Zq!g0Ls5#DfYRM>8{V%gi0zXop<+89obRQK;JmK$7hQjQ z=t%zmLGz=Ku-)Ce%s^C(^bPkl0Y0X4kc8u6DY zs}7%Y(bsRs6o7&OTwdoyfvdhGu0{zGC~)W#b(qdlJ*D3)$;KvSZPGZ)K;a~@2Q2=i z{Sr`A{ycNGHJSD$^N5X*F@3QhoV$p19+7kj$~lSE4?JW+YypJ*4I~$1dE#2PR$&#{ zSs(}Wt8-K6-IY#2tZz0w|97C93z*)l(w2jvtSJ2F>-yHu23)Iq6{ z&T9iKQ&Wn;?}Q)o-<0i_7g``{QYmMn^-bG{I`|$hk0H?cEz~Xa6dtpU{HDy=Uog`CaWuLAmk2!(Hs?>p`wc(g}Gz z;P~}FaD8J-U}`!>mn+SCa;(VkQ*ivg2dR}92%dkTp_mMfs1ki$Gv0hxNGvtg1D459 z;%ho}dXqJxBk`N8;0~3Wi zb#ImmFTIPB>|-#^^BFH@PIix%5+7oa<5K7-^HQ;{QL^bcjx5$ zAgDotti=2jW(p*y=Q?$5awdZm5#d`^Sx-o0WIUPQYppX})`&+brXcmmn+4U?{o6Zr zD$^JnfZD$oY~#ItA(GDmY#6hy0y}ia+!9u3oz003NlQsZpZCI~UR5VIgwENQk~56w z-|H1D3D)6BqK;DJ``*rRe$7IPI1e6zRH4(9*SOB_1W%Z_wKT-T zjCzhTgvmj4sirEx8urBoDN3mt1*t~fYd*B|Yo$mXLzuZ4q(PQ<96~*blVm$f_6#2K z{9Bec=tN{<2v_CfZR1@z(Mpu;PnC@32^=3ybueI}bkXr=ANr+2LH^jGvYEXZJyJ4# z@PNOVSb$B_GZ=CjR4+^)hP5K9?pA5OhNlns7j@j1<^?C;_*@r*C8xn( z$e-$sje_>WY$uKvqmnQM;(h!crfg+zFfq%P(DGWif*iT9Jrf{h2JSpu6V{ut)Q9^y z>338`Y0Q}AefKI~3%`uI|5-|}&^fZLD8j?}(+sdt<2Dv<`ibERLw7FlX?eX460jOw8!pB)kqcyFy3JZ_my3zx9oW%U#W+FLYLd;(oo6CU`NN? zi%83k9p2v?RmRn3o*hcORE*2!`k&OD3)oeAtC;Yh=-RPHtC&ieot3@x8_M3oNAZue_=u;G?N{+dt8VB-{d0rt z#+y!r#VfAU_hU>WUuj;dP!etgbYERsCwnrc}`RTvFve7qr+*U^*3gVQ9fI7u;k;)xiEcpe5 z6(P*DM>j2nE37^?m2Pe?!Dx10rR`mU`iff5%?EI{;Ah3z+0PpAAl8J9vQw=1A?xa~ zZpjgKW(XU1u1=K?F0`gYxWxN#GN13r?C|v~rfP9>y-`a{wHP}6CKmD(@}1m?SFxtG z`X}Qqd(V2|?Nk%XF81Ejh+KOm-FKAL;#YE`CvCLimMS~VAEW{NtiNk8032d!mVr$4DDtZz3e6Z-sSZ3` z`s)EOcp~r(N!;!>Qd402ds6d1f$Chw^!&K_9}Q6p1av$1KWd8q3xql;I%@BSgMhwo z4W6Efzxq2C*8r~@^NRK@LGV*iVwCHI?uZ>SQNNT$6N*^0wH5D~EMgr87?@$(|8ccI6eah2u36Zqx=bS1{|S1RS!5YM$y9SL*7Z5-XrTd!c`@J`KH*eatNH*Y^PKnidKDu6s@EUK zTGNX1W%H=XN2yJgkr}x4F1@)js&~PgkWucyc8SO`7o%*C_?$M3Z){8+l z^2hSQ^%JT}v9E{O6Q0F|{Ja^659>pod=s`^SSlW0#+7A)jvG-j5d zi@P?c^z!>e``c6zJj86;I(sY6hIM`L_9JWZZ6f?A?$a4PPyW&)k#nzi4B@%1;uK^s*WJq+BSWhFVx5L$X z@^Ul2bYt`DE|kE>35(GFm7jVOK4bOk5;F8^wOy-$pEC;Sp>HZ4L(gG< zVut!Ch_M4Dd+BDBY|LF)m7#pIoAh1A$7+UL@K7+#AbVP`G^J=@MB>}i{|_4nC2NC7 z@OXU1_nC2}Y*tcV;j!A{SK3fRo`>WZFJOrg@Ut(A=`&AF3$dFdc$Gfd-Ma=z<-E(n zO6oZ7YfpZHiCPDPEDG27**U4+U{|vRFY45*)KA2QDG6nR!qs);b^boKM!8otqP%=I zO8IR1{;Wx%rYunvuGXv}C z9VjXmB8>dM5{e2+~1|jhyHT(*Qpx^hy)fO+URqcO1v+b z-eEvEH+HS__Uk%&Fpe&@AG7ToydK}zMFSqyeNJnm>+Y1lcck+7oofOx!|HGlcI>vJ z|1#YsVm0GXJQ?^2HN$0Ry`j$iq(<$T0$01kPnx=V1_@53l&3|Yi?Zg&XNgD?L&9+q zNbuS1ZgOJuDQLCh8O_nA5kXj9=@zo6wxB%4@w1%-tX>LY1oH=tI(Xl%4>rsOntW2` z!Web?Wy(lS2}iVV?qStOsmKt0Wcm=+SkmeED(JfZau%lbHl8_Tq?!;e>DOSPL8p+A z7E^_+^K0q1pqC+ydR~c|rASkADwU!ONV7);=kigQ#x;OzHQUGQ%Q_f7=2&d{*a|f0@B0<>y?LU|L}D) zHKU;g^V5Vc?|2XSf^P_%|7T`>Mj}l4Tj38q*0H@^5ebSy>l3BTsruBKg3SJzVl17@ zNqTNy8;2%NN2;t-_Q`X9|FjGd1DlxWaeer5@!Gd+Tz4C2@l76emv%PjhyX?TNnu5O z{c;ll)zTuBP%%^VPel;{xVg0*3ZEv|>Bqim7pf4Uh+*iK2KrIbMPE2vg`qdg(9!_F=iI-ciTxhrt!iH}<;S^A0xZ!&rxzd*x*R7bV|%y4u?!Th$KH-m=Dpoq zVf6Doyn;U2g+SHj<$d|JKZBl4#uzu}n&M^b-x2!qj&2eb(&*{mOPJ+bKP?@C&v4l}T-; zn~M=Awp^fBKk(9_ZwAV*W^QW|Ht|MywJ$tRa8{@%Iv>vTl!JfEiVMACXj)`@@$1So zx3u`QGjFh&(YyqvSH?+5(Z61zfNV4?G^RY@A-oTDNdu#5XQT3vI{jEsPV+A7;T-=` z%?i;$*^s$@L9}0+%X_C_>`w-?AT9Qx{#Gl0Jtw`!%*_RX$Y44I;QS=#n>zBT$tTC@ z=B%ss`Kc(w%^l2s990PKVVQ%YYtmT#9Q!=>?$rJ45j~foLw5==!;Zsuxz|cQ%klj| z=xNszOppwp)9bwrS@B(YG+rTqUvHoHNmWaI6>Nr;6^LbBvKJupwM) zi|&cC>64H)Y_fJ*4^niR%ERWajdvr>U@@y^<%5l`TEu^Z1zT2+G7~z}Rn_>~u8s@V z3)c^pys8w*m^|wxeV5!%VxUg3eS;aC_7D5s;|G|M0O|I3OGZuFu8-%;?(-CX))PJs0B|h0<$LUOIQ^Lw_utV+L)qc$Ls)?IYdKe!W z*9~VeN8x_FtlL6wYN5N)B3{G6Db_nrOJLjrgI3Nwy|e^@zNh32ny5&wJC+$Vt2sBz z$QItBXOJ~5^A4ay1=rk{I#-5n{Gv`WDpqkKy_x6)Ig+=+QXvfie0@Zv#=7veQfz{b zB)hSt#OqKRy3`Is2?q@xrJOwKS0Vgh=<%1Ckh*<2J?GOOpTe^!0#C3u$n{aAn|Qb& z>e6Z6RCyR$#(dz(`)bJ5^vde4P1b#B;bR}tcRw(OpSDC*C#za21AXh$n7gyQ%VOhr z-;W89r<6vIsC0ctQCxo9iW5eYUfk5&eLV?Lw90})ET#P2B)kSV0S9~p-}z=Iy)5?9 zR75v4Op)DB`Pu~t6#k1@{q4y=hAi{BspVC~MgO68gcp%-T-^gMy{EwceZL34DZF1Q zcjf7dLU!LT3ywMu6;3@#67KHu9a1@S7PyW!%AnGb2^yC4s8w6~&PG^Pvln?_(s6g| zx&+&vH$N53NAj4G-gF*hEAU&8H*dnG2s-O;dJa+xa#wB>E?7-9h+~_B{r$rwhqi*t z=^i~%63-%Kd?O2{-wt_0Z-W7p0ogM$*L%Z76gvq!XpD>qQ&x_FeS^Bwv{7ZQ>H61) zU{@1+cb_HNbajgbY zEi%M~zmqf!RZKI~#23b;-3@ec2Pmn)q{llyTzYr8#@8XFX)v1Qd-~{?V2>??HiMK+B5lIa0J(vqV~ z_Dizh+VG~LaK(J--ayQQjF_9VasS4fEIMDZK0Pj#j6M5BeQC}3d0UX(!YF`_+(~-m z#M2(X(g+BwCoSlGSMg1s<=^E0tH!}atd7XviKyKQ0lfS^m^_hG3X!056<{Dm;T zu>Ce7MJ77RM~v35jr*x!!3c2AkStbyV(lfP6tua2{9p172rFM!iF=iIOj^`vL%{eXQpjQ$WG06 zzbeSZM|ufRBb1e>eFP`74_fWtVEmg)?)<&z3RqKZQ@__VUPBMSjI#ps+G2ebmt1+` z;L6m}sCMszKBD8bwd#lS(krN(8kNCKqv%^G9KmjM!VIR)jG;c$*_Ls{_36F$s98ig zHMmK8BFK%>qn$+bAjB%C61nT-h-;5&{^h?B?`h>HPNv=zzVMK_rhEbgg@wILFlb(W z0SiBqf3uh1w+T!vj1)D!3pyx%RVmg~ol$wB-YNFIxs0~?Euv}vdt$|Uk7CQb&3wpi za60-7wSc7t$~-YJzB>!deyCZnS!;ClC^7Q$*;qjkWpQtKIz|NZ9l!x9t`arYb-S*5TONkD> zTTzM6d@J1>Tt2kEH)Cbl7Qm-q6ryR79^lTdX$I2!AUL%Blf4SBBX+lX^$*jwWhwjF z?~wsDHFtd+)8_Ps%GVm0U#jrULDk*Z0y<}|^>S%&Hsel!p~a$pLt{Z^HC_1MvD;rt zr&My*nh;!|&Rp6b;I6etJ;>5?>9PjqYKA}GSkyfMVquX$~lkp8f zRZc6p*ik0bHaB;B^B$iWDO!k7e7%zfOIPF9tZ{=5jp__AK3w7djaUyMFkS~(fGf^Z z+SDK1!mCQGFG^`wn*>x9O=;T^eEcA5uJfL5d7t+3z8{e`hwHjXE7VXHHmh*sb?-SY zIh79vMF_MJBnTM1_;0B(fOo7*4wQ(fIThJpx^s@pF-`GIRlI}Akv6q^NrkyWi0TvV zMW(sMmK{QiLpVvqyp`)l*q}B75NM1%_wQ{z0M-c%P>kZNWJPERIAT$^r>m)tWo;}7 zS8T+|Bi9*E0lpSYzSG`I-X-m#5oenDWo&*UuVUbCyH^IDMuNEf4ZZMRE3$?1N(sm> z*r~1d8Fx5lrJtznRo&R$u@bdCQ%$Cf{I!Ea^zoh+^((Ws^G~Pr&b**kUP*jTcfI<) z%l#upql&JQpV=px#L4O_A&`io?7)g^oKO=u6!#r9wy`tK_m+I*QdyG#zDB{ou-ZB} z&xs`82$jYxjdTR6(lsDj`8_LHqtPnWWNK9~z!UnnzHRiCwY6J641W`;GI;MK@TKIM z?CE=K$eSaTXE&7MCn-jDQuWDQ;PRMa%5jHW>sb>{wyAw)a;4CII>{XZzbihp?dbXh zQhy}$D&xmx*J4~96_V8);Fv@Nr*qV|b)jJST#$J4ojL6IrWD&rl%OhY1_^EN z=Ht=kFbSr=vPg3-tr8H-;>KX+`;f2-M08qD17E78M z+np!%g&id?;u2{U=`!=u5^e(h(PHG*+0&ra%e=f{GnV0#0h=N=nz!!a@!dstBI<11 zIp!>6`yUjpK+?>nR25yxMlXsNSK0@&F*x#+C$c1HVvy=PC6Yw3yI%Ww|Fr)RKm^!t zI7nJ_zEHXI@d=PbW?ug;yLTjV#Z^~V;tM8fT0uk)tVyuSGZ=Ar>pr?&~F z9GyDnYFA@;wZrZV+cdWgNmc z^#q|9YJ=#w&47ltPv+f(4`MZ5x=unSL1lyetHY#;)b`Y6NDwr|CsOf6)n)@FDsMal z2W(3zd2!!@lSN+8&TvP|(+U|nD$OCbC#W|4d4NXQ+onG+BYOfst3lu)m9N{Ibx{k$ z%A6xvte{UNhJK5UI1aJ6CrEj|QlhV8*V07G^B`hwnzZ}kLX zu_z#-eaHD#lh-%(k6U0l{5Ac|^`tMKd1m_dx|IayjsSvnGp~rbxQ$Ju*7J8si<<@g zqdIndH$U4ZM?&SA7lhBPL!_jNSTBkNX|IG#@lR+wyBj%b?`QYAs}o8g$}P(U*n=Q9 za|M`CE7Owm3@tghu%k)}oJK~cW3&ty+m^2IwM_EGTJ?fr_?j@C}cc4A+x8&3BEW|Qvb&uQ+taiM+l z=05rc9mdksb-BhUZB#}^=aV4Oie+y!56D&I5s*Wbmt*HEf3WLp-tegfzMq_wbu&vN zQSY|&x1DQ3EYb~K+qlfucT<>oqvB004%u{)2u@qnkG{IrI1rQ?5@mqW(r+q!sXI6u zB`qSpz5oCT?CmTM?ujL`4Z>QadN_eG3BBa8AaucQfmGNL(DTUe|Fu0A+Ui`-Xj&wQ z&k+R@c5k)|wq*GF{&=n#)jNh*PBz-THDF(&X;{9Jynn$DVHZhk91Mqx%K~$6*YR3G z0jQ{`(yK@v6GbW(30IS}CnBv~XYz)*|6;!{6@*79qM09KJ{jtTXkM^p4C!URMN zZLO7ZJ0=xo`hQ{C(ZF`4`yRBjiJv*(Vg@$g5;M40tI!0Vu2r_YcSHb3Y;i4!Id$R% z6Zekybs%nQn0q{P!2gu+Ni9ZCiM_z%h6@_LUett8`N%m;ZnSnj?YNw{nao>*1{JIp zpVP=3AMW_F@L?@7K9Qx(0nSR=Qbti;4TG z;a?dM+FkCw-Uq)`6~6Jy#9Q>YA+ga$_E9Ef2HO|q4E2Q3YT6Ad#GS2x)Ge)}5dSLS ze0>X|L@&gppt;VKV{dF?n8+|o?S=P|f<6XuJOE75dNs6Lf8)|oKr1l6-)p9@YL;20 z@7ktij&MSgpPlO$KEArUSKcO!J6c z#Y9s{5%r7&w{}#`%gKqGik%A}&66|duuXQP%gDV`WItd^13b4s6O+quGwZ}$EpFvk zjv8&&F1f{CY^)(urCXflWREm1y|C;E^Z#zP?WJMW!+J&(M@q4y*?_!XVIRU%3rVO; zI2sv7_1&D?3No}?gDlL!ff-dbSj&U%yki3Yg}}}H+5f~AnI_KG6CHxeH3-fqcLX@p zUgR!F_AvARc|=P$UVQgtzyCu&9h=U#{=N0!&TjqMImBMhn{7V(O+KsMk?5Hgl?=iW z#=@;*z-u`@PFT0VuMOhYXMLFWmf}) zBgWr1W+a*8__R8SXe)|0ic;qM%xq8@;IlDAiZ_j|8Z6VdGNFNSsG z#Os_V7pYpnCkOo>u3Z3DGuGS-tS0>X20X$ouNay8g{GR^p{fO+BsvO__8&lJFZDx? zu>W;$3j0Sptb$Ul+WqBbX%cHU5F~bIGLDPsKbPq-vOYp~Zs_;T z0cOq%}9dKIe>my zY3`bBXmdwFH~(!~v)588r2eo;6_%>&;noPiJ6?+z3lcEVK; z%uR5g_BRI&?Y0!b(0#4RiRQEm%qs@24ot@|$-O;Cv0tIM?Nn)Lr<~MfRFvxgBnuY_ zJ@C_TmZpC4UV9zhAA=#W@5x(Ft?TjpcVN7`AU?NaXMFDS0xH$ALpTo$1Ov~EzK}k* zh!xa)>>0dE+bL9j=IxPU%#^Bd4dh6q2!NHjhLEbQ>gdh7&CZC5LC#kDi44p&eLLqj z_dmO4txu~qo*THuS~o(I69IKyL&v}Uw<7WR?Lukiqt&hK|9CYQ9*az&o~blVny45c#Y?E<@ttX4T$RPfAPp z!s&5W?$Iu*#!ar7SehvIjLM~No;9yuU{sWQo7p|L+LJ^YvDA-2Nbt;Q`R}QR3BOIC zR-Md;EO_71pQ-A@SaFx*R{KTY)?A5OamZG3+c-(Hams8=sdH{0qTy1kC!>ZMSOOy# zqCbrlCK=Pg3|duNA1f!L++i1`eUe$KMqs*lD~GlJyA1MIy zat(e#Rk{^+WkZ_(eRhDVY^YR)KeuKi#Arp-z^)JQ=9=-jxv^tQ7K+l-pzOjiATT09 zO1FwPGYr$wd;Muw-rYV)snxsuN3q&1oYH#IXlp@pOvZ`Gi@Z2s;)er~6n{A1As0o+ zi4OUGv)X<<7BHtLpC|2`gu+T>mc!jR9SJL$+tv0Vt9BEKbkqPs$MgV_u&iLMSxrv9 zK~IKQ0)ir$2;47F5QR1f{+nhqkhU<3SP_67o~Z#4MezXbu2H!nrnM!Le(=n4C;A3G zb`b8w-6b#*jR3Mb>ZxUI9QVPxo{b?cqqPr))SK5c`1dyScNF%Mqo$~H`OH7MF3n>1 z<{Fc8Pg%Bq4hfUgY`VPuBVcab!26EM1_q6Po=W3|Al3Aga%u)C)ahJIoo$TUZA|!; z0c%&y2N0dOj#$7JTh9yM*ogh&3*sG53{Y+#+6vFYfUHlaVs=+VI6aMVq_b;PiXvrt*ddH-pUS{VX zw~ze}U?A@s#;xSeG$t##@z4Bu-<~j05OfH_)(+D-{lV+T1$tv7W7S*gw^FxsOOD^^ zakp=rD_|_;e^s@}1^vZ8`RrJ9u`fh$JMrS;4|aFep6kO~;Wj(C_R?1r`%8RnkYJ^R zMenvZY8UtcGRT^PoUb@}(H-_7Z%ooPbH8*YK3wPb0~(u`td*NILO3BrVocn0)U>67 z1@B>C_C7Hl22apc1f<)bFrmb^Zs=asvfZWk!Ca9lv&?0cx(Em$2;A5aB#_E67$64> zEOlr)<+p*Z#sH8?3nEXS65-syFE;zbo$fPU@t>FT_)65cC_+qV59D<{!S67f5J*lH zWevaHC{vpeJ3g#>LHt;Ss-wuJ?2D4X zisLoXlcwftJ$pJK5NmwuwY)@M{#YQ2dWaSv8AuCZMB=g5`M}40%?YaF0!zQ3$lGGW zqkKD(>8)FvV=E@p(?3jebu=*C%Yhp2%ve%XXOwgQlLt7o>WJ%^1i#SvBcW#hu43u1 zorjKfsQy-XtP~ukYhcI29NCcNZ{?A3Q{mIs8ozso`+O?Z_E~wnW;8D-3cDdaj-3@$*gA7o>8c}1q(cktW)fSVpp4m62D@yi= zZok^R-0nWYAZ^wRgx+OMNgd>;!Xnz6L{53T?27iotC6UxmLYMbdw6OIr&H%}56mz# z?V%qx?L}<}e^1FPd7i6D3!Y~sN@N&{)&6EK8fG#O;UkzL2c;-UDOYX*ExLkg#`^Si zI;=Vw5Reu3YH31V&#rQwsTso%{PU@B#u0G`m4=_O|-a#Hb&q?3F3lvxk-n z$U65t3fVbu`ydo&8Sq@(pfJ%NJgKcM@ucYqcmr8n^Lg{P%_+wR!ZAODoLTZ&k@1jt zopEA;+@(5HRw&-xb_;fO{QJgp=UFYow#CWzBRB1F_aB|dw`ocjLp(Pf{Q!BRnMKFs zt*uQ^c?vy@IukN8U*@ZgNk$|Gbda%Z%`Fy@kqL0czk-)F>>!~b(L;76JX<^h5lr`+M(8gX{sSDa28nH5Dyy}Z;kc1}-94&=nU zyg2MHN;rAcQAgA{94)k>gzWIXT@o>Z{6n$vuJ;>H2|G&y#u5%U;5jgOiVa)K$v0kR zMWjy~0Ay)~Hx!Kvg9=R#P*jHeKV}5fRQJrRWwdHCgWZONg(y>vmrEJKu3F7KhZfd9 zw4`<-lC|o-|Tn&73ReYkI5?_ZGo`+)19og6eK!m7amBQUY=-w;%2{+Yr%EwRS?h0>HG)=|>T^?3 z(qy+D$eR|f5`BpQFDTra{H&pgAJ&$;B3}hZA(r&(OYH5c4?a5IWbg1Ff`bPxqx85c ziywLO8zen;SKCmCgIc4md|8P_sQxtVXlJ7Kb7g!`y2Z#P@H$zRYUYbpUYn-hz4$N> zBa#}hVm8JonyW1L&n3$WIAHfrdc624D6DFtbgZbrYg2-s?NLLnIs)@BNJ)jBcXF*k zdlCSWHHJasz+WbVLAkNAz9}r{)&4#Eht#f#ME4pX#&8u!av}r=Ow^o}C$A30?wWtV zhFiL@tz1TfE8(~LlY>3j@_%*;13ZjdFZ9D#{fC?Dq)p&jhd4d`-^#SwBKV8j+!a2#*!od=arD^00yx^m%4OC-~du|tnY(lI+Ef(|h`u`wo3*=aK$c&1zF2F+sT-jSc`~wL3 zzc5Y#pg5qT@drn{y=H?Kbo|!h3+I;wLRmj%gkC&|EuAcw0;WN(vjpBX?Q?mClK$S* zHUlCyK$(L%H3t_XUF9}RP3@>HVg)nJv!bzaD=NnhSuRI`ao3sWIs%5SVWXvEC(D_Y zA1(8LIRuX79%uewLyqPimzFy8tIGW2J?ogYbo9uNO0egNLDnsLnb|i;-N?#?h=^OF z_Bh-{js%tl=zka(W^7-MSaaWV%xNFk35y;a9-1kXt9@pU8wd0Z3<@$0GqN$v?Uit= z5RS;%Qq59?S9USgkxBuh+AZEZRC!OEG-J7xYq6GoR!6`(7uAvpC+Jdd;Yp0I z*)+##kvGstN|}(a2y@eBeZyW=dX!d5&+jWdUx6hmGs6S;%|;ByNa0ey20&?$QMeG3 zsIEl-^1)Ob%&Fa=kT5yn*&;pUOg_9jfFc3$p*Mg1>m!vyGq+OnpA4#=$?F4n^ff^7 z`KQ`T+;|LjOHh&Gg~v{5xLwiY@-+E9L?cB;#ZV0M-O0dkCv0HxqnaS7y)yR+3u5-M zrMJmra_<$#?0Eb$Nu?91JE-l=5r8NGEe|b7iUqs&T+lCjQ+t`cUwQ!YZ&pXdYb;CW zGI~o;{m-a*jO{K7of-y36n z2Luwzv?H36H^w7M!_65VCI~_Pv+T zZuwC7J^3`I(lV^BW)GQef>0QuwKsiihqg9lGQc$|`W7qfA9C8t^B>$U!IIFV37g5jKNYPxvzR z#?6HL4K_r=E&C7lug2?Zm{AY| zG|QOlh6{No4wR{=u+`m}GfyKgMy-w6E@w(i&Pex(C{@D_N5=r_TcOBB<6Vv+QYsO= z#5n8yp~+od_mqoU$EavU*8{}azLRURHY!b-+s(p?%GJGZuVX*8`(+o##y@IkylqAi zA9O&@^+}v#I$TXyoSC^`qf%qNu{SkyE}6hjWkM6+(o+<+c*?bPGfix7z~u^$#k!v% zK`D*oO1I@ZJ^Pg^1<7F{ys66VxA4+ae@-8>30n9NC??YG`)U3r_|iV~8vX zN;%%C0NWd!g>Tnxw${bM>jFT8Z?Cda6i*&5+>asdYBz?CGk>hoDokC~Xmqm}@TdHC(6i_%mTgfu z;f;GOsKnQW@9BGE#XXS-btt?YyadndMqc@uf)#EuOgD)gsMK-pr({jbCBQy9lTyc7 zFZFCsUu@7(5uG+)Q5XcSNjWeyLwT)VJp&nDhIh(Jnj1a{nm6|#15!pX&~)+DG+z*I zUSx7++#DP(mM)1MF%;%%+l_LEp}u{@qK@-KQapVc&^R#nC`d$?LUPyMEYELdnR5Eb zo=wZ`Po>rgZ#HIb;dKsE|6Sr&V!^6f{jtU1&wAwvB|S9nwm%v>Y7IwW0i5E`O5CVhU)9jfeM-LtQCHVyy?Qn2R%E4ntMYlU8n^PmuT7twA5QUPYuYZ9zby~jXLk}H>r1mGBs{j}rnhA^FcpV)0Cu6#|p79>mylh|a zP*}TV9KYffx7fRneo)kEf$51o6dxv8>W2ZeTJ^(-=5Mr&sI}wMAZcS@g0;m;B*523 zIS0Y_?y7*|m`z(^y6y$MV za4&-IDXY-6!Mjj4SOWX!8awEqX5UIahVzW0^Gv3^awo!sxe$Q9O$Xs1-!m?rx9~0? zkc;t*FRQ39?L!bYAS*8Du2+4ed$OST9r6?<|y6tZT|yIS=WW0c-!{ekhjmbeT9IuGZm@v8Cjh@7Oo zy3MI>0ADOI{S2X0ZN{lf*F2TJ5%r)`;&WjEj;x$ZGu9{AZ92yVyTf2nihdA@c_VGY zB)Y6a(>OfaYPQ&5047N7p;VTyTX(?qdr^FpKEgdXhzGF_p5_u6?Cm+MR|T}Mlv;9* zH)-!U+je@MnIK^y&^foVg19(`&vs{=OQTDUg3E-gq~62JleMV7cFf$VD?Hg7eaq;< z)nAvLzqmb6amZb4AEE$qT@2Lq1lh9cDi=f}z6UHa;L@(8z4k`9p$ad;Wj7aSRS&Uvb3oq6O-Nfg*)75k9leU}4iky}_*31C`nq>nt2guo zL;);6WEb#k34YEc2Gc$m=@kHKsv6&%@3PM{=ltGhN4SzbJ+)I`SG=VmkMj;694ZgB z6t`&fKU*BGV_<8~P9iTZ98$#k*3c4iREjV3NVlvJISE4sG9@btG5R2<234-~cL8TS zOWy^^i4L%|1_x(C#8&aghHUDb&(`1cNVA0d86wYe|5~)8cs{0XK-RHmYKrd#Al_>~ zS(yYm=PFK3e2dtHeY}i%vwP+uJ6deMuI6Vr?lK*SJZu&0s02kssOq58t;QM}u{iwW z3J)uA(^*gAP^{p5wj$HGGSIU5?b$y1>z@#|eq_szv2_g-{RUthqeRcPa|C={)GC2f z(U>WhkxPP`l@(+F-rpt<>;BA)E#DgLJ;&dz8r9-gjZ1>uUBm4-56|D11TZO$H$S3X z?b#l?lv2G?6O>Pl6CLCx1MdmCO_`suphfJJ*vTKOEY9naFw)E%<$(6?P;m+K`o&KQ5tFQbmS zyYOfZnq8`5j{J`|4DiB_*J-PZ-)NnaD{~TXD!r$8dW6J!qYC4(wSX4XB=@u-FKoEW zP4pk@Qk=-2PQ?;%J@@UMPXFG3b|>n8V1<0`Y0w@XFkYDov+9k$`=r0fSKEIy@7D#N z7^r2*NNQClAf26oN|BmY#qZ~oCPptAJv1qr^)Bef))e+~o-*M0Qtz6GuJ#nLT=MRP zRR~ZOPVuTxxRN&LQarQg_-v{TorBjhdvz)hgnmAkQG%ZyLzuH%A6IEy{(io9{fi%F z`>u+$As3plbO&~rocX_ylcUmLyFfD}MszE~Jmv&#d7QRL#2RXci_I>eE(t#QPmiv25wUJXmCL>zUM^+gnz+H`&-2FQ)7Dl^)L6n}lhrr{n>dT-uHfBN= z;sZ6Fwm5RV-9gR=m;ceG19ho@2q&V(U8gUipFFl!i0p_Stp-ok&uH_#eMq`l zHV6iT|7t#kxyZ69S<=GTD3y1n)?a+?hvgg71o*BPJr9;?$OL1VvttYm90VuKUUrn8 zdLm=rxLBo+hiY_CPPjF;SWxy$1{AmSP9~@?-Xd45BmZVZ zv$nTJo1tEkxjv&2@PQLuWa{_2PNdzq$QP(3kG|^htdy#@09p=ps6A{e}n!=nB)KJPBy;7^Z>)7t#tv8czPT;q}@-SYL+@F1k#-&t=tfp;G{Z`vX&AOp=tI;6w~u;CbfM|WBy zXn2tG+bN>bZGDECmA@Q?IQdDr&tp+nXg<+fyp{Wlq`8yj51X1Xv3|V*BbwQrbK`? zIG8YFdYOi3Nqy#yFKl4!>5qy}2{13WRO=^Ps7myy04pyegK~dsfiY0fkv|>}W>O@# zo^PR0U@z*=;y!iG`o4CY=ZdND)})J56{_!)BoqdO5`PRWRULP^KfwMog z1mT2-arNJIc}lxM3fAh8X4IGjBHuPsVMe3dZ{gvDGrofpih1G5(e`Z}h%q6z#%$(S zf_1sd<(=-M6zgA?CcDDJ35FVXqW?TfPsp}Qt4W)8WyN-}Bl=KR`%ZeN{~e>c+>{$3 ziFJ~}D7yfiO5I+_<^Idf+9SppyNsoW4OJZ?sSzOw-MoJP%}c1r_yqYg&nTlr2B%OE zOLH>@`;WCIf7uvjw#I- zBHZz#2DQ3vLJeff*J5RNvp!5_myDCa9=TBg7OBY2N&{)eS|txELL8I#Cy|O`w};8E zdew_i_X>921U~46Z7rh)fRY8PgmMl6pJtKQxvAFPli%;X!s2qh*McA{&UWM+P(9k^ zW;{n96J0l`zKaDLmSeuH<;>E)R?Wmds{Y;_1~LuPB}8JtzcYvQ1c}iVz?ou4$;Un` zL(@+?XTPt5Ab<9fiV~;m-0V@=rc_$dsc=}?9r=T<&QpY1xXiLM-#rW48TG2#;`^=} zWaT3&tjnO|O!o1+E%QJ+K-enjSzP;|j?gUMi%YS=8)r%(SApVfwW#`TN!h2v8a{+= zJ@xu&@OfB7RCBDIC7+UU8ZicgO{8Kf&tiYI7sdIo8-ha{O|jF|txF@{Q8TkUKkwbi zzUkgc4Xn30w0Y0+j|auVi(^S2aGG8mB=*|?X57HpFL_5VqXmG=%*BZ%e_wzS`L_J} z&%}d#eQ}5>;5@s7leQzPt)|t-MF!CpB+-j8=xa3}k z(a!uR87+9C^d*x%_&8?^DBAX5bbZlAQgtB#S zVutk=(bD_(vhP`GphPQ+Rj*gQJ|4`#-0|VZop~KSktPNku@}5aS0Tdp^sKB78p{S= z|4={e+J`SSUx>Hp!PNKi3cdoOjS*y?a3rv2gjEzYVC=_IaKL0#Wmf1moPy%<5O&~Q z>pA-JCYmX>Bmh{piqs?r#mod5s-fj&Sv=}K%gsQJTBNw>C1UK-Yo0p{lBd}WVk5>feNSO;@*N-mBSIBHs{Se&GfYf3OQx*YWW70FkB}LV(egN|@NwWXLFJJA^yA%dov&8^>gc#&y+N0#EHC)tUXmlw z4sRX9N6F*bsoxq7&{`rjASQ`{mZ-_kmt!ugkhm&HeEVu2*NkL{E@?PHY&8*$N(|M| z5Z;-y(J)r}AaW9Mb-k2q3-gB*gxLJdZ*EcHZ`d|BsB+CwwmqTO=s~2msVN9XM{>2G zYBN0U%0sak1?Vxnub+Kzfua~9LwK5hvK$G>LZz$WKz^;LC-(z$Cfjq(U4H24XA%QX zHp~Kq(ud3MaJ^3Q{#k%FzS>mmhWUy#B6+tON)My>*24z`Q9)bqmzL~;_yJheZI(BHyrfndcZz%#?87)>rX~*)h6Im}7&)&!Yk>{igm7=Yod2E$j}XO+Elfav z)$ukT)h)-sSw(|sJ5N=s-2FuS%;TWIb%4rnp@ULxw*;LlbHixThJsz1d1nAyhWKc| zV^d>Hhc|<(Efp%%s1HiGd{bSLZ`i zo%i&lo^qH(|G*f$qvIWmUo+E$XgD_7L63bC|D?R5TYc}GHFUy?J#F5*<~x#poW8FA zsfNy1x}1%Dw0?y$e;?A1vojJ>ZWq()X6esj66m}ge<@J!(~6H}S!1&Id4#+-N%1ai zgKIv1H0ihTQN7GDd>P{{-NtT!?cwM^edMxq?|-Rv!xY%2YhA7HEp$F-Q{5uvIA=WH1A4x^7~!I)IYO)3U#g;RtYE@0L;-A@eP` zzXEk1rex(ZITY^;Cd*9^+|wl_0^OFcnHg(&l2_A{*4S=0Q2ERp2VUWDBW&F-v$vxy3Y3mi9d2JVKWqJvj!-8a43kZ9#dS4lb|1s|fQsmG zhOiO%J8qO>hBpff=uodr$UbE*uxnl$DJ*J!6Sm`4yM-{SEt46kCYBTfN73Qi48K1y z#s3GnYhC$L?&NvXl`R48^+b(^h0v1O_P>uF)ykt!g4^S_gZeUH8xc#L`{pTZTG`mM zyg`hMZ3B0zi{7wn<{B2%D;30EN^`6}elyJHg-uFNlzBlcU_!(bhj|v{(x-}tOdq`I zT<|T)QzAOVi3PDAh^>bo%q+BvZFr{3(064bIP+%|947_J*TL3}kGTY$p+usvAr5B+ zmrpMKY^7t#*#BV02@DBbI%J0buiY3s#jx|tKCOfy_V$kA=BC{WX0h<*XAL3Gtw@DP zDob(GV2SAp=)FLR`>T91Yj*#a6eI1C=we!X{tC>7khgRVgqwO|DVvin&`$D~QK!2w z<&s?AX133+ApMPeS#yZAzl$z^i zU2n7bTsT01w5q>w5R~+c1%H3HM1FK7r}=6`=rq7EE?BFjDDE7Z`W1vhN@M!{$W`ve zi~X4r)2sD^lsYBiwF!dFzW3e7bs$2ndx7GRcEd@9%!qpgNVbybs>^IK*72B2<_` zCu&tfHm6z*S5hQypu}DT*goYsu%5cVh$AbNfL*lKoOt_3q81SQmysY`UFwY82+|)W zAs7ejj6NI+mHN5$&5L9~{)N{(%3nX#{~t8x|M8P(`j*K5VYy>^sT)5(o;bfHB1=ay UFegii+n1djFaD##J|OA80ovijfB*mh literal 61753 zcmdqIXIN8d*EWo0#0KbCP^pdu6$O#r5|mL?#OOGJ(jp=tHT0SyI>0zeQ3Rw$2LTl! z(tDz0Bp`$UfrO5c5+Q^@LK4!y4LJ8R_xs-OkMDbq=Xj6f<5$@Gs_R;7UFSO2+7GW> zwo%x;Z?l}7oWh0kKmQ^p_rrvo+&Z0&8-OQ~+3W7f$=#E?@bl?kBVA`lSjnn=xw=v@ z1o`z~dDYXnXNY^dPB{MX)2Gj_c-NJGmUFB`qz#HVJ7hsB56`7}Z2^mJaS8Tf56LEqAY#Q13YrS$E!>qT$D zeUj2FGFl)5NeqCM0Dn#l{pl?$zdlUuXkEAb>%&^->zcA@l+1YL`RrZqa96T+3NDg% zW@*Y8w->~(BgMHigs)k7U3@!0O0-UB_QfAynnZ6%y^-KMniDooD?#mvBMMF}QS z=f_Y2TfJw?@3qFs@hg4rRm0%pDztnh@K!Wj9HjS*wRK{=BQHsQx%a&RkZ6DuA0ZRd z=V|g9JFNvini^hz=ocxhT^Za%Cg?P$kx;^GtNm^md%Lf^uo{4&IN4W^XrP}ZE0^7HRfI`|)cm4}F%+wb z>*o75R(wailnfRg@!h)8=tzu&OT}ly8n=%|iJ0@F4XmVKUYiMfZPd!^{(~7hE}LP_ zSk+N%u!liXqm}x~^Xd-;kCbg<#k*oHR{ra&?+jb_jv|0?QKQ;MjmzW(<{{bDpx-yViU1-b;!Jq)~ysKe8G z>0sr{@%U8ln7Pebt|uyU9it6~2k&C`wtBg`W=F7|wAkHufW#kw z4(#~4gHxl9!0!5jo4*1_^i=;XzWz}w=_92CePW=fnM3=QE3&cfnr&Jr4P30_rE-X0 zB=h7%Z(U1O9;PNnQyl4C-yt91;A3FAv9EXxU~`Vx+`XDfRqXq;$Ea zVs9%h+G5zHzcb8uc;_659k{eWCMD?8*?N;dtm?BlnYV#We9^G|UQOtFa>@R?=OCmS zYay?novqvoA(X>vX@MrZioM{=F*pA{Fn?c7)I{j`CUzY& z>BO{>&brH&hxK(}gHpDZ>^Onm;+1Ero}?Dlm14MRW-v|Em$KqL*2p`MTHN*-G}st% z1~Iv)bPlX&o*RK`D9O;+RI{`C%^goo8F#9d+fM7~UY8hF16%*y8p8|y6(%AORf3;I zZ?}*V@+0On;s#=1joPY+Qp@$1Vm4-#9U#T0d8!;}r!78#BbX|38F&GDSUI3+|K%a& z0L$R=U2oA+X&!gda|3(KOkJl`w_#_J9rBjZAX#zbpQ35t{nmi?5pVR(08X!DfZYw4 z;F(jbxKvu8(b!u{ejK?yuKHSAJupsNV%(lZjs6|5W2%Y}ohnp+*rXQUAbr>O^V>P3 z+G?|GLZl_VUWcRve7@E|yP{>EtMn^5p(5%qZ&dKJ3GSC0gue>tcDhDeV89uls&g{dZmF|8W?HU9G6O5On*#^(%Tb7|t3+gvE;0B8HzBF5l~Na+Q)~ zoT%ZdrmSVMo%T6Rn z$6|EFbaU2kHAo3#UZcI8Rt1%w%8`B2ZLUMhEX3k8hn1;s**uDh>L9u!)k7Aa7ZIE- zrxfJtjM^87o zE~@e3WJ6!LfTG4kwGH(!wW%Y%73SmND#{t|C2no-id$0X>M@JQ*ZoWq}9 zIH%6}XHVgs+N{i2_|a#{50UT=kkKCbSIM{^V-5-N8mk-istfjbmP%-}n3~c9C!0ne zeGH(CqL*-ZBPc0qhy0((I7JneW#A=uFZcM=q@HoIAgZI=)(wua!tVV{C=2+rzn`Fi zSe^gfA9aF**1B;EP3r^~j)1qTtSE?FVU}rhRvBqmYleZ!>UEoYgxKj@dTjMFl@o6N z8$11vDdhi@d3?}ZCxFOAxK#{tt>EV3pMcMmbth=VYonH|m$)jh-;8yks{%Z+J>;7< zPKH>Tf1p%-;?-ES6)1k;e&otQ>^?fxqtPfsF12f+f0Ieup}Y5ehB;|!pdUlbS20`l zAPXfL>b4#S;NHR+Yk~i!iSf=TvlSD#=f41G`v_PTWdJAsWG#r3X9pF1?etJv8Cz}( z8_jCmAr%*aPZ&HKOSb|g5mV<@k<{_e{|5Mm9ls-^{{`?-Q4`$yEfeGJvV{d%D1CLi zSEcSzrj1+%G&%1nO4h`uB$QmxXA&#BtdOUWVn8dNG5?-~fx&@)K15b_{NUV`El9U=pz=3q3VN8^Ag>63DGEtgkCWXb9GfL!s-WTb%vl$ zkv`W2&cs^NJ|xDpaAr4&9H?OvWpwngrWKy)C)149OIeCQ6CRQ8Q;QkFls5uNn_tk)HaH2=n=yi z>x$b!PB(ZnRAVz;`#dE+%B_BuR#Zg)t2b_b9HZ54SVPOSX>O_+tHct0hXOTKqrXI& z#2oeat~$(`5WMp{a?HsHMlkC|G)EjUYGT z5IsC(VB&^fi?HRRFjHgHz-7u@%SmFvo|&m@7@cF%2pFSkTC3fs5U-KMVpi^i#93*= zn6jd}LpGY=B91Khy}6uQ>b)o5IB%n}3eylj zSv}%522P?&rj-iaral63k}@T|X=%|iF(oA8M<41sCQ)G`8>KfN7sy%wq8`;})it^b zN#@nCmc?lFp@3niLhN|;sEphqbVT%cR1uT1b*7=0oZ;dwq3IHJ40)T`HkURTIAuc& z;oLHzY-BsV47kn8uuqm<6k!YkOtXe(?^^E7o28%g`zH34pBQ=SKl~fF1UxNe<7w$_ zr_f;4%O`61%&+b>keYnKsGm;6`yg_F;1fk5-CzQaI!!T8?xlGe0{H#-hht<$W1P7r@vFPDwec z8byE*p^A1jdN;%MDaN^!um>fk&UP`5oIPT01f)7)mrp#SK`~BCg;2P=6;`mxgAk;5p&!eepReiJ*ItkatavQPQs!LbRENo+hC9Ryc?&U+QKF}tMLb0}g}}oxGd1oa z&r2}w%~R{UvVlX1+$^;+sqXcwP#>%o9YkELYjTdp7wm4XKK?z#vJ8Z^7HpST zLHlOxxkxOXv%*Hhvrq~r_huCW3dQ?XxWWIMm@!w;Ih~EgPF=Z{$`9a3oTb{orP0;5 z%A29ynMjW;=9g$>FwK;Wdj92g5|roAAWI9`4t7SW-Z!O$9T#It>|Mv4n-?5w6y7!oS`1toOpI|HYA7{q zyhPxgRdatA!5c_fD@apT8PBk3z+-haHuKdH206&-he&J6t2HRw5cp!1n{O6`2$7bL z&-g7aFJHJ=audqgFm$&+Yw8*YB?>f2W%6&pFUo*h92k|9-1_otM$jj{1I zh4{XM?tjAi<4L$uZ#8m_4}z%TSBwamwIS1dqLV9d<*GVZF&q#`Y2Y>eOR$^=9MuNU zzf^)#JO6WS!}egNl#QVE_AEW5EjznbV&Z!PwfmmO1+iU*(2abV0Te2-Q2|^CY6)Iz zg4`$m8lsfoM6D^Xv^WRsrfTgJp7yi;9hWGh;h}y406eo1ziy6o8X8m0MjTc3wO}?f zS^zy>h@@-V67Vz?74lOiYqnzl4*)YlE_&}YRJUM-wFC+0xFKdZ@5<9EAX&ZWof)E? zLdAZpj%mv6F%c2>2Yil2a8&)0lA)k*02lo2ugL@M@?J?JYD^sqHC&k$wBtaE|dw^zqTd7t&9!2{`}GCrB*Cwo3)2ZooG&E8)tXs0(P!?L9XAEhi+B`+;h z;(NGG<2-&^ua)0qX>cLK_f!gOq-#J!5p3u>*9UN5&;V9UNMxo@%vq*}rt3BJ9SCb9 zxHOD^GfbFX!V2a&C?7}*;yGL%Q|3XY1mceXEd^Y6@CW{rlT!-$k`k5BD)*Xh9 zjCzAN8n~sSV?3C+&i*uTA}Y-&e0kZ_W%XVZhn`25l?L%h(srgk@R&qjz&`zQeEt~LayS%kA}0}J1If$;e0P!UOP4=Q51tQ-eDv4fD0FPrNof~N_koU0*4{CCmX_d%(ku7%l z3InYI>WJ3}x#l*)H_LraIpM~|rl-A0*Owuw>JsGv?JrwFGN#iaJmg^8MAf zLuA;1V+WdC+msnGx8R?A0_T}rG``&G`2}FSfx1|x;mqO#HyJcB84qPDK^yqpfm6S@ zVasx3;G{S$oS`6g5`bG7N^uFu;1bpjpr_oUfAG}*N7CiL8w)K@B==s<{8)+%2wU%b zj~$(ui2$|Y^#dCxV!|peuQa+>QeEC?v3X+bhD4_AQ0gp;6{93$vh1+cNY$?ZAZ~nd zVvpL;>7FAgs2HvFY*v$K?i*xHPv`0Y#N7{;WJ_R~mY6LT%+I|2cgCiV>Y}OfNliF0 zo{SfB=K_{Sb>{U9+@9lFiLDg)kTQ4I?5Zdv^ z-Bm4w@f=!|$@6rK)YM?qkpg$wHRZ!gM{<=W$I3MmAIJ8S+ym(enqro)(hW&l7=kPDF9odYmt&{zAHk9 zxzb(k#xoh5p1VSpy?iOIUe6Bl%lj8`ffQwrJw78?dIeX-9-gB7O()#RKyi!bMPm{j zF;zph2NWrjF!X^emxMr&k#kp1`K$oY!FdV>Wf^@@D&dbhX;dF3RvMGftP`{?EdZjt zTK!#SM=&^PbR;LyxY-KkwUo~Y>$(s8NZIa&)hZA46dzN;GbcWx`z zR#`V`tw8(lRhj9fB!g$8CzVj!zVl@E|6Rfi-Tr_U*PemT=0tKMO`cU%9APpMw3ne~ zP64FYz=<~iNQiB&?9H1H11TKU^K@_FS0L7Pm!^^VlN`VFggjUdm6VXsGaX{K*c+#S zvnll)5U2u4pIjd6Q<37XsbvkhCj~I+_Es`AFYpUGDWM``YFcDTdx zDeL~leRMK#ra@qleNJEso3ZG9p z{Nr~q*7^X~tl#lF9$JH6>dNWzw8h{9cTs{T@4>lR7vqcTmKJI7yfqD3S0p~^xydVZ z`90#H6mJ}z zg-pZ|IO9*^W?}-AY|uBeN{s_pA<4htp`ZDdPDUf?DvBjleYIs*hmrzDV#T8vRYk(r z=qyoV{fIW*b*t7Kt|<_7R@D4bJ}J5LC1s&5bbk)Q(_Lil8@j zsiA(+`V)oRqul%kmO-Iz2eM>3IS>kJ>f4>y&$987BF2y@03`xp`wkkd8@9Ej&Biqc z1G%I)`7C_-!TsN~>6_c%Il3|vFvGi$jlEkjMOl&>Psq|>va8>s``p)QjmrDnM0HH@ zmI?Csh{f(5ZGnb~+j!}HNY)_1SQouTYfg2n$o3y4~7uO)Gg->xtgwyy|qF<8`G8{;1%u#SQkK1VwWf z&j_;d)}%BNHEf~ray7;m<#k!sV@dn91ZKl<+6=zjnJ_@_nu+1n3a6B!)fJ2O#fI6DcH0Ot2ju-!5DjJHVO6nkBR`_8QVmS!AiggL1#cwA z%=sRV*q^fx%*C<3;AwH5F~vh&TmLYx;ZFj2wIFPgm6Sp#q>p``5$6|`@@JJ2l8s5Z zyldiII`o)n&D%Ahje1@5;hXzd<2Dgih+FBjI6vPm!M7>nsxQ7;*5@@DGZDjyfzHgQ~P zS6J4adcaMbf><@|zH4aR3KvWU^4^G=OY}Z8(r!f_iTbf=?+XoNyfTng`Q9XWf#lU}sV+#h(|}$#4>yUXORqj};I}k1chd*`D+6}EiWReeF$tRZ za%6ZYV*X=bJ@+rT{ukmp0SBD9c=bo;q(LAntfGJ~*Hsl}6i`x|$P$00Y5hU!gVd8i zBsN=c?(RU4wR_%pnvVGxF*~$BD}1SLIY;eHZ%(Z#X&M!t?mrlc6Gfh=1O`pZP;W~( z^#;i9j;H0YVg~6Dx`1j@!&7V2&ABU?#0Q%oUX}Z~Lwk^>hM=+A&*A*CDX7iC(^1z? zq`;IsLT84EZuflfHdQq-lU_o^S9;=ZoV9<^4l@sC;%P|ca)p}f26?<_ZmR zPLyEOf7eMMk3pK3T&7JXZD}`*dI?14%WGMX`kJ-Fjgs)8s(TTA+K|dTPf%CnK66-2 zIj-bZB=`An=B$mc&_@H=G(74|D6T$XS%9bfRklbiJY=lwh;q47QG~EA(ocbqUY7Z2 zj?v<=v2eHCHs~g23#3ET02eW^BT4$sLYBo_5Kwg0l*lLs#-TTa;DZ2}VZCx6rhKYg6-3d5-V zj@^?~INpPa{#Mt5z2lF0vw=|}S0nig25M4oHAbK8%&@I#D8~_tkTK7B&q-IgwMJ z0(pJ#*Ux4I4$fZYRxm|ws!l_-t^jHuI-?(Y#V}=TM^dHq{{HO`cuVU_rlC2*=?Nnz z76`XAT2Tk<8gUgxMJPd+H=^!A39h#7T@H}nk3K2evI3c-0UCVL$C`4pJpuw#Xw!>6 zRj<}l2)8yFza)3lmL5>@ObCApRe_gL{;U5RE+USEbbU@L9OAD0pG$!sg6 zj)O{hrz5gukwWgarS;cjL553zs74GG$3@zeSS`KHOMyJ#luoPdze`_Ut5YnT<}G=9 zF{Y_7!j3V0-pr3kWYT<|YRDQ3!j9XQ98cirTAJ?*n0Se`ru5#Qam2CeiVRDaTH^`= zr>1T2)?jiy#&2l145 z;#mO&StwLFanf>cH73AyS?y9@ z-4go%Ej?6Y%4ppA(yVC6G-;%Wz#%;&yW^z6Wv+^gOFr85U~V10>>avzUxbxMZ^WV>2FR+ZJm zO!g#+U+F2L7!Ri$V-STkMY)!{VQXi%7)$*1-{+Np?4yLr+uL@0Eh(Gihbr2JT#tow zxQiF~cV54k5h$O~joUHqUe0g9)7lEAkc1bw6(6c$XDLesE{goJPJWzIa*m&PDM;x4 z!7wePjH`YrL1FN^;L&5AY2?B*en990sbopqLZ1KGCow~s%q9~+mJ#_=Wp+NFKHDb$ z)?foxS3x6D2Xnkf=l6i0jLqP;XyIvhRGbJg+@|1zxr%^Pth8dq&V&1X!9=Y)4Qq!% zp!qdkS>Z|NhOja z)7S?}@#MCs!Nv!xZn^|QOJTN^FiGM)IOjqaRpF-*7SrMRpi#>5Xoi{&0+K1SUy2C9}O2Bw9 zrYZZjSj_A>Hjy5K20{_TDJV#}(?Swx5^62lB%Df)cXiRk`H+}EnIIGJU$u+yNH$ur z6857({5qdpR54>%lqPzLtq1PD9W~}->6!F7g#=1<(cYhRsYAutK{|7$+mbA#a1)V_ zXz87^Ba>64{;pIz@4k5$%Trfm45e1encn9pb)Sc z5U(2mO-DDyiep$<6<28uYKx4+lDVE-VPrm}|8ZJ8hTs;{ppcx6m^`qCD@wwhp8`ri zfVzj6XAaPaSoqF}Z>h;(MHY_lx zoy+w|^yk(D3$v*nJ#><3SVu8t2q^OKBPM9Tw=LI|B!-)fK8$3P)SPqY{S~{I-+y$a zYoOub%#aSWyP_KB+jCn2+-#~hmF830UJur+hhM@~ zv$g&(mo3<6tZ3+m^xsVv52m`_*`one=V*M&&Hdecl|*xTlWAL&$+aKkSz%ud>|I&F z`@KEKqq{fk{Z6)U&wZ~y`Dd~4|B%Z5Z>7ukPH$jqC26cuPsL9?NkjhcW5qx*^qM`} zs`lQx(d%)~?P|uFcpJ^J)JJ%>+ zq_En~ZoL)R>$y7Xa%1&aI7H{R96sM@b=Kv^^BX5(g14{RqyB$siI4pkmegqaXu+kW z4u!ETGV|OGuZhm`>PiuRnc;lK_(=Qbiqzd8J@4;H%aghGQ}@0wF5Lx1HJ-FQ zltLH&sdKIOb#9#-vUB7yi6#@ph{`98^Ie`N)OTD;Rf&7Zm@|(!YJ1le9gPT2Ansol z=VRu^3Cq)|ci~rR=Rm6O_xtESGDiGScq5x$dTnh~sqW0Fv$?Tw8@rLq4gyp7iO0#J z^z|%<0i6z8m%{eN!)~QxUIV^AbM@=BCI-}3PJCO|Rx9pLpDz~}Rj9q)-~MSTBg?#b zJkBGe^n^3C*W-_Uv2bqNU*F84&B_>#YaYGQxz@)GU7CqtF=otbj>pEXsK_}?*PW(T z3}QZD#NQS7p$}bp30w=2S#Ap7*Irs(18NFd$kEY%g4%SqX6UZ|NDW9t7ILcLE+zh> z-^_F6T0`CAA(76Yf$C|lrfha-nAbS+3;6M_wdI~pRU~59 z^1@%E`yP+q&Zks@9#9gz{ZLYulCVBstA@Ex{zov3uh(6a%6+YTo!F$WlaeDP`hwZ9 z!In_hyE~Sx;GDxtSL#wqBWC`zg_2X&*?R5PqloSO8N2&Q zqV*SB!3l-?xFx4Zp=xQQKjW;BLet9KIT0vIUTKXHv}8nx?4>iL!&~iUPF?qAz8UV< zPMDaq!t$CBx0^r?U@c`zS^&KsNqsPA)VEIHsK$DZ$t1nAg*7%mTd~`>8sD;isEgWu z19!wQJ=^)$ix7w1*w+&p{X@#}Q_5Lil`|<~@}tusqdme$5FEtSRwpMlEDia>M^ZHv zhiD=$@Sahw&Q#nP?a43=SFg%s{!(9-)i--X*9q!(QDcciEe}6;F;*Np;AwZ9;(B8n zaAwW);q+O3SDesZ@!@7p5fcs+pm2=Xj?N-gSLv{0{pDvvCv?Y#tLt#SRuAegxg3hT z>~o=xvU{!Im%=V*ss4!Iw6DMv%u25*+uzVRA$o$^jZP~ufNCSO&QpK5wxY$)byHxd zc^wzk9x7XjUO>GVdx^>*`p;|@rW$RIg)f}>RoruOfFF&jnrcsXO65TiG5u}5p4E1{ zb$0pOJuRGkDYCUHI0u=zt^fS!1&sWAIlz)`5-+_%FGN#n#BT^}sOEJ>aQ$dYW#ew; znZU47)3t&bqn#9E65VspwTx+d(Wq-=SHL+?%MB z8B-@_ukoVywbF-D5bC8J*+hATnNv;A;!~#KLJOt73Zea7bK>S6VNQwfwNiOzc(goQ zzas5(;{ZHmRo7i#lc$yG;uN5mDT$CR1H0(pUR>XdJo>t=+FP_%;Df44XLNfFnDtKs zGQ3uij)bvg3`1W>?QFx7I6)TmbEcu0=>F3LllLxw3o*k#JokHfw=jLfh5T^#`y=GC zJV1yb?PNQ2M`M;1h?y-7!!6bAZRz!xN~KN&C${o2_7cqK-w+~^oJ&JVR;YsDcpUMV8$+UxEfID04xAt78%ogR?qIevr@oi>pkLAM=q zBNeV^6Lqx&KM0a$FNY67H#;_4jl#%=-f^t{ts3+Eu!jsJl@u7XBvB2%1g# zB;G>4GbH8n=f3d^elW_czjXCP&iVVP)0K&RHH)0_#frNzO@ck_sIF9$YXGt|pac4j zNvs}XbPkow`nn6m!mjZU{l{kE`aDMja2F6NRgr#<1@Gx^rRoco;2#pWx*53Wa-rYrBy>8?$y zegr)zlxI(8yemvZl(i+Rx)*n5$3U37n~ z(*>8pyY?=kOVZ3z2*P*6oSi;n`c;xWD507PT~c*@2ENzh${$>89GME=xf2ZI3x>D) z_IkXn_aS*5Q;L7~Ycv1psDGy*Dq@QJBJ0pj&?8cR@@=D^QoJs&Sl^Ld^KTS@FJp}8 z;-7%{v8|qBXw7$QNUOHpFS3XgpMNuU5o#69yBSOij~yBSRK}&^Z~^!2`~*GPpRsv@ z9B#uc`z59#ip+B>1N5alu->aF>*^Sr)KoxnZ}z{F-wpE@sDOsxm+CU+rpi`?MV8%*iv3%me)IcBg1D+7_9+OmFM5WeeoVU zBw-o|J>{eJBX^iCWaQFU*4r& z>i+*P7*^DO^*glhnT={tFiL?y*WHSF^H1Dq{f-FaE~EbqPR8HJ20n^e{PUkEach|o zkPH2G_MiB4?`{9xt?RlT{S$WT|223#MgE2IKhdPu?^VE8PFA}A#IdApz7jj47Fw_$L@ zpEA0#U@=N8M%ysO9MTqYb-zHSHHQZ$aN@kJtLK@cE$)?dM#xd!Mw0y9ZB?fG|0$Qu z1|4Id9M-%UC&ivbX{;3};x@_}_nsF2Wfb2;^=Yl2XU>H^rgT2CLy#ViC5$j~2`wa1 z9A6+ubjel*zKQ%Vah{5BDUG9#Zwop=bE>j40v}}V;sqr78s2eP3g?@ib6Uuou5jJ$ zIXBq2{FUXujyk=L5$v_i1o>0=Xi=T{&@c`7OfrBrG(z9w|0$-B6n*n~oiArC@Ri4Z z9r{#hd&T_^7hl)x_PJbDXgmm~Pn*wh`r=`4;wY&_Ji=V0P+5}Db!01%9Dp)YB?#J# z9EOMqb2pTtQn>~qLuN&&O8Imj*SI4F{+7@X$T>94S~THqs`izXS6hyU*$*c7aJ=eA z1bTOMuW1CoX&h;*ewGeo8k*}X!oOOix%ZCdRr411;vjFy*QT-Wytz@678rg+%s^D9 zhyqWgQbe!1pwrS<5^Qwd9pYG&!Y64BB# z3fCL!aP%VbcvKniK?wvn8szdKYPG$PK^6%WLRU)N^NBJNr2MRQ4I#h zknY>4Y_@K=M@n@?R}PdV9~jVkY}U!yD{+}A5-+8bMDIvc7|%dhTL~QyRYs9t(Cer< zz;2q69t2!vQ%F{WkuLC&?0=b%$6UShX0hYbd-_|ZH*M@-a@&iZ zO0;2wcjFb+c?R$f;qgtUd>Nl9*`2A})9$0U!9S0O9J(m7M3mPJo^#K{zJwL9sDr;p z=YXrEexH*rOjg}-^veFk4fpf1eS_n8JR;O|MGU|{`P&8rht8rXuBXHD?8hcWm4^os zKJz-=|C1Hje5)#yy!vMF`|(WU^VwP{5cI7?A2&zquz|M9n;)GofMBNLc6vMSgo4!b z1p3$pw&8`1BhfvL-8SP!zMgS1yH8Q&pt~K}@r)9G-9;N(;l=63c1kMmtQ*+oFk>Gq zq(W&BVjrPO0J#NBTV9(kni0SG8odQ!V$^^mqlyqWp*BX)prNNAX20xR)BPEX5-cN1 ztUY9&{GTnjspndE+(`^9n>k6}G-2y@LtX6gw*7J;H?r9@mE@!-V++a=+Qd{HG!Js^ zXo_z2b;khzn2Ms$4FR_ep7HAl7B@8vrtH?v^Gtu8DQ;27Uu-#ZRxgD$GyXMN8$fZa zhR%-2|VQo`-)Opp^dF^6UZ$X;AxhCwZm;1k~%y{4bLA z7_E1f1Xre+VY~dR{0GLe65*M{E!d&xs9Pt|J!lTu;|eQ*(cc(WCqlSx>=6A{HEzU7 z(N@c7_%R-UHr0u#+)%9i7STzHB8L8Elb~^uV~)n`e3edkuS3edAbLw$*n5+afszUa zjjE|n)Qig5ApY~Ba%jo;lR0YZ(v8D}yXC`z<$cxLP2ZTW1He7ya&X1luFgTmH1^;h z^hla`Z*~(AZ7+P^;~6ws38+rHXxJV5RoKHnQ#V#b9Sd#ppI-3B0EuOpcUTNvFT&q- z`Q!GL+wBz0r4yTS;>?2W+-<)|l-7Ro&%#ep2aMPr!78Ev*D?%!sMF%)H|x#L4(h!qMlcrvD%~5k#|R`#b>Ay8ROlL#IJpJ{mqO^-qq9Eh z*%rMymByl`=a_Nw8M_=hBA4U@!f^zj-9q=l?bA=ZVcJj^yc??-8gn;B@>XG>C zj)~d?g4tL)9QCEwWN{hqr+7|6e@7 zR7v}Qa{1XK$o(#l6_~K6)N`_^*#;<-l_OL&4TX&2PR&%_=%4(+mt_igVLzHgFN3CK}o z6jfMGfTw|ua=oL~TRh5nn%yi&1(YjZ%15~Uh6K7$&&;OjSd*9pS zWYMbDXZtg^$OfzZgRJsyE(-r--k@I&lyDi1VSw@~N{_{!GLPfjDmsd^KStE2d%Z6) z_y+_7BwM9x!z@x~K+6X?Bip51y5-t&$JaTa6PeP?a5t6mpmXFR%2SUXiHj)0;l1l7 zd0)|$Gz+!8HN`)+79pU>&s^r$9X{mnV~_=hy!)3#NXgN`Gid&`>6*)eAXl(!TabTs zHV;2LEf8S!Ecs=TrO%|QU?qbde+V@V>|a|ALkLT%#1mEAkn%vRQM&IORcKL6cchHQ zFV0fa7pKPuQ=gHwbZl6$gN*UX9{%kzjp*k!I#9BYVSKJfsCuZ2Qyq4(cqlt0vESk! zXb|ubIVjW5*4=D$UWRLSykjo(^TjnDyD!(Qo9sbf@`8NvU`~mi&apbIu={kbuVwvt zW@&JvK1c5T&XlQkLy_)S@@L*IUXj7itZjN+DZ8l71oO_31`ufWTpSy%q5BviP15Jyd zN#&ZpKPfb9q21ze4fZhoihoqE$wQLTGqKgx3I%fh<(6d01%5=K&$mHv;A^h;?v&l4 zN;r8x6jpCJ2_o`~PK{lc_no}g=!D$bPK)x+-onu4dHD;X>;gbs^Nb!&K6@lJRz$r^6(1IZ6`NF5W4zE4?IjnembL;(T8>hhY9gZ7y@1GiT}h%7*_dEZ zN295|FKk`8%Jp1!NFi5v2dXW)j%=cMz8uK|j6%+@=^w!c;Er`Rus^+jqxQ@7b;P9i zrDi0PoyD2Id6ZlUY_w$fze{-Kkt&!{+)9>5xW89sl6>#di9$u!=YtCgbr zL%Yk%I~q||qwuCG7-Q%;rUKr`7F=W$=jHc>du6NVHBTVQNDowqsK7&(xbBR{^|$o0j2!DLZ8Bu$vezdUjkR*~Vf+ zYFKg-!N3m+R{S)c11wdeeR-+Dpl1L>2)@JgzapA)$|#+0d|3{iq;im-3yF@<+oSbc zKTuTu5g&3WlLl7VhT-gkwQq^WQBSLOQVRU-=xzAktP8JA9UPMPm+i|&!kofydjv9UUQ7Z5ZEF^vr;n%h>DNDo8s zXjpE;c=jcTmhtmH7%F3TG6%p_z>3`Ig zerjXb-g_m+|L!$!YC3F`Ts-Zn2haYkwxYT7%tnh%Y^T!>TibiOV5*K|-SDhh!xJ|n zINlQw&a&^^AIb2G=(MWY8IU~W$KuqYQv>w^-1oiU-_&7|+w^?-F5&LCnwqAJNOhzT zlEx(cW^|(uCynYbm+_9MZhU5H)9=OS2s+>h_n` zbu&GQHEmJ7ez5Qx*n0E@+!UR8yei9(vB8GG=rbZymmLk{lBiN7e%0QRJXHNAFGJ;h zroY~$?5M|E(4^`yN>hTdV|LBczoQ1Ay;tn;`2*tY^A8rDcz(@h@h`5E zS3S`^@kIG9<-}L6rM}LVqu?jyQ9Wx3NieVL8beb>DqjPL6-{YT2yn5 zC3H4iWM(bP4Om~iBcE1hVd&_qF;mPywBC>QhQ#pP!2yk)ukk^r)N0(YOLp-N)+Mc)uyVQaES`UgaA;%`J&hg1s_=8UBsC%9&wxKQ0BA z>_x3nmrp^QVfc~?ti@Z|R|q zXdnQ+lyLNIsqsKUDE2Gz6@JYjYa2#^@FPl$|KF9?akxxL_-j zLhrsZf#}_`OOC-RoJbAZHS{^z1IS2A(z)LCN+WOS?v>pjA3KPsFtF2wScfufPp<|y zZF2Sn50TC|^>Geq_?*biBk3bpo|25Lu*kf9od%s^2Y-1hj#nI=}WmYf|0=n^o zeTIKY&9=4a6esbM(S>p*?{cVkBg-T(_iDyP#xehEqsfyJyPBXuhk}z965DL(LGTUd z!GvZasU-hM;4i7snZKR6s;}Q&G@S0Y@p)qJ|=b5Ft`R3#fo2 zB{+_tQX-;)QX(zXBnq0Kl*j-H5J*5ufDi%+NeCgyd*Sok_c+h(_v86~e8=&AKhf*B z_P+LBd#!bz>)b1iI$`1`$6;4v(=hR#Q|%7N=#OCR+DY4^6R5LfCY@D~K$e`o)?L4f?gN zRNn?gdCpAWTOYvt+I6`@_SPaw0U5~Ow2tX-In~GTs+Veu&$rYDUvX0$BNixdC;p=+IQ~x!gM* zeQ(NDv-z8L)Ar1J(h#TJd|^jG>K!kLG>?lzvaZPXq`1Mk_a(|!%a%0KHS{Fw_a2(F zx%)D%KVVFu%}=8m!5zC0e9#J`J}7nz!f@B!%qb&3E45bYG`-i;bH$8>DG68n3m(+k zp2_@WmT{V8b014s;Q7b!iv8H;m&mtk7zaFAOR|BcTusM6LU3a7XVdWIijl5^W;gWe zo`tWuX^7u5-{Xn8PeWhHWjurI$*6EW`Z;RLt3A7<&`8^kOpA|qcWjrvQyxCv zNXxN8Jy5o}>~Gag#ZXbA`X~cm&N(RF4DdR~HrA@ig%=vgA2SG?soLDp@$_{wC=TIS)>f=g|mHVYm{a_72qL`Q1$=B zdw5`XAoM`qP5e7I>dIwjE(9U(hp41*jY?7B9Up8>Bq!8gOjZo0tN0FQ#~m+k79AB& zX|*qz>-%2)V+L8Jq9%NAZa9SSScX?qq^h}LPG|Z)yjA7=T0ojy_|SC26QIH!NdfZqhFXD&;YiFu+77!g<=4M9>e>n7sTzE?Rh-RBrA%mfP>h3Q@iwwui z3oh@T8#_g5S%Ix>A;Md}xZREkE`D2Q+HV!%s?I;Wpi1+I9~`4mp3|K;r6}5oye&!d z?sd4$nyK*>6P3upNsQ}KNX;Idi>$-H=ZW3tAd~Jrc$^?(OxqgdNm}YBE``5pV|t3Z z(8yP`hak)MzJ#imHQK_i53KxGokGFnTG0{fhkBxxnEQvsuh`|c-q_&n0`gCuP;;~ACym}Wg}*k9V(G|@lwFKale=LCSgl%{^Cw|<7xS9XxKM3n z{p}AwE_n@pMkEho=v~3i0!(P)gE6>fp~z1i`35O!HcAN^?$zux$-kRRD?ZIZ0IBnIaOS&lyqG-yT82y{u{7rA9>eynJWcnjGJ;1V7$OGJs&`9 z`M<-D+rqFMKeOr>^W|%Ew&mnm8yj!2Euv9epxtX5c%^Y%$*oqg%h&!42qK_bK>4P4rC`5`his=5ZP-Pb{N0u*Cr4E;Zx`En6>H1C^+~7rH#bkin-ODIIMt?!?t6j3 z?US07N^o)|D;ax@y+hW07*zK=aB3m@_@w$D+VSX3;~VB;be@o(@9J_8?;?ToQQms7 z3EkiU^+3?UtUB?7ncBPT9erq;w9vDpHL6I9H)I*6JvMCNo~u$94~XRU6-6KroGS$3 z8aZ!Rjltdx{DCKA{kOS)RGJdNYCrmTs$VK>MvL_+z^tCCZF&P8VZf z8S7PUaY|e(k1$@Bt$Xsv47DZS{>1*U}5U;N~KoD_oWgoC2y}NJid8U~5$=9%M2w~~`1W>Ea)-b(z-I$^q5!h~D))&zy)@Wa2_#Dz*NvVs1dDUF>{OpZ`6L#enjnCZGX1aTZDk)zsKR1w>tj$k zex5yfomI9RfToH6#8N}@U^$e-N@)}OmF$THKguE z1uCV@^eyg&)JG36T}e{h)9dXc5nCLTEv<(i9)GEJJ#5DgNW72}E1M$|EaCU#XUR==TSHdrrpL*cdtnYOU0{s`IL6#2x$qt++1%4-fmm5%*_4p2=_#{>x=~jvXwd+cuZHOFtr1( z3rgoCAF_NHxjj}%)4188H$bi}88|u(iqr|4I|o|yc?8bYbyXh%ebt^)7JZ8O$lOjf zM^;!-=Ux|mTUaV7dhtc;#4}v521L;jop1vqyeLu4Jp9)qMnYNMNW6BPck3V852Dh? zkz?hGH~XxIOv*`IrjBKtJ}FS*+dKo={V4TR`53n!Gz^}a1?OR01y!(85~2^^6PANa zt7J1#o*I>OuDSbI7HhbWaz4|C5YQKJMdmUhb)K*hyCj>&SxDVHQql%PJ`H9Z! z#=~*?2Rt;dLlVwq7GCMJ*V@~S4K{w`D==Np6kXTV$#?9ua7bdhA}2+S!%!9N^uU?Ft{cA9HeS(UGsw{V{=SPBO%qQ`9VRc#tP{|U4i(Ty zkW9ujNTG*1d+-p%I0$*6Xb{%XM<=|4oUIrTjx8lTUOvn`K=uC(g)_kmAe8GD^X;h` z6`kbOKE&HjKot_b0SRta`wgGSmWpL1Z@rx%vq%qNe(Ukp-Ej!^Ba|6&uY7I55@n9H z(M11C`@I97d!uAC5*1p#=hZL7VTaBO=|KfKl?;REbhq|~Lh8{fU)RIIMkwJGUCc6; zS!ulX%RDcf-ML`JELH5o47le7Uew?k#Nd2SiNEl#bpmtU6Fp}*i`;bB`0F$yLS4Z* zq_Xap3DV~8ZoA&>5qZSH=ivf7^Q33vwP0JiDry#^hOZR|uaAo%{5+dWEgvf0ek1$G ztHn(j-HvKk$YK+!U37YY*9}Xfxhzguj#X*~qIrY7=(F`^9Q--I**g}#(pB$Qk`MzD z^KYEjg{@BOc*Xfv!O!H5ywW}=oN4)jD>DF`8Eg}Jz24ML{@*_3@y!uwEzuxS-mnhsE_uv@ zcdcpgEfzi`$P==dX|=vFp^;vlR{cI*{V%wWydZCEU;4Iky(@v_5dBjyWnF`I9T3;P zG0$|$#EeqOUk`QShOSKRE>jNW+CL=$_q?m@rtDJ1@zDytMcs|F$sVsOD){F%yg93t zBc@UgV`B21FvGs81JOg!i`J&UrMiM3u@2h~k*e4Ap$nFe%skz--)ZasQSDWsScOb9 zdNXMTl}+n~k?`$az@NXxJ5nIDm-qmgh(y736B7z9k2W<6HcVa}Sl-_{Rt#QrkE&?r z%?$kksSz3u((9mQoVR%HiUl1eqFrLEuxi;}T@X8AuT+cP$d-JRdXWg$DfBQ3_!18b zRO!VUYX2~7AeKl*MN3ba@#Zs^Z)Kexxp9mCu5;z{3VxU+AA(*vtlSJMiWn#j48YM( zh&p%vU@3-7IOn(BU^g)`g-an@M&3XA1T9maGZOz5{|H>U)`um^D7k!Jy%Dx|Fq|2X zTzEbMT%#rK^6Bs9TdwtmXc%P{+xF#8y~c1Gh}NFtB7Mw3F2ZDv!-k(O&@fXR@5>XK zXef{CaTV?bofAmcWWYgaI|jc2i%EM_WNqu3o55~0DV%bgi%tHA>H7njmU8MtzWg7{ zH#7j}&UHe%5T0o5ljHPV_rX8etZ-j}qyT%Tfgtl|Yc2jh+Cai&oF$t}Zsdkqjb+*} zo%SPg{zPmjXl9*DFmhSX4L{`8)L6c3a423SchLEdsGP@9`?I6Y8$hOPLNo0%s_T%@ zrTzw;Hy>cQIjBxY(}qpsNdv%$6d%I253ZBo9Suae#|<16-yCK|=XM}%O57G~EH{3{ zGbc8w8K3K%TJEOdrh63Xo)Fz5&fhdnB|Xzo>cd-xND0W;MQZEpSyZfYStt6~XA~<; z3w3I+cIp^DYIe~&;ESdU`!)ZwM2-WDr<5Cf=R%-;?KZNrk8_gyuRcOhaZ}$U5bovW zP5`Q^toG|Zl)`%5mK(9LTESjP&HIG#wWF~L+-^yvKy?B(ZBSV2t;Y$odz;OIa$pbM zHH1Je=*TyarTwO+aNIa&dYQL|4p_YPNhw5&uTDxkD4p*E90?F(faYfRKt&*t6Kna7U`-~stg<0$yYbdva zBOP(j`TkcTebRF(GQJsyPZS1y1xJQp=)P~>I<$4X^yy`(wBBXi8#i6TorVeQk;EJ% zW=^e5MQorBDa7`jcvlPG6>BrFo>4_+wCzzXvCShxan^@l`hJEP z$s-!DDWW_N)bJ3ojS%+;==Jg9yDv7*GCL6}@|xCG8+sxMSyR8l4pUQa5A#@q9U_9+ zUiS$QLw*4$`dx#hUgt_XA1kc;2=(`>KxDiGn{>KrR)1FcK^m7Ry#Bof2~n6}pwRoe z*9c=&Vs~Mk)`SeFD2_*L!(%V5=hC7VVUMEZ5iFy$p*a|6#^O`c!q%AdvsN=s;d@sE z+6=tQROiQG#?!hCttNdJ*K3lrR8#g7?JJ+T~x86rLcnGv6lFXR7pHCnuI zdEtQt-Q$UPivTMDn+;6rQKJiyD7>~_lC=u}j%xc^jufNI50sYqUa*6ZI}4VE!4JzW zxatsds;bUo6f#`k?$AN1dRl1vtK0OrC-IA!tC=%rOXL|J!ijiwX0nxXZv6Q_$QqNC zYRo?`NzPh;n-3zib4kKO2;UlFno5)p6u;V$_8TIN+B+dt$%t&TVi#i>(xToYaplrv zXWg9!_KwCkj<#GhTp1$zVS7e91KF}ivbTA?=CgbgW_9i8;tt#FE~$;=Si+W9{`*5^ z?ZutD|132k^J;~a?*T9@qOZ(Ic(2#T>opMg2yMtb`hk5PO~>+U3Q5a0Nz zmPeg^6z>rm8Y51hUO$Ip;!HqowGUP~hGC*URk66&y) zQ>hEzbirTdP*2-p*9ubI$*v=M0MNcFC0GM)OKdhv0uN{eqSV2^RN=#O3k_|SW0GPy zqRE`iGik5hdOjh|V^_c#icZu?tAC$+m6;=Qg(-{`)<&wjL>OYR|Yb zs6m*bBJqchqR+%SNqyMs2Z~K}c5P0xjUiPTcNMtl>TF42fw$50Dpcx)1?Z2f^g%f~ zEvWoOtY+xR0^`%n5GY>npS#DLsZuJ5n5y58%pFox(Kud-Re1dC_CG)A!ndA5;mPZ` z@=-YD&cMGm2Np^0ja7CzRy?pFOsxWU%&sPeAhRA z%M$E~T=KvY2ZYuDtI=2#l}G{%0|B#&r6<)MMUCoPFy;R;cg^Lb|Jw_F%`yQN@o#z{ z_K>-ahF|t-T;ThRSIjvlwOOf%j zcgxc`F=Wo`1f1cewUBA8R<9#n8^*(4nld#0X%nN3LmL+w_e`Ht_>y!ZT&14^pw*|B z1dRUTL8{V&TDs~4kIV5q*cMs0Fu7Mp$1)pWB1(+1e;jQXa**ot3sk;Zg?**d+_D1Y zH|>M1^Ujq0@}Q%m1OJ7w{^d2k!o?Z`)>G`ZjobCO6`TMMuLgrQ?-PB_InMR~c6Khb z(HSUM@v>ZKHq+Vu8%pbJ5qb^V#0dY>v@OiWExPiU)PqWBWNCPut(8WNBwS5Ybb#>S zs}0w_#+6RhaTBR#_XBaJogEj|1?ZIy>pk6)4O2vW&zC*oy*b9T*rl!&}PY% zosP8)8Kdv>E-eiqn&)EKfsDvmj}+p-XQgpHi1@%;fwm#KhaDP)D&>9b-ZB(RcK^id z@Wy;K{pKK>nc6I7Yt-1#`?Kn$o#fm9+(Q>1rJ;yJx7{Vx>fa#)JdQvALGg->Q&WIT zXcqiU0taYl<3#B5RMJ@`pO>xsdm$lEDugjuj3`hmtIaQGp5cfZas2jlWje6Re_5r3 z=EjFB1^a9ozo_=QlYHkNgKq)IR_;fS+|}E@e?!4ltc6am{X62V+S>oqqojUc0@}*AKSW z)p>tPQI7jHVy@c$Dy1OW*0v#RyL@8Y|G!_`+}OCnBIlrVY`tyIY2$sQOUq*`G9A1+ z4ZjcOudP>!Z|)65^1Ca&6`?c$o_q^@?TY=!|E8SUAmlYn2TD2T=R`T#2&m3Ykd zZknpxxcw>!uXxq&m{s2xyI;WOzFEIO17i-i6;RJt^yM=b#7q>waa*gs8{Xt!7<*pk!K+Ui(7B2^W+vT1N7!fDCl*`n;0vitGN}_kk$`Co`o@^YFR|r zYmV1h%{J1BD`ROT@xz#n6Lp3c-WwI@Fm$SyyWS&gwl=w1)rGxYW&Q7c`(Y_8?%9;w zb==~|NlQczs=wH`pVjeUginau^O?o`)ezAYnWsFi(>b<#w=E+N-*Xpz8(J64G)-Y0 zksF!YIDWhS?wVt^tF#XQf=FxM1C6UVrIzas-8#Hbtv3-l!wkbR^RHCWrDGF3cePKJ z$BPs#golMRX^*!5n}ejENf$2IvACli?eW@Ny_02cJVT3ahko4h-t35K4l;MrNDR$x zmgc@AXgtU83^RRr<|UE0NC|sUxkKbauB2h0~(4Ai%aL7d_KaBm>#@w?6K zedIrP9)G;qP1~AU<#u;Uu%m(3 zuOwYu+I{3o-|;WcWv70}fsS{HEAcGv^1$mA8%kyRi&SVkv`jMY!}z&Ud&YaSc5Gl% zsBE&~YF(6VmYH^%;2=IOh_*KLm66PG>}=xsIvB!NjaP$3Q$JVyOtE<0wVf9cWJ5X9 zh=Y{?3tO=MTj!+M=fBItw0?9%Ftj@hu_s%xLz^u^D@0iT(4_&-G*dp`gAI@dK z%1pq#H>M}UUM{Skvm1g1t^_c`%Pog0+e|8U@`i|mLkZ!{`)qSTYfplV1(CFz-{wU7 z(nC3FuYs~i;PG$WrG3QZ^|@QiKVvpqvUNKY*OuSt+C1*+MZ&ihmJMR20=Fpd+Ix4% zf)~-kusqF}lZ$LK9gb8iA2a#o)|?=$Izk-v`^Zgg$6=?4Lz4@4!KIm3=le0F_j^&U zCfb2MelEj<0$oy*6q-zO@^U3~H3U3}C$%Bw-dC0oN*-fXn68}WotRA`%ZKEH;ur+8 zx}VUbR79ht0LkshTZU}+STWB*%2g8v%wazid_b@VgP^CEoY4!IqBPjlfnu&)1@sP z_tjF0IsT}UpmEBAVD?XAwJx>>grTbCEEd+T0u^s#hvAs==H~e;7f8XdSPdi&eHn(AHW!?& zN(h0b69N`3ZJ<0Com|0!`Y=7}<<~Opw@ckco%nHS$m3m)dW)9d{)6%+-3e;zQr3#D zF~Yqa_FE0yZXH17fc$j268ien@R#a?4iM`)X?dwCe(kk6WQbSEtE0ClG0=UMz5Kdz zk{=oTHd0${tek1e-_xbVREQ>)Z(U+a6y$5*zuT*prK65X^{+WMzn^XJO7^grBz?i(Zd-;R}& zQ{K)nP`*DenA?M~$)xUVlSxJ3%u2e%WB68lbJ8JSEUU0s_$wXJNF;Yrhk3b?tGh8RzO*o=S!|S@op6R6cD!Gp8VG30jN^_ zdP4uoJa0nyMP#G7oAz+cRMq*;t*6uEB1A#iV00>m7D@22l|3aLxhxPxi&jcQn@p5o z+B(|!SCK%51s=|ATssy2HAlu~NJ2e8+EujJ0x`i|Qz zSuV1TgY1!bEzm*TDB|`4lD0GyRles&DH-a0&B~&9tn$lTR3ExB!-NyB#g8l>L!`T+ zj%T`-PhfrE`k$UQPk}WsMyWF%muZMG-k{tMY85~vXxjU&DONUyB-f%KONe!7w~~;Q z-tH^Co4o^nyonw3(y5-a%%Qb~y`S78^IeqfEp=$KxeqpKi8A3h9?YVmI>F1MV4OA_ zi8C}=3HxYLiiM+c12B9FSx^s`+Nh5Sh-UZUYyMCx1x95a`*t&cyIWh1oSM?VJ(8?+ ze^~IG7zqb)=i(5KP1{Et29-No6+7xk8Hty%MMUhn6`IGa(G=ndO%3#C+{#Khes2Oj zfJWB1xZn+=i#??wQtM2TCbGLg2sfXM>|xNC?m;^v6W4twnz;DT=b|HP?EUu6W+)V8 zurGm#T78s+`vO;*u@OpAd?6%2H>9?@lv58Pj#v-_P_>G}Vi}?Atmx+t%?#)yo7)iKBOzqoF+ z;izm(&KKD8rs$hEMig+sI&QE=r#57Jy)%4h@G;X0+LS(OUO<0zkJ%E{W*GIwkxJA+ zMs!(_&U_sU(@zQ6aQe!Bj+WjxLDA~nVC8MnLce~vF|=4*j-o@Cu+vM3EVL+^Z?!VH=Sgu>b znLMD`ERW@ylQi0{l>z6l03gr)#$-Sp*JP@!rGq=iax~P$wp<)WA2<2!iq-5!9J=~u3gTHj*?iMV;4zDE0v{AZHcZFpzm(WXuf$)i6o70()BCnG z`}gCttoodxmmy848SiTC2l2ZB(OT!Yx(>7kr8aNR7q{tSeFFq)<||_$D3$0`S^y;3Db#3`L%CuL2I4Cw!S0Knj1T?2CN)p-DpnK&JAkW|sjM`S2eHu+BZF!AfcZ5PV87jDh@HmgBbv1@_7yS9aB< zdGk8%+fjWW9gaTu|JH5)FQUQE{ytIwQ0fH6F9ll*XX2Wy&&di0*mkDkRnX}ylZ`PM z>CpI5+0=9=)hQihBRl6*L2C-E8PMCv9{#6 z>1=N|U&|8g9$M^2tJu23uG$#RSZpBEysQwJo(`*EP5SKu0pN#WniX|x_5F0sY~2eR zg1zd!QB2yz@l3DFa|Su={Ow8>u*S%D3mV6tHxOCpPQ4^an*)8|@Gd7G#HsI2m-V!EBY|=wq(@QV1GZZK7|kAA%Z}IBJO04#=N%hI z7d3+;p4LlKmapi-B!2Ch`(oc!c`J9LwZiZ5f5|)!XP26?YvCkV4Y>28*VvSETQ4|v zkUnJ(bMNpfQ1P-7<3GU0N}0_9^d!$UbIe?Z9H5kWI7Y>NP%Xy3X1`}$;m|BVQF}2O zh&c~-yGf0&*R@=*K^55vDfWz`)jc3J+DJcC;S)JX<%_A=P%lnx@{8v;ct0*oc#AYU} zpNYI4M_D)^@3(8V9kH>SnqdSo!3I_I#Z1A&S%gpkHo+!)%d`4RD%ED-e<37ltnD*o?=(5M&!XxlTY;^Y7)u<@aK$2% zzm~h~zD>&1%G)iydAAx8-ZJMP3jRNCTWm4h^%4@JbgPFLCd3#8Jg8I%AL{bka9q~D9iVJt$o3_}<4Gn@LtvX88lCX+wN<++SV*U04OGwo9+0x8ZqlC1) z9`Kb6tx(nA#jrc?X|~IG)%H{_$hr)AFCqOp&_8Nj>fMz5#@OWS+aft%?bIFhsKS83os9B+4H*=>r)l0RHv zVvmj1?Jr`w7|UPys0QxbSmS@4x=Cgh&=!sSxW*d_&S>=_w=G@rI>o5MwTVInzo3$>iMwN+C@km9_-h5Fozg+J$zwQ{WJqOr zGdIG@=wo_->i}eS+rQx|S4+poA%|9~EW}UtPQ>^tdhPoZ}rk0=(LheOu4d)0e& zMffQ9WmjqE2IrTJT~B9x={ke)LMz=AHe|<_*w#saB~(w1>F}-_G4F`jlIaYsu8-2? z=*Ykt2t1Toq9kA~z%Yf&wcQ-DXBzJ~9)9l9dC$5X@#o-KkbX|}!qO%{Tq}12`7I&< zD*h4mDq9KGr(7Cpw=3;NL8F2-2e%t+p7TgEimdAs^Vf=gSATFFr10F|tIxJ;Fvv@F zS{RhcxYqH}+U@E!Trqe?Pzg2T`{%q`Sbc@vqifd+k_x}&|XblRGw2|F*XV%vIH)b<4AAJH)>x*Yj9>4St z{H7$*kZ_jtF3mTu#{d5>nDn7?1El1>Y-0dW6vJ!zX$!w~ z;LP+{>tpl<)u}LI{6u3!Nb&esa_AXBWV0M?{L$fFj@IUniThWAnlR+Ih>iHI2n|J>lvE>(@y<{?|!V!&}llQ zYe=lm#1!Q8kQ4R@4gZqB9#hl)Qr6*nXRsIhuRVdO7m?Pzt=1^-?1A83Z}-aoQmZIu zX5^XE`t0gyrp?k>_052qIvsRP=vVE&*u z1NB0pQ0s^aD>Q7*PiU_7&_NqaN!QOs_gD)dg394{+koFbdqX{%a89l{eRMEno~0~z zrT0~1GhS~<+Bd#PKZ|1w-PRQu8<0%fZGn_-lwL3e*_3@UcQTeR1_FP=o}C~_hZv6S zqE2;cJ|b44-CPmy=hutwq$Wq%6?;~ z(cjx(IhbN?H*QOBFsw5k9JI4n8#_2aYomIGIejvMjol|S2%HVxHTDp*J}gdP zGDs?A2J;O`#mldVQ$3>y;0I4-d;TEnoPjz`&kBB5m_CFIKONxUzUnB!yCF@tlw zk)>2tl|?owLXwG?;t`|Im5YuRw>6lCYi3Y0E@^5YR>WW|@n#L8>_I`AEl1;0>_MhL(;5w$5%1yEEbvv1ElZd@bxnFz==3||Ax$w>-i z*K_lv6s7l>rYzx`@N-;_y#%-i*rinpMuHkQRBo}*`>=7p%8_1UVsLrxuC-g?%WkYI z$b!T6Z{8;S%(o@1qgi14ra&SBXyb} zCaEJLc#M~0!@ibs&}|A>58xmin?T@DZgVBoy=&Mde$JHg@)}{XkL7TLh+G`jsSahR z6}N5c;L*Y5O~#BOHZf19e7>2YG>(=^L9&aIn;D_i(#IMcJB2Mz9e|~Ikmhk5_wn}> zf<0`Tw68m21c_sjj??G+Mnu-_nOLy=Tac>tB4)6 z3Lm*dBm0Pkg@nuNw+39lcIPLj+}F5kUIPJq{O2I@AhN0A5p9lK*ZQC;e`a|+`OFKV z2TR!Kkmbj2;;X4vLnY~lgZMUKnJ}4T>38y;i80zzdaWBc>=Jvnt9+n4dnK=gbTMB{ zStuP0K^4=977h-lgD~Ydf`w9GIft*lNU36*qngAf#F!bAdf{-)*2S%T6Cb66n2(5$ z-~+M>>_z6sXC0a+NZnQ`IM1sUu(*zHNJhNVjNrUmEa(0h!0 zS&l478zN`2R`j%5#mM!kd!#OB=U`GKL}FH=AD&M8@EfX6)r$Rn^oh66a3dxek>g0FcV{MSn z=8k?e8~X^vmyA8XkT!NC$2y-S?6-lu^V+#~%MkR$4kwM;(5#P=%2kX|o0(z5vB)FG< ze)Wo%fD23im5&%t@7``x@njffS-h8tZxE!lEp@!<_qy1C_SB2@oixuIgJY;q>$0%q!YOe2++Kbq}`PDs`VPNI@;+F}yMY zmDa8;{>vA6h~EIK^T%f^hAVN0`2Fa@*@&F@jslU%RQZb^Vr*3Br0VI&%fG3Ywxixv zs**0@Gk1dNC{m6NJ?s^#$h%z^<}_?%*xRtvdUtS!XNOS?1}d_woStvqtu&qnm7SNQ z`tohtu`31EpHKYP_Ak%B@!X6JGkRjM^VD{_f#++nB%Ed`3jQ6ZFO2kJ0v6lD~NX>Z7HZ&=laV;Sh=}sMnEtM&}}= zqCJup17H*@Ea?d!HTKGLwCk1KWRY{2Q#8}R#MUDnK2m7}5IDLRL%yv)s_AfNM_Rk*o3oTt!y_4`%Uta*$HO6O)i?FB6^JK_8NwC zVb^ef#M}YA3iIft+;VEz=sDnb2YXXyxHB8~sPu?W?z(2q+0!X3J@vM_FS3vRle)rg z+NZE=@hhmGhW{~r(lzszOT~6UvXzDMDbh^)W_cYAmg(V?XBqyn134>A`^d>sF*}L8 zM+1>(TSJ#uWv5lFT4VvrH1#JdFNPNCK9b*y5@WJvJMrXDaQ)NZj-_&rMo&U_{7>B|w)@Z?!5`3NLL8 zQ3!u~@W_Vfyw=krZmZDL>RV_Up4%8F1=*~UbNmh}&(e*C|@^fy}-H#0a(&h~1V_Y-CvqR-;o43L$ zOy%Dm1nQ`z!jP4Y4RS*rk5@~>hTk$r-!JF=FL>eqRh96pk91dprCOB8MMwSsK@|yRE&6%AOx?l1~_0&pli3tgAf2Zdff9 zhyJ5LP*K_rY4epF>RfnW;(E7YF4Is)csl|R#Bw?#R&7Z6TN?uC4Hlq9p9{%Z)4US< zGt$b}<+$Y2mT}QXbvYPt1p~Ms`VWR0h+rTjK5k`1=IAX>T8U1alRVE5n#(mu+5_*N zKlBf#yu={a&Z=4(a^5ftR9D)%B3JD!YU0byM*^Z2xwqB-P~lWI9+I_|(U5OVerVIN zi@KQ1x;R2tr0|hHxiPd12|MKysgUoLiToG5=9NMIa)z>sKEiPwTsv-~G>}^y4BV+} z)@oh--l%WC_x~;h;2*^X0JvXiO>IwIwYG|ars8ML7X z8K>J+1}0zVlMWvt!g5lJ$Cej(Fe03Jhy}jBzQzBB^$sRznuU7Cj-dfNibTkTN7H%@(4$Phl-{sRfk0oq9j@hyirW5?(y*T4e z+J^{4AadzLIC}mFR0HWpkfqQYk5ltYQ+1AFqwPB*O5U-Ah~*0ST2b$J-%d@M z3=4!W6(4=?d{2|MJ1M`PsCcK3{HQGk80YVSue6R%tA@ionu$B{hh?3qD*(^ zz^%=O=f|CeJH=k4NTf|0{A4=S%g&-MIDgzDJ(5+a#yXf;xiY=S0y*GxbEvvY{W`Z9 zNB_@byjp1LzenE#TUShoqn{}I@wD6M)u(p+>b*``YMTJvmcQKfJITp=0bM3-Ek69v zfzu|$Le;2ryINj@kqRsNm?v37N9b9&hY%=BxVprC5*5^>hfYbW>N6tM!rm@GDI*g`9Fw!F})2&M_ z&z@191!(6+Xnl$wBo$K8#sWu-%(D(wt1ywjL`iVa9lnwgCa!uBFLfF#L4jh$jwUL* zq$%diu)XB zir8C8E4$-9y}2=(?D=ClgzvgO(Cv-z`nvniK*o1`+p@@CLN5wf{6gYB=xEI@r5F}f z`JfNQyqxqNE?tt3w;E+8%$<8}F(ax4;NVRpwP#P2VmUaESMO5G!r6{Drl4GLwv#MS z=2eF9oGxtE3N>sMc+W6IU6M*<=hKF3?~(O`zgs?23qkC8I=WHhmpB1TE$h#B*Iuco z?(lnkqf=?Tyy3rQmJ-jBQQys4NGmrUdww_^;Zty?=7wrJjlEcb!Of9Qcc&4>OwT%htQW>Ue}`Iw93X~J`&8#wvJPw zIq;Cjgq0r)ZDluURkF|ZeNRqKOEKRq)Bzjlxd(A}mv&5Hw&G6{j8w~=H(Pv6+y*dI z>F&hXtKmip4Ynw57c4NObtW<fReI{MDucB8 zR<<~k?JhwMwYqDaGvJk?auAB=W7vW8{t2+cu0Ojd;HSi+d_x`hpof^sbheQ#4Z)l@ za48Fa{(^2fikf$#<2)_bhG~Vy0;@XWHWPS!@zFlVEWDuj*6AijxGgR3W?M0v zc6Z_N@!GuGPn!0TUeF!_KWlMz`?bi)iBR+L6Wy&*laOaj~PI4`#kYm}(nT*pJ8{=-iId}IJe zlDN=0;~@o8>*85MVj{IA^}FmruUF&xc^H;+qE-LcjD$};vf&y4b5JYAM}oCqcK~gZ znW;f00=s?z+3TA4AU+7JPD+SwR0{hqs$&wUllBA!P?y1HP(TE%*cX@2o;b`Wz93Z2Px601x8C7l%8Rdj9k1;+@A+P+z6mS0a zJsoDs7^{v=GUtcOXQVf;#*c60$MVY72v)7{47!-ei)zC6Gh0R*iRMKBKt|*9Uj(Cy zMy?ndIZr294O*l=c9=Qyh*gFfwqwVclJh9hJGS{}`Nulq)L&J$8y!HOV-a*g@Qslc ze(aS&H?9enGX(jS>o~zy&$LkPYY%t|FdI#<%r2*5FsBX;xYOx>2`VNV_9UHiTNb`| zYjWWJU(&;Et&-MVH1YVREv*<)eekr5?>tPu-mbssv4nSZ6!Ze@xN|M1?Yy5#_~Ae^ zMF3ew%9CtAtn9MvM$gFsS-F`Ojsy8hf zH!lu+*7teoX9MJFrtnblx%@laA*qR-G_qs58l|rYMu2mA>!Lvfq~nHtx{IR*dN1HN z8u@x%PG`>$(3>Bi!NDgpiOiG%-PCFznvB!_i-URwb3OKSqtAu&(ZR^whGt>0zhcgH$L*OjG(pgCG3wr6js3fL3)b%vnQ~mk1BXKb|pHCEmj80cKf2MAWF2XiI!p{F5 zT>J%k4kSS z-x@YY&(s|!ME!6nJ$NfK%pj{#>aP@EA{bwf0Z}}vNBQS81uu+(5v+2W=xHxzYtv-N zZ1m^CU}}79<~v8*f5e7IYfb%lK6c!Ibwn$X$L-8!UfGh(yr2kWvspXVI_N_?g$9cH zX|K?!ov;^8YOLr3d0p@Htn7<*xv1Zz>nYl}XBM<33K%CCpvADYBPU$rzMuqG)4z*F zFP2yS1JoOW$@)~KcPi)Fflsej$}353Xy}bkL$2%r&(seNelfYiy_v;3R=lX~f|f1G zI-qQvVfTAEC~aRbenuT#8&r5?*<{r2exW8(x!rYq&RF$MpLY&&s^WR^n~2{89;Te= zzayK0aP?JHl-0LhV;`zE^lDqUT{gc`sX5~C3=CI?eripgn@n_kfCVfMB6C?E0#VR- zP7QCeUZwSB1IpNY3!Kh{b-+gF7_r&15wz8-i1(8jQ&PU|umbp9jjIWNi7_Ak-r|({ zeQ)~BLY^vU?Tq9|XNrWysczo#>bY|L9qSfG+jjIZebU_1}1r{g7>lzN! z#3p&bK3DLn;eaTtB~TSCGt}wMWQI5sGZz50pGaoDceFe+-w|+aA;1s7InC9lKh)@4 zvnRUI;lti*2x>#qYK2oEni@C0wtr==pxXVwBlgJh>pLyKnmu!*^PD7@b&Oe~VHMAg zM=0N0VLn6YxdcgR?1CmkP0JkL@anp&J7@WwKhPdl+dM%!Wxx&i#Jn_zox{0HqKqvR zbY(?5lx4)RbIfM)_a&2T{*Kl_l$!~vM8}7SM2Od?*x}Ard&4OL&|5)Tv+w~=Jsqo& zi-F+|i89%S*q^w|;$z>HB%AMl#6APqAKSld^BMJJ?gwd96IK63ocRBw#M1w7 zKl}9N{2bi*e?IA3CM1P~Xd>mCG(H6a!+lKujjI1YpuCF^kA2X zdcTQ!pPb5(BzemDJRHawuK>UfpzbKWfeVRfg(Z%MA?wkdYt=K{u1LRBN>pm}qKQvE zfMtxkzEIfqXED^O1nHM4CVCUgSqmT>QShkPYcqJI!%T!FF2oG|9xi&f zDHU$7H75S7EjZ;(cRYkUDSvYCt%2`k9*mAEghCs!2oaK6jQ)~Nw=<)<#7|hAb!%-E zGYaQ>sT$ueX#E8crmPEWjyZ(%U}Agj=Gmi%Kg6le%c3tWtBhTzTK979(FmK;F}vl` zsy2azx%dO*BK#FmX71vmoGw!yI&E8L-n@U4SE_Titlt{ElP#~tZ)e>%#$+{l{*jK+ zgIWFAP7c^OYrn9i*Ywu5fmWGcZYpK!wxMn5N6fQj1dfRU3fNDwh-x99#+b*eJK|mF zuaZiPLhKpEMbfCu`bmdsnvTHY5_CTI2QZiYn~)4Z=(~T~wPDB)8I0A{V6a|X{{f$| z931FNB_ypCXq8Yr(4$Fd(l>S8g&nQEjRMd1qpBCTWN%*$C{!hbjOZms?Qrp5pmq}7V<+ARQ@yB6Kxb~7ft z$}<(1G*X??c(^xYDLO_&zmCh<(_k0EP)12J|jVUa# z0ncZ%bE<6sR^NXy)ARuoUw*s2x-b25_(`qzmQ}3ZUKOtVX8VSx##!%Ffu?rEy%e+i zQIm@&+H|#9T;;xVH$QWDhNOsJC%_M>59p#*EesEiK1#ea#wnKIY2b!V%4tcy-(eWoePbA#m z<(gbNSd-?k)_<`h{Ug}+&D~e$Cv(ThE6-rBemqZmVl@7~QQ;h#XaIBHEp`vI)^^Oc zRoq407voJZ$uLP=Pwdt>HYE^> z9&_is?T6Hh9U=GB+HyE6kUi!ep4Lg|VNA=!OM#)p`C-8UnI$G|<;)Z(cb3g*h!|+k zjmBg1WIw3VmgS==(j@ycANEsvml5)Tu=V9}fMD_8Vz}F|NAsn1-0985^@S!s<-qGV z7j1}ppS03jy=7ssR^`R(?g)|G>?|r-=lf4Kd>E*5F>}y{My9AX zT=TK*JYo@U4bF-N*9lA&7=ht-mbceXBM|p{!NdhcL`9^$W~q3BAe7HgEUG#PG|*o4 zsi&xaj7)k4^{nu8IIhmCqNW+Nnvhqmf|zjEBI(r zYfn`CfYe3c^Nb>3Dr*qXcEo@e5TH4wE+kxN6-;e>w&9{iy;fp??}i)hgcF_G5}UwL zPdN1Ed&}(oimv1b!K;F^lDMxb2RMG_A;n#hsq$)^VXdt;e{##Hr=V`~wYPY3s`pkk z93>e6aL6B$!<@2lEMJ;Vuzz^;Ktas&>I!-pApvE_pPqV~iYj37L(42C?ip34Dc{cc zD7?f>#TZu&{v&ex%Q<5j!cvId6^j7Ez{Cg!W;HHjy{YDC<1t_xe%lw&Iba&QdF{z_ zvv;k_L?obDlZ52kT+MErzc^>cNG_B+K+-;PDSUX@yClV-mE zxsVXm>Qt#IOc=H}!M}tCJ0dI6CsV<-V|F&Pe#m4};$V!X99hs%m^)M7lm&&<2L8;R z8Lq}>cKz~?8$D-A>T|2uV#iusWjS*y7M}2G=9RXTp(GHBXK@GpqGh=8BdR6~AoR{z zpX+&BBZ^bGdMHi)0}CTSEV+kPooqjX#g}e^)f)@yDj}<2piV@!dfk=V$ag2Y!*DyU zbLz>U%%rD$sHWuGQ8e5OV6K_?p3qf zT)|q8)*IdaDaMsJ*f-F;6|8Dn7sWX6@H{sEow49?QK@sK$|o`Q^`%mPah=f|nK3LL zn<4xH<*Y2_$-k%UQk%)5DPiSISv#o!br9!m?uEgv@90N3s;SW@WAku1Z~!s(${rvh zjK-6fI~a?w6acEA?cfi*#n`(G?e&+2ft9a1G)~}$ z#2lvwDahD5{Uks>@UFV(_x{m^I)it`KYS^_<1%7R)B6uyA69O{W>*a~Ypb6MFJ5LS z5};aB#`c`+UCnx6swqG|?D4!YEfC`Vpu!$n_!~ts?ej}agOVYZO*{aPYMjNB;ky~9 zafM#7!OC`k6>iu$!y=8@)?L)Rwb9wU0*Gd^^_^k(*4a zAIWH53_NL|8vhsd*I;Il{MBsT1YRogdHoT|sW03zYIXgzUZzArdTk=|_E!z>bPx?YM z5Y(5_C)2OASsZ7z|Lo>4HL(WXK)$aRN{zmQ#wgw+-}4h7XS^6H(!R*zmOlK1;*r# z)(f-TTqbD8iZrkPqnL<`OV8I4;hr}OPB&Wac~P;MUs`+-1UQNZ>ntx@9Xx8C0|g2A zm!G?oqVg9#?BuXZSBtM`;1qs*FDEG zKTzhxc}`HDBKj_^3U#8)vbSY}tb@$w3XOL+cZYURH?s;4L2AdS*mwIuNMl)l^?m4F zg88v-rv&aV#VoenG?yLUMrr*=VV-*k-E25v#F}`$nXxf%?8DClIS*lJdKYb;`7+8f zqpNupi9O*4x*9W^wruL|_T>Xk9o|^irDRTPNORK|1ax8xg|v-U<-I zvtnA$^&K2I{ZsJTx=BTN8{Ef>(y8$FpNC3b??-_NrQ^B1}gKuzGQ z%RJL04InuGA0VFkFWKyK(Q-O@zE<%SU>lqqz|K)j0AkhLS@~u8l&x~k*_kq@aUOs? z0SKeQg*G+(EC%~BK54v^LIr|w9R&*$yz;X;Nj}w0F9oT*ox?jsIXO}TL0~)C{wdlC z?1F`WN!^%D8npV~=R3GBG?TJ&s+u$>DLwqs=~BHTdqpuKWK9iZPqyru7~r#j9V1vd zo0({4)0OkAAr%XkG2w|>F??i?5Z#cN$7mnhxv9-JYf+p(@UItX1TPdS0J4z1j9v2! z@hfrVH5q^9>3xVB@6&j}0WNbq(Q~r0{wF;<0Y5M|=|&%g5WiZDYf7Np*+!)~HS2_L zm0I*yJh-{m($+OSZ`Z>Tn-2k7v`oFcH=TFA`P1fmi|#zz@?_buZP@_+O{;A7Mw7@n z3&zg_eg5GMWagoxQ~Lz9J@Y<#i06&|$)A9yc_eK}w+^a5{wZtSb5vhtbpq?hcGP)9?f62v4)1T_CLPJO2HN;$CqM`uyeAZ)dK%HJL>@@)ChY@QX%^hMB&AY?+t^ zChQo(KWB55x?#P0IqvNJgTf}9_sL|^`htAarV9jWh$kJfj~=mNxDeG7w;I@Sj)hh| zK&JTBVR}ALlq@`@kG`0|<>TauB~zAfjIRd`77YXZp>0OXYyBfR_lr2L?Gf?@!_=@S zoO0~0s?~7VWhz+1{PaI(F%X%id~aPQ%D=;P=~RZahv0lbVxLT!n?`Fj zDn+a~_80W%S95^#voGTTz{xApP*#-=v; z(ADS5=8QB*7wGe)j>?7qv!Ze{0EFi9)nB0TwcptjxJWY#MTE?0XfNe1dMVihILaS> zTW}?)#|4p-DAAB8O6#ia$W!1yuLb5p{F$WqHx1l>Qj&k9R~Pz6{~e!Fqkx9rv9M*I&M-*7ZowS40@D;dPLwnh=-0CG^TJEx!yCUV?$QUb(D?y@kUH4 zek`&-55U@~=8~S4zBHVhDX$)kT)*dx z&ML9n^IB2SRLbd(QQsSwV-`Q85cWI@B1 zp6@hV$y=k50nQ%rbVwIr>i&MX-Lzlw++5P*30d@#bX>6UJ|PJcbDuwFUnFp zj7EZw$l2gZH(-rxY|FS9=ZWj;89~bzw5kPEsA`!R`~>U;;XzijzCctkq>$r*R-|4i zPVEAz#J2dR!4VQ2qV^AKd~uebbH39*AMpvUHFeBxkK6Q+WD3vQ@tuiw3X{n+1Fnw+ z4QL$aP1my9B*)`PNOFRDIsv+7IELXA%varu4~3*(*c5A_8)c;N_=9hLs-V7N!?88? zr$8DSJ_vu~J$Nr?+kAQT|zJy$~XODFv>TqH4Bm-U%d`5on zWx{$uFb&8ygZ>>a|bGC-TzOrhBbZIb#`Z@cL zIOAfmPekT3q(1i~16qssy;)>AkRy@}(-{u$WY=5k!L_suct<(6eB{z}8R7mo4ucA7 z$Dd&GrFqsHoz+eFE3&n2zN@tGh4%M{ z^@NJR0jGo=>63eKW$>j&MND8EnGA2$&=F9KR_uHoXZozshV!y~yS6Yl zyE>C4;XkjL-TX0W@5ijwP4A1#ZTAF%q7ouGHT0=v7ZJn~CwE%2v|!Q25B~X7TfgK~ zFAdk?$_7JJ}Mc8L0RdlKZy7C8uP1ox>!ScrH3~fg-vXYDykJ; zKDcCe(99!66${`pnQD~7%t*n$o&4E!xU-{U)GFNBcLn9^x7%LcU=r z$2aZH>PDr_*~STdI~-Mv8(=u3T;D!do>|+~oSUYt3S~QG85sl{j}+Ll5_Y2e!-Z5? zT%N6N^ms?LIgTppmc)QfQfVvfD}OB@TWmZA+uM<(J^Gw_eQIuR+hiO59>|kbhA;f> zzMDkF8AS|Z%bj$$1s^Ygo?|}gDss{x{n2#BSz;VilZaY z?V+qTpZ5e>wzL;?m zoVFbwo(FV1TZR!A%O&Nokq7=jPiJl7qoNbWy*>X@Vq?i2T<1MrkKp~&o0<&XyMtP< zr^U~F&wjI-40G`_5bo{Hn9{IE$gQqAHNa|z);pt}hwlS5;M=p3-0_R$O1?uQ zwoiY*HryP3Tv;)c_l>EXUU>+T!5;_@BDcn8&K|sg>El1phPeqHDag-Drf3%B3Nhm) za?mNb;;T2!xA{n9Kas({Pnk{VIegXrutioVYjz+9W2@Jh1S@iHUtdlUVMnuz_`M2HpscH|+K zCDLjN6d%$=s4{Vw&Or1}D6@26`$X{a@RTQo(K_GQ+9Vih%xk1IHS-roP)SXo;~Ant zzO&;@HvN%98@?Cbk)!i#qLIq;#z~NBzjN1N&3h8rmbv}K<&O82??;|)QAsln zLN4t>P$g?N9@#!@am6cYH5L;aHY}LpXT)cPW^8ZB>4+jjV*RzHxUu00AC`bT_^eqZh%`NpBOiwX5bn=G8?Way$T+4+GqUDJ6*_1ESwN5wmZC8j#EI=>XV#eUI@I>ix z_Wn!{wxLADcF>4RLgD3Q=c1bzX=qFXH)y@6o{=z^AboLNN7P4KF1#%rCyd!}$O(|F z3zxWGtmQ-N^xBnD=wRi__z___(vv=uKyP{LQey;l-Vu46FMOlFTJVt+Y(^LN03}dQ z)UtkC3WOL!>$udl`eQespYoFn?UsdBhzqdR_k@iwh)m9+f1WV_21@3uT*0`Z$TjaQ z%vhPsl)TbF@6rG>YQm2ZiCdXi7wzk9@kmN*++d>%4>l5ign6oHJ^xE3m65nQ((DfY zoCwRzs}2))w`NHqvilB<`ehH;3$u`d(*x^CQ4cqjgG)9l>KeuOaC&BXZ8)+4UQ>=m z97MkCntyZ0wWfH@)QqyE?A;r5;?u@9zn)RH8+g7mDM|csdKQ>Q)0RJWG4u%g%yYP= ztucCF85GOw57Rk%YPxmhC!b74P$g8OE_0%RR8zFmz`9|1L{6}+eww6VczVw!~$Rb#8U`49YlD+h~{6QCH{cZc_s>MQEj)!;h z+(m~NY;%+KWpLJ#DGtxxI?SBe9m4TTaL$Ss#xiJ+Xf2P7d=1 z3wG10IgQ~w8DeG?U7RQogOQ$$QT*-Fxw7At{Nw71B zvukXMC046M=rNDCpc1tNq$28X{mNJv%OCVl;w~TN*ti%KuLI)0KUmZe2a?Qvw|_w| zW*tT0{BC)`HE^%X%R4l_zCMZJ*e#Y5%i<94s_Cgf*ehR{-Tkva$YWt1<4<7l|5Beu zJ~KlrzD%noCaw_UWL;l8>epy|%66NtC7oRu?p|T??9zhE3SDyD{ymRpx!W6GkYabIosk1HPLXzRL*~J>py)k zHxYMWsYK&yxc~f8*8gQGJ%KJObTmAfXMqfW?VSaY`Z0)q5!p>efn;zNX_!5nhvkNE zURc7MXmC<#R(=+QqJI{3Y3k7Kuwl^LN{!`z-wp4Cf03K~?1p9_74Xttd%oZ7 z>|Za}jR;Qv|4x4KnJIfz%Jt zsp7(@j)H1YF^Fs;b8aAu>T%TuGW3vu`~Cb`YpxoG=2cmW1hKviAMY@4&K_4gqD`~tKtz7xCE3u#)=F6`Zu;);FpSVH-~OBs_C2`*=zkFBYmZ zw)O;JD;jL~>awI~w@1<(Ue#2CSF1{9J3dCfm-Q{>T$bl;k)YT!(oABfs4Q&g zfV1Kq?KciifEBa)7a<7Ll6^%7ZU)|kwfZE64BMNV2G3aol^ zaF6+Z*3SXtUyq^q%0s!8i9EHfr(L3UN<> z{Fjl(F>#{Zx;BGhWT^7hmqzW|Jh~)?o$DyG&dLA_diANwN6g#Rg-)qXjF=9nL$3g$ z8W+Ys6mWk|olmTWF3cL*`(b<+-Pc=w`vi4Wm5(sg98j{kl0Ph%1*gp#;@s9zUen(2K5z1sj!jmhulLlhj{kUdfZ^)?M`!>|zF=#FoeI z!3fj2HNe3Zu8Q=RGVLOT-K^Qz2d$y_#Jij(V*^C8?^#A*(>t3WFr-NM{n{EBq0hct<@#<;4?mSF~%4 zdd_XFs32r5odQ22c`R%B)deEm@M`)a!HI&E=$&E7#b<@zO`>?Qm^0H zu~@#{wAN+PU|&k6hawZ61J6a#FZgdQuyXr(L(RJn7C*^spE03?8?5lEx@Vd{)?y`+ zRqY3`H^05aTs|y}(CUxOrBr`3Xk!V;6)=dY=!QZdm?Gf`E5;30-#cq+(hyT%eNJr4 zI1cX`@#l`%udfRa;JU<3l*Q1Z{!u#Pu1T!QmDMRRftP~ryHRAKiy+$0!15C(Q={Ye zu#$K)XVx~3v*uc^b#h#9#)*5elT=iX69y&OS!8ahWzQ%TmMeYvZhfgMf7_aj*uU)_ zEZ`2nlVFlU5=()(x5n$a1E(oJtQ0jCG|rL_?_zl-lYBse{=V5WuuDxrh7^X6Ckf_< zjsDuOB13VI33QkIlg=P6_R>2@J5%Ky)_CewEoJ$GLK~Cj_rP^`^1fF0ZlAm3@=D@v zH`lDe;&7OtR&P({aLij++69xHy8u%APTJJFad=^d@~0qE(7smeEL;61R$Sos&NXSw zvBtyN1(|6eu=vVG;@c2QLz4ktj(~O15HF)zZ(PjDAAzLsw4ij`>)z;{1cZS4$z^ z$d#YHo1ZUp{T*-p|FR(P-%&gG=T8Snf4XuqE^am!7(G3IxPPgUg!^LDcBrJ-*oLjZ z_#}X<)KbHA%fc6&&*0&7N*fzxr82i&06T!j(|n-t<@)17PgWUSKl7X7)9*D3n;gKb zd$2%>g|eN%FP@%nPxA9!_%$lJZ+RCP@H6JlhLEvN_jcsNa5K7C!^ldNpTa-tlZ$=ow@(dAzPE%gdafA!kp| zX;v*mKs4gYGv~#K|BGD(`@2kq$hImpUa}&<^IQH48rVNt&hjWt9a}GCczzu3OrD!M zIxA&zfR8`?-Dd^rkr1~o-Bul-w*CMpp#nN};q1fz3*rBNmcYU>PTKdKK}2AT6PLSH z!}OcK4Q#Ug$AL}9|8Zc``hObOH1Lmep#G2hS>D@j9vSU4I3FZ{l$B^qhq;M+tOr#2 z6~J}ZHP2m^y$CmPhop0OQ;IKKWtW!-RFvdNrt)|dUzQ#UJ#;ZY7+^@74hRp>_?j|5 zBIoy0jI1!DCl3#foT#!+4X~_JDZ84qzk#@3C2YwC4fAeLmQ0oX`L|cbcK|FlJN`Vu z2Eml!#X;rQC-F+Ej;u5bZA(BzTodS;Y;h1DWKr#bI(*ro*U~l{db+F2GsXNLJN)^_ zKp|TGMN(4mY4&aEFEr8zX=J8ja$@M6D(9LGD}aA@tmzg|P0s57kJ;4hhJp28j{`KF zSAEdDjv0m83O(CwDSNW`#g=aiH7^f(90-e2e>rA%M#3N3rztlt)nSmjU05-ycxNF@ z8bR|qWYW_Rw`h7l!?N3tp2Z_Se%88q_JsQ0TN2K1cMq*3$L9NXRaKts1HM5y|rYjQM{jKT?-L7&2GbC=J{rufOqs#@@MB#4-2k99ZcQ;9_4DJenVi zI`2R5F8iCgT=-sdw|Rj+#MUJc7z`7I)@#*Eo_M1j6lkJSt>G1DZYEAdol5fman}iA z>n?&VGYmP{&j|@Wsny7~yO=$e4h7rxr5fhfV96ClrOnU`$OzaL_Zzu98EgQ4`>MS1JT?jOis`GHjRbK(~8vRk_mY9DJ%gs*75 zY^gO)2;0AmYdCB(#lN#W%>Z?S>1V09SG-!v_$jp&3*D*69;&Jbkh0x$r;XN>zK=y# z=EwB5WU0}9qL6T|3aaMWcZ0Kdac?t^R_r}3IZ_=`F|~eBl4nEP+4@Vz!FFf=YK^KZ`IVt{IlOeY|F6e)^j_9q2CR z6(#qmwM-g6?XOi$*q;E>`z1e_G*y zGafQn#709?(x^#?*AiFWh(C6TsCzjNR^4nS)2XN*p5Dz5PocK<=W&HAhb4x?%m;17 z#{oPawR9*|JveKkKBBqN|D-7IRoriR}-{J$m{ngM7kW%p%y!%F^$>+u^; zd8JuL+;;_~oDXGrT_R4yvgrNf9{vE~5~OdHWQ22MvAbADi=pbY#uOwqS(Hm~;I2hq z+g6whUD-u2RAxKnp+ihd+;m)EaZ$(~S*fC?BVwQhR#P}yuWP@fld!*@!P;31%*u64 zr80|JzlQL2yds@>^qx)n2VR778lFCa1+)j&gDdR!Q!Gx4r)cAFg z7*^`{{lV7G55#&@SO*JQ==JJ3aU*8UFsNPqV9Z8qLtf@>(^w;q$%J6AnbizYQ_>uT z5amw4*nZ18uK%2|tvK$P#RgdEAJ;?tvlKB&D^a;N$H6Vexg8IO;ZPbtRAPADLK{D# z1O%Q5I0zcdL-2p42B|D8HLC6DHN0-XYHil5omVkw0PCTU4R{gee_AJYn1755UU{m` z1EU$|5Bvyw9JpZX{(bHDb_Xh*#~8AMJFVx2T+^fE@3SzolqDL9`EOqX--7l3{bPsy zGZVG01blYvkw4e}Bmt~jyaDj~rY%s9zm@M&61S9D!e;k$TA`ok4KfP(NZDuSoZ zjye&AE>Qm#J$kb()XCT7t~5#(IPfS76ER?wOKW0*@7f&!KtkzEzfV4O9eCtY@9Uy7 z+&oyoHAQ;_Wvpi^tOjjJsAx^X5z#m>$0MGZAK`A;xAUiFareE3< z-U1fZ%#!v4I&G_M^;_xTlITZfJD)jyY7wwtjXMwNZMm<0rkxMyi83`FYH@=%8jJB#4yaJ3YB2GhrD$^7v=Nrs#V9AA6e8b51+M zjbEBP_b}d?EEBVLo8{>Ezmw>JA&^?mk(~GE`tYVD$30W;Ab5ob$-3U{^>`hc-+I)E<^0|IY^Kx0jONVoTBvB4VbJFO8ToDj+6LtS6VDhs2LNOg;r5k zFNv4#fpnpl0n*(0ee+6Z;}J=pK3jZEk6tqwObdpxQ;UuW?oXcz@1&&JP3SpMt#3t5 zGk8U?Asgn%2E!G55S#nzTDh7B9rCR(c;y`Fzj&sChn#-bL5D~AwpGqI3gn< z`gwy{h9Kr!)V@Vmb)ON<5OjjL_SXvPm>xC3?4G2tH(Y*&aS55T$EEMdSW>lUK;1x= zJoO3RN+B?mSi02tfpNYe=L)V2&nj0RTOx64vQ>4GcC^|EnmhF>t+5(a6w;|5TJ(qfyELU}CM9ZKe0DiE+qXO36B zlk|wiW5w`G9z7Pw!t&^BB||1lE4C6uy~ad$5EO@xcf74KvXc8%=Vo=#`j4j@{&+&B zQ;&_PH0~>X>*(QY*5E$ojSKRJu#k_5S-mXInD`^0-x|ULZjPBc?)0X=BPX0XgYrUz zOAB<(A`{@{%CYskI+cw@V&NCu9Z<7D|uBO^}@MEhG7C@W!$?=#z-=< za1*otadS!s%H&XnRP{@pueru29)N;v2=8CSk><(#zUQpnc%c=@QZ&nni?g3#12{Fs zC0t)kGj5S-nUugB;Q!cnIPm^4dHc+Ce`&z+=lJPjZywBq-&`8}F`gE*tAx=vO7-II zt*a?cprXp7Pb-1hM5?I>7qi*kL;Q_{v;gNW58ORIHIRN3MmmIv}Xr~>hq9IfY8#bWE90s{;_1iP{0Mj)pTisef zjdFG}hi`JTH*wy9EH<3=9fM>ysCW@Q;=9%ppZPxv;+{`VtYuNFF|23JKewXXcPaLX z3|g%tE}9fafXC~n9fXEEQD%;)m_LO*iwMv}1vDL=^j7!O;wd4b zleoKRa(|z>>3+&JhoG(XPp9}r&2!w&9UM?E^aM*+Ng3J5agT6YUyma%duaVF|FJVT z&9CJMH_xzbGJH5;VLO>=gOba?1>xSiQm=l7!xP5+!<$)%}8b5=Abmg zQouZNtm0VMYvSAA$PNX=xdq~bZ1H=%<-}QCP;ky3$}> z0es4>iniMknwy_Bl5aF~zgUWcqE*n4NS*)P8C1S-YX`i!S;;Pa*oVMaG@1V7d^&~TN4&5mr#NFVGY=}W&;UPk!y|+K zEyf2G{?9Y_Uc9tK0|yu+2A%Nv{!wQrlK!NI;J~_%_jU<(P`p?6^af3V*P)=}_+b2Z z=$Ja$Zi;kStoPRX$~@)I-!r;>0eC|L>P0)h^TDK(e-~wN!n4SHQn)RF{5e+OHQjse zj`g=u2uY&_qv}%mSc_w`KtzDi1PS)C?%oIjAWkRF5>Xt3?^oXBg*~z8{H4ajWSXj( z7oRQ5y48$1esN6>Xg95{drsCa0D;M$llpryJD)Zj&ggGQm$mGJoYXW=qQ!Ch6@`{~ zUj(c>LTvvUI_!GQN5?*hTVPj{MYT(UdwT7(PkPnR+^v75|37p9a4lJGH@};Di2#Jz zzP^6X;=|}8(vV)q>L;*@(+R5&pWM%U&@!P)DwY_O^LlS@*-1o&(IaGsbx7-ugoNto-;jWBHO)dDHHmHH za9Ky$;v3<$os8tDkH(1eYqX~P{3xF)uy*REW~RNFl1THt0-*If#K$jZ1xFXX2)z?s zIOgzR@t2VpNkV8XBv`$p1l+F-AA^xBif&q&;ZQwH%jRm2t=FH0Ml?Ea_k!%8n(Ujd zkC=>VeSS13zs4xR*WMg|j%HD#M{%uUxRFaM6pl4n<-r-Zg%bV3YIjFy6%7qr6F)9Q zNRT0bS3PK3-KzP7bbs1#bP(RO9@(@hJzsDK<)iHv#~|=4PAfiJ-hM<}cCt~SKs|r$ z5N4&S+(af~ttgBZ=dXEJm?PIvg>9~sB!5QfAxT5TU?zzPj1mQ&H(NY7TToI|P2!*O zCKn3ZKb>fvjFsL~91$R=`@A~Ns>9)Ks<6=Zl1ar_?u*v_wZ%uV*~cpzKa5f*pa=+T zHP*UIa6y)k==hF}n;C3ClwdXakNXf8*E9)zV7oJ$`iT4B+;-hTAf1C8Fq)XGpuIhp z@JQvr|21A39!@MkWF!?Tp*dCF&9)z`%@U2w3nGw0qRh(jm$!+D(=KaX&06NRIzlLq z)PP|+cYGZ@kGy_dmZB+7?DX;sMEfQ<-&9q5~2X z(Ji~m?(R)`G*R^Jb79Ao*Mr3qytgqIi=WOcMPZ{uA%2G)pX1!;Wq0mSn( z2$tElp?JN@ap(Lz+3GYaU=$hqf|G&IoW!VCR>qT~Z9hCGU|`#2cM;utrE!F>$D}9s zG2`r0$1Fyrx{>Xj^>2x%+aq*_`2r&p7J=4Qa^m#klb& zDk^n>xexc>j$Oxnd~WhK(`{w@XO77QDW_>OTh(_;ngm?av7dlK#MezCA` zeYts$#r}~fY*cH(5kc%dsZX1@%_#pf^Ds`>V<}ZUG>5HGNJA_$?+SnV-C2ye2x*c^ ziGMi3&H26HhaYbu_vVZ}wGC@5TmtyOf*-)M*-FE{V2t{cQ^8Ev&7`K^<#{p;d*IFd z)?*Q4RlbIC`zGThpM*OUh|+l1SW|+yMjEXY)EKy+0nMZ;eZeu~F4Q!CPCc ztGi9P)=2kf3AgW=R0!b;U;5;|(ioRb;yWVUig+j(G&NfUi;M&R+TgKA`7LwoH0x~R zkmAQz4urF)JCrjfbl@I`PHb-ke>YN_JlAxVnl34yfGL#3O1w}NEEy!yjm0XvU0EVL zqYSxPC??~VeUqT@MS=B`6HhRYx7)4QF8Y+COMlD`O87BnWc_znws4}q-4|SJ3L^;` z{2~hbbulNu#TeipS~t%CCs3n|phaRc;J#6NOwRK}SDBB?ZB}&%9><+9>6KK+l{nPk zj(BVFtaUVSM-l$gBFPTl!h6wmmDm=QrJ)7;g-*NeT`|G(m&AVAv7W5cEpMG$Cc65V z0ZyLJQ5nH*#U#i*vH8J#V7ziJ<%i6&+!E!_p*S*Wr_bh_13|eZgQ=#O*A@8usW?4K zQ*3yr&}pnGqn{6m-5J~jxwuB~-)5iJQTMd+%B$UleY($EEW_UbTX>TfnVonXAJPAO zVfu6H&1^qPS6bxH*(J3q-g$n~|3bNE5h=HOExYR4?%#fI{=`b;@!OfxAKVo`U3oG} z`8Vgp>pNGqp8IwF@s1ysnVPCN4`llZAqO+t6^~9%b++iB{M*eKmp4z0I zkLz9+9WA|O*>>%x{`pkfd)r`rqZcFexAYuwJ2#n-Jko_enK zMSA(0qI0LCzI-+RcTZBgVR}F2_&;d!AmRS~`__%cd*|{@e(ew^!JPZN+-^{25B&Jo&VPB^d4UuCf6p{-6_1;}{Q6$ODFQOxyY7 z2Wt*LHq|j}dS%kM$8ekH->s&=7EzMiy&bCy;_FZCmAb#DZ{fcq?-}-h2IKF7hm7Ha z@t0z!uU4q$TK_trx+0gaulK@kNuBD0Wf$|m_Ai^(BT${#cIVg)wbFvTch(xmU-j+& zBeGBa#Ps=H{9o1U*&kd5Ucim%CY`37ud&aZ3-d1cf4*_ospkY zZ85>VS+?c;eBdDprwx1TPj85=yS4XU$X}`eG z?$(o5{4FJMdc(~d*Z7vE8}aNHu%1vKbNKz+;7`|Az{3zE>Mh83iI5zAAJ>cBN1b%q>2F-7PZd#u$!SKWxHm(U;g767+ zsXB0;?I(O%VHg>^AaNn&R6Pa;i}o|~=11Rhh}*};P!Wd`TJw=t55SKA1g=H;Z|^NH UEGBB}bOvOKr>mdKI;Vst0KZl<-2eap diff --git a/profiler/advisor/img/gc.png b/profiler/advisor/img/gc.png new file mode 100644 index 0000000000000000000000000000000000000000..29ea5d986e0aaee187758d38a536d641cd12befa GIT binary patch literal 41689 zcmce;d011));?-a+8hV_D#vQZC?g`yIo!KG53a!`r`}O1y|>7wF7z! z>Wi-hK-x__?_3j}t$iy>l_W*YC8~!N{9GkPLMk40po_=f3)> zq~+`Uzb&)uG+)2D?(p_V#k1E}e1n31**<>h*DvJJ?2oXYYue_-8_C^8uns}ndWc|* z((S#DgY~Vf+_T{52uG_JN}NhrEYMSrP%VevYr{1vP&(KN+SsA`xNNKJW}R?MUf#O7 zEvD?OGv?DlfX2=*Hw#`?J~v!r$q*QG({HybRP<)QV@YTfM|nR*V%z!WR^iR(rW%TT zmNL=rbETM2FXtrIpUQP2`NzFm?fhQ&R{flG&bHI}^Ameu+O4u9Sx_`v_v%kuMJ*cc zH@3bE9P)6+u(@EZonLHTzw~r-{%Z5fO`Gg@o1fc?G7YV^zC;|<*uDAX<>>#LD*XyC zXNve>8eO3jV1aQPmCyRvYqel|Ie!B0t*@MH`_s}GA(#dYBq){BAhPYz&!Z?XED>aX z9>Bj`w54Y5pGSA?|5QBZ=sq8T-dLt%<~q~zK_Y39BM-SQ^3?>Wo?;R5DD@H^6yP?+=Nv(6yc20|c_bW*|oV~QmcfSr7;jSQm1T+UZb2q@@e z(cli>TKLTkWm1P=xeCAWKCZt47`rhd?%*c37pJ0`=1*ra3(K`-WRY;(eZCUp(H8=e z^NrOa-WJQ3rwg8#ZYhl&FIEq83bk?DH?J&mK36UI@ISv-+!837u~4y+=`Gg+*bV-b=4NraV8OwYau`Xt{JpcpM| z<;~({_a&S5x$SrA!a&YDkm$aVWYp*p1!R|h=ov^XP17>(RpigX;8KF+e$u0QwmY(u-4-e{_0-0H zmy?v!^a_w5c1}GR0tX}#h>qe;`bg&GCTZXe*Dm;HMSWB&ir&uY*OqA@;a3R-tGbiMh9# zVSZnhVLmOt-ZA_sCT>#Q*%^=;iKU%%vTTQxRf(S4p|-wJS9frT+T1~U1Q`d=mora5 zFK?pC=nY=-M<)oya=nYEzRh_j0c9t*ujeaAr=dw8fc8D8ueS7+7C_l=oDE@wuB!pA zYL!=N|7T_?8j9Mr@t3GLv0zvFpc?1PSeEeBC(cndRoH^EgJiX96{eTlx2*hSa(`yd zJ1tGsG6#6S2m)6uR|(w7MrtZ>KstJc5PEao87ux0)*7y@gTAzbOE|sIye+jm8boZ^ zWdR)J?_C2ZUpc8C1h4{C$TwPK@j+ep5jEjSS3b1{8kUIHgd=!Nl8QIqH~^6Q0%DAF zz>>GqY@uyOhT)nEf`I-sctJ2=7_Y+q>pJL*3pC{yZl5JtI;WPTZ(7j7;b^H=_TYPS zdI?DsuG}LHpTZRHVD9?gMB0^rw{MqE!~Zz<8|lKZ?$ zgi(P1x4%BBp(w$q1poV5?Z5k>NcQq9Wh0|=2WaC}r0H$+SW`8dga6#^q?&z_l&j`in~q-# ztmyUCRPlI3EwKFcK26NBjDp!95GYf|PcX73ru5oppT3l9R61>kdwO z!;pL!-T}$iQa)xHm2Z*N6_31B9C^2s(UUgl=_FA9pu%M>5K4w83q8P#sD%vh=dpYG zYqy^J_x*S=_D8hDT=~TN?C1VME@`Kn-MVR1+J>oq;o+7h(G;zqLu$1ApU2;*&pWVj1gz3WYenTHI=;_uac=m%OKzea|x-w=Q|ayvIMfH-f^vi2eS9Y%B;14!J8M1PmfI^b0=r^i`R#aM!@hOX`8Vk$};8H_X@x4cfmW9)HG$qi-xTpCyoQ8cumX- z2ZP=i^QE*F+(DbV&+|=2uexqcdP~0YQw>oQqm01V7(Z!qy!ikCL}<6t&`JKw&vHID z;f;~IV&1FoeujnAW-ZL!8N)4P?Q+oAtnpH@wu7RQrJ19rcc!V<*T>Tc=vfn#!A81a zNgLrx61XeN{y!DFBS44^IR}LKY%9KG`Rtr`+<$`9dX-elO{1guur%TnQn+Pw#)YZM zZYISb@iT$T7DRO9+%gHYajUZsEWMy=dlzirr~Ns=aabwv#svKPLO7rOZW83R@zflB z{@+DyM*kh4^^1&4FtcfzU}X*15jowU=F{oe$9NC9nuC!WuPe~f66EvKg%s^z;xIwAG>y$Bi$YWjox~^g-hEp7Qz>B!EDlDN z_NOIBEMlRf(+710Af(gbE!zqD%S0WLvzIfy)hsMI;!m0uQiSH3Ea|To>4OBt{B#Wy zv?nAV!CQW+l1Bwm|1zz!M@W{9DF*o%OT&`Qss4R9C{kOhpQh%c)$C^z3tAw>o;1OO z1jU%Q9{C|k2&d7~h-kJxT)zWgB`S5hsos;d*P=vKvTR8``z%K2%Y z_dZsra*1w0wqw?HQiqjj`Epky*%rOBJTm{tc*gRxLB3SP-gcj-pf+;oX6Zw8#Z;OW zW*u^#^oj^pE>8=Gp{M9Qv*+{l49G_myfyifS}x>b(=@RJ!<|ec^!M;lU!AXT? zJcA#h36}P6v|1jKcjc?brU}6HoHX+e^b##zA?Qfd8>gH=9+X$!7dRmmIcgSX+up_N zEU;H{(g?nkSbf71(3-g>_$>K|^0qe`9P4R}&vAsKmO5zy0;WOsK>$(~iw(#+eUnn# z5^sXmrb(Q@bJa_=B?bITF$gam-k)J=Czk<0?Wnx3=h_+!`2ZDzWp6;sX~t|gsAiTY zm&blSB2SU@^WQ{Zn%x+{KoqTQbyN%>=NS>$Q0ItPX`eR(6q&6k%~&AzC$0Uo;gOoC zExj(EOO_Lm03Oc38Fbjm<7{>0R zQVJC1A_71;j(J8nmh+ms3cZLC+vX&)Ho7|VK|@Adig>x-8-2p#8fInIDji^4B81*! zqWZgsP{RoO{E(!jPTy|m-xx8-F)7(ZVS~8;A?8pC)DN&J`%R#4eyTFy(PN~dgXm_X zR5-qpE#SZHonSbqdB&l~Hfi{4o~znesKYCz8xdxUhHD&(s@R9;E;Uv zAOil6L3rg<#m;OPYl4e%?l&3L1w3$Y)o$Hznw_pVQ7&X9B|9h%B|dtL`_X-V5pY^Q z*jebVfIE|>Pr;qxV0Fc2nADTKRB>Vz>K?%gP$Ol7?M+cFuS1oJPGZ|>=|$I)+*EQT zYE$T!k|42dNhki@hr-l>JuMtJwHuh#GLB}l1{4Fv7$-ZtGH=z;`yZ-lK|4Z(e3+`t zRn0CpvE)6Sc=TeY0yJlvn@RAmB;1ek${8Mk2CAgq5v=uEa)86_B=sr;^ip=+~_D!tlzk()$h9c5}BVXMnSq{Z3*V zhfyt4368q?g-v^^z>0$62Ev#4H@5tl$b}UXd6`}c3_JN1@WNZ zl|8mnyDYEuejeZ0ZiwV3bNv~#hEmgyT-mt~3^ndj&5ux=^1&X?tTN!H3O^!i%{77# z`C$ZVkumW+@$bIMpU6vq2AQH0fR-9-vIc64+u3TRp(V>{P^6kSSjwJGU!W8vp?iJ3 z3XKwo3Oe)A$7?u$xzA(E)1`Dh@^TL)cP*)AAalL`jg$bGvhTiTCadJE)*k~%$(=+D zdoqm(%2k!#U7(jl9V+p0Sp2u}|4^%ww>-&8#;oM*BIuKs$3d|hV+6FoPtEQE`4|g? zln~Vep}Jpa$}zsoh3-;KbJc_m_p$_VCG%K=DP{ah=J+PB<&NMHZ!xV;lsmrJ(`Bnh z-X-!?ZE0AfdhMC)AfbM#qNc6lP3(?npzcxeDlbLb&o#Hy}`^aWpZjDY(;0^=8I=A}U>=vc`ni_nAOdftqh52Q%M% z+H^|&Yk(tBDVsj2NZEL1sgPg>@)-;kGN?0gIRdhLx=g-?RVHv^9;O1j=V;Tmd0!FK z#8ccay&wQ0E8WIaD<(B%atBQ?E%bX?DN8ByKQ|7d#x_Vd;qtqXG-J3ne+t809`V*x zxdkhCPtx_2%MI#{AR$pLB7DjdDW00xgcv0>ed2EzASMd$L&MYMLe{7v*EGIFjk4c0`FAixoq7V~MB}44^hOlG!S0({nparW2B)*zy4t~CuQxnH&FM% z=1%H1+@a076vW0bGw16yT>uEVkb_}@wr%Y&{wI|24)A3euLUlTcreVMbr+%-f>7?71&8G!7Kb(b!MlgP?y?d{eSbRRSv<_&kdORX#U?D zh*tdI%BeFfseqpHPMFmFVP3UsqdR=^qe+E}n>cJ_lJe+7!rrY1XpC7{TYF4Ci?`Yys4+jFUa2It zX2pYh`1T_Tw*=;^0b8;%VQcoxCLRoLHA|$4YmFW|V68Qr7VNE!KJS*bfXnru4^+pT z?*Dri+k3}IRprC+!WN#;Ub?FVNW*cbr<*>?%Wc8-?Lr$St7HM5bUZc4#II-6gKvVQv4tv@o z;LSmQhhoEl>!i1aYdB9uoX{!LoRPo&ozwJJyv5UiClN*XPq&RQZ;vUGFKAH~$odaqB=g-a@*uRKg4eQXYn6TyO3*~)IqHkW13O zk7-Qmu*0N|Egm&uQ+i&;dENpa=>)FWdCu8b>kfzNz8h%z2(aOO+XHb2xBN$9!V|5* z;{)E@&UyEpwuustc>N4MQaN2f{+BOgVdD?k&FAc+Hre?wR$na08W#b^a_$T&Wy0=Q zQe|M&Q9PgiW_(Mql_K*CIc+Uk+32Y&P67l8b(e8y>QOjTNUQe8i~xq0zYZtd<>o>1YQk&nhalPvgxO? zVE~z>yY3{u!GfvPZ%-a{`~mi;*}vcQ{Q3Fe|z7x$W`7$nwjyjRrD zL1WqqWtH+?Cu^c^SLM@?agV|U)QJ&NMCRTE7jB&E64yGew`F^R{uz1w{(ZAaYY1k_ zvu^uEuK7$sZsuGojxd5qY~oc;bO%fX(O-%B>8}}jwS>+%+@igZTk-G{9(;2A60(`$ zX8|5uahEurOQrW24#UbXAHLLDSBx|!*|ypZ;Gf#=Mp+GDE-djym#VrIVsVmJm=iSJh%H1jhH;+=bJmKdeO< zl}`(;g4x5Zhi3{hOO7@?&|NeMW9d7@?wUmc2$kHZJN!Q?=^cUVv-{haiK3@>AQtAM z_F4=<8JVRUy`60^cDQ>ypH8(B8$Bz9Lw9hinZ(!`&dCGpmDcui*VwI2+>K{hE03`r zzu1lj6ouv8tGXN3SbL%8L@T191&BRkFkUH$EZ{}-8v2)(w%6zj>zl57`TIcTQd;Xw z6$AWI|4G3rDeyV-C8KB(urEIqcWaWBO1qZ3voz%G?fE~}<*~4kJeq;M=!y5SokvN7 za}?1->3)3SZIs*Ii#L5PhTqjio!EPWVf|DzznhinTU)4Cu$NVYdS`>N`1y1$BM@Nbp$Wl2ZrJz$90+5ycHhe5uaIvQ7Zc?oK+MK?cP@e8alS)ULV&SrV(BjU7t@e%!8vE!{Lh&Y}S zcveal>A8d2j`loUjOABW)RC)NOhr&p5|_lxO0fLvQYO8`uDr%#cOK{Lg&1xF^kDgl z@bCi1N@<8|3)@$lTM0G3SMnp#KcM-2)e_x&RqOjvg=A`ONJxg9E>(O0Ap}Q?682Cg@A9N>vpxg!uWJgU>-9> zr(0M2Q|8HEh%msSq=4%Iun~TsKD-S(uBUqWKJ8k6|-L0P3 ziUhB}l!s7mRPyyKUj)tp!FXG5kk|Sg@wkXUo){ARrdL>ZFYJfFLY8QyA}`j4Kb+H@ z;>ZZVw$*xf<1?p6I(L*y11;>El-E3;!Yr_*B>`V`v`ks;J&#eY>THB{@oAO8vHT{P7?vgSL)oX_)`-Vhg~ue*UbvMlcO+9`=YFh?AGVj9ZG+q z-wcj8Y#&176TK{qZ(T#XUhVMdliDY~`B=76UIz|ENrj=%C~syEti$Rr+E}m@q#xM( zRIy7ida*tvpF8x>vt0cuFmi-+l!WHuyai9oqEcA9QB!v7E~sHrRL5;0?u}6!T7f&q zz9{Zk(TfZM7f^qw=EdDJc?9|88E6TrGfs+X&59v;?V*jCFShi{yu*;gqCksG`VR3~ zor)VZdY7XALOGyz#Rg>q57F}jm|keIfsnoE5fBq;+#bQ;W7ozS2O=FKQ48$`SS>+g zPjcIO_Ub}I8Lq+U&n6!O@5R9GAM`v3c=jNlR?B^opx5={a9dwOierdLO>9L%DyqK> zQeVfIBbV2CGF>1Y}z>aAqmS;IDr)ev@LfZq;(qp{u;6igm6&

Fy6LI5Ad=oqXLTL9EXNr=e!<_i7WKy0_ zEFS8M(Wm7tOGc_~{W{yaTYe^yiyDTsbL42hET|)`lg!}4Sz~T<(e+|-NYh#bu8qW! zv{aEG16@<1cdeN9DdEX;mJax^NZo#mGq8z=Vz=d}8s_=xCe2d#U0Qe8v2hkrv%=oz z719XSghEv`T@Ro_gwk{GnWrJ;LB>7>3&}UJKfLAy_=HU^Aw!A`#)qw$K?Doq*_3nL zdw%Qs%|x@I653NVWvF=-QG5mWC_Llgl0Kq=?AEBM3pwIv%d@Lt`vP?X68@UAIL3P~ zzTQ|H*RR|zqK2~*&Syq*f{rmg%bS}SU`STha|!g#i4TI9E-Un*Z&16LS4!zsbz1AE zs={tipVq=ENEmFJInVmn{!>48t~Ys%pBi1IM~tHv7EVEyB_bB{^g z{A=coZ!D{1xYoyp@%3(j**(wqsc!JD7JDULs;#?WKk{?nk?|G{_Ks5*6OA-s1x@S5 zJj=mTX01Q%&;?|5_>7(H4%%NEP4uC|+iFj)e20tff`(VOF?{`1y+v8At&bq~BglUd zHi2<^Hgs5zQ+ICeko;y~#$fNpaUcB}INwn5ttZorCsq_t$?qr0t}du0uk`RCVqDhI z@b=cQ$Ug1<)+hGcXHl)tYAQsR8_N_;rzIW98ssfPsU?l%pQ>fSxh(7MgsXrrJI7R%doq#=6Z>exU~+?3Qst1u1u!GfD*6DeqN4luk6EnvgPSHP^9c z;8H{-F~qfwVSi7mlb=Zq$eZlYG+9A*iJH2{0*=?Pb+b1J!M5}TJwmwYppz~mx0;4R zMO7D;(INB<{zhb-Q*e9Mp-j6Y%rK$B{#^07OWp@@;SLMZm=RL>b7(9(Vrr_6&{&*V z_2SLby1XGa+lkZ{N%C6HtD)U~ZP69x-v-_y7qh~~1Hk<5dI!#Lq!`PlOMQwD1EEGS zBM||mOANHFh3Ka#&es<1-4xM8dM^VKF4#X7TgkmQ0<(lqpdtx>@FL#!N{{r$h>Z@o zJ_*T>CB-d~uJC@KHY8r|K2X@3n;7ETWc#h)))jpYHQ)Y`h4WA(<6!!tUduhu84o=SlvjG3`UvFV%U@F(jNH>$L_izT>o@j{ov zy-8db8vUv0eZuMRd~&&HAOl$otx>om9A{6(@B=sA70$G+eiylJOe!ZJju0K)Xcqa* zbF)wESbNgk9BghS0@iB?_2Bab0qrjTxFpNTwSijwRk~=9?#Znzy0tbP1P0S5EtMTY zyOBC=!sFt+HQ+;4EY>hlB+V$&rags1xS=m>vdVBFRhXLf#in{_wQHuRg)5y4@ajU5 zFDZJ0qH5At{YL@Nb%z$tTs;I5AN(QhrFKqxjs4}!h*KwO1FJI!FQ8utrue2XfGDE2 z;(PYd7-EQ)10O$vK$FmzU57H8z5O!^g}6S>gxIJY!Rh!+Coz-4Q==VxH3>^fb<8l}zGa+%Y-E?-x?4wb8N_o4DHq~MBtN7;b)*^5y zd`unE@5@OILEYJr5qAV*#oFKFjORv9{OeWPr~1P~XL3yIC3F5+BN_iZf^Xahx<{)& z*|D%=J%|+`%dy()Ug&OGvGQK4W6#V7kM6-6Rf34`gwOj{{8C^8vqL8gA%?hy5_#u! zwIRn->59U4u8|1T=t67!y@gxSWTet_zU&-Oj2L&k8k5t!3gpZs_9ooVs(5}dpmV|A z{wUX>HaDsdrl6(VSqeD_L(xFLjWN&xQ;mC_mDAf@})$WvqpjkN(YiCXA5IHF*K{OhU6^ z_;5lXA;aZdC-+u8g6VP{p}3r#IHxT7j{O@yro$jW42eUsaV&-D^TsjY)nf+7c;><% zoNS+#*SHEH-U!`B_~pP2|__ z5jXj0L<%fG*ji<)3XR#!CVh3O`^1caUKLxluldy*SS zI~TD`h~aQ=0jjcpm9pexxAJB zF!#Abd5)(q_ttioCrrv;rq6|b4?OM5i7;|;cg#bxCq+9PW%> z;WXI)Rv~~|a@wTvKje8ZF9t;z?e~G0`3efC8xs(!Q4RDA7gxmeO3`AxmN{j{UQk_3 zJYJ4q{&pcoL>k_tHAUDB5bY3;o^fL!krKg*1v5am?0cGCNrNds@U0p+p!@s2z*f!f z{Z>qbPU*i!62WK{=T*QxZ^YiOnYUv-rA4*ue@>y~f7&mKI$B0y2MbDnO^H6(?66qC z6rHca!-^ed+M2F2=oP5wu0wupft*4`%w*LdYA>dC$~XUhs+WO%&0{mIoU12Icg}Bc ze1+-#T(`Z>(wArVo@(9b)O{Lo1!5aee;{;3^+yR? z?ph@A`k2wk6YHz7pR|-SbG_2xuFy(bU61e6_!HWltABZp;hSO>FaE;p`mkCT-)!#q%{G<4v(<*TCFn)JfV>1S~4gYxEEb@vv5y8=YrvqEWwWuf?AP?k;)@Zs|W zg9itWp-x!mouTMNy9nE+us^opG;Re)r*cd0xO}j?hSb)- z8DAX}NJ7+Xb|xCjS+DL2sUvrPxMs1C)h{|`p^m}>8#fWw`xKUc?|3^?2oC|L-M>Qq zb0fO8Kg20t5@f9KSfnvO9xkSbG3$aEa=)jCV+0RT-U6_t8LPJ)M)hVWHD2gU5@ZHJzb-3O!?Luip;T zZ&P9Z6PFjyeG_(^D7%xyWK2+~llhLdl8GOf-*A(Dy;m`u65RR{DgI7pW++b02aY#=F)iV03Tsz`^p#FgQ^$cvAYWo|DN%n{pFnb^2^|r zm|oFc)st=fKMmKc9b*1P%a^Y~OUx2|#*Fr7T)Qr{J*&Mh|Ar*6hgw%#o)C5HCF*$e zZ!~Dpwwn=#CDZ(5+u6#48vPPCXK=$jsO@3UD)Dit>e|(rpOQ&)a0h5yo1J^@-Cd4L%|r3E&LymtMjaw3mJ>H+<8b*$5P zG%1E#laB-NX$_#SyjIEt>BdWSPWjO3-t{l@;hWhf-h15VF9^F}=X8P~f37gPPqS_- zxRtaFaj_~ea#CJk6PN= zx+hBxt>q-Pan$+4ISp^fZl6MiY)d$FIoHjZ7>XWspWiukN*TRPwgXx5&#@Ee{Rj*Z=$h<#E*g_cP{x9iPJEEITfG zv99-lZ~j^4E15a)e$>I*y?Kb(GePy2)cYBl)!SFp_dG0*^VdJ=fat!^g{gKpG3CK- zw@8wBz*a5CG(y~n><5QA!emp-0s45L=eIRkO>WK+usziyr8%6E{T_*^CUkyWxD$d+ zd{YUl=8lBRC1c0tmJVatS*d{x!D&#fTw1B}35J7qz*T>)i{I4y_Ho^rS53~Q$uz%*$L`Fo(ET1%=Np-Ep6FRB^-Ik75&YuO90kMkUjV$3F#!7l26UbRzk7I zz7xd?Ti11-F=24PK8PiRVmjUv_P)s$+i(wIo!3q_&4Z23OTO=mg;o<<($+I8pB=v& zn@C8Qi5M??Bsi~KOuFvV5NEirB!06Kd==&9e2E2zniGn!gNs{MUOYI82wpbj` z3}w1AuU|TZ9kpz}Qu}8FEk*B(ZNI}vFAdj(X-Na--?1}JiY%?%Lr&|2mtVUmA1ZFY z<7w_ani0`9HIiR5e)#V2Kfbw#oQ<3O*rL`1YLdoZ%=nk79tjF+hh=5^vk-#7U@Vn&zt zUDo#pY3tJS=j=WCrQve6(?bSnL{NRM3^lnp@t63`SwPn7xIp@&Ifcbp_EZowkUlUk z8P}ot`NnDOPJm_}Av1>}E+dMGAV2`Of*?LbzvC3%JZJj8#n$zd9wW?vJ~`9qXVX30 z>Z`+jfu(mlp~qUI+@uMs2I`F1mgD=>StPw7*YffSYetdi`}=7Fm>9$3Gs2#X$@kYq zaTAA={+vWXn(V;&beXJFko-#uOAXG6JN|I@F|SegF(i4X=d4 zNv!YsBF3hIydOTi+t~eR;MDueH<#YP--dKo0QZ_N`xKR0HzA6xL+gS7jXB*>d`@G? z`as}New1c5I~>_cw&^vC{)YO*ubkHh%jx4@j1e~t)^$rW%6L|_fVrPA+sniO5Hqg* zjD&}!ST%M)CJ*t#70-HY2hsw>K}-(P$&?*{yCkV=sSOS*%!+80u5xDds80x_6i*je zHl6q0DQ#BqUmEYJTd8OKz6fITNPI#x)urXEZWM@1dWTS#Weru@eS*o|(BG=AZD4b47A zgi>dnIRf`dJm=TCjv&6jchB5(l=hoBp#ptm>1*=`BedAk2@QsfQQMZ~9%PdhEUC|> zSh)n4XOHd-@oQfBk_6*Fg5lf$sl_v5t9{LZZ|7i<=BS^%x?k8+Yw~BiMI-xTfLf7` z`IdooV6a26rIPpvdv0=Fnt`dp^2n~eU!VDaPR6^^)31bxLv#mxBO)9V!K{EB6PSNj zyh*nMGrHd8dL$~4bBeceX-=BIQZANtO8kqu!#E#?5*d9$#@UvHb^6r~&FuQj51O-M zu-4|wr@~0DWEm^&V0 zexP$v&$fH4m8^5>U|#y1qhT1hH;Rfr+=eOU;jziV=8Fk|O~vU@1RYVPfSOWJp z=wdHaXGW^@YJsPQ>P^HEb`?Z#XF_Il-2-#R&8B8uSkW4!;8trCLenMuoX=I=g_U-} zU`Qxp_fm1g+un`?Yjr;gllhBiUPgGBqo=#%dF>1`OnEAhYk3#y7f}aPvsy!otNVPh zxkitND-`JtuXmq6|ACk|DWY8qYBt-K>m+(=k^No`loy@*-ik=$IGOYt6zTeR#sgV@ z0^jg>nciX5HQcKFkAZ!s_G4>4c!!G#4arXWHSb67VFYV&a6ll_Jj@$Q7fXX&Z=zn9 z)k8Rb+l5Wyv72Oj2ln6%<6xgo5zl(0h<22re%uR8ggLsvcQw<^n;FS*0D7H&cVXAZ zr{P|CAMn8Do*?mM%F$V!8~5sbYD!%&RS}*DuW?9IkTJ<94-ys_1@b6o2f~VKv8_v$ zir&=dGOCcAUpG$NzCPez=7c?u479+8T@*vWrn>CW2ltF1VGw18$mhm7$(ra&v%WPF zVWCcvH8ulj6q6gv%#B^49cwQnp+m^?yTFXB@_m)dPb*gbj6W65%DZ-M?79_yBiu?S z@S=@AROy^(ig=xI&B~Xye@HTJn-ocT{pQ!ELx(&R{rlct0hTK9s~7aKO|huIueGc= zrkcMGG#7Y`PI662i3dFF0f7ky`c%bDT#ml*J>=d~IO?Wjx_C(ZI4Bu!%q|$$noAHg zMS`>X z&8o?_yBg&R3m!~~1QP8Iv{D@$ zT0ls1<0x_MOw~~G!nqK6-bPezaAd^nnfe2&7h0t&w`1uOO|)Ij>_%{L&0jsO$@61JWyMXQVM@&}*Qn@_vo57!4 zB422i_o|&wZbk{bFSGCWUA|e!8(C&&uk*|*+u?d&N zzgm`;c}J>ubjlm4z2Y+Z9d$-MQ;_X1dKjnb!L{K92M>Jeg_~=)&?vsuB4&_E?{_ zFGu~%bFKi{O^Ll830915QNLm5g7EL|*h-ViitmN}iJc_nuSukZz7L7~$D7_LK-}|-C-n2^cBy%qnA%R6$8;t7B#r6<(O8w3*KKcYZp}}yy z9B~tvc`A3C!&!D+0{Ms^mdf^zY72uY1{Juq`KpbHq9(B6V4fHdP)Lm(S1uG>M-LRO5yHLyWMsIgao`rRG!=(fa`H<*1z)rB-ZGv zqjf{|mL(Zmz7Jq-Ld@y-SM4F>$A2t&I^Nbe(NZ(4edp%F(ogDI=)Y6Rbqi;vJm}Hz zKRyL?tiK2vUa(!UA$uOx=#SQC{wm2YTd=LEF{%B-NqquqLJf7t*pPpcE{nWPOl|i3 zu1*HMymoy19@d|8XFsUp`j?7-YV_~ne3K1ns^*<$Ck2^o)zF>5H3S{lTt&11{!8xM z+|SwXC~HvFv8^nvMLe^MCS6|A$YS|JS_I|4Tz!l+90rfikC4KieeSb|1w$A=ghdM`)>5OKElcFi|=% zwE8TSf$?T~VC}FgzyNnFhn{2N;jE=G9}{(^e$-ZdDyC&?ffMv}o%lN_Wg{Q7{$Nuz zAHUoj#Le47`00(h&Wx)CrJu%!RG$>ImBb5TMHEy+h|_7;L7=s<`P(J zbQLJA6Gw15@jot%L*x^0KM0;`h3{I8UF)@}fS$)DgrlmZ`|e#m##cvDmo~d;aSo4b zUfDhEll=|?5)8U$ze{koAvTVj{Ta}oKIKwSc&LQ)DbDbycxig{ai-Z9pf!SZ$fh{k zH&!rLsk`m>3fyMSrY}KBx?@q7 za?3DY63t3fpP{h3s}`fd-p4~xp*}Hw7H-+$b%7JztUlSB3>ACMpV47u;W^v3J>L9LgO@)pWf@0_Fg-|DE58Ltxa_N+bg}%{%ZqMvdjpD4D70)^K2_3O< z$qo@n4)4>n3-?WVvt=^O=6Jv0vc?0Imz3K0N{$t!IhYj7nPkqcXh+P4i zDN7iNd5}6Vrt^s%?CzZOr+rn+#8HjlMs*uLo|{RnF{bYSgeq!b2P@UP989mxqV;4I z_7ZC^j`Ab25!puaI7dJVPi*WxuAcPco7457bmK_Yx%Ff7O{~=QW9kCjR<`#G*R}D! z%(6J??uAhXU7c|IGIvwI>c5Zw!`ZThRUZHT;R*jINPeZ+Aq~a)I8^P3)p5;F|Ak!s zzt7#BZ(E+fy_t@yY3}%sf#QDMw0(HQ&E(~Yj5OQVLhlmW5J$(PqO=$@gZAYcK;``H zl@GT#n&#y5K``oH%^@e$4^5hcnc@*EnH!GF58-n#fFmg(#Trks6`fvx@*t7W=_E6S zN#hY?a~N?OH5fVI$frM7AA9dunSm)%gnf3aOC3#<_}ih1i`l8T-2hG7)-P~x`(YBm zb*_Pa$cgSHes<9*ROBk@IPx*>phQoss_Xc=LPPw1H>2~r(kJxncQ2k(m;kO&Oa=)_ z_H52+vKSu6T?@#0HysiG>9>$i{1nVavQw`6{IdT|r$p8*;&=QbxaLoN{3LAKMiG1h ze`y~%E$7?5$yi4N>hVuQOG;D6;W30x7#i>94A^bk^HJ%D6u_NlYIBM*ci(KOY}=VM zsq(qa!=`WSq3WXD92Ztzok3{RSqNnxN1AG`=^azdM^BR?Y7>Gyy=lrY`U!zY=zQ-U zKMrLFij1EFc5~}xrdNGi*b!qsVmtI&8C7o8EeFEYr{Yfy)nZ=WD|{tu_@kAho7aZ! zxzPt!ZL@d1hgIK@!Zpg>c~C>)=a^uA;d4($CFZXsf|Gbb z1qr*-x&-ua-e*_><+)=ek@qC1nZkEDvB#53(>x`Q&)Vd5?*8QwFu8m6)=b6E*^o4? z)LCgeG*hc#MJj2Xkt0v-o&BqoYG1+b@(x_HSpnKB!a8MN&V2Z`B7 z&sag);rnIJbiJF)%M({+pg=9yI0W%`oKqPmrx_d0mdT$T=|1do6i8 zb0IIX>wi)A=0Ry@Ti<9-Pw8|9J7&;0raKxlI7AcY>2xMzG}^|*5v3z>L?wzdB7HJ; zjA-IKlf*<56$s9#U=kJD0f|vTMI}lLDk4Dy6a?>+KIeO@?yb6S)vfP-ci^w5C?1}@ zpS{-FYp?ZNzl9%WUsLZ&&xXDEZXj-LE-fu|%42!pXH56}`kUAW$<>U)YXnnfzr3+3 zlR6xQ7~5!RSP3-^RzVgNsTZZU#^8}TKc%y?Nka-&?`(MVHgkUN#c>RHFD;|nRavs}< zdw6~#ShpiysVhO!IbIaoN9Sr1J@=cJa{Pn!U*jr<#u5i1C3sirSQ-OO9SCRJ6gErk zpoAE<{(z*h3@7?jQi$&8+}1qw)uS6Ceu%Xfpb`^S4RH)!jH$|YTn zJ>A!i4cYaGEJ>`PT2UvDQxJLKBc;EA21>OCW!^+ezg4aulWEp3CW$8N-A{6%%omBP zewWUAQoco0-SeAq=$pgjyxi$T+$hv;BDbhMU(!F!Id>%GmHyDRU9nBjVP$1wD#GahV}IworOHRaBx zxgy5&tC}p=v5asnRFbm)0VGa&%h>O0`=z=p&L#eURSZ1+jzm*O8+S#XM3vB~TCzkZ zeFfu^VJ%FL>h%$mmeH;%Kn>v^hFG+gNF{!zfp<8G6Pi>=B+-z}SK7!xg4fPE$$qFe zD)C2^%Kf<(99P~TZX+e|I3#4?R~!&Ohbq#3s3n`_Yu`{ zG|?J*V-ehNv`X@!V-{)8)aXGm#UVU#5>6bk!WriUUu^-Yxw00^??O8|TYP)0j88L) zdYvuTECX6MEYA^NucXmTpg%*qmb^K^1KE}#zy83Bt#=N94HIrLdGbx~BFiNffvN_<2xHT4+(u}~e2YRee-VuY1+pPQ z6NFndM&e5*OcH8yu<1!qAX_m!<(rNCQz&}w`HQ|i62onIE&>2+S%z)Y*hZ(iwghT$ zwCuIN(1zl$XQVMU3u=XN;rJD1Y&Rdz6zEhvRedyJuK&mjLHmmL<~$RE_wuyC_UI$J zFpi(1qB9!03FpiIoNA45dai&ulCPIs#K_~mo97Z@vMrsG#NpnDHQ5Q#i@x#BFF6k< z$M@mGN*GGr-pyC|8szSr9D}u+oyQxrqwuSg?_qO`?+VN|k<6t22iG@#f2eoXrI&Va z`V9~hR8{(IUGtUyxO7eJc3gQVF<^5x80lxl#lx)m;!3#|;#cpJ)|7WOzy=7%aCg#n?;;>q##`fe8$)(6L1P)C7z)X!2gVOx zw46)kSf=d9Hvej?Z?Cvo$8kQ!#S-v|?0A@BF0GYbeN~Zh3}6faNja|;UixVpHS~aB4ES)b*1u8N07%b*F`PT#89?E}o z zD_Z=)!?8!cyWu_p7Z+Cv2SJ;xlTUhcxT}) zxBEwHiL`k(nY8)?T|p8fUan=Qm9Qi}xoG?Pvd-4bMndIovB0FqI_ zT98r1VY13Ope#5b@W!Txc|8W_StnX#m8hmNugAw``c4QhZZh~fqvob#iFZXNE5b?5 zym;kxW{ku(Qy{eoZv}C#cLqRT>G+;;X(dXJ3Ef14Cj?Q@Ou-H)3MxR0N!H{WL^ArN zXHdx^?;2OM7E0kKp_`k-Rx^Z;kWUX4ErOgX-{8=RZ;Yy0vSbkbR*?GkN=?@V*VJ?6 z&pUt8R(rU{Ey*boSm6YJS$m5fAHwrt^>43%Qy|uqN=b6KlCReM*KC)?(4ZgK?NIBO zd4hD5`Re4z0!o)X*WCz1**fRLj&qMUTW9q8$)%awVbi=SA7$^RkLb!x$x6N1{=(x< z?u`v2cG?3v_}y2Ot~MSOj9q?KSHtGI}qP2Uea%v$hB%s2lYa+OULKS;&% z;SMVel^*033ZL8lL$alBC@F7Deuw94J2b7uL*Gg4eY64T{9>EJ`y_|Vcep*TlU4R_P7-@uKh1m5_!ikW|*Q{*nfnz*~ zFZgFZhrTp0cgW!g4}Pz9<@~5H!SO|C9(>GZ-??2QEqtO*WxN>F|WJ@F_t^oxDDfuc58x0%T#gshnG=+m7m z5lhKG77eX9_Fmz2HMeLf{Trb`0N0IOlFwwjJ|YbPUQ3DN3BzIRIU_l=MTDahd{=QflZw-YK^2V_fHBBa*esvhn^zFWQ>M7X6RYG^SNzSt zh)gIl<~YuTYuOZ@ZQ=I(Mq60;)K{~1&^6n@Rs>~?C(r8&MwQ;(Ddas%P+&qDzfW%Y= ziUvZ)Y5^9$Cc+o%;*=yYY;aOV#JI;?hkzbkcN-NJB-&O$vyF6MXNvkJm-+6Dz8!)h z@mP6Rvz^feHj{A{6Qy8X` z)SHATZ={rm^FuDQV-|C+D`9`k!ONgIEo@751gjJ+ZuSWe|Br#eO2+ktOG}g|STt_= z17qqz5QL^+HW^9#L=%(B512q)44SCr;>D{hw&Ht47aXFr2dVJ@x;xvAp9+t`{v_an z^uG!1;9&R@)f(&tY{L9{?0nttvi4TO8Dz3PV15;|_6@h8Cp^K|BZYV}nqqP@EK= zT`L!A>f94!sRx)9NgB9+Pa*}1kRPsO#SFSx&Ab`>u=P?UqN92KcGfxJxY(K|cg8UqoXHvf~nCNe7drTG5hI_7bu#zm7|cPTmx& zc%j6M{$`64bB_xuLabVXsLp;>H&Ameyu?v(Bg|34pBv9vLGh9lDqZU5kRZ=CF0cXq z6v64TXg?)^lvr1+JBO?2+IOPTG(VL#at%o%B>Uw4G)N3zwFnt7>O6cF>e1iw50NQoR1)$7m* zSG6H>Kjf`GyQ;$$KP|<_r2m+29UjoqY1KubJn&eXeax_y9)Y&j;nNQ9x4t>?z-x@H zD{0mHL2@pJe-3F53LKqi&I|5H-sRGU_6axsP&evqgc?Wd(r`8=r0o#9_$MZ-yG$l9@+ChpwXHqN(YL z=8u?{o_c|sW@;Kn)@wKPsF>cvS+1!uscP=fpM>2BT`!Tk3>TvyYtJZL%TkW;Nfeqs zzX~D#7L!$N6B?7OJkBW#wwGE5Zl6x4hb9dU6E=5F*lFSXm!|f%{&PPj-Xtv4Q;T_( z8_PU9mDAAm8Yk=!vIt}vIVxBwBISe@i4G_Q{Y8L>q&`K35c_uKP}(0k<;wznVd?mC#1~t3 zY9;upuk*}BXN$P>qkQ^HN^`GHbVUyLRdj3c9qqWCcg2C&-e1~cle76;=ndntmc7Bg zGV9ZY6N&60$pMiuwMEo9o<4)a6Un}T8Z;(FVMMliZ0LQX~f z7N@vS-a;LYM8C|Ijj}J6CpYQiyMNv@JsCywtTp$w&iL*O<5nI@4LbPHY%!x}6Ol%(FVg8{_pB|cO{WJlT4rRlFkD|p1oqs6DGsfxIHtEXh#QtN7A3>+t zo);#typO+mx-(aV9{a@>rjb)r_zedlH1|VMnB?3yvDK(ch*!Zas&gsS?OO$HWJ}xV z;OAG=eV6kUR#Oq+;?pa?r0;o$G$(iB4U>qx`vA$q4hJvK35D6$t_yAeg0dn{r$bHt z4b5Bp-nzc8E^SCcUzsc01tR6Guxjjf2;kqZphdQyGDu<5r~T))r!EGGP$Ows7vm!!)7f+zlWnGSAnieUE6mz;ze z{#%~jBVT-Hd7Z-69zY0VMKCd(eEIiSj!Jb>lHQV%gFo{DFR zx4r>IN7G-Iw^Gxb$M}lA)Kw&rFKO){E7F8Bx3Ixu$P9otcK2AZzL132!RM_qJ`dTp z-I1hSmzgDh!U?zBCX2RxZc1Cq`vBle(zkB6zuD-p$Lr};cK(!uPo2{i1B4PXbK6)0 zXDgL(X7_x;bRPS6M%gT!CccFc)>&r(5hUeWTkZ!4X+xfJ)(lZJTGSJS@0ykX08ucn z9c<=r$%RjMXtXo!+7(XaZ!}0Wv3L)IpZ93%ij-f@Hd)zX)rW4gsj9IDIII52DDYuM5Gwvw{wKed?-B8@6Hx3e;)r%2PfKn+YAxN92uwR~OPfKr~3CxG|)KQziv zbKEiCIsI2j6M*HuagWLuJ_@nw4T^DAZCHQ^6D=)T>+x+FcHQW~F)Ring6?>J-&n}ZO zxDW>?e=MplI?hRXG}bnI$j6rx>ID1cl2x$z_W-6`4ZhG_Bu{yW0Z z(!zd;&UFYtHkhqDZ|+Aa3u1!%Kvb1wn zi9SKecNE6$nB4frrHKy@E7j)HGFG5H9Kdsfvjvs8)WIueub#M9E! z-YIW61WRTkqVUan9*fDrR;3RT-SjhUO(u0hNMz?%a^$sXZ&G~BuKTH;+^20@4ehl= z{(i{!&VdI9#)}|<(AoxQjIkBuT9&Q;xsz7j&SDXd;EFR*=ulpFQtuF>9o^dDfpL)_ zZJKK^*`{*5v;%k}t$97iHpMp=#6RW$ev@#NF5!=?3dRpgd=vli*3Na|W1Fi#>sr1v z`fgGD;&+_&Uoz4s=>0Xk+oQ}dP z<|g9|uEe_UxlVW=Hkno>>Pt}$g%s=({S54v&N}%C@AQxIY$`YWPr^zxjRCzW zTR-8&5;b;ZU#Zi-5Clc$%ppgac|ic0cOomp~uSkk)5fEF}wQ7LljJNSw`$P{p)HFWA2d&)@XK2m`IOzIB5G5pDZhktyeU~ zf@I)yJ*cla5<`vm2R~PR3k_*+3D&Tm``F&HBzqbVeh*7L$xE{>Y;mSU#MS`ouIWi; zJkQw9WFKwuvfVa*ct=HvmB&W)Zv$lXEhy(y0L!>Oy{&apFWX3BY2{@y%#GL^J(FF# z+qa1@@v*IXrZ{maW7&k+C+|3b5v5PzBL@YxIPU9c{8+}=%7=@ZBjjbqBpR^2UCGCp zF*;itDIa5rD**!8JeNPIr1!%p)g!O?R$?}|<6yT^^tymhDIfjAe>8`tDu`*CbJXif zbO?~ZFzilxEpH$@{2V+(J=C7mfT@uokR;1`$;-}`EEoR0cD`neio4un&2J^ z{s_UEq7_iKwMvfa`Ar4)W41e}?749de)8bc@7i2MRNBTx-?s>qSk9&GZ`s=ea7I7| zWrK+ainz_zsTX4hgWxe|Le4NpYONROlL0gJFjTZGoxAOym(|lOVxCr^h4+{4q+N@> z`#^0eW#ps$M>GDP*y#97noIM<uD#{T0fID~^*pEh^U}3s zKwYx55^u%@X$wdjf%dNzU?u(6D1E3@*5R2pHXTY?qKUBO%`)uAJ=f|}@LO^ZV z#VmJDWjk#X*8BPH`d_#G^nuFntanjcx2=2_ZVbv5&R&xHOP@*nk|`uGn3ah{IiReX z6o~x{??HBq+;LK(!yYN}-^;(K@5>zZ<1K|#yg<}({9-91eUMie{!1{spVZ*PJErTB zKt9tn9VrRFnq~GQI5{D&Ic*c(NcI|)MFmL`Tlj5PI(dn?g^@vx@$@xL#GwB)MWt$) zYh}zxHwvT!mRHflww42VqV-eR=IpujX6x4rhAt$%EXcy=Av%9$7ZFkU#R7T$qEkj4;MK3)pk0 zC+Z@O9%M@&BibGf%}T=cde~IX$?gA$IiUSTU64)D7h|PAwIhx=AlGFvx`t-eZu&oC zGtiO?W^KJ;&sGLn>OB{mA=4(}??PkXBOlws7TgX!NIQM7ki}xxgUm5?W%%?eP~l<$ zXVTvJy2E{RSO+>ymwS8n@^R&!UFOAej9zj?l|FP7FWBe+R>zdxnRE4*bUUNLE%isB z)Ax;f!cQ|?ummf}uY;m6?UQ^%D5qNF_ZlTmH!qBu(c%Tl`K98#@0nCpTZ|Mj+OBVb zkXvY18U;RlaU(de(6XT5;wfPewf4@Ltk_9DHE6~^1R-=QWv4DRx z#J-8u8@R39+|Q>%0*>LzXjGds-EiMG=#O1%93BTzAR_=F|-2g!Szp~ z21OffC{AK;`h6_|jgwZEkrM##cplK5_Z-a-zv*%!)qGhUr3paAAllU7YezeD{CMrPjEvZ(&U23Ay6%jx1c0|aP63?B?gvQGJLL0e?b+2S z!t~;1z(h-hb{hvHzCw$QPXNAj>rlaX=8u2N|Z* z%f~M;1L_sVoO=L(vk%XesN!Y_BTESVN7!(WBUZD`y`_;9$uUxf z3y4=-c?xwM^d4#y$(A`BfzMCH42h?h4MJ@3Ze%SFcI04dwqQu|?Y4)2OHFE@wfP0? z32D3@v;ScCc3Inw9sF^|Kh{iSs7}GDN*{%~p2J$L& z6xqK?H}kU>py`!g%$l7J3`-zv;;bmnz8sw#dJOxNUYVHX1jaJs9j@0uaC#scskfXt zui4swXhlqgl0wvn0K+`h>P@yu4+TP`~i1KI4+wqW+fE?&l&9fPA zsxP$JQO7wETc{jo%7!|>Q>4~${4%Nbx|liL9F!Kz!3ZsJa{jtb@|Y3=)>g<7fC0=H zEv3@L98I3I4uF!d*Lk1x{G;(53{o&3AATTS^LI>zsaiLQy)=I`95?wP1=w|UqMt-s9IR>j@P8?{V7rlG&4 zCT@9u#fLyx>~FFQt>~x{5rfn))rMHDjip}DuM+>=7hXl`~h&GC_JK(T({M)n4z$2x4ef`3-b!7tbI5x+$pF$t6CC@a~t6-|NK=!Jbbpy9L zUU=VWzr+QyM{b1|)^~~g&du7~7D+SH2B0~}A;w@zMvU&f$hoD1(0UvI-jLaDaaq;; zMyF)Bc@eay8iQ}35u#ILhGQFP96}VWh)RhM%}VZl87FxDXm}?4FUHgr2wixu0dph0 z&d)e*DvOSjXM6l1_;KKPw#0GAHbbD_Lbt`4*smyRbMX+Ll74i%+Ddx2SJ7+d7e(lv z7oPCO*wm%LRrJ*~nsMgvc#a2oZ@^xVH*C;O5_ZDWdVC3X(ux#R zhWE17f0r1CyfD}ihfW;RerIz~5mMgN*y5B^o8gkXp@rLFMRZ9`?Eg#}g;X^dXqMX{r#@l{yVmsK9dTQJ@gcgy?7lQ>a54L;ds zF^!rm!c4rWhZgwdtFDu%?&bg)q(ivT%;G~^PR?8abr1?d1K?+%#?6XklF+0??K`>G+n;AWQWLWDYwc{#vbY{uUgFm?}d-Vqlj7p8Uv2xCw z@|k!oi*RxT+y$vON#5Ahp)`yKxLn_C<-Tdy%D|rqX;KaP2V&3?XK)ut5iopFv&Q0} z7cqonlGO7uRS;5)si6O<3-<-8dE9l_=^pK>U7V9l=}L3@qj7GW4o4^Jz_z1+GRtf4 zY5x<~{ibig>bCN`RTVvXeRX~G1bEyOvvEhk_gG;oT4Zm0)3R=pc6t&no?W{=!P)j! z0L0ghYrj0n?6NUq7cTM?dWI(&@^m$M{Oy&kjRN$?#IoQ@XNFQ}**R{h^6skn#ky#t?Lkq6 z8`73vX`b4UnqHFs+wX|Qziux7{g3bN5y7qzD4+#j@;P<6R#sm}?@Lw9OGt>xU(OGZ z0TOLRoUQ|@@Au;m%i2D}?$(GI;`r}mL-~J@hQWuY%ShA(vP1(nsRV|xbZE!6YyVxy z>wf}j%mt|Czl7+vfeMdd5Ven7&vpTb;(veh{TIkH@EQ7Fk0buC2bGR=isojC0HByo zTBoEeqq`r`9f4MWE4|&?p%Z0kYwu^@5zrcX-}0_r2jhTTL3SMS^ZVK1glHHjiDjQ0 zUJOZUuOIh?Hr#*_tL?%3DMT{qXgyH(^*a?fnYiCfGOKcY*pld*e|KeHC^ciNf4vK# za1KgdNsJe`E?5Vr#}b7X1q3wm(m>beW5;AE(~AL`Ot{{i4z%(|F|uzqB6_SedOQ%f zxZ~*e!UPYZeXHNT+49R@VA&2no*smS&h<8$1p`%MCwYS$Kb(y+8d#&;|9hOP6mlC5ev7DI#mQr}F1utU&^x{@P7rI%>$^E7^sT zqy};h`-SJSN6SI;MO>YFhAW86&vYsf6Umjiz}em4k^dQLpJ=09V35Ja)K(S}R2}jb zu};@>_6?L3Hy&ydjTN~cqP{!j>$?2Nir;))YU^(wis}~wM{621K9OWFp9=}3lLsAq zB?MO@;|QPM^~SO$kPkGh@rup5;-van+Lt1BqQ>>}m-6An`Z5>qiJQiCaff(PqJ}<= zv%s88?*t|HY0#cx9Wxj^}HW&s3e5oZx7~J=Nzg zPqGcY-WYt}|4q+2@67SUW%?tyb9UkjWW|#EfRUNhZR?Keyz7$_VTnqlPUZ{$8Vsf$ z_9VWu>Xq4u!#6RZ6sH=d>BGkbk`G%%M@*3J;wF=ykQr%SS?fGp%!ec@U*0fn(8M0% z6lKr)P~t+7S=k+IPD58^Ui`KoiuG)32TLs@W zn%yZ$x$@-vrDh-s1M7NY-Q1r;E!)v@?Ae=Vm$k0NxMv!KjrlzO;MC98mupM$uw>}& z^WyPJ7Gu*mfSFCb)8-CWH*{wdxPE$y_S)arlp|`h${h^stG((xWADCnhH+hw>dmz{ z!D`E(`(S>dfT0KYL|-=2Ih3Vf-ob z=N%P?z*e5Fg*q3^xD~2-``6HvLpk89fo4mgxaARr)e92$f!oS^x*hyufUqt~NsRh=CrIlC-)S z$PdrF;EziTGIxG?(U=?OUY*wHH1#iv{LdA19(VO_^9N1D(Pi*S#)Y{zk11!c_*Ag( zToQ-RPc<93B1^67Dqz)1{?yzO*gcPHP#%6hb=)Rs%qc0PpGc87w z$p#4>Lj^rO{mtE_m$*)UxV6M)Zq(9uL%|g3TA0lyeZYWMsSAd#!t=7WTEkRfCHam;jLd4?o&G<^E!|- zDV67(=a|ZaOZ#y&@I)9`x)vVnTum7l7rSsS4WQ-X3m~gXe}8>>Lm++d6-_jCTo!n! z>f&H*!QDM`mKau+pb|LeZTq~Idl$FMEVr)j`FoW?LL3fpLzj*(h=Ceuz1LVikvhsS z1{)spkv}$|++U5G$jkeHz3GaY_BnfV!8=GcjJtdJ)cq4jABsTZ!SPP*Ev0wDK-HQ@ z=erw1b8kYG?u26l2gGk!ml3lY_8v=T<}?hS$!MXcDzO{E`=s~-jSg0`O0Ch>E!!Vk zwZY(`AZWOaFS8Jo2SqauNxv4pcRi_tM8kiy0#UC-ZBLL~rBBd+9GS1JW5x|MDAWN}~~*K+|m{rRg3J$rH+`Egg%{fFeAhldMZgERMe zZ+&#iF&o*~DY=HK{5@rD?$-1u`{kxhb(6DqpbJf76;h(n@Z!AcEAY|pxi;VKDCxut zh^~2uB{tdZ^4tx+0${H|#RQQoaB$is$)eV_ z)NHs@ov;tr{PuT;WFy|CugzzE$~pyz?h5k+X+a&ws9QEt9)tT^lWj6*yiW1b!Kc^yk6s;WC+(JpY&5ya zzGc?Y9|v$y{9eQSlE<6<7Vbq#$ua8=eE5h6y6Oo|mOl^3e+>?)Df-ll(-?1pIJ+e8m8V%Rp9)_v4J{I#^Z0(0!>@2zV%TGT3IO36!dqzeE_@LG+lW z)`AYvu_^!F2)kaI*Id`nv}`;mY@m^&iPnE}>73L*s;R=qKm6UsMQ zrt*xzJ*o2BuRu?Mvq=cr6|%zz5JqY4UGi0BEAck$o%vFp(?ILIJVhY)(J!h|xP6QI z*IUOVpfF9@nvNs`Kfd*q>w*HOxESzc*JsQ>gr^OcP%pRW_|`3P20-!P=^DYl4IBYJ zeP{3t|E~D0-@4DSx1>0;+z$i76N*Kk9(tR{|5W$A|E5$J=r{K7!$-sUjVbS-^Z&{B z^9}xk2$=Za6%_*``M=aB8!pWKl0x5{eRpTO%irmII+s?oN4U&F{+-ZQfw)OUBBv6~ z&eZj{;_h_>Kq1xt##4RZOFsWVAu(7YzcMEpXN-O10*wHP>|mq~ZhA=kK&|soeq;$g z2!{y4gad>6TnvmnEp;#WbY~lzX|VP|xG(RZOJsC)l0Irz<~Ir9mj3HYv$J6(r!R9) zB&TiC *s=TLzSg8^y3SUGA(y5zY&L%QkzRr|lYeSCe*=tR}d56$wdd$`rA6}`RLq#YMUru(RUXb^sEZn z`_M1NS-23#%e=TM(vSr?cJRV3nZfVCrif!+D^Kqq7~?%kEu~J=T%X)E4f&W&{W%jA zY#JlZx9q?LsD);Ema-#}N0rA{Hfl+yYy;>Q*xaQw5N;wnInJ|CNuz)YWBg0*ykKv1 z8gks{-sn|TAgAMm4Ek*eZ$R2kjFg`Z$&2AMtdvP2D&-ZYY5c#256P3F?%MAH#M%iNc)Le{r(@;4=7G@g>TsPzv`rZ0vgPB|Q>+zro-6dCz-bYs(#sMAwaKhHJ5p z?QJ)32+$HIpG7PZQubup9Bkm9c71O2BLgF~ckS<#o)VsOG!n$+yS_23c3rM+X_q!a z?+{~qq*lB~O7*j5lhWke=z`h1{ zwLKR;(tpj5@N=zPyZz1E*AE}L79M-utk~bP_2R`92UJZ{D{7C_e<{$dzDK#z(^vBz zDCVl{H#g!pzk(Bw2C<1mXCvxKKckWcOkPs-Q2_x*6P}^0s;;P}KI9m=4mLB-)d>eg zH?B1CUc}Zc3;kl5@+sTxC*i)5*4Z_3EH$b{l>RWE-!cu5LJPc`%d(J#&H6hrU(+l# z>uVh8tNWJ9@HExt=Im^@D8;#YzWo`wqr?qZOPd0Ngr9I*3kP?QUJey}6ExdokPu*c zy`?oG`}804mpKir^y1#%8N`?jr zbOUdCb}VYS7eK9BRXVRAUIl9XKC<`6C85v|Eg7M5BgL1upK~+Xh;9fBcWTD_@m-IB zm_Xdg;Ql-f*-!uYc=_q*oLv#W=x)D7HgU-D;iH~-+^ZUYoV(LEB6b0&tVf)D89@_W zhrjV9IYq^F*$Zx7{n&NFwav;?pc;&kFu&pdWa8ah_eYE+Y(TS>5slL=>l1KWv#6CD zc9YF|0Z22*lctiRhwQK0qg8@-PbvMKP7MSV?55iK&{|<`_ZnX_C_b#${gFG_9zzWg zj$cNwf{bOXr;ae!i3H>s=5H4FmxzSbESr~PgA6sW?iR|KvbavDD5A)36NF`7mZ44C zx9qM*o^hxn-qASk_CtdmZ$L$(s56G!3%|Kov56=qe{gwt@7Rv~{IY#tA=EVreBzCU z|L`m=_7gz8^Nnroo{`qZKW-)L@!{f6%C0)GXNylnD*Mu_`8RZcva$%1^YcqBb2e(` zKXA=6Ikt@GpRO=onmZ_9XN{P`Ir(Vh(d$#xc4Y zO(^hcHyaqrPw>&m?d zeqH}j@vt4)vM4gTzXh1mr-L}V9Ti33`lGD03l>#~L$IU8&x_=CB~N6K^jW<6w{9J9 z{bWN^spO~q1`f5~1i+U~E}%ZzP7KdjkU7O*+P;g{QCks+A(snV?b7Fec^B3XFsPFb zS8^qZok^U^YvSop zlzT_zJU|01$tAif3W70`WT?c@`; zAHsh5!VK~Grt4XFT&177og6!k$nC+RsOt-4W%~DD?(=p1e*vQTKM}9~KMzHhJ-T#u z>tmO)^~n|UFgyFd2;jy0%Mw&mA+V{5!STmjK&eIg`nefDf8 zE5{Xg;M=D1NhY^oLH|@w8+Qg8e;4VSRH{>w3x2OGuM7Si4)B|_ z8DIfl6u7X_Km>l8X&*IM9+ z(=kp(Tch~X$df-SJ@<59iCVpV>U*mt*O2sF+3Qn=RdwxoaMBCtngiB z+Eroi00};WPZ#DQDw1Pf~MDmb7x#$^MG5HTiX?^Nwjv7ob^1VZG_^1Hggo zmTPX$O2h1x95Ze1THs-Lh_xF~pdgAo(ixpJlEg)e+WwJk-TyjbWNLA8YBHfl3DAF$ z{&Wu=(a0uNtlj%5^~v`GUrjknTi*6UmpeWxr+}B8+OYl#YQ|!lgpR$v7;hbpEnfKf=I1TxRIv|UGCt7=?9{H8^lxH=Se>V>9NgBgH z9ZD6FO2$2M#^%G{Sq~JcUMRn71AWmS!->z;+O%d`479tm^scE@XUR+V4ca5ub`h>$ zhlO4WRvi$*4W1oYY>nQwq7S^O)%hK~KRYoew+7ljhN;^{@h3QX@Xf8ruUey^Q|#{$ zOA^OWQ0&Paj{0u0HvftHk-KPqh6)xj%hBwX^WTOg1fGU7NHClH-iUbtpt&-4IBvC~ zl=K4qMWbW+yIdC_N`LUcZs)g-H(@)c)OyKy)`NTODaU?k?Q;4dX-%Wz+zIB?Alu4B$J02x!0xNL7l|5ITVn;*5x*luH6eKtMr_ zia3T6YLdAEL3)RT6cCUWLTE`42;2kf&VBD%^AEi9+y3ROz4qScoX>vR_eqCkxxj7$ zeN7Ev7CrHOqbeh`g)lwl?j2tKur>&#UKiD$D&IZL&z=ZwotoD<>`F#?d~{$jZ>w$u z;JmU<>~~WOOoM{3`I^4jq?#CxScz{E9|Vyb8sL|ok?8y`nfD`xV% z!>)}3WOQ1un)MmBBX(N2mW=tbDX{oOYSUcU0xz8Z$UcU@JQ5Z#^7EvHF@xc{(j8tG z6U@5x8aHsk&4MpB*1|)5M$V{;6@@!Mm5|{4guPPzU!fvMp}bK|L;~I&b0F+|I`nK< zn9K@q9~R=ZJwI~~O0nNKi6fKLdt;Wn_ocqfO~@lrx+3xK5F=p4=>TmX=zI+tcxsef z^g#(I^h{Y#mZ0f3zX2LYqe-1LQeACP3iz`iBy?>Vn`TOpf#A6o6X0HtYmwy?`xm&W?p(@zKJpmjcZ;*%YEul9`CayQmXBWgS4C@ zyVXir_W>mSIKkuu0b4nRCPh+3)tZ1PpS8_d`zS5#XiRTof@kJIRjx|BHhf+B>4SfM z^X>vvJOjw5IO`i(OxPYHy!TuKkWP8Bbi&;h5WiQg)3GWHNJe@Z9wdDz>@#uAoiKHG zfuj+zRfEpHfnLGxI1)1m1Ukh>OKCMy9hX&N@1VM7Na5DXE`Vc8e3cpDh2}xkY0#xM zZ%M|I>(DcMiH}X!TR0A!eic^?xu1zCPZqtLNQ^fGw~prL*i4=>_xGSca&B*oI`!cX zwv>A^!}lVu@?3oGFH-2P#d1y7K5`jDugbjP1xIEUt_0#C5JTKx!quu$8vF)C+;RoM z(cT6Mp~IeQBZ(w#&eRyRsz&)Y?OIqX`uxBQ%JPS#wrl_RDX+}^O84lkxNMur)2YRy zkHUDx?Be>R1Xu*caJGF4?ueLljIHI!GEH`XW{aU>w4*fQQ6|3WlyG2EDh`Ow<^^eK zR}gExj)a8Qc?^j21{pGUsN#b}wbzH;c(v0R3^D1Wco;jcCUZ z9#y=&_#-0}dlaj}et!nLA3N&g=roLTvD|XpDSA%7^ea)BKD^X8O$|z;PL3LD zYis)0Uj(#|#l7UBJ|~Hu;!38ZLQkn8K1l&|s;f;(3&d47jH?It#BOCVya4tL+SGLG zDBpHT#GQ$5`dHig$~0o zfX;zaWMlV=Ba7|d9MKO#&t>|UW=;)&<8%Z){DIM5&T@Itz%($R^Zj}~2E z$-Cp?Cm1LC27x2n6i>_pbo~aHC-sBJT!?$tGoB^4pL~@f_wJ~0Z47K)c}0F;=~5A4|HGC&ieu+gR`JxG&4v?Ge(1cS`QpaH1Ep`y+<-!bfmc{C)y3s7lk_{^ z-VsKS=pr3l^*F`~bQWczDxG4f*sp z7@KXi57$stXJl&-gfAzQl+;cAoST&fd?z#**?ops$=pvT7l(l+;U%p(a5X_zrb;bP zQ@Hpa)M=Q{jXRC^-T&v;bAoXAXea({)#R z6p1|+{7ARr(dT13#JywK-u8?;dBQ`&4scf=HFV~)2Nnd9aZplPUwpy$Gw=eulm7-| zHwY8}04LUZSdz08YQZ?JNC!BV5cF@(<+^#w{+n|}Uv?gkY(ppNsgnk#4z5-wLJCfY zGH#0xaueC=qwbRMv=aHvp!4!4&E9yEd~GQdkjb?ZaA(^T(vz)K?=o(g>35z_+NT8q#b8CpwIs&3j`)D=MOfZkNEOibn4bQ!WMUQc!=KF#xsF9MfF^we9Xoe7uOwqtvfMlR10OZhVCt}BC42JN2_CEvN=J(6v>WizH5 zKCR6w57))6h^lAurok0IT3@OTD1?wpt8iNbv8%In+ zDyOlx#uHqD4fOB1oS2fp9iNb3W&jaz>XSPC5OYr$PC`k*?UUPPkiHV|a|r;N-yD=qGn8 z(_9t}dh?~4dhG;5rK$O2=E(jwZ~N8jW>fRGU5Gc8kl@EqQlG74 zBmj5H0PaS=-55(*W{ibq-&9*C3^=$T-_X+)_9{1v0%>4oZpp=>+$C(0pj_JDj#3)C*n)x;~t=kqUeE_8_erkW_v+AUy2BQU+zrTV3!UAiSB-j z;E=a%)jO4BCJ2nrz*I0Lig4uz7XsopsB^oxwYWE!yJ1-ApX%x5%rhF&?XzT_T@O$FQrUpgXvU zF*)u3ekX>2MRg4nGXB^b^Mol z_$4le34l+QM?VQD4WHfgxSsmQ-(VaoWqod#P*-)wW@r#b8z%Pt9|S$H82|($u!jK% z-m5DtX@$bo^IDpB=K)H-wXOlxulx*VH^T2jh7Puv#vV3ie&W{FFb3qd!kwO3oSxuI zx4R!)r_4B3sq5p-j*Gmb>VvPeH-*F11=by8uaiL7&+#)rKja|fOS7CKRrecoES{wR zj&NOoe`c|_IG4nHm}~&n#Ed89q6LjFydM8JVS+hG zQ596_tu02213a!hkePaYQxnSn;A_&4w?hqG4Q@Jb2G!KM zeD0Z8*nHs}EUxQ+8oO;@%!+UU#}(D`#jDlGCI+~u-&G$tkOR)Rq1(8g__^K}#(;IVX#+)*;y&JvMcPymo%mUw+}w8-Kpu$551o&8 zne=7W`dPwZlqIv${R5SP9#+MnqIDk~KW5~b--eT{{`~=&rGZslIi<}26f$MUxfC!bz^$N>J9APx*5V?o>x{d|i~T!2=>G2K;mEuLprJdIQ>y z1}>^=VI9NX;BaNUCzfa$*S7OiV`=;jFP=tGnbK8ea882yJ65ne;mcD^31YzSXLaC| zkZ25!yjt+FjDTvnh~o#rul1IP69Dc=0J~&{0~7PCJCfLK-jNy0?@)kY$Nm#|mJ!D} zxSwg{Jm^Zf_8b1|QiJ{)2mAGcjEAK9t2v8QJ~ey9z(hKA8Q0F0>&_&(9o<`ZVGR53 zP@{JT57^E&6tOe90Zz9#x9Uv5H+I=e%eP<)-pT8!MEnwqhn! z68cVCry&L=o1X&4SAPnzbAkssP(r_0=DVpeeiNo>#^J#*=+4GH@keiQXqeG_EmI1n z?A;q;R%sNj*n4Ww5C^7S&eGY&pPlfFbWe=$i##ExyS-yE5X2cAi*Y8ZZpB~o$$fJx zc&bmTXccA~jALXPg0{c6!ZvKO$WRCeT- zw=`BHnLeBuSsdpA-1?Yyd~RN}$2E58278UTY4q%}nQUTd#aJ8Dj0S~G&79EEJhBQx ze(uO(*RB|3PlZE2VyII-Q9(iRG9XpC1eFr$9YjS`6jY>_0HH`Pp+gb{3rL9wNGCR=cL<$W z0-*&&dO{HbiIfCsfe;eD2-kf-e>`hF-}kQdKKJKZEOT=9o_%Ka%OT4j;~eKz*+%- zHF%kVF^e?zwNrBFvQabmd?W~Uiq2(ruul+VapZKx^TP+u$c&~P86kciaiOI-EIRN+ zr7ZqNjZQI{!IpVNAY@@!VdGFn&Ej18Qmetw!NK*3!6M`wwT1$RGfLEVrejG`y7ZWO zVuEHjjY8PRent)sl`OBSs;c=vOmT2Lj&~wVG=~o!+r8ot>Od{QHX8ryW+v_I?w!v! zuJIk%{j|1Uk7xJ8ckVypVgI`VLUz4jjpLg%9YEMWg}NDT7|?$#|gBS{lh zFz%L38=SAIIS-+CC#R3STg=S79L{?;Gq$Hu+N^teEKXfTa6);?Ce3M<0^7W^I?~=5 z8S3B;&yl0-hkP9iweQRdns0P5OC{kTCP^+YDvck|D~%Uh`x91pajS!^X*b_u~1+{idGmF7~3 zveR0>8DQg;nUe`R$tX^nrCY9I-f2g1CJ)J@XM`tEh_ZD)#TQ(en3p?&h3?cnoQsiSrB6!Wj*&mqRxP zY!ngZsievl0bd)1`KCt&dGJNgvIOq?y|asTx($;P%Cl6DlAx2z3c~0qdAtZDdRJL@@{}hI|YTZY?OVE@*)Wge`^vLuLAxyiRq@8a|9Jll+U~`B^)T%;`@bO>(Qa5Jemw$i)%Nmx(pWmJwIm28ZfFg zu8f81{=8?|EmyqZ0=R?Y%&)f3$zQO_P_xgm`u;=3%*$S%N*r*Wu(9*+S2e4QohzpL zPj*v1;T!M!_~RCy(npu;hoEnIwHAjea2ljk9))7LO#s@EwDv`TLVbaQ^uaF5^eE{` z$?jxwLr9Lr-e0ubxgB}>;6{Al4w)OI=xm{I-mJ^1gJJVTF-58KQKTze2!VP+xUE8@;lSg=XkeB;tRwNqB2bC(bNfTfSZeXWKFg}KscWvC8l{7+PrapeP4!GUJ<3=Xx;x+r>}qPD9`#}C*ki8Gmgk3I76P94RM zRVg;ms&m879QLJ}SZdIu$v30_h5>M63@`W3!cYj78act^zSzco`K8Rpmj?2Y&~vpb zT+eer4zIrS$nR7nY37l#l)9=+X0>EQmBYTwiETtGw8L2sNA0{OLsvI*+$Syg!0jyV zNCBJ$MNMGh%e{?n3Aa}3`r*{<{r4-ifWDWf*77nzzKa7`+aVBF-ngcsQ_F`7#*5^* zdR5+RgIkVJ8}x-be6MYn9CD=JGnP~SIqn}b62_Fn@2|=I%0_$QgZY!;xFFVa7BpsK zQs}3rPs?^A;HosK{zdc3Z7;nzO@wN&(nxHV?j=P1b_J`;=f?Ikje>|g<87rES*-Hl zu=YkSM}S>P(8ojaU$+{dXMy|2!{m&@gN??Vff|7B)^vW-?VGJyPMDb~U9wY8v-nke2@ z&Lnfnk5%9$sE+NM>v6VDylw{md=kP4E*Ga)l3rA)RH;=6DUK(i{)r}8E?!$IEA*8Q z5~*w~qU%H*w>>yGU;UsM{^DOE_~S`l9p`eB97D^Q@8cwQK~qUR2R5$WJOWF`M+)?y z7Qfci>x?f=x@B&2H=SR~F9(csVNliLZI~V=>4`x?W?oNsdY9n}aPCl?02`exth0D;iZ7hkbAvJ7rDX@2xS0(^{%663;*U-e>Y-ucHw zr?I!HW8Xg2lpjEsFEq$Kz2pV(elgo9Is}orQM^5*Bgi&qKQyByE`kxkQp0}OK;DeV zx-N)hKa)}1Uyvf82zGQ4&KF)z`{z#-!E3B@d%y zdVACRr9|a6A~!Ii@o6$B)ttP4^+;A29-jj<+;onONKjf{O!Oxp8SvNREvJs zB=I%e^~>}cYkGC@>&MV9xhthn(I3UHexiK!D%(s!1UBd)T$x#Nx@OR*Nc zn)215GtK-T1XJ5XjuGIsYSB|Ou1k)u5L(qU!-0z!6r!jJ)T=~Dbv23!RZ88A6=!R~ zI9K-+QE{TT^<@~(0Qw01Pq<+{79AxZxV!+smsuzQ`&R9IFTw)PjTmpASGXAc%B|UV z6;!CYSeAt8^lA1L)Z+s?w`qyf{D6)Ie_^6`G&*P^!Rg-=sm24R0ULJr-g=4ca3;xL z>n%%Azvq9zYBZ#nIDBhH>nNPAPZi?UI5rBem5;8_fS{{z>1aEcE!N1BQP;T|bth#| zk(X;p)sGaj5i6-q-aHljMt>{#*EahRAIo~L>BXr`fEqSJ0k1R|@4u4Dma+koxis#P zBao$w8|Sp#S(u+_(H~(BYEFFDbiN_iJ6efJ^j5EsipBAxlUAt-l%y?vJNxs5vZgCY z-+94c`ct^80Y-yl&JezESNO7V2XfVGYIw8*{mnmUdE7<-yObqZxZ~}|_bpVo>yVff z#a2(VyTza<1aZAPQ}^XtcI8V^jwS3rYN&CK^b!}SY)i(&+J83HqGr+TC2$ASw2L^S&- z;7h$LOPC{6|C5r(|30{0sr*fF;SEdVk}g~D_OU|WVzq-axjvCGscd~-?S9y0%xRR? z{NfX?_1RzKbH0HNJz9Q`jDG{#**Y#s*^dfw`ni@BzR$f&N85C4?&ng{L$vi;X(b=-0bde+zQa$~A zE$C0!nh1-x!fxL_?r6cUiEB1dq-%M0N>9-JWOlmXb%^fC1n_rK3Ma&G>(!``%~KSo z{LusJlO@L;og`_1bq^Y@%^?Uc+(U|Lf?U=Sg;6nDd93vrz9qqwjm;**q z@@3US!!UPmfiHif_*T{bvE8zO5*MKOZJ*xl!lke%B1r3dovG@Rk?GT%QuSmnTYXG$BMgI}&V^T=H%olev2vVDT%KrL z$_SnemD=bb`0VAYV`hXFCP~PIQ(^JLc^aFLGm>U*=y`ANuD!$v4UJ|Fja+F{1;Mkq z6kE1dy0qRpK$q#{f4I9Uipd|bzwy`ifB3kWCBBzCf4-srA4zxr!`XG5aDBrTV4XL@jd*iXgiT^D6@ZY!N>+@rdS{jWm-lc!NF}|Cz=kId2 zsxl2Bqyk28>RfPIM}Z|f1E*9uo_5Xao=LboIrguVy}vAdFgJAd>)P3>;IGEIZaG%& zKbn8HKOGH!gtxJ=aHH9#DPw8Ir{<(+Y2Wfl-Tw6yQs?XXklS6gN)`QMn##l(*9Ei| zyiGWB>>ZUz!|j7C9l1a8!^4Gsql!5_V00eQ*v-#<_Y48;aAsFxDsSu#%9{aBce~Sf zN;y{to@ldAsuN?cPJhCEOkKjTh@|(2?GC=_N&C<^tK-cFRxgT8YN=H45FLkn*3tn+ zoroVCHR#TMFIqp&n@PStk@@N? z>;&dfP36fWz838K;o7;$m;**v&bhO*Q}f~0V~*99xcT=`yEIkA(uu&1m6YIX)s|U7 zOUaKYLe?8minwe+4+v?*$dVdVOr}6mM;&TM4KxyZW9-{st3%5R4Fxb3S?W%-Q3KN;VI^bw zljs!nRqVF>D-o9!OcTF-=v3k*W3-&5wrk!?nzL8k>gVLpFBkidU4H~6>S#c|XPss= zB9iM6uYVyjcb}zvc}*g^!oN*8Whq%LpzX+U8KcSbv~|&){3ro_(n!|!_Tv$$u@<#v z^!6CHqs*PyMkQ2cvlJ=Qi`Ij9(G?7yVdGW<1SdXl9-SDmO2^TOUnC?5!D6bZtu3j> ziv_jydOS=&KC(AA{btPSg1S~#E^J%w(nF2`!gMT)nA zC06dA+$lY=w>2nPAuUIlD|M>gaW z^6C83fSYanm=PU6@$Jx|=q{<5vNE2Nm=SMD(I9xoc{20s9~_v216p1aDUcp>V8vYX zlynuTZ+wn^OaxONr0bajq1GLi&Cz%i)DBgMIw-%r;bXC-skS?7WCUfyCs7OjA*`wB z^Jcb{$+p-Cm(DvRMCwg2&%a_H(sX3RBDhti2bTs;()_xX{v}d?fGcx-bA5vP;o>f# zwN|!ov41GpeLx}F0iJvP?w?SG<8#{$KCnr~Wqy7dn#FB6+}(On^!%=$T;tb^yW0Vc z%SExZ5Er#~>aX$ceu*%cogVN$FS|>ad4){@?<0x??2BW&pRoa$;=50OKx~e;Vtc?Q z$74J3T@aIlBkak)oG)kOUM%w#@PBVm+i6T48=)T9zvt9REQQk%Wi1lrlh(Ijj&o<83N#N{C)99iTj-fqIqc3!B57p|$L@->%& zU2~W+uge~79!D9yKZjRm#Q(Qw#z}(L(PB{IJQBK9K>rQe={z7vOWcC~H;&8PS zoui-kZ1~Dc_Q6)zDE~i)Gk*&j@_l>#giRm@YVaI7n#UYGDKxP6Xabm#NP#!UoJI@` zJuiQ2S?oWgR1DPF5#GIPFAhHD_|Wt6hog`FVTAbR9{`_Y=--Fy|Hl6Q-_w-;GfKal z`H0P_cJixSqDs9xAYEnG9g4x8EVzH!@)5IRQwN^!UO&c_1nb46?l5;ug?3#9hf1v- zr~IF|QL#&6H;m`-yw!ZvF=BOn*j-|G&S%narug$_ZsI~&%LAc3Wl}-^nYFcIShME< z?N=A>{yUI~-xGVL*uRs2KLpuv=D!IV{BMSn)^gFyKdOeiaW%@r3st1ouW5_uaaX(M z>D(GQtUTHa>Y z!;kD1TCPNi-SlgPVroHty4lR25*YH8V~T(z|*)=gJv#)Ugq9LX=1W#MEdYA10f zTP&aM*}NSWZ<$!}?u8;!e5z4q7y5x5Ye`eOla)#DMecOiltpom!cy)0=0E9#RY?;J z#_m`6H)*#29>&TI&Xv>@gNjrbJ+4sFK?S=E6?jQ)7xzvviv~Su3{vok-eNyega`7N z63mtv=zzB={&6X=V(1N^yR#qL{Y=3E%!WW!>v(EOsbbx)ZEoFz)V7%9!Xhhp5$Oz3)h=N3nI^7Q6mcFd z%(LM~P3YW$eW%cg8hKH$aANN0BXl-na0zD`9uXE;z2t70vqh1FHyvRs!iWW1y5QMS zQshAWZyg0|P+ZANC3SM7Ke9e#<#qTJDHK98rnc5MeGg*}{@h=?awF(nsKb1dbr8jL z6%Rev`(T48Smd()F?ed$CK;sY6B2AN7BMGS^|zhx7HM=+)7gq#88!s{y%ky|F4q&d zkyg}WXaY>vmx`VtxSr0^1Ur4dbx8hncPD4Mkrl3O1btc5aa)5?#8bW))-J;!Xesz6 zvB7P$#da3^4KN z?N1Zf=I`!R^UP=X^9z{_zaaU^GoSNEV!wrFFtcL)K|y}u0m$%S%t~>3>TKA)=Wp!_ zKTSeLpTWt`-VXcVKAfgX2v91QwZ6}&4{V-aOTDYQkvcDjEH*CH0^3!hODh*d6?cl8 zUbVQq>9T)l#U~MQP>K|vH@?wSO+pU(7=0YP9wo3NPgpZhQI%s>*L|28L%^2~`DHb_ z!hA{xpQIJ{CX9LH)FQcV&|r}e`WH_~`M z2XH<9!*yKSX?sh2dS?2;7H>;Js={C4cHgArd*@}E%H{*2L6*qfekaO%BY2I+U ztbgU}B(Gx=tiuNSJ7@763|arvM)fu-n5*`==5Ua z*LJ=-4{dz$1|B(qlLdWaw-jbJbJvE^%pRH8*tJXd5?ArmmW{7ou{%aQJ14SKfp;=f z;hw;TjdZlmubs|VA&%cs0vx4}?rL}Ojky`9#G zK^x82xk9G0tB0V}^i z-*zaxq>PazTowE<)l)6_y4^iIS}m2(6uVM0s>{jP3bt-HxXp&xBO8^@H7UF};;6)^ zBl4rh!%yYT*E(Q<1!$`X_=x!Pjl(@DQs083x5t`2ZXV(o6TdMe>Su8K<;a*v0URe6Qx| zl2$_!9we%odi+gr!XIAiat}L>pkKvqfJV5%^!~zDefz(or);}}3V=7Wxn-A!=d!lB zTS|JbIa;&ffVUArXr8uc=27pr#P~e8bp*8rye`lDYQk$)w=YsP=QP5$I`PT-?q5_4 z-xtBDEC|=N&_s>J&GR(#S+){;$ceyz$n@IzgHTqtc^&Hr-dU#*2M3$37(j<#0Wj^P zo~|C)_a(M2HDquBHER^G^X$D1{l^1} zbNwGi+9Jhoz$(tk9WT=YpI+cjoiAG@@;Gh+9N8+?T!*l`sS#L!pN*ZN)O>69eKQ04 z)U{jYI7DA)8O}Shgn9Mld{IGOxAc#ol3s&TnCSK4dJwb21_I-}My4*&47X6v`HLSN zaU6h1-4CET_ZWb#NXz7?0J}8BPuktf=)FjXG?ZYCx`KSD)%=i0!`DcI(zf9I=F6wq z5BFKkb6NbxIyYdYY*SY5H0Z} znHnc;{1TuVv<%PwTPU`@*Y|YOhWP;4H4rowJQ*YZ`Hzv57MC%(*Je4Z!f1l4NP`@~ z6~B`k%g+PHW6tUr51z)fk0PPku%Tj`L~NRLlL@w^i|AJs@r5A>zNPbE@C4=_cJwu4 z5F6M)=H32cqbO+SUPBQ<|5Wf*qXztpE!7TLS1d zTphfKKIK2sD2w5CEm>z5!-e&D+=jJD+fmp`ZumxSA!U}T2xro-kybh^ms}$yz8t3K zy!MB?X3fH;6DuOL5_onFx3B8fAh@)N&DU#)b$>#c%O}|ZxR*hz2xsn=(K8FaZ|MAP zxx!Kr=H2B*G}AP$YC7-SEXW;u60@1OvCS$+8~q5BrwK8obY9R6*GgZE!D`;@xRi<( zz&b1I@@<=ba-~&jbQhURqr$WJ$t>QsLJCFHs>vSRnca_8K4 z#8=S!0_wK!(7~lVJ6Ts@uDYZ@`Qhj9haDdhnvl%_Vja<(@|E1;=QrQgF9ZC9m}!v$ zuJ9aN4y<=@@JQq3>wNZa#oG|wWBt?9ElR(E0m(hun{gZF8{R%z%va`b#4wMDIw*w! zww9zOl`%Vpr(56jiiK!}@IGZ&-25mzXhHalIDP$rT3}Dyf>9jWWwqX|=-5G1y?iFA zl?$m38IQk>JQguPZ1#9n5rinI|7reEX=P+Ho4>pZ{-%aU zax}F@L$6f2R}FpcAe$3=t;3E3gl%-ED`8%fdwNN*9KH{3S)ZReRJqpUwL?7W=+qFk zzsO~_f-xrk`LSNyZ?Zgi>E}%d^(~xBue+w0iZ+crPklzoG;cBxaL02CybwTtWX*4_ z$G71}b)%Wv!SLi63#lf))p%F9*!IC+o1#*4)N2dzuH!%Z>$zru^Vel^0zrq|yTVlL zhVakOG4*p|)?8IOuY#qYMRHi)w*S3eoHewu>pKKeN<=WfBRNugUUFwi_oi_>S`Tyn zT-~^vxnN$Z4(Rph;wEi@S;mGgZ31kgGB}&lBS)I)4Q+wp#_D|WHbb!E{@=x)!{3!? zAHQ7AbfLB(*6j3oYd_Epu}PDgAl@fK5ux;o(c<`( z_gaP+ws~aOoA|h5)L)M$?yDZT z=M+WzaIKjD=6nBc!n749e(1*{-h4YnBjm|@=Ik*bMfRU02(>>_;3nkq3M+CxRoGj4 z4|XX3iw|?ve9-i^KX^K+-&jX}XY~sgi2`T!H`SnA>_C?|s-_Pr3x*q99O-5%CuCGHkpRMIN^nGN+Y${u(;@-&PeQ(kCr`~S5(Q(nJz zDf<3+{C#%i@3(q;gy4!Y{gQ0Ign9}wsMTx zTgx6YZ_D?sau@74<}#8B#Xo+&-bzO9782N;rs~DLJm&In-(ESWynbEdAG`{6{C@$a z`ac67{%7dS)7~86sYU{qeO>* zndJ7SgpD~r4h)R0s2A}P38Qj)_o|aoFjb>@fTtX8*+Cf_e?ypm=nzxxbm=~3^?7zT z(XDgUiNHHtmiY_SDeX`A*PdncXdYIKG78AkfwC$G|HAMQavRXkk3JZ*`Fk8y;=8LcRM|OcH!{ z)Q*!aw9vn829zhKu@6iUo5E!w&Ii4IEW8RTy=lyDKM69rWtgh*JZ3IZIe2X0alU!8 zqzZD-%SK+ta2}$)bwfhC$4t&0#SAW-L+3?wY`3Z$!8NrA7!AlnnP3G z$PER-WexA_GCDoUdy+)fcnBS?F0oWy=&k1dG+&hir+Y#?hKX_qii@tr;V6$5zC(b% z;XUf>4M|($A<5X=Y)0P{VVyCPMTa`fvmjZJAuoR2LtmBOB2nxXF^Md`{`0Z}o%@EW z&1EV(zvSlj>l7^{;^B^oJI$EE;n5UaVmW?Mybqoq=H)y;Mp+kO^C2g*2Om$3Z z;;-q_h_elM2f#N68r?lW!-t0{^sDL@0--Oz59O2w_Nl8hFY~PO?^7Lhbxcr$sA=XP zmxuPbQl%k9x6LD{D0*`q(i`csz9s*wX&^ywmOI`8IUmsJI-i#N$z&+TSNKPNZi0iA z#;jtAvG)<@?s=U#9W#}r_!?0}u&r$*r#bo;rc$y~GOOL7dZaat9Eu6 zI3byuBPZM+DI9$9hjHhYnKX7PbyVJDMs^{OOipNK78m4Avb;Nm-HLot!$8eQ@VrV# z=p_=uMra%S_ad+KV&g!-ynOz^d0E?ef#W%)b4i$GWA)m0$TLM2zM^oz($ahlh7f_^ zf<{w0p*n}??aS>-v*z0oO(^wQcjvH$t7a2}4up}bORD!#p049dDkZ}?ztY;UHxj&Z z&w$xAC|km|kY)Iak{bm0an{s-t7D$Z4=*j4tMnhJxq6h;kORx7pPEa(3@n?>Viu~z zC>g;!lsdJ3Em}VtlIlqd1s90cp1WHPzjYB?hSq^OW-UZC>-gfwtJN17ky^cO_+f>z z68DT79wH5GI=`wg3e~K#YS9BnJRakR9pefED!aR{1Pbqq6iBhZ|0;v|!&uny>HsL8 zGxT?xq;;!xi0uTXi>=-kOL=!k+R5+bzNB`2>=+r4nnd>iwJxkj*{yjv-N`tB)qXpm zCJP)z41e-j3w^L&b4Q9XBqq@e|H`iy99{^WW_JN+}S8q~1lZ*?JjAw`_gZsUAjXlvCP)3u7ZE6iHCayw;%wFaPzsGgGSyxg?ohAw_@fQ6`vN*#Gc~o+( zYXnP9kjcw!SqVwBLhMTf`GXc9j<$iqPh?p}ar18hTnMdv%*;_^HUBSsQRYiPz%8#S z??WI)clzU=C#8wbDOtBZ+#Pw-o6-RU^r<-PU^-88#bwk^z-g5WA0 z>)A$vKBF#)NSm~mocDzmPM@0`iL^?-9}l%uO$;8)V+j{CaBunsYb)r#TkU?!Tk_5}3!$W%+(#e$Jv#zAvmTL3CS38ZcQZ3-r;!(#ONVQV zX30o%gwV)xBKFA;YNTRFKfxSZLFztG0kVBjit0jLf^?q6OU$2V6beR3da=?~K`Tn@ zu6Fe-@`}ZM41RW~_Tsf6)qy@P|58b6UJw0_32(4yJrYZLwG-;zFcR@w*Z9w&P%zio zn@P9zW86HC{WO2Thx%|;Q43bhGTFLP;G>pM(Nd&D$Rz3y6v%*RPm<)UqZYO1HK-X| z?Y-4us*HtDmG!$m;Flzz_GL89 zM83V7>~d9&+tByQBQ9?}-S5zezmw*-tB(jVY>b@DBc2rhn*9a+lq^9I6iK?>l=G|- z9>tQ|gc1h&G)E%!Z?QF@Lchdp)|_@X#cmGm4&Rx~_;pUe_KpTAGT&HMGU80Gi+qeIH>~n~Sdnd&g4r;n7+qq=)A(m>2m({<3&Dy(*q%ZtG>7 z9vhjT`Kiz++cn_M%ElQ%bs zss|iheJN&~YY{%rD(;N!?PwS;frzW1r|BwzK5naznI~y^s(|;9zlD?DG0aR-t#OSv z;RWj+LcwUzSJn{LSSYoe*B>u%ozVx#1PIFVmK3Lr&{fS;m+LQ;A@hB%H#JwFo)ZC@ za%+C;SZD1(8!=I-3MOepRyH=Hx<~;xwP_G?ijsyu0hIXtQSA(( z;ln_q9MH~~S?4+D5e0kRP7T%3D`q%p`9 zexr+LA$_d&lEA~%C$L8 zD)g4>e;7g2YYDuGsF3N{k3XyLPXd7w8j>m18m*^!(Yk8~OsSIt*M@3qJf%_jM@(&B zQcN2=>9b29R^vr;&cD`ALx%E)wIXVNPqHODAbO@T2=A8}xl~=SwS$FpA z7M}OxSAP>#Z594aXQ-{X#2crQXzVm*D*}-kl9Mj*N=M0WtQ3>Y+=ZHz?gxrgwp;+= zvzI_{7>4ZxCTBA7qRyTJ>eD~4iVr^d{A=^^$esDMdbx$0(YP?yrva2L0e)cq@<`XU z61B3F@631=JDZqXn_s31YHY<9&K9i0J#CoP3kAe2K1_Sez8K-AsjDCF`q#tfzepVd z8J$$_eHB&-;Cf)){H;SU{kK)-0ZLLG#qidbC7`@ykB)8coNFa*Sl92&TJTwm>(21u z^;EC~L2f~^Kee)H9)^&0e_$Gvf*6tVXq;xgnRV8wK+NZOv)h%6eoZDC~@UU32;ZcF)ke%RlOC2)SaBvK?`~v6S;tadQ8 zi}N2uxN$-si!`LyOBnUel&%0#4=*Mq&cU3Z_+|4FM6gMt7xOv?EqMc*X-%2*Z`w?>uMZSFuSXQy{W@??=k<4y4 zrn$W``{-fqv%t(yKT&ErjDlpJ{P=vIr77|26|eD6EZ z$_^J?p3GuH;CiKb+0dfzB>RPp7eh7DIcCy%l4QT(Iuy>(6+W6erKYsL9+Pi62hEuMts$ih8Qp)Oq%B=5-092Jv=D?_w3VoVYR==9MuPk zM84x52Z=?jtVU{c%(vc_yKu>QC7s}!$8iPgtG^*BBBTU<6;gA~{=*`%Oz+ji_( zfduj_$Ojo*mJ6B01l-hyS1{1V+bFJ(}blecoNQ6SZJ5j)#6EzQ# zy8wPLKTmiyY^G@z2rUei`q2*K^YTa_+_VBh>pA#E*fK5y5hP*JJG zcE1aq*X9&=LLQJkZf9>ZSC&sAFG6LzyxVTQ$<3pbf~!EoBOmc6^?2UuR_nh#CF5QP zJtsAM?32bV#60_T0inC=`r#5~t)H(r0x%rxZWj*wD`5d)9}4!%c7gyL_9prJcZ5P^ z4^5YfE|W_)4OPiL;xE~#_1`N#x-&pHyZm7>1wM9+`jumv4L)8uM|Y0mVE2ky@KQto zx0c8pD7F6%bB;XJi!(9S`O}cWZjvmx3&7N&n)cePRnGqR;r}yCxl82#S{tBsPb>)z z^~oGXd1cZ!bUg_B?7_RQNk1-`>)n_EEVtl-@9(&-E}__IWj%edRe}_5nmJ46DF3nx z^M8H=S;>)nG)1t{i&J>)p#J436~pn}co*I;uc;n?_M04zy2pyq^?DF-IXBpJ+NfSH-p>A1o_JS-gllV)~eG!uTPx$XN6n_#91sF8LEfeevZt*re(&Gu`x?%EZvFMW8SzQnOJIW+` zpb$s-@jc!-V`9S%Aop|5U>?a0G#gyJ?)$zjAHjJlx$=FmLvm-p-FqLW2hXyXSt|X# z|J-{yea-7u>8{`HD1D~{#rHk_N4i6~+SGe?X}qGC-&7TLg?bZV!Xt$7otIFe^Pa%? z{&0>A`pCuZ%vCvv=Ha+`eYd0P6@bCAJ>H=g=QkEJHEDTbiVIvV{=&qC-J|H{<02C5 zsB~T1U(0U4z1ly!G!nb=(}=?_(bf**@#P44Ir^=1vhRygrYnYAi! zC-;~&U(F)r~hcBM$S#h(E8YJ05(_)cE>rOo`=3 z3;WFn+K&?t$v@8wZx$=F-ptLa*UnmBx2_19{P7rjww`Fs+RNkZVnLFVc!vZ5=s31NG(C3$}W(d)|nC$4W)-28& zy7hX>KY+;xn({JMb5oHbk&IHdX43At z>c*E;!Tmk73j%0S=t+C80pn`HS?-w0oiDINAe_4C+No2|n6a@P{>7QrUX|~m?`BR2 zNv5l1wZ!*kHz2mRO{n|2 zCCX>#w$-Mo+UA_Dtq99yYJITTl} zJJyt;@Dgqw8eQ4y2i#c7SfCIE$^hk-ComW4xkv3CW5F#y2e@BWem=;Z{#w%dX7;mE z8;i`=qA>%xAj=jhCgohO1yp2xKQ-5|xN6*dsL`@JQ^#mQwMgchAab#4aex1)&P|~x zn9Qd0u#}l@e7ELn^ZQ&CWojMspmR?fc2X*W=99A5vRZz$$XozBnXbpMUk}Qr!Uok) zL%L;+gbj!kf${JtDz97pZaq??%@(ZDZs9zjG6??Fd(9f2Zsvy2%^TQa16Uie>nV4o z+SDKUWE%|?(XY-H>$SY?jtjbzjO+-|*KYDJLWO~TUe#xAH=K|6`3Q16)MFyp)x)|^ z(i_)ZcYV)Ok5)Kjm7VJwZUG}}WLO%Dd{_;<4*gC`SESB{9P~2978EDtgD-d*nD{uC zh6l>v$4mk65&&}==cbuZd2GH8RTGH8Ce04%Y5FK5HHoUJ*&ibPn?tqQgAaHu)~G)i zyq{1YfG(VTxlP1oe@uw_kPq*NmFY&(y=oijqY4kOc1z@P-f!T_cevEeO_ledK|PE+ z-+7M^?K-%OauJ$qow!&&0}CFb4-+5B6939=t&$?p;C4RooLkcLiQuK}aN37sz; z{ddimgl?G+7yi5Xk{(^;XhuxUx?`E=p#Rj3NY+DR^fagmVL`uMP=O8k~jTg7oPdic$ z&!3tD;0~|wB-~Q&^(`%8not*LKAw{|8ObXztPO@uu;o3tmb);CUS0m`nr@(Ho=R#X z#IsxG_+IRQNIg#olzgyI819F zVPK7mi>FsVCD37=S^WWn=zNbVZR4dG+cmIB@Xq8J_#(4nEd_kO(-d*bHB*>}y zUPcw1zM6^_2R3B~jG>DP*A<;qEhyaz-^TZ3w<)R)&vpM>G_v~ceaUyCo9u-|p z$EFcNPaA~0d0@1oP92H0ADSAv%Fev%30I`J4{3Oq-B$;NMtBCC%d}QV+w-5RuRdMS z4mr5<>`cR|u&>+;Den~$N4+X=`q*s-EE2u#n02Op?KqD6i4*LUBkD#W?ru%6L~=_c zgay?znd^Ikshm0_We~d0932;w3lv{yl1hqC4&R=Bfigxe&ECf!;V!~_iw4Vx*2l|R zAIJG=lS z5cY#_-4}3$ob>=LP@P`iTQ*Lp$pD;WYB!(>3d=%HE?lUIB? zIKjz8l!S3gW7+p*FV!OLH+0gp)4{yOHFbcznS&jvM6Vr$z_P?sWg@;MI!XMFZ{kM` zZAZ>8nt0V|SQ3A{KzJFRDiMmsK^njFz0F`2J3l*@m9`IQ-Y0b{sy?VOt}mkxlWz{9 zGJ>Gj5NSL^EpP2Ma-kCR=P?8u^9E2c+LsDsCVjgJ=>%LJ)gNn90A#FugWR)PhHma{ zjs3#6hl0%itm2#FxxSRgGq?o&+i!bSQl!}s?bZ>-xz5#{qgL&6Sx)XSf0?}1YIghK z(BHb&Qh-RjvO?6|cT%~k zF>cw6ti<-a$=+#^muvGE_N!B0K}xS_l~97=EOTHAeP&_g&4fie|Kw80X zYV?qK`SA~+)pd&bho-Cqo=Op>bDrCRjWbVl`JEm>#`!l3x839c&m{?yn8WGCM^t(I z?R3c%H+*Ki@RWF3PUbybOYFp6gKv)P7*uIW)mXW-n?L9eL?}$7+G z7Lj3ywgtTHLWqjS(28^N2;mz{UXXYlgmLQWI>fmOcho!ECDxiDjRxh$kAl8>cF1vu z_xsM|uTfvH9LmgKQ8#Ei&J%jzLtMyQl;3>|AR_9<3J3u2ii)7FCcUxO$ZDy)v(+zb zySHtXPXc)YwG8;@-}zy?)TWT(!!z8B1lJwVP|H~WVfZ9@ z4c(ySR}{>GvKqg@yz8Hx);VRje0*`?X|hm$fo|jDXDmu=V&8lx-D#`phn7CE1h9z+4axpFV9PK=0nuMt2e(9TS@@c@eGN-aEzh|2%RsUd0p3+wApM z8ZE=a^j71(GSH5vHuQK-`OO>7w+KiVM*?b#U-rmR!1@9e@js9p8( z`8FQk^#=ZT`26%W=p3K%tvlwkRwtT>gRmO#LpM3#Eg6oKP}=*85- zDk8VL(=$pjZ=q>#9Yu?u{|sihG}i|}(kWr*#D*o+guqP4+RBcaS>1~2OKyJY$bcJo zSIdU%7@AoGEzt#bU>sK0WL~YgS&pDXYT?wc@{g<6(b27scDmqul+{Qp(5B-=F3)&_ z3|kU%E3FaSP_LOg`3$=!kxf-Q!o_VF*0ZF|d|W<1MTxx&QFHpU+r&mCvA--JEzdwlAN2&09E`dLh~r@oywX4C-xfGYCbY0Kc9x4YCdr>^FPuAdV~M~ literal 0 HcmV?d00001 diff --git a/profiler/advisor/img/overall.png b/profiler/advisor/img/overall.png index 1883d4c97388b1cfb774d05fc9e0d368d0c66901..c95bcd2e49575217e6fa92cd8f88e8bf2c275f32 100644 GIT binary patch literal 8820 zcmd6MXIPWj*7gI44HaipiZq>Z#6k_dMsNfb1QnIu3?VQCL_>#=Sb%YWfYM768=y1v z2!s|8hEM{Dzz7KeB1VWvAk;u0fiF1kxz6|N{qeo$yyyFo?7gpNJl|No}M?-e%^i0ii2Ksi-;T0GbRv$|ph099!+ zLSJd|cu(j>R|EhYXy5)xbl!RG2LM0px%9KujVQ0#G0M|Xo7-AGB&R#-Q^CY%*4pn> zau)SMq5D^WLxhh5>|<%>I1~RzHoCPxT`z`cbs|=)9V9N^`04w+Q+@_O#7k0#{Bg<0 zeh$Ft{5eG#|Z>c%G(OHN-bWZ&{1pCMoD*qDD?l* zci#9Eqko`L7mPuSz7tJ$}bg-d}CCVLeho>KQ5@%BjGk1S_HAk3kXCxNS86 zeA^Ky{M1cH*1g$RMS*?eM)ND^s3ahtognbD0~{fS;tI*!aa(~S|Ydm@gA`l1}i0X70)@9 z)%@*??jvgjRp$d0%{b0V9vTn`Qg_B#{@W8g{c?+*A^4=`y`;$?lFKY=rS&G1dyT*P z{_hRu{s!WoieoR|NXNxDw+c`nKio-991LsR#ob2``{z-(pH3uvk~W?$Cwu9X$vM@xu8ESK6C8b(pvA~nl|cCX2MV2 zo0IeWzJhk}mXMQ#-x>t}ePAlpTQ;W(+P6d>Dn_jNTTMu2cJ;Q#fW}NWO1ROYmTz4v zNQ67}Y<|sYF&8d+Zyniwik7wzOtdGpM{Wu5Ni5-%*hgDL{Q21m)Yvb3-L@rw?1j&f zEuz_p7I3_|a99?sY}tfDMQw>z%Xlpt2eu{93FZR!mXJ2FC7O^;0t+`+lUQ4Gkx9jc z-;C$u>bSX{)1th&$N{Tu3QNE>o6DWxZ<}0MsgBxWvJRxSh*m1LKLwP2?%vuI@pgS1 z3>E$B%}?l|Yi}*6eX|-di2;R&m;vePW)jbE!W}+5aS*ALWb6S}KC1(nd#!FU4iao^TK6eK? zX7LcZ*@-OKy4Oz~_$vbsaIs=1x^?VdI8|Cx3K9`@H!_kKWF8y7maftsGpHa|=dApCg#S7baG@Cd9E)HAtnPwseK{%tPQ!J9PMaycJ{FUDKD< zXCGvZcKai{s{Pp>t4g|EL%|&sUZXrLqY&1@V6oY5%YAr551A)S<<^+!nMh%9Cr;22CB<6zJLHvOJoH|f#K1j|-FFC8~ z-B;E@&9yL@9xX_1kAQC}LsLc1YFsDptjoc^nXYb#@hne~cKNdw3ZE)sNq=B`?1<0j zTv=N#FCc8d)Y{pf5UHyrR#95tvG@>XI;N-Npgwk!2|C{UU{vz%xTwjIeB?n)=zjM5 z0&{1FDcjlEvvC^VdZcS-LL%JX1w>JlvSh(DQgF}a{$WyqT=OpFm*HV^EA;n zJA$25pKUhRq&wezJW;)E=h+y%h=1ZDd)-V7fQ;8$lc?pIE!>Dt1%8 zs{;;0fpY#d)yR0bN~87|Fk6>J#hoQHUzRpcoPTJwSzK?rt_Y#7NRB2t&iBy~6A^o} z-fhO^7Ty1=WpogcGG!i|oJ=}pzw=!tQ#I+LzHLmI?Ukq^l&W$@GhPSvy%#&kG9)%! z6|umW^xO1mh)(42&PT;*w+GilIynkSO9AC3!w2TOG`z@s-e5dxVvNxYc^P-gKTyxR zFo(Sw4#_l1wa58YsKb?@)(DG!q`zJTBilUr?%dBy9ULv_HFMV8Ugw9LehZZAj`=`M zH!@~?D`DZd(SWj|pWXY<@TM}#n0=yVHnv_!si2YfuDToZMuU`adP_^^=e#yxZo)=M zq6#ysYMK)H^}={sUYu{_yL-AiFRD1v>mBajrzgxJ-|we}>#iwv8_3PU4u9;}*ga+5 zSlG+53z0A<)eST!&Q8BDcI}EBR>J?NYQ%1kP<=j0q0dw2SXBw7UqC=jD@%{qVJp>=&X6tM16*N&u5m)W+4C%Vt@?0sRL{2hf3_rx~V#_LN$s% z?D&@dXl#6MA^T`v_{d+T%gZT)VEp87L1_24zbG=bFh9sk{m!xsTF}>9{)Cj5dQoXX zFz!oq(um(F${i4$SW1%{c&f)s$IxV*7hNNzUlbug|lp z1R54Im>`g^G0aeXz$DqEpEsjdp4k#u9Eoz^6f|UW>4VX2gyu>Bo|YH236%1VXI14? z6V!&MVEUYDhHEZwAm;4{jyK#?#;pJDU4K|b^i;miNKP{htyDc*ar)P=``5pJ;C~ON zb?g(e{h7+qoi2g8E_B?Z2_NpMu!j+t6I{F1j7KAXAq)Ad9K}oLO`n=<&DSqK<)k-n z^%8YZn+@dEF;i@ zs2h_?%kaBlx*2A_Adsp|fKupTeYX_w{+D#kXhFHR3iQ|}IepZ&xDNZ&$t8nGd*O3< zWO=VJ4}a)#P@4zU!NM8sz8FZMcYid_FD#LH8s%;I>IJf7DeJ)>l@>~wzHl`-!{>5a zMHraS#)l6PDBXo%RQYEUKc1$ZRxf`4_zi{F0~@^xXj}LK$Pn2nH{OS!(7796Jy?W? zDNI*=9f?fLUNmZPvH#n|@LP!p6)67Fv{>E87bx^=HiyY2?7fUUTH`{Y*cW_Bw~NW3 z)q=)4uE>9i=7osAVjL6EPvTlgjix)^A{R?!7??Q9Ydl1DoU`l9-c~cV%oo19He{h7 zWzR5bZ_9w_)v#w%nh$kbS=6n{obtw&mcgZ?mKoJi9uvEMK--3hy)eUZLZ#5Z<4ZFR znFO0M``U8k5X~gW%H;0s>_VW!366>yE7cAG`|4Ir^9{p!6`pSoepxEz<3!1-s_*S? z-bx!-aW*uAl?yQ0+xBH_*iF)O8`8Mzddf;Pyh@Ijt)Rh1O+jk(wWBkcuRZ-4I=Z8; z=pZ+2LTA(&hai?6KA14eSkz0IeOPw5JoQtq&*5K0$bB>c|K|6hPT!ol{@r*g#giTC zfzo~$3#TfjUTO!|t?i29%AJL@H#X+FE|{4&J>13I%W!i0wTQ>=C))T^xs6SkcV5OS zvSJI&x(QwWw}{#L=SQkU(HMx=zB%x!-p_ScJL_RCcG8zIX!^>XFF1QJ}rA zuZJF~t@C~QC=eWRxM8AjcJ0Jpmsd0PjM~JgwTt9&o(@w&gLjrqxT(!zoU^{+I9_II zu=&-n#C(*eCmHifaL5j0GIb^ALXOxMF#5c&aMccWTa)}Ew=^9a7?L7)-*(oc-Q(Lr z;E6UIMgOqrk;#LtB@s6j%|Kb$>4vGbIY+{)T&&(uT-i?qLP(!sW@GI$@SPUzdW`&? z#ihWuW@}3l872x&y}fUmq(2Uox{yg`O>rZSt{?rSkKgIll^FZS zK0$awP|T(&K^9#mo`4~{{{7esZMVPrgso-TS2=QZB^}Uv7B8dYKJIFLFy}p4cbLk0(a&JUrVHeL9RB%k#kO5H<=$4`6)#}TN?uXFq7xPWQ zm2|T#6epc~dBES@+)(2%d&#zA!bGE;NL1L*K1re9no>?(UfZW||SPOyg~U zy4kOw+UQQ)D@J-OYSq=HCmAtRBlFj%QQPj@Z;8)lm&){I0gpGPV2=Orxe%4yDTawR zjXZ<3s!cSwP6#FzOQH#m?qtm6wgu8)8^}FVj+&Ji$fw)fi}M{N2JGwX(}38n8WBlt z9wm3{XQ+8mb44J-SkbiX@g>}~O>uw@?=y-`Y}=??_Oc`;n{|pRGJ2J}a%$MJ!lwz- z*FvnAnx=Zt`NGj%Xqm5z2(O4D8;E}ybltppUOF?PT<2sd)*(8$Y)5N<43wJyz>5-s z{8^`62tWB;Kg=J{=~HE1EU(cC!GdbloQX;OM7xtyuFq36^CHWDjmRxftuEn9FR3NA z$mG+kV4tGqExUS)EDmthH$I-)TmCcOd-=*#z^k_;v+3%ZzV--iG?m|UXjr;+)@3r1 z93c@&)3K9}VBD_0dal?v-=g-3_R+z+1({X#K3ZOp4im2WHp9B%qG^`1^Hrhr$B44)WPXdIzX>HD22 zk)-2Yyu*Pq0KcE_1vyYM_Nd1SIX+@RAzg>$swyN5eJ5hoR-2i zY-U~tP^VEHlC7Gq6)F20I=nOp^D~sLuGi>&lb=I3h^NfcQmF`2!yN&4ez@oe!64E| zMFp4{&ZP=n!dmd;+)v1zeHKt&{i}J6qc3N=w8w2}nyLl8O!?}EGpT&3(S}%LI-?-Z zsa;g`!oVaXGYG<#4t{Hav)d`A9d|jAakbt(jf z%%jpQ%_Ke^O8-{Q^CiwAf82vkqTT-fA56f;LH#WagVUabmYgGK+xGwK!PQ4W=1Vhs z09C!@OMg=ejZASs@`3!vX#@i08|wv>i}Q0|@GfBS-$q#ccNz76%#F+~kF6u)B%;~qShI0cAD3BUq>WicMn)~16lcUUavA@{*2t1lMDt%=-49~68I<=a z(Unl{M!Peum@_>W+|=?Gd1=;yPpcGDK`!>55&XPspeK8d?;3<8)~`*?|DZQD*HIAA zd#&p}0hUxTZo47j10624Xk1}3I#zm1x!hDeQyeOEE({s|wqxV~x^wc0I5H%^OL8OI z$czlsWKjuS0`&y7Qy{PgP?-X)CY-&du2h7wvjs`AW*?Y>gnV z9II8Xri7>_^yk!0C*@ehmIqw<;ALlCYY|V528gG7=qW$-ybr;^l4vI{(*~lmjkoln z8m%MQT~sY8?a%LSL*Wix$!$Lz2C{Eu;TdrSPdvqZn4S38?c#(?S(H^Zi0sOpBsTNKUWP+;m5QpNB`o z$yEzF%G6SPXj>kx<-St*b@M>)x4ynnDUP*cej6Sg40MDCh z?$d*E>7lx=XRKmhR#)BBx|u{~{cuJ)dfN0Jv(aR6IILmyYH=~`wmGX8WavbVcnW)> z8D}@VFH~tR2fJkV-opWdk>M@fuaBOxIMAAyeF~zD+-;(6=7>;N%=ZqGKJH!+i~uR4 znQymPib}{Jf^4VLnw;DV_F}cJmMh^N{y}2UMY}#>^gX7`WMIsdgvGawtaVUr7_tN@ zS}1CHCQRxWCSdqJpS0=de{31kHV5mCNizFBc6%{Dr)4z@;TiP^u4XAryzPVOu|4N zqSFe;v^=t|p0J1z+D~Ow6$xyL2x6}B^9Z+ag#?a9t)+Jt>f2g2si7_!_aqK9Dg%Z$ z#D`{uT%=vyOQD9$-BryQ3Y}gvgo)Mj?YAxml zHuazBt!A@S=O8*L#Fc)cwIHuRJ2velyWK*aZPR%MB4JZbp~|=(8n0`Yy`(sD7o`rW{Y|L0U;&din08{Auz;c>cn~VN= z04BcWP^i~3Ym7wb+JT^49lfK1UVGN0pi}{4QqbiKxLifX7n|gkv)dYAm5F&d*i<_~ zbbe5JpBNyXxh=6j6DCbrD4HcsS3O-rapEv#(Zjw(mohvfBbxf80=_#&KG)U znt`{A$)lu{P_NszC&^k7G&gIl?{>^9JRWelbu+#^ z7yGgh9otE?&caM|5higDpf|3@Y8LYwZ<48O6PoL%UI51%iLJX)2LYWHD+XzaJa#$0~4A9y^%?bTjiPw=0>8`sKo z_L7JJn*PJ{&PP19^>w)tU!ILUBVL7#{%0;%faWuPUqR$628-YJ=4o z?N)slO+5pJ%U?x)+4m6k3#;q(LCv!}@3!y`=MF--CZQQXNW}%Tqz>&m6rNL%AUv&k ziY?R-U+;`IeiZ_hmExEl3*-E(Xy)Nw+yD>OGjavjioD%U;XWVu%*>~3eWG6Ded0Xj8@GymXuit$nenfsKW7go$ zAj-G;$6{W5^(!32TyrV@jXiK7;DVx3tp#dK1+j1xIQE)Kg=4Z>@&>!1&JDXYKYOkB zJ^OyS~DHoQ}L%7DVqI^k!ta?F_xyM(5^2pVh8IjeSrGJ rLJsY|{Hr()M4A7m(KyRebrXErv?MlOgh<-Hm%Vhs;pcK|p9lX3d=J;p literal 49616 zcmd42Rajlk(lrVz$ijjINpN=w1PSi$?yztI1b26Wy9ak)!QFxc2<`+6?j*Q#X7;!D z`(^*n|D2n1b8hGC*;2E*yK0P4;YtdUXvp};FfcG^(o$k7FfcDjVPHV)U=Z-cihKzH z28I+yT1;5Y9rmc>QI$Zmm6_*xx?Dwgwq=SkzE}nOjcO15q26oN-|5{p1ovujcG^l7 zjHy9oqmx(e&WT?`;?Q}`k08}B*wvM(y5pqJQq0BANQQs$uh)te|vg z=6NKktE+o{I=y)T5`qE4z>>ni`TI{}-o!300xvHrYOWlq<@eOoRptG7xzXD@UDJ7YHo~&Ivom$`KOMqgz<`9h zhrr)QM}2O$Qx&`JtQ?GtwhwD^o{#O;{GJ~L4cc!$xTQYdTD&MS4d`AP+HGoT?EJBC zGv3gUT+`*-SYO`^efvD$HTHa2SUU%;%gybVflo8kZI1_l<7Olej%c*6^- z7fhT1`s)P|T=^X_<8OF0o`!Cw{~r0EV<-zS6N#Iv@4~7prl%$HpC9&bcW}ng($l)R z%CzHjY$0iE@>^cy?t4~`p@lX!H9A`b=>Df2uo8Pr5X$-aT4m+d)6?~F6%&{d?+vaz z>_1&ug~7mS9783C{`tmV7BlVn&``7U2_KU&H3M%H^uLV*fW}NplIj23xB}C^+p)o8 z^8G*4&&3757lmZK@whwAd4}3~8*qDc?sEBxe1kn-bc6pz`p-Jn>EQIppikR%c3thh z#g!{hhhDLoY&qFs-Sm+dzW=*gM)7VGkSr`JCL=tG9xUvC1_}%HkA~N{d8pbiveVS) z^vTHZtnW(!XKqrI-EJoXk9J6W+~*XlAJPp_{1l&^Gva>T3i5l>d+TT1*LP{doDAn^ z?nqh&%VKJI4!uM(HARK{&!2Zkf&GD8X=r$RuCg;3_KZ#bG~a4hq$w6~`2)SUOaJa@ z*zSSyLeVe7AiZl=ezb@Y^XYCyk|AC2Id4?)q03`;Ek&dFDa5ve9!INVQ#QS;or3cI z>&(<}*JF*L*T|72(?3SAPXjM#KrR)l7=4x~II_gEpY|2^ll|`f(q1RHBzp@#bUg{i za)tGkJZL_V!&l3|Ym8TwtQFH#KNybIV4TU^-1zxKAH4Pc&(vXHNg+s}MjS1_G#$*8 zhhPo10GAy_!(o)bjed(~)mJ^_Piv1qpSn;`B70l&n%T7Ip`V`JPbG6knHYWAYT4aR z(VyXb%#bqUkp7t?DJeuZKwTj4auAE7Pz>5?$FG08vH7kGH{n~WOeaJ{9?5y~a*#!H zN8pL(7wli#OM?vfu&4|1@cZDtU5>k4By>z{_3YHA%mejoPN zp630$LT>yX*TSAYmMvqEt(0^4)pU(2O3@+*nBk}Q=p z#QAVOv6zti`Udu&-wqStLjf@fXgKBkDf}NV13Y~O8x~xd(Hq+v@b7~G29}E)CSY5l zB47TWjUXfsY=m;U2U@~^)^AS%!qBp$Q^IXU%PTV1+yu-M(r@umOe-O^>4bl>yc=LtBtmlqd-S2>T5{v`jb6$WH0 zzW#7~K8vd*Xk%x$<(SH>@3~);x{S=Q^uI@;`>O!Mr?b?qXOid55tc5!zYj5Vu0Zwm zTi`bH{{0mlxZ8jmZagI=+xJqPC+i8g`WF43-5&mT_X~CJgC7QuYODM`Yo+I0;ltz+w8pS@gs1#;o;!|JQ*&LA;9tJq{2~DOCD`I)h^zaFW4}0r9yi-Uw*~_vs!n`zr&k*4F7KWdp}Ia zoG+8)|BRo@0~Ue^L;n8j#&D!iMud8v*4ITau>Wc3DggsR0ipcmjkpvr0cNbZt{T&& z1pmxXi3s?Z6#AFXLs&tx7+OsdlK=I;GASTN_7yUqkzRZX20RLTYv-B%D$(gmaTuE~ zxQckY2<6QS&2yZiI^{wTZ9=U2xdnrfQ5{DwH2#_fJe%Jc{nsl7N@i;l9>>DgWuO4~ z05b}>N~)%Ly`cFelt%Z_jeA>`(ptN{_N&|mwSey0`1W7;$+ADJnObDe2Z^}IU>ci; zGQyT{m9~|Bx~T5VslWrP8<7BO$=Y(cc-0qS!$OO&UoB6H!>CP$z;nrBLR085v^W#+ ziUzKhh^&W>IWF4g450nixNB}+!pT0OyA6m{MaXcta4FFk(3E8ZCo}_VmW2(D$kVot z+~Ll=^QE9agN#U=^;)Yjf2?WPVjd-NTMmX<*r2v1w$c?!77dw~PuG>j0W*Rb3CJK! zEh{s}lPM~2-6<+^_L6w8O5%99%fLnb$0UnV21)ootlj=BdE#8hs|UA6+k5R8Ui}I! zYo541gBYA%WgbN9jW)l^cBJ!iwH!-IYbWoe?gD{sB$~ zT3KeZ+plBi=uNI2mV1xgRGS0LR{beic2kjhvsJDz5jug51@9I~)Mr8sIvpZja^_pC zdMCWcB@ON?E*A}YSX%CVBJ7{r6uT#xzOU8aDM*(y2w9x&|)2U854~XPU10E6b3Np5vx4NtY2RBJEuDYU-4Lf-H6DQVNzA{ zg6Z&2EHhjmQ}#UNO8aPA)t7vx@KhEmH2Vx@f9OjQevo56NtSDJ9$#a(51HYXGFpF_ ztJBpdoARY-B6-0Yrw%%V4CI3%9n879_&n*?5AoNSQ+EUCh7bJW8r#;jUzCGEvbEpF z%*~IgFg%=03&31f4O$T^GgLv3w5bFh?@S%vGFfc&k3>hGGA7;6CSOOF_JSduKd-QW zpg*;xje~1J+N8Fy^Y_fRcL)9|{Drs6MPr)#v=zwapL|T5Y^A_I6Yjcyi^-sh%k_Vo z)LI+DAp3%SOjd$rjBiLfm7~*65<5X_=d+oPNeB9pPK-52h$R2ZG%}y=eRBZj7wQ0$ z@DC20LYg$0F9gU7<)l^9XTu!GY~?`leVh5~5gW80UffIef$Ns*druDBV<(n`C|9Qk z1`s-52kU1jZuuBVJD=N@u*_U9B)#TP{IpcePTfQfh0C|ov!k$6(xi0?)yN)F<!ME2oRlV#+LK07Na0rMj~UIH?{~L)gD=;QZ74T{OVWywS6OOz)E=J#F0(Hyd?~_! zyY>N<^pkqaF8!}|SY5S(tXzF!Lu9z%mS*!p(+(B;mp+f9mw^}YyhMlJ%F+nAal|rv zn^tC}CpS8U&eJhjGpg5mwh^Xk)4KDyR9UYsA=pebllD2}5Fwm~?);ag*V2Zh6W3`1 zRBt?GzWAB$JrnRy*9CZ>_B5N(4e}+d>!i2Ln!KvHxn)Z-QJg$TFJ3-_P}+acq0H|7 z#UCoTc&dEe@4CWBO|UIz`m*h=_Y-|G*Rlb~+TvEj!rwW17iC;>$tQ2&aTJte*tVn) zLx~69ny56tP`5llt&Wp4!5Ao>iX!^u_wH@)$)LB+UAXjFPVp&9`(19 zu`m(46`g`u7Sx8g#zXEUCQ(W@p~1~6$yEGEi=ywzBULw{8ZHldLNbJwDIITz zbZnh0&TUh-Jm7fmjOmQR#YS0gJzH7CbP({GZbf`uj;fS$!>%NUr% zI8Q1t*=PwDi>0~3BPu=yv5M8m5`whiJ%ki+sHOP~&;yoxsXR6K%p3~T?+V{NApH($lGw>s6)eL?)+SMIP$R1&u)|j_I=fmDhJ3mG zTz-{Qxup1@%vzPctUM}KMEII!<~y@-n$<|lRHO#UKrG^3dEqb2mo&AN!Jmt;%4nF% z>gb~w`JXyH4XYAjRX&qLlMLzHv>RN6bCJ&6+Q1*$%E#8ewY)DCe4VJP2$}3rL037J z&#orrn#M8uP4V&z{eqe2+h0@@g*D|#5#3YENmbN*w7g(_x*JQen4x%vO{~R`-F#m~;VbOGo6%9gH?tEi)77e- z4OK}}>UN5YpQ5frsAXy&4&42`&PQhW!;M}};xf*y;{?7@7B{m^6l*B!&hH`MZXePq zQW9=1z|vX*$2ETvsG`M*=wC(El_DlP)_ljCfdh+%ryZUF)-du)WY!jlV3=gnFiJ*C z@@B7!;|Xefstnr;RKm#b%iH`I9k8&~85_1YGlBa!n6@kqvL5HxI=km}9ccR&FH2|S zSZo|!>W$i0+8q6$MB{M)jY#fzwFNm&cemvYLwy5&1j zY`HRDy6IPf)cL$E<-dD>&>m*>WA^WUUDVmawY{L~4WINb0~GcRU^=w zp`1fC!Jm5TCVkC0Ab=yIS3DRqECF>lLf43ehA?!!ol#MA%d#uk$Wyv^X+186cD!{w znKD&4K1;&VcId6C3>DdBTns8g6fwF}k&4rp_Q_LHlvxO`Vg9x_Kd_|%H)reX!MFNd z0aKb7Bf36pC2k*hIyn{i3l{qU~Ye0@CIkVb6tg4&OJLbwOHw zLr<}Iw~^B9-<{k zG3OAM3eQ8mg?*4k7o>}gGkZVCGi=#PjEAyET2Py$ZjM$i9lVgohVv}XA$q^2&=1O6 zP@8lRx473faa0YU@~J=VSu%L-sjBsiB8Fv`1E5y$R|+~RYp=a_Rm2t;M8~D|>`KC( z#xi-uzZuY%V$|$mlhtM1S*B%(jF_;-LoTC_rCrCIJ!k_}gUhp3U;}{7|FKU%9O(9S ztaM4@xL!nfrZW4OkyESTd*lO@`mV&dm$bhg#tLLF%2LqhXEv={DD=-NVz*y1x0yRM zo3RvQq7ORzwt7G!H2i-&O}JjDBB)`)rF_S^1Uynl`zyw*&tj!9M0ZT${Gj1s!++gk zFJMD9s|XaimgpGKP45)}O>*on0c%HT@an4#6j(GFZ_wS`4O|IEgzbeSlYAyu({7nE zP_hni@h{h0QY&l{1WEW#gsSCvrZQb0)p?>ww=>MYBH*Z75mjivsVbbY*R?e+J=}VZ!Ln zSgtR@z$*2_5Iz)2{3{s~hWx9ysuRv;R}v5;^4`W%iV39t5=raY>|etHy(~-4$>^Fg zwJ};Hbbto9p~?2*!Z9P z)5o-`UVJX0ZE7lhRU7QjG*+vLW{pxlgW+v~^YPEBfprJ_1H)(xN_=DhGV(HP*70IQ zbw2Am83J1-^>Rw6ZFI~A`d?Upt3M$?R*2{en)`&P6{Jvi9l6|30SimE)nccPaPn_24=MHZUA?qx6ajvO}vF}V+gre zq~u2M9(#hT0=s$Ol;Vr{G5ksD!-zgBqd@!LxYi0Y3F8pF?s=mQz~iA=N+Ui%zs=|q zeD~mm4ee>oC3~eJl|H^qV$eukv76u5pNK1VCWkuUMy;@-5hswko-O1s12MpiDCOIb zIWX|jchbh{mkY+$=L=3`Vy6l4axxe73A6wMRW+?xG+iISox~dShjBAiwcHT zm{IcOt>nAK&kfup64`Te6Rs^i=u2aioYfc$#wME!6;`2j=| zW3et_kL=3&qLOErZ55+fYcFwNMH|%=U}82M_kSj$&EVWV$kT*PZ$w4%ltR}e<&W+5 zv#{l|d?tRV(JVY!-6{R}^qon#JC1(&PR8@B<0%H$&sxP~^s>g*wF&M~fKzUwT+UgQ z8al3g{DQEgIV4rbi^Ot8Bqrv4#O82u0{P+rKF?~Vc2i;2x2J6f&TU&H#K$WSz%>ayUcKJASzA$?pfzqb2Q4Dw%GLm4o94(}YU!cqWRZ2>9J>^hIGmlv>kfQBA{b zP_~2;R%lO3_$p2CxX{>@e-k-uc^cZE$|S#e5%^m@I(^?yUecuu3p%na$0D1q5SlDa z^Uh&M!S7m~=MI^~-13U(SpyO9Hl+eJDVV&|*e|WI;?ky&>`W3{yFH(7TN$_sisD^bA9R9tAr%HikkM#oxX2*k%E>~}hCxd{zsCT`Jr@R^Z=@E(N^am^Ed z8oHog42_xQ|Ckr!X#FK^_<{V3V~fm(Uy3dm_Nm>*6dJ6qEko{c!8PA2t>FPvio{$M z2iy0KG7FOZd>Ria+n#+nH6bg>*9yTqc|dbVP7+u-RP6-~{iSUuFkX&{bgY-{b z5(<7BlZ#PbNZMD#b=nF;xV7&OR=3!`(nvvtm4=3~R&8qO7BI~=+BRQ|I{Ie>uv=Yb z8abS`_}1{2MnK@jFIh^)s!D2H;V531ohPVn^jg71EdZ`+Ilxyn9ak?0_O!PIZIkN@7qomR6E|>xk8LD^(9AOr}Ro^5?=s zjtqgf#X1Qnd7n4_qXj64e`_=tB_Uh2#wV@d9XC0pSx{8SuQhBVTGje~j)c0^=jp~G zA$nv^TeaQ_fnq+|JWiprx^k~>N6z^^=^HXby=PLbqI9TlT!zF{=de}QiF}N1=9|v6 z&ssXWj)(6wd&Ts63pRM&?@W~UsYh5O#xJ^dVn>8gY<+4a!3J0=c+$mcQ>nu~BH#SD zS@r6(Mg;s@Td5^%7SUl%$4PTFmb3aVxA(-9l}tF_Fo&17;=4$4j(wu96#h&K-r_@aiRf<#TWH4cgetcUiMY=?BOb7pe@@8-o=+tgymyqEMHB;ol9Qj zXY0qaJHxvbdwoi2{qo$|gVi{4-O8v-)D;_t)}uz$l$XysG5%z_Gezl?{>pF+_oS;^ zc0>?s9+i`p^i`>gD;ZC%1wo;L@@?1M(u$zNatcT&eMXNsu~KmP0n5)x+Wl~HH=#re znoY@vs?*xst7V}Z%>-smB*u(N&4m}L^AnTL_Zns-BC|ch0akF!RM5^ny9pq1!tcRC z;FKp(icNL&Q(D`qjDW}wJT8F~ZD-_Foq?n+yIZAZup;`n!CF*Ms%r6$_y-3)65~qd zXV+M40z<*mk4nc9EEG_-uU$rU>Jb9`QN@)D%xFysujVx(&T@dmBwWUG)Re=f23Ge< z9LDO9_D4{IZ-MASkob?LCvi4Cnj@?EM%Q8%LZT7J>(>4ou><25(W#pzf&gGZuBq{^ zxpL(QqbU_EI0CZXv2uGRGB?%_5Ra9dDdQSZ?5j_-xRd9eVKh}N(1yg=d|EPSmW4R< z()Es6f7r&(gR<&%AX@Ji6Xwlcz&7M6qcG-Jp{FP=c+2?VJQP4p23UwDOL}5sV*6>U z2@9sg#I}$SS14R^`L!Hlr?$p3>I5)NBQT5u~jZx}zU<;ye_(-(C+RBW-jyUAs)jUv8yvUTIArV zHrE*z-Ey8bE4_6CUwfY&{@>mucgbq4TxiHp;6~rr9tT{95w)1(rNsP6qCmq4m=WIH3>% zF6~VxEpk((g-b(|@=pzO$c%U8FNb!gP1w%vPiyIZ^SxJY?|Eo5nqrfa%$RZ`FE^Yo zSqu^%DV#Y&)NKphUOtY<2|L*42{5ZVj!=&gZ>d;y6tf`HZ(Fyhc_{`mt=8QN=%SV% zy?TGjj}TyHdHxks>q5L>>OS$Iir5#kl%@?7lsL4t5Hjh zEe;pTZ85YO*%0YXsDRfR`TJMx-MCG<3$<4P^eMvoS5c(UWm_ZkCUoD4uLCvB`w-dw zIFCyp!mmiFXIDjdC-z(=@N(q-jRO!z{hz#@` z2C9&K2d}GVrFEDIU#=BfChHX3?O0|s$ruwv0*CUQ0;Vx-aCnnW7$^2sxA!?2^t(w& z(o0bo-w7jxiVrS5DH*+|g}uT!G};x#E-r=LtsFcNfAgvfZvsH$x|i1|6>MzZ26dz@ zl`woAc|f@l(@a{UcR@_EVm~I$0_bBn4LKO>{-iyO)8(_20lFi#Da*FvNXJ6nVMJ(3>7q`{b{+k^$5tWQ851__k@R~nfTy%i#j(EL6f zNCH`vEb*@(0sl1%8sti~D6!BK222T6R!1BA#n$V!)5O5#-es;^9|7I-Va+1YTmpeo z8v!h|f90wH>XI7~0%_%Cur4!p3P_H?JPtc3{EDSTF#khped?lGrnM}2@eC5&r;>bt zI;-lGD!R2*RVCx}OXZILTSq!E&$N~zxD23bI-v#P<{qRXGN|<~g1EF4g8(o}AV`dS zBV{Ag=uUs4`dgl17_HW^Y4FR(!{NH3KD#7)h_x+LHGirsUfg4;K-9l#Fnq@|z_H-X z50QX~Hxx?Q^eSs>8AL~X_@$cXTZ^B*VPILFwCAz;__{oZ=+X*FDVEnyG~EA|^1q9_ z8&DN>$}|ql8l7#q5BklO`C?Z+_S@=rJWGLhwUxwf_1RVR*vs5}a94#g;^MqQkg@^6 zFYgEwx^<+8KiJq%>TlLIZKI?hS4N<2j`oFajm36}i|_-i>ur{utxE{#3te(iHkw2! z=M??=XxJAFGOH3guWE~+_`97OcXI}o3WiST{aXozXuFD&Z~JJq3DfKOt>D+S;%0sO zvUwCk;w~$Vr>?gmU+y#E1-dvT%9606t3~~1V|BZsDK4i1-5!iIa(PLe& z9#y@xYG74wEH(yU#R+FyNo8)7Aqs%Jv2fsx48k-taa4xP*qWCXb>2R#Y|AH~$Cq(r zg5edSC(R-r21>s8v+FDNzZgBnLkJt=C}-EU-IB-JSF}4m3pozS!^JvKv#3t zJGYT45gR$G%>tClSZqYl%u`plpfr0nzIHP5>{DS?I2sSWbvzBdxfHNH?CZN#MDXEH zY#B+bFTwAE-0I)Omc1s+tNt=el)Z)@3-A2}`;eQ>rFiWnduN?euBz@VLJb>cWi-Jt zC3fZ(z|371a3T4`m4G7aDR=fpL$+A4-eBbqms5!p;Q6SxF=hOS zvqD{6TGZZv{_^VLAA@0vL&}08M?~W%Zf9bU4?yG7+8;*3?ofu=l_qg{S-`dpDOsJ^ z;9z`s2jcg*Ii@G$v#XzvN=|DemUVwQPNSy_vwCC3`km%&zE6AvM-6gG!UDaL%(isQ4#YDDxtr z=Px1CqYKcX*|+_f-ztL&njVDjq5P%$n9Jr{<2xh5BWOluOc#yOW8CI}%~Zr!F6heK8q`8osl7z=5T!1GO;97l^V4Oant-iXQWVOB+L7&v>iwac2&7u&dK z3YJE#iEx>OO!@7%0(zB-(h_OI$~i!)TELq za;<<)nhcmIjcWBA(+2uU_LdK(fe5oC{j(yP?J+BNN{jaczUH2?(50&tg(>RO!g%<( z4NE@m{(PeR?mMfRO8QgBhTb`!?UNzg4=Za666_hX0tp3u6J-AOIZw{qB3%7~5!3ya z**S-HeHk%9uK|{~1nH8zmJ+HV7!~*WM^G|}AjL^fKAwzFy%^-fHDvOqG!c;!0=&ex zs{1cWvmvpn<% zXk797%fhKz6g%}!ADXTjeHQ+!V*HU)$pgc#r2LAw+vqYe8%rZehVZ1$LSLEHt?H$q zhVvvMWo2nrF=*85_AKekrrEH1mYMYi*7f=08QXz;t7ef^)*{fW8o2}{rG()Jpx|jk9Z-_Gu)md zOQ%ryBI`GE>%6A>Vji@m(I*P&LS%@T@#h}-jQZs=ywj#vlBHBKKOp7rodiW%IQU|@ zze|5Fx~D+x8FQx*l|^pzTwq_XMD&SX`(1`FFUHj>q0Fy(RQQWZnX_5kBI1aQrnfY#IGa`%6PKO+L`i2{ZB47$ORJSj z9TkVRwxLaKFTG7Qn9Xy%id?}*!pX=C z=1u|IX)hTshma=+RmB0p+e!ZS-?L@_55W9x?u;$CIb59q!PDa*ug6RPqEx_o9QLF~9{ z++fda&$+*2Z@RAao`FCnV)nbI3?Q{a=Rz0w581=_Z6}{v_y5f#-L?N;O(K&FH&->O z{9i)bF|ZabZ3QLty|=$V5*)y;WJrZj3i$(}xj!N`CXBiqD8w`-3esaE)Qt+HPUcD? z_?WOzc)7RXFr~&=w3I>JAVj@LAm_4{_X4B6(0vK3=4Osm-aK6$HbD=r6otXjUU~t0 zBkZ-%^QHkvON?j;X0X)01Ymwvh6giREFd*3BLe;2!;IF&`ZHy7_!OZbs?&>quiGbs z@>zRca}oMZJaRTxhd?4-v|9OSl=`<38mm1pwC0S$Kih-?1COE_A0GHKHFCyg(Ib{G z4B=e_N`a`NHBWXIGH+fMqb9|O7?pt#@TwEOb`4lsks*~s>b6$t;`L)+l z{aVHPG$^4mDY&_o4z!A;wQt&dBIRuy{xaauX7Ea|BpdmrjIWN6`Um)eFa}& zq8+1~^Lztv30Yd3@0p#Bn<%|mlSdtvm*k$6r|t|S5^*{gP+?y*R{A++z!fxUzGqJD z@Zpr=v9szJkbAbK^cIJV;tfG-@}5y-;{Kv1qNg^-cm_w&#d?RdV@3=Pe+|-b$7IT9 z@1;-dn7`9g%EJ01**Qbdb7jQ8$m*2t2so40N4--t$SfhIcB&gWedae4=2<+j+u!r6 zX-W>p2`GAx`&gMI=U}D@*G>s_e5GtCdP>ocy>FV<&MG&^@1a4~$5c49lSRwD#LGCQ zcZeTsFK&LrRk#_th5qieA*ES|%1LjQ5SBuaR@0XGjRu?sk4KW&;DVirGS-_ki?zKj zbWeh%utg%W-$XHO^YR9Ywf15>w>am1k zhVQ*~L)1C+TSE??h4Owdig>g|2Crr9Hiz39zM56(rST~W@1u@@*I>@F;bv_&#e-8q zPXxbK5~qA|c<@TO0g~ThL5;OCU;2bA;QT3GA92uO>B24FGbV&T(l5*3KNT*;oObrL z#hFOl0Mnc$>V}TRgr7#5>S{O^7T+UlmW`=k_?^#WIAQATIvh9MnBlvxcBC-H0h6|d zQ4hI#nrIB{LFAf8Pg@qjkw>F3JVwqlH^R0Xj7Swe)Pq$l@>j%Yu4|&7MNo1_JCiL& z9)!fG+BkwSmvWUC*kGyi#W1zlQVXd>456D=JXtpUJ}(XhI5z{r50Xld2dm_r5&PX6 zFuTO}Y(~jh!Zc=etadW1Yw^${(_Y%1y`yti-EdHe4ikzOZ)T_WUwPfJg^6RP& z)=9;Zd+8mxSsEf;Nk!@X`=f&%MTSed$~f)Ms=Nla=!qM7?$JMN({xT{&D!3Cse#XQ zUSG!_=5cBoUn4CNvOV>+3k|{K)!E*+tNb~R&quM(p<<_&){VtYS+bdMYgREr9*svV zf&C7PL#qYOk&eKz@O-MvaYD{DO%T3nAdTKM{bj+g-c6-VwPeRK7lZ)*WAIV5;KSab z@v`dtAME|4O|}My>6!iaw_D1g0{-9>W}C6red>S)yfgNx+&07t_CS}wlkZb_T7US> zL!PiO#k)=PD|*bXdF+2}0yzpOyC~i4gt!-U&y~M*l-pfHS<_MBBG6HeG}aXzcZI5I z(I(FWYGDyIQ{{q_23!4#TSE%tvd49wGh#z!#=VRnazhU6^ zR>F;JXedOu6dI9r(?)xF{PYLZ9fVHPUHioy#UAWkUgrpSKG6*cgtdSviiS3{J4D%y+gGEKe2z_Jv1aft@kIX z&{iLX0n3R1#86)BPUmI3?=F%`Ev*4K|A6%M?DBqiWR=T7$9tz2M8-Gg25!(pS6tAD zuj$+Bcv$u3t_uVo8iY<`Ovc~2tf}&A>4ljd{dGlI041Qu>B1`K>m@(Avc#{|kBC*Y zIW(aDwdP8sTNe~rP*WVg8uaH(zkbb&p>c(s*RHSqsy*wWV~r)c;EIjLszBn8!|&Gv zZBKiw^6mJ}ZZpSLu?-ExEsB_We+p-4-N$dJID0HTSIq5sO?iE^ykdCO!Q}fpS;rkc zC93Ho3+iG*R{H||Lhab|58t7I^A|>j70$)ZuFadDe)W?3sa_zQ(-W@zQh4Z@t2V-6 zQiIuoij|6d6eDa`f8=K)z+ck#7E-nZJ`1XOU}5C(tdR)*R<+z z&hNTN))^E2UdM5n(S25#*Rf!4BDg0mHnDRCgpk``dRXw`gmBOZZ&ZkZlCHCbb-XYN z7{NXJ7a9$B5Lxw~u8u~z5s6irv#p{KbsSr44SBeA#ES67=@F3bXWO3gSrWA}{22un zFlb~SSL|^F2mrKj@gI}|VHQ+CNVaIE?S<`*4@%Y$V9))`LMFR^T zJ2>Za3{2^UGm1vt6GWC{($!fzH=ne-1i2?ZU+a5{) z_Lk0#`~FuLjArrE7I^3OW4?DTfF3t+|D^Jdg6I$Q+ZJwJ>;lC7a^X^&dO(}qrRrf0 z06#`;c8aX~8$WJJEZ8x}fYJS)8_W)$ji@g4A`wU^fC?6ZA(TW^dj~ckEilfVV2{dJ z_xFHX*ZOX<{X6MzwNtDY@;n!bjYCIxl@)2JF-}7D_~5_mTbi@myf`Ox98ZDrpZ9>+ zsef{(!hBme*((2dul8@PQ_RV3=|eW)d|o2KaO9v|zwj$QULGKW2V0Gzi{<$cyc`A>f;DFseT`^ z+fS}*qjvmWY-fY2A6c5vm{Nb4o*OX=k&C8pSRL+;;{h3Sr~*fq9!nzqYymc{!Ty#j z%pHy}<1w_T zKOW_p0{C_GV~p6{v~HWXkr$aqRK5QAWbv^;RJ`iu5Bktw4`)ic>o;Xvmtq9rN#M%S zA*V|x6W#x40mwEOaJ0*jXz zf5p6)zSO(ORK*XBQ$9Oh%WPyyqd@@d|Uw_`L1!6RcBE0yvsP_9UMwfB9Iq zh~X#5Y@ri=f|t>e2wQTZW$tTBkGC6)0&QldR{g=WRwBsqW@rlt$9vTBYmrM7Ujwg9)u zg@aTfA7m``7@)52LRNREPqJjyLAi!ZhZ!v_*6tW{p^k&=(@UcOctHX2P6 z+4aJ7>~#M+H0F-?AVKHM(PffAPkp!#6TordUKm}^`V2r-qD$3+Mp5Jc!)AO?3u+@k zf!cQKSp}5%1dW<~SO|hM-W)#TBA%4c zl66$2nsawr79<5xFV?!Zu+Cf;lEygIc4P60I$-O?%JRl;kf{Uplrn(yC4@)v-H7N% z{hTQ!#3!3bf9~?6{gF&J-oteNITNW56&xnW08j=Gs^Of1x1L=M9dar>{#*-TsxeO8 z3FQSqXF~PwuCDhej98iZ^fEf=o&r-lKm}HDMR2~;@=r%ykFtiuT`k6leM)7>kgc;v zuD15E`3Lr_fR*6xa_W-p+AHeoGlCb;UW7;6j0ObBYoY8g z;=!rwy)Ev?>jI=wK2L8tQX3aT?-JHuSBaqUlXGvQOQVNo=nHlG02M_DPni~^Y#gDM zJ!7ofRs@p18y7-1n&9PSR2uS!w9xxgbypt?H4_V<875}LkFb2Bhl_j9F2O>CrFLtr zf?LC&%(BQ4!-PHS&LJ}i1r>PUtW29bfu9*_OtNWqx&a?a<2V%3o{RuR zl4~I%yQ+5tFt0^rswy4um*ze$Thb(nl$%K!p&L3a`n+t|*DH!X`$qIit*o0IfY-e= z)NoZqeqAY-vyh%pYpl`yscjH1pq`e-&>B0y3Ei4IiqD^x|Jl9%2NR|`a@_?Oxe--K z0Nb^~{EAWjY!#D{GW1+qx({ECi^N!bp9NoQZDsup*Vn?~tqO~c_lU0e!cR*$}YvnLZ2uiLA`du@Rc=2zKF1~X`&OS$LM3O*Es5ntqAtJoPBBZ_4-9V z+V7_^4kk+(_(S)ijCBHv92n&w;a>W$b+ge#AdA8JUaCb2)yLQY{eNoaE-!7kC2+MA z??3?yB`pBEq}@4-qr!yANxf5gFeQi0?m$P~yEkeg0x{uz=Sofb@<&EeGK|%$l=t=c zp<4z(Wk>u75c)nGETjvS3hJ-lMo+pr%0<+VUrN3f6{{)1)xsAnJ@d~TQB~Y?cxNE! z*J`;J`Twx@mO*j7+q!R%;I6^l-3jh)!JWo~y9a{11b2cv1b3IJ%*Uv*b80DI8SX-A1}F12lHxcHfOR#F3;l&IpAp$cvpUeqq!E1xW^QBfE)Tjh2l zY|EI4k{qDsB@WN$EuDDPlVdR5AFo?Ihv?20mdg=-|O5qhaGKt{LO0Fhi-|b@mh=`W;=+wTB z6%G>zBwozs`SjMR=B`qGWO@-C11o=k{7b zy+D}1sp$JiQQZ&QTKie_WVO08{o_dApxlc8ACv!ofXTbo!5x~w0_t>9*Y8-lF07aj4;y2UYDm#J zl8%4Ab)`MEe+TgPWtV3O=<@+8r$evcdAHo1nkb+Cle*8}$&P2Zd(s{*kccyy# zxu|mUf1@k>npFQwx?dHwyaDJY5qKS0nO~iG7oBp4)+=)L{;Irr$ZsO=qUa)YyMz{| zdDPh)8Bmk2c)GeeHg)D*7pPBO;Sb1XFc`guO+H6FpfeQ7rA3+;HXq3KAw{Km%x)}+ z{?-y4UG|4@PG{}{KdLyFQ7(p4*Y?FVCK~!%X4vcJvp%>HQ`aU@`%iuAGv~ruw#Ch= zb4+7MZuqEeiGnNqoa1m^ogopvA&^+NS}SL~-cIsR>}*d^7E+^w>$e@gT}yamcm>Q? z0W%Mq5hd2q+jC}dsAz2|TimvklR~6JZ@_U8m&@;SExtQ#L5eZ+^X$&;B@K3qz4*@) zZd=+C(>Gx$od;2;JBgnIEbaAPci3Si?0(qmGI~8)e{){b-RHJF*|RWU=8>3H2h`e` zD%E(Z4tzpzC~Ijdoo5Du=d{FOau z(#;+E0n;BYciZtjWK^D%)GU9hDZrsrKh0|-bH-!)zr@e)y>E25wEp&_ZjWbh#dX&+ z#5C8e%Q=f}Hhq`+GB0%(z7f?S73^&aQ@E-8vD1W+&gI0x<~_JRnkGKQ;(=nywl=OU zMMbYHc=%;wYmCgHK9V>f#RH-{1pZ$szr2jHe5;KuT_(I19yk2k9zJlRTz_uk*F09N zV8*;C#sCu)T)!JTTD=wJnU7ly;6xWW%C}1xpfZOKtb4soaKA6S*XEX4lQn z%lVTUnuO=MA7?nb!Q+s`u4l+Lxo8rv2xI$cX>c;e`NCz&JH|BfYgrHg-uB5P)H~it zNUjd0)8m(vIsgL-kge+{YEHh}HR$F6n{M%uY}$T_vQ#<>Qa;j&=!_=;zObdmbY+EO zPxfA-XAn1Z>DA+%c$9jL1+C4?pU?_@B<%?r7r{sVLcm8+zahkRY4?+7Tu|Cp2-KmU zyi#D?Wk3$j`1~||2;7X9`gHeW%!oV_i$(082G=6ELeB%5{p|IcZke=b1V}2$YhJn> zboT<#$v?C==Vv;t{FRp-c;bgx3EWcVMAw30Qf>lNIWzBwdmR9Yh!R&8MUd7>*6oe+ z)l+$CYb4P$tI#5=0xmTz*nrPelLpdh%%;KWSNSS&ewh+FuIbk_LB^Q0QiG`>duGn+ zUkD5@aqZ}c5>Qhaap`qpzu{eV<%{Agir&BfF$|Rq$YzjgGnncK6&jqs#D?IrN6yoUCZ-qVgZJ8U<#ibD+NUy_h}!rW z!^8kE9#dgHo?wHq`7l3shi&|5q2Myd?@p5qHvKiIHmvY7=1ynbw9Mzi``*8VdJ~FN zla;ZdmAM;hH8$?X+jQG+s{BeQaL6_Y;t0K(I6^dGCn^0PO3dy@e#ikK#FpR^Z;AB> zwTphWF;r?rOjCq7WYOQW2gd>K@trP0NJ&n*4-kgEVdF#{3wYP?tx7kcARNb%L8dnk zAz)IG)%j?CYV0%fhu88wYR_D7TQto=1WE-iDm28G^;$%BZNiA=&t#fMi*l~~Kv`LU zN-tPI51B+mrb_e4Ly%g{_mn!BD)uT%;@zZZAs|%8civ*3c@*Jo9xb?Q6@D7tkU^L0 z1Ey^S&TLR_K^7mlrYJ5azb)jR*&Z7mI9e3%OI@x0O!mKTqejll5=N+MchYW`OvIg z1Ltrh!l&O`HHxhcey0YEY8s1InbMwOQZJw1s{^X|tJY?fCh+rXLM7d%iXLR-BI!(l zd;U(-=pVaKb*My?Z1+@6+eFUBX20kZ6sh*VM0ikFoI}le(A%d^1;smj^KcCeA*@Ui%`y^~QBMlu*ah4py++T@T6K_vF7O>)X26|s z_-fNB#t|Jqq<%XU_96*j7HSCZ2#enVgG8bC#lbA+P~K{ilbgY4;3*iIj`9HMolsqM z@b((5Ds!R^*;{see!J3zk-NOiD_)fvC>OM~mC|^M@`!MwlDY6(-;2{qss(l!Y}Ft? zD2D!+QI2vr)ZThvM){N5#e;w!TZD*OJb}Da^}bm7`mN9osRzD>)bMS)4%LXj%RDvn z@@WAb9Jt*G^zaw{5f-y${))FKn$es@w041jB#g>)Cm-={A-5l5eV7#BEP|i}KLeR( zQ(Jy^j3R*0E&h}5s?e7q1gTmF?w#6-(U7PeG~`^4Xe790(7Pv4#Wy#jW>5rrg?c(oUU_9|1ZB5-96>w7(z-9^7z}>6BSxixE1VP4~2M2VzRp^$@$iQ)L!P3l$z;ymj zhXPD{c#8S7D9B^(>eP<=Df`jJS`WS(>!`}vLm@kS1kqaWlDMk87wzLB0`SD?IDK=o zU^><|guH4Yq73fR3`P5*a7`5%Vbb}P*yvwnwDg>;G= z8&IE{8_FEVgcE5YPFJ@6v(5N*vQylQ1GE`z)FkqL`@)zLUF6{Z=32-hbTdY^9a}$8 zvm@qFa)SuLxUnuZJ@^X#(l!4=Nw%|kuE(HtbnT4FaA8U{gYu6qtN(@8RPLWN7_y*aL`B=3W;CnyJB$46tRB?MWvYd zM#KY5IyOX5p&>HX|8^EraOGoEVO;S}bL}+it|oc_iS^W5p=VQOGp%nmWW)Q%CZLx0 z8<@_D;Q(zNxM~Ub;3;*FcIkpXsNxMPoIw});q@Tkr6?8!EnT@J9FQ=`CvG)eLImROizzVDm*lRiR@7c4f?3NGa*N7eX;rgX znv2IgSftY@<9R?F0Nt3=K?Oa!^NnylVC8N2Tc6=N)DMerO2~}6Z`v_#x|8nA%ProJ)J$ywE4YJ1OzR^A* zC8u^|9rF=>R{7}Rjon?eHG^3-UfpfVaN<%MHzWZ*mdW)YLUoPOadR_GkE&gNJCX&! zXnNPct$^qs>WPnOi;KJPt?5q5WV!CS;Ic-pK1Yicl_(aSvFO^2Fy*AXb&vT>LXQF3 zAcl_Vg@pyFwU*YrLqa^RB~gBw>r7STlETWkC@R{vu4P;t_UDF)NC^1yo>P%6sW4BK zO=69_2kf6a3Yp04oj63ep&X@<@uxr*5Dw2$io3P~Lcx~b-VJCGX(H^-B`eOT}n>|Q98v;Wy{3gPiw?zaR#p7S&#q;OnX6Y|< zEbqeipW0#FJuPW!P@~A&V5^vJOL*~LBbO1?iv?$$2aV}u`pHi7y%C!}!rK%(4;wuV z0P;{!#kX+dsIyNbiDp;a-d-880pE&jyffnttCJsmWOn@}e-&GL)E3L#Xa4}yhb=P- z=$>3Ry#34MF|iFtTPW(IqRS$ObKy8ZO3SCIHSmaEG-}ps^4A;yM~z9Ud05iBE$as` z?mQ4u{Zjcy3{j4`l{Zov^DK#7xp8GI=XbVL7I+?v=saxB^b=eRUkkAN!L0z%G9}v) z2%s`vGK2%>8R%I^eS!t0P3UX+;lLPmjyd{k7}XLw|7&RxP;yG-bAoI;I#Il6*vv}5)7+G`LlF;f1t?}k z)1WnrHx@y)ikKHW_dRbU`Z5cfb3z}HMG2{#2&7*nCXo1+vwcEal;d6M_^LyYtrQNX zW?~=S-VT!AgHFjwh9(o&onmXDEc=(vUo-lR9gQJUPpZ&SxuL({_b$ymI(A67FHUL7 z58w9pSbDW&*OJ@DP9JgmjOy~7XCj@P*$K3hAm=8wV>!h??hcFtX-|qDDuU#IkgN&Q zzNb5#QW{x6)$OBHD*)n;a23d6{el2V(bHL4SrGU@5YaO(yV`T7--sx1z3;gy5F+NR z%;dKN7q@QJmWB~aA0hzECxwq@39OP^yCryN&VsgJ^;K-K!HnjlA>n;Lrp=s@A_T} zFA8E(O{76g>Z{Ecb3Z6+3;PJ0g(*490{x-`s+cV?R0?a-Tv!X{%*GV!EAfXyRvl+O z3#Wh)>_7%#NB0uq;E60aFy_R(u^{vEDb!J^|2WE*WymAb*0Qva7#OS-Cjhlp>3g15DZg~M@CBHpyYSw#cCxZ+Xt;PUgC zurwVZWl@xsBg2cV44tXssoMe(4sKi^({z&AA(2C0Jf-uYjaXg{!-!j=Ryk3z(t3 z6As}+wPjE>3(`cF-dR!ltKr&AQIk_vHE?lp2DzWv`kZINSlrvoQ~6UM+@}p-vZs7Q zdv8R0?J9KabL?XQDT{`}yRCO^7ZsTit4bCH%JeD@sHl`M6cDHKuJ>UTw_P2!va@}g zd|{WOc>?C_8cqj9a#CdU-mvz74A;mfpxnW@=ple;$p2M#1LBuw$30HZnWVpFrV)>b zcn3HhIq6$(X!rrrZS%{|EzBLu>)71z%`%*~30}dd^&Jx>>!8d93~%CO~s1B?J9rwK#K z&0~JP7NwHD3M)=>;V|bnN)c0-2iDC>Kz(L-h{(x2_HWy&lb~3qWJ;$k%G)iNq^b*d z>4l$+@bvoiRx+A(XvvhfK$c=lpO(Q{Jm)r>pBg1s7SnN|(Y2Tss?H0u^7;x_;&Kj2 zm+IEahLQg7mdf>je}<~On+5op zU+Yz{{}N89Va8DN4Gr4;{#)~5hrX@1JYY*yN1>-EnkjyEJ z=K5~h5*Z4v`O|?Q4r1kXcbTP~jt9fOdCuIW>+f;-m0GHYMF*S{)IKW`6dl@+Kc6cU6)}M{EoPZ&;VgJ*^A6|{sELn`&JJ|8b1fo`xq+^muceSkDUw~< zKfwJ82w4x(R~`vlpPG&@l=AOu+61LnpOy@O_@T>8IGW5*fQ|p_%w)^RgWQaA-uevL zGTJ|{q~!;va6q@ycN)c$4mdP_$3^{V38{SV_x@5r6jnnN zTgze9{tsK^f!4%P0F0H=$$U-*Ud~&i@=lcn79&pUjT`z`^6e)5N)+%?Z6CE4c$E4@ zm{&6s|CQov&*d;y_e*U2a1y>j zc~y0RUBYM{{m;=wU2V9_Y^I5jSRu$L%zj+}n~z@d4H8BPXzKm|&D=laR4gie3L;F( z8*YXmBSrQ`=EMG*%tryhA}as}bFQ#J&M^dykoX?{q8G+=Te?Qke(&c3lT9k?^+u{A z^K$25fg;$BO#-z2X*rqw*-C+oa|GU;q(`B5KK4Y}9yh2}|0F(T%M^rqaPmVYnaiKO z1otAhtpA*=N3pI(l1<)+ENxopn&UtQAK{^ojURgpMM@R6)+4CupWCZe*%iX%)#{yd z0YZ=KJ(QXQfG2iGxoW!sMlb#|j2J~zzqTo43109wi7`=A324NRx0u!9> zk$TZVjQZz3vYTGIVgGMUXmD9*3@h7{O=zdLq#>6;6hH&ovFFjy{d^<*LoRU*TI*nc zaT95iWXtaG&KK5rHNF3Sv$f)XQTchcz3WrFLrvLrlF9EpAx8v$+mw>!17toeSxg*! zWrm+Gi5jy&H(ehe*G~&OZUsbwKFGeG zvu4$u$LNgDsj+16u3!Vu_oPH%h&p7BE%JFoF4s>iPW7>JoSZD5ge9+JaA|D11LPbT zT>yl`Ot?Ml;R00b^N-j+n439FafSw>uh_e`maM(lk7S^(q-#Zmc+k}i+`w*4<9L#* zxt_YcCuBNNXkdITescTVW{jdEK36*5B-^LCj|)T`igOdbv>6@iN}#FTxYIHIT4mP$ zL_O3$ab3gmVnBKdL?R6!W4_#xBWhKTFTyH>f+z%0wlIK<9@6w5zt&F$Xdw%p3~91} zi>4p*b2&h)HJ=+WE_CH+`X>++5zM(2@(L(($ulz0Z9CR7k5TsqTo7fH^6R)MBz03o z06x{xKcCR}KJsrMk9Xdy^7gQ7S*3< z9e<)3_n-CY1i8bg?Kw4KQa#>)yQTA*D@ac{DE~7+`8TN-(;D$lwo@@!*sK@s%pnss zCnYuJ^1LOq9j8IQu|zvSzKlG360r|^VNT=TI%0g~^0B$uwAU)_ zY`=U4#imKxbo)?NU#EgFfRO_x205+OxiVL-;yZ7DY1zOI(7>!S&!Xpp#DQa;%DYQj z{t=ae$aSUc|F{q~Zx_OILazvs!vPyFDB9=nprZPb|Rbq)YKCpyW@=Y9I) z00q8VQe(a#{m*-A7)R5iEeVkFy;5sd2QU)@p3TgIys8qi zLuvgBtlt^oY_G$izCB%$gXIwyPdFgd(+f;56CeMJIJfDIEbyxBDJAt7Lj`UwF}Zr^ zqsz76Tz<_5NM_OnIfIm}?@jXrk&^_cB@u|nb@F3|Lho-Rx_oJu5FtkGifd;6wJ5vc zvgxE76n^~Qgr>?@qSlk7yyLRRTBH z#uEVby0%A{B(2syn7T2mV`=mCb=m86o-?}M=IQ=BRv_XTnM1fLk_1Ig^q{}DrIEt% z7Wcgp9m=A8={KG|j@&C$hM$#>42t5q1A&yTQ%xxvBvh`jtVIiju@=>csBrMEKHpf% zeV&;gy*0{amk2A&+?b5xW-VR!vW_}#vaHNcwXCF1b@`srY1)c*L5rN`pTR0AAB4=~g90;f3NAI*u}=i@H+CQFTQ-cAdF&#ZPe!!VS}Q(4Q)|+~ z_><|}sGnU)QZS0Weq>mEGC^X;Jouh`rlAAAfI154f+-tF$@g^89ZTEbU~I}a#ZhSu zYTS+kc}+FYcKSl{!h##3YyR;JMX`?#AOQ#0bMXUe3~+J`5+EI8!%r3f!*#)fj9x_F zIZS*NILhb;$?jL(YK>%vkK@mM1Y%WqLin|9*}y{sY@oQ6W&;@f9L;%oAxi7eSA}K~ z{V=KBa@iyxQJ|wfZ!F*dsxl7Imoi|W^0U%L7Y-P}jeq%55rZm(e^q_S$3y)*O=(qcE=ko*|@XzfcJTND|48nhMsl4B*@%ep(i4p?6?~eE#zyLgPL)At; z15o!c&~q;WO7K)D?5_o2D-~0Ccm*ghu)c0$_sKE=Uj}`WlSfw}%=OlZG9)38M-cnc zIX+mQ@Hok>LjWiuCUd{yAUnL#{jrGyhMtHQxsaF4m-}}lat8-c`Fxn{s)hjSA_(rg z;m~)GNvMeU&~U(^g_>)}GQo6~5bW(sq1F7iM>uc$lEII>`tPx;r6 z26eQK#UYjC&Yxin?g@bWm@~qwDW9r-**fo4fW5Z5^|0X610qvCO_%D&5f|G`Qn)F7 z66rJn$};9|3gGEwvy~zl|Hf5*5bG(;lKq!kt0NU`3I4Tgz zqDw0D&oSVW5n$|DT6U0_m44$&$lG`EgTv6DR>OHnx4aF?b{WBd0e|$!hxu~boz&Hu zq7E&LsVxQ7LHPD(BPo5ztX2kIspJ)gbWtz^Alc?_$y*;6xzs(Z;m30BKb3DVO17-6 z!TLUP2L(7tRyI^gNaCF;82bNw3XVWUdkaHc)igl6Lvir7u|#_p?YQ0^tJUdF8}+0)wF)j9VJ69iyv6u}J`8{I;q#>kURRDdt9M;I^-_uMek z>RVWLt^3IPU>FI^0itc&@h_CPXls;!Y<;peBna_Ki1b!JTH%~t99KFIX)$UnkqU9V zn9TJhxlBC){qKhdO8Q&RK?P8I)^4j;(Ol_3uI5(_{Ri@H?~4@t$E$hb zs?W~dNGy^jzvI#+u)iG1NM;21AuOOF2zpMyEA>Ns3z&q|E3Klfay9p0WOIAfuF*-u zN%JWg?aN4j=khd0y*D}k9T@r51Su?GyJa13kYBJoV(Y=YdS`z*ywhMMN^kHi)acpd z>ow;OeL6KM?PgAn)#b5sr&ibZQoHvgtb((dZLtLM+Q~2x=(>(A`REa<+^WS(I%>&j zBV!hsw5NPy7c#p+dQ;fBQW8apr+tk3;k@HD$-8Lta6hE$`(dNM)6p@MdWL3XFz$>} zxb3^!=29o}>3$~+m4>0P!$|np&^xt)2wN`|nY_WE@VQn25^;Msy=F|;?Kn^O(>Hc# z3m%P>q(8?B(Q6v=tWihmex}7LrG2U?89B1Ht4xT9n`Q5a-!(vXg{?V*sh&gazpE2T zrlp6ckA7`)b|xh}+wstP7goPj6xK(DjAUu_yn3Tc#P!Fx{-ux^L)s*12!Cp_44)`A zf9hdXlT}hIk-Ac`Kh6NHx?Ui0k3kE*@`IV@Q78x>c_z|-Ak@7tONb5 zBX*Y8@k@Koajn98Kht388Ce>xG%9=?o4(Ld9?{B=9sBgU-R^R}uhuVPR3tb0-d-M2 zzpL2P*TDZL-b`Ta#R|o4ee_~|Dfm%^^Ukk1GOk5nb17AsL z>hsxS)2NDCUmHz@026vUwQ9_abM>swVR?#n4pOWXV{+ej)n3@{_;*&+G@HZ++DBrs zCA`5rYR2WeECI6^sUS$EmiBBsxPztOr4*5Os)oY^^mwAlv>j-gc5o2Gs!0M!C9EPJ z^SoE?S*0l}ea0yg%WX)5s9N9%wkaqTwKdlm5x1>BwpGx8Igjd%2^eUoUFr= z2;5oJ@bOq|j6m|8>Om9ylhHf9QfF<@bC&F(lhjmh9N!P)IDw@Pc4^(EoxESXG_5k| z!sXk^J649Mn0Da5Qw_0_;@phR4@2aXV77uYm{kpmq}w1RwW}5G)hVEsHkM zobusrVcg;F9!~kvrcRDJcH<>>Qx4Ja0I+X@PqQ#54incnsK0Q|rJKq)-2$a!%0tBs zAy>X;S|MVBqR~Wjy5Ud7I*_^bbSgZ445SqYES(`v7RVI6pBow}fgJKDEs>(CUoEH> zGqJwUq&Ai0U__PqsEMw;rflO?`6KXaoYr+MsB;^T|L^rdxha zCJ~#8V%eVhF^4i+oCy!jowbvkVWJ#O4HxDV_Y@fCw7nSnT;(nJHhk*Jy|jEB zX@o{&Q837uy^)am_;M7|V#Mpo!Rtv=c>JwuCLrcSvP+(t&oeCyXXe$frb289<FGB5q?mT?HGy=daYCR-R?si*<= zPP>Y3xyE{*@d}>v#rY+DBF6>(~n&6602mDwI0 zMEG8*)Haq@kI%6V!Fquvij+(&Lb);I^S1vi|clO z?fg+#f_jCNeoU&mma?1K#p98*S&8~waj%Nu)*(mhg&n%Lw}XgiLUS3HlH<@Qnd{WdPYkf}p|_Ei9+`x|9$8 zGdt|S8u0`V2BE+RIky73=8+qEwUr4|sHRtIhBT0(mXFxkQtt}6oQW6e?ac6M{OjBI zs#k9!#W-PPqR5}8l@U9_R=*prOiT?i1Zw<*tV%;e${Nvh#Uc<)qQ8G%q>Rbtt^FeA zWBY^Ad-QzTBTuA;IK@<4993 zr4lFX}*QSBdk{E0EvWnblt(1w78j zO)J0n4x`ve$ZUoocB{_IMkDi-k zD;Q>iH^KGj;^1}Q+EEg{aREJmu#=96tTXe_- ziS>H~XL#h_slkitlG4aWx>EfZvq{4WNP{h()miCcxF0jo zJ$C%ngiF1y%KM!P-7$FmXmPEtKC+E{@rnb40OnPiy^-AW%Q`k>R4<;H)qUqym({^d zkMOYW(?l?6i9)QV3|v+<35G0evj5_ArE6iF(*OQ_*Pn$0xx@!UmP@ynM?D5?^>V>+ z$K*O3&t8v2+5tQLJw~dU@#~V)J@k9rAgN!KY3THG1cOk11zX4b;yOyZ3Bkf3i{k;| zebmpwr(FF{R}gV>l!n#P~4Ktp4s?g)g z%{*v2>nKxbhCgZs&->!2qwpYS-<}r-Fz{6G>aoxvrrz|f8M4Gnnatq=RzNrV_cX|) zwWImkHkUb-R!Hb_;tes(gX?heOpSGG6RDwZP^v0@#D^hIS8M79Un>w=j)hGZgx!K63*0U8-M zxV<-7h3F?#KG^IKXAme&5$}-1f_=HUW3*I>G4h)#6YG(M0*Z5~0aexs zz<6z#fW`8+PoVm=?zna53bQZ6X3H#Mw~iIuIc*n9!I)Sz!k~ku`Uo^m{cpG~*)p}V z_`d*V=|m9Pe1p#1@r>T=(&|E%tFHwuPL3&u`{h0@BjDPV;k%ydt9o3W=~PfxDZ@Im z+y)*bR%+g1Qe0a7^2=?;xgiF48QF}@aMJS`E~jnD(8$H%;2~1%>bo!>#zLUV^ zRn-e0-N`5>S5ds^t(uReYBv}J40}PiOKUYdmy3sMvXYb+G)EXh)JM&M5#!b#yNp6t83ce616P*4mlb``BbLiRCA+(kLdF*?9 z@0f+YHfh}{sp=ks{%+l~5p=}7(a%(mZ714h3uL!>{W!r6^6&wx)RYHFzbgun19|+V zbz~)ekva4sO0tWhj5>LVjkqEh?ppEd1jB`R*RWDHowO1jFl$&dl^FaZw@$q0I;jRR zzHv%heqJ<^Dt4(Zi2|{&DK4bjAbE;Sqk)k>>GvG7u`z}uo~CwSJ>oe+IUr zkbSH|)Evu}>B3*?IGrefg8-L(iEteLjx?daTl6xSg7wuFXIjJWt%F{6L(etC0_teZ zKmJVC@6^6*Qes8s19dcHk_C1!Z58f?a=}TBpE4$WRX8dh(6YrUbVqUm+f3BKE^Mg5 zmqLyg*&L$kl`=C`jWT_)LN6>%LWTT4K5PxMHmq+VhMcAB#kSS{=(O~qk#jkasq};x z=*L&Pqq8l5J!y6L-hKQa?@QbLCl9(EDC_s$Cs7}^!nDTcIZ<4g2|ht#0KGSq;{VY7 zXJToBVZwD6f*w63lUGh^(K%{;-!1g`V}CJ(DBgnSss@q2DGY`JktA>D8MKVaS%(Zv zCV4MFm76J1fRJnq+MaH_C5)8lidEvn~I zAbw5VKBoV*J?J|~O^FTb;n5q~7|0}FBH|5jKj?@EMwuKK6k=fD>rG)3!b}#*NntXt z+TbMKfC3~bF6x}f$Tz3332+J>U@^jlfizk_@plf)Rh2fP16@g+zPO{b!ik`Vdc=1B zE=!2w;y6S=CR_XyAXpBx1DjM(<&bJ0?EMAhe7i0}zJqpgP=xULAU@}`Lt4XD94Rh4 zDDBpT1~u?Os(;oadWt4Pf?6NA@`P!a=w;pD@S_EDV{Shc22J*&aj*ox+afkbGF-y1 z3bMGZGQltIiQ`IX0PYZr+peZdVqb1|!mA!vh6OavnD5P<*~6(h@XJ9cnueF|p$`$< zA4tu=x0ZoMnn@63-)GBc@_C>7h=4NjL{z(I0qL&XGg!Q}3k$2gCtp4rF5cE~k{YHw zNx#IJe|@2>fj$^~l*E^NFhlkOCyce>!bj^=sM4xk#m6bseaR&*SuVJ#g zvcp9ePsHt+CY_8RyQB~o z{Shh2{VeHlyLOoQLQS(Y%NECUy69YU@jd9g!=1gSfT(2M^4jeh;cI&i%8!<7ds0ja zd1vuQN{y3JAiw=Ji*%zeNnO9}*akECVq1ezf{m_aa(3!zP<-o1ik;)!UY2xWjVT_x z6(%P5t5QIW4_Ne*pD(%^q@k4s-#(b4z`t1lObU|xBfIDL&SJN_cnAJc`@;$20Tg&h zuU96)C&Ri3*IMO$HL95?^C{`f-CzWw&q>+Y1ocxYIeWxRKYjVH%!3H?vFL z#k31!md{-gvFa`1TBK86nw&;N`*2YUWAv-)l1gtTu_~5C^W9j8ms-xncdz;F$t$vp z)x#5a=bbfmaTMC!pmc}R@3$CGr5u~D3M3m{z@q5j#OUK^4DJtjHl??8Ee&4Vf61ag zf+NvXv=v2ZFuPNINhCgA#Y-}tEPLAM3;&g-;6iwk1qB9O3e#z}J-O-i8hiTZ`_%bh zwqpoW0bM%VPB{H9-vW-d6^VI*NpVprcrKPYXp9TAf$ftzkG=b@|-X64o!j z-+>#VY0f4(?~sPs`YSaHj;`V>N+#qu8<$df>cr^>@B7XWc=#AnF!1x2ugqi#U4?`p zxg*H8NOaL4!Ij1H_g?DA=$W{x5aM!OmJButNXe`pt;6|)$ZF`|^YnUeXC>7Nh7%nU zt^J7;I+_M!o1-f3bFzjEW{?(C3w(lJ`6aVI>a2K<^`Q%EHkT-b?>mGa z!<@!jW1v>aCNgMy<{EqX-4cr2g_V9Hp@Ch{(154lRvU*_cm2GA=8u2n%t8&M+Fl7+ zqLra}CE-D*AM3QB zH^+mVRrOCN?a*4gXT6eh#2aC_`-Gh^z2M38SXlxMY9~LpNv3Va=;MB+rl}f^KW{RG zqGPmjsDNWVnEn1s3jS+sv2Ze-Wfn%=v@)Iw|6FMukHf}CaK^>04GuMNn!@A`8J7b# zO}Op^iXN#icOl9&>t9z(NBGY;aryYc8B0Xw&eF9}{M&+(k1tw4i7@?0fp|!H_klU*s?8C$%9+G6O zy*Ag<+4bP26!aAVxrArz+svEd)9(%m74Lwat69DG*M9o71l}SCVV9vhc!1YNjAJv< z@uc5l3=|aR&;Ar7D%@VV{w`hq8)W-kw+FY^=Q75C_3{_L4 zp8abO`Kbdw_UkL5_PoErt_m6apVvi#xn{amCGiCv>4EJ7$?%Yom_n+r3&pBI!$j~0 z-1D#XV$kNnd@@eDPY+naLW-`Z>hjU0M{GIRC}bN2jipvG^d9 z4D7JD9ICEPKu^yOUOA#@*P$2cgzU(WIogYNq_YJDr$b#Reu;DHn6tA&!V5Z?ZDNHz zEuO(eud{ZezZM@UyjYe5$tu(zF;}`4SK}BM+Rd;u6BFh5e4KLVO}FV+BGb+2%JAa9 zJZye1xF;c9$xEix7_<25C(5NnU@47hXvg72vh~+uzd*PRl`KW z`z%?SCaj~(+=S;5xwjHm*go9M@50xP2c6t+7u=cEEwM#lDeshsewRs4Zhc05fU8h& z1Ik?1Q6i^B2nl$ICS^ojWPq3K4#{+JC8y7a?>_mzIHPuUXNCl#Lox>|nsuaOMjm2c{_TBVtaas=emJdKg24{xod1b1!|> z@a@ac^^;YQwMzZ*ht&+6Us9znL+)RWHwhuJWZCVFBhb}K3;66lX_zFhw|_15N_wSN z(2q-oNoO`GYr05ptD zSP@p&QesuU@K=Z9B;9m8neO;Zume@7eN`L<6Y=DT+o;{=h4%!0Z_{Gm#c7%OiO+=={g7g!x4e0%qfLTY-XG_jI~KyM$?U%%D=PVLTU5p* zUTZoyr$JMxxVrwdAD&^>C9bk9dq^$LN#!_s`#5c2xBeCwG6pn%Eq_`r>{`8}z_Xx1i zA+7>`i{wQHYL$IqChK{B2E)s{)JH%;!fC0}gYm=Uj0q|33=JT0ylHNQYskRssZxDc z8rVO*e@X>8*GwYzzMK@OU`ow3>)@>oC+XytAWNAKu&|;418tJ7Dhv)duqp^angS#% zh41}h>rcBsz)a2|1Pbb#G(4u<0)1%rnG*|e)dG7<8P z0IXGCKau&j%_dG!fkrb%ZBTaVzZwa*K{D};R^$fAwf;IDn9>Om)<4%kCkcrqdO`YU z#b!~sWyf|AAMWlgrEC85a>`xl9~1YjU-?%<^Y|bTa?Sv#^V%YT;H+;UUH|hbd_9gC-U6*QI+*<2lAVi(6@B;{)u@T+Q{&-3pA@ zU*C^^tnC@2Mo0AB9c)d8^`r(AX$eR${`RpWzTKLD8#J^ym8ePol8OK9po2Q6t)}aC z@$-1kP*iQm$|)UDZE}~yvQ`xM#*EG~Gr|{aEOS5kE&dA=jFjc&rWPWRlT|TNjo?jJ zm*vwElT)gyUz2xbJW|CmFa2#O+@9>VkGnN2R_#*95(3j+3kVU)x|>GrdAA%r7`3RP ze_Hrj>B6t$9XBO@pf%xsDApK6YXGYqPNgk6p&tR4ZM6%Vx=__voWuq{s$1a^_0ZVy z;U9_bB8R1^Em~V6I(*qVuckizlDhL?=4NT6N4@)?YTD+*h1&Y&$uZ8YF`A>r<`k6v zWT|GHOA_2fLm!V;=-cijjHED1t=HDW;3`6|#(Q%W+tu|tFeAKg;;xR=6n-4;%h9{K zt@3^No^#f;QwLLLHnLeO=fBPFcYS)UNgnGVGxKM1T8@;p;pvx@@8Q${zMvkdw7;VMbuH~lX?E-5U z)K3G3r3GMp{&>`oeKBDn!YwPbl5=fk*}J2Zirn zl_rC4b}|#c-C!w$&45Bp4I={QL|WA67dQU|l0cURE_lU|w{q5ztV!q4c@+4O39lPT z>Z4wH#N+y{Lu%0>{~z|=Yi~E=aOm3~6h#ysVk;V>pSLTw#sJYuRzm(@`&5qU)}oqG zJ;J~&dB>MJ{JI{23U%W|TcEYLzvE}=OksDtE9pcyLr8xZ1W4I*-Oa2QowFx zd|dfma#FB-pIh2Ig#T0QlWAcAZEY71&A2zas=IMH+<-O7hEU?%?ZsL1PS8s9F04!p z=M9O7X*AzRmX;cmL3{uC`f$(Pl26qka?h@Psg9UdB?(P~S!GlAP zpuyc8g1ftGf&}*raGHEsYkh0g-c{$t0n!$p%q2KKURaSR6S^U6zrpHd?zjRY;ElOpz z$RjDGaEDu*U%V537nNf9$gd)q9sJNHM_n)PoQg4sJlQwlxQuotVEyST)Y0U@jxp(t zEIwit&@1p&9zW*sX%M(lFsp3-a7?(T)YjHCv)nV$;()giPG6pVTGXjb0R0{!y@nuGChi}Cdd_@;-YxszQHq41`WNAL67)5a?q=XVNVY{(s)O>2&_y?te`AKe)Dqo^!2DawX6ycM9!EBzG zueLr#5+4?UgrXi6YNi?7nhIVKrB2GrAM?adSG?|ZJaeP2C8?2S2Zwcf+7*c0KUDXbPOQsy#Umwz8t&In6<3x0Yh7aXIVr z$L5s>C$&g;zlyP-(rv!qvzLf9v5G0T87Dg*e$6ZAzFzGVshuJS+*4)KU-n>`567(P zlc2S(o!rT_{b<1*l&?n>?poc+u3CsuO+`)Ik;oxLgAqo@l;34R6TSQ~fWNqrE2{=- zI6tv?B^KY32zW3I4ouv%oXfQn$V9<#7OC?4Fbwos+53<6?DX1C*MxZSSxVYYDq%w> z!LhA|t&G>42W6ZYxE_kvIfPh`h$SMg-Ld0OZzXV^m7}zwTvF6%O@Ukax{PVQBsg`9 zcoI^EsnzvlEBHv_2Y$6#d{Xe$ogNfRw<9>Hca|w2VyuRB@pNm@Qd4GJx=W{_yBM!A zS^RwT;f=9;e-N!>DD`J$Nvw?HU7irgLk0LMX6>9P-;Wbflvv8ix;;zh$W(CF=4@#O zDjE>jfodaGbf2>GL?D+EgaJy? z_7+jSSI6aTlxfg)-g|mYz*lUUDs35|ANfym0lt;=y`~tFR%}W3d+NSC=U$_~Y|xAj(mZh&SEH1*qV4AOBFx?vT8 z={{B6ie{o#bGam^F~@T0;7QO}(>EgxP#W>YqCkLiJLzKo>YOSf)P+HIsE8GOThhMNs_PyO>{0-n4V~WZPV50Qk*ZXjpw4KHsn*fuqt<1e6K0hV4L49yT6*B zY0`d~H+4JNSS=>xbBFJHJJ8}g#{Tg`1z_dN1*Dxe$pErt4isZkf+2DU1Edh3+H3ob zuUlGkl|~AH-wL^=-gcjpT0E2fY=xjbi7gz4% z!j}Ergo8cNPmutU`yu2hUF85$uJ@uZkvr!StQ6dq%nv2U`kF3IM9&E46 zD|`A@Ezi|l4>u*~2{w7lOUx+^R;*dUv*6PZ+H5IKuJ&5fyXH&Da0e;AmIki^;icJD z`r+Q&AJ|X|Hbv^!`ekE^s5eh#!X1k%Sylk72POm*RM)XilEfU)-1w!oE=YQCg2jtG zC2vEYpV`LjKGvZ;(bD_aMpU)}qR}esx%EkXDo*fCoK5EiML5}7x)^})OzTLu#M4CI zo1d5#g&bgMCLjpL5(vycO=J%Z-de(wX|hwDr-n}7-jJpCIvetMjBruJ98c$F;wu8U1dP!ZvQo|qA*wm zt=JpYzK^fSPu!Bo4G9viKH>afvVC*Eg67xk=ER(AP98;b5(a=?)?L|ZfZfLncN_jm zx}(hyqV#v+Xw+*fW{q#ZSOVpzaeYg|Q>FR&`kr&2B0#-K ztI!=QuZdz>Iu^OI7L}WA?>U&*yz2icO$rTl`1bQJ){Ix-3_&Y%-DjX7lkOdsA>E#} z3$R0sKnbhB(-4YlNvF09Lw1rt4a)jk<{16Yx_9zL9VOCbvT08rB~;vZ`jJ|w9iq9R z%D1v+Yl_ilqXtoTHZ=7Iqt>T6r*mDzZ>I-H;h{Fg?_D)Ky!k<$$1k(n1A-xkE zypdTc$CCE?HI7ts=n3X~8{fpOK4N+URg%|w`TbNzyVUMyh-zVNLk5wB9A{RD-Zviz z@e2JuJPC@D5MGF?*gQGaRoXR9stv7??yNmty zzTFjXPK*g(X?8yp;_yip3YyJFP0jbA&(tcy`&=l1A46zX zxz)fFZI>`^MTPfLcNLWf9uXc7ugPj}T$J=)mvwtXbo|{-Lfft7Qq(r~_s{*ub zE$+(O#R6s885Bs*Jz2f5ylOV_AEssvlGIl{>hh z@vmlphQQ@pS&b($@wNoFB~;03!czZA7<5bG`6+v*OcG~(i@*(B4t z$YTSyX(C3Gf&fxJcgJL1cRND;e0unPm2Cbr9pr-c_CB#&s(e(w8d`X}_Z{l*LfR!X z=^jp8tCw;{-=~qX_EAkwyhCB%Aqc|aPzyxWCr1+qoc$DJs{7H?M4nB~w@%VH=gu4~ z6NFgWU5R5Q&&U(3A*!@wJ7BewHjdB38G-p-v{Q|!)+>oq(=#D@Nx?*Y2r(*51&+Z) z{h)w-uo)SHB{WF6SkcPT4OJfMyUT82&;_MyU<7a@39`p6nZz0rp9B%IqS)?6wLEx9fjIP3`nzFCRB6 zN=cjRn2n!%LVlNbnFySi(F~ftYakbK)0-*nvu!@8(e11*tP*n=QQGmRZoIS+KFtn! zo%pRUk%Y;K8c(y>9V%Fvk!h0rcL{zH#yP}Sk0~eLVNbN&knrWYvy=av$&vQpLZg>t zA!8)oSK1LM>v!~$P8gYy@z^HS5jp0kIZ>9+`0d{9rJNGq41VCgp6bFLL1X#E!WtZ`8d*J zsarE6`}>K3?lwNeWp6fO@tcQR|M;^Xr)^K4ihh=ngyt?`Y?53&pbM%@PwdLx-GQ%e zT2$-WG`wnMjYGiiqCivRS!CH9UMkDcvUw@mVP1TXC(ux2n3^Wtq0JGMCFYvG`LXp8 z6W4-LJuIPY%1-e(Ryr>kaqZt81aEgwe0@G!8_&mZqbcWmdwF?3#UEoUHW1f<&E*y_}ybGHx5^W92>y#e)@*XHdZJyU-*y5aM%)R3H_6 zVpPTK>R^3^VhON;n)sL^9CKlgi?nwOHUg4W+%KR}+g+l+L-v*7esd%5k-RPO?9F^* zT)rJ{@&FJ+OFoc%;w$U;ax!nDi|Oq5jVquY7GwxBA^Kx#5#50VZhQ^49O)$>)hG!p zUi$^m+kW7sMFzq*z41S!UjXnL`O}7k3IJ=rPHZ4@01*{@R3EEw2xxT`i_?(x=U`6@ zTw5)P%^8NRK?6+jUm=jdU8*IEVHLnw5DZ!V9u>$W!j>@-Bcy(KWVlJg(k_<9}cFLOqE76~yWnc`?JgY5U407~r^>Ma0lyfar!QP{;pH zhCFX51_e-aLqomg(C9!WTL`WcwfFR0m)3+{FoCnMN^hwMfr_en6V6`~vi`PP5lIuF zm~{i%KLtoMqY8HoaXh;auoGh52n!h6^Iz(>vxY3u5p6vUUX}vJ$1Mz$2n0L??u>Z(3 z@8xxQ+Hyg5JWM%}{?GRKA%6@LV2{JDbc_N>85#$m+|PEGzrB;@F)RSMk!5LBa3z&s zIR7kp03k=0owT#}M;w2rNVYW|SJ%G>a}WW02v{)j@5GXyuEOa+ofvDsf>yI2!@u?V zqZXli9M>O_>Nz#opuqS%=iYxup9;6AmfGzi{$>{W`hMxYO#)U`Ok`q^_>f94a{u)q z40J)ktxyDXZV{J!0%l|ay3b!K&+#Ye@J@FU8-&&fOII}2>=2WULqO!&gPD7*Ig+lj z16+`*HNu)SsJAa#oCOLVdxVIF1XnfM^hX?s+{rrLJ?e<@8&LB_50~O6^rU@*z)kZq z;(gaHQHBO1AejZz1TZ5)nM97wX%;7+NY>n6ZGkHh;=N%2*`AWrEp@;2TimDQ z-C%$wT6Ag@8to?!My!f)I{ctsJY%b6Bwd+{+q7mEeW(!4T)cCJEemg2kb3kSruo#D zcDxg{C@)56DfjjR;l970T|=M(29=w+NAxh3WK}zV3F_OSx+%{pY$nk%1&R$kwWEh) zhlnbsN(FrdD&yRU-EGV&AcbncqLUMdq* z{?_oPPpTYs{0dD>4F69Bxb@&uzzN07Ug2p`!91+{*(Ah#w_A^p;~vvb9yX$qQv#rW zjM%DOcVoSkF@ufU)SvI(3aq8>QVfC>YT^spJ>|-R-T>ucXVUgA8*{7{_L~*Wj5nMs z`}L)0KjG^E(_8pPyOcx-FKSf}xA*>VF?(6E2iZS%J&?XHx;7_D)1V{G!XrS#%0qA% zQ9$^FkH@ZEW;d%SI z&R0yCFI2DQr;+AwRs-3H%S;x$biy%pPaUIfNI%8M{7fPAX z+Zzcf6v=>~5wD*@)HkbLZ3f;`Vn}Hx+?rT@_GD7;Vlwb6pKj-d%pz`3(C%kV|4gnz zW{i!G{kDqlA^gMvgXOy8gu}v7Mj;hCNf7b~;VUD$6bA6AT5SqsTRxk$pD1bw^WuS7 zeFShKnFClLjaANv`m6rufTcm2L>kX&HK#dQfPXtuOiRQHY#PwDQo%Rmm`=1ALdDOH zHRpv7x3e-=_S_HpIWmi5STpiz_Te+KJI-cMA*h*zvDQc>On1gi*H0tF?CK=2c1+GUX;%yqO{YpBCEy2H>4z3 z$u}sUYw~ARyDZHru~-%Boqbc)vUTyuUN{^%*=3mAZqAdya~6aP!H)h6Z&^PDj+FnT z=U&FvJoKcUeq2DtcO_#tu~q?;JiLBG#cn;-Y!s9rtUvswgzB8C%mU#xTL7SjYl4pt z-kHOwBVw*I>w}`PE(Ox!&Civy)C^OdDtA)p_nS679qFm9={(IMfbGNUz@)PoklMrV z5-FPGH}`M9Th9?OP8T+1fPmo8uXX%t+W{}O!LDk-K_qy*dOdRB7T+oeB`RHamI0vv zACq+W`U^hAYej+axJ1&oE9nZwgx1>7Y{|Z&RrTZ>BG}?B!4U6;xpt8}u}HMzuFK{n z9Up2^Ts5MI*fPzTw!@72z3Lo#m9lB;v#7+-UcfizhA+2dS_Z25tKfxG_9gA;1p|R5 zHc#smV0(hK@^O?@cl04ppuIoQMsv$-uf4os?5gksK zI^3Rvch1rk5r_UVY?IXytVv|?*K>NoPUV@Vm|D}m^QT;Ow8Ood8K;VQu?L$03dOTX z?ecQup=N5bkYAXdvNyC^Mc4@h6*nI7F9X@2m?y8&R=9}PacY$XN{*^Nt-`(=L%Jq) zlg8OhV%0#&z2+T9jTVLh@lYA3z~c;Y;Bhl(9qu(a90feH!usK++COX+=-${;;a>fa zwA&Co&n!K@2|dS98?ZfXX1&k8%!$^t-!s2Tw5+G$pLa}dU3dozkk5gAonoad=@w+x;efQovsAW*WQP^@NN)80<-QlLt1|e+J7X14+6^2?Y|5`8+vjjL0K=E zC|?+$>J@uXRuVHZZ{hxW10*C5P{lTjBAEVLtQ-jnkqDjx!Mg(3W*I*n<|(A56$t{dF5 z(fi{zJaXJj$dnqsKYvhTw(J9Dkwx%!V|iw*k7EVosK!M%ZqCk>g_V!x@VV=LM1nHv zow5Cuf`4s^r&?0P*(6|`Ehv{AwP|O3AwGW6@%O7~5}j3k#OPmDU62@3tH+_f-+ueB z^G|D7M&<;N}I7?-iP3* zHUP@VnlE$HGgMmna9;-|`t2ZBTI65A^Y(Wo%3W?Ho`G@qiM1rT z7IYrnDy?0z$>DqX&wdQFm4%b44k5mq{L!I4YWnQzDI3^)I`JPaAQznOz;E|YR75wW zM&J@H1iNHJ6p%d9!OymR;Mq=B?a1at;SU9>BH6n~i}lmX`S^z=R!ej0w^QhPAcR@v z%#-F6E!v-IGg~?wKJ-trXxGQc(lxi3`_G?Y~XQDC}1vfmG_C!>oG@qXKp7! zqAxUETjN`l<8%k#=WPUD`m)I;3^~z?)li9C;wl*5{UP*7X zZ*+4&lOcLJM&t(!PHHXQ*jm#9)jGry2c&1KrdlQi;8q$a@6p&~GA*4@);a*tpjT83 zLh>O+yU0$?Z-V2_UKDMJq3O)~vl*a~mtuZlu;lW336=I0^pVkZI0TXb&Zj5q3?4}$wejU&ahCSBvCdmOFd08>ea%tc_bJIyW}!uR zrBkHN5`2OHg{EnuM$1*#thQF78Zx`=cy(q>^$XJdp&~;qPUcLKIT0;d}sfpp)VShAWKb?r;@5fB^ zyd=O$P4S)=$q&ii%SgTg58Qotr~8yreT>=0Nhe%R2MUh8Jw?q|ZhyM>t$|QE?0#Z= zX1(Ee-Z4{-eqSjqMq-IOIRi~UB6;~g2`*xemluOR+sE1Gr@^dxPj4!(~g_{D9md=gVZ1pmG$Z0G=5i%Sh7Mkt-o=C*#BtP9~^7L_lI;*RW{tXmp*hq2p zPEKB(n>t-qcP821q~J5Ga2c#8^v(eICxx|{&jEvLId@|%ywo`Gp~Hks@Y5j9&n1y7 zDPDq*%5R-W;dQMPGt2j*kB1Hy#+f?vxYl5AA3L?m;^cgLAKud+^+L2zoxA;wtDa>F zB}Z+jI19Q%7mjZVC%#mIiw?2YtS zg6XD7KC4I)iYniY4#`rNQ>PH7s4$HV?;lLg?5MuB8sS;c8P)%ItwS^5N-MjL(zi~M zoci?(l{X#1HbypyR$!N+G&+1-rxgOFQAbTDL5waXU18ssXIXtA)JUR-ryqcq*@&=L zxtUCV!OJF;Kk(8SmaG20f;sN}y@@XN{{7RwH02BN7%%^k8|R>#N7c06r&p4lFgi%x zd0jJ!k;leYv~A=-{Gjo%Vs>AyrgJGpHb40q(Be&CQcW0cG;w{AuC_mMLF+0>0Dqt+#~v=CI#@t0E}^A>rWMM z?s?*?kYn}97-LR^vGg-OJ&X#k4SFV-N>QmlZ&0*(ae)?zVQ%L=6(QHI$nPvfVQoZZ zvqo$D)+3$mS&Aq>kSf4p)>vuJSkKEV)FweKqq?{D*-kc z$unYzU_#89sehYmy!scNgdGAT<*E&!2PUBNrea zSXEqFB=)=%C2i8Bc9kryi~;5GdP)SFG~8xUkdReILX`{>#?<~EMdW5Ldhqf4gazpN z+b!rmmY$M7=PC@YDQc(>h^ZWUSJpoZx}MhKYb?`2Qy{B>Ur_}3#x^4En9J7=!pmv5IPXS z705un3Crs6BN!lh!6@w6B*0=H>s6_Eg~T4V=!W7B`Q|JDcNcC3{Al8C^?n<=?|~ z&|zI^>__Z~>7jYOVhKGIg}iO;E|iXQnHDdP|0uo6YDc*%&DyhFJk)z;b3L$rPT+Jg zy*BPq8=`bVNSgX|fd6$cg2(FuZ(uKJ$-;jl%iz%C8(1e6ET|a+T!P%FF4GT((*VHY z_q=6yFVe|b30if&88d|VHt&hoDSUvSJX;iF{Z$Y-kn`C2+N z#_c#q!#E2=5+f9lGf$pqIX~b1_H~sU-cJ+CzI8vhfFj886AJU<^y_jHp#S0$7rB@A zO(}T1jePUu_69Iq2inNN)GtyfJJMgvB&bV~_zJqm#M`od#fBQ&VfG_9Z2O#%Hb{xg zTrcR^D$qy8Uh-a3m=BmU;_yX)`s{FZyXp=3MZ)R94@+uvd>ucUxwKF67u)x$3xOA8oGX(${(|pMF%)ye9yN-EIambm(Jc7#s$VKxi zLLO(RmC8 z$qDekMhBPxYtl#4IV;zy=1aDI#pHpw;g+9ZQ@f4+;v%Pab-Z?{PztJiz7cXSOL3ca z-fW8O%XuDGpF=oO0|px{deRKX>oU2>-ajLo2D@$C&$Tz}h@en-zL(RDWAVIf@$ljm z{4$1*q8q!cE}o`%Ja*QMV=(NHY(5zIz9>>QlG;RtzuEv#W>3G6B3M0<^5)L|de+zq z61yt*ONHyYhfkiLjXKiz@#ij7p?=bX+LU?wwx^V>|4@ILW_R>pSi z%D~q*#zz~Q=TV_0M-*~wg!>sPeXktdzb5~x)x>Q`w)-BTmM^A)R)ZH1V=E3|Rr4&8 zc&OoztDD=C$JyTqzcpPTGP=^g7a($|8aYqXe#oom%h<8|gdg2-^C7`9X6f-pqiH93 zNzc4B#9>7W8n_VL&8dpS!sFCP9GYQ8$vbNv8kFXB_@%-3g~ey1BHYW~mftV(loK_X zbaydiTtofKP3DxB?+~l#RZvjfxrJ{QLp57QZ%oY!($(vW6c>KUr z7FhN#vY_r*JmH5am!P!VD7$e5p(6Q!MK`S&)?TCn#M0)?LHY8Zx1+-=yAs!)@)axf zV(CCHm7%!NE&(RcRJV(p1vm3~TuYsY`KV)+9&@d-K}4mMKw=h|BGS_C%O;r#q6aN1 zmlxZjE~sb@l6EB#`MlPGS~?vWpTcHWd0L7T&Ph53=*^$fzAUTyGOaV2RdcyrTQC80 zG{R+3)NQ}}+1ZuJ&Ls}e*UKoxUga{tcv4kW5vON|JohIXgvr7@WkrzT_xRgH4U2kB z;%GT*1TFDS5R%K8O7^{K?<2g%B1J&Ov_ZrEsMtHL`db#!PrBM=n3<*+D+|?Rx|+FB zM_$zj-)jra!&(1f39}EE%hkqD%z7`E9_XSTm`fqxDzB%FyzgX>ppTNG+5OfZ)>9z) zb8-{Rp{ZS-LpMW@o&+sF9aD<~G3FU2GR*rXvMS1MFa94qbwFi*3~Ak0Esh@$mRG?m zCzMJUYI$Rkajyvn%=WJ)BU;+30#9b?e1{tjRjN`1WTX~AA5C8N6XZLJU_;{CupyMz zj~f=XE#IY&3ka^X`7@z!HsXEJ;fWlz!bc1_ttx|W%nEQ`|JpI~MZw1c?#b9rSC=qP_;{HX4&s_`9Y$?qt3|JOh%8WlQ+?r!mhe=9EOHY}Y z#ZIIWlMXb^yta?ISR#;sQfgvw;)$^H+T(Z3XLek}f^E#E64fSarZa89{H~6u(U>hs z*Z(crMv7JD*qNt-O72rdaWD8MoHl)ljA2_NVU0=Cd(cl&pxTy$DzVc>UKyjxCn|k7 z@nFu~Gn`Gb=iD*$wHoNU7feoYt~^qpPrnoFU-xpO#Hm4`>-ObHPPp>*7GcG}?jJ>S z$O`9!i#YNi?e1ym&s_;f|17pAqz7MmHEt|lBdaZe$DT%&e+Urg0g1+RftPz89|K;f zeXbhb6i9*b_VH-Am0D7$P8uF5(lHmjEDtLAcACiG)PSS8wdeA_ z<^I*;Jq=J^@5sL*Uu4F@zt6P6Wj|J^qgviPUIIBa(asH;k=Hbn`_S%UR898!R!iK@ z2Ti8bOVI9f=W%;G4FiIQmZ6p#dLMhQyYXssgi@x`Jc8jOD_$BcMnPIb(GX2Oe=Ydo zjk>lpvl(XUtaud=Pei~^@6LUpMyU}k(*oa{tHCG4UMn#*V%;02X}oo|{8`hzM&kA> zE81J95NyyjsP`40^cwfKLKV`Jq2B#KJ1P>@r3v~9(A;&hT@S`I)px&5ode)Gv5RC| z1*BgOog`}F(PzLE2%gp`IOZOLVQ#28(gsZ5K2K2}CBgSL7ycyQF;1>2`E?+er4}8; zO|cQaqK*r)t4)Ky7q~*NJ+Cy6#S@ zASa-y<}l`D%MoA>5Bb>E1EW*BY|xZ^g`3?5=(GmGQfon1*_Upm&(PS@yig&ac{9VR zD(2#VfA$i$3}4|hIcyUAVs;XRniapW{$b*Jw3!d8x@}8VTY+p#*Y2-*3|ooV){$Kf zOPVOJp!lmFF?nsL*XW^2Aw5Z&#vAIQREe6n;Y~4lH#eJtn)GqRH3PBqUkO*DjDVRH zub?K8Icc{ApJ!6GJy=YF0H*q%oo`nw`GfqL?l~Mirw_<&l~=en2x%UiiB!9J(X9P`I*S0Hn7{NzC5^YYhxa`}#VTELojRWaZtzk=s{! z%f!Bx&*ZiT+S~?+sM#2G4NCok-A?{Es?5j+9am=K)<6DxB=F5MA^ro9jS!KI4q5(v zGQ#tF^nPlECrl*#1xj9ty4sfe^gPFJXSDtrM%7YxGL{_z0#Z>%LR6KZRkS$$Z{6@9 zm*rfRC}52fm9i1Z1h^J|4LtqR)+Yc4;&8YOzT&?8HfK;tWkW3N>@S+gA&y&*`0OG7 zqbYBbO=j}|qw{U=jN=AZm5;;4sO1y0ngGUsk?qSWq-#Ud#2S@U`~Zo|U8V>d zEXc15hum-o2LQyo{~kMVkoTSdfUc$4Rg=4sF0^SwhyreUD-^4&b2e5!A>HOy`r)n} zf=Oz=x^5>9-8GTAuY5`E{ll|+b|gJ;R68w{2Dl6;c6yWW^qJ+YVlQn`wbVKfhcJ2L z8|ahxGYc~l1=9s7yyfMneNp1uszx{N!y;fXZ&LK)E;wyt|n6g zKbRB^@w1$snQ7?qu%0H}v$yxgb(1awR-SG3w!XA4gLIt6W*AT&7Ktbj;`4kD>sts!pjTSuLLnoBgO~2+IO>yhvA=fyfOWj8@g^LLi8I z%1~I|B2=3O$o@0_Ot4;YP4Fv|=M(o}klX%Qc#2iaMGqdWF5Iu9v%vpR-p2M!EAtK8 zpjh&r!bm9FF0Ird4ROSxD!=+mTJ-eUgRmB!eo}UHU#k8zNpxr z0PgtyN#+Jf|CoDA`)$-*lGi-@p2BGsRm3GB8jhF01?r_t&RgB9To!0{m`3QnJ*()N8DFN~@pn?*!f%MNWUUr1Ga3YvuxKk7KfPwrDuLt0X)%}QZw$Hlyq-+wmUe|HHkz_kBam{l;c zDJpS@;uI=)n$m2%vv11f1()kk#VX`56it333Ut7FIyC2|UOnvBQ-9}dQ61RyxeXkl zGnN$5o&0vl?po4n^1$W52=i&Dvo(7P=#G!P%Ct*rsvvq8r^>B?N&7+==ieRC9017B z;}11q*Z;L5X>2B%s!C^4QBOs704_-*T;sP(nK*7|>(+!1Wo0m9lDla`Us5jO2gJFi z6jyllktr;S+lk2*nx9>n8Mq<|0338-G1ffjC{$;kfQ>7d@^y;!Q-t2LP$U3I=npWC z<#h}{K%zA0suPKc0?^;zO{|1Y0|3g?0mjC074hsJLiD25P8%MV;Yk1U?F4->4-z(e zF?J8&L+aM#hR@Kbx-qY+&?xF{A$v26xr9^P_`M4Ncr@n1WmYojJtFj<+u@~&HZcsm zIsot?l7_`{%Hm_TwaOFuV{2ZoQs48KUl5p)Mi*Nnr;-qhj}AggTsKJ0%PN zlmxngk*KrqxV6jwR(%8B=HGLuGXH<|4<40v3bNP}+W%Cw{u)XWzpVz>`27((0Uu8D z9T#xp=zpb<>8y1sviI=taE^+Ja#W*g=+X=9FO))mKMAUo%bq8skcPF!2G<)2u8;}| zonQ2unCw9ib*Lf}MyBunh$L;+Pwpo!^$u*;&*W@FLlq{_t@_Ro*3BXHLB+H^UG=W{ z%h&yVXfO0h--3**YtVb}i8L=d`aCQi?svsS9fPdNM~*4vIoh&*Mkktp!nkgc`w^U< zp2qrCznetO)%Wxc0+FPj(K!RBpk-%yFZHo#Wz>bENv9ShlECr2NCiC4+-%6>`t_h16VK{}SRj4a&FmQP%JADG-Ljg3YEkF%pyS z$AiPzcexYDH8W^@R6{4ZxueYJbz!CTMe4^G7i50aB;>sAqqM1wVoX8a z9zWNY@7mhg7#=Pi+@e9Dwu<3;W!mJZ9~r{+ED3ac%kOgjq;?_j-R-G-NmzBri$f(3zg$H9GH7`#MkgxO$x% z9V@IFw6LxmAw*BUYpwSpPLpmaUZo@koFa+P4yVL=#^&d`^mVt+ zp!MWob_?xwfd-{Y>OqY>QgHWn1Ge+c;Qgq6A;*bB#b;wr_B6xLOWMFTNM@j@YQCSY zwU(rVk>V0r^t6&4RscZ&= zYHY{mZG)ry!%U_%FlCGj_M?He3u)6iQf(R#*-~Qp1T-4fmVWQVWb3npizl zux{q^Rqn~3(XQzfmh%R6=qsXh6hd zL12Vi&axBAZJd_4SH(S6j5Gs>f%g)L53tz1{60UV!d2?BgF|oc;r4S4v?ZR8Dj?zZ0;dDs^w;HqnS{7E}0KJdpeIO;5 zj+K@Al61aw2HKL&r5nSi{AUg&iv*DCLAq5AIL*?UX#2U9uqB<2qcHBWi%kPz;of!2 zLACdlm2#B9nh0vs&-CeKk%2#(`K|T!HNg-Z`WqjJ%}mbCG&yYKe9O7pAi{@)y@*zy zIN9D1y?OANSna0;`FbqRO8BWL>Tn)G=VPYH;l6OC$}_xG&}3`$?B1@8%3)!eAmuXW zilk4Nlv4NNFVgt6l}{l=7R#zl%cjbWT8?GzHHQyZV6YA69Nh^5I`W$LYaPPUjt^=iXpFBGj9yXPuTx>CT1apk0ZsR#FhIhge+EaojR zm@vNMFgK_YdC{l-fUD_Qc8p!2 zIRu$I7ZugV3#x>Z251W*T59L$3mx=R=T*4-@I(#nr|Z|<;|xW?ITNH?+2w5YU4?^b z%&&UNn9=JC2aF5#puO9nv&AW1-f)j^?G`6$v>S$3>~zEh!At5G$j7D1y)5afgOOt7 zQe~Ab^levrME60&uqyU-JhO7PwjbrvN7H6T@^?0F_KE33!ER6`@6m13IFX=gdeAuN z@l_IRq{c=GKIPfj+!0w#4g#&wu5g?j#1~inkldkVNOfhU2))E|#}XPHAM5F7)gkE= zCpRc#`N->C0W&z0zoW%DbQl}RPUW2bl3DlT^gaYPYysksMWHQSsHFEiAl-7&2Tp%! zMPVI|KtcMM_dJhDMtwE@%C}7gzHN@nLcuewf>hoOub7_6**r>DuHli~$!ctEdyCtn zPmT8F=(*L={MHqoVgGg>8M*9V=_IWuZ75w$C#y#|-=t%G;CzgH&-u=(p8Z5Bje#c6 z=u-3t;jjEN4{b%#)Zgi?LfTMehK$jU>U7}up_WYinrAfTr>wo{1O6Yg59Ge2wwP<|&%(GCqmi#;vRjNfZJ1 zRhtFB0sEz&;6dgGbu8!9J$Uz8|CELX2KMwOM4XEEzvqlR(mHF4S@sDe%{V{aM1_mr z#8x$7RJOck1=j~(Id#AKF1GO0RQescbSO9Jj z2`qwkr>W!WOWb_iw!X9A6IOry#-BSds9(z18I~+Hja6FWfTb0?qHhId2?cU4E^_T) zP4^Tq5jEd$?B}n1NqnJIKx8WHN))Y2Yh+%`s7w8HiYgHL`y4jsq(O$%sxFhFzJ=3$ zr3P_s$t;(7jxz{W?af4u)CF(oVMbZ(JTK|t)t-3Ku!<6WTesHLmL}Y6&Qi`IDWpsB z#ktG}1}fTO#v%HpgSzUvH+(6xzethwU-6S)kZs})klEj>DB1l}7~BOjtEE1c}-n!}Jq~*efHq_t`RmZF_lSE18 z7H4n@?F|bzJMAv?sGr+b{rXC9b-II(m^^VR7lZF;uFj~El#O}wPj%2NZ#UZWpm49j zxG@NQhBSb%pVvLteFy;nh|2zdO#G4_q>%Y!WZ%E&M?8H9{={G>!0oQsMIev!r>^+~ z0C0_Pse669!G}_=2)`?#XmnSCyUGh2w%P%ZIVB*0S2pIn<8o*i3SJ$1mp49Zzh%U- zn4X@vO)847Fyj2;v#x7{AG+2t+3JL+3qynvW$RIRC(Fu^%?zL)HJ`(-(#Q|CF>MhW zUv~lAMn%s7|1ulS`o<6irx<^0)+oM;ODEVzxlvRs6mnx+V~acgu=zsVwZ0EscT^xf zT=piC`Gd8(@nO{dH50M?@#6#iubHnr?nNb`+c)(9nNLi({aY!+Z}7Ya{hsVqJV_wI=^eD#>f9- zehUp_?56hfDV+ZsFcQAaY@6j+{r9?y3T+o4;kHg~EpT?~#J_$2T007%K8O9KZgLs0@US1B2fuvbNIs@adjTTdSiw$=SAn zs0H{?c?Bdc$G5g*aj;$>Uw@F%Pa@YI1MYBJ6or6bd}M7NKw&TnrH;f2zDW=2fpJ+F?0eY|8w)d=o|?2}c}LZu#OU9;UQ$3*vbQ~Gms?rzYC{w$EPJ!^i_TFqD!uhvKg zzBVKCWvT`?A{3@~$532{HBzxjjfGoEOgB9r<>yR$$(teBIr0J&sk!=)AV-QPX>`s) z(YjeY;fGfKUokQ_?UX^lhX3&7JS~7k4?A;XeymE^-^MxTbT*%$;PZ&}UpwgLgPl`f zV(e)klLC^{opO3~K{%xq+uwjvc4}MRk>a&5)fBCjIqqP%Ao#_tD-0fEFkC~TFk8;h zKej?IqG3jL_@uSg;iWmuWBD$kvoi|!QnQf!=<~L!T zlnqvxBH!b6(4Y1RY1~cL0QwYEeo`_Rt1^pqH=C|OJjxnVL6ZrY)) z#;)KCDVr8koQ2wA^x4t3xx|~HJujU#i(H!?8ancrRcGwweEHB=CC|$thQe4+(=TxC zNvp&(sI=;TW-|_3fOIDDHG!o=;W3o?hfUaC2HMN(iHVXJdVD!m}evQ6Ns-}ypSxgtlgnw3qXjqh?r zwu-?MtVU7;hAZ54Co=~bc2O!{b9wp+H2CAS;E|m&MpvW}k`xLe=SE|-*OqjtPD|kQ zd>D@*llM4(h`*x>k!K(0n`!204CM@&53UE%zol(Y<+ZnUzS4;Yf&JiS8t4mz^jD@( z@VvzlopqSNUh}rY#8{v;2xI*P;X87^q#k8Si{8%GaApv}T6ugx5xF<)RR@@Kk1EHjXg24SgjI*mT+ zoVjGL?pGCYQb8+Wx)Hi)6cWP;0{nBATWIi@Jz_GSjzM*thkndyY)=SP`k_ky@v^DP z%zb?jI$vrjx7Ft%oWr1Ad%1Ahp2oItv_1j|VZvjcKe8@psMO zX5lCua-EnPNX!8g4Tn?KAqYWMjydPG3127}aNDzT}+?S^z_TjWleXYEWPj%78 zLY4*9wYwL8`i6ami>@QKUS6)qZWZN_xo_OwWjeXEv#TjvkA&^2y&r$;MB^(PiMBZX z6;x6Kti|ZUbDnBQY_!Sr?;|wX?b_wH@K%bzs<~;ZkII8@+V{Z|^CoLAHkL~UqUMKd z3#%KU?XJGP|E^Eh#P^1xBS*o}nvp*qhMNaJ1Lp`nl4GYzxc6MJUrN0fjK zsly%q>EZFk248FYEz~L;>##NmMm@&@)-a83_3E$+KG|K5<1V2k9=6GIZGj2THTW^p zbVjVLp+%`(PUm18TI+c1g+Lv%m0)C^d5=^|Lwsdz|9$UKDUOw0Wj`gZf$=Kc8EcYK zioo*jpNIsy!y%5-F3zH}J^qUkRq>5}*m^4ruZevuwoR=>7Ljv=Sxz{FTZaikL z{E%^_BK(g@9P3XVbIjhA{8x*I`mV&SJ!1ac7~+GvGXf&!)$LNOjgfLn$|`o~9}(ZN zUPwrY^8I6CzW6OZ_UgIRUEkgc{P$B#$m>i+!#kOs+e%vl-6?!9lJ+3E)LW>m^He1- z`7eNipQ`MZC1+oI0#i~HRD7Y=0J*P;JgR~d-_V)5~xms6b5pr^J?q2-f_wv8{3>8YLhyXs^ zmT9eJ)5zxZzi6!r2mo9MFLteyQP?L^f`ISuMQuTp%05@u_I)aQ`4`K!*2eOe19Y38 ztm(V;FEzgIEdU@d^^Y1~EQ0sJvX9#x1SFL34fFpghZ_RyUSIExgNC@)7psX-71K?j zDwQMErhX1ng@d5x^!(+R{B5Vhui-u}eydU_b3_Nz2lA$pLK0MaH4bHiz zv&o-I-2=*MK-^#C^_}PyYS7T?tP)6_o$SEqQ-KnFwUBMfGkjmW3*ve9>dfT<{4D7H zmItAv`D{(L6zfoMckWzP-}}BVMs3S4uLA@wWn~>gD7jH;wU1>B+O}Bj2SLXL$=eSC zxIsd``-98U*iXJrrzqaHy z7TeVc5Ry_Q@7JvRs~*<*NwYZv?1=tTJSBs89v$uV7^gy%0Nm350hX2MY8Tyzf}WR7 zsz8X398ddo3D%``y1OQTvD#bPDA4It(?XX_EU4X;a7~%5QziW3;vPX*B>926YGvQn zPRp>zU^8+*dpKkUzE+h1uHCkCMLyDm`~I&w&$n4Zu7QR4Cue9C<32@5N5X+>mS&p>9#1}oq&+n$!h;eA}Bt zzEHl78Z*CrL@q68@Nd}SiX7sULatW`f20SKjs6BK|Nl4nL}e9Vgjz*iohCH&I!I^z z2+|m9P}V!c0U1hX<5yhz(XwstSN7tsU5DU|d~qeJJ8oo{f~2$VCWak|>j66uPMr74 zix!22!J**oKXVaJnaun++WV(7>&9{H&u12}nMQW-b>f-!N-q2JJ(PQf*;H?HvkmIP zSH|*;TX(%~**J)fi}kn2jyqjF7Ua+pCYrq-@9Uy0BhjV5RtJ!;o{RAEQ}SF(GES?F z-I@;M4JgN+*Gs<(D$D$mBZ_xxRKC5Dqw9x$zBRj6WI_A=qPJp2g0)e0wBNz?4janm zLPo|uMxERY0A*}zk&`O8{nPgApo9~bLE^WJ);z&<9}af-LXawI))e}!D#oen&YeJ? zSC$~9gAZP04wZ$<edO#mh4(2q>7sR3~Nyhl%H=`8Lb^R`Gpjy1-Tl9K8+irMw4Q)?ylt+d_hRo z#-;T{yX2^5T$wEAX9`3!G~Ziiw~Cvyv>P(GK%1t5yXEoBS%17nT78&Nu%1Jn{y8D@ zW4;Sdq~2bRs-qod84!hFd1gTrlm~pJmXgmSQ7{zwCV{UNPZh@99ij^|+m7H8V8^?M zjUA@V)U-6ODN}L{w|_}y*k9x3=>14cJ*)f8+}r)P@{{j!_1o=d8>T~>acU|`)*XeF ze#0u5P@-nsT(;z>rqZO)k2+}_xjJBSJjeh}GGZB@&zST-jPsf(zjP1-{OMt>1({SezwCt7ew?VN2h}Ju)Vg5IO2ut6Pi2*)V>2k7rYDn_$w>h_GM-Egk&l1^r;kwUDJ*eUyWo0yKQr586|!X)mvyMQZ0NtLDMAtwAy71Pj0eaGbz?`4zw>#x#(`8 z?))7bBse=7KY~+wN%FsVwbu><&IxE06FqkVsBI$inx3C^GdpjOKxmhi*S2y)r4Q;m-3G`g|L4gH#d{P5)}UPVCr zj$MoTD!$NI=>>T&?uerOy@?^qxk*J4Z|BOIm8hS@29Q#^^3nV3zo2f%f9kP{+Zqx@ zTWnZ?z0&$EX7S-LcnND+_8l9CTUKdp zH@N^&c+#GuJY@`N-f^x=>uvrxJ7lydGullWN9;KnyJuZrfvR#r^0JBUL|699rzrP> z2jeDn#;Y#fr`W{1WTvB8LKy~KcQn$#S~rr&w{N7BM)u4qd6um9r)yAoU)0B(f?8d7 zKjlcw(xSeH>_-@dcKO;RW%d+pRy*bowvwXsjpV)mH+X2 zfzOkjxsQ{Ax`pxLL}{>}kq84^eJH8a-n-fLM#taujGt z_dKqSk&;XeVd0BT_G710k@;@zG|R_K$qZxUBgvu4z4%Yj@p0z!8!O78@El}HQ`LLy zRIs}oE2|Y-6zf>5s4@dHacsH`fAIcG5LNz4loPkLv;vOjK6zOXnmAOYm{u8PXp`|( z*$@M_rtKCL>KYy~0-lgWbyVf+9?wc@M)|LY^HeQ0Rv4($z{I(5l;@}V#^h-8pR^;y z5+pa?*MQthIXHoyA1fB^EK9IFgMXIUEneAX8DUUCpKd)Em>2q1c9}dQT%h|`o-PV#WOE_^)4IcQf74>@}T*3ZmDCoz9J|f99!KOSQ1I{bobIN zIutiwBbhNfUUZZ_6cY>&=;-A4y3@&lnFJZG2fYU2`%+J96{lvsFpEK|h)%@%{IH z#M^a8h;Ha&7PfF-fApZ|QH!eQspk1N*k*i_61&(Sfr|IKBuEc8db-Q$*7YB{VZop$ zL0*Iw%{{<1v-4HgU9+0D*T0XNLsVAUei4p2^4^9!naZ6zQaO_%xp#Q{i&4XvChiwP zCGb$}u@|@$eAS8Np=}FaR3QnQ4HHP-1)g{#s+eaiJ1UvglXFn}CqDgf zSSR&d^{zf)U?`k{VgT+V({BaX3(oK84)2_)X8cE!i{s**K8#+0x5 z%YCZ_)*Ss+=?aiJ|F_Dl9mUYvZyn?g3Ia;fYUN=iAzpqIU!Lj{5(vk;52`%V10+p0!-{Xsdf)v$D1$`a3v*LeCFguAcmuuQF};L~3{EIdS&Tla>Jy!uK3M z$r)u04PiWh{{-|>lBhhxc0hQn!dl#+@|lcZkNgWErz_)4{wXe9S`xb3dzz&I%?@U~39AC|e{qa9gP4S@N zDog&T4y#~|xSvDis+IPbd&K;zf_$^;z<=qAmDHI+Q+*s8`XO@g1O5g^Z{w)MkF!ON z{NNVj&@?pOnH0ocbD-B@f{KAn-7qHBn;M(%o5iOTyOTb;3G0uED#1f` zCtBsE1%4y1ypwNy*cj8DLM;gjtg+3$o z1xd+!kk8X zqA4$okDMisHwe?uW)g_wgJn)&1=ht3#)EOAGI;uTkcC3ehUrC2-};HTbMN?*uKQzr zc}F%}+90Rh5!2wqZc^qcSn9kzF5cthg)>j z%ljfKwDS;VyJZn{z!U3zOXQw}vs+7IU9mi(n+`oRW9$xbM0Mn}^{2;d&|$0iIt$0D z>Rry56l_hz9ci{?b5W4ceuNk`6&fN%bNhwcM>Jc`3TBWAHY;;C46c1*=>W@-T&~Rh zzVp<(w7JwwlOLe=`ge7TQpif6R@p^x!UQ|?UIR&sC=@(V*YcG$rHF62Vfe7ln>d1- z{yCm`z#j2neh0}Ov^&e9+Am6@cBB3&bc4A^8AkbaaO;q$r^#}pHqzsu@>_mZX#W|T zi_*MZD?X&px@#mX`OhBWMmFmc8C>uQw(RZd;`o^}-06BA!wUrKS<<(t)R%G&%GFCC z3)W;%_OPuXTT}W6Jyo69$yoS!(Ob9ZI(3C$BX9iN!|#2}x3RG4lpWwRkHvIC0e>W| zzbIb%nNWWnQPbgfh3;-l<)@A6uDZtzibRy(oFIso5^3O!u~k$o%U%}o z9LAtd!(w+)dnQ}HyX=rD6OECpD{33hhO2#^QyR|L|7Am^$=C)C4{Ro1V=AA40-X_Z zNGe>>@Iapg>qjWMu+8e-Y8Jj@d}>W}Hb*{%;-EEwe-H77SotH+JLyHjn*=)r(KY}+&{ zEXT-?m2ZPX)4;?H=y}(mFe(eIp|4fEK?3|@N$!G+x<{yF_m863)={=BZQT6K+fAk4`EU_k!(t)m=VsQD(!rnn{F(*mCa%ct z5e)w!t4j+7|A}`X)Xa>Z;hsCX1sQOh8Hf~kb^<66)P3jm{r;L(RRk#vL}x~6Ez*M}JgiziR2>@gqVGG>cP0RcT~m6o8rtQCq(1D!H= zj#FaXfKN+TkGbeWWltal5)O80xpisXvM%fSM7B=5^W8JwtZ_hJ*f6QoV1G7lAvsYo zDNLhG8HN#mJ{J`96+Cf&nYnS#zsO|To#W1Y^L^!lz2JMfcy8;@*D82e=Xg}mktF2= zmwkvG+QmvJU`MLD&KcV*Fdn)*bc)}oV@-|)XD+-i{gIQW<5Sm^pR*lSL;+Frk={CV z!>b|5Dx?6iu%@*XTN~WN(hcBo06!;(-AoaDZ|eI2{WB@D2xmdZ;xnF4ymETgRnUCjoARktKQ67g zFP!}9p5*o)8jZ?4$P>pUwH1!n@VCD^V;v+0z)GJ#k4i6tyX$Q~Y#zD%>WVPFtkzcv zV_Dr6#&LUY+HG<%e)a^t+0wD>+~(60dz#-GKlC_971IGW?MO5EzUw$ppd;C<*J5z+ zQ@QfFn4kXqB~CWed!H>MP3x-n7b1F^Pn{Zb1)-6hWMCfD zJ;|));Hb*$B)+7N%t60DCsZmu`q5(1P$2-HIxQ^33y+xY>*)`jn89+QVS^Ps=js=S^5at*-D)aq)@Ww(d_ E4L}3q)&Kwi literal 56377 zcmeFZc{tSVA3yp$tx8nZvR9T$sD!e&ptA2Wwk#3GIt&K$Xdwxe5JE_{5Ms>O%9?#& z$5^wDnXwPXa&GGR{+?=n-{)M{Ip_La=bT^H{4tr&=llNllc1O|boH-4 z{@k^P<_M4cPhX-KR!}OPidMhdnB`{Mk%Krn9lc$g$D0nJGfIq$BqdGy zU02Fg^xjNV%OaAJMyqhfrY4QI!wPdS_a+Bzws#c!jnagpw?ygtq)&5N{#=IqSnqwJW7fE_M!70s#(cQ1s ztW;g=t5e7vTu~)b*UhKHtn0`2U0rBG4)}>Tacn{_4@8?1(Ne*fJ9Xm4a36z|f-Fa3 zS28@$FnrcpmR2crTX(%zuXAr zjh9`S8P4fb!ZrQnW^cNlSsrH0I2_Q5)vxR+3RTFoC9#z)2pK9eVoaZ;cdXx5=TFz! zQ?}NOlT$`!Gw6!a7@AI%B1~t`Rm?62KgSLuYXas7sg@OK>F2*Il7{I|Gm{8)c*tXe zUT|edk?rMpd{y1uxS^WqtG$){l3YA)(<{ygasBeB*mOikSOfv5k6e4)QX(ULKl6B( zKEfS(T5SdDfF#eTi)2(E=P&CiNkFg2T&0*22iA#|%%~ECb9@}3uekIKO>|dTj|_UH zlezbz#3#z|$N1@W+yr9n3#?6c!YTM!b}&56O%`fB)ea#a-qB^WP=y9Og~pZ z^ZB7)G{pyK>a=vMjR;I+u^`X*F2@XHO&BJ!KER%5bf21ZcXL-7d78nDyXUCV#N{hK zy%;eFqf^&ivs&)=vmPCOobU`p7zU*xq9+Fsj!R)&dL+GK&o%FL=w zsd47<2ZMB+{hb3|NGlhIg@;hARFO#2xizHK8>+gja6}b{^2R8}O!x63-@5h6&MQd690V#c#uXvj^M-(h2TyN=bTl+?=U}X`w zlXgv@Y^JaDLh)_`S362p(o#g=X?>Am86E9~=03QHGo>&phuD`h5S6ROm2pK=->8O9OhXa#3)Qi!K4oEgjiWQ-TkbvWUy`OcPg z^P`O=OQX~X1-)0b5sxh(i3BHp{9ryc_xMO$+9G+m63!f$$QrIC-5FZ4Cqkx&MUZam z%=GM1(SEH{Th$#m$uE-P!@1HZ;Z;JaJ||#dUE!GAHfz0=lD$LJL9}9d=Qup)oJ*1VH|8!ONsaCMj`!rJvJWow70uY9-Sme;!yu`7Ae>Q{Cg zw;3}HZ(@LDzsBm8@f+yg&Z}+TSVd4>M)dhsh$U#_NabB0Z1tr6hl-Jif!v#XhFMC zZuC|}e&mf~eVhuOQV(Wa#g2lac-j7g>U0g-8Li(M`YJ=p|Yd#r5g|D z(rLw=<2o3U1A|V=O3$715qg8uPYHN1^db`RQgBxd>$MTftIrP|XH-a0KEvd&-lJ@` z`Y^LDWqy=ZM5*23VZ^dbjncgg5tPF2GY4$PM)!XFyA`QS&Ay6h*7 zKJmh0r}tjPFo%aYG$3QsrGGe8J|2wmVsA$Z$4+!=`MN6*i4_?2Z`&e`dyPJK*dPvD ziZ&C17Fu|0Fi4-v0-u;9m=|2fd7(wY&wZUz{Qwrq(;j{?XHFBpieK9^nDItR%!7Q=esg zM7#irmDQC`ZtEFb=6Hb!=VZ+-o7JWF;~l9q4;lj!(T_%P2q{Su`_OAc<27+Y(*#1? zlMEe+1<0`pkuM+e$n-F*T4PHBHoc0H z!?S0L4VQF8th`jGo5EyflrkD>yIK_>F12Y*pfwBGn(0IcZANaf?|09cLmZ}rGA70* zF?MTrRxu8Cfu+wqJk%GvE;)OvgFPj*Li?j7q#-lyGPqIA_%6O-C9BYQ&K978@&b>u5dc13r=>-Dcb0hhrgHIgT^C!IdPwlv8vAqm;Z=dCeDv@-V zMQM7!P}cy>$L$|8+a@nax~=mrW);-CT-lP)bHhY}YY{zz~}Bmv7ILpD{LLTA}sJr@{}a z_@brMR@)Cu1(!3V>D>*~XJo#~1oJ?=3f$L7AnDyZE?`m9L1TInlX(x_S0aO9;u?_g zZoiW9xbo@ep!tzXW_^ouojyw`M!V){OwB(&PLh}zV=C|O8UDsOojBK5oG|K?$*er0 zENf+w?N&P$4)I02&7HhGy-PiILeac4WGDyc6OLcB9gH8rOKga{*g4*DmSHr#!9 ziUs%u{}ACRnkNpC>qb}_<4KF-JEnW{VC7bE=*a6Tb;AO<0Foqi;eu%FzUo%X>193V zkai9}uUbi_cmakA*tpiZ9I2}8Mxcrpm)(e>?UbATXfzXYf0<%eM9+!i8vGf~yqod9 z*UlIv!qc{loEqK4QwYcpg0!)ev6btp$&^?0B>C z?MrB=L;F$aVAgqU`gEeJ?&0)vrsU4c7*%E_(%@QIVDj0KVOH3iClnaE6?(BSdr4}l z^Awsl>e%_t)fom6+|7+R-|5fsumikG9wlE`=>A$$=MOjxD^SySvr_0v3kkf&baug)$MGJ!WT3t3}Y$r?;kWt za^HVBajUKpc3{CP&mrUBQ+BOn)_}~k28@;-PJeXn`PtN_P3NgJU8_h|9Dn2)n_gdO z!`Ty=`aUtayoj@^kdtqO;5O#(lIfqttZR$-N#1tedLL8ZLFu^Jyon=(R%*Wq%EZmz znihxUc&69jKM6bMzt0-^Cf(lQX&$^N-HvG`WR_a_v@Axy7 z4MJ>l79q9@ned0x5q$yQ_3UdYsQmFc*f28^Ic$+g}WhM}dEKzkSh zbdD}vjMRF{BydU&A?3FF(>3#bHfOCrg!xoP>BMqQ>m+4qndjGavC3YN@Xh~p@@fg} zgIdB8Ljxn{U9(Tm?x04E3Bv>U_(A<5ZMA3p-d`S-tBmZ^NXv%_ZiMsPRy|;3P8@zg zxZ9atZ6K36&!FWGu?+x%LK^2bX(vilX@Eg;4d z$yFVwCx^U5$By)QiK(2n>Cg;bb4F+0>^>pc&-fioc>86~4HRA1UD-!RcZ@c9FT>W1 z`r~uToZ&YQWJKr+afyj)``E`;p2&xkG$g-Ds0PFo-*O3%>`YBz z%5IH%`g}Us!tG!tE3ONIcP~b$&j~NjFF zz>=VkC}yn{+T1i5iOs#$hDmkej;fx?G{j%I4@n7S*gd{{ z1BD`i-FjuOGw*2*?H@|pou+&AAlRwLG+K-eK--FL%S5b5Vy@fa2Kpdf4c}H)ZcJUc zQ+mpL4hk(6DBa(;hTENnJlb*mxdeMP$}SVzWTQIs&8%ol@7=)jFpJNwKlUX7>`U5V zTiKBv#KljM71(0hbE{f+zH!VxAp#}fUIk)C#dp;mQJ=_n760vW+R16l_u)RJQ?|4z z;^J~>$HF%$4aOVxN^i1C+QQb6u#)GO`cj(c-O@GH&xF`VqmV2Ewb1#S>yYI%X}n%H zMwK|2yOBq#3y(j(Ie0ql7GHu)_6@eHctITF-RVn+;jj>;=r*xH4qMqaGMsN8r+_)JhbE2)=dybkXV&-4G0^M`pcJS7$*r!!jS2(VM zxRH?6G&LQJrE4u|z2`5b>>dHDc zRp^Vq+=eKdzC<%)Q8^FG zg8gA@xHHbDvWIeJ`R}{xQ$}F<>Di7`LV_j?yi1)5Z?yFf2UdPXu9>aK9tjCs8v+3c z4|fm9R67(WMlc@aRlF(vQk%`Kx0pW^(KTN7nE17q@*LmRL{ye+TtZy zi5y&UG#_tEiJ?9q&kM`aM5DN~qhH7@JG|KlZ;nw}oW5t`d=<`*QH8;+>;vrfz1s4{ z2$~QsUFf|+6U}X8%B%jYfv0El&PtV{)p-H)MwghdtFDQYyb8mL!|ukr+;l7KGBFTG zozRStYdy8j2KH4J6fMCh6`y%~h6Hr-G=?Uffq+iHIj2L!v(F|>HfQ=l3pne@ zKgX0mG_l_=x!{_SnJV>1Ic-1v54l_iX%(x_o2C2yJKp+IuKY;+hVJlJ)y+tWUQ~)u z-Nmz>3S-$3tQhU9wm>dRtto2v*hr!hZA&pW2Tjt8-v?8glEYrDow(nTDtX2>*0w`8 zn*CNo_946c_i1P9GPS-q$6muLp{j00Bhwm`8);lY*6ptPxjp; z7;!YaZmFn*3UlQYPZl_hE*pWZ9kqkbr5=;&o6Ad8&=jl?2rcf{mg&5Ne&<*SIr2$1 zS9bkGZ)oUc*wpH*8JUBe`MPO zoUkM7(pYl_8gr1!=3+grdWtH-dED1s;?b&!Xq<$8F|05v{cs)`GE8q1`%9FHK9I ziL-4U4<1gd8`>Xmwyc*EmEq#iKOJ8t;|;vm?B{W%K6p3r{fK4T@ZswvBbSD82eL3x z`nhi_Et|K)X0~KghHKmniRAWAvnkVpR?FoAMe+7R>IG0t8I-a)P@Ctdv>=X^yV;+c z{cPEo9Otz47R~2L$*#Pa@pQQEtK0?%4WCjaGHP_Ertu!RVjKZgXjF27uH)$@BkAIz zjTd#F@O8(YP!jd+f)l(KT&X?1Hh6&jrM^xRoY8 zVl_gy59zWZp987*idp?q(CElep-(vB7`zD6hOots`vWsds+-_rp+5BfT5zBlN5Qp4jUv3Hs`L00OhLB!{o zL@5rnnNXG$)yLuCnk6<{?17yYc1kEsfBb>(AcMSFIH*7)ms%GbHbrmU`EGlE_?-jB zm)AlYY#5M%!*bhuE}Kcqc)NR;B0lNNS0MZ|Hl1~iGTxz~j*im%r9Ig{i8XTx;~?3Z zV(DynrSDpvs@^UKFv@U&K;A3ObH3O`tvivm2DMx)3N_7T~zzZdq;3v+pRFw8iHUpSiY z$-FDT=^ET_u`u|!QTyG@>ZgtW=cqK6YVxocwD6tYbEZ>2g#GjHSLI=Xm~BY>r$2A6 zSAHDiF;V!J=Xlqqsr~-B4rr1BirT10zP|ZuQry)zH}_6y+uxh_&kMgk_-j%RrcC9j zfA}>Vc;X=1&3EVKVyEBlv--m?ocRdusROl9d7fVZmE<_a(kjn8d8d;6Wl_)f{`qg* zRPOW1jNQN7wzGOc!G9A_ZQZ@q+X7U!iih@J^}_N*tNwm!dZDK`1Tt5P%^w>0&rJj} z`rOk)IOy%D4nq{#j!OUX{Bu{-|Ia04@PZnuA`aBpM#?4ObuXyUV zcFWH?&hwu8y?ey9rJ^?j1BA(fj~~tIp!`a2Z|_uuzgUsR^|xth_ibI~1E!|7ukYNx z{^NN$)1)PFYS+v2WbBF*;Lq~i+AvmgB)P++kxPF^iJvS&p3V6y$eD<(LlaKjzDpq>d>d|qBJ>3|q775o27p7*@;rAuDJ>f@?$m&J2DY$ROgcN%uP{ z?o>ta8OH=*C{?HxRbiUl0YS{_LT@0|y@hMb!w?er;osTNf5ucWbw4W*5hANB1*orC zb$iVndMrOr^@oN>Y?CT1+NWj~`{ulTKP_I~v`sFBVD{BP6e(oK`;Wv^Pf@$i3XGY4 z{6HdBw>_L~hc^A$xAGjpWkp{FLyH-wo%s2u9nsxDWmq42;-If?YD-O|c@FxG$F_&a z>NrAWQTK4dSN3(ulrEGsBa~divB=;>GRO3~F&0yI0dFFshdyB7JG5av!YH#iL3Bcp z5QO1nH+DG8*E?NK-}8)w)3nA!NKJ+7-m6DqGm%@ap(IHd=94vPQ#F}UdbY!k#7#KP z@sz%KDRyNxGlh7#nQM>Vq`2SkncugJMgUeg26iqi#)qf>{axP&a(V`sv;#zZ_(?N|7b#vQ-G)R5>Ow;L0ZCxSS_g=p7_bYrZ(>$4_~5LcLa~e zX+!oC4c-!^-HgHx&y5b8SY2P}eri?OO33Nfwqikg3`I%}Di6Fr=j>6Kn>p$JBASkh zHL&T)!yp-WhE@Wl`JMZdB0I;y8A{*C6lU1=z?TRz1!)U|kp1r`|Dg|ynLZ)+ zlXL$*a|33e3x>GYAcGgji=*#dBTHrb8s3={H(|fqPARzyL#&OC%;yJjZy6hgB@BGP zdZ_xZe+_IB+A=9k>m9jL5QH)~pP)CF59p{hqLK#z!} zZrEjV6>3*KFdapgVa)#3b(`i=Q@&6{;eutctM4VFeG>i2Vi@?H3&S${l30z)%o6cZ z^KB2SWJaHe1CC?a$62+B?v1q9jAgU@#WS+26Kxx9yYd{aJ0hth*f!oE|HjFFK)8M! z2$kZ>l$=hmye!p|pFBPG)(T5WVXSpLIl*~l&Csgxkyk79BN3f7V5tXL@}7MNO&+;{ z>YVna;ssg?f#8$Z9B`vjgOp;4Xh*5GLD$p;cE_}}R%-<)yXdaqCBmW$2k zubDA&_&CvnC>uN`9It`wg3d9K9laJtwTG`^PNm5C;L`$3wxk;#AeQr)$wzVgA7!YN zKP^Mxnfplcoz(JqGci#4=v!plGXCy5{@g_;3@Y?ldL17so_&vA5{RH;%jr~;lK=hC z9p~}CYJ2+yTcO(8?Y1+rP-+Q|8S}vp#zo5}fesRm1H}iWG{SxgRX?1HLGZ4!^)Zov zefveeEsPHON{Ls$Kqqf?SbVNxivD9Ktqu6QT+Mf$ieFQ#ep4`fwe_@GL~ zb?U`@?#s86TboQm@L@{C#g?jS!^$?7vb(d4i~78@5LBnYyvwdTYom%$okQB`ZCM%8 z!^UQ@{BhvioSn9IhEo?I_}bmOzP=!BpR(&NE%Nr9!Dly*yX>}ZT3lY<9E>#e^<4Y3 zpPFj#$vriTH1+MpVx^qM8+wf#z@24afVIASG`^d4d?E}_*Pk!}2X2Y|F?h&bkcNY7 zUr;~G+jUB7bA5FXa-?ubaIi_C(~0=9=gkw%@#J3$ zr|%L2NXVj0NaJ7%?RNGi!AQ)k*ViD$<|48>TzM%NtwO0dbs%j=^m$9PyNP+4a zkji&G{5>Lrzw{n`eT;jHI6nqAynjLpVWfa-$aY710%T>d?X*Dn9T8h_H&ACOnA&Zq2)KR@`piGFzU zuh^3y!?55hD$?;&0Wti)LtZb_Y)q-PR2v-qSL{iTw_N}9Piy%hpD572`rjchUhUS> zI%?AVc=BJdrw=zDWou9~mW!Zx`WH;{eWFh|H8VC-00R0SVDL|i`WuscL1UUgO+c2n z{}q!|QJ|dt8{_?_f_#1owC2AHDEH*i(reUe->=kqrPAuR$O3Uh2RckB}4i5eo2=p@yi_UpJ2qPt|d?J#Qrp>4PGw{d%i9wP)zi z_QMB#fP0*n7-t!u08>wrR@l`nz0fez(9pkxr;8jA zH`zv#&X2jb(tM(Z`T=BKRaI3xEDZP(2_rA8St}H{Y7n6m5BJ=l`T)ZpK#WgCGAfuA z*VNqDIgbjaVjzF+65F%sHa7x1U|MZ$Z&Va;HHvo@^B% zml47U^-mMpBG6mq>+4`S8TVRjXnxY3A4(249I_XdSYGNLG%l6)AsBjAVXfZ35G;b6 zYQzYMzz`s?s*#fys3fU8A8q&e2ly+B7y(CHCNAdoJv^W2QrWJ^kET+q=V>~Q9Ao1s ze%>64q!2p3y_;MPIrrME3mWWfO*IEp1_>hRrcsQM+BI0wH=0GNfQ;ClePwHBk(HQ1F(5OT#pVC`aYTBE6V%t``j&{)6Q3P^0UOTgM-|%Z+!Ar=NymFEXbASksfTy3apuc3HJNAGG=jFd!v z|5{GSKW%qMoi6P-X9Rn|v|Y`@YHQo*N=A6Q!xY^WHtSW(Y6E}V-%5>|*r4rnE`)JPFE%^H8LH1x{0 z*MVSHLQgJ>n~N`}5iMctpGw!JhS2MiXur{|h4|pccgw#TuHV|WgN1IiI@f1|(lp9G zpUX)p3ETPfz433C0zvA*VmJ=V&4FhK?8=I(P(Wp*h~)oTE*hKwbyQap7N7~gsaVd*3K?LezJBtEh*g%AE`jaVdSI{NA{pZFvh64WO(RO zr--21zJ(f-P`e}94XC81at7$RzMSrQ-E`OuhuO-_5Tzg`N3YRQ?cK~_FWPGc1Y<>|Woe#{ zbH_=?vR`2(56!{hiZTZMdE%6m_&9ofm?~}W_$D1~UGN*C z?Ckja{R{|?+rRoZ8y>LL?~|}y`k{}n=)Beq536kFj&P~qF%U+0)6&{mAJYAqHd=cTx-&Z9@O6*aAmqDHgzrnQ)d zAB!2yla%0`$5VAXy@}uACBpG+@$S8?1hXyqW7bLnB%iI|4WDo)@xIq2{6U40Nq4jJ z7%{z1NYZ9^k(b>YQE5pJkBNjny4ujVfi0s{E+y3cWgL=lKt1m37&lS+xSrYUXHo?0 zP|*+z#ew&BL&XEdh1~(}k2;M_l1Av$zcopP-Z(YA&|U7jS4#F^*@3U{vL&Yr2a&{V zH$_8IP51@wJ#(`mVD)lbCjDhqvYc01?;O8W|0VbEuNK843JT&wN~mUD?W+E-8p6>C zh{~cGEP|muU_kQdy^yUZ3^|wiX@3RbocSVy60w)_>RFSpH{_-J27T5ib4HblDE{Rx zt~Dki{thU+E&@hwQ|8%%OG&^0j?xuF38~DvukOj<{N|9b6hxs+ zRg3J%imJ;PAu;ht_l(m*lGevm`Oik-DK$BX*ucvshbHz4+{5YR$mNi93i)Pg-VR9P zF^*?Y1MvI=wHcx2K;wH!Be?Q26(XD=n5Fi}vz@^cH$o#?`^XBn)`sMk%NE85;PjQZ zh+l_e#~ki(;P~$?jjz!Y5ctLt(uT}?*(H6rXoi%0@vktyCBV31dMCwS)B<$JW#CIFXu#myc@EjCmmY0uQc)3;S)DL<8?kf(-B_f6=KNRq zs6)A~HG@3{!ta-Yi7i6#)MAFKCou$y8DvbK#2q|Q5>jp4zP2V;ZsRgJx`sc-`(Vn6 z@2N)@R8}t99C@0)UmX_Vb;8->SfHL;eXg%jE=iG5!#OFjRX6Fc8=2+6+?YB=xgt(D z=CUR0GIf<4r61os98XL8)Oy_iV^9)yLYC%X-wN%sDzD+=8>+wA*B)SBdP&$y%i2)ZMJD7;xdNd znNs*1--cw_pC5$cz1VfS9J2P|DhDp)&1C;P>8@9d>{+#ak({{dJfkpxE}cBYkY^-F zD_hwLHLrcZ6DKIS@}(fv1Uk32uXYTc)Geq!60O9u)XQkc64}l8WI{VdNx|)m1jW-? zy$bSD!#N__B};Z=L{VsfVb5HPkjT=JpfLdWrNuK=Epd5G$>D{x*@LmIPG25f#x%N> z+iJ&jXVcy@^H&00+WspJ#i z@vl|2W@oij9;F94Xh*2k1>T6()aY_4`bVx7LD10s(_aVSy5(H~ zaL&Wl5A?gBmvKJt>>O)E!r&FkM!W99*x%`~)`pI|Je0xhAHdxaSt~pKy8E}BL=f~! zKtUJ4Q8pGO6%L$#uPLa4n@bBkFb=PXRhkxW*!drHk5tDfpxhi^>q)8~>kxh_tGxgr zJKiDwhSRGQq@LE`GKjq*R>OBZY4)>N{;tWXaClRv0Vy*)hyr314kX>VtO~4GUD}E?1S4& zR7qt2qdilh0OfZKu0ck2S*AvwsGevRV7c%l5w%VTaZv8QS&wutMsIx$9O!85oyU!Z zbBtKd3MishEyCTrf~iVl|VDYilK3WYkv!qu+%$ zBYUq4(srza4MVoTLy==irCsN_+#K`1= zQUeBhABr%yn48;5+{-}R>Qv??KhD#uXLG)|5|)JRPTD7wwWSK2!iQ!BJKD-XPb=LK zt*g$BpvQh+;EN;6*P0RPmEonec@R<&=q4KzPgfMph~{>;F+Y}#nA{4-Yz78rhO^i0 z#c%r5O9_m$QnaRcL7bsL>Mh0tx1Q{4++Ei1d3von(b^QN`nveHvT3J3C8HI3GMKZG z6R|Nj536vC86X^3DC)?L5+GXZ#&y0f&eZ!zJUpz^@9q0-p}Vi#R2?~#))aPqwKNMF zPPdeZ^+0^0xjy?D9K|~J-Vx>6MIa@m^E-yK3>JMLImSGV)HF)=yDoLM6FF2S`;1}W zJn|W95DkhcG~{8G&V147XPdWLp+>Oc2s|d(apTmXlaa@9`|u<0au4|`U$s1x-(RYH zjqgpp9Zndfnd8*aFkfVK2;Vvz%r^3r9@wHguac(nUq_{G6HQKY%;FmxMLL6>Ml8xf0zBhyg9-{fjdcU6;RP`aLfm(hBNz z&^2NCRO))tNc2QJH=qR|>&j#3Qz9}3(swu(_=vYaPDv>(mFJmQ+{uwYlHMAdl`*D8+_$vxJm}XgdJmO+>GkHVk z4G+hMAbr0l@F(pA4IcV~6+%_f)n7(lAvTx)?gij{=J0T+wL3(_;VYWvBV}p6p$#*0 zmo*#f#w)At()iRVAbGQN3@8LxO62ykB{6iqOAgWzFO_e-VryNisnAs6z8W5Ph!Y{h zu>gWHkT1FP$;0^$1DTR7w9P(}mWIjqBWM3GcuQpvqrfU?9(*q}R`iTdZO8;jB3Uy% zo^z-MiZN&^;E*i=fLGt^9`ZeAwF;9BoQfsD!kC&YfYg@P)B*P*6 z0@^Yu;)w%eSfOjcS;o*XRfFzj?}>yszZd66xu!^0Hi=E9k241MT|KfiZ{w0V%p+60 zfPT>qqE)|^n9WN*PqkmOl&`rUQsG;n5lYRWsSplKm9|w_m|(^|%H|UWy|EK>ucn=# zPj8E|x4G8Tgo_+bu7+Ze!hrzAK@ z@1T)>gxam3*v`!OReswz!cOC$0xF9A)mdsN)^_QQKl>Zt0N^AZ&|e8E`0;(ozpn#q z>-KCg2Y=gNU`N|g1l05Bt7+H%VO9S@VYio_V2Cbmt zJ=Ss3zfe{8h$l6uJ^47JL6JK_iQ9)4Q!;wSjR|KG6TLR_t=4pFtsC%BjXh=>oYW?) zXJSU^%P;d734FMpevwQ*e3%YUF6S@E`a*5%K47JoN5QO`jmu;KqXAm4LO}->V4eW3 zIxsM>MVYS`QIB7yCf2{??+k5~uIHjqaBy)Ry1%E}6bWCQU(B}y^oB@7^I&8C9$(m< z;7?}d!PgSDThS!Q6^o1Eg8HRs*cs&J`fN+f=P1#QjSU|kpAzao?^_fqJ3AY2Y~T@q z+*)5<7&OAb#_Gbk-=_c+m(4r@3z)F$irdr;KwJUW1mIC~z=VPMfZ@URgAV|OJ^!OM z|M=o0Li;MI91NR>HUhW&v%yUfhDc#iIV}R0f^Vz=o=3lIYZEYG5|P5j9PI2RPQ%w^ zktgDtz)=4OStV}dfc2o|??27NIBY{C07M!L*$%kMnCR%}H;ex>DfM-90HFXh zy1A+0QkOT@>}{?U(B$^`Em_)ro}Z;h&y!|$P%fWx$)M6I z1YA@EXvr5EAU9BFh*I_PiZ(Y8(gXjlH<+|7<|{a@p5wmtEG?I(|Vx~%*CXwg3` zY+Jz0NB;}C=7*5P2%S6U@$)_3KvtPRHU9O`LtCh@T)$wU-Y8}4N~ z23iM7obH-?uLEzi)Y}WV?@r9U2b1D4r!;=NMd-WKfy(TBNj-7?(q#`&AZ;yU(`rE} zw_9X~5o(1}VTAnr!J!3lCqM{AU4vmicE5c}g_r^({@tGoT>*V?5DOTVN+$F6E~rrG zxced|2A(>wFSF@5Q;Di#+w?1zt)iT2qp%(TD+Mvey+>22o<-R{C8K~6^M6dOBL#F& zY};B9qb$;bkBCfZR`pHvMhur%8ouX+oMTZ{xo^KF zs%*LG=I^Q8P{>aHx!SrJDizn$vs_s{VbTYGCnx8XMc#nW=^z_;a4Su$tF0oESO@o+ zh%CR>%?-%eA|Y~WOl&8fO15)RXgi4;XZ&Jw)6;>*q98oZ#v=1|g%n|cG zVdLXGI5%MA%<1onE-fo7NGs+q&!|ycz>&6Ad$viWiR``Gt&a8T4|qHX98f2``I6PWWs-H*VRoVEwpgN} z7nMlGQ?Z@I5)vofadrX=tEFYovmw}d-hMv|$Bpjlf~{%ksQS>j$#aj>50|)|l#9mu zPLzok$Sjp31YD9<0Zdk}G()PN*dI#tAqED=nDimfMsi=l;1hUgp0+}LVQ(koCk9a9 zt#$1u?9U|1bR#A^p-^>u1b*_f4OmHY=2a&oo{?QjmhH62)GYT_sj!dEY)0AgvvJoH zjJt!E`KYTS7kk451yg4y^GbBkXfUBW4q>icQ-nLU>OEMr6*VlV*-6D8yxY-ag;wZ& zdi(oN?OM8eMWabV*11(Y&#`M+paS-C5q4jfhgJ>szG?v%Sqs4>dRC6Y+CImAdtynACUsddGq zd%3=CrVmP?_Qpo8{(JA+=uDi%JLJ6y9ipe{BMkGGqH!`GninsAXn`6 zKFw|21+8i}{oVZOcFcKPi6Tclt(K1Km>b_5-B@kiA@NA0rP-`eV4I(FP@8rZSVpXg zT#M*S(wTvLdmNA#Zvm?{j{a3xzZ>?cw&~y2v56S01;Fz{x8cJWB~%hRZg;kWV1$-5 zWb<2m(EhzwH4Mq_L!Xh3>z$*All8yS`RW=mNWjzWCf*B(%aX@)zIgl2%CBZwX}&T- zA5lOt(;$BjXy2Le7=uRfNIb=RP&qUJ4fQ-Cw^mLO*rzyOg;VqjEg_rpY0|yKj^;ZA z)qE-~-1;V;RVYqG`)LQ5V0B~%8@n(vtEldfMZFNoRbs*ZMCHlh*nqX5|1axjK&H@Iq#D@IWTI-CTz`$hp~R50njFy4r|xNnA9Mh(t|`mWZS(*NT~ z()`-Y%=he6@oxCHs9OJZ$b3L2`8VJ|OZxfH>tZP=h3B>7EE4WJ3s zB<}SJ=u`zqw?Ln6#r5mgA3g*ha29ZkiWUAPFTS_#!~j4SdX0nuit*3n`xFK`vzM{1 z9@k`Cwh6F~PEuK!^o2P90kGCogN~@e=gp>HC%|J6R0|$eKbI#G;22v~PR@3IE`7my zi9$lYjuXy~FzX^r-`YWujL?ALs%IP>1>Mhhs`Fdk%U@js`i58Q!UX{+LcZpJP(Qcb z;@S|Kgx*F<{TVJ~Ci2gz05>HF$b9Z|0Q#jYZy zxIsvs1L=iYrwTP4U^51%Vu}N!sCQa#6OQ57-+0LFr)}ev+vmamKh=Br-|P2L;i-Y6 z9jkc>3r5366aG9s1IACtWmSiK-z`jBJ#wjRULja?amJqhf!XzmuR?Ko8Xp!yh{>Vp zkcCmK-6!&dW4vJg+{g>?LM|pd<`4HBaP#hioge#@xpcE?+g?gE`Q%cnP>4jvO^>#Z zx95&4(vzkq*GHz?c*IO%kK+d@i)*ZQx4L|%Y)vQHGx_4)qTO7rNtQ+dYa)%oJ_Q*i zO`3rP=kg(*Onj8gOZ;~3w~yG%#a)IPU1z6~`pAPfbHLkXI_CoC=0`v0U2R!YbX*zj z>_uZiQGDAlMb^_@^};hq;ADYYbEIa5vU+I@TJy9D*Wdudr%xn;tc0v@&IDl@oX*O% zOMJ7ztXScZCOr9 zisuaEDyi~~LbHW*-gb;bWXl*V%nYYV`OUofJ>!CdenN;g`q*>by=iU}S;Fv(7S5%X z0LmMYhc36Ye~Dhgck4&KezIVzms>a|(2NmMJ8h!V0$xtM_lu)k6kUwohr1t#aXD>q zRg(qdnnBIlg_@}+(5Z#d`v;PPaZ(T7FM(HzaN`#8rgcg-mHFZ)LKW#&XI6PU><6n( z>Tg6=FGFb_SRZB$n`^TepN*JIy&@B}dM7$>Dn8 zZtjJ4uAZ6x<6Vv1yeuyY`>^Qb{Ql)GA>#_sAY~PJKOBdMJGZ3Q(ovivtxOIu|a=&UBqOrw)O7LE-p+#NN2eYW<~HA#hoM$Cc~x3%l@gS z+UNqCHBR>*8_1Kj!LPkH(z`!I*mx)S4m&&28YHS;1kcy4##iI!Mwmsf?~fM=&H^u_ zeb>X9)-J4>NM%39iaaVT3+8m%-?=e+@!jni&tyJ>_CDv! zYmI9Wgqr#jeJW&Hp)b+k&2z{p&>2FTS>?u3{`R9mMV;oU{YWgs2hTO9`!c-{@FF`^ z{$&(1JA})-c4%q5;zDQE(_8b0swS?sSL1NjHP<^MBKFs390X@7b7X0{NXLactsUMi z zWMX$$IeEJe5$FOh`vyjqGe%?>KAmxty&zr!X|8X)>WXhx_JldpjYO|5H4G2(3#dj; zm+|KpUsqOE#{VDez4upBUAqNpM@7VfAR?fGAVhlaf&x;c_fV87y@p-{M5U_`dO%9( zp+g`LkS?LOAcWA78hYsEZr<;FcieO2jPv~g_ZP=-jO}t~?WfK;pS4Iy_Cp$pgzZ1? z=lcACQ$a;V#2r(Sd4eBD8V`}4uQPB{ntvr1f86OgC6)<_=RhLS7kb_^nM}%Xy~@j{ zQArY~ZmtZH;+oelMOS-H*~XpoU3YS(kfprd=ru$_zDljc{(S9?Hu#o}IEAwN^Gx*Y zj7v?P7_o0KP&ufVuBy{B0itr zcb1q3zCMRN7R_+4$Rd|PJ_qGqNhA--yB>`tdB$;7Wc{7jt!_rP(P!MBn1j=QzjAC= z6m?7~m_qleMH%)_n*0`7&q?!F-Gs;28o)hzt-yH8v(@X&y6uRPb_`n+L^b2 z^WM}?ymy)-0%u1>M}@*E1^$u%Oq^J(uK3}8t%#1FsFFNxpy%gX`*H4K?A?pT7tITb zDIb(FiGC|!&tC|-#=selGnve-N)FcwtTpZ_3cTI=LHg0BxO*A8@8)@5nUbwH(2#x; zM1;At4o|tqIB{!4vmbk|ixFLNci-;iBvymJZ-#EM~uKugyC^nfrHCb76&cO+BeY_ zHXTZ~NEU^l_B7(%u}|I<^(z=D{3w+S+ZUCe-s>fu5xHuCf?WLp92XnzemhPKN|f$< zHt<%F+a{=&X{wU)EZ4xiOT@Ejix-#<`+lqkGrT>J(i-G%daWn+eh;k@-N<;@8sl> zq7|JJCel=tkP;m<^Ost%9XH3?Mc3zR(kA+B1RP~vrsI@|R^!7g(1V8T3(sM}0z<&GlML1j8u%9WY33c8&1oh3rAiS`48pw z+r-Q`d3lyY%69c`LiXpe1lU8Hj=9npl`NjUAlcr&)fWt7ZCAhU-2NZG_4Kb}CFgnx zFM8ZBWN+q}o_ipTyc2G=VpC4rdMG9@{;K`;xg+q1h|C+cbZK3Ph$atJh&bzhDCp^Y z7~8M#LKG}lJd~Jvq>oi%@}g_8HUIdamKuFO8s}@>ivU>&ml|J3(q`EUA?okLzIYvW z6prfg4RO=Z+|bvf`3P7PDA7@|qX!cmMuuFD785x9~R^>As$0y4dp!1%Xo%n&Da+RzLcE zv3_+`@#hvr`U_{6p$P~N@B_A5Lj;(XgbEvG_Pra%fQOrQ2fEvB?duMMprF<5y&A@~ zH#YW3LqA<--)FJL{h)_x{pQ4;(24OVVI|xLM%QJyqFC%cUUN6XP&1Yqe(vm*OOhIw zWhMsbYW*f{RhMQt;G9a&-v211dB=9aZ(o}AiVf2qkzBiB7S&ol>|TE<=k_}v)S#ag zuzHb~8#j+`L1=-{kHl&qZ}+k;lm8?_~WR+lvfZY=<}g0n&8`kJ2mwGUbMiCC|jK_W z;J_qS53aras`m7v3J(f`AFKCVFM)1beKcgTeYi!>Pj_=MAtPR7tP%4aIKz{`+pg~0 z*6SSA%$J_F)K=euNP?=dn$6lF#;fUMkOft-B~C%Jw)P(8C3E%E3z|ru%cjJO`KAzy zlStqB;VJiL6YoEk1(P}aCk zG)fXiy}7FVs@x7ozG3x3#qK6=;s5H);48EgMtzt5-kElHl;(ve4g!zSwi0e!4 zKkp>)FaLY&{V%ikw}0_rK4zZs3@<-@{=UfhG~5vF_b%7j{S2tqhKX5LJ5H9l(CgWb z0l|H=5}agYiS}sKQxNh=yLdb&gPud#BT{`suvQCLwe-!9SIn?KPVY8q;2H&^A*&+G z>77zj%TBZC`Sz%HUYBX@pd{BXBT^pPjBZ#c?2no{5*`VcC}t!<`!?S{sawmw>qGG{ z$?bopjnh}}`f;w;R&+?jOb44oWdmmlfwmaphOM7dm9m(PXVb%+cHEHhGm$}_4u{); zHS)80UDhxYhg(&wN+{Wger#PQy#)}SuAJDop~QxKtjXfTJqzh$ZGfOxhha%thH|i& zDaXVh^_1p*D;9kAV;r{5bg;MeVc10=X9l(w$R*zXo}{AfW&#FKb6*17b{FlpP zY;wA1qf6!)12+&_O~JEFtdXY2f$Z!Pzl4Q-)4K}GmemwI{)4uuF6 zE(A-l3BFy;P*2&hn^U$hkICJv`n?|5<)t0q(XlqdITmdtwZsYV(CXfG zMEldyeaWj&=fsP&J|r2axc-Dh(y`5meJkVhus-V1S6w2iM|Rn;>2O-R`ZmcHfW#lL ziL#tD|3ZJKA?c(YGKWH{C*73O!0CqpbSxYwg6l{zr2Y2sSSF=ouSlH7eGys@Rr_c!(=0jz&UJ zDkHtB6JO`CV!kV7d8NL57Y$l}zSe;7#;ajx)qDir?@!BvQ#6*GtAhMoJq~(=-bcf8 z1j6hzF%o*t!xGKQo%e)hThzwX7DpC~?RMUmzCBPO2^SH%2ljigme5P+l9I_8r6sHO z5Q$Z*NO8JCbFiettkCW&1?y6TCQ1@@bf<)87>086RTrhO?d2vy8`zH_%|Yk2;LHlD@t4~x2#3b#r<=K7 zRo~t1OG0T_EsGHF`$-g`SJh3XcZ=yZ#LozrKP!PyR=a;SJi?9$+xR24N|=>{Z`?C* z&?*^F%%RS|H~6^FvF`9L`j48nB(Y)uss(! z1mVR8*v36B%R%|(W>l;=NoQ57T9p*@92xABQMo3okp~klayNU9PNP5%E8C12(Mm5a zaTvVmJoC+qEMC+|US^qZ(QUaY=eWl!1Ljgt^rx_ETOvI9LM)ru%^^|+7ZC$8Ckt&> z&TMK4;z|{MBuq`)Wj8tQ>&9DT(0(G7Ak(7s83wki1?1Sz8TJe^PrpS!tr#`Hu!1Tw z`*LOi2Q(XGOMm2kQ;-j_cqmi_S6ZnVtS+R-GbLiL<=d6*K69G47T%wa?zOGNRLz5LcN<&79{03%L)`+!z2s1a=R~H`IrqrEV{PRy>1lTZ z$>T=bS7nf{Y5zx*RsnTy!kD4{@zF-YQm4#g`@3dxVXDl zJS>daNBA6B0;xFnShR0~aj}M=5hY;=6w3G z->w$YhGuX1OXT-RQl5?V+rVie zxhE{vpE;g?y!<4C+$ZZ%gW>BVDT&Mu!3eI(!aMjYvVy)9231UaPjV>~uPPo2HA?(l z@A&V$0g^+AoihD8uXuU1K|YN@7QI~XU`Im)&dU-3=DGLgqi2h!q1;Z`ono&NmPE#%}{6|bdmGQu@OGFmg$AuKCe4(4Q9>BjCQYxc?$i18u9}oR4qEu~v5An^* zRlv@STVMz+n?we}_w+01XVG7IYK+#xprE(uJ%pTil5U_b`D?@8HW?miAA}ALMoFMou&~R`1Wh4d3Ov6Z7{e+mD(KBLY!)3A@w~KW|4CM~Saprh z{PG~xrdsRIWmE4fiLE!HB&R#osT3Uy3Ju5?i>lO;RJOCg@3@!fM_(f$;wz2&OhXN= zuF{P|aiyB90?4W%AaAsrU^1;}Px>e67g}>*DTw*=2zdnhIK-*&f}uIIRl9%aZM5ElOgVGFg4>N{VWHDDokMs+CDS#|?cw z@zRq1SzC@ipV|3s<9}5IRXVv6``W^pGV(a#_dI-V<7* zf>-e}z5`;}`0JQvB_NU2qwu#$qMgvLTdz@3S#VO!zKiwG%k~dNfw$BV`Etj~NKUNwT4rhP#Nj%Xi1@yUoDH#dGreRuKi(&()|5Q1BIXTJ?VcKe2qNiUi&s2e)NieKC=L+9O!Hcww2I%>6HIM&`L*wqCX~ z|F5f$I9-7#Y$CvDUeaa#_b)$n7s15;vq`Dl|5_H~B!7>8TEvbr8UqEd3*dJaQ=AN6 zDo;8fqEm{Z4>J($%_F`$V*3Ttm5xZkx;|kKg~nbzHoTsd%UDD2qCnaFhm+om>)Q_& zY4Yu+LydD#Lp#S`qDDYQiY{d;I~MM479Olioi{<+?rU3u3409NeYOzt#ZdKlFfn#3 z!%HQx+oBr|y-owgr(4FCl<+8YHvPCUy!6yo8b+U5e@0aPDFT3QK9H$WDtrY-2)=wc z<>@>wWK+PQ4`h4oF&_vpPE27f@heZGcCp6w^U`IWd_VmXsDbCw!VM zChNg8VIl=KeHXa{ygVrgwx|g&C@3>}_v+HSjjsI`z1?a8!B;;~6O(K(CJqFM_R8e7 zz0_pGd67>Z+!UAkJB+o+12c<*fr+J*c=0?%Oo*{b(4W(09$tX>^ms(4bHS}A-reQk zy5rn}MJ(^_m%hx1q*@6daIFrEe{(6@OKCPlO=g=_A-M4>WZuy3=##v~27F|{LADrw z@JX5eq)%-;Z6VMr);MZycQRV7%8SIX`!iwC&9FCk?e^cc#c2o_O`$j)N3b2_%8rN6 zv{siq3$&y}K#M#bOcd#yl~NklOBNT<)ff2^kQ+#xG6NyD*e4O=ooX^bv(%u3AmgSaqz~_Y6GI0%d?~l^sR8{tNKsg^Nw~AbPokQSOLO>|_A8WZC){#a3C^QF zYi0Si{t|k|HFx{Ta$wDSwmBXuc`}aeMH?VkIhCMd8a}1Ks>HOJlgSsKuGaL$FKle9 z?pQYCbXFV+FL{jSAQh)bzr?yP`cmdjQp4Igczy1!rcbQ+?qut-=x??BOsFZR3-3C` zF66|nuzu`2_!M+u3%?&cb^oQ3-z&c~$w;S0$zH_pS32xs`ge(`_#CHF=Nk3+?OYw5 z;e`X43OZLghE*36lPvHMoLz zpZiej%iFol`TLc#f^XgRvVx#!uOtr->l!Xu*Y)0BVb2m^r0o8{o~*2CNn<$_yU7d$o~u2OQ?(+Z#V z9B^(p^F9URbksA~R8(zmv}?G`PVSHmG$GgHsU+CVnZ?W~?=|(MRo_j{J#MC?=V>*S z1KeZRUJ83SmhdqPlu`kJmzQ6j_n?+*>PqaxL5hX8zCS>RpLg0@+u-UN(V8IBXv}l* zL0J)EjNIWXm%P6CVfoD;XmCn(Ml_^4^`$e1G%+2AhwICm*%y3r8^#GNdG0x6@iXik zY0ti@0A1h%`4dBxaNn%1lxk)X@CbXR+(u<10pUsst{3!Lm%iuOb$+UOht|(s6|Ns1 z-rhS5s6gOvDM(C741Yzp5C$BR)rkO7SGJXv4e9$M;_xfE?yv`TF~HjniRc7DNAPkq&#a1ccI=cm||Sg;tLFE zR)vCV4Ky;4uc9xi4O2Q%lLW09o^Qh@&B}fZuX0Z>X>j{lg6Zu=%$U8STz*;=Q}U+< zQZL12KsLlF-*3ePlsM4rfx6P;0Wj10-Cl&_)SU|)V{)pNoRsJqVs9jd>q)Fw>^1xQ(+wvtxbRtj);QvfEVLOUAr zKrDFdhD<%g>^P4z7&~rtLor4quRrJg`sx?7sXjN{+ySw`*HAx1SAg?U!(1AVXtJmk z{f*fDd64;dFkF_C#SKRDy-<<8CEWe-tW8|B2Z_Uyh4_Q2pv?Zg5Rp>Kl9MfPu&f^ z-%I$?FYsWHTgLCotWTRI1MWUq^G1*9&fy4!;Ra}v(1&katJ&9X#>kq#&HXpIGiAng zh1gF&o1RM%!-l-0vftly;|6eRRIg!gqPg;6U?Kstpc53%Sqkjf{Yc6FYIo9b4m{b= zr^w$5EzSDUc28R0p*4|Fo1o64`Qi8G{EB{)o z!tq<|`iJk-Ne{8@pmf^+^x-*BRcPI#A!VVbx>1**-|P1V?|4Qkxb~puPclQ)?mnVR zqHTN7QA;1Zoz~Nz^6q7c^g7x|u*Gxs=nqaJf{$EpD38C_XumN@rbjq-EAaUHaqhRZ+cE6_-#BS$o}Mz@vp zXI}|W>a_c$YA?F3*c3Fbf|ncVND@uZOS`HDiU?X(hb*#zBVzBDW%YzhFh-n{9~el|vw$0i6I zyCu15r$`xhrjNe0yN7KE02EnO(`>sD#Vo>Hol8$p)D5bfEV6DZm4u-+Y>o@bn5W6#XFo5PDV8a@OS zJ55b(H$vAOEl_bEv<+WU?es)RrYHcwN^UBbci}nb*IynMRf^>-S`VZRmU1O&1SDa| zo=I_ga}58>>9p`G_ya+JyQ)>;%C!FNVff#X{(oZCstQm8LL4IbNh{Qb6QL~y{bTst zQA{S(;P=*r64NWp!OAKVn#^y8joG}-eTs%k<;DNaY5{kD@)q9`EAJS5S4{o3%5Et% zNzJvk1~M=Bxf43|u|xtdyaN2he|W8p2N1nPQV$UwhG7(l^je{2!ol%@FR|}^P9WM#^8<|?AplC3co9Pf;DOX9DMo3t=pK3 z*!-ou&j%4UmY;kYl}`&VeeN_RC7R(Do7aZSUF^%rx}5a{bT(DvZO84!P!ADkBprB& z3ctN|vG%YExr#u+5J7TwzZ+X@Pw6n?!}kCgcFNUx9cI5jP+$L1C29@IE20*au-!F0 zFXg1QHEL(Vl~R|^Tv}F;{?4nSPjrnVe8S&%pIcFRd_0tmYdiJ|IbNYVb?lgWQ;0iM5bIY{^+8yTA^QPTH)6^@9^#6&g*g}2@4 z3bXUF%hCk^cTc2ZZJD{oBQYHgLoA7+?r=QZd{>_d29xXj_}g~6kI+~?H6v6=8h?1W zpQ_#e=-YbRifTFrb1w$jSRMsb$O~Z)AH(8VwG}SXt7kIguDoM=4>B_=jJ7(E0O`kpNQRqQCnZuKqNy_YJ9y)c@6-#SV56+Ad{Ccp7QW@_RRofxY} za@x8({A&o^d#op?Syr?W$xlU%&ndoXz4kVPh%jy5Cm57YHg97a_02b;oMdy!^5EFd zSQJ!fwOTg{8L*c*XOXA(Yk}+AH7!2M_th(3&|3o`JFhIt)e3gxfjxMRb4k%EoXd|+ z&T4@_*48>vUgK*kLMzeZTR4)Qm)jBDgVeuJ{gtTKx`2j#T!_nqo!^tNtqP%q3M+WqreTRbXBuurwa4N_U(an*{&cuuNX%EG z3<^7KFS_290$~ASORTKx#p3nlhlf}@kKUGopr+%(dHpPwVgO26s_h3ptK)eD+@o_9 zpc$UWKIGH@PS%Kts101>9ft3d!P(i%uWT_^%?k9ypMPURsUaMdVuasy)MDX5y>Tr> zi{>oj$l5Aln0s*ZwEXSU9$1}E4sKHS*=Q!3)#hXIuAXGhDEgvvVpj{K9ijWsG?-i26yeeupVF!~e|+uN!eBzI z`dEi?m5eE8TgCn<67Y_STp(Of*|4Ep`aF1_9gk7X{DJvhYu0ShNv9b#`p`$hjVbLc*5<{8!Kp@u?8)c;C5QgJlHpvY8WFBiCP}#5B zmIYkBfxbJ;t%ITwj_XBKRSXf(zfo>^I{0fqQn*iz#|S;hT3!WZW%~6==v&Gv1e(Z- z`|4w^bS+LvwYw+iE^Q_O{Ngv->0ZRrO`L*oJPTi7#nxtL*yy1gOL~6QCznTE?8^2a z@wK~aZzqh8X5O5M29Bkj+HHot@MxGsrlN#!2UMr$nLKEC9A5o-vSgGh&sPkKRa43V z6gdkp5o+hwrgN=4uk1N^G~Q&ym~XSv$W}AwkPF_Xc~NBxG@gm zk`c32v%74P`nknxG#f{Tr8mBILDELt8tRm^^qqUotzSsqJbcG5Y4Z^B6#_u%* zB{K}cWvX{ZU%B9N49OyIqks80_IT-+qzeRS=NN1B@`?AUZa+Yu2=@}c3ZWCn#HQtDHy>A^U_&;2Lm1lU0$KNtYTGrE7rSHsghD zt!7s%9t^6ME(2XW@49Ok%(Z9@C`kX+HYm>8HyT$=3e0R{P z|9`6|&IKP!-VvBa#xg@Se{&{ZblpNOiV1-FlCet0ZfD!mc}%D&(U}_tJ1^FM126c2!55WIE@$$6T(5X9?TR`48O%8vO9B z&Z`39PXY1K-O;g7&)!o~2nRV0L0tA(8)NG2dO*xC*_=hZWSK<$i?M0Eo_q5;Sto_2 zq1UV~k@2-__ir?~&N5E}(0!4DI{vHzI#-ir-;FYt`uP)mC9+&?niX+D)SwW=oD1^; zO%U@Rp0#CDi7(gOAIa4|ny4)PTBl}J)AO>w(iT*Qwe@l<7fg+&6}!rABfJ+^GU@en zig-KE4t~zj^l6^mL3hLaPisX+*E2bSx7mQtsw7P2B;B?HBB23tXqdg*6cIhzW^eFZ zBcV5B=(KLED)au@H8VFv4usvNUxtmgh8!r!KZWOp)etL4TW9R1Rr4p=h!oiHN$^k( z#OqX~BF9k_5#>s5o%(^#19ay-fR#U*Ua4B?orYT$PS{{DUC;SaJ{#~EF>WlSdcnj| z;b5~I2o1gX-IO!aqP-E&zz`C@JDgoIox5QF?35e%UwDqI|0LDlWNkE`)|VsxhC5F? zCi)Nm2^m8Flj=g!g|lnNn$-Vf1^fFgr6Ol9;C=2tsXe{V8C`X_erE0PXzkN~vO>+f z``s)5%L3gGip9w7bljc9FUO&APt^QxCSCU8n$)_Y)3p$r|X8HU7 z8^gc8%zt1E+hlD3UEwWm&9Wy4jzVgYSNq^9$G1fggMA3N%v@)6v36~Cx698HibVhm zwI)M*Xphg{21I>O5YiBNZP?%*zmc1pw#;MgF|WqecZRhGiQAj^lpRWcL?=t97sR4$ zwnkk33|jpo4tzOc1$7PWd(2o%`F4osOQo6F^p1$XY;r9XFvph7 z=qU8YQ-FqR7}-vB?$HPT!pR%x0T)0E98i9eNOlloAI7kY`I4fAfs&Q>=)nDP-hd6? zZdSjLD}D@T{XU-|OB)#ef47tQ5Li41O$ZSkL377;`;oyfg;9UaL9`)Yi*>J5#?ixx-C+> zG`~Gz$L?zDYRw~WfbtUjp|GD$LzJXiE{fd7Pz474PP`0p%mIYUks%enT=Rm(b&NJ= zSbt5`xn2Zy<^1A#p0N$o+7TKvsQ9NV#@?sMu*RnC%>WBXq;%kh0_A*k?tV6}hmyFW zMJA@RYhB6AQ%c*gsHA@<5I0GT*};Yg9R(xdZVo=v6^?-(UzuC%=dm=uBoFo5aCsy8 z30MKQtnOK=1ek;Q!+3+-7#Zm0`}gro4^XE9z>Qov9jZ_o68!w`9Fu~P3}IU;yNplB zNa8__O;!JM64^wc4Vm|@sphi)E$yTH=Dbc-)J`d0o`5E_-c96``@5>_?Cu+DHScO7*BVqa@`jga~7Nd5|8_aCPqdL z;;YM5+yeYicFtql+}QWVyId<@Af1iki(rTskkTKkd*0z28oEw@M%uDh2W~hXh^DaA zXzR6rddm-M$38evI!)*A2Gn{S4fmaLYM08~mYib&UhAO74YXrOkdes&bTxLM3b^=d ztK3Y6w;W-c>FhZ?G!$=rdM!mj|B?p2xM?hdWQD~ec*z4Y?FDKk?6s|mMu2A6XRfH2 zaVLtzi|jQ_=`Nnt%r7kxo8K`aKU^WEn$J%@asZwM#Q<%!;k@&J5R3%!``N4-tMiU7zSns3y3g$*&5{JVE^D!a4uHsA9T zYk@*SUw1JnEWCD zKS@uUPbJYe-sH_#ZU>%C>Yi_{|C4gl@QieJTDD+$edJtly93uvUS0d-k~A~t`CaXe zW}|dvI4-8je64xWCGOlJ+tC+wEe82pe`cG7!)S^YwLYtOIvVftG7Ej7#w%$BG{D3x zM^0XcxL>KtdYGY$xkTR_$QJI9Fwd`UbtO*{x3l>!?3rv7=1xo(m^dX1`VwiN9h>@W zAdvPb69yR98+HK`mClg%UF%GcqD{I&33hOE=&lW>T5Sfh+x)_2XX2s3&irJw{A@bu zSU&;k z+#L-4Q|uOy!{j|K^FF0j$S7d-{)moM-m5Y-RuB;WMe_<7oQa;-Dx9sc9N3Be*sBa7 zX;RJ#wo6y+AFNMuEPKCG>-uz%uVPDz9VR|5a`kBXsVpQ7p~k8=_l%^i1#|1*-SFC< zPyD^e(8tpNoD68;SC%e!1u_T;N!{W-_y(MK+naUgQsSUCT_&?&i-Wb!8ms^r1J_n!92JiXpFB@bUZQw! zDZRz#=PTl85z+iYi?W03y|z0=l)JmkbAHa8&rq5R?RzzY+AdEX)b2J#kd+zwi612ElodK=VY=*V8ndXQ`$-`!vQd1g(gvCy7LQHtS#{$9EH@NO-wmJ!Q=#kMR6h)YZ|m!jdM3Ws z2wYaY`F^I9ffaT0OVanfbURRQPfi+G`v-uprcG7Awo4yHkvR+GP}tnV3act7i^v|c zqrNmjF{zAk_542Nb~J7!@k<-O3R64D?2Ris#?X;GQM1{Vwl!O}6NSJ5!p|hXvxgX) z$>;`=rFSg#@=QmkJM=t%Z#{q_%AZRdBo|f!f1YRrtU3ah55YY%;}Ld#IM)4x-PR$X zjV&EobM0%omCE=}g{UZ=rs`+H<Kb1o!2h$>o<7>;t1{~_pqQ3Ft zf0(c8q8t;IRbCQafwJJ++-i#ai_m;j=W`i3yRS5h4trBl{a~rYhbzxerxcqlFo08T zkVQC^wr+orwOo+?f$8GFg1aP3^*S=shhAJa)HQMUrf?}>vOtDx&Vf!pdz!FHExP(Jf} zzc!E3T)<8qW}QC60tRMoN9W3bS-0D`d2J4o6@JRiJ&_A=h6#ia7|szplZFFEEy~kd zqhU8rUT=$K>&I`<`?TbO{eE4ExNQFyX#Eoh$N~vG1iAscH+q%^&Hb1^bK2yc9qRxS zrt25?kFwB#Z#E~2!`@>WU2fdgxKR=r_UCze9u~}hbz9o*>eP#jC0oV%O@G8779t01ccQloB7WTA`$toF<)U9gRsL1Gp z5d+}MRcv)$!W*Q=!`G0?fEtf4-FUg@Pntm*lgkANK+?oCz>m29un#{j2QEyNVF7u~ z&B$5H2{WLeJR<{=~Zr$BI9n4|>>SAu5qsx{(vF_LhH?5?9g)G3u%t6iMJKt9< z^$%UXOE;h}W(Mt2FIoU3- z&G3{BYqwwkYiYG9!1sDaUl1w8jKgs)L_LLU5&=dcq1P7u zL!+RE%EXp!7T~QC64CaBvlLFV*a;0Bu%iDIjAW!0NNM z7nw`66FFk)mTUGSd}zOZ?j<|!y}L}{;~l%GQI|RpzHjY?S)q#6EA)vN?_oGka)fs$ zh}U_yqBojrCDhx{N~LecGx%;g4dhd>FG`RC=4zIDJorSV#>-O;?3wN_4Vt|Q5o2MvR-g|eWTqkBE zI{7cU4cJ$#{83}|axR}01ZJE@pTgOnFYo49qu+uK7S&9~|0~@i@42cSC!1d1!D0mh z#y#^+@_wMbVxtqDVO?Pr|BPFE7hR_Kw-x}Z`@5HrybLAE0<;sLdBk@B4Hn;)EWnQB zsd%kyCl^xYzdc|k-J@BOo%Xr*QSdL7{sbRDadaL|g-IiUDVAW(7G`Ays%TV&r?*z$ zR{@1erL3*Yz3Bx#kMPhFn%_b5+q92T%=?nfX*QLOmLULR5&u195M{RtP(wg|C<*^q ziY3St^=$+7x!PXH^lGg$^X7(TUpUUm#fLY1QzOE6E~my+$jnBe*1SPeoXR?&wZzThJBOQpK`~_g2##W{EUU9fcn_4iKl;-kiOF2 z!!M$4d09@kI~+v;d+nEgv^h8P!%3Qoe(feP&9Dp#5z{Fa?67_1eDua0f6K~;qbB*@ zPJyg*`A$vkM0f;y=cLh9)j6}=R}!d>hF_pATK>J4d&#w?xsme6eBa|)1x$AS98hI; z#9)+$s;_VLcpH*;Kk+gla&Aco{f_HiqRz|@2EYH)|)Wfv8H_~xBHNy1|3d+!I4WCZ9G4apy-yN%P1*aNcG`2>?iJ1FB&Q5;v07 zQ=t)tec-9uGC-VaG(fyg#jmrgA{*nAu=$voMj4ja>oFM%*<7%Az3c*eUbFV}H&t#z zr}I4k>pW9y;>VU)J3d^i^^N;#IJ^d7yl2j7jn;oo$CB{C&ZJ8zN#pCSg=jXjZgzWm zVSoevR%9#U<=kxFdp&j~*;fJPciCT~8BGSu9<;S}q30Lq-m8Q0of$PWiWNxBIpmWKTy49w~$n-_0*X&MUvQ{=*WO)cI+q z{2vND;GmTaSFsriOazo!1%;`_OQV^3!JG5SrRdzGUm9#-S&f-wpMM|rmNq1sln8T= z4*eEi$&A&xY18%lDfS}{F&>>j* zN;O+8-DWnZ9h=EuhpB-}i}P9q;#KGUkOfQrGobFRj%zEOvAAtFZPP0ULOZ1Qc9;q%y+ z&+?x-fhACbfsr(O<{KEx_Be^dzQ3& zo)c31hgaGUx!Y=$ z4Pax}LkD^Ib%;GNe)QGyxZ7wF(ggAA_Llh_b*26Ks1DJbSSz0glnE}Z^aot=qpRMj zWp@j#tz)>~)mkbJOMWwUz!u>8XDKr2Wpch|h+=KgkF+o3C~Dl*lI3rq#om!ZZ^X!X zFC_1L1c_a`Qg})Dcsep{p5GDVro7#!a5<%rk~_XiwmyfzZk7%SaC*`b6&)rXCU|uF zh|9gv+v;m+@7J4knR%(DNwDG~RoO#Nc488M21(x({>?4CnljO|H;qDcJ7ZV}+(3J4 zUIDe>2?pe3Cz-8OVLk7kuwtve%7|Ao$s9-(a=S8eL=B|G>CJ*sDvf(HS;}E=>ft=L z($p}&EAw#DPU2T<0n;_!bm&lV`(lU?dEn}e3@>hvt@7vTLhe-P&V_<#LQrV@q0`#A zf+)!Jw_vBk_oI#Tu5K(;NlMH4jkVH#4@Y6mifcnMOmRj(ehwA}a$LDX(!1nA! zslm9;vP9G~ie~GV=SPM>%Mmgr(g7Q5?HO-hq%&=K_sDj~(9wXr!`WuLUmMk~ z(!QHhViF9RK=HeV=}rD6TNrabp9bEnJM4VH<~|Ozq|`|>pM+6Ry|2Um$~z;KD;AaS z5*}48Hb2!R+c$>j?a6(uS!i|^j$LmT@pM+WS@05GB(&;-FE);Qjq)5!eyGc_wR!!O z4ETzIdiY*~DD5=E0zt1n-=6+bpIfy1@8dbmuis35Kk%D7m;?Nv z7g1ohKo-*LNr7Mdvbpa$FS-@P_2bxQwpYz4ot!~Q!u8|CpRM^v^&Qks7?g?3OW+&| zV9Xt>e)C@NLKK>6aQ+L&=yu`~&5^_LF98x%j3cXlwHuZoRObE4_n& zroy~E(Q9AQR_>`^&CJ!Yj^LDPH`mFmm&|^LMU&g&N{mdoz^vpIlMD`-!hl>MUP<~b zmS1jcxka~q8SbnsAw>C@UhqPU-W|7k(TQZbuyDO}O40t)@tsQ5qGtO3(3hQWTx5)Q zh_&yz#vVGLYre`7X!*6bE^I%ovufpOI|^)#W}E3&*w@oha4|gDGzGH`?x&9<>*aQS zebZbUOdR%)l-Z)BIM`ul>?$aaE)69Qfe~F1H@b74vuyX~ZVU!5qfwd!>*ujVy6f&rkom)Yae= z4KCH|Jc2>z@e0XG-gjtHtzIxvn9$%CMIB!&x%{INhth1L10Fs9<{*9d?Z5(!x>T!7 zaNs3<|G7tV*^gIU4}M70BUW01KUUMXN%hv!^Y}$mnz6Z_DBoZABH;>oP7a|N0}$0a z>I3MiZfPB1g%HhWhiNiX>$GPM~m-(5<>DEZ|nF#;@^JRoa(VTRqeoL35 z-*UJScgJ_`^Ge?i{=@C2B_ESkZ@N8q7^zk%W<83B(QK~p&nGr{g8k5;2o|TiMkm5V zU9cV=8kuze+wS;_oPJ|$udj=Ct)ErXa3BNYXAXCXv7R5)f>x5n>Q&!kpm-RfPdcX& z))mLsO0vT^_5}Ao3Ej0ZfPG7NS+0Hjt6eQzfOteS+<+=RX?XjZvXIOOLIWaOz}hcn zNiR@LSMq~)0u{dB5r-QDgmu6W71MEbviRD0tqVxXXSJmFy;`&~bzo3B0f*Q@TqQJp zDCr$H9o4wMG|v4$?R|GtQ&|_UA_|TL1QkcBqJjlb5KvmghEf%j&_P8|Ne!cxyfWa3Q(5XP>?I z*=L{M*(H9@?ETz?o>K4jwoPAlw1v5Vz~XJk*U5VlR2uXfCnn~S;l2l)_03)hYuK;G zz06F2X;mjs1A~`8VM<&W?QXac5>p~;q$Z*@bfr@HwN-FUKH}opqW0W~p|fIp-G{(J zlydumwvxNG$D-sqE`BL6JCDiA#x|tYyj+|FH>%>?r-j#Cc+U@qlILOZNqZ>9$}-sYvlqGwg2hv z28%6jKfOr@s!o&)DiZfJS!7*1j^}K-aD1OJ-d8E5Ow|G=e|p?nefh}^8f_W}vr6-J zZ@;Lv0^}WtyX0}@jclF)n}U=*SXi2 zU34dN-Ral^3&`DsL{-~DcT4@CIL;TUybi|oe0P!g%={S2nD$$-+?ZAc2xqRw1^ks@ zZT974^+)yg1id}Rw;!mSUauQIUGrGyVDD`^eEs>db9#gHkz_cc`@u5K;Z<4oq+g!1 z!}aFh=QX+d-6Sh+9An`k%u-M92f*N)-m2?o@q8nR_r0&Gmn}OongZSx@Ct0lwdRWN zTkex#;BN4pcg{=~-Wy}>3!uKfO1|?vqCF)b>&x)-BHJhJt69^=uA9bd7G!BA>n@x^ z!Luax^-E5!7;fRbx>(BId;FjwX1kx~PWK`1Ief@JA@VA&8i;wWSi`yMGpE$*hw~h4Ucm2G-osdeMWx0GP0r@>?*$H~ZLVL=wf82=O*gEsGtIUW zpnp;v2gg54YzU5@qrwi+&sinqv6QHx-q$bSX;!eHg5fcdXO0e+*4p_VoSikAt$Dsi zvS9GV9q)$lQ%*|AXl)_23v9%K+kvh8_bURlZe7;%;-6^9t#H3yE6D22Z7kk76SfZ6@_S); zr;n~6PPneIHe)$p`OxONT$u7j5BBDcoSv0nZ5@4%=M&tMnc~DJbM1o?tn=0(yBt5& zo#{wHrt<_T@k%ZYs(LL0`Nb{)neEqAR2VWMR0|mxAuWY1*YN(Sy^7l{yE&2Qe+Hb( z-vzH#reRm&+3=f5-X{06caHO13j@Xb$+}xljFqLih3wj$BXw2>%x;i9TAW?%m_GFj z_6e8}9Xqnms`W*?sVL*XT0oZv0*pO(Cy(c7z>WE;#}OOkcs|$zAx+f1VEX$eSDC`l zdmWaiva~{q^zmUftKx1K6~T>5(27U3EXMj=_vz$WiCDTdJZ)Azw+XA$UUI5lp{hPUIZHgMV?lydtv|EZdWN53Kwmk6UlqVoU}R#C$9Dg*n{O74En4 zLcsH?WZa{vt-f`k6Ajl^2>TmaD-Ud->W|$OROq4`YS)(&KWs^bafUb}m43*O62MjO z0{3Og!Tf|qruL__>BZt+4hAuXs95VWy#&4NPHZ`RAyc(8KciR*#&&JIeJ%l3p;Ms- zfBJ3&-XK)oay9SS&8x;U7xNLcgGdo+{9`|2ZZ~Zfej^Obo=yLNF?{AwXszaKS%(Sl zyYj09h{rMNGm{hNIG&U*;HjNkpHSpC!<<_I;*b@m6%Mxz1kVQ7GE4k z{+qx5S=q~F0vx&qaiX$>7JOlzBDDFNs2f3CNvhe;yrXF-NHq0Zpmx!bcK&xuxnNZ$`}(Kn`@kOk~O9=L-68b2t2Rw#Y9cwSb> z9U{|Fz=Qw$^%cNel+;Fs_<`?KXxIz76fp>B_AH4k1b0xUrh3|na%&eY5()zrk`S=`;4w|(|6ur`ZIdm;F<;*b}FjiBPe9} zLlz!Pp>FZ%gj`4pEp}5hAu6a5VxQqd`ozZXAzP)!5Sf>g#A>ajx z!DHbmRKVcYgXY7bVb+hyFjV>PrHuNV6A*QZK!qBSn#)~Sz;G!Hez4y(Qyv(JSYV1| z^rt6|bb@1YMN%Lu3WBysUt*3917>{jVbjsD)T^^B)1&w=UCzaWEPaB|=q^YitA(9p zWEaQ`3}$01SqvtaKW7}q_w7D016AcrFe@V=pcN6s?*eqder?Iu5dcW0oS6Xu;_gUb zUaE!pr0QzSWxTVDtZYy@M(^PKm7Dz%ln6~PiOKJbaVw$R_?bEmYM1@7Ucby zOP_EA6UMI*u;dv5%~I;p8WO*269rtL5sWN_bPlY*4`0R-`Po}Xm61D6>=Q*F~R=y-5pAB@dq}d*4l}g zvb8pA06yn;r{ETjz3Hze@du4)&MyWp`MGKqPv4K@^*Ww6f)rT)DuMMR+ZcxtMoRfT zzt=@rV@B5_`LLvpAXUIMWkl;5&r)0KKc?z`rgCxg`@1SBBb^mhHUXXCNFg0BR-vB7kXM&;AQ_(w4;C75!7-kn+$=J+vKPqWyWnXaHD8Zgv_0TKX3E|D=u#;}~3ck*)nRNh4m{`#LtF!{kY9)X$4 zC^Amt-UOy+oDte`5rOj-iNAPK_+$KrjHmty84IQcgv7IB;l2y`R37J~KX(`tV2B0) zPKZCZHod(NJ>6CzGoS=Kr|_o1kR)a*r^%T&d%5VuZ|f+0PA*)M?95LA4xNKE#pqt4uxdGM+_WGa=@&&*gZ?+r8p1?|B6Awa`voCOed z{rSAuM1`=0=y}F&O@GW$6xZeWuV4@@e*y;7ka)G2Re@raXGMEyXh_L63DUKC;t=wU z?undjrj8_zapya8ftKJR+Sn~a*u5RGM);WLXBh|72~2AW|B(%UUM*uTXM?B*v_J8u zyR~MlV@wz<2j{7s6MXpdur8)-Ys&U4GEt4O5m;aodrSJ|6v3O>h&B%%C0-Q+gcZ?) znw|N(aW@cKg@!U9x6ssci(1H_1aDF~5D$hp>4jC!`IMJ=^9_0u^0&)T42WDQ>ggV7 zv4~Gn9?%L1L<%R( zSrm;h)%f+zm@-G_`TTmzZLjhJxQRf^pgW99W3ki#&-nzH6()4SJ+y<)qVmFWb{II4 z`}rMhxQH356y{NThSS1$H1NTOOSS-_R4$4i#}d33&{G(zv1a~sE~t8aENeM_Ag<`nKGO=4K2Wp4Qk53>v+

1s4)RPwK<#{pTPb$MA zrj6=JUh`#R{naVdBz!SYwuQm-pgqC&1T_Qm?j_R=K|IYe%Lv29k?EJ5rI%ifQ}yQO z-9t!PU)cOxcKGZLNJ(y6bfyygw>B`pfmr^tG^zI&vuM6omQ0YzqtZg}HmgI9hpp~~ zTuv_Gg2UB!y|%>zPz-nkvW#oimY#(uHUXZ>kS0H2TmvU|(C|$C74&ZcO$3wKYw(jc zzGgUOP3Ui5o2I{*SB2j$@Sp!qY@$6rmA+(&^vjb+4bKRP@&G!8>S;gxs%mP%V#mkb zR$fBaT2^mfG4-hN*l`=@Kf>-*R=l|9cH609-+;<~%CQyWhYt&>sI7W7er)r@0P*|n zurs?8C1vN0Cf?jl9E*z)z(6qSaGLUN0pfzPuoD8do((7!YqNc&Xn!&m=k{%zkEQSrHTLSjQhcZ-XYmF>{Hmp000 zw6w=d?6RCNy$2NZi@AmoEekLaMnIMmuFE5_u~Es>6ZSdlBOK~oqDK`H%$3~SvdPOV z^T?y6XZ9F4+1kE$RUbtu$ZO&azna=#bF7*Xb!{ExQX_OVG&S}0(Q3)5&8|iH`8nrX zo|2_>2Rc1VJaQ~m6FdFN-d??0c4pT;4UK~b=b4o9AV!}rRLd^$%Brp)hPk=<4mY zdOFy;dp7>RV)%>JC6UjUKPl}ZgD?KFJ>j$tv8V$4ZdX0ct|au7HUZsxN#;JPysYr_ zp7nk`6V*96%4z8)IpCwUJw1aH6BF9h1CU&^di3iQ_>_O%nS16i2fMn9#cvOv^S^L} zfG*TFy@Uv|$DZk|r!ms?YA+V=uCF(_d2=S&bjJ<#)9Z8E)80gx9mAdYWb4nLQgE_U5Op`%&wFaNaCH=M|$M6-U?&d(U&k1K72`zR;Fl;duKTz3#zbP9?;)W zh;-}IT}@$$dq4Pq=%A?eYS5SeEw-^lIe^{42AQr!G?)p$PsU1=36SH+ox>en*~yQlsA%EfdryqQXiF zloi_B4Rg@iN1!<2{6l)3B`OV}SRrcsBPIzavM8&+lIh>9Mz0fC(to2DeR6IE6e|{m z`map-|3bZT=>IvPMxJT~?M4vPvN)t56Uo2d$Im9$y?Zy7Wd!Ewgs(pcWeOFIbPOhUaPNKhCYE$j9t^B^+N117ph_FN88A&rf$SDPtWEv?XahD?4o|w7D|CZ z=>G+MX;$w^hd0cDTtc$#>lb_Wz0Rjan`O)^$d-Sdp*2W}l@1N64z z7fA8_pl6Rj+4uB?M?QIjh3x(K(MkPaMy)FxZu0j0MgL-_e5OmY@M!0?m4C1MuYZ}_ z5;-{Gwd01hJWO@tFRgU3leN6BKmXgm8cDu_+T8^?qQZg59%N-q>*ahzRLX<&w_(e^ z?NuzUApQt^LGO27uoUzccdpz!S-u_09EI*tpcvyJ$r3d`L|B6`4wYf7F@j+Fp(_G+ z5~@e-5`nWw{`h2DRTTqLX+{b`dMV$fH;|M&m@pODNdoKlvQ!$V4})n&anl`draF<8 ze*FAPBu$452(5`61~-X}o1;qG;G==0tBw%QR4rY98^I2<%posq3`9flL*&#U0)=rh z5y7_aW1XCwaTDr_>so? zHI=A|!E!02Iar%4bRU++tZAk^W?GVcW8g66nlms%H($9h{kry3 z8Vt#&)+3!Q_T&0a{&B=8+8&KxkT#0LOoA=UPlN}uy671J#7#2XkM&KLi@V+#5h7kP zr{7PWaCi`J(bro^Y*oVM>MitDlLh!OZRumQJh*)mc%!_9v?a(bXErQQYKOwqx5 zeu#{WJkTEaE|xkk)=F=r#+M>5PUe8GZs}D2h&2r#v&=VUm`lOpGN2T8asU*#BOPJJ zdzz?PqgJD|4aWCQ)xdCvvEGtj02;Fs-`8;5#3)SBLwp@J)jMFE_Fskt{(p7ufN%$lyxGuGUj zPB+o8ZSD0_w#%^k?z*cSG5+VCdcL%vdc~^GHaQPx{h=|EPsycleq|Ggz_8WSZKN~9 zt5-VU>1n>rAqf zSd3>)s=oVS;=(K3fF&n|UEow%R6oV8{c;vx!)IG}J??Tx4jrtas8G^l7_KN@lp|xX z&7MrHDjlB*O@Pw;3`1}dYxX5Rc$nI!^8{ZR=fyo?=7?bgSBSC)<&}cGbjidnZ`6PV zfF8nP2l{do!WaSEg*B3oo3xf6(G7e+*{px(L9D-zJC58YAH;0S(4YjOC40M#f&w|) zB#YBj6a)FnmDM&477&vapPLeDr6U5nsB!R~bdJ?y^x@(1`Z(YdD||f#s2|7D@<2Tf z@6qvQO$YPJap>8MZ1OyS6w}!|J(L)@g$Oj?#L3HgA>%)EoQZ7 zH>IJqsr;8`BjmRgR34O3%kxI-+YQ&(^pX|&%Jy!|*grWwqit1KHS5mE*d!A&lMrZR zeHn_zZY6Lt`*K%zV{CsgkmOsTr}H24kUEFEFlC>`z!Waniy>0IGqm;|*sr4#y1%Iu z3|K-&6cKl^LbO#M#7iWZ*C?)Dy&u;V@aj`g0a9(Lc|hQrsE9>YY2m6~RhHMNKhE z+UUHrRZ0W0a=p3nGTyKbu^jTeNKC$auX`ybZ2axV7=T-szEbxvASb_N7{Z&JTM3G> zpP~}LZHt*IWIgQ6{8ZRBth$gz7(e!Q;EJ_E3^5RuD_mX8BjDV8_%ER~V zq=Gbg*gn1XX9ZD5YMk_ETBvr?iZ6g(A0R`X6!5s@df*7nN)lrdE{98?XOop5lF+n? z{F;YJo5L@2-&N|#Be9cFqk&vSP43&bSZAa`&z;OWlDYjrJ-uK};DyMH$$3xM9VsYZ z1GVY|o&(utUidf3+o0=n_e;?iO{k?GZS|uwnoJXEORxvTUN1~o{-cv@kS+Tm)LO0H z`6I7s?J!kb50YZakKCs0@@HXY5Dr(>C(wEM_d{j$3{o^5l48UUkKT@*L@0Opqt7oO zqNoDJ`@dvUJ+Ccn0Y&j4{J;8npkF$wI_@O-wK^oE?#(PVtnc*ni$G{E6ofgTIYKq3~cSg*15meha9b27{mq7yA3_hCR-NH1z)_l zC80LEE!z8uz8ciT)qX#;Bdrpk<29_T+S=OKcWdWOca#(Cc5M{<_>`O*dmJ3mdKb;| zvPzP@gUthz90?7LgoFfVDlv@RBnt^s3BO)9$N563t*w32(BQDEPtLIM5HMO-upSX5!%{X zH?AI5O+mkW0YZ8HdF@13^@F-8scC5oL(C= zA<<6-@wh{IHhYqc$7$0;?~bc=k+pO^EI_PqZBGFnWR^A7dUs;o&=cqhWUd4;eYbbd zx;%(IN|~~!xe16AX6b31_82o1JM>@G{>l$O^gyWbRu4#I#u~}qo|oKX+<7AOgQ7x| zrNwLenfA! z+DhspD9sbz{v#&+X3RxggV6oRO2M7YzgpPJZQ)T+q7w&&`Y#7*!NLFk3l$V9u)=Gi W%r8Mxf7cT5=j1W-qXmaue)}I{>dlJ) diff --git a/profiler/advisor/img/schedule_1.png b/profiler/advisor/img/schedule_1.png index f1432a59589c128ab3bf7f2b9a067d47203aa06d..a292cb46d43b1b0d4894638ac1d3eeef5f1e802c 100644 GIT binary patch literal 28123 zcmdSBd03L^`!{UUV!Jg?S-D`-;-smSl`AmTlw*rjreum#Zk21AxFYRNS!#=#p;BXN zYD(gQxIk%&JDDjW0+AvrASwZ}KWH=G_dR~kU+?=I@B4e6KRSf_zPYaRI9?&b*KgQBchH?Flpch=LVl`hSbD^L!*ywf{uYldPn@qI&n{no>UpN? zueYA>ym07A@t@v;O8%Kd_K4E1=S6TgVc@x6>Rvzswoy7{0R|@?it47ub`e^ivw+#W zCmMwwYB^FptzadyR1!Sh}I5O45|P z%V9xy16yawpFb9I`yT;EiI_Z`yr*Sp(`>~vW| z%Yp-@YK4y@4qg?BAfTVsvhuV$!B!^+=IvK!@KFA3zAY>1*OhmpdhVi#K!ZS%y^^Xg z&~c1be-yC0XAYK`D$dR&L-Ndx6xZjJy~p$YJgnAeUe&6bXbSalMQq0Lw0jgx4;F6B zJ@Jd35SoshNC0g%xvJ_lof$>4YizLtx(ibs-FBOA*xcCpUfdobOfL6vI)a?wHKF9g z6TkTYH%!IC66iePgRCs1>gnm_%#}D@r`h(1Bi9>_rzKhCbeCn#$&0WsV_{nXdl0F* zn9!NDgy!+i#L^;uw2z(JsamkX5X?1aiTJc#;?&vNk+K>0PuJRDOFxFgEbG6M?w}rE zegmmmBKSj-(WY7VEK+wu48bx&1hNvg#UGQfJyE@l+S4pL&XPZl2G_!IaG)+Hw7qoxDzF@uvgbL$bAv_-&*p69a|Z zVR=(Y@{td^=OH=Z%~OpPG{C$P-b*$c_>;taY+mQr>#yQj-??Df5TaE31go<+YFb|vRZF`f65lQK^3a zxoBV}kPZWc+OVuvj7pu_h*cZfFsk9+%jl_Do0vv3ueI}&zXKj6o*x~W&UW$qaD_TC z{fQf2wvmJ!o&vI_ni^^&k-%?w0nLS<)yiJrFz`%T-eecMvOl|_Q{}`7FE=I z7_Pi-hxp#nlC~L9gI?!PntECm6IUc0u(~R+7LDB3XFg`cImn0Vo3aCGUA3l)XtwYh zl%Tb^=`qg19}{Jrc9tB#B>8VB0m)jXF`uKqna&L(+2=%YTXZn$OLhpsHy2}Tl4Kj3 z+1HDkTQQltjON08(a@>5UMt;3wfbqc6T5~?+yst8!fVUo0@``s05{)e=B`A=T%XyN z>0uAnv=RDo$1L84X{IklcTrfiLp3mRg)`#wm`xCI?(uYw%(2Z{PpE7&$d6F^%oS;H z@wuz!g?nGKS^9^B7pA8%TKd`JjU3>MUk(X4B3bZzs1GlnJXnpCCB<{-G@5;6Ot?lmAGHHj`QiF!WMhHgdyA3OEtIH)5 zYO*6S);!1c5Pcy~O+2-|7woVVG4eU0+huB{UT&CfON#)p!Qa{^Nkwi3184nt6MWeW zC`mO58#WGKtC5u>R6U@`UzN?t%wbUA7JrJ5d?4)($}ZZ$fk}uM`vPMRdRvDAD}X!N z@=Ax4M3*L7Z8rtDnV7^PH;YBkM+++s!NZh;+ZI0ZfzCgy!SLjd*#7>gzW?I+Tkm?8 z%$El`IwAZ2H+rP%xLTA9`ei||3+}Ml?!KbtiSWb)iFFQs-oc93TKK3wqzgDR_rssw zowg!Gs1xlpT{i6ODDID6j`++r3X#^zG|de7$CYicSu9xQI@g|ta*~b!RLd7GAMDsc zidD-_#i_aAg&=?9QB-?4Ja%itG@7;W#QURlqk1F_a!w^oT6jXoaNXciV{x4pzis!@ClYPQvg9obqnsC=w+eUlfj z!jWdMtR7F3`NwWFb9mtRoysXZX$?uk5K6Xx0=+X;_(2j6dS6@^XKhwm9>H;;jA>z_&wmM}I!Oz+y!XHUDz^=MmeoZBgw?r$_)!e$=3)6>GwW1L6~H# z98U)4kj&~kZZ7@D+a4BS;*b5~i(@6X_>;N-)ZA^rKRIU5>3^}VbdjN_3-N#zE~y{Y z`;`iPLbN+dznmzrWN*MCCnD(vB#8(60A>!|GOJH+$`C4^PUFB=C5q3}IE(TX(A?Ss zK5@|^+T#dXD1N7c@yZeFH6?0$9Ngp`$s3lwifHN8yAv~-Dafd2>HF60rZyc9cgTd} zdrOY{_77O26bINRW2geyF!9qy%n zgK$Z0_|@l{G(^^pZUT@F!VZc$q<2cC=6eKd!o$~~5Pg_C2GGv(Bn@#HE!!MeOrg?MsdQh``5KI~q+RbW)J0G=`)zs>;t~3oR=96h zTe%N$Zt)>R!jBjkV(u(?iEcJ5_gR{SZJ04;f0kB(nz=NGMP?I#_Fydn#0liCMQ z)kY@NP9nt!8;^vVTLNOtaEG(xMrn9_i}|`lDQY#f2UIVS{5IH4 zBomAg2ql0jSwePc3ZCLmLan@S(dEm(+FaV z@^R}qUa?rtVCw-^K^t(gw_Mohw%g-)BbcDHnpt<7p8#5C1z*-{pDn#kVdbk!E@`?O z5uP>MyO!%du`(8uN>w`gVu)9NAHj2E}=;oChFbs401td4AS-+;%d|QwPUDXE~^&3`u@w@ z8`C<~5~Rf!Sa7;AzSp8D7<7#{tGAq*H2Gp0$GP?dyks~^HR*8x3I~*5`HA-wfHp2t zKFG+`rBz0nJQ}JQt)2l~{&p}C8QaW7i&T<`ZoXaTI`xk56{yU(c%ylrLmW?Mm?goE zTC7u}r-dv=B2#3-MaEVC$(fxlp{I6ZHex#@ndl*~8)12CFpwPzk2{Z5{CS&Go4X99 zV>6H>aN)U5dD0D-M?mZTAn}N4cWsSB!Gd9j*7LXd5}xl6Csec5)avl?m^D!Ts!lKX zT3fEt`4ZU~5sb-7hPq%e1j!VyIiRzsJRkh&BX?%r^atYmkI$}^jimwSWqfz^7T<(G zeXzfiDZY?m82MuV1o?I;xyi8na%RBf zxhwqvcd{Bl4ZEgpS1+Rpilkg&4pDB&UV~@-Qcl1sJ01<_C3&jH53n1ShkU5X!kV>NEgiUaL(w{2=<^c($-#FFHWFTMACGgA zTFvfqmK^uppteIMi?li`L0Y9rvflhONm8anoE!pq{OOY@y}scd7qvaI$QhwPvKMF4 zJv2)X$#RFRwVX%NG+-{eo)-@t#MbL;mEq9F{UqO!H0CDevxKD!Ihp^&)Uj0MHgavB@EWtmw0;A~ zdeRVFpqhl7i0%G^2kRcW&OAcQH+8^COzRQf{MaRE*3VX`(5-XbKp4o8%S3P$%>a@% zDeSmwRy9(jPL`L5S3W+Jo{NNa-iTpDY2!jGcukhNzDc}yB5h%@ds0mu&Em-4&*ff- z;N8-i8px_qrYfaAXc)C*Q{tHuIdx-W(6ZD6=Vz^EPp-lFnT4;R4$E#v9N}FVn<9Kd zB-Bds79=t9HAxd2|ILtwC+7 zh@p?R3kK@jY9;|Jd10VV&8fhd6_b#b0MyJ|BS5KQBD|^JmPAy= zKb;$X&}lEiNmWN;yzP#4aH|TeZ6WtApx*c&@h+#MMFa8t|s;|EkeD&j^YvQvA zBP-H4Lvu|ge-y)zrB37IqHWf3+R(wkTWKObtR<9vZjCF9b2%-zf2uBfUh~(%uW7fzCYDh^C><}v~?Bk zxUkGk%1XL~9D8sD`{plJDROj2^P~46#<^1Zk_%% zTz9-*pE}VugiqFn=%sDB;xu1<-(IC75Pz>DThZH^5J+u#uXl!c8IjY|H#9%}yJffd ze>mX`Sw|S#KrD4P!W#Svh>Esn9!?*Y%`Z4z`z!>FUBMX_GjhxEZ z6yQ1yXp|Cui<8I79y~g^3ddFN^g${dn=G((PAiLo1hFlK&B0;yyCz23kbsrpv8?&L z1vs79OG@yFZK>Bd`G{rX(dL%@73E&4H>1aWO?=go;3o7`UKs%I#T|Z7(|F0FXEHVm zp!&VGZq7W{yBXsFz7%&PD(MS3CwH5MZU!22F_~w325oMI6qY8fj@o?G1@h+t#^En= zKG?Ti_(zX=ZowMgZ&`ucT=4yHdKSw!Ua(MIzK>sy+gv0DP24qr9T5B&Ne6K#z1uEE z9T!Uj^ly~_U?9!u_rY#~0}F$mv-$rE;Z>Eb#<8>*0+;bmAuI!%s7W<8Z6HZEs?BrL z{N`!#!eO}LyHi`zR6T1ASkBhNo3$m4CqM%K1p>3O@SNq`yrlU0TYQ5g5qerlL7A8+ zr$*S#X~q~ZBCphGejpw61WGe$EHbMAYX-L!B=Nl2Gd4j@T4=c~aioI0Fx+5-N1ivQ&V%U9Y4BirbBG@5 zgg3YOV5M5h##=Y6n@0+)3+&YUz|z@ab86Wv!<<_- z%QTmk&9cmO%V)p(xH!p(~K$+Boslv&kc0`?2ReleFMtVfpVa zzF3X#h${8Lcf^$D;8#>oW&07F-*54c83jVqvZ2Bg#lAYesuuwsKTzF4?+mUdAuL1a zX)pA>Gnk=8g%%zGCFIW12Oieya&s?PHx{)(kG#JZyzgIeU@?Ee>_3^-pZ0jWqI!Yy zwDWoK&^Aj`-hA{QD1GYJf0=aS%jv3-WYT&QdpUc$z_!tGSTj`m<45z*G+2X0H|de& zgAy$*n5+kUQTD?wk~^gkP(vREnzLL>9~jLmzh-r1sgYfl zp70}4m*%k&UNA#=urNl1or;2zWyL**0kdI^f&RIW0KlK=D!PGho96|?B) zq{C5cGFW~(R?A<2m0GNuq~e7f`Jthl9-!)a?7u$76}Fd0Uidc-j7${e{rnFPL?|(> zmwzAw)L(V~#hz*ZYiAn#^(L__);+QiwLejjEn*YA{49Z@v$fM-RcthXxYa4078VQ> zL)l%o)9!^M_f+zKPu!W4qS(PpjFoidvx^~&EyTotFaR_;Sr~VSJ}euTHNCM+TL>!+8gIRI)r}`1gN)liu4Er&_{u6HUxPf zG02x&)I>YH!QZ7PZ*>ui&Y-aMaMG8kA2t`Fqb>9vvA^H{ZO|xxU3%CfJtjQ)b;e;M ze`wDzfSBo&jU2ztZyX2wP@v)O>9A=IBLu;WIU%V&u+}v_cUit(gQ_(Ij^wIIp=B}SKaN)h&)nj_C9Q!}AEKBw(ey|* zmCUXT{JqMcZ_${Q5LG~x)7qZ^$`i3)kui6|fyzIAc>>;Gl~ZfsF<7!9XF9-EYyac zom(ULj#8yZ)l`>lFjk6v4NBp+)c34S>bE&HHSpAsZzNy zHwdimRVv0~Xkdcbv@YfomU$%anTS?d)J$N4R444btC45$LMOyNCv}!>Ab#HFe^1Yi z>AdgRW;-`J)tdmlIMY=r)!_SacFPm_rEDLI_Ya)hm)7=zxM8c)n3_M5R0I*biGi=V z)rgpr{88$YA;Q#MBr@B3^tr`$4FJG1&@>yh+Td7pjw7H40BLF)b^R8OSMS*)nuf7E zXXM5sm}%tnR9`QVl3Wz-ZE3MA3(DL=%*fnzV7laq&`H_v2H%}az*h=<#gwl}eWG_8 z{J~p@(aPQ?2@u7f_)SV(3FY>dB^NMt>Bo}h!>*G_tuyboLdhEjctf7XNe>c;iN z^js4j9exh@%6Fv|$%&UdCJF+mLt@7TDYLHy8R`%Dc)3YXCOQFnX$n8(X_ z_G+wCAcw?R%swtcGS-<8%+!AU`Rpz_I&?l3UtW&0Ar7oGB{v$(q4f^(C{9uP5*xiT zm))c|Z86bIuZi9h=VGTgr!>D~_cEBoe7$m&fVg2>)u_l{O3jC=zQUD8}qBN{kHRJsH!4%tiV|tg_GppZhF3{ zr<*#)CC72*gLyqz-=uM`x%(wcm5oDNQjo(gE~V2imP@B z%GBvUj3@d#l|uARw=9$N8EsYwNSN!hiP1Xudp~Zrr!Ejrl=e>tYC; zbc1yYV3G%NW)88m{$p|?lcgDwSeSpq?dE%Ex-!=gt%@?0TMLa-i#lTr{Yf_Z$H*ms zqpFbD?JDcl1HEAFv9p?5dwoAu6db-1;*{w{JUdr$a^7;sme4lqA-5(437&tlUV%R* zZ0orbkm0$UIAB`;%Wk5XQ>3!jGdAbIj^6~xEw#O2MFA1$CA4P9&Z3=5XzdlP4cqEh zo+BZo6U=v2v;4QhOkt**6u)}E-(88_J9Ycz+~jQV!?y?8@(8m7(|_=y9qlL5el356 z+D9FWzs=8SkM>t9hH_iMtTwz~bGHg#m_R4?yj z;4=DD3A!nt47no;I_4a#@USXVwV;O2b$}l?nyG9i@5>&(62&l_5rX3`8t+)8+NBxa z2Ys|Zkos|A|5?^ajlbMc1DZvo169a@enl@>zXudsC(iBN=Lr15w3$~-nKh)3Z8U~$rhL7d;w>#pp*kKkYC=K5A7wp>9m*+gsGJ^m_3#Uy> zZ#B}T%X$P-L!r``maxemirgR#{`uZoJZ!W??@n@<>_qIh50}x|-%5{_n1!b8RJ0Md zC1B-6jH9_R{!o;^o%hl)vnG8!qb?r*ebG`{g>|DMpWO>l`^}C#U(9VERl9a3xt9Vw zLy?DLosy2U`C&qV9ALL&a~3Nhcuqc}F8~}eW~9zrgMpqi;zK(G_@bfYt2XXvf|{sI znC_6rDFj{Rs#Cq(#QTG%#W*@{?7Ce7{iFy`p72$Gzefh7VoV+#o*qpW=Ed!IQnOpq znxbn`^1d3a^?Pb|ERD6!G9gHDz^u{viukP1v00_9JczTBBW=ti(#+e&DZ;j#7rK9I zqub{ihlF39o-l;I+R$5H2-eR#=IgaS`9($|y;{Lm7JZYHJi~kn)h)0IB5mpUf}AVJ zkGWHH9;FkvKPN6B5EabB#N6vEQ^)=4!?B~Q+7s?!?AMz-!iP5I{rb6KEvFH)g8SqG7z-PV)}`g=5WtC+aR-hR z3rtd3*&xmBCf$vTrr&m(TwR(~)U4m@PSy6PkSAA2fP9SJ#>}D;QSFMyQk%9U0elA} zABB)5=j$@ny@*=nAh=u|TiL9>f{9uDEqJ11H0{KW)0>5;1jW#sCSmd0izsz+NxokuEw%cNNWXml%L3FN%h&q-hapQgSF??V`*D+g9Cz8 zHdPEf;9*jP4I$HHZu&i7={~CH>%ojJz`1#IUFV;6rmrk%+1i`IX2zDoGUBcOiZt$Hk3`loM1*+ zZ(5eAB5|(A556V{EQ6*LMcc^fVZD0V!}jo{A1G&q8ANcR=r$wfO^hUl;}M2-ine7^@_@#9AT^#AwScFbB~1G1PvnD} z6fO(g#!HN80#=*y&i?>S=k%Nf8SpGM3e8v|Ts0)1HgS%MQEp#AxCT8Ky&#gfYFU}G`l14GD1=Y(J8Ur=*2}rXZQKOQrdS6;aED6aNhHh9mt-ny|5f_%h^OS{U zh&yYKn3)i?kZ7z@yO6@?(>wh|{ff%cJH+_mhWMCc;$NpHp_Ue4<%b_#h=Y4h-t*=VJj%T} z^&UO+JA=H$E%ozU&W^Q}rR1*Cn8LHxI@qLcT*;mVb)n1t!{EeNAL7p+J7<*a>mT8G zw3Ha2pc!xf(-C>0`9Sspd7~Al-0@?%(dBf;8}orX3k0iaoazfz;EmS z``boBwGDU_q@?nPVLbQAs|OrB*p@WO_tMd^``#70NM~^4E&iQ$kd6+$X?UZ~jnJ)a zZqNCPb>3P2A9RzSjJ-o5@}ESs@5s6%Wo3I4-4m+eH=5grH-1f*cUZDeHk<6+Cm zQJssd4lOJ=Vv{lH;(BHvBO(ZaUR<5?Tv(VGb5B}C#|Dse=*`0$?Yd^8hAmekgbf#c z@pH}(pW=OL-d5`F*V(v)w&E%B>z>IqxM1{wMNGldzUaSTSC$UswJrNAoBmU>`+h2K z5NBGyw!)%JR%(V@_J@Gr^xo#7X_Xm0WUh}G4Qw6T`Nenb7Y?*`YqWfdD1>y(J2nZ|)$7|uh z_D$%n1Tiu8_3t~!r${D{u%I`j`{1}PDkn*sswdr(4 z+i>j1MbTBHxn)}uY;dcYbd0T{I~>I%JixL6EtEvx#8H=Kxj zvI3{eecCCz@ZIYosLsjLAtn!7{{mt)VDW5k<{IV*1*(8AW!;TP@EY@B9HTEb(xxh{ z&qQp}9){2tMfD{9oLhhMuR|mKcZmDfyQLXgcb=F${&bGN=J);f!Zo;rj!*W!RbdU3 zH^xJpp9T}hYTm!v(A|H2-^Eai?NjKH9ik8Yi8X5z%(`4gA{;Gb*ri#^xUVO;-9GjE zHQn8v$wNoe*BH6|GpAUWHt{;fKOIOv2zG{_`0l8$Njui2yaf3D>wVqRjhD8U-k-awkP!P+M*}t`%8#V4 z3DIuth2TG;@GVZ^zrF45nvsm&=YKwI9C~HCad*_j^G-@V`?>>us^`C~B;yv}Q^(jc z#q4vgYy7TZx5bQcR@q1oNZ7C~;Y>Qi7v+&0+XSoII#J*L7jH`_sX?~Re=(h`bGF9?~N`yazTC{gf~@V|a{>2wZC<2$Fg2 z(0AZ+TS}4T9Xa-H7cn}?_e~uy;`EG;&Wt*gZjgBXy&}9M`!D=XX+{6XhU>#EYYuJQ zl-GHpe8kkTDC%~)cOJS5AAb2vDSEr}!FNcl4K=?(g*iGIgpMBO<;*A^-e%9%b*qm> zO&wYXLuWY}I!L?Nt9UDLH$#722&&1a<9=q?x^^#Z=#>*zpv=}`CYF5a1|dxRNtzzX z_QnH?D15*2P5#!`>AxY*X!m?xv0qP_1HXRCcHb$YU5yKN+gyh33N0mEnh=``;{2-L zI+0JnH-Gju%f|K;M19JBP2`x?Kgn*EI#qo(55J~ewzHjWoa0_Py%? zzFjZ6u=xGab^c~2B#YK++q5nSwDCT$#MB|~g=5kahoq~rxiu4g9W(Cz*+s>R25lFt z;{76RN_(42uzY`FDkJXE=oS8~g`(^9qO~vnnM7O31qX$#I@R7->Hy|;HHzY=`?u-v zS1mZOHo063TcvGe{V}UH^NiJKdnx~$ZvK6B-9l14c-)G@&@(jj*L*JmFV(qmnDT#Y z8vLJ<_NDN#8iS@8Uh17~RAW5@T&G^~DspqQOX?k7D!k2_wWKKZ?zYN(CXN|iH3y5D zDrkF4Mo#cXrsA$mo~y?AvCjrKHdbKBL$->~EFP|awaWWS5p}NXm4lhh+GeS;JcJfK zIDrMeVGTyb!?-<3U#_^9O(?YKOY5gvVG4Uft&K=w>%+DW*@k(?_f=n9M0R?9k{(Mr z{F?DSG{)OmVkL;W{2a)QoD432l{bqU-_4cNaiV}{Gpg%*SkAnsf{`_fJUEE~ORiU} z9bBwfFAwuiZ=Cb%8^|rv!iBBcW3LO|{M~ItD6i0D2y3a*rj~@T)W%l5Nyp1b9J1jwON0)@vlb;>0Lz0zU=q>r zwLHns{$}&ABqXJJ3C%|E``3u2p4t~#f|k$>1K@|78TZa5^AeoXf!7}azQlZzFUJ)P z31Z%xtn=q)!AG^Za{kbRehH>+QUUi`4f6+Vef!t2eZ~S~D=_^I40KgIs|C5e(@H)hm7E(?$E@?>R-2Y_|==)5DU=QW2L5(%o}03b8~qS$M1 zTbL7WBjD|7JlKaYnE~2TQRQo3Bgc{4!^L@o-~wSz=YyVC*XiTR-+a+0X;a31;Ui(iN- z(RE_OFmeSoEM9pn>W&+I4Ki!Zm*S|N>H$mF$>*6D|G?~~JoRU^#Jd`rDFD%_9)H)Q z;xWhmwX=Z@{*>@DnYP=r4cqtHqALxb4|Jd7-YaD8Bs#wkNMq}qhJuP4EPGD|a?j1I z2`yTiq9WV+5JoJ+zCNL(VmoTFhtSKTT*6Q@_GtT}S(OJ%NfrkW zpzl*Ma&sY-s-U5;@tj>9CY2LpNm4zx{X~$$C$4T8FE6z61ij}vr91l&+In&$7Ja3( zZg)Pmd!Ql+V;nv(F_rM8+8YqeCuI7Lz_J}YTL&uL^P1G>U(~n`l%^4ephNR^dw zgTEe?l`dDicAP_#eW74!Cqk0S3ywR05n zA^#~S?nzCk*sB3q7?Mw^@bN-uPmq3i`@njC+gHBaW!8F1Q+&8 z?y7uZ7$Lcr>2U;4gW$@)~P& z1sz>p`n|zxJkxA%;S}Mk|4u3~S5$Cj5ESg2_dU_bfz^Rzv{?h{5M)g72vOJkE;6+w zh*iU>GoZZ6mdGAECfA8yd?sIU+@RP6#F*toSxzLKp^1QJoS^lgq|3=Cyc|@gL1_*+ zPA8n65d!~sOk4(rSG%8SeTC?9}g_dR#Ma9vf->Z(FNj{T3q}mfw zfjO2~CKf|}rPmY)o+8X)X(CD#Ro(QaE^ElCQpi`!<-2iq$t8mP_+KXbL0Q_?v@iF> z+=Aek?*z}ycHvno12(%;&crzcxQcUby8;RYb?2B<6j$W%5DH@dnYgK%lye09Y+&Mx zwTDAkA-^EFhz>xC-9~$E4+oV6a3q_|M@sUpY*x_s9^eca^Ld{Ui=&Op-NSNkxdC?W z{O-yUKMBJmr9A?Xlsytq`ZFmv?Gt4_YM>TAM*80n?hRDnFYbC_9kswkF*>qAW87!3aypV_aqbBq? z9god!sX5Hr3(KGMy(&5pSE=$I1l5PtQ9{taly_eOPIBC_j9%{ff`muh4IK`JdAWY6 z{e7TZ(ax~aBe|bPPEL#<8s+j`6ZxkVhG*d0dXsPgX&V#`-jZ)b2h|s-@Q?B&Wc1W` zIQ`USsF-gt(uO$E>9t$>J6!Ke@#Pb_4^-&!V9_g%t4vXzbzLE=bF?Sl>u}U|b^c>) z$J29}e2#y(?*sku`M7ozFR8#NNn%GE_kW2Sf^QeT!CjFjb`$78{ z^zW~+YgQrmUy2(77p%K56L&V*I2R&wu#y-HgWrH2htM-?AZeI9+ij1E6C&4G2XvrA zMSO`Y-cr6tfPN(^<8FCVz-_e~p;XTZ;Dg*Z@dI>3dFfU$q9AxHUAHot8(<#tGzm0o zv`gYHhaF(&@Up6CpCkn%f%3A1GVAMcl=Y-opQ;FckoU+b=a%doY;*lTsD2`e9x!9$Cby2FVA~X{p zzP1!id}T=BpA2B{d`Ay=3@LR(Lp!1`qdc(&#+CJM#Xb{3HljLTTgT*Y`>Y4bK*z(o z9LZZ~o+sa|r^Z!atN5F{2^+Ek5KN=+R4?oB>L6~zdE~&=+kq^bQio@XzlDj%=!L~m z+y=pSsB`(+f}{|+)7AX3xw3sw@kx1A=pT$q&WrY(RgB=k!gt6Zh7Zd>^)Uk>1O80) z^TdywWW>Y@2FL&>DvlcKD@oksJs%CZE^jABvR{b;%1e%mjUQtv&p=ZHrC;>ggYl zk33G$HSZuf*qwU`sUUW0({v8BdnSRTZ}4JYmHgY65Tg>_z8VM)3C}tps9&(*ePqT8 zXKl;l8ZG`B#GBUR8a8(gfAY@+IQBCjQO*6A89LnuZ0jGr!u9M9&v_EsH!B`~U7ngj zBKR;r_7B-!i3X;09{+pxV1VWDk}*Z7Zm36iU#K+9*iisoed_b6*K`vZT3n}WzlO8r z*yQ@1ysS;br8VtYzw?PJgb#?f3PKe!ez+uc0%Qh5UjYtW6ZAZEpl(!X!96TnbnzIF z7oD}Nr+CiDAtMLCh~K@tyzh+S#Q4aEsMnc!MNcKvOy9)>kSi%BOJV2$&9CuI%G#lN z4Aw=6j87d`Ao((Mp0t0)QOqB}l8m2%-qsvf1j>Y6Gg*r4T~%@M$`3-^`sf!aLwi}! z$5q{Z`yP%z=l`#{-j5Q91TTV~u^*iTfX}He zDdj!x;IKvSTS6v>CDUo>g#A>;YrwDdkW= z6v0>V5vH1qVAdexdaRkd1gO1(&913h<}`)2a~Ung?~efo>!D_yOG$nhGN6%bX?0kk)-yQ%*~5Q_`=X4izkZmn@A%K9DtykCYQ~2qM3qMpipMwNG0R z$OHw8I^m4CROBpSZ%*8qxQiNXku(uA>8q*c4i;Z-oC->@{>FZZN_v6~o*vw!u!#jE z4Xw}_xR`&79~hRMD(vS58;YcewU2us(yGQ|OI90B{N8i-w)Vw5%NoXMlZbKL9mWtQ zp`jw~FwNvyR8k}V?TOaSh_RQNQv;2u1JPEiv{fPht}F}H=;m-S+#d`fz7(N=Z#zG& zY|xLc4`Uf{)vKAw7{pIEq6eRwCa}i36pqR3_I$ESk7Apyz!{7Q-Zdu~_zUkpT8#@D zW35MS{`2Owd%5fVg?Cdd+RCY=#MN5_)^|T=1;o#4dlleTGF86=?}X+|9wy{Ar(}oV zF{{S-85v`HjLzwYAYDF3mRUXbXYlt<9z0*3wfa`4L*Dsvyo2xg%<*4O$}d^jtN+g7 za7q^iMCZKv5U@AhR>5C8ew+MIUR51d>09bsP*a!wY}>AwNaiotz{1k zV>+!51E8ZrZW;Wfl@C~Y)jjGyoV5aL1+DaD z2n8xLuFn^#U}QUI)8*UOwM%snIGvnGFwOLZ3Th}BHdl&HY}Tcy;_8ot(!SLH5>_PG zN1aOfkmM`5d2Q?~IlJRi+%uzu!gD=&^4DBxq--|vvas?By}wQ@+jk0sKoubS8dE(( zjj*=4rj8Zb7v!gIm4dxPse;kFE%%4L!X;kHyLh{;Lk8|qv#n%U-sRf{gB^+J>#92D zL{{5uQ)#EC!|4Rb*Kb6dNC6x*5Q-Uno-K}4eUklswFW)m`{r71B=Yosc>z-X4G#XU1^P2;xH1DinW>C6?|B?4V{Vc3R-qa#H>(Z>4jzN%{+GiX;xRXYSoXTbe#h?qpftk^7C=-Os z3^(u*daJ-4YXGmz#rew21kI=<8P?UkarhUdtGViRl_ zx)z46$@Z2w2ckMdM-W{`elDTu7MlG9!kYzNGVf5Peaz>5Uo=)pE)RiH1FmScvTmwO zrA2?htl%Pf`Yd6!)u6I$3%APOcTS#ju61;3ILch)*Q7o8p2I$-7~?J&y6 zTgFeihZ<;8Yu1!~-C#HY+p1-v5L^7M-R9bGm6IQAqbl%@74-I16YW`wrtFEF^n#HA zj5H^`n7`BuV6FPuD-&xUHrC{gJk5c8<|854Dq5zeQ-#A$ZIXK6p4f)7#c# z=wRbf5taFOBM%ufxk@L&5AQ5_L+DM9ALjKK+I%K_JmWwdBhxwD;OB=QK1Mt8!%`18 zC0lgsD!6;^S15jaOSRWssTz2p@!y@A1txt%v}0|be)lUDjL{9oP=`gh3hMjU66`NS zA+d2oAhCTOP_Hlk5xwS#1U4~zF4NkZ8L!FZ?GLgQkLTGuLJ*Gg`Y{j{^SP#;j)=+? znO9P7Mp+kKPt~)6qa*2oS z=WLlQssrAms9J(Jk)s@<_fct-O87ko;E;9<=`@iL#s{_LL1>7u$um}c-(s)8>RC%= z;=nO74C#BdUM86!G@P}gy6lFe+17w^C8gJUIKzluZp-Z0Ycg-3xKHZ1z0HlkzirC* zdUaDv&G~_x8wL6?bN+4RP^YO+OTT0^n)2ceqyYHx?yMlc8mxPPHTqkUjeH11gUsBz zQU<+>e(i=NKiW-<*ck2GZx-{A2y#tj#DI1n)JGi3iTOP#c{NunBM*Mp#GEWOyGj&Z`(~ z_N?MUP^iA8cV_Xa);P@T=wqF*64nVEvdGC5lhr}D3dX>`GW+QU)Z?kZ*#tjk zazS0RR~CU?s=aV-+XS~?Awp^ZS8qQSIC*GeCYGSikBlCw&@e+g2w=_b zP}U-bs)&=aM*QKBCT_p`_{b|^S!RV|Qvyiv?u1P3DuMZg@3aRT=R#5Fo!QPM*De3( zd@+(%29Z3UxOQ)dC^Wj?uPBpLn;(t=v>eR)Gd*_@ZCXFQW60skF|%Xwvtlfj*cL_3 zjXl)kZ{0R}>AY*l=~H!WX2`u8JKvojsPbOe3;anBbxvVJ3kb{eMOzCK_l^#c)Vt|{ z?eZ4u7iU?Z zdfIj9U@m3Rg+~KF`~Q#fzB8=JwB0s{$~b~J!weurbW~7jQltu~IH*WdiqeseBvb=Z z0w^lZP^A|`q>JYisc?z8o>TS?ZwjGm zMksiVRLux-y1amPE^tHRIt$9hu0hVwjg9?0ImQ@hEl1Tl%>-pkQwdcxQkaJnGDoJj zDUx$92vgUZyZ1rPJq6A=-3t*P9)jW86%r?mHa4f6i?zMFM9uo#GYXPRd26InQ5tj+ znJVyH4&;AIetz#&3sE*|>vd=|{`0zNy@0=Y{FE(yV`2;q#J7G8{$f!Oq9rDPY97cR zj=s7vCa|($@#l~hM!pUb&wQyjicTNaChB)5#A{TJH{U8M(>ORUKq=yS`s)p}lS8okf6T+naK`5FggRgTT~$@r-n{gH zW^cF|MZ}a%0-H?iJw|Ou>g^1CgssO(`BKk8(ChT3rA{hf)PoD6;DimP+=4eAZys@~ zY8ZNCyab_6kYFiBh-jWJ2irp33k0pqwU&v!os0`81v9npBToja84cS|^A@vc^*Ehe z2Jhkbt#KLdv=?WowV)N&MEs!(Tm>>umGtVt&fW zRM~l;kHfC(tubbWX*M-u;!Q-*V2#7ljir0x=NYmNm}K z5PyYQy&XAkAcOqO4?}RdQDOCISTil*-skWsanL49t?#h3QkAq}(y4k~?Z_8r!_&_K zF{+QC6#?An3*Y2W{6nxL5^RE*L(#9hh5^#MTX9j5xQ<&5&gkI{ef40BTV%X|!mwaS z?=gk4PZl#|Ho!)=hiF48=oWQ8@ zs^Y8X=heMqz`cAHZ=SYJFJX4Ho$E9k-!4*~Fk~l2Q=91f=X9jW!x9|Sb<_-JBP&;V zsV294I)ML+DynFVo--Xe_!)4q@-k&y$HC3OQnJ|eKNz+^Lzic^R)ZAOiA*Bo`pi03 z-8J}=eskj7!U~?KRCKMJVftaq-&j3IH|DT5Ri%M(LpG0pM+LZ(2rfj7_xJC~!@e+- z3ZbpEj~FP~MSgL6c6>#{fv*hD&|edLA}(j^hL_kOdW3DQV|>&fA8l9f9MUL6`A;50 zL;Mcb*s&#ZYHX6MHkg8&ALEAIJ1S=92cN}7F(&z7`9RdjoVru`Io}c-d9Bnf(h;{w zXKU&*ZrjxlYYaexbIK@cs-~s1nP$P^zy^h>*@>ozuJS}Pw5X*kW$v`w8lm-!GOx)Y zMs2f{j4vIMXxIblBSA8Ze1o(jrtkFPiMQMWQW*tNuPx9n1iyCu-l1~+OZ929pBN3; zA}>{R@MI(krj?{TB(mmnN*bFU-N}uMSY9Z=jYj3xm9kuVlpo`f9HWMi+UKB8Ex?J z7cMjF=LV^kMyozcxuH-s=pbCqFfy(cr@I)Rx7iU6V=Uf8IIjp+pGOz1%jHL>=LHnG zVC#vgi;lt6rM#g~e9No+h~gJJP8QS-HQO52Reis?g`G5iuEyOZb(AwKcS%Xz9bpl} zprFS^Qlt4}QQ1Z&ig6>UG(2hT)YR}GH zl({`xl^rX;>*cN6H;t6-$K9B$Fm}V98{Ky1SfU!dm42GjVVUy^iX)GuIlc%+buB{k z=A46KPU~B(&Q0o@eK=>i89cOsWbx;lH*amg%3*XtraaCpoJly&E>ufLyJzdOl|lJI zJiuo#K1drfZO%KJb!bG2&vLKAy4gR54;@|9{55SIyPX>N!_HX#ieu^_gdiOu6H6CHq(1I1ljD$!>ymT z`gpdK%1HwlK8>i_0;J|`PCBtEtT^N%?Cjj>){(Fci;_B|AkS6Tpf`+qfBBJ72cD_* z)iqyknN;24LJK<(fB7!8BYbHjy~xefXwKzNo_sv|ZaR6vaMpr{4Xmf)2RVzY+DHgG zF;o#vd%s%HBIqSnoieT2dbD=CX~b+YVqqJ@32>K~)ZrKmhBsEFGUJ!Cj~Hs`_s!ef zn%eW`Qm3#IsBvltLe9B(u!%l2$1_*k^7u0%P1Yw_`!TKMhQO}OV@FXFmc6x!k;W31Z5qLLw zoUoisWteh0=Z&L)=c9DC#6!R7KDHxPEAE~L=R-SE58I(Zdv=CYnG zZbO95pSR}(pmfiOM7ogc8Gj$r09 zpd)Yt9q}wcA3qZnzigS&NshM>N?-#L%fGmm2P^;>a>J&IA%zkINB;TLu!yQwtd{Cd zd-vmk@K-)-14%^YuZ!cF0v;!x_Oq=t``Jw`-BF=Ne|vS! zo9}WDbMMzH0q}h#0Zv5o4p-kO89D2^5l?vP9*Mq~D{G7kj5og}XTUgcpMHTJWUn)P#8N^#sJ#WAk zxsc^Zi>PsfVT&Ik#;Yo9dlE3$d69lp&jOK6?@MRg*j=t@3&4$(4;XCxMPn-xApDD+ z(T)hX}+~ICEw7a7%y4W4^>1Thkll8&fSSU*-`l8^QxNP!HSAAZhwS+__}g>n)UeQ33wwENbzmc??*~UrjhwkV>;y3 z$qS!fyweVjV-Ytw##+#;Q*b*3h6dDXU5BdFp)YSfV70Da4O<%G>2Hf`0^UOZQtR}G zQw>DHdmdGMX*{q|k5xOcDS5$zL$=9~V?L~V6w)H$o%dkoh7n+S6tU~wdK+hTv3ay4 z;HL7G`m@#=Xv%Nn##xJ6ihYOe@ddb$C|Y!SYoc)hd-+zNdorw+OJ|kQ;_MA8AR^y= zZvJLdMyF8+_A`V$R23+buRi3zhn`mg%gcL@OfVgsg#6YwTy6|seHE;hZN-ge*Ic+^xMm9d$+#f2?wxa-CqiqpId)K-o)_h(SH!h;|9AxzXL%K zs4oA%Q_sn2pa~t}F%^6(2HUR7`dkek7YG7j{@G&x4{5zGO&psvp5UUj>Kxa~$BtVr~al5*;znw9=K^9)D-3bdR2;dt)%%~{@8khdJ`iU^ovyM6B zHwFCA^uaX0++fVCuAL8}os}D+*&LhMF#WTd;aiAMQqK3@F7N{Zj?kW1o`?Ak3-ft*}Ovoo6u&!R<-Zyzl%ZAc;WI&UD zOX9ygVGBF4#qr^M)e9%9x(BEuXqSFO-sXRM@_L4=Orl!-&E!j7oL<_X&g`1chlZ|9 zXPj(Jba1cBZpt?*ZQreikKRYurk0539bIsk>LA+J%gid_C8VgB?rft&uT{R{_Q+Gi zqY)1C3v6$d!KD4|MPIz-AgSn4R6*#yps48xX%j{|@ys79S*b`s=Bs6PV)oS??CE=! zz3D@w(f0UjiEG2sIs1uOVT9fWz%^@7v-b6;2!0EboWfg?JL0@FoV7x0jBZc&SIbx~ z3X_kfwpCMKL_83j__pl2qkWY2KPsA?dsrR00pxSxS+s+`Zz$4+xBt8c9-!S*`qAs- z>pT1v-NsS9XvbJBcYEv?>&5sMvGOIS8bTB( zsG=i=r{k5M1rbxZo!}~a+H4clF|U%A%Xxz5!wJGjh)ZA^6@5T%_g_s5)c<0ZxO2yI z^oi1Mri1&1gO&QyltiL}aJP-VHSeh0aR1iRh>{pD3{#dgBo@_F+urE)LhXFgw`qHG z1ba+ErvVj6=k?%SgQ+D?XO6_&Q(?FGD1RCDA9z=r zmwk$^D_BdYun^=zd}96He2)r^ZX5jT+$bY6QXxc~9s z{)MlYELjp@tE;tfrbLMZGDtyyv#nlC=g0^CDFjw~Z`u3Dv_>=EnWb#Py_9(bdv=%r z_RVl@a0cU@Y#;k}mRoG-s7;hqr5f@jyqj>6+e#p0;0UTfJW1cW1k% zb)@!QLe!z$+2MM9hoi<#e%k_JzKwe zJwz^7It!;;&L)1c%F}g_Xzz(x}JSTvZzvh+WVJ7xKLC2bC~AxoSVi>lC? zcbN5Cxm9swSD!{4@_ID>X%8<(scks*dY-GpvXa!7`MTb+F#+|C`&`T`e^D7}ufdWa za91J=7cszA-v;}NtR{ZpyY!Wb?#2_5zLGh17jnc-;(FV`i-Sf>nH`v=uMx$~xXx z96-a`d%X%XsaYxil`=y%w&KsIN+A`!K3b@KVW%Wt&3BboiVSwjl{GyUbI^O_#GDnn zMw|1wA-t8o9o20fFf$5?(ewYWCzp%LpCzh>&G-x98Yh#)fa}D6I`{f}fD|-tuxc=? zsx^`@P6F$F-+7=)jfh>{cYCeXfGV)g#%M_2X7Mw(&WLisfD}pYfB*U2|ch(MJMtYOG&^TD}Hkv@&8R`Afj2I2UU^N4?8XgsQV+H8bVopwUD$&wgkVT83(Df zB&P2icIoS*t*`{uPiAueN`>A6*Whm@1204L!dP~`orK2iBiQQrmJ#T{ZgR4LH`pKR zt`|qEjg6Tw?slNCNscgU$sN$G_+?&vWbu#qkI;IE#Er zM#?drW<6^S%}`6tV}gmzbZ42HSxp9x!J*`bw$f(1C#wZW%!YkEi7Qxv4c zH8aWz=Q7J}tw%EWA2A_Y7h{YP5`mQshGx)2GXs9Gf#F`=ju;H0o-RE3i;@TSx+qfI z+9r%NuwSkNo(5k@y}>oq7YvWY;+M8!B14a%mAr&NDUDex;UO$mN-{L0U}kxe$(a!u z#(&Td@|8=Q@1>u9l4=tCfczmQZ6n`J^rM6&+&7GuYzmnwFk#&T_T5@f6BR-hs7&1i z8iqq!gToVGia?Vf8J!Avk~p>&@A(|3=5o{W8P_x9u)aRYJE^Xt`&E%~gj$e?6g?+d zwuPAL&|(#x*x45447)rRj56oDT;&Oa_z=oheP5PJN;3Ssfk|&f%0486snR=s)r*-A zo8mcmEej-}2pSWLIy4&b$cybg^E5cAg|L6_3dLsvN4&*djRqB60EjIhc@mc+xJO=d z-%P$=?x-%)!br3ac>f2t=zzy{|HAqr*)|cS(@$S{>GEqAo$}&Ks;4~9gN=j5-;JUI zs3&}$iI=W4?T5793O?bAJ192R^M`G<+Xf7kZm8@xp~Qxcah)AitOv$gu1y@aU2Jnv zFE|MuzQ3ur2CIN+OsVnOG(l2DG{H}8F~Vo1;!S!G=-&IQUv+kH1F<+8Jw*_!44#gt z2(68`aE&wGiCl6!si6=)$kV9S)??duC^#3S&v^+?R@PQ*ytz@5YCadxI@uO=xq5ny z8(^%S8-5)(zjmGP_e

TKY=QE4+bw|+VFstygI(Y{bW3N%n01l^@_bY zFJHiRLZlZ(reF|v2!5rU%#q9Nz*HaiZEQF~wDHhx@*gx>ih8;5*N^mqq7^fPFWGxb zy?2d?hS4r49*UjkGxg)ZtfnHb$UrU&V zz4xTO92yj)7DQ@8Qm~_ zIq*-}1H|Glzfez&EA>Dnk~J0f<9+c8&9Is@GUn~#9~S3?UV5Vj<%8`GB_tE?18F?8 zX1SWe!q%?~CStP0K>W{urF(RTOTBF{wcY)Rc8MROt!~b%mxMW20To$>I2hpNqTI6^ zOl|9X!H2_6H+N~QrBxcqHv3d*0`KZcrZm|Wtfo??WCmx(Sh~7O!lZbhYoGcrX}+*a zeCv4zg8NL(dR*(G=PZ3iQrOkVV-i0s_PUOfDc~J~u`1mbFl$~Fi7u4Ak08qJs3#(y zS*w|o5rlH(8vC>xWJ!rXD-ZNU46I+RI&gx}p3RE1lW%4p4$Oi(e?s1!RL~9lVqBvc z6sHl4oi_3YH*W1OGlnYrE)i8Wq0bEN_iSEBx) zd-8d^^tUW=sa%P-GtT%m?5I?r790jW8}{Om1- zo06nI70|0IpuOPzAij^_0=F-wWupr_dz)|k{$g830iQ>sTFCnChmzO|a1pe6tLtzZ z^zvlPguw1YvFh17?uIJ-M~+>W2c;zWGAdHt#sFAfX{9)JKUU-5Hx}=jpocCq?VC8U z5`83J{L+etv$#jU2dO|?`vUg~HxYSm23FeU^kv`;M*!qKGBNx9j$;sQD4#^GFJz*Z_l5BSn|fA< zU?X5UdDf3epeNU=EY%GRMoM@i3?%ca&IC)JUCx(3h8ckjWFKC8z2a3D{b^Yi&kn5f z35MKUQvQCPTOce{_P)LY$}bYr6kPB)=2R?yzVN2Gp&4UQBUqo%Xr|p5L$i@XX^wg5>Eh6Kvs|IJ&n){4*MOixJ}#2N%nl0Tr++(- zfuL?4W^Lo7-f6v%*0iwIrF6dd6c9dqMsb?F=Q&XY20YLHRa}_&a>hD&U>Mwdc_+J%>)(iju literal 73378 zcmeFZc{G&$|3BW!L}gSGvZW$olr8IMN|Y_T>`U2qvNOquWT}L*O%ft&Vlb8v%3Ahy zCVTeXm@&)mQn|nHcX#u-@83DU<@?9?9H(=g>AGHXy`KBy@qE7K;Y~HgLv*ZkyLRn5 zbY1DH=B{0! zFMd9J`u{xqAA>Zr#;k}`EEzGFR=jRwv##A1=MvV{yN8HF>jPn>Kh zY^{k!&(;pjNpTLdCr0Di$G_P#_{}Igy`O_`bsDu2%ErF>@D_CCK(V35GWUi;Ed5Ld zOE;!7+>Vkgx(BfHGH!YHW^!1S<23c!q*+m1Fut^H>ib6FtzSkg&jT18YbsMFA(gmH z;6uMQZHcqBGjh@KuB?3H_QBi-{%SJPD5d)8t0RaWkUruKMTWC&u?G+H*0 zvG!)~6TSN$SsSb`P!citG9IftkQi|ks0NDL@vdlHdBhia%FE=~=tXhZ53f0%Ca--u zR}iP5+{3<*NSoB-6}++r@d~($TDtD)z4lYRE&nksiohq;VsN><4BNRGrimqe(zk3FQgfw@m_z?9w_@X`An&WBW6islBacIJ_} z_M9v~zF4iftSiD6-vpB+u_QkNjTAb`(W?Ip8ln~VK;|{1&i5nvlDtYkMUo<)3`*6f zXOpZJAzpK}=yv8JP3K{AZ)vEoCZw1YP0ai59mTT-goqrsS(C#Xe>schwKDbCKblInO36iYO$P-y&+p{*BXbK zJ!v9(Sl#?t%4nW)5&bd!!vQ+*aZ(}LcnULEe(rhrOW`JBUNof3l;T9_8Z>N5>tK$8 zcf`(i%crHOrt8kzW$ovjkvb;o%3qv))+PJyVKvXzIMM-Lwmj1+hZ|6Sq1rxfoz@U= z&u#kIMz!kc>2>Mo%;vgf<~U~|lX6vor_tP$mlz{Zp|Rdc(Ws>DXXx?8T({D_rKB2--^=E01(0q(uibI^NuxkoMqT zyc>dh6_i)a5w5;p!(li>+eFQQ&H0~dO;nuznLN4_#3HW)iMLhh|_=LVwF-ChkB7}r_~$z+v+cs|T@*qz|c z3ApPwKb;@A)gh0y9(+5<&nP1m;fv0@RP`->M$;KvxH-X`qqA8?m~Ya3l&tMvyh)Qr z@CLkm*zdG%eM;M1{V7gAo++;fp_PPa_l z8}b(uaLCrlHQu{)g}h%91=&C87Ik!jIU=A-7&T zy*M9h0Dt|KPCOi3dniLwRe(HS6qJy2sswfi;WF0gd$(^?S$Dqtrtv^j_nEX@aoI`| z1d0e{P7Zv3jo1%Kxsk+DG9BG*xgO!qwAQDOsxIC=PGAuU1Fht5s2p;ow)3fhJHK%2p_C8s%pRAS}ViL^X zl!6zeW+PP-p$~c0KhJBrGB)fQW7?p_`=8*}p6A(I&&qrn{KjDEyYS4sRG(FSd(&8% z=WXNZd{skrrgqmkC*^12-MP;jQ75owbJ=<4JkR*1w_f9Hj8=)5?jqQeN;u6j|b9-vuy*vsEQI;^@SnsGx)XgW{!SmTIpu?9-EkXz0 z^ZTn7&#Yl)Yg3cR=2NDG@I@s{uTBbkrD2{NSh?4t*JVzhKka^>RQI0Mj-jv8gVj~> zLY3}O>qv^_iJI<;dF`Q1F}<+rBfp|Nhs18i`_Uf1imxCkg^m$)fN3S$7LC`05vWd3 zI(V@}LKaH~Ew?&BQU%BQJ&;l1~I~1$J+AxQJTTl?jBV z8JTyJJ?^AmEU6h?lGbnom-u-Qva~xVDMC?hTC&bd*~wFROm;7!@8{@dt<-rZX&p&NLx413s>nu{GNKm(oV^`kvLR`We)XotJrN=aZGO?q_(zbeSWJ zUc1GY{ivTCiAFRrxk42V$@6Z=gd3P-{_ zWh$$U+M`fv*g`B#{I%{Hy{lZs@v8OSraI>fk2B=I2_5EfZ8&EV8Uxsi7{N3LOE87c z-zeS9>ABeJJP}mwz<_1crI0)GM%Tq%CiFLjw0Gr&gBJozWGkNPIYPU2Dsn_VGhG*9}@95c?l-pcO$vPt<)N`kaZMffqtxsDW_ zS=558C;7&5C6Jq9OgYFme%0CDbNjA>m?(S!a|7+WkN(tl5%*1v!9&>z-OxmnL*?|x zlobj`Zj>a2$L_ZjTgaw!lYzYFtdSN#XEhrzl!d8Rib+5P9^(#F)Y&-LDU@m2N+^l6rHXm=!s(dm(uku|uZ z${bqjyLVTTOsvw}YySMC6WVuXKRSha+fS}Cl zfePb4=B0 z(RUF1nP8?;|6k?TMPj%_0k)(x6`K_6EE#EmF>b=SIWWR>Ga{C0d zv164jUM|p2S#-;ZC^+$XeJfvk$_GeG76|b0VX=7EoKBTHqfC{qI=E9!sD(plQSG7Z zugV#lCSK-xrKvz3e_lZ{H|<7Nl#LzIUVb2K+RAR9027cawrg%AYGw>&K;J*SJ2A}LHV023-fG2}i)sx=h8gdK5}4bS)Ct68FO(eml!?B&s`#Y-M5QTC_*GBN?wqX$!Ur5= zE5Dyv7Jut+vTmYhUHde%r+UMrIdWgQ4Kq``eSsBwkT0#wR;#SPt-A2$t$H75nZ@cL zGUL%|mCV}3f<3>i_uDcPn0p_m$}Idl+~2VG$*-uQb@ch`BYL3L zK;iC$SM+}0J5t2VNfoyBMTWSO&eP>?-tI2wE+@(IYW}Ed731i1%*yDKuIk;h$$^y< z2jUfptZyj(n+E=eG1)$H7PO{5i@mZDZ^Z0(&8EzqE-2jlrnK)>{Jwdets=s%hJ}pi zMI+>;J6^HjFzRHe0`bdA^ z7gIv7E|0iL;UU-B=YB_;M@9ZDU!SW6UFUbb3*~>!u&;7Mi)2MW^d_n&tCIka@?sXO__djLXm%@iuC(du-uV*1`~Oc$tz$9qn6onN+EPx@8If7!L?6yn?7h0~NHOT0ge$CowxUzOXfd)dw+ zy(`aMn}y5gY*l{SRJPbpXu4=IQ2-T5P%9fvtS?ONkt9uD1AQehtrI?n2d{^JT90eY z)XGbMZq%aR(qkPd_T{|dAN_NT3@;?Rk*p#MyZmp=NuE4btCjVUC0O9W+O1-!adF5e z$#n*_L1_qz_^Pij41 zNx!j}iQ>EW;NN`6Y&QQ$w zFniMot%APsN_GobqF;y-I3+ki$^FFXE3 zsauifH`2E7*k77;&$`X-%p?}xxdbrVDH7LJL#^`!yAEU05umMze7yF|sLUJ}o zpEZp;_qb*EM%N6KT%4~T#yPz;+h+56rF77xnR9O;Z|@?0WNwZC{Pwy3uROMp^k8F)WunsI)4*lMORl+>80->GIV@{PpYlZV8 z^aAuX4E%Ie_J`ByTV;=-+$3d&#E-A;U6w#--$g8OZ8XFh z6qB^|G{~XeVluJW5#UkCiw?ixFU4zFb!ZdA>MFao)$f89t65~a5BkhV#AM!}{A%|Q z56M`t5j%)S>kO*vctFGR>JrMbc(*7~GhPqq=pd(mp{VonK&7tku#?rsPfcHemft0L zt>vy2oM4sB%A0JY%MytIuP_jz6l^cQ&s^%ymd))>?|bt}N9TA%lRlumY@DhftaR(v zrNhcTD#VdIt!jq1V`6tdHw7{+KYJBeU-0;2*Nd#;h^1x}Bt&5bM(=lY)VwQi;+8o(qzPH^H2rPLVZn_6MP94Do;%R3$ya;WsIOX1mnAy%X!@2A?)oJSm=D@D>2u!9G?wRXK8jEg zQz@x3f9jU8j!FB}SS*3Rd8s#i@H7%G9EahouB%-iRky=9cF=MV5}G`Ok;pbbV}*{g zRig{2<|$6%WAbMA$~R}#{6KMVCFR&c>E}eW)7}Z|l{kGb`}G9rDB}1ej~;>k`fWLA zNAEj(V-~4YF#&a#ck~SjJA&O00m_9n(}(m`B>xjXdBJ^FS6-?eBBdhtH4rRg(i$Fl zW_4xH!>8X8o?l>NgvmjlN}k+0&E1DSrH> z_GonMihwWdqd;>^5U(l6*Nxz@d@>8xxgM-O60R=GlY z50aQK>9SGA(mL+ms`ePtCP}y?D+NrT%|Qfm*colen?v;8E>;t z)o$n9_d#O4OJLmzk2ss>=%Kt5wCGLc5#NH@e^nb?WlB=@>TYH9jh-Z@*6xzZ_^2ymG@){4K7WJ@t zp!!a0xJ3{pt2?oTL#kIyxaaP8?}XhTfgz3G#k{w$++4x&J!Yj~T*?zQJLH!tKg0YM zAoZr6bX-yxky_@J5DFlAite4!dfVgRv!y>9dAsmj@fL&e6m+O;qIcwcp{hWS!%#U+ z-`-Z{?PgxuRBYQ%4!r#46=#Fxxt=SgIxRVdt_LLyHYh?pJ)9Y7PRG|%Umby0JmY+H zA#haw+SiS`8+P3>;iO5>2Nqkir*>5Wznm% zrudn!U3VNQ#QvP`^={(Kgt9$A+I8g#rdw{+XA4p=PuU)+RxC4Yb&fPxs%H>Q^iJN- z@*s@7uT#&{N88S(Xw{GOS_~e9tOJqQx78RGR6E~t#hn@Bet3~MoX{a=RNB|IR)+r5Wg{^dm&=q)k@{5&AY=J3;j%dcjJ>A!kw=`6R; zai+lU@8Bblj2ciMG-egg1Pq~eY6f|;8hU_TT1vSSG{kF2PjXf>o+)Qc+;^uGO2Q@v zr&c9wyw_b=YsfV0b?C>BWaf^@w1-5gG8MhkYe8{Ce?c!0Sbq&zyD7Ts5&6FU5@n;u zWaph5wdD~@&&7voW#^vcd(voJ?iSXp$?uLSSHIISY1?*Tf%E2HTZt6ng6Wr1PTy?h z-RZ#CZ~<|e0aGL-JVe;Y<`Q~?#{)sfxDa!orrJSKXf9i%#@pN~s+H4IA;xhZ#P;Q& zOHbklnw^p{5(qywMDdM>$X_**yxHz;Hf<^8Xi$Pr;-eV9UZh2(3uLHtZ5fkW9XYud z(&9=Q+@_H3dhH%M=l*KEV#lHOS)JczO{w-M*`8h~HoS*K4-u%Okj8f9d`Aiz5IeKfot) z_;$r(Dhr54{ELsc4eN>br&vjWI@%Ubc z@a`u^x&PA5e_x0k|7R&SMeJqytK9fqpNDw3U|{awKmF@{4*<@z@7aR`VM(CBbo1XA z&kO%SiY>yh(n%<_J-!^-H)?#`>IC;ki4n>FUip7%^ouv}8i+iW$ljK=upR0DX0rWmW*x{i8ia#vha9p|TfYV<19+f~dS7_@`H@NQ4&3{b8RPyU&1k zW&yU>w~_f5i;0?Ojz>^&p${JW|Jk-K4xelw{H5&wzIeI(Z;aFX`V*0zhQRM4?F4v* ze{%6A5BC$ZzZi%A{(d0HG#vH5%?Jy$sHHlm@SGbBBR#E&#f^>!sD4}T9*HVS;B9Yj2M-ULbG1o|ApFAv#^6Z;T|od9BYJ%P zOQ*;sVPSs?`Jg7EEE2qO^ytyCvCp)QnpES2*}{IODYv~|i)~yA&sRa_{thbCM=NCb zgBG_4!2Q8{_wv-$jagc1A&*@~JPy!yw5tAH1HX1=#r9IOd8z~MOaiG*Etlee5Tko%@f{;Rwot)b}y|CpzT9AFl)Kiv{JOX@+ zxF+9loQg}z*8v&-pOR*}`JPj}Kj#DfzRY&w|Ho*}LVMBVXE*)t3$R}M7eVgOQLrQ< z6|D+e2Gz50Ct*_fDJW9ckRmAAaq4hY#a7GEBA-1KNz0q{0HLJl+KUro@8N;pVs(@? zv%36mGf~Za!hS%!my85YqR@47#>XGJhEd6&1#h-v6he}z!1d4X>ut23^#bgBh4)H( z>ug~0d(q*Vcc!`aUs;X2qB;h5tjvBY`Oh36>)UZeh< zw)*cWNcioZNG9&vYKxmXH*NAs_!bOATKQb9dIZ>DLsn~m@F0Mqb?~5+h$*@8fmQz-V@GY#hpUUS;~&syaA<>zGS@Ysd5`s z6iiO;$F!BW``sT6wEMiFb*2${oEs5K$~1+tL`EN*q|4&l`|0dFU!o-5K+;M7o$F#* zbN!X3_b2@XzsD`LVDgnOXq2XD^RE{j+RA<)nb6IbWwGmw;o1K#g8q_jzXOi~$WhUR zj~4`|WEN?tR;!^MT=;tNEwy)xpc zr5zSJQEdd+9}Q(XsISd1!FvtVk6G&Qm{4kIow}W)TrJS2KqRF6HM{OS`M{*a|U3-woG5+{jgODa}`Eym8?3jN)>cIxp2c|iKiSgd~DhF;sr z$C4WZE42~(8{1JP zST{4{f`O)a-T`})#2c}}K^qMB`=25Ex9(O<)PhH+CE8=-6{USA(qN5YZ5|~k;aFHf zCepDFcgv7F{JVoR%KD1reQ!dmd-?&Us~}gC_CJf&EO1+@-&>O~%Wq#wnQpCGvtb|% zw8wNd(j;l5)SVfuEb{_vU9W#ZaSb#-o!Uqe#5pLTPWt7Z)X6zd%hZo?5Uq3fNn|_x z-m0h)Sn5XPaT5D-KOYNB!qZlV(f z|Goa@=NRDAjoIMK(0hF}x=wIX>`MM+gmAx!rm{H4W_efW%bVo3B!9t^!*5DsH4%?n z14+olxcZGFGRE(G8>ia}oP=PYbJV`ohuNkSlq$^kr9KAkWP*E}o?cD_0%~C3R*(o& zCnK*SsIuPyAa4>VwmyY3)gBYh30|DleTj$kq7*>sm2-m|#O#iytgiOBe6s55LUbGR zHTiIq1%bFVD3N5KCy>m*OIHcL5$&ut7ic>?v*OS)vT(n8CIk%gp=K1YmyA3J z#Swi*woALHIO)nXLVRV(Fxksga;+xvDpzrTSHu($uesJd4+&%&bq z;yr#_q9yJh@YMELt(5?>ke%vA>HfLOm{?XtoTP@go(uj9?9rktXDR(#nAwR8KRSy4 zSox17f+TlFi*}ye5r(kgzk$}d{tcdguW^2S@?lKiA4(;9DOl-1=j!GI5vsb7soEnu zJiL7)nS_gmu&b;ii(QP>gsvba2pqw)HN9JCh8H<(UDOYHrDdw zW@EJPI#8YiUs7UH?y+1OLXY(yqvjW&mM0(T>6r~6#^o#XDD0h!y?G{;$<4!TtgHYC zIKhb=-Mr#{>G9WKe>v!d3v%L~6ig-O;!v3@$+rZJPD@LhZ;EZs&CQJ$@}W{W?B3&k z+NG?x7`QPiD$2nN7dhJtuW+9tSNd&3lqAr|WjQD(W9_GZ8W-iaTcB2eCY7Oh=UA}98jug+}9=2766_wZ4*vL z)q(Q^#h8>7K0!=ijQvp?WA){M}e^9}2>0LBSIdxb7yA(0mEZ~AgSYq&_K zg{TP%+7?4{^8{gQ%bAIS_F0Xi4o)BKK{>5s%VTF3tCO2a-cxPBP7kgGOLBhtep!xF zo$=ZOV12ZgvyBa)Q5Ulh{}U<(FcW~ewlOOUM~V|ASknjYGzH_E=TTv7Xl&U2y9NmE znhX{9*%c(obb^gtP-l|2p)pe~P(yzZnF&gLb_%fJpNN#>cdOko%MUW3XZ))opBN@> zxzdF5!=MKk2B;OL05aG=Du#a{i>x*Q=`>2r9v7#ETtJ9DI(P6aL@D(jMv$8*k-xFF zW>rM}J(B2*=BLZ%7M^A3n(#5!e%##b`~F=f zsoI#TcRk$mpYeddHB2~G2*j|0mz)c^nE*2PxP5v#OmL^$`L|UjFTVuB11giZ7Yh}= zbm>Rfl&g;dSkb*KfAtIg`>CL7$CWZWW&0f{xrP9VaSCzGY1-#g+T8u$lgQ7)l=V8- zFa{ITiaEk9s+lImOf7A8?(dkN?_XFjlyII2c}Y)|90wl=1D|??qLD{hJDYI@O1We^EtT9FuSOYa-=0XZ(MLY0y_yIPi(_kBttquOLw)#ACUw9w<{bPK+WL z5QxRMy)+(VaOBUwDW}t1P3ynxf9cbH*7LB^&A{Xv0flze<7Jcz{p0OP zeY$gLk8kAm<51tFCrua5@q#8g;NpXWM?4{6dSvDH_^Yx^uS`IN-GNnBxT%mx6#R?P z4JIqG z3{HHG;&I14Qu;V<9~xBO&+-5`IWlyj<_x1~zx5?yV}m zkGrPG+6%!2zU}xGsX{X@zD+&Kc^r3GZW$>?zYpv@-BJ%xXP4ca>P>_fx!rOQ@#WSQ z*JZCPeNBZr0w{+=Ox&jr9y$g}c}PBf71Tf1yVdiI9}lPW^keum!dj!i^fVS?l<>?0 zjIc|>8U;8^N#~Y{KKsFU+RH22{AZ~0h#&SG+ZNoMj&QLF2yhGRMxJ2zD#dj%9!WdK z>Uil3YaBTZl!{jxe2tBY?uk(KzB!;ag%4;)6xKYQpmbFPVEwGCE;(?e?e~UWI{oKx z3);_m4rT)oBxRb=^fEBrI;jLlNnd*^1~6LD7Hh=RlAD@Xu_)nZOL3cbuYDB0p~leQk!%m9U$sniR7%r zr}&d6n-Hq8$LqyUDC`m2%MxP-AF0(+!ss6)Us2_B(ejLc&Sv!*K3MRfr*%naR^3y_Mv>xK{~t(S{Mb*V z&*uyX^-~FZ|2t7{olW=8?~w60^tOSjDS%#4p?~#-_Z(v8zFkzA{31Vkak#{GeZ>NI zA0SJilo?D6wri2Fd`tL=+a-Pt$5sEIL}@-B6DdBA2!Q>udh))Jgct>Y#cF%ROBt#t zSzAF0nr{7|Pz7-|Muh# z_HHTv2d3owchHEPdV7xbS*p4K$!tSyzsriC-f9Xtb;@l2pW_wRHd2uidEw#br!MX{ z+mj0*yPrKN;>VA{{^Zo?YQOKSA|~7b{JkJ2Cv|aUi~-t0kQ}78eJ5^uXHKt*U=051YC5?InAC2(-S7r@K^Q9PFduv6yK6ad&7uPH7b zIyBrik!M^5 ziWd$Yij0g>i-=ZtC8kQA+{U6vB+&k2L&Fy_08$t#4U`=m9{&3EtLH@1L{CqTa{T{* zDvy*74Gefr+yKC533>|v&QMI5JHQq!7Z#pV1EwvJ$R7<^Zx^#~o;c)qERhRbp}L!0u^iX#mK~tc$qG+HA8)CIJB5dCCoF zV^(|48vxXZy7*vBs}&keebHOT_;kMc`T6Dw{%OABNF&xW)Ku7Yz6Sj?$7P zH->d;&v$Ph(v9&TF4VcRAGE8MMJ%*<%;HAxJ&*SL_; znINDiE3x$^>NM^P?kRX|bh+|M=` zGe7`=0{mG5rozMXKf&dQ$f!8}RiN9f#aYdlGvBU-xOsySK>GlEe7Nk&K-Dui_T7lZ zvy9WOiayqOuJ;j%pq-}{pt*!+_bew6TT%D6+YS+v`rjV%rg#^Y;hSOwegb~rK~-wTtAbxAx+f*aeE*YOWQOQfQx|+z2suZW90KM`^^{fcl%NEtOI*gRaF6~ z3Rq4%(4rp%{R|T+E}xE5!MMBkte$d_j%O?N^ZGeB04Q+vlo{{u*=XlA@-r~P?q?`ac?_68`d@v0z;py(g3K-a{T)2sn?FmYHt+)kZotWa z$l@y<516KQ0;4&-? zfD+BT6!!#_YM!^7YUBIJzK&@Rz+d6BJ>NcT&z5~UMa3KoE^Xs@+5R~}plEjpv{t2( zz-A7-9vmzh9tK#|F8WlCClgd;WOf0d(~n!emA0nxg-bU9JbUF~&Tk3wuRwJ?4LId~ zp`_fMS~h0@#$;rkeWLsS^#xR}brCfToy85kjpPPo`ooKsQNz~FTTLm$vcdDDkMZqh z(I!;7FNXwy$?6;*1yIcNCfkld{&&t6eGkas-6RvYO2(M;pZEFY3()iboS=sgei2pu zXRnaILN987D40W{*k!sqQiRmZ9=C3IT1i1tpc_3IcMjy1R`_72^C8wvB47`z0Yf!T ze@A2gR^WI`X`rENbMlJ~NYujSRC>3e@z;t)ZDqfV3S3*W;FO%VrE#l4R#(Se_$A^* z{YsuRu;NPS-R2`_MZBksY|lP$*f6SJ16^-EYGNW)9v-mnX=PA-scV^P($h9B`&_I_#^jwqD`W#_hzL-IPn>63886Xz{K~%V5HH(4C%ur&n3}Y zFT&jWGxnO&j9jiyaLmMp=f>R-446nuP`y)>i`}Osq1btjS7Cn$`&mKtC~%KMzUZ6wd1w?5#=YkEjcs@M%@ zeR(Rl>2<0;!C4x=<<=O)2UF6|Ey!HuBx15=d`LJbb$+f&P#`{Cv)MFkUA)wL6F+)Q zZ$X-KO{Ub_t?@zaLVTY8gqeN~!`_JR-o8bSQyY~=VN(1(EgNkH5QHPIa-Gnfq8i%_ z0NIa?*K?I`S}Zoz2a=@Wf7aX^0RTDHvRRLgfsAF~D_^7sDR=IkIE$c}JzV^(ne?L5 zS;6$9DWT^@$wpjJ<3=UfpIe)c$ovR=kr$=-w#rTPgva-luamXm$~cmGnGiN`ZKA{e z5rs|AS+*jp^mL2lyLnC~Qv(yfX+!JUD8`~Ldh_P58*Z9zfuYV8qrXCG6Cu*a8~I}*;xxlm_o6vfk_lNftRiR zjdml2m|Qz)O4m$qY%I}UfS8|t1>nEW9!+0qf>#d_O!l##9f+#b_Bj|xExiV9clQAw ziN{h5H;u{J?gn#hDGUL= zf_wJg@nEO)oYZ>>G_hX|uN=zflnJa7lY65vJ&3C*P|$!Ns@b3WU%Z#_=H<{OvF0?; z$n2Hbv-UOc1DWQZl86Bl;cz)NkHg;jhystr_{;G7Z@ge#Q(WFYrKjC}6({nD*QmsO z4*oCq7-8OR%@o(p4QRF3g4>rVE<+B~6uyo*W)h~9ON?HvbOeIJ06IX+d|*7h4BWK$ zcr42JD0hy`Cdv!~mo5;OxsTx0XuQ8MHfRBde=-8qXw!x0o!{!5Zp*p~s)jP=^mwk| zmOq0n(Cbuo4(ejG0kW;4zCN^ik@7IvNmTx3^H%jaA?yjy{hvGoUBAWFj(=9xbCO)T z)91>)IS{C-eG}s_zhV-cRANaOz35HEV>>ywLZf8JK7bdgSPCJTIu0yf|1$M*qVlCh)pjId zkp+HhK{)KDIel*EqoXHZYZhx<{w!pU@6fNT@=W}ir`2475aAEK1?!FxhUv4a4&Fq! zjy}E4gXnS(H5i@J!OOT?h2|Yt_N#HH;-TPehAP%~GTD7P+sz;NiHErbYYXeDwFw^V zK6rLcn-?HI{LA+#V5VcJC#C-VaR7&TeNaVjL8^Ey_R6vx?L@psVBp5=$`b@5jN2JR zxbU$gNzzZ~goN!1u&{v^;NBa8#;CF2B2&{SDHe|m zNFe^JbZf|ykZiIcree|Ky_I#w>L}h4?j+=40wOC%(4Bq3!(IH!2b&eQA>8okeL#Sn zb4Q+*J&KdOV&U?5+wi;t9Ql*Z0w=)f>bNTla4O1%!ai4gg3T8Zp$Z^{C(vUX-7Fv3 zl9}5%@pV%8%U?bP{*ijVC;v{qP66~e)0UbxxrAkb(q*k8hgywv{2`FGpNPKiq*kMM z0_Q+f+M5Tch{)^-Fz|T%`W0!_KkM=Tyukks5(wJQ&MmY2@o1+sn>`hnw0lnxcfsw? zPk}ofFcm)HYMtVL)wjQ01Wg|xBCV0CC=~8?5z1+M688i<%Zndl(*QfcN!Ub(=3N+L z`q@PW5Gj_3r3`j;iB`J5qc+;WwbZ#U?)NTK0tOhI zjjG9G$d|OIN3II*zmBZ9m4hZrnPfHs^ ztDGGiCIy&c=Z%~IIszD2HwUJ*3L_pYVmV9waGpY50Qq6xo3{>h+;0~Fo!1idAoWER zf;9J6p)#iiitV?#7GT(7s8Sl3x3@i(7SaJg#5j%Trw$%INCd<%ZzR-<#lqiiVp|hY z)LElj$4c+MrS#+)!6|@j8>PJfX21Q%p#aafzWk_m zfGh(D>lgs(pCF$Dycmue3$PF9b!)#sUJevi2MB*31>$w@FI1WdK($mP1kV55K8Gi+ z!=Cpq`!G?-$XrWGzzB560VH)2iyxqyiBzf~(63`D+V$rHsa2ehZvG3Z_mS`ow|Q`R4_0=(LRLQOQ#UpnuVje+NoCok@)^ z&)yzXr+R|p=SeS3rlL)1hx*RL{zcQ^d@bMSb5ufj(6$-;)9IxDm(AeYg{bZGEY#fd zCHMa9Y<%>1Z{t~*jkCSqay>KBd-UNm%5N!VN67#mu${Lc;u5B1Zf8^!uY&G29&riKH*k%b!fiX6KQUkb7AS~n689Aa&K#;0Ui{d0K6(f{SqX&3DdVrn< zh}7uQ3f>1#A&4IlLZHY%=1WwLw<&>b?CiD^U=^J5bUT&-sA!U$sxK?jPP^{bW#~!+$1jb_MM} z^#y1U40guME^@M-Jo%#_K%xpGu7c>tSzS(02^?IPxsug@GJ=G10tWWNz^{TL)?iAd z`se?A;w-5CG;maBdj|6%IJ?2&2>1C6^oKGk#qoXl@=Bm@5imQcS_yu&J+b7&2&=8P zVEnVB&z}Vjq$;?9Hxz*3mj7wl?Vi+Qms|SO6ucbMH=q3yzP=uqjF&JB^e}Of#C?^{ zC)(MgN6(C#>S+2 zhhg|nIWF-5LD(H2TURGvl)Ra`T*;N10V4Aa!0EPzNIot?^yh$Kek(($uVfFid5r=H zdSUP8Lr5>ZIbD2-&VMenL-l%x0r;qg2N8>gK3a#`NW*V8W^@bCwHD0jC6;i?03$-$ z<3%sr(Y-RS1CD<{a@r!moCxdbCOH2d8g@p2sp2&?-X(wc@CdKfF6i>aDTGw3#f3ka zMkk=T7Ei>|%)^a>hJfYo0%AoR=Lg;1iy5WPe=K=fSo_plzGldVpHns#M-Oei9XI0p#@K$)v8VayYAC~Ga3 zjaXU3==Um|}=iX{IJE2xcAU^7&aU+7ekUpZkAgzkuVF!0EM< z+zyn}IiGq-QABr6ql4EZngAB6>`5IUc}h~`#RmnuNUbB@tc4DpL6ACrmr^xgw91o^ ztDi0w0hWT89Bf{wiO2QMk!57MykU>$55+{kImm9xeSS9IXIAC0`^RYR?IUC#H~P#G znLU1Z3u0H*5b7eHE_#D)L7-~19A|ic>~j)EgRFdF@9nzjXYr1bMo|LUhw~S>lmw@a zsANk$QmJuV0hv!XD^?wds$ThEz@T_Vr2dQAN#f}Wv-`|eZ3|Hc8t*o>#~kckkYht1 zysyd>l{$Jdb50k+uVhrUhNoOYmg1TiD|@s(Mjvj;tUWt=?2nQ58!zCcRuZr<9TbE! zAl;;s*gR736ysY;R3*N-eG-z`km8l%lSUL=PltqJ$%kKs=}VRZ$5if<9x%e{?cf4A z;DC6_%EM8;fp{nik_|x)8WW%TX+MZ^gnkeC;=f6;UL*&emd?J4#Zj~&ElTt8ap_wy z`U9y!JclfVtkr@hMrt;`#zu>B8OIM}DMa4Vs}zU1;7I=dXe9>G~+ zi&(OiucQooY}ksau9ybtc)2bSOUw|~?G<>}0e&+RKN#p1^5d|CB9WyR?iCccY>r`LCGHKJ0!Vl~o^CzD1u z@y+dx5_dRF{IqqFIMtC3pWad$QPpP_)AZ1NiBp{LlJ)M5vLms;6ea`ebfkIOFn&!0 z8i$L(5*iE6=9YUdmQgV4U-vSpR1$N=%M*cF5W10vQ)^zb!+|6H`Ox*)P$GKq(SSA` z1tp2unFSbG@-x+iD!GuIkuhRU$uxYPi!RNTx?2E|IqJDc3r{@LMwxfdKZz-KC9T9) ze}781U<&+t52ncUvjQjj>~&ThN5&`U#E0WCCy`r?@tQqTn|)%qmZQ-m-uSIT%B0s@ zt^KJPGs~;zM%J3v6}9d0%bT`O=iL4MDc@t`1HI>8R4PJw&f2Lvo%*}X06Bg*yHANb zM?7R{!nzZJ%ifQ(JMB)#UCAs5VQFqc&*}JI;>vGtI#qSPKXbo@tyDhQE8uNMFw);) zaoX8`s^JmpPVSK2#Z@g39CyWbtD3Y5?5` z(d5Ri(=Q5yNu+$)6zqfQG`SF$MPPO&ytb)eKFq2#H{#j z;q+cGV6)ZAyj@%0#@b~di`EcNs9cXgZP^hf{neTiF2&x-Xe+(C^x$r6_Jdc5?1Cv2 z*weEON{Hl4>a62~YF{r>8Y^MhP37*PyW`T* z((nz5_RqAc$3og^*oVnIJWnw%dIi-LWX;uLVSWAh7C5D5QWR0A3vSVH-rQlV-$wpeLxreOU}L~kCV0Gg_R-Ah@WOuk z^}AOZ8v$x6aCxgYrQgqv47u3ktZn*T3uJdeR`C|rP6$3_!My4P1cD%c2d?zpQ4=We z=wW6#%7k^*YHc$?IkL9agUrueyr}0cvVihmjTSsxu}Uw zgj{+T&5V|)`otG^3BLR!Gmo;QYx#oAi&cSsoDcN#Eiu%iOMPA_Wi5`4X4wvIdikX9 zy1ohH5w*>F`%d#(=j%F0M4Dw>I;R%i=gg>8O$Ee#o~ISPk+6i5BzhnJfzab= z*Xsd>-ZF^i*5BRjp(|tB>a~@tHmr@*8!-)28egt3m#~w-szFLyV=+WN7(jdm|kWF~a z{+c@FS%&;0ox>rLgN6cP(a=rkA<~(ToAoXwCERo2+k*I{(vEMh(DvJ7ql`Hc7SHD3 zYOMw#k0V$4Q)Bfb--~)As9k10g~_7uypCaqmww9S+sMlOrzz?K`^bTLDYtL4&Dp1{ z1PJ(I)EmI%rN?;2!uif;7Y-S*YRmv;n(#X^dkP)4w>2D3#q2C$Dd7oUBzwDI>u+L+ zvk&4=8SLk60_NR`$?^1j8yeHFF4+rqTZ<#f@SAJde7X^k=WblPpbJ0RQ197%y0tB@ z`?RqvJ$fV*ui+r}nt=K`F@($0nO5>B{bU{n>jmuW@Xsq~lRKP)~$fkJ~->%}$enh0W=Y=DxlsyP@vd!=4DzkkCyC zo6X3Ysgr41{U_pz1n$oFtez)|N0=gvzQwzldv$~sCFjZ(uw(B~Pij`}B}{wl&wN%p zl7V8AI~aJ6^e%9vNiH`0$7)DtPukj8Jvq;to|9;e%uV_2i1wv8t%`G)U*Z9_8w*Ur zNpS%*n6$UwR^F-iXv)EcZJ;epoStM~xJQP|2Yl>u>XH@+hEkg(O!RKJe|ZdYU~$`N z%p&ZBE*;Eckazs=1BpwIima;uH%uRiHng1=Y=DQ4ZtS9QJ6b?hjH%XcwLjbX&I`6U zpI=KN_e2^yN84rLJM?V2H&%ott*XF>Jhyjn96(Q?$csVG6P@!eZ~EOkySQ@BpPlXL zeOfjzg3-E6yiT<-Jr9OLcdGVAqY$oS44bY_(w;Jp*3vwkoUillMyYxD3Zgrl_gkKQ zMSqCc1mcvFwEQr~&-Q}J;)0VMCZv+oYio9L35Li_?NG@hSPrg*!hELg!;@ zm9h1Wmre~=q!Tk5^ zU*%y{^A^&#Y%Orn+IaE21D)dRba?;bI?!NfUb%lTy#vM z8^_pyjVm!4?T!Rk>l3rOx3~LyEd`NXa|@^mo!u~GQ;N`b!P0A#dsD7QN0u9BM@NJi zQq@<4$0?vaO%Ny|jYwUQ5DpbqjN82sCkYu@&lmq-_fIEy9y`?&&8NCzYj`1!qv*uTmJGhg%yY5Ok5)ffM2 z!T}UWp;tmqPx)&1w@A3%6{|Qt?k^^6LQgjQDc!dE{VC74 z(+_HoO(}({##hy}C&Vc|4reeZ>yaCELPseVyYlDr@-Kyk{!ZLA?;5mc2PL6rjlg7E zl7OSG(6e(~dY=b3J}F^ZHtr9HUbJ0|p0Qr|zG$9Q4t)1lffy1Ujsj$nq?}A%*q$Bt zf<2C4>R{-_F~ah6kr&vBI7Nhwvf*&+{jF)wO{n9v$4QlC{dTA4QH*DU<0i?)o;>Tt znsJrq@nFjNJngCf-xXgCd-8lm9(+F5>#;3=vDXXS$?ky97cRH!nXqiUn0BS~*m}P0 zaJoN5u}5}=TZ`|xu>C!&sJOLGcs}PnJJ;unF3z;={mXP zX_4pI$lQqT$&xEl*L@!*05qHPJ)IZ&+nK$NS}FUZCrKUbzMrx=CE&4Maj{=?aXi60 z?S$y&g`c%hN>InLa$pPjZ=Jhtp3NW|J$Hr9=Ps$gM2OY%sPm+#@%(su8`0JVp!v=7 zuQJ1Y8!W7G)&R(;@pOUpVqLxQ7(lrGi1K3FAAa%7E%0`V$a2?%dO+hY&gtG%{R z9e!$e;wx5Nise#tXT#c-&?A})>c1PAD3nXB6adzqI8HkwZ+VWMF5POLJ?-NBtT2BE&Q4w}ft{NL;Gw z|5Au`Vob=aakvm^H_6@1ApeF>KdQa|uC&ou%llhpwSrml)QM5I*Z*p1so0(a)_-Fu z{^;+zcpViRSTld?uO=N|I2ADJpS_hbG^+#ujuR?iv%*CFr3UqDV{gA}^gievsLGOd zcmIAeL@v!N>fiMl{|#W!-=q?n5O4gykez}53-Z4}`$-bV7Zn1~%fH&oRQyZ7aQ$B% z|MdtNV9d$r#NvJH{(dfG6#l=FB*PZE#$N<{nsUzn=)Cv(9biv)L#aQ&pwnH#|L>FW4Zum?D<;P0 zcj*#%m*iDQ1Z;Vd?Efd7|39!Zi^Xc;v+Yt3@ZE1KN1366*+u#*S!1^W-pA$xdeV+3 zNFjdBdj)&4+U`qIjgZ@d)oKjeVA747f+(1YzE`2tJx74A-kf);@3LH6N{E?YJ zF8Dy#`3YBpaGghtf8qC!Cj{JyaW!9Mt^w@6ik!B>LjWc-QR7daq{%g4K<{l90i9d5 zw?j2oc4lT6(}e<<*&VI~kr?$~P$WjKt&-79KkYU~Ar{PHIlPM^Njgv1JoAn_@$O+7 z$CW1eF#$~fo-XrX;vySyWE4Aml`%A+jLW#8!^UVHsL}K>H(r zg1Rs^HHA*irGtB`CCr?K$Qo>we9|cIxQq6pgJl%Q?{v^q$#%Lm1OI;g4^RK~#3J$4 z(72Co&Yv6|s!n~Spe0cvIoggy-z}NH595%CJQIyqj4{@F|Fd*y4jl4`TC2=5fMqV% z-4}I|obzA@#l<;I9)Pl4n8))}DLRHlB7Fd1yZtVY!L#zc2;x#> zH2?z^^g81Mu(#iXFgXFp{+WP3%(;T8$A96_4qZFHr_&!Rqdd_gd*>iXU9NbceA2QY zKKd4h#AB199c4@#SQs{;{&&z(&I{G#ef&!+;$8F8s+y(UTnG~HdGrATkfo?_A-0PW zW@!WK2;WLkql=_E+WE{*z+;ef1_^(ztdz@3=VDG#DtHtjGW?~066OH?RPC4;c7`s= z^^7-HvZCs{G&lls=*wt;;8cX=qz5G{+?04MBq+%w!2I*QwF_>lNn#1MTDfWjH~q!r zM6j2P=9nE{G7I?olgZ}nt`aT=Prf+z+DY0wNZ-0{kXy4FnHMI8euXpN70%0bY!l15 zUH5jQoPcxDi8`HU?+Ejq9@HGxb`aA?mAQrYZ7;Oe8Y;k26~y9 zqDA_WIAm33;tI5RzS;p4<7=b60H|r$4hs?@OIzHv%1KW(MUHT{f(mkB(s63CTvU5DJ8u&4y z3ut`%4nKIwQCG9x4F4hP3&XNvh_;l7bCN0|xV4)943?*axmy18F^>fGD-Gbyy3c{= zB>HDcdo?sTQ8XLdS^>^CwqA3MFNS4Iv&}(~Z7zbc?_trV?+esiq484vCJl z1nahAy)|vJP$yR2X?)CSfY(r1k5dbuZ7pLqJlE~?gD?Y?@ps2bTKB;tD;Emo?7bt= z8v}1V9?=kGL`52gaUPNS6%>po(O(6rFM?xm!8JHYXcJbxdFKz5B zU_EJ`U0;3%xCRD2IfHJK(mL$rQHD+=Jw})#7i~MJF5B|1DG}yU%l=AL2C7W@`4Tv91VeIE2QY%*Nm*l6a3n#UF8(dw~w0_bW)qFIs!@LaH#FJer=f~RKBGR1f z?L;0`g$(k!AI>V6&_7FLRDr*qZ!Fq-c|8B3_MQSICSF~_?tzAF2(@67C3P47cbgBF z8d846W2iCdJ(Ww7(T??NN<3#LZSNn`GH_;wp znT=8rlRn4GRr6OK$x21^G9K;&{wPPkdSV8sW{+;!Pp!G~$Tku? zI4w}0wO`{)$7_sa+wm=v3>7fE%+8$v&CkI1bJhHAO?IiTIM-`_D2_-N-$rNc0l0k) zg%e=6m6Henh=-Och2aC7N5$Pt;T&5*^<5~{TMI9Zh0C(cF#T1S7G%bW=X5Egua{5h zT!xk3fE-5O0Atx&Mj9Q>dz~mO;#E67;fnWOjZZZcffF)Z(_8^PtiN`dR2Tz39w=hn zm46!2)*bH={|Ie%%|8Pzmq9Z2cZoVW3ZqxNmtIE^4}pmoAniKH0Ok=C%+k!4lstTL;vBM-*lDy zlvO34bp7vjn2FWB$}PW{5B!E8zVa(mwn4{#CKNzj(Fc~G^zb(_Qz1d7zj^^senzpd zj32)_c^wt~^6GL+^o7@NfD=?#dd<^ic>EhG8#h3MGP(8f@7&&s=IUH0wtr1m{D+3? zsB(Z^no)6EyEOU#;a1aSssz+3d6nsW|M)ZvICeKpxIX+At)AJHWJkx_W!rv98_<3Y zK>LYrv4E}&$W``78giw9Epfpwe)HL@sUCo{7?pPKAIR5Gvaho98Nxw-&>y^d@(>`$ z7*!sK{(jLocolK#YjGw||1KH4^#^#mPAcy5yUfC7btO&oXJS}?luLe=EwFF>p)|#& zR}0kyvvdAjxQ~;_<7;JWX>4q~xp;g7RO$Pk7UYG|x#1CT7INU|=-_}1l~WWO-(Ozc z2)W=*&UZu!_q?qU{i|IDofD6vtW&3tRo+;s#CvkHH=k1JTwSj3 zW@s$YMF3X9>Uc#8_cup?uv;&Lu-UtlAlB1V%p*x%ld*(t#D_7r1VYf5m=S#_52Xqx z)56pw(LKT*>0p#`*Xa(1#brBn=*yDN;x^6;<{109Hg6QNS*jd3B_a3M3MS~=3~*aO zyyHj-Nob9FLPO5t=!aEFSD7%2?D%L^^E*y@pc^Ho{QuBYyxK19 z{s&F@U_IbwJz~ZE^qgM^lv%XPVD=cw`;)5ha04Qo2Mq#%~H>YDePuW5oUD9TXAM%|a@ku>8UkYI(bCy1dR)T~%Xp=zZq9(D#{@Sy-vAB@e;Igq`9! zq(N%3btxHO61F@s4bzPkNr=NrY*)$sDZ7zKwNQU3SA3O2W8*C4hij|`g=T>1uoL=5iDZ-9@T+G{MO`#%MTmT^L<*ld4<**LT5M7D!QCg zE$7TyO84o6C~!8qf|~Y;XUfdUU3*iMwsD z8i+F{U#K;hnQzGp;))NG!#hwaDMpV`gE#vY*49yaJan06n>CeKHEigL=68QoldbOz z>Nx2H(Z>mp@NarZYZj2ihP_I9ybc))40nmhZxhQgAUxf9!?E}I_?nFG16ElQikpGr z1J5;6y3ey>S{qErMrUxMUbWnn9qM-|pXTEequ*~vUEd_UEj9e&-5z_6uQX>w_npr< zwE|9X1KoN?{8!XW7k&k$`DHq5}orxjqcN?U=uNB z7l$t4LA9Eq*@J|kQcGdX-Y1;87L-CP*3V+uZOCy}ZQcZso=*Aj8G?y6Kg!M4_D3qx zU5?*vJy083GadNk109ObWlh0oCFIff5g<~OTL0QSUN#PYfLd~DlRalwWvjCC*x=xm zT(8SS^rn#uh#R`>Y!D9(z)DRo;Aap=h0VgY_Pw%JN@IiM5^Wl=s+t&*^5!IKMB~D+ zy18CCl#31zy2F@}T=He=g2|TSG-|-eB3L)$q^4xJ;DJ~D@L?eh<>p zC)gL7Rqd|ruzze zBCXPtN~LQxif7ScDPkNo0h_S>!G}*pMO>zrC1-|M8@Xi-2gAnLyL8#B5>#$2 zLM1Qb?|NHbn?Ee(O>w|tzJq#=#3&tCB0fWhxZHanw-)1>W42GSI_s+!ql%bqzwA7K z^X3mv3aqjQsCKua=e2<-6ZA(_`M^+5d5Df}$85SzzRNc|is?Eh(-EA!u-OKJsQYM* zFZ*C=Ol(J(knp;?I+LTOyM79??39wm7U^qx8b}%Ymp`_|+wyt2?vKD$C-iwQBocXu zRC-5OUUu$lBF$z?OAR|(X+X7N!KI;^U7*{7Hv)a5Y?z;@!%6Ck6#OQteu&7}wg)L< z3&-?!A8I!gRG*|7lI1aAH1}x0CWEdg?I(d2HlY1HIJA$6RN_di6LdnGsfb05YIzd0 zI+rI(oCSG}ZkiEUannNh9@eRFPmnOe1D|?M>bgjcHY;TxYgcN2s0k;ET+`jqW*N_W zKBQLadZsSEVToAEa*u-%sj)9%DU^RkGMr?lF(24uZR89&k@4Ig(OFpbk;`?lOMg9M zKo?xOAweW+jI?cZ(}q3jn$V7;JtOR^YSBIaG}zz*xwd9z>56%rBjBI!(b^YF^UkjA zmhy=8=R$=>R9Lu2{m^sc1~s=Z)5X!jJX5^GhcVAEa0=d(3|SbuB+zqBWPdlyi$`p< z*?tT$jNbepXR#7j*<|+4L|v#;=WRpsV*d;AY4?OnZ?rM?h4_!S(~p785}Vqs_~|Ja z{3MqVa?H>#IAD#nYPz%UgAlCqf@#^T>J+neKKL{yM#LJ0$Pu}FQSHf@?Fxynnd~-H z1v5MPDYBNsymQqewJB`zn&2D=G<%6s5fXJ0~^T?Y8$Zm8rW1VR7ZEQ(KsL`!XD3sBmkQZf?Ew zbHf`%rJZQjPU*OTuLRa1U*gDhV`a!^lX3zh^%2IXaUgu5d|Dbfm&9tYPsnU!4-1S% zO3AMq>g#5C+tJ))5ES#>;_}WvfC7`GO8PHj?YvchnvgHj~w}jNP{K*8EzO#%+Bh zKrvWkF)r7b_7Pv0jLf+`fYnS@Um!Azc^{QM^h8gfF6CrlV%3>Xb$=;|L!?E@SJCm6 znr#iPedo8-)!x@C@{pIda|=9oXM+v z$9qNO&qM?L0U+?_*|>Jg3#w`o9LpDN$c`g3wu|bzo;~%?O4(EMW=QqQ+Gr~Q*69V( zJRFHSVWYw`l?qp0nCnfLuU(;rn(-n|x~^H_4Q1$og?)E7(Op}Ushcn%QtP1)vApfI zZTt2Wn>h@LTGqUiebsM3PYZ3v6nTfL$yoX6T*U^-9$75cl36}W#whQ92=lpBL2M!W zdF%OgaAkZVqZBnNHl^-EnxmaUt|OQCy(qF8oG%Mss6OjD#c!MTzAKJK_dY^sw(lC! z_mnBGPE~&K?v2_sLXq#{$GW`lu;CPU1W3+vjJ=;)+_M?&QR`kE8_G-0N;a(}qp2wJ ziMlYd{n&<~oWM)#6$evg?EtBS@p-vHJ`BR6MjYr>BR~0Tvy)>?*xgsI&{zRiFD7;^ zs%ml1w?JqEO~oEm_GZ27d70p_%^T0eqaEg`pd)stGZSBhLt}@r_92d)P@4u?H1Cmi zNEJ%gVvkgo#bnu{T0>2(Yr(B|IrsM`9|0xQ&&DprSijR&Z*T1UC%y{IcfVRd z0@*J$=Pek#$Q`M$!agQJf3iS4V)%vf z`>#)iNHf1r1~1NSd|~S*6I7CYb4S_CK09B0zqycA#kKMslbLc{-T|2!WO(`qBQeY@ zuH2M5-+=2q9i!QHoEc>2tc>2rK*oK`g}A~c;y&a75;x;I$Qk=HVu22g&_YF2t&m3w zV~SI8L{my_=$LQd&ExtvLq#go`7TTW)w^2B8OQU3B9(gYK^V&W51#ES$=mB|mo=(? z)ttl?IK=3z7YPFQ!ghd`>Dp&XIb?Q-O+3uEK+8GQ}+3MpW?cWayZfp2a+Cp)XVKVkDDWy}TXu7O>D3XDg3x}S$2-%zMo zszy+76$pQcTqCXtTNx7s*PV@)3@GwGpdDC zacE|s#9n_TFJ34J4R{EgG_(oh6sT0iVM0sk5^^ zQOt!MJBp8{&0OoQMCj0(N1qLcQG(k*cdT2jUf$UanF?wj`|#Xfqkv3(!T!uBLx#IG zso@7K63=zaJ4d5q=e3W=kXB$&g9CTgL5nDZ0XN-h|F`Z^6JCs6uR<>0zFiv?^u=_g z&OZAMHdN=Z#(qW>_K0yNe6w~^kv2xvQB(hloBUJqUP%j`SPGB6HQyP`-^}KBXAD-9 zF6a!*aM49zQFb2DXfZZdeDr(-O?Kp7zmH^9eto^MF1ah0fl`X(a8h38#Eq3k_->F| z9^SiBYjpMgulMLtUG-q1CPp8UiFO^qX$PLJCuYx92ppBasGJ3CnvWP7FJ_oH_XyO6 z2JhnYrM1Rs`VNuC_gTTVd&24a&OxWcJuy_2u^z{SXHvmKi~MYj>QAfvL0#2#qeb!qltE)>$oXTxD(OmE`X|sNb0d*Kx_oG*} z;J;hhp^=z)PX`z5g@3=7^F?SXlZ+iN`_gD!CqE}2D98wc#Xq^7$7fTv@k4qPrQtlt zs=N_$0E0PVfT>h0U-1meM3(AnRK&Ea1qC)Hv;<0@Dn0SIp_J^(i?pqj?9J?He`+Y) znOpdr%&?!*JiXv)gnjo+O-f8}0K1JCNbfvXuMxw9rs88Cl0|leW95mNhU!;dq>cFA zzSFXV%iT{TC3!{@>4FkEj{R&WEu5mUT7H{yxXGBxI>>?*lV z&~^G#^HKY*b=oYT{9;YFK@7sZTkYodbi9$wNO{U`Rb+0z4J|e0vkOWWsT3!E)fDIF`{R3E%<<8ikqTw$xX=xJqZaeiVl9)||KUk^#sJBZ3K^6UfRN1!{d zgjw>*&VJa@=#Fi%M}!aFgUJo}-Tt)yhT;uh&_J(p4a#m3`sni@R zJa74cWvsq6`QG;#_&tw&Q?tc-m-Ee)dv7@8T9QbxML=n%NSW+XiYb+7J zCkGD#%Ds)U)8yXw-1H;77x7}f)YlZz{@%v@`7l_A`6%%zOI2&${v6CJ^VxCP8ENeC zj?t7^l`X?2LyUc4-mn4_Sf?6=Uj=GsMlwZa_8^{%RYM}d1Co!K(XS$#z?eEl5oAVu zea`}sy%-6AYo3Gl~B2}F^Gm#dn*8NqYObZ#fIE?IW+$Is> z6B+k;1$P`>(1+C17PTvEOoJDP z(ws6)aUs}RUU)+)amqaDgx3_*T9Y=kPb#H-khe$Z-oIM4p?e830pX}wvTU~#( zGZt>*67W}!g*~MaqK)>{ zKRlQ$Rm5jzLo!;#+NOXP-Oe-mg8Vr9$0y}=ceq!)=a4J6h)Aim*>JObmSYczv6(Dtdqx;aCMU4@1W8#U}N zn~(Jo&F7V0-NGAG`cYf|xrR#xZX!RoH^0v&`QGR=v&v9dAO{xYrYe4L-Ti1d-{zaI zvu(=u(`DBf7XP+8NX-}gwt0zS3N^BR9NIyv!!H+AOO$vSQdQB+qKDK}R|xjQM4m(t zzH_ z63u3_j>$SCFX*#Q)O_iFhVuh|U?{FA*; z`CX%I{M7@)qy}Yuv$(>Xxw+fFYA#g3pvQOO$=`Rl!g}Vw{hME;$S6JhTjB8X`VO)k zxVlI$ikX6G6>DnZaptlV8O5>+jgYsTHCx{;8=6P4aNo$Sz`H*U7TV0l-@~peUp!ai zW<&+|kc}7MW{N%Y?VALt%}75`*I-RWpt+DlZKQ6h=x7)5lPaT?)PH=wFUOr-qg4gd zHL*Q?BErWfn00e|jj%a?&+FE+l8kRZJCZK1`0RjYgXIkU%SO9jH-3=IYv@SAcjsg- zcfnBXqL}bA2k5wVHtNu@i!(11cGWPlyJ|+7G|BU1ILqFzMNOeiIa_os#>xc4ePUQU z6TidHO`^s~Y@UH(qt@R}yI{d}%EQpHv1Z%JvZBz<68j6w0)l!AvK3y~w;pZ3TD@)# zjhP-OeBRw&>NT@-gW4OIGhXKl980SHHF_`zbPTagm1Hmd#*+NZZYFsWWb^vsdlEl^-BTM_IGbyIH7GsMN~0d?(pV4PlHE?c2bxj^YtOeR z`3iKP1rwU0MlA{z8R8UP_?km!P<8>u9{8!9!t;`#Xbp{qW!{@ByX z{^Gm2M)CNfR{cim@u!G)%`&7BH{X0sMVGdJxo*GpN_Vd}@Wc^`EIhWTNxGMNEuJ0M zyPcNVK++!h#*w6t7uu@#srPl&4noQHxD6|DJ(9Z+KLlny z3hgfnSAKlm=<0;Mx_kQS&CBlhK(nO*U|d>&@i&u4{z@s7Inh60!_XU^9Rwurx>o{I zLKg0ZGo_xi>?wkD7mH0FxCyx*Hs;fuP`rNsbadQt_;Hw!hLo7ICb%Kh1i9*cvhw4w z=q+8Lf`7pgxeXbHo~n)GMWOij1~13>R-I5L zJdK>ILYmJzCW$ zA9}F+)gG=>#5_LAv9M&b#p<^jZWZUJR?^!6HgWg0>82-Ik%`IekC zcG*?R1auke57PUfVZ3o^?am$DtA&ru4JPaF!Mb^V$yx6$iXkv;r#x8`)1}rhT;u%- zw8D4#0FlvPnQaE`+Ss!$K=`=LuprUeE1xnB#|ChgS%B^s9+hy-FoV0_x%MplFmiQ4 zj=saqFFgd%A>m00gj97JhtsVFe+ULTV4+tn1vlS1u)0lBXq=keT-q}xHM<#-_v@A) zXh;g8^DQ2A6;p30uJHEO1mHI#Rq1&g zdepm-HPsnxo|i36oL7AoGeOyUW2Yvp*E)_Xm!BDvYIDWf{c2f|V`dZZrJ{}Ya3A(8 zc;SSAfYU>f>%7+4thIaEFeIMuo81rrS6+xMCtT<70DJeoHqvSTHfj=W*pmh0as!Ii zDz(Zcx0|1+@DYtINtoCK%owers7Q9`0&==dx1qF7DQGGU9#+TsSS8^ z!JrMRU2Qmuz6CRz`m|MMi$Y3ACf(c6==?UL{&XiXsQx^!SAuZv6Td2{lH^+HOU_SK z@}XF3TdMk<`Nplf-;rbT!%3tDc2btf}sHij3rqov%i*B(;LLJUW&kxE(khQcLqZ>T#VBsIH9uosRA8m!#X1V6c3H zS+hdRD&=PKwLCKxpUK)}J=5y+j3TQxPk(c_7g4=Yhu>g~etANt_^Ej{YhHPM;xs}* z#*{I}0TZstqi)XfBwv~J7*$k9H`BD443jS!HpQ+J9aywdH73N?sHhx$@xedNGWv1q zt$5jgrt!1;8xkwdn4f-9UDy2bn`V?yW-;xZxiw71fBKae@nboC!(u4rP&9QJf&-Na zq!AXnM!06eCYkLdJ*s1>Z*9jH9rER8_M{A~QSx~u-v&48_cJp`1K$dAtI<()NR2QM zvjps{&p2%O;1;CtE?fSsEO7gPbtPF;CYQExI_-jJt8hj)kVHl_hU&a*E8kE{%Up?w zNI2O6&C}d0bCvNXjTlleor1GPRg23EmQE+3x0+_}93eVi4+X3ce#?A_{xK(K63;H5 z4L5p9%Rz`~Xt<2)OfBVYskd|RmZ2*(9cB0rb>KU|QOnZFxG$U>id&6Rk61`fI;QZQ zJEB|Hd|$9MpKrRA^*B^osuAwm_U31}KEN9AxxEpa?Kkp0mrCI4LKo9g9#(Z&r0>b; zu~FO9f?;XY-G|EN={L@)M-sc}kY2ge)3W=$0n9fum4htMJdATx>p(L-7f$_l<0YKv z-3yl4F?i0e!%D(xv@|kluA-ED@KxjUelLgCpHmqylwBZ4PLt8O9%H{`u##88HAg}` zw{h&S3{mS4N) zIeEHhjxrlBpt(?FuZR%6XN((#YP*RRoJSab2=p^GG*eU+OVb`vr00JB_ z9=2oIvX=EoOrPZz>vi%4(Hjk&AL;x?VtM!zOKqGRD_NCQNr`0)??m~*)r(f8L{$Yn z4rkpCrX)478v~2@ZoZxxYt|XxGVEW?w_qeb*#4S3K&~ybG|vvZBMQeOQjev7;wozY zyqBMoeKiCT>KLgghok?^KUFZmPDNpN#fkKF3OcNGFmbGkK?sdlz=d<8k851^ z+k|FgSf>I)L#WjX8JQeL5|ro`a4RBnd5KhgW%*){gR5ZgBF#Y2c;ugQc3dD9>hQuF zefvn`*4f$s->kd8r^7X66P0a|XDx|#s%IZu+8sF>;zK5LRx|KG zh4ZmZ?%-tJr_S*JQZDt;M@7(5LvH+r6G{1iZ4$taT+Sl_4GHA2MqIz!yu6wM(9}{y za)w{;>;!97M_S~-C>w1E)du!1&N$K-)e33H(YkjS#pBIP2TV^9+Cvb3_fUjyL8%Qq zaIbQbG?vLQ%6FZtb~-O%d62HQzae34-QmJXt7wnW!XEqiTf>?|ZIXrrm$%L9PiHAw zUfrM*^6k5aCj0rLmRVBnabD-MANO<_ax4g=Cgj6Zh~iau2D9)D^a`kKRrb|*Us3d; z#o6R}x=`>5B_fKV!}s|;3P5WbqpOOY!8O{u-TbhzxUvUCR=kq|jb_c=uhb=5#9Iyh zRQKr!*;};szQfI*-Z+(aj5T!!!5-$lx6bnIIKaDEN1uhl%L`g)hwrD6xx*?L!c8Y5 zvboX_NSba1-WZYPkS{j5>{3PR4JZ56l4%mPU0}fB>)p8W#c%cE`%(W0uNjFo+_66& zV}GhA%#1G2AUEC5)@ljon6k7SAv&Jg9r!$*!N9>A2pXtdoa3**=I8oCsVYc< zwD$t}&a+7yO(xa=SfHPUf4fmA#dzQeV1>M>c zu`Vro)fGz>P$lkI^R}+yksG~FF_+13yz(^@RdwzWat%fa5><`cg;fU``165ls)Mi( zC+0Qhru1X^uZw^e&OEjm$yZJ?4=GJ1qF1%DW%cNrUrYuo30Zlr-Ksac)o%QqGc!om z6BOP~S|cTMO{}2e)tQ7m1}w_}v%_Kb?6h{=N%w@qJzb|&&7OG#9O8>t;UP)iqAGo{ z2+MQp$Uf^>JoVdoPVv4W$@F7U*ypIpnDV5{8c5UKtHw{Om&&1k&dELTZ8BBHEZCQ- znAK#t(fXLrnY93nHGs372e!b?na+s02A04r5QI^jifD|K8WPoL)#K0@ zLA(6P;M9;gE>O;Dq{QMW;%lCrT+Qmbc2IGa{glN0=kHSkzfKgsa#^x8bal(oW8J6~ z$)c8>=iS)MLxLejc`Ur9YOQD8r8Z^n))=`E{i=E{X2!Nq?<44%M9ofkKwlN;E3{?E zDP)=Hlb!C!7`b-Eo5Jdjnn#upUgfDOjp(&Rdol%hh&8#}(dd^i5#bd!u#F9H(zYmO zwiLBb>)EmIcMJ_lwaz_$-qc4e(;Ia`v#Gqt_pyd31wYV`(7m?v1`Oo$e);KoS8z_5 zGA2RKq-O$WJ*yuJ%Y{?V&}#PTov1k>!5sbINA z?T+IhLm7Cs9mm+3n}PND^b|bGSEb5@uBt3RSDk@4IiSv&(zSCWj6`>HV-voIb~;Bx zaJJ4~3sTo9Q1x8}UDqtZmd&Wsm*m|SQ&;`E#v4aEpj&yT+tt^N$uRzd<&b#>zQx8u z>x;&~$^E){wvb88Oh>o6a2s_TmF=h!nBe+g$Zp2{ic&48fp#lCW#9QZHFTOH%PiSR zEYX4PEX0IpRCphqrWZvs7H=QZX*=VC?_B`D`#axi%P^nngRC z))kWmH!-9e9^F)H3}V;yt<3#;D~hqWF2Ojp#aal$6b>@9sGStJPA0u;ud-vu20N~V zh@Wn??o3nrG1yXB47Aqr&+OhAD1S9IAZ=Dt3Kmn!B0QieaYn2s{sd%skdn~zhZx&shj6<@haiF zzzJ6Mc1N`wvBEyu3VX*3iwccJ_2tR=$HkM`vk(eqDb@ff zyE+z9I1bcm1zV!L1h_X?jg;KReWXg#OiM~u^4NQWd_PT}E-Ks*z=X@dGV#gnyK8j&PlDQJT zC9eAt`{RyQg-r%w7+2+gVqOGwpuW#==SNzYouGxdIeNV1gEnQG_9uRt>o~lIkh(qz z9U^+vqwl_Z*|Gi{L})c7K`iMir`#YG*H<)N6N)c|Zb6^Uq9_+6n7T*69y*BRxg1!m z${su1c5Of~s23mtxMWPa)1edJ$r$SfGRH@$0}sW(e05~fciHn+s6J$Zh10$z2Q zXW(b?EUjFIJ3k9GO+1)EjK27-XYVRbl+4b|HwDvY8o^=>r~+rQmT{Y7wnPb)cFyiZ zT;vc@_t&0eo+lC4x)mg-xioj)AH;FzTOvwJ+KVk$HB*A+Y|Yx8UE^uSF5SMcUcr^$ z%C~=~$oCJlYba*7Nnc~@GZU!%!I&nptHyMVxxF;gKh%8q5s(SX7OVVI!K3o>y2=@! z_hMYh?mslJUA`7=j+8d-_e()YYpw+FFs0e&_nT(eB!P7i_Z;|x|B*@Y<(pZ6xlh5R zH-C1K4+6!vCs1nce~MW^?E@&H18q@p5k7wqgLzX9XrBM!=i~K9KfU7BLi5Ctz28k0 zdJ_PJZ%;4>e~*QGKfih_3fqT|68OJ_ahd*K5>3s5{ZE1g#HEQ>#B_GwJNTm<_=5@9 zGBNX+`R|o7z}P5&W4nw4|8MRVive<|e8aYa{SV?-Wlx~T1DM3%KL6eJ|3{8-H94Xu zfO>w|&!FBf51yzQRDb?w^ZFm0xzeMnp=ZCSfw6|WFaH>s{>_%F-(Fq~h5tFM@(1Nt zTXF#gk4xcYJl=!1e`v!0v5nf#LF|9%`A3|;YLA#)s4o*mMnm8uStS1_gx{E>dJ}ng z={;5r7Vx!N+6wzQRR}1q7YZ=IPp+r(&rG)?-Yq^EPTZeZkmff56EA9|8;#9U64YTn z|KXZ945yffeRNk5<7-o;bpKZ&64-moGSYzP~gYcO;CwP*Mi*1vU0mg~8xpPe^V%^P|V8 zehtRg)e@#$0vqBme(>S#YCtsz)z>Am9?PKX*LIo+wm|f1z&l&`97XFCuBz({`nOq1 zQIg~%6d~+rg-WLsB=_ZXuZe4dEIOBpHKCVDz}ctx9#CtX)spRY*kFO3FTE7X!Ys&d zWt@|Av@Ts&)4R#%C-b7rzMFNKUgwtHrg_?c@y*9*`N4)jLg!SQ_U+umSRG$V|5L@> zIaP;F*Qg+G0eiU8BSz7>?_CniV+D#1CU`_q)R&4MRi_Ml$kAH`ow8wE6>MRyn&Qp>B%nhuOy{kXsU9*2m5J;tJ~}q7((bVj*=THleT$$MiS#c$POH zlU7&3_^xd7?YgDC@;CT3H+LuEG~+CqlJeH;p$-aMdSk zQAyvZmgT5ol}>vH-&+y#Dt0ak?J6rpOYwJW+EE$Xl=#pWJR&U1bLNT_4EWiJ;tl3c zKsw@=TTy$pk0|aluHdMIwm(Im{f&H{&{l z$56_9bL0*W$|xn!pnTuo>3q--4%Ky$Ir_lt)grUG79xeerJOPT9%KsWi_95$^*-1- zJ16w@=BAQmZvXw8zu6f5$H%_4DL_K@_k5qhXzcmLBHn~)2MJ(mV}Cj3@jHIohsQEq zO+Uu{J>HEkv(r#kU?-%QdP!7|`@+cDpz>`f!}u{Ak-OE-@PwO=sf+)&^n*mssOpSJ zIEMMR@iH46dQb=JHML*ZEm1?9QjltS>Vm_dD=AUtV zaP`L#iQP%_Ct*@yW^AkpmljJ>ltbT!2C$B>++Rbi6;VQ%q_oMTn57R6@S9-s|O+;hxQVENuOdU~ylcn5Ax`m4EF;^IX-(+NR z`wL~E&c)6-z}Muw`ebV5dNzqLM55Dq3`nGpG0|74x;>QFw?X}S0SzQr7_6RRFOkl{ z2I^S0S9xl}q2Qt>bw_oaZcQTw1DRy#TT*H7^kk72pQv6psDqWFvzfD|ev3+5%~zQJ zU{OjbOT{{{r9&wf!2m1aID7L&urpd*Sli1cdw^TI{b;|7-uGh!;Y_4grpu`uKe_5117R^hZf1wqmt zIQGj)D|Pn=3WzEtJmW=7;JLCi=pVk+{bc6BhO+3q=YB6@BF#F(dR4BTk)(X?T0Z~B z2=q<)%&^zCX(~Z;Tr2`BUPJ1ft@^4gadvk*@G;E>b%NVP#ypLo`4y|GXw~NFc)E4G z`Wx42Qn6>0#^0CCT(2J6wZU(UZ?p{bSm}88*bNYuFHmWBct5`R2Kf@p=8IPFIeS-H{U~~BziSJ#Ex4@mI zqPlw+%jj>7ctKU$F&TP&W?F#!4Ids#BRFrQfhnP!eGjcO98Abwu%mSy-goV#mi6L8 zU20c8`{EKqA?IH;zny_T!@kn!i*}Ot;<+H{^h`(v(S!@?dkd3VU4HH|PQ;5^rj?!q z`^WpW(c9is8WPu@MNp9TrbmCO2`w5@U7L^)LP^1p8tceI7*agWnP47bH{^H1j=gjd zBV!k8y5%&4I~{)KKJ65m;embFeD>v_kmvc2YLE-V-2oaoIYh9pJkJyvoxH@$;UlO9 z%(@$Vk5M@vc)QDo_;FCV2v-6V48#YgW@QO+AdjzOd!<^DEM$-BTWLq$%D++E0@O4*WV#cmRk)96ie!=^?Yr0A_C&RBq|6R8d0Ixx1NzV{|23#qsH9 zv8;4X;Bpi<3&| z|4muu#Nh&y3pDzCb-eFV(K7$B$nOA+v!E#a#|>qpzk0jZhv@bj!pepcMTq}mj$h32 zEsP5baax#INByCc1A7^8FHOa`B;M0&pfiVOJR^JP8< zn&^QLU3xR`tK}a7-Uqv$e3J;J}<=A{HdY7~_c98@w_A{_jdr6#JFPwBXbRgJN0@^nhu08nfoJ#mdRt7NV!K3`c z>6dLv#hzjl2-K%~b}>oJfYSx+zW}3DGr`(R5xmCJvxHi_BEE3CyhUyTE35An_N`0t zYn&UGpz?nX&P@;WAOEh0U|7=V1TOpa8BC~ek6RKi_I0LZ_}@;sOM@?iev zzsa>YN(u5#s#FGg9*%S-)&&-4Pal?A&>jDA)xAZ~LDPR1V*f#`WPz`_{Xcxo4F;I< zoE-Ne;fg37+vNapOF#OPEtVh`i2t8H*~NtHLb-g3mJ;-V#xG9x0xb$w1apA9B_#7= z+-NUT%F@vCCSHOpVEpI)xZOVYfZT!g{pV%pvuaBHkJEYXj>djt>y@||6PxpP&sX4F z&SaX4T|V}kXh!6v(1D&Mx%8&>3W5Ga16t|AO}!+L_%Sa1?~757BrxVZPx?Z^CJQFW zED6`A%gOUUw4nXPG`|9rTkN&V+JQUTw4>+m*Kk?Q`I#qgldG45A9-#P-D#4hyXfq9 z1~8vL%!>706pXw`ai9-UhFl7U0;rCEdiJ10E8W-xm(`pvsSV~oUAS0MFJc#I&cAEm z1}inpZtU%OVijSq+|D1Be(~v2Fjs+;{*jKDP3>HjkP`g^MmV39N!KsCaG^3JZaumj z1A+`NmNucpms~vf?i~A(khZ@R%xkbV{t7P;c%o06dO4YxIGvd$7%yv2X5^&EnGAkS za0AJy-MGpnqU$c!r#;yFen~W4B!>cuUjs-`J?zTrCF0Z#^n!i_18V)7dg`-IJ}C=|9WaIs-tS8jjo5G+`Z+XhaN8qtG&vt zm;`57;Jy z#lYWS0j;)WfW0zKm7~6$l-p&%$zy%O@YJEF_pS)9)34eEhRMn-S<3fPPZO*2O_~~U zmv8!LfO+A)iW_TU*+}|g>&r*A*W4qMGJ|{5uH54eOhk*BxS@)RYmVkUOqbnIVPxyl zKm0wXqa(4eYKO>2r_G;Rwvbk>V7(f?zt?($=WRk$tx#P#Q=AJ?_?=q{cBb9QW9vn>$dly^@r%*UscW z&wpB;OF7PW+Yb0J$vk{d0V37-h4_4JNP1Te&;};$=5p`6+`^ykcpCD_^&c5Kg|XF+ z*;ZL^ElWYmP3x9f@IgAf3aPo0R0+BdA3jvz&2!gPJJ>1z9CwKGDJt6t2&~y5Uy2DD z`xJ5aEC3eUz$U^gs9aHSPnr2+P9$M{-6W2*Zla*~yKgUn$o;MyTWM|EhDc{knyHKC8dz3_T*eDCvN`2zp;4;Y?QBCAe{PrkmJV(4z z*mQ(zDV1S#q}DZ!FJbc4H1)eWRni?zgMIsTXhl!Zl6~5}=}@IL?ph$BYX8g^(9;oPitwfuBYEhNOdlPMEW~7^we66+FcuZwX`=Kud zW>jf^(E8~X8%+D&Z5I?Tnel~+YDrAoT|_`mTk@Bk{$XNixx@+49fI~?I5qHZ>GGBcGR?smfX>*`Tfpl+H=ZXb#G~t zbxw>Jx{vuI;nxEk4t1nXXl-?+d7bYFN@uU>>otc0I|gQ>gJ0 z{=~ypbGL-vcWrk(j6mJJZdb{*f6-oS;RbYO*H|l^iO*<31~#r^W$F`5d$!i=PW7*_ z3kIX}X*g#Kgy9-J0-u?%jJBvrRjW^_s_NL3!jxh@Orzb?s`hZ$yV0v?;(?h%);f-} zc!)FHcusF$@fHCl-jFl;*Pm!!X<_XHp5E18V0oQ4W^5gGVjZIvx374a#-t;<6?A4_ ziCfh2TT*o}z%$Q`bHVexb)`r@I10bNIP*i`vsx5p8j)j}pH2O_4>r z!dz+p+2n2Q{jeYN*LJXko8|8HF&(+qn6unV_}tyuKP%{0l|nFeT0~yoZ)tmB8&E)) zt2lAIn&i)~3Z`{e%RTt5&iw&F{Z;_G=^t!D@%1@==_3wPuZn)O1^L=4XJ!`9rgYt! z&PS6E(@?g>U24CPm%%-DOEX5gCU2>>oZ_Y|izh7~24SdqmBv2AtIJr_;2t3FVs~$~ zufw4Zp7xDcB6{LA^hk<|33lb$16Vum@{)&`{6D^ z^aJ;)X3~!`KoD?enHu6!WtS8LZ2)9Xn

{PY1$%|H0U7 zSz2E4nruo6;~ee9PA2WmGe`>+*fX!mAFY3}{3L?$&Rz!^YiYngLeRT$)fU)R%{Nr- zmJXF_Q<-w3V(>e^48Mw1;rrgAIoQ5O^-D{LlW&@Ag&)&YrvrgzU}fNRHno&rH?^w{ ztFT%Byr}1%`ef+fw>PHXhjf@_^%2)NUvH0@gl?k8SFo<|fu>3HdwFS3#sj2LCIfpW z0>&4=PP)J~HQ&7UB4WYF$g5gwPUSk${9~6KAyxV1^{0H4u}N(wUv!hrhI`*1xV$9< z0SqHAvfFU{!YS3ebsDq6RRvQose7SSQ?xxOZD#`@qR|M1(=&r_P&)v9~ z4Miy1c`~qLkwN7zR@Ljmdb$(4I+un#SlNCJ?*3-gsns=G{x7`}qb42FtxG5F17IQ} zTL~j%cTI)8RJ>+5Q+I-Dz@>EA(IEpbah4#UHut!EQ9NskBODOr9c`SoGWseLO$QhwoyT+i(6X=G51XJ{o=AQkwlSM?7@A zUySV1_PgS(pqJES|3$9rUYr0nIq9%2TQ=5ynfuEOmVw?+p?T_>XPT$uYimpWIxF*% z`1K0WvcKCmv@|RFRZXnfJTinmL_J8EeP%(LL|`?21bgU9PG7yc2QB1%yvIOgcGt7ecs)pseV_1JFw8tF(_o3mx^UB}PfvpjoR))oWn zltgJz>R*7$DuHvEA)5y;LRQ|4j4LfCn=<-h*WYK@WziU}<66SX3Ua2!|NhMO%Jubk zK^#d5ey5+S>Cja7!?Z-FZEW7M{}?~mRzX-Ds7Xi}$g@(To@bK_exesz8c-)S6Pe(K zxL)ZjAjC5y)e@7$DnjOe_xUCL{OSgZ9m(ggJ*Vn|sFy#QOP9VzB6@!6^!kn;M`?So z6Xcus&Yo_`$llm#o^sE)TE|%#{oYX#-rv0Pt6!k9>XvR!&Y`E17wd^>aoyZ*DH(s9 zWOk5IeO5tdq)+Xdp6e3-;rE0&3`k2#E`a#ZJ>QPcJF%UDyzpWn6irwV0)nK)o@%oO%!7|n3i~!}^ z6u)6JI*8oI-{#zZ>xPP~Z~gWr$G9I^HWHeF=T1iDa1ow#1qB20^9$!MZoCRrJszU? zYnl86)ZP9B3E8?7qJe*ydu4B)+r5hqY^Jv|YfgJ?;icJ?Z}S~@&}Q*X(-N= z2os@iRV%Xkyz?o>t;#;Jcq+s7$sH_{Sg_JL9b^?8=(Lrx?P5!G>!1ZoeeF2PY)d3r z+}9(x|7FrE99jjptGAnj{~0@K9|?g8imvrJ3qMR7>3CU7IF ze6ubSoO`EVKmcm4TRUsd&{JakLHId%Ndi9b$hPN7{s0{xfIm17`}D`pO1Mm{Do_ma zr@3lnoo9a)26wCs&_;hY5Wj54zysjJjW_mP?$-@HKv$H3E0%Ud_5M6irVZd1OLG0g zi+6i9-#mE z(xaWl1)g!I4fRD(gSW&b-yaYld+BBZpYzQ>$=QDaa-8ulTri3Y;+2;D=cPxhcP@z* z9xKsHc;x0Kgv_`PKS({|L%5x zvh$?xg$lf*Tulo&mlODM8eU5G4^HvyViNw*-WT}L|MbV7ixZypiopIEJ#tz24-5io zZ9xAIt3g+Mtn^=UHU6`Pi+?rzcWeG9iTv}w|1F#UgXRs0!8urHI;*dBbIY3|ey!AA_d3$bblzf@Zzw}{%KW%)I6kgSU;lR)B8snMfd4aBNW|zNRcW=p1 zNT}fd;4bCQj?PdacjpXw0*oiDD9A5fFATZ9hKoLnMfOwBgD8W#;QsR=BAmvtmD~+I z;{4`auX}?dzB4~NJEDdByaWnP<4oflh&R&8Q6VeR-fs0Ld(JOjnA_S~IXc?t zPb;$oQa~Y2a{?KIe-YVz)W%=G(Dv8QPJc_dm>J9;mz0~oZo}Pvh2ojMe$A7@3q$wsK^ew!9hb;v8Tf9T= z5|9xw!lo-ADaIqTIE~-O?Lwvh+8$?9dJOh`Gg$0gF=Oh(>SM{WgvBoZic+FvYgit5f}v&j5gA)6Q0)r z{cib1`|hI?Dx?p2&*NiLzQq=WARl94iYLP01wSetBnkNx(fy@Ms#kH|fQBG_0Vwhi zRTM&em-6?KTh;Ay_wI^nLHe`d((=u2la|g??uxj^%(Xqqwis{}m zEohCZBaPlcEqsE4-3Kb#=~t~eA9I#FCe`u2*G(tn>Lj-m+6*MFe21SYR3;mY-$`zIUwek03x9{Fzb zuex6CmT#Tl*KrpzLEfl>5jvGD=oQ0(?kZ0(*B5I%{a0B}MWMgN9^ixi&=ssQ_mt0n zGDv#t7|B2I0Q%#=Xj$O=vr12Ll@LznkSF&C7#{o(Ccte%b+YQBL zDSK)+`M4yhhBrKCOb(IUS@Qi?V-j_3!c59s2N~r{!vOcF$!&Khx5xWRP4jg5yYq~e z1Z#XpE+h|DcwFxzsvy%GTAyV$mm|lnAsgG5V9*+Q#UT-YrKX!4IGF ziKo?0w|)sk&5vkyJxyO~Q@tVQp)|}9A0WuGk6+Jc4;OW_b4l^78ee}*E-)U$&ZAIw z))?~uZLjXRV0V5knnoSJoh6(0esGw&28RB|R@UB}AD0gWJ?XYqzkEQN?&>%~N|%)R z!JR@2YddlCa(8fLT~|EQGFsod)$vF8a-SSiJrsYG%4hZ(;_F1DrT@gR_ATutCoWEM zo+HJmC$>GNp&yf+cm3L)XP;(i{7T6_vVQE|-D9)0mCZ!>O?Bz}rccxpvV|on&GZ6Q z`AYiM&PC(*ELK~oWl7WRx?XBavk92~r;`ZhT3FxHTT2s+EKxSoV!MP=c|_d%%I#>F z%>faCUV$;6OSg}OT_yZ>D%;ZFL8rz|DXgCa&{&;3$_h#zwh&{ImpE#>%*B)B1VagG zT=&|eufe78P(^G4-j?MjWoxFpr=YQrSzB8%kITZ1@vZBZ`|X7DYU-lazb~_i4%XUk zr&-5J2W~mk{2K3{7$X=*I9PL#9USDuYv$NC98|eI`PZ5DS~&VOL_d)lb{sb#`GqTU zbkc^_C0Ib}Y=kQ$s&Qj}KIh@<1iIVGp*`+P#+A7wF-OFjr**7E3)qQ_5F+zvwFN$# zOoAJ^2iwNmS!oZ+$RC*s#SzgwM<|Sc&9AHIiO=5SBn?jL#d=dkfBa*Ytix0p8?{=H z*YSZicPKT?NmI~6D2H4?A~2lS%!mg*AA)_T>X(g(A!S$0S6R(x7 zjG$ePBf8R~5<3}KxBW&_T7p9qMq>4zjA9W7!m%l1^8TEGetw1k*c zsrX;^3#pPXaM#DXCj{Pq;TC@)-o{hcmc^xx(`e46Ebowmn=t+>w!fNUXf#?!;st#7 z0T+ea(MEsgd&9A>dh2vTH0AcO_0ARSl|qrVZ$74+ys))Mca6sxJzzklyo@dvfkEZk zUfa+cNkq}kYlNzH2J0aI8||7aLZqX*6FGIe(8us!Y6Q0vU*@|btS7OQ+Xf^mg|zWx zYON8JWcuODYj5dfBv+ri98G+&h+5iy6gKS2X|Al$FGKoj&`ozY`ZmXgrLjm&k%hMF z=4)=~Gt!JD%N{rzkJ--SL%XLCMBM?yUN*?u*_R%gyjw4vpD}lM&tJU6bKK!`xhqU` zvsz+R&?6}xLb;pDweO)Ff0f~^pv;tCaw#LK9J7^KLN!zg1@^hXRSMr}K}!>tq3`|^ zmZAh4au>O#iCovyMi?d*>&ZWts+#GU; z6k4XOcmKNgG|1k=rp$GMVf!JZ#P>{u(6q~sn#vTF?S6c_*L7fX`ieQ&gk92F8ca{Y zRmP6iNb(iL3tcVA5Q{bm6da)S)|dpvlgU?P$lStmx`tb%*N1;8GGx;#kIztVw6xt@ zmBit6s^eI)9zXJD-HDPcudKr$-K>BqiTB_%PRLcUL>c#04tta+ATZy2%%eNBl$IrW z3gIxCT%P0Sd<&YHM;Z6lD+vKJFRfBbYdZ|mD? zaAWyhTlRm%0&Mo_y_*RLv?GTfc^xpbe531B9xsT^4aT0SPG^w7XyjIfObEJGx_=<} zz3TNr&NwvC&X=OKYDU0kX*uDG0K(N?%73~$-QYQSDlxP1WUnKoY+r;Hr72=9?h&^H zQseI#*l(A~DRD8fnjNmkODcA?+&evx>Ca`gsT5)%32QDUP!@PhivP-|ygswfJcv$G zrsnQbh>y^5o^5`%z;;YnU3o$J?18VLZPM@9uajLgj~L)xEljsXTcicp$iVKxQ0e`> zku9z|t8VvY|6`HL;^e1Y^(|dBbU4s{n+P9%Cs=qrflvfFs+(rlo~nk!G;_%_GN)@F%LWyIRd)P!+KguS3VxP`cA7&ZeEKrDAj&TM=fbx zAVlI%ubhY8^u=9wJ|3qrCDJB5&?jE=ver`O- zSAVuQD$g$crh1fKnE0=n?~1o5ur%{=pbzupT~7~TN9P-}xv|#fttRKS(-|$X&5tmr zFzuXdBa`>LszKN@&mnmx14)f=L|7b=Z@9TjCGCXU<9MC3W-&^on8ISoQ6{O@=0~@k z%ISihs&=3&o6FsyRKgm3mJh8QR`;6k^i-)7bckbb3Tv5J9&|{xZCDYbrM{NkcGw*G zU0~cUt$gXuU)(>ewSnmOc zqDnA$xshqx=M}Sb^A#ql5R=mgx~Hp$(d1SGDskKRqZ9evw8d=qbtU|nEJPs^x4#r@ zw~Q##JXjI3DYVTOSe>7_ce{x3DCYwcOEuauG3B@Y?N?`|L3E+CVaKZ12_(d~DpQhK zl2?z@*ptTT&E=pmME&nHWclAx!)V;x)`v=WQ0~|msJMXrYgxc!rPpc3?ek&2CR|fu zb`_Z(F~Il{jY}`t!c%|-CtVAi7Ha+Xv)-BI7GbPe|x3dl3x}I)-t*UxB zPrM|2t3T;Y*GSz=T4mB+#a)LG((%?i-C4S`G(C`|+n#nVnMyWfRrV`->F+omi7Ml7 z&OWCQk>Bs6(~da|a&0!sK`Tg>noB{S;ByqAuoaSx|8z|n?_=Z^9d&^5m6279Y{7rWrGTru0(p{mn&8aE_u692umMOEc zb#_m*T$g*VpiENoP9N#F^Il%EtB$|woP01RYDu~a$~B*+|^P# z8FM&O=kp19D^1`H^;hfm$L11kmRj%S+3q;o^)F#uo8)X3qOWIQ@eF@lnw_hA8fLsu z9i1krvkj-vFn8NHif#-0L_y=U+c%jSO9>&Wuhc+TTSVD4-#>C7^6mCwBszP}JQHI* zrRAly8K%E2;LntoYrPcwF!opuY}8A7u`1chneMVyWh)L5Ox3FbJ(?1(Gko=LRR}E~ zjp4UET68FMTJbvMmHk&YVG3~_m;j`du<4;GM5y7+`+KFoYLkzTXNKw9V9Ia#0P(Qw(l}1-ZBff3!3i;L{}Nr zaqCG_NfLar{w3thh z#9|7-S-xFXPD<|;$ZBea8N5mC%rg-cde z^J_!=N5b|+gRaEvxp*54sBGt0r*08$d(86*Sy(X_=xf$j4^7(HXyNSc^n>?E$uFJq zRkwE?t&g~Ugtu>B>kA;qy>+ZKSo#bb5x1s4b~=0cKj3ql!oQ-fCqc(qB{1 z^zEHDR8nrLPB&IU`Z};t|M-i=LLfc(K|&AlKD*dkxA&pgj3(B6J67+;qh5gRKxNsZI+(bSE zQfCvutf>EMq>Ks{Jo zEhs3+!NViORbEn2QB$%rZ|kV-;Zc6pbx3CA5mr`HQc_aEEyT5deKUU|k{9iG4{JHrUJbVkXmdT2+1Xrq5p&0NJjfany)cN!H^mnVz(SFmfU+><% z`#M@~ef!p}n(e?Zyr3Nissho8sc+WA$f8k#uq<#-KKxVJO6lh1EI) zq7OI(bK5P@f!aw=3~_O<4!D?@n4drR@vHgE0%~vsusf&~!4U&Wte@FK^yX$&9FZPR+(Jvp=JO7g)7gMfR3Ujf`(wi9ucEdVmB2L;%dPhQCN{icBIk<>R~k;5C$PM*wE?iK#v<_`(8tvj=#vt?||VTDPO1c8N*F z@>%pRvDBWu-Jb0DUR=Yvu<#eb9yOH6jF<*p9PlnJSoa`8N9!QJW_WTq#-;Q|U5MZH z6|-tKwP3=n%9+WWmy_X;6gt9@!cH9Msg?zM+_SSR>st9Gy%4cGH`AQ*U34e6p1l7? za}@M9id4-Sn2vr=A?I8{mPfh0{Yp-a$NJE2uGA2~apva%+NiwSUzV}2Ua(J$NBNt@ zT}J|Y!b)gNgD_Nn(3OQy8rw;~^cjWMtnJ_=@eKX6o~jpuDQA_Oym0HgsF=Ewc3y-B zQavJn4zdBu=Nw^**q^K;mz;sE7+q0enk*<1u)u|C-M8iy3@2X^7q+LBMQ}@UOT7-OCD3vPp%oX*RmFaY?X_h0%C&^jaaOg0fQT4Vqh37p8h8e?mt~ z&bb3JJsvGLS&+y@4>a>XqFW*C!h-J_WJB#-ElRDe0f5sC*j+*%B0YqC0r(_vC)CZmFOk2sMuAc z3*|rxI6NT%*bz{dcC7SL%!rmfi?&S(2R?1OQw$vFA=~0Ud^b+&{@h5Ln=U2oKfC4Dbo zX^CMR&yiNc%`fukcn%T4UO7oyRxo?j(6N@<8qsO9%l|XC#w2Ok)x_wYzx`L<&?h~_ z#mj>dL*YNC-ZO6$zA$?G(=P;W_e99S>*TpnRSG~TQ?Q1QGMaq0j-qfHLuS$1uOFG7 zEGZn#YEjaMkV;@3m`bK!x5>v=vVAwBu0sKRHV~6!a-i{DUxxqpiI6B}kEmL>jRAUIR+*qlZ$!?Uu%|;jT|~owF@JQ#BtHy9?pf zX~7ET7t+%lKZL1LL3C)L1VN8p|NeUHtS2Idr=S=R+T{0 z{#->_X1eGQP~#wVt;O>qs}HX%6HzK9+P^f~<#+df4#`p``P4vh`yN;z+F$mp zcgN%SW0PGqgh-_3#L>-F{GL&H=dOILSnZfo^pN<)KYaF-8un_#zy7HzgF}8*dHhIq z6`I1yk^oT%=Cf55<^?a8ISsSG?96rvljKr7XpzZbxbpgFb&!88!b9r2Ii^Fk(97j2 zwc=R27k5fqkehhN%0L5Gi9=eMxs;zyFJbj7&-IHD=azY_1_Vd^0%?J_bVINsOs3m= z@tUt;m}j36MlBn*B`$Sl2=$lmrY-GC(1cIWLEapWhpe9mnAGc>-100s!w`o|4{g8; zgr3ogXY=awX8r(CB3?PUW?GX&Htb~Y!cnjF<2%9k45j)dddb^xpagTQ9u5?4?DKkq z;&k~L3lEP|*;zbAk$TuMcQ5$SaGhN)ZCtR+)ZS{S3Wci8L-d)~p4+gbI^(5&?<7xt zr`x8SBJA%CIxFDo`z9Ywk zPZMiZ8;6JH344UzM_%Z{^5LXpSn6tPQkF};O3Tmrd|m#D&CR-TCNY|C7SNh8)_QBC z5p;T%K&&7@B&!rHTMAy9oE?32s1NiqEGN@qmDe+K?A;QCW|_8n1Yu>FTr8BK;^%zc zyfx6g9C3(E`?ilqZAO^Kq3nnIN_6bM2wVJ)3bgrc9uwAe_c^3$oyfiMt_tmWEDA69 zwk>#F1nBXc7kwU%wB+Q7IP1W6NnOEFgoMF|`=UPgW^|m9`WxO3xIm27Ft&rH$xpm4 z9R@3K6q{pFFZt424Re9mr#jgdSS^Y>FoT%T8to+&ZIATE)0O$j0;Re;sLJhpE&EpH z!O)_I-?qh2o|m40v>`Z@G)(sf4JkL1;JeAQIy5;FF(^L7Em$h~c12sFH3YWulYP4e z66{(0oR8f-xTEXc z?d5PoaL;4{Pjki%Yjs6+pE(qp&-Y2*{56XhUglTz$f+Va1su3qezk#hrFb*dsjy3c z?{r;pvMQdO6}M}p{lux_^BWgLc2jyz-17R#=d57m7#X5>fNXc(^n5gtyUX?U3ZR@q+Trvd_`bNz=VbQ~y zqZR{NO@arpA$ZkrAiYVoshcdOwf>0*U z?ycR$9C(~>!~`78O^(b8@sTonuKw4Cm7SaFzOb2^aFHfKD5j1e2n>>!q+OaY=kN3U zs=9UoEUu?~$1zw%ot7TYp$T&aGtE*r;5~`+kkw%VDqOYxeeM>6ggJ z?AHyhHvtR94pl2vsnbzib|9N?F*s_+Y8cE#|L81(OkDw>Lg^eaRC|P=7cx!Oz^ho> z-bPp+wr}v>B}zkT!VI}z#4l>#K;>46+TB8RG%UQ7osqlfL!6h-z7sAQ5;nt2)%K%! zSx4|{^w=bg8d>EpInVJ)ePGj7M~|mBCofNo#MBNIMDQXTUl4Jrc=321M8Ei{&InRP zauuaO`agq2osDp!?;_QguNyuC66!w*rR22yovx3pW7rfMZNR1lz$yyzu3-PTfJ`4+ z1*zic^WY$V=%vPg2r~DI=@5w_AMgK7;`jsanr)L~QX})?fL0-4waxFw5*&>GqL?eO~EJN zbMS5b*?xV(-k})*ayeoTK>)P8u&P-Oa7zsRgJ&@LNa1fdjXQmP2TlIOHdjd^FOZ*q zMbTe!o);`QLv-bNdna7GMomo((8q77;=PKBfY!&XI}a9*O-_zW^5kc?+3!r$wW}O0 zx57N$k~-@WI&+`(`#0z-0D<`d7F=#Ue#jA1?Ka;DP|~=?MF0~52r#~7lWT4EbJCB6 zE;t;Xo(}NCj7N{^gHpabjWli)GX2LK?0KtL&++cL3l9wR17Lx4X9u$YTm`uF`a~_C3OzW@gY7rHkoi}+aH7DM_SI5e?XQi4 zh;$)<uny$#=(OCI5mLWXvqXzAL|@$Z%kMGo7!e)!@jrBfk*k-H$gct zoeJ&?4WZm6JgL+QzB}^;Vpm)YKyMJgo{{kxpm?=NU35Tll;Oq#3>b_w@1sXtd`C+y zY1J9@x7lRi4TNH=><%K`l{Wd!*0Egk8Ku5z86pwUIWziU)imTc=T}ZqHC!% z+O&Vs7#!!j$=?*{1wZyfI4eNi{cnU4pZxh&fSg4Fn&Lc~osgc7O|r-6FI_kn!C%R> zt0@01g8w}tD0lz`EXda1xEEB;XCG;?oCCrN=+AW-IKZ(Xo5F_Y_cz|>b%P)J7z6Hi za{gG4!^fR;oMtdKEEBuUvDCx@C}mQe7=pBniFVeppfxpaCDmKVSpG>EgMDd z&w{G-QvVBQafhMNPaFbvEnpVykWL*Sw;Y5iG(XD6#(((Wakz6|0TvQ+ex?X0hhk#O z=YQjhh4p5ylgUX-pKi8Dr9p@R&GBzYi!0uP_g?{3e_jAkBORSAEX@38pCDAQ4F~N< z^Y{O<8ViIeNJu?iot*-r?0t;N1T#+KC11g`P_KwwYCsRhq14aWI<1enfD=rQ1hjQ< zronzT?_xPc58Hx5z}_AApbSae@udcJzK-%=cic-mpo<4v1g1wt{g*Xcn0Wnimp65H zH@8)Q6()jPMSi4yc!OLimqj*gbjwrTYIzD7UHr84x07B6%P37xo=h(+0E0n{lHko> zWP}Q#0c!EvOaa=%(7!!W7IGe5@V9Rv=Y0D^`7#+{+Rr)Xq}`yO1#?Rnj!vm+(O}U$xNO@^?7tcLvBTRH74@) zZf7mJ+L>{$O-m(U=H>fN98aks3TJqlYIV=s=v+;lvJ6D9YkqBrNVT-KKY89XBb#1F zin==9Tril#)E_gWj|I+Q73S=eYP%6*l_oFb`Gy>6k`S|Bob54qx{^4T3Sth49hnBT zxXF>p5=RDW>v<-va1!}e|m2(JJHF*er4oUk%^Y2^>Jb4 z7Xq@hQvfFHV&?P9j{DQy0MNFU!dY5vE#)q}XY}xUXiyZ%Cud4krpZAo-AB+L`wkwl z!^Iy{YOA%rhVd!)CzVpe>ehwEHA~&#Pdp5c;^?dB_juQ=4caFk!H9VoEf0dY@(Ymi z1g{&!FkTiwC6&Tx&wp68swb!194Zob9RoiHM zY78-8$}e(xvY~DA;$(y3wnN@LBgdwFY3+G{q+PRt&a*JOOzWkZhunC6PgLsy2e@uW z@c2e#Z23?;E6rYxra~DzaxILl6zR|O0{fa2<_oN@n!p&47eoKNmot|lV57Fd`Ie!xu?M? znke%b2>_8lvwr+mf2HkRd20M&N@Vl#$DP3T{b8KOQ|o*&H@y9IKt93=WLlW+lIJ}E zXZ7B9C5Q2TV40*VJvDg>QRSP=D=GPmVIV{{s*8}|-=N}o*{}Sgd0~;$uyjD}FZd*{Y;sXmn#RM^rVz#Ps|%c+*RR+0N3o@HPWm(v_WWPl~82uW^fwkwRC$? z5o!M#LKyTNv;{=iPfwM@@8Lu=mAT{lGc9u55ruI0ArG$7i$s5!1EI92O1iaAbC163 zlssm<81HfC1W&bly(6sfQ+K`X&#!F{;AKWuT~<%HIfup(<=^RP0-%8llS&%leTRy~ z#s61r=Ngn$8U|puyf9rcNh)udyUEr@cB5h|jm&GIlrcprR7OT-WmaAaGIeZS!=}bs zvm_VO$Wlv7%LG@qCT*-VQ_KsM8fhsZ8mK7y9=9L+<|L&h+jUd(0BixRQ~eX0(vCs*fBOF6f1eIJdmU?>{jV zyf?Mu50=Em!P>(yuD{T1=T6O7@~+>yA>L7%TcoBTFAnxqy>_szwV+$fRyuqdN{gCsLBa_gFg=%u!YeeWuAlZEW(jtzTl9O7aHhVS{m$-=6T z(7)v=C5US^xqH8eDLtErFPGNcF}r3*%x2Bfzf~RZ>(AC@gf^t55Hi$hISk(mC7&aw zEg)!F4m46ZUzD`cLv&SoyNw-Nv#8QvAP@Ply*PN$TfeSZWzOp? z`9c9pw$W1qkR_J9ug@WRe20v zjhSQm43T>Gioo>+rg`LQ7O zu6B=VI~kM}NR`bKp#Pra8@v^Y5n`_ke-3TJQ;@g2}gs{A}`$jq7n(gBjn;gl>s< zAjRAS{p#Z7!3g`ra@X$7l@8H7GjOw-MDX7ayMDn#Czgy$z%)-!fjVY`# z-~+2GO)J)5{RaIQ07j#}GT9P68#RhKV~K!h3?gNO=)E-4DmNtsbDk-(vMeDUHpEC{ z`%Xj)oru2s#e$S14DG_b9Vv}&ni`OdF<#kw;uB>UltJ-M)^^AS43}YthtbV41`kIL zq~;kS-`g%Mtr*?U;+?*^S!Ebjy?3vpVHIlC#-Psv{hVcMOOAUhuAXVX!Dr(cPynl& zk0hYq%D4~~Hjqu*O+bhyAmND`&61Zd0TK4J-q-9S9~-K_6~pFt8U~c21I?em#Ssij zkr{uW16<&LmaU~u&a%Q{-DSn4*ps=1X zL)SsX!lo{qSYt4SC$&mfFJ3%NC|px4FlZfpK;DIdeU)-lx|JpVC+XHGrk{vm-8NQ3 zt4X=|(uu)v&}E_QUj@VVw3O_Mr`b=MxMPM;7=_*lfs62w6A!-nFAUJjJuFJa@D_8AqVSL zd3tQttqW(J38LMmAk1AUl96(|WxwMxF3&+c(NR?tdusAruKK46e$l^~4*7#}dgt;( z;!;$F4Y`-=17-)D7u?t8g%Zl2-hEz#@r(hRfjCSe>po#IeOw&?f_AfS$F*y7xoRw` z^2uckk@Zs<$rClzf_ULF7J$LqX^HWAI|*6O7^2#gq;uh^7*PP30?dt0zJ~Vyc?!x5 YX6gks5mbSbhY@@{*VA2dX`EAk0he$G)&Kwi diff --git a/profiler/advisor/img/schedule_2.png b/profiler/advisor/img/schedule_2.png index 6363153f9709b5dd561934d7fddcf330cd7d7445..6669ba38b6b071c2ac513c4b6f670aef487d498b 100644 GIT binary patch literal 65231 zcmd3O2T&8}wlKS{%kH9}D@CQmwxXgSB1Br^iipZ8c2G(X6jTBMB7_!Vxqx&PkrE&Z ztB8~kkrGOxLMTE&R7fCA2qe@52qC2ZafQ40&j0?IH*fCz^D^U@eD$33o$vII#8W4o zR{yf`7d17t)yIw=KBK0#60N58ea%lls#5y7Qx52pT#}EF$pZHx( z4VSJVyS!5M`{&@J7vXAZn$2JTeb)wiaYao{apli<}M zC-wV;SqG36)!VuKC*P^9;l8_rpuzt*Ikt)IyAihT=-Mw^E5B!@e&0EE`-i7r>K@#@ zg~^cDj$9xsqG*p@ zG?uuqaj9SJt`E+7#*up};w$~Weh!xIvRa;f>JV^_DYP`(uF!3fu_mO*;sql;$6uvb@^FOPcsCv^()bhdR8i%wnMv+@*ldBSc# zT{HwWZTa^E-6ZnT+*_m_|HMzXh`MU2*)7G#V(F{5`F~(AO`pUG&}Lkwe!DWKK0Xg6zZU2c*+)-RN{jF6wMtiiT;Kw1*=YcLd1cRch@rRGdOiygO6QIs zc~Xcy722bR)?#j>u|_nL6z?%EdT0kb-|p_Zvm(QAr3A0lpmEN9iFnlGG*=a5zlgAH zt-~+IsV8`!j=U8+Pz=(Ny9RTbd4pPs$U+~%S`iikemJ1aZV)GF6#5z-nAt2s(Zjp0 z>F0%jC0Qm9eFAs3R<&Ju5M!J~)~h3SrQzWdY07cX{@3c?cDhww9Z2FN6QZu{xLII7 z7beOg(sUxtWMi(3eE-91=WBXeq(-vGwoKH8n=DG(@bz=hf)7)O2M3r5xd)Qtru#RD zE8u>emPDo)871ctp2R#|{6n{}2ZL|+ zw2ukDEp4+MDczlh=g0DjrU`m7o(@m4b z02VmTx@b9uRI?snH615`mr8m+w7#Jka>iljN&qu(oN7?KP0;8s^ z5JIO0SJXCZ+eox@>H}bj&d~fo6Dx?JPHK}cuYjf?c}9;y>O_+WQ!NmeU=}Nkk-1M~ z)dA#>r4LXr_@|ye*GANC?@}K@IzjRD8d2j)h`TPj2=W6UMENIVd@a&wL7F@!+~ltX zStma;D6N8ZP(q<_ocIp5K01DyjH~!*y1XYHkCJN$t(QT#{S|~6yzP7b8UkMvA@~0b zsE}%ie_34=oV!j3%^nVt;CEQ{M9Nz~z&z%gxTO-q52^XG6}NkF&q@5KCC0y1jzgk0 z&AeI3`=ktIjY#jy&F&^3%=D68JD%>PF2>DS>7u8H%b^bcyc*2fR1$|72MANPQDAV9 z!6$6rPu+EVWsidLw$}`;BDGhEpetN<46}FR({Zk zrN~FQLF=$Nbt*cgCn2@oyPnd{v!1GuGh#8B$z*jpM&5}WuFA9ENipk)QHl?)4Et9{ zJ+<}F{!PC4msC4>+xoizd9vc!fn|0{4U9=`O`umN!1>oTyX$PR#~{*~rvaKS+Ab=@ zy>>3RpMq8-3KnZCuA!!POJyIFjawo6LE%(}g)G!v2W^~&kj*^6vEKW@rETkrToHd3 zf?IxCTHPIrW(~yLZkUI&LJ>^FK2syiEZ>vR;Tb)1$QRSS^6OQkY;F4-Ys^UJE!In? z68ai4^>olprYDt7t43$LjckV%LO9^$FXXo?QM2boazCg8%Rdbt9VC4z?bH|vICK+- z4|$uw=wsOorzJz2wAvmLV=7Fo5TvtV?4p=`=w2u zWNM?EW_G2zLLx~`GXhNQN`Iw>zw&shfZL-|;$Y_7%4^H;bosy+!{9sqTFCXrBH{q} zzp~A!BtIA3Jh_Rb%r;LBSRFG?(xVM8n;2@p*HEX#cdAg&;>~QN;EiJ%J z=7}3rGp5&CyI|S4`S9|eHi_D!e_91E--S5>ieypfZA!Mu z(gEFCJv3Rbq@ZGH@w#V0|JSLwl&`vnLW-Bg-DQ@6 zKDQ`#`O3Q||CS~7pS0}%MGmEySxLda9{sz2-#@wRKVl|BwRF(L5zx$o6}&g!jcYLg zhBbpXAfZ_@Hi^d|3Im!6Fljj}m@gg7s_)T)Of!`otPr9i?E@c+hY-ikh~73Uzc5k9 zsNrv#xB%tv!iGYam(6_Smp9LNKBZ(s*ZSm&#n%m85@-W`QIAK7ud^YB_w|L-Hf>BM zb0zrz{Ac8sRuF4IZh=1v{nWTbZ62}%a%Cyk_%}LM8S9|YPN?kG-e)$#a_6mNu z9)lVaD>~1sMLDDTBlwUy=bF+N7k5`G?n>FEv29IcRCq^m_-ks+tPX5lh;Y!3I_<7Q z;`GvT4Pu{Gf|q5xx94<@j{p-pge=^#vr+j(dbUVDwo62V0cb?7##3r!sx(_kWHoll zCqA(FcowmV#pea&DjNhOPN!m4)Y=pNO6788@_>#7M__QGcr`}D7XZ6}8)&FBnL|s@ zE^CYaKBJvhsJC*3v~74e8Vt{Ac%oEl8APz$Hq{s?0K6s#8Y_hF8VHv3fv3O$F*9e^ z#Z6Avx|2BRNcJ#pOu=+!z)+#GOaV4_?tR5xSk4Yz)YOa~9f0B>3!kv&5v22dQrbV= zCfh)mHW~&RV_Iu(lSk*WhvBnL?|LVJu{}-7v_(N^NbR}LjqjVxrYIt5uReI1DSE&P zhQUyqWmckvWW^iL`D(L2<1UrH$&Ea5lq;h1QA1j-C~hoy4+}3_7<^ZbNR@sumdZDR zFsEnn^FdQDeQ;5%n%xsHC!BbOh1ZHzXfYbhqPPd*5gt?mP$Fk_={+}-a*7AV`YgsY zcLqU^1}lL)7rxb?i!9;xxH5R=H$;bJR=q~Tfd)$f8dA4U3E04^d*7r?0tfT8`2b~~ zazUTRK*i#vU#8Iu`(DCSc9D3csji_@RMw!#dj$K%3GzS3C@gtHnUZ}2 z(Hq@d^R5ABl1$Qrm@nv5(*3@}Nh;D8YX#dh078)mg(Td*B zfgvRERJ8RZEJDe8thyiICsQmT`5@&n2$>AV)th2&N zJExfiC!*kPd`g`&n!PG+t`;(-pf2iIeY<^?pE-Ky(bCO@h0|GdfIO1w6@i^eYdq52 z@Il@R_F^a|2HV1C&(8w(PbE>B4&AQScA3AS;VH|E%#{@-o{=RbkLmTE=2`Gt3%;^x z8W9et+oI?TzL~OTk5wDiDv_O$B+iQ-4F)X9wLE6N z^)Ao88Jk5y9eyym*pyWdPV)*VWL`H?++C8-m2p-CDzxd))|dqx*3&NbaWr;DQwNi+`$bA;$gC@8r@RmT`@BYoJb8?$Li+S#*ZLy8)MxK#s2=3O?zfM1(QL@u zTK+-8h>u80@fbC2+Nz9797{hO#gD*5^Zh}ipa`}e-ARM^t5-EuR;*mKWiOaiq%8De zloS3E@bmRe6D?9p#9Iu6!&Fw6GB8G*zNL^1HCEpXN_gMo(pY3ACUF?9^(<`+nyI0Vqd|VWi@3i73y?@m%QRl6=&}+ZdreKyg36|ekjH?j_O<9vTwaR#Dp#0>Jif~ey z#}3Kan6+X7if!5_o{ghVHQYzO9aJ>mDODjMa&=k1ARvzP5ZPi^C5?|4$=T>gu88*@`g~X3-aQq*v+-^x`o+WzBcg>9kz>drHO>RAqLH`gkbm7wo|F z9fa(*^p>!G-8W|Pe!AM!s*O)A_ic2I!0Mojz7GLvnjFo5Sm6z>$#PzsP}q020P6w58-B!CPCeWGmSL3iW~ZL^pcHd&|$c6 zq4V6-EWqm#e8Z?*uFiPY8!L$UDu{oB*p9`rw_J?yy1Zl+-}+hK9h#=R=00xUGNJcD zqF0LcfD*>$D%M1bEN8KaqX1#Flos!Lq)VC!m9@<$Nnb(^I`Lb*ihZhL^6#CGNbf(i zBUg&poDjGGaP3n3D%Rh(P-__?^8~x76m-mxXxKHfz4|vcR3dbZ#WFrpuTnjn`GtufukS=*0IflMG&={#EqP zr{8{x(1RatqFv-%&rc$g7qmR~DH7B#wg`zqk5_(y$OTIC#v8-FUo~PBQT|4ChvbS> z=l2%l0x%ra&D0l(5xaahKf>fnS9SmS_zW7}q@6}}}Bp4tYSxi`qLdLnm zmyYSIRh7z$!_%^3PTVn@N^&(kthyeRK5(A>`+DVK7ArNG)0Fl*fh&GPD^6}aT`Kj@liIMDglk8?CJvtw@n&F6HP44% zD?YHM$&u!>PODJW)+gsm{g;TM0lk8=Hie#Sa<7HY>bwdz{zjSBtE@VDlSs~=Ld8h~ zcogtr0#6Ps1CAf=U<&8Uzr+>ZqL0iL zsuY*Bt@N{j2Et=JR+)kN!rZBtVPX}vDuYTP=wztFbg`O=i@u&Vip}q8iJzJTTeIr5 z=+xI)*3SKpsiH#jlc+`wtjd1cCLPR11*4{vGeeG-k@IcP{&wX;HIc2g;+w7PO@s0P zKON3du1Iot>n8iSeu=y_-;p>?N1KW$vb!NmO_gywtoWPpV55j3OT4SyQtO-kC)CEQ z!+ao5HlfayUp~0h?zU33zz9040fKUZT#Idsn+7~=m=7vijv7P%6Oq{eK#UjXX<~(% zBVcpBTQ;t(I#J4WWwa=X?5bRZke@6eq=p`SAFX`0#|ndz&E}g~u>>v0EOsn)zVXA0 z$1Wo{^W&->;;dwn_e>9RQ^{u=)0N^ktEXMc%GH+FpB!HYmK87Pk%8b~s+^m+jyGiC zPZtZg2m)31HO!B(7v)6{CmPf7RsM_;b-+#IGytM4%_z(X5U~tQB`3z7LJ316DU?WL z6QV7N{8y5JqYtZd^@<&kQv^K?#RW01kk?;hqolWDR?^j%-Fi0|p5VfDy~Mi zB6A~QY)X0UiB?OryqdC1zW~$EHBW|Q!4*@O44`o~5rR>ZsUxAO%27r(s5 zgv43b1Li0cBPVznA|x3p0~3~EpspcQNKtZbEh8PZFvcSsVUP6~nXg#(9HEZ^_JSFn z{Pr87vg|w`pObl$T0@um)6KrL_>kP=@^rhs|O?N6g0Q(Mz$7uc}CX z%Ck32HZ7s^#xIID;Lp$iE5%=3OKh&Z!M4nM8CUdpF<5-eJ($_+BS6C%FaNk4N}>Ld z4qnKPp6D<}RD4ak3_*SkT={n=x4R(B#cuY8Wv@r=&4dvM-kJ|a#i~4JRi-I-!{1JC zzBO_%A0-R<+so|Bj~(0m_s^>_s_f5$gMT+<{)b~?(NCzdFwwINQG>MGe#+?jkD;-_ z-S(lpKO3Zwm;LO?UfT9}(GuwP6~`gDhI$sF(Z(?j!{`jGs*T|ZtjvL6bH zc|%35P$hFBUGwcY-K%Ct?H50k^SoavDU2*5yI#Js()JIW^>X6Pn?Lx)M-ax7Sm?xY zzR;WWFH#Gsjm>F_5BckT(4pN7#GL-DqK&5)cO!|sap@3lH3h(v$s)xY22fjQ1W(VC zKW*Sqf$@-Bg)luRjAd<-+{>kFxx4^bsgdycI-^qDCtiXvEXJe>Hq0c}wc#KJo<~*{sFyQ9d!WX_nG~*^8f@~F*8g2AMr{P13Iq)_*A`N8{ zxyRvBo`y6T*~8>lpbi?Ey-!D_9?eLGvq4&*m@H`_6vayANfJ)BgxC+hwvC1jBAj)l z$kRd_QLQmA%xN^>gpvI;&o-=~;W!^Hy$aWc|9nm@nQ(6?m!YH7dp$3Aj#q^EM5w_UCR%6B9D78v(ATA>eMEqS#JP)TI{bSTk>I+}5vYU6Tit$0YL70VF;v1If83 zJ2G%?JNVj2Z9jQKB+LMmoP4!dMa#Bu z+^2CsLW8LP1td9ED>$Z6osQ^kKUDmL-cR{*-T7cu8m55A)Kn}o7b!FG>Mj)Q>A%F$t+491e5m3`&4E$Wp2g-* zPIMtyp{yrbiJBOUJxNd}z&Sp{epj+XFtW`x-`tFVvLfeRkNRqIc&3bp0VbcTZbkWJ! z9^`CnDOTl8wcr8uMA)E#EX333HN;_&fF_rSvFS{m9)2@;jUzd<>*Ln+ z$Nd$#tUIds+u{m#HX!niCq-{ji-~Hm-K}&uHcvc!pNj*)Q3RHBwJ7t`t4(E!yC3iP z2xP*A)d;~rNP{xnih)`qt(QvGuRgVtC47ql-y&DPC2`_Wma0RVdTlE?B>@YS?>C&D zKZE^v&nx2z$T^9ao)-*MQi&!K8_@t;)Jk0QnDqL^EqQkI$kbXyp!;iCy);^G&gjsG5d8X7zEN4CTH z5uuU&IN?*8c#`M5%c)nUJCZ~Wm2M?_IC4vOAMSit(-pRkM(&;T8m16`3o`UM!lR#OsNN9F!B* zriy}a7T6}1&uIOu{h1zDi4-|n9W+P#JtpWgsG&jDr?WSmp+26 z{tEk@`%x+l&gI44gu*#pqK8)MF71afvaQLgMB6x|iF4(dkD#e``^p}5^!fyf(_x}o z5a*O!J#JBuSXlEb6S+G*f1g+H-co8_1ty&pon^aAMB#5q2LZkM9@d#Ew0=>|$4Ap3 z7ZQ*~mXln?pB~Y(GBZ^~qixP8+}Mb~LULQmdAkHzBdO}K>xsvUdkY+jdBGIWUIp$$ z%S#d`vmY%ZMe^y#5F!U~FvYM_Zf5q4ELNXX5f^m`!qrGx&@lT{o!w2hb^Q!AEq527k+`W^}xJYO*U0<-h_P;Kj5sY;R*92nNbSdSZN zN3PEgWo4{ntD+8mgbE!@_2%K|o~)kzB}KeWU>8>2juE2lgOhngjp!8LAHvytbn#4M`!f_|$9d*+qxu|kvH zB)+?5YSErjmJ;xaIYsPlT%Ai8llsa^rl&x7crULqEWz+yh|v2kE8J(MFdm$JR_klnJ5=n`aP* zkZ${k1vlyO#4-6m@5W15l^e-~)djdP`8<=xcO$=x6V|#@@I8t}m40e2o+I55C*2>e zBMfVTB0ef&hN7gCMJzb0Lve_3S!T9#iJdxH6|by!T^e`Y@Ds*9@{snLuvt^Z3Qs2& z7MO5hi^Mmg=zKB}@(6YF!;W3Zh0h(&5yBcGCuB!qplaiK_Tu4K#ZPafmiWSjf%h&NFzi|}? zrLi#y(lp6$kz&+_;C$)SirV@~7yC8JkWP=@1Cjh{6slgP^NI@7pwJ-7F&eKv1AI=| znfwv;X??y(#U>P6SG*;@CpSEkXA6%U0R4Y_y5wa?qZazaj|Khl98X8 zOx(qE>NX0Y)Swo-l)D7?Y$aohIkLWy_o|bzLOC|MEU1*IL{lUR)RYGt<)v$cpH|cm zs4+8bg;drVEcoaPFkxZrm?$5Bh>KFzC-d`ke9xj}-On{r)A1Hstlt#WJe1j@A+Iv( zw?p$z+lm@Rs1LI$G)<)7P@vo<6ILQxg9(z<;i6q4g-L;tEN3x>F6df-H z&XmN7=<@HT7fczqtx1yh1_YNW4{T#+5f7&+?n6F7EZU}XMZ}k&urr=oBXzEBFyj%S zqhj+%W)ktrH}=%JKsebY#-kk-3e zsgA^HQ!AA0HlR-WaMoRh&AmE=m`o<0K7I4h7@vRjt+{iX*UUj#=kPI!=f_@vr?%wX z+)bR=gy((kNtdm;OnvS7Cm_+S2Bz{YKAJd0_&8!bG1;q54{OHtbF;{^%J1LmQHj$a z8mCYB)HwZvB$)~=^ zS!qi4TRh-`YM;Lj`@_2jTngrt=#3Z(${}awpt?%=gawyK(qhnJ`4IF`43&hSz@x zDP;0^XGeiEkBE_>b;IBTVhk2mrLHoID9&aK? zdOz{Ta<=m7ix_4Q*++EcM;z8I>&82E=if|lVbYsSf2St; zz&nYBLr>R*c%*}-PTIx5L?#1QFkw5bqRAT%gkWu>Dmsd;vEdT=y85GD<0>B}wfH>f z5|^F>H_pY^S1U&YC%)hs=vt$IIzglzGC```kLlAv+oBkvb*bYI&hiqpo=>(kyzF`V z0o9=3heGPj6r+O0q-DgK*KGrBsaH<*x zH)(s`HeFuH1KWXObuMW;7Y|7FkwMSDlRg9WP{W9gLV#(#fp9gA5 z95c|(w{D}% z`rkq=xTOM6ajQNrGYb-~6v_m=5@QS$vT?eU3EBktPV{y;)X|T|awYW@SiZ#OYmns{ z?rTBlxL36Byl7a%td3iN?Efo)zG5~L26%+3O+e-BUxMA+UTjnGZ?oNS0~*+m5noGc zRl$!){ae@wDaK8Th+51#8V&jz=btS-{te*X|4yC9Ez5N`{yo}N^j{QyJXr!xZIVoZ zBRwe;1TvpOiB{Dl{96^sidipJ)#}pNH@O;?|BO>*`oO;=cR!t3j-^m{KW&|yY-+f) zoCH>zTK&JelIXuvA>!3rRqMgiG&u0#z{tt<7=A80p}x+PNU{$=rWwwpGzde9R+@DV zZPLKYUYZ`fAmnpeO=-Ybjyv6W&^LL#Xut2tmwN^f>Z=2~^eH(h#IUitY~_XkSSGSE zF0H{qZV)^C*gvI5R24=h$9vd%e~IZFQs}A3eb(v)wd|MeZ%rBJDkjMbi4L2Auc)uX zT2;ks>YhP0+(rH0wL@v|3Rp*QcrQ%pp486Vj7!c7jGd>S4rokv)dgAkuBUh*3tR{M zJqIY}uzajLJP9)1=q|k?I91i1VM!Utuf{!(dpk`CZ&$<}P)7#!3kRz+<71$t$i|Za zNmABEx-zKdd?>JtG=<+=&}LYq>(reej5KhB~FXa;#N(DF{k7w0}5YaB0 zPK9eP!ux8&Ag;frljBj2sSmHluec=-*qTIUJ3?6h*sm3uP&dH*c`n%lydQI5KgS#D zifFEbsmcbcRJFiOfpteF)GNaBxt@>7luP;m=y9!m{gx3j%}h$Jj>glu&8NZIKrWwG zR#i`*G92DaIl$FgZM|lC;RbGC_a!Jv-WFB1lZQe8L_`|wByS{URSJJ@KH3a!zeK~K4Rp8T=~j8P%;_8O z@71IOKWd{#vDkfju+vYH2-oyyv$7;}Yo!CoWV1@IF|guLB8_C;(za^0-HM$R^u)I$ z-Bw(fJa{7%lEY0i^yO{e!GP4H4{s#ACe4!c@N^fWt>law7Ae&pree>#I307^S_S%4+J-4@-n+3u?XX9#dL^?V3V1wyt4hCspqeeEALj+^A(2 z=iG5_?Wcteqh>8vM$b$G=P}*K@{|YCLo|!IpF9E%8DQ+xZ{|IIXP^)NaDy|$n}~Vu zyFsv~-ys~6XnDZGwKsQCthl1d&b!*(_kDD^eu}I$U@IjngxN)I{y>0_)!{T(I}}{8 zbAZfUvfu zz6W4)T=#m-(b>;+&9Oc*zXCF@b8$G#Zbx3a18X|_b3~!=ve3Rp>yWb7`p-dKywMPODU%8j*|^FF-Vw;bxv^(bS-LRD-7_tBRmas24O2@5<(yI)r4I4%2%@{y`1*H)Mt7jfae;8F%BuD*|Za zDADvokMlL9h5=V?qfI|fhIuv&=A}rT`w{=3BgW36D5B%Y6>n??QV>>JkZtDTm9SXr zRnoY?;X|f_slbOr$?t=>wX8wNmo|1{GGXz|dg4!4AhUXxqdfXz7k_=3YODNhLy9!T zT$47U_q^M4RF4@Uqi4l#6_9pwEfPx-jcqQ33O-gJOM5!388}ThaX|U%=MeV;H*?w9 z@!6?_=bV>kaj`eJCthc;kB08!a-3=cYgmH|=dFu%%@aad*Mf=`H-ZXYxNKFn)|9{_ zQa+8jwEDi*_sM&Ja_%$e48IUm#$JV_2H57-x=TRL(nrIk>pIwV>gjV^xnnCzp;DSH8 zdZ2urwJy7H4ebgfG>F?J-UYwT*`HesYQ2nZWESJAD3x|FyR{hx0gteY(O1`F^Ln;z z2e`zP&D-6dY$0^73bepu0`f{D{@5P!hje3JcRpUXFl51cr>Du)hRr|P!|r$l+)0T{ zah|uB(yQsP2GCMN;@qagV{Ju5Tu5F%U6LVwpPvv4%E-=*DD=G;;98MUiAl13T@A;c zz^-Tj_-d9%WP0%4x8RMjuFU&deHfg5F6LfSTI1y@N!10p@Xv@wpcTuZ6rqb*5_Soz3 zX1Z)Yzq)KK2q?!)G1U1~`ECtF_Y?5QoW{Wz{DwAz+RU0xLbIORReCRBT<_B&iCQ^q zF#suCJ9{;!)r(m$ZPCtbPsJOB`gEhS*9~MnX!x9st{^H{37SHh6zKqQP`1WwXfq-A z9qn=nl4sYmuL!(sATMFD`|$3OC-pf7x;6fv^Q7U5PHymZkbQo$wb-SPjDrRAcy2}H zZByJVSoaM12eRS4$9Zzd$E*;;7kdrd(Ye5_F5$e?7}M@%s()meII@HVn|Qk&krD#! zsEe;YP*Y;7Zb6wjx*d3=8xFVi?=I!;)5|jMK4D8p8Ioy7o^B2EZSkTP@GY`Iz-A9W z4%veh)!n=hIS0RXZw_YY-s5@MaJ%nDQL4f|lJDJFk+=pMk`L$~-n-qtfPiXxOcp1m z&VEKAIHCHmT2AFY-Zj>|#B!v$YSTg~&t^w= z=Babc*WL9S9zdr0Qx9>n2c}a;7BfSDv``?hSMLI^ll+MDWPTJFtIN+y4w#7dO|b-U z@2~5nEMDEsIh;m=qLcd15!1n5RGw0~w7NU zT5l_==w0gNA-mUYrz17}hkLE_GL+{S2GW`Zx@BnFv0-5A3!Md=aC#T~d;!}#4`bDR zIp!IBM<&_TjM(}~csiQ7yT^fKU>UE77t+?xoqHcxbVKt89O1{Ix`}Lula3x?+$Uit zZ{*O|vpdMsdDhs1&Z#^SH_Z;%c3|H_gBE0U3cF1@I{d3I`6nEQy-K_e*#Y-$&o@9k zpZ>%5kFZIKk;6R*_eG^<>Lm20(Sq-S{2$W_rB+)ycN1(3zBkPR#CMq5r6x323$Y!7 z_t4Vy;#6{ugDq?y08h`Jo#vI{|>7F@{0ox zd%p8-3iQBP-v^pDuS6~m-w0CcGRx^A_f|rbD}gdwd~lA z=Qki;WA8bq7&lqJC%OJUufK4@mroTxM7B)(`lhgBQt(E@R@@U^i5?n$w9ji0Rs6&S zC4bnU22lTu4{3#sd|vNwW&D6H`%HEZ0+r2Dn6fLWbLEi2&jC0l&hWB06ngA1*O(}f~y_Rq0K<6})t<^I-tl*sBE0};$f@3#cowUu?aT+xW`!>dEIEaO%aF{@nK8)XEZ9i&xwp zcZ_bld>=A0S>OnYc^k)WPp)r#;?6J2z_>$3a(u1g&Cp}c#|)ysYG~R$ zXZp+tK{j<@BRPxnR1rZq5!$>>Vl@0KZ6auRAewB)P-YEI|= zg8aucd2mXBMK<4Wy^wA4i7x!KMzFojy5iV_j3eun>(h0LsE{tF{XD;hSbuoSTRzc z$hp$UtBE&HrHD5(huogQU*5;Q@o<R@3ie)g9SNWk;55nPZYNIqlu4mf-`eyl8c@C_jyq zwzb99OOK5wcSXK>tSS_?ekbS8p2W<-WWK@_!Z(mEhu%}z|GdT=h_!xX6ssWb=mp^^{D>XKK8GZjlQGTjx_jroc3z* zm=x4L_hh*FkBppZJUJEl5~^~q@9JRkFm#148kBa~ki=HM6^7eP;wXj#P6OBXl5Fsc zX^a$a`Jdidn# zc}x3Z>v4}-heHj446sd_4vOdYv`1x@ zbGW;RK_hNONP8TS?)5&v3UVlRHE=`>N-^8|I{!h!wU{6C(~s^nvSvaX4iH*eBG`>> z+^V$bO!!{i1&i*}J?(hpAM%{Hd4dxBxsVehrY!{+C5ZDblG|ah4Oz`@A1Tf33@hrn zomuSSQ6bpHF|XpaR{Oj*9DF8tWYKDgi*JHzItRRru}`NEZrGM#=;6-dh|^yBvyEeo z<)LXU?_$~9x%FH6FaKduQFTpmA(7T*`JiDUnBt$6E-xSOU7c|LAjUm@yv2C6Ru#D?46Pj@fyEE8 z#Or|@Alrs~f>?G*W-o@QIlh;qKXjZpi=>Bm#Gz9kr!eZTYh$>C^%(|}VpRrR@SB^w zlTg%y#IULHdG}QAeMS4*Ux2g%gLEib`4GqnIN%jjF{1eRS(1}o`}mSNNXsX%lgh>m zvBbZDyhCAG?QMyNI?m>xW*)_(7@$%Y`w^VM@6{Th zzN~>fOOJ?vpmS>8dcpaZt~7qf{V}>JF=2dz*5!i{RLt)9%Ppd>;r0F&BNHoEg{!|B zb*&=z&tD4X^b9?UFWgs1RsL+vbz}R!^EUib zX>qLLtjTeOf0lM(|M@wvd<6W98tot7hGOj9pqwN6ZzM-q^8P1_R*-Yh=N}ro3w%xN>xpvw|uGz@z1|$5N_~yDV|Tjc_?9=##(+?Shk_GptRf*Cw_Xicqz)P7I5c3 z`-czyr%8_g>+dhTc`oHZ@DOn(B!oCu;J^!-{94xSt^QY)@$M}k?I&I zoE*YwVG}pQcf)q4h!hj(UM^?y;2`^|*Se2w_N`CgYtoCJrEiGUVTg1{*)9GlPal1K zOFL!qSFpbF?aj4Tf@GGeD&k{+wcTmxNewJ+)(!6y;=5zaZ%|d@V3gFZTgr&0N2mBM z9*Wk31U3FO@x#-vVBR7>-udzfaaX}uKWH;?Zhj^4y4Gc@yrPrQj}E*k;*@1kKU5(5pyxiRpwVQ$C&f+;I{0V}LKhvxEC2hA~bwE+**Tp~pa}(4@MOIJ`rj2Kra-6K(eiDhgRq`+jkB;Yx*k;j%jb#@mkMWaVU< z1dzLO5J+e#@$GwG%>ajr4Mq7eN6opF{9;MWwMFU`Y51z@$vxP*?%kUasdFomlE~b{ zT-%Xj02WM-RunhW>&Okc1j4#_CFX z417&a^cY^qk}BLz*t*ghjUHdT+BmfFlv~13%qiA)P8BDyS4$xlKB}V$rBK#{cMs0z=aBWH=x2HHA;y7{=-X#ZFKmDxDDc2u zZzw0i#(Z?ArRLkF!`&i1=l2QRs`){kErhQA*q*6^rhRGoHPT3$@z?X9du>Au-@)(E zOJ-ET^{$z2qdmB zu=3%&fNAqNe2L{`cIvhK9O*N`wcJ@G#jtAL^JCBN88B9H^~q15l*n}#I-Pg6_I@-7 zpB!A@SvK2qFVd&tdG}eeUC=`A<76y2;W@}v?1}8kVQ;|yEFP;M9r5alxljN!x#2qj z@8x*64thC;72xhjqKh|%2D)MR1f5p>Zk)oFDj~RFi)#9?8%oYstiBBRKg7LxT$0)Q zKitfGre>zaDa(BsTeQsu4HXxhGPN?bGMC&aF->s^aYF@XYRYmWHFJm5)Z8FaR9vu0 zh0KLqaY30xK}9r0L`C$6vwePlKCkcV`99D6Pk)5xzRq>7`57!}!(B)kyBXwT4J3B@_cg*B)<#3Sfx5Och~-!MD&UQk!|`t<~( zv1XsM$`H-FZ! z0z*!0S1LLfuV8DW8@-P@nga*dj6%B_ze*X3anuOS(K18PB}!VR%qO_X9vA zaiB^BP?t2#ga^;}uR@UEv!o4TTX==~Lwbm9zRfCAD3%dV3qUo1$$c<0hE3%B^}$$q zXf?Ojgi)R{PUw7P=W7>9n?;nZ23f(Krd<8S>Kyu7a6mER)t!{Vykf>vaNPOOZtXTR zkk?pNH6?^?5<<9MtnNOjq~8OpO6*kP!&z7cGjO#I@{Hhi)jHYg3&e z$}p&47jgu4u9I$y^!yD@eII*3Jf~0<{)*sIZim;6;h!vD)Z0@$Tt#P-l^I?)Gqd31 zs?=8&%A7#vi?*D6KVns?hu_8~YV+~2)@VJ1s}k*g<(ApK?QL|+oKT@oo#FO!`RyFK zx!w-prK%`m1ntSr(}Z3feZor%c8}TbDLvv;yonfb{N9rADj|3l$2`dr-`InzD|{YR4!b*LAsd5_8TG1& za5hj|dgo+p$U(AoVNeDwnR~2GbA#WGD1ga&()U)@L-k+WmjI}WL7naDD-s_Ij2&$_ zJH6Ed-5@Qdd>Bo_nxnmH1R|Rfm+{UB{N!i%xSqu7Z{tI;MYW%khc}SI!ncc*=Vv z-|n4fl+H;1WFH_d%VQ@fm?A>6Ivq?z0$NBV?!ijKv36*V&-BNS;egWcB^uHrylZ-? zn4HQNBd$){FFIluoq+I*+*&{iy6Pf%Q4mZ@C-c1RX)iu>LC3@Kz3T1|u}|zE%AeYx zOfhP@I!_5135h&A;*cdW?wP4B;|1$4*qaXBfmG;1yYFtyP0I1#GnIoJv*9bWD?MCA z9^ANA(V+!54;z$JdOV6qo+Q2h=znKy*u}Uc;e>x3N(v>Gnx7S1ux~6q@e%Qza>wP& zYZAA_fNLh-3%@^kcU86_u^fKL23oFIH12yGO0>U}yLCvlc0cr^y3vGDzs@ZKA#nsT zv}mEC)sJ(tna#BAR(d~*pam3{Zx^| zaO-TGLG3+obpXBM$HW&L{PX^XHFhjPpVPPfn^}Ms=L)2*0sdrj2DK5#z{=#Kd#dM~ zkk!jS&I@w21P z`%u1^xSnC9P;m7Rj0czE0-NK~8gJOc*qQ5I{#}xa%Rj(KFfHzt{E9nGM=hdpt1J?? z_3Ms?Il?g$8a$TFyVzphXdjb}+_#=gx5#LmVyT*^G*E6ts~su^D)e%FP8pFHo@ch! zj7MNuDW(EcYULhi73VHB^_JrcX0*y=S6g~gOvUQn0P3^(Tu2CK-KR47xcb;{(w&BE zbx2eHYSi>)Mv#%co9|OqH2$%C%E*zFmc*U963s{OLiGP0o@M?7Bm^ZMle6Kj?l6`K>OmlqxNL8xtyzIZWwaZz$T#q5+NCy zIs(&$4c0<|WH|w#?rAJD_6LOd0x|XV4BffH( zDvG+kABtCGPZchR=S8QmVX5ernWLq;FS6OI@JKeD`R+LAbYA_HbEQwq_SUD^Bv&WH z9wlCcS3MUiQ%-6@>%zsEm^)Q<_ge1tEQ>F7$a|c(OG4~~ZJikSR<~M9DyFGRNyb;0 zLC@%k1e97IID1{qMHP`{ga9WoSNLpLYG5XQdotFje2DQMSn^d>qnVUNkyBJJTwrND8=TyOAXwEmb zd9(}~6Jpdwx;;hhif>vyE`H99SrAPh&gVuyG9DlSd%{~F0`jR92KPju2vtnJiFyi} zP03Er6@46z_B5OI7*vVTU(8-C4Xbh&=!K{tompND)@;y$BuVRf|LLw`!v6u47*p1C&~e0?#}8x_})lnuBOy3`Ld9-_W{ zEeX*5>xqwSX~e`$Zvan+G9@iMZPqBDlxy|c1Y>}#6FD(n zfhl=EIyylV$0_FHwF-?)8#;1!I2=y8xiH3_gJyB_^KD|VRulScl63!YuRyUhLbe1Q z!mLD_I(K-BA#TWQVCwgz-znB@L3;%RF;nw!F870dg2`9H!c*E$-px0Dc{HjH9rhc9 zJDvSfo;ddr`kV@G-Mf&eh5bqQ_EJ0Xx=HZiNKMw|;JsS-wB|P#LnVq!HA@lfK$^$F zhc7T?rV~CrPT=&!$n$n%9YWu((q@NGi?Do~$8!p25Uxqx#vxox%l-UIS8ITw?rEJF z6J9<3cAGmMsR(*i1l;KIRF5|+<}M6ul-+_jRsz!?={Jn51u7x5bE+;XaSNk&&tUgf zbu}d*FNw2U_fkR5HSs&YD z9UQu@Kuw=F2nW03L!7yqCvpW^cxlT244=Q1-;O`3@YJ7T+wlfjscVcmw1M@R2BSn{{wwYpJk1|r0N1e z+*V!F!*4VhuJV5nT=SA|?%G|Zw^HTt7 zb(0Dj*hw>oB%?EkSb=MVRPTSU_xwgxN9)afm&f#a-ZxqCYh~IwppBxKTT$pwERqwAsMdwzAA>&0+vvtIR~1?n^g&nc zXJ+&Em)lQy#5ptX#Eydn40v5!F!rtdg;SvT<@Ld?aQs!Tq~yqu7n68U$+>Z~?}5Fr zm!{4tqSI`GI=ju|1g!aNq=)(AvNh-R*p945lu$*9{5{Bs)1X`=L^p?~@Yp@oLzfzZ z=5lT}so7N~|11JPyL}j|KdD=Oa2HtsGEmPavGA$|JF2lk{5q0I32|23P6EgcRU1VH zjgqghPjSOeqj`e6(1pqCu%UaIbQ=o?Nd>4dR*oL82zU8Fs5*sGq2|qih0JhJM|ZIw z;WHk$IhwGcjfEoniQ!a$lHfXGz6|Eo@?wQ+l1Ua`Gfg=at>Zo-IIkWOP!%Ci*(ggS zM@U5cE?Bhj(7?tg)PPze0G_|H445M9Z?7*MdTyx5?o#m`3hXr2l8ToH7=`J-ApmYk zZ8EAP)Z|Eua6JtmTABgoax`%mJ%{Q=W#6`OB6P<42=91*NE0@cJzMAC=_gGfMCS8S zQY{E3;V7qjf>dn$@*0OMH$;6_B(;F{K~;jHyRy3(bsov08FW=_Me5d3OP9wxl3G0+ z&){huW)0&B;^}2Z5blGeeh(Abhe3RB#MSvU&qa*IdsmMJ+l{`&%|sQfq-$$+x`y4+ zu%ZbikJu-~za^GGDf3t76KH)jd2MHwZ-B-lcn8r)JpM`K9&aXQQ z&oI1Ac3adEF*+^N>ottr&B|lk?g_t%jqf#~c^Q^;uWt(Sw?c2--CQb?`;^GYhW%s8!qvWcR!*=k`&p(VI!tp)-$+yM(2T-%qnM zam8>kdmqaz_uJB2Fc0k z@)rcG463#~UxaV!Qya+D)IY?^f>$C%#F5c+d_oF4g?4B%k@jPEg}+Jdl&Q`-r5qG- zPnM8;%;@msA_gj~sB=m#vW^H`${|5##vUu=PzsqMZfAX!j<4ADN?%=Jj}xmDyeAzQ zECeXgFdRmcZ(9Q}h)%1YV1q-&VW3k<4E-Zg!;AJ*IZdJDQ(jc*qt|a8sH zGHltdH{5et%~Ufi59F* zF0)(xf(hcJE~kOGkwpXfTNsRH2moFd^9lyM&`BotSY%J#SRzfah4%hT=hrCy0>_?U zWNw&$Oo)`VO|wM>U8ZW_>E^1rRX@5LmwdD{vIaeFfbzms>pG@iKI=TW#}R~=#i>&; zcnH~-4^m@XdQxWQMbP9hqiiC~`oaQ>B`|_wLvmo;{4- z)@-8l&)rm+UK2}Tf)~;Qyv2LM)OxesBc=j_%u8RWt{J}wcCHU`?LF$0vsL)XL`rx2gmkvMejUf$iknf zOdMi0{JnS5vFZ}%9S1k8(Mbd=tKT#ZSbnDL*MfS*4f0T)7B_?O2|@s4>x`jpYVN(t z_y}~Uyv;7x$ni{b7GRgNBVvCU@}#sVBeru7v=s8`_NmkG3l}ct1r7#LHk9a zC1Apy@{I5U6%|C>Je|6AL|%F0;gy7L#I-;#3(5A%FUa%xTMvRJEu1g5kuxS4b;D*- zRE5J^E`Wofq)aK$%g`-R&rNrUw7|M2^xLyW6N07Kx~e2UUZA9$U|MUqM#=sEx>Rjg z`(L0{U1xI__eSq((m&`_Z%qG%#M~wK|61Ame^j#i=RTuUj3!kqxg9n6?kmw#cP!C@ zrnYl&YFS#qny#z&69;J>0B(ctf5?9_`iMUT-xHcrAn?4bWgnZdsyYv~&R@0YWtTH_qg3Gi! zxy!ENjA0}%cu_vP5gz;*MA{iv{Y%~3UG*pWx*zP3*h>ZM^`72KVo|4 z7)%FYb|oquMR`6n`>f(~l(+Vb>s@s8MzlKJNyp}MmwTtWXbsO<-NInYyms-v2v+0$K4&OT@voaYGF6bnzYtJkuo(Mv)NSr>KM8_4uhfA|o=@0ii$ zxGVCdZt-<%z#H<_K;5Cu{YxJNuSX(ymFW3!X8Fhb=D5f}q?!*EtcO?cSTbN_U1Dgu zxwkLAO+o5JU5O|=&@OFj=jkPc3=N{ce*eO2y9A|SH>Y`U2WFfHD*%EGrP7G1(6i55Jg!Lw2j4wVl{Xv22k1uP z^r%iI>X!Rj6}s22!~Lj!B3+@R3#_FSV2_{$h;&y%`UQR4F>`qU1TdHZGrp1A`UrMv z9%s&ys_hr;+U8$n^v>$hbbIQP1aNcpcASR2HsL{JMvevihigF`{$)q(rxlPl*#39DKR@w5* z-!aMMO_(p(Ku{FE{zaU~))I3juUn-#iE;t-r^h3&@R`O$T0I0BC7(C@B(LT+I+92xkz|h%c4d$M z!dfqJf2&D{ho~}qG%GxRwr&jeQn?O;Qh}YKMW2mE?<3bM@r~cZ0}%IW4e(IA7Jh~E zV|Z9lgCta-jG=j@pI%_Db;Vw;z{>6imb$wHVu@-uHyw#DsJHT4M@fgN1&rd4JKZI3 z##fz)o$$@w+SzcyCqimqtUtO?GF6-7dc`WyIj-pwKG{~K!URK- zDUuZsT%q)Chn3svU6zd+l456E-s@_*TbMQG==5T;&r)7P+77=+z^jL zVNhQ)T-D>w*-Sm_c&qJTAAw z!&I==y_)2`RfHN;km^qlF<6&D%ebJvY!MKSI@BrS2g+}q8Q7$C#lGO!)D15EX@2Zm;Vp`Sypz-DIRwav1UQZWJ_Kr!mS*XX=|T!s z2{smvn7~7`>54QA!~Rb26_io~+6gi_e7UO?9H8pQxq|!VhLZ)E=*-p0JY((>(Wqk$ zReimF>oTtBo=SF8Hn|+6-nGSWosDhNe4?ktSKe`61D^o$t%2%N0CqZx}5)EFV$ zd@&h(acO#rnwe;Da_Ynz`qRfbyaiK0_qfld?a)ILzY=fl(61?s#}!wCh^*jGlVV26|cM@mp3y{H63i)u!|)K5s4Vt^4gB zOU~!l#L;h{z82;-BftXAX++YU;5K+=d=FEhVLf=X7Oo&u{^HI)&!mcJK(&2Z>{WV; zPx5$A{&+~H41Cjv2kX%) zr-rO_dd>%Hr!*k#F}OCboZJiB5lI<5jT#?ST8 zRm#F8;m1#}T{Fu7l(F*X9<*pvZqCTF=v zl@f>QdfAEe?~g!R9AcZo&T3?Km|4qJ#{00gSo$vB3;pYd0uU!Ra$8l1 z-0y84&K2kKZ^jj=1#_lbF@|Qll8hF$C3vl7hS>h2R6U@NcwA=hXAPZ` z;QQZux9BV|AHMLhn1B8OU`^!Jr|@2vgXv#K)Ftu@VR#Lh7$k4jW_bgfH zZ0zi5AY$p2~!iH16FqBMq60->deI{;OZYop|FWutP8^aHOMw) zx(pqg8xeo@9yhw(f8mB0%`8upw|xL={bB7@lH8{$6s@Pq2MEMBgVgO}7D zR|}RT6^A2#$+sci`m5D<|G)HC|G)KaI{g>wHv4}s@?AYoT(36``{NhWPit-3bOl^U ziC+l0NwxpiZ{nFx(PK|75w!K4kTw|iO?-+nj!Jgvfj+2HgxAUcF@9R4#Bb(iCG|(T$b=!TfpR7bHj6K`av!!8x zemDFI3G^NpN^=8VZM9{>pNZs;Zbx#v1@{`OLOn+hNJu&fcSQKO&Y?%Z7bccpMYyuu z^B)SyI*U?DsoZ{k{bTL?BD8%Wst(GhyJsQ<8F;A(d{<@npC<$%HXhl2KM0WC`ODH? z|2HM8K14E*xerfoU37k%a6}MyzP?1}NZ+-ASHn$%e^30Rp$oC6-!s)5Q_I@q^*PHj z$jQ2fTD8+Ei=c5Rq#1{YVI@~+}pK6<8|7Sz|IL=3(|jNo-eWsSaH ztE;JGNe|EVuFm}DrM$52_}W1jbo?pe zB4L~ZV*j|4Yx*T)p6HD@cB*_z9UULv`!nljw-&5nuzor^>?@7g($%kQnatJq3Vq&I zpTGKD!6s7~rcMmHmuW;jI72LhUL-yjJmZj{-7uB->+eP$gTB+`vfLhKdk2kXCOB3t z);9x!d%+**%?cH)lGWs)3!3O=v{}BcwvO3rN%wmz6RqByOn9PXTB)P`oT>D(KX5e@ zBKr+uA3Tm7o4EB@VZ|ftBlo%%IL@q*km`13vf9`hmM=^rDN>)0*i5S4HUrGhl?+8_dD>`&@(OFP3DBjWBw+DSV1t@VHZC^Tl15Vq2u;T~mc>qO zI6*_G`jvZ86GKUz&#*S=mN2_8D$+O(^n1L<;^z?F{8A{cTBrDF z?0)!KTs(*m?)0)v9~Qv$u!$w=2V?SD&-J_}T*%{d2YJqV9Ns6mqRkfb3z4?|lwFsS zZy}mvSci3Ks4143=r-TkFDmpXC&`fp1v&1vsxg$W4;JN}QFXuF)jbSOIn$<#!R1+W zz?00ZxR{*ud#FdC>+2tqrHPc=@$=F^^Rs4n{rS$iQxg|!+mZtpG@Zt4wDiS~prtZi zbG|i;TBw&ZVdTrAR6aXix9VBz<7{51uDsEQUt{&2j*x*I(yU9})8K(R{{vln{&S}b zN8BAxvD!Cc3>$JJcr(dloDxR0qU#Pi4%u^$3^aKrW7ocfL0(}HpV+OI2HKs4+Ecdz zxNk!IyjVWyWN21xF_lsPKz^P=9!s4)jd8Mig%0%mVn9dt6jIk z#=B@4jn18`oCZ-SuZ$0+j?(d+lrE@kKO?#!dGG>;Ei)@+I+*Gn?`1~U?e^)#9oj?eGx&8mAJnW!Sz1i`8W=9fk3TmW3$Y7+*~;~vFMuFrtEka1Wz48eO&!f3&FD+SR&xDg zCTG>iN3A9lq!cpRTFcLUxa7*NFO*#2T{OGBUu-y5*yIlL0Ixg&k(RUZE*-AAy;)Ws zp{kE+tv7w`<0JIr0g8=Gqi#aR*y9f!Sr!0FNziKMtBBHoGL%_ghD$Ws>+!W~Ff_j#Wq4r78 zL?D2gWl@_CL1K${KB8smM|2llMe+-Bhokj*t|@gjY6w2IxftXk*$(kU44^d}*bP4} zeUwef}AAKUuxDBm4!&9{(^EYjMyqxH7<8W<3?8=bb}|7(*Ot zS4~O1#gp!^ra|&n|M`JO|*eiGti z6N9P7&Z?!uapyQX;~^&M zv)?>7djZs42b_>;y}aoSaYW~hgqJfeMa*XlPmYd>i=Cgg571KcN)4!b^U4x&Xyj|8 zUQk^f;@pCZN-mOlb>c0b6t=(ED!ryVD?O=pydK-Y7HzBH9VMH;8HaDCGL8F(-8xAA zgQR?Vh`(o6Wvm>HsXzr#GgTr7*E{eSfJ<*UNZBlvwle4z;xJ)@S)LXgeubYVrn{anxb+RdJ6$g6Rb5+FW27?8IaBVxr?bWCXEYAs^ zvP_*-UPz%_JSULd~d%G6_#z?eahX3Q8+;NEL8{BCCsr|8WYTiEKx1++EtcDJ zrxCN9BhxmTOwG*xM$kkjYy)=p;=D2+2U6bhK7Qy=0%mnDm2;ytdgk*ohg>-ImS`z? z7JEPYAZipkAjr!WU6PD`;E=cUlwT@T@#Q7s-G+2KF! zBOU;Kx?q11vb=dnoXnPzHU6pj#r*E?QO8^n)!M-~x|JJO3a+sRYH6pL_vN#Vo)K3x zEmRr9nvt7pB`ZVs5f+LeQr$l_PhT8e@8ldsJR~3}L5OEDU*0SSx8<|}YZWm@9Gw=3 z!<50t_>IcJo@}6QcAlHS^N9s$>$fm%(Ta)>_jt9MRZL9Fz^3a*-s#zpA%&@&neFWe z{QFhTaNUZe449>d$?b5jZNB5`Zg-Z}zHKpe+|zGF<&0&cX6*v-q-RU(s%1B@g5PK+ z1q~=6lG0%PQ4)LANA9knXNt%=`2cF7tNMc`h_Ban0kq z?F0Ni=*?u<`(Yccq~4Usr1Ko_Eb(y`nT(;`*f!@oP1ywNR-$+ffRr=cqEDq7rqYYm z&$=M1N*P$f2)im8?~+eADym8d*P8U-Pks)v0B>%GGB2YC(Bo!|uJnmg7Ix}%WS4bB za1z@u^p;DYoWyF(bYxAl1KmpT1{Wbt)%ofp2c(zC72-YO^AwPYs~O0@*F*ef_-x3} zLWcLDG)3`#<*kc65 zc!-0VM{URf>VwqSOK(&B+RE8A%NlCO`y@(7aS>;UQFiWvqqMUdr>KD1PU@#5{wHfTcynvTeJQ?VVR)#LjFVLb}eyvG)1hli|#W1Kl=x zblb{qaXHU5b_4JQxX&_e?UE7NW$rDkC5Pb{ z1*3Y>g@PSO;GLWsdjfpgKL^{~%brCm9zn2#ce`!<%_i3Le5OIsZyXCjH8WMGft?fvb~0(k*}(CMC+>IaHyk21*oTeD0}#xBcs=KPFVn|PQT7aU}b>?7_M zb8ERL5QChNl zH|g999rq`@wl^IY%v(6}YFKLcS>U>TPp_UFi1Sp4TN$q*xEO0Cy z-VU3vWh~?Y?{!-R4w1c%l-|fHWg1T+PGa|>(6`o!T;v?H!c&-{ zO@Y@zb%m_&);kEPQv8D07=8LG`PJmySVSl}_&25-`x1Vf3jCRGTFLT|Qk(yxTI(Q% zmKT1}2#FZvZc*>o6ZaYo4JG?b+toEar2~Q=2#nUV_xwi}gkIn6JaMhmh)!Iw_QF`m zx(GLcH#a@ST}=mq1}*^e_#wdvSTIaqPT?{ETX?Ndca6cILC)0tR{?YhqO&#k^1=xqjP2^bN!-6VBrgy+@D(iET^ zFdjX2pSh%BhiME%_rcn-j>VYM(!xs_Z&PIulw^|mO4UZCqDy7)1RE{Rs~}@0`vf^j zz^>e^V*t7A##>8uw0z=fQGM)HHf0IU4+o*u3iyh-g0D(>p+Z zmM4Yf%*XHp1nJpa>(H({6lv%YY+s3ba9m!xBuo3@Wk|c8%q%l|r2RmH!YfcMy_gxj zw{pt>Dl80F9DNF&LY@fK!)IRoYdNGeB)2W})pew6b4!nY>Toi{q2#+Abks^IoyU)=(WBJAC!< zUq6x-%eMuR#l}DBjd}7d!)b*AYbhyn^U4i_f0)^SG0F4@eflT)u`;1$LM}g3F3(8H z-m_dw{po#$mrTrqWr9oY*touPfoIK)HBzVmAA%0D2px3N1wyE}BqGT-_^ z;z)CEfrjP7^A&THhTs$b(B$gT9paYLYyT!`B2F>7m0kYgw0-s0TXyZ@Aa{709UZ{4 zTfBAd_!-?6<`6_=C4Jd%W82TiuKa^gd>f1Kqc><_quxvA$y{cqZgb6ylhVJiwzqN7 zf1wZ~MLRrDnO?E*xFfj`DPpd{-_Z2CenIx+f%!caTlF!V+0rqQp?KrWf8h|$ma0cT z`ELt}A|!A%?)aD4Qf%RWyFB){`yTu6XeSMoFYlyjcb+aw;>e|cMO91aw!ZP#f9%>N zxE5%b_O|??QGOUV*9U76o4mfhVKNg8&VDNX!*1cZdis4Xu)G@>pUc-8@7^=Gnihep zhg2_W5A@JGa+&AZ2D78Bv#xmarfMQ*AmvVD;Mh`?@OGsQEO^Lz)vTU6K9Vx~MtXuw z%+PgPUZgQM=ob%YB5#9`dsvB1=By~BhmZ^2N`_qdE!iHmZZ?hse*c=?E^ECGzzrSjGf)?-cWQy>Yp|GxfBBABT=yjcoXi zoOC@<=g8{D@(A!lTayb*&CZ6lEFJ{ZYTwK=Q-tMmG=Ot39E4 zHpvk^IYB|aH&HKiU)=~i@ON(UjWkKBeEHU+vrl{ykD-;@O&+Fk;Qf0 zF)|#%HDmI)VNK!NaBKAn7_R5ejC&nJFxd_VUc_2@tdb9hq@y}F!^WuLIJ6uFwdtXX zSlE~i2k9kr^K%Z2xhVVD5*&Ha*Ihe@fWIpfzLA9A8@8};{?mTmsO`)%TUSRZ``ar; z3a1h)T3d=HuQ5Yh%nnIbQ7P>?*HjJ#ZG6p-Vp4udSE&OWPHaF|ew|-d@)K`tA8i2e zr0~X*0@d6PkKjqiAsw!46E&eP3Sh8^G|lW9YwQ?x-kxeI#ZwrX86s*Ez0Wk5>?_ml0oL|^pr>Sn*TbCG#c zFL893_bH%=dqotawo@pOo*^f8zHSaN@w9(IsqWmlFnF=kNu?teA&EwJ8c+L@Cf6n( zycoBZf1zpDK0!SKrXMay15bg8^?Hh!@vb0UwZ@DJ>p`<8>`wUWrdi-^4%lRZ30Nrv zSAOlK$=!eJgS2m=Uo+4J$*Z+HVoK8415L3qJ(38Dzntq8hP^Zw)jC{B)=SCEI|JVu z+e~QcjN{QKnI#Bk1B)A;TP}*FB#ai#E4V-TOpnt~j90?H5bNJi{xqaw-Sw-c{b1VvDZMn)4S@#Xsm|}u>ESa-HrSFnxL`uw}*$K zKdHI5<##2g642^n^&v))(p0Lxe8HlzA9X2-hyl$lW2sykR*Ee=<}~C`+1ZlH)!4ng z=tLbssc|C)gm+hZ)7Wnrcgw(iD*xmWhg0kV!3r^d}GQCsmWz>P~o;F!k#_62)NvKc49Vg_%EvUaJWXO{EUD+7H zL@K>aW<@<_Gq(0o6IvX`a?aK9tyrwBLIw}yxcO04G=uhK1M8in&F5i)Swa!qfP+(sl(2wWLB7pf0VPHI=b3ntlYciqUaZ#|52hJq$!@QdkVkt7Z}HF zX+x%A+Knph(bt7Z1M<$J7gDYt?rVREkCIYq!2u;OoZJ}oLhAM7SAHFf(1 zbJD0#cO0m__T`6Ih(=FT9?GoMxFL?#PB_PTCa#%L#Xva5IzzX}0W(BbHeZSa7G$*g z6vOJL{czGP`CH1w=iDU2=Hr~2 zH@+mri%=PI8f`ID@9-Z=m$$2e11(!9xFwFzjz^G(q8b?!-1ID$);zgJ_J1oOqKa_ui7=-y7SrpzB>B@aa>oS zicW(7Mq7qF;))@1e=23yt{C|pr)C&!*$X0;S@|wCJn_)@yNUI$KmLE};`~EknI*j~ zk@}B6`p!&Bm-Js>uRyxn|LYsd{ExkKdm1 zJ;hzG1@Y=@Kf4Elx7XMU*`>G%TsvoR0^OLc@|Tu&{hZI+UdN4R_m_%7i0e`U%}Cwm zwZcYz?HCic*@3IE5Y`_0NA4%hY}Lj4aVs`)Q&M2uXc11!zCNCw2j|jpk#w-&s(YkR z$Ur#-qSuQvnRIMSkNHxY>;iM7dST$9OoJ>9(+IV}G!)7F#rGr?;KP^jn-B+KxDx~- z40ncv2*VjD$H1*=_YkrXE2_15dAW0;1sgYEE}Tl|cUGW;QWiUxNmSb3vhOm4*a(F; zAud9p6U0v_bcRR;aDl{==BgbH@xz;mMyxPG^G1Mp>)F557_yURzJNhns>im9hyEVs zmn;Hq!=#*0y3tThAZ(aa5b8D>DhMqLiQd)nUd2muX9)H3=q-WaLCW$%+#H{!JqhvU zuV4H}@g@ZkbG+9S>>RJ0LYd=LQYIRvjwneFz}!|suI29|E=-9-lf=|&)3#Cqv0K5S z=l=1ef7TtaN2v}n98sQw3>V7+WCtOqIavyD{#F_{jPj+3ClD|Gwp*dYKcUGx&c82e zIP%Ch}T{3DkB1?@qG6ACrRxRog<9ah_L5BU54Qf0iB z4XAJ)4?m&|`4@|59{fLz-v0OM-X{&uM6$v&8|UA4G8-p^I+1_8`)7i~a0+-m6%*Sw zj^ySqWQH^kzMaeFNgEIDuYI|L@c#Q}x9c`7SlbkFdu~WW9eOE|q+{Y3D5oQh3laMN z{=_4+eB+m1bHERHU5;xg?H=PGZte7&}^zvm9gl@6ugN_pjsf#5k;b`gebi z*2v%azMg7f?_0zyfJ9Obqn zDu_e>%Ki@tt*EQtp>8o~Rd_X_8}@Hl_)B=iMnSx^N&M@7O=Kur*HCP6XidsFd_B<1 z@85W|(jcK*`Cs(BU&5s|a?8J5-1{-z341VmoL5`F|QH-kC`I^fhWLhhw|% zL{hT9I2x&*16xSr#q})HxHTnqwNp510+}ovIba->ZaSD0IfhxpR2L%~spHxCrTGKx z2bUwil%y|gN`uGWUyZ9E7`G%!%S*}p21$fP{9g(aey~Rw^Kg?8DQ%W3_p&qDNk9kzfdmK;C6EB24+PS3W7T`!@0|O6 ztn zLcbepQg$w)yY!2{*`XaIfQ)YV)jyr82fs~t#wfc%lUEevf^T?21NN-VroNV--Uk~J zSl8Nz7L7mb${=gq2cuqj2uV!o6FB);rhUL4l&-3w%GoqO7tz z?~Hveff2Ty^)=QqgJ}1730dWh32PiW!%YiaCLmHq=9O+h_kq88dAwWuXY*ErB)rM{ zX%3Pq!>f%q4BRqg^C~^PuO5J(l@QA40GS}Qq|BTLp`pU6!;adgslPqOKij6_al$<( z2BLcO9HkXh$929Do-h2<3$I#o{z>v`Zcx=- z#n%sBuGYkb)Mkc#zk6dU2K9*2eMSLg>9rf3K0)C1;Gs~!OGXqP=ppVEx zR4d;ED|!5m9Jv||0fe7dH?^xWzJ6a{cltM|CbN_4bO~_pu@~7|2E10 zIK5FfB#ln+5<+{M6aNfq%UmW%_(@8_mdYclMUYKI=kd%i*+de(MHK#@?fxctb54Sd zT4SPQHm*f6)~I8aI<3=zP4j?tjkpz!dE5TVgnzle_vX2&5zTF)J~dH#lbFvQiz~eF z<6ori9r{^P~;i^N>JKnOuCP<+O#>i`Fr5Mrfev%RdwcnaWjeA zVg1;_znTA4^QUvuTySC2W0%mjU({qL>}oItVgB|1YPZW;-~Q|Cgs6DD{&5|;@Yus3 z{z2`g&aYyP8L`Sq@jIFrZ4zw@TKkV^37Th@v>JBJ1GO6Vrw4k^V`iA}R;{GU^VxRa z%^RVc3o@HEuuZCp=-dv}r9wn=5 zk`6(x)Ss6cw%IzLH2GJE^eJ#>-Jg;US2q7%c>f?{tB)E%aBg^n<5s4m zQpnZX$Y=qm$%8k5YeFfa<|VLkRUh9}9JJto6rjcoP;=t3o}7&aacpG;97-$m0OcVA zO?it8Z|LG0fiRd+yO$6~4ku?(dB!nm`8T$d&5_xw4;MHq#tkj;BQbP|@|cr4nAb40 zQQeq;_X*2rkgf>TqszS#?yZjKo42LSN3&f&b)czk?l@Sg;ArLPp+HF|S#A)b^_G1C ztW^Bt%zHyrQ$A(>MnsRqK$-SamqkwP+4jx$iv#-VkoBI@HCG*_P+ZLy{iyO#DmM=AH`tR=4Ey%q^exP9U-nLGz*5Aem@ z9ZnE4PwCEZD~bdC9958X3%}Ob#BR{yKQgg3&8}MduDFJ~+RgDTQ7aSWpO9XgTB9;A z6C6iNV)p0rrOy->^x*9sSsa5Rrco;>NxuJ2Jahh|&tzr4sOS2i+`+DzL{hRcZ^Vl8h5T;@WB*q{{~q zNOSL#B5-4VFgz)?^dhE5diT^?V;W9235~Kx|l@Ok5BKHb-aao+*e@ zodu4_%ZY{#nuYQ^Qb&Vpx|l)Y5|xfcp}zAlt9Wr=C4W581>{{o5{~_ zSx&pb&rmKj!u)kr{+j$mI{U9G38jbJ{KUsw;~8Nu(mP>BwzHf`#lemgbwm1PkH;G+ z;(mB5O#_&=rf4TR7-D)sHRNbR`9`kLnc5Oa$BeBirZ8%$B?o+0u}{M~>Woscs^4^H z0C;vq$;u4X+W3i==wOus(M(OOCn0xe8Q&wA`Pd9R5oOaE*T7L2;i8BP2S{fU^K)*H zZ2>0$b?L?Cx6x$LW5}0C*E)|jSBAS%yelh!bZ~;CM((&irI_OiNniLF-rAyT5sdR} z?vyk~J-aVXxyV|_DN8G6FR=o!3=o*i613p`gu^3YkT2QsPjefp#YJHhKe%qa(u7vN z$Bi$!^XIWJa$Gv&hoY>fvShlCk%Ff_T4D9|5aSxK5U}6+Zty+<7gd^{NsY5&9p?oq zPxNitz@bxTJKc;^d(Bg==csMi&{X`6B6sbz~5at|cGAUdDwZBm5+ORvoU!4@bBg%6O z|M2V=6%(DQ>{HYHlzMh6Db4`XvFyk$d&$N%#~Db42=Cx@Q)BZ! zE|%5#L)ny;(PHMO+k_WO#J?ET{A6Z7+w%psnZ*qT#Z-2QlqBV&97wd)!+H=ZL)aXJ z52H`BW?lN{4B;kQ3qHN3uPIvy7h#8Ah#PSu4J}nGlNaWZ^r(hFJqpM zI&%;)bYOQuSYXH6?X}k-f#{a=`wR>BL-Ijrwnu$kh+jSA7>v>PYgsxU8;iH}=GqGKZ&7t$*f~U(G!GL6j3&bm= z^3=5j8F1<(rU|WHHZculvpkEmLIN#OgM2$J?99HU#-=$ zIyUbD6ByR$pqJ&l{GmrIX85**UZ@V;fs&$0cGmMpxw%hp42Lt79v-mG%J?|Hs+{^Q zk`O!%sS`K6WW0Bq)=Y3v(TUw$AtarziheZEYbIm!WaRteuK{hlC|Z0-sqJS z+61CWe7TA1n4^R&NBiZyZn>^RxSX-lLC@<&=oO?`PF1$-ObvP!mgnk-sB2gur?i|1%-u^HB*-u+{oR&}WQvSthj2 zb}_+uVA(+p9Riz0km>?f>1)&vYkzX&R>4Z#<>V;KRFo;m_*#rQWmf;TKkZge)4+lp z(3T3jl8i~I8wj%N$9;8Es_nr-_0rjyeJ8S#A_FJa+5+_yJp53~EwBuE$IWro zgMs++!OF`U^MGxd%<#M(rv_XqC}*3VRCK`*X8L1gup?(bnZ6pERlhfx7eJHaP}9m+ zE-?AllqjF^9Wt!QUV@=&SL!vDa?i@Bu?x8{cRr8En?DGs-To(33)tPbH6*NRHTG+E7%0INDJhC5leq8OIn1&NCV3PsGv<|R zaE-DzEVg!l+b5(QU59{c0{sLGbAiohg^K~nUU_Vpp|o%%rYL*-o(3_~;Dwozx*Xs9 z!=2QYd7V`1NW-C!x1a_lW!4~DH`8l@h9H=P!M_XYsF^JbtFg`;TF`FYa&SBLJhT2| zp;3nPp5Ztp_LT<6qyRMz8aFWAtf&jZ+cg2qTEFOygfpE7MZ4!&kKq+*$BkOC#r@0X z>`TPKKPId_>nJz&kE|{4#D;<18`*4PqqKiRXlwId=r%mO#ZTi5Gf$7i4b+`tjnr0o zW9tsp?d8B`Xja55kbU#M_So5atn1qH@u)vm1@)XD#JHb?=>_&B7yR$yJV zo7w8-PCy8PoiR@hDVyAiu>E*DUHTZ3VyL#de05%vNl?xzXvj=27j8@jA1k1JN;F7? zhopR1o&iQjlYSKenUOA7aQSE{=PK)+C_?**`G*q^kg#LKROV|?yf^)hbtmk!K_~lw z^E>^_b7Z{vd2ZQSWr*QBm84*i1AKofEY)JFXiyCo;ChuC`Noif+0_?z%-{(CS%;I5 z-bnogP^~)=8t1bq+Q^$P70Y7{l_#rYBv6wCMrydX_NhT+5XVzmzgzpF>3+(vfumdg zGB+~+5Q8m94P1JI2}O7XEu^vn_YGob;YMj;*j@t;tmz1ltm@l-0hzBkToqbNdoQ14 zkgAc@23!Da9>N@F+}vDXX^q$tPg83Z*{jjJ1Z@4WUZUU=b(0CCU)<6-Tw64NKnpyw)Pv2KbTqh178 z)f;WD5jhO+V=P~^X?S**L|AacBL-x?qp0y4!=I5_Pb8uh9nwt;{cLtcB~+>B=(e*g z$g_G4xXcub%AI4V;kZVzj6_T?sG_ZC~zPpBMR=VVz3aj!lWAq zj@@&@tV^sRJ#o1XQJ&WsPLq!a%_#Q{avmJ2WW0ABBKCo6m_;-n)8F=tFuIh>HXh7H zKB`BrI1<81x-?*=68GX_UV>H*c|&rDdbYk393j{MCF5tlC7~$opqo!OgeMTIPT*Dfq{}db-^A%ws@(FpD zW{UMWfv%;jb+A2O(r928;x}HI^iHEo@M9yny=>m?dx#6H+Y zVs`7<;W?rcXw!_~!3xR2`?FGb=HnHEA$@~50;#Nz(0Xk)Fu4N8FV;#+rGAlv+0@Leb&8gyq2`6M5AQ%BGr*D=@AAwU&~T+m2~Zn)$xt)yms zqUFjrf*SGojJ|F!_S#A*12(Ozzp0Hd*fSzpZR{qRbZ1-$H%Pr;kij08t+HFMNs-62^A=s~N$ zHa^7CBFW>GR7p!`hP~Nj zL@%V&-$5K>aQOLkf_36IOnn9Kg&=%&nZZG4t$GaGzDQaD!eS6To#ZE?iD2jD)cLC} z2yq@(|7MTN!J0w`#GC!|H>N~A+T zwINjmgGsYh@B+kDW>Q{|eL1zwaShiZ93_BHks@A5U1Ej4=wU259kfKLaiZT)v%X%Z z1^E;-7^@ghli9dQEw+OvK^ z6GHUX<3mCX?RYogd8;TTvnCI`VGa$X4vdJc7~`FtLkl$5=y91UF6uycacmIPQm~k? zCdru9oCN6H8Hc7oJMWx zGCsIs(3!RX`KIX74@a6(cs3PDQN)vnj?wZf=M@(S*gonfYw^A>Z@Hr*X>tZeir5>4rvX7LBhG$W zWL}+<(pVhSDsHCfB{S9`0#>{HAtf4a`J782m*^kXXprRpg7GE5N1e@!;M1>GTBJ8onlc*Crhlk+`jOM~1tEUY{8v?_X@ zOZ_b@#UE`<6Ptc#T~R4g+4i&YcZ;C}z?};^B;7i;4d(sJ7Gv<>1AOZ4t7U!eXYA~t z1o!?!0eEbaI&P!tBp`nCcHIFgPKzQJW#MxDk41VOOo3?vtA$g`ayKaCyYv@B@od>G z_YnV1A8xs=`|rHre~;3JRy?7T2&UG>!H_ANx$8ZB?64F{kbue};@ro3a)c1(ET^uf z2LmfdilBV#+9a+i2`T7W;1E;D0C}n@&D3Lj0pcF*{(u?O=d>F|{!MN{g5C1vUH`_o=ED!> zc}vRpT)Yh=6^yz~bu)wcWH%c&x7678Bw$ejyI#QixGX!j9H%B%o&hYbEwY6Xc1_62 zsK2PKHYXn}tVE1F$%f;;A%g6@B-PXx6aR%|t>3OyQ&VQjx#dd5{-1y>gVUd^4na$R z1{Ln#JN3w@L!K_}Ub7VEt7u^`4oqmQC?PzUw>qMVGL737md=QIaKiJRwzJ zE$LRaTy^ux$ZePFp{GpqVOKY4@0#5s;G(hzpKhjygdc#%?AiwdMl=oOXB(j_$-Ck7 zRA2m7>i+_*`l?^FpxEgAxhk-bRhksx`1Kf47FaS1HL@bcIS=Fxu!F z7Y7c#&fUa$9}E~dYW1H`sfQ9&3b8Tw6P4O^6}G2r|64|)>cs$JNm|gMyqijEuVmKg zz0w9|b2qS0LF=MXuu0@B^M!&C5WU)QD6f>FV(S{V$r((lHq0!2ek2l5b zt~BX_vNV7>2B+ZAk>+U8H}vPG{W56*B4zzJ!zz&oo#>J;R-cc7&axUJ-02g$IwGUhFjb+A#?z&{&HKNeP%Fq~`~{ zMqd`1Z_XIf=VegD=)LceyRfRt(uV1@Td@+m%yw^B914t1AuJEG`=9FZUE7T&omGgg z(lgnic@trj8@yF^v5HxZUd?kME3};`WxLb@}z{R++W9(sfZqp<~0;KiH0l)D&3A0#Q)6>=tYeIA)0C~QTy5lFam^k>jbz9^7tj_=RVPw?2|tQTMh@kCrUk8zJIi(b0M$;|1hl% z7J1lCT9PUe9ajH|6RG))NbS6JOupX2V3X9kLhsz>G`}I%QgjIso&`izFgAg6V0<1F z0#^&_nmP)!!J%TaArrjj*e!FkY-wQ|Ygvd#DPX!7Gh#EVVT_|RZu$QQKoQ7aQ?MQq zTq+KQF5?{XS!YW;ffHd4PrZkH8v0nU=Pso_`)3r-b9U8~05Nw&|k? z&d;7>RSECOSz--xQA^z0HCu8F){v$aXHM;;KsA+VW{?@i^7B+eTP&7Ns2^9!R4H>Yeqh>a?f=A?7yYVtk*2EmA|-&K|!w zTWVk4-K!e#_Vp9}0oRS|7b{;qON#7|e2WQ&?8~F!*J2oZfL=xg9}4#v{F!s7@Y23* ze*2Qyv-!(WY6ZM76WYn>6EU|nj{==MQs#}$Rh7EH?i%fB6Y6uUEWI93_}oKs>yc~u>Fv2Xi* z*u)j{X!Gf?;_KgTw)ntM4fLCbjT6O3!9y>u-FU2cu!Lp+-uiUA->X-PN68~elwDfp zV$-i$&uJe&Wj%{=g147W9`3@|J!bc?WTqNY_45k$DOofRyyp(}!0g2%x@aEkJO}`m;}jFsYNgCKoK>gH zCrnL1JC8itf4Rf>OBd64Kj@>LlG<1i^XMI~FH8x8d4QF{MRxI>Z^_tTHp}M5b+5b= zZ5!tTBY*YJwtF}c+SMFYWSaOm$jIBhLFb|9{1y?wK6?DkkCCP0@D07hJ9RD-xH~RO z{if9BoUn0wH*ya(%(QN;wOGboI(A3!V{Xc1Pz9CqNzp}cNXKkww4`S|*gA?Q>*ug- zBYhcCe9jr>^#bvE{5Jc2xPih@;Ocprd3cB5nsAmMGQ1~%6~KG9!z$3%*Q#iSWAs@e zU7_=3Z2elf{IT#kCDP8YXX*0qmaSa$YKr)-o^{vtj>g!Pe{nQl`aBIhy9{}R6C_Yj&^O45*@*YNH?MWj>8`Y2eaKKDySch}NSC5l+y79gJQ?)mK!bApUoP>< zCdG5>tpN%?qdbg~ap&5^s1iwXpv7yS$NYt(6zv|;Dog6;l>QcFDOsj^?{+4o^z~Ro zKzHn}I&hD)`x|5mhWOrA{OhngF&d2df$9h+WhASF6Me4g`i(qT{39)CJB5L`%sUD0 zXIieZ<&6Z2W^o6PLKeDob!L?H->cxlZ@YAqAk6D-5v?mhfcl8e~aMWdDCI4k@tf7sZyWu9OIUB<%0&OZw~n|Nn*o1Ukn z@po6_-_v;{NJ=L!$V+&jWxda>5XRW=Ws{rbZ+3&~r1UL+Tsb)pK)-S@Z*}NN+~GDEGR9FE-Q1!G!UNDK1SP zD!W3+<5jY@L6E|AD)}ZjJl4&&gP2I&i>ed?!aH0#iEKozc6igKhpz?YUgRU}c;*U} zBR{`43>Cb>Q3mS`dAuuj*}G*7nbQB7h4+Z`hUs$cu>_zKW92z3$8pblNJ$i@-a%>A zvGgiiO}zcSR(MvY4&;MS5w2BI$~Cv#lZG;C(=OVN4R!jZ3pE_mdyHXjiJHN5x#6rBk*%}luZcwYi$noB3Djf znPvXVFl0Tx;WZ^BW}I~A1nmq+6WV>wQ)=agJpM=;JD(a=weOr0vccC2bwrSZ4h4 z-l4a1v7rvmoZh=6)j;=NDdPZbh~Ch0N^+2$Qn*{DzfzK^gcm*|gOIM%(ph#fhp~Pf zsgzRRpFvN6sstI7h;RyZJ>$=AL7RQ`uP?E!&x9kcefapO*xKu`K)aOsm~QAJ<6^cK zJ$uayi|AX5HYej&K+-C@Rl9j{u?wF^?bSZxtWt6+(2dQNTfdgLyDahN#pS>9@DQ*e!~9s4nAxgp;U_iSrD#%V|gv)Y|+EXGtYoPsMF;7!VQdH{Ef_Mf7m z^_t?r#^!ad+EAoQnBS_OarvlhiH}Y!B_v|yOg%qhIzLA^+7TH8NlxCQXAd9r48OK7 z#`M0tugc+)$IfO;RyEO> z>SmBO>0XbTR*oz0V%82)%7x9X?KU~asTd(_s0n2{#rl4V@lmPC0l5CVA-WuQ)kdp%5!qB&Tk&MS=i9hELSY@^x}q;Y%}zPu_1I5 z#poBy$&e?YS)@T#L}1tk{Mp~W#hLH_y3ap4=h3sgAOBl^;IwTB7#?xU4dEyi9%(~Z zxv|p&tx@K&PFp|j%WqhYqXe;VPxNS9y8ux$f1FRpfm%fzB5@t`eQb9dT$&>w=EV8n zIt!2fErdtSrp?e+J&Rg?ViswWw;_Har>R%f7+NZ4xA{I+ux3!`V941{_~?#u#r@8R zz9x9u<)fw6f|DbEgvu7Y1LzqGu;EfTFPKB{jhT!yFrGxR6t? zp!rp&pIP{vv%Yw*04`jCFG3E-+_}p%o#>W_7ca&uafRE4H{GLZL=O3# z5oSF)aZJj{pMW_$y1QZ7Wl+NzY`CZp+7qquw*l6d7+{MzA}1#ouRgf8p%ME~D zal635u38a%dElO8U9w4>O2Zs49^=4zT17os2^u=QcKmV*;{j z2Y3HmMI8`{Z`JezWS8cHi5)13llw$$iS2G!zioqCI%zc0ypkSBoRA8sDsMr2J7?1aGQ(-9J__{ALmPRTx)CHYRPZcvmY&~gUA+wB&doJBiv2_k2jiEB0d7|7cg6D z>WCvV_t*9#R^Y`fC;_iWP1)E#>l@-D^gp*^+#D26&TH+Vy(|UHn7%_DN5KLSL86Gd z7rIdwc|9LZ7b&4uLSL-mWS!oRUN+_U?G)ekeQvBOpA+2FwfpigN=FT4&sCk&2L_)?-Lwq|a2_(rAd0o;!L> zX=fiQ#HZ(_(mL-^z0_`NbXm@VOQ&zerIzuT%=K|q4>_f}DpuY4dWFJIb7!*09Y9Z7 zx)T&P$hR-Z>W6+B$hrK&PmLC$k)f zUFvoc{v2cLW^LQ@;ncG*L(ns;46+sGI&ozk4OXcNYXM$o#Ll7XY*o^;z~_2ge4HB+ zephcIHmMO>8!33WZ7a8_v(M<#nd5Y8t_j75^d>cW^g)VlYJBTU#QgB=F3eS)_!g^S z^mWT?_Ai}xb%}Ly^S$EpE->M_tF47ep1-kx|m)i~YElAYD4&*S%t_#+;`%XSGH2-aNC8`5Q)BS@&C`WtB@s zx^0zDdn%@|P9-drtJNRy#^b5|O)D#vFYAYV!OOvc{m+`Fc2m2al~Ly&9+nZv5@wCm zIOl9>ZzpJoa`4HXdw#YV3hT8EP3^ZxX5fh=^1HNon@Xy0Mr~Bu#h)GwTw*Spz5`d{ z!QyRktJm2Sigc?t+t~(>N+UO<&~-BJ$;S=Q4^~Qzx2Whu9Va&G zpH#zBaz+WMv2wfq#MI~8vcAK=Q%|=12LbBRH5u+ zg9`gE0lKo&oSYWtr48@tE`>dSEr+Xv>{Ov*3a>CF>|(2z_&jnDZEaeOG%Ju^p*YPg zKN*Ue{sJ&`Q>x26g?x+Y?(Nx;n!c_+#DdDZ4|w}IL};_vX}&A{_!Rk6vQtBpUT0tO)zZj_Sv)-#o>ap?y@&YDYLeMV@P ziJHa!-AkBV2)QWA_>cI&r1ZG7kfcd~_R{f5cZzcGX~T|?VAzjF4(|%D5=`j0be3Bgck8o~RM)P(>fri0@vPEi?&{gWd z!v@U*b9?LGLF4fA?KjLKmi< zp8&mX$R#V>lp;JyiOmZ0BO8<)c_Y8#3A;XBv6sqfu)WOGV?gdC5)4()Ar zY;&0BD5%E>6_R5)@Ph=K_uEkLa%UanJJx;opQCqfVJ+yHN7QYoORldG>O3X(nn^W*g2Z?2 z2dtOYM!4Q^Z{0pJxCQAY%q>l}S5olykN_%1WnXl4M_30`r$lq~|t*(HFBUoo- z=`R_yVTFD_ezVP8b1(A{i2)Iv-jSdpFo$1O@j7PiZ_-amo_i0>a)b>ecc>1Ooy|ND zd$#eA5fP!>wKqnpYK;eH1>uRm<0+H&Y5Gclm4cl4C}E&4NF8^AeYxUpfn3>~>y2`K zee?+%b4jDH8|pPa_K<8RD6tVSx7z&CJoF>KZ*FJ{ReIJ}Tjv+*FkBLw=r*13s8!td zN$(3j=)D{%3DXjQBFhgm;E!UJCPCo2M@CDAa^RF%bn3gcljc9Y6+<!cDiPz*AXvN{J1}#`ZH|jYZu%#jlo_YKl)P+)m2!idc=NBbc~Z; z7hJZ$BFD2_Rc`E$tTf!csV769>wq?nnd>U}t;0~5BP*tnrLpYB^yx>NFJ1LCK(|eM ztQt(dO{;5${0dLJ$vN0HXQ+H`(XujM@Tbj?~9~YH-cTD-@r=Z?M+Dt=9l6gUA5GophBG2=J3Y*?UdH`?tzqytL5*)^{>j5^mp;ruJTy~fM~h0!lqM&Scpe?%yvgxwF>&2gr- z0ed!nhrIn-IN!@jzFlbblob!j@^dLo?ud0oP2??mgLV8dQ4 zo7p5RIxfC2p4v_6+OycS-}^A#%Zha~_NhR-+wzZnvVzz>PtCdx49QU%iisl%VS+~B z$^mY1-*sYR&`eYf?kY2|>Sq|aQYr~jPkgmV7k^w^6k6;T%2%j$s?hB|Nw=rAUD;8y_&nNeqW}!R=npKF6<*1t$?Zk^t_n3bIlTuMjn5X5O~|K^ z;z5fgog3g2>~ig(@_Q?#RM}=^)8^?*;$mf)B%Wf<;(9)kN_7IIw-96Xsf%&)E}wkH zlT(}k9SIhfBE_r``3Vuu_Z8gXI$ci@iY{Gn={ zP9bb^$Phyr5U8>v?UmW{quNpf$teyMNq>R3EfxNC+KVlsLs9Xb3qtm?UC4LmyC?kA z%B{5sB!|s>1r~*WuB##8;TPiSVI~HrS3j`viPKKaN=olg8Njx$Iw}R2w*)OF>kON} zkkc?2_ybSGQ$}X3s*n*m7Re7%5Tr|aGP5p|`zBsq)bHD$vkkG=19Sq5g>q2BDoIhE;em?OHzu)T61H5XrxNHvePl1-}?eGms^_jkAkmxRQF%^MbT z{~6wRSo=4;5z|I*j@GpiN&S^pm1hy6L$eJ{#SMCMG^v!7{17s#7$Br~$Nrintk8zX zbN?8%aVAEknKt(T)jby5yyNuvs-qijxoklN{qG3%C$=z|RExlT@SE#otsIqmz+35Y zeLk2}n%zBquSCe6UP=7$MxY+Z~V*QRF&M)*?6+ zmlVR-S|z$aXqg|keq{1;Ur%!UWI0js?#SZ(6@$jtB-0bYJuFm<1Tm3J{twMhl|JTZa{8CWaa7`e1x8G#8dgEF zg)!N5kScKliA1T)wT2LVpqlPCX5=V(9DtUxjEMF z{Vn=2F;B8cWH2I?Q=_0~TD}yqjm5dA#qGEzdSimEMfYl7LJ9TeuYR12c_$qG ze*7OqDnT-Y!tA63Dg@f<$+*jVXBn`xCd}phc+APsDKB2*kMi~9&X(_{4aR(0lQRK- z7h}KF=XDqk8kT42)bcYN{gg@|U$;|dr5z8WRFY!uG9`f`OBp8|6)1LVk7$GRxIMiP z@*BZsGmvVqyP~gAZR&T0A#4>Jy)HGIb&j^J5CX~iTWrlBiI1UP-xpQa#q?afmGy^C zx2d+94{6wR5|sEiCNlQ;)9mw=3CnJx>xKV{hp@ixt|}o;>~deIiF&#CC5QaZ_Z+Yb zZTR!)Z0$MjN~}T*7r9S&+=7%kKUxZ*h?W1fVxGnPjIXZe78UV~72kyHd?|L@BBvj+ zvw!@dq3BD|v>)eBbswT9$7y~VzC-TnzaWquUjK$b`Y;j*1p7ZA5XQ_JNs0e&2t@m= zeQUBdj^8LWN%Q?oV~Q>!u^XkXjN<;z{Yrz;ehx`#W$Fyt-0*M&M%c%{p4zq1;IXN- zxHfVk?yxcC=Adp)Klsc!n#m!lr{vla-Ya?$20Yq+Y16vxQI(~e%*tXwVI2)`I?#9T z5)@oGD!6NY+#LBxMDenolUiXE>OkgzT>bBpR-R+Le$O|=0>r7-=}}EcS`=Hcj{AfE zYk2PZSBfg6M2cdz+DXnStW}UByBbWGy+=j*bktHyA^5JU6^==ZPA^B&PlycAHjN(v z9AjmT*Ui6<#4K=t%0Mex~Tcw8)6r7_l8 zu|NqU69T%YEB2Ej?Rw=YnG5M!FlY;}+u~ zA=>PFuA2fUB{}of>dX^!#hnq?dWj@roO3zK%&eYGw`I8W(qsD@#lj}^CGHd5gccPg z{Fm8McBo#~9Is00laUlaC~6_05K>Sozgg(h-zzsD91&hsu`n_I4A3!{<%za+2SwW{ zA8UX?Z`r0Cr6$J=FBuNM5nT#9o;*pWW-0q^(i+=Rmz6bR$^oApOJhu#AzSVNw%2I0 zq@Me)W7@RE>cAG*KP5RDC7HiV)Rh#RgpW^jeQxZyR!;HbyRVQjrFR z&KghOO*VBF5=SbW2Ck>MRO=F!dmC!drY>$r>QB;fv%*Z{4+f#8hZVG$AGvQ-Mo>QG zDIUcg((pQAcZaqHndXB+cf-3%zs6TKC1@e`ke3C3cKg~4GYwlb1hZSLup|70SHX19 zbj$Ev-mi79c48NRe9>R1K7kQXR6os@UGtz`75Q7?k4^f&3JQ|G?Wr{|DvuqE*X{@X zlQ@ZZltqg8MIZF>$Z0iB<+Z0N-k&Qgtv~m*tAfK}=~U*te)~h+d$27Mz5TSROOfiA zl|)nPZEiGFZz|?Lo+wGEowj73YpXj9?1{Lrc-kX$JlOq=U#c4InR zw@CaBuaL>+ycXi4?UKgDFhpM64E2li_LZdO^9Fid9z}c#)$8K;1Mv^>nMb@&lzGLA z9nGX>h?iT88k_Pe_MnSzh?4P&+g!Vk8r;_19OexS#CzF$$}f z&+s24_jK6U7RXKfwWdXNa5a5d2g;BR?4G&{jcem!4>fU>A?=gYVb|*5CO_#tPtPqy zgO&bK?Y~R(swy;g$bho<(w(t;Z+35`wI?L6Cik+fir$t+lSY6+xYCQ^2Sry2)u%Fw zcj=$BX$T_xx_@`?#Y^<7g>`b3hn$mzF)v(>H3Z++IkH~Ee0rnvQf3llH?@GQ@ij;V z0x(tqWFBtlk3Rj$gqQuv*G>cJ?s+DzWxOgyAUz1Ixzhk}Y0ll3BLx}ka_A6rs+QNw zi>ix@aK)<4ZV9)8st*m`C?b_}Pnn0&hrivRx3#=_kJ;eLb--APJV4>^nziCr-8n>~ zhdnjdVL4bW`corc7~L_ea@$K=P{hIV0FS3;uc}@Oi`IAcIeEP@y&roKe3C-xv8p@N zc>^-f35!bAN1eiqi?F&i4tDw>8xr1~*Xy5IMJ8!$R&#qcyz2`9Hd>Ibe(88bN`fd4 zW(VG*Shu}ZfHNrMg5ozF0P82q-lI#ueLMSl2Yx0qienyga5p{lYjdE-4uSs`KCEK3 z`D$C;4@>5f@4>c!hW5C&`tKz%RZ)8E=E}yx0yH~1fQ_)$Xja|WGqG!4sgv%AX5H7X z=W4n?sm9tCv@mj4RT%Qj-S3NfEU!OAfY&~ORaBZqa zVW4Ml#d2@fq>V3Q{*0AjXMLt)! zuk0UVlD|LfL)@OA{stH2I4>>VTQ02$=RgAj&#f;>^8S1R1{^Jd7> ze>+MmnUZDd^QVlgQc*$L!q6UBSM@DeNclw8^*%I2TiJz%}kV>w#>8kX&yQTZtLqJ=DDz{xO=CQZO_DXU+N@X5T*8_EE~G zeFPtp6=wFg^Etjx0#bEss7EgrHJQ)|M>_J)>-0YO%=AdX%&uBOcv3oQo`IuW8t&3Lue(S3lBE;tl@bYRp8?`7ru$>Nv-&0`8Rlr z*3>XlcSLCx$e3gq5`3z0e_A&vZfO)LmY9)~a7$lDJm)GyQV=zSspDiljJzGyhQoxR z0n?bjk|-eI?yW9!+8G`cKb}6**~WzhdBw{dPXo`dS8eS`iIE<5WWs1!ei#L+^U>ik z?-PkyYlyUhTo0lZ;^|^-Zt3z}maBWt>hhTnYg*^R5D#=Y`IGK1ef7}y z=XB0ZkW;qSP#2_w%qpk2&3SEB^-f6pp=|H=eUl|;6U|hqhG>qfidT&=klX<@m<`FUGbX=)TLkiHG>*>VM>=Yyi=E*t(G$sAeSE`d_s z0F_q*;p z#;|?HcGX19{~xSvzk)LI&Bq@sq2H(Yz8a?GI`fvF-VWSlm)miA(e3vNIYMx^E;dfh zy)UC9Xyw&6K~e!~9HnLE>8d;@SsIM3D+517Wvdyvc!Y-E*zwE(j~x~;)|__DnT}^U zchfqM1kOT36M6YD&yrnI7cFU_Z!?+q@$0TSW525U)%z;rti=~+A}a-HWaH`K+dSYa zOsH@=*3vF*U8-I3X=tfK&l)6fTH~Vr#tzVFCqJ%v!xTEVSC2Rca1fvuK7Ng!ZqJr6 zGv0P3q2V)AkV?MUG<@Ri=SZTGss9(m(E|Ls;H{^l(OG{;yu+^ zjM!#CHal%ER}(j!?IrM3^|luh29o?O^ujUNkIsuK#Rfp8pBzi60Yk)G79cvV1Rupd zZ`MfnW)am?7zq_am+$_T=gVuQoYCCOFK{tv67Dm2*<+yyWb)&f57|}mZH+%7{GP%( zFef$?-w(4^J`5Yqq_E7}1jji`Bt+hOs#cV)8`)MW*6YO|YO7A1i0?fMHYKaH57F^+ zb!ZTLb<<+twob`;6{`3IBadJ3$E6MfMaJR;JN%Gj@0wxQDo&G?e8uCQW&K^PPP`wa z2PAexlVBxfPNMLr$N7d~8c-3VG7J*gMfq^0EzlHEft)@Rk?BOA&8Q1p z*YU^T)jf^GPaW#+dzy>^!QX!e)CH|wW%3n{-|xSO2%?K2?Qhorx&@#5^>a%smj@?j zIR`d!3zz1Sc)qZmARY;r0-Mj}@)X|)OB;gM2>8t%GBcgULzF;dZFwK zc0Ods@shF5&NKmK&~~FFFxK-A%x&<~GC35`J#H7`o44?9IXBa`P>FL58eui+JI2~o zivl;iBTG3~ac%xzFqiVb#@t-(K%XSgS7D2Mq&S9$s<7x6A0v-79OR%O!EpaW>X%l# zC^Ep^ZCV2K#^8N_|I{xOY}ta}?hAq5^&Rdw{0K5*|$&uq88-ZbmdQ0k@; z;43{h@H0=%aYn{ooS$|dyix_K9H33C{VE)xkas8yf5+55L^t@kk2l<72Y=5_qRh4D zsX`0F#b$T==M|t8WI0&FqsG=n#t(v0{lagQk`LjW@luLHjYk|uj2HIy&s|)9VRJRo zNi#I7U2jwE44(W-kb7|_BbC3X)%{)mbWaaEGd09q3G!EoW)%6woqg)A=Z2>YlV{3N z?hs$IL5-rfK&wf7Kv4Zui8Ed3LJs(B93C~#9#1%j4sS&No+sy~C1`g9v zr;7IKKt4vpt8^D-M1d4E5TU)Ib}vX5GrfAppl6iaw9g}CIT%CWgmonD9^wn!D>^_TEU>cQ=J9fqYOXP15TQFl#YK58c zJp!G3Fqa<~$t?SxwL}RD?tsHSzND4eHmgw*H0#N3q>DUFu8hGzX}Wir>JxX3neLx; zjyZ|cS81c9)Kb~*caQ4369|FH-8(|V#zH@KP#<+%!yE^0oqKURRL5BTe7u^&#FG8| z*QsE^%^+7+UE^l_G+3wQxexClPgB{4jZG(GYmnO+3jC8)^@4L^J|EegSmk-eXgMoY z{;Cv|OXTBA-6YX!tcPNDE;Una3A?jjz3taUgx@~7lDFkw?XFaD8lE`iRphd=yBnN7txmUO7~Q&)d#u94zJt6;85@BlUN`33QXvLkDow-2`cmy^Xr$wo6Ae zFE}JSAi+*oLM__6nD6aQI!?N$)@f{cVHZ{C>G`aW6xPr%O94M+R zYBHfwBO{P$__f~dnM%JUbM53}UyXi2z3K!xMLFM?{@r!T!eMiwp<^-<$&Yc4M?8%_M$J|Ojb)(Va z1RLx?O?8&xBDsM$7;q#pg3xUjWfHvif~4ckqouOd$090Z-gD|XuuF0iwr#=&SXBzN zNr)OwNT<354S6ObgMd1SB*AI(v*I7Ega$YZxUP6fZP!fWpq!umqPT=%))o8|M0AK+ z3<}opiA>m7D!Qd}%I%)EsKr2nNrI%bT<_iTz`D2-K7Sr9?jT?E676i!(ViMA)Rb&_ z#l3BDR_BH4!1#zR>0y8mFE8Eg%Gz1}*H)K>*P+?puJ6(2nCFFM0?cMk0hEl}Q~f)B zzR_Ou6W8Gu_ua!)A#@f*-PJ9+B=)o{whEA7n@9r-@LOfuL{`$1Z?nfME=O_Uh6e)9 zNbaGOJ|b^&5%LGTm_AHkRXIL26n+W(q|B%;SyQBs@T^X)E2vJW*}_ipycJwTQLupB z@yOKU)peh?{G1X4+$A)$~z$%>3s+{DbOEd{10iy#URAnJ=%QGhE9+S z-PVTvP8|g@1Me1<@dJyqRwJt%io(+GSiZ8D8MEOfu54|O-xye)G;Bsz_TTgBmObw3 zbW&S-tSu}0r=qx@jdi}OzNE?LK4p68EjP+bj&`KHB6O&j#C+NQA~mY__|mC|&JT1I zmi6aK@wdx9dycV+27jqbmP78ULj!H{e%oE;>UyJg8VSA}&2_Hs|6YmNUYAe3vX$N0 zz8xHE#-HB{6>$X!|3OsoV!C`oP5($=NQZ@UiF?yoro*Ch&8LZYpynD-Fg*3!_Q=?> z;sP~)_ZkO-_bcS*1X5m<`^G>2XXPfJdk)YDD=wEJJO<4=x z*%yPkrNi>p9GFk`EzBmQt4}DW8+i+kwqHfsmU)we!K7g>~0$fZUla zzDsAJmkhL*q}wOm^2+gPhyj0@YRBGr5;v^-5*QFMxLo(mBhw!|_Tkf3ze|R57|5Nt zi`B!UwaRiBr*V6B=!zlTLFRJ+^fl)oYM*+O*KU-xK`Fv^)znWex}JkL^O6>RBh=E| zgC^*u#_E3I#!wEFe{M=&O%_CGIYZ%u5$_*zS9<%+%Z~t9+opWCKn$ZvSFKBuld1r8 zbwAC%XyH> zfzB;54B$C}k-bgJK*-oN%C1v49^#uFC%)>-uxc*uV;EAYG5FV)BeA8`k+37Ezc`;3 zp&FgpBT;BhtyVj225gg;hS!ImgXX4L>er|1YooRjDHY#Xx>-I0o(gdD!dn3yPHPfKDO_oS7sE(L~*aL3q(I0G?FHxI-?4R!IMj( z@&4$Eo!PUos=zJN(1r~ZYF$lz>dEcZ8nuSaPQL(3qmD zdgx=zdh7?4937Jv2HxlRp-*RGP+zYTJL;1Mmjrs0P+l=QecBi-J?e=TE{Q1URhd0R zy^v}7uBOqNhFqwjt9x%_xw#Vup5gukv|90`l?$Yu@i4&NxN2S{%jnBEv@G3OgZmp& zHfV1!!y0RyJ#qXbrVMJcYMXLVYm?#jNlPi`MQK3!r;!}>ElV*hFhafQ4EXJvjhOp8MIazD;!R6j0z8RsPUwQ8{G_I=ADj)$l)K1GH0$0`IYym_}Ib=c!g z+eNCA8`E*?_?3zi_ty}TYY$iLK)OovKtt9HXsx`WI{KH$RFQ8vOL7wXfYhj}UhS8r zh-wQ_?ZfU#HKxwtbXeW#bhKI}cEECk+t)SAuU`I1zo~!v-t-~lPa`1AEsv*1Bq-X4 zJZJc1RBEWju@nK~E?UO7t?d}q>YetAJN2*YXdzSlr16|Vm{@RM4|MOuDaF51``##x zvP)3pq+obd_Bf;U%r|Jno_RV;^@cBUIDCXl{B&V#_oS<5M5#Qmh=~hpL;LA;C1$Am z{y8%03HJ}xbs{u$>sWtiYhu43?Gl^_lI8#QB1g}U2PrIx@qgI5hw%+wvt&pQR`s~i zlycrFTzS%^%n0MVNQzGB_T3b3PD^K(T!z$9cN*Lbv)fjeCpF&R@y~TTJsA7PYU0T6 zIXpU@aHOUV@|sYA6f+IVVW9Q^?C8~+#R;SDt$87o!Xh%HNMHYJjQHNmzYE}n?l5zR z&74N`v5kz^bvq!pg8eDC=bw?@wk31bc7Zt9DOJ9W{&&iEm$u0WVJZ@!FLH5X@19Dp z&HpQ9Ud#MkzCTZZd?}%Cwu>}MZB83x4|uMWY$yU~joLD%M8X#Ha^Vbya3qQ^dwycWz6Gnh( zxt>nfw2XV-wbDf2lUCaysOZ(5Gn*5ZwyqelmoA|`5!n(X64jEJsp{lb-z!)A<>uPb z(gyL*Ey*QuRcYJ|u9K%L;Er}+f9ZIUP&lHBCcUce_3?p0&{@-E{S2cbjb6@_DJKl8$_Od@u|wy{@5nBAWOiR+ z>siJrw=4R!j;Q8sBA)YLtkDc>t2t+BoWrEuMD!p;*_g06_eiB72}e-ZD9vE1U}do;m5Iet6t=+_m!;Ig9c@!VKe!Qh9-&CvnSKGcwuH+ZE?D7);h;4 zYgyUkI;k0$E9FvrZ3>NQ_4cX{TBYHL#s?b2;cF9Bout6zl+40ek{vE;zuXF#D3Vum zs0d(w)(%XVLE7zWL5%^P%aolzd8*HkG2@Mn8qDeNaJq0$NSd|~=>ipEDz+Ym?d6gSl@u+f zdA58jj=M`__YGQcjRNg(<<~|h?4q-|Wu?|DhiLLb9;x;|yZ&eGvumwJ=L+!lZpYja znT4G1Q85J-UBM`9;CE0j_8>@NT8Zm_1%22y*x<4|mJrd{*^P(+?YOu5nswC)kTqs{ zR`qd6pGvGj{>o_N)aP;dmx;lI{4}UcQ1VD1#qwrgJ@>jMpyWp3e!b|;*FPk+1L4zl zo zu%hIvOq2e`o1*I}IXbw#&8_pIgbi(>f-1kw%=n9W|7ys8!KShQ8@lnMvr5Kc)(B8I z(_{tdU2wzx8rW~h&hI<)ehLeB5Sx>)VHO_YmgqK1Iaz6Rkb53=ZiW;`JY2lPt|L^~ zGlO1EV$%5EaaJC!A1nyN{&W59|Z% zgV^O+d0a=Umha#PHnoI1iC)d8%zK^*UZuxrU_xDqmEwKaf}DlZ?$Ofi$k(9BpaB z-2LH4?#_K~d+56MUVp+Kxx>^B*=<v^aV!XO}G)gbcbql)p%cYkHJ%x7$72NKunKP1-YbKnw!-KwFjIbF`?>+PYs^+EW$jSy?-k!z*jiBzK$Bh| z%AZSRr!fSg%{T{Kp_un`cDidP>;~b9RCI0X$-5-0`uMUMIt-RTCtHZk5bpp!n--kG;MkNI1w|B^DVJUL0LB>18tc3y5#U!_k^AzorW`z6d zA5K&yXh0^MmjToVgcm&i^q?M*V;T@yb0#(O`d{L29((AXVcDxy*2D?}TF`)*hYSg~ z5Ul|xgzZ0lcQCS}%`kw`bUu+(OWoT!t#5SM@7+u!uXokVXYw4fuJan^rTNaT1j;olI&G}XM-Xb?#-b>mFgu1&g4Ua(GSNA=E;|yu|RFUSYKeip$ zK^%bmaam)omK>hXnLLm&9u{f6s9II{9Y(;^bf6`ZXBfYayA-}6jf?`NANkvGGl$S_ z24Y1kOObnMIon{RQMz>tXhQ%D2S{m4v8MllU6*w4hSxv^c^dBoA7K6QR*OseJN1T)NWR=zdgbnN8Ad8j_&8a>P?LTz^|7P zBLgHc$+XMy)~VR)F8@H^#gDKPVTUQ6C8&6Eb|)n{35RRz=r{=lI-T%Ak5$qX69TDQ zgRU0?w^M#CQ6}mI*Gu|*{6^1oCf4k4t7<=;{EqK)V+l)7c1uZqk2_b8a|KhgotJ#2;nhQp6tS+BD8zm2%OQ`CIdXqS zyj$3SUcA`HsP2hekeS0j;NBKo1AHq*NsoPjK%!}|cSl08AD6G^ z_fJQC`~2;Qi5}>%S^KYAGa-wvNc~37fb~w%JwYJp9UW%JiUnZ3sEmdEwT+L(l=iT{ z_qxB*1)SUf;5!$bl(8kk2R6oy_av^j7?9L+t80T-UbBja5Sf z708V!Szg{Kx-im1re98q!@+<~^1|A^mI8@yNBB4L?-?z7ZH?Fzt8grg?7dPWH7Pvr zz;lE{p`rY`l?laJkVjAjel5as=f;rjmCTGvWMr_q1{d-pCf839`J0vf`SPV-8ICE^YkK0xe{{ajH6H*fdO4SmZ%#U?X{!MM-xJOq*q%DR zKjZb-%ohM){`L2dE>)PB0{}QGc=AxqFvx0^cCL}lsA=~Isf2Hr<38AH- zZn8P5Vh8@xStjv1j>8^46qeTiB4UuJbJbYjBrz}YX-2Txul>Rd^L;{b^v$=hlS=)4 z-#{LcWxt{i;z7{c7msR3%QDQQsuwA6Hwum%s>q;B_02$imUeZ-Rxc_6npyq4k30KckQ2vzaRbp7|BH-Lic>kYJ$Gg&i$rFL=rt$ zk6ve6z~X1*e$#C9?l9#NvKCmTK*;}QZ2t0`Wt;7Aoss&tD*i4J&0i!UDu6izx_|DE z35CzkvXK-1mzMzG^?xh=*2$EcjQCqAe;0{QAAkP21|$pu>NR%%h$}H}csBQspc3O3 zRR4(hBN`z8&)Wb1_sRZ0EwA%NF#je;Fw*(`;ZQvNZzVHo0Iok|mJ>^VE`$D~YP(7k zU`POZ>`X%-Nv5U3vSFc`&02&oRtd|wXIJ!I*}Fb#Kk}}m%P$Fb$zO=_)li30w_wU$ z_)a0(p+|=52)ajetqOe(q2-&`7lv7-O(OSU8y;>tpc(Y$!RC$}@*c&)y`Y zFFWPj)Z`rIqY??m-@sL2gxh92B$c#+9nTal3Q@#JQ#a$56t?z}GHnD+1fKqYT+rAX zuKUu=Q(2bo*=H&k_U-C#5kv>#l=@kch7h6U7Kh)@NQNBQuUl|bkxnmj1}25Bd?qZ+ z&AUf17un{_J9X7gXWdC|_rD#D|6JnLLg5i*RiRY+w3zfI>Qmgult~AP-Fu5Bglv|z4O#n+jD?BiG{bP5UIhnurFiK11$03v9-EDPA{OU%PwUEb})B;hU zvo(jy^XQ$n+*ZCMMf-}z?<#EcmI?s-L2|Q}{Hv}G)apV1c|7jm)sTJ3^twJ)^~GiY~o_QS_Gww^TpH)s{vX#oQ~ zR!nj`e-lSgxqM0!cCh+R6*4W{G%&I$dOqa+8 zQaaL{xDypB_aYV3vTv#5e`WJI<=oehf79BO9OyB9gTwg9G)=^uUuS7LnVwc!B}S%# z(G3o`!Ph=REwR?Ox1N>aj^>i{7v@;FZ?<%YdZe!%?IJ7!Q1i|Bu5WjFlB$pJ5814uGd|Y(kQ))s@{k5*HiAvE8x6la< zj5&`+BOr^`_@7AE&YoJIS%|N%c*b`vR)(2lmFI?J@F`M%}~ zgO2Xq@iUUfko$wkfp_8fURbZDa^Q!?c>|u*jiawC(HVs=(sePS&nghQGQ}!nMN65m zJioNzmK&{d{V|=pukfA4?eQ{q{PG-v$~CQ~4nFx|4!MG7WGPDNC5!@Ddjnc4Tn6bN za_&^HWdK?P)oPVpimA3Nt->Bcv^U2s<7f_pKSEFZm%ejBgaYm*v_rcqrC@LavFEc_ zb7qj80ri2r`N7Pa#fepMEF_#aL5U-fYb^-R__YNQ|2|VNKhN7e{ln46^Qo^jIKw}L z0!PnCy8OlD;-dcMX?uh6O*U(qS?6u?{l|qMBm6IA#fyq#Z_X1Ezb89&!F)GgF}~>t z$6N8G7lfCv-d_b1u zczzY3bn*J9j+|B!;RWHrHT^2!o9WiHN}tJ(I_nlgTSpbF3kK7T8p=leY~ecr0yT}K zYqi+kquNh7F(9nEU zeQ$N-=4wExuE>J{E${?>cVd$)UPKO#I3A9?>7FzcgvWk*``X|gCo3fbouC+|Y*$d} zD6URwVeZ-Oj-><(K5LV$%M5+B-?*(pv%>Q4gqZ6S+WH*p)g6$5T8Y)-N1wHh z1gKWdODEw$v&gQAcAk_@bI+B{`<84@s9!5aU!tPzH`ywBJ;qAP{Id^OhoSz&vNN&u z)lQ47|EyKu3-MF^q~xRB)FEl#j??T)dx~(M0ZF)fr}QQnF6FTQwWWYxV0W0@QCNp3 z9_HhI8HR6vZ|Az%vg$ffufGNwMCeR7{W^`Nj@yl2le?QbWOv}CBAh&(O+BZMAIkRk zu(GLEj}Nle?7bJ$InJWpXLbHGQ9;D&cDYl2nHBeTBk2?6lpOU_l*0gJEitzGop0xu zXu_!6>vtf#`C+paNJ@0SM-yCsy?(p$9A99t?35hRru_Ruzl}) zmd;-=Htlcme3BymYPeQwtN=^V(T-erT6c#fJ~HfZSzXd~^cpo?Mx$5R|Gnk30Nq z1Qc6O2tGr4eSzzJG=0MdmfSl?5LgLqA5P9w&pu_v_y4Zflu@Q|kOQH+GoWX>_tDWi@VdUuJB9Dw6FcoUN&=yF~h`hm6SHCPR;q z(9&PN_a4e0>Ix{zZRh*UT@&blhUHJe^%zebZ^NQb7KgB;^{5|!lJAlBnv8>2R2B`qm7e$XKj zr!zg5k5@SDM<9jmhk)mb)O8eswG`KidY4o;1MP9=yqDKx7jM?yE}z#)ygSQ(!b%?r zSAKl1X8tv!{Xo7>eO_2hXC$RytXH(kr?bE(Ug)TDY8q^su5ReiyXR`oYikHVK2?Y9pp zwY|yVPNq(NynAY9o2{uOZ(gV?EF|(12{Bi@4u5XZ-u#hXruv$Y0`N)>7)>Ld3Z8$*W- zIo5M1ab5A?4Xw4UiUbrB%#scBQF5U@eR8^J*3LSqfw=S8=iu7%dnk{QII>9G?WDu^ zFj@jmIItoxik6>u&Z0><=b!bu#2O+V;o+7pEgSYkIcyES2^U822Uffg&$JpBQwW?$ zP?sNRfjD-VQX$Z_FHZ9|1<7r#+S65AvI!_~A6i8*zyqi=Ix!YW+^pxY_v0K`{wf<> zU_M$f__M5G@rK;i&u2K{xNs3+P$t_q;zv^E%)K1439sAPNWAD{bFE0b*nr!J33%iN4@w$RxBB1));_#e8k?eW|)m-QC)F$NPa7I>HBSw_0}g|2Q1FOE^GCktXzDUkwiI ze6Tlni>)ieX{APRr&B|=74>#Na)9YTc`ssr3Xvx9Aubc)08!k@v7ETf9Y1!Ya9ypK z4@3>57|r-oK}zDNQR~U{T4KtWQsnfnP~uc^;mpRn{qnFon}L!y;esFKar&>rFD#}y zC3r`DRVVTSzAA|DK~`vl(zs#qj)EgoyY)RGe%d~1zHyLUtqq%0hhOG#48)wjhP zs2BRV%>6@hsrJh)Mh1tg!Sb(r<)p+?Z&?xwfUY{NJmGa&lqWlMB}!F31CM+P;Xl6X zL)~^_ieegRCo>K|t^}SxuuM&J$5;D6oSJ?eo1PgyDn_Q~&&CS1I+I(u8W0shjZb6U zdhX;`zkDU2i%`6_V15{8m!%$ZG$2W%dGDuj^(l*4Qv8J#DFOnbC@hzxOI8@y=k3bl zi5#WaVSpXWc%nTZ#sPCn05-3M<-P}}h@rkn22VwLpG^guMfV>14}sGhK& zXJA_3Uz~pSw>9I&QtYyZc`DsfG99&(eZ>ZsBR$e9vQ*u3MyekfiNw<||hTa4N}#$h@2 z7<*{WrJg@CW*>Y!LZmv^CbxI_#*!<;uDGshfsP`Y`?^aIg!8Tg)B;$M3EF+M9R3q= zUg?NFcvW`xt-Sr}CkeMEOBw7@L>T6VN&r6i)3KY2?7FPbWNW83+^uK>VR!7Mj0oFrcHHkm&L{MBX;9_uNMTw+@wT3o-rfs)(wk+;SDcVL zVa8&CV?AsYacQo5q9z@$W?g)^FV}DFkvD(HP-7$yd#Dp8VH3`K!_ph_RO}RM6buOM zC_LKzE|LNIIE9o8dWMEem9g_Ai7OEH2~OxxlI=cNJk)wrj!h$=@oMEf*&MU*2<)@E z%KBA39dZZ1weWVS`R5&BnL=oeuzlMGS0QG?>FA8)ybInV<55m6+so@Yy8oD!ZO^7; z`X)SM)i$3%)hKzmWJ>0AHx6-Sr(~^EGLN{D~`03x^>Z%vNA2Dh3Z!> z*(7Vy$)9gNqzyST()DvVf($hTPP8R-T0Yu?n6I=~ZBK(GuSq^=3%7?(*>rhBvt)XM zcb9ve3J9OCTAMyr@OY=?7d1GU2+O%?4Q+1wb)k?1r+Nf;7c;AC%{(7_&4D9G-F~cj zUtaoX2Nc*u%O9^NVFftA@jukD_cU}GmR!}~OB>t&dd(+2(9%)C5+v zY5PXbNA_6^=dt_n_73a9kFXxiF5_Uz?5zdi8?dVhX4IxL^CIy>`&E}4S(VhQuU&*n+v(;?ZL9h zrQ!#)VYIS1qUTt0DVLr;DD8l`aj%!OVBJs+5ZL-{g+)mDTV>90&6NhEb)#=f$X&L0zo|fA@ z;uZM%n8UnB8?Pf9KnI^1uQ8DPRRhxydcNT9jL<}1%{MYy6hgTIm&;_8)<~x_ocubv zkAcp;meVAg!}{_-@CMs`Vd?K5mK{{athVbmgpQ|Del%}w$udUxY9|a6XU~EW6_&+& z(xGKNJDccVV)7mppvjbW^Lu}sov{$n2(cDD=G zp`61x7W?Ed1wwn(KD%s5{iD(8>56r7b=CGcwhoiEP{pnOAK*&2RP9oav`ULj&%xBQ zzo6FZsx7*r2a$?}sKE?rC2LH=p(;X>u;nvUxo^L%xSol|Rhb>#4pgpx1Vt{2{HOp| z(e~PLy?f~$vc@BsM_kDIiT7K#sfq1ij|3lSEEo;zXfuMLqWY|EPE+0kQQFfV3qr!D zHm45r+{(!pjB7m*#e_Zjs#mVMFS)hAdlN7BhS@xHM@h$E;FN1LNi$%XsFr$+zojbV z*E~|UAJ*s5J~cvp7X+6@d{Z)@C_t5kg+2FVb*&D9IEGlbwGTWX73WH{_55j7-7U@| zBt(@&kxDGN_7Qi3fwt4@A(SFN+)`jvf0ybBby5BY9_`N|l3jHBJgD-vJ8Kxn@mkmP zP|94K18v`>_}-*vk=EhQT1lnuE!t1|P53j9wGr-_f3^ldL}Vo!Fjv;DXw`1(P6Bw- zEMryirZ1}U5d1G#oQvCk0tkC&;jwk6FLskjSRp61J&jfQ4_;eirE(@W5QVB7PmdQZ zP3r=O-S3h8U3>1)iU8gASV4Q(l5?N&gx`@YE>3pihw|^os=)1*n79AnJODs{IPELr zztC?}-2XWk_%$u$BNCyn=k)2kKQKE(;xPn|K4-}|(4q$MA0V;*3%Z|w_?phIhxoc8 zdP&~vZ@~Yu*nSwRszA^m%>9~;(5Ze*_R`t~hW-}PA5dOGFNL9BTK4T4MgBini;h`q zB_4A#3}TRfIIlSiG4`u_{K)!uTG`~5aH$4f{O>LpYM=TS?Y~y^`D`vp$*(BvpYTC{ zVAJT3oi!am4>LV}^rzB@|5T*tv)P~JdjB<;x#D~tbO7h8O8?gP|4a1XKU@3X2~5bk zW%^l$WN`wsu2-OKrU@J}7Hr*P1;WsYz>5KoB(c~a* z#=36#Vq4dA3zyN6m$jq&74E{0jHpi;V|5kF&BHK_uKWzihg*3eL0KXplZY5qQ`v+} zQA=|oFicJ5l|teb7^JM>W<2P4j3joqD*^Saie8M&DV7`y%?eQ_t|rP;Z+O#v=N6b_ zBR19YgWEqH|#)Z-fOnF zy*#sb%B45?`kjPGgUV#K%bVz`L-DgG(nL%~NbtNkuh>YN{8FO3rr#{D+)c4>686EowVX}tFeNcTsK&CSgg2|HmFR?u3oW=hp>3n2 zq=wSlp8Ksgt}4)`pjTjyVi&sj&n-mL-_-lH;c3KqM6o8LDpmq6GJKEQF|o~+Lr?Jb zz2Uj4-SAU+aJYDTd$cPR0$q6t$!6@^Jf91_2#COr>YiBQur~|x;W-tv>NUp6&K`P# zka%vM*G#yTF-{ZZ5GY5O`2t+1qVHim%_T1-m|1!oM!CuS!POD`{*#DFM!iS$VAwstDYgPh~R=)D6=vWe8`n1_c8D4eG4Bu>OONrx?w_txY zXAD*9mPL=m9j|^hJNvm$3j)5;&#uzXGfTvS2s68Zi9sRWHKeDjEh91mudqx@Gr@bW zv019V7h$xXem44_8``NHdG&de1lFER4GYK|=`U&@4u(5b$MVsvcQ=pEWy^?m@U{W+ zL|bJ_MWsqLjcHsV2>@`;IlQPkJt>+=L}__F250p?XY8^Smsh%3IfXVqL3Y}QLJhNr ze-Zr==zT2~I=q6*I_3Q1DGhMrDe-JrYZc#h>+tZGQeW`}o1u_26Xbk~wxbu2&pvI$ zGTwcVmK9d4Ga=8~q#J`;I46Me+5$otW7lvOwE5+%2NdC3*@EgyUk&Jjy6%j`-OwLh zyEk$H`!Yi$A~+wK{f5pxXPuW8h<$@P?1SQ}XJg{|y_*b|$a*Zr zeyzUSpeDYj zQ4WpaVmmnvUz@f0tWO$>+N_yAZI6;{Abt%uc+N;jO-*9YsQ|8#HX+lWv(3K;VO#i& zWV1Gf&>kyk>OPY$^`LJ&hI#Z#I~@+16Dqz&G^^FuYhE1MmMQwo$1v7SG>mQ45cN9C`3X+&W{e}*JM zj|znB>}_T=2X)`Ov~{I7S%Fk6X{cTw?})n7L@R(YIc2_u&Dsa=LZc;H1bR|Bd+$D( zA|Oxbnj0vyWcx2wWOmCRgVRH=ka^3H745h4)7Q7aFRW+{@U>+qa$8wg=P}2qVZ_+pjyGvhj>;v#reR4EWS| zBKNkr0?fAklG$`vGA>)?CJZrgDeY`8Da3@@>+}bV)bHZc+8uPvi=v6e%Q1l$g)%`T zam&E4+mjC<@wZ1o*pO2I8^8*P<>E*)U!U5@`V*%|jG8DD1x62Zo;0IH)!Si4R`w%5 z4f{??djvu}neUZO$xx@teO}F7eZ^I@$@R%2&AeMJy@m65O(=ij{d6<~bwhCE;rXk} zLgYdP{86}r#^fd;!{8oWa>zXgFucb?S3oS9X$&{@Zb?CJ%mG}_p`-)KtZlRMp$u`fq!A6iO$CvfC7lK zxzl{n*Rs+V)llLlH(biW9m1iLVuzJkov(D=IP*rVbOB$_DYSD?Uh6J%r&3Aix$9w9 zS4k0uODwtX{foVa%FBX%+xoutFHnQ_C;jg?WN%kV1EWl>nB^a*WI;c#X<5VVnR06M zj8$qRE*fqZ?>B|2nca-OGs!0@3Ssja4$lrZ_NgyG+n8mtn%7D+GzE{1bqO2HyV`(d zM*sF=Y+*occ;?n{8(PMEaC3!QC3=`wCa4^|794kdGrV~+@Ma2zW(V}QHa#fshh8!H zT%{DUX6Dlv)>lj1p(Kqhx1$d05Aya(4Lqly`KI?(M1siA}YER*XL@g@; zfbB3Ee}wYOs?hExQN>ko;{6}-zC2yNn1M4Ihn<}-%z!pQXx}L*zL51qpwALkCbM3t zUyo6&9c?b3Zyp0?{rX8k8~sdN(K!HeksM(v2D8^W|8x<}e>!R0n}F5^bF&)s4L9-4 z?Rj~&vv`mV1TLx!7pyW=*6`-$vYgGIiG>?s*g8jjWr*x#pl zDSEKiu#d>L}xL-!_Rss$V2-opg8 z25pXIvTAvubVugD70Ucn%6O)Q#%5pV>W4tNqdKy?G3p5kU7~ABSMVUuo22qH${GT7-ih~=@A#*Sis5UwNJYl5#x7KI zdNzMOErca5q|?l{&rvq<%FYQ2<3$K}=2_zuw`d1<9T6v&-aE`x)JG5gQa z2irV49Lp7%&RF6`*IT=VpoaGwGQ{nhfoXReGx=e}x!o~+$!TuRqqZ$e(yqy!a>kJ{ z+gYe_?dpR8=+`lIp`ht^=Y$?*LyPT(6??%c^;_-l*7~1sCBaW~TUz+YFWq-{vCuJU zy^}qay=qlVo_N>GKt>;Xv-_~{A$nTkxnxEJ9Y&pBIpTwdc3eo~l~cyU&CbL8?Q(nH z=AuuKnU9<1HFvNIdLsEYN`d0#ES(7I)*wc`>8ELbw<>4~k8 z;<4quDSu|hMSm9mhrO!T}|*mJ8mgGsbHfPMWBgdrZJ7Ao z|E*T90(*%P=#hg_0pR)@v=)AV%+ z-j{~fiY&t-Qw*p{PLp2cL0Ea6`^$VnV+K@x2Ya4guSCm3f-b*=2tmv&!7n(48&t-) z&F!{`5+npGW1N-UGfN1ndF_SO?jd5vOW04%Ficowb;T6TPq@hEleJvpRnBKCxNfZW ztQWfDr=5%wyL;0)=yJZp%5pPRjlGp^Auc4^=YT^foAmN{4iHKvH#}b{IXiP1PmR92 zzg#*O7qlTWIMqtLF?%Z7F!*L2*ll>{XB>A%+}TdrQ`6fM4LR>xx$eCK<_M5(p5Boz zAC4lQ1*od@Cp@oPN`4f^o1|p>o*R0Jt7ZbLq;FGj0ELDwhsN}DaG1y^ta>fGwW!C+ zSm37yi<#v)miaDF7CX29>Lwbk4mc-HObK1jwki7J0Asp((k!1>PY?2w_1(wzY>#P{ z6GFBL$y$|NX)YCFj++ZmRfRUQUIS+LQV!AK(KzAqlpa?Mb(0Vxuj%ZUlymxxaeDY$ z=R!-ZN|>OMx~DVl5YoCz?;xdCp!qW@hHA~OzY@lGdSosZ=B-=kJ)PNwnQ}fzN=$1 zoQm*kjc2W0qKGXKmHoZ3mZ5jn?#2T3`P`mxu&QWtzg{)tYR!s1sH{_5nhxMqhwRZf{Qr2QywZs+m_65yhAb zWZeFG#g7!dv|!Am8thAxYq~YnJY>M9X!apmpobzfngj!AL}!l`L9(~NTK&0PbF6e> z{TwH|$ekVshl|sJ6Wl-tfo9@!_oiFXm&%Ln3L@i+0`+_z7hZ54kGG_0$*pTt|o^Mz*d6V;3Apa!H~Euk=r3pSFQ@x2MrOMxSlB~Lzu{}t!>;0pB%>0o!STVH74N8&3Vi6L_hPKM}}fL zkcCyvEXi2tlaV(9lYvd;=!T@Zh8jQ27SGT_mIY(i<_7EM^DN?=%QP)=gV=Ss!i`Fk zWO)s1x1#Ykbb$Ubat`;>it5d-Q!WfUCH1}vyKg06TQ9bLWceYQyYe;!*J^Yvc`+2? zVljREx3O%659?45X_7Aof-JKzHRtpzc%|Eh*(EEqyV&P&CUobACD%hux$7-`#5%AK z7Ad6|Q8gr_;N^pw?17iOS)UPS7Hr)D9r|i&Mn*!<>R#HW?;8dUoC(HKn+m%^d_7(L zN3BiUFC{f&Z9;q9`~@RctcI7TYH(C2pO^e0d3{HnaWR^Ex=cJj{NT|sL* z=&(%IX}kGkiKb0ej9jR_)0uM7mS$Z#nwr)Pv$QnJ96CEP7Cb){bJle^W_%^$DUKpl zi}X`T=#pF6;M6T$ozt`!Ee=CWGDnwBh5@p{M+TrT@$^wUFK&fMI8VHeGDIhvunpzS z#B=4%XH3P-lH(`x;#LdS+V8M$V*669lp#7k2j+jxPZy?)Mz0XQbwQ1rR)@WbZfK21 zLrk#P{BxVhvU~#uikx!_YCjdcyIb|hn01e=A{sMlVS7PJ@e&ajWU(|#o3gvU1Lr-Y zbTzFfVf&W&kjZLfyU%mv73DtuP?=*Kk!F5$Iv2=Yb@j9dT+7-hq2f@uXWQm3?3FyM zfU#tupj3?~CD^0Yt*<;_uiJe$!@)>M(N9pR5|I_;;Ach+0X-6k>7^_F!Yl!jq&86! z0(?VMN~Etqc)UUa1+XL~96@`)UxW-Uj-$cQPTa#d_4JC9_=aldtUCt zhh+Nm++56WQyMd1^C1q@>qm-}4$D-)G6u2n#s&UUHymGNQ;L@80?V_JE<}%BndMdC z&!nbnQ=hXiI#K8)O!7?EvDQK+i@jGuFq+R<;neYy$v!u)*z2q!GnbkzCl_lTko~b% zS0;5KJ5J9-tiZe`MulYtumuiV)yXq9X7Wru?LTv$J^S^F_Z?8b!9B9YgiTnYT?%`d z#~5rU6UYT`dWp%}cW81rUUUlapsm!==-Z>VA?Bq<0on5IgLGj@PZxwWgJ<_ zoZrABxc#1a1F=<$9Ik^0uTN-wa9;d$+e*i&VWK7Q7zFJX`mY1!XhelO>5Dp1_~Tee z$ZDyQIKshzN2_+t_RcX!ePF?|ZK3WP;9XM*i6!)B_XNSPmnN6je{x@ihztmfeo$;= ziPksuydPg0J#P(ICzN`h1E?Miu<+b?QP^eBNtc_x)Rj-P`;Mr9`*X0}zw)Jh~kS!yH>K$vQwyH7v0F|97J;1Ov{ zS-8ICXWoun|835r6RLMYj*n@7mIeTGU9%JcOZ-_eyKq15eQfuZ{LBSKlI-bh`li>% zT>=1dxftx3e;@Cjy9$VKtQGlv@VnpSZ?O1Y)bF^!we|2IJeYC?02uiLBGY4s2xDjy z0Kjm}5YWW6*0f_4+^&GNJ^B-3u<6_7psM@<+`azeWcdGEZ1)eO{=aeP{C}uACEdQY zz-|F!*VYl31NEc>NOb&`hj)AX!|jMHZU!sWy;pk`R#4LYa}-EB&gqD|(PI1b#p6-B z_xqf?pl+Dc`$lTnBT8kY7GqyuQg;$!*(>;nF)F<}UjAMX9pDGA=0C^dBUEUH9`IfS zV%<2{g|;75=buKi$pz+X)eD&}>RwV|JZAl8#DC;KJNk$ykg^Sk+9kg|C7A0LQcM^Y zNIx{RI*WboCZFQ>^fIm>C%pH`Dc^iWG9;&~l|r(Aam|U-rEV7@j?;SP-FESOPVyB{ zRnBY0V(0GdZmJGxi_m*kS$2!7FFG3`4Cf4atSi-<uS=dc#5llTGz zaVU0<`Vr(KcTcG&D|e?F*3lXpJUrc`pIrsYz>!mrRTp;QcLo77EwLN6hy3u(DWHBQ z&n2L80!R6!k3-6ua$@ukSY>@?EkkKHnJ;!d9Tp~WJW^&~AP*-V0UUcwbmZ&(*57AxAa`gjbuezx0Sdeya8^y1Pqwoe! zl^h?ujojDUY@GaP6@KB5FA@0MX<+1peevALPZ+2@ zjRewg&e4x&o8zGI@Hw7O;s$xGdJg`Mrq7T2Gbhr{(cU!e%g0;uxVH+GsO3ZtP@Vph zNinat_jQd4yiDWTS*su&CB;h!THQznHmQ-6#l!e=c`;9o82umTfp#KXN6#Z(IF=*>PU!i^w#%{t-0svUG{T+7cXTg^B)7Q@fIFb%` zayso~Bn}64J)bB741sp-M-JBTrZMZJD<%-0Y&g>Iucq&n4YxLU`@7X2vMBvz$Qrp{ zsk7(^T879v4%LjGqnl_X^xiuse!fxv_#|;%qc3i)C}`Mc>Z;g=h}_Q{<)$d!ejfZ2 zMnEo!@1KPcaaH;Lz3aX2PRv*Loa-7o-ig5%69n*ck(nO1dKP2*%cZ-dz2MA&U)fsDS4zN(j$`XIZU^nF30> zi5|BU#EX4i4XX4VWhnYv#c41ARL}j6DA()O?O=g8E>cj;KS@6kbPpl#?lz(Pxwk6W zPBtBnbNXa9mJR#ViyaW5^cVL(i8nP_%n{R{LIoFd!iCskip`YQ!{u(A&sGV`w>TKG zzr6`T%zu3%_wMg#An@$(IGwW`gfUp7WG8r77!!@fAM0Cx`=9V&E-s7Fzur$ZG)%}J zl__5pJfNKzrqp~q#V8VgTb6&yj2g@T$Go66ta{AkvHkd`oLAPZwDvrN{C80LKl70U zwPrV8=q7}X0RNwMCYQ#{CDn#_XHd7@`QKsere%Hz*wH%nV7*tN=O|1vB76!s?fS2A z%B5is(qQ8zH$Vq1Jl#hN|3*gtPi92@560;@1?+!Kwi+Xsk~$YUlW{;~6==svT#0vU zxT>2vfueknS?PI2rwU3L)0x~MBb+vOa=v?>P|deWV_Ynz*kkEej!iZ2sT7r;;_%qD zQAcqrjNH&Uos3l+atZqUYn1XS@`|eLmEu5QrEl$@;^ej#BGXr=&jjH0p-SWD9&98Y z(Q+dL-wem!*$Te@kPhjz0HicEwkbET#)6W$ufyLOc?uXAziDp7iK0*{nd$rP{3w!F zbWyxG>7HlQeXf=aXJnUscVcwRoh@M8^~z$ z43~y%ZMfoMS*%4ZEiDJhvC%~8BR5Ooi~vV2d_{tndB%YJljFU*l^k_hub*9zQ&HqM6M^Hh+Mq((-$ zmp@ln4)`4$h0Bo@R>F?;2?;#y(|w&_2OefkT_4PWwBl zs9|8Yen2RCNiv4o>UAu!E2XX(rntN(;b7H-AnIrDPg+do|oYLnodXi%-eMF^394O4a^+Yj!q$|D}|QG z>H5N#yppGTfiT8LeVH6Jz#%%|%be{hMZ^oPqRfy&oqBHaaJ6;0zQb1?d8b?HvrMF% zbP@9=5V-uQL5^Ob_+L$1S?`)=lbNMXg_m0lb-!j%a_7oQbj5W;vp$&!9{0{X+%jjR z6aHclEh#V&LM;xM!(3-X74Kgo#Nu(f6!7zs!>qk>f$md@S`y}!Ew2rRp@EJ<5(B>^ z6^2Sj#Td{>m}+6Gw;%he_qmqM zAV(fI^el@tc(Q^y&G!;@I-n+58qH9dtKCnW!c?REzE{_r=P>f!Lwl9=7s)n)N?xht zfWGYCeFL18#=IfS=kW^Wyok{j*^-(zA(PK=?N(mBR|=f6ZzEy2FDrSjQ4GAq&Hy@A zj_CY+Z(t6LIDByv+ke$GO|*P@=_R%srf6Sj2x%iM`?FIN&ocBir(w++=8LZ1e@yEQ zyrLv`bV%Q{>0l_NC45=bQ-{5saI;mXg)6XXRS~&$pm8o7A38=61MgHjb$6B>YI`RP z&U}^4uD}COTC!xNzzp9-(K4ylAr)0@n%v#D(eqo%CCvO`j3ZYnDayW{{mDCdLP~oj z*X4}F1|vW5SUxvZDUqPVI7WM0UFrI!@1JHp(vDlqMI=~cRG_IXl)1hfkbl&xuCe9i z;a`$tN_Ph0GYX=OjyI4DZg;c3(4$N6hBb=Brh9K}cx(~v%;K+_$a>kPkaD2m9y&H9 zojX0oKLg~KUGC(XQ0g=a229;Hx$K?8m*Z z^-r62Q<*r&1$^4XZ3sQQ>@H4HG*T{qCp%DGG2j4gcDJ+^CM(J^Cl*(4KWY|YyEn2a za`yS1T0h|1lg-eTt?ZUp>Vxf%QlYF=>QZ3#eD!uvb-Q5kJVt9GjZ+qP=QC`*dblQP z{;|`+l{DRFVs;1A8QSW>=jCy4I1!3a{o4LfF#p|FV!$|^XH})!ULk5EUe>A)nnm74 z#rL3;#XsQ(aXq&3ca|u&*~~%8d@Pl^M~k}8i#uOEQ!s5KHyFfn> z?;ATC60v1aHc;~cqgrq+Gl>b6x8X4A{`f*!rME3))VpP8c$eV8g7EG9Iv0S{FwZyI z(~06jV%vq`?JdO{`;%X6axSpMdXER_9Et5I>5BYpX_)>te|}BK0BztHdTZm0b9Rz< z>+GADW%j%(slK6zT@dTp=h~N-A8jCz1mzM9$nM%kH6S!{TVx4A%SOQW=_ciR= z*$+nzvNLF3T@<9|w>qdg>jqddl=u&awQkj&^A3o{m3%jOnr z(qdpCEYduftbu=TvXk|UTt9q8C1Y;p#z%6mADMyEYFQ@7TGSxi@5!qR7`M=3^Xu)% z#3_J(bnW6}r^bC=KL1=3#QTxLEBdH>;JfAMghq2WX65HQe0AodU#~~5M{F1Z`|}`~ zIxMy!w}IDpOR{%IZ*3lEMtkFO`d7%;t#B*|Alm6y=G@%c#Ms;Js;%@26B1`-mqnNW z1~ovrq~xg8yR9w0>Xs|+#h08l42`WA_n?asi47JX_SM02);Jl@a@V; z_~1uc~;T$;w8W6HKM+OmSs$YCQvtJk8HpvBY`_UA17`>U=gFFX~(@3Al zrtXK-r@umbtM|g~AG~VtOs$oEcV_$69J@U2%i6(97Gwen|`gbW0AM$C3wf7&a~NcVqiTt+b{2XO+4DG0wcRDrXKNXE9c_X zlU~UX!^c>i$549IAQo-Di0s^G@m_`s4ke7~k#lhPEkh11uZXn+u~nlhrylyZkYqS* zD^cI>p7Uf=QRNbQGaQKJIgW-Tj%OQf^!~)^SPn~xeTLbqF3;>hPK+Wp>Y3bA9m?cI zwPgnbg3sy(G)3YJFQ8;SpZetvIQLgSNLr;KQO9}R)2BDYx zQ^BZ%oxrJLu9qH4a;Cj^6-10Qg5iG=)o^rFr%@ZaT3?@vh(+sDM&yQDO~BRio>?{b zXNICJj)vypL-SGtVktuk&tV5G%@*HEdMtDH0VZ4rBd7I@GLL3%$~B%I?2tsQxUN2V zf|-Gndwm-j(Px<4S(-!}edmUAm1^93SeGw9bP3k(k_rP9jh4g4-oeo`5^^>w)WB02 z_36N$`sBPv8TOc0AKp2PrwE~rst!D=_bYq3&f5alD(IzBz*&pC`iaP~^DDn{jY))e z=x1Jc@xgmwrK=^#?mm+NMpiL$4O#A0z?xKzFK;WWWqn`3eywoR%Uktl6IC$UG z$}=-0bHhES{FT5KwAI+3@)+t{u3vsWbEpD(_q(SzBf6z7dttu}L#>}X$V>^HeUzw? z7QE%-yJoJ`YWkufot+s0GwD;asRP9Y|ZG}=1D@J0p+^X6dtyOz(qE=#ss-nb>9YXDh*g_Eg z>HVAk7tf2|=XtId`D9$lnd|&s=Q_^saU8s{9t$7l)ngR89Qf6=;zm?tysrHk?-yJ< zl&ybnag|Qc%*SSL=Tn|{u-+~wWhBi|;pO|VFtHGAMv5S~*tIO&HjhNP@rS-CB}gCrrwwFpaxeFN3omE;ht!BA#gex+|RXPo&Ox#TAzf;Mcs zvsCEisB4rGm+D;Qp3^vUy%gXYx7g8&zV805Vt%`aR=@itIOm9TzQbyN`Fbq6gN}LL zOD^R}`ztv|7PHO7XHaOZYAN!?7K(3HOY0f(r9%?sZ`j+#Mox2vpcR*8N=)DNy+ZFq zT5h4$UkdtgVYg+*y>B0S z_#vIw?2WmOR&zD!p}lpFqb1yRY?2)6Y;bh%Qq;KhFqI=Yj6KW?d;zPQT;n50{Bq*_ z&o`QV5TP#IZ0;{NAWs>in7`jbakVGdy2E+VDT=h0W8Z>7K2WcR9F=`Vc&7SM3VX&Z zQ?`j~bi3DXcHC7t{f$nC*98_&%wU?;>*7x~Ji%&H*tfSY$7uP8EF%D!;gG9#`6KHy ze@zA-wt2@)S%fXHK-F4t!S3*k-q~C1Aoi&;16G+leN@&2LN1)~>UffMRs>Tqb~m;0 z_Ux)xIL>Z;qbwDdvF3W9{s5?NuXJykPO89Z3Op|j@cLZ-Mb1ET^3bSUNyPX&ZO}}k zi^z9Q=gsAT*DHx)huCK}4w-Dlwv%#~*Iqt>*pA}uo^$XbPTY~j4>skryBqK=f9``P zRLIVYAnx@+U zhN#`;U;X@L%`Q*$S@FP5=A4H|mI+t+lOMXy|FixP{29f)_w_dj7uExROQy?DVxV0$Fv;izbPr@8g1iVxKZ zErz}J1&hA(EEy2 zNi$wnx=7IKK!3gNJ8}WCOqr%lP2YHnT8qL}?d)8l92VyG7`yh zS%(JE83^UUb9!qtDRW-DXkf5@@Ix>|Sl=%0*1_%9tEEWgsN#%BF-x*UFG1Q;dwXX^ zna(k`Yyx%xDWkJIOMH3>Gx3fxA&|nsYkp^H!Tn=f5Aq2QCdUXlmt?q0(<~)8!o2x) z)bgou#5q-470~4h3&Dn0#+@S>VJjLVqmNi+{7sB&PXYr451QS8_KAcC>#ocjMHJvk ztiCc_NuoY8+l&&%3y@>&TQuxo%H$O$lh)xJ$;-sw$6Q+MurEtKq=}MXsi;(T9y}ZRMjgAEQ#3|z3iR5KB>BipQG*6Jx zbtN@ggI)y@81+%V)@tQ|ssbIMyypO~^m4cDoLl1HF*w8Wv^;d()>jELJm)ECZO?5} zRKUXeAx6E_ei7t<#r^8NH<@uWlkeo0cOd7JXQzPB+I#`8LH0ILMyDxiF2qn-j(Hnm z2T(3qAv>xxR7cmP;Q1Sun4xNy)+N*2=mxp%l)^>B;I)iuK?|WV*o2zQgZAtE8Up;6 zA5JQmjfG1Drll&}7HgP*B{rp&t4E|`g;D!(6RY$Z;h=1Hv|HkHndg0fptA%D;7PE+|Eq%9ZBuegB2SjtXSH3vvus* zWelp)HAKO6UJJAjLkmQJKL|h+WB}R08x)`Jvc32qq>{tyE`$kE!zrw1hbBkVs6<)9VMW z_>raVF&bfZbCUXeii^1{g04ER1+@52b?3r!o+S*xH^hTs5WwwpUQ&2BV5PoljIzSJ zm5wndY&)o&iBc&1Zu%4??WY8ywnj^oLv{ps@Noht%lxV~BWJ3F5^Hk_FOJHi?VRNa z`1AdRoln?}nHzWD83*8ir-*C%1Zta{#QiGsb zxQvVyIS$s2D#MBZyxzS9DUPv6b^^D{cVEz=cik`gHH(7>&&Z3H9Cj?K9JMkoT`t+a zfxUZ!HrK>(K6u$h+E7F{*|aza*7kk3`h;$6>j6?E7gx9iDIyPWgF&KpAu!6D{4aL#;`aig&gw79Q{c6?|!H9jueQM`g z1(+)GK{y!b1it*BZs9H6pz1rx;verRT7pj@$-AQh>K#h?Pcn@1pJWPZ&>#m%c@)w)9x4r<7CEvYXC5U^7fRiTG^m4@-3b?gXJ*l zaDuSn8#FL{)4#&VyX_&^)14TFBN<^5HX3kTo-{1?eKDK-sA{qy!|(P?0niy_D&CyE zk{VDjFkD<}&zcG;rT&=`?}*GaJ#x~z3yZ?zrIX5DojqThb4MB) zsv1>90sNo`(v2XK1+j+5%85fWT-C?zbMud(Gzh5W<;LQjkBCVDutbqc(=kb}66M*< zSO;zvVK1R-Rp$nRR2#DN3)xHL1{%)&mGGtM-It6PzQF@y^yp*jiU_$?j4r`R4Kr6% zRKYzbvV%)w5BlC zfy}K+caMJhnX4Bp&<=?E5@KGOPmxQ1C?vdhDc(+b`SJ43`7Lv^D1elnu=;b3E-Y(Y zeVq9N6|-Eorq?tp8 zR(NiiJNR;ZsbA>w;B7Gok9zs`n^O9MGO~5dUxnzbsPvr-B8J{JU0jP*5mk_V`7~?# z=D~H7=>BGm#&QOKvR;fVb8yqk0Fzf9!8XB#lil`ZvxA7MCGpLYdEs`_G4Z2(Bffb= ztV-G;4ijlfMTm_x6>v*@G9z=Mwl)zQHHwg7FARI~QyirnXwGVTIS>dZ=MyRtR6NN|gs?u*mP)zjfo#-@;3hEZO_i_G9KLE*I@S2{X_t zWO`r5T&9GC8flf#L{;LglGBeQ>&S}0_@rrC%pE%mwFVcw{}6-%%Q*zoOV#vl!(r6^ z19+LBPvqj<;yxnB)lQ6hqk%vK(EF-m*3M9Ak*~Cda^P+!kZGi~fN8`qj(P7?gHDW> z7ap?ND2*Uyl}}wBh4GI4aTy?`Ba-g$R zKrsrJvGR1hkFK~{yDv*5El_$H_KKv>?(^95Iyb?k)|n`Mvj8PfX@?Rttpl!XsP-zP zuQ){Z3DsPh!jl_S-~smf3+ZnDD7jucKDU(S3a?saCp;tsI(GNr?F?=?ADnL>-N_?h zjd9}elAU*14QzmZ^iH3<%jEUrO;6REm7DzZpm@WNZi3;eeR7>;`!a!WvW+(UTq6)P zS6||e&2^ju4h7`;1rZONn(c|hOX$6mnTK@uWBkKO^u+_9(^@4^j9oZs98E#U*N&sO z0=la&BO-SsV4WA{GA+B1L-KE}Tm4$r6b_jeMu{bzfV2LQ#L}f@8|($}3(%$pE7oK}jov@Uw7>vBb+u;7`O);44yBh+r(CuSwh${9WRvc@fIZ@X{z=$IH@ zD!?V{6w@a-w*SlK z>Z^E~7+JI3hpTBh>6e(zei3S8$#kx##+ z?<9rgwo;=|_lYQXze*M5YRRU6RX6=7)fTw^hb#Kvzcys$H;|ikq5E7l@uX`vVU-ei z7awx`6{=Nc-`M=gA1btu)O{<-l9t@2K^*x|HXM)JWRw&Y)ol&m%|*$jsn3L#;s~R} z!<*u;7D68=@FV57yHrowP7E1+_o&rGiq>53QDuX~5gmxFJ1?kcS%P)tgnu}L8Uhpl~HV8 z&dd+G&3lELSM^i|zQ=!@=GALUfl$6FV^aSt-xCMzCGQwZU6ZH7_I-1s?y|FIq*c7~!!FJl!AZYcxLI=|P{ z>57=+4~K7J*4Z)n_~C`c3Eidbd;Idp%_m-y+o1gX?F?t}&xqOrId)pcd!d!S>Q zud0g!yeCJp53WF=3i_KzpL)ct&SJ7b?CQGF6dq~`VWm_XTL0JCH!Ip5Z0>^GG#OU;gmT9yeI1_YB&pL3hex$99Iy9|D5pWaJx&_cynbGykm{ed0|MLqXKs(`D>1fsJXR=O_AX>RD|eEwflM<|WS00j%iR z;*!72&-GXr{EC5Vgr)|qe|S?^!$UoQa~3&@j@<%fqW+h_N)=;rwI|Q75(w) zurox%uJeju4WV#6@j3pL8#&FZX>)S+30gOJmhU)5IJWn6TvntqonS~-n1T=Q#=Tld z-<5nhA?jOwoBZBBw&LqyG!!+-)dtrB={O0*u?T=_+H1fK<2 zoW&&cUu&Bk#W7EqOWe_;^BL!mx29wK7-soF<*oc#Oo_MpVN-{xz9)~$+i_l_{s8Vq zEfGZ>eUv^)Oq`-jnT^?E9d8=& zYRRpT;Js;R?&!eU?^t44@75I{B@p?;3E_$YDl$ zKhwD%6NF;P=+uF9dNkvEgV9o{~eBm&L|*=AisZy^BhCfTG{Bb3?BOHRS8o zv029cCl|trpd>1Dh6@ios6%k4UFT=pJ2gO3E)iFgwSE_l%Miy~o$GO1|CJb3~LYR6GTo=nT?${;H9xHY)M zG3Ezp0OO~{aj&b)&mVX|e-!?G%wtGPX6Ki4)GULEyJpExUq`vBzKNsxv2cOes%bK4 zbRma_C%uLou@^86*Q}%;?M$TWEE~k=Z*N-^2c-=GyZK8;z2I$=cXtVz*#f0W&l4Lj zQ0_p-0oH(XzTh2-FoA+&V9?!XwtYk1lXeDiR$0@j%uP0uyz|$jl{p7f8Yf6IIYnvN zFFEpu-cqmlkPlLh7L~8Sr+Dzj_W+^04TUX|{3j6#QD|QGB8uPC#XB~?mCps;#Yx+*iT+8;w>NBj-9 zs*ADAjJyTH>jT39`0B^dp{+fH_Bartva}D9zisJtOf(; zP(a)6JySf33G=cn7!Su_7eyT=Wom#O6{WSNm~4T_h9l@rbK&;kf7}7FY(|s}hL2?` zfoLRvIdG_KXU{qm^^|#i^)#HU*m1#r#I^sz%A#G?$=CSVmBvw=n}t-?Pype8+lps5 zm{(VZCxP(?V<`UV{-!eyc7vx|p;402>#wl67)?p=g_ISvTRa}G94j7_1^ z*k39^Ax`l+u&nazgR0cC%llHws@?mkmFxa}jV*bhLlMVsVF z$CuT}R{=-FAEv}RNO*_4GwAzgo<;GXNfVt;W}UHxBTR4} zCU%*A?0DRc(vpvqKn@lU%`F)E!nFsUo4Vgj`pjwA_~~`+*(6bPLhI({%bp&wtL`@2 z@Ip*(9&aXKjJ!OHaY&`Bo?1w126hU?Dvwdv?phEMaZ`w&6{D+i5az{))aQ$adjYDu zY^vsg4@23@Yk-tcRR6Ovvhker4Nj|I*hmive%PlWsKHxqufVDH6(Mokx3cEY0`GYG zV_+Et7y_>6sEdps)=T3HNNNT^heq1qk03cl$;I>=G99!oxmYimuQqJ=dlux4WA1xs zf~u$$ckyr7>-kb!X}@MiqA`Y9Z(PFPXsZqE%VoQSiQp9ZN<{`TlX_m4fwW~y{Mp7{ zkqyYOoPCabEj}S=g&Sw3IFsoQl;f{R4^%qCkXyr7oy%pFP_@XsC4ce^?R^=Hy4 zPdfs<`U(_W*Q^xXSiAcJ&6$5utxEvP7$V9Wl=G^unGyXu_ANr4&`8)qy+rgOF1PuDR4#zEVx~ zWK_XEkjuKcLbjH{!Y;Xu^M>X*WM?6F@vXt#i@VUYW~@T;w=~J72%xRxe(47|=)g}I zPmTx8T2+H~Dc!`VnGBP0>W$+Nc9HDyWmwv~9wYjVVKEkK-vJK{^OUWp^u9>__hhW) z&jEv{j2m8BiJMZe6rKRdwu47?3l(fcU#g~o^;wE^&6~DAv($z70w=DiPA4c$$_);; zlRzYMymCyiyXcr%HeWq?b~BD1XgDAy7rz=c7UC7Vn(pYKNosHk-%i@^_0W^yh5W>} zaETat3WBi;pacd;dC)VNuY1BIoazgMN1|wU_|eW{0hgFE0NO!6T(Qg>{9_i1K6o9Z z%#wgK|5xW44GJF}2V5OQ1M<59edpEE84PC&<-3W)j3JgO8_^!ndrfVJ7PVd^)YxTd z__1?<&n{i>&dm|BfW&ciYH6?fH&J~W`*u1HxYhD`L$?_zn|FmX0WighS)J)Hc<#M3>IUl&LtP<`@~x`+T*!FOZ4b#OTDi@CXPM`ox0uB zOu1vIreIlggTSlK8@LGN20g~!g=wz$ffRKa`ix9(LprAW!W^xEJYP3{f9*w-f%usU zS1$cEvE2#R9tU@GLAv*-;K?f|re9Fmou+=!Ap1F3T;CFs1h)Z@j5SG3FR{44m~?fQ z?lOu(F37johz7l}ePOpI0atcwUy5p3fi;O`s4?rs0P0#mRt<3PjYCI!8I&+Wcwhuk ziejeGn%;pUCE1g?HGWkGR%8TiKrpX8j{06ZD?>xeP$3cIvKO3{DolUoIau1)<+`Su zV?Cu2jmW2VVs0;v+G@^;#OY?(PPJ|76j$pcWH2;7*tqMK9gTXQB%E^Bx&6aKh>fBE z=z!rP_u(qC>AR3a;|3mVR4XcsN#-X$b|`3~7wA>@6#t1_%}+uV-giaR?N<7ZAv0i2 z9s)_vu^|f0asc9kA)0tc%yhFv@Y^vJA<4SFuV#8foK)s5=qqP_f`c4gET|MO;5$BK(ho z#v?SX2IC%ON$**l0R@*%doDYZ91o9E6ut%;b_;($@&jk=N~f%ttP>GqD95R=t@o<2C{&0+y}E>s5l{LE})rEZNOP@liP6E4SC-}y=R<;x-0%% z4RUE=g&yq5H$~FWaC~6U>$;E5OQjwM(NQ8#R&NLn9fsezxUse*h(!QOW)s#?dmSGe z`XPNxov&{$-;r*48QWy8usIAykPAgrbQ|-)l|8$~Bh?l{?G$o;f6INu#@dMH+QeNi zy?hL-e`aY=(IrSz-=|Qp%MPQOxq+-NmMs`YJ*`lomup`_((^bgoyo9ku}8%s+y?!> zt;!r-b73C=dgUz$8rdhCLE}|b5#~`!r;z2&7Xp(pr1r#RCplSdHZ2>_zX?Ql4}qBT_sq2<j7Feg=34jWJN&bKI>RyV9HDd)cGKw04lzD#5@Ze=PvF z%OK`C)cLnc+8;RQUZJV=>U8R|--}_kk()}*C~T&^I9T`ajZM7Ddz&eY#!3t)KVqV} z%K00jmqm4!ikcDl51W?aD2{*msJu~&{r7V!GmU>^M^yi9wQ^(m-=qb{Z+HH~o1pnG zEakKI@9Lr%=qA#8r<)hk`u8MB%Qg6R$Q;|B zeFJv5H`%k46|62OfjtKXT8mmVcCG9qt!2*)>{@Dd$#uMS5c?nu&Jq2zX~c$TdyM$4 zU?U1?=#NB4CnGul-zbPt-2$ct-*mQtsh;{KMBb|Bv+*Cg6sb*?TO(IKzRr%;Gx%*RgVO;~^FxctfV)xgCe1ec z;8wMJ9Np$c`wBaf=jK@WIS>=j!FbsS*~N%( z+>iX}CmBb)M+r|c;0^JSgP>)m!bK*}KkGa0o6YHv^*e1J&po0WP?b}a13so_7(6=9 z%~`IX39Eg(Za>Phmcj_>we>h#T}ic0Qg`}+Yb(p+<+|4%p8RTxYr3U}Qv7(^E`9z~ z0vo^R+J>X-$<{&7(DJBRif3`)ki_!FyJ~_H4?&Akz7dA%Qnq0tMeXRR1^zcoo=9cm zL;?nX<9%ksklbxavm{p5!OL>{eX-~&!$?ibVjO)q*lYCuKF_ubx!)@Nnh zsr>=ZiJsumuyf$~|`p=&!Jy>Z^yH^LqXFp~&TV}N^73sj6B&(Nh1bRdoCLmaUEX5Qh zCY#ETFi%ir&TAPXo5N(GAW13CwKIz+or-<+dO`Ruy>^^*eUS6fL6wtYie=L8 z6-co)Vy3a^T`n;vTP0ECW0ApwqNS0n%RsZ!qo@Zcy~Hl=b-J5?!p|6Ngt|dWQBNNt zMBO||P;hi?O>N+H1VK;|nGX0R@_I;Sb&UV)PzSGjzs@~n=jo_}veOhdMMR8~W$}|1 zKYyi06U(TJ9U}=WTQj48QS-K{L8W}nBqx2IO!@*ySIx3;RuqB7rv#$I`I(E>XC%gR zT~`Bj6AfRS(ywXlaZ2ZRfo_aOFG}S}c(uga%0hni>w0tEFIoYewR;7u6-d@E+12-* zJ+IMcxaaM=a!O1Duj={yMtIuj^jgXpZg@LW7Q_yT!bQHpQOTnxGRCbxAjv{Zv!_=n zF^^{FV7j(HOqV)o^M1Tv^?}N&886SSa*2Axn%Ge`Zn_S@>;jawRruNu~25fXuak?x^5zC%4GWq1cp0XB78a`LL{xV(~0nb5da zD~|>#d2WbuBQH15hK?aY?Q{J)r^5F*IadZ8{pFIz~iRAMgv`+BR?do zIUU+)(Sk~kW;l~F;ti*fkLsV-WkjN5LD??$p2Haxyb$@zVJeYa+{smGm=m12mwf1b zkVlvwzob5L=Ic>YtM62;=74{)fgB_v@D$JHVX#TdZazF zaBH$a;V>fX6^7@`3+ZdXrByO;ck%__<1a#z=3J7>yK{+_n~wvJVRMg5oAg?|ZB)?c zBnw-Z5V>id93)~6szwzjx7ig1qEah&LK?%&=}l2drIPzZ-HKcyzU3(WOGUUFbi1}f zEf^(Igy?+@#AlL5S_j;_G(W73-;n^qkd-^19e%aqz+J&sIAJ~r z=20Kytzen^W(}9$wKd#WaSC0V&yDv6n_T7k-n7dyv$19aud610|E9D+<#I~h7B<49 zddZM0PeN}XS)eG&Z!-Ocir1S8JAC$M3BHB?yqb0Y@(=iYR7sZXU;g&fUS-OzAGjNr zVaGiPRYlZ|zOH-pPF5S;?zUVT@T*9wG5tBRtFEZp%nHjC>P8RiCy-;?i z6!OfFk7yZ(d#Q}5{zzFa;Qjd#1L(qD0vdghD<7`km?a51^!b#A!M2l3e(bwpik)~w zY#9f3oPWwk%^b1PfMXz)9Py_8#!U%RFuhA(j}&cVQoh2Dx4qqro315k_wVUV(XGgw zcxDu$yUT-}oDwbzO;J*8ZbXk07*wxNTN#iP2xj?@g9RLJ@n)p3{a7kK9_-lJ&pWl; z#p^<^CfZ$+;ZFvw_80-mJmYT{7>15&P>woxp60 z99vx4kJ3dsiGDKDqA^cVX`T!%oUvR`z``m66yM`*CgCh)CbH#PV?Dn-ZQYQTvmthF zibM|7l4_7OVWlsCFgF6WBcja$EvJ^5l$Bwzsf;jgf7@KGkR_n@58MR|0M#$evE9-$ z8DmDO{HNp?r8j!jOBv$mlppk+1C8LHWNh^rWW`0yGsmtzvSNBJCbhm@Vz_sm_l?*0 zU;gK>VTW-<`$@`py?$~5=@I7Ifi!OBy{mK5G~!(xn40c?J*i5c%bu0)*0MsNdZb}$ zy7C=Di@FE{>JjJrmCSWlV^LG^U9UpP(Eg~WGOzNI9;|&{yD*z7W3F;bByIQmk8kc6 zI3t@t_=`x}uUFpvQU3_noxrX4rWGFWizy1biUXa{WPS?9_jj4~YOV@GUC{_A%hy4Kjg3GEo zI+yCzZPMZAX>c-VC2NJ*+_TDdS9)gT?OmvHj{i& zpu!Cd4|=G`kO9jy-o7b|Z`EL2?@h6n)*4W@69&)4z08~9(rfzNm|@yd$`EA|Z52IM z^tEN~>lLXdsNd5SMzE=)?y@eg48Hj!U2g}U_fxVA{wsH?I7a5P(lMb;%`D^v*Ns+!TQN&9KJ+OlDcLhsO0K>zBv+Tiht~FRS~i<{n}!+@&77 z-A!Y6mt{xCGm_+P`Jx&lR@k;Pockl3{q5xaBPnzFY%s0!Wft)F(|A46r6(5vRNt1SNT6_cswTQ;=GO&Am9-ExPL zBazcH^r7IoTKyiu@g59_8SdA?k~lIUSnvw<4LO;Sxp}T`LjbEh9>8w)+cr;aiN&5$ zn?|FQ#>NY>8Ngmn7fqSE%i+7F+^>xb>GFTQwWUq3oK*lF&tDej2tvs5))q%vCSqHQ z2D~8a7D%eNjk;vLnn6VEYtGu@E4f@|{S;lx<;(Ynq0g4bj(iouX+7}K=kv^0nV%H} zwdZvDv0e_(v%-O|6My)X0ObNnhtW`h-;(G|M*AD-SzTYAp^ zte;Y=`bQdyYm;_-8|R9`N$fwka?Ha?zlB#y?yTNqb$CC#dn=)J1IXIiN;j4Rd!ZtBoPCGjWsPw z$rK02hg|*EwD}>cY0h;eea92?+Z|(t#NMRzOZ)XmOtR?vEUN?HwBc+0G4k5+gCbZm zEywOu9>g2(xcobBV=w(Ee`u_+cvzn*D4*XoBP~b`ZfgF8u7*MnAZ_( z7?9~ay;&;bBVOwHn~1>R&&54N2dItPK*JvwUg9S;wnvtcbp|b~a^8Bc!k27*p{MqKWwCCW=oi_%KE=V1UFrl zWpc0Oa66nkS=X-mjnye%mOV&%34SrBk}saxx+OD}=eC$CvOxP(FJurR&yPVcTHDZi zi8%%)pwCWLVuz0K~%?zTHCS5(S?nO!?aW=C` zxg6hy;J&`;7)f*0g>4Ma)34}JhEJ=3DpK~zDa@1S0lUMmkfpU(o^Vh-JAQwvq{niV zqLzlNJQFc|{w<_+dnCwQrP+C}@BL7eK>yf%RIPo5s~5~R-l&$#`KZ!37`az?-XL!l8|m~NFGU1)8ZzaizHCN zeb5FSI=Pk4tL?RG)`XS|1#Drrw3FqPE7}JZwhisv(Nc@G=oa8NZzN^+Z9ou9Kv`K1 z?KyiS<(YO}BVY^#{|!+MTc~{{1_rQ@B#m!`yHsoq~115&cSGlseuV~8lV7EA(eIm))?;mZ~V~}=m z|I(^CmaE~n`X?n)Jd0>TYy|In&0gWX?e4c5#tC{k*Z5F&DIsg2*&Z<+a~AV@OvQOm zO56QZ4}NNK53XeLtnTq-W|}O^jL;5)7rn!27~C)luK=vYUJu6NVOlj5-HRPvg4{n>FD z6z-#ou(nqm=w_b0T*4&KtDCo98pQg-Yrv@k=gLV3PpIaa_|Rm>i<9{gZ}Fq{rTN+i zG|I3{Otb+#0!R~u zmyKIM2HF&wNZ|5cumy&LIguHnbp6i?^l{e%5m~{*$v|;{E^CFK> z&`Bw<>}P|>>jlO>-hbf_O!)zO9jn880>MwytMgy5$xzOx5>{vO&CbgC1LG;Y$xQZF ztqUDF=L_ohDTqB#iG_6A?EP&50^$B!zh!Vhj!6WJTLI^1&CV%N4FfV2hbY=JKPl2) zl;48V{+Ag;_J5GU|5jA_FNf!@H@$CLpt7zre}kBm@Au)x&+R#!9gXjh(?CgG+@ZFo zCo8HL|9^e{%D$E2gfT~okaW2&_it+w&DOrRFlZ*T(!bx)ZM~9>=U7ny{u?|dFk{9M@Lz&te}7cy`L9B=Gmgxul&_@!$48@!};sn%MKp%5-Y3DQKvz95$XBKhuuXoInev_~;naO55yrfPXh$?43!#F6|7;#ZaK z18%cjtNvQ6*`ES!3ReF(4lK{uOs^xZm59 z`8!37YOr;Xnq0pv;Wq1m!wL-tPP?;$W;YPNgKm=&DMAO`Bd+M&oY$D!q>*ZNK))9e z3dT9+vTYVIPTimN>wNtVFWX&ApHwo?`VdVRhmHrNNCtmer?^6|X75wnxrMD5O)Z=G zUXC`~H@G{CO>$}nW{3?zyE8f1fpyo zqx`D)*K%~VlpWmdSW(gEZOO!Ng8@n3MhI_j3|h-Zn!;9W$z+-GcE259{68X39>`|O zHq4TGLZmq0O*d89IL#V+yXT!-R%xTHP*QeI?jL>IkR#y-p=Pv11FT2T~<&@*$%!s z$%9$SzvY3I|GG?ygW*4F`3TlFLp`&XOZcoGyCyb9sD57aONA~vSz}<5@7Z$xvpDgc z5|Yb==<4^+3>uOAEHDSrdk;HYg~A*Y81GIFAYC;S06P>dS+B45&GG38d@=zdh9dA? zudHA7a-N0Jp}Qe>E8V}0W>SoxJ1=EgN}qm-8g3w|@LqKfcXH;k=wTPR7?z?2nwGAf z%h}LiO$GR)p7i-2w0MwcW-on(CSP}d5pi}Nm!$5ufA_LPT$rSG&nHpJ>M-r^0wqD7P+VOrCYf-$z7HHyz zrFeFFy+GZ$2%!bs^CFur+%%lLF_0zrRQ^Q^3nRonVI$I{&Brve%|GnMBd_3cipT!eQ_9Z}lUfiWH?c&E6e{k`KxwHiwcrkYul^E>}sSD!6e z+`F2YPohmC&={`KCc*yZO8Co%6I1u_NeW19pbj&$uZjm;kE4Xz&ERTx8pTO9ra;h_ zM}%>mw#0c1rEfg2fLW4YKfJK3{;yUr~Z9Hda? zDcS+M;tx8Ou8wl@Zk2aJ;{|*;q2i$H(Wchv_F>52H7f0fbr0fp9ST6YoTS5OMA5*qj zCOgvWB>U;XxzAO(LE`{0&kZ{#zPQ7f4H^PJyF(?ZbH@vh?QGNheE8#2D5*W$=dO`m zNzm?nRN1iKi$WLChs6X-RU!X|mU}<+S6hTNgKP3R*1#*M#B5Qy7^9aop zFAE86j=0>c6Lf&&wtNNfO(b3SpZ*zkbVCSA413P=jEPKUz4nj3VxM7(3Uc+E?tUgi zxRbUt6R&sG{osp2weaLzTJ8ulW2QlBfK?8Bw_5e;-3w)XEfg{$ z8pN&SmIr^I(WhEVTRuQyVayuKyF3WLSX)vgln%htj|;+!PFG3^N8 zYMVBGenoJ$R0kfTAFDEpYIxO0t-`gr~YeuE4ZKqhW|ZM~*QKiE8UJ%(6_z zgy#HX%9mZqntpCC*S_<)!TNMiD*^?0_>1epMg74>o@ZYV8bZE0><0~HMsm2}vfQ=*8#k&>~o>8RE2X{3Ek2c)Ar+#mp{D+ zHS5HK#=_{s0`YeO>@{|Bf4rA% zlD@wp!upSBgm@Uy4)#&Y`#@PS=@W2X#HG2(ITOBs>>`H62sJ#;Lrm${0Wd6j`CL#5 z7yjX(lGMIu-;3p(+~b(fHF?v+YFDLPqlKQ8wiKa3`C)1375tu*ayOk_k~j;qJLsf0 zf<`(x?dZ<0c8(L|Xz*OYr3rbaOtaqbr~4njC6@_MlOJ{TYkRO-UK+erh0HdWck=O~ zW4V5v2|wbViSPd`&tmZ*Vf5`2O~Asmv?JcY`BV@MNb84YI&s}D^+{1&;x2?*xrU%0 zk&;TjWqh6Un7`BNW{n0Oeb)Qldd`6J*J7D0?h9KvcW{J&7Ww<2MsPLZX>H4o#&uXE z&Os6fohJ&^-|IwXvjE5c(a@-Sd9Gw*xCUvEpTZ;8Xh?Bjj}?$=gTr0f-Nnxy*j4Fj z$4)8S)GZ|?@xy*?iTUmh?-&Lnk&Lq45Jggkezts2y|X5haDoqIw3hI2mSPArIrz(P zXcirq6UpoI%#aJV8>`qxhznlLJGN84@OEhK#pA0|<0x0^vSr0QKzN@j=i<98K2e7( z>xassf{lQzSDB+hzk-Qx)oyaq~=* zf7KTo1nrX;~6;da57*^jjz`%83awdVLF(vi}R z9V!F=c@E*fb5Bg_|M@P}|Gf}11-1PDSbOiNroMJ-6afWADWW1GO$00y0i}l`O?t7= ziz2;A2_;l1BGRRIq=nu|qyz+n(4_Z3=p}>@N(fQ{H@@ocJ?DPsJLB9t&iYHk9?4qk z*?X_~%%{x%-0SkMJNSS5Y8-;(Xzq+K_p9Ud?@{~93yo$H>2LnqaXNed$bp0qVKcIf zJ>9}?{=1H8{Qua^YS%T5sjtqtGk(XIU7Y_ZxQQt0h&TNE-H$6wiL#2n1SBm7iJF=w z-s&bqWyrm=tg{mwpcn#TT?uEVZp6~<_ZqZ+%fJSM{TO95i9(SJ{Nf?uBV>S+UsWH!MnJ=n zF^RD%#gO`d>%M<-bI0XYh93uiUE;zL(|;*KvQTSpr+*`Mcm3DMRwf7sl;@MJk`Mg7 zuxyx^$CAqQ=-;}L)(L*BG6(E2f3twB#{(|^)7y2hw596ouR%!uzuD=3qs@2KLGgr(*DF>XL>AV~upUD3ivya~>D{rxac0$2a6_|dc2HckbK0ci8joF@E6$v+D> z`_&sEhsa6%!&1=7c&vY`*FgNeWk}AssZtEI|J^8JS}=9s+Mllt_}jRe_-qG5y3Mlv$H(L!vy*uKZ)}Dv39LhyH;wB#m^VIwSl?FeK+_XwGx8Qy zftf2a>-1{0ulC0`n9gOkH1XE@jl-x| z0~fFeH|{N`#ep}dof|(?f6jvLE31r~HmHqhiL5BQFiN6abVl@!vhV~_5OYS$TbSDE zu;5{JafwBr_!Gd`MDJ()g(KyHcLIAwh+5+Da(HI8<9BkYmb=Dr6)`Kli75-cabr$Bf$Lb&b{~)ao%T(p>yt8W@T5Rg zvJGrs!o}}qX>C?-1bTCAqWiM5)P()K35D?wj~mSF(Y(q9XOyHhwvWTxu<6`}<@cyP zH|EzF{QWa$%+^%T6cdeZl3VUq7q`HRlPnuXsC6pbwojhw?~Z?+>GNo4URL9#tOUbL z);buz3GWR)>Fo1ZiNEVka0u(MPnmgSuG(~pTPqTrV32GmFU%B@O0I}o?!_PUwv4_e zog5OJwE2$4Tzcug0_zE7mPYurY*eRW{ZV)KAKc5GX#kJ6&N923^nph9y$p6?-$BEz zd!xeH==_>1*T#S4a}h^&BxN@AhpeFf0X6nW$1&S!Ck`n*SpeA&D6>64>zcEfTe6zufh1$&sl>3ws`RPGB#Mz!e&8ekm>oVIT-wzQw73~Au5+h9b za*ns9rAhCEsT{nuwzh^Dr~`KxiFU-@#5Qn$t)v_O;>kk&8O1dS1E=48-sV+>{yq~M z3*tA@MCi`)8gGc0_Rn6pRru${EfY>Fp1@Ca*=bx z_Lc+0$lF)cNuekkP#@S4H)01M$4vyKK!XnU4~bVe|DS z{vc+H3DZ9Ekss$TG2esOIM|D)J_xiBRgQXg{*w}H?QZP8;j)5ZsM{gu>TCH>R8@;o z1S*EGsS0zHKo+0&@^Tn{h>QNnPkYmy>5(k{!p0wJckmAJY45==@;zJm%sT1_ixOE2 z0g;7dIU(*CZuSPNRSI{-VpdD|sBXbgvVm@_t0K)OXgcn*dKReinR7QUJU z1GNjkOZ-D4f9-ev2Judk&+K$UpK~pkxy0JFDnv#sFfSU7uOF+uldO+0b<2!0Z1MYj zvyqe$--@&Lm*`OqhOa=w?(+AQhA)AHMQnRn!{ws9veMX`O|;z6i7(zscEvKU)yU0<(KQ{hOB#H(Zy?r8!lG%#%v0 z;yE-<_r7P3G(^Ni;S|Vtv2T=2SC*wUFK>a$o=g9pO27^F03=o53qG_(_jN~2`!wX@ zfuRn-rgymP@g2(ARSIucDG{&l?U^VdLF2|(h4FWLT2~;_z}*7%`)QqlVb*saPGcVo4rrE#|%us31am1V;DtXD@JDVXf`sDQ%#5lxDl zbV~RwqJ}2p1&^D{Ql5~!e>UeS2~0P*$mp{oVXjq9d^WQ-Ez!4I@`V71)?Bp^mF4nT zIblI+@Nrjf85v?{&3VwJv9w8GU6xsrd_*XUc)VN{A+n0Z^zx3U+_Z#J9@wH3qr_6) zi%fKkI{fpxx;?96YCpyy<(muc-SZ#)C4NLoWX@UV-O!?<$XIAT4Z0`R{Z5=c?4fBRFx?`p}{g0MdF^!Q^wf3n#I|pmsSMUq5e0%yy z8cd4?dxNsZ%#WdNJ<;Etf(AHu(vqnIvs&`xn(cK!@q=RZ^oLzQ5bc@=Y5-^(=?1#S z%oasjYyP$~f|_{Lh4<-Q16yBQy-!^lo#C9(cfW}(J1DO^G@t;?oM@~kwV2Ww&&yDZ zy_Q1|27K@T-2~H@uc9Va8`35N#UpD)+yR zLavWp2Z3d)Gpw zCw++sgqJLP4-$Ff9)b}qOu{CsJVPU4p{wwzY?NuQDe}fG|L({#c%Q{y2av1^y$k2EOxX8Zy|5SI)pkUma=r74)0vBvRhR{z<%b{(rm+ zvdiai?o%ES>9h8lZ?cv;dkcNN-_8z)RChLWs!H=4rzwa>+vyInki~9|wVVVBR7Vq- zeosT@s@Pz{t5una`$k+eru=9~WNZ~jW4_(=8s|oKiTAO|BSlF0J8#U!w(mH)3nHPgcI;;(rh$fc4N41_<|Qb1Om-r_TdGTAN6s@0>U$ z97ZOTwz#D{2-`_pp+8PDOHl;aCC#ev1wC*Dmqw00Zpbp?=jhBw>+5`t!Zq37WFDQ$ zX^1lcw;rJ$_wu=mTS~1+p+3i@A71H^o3ZFB1;1bfZ5TTeeyuw|_7kz}Wl&=MJ(8-F5u;CoGG(b01XX;JxhHAm|4^o_+kE}OD)3J`~GnoP7z*_E40Bh$i) zQrB0TAz8g~;h42>iww~$JK^NApz>9T38&B6OX>5FtK;*TN8{~c4(H}>%mhK!5 zQg6KFTMZ|_wnyqa!W|jw>*iz#7TMBa^32lXCiLRs)HFA@BD;|j<)~Z<^#M~6vj(@V z$0@d;qo^b?`1vgO`ynF#h5}7hVQ-2}_4%J}}^O+V=^c88W4 zVlc{u?@~lk)WU|e$j=Ht*Z$hTdB@~V%aEETnzC5+ijSLn(ZbnSRbZTu2{Vo+aIGkF zma~&ptf#akE#1t8Hklff9hx zn8fpPp4*{=hZL!8oFCXfdWhb43PcHC52GIDpV&j?8QiDmk%QVlE1Sq?8UsyxB;U41 z4@Ox{++BOU=RRWYAMI(;C~0N70c^BLJK~2%*fTmJlU4NJLeJ5^b;|yGzVj0CTA4k+ zW4GonG+q9!06VD!Zh5wHik~`yH)%O#Z|^OMzYX?HiMdF0dG0H!Rf^tL3G8$R#jwqW z9!92#b8$QyxEg85-=ZOw>n+iEwbiJ_7Z(yV6MlQ;(uC=?8PgjpOfTg~@_}sKlfZhAXvA;mp1X^(2e>iXO1>qQAh8=t5!7 z%RWf@BKV5p1%_Bq*`Vws(8Mto2jQ9hq$AWxh~Plgazfyh@1ky8ik>QeOVUz)-gafDL#+S{WyI zJ(ohjGM!y_eACOZ;7kkp|Hjx&eJL!sY-*N36eHZZW@Xb^4#WD1h_b~{irlnis~$RF zVyZ^-e+#mckPJq($y)_ST6@)ftjUplE_a(vd5rw68kd9^sBpW8dC2kh3vg4<)`drX zWO6{2+iQzK%u3J)cE-X}UX(qJEkx+8?m)K0e;6DwD>8FoW8yDreEFwa)a(s!nn<$J z|ME;`67oG2Op)|~`5>euhb^R-ibg(sMM=78=~DdQ&DiDGLCm{WwG7ERWoRNa!N-F| z0GD9E1STs53zj}Oj{*jWUK+%AHE^58S1sY=fD`ch2E%(_mx$2-jji+N)G3tXXwIo? zO7JvR`g|9POgIyL?#o@THiRv;HrcFp1Wtq)L$)PWp*rqg?$n*CT}rFu8Sc%OHoWij zRmPXt4^~LMYVWp>|7cD3t_r_g_36E${c*xFh*5yA&i;JgPuC6m&oz>kr<~d%zDwPS zUzp*-2QW9r0<^Q&@585cHO%yMxcREsCIDMiK|)fd_!o8)gSbAiy_BQ>I@XVF5(&OR ziDwI>%PCR@iq}?h{y4>tvWZfHE^4f>EPGI`Xwe^HEZ}`OiFk}TYDTo;4`m$C8u|}$ z_`hCC`9Fc@xcHW{;}$4!iN8k><2hWs^HS&3e+J)-BL&@a1Y&5q<(~_$6ED=xRqPIY zSP|cj++!g{{rS_QkAD@+c>iPluJ`HDL`&-Hwf>CVX}gNA)3d#-GoD6wj9?Z9 zIXnJ`-Ys12%8iCp$g>zXjuXLS?X6$5EWDlMh)DQ42Ci`hl$&6WdQH`6GyZJdIu*6V zBIU8lau}SY=C`r_JGj32Iy8#W+%!xQ<_9A>6u-3c{%h=B$sHm7)9hH~?TFC8zkn+d zJwN*|>34iEv>D@oPw2w>A4m{estwwb4?dq^1BuGFx|05;kh5ui!s$3RAI6?dH;nyR zRt5Eqefc5qhrdli7tf7A3wpTC?PpE;U_;ZDv@{fX+E*<+O1 z6EbYS;|93%XO#m|#0iv0xc`~zZ#I8$fY#|;0CGnJ@Bhaw+j?fNAXaB9iP`-;bLi!9dUSO1 z$~=lGKKEH)E zuWbPEUWj5hpW-Y2`@X+>cP^e82awHf-u&KofwSoxVOz*!o8q(`^n9hUKApZaXU?kN zWu*@}_mL1@g1|_OdHR#XsZPxs%(BvutfBDtJG9U-R5kLKtLat2#Vdd#P5io_qez8jcOyv^h4Ch@a7 z2s0OHJLU;e^-r6VQo|VXHz+T!qPcow@ns*6Hq#Ce6H!VnjVjA7jWpx!E6jq}hp%Y~ zkuK8{QAu706}H&(gC3f~Bbg(bc)FnRaQ<%b{kh2~@%MY*a_4+ymW5sejw@$)nDR%i z#Go4*j~x1|z-z?!XEyCQTp~!B#i&dpctGlbw^^j01=2h-p!Io+fNb$V1K=sOPnsHI zokBC}$};NP$F6vy(@bFUO*2U1Zob3X0%za>DHF8;!uurf4tVUKQ1$#n!%^Y01^ahT zvnq?}SG*mMw0kuofb-661m;~cuK9A!w)TT`7PV%t-SgT$KqdnDafV-{1I@Y!3(P8a z#&uD2;JgzsVw<;a4p7D68cXQXEFhi0N~t)qtgu+I@UTjIo2GbhzgK&_)uq`057|iG zuBDDuN8}@#IgXhh_O`f}x_%Kp>Bla&`b~FVAPGPc`}QZe(Tu4d_r$kvhE-zDH_R7h zbHDKHHKJ8hZBAO}yiVW)N|=mN@p!iE73hwlc?D&CKa7iWjvRAPX7_}h#b44nPmz64 z>c{21m`lcyi{#4Kox@+zwVefl%nO|AE~B;Y+_c*>Pz)M*K8Sm9`YF}Qs{3FALVhnz5Tsr>w4uK|sTCdi(ZNyTTcjeKeq^qd8VjeE7lZ{YYMI3n@+ z)|ooQYBPMGYc@^%?NhjWJ-j(T1mN!)WST<1Vo45oxn{hDI$V9U_t>`9=(!kMQaQqX zwiuZD?40?8^CIjLl+uciWfDZ|%bl)~L!A$ec4OWj3)=L7?{iEcH`oEDt0Ew;Tg7ra zwQ;>!Gkzx{&j!M+By@A;rdv2rgeXR6Di%EB0_lx?aNO6>9q(O&$TvG7)-2YPDeYm@ zBz`UD$q1iKf?pOiDh~MFC(%T+88V=?)VHZS41Cf^?Ss&Iwd1LR&PnNvele?4fi=DE zv!wyv6~DNay9gOwZ*cFOiOBsLN%`GB)Zqi;=33anRrF)AOq?*?6-Tf938ns@Z8ny8 z2a_VfXKDipEi($hdNmtEEiH=SmLGghkft0Nv%J&YHNJ`>q0AK#Y$$M>(QEtEcV+c? zHNzKHLizM7d5ni9qvc%DntEJkx1>mnO>cQE%Op0WH{QX#)Y>@jOe5iSj!B}bJx9Zd z+A-$-gj!=}YI(&bv68pN&v$p+s~9l!04mlEx()df-LM2*(G^#x*-N7mNi|pV4Sfm{ z?|@&?_n%&rIJkGCbij6xisYM_JP9m>#aAth(#pG_@3_b5Z6M5|-0(n9fI?AI1+3nh zJ=eHaKp{vGz1O@r6XoF*Di(HyI@!%>WZvUWgUH5Z0++!(dY{MVUYv!+H63S-h7Bmz zi#b0kp+*9uS4N@|G>$dYScA`=ZAl4tITwafXpXk#{mg#J`phF05sW!HGi6Qemk0`w zQHxxw^8eJivjG(U+4w+wVHlRF2BUj2XDQij((4Snj1-~3njSh?r8Yznpu|pIVd{Rf zJe6~;f)?>z4ZJ%SUwZ~%FC_o)PNBN~ss8+-=kp5%aA>Og9qBv+iVSTpA3aBw{ZBq8 zhWi>9g1q{W+S%S2Vc<}wgG60<#VdQIOM^@nZB@1szFYaAy#;en@z%o(rz(H%^RU>J zZz8WWDPnb@W8%Ksl#Njiv%}mDbDNd>uUf!8CwIW5xNcprclf)-S3rfVkK%`b{lVFt zJhf-D_*3xpXi?}LG)B&pyc(gT4|unVXmZl<)9n_wN(CV&iVl0oxl_yJY^NEzFX9hC zEMx8&QYcL|4kUNHqqNw`G4qo-k7D;e4UT)#b#wLa8hwO8nojVoFc+s<)hXYJlCxeI z+j76hwjTn)sgys*J?^Xr$(+}`p6P%c!gj)`WSL)}zg@9?IrBjUo_l>vVb*ZyL<1%@ zR1jG0Q88-B-Az!7{V-979LH4$_v(%-<`n@Apk=P-i#pr9F7^zKVz(Vx5f?gaBwy4Y z>2QngX{R1fMhGQ{Z&Cthw81RG+DRpK=ijR_Ywh{f2pPl5II*L{4zag%^{}{i)9xu| zd`%r~*}&SC`-Z1ED`RlyVnY4#MGs~Qxbx7?$0>?48kE|^61X8yqWvn&n<^P&lF&=H zC3O%vT!a_tMllE>!fmNK#Zj-h?2D>ebS^=2uY&wpXXqGC{nFk1IVnlF52yD7eVblY zWjsvKTbuHA8F5H?P~p_DG_eAnj_T)KT0HQ9pJORt9&(5T0R~D(mfbd(MJ7khI6Vqd zmO{0i;x2MJ2$7Ib5~09#U?tdj2s2*T^_^SK6t_HuV!>LN__xhdxor8iFMYA)rz_@a8$D9o}+%M-gWMxP~aJwWS3&dqN5 zjEwI2MNV5{m8Dy$Y;15{3-Hn|lI*xx9V~mR0d|jxftG`QL0N$G_pSBU64a-q6EaR69fhWjvm*r47 zx1Nt#ZV+I0y2?;@jwB$Q2;xpqVRK&89|3sp+=!QFhb**9_`Z(Jv6OY*94Ew&Op>pC zA+hV{;$!zn{;A6})QE1PyL zTj@c&e;X&8>j->zc$PhaD2&sHKgdECmB0UvhTzB1bz?(RsV=+qRJtdw}QghY-4uP28vH2BU#wt2cK^cil5knUzBg zAIFtc62hw#{Tg2>wFB!l46{AaGIy92DzhlTcGGtTq@Xo&2b(lgjg4ycuh36SAx&H# z(d5xV_$NSTv8t%iBM2;Smd%}|MsviWxR|y6_@vmuU<bnR5C=YX+9p!z){j7m~vq$~}cuN!=%T%1CgHy%yA4m$=vgNC=xW2iCztB&g{=&<3 z*ZV~=qRvee>M9Y0h^u@t82-T7?klW}v0Ql5C+HPkCGmBh{w*{Ok}Vn@)05QOb}oY7X*$5{`Cx{6le zQan_6pqYtMc#nQo|AtYYAk24PaAKO3T7Gj0f&B%-191Xj@xe!J3FWy3^zeCmqN8Yi z;T-POBQH8t(t>sHrt*sr#Gra=@UcG!l?sR_ExQCOn{Y9cMW!G##NJHU*}E8n8mDBM z>_;}tre-=;&EL@VMHKbJZsuivbTS@?K064Z(tfRZnATFF5kp0KBzu%^-;ev~`_0^h zr1?gg0UPD!n$Z3rN;azkG+Umh6EHW?D6k&JR8^03YVNO(oW!}?$)BDNnRh<{ysP3G zX@8`5>IUP7aTIkNXSB}py-X@x_OI2gBA9M8OnyvYYRQO?QAk?n{6w}o=jDe3=$yYekMdeB)l&a0v zI;ei2zP7<;?{$>2y8;~UTLA{YTYZ>uEJaOD0q|}&S>@7gyp=?KBgrJ96jTN9@Gi=o z!rb$w(A_@1di`K0AAae_M$mDRQDp7dt5H`IQ$0dY;`|#VOCy?&B;YiX1T>;U!?>Z~ z&2zq$jrP2-JxT4bLp4|Q+Km+5L34eo-FYzU16%|TrT4=rpAsVQ!WUFSKPXj0olb2! z$TS)*m+U%VWGm3PcKaY_2!1i<;DnD&ppj2nslnEVVb0OF*n7zy zEw?U>q>+nBtJBot3umUw>+p^;0w4KRDK7o!HyT<{q$ zLc9zDHTwmH!?Y^B*Z8-hVyIL^Z3*oT!Yf1AE(Eky>btRK;?ozKE-zRkV z+QA9m2wj`KZryWd^7s7@#D;+zsq2TF4St0dU7PJ3rm3Y}BR$aE+pC_s2ekvI z%`Z?=&5Kn}M~>!CIW_eS%h0MD`r->7ZA z76WV$@)VV-n@rXTl?QoVPYGd54=%_!X!KW`lx-}#)^G9VYf$GP{La>hXnTZ1Ub<^d zHsdN+4|iGCX@$35M;}=wPPu>?uHf=i_~lc#oRXHM%uUM_rAhA#_jn_Bczxtay<3!_0HD8##@S`u zQ1pdiA{*S|VCsyQ{%VUq(Z}Awe3c#ex@1x6ZCB62atvO_?|^`fmzd`#_T5#9_pD@_ z@1xuYCEI~=IIjmCp~W>v=dxro#)68EQxJ`IqhyUR&Ys1#x$bykcJcF!Z+@>>KCW_p zjRe$3o~MDj_Gp(u%jU3Xz zoM@*lh!soxdOQ{?U#<)`DXPBmk}>*KKHrB{H3Bn}p3Q`Kkr2`=sl=hpk6Vo~-rmmJ z&UA-J{v5JZCOupE68b1Re_{qe;mhDQ4>4>yS%{>$)D|*O(os#1~ukqMYT;wqy;d~%&oABXyXlc0|75c zyLlyU9^xyT4ciqWy9R@~W$}h#R$8NM(flu(nOC$|&Yr-K>4f<5krdy$PzMgGgPoVt z^*7dCkCP-V9O{sk8m>@^{fsrY)Wn|>$hGuIc57CLf#X}i?Xx)~BKTEo?b+M}tdUvE zh}=T-WSR0a_xw}03qs*C{XGGcve)j^TnfDWbx6P4pY0vnBbqj{4aJ+JpT{1(k7Z|j zYNU|K!?k!%pHh}Qi1q?!cxwPMoOR}`VI#k>xPi!<Je@E-Vkiu9YB^N&a{Ea0 zX~dOGu)9AypJLPWLQkSz0O(l_oI6xX*xnc$F6)JT)|~8K#3jKTb8_n*Q!yzr=*eKd zu!mW1!Svp)B(;wGmC-BuPK8w12Q17Gx33=iRfZOvN=VD%+e* zJZF13l-PMMn}h}jV>=~*waW<|(_J8O%K%1MY+b=(9ttWh%i8*tO`KbgX{~m+es;q| zGxqYdHw8wA^THO4RL1zSf?@U=cUKrMuhQGnrotN!!12tCpY3iobB-EOg%(~s=YH0j z%XuUfCQaOmN0Z;km;=rQ12jCp+AcEY!9o(jALdjUE(>gCScb$ZOOlyd$thjtI37pq z?pZ)pjd!Op^vB~j8dt=Q6{PYIRf4vKdTJfNluDmctqE{_DYWi{#F^{$ zpY?gKI_tf6}&4Ky#DRG=*`~UvIqp)?_?URZY9KRO#oZnvL>G`9MV|a_s z-npSg0@h~rBl4!1?*fA@44`Z%tvRR-yAtsF4umgWdjfbNiP_)?#r|X11QFCwPKN&; z`lP{ya6KdQ_`PGsRWwud`nMC|x*tbU3e7dvpQTKhMhuCQ4CoIa^2{W#b1dt1HOf%_ zv!_rI+=H7B^nAp(M{dCwS`Nv62HZnGjP@AqXX|01Hu954gQ#vwzO8o=RM?$UtL%P zdxu90)9(KX*FOrDk*P>Ih&I(qyEVG;-YWhofd%zs&{x=Ir659u#Ni@#h)dJEj0jvDM6Xd6G+d}1;IJbGWqI;@J| zwc?DCzHUJ$Q)}+%Euf4^08incLV4APOuoMJD{^g^)^^zlRGag||EnPZaql;!*|H~k zsLRSR!?-c5wUN^~x$m7!_elw!e)3MVx1|*sl%OaLtuAxz&+Cid* zeKpy`+QuernjVg8kiGbFM3>(pbg^boXd-)sg2s+MUQf@V?6LI|j(}=U3rN^wV}FJC zqynw3fEb6Mr$}jkrQTd_n;TC}E_(9gZ%w^km@L?MX8<@3vX2XVJN-c*6$Xm2UQr*8 z<&bdSx;lGIxXK%~5lOv~j=!o{O`P|y@qvnR9oWWFTtPG3;Z#SN52$l=DvZ_@;xP!b z*%F4w35pF+QN?-&7GZkG>G;3laF`6;tb^1pyE&yuxteDUq7~h zqK<6yP0pbQwMrI*(6XETV8XiQpzKVX7Ssv<4!$ivtKQ-8jPpiT+&Z_D4BJn zaY-FxMR)l8)9zKL0poku?nHXJ^BY_~a4<|NyZ4+sFNn|F%8+eYb=+~EItUW9eAAT` z{^8_k0b$)mU5(Pfv^zL??Tl~^EUbl9e~E!NXRVg{e!#v4D;T%eVtDJNwHP+{uSiJm z`|(HD?DraRr}5i5XEj!;38A*9FQ38g-G3Ne%QNe3i26M6Q_INk#8T4+>4Ye>NVD{v zZB9gDP3F`aU70KTK6f-|j$de-P#s>F^a=J2p4L)y6!FRx=b09QtexDPEpD|lj54{m z(zPU=vn8T(`Ur&mm=w-+{ERZI~3qdp8^h)=Xz^Gh*YyV+#5*9%K- zYM37r;xf-g3MiY@V^0BN(OSgj8NbHOr^Fkw>qzLvm;SdOI-zTGTBX_fTGG&(DYJwgb}a4Mm~PYbNvr9~)fZUEBNU($p;jdvY zmTzTYYknR-rhCv?5#}(A(8@D3f$tk=V00XkSbO$Z%e<>2eXpTEF; zTqLgRAe~D3&l{R0A_SSQlZv_d>#V-V{k(B-`r}5J0^{Sj6Em7_<5ZZTy1cecU(`K4 zQ_JYBQVPsCo^NPXtjjsDLrR|t)$v!A z?sf&B8~oy@-X}QL8-G&NMCAiDHI7wd%OeUI?Bp1ueP4oSA4$czrJnaQ<#FALZKi+f za9k`|P}nY?=M)AA9IJaF6?V|O@4pUz%QP_|8LN7+gp1dvIa77|)39K|BKylypy;a-%wKRfra~R%2-ISZS&2%~+TnY86Kjj^cP`8(22U?ORV=)}@q={$ zR?Q>R?2VKEvRZG57@Kxn%Xr%(q9bp`q|4#mcW7m8XR=&`%=%on*9Q3%X1}N6yzzAW zRSB$R)}!TZ#h>J)0CoyeknzXpye7}D5cv9#Q*`&u9*7J(0P;O_YJARicK$nm*;w55 z7aer>1lM1q-KV-drxv!U*=ZJ>@jf?L!6iEJ!^pD4$Y-xHT&(6oictG=t6k&-QH5Ij zi=zr(*{?r6c#Dt(%gghZDqBH{zAuj&ntH2fJmGn-AL?zb7ii6MTUWB1=80f{y=K=v zQjSY+XZf}Tqv=Xa&*Th`!1Cer%}`UZ&_c#q*`x=DNp^_maYgXr8(X5$^o{!%_0>4u zdzT~;S_du;rc=o#L$?9`tyOX_rKN^nsp)@0Blh{+h!zHDTO|<=sdDU}0i&kcau~xZ zHz;#*{CfI1rOTVW)4fjnr`<(b-~-i#z57DGYv0$Hy&I%}h=PxTPrkAYq<-_Rvh|vT zN1B?`Riuih%^TT87__<1Ru8hPz7cQy?rI$SC2bgA-DE5hAENT0nr4=LE#mW3-phXT zxHJE~dnsSt@32D70r?Yz}<9`e}7HpQ_~~!-Od9Y z28R%#FzY$42Ru-R5J-@g)q5|$`!iJs`4$sg-#^I5V+=2iQu#X2TxXHtul9Q@!5AcJ z)7J>D?6@$XPJ6!xK`x26$e^C;nrMR%+^nKXs-n7&Lz<#{_qGU2+$xXiI(CNzp9BLk zYp@-*D|EF9Lq9dz?xaQrO@X?9W*9TdANz?s{FDeme`uq77vMBoTBLo& z|0a!H=B6FAW4}hju;~#Inzh6C!^aHq%*hCY$`4t1$}dOn^3AaLd*PFIB<9P)?ef>z zu^A)=MrI3v9d}T(r7#uX%+E@&e?=vCoB4aOIjZNME{6m@rC@)#8xt6X{jrft9%rU@ zN}>%hzIbD{+THl30Y}iM0d(+-`F8>!rxtCNd;(-B^{`S=UzXtB}`qS@$a7%tO@ z=RLFTstLWERk{1YG0Brhb?C`s?R>d!t{T%Rt>3|Qp}>bu(5S&Scj3oRo8i`0d5bhu z(hq9o#Fw*{vSQmBqZ?27_Vm$0Rg>;Q;{M{=6#R`U> z=|CL4kDqPB^vWa!A$KIW6;3Ih=I;mR<+4BIgV&+yyt9U(TKu}GIP-}8ont8`@iITdfbnz6(5JYMf%s2) zm)g?`_3C@Sbl(xX7RbQu<+M#6XXzs1uk$QQzZA1F8uE6rwJ$Fsohz>Y0fae-?kDWx z!wH56W_0r6LzFzU?=RP?`Iox*Rx<{dG0Q?BL$IHElcueeT72 z^yb}M3;b*3lM*VYYG$A6(-%q!o`^T z#jaYboT2x7GEG%%46jeEVp5Q>7Kc2-5f<<-WJ2=c8gYX;X=$KV>on4XfcV09TskCU z^i2swt!w2mt|DHk;<&FxNpo7gmo9#bb1idumV6gqIXSSlPKZd?()?7kk<8@A$JEA2 zw{)2NW(d80h(+zquyZ4wT`@Nu#sKlAoj2!zFdXn^lEqGy>83#PrK{=9KH8Sp!i$U? zk~cMtvj>PMr332j^~oZ0x$<;=+3Qt(3g?!7=X&DT)$y_DQI~%Ir*1-5TG)GaCVOpe z;4w>?8PA$jOQzf}^K945I!f3W0_-$JAMWf=w zth=`$o}(sr4aQIs4yhz@$;H09FNon?jps}N(EUAoTV)A>H(JOs0cXgS850L%&b8T^>B2-OOg(+A?Qp4F z+KQ%;ZM!hv##W)ll1JE9NAR04o02!vO}EykjwNKhp}P01ewzv}bar%t8u8!9mTzM# znul9Hp6;RIPM=6tQfQzZ@VU)Nl4ods0@Y|Y^lUGs{3FfIh$|3YiJ`?WGv(m|2)!~> zSG@}Dj77*9e)25wB5XZ-rK+5_YM7%=lAFG0506duh>@fR&^vX zp&8B%iPQ&M$XVN7+d^#n53bqflfcw2R${)G#DC;Qq-M6r7Z(B-WYlk1hf=Kz2|_mh z&5K}|r$K=A%m}W#${8%ja+i?dfG^g&T8ujDz&=SzVD3iuqj*?Ew`OQz*b&QWJ2k9S zGW2(^9+Kl9bTuPTp;}>lhZ+Ir!$CJPcHYBrvaOemry?^P{G;cI7-*AhX#>7md^_PzNW?} zA};w_`mN_h@4#ku5Ja#!o|ey>`?@(@TL+f|$28?i{vbB}gqBYLV~bBlu3(dfwumSF zI?EuD9>^Av`KQGkQT@W|H~SGl`%mr5zvd%DLityKqL#tL8kflCK73G$GiM)-f|UzUjjS8WYT3r-aBau2(p~g3Q454M9@8N z(RYV*WoTZr?UPa){#Nfwh`0YZ+9{1nY?z!|{zbQT}~@~M$CSRwSIOR9-iY?dyQ z|1eLOInFk)Zz=S#w~tQVeyCfDgnZ3}>##WJGi1{|!-_}o6J%m;sN)!axb|_&V5Rgt z{)U&0_%WB#{G~~}%!q=-M}_5`jO#i0DFN6QsizL!5I#3WtA@^WQzZ;Nd ztBRe5@(~(&Lh@Vz-l#DBrSZTwht3*$8$*GKY<$zh9M#77i81($^nIj14G^$=lA4F_ z88MJh+4lI;55bIaQR0>k>0q+&PHr0Co-L4&WseTI zloL)LiW9$SFR!nG8QyexW`_KdpYLF+d3ywXf%ApBDNH`A!t3cq5tUA`f!}teuxwaI z-ndH8D=pI-?uYj9#JG66>5U556JMScG+s}?E=ufVG>30lF5ThKroz{5byMX`o-9I&D zW?zY`;okgmbi`(S@+w?wnhJ;iu$$->s?dnIt=Y3I$bS_2t4 zvxi47YLegjl9KX)lTEMko){=16{qb~xzi)8<=3;v_nYGYwWAy~)lfx}{KIOE!}Q`i z@&l9kEA^YE1Dd);4mV=bOC}h7TQ-G#O%Dyhnx}pbLRfF)A2vl$RL7leWFC~E;)rAy zmtm5W{V;0<)2Qu42SnI{Q8Y!5|^i`Kb!aY-e%fGhdSl5OLP@T)X_j;D;=6ilA&4LJs+O39Q&o# z;2E1kGj0GQ%@BM8JA1OFPLqd42i{)kWu~j*Q$E_^$1@=osmh@fmOrry%KBb;7APb_ zCCR5Vba8s|pyzqr97MZU_{UL$FE4Bk)|}?PCT^0#84k=Uy%JoJl-FK$NO??I_a$P+ zaaR$jS8G+Bxpz`pO|~_C5tjdvd3v!aa^dGQmx|oA0j=YG2?Joy{lQlVa!g$|-!n<3 zO9^tZ1jxJInO9!r zCgFd4EDtU>(IhX%OwZ2zV) zPX7aqK`Q%zdOH-Bx~`dYhD|E0&y#Pz=FIHfMex1hpT+gGfZsf2t8uc=f%Sxx0Mpk)XGm6~H=QY4yKjFd*44Retdo#(ciLqtwJe*K_5}+r{RT9a z?%DjN0-fROs0ZVQ8@R!Q{Aap~qv1r8!JCUm9B*bRi~k^P;r`rk@NDj4Mjhskud4h3 z?mzJsy1HHJjJbwKqIJnWfTY^%`4$hVMivT0_Do83SaNur2J4qa_k9`wZ=!AV6&DYP z%*EjPrJ$|@iSD}Zt}bGf-j0r0%Frq;bW)P+bcRZ(ks1OtLOHgAY51<8bPuM0|ANqJ zk&8lrSBRYAqU89#*oekQpBDDH`_BFkgT*P}6=5fSc6Iwq(Y0(&(s0Tr3b4badfmkO zk3M_T!Q;Xc^??o94j~yhn)_%L}aI_c4U(nXeRJC0Y?}CM!BgQ z_wwL3FvI^>wt^k^o2@8tFi2<%#yi&8%U`k+IlP%B&|pEuU?~dY)M8lp@PcwS&S5Y8 zDjlqnD`TqQYTR=tU(9{6u}X4F=B+QUFXmLeGupW;J=#T54PejJS`+&}sS1zR*zP;j z+CtVrd@pzTCmwGQDK?<(8lk;%umxQ0s3JJ-`z)$;bK-B3hY$?4YLy_Sxd)uWPPi}?Jy!@_DwPs*GdKH3B46%jh{tsvh#+Hmn zcj^$$$CJqEV)Ir8M1pRE3n^{*?SKNg-P))D@hqbc~ zi>hneJ|ZE~ASDe_(x7yAr_u}}AT1y;z|bO%bcl3^(l9hA-7PhQba(f+LA{^*sqg*X z{)B?wXgF!&);&aXz@5_2OjV4Uavgi@72?$FU*kaL? zNKY!*LadtF^4Uwej*`~Kjn#2bgC{EB*gC!G9ZbWF|A8a^r57D>eV|@PJZx zd2dKs(H3R1_NR@G^N1~DwE0Z-o#ZBbVgYW^OnMvK-Eqy5wEJB96~)%V)A0)yUyb%X z(o`QVxtv_@;SyQ2Vt31Tk6@7zMYywSFovZTgtR%~;161n^o+{4?QC{YOrF$ED0vNm zr&V}fORBsk&u^Z_SvoQch`p-xx5u1j=Xf5^`r0x&^(*X2vaObYINwu>tyi)zmaPi% zb-0iU(}kD)>25Ei(@QCjS(L5SnC`JRCLI5c51y6+cTvA(N^-!Md50_EXk!5wDaNh|OAk{5f}s@+bS#t)5h0?e;G!uFH<^@u6jD z*!zM~{M2}!TBoXMq(hW9BsXqY2;2({jaA6cM|5##4|wH@-F**(R^z_gg;MKo#T5b_^n2wo$`T*fDYWphYVCk_ifZ` zzZu1=+U;I-u5+zH_iCbMHs`55XOcG+Hl6%(8L_|P>qeVqg4jQZ$snkL#;%Q_ZLre5 zK1zn`I(N#2AY#M858tXwpu}i4(NxK`1me3#e`E`K%$}tI;|Ss~W379_g>EawIm?;J zmjfq{mv$dcg34FuUE|)+nk+txchD%=B8*f)dpkLLD_roj{s-YA?SB$3SYgL48GMVX zAf0TN#YO_g82{oJhf+1>L;in}Ek33HL$>h7=D$-dl!7_gzp_M0RE<&lPRTktP9M@L zuh?>R3ZTvNk=&{nL@n1amkWfnnpb=%?!-0omEUQOsio~CeLwzQJKWYpL}nSiovQ@J zJtPuN%P+B$Z?GToiKB-(wR*c9BG{OK3WkQ?Mo#vHpL36tiou5%EZ8!EHw{Y+T+tsd zelRN<{!|Od`dlC6soD~lbZzYhv z?4!Te%*+Xj-X&DcrVIj>jxHmMNS*#3ck-^l_{zEg2C4A4qfP=9OJjG@b`PgXYmmd4 zMEQnl%ecSCSh*&=z4a=zybj>wUry3ku*5qAFP$^;ZH*T}eHXy6qiE1Odc2$iNtYf1 zrw=*-ZkE=%P_~;0{2iFJR4R)tKh-2csr4$$AV7=FZSgRkI_GP${5?*7$)4`lhii@J zPMW7Dng(kI5;tduvNw;(#x1`bab2j+5^#|X9^9b!EF$qO-&}a6$YZC?5JE|t$iO-y zgaQAoTHqOFoe=V-f5Sa_%Av9o%u;%?GDXLw${Dk-I7#g^WWvugQKkd-IBxFa(U&MdfvQ^^(0{&JeNlNuL|Xfw&eHzMLav1`yfwCx&kM05&+ zAmsCzLGV4yL$-bQZ-zDxrNl?rUZ+3ydGW>|`wDo&Qx5N64gz@KJp-}vkMduguV>i` zY>75KxPtsh3!86zl&x_X)cl?7p@NCk&{p@%htGb4bJ+SO&L~7qx||aI8b}C-S`?}N zThw=g#7j89%!7A<_-lC2?S&@8l5ardrmXXcs_AQAW(TW;`b_+xGyB)_+b(18x@rdS z+g3FOtJxh|N280~mi9tHJD4xKB107hY``w7CL8Ob>4QMTLA*q{-chU`-%r%;D zLiy>dxNN$%Rr(?f8R95;?)CXpVS7BJ%lZda62#n%id($X+kvIECEyTfB=m=681>}r zp5=NXv4L@Ib;>M}zT5>{h4tN6m}&z&g2#~R=);(?BsNyA4xGvpsV9$8W>%zGi5lOa zCQ(1*9HxL=VOp;-XIWj!mK71-DGr7<4lsNq*ywRva<>|4@m4$v%{M+YCqcr8Tr(XQ zp*8ETD~XnCw2(Veu6#E2%DaOQ_fxjOBjxQoX3sU@j$j6kefvJb_R@cLa1g-*8sveG zBz~D5sOv$OyCnB8iCq$j-dnkJy0O{S|KQceeB5uz;omejfC74eYPwom-Kf1U9?Eli z79pVQQ58~`Hui?6;nLGX*J>hbV#BQ=g-)_RjlMUm&LQ^0edoq2)teyE#k|9sS`~`? z?SWCttFmYg-GP3!&*Uy3o$I6Fjiuz2l_QgN-^&|@iq>qv@v(H)uoCaOL*+dEIJx|hBKjvA z-bS=i5^&+Sh;=4KSr5j!c=F5=#WMsecL|W`$e$ zQKe8rwmr-$Zmkd7W%b60*r2G(y#Y7d0F8pQzAxAq?m)hIJKMM-QlN_9VJ*5~T z$KOC*zOJ(!kFd7z6&e!U-_H=V@$deWdA#A^e0)0Ll@l+MLKG_Op_s@?7nBSo%K`s9#19o#yt` zdJLZBCnzZfasW7=WIz! zVS|({w)zH~3_yF$Ht%o^{V1K@XXg=Q%0~Giy|754WbsqHXD&^lQHGVs)EJ5V&69|# zP;5~8{&_R))gB*n;}n+^Gz#!JkgCZ}?PPU088A@|W>2&C)1~lIHCB#4mR{PH?WkLE@LKI(EK!7;}r8?v{Obun!esvGfn+i(_Hdv6djU({nD1o( zF1h7Pqr&T}u+xA_VG3!pk5V+)U}bN7w`_PIlG5-z zSw`V@vsk04!n#3tAM9#t-Fd?Kl@aN)(u=v7`qGgjx}pUp6-+49NwR@7I~!zr?-gwm z6nF8tB-u>)Cf%t5A~2_9hM6zUj*bl8ez~H}zQ`xFpgbg@Dwq=OcpngzQZnW=nTcBd zbdy1HjJj4Rc;nh^LQet2{~mNNJE{EOW08~cxCz&b4~##hmg1_wlSnJR1rL5m4l0eG z8Sjg%EP)=x`k0sDet-IcK=Rxs=vc0ZH(E zG&(!!r?kKy5yV&4{?_MefM3yADt^5c>Twgmv2v1Y)Xf4+hH9luNOgh|}!$pA>yO2#~GS8+(1(M*Ty3`NO@g$RwLzts}43z(kN}XV`ajIHj~^Qg#Z!b-#0c)t4)WJI5Ffe2gn!S@k%?y9nfubZW{^ll&#Uv1ss=mIj?GB zWeJv^J?;p8zY&+jD!g5A__UUufKmReSe#WH+HxYyaxE_rrNzhCErHg##)JCf{`0Q) z_2>(AHykqjqt;SX=fauVj8kEUgbK%jts4%3Pn}G2>uVY&Sbf)qFcO>`#9i%?_^J7- zJj(1;Zg6CXv0Q|03e)gmy5#~zf}*T(m*9P~h49lcOHAuQ5LTQ4_TmpoLw~R{Vmxxp zhDhv`1I!u>(ih+m%Q;f^U!b0-W0CU&)iMwN#O_LYy276{!N4^Cxx9pqWlZ>}p{opa z8pAO}NH|*4*!l31h~Xg}HS^&r99=I+Uv_|N@b%u|HRjx| z7R^@C69`Li&3^u(O@8j(z>)AV8_=HUe+jUlJ{Eg1>F7^ddGt;q2gduWN)RChmJGzL zw5XHsF9PtlGb=+MmnIQsH;cLO{GgDCr4)YM+Ws)f!LJ{`EbbDEe0r8lUQ&}eV9~t} z9b^9f_DV5weDFHfs+Iz3fY3D5S5V&oqV~9d`7ytEjhvOXg|`5`9~ zb0d_$>|7$3<@4@C@Hontx@U17=hacAuZ3|C?zOKp{91jpq+P-gk;Nmm`FQ4LZOOWe zD*KPVhR0egyErUwJ!@sT_n-RJj~g`(=^siGF+#yzk|F(k=@RY zXoxT^l4kw%h&yolET_|O@iv4SlcKi_1I>tI46;6~rLrnXYT=bhBhT@2~xiM+Hh+w^EO; zfgcFivrU)FR%U(6gKcK&S8OShsw+vHE4mrk%qt0BRa?JYJQL+`nHP==U0_;|;$O?N z?ANDUcgK%!2;h?RMc-SG(j*^otUuOVzTqlhsl#5|=lg7YU6Vw-Ps%W~)AJlFz4+>7 zM_<7cw9x&TKJqzD0}$SoVnt4)xZGAm-bE6&$5^4T#+J(qVr)fI^A#=wzw>x9+_kw+ zM<58ib%I-t>yG=~puz=tc{OM*O6z_5BKMkBZ75BIA;4wghDrb>fU5ArLy7alF&T(}OxLJAV~R+url zSwrg?7JSC;I)qKg{nPf76C4+?-nhXw+queh zfVHswr(o;$P|bdgPgn)TQ!Pil$zO-xo>6M>97daUe>JMIegH0CJ(hepujUn8)xg`% zw~UmePVK<-!kxAq>6B+-0u(*=G*O~&72obgfw69MrGL-^r+D)E~-$Bp@uz~%jq7+S%ue$AnKAh5g6?j)3f$p zGX{?La^oOJ`Q}GsZBt|8x!euad4G=b`Pl9^khYrDm#5#r58IKFZe)T_7z0cMGueaz zWhiE&Xtn&b_$fgm*laDw|FB)658*@V1yse^{iRUryv;*bS6RDS&7;iz!#e$dY@KTK z#}!MKSx_@L!dJU;LO11IlKMQWpn?JI}_gwIX10zSWfKc<*;mL}+o?72VSF=(xudvHF9 zYx|Q9vmh|J64l@#9~T%MU z{5?oLZR#AQ5|89f^S)e+HGF%uZ~N}I?iHC(l)B zq<9>q%8_fz5iuNJsMuJNp{sQAlK}%UzVcVR=P>laJ$;Oo<<4oTy*>VMi3cT864U|7 zKFvbd14PX4vOml-`g|9G#XLV-c@;P;;GM}&WL#@H=URHAG~T{b{VsyfeS2o<`6|K> z-aJi`-E9C+a|L$@2z#sOSOdRBN8m3W6aCV-D3E?z@PAK-+w&D~J7~NfYWxN|J5I1U zWc~dcxL3u0C3gNCy8HQ`O*dZNOIHg9Yu}a(%-4Ko!kf>VSRH=eiCp)_`&%aKo7<@E z`tIB^KHrg+{$P$TkRIH{f%a4d%4H1YQ7hES@l4zS>1UEiVxrl*JIqYq=AMr~4wK}s z5D+i{=~%kMi_MQGr{W%v2ppW-vM%Y8t+CpU`dS*Z}=mYWq`ky4wplZFZ)IT|A z#2zZ*wNBidzccjh%ntVeJ`*RIM@~8p0rWX*F$8BW6?=5OkvvjGv>gH-x8XZU9Z#9K zYVl{Tf^-Ey4yIdZ)TQlpX0G$C^Ufy4Gjz4^WuML~n~1pVGMjfFn!T(spXrzTXi1RG zYExRIkm52P1(j;ohW#8t)NZSU>~80wNBDD-VKL7%iY!3X-}lX;S;6|1GPwi+&y3sC z^e=T~Rv?x*GrMTB^L3|rA?jxb2Tv+AlQp(i3y+du!~2iD&VLJda2JHY_g*Ug#j^IC zniP$c*v5FJBWy+b5b66KP@6g=Y$N!h;L`-MD3J_pyeh^v;jNG_>!!qZ@Q;m!<_X#< z6cKZ+9bZCo58_r2p5v=AYf&Uv4($8*R{&P|dqovPsXs!hy3$i8XonoG@KiycFRc|i zprVvJDUBIRfQDC_g0OMV$o&kDSvd*C;ZeXjSaW>$1vSjj(fGYbnf2+2lNyOd|I%Q1G zxD_*c*r>~I`_n<3RNYfmeB;Un#j_%e3lrn7e#JvR4wzOTJE!V0W}4-2Abi>`8j7?JU%BB8bCL?As~_ z7>X#?V7#TV!)C@W6Uu&KXVph>M56p&PP3oD*~KF?RDITkRDzfR%R;9(P}jN=CjPR&De^V85Mog zMX>!8Z9ySPX6u@jf8Y804rLrJ0e3AMhS&a3o*jGae!5A~#u*@V;5P|8q~*sgTV1oQ z$NTI|Eov(YeO_TKHf-z9FrMb`&m)jrOr+bQXq;yHTExq>n3=~m`};C`#q-A{(*+s6 zQ@b!}W?vUox?DeS|10P-@1ch}d3)bnr(Cs1XXwif(>>;Txq@B7{ghmtDc@U=3UQ=( zT>Q5~#-Syhbg4vuQ&h2@#oms00t!PosPE3r6V)-&BM#!hGXK$n*73VM^O2t}P z@8EnS!-y_1{GfxIp4>1!QJeGS5Jy<1x9A9h_57h&`N3)$A{(jzpHQ*L#~v z;xJ|#To;e#E=K&c1k3iJ-DevT&WV%a9T7e z1DlwlYJ;c1i4}#+89z5H8GqdN9%)4Z{P0@%NPXEnX^V-fihRE)w_iXj8`YSVL$)d!Ea*C4WTNKt%=!DcNuQfLOjuUB|N7p+a=FdM6CUW3` ztFbb2HBkm!Uhd zjvdE*T*K=fJRl(9e{r-xZQUB3MHlP0-D%!P*J^BY{hpp!Ey?J+QZh*ubuTnx&k}lS zvAr-MgS7piY=-8%UX=-sAjPO<9m5%dEr`S zzUOmO!r~I{&v};8Hk8z$u07npFUxUEKw$`6b*_?@0v_#l97KLT4Vim*9>93{A)J)D z6CqmeRd+(TDunq0e*H+(+$6?-3@VgF%kCWHyY@N;kObdiU}lCRAbW=_Q@Q7hUqqT*z1VCM{f#cQq` zdF#+!zvzIi#S&zqJK_om&L_!e(Sr4iuR6dSNOnU&e#bBvJvo6wExMB7@kg8|&n&*7 zR#J*HJBG&!Qo+)_thtFd&Gi+ENu-A{@MuO8ltjIcC!J`=!Ce-dfzt+DCE1*+^vWTO z`)rTIS{$$*DXr_P!|)+FPFBq^?4eQ)61BhN`A#Ati(y}+a}gOcWX@9J2JG479*1j!Dw5=#4>@!u+1^owW64-`2|&e zrIr7&Vn6<%zH;_am{gz$7~(R=Wt^L3XcVH8E+{8O-hpL$2JoKohk2%iBU#TwS`i)ip0w)ug~=MEsug&hQDE7NAtMb}-h)YJTu)NC=bth7fw zUz;f$B#4yHT73z8-#6@)9vNlRtp@ro)W@K**=1!Fr-YS@cl!vXIKx7z-mXyb2yP8R z&SBcZ6-No}6L4KuFO{Sz6mhTCDq4z`M1f0CbbWPdKktX%7d9%RiZ7p5Xy)9hHY_`a z^}v3I^%=OoXEy{H;ZRUrknFAgD?&%7h^0Klf!uH^4v$T)3RHnuB0OZz1f9MoRtXL; zXDvA!aQboGmMHykHyb7r>2VWID)3MkN$SBW8TqOBUf0HCO0xra@SK=o+<8uTxEPwdpB12(;*sD; zz=DrEmO9}$4VifqX_H<8-UsU4{#tWg12*tiBln_A+-Z&t)S1O9qJ$&sl)%dSM6NoE z@q!Z!uHLg9A3r83H?(^|g_a}CD;;&ty*m@KO={D?624S>R(j_r%sp>@l7M;lRQ1z6V`I+AM1aow2C}~ci{P|LQg1u$j{y7w;{23Bc|G} zliCnHs@Yt~A5oaN_%hLSJ+!Bi(8lS{?V|D4T@te}HyZGWuE*B;`T+|;l*scJr~J@r(Unq4JskGr zFa~_yOypG{7DM~?HWss_G%!ZV(RNz6kyX@Ob!#wbvZcy<(7ncpQ+pir=v*Fwr!x=r zi8x(dMYlTB!INfc1iG=Zu3jd)eh#V1mgkj)9+|QB*SP5kYl}Wz2~LqpusJqo7cdn{ zze+FENM)wzi#kkcQloz|iYlm%gA}9FgY_`HSUWjWWrB&d>6mkl_35~Gp^Xwq&r_OU zM-w&PSfK1c;24+vbJ31d_IQUZUqOvKpo=t$RoTwi&gfWUS5i;iQ}4Qs!K?xODsv9d ze7>Sp5yJ9^pWsv3|AstR+h0$njNXu zn(d~N#l=m%ukgjQyb62zf9)?ZL|AGa1Wm7c@ z3%4CWzB7;X_GlpUEy8Zj6g10u)D_J(a5mStW!COp#Cey}94C9;)+WgO$E75yyMSlq z0)Amv^%C?&3&b&7#mNA}rrQyBf9j+(HZ=F$*$MM0HdnvB-LuMminq6w;{cr>P>}%} zK>!>F*SY4$(-TGlQ|IHVX~rVfxt;L!;Rsg}kaN;Z+H;w2WJFA`nY4njXulys8HWp< zFYVW<=__iv3|!8Gi)llao=cMCO`f^?Bq2^G*M5Rm@(Vq`@4nxJkm1ik;c54fT8XvL z+mVjG8@`aEE8M_I<7w37@!KWc=ru*hJVXzWpb^NSlXhYl5NMo8=2PdML2z?>0RkMbNn<30{;K9X?o2?16d$67i4C_J9xM&Sx*GUq4VXCTU$yzAh;d6LxNwSk-u{~HaSx(;owAUM1K93 z-y;kMNAb&(^4GtC17cNZl>Im7@;C6Vb1<8R!Nxzo^!g4D0TKj=D?R1;; z-T;*^GxLbtphomjOgq1YDV7N)GK+A!BM`M9lPiAoE6*~XmX%V)B;J%gVajTFZ)9At zr=;3VD}xbx#dQ1St{c?ahsXFDe-e#e{DO2s{Y@teyjrEnpDj>FeaY%S3Pk_{ zJfyeI^YZ=VNa`pc7DCb=x@LzIgMde(+iPo%hJ2VcGtNlT z4(#;{&T(<_%wED~;I-EOKF;F*PjQykj^A+>9<+arvy`Y6(I-?~JW$dPH*x!dypueM z<>I_>%S%Ch=5bfIzGT(al)PYA@BZq-qB^Trn|zyEnX1#=F;2k#e5ReGUXf8d>a^{_ z^haYk74!bL6w6V~qd_=0T3cBO(HH-zE*JRsx}5Uu|A)HVM0Xr9M%%zcvRmEsA>YFn z?Mkk1sJ6yC$9D;POFJ6&X&J37y&D9VoL=YKPs$V8{TaDbl79wt)Qi~ItWqvQR9}Ct z?FC?~eFsfqPsqR)t_R->PwN$#tI3mBrzP(o9UCM>KzapM&M*8fd{rCwG$dij8ZtV8Mii|b5fw(^jJN&qZV7{s0X$G^V!GbOxMBJRW`61E6I4iG!6$Mei%HZK`Ho) z6Ja^&llD&+JY0)on6pyi;BZAJ3-Y(F-N7j=E*Qef(dgP_7yB+`K~Mv$H_B#0{Q|)y zv*-ss*+OT)_B8Rl{Vs1nVJkJyXCQ1J>W_wv`K}Yazv!m_5ta#4z+ghwhCk zPbz1hq@yE2NYUqmcCfAO950g^2 zsn&Vh-k z%GrOR7Y$3o@zzhXp0VT%Io%%jYX;IKXV-gEW!@A^J5yG!Ix3J`yi|E)Ecwr%t>> z+&NJ&#XLbHOC`Ntmzg*MzE4r8?M=wr39;!2lykcvvk9EsU|Vr?-|bGQi&9_!NGqz( zNEK9 zsDriiD_cVCJX79mT?I|sUYcwuWv@@?g|jLw_|n@z_;O;duF1 zwgi83vGY5MC>3Gjo8Y2lq2!OoKSS9z^wVdFTj6c@c;6tj=1+$v)`)yb*OTAdces4) zWbw&zBb3_o8M13)jteD!*F(M(<7zxVC&O{wg!&u(v?TufCb`r()XmbB(rq4&Xvc2f z>j{M&baS}$#-ZEnAO9@KeD~oYiW2S3f`Fr-)idWleB4$%DiVt3DGDXOR3r+0%(qx> z%8WyuW5kh-qM-o_a_}mRcA6L3B$y=A$r|$C;vu3#zv3adE|n(!qU*>r0q``7cmY}L zE(@r>Od}TkFRCvSZFHxzrl5F}7-C6?%V#&syIh+lmJ<~y)>POg7qRe1Vg|5g6g=t; zX=oICKvY|u>qA{&Lo0-c^g6{svuKJm)%&1N7(^C%KINO(6ZnW$wG+EWex@j~3ucpP z=`E092KoWeMIQppB}o~PbO?n*0v#5|P3JXUD78~XHeWq50`;9)BZaa3{w}<{;o%|7 zT6?xf_{BG}&3W{!iqfZh5i?bj>bj`SzPKEdl(=8BSn( ztcTpKV+-IB{H~)_xFTxlHkPrZ*!neKXEps51{1~`^zJ>b3oVuFKyew9M{NCx9 z(1Iy3hw=9A2<@h;2RW*V`43?~6*gE5Ven6n?=T^vz7e=CrE}ydh zUOOXVR7SDJK4#Swegk=*ti>~Wh{4^g}yjbH%7-?xp33cRXq;tFb__BTP*!o5Z9&ZR7dd$ri_W8*d2K z;Wd}Ft`hN6!5$ZmX>xc{PHGMfmm#|h0-{>Iu=9=&!g(3?-8mY>zxPar8?d2B-HBh{ zv)!P2^xjKQPE@dCe+1s_g$y=P5jbIz26r}(=M*-gQw;D_)}Vnt1xE6fD_foXtsX?n z-&31GTJR$m?aE(mi5@dpw0%HV0dg9dAJ#t~0eA`3tu0g{%_-+kpRstjN5MX6SZY&j z*!tpO^kzi?St?AV8M^l95jL1>Uh2v480?f?JzPTD*WkU5zbsI3U3)UcUWAPT@8C25~^y# zNwujCD4#*7DOzCk%3wyJ8?v8RT%B(B_((J&{2>nGDfGnKaKNII+4twFuR%pPtXJbD zd;3wwt_kg>sU3BnJyuIb%jeR-O*F`S9Y>J{FZR@JD~6U52>{!PGj~%vUOx6WDrv?F zxN-}f=?Cg#2Q7FZph?~BXDi#{ER?O~4BX~i&jp-1(A|px-*O@r%5w^%kewRPIbTRO zN4o^;qc%0{*x|8WTQs#*)(6xPw(Icj*8oAS;dhD5jX-z#SBab{y(MkIy8^(p;@iAq zO&B(po}peG+x06w0oC4SFqW=r`6{+Dt&hG6GNgAf4$p;eo`_+hBrKr_#q{n|SqbA_ zr@tp%JzrjIJ#uo`^QZ`yZR6zYHxkCqI6L<+4f>dvIh+pzXbneOrmmj~#vf`9P)1Pm6eZe3_*!{fddK&Ng>WAhz!Jwy*`%t4F@^O!4Jvs`qy9lJt zBl8B>O;i|ie;jr8y5IB{lJLvZaz~V7A(DT=J=_H+rW;shD)|unqUTR^TK6N*+LB}^ zLU~-s*VU`-0m1`jTP;6M=+(@W5a;;OoqtZyjt>~oM{cRz<_`Fisx09r+5AC7WeF`LA}2qCvUmrJn2_KzqlNvDR}}fs_~7nht$-* z3de6erXN7)eOQ{e9{Puu)D>rG^7Jo+Z?6*2`OvBOA1qB&)@o~OJSRILlJ>b1-&ka2PRoHs{wK*vqTyg*J zRxl{lK@c2taxNgP^=6Hbm&n-E#<3)Nw18AI3ET_!DV^J`$MMJ#@}bQ=*@#i{A(L&aUQoa6#;WXO zX_#Yj2_eHI=vLZ-t)x)bSjLp$kLxaGt!*Zd%(1oqQH-6|CQlrk*iO)8r`Pm zaTJZ+D(ZK+qr;fF_+`j^wP*4;-4W5RuK*5-_)q)!nfnpHJQ|#Ma~tsl z?|uf&$iQC>=>JgY-n0C#h3-J&uLd-VWl(HRUnZ1^-y<1o`qCR(2%g#F@#x)<;W;8@4#lZRiLViq zDw{vwq6w+`{n6o6Fc1#;3^9L>U z^&;2KpWM+J(w%JjBugK^!?aSpnqF7_3!QH@c-6+Zf6|5 zD|tDara>y8`7K_UE_blP@TzPL0khu)w4hrfIu93KDz2oA8Ec=$vJ&Di3KM-7QXOU_ z?s2ti{mdA7!Xf8)4q91vx)!;zdmu3pzEr}RvXW+Y(YhRyvo3JJ)aJq6!ou+XT5#yM&PQFDu)t;RxCAg9bxHZ8X;&*b z`Y(VMliJM`s&s!!+VO@{`Fa*_g0Nq}%9}`2?lX%mC#&w7H~~5<@irEdcr94B6@7`R z=d`R(0{EqkfbK7UGcZK=>Xk-QN*+R0h$;9tAj5i~pr1Nj?$tRSGvDjM(EUGD$)wDk-76@ zItM>5U$F(!qOb{zy42C2nIa(F)j!hEQrbm*zi{JR^$oH{%@UYFNhm9B6iD*3O4e|^ zHvft~L3W{bxO7+Wa_*cL(iR|_tk)k2c}*4S?iWE+cGsjR}u1YVf4 zmQfnUIrRu09fw-whH~NjzFZqbfSY_PGCx^i@D@Ad5N5>Tw}1GQlNrEN)W#t6R1RZh|xu!zjhJ{?MZ}26tQ$9@Lrx|JB%aB&MLiq^l3b*#;D^0_> zh)pV74f&mW_d)|CC@Ma(?Fei>=4*aZtSYO1JvN!DCglec?2!s0c?i%t6r!xLQr6T- z+~rOzL=AchA1EpEntlRE8nQ3-p8$@<5uh}53q^9J%D_5UX*0#q{ab|NtKgQQPj!2~ zV)&(2*u^$cS1~ZC7)&&`?3=*NEXj;ty&ZEcmm$NGbgNmlRM4uWYpyL0B;tq~)=AhR zYcwP1%gp8c_PvbiOt#rGx$uG9d!!gG0x4(}BceLT0an5A&}pmac5=*rD~Df7$t~Kg zzkmD@??L?&>i}#2`kI4vtyXBkW;Oc6@yij;Zp&l>&qYa=@1cI*iz$Ck%Tt_N?16@W zUOnQb=aB8wF8N_>K+xb$J02NrvYq=70`5Stw4-fc6!k;cZ8DT_2|dQ0KuPsg!PnES zi<;@3^igmBi0OKM^3e7Dcag+5s0`2WA?{!-HJY#@Ox=t}`zo&;5;ZL|tu>oIMDOFn znyVk-!L$`)^ZQrsn2Zc$A91`l4FweX z*N^pHUTokCJQj(otali4Ic^=t`h5auNWV&BR6n#|Km*fDD{+YE49lKYY`Udcx}3jP zWkETo5B{4R+Jf-=T`(whd2tdta>UMe7*Sqs+xyQCU84Qeeimc+3}BnwAsMHXZjp>u zcSy$9bIvwP@Cth744W&|TLb&4{d6#Ns)N` zm9LZe#ot~`65Tfy|9xI;i?=@cA3@|IKZ=#H+4rzX+n`WIc%M!h?5qE@z2k;ICW zxKhwYS%L1joAUgx-tlYS?g1H9PTE7?eE_pa^J464rM*m$g^fW+)qLB*#9z1q9E>;> zeu#vqt0a@}>ovmZuHf%=doq46cNe;x`X2ON3jFCDZ`ys?lJqbCX^S1-iO2izonw~& z&^Z>?|J6C(o%#7`1kZQZE$qt5y@z`E?MC_G{U~n5Zm{@Pz{8w{iA4H#hxu2Ao=y{- zv#3Vdu$CXbn;kgum|1qS_a6|0H+8;pFI}Q%Q_DEL8{ccQ|Em^W`;f`rpCsx?@&-0$ zP+)m9>u1cn(i-{ax@QFphOSxf1g0KG%zp^v?U|RV{)CX9)`XE5_(Y zVQ<)gtZlHdbd%uLs|3hNYAG4yjaMLN4v;^R7gYKuoMI-jm93iCm#H3{6DA`Ec|H4$ z^*?2?(&p2@;}n~;UzBYTYW*~=odyp&y%<;;~rp zHbus`=~(X|?_YcsIFct^$Swj2d&gLDD3_ijX-qfBI}{nrar4;l*c3nmVAw?Jbs>4O z7J|e5ubKjln<8?#O6qYw;l77?EAe?rUQcJx$wI25f||>p95K<&ROxIg!O@fxIE^C) zK(?8tvTgKfa@?2dpw`8WT$$3uFM9#o^xf+e{&Z|bIpz9`r<1Rz=TXeHm-W-@59QSXV&fXJB+g^u+DQmSq9ig-=kj zd|GIr4dQL39JUcd;7e^(^^H1mLRe-lw1A(!bm*^gn#vHDQ`)>F} zzx!kDZ^`lIR(kgU*Z|2YtJt^u)7duAUQnZw*~8=YCBju7A(S!;1OL8*PcS^rllM}J z-?Y`WI3_yvY?+A){hL3ZvR~E8uFHl-2;g6y^iYIk{_aV8-9w<%gCmqF?=Vta=KftQ zr-}P8ZM^i)yY>|dPouhn58Qj$`JM7n-CvDj=Xr_bLw3V}WNQN|@vq|-GBV`B zB^jis>n@)zHdDT67r3hqMQr{7IQTuNaNmUP|Eku39`&DNcR%lH-@hOEe>*U3w17=o zb%{DXGq3?S(gJMxF!ZY_940ooa0?nJ`eT3s-dhD8;gAgIq~CE1EF*6Fx>(o8f@+8L zmjRM7kvEXhw7&!fBR4(yjTOSodDyYOl(AmKbyPduH*vx zd>a9P;rgBAj1rHdx6sY?{b|0eG2|e!;O_KI8vzQPI#2JmqwFW>W0Gg<%s`_$R}x^9 z{ouIW>Okppb)824=PTiX^%1U$$bI6p_>>>KgnyjDw|`v3ks|fvuTrnz3BAqG)8K8+ zTejo93(N@=4oAN?3UmR7d;muE0C}^0cxEed?K44k4cP~tzjz4D)RDAyiQH@8w`O4@ zwUZxCs`j0wcMkgG=ViXwf2R5+llsB!N_M3&ndhMoA-U0MD zX7)<}J9vC?l9vi1@VtrId-E5ppS4#>yS2$>y_JmnjJf;!VuVmzZ{@cRq@s|f4DwQw zhO86%7rbZvxZIzqii8NAg`4R9G0oiaL_~J*0IDL^^(|GA((jgwkOhz%v<-zn0f_r+ zKONfg_N%C8TmPrDbB$&*TjTh!^a!o!h)dmzmZpl*K}2zsxJQ*Wm1!c51EDFK36ix}ak^0v}T4%vILA+=9+qX1!(EOvWHZCvus$)@}-X{4Nwqg{i2ty#BEoK({4| zCc}9VrC;Urj}5xFM8})8b`QDG=ChW_6+YyxOXnQGhg3_VfG1>O8s|6IB#!G7>Yb7+(fmJ<61K!oT(7>4(xkuJUg#T)Q>`%Vidsq)sq_ z1>m_OP^Gu$%)PNYDRi`QUSaX|R4n{+6^*RIG9~+@3Gae86#u`waWq!yLNE~W`sti5 zi)uoqV3|CgT*(vfn%c|OPU}^#1t{QN_{LC!!`#$beCJBr{~VJT#k0vQBAFq0YwDs#jY$(JG7dWhpX;Z*a@~(efE{=imqGl74^V zDd>_-I$B;==nh=xA;F|{sMA7{JE0rv%I=LP4lrJXkuH;b?R-ra!khc#p@l}fmfx97 zkFz>{|MIT!$4S@!YnLq6VwLHchQ{&Y`fg{k@S3JH!o;@SmUz_+;TW8(2mfuK8BZmzaZY})e{jslFBdEUn2=EW=+@_4`q8f_WhAHRO#`m8<^Ue)%ys+etESC9HZ61OPG%|U^8Mh zX?j8mxEa990|qxIwWz$jeeQuWGD#u>J6j%1|B#9W&mBkuZxb>Kk#}MHVie#fSLJp4 z??(-tCM7?VUG<3rN-S6?q|J07DyACC_bqIyNHhrgDAn829;}qSr-|{ z{mO0^(>kxL&8+fkg6GZKM;@DIuaa7D4A)uY75B{LBqrH4y}!ob+nVU^nMUpK z6kSa#?^9DhD1zjRy<4^N5VgebNBu$mAsxkopZ7Wp5rQPe>cmN{o`45N3HTiRbIB__ zEXwVklNtS~m2K=)Agy18cjYd+tn=3Tof*S}Fv)=s58F$7VC&QNJ`Gv#O@uV0 zo&WsRwuukmeXVqi%kD-%ZD*}Vuq8QyP3+{&o0)k>v|m7$a=l=v0D6>^8Wq9$aKQZR zdZR{VHP~L2)8(=KK=^eK+7Me_%&!yE_cRpXOO@jIRKbs;lzwlix^HM0YT0${rmo+O zhQB%hLj<7`wJ4YC>p-;ox7-Og+~{hLQ;oQ~UqcV-mVMo0h{Atenq>q_m_OV(yHv(R zBo<=%MoEADJO@4WIzvbs?4#K+LFtlp(@RnqHBy34-Hfa&Xk**VM?y5H9G_EEEwyZ^ z({8)mm-)Z+ciWJt4}E=7q4&SHsQ=9|ZGACtqt;aNu7dH=bO#Pyw%bH75IN4+elp?l zaU>uElHh=Q&k%lJ&D2i^%&nbe?`Hs*wX^(JMSl9!mh5hH{k;HPnbP`b@(nTPlg$Mxx1cH<#kPz|(_uglWd&W5T-gC!y?)OK=JLb&$&iTytl;884 zNp~zwg#Ui}?*ak>!lt*b-xUxz1QrnZ1@+rue$CD7&_DQ(UqbGhTob4nlv(B%1bwbr zTon+gO*^`G|8M-V(4$)pAp!y-pAUY&-~%f>1q7f+O|M_Ijc{EfO{FQC6f8yatYW)v zUYpc#+VPST`Sr2jxq_>OBTk35E*YY=@ZU>bn$(+&e~Umg;^T$CeEcA+{pH$-Ga@f& ziew?N7T!n-Z}p0iKTJlg;ti!5``z~DBywNgbEvYr2bTxdsn~lu$DYWQ!ns!2kseV} zb;9{l3H;f9uH)B(XXXp{U--|Dw{M{j>fIF;_~R20K#Pf~9z4N9LZ=R%$iD=S9z5-? z#UG4HK;_K;p3h|p1!;peBnFgNY(sLf%eBxCs%yQnP4!@X6e{WWfQ92^R@w;MP0Old z{9ga`zY&jd`lo~^#gL@4KZSVn!hP-3KgIM~7a~{`tz*O2&ymC>P{)!UZ}n$xk(M)f zU*Wq-l>DW#Au@sceRzv>d9IDN-)7J%UUYXMS&&vnKiWtl7XU#Goc>)K5~yg$Hg@KX zY8-bmlm4kh)FM9w!m1R9!9bvpAVcb0E}aT1T1q2|BgK`yb~8vq*5qy4mXO`bkYv3I zsq}Jri~KZmm#%d0VDdhCuc>vkELV|42h)TeFYooxQK<^F%kWRH6zwK6fl2HXU1c-0 z+*<6T^^(l3A(&_kqYPOs9k^%CBFpOszOtwZwThuu-H6v5!}|>Obv_d350aE!1rbp< z+7u=EBy1k6h_=6#5J3$F()XhmMQ+c~`-()82FkIaFjk+62Zi^QJWnlb!nGcE%L`qc z0o?y0hLqw)Ha=qlM_3`6xmz`%Ae@DQw;S7%^_Rn79gp9ITjqMsn9)XohA{P@gK?yh zA{bV1T8F8g#C{cFq2xwH8GeI9BG$_2afW9>WiTvf+23uGIOnQAFYhBc~Bqh^A3zi~<8bQYQ&h20nfNoQzhx*G+p`_yd(qze_8 zqw8R?a%=kzan68fqXL%1go%8SUYHLPk+Vf*^`*(sL*m0mNTlW22r|LmCX&Y3B8l!_ zYZu{n?|zQ?%AFKCb#4SF9=iCFj>TEKVSh+$QXb!|yOW^^j=W}rRTo1l%B#t&42Tzu zz-Z*<75_~$^}2?|1zhXFNpiPgG8H9sW=6RT*kJ#DY)15v2RknBNwMvbS?8tJudhMk zy_8k!j{V%TF3Yj|q8l!6R%MYYBr9Ca%KJ*iJ0Yk(%EWrS zMUQxaGMyXha1$rq_(unu#$P?3mi1U*<`I6kXbXZV`kfWmfwh1lPrzBPZk>(Fwk&++ zwW(0NBa4*A&2YCpb~v`=_O&X?dhnWd2b*m?C)5hmz>OY^=fgdw6Rjg6kwmGPqB^6NmEE z4Qv$3TA?yj(!yDA#>hVF)APXZ261sL>PFk1lOX5%kGDH&@jCDp4z1OyGHrE1$Zr3s zG`CrqSJv7Ee>W`YW)mMqp|-9Uo}ij{t@fTH@p5)3gZ%CVm_vm#9r|a;T^mVedjuR% zXg8R7Z}DLGrukEB7dOy_AhbTlyB^AX{QzEKwa%GW2)1-o|q=|Qi9gf=uHWosJ zJ&ETEd$I?L@cZ}GjdSru%XIyFoc5~u+ePPJ$v@vAcdXXyXu_GIn)7)*lyEb_h5CNV}-kM+So3Y{FSs^D(9RP znV7jumFMe?Jod%N*)m*HD<=}v@HT*_(?X|XV>0%BduNG`(t%?+-Q2WriOyDcW2a5o znngjCozbMPMSjO5hWMwR)48L=w@v~AGS3*e`2(XMfVlAgkM{p>+MeLg#-c`}aAtkc z8gL`YD(9yq67Ub$Nh7+gyGG1DrLSBb3e7Hzshetk^wuDb?hK=^Ivfn>}qbaoED<0)5lQ z>cB(_oF@IC8?1*0?ro$2dxFs@IQ^8d*J^W8~)#1 z#sBOiw;Ki{mWFgAZ=L-LHkgQGqH+w$mg0w`^5XzNSo-(&98my}d>_ zh}?`b07PPClfMB64)#7e{_{)sg1EL;O}|bPD6*%6E$QGF3!jZbA5=m{!hfGnqQ!{SgvMW12R^ z>7|hA;*STE!Mq5El#mP|0tL4+C=Bl{h~AuH>J5GpR8eSP*$yfncz+*fVZVtX6IppQKYtQP+WAkwP1Lf*n#3=Z+knD)HKV)p<85m^^#2T=1q##mh zAGg(yYC%a+BQk`(8Ig#C`w=lF>XB>-N-OxMo2l#30P2|3Q;F62!kxKS_{d_v@D6kM z6R<5YYRFFHm@I1oH? zuX(e^W#7OS-fknm*2f=o(hgT!Vn}uNHQS}JOwg3OpKtETI@6&)j7lqVZhplZBw>C; zyO*o)LwBRyXWB_1*O=)cH)iyMAtTb7bL0n(GGs5L=Z9b2YOKzW%4AYTT@m-S9&Ev$ z1S+)lDnaG*ZopP@`$+Y^e)ctm>#@OS_AJlv7hHruPzJMHPMG$MkweL=X@`FqT<6aQ_b&ZN>3ER*%G@3|7g91wseWO60KYlL!=kdIcF`0 zTz4C9pS}NOW+54u#~jCn;Ud*#^LAVsy9F=sxa!sc3xc%HKzUlB%u4<5xFgJUGHS1| z@P{*5C)0E7LWe%5o=)t_Zuo78VY(LBWpW`pZVG=pPRNw*iwiKVI}mj;Tp8?Jg4-J9 z&h9kFoO9VP{Hi35)LW5#Ls9@!aU`QZfDP2xY`;}Ajbt11^m&?iXkAnhs8HE&j2s5N z{bCa)Q|}2G6Ey4o0ymq-C6Mbmtmsx;^j#(8i&xtpg{QzlgVBY*;SynU8;PwpnnHKw zGpFJ43@*0lvQPnK(PJm^kkn@^&w!`0r#=fI)AyI_a&S>(eH!gL`PsTlQVV3-k?f(=OY%y0o>*zm$_lRwd$Y z?c!N=X-@Pn>}}-yM$gCaX8%z`p|TZLkTEbbF2aLt@*7_?nP+O2=+V~m8;4!-ZRTPGko3!b%PfsoF zTA)*dMeHolay>I^+cmDMvD{IOmgqq|EaupRL_vfV%)kmX!}X-%Nn~JQoVnGce|%WB z;f!)9yCi>tX2m-zGX(6Ga&B%JDQMZg1tU7JQnMpv`I71CPUX29Whn!?aQ#)NQ#x^0 z4~&KEc5ANgvo8wSslDtKLjfP_DG%jlVRT{)AH9MbVHvFf$zrM$*k62k=sj_6s25oD zHAM?*@4O-fCqsFB_Ue>>yw`qv?d#b#`Nj3|Q|d6PIj&sw-@e4G;GrPGQNc~*ps&E zKwa1BSTSN!fG2Wq*X3&tbEbd^6#dF2NObnCOFDV(j6r77_G%16sDT}PV4D00SxVsD z_5KflSgWN9MKHThM16kUcg^|rPRU0{X6rvvx7Szi1C(o#QBN-UA3@eVBaQ+>x!DHIL?o*}HosNbT6IJ)lP&w)aYvBo6-&Z~4_#;EV zt~N(+YRKiV+s`sAvtxbcE-8bDo(`~*@I3pytma$r8}PcNK(|Vbzj6%i+eb3FHB;J( z;E6`L(>>k|r|uT##7UPj{&off%eSmZLjxmsXD?a##Ez3-H)6x0tZJ%}Meu?l!%5Nt z&ir$`0pkJ8vfu1^Shx|GEOvqNU{C#E18Msl6-Tmh`%c^IzFs)xP<}9r%Z|&q{e@<~ zu~8F_8^+0No#C;gf3N>E8Hc|OzTB~r=HsTfE|xtT=*Wc@ZaZ@aegIlf47emDsF}kU z++o#Y3l$CNPN$$y=*@TVy8~50XeIs^wB|h zvAGre2DcaoFBSh*9tYqd5~CWucPgeE6v2ZK7K~+t8Npw&iYWn(WW>ur6`p3Q4~l>Mu_M{p02M-(p*?fUmoXx}8A?=C(1qhZTT{n8q8PwY5lO8mfqlHr?aOyz;k9`iw zDG)osjT>4I<*_E#VLT#^hx7g!T0I(FHH&93odUyn8*Ccss7BjY7x-c>{ac1rxPgRv zD0QfSHdtWvGZZoT_Fjlk1l(`p{CFbT0-K3#P zk6{(ST6FezOn!zs{9py;hMRGT+m9Slbou8Xk%jK1u-d!n3o|{z@)DuA48~*?DjbE~ ze9r3a(>R!zk1zQ?^1>H|KDhH&u4NWy?kH<5UVZkTe)a=5F;%>x1Dz7DJ{;^8Gn=l4 z=N25q(gM2AnQ9%kor^Qm)qWOxnMVIa6DjfkGbrPKGeQYo5gdIyQh2j53VGKwUJ&aQl^}=JjuJ zn;0E*EgGZx4?@y8;&Z?e1K3|hyv+{#8?ye)V0Gs0{w+Hce&&%pwzicLzrhA{F%Eq& zkclQACCP5M;=|e%h3v4_)k4x|#gO5eHHo|qS1a@m+OjxhKoiGJ8IgWyaFcf<)uC}dH|}_Jv3q_VA8sPl`wiFou9|V+FdbSg zLJ%q~DZO;>*wO9toxL;15w!amIvHMhtsArTeUanog3u*=L%_36VF&UwGz6U;ccXET zG{mUT0eBZxC#GLTUH3`O4kG2=85FzYwECg2CG5XbnHk#h=x^`G=?{q?ycA>CnHsDm>dU zZJy5%f(AzSm!qe=xT5SLN=ey7-st#4iD*cy1p8HMff zRQ>wRNK((u&!G>K?n!O4H{op!jym>SNkjBXKhw`A%Yoj{bpjT>B&f&64*dE4} z*4;Xf3*Gl>mcJw1ECwD28C5b0hZTMsrq} zsw1&Nv555+6(Ey|xr|V**1?3=jVEhWE_$NNp{TaRcd<%pA)dKFlz+b$@~rjDO1uqvo^#Cx-_b>BG>)iwEVG2TeNSo?P1qf&6xGXi`ipEr0|du zojK*o&JJPmu&GhV?pXWYcZms-pNkWtFl*i8;P)X5oswg{l28mG@&2mXn%6>i+FfVXLsQRrMn-9VW94I1+YiEXhN*ILs}{^L_a{XyL|E@ zpFsDaIy zhn4+tuZPMGrhF9|e5Kt^eArcH{m%NR{*}Uq8TiXyd4myaQnXM_dTH>0RVAvZHZGNh zuR=b#>aU84eFrcn8o=U&AeESxY|kQtoHiW`6)NoJMt>fwWPaG& zS!VN7f3xQ9uX0M^V42{Tsf<3Xu=u3a9YWSpm1+s?;*@HjGbB3mXibH^>i)`(R;2Y} zvN;PcPYi18R=MXYkI$Ms36P)_tCpt`18%;SESDKNZY*`VRYQy5tnNDmmS5Q6A3nDF zqPwaN(}3&TFcB-8qhm4=a7uara~WpAR3f}KIzlk)WdIu-m}J*O7{&t(=v9GC^pK>z zo&=+gc>pg_uJ}_CF-SY|{Od0@xo>fGh}uAkR(TJgr}i}QG1R!>Lhmt#6;c+TuR?-Z zCrXVJ-h`kSL936e#IR}m`)api(iba}t&h6d- zj4c|^T=K(eY$g<3=|r}fjsrCMmMJAY$$rl=)s-h*c0QW-=J^MjMt8F%@e;)Ocy>Za zeNo^VeMzTxze#>1!u37sLH(&j4@(;HgH+zB&k^|?7 zW<&K^lk9C``9d%?I`{qh=)9dj%tv({kHOIds4F?5?OJvA%O#wnfp^=h`}vizDgnk* z#npe;IWpE<1kNsDYuzPGwcJ!=G>x$x8Uj(WoIS^AJK6AKt4%AC9|o-7hm1ze!~8%jm>SecfpN9p_q$;G^21vBj{iT~zPdcHB4^ZGKZ zeF^+(0qRAAORsrz6~mGBK6f!LPr!`~n)>8O6!Kh$=ID@1RIRRuVJ})>z*J`H$3jAj z3MK1q5i+W)SlU9Zu=j5^P^Vh}G(`$;9l4W2eTSXkPo(1I+F>R2cflb6tkN<=T;n^V zl2u*0Bwmpr41jr!fZ0)XT3L&5=fE3z7~PX0j!wpzk+C8AZS z^H3owoy1CF-0N(!YnW-YUYPD-WG{h}cSGL$+dT_&a!KqdrM66)(MN zF=_v2nOMmit)E|mwZlU~aPN#Da`+%xbHFmrIeUZ0)%~`CEjR*!dm~|SvP%&d1+a0e z_g-g=9U(pBhmai*u}pI~4tWnvzFMOROA2Vz24!qrs&y@y+=yiCr?I5$sbI@N>YZ*$ zPDJb@{L@_Ujj?8wYu;V%=?gS}my)ccT3dqUL~Nc0}}o@(7`g}fZ8Mntuzw=en`J2YdHIUY3j ziG@)km`vYxd)-=8?nO5TV{y1N28DZATs@QCEPaIpD>16Zx17s!@A-2%Dx6+%1Pbo_ z{A;>+`wx#=B@1&lS^%0Hs2jJZI9mDiZl`y+HBtRtgC-=p#f_`iJso+&9JzZ6pMbUZ zn29(s;C8;)V(aDwyxX$bjpnlazpY7afe}8YJUp<}?Yc#6l`O1RTN?Fb^|Tn&dRr~c zqGs*&ssqu84f7(Fj8+TEa(Y?Vzf>?aW3UOmu-AGhMUdrVVq9d zUx_s*m2YXer-4R&`l;9R4*ia=<>}YJxs@W#g^5ur+$-RqdK8VI<(vB&Hca^OT35S~ zsR9j@$;LS0%keq!5(Uiq1>RurNE4#awY>(^Tbak z7<+6E_^P_(?M!dj6^Op!VA`o5ZXdycZ%Mek|jt zmc84XV2_NAsXJM&svfFOf+jsS_;kP*qX%L4fge8!Qnh8%Y5sXMevrd-^BsO>9NTV zy@=~Pi*CatVnU$B<+&^~_=6_m0P|shrPAaVf;H_l#+FbWJK))AOR#o`?qbl0?e`Mf zXYgBVk~vgp7q~QQD=W3RB82I;H+?*Pk-9Pe2modtr1BjLgd2a#dQuw|8CJFk^G8%`OJ^n zbjpVs4ZwO7g72$K=zkNkbB(oCKW$3-;tV(b4hyRF%vd}{jG%u_tL#!9BytO7ovcw| zLH%llmB|Cv8=IZF$h;dtfO&54+AEzVf}Y4i_E5CvcB3I`MxyT#z)9zE7BhtbKv)g9 z+3;jfgiL5eVpjqu7UO9jdupEknDq2J?3~BlYCO#?swBNnl-g)M+E)=j2hmg-P^_mhU`fL5YL;PN80^JrL|R>uC+>I`rEcn zE#{}vcEPXxPjVl)l$?s5F^O)Hu*W~_>_i%WP+L)(LS<`FbfnTe>20NuK~(Yt6z;x# z4nwR%NFaqduaPMscd_1v?cRyZF@^>`XpvS-2VgS+`gK^OC}B@OgjM2#l$B`oXgf+w zdrjNZ|G-gA^(yGqw{%?BxVDimx$?j*E)3Ir`YAYdp+ELH<_&5fICgu(6G*UWxYQB< zVZQiRk?dY$>Ti^&!vf!z&UH-s&bcKZdJdm78T}iB^*w6St>byzef{*N!e0b*MQ))R zUPeXd#I1F|wQpmn%P>5j(4++ZggoWPcV5z(+7R7epDTQRxNFEf@sV*%;A8E7PTKt! z5p{aCB|4T{lzxrx3^mMB|AL)2I@iVDA?(LN84&UiG z;_Y*QB)RALyTmU5<#swa>v;tSx#2(kZ~iN1H^Fb7%bK}t@DpK~O8Nf=GxWT)MAP-d zYNNmZ#hr|@3>9^5Be>{Ceu@yDqIdQrAF!C(+-4qTJ1h>;v#Z81wq#}(pBuELbua;+ zhWW^~;aK2nQ}m_T&K~-=$BM?ZR+??m`Sb%QOXj>7(&$`={)eAATy)*JT=$+dxSrcB?P9M4*HrUl$gj?akvi*pBJM+`v}(&n^>TB~-MIlyv!O z72U)dKZoIGXVHa_Z4tZ4iX;oy>3<-k8$LW7(uu!+?n2tr;QPjU0Ro9pa|wYHt&Wxc z!#*V3{d0QO-8`gr-A0=SME`-LvDs9$P_N9n*fSlel-uwVVW&b)a+GvdZ{(g;7v{1U zr)j=R$?wlGAu?vFpzW$N%+$S|&WI7v-{%>UTW_bJ&k6#}@WDGrk}7u^yKge@;c?G0 zYxq0My&ho|UmAzlOf5FgzbgeDQIpF1=vwQEfq}&FkY2j4(Z3T8yI14*q5YRclCt*Y2ouS zn9iF6l^rQI02F_v^FsW@kSpD~Dn5Sb>H%*hzmw(SX{WaGqN78mx8Ny|3|>)xfUb1+ z*Kgr{lHLQEC|n_;&qA1cFkA_Tt#F#E+5oQH>@NjDS zM*;+r8yQkt?`H4M0j#TcI8Mj8Sv`+DFTY~Dm7O`%XWor{3B=zH?(bA)D!Uo;QwD`d z1BGksD5ulXQ%#E(!#vjx4kls46#dQHJU}M7^Ycv3uCPnP>?`S3jP@71Dc?i6+Mj}* zqw;I}$BXW)I|>mzrz&wC(`UllJ;V*)N*Y@=WU|k$s4jlVc6;qUzyc|VM7HlbS4E~k z{bZ5@O8EP9ThCkbcaBZUL@FS&Al1cCId9|={R-lEc^jxJ#~YGCT)ww>Cu_oDl`je* zI8KplB_OAz{b^e*;3n zBrJ>{SqwxZw-ru03!`Y@Atj;ip_v$7GUf%Q1TvMpc5I;@jKsu770VK_x$cZmULv3@ z=tHNatDEVG5*yN?IyINKM4^|*z)sbOMgstzs(Rss+#f_ zM)&Y7w1~Kvs*Kn{cKmzO*^bG;2|UC1v`%rNVaMO)jq_@EO8zo&PHB~@dd#pMcyi-F z0V7oYXX!)}lY+Fxyg==L+YP8VatKZj@lO0{7M%S3;~67~^8fKS->itv#Nnc813ImV za#0nWRUgn4d``URl7T3H=mJxo{017yQB|D(M_$pmzEie4cbSsbDhk?p6HQ%fY~}5J zmT$>+i>i#?pTy9YRqSKZ8Lb7fMGp>y`Pgt^iTGrc)t}^_0iBs8*SIwdy^GA7QFfpg zsAfNKX^v@b-5a$zkOY0NLA@jR8(xqMrxwstLHol$Ut$Gy(1<7)u_$gP#Pla+$S;|n zmHXBTzVI(zQVY2!YHiHE)!r?#D|%re`{zVJ{k;Awu-?BOBa}r+S7ilV&3@3Vfqm4I zMcKFfnO^YA<6WuKW%5j*xP@yjJ{v`xiS_x!#|(CR!<<9vLG4MkUbkC^}Fn+Q~-R);W$t{o3iNxa(Ng_lGWb0M;TM5YcGh z-xltoHB@8j4E_4bVi13NONBYVVnd%Ph&bd{2gm(Oi z+PL$mwr$X%)+s%Qz^xCvf>73{n{e^Gk$cszk$&AZ;t^(EkQSWK*E1H~_M!P=NUn?7 zaI7o7oXqG+w=w$-`R=^JsmS!O7weX?wo^y+yua(}X9?LwROqwaw%EdWSrcfzwFSDt zRIOukPU#M}FE*K>ZKVL_AO!h@YJt^tOp^-}rgno9A)6@p)~aT@6|1`Av_;2ULM8u9EIIXa2@S0c;`GB2y~39{sa>F@n^=^ir1#rR)ktnr2&!6}L5I>4{>N7~18j)@+j{i2j~B#W;Q zRz%J3k1N>r?l66+_ze-t3reFlqGaR#d5@4PK;acvDGA2c!Oq|)W8J^6^$?R&U)BPU zfraz6lFAdfVoF5PN{w~<1mxFQK%`WLo>@(RYQ6hY9|TGkomZh@7V82{ zc=-i3285HWjhxkx2{M2gDV8y3Wd z*St&nlkMM@hw`eFyDzQVJDL=`b;(FW*`J<>!%skMm_0e4vvOW-X@LmA{@V;@l66%3 z1EOPyqa&4{zyLZyEA2a5kbRDB_b@?VeClq6sgT*N_ZLk~hAi_vt||1cD|Vdb44d^l zt!UkIDaUewLAOzP{>ci8R-TjM$UA|B5rzA7^Se^Cp7i1~FHbJa0UXpZ%HsW-l$^5w zgx5Z6nbkipsq2fNuDf?hw`PG(&KcKdgU0b8HoLRGc>2c{%ywKMH~^1*-WVl3G6#+( z=tB&~j?f<dIs#o=4KwM80^`2Ir zdy4{v_?stDG#78OOw;b6Nzes+EkrLdGnTzS;9~as;&?A6 z2Mn#YRl4ORCJr|JD>nk%lW_MHAU&ndc{k|#fZx=u(Re-~`u6izmEm{*Mj16rHLU5{ z73Thmcu(OwDxZvdNYk0KXO7GNy?wRgwEQ2I5zkTN3Z^Ty@lzy4G>8dPgJ-JP_ zy$g7yl4f>;Pd9p#(LQ>l;F+TL)Q#ed4~k&SUpBtJu#0D&qe>&ukc#wKFwdF%XUWpI z64*5AIO5wtta{Qd(UjYI4zQ>SVd~0}=QZbmG8BMW6u3w5{uSV{zWMVP*>~klLMB#r zyhr7>pL}(VYR|v?P^PsVAjzTc=tFRc!v7eS$0Q}8()<_qB7HN?V*J3*HCxiGbKIoX z4ebg+jhf01?U(__(xFTMyRH}!liH%665hWt5!C4Mp4yKHg2y~r1Y?xJZ#7;98q{=H zD1b8-d^euqZFqZ4KHXBBqVDBxYotX8LUv44D|>U=oxXop+TMzBCe73^@PUan*9<>U zyV4R`d~R^aqI2T88MG+6%keJdYZ-oyd#>~~Rg(j!g2f~Wk9{468_G$?7?rJ%8xJJVjw5bnGDz!(v4MTc0M;XqK??q;?&a@VK z_}f9Ghphas^oN8Y+SpCoGK&Cf?j)*G`2l=MnnC?L0Jq`UT_LwhOih{4LlDt|{LVeX=o)SWY9X8m?rugz5`DU|Fw=%JkifYpBYs4+;km7S2p)8%3 zmkycq11!+fhf0bgb_l|vFL&-#I^Ln~{74a1G>f{^btJ&yrs1(<&5aJ=wgYfUc@p`m zPmbAQIl&w?{?PFDHsg0u@90Ad9xC#ygvAD3rNtU-$7Su}jW*;E-OyU;LuR%VH7@G2 zSZ|VMZF>yGBNfn}itt)73y-xf2Mks$rHNDCapwC(!j=XyoAx~o7EgnZ!@~Mk*E@IK zHGoTlN4CIB3uj2Mi%!?K3;3`4c2)Us`p#=!*4Nw!y_Uk0#vDf#B9xGTaENm~m3PL0(Ej1+H%GfjOHi>lfG|3Hmm?-sXz5N+0BwJ9fx&Cb1&iC@I)0Mah;u z9%b<`$|evC%x;r(?ET&AwHOlEvEx1CW_0-SnJOhPWVe=-Bj|^TpVXC0By1?Th3s1}pD`Z({Ic*{4=WEzga^^;iJJN_~dWTj( z%O0dcGnzkycdbcY7B;iK$pZYH(WiGT1k&xqYSd-Gcjy34@bx;l|D z`j5soTLVqIw}ZFiQWzf0x6k-0?db%#bl-BQ#KtvlgFO*1Xy)2?!`b7Q5{mS=0>6y6 zLk`>*uBvMx&9Tkk^2@pl-L;}|L)F1Ee&&WoEMdqP)HV&kuzr;*XEXGy??KHF!SVUF zqrKOvGJU)?ZF@sY0+Knukh8`cZCdfY;%5bf?MW%FSU zF(JZfcIuOT`y0C&HNnQUA^1^TLIY3bjKHdLoM;{g|5ou98vU^m%ec=RJc7vL^YT3h z4IbIPaIb#5v-ISR_Ib^XZxO7&f3yR0NGgeZ1oe?Q|G%qoGc)x%%Dd~3>wba)eowf2 zO`81cDBpAbUq**LwdVg<#8{7~{gVec{sfZkUZ5Y=d+g^$j@7YRGc@+cQ;nR*2a)lg z?fQ2AV#&=bgB~6}pqvRXSHOy3`MB+uQa?Wdf37wCH)8xBHMh~gz3+!E9}uJk&*_%~iFz_pD;`5&^(UsY#NNGzs{K~?t}A37e3oP75+?IccIStFdNQMhV(av=-c zGPnZ$!YW^Pd&I||uRGifp{9f_P!2E|`Ym+UbD^wuDHl*j3(aHKXZ1Vp@SN+=WKV>^EuKNo~np6OWmU3x)F{3#k><^R%M?MA#-d&25YKQrwB z8+YgLmb<`?mT{v;nhK7)H%Df6%3#M=Q)?6hQb;#p!wK5tJ8rn}_pQ)jTKl-`#jU=K zo1FLNHXaJ%Ha_E>R@z_j%mKl8T*e)$&8zWPZBWBJ>6{{`bC2p&Nb%iZ?uLT(@x*5cAjsX0m2b zicOx&iTX!KG3DG#C#kQvY7~u(i@Q9^B36$+94fuR@LL6Ve+GXV(*MPD!5G)6D?KO8 zR+!2;LF@0Pr;VjcQX=k-dZlR4oKh0Q%(rpR>amvIME-{P82(3xbKC$52Mq8qmTI3JoNClrsNCO-I7zEe)*Io9 zN^dK4`)eoEq9i`=*>beKdJIe7DL*tt`!YBYPbm(&9n&!?HbNFwnOo$e~-dN@|wc@rD;P#a?Vru1c=Xd-?(`ETHEr zY9`)Q^hp-!lTRDxx@)kJ7BaM)VT9 zdx3&~F^TeD-LeQ52i2UEc0sumt-I;_XshpzZw6g?4gE~4)Y%E5fI0#4W?6otE7^?K z(DMzoeDZx49(rr~c}vQMnmKwEFiG`_aqHdU_b763&@uJSW{omwUGtQ&w1>KKSL}+`JG>pU>&__VT6|_~7PT@Q85MRw zrl0zguMT|UGhJ15u%8pgRfZTSqIO^AK&`*|v(<)C8e#EDLw4^K{{7eli-7Szl0!zl zBhBC3EnvUr#bbf<}{WYcLG5YPI*XqNI&p?vxU*?Z(JB6u+4qW%$6E9_g6} z4XEDt5MF@V448I3LdYiqfcMh0yjApiId_@%_(}|pFH7F__@@x0l{4GZOZ_Lh==joo zb|-j!^#k_kflj~jN5`ky1z@l@-0y%1Kd$ya3oPq}3lRo%$bSDeFXM{fBF;3m{~!Fm z1wOjUwhGe9MM3}Z_ks0kIW}^4@~N^ z3-Z;HQ%TK*TQYBBzGgvISf>=@~XJ&(}-psgWLeXpW* z7PoK4YR^s~YA^kx8{4%R(e@3#j@sPu3u!~FI|C<3`)-`tIGQO1W9(^07fL+5G5PdS z^>l1Ul>gsiEwc!5*K5fB4N1_2(l}bc#1IB!vcLP~w88c*bol*|WL{Sy&uctI?n`&* zVnSnOgiPl+Jdgf(g<}DeQDtD&{b^>MLk~{@f>X@X(u=VuRh3u9Z!`3%x=Bdg!`-^O;5;v=` zHIISlR--^6F;zbSzP6FZUD}c^!Tr$Yel2L3T{AOWP3$Pc)8!0e`r`#x%8G zJLXFqEn7dMl{m|0)PCENTX%4{d&P9fgdadbLnpS>9Am0ie-%2V%~v>YT}YRhpU8gQ zd925Vwn_mJ9C6X;kf86PwsAJDSku=nNgB_|pD<`$*Z2EjekSqvz=qi)c}eZgNA{h? zknl3p4USxWU{*Ku32~13x+QLq?4;vGL%C60b!JeL#Osl1c~|$ zE0*tN4wQ@=b|>q#KUME5uPf(TIJ(&`3=B=@u%+)-7@z~0`_s1FP1Xo%Rh| z>KKNmuzkaJDR5kJZ3;saGSoZDQKWsLRuG1A6FsZff5+l3>zIZBEw-R<+4avWxp#6a zaN)6QXENoMaDBvf30fm`YC8pX=!9uUmI$CC3RjSM%uhR^8feCPww`qxSY?+TO$2 zp@U|-;8i^}R9A5IJmm_pybU;Z`AY^v0#I916fCzFV)h}A;JRx#LJV@iAvkLipY*y~ zWP2Uq&i^3e5{m92t0RBly=%zrn;~~=q$Y}H9kZ#Wl-m5Xd)kM49HEEh^I`^aSw>F+ zK+ap@LB5i(OSrP_77eO?rbSqDcy&%EF%@DgD>16|DpQtb&T2ao@IK8rcQ&TVU*<61 zQ`k9r%e8am*W}GS*YoEBua(Q&i7GJIADb!{!pe_WxRx27(zp|Oj!xTmU1cSU@+$m{ z5hG~^fe=G=3IO=1ZiCJKw(Diso7Z;FJ7o;4URd6}5MVda+y$sA0^A(S%b`<_LFlTV za}H-7vHI+IDb5b^cW8-o{TN$$s1cIRf)tzEu}%$nAyLm9g0H$9sZE3IV%rnPbZ<Q|1~D+0>i! zUT-yXtxMB^w?zzrf*5^ZNFdLw(9OSv$ULt%*x7X zp($z(mCdGRPBvx{#=leaclk&-a>p9`X)<5U!Fz??GnzVBgE=mKI z>a_51vXzCYK}&4plMn-?$~nKHux5%SH7hQ4jy|Cq7VSwNHo!YaDuVl3off0ltOa<$ zXU?glq!9-^#o~qGosSfqvsGp;hFwoJR7nxzW{S22Ssxmz2O&1rgAvjo=<%h1{3keM zcs^hjzyeg(Qk;FO>_g_4@5+nHW}onN%w^%EO|7aeG0kU`#{AdoysEzPa(jJcADYg@ zyK`b2UPJt7)fxmR3D1mP+UQTS04sl!?n_mIx*v)r`BInjpd9Jd&Saack*`cbfOmQU z6wzM+kT=Nhd1M>sAP#rjrrK4`FV>I$VPd!YTy@|RpthW!2RQ2jc}WMjzx}KB_dRz5`G@_Z;E|kO|Q%k%C&e6vAV6K3it%ET^8pd zI5eFkhC}6QOka{VJzW;X^cTqN7+S+M7V-}g0emRfk3A7OLHSC4u@4i0&7{zzFUV&t zseSb06x$N6ZLCveF;S7Y`blqeg7#bh7;;|=aD)(V>4zM#-Xr`wI}JeI5X?Rk!sNFl z!o!OpRiqnzEw}I45ZosF`a^K*AbcDtTs+Vty&UA-G8UGlLTtu~{N$nZ=WIOnR&qvy zy+dTgcEPMot6z|JwD}}rrwmwiSkpBLqXcn3B(=ABL-}251B=#Mnq~X!7Rd5DD*B6Y zkmdD-j~{mjY2s&wvyhQ%UXq7&d$MOBEQ@zMH`Jy!)=FLVw<~@j$rJU%{gv^b2YNk6 zPlemBud_03a7IwmMWmL09Hdg(8sGH+s2MQeV6t$Y18I07C1JX1d#4KeTn>pOZ`+-2 zFaSqM_lKEhz{utvneG67vKKK^Fy+u@UgQQ9x$*7Pu<-R3>$lzwK?}%<7P!~I9XkWL z&Tt=(kVRcW@598!Tur&Wh~!A4Af+IsLhX%Y99{*NbP?GXJ&MP`58nlv`R zeX)j0)YpIj*{j`wk0ido9Zz-ZzHn}fZ*99>2`W+++EajDBfRPY=sxJ&0f z?=e`}uc!-P+E6c4NmJhD_h?w<9c86J^~tI6&9YKat+nR(rExg8)Me$jOKaLW>0*tD zXz`m-BCr2qz9HO4VKdhFdz>%2l8R{qw3Lc49<-iHCgY7-N@U`U)K&@-V0OfVv7vog zrAnH>rNThByo7>{bWg)|3nBWSC(Vdyb&r=`&BPA*8`G5gn1nrb31-Dn!FAdIe@c7I zxbCz+qWB)Dk9G2VQ_VcC%t|Fr+y(@pLH>RZgmJ zvs`q1V^4ZqC(G|gn8!!sx=_mOuKiYPaXK=|bGTTwCck(J~4Y0($x`U`igP2ET45m^11)I~h zZ1YPg*L~CRV-l)(zFX4|Ycy_Qd2?NsAo;^ExB~Z*Apq_{U0mceg~~B2md!hXg+!ub znan%qY}?gv>6EU!8oZ?uTS=~L${hrjl!2(?a3DJj7r(aB6udqU(5Qh&o8rlu9Mz#S z8Krh*bdw(m1um6}`;h09W?8>P()1F;RgfrEVI>RqMW@$>YU;IC`6Z}orO9uyW+L#sxPR_66_jWciE2gZC0&P~YO8>rVS#as z0>RE>Lt7y@YYs2Wq3U#ze!B+_=}l;SwV)?Uls9D5zm{v|EHNLB$SDD9WO|=gPUt|? zgv^xbCaDn4)9Lj-Att@uO_iJr(Ls83?E19>n&qAj@$mNhIVW=H5XZ4&;IJYGkI8zk z^g087OKF6eE*lLH3?}Xe)$!q%cqx0Y?3RV;T4;G*`B^b@<0AOMyp1oW$>@L#=_&Po ztPS51g5`MwOIIUp_vDBVKFk`LVQOhgXhyWh!xcGBIoAQ~38XyIf6Qn57gr-n3H99! z^AIk8?Olq0mdp!$v$pEQjeMB%?3Lo1DLzZ`=xu5r(uk)u@(C^7anr*4SgiqCKU}9u zsm{EzqjnW!+hGBP<>4B*|2WooMJ-b&eCV=ilX$D;0yE*53Zxyr%S0d&$v^|Cu`QOnwM#* zV0Y|&FUP+Wl47__uZSx{yz40Qd90I`>sW# zlsQNgePeS6!^*>cOiKl2wG{G7(+#`7Li+(0!{~_d?~U)n9d+RE?^47khvw@4@Wv|s zEcnKuW)MnYjZ^Y>|H$OSMu-}mh^6OBjiRNsjDAwRt8FP1bj$qSGHBaCC;WqJ$BvPf z@gs2kYEOs9=c*6F6lIh#!i(kvvvf|k(x!0?`F^tu|8|aHK^<>(#CVQy&7D`@OctV-MO7THp2zFTG`#8+!G$vO$yLTO4j&Y)kyr|LV%RE$@&0LT46BcKs!avr{uOS#STH{V`vn8)*+7`KcN{siU(xdAnsjO#k+x zc&zBJgg`~J+@+7_Bus>zyOhcWIPB&eyV3T)?eYF?9sX}i426GaQBao2{n5+(y<*YH z>5mk!UAzI3OBW6o_F*bmy8 ztYT2S+0CgjBWd~;uFJZwDfLUcKZPQ6@vKfz5xqM&JgB5*7K+}enW50{|V8(aXAm;*7fm};h;L(=@^)yiJ@pI}KmTvj>=e-1j@7#Ao9rd-EsNe><{v+43 zT;T$=8h=$C zHnVBNLgcJQyl<=cZGUGV{h^j`7#vOst1hZ4*4e`ZsoW4OksK&R2*30&m|JDJJvjdc8K z$L&c&2gjDYOG@#j8LXqNqKdT()v2gMV8+DPm$hAmaTCoo{kzj@MK=wQMXR7xtMcNu zSch?HizayzaY#Paz-~8kMhL%XliO#O!()0e!j{i^a^&1b&QFM#9M(t$EUc4d4>|rc zW1+CR%BMB4jncl5=!{L$jXg~QQ*}PWDk;94^E}j4fntadxL$*S6iGOtiLhyBlYt;1 zMkrLDyp_~d^j&&idX%%Zv(bC8wP{Pn;C)^>+>qXk+%114w;ni=fR<6BS-WIbC7N_A zR;e|U z(fy6F(Z)W5-{^9`IEH(t;?510L#8{mdnZ*H6Cf%7Se(QbJ0Mg6MtRW^cfOKj^gj|U zG~x!g3;3$vUJicoF=%tIsi+CFqnf!mSMwQxM+BLiT7YhiDzBgJ*b%be2#RylRjx6R zCBlI+V|u{A7zpkUbt*Q^&J!M;=k?{hR!N)TO{ICe5VQPHW>h;ccCzp-&03E+TrC*G zKLTVx3}ps_k1g5ACoxrq9-^6uHgzKARU$PZBzgJl(CBV)2IQmgrN~iVU0k>4QOw$i zBsZIRC1qd%U4ATX?bV2p;?x7NwNnL&zrEJ|aU^b@#ub7z> zU34a9h;8OYWe$wA2i^^pw7Ivm3R)?d?&AFC-T7}vMwHjSAQvjoh$Xg6`uK z9kasU_YldFLF@5{1pV7DugZFk#ow=d85?-Ha_5)%6;ZS@CtOWI`2Iw7|(=Q+-i{oyRwqx(aJ8;Lo* z`d!%4P^P;P`}cfCnNCRF7&z2XbYPQ!((7I6qVI^O^k|N{>{Dq_k!6z%?^qXxJxGVW zr@{K~d>^?1-8WD?T^b9u)94)?H<_1#nl;qcTE;vml!u@O4bR_NPu^u+aQV?b?Cv>M zol2-Y#LEc)#v-ebJhs$)ZHQrbjZb@t=PnDnDXnSTz4cIfMe>8GtuW?NhmoI+yQ+&9 z)~qz=#r)Ia%AlVB3;r{;BXI&Cg4cWu;u~@i3%cuBn^d8 zhzjL;DM`Q_{_`HHuE%+hlTmYK&5qI1=Hnq{RGD@g9kU!}B$%)!Viq_VBR^a0Ii1fX zwWYf34jaRAcYSiSAt^o@eIY)sgG6n?_Y<1^aM13y{LS20o`Wy1M_x4&w_*3qk7LL~ z`_oxi>JmdM$0ej0927md76-`J62y#u*6XX+7Pe5cKDUg250!WBL#2)J7>(BV`1cY zK&2ou8roMNiFVw~BLDq{A(l$%#mmDXv}5HO0^<12%otpvBZavbZ0TSrZ@#(~1o#5m z?`H@ZlA#BdhZvl*Fw73uyLox01931A+5-V?;|p()Sz73f>7}PK`}=_?tm6Xk?IXGc zA*I=trxm+m>hI~~jG+Vvbl7$$#?!kCC9v1?ih>XhyZt!^{e^A(=a_M4beI!SPk4|` z7}*Vq*ivC$SyUA}#SFP-lJu3~o}x+j*Ru>zZMk9g>rgma&fz&amD~6!c4F}^vhRrX zh5(LVlRJpZfe8LcZFZJuL~;luU7T;W;_o$hB#ENE9iB^Fv6h#~FOTd8p$&Wm+(vtY z#tC3^yqYibJ!r`sN;89iUDIa-zl8X-M0Db&7M{h%L{{BKl*mvR$B6A{t}gk6Y1KDS zyXOgt^T=#ty$z{q6T;(5{ef&&)vi{qADG@U21TL$S`&lKiHR5ZqZ`iq>(Ka7Hu5#? z=xz5*+vN88#ea7zPMG9h-3nlU?YNOSFf@@B=_3}Je`6oN%lbR<%@%QL-WZ6L#d*?l zr!tXFNOX)GYeJ&ku!~>Z9U6>dYp;%0xY)F0`IOpeOxN|k*BVs4QwDOsfOy3V?VB6z zx?GWu6!*)6aNOieXV^t64MfOuG`6JTbDs9ZlZi{&w`0gfDLdC!3x>`LLFDU-K7=ps zi1b)QvfNgO^8F;OH`x2IvuyNPQG4X0S`WyMWc!b|oyaB8uXFAvJAvs*T62%Dk{%1A z-ZcJ5n#puzFSQrY$BLp9qt%MEQbxh{TVw1RBLj8xyw9;IM}W&!g$xqrOtdLxBFGO2 z>ce%}ubg)ybQwdfhEUxZVfo?pj{p%rUpKc^fFQI}MWyce9#YE5ur$5t&hdFehT?1lF6gkr~!OEZa6 z0f=jh*f*{@XUVVHagAX={0&sm*gnCtIlO^qM6sDOv*Ijmq=)u8><5z*mw5zT)Xsb& z5PjPy4ZLB!93-yoLkZGMYScaaaBaPH^FZUh=3K`F`n1l_C;P$T7ufZX@Q+S^U-)xh$C({J zLMBEw>iNulC$?0ZZdmj927jMj+tj}$oEsBxQe9z_)CxQp^yIx>`MnpNUOIDEm(SjJ z4^&eHH()y6kMA1V%YiRi7-=8?j$sA_(t7)p(^W6D>FA#|BA{s_Kng>ZpkhlH;auO!nj&l zS$jK&fA6p#{d%9j{?C}ve?yx{MD>3`8}T!zm9&J{l<^k-vHi<~)AQ2!Y3S9BBA*@G zPRS>gFv?xdf2f3wC$H6UHf>|y5p#ZQlA<)l!(TQ?Nu(Mv#@Tg9v=W3#lREgvnUz2% zbq3}ta}657>pbC5k&M%@7IO0L{mEa}4q#FLnts7*>g}DnpL=w_Qo6e^Ab8>oJjIy4 z$6&v=uH@6`!Rtc@qlW@P2A0QWlOQeyJ9xpV!lyR}462kmw&X8PK6M{cZ}6e;7a=51 zkRG{zgBh82KaF>fkTi=0wQs-Gzs*U$m?r(kcB4dgwO=q$hDP6t#HBH7w!QbDZUov1wGDmCFPvV@1Dj;!($}KFV-V& zXkLYlPs%`ztX-y6o0n`*KsCib9&08ydHvY>_l?dB ztNG{0)_x}3d99L)@{;MEQta&)dH@UBE~3m9j9P0JkXXw_IW-OI)37mJp%*s-4De#D z*zlnQGz)Y#JYD~js%=?8>`ML`Jjyc!F3E;6Ra;XDqJ8b@7p)klqd zYx5!R@^*{QepD53f@Il0lgSerbtdhb)Le9el>4t3NbjuCB?e0Jy`{KwN-%Q;yN#eG>W7 zBB-uQ{brvTkWlGZhPN4K=|IQOfMo5Gl<6kQ173#)%L-NyfVsuNN-47lOuGnH7l~lP zU|93~f7DLusN4dtG0!yL%{hC-f6A4oQ=SLQNFPeg z@Fp2<++QRl2wDj%-o9!dF*a1BEfR42uDx)kg~XVET`t6&r8_OWa&bmcgeeAcH6y-j zQ5ZO*X(bLP8ie|RO(l{iboITR(E`gua@ypUZx^Od>y zIP~t+29`>X^@tzP4(ZropC!mwJt@BvMmr)5BB5K0vd_cRbMI+Q0d#SM*kjNcb0KlQ z4loy?XEI&t1TGfx%plk3EiKtN_ny8LcOcE)nvnz>AelS46R-)(KEv@)E4 z8EKOe<7;I?u^Zw>rfy~lJsc>F660z_lNLd#4G?{piYn!2Jc73 zkX~GcJ&`KcxAMwSMRJ0;dm|^Iz@5GW>m6}Q+umu_HY~W(x!v>eGnh2O zB^2)q(B!}K7&hgK2mPgnBo|#`3uZpPx!oQ5Y_aa6{}p(xB2^O6Jg;{wvSmlCvfxvo z{Gokzcj-vS>~_&(kp{ZBsVe9(VsAE`l4>hij)aKOr4PV*Bj(u2>BI6Cp{%aPlenoL zhxFt!w$Pl)93pL^C`CF5>F9e0`j3=$xM1$1!m)ivW{oZs6GrM;sURI5gWJ;2&zy}9 zro9+W`(rj{HUq%Q0aCk(%)+_!W}8xf67nd~ybklsud&K4|5rcrMeuay^S$Zh>@qeA z(6|RfikS2@gRWgdganPO1lm>=_xw^RD_WC+=2;tBrFZP4XO3IwT0{y{3-OI39Ki^% z?wmBr@gObeaXGcCX`4OGH6~$sg)^fI^A7gqM%OS68{%C|c;8uq2%q#5)=rCwghV33 zWos+h1%_W1ohsH zwe~Ns99#c1_6wNIlud$wb+ms2m)8H!fs3~!p!=Gmt+u+*5dyn7K=PuHb{L27b>7pj z^x<9T6=!oyOGdb_q3b*IsciZgymzp+{j8R!)9e%~mrh>>iFH^LrrED3T0&4m@qlSg z`HBTU;mF3N>!-w{J<>MZJT^Eb6eY3NJ@gNk*MH_QXe;LfUHL74#o=I6l*Fy}_;US#9Lf$W70#WZ zd=@wWV@3eu>l-e>B>8j}3)Y|RV*6OV#*!bc=la0kZS9E1tc@YUhMqWEHvJcG0sV!i zu`2xLs2!xgG$A`gbB@lztA{-vUKC6>Ja6+{E5Ox1!B353fQ?H@U~RXd{0H}6vrojq znbMKQ!@YsfYjzew+lgHIK!4B!<_9d+Bf>c=!LAMfVAfTAOm9Y>T78#1jGTePCWZUI z81=qHq`K(t`~gb1z9Y+iUg|;GcD{t%+XFrPf%9ezh%k4*(h=>n157WW{m}5slAK+1OVYbxE_&bgEjG6&!3TbFk1X|H5gnX&7uTjbKOc3$T)F7DJ*Uu} ztEmAx8aonM-(=CMsowMqi%h;4TCn=-8qXjrb&CBEh)xU519^jb;F>oxhcNN5L?8 z+f@^Ad+A^Gk*iL3u(yUT%5Qcs@l) zv0ManI^>JH!~V+!|9@_G|7H67pZ-Ita@l)nU1<|{RkCLKe(2Ye=TCc_syZ2Z_dft{ CpqQKh literal 82264 zcmeFZ2UJsA_b-ZAP!Tv5M5)JvihzhpF98IlH-U(NfQt0qAt3}sK|z`jlnzo8N~DCI zAfPC{69^DMN+1wI3xR}k<5!gP{qK0~zI(?VcK$1je}}w{G*p>Nx_B8^n2y-r(YwRMR2sv& z^W-Sw_X!V8GjAp)_AfvG4#7m(d6<~?bG7c?F$u6*8fOav4n8OELXSU>Rot+1$l~i` z?~NPg_GK0`jC#;CEPHt2TGzGnPgG`@PijLB2Q@^9hq&7uVr4gktK1=Tx`ayDr5;1{ zo)l{>&NEPh+fa>J^NU)(u{rw=UfXAaBn>wjYMjPw7R z`{*v_ZeI=leV$3aQK0MF$+_L*3x|H6oj=9XA#__G@VD2d&1`16XLF(NXmb3%Q{~+s zeTRSljJ^Ew;HO?b@wXPVRR5np`p&B;D{CZCP*O&wPRH0d{-luZCkya%#~8=OFT8a| z+3(;~hj`1y)9-Ke(Nfl63nh{TKUm69J9_@PXG^eUV)5tX+p&30S~2ODe|I;vM!Luv zxv}A6`|O!RPv#x&n~y_`jE!OM-u<`{yz#g3)B4*0X`WF9{ceWo-Q_^10UW1!RScI{q0E36! zB+NUs_Fs#Xq!yZ)&WI|AR!^uvqS~a3d%~&~$0?c@+af^?$;^oTB6F~n86QEyXy~hi zFFipgK|xZmIx9`Go+&fdt=A`Jpp@rG7uXxt>!DDh+v%T$p?~890 zTV?Zn(I%@91L9r*Ra8pPta&}PTd|NVe!=}P^``e*QLb~Z4ub@33cSQKkT7E}$dRhfol(mnxg(UD=}lJ$ilndQ`1b@_O{A zv5Vf4v-Q{O&g&$HSq6okH`si&noV%NBl_)w^yNazH2G=Ld$FFeY65RUU>DH*)1MP1 z2m0~uitr`K)OU|_!5=Z|gDj!bIIJlq6ISOuUKY1K$gEds+bLhRg*ZwYHOdb*VxK!3C8jj%| zuS?3GY{#cc{8YdHIxZU$;s6Hnn9ol$@A1#9;F03QS4hSzP7v>0*e)7RDhQZgA)$kav1m>Ku^wt1cnPupBE3wnBQv4q)w6DwjLD?ov#{g zzElk?MPB7y_OC*6tVb!vX5IJB?s{Sw=N_FMgi}n z;@Mka*ltY@sxik!0(l&-bXeZkmuRvVP-Md)t|L8CbzpmW_m|csv~RW16=#7#KAG+w zU)JT`{uDpNjvzE5J#XoB_KUCn9NS@^{3~NRs%*P2k2m;b4=DyB-I5foaIv;PxZnG%dSMR7W!sTu#f_xq zZ_dl{Jk8#9$XdNdjT4*1U4tNwi>Ji)F3#7~1@-=2&baBr?M?Lz6 znMsaIe0v>1M#E;5KG`TVizD+`K!;zBek3l{tjM_pX&U3UPyL=Vrc7_fCP?mCgiS5a zjl72#cni%O37vlH5|yTCW&bLy4B5X|Rp?T4XLK*S51XQ-6NtUu$1QW&I!BnW-9%Q2 z+N>u+O0xYN=vLNEX^F*6H3kmo@&<}Q*ZnCE_pqH1#J+d2$ddU&yE}YH1_T{Cigj8Q zAzZ)$vTEKFsxLtmHIxzSlpuel6lG(B;z-Zn6NPISCLA#ce&$tAj>Z z?PWc(&YcpoW`c#^1X^w_Xe-F2x)l6&%1849OU&yp*XbTvFND9TX^t z+DCixj$gLV=k@q>KB^>9kB<7uJv7x+yd3PA0p$pP>DvsoeeWv-K3ch4ML6J z?|`hRXZ^u-w~9dhfMn`-9o)JWWQM@a<1-&>iCg)eW^UfM=FN9jsS8I_`Lf4MLE*|t z_s3T|O3pV1QrbC)L-yK0QmTb!rWJ||BQ7ZT6Jy5*-jz4xidK-u&E@J?PmmU+th@JK zN);?6rAWC0R|5X(n$k=|Ogt>MSRO5Ms$=`z)mTO0Z`|C;u5zm0-sN?YQc`e_rXzKn zzi(jLa(&ofwXm?z_We7RS`U7CbJlmr%--?y=jYrB=_PO9p2}5#b}qu}DQu>1pj1|6$Z%Y}|J z)FiV7y1xMRN8)z?j;bJ}5a7-M(|=O;NqDIN32U%Rr}~Em_NU-k=k)?6rec-k3#f_l zcGX~O0}xsKz{f*iT=gjpyGF)?6K1+M$zppUnDI;K(|iMUfcdlrydib@Oi(sG_`nmi z>t``_P#1TQ#0>hxBAx0kktu^awZH~7B#UK*q<3*Em8T46>i0cNnJK6L*Ek;k11sTwi$`;KogEx1XZy9i*Bt{N{vN>)tIG`4l&H@E z;P1UL)nMY`PmlqRIL5y|Cmzz-CE}8 z2gAK}>#Y}keSKqPRU0q))8g?PnSYZ8AE9`i{cQs%!^4W#Pm`Z5p_)7xx4DFtdm{1W zfRHndlkWbR8B)ICV>lsr3GH77@ND}1q7Vr~?{bAJU;`Ek0C!NShCGuC-l^v4%TNrw zeXdW)BBo30fUv2Q-KFM`PXOe{4!)`N>A#$2x3r}t`31(yUL5paL!LRw`-(ihK}-LR zym-*t_E295UvEA2A6~QlF`f?UOf~s$O-O`|hGN1&y*vlcGX1~k82=v&*}pa|;eYGO z39eu6;OEuPHQhQ zZTzFISc8PkgL5Fhlze?BeOY4!npSw?y@AnGpc*XE$$AAofmU4~dT`8ijo{GvakcGa z3FU@mAVg`nXptZ1XnJ*FN(?KCE%ALcLlFt~h|EiKHvKiHAyx%b!?$wt^F;*()k2T4 z^@SWdoRFN1NJu#6uN9`$7q66w8}D9>9`SeF4l^Qce`hG05F&>OJczyydOYPKW@cKV zrni}^RJ4cYGY74An$x_B^@m;2cP2feoHi4knx|Q%7UTF3CoI2Va;vNae!ZC~t)E4e z7UE~$v$qIPq~ayx=7F@M@_ z%ZJJx*xus1j7ko23($V?8ACuJin>aU+FDj#+eyO95BwI<5D6pkjChQGF30tOFbNjZ>SQJH%}D7%ZnLr?9wF*XrxQ!L(iUR;LqS zcS8pu%WF8da!VxHGc@nf<7q-OP-LW1%@zh+F1=9NP-kT;>Xt}F^{RJ5^drUB@8R^= z5P?;9m3_NP*U;vCzS6er-S_eHTD3F;^X$tE=i~DAI_+GF$dmrI@A&bCnM9yyg{w81 zIka5LuSvHv`Nu#E9d!LfmnYsNn-odztqk>p%%jXVH-K%W6g)9?qf2gZk9-@w2C6}9 zg8TWOU8654M$!CABg8L`{Ge?&bQfo{I;GKqwXT4vbr~|)>=j~bvf>Smq|y?Izhy=k zcNN>-21$ERmejQUz&7eT_6n41TJdRzRtBk3gHXLGkCg`K$Z?%>q`fUPulTx!tF~f8 zA+|=?VF{*iT1fw~4(WWpnII`^(;)kIC%fPDo{-1Lmo+Dyk03h84@sm5FK zvQ^1W%DBRJC=Ozxy}5wW{udzy)Q-OK1mh!QK|R|`2(Zs0V(okesbtYc9efk}xuWs0 z3fT>~dkD;bYxK!O%UM>?XV;ibob*tk9a)FZUA&jrM%i#>D1$^VVBjJ2R7GeqS^>5B z{FOOI!AZl?1A;5Y81bB+D9W9absb(_CZoObck$VQxAHI^i+f!xsG$MKyhNk3Ic3gH ztf8|;M(KFktzT2oa|11(qfBcJ9vm)9$Zu+R^S;L-bY0S|LQKI)*U+K;v>(~z4K`9i zd-5~8m}QmBIVYay6ZJW>Jfh8aY*|qvV82(D0-p+dMMTF#M&kp!P{|%v$+|IX@1Wz3 z1Y9?^;v4p8cz`t+84+H`liQ;0x)rnQcjXSljj!Cf2>U9hVVYe|=8SO~4PWhQlbh=) zI1=3`C}kUzUWrTIzw^h7J7F- z!}AHKZ;)$G3L>}w`CQhghA6Z;A>mO>{!@|&EO@S@y7mXHBJv_&dYeF((fouZe&MP|~u0_Q#oc6}~-sAO_AJ+`G9_E)7EBJmdv9&JMoZxsB6XQ7K zAYNHAC|Za35ti+kZSOQvJ4qN)Ruq=4pohyygY9uv1tTYTfm82lj`L|Tb5Li-Mv+~rZS z@F?;Vn_Vw*zXY>7wr z^4P_R{;~F^buasLfR%TecQ;#!w!pGUVCre6FbKh%GqBOeREO^|>l>cfi^1C2Pn%Ws ztUsyp;IEld?hwq=WF$=9TV$ZMbnRE~*Z z^VK=)ngFfex~wukmb>!hvkev$B8{eg?pRXR?ddgouRQ18i%S|S2V}H+;HJKOd_3at zmRg|svF}_7QQNiJHEa&B{X%;JkHj?_;ZM;$BFlsJ+dTCATPcp=#C<+SMAIF|HU-kv z>;Kd%A?5tkV-THDNZL(JhZ{zgjKXuv?H8X5)aR5zq~r%;b3cZYZppWcP1nLnzYvj3 zgT=v1kU4_F#aXsala6Zv^Veo+;4vP!TpOQFdvt_3PY!}R8gpXgT5Pz}^2dBE zHcxd87OcCB+^0LX)8=azjSGo}>KXa>1+!mFHXco;&$uT020r(~%mdiYh*{Uj-A9gu z->g-SR@Zx7;5bMY9Sj3<5ZMW1$P{|!gG3XAGy+jKL%UpL6&}<)W?F1q(Iltgyz-D)%3Zn$(=L3D1j!mUWmy?VtVeA9KPuf|wvw-Q|8GsueDYq1*SUA}8D zZ@kxSCoCo(>t`!M@~(9FvdD?gZFEmi9=z8DPuCoesn(9`H3*TMO7X@FAvM*y!$+!+Ks33^6<*mvxCK$UDN=cSU`^y#QB` z4o__Re5EY-UF24vU(&1l(@#8b{qdbv9E%ir&fWw+NB$i@>M=sHFO=`ePsm*NgSBnm z`%)_gsnoE=^A1L*vU)uY-)I1w^XA`Fu~7TKfqu|lf2q#qM&wftgY=BzCzbPD1wnWG z6>E%v8cSh~Bi!I^N-~&WQwvoO=vwz%a%H=XJ;n+@%e)WF zK4~G{e`sb8*daRD^vTn^V#yl209UX_IOzJl{S3%VCKZ~T;nw@Jtx?TyFQ6ghX=h{) zV`XJ1Z$|q=aOcTcF*oAnO>FFy^SH1|SzpT|B>e*38rGGnk7kU*xvi1w`qmj2^d(Vz z3tZPL)_cEM?DTUoaF(rQU(o7ExTL?=bFc6(0q3T^TNRHPEsqKRdqZGSp>Gglt(r9) zo8&5&3hD)LzeMYM$3N|JzdYSmoO+n%A{Q$X*Nb*MLt` zk74+eY0w{s0^fU<3|i#o`hC4sVVI3^X@8^HX}CH6S7$BhL^0Hp5$E2Hb*s7B&2=0L zr^OGeZ8=Y*d(1D(85KuN443sK#Jdu+Q%VemZdx0xbEB=#0(O1|o@;ar+^+$*HGEHQ zYDhiim0X)%6|B#6Hg3FDzFF-w0m9c*PQFcKnZm>;26 zh!T^mr)WmCp&BVGkD^##;pwQU1um2O_Ea0Cs@0Me#o_KsB!P6ox_ftL-5a*B$OkEq z?v?9a^c`3lfQN^GAIfplNh{4K26^`HIs0Sh3)sLh$+ccxmo{!T%f*}Yv;=Ki3a^X= z!%8f&P?*yk>Xra2IoHio$llFY_@HTtiYWk?&8R;?Dk$gVRfbu#GkH}|5YV+n!4*mw zG~|eox%uJu7jOiWQ1#VZE@F^{z_>R=-D%9fN$uQ{S(SFKm4b_tVdr8B<%hcyLkhSm z)-bQk*V{{yo6&siOFhL_De06K2p6C*Kxc$--eGObk=s@#>62nVWIl5mE96{2ev;gh zhTAL0O*Wr!gB0~8`-h6((>8XqdM>6wB1YM7Y%K2iSs`Z2>!-~MNSj#++2oSS(bU=J zeLaT{IbKwQNz`h3+n*m1i_S_~wY&36w3&VoXD^`>{P3t+NZa?7vEe8wkE;NI`NvI@Y*jsJg6Iw|ozUcP6HL9-u%xs4S?bi44?a5ydXrcWF*g;`~K627K;j~x&Cq^5&* zmZkE$t^mg36lO=;y#4ziSRq20vsBxn-o!&Vo0USUl7)`+gD~(*js|Vhx6x4kal0=8 zdKX>0vr0{PDow%!B2El2?0SB*yJ`Oo{z66S9}1ojOah5@_b}q2|NFd@O>Rqp`?niz zX&I}M#e*5;qp%iyg=YNdpy2p+hkpr2n(^ZaR`c5m7v}J#PVb#q>|Y+GKuBH3{-rrg5Ny%xFt3{yOjzU=(1ThPvUxn%w=tHe)wX&(B2Ycu_gZ@{EA!o zojQ7E7+0HJVZbVH6rW4mtyIg*CT|<-)(+2g)T9^E(r21-(I|72%X$z5*6+p^y^4qN zH1zsrU+27D-R;QJn1;n*{@O9@c$?DPxV@rP3*G!+2=LjW$x?3B$aW1d0_IU7={bsZ z=PwO9Hb)Y^H4L#+gE#rI`_!GZ89vZsDE<`Kt-Oh2{6o0Ld9Jg)zeX%`*N9w4#qU39 z*BTrfE6U!d?&DBh+JgiyOi8;?unt%7ElVMNN#gg6#le8S`oz4V)7Jf1A>E;l5pO9< zSsd*Vl&TiDx01d`Sbl6^x3HK@*(w&!(+9vrHyn`neZuhuA;a_8wF6;{`A3 z`?GQ{-;sH7;g$SQ1R&rGo03F$%>5}Tnfa2V#=b-L%DoC1A7u9QpNsFKZ_py%58Dxu z8W7+DbkAhpcm9(FM^f{;FO@q3O_CzA&dreYe!=d9q7`l1_5u3aH^?c_Ad8XOJAy++nbjOq&x9P&*%sOIESS)iU5I1?g zX^e~V)hR6#y6V1+n8xqz(9H+=JvYzQRxJT_#6aiXE& zXJGh>loW86Mlsgbj;On=)>ax8W@n*)@@!YiPZ%cy;uPB3-^h~UJFcccYP$@mp9Rh0 zq&8eBb*2vKGADxfc#F*>t=^pa^vhOc%Jg@X_0#=3=q6KOV5d335>|Hrh{*K+I~(f% zgv~SiP))5|Oz7UfpuUB217(!O@9mdq-iqy8v@if*dGX>!Zb^sOW2Y`;4h)#&=HyiC z-0t~XRQmFz{~I)-m$3(e?;-!w9(~6M;Tsznxg5aZLYDi-#$0p&g&JzKJx9mNBfobu z6{=j|S`Hmc(#41JwSgsb1QOESbbeoF^5N)@#H?Kf-)SN?MvYTxzpCG5sy35DssJkbIJU z&odKK4>;*2<|naEPj8R`y1~-ZOYVdotAX}2#0uN-?mFO1UkVeRb)nOk^=hWS8i?WG ztmW_woIMB<$+!Op@OH|h%}O()!-@jn?n}9eXC>;1gVwRSLlZNU9HD{(^5c`hH>8BJ zR;G|ACfMfm+(8WP)70+8vwa#Y6uej8DSz^!7J2t4hw)#>pZBd+-1atg^ng(NTaEQq zTZd^%0)0HachIAu`0YhDU-FGcnOgFJSN!?pYLNf_Fn+`CJNWDnl{{f{2;Iv-N2WN4 z=qu-X&UXD$%}h-4t*k#fpS}omUw9LOpYM&uzh<93F@HcnEn5Y?gJ;u^&qwid{1@PD z-f7Pelg|vAt~kD6bE)V+AABAd99t{hD2E}Bll*z>u&ZQ{Qu)aOiUsFTjE236oI9_MP*Ubdq9*EJIf0m z7JE@bVPLYyNEL8JW~5D*&28D0=#T8FN5L9lJ96gb4)tX(oYC0-P~ND1b+uv<{2m94 zgfJS^K*t0uVYv5pf_v>^~@tr2i7X9#KhNg}nbhL9?}o z4(^i~bnM2}Ns_Y&sNGJ42P0$+2X^9(Tm$+7cQNlx#vQ5-yB-g!P~*5>tQ+E@5B0kbjgaDq6QfzVGK_$+IF|Tw`~Y7Lt(1clHp|NQ$Kp_1%q)Hm z3ka{lQQtpH1i`4exV!vBhn{qSqMk|)2uIk3aDPYAv^bpj+Va4XD)bNE(4$j^3a3Jp z=n>l>VoemT)>g-3WToi5b)2g;jF}O^A30R_BHeeHGPUTF%;sy|fYhC}Qpre13UeqG za#>e=qy11X4t$oUo;!*K7LxX@78V^3=*CsbvfKg(xo_=gcVUxdYyX%%88~I=f29bn zKI-0}F}hSRlRc-D|Mk5s+>Wj)7ARMJixdGyJyB2Z8$*4{c2~~kF(t8;Y|V8=e=x4w zy5yR6+XT2@X@*=v7!Ecb^S`lE?>RA)P#BrOV_!z1QT>Qt-&rDpy4iExl;V}GzM7tQ z8dy0+1*AXp4rmYBgQelt@1U8N_pUn9e5Ya4`wn+^b^DrmPMV;dUR9D-UqtbHWU!6C;$zbn^2y1tBV_y>075C0hG+P%Na7@R_$K7XgdFe9LVCvkDCH2Jo3 z^vQ;&&Wj!}?MjlNnJvsouREwJ?QQSNH(y0A^!|8pP|(?cVdFDm->_4F+qccdn@SZ$uePgOZWXgq zw)d75a&4qUpd{0i!MBYm8TC1R5Vovt-nQ+;l2(DM)QUw0q_tnIvADNhb^05ptPv7N zM1!(7Or3aFeJ!`HKvh8x8}_X0Oh}3Su;2}XdEMRwWi*uHr07_gW;C6_jv$ht*mPqq zCsjNllz#i4E)4^+F@3c^&s!y|&g(6T+3@A$nvn%dF(V+m!Voe%gSJUp+Wp|Y)aARi z?cC5KfKk+**z&5@x9AU8F%{XJ@O%1lS73o-pLBh9aHU(cztHqs2=dW6C*%DYzB+E8 zF~%658?pI*;=@d3gK|B!!Q_0AWUdSpk)hyP-I7+#fFG1|7TvGz7Vpva zb8R2df7<;ixGHVu0eEkHf)T&S_GvXOZ}KHFoOQFv^T?Q;w}K{I%JeiPsOR&plfZj3 zx9p|&-b=fVWg$YXc#hA1Zxq?BZ*sJFzmpqGZOS&MHHc8cZQMk?a2jdB)WZD%L^c*$RyNRx!F)0`H&3qn5c1O+)M8rzQ$47TfweiZW4=l&jqHqguASaKS;o0l zLcHLaf3A}fYq;O(u$aD=-Yzwzy5v1+B1A)EW4W%fh}rU;Kt9%M_6CfvV%8xFhTF7@ zvb@EN2)2Fo0K>Diz>|_h2!d%(L+oem#3mj(9v-;H;_sw3w!JM1EbXSkf5LM0e1Jbc zS&t?vpWRn{+rkYi3aFUfS-d1AGJ#g!^pta|dXM;s>R*2O+1)C_TByVv2B=dVJ)27W z6RaDjK2k;>VPo(V&5V7g(_)-v^BDAR!$R7Yg7f=i2&{`=(XlPru;_)j{MTAb^SX}% zRCCFEF;}pm^NuULwMKovwm5-=o#y+6sq^;)u{q*m-wc&Ns|qu6aaTH zc4Afimi7Nc^Ri_I5YM*lfWEy_&QLIG0VCpy`o_ok_$N%rm8|q+cRc=FYOG*p1gZ<0Mj%_vf)y!lV_^mr!y4W)7N>yji#!=CV1rj%(YZVjwL_4~lLVzaiq-3_@4TF0n1%sbO z&47-q#9obAv7VNU;wG^5TmM8S)xOOj`-`0g`mc+R`9|-;EcZOBIJ<6*ee$ShGiZ+H zFEZM87I&Cc^LNC*9RW?UC;)wjG4pLkQ-9ev6wYm_21~H1tORrzQcFMBSWOR6)r*q= z>Pc<|^yZ$}{eK{w=3xDQpoaB=tOY>?i>bxcCpA{3*jvQ`W!@0iY`)a|k8h?f(GNEu zqbAgK$GcWarpl*0#ysaA7c)0URt%?iCyz!*c3=`$D6aQ*-9CwK)ZFYdtQuP<-$;DZP=iZao`4p|ZKO5ll&dc8(yam;p|L#l$WC zflgK$L+2PIDZePKrDV1Jv4+hn|aJjeM z-ftmRyg)#@MAODq?h_b^P&?rkDnbRVN9Dskd`d`hJna`c>vqj&+1N0Gt^v%4r8JuvEMDw~QwAy&q6O_YIXBWIx4!_tY zmR|I6Qil&~l3xalaiAX}*T1;5MtVOYbcu-ADBpJOE<|>jCPGuU@|>&-Qu56tA`&Eq zOY#_4rK1GU#gy_ScN6@EUV~k{clNwOd!%tT&cGpvA}vcmPo5-mvkwmIpcTGcKnTlp z`z)7Fnh$+=(X-pn+1J-B#URIM_)d3nIL)@#nXKMVi)I(@Yu+5IbW1{(T1ogRoptK< zRY3Vv1_u1N0wWdnW`$kn1S~=9BHb=x+v26)5DmdP#&+|KY5xI*c5$XZ5vcY|BwJsk zSN(Q6AsS?kg{Xm3vyXz%=aIMqE>l2OI9G?sYOWw`HbZIr^f0wP@%<%I~Lcry8RX)qt5iuIy)l9g#mVc8S!WRC)8Oa58O9h3LX{=52pj``ff{o zKCh4izXKe7;85)TVQ=XN-G|OOG8KN69gAo?pZi~+)q7{9FB^Q70pw74unY8<>wRPJ zd)x~3vuf;D+R<4Z3X}Esxi1a|Z{*7pEC4|Bx|_t*JY~cgI^qOGIr`RJ&}2C`5N(NY zG0Z~@W%p3u?~B@rB(uFr#RoNh8?TmQ1Mv?zx5y)cbRcHD#-40Zz+?a6 zF}S=<>9rI1Vj#DFX~S+xxR9R%DV{n%G~#zIYMc%fxD}x;FDvxoTE=iQnj}Na62GX0xc+W0204c-R{@ z2LV}^P`RgOq2-?wo7M32@LB>wZzXcv)>`zL%y)7bW+~RJFY82CrE9?K0SXwpuOou&}f=WYdp- z0A0(m>m1!{Y~d3ROUl}|4u7v|b3B#?ioMj91xZ2qESu;xkod(cDK9D;N=>E0cd^yJ zBd4>1bnrJx<4;!@i=GUcy{B9;dAo5_zZf!aH@@Xk<9_$;&s4D3Jj*}oNHy6GMG(Vv zw_x>IUq2g;@4{9&qe3%Z-0J3@y!;m+TWp+t``6H9%FKVg1X1DSX#zaVcYgy22hXhp z>1~s-q66WI2pQkG`Bjm!Mjwyw;C(}?oCrVmRkK0<_FuIv*~^Cyk%p!WWvoXqHEU`+ z&94fro~v8Do#b82aP8GpEtIREC%q1_Y8G19J31{@go+Jd(A4b0*<#Swjk5RjGFO@$uajH|^2$~BC!ikDSHE3gYnG)gJGp|eSd!v+jRDyz zpX+DdlMvT6nV||UjZLT7M6diSLf7KbsU$~tJ7UO4%)(ccfx~LiL&4j!sS|mmZjynz&LaqCn@PPjmgRZS^qn}eS_m?^tNb&$oiCL0rw*u_E0 zH@f0Z%hP!6VM5u@5R(tH^?S0v*J!3M2Hu?CCEmrUqBmu!A=IuX48M*>Nbez)UuA3{ z(PjO0D_gY|PRU_u3RcD)4j1-o?=NNq>PFf}nMRXoWVqvqet%fB$&}PO}%{!v9i6RSh$99W8o@Ao{!^IC0mHQa zDsYPPFPr}Za19#y1-KSBXA6+}aQW894NiM>UFP(euAf1S=H{m@a-KA1iRl%KoIbMA z@)O;?f}^7v<%S|^4RaAsr!BL3&zFFupQjJN1$|rnn@$HZ_>UDnU>Lj9%h0vYQ`|5) ztb=Rr_+&-aPPt9TEE$2lU*4am^2LRvSi602n`^YJ|Ef;I{#$M3vX~@Ese(BQXYz7d ze-mF;I#wvC|BsoBj6&Usg2UlXXH()a1WAW(5wL#O+GifWXD zJ=N9cHNgs>)T%%X&SjEiU7N5OY{DT0;gWU%$#gchAe0 zxaifC7U&2(di`KmfDx2Yp21wq3(7*L-*3<3=-}}(kwpEeh^77$+1*P23H$nrFZ)OD zBA9J)sG*siUA8RYKXd~gQP#3+eJgkM?UOK*{KeyZ%kgQ_v}2qZgvHYF?g?kKddhi) zj1(oce(Pw!6*ZlU-1IbOh&SHUsToopO3(O-y@Ik&<-i|1#d(e03=X*s=|n9>-5&h# zs6jXrSCxUqotZex3{{HT`b5@>&$OYazlWeNJ|3F&Pa|}0M?yLZ$E?d}dH?82_X@t7s841WrX59JT(7n$$rJvU4 zy{*h}xIP%hNFkj;t*W4%HkW(peo~|#rEQBpGWdR}>{Kg|zb5%RCaCl$_{!e-qwT>| z)4gOrx3~Uza}EP9J;;vO*;m{O!-;je$zDjS9E%oIdXufSXUd$_7+S4OLsJ0m80l{O z#957=PjFa+a5#GTZ+;|ac6fT@91dUP2KRsb5(8k+`}i~Bdng79RCBSYbWui}GrOyF zHk#WLc78%K4;Fs1FF|5ksgo6TdaXFNs*e-6jt%YM+Z~DAtkjwLJSxX{h zOFE_kMLsB6w?6>n2@}#vEQ9(JoN=G0*MW*xHAbwe)6-{%7Q|wp)>YQr`?zl#Usr2) zYX}=FkBkN6+SVww`aJKl>vfP`C4=QSj|t#u?_3r(U@wX4gv%9F%kW;jrO>f9&LCX~x zbq?=V?hGI-;oGykH;FqSvEVfp^B}VU6tDTDRUzG;({br_kw@BxMRy{xOQ>S5vVcXH zxA}%e4s1?I5leq-TGa)R^>q<_63BwHplRv_&aj9ri<4y#0n2LU<@GSf-xAql|F86! z90xZyzxR20S0S4V`F@LzlaJy2)%{)QbSnlj)@5RKm-%Pei+>&So!rrU#voAb?c28{ z9eVy`W7LOWWSRY=Cd9M(UBH4ubC-Xypp?jXncRgQg}xg+*^iii`%p|OTngFXKGOz8 zH4c{;{*a)8JjJ!5s~N7Qzw{at6NDMC03QDIB(U;aV&GR^&a*wg#Ds~dwlN7X1_9-j z^@V<|xFiU?NwES*i5a~0IS@6TXSd(*p#HndKamKQ=J5kFyZ(?k^9=`VG7~{cH{{?3RGjBhvl@}-5@Ba!IF+|X9 zXd1Afyft}vcjXmVg+JMfkCA@qb^un*^ghPUTX1V)E6lDV6^={?myCmsarnD4(^3fC zLaD#3S|%nRp0h(p^BSX?ni@A1ve=t>E{AZ z@A>cMqrGp~$z^`762r(jJJSai(h)znSZ}AwdNx^t5Q6*^XVIaERFDkEQLSKbU8rfA7b(C#qnvDLVz5qe(b@Ci`CW3*T2_sZ{qB> z)Zf->*YQ0MrKHAsfrI5%COaO7Zm087(+)P_(c%H_p&E*VLJyw(pGP(L&pIjpWLA8G zrN=$dF|tt$74y|Sm`{x=oLSriMoRjUWfiN{J+aVx&i;3>ei@P%~`}JqBFL**Au5RBs~#HJz`={jEaKshhuB2q>&oE)EJ3s8QQS}3gB=B>Hfvq# z`U1_e-Ftqt0z86$D~g^@nxUq3jwh6l-H!uea+d!#VETGg-B_Yc&7V3`2GRvIlF?ww zk16v(Y0W^jpx*sBCp0CDjv#Uj$chg11O@vM8TXvqr&>_TGiW@NO9)9e_7 zA3tf4LT6YNqqnshM+Q2nc*MI)-cpMZLAK!L0Y(seZk!MIsnTSyQS46X+>;pDE(bQ+ zI9Y!lxi8;d^1!iqNyB1eBPP`tzl|U1qF(jUDwm=r37KqtolJ&zUy(WrB6^7@yW1}B zznn|x#*zxpI~K`UJ~!@mP6OSLn05OQue4tOmtLLSyO!t+hz_5+QsKONDyRdStEH!ZtRytV&RqvI5u<7Y*L&-^f#V5?jKoa=){*L)Hd7MI_;56LUpN^!9b z{I*b$gxdvgwBfi+7{!LxTWhx%F@`b~0#vxYG9V9_u3O)RSwQ~SJJZX%gBvZ%C=MY?=7Y^Wc@LC zAr)!S;oMc`l02RmD`~smPxakTxHT1m*X%axvX$`N7sLt;p9Nr)_sfjJUF0_B4_y?& z{3&E&@SH;w1?ZLbX#}>FA3`&VJv=H^Ls?J}@{eV%NK&EBsW!?_BU>hm^syR>N$b|0$elR1z!tgm_P=<1mn*(icr{**qWvsvlPQ1)X13WOberzgFT+s)H zg=Kadc6}&F+LDNXaUq1xRy3T;t_pC;zB=RhK`bR}S+r6C%A;-K`)5L*&`rjNM)(XF zT1Z9@EZSHsnU5r|rQK@tQrir1f^NQD{@@xm-eA@~QQ%p5#5O8DZ*-SK?on{@$?i&D z{~Ya<<+>5RlKmD1Yr|LA?|oh-itct?fJZlt%J=A~w(`S*UlOEEj+UPnRfIs}^4?PU zn;Z-W^B69QuJBFz4cCHGd^uiMyLyX|!@yrsz@#-Oq)O}13X~U3=*2qjyPfS-*n}Yv zHy%owccr)y;`i_5iWf;Y_l<7%x$maC`RtX95fkm}}v2mQmNPT>;}nuNFq#sdglXp5Eglj0BlnG5_M#Ha@q=t~e}qiIIAe zMnMdJGPnJ2)V+69Q|sIQi4_q66$OzJ5j!F_dR0_VnhJu_qN36X(gK7e2ns3!q5{%E zItWqeC5ea%(n1d;BmrrmB|u0*`s93H&(SmMo%gq9&8#VZxfbqZ=h;uYpL<{T=R(CO z_@jf8)XX=r+@6FDiOzY^my4QvIAr0;8(wg16c(K@$8C}MK7}nEQymvQ<_R_ez}}8c zl|GHJKC!H1Sbr9%p2yzwN>vStO*`6rg>$AGYmxJ`L9%7lRb zRJz&@LA^nhtsjuw8eV4uHmWz-{Xl8CB?-5RfP87`2PZyNCaavMwT0=j8CcVijwjJX2J&ePoiD>V+S z%<3N!Ccb#$752+@XOaGHOY62(TEO>(XVK%v^Dc9)BvOElRwVen#Az!f#3)IB!ccz) zSD4pe;f0y9wOkSC^+M4h%2p$b54>wWW!sF36QPW?+qr!Ay6(PDHv&nz<#5I+wJb=| zR!n(czi%7B$=CTl;NI`Z_@ynfPzi3@RtExJN%3AkTJxTEBik)BOiOv+0o7!F%hc)W1JbFH`O+rr(D6E#~ZLzKh zh~j^<5&uFTxI6+M#|!MAR~i0nj#f)cr$V6aHCsDL%=T&$)8cue&RJpdsnIR~5Pp6h z7;a@v2t{r^FXi-mw42qq0Q%g3DF_8eO(6~pbm47%Q0TR<+nKj!c5m9yrPOmZ`vtKh z(WD1rLNwhL1P?*CGe-Q=pPwBsLm6xP>i#5$djFWxgyBK=5+RSx;Bu`#C&L&0zE$#f5wUij@ZFoCqkV>~PDgn9<{8iP zjWd?i`!y+LxcO0vWGFyqfS22zVtc2kUzCznKPo1ROK#>Kpc&_I%^~ zJF0)FkJQT-v>%v~s-1d#@H8k4WM2w~%U+pHqIGRTAvz_n#ym&nTaNKq?RwrdB^O z=gzCh?zshQuPitcRT-G9wt32H0Qe2I{vF?aO}<}9lB#o-x^lKSm?^9Y1LgV(3m3rp zt4%eF6@9svIG;}4EB>i)@SkB4;X6=f!fsNfo$XBS(w~Q9z3A;2cBWjPJ;%{Botd#T zipwwfVFAq2(Kj7I1ssDS+e;F2I%z9OUT4^!V<6RwuW#4;*n2FPsgG)96LRHUhY*$C zOPE?xWq_d8bHya=)LC1BbQ(QrY?__GD81L=sWxIetCx`H>8EjFai3sNrg0lqC z4mSkeF++x^s|dgr-_z?KEos>f@p_YAC^&yAJ`$1{J?XUq4LGaW&0RWpWBjQ2o5Z4{ zQWzmwiq^dCK2X!t+GiU8llG*!q1$oE+Z{g?#BAtSsCy*TCRITf^BU-G8Wb7(ePepo z|1@3XM_;T7JKTABFt=tw`+6RDqPxx?IEm@5?$3kt6m*c0hpGuG*ro&OxMat-Fqm>elnaBa_RwbD_bO;3V_@WyiqBd|7t}wq#`jiH(R93h4cK`- zht5XZHGC_zXBL0gYkQ&Z3?<#zJHMIJ4Z{Q*W&^Sw@AJz!Kc4qn5dHEvQY#nWZ01(2e6v6-+%6gEb0u)Yg_7D84r}*E7nic2w$(M!Ga) z7c_bKibpRxX7zs>8SiO=k)*1_8xcyx(TYzS_{~cygMg=CHPveW`)k^+o~76lHyV0(;v6(MZ9{P3lD^v2*b@}8)v$Z?F3lv41i?GqFo1|BEQDR1O z=D?1zWJL))9R)I5AquCY1xP=y8bQS>17#GnuNN69Eyo=nD-NqZ&mWmdppeRTI{Mmt zzZa+5ijCtN-_YI`pT{K#u^^DW7@lDKf!eL5;u@r5o{8!H4XehkNKfYCKs%n&TPU@7 z$v&z*tNs?vE~#|xHgdh@7O=*qz$#yKyQbk^41wLdVN`5kfocYHu#{L^W(+N7cn1&# z4g*oUV7U{4hIlgPLnWGR&G5uefahijiLz5`b;|vu zS@FmGr6m3rP;}x7W{Z>rX}G685K&)0;+&S2W~+TClJMO3VYdQQFuoj2tzc?vQ&=ksI6Ea?iry07B*4-g64stg56Z#mL)uftNvOZ#M> zo)uzyFf)7a?+sd{=khQ9OGQP-7WALZ`y*Rv%=}**U(Z?pk?s(A_5Vr#YyCd$q&8wZAmi(Ome>){CiqY7bCnOQwOttln` zyLTh({`+bN%cmfhGv@!v2%4FJ)`zI)8wqh|@;!~pNE=_>&_0Pdu|Km+nfK3!N%s^n zsE9`!HrI_e-Rg9wx|oq15NyNUzY89i`C_xktksQwBDmr|wcv^v^F@3fr?I(d){?qu z^-zv=Xghv;aPHco*W%p(8faUEzECw7rwB_(szfdW#Go7851E8tQZXG(Ssv% zDTIS-mzccYO%yrM`It%p;M0lQLP1~He4exuZxQ z2zg<`#Kxt-f3w8MSNO(6)x$f5namTyOlGeBy5IQ#k(@FiXzaoU;RmBvKW-oTBgd^< zW*`KOxmo-ROKd$q*VL5F`=d8R8sz>=tcm$ zi4R9_{B(-zPeNqA{YJ98>lDSz7Ox=D9XOccGuap2SQRu6f|6n`Y23Ve4jU|A2k+6M zKj-kyNW4wXUESsu%60ymwrFfWdqWv|Z&(gA`%{Fo%`b$Cp~pMoN}ZwF3R`QkCu^iC zG4K%mn~grF$CtR}!3E4Pff_^-y;e^&RHI&$9(5_XkN!{wJB4WG;OG;#XEQ(5;^kJq z3%qNqTlYxGRgH;1eV5X9a&;9#-ULnZ@!BC}6F5z5nLV`lP<}1U{7W;naW7fm^c}2` zGlUw5=Bq=oOdYq`xI470DauEwQCBRafb2y+-d|IKhHC7{W%S2;?IBEWMy;koe?_(A z&~0ET2|1dY^JH>G|C%`Ae0G1uho%mNA^iQ|Y?9Oa>?T$rF)RSpE#>Q>yvtV)dnb-V z+RQ;WWT2I%knVW~^l{Ql*&dE$_-d=D^?S!QLR7X%GhD*cMuJmbX$M>(n=Yr1I? z`EBDjzBjn0bxCIr)tyOXTui6x!wx>ue|4M~btbc(>Bru}XtQ$mv^Ovp$4h#8)q32% zlzV-_v!`rM>(gQmabp$F4dsbFUWp;@8c2_TaZQ2?YEQ3&u4)7}`A4x2Zy$i1gTylW zONV_<+@HSn_bQ?7vs(HR*7kLGvvHR*Y@ht5Dv7J)NcL?_`(L{`d;&)<q&mlNyfa6$}Ubi3rrg06^T5bcx z^Z-Z$D+Wr9f>2GTLG|Q$Kqn+G8tNMaUrcUWPG4NcJw~WKo>N~VE@>I>|yR= zNmuAMS0{5pXgw)_?yrZ8stSTF8I-|@IE|+djac!$KB_y=06ymcyK)&nQRA~%mp#Fv z`)=a&0*mBg7q-ZWS?is=@U0(dnoYdcCHVL)#DV*uMx)mMg{EwffeOxw?7*I@{AS1t z%8V#x?0{x-;5NvDDIHctOznkCQ#?ATm*Y=~s+rTf5(}*=j=g|)Kc47wBJlmRet6I@ z8}!h&F%C>>#6!_Z&i>SDuEewCHjU>6bj-ovZngRL6{OM7Db7WyiA8E6@$OTw+cSk@ z2KuXY__ATiA|^K*VIls1A7c@*43W1vdsMe zwIBUZ4QIt)t%J|u)Ef$3V`1vMmWtlg1}`jf%U6KH?+sei^+z{ihq1AS_nXSWq-yWb z@;uW$Pc!TdS78G8^%VC_Le0Nzuyc>l(4J%k4WDWi=_QDi z+kr!iMm|Vqo{7eYrtI1b07nfmqc ziu~8wp$EFfWz3W|CU0ts-lj;S_Y%xLP3Da_S?xt$=m-8zIU=f@^I4+Afrxlk{VkaI z+WDIYH4zQy2ASICynfm#CPu`RZ;=QR5IO?+J+VQl_%kS;39JodG8+|;sA=(F+myBc8+jZ;cf3uiG z%CXfnq(;6f>g?O|p>@&OsnN2e*Q81B`uA3`8}Jom@%RAXx@V_yo9ZA4cPp3pBU&#o zWdEI+;pK6^fz6!U?Yz0Nz+-sAKwp-#raSt2*QVwif_Ca_ka^OTXL7sXg)pO!L4JY| zpms*ybXiov8`9uL@l#ct+GBHC%4|XF!YEJ4`eZohR0djaw{eE(ky3uaMZBL0Cz0sB zsoc*{@8rA8x4a96J-tdiuVX$x5~Wc!z3aj0E=!5);P`MUFt;TnbDK=Tf`X%873=n| zps0uU*Dz5m>FL(vwcq7PP4qn?;o*CvHl306_@Ua{Vh9@iZr5S}BF>F{`re%>9}qGN zG)ozw3VslX*SrT$kc=jyr&~46$v0I<(ns}y2*V1o#WT8R>4FxtAY?)cupoaL54RAL zj1j~KRJb^&K^0&=w}ivSg!V}<77QbPz@b9Fw6uWq%!^_y0PlVt<_1xmXU3|wD3+w( za5*w*ls%XmqA~nnRG9>AiGkRh9u`7phG#c7F{Ifp%u(r4;Fq0T)YZ@P-plJJ8ult~ zNmdy5o4sxMVY77FnT9yd7%v4m_62Ojrr>L;Vb%>MZX6WG%S#+Jh|_Q<=%M%Qz`2oW z4ocrSq3zY-=3E&o4lx=)?lUs_lQ09SZVW8 zt+LxKzdL>b-h9$cpdvQ}{K6GtO3j$Ebp6St1kQnVk8|-mvrgQfk41gZy*syyYJl2T z9Y-qp3Y-6e{eebb#e0}&8?Rz7X3#$3u$S_v$fviiuMHqpDJqYu^~Z%9G}z`Ponx7BUMKyMjMH(t!aKdh+1?2$Xf_ZYl_)^ zS7Mg`+-8Odp(@F;j9Ukd1}-!8GQNSKCK*r-suW5oO>-Z`WT($l90J?_OXi~8pRrCE zi7-tQ0KR)kHo#ky%V(#iB`)rz?4Cj*4ocTtj2@Z+-revBsBksb8p_Nz37Cq3^s=0^ zjZW`jD0-0}Meg~+SzVv9WVBkl%`(bBtxMq*25ZPjG#Y9Psp#Tne6@;~nHe$0M-fFQS<37aJ&z z&5O;Di4UHJ!S)jvDg`?S9h)zqf=o#W^=aX8$2{F>aC6VLZ(LZ4TVuz zNp8%RZmg|(2n`o47OQfeyksK%5rv+j+TKM6jR!o_Z>1r`M8lWA)tm0`F1o))EYaM1 z4Wu|QB;W_x9_eK8Tv+q;FCFKl)HhZO;0cksOZ9e`5v}USU?f*6qE5JC?W8DsulJQ6 z3R^FHX{-l%9-l6vc`w5%G~i(?*+5C@lA-686`3zB-&G;u#in6qU$2sTO$yUpNaHc$ z=aXeFNjAdL+)1Dt3j#oYe?7-QiDb5JhcZ((=%*cZ@Ao}@2FJEbS&R7GZKOzOaA)MsT7qR})`0IMx z)P@MT#OP%B8r`>#->`gvc~)A2sL8!Di%h z@%OeG7~kH&7X*AJNl%BQAJLl+&)bd68642r8@JrT@feKRT zg`gHtT4IW{oWI*l)xsv~GZ@GWl{R6i({uk_^b8n(9uo8Mm)?XebDsUO?an6?^New* z-JQijiv}Y$JL|Jum*`*ORISqE_tX%qHuToesy{c8 zw3;e+fNf)X6c{MdNHOM6Ar#$TzFg?BMHulfH6Ib|0(^|lw<3c}<46rT#V+-^FEvl~ zGTX*UF^BVv*Y9k&bSiK)K<%-q=Siua$AvG=mTJ#3GUg6#k?58uaU&Jeh_}wS416|gXyGIpQ_hTq@S^&Zq-xAte9{THC?d9o%Y0^|rCi<_^-Ion zphFJO7ZY=eP#Ss-Sww=LfKl7pB3hR)Gow~J0-%m){=>TlvxnmHe5?WM4Oxp;<}X`bGx>o=+&C5X-V`HG*}AEPuhdNSiq*3Y!hb!ZS5&w-qZ3bH#h_{iebexfOg!CQL-gRi@J=mzT&fhI(89Ue#bMW)gqZ1xKTj0 z{;J=5Xczs`JA$XrskSf+y<^=6%7$>}Ct|*+8k-h92Vo2k(;+0TX%FBy4DCpkWY zzw*;Bx2@(gF<_qS8SjUxCW@SGn2&PqQi!h!YF3cI>W4o=e#|- z?0UOUW+6>)t|o7z(uXZvnv_YdPy5>KlOD+f*>BZQN?*@lwxTrsWA4?`-z=Rn%D06y z@k4(g>nZ*Ge(|?R)&Ml%JzAIsXw0Th*TAn@5=S{gz-?GUOxRRHW{Hi9^KjTov;zn> z4m?jQGo4=*&R{Dd^-w-D1Us;?_b`H&A~TbcH9lXCQhk&=Sy}WLz}_;5sC-;uiQSSA zKIWBzx<6(*AIAR7zUYZepZBv(x&8R00_Pf$^ zP{RHGyO?K}(XeUPeLev|el|-qqSU0QYX_x<=BAKOHzq&5nxJpG-=K`3oQc=CP&J#I zPQE(aYoSiR*}|;t^;$}QFucNI?0tJ@Rt2(}9$*$?8;~mizrSjcC&M^a8*V@}lA~VQ;^{g6&tED ztGuD)rYdDh_ylm^+0^)Uk~@*^nz<4UqaLN4!t0QE@1_q*(P2zxpk_-kb)^QPWcwO! zNc%WpLLzW$HS4s3K9<}!s2&mm&NiWMBs4~pw_s!`-J*m?tX7CT#qC$WibV9A+Rsd_=2N9{dnkL!YA~b&7%8LE zM(1Fj#B8Rd)l;p(qnN+17JMHuy7H{xs@e`rg&+|6?+?uV^M8)oxcRT&tCGQqzrqcfFb7~Ov>P2o`H71FEs{f2yGSOSl_kIx=vd_>7xOQ zLzAM+$E(*YjNvpc>oM*3wLPM#A7F#+N$p)-K^wwAmy;{HJ0QG_)k0B&KeODcmWQM? z>-~OtD_B8@=|zUrL1kn~$iLhba;WSnb$E&KzMjL8Y!{(VV3O6JYF08{C|Mpq0aY%h z(*N~zpxLvZeg0On5{t2zKiw6`oOh5urX3RYr?sFl$?ccBg3GA4Hs@+6`{n-N%{n--J5KhSazy9dbym*cj3;Dm*UIFQfo0zX! zyWZI6v$ZN@QKlSS`MDxb>u6no@Wt`S&l4SmBSK3$WOC35xH_ehS{<7q^P8l#Z)D4C zkwM|l9W~p&`p@p~+MAi1hAo_HD6y%&_nJUFnqHmXfhvO*8W>*Isrl_9wXgo){(`UG zxJpQL)hO8iB}Ts-Cm~TY5b(E$!xQO4!{v)^(I?UO1J_35DURjx*~h^h{Ys>ry)DGm zcaqF7O_L^j#Q$CqTk}Qy>x&8h|DTs!r z0d(^*g`Fc{vH}+$NivxGK-J&5lPjb^FQv5wc+LkQlw4jvKg=dhDW-G6LLf^{Ui8ch z@kY~bNH4`aNpBt`NQT9)a5rM?L1J*hoFVO^E*Q#Tx5-Kr;tB5hDo2qYJY+|>)ysS( zFbtrkBn}=M$u9fZ1C@qKD3%Y!^{(8^OUU~MUc>y>lN2E(=CP{(J)I3aJJ-a0LAea% zPkK8u9pY?G-83MpkrVw-FRLYh?pBmSsUD${iv0|o zKoUyjZBJ0g0SQ8_ZXLkwRz2wL^IH8wV0#8#LHaumYc4tY7%26y#@PXpl*@d}uBMIz zS}!SNz7vBY%+GEIZ&y%)L+wjJ=f@B3TadqA#*V4D6Q>}Yxq5&Ih4Z_vE7+}CDI zC8IKT7}Q_@cOsqOQe$IH6E285*hiV;L?o!^Q53 zh1h6^lUCEjrUC+C_g;}sYy$3`<}zP;B6UBk3th;Wg0n2#)-4FfK{M$-(A3Ixiqhhu<)ok}weK_~qREqpSxKcYrmz_&5^~`UsZ}>W& zI$z8=yUXB8|H_-Z&AjCZ=LD7gW($v>LR$xfVhUkCPCw>&=6$SQ>0$sG1&!HFew8Sf zxX$nKo^>C+Dv6*S-nC7{%o}{)YI!?Z(B*p@AMH`DykqiE&?4(R2y+lZYZ82Vr=n*y zCxcB748JRPd>4%ZSKEQ7!o1*7dhIPkGI_FV3ME61J4kmBiJX(h`-5CbMq#jwtmS4= z_jrnWg10zBFB(=tOVIwkvpn0MpKwt3gT+m34AR}%F*m683C`!CiEqD(6XmMFk?<0) z#SPo$V-%8T(d+>jozSG!0Zy9yKm?n5yC*EY+1V^AGfJKRyq)GV*j3orWh*K{>0#A| zaGs?(e3n>7Vn+`SnfxxvFlPRBJKqFq&H1%E?Y;*eFM2dzTOW1_yr}0-rYHYIDb6KOIKd2PU-}WQEUsaDd zA55m5*n+}?x#`uDq7P7vn^OJ*XAPF@DRUdFt5V={qFsMs511( z8|Ig<{%YBJHx7!12e?}+OiOZ*`Ee)JwbdCPmS#FLurc6Z-{K{!;t~6DrvR7S0HI>v z#n^k=0$%=I%Nm@^UyB-3n7gA$^%$K)5p;tYQ|U%cOHyToEX?BF87P8=(V&o0wA8sz zFXv)oV}$zt0m%Na)mk9F6%^)gTC>TGq~d9NTB|!s>4DiA!1hXRz(=3MM2?>F>QZE7 zb{%$z5h8^tG6L3f7InU7O72f9DH<%@v$H}_?s(|v%;)Z6aBj6{lZxO_fi^h!0f-g>mC-k-5Y*2Fx+$7+@R=BUC#5D=CU3< zOI9hQd{I=q!O5B;kBpalIF{TSW7{_?dib9~-Glu`>6j_IT+l%R=4+Ep&~i-5AS2+ckCnvjX)hyQ|n3&Lkk7I2}%aIA zXbCORxUFi!P}VRX4V$wSR&IX&)9jbMvp*Krmw!3e4+S<|h3?D8baf4`-0xV8L%=gX z{81Rk-Xh%%3~AofFu}lU-2J-cKgiSa{}_qm{hemz=mRkF8{h^XxtE{#}1amj6$2qMb2ldtd9g)V~O+ zInp(nzpqI{Z<$J$=UF6+)ZrC)g`!vr#CHG=vs?#I6( zJbeqR^sMHyGVJN{jl{!Hc7Q>6x2FNuX7As_KXCVXw&uu(4)JeI1@dK!8PBzMTaZ^|Jz$OgG09r zMY+eOd-7T)sN{yhzZ5mdvGa{c6C$>>EM-2E%Kyx(sP#1@5j=5AO#op2LBw0=`>V@W z!2u}`P4I?m=`qO-jcWA`bi;~%`PwR_4ACfha!10kMfvwx+s5W$?7ImIjYW*VwPS93 z&$EZ>&V5O8V2C#oV^VK_-}tOI`UNZbgugk~BfRL#w~ByX0%G%qE1+8j=Q}DWAx}d- z6i$U6zUTfi%a0-W`ZX)I7K0ucXb#_$hhu0L?=5l$+J-!&ZkaI<%tS!lzk;4V&-R-e zKMU;{TcS#73l`sTiYhKqYMpn6ej{jd{}K^h&A~0}dW;lGmQA}l>StzbW&(i&pVWe) z-ZCG`mV`SRv9)a0kbJi?12p>mK|nV)VPR$9Om@G#VBmBys3TOoa|SUG8jLI(P+Y52 zJ80j@gUvdIeJ;rpMfGRK{apsxEu1suA%dwXO|zx2-C-SN4*toU^CY*Z5AmCtFZ+*k z$25BVc~PJ>kjUrMDJ>9>$fC{!c9YBj$$a#cz?Vt6Bq18Il9sA9B+wuKC)v|;?eXeE z79=75s?`P2Ps+zoBlEd?v#@%1eJ6nkNr_e^oqPmH$&ZuT!$tQRG($-{6#5LhQ3~Ta zj~Dy)zmGrVjc!DbG5plo;7DX2QhCy(wa4F#CByMPbuWBi`BFxL06FeDzsBwB)<}hA zu1=&oY|B-EX5?ZQUv|)7-gY%YYv~3--G0Q4SXyN^i1v0-*edC=R`>v9XL-z6JuALr zXt@+<>I@+@yf zi%Y{_sA7N=-a>f5K=)u8Al%?5#BsoDmF{&5u^g|}%LXAUA~{k-$B#IB^N0A-7HHSD zAnF^F839dbkgmy%onnj9bZV6xhAzLG*&CL9>mmNNqUBcsFE{uBanfzRsXFHVyZPhn z1{O;1li@p9)B^K@5_~`dM9){vHHjeh4=1NJbQIRtjk$2(;y9KCD9jP;WKc8l5 zaCo`(<8Wyqe)L)QSMU!Wwi< z-oo!JwH50Tr?vHqkPUmg&ES&WmCtF^G;kPo`O>zNK{bvONK9%Lo3yGJ@UBw>9ag_< z?>iD>jR{W>tKLtf6;g&iy4|z9U*XUmar@8v$+`oc# zIk3?ZMFFgfx5L{ycrDm1y?2^RVYujxW$$esDe5#4Q(J#iu#76n9)%`?(BK@PLo7yN39=?aW1ed(fM$GkY(E zqmkGe)b1N8XyW3+_1aZI{dfsMI@CyTh8291zYIWV&;lx-yb9YMDmHcFwFjRL<+xkD~QCZ^SlS)3xs} zeN{iK{owYlmuD>8XX?pr$BnL7Pev}^JE5E#*!A^EAh1Gie)kP%Y}m@ncD1V`-|a72 zgv6_q+3YWc$-a*CPfCCL?8*BbCU?CfORgvU?A}v2QJk}F!pId#2T_qjL@z3`dp5u- zo5F{c#*+=Adfsx(NwgaTY*-3(1DJ+Du?-oB#v?@9Wl)v^2 zxHXDow06UQ4wuL{6bQ6R=Hy9)>1_dCx5Po z*mM;b6FeH1Q~(6}w6a@Q05F?48YcWooReHohMseF8g{MV4ekNXur>dMYpNe7GZmE0 zE_6Z%T2Fq%v?XwZgc#h8gshtf{a2(Gyq9FqOmV>_*c7S!mHGmuYi=b>6HSnsLI|8k z%A-guR2`Jpi=HEFU%Jdn1Xyz^vhbUzB@br6VU*_m2knBkbV1Ir))WWMr3Qz%exd$A zEbv?AVBujYVk;!rtkIixr=)p7<668mzphk8h*x!bNM+*yZJ=WKC4neN-H->4g8tIW zMifNvgG$T3KiXf(?(0+L!0ZbvQ+th;pLmK4&O}<`i14eD#cNJ4P5aNBb@m)Pez2xk z&>h~weR+mcLF!-TyLrZ|h#Pw(8Z)GL3)Yj53IW41N>9Yr$RkPUsV*{Qo%BiDgPPN# z;RE0}!xDZdyl{);;UlNzZ9A%x8+G3KPc%1r0yMnMt zu#;%Q>v9@8*XVmU%0dgKlslJ`Yw^M1VRls={GL0`gDEBKAv6yh`w5ejQu#p(a(1rj zz3$X~LK{eT_SuQ}>hDrqDD+H^{OL?m)xFlu@J`P>x99jNka%Pmy-JkhN$rC7$vDFEN#cej3jHAXA8xAfJ1C}~EHkW$;dM+5IH<^PcSL7t$rvf3VO=LG6< zlj$GvZxUKI*U0q2ljIaFwld}=YGp!}PkKy}_zhuTX~hF43|;#^H16ukY#)MR0F*<| z5fBen*}5MvGcg6N>Uv4vtK{=_nzb&Tp9^kLuiK=u?*q#;w&U=hK6!;T#$0~wa<7Qo zXa#!Qr2Aitr@|C+g=~wwE3wK4(*y`6Q}(gd#@n0dnneb zT2LrSanrJ@qqBZ3q_@RX%(C0wi}D;{FAF#k_t~AplBkKpupbLRR$_rfJjr_u6S?Tb z(&n#8p(!wjeTU4}Og`~Ikmg%(xrniAXwA9@Hf&k1a~QX1_=m3%%%$29yfHTXY<0a7 z;p|RY7i}fLh>2(IW%NsZYL&2H8JOsT7Jah%cP_Z8C7zMKK-4>V*pOMeRCk!29L#3VtTE2Nxzn6Fn1X=-H4Q^#$54t3nnduv_Wsr{d5(y=)H${T{P>KZ_D$;Biw)$v zP#^j&WOvACbJa!67jScZggO2_Yrav|tJ%%zr3I%t``D&Dm_1DSRUdKGd;2X>^xI9O zVAlx8SO@V>Vr%nct@6u?c^$a!ZZ2tVDBU-y+|4O}OzY??JqHs*TI*>Vie2<(bK^w% z_VzPzu7*N?;>j{C-;j>2+uEgkwa(PKa`333HGSyBeCbqD)Qskt^R0L133_m@p1=ia z-Qi8$WpF|jUvuSL8>qW&J&DoRegcpKVmvINsFbveR~VX~V+eF7zQ?_4x0E@t7)AqY zhuI-JF7$+Lu()Rz7N8idb@*#g@cT5S^e)TWzVHSB`tEf`y>OxriPL6n4)NH>SbA{1 zm1t`?g8NjGjnZj)Hh>mnhhO7MAz?7=y4CCKwN3yJ{QB}fSDRjJfq)uu)COc3ilI8Q zOcDXSH{J`uBO(T6e1=r|B8Hoib@uNR48p(kRJgS=QNL>r{C>-OhRG2gYe2ch5( zLm{ol*6BhgF6D1?(({pQYVcP_&@(YZf@hQm%A#S8b|Vg~6!>90iIfCv}dHAJJ) zz%0!(gM!;nhx7!6-0$j-E2le3$cuX!&pajui!>=$sYa}H;sTBI8aPtk{*5_{cD=qJ z_)a;Q$))mu-%$Bbs&KQOOWztvwLxhf1R5S)Q4x@-!O7Qw&=bSmfUX}e*hNhhM!(kK z;|Dj+dWBXFm|I-&RX6afsqnKBjdh(*nYZmMmrw<@pE=VYF1^3v8DCUlaY7WLGs(hvuzUK^rI|6P~{E?yEWieU3WcR-yFdBwZl4xT~5Ig2E#xyI(; zfeFDW+7q0{ER+9J_TG22m|23lrN_vP%mn8jM1)cW%5na@4^yDPOX3EwA1D}{(1Kef zJes>1sX;^^Upez_4GVkk8P4XEKAX-KWN_7~oYuFRumUgAjDy(3O;bdRWXkYQMSIE>e+FBlU-DMfkP9qOC&P zY+?ap5a0cum_7L8FZs2|%?nDQYh%r(lowb>3GUD<62blR@0`dC$yu7Y{N1--)t_`v~l^f8=V^Dgb_sUu(pD zcKZ;*$~mLJm=Sdy4ViPhH%;Z7L;N+1Y})5<(TT@HL~ibTu^Oib+O_x`*?S!BdxC$+**Y#pY$6 z^UK+wrVNc)(b;9|%R%%h!0GwLWVc*?TyL9h*Mlzb`c)B`w=Q8V?`12ZW(~Ie1{WUR z(D|oXTF&M(|9q!#Zrse@@~4^C$$h`?DE!|&+lTV0UrYhjGID>%Ga@3#jvJ^{XSUALswma4)jl>c&6pxc0@xBj&W{ zYtcL(RtK7EUim`X{F(iZ{DdBfi$!&Ryr#*AFbUu(jDU*~8_?Qk27INl^TBURMiFB- z{sM;IslR0at6~N?x8d)DoRFMHA&d3unh9j#%d(*_l{Q8GiMh)2v!$Dh5o`X==p<~Un;#{7L#k!@Sf{ZTh9vib48Kk9TvHk|(#ly05!pD!*xM{Lih zreM(fK~4My?urrbJ>(1Xmd*-pe&sucm(M}UE@+~;q0IRlD8+3uWpJ6nJ3u1NGk=L78%#Lt{=paIX;&SQ8l6o$FQoQoXn%0c>wIV_)z0oy?345py%Tx@(fP>{?ptr9^T?c{+to{UH{%BzDnKUxJo-J4&j?5R%5nnS zNeQ}o?bgS;Pr|WaXiE5kE8w-MQs`HGnRJ)dj?~K}kUjmqYhrIE!i?uXs(Z6^LxS$o zLeVaeV++1;I|+YTr0=~VD5_5r$ueZIm_u!qh(=xjLUvZ@7sCUnxnP}^oL79f4QBOACTD4<6q;>OuVP<}1G7H=pg%3i(M)Z7Tf=s6wd0AOG z^kM@a^yr@99p4Rhu7L>mcVaTvQ#TvwjIPebkx?w`&E)0K1-I4PgX%9G9_@fgqNW

qQ3X?KyCRH`gC@OAFSbo{F37Q(gVuAAfO#MLcxJ0&LlEb*pIOpP+I zU@DYW&H~;MRUv&cEc9N>luC%>{la8Tmu={c8w%n6x`?3S`<_1jrgGaaJ1}*3GumiwyQS-x}&)>StPfC(B)2dbty~e>{DnoY*_<+ z|B_y7*(Q+}DQCg4adw7VJvX9tE6Ov-NBhj4?G5u)4(MfVq(08SZa5T!PMNgBEEAjP zXKx%L($CJc4~DQhg0`zJ8`P94ECXIIV>caWz$YcVEY}s0+Hs*|jz(12GYL2pVs;E3^$sSeBzht(WLeVWo=N5lR@T8teVof_Lo5_f|@ zqs(XD9Lrf9?Dl+i_|*peINKE~*s4C=VKzu9wDQ(_57BYR!bLQ-cUml^dd|-e&gw$H zKBccLIlngR!R!suaXvNToVa(-UC+nom>CJNotb093jN#nDfB_tt(cSPo3R}QJ*E|g z`7{1<5|xGDU=E7>p2+R9+m@AlHzuUyne9vMTq<~I)1VW+`^Y{F*S84o@1C9WdAxt^ zew^1{huDwpclzowh0J=;*1fCDmH&c2F=)$jnhB^WPZOjm!eBN;-^`5nAd`Z~*x8zK za)}SqR>YlmIEf!>787hhc0+}1Y4G)p3%Cv$(C4A7n{xBQk^^)oD$R76mk^il;@?gU z&!nj*Aa%rZL7Th3%^r zc#Bl3YXu<>VrZznRX(GO&hd0F*!LJ`!BSBENNv-vchVEuY7qCqnoqBL7CJ0m8Vthu zCs#@4yTF}Nb3B+5uO9QNrzx3Q2)&meY4ylDTxgg$i(igGOnmIAfP*bUVkM`_3Ey{4 zvCM}J)tQFdPaE$bo`qNm|El?MaB#g6XFYauS>{ADNPb*!4IG_Po~(#*4k=Ev7^a3$ z!)-h_n>46bQzb^+Ov0VZx7T|A(8Bg^kFiI!>$7Kr;P*k$W==5H@Jd+(Ud62`KSVDC z%M^vBGwtHHizTYu!b@f6-)(I67-HzALzcaD9IJDU9Hmmq>7wJiYVK~Z^7yWCWA+D| z#Wd0y8Ye^?me~83%e-6CnddG{zQc~gf=6_m-iH=+XMOWh_K#Xx>#xPhZ^zR^@LZiR zrD>-fpV8T2g}QeQTcZ!VB2=P7Yz>#%c5egnqQpMe?3?I52z4gr4H%l@#YQ$CjvK1{ zc4Rc!1AaSFE61sD(NBG@a=c>DwYb1wCXSFOwiOiG3&5QbSs$;zQN}Vpy&Dmz-+OHf zjAg>EXPJ~+d2n{68hDC^3^mG5ID2O@1vh3g8P&m*kLnx7q%|NQ8_)kO5na62V5^OM z_W=*JQwj>1MV5D*RZc>o*D~h)*mYBCzA}?kOj?_@Jx?iFT!qSty@G7ZLCzMUvAki za6la%! z@O=ai^ETM-Jk`kKuf1@-_`h|q^7I8kgmT&st+E3;pUea)4czhHVoA-{DBgvoFr`dp zNlCl~JAL^3-)yuv;L9Y;YPdt$^HT1sze4H9mv$|L9ktt-+pYt%z<>XLj$7;`=dV|f z|7(c(*IRJdYyLx!`(F}{|1TrkfBM+r^`+%45BlN1zN&bu-G)?XE9;30n+~{-H)2_T z?Z_OrR#*f&be6!5RIY8eZ$=$p(@Kqc&&)q#80K$HzP?=R{jgmr#;jDkZ1`L~MjRRq zCJlA0$c0*DY;`&B`K>pOCa=i5?PfF;KD|vvZ)5y zg;=I;t;%ipscmicZd;?MrJfxvk^J50P)2^}dBPSA-t8s^9k6u#>v1aF=j}KBgWLT- zx2w}GZv1%KjrDd7St+xU3)@6-REWOD(QJ0a#nYEG&k=kyXB(!$R5+ZatOcP`;M=20MVIq^ zt)6z_U85OTvG`!e2rqyy=&axFky_v6#*4IQaN6o z`J22cFPQTPLVfN$9`|=l-2uxZ2te1gK_-FWyd%}BrXg)wqneuW0+9!vdpAE5Ozr&h$Z^& zD)DypPajv4<2#aCRt0BQEm7b=lbB_*_7FxM6B;M<=HmO%#9M$3t~<{AMrGa8Mc8=$ zmyf`H2Rq+9b=dy6{rB0Pr^AsdSy$#)kDc~$SGngbIyX}kgqmFX1Dl2}XB6-rPsuXd zU{MME%e5C4K_K-vB>8<`hSX@b$v(^bWM)DtPA9839PyMEzYH+dQ9hr1aZpVA!<3s# zmy6I%JypJTIR%FV1Tee82TH}F=4W(V1Iq+^3-sVU!l zdOOYNkb**)l0-mxK`*KbBudT^19f+QxBK=Dd=o55nz>~pw&dk|4XJcej%*Dg%<3+2 zd*HZV(XryT{ra5qPm4{I9bNH<5f-xFqNqnxgv6UY}$cNlk@7> zY+QIAv?*bMiu4pLUT5!dc#mf0Qh`wDzuk zTH(RZ$8Y*}FlRYlbvZ=bgjZ`g0}+DTW&a_p`NW0qtH-ixROuZDm+NYmM#T}+8C|Ej zFS&mSde9rZb1owU{`7>W8|C75!@Ky74ht;!Tgp7|5#4$*Xr;oL29gVa^0?A*Bx!3kwzIy5^)5 zh!IoKWH(801>_0X6rL#qOF^&Pcj9u0FU}wIit^+SZ*fw9(eZ|$QQTEf=aez?d46k| zhX%nGKWBp|bMI9vC(h8R=sCx28R}RohDMpezJHY?yKG%|n%T;qvvE>zk85o42w`#V z4%Oly0EVt6xrwRg1FF<7x6jbrg^v~oo4;b6l;-nY1bkLqpC zY09AC^8D}gAwM(qrPZL-kg3`fWO=YP?_Bg46@GWyQnD9-?}%2+osu2aMdPkNsD~%` ztV0@uiM@gEY3e2({DRCzavJT~i!zU0f!^Tt6vbkv__3rEPe(LKPsYP9Bt5$KGZqY! zPhW;f&D~>3&L+neUbKVuZfP?DU=ZmTMJcrVqGVFwWr;IGHXuY9Gq?m=Plw)7Awk`3kl2PZ{Z%L zZ(#_ciEP@dj`pPvtcit+4WNH1BWkxmGP#veNk>)_<#a+WZudNeckf3oQu?PDAE2k# zP0h?j1>`Ujivg?TdKPy5Eu#w>NW%BEvZOT|S1hP0WQkj7OafMTi+0x{8bx`Fqfxy5 z(no2x%hkG{25PVe`Q51aaLjetNp5B#+Q+QTb*E6Px`zkfTgQQrcXcDy{#T%SffZSK zV~0ntni|>q9i4r0{*_MC2M?xxrX9>S!|JU3iorc#pykSPf7ltDo5YS4vY&7W(^eW> z$S`!c@x)u>3ln5)i-3msM1Dq8LMwHTjHFbak_U~uC8<=fEU9;v>t?#@iC)7lrT&T% z0p%p}va!>=CxL5avod$-^5U_4XU^VfUBw(w=;8yvF-D~YFebq`5TPUeXI;E`s5JQ6mh4u3V^ zZFp|(tHLC?g(9!w;h~k{$QH63VL9+nV&%{QPGkgs&xBCrV z;r3W7P5;#OtUywacZHnjx~H8sw)-2P;#6H7%5RQO`1`vmffATSPJhTKd+T+KO}pFR zHwSB;>o*QletYwEgE=?n+Ea$~1+;Bj+x5J%F=^Q@|3i<5c4Vr3NqeJm?@l{|7?v<- z+BAhj3Udr$LPVr_!!HnzQLvCXM_}*14mZ;@)~!7mDVn#Bp1Tl6#6Hw4N6HM`p)QeO zAYtAxY9}pCU2LB<__bMZ+OtVJMApvdcq~w zVmj6Plg5@i)57R`nU#Zvl@;!}NTlpACNCHmUOirjqsQC{ZQI4WuU2dWdrpUZI=AE$ ztlJ81m0y{j8{MAr9lA6ETQF89KYkju6ldMC}pFGCE$-coLr@YN@UnM8;o~a$i zRZABI!Dy00M^B(vHxx+jH!L7C6$S@a)|GE3*VgIWKsLz1l=BKekg2I58OC}Gbz1*Q zeWvRD+63;+yEa71OS{bUbFqO%n=$JSl_~XN^>JB+OU9#MmoEP8kndX=JTVw*ru&P> zZOblVcGT5#32DFBw)jCUhAqiY!bb_WKH^4Q@MCFSl|3=yB73dfSOg~s#*_v!1DZRA=ekFhW#1X<$auohSY}o%Lal*luDkF;lUx~o^IDrWK%-gUD3Qh zmfKzrI2}C@wHSGj#4T&}Vgtj>a9nF$qwd19_lu060%?@K%zX+-LFK%2~3oix%TTQjmRZH8wxlrXTt)HUsmA zd7gs`qO(xo;~67g+A(D*fK>c~3`6YzS4updVFVnDr5GDDm+zj6vFSXG4m)-ID#C-w zd{0|W8Q6-K|Fv40)Y#$5Mb%<>nr78LK4NvVti6$N3H{Ts(GqvMikV%OD339kp>W8d zL5{!GTC;cOSJ^+nHV1s4y94HptCE195>vie41=9H524sc_y&joyzzObxstdLPuBjF zfre^jkxx6z%^Jn(1WSNG=Wff*QuhmTh(hsDM=e8`^povWjZ?_<+jMIkZa^uqQUM~v zN*J{GqT^ZX>4dR;66>DmrfZ=*cA>u!|8&T2{9%1yjxNhXV6>?dPd$j-w$N0Pyhmu- zwFR@7C{#YZ&{T_B(Rco%K()s>RDHD$dGNgqcuO4X#J8DyX6+c!R&NMT&`x(_W+bls zBKNal;!s1yUPl8&fuf&W3?n z>Izm>w|xRlfkP?Swveo@yxYo!PTg+lWfZ&ip5Un5U1)_M!}BB#e;4zq#T!=scXnVCr~|v`hoKT!Y&@!?4RIyYY*0GcNLikPGc=U zB3))S!>4PFvQ3Yg8R)&EYnB~834 z?b@c|kW?qQl<2q?^%*OmDmW$W!)cO{r>>NjVA`_{E}C<+QWtS`A2z@c_`jl^ty%fh zOxns|bu6mu`%gqsah$!}Q)W}ztoVsF4$e+`cKe58{WJFVbCzQz9AK@MFsT@PJS;!e zglRTxR0j8(Qx6RAO-NQu!TU$q23`e*uMMl5sU9^9mH8I`AmWwlw(X!+WH59HCqIl)6Yr;XJfPYNb~KIvLU3_kb+&g7Z;o7BDW}}c*=9nyh8tSY| zi3xaf?QoTDG&n){&6vDwNE5Ckb15Nz`*W{bQ5ijmfS63Kj@7^8p+|HS38La@&$wCm z+_a-Ri)8Igq5VwZ2as1~R&?*8Jil}o7E@~*kd0zOmzPuUN!J_w#%KW0RB<7EBz5y= zSpE82TqM6}@N`HSb>*fQSmQ-7shN|0c;|3$yWDB|tBkt=^UvH>MqR|cHH9$Ry}Jze zJi833P35t#_a#I{8DSyDx|3Tf{RjxLIT#R)W%BCsiu?jp;q%VG;-9Y$MA%mJC}y6w19*SyyVT-jl^QB zw)Y6^toGv4A6LBYs8}!U0L;i=`ji+{iK!cV4Xe?;uof%Yn=9GEduXACCY(t532Gi6 z-L(*OHJ=2%p9O316&i}vR@JVGoVg$VP_eM!{u)AN&7CRnsBRbMhngRs2Y}SNN+M}s z8GpSZgl7+MUG{iTgaK68M($=z%-bQ%GFtku=F}+tR!;wdTzu4p zpHImM7Y96xH|R(!KZtqyy;VOSY6f;O_LpmZe2#I8FkfK-Y*c)YA!TPPs0>1E6LTXi z_(drnI47{9v}lm;4vzzZOKHFISo^^H`oCA@T`MGhugbqV@;|HDqp#TjSg9oe8!yMa z(EOjMl&S}Gl6>GW3#EB-` zz|M>Qa<^PFfvU>j4NCtFy z=UtJ(+>)eIT0eRK9i)yvZ>;vIfu_mTi`9X|q{q$qL_cVh8Q@xf%eWXP1IC%L=jn;J675}9^ClDONdxTZV-ns8CF&zz!Gk=PI9yg2MBAoRfXKCgM zg_i;RS+zV(*El1z?Z3@j)KuYk^hvd>6ruE4w*IU_hk_S~4Q32ucRpZw?9(0}nswL$ zUpX5e533?IDSQ<^m*}l=PspRQ8uj9XIn?W6SI1fcvxa;qv0lp|*ymsu`}+3}sFRH}3!CJc=*!bgm8(hHQ6{Aac>ZFOeM}zxmKg8j9X}O;( zgafl%i-9`VdnqPgP|3xY^>zYsBO{4#I5dE4m7)48@@GA54n zeBC$J?boA`nR=+fTC}yqMOF236#!NqM^2j?3`q->!#shw61%QYHaepi-vS(%mip$R zLwE2YwOzoiwn+Jz5IrwywF;lchIG;?Hz(e3D-}B{Hbtas)UxdL`f%u_QG@S*TqPrqDIVrAb|hDAgUO4Ylf}_)gh-m|(_o%T=RS6J&L9GSJY#chBKT_pbQi1TeVsWV;f)AP>$> zIHHTW#w#Q>vsktPyq+8{2uREg@CK_NEy34YNO(ZS`*Wd7DkgdkXq}RX0i}6Kcv@5- zs+LcoHN**uy1bRRRXSLsPYGUR_ZU9K6i752dA6v2gQ4_uXil@I5x-6MC6r`4KmNd# z@4*>L{_eHD2N^@WS5BxdlD4{#2<4eltM2CAKJPb109S%Kb8&ZLQ#k34(sk!NNSidB(Wk7$r=qOw6|isFRNw*2YG}lspqQu*o{Mm`Hi~% z*zo|+awlKZtar)ViS`T)y=#^oxOZ}HW>JNG9gl5b>bDlB`q>1yfmSg2-sy9P7W21y zECh>rl^mJ#-EHn4Xj9i*?Qe;hh)Neo(x22G_~UJoKHAOz>plQNFHqcuFg~Pf65~7= zX+xHw4foQjl+3U zwSS{9uP%-9Sdwg_dB#Z)Cc;#Zfm04EvUe|3lH%1%0ZyK}IAu8UW0qyFJ5Vr*^|&m~YfXn)_#1IFBWLC_+pHu@5NR|0;1B6GP` z)YOTJO6u8S1mf}{EEvU3rU}jgJtVUrw7exzynl>K+X^04;5UhP&noo$`qf+66fG$+M{dtaZ_`VW{_6CMI()HdTe7yBp zFqcn=3N-Ba;w$bg+^~_KLID5bp73Zl|D_78M9H8#D*I&hn4|;j$TMiHq_0LblUF@6 zac|>$wV|g9F>uj=<`U^!m9}l+zD+u3cRUo?)5R%X?9`ia;gLP2Ub#+z3Sf2wl7>2qy# zVwPFf0;Sl=-KxGz<(3rc7O`(ltVV3blol6a4_qjJ`?Vdv%oy>@gC@;*?7f`2yD>pW zh^z#nqHJT<+cQBLx-;p!2{8;3pgl%AiR5IWuF$M9_Y=XCFzqT>pL=iIaXl;PB$5Ie zZ@{ZWlchmwJx`W>+B2Y`oq?wgiFc2>Uf4=q306W&N);tRD>qrq>l$YjmWx9hQ)a8~ zq`yjSt7#+G#A^8imkU8^64|uYJ@{wfwpiko&zT^A3c@1#Ye>-G;vz)ko8q!+UR(HH zL~J0=e1`T+XgMOS_f5@Ov7ST9A3`{Li)k)IOwz_VB)mpUB3AqT>PR8ZW&B2$(p?+$ zJoPO6t(KDdD8;+}SF;3h-d_}^-_Y(;d0MU@ulVU*s`}#j?}+!TWU>v5LdP=|2+qUZ zAXI%*ow=a(foY=E4agUbRIs#Q8H zc)a50nn7Tm&(UjQK{t~O6(8FkmVaREv_RWm^uCP&Kex^z=TMb zR>Z~A;FR6ArXX zSpPc$DN0aT0AAbf-Ka`F#gtJ2V=F5?71VI?)V3{L`j_#1jYEqe3RB=g@SH|`%QC1>-EBw=r)I1Rb+WSh zPm%VqRBlb|{`iyG{>M=mSdKa8dt#<9e@~X%&lnbi6Ze-@$(uzGwpo|xj zT&n(?U`oAZuuWa@8pX0Jkj$qzbXyqx-KgH300w+9tgyMEG{d(pYO;F7UH7p?3P zij%?{ilCq=V9`>zAoK<3y@3-guM}~zZn%my)ceY&c%)j6WvPa^I`{0otW8rn4Ux*! zI(kdlt!J2uP96=xhqXa4qXn4__-@zkWWgF4$v$fLz8`eP|G&@~!*={HI)m-}8=di} z4G0`_KTbjnS=hOYZP!S{uB-7P9yNniNm+um0Z>kdUt*?@kD4f@#SxA%;cHV-H`RDYZ_*HS&;U;cST z4dC(N2aSwRK}D=BM83y!n8+-hd2_f-nzjLrLS<8P;ATr#bkZS5MR+bR3L z`N#(v_bTymIsr4;VS{u2+S9FDu1h+ut|Mf$jiOV4QrTN3m9&5ZmB3xoM^+!9 zA9|FMUO42^0^g?|6z|n}%}qR9>p2>=6$OWy!_^N5M^}d-EdYv%V3s>&i0pTUz5VWT zBE3xb(49zd0(b_l-qa25R0LV=u8%r=qUPh}p8i8c7Rk>j#oo<$mp zl%EP2UUUx%m03HFl+(g>63@9A%Gx8h9g`=T-f8GIyCL^$g=oL_aGs&uc=IGA;kKkg zZYA%HQYE)GLz_UW9@{%$JD&{Q!BKwIW08AvRlGvuXULcAbWXZEn;Qx!3#Is7F0cGK z=bOB5yhb^vItzIM=7+Kdcc`-h<_a&c7E?j;NH&8IbiejZf94Z+K z`^AN6brXqNHcp;X$VuxFp1qNk8udq6?=A|=`_44M?u_5Kmb&&4x*Su`G0A>-cf_TdCC|ON43@)$dBfo0_##fD;9qGx?TGPL&gbi;LBRjBl|2 z?)QJ-^QNU6kQt92IyccO)btLNO8lW1#eU)p5MdWzlD%g2(6{!dxi@>;WQHCqv6UoW zhZHLhFI*Z0VFaabNRT&Q11&6WfgE0+hnsh=9?YdKF?HZxs>M*r!KC_nH%UTP4s+zo z7Vpn|+5V#Vt=Ng2{>4=A1F+7h(;FdZN1sdBU7OVEjvTHVXV;?n(wy3?5CAJx$h`4G zhv&JmTI|p^K9bWWCi_)dU=_t-(TjZ}djH2SnErXz1zj1_n2tg1#g)}R>Zitk2~{vV z+ zv)HL&JhM};0=BuuSYGhAI!3Ei66;`xZ4ccPq|H4?Ei1vX84HOwPow{h7%^FjMtTlOD}E=LWIoxWgE{P zF0zRkjJo*VHtO)J*Tl@o-skTCe+XLH5TbpihEELojVYjt^W|P02Tw%^pOT1i3WM0N zuM(S_bn?jhImI74Jz)#o#scY_DXx1OXk%2I1jk|C{d{3zN$nh^_fN-`NFLCR(=%)-R-&re?pvCFYmpfb8tlrutHG@h1F|=2%XH;yGBwUq&Rqk? zis#m+$s~uWE1vPKR~z#ixHy7sjw272I)VO;a>iuugHjI-E1*K?RLi`3rwSx-=F>Xu z!gCXOg&+cJAP*^HI%<X8>$QNU#VjKUrlCQ z&7QE;KOU2WSX_h)_QQK&(-_f$Lhb=rpP2iLm-=|MU5jo>IsAYlE|O~G|YEBS5_}p zhv9cE_}9w^wGdF}i@*A3c;iBBd&*33>K1%_)`@Pmff{NxT4f@ycI^*m4G^wKIq2^= z1&gi3CLeUwFc{eN?EKorndX>OZ^MWC?q~&K1>av+0UF}j$_O+NsV2}RRI^$=Nyv|| zZipF(pTd!qHfhLvL}O}ZThyMJkSscrjX$SdD*T@}3gZOQAY(m!B5^Jh+q4)Q&28`| zaw<__t}nv2lv>udYZ;7W*%a+(k<;w$F4PW)M2bxLN`$2Z7e1>d$J%toSZOF>x+;w= zy@t77zMU*ef38w1EuL&T6o;J!;Y$1adj2f)B&X-5d+H8bY_WLecxQ;JGkzuQ44>7Z z+DV^oUDeA_(yQDRyN;bx{}RW@(5a+35R>FY3{{9JV(}y*@5y8rskQjn?gWov4~fhk zPyPaoXPACy+aiFre~#~Cr^o-yRdV)R^2A&4?ft~LO?Eqn5Nzc3+tu7wqv}5*j(LmB z%IH4H8-CEOwnJ38*q7Yn9MPMfyVLVrEO;Vns+1diL293INm8X(T%nXQlEaAmZ)^qm zgRLAq=$ZII2G$ql3@-T*JeoL<^Kst&Q{dum`Jg@k7Zj_~7Th*>B?U{lx+*kCT}aSY zv8m$quG4H+5iu&}T4Qrlk-N*|>bTmkhX(duf`-&YfuxcwxLsNWZrh}8Wz$Y&bgKW* zxv>6>C2hoTJ=M!qWBNO8BB`)JFSO~Ba`Y$>iafp&tAX1p*>`#z?6xRIbw2cW(4tkL z0A_9pO%N4;`0a4=%@C!R#;Ug^NuPhj`9m)5&Ms4462Y0Ov_+;#kFbRxWXD8FDc4lF zV6QVJt?B~=D}W)0MQ!{1=rl0bG1yw^ZaViN&#Tlci0tr%Ew^+PX8RkUU;DIxK0Y@c zqZu9xpoX-&A1gmDIgHcp)+67yF~7~ilKcEDaqbT?b>4VNjc;4g&-}F>dD!dG4d3I~ z17wH<$)u(Zo#Q?b^~64(XZo#)h+DDK+HGnt(`S~cr$bFl+ZTSMUge88-6`*##PG0@ zsMe^bcJFEFoxTzer*}VIgpA{NWntVo3K*=DZa2*exBaTP>T5^4e0##@9exT~PBz0X zF_@bira)1Wi#ZYCE^O99vr_jozQWw0sa&k4Cp8{@q7WnfT?os6YKY&u_BktZz<;1; z*7MuW>c0tDq6$?yYjRBu3;jh**J|(AW+FYp31sT}&~0TY6(!;NKj8`c%7K0Jw!XS& z9Ah0|G$)z&CB?Ar!?n2RPb(uY8+(rXrg=d1^o48tZ_Q}R+j$&=`7r6-`nqwxAKz(!uf%#LMS>M55e&lvi#}J=^ob@fgqWe3 zj9*e|>4WSNU02WPMTyso;Snl$pQCEGZ}AA*Y1fPSLL5ajDaO-8D7buR0i|~2@Ze3RPSr_Q-BD?eU9McC3b>2dc9&69ABjOO`wOcQTFkP_}%AWAiZM9VwA0JsMCch zmHsrc6}C!X;#ArM&UlDdx$Bar|NEMBWg!pIuVIs&UdS)7$7{hq4sdcTDU>n$@{>Ij z+0!PjI{rd0{fhan1CQFo&#r#%X8_NW)*9B( zDqwhEX+L<2G$L4B`SzRAjF%jcRP0SPHZsRHkkpMzi3SOGPgKeb6n2eQgUxr*Eo%vk zz_?N?-^ZqKHU<&EN__Z_7$l}U84Le~L5fSR*sg^=BMT3@@1b!e=|P1H3M5G*?v$8X z+za-6DFU8;)33n39BQE~QF5w+9C28{^!qGCQjsPr58_Ly5n-FjQP)N38z>gX|OG z!u?S8U=`!?d#v{YPmb3=>e$6s#bJ=h7f^Dk#n;&GG2KL3_=hTGbo5VE3T9<paGjrnCL*I1OVhLb{J^^S`+;34RuK;c1w4@{)Nx`Lo5oRaWG(rohTUF1kKh|?mW=*V%@G4IuwZs*Oe2sFBNC#RS9QW!=l_Dm-)Xc^5ygm0O8*Mlq58 zWBMvQq*0R7o>k#+Usb70{_c zdM}N$6Mc3oq)KVflB(fTm-J!x_(}`8(xu8EZ*u$Z9XN=FI06e zFgYb3=VbD_bNye@4aWwV^zx3Y6~;Hj{L#vNu-kWoI?yc%3FiLk)lUn+!qZpESy(6D zmktl{D8{C)cj4NGw^1kXMsnR{VgBb09GSPV_}d43-;?@EdC`(8IMjPF_uv~h&SD6I z2R>sW+!!pSbG6j0j6Y0<*^z? z^H1H0X?*2JYKncyumgk}){_12@rGk~pNv8~Vxlc_a%@3k^II+NYz)AkXi}m#l7&^drDY5;X9xB*?rcn=Y0oei+Ms`)q3Ts*xvA1H&+H&Wb7^K*~ z=hu_{PIUhp1<6Dy!8R!{66ba-LNBFbXU%S;U3{!eG3DJAc&~>j3iZo=br2boq!5of zJ8Nw@5!mIp^kuN{)_KDFub9MP8kGHN7psq(j@QI7bxQw_5E#3lA;To+pq7E!`P!Tp#o9bocaYfopX9buY`W7R+O zTbIAgT&_#`^&=&o8v4zg$C1VL5B#Cx)<3j?_|GOf-GRaW@{X5D<)209OiebomhwZ( z#{1)!_c?0CK%YOIxp+2SE^IEP#VKp)2A~}DxP+@x-A_CDJqEh#jZKB z{MlNTwL98V6+UM)sQ zeb99S>mw;6RpWrKp2{aoozFD)afPe;PS}S(T}cJ%1g$g6mIsb0psh? zvr|TccWgQ#R}KwC1|1iXDNb_cz6wiM7Lu5yry-%H&b$0s>G2u28VX8dW~L25!N zbssPggr0R{7w?guk6IDMJ{$kL4xQdlHCoFr0D!&EsR)_>-Z zR;zsLCYUg~ts=|?^AD7v-Fb$~B`~5I=~3&mwK4Ga%gpLIpg!*ZjWX!px>F*0Uu2N% zjBy7ZuD=*?g=XE1YvMpR!O1v>@J2HurJmu~~k zGCd3DWNm9R<$4S3PJ;vIKAyqK2~Q8s^X`w^zpJMG?xH7+QotyAo)Km>x@=sZ4@`Vm z3fEhp>fj%iQm3XJtXW&_wPI1XA!>z$Yjcf3^@FG~5@yM{e!OVeNA8>-p$PX&OLSlRcTbM@ zw+*#3OC#-dFubhFNQ!{+^vy@X6+$cMB8h{WxnwU=AG24IT$g$DTy!fAp!$Lwn&*Oh!4%dG&2dx7i zlgJM4O28+hv4<7I-tT2wa_m3+vg9bSw%C>&SEFL!zp&)!a?Gn38v-S>S^pJ090xNu zXZhm%9heVHV$;qXT#K^#NAw`HGw94ev4^aC|CT*mtP!C%OpHFBcCyg9lJU>rVdRYG z*j-#7I59NzW1x0D!=PS-RJrx8v8Ht0n$pK-Kwvy@*)*?e3J8%M{lIW2bl<7}Lkr|v zpS&f=Q(~v#iL52Ph0V<{1?PLeze~~-@lVttyyHAPx;WU+ zmU$$^m)^WfPf#lD^faEnOA%eZ*GWkF5*eKpG6{V!Ag%bVHk;<=+c*Avu79eTU4DA$ z=jD4kr0i!UKv zOl8QNzvOM9f@fq{(F%E-mqSUW(z=n1yeZwI6z4QqNDQkSK3 zI%|0RC~VzxwnC1%m&`J^%ZI>9P@We zj=eLH#(JF3h_axg?U`|z(H{m3jQ>>lfDvFJ99!Y@j)yQ817L2wrQ9%kX7zIF-%CO= ztZj~CZh&Q7g@%RT!|Gi46l8wwN^;bwfQ-m@kNtI468#<5lXg4$D^|QyDjR`z;ZWz{ z$^!v*)!9PCI@ZOrx!oRsc`-v8gI&$c=(yHC*GI3RP3W~luabvfHRwb1tjt0=8|TTx zZO-8rF&R>6q+g-lvmJjIgDj7@#SK&&TzPE&uhbwrd1aIZ99=xRwR>MCFy$${^apy{ zfxBGLyny(Mx;js`8pOW&Z_b=9uE#|Lxhg-$1cv8Xx7TjP-FkD<&(&!$vYD6&cv!!|xf9y>|SGZmr>%s$8lhdn_&|A=( zWcMvDSxvXLs1n1X_T-{$Ybzbfy=? zW(zneI8=Ak>u$g59badKuRhaqwHbYEC;i>EmDvor@_`wbA(*~Y$2yK zJyr+!b0^_A=E43giUK7lHQtQjV@Z=k)|X;;)m)8JeCyEVUVX1g{N2W76=MVLMK7(- zu>3~x%#^{QoO-qM3y-?p9W;z)`?iS#_}LnR;f-E(DL}Z z@5)J`7i(AzoYB<8nmZc$HED%u7hw%n#yO-k0oyTwvV$I{-3bLi?F!{OM2ppZ>P9 z3EMK3JnIovXqAoP{OWZ^$pl~#l-4~Oi!O;S%q+{HO&p0UQOjdN-n0q-jGgWs|ILvj zu}q~XopGOKS6oBhx7b@b^3GgFY>@_jsXz*Q1$MlX7Wsb9<=H|GN_;8*o{~rBN@8_y zsY1D&ZQ|5MEA2{(TA{OWWSBh6kR7qXOvFB$N-M&Xdg>1UrUHRPFURz?lJ;}n-|J;n zW{XTtc~#yq{tfY9M`5^)nK-V@Kz}hb(u#olH#(5f=D&0xbpv-^Iyz^8UB-xKU_IR)!Tl9Zzr&I#!GAivnRZAya@aN)57{tPt89YlQ25>l(>H)nD$7tz;FcR zqQFoXr`g~cdfyfMxM)D?z`Y^r4?83-p?c-=;kTu#v`wHOHHa0cJ=o$HT)RG+pLwCV zhn+ctR8Iuy#>X%oOs$uPXxO0TCf4UqzcXYfg#GA~>|Sa<%5M^g@{2bmr(WhJo%qa2 zZ%9w*73LWob$2@MMeR?2qW+~4Q>dj0%UUql)E9%))*N1q zh%f>$IH!b=!X<+Xm0Ed)`{)Ip2iEBhBX5F7rRd=1!UCgFOnYttUBe&>;YggD-s$d>0ma(t2}TvZ$gm^( zyL?`@YB!wb@;P(Mu=(H+$B%#w-cm7p4RiXz3X za^=A-VGlbXZE0my)#VXIOuPdjbEXf3vb)}?{9}{48H&Op&CCfnz@_`*&E(|`5FWz0|p!c zFT#7jdh@hv_whTG1=s}1+_cpMOdIKSdMa&~GiC}zW3$YDzoGqQz=d`I^%(vdz2NIQ z3ivCleCa%2RsKT3F8?jC;!#^|P8p9+v-gD~h;&F+KecG?MH`g06ASrVD?m}I$ zuV7{3jyNCWtmNLn0};uo*NcZT7ru*%k>+@CQH*KkouR#9wqJWVuCHk0`6^cDHJwV!KnsEPmzlQN`yN1us za;Xo`KhJJsPWFU$;%R&}hCi<1JQkiRGY^Fs@lfOtogtZZ#+oVzj!r7clp{4Xi{^Sf z&xDdOr8AymR-a}N{D+a70@%x^`#kDS@{j@kUCnfSrE+3WHEhFon$V1rOCp<&{^1}p zWdmfUQ&O0}&MQmIfVclU00sGJ?rU8x=LgU~trG{V#Ib{&Z7&ErHVl;!I{k>zVh)Im zu*{g?Zorl*WN_JXMNIJfe9NH*7_~QU5eNlJ|9GR0Y0s9Qc9kB5XubXB<&=TQeX73M ze)>ieJ_^Tc_e1H?BW;mZpPUUQ?Xa?dhH;uP22E`p1n1)7oMM%LOpZSKIX z5a2{sVsg6t0kQd8LjXiJV__%f4RVmb=GfM}-J2##kg9B5ulkXmAF0{n^`(mf%ESkb zCx88waTfsise=rIM!<~O&Pjj~duIU?tMmiu1gLdTSJ%h3F^i5W*)(;<{NNfH7SHUE zU8PLA60pmtws(W*N48ojA7+%}x>am-tyQ))o%T<019Cp?>XQekifok63fv)KoB5aNX zDC^Dd$%6ACEdsk$3ftZH`amB-adVm_Cf~o&o-Da!yb9SivJ4+NP{y+9{(3*~!h|2}5mUH>T!b+`Z{oZy_^Hiwisz#`5#Ak5#ib zkPcgUZpFdsgF~@*#)9FR=%u!nH|deroLDhT5ZjY%d{DsW=`TXx~2*WE6X1n&#Ys1a~*-#ku<-h=G_l8*8ZVcxgEJZ06SRWd~5`g zL;DR;ljPn#%hCePMIQ-pnRaD9@mVZ0@bJo&Hio{ol`iqU_kHbKHGoy|Tzgy=NOl&#nV<=PI=cBnR8z3GGA^r00qk6Y&XH z=DoJ{HU>{2ZnNhN1E_JW*>8GRO~B3XrjQ2*%ApFPDL<&gd;D}DN__ZODyib#=|8uY zz3=o-*EYO>u|6lY{^X~SZcWbICMu&sDg!@kzVo;w%ht|0Fi0F|=Z?T$`luCRPP5;Z zO{|C-1|bqsH!iX>HB!CD@7F=k)0TN#qyyN4u~6mnae2?CQ|u!30UNNhJ)uEmz-V;w zbC+$Sp%{lqyqb905S-p+I<9wbTP!ur@ajRbFgBqJ5V_7SoN(+$9 zat-AXS%PUMWW;uqi#(NFJ!06LRyG~x-;aNTk~0iDG0^h3K2h&D*>UsL?P-UeiFkir zt?Cf9&4lC6G+(Wi6wxkk_s=XvFr(k{X z@@~GZK;<)hLVa2UYl+F~oC!Ynq4`ls-o~eQh{6<;52;mECS7$iIB;+3OM_ZRnQ?-` zOPu0+Ko)JFjh~umdL_Kr1y^V3ahDQ#zwY8 zZo8XweuBC2b(M)S@c%fgpeSZETJUD=%dqdIQjuc=xY-@bmxV@vJDs zMk4Bs!sD8k!8zPl2o1LrTp#?rg&Y)j`)}CpaSePhHv6@zoSqIEv=UssC`VMix3f9R zrgxgHaqC3L9UY+;bX*)Jt+SQ~c9ytWz*QG$g6Gnz_nai$0rwzV35%bZdP-T>U?*Mv zz%&+G&kc_+K-MgK87dtLx`Zi7GbXx`i~`i>;28- z){cU-8pJ8zTk$%2BQc@WGmQnGJ;NsLD=MLGzD`*;=cZn$t-2Clrw&+^yU{PsmH|ca zXPk2(yfq>|S{Iy%nirJhuE)03%&Wpf?#9=2+Pxe-?C?snl)N4oFg80|b%`%KT@iZH z>yXuRuUy_(!PBFQ{$86!Gz1qE6cts9k~7$nbOB0_Lq8As-kl`hQXZ8xrSohl@)2NRl`Y2aA-`Ks?Zk9@8W{yd@nR2Td;7e38_03| z_ab61U_1dsSWo#jOP;Tx@1hgI)=3-=Q};?x6*-|Yo$?KBT6dZE6ROR#`j+BMo>3F; z&JTQb;<(HBxRujA-%VrIkNa4vAf*~A}cH0S+e#tDoN;x*7q7=SXwy#^uo zi1XJlZG}?&ug|#y5SR@19m_f;q!VUkz^OOPC>C z@ws=nLZxX4gHEBmt(D6>yWn62ky4qO zijR%NC#6?f@z&2iICd|{0Qa@Pyv>qA+I?Rx$}Apq9$Z=sKnBe`PF!^Uu%1^pI}@2= ze6IEelq}@9vbXML+@q<8-_}G+*KMf@BTPw`F(8KLccLHkWU1{b z5>yoVXC4&O%b+7nOd}vlCT8oXLe$u3dUv8dXmpWJ$$L26qy*!i6|wF)HrrEWHklNqT^>2{VD#!;!@}8e~$I()kJqHovWueDu~A*4-bRx&oW3 z4j#)q!PBN-V&UcwyTi(CerF$sVZRz1echj!$^O!^Gi0tHF1*@75>wcF^m6W7yu}yz z;MIY5>O5!fzFw5Yew{P%6z-$e4lQ6W@Tj|~e~8DF_RG;G2KRaU>9LVVB*VvHyYyCB zAAJ0THitK#PQ=v-yif2dZnwYweIr%2*ni~MuQ9u*1`Jt*ep>YozU=Vud==6ZbzXPn zYU*b!Le%5l1s zJag$v@mR~nl5SI7c<#u`N5F@?hhBY;>3k%mOeLvKsQ|HjevimEcOn3uF@No9q3O#y zPsxR)Ndp*1f>jPw^(m|}kLx8l%;;rpQjY!*D~)u294B0EO;6<;SQsNP`mFQB>zD1b zf{pb@pz4-V;j$xRtp2taqOUa0^^Me>s5#|59-zwVwZi&58)_cVj_uJ8%mx>UOs5D( z>QmJF@QE4A$2>aW!euYBJm^ye+Cal&w=gdxpyk$0p%t=_LUeY~6iYe`|u z0)t)_;$vV_u!(elN$OXR&Fegm!dDJWxJ6Sz)t&A}*V| z#YCM0xzfh_rY;{cLljh*QV{jmjS?5rxuBl>kNt&BWrCzho6^T(^m`A+|7pX-x& zG*zvyzJfkPmuxxt zv@%u8{4R@sX7F(Zl=Mn}w$t2NB(mU4gbL?_F$bHZ`lo8eJKEn=%Vq~h$hC*JatRpT z_0GPrWU!-C|HzY-GVYYWfWS>JeVGlfjgOHw^n zA%X9;ZVs}Ceblo0Nw0|ovGwABR!VC5JG(3_=t_Hl4aKmAb*l96?7S^2&AWbYGv9>w zfu?{u>tiO(y|p;0tYxuZALiIqK*al((wq;CNq>WB?~pY4m+$f)?j^~1EdsjS>^yV5nTjQm_I-ODtUTKgloLJK9@QIE&kVnS4}DMtnLJsq;g- zEsXPhdQVHcp|{p+Ygn~5aQx0fK$pM;gT%K|?Ms;{-|Ka6_pV+3d63v(0vrfObzDC$ zJuYcD^*)nxFP1X~`afz?*ax%He^mazYf=r~%ZQh#({oM9`K0RV<#cEIXt9X?*M;_i zu0D>BWrtB6yHLsUNdTLiEt$fgAKJ}o$tQ} zc7fxgfAUl6ZrFC}Z*^Ia5)*MUzc`HALp|H>5~pUC>KVWlg^>FiTx7Z>j7U8mO&e1l zPN4$sNu^h>T0fP}6Pf{64j&N+TwG%pmVV%%Po2o#+G{#p>QC{A`*6Y$)RHsoe`iaS zR*?>)PsBpY264X(9dX=%T;x4X;&E~%?Hx(u_iXx+xA%@mTRHa=+5wpa^-Acq?5+*iL6V0%guc@ldx^ zv*?H$6+=93AOjvKy)|X2F-+-YKw#{cK{R~qQ6iI~nDqcKu0(}w6jAJJeC<+gj`*qv zV4v(#9OF;8->DctZ)hXb+aBXUa3ikTkpS55BICXKFR}BX86aqUw3{PVKylb%iw?%2 zUf`V0_J_?yB4#x?6iqKNo8HY6K)^=0f(LoNgLwsIF!*24jYO|?J2aYlz*H@ zD@zbJGprPtw?GgJEP^oObuvrIY`ZOa#`NuhmTO>V$#vYG9~Ib@*KK2S%Vw#k-rLkL>6Apo~BT zl7Dr`ceu;?XXTM%>ep}1tiZwGGAlS3vF`JEGq%!Vq&P^&E*^S*a=FM@1c1rF0daE` zv}uRkT3nj%3~0F!Z|Xl`lFRjg1sxMq_c#CV^{uYgBw#ktQq#SrW%mC^tqTaVxQ=?^ zhvmZOe4ZU|c^y^;-LQEJTYtr7wIw$JD4FJ_-bl{uy1bBfKShnFtRo`rIQ&o$q>f~g zra!Mq+C;vQ5V3#@_IN23`!Gd2g%Evf`4+Sfa$Plezu;fcG)bZVPCj$t{7XLjly#{e zPpUcftesDypT*tI9lzv>t@c!yl0GzRh31K(H zTdAtEoRuzt2HjE5AsW8XSpJ7VDYv&?THc3m&m zll>d-RYUh0s0*H3yVY!Pv8ww4to12P!9%Pt z1TurX45X-6`W|2>)lW~xMxD_Wf6v}T(U@*!-@2bInQSJ{Yj2ij(xSLeI49nmL!`#{ z=n6iU7>4313h#ULP2>rk-6M!RFrD9fIra`DYf*bBDi56=C5(!F5p-zm)KZZ&2P)h8 zf6~jo2}OTXH7AB=xu;!@tH7_4t0En953G*GR|1VB$`F`e^kU9j^;aSKa&*4LBj4y# zH|V9~9@%f3nF;ld{Q;IMtLD2UB-j&+E&Z1_9vR@57eeg?LI-6(<;6h1O;g|_{wrjQ zN`M+_O!c;ramTm?itahoUA@a5; zsr8TB=y^~`z|Q4lsvrrIQSSAN(hD5#RR8!0vJXPMNg~*u@8Q!0bxN-mjEKt2hMSCi z<~>ujI@B&uJ!9#4q|_BS8In6zQ_9haOj5&Rn$V5GE93FO!V2Hx<6`L!T%4 zIuKrKj?G?)H3FgjuDAKTBV`asN9u)YHot=GVMqJTS5)CUdiTEiXx6k0)pkkaL7%Fc zSh#SjuW8wcrzNX!AG@rJ(D3Zd$k-9ii_(*CH5QnUzO7DB&sRSGt+z3mTKi%Qehfqh z!uGE}l^DXvh0kjbzLKf4IH+>{|5Ldw*z?W{GQZV*Vf`~ZK|W}dt0Q)$uJcO=%T+K- zknH=qXY}T=@|;^#gn{~KJiFy3O1}(c|5)%y$FR)?vczw?PHi}GZau&!!^}=e%JF8l z3Fbvwejk)9tE5+exY|4_i_#zutaa;+@z7K6i9hZoU&vkm!F5I%M<=~5j1SrOqj;c|W@JB-2$SJL>og8hjetr-6ZiXM<#&s-Oi^ASsM zP?NFU&3WtV(9`PYm+Pv(e$1`o4+1fatIL!ccn&tV_9_4e6N;Q1_Xxu z^j0KOgH)fv5}?m^em$-wSimxRQ*kE0KZKWK3p%>T1#!FA_U!0VM8p(+M8Khx&7aIo z>AfkA^7cfYO@RN|RdS+$fq_Q8Du@ZnPL<)QBPeD<;Kyip)eThcwv@APT2V%x%(o=# ze|8nYgqXpcUS5cWll&>0<=<1afrVIp=x z=^AjCeA}=kGLIAL5@iLe#hX^}Zy7$?6`;%*HeT=pnSFmQOaqf3(^#4!?f$UUDjy4^e_yl|WqEIO9_J{JLG|NcnE7mfd{+P0!0b2#{PM`dI9zLv%zjEZYV(i?vQg3dkw*hixZ&qj z{%Q9fmBbOd7Mt$|f0qW=cA=5Jit^pN^B%`^O5@ab;WQUk>}^b-uEli;GSXR?;F-kW z{R1nwv2yn32QdEA6TvF8-;V6oLKIoRF=Lq(N=Rv}!-aE}nx27oyl!LGCSqF+*lW*6 z`7J|u)7uT+b9@@>)XNAIKft#_LVWf6i0JoET=pawa*_7q(O=gph_abtj>gZBEnVCe zA-sFhEw-*JYfjwOuQk-ZkCIXe-jR1R zV_ZCZ-z1R|UL3eZDmbayG_9T3gZ2#j3Xbouv2iqAxieUTch1#BnKH7yauH|HKX!r1 zC`Rt`Mc@WRLJVP z#oGbT7Cz?-+j^u(pL}i5thf^N?y#WuFi3kD&=v|_)#oUsCzxe3zdx~?R>wYcX>+<` zl)P~z`<{5UdZ?~Xie>yuD@1=sx1aLqhsYLO+O{!tPmZBdfwE1(yfI34H$b(1o&@RM>@;NBx&PV--x3d`&|OEcEsUUh_2 z*mQ6?mO^o(!F{F%OKTUZyY7BaDW86HjZ;notdwkfu*)$pg}IKjn0yDZ+DGqIjJunS z9Np%1=hn+S_We39n*O9dN(D0XHHrqh44Z9$$WGYBH@<9LQ@3qflh58B;d#}m>;vY{ zO0W1$f}1Ft0;P*UgKO24Dw1PP6UXo0Nxr{OFD52^EJQV7$t1+%L>*8~ZI#*1czwBx zQdi<{M?0}S_kG%gaG{aol`}`=Ss5zJd2t82`)zzXercIN z$4)>Xe#j{>Jig2f|5#P!Xy%TTvG@O1F>}r31`qNiK6XeYMF~5_GArrAqB?!_i*J+V zme2F7P>_dYA{b^Swyuh}A03YnT1ZMv;rqTPI(^JAxQB4oxOOsBl_#p`pb3rg3?_12 zeSp???hT)nX$DnAspY!uae zj-Ki=P|rq&R3y)9c+cl;Yt}tATuX0l+L)iS#h63ccaV5vg(BfTYs|C8NJ3qFHv=9p zv-u4-T{2f9A0V$gy%(C5OgiH|3>DLdo``WPFIriB&!DpJiIgpNd9;K^V>AO#>?dSP z^9^7g*LcyOA0q8HA!b}Ho9LB;tbs~a z%#ly<^5utshSQxtcG}rmVtqiCMXpZKF?%1kZyj zf^fbSA*J=sjuWQ%b0O)T&2Rbhx<55D zz2A_fv9EU*+3z7*Jbk~#P=ks4ngNb0wYxgB&F$JDHoYSkn#af|<(CiE9nuYuir09S zmD^NmhQB3(Te_CHKoMA1>{c$8t=11TP?kN4E!|;RV-S)|>ey=kIddJ0{nby)>m#S^ z0x9BYx(<_@_1X|2cl3 z{{1Qy%P7uE?YHpWhfE$&N8$t3Z%WU+-*>D4O7*kR^C%*$-**yn=(lQE=AP)?koq35 z$MRK2q%7Sr#>(y*to45HTY2@jkX`rfUhc4bxw2I(eQoNOi#_bX?0kHcf4yD}>!+^l zr73kNntQ1O$wZaG>RjUXHDov6s4cM9JOW-5PJc`vnF9olNJY*`= zKdUQ9F|T3mDNk<68q36r9OZm7KUT3N&UCX6l9i6Xm^_VjKIJUgNhF%6{Pxz`pIT9| zGkP8T;>=CW5Kf`Lg&j*g|4StrkQYV&3oZNq+F=cPUz8eb_}hxN@7sESRVfReSW68C zo#gbM)0UKs0{n5xl4%ao#go?YN|UQF_*{-h4Vf?ZlmPEBt~QCb2#yrS(=3~~ZN!pM zRAwGq93ZLu@}40)$CfoaQOc3kk`d4fZ8#+Ev(;m zy4N1ayjD+&K||)-i=b5JLIn83N|u`Y>vCy+mGdh07jct&0#>t;+g(|fwyzxk)vgJL z59v5zDP9H(og>eTS$IPjsR2mQ)lqcW08ZG|m2=zOMnM<5qi(3q-mrGtPdIezKlYd< zHkj%EIJoQ&r-T_40y~qz8FH9oM6S|9uaNQSvafuYBfdbTnSL3lf2~81yyVkx${aJF zc#yXt#8Af^RXhp1FK7zI=!LW>k8FMSQ?k>PflP))fFG?#;`4k;TQ{Ti$M;@>ORn=r z^=}ydKH#+f`VY6?uR1epQOd5+U(D?7gBR&19;6B_zc1>Ex~Y*64k#~TiuW=TV8;K4 ztfb0GQhwdA2ggi!`AFIIL7gz8??I5q3-Q}3=O5TE25;1BuPdA31FL2dg8mD|gHP9?yX^@x*5Pz*$gFp@B7 zHZ}FQEX{SG^Jsf-$@&8RzhIj360K+erwK?Z&?~fvBrBtbKrb@&=>%9xUJF8|1v?8Gz!=n;Y-v(A+vJomB8E6F$2%kVfgso5uFek@5y!5-qR3E%qi-{@+p(XwtffJ$ z72+=7zAT7&;s4SP(q3N(RXATUn{LzwJ*imdmtnKDw%{OAQmhoY=Xx3s(Wf@VY0CDu z4Yfh;S1TIn!OvDS;6yV%jJ`(zC1vYd57#zo%%omRS&GVAuZ7!cpAPrrysJx zIZd!R5+j#uLvydTE$SHiQCZyP+TZb^k2om5(2rsNfth+t-{x>Ln@k126H0HETo^WH zrvNc^-ROK0THXURYUlMh9e&WRGQzhY-Pe*k*w!g(20l-JeR>*pFwE7{rjVk;SZ zVf4wFca!27^~U?|kIuDv#fJ_5L~+|*LHmV4t3_tm8j2FUH%J=w)!N@l!*#>w4$d9w zJCdllP@yk099+2m!GF|y%Equk;eEx}K*Bh#sIp!^_FX-aH{w8ZT+Kv@V{&S&dbG^I zYYe+Ie8h{|0!~!EqxXXLD}L#iVm9a_hnx(4v*(>dL=WpyAUL+t*mWb&1PRIloxo7IW=@$xkBWb zK7IHuxF%I^x-sz$)72#2|APqhY{n}EO(m*g2K7p8%TM{V*D~7A$jE)mwr85P4VHRe zfQo09h`t&TSYXJ{TcYLMRXy^_2I(7L1|l07n+eg0%mK~V1v{w>%BZ}8_1rShca;}O zhHVi=ALT*@vXv%jZCk`5Y2nKUw`UcOj|jzt`k~+RhVN5gE&Y@) zGn$kV?M6uhn$s>i{i8WeWT-Ptt>nyp#iDJeoM*QC_({~M%QXBQYqTMwO2A{=ITxfM z_2&cafC5g)2@mxT90DVIHtBMTK;U!|e8h%wc1Iz{qhLKE5d5~&v$h6j3E(M~AO*e; zWNw+RC=VuVM+tiFBj+kmpqBXbXMtKIdMya7dGv7uI4m7|8OGtEl3;?nGRD69CHUZY z0rb-DK?$PA`UW7N-;qXho9M{A1$@9aXgAcP9%PCnSglJq1CCFu1}4RY>|Bl7^$NVx zHYG=U3MgIwUJLpo}<_7I~s-2ce(duuW%`X@s# zVBV!z=<_wExL`ea@|t+V9V05q&dxPL)!dz!NiN#0A-H}|Kcv*8F2iAt(M}*e;fgVW1_( zfZrJA(UR>LSy}tbsQaxAs3X4KfoK>#u)O?L?!p?`Pig`rvI=$CZf9HPrd|@{1nZ3) z|EQQsCsC?mX6rhBjq1b6(38kwxN5XA3R(bBi0vB$x4GPn1ZY&%J5L@9y89F6Z8P4G zY35N~-O#a1VW$DT8-34_&jb#}}I#*gt;Ig+}`R3!hDm3FeCxGQy}@6sk6 zrArqnj&jQq+(X#3`(1L|tqgeN?x!8NE;UYl46rR3Gqn?`4DoffLGBDVd$W*q z?EDPCrSh4$dhEHMOBnTWy0#NlEB>Aeip#2Lw@giuD&o=T5Kul_4^UI%Mh-y*=UnSf zIq_!>X5nVjHTZcK_E9IF&)*0Xbv>pwciFOD?q+<7W|@7J?#MjArRr#wG)Jce2N61s z&Oh14QaLI57X$Ta;X~t-g1U0^aeU(H8&+f?LBg`% z?B3_2$sMEKn%tH$Eq?)&!g+wLf@>{^Zc7g+N{_DIHlSp-e!7>7NegocURQHrJ>tWGS9iKJX}d zA~c3)mBT8!S&d2#?W`o z4KZC84`&~)oTLnd6N-!=Sf$mXDb7(pt4HqGKy{BT)zS5_Jon}C8{XC*ZeIw`&!IDL zv4v7j@kr?$KB$e)NN$Qe?#5on*5)s z+ddpOGr8tEDY>NusJIKyBV>CV{hWu6eWIn;x^`9;i_NSVuruYXp+nCtruT#tltX(V zp$*=Pxc8(Rglkq?gqupeU0-r(-=no3 zSj;C*jYIXQFP0zWldu=a~Vza(R>f32s0S2)&HMN?yqDpF= zBVcKg@&u^RScl16DWFR6^tz-`FEYleTeDnc0CFD%Pn-h3kcKr= zBA3XQDK_hwuAx?fp6-KBwj*2`A~mV)jw>tAr6yt);rGw7adBP(r|!yi{~q*q(it8R zwbfR(4Rs<)gA#mnZ7_7-yx8udG~%!t&TKYPwp@N`F3R;k*Q)6(SX=SBYQK&H&m zUH0uJzpBn6s!N$D>0yKc_VtwgiizBe*E7!WS0~n2!5H!hQQ~eR3!8^LI})4sAxU%12&q zZzQ$*fL@n}X$c*@6dnOvvRpxkwbW4+tbw!cU)gZyj?9)_Gxp#?Z&Uzo|2cfCqP#5p zcp!L)EA+E$v|ZGnz%F!!5Fx+|cDA_GsrayN#z_=O4Zw!h@7()B^5_xmyB5DLYr`&k zQT4ceLWjHE(vmo*cF*uBrL5l>$U5-j@WvIfm88*aHa6aB-;CI+x+@Qb`4zuRLrKLT z{<`#qX2|4MR7y&Q@7=8+_Y3Mkn9+m6>2DXQZYSI;`4<;^SI6Ik8-P+J{~xi~nOM;K zF*e15WIU6Vqq!`={0CBM$HU2ozRcgZ!uHEgo1Ek>K$@8cYUgKB^UUKz`1Y5i;`Z;h zOx2P@00Fn=cl}vq4dOZ@Z_{_9hJ`GZW!psV3t!`p!=5vHCdB;fL3s;s)AkJX9niWi zX(ZY@`M6Pfyy6}+pxPA_DaW@oz}jw+$J@`Wq)VfRHf~T@D;emAvDkojwBQ;%X^f|derMXMi#F{TF2H2Ydl_TqM&T7DJ$AF6L6d^kICHg36;otNk`Z(?|R%ODgu2?Ym5n_ihiKwUCfGVD{M6T5S z?iAGV0Yh_8jJo20pJ8}{{Giq{ZY23|>)8ikw2h@m9%nsz&vvEGML^2G2uMbg65fu4(^&2;wzFA)On~`l6!GpP<&uPX~Jl#G!qRDwJdT%tR zi}s)T{mHkpra#-}H}`l+p4zU8e^?5!J{2M&$p7yd%(0(=R!L-N3mh^dsa$5i8p-IL^aj9VboH- zPPsQtp3xls9)$n${inApW3{6h;v8s0XZx4QwjrnP%ccSvy>?d-R>lK|A~lrTF+?oI zd80D}e}mZ`8UYWd@THEndzhTWfuyWg@SWK4YI_WHvHIeqyDVZjh%sESJ_}HpGeM3z z$oj;Sg%PcmYkMPd(%zt{VCNdSd{3k7;?ejQQu)qPafmkFLFm~@Pi3Un;H$?Ei`^vF zShark;E#59SdwyC5a0YETEx05RuW98%)0I2fug^}AG_z)MVy{vuBVl1M&)UU%1Jo( zEO0UFCl2jO8FY?Q&QPp^hFWJDv#63aOiL~ZHR;mF%Gx}B@` z8-rbq!#Tt`x@#WWd#jb&yAtkG1{!Jts_f|nY$tSmCu3#$C0oLyxD2$`u^6N1M7`Ra z=Iv}#-Zwrx8oUR8y{##@l9~F+JH@Mbw;oZC%JFqEfgPCw8cp!s2*k)-PZ=1Dj|=IF zHD%PLlO-DK_=rXGQ5G6dXNyX_6>=K-8PSOpC4%A24@?D%F;7dlUk)|NNxV`bSIz-4 zjfScQ(y?<}7SV8vs(y4o2;rq2KJ);s7Rdh+Z7 z{?AEYf1{$=5cWw-fV8*#lf*sqy-N8*&to~2Bm|p?7>n~K5W);>ukFgTji8I(qUMMwkr4o4 z>|`PL4#Gn>M(Q(!MvSaalU^fC|K)@kSuAh#E(bkp!bP>-;6!^{t2)^Xl={Lk{^g>c z2PoF~;#2@N7AU(KD^w+q1R6@H=9l5a$al&XmjP4sjCO!{h*8L(o7e&9==rE(=SKb{ zWF>I{07%qL;%$aL3W~dHg3e{Yr;!zDVr^O*NsF%A0W*-Q&Ni$mxCl11d&vVivnSi4 z{r*L4oz()h1iSoYpzr#YBD~Tl(qXb@xkW=BlS%*U(Q;D@a9QZ_bt5+oFbdR3Wg(mG z7(%D?C{UOt6B%yM87J+RkN=a8iTIbmPQP%x$|>bw~J&+iv)uY$Ksd z1pdKS~v*jeCatonqrpG}NdX9zA zC@X6;Tk1n)Z{#1M31b(H2yAz~rh z$o;RKaV~6Q8IQMn=)aE$(uDy&RzAwg%Av2hEjmAcr)G!_e^}dq{|Xns;Z7bQv&F{; zMSlpOxQY_&V^S4n_qo`v!yJKXdOnYlClPL{=S0;#Fe%zf9hjzvo{{NXgUFoSXvyi$ z-M|@+S=`mlM0=g7jyd*B@FAmpaF@&&L*=S@{Mn(Xix1Uq(x5~+c^5~mMzscKAdz^= zW;>B6Ui34OxVGw8*3M4gG_fac-mZQlo@X`)C+D*IYWkt;CPNFx{M1fK@-R?$gPup$ z7=ME8kcYqZLO+M=aMy3Nk0|AtMCBaroa%-nm0^Iu$7gC1CKhL#t-$dVOmLXfNpLqX zyg!j>i9@`fzzzv_cJ7?q;k%~uvSy(bW-!vL+i>}7bE@T58lEh*T-a39B_?ece9J#e zw$?XUm;D#n`Y7Z_gzUQTT;6Xxq+h+h&XeYE9fgkON2`!)R>5){!?uETpvB~N4 zE9Mpx3@Ux}K{-*j(L?D>AJ7ayl^k-GNOwGY5h4#2R`$ZgTRure2qfgSS^*uCqw^5lqd?^PSWc-;AvNU#(!zX49k=2Q< z(m;|hoqKkD40nH`@bu1^epLKif~mi}0lc*-Rc4SLIOF?3ph4)^=Z63jW%sZ}0mWl(MA<%hKEW1A1oX+&DU4X!TPh&4^vzFq)U9dS!~jC>P({ z^UTi}mso^9kaQC-G@YQb{n5No!Z#jg9elxwFqAD&*OtXwT;Pz_-m9!cDtXhE)gUdS zehbXNxuhNR^MhR_8kvy8t`6)-=ehuPR~>rPeLAa(yP;YHFqM+$6jiZW@4)O$*udBW z2Z;MVt&z3}XRJ(hKkD)>+~ri^AprJf_`^3FdLYuMtq5X5e1L6Rg_SL?(t0=*Q9~d9 znA7n+hIQjB&GI)$G><>EAN=wL#~PP2;jL`?Z5~QV^s=ts^2c)jzLz1PS2Hl1zgIFl zUI1(2?#G=41&Mz?8`I}p9i~Hbq`MT0_ayJR9rcv@8l=vyNvuwR-oT5JsD<%J`ik>F zms;$4y?Bd+OT2|x&I)EW)}z%SByxT`7+^PD6AyTA$_IaWZ#Zuz$|c6~ifO*ijY~H+ z?pqkrE71R3aIUZoT3%LE+vS^1DJW;G1i7n!fSjk&=g=6FZggT4>Pk6*X+dG=<^*jATf&P%-zG`z2e>mZ!U z^zSU9YQm>XZlwWT_Hz1?PFf=c9uKm~d4O~4k0qlC^mK03%V}m^Lb0V{(uG?if%64izn2=8 zuYe105Cys8ce*}dGpXSDZd$kMmz^TK7e8>mcJe(;-;*dj)Og3%pCOC;tRG(hi53=m zlPf=6J4AruK_xtssE(6gp;eGOOtU2`7hN7>@_kkdReYmZW$h~7!Ev)%M003$zWT^uh zf`&Rz@+`AFhBO#DP{toki*y-x1g5fuPlOcK1-f$JASV#;M-w+FD3#+5`ve|OaZ(hLKxj`phG*`9{tBwR?|Cozq z+;3@n5&p)jXHSlVl0>5W1z;YLvIT1iD*G9Cr5-f>KfRr4Skmd<$1OGUsMVRKOiQhq zM#m#+Mx$jeHEK=c*5tUQxFO<(yC{-ScE&9C(osXhGPm6K#09NJ4K=qE5Kzn9L2`qX z(C2n(&NJs+&-Fa7o;UD@>*50b{IC1If8X!tvoeSNKqS5ob%lO^vSek{Wj&PEx@tVg zs*dnqhml6E^|SJq4eOe5u=+2Hz{&r~wSHMMNHYdp<*bd$7s4o~%>o_>N}SY}TFYeP zgRpmesG(O&4=#%=RjKD%3wVM)sW2AwZ0k zh9c3671KC%)0CaRW4ZgvW*(whC0eZ)p+O@n5#sNZO2fu_^DN2eF$^ZhvM{s4Oxm8I zeXHZ283R&Yh4E#XnumW@!RyfY%{3(+T{=3~r+rm!&$pw=9=w3tmoFu{Mxgblb|#T7 zGbeu94^h#LS$kvUbf8@v?H4Z{di^X!xDG^X7)}0`dP5Z~fxQMT|*C zVD>}Dl&Q+mD|f2Mk~Hg>ACfK%6MR1!frGUw!N9X&<_5;zu4aT4f8zs z-OA&Ap8H68dtM(S?0fdG9aBI1s4U>t{r64pa(GdshA~BhS#mzmKhc2x=_D0#_@^=q zyXn-_i0pNMWWF-3EJC`ur%%?a>buzDcxk1uD!XFyj7pSz7>wZWl}PGtYj@$Z=yn{w z!tU;!t@NIl7ZIw9NHtD+MIGn=Khelk%WnabY5K} zKtfd~&MW}Qveao1{`Y^*Uj>q9HIpG!lJF=?@bi4v2(+w6jEDu?D5{Y;%C&}#F^S}* zqLw^Wgq}V!7YVy&3p5Z-$0TQ8>L(-=P`7r~9L9~;!WNXkWx?QJy0@g0>-xV;=|V<8 zoYu;oQ}g>jJ8FM^p&ezxst>rri`6A!#vO3`&OW6K>07z({#5q5z$aKU zZWcV4^KEle6UEG=KFC$_f_ROa(dy zm`lK3=|;Uho&apw`rF4p9qCmIzS$_Zo*TQT9svj434fm@tqoG7ip8(=uP1 z)+<3|z=?#!Q|zFu03{NctnEIP6{AQ~Q8{ZLv3vL0$xhL}D{=nSr7w6m%TxG^2oj}@ehq|^eYH)JEzI6U* zRVIb$c$H^WLb7+Wm$%sv(y?U^>D9ASE<3*TAD~59Tf%h5B^VtWb%x@X=Kzq|)#Ve5 z0gPYTU62x&Yi-leLwaRA#%CWwklbSw2^xK!#1m>-Y8w0@r82?j4b zar+lk?}(;exLeO&cw&1a4jeI*w(&ny^-JzTluOvNd>O@#^(XGqo&xQEP`i=_L{`+20~;M}H9_Sn$)cIGxQUX%L{_ z==fM}qs|vB`sRIC>uO5&e**=j;4BBuA6@5J+s8kY3yokzOlA$&5?>5CmN$bVghgDq_ie}Xhb3*q1%>2bn>KIFr%1u` zdabb}D?#9Uw_w%;(0s{u_)zp8s~d|?Adpatg31E1@Y&kY=1O@Ey(w4ci#Cn6dm!EyFs5|2RbLIpx3XMyzZsZpLKliM%B+=E z5**%S7D+z8R6W!ne-JEbcwvR7m+Snav|4G`!j7Ti2C6ENi@EaswT5V|^L|@(w4YTg zU`LZ0A*No%4=53vr)Qd#$IrxRMG(RIdy#h~-r4dQw`kdP&_-p2?JQKpc5xXw(#Hg&>(?SWnWTjVNt-tMV6FK;E8(&GU$jbOKWCngs?2&p3H%rFeKbuMxJ2 zXVfI(o3sNVY<@(ebA>=(E|gP_hFld`3-~*9mz$+qMXsoj${ktz)4kEYKq|)j14zZ@ z#!2*~Kc!+KZ&GBAxkJVBg>>&NtJpZ|KdoZNzintCJGHf$e*2A2%7@j{~)dJ?zUr9-ws{2$5|^jhRbW zJ(KNda8{#R?#ir*k8X)TYy#TV;S365=ptp>w zoCZjkRZAaatfXKZSIG32uNf3XoO`c?uN6YFL#ZXUxVJ>{qv!X*O4H0&yTqzot?IZ6 zIqQD~bq_y%;iTH5&#B2tpEf{@Q;Rdd7pGCYni!p?@qzjGP=sYhY75fH*>^_Mh7xur zj)k*tAOu7$=oKm!Ar8-V+7mL%nro9?h@5&gwu4_W2CuRIO&)0HFz`B+3lvQstOc># zzbjUfPv@A9G!MUFXkTsa6r4wXpFxnhM@F$%b$lDVDkE?Pj+edz`5naPIZ?8DN#vTO z^{8u>2fWox)xn6?26AxP(=8_5ICdWfq+cQ-8B$;W z+U(4hCM66Vr`ixQ-q;_AWGlc7B%xiEB3-}lW84E!4Dm%wajIh>zZ_*J!|-2iZ2HI* z$1|l_5JU(+8l?JM%-jv*K>w!7g0FOKw9ot=JPe+gC26l0W#5jPskWMh)mRx?SJ81i zlB91fb=*OvCj%k!LZoS+o2P{D5=(>V7!7q#zENuT;B;R9(VigJtZIMIJMr8xGQ@DY z=;ZIV4ZDhCAKlOhMk#BbC)H$qLfBB#SiC7jIm$i69r5`_slS@ z{?ya?GIM}n%%ihS7s4)3FbuL>pdCQoc&;*(O;GyShizmWv~YcVC9otU=K|wj1AqQs zgXBBwhRr9tM_ZxtLhN5gSO1|rvc{NdtOh>yk)Cm6*+4y zqEFXLDwIr&T#RP-Fr34sRT;z)-mSP*&Pz>E6(1venE2g{V|0>0rLF3@@zCC3Pwo zL97G19=r69zu(|&Jhvp2*nv>_*)v&n)J?`Z9tJoBA+JQgM&d+Gf&- zc)l`=fZW{Y;h|oaL5;nNoCz~krBt{aZa0h2w{iRm)8_;AFiS5nuae2$ubXjYG-70a z%paVt#Tvg#8alN7amvL`Jy3jkMM4Zsp43P`nC;9AnRT#EQ*JT(D4J!bj&-6#pm%;1 z^r1T9B&$5kk-L{$lCBu|&*6kGK8i=`7^v+ZWwVdv4y;YVt!4s0ldx9K&zHP`ojJ+l z%VyvJ7;$gE+14W{2jK2vwgA76iT@7xP5Sx6#(rNo?$eWwy4lT_Uoo*TF8$FZ>c0R5 C*b)5z -- Gitee From 5b1bdf29ac3fe6292326514fcbf2672b94aff762 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Wed, 4 Sep 2024 19:27:01 +0800 Subject: [PATCH 695/791] =?UTF-8?q?[cluster=5Fanalyse]=E8=B5=84=E6=96=99?= =?UTF-8?q?=E6=96=B0=E5=A2=9E--data=5Fsimplification=E5=8F=82=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/README.md | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/profiler/cluster_analyse/README.md b/profiler/cluster_analyse/README.md index 9b8d79c5e..562f2bde8 100644 --- a/profiler/cluster_analyse/README.md +++ b/profiler/cluster_analyse/README.md @@ -50,11 +50,12 @@ experimental_config = torch_npu.profiler._ExperimentalConfig( 参数说明： - | 参数名 | 说明 | 是否必选 | - | -------------------- | ------------------------------------------------------------ | -------- | - | --profiling_path或-d | 性能数据汇集目录。未配置-o参数时，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 是 | - | --output_path或-o | 自定义输出路径，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 否 | - | --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | + | 参数名 | 说明 | 是否必选 | + | --------------------- | ------------------------------------------------------------ | -------- | + | --profiling_path或-d | 性能数据汇集目录。未配置-o参数时，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 是 | + | --output_path或-o | 自定义输出路径，运行分析脚本之后会在该目录下自动创建cluster_analysis_output文件夹，保存分析数据。 | 否 | + | --mode或-m | 数据解析模式，取值详见“**--mode参数说明**”表。 | 否 | + | --data_simplification | 数据精简模式。对于数据量过大的性能数据db文件，可以通过配置该参数将数据精简，并提高工具分析效率。 | 否 | --mode参数说明： @@ -63,7 +64,7 @@ experimental_config = torch_npu.profiler._ExperimentalConfig( | communication_matrix | 解析通信矩阵数据。 | 否 | | communication_time | 解析通信耗时数据。 | 否 | | all | 同时解析通信矩阵communication_matrix和通信耗时数据communication_time，--mode参数默认值为all。 | 否 | - + ### 交付件 -- Gitee From a5edab77aca42cd84078764cec70cad6ade83760 Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 5 Sep 2024 09:17:41 +0800 Subject: [PATCH 696/791] update branch --- debug/accuracy_tools/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/setup.py b/debug/accuracy_tools/setup.py index 01e399c46..778a7e4cf 100644 --- a/debug/accuracy_tools/setup.py +++ b/debug/accuracy_tools/setup.py @@ -14,7 +14,7 @@ import setuptools -__version__ = '1.0.3' +__version__ = '1.0.4' INSTALL_REQUIRED = [ "wheel", -- Gitee From 2a8a206b6e34f6cfb93a36dab176f3067740748e Mon Sep 17 00:00:00 2001 From: zyy Date: Thu, 5 Sep 2024 12:05:08 +0800 Subject: [PATCH 697/791] =?UTF-8?q?=E8=A7=A3=E6=9E=90=E8=BD=AC=E5=8C=96?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E4=B8=B2=E9=97=AE=E9=A2=98=EF=BC=8Cremove=5F?= =?UTF-8?q?path=5Fsafety=E5=87=BD=E6=95=B0=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cluster_analyse/cluster_analysis.py | 2 +- profiler/cluster_analyse/common_func/file_manager.py | 3 +-- profiler/cluster_analyse/common_func/path_manager.py | 11 ++++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index 9e0c9d154..c29f3750c 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -47,7 +47,7 @@ class Interface: def get_cluster_analysis_output_path(self, params): cluster_analysis_output_path = params.get(Constant.CLUSTER_ANALYSIS_OUTPUT_PATH) - if cluster_analysis_output_path: + if cluster_analysis_output_path and cluster_analysis_output_path != "None": return PathManager.get_realpath(cluster_analysis_output_path) return self.collection_path def allocate_prof_data(self): diff --git a/profiler/cluster_analyse/common_func/file_manager.py b/profiler/cluster_analyse/common_func/file_manager.py index 46bba017a..26e6607fa 100644 --- a/profiler/cluster_analyse/common_func/file_manager.py +++ b/profiler/cluster_analyse/common_func/file_manager.py @@ -128,8 +128,7 @@ class FileManager: if not os.path.exists(output_path): PathManager.make_dir_safety(output_path) return - if os.path.exists(output_path): - PathManager.remove_path_safety(output_path) + PathManager.remove_path_safety(output_path) PathManager.make_dir_safety(output_path) @classmethod diff --git a/profiler/cluster_analyse/common_func/path_manager.py b/profiler/cluster_analyse/common_func/path_manager.py index c57092c46..b3ac78366 100644 --- a/profiler/cluster_analyse/common_func/path_manager.py +++ b/profiler/cluster_analyse/common_func/path_manager.py @@ -156,16 +156,17 @@ class PathManager: @classmethod def remove_path_safety(cls, path: str): + if not os.path.exists(path): + return base_name = os.path.basename(path) msg = f"Failed to remove path: {base_name}" cls.check_path_writeable(path) if os.path.islink(path): raise RuntimeError(msg) - if os.path.exists(path): - try: - shutil.rmtree(path) - except Exception as err: - raise RuntimeError(msg) from err + try: + shutil.rmtree(path) + except Exception as err: + raise RuntimeError(msg) from err @classmethod def make_dir_safety(cls, path: str): -- Gitee From 018f4397ee5edbba24483309927426a661f536cf Mon Sep 17 00:00:00 2001 From: makai Date: Thu, 5 Sep 2024 13:14:30 +0800 Subject: [PATCH 698/791] add report2 --- debug/accuracy_tools/msprobe/README.md | 4 +- .../msprobe/docs/01.installation.md | 18 +-- ...marking_validation_performance_baseline.md | 146 ++++++++++++++++++ 3 files changed, 156 insertions(+), 12 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 802f1e857..882030b95 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -90,9 +90,9 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. ## 📑 补充材料 -[msprobe 精度工具标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md)（待补充） +[msprobe 标准性能基线报告](./docs/S01.report_msprobe_dump_standard_performance_baseline.md) -[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md)（待补充） +[无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md) ## ❓ FAQ diff --git a/debug/accuracy_tools/msprobe/docs/01.installation.md b/debug/accuracy_tools/msprobe/docs/01.installation.md index 7463ee2a5..9a84cc8b9 100644 --- a/debug/accuracy_tools/msprobe/docs/01.installation.md +++ b/debug/accuracy_tools/msprobe/docs/01.installation.md @@ -16,12 +16,10 @@ pip install mindstudio-probe[==版本号] |版本|发布日期|支持 PyTorch 版本|支持 MindSpore 版本|下载链接|校验码| |:--:|:--:|:--:|:--:|:--:|:--:| -| 1.0.3 | 2024.08.23 | 1.11/2.0/2.1/2.2 | 2.3.1/2.4 | [mindstudio_probe-1.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.3-py3-none-any.whl) | 7060cc141a5b98ef770cd9220995d299393f32a61938261e632c7e8b5160bef2 | -| 1.0.2 | 2024.08.09 | 1.11/2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.2-py3-none-any.whl) | e4a980e5d98c426ce5ce9842520d9bc031d3b3de621c74b3d59414cc6e238e0e | -| 1.0.1 | 2024.07.25 | 2.0/2.1/2.2 | 2.3.1 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | -| 0.0.3 | 2024.06.11 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-1.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/1.0/ascend_training_accuracy_tools-1.0-py3-none-any.whl) | 5016dfe886c5d340ec6f60a959673355855f313c91f100680da814efb49f8e81 | -| 0.0.2 | 2024.05.23 | 2.0/2.1/2.2 | 2.3.0 | [ascend_training_accuracy_tools-0.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.2-py3-none-any.whl) | 2e35809bde559e9c4d2f16a02ccde779ed9e436bb65fded0b7ebaf6ac2c88d93 | -| 0.0.1 | 2024.03.15 | 2.0/2.1 | - | [ascend_training_accuracy_tools-0.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/att/0.0/ascend_training_accuracy_tools-0.0.1-py3-none-any.whl) | 5801510d4e827e4859bc9a5aca021e4d30c2ea42d60a4c8ad0c2baab1b7782c9 | +| 1.0.3 | 2024.08.23 | 1.11/2.0/2.1/2.2 | 2.4.0 | [mindstudio_probe-1.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.3-py3-none-any.whl) | 7060cc141a5b98ef770cd9220995d299393f32a61938261e632c7e8b5160bef2 | +| 1.0.2 | 2024.08.09 | 1.11/2.0/2.1/2.2 | 2.4.0 | [mindstudio_probe-1.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.2-py3-none-any.whl) | e4a980e5d98c426ce5ce9842520d9bc031d3b3de621c74b3d59414cc6e238e0e | +| 1.0.1 | 2024.07.25 | 2.0/2.1/2.2 | 2.4.0 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | + ```bash sha256sum {name}.whl # 验证whl包，若校验码一致，则whl包在下载中没有受损 @@ -31,13 +29,13 @@ sha256sum {name}.whl # 验证whl包，若校验码一致，则whl包在下载中 pip install ./mindstudio_probe-{version}-py3-none-any.whl # 安装whl包 ``` -若覆盖安装，请在命令行末尾添加`--force-reinstall`参数。 +若覆盖安装，请在命令行末尾添加 `--force-reinstall` 参数。 ## 3. 从源码安装 ```shell git clone https://gitee.com/ascend/mstt.git -cd debug/accuracy_tools +cd mstt/debug/accuracy_tools pip install setuptools wheel @@ -55,13 +53,13 @@ pip show mindstudio-probe ```bash Name: mindstudio-probe -Version: 1.0.* +Version: 1.0.x Summary: Pytorch Ascend Probe Utils Home-page: https://gitee.com/ascend/mstt/tree/master/debug/accuracy_tools/msprobe Author: Ascend Team Author-email: pmail_mindstudio@huawei.com License: Apache License 2.0 -Location: /home/***/miniconda3/envs/***/lib/python3.8/site-packages/mindstudio_probe-1.0.0-py3.8.egg +Location: /home/xxx/miniconda3/envs/xxx/lib/python3.8/site-packages/mindstudio_probe-1.0.0-py3.8.egg Requires: einops, matplotlib, numpy, openpyxl, pandas, pyOpenSSL, pyyaml, rich, tqdm, twisted, wheel Required-by: ``` diff --git a/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md b/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md new file mode 100644 index 000000000..e75196cf2 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/S02.report_free_benchmarking_validation_performance_baseline.md @@ -0,0 +1,146 @@ +# 无标杆工具场景验证和性能基线报告 + +## 1 环境信息 + +NPU：Atlas A2 训练系列产品 + +CPU： + +![输入图片说明](img/cpu_info.png) + +Torch：2.1.0 + +CANN：8.0.T5 + +除上述环境信息影响性能外，API 的数量、种类以及 Shape 都会对性能产生影响，因此本次选取不同场景网络和不同算子进行测试。 + +## 2 模型信息和性能基线 + +大模型在使用 msprobe 采集数据时，建议先简化模型层数，减少采集数据量。 + +以下场景的性能基线测试数据均为多次测试后取平均值，实际运行时性能数据可能会根据环境状态稍有浮动。 + +### [2.1 ModelLink 模型](https://gitee.com/ascend/ModelLink) + +NUM_LAYER：1 + +NPU 卡数：1 + +主要数据类型：FLOAT16 + +#### 2.1.1 LLaMA2-7B + +softmax 算子为 FLOAT32，输入输出大小均为 2G，为模型最大显存开销的 API。在该模型下、对无标杆工具处理模式、插装范围、扰动方式组合下的性能和显存基线进行验证。 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|-------|--------|------|-----|-------|---------|--------|-------|--------| +| / | / | / | / | 0.24 | 13.69 | 1 | 1 | 混合精度模式基线 | +| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1 | softmax 本身为高精度，跳过 | +| check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | +| check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | +| check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | +| check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | +| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | +| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.38 | 1 | | +| check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | +| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | +| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | +| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | +| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | +| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | +| check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | +| check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | +| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | +| fix | 前 | ["softmax"] | to_cpu | 9.70 | 16.67 | 40.42 | 1.22 | 不支持整网、不支持反向 | +| fix | 前 | ["softmax"] | improve_precision | 0.26 | 14.67 | 1.08 | 1.07 | 不支持整网、不支持反向 | +| 预热 | 前 | [] | improve_precision | 155.07 | 24.79 | 646.13 | 1.81 | 低精度模型基线、只测预热的迭代 | +| 预热 | 反 | [] | improve_precision | 72.29 | 22.01 | 301.21 | 1.61 | 低精度模型基线、只测预热的迭代，grad_output 为高精度的算子跳过 | + +#### 2.1.2 Aquila2-7B + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|----------|------|-----|---|----|-----|-------|------|-------------| +| / | / | / | / | 0.17 | 13.66 | 1 | 1 | 混合精度模式基线 | +| check | 前 | [] | improve_precision | 1.57 | 14.24 | 9.24 | 1.04 | | +| check | 反 | [] | add_noise | 21.05 | 14.19 | 123.82 | 1.04 | | +| fix | 前 | [] | improve_precision | 0.95 | 15.55 | 5.59 | 1.14 | | + +#### 2.1.3 Baichuan2-7B + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s）| 显存峰值（GB）| 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|----|-----|---|--|----|----|------|-------|---------| +| / | / | / | / | 0.26 | 12.12 | 1 | 1 | 混合精度模式基线 | +| check | 前 | [] | improve_precision | 1.02 | 12.27 | 3.92 | 1.01 | | +| check | 反 | [] | add_noise | 11.15 | 12.67 | 42.88 | 1.05 | | +| fix | 前 | [] | improve_precision | 0.95 | 12.82 | 3.65 | 1.06 | | + +#### 2.1.4 Bloom-7B + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s）| 显存峰值（GB）| 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|-----|------|------|------|----|-----|-----|-------|----| +| / | / | / | / | 0.14 | 9.51 | 1 | 1 | 混合精度模式基线 | +| check | 前 | [] | improve_precision | 1.64 | 11.58 | 11.71 | 1.22 | | +| check | 反 | [] | add_noise | 17.15 | 9.51 | 122.5 | 1 | | +| fix | 前 | [] | improve_precision | 0.87 | 10.62 | 6.21 | 1.12 | | + +#### 2.1.5 Interlm-7B + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|-------------|--------|-------|----|------|-----|------|-------|----| +| / | / | / | / | 0.13 | 10.76 | 1 | 1 | 混合精度模式基线 | +| check | 前 | [] | improve_precision | 1.19 | 11.68 | 9.15 | 1.09 | | +| check | 反 | [] | add_noise | 11.69 | 10.89 | 89.92 | 1.01 | | +| fix | 前 | [] | improve_precision | 0.75 | 11.68 | 5.77 | 1.09 | | + +#### 2.1.6 Qwen-7B + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------|-------|-----|-----|----|------|-----|------|------| +| / | / | / | / | 0.28 | 18.41 | 1 | 1 | 混合精度模式基线 | +| check | 前 | [] | improve_precision | 2.34 | 23.18 | 8.36 | 1.26 | | +| check | 反 | [] | add_noise | 22.07 | 19.47 | 78.82 | 1.06 | | +| fix | 前 | [] | improve_precision | 1.31 | 21.11 | 4.68 | 1.15 | | + +#### 2.1.7 Gemma-7B + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|--------|-------|------|---|----|-----|-----|-----|---------| +| / | / | / | / | 0.15 | 11.06 | 1 | 1 | 混合精度模式基线 | +| check | 前 | [] | improve_precision | 1.49 | 13.17 | 9.93 | 1.19 | | +| check | 反 | [] | add_noise | 16.69 | 11.06 | 111.27 | 1 | | +| fix | 前 | [] | improve_precision | 0.87 | 12.25 | 5.8 | 1.11 | | + +### [2.2 ModelZoo-PyTorch 模型](https://gitee.com/ascend/ModelZoo-PyTorch) + +#### 2.2.1 ResNet50-Cifar + +NPU 卡数：1 + +主要数据类型：FLOAT16 + +主要算子为 conv2d，每个 step 有 51 个, 因此对 conv2d 进行检测。CV 模型、依赖 mmcv 实现（如果不修改 mmcv 代码、工具无法获取 step 信息和反向信息）。 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|------------|---------|--------|-----|------|---|--------|-------|----| +| / | / | / | / | 0.09 | 7.63 | 1 | 1 | 基线 | +| check | 前 | ["conv2d"] | improve_precision | 0.889 | 7.94 | 9.81 | 1.04 | | +| fix | 前 | ["conv2d"] | improve_precision | 0.328 | 7.47 | 3.64 | 0.91 | | +| fix | 前 | ["conv2d"] | to_cpu | 12.23 | 7.47 | 135.88 | 0.91 | | + +#### 2.2.2 OpenSora1.0 + +NPU 卡数：4 + +主要数据类型：FLOAT16 + +每张卡每个 step 中 linear 算子个数为 257 个，FA 算子个数为 83（FA 算子反向无效）。 + +| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | +|------------|------|-------|----|----|-----|-----|------|-----| +| / | / | / | / | 0.99 | 17.61 | 1 | 1 | 混合精度模式基线 | +| check | 前 | ["linear","npu_fusion_attention"] | improve_precision | 3.88 | 17.61 | 3.92 | 1 | | +| check | 前 | ["linear","npu_fusion_attention"] | add_noise | 3.46 | 17.61 | 3.49 | 1 | | +| check | 反 | ["linear"] | improve_precision | 12.61 | 17.61 | 12.74 | 1 | | +| check | 反 | ["linear"] | add_noise | 9.8 | 17.61 | 9.90 | 1 | | +| fix | 前 | ["linear"] | to_cpu | 18.83 | 17.61 | 19.02 | 1 | | +| fix | 前 | ["linear"] | improve_precision | 2.83 | 17.61 | 2.86 | 1 | | -- Gitee From 8f888bd26d732bc7fbc841000e2db6054cef2e87 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Thu, 5 Sep 2024 15:24:57 +0800 Subject: [PATCH 699/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E8=A7=86?= =?UTF-8?q?=E6=84=8F=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/file_utils.py | 29 ++++++++++--------- .../msprobe/core/compare/acc_compare.py | 4 +-- .../msprobe/pytorch/online_dispatch/utils.py | 1 - 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index 0316a47f5..3f1c2747d 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -202,11 +202,10 @@ def check_file_size(file_path, max_size): try: file_size = os.path.getsize(file_path) except OSError as os_error: - logger.error('Failed to open "%s". %s' % (file_path, str(os_error))) + logger.error(f'Failed to open "{file_path}". {str(os_error)}') raise FileCheckException(FileCheckException.INVALID_FILE_ERROR) from os_error if file_size >= max_size: - logger.error('The size (%d) of %s exceeds (%d) bytes, tools not support.' - % (file_size, file_path, max_size)) + logger.error(f'The size ({file_size}) of {file_path} exceeds ({max_size}) bytes, tools not support.') raise FileCheckException(FileCheckException.FILE_TOO_LARGE_ERROR) @@ -266,11 +265,6 @@ def check_path_before_create(path): 'The file path {} contains special characters.'.format(path)) -def check_file_not_exists(file_path): - if os.path.exists(file_path) or os.path.islink(file_path): - remove_path(file_path) - - def check_file_or_directory_path(path, isdir=False): """ Function Description: @@ -427,12 +421,16 @@ def save_workbook(workbook, file_path): def write_csv(data, filepath, mode="a+"): - exist = os.path.exists(filepath) - with FileOpen(filepath, mode, encoding='utf-8-sig') as f: - writer = csv.writer(f) - writer.writerows(data) - if not exist: - change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) + file_path = os.path.realpath(filepath) + check_path_before_create(filepath) + try: + with FileOpen(filepath, mode, encoding='utf-8-sig') as f: + writer = csv.writer(f) + writer.writerows(data) + except Exception as e: + logger.error(f'Save csv file "{os.path.basename(file_path)}" failed') + raise RuntimeError(f"Save csv file {file_path} failed.") from e + change_mode(filepath, FileCheckConst.DATA_FILE_AUTHORITY) def remove_path(path): @@ -446,6 +444,9 @@ def remove_path(path): except PermissionError as err: logger.error("Failed to delete {}. Please check the permission.".format(path)) raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR) from err + except Exception as e: + logger.error("Failed to delete {}. Please check.".format(path)) + raise RuntimeError(f"Delete {path} failed.") from e def get_json_contents(file_path): diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 988e5ad39..642af97b6 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -7,7 +7,7 @@ from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException from msprobe.core.common.log import logger from msprobe.core.common.utils import add_time_with_xlsx, CompareException -from msprobe.core.common.file_utils import check_file_not_exists +from msprobe.core.common.file_utils import remove_path from msprobe.core.compare.check import check_graph_mode, check_struct_match, fuzzy_check_op from msprobe.core.compare.highlight import find_compare_result_error_rows, highlight_rows_xlsx from msprobe.core.compare.utils import read_op, merge_tensor, get_un_match_accuracy, get_accuracy @@ -235,7 +235,7 @@ class Comparator: logger.info("Please check whether the input data belongs to you. If not, there may be security risks.") file_name = add_time_with_xlsx("compare_result" + suffix) file_path = os.path.join(os.path.realpath(output_path), file_name) - check_file_not_exists(file_path) + remove_path(file_path) highlight_dict = {'red_rows': [], 'yellow_rows': []} with FileOpen(input_parma.get("npu_json_path"), "r") as npu_json, \ diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index 52b6a637b..6cb10887d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -1,4 +1,3 @@ -import os import inspect import psutil import torch -- Gitee From 36ef309303167214de38f631dab87d04c3b9327f Mon Sep 17 00:00:00 2001 From: wuyuhan Date: Thu, 5 Sep 2024 14:19:13 +0800 Subject: [PATCH 700/791] =?UTF-8?q?advisor=E6=94=AF=E6=8C=81output?= =?UTF-8?q?=E5=8F=82=E6=95=B0,=20=E5=BC=82=E6=AD=A5=E5=88=86=E6=9E=90?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E8=B0=83=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../advisor/analyzer/analyzer_controller.py | 52 ++++++++++---- profiler/advisor/analyzer/base_analyzer.py | 7 +- .../ai_core_freq/ai_core_freq_analyzer.py | 2 +- .../analyzer/computation/operator_checker.py | 19 +++-- .../pp_stage_computation_analyzer.py | 9 ++- .../computation/profiling_analyzer.py | 2 +- profiler/advisor/common/constant.py | 26 ++----- profiler/advisor/common/enum_params_parser.py | 70 +++++++++++++++++++ .../advisor/common/timeline/fusion_ops_db.py | 5 +- profiler/advisor/config/enum_parameters.yaml | 36 ++++++++++ .../dataset/profiling/profiling_dataset.py | 5 +- .../advisor/dataset/timeline_event_dataset.py | 3 +- profiler/advisor/utils/utils.py | 19 +++-- profiler/cli/__init__.py | 4 -- profiler/cli/analyze_cli.py | 46 +++++++----- profiler/requirements/build.txt | 3 +- profiler/test/ut/advisor/common/__init__.py | 0 .../advisor/common/test_enum_params_parser.py | 55 +++++++++++++++ 18 files changed, 284 insertions(+), 79 deletions(-) create mode 100644 profiler/advisor/common/enum_params_parser.py create mode 100644 profiler/advisor/config/enum_parameters.yaml create mode 100644 profiler/test/ut/advisor/common/__init__.py create mode 100644 profiler/test/ut/advisor/common/test_enum_params_parser.py diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py index bed45679f..adca23bac 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -3,9 +3,12 @@ import logging import json import sys import os +import platform import multiprocessing as mp -from pathlib import Path from multiprocessing import Manager +from pathlib import Path + +import psutil sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "compare_tools")) sys.path.append(os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "cluster_analyse")) @@ -17,13 +20,14 @@ from profiler.advisor.analyzer.overall.overall_summary_analyzer import OverallSu from profiler.advisor.config.config import Config from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.common.async_analysis_status import AsyncAnalysisStatus -from profiler.advisor.dataset.cluster.cluster_dataset import ClusterDataset from profiler.advisor.utils.utils import Timer, safe_index_value, safe_division, safe_index from profiler.advisor.interface.interface import Interface from profiler.cluster_analyse.cluster_data_preprocess.pytorch_data_preprocessor import PytorchDataPreprocessor from profiler.prof_common.path_manager import PathManager from profiler.compare_tools.compare_backend.utils.constant import Constant as CompareConstant +# 以spawn模式启动多进程，避免fork主进程资源。如果主进程逻辑较为复杂，fork可能会导致异常。 +mp.set_start_method("spawn", force=True) logger = logging.getLogger() @@ -41,6 +45,15 @@ class AnalyzerController: self._is_cluster = False self.analysis_process_resp = Manager().dict() + @staticmethod + def _set_analysis_process_priority(pid): + # 将分析进程优先级设置为最低，避免因为分析进程阻塞其他任务进程，unix上19表示最低优先级 + p = psutil.Process(pid) + if platform.system().lower() == "windows": + p.nice(psutil.BELOW_NORMAL_PRIORITY_CLASS) + elif platform.system().lower() == "linux": + p.nice(19) + @staticmethod def _check_profiling_path_valid(profiling_path): PathManager.input_path_common_check(profiling_path) @@ -91,11 +104,25 @@ class AnalyzerController: return step, benchmark_step, target_rank_id + @staticmethod + def _init_async_analysis_env(kwargs): + envs = kwargs.get("async_analysis_env", {}) + for key, value in envs.items(): + os.environ[key] = value + def do_analysis(self, dimensions, **kwargs): pid = os.getpid() + AnalyzerController._set_analysis_process_priority(pid) + AnalyzerController._init_async_analysis_env(kwargs) + resp = {"id": pid} + output_path = kwargs.get("output_path") try: - self._do_analysis(dimensions, pid=pid, resp=resp, **kwargs) + if output_path: + Config().set_config("_work_path", output_path) + Config().set_log_path(f"mstt_advisor_{Timer().strftime}.xlsx") + + self._do_analysis(dimensions, pid=pid, async_resp=resp, **kwargs) except Exception as e: self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, status=AsyncAnalysisStatus.FAILED, error_msg=str(e)) @@ -107,6 +134,9 @@ class AnalyzerController: async_analysis_process = mp.Process(target=self.do_analysis, args=(dimensions,), kwargs=kwargs, name="Async advisor performance analysis") async_analysis_process.start() + self._update_analysis_process_resp(async_analysis_process.pid, {"id": async_analysis_process.pid}, + status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + status=AsyncAnalysisStatus.ANALYZING) return async_analysis_process def get_response_by_pid(self, pid): @@ -145,7 +175,7 @@ class AnalyzerController: return job_list - def cluster_analysis(self, profiling_path, benchmark_profiling_path=None): + def do_cluster_analysis(self, profiling_path, benchmark_profiling_path=None): job_list = [] # 单集群profiling分析：下发、通信、计算、显存/内存 @@ -395,28 +425,26 @@ class AnalyzerController: job_list += self.memory_analysis(analysis_profiling_path, step=slow_step) return job_list - def _do_analysis(self, dimensions, **kwargs): + def _do_analysis(self, dimensions, pid=0, async_resp=None, **kwargs): self.dimensions = dimensions self.kwargs = kwargs result_list = [] profiling_path = self.kwargs.get("profiling_path") benchmark_profiling_path = self.kwargs.get("benchmark_profiling_path") - pid = self.kwargs.get("pid") - resp = self.kwargs.get("resp") - self._update_analysis_process_resp(pid, resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, + self._update_analysis_process_resp(pid, async_resp, status_code=AsyncAnalysisStatus.NON_FAILED_STATUS_CODE, status=AsyncAnalysisStatus.ANALYZING) if not self._check_profiling_path_valid(profiling_path): error_msg = f"Got invalid argument '-d/--profiling_path' {profiling_path}, skip analysis" - self._update_analysis_process_resp(pid, resp, error_msg=error_msg, + self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, status=AsyncAnalysisStatus.FAILED) logger.error(error_msg) return if benchmark_profiling_path and not self._check_profiling_path_valid(benchmark_profiling_path): error_msg = f"Got invalid argument '-bp/--benchmark_profiling_path' {benchmark_profiling_path}, skip analysis" - self._update_analysis_process_resp(pid, resp, error_msg=error_msg, + self._update_analysis_process_resp(pid, async_resp, error_msg=error_msg, status_code=AsyncAnalysisStatus.FAILED_STATUS_CODE, status=AsyncAnalysisStatus.FAILED) logger.error(error_msg) @@ -432,7 +460,7 @@ class AnalyzerController: else: self.slow_rank_analyzer = SlowRankAnalyzer(profiling_path) self.slow_link_analyzer = SlowLinkAnalyzer(profiling_path) - job_list = self.cluster_analysis(profiling_path, benchmark_profiling_path) + job_list = self.do_cluster_analysis(profiling_path, benchmark_profiling_path) for i, (dimension, scope, interface, kwargs) in enumerate(job_list[::-1]): result_list.append( @@ -444,7 +472,7 @@ class AnalyzerController: if result and hasattr(result, "show"): result.show() break - self._get_analysis_success_resp(pid, resp) + self._get_analysis_success_resp(pid, async_resp) def _communication_rdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, benchmark_step=None): diff --git a/profiler/advisor/analyzer/base_analyzer.py b/profiler/advisor/analyzer/base_analyzer.py index 6347839b1..0c3f5a6bb 100644 --- a/profiler/advisor/analyzer/base_analyzer.py +++ b/profiler/advisor/analyzer/base_analyzer.py @@ -18,6 +18,7 @@ from typing import Dict, List, Union from abc import abstractmethod, ABCMeta from profiler.advisor.common import constant +from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.common.version_control import VersionControl from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.result.result import OptimizeResult @@ -29,7 +30,7 @@ logger = logging.getLogger() class BaseAnalyzer(VersionControl, metaclass=ABCMeta): - _SUPPORT_VERSIONS = constant.SUPPORTED_CANN_VERSION + _SUPPORT_VERSIONS = EnumParamsParser().get_options(constant.CANN_VERSION) ANALYZER_HIGH_PRIORITY_TIME_RATIO = 0.05 ANALYZER_MEDIUM_PRIORITY_TIME_RATIO = 0.03 @@ -37,8 +38,8 @@ class BaseAnalyzer(VersionControl, metaclass=ABCMeta): def __init__(self, collection_path, n_processes: int = 1, **kwargs): self.n_processes = n_processes - self.cann_version = kwargs.get("cann_version", constant.DEFAULT_CANN_VERSION) - self.torch_version = kwargs.get("torch_version", constant.DEFAULT_TORCH_VERSION) + self.cann_version = kwargs.get(constant.CANN_VERSION, EnumParamsParser().get_default(constant.CANN_VERSION)) + self.torch_version = kwargs.get(constant.TORCH_VERSION, EnumParamsParser().get_default(constant.TORCH_VERSION)) self.html_render = HTMLRender() self.collection_path = collection_path self.kwargs = kwargs diff --git a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py index bc0841152..1c6aadb15 100644 --- a/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py +++ b/profiler/advisor/analyzer/computation/ai_core_freq/ai_core_freq_analyzer.py @@ -33,7 +33,7 @@ class AICoreFreqAnalyzer(BaseAnalyzer): add_render_list = kwargs.get("add_render_list", True) ai_core_freq_checker = AICoreFreqChecker() - ai_core_freq_checker.check_ai_core_freq(self.dataset, rank_id=kwargs.get("rank_id"), stage=kwargs.get("stage")) + ai_core_freq_checker.check_ai_core_freq(self.dataset, rank_id=kwargs.get("rank"), stage=kwargs.get("stage")) ai_core_freq_checker.make_record(self.result) self.html = ai_core_freq_checker.make_render(self.html_render, add_render_list, priority=self.get_priority()) return self.result diff --git a/profiler/advisor/analyzer/computation/operator_checker.py b/profiler/advisor/analyzer/computation/operator_checker.py index 04cdc7a7f..d118ceef5 100644 --- a/profiler/advisor/analyzer/computation/operator_checker.py +++ b/profiler/advisor/analyzer/computation/operator_checker.py @@ -4,18 +4,19 @@ from textwrap import fill from typing import List from profiler.advisor.common import constant +from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.common.version_control import VersionControl from profiler.advisor.config.config import Config from profiler.advisor.dataset.profiling.info_collection import OpInfo from profiler.advisor.dataset.profiling.profiling_dataset import ProfilingDataset from profiler.advisor.result.item import OptimizeItem, StatisticsItem, OptimizeRecord -from profiler.advisor.utils.utils import safe_division +from profiler.advisor.utils.utils import safe_division, convert_to_float logger = logging.getLogger() class OperatorChecker(VersionControl): - _SUPPORT_VERSIONS = constant.SUPPORTED_CANN_VERSION + _SUPPORT_VERSIONS = EnumParamsParser().get_options(constant.CANN_VERSION) _MAX_TUNE_OP_NUM = constant.OPERATOR_OUT_TOPK _MIN_TASK_DURATION = 0 _MIN_TASK_DURATION_RATIO = 1.0 @@ -138,7 +139,15 @@ class OperatorChecker(VersionControl): return True def is_dynamic_shape(self, profiling_database: ProfilingDataset) -> bool: - less_than_cann800_list = [constant.CANN_VERSION_C30, constant.CANN_VERSION_C13, constant.CANN_VERSION_C15] + cann800_major_version = 8 + less_than_cann800_list = [] + for cann_version in EnumParamsParser().get_options(constant.CANN_VERSION): + if not cann_version: + continue + major_version = cann_version.split(".")[0] + if convert_to_float(major_version) < cann800_major_version: + less_than_cann800_list.append(cann_version) + # CANN 8.0.RC1 之前从 ge_info 中获取 op_state 属性，进行动态 shape 逻辑判断 if self.cann_version in less_than_cann800_list: if hasattr(profiling_database, "ge_info"): @@ -305,7 +314,7 @@ class OperatorChecker(VersionControl): return details def format_suggestion_content(self, profiling_data: ProfilingDataset) -> None: - if profiling_data.PROF_TYPE == constant.ASCEND_PYTORCH_PROFILER: + if profiling_data.PROF_TYPE == EnumParamsParser().profiling_type.ascend_pytorch_profiler: self._SUGGESTION.append(self.PyTorch_OPERATOR_TUNE_SUGGESTION) - elif profiling_data.PROF_TYPE == constant.MSLITE: + elif profiling_data.PROF_TYPE == EnumParamsParser.profiling_type.mslite: self._SUGGESTION.append(self.MSLite_OPERATOR_TUNE_SUGGESTION) diff --git a/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py b/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py index bc02b4c3e..64a6c2ceb 100644 --- a/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py +++ b/profiler/advisor/analyzer/computation/pp_stage_computation_analyzer.py @@ -7,7 +7,7 @@ from profiler.advisor.common.analyzer_scopes import SupportedScopes from profiler.advisor.display.html.render import HTMLRender from profiler.advisor.display.html.priority_background_color import PriorityBackgroundColor from profiler.advisor.interface.interface import Interface -from profiler.advisor.utils.utils import ParallelJob +from profiler.advisor.utils.utils import ParallelJob, get_analyze_processes from profiler.advisor.result.result import OptimizeResult from profiler.advisor.result.item import OptimizeItem, OptimizeRecord from profiler.advisor.common import constant as const @@ -33,8 +33,7 @@ class PPStageComputationAnalyzer(BaseAnalyzer): return sheet_name def optimize(self, stages_profiling_path, **kwargs): - pp_stage_processes = min(int(os.getenv("PP_STAGE_ANALYSIS_PROCESSES", 0)), len(stages_profiling_path), - const.MAX_NUM_PROCESSES) + pp_stage_processes = min(get_analyze_processes(), len(stages_profiling_path)) if pp_stage_processes <= 1: for stage_profiling_path in stages_profiling_path: self._optimize(**stage_profiling_path) @@ -60,7 +59,7 @@ class PPStageComputationAnalyzer(BaseAnalyzer): pass def _optimize(self, profiling_path, **kwargs): - stage_html_record = dict(stage=kwargs.get("stage"), rank_id=kwargs.get("rank_id"), step=kwargs.get("step")) + stage_html_record = dict(stage=kwargs.get("stage"), rank_id=kwargs.get("rank"), step=kwargs.get("step")) kwargs["add_render_list"] = False # stage 并行分析时，避免调用本身，即SupportedScopes.STAGE_COMPUTE @@ -78,7 +77,7 @@ class PPStageComputationAnalyzer(BaseAnalyzer): stage_html_record["html_list"] = [] stage_html_record["html_list"].append(analyzer.html) self._stages_rendered_html.append(stage_html_record) - self._multiprocess_result[f"rank {kwargs.get('rank_id')}".capitalize()] = result.data + self._multiprocess_result[f"rank {kwargs.get('rank')}".capitalize()] = result.data def _merge_multiprocess_result(self): self.result = OptimizeResult() diff --git a/profiler/advisor/analyzer/computation/profiling_analyzer.py b/profiler/advisor/analyzer/computation/profiling_analyzer.py index b29373e87..20ebf1da7 100644 --- a/profiler/advisor/analyzer/computation/profiling_analyzer.py +++ b/profiler/advisor/analyzer/computation/profiling_analyzer.py @@ -34,7 +34,7 @@ class ProfilingAnalyzer(BaseAnalyzer, ABC): """ profiling_data = self.get_first_data_by_key(self.dataset_list, ProfilingDataset.get_key()) checker = self.checker - rank_id = kwargs.get("rank_id") + rank_id = kwargs.get("rank") add_render_list = kwargs.get("add_render_list", True) diff --git a/profiler/advisor/common/constant.py b/profiler/advisor/common/constant.py index 956b287d7..d63f053c4 100644 --- a/profiler/advisor/common/constant.py +++ b/profiler/advisor/common/constant.py @@ -40,8 +40,8 @@ MIX_AIC = "MIX_AIC" CALL_STACKS = "Call stack" INPUT_DIMS = "Input Dims" OP_SEP = "-" -MA_ADVISOR_MAX_PROCESSES = 16 -MA_ADVISOR_ANALYZE_PROCESSES = "MA_ADVISOR_ANALYZE_PROCESSES" +ADVISOR_MAX_PROCESSES = 8 +ADVISOR_ANALYZE_PROCESSES = "ADVISOR_ANALYZE_PROCESSES" TIMELINE_OP_STACKS_DATASET = "timeline_op_stacks_dataset" TIMELINE_BACKWARD_NO_STACK = "Backward broadcast, without call stacks in profiling." TIMELINE_ACL_TO_NPU_NO_STACK = "Incoming flow is 'acl_to_npu', without call stacks in profiling." @@ -60,24 +60,12 @@ TIMELINE_EMPTY_STACKS_PROMPT = "These APIs have no code stack. If parameter 'wit CLUSTER_ANALYSIS = "Cluster analysis" SLOW_RANK_TIME_RATIO_THRESHOLD = 0.05 -# version_control -CANN_VERSION_C30 = '6.3.RC2' -CANN_VERSION_C13 = '7.0.RC1' -CANN_VERSION_C15 = '7.0.0' -CANN_VERSION_C17 = '8.0.RC1' -SUPPORTED_CANN_VERSION = [CANN_VERSION_C30, CANN_VERSION_C13, CANN_VERSION_C15, CANN_VERSION_C17] -DEFAULT_CANN_VERSION = CANN_VERSION_C17 -ASCEND_PYTORCH_PROFILER = "ascend_pytorch_profiler" +CANN_VERSION = "cann_version" +TORCH_VERSION = "torch_version" +PROFILING_TYPE = "profiling_type" +ANALYSIS_DIMENSIONS = "analysis_dimensions" + PROFILER_METADATA = "profiler_metadata.json" -MSLITE = "mslite" -MSPROF = "msprof" -SUPPORTED_PROFILING_TYPE = [ASCEND_PYTORCH_PROFILER, MSLITE, MSPROF] -DEFAULT_PROFILING_TYPE = ASCEND_PYTORCH_PROFILER -TORCH_VERSION_1_11_0 = '1.11.0' -TORCH_VERSION_2_1_0 = '2.1.0' - -SUPPORTED_TORCH_VERSION = [TORCH_VERSION_1_11_0, TORCH_VERSION_2_1_0] -DEFAULT_TORCH_VERSION = TORCH_VERSION_2_1_0 TERMINAL_OUTPUT_HEADERS = ["No.", "Problem", "Description", "Suggestion"] SKIP_ANALYZE_PROMPT = "Finish analysis, no optimization suggestions" diff --git a/profiler/advisor/common/enum_params_parser.py b/profiler/advisor/common/enum_params_parser.py new file mode 100644 index 000000000..9731a4cad --- /dev/null +++ b/profiler/advisor/common/enum_params_parser.py @@ -0,0 +1,70 @@ +import os +import logging +import typing + +from profiler.advisor.common.timeline.event import AdvisorDict +from profiler.advisor.utils.utils import singleton +from profiler.cluster_analyse.common_func.file_manager import FileManager + +logger = logging.getLogger() + + +@singleton +class EnumParamsParser(): + ARGUMENTS = "arguments" + ENVS = "envs" + OPTIONS = "options" + DEFAULT = "default" + + def __init__(self): + enum_params_path = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "config", + "enum_parameters.yaml") + self.enum_params = FileManager.read_yaml_file(enum_params_path) + self._set_value() + + def get_keys(self): + return list(self.get_arguments_keys()) + list(self.get_envs_keys()) + + def get_arguments_keys(self): + return list(self.enum_params.get(self.ARGUMENTS, {}).keys()) + + def get_envs_keys(self): + return list(self.enum_params.get(self.ENVS, {}).keys()) + + def get_options(self, key): + options = [] + for param_type in [self.ARGUMENTS, self.ENVS]: + if key not in self.enum_params.get(param_type, {}): + continue + options = self.enum_params.get(param_type, {}).get(key, {}).get(self.OPTIONS, []) + + if not options: + logger.error("Key %s not exists, optionals are %s", key, self.get_keys()) + + return options + + def get_default(self, key): + default_value = None + for param_type in [self.ARGUMENTS, self.ENVS]: + if key not in self.enum_params.get(param_type, {}): + continue + default_value = self.enum_params.get(param_type, {}).get(key, {}).get(self.DEFAULT, []) + + if not default_value: + logger.error("Key %s not exists, optionals are %s", key, self.get_keys()) + + return default_value + + def _set_value(self): + + for key in self.get_keys(): + + if not hasattr(self, key): + setattr(self, str(key), AdvisorDict()) + + options = self.get_options(key) + + for value in options: + if not isinstance(value, typing.Hashable): + continue + getattr(self, key)[str(value)] = value diff --git a/profiler/advisor/common/timeline/fusion_ops_db.py b/profiler/advisor/common/timeline/fusion_ops_db.py index 64cc84929..3f9a7b59b 100644 --- a/profiler/advisor/common/timeline/fusion_ops_db.py +++ b/profiler/advisor/common/timeline/fusion_ops_db.py @@ -2,6 +2,7 @@ import logging import os from profiler.advisor.common import constant +from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.common.timeline.fusion_ops_rule import OpRule from profiler.advisor.common.timeline.fusion_ops_rule_handler import TimelineOpRuleHandler from profiler.advisor.utils.log import get_log_level @@ -50,8 +51,8 @@ class FusionOperatorDB: def __init__(self, file_path=None, cann_version=None, torch_version=None): self.timeline_fusion_ops_yaml_path = os.path.normpath(get_timeline_fusion_ops_yaml_path()) - self.cann_version = cann_version or constant.DEFAULT_CANN_VERSION - self.torch_version = torch_version or constant.DEFAULT_TORCH_VERSION + self.cann_version = cann_version or EnumParamsParser().get_default(constant.CANN_VERSION) + self.torch_version = torch_version or EnumParamsParser().get_default(constant.TORCH_VERSION) self._supported_version_dict = {} diff --git a/profiler/advisor/config/enum_parameters.yaml b/profiler/advisor/config/enum_parameters.yaml new file mode 100644 index 000000000..030f0daf5 --- /dev/null +++ b/profiler/advisor/config/enum_parameters.yaml @@ -0,0 +1,36 @@ +#ͨӿԶȡ³κͿѡֵĬֵҪڶԽӵ +arguments: + cann_version: + options: + - 6.3.RC2 + - 7.0.RC1 + - 7.0.0 + - 8.0.RC1 + default: 8.0.RC1 + + torch_version: + options: + - 1.11.0 + - 2.1.0 + default: 2.1.0 + + analysis_dimensions: + options: + - [ computation, communication, schedule, memory ] + - [ communication ] + - [ schedule ] + - [ computation ] + - [ memory ] + default: [ computation, communication, schedule, memory ] + + profiling_type: + options: + - ascend_pytorch_profiler + - mslite + - msprof + default: ascend_pytorch_profiler + +envs: + ADVISOR_ANALYZE_PROCESSES: + options: [ 1, 2, 3, 4, 5, 6, 7, 8 ] + default: 1 \ No newline at end of file diff --git a/profiler/advisor/dataset/profiling/profiling_dataset.py b/profiler/advisor/dataset/profiling/profiling_dataset.py index ebd90951a..16e6aef5a 100644 --- a/profiler/advisor/dataset/profiling/profiling_dataset.py +++ b/profiler/advisor/dataset/profiling/profiling_dataset.py @@ -7,6 +7,7 @@ from profiler.advisor.common.profiling.ge_info import GeInfo from profiler.advisor.common.profiling.msprof import Msprof from profiler.advisor.common.profiling.op_summary import OpSummary from profiler.advisor.common.profiling.tasktime import TaskTime +from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.dataset.dataset import Dataset from profiler.advisor.dataset.profiling.device_info import DeviceInfoParser from profiler.advisor.utils.utils import join_prof_path @@ -20,8 +21,8 @@ class ProfilingDataset(Dataset): PROF_TYPE = "" def __init__(self, collection_path, data: dict, **kwargs) -> None: - self.cann_version = kwargs.get("cann_version", constant.DEFAULT_CANN_VERSION) - self.PROF_TYPE = kwargs.get("profiling_type", constant.DEFAULT_PROFILING_TYPE) + self.cann_version = kwargs.get(constant.CANN_VERSION, EnumParamsParser().get_default(constant.CANN_VERSION)) + self.PROF_TYPE = kwargs.get(constant.PROFILING_TYPE, EnumParamsParser().get_default(constant.PROFILING_TYPE)) self.patterns = self.parse_pattern() self.current_version_pattern = self.get_current_version_pattern() super().__init__(collection_path, data) diff --git a/profiler/advisor/dataset/timeline_event_dataset.py b/profiler/advisor/dataset/timeline_event_dataset.py index 44bbc141d..e0a12fab6 100644 --- a/profiler/advisor/dataset/timeline_event_dataset.py +++ b/profiler/advisor/dataset/timeline_event_dataset.py @@ -41,7 +41,7 @@ class BaseTimelineEventDataset: lambda file: file.endswith("trace_view.json")) self.dataset_len = None self.step = kwargs.get("step") - self.step_duration = None + self.step_duration = 0.0 if not build_dataset: return @@ -196,6 +196,7 @@ class ScheduleAnalysisDataset(BaseTimelineEventDataset): self.aten = formated_atens +@singleton class ComputationAnalysisDataset(BaseTimelineEventDataset): collector_map = OrderedDict( StepCollector=StepCollector(), diff --git a/profiler/advisor/utils/utils.py b/profiler/advisor/utils/utils.py index f41bba53d..71d3dd173 100644 --- a/profiler/advisor/utils/utils.py +++ b/profiler/advisor/utils/utils.py @@ -69,16 +69,21 @@ def singleton(cls): _instance = {} def _singleton(*args: any, **kw: any) -> any: + # 适配多进程异步调用场景，确保不同子进程的单例类互相隔离 + pid = os.getpid() + if pid not in _instance: + _instance[pid] = {} + collection_path = kw.get("collection_path") if not collection_path: collection_path = get_class_absolute_path(cls) - if cls in _instance and collection_path in _instance[cls]: - return _instance[cls].get(collection_path) - if cls not in _instance: - _instance[cls] = {collection_path: cls(*args, **kw)} + if cls in _instance[pid] and collection_path in _instance[pid][cls]: + return _instance[pid][cls].get(collection_path) + if cls not in _instance[pid]: + _instance[pid][cls] = {collection_path: cls(*args, **kw)} else: - _instance[cls][collection_path] = cls(*args, **kw) - return _instance[cls].get(collection_path) + _instance[pid][cls][collection_path] = cls(*args, **kw) + return _instance[pid][cls].get(collection_path) def reset_all_instances(): """ @@ -182,7 +187,7 @@ class Timer: def get_analyze_processes(): # n_processes not exposed to user through att-advisor command arguments now - return min(int(os.getenv(const.MA_ADVISOR_ANALYZE_PROCESSES, 1)), const.MA_ADVISOR_MAX_PROCESSES) + return min(int(os.getenv(const.ADVISOR_ANALYZE_PROCESSES, 1)), const.ADVISOR_MAX_PROCESSES) def format_timeline_result(result: dict, dump_html=False): diff --git a/profiler/cli/__init__.py b/profiler/cli/__init__.py index e768e4cb8..e69de29bb 100644 --- a/profiler/cli/__init__.py +++ b/profiler/cli/__init__.py @@ -1,4 +0,0 @@ -from profiler.advisor.config.config import Config -from profiler.advisor.utils.utils import Timer - -Config().set_log_path(f"mstt_advisor_{Timer().strftime}.xlsx") diff --git a/profiler/cli/analyze_cli.py b/profiler/cli/analyze_cli.py index 01758f7b9..3a2cb6dc3 100644 --- a/profiler/cli/analyze_cli.py +++ b/profiler/cli/analyze_cli.py @@ -9,6 +9,7 @@ sys.path.append(os.path.join(os.path.dirname(os.path.dirname(__file__)), "cluste from profiler.advisor.analyzer.analyzer_controller import AnalyzerController from profiler.advisor.utils.tools import CONTEXT_SETTINGS, ClickAliasedGroup from profiler.advisor.common import constant +from profiler.advisor.common.enum_params_parser import EnumParamsParser from profiler.advisor.utils.utils import debug_option from profiler.advisor.interface.interface import Interface @@ -29,21 +30,23 @@ def analyze_cli(**kwargs): help='Directory of profiling data') @click.option('--benchmark_profiling_path', '-bp', 'benchmark_profiling_path', type=click.Path(), help='Directory of benchmark profiling data, used for compare performance') +@click.option('--output_path', '-o', 'output_path', type=click.Path(), + help='Path of analysis output') @click.option('--cann_version', '-cv', 'cann_version', - type=click.Choice(constant.SUPPORTED_CANN_VERSION, case_sensitive=False), - default=constant.DEFAULT_CANN_VERSION, + type=click.Choice(EnumParamsParser().get_options(constant.CANN_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(constant.CANN_VERSION), help='The CANN software version, which can be viewed by executing the following command: ' '"cat /usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/ascend_toolkit_install.info"') @click.option('--torch_version', '-tv', 'torch_version', - type=click.Choice(constant.SUPPORTED_TORCH_VERSION, case_sensitive=False), - default=constant.DEFAULT_TORCH_VERSION, + type=click.Choice(EnumParamsParser().get_options(constant.TORCH_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(constant.TORCH_VERSION), help='The runtime torch version, which can be detected by exec command "pip show torch"') @click.option("-pt", "--profiling_type", metavar="", - default=constant.ASCEND_PYTORCH_PROFILER, + default=EnumParamsParser().get_default(constant.PROFILING_TYPE), required=False, - type=click.Choice(constant.SUPPORTED_PROFILING_TYPE), + type=click.Choice(EnumParamsParser().get_options(constant.PROFILING_TYPE)), help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @debug_option def analyze_all(**kwargs) -> None: @@ -55,15 +58,24 @@ def analyze_all(**kwargs) -> None: short_help='Analyze operators dispatching and timeline fusion operators.') @click.option('--profiling_path', '-d', 'profiling_path', type=click.Path(), required=True, help='Directory of profiling data') +@click.option('--output_path', '-o', 'output_path', type=click.Path(), + help='Path of analysis output') @click.option('--cann_version', '-cv', 'cann_version', - type=click.Choice(constant.SUPPORTED_CANN_VERSION, case_sensitive=False), - default=constant.DEFAULT_CANN_VERSION, + type=click.Choice(EnumParamsParser().get_options(constant.CANN_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(constant.CANN_VERSION), help='The CANN software version, which can be viewed by executing the following command: ' '"cat /usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/ascend_toolkit_install.info"') @click.option('--torch_version', '-tv', 'torch_version', - type=click.Choice(constant.SUPPORTED_TORCH_VERSION, case_sensitive=False), - default=constant.DEFAULT_TORCH_VERSION, + type=click.Choice(EnumParamsParser().get_options(constant.TORCH_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(constant.TORCH_VERSION), help='The runtime torch version, which can be detected by exec command "pip show torch"') +@click.option("-pt", + "--profiling_type", + metavar="", + default=EnumParamsParser().get_default(constant.PROFILING_TYPE), + required=False, + type=click.Choice(EnumParamsParser().get_options(constant.PROFILING_TYPE)), + help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @debug_option def analyze_schedule(**kwargs) -> None: AnalyzerController().do_analysis([Interface.SCHEDULE], **kwargs) @@ -74,21 +86,23 @@ def analyze_schedule(**kwargs) -> None: short_help='Analyze operators and graph.') @click.option('--profiling_path', '-d', 'profiling_path', type=click.Path(), required=True, help='Directory of profiling data') +@click.option('--output_path', '-o', 'output_path', type=click.Path(), + help='Path of analysis output') @click.option('--cann_version', '-cv', 'cann_version', - type=click.Choice(constant.SUPPORTED_CANN_VERSION, case_sensitive=False), - default=constant.DEFAULT_CANN_VERSION, + type=click.Choice(EnumParamsParser().get_options(constant.CANN_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(constant.CANN_VERSION), help='The CANN software version, which can be viewed by executing the following command: ' '"cat /usr/local/Ascend/ascend-toolkit/latest/aarch64-linux/ascend_toolkit_install.info"') @click.option('--torch_version', '-tv', 'torch_version', - type=click.Choice(constant.SUPPORTED_TORCH_VERSION, case_sensitive=False), - default=constant.DEFAULT_TORCH_VERSION, + type=click.Choice(EnumParamsParser().get_options(constant.TORCH_VERSION), case_sensitive=False), + default=EnumParamsParser().get_default(constant.TORCH_VERSION), help='The runtime torch version, which can be detected by exec command "pip show torch"') @click.option("-pt", "--profiling_type", metavar="", - default=constant.ASCEND_PYTORCH_PROFILER, + default=EnumParamsParser().get_default(constant.PROFILING_TYPE), required=False, - type=click.Choice(constant.SUPPORTED_PROFILING_TYPE), + type=click.Choice(EnumParamsParser().get_options(constant.PROFILING_TYPE)), help="enter the profiling type, selectable range ascend_pytorch_profiler, mslite ,msprof") @debug_option def analyze_computation(**kwargs) -> None: diff --git a/profiler/requirements/build.txt b/profiler/requirements/build.txt index ec879a353..fa00e62c7 100644 --- a/profiler/requirements/build.txt +++ b/profiler/requirements/build.txt @@ -12,4 +12,5 @@ sqlalchemy urllib3<2.0 bottleneck>=1.3.6 numpy==1.26.4 -pandas \ No newline at end of file +pandas +psutil \ No newline at end of file diff --git a/profiler/test/ut/advisor/common/__init__.py b/profiler/test/ut/advisor/common/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/profiler/test/ut/advisor/common/test_enum_params_parser.py b/profiler/test/ut/advisor/common/test_enum_params_parser.py new file mode 100644 index 000000000..94b2c360b --- /dev/null +++ b/profiler/test/ut/advisor/common/test_enum_params_parser.py @@ -0,0 +1,55 @@ +import unittest +import sys +import os + +profiler_root_dir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))) +sys.path.append(os.path.join(profiler_root_dir, "compare_tools")) +sys.path.append(os.path.join(profiler_root_dir, "cluster_analyse")) + +from profiler.advisor.common.enum_params_parser import EnumParamsParser +from profiler.test.ut.advisor.advisor_backend.tools.tool import recover_env + + +class TestEnumParamsParser(unittest.TestCase): + @classmethod + def tearDownClass(cls) -> None: + recover_env() + + def setUp(self) -> None: + self.enum_params_parser = EnumParamsParser() + self.argument_keys = sorted(["cann_version", "torch_version", "analysis_dimensions", "profiling_type"]) + self.env_keys = ["ADVISOR_ANALYZE_PROCESSES"] + + def test_get_keys(self): + total_keys = sorted(self.argument_keys + self.env_keys) + keys = sorted(self.enum_params_parser.get_keys()) + self.assertTrue(isinstance(keys, list)) + self.assertEqual(keys, total_keys) + + def test_get_argument_keys(self): + argument_keys = sorted(self.enum_params_parser.get_arguments_keys()) + self.assertTrue(isinstance(argument_keys, list)) + self.assertEqual(argument_keys, self.argument_keys) + + def test_get_env_keys(self): + env_keys = sorted(self.enum_params_parser.get_envs_keys()) + self.assertTrue(isinstance(env_keys, list)) + self.assertEqual(env_keys, self.env_keys) + + def test_get_default(self): + self.assertTrue(self.enum_params_parser.get_default("cann_version"), "8.0.RC1") + self.assertTrue(self.enum_params_parser.get_default("torch_version"), "2.1.0") + self.assertTrue(self.enum_params_parser.get_default("analysis_dimensions"), + ["computation", "communication", "schedule", "memory"]) + self.assertTrue(self.enum_params_parser.get_default("profiling_type"), "ascend_pytorch_profiler") + self.assertTrue(self.enum_params_parser.get_default("ADVISOR_ANALYZE_PROCESSES"), 1) + + def test_get_options(self): + self.assertTrue(self.enum_params_parser.get_options("cann_version"), ["6.3.RC2", "7.0.RC1", "7.0.0", "8.0.RC1"]) + self.assertTrue(self.enum_params_parser.get_options("torch_version"), ["1.11.0", "2.1.0"]) + self.assertTrue(self.enum_params_parser.get_options("analysis_dimensions"), + [["computation", "communication", "schedule", "memory"], ["communication"], ["schedule"], + ["computation"], ["memory"]]) + self.assertTrue(self.enum_params_parser.get_options("profiling_type"), + ["ascend_pytorch_profiler", "mslite", "msprof"]) + self.assertTrue(self.enum_params_parser.get_options("ADVISOR_ANALYZE_PROCESSES"), list(range(1, 9))) -- Gitee From 85d3a3015092311d877936257d0789d6e5f380ba Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Thu, 5 Sep 2024 15:40:21 +0800 Subject: [PATCH 701/791] =?UTF-8?q?=E8=A7=A3=E5=86=B3=E5=86=B2=E7=AA=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/mindspore/common/utils.py | 3 +-- debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 5 +++-- .../msprobe/mindspore/compare/ms_graph_compare.py | 4 ++-- .../msprobe/mindspore/debugger/debugger_config.py | 2 +- .../msprobe/mindspore/grad_probe/grad_analyzer.py | 4 ++-- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index a0dd3aa2e..2b9cd1b61 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -16,8 +16,7 @@ import os import mindspore as ms from msprobe.core.common.exceptions import DistributedNotInitializedError -from msprobe.core.common.file_check import path_len_exceeds_limit, check_path_exists -from msprobe.core.common.utils import save_npy +from msprobe.core.common.file_utils import path_len_exceeds_limit, check_path_exists, save_npy from msprobe.core.common.log import logger diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index ddbbe6cfe..36f7224fa 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -1,14 +1,15 @@ import os import copy from msprobe.core.common.utils import check_compare_param, CompareException, check_configuration_param, \ - task_dumppath_get, load_yaml, load_npy -from msprobe.core.common.file_check import create_directory + task_dumppath_get +from msprobe.core.common.file_utils import create_directory, load_yaml, load_npy from msprobe.core.common.const import Const, CompareConst from msprobe.core.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator from msprobe.core.compare.check import check_struct_match, fuzzy_check_op + class MSComparator(Comparator): def __init__(self, cell_mapping=None, api_mapping=None): self.frame_name = MSComparator.__name__ diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index ccff74993..75890a321 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -6,9 +6,9 @@ import os import numpy as np import pandas as pd from msprobe.core.common.const import CompareConst, GraphMode, Const, FileCheckConst -from msprobe.core.common.file_check import FileOpen, check_path_before_create, change_mode +from msprobe.core.common.file_utils import FileOpen, check_path_before_create, change_mode, load_npy from msprobe.core.common.log import logger -from msprobe.core.common.utils import add_time_with_xlsx, CompareException, load_npy +from msprobe.core.common.utils import add_time_with_xlsx, CompareException from msprobe.core.compare.multiprocessing_compute import _ms_graph_handle_multi_process, check_accuracy from msprobe.core.compare.npy_compare import npy_data_check, statistics_data_check, reshape_value, compare_ops_apply from msprobe.mindspore.common.utils import convert_to_int, list_lowest_level_directories diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index f2dc0085e..974e420e8 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -3,7 +3,7 @@ import os from msprobe.core.common.const import Const from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.common.const import FreeBenchmarkConst -from msprobe.core.common.file_check import create_directory +from msprobe.core.common.file_utils import create_directory class DebuggerConfig: diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py index c22ba9b62..17120ca6d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/grad_analyzer.py @@ -13,8 +13,8 @@ from mindspore.common.parameter import Parameter from msprobe.core.grad_probe.utils import ListCache from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_check import create_directory -from msprobe.core.common.utils import check_file_or_directory_path, write_csv, remove_path, move_file, load_npy +from msprobe.core.common.file_utils import (create_directory, check_file_or_directory_path, + write_csv, remove_path, move_file, load_npy) from msprobe.mindspore.grad_probe.global_context import grad_context, GlobalContext -- Gitee From e1a0ea2c0b8fac070890dd06d37e2c02dc82bdba Mon Sep 17 00:00:00 2001 From: zyy Date: Thu, 5 Sep 2024 16:08:49 +0800 Subject: [PATCH 702/791] =?UTF-8?q?=E8=A7=A3=E6=9E=90=E8=BD=AC=E5=8C=96?= =?UTF-8?q?=E5=AD=97=E7=AC=A6=E4=B8=B2=E9=97=AE=E9=A2=98=EF=BC=8Cremove=5F?= =?UTF-8?q?path=5Fsafety=E5=87=BD=E6=95=B0=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../test/ut/cluster_analyse/common_func/test_path_manager.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py b/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py index 66ef15cd2..0510cf4a7 100644 --- a/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py +++ b/profiler/test/ut/cluster_analyse/common_func/test_path_manager.py @@ -71,8 +71,6 @@ class TestPathManager(unittest.TestCase): os.symlink(PATH_FILE, link_name) with pytest.raises(RuntimeError) as error: PathManager.remove_path_safety(link_name) - with pytest.raises(RuntimeError) as error: - PathManager.remove_path_safety(path + "not_exist") PathManager.remove_path_safety(path) os.unlink(link_name) -- Gitee From 4fade3f8443623a96b77ca6c8fb2cf8d39de2b04 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Thu, 5 Sep 2024 11:43:31 +0800 Subject: [PATCH 703/791] =?UTF-8?q?advisor=20=E4=BF=AE=E5=A4=8D=E5=8D=95?= =?UTF-8?q?=E5=8D=A1=E5=9C=BA=E6=99=AF=E9=80=9A=E4=BF=A1=E5=88=86=E6=9E=90?= =?UTF-8?q?=E8=B7=B3=E8=BF=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 6 +- .../advisor/analyzer/analyzer_controller.py | 66 ++++++++++--------- 2 files changed, 38 insertions(+), 34 deletions(-) diff --git a/profiler/README.md b/profiler/README.md index e6e894525..0ab2dd4ba 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -105,9 +105,9 @@ Successfully installed msprof-analyze-{version} 请通过下表链接下载profiler工具whl包。 - | profiler版本 | 发布日期 | 下载链接 | 校验码 | - |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ------------------------------------------------------------ | - | 1.2.3 | 2024-08-29 | [msprof_analyze-1.2.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.3/msprof_analyze-1.2.3-py3-none-any.whl) | b0348d5cbf82d72cc292fc979a9f385c046e46a1edc5a0cb38cbea515b239845 | + | profiler版本 | 发布日期 | 下载链接 | 校验码 | + |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ----------------------------------------------------------- | + | 1.2.3 | 2024-08-29 | [msprof_analyze-1.2.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.3/msprof_analyze-1.2.3-py3-none-any.whl) | 354a55747f64ba1ec6ee6fe0f05a53e84e1b403ee0341ec40cc216dd25fda14c | | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.2/msprof_analyze-1.2.2-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | | 1.2.0 | 2024-07-25 | [msprof_analyze-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.0/msprof_analyze-1.2.0-py3-none-any.whl) | 6a4366e3beca40b4a8305080e6e441d6ecafb5c05489e5905ac0265787555f37 | diff --git a/profiler/advisor/analyzer/analyzer_controller.py b/profiler/advisor/analyzer/analyzer_controller.py index 6cba61b19..c0ff7e1ee 100644 --- a/profiler/advisor/analyzer/analyzer_controller.py +++ b/profiler/advisor/analyzer/analyzer_controller.py @@ -31,6 +31,10 @@ class AnalyzerController: CLUSTER_RANK_THRESHOLD = 2 SDMA_SUPPORT_SCOPES = [SupportedScopes.BANDWIDTH_CONTENTION_DETECTION] RDMA_SUPPORT_SCOPES = [SupportedScopes.PACKET] + COMMUNICATION_MAPPING = { + SlowLinkAnalyzer.SDMA: SDMA_SUPPORT_SCOPES, + SlowLinkAnalyzer.RDMA: RDMA_SUPPORT_SCOPES + } def __init__(self): self.dimensions = Interface.all_dimension @@ -244,13 +248,10 @@ class AnalyzerController: logger.error("Error transit type %s, optionals are %s", bandwidth_type, supported_trans_type) return job_list - bandwidth_type_list = [bandwidth_type] if bandwidth_type is not None else supported_trans_type - - for bandwidth_type in bandwidth_type_list: - job_list += getattr(self, f"_communication_{bandwidth_type.lower()}_analysis")(profiling_path, - benchmark_profiling_path, - step, benchmark_step, - scope) + job_list += self._communication_analysis(profiling_path=profiling_path, + benchmark_profiling_path=benchmark_profiling_path, + step=step, benchmark_step=benchmark_step, + scope=scope, bandwidth_type=bandwidth_type) return job_list @@ -452,36 +453,39 @@ class AnalyzerController: break self._get_analysis_success_resp(pid, resp) - def _communication_rdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, - benchmark_step=None, scope=None): - # 小包分析 - kwargs = copy.deepcopy(self.kwargs) - job_list = [] - - kwargs["profiling_path"] = profiling_path - kwargs["benchmark_profiling_path"] = benchmark_profiling_path - kwargs["step"] = step - kwargs["benchmark_step"] = benchmark_step - - if scope in self.RDMA_SUPPORT_SCOPES: - interface = Interface(**kwargs) - job_list.append((Interface.COMMUNICATION, scope, interface, kwargs)) - - return job_list + def _get_scopes(self, scope=None, bandwidth_type=SlowLinkAnalyzer.SDMA): + """ + Args: + scope: analyzer type + bandwidth_type: analysis standard + Returns: + scope lists + """ + scopes = [] + if scope: + if scope in self.COMMUNICATION_MAPPING.get(bandwidth_type, self.SDMA_SUPPORT_SCOPES): + scopes.append(scope) + return scopes + for dimension in [Interface.COMMUNICATION]: + for scope_ in Interface.get_scope(dimension): + if scope_ in self.SDMA_SUPPORT_SCOPES or scope_ in self.RDMA_SUPPORT_SCOPES: + scopes.append(scope_) + return scopes - def _communication_sdma_analysis(self, profiling_path, benchmark_profiling_path=None, step=None, - benchmark_step=None, scope=None): + def _communication_analysis(self, **child_kwargs): kwargs = copy.deepcopy(self.kwargs) job_list = [] - kwargs["profiling_path"] = profiling_path - kwargs["benchmark_profiling_path"] = benchmark_profiling_path - kwargs["step"] = step - kwargs["benchmark_step"] = benchmark_step + kwargs["profiling_path"] = child_kwargs.get("profiling_path", "") + kwargs["benchmark_profiling_path"] = child_kwargs.get("benchmark_profiling_path", "") + kwargs["step"] = child_kwargs.get("step", -1) + kwargs["benchmark_step"] = child_kwargs.get("benchmark_step", -1) + bandwidth_type = child_kwargs.get("bandwidth_type", SlowLinkAnalyzer.SDMA) + scope = child_kwargs.get("scope", None) - if scope in self.SDMA_SUPPORT_SCOPES: + for scope_ in self._get_scopes(scope, bandwidth_type): interface = Interface(**kwargs) - job_list.append((Interface.COMMUNICATION, scope, interface, kwargs)) + job_list.append((Interface.COMMUNICATION, scope_, interface, kwargs)) return job_list -- Gitee From cfacdf74e5b5699abcdf5daf2056389733533d87 Mon Sep 17 00:00:00 2001 From: jijiarong Date: Thu, 5 Sep 2024 16:55:10 +0800 Subject: [PATCH 704/791] mindspore compare bugfix --- .../msprobe/mindspore/compare/ms_graph_compare.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index ccff74993..2b14e546d 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -91,7 +91,7 @@ def generate_data_name(data_path): statistic_exist = bool(statistic_file_list) npy_exist = bool(npy_file_list) - mapping_dict = [] + mapping_dict = {} if mapping_exist: for mapping_file in mapping_file_list: with FileOpen(mapping_file, "r") as f: -- Gitee From 4663bc8cc9e47c4e315b8c4f8a757be6cec2878b Mon Sep 17 00:00:00 2001 From: jijiarong Date: Thu, 5 Sep 2024 18:04:09 +0800 Subject: [PATCH 705/791] bugfix ms graph compare --- .../msprobe/mindspore/compare/ms_graph_compare.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py index ccff74993..01f931fc0 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_graph_compare.py @@ -315,6 +315,8 @@ class GraphMSComparator: else: if len(split_path) > 4: rank_id = convert_to_int(split_path[-4]) + if rank_id == -1 and len(split_path) > 3: + rank_id = convert_to_int(split_path[-3]) step_id = convert_to_int(split_path[-1]) return rank_id, step_id -- Gitee From 00896a25baf89e24d96d0389b851277c73a99aa2 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Thu, 5 Sep 2024 19:44:29 +0800 Subject: [PATCH 706/791] =?UTF-8?q?=E4=BF=AE=E5=A4=8Ddt?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../api_accuracy_checker/run_ut/test_multi_run_ut.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_multi_run_ut.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_multi_run_ut.py index 27126cddd..8913c7160 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_multi_run_ut.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/run_ut/test_multi_run_ut.py @@ -28,12 +28,12 @@ class TestMultiRunUT(unittest.TestCase): self.assertEqual(len(split_files), num_splits) self.assertEqual(total_items, len(self.test_data.get('data'))) - + @patch('msprobe.pytorch.api_accuracy_checker.run_ut.multi_run_ut.remove_path') @patch('subprocess.Popen') @patch('os.path.exists', return_value=True) @patch('builtins.open', new_callable=mock_open) @patch('json.load', side_effect=lambda f: {'key1': 'TRUE', 'key2': 'TRUE'}) - def test_run_parallel_ut(self, mock_json_load, mock_file, mock_exists, mock_popen): + def test_run_parallel_ut(self, mock_json_load, mock_file, mock_exists, mock_popen, _): mock_process = MagicMock() mock_process.poll.side_effect = [None, None, 1] mock_process.stdout.readline.side_effect = ['[ERROR] Test Error Message\n', ''] -- Gitee From 99104b3adb58d0b7acc777317a7746c8dc3b2193 Mon Sep 17 00:00:00 2001 From: zyy Date: Thu, 5 Sep 2024 19:57:22 +0800 Subject: [PATCH 707/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cli/cluster_cli.py | 2 +- profiler/cluster_analyse/cluster_analysis.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/cli/cluster_cli.py b/profiler/cli/cluster_cli.py index 261d70dff..3a10ed243 100644 --- a/profiler/cli/cluster_cli.py +++ b/profiler/cli/cluster_cli.py @@ -34,7 +34,7 @@ context_settings['ignore_unknown_options'] = True @click.option('--profiling_path', '-d', type=click.Path(), required=True, help='path of the profiling data') @click.option('--mode', '-m', type=click.Choice(COMM_FEATURE_LIST), default='all') -@click.option('--output_path', '-o', 'cluster_analysis_output_path', type=click.Path(), +@click.option('--output_path', '-o', 'cluster_analysis_output_path', type=click.Path(), default='all', help='Path of cluster analysis output') @click.argument('args', nargs=-1) def cluster_cli(profiling_path, mode, cluster_analysis_output_path, args) -> None: diff --git a/profiler/cluster_analyse/cluster_analysis.py b/profiler/cluster_analyse/cluster_analysis.py index c29f3750c..9e0c9d154 100644 --- a/profiler/cluster_analyse/cluster_analysis.py +++ b/profiler/cluster_analyse/cluster_analysis.py @@ -47,7 +47,7 @@ class Interface: def get_cluster_analysis_output_path(self, params): cluster_analysis_output_path = params.get(Constant.CLUSTER_ANALYSIS_OUTPUT_PATH) - if cluster_analysis_output_path and cluster_analysis_output_path != "None": + if cluster_analysis_output_path: return PathManager.get_realpath(cluster_analysis_output_path) return self.collection_path def allocate_prof_data(self): -- Gitee From 33305c62180c1e92fdc061999b861a71148991a2 Mon Sep 17 00:00:00 2001 From: zyy Date: Thu, 5 Sep 2024 19:58:31 +0800 Subject: [PATCH 708/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E5=80=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/cli/cluster_cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/profiler/cli/cluster_cli.py b/profiler/cli/cluster_cli.py index 3a10ed243..6d7d8afb8 100644 --- a/profiler/cli/cluster_cli.py +++ b/profiler/cli/cluster_cli.py @@ -34,7 +34,7 @@ context_settings['ignore_unknown_options'] = True @click.option('--profiling_path', '-d', type=click.Path(), required=True, help='path of the profiling data') @click.option('--mode', '-m', type=click.Choice(COMM_FEATURE_LIST), default='all') -@click.option('--output_path', '-o', 'cluster_analysis_output_path', type=click.Path(), default='all', +@click.option('--output_path', '-o', 'cluster_analysis_output_path', type=click.Path(), default='', help='Path of cluster analysis output') @click.argument('args', nargs=-1) def cluster_cli(profiling_path, mode, cluster_analysis_output_path, args) -> None: -- Gitee From aa2f80561c4e0157f39ec2e992e0541e3f32b9b6 Mon Sep 17 00:00:00 2001 From: wugengjun <451676383@qq.com> Date: Thu, 5 Sep 2024 20:57:41 +0800 Subject: [PATCH 709/791] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=A3=80=E8=A7=86?= =?UTF-8?q?=E6=84=8F=E8=A7=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../mindspore/free_benchmark/api_pynative_self_check.py | 3 +-- debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py | 3 +-- .../accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py | 3 +-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py index 59016653a..a92b3ed4c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py +++ b/debug/accuracy_tools/msprobe/mindspore/free_benchmark/api_pynative_self_check.py @@ -5,12 +5,11 @@ import importlib import mindspore as ms from mindspore.communication import comm_func -from msprobe.core.common.file_utils import load_yaml +from msprobe.core.common.file_utils import load_yaml, check_path_length from msprobe.core.common.const import Const from msprobe.mindspore.common.const import Const as MsConst from msprobe.mindspore.common.const import FreeBenchmarkConst from msprobe.mindspore.free_benchmark.common.config import Config -from msprobe.core.common.file_utils import check_path_length from msprobe.mindspore.common.log import logger from msprobe.mindspore.debugger.debugger_config import DebuggerConfig from msprobe.mindspore.free_benchmark.decorator.decorator_factory import decorate_forward_function diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 3d9477cf7..f708152d7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,12 +1,11 @@ import os.path import torch from msprobe.core.common.const import FileCheckConst -from msprobe.core.common.file_utils import create_directory from msprobe.pytorch.common.log import logger from msprobe.core.common.exceptions import FileCheckException from msprobe.core.compare.acc_compare import Comparator from msprobe.core.common.utils import check_configuration_param, task_dumppath_get, check_compare_param, CompareException -from msprobe.core.common.file_utils import FileChecker +from msprobe.core.common.file_utils import FileChecker, create_directory from msprobe.pytorch.common.utils import load_pt diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index eecf51c60..ca3c66706 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -7,9 +7,8 @@ if int(torch.__version__.split('.')[0]) >= 2: from msprobe.pytorch.grad_probe.grad_stat_csv import GradStatCsv from msprobe.core.grad_probe.utils import check_numeral_list_ascend, data_in_list_target from msprobe.core.grad_probe.constant import level_adp -from msprobe.core.common.file_utils import create_directory from msprobe.pytorch.common.log import logger -from msprobe.core.common.file_utils import remove_path, save_npy, write_csv +from msprobe.core.common.file_utils import remove_path, save_npy, write_csv, create_directory from msprobe.pytorch.common.utils import get_rank_id, print_rank_0 -- Gitee From bf049b4b62b96873d76624622fc47c53223ef24e Mon Sep 17 00:00:00 2001 From: wangqingcai Date: Fri, 6 Sep 2024 09:48:55 +0800 Subject: [PATCH 710/791] add MSPROBE_LOG_LEVEL --- debug/accuracy_tools/msprobe/core/common/const.py | 2 +- debug/accuracy_tools/msprobe/core/common/log.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index d09f3641d..1203e90af 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -322,7 +322,7 @@ class MsgConst: Class for log messages const """ CLEAR_SYMBOL = "\033[K" - LOG_LEVEL = "LOG_LEVEL" + MSPROBE_LOG_LEVEL = "MSPROBE_LOG_LEVEL" LEVEL = ["INFO", "WARNING", "ERROR", "DEBUG"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index eaed07807..1afa2f92f 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -4,7 +4,7 @@ import sys from functools import wraps from msprobe.core.common.const import MsgConst -LOG_LEVEL = os.environ.get(MsgConst.LOG_LEVEL, "") +MSPROBE_LOG_LEVEL = os.environ.get(MsgConst.MSPROBE_LOG_LEVEL, "") class BaseLogger: @@ -37,7 +37,7 @@ class BaseLogger: @filter_special_chars def debug(self, msg): - if LOG_LEVEL == MsgConst.LEVEL[3]: + if MSPROBE_LOG_LEVEL == MsgConst.LEVEL[3]: self._print_log(MsgConst.LEVEL[3], msg) def on_rank_0(self, func): -- Gitee From 622ea57c51ce7c088f665c5b42cf45713313918e Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 6 Sep 2024 21:02:17 +0800 Subject: [PATCH 711/791] add range methed for config_step --- .../msprobe/core/common/utils.py | 1 - .../msprobe/core/common_config.py | 30 ++++++++++++++++--- .../mindspore/debugger/debugger_config.py | 6 ---- .../pytorch/debugger/debugger_config.py | 13 +------- 4 files changed, 27 insertions(+), 23 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 9eb3d88db..a2f126919 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -58,7 +58,6 @@ class CompareException(Exception): INVALID_TASK_ERROR = 20 DETACH_ERROR = 21 - def __init__(self, code, error_info: str = ""): super(CompareException, self).__init__() self.code = code diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index 688734be8..6eab2025c 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -8,7 +8,7 @@ class CommonConfig: self.task = json_config.get('task') self.dump_path = json_config.get('dump_path') self.rank = json_config.get('rank') - self.step = json_config.get('step') + self.step = self.get_real_step(json_config.get('step')) self.level = json_config.get('level') self.seed = json_config.get('seed') self.acl_config = json_config.get('acl_config') @@ -16,6 +16,31 @@ class CommonConfig: self.enable_dataloader = json_config.get('enable_dataloader', False) self._check_config() + @staticmethod + def get_step_from_string(step): + borderline = int(step.split('-')[0]), int(step.split('-')[-1]) + if borderline[0] <= borderline[1]: + continual_step = list(range(borderline[0], borderline[1] + 1)) + else: + continual_step = list(range(borderline[0], borderline[1] - 1, -1)) + return continual_step + + def get_real_step(self, step_input): + if step_input is not None and not isinstance(step_input, list): + logger.error_log_with_exp("step is invalid, it should be a list", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + real_step = [] + for step in step_input: + if not isinstance(step, (int, str)): + raise ValueError(f"step element {step} must be an integer or string.") + if isinstance(step, int) and step >= 0: + real_step.append(step) + elif isinstance(step, str) and '-' in step: + continual_step = self.get_step_from_string(step) + real_step.extend(continual_step) + real_step.sort() + return real_step + def _check_config(self): if self.task and self.task not in Const.TASK_LIST: logger.error_log_with_exp("task is invalid, it should be one of {}".format(Const.TASK_LIST), @@ -23,9 +48,6 @@ class CommonConfig: if self.rank is not None and not isinstance(self.rank, list): logger.error_log_with_exp("rank is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) - if self.step is not None and not isinstance(self.step, list): - logger.error_log_with_exp("step is invalid, it should be a list", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.level and self.level not in Const.LEVEL_LIST: logger.error_log_with_exp("level is invalid, it should be one of {}".format(Const.LEVEL_LIST), MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py index 974e420e8..f42ed2f1e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/debugger_config.py @@ -52,15 +52,9 @@ class DebuggerConfig: if not self.check_mode: self.check_mode = "all" self._check_rank() - self._check_step() return True def _check_rank(self): for rank_id in self.rank: if not isinstance(rank_id, int) or rank_id < 0: raise ValueError(f"rank {self.rank} must be a positive integer.") - - def _check_step(self): - for s in self.step: - if not isinstance(s, int) or s < 0: - raise ValueError(f"step element {s} must be a positive integer.") diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index 7c32be7cc..4f926de6e 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -1,5 +1,4 @@ from msprobe.pytorch.common import seed_all -from msprobe.pytorch.common.log import logger from msprobe.core.common.const import Const @@ -46,8 +45,7 @@ class DebuggerConfig: self.port = task_config.port if task_config.port else -1 self.check() - if self.step: - self.step.sort() + if self.level == "L2": if not self.scope or not isinstance(self.scope, list) or len(self.scope) != 1: raise ValueError("scope must be configured as a list with one api name") @@ -71,7 +69,6 @@ class DebuggerConfig: def check(self): self.check_kwargs() self._check_rank() - self._check_step() return True def check_model(self, model): @@ -85,11 +82,3 @@ class DebuggerConfig: for rank_id in self.rank: if not isinstance(rank_id, int) or rank_id < 0: raise ValueError(f"rank {self.rank} must be an integer and greater than or equal to 0.") - else: - logger.warning_on_rank_0(f"Rank argument is provided. Only rank {self.rank} data will be dumpped.") - - def _check_step(self): - if self.step: - for s in self.step: - if not isinstance(s, int) or s < 0: - raise ValueError(f"step element {s} must be an integer and greater than or equal to 0.") -- Gitee From a170bf6b7685b95820b2902fdbc687047805904a Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 6 Sep 2024 21:31:28 +0800 Subject: [PATCH 712/791] add try-except --- debug/accuracy_tools/msprobe/core/common_config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index 6eab2025c..a2ca27452 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -18,7 +18,13 @@ class CommonConfig: @staticmethod def get_step_from_string(step): - borderline = int(step.split('-')[0]), int(step.split('-')[-1]) + try: + borderline = int(step.split('-')[0]), int(step.split('-')[-1]) + except: + logger.error_log_with_exp( + "The connector(-) must start and end with decimal numbers.", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR) + ) if borderline[0] <= borderline[1]: continual_step = list(range(borderline[0], borderline[1] + 1)) else: -- Gitee From 638fb1aca41ff0522202b15f28b676a949efdbd3 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 6 Sep 2024 22:38:11 +0800 Subject: [PATCH 713/791] add try-except --- debug/accuracy_tools/msprobe/core/common_config.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index a2ca27452..d42d5aee7 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -20,11 +20,9 @@ class CommonConfig: def get_step_from_string(step): try: borderline = int(step.split('-')[0]), int(step.split('-')[-1]) - except: - logger.error_log_with_exp( - "The connector(-) must start and end with decimal numbers.", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR) - ) + except ValueError as e: + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, + "The connector(-) must start and end with decimal numbers.") from e if borderline[0] <= borderline[1]: continual_step = list(range(borderline[0], borderline[1] + 1)) else: @@ -67,7 +65,8 @@ class CommonConfig: logger.error_log_with_exp("enable_dataloader is invalid, it should be a boolean", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) - +xx = CommonConfig.get_step_from_string('6-10?') +print(xx) class BaseConfig: def __init__(self, json_config): self.scope = json_config.get('scope') -- Gitee From 0d462cf499e5966f05c8614fdffee59fd353a202 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 6 Sep 2024 22:47:11 +0800 Subject: [PATCH 714/791] add try-except --- debug/accuracy_tools/msprobe/core/common_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index d42d5aee7..8391f6afd 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -65,8 +65,7 @@ class CommonConfig: logger.error_log_with_exp("enable_dataloader is invalid, it should be a boolean", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) -xx = CommonConfig.get_step_from_string('6-10?') -print(xx) + class BaseConfig: def __init__(self, json_config): self.scope = json_config.get('scope') -- Gitee From ca6951a200e7cccdac7c7e4ad23e8106e6a18a70 Mon Sep 17 00:00:00 2001 From: makai Date: Fri, 6 Sep 2024 22:59:02 +0800 Subject: [PATCH 715/791] add try-except --- .../accuracy_tools/msprobe/pytorch/debugger/debugger_config.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index 4f926de6e..0ab9eedc5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -1,4 +1,5 @@ from msprobe.pytorch.common import seed_all +from msprobe.pytorch.common.log import logger from msprobe.core.common.const import Const @@ -82,3 +83,5 @@ class DebuggerConfig: for rank_id in self.rank: if not isinstance(rank_id, int) or rank_id < 0: raise ValueError(f"rank {self.rank} must be an integer and greater than or equal to 0.") + else: + logger.warning_on_rank_0(f"Rank argument is provided. Only rank {self.rank} data will be dumpped.") \ No newline at end of file -- Gitee From e7b52a1dda110dc5c9b3a30b43a03de2fb694e43 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 7 Sep 2024 09:54:11 +0800 Subject: [PATCH 716/791] modify config_intro. --- debug/accuracy_tools/msprobe/docs/02.config_introduction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md index 0b6a4a137..0e828ece3 100644 --- a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md +++ b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md @@ -15,7 +15,7 @@ | task | dump 的任务类型，str 类型。可取值，
"statistics"：仅 dump API 统计信息，默认值；
"tensor"：dump API 统计信息和完全复刻整网的 API 运行情况的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持。
"overflow_check"：溢出检测，仅 PyTorch 和 MindSpore 静态图场景支持；
"free_benchmark"：无标杆比对；
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)。
**配置示例**："task": "tensor"。 | 否 | | dump_path | 设置 dump 数据目录路径，str 类型。
**配置示例**："dump_path": "./dump_path"。MindSpore 场景仅支持绝对路径。 | 是 | | rank | 指定对某张卡上的数据进行 dump，list[int] 类型，默认未配置（表示 dump 所有卡的数据），应配置为 ≥0 的整数，且须配置实际可用的 Rank ID。

PyTorch 场景：Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，dump 数据为空。
**配置示例**："rank": [1]。
MindSpore 场景：所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
**配置示例**："rank": [1]。 | 否 | -| step | 指定 dump 某个 step 的数据，list[int] 类型。默认未配置，表示 dump 所有 step 数据。dump 特定 step 时，须指定为训练脚本中存在的 step。step 为 list 格式，可配置逐个 step。
**配置示例**："step": [0,1,2]。 | 否 | +| step | 指定 dump 某个 step 的数据，list[int] 类型。默认未配置，表示 dump 所有 step 数据。dump 特定 step 时，须指定为训练脚本中存在的 step。step 为 list 格式，可配置逐个 step，也支持输入范围。
**配置示例**："step": [0,1,2, "5-8"]。 | 否 | | level | dump 级别，str 类型，根据不同级别 dump 不同数据。可取值，
"L0"：dump 模块级精度数据，仅 PyTorch 与 MindSpore 动态图场景支持，使用背景详见 [1.1.1 模块级精度数据 dump 说明](#111-模块级精度数据-dump-说明)；
"L1"：dump API 级精度数据，默认值，仅 PyTorch 与 MindSpore 动态图场景支持；
"L2"：dump kernel 级精度数据，PyTorch场景下须配置 acl_config 参数；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 场景支持。
**配置示例**："level": "L1"。 | 否 | | acl_config | kernel dump 的配置文件，str 类型。level 取"L2"时，该参数必要；level 为其他值时，该参数不选。
**配置示例**：acl_config="./acl_config.json"。具体配置见[ acl_config 示例](./04.acl_config_examples.md)。 | 否 | | seed | 随机种子数，int 类型，默认值为：1234，仅 PyTorch 场景支持。通过固定随机数保证模型的输入或输出一致，可固定的随机数详见 [1.1.2 固定随机数范围（仅 PyTorch 场景支持）](#112-固定随机数范围仅-pytorch-场景支持)。
**配置示例**："seed": 1234。 | 否 | -- Gitee From 59b4b21345b4687855034af203b2f04a689ff49b Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 7 Sep 2024 11:46:31 +0800 Subject: [PATCH 717/791] modify test_common_config --- .../msprobe/core/common/exceptions.py | 4 ++-- .../msprobe/core/common_config.py | 7 ++++--- .../pytorch/debugger/debugger_config.py | 2 +- .../test/core_ut/test_common_config.py | 20 ++++++++++--------- 4 files changed, 18 insertions(+), 15 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/exceptions.py b/debug/accuracy_tools/msprobe/core/common/exceptions.py index 5f67686d0..8b8b9eb94 100644 --- a/debug/accuracy_tools/msprobe/core/common/exceptions.py +++ b/debug/accuracy_tools/msprobe/core/common/exceptions.py @@ -13,8 +13,8 @@ class MsprobeException(CodedException): OVERFLOW_NUMS_ERROR = 1 err_strs = { - INVALID_PARAM_ERROR: "[msprobe] 无效参数： ", - OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数:" + INVALID_PARAM_ERROR: "[msprobe] 无效参数：", + OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数：" } diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index 8391f6afd..6fddada5c 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -30,13 +30,14 @@ class CommonConfig: return continual_step def get_real_step(self, step_input): + if step_input is None: + return [] if step_input is not None and not isinstance(step_input, list): - logger.error_log_with_exp("step is invalid, it should be a list", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "step is invalid, it should be a list") real_step = [] for step in step_input: if not isinstance(step, (int, str)): - raise ValueError(f"step element {step} must be an integer or string.") + raise ValueError(f"step element {step} must be an integer or string.") if isinstance(step, int) and step >= 0: real_step.append(step) elif isinstance(step, str) and '-' in step: diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index 0ab9eedc5..88853dc28 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -84,4 +84,4 @@ class DebuggerConfig: if not isinstance(rank_id, int) or rank_id < 0: raise ValueError(f"rank {self.rank} must be an integer and greater than or equal to 0.") else: - logger.warning_on_rank_0(f"Rank argument is provided. Only rank {self.rank} data will be dumpped.") \ No newline at end of file + logger.warning_on_rank_0(f"Rank argument is provided. Only rank {self.rank} data will be dumpped.") diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index 8b2138a48..e0e95df82 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -from unittest import TestCase +from unittest import TestCase, main from unittest.mock import patch from msprobe.core.common.log import logger @@ -32,7 +32,7 @@ class TestCommonConfig(TestCase): self.assertIsNone(common_config.task) self.assertIsNone(common_config.dump_path) self.assertIsNone(common_config.rank) - self.assertIsNone(common_config.step) + self.assertEqual(common_config.step, []) self.assertIsNone(common_config.level) self.assertIsNone(common_config.seed) self.assertIsNone(common_config.acl_config) @@ -56,13 +56,13 @@ class TestCommonConfig(TestCase): json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) - json_config.update({"step": 0}) - CommonConfig(json_config) - self.assertEqual(mock_error_log_with_exp.call_args[0][0], - "step is invalid, it should be a list") - self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), - MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) - + json_config = {"step": "invalid_step"} # 传入一个非列表的 step + with self.assertRaises(MsprobeException) as cm: + CommonConfig(json_config) + the_exception = cm.exception + self.assertEqual(the_exception.code, MsprobeException.INVALID_PARAM_ERROR) + self.assertEqual(str(the_exception), "[msprobe] 无效参数：step is invalid, it should be a list") + json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) json_config.update({"step": [0]}) @@ -150,3 +150,5 @@ class TestCommonConfig(TestCase): "data_mode is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) + +main() \ No newline at end of file -- Gitee From 6edc6f4b7923580446ad85f25f256d5c15d2d139 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 7 Sep 2024 12:23:23 +0800 Subject: [PATCH 718/791] modify test_common_config --- .../msprobe/test/core_ut/test_common_config.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index e0e95df82..b877a576c 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -from unittest import TestCase, main +from unittest import TestCase from unittest.mock import patch from msprobe.core.common.log import logger @@ -61,7 +61,9 @@ class TestCommonConfig(TestCase): CommonConfig(json_config) the_exception = cm.exception self.assertEqual(the_exception.code, MsprobeException.INVALID_PARAM_ERROR) - self.assertEqual(str(the_exception), "[msprobe] 无效参数：step is invalid, it should be a list") + self.assertEqual(str(the_exception), + f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}\ + step is invalid, it should be a list") json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) @@ -150,5 +152,3 @@ class TestCommonConfig(TestCase): "data_mode is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) - -main() \ No newline at end of file -- Gitee From f95e974e4761f0e2a2c0a32d1ae091ee8aa136bc Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 7 Sep 2024 12:26:46 +0800 Subject: [PATCH 719/791] modify test_common_config --- debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index b877a576c..2f476aba1 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -56,7 +56,7 @@ class TestCommonConfig(TestCase): json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) - json_config = {"step": "invalid_step"} # 传入一个非列表的 step + json_config.update({"step": "invalid_step"}) with self.assertRaises(MsprobeException) as cm: CommonConfig(json_config) the_exception = cm.exception -- Gitee From 99d091fcdde21c8c85cdb9ec17bea2619fda69c0 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 7 Sep 2024 12:39:05 +0800 Subject: [PATCH 720/791] modify test_common_config --- .../msprobe/test/core_ut/test_common_config.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index 2f476aba1..275a8d757 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -from unittest import TestCase +from unittest import TestCase,main from unittest.mock import patch from msprobe.core.common.log import logger @@ -62,8 +62,18 @@ class TestCommonConfig(TestCase): the_exception = cm.exception self.assertEqual(the_exception.code, MsprobeException.INVALID_PARAM_ERROR) self.assertEqual(str(the_exception), - f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}\ - step is invalid, it should be a list") + f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}step is invalid, it should be a list") + + json_config.update({"step": [0, 1, "4---6?"]}) + with self.assertRaises(MsprobeException) as cm: + CommonConfig(json_config) + the_exception = cm.exception + self.assertEqual(str(the_exception), + f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}The connector(-) must start and end with decimal numbers.") + + json_config.update({"step": [0, 1, "4---6"]}) + common_config = CommonConfig(json_config) + self.assertEqual(common_config.step, [0, 1, 4, 5, 6]) json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) @@ -152,3 +162,4 @@ class TestCommonConfig(TestCase): "data_mode is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) +main() \ No newline at end of file -- Gitee From 72736e3e32cdc5bc94a114153f554d329c393601 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 7 Sep 2024 12:53:21 +0800 Subject: [PATCH 721/791] modify test_common_config --- .../accuracy_tools/msprobe/test/core_ut/test_common_config.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index 275a8d757..ac8bb84d2 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -14,7 +14,7 @@ # See the License for the specific language governing permissions and # limitations under the License. """ -from unittest import TestCase,main +from unittest import TestCase from unittest.mock import patch from msprobe.core.common.log import logger @@ -162,4 +162,3 @@ class TestCommonConfig(TestCase): "data_mode is invalid, it should be a list") self.assertEqual(str(mock_error_log_with_exp.call_args[0][1]), MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)) -main() \ No newline at end of file -- Gitee From abf317755d05a17901586c7da2ca386352ff4974 Mon Sep 17 00:00:00 2001 From: makai Date: Sat, 7 Sep 2024 13:06:17 +0800 Subject: [PATCH 722/791] modify test_common_config --- .../msprobe/test/mindspore_ut/test_primitive_dump.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py index 23c0b5ad1..2b9c16f62 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/test_primitive_dump.py @@ -69,7 +69,7 @@ class TestService(unittest.TestCase): self.service.check_model_valid(model) # For the purpose of the test, let's also verify the expected exception message - expected_message = "[msprobe] 无效参数： model 参数必须是 mindspore.nn.Cell 类型。" + expected_message = f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}model 参数必须是 mindspore.nn.Cell 类型。" self.assertEqual(str(context.exception), expected_message) def test_update_primitive_counters(self): -- Gitee From 79c317ee4374e795a3de29359188acdf499afe49 Mon Sep 17 00:00:00 2001 From: l30036321 Date: Mon, 9 Sep 2024 10:39:12 +0800 Subject: [PATCH 723/791] check input params type --- .../accuracy_tools/msprobe/core/common_config.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index 688734be8..e4b2a260a 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -1,6 +1,7 @@ -from msprobe.core.common.const import Const +from msprobe.core.common.const import Const, FileCheckConst from msprobe.core.common.log import logger from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common.file_utils import FileChecker class CommonConfig: @@ -20,6 +21,9 @@ class CommonConfig: if self.task and self.task not in Const.TASK_LIST: logger.error_log_with_exp("task is invalid, it should be one of {}".format(Const.TASK_LIST), MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + if self.dump_path is not None and not isinstance(self.dump_path, str): + logger.error_log_with_exp("dump_path is invalid, it should be a string", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) if self.rank is not None and not isinstance(self.rank, list): logger.error_log_with_exp("rank is invalid, it should be a list", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) @@ -38,6 +42,16 @@ class CommonConfig: if not isinstance(self.enable_dataloader, bool): logger.error_log_with_exp("enable_dataloader is invalid, it should be a boolean", MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + if self.acl_config: + self._check_acl_config() + + def _check_acl_config(self): + if not isinstance(self.acl_config, str): + logger.error_log_with_exp("acl_config is invalid, it should be a string", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR)) + file_checker = FileChecker( + file_path=self.acl_config, path_type=FileCheckConst.FILE, file_type=FileCheckConst.JSON_SUFFIX) + file_checker.common_check() class BaseConfig: -- Gitee From e0dc42a3f5d603b3bc4d71c82ee8aacb147f1f26 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 11:30:03 +0800 Subject: [PATCH 724/791] renew --- debug/accuracy_tools/msprobe/core/common_config.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index 6fddada5c..4d05a891d 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -32,12 +32,13 @@ class CommonConfig: def get_real_step(self, step_input): if step_input is None: return [] - if step_input is not None and not isinstance(step_input, list): + if not isinstance(step_input, list): raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "step is invalid, it should be a list") real_step = [] for step in step_input: if not isinstance(step, (int, str)): - raise ValueError(f"step element {step} must be an integer or string.") + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, + f"step element {step} must be an integer or string.") if isinstance(step, int) and step >= 0: real_step.append(step) elif isinstance(step, str) and '-' in step: -- Gitee From 5b7d80e941c5e40e3c45eaa57db058a0a7bcb897 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 11:46:55 +0800 Subject: [PATCH 725/791] renew --- debug/accuracy_tools/msprobe/core/common_config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index 4d05a891d..0c8619bbc 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -20,7 +20,7 @@ class CommonConfig: def get_step_from_string(step): try: borderline = int(step.split('-')[0]), int(step.split('-')[-1]) - except ValueError as e: + except (ValueError, IndexError) as e: raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "The connector(-) must start and end with decimal numbers.") from e if borderline[0] <= borderline[1]: -- Gitee From 05fc7e04c55219e24a307f30fc80452d932af316 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Fri, 6 Sep 2024 16:44:35 +0800 Subject: [PATCH 726/791] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=96=99=E3=80=91?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E6=BA=A2=E5=87=BA=E6=A3=80=E6=B5=8B=E4=B8=8E?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E6=A8=A1=E5=9D=97=E7=9A=84=E8=B5=84=E6=96=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/README.md | 4 +- .../msprobe/docs/12.overflow_check_PyTorch.md | 76 ++++++++++++++++--- .../docs/13.overflow_check_MindSpore.md | 11 +++ 3 files changed, 78 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 802f1e857..2ee5e9c63 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -55,9 +55,9 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. 溢出检测与解析是在执行精度数据 dump 时，判断是否存在输入正常但输出存在溢出的 API，从而判断是否为正常溢出。对应 config.json 中的 overflow_check。 -[PyTorch 场景的溢出检测与解析](./docs/12.overflow_check_PyTorch.md)（更新中） +[PyTorch 场景的溢出检测与解析](./docs/12.overflow_check_PyTorch.md) -[MindSpore 场景的溢出检测与解析](./docs/13.overflow_check_MindSpore.md)（待补充） +[MindSpore 场景的溢出检测与解析](./docs/13.overflow_check_MindSpore.md) ### 5 数据解析 diff --git a/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md b/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md index e633a3f24..a63c5f677 100644 --- a/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md @@ -1,25 +1,79 @@ -# 溢出解析工具（更新中） +# **PyTorch场景下的溢出检测** -针对训练过程中的溢出检测场景（当《[精度数据采集](./dump.md)》开启溢出检测dump时），对于输入正常但输出存在溢出的API，会在训练执行目录下将溢出的API信息按照前向和反向分类，dump并保存为`dump.json`，前向过程溢出的API可通过该工具对`dump.json`进行解析，输出溢出API为正常溢出还是非正常溢出，从而帮助用户快速判断。 +msprobe 工具在 PyTorch 场景下提供溢出数据采集功能和溢出数据解析功能，详细介绍见下方章节。 -工具支持PyTorch版本：1.11/2.0/2.1/2.2。 +## 1、溢出数据采集 + +溢出数据采集能 dump 所有溢出的 API 或者 Module 的信息，包括前反向输入输出的统计值和实际运行数据信息。 + +### 1.1 工具安装 + +工具安装详见《[msprobe 使用手册](../README.md)》的“安装”章节 + +### 1.2 接口介绍 + +溢出检测功能提供的接口与数据采集任务一致，详见[ PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)中的"**1 接口介绍**"章节。 +其中 PrecisionDebugger 中的 task 或是 config.json 中的 task 需要指定为 **overflow_check**，详见[配置文件介绍](./02.config_introduction.md)中的 +"**1.1 通用配置介绍**"和"**1.5 task 配置为 overflow_check**"章节。 + +### 1.3 示例代码 + +溢出检测功能使用方式与数据采集任务一致，详见[ PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)中的"**2 示例代码**"章节。 + +### 1.4 结果文件介绍 + +溢出检测结果文件目录结构与含义与数据采集任务一致，详见[ PyTorch 场景的精度数据采集](./05.data_dump_PyTorch.md)中的"**3 dump 结果文件介绍**"章节。 + +### 1.5 其他说明 + +溢出数据采集功能在昇腾 NPU 上支持饱和模式和 INF/NAN 模式。INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品（如 910A），默认为饱和模式，且不建议使用 INF/NAN 模式；Atlas A2 训练系列产品（如 910B），默认为 INF/NAN 模式，且不建议使用饱和模式。 + +INF/NAN 模式的使能方式如下： + +```Shell +# 使能 CANN 侧 INF/NAN 模式 +export INF_NAN_MODE_ENABLE=1 +``` + +## 2、溢出数据解析 + +注：溢出数据解析目前只支持分析前向 API 输入未溢出，但输出溢出的场景。 + +在第一步中采集到溢出数据后，可以使用溢出解析功能快速判断输入未溢出而输出溢出的 API 是正常溢出还是非正常溢出，从而帮您快速分析是否由 API 执行异常导致的溢出问题。 +同一个 API 用相同的输入数据在 CPU 设备和昇腾 NPU 设备上执行，若 CPU 和 NPU 均出现溢出，则认为该溢出是**正常溢出**；若 NPU 有溢出, 而 CPU 没有溢出则认为该溢出是**非正常溢出**。 + +工具支持的 PyTorch 版本：1.11/2.0/2.1/2.2。 操作步骤如下： 1. 安装工具。 - 详见《[MindStudio精度调试工具](../../README.md)》的“工具安装”章节。 + 详见《[msprobe 使用手册](../README.md)》的“安装”章节。 -2. 执行溢出API解析操作。 +2. 执行溢出 API 解析操作。 ```bash msprobe -f pytorch run_overflow_check -api_info ./dump.json ``` -| 参数名称 | 说明 | 是否必选 | -| -------------------------- | -------------------------------------------------- | -------- | -| -api_info或--api_info_file | 指定API信息文件dump.json。 | 是 | -| -j或--jit_compile | 开启jit编译。 | 否 | -| -d或--device | 指定Device ID，选择UT代码运行所在的卡，默认值为0。 | 否 | +| 参数名称 | 说明 | 是否必选 | +| -------------------------- |------------------------------------| -------- | +| -api_info或--api_info_file | 指定采集下来的 API 信息文件 dump.json。 | 是 | +| -j或--jit_compile | 开启 jit 编译。 | 否 | +| -d或--device | 指定 Device ID，选择 UT 代码运行所在的卡，默认值为0。 | 否 | + +3. 结果说明。 + + 如果 API 是**正常溢出**，会打印以下信息： + + ```bash + The {api_name} overflow is a normal overflow. + ``` + + **非正常溢出**则会打印以下信息： + + ```bash + The {api_name} overflow is an abnormal overflow. + ``` -反向过程溢出的API暂不支持该功能。 + 其中 {api_name} 是对应溢出的 API 名称。 diff --git a/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md b/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md index 2d8eb0d53..79a332627 100644 --- a/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md @@ -2,6 +2,17 @@ msprobe 工具提供静态图O2编译等级下的过程溢出检测与动态图场景下的结果溢出检测。其中前者检测对象为 kernel 级别，对应 config.json 配置中的 "L2" level，后者检测对象为 API 级别，对应 config.json 配置中的 "L1" level。 +需要注意，动态图场景下的溢出检测功能仅支持 INF/NAN 模式a。INF/NAN 模式的使能方式如下： + +```Shell +# 使能 CANN 侧 INF/NAN 模式 +export INF_NAN_MODE_ENABLE=1 +# 使能 MindSpore 框架侧 INF/NAN 模式 +export MS_ASCEND_CHECK_OVERFLOW_MODE="INFNAN_MODE" +``` + +**a**：INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品（如 910A），默认为饱和模式，且不建议使用 INF/NAN 模式；Atlas A2训练系列产品（如 910B），默认为 INF/NAN 模式，且不建议使用饱和模式。对于 MindSpore 框架侧配置，仅支持对 Atlas A2 训练系列产品进行设置，默认为 INF/NAN 模式。CANN 侧 与 MindSpore 框架侧配置须一致。 + 溢出检测任务的配置示例见[MindSpore 静态图场景下 task 配置为 overflow_check](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#23-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-overflow_check)、[MindSpore 动态图场景下 task 配置为 overflow_check](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#33-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-overflow_check)。 -- Gitee From 8a42bac0587dcf11db5e818569266e6078ad0382 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 14:31:49 +0800 Subject: [PATCH 727/791] set log level --- .../msprobe/core/common/const.py | 3 +- .../accuracy_tools/msprobe/core/common/log.py | 46 ++++++++++--------- .../msprobe/core/data_dump/data_collector.py | 4 +- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 4f686598a..d61fa8ea8 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -326,9 +326,8 @@ class MsgConst: """ Class for log messages const """ - CLEAR_SYMBOL = "\033[K" MSPROBE_LOG_LEVEL = "MSPROBE_LOG_LEVEL" - LEVEL = ["INFO", "WARNING", "ERROR", "DEBUG"] + LEVEL = ["SKIP_PRINT", "ERROR", "WARNING", "INFO", "DEBUG"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 1afa2f92f..ce6c3f9f9 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -4,42 +4,54 @@ import sys from functools import wraps from msprobe.core.common.const import MsgConst -MSPROBE_LOG_LEVEL = os.environ.get(MsgConst.MSPROBE_LOG_LEVEL, "") - class BaseLogger: def __init__(self): self.rank = None + self.level = int(os.environ.get(MsgConst.MSPROBE_LOG_LEVEL, "3")) + def _print_log(self, level, msg, end='\n'): + current_rank = self.get_rank() + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + pid = os.getpid() + if current_rank is not None: + full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}" + else: + full_msg = f"{current_time} ({pid}) [{level}] {msg}" + print(full_msg, end=end) + sys.stdout.flush() def get_rank(self): return self.rank def filter_special_chars(func): @wraps(func) - def func_level(self, msg, **kwargs): + def func_level(self, msg): for char in MsgConst.SPECIAL_CHAR: msg = msg.replace(char, '_') - return func(self, msg, **kwargs) + return func(self, msg) return func_level - @filter_special_chars - def info(self, msg, **kwargs): - self._print_log(MsgConst.LEVEL[0], msg, **kwargs) - @filter_special_chars def error(self, msg): - self._print_log(MsgConst.LEVEL[2], msg) + if self.level >= MsgConst.LEVEL.index(MsgConst.LEVEL[1]): + self._print_log(MsgConst.LEVEL[1], msg) @filter_special_chars def warning(self, msg): - self._print_log(MsgConst.LEVEL[1], msg) + if self.level >= MsgConst.LEVEL.index(MsgConst.LEVEL[2]): + self._print_log(MsgConst.LEVEL[2], msg) @filter_special_chars - def debug(self, msg): - if MSPROBE_LOG_LEVEL == MsgConst.LEVEL[3]: + def info(self, msg): + if self.level >= MsgConst.LEVEL.index(MsgConst.LEVEL[3]): self._print_log(MsgConst.LEVEL[3], msg) + @filter_special_chars + def debug(self, msg): + if self.level >= MsgConst.LEVEL.index(MsgConst.LEVEL[4]): + self._print_log(MsgConst.LEVEL[4], msg) + def on_rank_0(self, func): def func_rank_0(*args, **kwargs): current_rank = self.get_rank() @@ -62,15 +74,5 @@ class BaseLogger: self.error(msg) raise exception - def _print_log(self, level, msg, end='\n'): - current_rank = self.get_rank() - current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) - pid = os.getpid() - if current_rank is not None: - full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}" - else: - full_msg = f"{current_time} ({pid}) [{level}] {msg}" - print(full_msg, end=end) - sys.stdout.flush() logger = BaseLogger() diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index ba7787489..40178deb7 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -3,7 +3,7 @@ import os from msprobe.core.data_dump.scope import build_scope, ListScope from msprobe.core.data_dump.json_writer import DataWriter from msprobe.core.common.log import logger -from msprobe.core.common.const import Const, MsgConst +from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.factory import DataProcessorFactory @@ -130,7 +130,7 @@ class DataCollector: if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) - logger.info(MsgConst.CLEAR_SYMBOL + msg, end='\r') + logger.debug(msg) if use_buffer: self.data_writer.flush_data_when_buffer_is_full() else: -- Gitee From 322a72d1486ee4dbb27230f4bfb2e0557509ce44 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 14:37:05 +0800 Subject: [PATCH 728/791] set log level --- .../accuracy_tools/msprobe/core/common/log.py | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index ce6c3f9f9..af25936dc 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -10,17 +10,6 @@ class BaseLogger: self.rank = None self.level = int(os.environ.get(MsgConst.MSPROBE_LOG_LEVEL, "3")) - def _print_log(self, level, msg, end='\n'): - current_rank = self.get_rank() - current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) - pid = os.getpid() - if current_rank is not None: - full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}" - else: - full_msg = f"{current_time} ({pid}) [{level}] {msg}" - print(full_msg, end=end) - sys.stdout.flush() - def get_rank(self): return self.rank @@ -74,5 +63,16 @@ class BaseLogger: self.error(msg) raise exception + def _print_log(self, level, msg, end='\n'): + current_rank = self.get_rank() + current_time = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime()) + pid = os.getpid() + if current_rank is not None: + full_msg = f"{current_time} ({pid}) [rank {current_rank}] [{level}] {msg}" + else: + full_msg = f"{current_time} ({pid}) [{level}] {msg}" + print(full_msg, end=end) + sys.stdout.flush() + logger = BaseLogger() -- Gitee From 8ab885f11808d8dc9ee1971d6f7d987b29c7280d Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 16:00:10 +0800 Subject: [PATCH 729/791] set log level --- debug/accuracy_tools/msprobe/core/common/const.py | 1 + debug/accuracy_tools/msprobe/core/common/log.py | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index d61fa8ea8..427ec2f9c 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -327,6 +327,7 @@ class MsgConst: Class for log messages const """ MSPROBE_LOG_LEVEL = "MSPROBE_LOG_LEVEL" + LEVEL_ENUM = ["0", "1", "2", "3", "4"] LEVEL = ["SKIP_PRINT", "ERROR", "WARNING", "INFO", "DEBUG"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index af25936dc..5509156a1 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -8,7 +8,14 @@ from msprobe.core.common.const import MsgConst class BaseLogger: def __init__(self): self.rank = None - self.level = int(os.environ.get(MsgConst.MSPROBE_LOG_LEVEL, "3")) + self.level = self.get_level() + + def get_level(self): + input_level = os.environ.get(MsgConst.MSPROBE_LOG_LEVEL) + if input_level in MsgConst.LEVEL_ENUM: + return int(input_level) + else: + return 3 def get_rank(self): return self.rank -- Gitee From 6c3c258bb26d46a9d73f9e9664872916c6c93ec9 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 16:26:26 +0800 Subject: [PATCH 730/791] set log level --- debug/accuracy_tools/msprobe/core/common/log.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 5509156a1..2fadda511 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -10,7 +10,8 @@ class BaseLogger: self.rank = None self.level = self.get_level() - def get_level(self): + @staticmethod + def get_level(): input_level = os.environ.get(MsgConst.MSPROBE_LOG_LEVEL) if input_level in MsgConst.LEVEL_ENUM: return int(input_level) -- Gitee From b4346888b1a1ec7c0d26a1c3fe2c2d513b247e6d Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 16:31:49 +0800 Subject: [PATCH 731/791] set log level --- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 4282d8ffd..5a50b8602 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -25,7 +25,6 @@ from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.data_dump.json_writer import DataWriter from msprobe.core.data_dump.data_processor.base import BaseDataProcessor from msprobe.core.data_dump.data_processor.pytorch_processor import StatisticsDataProcessor -from msprobe.core.common.const import MsgConst class TestDataCollector(unittest.TestCase): @@ -76,7 +75,7 @@ class TestDataCollector(unittest.TestCase): msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - mock_info.assert_called_with(MsgConst.CLEAR_SYMBOL + "msg", end='\r') + mock_info.assert_called_with("msg") mock_flush.assert_called() mock_write_json.assert_not_called() -- Gitee From 35590dcdeb3db194d823a2c2b02b2d589aafb2ba Mon Sep 17 00:00:00 2001 From: zhaolei Date: Mon, 9 Sep 2024 16:35:42 +0800 Subject: [PATCH 732/791] =?UTF-8?q?mstt=E4=BE=9D=E8=B5=96=E5=AF=BC?= =?UTF-8?q?=E5=85=A5=E9=94=99=E8=AF=AFbug=E8=A7=A3=E5=86=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../compare_backend/compare_bean/kernel_compare_bean.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py b/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py index 630f79341..06d978397 100644 --- a/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py +++ b/profiler/compare_tools/compare_backend/compare_bean/kernel_compare_bean.py @@ -1,7 +1,6 @@ -from compare_backend.utils.common_func import calculate_diff_ratio +from compare_backend.utils.common_func import calculate_diff_ratio, convert_to_float from compare_backend.utils.constant import Constant from compare_backend.utils.excel_config import ExcelConfig -from profiler.advisor.utils.utils import convert_to_float class KernelCompareInfo: -- Gitee From 2fd0c2694959be8c2843a7ad69aec18360d344fe Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 16:50:20 +0800 Subject: [PATCH 733/791] set log level --- .../test/core_ut/data_dump/test_data_collector.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 5a50b8602..4a90efe61 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -69,22 +69,22 @@ class TestDataCollector(unittest.TestCase): def test_handle_data(self): with patch.object(DataCollector, "update_data", return_value="msg") as mock_update_data, \ patch.object(DataCollector, "write_json") as mock_write_json, \ - patch("msprobe.core.data_dump.data_collector.logger.info") as mock_info, \ + patch("msprobe.core.data_dump.data_collector.logger.debug") as mock_debug, \ patch("msprobe.core.data_dump.json_writer.DataWriter.flush_data_when_buffer_is_full") as mock_flush: self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - mock_info.assert_called_with("msg") + mock_debug.assert_called_with("msg") mock_flush.assert_called() mock_write_json.assert_not_called() mock_update_data.reset_mock() - mock_info.reset_mock() + mock_debug.reset_mock() mock_flush.reset_mock() self.data_collector.handle_data("Tensor.add", {}, use_buffer=False) mock_update_data.assert_not_called() - mock_info.assert_not_called() + mock_debug.assert_not_called() mock_write_json.assert_called() @patch.object(DataCollector, "update_construct") @@ -117,3 +117,5 @@ class TestDataCollector(unittest.TestCase): self.data_collector.backward_data_collect("name", "module", "pid", "module_input_output") mock_handle_data.assert_called_with("name", {}) + +unittest.main() -- Gitee From e9f5aa81ae8b4566f2c76208d79a5074fc858fb3 Mon Sep 17 00:00:00 2001 From: makai Date: Mon, 9 Sep 2024 16:55:15 +0800 Subject: [PATCH 734/791] set log level --- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 4a90efe61..ccf249ce1 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -117,5 +117,3 @@ class TestDataCollector(unittest.TestCase): self.data_collector.backward_data_collect("name", "module", "pid", "module_input_output") mock_handle_data.assert_called_with("name", {}) - -unittest.main() -- Gitee From f003fd67924436a6fce2cd057e5e4a19c498ed23 Mon Sep 17 00:00:00 2001 From: CSNIU Date: Mon, 9 Sep 2024 21:11:15 +0800 Subject: [PATCH 735/791] =?UTF-8?q?=E5=A4=96=E9=83=A8=E8=BE=93=E5=85=A5?= =?UTF-8?q?=E6=8E=92=E6=9F=A5=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/core/common/utils.py | 4 ++-- .../mindspore/compare/distributed_compare.py | 2 +- .../msprobe/mindspore/compare/ms_compare.py | 14 +++++++------- .../msprobe/pytorch/compare/distributed_compare.py | 2 +- .../msprobe/pytorch/compare/pt_compare.py | 2 +- .../msprobe/test/core_ut/common/test_utils.py | 2 +- 6 files changed, 13 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index cf62e6d8b..95cd3a4d6 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -156,8 +156,8 @@ def check_compare_param(input_param, output_path, summary_compare=False, md5_com -def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False): - if not (isinstance(stack_mode, bool) and isinstance(auto_analyze, bool) and isinstance(fuzzy_match, bool)): +def check_configuration_param(stack_mode=False, auto_analyze=True, fuzzy_match=False, is_print_compare_log=True): + if not (isinstance(stack_mode, bool) and isinstance(auto_analyze, bool) and isinstance(fuzzy_match, bool) and isinstance(is_print_compare_log, bool)): logger.error("Invalid input parameters which should be only bool type.") raise CompareException(CompareException.INVALID_PARAM_ERROR) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py index 073969862..167df1e18 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/distributed_compare.py @@ -54,7 +54,7 @@ def ms_compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): } try: summary_compare, md5_compare = task_dumppath_get(dump_result_param) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match, dump_result_param.get('is_print_compare_log', True)) create_directory(output_path) check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) except (CompareException, FileCheckException) as error: diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index ddbbe6cfe..7a38c6282 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -187,13 +187,13 @@ class MSComparator(Comparator): bench_struct_out = del_bench_dict.get(CompareConst.OUTPUT_STRUCT, []) bench_summary = del_bench_dict.get(CompareConst.SUMMARY, []) for idx in input_sub_list: # Fill in the blank value field in the pt dictionary - bench_op_name.insert(idx, CompareConst.NAN) - bench_struct_in.insert(idx, CompareConst.NAN) - bench_summary.insert(idx, CompareConst.NAN) + bench_op_name.insert(idx, CompareConst.N_A) + bench_struct_in.insert(idx, CompareConst.N_A) + bench_summary.insert(idx, CompareConst.N_A) for idx in output_sub_list: # Fill in the blank value field in the pt dictionary - bench_op_name.insert(npu_in_len + idx, CompareConst.NAN) - bench_struct_out.insert(idx, CompareConst.NAN) - bench_summary.insert(npu_in_len + idx, CompareConst.NAN) + bench_op_name.insert(npu_in_len + idx, CompareConst.N_A) + bench_struct_out.insert(idx, CompareConst.N_A) + bench_summary.insert(npu_in_len + idx, CompareConst.N_A) del_bench_dict.update({CompareConst.OP_NAME: bench_op_name, CompareConst.INPUT_STRUCT: bench_struct_in, CompareConst.OUTPUT_STRUCT: bench_struct_out, CompareConst.SUMMARY: bench_summary}) return del_bench_dict @@ -206,7 +206,7 @@ def ms_compare(input_param, output_path, **kwargs): cell_mapping = kwargs.get('cell_mapping', None) api_mapping = kwargs.get('api_mapping', None) summary_compare, md5_compare = task_dumppath_get(input_param) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match, input_param.get('is_print_compare_log', True)) create_directory(output_path) check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index bdabd0ee7..304322688 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -55,7 +55,7 @@ def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): } try: summary_compare, md5_compare = task_dumppath_get(dump_result_param) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match, dump_result_param.get('is_print_compare_log', True)) create_directory(output_path) check_compare_param(dump_result_param, output_path, summary_compare=summary_compare, md5_compare=md5_compare) except (CompareException, FileCheckException) as error: diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 17730d742..a3138e02d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -28,7 +28,7 @@ class PTComparator (Comparator): def compare(input_param, output_path, stack_mode=False, auto_analyze=True, fuzzy_match=False): try: summary_compare, md5_compare = task_dumppath_get(input_param) - check_configuration_param(stack_mode, auto_analyze, fuzzy_match) + check_configuration_param(stack_mode, auto_analyze, fuzzy_match, input_param.get('is_print_compare_log', True)) create_directory(output_path) check_compare_param(input_param, output_path, summary_compare, md5_compare) except (CompareException, FileCheckException) as error: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index c923b9179..cbcd8926d 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -213,7 +213,7 @@ class TestUtils(TestCase): @patch.object(logger, "error") def test_check_configuration_param(self, mock_error): with self.assertRaises(CompareException) as context: - check_configuration_param(stack_mode="False", auto_analyze=True, fuzzy_match=False) + check_configuration_param(stack_mode="False", auto_analyze=True, fuzzy_match=False, is_print_compare_log=True) self.assertEqual(context.exception.code, CompareException.INVALID_PARAM_ERROR) mock_error.assert_called_with("Invalid input parameters which should be only bool type.") -- Gitee From fcbed25bb365549063ff49b39b20ea81b1d82d40 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Fri, 6 Sep 2024 11:00:25 +0800 Subject: [PATCH 736/791] Add readme for free benchmark --- .../docs/15.free_benchmarking_PyTorch.md | 164 ++++++++++++++++++ .../docs/img/free_benchmark_framework.png | Bin 0 -> 259617 bytes 2 files changed, 164 insertions(+) create mode 100644 debug/accuracy_tools/msprobe/docs/15.free_benchmarking_PyTorch.md create mode 100644 debug/accuracy_tools/msprobe/docs/img/free_benchmark_framework.png diff --git a/debug/accuracy_tools/msprobe/docs/15.free_benchmarking_PyTorch.md b/debug/accuracy_tools/msprobe/docs/15.free_benchmarking_PyTorch.md new file mode 100644 index 000000000..ecc214441 --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/15.free_benchmarking_PyTorch.md @@ -0,0 +1,164 @@ +# PyTorch 场景的无标杆比对 + +## 1 简介 +* 本工具的目标是在不依赖标杆数据的情况下、检测模型训练中可能存在的精度问题API级别算子，并提供升精度和tocpu接口快速验证； +* 工具基于**数值病态分析理论**：对算子的输入增加很小的扰动、从而放大输出值异常现象；检测算子原始输出和扰动后输出间误差是否符合精度标准； + +* 该工具的**特点**有： + * 不依赖标杆模型、不需要离线对比 + * 提供接口，对可疑算子升精度、tocpu进行快速验证 +* 推荐使用场景（针对**算子精度问题**）： + * **暂无标杆数据**，模型Loss异常，要做精度问题算子排查 + * **验证可疑算子**，要做进一步确认、验证是否对模型Loss有影响 + * 低精度模型效果不如高精度，要做精度问题算子排查 +* 该工具的约束 + * 仅支持Pytorch2.x场景 + * 推荐限定算子范围使用（白名单、或脚本中工具启用范围）、算子范围越大、性能和显存损耗越大；详情见[无标杆工具场景验证和性能基线报告(待补充)](./S02.report_free_benchmarking_validation_performance_baseline.md) + +## 2 工具实现原理 +1. **扰动算子白名单**：根据算子名识别需要插桩算子范围 +2. **扰动因子**：基于torch.nn.Module的hook机制、在注册的hook函数中对算子输入进行特定类型扰动 +3. **误差分析**： + * **check**: 在hook函数中二次执行算子得到扰动后的算子输出、计算扰动后输出与原始输出的相对误差，查看是否符合精度标准 + * **fix**: 需要做验证时、可以选择将特定扰动类型（升精度、to cpu）的输出替换原始输出、观察对模型Loss是否有影响 +4. **精度风险算子**：不达标精度标准的，最终会在输出件中展示 + +![alt text](./img/free_benchmark_framework.png) + + +## 3 操作指导 + +```mermaid +flowchart LR +A0[安装msprobe工具] +subgraph config.json配置 +A1[确定插桩范围] +A2[选择扰动因子] +A3[选择处理方式] +A1-->A2 +A2-->A3 +end +B(在模型脚本中开启工具) +C(启动模型训练) +D(查看精度风险算子\n观察模型loss曲线) + +A0--> +config.json配置-->B +B-->C +C-->D +D-->config.json配置 +``` +工具的推荐使用思路是： + +1. 配置config.json开启精度风险算子的排查 +2. 根据精度风险算子输出件、修改config.json开启验证功能、观察模型loss曲线是否改善 + +### 3.1 安装msprobe工具 +参照[msprobe安装](./docs/01.installation.md) +### 3.2 config.json配置 +修改[config.json](../config.json) 的task类型为"**free_benchmark**"开启无标杆功能。支持的配置项详见[配置文件介绍](./02.config_introduction.md)，默认配置如下： +```json +{ + "task": "free_benchmark", + "dump_path": "./dump_path", + "rank": [], + "step": [], + "level": "L1", + "seed": 1234, + "is_deterministic": false, + + "free_benchmark": { + "scope": [], + "list": [], + "fuzz_device": "npu", + "pert_mode": "improve_precision", + "handler_type": "check", + "fuzz_level": "L1", + "fuzz_stage": "forward", + "if_preheat": false, + "preheat_step": 15, + "max_sample": 20 + } +} +``` +用户需根据自己的使用场景、对工具实现原理中几个关键步骤进行配置 +#### 3.2.1 扰动算子白名单（确定插桩范围）： + +| 相关参数名 | 是否必选 | 可配置项 | 适用场景 | +| ---------- | -------- | ----------------- | ------------------------------------------------------------------------------------------- | +| scope | 可选 | | 需要通过指定算子名来限制算子插桩范围 如：["Torch.matmul.0.forward", "Tensor.pow.4.forward"] | +| list | 可选 | | 需要通过指定算子类型来限制算子插桩范围 如：\["relu"\] 会匹配所有算子名中包含relu的算子 | +| fuzz_stage | 可选 | "forward"（默认） | 需要进行算子**前向**计算的精度问题排查或**验证可疑算子** | +| | | "backward" | 需要进行算子**反向**计算的精度问题排查，不支持仅反向验证，前向验证包括反向 | + +#### 3.2.2. 选择扰动因子： + +| 相关参数 | 是否必选 | 可配置项 | 适用场景 | +| ----------- | -------- | ---------------------------- | -------------------------------------------------------------------------------------------------------- | +| pert_mode | 可选 | "improve_precision" （默认） | （常用）(可做验证) 插桩算子可能在**低精度**下有精度问题，扰动因子会将输入的低精度向量升精度 | +| | | "bit_noise" | （常用）插桩算子可能在**轻微扰动**下暴露精度问题，扰动因子会将输入向量最后一个比特位翻转 | +| | | "add_noise" | 插桩算子可能在**轻微扰动**下暴露精度问题，扰动因子会为输入向量增加一个极小值 | +| | | "change_value" | 插桩算子可能存在**大数吃小数**问题，扰动因子会交换输入向量的首尾值 | +| | | "no_change" | 插桩算子可能存在**数值稳定性**精度问题，扰动因子会复制原始输入 | +| | | "to_cpu" | (可做验证) 插桩算子可能在**同cpu**精度表现不一致，扰动因子会将输入转至cpu，需要配合fuzz_device="cpu"使用 | +| fuzz_device | 可选 | "npu" （默认） | 扰动因子不需要to cpu操作 | +| | | "cpu" | 扰动因子需要to cpu操作，目前仅"to cpu"扰动因子 | + +#### 3.2.3. 选择处理方式: +| 相关参数名 | 是否必选 | 可配置项 | 适用场景 | +| ------------ | -------- | ----------------- | ----------------------------------------------------------------------------------------- | +| handler_type | 可选 | "check" （默认） | 要做精度问题算子排查，输出扰动前后不符合精度标准的算子，支持所有扰动因子 | +| | | "fix" | 要做可疑算子验证，用扰动后输出替换原始输出，支持"improve_precision"、"to_cpu"两种扰动因子 | + +### 3.3 在模型脚本中开启工具 + +通过PrecisionDebugger统一入口开启工具，示例如下： + +```python +from msprobe.pytorch import PrecisionDebugger + +debugger = PrecisionDebugger(config_path='./config.json') +... +debugger.start() # 一般在训练循环开头启动工具 +... # 循环体 +debugger.stop() # 一般在训练循环末尾结束工具 +debugger.step() # 在训练循环的最后需要重置工具，非循环场景不需要 +``` + +### 3.4 启动模型训练 +训练过程中出现可以通过一下日志检查工具是否正常使用 +| 日志级别 | 日志内容 | 检查说明 | +| -------- | --------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------- | +| INFO | \[msprobe\] Free Benchmark: Perturbation is {扰动因子} of {算子名}. | 扰动因子与配置相符、算子名在插桩范围内 | +| INFO | \[msprobe\] Free Benchmark: For {算子名},{原因}. Cancel perturbation. | 算子名在插桩范围内，但当前算子由于{原因}不支持添加扰动 | +| INFO | \[msprobe\] Free benchmark: preheat sample in step {step} api_name {算子名} curr_called_seq: {当前采样数}/{总采样数}" | 仅开启预热、算子名在插桩范围内，在当前step的采样情况 | +| WARNING | \[msprobe\] Free benchmark: 无标杆工具不支持当前算子的输入类型 {算子名}. | 算子名在插桩范围内，但当前算子没有可操作的输入（不存在Tensor、list类型等） | + +其余日志均说明当前算子未能正常添加扰动、请检查日志信息判断是否更换扰动因子或不考虑当前算子结果。 + + +### 3.5 查看精度风险算子 + +check模式下，无标杆比对在dump_path目录下将扰动前后不符合精度标准的结果输出在文件 free_benchmark.csv，如下示例： + +![free_benchmark](./img/free_benchmark.png) + +| 字段 | 说明 | +| ------------ | ---------------------------------------------------------------------------------------- | +| rank | Rank ID，int 类型。 | +| pert_mode | 扰动因子的类型，string 类型。 | +| stage | 前向或反向，string 类型。 | +| step | 迭代数，int 类型。 | +| api_name | API 名称，string 类型。 | +| max_rel | 输出对比最大相对误差，float 类型。 | +| dtype | 输入的 dtype，string 类型。 | +| shape | 输入的 shape，tuple 类型。 | +| output_index | 如果输出为列表或元组，其中一个元素检测不一致，则会有该元素的 index，否则为空，int 类型。 | + +无标杆根据参照的精度标准如下： +| 输出dtype | 相对误差阈值 | +| -------------- | ------------ | +| torch.float16 | 0.002 | +| torch.bfloat16 | 0.004 | +| torch.float32 | 0.0002 | +| 其他 | 0.0002 | diff --git a/debug/accuracy_tools/msprobe/docs/img/free_benchmark_framework.png b/debug/accuracy_tools/msprobe/docs/img/free_benchmark_framework.png new file mode 100644 index 0000000000000000000000000000000000000000..609242570a23a20ea91004da0719f68218d56bcb GIT binary patch literal 259617 zcmagFbyOU|w>26eBoHLHyE_c-!QBb&FoQ$z!4otDcLsNN_X*D6?h@SH{gHd`ci-=O z|GZv3tE;PeRn=5io!V#b(;>==Qpkw-h;QD!L6(sgSAFvaF7(Zt_Y9xlUvpp_=qRt3 zw@#{3qHoGa2=`uZ;LJr7MBcoqiv0X&{NeQ;;k&f9)0;OaUH^V>`|JzN-n@CqlMxqD z2OAtNfAGSPA^5ZAn=2dsE^z($-$8hM%pBbE-v>XY#b+1$%|U9*lmu*@O@6cVhvBOE zWoe2Q_LFvYiphP_Pk#S#@>NHY2wMqz-5Ryh{b#|W?RsCN(nPMz)nN6`XomI4a!Ub} z?EiYgTQyHsB7%_jaxmjVC?;Z{gUlB=k^en!V~eJ(o}g^-p#zC5k$HJ}KHC3V_L{h@ zArb(8iI>Hq)1sVxG5pV>FV5nX(*K$9iN;UjKl?1ogT{#dA5DGf3^9uSkN@QNMG-Fa zKg$EYlWS4_XWs)bI>`Uuu6}_dOZ|k6@Sjbr{r`4h{$gje?&f%Ta+TJwT_7PL;U4(E zHLfiR_&m-vzBYO44FC;U=-u<85DWa0%_IL-tiIVFEe2II`~S8FiY9#&Y4!6vLq+aU zE$SYhcn6UOEs=i`wb>mB;jo?=0?Yjyw#i(DlmwT989e(ZJ@OjqF99T5(0;~yx{hot z1`XWA*C+7sIV^sJhlihO1oRObw4Pmwm5iTmZ?bLZheAWUY;NCg4_hRMB-Vdu$VcL>J4uSDgpXVICfdx-82c*s&vw`FP-C*B-o z_Xbmi@Q3VpItS_GWbv71*DUIQFY&zF+r4S+KNlXT7iUwq+ax7h8NXS4BNIg2sNXF* z^>~ogWSQB;FG!%2hxpy9WEgG@FDBZ-SpjK}(quL;M;q6m*}bD(1Wj!eGltI_CNyLI zZU*@4ygWwsc^u`BLS#RG(w3HHyPt4^SA;sS6u4x!6#W7n%1$?a{hK*e=xUX_=lq&p zVq)$-Hz-8Knw;fWaAfj&eiEwOf%yt4`-vY!z67l5mg&^ZwRtvt!uDnf|9G&VVSMS% zVulTbcvuQemvt`q&>s(5Np{_FtH3bE&Cbvyu`V{|c1IpPDTws@ka9&7`|R;KG_S0v zhKm(9^ATgFGX{S;Y(a6OPFsXhw##-dNTox?p4&@xr+;2XuT$Z#l%6Q2jZ2;NBn)8O zu;sX};~MCamMdj!P=82oj~|K=x15X>M@;Xb+~mTrt~JUFje(PiiEf%9-F9aAUFYDG zYSGHWS4t;XFDMos21gZKXcDRrKqgI0>vaZsdiu(d0e4^v;O9RWcQMPIW z9IiH%wJ0v(=>=4f^74?EloxA)C=Yj?Acm3`T#G7_{&Ys?b_TR>|BLUV*VF06P1kIn zX2YR>WF+d!$_n@aZe8!3gdX^KS8?t6=+KcZ8II59dZ_hOP5Nood^}r1K8^1il<;-D zrwDM=@tJ0o(v7~B_)4b>xZlQeh}@=Z4JIlX(tY{2J_DeWBsh$S?xQdXjHJ37vgdO` zXI()^a)^&O-+-nUm6o%cq#8a`YoKg;5@--*eAt(~&eNv*+wYnrnjDizgvZ7DFA?|B z{y{go@Z+E}xRq(cC+b2D49#yf+h+L7w9w$pIGp^}PiOta+bLpaZ+9{M) zkBKhX9;02p>BAz2`)BX9Cy~bIGQlE+ z!>DS>k?L`w2=($h0QKG+$n8z!NxYh7=Uk)ltAp8pNw_tZt5-{*p3zw6D;SDXzIDsa zl1p7V)JocgQjyzq;Y-5U%vpw@fU_z|<+h!%^$|};eAs{3bWhOuBg?zmW^YE=tfycJ zKX8(Be7aR3}cOqdrcOv%pak5t}o0^d}oI8v)z$w6!TA%RAp; zh^oKw*a>^yT2;2S;+CW9V^x_J5lZ=u(_PlEp9fV4gSaW&F|FnZ6d9_Wxs3a_g zXejeG9oVVSV1;r%2OLZ^CE?wY{Yc+=OlG!am zuQ6Hn6ejM!yN_#O8#qdVVt=EDFHW{<0hY}R7mw~xp3^E*+$Mr)oifE|Rcc4fBQI$k zQ!qCNAnzEK&l@J&3_nVa)vjXesn1{@wK`{jP~bmiGCA2iXK z4CZn*vk?f~5=49?0c5qO{o9hhDF16q^lexb z*4iB1>gh_Vy@tEddr$F!_nD=GeA>x%M5_a2W0lzr_tFB~Ve8?VEwOq%f24A=D$PPs57E>bBC2{5|(lD4SMX zjej^jt%umG$doJHneyvkq|Z(O|739cF-lR4dFtwPZH=pb=uQN6FI1!KHiT10|5VE z$>TN0vjC4%M2qoH@}~8d#i+?fC+|_kdb?%dCrp_sX1%%g_+|47$EP{8S2)#MH`^X$ zgJ;WnVwXNv#$F2Dk4|kSY8h`g00|ig4D3A|-y12!?2BpD*`+^*(flgS3c~Qn_?KG~1yoe2 ztxe9CCgPd>jA$Q{!+{sayuDw?N6|YMb7Czg?TXoyb4g{!ufx)~!Y#)WijeG8qHfm= z$=PGJhUFad*h+R2(fWG=FL=mZaN?*kO@gt_I9-KXTEVnwd~TB;e^cr}2ZxSUAkUNa z=OcVXdz1N;+Uo(`4RyVGXlHIkUMuIP*PP7he0GmMvRky=EBMV!h-l*f&=Ylgg7)V6 zL7x~Z%hP$7!SZ~AQ$hbLP^5^~{)fce9*)>UIqXWUwA0SGH_hm@BQ>zZv^dO=N2eEF zlt3!|RFZM_ebkzC)--^UPAZuD6;h%}+D~!{?I(9aT+QQ>8qkMT1I5v*hdbQAy&E*; z(!=fy1xUF{jCI=n-@Py4xR>Rshh_qYQ#@lJeTz5KvR!c?i#B!F4oyT_nM7Zj$rNX^21cc3-KmuD3q3Ue0v}{EI6v8;lgRQEDCW0htl4BSTBazy=4t?P( zbqC z_fM;g(p2F_=6^3>UyO178+Jdxar7LxOc`xgE12Vj3V)ke*zXItgR-%7?@qdbFaN%y z_|3PgDPxMQ&~g_=;8bkJJ9V)mPf7+s5qHh{ffW2uFnL zHdMPj(GJi60$9utsX)!ch<(`{<%vWaV}o-$mYtQ+6nJ;jfKbJOIBqdvU}<#_Wy3sR|+{%%*?Z zi@t<9@oxw8XphZ-zHNUxbGSQA;9Ldc^d7E;k1Mn1);s8H^wQ6FN1z%<+0Cyi>RB1- zjz{M}W`e=#l!{Z%-u0=sCyh>2CW~7ok>AyNY+G}8Ex+?SM#=f85=e<8SI5JMg(e{% z0{N6cu=Nkl3|6>sdLXqV;)|UzK68O5+qH*(4$UK(pL)=yo>L%U9UuAk|6@yzU)L~$ z@E(R26VA@-w1uSjK0hV0nM#gV|HvV;$(5GvDOZ455x^M3|3 zNTu>Rh-Q2sQR8*Lhqw3VwBV$lGUnwUs4NV!Q_zwZEehjery1MA?(1y9_bwvbG68ES z>sg#WYz7+h*y164b^}hYD1_N1qxQYbe&^A^v*dJw*B`{r>B+q+B{T@6Y-esgL*ewo zCL`f#Fd@AvQO{K!9vKpTLZlNO8Y+5oWNvHX{!&eOH(-6$Ru!=(7Q{+f8n)btVY;oc zTjDQIWL0f@E6VKg{Hy+#FwW?*#6XVdvGfYLwgP-+!bV))Cn`BC3RBU~PpfolJTXOJ z-saKO;qe@)tF8Xy>7k^Sh_={+_1s9W{_o{NKc9M!(6-)~me73Zh^Fm+Ap7L|>#OOH ze%Z4tN{#<<({6>xzyeME&tfOingwLY1T)l{qE6m_B&e0-SLg$!?Fboqhu9e659A(j zUp;LZl_YIrbNk@5G^T{LY0VGVToZhzRguAgQzpnr)gpI1y!x1T7Y1Nm~G; z0$kubr_Y}mO;m=m-QUz5e5G+Ms7SBoOxo?UDfJF^@HPx9ba6HHWsScFd#!f6{ z3HC>6xVe|wHI@Vsei%$=B{Qaru4G^V7qn;W`Ixm9s- zD4V6+ant-76BQ;_v2WyuhR}0q44)r2Q5oDPjO@KyHo0Ih<7Os9>{OGBOhb(GxQ|L+ zSD^r)N443VPNurt*y6#G2Ts6j*Q8AEwVUC$8EY1UdqCckt; z?3wbS>3niS_6j^sgFhVvfr>Cc^8NQ%u1O)TOW5N?QV;K# zyI5*~gNZy4a8TjH0UyfIsIN7gYjG1h0JnVv2*a`&*yxuYh^G|^sfdr^gg7jJ5xC1X z;$$aK+5;SYvNM(uY96IjfN97tRc3^5ypY*<&KVljvrGg6sr@4>(iamLd;ca& zEHuj-*aBGZ20G$g9-l3uxh@6mmyu1)U__}hRuAsG&iJele-?^Sd@62sF}lQ`1Az@E$cvk2Y0@@WE%~6o;p}>sW2lW{ z+1Ve)N=f8H2l5Izw|SoL1vzfP;Jp^bpLq4IXGq~%Wr*JjeW;{OvAwUIb(T=WGp+8< z??Y1C_2?(vm*01;(?b)#2&C(8{kd;KI$20^>=~G-+yQG4HTkp6WEWIcNvwJN#Ol&f zD*!22Nz&Erhp#vKUXsQBppnwz@Z_oq2d?~%ZJn$WrX(xZ)DmVfSv>sE(zs)fOT6J} z@$Tsil~W*?Je`Rv+9XowaL+K>k7^%q)#^~HvGPB~*@h{-sfA%T4EI;mhF&$hY0k0|JqGR+Poj^K8b}dBr&>!Z0o6f_8Ckp7_T24? zudP^4HFy~@z8EKHy+RJnl*qq+`ZFmN3C2`&e}u2V%?7@WM5#1d5!eol&y#YuUD7=A z_{o{SxNW}?VXyUPn;+L-1%81RA`EJVS0?SqXNVucw4RA|1)>N05*LL|WqFNIiYL!1 zf|Nq)#c9-m%#~X7O=*%AN;mT5_i6;YY)!cl6|g zFK|L)o^mPBT>UhK@h(jF6*_dnh-6|bK5^>0=32pvP3qa#B zzxp++m6{84CWLB)Lm1J#889u;aV$$Ovd~?B5631fgj9@J9tVydSo$eiz>tj$=CV-p zdfXWG!p}ZaF=il!jM~y(;nUM;s58F0k-2@O+O{gDjD6CRJTVKZ!pHgoJ$%b4>X0Cs zkLGN4>n0=MB(G{6;gO2up+=d5#G|Im_n_lmM~;Dm(eb-Pt@YId(e)tAvK z)4u)07cjokh@z!LY!zx@>m%uBf?Q5sqoPLMEO^fCr3sQBLz4~Wi{mPr?3x<~&uGF= zBHw}n+b|+#9N%4Ek@@z8gZCnm0io#f?ak331ahD5eYb&nB3_WEZ*%q8G}tt zScqZehc+wF?VF^DmR2G=E=tMfKM|Wj&^b{jIR&p zuLf125C2UAc6=545lpZW%?$Kmfs{LDM2Xi&kFGEL7@ks?3rqV+Z?@f&^!yjDZ32H} zE%}YoQmg;EK)c`2QiM3dx&&`zaU)wcW?*mXEN6>zc!H{0qjxJwQCPS@9rWX|P|z8v zi9&uiwJ~}2ql|FIJuiQL*?O|l)b>gP+n}=lGt=k2kD*vTzI9g9Ca=u%|KV(BpY>ij zd@klEa7Tl=TEUrDdQn~Ts;fv4k%5fU&S)2vw7%n}Fsty_jgIE}Z@Atoa>6dpQ*swY z<&lcungUJRdlGN~zrma`@Z1=r(JFGXzC3?Y(7|VD=vuM&kENd~AgrP}kO$=#6Wv;IA`wZuzSo$b}FbVo)mOZnb?fR>`a_9(!mc2Mcnc0g~ zb-7wUk9$Y{W`AB%-S+{O(uLe_U@yesk4Wh2j?k;duu5YmHF_8iS{hoq0{H-r%Ih z-dl2j1$nxPf{E`}6Mm)!G_N{_T;@ZwEiCk!e0Iqz{K$_jNyYL@`XSm2(X86|0CUAa zZ=`U>t6*Ki_S@9T0KvWAns?}U;-=dSulmj{8HYVYg&D_N(F0fW@ssR1iqLT2nFKi} zvk3~eC0MA9Ui+SSV*1m1Z~iH6c{-6~zBJZ7%ZJdZB@)B*B{8M_hMr^qL)VC*sr=g_ zMZBSqsu$L)R{K=4XWPhQQUu zRa{2ZCPJY44IX$Fx2rM1{c2u#l_|(5djHBpXO?gnsBbC`QuA~8h@&#PEgom2qN^^q zsO)_&q5hl0S&tp>;>;G}q*>gh6O%8^TivUPMu_|4Ag5$08h#?7r$9WOaf$CcF-cju zilI0DL;g3hfeMut+iV3$^!*QsPOC{?zPvY)qbpmnGOb!yj@+jCgntxrNEL=fE=$m7 zgxHl^h=Bu1N3E)Z(mGWvV*5#Yr-}SAO%`@BK5XY0o%3jQ7SKH?GIrqj{ZW=$*4tG) zM#QzTD84Yh=RA=NvD1r7?o@)fnYl$BaSy{+4o2B-N~rhSy{zCJyU*dVM*Opso=b)V zIX(;D5R@us2JFv^X<8}mMyQmQ#(^-Y+}6w)DrGw z)$b3AvX-;Zg)+Mw1jS2+vBCeQ?1-G5WSY4CHDNP`t!Ltd#=ME|q{GRvQUQ5oy~c09 zgPk{i_1Um$k-QRXV#k%43f>#_@Rxe7QohQM8uW>}gaY4xyl-|qzh6A@C>558S~yjt z1p?xP&-bHGd58IDu5859Do6 zh2WZ_balL6jJM~aIck=Bs z`r}~VuX+2AKN{C~emGn26uLeV?jFZ5NBT(~G84=GkoPs(PJ3JWjK1gcHD}wOASf#w zn53&iirPHn$E?4%)Z_%-6Ouv1lzAw0uks|+SE!fCTP(C~jR84MwiUYt7mN*?=_Vf6 z@DLP@>Tyij3HYqxt!%3%L*Ob+VK>**L02bAFBS1WP3JEiA=2zslXZd#}Zv`_j5QtEG3&qM|x?4b{Y&C zrEK5ju+^oVDCEwShQg{Z2pGn=7T|0rTkBl&JErprmD~94Sf%%?l@|QMntytcq=aZ)HF% z+^sLxD-O4OGqK1|`ACS(8!Q`oa3prP)D-KjX^8i+YX<#v8{gCA!1cJT|A5q?=SohT zyOu;|1D2JJRVa9ZFQOIIehtUPcp~BR@d|Y>DBq(!&dQzNX-?NIDoR_$X38=Gc`!8a zn3T}Nld6rHs8h^04rQwcvaIJAovsihF_1mQm`qNd2SB_CLxylvql5^hneO>@wN5^P zI9^4M%5{zm=g5M}QK``5X=%E|wYWB=kM}qR_j^h<&>E<%@C43R0ax-Uc2>18=<%j zC`?PBx&#OI@7?)WC&v_S!F%Hn_UR0Q>uQjH*;(j)zx(<_Rzj3P&XJcYSF6e-n_xjH zYa2FIyh-?-rkSw4Uz>Qv%U-@YJ z8GoIm{sKEvY4h8gc=CTFhF*^oMXyVOV#5w0X)c^`Uo~bG#Y*(cGl-nzdj^I5C@wC( z@9cZV?xtwI6e zYlRf-T_bdCu7-_YS=a|Vemu48Ch+FlSvaaT==C+xZRPShnMhs4S;o_|?-5mo=|x;P*$;X0(JRoCiVMd><1ji9UFyR+k?`2w)7AAntg{OiV1oXQS9~_6V!5| zodGCXdj}l&OA(P<8N>(fWZ{C8PMH&pWnp#t;(5fnr@x%PR^=b=dJkF0sox+{Hk)4E zeG9|qOH{kx_!&kSPgCt8X^Rf&^>G#@VahdLhAe5qU~?WH7U8k(O>n9sbv;#$cRMU- z-bM8=^p6n@S{g41a_L03WO=7lxQT_97}^byw|!-fu$5orL{)*ID|>F+QL8H!KBW5d z0y62B-P(|a8)z|RfotPum_As*q3jCuQncMu@Chct ztQ~8t@$NQ@koieYw;R(Wh~YtFZe6pqS-27lB#~nUF2i%#%>B6s62ZzjpLKxSn0xi& zP@(+l{Z&57ac_)TyB#fnYuNMt!W{6Ggcm>Z4v#mLV)s}=KVHPj1fQ+nt2TnsWD^*R zSR{U9A{!dEf>nMe(3Tq-iZ#L5V(#8 z#CqlH%ZTvqolt)T>u#IVf|}#ym*w$(;3lVN8OwVxb}r*$njL3a7eRdo3ksXlkclEe zA4`q2T4ic8dhg>m2og z4Pd61za)p%W#B~SJG_mjAKcpoRoP#IN7SWrJq_zJcv{VvP_O1CKxU)qBb1?XL8c?` z%tFrRX@?lEV65XfPT6;~A26Wj+#5G$kXVsmI|s0;LZ`XQqaD5wG2!6Y@aU!Hwb?*i zZK^<=-mE#mV^Q$wRkC5Byrj_t1~@wH&mY8bEA~fM@kKJDkL?!!@FQ4t8+Mqck?Hf zV7qT$m<@LmNHZ0^W?#G#4`lz z3bjbSJ808pj7iMh$qr{EVN0soNRF1&9R5bhDEHFjl*DgO7OGily0{j5s5=&9L;aR6 zl!G&=)%$Y|UmT5Z)5wq^Tf*=0Vax&ww|;TRLS2zs0SeHGrze%jE>~VR5rILoS~YpV z=T0?&S-)dn!IReo#BFM2eruwk3Ws;CEvKw|lj^cRl_!}8h@<`XTh)+>idM5O=(%pD zNG*ljQW%vGHl(I3-k`8hVAsWUNsN`(Vmk#WX0auho<1>b(O^^U1I$o0NGl*b9TbXP zsY1f_?|e`K)g<6i%8MS#MgFvL%In`~&F06&RvH2Er_05jsLS@aoU!HlrYAnq105L- z2{$l=+cBnlGd5?+bjK4JtSw_XtL%iO4mU!W`^V9i$J#^TE2MC4{~q^!Hk?!!#?zoX zR|T}hvf+8u>ge~h9Ew@J(B13hCL&XN?949AFyV$ z4Y%f$W9oK{?S>cwQ)Fgq8l&>pGB}vYDDLesmdE6qv+m%6zi;`c`0#p)N}>1FU@wpP zAY#o1CuW~EUP#Z6srG*D!D~o5oJ{wlPzpxn%gB2WDIc97lFLg`k=P5OW*_a^Ey2M{ z=-PzKb1*gNDRt%t;a01d7#edK1h*U{N^sZOQ#5BGypV_-+NGuOF=XtuSipupv)x_e z-2;CEk941cat0f|cgZR2b{&NoI-vRP&1K8WE&n8V{e>Z!}(cq z`j0`GXwJ;8K>^9)Gfe72f#s@VzJ62Oj`&!?aoJM6Tsd1^ysaE}=qE~3Vbx~z+PfL^ zntC7q%&uAmwb(6^Fp5licGka}Ow?GRra4@)+_jkD+cloxGahF#Pgpi}IK*j`HLd!; zHhqScEQC&yu1Erc?V5|9B#w)z9QzuPLn?roXXn>F&t9sh3#(0gyh`X>i3TU#w1wb> zNvb~?n<{lTjE&q>{OBn_?IViQHxk+jPNj{+yq8m}IRby@RoON5nB$FC{mMpdBB;2N zA^RNMWWna~4xPZ2(rK9QX58~ljz^Wz;`9PcF~7MF6}X${=I&y?{iip`LA2~}9M zCwM2p=NS5^UX~tY6MmpFCUEGjd12GXrzjmgXpb!_ohvj0h)Y-nV=2GJjP|<8X#xU) zW4&5{NuR;N(~APGOO#j!P#6e0Dz`8cu6>XmcYgFR2K80?Jr5V)<{l4AqZ^mhSxw5MJR9)_BEXCiCikI0Bzn>YhiCzez=qA)kB+GXn)CU;*S8X}U zCUi_vco1NOOyT+_&M8BL(#d?BMQSvCY$y;eGVYtoo2s?qWXj7f5WK7YXz1v^lI&qj z?v!BrCE<9yI?ue9kD!wAj@eu~0BsHG1VQ)Q`SrlJDAyKyMZoGN4bsbi3#Nq7{?lb# zsO&*Bu|CDV6-F-NiGpme*agGcfq?%~`v*GS*||oRtQC9Q-u_soR<}Gc*~xYF+E?*c zWxU9M=DyS54_IzQrmPXfNVE1#>yPn`4bXTW=5f^l;ZE8&rLV$A6U|f3+$Oa zH|{GCwjURc1d!x1qIT(9+%#&l90&6R0H9w7Mf!77EgEs9^pmQwSr02ghGiU48%vJ7 zkp0G9y-;(lk_5q;>G}$4v4FeKS)b8O8uOZC@ zU}CMoYt5g%x~f4FoFKQ|(FFqH?m>+p&~VD6fK`!NRQDPz4>1O-_~A~dS+znGyR=IY zi;P>l_O#I_Fdptl=!eQ;|Ir-yKIda>dN^(ffVeC)F9(cT) z+lv0m?<4p=Kj~blSc+^Ug3A9Xi_+qQ=PgOGEjN<)hwa_V{N}hCJoRrOjcWO2u)kFl zyWb>6<~LZz8??7j)baS@WY3Vx)Tf`2OL^8njoC%=5^b*E5Iz(2B+Oi}4sr=UK3e5- z1@xsfE0=BQ7zE>&{Bq=XRJX*p1x_g^s|v;HUv{MY)g6MgPR?Z}tSRSWPpft_Gf?k# z$|p~iDlB_k6&bd?0|p(VDorOYE*zMf-lo9ewPeZYk9GEb#_*}k{A{yC0+Z)yYwiZ4lRL>^p>-nd^+ zuqLedN-3W2zY1w1-+$6+Y)x6|fNt~Mv*FfWx>@nUQd0Ocyi5~6;H>Wyv}oECf<3@^ zPM}fHbt0JK(vxx3E$5F5t{BvyiLhG zou>nWAccz!sDd7sWAW*ihy0|;4@q%vhx@H5x^51=w$p$6ys@_J|7fS@rC10-;P?czE-Rd@@e2*| z-FsVr+w#e$!Pg*=FFv1MpJvSYSTX& zDlz|-&+q24)EHpU^0R{RU zQ?kT;w0(S-E`jUA{J?bni)_~+5MGFoU_1kh<(<%HSy*Xl_y&R%0>US8)A;miWcYrGqaXp9T#6+GFkOi%hY)Zs0K4^wTX05Le!Ba2hss?tn z=()QY^M_`){UB_p20vM@>@9MV2?$n3!7yk5mfIk^m-caMkz1w^P`^@6&%?eKzBonMlo9Z z(#fJNqH^m`%IRvw@i}ksU)Cx>|nM&NqI#pf#6P8cV&xXup>3?>|b0`Mp zJ}e<}aX%}5TUrJD%zBZu$*8;r5ZjS}ukJZ3x&{fB9=lM`v%tQr%h$;b8J&KE&3q$$ z4bk;7H!e5qF!7ik3WxV4m! z`hKN#Zy2O#eHOh(6+nVFu6 z9+g|bne%JejFx&No+hoy--Fy?E;QOQ)T_&ks(kDjDfo4a4u6XCb?t}z%|XwG>5a8qr*<62*8IFicNX1QF-Uvl!K zUy~HzA`<({^1?Sl>7l`CZjLq-V;ZBrg7+d`{v88)-mrXqsWvQz4Veyo{ukae+*(6U z$J3ur&5Vser1ssBfSJ3y0+*H1YVUSQP<-Iw-o_p=DcLmt6iw0I)D;L;qzpxp*<2x9 z@j!C(hSOEd2GkWA5w@@b2s~D<$a7K9Q3VTu~3fr`>&j);g41?_{SZu0*6Y9;_uZ!$JF+_klOf+FG zVQ5du1vH9ew?&_5-_3NQo z{r=LA^6IPO6xl#G(S5Syj$Fgj4k+qv17JOw9#hH9UTMUNHW)4TWyl6UrJ9c##MZ8*N-tgluQ2!Xs3iBJM{pSQ5NL0{)pFnULJhN#;csCPn&ffYyAO_Sp(&~oFc<5OC}gDy$1HA7xNrbp9m5w*HpEF;)DLsn9pHS@l&YIdxg=wMY|)+V#b-r zU-mHuP7;^5Jn1JA_=V|@QDT#8U2a$|&DY$)8LSzK@uKirO|+Xz@ky#JO!telo^r(} zoAb{)ky}9(MZP@KKD%LGD3!Pipdj!o(@M{py*VBV@9mG!B?Is@>5+VwK@%KoOmLH0N82Il*9chY@^}7)* zTiO{-Wn&3@GpqfUUWPd4spF|G>2P?v*iJWDWj6=qR$7BpD?8N7o8ABJ?e#vdZFS8O z$O;Xh2xLESN@<#_MpfH!>zKWMV1)Lpr!4?sVliQdk9`ftmxBxSp~u@+)UIr)4rVac zY~JKD8%*IG=^)>fvE*<^ei&Wd@o(Th$ynK5S|cho;0gEr=$hk>AU4o=vWFM8dV!yj zSfB-HrJ0OqA@FBMQ(rMp;&ny|P6wZQO>!g%a}a(9CP4aQ$e6!@8gwB^a%>e(Fs zzJW*jr4v;3905H$E#F=aX+pZUAt^#K=?Fm`Plpe%$FND(&i%qmC!D-+^hP(*_tz~l z%%*707a=rf&#;0;?p|fUOrR{v%~Q+wZ|qqD=_g`vt!h0&aq zk!%D!VL?g_ij_XkU+{4h_$<2g5p5F$@FHl z^uB)L=CVc3;PEX>I+`4l?hKVZLZ7W1Is$1zW(#ojY7!_R@s_;Iv#eo*t6T29BH(Ux zu*K1LN+N!{$-|vCh-JAL&M;I;p!RD)+~Ui#&-3jAf!tSVu$%X{N|=ZDS(h-aO=q$N z?c{hN5HFWfHv9xjqk<e^26kr^t>mIG7LLl!)`Y&!X=d8PHk)uT7RczcJ zn&TwwY0^bgYw2SUokw2r9|(dK(Hysx*G57%&NJn;3m ze4lWtC0y5r)R!%Yr^jug_mg!y#A@szBru;a{W1;!`zP z70dl-VgVWPVH&(gfPPoSn)XiB>^39U-*S4wGq2NfYYpOy6 zx}Rq=JD4H#syU5k#i`;O9nmiQ4j5QpL~gqRPHp<$3?!a(v^St(X!Q@Nj+ znbb|gNStru`&8^ zmX&F~XXLF-mz@RGTxL~NMbJ_h7wM=gVmADSaBafNdDLe6)Ky~+CBW106QJjgh@2IN zTdcZaPA1x}-t2sy6#kI9vv+Yp|xg?gwf|2V8LcR~!`N3Tg5u@TcY-^*JMJ+uK)Quc~&jD%g6; zJ}=Dk1LJ-59(yKb{T)7lw0Y0JOZElctRbTCyKY|5*OT4r&kXNdl$objHO;e^DkLWLy_9kIkbd%mjuIq;YR(;T zlRS=JgOwt(K5Q+@lRfU#Z9*r#fRoH}>wLqOr*#H2M+QU$3xnl16pf~LWT*G>X&}D_ zGoy2t=nX(@C|2K`?e|+RoTA8W|FMP_`XuCf*U;x0pyYuXOBlCx&LvlO*W-Q%cV30Y zZ#k3jCdOanko%KO=DNiZT%j%Y8wskBIl~ZAqgG;!d2kjR;*Jr;f@dqUhdf^N_guil z6l&Q4Cimvnpv8|c-^^f)4j)1QSnZ>gq0ejoEJzE$uzgruw{iF;V_|Hl5=3XrlX~<> zSFgzcrhThFhgu{XMz5_Azpc@REl5NBSq%IA+I;Dy2Si5jT~}tHN07?bdg}tFxO`TQ z8%$EMln&xeS>VjYtk*4}p{a@uVDhMSPQ#R3jrU1n^?;BULd?|++Z%C1np=p9$&`V< z5?a2c@g|d%U8*e+ihnRv*)NqYvww4jESf0fY^r(&lMb!X6Ur75UR`p5*PPV1ov9$O zrT>=v>iOB?KwPR^y??3BtV3gtAP6Xwg*wx0r{i(+EJSt8_VyzKr|j8zrLoEBd$+^6 zbU7;JfMyM>JahV9UPaaz>QQyvW)8xJ!Gb#d%@~nTS#!XRtf}H!Yh!K9WB0P$cXeC2 zy&|#xNov;18S#uDQmA*PvU8Vu0#{u)1~catJ+L(L^L;^`o%I^BKPXP1wxkvAEez)P zWaii(^iHenEGs-u2jPb7#VqEhvKey#f=Im!BJ|sG-c`aHEVaH|vl7m)emJ$+AXWug z9#(HmfWcLYt}fQJjkwRU&v7b;SNgSF(_2>}T2bfrpg}jgUX%4Lme$vxmG6}#Mq~E= zHZiqtoQ<~^Es)Li12ch5&r+3EvXIs49*+;yP+r8K^bIenlwxA@&u`$<%0G|J9BRcJ zW1_+!D|y^A1+gw+o${=16I&{K1Ps%YG*nM>BI=K+JPjQ$tFBzTar6X-IbHaLl$xXq zKy}S;((=V|=e_M0)Q&Kz@&%K4OQ(fXS%inV_A+I`&m~6H-M6s9CUxZV4I}z8EowXB zanY%D&y_D-INc-kjDV@Rq%r@&*@0oCTAY5u9LOfffPo{!)xWt?daoH zc3_03$J4~VS>C(Gn2R7-$mer&9qSMA^a!Zj99#b`%?Bz&m;^uVm(M>R!zy}rUnup) z=5cfod7JggClYiE2TNR|kKBH(#$;Ag()Ba5eqffoP<73VlPlI6Z{GX-8rM|Wf5FBT z3EbMo@Xe6$d0u-{C}c}KMdTs|N7P5BEaEu*$E{D1Bx~>!pJ7@MjR{e;Y^^C`d1>0f zM!N?lgDvAQ0#E3GM(nbDv7yUq(&O++^}OzYH`j`bXG(VnwN0ZC1a9ewOd^|{^&c%x z39Up8EXS|By#5enhcmWN;t1>eyJf1e335ckJ{4(F7y zhl2k6C?Cc8v%v4f+L22xxkOJ~xd%oD9Q7Ua{WX9pN-^0?-~$04O!0+Xp2}sRY81{c zB+*o8Z^w))71VM3T~e70v6Pq5!$VLsyf5eWxNB(VYuLcc>~Xp+t~^;k16vMu!@Lz7 zFFh!ehz%0O-zUNFn-8OS_QMuFU5u21R2XDUm;X>TjPZ{ja>1>3btM|*EI)Opve^kK zOEHvn?r+0lEkpv0AdpGM1%5L!8P8SGze{P44klS!2x^KS}#Jc>of6ko*0!j&k*Vb$DornsSXt~D~Jy5O_lmH5|~ zpc1LsrivY4>)j$_S$U;(5dwOzFi5?|z$WK$P(=r0XTQ6rFlyQvmSGCFTwztcb``w^ z#85Ic)b~A+qub@3AQsxjj)1Ja z82}@rV|Ox$DfCb!Mk6SX<`OE{ch^{nrL2981nM2FY}_Tjv{^)e=QqQgK6N~vot|UV z%1Re&=Igvlbe=2!P+pWPVc~})lYO4IFdF*}-U_$%WFX9E2UF<2!1K(@LMVa>FZfQySi7C{{{V``Sa2&>1-I!=`>S zLE+t6o7)o(2w&1ab@ja89*Na2g~e`9q1PWiOaGn*bZ2GrQ=&EEALlwbiD6L!gn;2= z@jh}dU(Vst8=(7PIp{;*;_BJ$-0xq;?URfoeCeOitCdzx%cJ|6HmpAc7K_y$c1kRU zMO}x@z1kM6e&w!j6Q}_X!_F0WvP*U?k=M)Vf^)~IDJBTOh@Ynwde*F%OeX!Ne0v9n zI~n2Mlv7da-_A4KOm2VA>Tl&%%E9zX!iXbwm&p74;x)%f z7lKZkEngt@w5wd96-8kpH`d}lu|BqQ@WXaF@b`2G8g;((F|B4ZkX$O2p&ZS(L%H4k z@doe*1Oy85fkecf_xu+yq}kDGxyqi*Y*DO@4(5Y|Uw$SKM&~9vp^sWa?wG6NP4D;B z+qv0_dpiDfcPPf`0ipyRL*4$Cq^S!Ta^HT3c0Tru>nLWH`N~BLr!uJ-I;9FA<2TZZ zIZcon7#&o&`B+P(f98Za9X`*P_XV7t&TzRDf+RxGm%kTYvn_KX@%pL1ikiu9x&Jwe zsxMQH0g`gKcWWD)fquYJC&FT2K#)cxhk zS_q>LB9gJ;?YHQ21v1$No(pPoHz%6+|tzzX> zjnF{8I+hg#jdHB;ja1dkAnn-xnoJ|Jn-dS4YWgIO7?Bm|xsr`kE^R7sy?ZZ94UlWE zO-2#U=w+uKZ%rxCC!-DM`D=LUc$iXj33pLgj}kS zLV3mi|C&?YztlBge5yxyUoUaNMhqXr9!{UPEj2$8-AGMNB`uNRcI#5n#G7{J*&iC6 z7r~&?M;9GXr2t$yRA}FtlTSB1$>hpvisgo{mRfcal-Ia*>jKf9CMcfQnba2}P?=Qp zk7E;$%g>&cn~+JrWjP2qL4wF%z5^f48#Xh}&Cfas6mR3B%+O6eBXrkI=%t?wAd)4q zL0pJWShK_Or94llaeP5ZP2lm&&>}Gk zm;^t1Ptkl59am;2a)xPeER5iD?PGQ3y}IV%l~q3@FTW&83!R?&2dEMu3y!FLd+GDs z*O8rQf>y9>z^=h|5UuyZ!QeB&7*rF+)K%-Fyp(WqSd9I%i%xkCw-VhL6hN(NdR8jg66 zYi!I2jQq~*5#Wgy%$hqRE;&iNG^^tSdb%3;AdFIuh***2M)zxuaMO8SXBHn~vZqSl z3q4qfw^Ud@UxCAus^O_v#O6Y_zxHW6$aM;SCsILuZuY$xMZJ692bJxei9D`NCo$pa1s-sTfCYP2>TF9G zBM)CxQGyEHeKW*)Uo!A0q7yN)=cg1)Rcrj2sD(haebE&130oPC7ADipq_oLQ`MhN? zI}!4kBH7;@R-;N|(}k7e;a}xIKa&_87nL%ebr?QoY7@(lR5?6>fGI&;YocJmzS`cZ zeK)yP`4H0^yTYVZ`Ew z^PgecQR!fI&sL54bZKY?V*!A5FD!Pj24NPLou^s0Ebz^W4?NIC3XB|%RDxI%T%N{D zEcm805xkDq+HTuNYR&sX889yDBlK9wh;aK==-4ai3;7ymvnwSalHRL}mH7LfDDnbXZj z`$TANE@l}uS2W_T)@(jKeDw)nvTOQ;QS)dEqO4eqX#hvg$U+hVvivO1~yVm(@8NfGL&_Jyg2i{~*Udbf{D+{+fZ0D~pVP+qIk=NQ0;BT=vDFmnPdNac!6>qmB^WqnAiCiPF^hMs5thWZIo3g|Hd-orTVlgK?lO zIZVRH@1_xaH+7J6aP(cBY+UDdF70FmOSaxhqjSb{Rn@**xFdy8SL!O4W$=2^_4?sv zP2_&@ej><_SQub90yUJv)3^Mc7K<^L+#=Zb;*RL~K6R%`dPX*i=pl7=yy{*qfCVc^ zd^Vde`+T%`J`(|kOd!XtDT@&7tv1OCZ1WyeZvfhxU?_A@B2h9! zVcfwOSNX9!7*n1bf5yZgj-X*sI)gB>z_VI>djGwspCoowi=C-qw$^%LC_6+m6 z!hPyIOGG8`XA_M*tSE&VNlq0736#jH-xc9B7iX*%tB#P$u_Kd=@^w?hHn)pBIoP1h z!hd!bgO@I_hCPW2dZ-~~BZC-i5AdQE{gsx+AJ9Orc)f}>WS!DYp4*$fs(^%lfsb8- zsV3YOFcw!N0*5l*eGZJsJn~;nXG%p!D`Kfg0Y9myHypOf#uemxgoV66_YM{iOlXSE zId)^JP;yM5D=h7;~6CL6S^0*f_YB6Gam> zDzxBV1-Naw`T9u`>kJz;5a;CN3$?b?XTALlwPJkRu0hxt4Tz2JBWT)174gtXD{y9R z1G7ryzMc3fGfoRi~v*;fH$Tv*4bZ$gMRMJ{Tjn=5o zBFfy2|I!LO5*fJ^IUKXh=ErIyKMPK=AK%2j=l?C!_P4diZ1^SyGP!@~TpVE%sD%As z(1_x$cE#j@UP>{bzu_G!=E7pJAwucD4O9o62GSx)<$8&QE$@w=KxOwp^~DhCh$?Eu zgg#nGMl&V1uN26>@1U3**>^qi@WShZ$nb)N?|B7xZo6lW60sr`R4Ie$2};1VA`lcv ze6T&=5cYB_*3x8igs;yd=TNOt9kh5Ky#5ubBxbB$bC7C0%bNgCJfAc=N+SmcFnPFb zn^JEsPj8Q3`1#pxpUVCxP}Gz8@112R{82Ry-}jm!^1eZJG}KtFUsE!Obforma<{sf z$l>6akHXtcrFtHF8_B={dO?L9*6wqZ!|ff07|1 zX95u<(93tx6|~n|GDcp(T5elff=V^MDU~rK{F4_A_$bj0BNKQ z&@h5=li%BIpzpXne$VNfdi)nyGx-sec>>~_`*QnO4WniZ1V0`X^n9TT@=apU5JQUT&lz7G5@NQ=f^wdRH*+UF7D0nS8tidCf&WgVrol@FA9m&sJV^| z5_$A&_jj7L5zlp3lnh8_HT01AAD8`LKBhpSLw8w`*-U}TAoKX_`B8k|fVrKfnHFsW>-9W%F`>z;!f9nzW z`+VE_aQ_|*#}x52&_Si}(`y|USA;5gQ}q3!e^8JS%q8#Y&L3sje5Bhrp=XV6hG(bp z5XOJ~2Z-xS|MkNxkNS@K8LU+;g6DG(Lx9QS+t5X`*i)fUvs1UNzqfIJq_;wuc z^is%BsVyS%U~X1)Vm^2pE7L*q-_J(;S3qSc;z%T-1v+T@sF8iO#mV{~^qIpEZ`OGJ zsmRoMy-G0F43DX{fQOk);*H+V;>kTu5-6Fyx{w$6Q>^4)J0Q@Y_^-ea#{U&wVvwIl z$MRxM`RdHsXrKT9l6y-ayojJ18A;Cn|?#Ae{K?t ztNv6NDqEhXjJD9s_0_ElC5O=tu>PqBil+c!|AU?R*5mj0aY)Rq&4MsNCfQSR%Bn_e zY36AcDa_v}UF&JrnDPXQLOrG+rQX)V5yySYYyDg;*-tV#Op<7HjBIp@>_3Zc^Vc`P zflgu84oAaNUte=o7jhRD zNTyEAr)VtxW0;VXv(YJ`I-+{|bfOM^kwJAZn?xQL_niAezhb8QofDU5bGTo{Ebou#PEah1f@F%>@Lxvr zme^mx-HIJy7>@o}i3-W&`nR2whNoR;L&Mz4*RiGxJ>DbR_{mA$j?yV(wyUeU|7n!A z#O_`m?M~a=?r&KBf}(ftwO?Zg^`E=~5b%Z;s`PG$y}!9 zC2xndZ8y8XKYxZ>?dV#GP1+=5(SO{Lj%Kw?5-je|Eu8eNMGS3-+U9abVd2PUS_@B7 zD%l-!Q*QP4$-_aN^woLh7vcx$*+X5PctZG05&WuGj{N^S4-gGHSndZCzvqBfE%K_4CJK=SNF$053 z!7D*&H^WtTc%4K+($-KI>uvM&%O!k+Uv~mFOF^1?u|E_`iGszb&#?CN7h!$D@t17< zv(|B+F7GAc0Vst)EZ4!(&Oot)fEABs!0yZST$ z59&VbjXQbVGgEv7?)ce!Rv%Nd1Aax5*FJ*NPt`MSJ7P{|)T_W*JTqCN1^g9YpFfYNxNr4TlM$D?mMfut zF+x+OjXF2E0ezHt0-=bYwez7nk4K5i(^tm#cI}c_TM;dem9wqivK6f6;R}5!VSUc1YO>;ZvT_OKFAr9r+%Z2; z%zFE?Df$Jo;rf&36J8F6y}?&54u{kB( zquGL{MJcR6anQaeX0S?J-9NsJDw4!p1H6eV?ZuLMXWtfQ`iCx!tlT)?tM}_o)t~C0qCv!WM0C(9LPXg!1 z8weZ-raYSrw3VAsZY0B`3R)i=(2L)z&QOjE!d6fJrIZ+yCx%N2iroOoTuF3skAIGi z>Xa$uFtV+w0Ndewn12n^FSQ(AUCVN9^baqhuJKSRZ*3?L#@|inpYhh#pPHZfb8cdk z4R=O=o29y9DU}OguPpV>vn+i$TnOfE$Afmjr3=%4MeBx7(A72p;sI`AUO1Z-rQE}yvjD@?|`$=sz=V@Bt*$kUn$#gx?vFkN|xDY zNOz94cG&u|z^0%e_ls)fRRz!lyzyV1U01e&{9@iZ7Q{P=9LZO@(#Oa&?%{^&O!-e+ z;Lh8dcj*lk8J90@(R_H%(?tTGwa!vRhM!hd9A{sz=V`$Sp2G}v0vv%G?p?0BZI@Ze)62WU zu!EfDj}yxyXmR3A-Ba0U(xe#@Uih}aPzYyR4X+Ne!I4gHEL$?wV>d)8Dv+@)pD|xj z7unlJ}6P^mLvKLAa=9z}6o|H0( z>|hb2Jwgn&kswUiSK9<8y*XRCH|rRFWlmA$Y}6#&&|4-!HB&@W;pj5FLatr+ zP)!0nYRSEa6E$~n#@PrmLw*!6LN(r_K%n^d4{$Jhu4-R9xeSR7BGiuj#ysTyjA((0^~V*X4T+U)CQ1KP<}9V5 z0G8y)8Z1+4HI;4Su!fQ%3$y-FhiNzv*tNg>la17tQ1SEjX!d)&!D_Hx`E!LxL+|Es z{F{0}E-2AihcVQBY=g;_D$oWQZG8ZS8;|#Ag(LYNg+QD=H|1>xLlS5=#BZAeSPOWK zK3g@@c|5^(9F-}Y@xHGRhxm3x2xj3r_?ih6qrxNMXXI<`*NLK5e&=>H^JBd>13Jj? zSP|W1t52BH?^HVth&n6=K)DEDAH@Me20#Z*>(>VZRg8Hvk6!3F59rP zb8OB)0T2IZof8oL&xxQ($mivavh*Ie=P2z)$W(TPL#UPO+5UXcWW>FT8`2IW+tQ~; z4kk0DXk|;aS#}Yga3?C^`j-H?nsYd%^5vd6&1Y8k6W~7m`o`Gnv&I@z68oZjG`9sZ z(Nzoo#p6M6NlBHv^GF;{g~NG1wwH86;H_mk7{t2n@ElHLJv1yL{_cS+{~|{<3^Swn z5W;$Q{8u#3pdXW7f}nXCT~>d3=OY5}LH;&6X^7qBh{=ajZ*u_vgh-{ci~jiWk7zUr zp>#TDAFcL28gADJHH(Z{T@OHB4yZbHacyM3^77?o8$tAn)F#`}Kd{`yCNFk4>YZqt z3N#wT9TXwlyca-2lc}J^UlQ`0==wE%o*`(M$e-zOp2z-C;@$V#yUr3f4G<6_iVbMg zW$ymg+w9iP#v0J;1j~5704Anp3=NGSl*bGat}jMYIuiDLgg<}HM7N}$YyZO8-RtF~ z9em3pc38BZsM<|ZBP#rSNOai}M)c-TB(Jt0h!=K-M&4(448dua7%FY&h8N^Y+ucVB z6#Lc#T|s+%IZ!mN%6sgJ-oK`2)42EU@~4NBi5ozES)?Jijh%KulUD1lCikUA_PWwR z7DR#YC$qp>Nr3O?;q1!G@r>B*c_Yz!6?^A4zOB%9Jr(0CaoF$fY4%NqtS%51e8RJx zUMF?6onqD=0rvX=A3&675F9zz+zZp-w_~g7DstQ#?QC1V9jwTT2SDjfZbNuU`APX+ zHNULCg-dfn%2c3wmVn2t;~Qmq%iI-ZS>MjHKHsf(ohCcOwWnM&t-3^sg#Y+ALq=3F zSeg-Hx((5q^fJ7e0MULeBr0jKL)SP;jT|OgSoXs;EO;xH+w8Leez1J=&f+%yK{L8G zF?hNXb~%$R`Q&oWKaqUv1UQeff6>&FFjIH}q;iIx{fO)H;GlE~fiO0rNHMmkAZ5%U z)kv@k2DusaP`+4}7w)?ZT7}2YbS;6WQ=Yq<8TyptbWjYZkin&;5e$V2l7JUOpk-?M zs^1W}ZDiXIgXS@g5H}z+Babpv!CM*c8mG$*hVa!6<&T80^PA02U$CHS;tC8;muBL{ z!sK!z3VOQOSQNYGX3K@5X2@!!{8o${(}?=|hxymM%T(XTRL*bNh7W`sjG3*u#bCo=v3yIUjcH;&RPEA3oMsrzkW-NMCh4g&4rm>HkV> z(`z}wSIWrL>2U9;wX!l?GOKwhRU&Zr0v6Pe#diamwe?QsgBwZ<_v z%+-A{l>ej2iY`crbLP}YYD}mNlgXBZ*ug;M_-D`jtEwR-^^}>fZkj@~g6Q)#_RA-X zARZlJr8njW#H7w+4sw^v4Wx=Ysm?Y@tr#uqJ6!~r^Cy=VLZ zxdAOWS%g3NWO*OtW5uFSAz`~`8BG|DkYeZx^4A`okacjOS|o;07qB5)KLF=%5ZEh@ z(m(&a=(S!z6VB{|3-90PU3EtEoxan$?}~`%5M&*L(m1Ig-lb2POjwIe!9AkNf;e?msAln zs@-Cj9Ip*j_IF6?d8g&t<)1~NN|Jga4$8oD+mW zpm-_+g*~!%;QemCmvD5%sgK~hbu-mI|JIqUJTPLnRb@`o z4j!GONHt+t+EXwn2}J}Ja=Tm!BO6()!j%lP5Sz}w9YlKiO0b$zED;%I8kXlEXCe{y zBI>E;qy2n^9#7yPeWU@mPWC8!ylzMh#DdwL&m^Jp!tcmOH5Mp{M zi6faAOI4U{0oY8Dkomr-^Gj8Klt?%Z*heUDuFp!8Wc&i?El(Lfq4A;U7dYG(_aJ%>b@%rf zNA;x$N38kinNE=+XCYrAfMz*-qp2CVEyJt;<=IILVZdjn3nuAHfNJTk*v%=)pz$c2 z*Ee(Y%j${pL`|Z>Eue_%0z>BVW{$sA^7+Wv!`b>d>9X%V#Cv8}ZL&=mk$rwR;MGX9 z*iBgX}l7E?ceX3>yw-8v!P7uXPB^Mq?eSy%JuPk=K9O?mg z@>mmXQ4IYCoWJNH+PMig1|YwiUa%=ox+uzsZ68;09oYS0CkG>w($?eA7+yBYG9d&OXM(vjz9R-5Hxby?r7 zH!^e`Px|45O2Ah6_U%kGM2?tKuZ(QhmW1S>7h1{6<;Rer{6eKAP%(N?HfcGM;WzOm zO^lq$0y6qt@sDut+oVcM%CwSQqOg-MPL}QS9bRBO8)q^9Y#9vkqUQ~C{S9B>!LWS< zE$P;6zk%9(mE*axh)wX|C?BmnUN7LoHA+0rb_EOjWVp(snH_SAE16ys&p`l05r(@FR<{@GETbM8(pcl z>uy(0;yPOuqIgAdLVK1CAr+gwrC4F{Uj^fdR&MzY{u&ZOJl6|{W85bj7-V3Uu2rCC zaH9!h+7S>zaN8al&@TcU7Q`m_dHxyUGRimLs4=#(8&Njg*fzL$qfhs3Vj z*J;(aS@}HNVm_Y4iVM93MLMdVD%*$(ECCR5u$0L(O{XqnHAS^oI~XQ;!9i!4DAIDo zWDIcOLDXvP8LfiLQ5wlttPU74mQPU1VX=Ld?|HUJ%b^_(KQVgDJj$)R+TTfZRY=6| z`IBaL5WA{z-?tRiu?;3PCmIpwz*!ozL+FJ~dM}msF&qnilJ7zgtV8;crgZ z0lO?-M{gcg`6B@32?rKBl--!*9Tl&AWQD&6ll;!2(sIG+V-&MA)shQ8_P1$XUL9*$klSkhwhf^bQ-1GP&YBulbb9#X8@fqhvozaTF zLL~@h`c+dbk|c|^I$hyL}f*V=LoO) z>7;v+Vkz{G@{EO1n6S4mf&L@7VRf~%sH)a8-@>reYB0}?NT#(h=MK{QFYZlpRyBXK zYa)9yk_j5%ii@6lhJu?}>Sz|y#YL;?q&I(+CjmDV_f7Pt~wB zmeTJZ!tjRsh*6dD>T)yq!Ka!8+fv$l??=FpC+2*m04$A$qMn1*lTW+CSysG%jcLPqA-2&CizxK}&jGbc{-Jj~dKN?|@hrh9px;cg#OHS{-F21ZV zI11K!TUBS;*(KMdjbN>3xXnJe6|Upy91UQZb6y4(VqABG48>toyoR!P{xd2(Mx!|r zgsA-tQh;g;gyHd~gLtq_9-VamW|RiIw-@N}44(e*DhHA0VR-F=t*2~9V4-%#lbp#! z$SA$f1t*hVpYHJyi~fZae?rlMe00^U27)=R@GMX}1&uoNv&9>k`%o?jhPJD?b@3v++spbp_&;^yGOV{3Y*Z&TLwZg-n0MQjZd$ zyr5I7QkJ_FRB_=ghR=?%c^;yofAOxs+UYv7$0C5h>ebxRQzMZlgIH^9##9?#*kf8t zC|Zm^iKFzhhSuOzcm9{icd%eE-+Ii>0C{y6jxy->!;wfHcFC?ey}I~b=GxxbR z7C`v3TxK`io`XnF`)?)+)iOL3=JoC@d*wZIo{?~u*TSunp|c{*LfJO(wkN5LKLxo8 zCmpe?IONyTbXfDuQrqi?ZqBcl&_)rsTN9m{<|-b6M%53jgIYFEk(Ox+2q!wHQwArk zTc$4v$uR}4E^;O3l{RzEO1SQRiep0xV(-JRjc+J_-wQTYUIvp8{GG~EMU^l5J8z1C zyOS7En)9>IgAN?Ff#iV*ww^N6CaeOz>U0^jgPO#?n20@^9ejk+ZH`pg-bkA{rUKE! zr0LSk$)BMV|HVB|u-TH+n{xS(KA(n7^kMX<^T9*7)XHSybL1EfMF~Kqv2%w3HZg|P zPBuF&o4MXsPB`cBS2N`%*ITzCrnC0&RmGoQnCMwXb$b#$Z;~A~=3Ad+TiKW^(HEx+ z@^>z_dpNyx%n#Sb!#qqTs!$wC?2Rf0F~7s z-^)X{<|JwIge6nLHXdZ(*;z)?!LSpEjbAJDJSat+=-s^SL@Gu@H$n&P$8$PQmTu`G zEA8FK#*I$m$w?@tgX$k8I4F~2vL;#MtyH=nTnFEI;61Owe3j0!LCb11M|G|qRv@oG-FTFYD{s1}v}kJ}xcodR z357>{i^6Q)DiX=b2JZ?aeMBs>D0XS8hb%MH7TVQFma-5#So1q4?Qfa{+p8NBohb7I zR?PW$^ij4DL$?p#Na?Zd>It2V0Wjs%u=NCxe8&eXLo;H5CezVMR7Gi3fEdCCTM~nX z=2}r1--B?nW}ixX&k!yqY37Jr{)DreGmc-MJlH2~-dy<5q)1>>R%t@}2p$UxJF&!P z25}wUkJ4wHb%pjc@Ym(!B6q>>wpsM>Z@*z27#w(#Ne6nZx;HCl_WkUHJ4qK_{C@HM z?m9fHCo~g+2`c=`HuoRJR+o~>z*tnQUyoD03$BM_m?q-e8R{_q3{A#TOGci-5plHq zekcgVk|C=-r~m0N%HM!G;D;CBY@zr|%EN3UH}J`@kC=Z9ZUNKk71c9CM!+MbEIP+F z8sW*dcBs;)1FkIZCf``LQSdE=W_p*{3#QvyD!2y?i`9T)^eV~5uA8KpnMOKEO*651 zA6&HLMnC=Re3iB73+a~E#ztxR42`zxJzk{YAb{LvEP4kP2Z?h7TYR6OSMw`@+53L$ z9RPR!^}JN);iP6k@kA)a{qpAc233?&oi61JCtE#K+jl50RUooQq!rQoA_SK4)c@{d zcI>9K(gGWF(6%a$6&9*!a;(5yhGZL&JFy8aYtfS)6#Ag3k^@}-SXquH89noT`Hw0g zyT)BB+giSb-VjxDIi<6vks#48sxK5!DZb3D?6sjT^5hH_T05`k^je)k315Fx3r2eb&)0m? zx!n57?5#ux^_Be1IvJ^v$t97FC4T+6E>ZPZ#YuUwIMiT%>eSg5^64+K0iLy|H&9_+ z(rBnG)gv1W$G@6VYFpuAWRHTHR+x~FM%TMa)l4osIzr+g(~pvU@;|fklmqm}8e_8e zfXos_LVU7r=x?rst!@vfR$=iQe{?*oUqWg@%v$X<7{y}w99u<*Qy2c$&_xj=WJyF% zhuBeaE#%_6KerzCc+P0e>**K?;dqAcYnmrZPQe9l^*iP>G5KscaQJ1eq5L(?B&SSR z&$Aqr-0qRi;{Dse>))0+cyF_?SXwnJ&>g?=E*qw4)3to<($ZP!mI6oT;1?wE#G->W zE9VKMca2MKcjN_EFGtSFpiS$DCTGwV&KCSerw&O>C*r{7`D8HO0;ET$#UpV2n2uPM zMW!kdFO|)^4Tq_=oU8<=9_j7-Rmk1_Ysm%@_lU$*6QtARkG0mx%{l4_2LlBc&e(t5 zhQlvJoQPd9`dBy83Z|7(OlES2J;N2~CMQh^$8Dg0PObzd4+n?1Ep(RA8wlaxlhE>FD; zH__Jvt`wHRc-6G8USz!OakWUcoOx9Uu|PaQmD9R-cpz(bvcD$uGk&>I&x@Uii;QUm zu>?+C42unp(s2aow1;Tz|2DjMGHeE)WlAS4I*UJm@xRkr_d3a?*B&2p5a9FV{A-^a z{tf~bn02Tc&}d%WJK{J+Zo8$DmyM~-+0v2nG=IJZc8^Zt2&gj^A zxZDN<1KT(`DfLP}CzXupc$m}Nro~h(&E!572?U?D0QG%#>NlvJxHkox#>-aJ`dc?U z5e|($3heVECTY?UF%;akcc#vjsb;)rJC2ruu%p4beF5YNur0;|<$uPMcjO5Tgv3Xj)j+wL8z7_lH}p`KO`k&|?Womi5r z?5E{^7puS{2WR{oxyiFd)i`T#5aX9GQfoiust$azJ?9POTTFSv-TH`Ci+!I;Iw-tLbG6ZeWnO1B^o3i#S$qnwx(o)EEioj=XK>ahp6_A9#+ zvpFH2(b*ojw!-G>yYC1-aHabNHoT2c03JQaR;I;6U5qSY%mSOUoG!w(k{UvPlRZInj{|_Of#ofE4HHFT)mp2OVUEH(Diq3-#=uX48sc3gwJrax%5S zqEC@=q#uln@8bAR|5v-JmLn9;ky*TjNfNq|g4^v&NlUorfYx5qJ7;8?CZC(3ZhZ9+ zJl;``iy|vpK8cEZ-_-u9o*%faQ&%P~9;hfry6>zD&OEEzcQW4rg%&06BCqpBg1ijflN@1UcO>wL6O)Zt&_U{DVtNx}^#^UV1>t@LT z;e6iF?<}6TorxzJt>ieJ%6b7cS)Mt!++&e_+b*dzh~TiO*(y!8sGfP>IS`qFcDxgPKT5HZ=!-B0m`td{KwjU00_>)fYOtN;_E2VcP|;9q^dF z2c#U#>(cadrJ68!A`LRb8_g+N$thv?l-i6AAg7$Iy^~`BKsfPN+^H%B_WXRYxL{m1 z+k^SP@42@u8=JSkqDoX&AkNzAPlHB>5b*}4tb@j@tAc9y?zvrMm^87)@*ZXlTONB0 z&%T$tD;lmQ*lx~(xw>ps)c@iBT~@9}L2mOZsF9>SjHv^)=iKn3pn)oofzptq!6|0V zH~-J1h$e9;i=Oo8HC{}j%LqecVc#*?GD~_lK#)yMY4}CeKq;w|2mq$pOl4Cwes{iMN<*wfx#K5q*p@wyrTDmqQKAyv?=#t%)|IKS zVA^p) zi2hT&`#Gh*=`fugRPi)E>jErWjd-<`0&2NO#+4EEB?YD@H@F4Jlsp10!Sb_ri}`6x zQn_vY6TYj|F492O74K}+)c@N1vfqHem*m1+JUbMiH*>0HWeLLed!#UffF=Z>SuF1j zNa>bXe9y9O&;x5xY? zVHZ!u$?);p3qC1>i(VV;nVuVtc*Ue!hQv>>byF+Bos5rc({~b*Vgd5Vg7`T{Mn@Q? zx#E-Fsz&W2N`92fe#I>u7I!*F(GUKY7 zJh#cte31=T?}}r3~sz+c_3y;b|mQ;q%2JpN6tm1QKqbC|{gA9<; z{&`mdo6XoL_s(s6b=Zi_O-Wfwdt(mA3aBJHpx%xEpTeAPx>f-h^rBgA=e!ItXURRF zIqtkuwC)dsKEKeo3#{Omo3z{|OMa}#9M2?Oxe(_Y{T_m;OwI)pAwF`j5R%uxRW&FR zmYxGKLE^R5*g?49P?AU=K1Opon+AAU z#%kj?$;%dXO0efn7ZH!E1q&6A5R9vK|}}BN(Vnt=!=ue#vDBPfmE=uek~i zox)?6UvlmRss*De98E=gBUC|y`C8{o6Aq0vKKfBNeBIi`3182fD>c^e^SlqKc0R|% zjVRqKE!kWn97m{rR3*b!l*s9DEyH+;j=|ukP@~EbB~|SW>E>`rC_$Jl1{R3dSz!X1 zLyz#Dg&T5Sdwg2umm_NyNt8wo1+<*S_QJ3N4$ zMDPURFXC-v+f5rWiR?9ZmHBp3H1@RAf}uRnUY4|{Qr+n#&e@{E!=x=LGn~mA(I6$D zenl$R!W7*rv{IYUUCJ*SbPXs^ZDfk;+LK-TG7Im&L)i<=Ap5T#u%O-V*XH(?_k?P4 z`B90Y6|R(5S)Oq+<2QHNRWuqL57NLg*q_@Dt*3Uo4(H#LZMgTZT{k|adcJktW3tz-E?HE1V#f1e$_=18hcy*h~7m~WxGvGjZSKD&`8Dl-9+W{V$W$l@L z%iK%*y_uY|hQuQH!KS3t?A7J|hD++y(Z))asofwv7gW9Q*kC}OpYH0{$BgqXp*}j| z=<>Mq;>JU(l6$$e)oS?yF#aOVWMa(j1u34QQdy-ZpYP0~H=3-q6PApCdXp;o`kr`0Po6 z0-)kfMSTNXHpW#iIUV8h3EIP@5SZg!{n5B1NI0eUkf#oa5Ur|g4>Kid^(wJ z@{e)u5qR@HgQ__Qk0^rN8%11})o%L^`u~Hfw~UHwX`+U426uM}uE8O=ySuwP1oy#$ zyL*BLcMb0D!QI^<$UEHU(^>OlV9lD-=X7`Ns$EsRTWyrs3=18Hf4`xzw~&Bhy4FW0 zsjS#&ZP!FTNB^5cnK)BIMFyKk(deUAIQOnPwMzYvo#V5jw=4)^M~nGr*yOX-xRYZ+ zf_tgQMBmotOYWe7ilmSFzvu4+pB$A4C7BI}q{z>@ zSRYc-qj4jbP_5N%IvLNNdV$9EK7@6D%J#%{HUou$bYIJh&^la;GKldthQe;AB zP3V)16S(=Xew`PKH#^eZ{)?v1fv2V}sZRU(CEaN*zVjy%BB>O@fktWqz0RyBep`T< z#Z0*RYqflFx+{C!TWBJ;s7GwLOi+zuk-E>#wJN|R9Dkg@mV7a&FmI3?FJpFy+CTPy zfzKT%y`XZ`j!cuNX0lRXk4U%EKjfNJrddFtd%13WyJfS-G#j?x%z=5-kUV#&;m4z7 zCbD}QEL)=Je4>`@azJ${+FBP}?UcwyU)$RIO8vX?NUj<=r@H|?PVd*F94MjFN zPnY69ZOtD;O}s(3r}2GxkSamPB1|;V>_tw&PufBDh}uKoW!Op1SPw@-g|cTOJ@eIcZ2pvv^*EBX1;OpnZ2-gUg7I3fID-ym_}2;|9=YN?WQ?=tAg*e|+ZX!% z2|niFUuEy^C}?MBZvw@jzo#zq-p-=VX!##VQJ8-ho5v0dlZY24MsKvo6bxYO`dISB zxa=2S{}g>HcI@5@S>opS!@;w|;u_*ZZL#N|{#B3n(3qz!yZ%zflu_n_3ms?oJ0l4Kk=)CgX*I+F;eyxjRphs@Vdr!F`g| zJLp!!UlM8Uq*VO~|25xF$3gz|?+kP~Zg;I1mm?-V3fo_#L9+oIv@c&|i7!Ya4_A&!qTX~#2Q>t!=KPcM_Kog!G|#324* zeLh@_NKkLJ`e=zy%hu2B-gW2i-_n$(K9*W zus?uhQU_O!kudB}T>+7{fgHzkUs$WyR62Kt%lW^7V$eaPsCd#$)9jUd@>4ED;iMv} z3E7gD2kkF^(oW!f(g=6-Pv9HM5@~P1-cFhcbsLi;V+X9Y`M6kff`Ec)xLZ1?Kq-fI zzkYre0R69EJ|B-yT=o4t_9X=RytvHOkJB?0Uo&uxikYZ5w$Lx8ZciRx`PakxMs5R= zM&i#Ut<}AcfLa4(hK+`#Y@(X6i4ucf*_))Yd{1gjB-FF?>C65ff*hzFQUyH%3 z8o_l-(@JT57*6X4%=sIC3HO+epfapPx~@CRS?698;A<6R=gj9GFQH4qVME`(FTy?8 zZ4NG;`*6EYrJ&B46?3yF{@uO}j0O2$J+KUn$q8-n_@!`2h{9h~O^mIv zimW5LfbSAzXN*NtCQ851JoRh^n3!$DD7RgY8RdSjqa4D=_Owpu8z+pCP_BGX2>5Hp zjAvLwFlz1515Pm$+-~@kYa&n+`#Y;Q(U@XFBb%^6_$AUqF7h=pSUiCBFB^ju7rx?| zZGrL|LMv|v1@-yWq@^E3^y_cK*fSml*!%iR4!qnS10>F1B#-W@FV zWeh>VW<;!rF5s{DX*ZIXx}3Jmf5~F@hfyoJv8X_NQ#TCz+;Xc>;X$CPAG;U*2J>!WG^@J*b5+IEf)lcWP$ z7bZ;_A52>AB1%3-*T~C?{*0Fsd%&PthD{6e;uhi(qG3y`oOE0ar1N>Rb(%iw4M6s5 z7Mo~tk7xd?Ak$Zc-KKfnAHm7ytf)sgv8HF#%N#{rCow^+uQZaqt~`8YBaYLc{T4y+ zn{@YX#`l?n!74w_X*139k&)M0iTVLN^-({JYd_3rKmW&w4tD2Ysd?jw8B>JY}?f}z+sT+NH6T?ookrT-?)J&!p;oyMrBB#Iy$ zMWrSpUart+MlcwqjS`PnF!oMBBCR~;OCG(^MEmVgnq>C7k@7qeBGUp`R0VPbic>n z7+5K?>6~Rzw0!+x#t)DECSemSjwUd5wMI>D4Mur@^j%IrF7O1BHQ!m7QbCf6eKM|Y z;CHKE__zm(H+yy{vqOcmg#{xZuXv$O>PI`9ZowhA?08EnTxGP03QSb`=5rY9%=qOUK&aUGq;IH{&&cGwVaVAH-ou^r#Zc##$HE#lt>FOB{zr*Qz)Xq3gSL{S%G zJDVoy-ULU>AH)|goK0?Y61}w!q>HYZX_ji>OcrOMuJTq-N7GT;JqLPS(mO;DoKAIX z@ZUxb^`1icuWs5b04%q}^Y&!pEAI!i8SGHyehp3e3K`Oe>+jJUXHl`345RK=N4_Fb z-&#d_pnqubn&1lup^Es6_s95gzY4&^>=SGSELvZx?)&S`6u2NnZ#)BYLJl9kET7W~ zH3o@-3URc%mhn3OEdsmtXqy-WA=R4-tk$?akpGHoXLATtU!9B#il4Q%W5B$DWY>iq;-lvUKw1cob1mq&8h;HFc0gYP=C90#q^rS>xX(1u0*OmsYpA3OdZ!V1l5 zi&6AD){Y-BOQOofmnwGJoe&gR*kKye9yGF#gWnv8ihtt{!T!(^!Nn;H#jeyTQ0JBL zvt%W{*bJ}es;#M-C#XQN6KXEB!Ow;lz`?gIN}R8Z{HS2cy;u&rHTU7ch?-~(A3&(@in{@?uXZ_2C2dRMzPIUjeX|XBCh!lN>@l@yDp@t6 z&M|JL^GCTt;XW(i{xRgYp1s;v%W}IhG)W~cm z{Na}i$5Rh@_8HijNgcsuF#;WTn;Ua$XiDZ)Phtv-+&$e|auuU5o{VCD_#6~e*6!OEZ=7ZHkFE9I4hi$pVXp5Z2`s_kpXIwyR{ zVwYHfzbs3MI(S5e{mECyuP3%012IGXzP7}8P%sBHN~SEu_e|jO7IFId>w3QlwGcNi z1bPoCiv%pxC=E!mqmhY*wy$7W^8 z!FOxX1QtvCt8Jxzp-VyNlM@Bh8XF{=FUi++e1lJ?65Ii9FTP0Q@_W09WHca5{i=QX z?#E9>S!Fr)bl)N??3joM9R(QqRfvI*J*$Y+E{8Mh4Q)aDB-i}N<|XpnC^7JqZ~FY zb9QB6D$ln?avT*N}chQqFqu_|5vwb-C5F6)%2WgV)ZNVk3Z562!-uo++uM65rQiO2(Yd_uRrwo0!?%TL zWC8B#TCA1cw>FMgEXT^l5gK`f6ibLK6uAP^Qu_dA5LIDvye^BYL+Rx}1XqPf8~0K9 z9)Z2;>r_+0DX`ui6=y8{PgP3rY;cQ=4^_<=s!5?bk3G`*8;^h;Nf4P&-;BdEuN!kV zw{ASxikD20jWV0RfcdPxPn1-ZFz8beCQlS5z|R$9X@gejw}zLLqFn!0sG^d^%6A4Q z*TAy{Y`4=kh@KP}(~W5CX?-_KgDRY(A)T+P~J% z@qQS6Tjs>T(8vzWQ;>Gc;YIo2s2PZHj$wT&Rm?2Pgln)d-p*;X!WXp3h#u3P_m2WE zHneahLXaD(Pn{qwbYhWg|sH5?!M;gC59^iO)_J$@(tBE3Ylkf}Qvvrj)Pr{Rk zZIy<&xA(L!{vgE=dGG#kmXFR61YpSlBW_y5uv9gBDeAtacC|zoJO>bG#|Q9*yf3U8 z-@-!E$#+H2&*>L1w#K<9#Zkn@82;2l&+Gg`u((}?^di0_ojWIWS0qQ*ARS=KUN zpH=k0i`eA;io-Mhw|3OmQIx4%|1gVp7aE(CyYOoLHnx>;#({1~jbi}zeLtGHaPoI{ zIZFdBo>xP`ZmvUVJY?*FHQm6$^=1+60OAKm8{{!JE^al_qwVj}M6EEJw&OjiWeeA4 zBBqD}Wibb$>q;yGx5HO2PdeF1rBUij*k4v0sgbZ-2LyUD=&R9)A4^jH^lgX2UKRPG z+?6F!D}yOOt-?fcgH3)?U{s)Mf)oC^uj> zLpCb4ZECqWzj~R8;Q7OdkI>GOm zpT`t$yX?o|aV=0PA|5rP%tu#*3O|^ub)6zU8*%y@+q0h$HQ=T!gpU_F?3RYxukwN< z;u0;w6wdGj$BAj-(bQ;9RNEmig{t zoln~u7RS{daCe8cSt^B;@u>b=w+0MUQlgn69)6`mKI9Qcq=ltB_*>4%E(7J4&0~MzueW$bN}zeGcWLSrJV&uz zU-pYlFtdnl!=^V9NF21U(94z8a#C+xw*|eOiGQ&1SB!CDY&~U=f$@BmE|kVVo9S*9 zD-9Zs(+opcDn*#_QwEgT057r2B1!*`~Z%Vm({l9gW2c1s`Ob|iQfh9!p$j!R0Fvp3R zMH8Cviyk4Ue(+hPki^c#wwc&M&x3EQt>NHR?FdQ*Cb>^* zPGynzet}CcF*D`aAKnOj59{F)p$|t|1H3UBc;w5P zmw_uU#OYji88!3&hEkk{hC!PFomAe;U&sI2H%z9TD#(Q3_{ivIbL_$TN%tM1yXaUYjU~uS zEmES}e*YVn^_^utJSmbDma+t0vk6wLO*KE*xjz$-ynq`eOw+NegYNI0d7{K$tp`Exww+Ye}iaP%)WfAru?u zNt#dVC!#>8u=~(f?Wvuf-d}QM*?FMR%-(wAwtx-7#+P@7WLCgCK8WP_@_R>DkroZ( z6=_=|r}nKM3R6HZ6J+8s_d)mKw2Uq8TuOjX1~Tt}=Qnocwp%kh0e2#<*eNl71bIMX zLqT6&xeYhx;QWGZ&G++_r#cr;qI5t%&8n?+;!ESHMuEqGSr4F=&AW0fxKg*-WcMz0 zxGx(r?W_$~H~qO%cc~~q`+irnSUz%UXsF1!)4MdSY8JnOIP@lN!XKg!{YbzXY&;fG zcvoT0d4n(2Fj45n69dMbByfAT#Aq;j1b5b!sY>FcQ11Tbl_r-q^7eaw8$wzufzI{* zIJ?@?j95^UtP@k$JEnvNdzjJ`7>u=>HN^)NJjg{w7}%#bv5qX$h4|m?t&0t-Rxy=h zv#W1TjFXa#F8z*KX~dAZH{qPd1Q;3A$Rxh3G}?6Y{H?ixC?F%$r|Vf*z)Fc)5efIo zHr|*m9UpIR(*PDpTeQ7=t=8{?l!zq`{El=|U|kY~%Wr`=NXc9TTOiu!Z8hHodvd6F z2uRWvBX$C>X!7cw`bR*yLd!fgpur31yo_a&d@y~yfNXh{hTLzL8vM&^yLHUMFuqyLD~#1#;K83OAPSeJt-oFzwonK zxbY>+%|pVnWfxx7@DoZwdelnzNL=4y4p7V=I4pEO3fLTliq6K=?uXJ z>(PHV*Ymm}SfVz#GyC*HquoY0s(v0!v|*B5rXG1ypMZHpFD%7`nN&)U8Ji9=S|Xrf z#s_3zqL)EH6vWdnV(H}TE`1aj{pc4``KG-BzCV^K{! ziBkb?Ca?GO(vrAHl><-Li+{M98M1EsU2_;ZfPH(moVu;iHt3+T-L$`BLTj3psg;uq z?mwKwobV9EK!8GJgBAN6FRI8p$4!Kf7KaIL81hk6BXPlW;{?JfX33z6*icR7&0h@( zlVvX4Kx;M&BMIM(U<0g)eAK@M+gH|+Cm1PZ{K}J3h;ujkEMG?IXphtkI80U*N3fJ! zo`23|{F8h@~WtgweJ}YBRkQ9Ir=fkwV;U@s=)sW0h27?{30DBx_K~8XY9DQWs_>T-VOSd*I(0b;? zYV9XnX3ehxdOJ~v-Tre&mg+@@}C6lyg~D)w^dGgrP& zM2`l(!0G^&UQvPFm)7bi*Y5jhvHyI09 zy`T_1jvF?N`J|UboZ+fy4M7$iWaCDN_MZIR&`(-ZB?n+IoW5YzFsk56hW?eaRjz#zo^SQwB~HU`1WPQq7NKDwt-BBCA7Hh*B_kB5fz{QF7A=A( zN1KNtnKFG(9!dY^1#eNw8>9GDRqcH3hwk8yH#g{0i`VdnkAEb*6WE> zn+EQ;yev2@TrSzXGQyy<*3fKpF+{1|#JBq&(^6D`u=8TIU__}zNNkWwh{Q+uBWGWbu>dUUu{V8DeP;jY7!R? zI?`zkIs9(jn3VYcj8({1>=RnGI zOz-yc$xsYcV1P;N=|+T+LV5~>eEOb;gtSi)FcnpKs>Wp)@f(YqFn@ECm2wf--IKV@ zh``oWulXcNf%YwO^22fzjk@=rpIh*slYkncofKygJymdw^#& zczsvZpHiLu31sBmBt5Kft#TL`Hd)=HB7!qbqgUSgf_Ljhajev71!c;Ef@wzxb2FZU zY%P^*F$L!S{7gc13I!sePi1F47qWl-%0ogRLCT~(3!qF?yq*MI+HTmTro+m}G5}cu zW{v%m9Qy`>E0N*NPck$I(SK7n(rA3ADz+%S{;B@+F=3#^G^U!m$2 zPxeGk)%qf-;@sg?=L!zpx~1ZfykUe{rF9L&`cKT5%t}958qri)J%(B>w0t?x-w#t$ zWnMv8u`TJT^OIcWyoA05NmN7J@JSn-i)~s=us*s_8ut0BHER}{)=z4AF~>|Lp*`R8 zFcG!GPq{(k^f_D0Xu9DqqflsUNs22fn-|3!1A=e=dXIB)?D<$q)iNR1UC9-r#ZqR{lw4;R8Eng^ zmZ+3+LDJ;~OQbHp;~agcNmtZhgn{TXevJMxe&I=M7pQ&+iO($Xqubo~|*z`COxIlAD4DVUHsui-<4Ll`2;- z*=V>0Cq#HvN!m`k?{%@aXGr%!pwh7fhYI(4eC1g$H+A+@FnxW^di0r|eg>lIJw69( zg+g$8x6xQQj4pj<^$Zh0CCH$4_Mcy^HIKnyvxz{!KQUw9Cm`pBDplh>KN;zO7&e#C~tBnkFwUm<;&lzoja z$^Aq1nRr^cLjCo^Br^52oHTHlMQ}95)r{%M9F_S5*g^1ztZ9?(DOO4U^*x7jns|Yb zliC!O|CXk!QQZ%_HI^Ln@`69iF&Uz6s)W_&ooDa370{{QKssF`(UZ<^#lJ@yyer^R zD`1C!DH^HrOL_1YA)Pu?c&~;xs>l!9sPOI5I#7DV2AIxwJ6(GCy&tnAIepwM_TWw? zN0@(cM6-DK4Hd;cHj2;YgH zEKGaYTQjek#3A_(XS>og=&fz}jb9)e1^Zkih&HXZD@1+p_DssbMC9DRMahL1I_lE{ zyTHODK!{`BY&|xypsJZsC&SVd0N8AMOfLz;99xGu_tOzUK!Np zNVe|xofeDmx@SjS1^P* z#xrm@!W$gK`(*@0{;d!I?h#0fb?m$QAe{a^>!U4am@F*#&0nECq&2;GE0diZq-p=Hc5 zz+M1moxap%TIe`VmR_^BuZ@4OnGxB7qX}K$P{X7z3SP_4i>cg}mMlezGztM8u77%V zW^c(nm+Oh`HQCF^5nnS&$ZwLI2YBk;Yg|UnRp+Dt?ldB|5qDEPq!4{)dq zU5JOAjx-B#I@=6<%X`+{cMN5wh85w-v**7TouFSKe|lnq!kG!2HHABCBMLpSAFM1h zEAEeP!i%eBdx!Pf86F=hV!3Qbhv}-7&eb$F#31`+*t;i2`ZHvy$5BezA5QcgYvop= z&nBF!IYowPzRynZ7QZ$od~3BMHA4Q*LnEzTir5#t2p?tWuN6WvgDP&Z4MgPVGK(j_ z4}A4e$5H1!szh2$_nWK76%Umu({w`ww@DvWuH3NhrP-*zt! z9Yi`#b(DpJgOjQ39KO>mhUi!^JO02zD~E)+q(FLqVps1E?3Cw2`j9#-Z=)$3^*KWG zW%+TdC5W8Ciy+Hh+8f1){8BEkO`I4BBX>V0E40-HE;d2r&9n#M2IJBExhVi5J-OMj z=Yng0S}nk-{$QiMw~Q7dELrx^`xtv4Jc;#RK1NyGFr4i5^Rv3Iw;%CF2{w7YrS>qi z$*|7rP{$w*RJ)bSrTVOY`iXlQq6m=j!vGSJj7F6}m|nB*uz*jX(>&MIk3bB;eEM4p z{2&Tt`91FFRXfXZJ0eUM{BVUw_v3E()$xcms1`rkDMputr=YDYM~$65H7@u*c1oy= z4zS#aW58V9eY!^p#KaNy^~Pdo>e^nC&lR9ux6V{o5s4AhdMOvL=6Q5GKD$ZLEP zCXcIOKoZ|M(36qGV%!U4ps9C`mXMVV{tQvbS?hq`E6zRv5;P%uh9!X?kBTLHYFnpoeal_$^i6f$4HOIBLY`&+stJ3ByTsS8GR+4S8hsIKC_~PW0*Ke*mp|H!ZJ1#T8Fa*+G!kgW zT?SK?3cwgX$Vnc3$cPkA85pumDc1`^tY&@96Qyam^xa!D)isYqwgVetEE8#p|2VJ*GFmMrlk)0ZR|aiV*d@|(=U$dA69RcXV1v_6Ms1={hwFJ9##&0@Zv}<|M!1 z?SA#?sCkj`mJtALf((AzT~!)B1;xU@w-+$)CBN2kIaZ@w>Sy%;Ez;!iTQ)<=h*wB8 z_aB$j?XTbvR^-L5d{T)6@=#45qRSt5=mLDPgoROQ1%19QH>pIZ3Dnk8fFM6Fu|q78 zt8=7=*F-K9)t%krzejU*Ox_N@u-g5y*~=}tvYprUb$xFbx1)#!0%t$CUDu02u?@`J z(bUyVT&Q$V3vUxh@2*yG>Lh)O*i4u~uCiVEwkyWO>6W& zFSJ72vlz*e7|DNLjLSmL)rZKEus6zTX^{^JLLo4K4@X)B}Avw=6Gzezg$i()a!d z_zeC)OA9ut6yblN+cv;>$pVq8kp|iC;l#iy(8Qj~)XY_PVMk9ia`=+udgs~oo8=h2 zB9shPe0ab>R!iGq^)(N-sh4pBynBCg(1MY@1DLQ}TM~{r!hXnojot&YNFO{Pdj`L@ zEW576qmvzO0+rvE?Uialw4n?Fzj>;!BY=6&Jqr+mWBe*3hyfm>4r#RMPbT*$5E3>F zkvRAvdUfRwurg0;NQwUj(aX{#fdXHo)(b_elSE4zt>Q-|)0jgp7LQXSAxEY=EOzN^ z_Y6bWb^uq<$Z@N`C^S)q<9xoy8SQumcA?QK6pjA&eRrcsa;=+^b~Go&%BHY~-xdC= zqwn31z~wryY&jK&KPoriB^nHah2l{vslOzzWHK8jFh3*S)-aJWv1mOkhW{u2jV#JQ zVLT)P@e~&=F8{D&CvAdWGe-U-RhoG=baAaTfMP&9>su0J&jHe>6_JLx3fpz;-8err zcQP3+R(N;A(E5KCMao?a!?YBr<-wGZFsag{A*3mu$zSIoi+7Pl5vU^4*^tpVPo|dU z+6~$w#!*+z;k)$tpf(4HPz{72dz+VxH%zJS{9j+IOMU;49MErylxh^6D;KbBZY?T7^%{PyuNl;SuX`qQ1cl)D3$Gc~RGg~5;GgB{-Hpp4K zH2jM(mAviV%IX%cuZF0N`YH zE6W3#(4sQ~#HscbLZbAgH{LJnuw^PpG$$u;kuzviAb;?CC$2vkJrj3ggM0MWh^iIq zSTaWz>F)>u!?8r+zEeKI35-=yjlefEi2Qpc`5U(k9JkysYFoXZ;L-)(#w+OZudmR= ztnc5^|K{s;+BnHnnxo2=1zt8Hr3;wD7njnrJ#I-Sw~=N*fNMH#wPH?luqzSlp&osp zjOBIotTsEUCUZ?K6=meE6f1wQe{L{YN@6(qC>;>b^2wM8{s3E{Y8p-x6uhY)n1Kny zhXAbv!~*VgT?uN*W%srvAY=uD zoWZ3SpizhOL_bt^0wW1~llY9b(mMDOA$l=xjhq<(G9?yQBLon!VkhiCc^lOd7&w@G2b$% z9NB3htWz?k{X{)CgzNJd*-SwF!OMZx2Wqu-WtOY@;^P-~QvvHn40G)&SR^JFkMd2> zzhBaQ;1+?rC1EZx!Dn|lp*_OveMg{V$>=a@FMvA{50uEFOpB+qftTfp`VYvYE+_}m zxJXdTl|;}PjhQY3MDT*({C9%6t$F@(X)`5xVVFjimh_PRbwb9Wg>@&_|gD=vBYxL?6UL=uw zod$*WGR}~+bAP4{%Ut%c4q! zy}BmWiq%DVQ?<-*c1E^asSFJx;&byiF&S>V6E8)&ZGCF(t2;+cclZW)6KyyCR!)60 zZX}Rr;#=`s4C;Xqjk`Oc33-< z_&y(e;;Pj3^)TgKt~*+Sj>xx_uSB{eu;7`TG^+KyGW#~PL^E;Ad*od`)EjcY>&6*Y>{a zRk&!IO{991wZ7(~yf8LtI;h;-T#$~0p$i*GcsR7_I}Am=4dCg(;s3*suNc6>?8=H! z_K#ndg!vJ7af0*dnSH#!wOTtmIxerSy3U<|#2TQR9XBOF#h}|47n4iqH|Dolwj%789B{>qFFmmd#`8R$)A7)*Z28GodKH|_t}>bX+T+T8{AoN}262|L;m z@!^kEd4dl<(5Nk=Ee?85&rb)!-0I=4x!ST!(FZG$g@};{iO~hIhrJoXXDE0fYEQ#{ zS@^=OAPnC#bU5vQi4qix^poie+1!-5JuUek;oSiOD=0Atov?Vmlt9Y7Hsf==8f zi(^b@cSfpukr4T52~Gh;E@cv%xFFm6kNNRmlI3*WbB*1cPFQ<@G~irCZUtZTv5_cmP6V)O!%c^^OH9dWZko zq4O6>KUUXx5hUJaMISu=LH4~`C&usavX70@RCEWSVa#-=jIs2e*_Q_^kNzufPTz{3 zG@b^6u|SY^xF4hJ)yoLe$l-Su;DsMdXJ_K1;EG+mj0OQqCbuiCjR*c zyel;3){{$A`JWu0WDogGHJ{TKSeM_w!Kp7sA>w(00SCpP#=3G&Ew%RV=TDo^WC3)a ze393%>$e+sF)vHVE~;2e%Fq!?1`YcDo@I> zE=7^+>tfRH2kuRY`BPZ4Ly;5FsJfjei8RN7LDbTZob-oXsgb6mx?tY_#seL?DP znHNj=^@Zp18(wnQ_8|>k|8q|0MbmKA3=`ag7DzZ=Vm?_QjBR%WjkO z+92{bUw~l^A~+_^i{XAyMvY=;TBV{RqQW{a#*4DW_Q6`arp zpzF$)%S+ih1{N3KPh(pk3T||KLLchv9|Urnlt6}%=ieiFai0VWM5OM2Ni_i3Utv*E zNY!Gw?ZesPva-lp)1ezz5DdeB2=jISu>ASiE2&QB%LhrZn|eGfBD?{Mmk}6vV8DHU zc$^{XWM^Qo?062nm?Sx!&jvf$vBZBvZZSg3!Z%PA6O;vEIw&MPoOE(=**R2Ms<%{J z{`>d(^8)xh8}`IORh=(JUb6q^R%N0m4cAlZYrn_TGzp#`_n|z%Lud9gh z6J#ZHy}V5#`L*J0nPhb3xTONW9BkayW6oE$wgMMga#6_Z^(BH!L;k@|0!^ojJ+!mc zTDXeT@qu~gKGqsbOSdd{Y%)P42^*6JJbgZVYG)tI3~hNai%wfgMKl zh<2K|c=m$KUS80e0+K>~GkD_lzE*Bz4UHURfrMveEe_WfxLXz>K4UN4m2Xgl-o)93 z;;LuuL`QFbcuTcA@ypfAqGhng3hZbtR)3@j{e`b8H8>2gnl0pSyCV)XE*dmFISbWl z^(S~BY7bnU;=g|G(f=0vBt0$r0)tcS3luvFL>Iq*mJx9wM#@o?Vsg4X^#ID68W&cE z)@fk-+N-d;34n}?=pja;5L24#<}H`w#ubNjEk%j_N<(BG#oncm=%FPvbZ@nqf`FXvBXJ50Vu-G?Xced{zx-%ye<9st3-%;B3wZF%-ns&nSwp7-11$#ycz%Ma z?i#wfciy|=Zv~r+eJMJK@H#rn{ys}+e`-RLwV9qLi$d!lZyO{K&d9Rh$F`DD?r#lX^u|%^vCjH#dP5tJ`lSi{3I0Ar>)0l%X)aZqQg7AxDc9~%XD{t zmtW!tCMrsSi-*@Uy^KpmB{40XnyQ9&5;?^DN4R%tDB(G0;CC4;n);jI_=OHSzHg_F zj)oJHDp=S&)L!)?U)bd5R%-1AbBqp+V zS%+sy-AaDa!!$1V!mF>p5u9h(oAQ|HWUWFJJ7TKM5JX%4fd3=xS&fW9y~c`hTq1Wc zH8=Jt#n{@!VM@HuVD)Mr#|6WB?>wT?vN*qk>R#??^)r?Yh}SFVRtF8+O@IP&kBf7s zIO&r{`UDM;L=~2yT$Nukzgny=RvPJ>95*MY&~6^{jg%K7o^G&7-iqoip4@_{#&5C0 zatc6%0dHx{1p-WAhXM>F$;@Mo%{a1{$NypMtpck0-mpPM;?UhC-AH$*ba!``bVx}z z2uKPX1nG`LNOy;HcXvqhZGQi`n37 zf}jljGP8F8^K^7S!iMrKuU}B2#WH=kK6uJP(Bn4gg7%L?6H}dPWQtKOkjr&2iAm8-K_qwCW8cC-bUAwYy8LQ$(Xo+-O@@Qp z*YUM+hS4gjfAUU5PhM_LkvC19SAZ5PB_`t^L|W|mCHFFfwMg| zJ~|rP;uYQPA;+K(VR<{-hrRv+QFPxPT$xtY7{upRHS3QPAGbOW%y^zVlxEx&@MGf} zS6uFA;Xj(-BB`m*PxlfiD{*)Z4T=f9W5G_d@qmkKE4ODgT0f-P*Se-fR8~(`SPvDP< zpOaG4KDPdYL@kZCe@r#dxkU-LQtEs9HqOpr?|+>6j5pl^=X3qJ;5t2KD`(nEYLHmI zTn)uyC|N~qXsewry|+x0QaW(@*A#emG65aoLh;OHB}t z|M`>T`bLe^GV2SD1@YiFD(gje#H>Q2{tT``Sj$AAzAWk$mmV9oigFud!9j(XO3rtU zB}_fN_sF?a|2msd9JLt^VxGgp^Ff|D9FFR2b|bBCl1}J|i?nzt!3MI$JZPer?+s1~ zFy5P?IoPnI0x_Mv9C=85F@`hLTMRk15#8;lv0}s%zZF;Sa4@1K!snZIF%T}6n48-1 z6k1Lss=eo`%hJgj74LRztU2!9#k4r}Zd79@LTa`A#=t7Ss|&)><52CxB0Uk!cGGXH z9SYjQ14pa?&1rhjZ*O0de~BuzoqM>dJB*95VD`KA{Ok9Y9Q#3ERT^kFth?fA`@h3% zZ6O-Xqt1svcgu>z7#|-jymJ-e24_jWf@GZ)288!_;zj(TW5NnQN7#Ba_&Lb_mmk%z zYrE|XVOQwzlLeH#LA(Hq|Ne4k%MNsZRkh-WfWtU2ULes4F2lrStRbu>ojxjsV!SY2 zGd4Iw=P@!rtb~j9ITaneZ>%b_+5DIb_0j?dz9kaAezmEx{`iw~B_e+d6H2INd}*?2 zgFH>Kr4Pn>+sGq->2+6(_nOw@nhBKHMG6O7*QJiiAH!SoK@fqG7-D$o#hc-|7$LCr z$=D26FXSh2d|$Uxm*ct6G5&E(@>giEi<2VU_52II#&43|j5w4Fnd%u}A)9X12ax6x z7`M9{?MqU)$=W_QiR%@)PH+GgAgtIX#rlt#uY&t_Rq)0m7pmq08g*8-W!?ECJNtxR z97HkV{s;KsjGk!e3tdu5fuC|(A&y^?e!x{9X@0aP^bjvK2SgUC7$1r;CZ=XLnSUu9 zx&`V9Aq%Xv;8?Z)wznX9yg<_=@hMmZ*LFWeZXc$ z{Hcmu`mI()ySJ@92L9Qy&SFgrtMPa9C!ZqZ5d&!zC+@v4GR{qRqKLap6{oV5=5m9a zi)#kqLv~f~xy$Z6>kZb3k7MW0N$ezOJ9VtP)+TNOrkAMI$GFY*q$l^btcO1MMk3(jL&a{w{yKs(XC%P!>H3BZPJE zxqx^*8X(19#wSwBM#-%JtRUcm9>@_5^8Afs0?cQW8IECs5GG5mOeCu=8%|vciQkX5Ie|g(n5ui_VX04NwSAYU5%P9rm1>vCSP6tD9e zqK!gFH0g2(Qs-2ypGt(w?!9iqtlGKN+FfBY&pAK7N43VHjROycb>QmE^!}YOQyE=n zzeRBhs)()>*Q{g5VI~YrvO)fl6a21u=fOC(`CqFEF?BOTDzd zD4G6S_;&n9sZddVNX1u~p*(ME+f3T>$PPS3Y=%60(W_tLov$GS^XNPJrQjMn&c8(} z6c)w1P`=L`KUHceAIewbV6vmAptXLPoO?eKp*m&vR}`pKA5MAFZofbv<_xE@*nz6= zkjq2fC662-pDqi;l~$T{G#n&)ul3x-W6Y4jCm7U5dj2nz_fR*)Y?sYXH^GG2MQH8l8j9%9ec zhk6j%5Niw8UqC2Y0uzSGXGPEWj%`L(a*rvI2D(VMCNwGvV$EH4{amoS;L)F1Vc6`J z_S4A7S7s%Q_}F`uZ*?wZ6HV~SfGvzAEVt>2yF}wDK7EVny#0-u{x&OwwLF?)cf;dp zt`GTnm$+k^wEAh$1`Sti@1C&Ar^+sAPb=Dc9HGgUpB*EFf|BCDr^A{6&td9QQMx~P^sNjisJCh3l& zF%w3TNfLKL4?qbK|MAmbB+LequGDIS3A;_apPVQZ+({@R82j+J6^x z5?~)vd<^7DIIVU66C6ihul3!g705Xk6uIEneFQpvi5fVAY>Z1n$*YwKxBsBRr(n?X z3NI?V)mUGKYmPsLb7J?=vhK)FUd$_xH8Ya?7MY8~zlvpse_(wpAAhmi*c~QspjMfU z$?rNPN~O1;e)<)lq)fHjC*XF-k6laSCYsZ$gnwZ3Wvp>~&Mi;hwYek?aV8FxHA5w}|&Ns`g|)p{4#^T=2=R z*;ZC+)tq@l@jo%fQyrWnUqaR*8BF=6C*;%_LU5EQWksbn;y)Hd4*Urc%f=tzucNls z6JMdYvi#pm2Knp@2XHHr>PI+Ym*x5B@zI{Qo6LJ{@^HnR1YTMudU=@e?>^1pBiq&W zA6iW=>*8N{yfPx~FJTHcyNg$Dl)7-MlQKIZZ%AZge&)){yMn)uJ-z!Xo-7aRO6-ta z!?9;KlMnoLPyjGq`Skm|cZ#ooWz&{63W+zIhT)L>Kx8;l#@XAM_pi}2)PwtvtC(Q*pRu%!6#>rF(Q4}<5>9Y z;SQET6mi(sek7luYwTVP6QLZ*$WrF z?L#{Q;T+N_@pdJkn($pvR-bwh50*?i!rNLTVfYz2dGjElo8+V zK}X@qHVF&0AqP^%{&aotmIDjGylF8K7~e>7{@)?nvy4nbIzU#^^j!d<^W{CrDqLf% z$jJLRN0uZqHxRAIoJFls3?nY^?07Aza)7!0E`5OF8yXr00P_RH=BKyD@qiNsh0^Cv z8Tbl#GW$98oGNGkxF=2492c8=G>$JWA$bi#^5u*szw-Nc7WM_tC2}g36WH|iOOmgp`$e`stbvE)2L;9KB zHkG`!GF<(UN5>%zE6BGQQ=Bk0q^W{zgh|1~8W_X{Yu|(E53+UN7FDV?Go?{yu!ko$ zpwQOU>zwbv3bk|V-~FvRAapEo_Btb)-N)`CM-GQ2>})S@W~pcEy2QTIdU%H3&UC|P zw!+RiSt##EyC0}_Ed99WN6++Sx_1-i9c!P8U!&pm;12RlvH%C$7JYsldSgKV_s1s}ZUY4ml3 z-;Dxwloh8F_vV6SU8yq6%HDD3=)b&)#!o0ez7!~TXa}Qm-)<~VThn&6I-iyBUlEL- z8x%q(Pu@Kxcq>@?E}VGZr?eknv@?00lW`nGTimiIe*GF+I&!4n{)~ic$tiwf$lxNp z2un4L6_(O~k@s`1s=>+ZdUZplJO=pap0ekm=V+9EZp*s4)v+U8q` zvcwR%I5WN3o$ZZj@();E2FVdCdv(vI%hIe$%Rzw@JAz#e1Ty8Qa$8y&;(*^V+qiV; z#dG0(v{^LpgVf5Ub4ciTW&T|o;8irHaq~YigZQxVT2idBuij{|YIr}*?0q1)aH@?W)F~d9=D(Ch#G`g0Z zyCnNzGq~+>joq#g%fp{_SN}XnHH-AQk+^Hb5+;%5G?h83E+hI`Zm~n`WB9*L$@wR~ zyy59e0j?(Jco^$BGr?7vdX?riE}{$-8h_BCn_AfB%PCA6ud8JZ8zCX{W69UobZ|^) z>M-81_Ud~W_j|MSLP6cJoERLOjKE{e3ZwgkqMKN(e?R2=QN}+0JlA!lS6lJT#(~|ds=oUpIi;4o>he@ zw}cfLP?jprK9W64a4h1AF%DR$u;%IU0%g@@<9~I4OpLz!p}fHa=)MuA|1;#Bis^45 z|IOc@`)JATFW^5JwaM3dy(Bn~_9sPk-gM)hYcL1;_Zs5qj%QUW$%7snRwn=I5+WS9 z>hqnA!ioCtRLk`GFNdBwSgv~%y$j_l>rBLEV&krBC&%OFR_ITrQ%g#*(RS@5)WHFW ziTSpB6|~96%GAK5s+p-UG?5^uoUa&$N)PXRQG}$+R6i6v%8SlH5j+K#rE=y(D)(a( zw_r(LQq;@b(9=@kh|;}6k>*IOT)oAVc4(kod#|v0nN$T7>}0PM)1<jB*CV>J9Zl z31_K3o8M>}D?j4rJ7_?slxj-OsLNSk)lY5ADN2yUKkyyQ6c2D8z;g2a@b*#-jah?HlRvTXWh)ppCMFuZdUvVuCTqd9dg~6%Hf7O3%O@(J#IrHaDgP+KDa@vS3)-h>5?W1yi6E@0J!O!;sNZ4zm%7c6X9Akdq}> z=bRr(@)%_z*X*;ubu(VxI4;}V5rpuu1q#5IdcP;pJLEhzotbI*q3=E>24a({OuMd& zUrqUicP!vxMB)3fWw5UMu$W%1M8^*C{9dupXI%GJTE({b5an8)Lym~yuh&32BTHfA zYcIrsIinB?YHkFbERFPcM;zM)Q8ZMCQC(%L^|W?<-(&T22tK#_;mja#1eHo2ZCf-* z-KH6~ae2FPmt3KphNd6?X#hR*RAI1r{H>tA$9)4C);_unlLKkmK>Hr}2P*hJ*M&^Y zEFwe(wcq}QT+M;W@nLn&=^Rn1-T-=#jePrv56$7qS{&a8TdhT@pY!z>ldb9VnDKFL zy%z&!N5y@>{s|HH`XjKRRL_^))t--NSkv#@nv^7un2*k-k_6vV#CBnCm=5~QnUhvq zb$WPFFWTS#Lri1wpz*7%arr&~zY@+7RG=IgD5o~*aEb4y@bFt~WbX)a%SPr*g_hpY z^BIZOe|^+7C)qpn@6h7^o@DL~oL&3KY?Vlz*%G8=DF?det}+-0s+Wzz*g6iA^56Xm zG}Okoh>|1f<_3kMbqp zfy|KrsypdxwU4*eJSAWBe;HJW!gp_;Rxp#jB;W<`idz0~(z{oNwAeFQ5%%(DSX?VN za6B^4nQ4vaD`3@HpcpHK$+Z#SfI@M zk;Ii6hiKL7gfs<{&yU#aWb=He2Ku5R~$Y@ zpdRjV7td$54tdg>4;+`_I`1CGbo!<*c+r$fA4 z1RpeA16lSt$4>QQB@uDVPEMn}a2G!}78_G4InktI%JAogPJ z&f*nMJp^t@Hv;|gl>W9tFA4@x%lIhL7sOT`$fg+iLA}tL+m~7wn+4!famJPF1j5+G zZ(5v>gROL)|M!F_LBD_hz66B4%knGD?s?_qqYAH45X?0owKJV9(^=*qqwECwx%x-s z4W%;hi0nYB2H1i~|No&Vb>>UQD+84CEPl_pN`**45%Frf; zTJ2Ba0F+`0C`?Su?N7>ql^SOX#w;X$k7EjZlXK!Oh^~%&oYf=#n}uh%J(uG9jSQ+F z0^2YOCDhv+d3XP3yw|e?ik_?Y0zUzzDbz%xv&g{^cETEupy&%w2AzRLApgXdTmRv_ zS2ftCGlbWWTN0bWXCsvPc0RPlW$6DBL>u(p9wAj!^2Za4DsD^UL$EJE_q_3hzrIWr zjm@AkPCOMp$hk{T$VvSF&=^yNpmmx4D>xRK>Fj(@C7Zl{2ozVJQOv@JJzkId75MQE zy9ayHGGFk4`#s592cHokDzX#Y694fv)X~Irmq&-wiNw0ljH74V#6&;gK&z^0vQOPD z=qGIKjRauz2OK{%v6?U-qPUAwP=?Fg1C3LlZk(h9G-@T~v9{h<=dS7B*LrAmazCS( zXtQvLVd^sW4%!*Ndp1J4d>Wq&g8E;pl5*wkM(1VZkqVYx`v5se!m+HHnAs={2m`3A zz`_=4B6@d;GyRX4vHZ5c2wb!HvYJl0#qUM0t?l5aB36 z<@UZANJ@yDtt!mN6N7#IXJ{#^syZPrxI9y1bDfrUJjchjh93`)E=fAjaxZsC`VMhPiZ^ z{;&BOVh(LkjFHq0zIU|d`wJ7BAZ&=PhM7jM@!ojt93=6%VA-vt%dz9uS{W>o_-^Ww z_~LlgLIC1L6faP*ewsQ9lH5#{z`)CK54W76P10@K*cjpJ`7+IUxf_I!oC2_EfL~z? z0urI^DoCp2e?LIVpDS*{rQL&?|H@2zLUx++>8z85noRvy2yxp8Q54!RByp(jW4Ry^ zX^SqUgUR&*Azw{h3wUrF!4HJ8Q;5vLsWROem-)ZuJ!SrX+hf`i(X+!irO0N_MYgSE z0it5GD47)y2AiP`B>x5F*MFol53>HsROJg*;9wK7t0eU64;MKiHDQL)-PLn%cmi4v zy*e3ngKrkjiP?W^{!INAM2UI~2k7o_K!0l@657M_IS^d((i?+B^^LWOEl#Svd^QA5 z|3G3hNu4^jwm~7bmxx<~xjU6Z{8;0i?^QPcPMg#kj0=T_x=y2n!O}X|j48Y&IzZ3m zKb|sNFOY6ki<@AIy`bpIQX$xtKy{#3SD!~mtQSsGPDz&GFEP`l@6%KX9ek&)OoZ*S z)sVEOm$v6(xuIqugZwQs_OI|IdfQ+SS^9q*BjqWRE5Ji#VC`2`A?P8agxTVF!1_y@ zoOWMGm+^u9_Ju4SlY)&Zwm;L!eN!?N7>tgUfB2v|wus;X`vBa!|F?gY5JC#1Iuu~Q zis?y#0~+eDyE2UXuP-n{p(RNQzm1@;XOX{Mg^gbrEWQ(jRwM``l{(Zzs9w=sqn0gM z`5160O7-89icP&~e7AG0+XxAK7k<-$b4bYo2-Df}I1`lE+`?Z`9^DIQq9-4|j2KarwG3q6PGm$Auhd8}tG%DiFv%Df9^tF`1Z>>l@bXrmX-U ziT$;9VeXTY6J+1Vzb%>nZK2Ah8t$A-CMK|i&u-f&N%viD zrU6(>ube7#?5yBrsjZzS&4?*kBjx)G&j4mmbcXh7^lMHvq-v=bF^x;@?n&Bg(Lp4h z=Sf5|3_4-y&$HIM+n{fZtK5U;HFsZqHo8Jps*FV)b+dOzGq-voF=>Gc+_2TFbI!8T z^KA7(4Ja|~5U}Zclq^s^jD}9th>3`l!4mKlasR!>BOxQm#+5@_^s4DDeEIvSEhTsy z2cKJvvu|Q|yka#{YpP?k#-=^v7Wy7=(NOtj=by(x_B0&gyQW3p@|$rz)FD4(t_JQX zl9xnUeN)x~3B7r3OOfXoO)Q^&~ zG^ThTiapz?ee82guw?r;qC9`Y5h1bnLCU0>_x8pKMKERm^XAIKiZHQ10wEggY8NGl z(NFhO_8{!!DLI?(x%+8=a<%Hnn&(_)-Hs`@&)x_(FQ&%>pS8ZWIZxlmAVFGQ@h@m# zqw9K;dos-p<-sB6O@i9j%FG?8$TK?5FobqMd~umBR`m-X3?ZRyJM(=StiWgn;s1fI zgiQ`-#LVtlgM-;pv&Q2tXTTAWlNTI)l>&S=@@;n|sv>T;wzVi%&`5y7krjrvx06kuALB-NQTC6CR_OyF8XbD1^+Wx+x(2pRDhuNpU-J)6FIef7ujX z<_@$r=Bed-1FD&V%R0l*+S((rlSIv1cq{Jgj?Dd$n9ms^xf*D%$o5_1l3R~(N_VeA zt_XD|iEHq)_UADP@^2l4fa-yHZ7;w3F$Ey43@UFU;m$S;(5)H}QTvc`HGf$jVotQA zySKqT<3!C37agbX(r=JQuWVji+O6NJ_hq^`I(Rk8khXrCSpCCEZ5_IRBeLP`r_c~( zk-n<;2ZU|@%b)Fi2%ut^tq_P2@SfeN?klOPb9owFcKkVZuOl!=p`@s({jf1@0?CQX^jpTnnvnbl?A*a z+Tci_?EkrfmTECSaSD9mf1^c>k;4y?xeS);ERx&u$PKKC6svE>ip*RF3u-W%+jc4Z zmh`e?jbirBq*aCr#2o%D1r*2N z?k%)@4tXN}2?tG{OiI4CI`OWVAW~e!_Onrk*yEP;%(!w^ZY-hZO zYT@zE=J}Oy6ihRJX7&6h??_S>JY<1r@dZ<|?vtT!nDcwg#Gh0|h`8b=zSDPvs5KqF z+PpT%Byu%$zS$#~{Wyw!J%d-yeeA)dsza-r6EQMrZAWCAs+$ZcexId321hLM2x2+# z6Z&$;N^;sUr9rNH_7!{4JToPShu{InW4ZeI`wFpI`V{5*Zrj$x>NAC)PyN#Qz`UuV zWyMs1hpeCZx)d?fnuaoPAQV!`--KU_!1j zbhuQnr-s_+E_9-izf|;OiPcY2&`{Ozmp?pYF#PC=PJ`S_wajh~-FaSnp)4?%7_|D} zEi-K6CdB0a9`hn>^~K7y#O#OJrshEa%Zawhr)sTks2%>%t{_Z?N%}PWRvk44VAm_98~Vd3+H%UHmnS7n`=iW-l+71EBH z6Xk&B12*VDqOYFC^;I**PK_k>qByc{eR~!u1^FK5=#umf2X|ml^l~3go81iri%Jp& zv-d}%lU(xM*r%uXVnQOLFeRfH7|bxXX>k^Zf9$`I&VEs{REbt&P$`3RHT=Zvh=+sq z6;m*%5W3iyT@pWM#Z69PCCFkLYHV4R2Td~({QNO%tbDb*&-n^646n)w!ow+8GQItY zInP>v97RyP#^=4yew@w^U)$*YF~KQW^0VU`SLDlQeQu7!V5xzecm?f%-7Ok_l!>{C zZHzC!b-LC>o;b+A=aIvRnf(>8qVMHU!B`MK>n4jHCldwQS@FLt;PtMO_Y!1nIKZY` z&Hgr`S~Qu=XB-TseK&hvj}It?5CT^X`{1KqrPcL|p}zEHbn>q?nE9~Rd$7(jmx-4v<2b)IinL! z2!SG19Qj;lk7h}9r3%x`CJ{!%-~*n?-GOZ5Bive-nry~+sClmU{7kpb-raXX@Z)d5 zaBU#&i3>$^o@W=_U>Ey>@>ao+xvWgQ!SMlY!-j27$wT>PFUtcCHmRhR^M4L#shdeV zQ)zy#gF6#3Y0QDQ#F-Ch(XN(1B*+yH(*@3I`L8p;Fh zGe&D504F6OCzWeJVDMY1e08#|y_PGk z1XK_m+V!ns(p>W^86#lC149laj@GS!Xl!VM){wIpk56oHeuX6|X8GJ)bYPD7y99YX zBixbp`*LWo z!%U9ntLGsu6mxt3JP|>ljtova7RwPl$0cBFZLr)AX3@NXyHox~WtO#2h@zLm_fPdqaL1QgMRSfIbEwwECj|rqQYkF@j zMnmYjxZYOeJgT-Jyw#VpP8pe^Uq&faYyoCJwK*%MVr3?*0iJaWAE~Kf0e@fVL~@w* z!^5bZVXKX&`x_n~H>$o^LU=ehUmzJrl#V0XAWd{_3wVu7q>%0!8zYD%;Oy#)MRdN{ zge4My5IsNs3yqDX;&ZbO2N;!?@dLbC75buAy9&m3cCmTla2peO*vk#&dfccthcn;u z%}AIb*aQ!c;V`4LbfmPV2P|;x06k5P5-V40aC2WxR%3|HRE3<6h|`T@>nI7A22$c+Rd1qERB_G|s}_SpV*3OJoNXMjWS!&9sap1G9JsXvxvBMjFkr_d7dS zPS(Q}%OfFP|AbDB^n%0sO0LZEWQZ9dUr_&@t#y^_{c}0nC8*=T6(u2gKcj~&BFF70 zuR?|8H%%U-P2q12fhOm0#Wqcr(w2TN>ZKJqt7yL^{EYyE{9fB^nGSm)jT4 zF7SZ4T`&6jS=3x5{bu*0?A=Q_8<+h+{QiE6^-KviKm;F^-;@9M&O=(eFowOsUMGml zAq3*t$kFz-JVxsiL)VoVmEUbe#KE+~`FbZSq*=2IKw(|Yv4|T|rbFPjRDLTGd-)o5 zfhD~!dMhy+6^6TfaDmX@_l5~dzv6>2>pxC_f!58_q%I$yK?_U#{!(h0W!U9%d$Mny_5NbhTk?9}q=(A)XrY$>*VHkg%wB&yX^~bX z&hB^)FwFeqeBVS8iSQ6K9AZKOad30z>`tgCMas}@`?Cu9Uf^{gnoS?|6! zP~Tr~W;V4WBgk^nlG4TY?p1-_NUk5KMz+3ipoyuQ-+ghTF1#j*+EUWM4IHEsBJPfA z`%~q1BtBEMq^gvKKaI(NA|O2*>JFG&ej%wP{!{(@r0Y`U1535I+I7Vm=^k)OU!F79KFfm2l`(4iSJo|YAB(Ge*@tS*b zg@-Bx(|PptUVjs+lkM*AvYjdKy}bOr=0lWr&(K|}Bza;+mZS?7$ft@Ve=kSe`H74h zM`&&i98jv5_Vw~>$>Rr?Hds7CUzFVuneq{doT1bw7Xwh{;RhDE6CeBZXqOW?7dwng z8bIxV<9)r4zPy|P*5`#G5u)vXsyzIXK*05G>bIxia+3i;XQ$I-k&0kQXy}Dp?JQ8t zH&s}5hv}>j?R=@KSHPo^CT3zHn=Nx6~$Xq%# zx|r^;9qT6gEc`hTXscOq+^n940;QWhjRzE?dwsRWZy3u7@iiuuKb%GB z*5eXeYo3|NYh};i*^yTJ5w*Wsg)eoxAKkQ7XnGZO!{fb=b-L;eRoQ4&?R(%IGG!h9 z3M+}mJVFwk^6yDQ6cd@B<7XQtza@k$KykYnJ%c<)&Vu7;7Bv>((x9oQchaZJ*{#$F zD{MrG@o81OHMM$8nrs9CTa?6ID1vL z$K!QPu^q+I8t@PKxEq?>Zo4`bQMXrYD903$<1cDIfa&2FXkMCiL2Z+tD$IJvvj-pPFi| zOZKUenk)eD=&^Xug_@A5XNJW@-&37$VQiWD($a1*gts|zkaT|Y90Dk^&Al=90F9Gb zN1*@y%rI6o4o;BN;tBir{nb&F;Z$DZn?p{0)rK-I`+4Q%rmdiS=_s%V6MNh3-u1P_ zeDxp7mw&PL>*h} z;gDDQEZ^3FGPJ10BF9#G3;SVmM+|b6A(3*`3+%^mkFJ2yPhH_hHE{dePI34QI3^pn z#Ayq4ED%p{Oh@zwH5S_wL}fieF^ma~qfe$+Cv~PmpMEtrK&CU2uP`YJlLLO(# zA!@(Jj32IT{7xz#A8r_Q8|3@0Pn=KFN2k&0$v?+FMJ-`^?>%@ zY=&N1z0`7Ou8?%8-s$Y8YE?~?WUXZ^8c79T;I^g<1^$8Zn^HsENIatXwnXg-&+>Jh zIAfwF`6RUuHbL9ouRSPVh+6?P$WtJI^JN!9^;aV8dPz2_yX9HmlZ}G0gEhRI9I<~; z6vUW6W~CYq(WAg{x8gJ&e4*7fba(pe00E~LeMfqWHL&-iuSRk8$cz-Hn9gQeJA4Qb&4WTN9M%m=fuH_hXGQ(6+6?T@eS z%BjipSu9TERV3towR>Zx)cJNx?s&luK4^JqO62(HD;~EWEiCfScssHZ~ueNG%H!~0K_O4<--GNVlX-g-tb03IjsR5UEI zOlulw1MpKUDd!}O08wIfiEk$7<)MnHtyogR#@3;twO#|3Zi;*7JkuvKE!eFtkd|c| z*5WKl^rIglRH`cVH!-@vuvE($O}H@HDF;q2qMTTS!7qP6$iymTvYg4~8B#8jz27ya ztG--P7RvXw+1)gQyx+Zhy_S&>RTmn$0Ye#D;h11+{eZ}Kz6`fL)zYAWViV@-x#-m` znEi*7MTBUi)6xmWDGa;~2iheONORs*5aLMV$0^01Zl10r?|cnBge`b_q>?}2goQKm z?+CsBI^nxYb=NN}ilYX-u{q@6g*N(f3td&VisjCj^irWdhxX;}tBQL*X2ewo^)ZoR zYh23);Ws5u#jWe~E6BG0hw*##xsIncr>v%O+iFS{$(b;XO=&+hzbomF+*E;u4XKNW zdgZ2qtnT{4F}m(}(sL+cy**1*9Fi%ov!+MK)oD%EctZzPn_dk6`dgGbU1IUb6p7Ll z!`BhnCrTvwAx+5rgZ7Vspg|(O56>|6cByn~efi>W6zY{fFJ@T>V+k`!H8shZ2d+xz zC%XWVXHc=)729}v0wW)LU3zqSC?eweSoR~2{}aGL@dh&BV0oO_`zwSi&UCF`5kg_e zwLM_TtBs?~O9@79W7Ig-C&^q7sZ$;du@~Fp0oK^7QeE7^wCA4ib+iEhHbc@xPI3B0 zCl)J^-VQH1uqN{k*_D6WogU;{sGVuc4miRuVy*99@o9g&8cKxiOX*^y6IY@zAf1UwEp#Z z_ZL4QAvjmX;!70K;|R;_qhLENN-WWy-$n^uD}9}RMx3siQ5%1TDF|Uz{6)`ZNTWjD zM&Qh~vX2yq3sr8F*TG_u;qjaunM1>dO3ix>6vbnMkn$lf1^@GlqcnJiw2H!Szw}S6 z^}-NaIrLp_Fey(i;E&PCde^1bK<8(shPp1C&hP!0?GRF3Hdw`V;P;R|^Z&qVJ@E0` zaT;FktoMXJgr&AdmSWQ5%ZMR`6cpVRoJyZ8diKJ2Sb{zI`SrDWo8Ns>Ccr{vuB)F` z^v@!3S;?kW%!<&dwRD*d$fr}IHhXzm@Oms1AR?~a4e>wXGP*v#NB2Bk7A=rXxm%2} zo3CPCZEueQ>LyXOt2wh^`;Pq8SCX#raf0!j_8^|`Oy+=7KnPHhnNCsK$)_Lp%)MkJ zBxv(sw}uxg@B{VuiJ!x1X9Eso<1r=EgN-Zro67QYLX1wkzaY05@&DY*$t z%?I@Vhd=u9AVoB>3*a;mx9p}{q2Cl-R75*JKkrzfjh&TyJc25d<==Di5Jk7_#S@RsomKd{I74;qi7vFcMB_ALwDPkYmFVyqec z5&t+B71^Y{$_^`sa-Vx#M7)2Pao!T!%!Z1owIY2nxmRV57VusE0$G}Grjr1b>UfB! z`*`m3K=dPXh-GwPezV4D3)v1KNt~9j`rt#qy4%LBi%z5h9D1YDCqQcaCfQUdoo@AZ z&Ej!^>W#+j2W)Td7LRY1u59U{TpT9RVwNw_HkMJv5G8#px=*VsgvZ^Jv$m#$Zd}mv0-k&Yk8wvl4@4CA^oFsxCjr?vbM+?nq)vpKe{kdO^$>eqA z8ySJdV_i=|dAc|Md)7e6H4Z#@hFY~fHzNM2+pwB(vW9=#V zFLr5bP9>#$f3kz`15KY#6}={G?@+3REZsaoHTNWkaZ)9vQGe=`bKTz!|20!&PVR-| zjdR~GxZ~-Fy+-u^j5ej+`gF4s0zxK`IHk%vql9)$I{eT%PfGG#OWE*Iv8p+;lp~yw zOUVPh@9AleCh)``G{exZo(LLZSv>U3G;$-Tm?LJNGrCw8tQ{7)I49bj3@Z53J>a>K z4puNYhj2sj7sL#w2YlHHp zAE=Kf&M2g}x9neX zCm|tW{QB}0k7okm;Xx;}<+qX(KOQTM?EhjWj`l(1N38UG5l#>9O;t?E%R{kSOX9E}pGJ3tA9M~TJm3YIm|0{$gQ-U9 zYpFTD8(0sSF1Wtq>hu_L$%7_f##?`Rk1Oyepg1eQ6^n_OJtg(W6Jki8GjSW#a2RUO zF>>_8BHefHrqDv#rC(Hwp2xwbe;&;4{&LO>wIi2F`p~X;l302=#i9=vg8h?-jLGLY z(}TWiecG6R5uC5JLcJyZ6IhBnn{Sqw)5tbmQyOP-^v`5C`67+81@n^i2P*m1BHkZc zD$R1kaAZBtt@F@Qo<9vVQT;H#B)Dkje(~zoSqIG$hnuXX63UUP`lzE7#*PnJz4!uYSdl~J(IP8m1 z?T*UYuZCg0-P}Uvn_Ov(zWdv2MSO$%moJBl2&y%qonbbxUqAK#@Mbw=H+6Qt(LHi2 zGts0r>#1ORMen|EJn}T!;5@qb1ycrmGKp^CkPw4*L+Qmcj_knjx3ATpQltAgNx!`9 z8wtn#)sZTWt~1LoL}4*DNoF|?RS47u*@m(hVl(W4fDtBP$H**BYJ)e#w1PoeTNw=; z3gBHwn?H2$E}LYqB9P+4#Bx6nGmFX-QJGg)>vtHIrQcyBDTI!qML;q`Ld^N|01pkq zFiRN!0>WHs*#^S!12r0xr?idzwQ4Ses?ii}F~q-w+=PO1_$9mFCpj9#C#RV8wN9pb ziHwsHBM@p&cKF@JnFwD#(s77iWZWoMIx_urW`eVVoBs8lzy-Z7lMU`_ceMK$-a(m% z;`i#NpEDtQrtfiF%f*>>Hwjh!9awi=^x z+9XZVq_J(=*s&Yiwrw_z(b%@FbN6|_aq@4EY}}XDwJ_J5VFd`i{Kw32aF3@ns_}3W zSwcg7!4s1m9z>l~7PvSeQ?&)8&L<1$n^Qvzkh7bctOwIP9*vPmhGrHBEt*8Q4F*G4 zN>!Bso{x@?spWGW8bqG2PY~G5+B+8Z!T(`kxaTmYk_IY$tzc$A$i>Fg98}3+ucm=$mfLs;*M`WQfm0&?-x!#FXSV+cUmhSbHxXpPi;QqX$nUsZH ze}6o(SW^ul5Z-8FASfpX$J!ctp@H6DwPPW6WM#D~PowEX+Euf&9TbK&*WYNR@sPSg z`VXn&ME`g^3?f7e8BUI9T8)*2Ca2&QadJs{VO;=U0+<&Gpz?UR(F4)RX`wWh7hwGT ziDff*h8kRq{kn_Gk6h6g#OhUZvcWf;u<6=pgZVwn2dvBJWh(W6i79;XOn?|-rJF&G zUUA!Ny)jZSfx5^g+{X(SdZ-xlxN)ok2dQtNx=iIw*k}au@I)0Rn0JgJ_luWaD1(zd zyC?dX<&k8!KRpnUKpzdFDN6eU)U#`0pu*VzjLeMb(k!aF`r`e5nc{MUg`VT%l~L#3 zzP^Eh-R+TVbBodWFr~G%vg_@U0O0oZ&c6TwJL@_Uqn_y`D3)-^;H_~#B+@mqZl*QIn=dbZF}(oW=VU zGFAoKKr^@zhtCq7zd+%Fx8Z{#UV^CN{x4IlLK?h;(x4jj#w}T05o`LW8`isx{y4~0wI~Om5YdgAHnA9 z^oL0P6NvUMDbhxJU)f)SGw@QBKWL!vzB=IkCPy8JA}Gi~AZ$Di8^um4MCspI6JS>= zL{yC`IzmPP7DWmR$cI8h!N6<}gf|r{kAbUx6Il}9eM_jfJ^t$iEDMp4cf`Sgs*>H< znpL=mx3lZ}4bZJi6^1bdZgIOS^eR&x!w?qMYcW@<2ef2{v%Up&&qrUxKaO{KJE8Nn zId}ikLIGJQLVkZ_%r1c)!j2YDU2@=NqQ)1O-F#2G*8W`;>IDAylp)%Tx4=l~3$FhBvhr#`GyWgyqpbo#mXdT{NqSY%IW0GncS8;Q^Lkh(5C?&)0fvM(&r!6+$MCRZUJ<&pYBR)Yg!pD?%BZ7rXRVxuQwQgKXjXSi!Aj`g}=cby= zSzODzi|*7%tC(On%l>dH)sB9vU?ad04s%Y)4M}DXUSXT+m6XglXXsimX z^6(rv38oM`Io=OMmA(rr@Km9kIwT_jY=j=Y70+_t<_l>|Q#-iU|KUD?MD8S@5~XKG zE79Y>7?;n(s-l7p28(R~&_GYwCF|8zu>Ed*b8Js-T3H#g_4;?6vZ9z6)OOEDqMxOP zHsRD*%T85}W>$L+BZNJ<`F(pcTHRfKng))vxn_P!*@MiL8X~o-~%A-CUnX@}z6FBo}oRj-rtUl*!@jD)9q!2bR z3okaBTJO$o$2Z58%A}ct9mfo_7p3{r`Aqsn7AU1kz|Ed(6Vmp5UZR;UWrb(KMhv^4 zc75|dt^Kl9O)etH>ki4sWz~N)SH`aO>C>m?{pXj*x}{w^BOqJ8;$Dt`&6Ka|{rWT! zuw!X4nZf6hD-}z2uvkY81nm9Hq~IMys818;gHAfM5pS7U0}bN*gr~p#MV{EB>1^Vj=G@Q@K12E}2%81{P&0 zl?X_jrrXE(!Mg^Mz(qp~MSawRq8l8gv|Pqmw^59EXDSHdO8Q_!-RleAyko zJ=HgWD~qr%8bCHMHQ0a`G~NQ1GFH17>)?KWU_!2F1KavxmO9!67+uXPh>=mLBl~?p z;pKgyb-593&U%LplmIo)I_0C=0a)c0EW$iCUnIeQ4x8IG)oE;I#>t7;O7*o6Iu{#m znj8)F!mdKcclX9ft)W2)^eqB&^D#iGA&^ZpufZtk0N@*{r!kq*5vKWW^k9@DKQ~62 zpl$ex(W*M&u$XevUo~!(ke(!5HkBmp<*~lLKHI{y+jB5f@f=O0HW>1pb9|uWerl3_ z4zuEz67Ncz1QLHjfkr>JxvBbO+rm|xHv9Yl^0_^7syd3$F@izY)D-dBb3h?i0%F!| z8JF=HNxZ2hZmL2Y3Zq;b8l+LUx${he^Yb9G(P`GPA^LVZXf{}i7&_>L$M@44waW(G z{8ouS(O={1nn*UKp$Htq<$xb+} zndG~3IKsCAYSsR=siwL=Emf&r+CTHoniCkDbzW!r$1=Q~=jOgQTN z4Byoz9)p$x_LROP2hJQ}RH%&-oQ$4+DU{KUK)SXQY7>)I|EVHhweyj)>u-)t)kY47 zc23dNxhA`-Lx_`Bdr%q5=}(BCL)0lo)+|tcv1ljYh4~}6LY)25-`bwPJtOx zAAaWvEL59*jyf0eKQ@Wp+}KfTKA{g(_S+gBI$IL8(Z4?mhsE9#&v>iQ8{nscTa->u zXG4}Aik5AY(Tw2p1^__+ce#GF>j`ew75lCWp@<8IwW~z(XILuWm@26=QCxe}d3U9l z(9S176U4C9mIE+q<-(!}yNZa`QRLfb_h{tmKIf6zcz520enHnqGRCfmC*l33mGq5A zM6_2C<;yGO!iz2R9#(a2jbBBDRbe<4pAv*88R}fA+=@Yp5 z@KB+w2QsrkG$ZQxIv4yD_V$~`ZkiwO8>#7IG3Z7u7rKCqC8Jk5o{2(G4|GfS501u2 zyba2YEcVN~7GXL63=_ZSS zFF!ql-aW~(?~mr}e29eQMlOfk;Hr;{AD+?`$dUvZt~B)#k27mn^la;m(EBExg6#5S zLEKZXlMc^~E=NYVQ^S84&3Nym9%GGJr-&d*=;|YpAgs)1Rf-)46PjWTjVL{cA9coK zTmE7*g|gfJvfv#GPM@5-JaX)OL?onLi+4^D94z{9f49*b8&l-8+=MB^Bc<7gFAx0;XMF&rO@a8ZgS4>VQH_G zOtpl?7M3U<`PAdB^9*XV3T7`IY>ysfiT# zV?5`$(VzhLoacwc8qfHCmfGqdqt^0ajS}7wCAsXMw2>z%C%R?^$ZF*q6wm=Zx-387 zsjw(NjYTFE7%joEQFwxbMjnd-FE`KM1MrhWrt`n0LDSQ-0=Fi(r1|n5W#^9ZGzymx z!onh8-8--RM_JzVWU^|1_VlQUv5fy8>Qi$Q0Fx+yUv}3O2tR0Rx&9IAv(atAqI|ZL z`x!+=|30~e-DVwsZI=9N_og^>YOMwnz(Piu?1{>x-QM}>E z`mcpK!zOt1uQ7;+VV}Rm2|`R-Z;;d>aWpToIjo7Xe*xdU6((viCnVWXoF!Ib^6ns+0`jzN&mkZ4=}g0% zY#!|)g3|0|m*tJSUl}S!%MH3NZMJ7&^Si2$2iu-gMetidV;`d?*b!4Q+o$Q1I<*lw z#`ywtON5xbyH5Y~UL=d#35R9!XDr3Eze6{DUw*?Q+oks-#TLmZOfFFG^n8;!;sW9C z%k-y<7X>~>#FK~yA*$A!f{1?$OR2TeZlm}UCCh_Bp%UbhQ7`0Sg86W07OLunZxDDj z1&pIJ#kMg2_5@BB#J^3E@NFQ;S56uck<3O64NXvqPF!i}t9niq$#q-Q@;O#G2dyb& z8F(-zIZ3R^2|K>C-?X&8#)2an)gXdd*0|Z-Qt2-uB921FH{# z6Z9)9O`vDalR>Sj6Vnb$%G6y}%)}SWGKs#aHeu`GS>XpqZ+uM^#C3o%|BG!hx;3;mss2$ zDwi3vM6uG-33luq1TC-i)2YT$4kK0#NBuht+HHM>7BcEDX?2KqX5v?73M1t5(fcv z!LSbhwjD4`@4!&;&~D&%1%b?tUvn0TtoQ^G5hJ|&Iw{+AzEa=yk}9yp1^y7pX=@E z@q3?N6^MA}V!XJ34e035^3TM%yGyyeF5;C9WI#a?pJfe2S>lqLsOK}LsD6C_<;-^$ zQ0e*8@<b&Acp(@=KAHBrG^G% z`1@Dj`}`Q=DHvr*Igvpvu%vnFBrRBy``yMIov6}3Nd=P4WSrrqUpA3pv1KCG*?0zz zmD%DiI9kITjZO>;w+@5j-^%5)sD)vPrhH3K0Ir~7QH zo3vb#Dd%&A#b(kBnYt0zpg3A><1%DRVcCC=fcHZ|F?ik*U#Pi6>y*@hd6QEh>AnRC zx)oW`3H@g&Ako9o{7On_EiD-?cBP9a^ zc-V19|K^$e-<9}ly5P_D-dH`zFq0on%(WR0DD$GZZxtz(Y&Cl=%O!u)I$ODvTx;ji z)zc%r>5nviw^{$1=GEplll7cA0;kvoBMXn$Lk0{8_?0$YG6fMD(9u15k6!4Qnj*>U zJicF#+5Ye0f4f;iy|+JC>_>~e(OD2ZqPBJkF`DOgA+w7j^_&JnscH=tvtfx+$ve)` zU?hpP$Irl$hn!(Dm|(;e56{5N*?`;{>viPaPU_b?#s*kyx*X6T8WwP8vfxD22ZoB( z!1ZK6X|dLN@r^D&A%cXfY~9UOZfFv*Wy3Hz%=w+kL< zw34J^F#GKVUge74a;kQ26TKL$`}`)>bLqOnnajaO=AYfAE$MEvB4=s!!sj0A`k*dy zc}RlgWF8Q^nzS9L303TAXT@oib!@Q?x2*Yze0}7c*$E(lQP5PrB{CqB zlauFAd($bG!wLyuYIne+x&MIid57iqAqoQe7~usjMUCF+mCJj*GH1Jb++Mg*dwNQX z&ktR4zKW6ugqYz-^a{@@i6T_* z^_?}tAu9p)YC~RN{o4Hz6Y`C9aErxw-yarwd%it!UUrg=ZAOB$Svd54%}`pL63>g> zGe-VSCv0mWrxYc6(9I@X*BN$Ir@h7&to=~ga?|C`9sR?l4P3yygLSG#&=Pgkdxr(r zzjN_MU9DhAmP3;=n!>)d)G&(~K8*M@7%8t*EH8fWHQ9c5M3@00C3^!CE#BIe(O~PrAH6T71{+ zUrQ@uZ8y`#A%U#o;?uhy+Y!Kt+dBvJ}i|qu)U?DOS4OLY8QK{o@}~YVEz$>Ux*rA4%BWly_pN zYq6r}F7P$i+qN`xj|3x(Uj%zaXL9lK2|jiT^Ae%40HcNEbv^+%+4L#S$+$lwqjbvn z0Q|RLqSu4T9KP@Au>R3tG$80IHyrvZ&td>32=Pe1BS6)ov0p}Tt%r*d!@TaW!a4TD*MZimLjvGm{0K-B;{;+eRi@&VqEKEySvh?Z* zit0F+5~mX4Bz}0_(dWIN`EQA+*l++u={d{-zY%AYS16GUjc4b^Zk>s1p^3-Y(E!pE z)Q(Mchm9hK3okNr!ekLlP7-#uQ6E;a2c=aQ3afq|c)^Ie9IFcacp8taEZ@7mf>>X! zj*zjRIWRAFwNduWz-SKjGD?*Tz8 zLh9e-0JKE|_?$MYd!#q{3>3pBE05@>OHMM5V3}ZK!Y_f%cc<@kz<;or-n#G4;T_Lb z#1E!W!iJBg3t*xM;C!=mj$qaJoZWkerGfz90fImmOu@H6*h-KLgZ@8Dye%h!h!8x= zg8rULW^HbOBwDG#h!XIE-64sX)(#b;olD=K=`Z8v&<##2xOogCZf!5ejpz#5xKFaZ zm_#1MYa?P7IgA#ET4#5dEc{7dKV9d9=knnU@8g$LkQzJg#!Ss2PS@+t`jGyIJ_Ql` zKe&I7k-N9nP;!_lVF#82wx_Iitee#^Br>3lV*QAJ&~CbUgCr)nBbtOSL#{TXtL=xs zAmTJ)+<$7`jTe46dir=KB+{Z&N^pKB)9>r;#p=H8@YaNNc?Y!yE{+|W0vX3z=)wZd z1e_EFn)bs&+q}?Eepe>#CqJ8Yj1|sSc!8%lRs)II5=HdorlKg-dV^qU9kwpV!)cM| zH=xxVWZx3j*6fMy{sZ|N+)C>lKBGqCbXmHOIa-~iptR^em*(G;F+>vnq#(h%4y4U? zKqT|`tFhbh)EYL7nUYEHuNocEeHl|tBn~lKg(tt6QkT(ogAt}EA=+&^h_xu#8`z zJbjS2JBsv@u@;qlAigm-Gn_>;#mNb)Ie++juI4u`UQQ;C@ROy}+kT+K3w^Tl*7n5p z2Ay}JF-UyMMnoguqX`M!I327q%cBL=RTj=-I4(YH9x+s|9TgM%!RZ|&NRB^}IDveZ zEJP*4@QqBX%~|}5?SG_w9_cJqQ|V?IR!%vLoy{V1@Xveq zu?M6OkLg5NO*DlZZFGg1=5W{tF#+NrKzaK!cx_#>nQZC6w;<== z6&`TOl`RsIMa-2x6-Yp=(+&fLJY2gI)G=tFNf9vX+cu{hOwJ*e??{}PT5ij2 z*mf@Fbz;RKq`q>V)fmag4;C;e8$*P44W!354nAI><|c1b$My8>ho>QK%ra7azaV$e zn<*u4JwfPLlK?2^X0TzmL%v1+BLou4i1j)-3b)8k08!IbH2>m!Sj8OYS$a}U{nSykTlHUx{OIytkNcwNxle``zj6R zd~(x0Q?v^RHO7l6a%Qm@nwbJi(4+eTRlMo2&zAY?UEI?De0z&WrFgil*pkvP z4*s6(*G=^<%#^*S_vO+2H@=l;qLp{tU{wl<{EA=jVY4;vhAh7#)k<;}T0HPlcRmMP z?d#ppX+uUuMcFc_LlG8P_E|;83`4c_yzf#9D0JX(MejLOnBImq--Yue^Ncv11-rM% zw4@!>;&i&<1E5+WlVy%#G{Y62U6_jKqh;<2o|Bekn}&M_oGgC0xU@=CJ0D51`7#y_ zUW7x5NXb@QdOh!JhZVDe6`it&Lo-!|rBNGk$cEGl~ z-P7U+NI96#35QOw#A7Mtm<%~Wv@KM-#CnJ=oK_5HifWUY|{;Lr*mympxKha2j%Quz3~4*c_NpvfO;_%mv(NAz6Hg^ZM4 zdyNK_(jN(sW52S_9;;HdmUnXC!()7%Shb|Td>mHSd?+_sE_?D@zTVsNHzD{Ivw*g@ zx6+Y40%L8%-;%==aA}hLOEF&)R^$N-zGV-$L_HL?1->OMq6cuV133{J2n<=67oqC* zN70^)vuz4)-6If%L-LbeKtn{TMELf+UX26sN$b6DzznXGK`;Vc zSFfoum!X*e{vl^(@MCBiOVO^!t=JwYSTFmuldvJufkmAY71_YK_oy0y0YwY^$WiID zqGzpR%CPZKn%deOV;(C#(zB{l%xaq>BI#Bxr?V61teWJ&BM3vi74WiE%g@tep zDg~x0V2lVmnYat`7>a#!(il17RX;-@UYkXI0YU9JS~!Sue5F%6__0R1TD<`aH?=X~ zH65XgC=O*vce$DUg!;^PuE&C<$Z8@P_W3)#E(Y2Vx8nMC@h@`z_~$iDo+&^SLK|ZbZ1Z>g``(}F&b`=l~IFN)x zbNn^ckcLkes_}Ab(>$Uw4q3R19W2}8WZNlO>v3s*!8yXbGXbbm(ToOpCMQ*{_-a$Q@l?U{9nO#Fw&tfPx7OP%~p<7 zd1gmu-;iX#q}wYk7|zV}-5p*%o8RBU{;#UEgn@O;KUYZ@@{d0hs<%EQ^k0i;w>T5s z)madZN|m7TsaGIu&F7Jb{=q3-x-f{6fe9)_SoP_ zFWV3bW|eC@2udLz%-A#s%db6!KSMRv`}8%S`qgv#JPqgFK}oy0)$_wq0u$oU^ev0+ zoRe!&OBhY?pZY->g?8?+>k1C&lQ}GQaVeEztW*w*Amdp)6>)1tT10kCQ3zR`x*(b? ztuma*7s^B~X6EuL?r-sjQLCA;=7GE$yoF6xQBeji+oC@d|Ey*DuFi7Bpk%=}9h+Qi zo>@1`l?K_K)eR#v_@90}4W7?oQRRfkZF2D9q3iY7C-QW+gUgEI>#2%?oQ|W{Yuat7 zF2`DD>N7Us_8*iBcr5iHM?u zNH!!j!!TV?e=Z--cs!fyV9p%U$YwbZgEUOYUyq6uri6~V+p5p%V6m*}Xv*q`?(>41 zPqX^>2?J0u?C^l9M<8Xx)H`>OkXAq;rL>sW`&0$T5<_d$ho3A&7z0+j;-y)kMMU`*Mq(c{Ww} z1Kvf>Qzbl>RvPQ&n1?JE$ORVb04z&#@?TwngvFba{bc?p*>vt$m6&&DO-;>a0~LtZ zr$zmND}RxyH!zq+p=R3a;c01Otz!rXfzkIUvH-KJe{(Rk(7}hY)KrJRryY0r05u!Rzaa?fZx% zb^#ZwgkD>I(aC@@K)|?SQ%?N;1oY((r+%h?SI5SrSMCR3Dhd_qp8=?_+JP)Sr=XXg zi=cNeMnEjv4ee2g=HJ9#r7c>uS_&6IkKyo3lks$ryH8+cFb0v_$x;IXA+O_x&JYYP z$utim;8?#nJJy`f@$tW$K{=%0d^b`&Ywv!zf50p6o(_)X+CN3u5wAg{b(zT4n_3ON zfQJhtbRneAB<+}d_W2TMlh&5Id_#|Fbm;$xZJf(8*SinbT|$Ie^YG(Ct@EOv6Oe3| z`~U~eAdI|wy(8-{#VuS_kFuAz(LTMm>hIr5s?lB2ZodzYf@PCZ!MGh9l{n`VcjWg= z>^nU`9L;P+IDaTwjvZ!;7d>9QmVYp}ky*IIDx5Ph7)l&x9pR+cMA3D=|JwW{ZZK~V zLWgj!XHRdT_cSpiz=F!`BUacLL92n*IQW~`#57NyCU^XBhdPy7j_9p{DV5VI4Csx> z0ayoJ-(FfK(^}iT(wgJ*niYDEhfO(fT|RZDenF1?oXQ~R2Zb~ZM`-4Umc?||eTn^1 z-hF+jGXOkbQOMURu(8rr%UQ8kLd(pDoUS_q^0DO(6jT|ilMALYcVvd4At|Y64 z+fIIEo%8@YXd&C@4G{n3&+g~H-Y4F=(U&God36Geg#I1~ZF%#9O*GuL5S++NP>SA! zv5>_0p<#K<*-O3ns!QC)Qx3OtxWh}&1otl1A*e-84}W=*Sc2t6+8cywN8O<8DwQwz zIWaclTknjx*ygm64AfX(xa%p@wsf7n2mwtMQTg&Ynk35K`u)E*w#b$gQ?dI z@LL%Wu};wUe9WsbXUlw|!gprGE+mQ_0FSHk1K|SfI$6!QD%T#FLv$QVOd|NcVF0#P zLA1Suxpv$K0>d)6$^(XMMP4;g^z{{TdOi#0T9>sKrXp}?<5TQwD0h@_PnF&7k_Ds$ zZPbd}%M(_)z#EaVQJb!A_@|XO3G@e7=EM^(sAq}Ssk|S9IQnf14AgXU!@sQ@$oH65 z{FguSOj<9IMO0i8<gX5_T`#D z5JE2q*YN}*suXEDle^rDgdZQL{-H!eCJ{vtdUv{Y0F@HEB8E`^)*vX3a#><61Z85f`SbkPMwq0r(@-$PI34 zkoB`zVX$!VV|b+TcfGf?5$%U-T5!J~!&&z3zAWYn_}4=si^Ml@h}4CUKHcF38T#f0 zs;gu8$r`|k(?%@i5cr3iOBrAZ#P_n2YR)6*8R#RsQ|GoPGovLee5Y~!FK%j3;88UEc`6)wUPp=2 z??*yLB0qyH#wPQXK?`7?!AGm$iyokSdA=O^8>!AVOA^|FPeU~YkJRC`W}()aqLe35 zWO;G9#`=o^#)uj0M3LM@261oT_9MGWG2(Sk&6lkJ+&4~DLj`zYT7|wKq=Bnl`pGsI zLvU=4-itaxC~L(sKj1#upR9ExRp}?QR94v$xI^=8j_ zgM-e1emhmI$pq8rS2Y^CGP`Z);rOq9eZd3@#W5``l3YrUmPhq2E9Xc5`kjNko{_1R z*J&wpFzWG%ViVS`)HF!z#7{K3@~?eD8J;%wqd&M25Ftj;d^41p$l_{6A@V2v9tYNz z(A=r5^=)A|#I^#88V?$}vEpX3y`ZlV&3eFD!&ckrv>-{9>B|3!wT97PoY`IaH*|tP z3a*!!S#$;&KgKQB*>K7cM~3^4+;tww06v4&8Zyk)cz>WcQXnjd=WN0%;Eejp08<=G zJhT;_Uu}hmuxXfuldhKL*G1FvR9zndKU+!U<8xxRUg5CYlNi?5aOc41Ozn^m0nHep zAg2z}>N_czu2+u@MGO|a8H|*upf6TtFmA02+uW2Cm1`^OU5qa-=Y12uf~J;-*O7^W zp+$%kvA<3YlWAp%4tT++EfM)L*l4(NqeRts-7Y)Xnzlm(AxB=^j&g&4_v&#p;<6FK z`FSo-x!z1LGL*=2M@VuC52EYA61QMxY6<9S*by*5)OFwYBR}kgzJAv6A~6ZAAjYSUORX1d`yUC&EOH!3*}(`h5Lo@oZa4%A0*?;*_gliz=4M|} zyMvE8`8KpV#;us#NR<}*my+pdh5Q4;wHNp4ou*{|T|&Ii-oE9FXS!3l#5g?m7~>gj z$y;W2t=ZJVE@%8-(l|3%tkd~U2(9zn!h>;mjxhJe9R`{&ZDP8*t&~b%Fl^SD6c;3VO}z-8ioOaLVCTv84L~nwRpDu*n}@wxaxgo)HMmqrWRc3AvhfEUz{T(vETe z;hhMj9FZ`98QeJ(!Mp5*Iy%Oya6!V~92Li=& zDzA{T28B&1BLpqzxO~dyW>Io2AY~m=@Ov(k)EnNq4xw2@|MT}QU@;}sRKDfo)A(Ar z3swTF;iGUmUn^c8zERNgCFQ|PGixg%eiRgSM@+$;t))OGwnJ;;m;Sz2Rc<5Dcdxs_ zv}0_$yAxc3)jA05HF~``g@|aC2|F*77H7O*7wi5+QI}0dNGBYus^C_heAi|ds69)b zz;o!RVhy1QAscI1pm>2vqgaRtkk0-o`Z!lM5A}Rk!fG6WpU8N<<(*9yLCA-jCFnI6 zOFlFs7BLm{gKlNDYZG9q9ZstHh{bTRr`SrP4ltdhtJdgac6#6Tw!2-ECKRq_g`}k5 zHd-!1qeNG&G-7WaDq1jTp9EfRuhy6;(tJy_i~<6j?QKb}v*f?%Yq$HL!U*prz4)M^ z=>G$RT;3a)8lb=&i@{Tw1z{AxocET@6xpRN{dPJ??!)2yoM9_77A-P~X#%6LdYngjZo1O&C+-CqVjLr78lz`@tZ z22|v$YoEOKLW6>WT%T@e1w9{nC6T8(mC98k?;XkHDuh^5rSN49W5LgF1zlhxwtu<( zo(CUNzZ6kq!#{D@cZah2NMQi#K}L+Fwox)FohLR<{Z6Ku1y$J6GN3%QGaZ*SHrOFg zzGfVdY**M?eWN{K*JTNcCJ109f66&ON(+g}4-W8JK`|Bo*reBRXgAumx?xhSefEi# z3nR*OC8WwyqEG+lm^o|!Uhc66VA95zOCz6-t#E`w=;H!7xV~Bo$(>!# znMLT5pUve3RP*fM9eZNIAO4MG*%xdq6@!4pQJhnU>G!QX7>VucD?Xl8Es=m;kk`(4uX zwXP2WrTGCviwOtI`N(*8>V88#<~W7S5p)G?2xNvdXXQH4(kq1(GNKz#Jg6_)LG9f^ zq4XEs`Or`IbmXq4AT+~DHNwDQA!)-<``fbK#1abr6PEmsWhzw?6PW@a42*uhhBzKp zxUccGch~!JdL{Knyh2jhbNJ70tEn2Jg%c8F=?Swo}31*+*CcS zWU?$0^042#+2NAldj@w{9<|nm-~xg860&$OnH_!+P+}xe`wV{+ zPxqDV)6t33MV(51EHmlI(kG#3lYPMnx#as}zlgZ`o|gl&WbSjn-FUxYO)rA-OlUai zV?PeA9Vnxfr_Nr>zxK&|#6gcy^^E*^ku-$&ZbgZbkp6mXB|;~`_Ctp23dyI-pN>ga zH>)2$_e(r(@)hLi<}(K78W2A$JFLe&i|5-2KyuCsGD!X^@MN6##qv6b|zC%Fk?K&XTXPkkbaO?G#Mb`DxX3SR* z_60+iP>Qg>4_q?GM-A!fE6|C~xzPMD2HYFZ+fqRO?_v&Tx=3z9Wsje<4`KO@Ofxh&iw4eC+qf$MC-@LPkf+kttmpZXF+oS8BE80uVbI<#L>^ z)5q(5a(;dSOgg0v#r=baL$;l_8CXoR@Cdd4K{WnVnX5#R=li}Ne7@%QM;dT#cEfAr z-x@@!m7&qDfDhxv5l3Mez*TU#5Q{KVL>8x0_BS}Q98RrcDQzN*dQ@B{LGQ|j7rf`n zmK##Ay&Dfb5)Y_$l4l;}zkwApc+}P4O7TNbsoEMzpx0~*TfVp6>W+s-4a+;E@=98y>M-SUtaUV1(p zU>yCjB`EmoyN}+E^ao4_^o)-WZa44QLs}Hxh&J40a@SeYxpS0<7DDW8-`q2pvb*7b z=VlhIf=8Y-TcxZM4OJ$uxqHwXt+WWZ$9CTCW|Whf-Un+v$U&L~t&!T)IP`yDXgv(j zWZc&qy9i^k7YYa!kkCJu_R-7n!B*xKO2|3Lsjg-b3=7tdqp5d3RbSgU1?sJ)tF4~_ zCwwHF0>e-2y^soEG1|@xqcM4yDUxfK+CDoQmpn(r;|RfSe7Zl62JF3H6?=m~0!>)R zNDoj#y!J=!pb`e4pH=|kkR9M4^L=@|zP7qQTQyy*Ew~sLLP4vmY<3eFcuiXz#dx%A ziPMI^@}bR`m_#Yvk=zfk+aA;ddR0Z3$s2SFu5o=3${hu_-y!KUvVIg?zhmsU`sjXg zmlNX(E2WU#lVeOB#e5XC9$63LoIT%IkjF6D0IpxY}XQSLy68 zeO_^Y!S$mv;Kry5HMZi-*6RBul{KjX+6wj9M>d!kf91YEp=Jng{L=gcDSx zJ`6h(aLIo~^g|m8mdot{ld3JMX|o5eNl3U^8zoMe?~`JkV;lJMZ3Akl0&9 zgg3BGd~%!Pg)LE})XNEB`5GN^^%-@#T2Q1k@msCDSO|jyNxQKZaY>T7djE zbOl>6|AF)~Jp~B{XmI5U6ZJNVoz;x_W0~vZ=B9@yLdX<<06Z~*)BNGrn5rn)|2!Tr$`xPgrQZXNk(-kF48g-kMG*Yj%F6vR-cq z2l(5z-#Xfw`ywf=SKE7@)3lvhz+MLQcIU<~>nzm?z1we!uF?eZIkzOEs+|x;Rjd64 z0mrNcw)jfJMwEcN9z9HfPf%_+tWd^RSS}P4Uc{v@bBtC!*RJ;eh&s!#DBCt#E8Q`4 z!%#zamvl(CARSWDozh(r(lB&~h;(;KNlABigRmdpZ|`IOg&%NC+|PBdYprvoV(%=u zy*okqt=1@E;hNq_qE9y$Dj5UKBtr4Xh!6I`H<^2mEG;;3xQ_4c%hwE1j>#wm_sq@V z1GeMMf@q|uZ7Q{}>04qRo4#oIWSFUT9}iKHm}wZmiP_JVNhvI z%nv$S>%0n^BepVXZ)>|BqF&-d`Au6FjaIv|yc|;W6(drF21PA{2-)2AYl#)3_uu;q zGebxLH^wFqr5O4eTHYX?%2FstuaKL``&qAC+K~1eyTuRPzW>>o|4oDeB%q2O$cYuJ zqBV7}bXet33ZEDXK5LLE?R%(VGfL^l2IAZ*)mRc0%_=>302LwCkAkNP_4P_-$v|(J zyPn6{hCc|$yl7Kj5NA2q2U_4? z#Q@4pH%b^DBSWY!&+>jW#8Ze}{*^ZY>dSmb{s2X0PIspUin*fMYcHGq(RF@L?xOi+ zfJLuoI?+@Oz-Wb($^Dn$Ff(J;t$nxuQ2(DNTju|Ivi}p9*iG|A#SqhvJf!D5^->=L zc0I!{9R9uSL2v|@Z#ntm%f4wW&Ht{@@lqj2$K{cH5vO5Qn?--P1l9v@{^3Xp0(l)) z$52_!C9>0Ixq*V3A68RAHOw+aPYuwML}X5BENe-GkskrXGcF)kxqT|CaBz%VpFA3F znUv+KD>DKHX@t++Db({~^H~@)J~ZnMtR_1nY=AiQlQc8g5EuW+mG^5_mOJV!)mw$qct70-w z9G2OrBa6>3E?&sv^n8t8nl_d~%y2(8B!qD(d^m~zC`&eMUh{3l$dN7OCs)q=;&QRs zQo#=r31=W_*u;XzR2F?xQq*z`HGpTm`YTegV>lOh*N1^y5;+@S|W~lZ_ zWwUHv#nWZ*moB3?eE5)SOLB;ofS#)Q=0lMz|k_|KhaGV13@XfjO+ z#lL_o4RCdxHu*jI0^(-SstW}Nf!!buIzo6#uqja`R$xY)g$f-ue@b~>J-tg9Z0Q}< zLJVo5L?qcFu$%ac7|jnfBF=3BjtELve#X6;6P?1q>9!%RNcG;Jpr8=LbG+rD+Dt$W zGr?VxPVM^+kb$o@rvU_z*M`EAZo>WVLU|nXv9v^jWK&e>kZNx3AcO^62I(nnbKDnt zKoe1vG-LSB$#t)PzfRDV5qCQ`ni)VSd2%77lRa9Ck41~iLD(6t>KWjVsiP7g6<2Nw zzH{0eAhkbFK_z2|U|eVVpBq{t02bxGxz+iJFwcx)=iF`U_O;?x*uW3S-TeCY)3kqN zrRj;<`8C;DV1CrNFCPhowK| zP*NT+D&hw2?dDnc&gpuxEO|@Ok2vu2x3DZt2!|V`YI+;r)&ATUK10cufJI; zO>^UDVVi<%IGw!Fu33zJT2+7uE1M~9dGEJB6St+A_Du#AL5tXFzNqeSfZmtFwXqIu zrwd2^qt)m z;Z9|FUE-xR`1g|U^>vAfw;ZTgrYAKc&t;!=py7IpAxGyUS{0_6ofi9Hr*vj(8gvAy zVYiv8{(S@0WVQENGOkA7F{mpFy=lYpO zlfst}YDt%1?N3+k2#3f($-@;2wS;99m?yxlS0M{bpd`wuZVOsmI!aj^y!KvkVs6A? zt#Kf!xXgfIOLddVL0fA^*vM<*@ZuN&YcKJ{qz$9H_;ALaamh z759Vf^}AM1-sigW6)~g~`W1=b9o94>ojcDk*@L*)>CubucUS74bCb>0~9>Dh&i!Cz7p)O7?HIWOnVAQ-qa za-4Tjp^fbYU)6`3E>{!OR&il4?Yc^d&LscUx1gZ_ULt~bT7!^SCRS)fUNjMq=qU*L zK_wpE!Ol4{N87ZxxOh>fo2)LgiSpM|?g!OIh{Ek4d{IQ=qq3>0ZvsR-Q4+ww;Ctso zQAMQr60Lcq?-L+$1csn*a(z_T@4u-^smTHWjO~i_ep+1Y$(36uRRA~yUcsdf$WK0$IVXWQ>P`cxA?%rf!e)-y=vFQnc+9R)0txa zk?fqT)8zF4xl6c;x5C!Ck0h0HJUr0H;c|{-w%Xx1(C+)k{rW8#&9(7tzr^>c@CJN_ z(TA;WRuB|BhIb?KOU*p}=VoA?f$#yAHkL+6= zFD%}07l8a%z;(*4v7X4C(K{N#|3k#Y=LcZ8j=bcE!URPFatw=NKf0Do0a7I_ zA_kI3CngXbGbxpX{AcL@FHVGCsc_SK`QdZIhZj&L9b5PKJ8?R-P%!sL^m;ps$)*on z+%q;e(H3FizWGo=q+Ifkk&LBVmwo8aa${eweb8@@oo<3^Hr&(r8kxJ5zewY|&17HB zT7{YvuRi9iXMPMtGSc~2aKMDQL7vjue=~}=oHy`>Gu=|oE~ag~#AL8oet)U~ePkel znNzlcXa1p-d*uY1*KHJITnOC$NuD_>YmQ`VB+YwQy^wyX-(Wv`4Q`HyZYyg&(fwhQ z{F=!eCAu8PNoK~w1XghOjc|?>Malgiyf(ni*&v8TL4h=QF#-!G^i!gf%1?Fl8V{JF zqT{C!Q!e6xU>t79G=QX%p@0U1edL;`;tTHc4z2!`j8tSyJ(_K8ZT0HN70e={xY5>y zsZT*#BFAdYB(2SUpru<6h6e_|&4rP*xC(fAX1JD-Oc{b`1uK@9Z4#v$88tggr;*?& zs8Q!qMbng1ebuB+To@(4QNaqP( z_J-x72X+o)Y%R2_^>00UZ9u>^1vdpjJ06lx3wm$!!P{VhO%PvIAVIY8n-7|zBvfH; z7Z`-o9T3G(a=!;BWB!p8NcU6ET)GP$woL#jnObD-oPA0-X;*Ns3y0;kRSY>FK}lx_ z_(0-s!QG2T*^TcfSbJDnLWbl+8Cw$}D7ZOlj4=J1X;AJ#rQWr*jjsjt-nN z)FC$prnz5}B{Mo?rStK(1a!%b=?kOw?%#AB@Lw=+Oew^4V$Dx-? zf6dLMFftUQf}mKfb@;WhOO$;ihJ1~9<|{>e0px;2u20H_%g}3{@=}=tdzuL=I0lsP z!l9B7^-1moT`8~V>t^nRU!cz}3$-R~eAq?ZYDK(^Cn|>Wbp=JGhs4OzxIywyght*u zVZgKQfEFK{Cja5_@sL)chz`1xD9wx>`~S8-^JXy;pC1df#G%Hqn>e>s?gIpB7@E}? zvgZqtVFR#WH-6uMXT9vPJY$WR+&ZjG%-n5&qOlYiT57f*d~Vw7$bBnpBm2{MDhAm~ z``tj5d>D+uuJK^C!9s%>@!=v-SChGs-EOgmedZU}Ouf8_0%}sS_jym%e5ykZrp`q@ z&ytP?RaQk)O}N!YA+aWN`KA4_f2@wRi}J|$CV6FUIe8weJAN~w{E5J_U(O5TT&HjA z2pIH;$8>xu`ywd(W|jNzUqqN7BeQQaj%Y-@hy($|l8=^{GU!A-pWJ{S;OzMmO5WOw zt?~ID8n*4+uxV(L3FiJnnJQs9H4J+4S5!0U;>^C9ugr*%S@e>n84yDQjxQvTwHy-k zn#C{gnk5QB0RjxE>6dEP7!PoO#V>+SgVrgmEfAs=tFUTI$q_z11zCxf$_9dYTiwBk zz7$^gw8=rmf%);oeDJ9u&E9FR3h1pwsYAxn~0+0ffeH zlJ9#&?a!aPB1sV1ScS`1R_+{W&Qni0U1ib#vWx!-$DwEO6PxUO_|S_XsARa)ZJkcx zInFhL&mdY)Po`L6vq^xfBh3a-HPq}vmB~%|?Gv|D(TJ96k%Cf;=w{%N21&kSY#bq% z6MIG~>yX%a+!RVfU(5W7T}k= zZf)da`-9H4H_^Y`T+i{{-d?V6u5{K1!KgCAVMee=jV??REV*HiS_>qPDo?c#!j=z} zqC5um?7g3U^6cw?k$$md4~~0?v#$?);~sc!Afh~9QOff0LZ%QcS3n5Q*8(^f=NQHA zo^9*?;yu6rA+oU9C$S3Oi|rux*KNNh1Y{LeEvoxRp8anno$}%=0YUvDGEl#8Y^yal z9@Zx=3b=pUzgK9N&>n3XzX=S*ikL}{6tl<+21*L1vJzQ{HN99*%O)0c#_XE|*BH*C zW+CH=XDX{dwZtPSjFDf9M)IxP^^*yk5C&DU)+zXp>|}NF+_b3|X44|5TXtm|wshw^ z;rvs|xaAV+U*0Kh4#*|Z$4ds27AGvE$`G8oTvx8YDdR@9!OR+phjM?O-cKE(*IK_J zr@ua2&2D^*sJ|hNU8xx&mPGezBDG0G3*-38h#RpUh6b`1#+Qej8nwhyxI(qUyT@;s z8bc5`?B09y^~opQqs1Bhfpdk}kLnO9qJUhu(jtB%VaTN)32&W+q)e#11w@#hXe!N# ztV@}jP$>ezT}|n+kr9Rc*j!)_Cpw?f!KHr9(9c8#`5V(m!uVFn(iWHvr1mDV3pz@g z^6sj1>w@Ab!QGLH=Z;#XA&0F8XP9~TLL0GtM0+`j~ye{?3!nHB)58@jeH6CL=&Fl zYr|*^x~aVYw|k}Qk>8k^ouMusm+%~W8=^5xrCr$%y;Gj4Oyx1TVR!|MU=2=AyWicP zCc%G^NRf4%pH#6)I_;z||1{<3l%1bEE02lX${!UAcnzV<4#vDKf`3KEdlqdEKf*kf z$;nm5025jaAq+BKBugusOy|3bQk#^zifa9&Ed{Zz+_6uS1N(~;&l!;#OGTz>;L8$C;Qs)whPNQA# z44TqZr%8@cdqSTjM7335WN~omoUQ08MYWtJVRA=yC-k!ciAkLi$+0^*!c7#0RpW;! zj*`ZL_qz)e?*jKo!px>Gek~L8_xPP3B=ODiE++IU+a5E^$J5Pd=j&iIkh9r^#{r+m z<2KB{mool97#8`OjSHcMn61CL*^LslEr#PJY%|$`e|C3{{o5aC|40>~!66=<_E~hV z7)}R@!pR@gnMG>|{Z{pn0^72?Xs}LTT`r1suR)`;w)EBYr^({qjB-9FeJ>mR{_;fT z&tA3D53d^gbb-?iAZlPhCzsh=26%p&+LY)Lab$FM`SX9-Ja#^NIb1M~T|R)} z@?|86M7l75Fl3Z^-yHrkzf=$;3^#PBX_(itBuVGsq5>H7*9tb8+qBF~dI0qWS&E8j zEm2GdUl)=wHV87J$4cXK5XQWcEej})>wKmHzehs+_%U=M$zYxAI80L6PqpRLH|v>{ zj#+ZZ$xEFLmth(!&Zz+eTX!3499O38G|wK(O&!F)n&8jlOYw2wkf*hlF?4A+$pM?O zGTZ)dUENE$rc{QGo+}^z!UImQSrD};1F28Vm#aE6e)nS}Z4XAcx&{2$ z;*%}-q0-$p?DnVx&%%&+l{cEEF1mWwvFKp%eK@5$EbOU5X;0&ftRrpPo2VU*Ag4s$ zeZxLXp#i0!dJ+RI8b>Qm9K?9?ezoiq%HsK7q%Q0=&p|E~1X3mK2X-#CmkaMpo9a?o z8gn}gR}*0fgQ5RWJ2;%_0AAhWYYs%MgCUalu`pjP88cFt1|zXa>cANHLZj2)@5S0n zA_1Ex-BAPiMK+%Nq6bHdo_e$4h z#wPmnpVLE(Qa)agx;Wp#OGs|)ecpVQbeo8D%uh$oj6L2kxeW4j&OSG)1WKQi7eWzB zv|MO{D#>D|wsGm+3 zW4*{24i{M4zq>366FN5crmA8fktqK%ytS1Az^s@!5VbpNNx|)7M=4qko$T5k|8XM= z!?S_LMB;*jEBD!b(8`Hi=I0vypLsNBQoFvi8c)MM8D+lhtE-nXmuB~zXXwboUQ-^I z^|NJ&iN@@Dxssrmzf~#4z`0U%tWE66ZB2cWGRh;0Aq1Yz>8fs=oc6`8-wWTR49hWt zIUu5?prwPEk_RzIkSP+UTk*5*rRa1aXaGKCtVEMj!K45;8W1Bd53I~Hl&}Sor)3z7 z0qN_~MV>wXPL+^2-Ko)jCBrCZdVMoUu@+dlqB=M@7#h$!y{9fb(63@H1112x-F;8> zSuH(Ku@>_GzNccac)ms}y~A7I94{~R?`=TUnjN0ryPV3ZMziBo(tnZIcqHN$V0OZo zyKc6qRk@O=q?X5hiZ9(%8;woXfB!yrS^UCECX*+5mhD*>vm7C~i9UBnj(uQM_*?LL zF%I$4s3sI4ebj8S$v=%>l%tBxTYKc(wmj&|hcL9EY$n@Hy*O9)-`H5RJjR&F{~SN% zPCI1_gnlja;*SX)j_7PLkFmtiMD2>41ECz>uhbeVr&ifate>c8b3T*Av1SWheZWO- zIGu^#)JLn6jNF?bsXgtYZA0b_EP^7@6@_hXR~_*;`(fzqL@(RlkM4Q05KzX~b#rb8g z?q=a>0*)tBYa{OCHYX^2K-cw}l5Sq{mqBcbqZlW-k{Ma={v57y$?^~KJ}ge%Pa`0r z2WggimYC+Lj)h4WB2L*mp@j@bf*QM_#D1iWT!Rbo#OFf0fi+~D=``bqJM5-(y- z;u4rC1@url)`2*(Xc3>=b>M-`;f!aUiOVpXFO6zQLj|Q}mAj|nkk)9AS^d*stJCKH zH>0L}4rp)`wTRi(@Ftdqvz83P)KV96-j0~OVk$hJ4sJ0__*FZj2+Q%O0_(jhPslP-IdzUPd5Qpe6GctFJ%$#R zD=CJz9z)xRWqi@3I`TwU?v? zpM5G}?mw!2gEzq?wcj2amI85por~-mu*EaK|A3asOUm_0s*y-p3)}k=Y0hCG{3dgP z>W9J_;nk-J!Vtq9F673wxZ0~%?hQyozmRXHL-yNN|IO1meiC0auq5phZ~D0h zC;CBCrWL<=KMMJkp1{%fm-n$hh|G)R;t@ONzTD(Y7kBBc=HWV4$fao_$b)TFRyR=F z%7|fFNhKIsN9BR^Jx}J>3t^)t+XU*=JH#-pFN)e2IB~fI;E)NkUz>G=lSP7@M<1$K z5KM(|$8z76Q`B5szVBI;iwVF14t?0D2`YzNfEtV}5*a?0HJl(&t@#5uIj+`>hB|*V*o>AzL!pRv$73LiPNYi0b3GcU+*y(a}uPjz5kO85tR)xZJ`dxmS zhL?sUI!aR&*A-MkSYoB8Sjimv+3G33neCoSc}Qf%)i$%F-KB@Y0;NR*+?wr6pz?xW_;4Ng#MLgW`R!R zyLp4`T$5g^EBm~6eg+m8mf_m)BNaEKI4maq1oJlCBb4syEEAW7_U?cPZUS-H#nl5(&Rl|$v;81|ac#vB=D8a^b@ zcMj_4;S!wrhrZ426}h2|3f$3hl~Mp~Sg|d$Df+L7Hb=9=?+NqBmI^UBaUq6piWPL0 zJI~0dCFJqe(a#co5Xo}9*a1#VG`T|SnPT+Z)f7#+V=C-$aDb`gCTdEsnAfFwThSha zLrR->DGDaCj8UgQ-ma9yAy-keb+mLcr;Hl^w!W0LCGO%ifOhuJ1Rz0WHYLj0pwzf! zl_=JH6#E*(c91tUl~$+2S_jaYjz}S1hI6;^P1x<1g*)Kpi8UJHW; zA^8j}lR(I!$&4oU!u;Qr-JbxoAebsMn(8)HE}v+1rR?OL>|lt?NG zKn;x%OUIgbNn0zRE&hnKU7{3q(^E->2W@MI`y0;IJEGRkei4G`+?X5t;j^=gPU(F@ zo@y(RZnF)Z_Q6zYe0lhFB9Lono-2G_#pBRKeM3pA*O|Y;vro;H-${3_tN_y> z&(6^0UzoRhpP=?d-f4b3y89;26~IUlsVX%AXlDfq>GX&SRMgmEXr(-=+*5#u`XM4q zA%zjS-HAFy;X#uJq-W))8D|8GtcKeyB~T9cK%z#=3yL0U%a9^1-nB|V5 z$-m}SaGH@~0)8F!Zu*wu66mWKy-QM+-u&RJ+J_A+3vk39!2P$@U}%I?<+c<)_O-z9 zvk%}N<%qMk!7j$i;dxKn1|R?dG9(4^r7LM@`%|jf*`Yh6P+puD5+VL4)$G?a-4I>e ztGkH(HvfDlw#Ujaq8iM2`6`3?$sD<#K0#=4SL=c>J@Ktg{L5d#JP$6}1tj1NWagao zLeW|uSx@_elX=r?FSaqDyAw#;5Y@j>EkA7GS*GPn=>)m3z5-txdgB@OfNM0sy=bK( zzM}SGOEHU8SAoM)fU0C9%koXKT61GCHq>gy8^XJS8Jr%u+r}i>ed@yA94OLZ_%Rr- zM3mAmm`GQs4#Do4&@b%4FSxa})qh+_HR5p=X9w|k(Gg*F8pdil-!o)@lHhyzSPNR1LuV;Z zt}P#inH+~fPLR!No$AIFN{M@Q?KYfC zfxmKvvB>EHBQ4a=a0mlX|B-Q$%cTbyQab+r~ZW$MJAbu?JTM9a0SMoMN`3|epjWMVuOp&ANtl_a$dEh1<0?cJS< zYL@~sx5ZU#5i>j>(>|*t@aO%S$o70*hmaeP$3trde!LRon5eAqw z#j)8>Ke273OGJK&cfihS<=H*<7?@q+eHQNI+pF_-HBuD2g16Bi(IqKccej5l!(Y|E zsFRVr+63UQ=HA-v4Ull}%9Wn*KM=lf0F^Pppj=GW{<*=owe(3R%Ts0q-m4Yb8r_KF zBgu{df5|&c5w|BS!bo|_^S-1Dj7bMN-l2r3TZSq&ld84oN&Yxp#cPP#mW_+hNc;In zEHN`L{_D0bw!V4ma)m^kJuv;lvW`&Q80j1@Z!OdkKknO8vB^SyMXkLc(NSYnS03P9 zbRKf7Y=3|a%h;)8F#&7!j9ZZ4E9FLP{B6m1E6{@&Ra0aIFk-#jLUzaXV%&tXop#a@ z!IjSWavj$R5ua5XT_nx5I{V|A^{F)wwBCB&jKDI^d>s15G$AEAAO-`BsBgSBRHc;F zM|O^arn}_2sIWuJza}4c#Xg?P{^_DA78%mQ_p zb0z?{RV@fRQd;vs$*d=vw6e02-Flivlvas4qX^;08B~HeL+u9>^FIO&6mq`xKz97W z2c~A+rNCMMC(?w!v04Gv$I?iM2v3}o?WYvcyOv~(L&E}b}G_H30 z#l$mrjXFMui@Q;Sf-akuS051zlq1!q2nEPQ$|Z*tUtO~WMP`$I|5P7uvpAh8v(gpO zM>kZ;6R(Hw<1*yBpwHAP($%t9!ER>h`x$-Ca0h|E#|Of5_IubMt;ppAbbK+oVl9by z_#a;JRcv3-*z_yBn`~wH)#-s%DhtwGd}VitJ-S-O10}Q2U-aqvHPPcg-B>~2NsG}>Wg7R!*%wRr3ItX*quXx`|Pyo>m~a}x-l zep^z5X6)9xv%ymUL}D>WZJ(!2DDC1rU*g;DwBGHv+~NJUB%Zvhy6ULTX4rQuOBjPX zJf^$+GuQh4DGK>xM#0iLv@}~eXub6h$L9&o%0J(6T6xC@o6z~72|F?`p1xoA1yAla zFtT)%B=#O-CvA7A%@0}#NlNB{M5^qyLF-!_rVx|Y(p0Mj6H;x8=3tc(LDXpO?(0}24n5#9 zLiF_ZzO??p@U}=X@r|UvWb$qb>5N^^{QNh*b__JMKA;)h>up#}@o=6Xu!`+bf4q9Q z`Xss(8A1hWgq*}mmA3obUG7{a_4I~gK17tbpDdXGALr*sdG_gB1qBRX)H?9^5s>_t zN<5UJ@X;myI%4FrMW9_OK#Z9E(#NB;MD0fblt=Cu9R1O~|CXAvLHH5%A1ZVPewUX zp@tdpFgLs#^-=oe+i#ZJIp9JI!67S#fty-77k_veRxukL&?QaEP-=E>CDckX^Zppk zT=t%CD+FvuO>g0O3OA69jI|B^;&VR^J-U;HN}+k!3E0P<#f+emUzsLbtPFbwe5Yn6 zua+)Dd4IuNGz$0r%5>=@IPD|zO>U1ry0-Cwq|uAp6O5M{b1+?Zd}6BVV>BHoh z*>M|wf!OWO>|{yXo_ymjfc=d zxq7f?r`7Qu}bye958_Y2^AI&B^Py}6I3ElJjj3*9y_$GQauho*H<1QHE zmhD1w-b`d2>8^D>`(>iYRe3x$wj53de=f>TY;Y2`g(2^{Dj5`o1Tr3~56}1f%7lwO zH#Y~22%sp)V`vXICb@7L4PZZ^h^?>ZI&+`_tbEW2a7rn+zdF>45yXi>%HKs|-syA4 zy67*xq`I_xyjcH`%vVmFKy3AG93UkDp0ura@wocG`ji?kb5;u;4rtH_S@W)JYHA}H z--oE0k)a2KbWY$=RPz12mCtEM$Ybyq)lcTwmWNg2-%T8)y9@Y94$?ZmHDAs{bJ;Nx zbafs1hJ$KxpzmfsXxh9fBkypq>3q_yziM`r96jkBMAiLqN#3Swj0rqa3du}0%!iEM z*3qCc&ND|*SS#h;7Ww>WXemF5hkq+tm$(a>r z7&dya#qWI$i6a+-{bozg&Ls}m#3e(?+o!%i$T6qSK}H<^lv;gaUTJ!JG*>-P{`;uz z=>EeoX%%3=0c&iRX;Yw&SOy0&vt`_!1b5HOucwV#kbV>eDfweZA!%&e@{^J%8+^Gs zWFX|Qh7sK53!*U~PiFqgR0CmnTSL|XtUDQU=Rp`opRd`STkE5VnQUE)RU&GQCg?`} zb0ko3{3s7kD>Ax-huNOMFNs+>oKp$MQ{^fOV%%?E2C!B;BM6BO1x@BpcL@JfVO;(h z_-2Y1IYx(B-f$Xi&r@F*MQNg2i!2kD8zG-GU?x|h;JD7lYPz5epwhMuyI8Ad8W!M) zoOgG)a4z@8-wNUU{dE*3kZCfoOb#i`Zm~#14rS65Z#d*ptFY8c9!WZ#KgwDl)_R)g zx~?v2_5xfZOYnb}=H7|wi-5IgwTF!AVIR4fP|ToA!x1d1dH$F&>74M?LR#;@lPm;C7`0W>KUlgc!_zEMJLvR zWvVv%fQR<}n!Y?krbBLbclY^Xf2IV}A27Z!r}O`7y)AK49U+a>)07Y^qExf(4uXH? z00Nw4lbOyTw#be$%-5C)6#M&ALyRg=|HhAFM2XkHcqqW?DhTve;@vI}whxG{q zJ}OP4sI1T{@iiZ_hG!m(Tt|RpHU!h4uOk~LGd{&AwDP@t-Q7+M!x!4Du*%G(uJ^5L zg39klqMIiicF{uuq$|?`ta!0!v9~62%R#4OwOwuRH zw!)-$sLK09Ov<@swR^i6 zCXfiZS5785TTM?|`IqMq!X9wN4RfM!?lV6OX-G1YkELSc7A(1vJ zb?D-~l|Ri}{(4v1pTN`rG7YN988De2@}z=&BHo|u!Uq+A_owN^*w#y7PEBU!=2ao0 z8>a;n62G~So$8TPAuSTp4^F5 z6qOiA8L5wJ{TslAbo7Zn@T-LYL7|lM?O~ja>__pz;Fu|E63xKaI8UMeTCrKGH5?M~7ZrtV84xK& z%I76-)s^{=PvQ4&_Rd2CD&V~IxW(PLmJ)EKmuBCYh)O|V;@yvXL+N##PPMRDhQ=pC zzwcBQ*o$gjI;s{KO4v~bYYrMj4t1fa-|=}KR77x+@r=&Av5g-W|C9A$()YsqqwXr4 z06Mi;sU99oE+ux%wQ|Y-^WpXVYdJ6;&*0f#Z5MLL$|C*e$BpKv!B70~GYkeEMG$_{@cZGNYlLL?|~mHS&MW2CrSI3PX-kp+n|cxH2;E#_G=4dPSz?8 z$v=|2S`^vr3=A=e`s6$NBO~xq`poG8H^?M~Rsz94m$~gbFtxB`+l5;E-LT(sb6qfr zFOH%VQs~{@o#V|lLA4IYRDejNEZ%vF7{VINhh(O$S5%zn>+Qw*PreM!*^xqBn{p3J&#+jeFAE=s#5c#qdOufR5O1l0G%BzaijsIpz^y+wv;tvOCsx zz>^YQ@)-^h@#^nzDBO3Q8p8`~7lu$e^w|G`FU9Fajv^C+s zSWan2mMnFz|Lvuj17!I`|tOE_XEZn2Ft7)0`F_;<0%AJ z+p+eukG`0h|KHhuihQ3nVq~mzw`(PIgXT}BWK}g7JckoQzszI4+d+>c7q~a_|Gn-O z@}F46R`%?&l&r-&v?s!)4Td3fJA_|L=X}(+c3k4GQuPct{MsfGCqjxcRi#b z8ZM&=>Q}1Easg8-lW4fG@LMP$H-Ib9H!jV2mAab+Y_uCfYww^8|OjN3aI$yEq z{dkwoX;EwQn-R)KT&5WXtjn*C038#{yO|%@ylJ|LOMR;g?dGjgz=(;%(h{ zG6N19p<0M$Fao;g)zP*Dc=lhCV#^_FM1fTy1qE9o8H5u}*g^?SdWm!8Q3}bd_L}|i zT1*(zR}l2tSaXhn)*b(;+A*4T;mL3FFabLtH6dwWDDR zdNkeH;7Y{MTBZDHTj^R2rl2L)W(iM3(;ZQUem9jY1}=*h49_91tb2h~qO_eFLbk%P zWChA+wK6?fO~x6l8+;Lbh4pA-OZ##SW|Zbmay*y$cj>HG>}}G3`{x7MqU%p$$aD{K zcKO_{;k;B54;y?UqkS?3W*)~~hp%)d)YWUL#vP)TM=N9aOn!r_XX6mhJl6V{+rPGE zv?q{}fnab|It42M9JG7+yA%#`O}6BB@zWG*%Z6L~v#eHJ<|5?xefWw!7u#K?W5DM< z?k863*`t(!5cuq>iOEb@GGvnJYN}HCv;XtAS?$IH%_ghMW-r|PS`YBjPn-Tf-ZU)@ z$00k7(}wG&&*|)Sa+q?ZK)Lm&?=e*emcf0+*9S9K9TJ*!Mg%nFs*ygIr;+H=$G9wd z$#>|2UY9fubf2xQ8OCCfrSYebX;>lXr|aFpZJrW_vUOByQPW+Jow!>avv*`9plBq8 z&(TUPt_S!2piVhR0p|nFY)FU|3Pe`&z^*QoUr^hXbpL+7O8=W!*l?&~b#PG7Cf@Kh zt>vOGFsI(*5kt1u5hiGs2JDH4%oOK+FY(RTzsAAbSze3N9c>Qj2H>n-?wmCL(N2F` z=&!@x)~hFshW|*DKI!C4fQtwAmzq`_+NDGiKcrmNZGB_bD2yLM%W#>(fRdg`+=+0?t zgWz5|Tc&6zl~3sI)jFCcnwoU{@5OCD({krs+MgO(&uam8#_|aJBDu|Qt@veNSp5lBl+fs_mR8NI}B7WDp9sK z!-)tf4ipgvCH`l+9}H>v!}`&e2n)|(v__w1mm+^Kl(7%-G*<-MJ~5jDk+yp0i}V7Y z@5?59x3}s$v?oD7c9&bbvAIU2gZRUAYK>O(X@Pvzp9@oyQ3X_#-LG|-qj0p= z8IY^uqhDtVZ^W1z7mT333dop4x%Z}$GArAWQok4Ep7SsN4hQx%0G3L#-Ldpx!*6u( zvFzrF(r9Fp4jJGh5UMnpayqAUnkoC&AT;Udy%<(CHe;=Fegz9iI8`rWcU&iI0yy9< zGR<%;EcX}ncql6@BUNym5L^N24ST+<;i&;=+lmj4iO%C)MTWE*yDWk(iFGz&^Hocr zAMbW;@;P7XchXpPAKwpY>4N#bYSBjayg)nNz(Vz+n+$#=4JTmP>H#N?E7B?77pFc| z;wdclJ8{tjeIAa}6)c0{G468a77jjzSDt_l(AQKA$jT=e(eF?X=%_w!>vPrCD7W@bX{IXxaq;|D6OnlC1aiU)U_N=I{ssq-uNyAuk*)Q{pdJp~nox=wl5R4szRTAe|~+vJ+nVF z7|~Y_Fc_{n38FeE#Uq7NT*a-uG+%N;xR%JJS!nSXM^!90f7AecBLZ)n_)o@Ih7b1_ zHKTt%eUT@fGG&0*%AtOR1U+Z%msxyNEvFJOp>k5wNhYQ~$#Yc%$>Eq)DyP5$5g%4X z4s{s8M%5yD(9*9g0qW})6a&m*tlYfP8lXb?yk;xd-s)gk>Y&~!yZ5%;J3M@?-FRXp zK`^BO&s*bLZ3UBB^Qic90#VPx7->2gliAR0hgEV?>qNz~qqir;wVyT-VmyWf(bYm@ zfOFGY3({T%#W`2Z(@9hzyT?#;%Xmf4l+5dW?&KW-d&inPVH^N5Y_MzA*M&Dc{uoEg z+QGhl$~V?zoA0*@vYPuY?3`wED;{|_8V)@?_VJC%kWcq-+FV9o#3>qf2~H;RoAUgP zdWvVP=Z8yvYz@yMulO#Mf)%k2xZsLb=@zL=R)M;^mWc(TQ)Or9q!TVMQAz*)4@iqC zv_vozZAx@}KOZx@1?)7QgKPTUNsE@ABT~CTpUlPuvX2Rd4ypbDUPr_#nSu6Nu_hJN z)eb+VwnrbN$4jJmQ3Rv8pB4Xx2rPY` zzNndAle+{63^}W}&BvaFj51d2su)R}Oaj1=!7}ESfy(tOA>0Rt|H=~tbagbBl?rY$movA7^`o1`FXDQ36v8fj=IskX)t{o0JdQSo#9@{NPtBv4U5sm4C0IGM z8@CCv6PWFi_x)kl*F5Hetw+?J_Zrv8%?vpXu6d_v&E8>WgLCHbOHQP;#mh!wPhGIY zax@hk;1usZ0iJ|6eD=%hh|Apk^jv#=?-F?dpzvUcO0MTCnIAj`v8QT=F0d8!a-in? zO{XSFu@Mfiy2-^Ho4s$!CzIHdQ+Z7LkQCF{-)fX7?{u~jaakg{Y=t9qUvG`)h_tR2 z9auO2bDF1Y0I;FKlAec$hkmAFwMRl^MZga?C37JyObdr>^3Sh=czKPOh%0*kuvHLX zZYtAIiIITR`7{f3{Jk1MS|Z@o{V3VkqYh{AtOw-;U-Na7DNzER(RhQaEb=u82ONM4 zTPLFTxh%ZIbs%`S`RPb%jgJ$A2^#A$YFo3`0mMzgEnvM zV=~^#nPs@C9tE zSh7QTV_ykO+zzQ}0wg*E;Szkr-A~aJ$(tn(aZq&4gW5i_z|FIw(KgQ5ob9nUKM{eC zo{_2&U>(~rX13;yx5?pj1p%mq8JyM-ehOHwQ{W)Bg8iCHT&jO0LBAjERlQX#O=S|2<-C5h}Spa|vu} zr35`rKnb11DK9N&(-codmPQNfNCQWkAty6xq4YZ!!VdH9V#GbW$`6hugodAEn&<07 zdQuqerD}q}VD35L>d+CBtRgH0=QB1fWW+;=aWZc{BvBtNWYKzQ{e zlIB^kzx5hqPGC}1AysN^PPX>sFBa9=SlA;{7Xv zw>xJJJC2j15M*5`-bCzQ!`AySNwvzv`(0pb#K#DtC3bNfQ?D&HpY>~_S!H_D2RCPr zqt4)yqQE7Fb_#;=R<{yip(F$hNNOfRK!bt3cW=pj ztm~;1b)=R?t_!8hf3P0}_*Mx7UkuW8aWPj{SAA`?Ok=BtCu4*I_42~oqeLZaNshoo z&;%-}n7&xj8Q^ads6cM147_i&Y$^;^aO_z!g~?w|9fe0;5WS#)0z6-WIx7k+MZc=g z=F#QbGVN9tko-b#m*&Fe^U7sV3cxF}&0dp)T38o@G>>hc^agxc|3JcYe^d&B&dm&s z?vF0?b~({PJed~^61xD!WTnJ&{zdkmMmmtXKbEQO)@R?qna%7BX9)O!2zOY*`kFXe z6*3}oa6n|KR-z|CSU$%Xidh`3RF|?i-tB|5qBSL_g3{)H=;g%J#YvbV^eDFn_>=VG zC4%4`@5t`^Tg6}|-HtaoOyA}1Vk~9XZuQ`mVOR4Ad?)~h44^;tX&!QsF;i_3?m0;a zs&!&P`mlf^`5n2{qCiICJK@&;mdo8dJ8AAr>4Y|s5z^r1V^7_Ac_~JpxJj)AKAWRA zRFtI99ivYCq}SwT?-xPm+!gHn5>u8{KB7ZyVa0~^q=#n`i-Q*2Jrb}4$5FhKH0suD zulb#;4RzZJuNc8DC)N2a^zV|6Aiq0}7DM3>G%v4Se>Kifanz+C$y&j4Nu9)eW(9`m z2;enyRUq*<9f;&P51yFLP+3gXuUu?->N4d)cWeIf;*dkkV;S7<<1E%x$se-CKMmb1n<9O?N@mm&!Kr& z-aztaC@m~Su}b&xqCl*!KL#n(P9*$YBoe)p=}RVQ&r^dGilq(u)6?_j$pi4EYyEa; z#vQJ#jc@80FLCHT#=?rK`o9E_c>C;AJO1*C14@LMGFS5hc^DMlmB{`p=lR4cBh=O8 z)tJ~ReT9gjfhN#J)O#$m<`Q4wFZZih?)L6c(Hz(Mo)<`_{pbArFfhAmTF6-slNl(0_6_m&aKY6*y8xh^%6ox+IvS zBml)*YaW~!qQ%FNIYPw23ixa;l$U$+Xvcy?6$u3$=$Zgd3{z`F9jlFWWMF9Ng%nSGei0^XN9 z2+SWAQx-K6m^EP^lvkTsoKq`~ zH#IrbCokOhGD$U~VjuxO{D?FUNz`&eEgWex8g74{_$D6}82oa{+R7RHpJo~aF>!>( zOw7IKOeoXtg&^YZb@*|E?#)a?Cm*oVNy=67KjaQ|tc|PA(d|9-{;D>)C_A-LE9OlT zz>ICcG2Xlv1kJb;B(ZjzCS%99Yj@o*vV!V+wm*vHt=JG#C`NgR z@aC)+%vZLRUIA7Lvt7JwzI*f$dz7ixit3pg5M5x)k4I69B#c|V)71+2=(>DXh1gY& zFQI~bL9^B5DlNo)LnQ1jtgU;2Aep36BQN9xocB0CP#lbL=fSKq=_eme`SzrnO>MO1 zINsKYyj{}2WWuBJm}_f`9NhsS-#60UVH5;lrfnJg~5kf zHAG1d2KB8j`;lLmJG0k)#I?YWfrkd${ zc|dMjBo>C)vnq2S6)KV27b%H7E`h=P5G}dmBth#|)HB<;tYR?(^jPlH>VSUL@MCD6 zf5`!uE=XCCbjU4I;k&uCX_Af8M4@#y{zQkdI%+NRGi7Q{&5JU$Tz*8t%cJ_&s zq+*LorY2lVCI?HgOoGEZR2mF`T5@Q)4W#jt>(t*!)C+5IwEKe>7B0-F&@Y1o<&pTA z`UyemS7Iy_RciL}!BsS-7PG=#GKO0pxqA7@obCj_W`4jY>4bKK-n{)ruz0H#1r{7A zRDaPqR3BLYO3Ao@&5rzAeFz4!%KI&XRi0)%w-{lOT-cjCxye?2B!(aRX#ELp8H+_g zWiM|V6U5(!dyHz(lVyUXLl|wMW(!eTYmHA{+0WDL7&?jQ^ow6`Wp%}c7A@ko*q6x#d#)on6`L9?68il?3^xIrf!WrzCc2cBbMvrP)hRB1*AaCm+*SuU~ z>j{(J%nrES-njla>z%swuRE>w64=UwUaIhzi*B9y<;&q5TCM|aHwb%BZXV!%9QFXC z5aux$hPY+&sZ&BFv%isndlGU9AA@OLgNZ+sW!t-oN#~D)l=*h9wbAb_CyY2N=3A^7 zcM;3s@}snlfq86Mam_x+Q(|B09hsX|v=#Z4om?F_DGD~amF#WkSURnl3@IcvTJGpj z9Wh64ULC?I38iY0#3U0*4rOx*TX1CATJXV4eD|f?mLSUEL4?*OwE9SyQV2lbMkU}E zjJRI=PhG@0bSQG$>@Z8x3TaTq^)>g2U>HzH$M2|r>}g4E5wcO6f(M*5m{6Hi1L3hJ zH=I3yR{;p84m-U!D}9{xN366MmeI7}oiaN$U2XFrNOxTOcr~^+bYJN(?$ljo;}v_H zNE(oi+w=YkOM-@I_#$APGIr?&*=iTo)OQ`TuAq>|BL0BGi$ck;GrZT_?$YuSMRCmg zt+c&agu&`!_yBjSzM}=l?iyL6*!*ge|Gn#Cq0FPKPT&Eiqfti8auaM~#xl4)^Y*zF z#9X}|(VQU{xFX(Rpm1@tNb2E zBKR3EdJA*Htgt*(3Ljt?5sC*MoEbfyr3Ac=Z3hD_^A}+ia#UZQk{}XC)=6F|J=1@o z#zSW_W^mP0NLk@RMl6$v12|SGP#C3tHSnUG5;6(*llPtu_mvNFt zj}|T>MLW_mTHU#PwQGY!Z*eoex$~wmFF0OJH|6mbX<1HrU9fdH4UbKf=~XjT@L5S@ zOUu3x;xPq&?8^ty?Z$in6`^_uCrdl_3R0>CsrU&+JCq70RqATbeWRN0MyFbC6j2?v z2CewQ3j3&v2WjKgY}GHr7SrQ}-%4x2M1WI>M)3&=q-=3mRk~k)sw(UaM9ohA=*{oI z+Czb$ut8YI*9qmDCZuD=gf|81kRF)Y!$}_yA?4B7mtsqKd$+$f{xZ*JbpFje|Kl@D z-c}nxz%OurDW)_s`S{2(-+Y+`z|>oF9@l^LKU$Wv?@97#24ZE3TFR}9kQJ?)N%Iez zp09}y{<(4j@Gd$kH>`4<1vJkHb%Cz|LJA25PVU8NHhCk4*owfKuHg_X;4?JFMgEs< zoc?i+1}AQ&2BN8ui_IYyRM8gU%MA$8c$Q1sjly1Mi@tIP!U>8k$=dRpi@-&OTCq7wxHO*;^ zKcRFS9&gB(H#&y}KeU8-wV*8O;qPzlj=uE*o6Z$GaDJ13&Tvfy_i7j6yyXV{ZJ`WB zT6gu&1>&UsB1;{fl%o$y*c1EhcbFnxzFva%UZ*4&gg$^KCuR`EWz=( zZ*eBp(H*74zrXwXkNad6-z-$`0%k(#%T(TOgZ$a{2v6myahs-`>|BU2dD~66m$(1M^h%Q z0)<9&oz$%QE0iOi>QK4+6^;_HtU1BmOXM2cpQ~yvXAHiia zUG(;7-ZuI>cUwMU;M@Inuk*brtL$rye3O`4h#kl+HV+k5vPBOXOFbDs#5swB;&Q(s zSn`D$zxjftDXu1BjL%cyb3P)qy-v7``JSoR_Vs(b=T9{v7Acd*Shc#};7Z_{d!N4X z3(KceqXsB3Ccs1YTBrpsq8TE`_)D8MIU#+GaF6KST0ke-jW|n*kCcfuf+!{A)%j!C zcel5-qGo$@m;0)~(*3crIu}-=P+BCe;=ovB@$Sw6NsfV&hp* zDh4BHq89@BI*DZB7*IgzxepN8U~;s_Bheizts@mzsSeLCs`&V`MV4T5`zmVw+}avS zqVV$x@FkQgnUD>NCyip)XpEZ2(+g zijfAS;roRlHgA-`IQ~+4XMphh&VL^6VNC{7bknanEm5wq)ilT*>atS1`(hMrDMK?R zYX}ik!@7{_(EE=zye~3~+vzg*5rFQ9IW5;79wN0}MJZue9PC{c!reV>ysfxCqPS++ zD)ds7-uc#qU4n@wNeyH!Bnq#qG>*399g`C*Vq$m82vH)4T0K$bsq-;9NaH;kwfFX( z8L^~CZEDPG0n9_p$psUbG@)39D@S88W-k;u9x6hczzj0pPMI|=L!st|0|efQ{7Fb4C#8YYi1Y7MVIt14`5E)0DC3bP|@ygf1v zwb*>d;}~!mAzH#_qcaWfrEC!LAp_n@S3X~9TgpbI26tqyF-|=X;Rh>`$A=TCe1e7A z*3`x+qNvM90tNg4WL`LBshUdyitz-D*bsj^vytP^I)MSybOZ62pZ+(+56KO3;j`}C z@WqvcIu3VZ>Pf10Pm>77h~eZT7g0aHzb#y=WidQH)mqhIv=#j`>&6E8HHOpnDZ+gB z_#7gjX|CP;oXXkW6x5iCPg?yeyi6tFE?Y2|2G*nQ;9P7jhMcbK+(SBWF6RbcQOjuH zw;naT5GSgpbhcF8<}quTy_q@^PD{HxdYTYzP(QbJz9@q!9aE3vxMT>)is`&RKWaQJ zPK^$(B0RDdid4z26AZgE5L;IrHkkPC3A@-)cA`qORae^dk`XM|r{*S4FZVP2Y%U*u zBso#n6V#FuKE9J$+d`U0RU4HzqJk>yH7R9cMLVZ$O=frImWX?dJb7zu%KrjJ*0c=} zzVnAH;0s%$iAILp-U&m`5GNJ~Y}AH- zwKu-aOy-i>Q7VH&Xb%Y6LX4H|hoC?AA<3;2RN%WKZ(Tvl!AH9mMBAq{i5=Qdq9^Vn z_&})^VMfUSJFe|8qGc@uFuFfATSN&5C#J+FER?I8W9<$N!RJT_j&k}rP?q6!vkKUd zfP!H1mTeIfe^b52Eq+MSk%)sP_8;%_5S>=~3l1G!ni;Ue zdi%hzAvBDA1(IH*J@h-1bt=!vQ5&aT$Z8vF_r@jwIqLI*o)FYSUKn_;(Id3EbsHoO?D8j5irXbRsd}lHp4hCsI zI6sg~0+;&Sg(Sw)&9Pvn?O11i4(zI4r#gZ`SiM0ZtPzA(S9UA{QfcT)Q%_daLyr*) z@a{6UKuykL2dY1GBS)0pl0zz^a*NQU35K!qOzS>jM%NJr}%y9$GJto6TJdhy>4kJ2Q5e-yJYfsyP0 zCt7o1Sy96|t{CU^p2yKpVz5E#cci_^;mTlcAaV?JYs8K}d_^#}V3&He6T^u_Gk?JM zxD7abt-5`FQ#g3maziMtn+M%ff3R$f({_t0byEIEHSnHytSb5Xae4>XKyZ942<9kw zdqUG*a!t`XeYV_4FZ?F(<_O2>1w=##r)))yBBx-p}rw z*om)+awHTaKptqhc7`Ru#8I}CYe_9d3F+zvE#JRiOiMOz@1eI^p0=FKqp6WBlX5}kTsS4S%xn3zqk2jz}pUgvfY zDgS~J(YC2`@#WF_8WTRQSD+jBu68D0s4Tc+V}1#X$NvSR(6TKp>Uc%S5Wg$cTFps(iTvsL zqcp8FRrpzNyb+td&H7vU@A`KY$)2Vjg92hH3X~(c3oNB8kl%jCc&WjZtfOJLv-2i; zh4u)MbfPW4=WwCW{KWRC-Rj(o%!}}#OuhCCO`8>+fu$^FOr>F;%^qJjs^$TgK-V4f z{>Qk)V8KKGokU?*GVyHPjpm_ue?fBZk!jzdQ;$0(*jyTUr5)}Wk=ipWkh(Kc@`2vO z7c)L)o6vM9HCV^pk@iTJk_=RW2{>b+hAi~7`S2tMr@Hg~Y0xmdy#C-PQ31g@+i+_; zf)dlx%RsvNmxNeeK#rEXC7BwxCFeGa7rmA*Duf~?(>I03W?M2uDED^Ok2;#V;JMV+ z%%xb13MRTIA0ANH5MGFd?|q;FdXuA%oc>RDst2<8zQU5SNsCfF7|-%sD3V%K;Q%P~ z>gNo^HmU>3mWa!~t4y`9)@fa+1Jl6%0Dsm?UIiOZ(qVcd!D#kOXXO!s=apP{YOxb1 zoA1swWb>Jed*+q9Zb*tw^y)h5@f07~L56|vYBo1;99YdLJ~d#+d?OrIS=sv|AZqzU z+=hVA>(M#Xf5=@W<-Ba6l*sJQj?gLg26TC08*l~2w>%!lAwX5Ith1ZT}qj*xc z;y$IGqe-*4wWBR7p2mw&_hdkx=yL?UBXTz}RLY8KNA%z&FN^FfRrHP#Vp%uq> zeAtS=*TX@|p5y&%+)LAIAqx#qpRw!ZB{ZgU z6)R#S5w*XbtszDHmfq@WlM04p-U1b5l)PwDR8-U+ji*E>jC<7Ss2m9lwr}w#mJ%UR zKxJ@B-jOy>DR0 zjE59JzKdZXkOP)*JYdfmWw~Q2!40M15Z57m=?A`uME(jKtHzn}^daD(EmPSC(ygAJQAkVF=I`Ys z$UXy&E{}LL%;>UJ zAxWlUSc_~yuTuk4Ivk$nsHk zsL~PLqX6G1A|HMnRnCNmH;j?>&*#Rz3g+jUG+5EiG#|ftED=mPZOLNUeoa`u;u}k6 z3i(;RlGwLDn5`BX5ITK3&eDlhKjvSNf7zc*q`dJsYOPK_wT+S^{9!dW zjm|PwZSUx3Wkgrk5FOrdSX(Z{l8JK)Uedz(3ua;v&p)@|LKz&0b`{{C`2J8bL+CPS zCyNMqo$___A~*QqSWu^$7yj7)&#VZ8nrMR;7u&>gu)E=24})d6)>>jLRNJ@g#!2@Kr>7O%7pA#6S8Ej<4m7 zz1$rKtGvSEZ`cCpf%g+7jZ*f8e@strZ&=P;NN7ftelw!_aKs4TBoN^hgUg8N`w5j8 zE?Ps5cDhI+3bXCr-8K=RZ+HJsB*!b=(Wt(ue-;s(MNN!Tvr_lT1cLYNDD#*lB`T{% zm>k(^4}A1dF`4ogN+d?gp2?@Eq5F46dZC{-}c?uK!#hGFjD9eE6f0=$Bh2xmRgpOa9@Hlh}Z;u}}||7fL`l zCiBg5bRd0-n?r%9laTJJ!~Iarl%9`4#fKwX6)|kZ#pMX*ti~qU%Ogjw z2Nyojd-~XB3Lrl!3s>Um*)Ew7}@a{Y97} zLH*wEGKet9m}Kj+Sia3;lq!Aj9A2Gl=mewM=An8uLf-~DSTLuDs^Sc_1G+AwKp0>i zQ+=Au6~2OTQ%>Vnfhnayi(Ie_UGVQWSId+>p(n@wGxkYj}<*O27betkT=NM;fAfP>{&Lvszoz&Ud9LAhWy z5|)d6H9uqM`s};H65v$G4~>ljFj)9&^xfuzNRyQvkCXHw@|S2 zk3XslS3V3ZMV;91uRX6$o^00&_#s=J6=;e(IU?7lfsonrYhOq6^pIM|yKgr9*+ptHRB{r~58IWub(|yd)D$!( z6U%};zE3>ygBhsiz8?B_VLQE3P=gJ7)9!EV_LK(lqct~J zKhuzy;*w1YlJsO@46(A{=H@Y+9Tu%rx-+&kZ+7T57Gc~yNd$sH5x6~KeWlLNlwBlvS`<<1`d&;=rB7aa`eYK%z% z6R3sbEkN3gDjXgKDqj8ifk>`0>0T1yK1O`daiBXtOPCJmD#>L&B9qT`7=EZIea-k* zFU~sdHF0zMDeze?(4<5e;@^sC*p&Rf>XTupfN?tPf1$o2!zHzjz0(}}dI)v5U)K59 z6;rC8ulIvFzl2@&R0yNg!>>QAy(CufgF_W_0T6#5p$h z5l!Y$jTsezC02~*QcxFM1YbtI9kngbdr9x1_Syy?b_Q>JRC#GQ9RgVsqXyD>PM`Ss zFZ0_WZ-cwy46@9}qYxnP!H(r5D=+@*IQCGvQbsbIw{&Lni5XLR)Ma!%pQ#Dk&*f$sbegMOKnvHv${eoc<#U)3r$|DDnFgg6)L zB-EiGv7piG{23s0+A0B%3=oaB`#jz@_Z;7NcfR+35#IgV@H2Iy_=&U zxB7^lF$Nk71DEnmhRnA_Wm*>CZVxK&8{|-505ITvw^TmMGAmc30&5)cv%IFK>MOj*WC39;%x;W> z^ZRzt{25hb#mZaKKW77W2_fD;tl5MbhmgKU9l08H-n=UiKr>%1OJdjxEn%%k>!Q~S zQXVWRSh;(?Gf~Hc+DmxWg@7$GryU=#uWgO+aO80%lfaUyUrnfs=oVp)(TJ^n61A2> zWh`*;$5zpstOgif1uDSJ---pOLT0>HI_O2+4+S z8!g9GcOvl}PwZyX!0|)3x#|f9G0%JWlR`snU7e{T9VVkvuF&{g;Dg8C{K_$Z2~-lq zdTQ06OCPy(rZ_+cM8pBnHMVI#&_4AAUed)2^pI4!mLjd^Xhk}6IT%M8Sl3EU`$cp? z-)obg#amQV`{z=HF9Xm;K#2&<#@PH?DFb0qL94K1u2^4^+ zeNj~QdGz)H7i#?tvra7;L)F)FfE`n$n2s72_N;6;zt%1vdy@L^5fagRA2OtdX?n_u zk)yhcnQToTqVk6izRK{OYE%^TxQc>K&iS8p8q=jDFOz#YRLhQ%yVd55kyP`o@W|!Q z36&vjd%37v0LM`4%aLOk{bor(!vnr`SCj5IrGdnzT&(TTK3=j9dU zX~~izFQ{B@e|oTS;j`FCbM^7W8&WYSo0iap{N4p$%8>gwL(h|7B;uCEB{K9BdrmRx z{EF3uqyFxVnP%F0&N;C%7{n_`qVas?8k9!H+R>C0lpkoC@M0sij({xDn20Wn|ltQ()x z6c#vM#tb_k0R>nbHjD9a!1^q5|3F>I6vb=UAA&}%BJsF=h;!moasNj+i^8o5$c!D9 zT#Sac{{j1{)>$Gh8>$mi+^iyeaD?dUe!7Rs7c;(P%H!r5Yh7hCW2G{)Xur?w1{Zg*MsE=h)COg`~YAGY>mzhJWA;L zMU2Gi*6*0z>6hgMQo-A#AIjz)famA$uUc%Yxo1GT(1KtE5(=NrYRbFM>$X{}$6JoY z46*V&!1FxISQ(~*~uo)?mE*q(=Gd;#oj zuhW5|UyrJJuA0=#oPZ_Kx?{Q~IJ6ot=5PNhojhQb_ioSr5jl}uTA+F+^qX_eZbD$w z!vo`FAa-!qZb7A1h1{UkUG~ZDpVvf0As)tS$gfDw+DIjBk?)@f&ao4Ee!>d1u@aa83F8cvCI3+$lu~I-$*!$%jHRYpU2kK+E;-;Ryn=MNE zm$$!7*#bvWlgV2hQ9{MvE|u4_(EJh?ISfS5vhI1i&jwFRs!YB5np2j;Y%S{N7T+6` zl6Mo0LZGI77FPaiv|GzIY>T+Gmh{qGsh+h}l?fjy!NXONxH8e?NDHPt9Jzlg-&Y|7 zAS!=}4;J=sHcTGNL@(Z^qNsBg*U&4kZ4eoHYH5U!KQ)?Rd>1Y2qm}RO!S6KZ9+|I1 zQ;*3t6i;j7Dw}WR9B^1i35%kYOJj%v0%W7y0*SG66nc6Az(yv%L{D@~JgRvLN{YVD zK6&MQp4MYLK9@cI%j20aF3U4L0t$*`@knZ&`x09Ni4bD@KBGw5JD84$T)R&~(w+^Z z+$8>?gV{VDh?3E`&x|9j;b^22q1Eqj=*@pO5kpk#sbf7-9(5OW3f1bZ@H^d@f6tUy z=Fc3@S7GhD{xjXOTM$An3ZiN|Nnw1(*Up^?KUiDtp5Qp=yn@uSw$5`Gp;Zx-;8jI1 z#X9g2uzUV^8y>V;iF)f~<4q!3OG{YDhR8^33g%EcaE7K}R1=`$894D|H*t zifcUk!Y3#qsbhhVo8Heisr23SAsEyw)=ual-u&Fu`5n6@-1>hA=KVol*gT|rApaTu3<6IiSx zLBQjlZ~Q}?=Gw3!0|DsTWD5~XIW0%qOGX#yUhT;qrKD;A;etA{$#Y{MU~G$4V^a0; z)={vtiPZ;%OnEFTq&aYS9BH^HjfhW_0EW5xvD{6&d3IK%eFW_oux6#{>a8LC6Jcz4 z9O3iK?KPmzGz~US4uG0+7*NR2tvy=esq^yenfiIGQexzfn>S7@^`F%&I}kBSV|OrW zes?1>oRZ6Ya97gVXcr~Qjn|JvoyLlr1trC}y4UoMB%_CfTs?vU=-sD+-_Yrq}KF;?iSV4$N@k;BXa|EQ9ArhXqQ#iEA>46x9G0v#MqLJoJzs zscJv#Qm58!@?*D7DL4ba=txwttUUc8jn=9PFM)z(+mGu=yo!{I`w9Z-B)ZGM({|Cu z7%)l4q%m$@P2>6FF@=lU;<>~Jq*UwU-L&3x0ht~Fnc}t&*N2PkAGrAxbHabuH6+Ey z%@ME{Oa@DS1tl)`y+(iH<8x8X@VF0;Q!Q1uZVzSRlo}YY;r0~xhRYBt6-U&&vZ9~; z&#q(hX;()*o#QR3@KC|x5E|IHr%!&c|9-#T8RP;@EfZEV8$rgP$siWmL%OG4+C8Bk ze4y34Va!-(m=}khIjnnU{MJ0-2VY;L4gN2mr+PPPJ1vQsc|m{p`pE|!jUp9|@;DWC zs-n_~uBV^6I)*rDDiye5Z2fzq(`3?~Kd;(bP~6UVpktwH(wVT~QBZMSQpxQq18lO` zF$YUyyGmYF>A88=h;w4e?aq3e`~s1ka&&heqQX=Xif#1j1dd~g&omnGJUbK6U)7-g zu0QEL$MWe58}pnQv{Wi36Y=nP%V#|f^c6g6Vz)_R6XpeeObq*_(}BsIatw%QXwuKH zC>sZ#(=4&6cEHAQVa~zF?130Y?E&M^At~V?QJQ#C=|2SZ8P1>ZA!ubH-!B#1AI>*J zcs{cM>DKX$re9~j(?>6qk;hZVD)?L#oMc=82~|*TcSV#tN48V7;^+90Z(M3QCr<39>=4?#U}Y1`Jl z1Ipn;OcWR@14`NXaqnhAW;Yvz-;8{{+2C&ZI^y#QVv(rL6ao(+|Nf2mHeD!Yn=Xa< zPq)kXO76}(+0TgZjB>>W$|gS!dM3f@*U}j|9Z=;1Eqx`(Vzw*q^8c7LUJ8tvM4wCj zH1|+?50+KO>Bf{PWE4y#f1QDgB8gL)VK@#CJRGdyh2vU% zH|9MTB!BqFFgz1z+Je<%aJJ@_Z2nLh!nFI-WNvVAZg>}UAlfk}&BYbrbJmll8Ajvx zz~>_z--*mp?IjiaqVdl-jIe0Ez!p)LR#Q&2XCkFsUA5lu!w$2t5lE3|r{svu0d_}@ zY9N7>Ii6a0C;DimhVB{LO_}g(JW;OPrwfXShX%3_xUUrK$s|QfTKO8 zm!XrvA`c%+_t|Zg{#S;?f2PU(G3#(b;zMUiz)4cJ-Tj3?1)@{z);HfhRh zg;V8orf8D|i^1bq*myNmtpPu&lHz0UiwkxNL1k~Jo6tp4&jqsqQFmsTAZ7p%M;_ke zxNW3!q5~Cy`YUNmijT z_@RmCQ=7{0BGA23IqCFtR!EitDGF!=YqzLb#)3GH()XBM{(0Neh9DI9#`f-4#H;k+ zY0+zS`SgkLI6ekD&C+bkd&Z4UMvm5IxumClkVSu2U*BxvA!o7mE-NO#tX2g1>xkX(8)rgRwDPpo;G!3@yW0u+udkCb z(e4qG54Z42%wf}Uv(6x4ta7;V@T-DZmh`>u^lHsOR}|UkIVl<6qjqX(_x1<)W=!jY zAWC_xI|tGrX2~itpToC5zRPb2*wU(1wL-JY@=2_eHSy#hQw0XXT9f#Nf$um4LfG?v zrkpHtmQ$ArXKjtTfsDC8hBTni+!;*N+r7wJ-jGZvDf8V68u-{!I)=neOQ+?fPF;N$5$96JsCb4D_HAWF5*5Mey-1tyy;2E-$f}LY;_|N zS&6JxljWSjhBjf&-NRRlF2rn8P8YN-vbDQg?DurX`a`|ZRx=|rvu;IJ)UcIZu0;k& z!eaaJWfhr_171Fjv3IPH#w1Hlxsi*+F_$l_AL?-vkW`jwRek39EFj?jfbDa+BMR(G zR4e`ATsYoK1jKZmw>GNODZB@mlF}eWlAfNPL{5FKqVZI?_I+k_>EwyqC3Y9n1tsrx zzo@x#^_aq8aX^7lYyBE9#^d`bm4YxB&BzGS0wl)0JXz6Eb*?3?CelgpBfqxK9<8X0t6Jvd`s}`M2l7JD;R} zp&ZubBY}u_hi1BW2p8o;D1kei zU{kq1SGw_tZ-oXVD=dNeluadC`xZ)K;Uj(z`)9PFe3Y<)@hK6A9Fu#e>q;K6b3&0J z^!P?M1!3pV4GdP+g9I)YKUAf!o?(YK<3~k&=QDd+oPZo+hc$aCZRa^Nx*znRw4rMC zFiCT|gE-JnVn$p%?-|BcTuGe7Qd!(4f-o&d^JC9*m?%NogXqd_2t|a=&IY6q zCJ%WPwLhi5#{`5C7g}#PzI0^F(Q(I;2Z^TvJdgric-%it^vp`Z`sv9mbaQXl^!dPi zV?@*>%BI++us-uokwYN;_k|)B=&>ZLAyLNI*wsC*SEUPe%7JMAt8O?4^fkzgg;J4@ zxqsmxgp7paG8@Gny*9FEs7PR!>)~Wx?)4U7qqmhPWx;;H!_#mecnyo{$jHbn2$(eb z$`5MXQdTO{Y&#oFV9;1ioI9CO^iGxg8@u8{=|iTCQNH34y3d}XB9S;+Wzaj2!R5~f zrtoM8+xc54{Cl{6;Zw*PQ-j{*v?Kp2*1fI`0UKOy=r?|buQZTT1_+@M0$#GFkG2$F z^?W2KD^(#Y7!Oz? zkY>^CudvgeeTAiN@*jN_p7%G@A~D3Onfl*walN6hw&TtON_CQ7fl`Z@N2n%VlQ7Wd zXQ>l+pN@vFG(=fPeCZIIuGy+tBnG0|B{Vd!>A0O}vIGF2(j_QPBRqDOo}yFVaqc(Z(7yfBe@I+E zUFm4F_`WzggAY%vUi7MVe+&j>zzYKvt~9vQL^b7l=(D%RHReuyb+i&QIduQ?3CC=@ z_2xLK)Z+ajKhavW;a5Em{HzS>%0;n>T&K-Vh+%UMj~UO9p~hD|6#K;%)}>b0q|}A& z3E~uoDHRYagXQY+c3}b<{f!K2nc(XK3Xw{SD^5sj^jypCP|)>~8_PcCf6B!Xy|SpG zcEsxoESHzj=Dsk6%K2N{yrE;|h}4ct(f7UldA8LW4B~W33cr&I#jxDZba1L=6Uuj= z2A(d=rhyuy^Y(*BO0~8z@U44VX$lf#bzu@=J0JV1e}~0%@%r<*^wp+#)7&kfY=jc0c)CerRz&4!*hZU}Q>Q0$mx~<4f+HE7WoXOnnQ^V&K)}xq& z1hk1<{z2$3z!P^ZcJ)w9!#$7uWw9V%^ z-^e{Bnvl!E7|GJf9Lb4-ep6dhCg76{sN}ka7V+_zjj*MWBbHCp^ZX=l16|2d<&v5e zbhDMiZ%Q}pg^i7|D|BQ*vLuKpm0fZ-LogOL6FK}5ZN;NvR@gO%Y}1RpyyWK9t2}`( zZ9F~ys`dYUhR6}va?J*jW9i(dwC!QC$&-=B}d==;?mM|X_&qN{m9?_WSR4}GoP$ERevIzVTG2Ld_g(Q?;eF| z(Xcb%3-PQytur!kfla4w<$?2Un6oxPp_d<8fiAqt-0Y6nO9Q-VDVQeC`3 zxP87d;}z*zXM#N~nI~_tJe)q9bO^s2lVQhL&v*5AuCIwE)ko~=z!=#M3hMg`@hi*F-C!5JcjgR|)MCx~-xX1(qQ|Q|sGg&+(sRmuz zcagWNEk5cXS-yaP0Pn{=6gR_SXk8?12^AAEqjq1acuHcv9ANpj{;o6}*B-+TQZHpa zu#5a(KLjLxB^@JKv(^?Q@5_B*bZNGI0FR}v-sbmE14fafo> z#bXcr2*=9!m-ubI&#)7Rnxl#cLz|W|UWEUh$-vzS-%G%aS@Y=(X^)BVX7(Ysju$_g z^Zs-Ch3AvIjcy4p?r_eZ!tmRl5r58*(|#1;I)G-2OkaW6vaPz?+gqzwqToM0FSU5_ zc52Fvgm)DAq$@d0v#VD4Ez|1+h-@6`tZM z!ro!H^m!$5{cZsI@3a73aQlC@5^!SHknmPRzqrXoOERtJF8n{jzA7q?c4;>W5Zv8^ zyF+jYP6+N!1`p2Q?(Pmj2MO-(t|3Tp3GVJLr?dCJ_xa9$b1vX!)@qub>Z*FH9zkov zOg1?^VqE#+J>8QCY4GbyPNauY6&p9Bt^U$Zs(gXR(~@sm!Gi$%-?#qnm0Y%fD*-r| zh1VZDJ4dgHUE;3H*OvtEw3CaQgIrne+1NOYUI{G-Ma9Gxgg$>ZC+tD0dpUUidI9^t z!s{0)1CE4*MD?o)JH?HU%m)tkhnKcs-(-oOvUK?y9qiK|C%}=BQT-ltP}TutbJ?-O z#=xKZM18^{W)1bfzG&=-Y2tRu--$ne#>tqx|Mi_sRfF9tZbUU?NpUH*#oZp(*cPhzPFJAzpV8OJfWgATvJJSTEM%6bmG;OoCVWOX=-LFFe?b>p!$%E}X1n!w zfA{3YuqeG;4yN{;ZoD<7T+x+!zxd4Y{YMvSDJ;+9%>_LJfzwW)7qa9yJ5`m}qcf-N ziY{>6;j)|!G%&_`VA5rbrWF1HiyF89^j17R`b3`v0E1Ym-eBZvV3a=p1B@k=kW$EP zYbaTG<9c(1UxPR(O!nW!{c4*V6gM}wvktDwNMiR)sm6lG7JxC^FdwOU=r(eF58d4@ z9UMG{vOyb?V7DSAzR}cq;JH-)N#D4-(WDVm3|IiJt*`qLa9OO~oo{aJjZf@?0H4~0 zVv90^7zpFt0hJsvS}Pm zB8Jh+n70;V-~9pm)YG$WYU*!+66UjN&okcBu?|*bZ56B&N9XxY^UNofu(!8&x_d>$ z3;v+fV0F~Md@xrT699`!u8<|*dbJnFGx9f9{r{zWA4>)WkZlGBUNp_(sx`;*Lgjfw zf_3NJ4tBYq>ln37WgyPAcou}p?$7Yec5QM3G#}M_rz=ftWEz0 zivQpTMxMZD+al|=|I|Dn`UfYTsto}X=l%KDw;iMluXG|gl`OvtH3Tckq*<)Q6cHK} zQSXWtH4eN|Q~^rlt5+JGL=Gu&z|}^3K4b;8ZZy$fzml4o+Rkyyg!UizdM|iy)KAXO z&!4KJ$@-QCPyp}#^$D;5?*4`RLPy&G9r`$;XozmIGa>Z3n4K9R|HrclZS`OMyXWlq zL0|pKik=sZUHAVUrGYD7ww?nS%CS0gfQ#v!&_gK2utLtMQ~*4ac<|N!0=r5H1A$;l zQhj~5Opf$l8WG&Xl@`%w{hk0ZG;|W1Dx*)Jljve)GFFNH+?Y|Nfqx{EvC|RY%|+k; z9jBSdK1PlqF_dORtAQx;Gdwz6R5}*aQrZt&-bvQaeBB{U*DbFHHs7;W(7V^fnIVi(Yh=rt46Ot z$C2@D3uv^AqXL!1b)}U7C|+BU?)Pg#!2W{{s8N89_SR?;VwQ_wU))yEO@ifOoiu}N zaBkumf&29ynS(18x3}#oAdBoP67%+dgaD(QjrOhn%M-msIz8<+Gdvl@pAnIvzPGhz z8`2l&x|%-z(DeVT$le++=GC;o!Lq8p7MHmvVRb>&KN=E*kn6cnGn$sLOg2YZr%2mG zmZ>DJ1$y?cP8-h2ME~3z0Czm)Qhv~~r>0zTa`&u3Gk8l3p^+=0+yMC?=JJJ8{XjpLStrjp4nu)lhiLs@S z#_PpkmFhIvjK{O|fNKDCzv%EJngJGoTLteMzdW*su1<2?}PE^|hnF4rfvoU6c=!W;v%o+kjK zW9LU;@-jMuO{lu!dVQd#TBeB(IG`eDzOeL&kgC+%9XDfYdJn=(@}wH}QDfF#eqr{eLDeBk)#f z4EzW!Jk?9V#$$4*cR@kDzeb^Fh%jC|Htn*4yn44ksLgFasKUD@1@`Cgzl%#r`3M)P zB!ByVU}ECHgzIc?k^-pOZZA*Gu73kDUJ{I_!3lAo%T(EK5i;oTv^t7$!9gcptT!T_ z^we+10iOT0kyL8eZ@>xGzVB?hL=6!@qBMcee1XGePsjVsm3s&Cxd!VRH~_MblSHpE zunqMHgo)@QRr=Wut$JB1kD==j@%7n=iRb;377$xmJunA_x2Kcxt|hSM@Lrd7K(N^M5i&GrE8omAWnIhuzD*%2d@bW;)YIk(rbyoi|2T1t? zFZSMYG3L)`cl%S|2#hDx`Rcn=sF15N7(Gnwj1GK-W`pZnJ95BbFlWvrI1;}z#LwX$ z0eQ)PsSzpxfC2FYEwGqR{zL(DxDE5)%~fXHo~s@Gf$*3>9yGc;1I);zHqT0uHl%;U z!z1)ea+OY&>u~4GrK>HLN zL_eFB{WU6Q1cwrLmaqEtXucZA0AvZntS)0jv$?Nh-+CP-JubK4D~Nd` z!!W7478kXNVBQDK#RxUi{Nv4Be?ocUGDL@QP)O%X zYUUv^Xr+#(&gw1IAY;6<42O0s#b-B$J32b5G8!ADY;`D7drY&|VO7=q@i)PLZ&S3h znz5LvayYT_L`1}t!h}s4v(DtrG{FW0O+FLJ=>v2GHt%H~-NjDSN+dK-*XuKotDt8n zZ>^bw2Ks-~12BQYdnsYi1luN7D!%<0f;_N`8s@UQ+-dh#O1% z559LIWKgX7fP=ahQu*?B!@TF*`-X^+CRz+ICech0`DakRV7<^+hb|Mgx*Ypw)^p+9 z97cs!+1W?G#8Q~`pFxVWO4Y*?Z2|9Njn|jwYOmQyao+r)xH{8mY3PT$wKYE|I^~Xn z&lS4ORVJ6F%e)Q*u*odIgBhr&0IX&I5R7KNqK*^Db}3EddBL z?0fjiPW>{kLqI|bnC$+~)7e#!@?#mVTx)RY>7+Q6Fh~D zH+qB8(Uqia^Ukgvt}!eRR(!kJ+m)qJ_F%9f?xVNu%2%b=4)_CR$@1IX>w{>{V!bNs zL-pj|ONPIejPx4C{sjRKo5RSj$2V2<_v0Zk$~t)aMj+f#j4!KRpPF<6?SeCCO{nD*dY;B<93 zF)||ak9~3T5ELnBw-2RgsR+|OX7P=J(k~_N<_Ka$potfF{t;w3ww8$!Cwfv#D)=)= zBJ-mco8g}Ek$VA^GwnNj!xg+pym|DKZJFGctxGr@v;JQi86>NC*oV%8b-z;9h-K!e z!Jgh#q*JZaV%PGPXXM(~Lz-bX1IZn~G*gZqj&auH^D7PsVaNlX*Dq%=V+1PJpTYv> zo4;}4{Q4e<4p<-d&U>AG@mvogpvbDv_XUoI z?I}rw9tIJo@5Sr!>}HrE(smZFl0*;(cpO3O3RVkP>eSRc?pts_Q6ALr@bN)&W0|zt z`EaIA$AJM1$%8t-v9IeUzsD}vA$UXmRP?h7T{&3k!`TGerQkb@P0NA$2|%f2GrvY%Giz<7R-%I&eb4 zo5%Tzmfd88a_ql-XnZjFJmPL6bJv~_@kE=59#B7UqqLoS*1lQVznFyH6>W*(QwzLS z>zqX>lb`J<%dXr7Hv4QDkb+*ru{Aq9oAsx{%W=>3Uux&czo@{-m9|tPs+ycJm1()Z zdrOe=X_dsPrzYX_lbG_O;a#Pk)}D$OI@kewhv-7ocbB&;`y&j07Q!4AguAM06|b!I z4nwF>{p-W*Zm0sw@1W(=HxC+2G0Odj6h2wYLmG9Q^y;M%158VKI;&)O9Bw!`l`E0$ zgDh){5GbBox?=S#R)LjHJdPg4Uml8E_p$_%Vw! zCDrsXcq&nnsV&w}58SpR8y`2azZ;FNG%8v_@zJJ`fcr+gpUhZq)(N{Q( z5yI9pjB~ZO;x$EhG#q%!bVV!c6!FU={R^E@oWE#qb%_&lvXFhLq*9L}@+;)F=|j`o znw)HMd1f-hsofNY1b9bDl z!yXD#i#9g>#T2M&tlSDu!@m?cZIu`+!gLeu>w%VR z^yoJ_kc+#XJ#H0q#o^D<|5Iz^m6f{8#yPLDs<2w+_50fkv(7Fq`ix?J^3-|1Z{e~C zggol+G5%r4lKgh~)%CTcs&LgI3DVST$mi_d@AxXsNTd~VP$w)mR&`GotQy^@^FBxB z{(^S=2Opk`q*r-P*JE#u0bP}NwaD4GQCEqm|rzy3B?03hm)1(TRwg{^{nzKgw4d5KE;)A_mP0= z`HMjN`M_>fpRFH3nu^G@^(+z4mPjRx@O~wW9_+(Q`^gQ8`L@zVg1}E0wrL?7%T|fl zhGNsVXtFtBJCD*PAGt9Ymei@vNItY+e?5XdyMFMT2b((C4E0{M#CH<;Fi)y@V^608 zY3Fu73{KeKhw{l%V~kVZgI*4M6P?PBHDKX5lqE#`Q6VkxC27P+yqynEwH+aNx-5A8 zq~6|s4yDd&F)_>t&_~34eYnjjuF@xQImuCV^@2u z5MljHK}}&XFty1y*as|9*Z~eY5xXm^K&3(dw6pa@r6KPXqY8#m`bWb-BmGknxEt@E zR&j+vMoG$lcK_4!yy9V~@R$w-8JaNU$gay)sTc_RcmxEf{MgJ<00&PDt<4(0;ls86 zdW!`aw|l8c*&|R|SG0$Y&Jv8MU7Q-`x??FVxGN$Z5_@+xIfvVEK*xF%l9~%Al{3($ z&@G(Ly4~1C>LWH)_*I;~u8-i}g9wisN2gsi4nvZ8hPBEEdA^q6E8TA{fzCE)ndvBmM!jiihK=`irqjP=rfd?vs@ch-md%IAwMeD=XQQcj@RNlf*wtIjKgl z$sn&pIP69nsq%V!B(JZG#xEROA02xcvc-Dj$_%rVxFQgalSqmDri7)==MqkL!G+%q z$RX+qtIFTjrghPJ2}T|nAm3oX2xVl3Ra=r$m41DwJlBYjfUch|jHj(kVM)WAZ2w~B z>G_PXPDTcY`NI?#9>OXBYP8-AaiW+9Kb$u1?Ek8dYM}%Odl>>!{cgYyRtKpt;9`Ug z49cSJa655%!CPH`oZ);}8^ENwds-J1YWINU0Qs9+~Iza*03%inW~#ra3E*=&k1 z+NMK&U-hOCen#Du&omTv9!wou?nzg)UJ5~ySM&I~lfj#MImHP@99{$sH}pGqY$A=1 z5-(yGvM*xqXrACcM*4$g& zWw<9YJc)Lm(CT*gT&c=xo-=K=r+m1!K5xboHajXCOJiIuoI}5BojIeK`1@2|6UJcF zqbTNAF0uB5Z6!`y(=l%p_0_G)d zPTxBfU!0$d?+jD(?ZKD+wSNwZ0k3)ckW2Tw;*LlhJqf?-P1KqaNax1kPh0 zziBYcDR4X56_41#)`x&0rN7!1Y3A2cGzt&HfF1mWv2$4F z*G1m*mOSlHGEEL#*LJhlq|Y@|6&_(26~e8qUy@hE{t%CH*DJ7-*l&NUFMAi)#q6Uq zsxN@Gts9AEZ$@(P-1x52$(ZqTS&7kD1;)I#ExU~r9Z1g&VT{Gnf6LsK;&v$}t6p-$ zGn9xr?WH$3AWu=Vy*={N(XkdP#rU1bxA?av7TCcY3!+%0_M2y1o@mUSMxk}wo(c- zGUqd!*WOSD6EOLCw5qx08D(RtZ_})emf+mQv6gl1iuEeF484V>FBqElq`Q$VOWfN0 z@OtrvKl!Aul+$Ul-TE^nq-6X9rHd*`=9m49?*QraUzD!r>oGvLYeJM1FHuk>=hIti zg1F0|ky+`99QV;x^oK5d03VKM@R5s4Vlr1^eQz%m=-FBA*8n=l289pH3821#wT_6( zT(x!vuzhRnW!7opP5_avqnFe7^@&Fl5Wq{a+T~QA8f#Lha1J%;@wq~t^M8(jW1S*< zQ?0H~YS&_Aee5g<+iWI{$nslyWqYd$&vwP%-9g>W`CF$&=*(B72EBmPSKW^YdPr!j zPfz>QBEW4L9A6%ky0)D}udK3FmAcZRLXcpUOn$&*ZA}4C6R|zjJ)cI2mGO(`ld3U9 zP(S9l9D2_`eLz!g4Bz7gV@bx`p$7G7(-q5s>hReJ@Qw5JGsQ4f|)vLyfOp(`^zwy{?~&R98AB5f&~DU|suh^GsbRzFG1U;Nn6C=QL~?B4GVVDLmLlo9oe5=$5Y4av#*(9ogU3K03MvFKIdPEJl_V)m~^QT!yd@3AgQvI3USB&vFNRY`SE3wKusc=?P%S;D3fMRvW>ps6oAZ zH{YU^g&|ib2Y8x%ikf{Tvt940;^2~?ZcQyL+w$d);CUtRNvPdxKFc~oR@;UHHoN2@ zd4d^CNULc6z^B2$FQik(hleYq=s(vIs>ZBB9cNu>n<;D9jJn4^kCbfAWEc-pNr!(^ zQ$ImJB%j1!uJ%SnDLGHumszBvC`&;5cLs{;y~QTQNW(9)W>Vdji`~3F$DVgop>iw= z_1#^7YT6g0(wH66s71Np?=xf|mjy$HyQqhld_t*+Jq)V{X_Sgs{dJ<0G&wt%bvMi3 zAoe~dK>?YEI53pKaBCwP2G$Hxk_n9ti0ODAZo5)xAZ$ieKV4xZa>RXl0U<7`XZK}D zJbYENz=|%nE(JgD$+6yCmj$hx3B6UOsnd1C*!=meWpK;2{s-d0cw0|PpcWQ>q*!{( zhs>&<YsEfqZvB2-^*UZ_#@ zJpIe^)CO_|^e}Bhh&#hes-`s4l=1gSYf4iRNt ztqudnjUdk;p^dTdvuO(Kj3o}*Wnm~Q5uqwB6iR1^vSGl(<@i_AU-^335&}zpB2Ym_ zSrt^Kqo@9DJDoS$*BU$#FQ2PJg(xj%O8R9QWs>NF06u?h#2LUe?pa;U-o~9wc?Cp0 zAady+@qh-51gbd;@vrBs$asiR{C4)qtsL*|Y*Qaz=>f$-RH5dkX*9(V9H%5SedFo+g zUnAg$Q>$$bcAPw11D5l5r=718oXEke?b_WZul5-4Iez7qG4`6L2aNkYxp}blth@RUK#2t9&+8fpL&$ z5h2#0TCqI}CQxA}m-&u!{E?`CXk)cXPlb`M7Dvv%G1utwThr&X|=hSAR4_1e&Ae|tJL12d>tng)CN~F>6ToKKB&c}=W@)uj;xrp!Ui;{oGhO&Wrpl2mJUC%ez zhi2#^%bcNr230hfGhIXreFt<3ORXviGfqA zpNpIr`UEDdP50wLp%mU-sTl&U zc~lj(i$WZhx&pfqm9VHu)#4(;Q4Cj@ipY!ZR)rE01LamuyZTvO-*=~|Hhm&WwB)pV zPb1}DLF#A~dR_?hIq}-N66?AxU$CNTIR`_>`ZXWji^Gt*v!A&7hMGcqlJKvCybO{6 zpd|Sjbg(LO3_{fInl}>q1kyf2(x>~2Z711+*eSZ5QW;H|mm z^|fSqfgSK&`?V=EgZ%Hi!hUjpZ@mw5@&G#=orEMBc*Zp!j2^OEisHW7U2J`(rku+d2J}KcbfwQP{wpRsvKor zZ7FaOKt7J;%>nO>4&RhlSF-{2i(E)K$h}=U)*et`Xmvg|3^btO-fANY!H2h_)YWTs zj)GCY-J660G-^$@!CPApd27m^IK0CBoiHSoq(mSIT+X4NztJg_@9F!DXn#@eZGihX z2reJbj#uPmGkMSxaDiCv7Z7c~9ISPr`0 zR95^HuIdKvN#g4sW&}wTD35&76>@*;M>@W?hwO7h+e&VtG|E0eYVp;7`S?9VD#1Cm z-2wLp*XaAD-_KPE;J{#Y<*)z@k0u2{U!#zeTqj8;kQ8bDO_7i*n|ir6l`KP*@fa4; z=?u;_{Z@cVB1_w|TBS~!m!vTmO`kCQYn#V8Nw_Mn;t%;Rj-&!8wK$b%uin_nF}Bd$ zm=&{>4Yzk`K`$8wY{5CGr; z#wQryoas^xRA3lj7^5IB-IlXUQ|&(+GNJz(iO*pOU}r|D>H$@3V}Kf?QK2i6E##G& zSNLt}N12HsSQN6hWvAWd%I2^!@v8b;`cRw-3{dm@0>nn+DeEu1NY&z;8`ztD} z+uKUxciZt)KdZ+nO!+68;x>OZM`_Z(9Iq?|!`VL7DoSAu_XQYbf4Xc|L?hN~#&z~CT>tFr{1e1aU|sc!O|GvyFbh~-Z?0L}2VORQHOxlQ zsx^@s^OR63264+SRl!+vlq!c~8Q6g4UJ@%9W68W`Z14n~86PJ#xuV2dH|Wix`e+%J zL(IqWzVQ$!Jvt>2?RD()w2tw)oG{+}G7m>OSTjvyAQ+odXUN6;=yuj$&qnT0<^rsvX-ORqZX^f&az2$R@hoof<}z7g^b(jj4Yi6$sk zF>yRrM@frt>hm3OR^-Z0=^E`}bYreC(lw*ys>xJM-LZ}aF!!;LZs^GJBz;X-s;4pQ zfk@tTRTQQ$WSQBL{J7IRsLQ84J`>6VV{9W?MrAUuAER~-b!b7H_>7s$vAinUJlp2c z_x#S)mO?Xi$WSsRGRXxE*lg~t?ioLbQ>TZJYld1=S4*hrlNf?>VJ`DNRfCKEv~j#w z(DoOcYeS>27ds&ktTx_#OElm9?%8rTNNtZnccm3!D){)#Ac)4Et#dK&rL@P*_BsaHX&aWKQ)&f|&$^r>HObCD{U%MX#j~nps;P z{Any5VC2*+sj|=Ih(I|%0>Ut($KvFG$XMMZmgbNUpX=Gzo42ArrBUQF`TDIQD6x2X zdDj6|A;#>X{{En5i0N3yu=Znuwj4Qz$YAztLQO=SQVIl}63^b70;~IMzV`dGf34lR zfNWMRq3uniAqBE}Tm%La$jtVA2r?i$B9Sa?gf zWTJV}O8RW6W+vS5l4=2Y{2^7A_!-{`^)lUJyfT=;X>da~kj*ONch?LTb}nhfVCZL$ zv*7wlSYwo8NK<2inqnWfHQzpr!}?=e<`{h`ZJYNoMV7GfU%nDnwD;L@dP!ddYc=a5 zGZ8wv!s(`cD3S8v(NLQFb#J?>Ma)Kf@@Cg}=ApZL%y+5v(g=hhq$pHkJv<^kxBz}dVPa5#{528Jaz>wDd?hLdeO{U~WL_a+fp=vG0 zD@)5*Z@ttYD-L;MDd$)&92GYtVxp_cSD=qt5=C_$^sd@m3m}9UiKdX<-5e7-Fe`l{ zF`=FmlF1fo=fPz(Us_qoN=n-o^$;xTB@Dt;ULelYM1nCvn`EQt4v*$AQLqM7Si^ic zV=j^a45^kI2#`4L@9c3Y#0j!Z$jSf3ja(g?lKZ?8xcnzT@K&w!u`lXf$Ds52-5{v} z0g~vJ+^B^n`u4%i0Lpk`LkNI)7|WK27H~f8fjTE=m&eMs^IQT5YE?Ox#|Q5?u{us? z4jb{av_Wk|^5JMVIOF=ycPICm(h36ZIFv6{R6$Yg>JI2oUZ-{2kosv(vv)h)9^oN%7U(NHytNG4$jhOx-=>j=h26fc{)jqyboHESFhWJQ zEW8SZZSvN5?#;$$H*ts{dk@b*tLx&0xtqX0#6AF1qvh8&M9m>eeXCydH4i<7F|NB$ z{M{;UWk0JmopRxq%6hBb4{S1^XruTyZ(grHi;JiRIF6OR3()N@{^T7(0p)D%3sOL-&t4ck8=juTjyN z>ZQsjFDJ`mu|0?=fa$iri@oVBg6z@K+w!HW-T@k=P|qs#_N6+>LBFZ$^F8FYJ1m@D zDr5B=s2!xz;_XD`wL=e5=o(m=IC?$pb#sS!1l$PD9CiwDrn%X!A{+-i^YlsPNaE1G z*d;Lki3EgP*h3IZP9|34KR9gkg;7RG0vrdE^0VRd{PvFOa$%Hp8$%bG4hocL_kcX{TRc);M-dAtnV}C)?3%h5rb2} z4H}qHp~n@7xN8`Rcko3L?{2ab$GBIMF0uIdkIu-&s4qNF+};ixc1c6=$^jZ!6sR4+ z#lH?9&qYD=_DAn*Ryv|e2OGa?H@;}z>L%rVMoEarzAUvt6z>#{nv`3fG|Pl_+(Z0K zNkGC70y-Bo<2)4h7_#s}$U|_#;bz7`jF}*5*H^u3uC}Tt-M|DvUaKe)%E%2uaBSg5!8#+;QORAcD zagGWM9TMn`yEMvFZrBuHe8U^dufglMr>V&>uk+~~hTB$9BS-}CQOLphajUuM<8ZjZ zf|`l`$X<%;17|epOu@&Y_!+8TbL=%YT$X?Y_nU{{l)_{~)yjrc_x(Yx*ca*bSSnQA za#c%eHW%N&ILj{ywa4R+$)I;SxaVx&^wz&Z7F#77@Z*Hj17W%-f1+Nw9$ge^D7&5Z!xd0eJPy6ZFTmU^y@ zU7Z&Irv?-tsWf+{0C~%_d8vtK9rEwJw=KVTEdvs zU)0cncNiUKE9MQTqr3AJ!2XWe`it-QKf(MK{c#0|hliXCjU`y~*jO#Z&?qVA|4PA? zxA!!7kgv%a#12J55j`qAXNA6pg}c1h%E%)fsk&`y@PHqk(-79`%`ydW!qoN1?u2_tOw z#EFO6D%S?m?QY;MkJJ>Y2qpMEdw((%u7ArwMdmLeSt?hN+pg{sSM zX3>b?)4@;Yt%=adAJy4dV&%X&-SZ+v**j*}S}*%V@+-8|^&D0UnDa;*E#VD}}+O zH&Pah#s*ga4w>(i0j2BI$562S71MKWh3S@~y^D*Pe-QJAwvYci;$ng5k*+!zdAr<*MQa}V6|pi;99I)fU#SWvr7?z{%bJJWJn(FhK4iF?FIh1A6e9|5%baSlLFLxinST z6S&tGg~bOU^APuOEmJoU}#xsv6Q@zT`wFuxI!QJ#?zJr3z!J z!I~fB_mGlS#gIBxOuX1Y>dU0ixjq$jjrY3^0|A#d976&TB7SU>>ar130#me%s-b}_ zHHFZxm>+m1Vvx@&Gk4<8OQZkIT8Hwa>?r2OdMWR8WQnhl|G;O7t*E_JVB4EGEC) zpzu7KTP>dL0_|J#RYFbPd@6b@?6LELFA!PK0eJ28)f4j!5z;BFUNkZ0-^Cm2}Zhlb4P?IF1O!qgjCVjOEl=- zxO&;6CP(gvx6FJuDm&AQV0SW(E?~8rWgN{~A(XH-Rt#?MS;r?1I*ggc`{;rxzj7lg zU1DLbD8!X-imzBM#=^;f7qldGTZ;$x3s6UK$u$xWvpZelDdRHWBJR6?jeAt|tIR5_ z$}w}OOLN6Kozd&dV8}Cg)vj`_!=`w+Ho=CrI+%RRG481fuijHo-Jamw5Ca3=9>1ym zB7qS+oRbh?H<7veUoGYUJGg(>3ETXg|!6`b*>T*R4== z=-|mj|1O-88OgWYr*Jb|l!(DwxRb|XacMozc26ubPjg~QyJAJs8E*_2%2myjQAlXF)9nyvHY}ZF%%qdO`Gev zKd+{Cy~W?LH^;4#XZboE+V`x4W&UdP)rSN$(QyTNTKX?j0|NuWNySQGc<^6T`J`-e znsTaa!OsBmoz(@yx|Vq|d|8n*TO6OBAtGYFj~sy2-`uLwbDK|muvg`#bc`z!(~`mx zmnSE*Ml6SI?w*d_6W5Za%mF-p!(ozC+_FAy@sU@DgR>psKYt$z+D#67^fkkI?w~D5 zn`n}t(}X9rd0py+;`Ww-QAIy?LR(_F7co>p`1Rd5A4!mycCFP6!(C1*X3Aw%WFww9 zC>&b~!iW-j;}M>~0GoF>PjN?S5i&C2;`+f;ai!@V>E(zD<8Ag}&od$S{8sUWt{C0z zVC^BNH!iCDxcDP;t^LY4LYjlIZ@Utwzkb%>h2J06<478Lg`@D3TGWktPA2d+#txwr z+1WNA%QH>vP~iposR9x2iU?`)-sC&!yqbH+(I7*04`}UiUv9e>KKo(zmmDYlR%<8N zT>rNck5Xq>H?aof(TtK;|70QPOE&^f^0#P78|Cv2<@CUe!6Z1ZM`-SSf%~U~gLk|= zxH0_>cXI2?6(?FBZap9bt+#MAg-#-K&4p(RA1T01yj`>phyvr~16?+l_BY$SvKDht z4$-cn$wD+>_{*p~|1~!d(|ecf=tDLci95H9OQc%)@8&Rk%`Zh0w=(aJJ!YNIPv z0})TmPcuNm_Yp(7z|G5QEByPWhN1o-x#Bmx)icjq>8mUUY`*xIt3JF6t>2~Ejp?SJ z+Rqb|&iAbxk3M1`y6AV6ew~S!pm(s2)TxEDcF^u2w}oZYohFnx8cw5VD_mI}c)?C! zGhLzcR9?(6Y$Z!=Hi4CFUg_Hc_0FR%EKU_&b3@|c6Yo5ouqF3gN2@>-RcB4O`qreO ze8}!b`)Rmn5tkooS^QTnf8^-9Q><^y8QfJk#j9yn?H>si<z8I+Tn@04N5MzuOtBDZZ$|SRa}hZJWf|MjH+;kp6L}$U5%!rDkF^oD)lmkg*7f zZ^r@cKVi?eyCG=fd?6G&3P-sd1nf5L9~UJu1igT<_SY$@z(q5W7r<*+`6I=A3-+g( zAAeZgPqZ0Nzmvu^Mo%u@=2cxo4%7;enxkeYxOOKw-~Al-dLm;-2wYtaSK4?&cZgXJ za5v(N^VmEaUEg4r1YJIB%0geb%r~w>dj^?E%Yt*Y1_|t4yr&T9lxGNItO=7re{Oiq zH@>T$?~iX~V?}H@?4V7H1T1a=cIMhdVeq{2MayC0#3Q?8Z^HuL%MDG6GGl|ul(Nuy z^@r|7MWwG?5O+~&U(Eol@aTd#zH~KY`~ethr#q45{pP?ZuY;JxI)wC*j+~N_nFE-` zjb+l<91g&$gqk*qP9<0{3+(H74oLjQ@K#Dkh8?qg@7rzw(5i$0T=CZpOj7tau?-@4 z&=pUK$B*Bwz&5rjp8c5B0)f2H#!husvZ89eH{@_RgmPcTF0HJC7Ozr6bxa){bv@9Tv&*P<7W6ZNb-)7alk5KYV3(tFl)_MV&*vpF}igrh#Ny*@z} zisbHPlq&7N{;56{#r|8AiQ#q)!w?VZO5U3lzdMz;8vJZPoZY-mGT(4WLLPK!Rj*#Z zPI=$le&uo{6hlbnG1$+;oKcG!Q!y~|HBxt(?G`VyS7CE{vbg96b|S{H2U4m0V(w~k z0*{v+%S|?4L;wdNS*G0#_WjM9fqK#)L4!-skm>c)B@a*D+daOm^f_-Ufvh^&&tt1G z%?=;2=nRnS52q!LEwB|i%`7wroF!$szJ49EA>N5MAkoP3RT5Z^EBs3iX_?K zWqdMp>+xSP|497Uzx>dccSq5#^}Z=aCxEIwqF0?h$m?YQuk)Tq-a0#``}iOnN$BfAVqkhb9S1X|WBg~M6NfZ0Q!F}d zEQxO;>H01^iG2wBmtiq<+R$1=j^$17ylGMlf64i*H^75mxsi@9@#z;AG19)MN*0$n zzy2^M@9NLhtg5d!8B2G&+^J?ZXvx`jJXuZ)L#KREbi+&%{U(Njdr&7CjSKEjgk=Nw z#6E@u5^Mmvv}MPF3*TqTca6lduW5Eh(|Q9jkGpK-ioOukPtyz{C#6pEDbjrO&kzI- zd}dT;l;LrN!;9Kp8!=(p=45P)AN>F!43+_j=@SxdZXX}Mw{WX4y}fC%QYTCp1=7A0 zaiO!5lWeBM3{2V4r0{N(!AkeUyzPbo;F3KJ5x$cv2qr0|b8$a;me!RAMKyCka+E`_ zG`>Fr|4sLoFQrM3+5O2B`muy2V!i6yU$GCkJE59@bH?t&&as5Bc40chn)z!Yw@p-rhkiS7qI>HMmyYNO1o z<6}k;Ih$10tLfLtuGzhhWrARfnc_5KX5g4cm<#HCpv_#+-PBERH6EK`vl17~$8q+Y z-;_&j-)=4um?5X(8_Nv+s77G89BiGsK+a+~;F|uy38KBnZiw=cv^jK%gxd~)r60}L zyggJe@A}=}b7*uzG@8W=#bJu_30n$3vUc{<9LlvKd2PF-W&@H|OT}Psp~5+*$ds7X zQHI)`rI9hSipo4q)I`}FR@rI|j85RU-rP9zxzrmq6R2E)ULR2?ON~)2jg&`}+XQGVzCT2es-UB%mR1Z>rEG)3&8YrxXdh*$0HdH%3WwoeE8q7Ic9B zr+>0Zi8n&kmJE2QT8#n)ex>IG2C3fJ{fWqQ;Y6MXyB;rueKFXoVxjVc`{uJ~Nyfnz z1=@a&?!*74JViw1FK*AVo(fUCK9I)T9AykL@)M9&==Q;|Of1oWDmEHTA#*w@v%?y| zBFkRK9W{sm8b_8sTAJ?-v{bs#;_H)H$7gVJcPL~&0qsl7WYXXKlzW6^JOh4mRGH3-l>$eM1n2ZzFQ8+1wr+ zWwV7~GG$78vJ;}Qz8ysNJ+Wm@fr+qsv3RaW88A5X@G ziAWaTzra>1&_L0W_INg@tPY36U9=c7(7qDmOccKqjzWWDQ}1G*yc6o{625 zrjACi3E{PT4kRV-MX~0e&(_6&vyN(uGk{o2J{7wcZg$u;r4a{<^uxe2VN%O}Jt27@ z^a@j8B$qxTcfqXhF7%Ro4DV||r;zMB0O$63VzlhpW0gq~<8JtaW2Dz8Qn2Bb9z+jq z4X=8&^2cxNyouT87R|K;1@kX%c{;AK-pzcM=@UGfcS1#1rct%tub`UVlSF?a(737W z&NvsFDD`q^(x3|v4YDwN5maMxa;)=)oC}lH2S=1Tf6*XlP=1yncMy4~6MiM9t*!B+ zs=bvIr4ahNCwn2!m)5DoXXp;S5(+*x6Y^o9L+NZMY$f&fi~GXGSz1ap-6smp95ne@ z9Jf=LU0+bupp5*zDPKyP<2VAJuFQ&Y-kpKs{xd)<>kGn0s3!`CpN4oyV{UQ2ru3s@ z8kaA@ZU%X=fsA3X926115Q#xL56*BQl^h|g+}vh1`1IY2tuB>31pELq%4j33w3NXz zTVdO*ItCOR$W&n{+5u{T<>4u?3I;0U$x*MfIrKOQ>knXzM@Av%9IrrW_0AB3P z-p_uX^;zq^xQajM0eJs3_NdOBf0M|2CKeXBPxo+fDCw*&xDYIBlqMe-7+4zW2iRs@ z)(1(FMPTs^?DgOmY#2~xM4j#L)UOdC6+o6a%FQx7q?}nJt8!J9i{1C%OV7+G+KG7o zu66a6tyHr;My#6ru^-vHl^h{sESAHoHGIhV{nHxzv(dcy;f`lLP|82d);aQL&0E6u zOuCL@5hoXm^b`De$I0cH48&7Qy;uH>MRa(6RZ{vNzhi@C}sK_x@HtlZpt8WYJ7} zuAxr8(=E?vP8_SIPHnvIgl+8aDuE)D9m<=|Lg6M-wpt0A2gm~N5YoaP@A5qun3Kyp7*a@^ii=wS*ZG_tkD!F4 zWU)^?TI;r62?ncg?imjVu-JqfJ=~{myp7)If#f{vh-(u&V9>gMe$!wqoim2klOZGr zbRoum+De(@@wpV8C*YUKdfHWWyu|1=-xOA(=xI3~HDD8eCG_>aS~uJrD7 zrfhouYhEU4%?nOfN`!vmJ5524le~|tI6X|}!4SU0G&z~+wJ*8zwBQ&P6=quLXFo~c zKd-f;#2ubQa_giQGV${#u7glw#r_ZX!k&L2xDL>MPWOX95{WE`AP>2GBYHI1its=Nj)lIpErCNSi?`rjlciB5Hs9)>z=G1 z(80Wog-9pJ#YPXsU7uY57``71_~W15r9&Y@JTJ55lT42U);!_VG!ww9#zxg1#xUHoi_@BGl|r&b*;7D+*@j}GrWH^B!mHH@^fE2xg0(f zI0mqFFKu*}O3D0X0hhta>2pIuNWjX)ckB*R-H}9T{e{>bp$tX;m;uwbfK2s$YAPjE zJz+$a(e%g)t>E%1QI}g&;K!yhALKp6F%60t1zrN3*{v@1He4CaIJ_?hGChq6Qf^&ND#+|@MN-A z8Qz1#x#vgFXw7pFXK<#F4ln7eKN|MM7~E{0U-fI4h$@Szs3P@i)vIJyh?D#gulT`H zaA(``&v-3r3xi=1wz@l7R2 zE-%^kkJj0Y(r?rbrtuXCr}Mx$#@tDd&aeEjn3_gPWV2=P$`E@$FPck{;^5T7#%+T?%VBVs)z7X&5?8qI31C$T+JP%**h+TC0E*4yZf z^%M4Gg7rX?!&4>JrilMOj3>J}!$16EyG7lwTyKd1BukUq8a5&3mdL=JZ0--hPHFZq z&U@-o8;>PxfhXeF?=822-(+iJ+}hUGqqrT8MB=hjrL;8CR`tX6tV=H6uUOgm(F0|UB6TOu;>2l4K9^m@uHmukl;#j7 z;*g~E=HZ?pR&$-f;+}kz3E?(UJ%fa9*L=2gKiIIDwj=vFm>6;*b;bG(xT7|&-!A`Y zuHLz2bdl~!K8u(9D{6u2&hAq?A(nY;N0ErSsH{~qRZW$^C3^a^r)gxZ?hF_N{0qwa zkr;6((?V@ZO*^uh%!=*wW$6L~MST(8lu4$*_Zo>UbuHZq7J2{gv}>eD#${wS1YzsR z%gn!|#UyA7DrTSz712iegVn)k*spckb(`Ja{)4WDDko^OO2W7D@a-nTvK@&4G@yL& zQ@m5j=MzaW5cx!Fm@I+B0nh>8-E=KCqw4;QB~gsE0z))8WY(A%^b1%3F(MuGYrkYa zQK)`aJlD)--*a@y^J|f!rdL2#*N!#F)VQ;L9a-shW+KjC9#N&zn^Wpwzi_&z`9WH0 z0`Bx!K7vf$$vEaWkbdi#dhPcsbq&_wO*AU*#6~WUxxAgWpx5x8`rW4snR@&ho+1bJ zt%JVRe{ej^8({I^PQXy*ZOXT@0&ul{;1ADJreDc*(0RnGr{0~d>)RXGQO*tB5|}2; zO?LhFvvVa6Z*6T&1QwS!o?mp!21TtINzO2Y*N-$6D;Du))p1~-KYs3(wbCYjr$#q? z{cbc*@MKLnubO`lYqrB*%I%^+B?VvX-JY4%+iU4{o3&`V+)V)gTAGQ32(IbmsapWm z@+34RLFmtS3=}v|0V>J>S{~~Si~s$t4PYq@QXc3hsRepxnUYa8;oa=uIF`Q15Qsy+ z1gzpQPA(HOIMh%3ib^D;xHN`0;WRL$1h7U{O_2i5eaRv&59qXJpP;X41Ay4ISlG7l zeY?m^I_rpwi#zGDKcpyyE)VPM?MCkz6Gq1Ih_A8bSHps;ljrB>fmW?}a;2N-L0r~a zaYp^Sr5EB)f|ksna&p#p9p4_5`a9Zt7YyT$xX3HjvN7dxtwNj^KvK(ojs7Qst&)nJ z;%Z#WKJjwX8Y``9Fi@Choa5^9*GI54F}e@m0Z1$)qw?(L(8SA>`r?&G$esZ$clGox z`|h3U|ACTKj{Tm_`Cq6Qonw{FPnS8>la-Y`y7Vf;fWo)^|3&Y1@I^|*g ziVQw0Z$jqko$H7iE>mSnor>Rc>``%HmKfE_sSpR|dyiG{@#jJ`&qO}?#TR&mP8XYY zNZ$O(WY&0oE+NS1^?$f9=X*0fh67Lj<(`!F2LM;HrR76G4NFy;e0n^Oc7vla6Xn+K zv%CNs+b0@BP_P80pnbpcAPX#=iwFKqv*qQAY}@^*l!#k#L5$>WmZa~XB+J`RTY@rH zwor0SF~1$AdZ4mR&(N?&i{F)HlpLGoT6%w1sZy`bmW16{B}0%Cx9@X~{T&C*8Knyyrt1L>hNFm#y(4Gn0)UO1SrP}A-8B-Wy+o8YMo8e=U-hNWIelS*u zE9R&Kikh$yLwda#?FjzK#1N%h!qED#lcNQa2&M0wr@&yFQ0VH#)2wmF0nFGFXxqmr zie=Y$nicQUq*l;^ftBQALU&dlr_ph)?58lB`5aQ^LI9lmzg8KoRDUfFW#w%YrNPrC zWq1IskcLnElgQ3QU_l}80?9Tg!6`)DS=gBwHiuKVznG3$%6+)o_5-RH(-PKzje{y} zo~c^DvFt;5NC?kSSaZr-Sqf!hcQrH|2SuY1QPekChx3Jng&xH6k7U(e$m_&FcOdq3 zjucP&jJV`aY8e$p^}{S@yUE7X7~ep-PrrzjjJ@F_k8#iK# z4;~r7~%&7bqbm3YDf7bzGd)VQuq^K^Ib2FO-vhHn9yPw~sIq(E0tPZOJkq5pe) z9N+qy8q2_`H7@$8U!(boXNQKXAx| z<^TmJveN<(cZY+}N>rmmxY+FSA&lVi6<4LVNcpU+EPG$2#Jq1#;HjU#e)Tj}$x$!E zGwD-^o6>4?k4134F(H1&7l(@1)kG}qpZs4q&Xb}OU}hVJ)i@b4FE~LR)0tl~p@iqb z&M*Z^f}@=CxuH)H`sCQ+iBd4Mp92ZZJxzf7w#l>jD}~2kUNr}}xcF3+$dwdvApk}} zAdu8Uxma49Vy0p_%19+>sZEq{?Q^K4TDIh;m;}Q}bk^CdOcA$g!OPl(*-!JXn?o@` zbcv@5*_?ye6mdvN+&GCBixABp6^$1w0nFwrkUPyyZbZ1~uUkfNCd4OzAZ&dTTGrE}rX(XzX ztRRcgVcSQ(Iby^8v`M zawM?O1>x6S6rBN|P)jaWS4q(6yv}EZg)x@IjQa4-b|NI8fw;j@-{c+621exz{Q&>~ z35~hlgarz9`2J%QH>*k5c}o6a*8oP>3+KaUkds(aOb@~2>xLH0rwi&;TGjyIu(2ivI^F6KmpJ)gPeAg`cF27|;V~bb6Roz@aP3E#YaB z6>3S=0w+{JJG>4^hnr89>Z_*amF9QHa~cY65YiVBCVfHRzIWY>=Yf^efL&S0Z-8+I z#|NdBRyEW99@r4Gt*OQeNCrRWpG9sT*zQnLsj%t9DpIJf9nv?DM0)7z>aG}d<&h${ z&Q9}vB?%huGqBsL@M~V}^^tVw^lwvURF(uRR@Lb>u44Ti7^iDVdcK+AEY<3DN=QJ% zWc5B|lg8oztN*G^{hNzU@z4BzxPI68?ZaA zP8tiDud#a9P@JZPxqMqDOP5}P=&?i?66jBNm$>-3q2+I_>mrLSc z+H;)f>am%&cp;vjaJ*Yb?8y0QrT@}Zlk;qNv+XcT^l!yZ@$~Yz9^XBVx{zxgHS6^s85 zJnv?IY4eD9Q-4~>2ww}^?EmbLB4cCk{nUq1445|{e!+fu4QjuVA6z{z3jSML0C)`5 z$NIA*TD;i3bN;dO_+IyaTXBv6HUxReduujI_*D_J!`e>?e9iQM9nQ1oa#);+FJYXQ zO0BtCRe;@=PY6|3#>tM*rWxrz+@BMgRI&%OK90nUzuIY(cY{{;_CM!gr}MSO?ow4F z7$lJ##1?86m`M|(;R058qc2{d*JcB-&H>qSKvqD?BS~K{C+Ftw$NyNL2D(LC9u58$C-T`d`$Wg?t`b>IonH|o^t%=O3ci;K7qXY zR1!!5m@&2*=g9>Tn!mwR`86KUO)0><-`Ri6miy|j8@8RGQFl>@?hCf|m4N}v#(grn z`6}sUKdPz{Ye{m2*}Jn_Tnms0@wUVXy=L7fsBpreYVF{sCm5O9RdWz^CB=zjfE{hCB{Oytp2wQ@6Qfh~Sjz~GlKeCk1T76VC< zf?_cWtFz)Cd8eUVj&L6UGfLz}gxDtafabgMgPV#OC49F8fxsGYlN<)w>0fEf=8#ci zhwIoXP^dsH2#I?o9A^1z9T(E~g}@~r0cB%t3y^TR*}e09&wy>COS$>z{NfI8ZNH6o zP~4MEPsLcF8z!^<*_%pEKz$j??ycU?4*_PXCuH2!$7<>C>hL;O1Uz$p&O=l* zOujYw<|AYqV|jjzk)Pma-rP4L$zpvvzR|vZJD9tBh}iSZJK!(16PW(+VM)^1EY&*X zX4*q<^zNYP+vO{-6`D(97G*oD$WE>y9@2$)MU(cG+$%_|%+jXIfA&Cte-HT#3qvNs zi_dipe!klatW;@Oz$#091i}vYoB2#=<61T1%3@>Cb`LlgEz0&-8~#arrQ)2lv;n`6 zNT6(plu2#fv^EyV0l!{1m@vp1>;mrCH&;Iv|42WpB3>caYE83KfUcev0gI&`R0CWF zEdYbq5;%>i+R&2`H|0*mYlmf>bcW)xAo9?GO+DL?5|ECIZ7v@r^qK@7OfgzBQMn0(%KW2Yw0rmsx>vG zqYIUEvsBU_qHB4bS5Sw}+s8&O-b}}Fz}1(79ey-Ji_&%VX>EoWXSnwU;X0Y8H{tg> z%?75G;dX9hi|0Qy!Ptp~Z#$6-pL1^Q@y~gZ2inDo`_0{#_dmSL=vs!7{#UxfXMO>E zMWi)b0Pe;xU!c74JB5Yr;%5+yY{5We!S zRbm2cf4>B#MkXU8Q@TnK34@-;7t4PCEf@C8oO-f9RBi>;Of4BHq~oXHNNp>pp-v0- zhcD5Aq9IC&F0kD%$bTB?8J$ZMI^Ce;7>u)Ca{#mMFZatY#BKtJulFI%Zv9EHJVFmc zDv9^}R5i7=uZ>6HFnY{{+)*RNU*}g7zamajXfL|<6B-EP%O8K{`ul9A@ndxc!4jtKwV}^!ZS{zpr z2Y2dTs$<3}kRd5tsW{-{g&&^4az8N4D2Mqx0SID2*hCl`kl>QCDu@oSh z{=)hv;IBz@v-uu`!6P6*vbkg854P&}h83DXHBsN>9HIY$?AR>8WK%NU;5K)R*Uin1 zbV9Y0IH?)2oy-K3GFk~Bq22LIz`n$QR0L&{p=6Y|{*heHTM=C%R(ZBKn975hfi7aF zt8SFou)hs9!~mYNX@Y(%Nhue2{NA{_^D=EgVhUS+!A(8hgflh08}uvfJfXm7Jo>loXa{ z16NlVpGjnUf5fFT$W2sGahQ?pXojDiQf)q@jL(oR&f{2^QDgwODQu|i=uHD>bWHsB z-mnFxEyVz%*e|UWHlbie)AlzFA`o@@|NV;hNU@SqjOiDU8nEc-zl!JeSo;^81j74q zKqpW{ewAD&AtzEa6v2dOdwPEnZqcOk6-U&tj1B{+vv)&h-y; z(&?L*1nhATYdYs^^}cs_FmrwIkQnL}Pm>i*Y3nkBxc?K?Kh4zR>Fa@Y!Wk!esab_#$bc zYzWp=E?~3WIg%q2oJthdy!#Krnn{&#IhhTk2RN05cI>apU$2`IAFHM*&o2CAVTeF$ z0*>YV%~{()neuBta~&AX6^LBX5hh@fq~afhk#fMHs5F>RBy)wnUW-rJtmvbp!BY&2 zgw#S6J^JGAm~(as$8P}uEL0VKnCHsx@SVM+n?vsANH;2;3OW*Sk^#Q=A~8s29e^48 z7HR!vUVg7N%le~(o|Xj+-c=@Va1o{4TuUMI=XL5pUoEeoY56ob|EvUtfM@LC`$fq(ND6Fk;(X^()!gcph1Ov5CpFdBsFb z=(zJ28=|>u?XO<0=dXt{(?ne1v~(8)O-b;OvA-4yxx>T;#$Rt^!hrY39tD?8NjP0pvW{6S(m?y?I3Zfv@;l$8! z`F*nsQ}+vS=@%rZQ&%EV(2;zfPuW1<1+LDoDUl8;=p}?uu56nGrOhEU9@(j6VN5BZ!M)SmJ28r`Vwg3`d4iM!0`Dtxyi$zlOU9puNAjg2f z;Lg7ZU&DEWi;)ze@(CZpW*5H3J;Bz5Zj1;fzMOQ0)CF3(=RAV53(6 zPv$k<%mj5gkb|+$yw5tH`!_Q>UJX7MYWf5y`)O1^6&M(xxg+-lcO5{U+66XEcHncU z=7rBjJ+Mps;G)V>`-zxKT{A5!BZI1?3<)TfPvm+-t?v4?>gx}2gaS?C+pu(v>NzuaE)cKB*C`0 z^2k-#kkjmCNXUx(NL)3~@Mcy-bkyrjVm66bWMB5}(GHR=jx)bZz>T0M)F}7cx+hIL za^xjnL3o1AY#j{q#ac{zY$b6xp2v24~Q#>g!wYHI6 z6#BjCK8W|yzwfvF-SwHv;gSrgn)ITtcF^WHFGXwES|TPTAki4`ujh-*ls$+a}s^Y@P6%zj4sD^21Z zHKIs`YuQeN``au1L&0@1MpBPQLcZIYZa?}6Oc@x=t!;zoOZ@+kDTN~CJIIgdEw4^^ zTxfIQsPY-D7-Y0f>$DUBd&MQ)pU$T}iu00wfU9m>vs%GixNj9ZX_i#3X!%#XgiCs- zQ`Z3-X(&v=d?Q9o$rY3iFJ#G(e_-U!uR^k@r8uv3Djb!RXt={KsYT&y5a{`1 z?Gu`-Hh&yo1*7{H$)<`VK^Au-m8*f?Ub3&kL{vJRNsKC_=ue+kPoAuuvg#Da3%`=; z|1P(GswAC@{{{+vSz%(FNhOweBgg2bL#Z(I-1RJeB$04Sgk7ijo59)mF}yTytY%ty zbz;Qwt^ermGO)u?g#pY&Ms%Dz$I(0tc8UCINj4034-a6!(FgRIq!L9M$(;7ky@=O4 zUH?JGlwRV!v>B(dyv#$!)vH{j#t=)152Vm6x{=WyAFop-+GSczPb~@kNcfsL!c66U znHCU9U%ohH6Ct!#v$Gg2Y$~?Bwtg5ZokXl#9XlLT(;(Y%OqVS}X_c>1Vz?DD1bMo-UGl zH+|mSy#4omhmafrzHAy@5yIu1mMn3E<`E>maF2lahod@S-OJ2T2LU!${q)aEBd>&e@D+L$AN zl8$|qdTSSpDQCNegxjeS6x(I9%Y)SW{M8f_aIhso7gr*2Nw#N0I#(PJctIwN&1%rx;VL7;d z-T&c+rlo>F=c(f3B`E2rB-C10c6+{wkKL3ZcW_vPgg1kHi`>S-w&1{N(dCl??8Siar=r~D`I z%bI5P{FTVyv6@yE$1l1FMGNdJ%1v|rBs;}+t?CeV&?#0|{W_7-@on_c^%5+4`f`HD z+t{i6+q)UZ>HRi>Z5j|WTln6rQJ1h%`Pq+YS9I!X$eX zRd}*|q5hFf5SE8>z82r$4Wc2Rc?u2;KqDptm6wZy$=A6MbKVZiavA;oa9u&gG?he^ zTKgH^Oi_=hq$)=A*^8YqK35nZ<0ciL4B^?P-abA)dc^`(l8CSV;C*1&78}_txGG3z=ShCCe4m4pz=v-p^FbeB;xj8<;@N=%HZ%Ph~>8#&m@g#0LG_5+7IW-v z*sckHoABCS zjzNl4z2?imrTa@ip;+DCysq?$GgK6`RBia1+PKXt8FslcxG}IbLjh6ocvhq$3L9N9E)%8f)MhfQ=6=@8)bb9FRtk`7Uy4g2V-?oj6J2 z#vOilA2Qqt&sZmjdQEUDhCpDPohuRbKN?N_f6$2D$b zz`+ToYGUgJ58+J4h1|naishT7=jERyBQ2S^SO4bUe7O>3ieuk&S-GSz^0x@Jy~E6t zGhf2a@TAdbdr4g-?9b(XuhoLwvOg5(iC>6K*xGcSIbhu&0%`et9mVuK$o{^st>}F6 zok_gHzL1Wb)!coB7;;*xkWh1^A7Cp1|5lzRa&FIuzquFy7+^AAf6t?Bj&nd-T&d7I zeW9_Mspc1WvHtCaJ(VG*!Yk4|%klHn);p2v3j?I3qOPUQs;qM~W8*@z4nZ`{9~$k? zb&7&l)Eo!n33oiya}%EH9x!^G2Ho?U7lQkQKBrgvVF}J574-fe^I2B@TGZN#W7$>s zODEnEadGZNyL!9QnSQ5dG5G>v(=7>`_1MjKshQkrIOOojgT}M^IJ@dH*X}i74AY<9 z{5=yo(RgZU?k+PtYQgzcQ3JC&{s^YJo(#_0i{+uGLfH|y#?URP$ zjmJ|RkDACxg&cz9SUpexdh`(8JnpDWgSimYquDBP!@m5;iI@*rd#ATXsZdH9L7^7Sr~4zO?GnxH?e znyagM4GTSeJL7=QOn3Y8@f4u^sw_XAdQcbQZ+b9s|H@`G4tuM|n~1YaKJZd103Kt; zadXt1BJ#U$g(v|r1WmpOqJNb2oSTp~?k-4TXG8f^&xc6aK@dWuH;y*PqWS z#hr0BhJ8bg?SqSN*9mt_@DY?6)Rx971wo_B`<3l~@rZ4rzL8?^`{j(q@Gh}(lzYn@ zd(mW0iU;#63i#}3!7jzHiA9u`Gmc;Ul;}WH_YPUNzV!CSG0)Cwd7R)h8Np$Sk6Prv zFlJg>x6*swj9%l}Sy$#@0Ml_%jHuM`sCdO+KoH{VyF#ie8TRB|=PLW80=(&`l$7K# zNKjQT?qR^*jKPS^z`&q$ke+x-=x>N=?z^o&1uR;)Qw6$$x|Q|SZ>C~N0o(`QTk+V! z?HG|ZmCkda>d8Y#8&-4?bD+=Vj1<}bOEw5g#CV1MF+;-uzynMDCPTL8UWF2L*fX8n z4j2i=Hon!i;g2L_P>hW}>5;a21x4FpCmmg#*k379%V3J#*#MfgIiw<#CskEdQ+edR zPq+XjQ;C5v=Z5nRrj{MB2zkJG4k%8nI&5qxuqAGs|b z0+YKj5XzmFvf~vn?$x^I0UeoSxE40q>nBkp*Q4p# zT}qGqUkUO1J%31YuG%^vLK34udnXDW=3IfO5)swcoh|DpT<)bjk4<+7+Vm zv511gG8aZBED+x(FZK38TECj(XzrN}kQ0~9jwH7#bCC>as0MRWPprK^#k?3vd*Rd= zEoE?1itPGW`Qn;J63QPJCHGkBe03?=&t)=F7%EefY1cgnV~sbvdc~A)6Pcm$n?Zgw`JZX!k>9Ii*HNqX0;AY1+c&C>qwd0=}qZg>aE}y>9(53>2BN* z9K3UvpBQctd>Um~vpl&G5X%*>1v@o}^$fk}EE{(r*G~>TD13`GKGvT&{EEKpMj=&n z>RaEqCE+`{Hc5x2@qXNrUX$Mj_1N?o_}}}m1wds@j;=JRYBXpmF;3IFt9?f=WOY7C zUJAE?@@!NA0lqQI=t=Q{_O-AB&~^4L(?I8Q(WljX zYzKy>qKV=uv3g7Cn+$Zuokj&lU+9pguwYxf<=h))X66m}&O(b2k+RLg%AC|%V_u(j zN?nmgc2s`z)!}lm*KXhS*IZ5gGgK}CVrVfyWNec=a;#7VsjakD?H3+QrM}VzrQSK) zirQJ+e5+9y&t6w}=qDhTE}s5FQ?1Ix4CdE;!d!94s&hvf6>>3v>qTA`FwqGnmHKm^ zxrnZ^ysMnf^70@V7#kmwa8+B%+lFmLUmX>`A@bQ0gK&T7C5|FI!r}U=j$I*s7D(DI zty>%-opDJZp7b5132U_e^)r+nKOFG7XLg{6(r|I@MW**0(>Yb8Z`G~wN)~#RD?_h-A~sne}=1_X0B72V0n60-zFR^PEQ+K8RR)mMIx4q&k; z{C$Y4Olu-xO8=3BDgC)WphB*1>2Ai->ue9rE^VVihbSJ?x1^C1Z z;lCsKnZ_`0YYMFk$wxIT*64Aca1X?+mpt4!eP-W;9N!5D()4(J?SB zEyU^{+P!*&BQd1-k;7w2%2QbK-QeR{h*hlZVTJ^xy(XM4=QBIBX91LHf~K@(tY`9@ zYd9wCsLG8e^cmu8m(FP1=kAs#vsX%PE#T;|-N#7pq0CF@Gd+FCRN6qfi1U0Q1rZp3 z6(bn+KA$JK;^$NPVIz}a^7;K9DiV$W2twx zW=qIh*E%N@3AG4@h$IrhjAw1=&BNvqYYKV3((3usv;b#(pUbE*0GNnJ0;Viblx*=p zU_n?n>Fhz1>v|_3>2;5$0_v(2phBfA45bUO8*1R(eKrku$CiP9M#)7?h7XPwCm7lH zy~1ny-d`eDcB?co7B+~jq?u<`GD_OJIloe7|pGc6C z6am)j8JTM$yp#MNOM-7%jwfwGGmC;c{3%|huXXOC)CfVtMQ15aphIj#(#)E3rJ}lG zSBIcTrW6G*&%B*sS@Lv#8pb6N{K}L8baqCf&P^Ge*0*}^XXo|0U#E%WJdz^g7v(n4=Z@CnJO2gtMTcuj!~?`90-A;tW*4-Qg3~% zg*amZ+U%AWOGN3%qSOoB*9nDl*z!Ug_?q9XGCx)TTZJ4_xDjeH6ReTe-bP_%el}=d z8&ym7cgkl8x8GC-LRmRK7l_-DAL#ju_!2h+*(E<@7I4q}1l?6GJhAIo?b#ET1~*~MWU5$M1rpGr$Q3raRrME3_Vf_6;Mj=kDV z3Bln;SQXlsxftV>H_AnsSZ-v0Tc#Ae-Wh&_3I**pVH-l8A`_-89pz5>^r$~@&%?t< zp1xRp2{m7ipcKLMTlh*lH~_9xi|ZF$UFM@tscdNfZm}wv8g*lM^^OMesx`11Dup(w zs^DDZu1PtN^V8(XJ)a>?Tx`hpqy$eKv56Lity%x+_TII+fLhn1i#;AP6JvG8SAfl;Mr4`ttVnYYDlLoy5aP~QZU>^Iq7lFRtMr5m zcJBJtR|kKp9z^^x73eN1&4<8yO=lQsUp)KfKi%+jsQGnkMcz|ey4g*OO@BA~{7|B6 zu@@D+ZEx=Q1}BXdyu6H)D%zZ@cPEo7&?kfO@A!>G>-@Bx6%(IHB zOeE^xaN^HD+{$r$6wQ5I!?iV8hC6>!G}bI!CtW2;G@747@wc2jvWZN}t}*pgZrccl zj)c1B0~@pimyZ6)OLTPUjjy@)@UooB(f1XKexX166{dO3W2Ah;RP8VlOG4LEVpv60 zjBr;CnGD*cUhmX8^eF0O`O!tEEg8;jM2uY9q%`WtIKTY$l&mWM))_y!fgw=G3bdD{ z4k1Am%IPUFFZYtA{y}<6hi%@txr)+v+LeEmc!=e;v%q0UCt*R)7lz>Y$@wY+!9T zR<5&+TCr*@Ix+YI%SmuOV|_K3mg)8tyc^N5!c4abOlB8*AmH~bPn2UmCq&Jw^2ec- zD@o`3Fak58h<_$s#m4}y?@jC%fX7^Ur!l9z+u$%K*k1>cNQ{BqBpsKqct&-0*U{vM zp}%lVSIZPeVWf{Y#}h@(-e{)?2AxwlBv78&DND3nW)I%mkSLaeH;23UtqnQmy&sWP zWc9^*Khb1P`ShGwgJJD&6=@3hf%NcgRm-(sXThT|NOSCm&YPp}U5Y?S`D5|&tj>)z zpjuh6O>*;s?rYUEe2Yc_3^m2=U7qOsM>Q8OI(*s8!sP+oDp-c7ZBj|Q&xX$3gtMw!4=k} ziqqby`e9ww=q-51Evx31)7AZ)#dafRKc9IeH7+Z=%r<1VjJ!!Gs(8I!vl;lRJg*;1nb3e}%FUE2NVj-)arP+HyRHk3S6 z2abE+61mQn|Cm#%OB{7lnTk!MR)TRw-*mQJeHty^_u6ZJ(D(9izBj0fXbwfZQM2#@ zx<%Y#G`&eMmbP8X>=&AoL|(C|gg2)kHhI+GK6zjkwha1kd~C|KXY}FG|Cv>$0+lv@ zNnFb`OVZwWxysUxhG!}Qai%vv#@D0rJm~st6(CCH6418BAJe$!kpp?6}HJ*9(_1Tq{?jJ>so+c z?gzvfbjpn=4lG>mN8TX<1hF00U{Pv%QF#7RFNOHYd&p9HuWI7Iv#PvXroXL_uyTDC zph_?EJNnDgl}+eJY=!rrGte8FBuy;S>g{MbQ%~sS)2|6FsJ6FmYlf~WjjAJfzLMTi z3&Rm?6t#Zlz*9#LIEk+kQ1r&*x#SBx1f}rccmoE{@fQ!)2OLXz@4AHVTXI@tzDf7! z5*t<_s#;o}5VatPTUtnPTY5UXNn40y2n%jvkq>_sq*pXYo~%Cj-HHD>cWf}M+P~Y! zZmi?VNxki(Rg~Ql6Zbt1@r}F-*^W?*u!`>uU9z&BOCY->F7*12(&pc5G4JJJ)=S?B zOvz-$-&5rA{Zd-dl(ze5CN8b1;ju2B;y0h)7&aKkbS|Z_({vIb$uf}JE8Y8QqpM>I zUT)LmN0J%4#u-E(6TgWou0<4dAG}+!+F&9isSBEhCiIsTCgnrqr7#vDM-GrQ>olm% z)x2hD_9e)Dn8%mlS5$eAu5PpYoVpBl6{_31WLV+mK1Gl=nrsc?Hj2v~!%a9dqmvoQ_GB6~l%RUjrj z7|zSK;)1gAEp1bnY)1W=>NC9~Dh~?sZl|t_4pO`2)?U#@7DY;blpMwfnZt|8LH-d% zI7g>+c?dRW13}3b`Bqc-5DnNwzMG($E3GXyQAXSxYsULvja2KY6d$acZXjRD;%(kx z0{Yl?!W(MT&IFKck&2`7)+ISH`_{aShIQ+cM%52vYjLqY2?&Zfzo|TnDR!7iD74u z-d)LIcLX~juKh>hR2bp8ukPy4_9h$MwVT`-shCshXB+#0$$CUe+LO!Uq10H$e`EV! z1QYIRT~D>CUK+%sTBqQTq!heq0%=m@UIuM*SF5o6Ml;obNFTnnswc<|9;~U^Bs+fk z3sf;|-`9g2iL&ZF$P9HPCm2iSj883Lf#9-Y6%=GJYhVZSq}u07zJAla9Y~P&Tx7;O z{xqxNwV{Yu73qt0Xk)`Whys^H9C6>hUX>q~yYJ_`G;O|TSq%hEr=NIS*JU!r>PamP zrfIjjd7s(NfZG(}zUOJ1S9&}T-e*4lUgODFaCy+TlBVJ;N}EePuzE7}O^^q!cxF1l`0_>Q|NRjxalI(>f@LM;|OGkYe2} zQHJY#eN0Si^<_f)31kar2wQ;m!Rk3}z#`;0lqthjhx)h&V_~>S3CSvaZ-OcqgA%JG z5ms<~tk^!W4&7b8pcRShzb15vjqqM;>8?R~(V!rj=mG))lL=8?VJJ<$6;}DyZ&)Q( z+x{4KZ>Y6EpjR((LXSJ>l$iRGG;k{9sRiilPfw?pY4XgUq69p%M>vC68rHpj57F*l zMTV27ItZgXFlu~(aj;n(@5^^*ZE_z8b7amwbdMnuh;h~*0>fM$>rt(v%#MFxA*{L` zGbP=^&NignY@alfjRv`Ny%o-yeYZiJ?7yfE?im900y=^ezPRN*zAPH? zQgE!bB3#S`RF74;M-z$7^y_HmbOqC>vU?mnf5ylGMw;Apnv5D#^J^iC>M-ssX};SO zD~_b^<0Y|;$xN!uTOjue-@wU|jGq+d(U`vhQ@88ZJ zZa7X|-iF{439hD@_^M7f(wxlA24oXQ+L$ZVX4GOA4gKv4tIOKb*Ge!ve<1jCb8)Z^ zwU3bgNSt6K(xLn%1!o|9b$R(a{sR|0f}GXq`YRzSqEY>0)a!zDecR^@;CfgqB0>nH zgrnB#9ziPyX4)jCZ;!u~?M`5=fE%0AH^&d!L0|8_qRX}G{_#aMjd#ym4rpf~>F$c~ z@J;+qP|+4H~C0CPrgBX`IHk zZQC{)zSG}%&ROqYGiy!OB)RwA*S`4q&XQNL`z^mc<7|pa^l4EE>E01}&nZ)_+-Q@9 zDh87z+A4+op|ZKZYv~?l&-9Q$z4`0CMB^+-w&V#03wya(iO2iKFSX1vVvA{J7J*D1 z5!NV!dC2shv@UG9uH1cc}l?UhtN^$ ziDv6cr-#4!@1MSD0{Ha;7B@J0EKa%Y&JDiYUV|TNx^v7IXPOUbzf0W0jjg%$eDy}w z)0)j3YeA2#+gmeQzbMmkH)gOYQHM_4o|&yW?vy7$o4OR60!up}quxBvi55>G%_}H^ zWcOst*6)gGx@p9&f5l#>vEXak*F4JI?sh@ZIFIb!cxO4pNPrUQazRcNU=|v$0tc4n zfAt)9(o)lhAwUat2XEdBk=%KFz8^pfcj>yP3e1b1 zKZ?Rs3%`J)!e~5+_ox$n&5(fE5JU{8_-%+PT4RY{u|Xs>e9j$z=H)}E9Fg$@d{6<+ z7~4J;IEDu>9qi}_^EG7me_=hm-S9B!H)A9z_-lQ~q_~dX@`XxGfO8TCB2`8E z=`sSE&dUiGkgLY|7za>YnV_kt)?E9_n)-Ti&pi zD(e~&KyVu=D)v0|Ap)`nPY{=UaYO9|Q+bjPrY=d7%L5q1#PMn?<3vizVTl?d0=h{( zW3hV}w@4%?!lE$O-5wzeW# zhx;WUYJ_X$Z-{)0DKh&L#RQIN4gUw=kU+B6WA5uQ;W~I9?7Q&=A&e9GJJFSFU^i5) zDX)+ipFPsJaDc!i)3P_%>bUQWK0oxKd|RK*nCXK1y=?I`m<9PKo|6 zAx6U67o6@kAS`Mwdj*6@7Voj z${Q0PWn!wbAd3;#G+wBJC2e32VwD|2kZe--ms2&xWpeiyNNN8$ZV%kQASM*vzZk0) z5yK#j4plQ^QaFY^edxDnB8_W@}sZ13?6> zaQRbb=eIC_B$Xe}eBH6(Jss-a#*Q^M+a6aZ-2Ar%r~XuS*_ozemx}autiTu-rhLMu zH_AxkOQ)k7gDyCkpDU5BSm4AKge~mjqj?kEAElPTO{Xt@1qh_S6CQHegeu^da9ww}=n-lg- zS)I_W0lPmKv!CNI%lwx|_5kXJ%Jhor+2TSZ^NUgb3!_U)$AUhf*D7Ag>W zHPgrCbC2NA6&uj%icxYDDw5D1#F$Lp^ac|<-b{DpefKPH~a zPoUz4V=I?_a9RL1lJUyUyRZR=ULKmWZm#VMy62D@%?K^TWs zoT`jvKQ<-bxmW=G84|5mZuC`UtI)x1r0|+nLKEt~j}Hd>x~7Q=;}Iv_FC`scgAu9P9~G}y ze5RV~EWM65M_>yc+2h(rfk02_Z;v))+Hxh?UGwc+7Cp2m4{3g3K6j8#Hd?l&o9y4 zT*GPha=+E!67F#&tBjH#-T?E0;(?UYA@x!?z{s@y9-a~s$g6j7t4;)P#UZF&ngiky zd^HynFu3S8ScAQr=lZgRrX#!O`<`U`tQ&X2U12I*GU7;Ds;3$-Hfvy9gRv>MJRyOxHQ}KMYQmw==K9B1! zpca{!TieD{OzFf58L3q48x@j~x;xa}G*O7`dyI8o%-!Fj$)bz=jKn7My91Fc zaU$v>4h*hmi|RTH^i3#Jfgo+Dy1u2<$#f9wrznPFG__2DF3$vT@Hx2k4HFPBxOeMH zcKvLJt!0mAYsGZr+bFfwUHsF1ncs}Cv6wn zDRn>Pr!&PpVF2tP9nYcYcRn&NSA1sYdluIZQ+t++)vdw1rUH zpq>)5XQ{*r*uCo*dZww7n24!Lt(%3ntlIFu>WNNe%}2BO5Eq%b+VwlgQAkiK@#24n ztFxFefsAVRhe7VY3sci09Z9_dyG`y}SV+Qq<(Aqk0Bv6<)3OjXfq(o9gLaPqBmy4C z$}(~$rnt)eVmPVU>-!`1GA%JQJ&+g0K@yH`DF{*1w|~_M6rODNFk=VB(vpZ|@6*zm zVTmdeH#QrN{nm%)o8ndCqkej{E`~#X;OCoEi0uAcxLele}&es7&{u(-x zU$dgBpl9w^G*7w^vWXu~kZwXJ*boZ3(g zlP@7hfjazBfds|>HeZN{wCE3 zv2K!Ld*^qYAPnG#rU@V*vF+&SIlzFM%z%U4UCuz{RuYtriosBnI(f>GF1EL}4j+uh zt0OiwFfafm(E%B@K%Zm<%|m|9xP*H~!%j2)+(w6u(JIp45y`rWrT0StWYrYA*f&%p z+2WoPpfR^^rnJ;=_`lB-WbP~z-aviX{!=V4AJ$8e|%+ndfvrxz_VI zolvAt8i4=&ZFCtPHXG90VJ`>*y_60DoseOht*61A6*iIi{Y#*ui}-T9J`9eI_DN)o znnx1U_E?_lGhelO_5p2vY3OH`mRgw_HHY!<(821K+<2w>x8DAK2wIH_BmMxDI|g&i znBm7bh)uHk2p9&?NS=_E7Pg3ph~wkAb|ah)h!wBY>EoG1c0{qM&|h8*8ZxMOngdlE zm;1NKuu#K^DYAKgQOJvwYg8&#Xw}jMMpRWX&@RS-bcq0WprWT=%c!=)H`qEfG_;o5 zb9rw!0+3@@UI4ZZ}%^b4NF!jGpGD6y4s|=hHp0Tt( zmiMUCi7trkRQ;`m89n5S49Grn?<5iP@rrB`Rvwmqx7J?AY6lI7yH=qc;!G*ZW()c7d5LTK7a8EACuk^2@HNNyn^GP8=>|nsR}rH!2~jBVw#w1 zG?J0M2*x4&h$Q|!rPPmb8WlWCp||Hg-5?^PZL#7u|9Y%=EAeV)?c@jkkwn62yh~u* z|1HbVQM1Rh3&xOdvSO{16v`3Z-}~3&RKw0>z0b_xYcenXe-}dp9MCV)wnYg<*I%$~7mP$LaETu) z01jn(o4vGtWf@C$j2V;N8t}wC{;qfnwHE3oR`OfYg^Z0{-yjN!p2u|LL5^5!*HUY# ziJoWbR(NrV3AOzJH*C>>^B)?jzMR4kZfmMVPTE>s#F8S&v(J{dDffB)3s`%$(LuLpp~l_M7%tq850zk z1(bJogr}(-aV}QH+TDQ_m(m{tGR?2zGx2n`HVE(p3-e~VMR(MnoFDU`e@0Q+WVA_= z1T`TeF%g2*g*B1B;Mu`Kq3hNE2%tQTf&$Q%*3qGo_(L52m!KRhxgn}-N| z3@7mOSBeJ#G4iU-CvVuUACeAF&Ml{~?9r!f(H$(w`ptP@=O)YjRt>mmx%GDc@L=jo zBexqt77#TcjHzsjZoL72d|gs9m+X@ov*|Kip9kE@JOaK~{xYlA3e@Hekeuq5>2_ITU58ZI#$$8i{gdi2UW zO2Zb0!rb@#l~G!)WwMPID&boTQl18F@G+evloM8=5Wp|4cT zUeFwf3}Neon`DOBh(~#^M(ci9NKxg%nHHJyw+~`Re>x$6v=oHVF&NXCYz>RVce%N!%w_C znpQ5|<7j!^ph{HhL=FdwHJZJHTB%yTfBOEDk_PeLO+XX=R`Ye7$=Oc>&CwY%HrFU@ z7YlgyVQnTa**~xOd~wzF(rdF-$>gFvh~Hu3))maMRvYc%BX=KDJ8c>H6f-4CY8@O; z=Ah}J5c+EdWNX)-Xj!~4s5aaTm1Fi z{~+oQDRztG+@;o7)0(BHJ32cDK4tT-H>Jte-eVb(f@(5?NCfQcsn)hdX-Icg`SdBm zIa5>9*_&)3aIs>pUx|{&F=9;s(j(8KKZd)8%7=&Ro60w2PJ2Xjzpz&3u+pF_gZzVo zr5XRd>6>G-zp%QL6QPRBO_uuBQACGrUcHJ+(7%q5#iWag7-2pAU%ygQsdPZW9kvo> z0~KQ(Y8i#*&XrMV8Y{l4vcrlH7TH=}P#6R|6UxsrUU$49IM2zH>YNUtO5NV^V{g=9 zCF{K?z~~_X&uo?aH7N1Bd>I9lLxHxJU#mkymV&W;JW)h-er3G&93~%T1)am*fHkgIT zEdEpHN7dFBa1Oz#q$j1(Ir-z6Tmi4j+o|0hGFhOIkq}dV=ztc0EC_>D$a$JFcvMb_ z?~ZM-f_)bS8#{2h{)K?g>KNzGZ{5W7T8;!!@SBrznAJg<;yMd|e9Y{3!<#7qZU*f? z5caF}!nHCnsRut%)NwgO(-Tg0L=N+|@(lPje*9ibKPuW1FscM7B zd<+_@2d`P?=`fe4%hZ;uFJj|FynqX-$)c?Z&|o`=832yGh8g@C(C26}NhyC(VIZ=f zuixtBDz|wFS>krVY_{!RAIFvs@+kAn28)0UDy|tRfOzYr$Z;_W^14le;M-!{8rI=9 zxcGm5K;AxjfU$h@M2>&Jds|-mNz-MaP#0PHkMr;oTdaIh+U)1X1Id~+WUQx zm*nQ4Oin_i-1Jp8vj#_?nd8KsyDJ?K)v7nJ+5{*_c0Oq8e`^ygq*=O~uxEOi4z&@5 zgp8p)9CFktjPn?J1~Z!AsN`?%;NoFD#5@NjKWxW-+@Ed0 zF3j>p-cK8ExGs*#xA1o-3xve;H03iP10vD)v;lqJ4WQ@E8h@W;-bp2P_P$&yth+4= z)!`ha`6i3WLD}>RAi4wL-%*!>#J*#81Uy%#Ul4#ZwulAOQaq1okgv5TZ ztv1>Y!Etnf&)(n&_;6)J4%@#|EJo5&<~b#Yz?;5Y{-iop0$lL^FoLYx3dzYKf{5<^n$5=RI%_l)DFqT<+%Ov6K zI1*+6ygIH$m6J9hD}(>PMEBpY;~N9H945lcws*k&Fy0-7LeNv4RbjCH(CKe@$KwIZ z^^La*%^I=8Tv3W7p^=x{s7ButK{^Kk5kt?W8oq1V>+R&D^9{cZy%tNc?4hTk@oH~)8y$4Yf8?1vHMhQ_WRJn;x(&rxtra7#)>{Kny})25M)P2K zt7;Z2PYZ&3Vq5!B+gAj!`bK!#82SgQQON{11DMC3YXFfuV^Jl&5#mZgtr9Qhx5jF< zcoJ@v{_E=#eaKP54>C#1Z>`b%)g079ODMBEUFfCXw1xz4{LIk~ql2uWheNr=_6lhe2 z#Cu*k1GLa-AePK&oj}N>$KJRNytDS;(oQkEGxnbpqw}Rb4TlJb_Vft4?Fc$hP#_02 zxbBT2Zso~-wNX6Y`C!^TnHM6BX6hX{$nPaWra^UVtY;Yu3YTX0%qk&GHfnt-!F`3q@{d+ zV22S>c6ScpKI=7iOCaerJL0n9rUq0W1gDk6n3lQNBdpc`nm}uwE8vH_4k?}fj=|cA zR=0);hBr~9i-Oc}mhTWfOQrFL=XB9LG@P5ed$I~FGj&JDyfSZ~BSXQn6R%tmqPzZs z$zw8gT*E4xtqQs<)k>(qh#w)VJL(sQ$=+*j>q+co?J3p57@EO2&*h`Dq$(@B6*tnL zhK0{k3^minEYiVPviQoMq-pmjTB&|crUEx^4(NQAawTs*A92CU0&DaNe~D9V)#BV%|xe0yrjVX|a5SUVlUkv)>eqp?U-fmxDlKD-hafyS~+Ngf=SV4 zhWeoM^9SWG`B+AhK zcO(cJnaO6JitTifz;&(RNXG>lR+dx*2QUtWOgP?fh%BaeHs8W(=JQoArOK=i-46o%0RgJ_)m zhTA6jBs9jb=SUa#mXEE-86&cWdBNfNmQfH`^b4*xBI7H-9%RS-!&wYwS6Qi zTTP@S**7kpqa~ER%|+&lUlLw;JZ{k0V}-!x762)WJV=yE~S`>Hl}l$r>aUR+S}Le(zP zxvWujizo%ZiHvD+ayG8w!w$S)0F@Ta-^4gV??FT*J=;we5Lu+r+2E7g#n{NEx8aZw z=an${m6UOm0C$HuR{n$lJF=^^?;NAk690; z!;F2B7hW=Oxv9VYf>3ITpxsNkE?5f&-WD^MO*sR@-vxD6CCTzmgF_3(o<~E(e&KL!-tUK2UBs>tP4)_<8WE zi7sV9Jg(T~x}N{XRu_E^P_@wpr_0 zY+~@&#IM-;m9a~x*PWo$3}wJa90W6!c*PAeJM{iji0w1M7hOC2{DZrgOd1#)X->fR zr{P&lD%CY{OeHQ6rNoDDaTg^;ELcj0<}*r5EI3R1^3u^}4GB^*;ua zV)SENZ~Ha~L?ma%`bxNpKrq?xhD#tBYd*G6tyLRnJEAMq(lM68kT-|jl=C<&9jltr zSVOImiIICtni=C!ET75+dtkFL>H4f^0WA3?p;_;`Et}Mfi zlwzc6FEy;En=6FV#2B1Ck+n#=@?SdX_6X%^5IPdtP00PpW!mN-oWK@3H#iwZ>*Xyrf8*Dq36=D2YjV$Ls1dVY_8f5H!qm1aI>hWglsBS&GLJp4_Q zjB{n121uan1TUP#Ge27BR7RUJXTmtEVwstYx^S_ulecru8G4WURg?x7(RuR zK_K4KaXPSZu?cc4?IKYs^*y(0P(XGVly)A?vPyI;10O@+$>c!@bT80A; zZx5{An%Ul;7OP}#b9}zuL2w(kC8L3bV6tT*d}Wd6^#KW|xG^LpZ}Xgt`miZ<$&Xe@ z_?^Gfg+6Z}-4C}TvuK&KWu5lXr-?lGSie0nxnJrn1{G(WQB@r*SN^&Yeb>X~hT7-*u~G#pml4Ij8Zha|#$ ziNF#NDpaz0vE3A;UitjI!_O0qGt}_L$8wmT77_59`kx48G$?i)<&>g+OJ6KBK|moz z-Lr6n;g!q2(OEm$FXj63Kc+$ybu(uzQuO=yjLV3D$AnEd7$5BX1A?(#1vvd#XlkoH zfdmItSQ!C-jqNYm^rYG#x@TBIKV=U)GC|LXCKmSP6xwJyZDH*miBSJLc;*^;(R+ht zBvwhfUUp^AsO1_w+18|y^lv&=(Sxd-#$#lbR-N~Qy{??feXfXZ=W4;DMJtPGjGzaWS=~2Gl()_42$rxaPwNN%KxfwUddkDf)6bEwX@sJfLs#|2?+rP< zJu6pSC`0*l!s;}8v+fl}*O6v0=kj?<#F4PDV6t=)A|T8iXv#<{2Y)I1T|0$%!|CHh zItgbkdkw$`Aj8fPg@3-_1O(&(Q$~CCQu_T}Ust3e-pgNmvvm{c#*uX~?L!PxZ?_pH z;NELahMDnslNXShoBWv-zqG|fI#SKm2aUbvRkXlH-6wJZq|C=PZc>l%9o{Gbs~W)$qahJ6jwg#o(tFF%u6@y6wN#(C=(CrQr4rQA-nL$>~(N zuV;km|AmsJNc3vht(Zp&SN*{j19><{VcZkA#^hTVl5R0V*2LeKN`b`imIcvS9QCgC zM(RZ%5_$H{Y0*@iNHDbKC5eg=vZN%bxTM+}Y>GQwUm|hVw9mAKPn-GhyOxH(BYdnJ7klDemhzuY^o zjwSo{o~VArVcLYoxYjvg#|?a=qQd3-DT^+p*&?5u0EQY@RHa}tX9h$@6s_0?Z$!3% zO6gg*f zWPinUl*QUDnJ>N%EeOD!A1^D| z+CLdR+OgAnWm!+44mRZF@q++xzar6M~XKp<6-*0RkS;b{|MiobH(rjdh&i!s!dDvmaCRC&FnD+&hh>& zY#)qthgD+KLU%j45*(faGFvu@hrBE#9|znT+6{A4oT!Y%0Y>2bXW^zS+K3~3{^vf zg)LVqVQ9Qs?rA6`L^Vl5A&LbpJ?M-VX_PP8?OuWKI?nWEAs*xSyuMNIS?w@sj-{H& zt)SR2Ow&L(raG-5dbFRxx|NAd)YNp5Hv+~DbcXYzX!?IiN#0nk%qn2D+I;q3r2Sjq zPjqLouJ-Ud%c0sA#LBGy9P$=(mLy)TQ@DO$CJ(V^TK4C>KtMRcJTG(3=(3EMV&| zf$=rmV%M11x4I2#Ad0I$-2Vg+(>&@1*vlC@tsQb(Sr;Hpux!-&(6Ua}R}$$EaNq5y zhlxH>FMAAQaWagD)rI3i@K7)<$z!2HF@u^m!>Zpzs5B?Z!m`di^wzvli5{}1i}aOY zQvH}?8Mga3qnGcFbp!GfQ;@HzWMaU~X}d`_6i3leS|?<;UXXsQtxZ5vD8~Qs#&;k&drS| zQ$7ofJWKH>Hhc>ofN|<7ASFdpV$B;vScQ9c)1W_le6HOMRH~xFh`bH&`mnwSne67B zqcEMyHPF(PXpAj-++P!CgOnf|N-;^?Ry3W1#d`wvB5kPkJ9rS}7MJ=Nuy#|7=}`G#~!2DGYM z+pJ63$>Od?dGun@U?Aw_e@6UG8%Eh}80tCmp|=JOY8!{!c=e->%kQJT*M|e74Q*e# zwkVc``g;adB=KbaYZ80c_W2H;6jNCwgFTR`m$*j6+>#QVRfS>gcuifR0}65a+r(*p zeG8_`7F*!Yi~9>ir7nlR_4`Y39!ykfunV?}ViUMU#=oh9g{CkuY?=1tbQgLl?5c8z zNc~-tC`xq~QC9~2frh)iMNcSNnvxf&E3!(r%aUBK_n3jG2XPSieKD*(RIA0DxM^`1 zOF~5q7fQk?{(EabQF>@dP}`e5FuppQsyWf55NFCJ$MtKF%uUY^yYcQPhz0-j2v=m5 zBZ{yDCb;Tr#gM2qCyf2VCjD1;hbA*TjlV|&IW5>T@?)%_#E)WyDwV(yGPu58&{(<9w~!vL z5@#Yv3GYBnQf2hCQIjTYXDqY1ep%;uHMv@sZ?4uL;P>V#4p-{!rJreuf$TKc3?e1_ zL3<;Bl;s1fwJ5$vgR>-`oe4DKjy-`tWAfGr)GS#vx{M0PW^EUIQCN!=x{(BRW`ip* z@Yi0(!#JOyRh%COgW#V9ZcPgwH)z=+G{U%{O+@+KoPR>)Tk+)*1!KEDj`do7aQ>AD z^NI#QR&IAi#CPws!4`C}H4oC)=}oMjTN()W-jnq9-JCAaR*AD078Y(lR_aoz7EmlD zaH2D);(CRK+Bp!bi^h~&f3;2Wb}I_akB`YojzNk@X3TMG2L&(UI8a@w$MOGjkx7&c zYMrlMggb<6nLN~JI6AN*)RiEIot?hcobJbrOjOLevi!RC8M3Qr!y7;t1Dw{O*Twn; zYcbz6i9~!E@$}(<6Eaf9x$u4xA$`rczD(*I+LRY~rlAW-zM!#?KFlPP9>xkDo6mjs z<;T>6{i?_cc4UDlR_%~AU8T;-W<8I4_ueT&;T1~y>ae$vSILDzCwk(huS}oyMKX?( zOqlFQSXVM*Ws8L)_+mK3+XIavASN8Z7F=A)JC0V)VIOFVVSoMXX2ofWqfwz1P&Bui zkK!KZ!NcKLy7L7u>&+7?USgxKLoRj&FLN?}O zMd!dGC1F@FCx@MKm+y@mNQu*GM#QXi7d5|NUs(8H@fEI>q$0PYc~@7KB^V#45sj4<{J7$RM>16bm!}c5MyyjPw;>k!8HcDTCA1b=UU&(3hI7(Nri8kz zu4u-bb#Np7PG=86rQIEAvuvnQPF-^fnb=7>yHTnwy9iQvFmIQ08S4mId?KBBEc7$? zJ8HRuO$@bU^PAO>+P>g`M3;*9iwo{p3#k+gmw{FmIKu4yGKVu zp)D8;s>%NrOVin`BUQ?rSsg$`6k8HYn5RXEA2i=C{@X;^DJ%jQ1vI*|`&an;&s_LX zO2f3e2q3@L|6nzIlMOv>vUn+s+DSP4i9RZ8jKyW{{dEym@Xs-1Sa zeb2YLh&J0a`hKp-1OHMmajN?8GQY8kkeC3_ZV3aJRjyM%kkd%K+K4X47`{nV(?B8E zf)&&g?e+DKV}p&z;_G(cc&8p*uAGOb8ORyV*5l*jAq)h(J~^J9LS}NieQ9g@-k)v` zzr$>bx-Oy6olfAQ^f+PuEP0Z*2~>9E!1%T^snG0Q)g5ZzR^vS;BDeYj9QDD3>Tedl z^a^nbFs&1gF>;f{^o6Dt#Xm<{@YhgwK)I+iNO_bbNFBQ!&zObUz~MB)IjuN}N*O5# z8OlsfBI_B&pv{3;x(P0wz4kr1jYik(`DGd!Iu%4v*4+Ue8)Qb@@`b;=@MfyB&f%_{~HADD%Cr9wHJD)K8TwN|!Q-Lm`f z@W{x%y)b0u_I%Q%1W+q!@|myida6yc&jW7V_fbc3ygN*i+A zV%TW>j<8^z`j&3^gvTfePNL24FpLJ+H>DIHg!zN&HU$MWa3 zz~T=!%b_k}p})p>j?dG+=PR9Xd%EbBcaS50iJY82=s;rE1gwV@fm5$$T8o!490$W* zvH%}P7Tkr3z_RF)J)8C~1EVQ{q5JfW3I)upIg- znOqaExX~*sT^KOK4KG7=KRb{4y`T-swR=ZZ5ED3cryGjVaM|r62KhDiAh(dnBV7np z`1BK9PJ%%YnfKrae;3&IAAyMo#wG5%*OJWVI#1$vV1mgJFuRgr%l~%8xaXM@wGvpl zm;JQTiQ8Qs<4drr0bZ@m4PW!rkD=X-CO{8okSd;~mP##^=JjTmAEO^AaO0gVRVKBv z`S8L>o}at{$uoi4pn8ywAVKua?unsTX4BYHG(}odx2DO`fgjKAWu*{2fVRxc%{9$RUHu)OtrTXhAx` zj-b!AKuw<%@@avHO;-Vmn4m`>BFq1qGzEY@F7!5CQ@MdbS$F?=^PXAtn2l8*MX{x}0v$-p?;DB+SeJkk--p z6;o<~GCa7>WWqm8p zN1vBeiT_QWRl;zjD@_0>bjm6prBqb@O`cJTg7_mV#^xXm7Tlp&BFQus^XvN))hhPy zp6@U0ocGn=M2|nCgE|2W+W@r%xONpWd^5s9~vF3l{w-q;fky7Wz7;t{tQ>` zWOlSU0LG!&I-Of1^iK!6CM6%wHbbb*sY-|4v-)UMhUK`SMG>jtz392%{_$ux%|-*; zD~cQQRN|xybjE-a&OyTxn~F7ksHK!jyf#+qhrQrk2yJLZ0B|lE_blvU3+6F%Q5HrG zqXJ%-VU$g9x|jUJ<%v_RKecL6It1ZjqvMZR3K&NVF9nrhve*5BV3gs=ScnnelZF58 z7R$Jh6}7Ji3Is9aKCTjA>u3<?}U!)KgUPIng5&ksy(cQ_Ie zvOZy@2_cK0xk^k)h}TuWC7Y(c-<@(0hJj){8FTbS<3`ap!dE`zKD%dZwsD4;mVJ1_ zza2t>;@WDA)==8Rq|X8{Q1kQOP+;=R^&rg0{CGLErCl)VkMBygK#>tR4A9UHni}29Qa`AMe%tm;0Aga(RqcSds!Wab`-8WHrtq>pa0vT@ zz6858(xV@?CnumY773{P2+qy@_IRfm9u*a=MJA)`;A_;6|NYOF+Q5+F*KwzbjQ!o; z##Rsll(`_p2#u2XYtBE{_5a=PiS66f7?D!hY1>md#&*^N4g35On3ymYJTGNs7UE*a zXtd>QiWTh?>gdWzT8XXrzkJZo{RAi6PAoLD@k~oyve1dX5 z|KmA;UJ|h08cdX}tfDhK2_w%AH>B%w6Jq zqiT?}T!%@P$~?---KuZD0ax{m#Kcn4(#AvowV?jLzxwaov+Y1WNZ#l1m&fZqmc(I% z-OKN;_i_m4v5~0<$~b$ag_jZr$t|+a+wa)vy5Gda&lLIkJ41Z~0@6K+T`hy^%DeP8t#dEh6^Z&clSX2}r5vF~Ve_*o zx-c~qw6cSCy}fD?nS0qH6%K?OP2y~8vGI{H)422|Dt1tKuCH7zq(0>TzpFE1DCXgJ zh9nT)q8w9LdY51aiqoxQ8BMEA!#UD3COuf-WL=vBy(3hy?{X-j^B@+mH}E)PHD^uuCdQ3cW|of) z(cufzbvIPw{tC+{KOEit0A3g{yy~-QbLqOg|P)KU7@z`)KSl_TOVc+ylUboLl#ZIO?(eyGUInx1EIqN!3w4V3N# z&2+iD|8UrXlRs4p3KCUkvs=M=*P}FAd|O%G#q0Tbf|u`pR0E=P*mdA1@&<_E!dy(e z8n-{9*PHm&6#}-QgIuAFJ)Qy2TPrX#uXpE-4maaLy zvPXsD@humt)M#f7ZE6&~Z)nhRW}(vkUP45y=o<##*5P#DOThONFZddQnhl{+`A;h= z&!bDo8XGd5b`yBmRQBMyMe%wQ7b=m-MDuSgZwhpdqRKe5i?i<%1xnPZzFg5oXcQ)| zV-h7jIU|~$-)j18zde87(*vm}G_NSarc5C0f6ZN@k9ax63&Vrh)9$Hk9cR&q8P~a^I%ge0~PMMYOVT^h`6d)%lV$Xh@4$nD zlF0jZ5TTV^3j4cZMCov0F5K5ipu@=F8O>HOP+_~nM@q;Kopg`4h!;KAxH6Tgmj%m! zV9-~OM^fQkK-dQSv>Tby1$SDIa#DYbIIb7enLB@AtEfAIwaLws+oVhQbR&jdTcYsW z`^B<{fL^#{3T7EnPH?t5O?Pd5n)7#BZ88S09MPIwG9A|{DEq3vuTs~`wqPJZzZX*% z{qVJL(A`eU%*JLWh-Pxj_)AlyVz=r3;bCRUZ~(Xx`s}0UCn923F}j$84~~kIik{OA zx{6_$XQmfGKHf9ljl$gQqQ#T6L41pkZ+pZP@&8`32kyBV!;qpcYGF>TghhHj&~<0( zLLe|vQpTQRX4GVQr4;H5_K{*%|H!Yj^J6#s7Br;JG-|TgNsFv(ixS6o`9aH<{iZM` zw`o4T$Xc;AZ4jP<=p&QAfh(LkYZ^Tsr)H_qY5yN# zZxxhBxOQzOxVyZ;-Q5Z99^Bm_xH|-Qceh}{-Q5#3xVyVMOy~Re{$}Q2Y7RJ|ilVE! z`jK_7buGBW!Ik{CU0P0N((AB;spSR;iXF;c=OLCuefOJeq;81UVe`Ch3l7;eihk#b z?}xfxAK4K2Mwf&+{_8CAX)&3f>S5~*Tka5M`~5Drb+Mqg=4Ph29dMSL`R)IxOK_WI z(prgb|6NanQNudS-w6wEcQy9ROfa~2#}b|FtexY8z!Cf2SC&=~HWiCnu>V-t;(3Ns z^)|x5AZpzC;a{=<5Yz@qw6(SGS+`|iUeXCgePK-nY^<#pC-sXI@`c0T-uj~V_{b#W zO)M;yyIxe49z;y|ZAErd8ONTnfuk8B36~~j&BP3_r*CEmUSo+dzv#=zz>ABE>(Ks> zo&ubTt~rT>dZKr@oDbvrkaBxEC)0z#3=P?EnLndbep9jC`C6FuZJxp6_YQd4`94(h z7>5JB?Vu#_lQ%)+Fl~B{8G6mJHHvzpbV%B8JCj)+4}&ke;LMj3rtxu{+&RT{x)3n2zX3LMN!t(SnbKDHV$lP{n1+0FHrdjeL>yx9V1fZUguQ`X2eu=rKg+Z= z-+vS*N29!K@_Gkye^#Ujp`u0&4~LUMGf!jyHs1@e_DhyvO&t9YMtOZAme6m4iWldb zZ7dE{^=t0_m$|o~7xjPdRaaLR#DZ^2La>V+uPaee9Klfh7Sk zQ0d<(H>=c}7fKY5PDhg62(@=;oo14E#n zGi!V(AG_=q7-19rkik*i5@@a&+w=akY}FHSa8^u)xlqRAr~L1{IfJhZeC|J>kIvqu zqRLD@<*2F3@;M3YG|;&s^Mob^p-8!V&El`Uj0In)ld`4P5#(f_#vAD!`ky)R z^db@3TiX&g9P@s8$@RX`koL9zvLKRflj74mh*`dsN!;FyP+QGt{E@@Ac%yqWRJ0Ht z7qZcx4M@1<3h1gXtPhA}D5rR_U^&g{ZzleaSM?~)cWmn$%aJ`UfvDO`5Z4HNPX>qb ztI$I24v*e*q3L$e0inr8z!dxLtlaSOYbU2V+N@es;NFD1V7%62n-wEJ|IOeq`V5M$ z_@}M1Ii{8Sgv?Z-Yl7!0V?$4Vuxk=_)LK_ku=A(Q{@^>9oYN?i`qTR=gbt^inrnB6 z&u~PpJ7}Wn(%8DaoZ`{n$%#<-hC96?e6qY<_;FN!g+*Xus(DSF)6HFuba~X96U?t7 z0n=f{O(EIi=`Xqxw*5P4)&oUsWbO`JZ``l=nO%tQuxtot)xlwy@x)Junm>iB@xa>O z$5?J?FJgc36;@PME=`8L-qIPxaH1}$h#8t`qIytancmZp4EM~dvH#!qqfvo|>De8k ze~4XW0(!%=`)lQpu=+@5$NBH4tF?IOFRl^~=Sv!s(;6*v@R`^xhL0KAl1>Jwr6Mvi zxXa5MvX|!wF?=zB4wds3dK#Sl4Hdb-t@#P$PJn_&SzWhkG&8=I9u-wiv+;Si3%| z#(U4(r%^iH!;x7&SIDIN@GaJBNO%0t#ZA*dq8q71`k7ffwt9Ow5HED`bZW?8DlE*A2DYOTKx z)sNLKSQ0E(F%~BkwtKt)EH8VLO?-XC9c`%}s=D55?8wZBGD5#fz*}v;kUgY_T&@=_ z?wN_7ccZ3|G72J6gz48}pngmL)x$jeyUy7m?OM$JVVmBI0Va|w|Dt9K#9zUR7oB`i z8r|7UkeH$yQeiE_g*_KERlw24#u!nJjg_bWFv|5zk%>X=Eq8dFnRj#wf2mHxWmdOx zGPS-8YFqy9p~3SqxnGSH3Aa~o{te?syNL!(3r6&oh}*BAn9 zqnCS6KD_SO@Qq_L4UKta-@y9O>}~ZJoe!soBVBMh!3oJ6!*aX~?Mo_Vk~N2I;jh-0 z@I$>_4YT^qPc>^k-5g9#wt6rOmZ?)u(Jnzbb;xgx&BK=3R=^k}hU5(Q-sI-@R`BT8 z<3Pn$z=v-*%5FTOM1Wm-V;nW0VKeCV6{+xWV&63e*I1l~-&md<)aFH#qlt78Ne=}E zN7J+XCSkc`%<^}In~5!*hgG~AZ@IrP{1Jq7MHj=r97U=ZkcU%t~TF>5nU0u+LHfhCgDBLrUHQykw^(g|^| zF>DuyD%TeiaSP>7Dc)a50s@7+bAy793pc{zLEH_bcnEb7Vxx9QM%!t3?)hgyS2r(t zEg7G{T(sG8>+!WI*h1Eajnn=XG6IR8v0yR^VqPi~e}DhSf*BC}!$p5fhHNei2^gwC zUCTD8tsnQ~8B2uf#-}c7YO&&}>D~1Gd+o^jIzg}9hNH2E{N>97m0kxIAV&OTIJwE> z1SWhpPj5UHCx<<&t65f0E)iI?A)9VbP6!3U{c$MRp+;kNzu+8^27^LL`M>(0@ysP; z>}E`Y3-kL5P8W#%kALxhE=+s+JuSUd6)O#}nSWQ7r|nr30$ra7G}1kAjQ>EX`G1|k zXRo0OBN2=ES>MN$2(_bX4|W?dC;s;PKE2r7E*M@@IJf#UI5>T&rdv-2e3xb4cx&*P zK_7a-^@z`<9J+1S(x}v3-jH%&$kzoiEKtuPkt!bQPWS?Yj$EBYU>n&-{HG@T5+hN1 zUEW0|XzknqVn`r&l-XAx_!&cz-O0y-X#>mRxX7wGAUp=o!9gmH)k{#F(3{~9v@>qZ#(NX(= zkRTd|)H>6NaMKuG0S(sagRkTmDF{&$X$>StCZ`irjbDH90&{z}v^z%)dnNfSG@$+( z=>uXl;9#Ik8JOCGpDw#>zT_m>_=NdFKe#>zt|5iEYNEFPW&nZ(+4DIQ#u0OC*!=i*{$YxwS4hOuuvzn)RA9--|x}i z@Bjt&(7DxZ{d$>8gmwhHk~cY{GI|0nlfLs+x|we(knG=Q4Fop$Jx6@^A(hg%e^X*c z#$ymx|8@F5A#dQTej<&hBw`SPak_ye8#asH6al+DxmEO^Mwh%5kFuJV>ZeM4eeVw2g2b?}k%C(1zF^&9U9 z0wRW-Ku`q_XIc)Ohmz?gCCx5UYu!_LkIF3C=shG6aX~s8L4d~J$r;0W@=)#jBlZTc z5j&8O&fkdWh7l~Yw?~t~^e&V88>!2uze=|kJ28J???Elzh#~4~D&mu2C@FVChAZ=< zrN($M^*15?H_}44+6@Q*#8iE;z$2|W4EI~3Y5mktOi(?0(8^!g8J0z*C|jfKQTWph zahG4cc0{Ud?q|6VMt`inO%a17|00r#$xgFz->F6sx<*riJzFabJ-sMHb3`cU4k0Gt zy6FF`ztJ7rCl*nI?W`));T>im0a4vS0$lHgb-=)Ns2F$hBs-c#qy&g|a6xRQPWopc>^Rf6Wgy zC`#sc6sTIyS1=-v=;l@4+ovuei_VN-kv+dCSyK#k>p7&dC0PgM?76ikEePJebg8jF zNvhHi_YQN5rDtT7F4PDNx$Udg*oP4rnF*53wn(waE_?9>3TBao(wDKiyRgN!LL}OuY!6(~D%P>BbG=0KyFua$5i>T#LO7 z$WvDL!Y6uK+AzpvDa;BYdgE)W$Z^`KSq0#T-L>dH`Td*tl4bpB>EzRU+1Sy#Mk1j= z$?yC7H^E8~(dBry_k}Vxo`mGsVIB4L93Qu@NV+yM?wCB0rg)FsE^VW9XLlgF42onC zwKtNEr&#I>Tg*UPC@d4Z`ml2K)nPuJR1?|>=QJ*17d9c82V(b1Zf`Rkbg1y} zUA!-7XbHpzmO|=0PS~1=&kAsuQplQV6~jtosOZjVaV9d>e#7^EHRk%|Xx? z31AK8C=*iooq{Pn3m{IGW8u8_;LMip=Oauv9zr)Io@2EX!(`yopZfD$NaL(tKl8Fn zwvP{{`Uexqdm!$-C+2;qdFXSJKf?E!@Pf27YPLMniG3wFCOW6uhb^pEyRm%mk~}a> z#R?Uaqi`3nnvReg>CjRqH$(zF6_cpr8{1mRUOAd3(!;;7nwc~%tvE2AxAQA~koh%g zh!So0X1;h_q>Bi*d@t9qAV$d-PDltqmd^#Gc<`$(l0Fz%5oL0$F+x{V9-fXZ;eqRj zpfn9{&pWhlG+2=^11rbja6 zFpIH72uve8RC5kV)PyVWKmzgJtT@lFmJ~X9h?Ve>@6zl)5Js6c8!2?mc%Sk+f0Bp= z;FQiyexAyQH?s=Hx913F0C@aCwDzLqc0f-cD)A!GP`eowPqIZ5%Op%*GAOGA(I$Ir z=Beg8GvWy)M^OCVbtjVZ4L`)PCcfW`v=3TJAzGu`RCIA9mwU8i4mNO)ajh*xiaiK? zT%U|DI7jM+6csgB93kofxLq6qxBJ3F-=Xxpu*`Lw_EZ~T8E8wJB;N(n>_RfX)(rM` zgGR~2j}*3p%9LOC86{RTtY%+)Y{CyB@trjvH%W6mk@$EH*E=3gak0z2i9nM#P#VPK zLG>&1V9cDBhm25uD^DLZO9xk&hc+&satk^6cELK#QvUSPE$l9h#8>`K{O(y>3Pc*L zxA*fFJW|QN-p)`$ba?v%h*ZEGS7VB&4jXj$KBHME&@wr?=4Q8d-4FTSt6w*+!w5Ou z=>lY7)3YBuxQuu78oK$GWEy>2bY`e!|1rtYHdssLrHD$(dlQ7dZAE|HM1SpDI4Bv4 zZWzlA8-s;0DuG@Gt__b2_>tZQSr`PjIGwzH$}CFVL<|AF7sGfU%I=0Dl+w$3T=GEpKE> zSejn=5)bT@DL1=x~nM9w)boME%)(cumFo*t7l{p zv=B?UrKK_%jXu0YV`Q_^w5O#Ksz4=Lox|twax0})GXgK^cy8WomoI;@4)`9A=E|{+ zNJuVE_vNa@?sNn^ArNudl{A+=Zj-2$q4sFGm=30xtHc_AhUBBts4G{RWB*sAN!TD0 z8zzy$ik!Jwt$9mE#cAiNA(34@QMSKW!_HYiQ}lWjMyRfc?aMAU`p(MAyTpaZH)K21 zS!2iT`BQ(g&v=YxHKJLJ2bs~DK%el4WXAW4$EmXQ7d`^L>LRKQH?TI$4>ciLa9CuA zegQo19Wbi6@+hEPkk|W@x)GaVV=F72Q~mid`Q~a3)#csSiH)c|9A#ADWf$TA>}Jsf z2e?^=oF8Q`N*ZHJu>Cuq1>*za-m|Pw1F@9Es{}98Xi#p9^7cC6&CvII9sg>9F1!%+ z-tNO3zNtUSYCv3-&l5G6OG*CJ&#U*L94n>^e5waD(>#?HT@o=zzC!k0D;nR`=JUqo;sa1kA;m7&6|AXff9 zCu`SNqsdy+m8)a{PhVf3jDn)?_wR@|d{g&sckquXro*xu=E@9MzltB3>)m`WpYN<}4+%wZ!fbrL8oy_Q($vrW({FUP;@r`p zJ6(+)uG(SL86!9nv5Y-S{wiSDnxpvjY*`jEA1um*Il~vao*-HD-`0LtZxFvNdULm) z0i!=Pb$7ku%`tTmFp~1{L6hk?FSHMZJya6XJSP$~&?1n-NeW^hgK2L@hBvG1DdQZi zvEqZqJ!tfgj^nlqc(uQL)s~!{%P*!dn5TyWPa+l)nD;L z=iLI6S!$(lpnDS~6p&*e<=0RWl=g_5)D)3s2bfCvJmDH8QS+=qlIr&2jS`~s#KRDS zo(t7chKdDkO#+jyO^K%8zPQLoES$U)^j{y?XE|0uDOF( z*uE8&U46lO4M?8O^TkY}vG}YjM-1W3Z{_X(<7wA_7~y7@g`+@(BDpSk-$dyP1SFnf zwi{km)yY+tfxh)JH&QZNpu>eG%gf7_%8qD3$x)78r4*%H4&@!THq80c2>A1%s?}DH zbByX}x!1$yQZx3-ms0(1{)Jzjc(m#vD%#M-^Hm30D&VM&#Z8vM%gOj#XJ_M$zX;^z z<;Sx2VlY#yg|y`xVY84cG-7Diy-=U`;2RoeY`JO3+1QdQ>*ihhOt{=GCK`W*nEU!# z^aV3vvCm2ve&hO0|37%jJR{WL1k8U<2Z~aS+t>&(SqXv7DQh}kefUIicEP`0TZdmS zSd5;YYpOT!3)i3u^0Vtw3mx1Yoay(gZ->q%VdKhI@+aHY(lj#Rg%Uc3ek1AD?2Cc% zJWS@D9fpvW)AX6Q@o?yS26E+N9wX-jE{S9tOo*XMt=kBtszqLkCh&KKBlolDGWo5s z%{D*kwJ3hAJzS;>D{=05tf#i9za`plr7HV-2VNVt%hR?sjR^iQQ21vkFilUpk+Jea zB}^h+8<$gu`6M!X-B|IPNIv1uj@2I%x)1q_S=pD?hVcG$Sd%e!gU@^=_BS3x+e(h; ziN!Gy*;*v;v#1atj=od+$mu4@V^kE<%OPLu6(2nH^RAu)5VLQG$!BS#%Rc|Bn6ZQ# zLK>_$GEo#_BbhAGh_d$kpKo0>8edINwuJ+(?#FSPPCfK$f#y^MJT8>;b!f#>XG_3Q z<~4T7<=QVWLJUJ%tyDww zDKpP#x|;^eRQMIu+*LO?M45J$);3L*X*7xpFP0%|z{ZmO)>e}y@CQkvG*SG!H;G}F zAWS|HEb_d3@3p$?;nxPnjAe6AMOwuELM!=8cCiQyMUn*3A2F)bDwSGb;sRs&biFfk z;7q3$pIdS>;D;$QYc9ntqns)cMNDxS+M>!BX095xbnw!&B+5pJVq1fkvO~9P^5{9v zvtzzL|2p~e9zW9OU}ogjl0DP6S=r9?n>oACpAIVa9IC*gu}-tI{ijq|c#N305m*A6 zn56VKvreA@YP6qB6p2zxr9@&j#M?GM0|V}Pv-CWn#B_m*DP7kELh1yg0u=nz0S0-J zK;-$-%2jPMBtH53cThc^Q!lyhz|5);+S?Ry*E z`*x&BkDF^1Dw5mtHQd28+^mNLu~VrsxUlwB|2-MSW<5SO061nfj%#gg&1AjE^k}^i zk#-3Kyjt#BL%tJfE}v&-z`B`svgLmnXP~p%oC;`K1p1v33-1s4L$GNiqaTcs{Qo(& z>=BB0GU+_u2?u7uh(Wk0t60oK2eQD5L6F>+mY07tz(SfJIx9Ggz@2Jv(3C5PqNEiH zcZOK(u7Viom-BRAI4qP{DTPsm!s@D49PavfXBRewy03}cqsGLkNGxXibvCTHQlb?^ zH@~P1Kkl6$ss%z$(2g2 z0OPl6Daw_T4e3U|>kJ26VW1G){pbFr&6rMl4L3c#Z-7PVi3;h8jl` zm5DlrSma6}gj9y!FAV8qhUmW)%>6~uuN?pI&GKPl5a#CQY__W)=u}2induK2(d1jb zwzNBqFX1CAV>_8}m_sve6cbQ$`f(P0rU-C0=|2=n#3FmX(! z{vRL#4FDEeYTP&}zNLMVk&&rlzaX;9c)mZ2OvGIvWmdbwI#nkkS?I<5vq)YuN>XW%X$n}W_yW>cINmtQGY`}vHjm2SMkmd{excveNy@sB&yI}5q8HXs zAo^yv@*Bv@F2oS@4j)Wvj3iHNZY!NZoRy&>9)4EKzyIo>)u*Ylmt(!Gr{V~gMpAY ztcSB&6IDSZPa%I0Sc?DmSuuVSP5@YmWJY}gbPCxX)Z)UA*~wU|aX!Kv{{E^~r^P9z zgd6g1qICh>`D%&TR21uU{=Jg^=`i!TWd~!yU09O5J<3AyXrmjS3LHK#2gPX}t8?yJ z2XF$N)~LIac_DT6M%eo9(8fmg%!-%o^|cm%PI|PCXL2G(XJyygzG?*dJ`XhR<0aUI!r+U#Yjx{2CXYITp=!CF zuL2ahi8}HNzmU#ItnG1dKRkYCn5L#C9jo`eqN3oP;TRzbG{dzPW&upJ$IEL|JiSh| znPNGSl_s0k`PV=gy?_3`ru3Pst0ku@_5+maD%C=X0^wjdjZRPYmEwtTF;NQX_~{O@ z|Gr_rz=4`rcU;hCX3iW;W)kp?Es@2C@~rSa@T+@WCZlA2n#im9hfgtCeqoGD9}4_& zLFWAHT)umdb!C^6Xg34c{15!ZTcNRUXk-JFZ(IO2jtgwPt9Ru`wd8IF5Q^N-MURA# zEm`&2;MYjYG#=gbhw*$1$-Fj}jL9Mwf%?IixP2H;N>hgLzzR$oJ||Mj59KHSaWH$i z$MY(nLepY2?lk%r;`r)v41j;I6NKJ2{lVQmE*R!x2}6GdWu$Se#MZm0)A=__JJ*ENe#ZF~a=v`@sKu zJGwp~8al%aoHD#+@J_OqOON;NnJ~m>wy6AYgDg7GmdJO>JmMluZM+~wAgCbwbTB_QJPRptMJ;ml<}4X#Z?T0DbR$uZ&2Vnb><2hNS5qe!PW!M7XM zd~gIUWn(7^e zaFk>@1gA}@H+{$D{63IUF;T6f_?w78sU-cTShUtCZJ~s6;%xa&Wcqw>tvzk73|z^` z*nWPPGEiOpmpQ6GMeBd?tNE|VBimO;qdu$cZn5>yk0AO0I%{1HnKMASmT6u`UX`PCt_*AyAjj6k|Q`EH%z{F zH)t@^2Q{IoeGugo1aGyFJ=J~<+gEB)%I6KFIa+ll>q7;OkEv5z-OZD;Xh6`jMG77f`@K_5zzPTd{NOM0 z3_xl8@|As?*IB?U@%JlkW@q?!Xb*`?>g-z%34%2VPY*UZ0OZ_8llVd$PW` zOELy-uq00uw47?H>|?^=CBhuQ4@ta~*YfuWjuuKe+U|`;${j@*?ek4va9)3-sUX(G ztz`_cpaPCAbbC+GZPTE>!^a{uZ`kPbHuwFgleZwVN`*+;og2qCR}2%D6cR#IOX_N3l?M?MCCIrt1n)EzO>A6xICU zk}H2}HrqI^A5WOHo*}@}NL^Qay{~zq<=df!O!VMJF>i?)>7vQJYzD`;MgL!RK0q9{nm`_fJYo%L+Vt275@RlZfp1b1?OijI`-m{PUNrp&wm7SC_ru{|;I&`?*48 zN<|vI7S9c3kFU4Pw52WczhV^cjt#t(&7*7;prKP3#OJQ}9PW-)IbP4*JTXkgb#?Ip z0&ib9I;FyD+oz8GiJ`Ez#-Ay4 z+4=v>l9Rg8?jk^5n|KVmZ39c(W40DZXV)TIXXO^{?#?(kY`Nj^slZIQLQKMn9^{= zJHKX3aL0-s(NO-yO40Sr2)H5e;H*h=ER<;PubEjJIB^YG!JLv}2bJ)u_Or zDUf6b~O&EGTw%d`Rfe#>DzK+Wn*KA+hj|M=T zC*T0sNP!!7C%|K2u9}YvvqM9Z4G!0jriNortns)p*>Cm;NdJ_S4@V=f0j^R(!68P& zG5DqW?O0vja6fWVuug)}(eTFf#(dolbzI}Vn*sXL@Qd4phEVCxPCg@&KE;$LomL~r ze+1{pO90oUoB1-7D*>G97N=PJ{QNMebwIGSwWYd6a6|2_Tl1CL?)SIUnILT{m2*KF zlZpP>dKpnCtpC~^-=${_4Qzvf+BW-QWXn2`H@Fi-al=Ceb@!YnnDkxhrWpK?d=YV@nek>WUK=bVh~vV!2JS=k zW;h!4stP7(_>PTk)dC*hNY7goE8~^MwIN=W+0ORb7U0vJSd`-M-voNy%w4J>UhX823dU6 zHs)9h!@Ez|&`6|gXb}1TGGp?FZbe|I-b_Y{9o05B4-bT+I{;maP12aC3Hy_|lb-dj zhHiyIxtAiWqa5~k#uR48v2QhVI_x16p0cVk z%UGW-Z37ai?1rmtTSb-ySS>Frqif+Bhx^s_p!z$%ZTCSERvasV2?MEd_fuY0Ukv>9 zHskX_$t1!C7-d}`HmtK_@AU*R)=eQ8>LD}L-NCT%&X=&+Y{sIatBc9u30Wl}MeOMV zGb3(q^TnDBE9mHM$ zk*84i7qt4TmqpH3$hDQv5j$vi1X^P)6QitC2q?8%?%%P$i5$$zm6F$Pe7cAxY>>|i zxKCE!G*|N*@Soz6+kXi3%KiXVQ>XO&P+@>ep zg@9bQB9OcJKUa_Ihu%=bXg&LfOM}QOluW(`f~fouNBolF#_@FaPvaSggVwIzP@XDm zpFHR%IUV|HmX5l3iqNHD=L{`O;kC7`(vDD&eOy2_J7Nm;7a(WG!Vu^a&jQDAH0y@o zKAOS>3O}-92!oAFPpZMDWTAz-YQHoT%}xtOfEA9;X=i4~$e6MI8iV$gMlvX`n_F=Sc37!_azj=NSJVu5CkEhkktX8x^rCJP3HB6 zi?Nxb{#OjK6P>C0s>Sk#@)z53<`>RmwH}F1N$S`DFFEw2 zl-2$x=y0l;BEsX5R|lRB7#yb!yR{0qioe{OCgKL$KYkS<UDCVZc%CsL)v_DViJ;UhSYI`tG)Y2%FIp^-gKI+pY@>Sx3&@eI$*+#y42D30~on#U9t6~wjEY3 zU>{At=ivyTxK*w8y=ZV(WKQ;+C%)8lh2Orz5 z4V(G(qgPpXD=*VB+LYPdcN*909>C1KY&|d|IePHw>G8&yF7I(NSCpoH)`YO#@C7e3 zgPbibo@8ffpY9+iNclFDii%Y6ki6PP(T6Q3e3%d(R#mi0{6qaGc;9f!4+Q=EW^aMn z7KV7R>sRjE48%(12 z6*#|{iA4)!t5s^%=8_4mZq#%^pgm&)X>{MW=ia`)0IN4ASOkQi#6eSm0C%%_;*ah*#8`>Fd~#h2_3wj?OSqVMb5%;YdhM%RC(Qsz7Ut&v#JEOJfzmz z%BnkOZ^7>{z}I*VxyxqGqWG7G7!Mqu!2IFJ6N!0rORaHPC&By3yQJ7U!IVfO@R@41 z)4x5NLoh{{%~$W>_XJ+=TDAe{$TzxeR39qDs}=l1#ZI_iDo-(U=^-Zx6_dx*pBb;d zB2j5I;VlL{Zp%M{$O2MazBuFZe=b~It+nn)I^}alJE&go3oMzuygWzL={yWX4@EYZ zjHyhU8o>#dM>cpE{L|t{i9?=s+{4oUtGu7copae#Z}|IJ?Y$+-&-hq})e{hbm^5+p z&G5t$0oSdl)F6{4$Qw|9*Vg#5$Nd*NB}Yv49Xv$lEi>=Q*)V$;1;!4YW zHWx~}k05D`r$VI4`IRoNgS1UMaq zlK-V2^IwtUAq0lg6I1QC2gr5226-LIQ7mtvb@;_n&LU?x1^lsA0JLmU2NCK8gY7`6 zYc|uC1?X3lZXmhOalv#LJ6DYl-s#50@X*eJ!>kil6MwVl1?Ku9cO`5^+JV6d)=aId zmW%f)6R2_}wlAW;83m)-o)I;;voA50IWGKdbIFAYi=ZX6S{efPnvC9PJQ~wmtevi2zMQhGc0JO`yw7k7vLH8kj&yBo?pEO{+#ZbviwxxM)x{-;2H1)c zxq(U@cL>sPfZ5%zLZzi`Dhor1voRODM{x^YYZ~5pn|}Ls{ME2};UzS02#En!AMh{R z-M-=m22|Tcj%F{?Mm=qBT<(`d_tod{Uf)XMgP#ol6jWZq|6x!P!Urg#+ao|1Mb3JZ zD@r{M_9uCRzzG>7lMGjwKc1=qt>LiFGnLJ}@9@F}WGGsTJrpGh= zdC1vbXIx?BeFtUkA(~$AogFTk!DuzQ1h1^e>TD|rqFuovCoxd+c!eAgAKl=8qOSgC z?B1R~Ru@|Er{U%?<9^2oqT9XL=nI^AC(OUKhp!`Jh5|8LKqq zTF?C)ZLQF)ehL@_9Q61tiIx8ldrXMT-aJDm7z$Y;Mo$FT5c%v z-Ru3o_{X z0~J=FFi+SGZo19P@2paPFmrxzin^bEpix8ZDVF`*Zbm-gE7fB3`_RzxfdGWWO2vk{|RBRaA{`m`US}@Avd%HvfilZ-xxcomNj%J zHmTOxLVNL_z9C%!L3cOT6NnRKea0$B8)>Heu)W;)X76I~R;=;U!GXw46p;`A)gtx) zL$Bx^=E4-#n-}rVXNkNCTK~gz-MEAa2Wm|JE`H~<$)q56GE?8uyj|{&hl`CrQZ6)L zu%jeTCceqDzZH~AXwb6X0;+Vq> zj6fV$2=K|GVqgRTsrGu!*AG$2WY&b#5rANH=X-s78$^`db5kx#M!sXH!NsF9y^1lU z)yktsHp(%YEei9$)AQM1s__3ApKCCdy_($Yn7gBY8An>Nc=cW+2u7ommO{scQl*M4 z5EsbB$PMcvy8uyg`KOACFVnYrH@s&KS6 z8mrC}?{huW+*jl+F?r=De!9oun%5lUZTXY0?go~VeSxffEiHQuHB;^qJY1ON$pWij z*nxfS{&icvf*mJ;sMpa??9pRE>E6CZ!AESG-0c-LrF3LVvU(>;pE#^FriA6+aNhCh z^oBGn)aRHzn~a}=(XT#`cOx+g-Sl^op+3R^bhUo>e#krV0HOWftNP-?OwYu9XU$nX zO2}94&To=&pu|s+6SxlF)!4&!KI=B}q%9j^@vB*X9v+xk^d7`=s5Q+8gV0~!wBlf(Lq6#8^#~zEHSXbyyK ztI`)y{%2Z)mS7zm$8do->s#A47XRmu;G1DY8w{jTUpki%z_R^r#%9fF=# zT3jx2{vOSC!2%M~-b4PWU7{A;y2S8iSSq&F`=mZqb|?-a&J(2~V?f(@d7pH27&mVk zTTZl0v+b7Jx6@;|1~#c=%h~+$@(Erd^}&W6^IEShrdh_)Of0PwXIrEUZq8x0+({Hj z!r`?2Xw6B#gZ;!!bmG2taO3H$D0C}s?B)R@-W7fj1B0GS(4<#0f4c6dOs=gCkcv$h&U2X+;-OmR4|rsPt?9xsS9thLjbfYJ5HVPlGPP(qTBV|&m~8IH zHB?7o-&KG22nX(J)(L0p^)Ir#_j}GyBrQ;R?IvghdArB<)VS|i3v)J z{Iq*?ESUL(K}lBZ0sg^KhjPfk{YmbSWKi_edaw;s+n0X1u`oJ*f^c4Lh~?#}pTEVr zMBG}@8X6h^>F9G492Tlk2B%$2r`Lm{@(11ftpZ!RYzTmTsObl$F=hfEY+(-?z?mmL zD=P|cDAlW8UxT?Yn4=1jy}7mbQe~9OQ7!wsC@9ZoEu#P#u27`@ zxm35E7tVr7-ChbtSXfMfweiQ7%vUX3IkWTrUUELgwKC`-`Rs@?WlH0pPRy4cU+oXs za=zpSy(7}(nZZAL6;OpZLXFv%c?l{$H}0f z-fxlf4@#zGCK5}}yBO*>ahB_Xd3GNalQA)cwpM-Gt|o?23)dop{Fo9>(~8FvZkxAxNVp2LOYB*{kWWIxtcLMgn#H zesH2ilmyIxz-z5!tLoMNN)w8NFAV%BG~4Pn9k1p>aCg<4Pnywwii59BO~pn)@Z}%d zlZe}Bzdd5rPN>VvVTi!RrJ>l3>n2BtPGB9`pg~oZ8dXQTmK@c#04-OOzBM36pRPG* z3J=)QJ72fKT~uxSJ~RvEefs*0`6b{?KiTUGs4*urPvmTgGqqLh*+U>Ja$dY0E$bQh;Ylx@xgLPLurLJ{E;8ylM?LnV3_ zph)ACiQF&W(4rgl%9ze#?gqPWDyuF?l22n1lCL9*iH9D69{sPN^~K7UNIiH13`QX_ zH#<5z{Wy4@KVfXIC{XjcA)In4o2%Pq5tq~nq!xmVW8LME{(7NXYEt0(4+s)(-15^7 zGu*PYzNn;Hay{&4o@rf%s9%>7t+_R&y4pdNo_69tWTBIv8GEIhJ(&{uO#7$qGh+QG zu}rs=q`3HssfRWy{avv63>*x>k=EaXF2n4-5j_Jb8T*O0N7H=+_%jeKLWZ@GNu=eG zt%H4AK@L~@ptwLLw$T#&!>#S`o?;n$L#Z2CLt_Crk(>Tms6dCmMnp*Nu>pLo4$^@& zwO+V%KFW}hjFn=M&Uz|At0DQL;K;oNPmYH_lY|CZffC`E!1*A88s7LHNqRR|9^YM> zs(%WGpi~pv-?xNk)K3`usQ>x6i^RhTxepZ*Wh8NpH!%qUf-Z(Y#fYEE(#pxS;`3jyu$t zYIyX*+Y9Y~!=&qvKb(y~-=m)1)Mmt!( zi(22R6LewhxS`7{%_`Ixc}$R!BQ>W2HXLomnhTcCZ}=?c8=oPt#+j@P?J#1xg17^R z`d=DJ+_a(4a*bvR0%86#_NN4D!Dn_^}T2${lg z3H30VE0(jbEI=gKKsqy{GtS0L`DPU`&;5%F3qs%u_WW_! zDOw<$?UN@fhoHnysOdxI2$m;rQnHvYKOhjPXI&Wrrq^z}QA4#k^#w`jpd=)WX>?wp z4*4fPlZrK)^wRBK4lHdSz9gN=_A#8MDTRLgjkHs}jYWEE4u-+m`W!m@?(ccTC ziSZd;BcTJ0FsDit>1UbXd%Nh@7aO>nT~Sg=XboPATNm3r2son3AVtoVsx<4!Q*1Ma z8zy^io0@J7LjfJ}8m5xu+hLI`NXg76TTzxZ^dZK1#7KXur=bhfvCn{zDLXV)g=E3x z3)Q9tc-VwGnQu0g2a4U8k|B3)r*Xp;Dgb?wF3sx)$gacOV09jP${7!75-7?TSp%_r z{d{St=Y}^wZV4S}ukVhK9cE1UH(3t_dUI;mK?AU;4b7+-oy<3a^rWDxkvxpXO@l4O zghvayLTVYWS+yXx#mZ>QLjJ#zgjUR)tT{;5zg5kga8&7S5N*!iDH?2G9}v$i#DY@1 zh470qXGIKs$zCT|UXlQ#2->BaN8heE+OC*3QJp$$RrZp~NlF;V5ShC#Prw+T4GrV$ z?`1icA0?Pp5l)`JuDAf3^t)jHl;dYq0igfv&ptCrYry^KQvW{Z^I8_47Zz}}G^LH{ z^thqe2#O^Xl9o?EDFv_sg~r1%Iwu#wy9LJ!)r8tzAAl#AO05FKN$%%9wo27SK`tai zB`RkHn03s7b=wafCnTl&-vUk=-GcNG|8lj>b0PFoKL=+IydE;Nr-e$rk8J|gEx&T zH{WUEL}FSRk^%`7IX<}nYj*BuM3I;18bgcD&14M>cA3mQ*`(Cr&GB)G&E}O$dIUMN z$f;`TslS;0Zl1`pSAeY*7k5f4725I^SBJL-UE7tQzrXrB)q)2Wmb}gAs@Gx>G5H}F zZ-~7A!ykyWB+D?@GwpmSz#9ds!NPyG6RErL#?w6xXi+`~aXm5h!MpWr%M~hY&UlT` z!599>2l?JW8Loz7+}?flhm9$FGeFI;!wvU^XCPgw;!Cm43=C7w_($-^mZ-%D=_&kQXQh&8)G9WXO@WT&zP*5~%k0{a-AdV{|56 z*R8vgbjL}@>Dac-j&0kv)v;}NY}>YN+qO(>boq?>>UW{o`QPnX*I%E6u4Xu|)h7^@lgwk|zWFle8h$<*H0 zq<2md1?GT`Pnk;9x<)ifwF>GcNwrYX&V?-9&Cf=w_MgOV#Y$24XFtjxZiD|Bs=$~T zNeP8-fjv;YLIRk=fzC{lO>BTkJe104BKwc;I}lZu0|>XvO*XiPpamtH*};NeA*X|$HX)_|etA%KX}O#U)8l%m=m9%U4Kwu<^UCi_kT7x)KF z|FVQ$pn+d_X8(4W%IdRJ8mE)J9&MY=4@^$O9_LUF_@{cwT3u@87dlPTsDf7F?CmI@ zqHrm1@oN^s{aOByxOPK@Wyy4&IXzY#qnPW<{gwDCGZ3PF>e{r5f+F%($lZ-NmFs~? zK_DOt5{GR-_kL~S%w8gzD|k)9=4-WA@;g`5ZZ1NM?}zenn-lCh#iv_?O0mgI_IKb* zF#_MN3t@z@{Co!r_l=y5VU67+v9#Z1XT_WAa_Oc&>xLOQOK27^L#qq=Y43TOfdYiI z5MFLY0Ji37vDfN=Gr?vFX+xc@dmmg*CDgac?)Kay;dB*sz`wm_KMq?Mz66RzsDTMe z$G}m~SQ61r;`)3V3ddQI) zz$@py9gCQ@6~gxyRT64Sp|LDzhcIQ{Dv`jYRG_afKKnue8l{0kMk)p`$66kIpNQGrW;vBrjP7Sl!3Y~|Qh?V|@ z-_`e@MS>TygiDwMd*4`4AWf7^RU5ba>OcEA6hVHT{naLS<--%-B&oe0!uLq_u78zT zmneKZg7zc`QzWyogl{3PdLPn$dq~#=0yi-_9e;l~&fBYV`-KU8VJK3K>Hv0guYJfX zyTfb200#I;aWOal32RPU!q{O-kc1N!R0B-(k7wADZbYZHMS<9WGjv&R^hR&#k&oSCMOieWuKokL%0Wog(oGS%WGDQ+Q*HbB`JC8hV0)*ZT6*~LUxDs}@CTv=NE$X@s*H@r z<*m)U+ajNBZf>rIYz}2ZL`Mgt1RPW<)ozk-2M33@3V|6Qr*vh{z}R^3GI@&{89^RN z6Umee11K&PwojIG)TW$|*NQsi&n$;TY#ENy`&a|K)HG63(R=3m!Hh5)6<@Iz-532b zWy6v+ob`QB0U5$?SQURff}mz<0!1l&F+i%oZuwJLP3`yZT%C=65_nq8Fv$*YlD)^c zhUG+QG6GS7%{`$6Fguw?gPomj814V>h4BWV_NaGA4?Xd}1OBKO#eT3jm66^LN~jAP zu)l)5}dz$fO~?95My4j&K%h(PG6 z;Tg31VvSt?o=Z}{C90p_IIronmt0n#O z9ZRX~TG59yWK}l3#VDIO)w{HNT#ut~I3i0yJ01viQt5?_3TurQ^FZ`;qm@sC6`@gB zepJn~3rRYYSUdyyZT8#g+IR*dCudozLzWwV-@Sea#V-gj5Z~=DGZV20lieE zu586S^ioBb$o;zp7-(QV+#KsJlLpU~$--Ay%)c!Xh&9l6-qMR5O+jFzl?RK2H@F7d zp`fsCo2&0mQ41AA`^vDv*~x>O$wd`Qi2?=9!7$~}aT2*jDCX@GGH6MaFZZ){kW0P3 zaH9WiPqUv9g@)L0QH47;q;7K3F3#Q=R!x+%Yndu{LBSf9{VAZjf#vQgrizzscQ!J} zo;gh-@_65pyAT2ppZd5QHt?Fbt+%6h;6tsQHm>j3(a@e(gMGnMFTd2CcAgN>pZ#S@ z?+z2s1T}TsplFwTV!m|^x95KO{L+_W`lnoW@MjXJCX+kV^9|?fj3*c#KY8+1Ey;Bf z5{6&;g10??b|JyT!zWRx`86~&P@#TUOJ{Nj0j>|DE0Pm#h>@#1R=r%sAZb|;CV~H5 zDpr1Y=h@1+pII0VzsarcW;>;f=W|SIMW4fjvQz%v`!61C{VQBQk{Zv7nEf3qV`}%?A zABqJFukx6y{(xg?7fxl!jy{ntLG_y>2z!dhCeQ6FVvW_>c8~L-ux#jv;$KjK1|^3= ziMY>W>sL2A&o)?|xpL0Vv7je8=W`B6Ge#Tik*r08l%AU>dYf;iU8!>PpGXni3)B^1 zAJW&N;F?U9Og2oKk32vBB($%{eC3*1rr%B3Dqlr&EcmEGNO7W!#^dNMOt}eFB8U<| z<{%aA7Y)D=9?s_N(j@T)ruhdKw}S6iKi=Q>uj_yk07JtEK8}qv@Vj^@YynZ@4=7~1 z$9FB!t@xOBq24-CykGAVVBFlIZKs4QpA^%omzxn>S{!p!D@ziyraD*!7QVAYW2+g9 zgpKw$2AT{eqQVO}k;9dr@`khP8&hctSNlhFVu}H21(yDt1pjI&Mf1$b{rZ;cWmak* zp?sZBhLaY!!f`m|DZ9}eDQ4Q)>GQX!s5-q3@{M{)_R~QDhxgLvRHiP=!qqxGuomSm z7SHv%XACs{gxenasl#Ur>q~~9Ki9MnD6s5*V2`YcmV%MGHHq#{TL2ix?BD5D8sZIY>TJy(zNl8vhqv-!CYuKKD?5ED-MWX~?@j!Ilo zs$3YLq_`W!K5&Lgm@0p|bZh%&9R*H;Z<0z8@mmWuShk7)xO~6*kvFHR91iczT28GM zoqHZCcBm?j_U6K4AA?s@&)Q=GGpe@t1{)ala9x7Zt30RAyj;CMG*g`YwY$`aL^mSI6sP?39of}!c2q}4vOdWvnWBP z0g0OmrhjVV*alkZa_vaCrY7}=-^ZSRdw8P?Wkfui)bXDsKz2DM4Jt$8QXN%wGsud` z=FxL?)rTLUq2ADcY{jeZW%qxOpq@SIKQ;_ARDcf^)(;sCc38;;QyhD6Wm%z=A z3{d^4>(4a6}Cj#;lzOe12FBcMtaR`>0zdKB>E;;E_8-)?&LA_Q76Lg`Aj!t(@8XJ zd+BrJF0RdMzspjEd!;K|hN_3N6xO40mABAb+h{XJcQIkcYltKxNZ3F%GN<>A=0vik z#gmLt+K10*PjpnVMAZ7l54uf-zS)XXxSVUm!p23q0z)Xc(SPNg3gJPdN^fh@&9N$B zvO2k1HNMJ$_q1r*!Dt3udGqP$A#x9k9;?i;J$5zJqkZWF((~&t626rP?e+MOZD8H# z$PlcDo4X}R)=BTU9#b@Z+X3Yd-8}_HT6MEQYx78LQz2fNZaJ*cdcvJ)>=a=R>ZRMA zOY6a=Vv!-Eso^;<1k58BqTmMH>`jmB(Tm@em9+F@YkuAr61sPuw0WAR(@ryPeMc7S zLR5c3gy{bOkm>z+Uquwlv&9NP+Sf)^&`)zbnT-XYw4R=xhP3HhzC;0H_@?3oalj*h zu3>;$a;ga!4gvpKo+l8%yf-+Q)s-DX$Apgx3Fg-YS;Yx{tWv@79UEE<+lJ66iIPcZ_ zC;lj*H<&^K%y#Bg0^vo`VDZefpL=M*W;&zm2n}L5k8!%Gna+Zm!|}|#lY<#9h@}As zzdO8WErJyVyv%=6pE4QDx4z|{H3xLGq0JO2!(rt%*OQgsO}n|`wnk%1>^ZtmPF~1x z_SJ|FVU{>$jUDiDcP69GvFBIpp+9y;aBLyA;vv`m(~IH}=mGDU$`!dmkt5w6rNUS| z9f91T7P+L!}sy#7mSfDN8Z`%CZ&{ zsz!u0prC+>W3|z4vGk}l9AR@lM;{17i7#^+V8Y)NiW7a{jFFzayu7rxtV&=cEBwuB zok7(prauEzT_aiCZ8b+*c>Q&vWlMR9<{Z$#dd3w#^A4Cybf@3xp0hPNzF<%?S?N)8 zoV;N_sP82ro;F*8Di)s;#grpKH!fFvW))u}Y4XSv0Xf z>d%dpeKx-PE>z7h6(E%R8qso1KVK(W02c}hi2Qv)j+d+vUaatTp0+dwZT}S!-r$QP z_OktSMX`s>a%#RzHwOX`b_xgK^}O?8`$Q#L?wW>QGRB=lx)W?5t z-trA8Wn=t)2$p}FgshMyD;G^;)&X{DBbiSjU$C=vEwu(?Hl;bWJg=reW%kzDq>6q%X)teckmUml(X}wMn*NC14U=m70Vw4^ zMiQ-YOMiToIv=T^%rscyC_<(9o>1fcJ*ro|ej>5s^@Bg0bf{ji2Zv@L4PnOby;ZNa zV2(M+2v)Ui349%rptiwAvtO9IM!*H0*_9VXl4w_b_G7X`r_aTnQ{Jme8|;<|*?F)! z6aK_()CpD-wagAUq%LerLNFQT+kL_iSpR)i4?V zl<08wMuR3TOfcKKAD8aun-eKrY?j z0-IQ1rDMtxll7ci`IjV(YvS-xATRFQ{vMq{+A~erByi39r5I|;QO(0{*~Jnd8{e)k z@^c~TOA8b>&i14?|9hM^rn~#wyTIBz1WYR2 zNdwrAQ&EM6Xdo71xfPh*CTD5UL}zraudyPsg67PHo<)=|6Gc1WN$i(M4{vvy-P=zH z;(ED_rOiE%46P9N4Yjhj53WD3BZ=Lo&u@vI=&m6;u7Mbf4#GCy(q(tUf_$wUT$Nb; z;vCDOHKDw5N5^6Oz%xF3i*MfL!#cX`b3PKYRH-2KzC$I#sfTx3 zHq$?@86P?w#zI^~5AHk73%YPIxS?Tdl=-@ausdVIvzr>|SMvcm%M__e=Y}r2jkH@w z9C%m84H=oOB&uW%a+QG7Z#x5XhDMA1Coh2|?Zae^?1y^b8%jhm;|qq=q}%oZvzQKh z*O3D%s#yTfQmGdQ`$L;Z`e2plgbgW}7`%GL3GR&gErin?BD55KV6^Xfmr14?$J&+l z)v8AxN3XrxIz~^+U54~BnMW1m9VWYk1jXrRl&)=;w&TTelJdl9JmocsiuMoPR0jAT zwMGP5r-npfxM-NreIbALqG>~m>0G}vsv8aTZ=rSl?N&|h6OE|z29<})22c`XVNbx% zgD&}j$`k>P783D<5I~k46;P6_wcg-(VHV5cp+DJi`1u((R!H6MhKvi|#*sP(YkEJGUwYh4GEPJokHnXJDsF=$ti}bCVO8lGV^ypNFfb2ijDT z;(*Q#T-j$|O&6M&>BfXrfA(46RX#gc$2)HHQ^c+p4mr}ig(Ycu4-?Z6@-gkiK*^l* zxIfkn21DRcGTiSdH^?eY`F}okKZCu;cx|AN1udw%Sa_HNqtb=x6 zQm5(`su@njs^J5{NgLFYF(r zl!uWJ<0cC966w#DnZ7+l19-Rs)LzA@HW|Zv z5)}eS(xRZskXy`F%P8YbU<@uZ5XRFvGg|6`>6{!rRuv~g1c^?g+-{65*IA;mHB9JR z_@UUJrcETt+01{~@M;dOz5V3pZ+hNu4DJZ>HEpfh=Z5Mb@)9z(Nwrvh(p-$B5PT;S zVH3wHYcL7d<1|~FxI5d!lGqsf0m5&|A3PLwbD!!EGL+>D=EiflJ+3Xe{It0I)i#s& zaCf*@B-3u!F_;Q$^KZNU8QDq(llEG2fW;(36c3sV=$F5F#jAMfK*cHnI6EghI-0UN#F^vUfY_?B= zreyW|)alitpsl|jw-yTMqIMq51hYyD#4a7uMm`R0rNlKq=2UaYRljG4y+G6^RZ1*^ zp-@N5B#J*DO5hy2cESf%;qrr$?9Egn^WRi-(~$c{2H%wW2-iOwwCGbB>-DGU^5i<3 zM+_|&J|Bz<^CgZx4>4-8vyS(CW}#}`JhtO3L45Tu{;L&IC;`XOQoxA1_|1aZ=@=wy zn(!Q%+1o~;Hm$DqPev;O-T;FzdAivFGMsgg)_)Opja-AS4{>W|&UK9p9>anes;&O{ z0dsO_TpW4H#lT_ZSj$j3=1C*zy3$AuB;O!bR6YQHLjuY0Iby+N@&~lUC>vN}v8k&} zU0$qK!rm{{M)_+Gbtdf7{exqxl&L6O-c3`36K!H_)Sw!Hch3oQ)Kwfye~kTZaXsl( zijCixIW~?}6#244l{W#u!o3NTl9KFka*QX^b`B1NfJm8racz43NDSuvt~gbqP!4)u z#hVM@*{v;71l*F56*IxoMF!W~J+~I*YPCd1Q`yP!E?;LUfrp#F^#6PHehFlfA)!zt z{}WF{L>8mnVu6h=A$Dd_R!vR+tR{^eF$`ymWa>ejcNJHIjK&IheQ=nRZVTzg%wwonFLBT%aE9m6KJ@P;L|2o8(9l z(xT1(o~uNL^JHP-$~_e&S=1&$TuKdk6OakUxM56bFcv`0Vk%xPS(KTUT0x=JJP zUo~#*Bg&%3mBe?yXQqk>lJ%h?0x};OKbWhs*Hvaof^+FB)o)k3aDxrH$kOZ(qLobN zSKJg&Q~1`(XrTDodhMk|v1DGn}*MXq%= zU|rkhiYETDsXtYu&SXZkyWJ9oiS}lcUq}Mn(8KI5AfTWnDpgsU&OEKLaf2%>D>uh8 zglf~Fkb_a!lwXmMRk=WF->|l-eDey>%038>sJK{~&Hv;~34+W<(lh<rNOrSnI7t2o|3pVGD5>HR z5>yc_c>djn9INjGTqkl#Dws5pl=TJl#-|~DRU|~Ea-_+)`%M#96_o4E>aGxl^Z-c0 z5Q(3PET&dOC^F#$`;-n}_=U4Pn_0xtk=JiBiZuw3EbYmx3LS5FUy0KmntkSrWLW8} z_;#`O`NQ&|`;m(n-lRbv!)_H3nBL&uX=Ui!47}20xB|T*YpBxa(x?`Fn=>`0ju#h- zXFaLFt$BA#+PDvbXm#Dg6Ergy!_sK_mK`rDb`txg161$90nkKk*(&hXFsjOq!scYK zT&^~K_{PA@Tp(W{erqookSn&Z0m#3CfFdlAP1@S*{n3i$?EHAH=6tm|(d`R*@I=*! z*O3H>k_0^s|GQ;Cp>akgCSL^aZ*C|YB`v=iQ!=URt}KDP{y{27Txk6iln>q9y>6%B zdNWYIl3q6_Qv2SSR8IXm$L3#Bvk~%0qfsn*GzGBQU(%>vaIs>9q_omQhw0uAmknVLd1%7W9(eltT-lHPm zEvQL6IYwb7z?jlTf)>Jxsv#4Fp*kRR7v#cHA)+E@{@GlG~q)2-DGQOi8J zz;Ddfxm{c={^o3@!5Vo%-h2Pu=SghC?E6_^6VXZ?HB7lq)F^dut?QK%*M6yU4LH#2 z;LmSf1?qv(P2s*^_#!~aoKf$Pv=5FCfvh3xF9U#9-ZK>Y9h_s zSw*mQXmRYdbv;ff$(&Gq-0u2vs&U}KyZ3O-(B^_;T+{=nefP|H2?8A8`4v+N^ADS; zGkClL`if*~oD^Z@g{HMWkq=kW=Bo65^5A6k|07;{_H>5d-+Ar1SF-~>d(T9lVKD)_ zA_Fa)WWs3*;U<--F0dhI?;jhVRi88~s*jL4bvzM8D=6q08NL42z&$bx)>I&q`i=5n z>5v42X39n=bG6}sVyxlJ(V!2>hX*TN^dqKm)#;>b;vu+0NOG1*f2Y$4rYs|^Yd3EO zU7Me%HI!!<^g-gJ)Fhndb4dUslB{BSBz7;#bxLh)j2n}%0WEy>*Ds#mJ5L5FZzD@P z5s7YfjJF?M>veEXENc8`VH+vTs$gk*thzK)a5O_+tG1giHvT4olDyX~FLrj)w5^it ze6p2ElG#geXI`Kss`&&~L*;wERtT-*%l$rUN7Ehli0jVM1yYF=&5mc2{Tnz$)N`3e zoJ;hLX3~B%HoCLZBj;|f%t{wqf+0daDS-YtMtoV|DGb;J;`?UbKk|+wNFb@>f@2lk zE>`M>Yd5~IW_?$UoIs;+V zbiNKrmg0-MS?5v57OT@@VZ-NA-5e5`qlNR8w&aeui!7PE1k`?N<`8UrD-RdO2IsWw z;@*;kBW=AI@8@pP$j>Q7J7+-Ysx>rmhTLG%o6HHeGrt~bAT)pb6Jtf5a6 ziyySbX3^}rTJL_pSUM7Mze-Qph-(~)aJh)sd8i(e@fX}B=zB&W6TsEwznt7tYqjTG zJ9^5uoLo1YNC_3sB&l})l5?j0Ej0E{0v!4yj8ViD+NKDsR}%hB`N2k%#ex-VdGM4n z%GPZOIA3jbzq1>d#qDSu@PASRyE_^laVrlHuA)gVMFCA);ovu~xiyu2*i>S*Ss zR!WAd6`;s>w$a8~N|)FH_$c7vf%J~Pa<#e?7Ap(@^bj{tw=5|t^nbEzieO(IA^5V_ z7ZOK(bLDS|E2*9X|A*D44E2uK6$h2Cmu=yh1mR&@XC}@ZDxX`WvSFLWP{M;*y92qgIirx1w4ne$F{{Q;|$= zj0bHTHM`zXr8l++e|AEAx&dg-DSvQyf`(rNXYuC%8*Y#q%i(W zn91HOWUN}twXk#`;D;dj&hwqaPn{+YEbZm=OX$jN1IsrzG_&m~UnU!;-ipy*0)zM_p6C_w=446M zHEY28X5o4^bP9HNoGob8_5rq3{&&}vov%E{jsi>O&M8jFz`L8OH~yiiY!*+z3a#Oz zG7(jfjP;4GwE^qha`H$Ca}_r1siH(06FD$&oG5YD*4BpU;gc03 z%TxlT;b;p;8fw$__V&4McZY+E*&?O3Nt7xym+LKr(8hrKl=-pWdOU(XoQZ>b6}?oM z^cTeER!z=*$Y#B$(QmHu*$zFeE%Nhcg4BblU&Em_8{|r>fpN9Wc_XuJWCFq{K}YHv zEkY^Y{)wkUVMX{jY(aXb;T`o9Pc2=J6Dt{E6jK3*kPaMEJJ&_q-%ERmU@wGOi2U^w zjbpp;1jpFHrv+kh?lIvZ%DCX52Iu~wb zk4N1Wa~imc)42K<@o_Kj9gbpgKa=QHFmxWR_((#rc}q5D3m5$s3(mzl;a$~E1V}WU z@Aw@y7V4fHHO^h`Cc;^!vr{H8V+LkVcx^py_n#~pO}cfr zAvJS<56Rz5;Bo}}!jMN*;)1wIs&7hpf0w!FoeqMYYtaWO7sPGb8A5_$DCW8+uV~U! z|0$zs7%@bVjtW40NPnu*`9$oCar8 zjC#FbE%_bD493Qf7BtlWZ6r?Ol2j}%0BEcXh`xMuB?3L7P(Jc3Rp9KP|1HpqCn&P= zkHFpn&+&`)Pnl+2^oT(I)R;!h$GiKPSkZGjlOW`fdzy`OPF8mJdn^%w^JVV!zKltb z4cve;UiSy5lq1;Qi_@2>Xuu`MwJnv;_)TXr0%veruqpxxj5$^PK3@gj!VT7l>onq$ zPpI#kmH_T^3VTIR_2>h5e7rYk!D?Kybf{?ZKx%G*glv@*YpbZZX>oy+HUZL zkRx*KWGjg2K3QOtnPdxYcwVQ6q!u~7Y(-+ywTfzFa#fs2K?|@Q-sZfT#?tA9_Zzo< zFvPWY#zHEq5zE65O*?%tD9#6|uE%6Uz$SqGEh@XCdt`!;Ed5votS*~KkeO=`jo-%> z6V9I`R`=bwT+7MycB$JyOy~XFhYzZHd@@*Q&sBwj$341jSt?z~@6x~yqvhw?9Mn4} z)4&m0`w$Uw%j%7}8e@ALgeicGtsz)5i0o z`IAJoeJl}==kah0brWE<$h0|M>>m`#WXDmdF}Sfa2=F+J2wn`w(v`MsSMfvbBm}6C zi+PIny7G_XyU~fiERH5p4t09HMO5ezXu)S@W*!^>IU!?i^PK}beZe;m55#)C0k?O5 z246Yho6j$OBTM4=U$7!7x*#v0jg>9=nheyT8cxR(cASQd9MU&ev{F2`R65g__PS|E z&^`z*ioava6gH50RpkZcH2L*M7TwsYxsAhG42eAHANM@PZtX01?6~jX(1PTDPiG&g zH6aO{SOPB)Pxp*iUk>dx))PX5jb4N)Pz;~oqIbVkH8$_R-18^5#&zcGv#UIo@Kxq? zwc^5CJC9s$_P{ej^2BL3zpTswY@3>}vw$qP>mMzQBTq45XpfXzL$wGHl%(1fh>w0#feC{eh5A(F7cr zm@6Ump~#PnhS|BjUhngz(v`KNBFq;`N8AJ$O(w@D()CC}=|lEoaXDwMPgb0yF}*Q| zqHqqDTiV*;<#iK@;1e+${yBZ@5;Z7pzr7=$p;7ByR5EfsBwdme*GOe#owL`*F0eEH zo-HC)NSaUax)so~7~NIb#d zN0&HiLG14gey_&?7*a<-1Wl@MB=%d~ULx@%@u&iY4M`lwvaWHiK!I`|OT&|9T$Zw+YksZN>4D2LP~djB1DwwNr1uf4#{DCCp5*l}_= zAR+FPzkdEqRj8hGO*4s3eP?;yBNSVF<(+3w zyYqB{X6l@x@9+e7`pdq`qy{N!WV3!tYS-nzGGS5-0%M>ygbZdR<^Vq!i0YptDsoB3IBoep!Wt|Qj0ixA>|K8viQ+FAYF{1Oc4Xs zZu0Klo*;m|TB$RMrWzGRYr@4Y4xqE!{quT%LP#35S!)zt^Dq=!+9X!F*mc{|+R`$j z^s`dCgW`w`xb=xOM_=JGLC^yOOE|Z+4(-iE)vwM`B>1QUG{9$jymX<4!#r#84qkRs z&5vUPE29*?{YH(dH{h8(eh)oW*^uw<1_fUG^&d1T>W{srP5-9N9{ZvA~fBJg|>-W&!KNGE#gD$nlw?Seh2^?)! zW($d7lQx()Ta>qBs@O3bA9M3>tcwm8^>2^Th6~692InD5Yc9+Y4u2021d2%)Zff%{ zCBz!@Jre?N)G1j2=d>d6<`IHK3MZ4r)w9SFHd2YV`b&T2>S(W~q#m~>=*&9HzLKRN zkZ>|^RIbFGx-@w_#EYoNJ@bii@_cJx6I6uV=@XR}_J|&`LBWlr;-WXc(ka!Mn{6$l zmy5E4rJ_)2b=YK2yX{rB{pgXdtJrjo|>eV7$|SQ_biATW9c0o(Q9E_qV+?JQAQ^bR}}-7yJisM-9nKa1z= zBblU_CA**8iQr$r>H_r*^V^CH4gJp_Q=n3%eLGSPxK3+zuk%FFq)&Q}kB&&;oxOjO zD@xuv$Mu(GBZLt}6NyEPCy>WyWFLR2k;Tz5lQbtmAfo_HRQ`IT;YZX1*^;?Dhz2Ys z)hpsBiiE>{nZ$_u6Z!YQ1w;k$iF)YReaD*)j*=>2tQvP}Wby-fv&|>tFJZDoIn|^6 ze;e{lQ+r76*?2dXCHHnK&|8;a8JG;e;`iD;z5fO-AYp>@Lm@6;kHsI6x_TSY6^!#{ zXFs^hT;lK!PG$yoS+_~sZwO|L88FVG3ygeyRAFatkBozhj@a-OG7eyTzY z`$QI?dL4_^L@l6W7`b=vI9FrP?}u=E5rHTP15Da1FIUmq7Z=pwLUFFdkSe@>j!hlF zsHZ)I#?fRZ(6Q{636-jWEfDYx-Kk6bzXqg@NM=%JTcg9%<0o!)M+UkIQFD8FL|6uO zE1l0K1ZqGHdq?<=siHaZ={!?98<(b!A&ZSTw|$BzJkGe7W;>=A zR9Umf4GCJ8o{IKB-he0?R6uATgr}G9Z$efFEsH4sctU9w88*8^2Q#0<7G_99j8$wa zI%p5K1ujGFu_(ognNQ5c;1#s@NO;74Wx$SWsTr2G+2LVe^VnF zV8`0C$dN&CyFZTgM3Y7=2BO)M zu6GR#Zi;KiUYlj>a6*SC;jN^?_%rj~FnpJ({BP1y+DB2>!~`GSQ87vgy{n#^{b0X{ zgRAjMKUqZ7PK&$hX}suc9lHIqW-M>}E8_NG%x8g7okE-d<@&$wi|J4~-Br_MpVi!Y zvDKaA<`JV{Y#mA!>(s$ON+|)D)U^b#&EX1^8a8^pIRg#i^OZWI8xlM`&z8aBm|IgK z=?v9LZlZ%)$Z+H=R22E=bfQ~P`ZmBI$K2?RNq$K0a=Bis5~sLO0r0K{arhzd^cR@ zZqkMVG8Q~(X#Uxz#l>2pr6%Ug#3hqugfUntsZgMliiX#Q`tYRYGr&C@(i+NFfYfKv z$0?sg+Ci-zm8-hO!Rv8__-$?)@*fo%QUbAo$-RoM3aarmI@V#RBL>y$hEj&;0hn#& z@y59!IZ~IDY5w(a`l1hC-AnD^`F68Y`+>;7<+91G7mOcC?WWS9KY}r}gk2&y3jkxs zTvQlf97na%bxa$>OD)}{4ZfGIx^Js@RCB{La{RX-(5*VP zwAU4roSvDrF}R_6jcmL|YFqN@VJt<1Q}6ii&|NpiVTL!Fn41yLMBB^8ly$wGosJOlL^&NfNt6#R%+U;IRRjK&JF?)_;(KI7)3dSFmKgXs8mM(l8SIP$8Sm z_Skq636R(Xrnm7_M*FsekXImi>DF1kpb!#?WO(eqk%=#FSRe<)e6f5a248Sr+?CVi zvVZe)jC0jbcYMC*Y0Mwys_qipK^79|L09R@6%v9+SMAM*RyRff#9$YEB}g*{7qtLy ze{39SY*OaY=9Xi2R!N%NRP5rE{ko;dSq8!iQZG&wIaf^KrnsJh4}bcC=lw`mcdfo5 z-RQzZC@J?+x`}~$kL3Nw>ch_V)*8p!?}6-#NeDsgX~E+dE&fG2!bMQK`&5#lM29Iw zWg{s`BQrr6aVW+q8X-AxcwGKJVHs$=gfU2kM1=Hwx3jz7@JT1R@tYyM@I;1K^4;Hr z))`CbvRGwoFWo{iwug4~n1@yVfwu8@d$pIjxkW8F9M#;VnIIGRC6Q9P&y_1_yjHc? zczzboG-n#EUCwd6<};Ub>6*Q9%wsoSv;~lK<;53kwcy0deCA;Bt%^`tE=4$N4b?1t zi0tQn!^z~Hs)MkJ$_}PvwOQXDWW6AHTF-U=0&Ql>1EFBFbi%ZJPoJOg0d z$TXX)srVYq=7dpG^7|L1rKJa{34nJR=l(>x3JpH>g@Sl5HSrPZpU4t=7%72(T)>gM z6*QO_iFC+dKYtFvlKil=1@5ta{L$R1Cr7k4Zalue*G?xPb#Ulpwo!^xfpH|{_Wlig z10-UTgM^Lsj4wM)h;Tv%IW@6LIW?)mTw#R0w7(l8i|0cVPDc*`ZHP_hXYmQBp8zR5 zRNxQg#hofaAjFU1%I_Wn=n5_7_~Tg|xu3Ju1xST+$W6Z?*+f?TF*1_S9Gm)-1n~`U z%%d(}a-7WJ;DE#JCGl5U=I~Jb%3dK@dSIp6{`zji zYOUECakf_c9>uk?QiTB!oL+>=_^za^w;qe-3Ual}Zu@7S#j?M8XxY9ufo*$=E zIA7Ee(PY{Ekx00H%THWkaTaPfbU>$x4u;1a1!mAHwHP0N-kq;7wrJgAy&?de@Ijzv zJ2?3F53j|t>zt*cY*I}C#xsnG@7C*B2$+G8Bl}CEU!!G$tZ-~j zg;qpgjVNdSt}F?Z+EIk6lta?A?A^M)>2knqMT;zx2Z<@K(rOL?gN=ku3!r4i*q?&4 z-fTm0GlYgB-&P7O_aYS#&!v=_ado;iCsSAmh*{ZZuD#C1XL-LSmE8mgUrU#Q}~)p+Hc@^ZKu82#zxL~ zh-kGz>QFAoUk2Wkf_8#8Al=)Nc?!aXju};J#3Q(AK<0Z1g807JZgvOa1!wZ1umNmY zQr9uSurdb7C$&v-xBDZ&$ZDzGS@zL-i-SE*ZZ*nsJ{|MOg}VH|f}`X{v1FRe8kK8m z0ee)jkai&uU%bPa9XnS4UoL2%fwStEu{~Msum)I#R${6!0h*SLHo*} z2MLUzG;!6NEJZtA zvxn*^)?gCK<^p;gNMt!1kdsRG9%hi%7u9zRPkL_pf)B_7WI634G^)6oy>>&|z0X%2 z-eI=l;iIP<_OIn=S~gA2rs0@uHn}pKQl4>fvM+1(6pJ9dDpKvQJ3OiQD?OH-4v^AZ z-j3DEzr8@Snk*pcc+yPdyZ5~BPvKhr2xBs8IdgHv(GyJllu!dWL&el)D!-8C_syEf zG(AG7O=E|qE%;+AC3@?t^hFD%-0tf)uiP#W{wiF?CD3Sb_$lQZ`wKIb>SRLc+$R=_ zQYdV;dTgUSUQ^G`d5)@L6fhrb4(dg0y>Z-=gbC=0^-8J%a|#~&NyO~Vrx3S)zbLJ< z*pZ|YI6a1t-oQxO+wC*BF;(i^H{js1pf20)6(u}9bgZ*K+B*zQXn1?)Y;j^bk!&tA z_K#nJpE;0?w%6qxFXtoGPsZg)sRYu!K@S{BK30u)4?4fDCzYd_Y$k0FYnThvA7C#n zbkh1uX@|byd|_|v>&lC(jEl8L1eRzvl%DrruTaEthW-%0+ea?Os*$O^HFQD55z~_c z=)j^rXN%NsLa(zzBJB;1LqY7@!fXC{4Bb>KhQay%WIT=DTt!r7sPDXAB>JHVB`t3;&5~J$8RUNAK0OuIt2{$Ba7| z3x14;>0XNpPR)?nSDwh7CAWMGOdlZ&O0D^Pjmk(+prF|s)YX%=57h$F8OfuCz?Tx0 zH~SR2tY)+8nq;c`{Y%A({>71cwM5Gh9!vTYP*3|+6|$Rns?7`q#oKUEy1oZWR!pX3 z{LuP-uB;={9q*>1$)g*O3X>Sv9OZRG^w>0WNnW!%NLZW{h%S4yM%Q4e{US1!eUebW zgXTI{S>}^V&CuA9OSzX42u(5Z2VR*I<~$^2x z=8!y|2!&li{obD^Ge`#FByjALFBdx+x-p4Q3zs^gJ+xI=V^2`PVnMiFXg_&CqI;2$ zs*l(vb9M#~{W?t7YUR@Icss+r+>{T+cK%hpnxe8dQHb+`N(697$5#N3>NVb-1hh z*T}7HX}X$#^r~A1+&QQ0fhY!rG*8M95921DWq%Ai?n%p46n3M>Xo`>s7BhB`?u-XU z#e!7F%BGh(pUi#rgDBQf^A1TlsC%>srl_OGr&AwY6xRaTELDLpV7(0ilyrP{^P> zhC>M=zwAtboe%#sT_lg^+a?$4SMAnr7BO9*HxwTdB-dyd$+CH#70wdn+bT{+1I-Q6 z@=jd8U16v(qKz-9e`dv~PsEL7El_H#)hX$f=iVydY5+sw9+( zI(9pdVG2PH`==r5>9kUQTh$sZO_vE)&i<+wc#~)~a(g_R#0hJdL_{I|%pKrRBvrjO zI)EGRdQLI&nkZBWp^|78H4Fy$21&;oGx#mg8@8tx` z&DO|_Mp*Xpv)5{#8AsgxEain%xz2;HtE>$qF{d!~_X8H(Xytts74Q9C^Iry5D~Y5a zQ|wI6<<5k_d1AqvJ7Tft-9@$j{u~TPJ(D5Xo6M0wZT3@>QPS{;+?#emxjVqfZ-e3@ zKHG~=itzBhpbRk(rEUj#;*mwlO5!W(#un#`4dra?v%S6KX~0lL*Aby(hYdL+fsbb| z_+L;ywY_w>qL|=VGXe@ys}q;YlH?QLxiYjb?~`;!$b4GG1d#@#M-S5kxb!}b-l*78 zE!2>Detv>3;T;m~--VO}?c|9aA{8>8v$nY7y#`e5Iwx(*C*@^7*Uw5)9}c!AbgN?K zF9olS`(h?$UQMzF)RIqxQ;00bGEePK*_zphUStS`+E!P2eoe7(A%>DC4lb@a9b+X| z^Ivg!dB33)H^5cXmj3miqvibQw`TxQVyfWAQ_T+79k2pdUgSuq#Ye9m3#YQ#TnI0P z?n4|-kLXSo)IlujLvwGBKI4ha^OR<4@nSR-)%xeMR{@u2;EnX*rr{{jjqBVf&4heK z*IQ$kIh+c)1!j#oK$W%#5keX+Eoc45h`$%F@tLEGUTh}NkTYYlZ%;pE;PaO}ilbHbNLRXG*8z`u&c`r5vE_mtlqpBy)0U*&$4ay7FzE*qRvNgUv z?aNs;Ne~(c9_RmEg?S-|-r$Z+*BQGRu3ml*-D#9O7Ar<3KyxI|T~h`(ML;zhn^M#y zi=|b$yKJdTH)IN|nVcmVsTT(^XN)uv8Q5qy#FQkP$)DblA}}qYr0nl2TNlzr^fR0s zDCYBoo_jJvZ*u1Vi;%OEM@p;YHQGO~13u#M#tl4M$KZTY5$zPL9PH$D`w~?Lw!RcQ zWvy}jwpW(O$t{$7TXA39agbl%gM}`ss0O0{O)ilCd||cJ(pvGvOcM|e5~>1ECz)pn z%X#<1cHTFZK;Fej&}bmaJCx8$LM<4Ni;i<;3vOO*xRfsHY<@t49VreDzghT45mvMk zCI!)4Y33q<;16#B?3=<)G4AChkJlo56JH0{bUCNp5_;qejJ*IjiN`bD{S8Nq@=5m& zJj|dmc25&qTe24)BnhnV8eGHiyfuRu-rvpT5{o{O9 zr4anRrp#b<8+disab&A7zJ29#tjHI-Dzr3lsi~4O5pJ^p7CE9l z+2AR~8`^uopu9Q=aT-`%CCAa4Np1=Q$VVCw7;km4oM%INHjKfC>P2S>O?nT zltj|#YmDCbAtB56QglUa8U$A;Y_C+Zh1V<&;C-14CfCtncN&mMN-IiY-_2hSKrW5( z$A^#r%ON_AH6Jy;nbIWDVG*Vjql$*n&r`b{vJ<~QAX9-J4;E%V;YNf)ZnIod?H|V zaO~XRg`gQQs@r>Ia$~U;BpjYyNxYg=sZSKzcb~g`Qj?`9t(cFZqN1jYZfc5SxjuNy zYX+%MA!%Du=@FeI6eN2?#O(xEiGSiouo&{X0#)0O7ZKug*15Th(cm$XR2c$Q;nNlR zhaJvQT78TO(d~o)(Umb$Wxhsrr$V`&nX>6^u(9eFepd}?!o|)YVh9qMPiksv713Ta z_dWnO97I&_9LHO=aNFGuEdq%}=LUzBMH6s_zHt%qk*e35^Emvl0~+aC!uBlHGAT+PpDR7NHA$&14924!DNQmpD+<><`cGNXU*u=t!@29q~%u4^afLWKg2}to2!k! zvp^p=)o_|Yy4bm$zkg1{)=W9B0U{{?{6|DCc$hG|7f&H#=-i+9Ipnw`T@U{wNwJ?U z?Eu%>1!ES2hBvl^aj?zuXH*4>jpOKc5dM!-63JXmJ`#MmnXA2}>NopjXS8dK=|!I6 z+A1#S4x|9Xu1?SRgIYxduk5d8#e-|Ydn+k(QO!`k2h|o_7SqC!6~*hFopC3vnOh8z zLm5Kna56?4OR1Y}i8pJU&G!Dvh9-lly@#Q)w#Tg&XzTw{gDv)~dr5CRh=ZIW&#tHD zeR|MA@b?@veke^#v<+T#(*CX}w>>%*97AC(A$nI2wqs;%>zEmBg997g)3UIo11gy) zCn}Rj_k*`4&2wQHsY)BV_1rTx#u`3N3RWG#J=n{?YktF4q?SRSt1|8pt5U@O3O7&~ z%1tOJaI~H+j|b`{02}}u<$n@sw1$iD(i< z6VP^*yPkj*vsJahFbW{^Fwl7B&W*hRmghS?w>}4(@9LCKA`tGw(+zWkGDr+>Tv~86 zb}wThkJ2?l^hV-_P#Luc9^98=7J;wIz4<(WKSROZ5rFl+zm73GZmT<0WtCF)J$r8y z23MoTZ?KYpL;`8T>&w5XHASjcIG4{S`a08oY3QKzIrY-RIz1RFdGtOu?{U^G_0IG)2d4|_T1oHy0^`z1lel*V)Irj{ zL{|Uvfx^;Q8()^z^+6PgWCy>{%1*MVVa0COLNlYL1z(2;g^d^jm$;B%6)EV;I9;hZ zg<~FO&*rz=AV!Y#b0U;*jd3oiB;~UE-Oj=8l}BT-CH&sO2O*;1Lb_4eP;V+bYkVIS z1y4^6LmUhVIR>GNs(WYQhHfZPwhn44r#yqBp_S5d9*leP8daWJ(gdBZiF`tB{#6KZ zVXC14pDhn(bU8`{9piN=ecj1>uV$Z)1VHq;eQr|m znLvEb5&9DY{6j7sS*9*jP#SN zg!rAifcd|bggWf|w5OR=cKl=t>fRnxlAF)f26)MB_J3WuJ#K_6YCFfIJD)YX3qkFi zyI%ju=g`J3|72b`O+9by=5)_Y;SYU5R@uv*U4W2GW1TeKK2#b>fy;2++A|x!X^QQ> z@mLma7{Tq-yEiKoOQl0Tl1WTjwPQyu8=9yS$l7RC)}bAr4Kk5mik_tmajCpRCP|Qd zeOZ0!GPJoUinP(=6ZBI-f&v^y?I%J&R4m>)MB)|=HkRjjx)~6Z0ZhzENLo-{So4d= zdHCeGc^vVB8v%AFJj81( z;*|z(b(^|UjqxQhbvxa6u4Kb3P*W|KhUR!JUFgC+MzvBQ@6k^KtjuX1B);MWG z1Vu4DNY^9bN%e%REMu*Ak2xsf?ZMyg=dC7-wh+cQNmk|Txf|V_vuYy)ebJ4%R$X3p zs0_)f15gw3>q-e{B$itc#fTnd86)Q>v-nlnDlAw_82!4nFt{_4L}!CMV&cns3I*Ex z+Q7D5T)4Rc&i}0va>XC(?{?3IYP3F3we+l_YNP7ap2N{r+FsBpVYj2oU3wd_0PVJ<;4RHoFtX^i78cU`!#eqv&!^fjU>AFt*A@_N%IWZp1iyxjb~z3e2{TmygYT+R)DS0#V2w}43J7~3+H)!0g1F%lXl zhDQ(*F`$NCc-SbJT|-hjESkw?DpPOBSACT1>?OK+<79P7MrG$RB%_e+aCX1MkXg3YV=D=ksulDhkIX2609+tyQcQ2J-s&rn@5km{NHbVSNq7lFuZmxB=c4sFov#g6%~L3s=K_rVj~(M3CBh4*PA~qZDw12Y z-yA%FFxq^!@Q}-Nu_DF){eA++n+!&{P6F5?n7GSj!ZWBmHPmUtl2MW3PFGUWK=rx= zM?FGhhm=wS$UQVF8QGUzWb9twhzBL6vaE1T4WHY2$xfFMvD&WSL+#lH46p5CV(+@S z&f1~zEtl*GL$VqK4Kh=(c2jA4Xkp{~vSVOQwNP4QJXQ8d1)?AL+!gc-PAA4+t6^{@ z+e%~1b2fLQ#cIu&qHEd*3wd;qQ&}txfStdNIf1V@dp6^!RAk*U{*8s9Z4R`sOjQu6 zn5-Sv`)P{6{db2V^#O#=6%49MnIOd?Qqk{(}TqIwR z59obA%#eV2WpMd)sZQcfIwq#8FBq}Ek%iA*r})ji9uXKfU>>p%t`D(rt??y#wk_fN(dmsp)V0o98x&{7u1P;$astZaSU7F)$6B2ob0)x#qHCCn%nc@ zx^ge2(-T3GWw7{8`1!N<0kU}9cX^@#sj__PIO%(?ZCaVJ5zQe8D|uTbN*N%S#4hb%mOBkg?JfV%lE#6wG7!W}lH;p62uB@A_5pq(zE<(v+U_ z;RfAvz_A?>+g|@;;N1xihXmZGnVUfHkRJr1@#2)w-OsCAkI=e0IGe45T~kwu1WNto zda72}+o1<=KFMqlsVs%BjPqywPKVMxbpQ%wy__$F`G4Qx5HbkcncVXg3v(QG(KV_kF}%ZekB7BP_vDY^9EYuA|ND<+QbwSUK#Qo-iGzKtGu0LW=n z&F0#I)3j~lQN(D%%|ug~Kbaj^?0K@nY1D2uE__x`r7gZ<&CRtH$($j!Z|l7G-UQEM zn|cZUKHspi9YAOIlZZU&ByN2-`ipI>ab;21WTX?vZc{f;((agA+XO{`ljvf=(>%R{U=KjrfQPpYjFzhHmQjt()+5*0vjJL zVBSz7az_CF+@{p)?dDUyxm(stGzRv7`oa0BbsDfA|MOYFMXvMoGba{Zds%+!Q?7q5-o1Gi*C%o+3ihR3dcp`5~#0lVA_Jz%*ECBxy3CbgyE8W5vI(Ey4CM zoWx4IbtiqFWaW%2x=kHvYC+{Wj*5VNx?Hmbd@h;IXl2sWvMSr{ndXVZUi%kr7JpV( zPY=>yFwcsgvQ9)@7gPi4Ijxv#7)tjuh^<|RpyC;8rO_J(leyA!mNI~bOW^O+=YCoj zw_04w%3>!(H7T|br!IQvlsVrOQ1V;muY*mLxin{A&GwKuDI{W1iTi}Fu`TYG&F!^g zBRuy3G=`>2`{wJOofi;AL5VeF49sP@5>Ew`*b_-1Om3B9vk^z#s0?K$#Hze8w!(_o z9D&h_vDdcef*Rxs`J+MpCrt+dLcM2USe)Lo$0Q&sZ_FQI=i`a>$#yjEFxP!78jCop z9r~%PwluAr#!C9MA{DC}TY~~8i`4-Jb_H~vf?lrn=F*4unf*QF(>-yAOo^)^juRb! zGLGG3u0)Lj;vMu4WpCVDn|=MSQLFFLQr#s{nJIPOULNHZK+;W*@HclD%P6BpljX-j zOU9Z1cNLfiF!m-9zQBXJJU0G$S8x1!xHeyHjt9QMXxglV(bO~>#|RbY3kJ2ajI3@P zTGty2as|@iquFCYa5ZS##a9^Im#>v{On6pZV108m(bC3Ky?k4T(*=YQ36g+eg5MvA zF~15yr5OW4VZzFMI=q35g27A)mE0r$55@V?($WHckR?zQ_X1LjU29&lAL^7MT>raJzo&f}i-#V%cPmze)Fr8KXOwKA^ta>v z|1He4dA8Kmq6oZms&1_WyI;j-4>_ONc2P6(-ApL-o{bpiiKpx5pkGSH^%ip>lfbxj z$N$Q4j;y9DIn=!HqVHE^?Ou*F*QUBBpL~U5sCO7LZpJL>JWu~|&bi`p|HNdqs=ax6 zmDj1YaoGiPe;-^_V@$m3$=q~i^>1`IblGZsc-Z4=x2qH1RiKOsL+ao^BZ)0uLSdm& zSxVXjGhppmX?2-iz^D;z;?_guPRrzpf;g73efHPcvWiW^uT!7OQlzTxjRlM3)yi1z zqm%i;U53>=VpmZ)xD+)h|13{FxPM->KUxQUL>K*PIX9C;8y{D)M86v%Cn`%+Fgt{O zGq%?kL*2Ica?}51E^f|$v8lboh6*|*RkC=_kLj|x(#XUv4gW>-miv9O&6wrRH4uE< zG%e)TU=Rzn$yWVeI&!TUL+~^X#@4=NSI}h}Df%9hP+#C}Iw}1QG;#s19{*#dUTfsZ ziJI@)k2prv)^P7{xh~27Tg*M6W2M(b^&BnZ&lL14H|}ftkrf!p(=m#~B^0EJ&>qU+ zJvmSXuQx|9qS+zIEyC?2*S4x3v*c|(U<($ZIGoC3pzej9 zYEob?AePBUqg1UPu=qG;c@dz75~!qhhmppJ0op|X@gu(ws*FGmOw1&+q>u#Rph6zv_XJX{Eb5C1fjm zbbXy6f`S>I1WBC0p-outX>-kC2#XlSUG{xpfggZ|1?yE z4DQ9#M)12biJ>^cY})9ESbQZzK<&E)cok`Vun@?2O0?PP63?k9M4f00yKb+2{jaa9n zx@xNQsZz&@&g0|Jt3DWiSX1q9Ot(3hD$yvUy|k3DDwc8Ke`TXMMALbuf!OI8a0Q0w z5%6yPazm6McDx&icjfF6s?o(rGG^}L>s!0yJpY8Y>5M*u^f$uVF6F&Ik{xcL=stK+qamhO=*`DP=lA&$l?bu7IFFbFg1 zmHc9NdDtY7;qtAyNR&)qy}3t(2eu?QAIR*@F8^JFBKE8sm!+i;=ffm`gX**3^E0&T z3R`z_H@LQp0ny^Lml8*tm^HZcT@N<=q$`X`d6Jkyre#sS=;YHe6rj9d3F7QsiJjSg@Ot^{gsaYl0Z8SkRFUhWNdg141e++pub@P*x z%DTu# z&Z+%?zvEXB-p$glT2k3$KL}Jh8L^_t5_FzYdg=NIR(3Ts4j;LKGLdy;v!qILQ%sH) zY;=KX1JOYxU#>rKI^~D!g7G2ZdMI>;v)}VAR=c}=NmRTpzq{cAr*^nmDUVB+)(=R~Pe# z1XJP3@U+`F>(u}b&xV&FSQ!RYXZ0h-9yGP6vxyJ@z1j)Y6O7cz*AiteI;I{{S zepwLN;qw*R0M%Yxt)brFC@u`}-S6*8cunQD@$Qdtq4Rk909qVu(ZRAj!g++nEe=68 zyf@{ZY@vCp%Z*o&?UW5O&NkcN9X5PaRIp8x*wN$}o0Bjzo{ty5>Kh)4iA%lbr)id! zA;mv0+G|@`>Yr?;jYy)?5%Ls0hi0pGM+LN-o|5NI(bSo>`Z%?}k|V#UEgB(9`(w(p z?>nviglg?cv%OuoV*R&p!8D_P)*gGP={rsZj7_=yR0shAfcD4nxMaLs&xFZLxx4Ax z%E~HuR&qE;cuxa}F2@uvDEwfC zkN{lRN4wpHpgM-{C2ddB7gR>$RjJUPqVW}uuHK>eyw&k>!}FQ@{KY^l#K`Q4)?nJs z_D;=PqH|o&WW9SEjrI6TvfH_yc>hgjI38Q**pXOFH+o=DgeXrCe|ki5Y=Ast=X&Cl zVvrDUN6^_jx4HVb{e=mR>3^?Q%ZmIFspX=VPD>QAE=MK#D@Y0)FqKg?HaBCmOPVCp z;*oK=G_KXg+Jl&+@+(C9R?hm z^xWqk$rHqN0JPZ0%pdd^K4h{ktdkO3|I93VO>-j)EQU`IL%*zZZ@p1u4)m({6G6l* zZrqMwub+d=6c>oZM5s|LV5!A#ZT<59|q~y^H z?2W=r^F*%YA!aEQ*|4v~zn#^KS&w#3#QODL6 ziA=akSC}FO846cB?y^R{o-Y7>T|oO@ZMwr3?Nt$28B3CiK$Q|4iAMArO&cqKY4+peVdygA5iF<#!>q|H& zFp(#L#o-(YMWqR&^3m>MZp=8{cYaXtRW_ zKGR%xj$p(_eKBahZ*{)=JOg1P{(`8ZmO;Ymet}-BA73PUMA6g-OgeALKlv{~6bR~M zid<2~^xz|#PFLzGe)68`QG!%wNL*@j@qGJf1dH=P$Kmepq(;Yw^5R)b14nBoio4W+ z-5dA0Ey<%hrCE=7GlniEq&+2Y#cL~6k=w6rTBQ5J2t%ZOQ{wkup_pF8#U@#<+R59%Ol9@bwxJlJ{)5j-0kD0J{|T)Cd_ z&jIUfW0qK-o)5+>q9|T>HEQfHS6Udv0Ai1b{~iQ5es|{ewmMzoyfL#m>pu@C9)Z+v z=$H|PH)A#CoON+SCSg9|?^0*$w|wcO*n_8D_5&r`r*CCx9wmeV=>hu8EMXL4u(g9Fo~u5L6ykGcU9CN8TEaiQgtAPh(LPIwA-pXBR4s1a4RjsZ;qL8I3v^N z+jTT`#O9BjFo6`#Z!pI(yF~tF5aM;>lBVB$(0E_Ub>V@CkL}ZC86+x<`2_@qo=_w4 zryIjGc88$_{0GGmXjC=`lLz!0wnd-+P7WXILa`(=RJP*UzqfI_(8fn$;yQfAF?9op zM0h$3h~HLP|7p@t)uUHO;e#RrD*k$mc-Qq`)==^1HKRj+HT0s6v;w4V9x847b?$Aj zcn434INuJcM&rWcHZw#bGb;!UlLJvh?av<{xWVJ=u$6Q~wvM0`&{)Hyy92yd1MKmv z_~AQ4kL;i<(A6b2X=%ZCaB%AnWYm{TE$qu#o_JA+aQWJVO-M7qv z4B_z0q#)hStjoIIb&&K7`K*rt3oKnJ-`DRI?BlZ{mIt#DgrXOVzB0{x<@4y*e}E&^ zSO{3<0w#4pbeJSd`(0pypHyWcWV#>{eIBesJjf{8Z$KTW89o4Wg<|uL)QE<#1zMxo zgjEojYsd+_I9TkRW@T(9P6?JVH&G~K( z*J%^UbHkqv9GC!6KE%#ST6t}h-M2d3g$UFnb=B;qx}b=@7rs-Y4B8iR=ngo=FIRJB zQb2@}iPFI5dm~&?h32N#wPHCPgaBbasG={`zRDgN9qOV!}gFBLnxs&vn6@BDu4Eh$XiF#4wu$@@{-sQ|ph=>${BHbcw+A^jV{%aFHO2psk$S zKG#cPrWm1Rcwa%ZKZdo$;7Sk!ZIBWTpn$*O*|Udhc&JwYnU8ci$`i0cYcZ|M0JH-E z!}n_*o4Yw~F8-S8_ja#R`g$7Lu;Itzk&||dF1LupdyoLZfc|(vj(O~ie=@B1u>Chf zi5pVq(#18jd$q4vtkdQZ0Yts{$f(rh^Yf;a3QTiNvQ4bA?OA*rg(Q%GLk^UbLI7|u zj^E!r%#2Q=^NY<+s}R7HWvK`E^p7$%M&SD|DhIG;YV-EDyq2(?p#*8RlOu^=g-21X zx^bSZ@6GgexBd%VhI*=bdEh`$Hj`15fceamon#ui<|XRDlQv9sNETsyFDU9Cyc zqw%R6RZs2`Bj5&nXc?|nj)}~@rnI6MZlYZV{e0<8e0y}epMzPO-Gg-}9kvlt z*RWf8qP$tevo>K5ob0$SueGE|N-BPm?yuA~`W+PPvCuo@>HN?=->Sf{ab+?$^Qqxo zAoKWG$(3X#&EkZ;)F34k0N~BJ84}&p_3eqF)Net;dmkCaw>h$iKsVI-+0$BOj zOShMzNp(1pmXtZYj>eBsalnexM6VcT;Q*pFYLJ-%L)2zhjl>Oudthe2do9iCEv(`&K8Dzl2H9{%5}zuw~QKe!Te>KSUp{GB$P zeDXs!cFmtU6k}!@=NCQmK)PjgfW8=oT$Ta?<;j+wnVC#$T=F2Olr!&$Ip1)Zn(>q) zDWB;orrsE~H4$2G|Eu=NReiQ)5o=$YKi+}A1_(EviMf7C3J)8HL)lv4&be4XEN31B z?s{rwX|C0MR<~Yx`|Nx}H}F8}yV!Ef=7Jv={SA^x zYcxFY7^`uc6PCv2{CB8Hk;GxoCc~JYJCKYmpZ7J-DARDiF0_>0{m)|R9D`w}#*-A9 zMBrPS$y7UxRKBIYz5sH;dyCZHCI+UbrYOO@UmI5#D3U3ocem=S?M|4OKO(oH)PETk zF~ow%i{1|jON?;;NhHUCPEBL~T_jl>TUObL^5oy0KRaNuCNt0gZRSHNZO-p?7_8-$ z)HLm$zb4LCH^_xsn+HhA{gDZQP-=3Oq7dRpC8MR7+&!ad@o~isY02y9u4VbwbhK2X z{W5<&kD}5ks%Xh&h?7e|EDc<6%&=03vIyWO59deE!BhNxbcQ;^d{HfgO3lxM;Y!3wqry znQM&#A&Vq_U4;`v*|0gzLy_p>Z&E2pk&8RIK0L2<`X1%>?XHsyXp31Ny!7wYuTz+m z42*xZPIuOsXz!Y6(>qs{FF|?&n-!apljV5$Qw|2NZbf$m2-2zM z3`#YZjMd)7eQ(x;L;iFh?Y$(0q@CPNpWQIk=B?v;eXKlCkg2gx2MiI45{mX)Ky5{n z{VsZlAo{>hl4|XwkX$yXXc`IEDfaz>>|x$~=t4oPc&2({z5~RBy!S1l8}dE0$8-7k zv_5$wx>^U1qdN4l9j#t)c}f5~`|<@itcgT*qx14&8f^&VPxJx^#YSMppL;%Je*8Vz zydOsyP-vq@7smrtwF*jbUnFiM@TY2DTO`;bKP^`Z3(joWnQ%nq&yYYMh&mafxG@>H z5ZPdxIHpUq$=XM@3JE=Tt43BBGpWBmO&2?9tKDLY?iZOQx43|@ora%@t^dQ9`fQsY zB6H-5c(A&(h(`T2|8Q8g#eu?<*+P7_PpWV&AJt>dJjbX`-}XD|GX3k}WD)Y{^L$8@ zh*)rw1=h(H5rO6=!Vc9X3`I&H62!J1>iqXjUc86p%Z|Em3uT@x+RIH}x4%_PoQ6%Q z3=U(IPczhl(U1lM2r^GpYEeyVzY@w*LL?#@`^3yXS?;scoE=<`=UqAAq^&jvOcU=O ztkzJgp0uAkQ3Ba^0Dn;zV(KrgG!_%2-WrgS)YgU}BSQlWaFpp_wpDWz#Yg{l+xt4Z zy3)HpSc7s!I}7D@A{*8e7{;Acv&adVG2{#jKEL zod@Ea7b!q>Qa_HIP6S`?P79TCXoIs?-X!8|uBL5!J&D7M{RC7d``LfQKf~m8U&z^` zGX+S0Z3Z!0A@G~!l5Iy-^#2?pjI`F1Nsst$HeP)=A_=MP*>Cf5aQCItN=Pg_K%ULU zo!h~@oWbZp{EXtX%DpLmjLCG`Yu2E3OTl1n5OKsbF6^fXz^zHh%ju_c?^4PzgE{6V zK?mg_Q0v`aaT5yo-Z4Gr%h0DS@7eTm=$DpmP5YW*X|RSoqV}{3{t5y>)<_cIkdR$H zJNK|?xrFmm*O1wPKgaP8rwS(jBW-|#tw%OAu;}%LMRNrM*o8j8N)^WSZdADS`yP`M zD$WshK~Swp41mw$6{n_(GhkAB(9q>r7@@2wfm+XP@2pa!W9>6e&<^>_>g}42%!0#J z2NV=SkDu~RY#Op8YuIxcU;L1jnhG`8+0NXIYEFvj+?C?f*vlw>p6^H#flzau#gLx-jBr&YWNV#<75T5Z-VDkw0R&tUE11-mH z3!GUr1khh&q~nJt^Fhe2%;*0`ib@)H$j`)q>e=guwzf90bQ)=>;MX6pI{>GkFQZ}_ zaGsJ{?1z~&t~Oe(fPn6tI|mB--?;VK(`NJgUdLX&lTViS zV?+;>bF+{~F7b|hoIG9ii_d6(syqQ|`>q+`w{y_k_9uqKG=ca!_%iW+25DKUf!R{J z$I*-`n==$Gw7@<`|6_TEwvf;gW!p|t=pux;0_Eqf!m~2{lP-#v}7 zDMga!z65O~LO=+_0uNy1uZv5VLD|q$t(yAYm}>WBj!~1_H8apu>@G#8n`f?Nw(o{a z=?xHP^iHIco%V~YwCwAY?p!DSef)P_;`Bhu32<4wLh3?MhGs_r)>rp(gH?FhdOx5% zOJ+0y|<%}$@aov~vg*k&_0O_h-?8Ay6ip3SSI^9 zQ4Z4?rvdskvD`>4);#k%uqd5$&VU+o&xxpE&?4dXw7a^+vk`lB+~C}A=439ZHp?dU zued_A;VtAw+5(bQxJ_8Lw7}^1-NB4S?_U*B4~)oUM-F+4Mf^Kbdaq95Rlq+M4QQSb zBqb#uR^fH{_*`d9frLm@L!C8==pFVa2su!rp5!7*_Z%4614una9vpCzTF2$g^gqiNkNr-;XeeZ3_Xsw^-@uUG^22%o|r3;3~BYCWm-h{JPSzC*>#rw^@t zEjdUVpKFu?Pe3|s3G(kgidv)WaT61ZY0gn)YXg8qev2(8#7m}Z!DrPRvZjk!I7y1i~uj?l-HwWdpzrAVRx7b@{T)wpJfKfl- z7>=b4zCL$t^ItecaXOeL&}feV)&%3l>OA-D1GmLy2a3<1KSwV;$nOuRwguQoeC!m$ z#GWASkLJ=XSDPoQTokL>Jh{&cw@~#aVu+v}$VB<&_4JUE2myCRwF$)H$P(RWW?e(T za-tZNsZWU~mo3=pf$7)revXK))ecR|X{Gydbjwj7W$_GrIz2~z(`CWXNR4y`5o!GrmXR#d@W=?Yp0P{}9-y4fRHv%4~rT z9B7P|D@g!}1Wo;$4^JpkNWJkSJ`SfdFF=pfKmYaLWDe+k_<$DMPk%6G&}((&Ww)cl zcYpk=XbhqMN3lYR6fz;p7+ph(0nylV=AaDD&(})7Y9faht#fFm5?dyq93t*C#2Cri zx;L;d80@XAGF|?;4Wk}_)(v=?Vc<}n^PPWpKxf?ib%nA-a&owwlmVjwtZ#C%fSi|3 zn;Qo6Y566~DLfO33$8KzMArZhC;fgZFL&G%L!DH}86a^k#id$~O+YDyxZqKrCYb%oYBt8b#y?3iDkua$(-gJTl7&Owp!`oYnZPG1YGZF(*az(6Uzc4R!TD5 z`C8O^O_asKg24*1zi*Fb^~chZ0ma=%=7e4((2a>@aIyjVy58B@cwjrHV~JBl`F4(Y z(7DPbp)ym!^?IzB<mzmMZSzo2Y^$tIhnp#J+O}dol1?xI)44_v2lz0`%$W-mVAr4n{r!W1T3n;Tn z0`%BNR|BmGq>c(7Z{pOauG3papErEF&KDgXA1l_~^cH-*|IDwj;7r%qIfX4lvpUx}!g!EHJHcPIDKHn!!pA7w$N0m3<@dAd z;L!>pG*@iudf)r%kHSxnBmiFsynf9Sn=a4PH~|gvI%i>aXA`bi>;J;+7pI2A(Ce%; zT(0g=x#1izj;J{XhPDF_W%0|i_{6f+rd3>Daa_;wyRT`mw4`ANd#z`Ks#g!#pRSa4L?)@0DU% zpL;OdTy(4F&1Ie;YS8d@MU&Z6@}-8Nr?kTNMh)uk3i~UYP;-=NAz6oghFpr2>#5vu zU$~mq1hf7dkoJK6R~kFP0FxZwl9GxlBE!oI@Sb&TRE#%b;3ez9AFL5Y*a7PZiMT(r zG8jj%6Tsi{auN(cC;^L5A>*xh4%!dY9>g;U374lwAeQhcQ@tQlHb93f&>w`c7__ks zV|BSEq1B>t9X%8(C(MLfVAs=hm9i9qtn>M6#IfjhkzTdGPg^BH%H#Frd1GI;kTN)y z@F59-SnAK?z4LE~_TPYf7|^CB0!s+%y%BW165x}vvA5TMdAQ_sx$^t`_4{u#%&M)4 z^~mw2!!Vp?L^v3gyo%vJ=|7Sf4L;Q!Ddqegq(>^++8;Ed|M%HRG|dwL?9^ zgGOh6k2=w|>wnDQ^Hx1v^Z&dqNxUC(!Ih?K+q=>N8!AIKC#|8jFF=xL%sXWE4+g~x z8PFB}JkZ2qOu+yAiNq;KODFV7@Dvz~5#QRDvHmPqIH=L^ZSl07Q>Dh7i8Ntx%35@7 zg~QADJrHvs#`tMt2jXqHEfk0wf35X+L31Cob#jWV<>`UNrr(RAw+1XoG*7^Pw&lqy zJD_CE`Q7OQP$ZsOa0!0)Fv7yZ+79FrVu)q$@%ZlHsP|EBi~YfIh%voUdFC31+V}U1DSfSO1Vd53!aJr

1{S5xrcS^WO29#4!w80kI$ zDFdRnyR(lQoG;!8nDdE|L|^tv2yo{c^5xb&z*2Uyx;JlUL``*19x>E1#>!<)1mL=jy7B=vQ`m!fB7q2no~d3Iw~ehn$0n=&Fn%`An%GrEh9@lA899zD-2e0hCA0Hy zTzhzVh9;BKy`BR{!0PAA*U+oxq{sNuU+(j)y=X^Ls3xR`p8A{x*00B$YyT(pj&Q7* zjBCn_rIx(tEyJ%wI$K2>F4kPBGC)Gab08fBpe$_7_f1BaAywd5%3?DfM;g9L#uILU z{aZIuNM_d{6YL9b!PTQ%AO^Q&K|lLC=N9C&@$x=T?&l zZU?_a2+{b3)$^wjFr^e9H9E-Z%R8i2CD^Qwl2;}OqlIfk+6Ttv&30*lKmS$(q2>D# zRYC^L;Op$kjGoIEy3!rA&}eV!&S#T1bw}_LdNhll!1MM+V|&Hc{`k%l6!iMtv&YD~ zPZI|@B`IVGFd1aElLS-qq5SESRcZDnT$}eGv~&kE9lu85Sk|ggJ+`cN^q>Mx4CWb# z%>x|u>~BeO1noXDY18i8H;m9Z4Cj0Z1maup7P!Cidz5UH^bB&@o~p=)7T9Ob!B2vt zpT1ty^M~w~q%jDI*6xo{!ff+fs;G3Ye{^wi0V<`d4P~H)0%`ihTF0AWsN^M#M+x}6 z!~x$6BP-lXKENT6%k~6}!LKoi$ScC3P$PNNZgouQLq5l+NlfP@fG=L&IXk5Oa1v~;l%!$jh zyI1#j>6nelflePio^5Zclv0dxwCZy8dt!tb$b~8-^Hr#)+_m=xWcdG8PX_ZP0Q48* z*)r8h1_4I>y;2OoH{L%g)tf|{+1y+YJ1#>S!k2%`Vs-B&;FR5$BS^qCD;*y8L2JP! zPwvtDhvzn)&t>-2)_u-WMDa${iEEp@no;5D;i(Z6&Sf@`f3G|$h}Xis)fyinD@cN| z!gF-#Vu$07iHcA>G_t*6%NLITe|^U~!7mZMiIzCEZjDu|fq6Xqc?_pDNj0b{P$ue= z(hwRld;yt+tm!u9#J)guQ;qv5F{c(WCf-?;w8`}Rp#E}CUVYQx$hq3dV=#|9TB>f` zrlz>@P&%t6*+I|Ye8mbvU@3sN*k;#vt!lN>{W-sVK$?`{_2GNDY>_j!_P?>ktu4l# ziD#17)i3&!Z!S)i;p9x<_QLhs-@J15;B<+832+L$uEZ3@|P zll<{?IO^Vj{@WRvPfa*OxiufLxZSI2HbQ!$X0cFlk988+B3l_Ougy$uoN{FPU5>pE z4O0S8`;6US;!5}@y~bcz^1Vzndsh$f20zVZ8y>BX;c&-Mo|H~^oB(NZFcNsnhF^}?xS*+#%Lr)AohgQ+n(T|0L=uVU2Q4dc5gtu>jgS5_D)`^x ztq6g90|zh9AK>w#)>lR1?pSOyae$PLBqCv`11bVeh9+a0Bo62z%qHf=4AwBrg|0a< zbtq(9y6dYgLwm%*&35}ta=c!QV-!O=P5g!~*168ZLm~u<2aT|p8wvIH(o=1ny}xh5 z)jI9q4tT%Hn?BH#7DcM5A29z-{Q0ZZ09*QIorkkJ&iP8!ZYwGQhY}Xrver3EEV{Mi z-pz)31%1MBWuKVmF~5t@lh=bLOD=LOX&^E(#UyuY%wA~(86k21BLaN6;jbs~W6r0r z1ll6zY!Q=O=?dH;(D6rA(i|)|UY@QfcgH2?O}2HxvIu0#v+47#UiYUvMgZ^wjI*8t zAg&`935Ur@F3<_5|8*XU?x*j@u*8v1P?>ANHy!4$2O~CP7wFQ}QF+O)B9bhtR^bi^ zX>x9l;#J*XFzE~j@OjOs)G+S{z-HE)G>c&}9*H5BgMawzEOHAo$4^vi{S(4;@&SiT z%s-OWFQrQw?^qoe)qb$X{L=aKt)COMzjyJ;#M2+TGJV_;TiSzD@vzLeb%}4ICGm6844*&r@S8} zl591HH%kXOZPb#`v82pJXF6`K|FJW^@>hs&TZcu*Ul~5cQaogJhev6Y z%k3keYkF#$$q|B3C&6Yi3ZcOUc6!QyT@;mWkWwM^HJ|BX%kj3W64KSddfLtE`LDd{ zaj{q?SwfA7Eijy`Kxii&f{UZUwN=46a#lz!`J@PW&uniS1lyr)?_B% zT&v@Fa@n~GD7+CQnpfoeDwaT{`~wJ)z|sU~6cC>vs?doRC2tEi_=Kh+Q5#dIbvZ! zk&Fv;j5JUvHt;OX3`Q?=Q-Bqq$u5N}mWx_TDpN>W3|58eY%#iyh<21ZQ|OjKbHYxN z(!m3TI0NH*CV{gPh)8P0ccvy`{*lbGs=FQ?_}U_p8cfTfx&l^6*J`n-A@is8gkm3c z|M*@i5IO;DfigyLXg5fF+J|Q_kQu2>)Yz-Jl8&wE+=(Iin823GbfM0^7F-re`6Cna7Qm5GOFVi6Yl!;?wcKS}n zo(B&RKU%Vq9XJ+Do%o_TdCQ1-xoNg*I`J(JlAMmehS=5KSD{kLFY6l`Hj5suUp=jl z7K2l-S^ka09jhVNFUv4@Yj=gMV_#+GnE*Zb1UHQ85ifvB_3j+s00R6C7R`OwR@EF;x)w`1e`xJ+TXU;3Ck{HzURRB$8ztV5mM^T~ zej}-JACUN{_}#tiiv_3&dqAw_RE4Ih$6v33$~F9f`Lo^yRjKDs3vQ3|3DzqvKmvA= zRviIY#JDMb^B~`@#q)GDUO5#_D;yDW-3%|tc%QF_oRHVfmz2p65k@yv+yN5*z z_Ze2pl~(uUWN4i!y?mpUx?$C(>t|9#gkeq8Vx<45sqe@Wq!0aD_=`ZnS4NL(%to8F zkw6%e-4X9ZT+T^GgI$16>N_xN9&Nt5i!BQ^IaRY`IFeXqvUp0RUT4C3DsavX1vL2? zp!mL7?QbX_&{g(4Hz4c%%Y23Ar6dq=!^r-6a~>Qs^9S%x8O;_eT-I)$5mHcu0ML7_ z({F<)T+Y5CIX*SDlsF<0CZM{KD8OqMPohb+x9rZ^Y#r2Gzq>aqQBnL{6Tiyl`{MOK z*(k5evVckXA0wXKYXFEq>kR&n;cg`@13Q>G^q&0b47T&TRMpzv9^29n3A{mBofoWp z$tWOo$iDB%8n_m!CMv|{>!-EPpR+$x>FF5&iTYgifnREu!8*;Fu09(Y65)i?b>p3r zF?c+`$jr0niL29$B`_D$iFHpKAShz%_;~EbWWWo5d1a;U1V}8B3q%@Qag)>0(S<}u z2dAd0H&Tn@yFY@+`TL>yVt9>}TlC!^$GL*oBbR$)cQ^h_t`~7vG;Dk(uJE`+?Y@9w zn4kRuu82~Z>?JccO?b$3k1XGek)*8|QQML+{}J$xorK-4_CKa?<_S-qCG)^VTA(8dLK5D8 z)Lnp8VQZ|eX9M30g(M8vbxoilq9pDf$|80bVJXpqFNrA3eY?|_tHR`}dhd_3)fzjx zOR1BU2qYniZ=Mknp>OtXnh0#ErZDh0B5>!fYVNgbu(rOhi6VJmh@L0P~V z#<4OvrVU2%>4hH()?h_wWGOG@JO0WGI)Em+JzP_!u~|a{p^=sB%==eU>+6};1Uznt zydKwjE1dTB_5xmBJY!>Hi;u%2u}uW$EEFl$7}Lu7ucW)0U;|Py6tQ(Dr=->j>$54= zui|C+d?I#Upb@OB^4+rmN2DZYe1m(SD(cJxMgw(sf81kGPV~dm-GxP$-@KOlOEdR))EIQ+%_YbdVdd8X%anu@X zNnkMzHN7ncl0a7@RIglvMHgNvnB&Dp+KsnKQ@Hf3J#0NQZ9`D>`6)XTJ<}XQY6W}K zBE>Je{MI7$+lts-QK$_;jPtu^JH<7KN@R;r4ynY=5K%@({pny?}H>t)5gHh1_4_pQ;}py_}-t{vsPSBIMw&BiE=` zYGbQd?y`TR-n}k2%(4;fOrk}5Y!s~GOh!!lJ*M@kGw)YomGhB}+7Wu$;jk>IACIo7 zM}#|czav7#x5}Vbp_2D*JXVbnx*<>u`J5Wua)>=?}om7&f z`H?inI=Ib6>^5n9sj;exo`#lo$wdXICIQ9H&JI8)2$hwUTJ(gkbPx~_Jl+qeK&G_7 zx;o~CbgV0PGh5rqMg)AG5P;5Eeewg)pg!~Q;hUJ496tjybI(c=wv)em>%`5?4b=9n zJ}IxaUO_jV%oG7m->uWriJKQKe0+SZBh%3Cyk73jZE8@*uqur?TRq-i;o;;r-b zE+Z`h|JNjO%1_-)7xjpxc8DnBh6FtG5WbAwgbrrAVck9*6?Pghl#^MjSPgKoW9+=3 zK<%PU2XW+IA3wq<9_Gt(>6I)Y^!-&{(Ipi@i+$S#Gjc+2|Ge!1VangnzG|&*)GjCMk!xWbPHQOY z<3pbS3JQjv z3PptKk30rb)smD|YAfV|{|p}$7E$I|Z?g}xBC<$C-}&U-^>0NBx3%7&3mCgV8B|8$ zkh?e&ZVX+&+}O)Y#7lobT*mNYW)1p7>)xAEDHXj*ETC5kG2olC%l>Y(FRZDcnNc50 zmybSAJcDEwVez7#><3|lf+kq=pkKKh_jY&>*Wf5(r+&wiBB2TOIr^Kw)QGTB$QtMq z#H^4A+P@`Vf=uOYxTtyL*pv_f@f-&?4J2Tx5*C}KD;?J+_xwZpXs}Y_d}vU>L$*Mp zj!vrVhUlKrmr#>_L8NlgdhVp)*n@BNTNi)q?uT%HV+pb9whv}912S*-!v(e7GYoTv zELx-Xd~d%ydTB9Z6)FN*B*xbKuPe-|#}9*9ViY?1d^Blv0}aVGYSXJfJbA-=FFMJO zXKTbBA#xb5)|fUh(YUXL0hQSi?U`#%Fo%V>9kX z2E@xdv5LLC;t)7^yulW#Lbl@V|AyNH{dVBNoW%@WgFx8cd|@>!xxupC=m-Qd7Z;xJ zb;-iXMe#b~2>#v5FnEM0p3n04wtDp`KgqI(#wP%`gzx(y0>fZEhAFLC4H=GJFtGY^ zqtK|sFRjS}!Ap%viNeUaS%Q@vV~_d-nsT8@MWJ29R?i!yxB9K68NDk4897%xeMxP# z3A8j?CuDj~8nua1?0QribKMks!lR!W{+K#fY#~fi~f3z)&qR~k8=uV$63r)=ktud9CG*e3xR8o z9#<6A-yMog*>r;GCggK-zeRST)eME{7*?EhAO=}qU^5h6X~b+kCRwi?{RO43)d7Q! z#})2f$DNn2!5xlszxJ*>$VN(`PY=B-%Q3z`Xb=xD==$$J zFa{JxPPHp^>pBUNf9#y0z`e8m_$g^lHL7O|CwpR1=FZkx_0ZP~w9`*!>Zh&lwQ(Ls z+Tw4jTa=V+kD_(0PND}V;jL_lb6NbQIKfX6Uy+Sj=ru3naL!ASSv0DWvgf!G>`nwIplBmC>G9z^kC%*5y#Aupa`%fqv)}`Ow{f8J9iL zpTn2jSdnedl!a<0Dy8C(!ckj)tY1XYj@^A0XbL$7C}T^FXv{@&gVzZufq1tRyCu7(@yk%yWZ@h2!6^<8ai=9Yvfy{83hdZcVCY^ubF6vCh7MLQ}qN9oV-$XzkV5QxmJ4&u(d_K`YDznY{6-sNubOr;QW?Ft;qBPD^929&PO z=EGcB^TS+o(4$r@xvfzMTzG)PDjEWKX=K;>l^8>3@J9#5TyZ@miHo}vJ)QbbY4bGs zZgaz(es2ofR$Nieg&PkhDuc<&`vJMTfqd%#QCMX!5}@^s$hy!_gSztDwe6t4$x>ws znqWx=?#VTt@d^My*w5Y&0g+L=(!W8|As# z?4)pc{dl9_68`u*yX)nuA&LV&TDKAVIw<&eZJ2EwP5j}{DU1F>?B0vKip9&>3qj0w z?e%!iqZalVySi^y#*1C)aNrSYgtT-s`nyECR~7$Qsdwr|-WCxr^sAD5U(ObON7tRK zNCRmHKC8DNJeYA=Ak}Tle0HF`4g<{_g0WBtW%lcBS0J;g-M*#$7Uyi)LU09H{9O!l z3m!UuI8PYMH6I`m!V07m4e!}7z>FF9XaZR{)%!l`+T)HdR1Z#xL>2du74@A1#mouE zZov1=*!9Kdp?=UnUgPMO8#cCHyr1vXz;W#tk-R{u3WMK&+{LgfGp|!8tWU--}xF_|iQUy0m zlKZ0EzDmmBgA0dBVFf33@3`uQR&cysCWwgw9h<`mZ8j8(x!X^7@N#qhMLw6wnJ`%@ zwhygag0iVg)!%qe!)&&AS~ZuDO5M_wD7H#%8W!;=j)7^hK+*Q_&Waia`azUmyHxa z6csB0paa+oFW@z*u7)dh3~Zs@@%P*3i!NsK<*7u=dix@U;&oq-{hbes4{fKNu}K+(`|@f3NLl@+&BRBmE{MBb zaLnvqicL@zRn}_zBf(+1*K~*H#?_fnDo1F2yD2l z1QDYIw!IT=MeJ`tkxZd#sLAfHI>STQxF zR~BDTr&8D?{kmOVHj6@Txptw4$k-KIF(WQc#*fYGxy1l1+6v}5SY!?MUZ9Rzjhnqo z%?JN7=FvOmJBG5IkU;g*!9HZ^6P2j7ctpUiy$n`FoB5y9C{c?{N&-^C(C>VD5Q0w& zOt^?Hi!K_*q{;glPQXhRD%v*T`i0^L-CaHp`gs5%^!JbhZbpwq4<0#rvWQ)Y(?wPr zupk72OuB(npA>VWNE#N@5qn85v%});wF05V4sTC<1047kw;J%KM5B<Qi(L^Pa6ob4_=AWF@KjuPS=(iU7@F@w^{ zp5K5J$Hi*XpQIZKgrkex@X@{xM)NMM92}_%tWgagWfhPDO4W{9qY+e+7i*IK!Ww<5 zC^{7MVxIY9)ZrrIPqSbPm1V8k5QvOz)+x5p=aj>5^UBb5p2$5{fBzj{KeVgqoFwd+UF(Czr6o*aKQW^XDlZ*P5b^I_}pa+C2iAU0p9 z-Q#BS&!3SSi=87EL+>Y(3j9I~d|Ngal-0b#0oX&$-nFtLJhXE>O z=Kwz(;^jj(J~%B}nU`3S06b^S7Tf`|-y4VG_# zwvguk-Vuva^V-Ob-*#z=@}-olK)yxzJ8Yi$ZpgKVU$X8tnl(PO^Lj)$OuQIv=!KgE z#`V$_`qvJ|R@?T7t=bz*4i|c!6n){aK@;7dBz~tnCP(NO&eVo~aw;fX3lTi5T!WSL z0=pmOa_`g8IFNPWcz@3KCL3EbJnfQ0(w1f$ld|%3IzSMhH5KT1a73NfcRvK7*wID> zBl{z4%4Zyzt=K|5?G7BanlR;>%JmmeU`1fl5A;XkWtacWcjN4(H=O?l=hhAD94EYW zHHhfaz@+RiSj{qG$Co+h@}0D`QY)CRF?Ep2m~0?})0**)-?ra2;zQ%dzngstRO%59 zM7~U#nwph2jtql?$e~~=Dk>C;g|I7PzyWmm{qf~{h*aF>%Z+6UXEi?{bcgeFV`(0A z4EjwE$kE_XF??UG0pP83-&PjgnBE%s!<2b7B>N980nHYqIQj(!*I8`g2Ayvr-qeD( z+zz4=r#RC&N1|@$$5VNA_)&NsA3{}#%Ugb;@A2IWjut|Kvfl~>!^wSCrVRg19o#Lz z>fK23dr3!n)rDX?!6^0v-^!=lxl+%VbEjz^f9fPQ6&{Zo1o#NdZz4@kgo=z}J5bcs z)wZOE%+b2&p{5Y%dY2`PfkcnzfhVjn} zt|a%s0E!tCN*DUlFo>O>h7WU-VGEHE{-rxL#eZZXbY3x$JHQ`Kc)^1WPZ%HCFxMJ& zzGO&cEE>gzX}d4__<-DUkg*O1E3=dTNVWDC@?O@z-!U^wzzalK2{*7F?LV)j2@gUG z;AnVc65L%j>o{hbZ>0So?tifz#XU2&?7zFCPNQZI)y)J_z9QY|N(%f24uzZsq{gM3Ez@^IN zu>FuHoq~yG1EDNqh9nF%f6J!%>MLiBjGj@zCiTHZk__sFo|ZMN%D5%CqcTc;CL6T# zP3qAw(uEzh%WAZ$?60^%VmdD5?*cV3SCM)KaSwfNR7urriaf3z=roq_1II`WIfM*B z{{DE-$(51l4dLIb)?h7dH@f?KE#*o7e(XVY9sgD07t|2BxDrFo7x3#G>97{0uUJKIMoQI_UP1Tyd8i&r7;FwSvjW}i zt^!@kQU`HsgJt-zDd<<5;HPIZz!1IfI-0Tr8|Wuyp+H=D_( zJA45sBHX9L!A6<6xv4yH!n7z+AE-jJKW*+;i+_P&BYi!+u12U~+xq(Y?1~JYLb5tv z8>84hf{80J{Rait>W})_@GG+6n=Dmm(B8jXk!v{k9kF{JtG?{~%$s(bma#FK7G*I_ z&P)c1LhKC@g-!ugCR_rvgB~S3T;Y#oJ5{VyqiC@8?LY3MMf4VkLj@G#0ge@eSrw_P zC5%Vt^k5Ey=XgYyG;NJ#HarH;8e$7{+7Qs?VMC+*yK3hTT$LRNNNx<4P3Q-vsC}9G zd3ImsLDCzeg8X$H3?!hdynCmeFYwG3F#THa@)%vWLBNKZgEHRxX<@f+U?bC3jUcd?M5HV)N>jF?!cZN|}A0IvXH%Pn)f$VPaH;_mfv z;(K3R#EL(s#n!;gEv;q$UbhW&!EMt*Kuc&J04lPo;H)Vbz(U8Q~w_WMfBl^gk2|H3rb zj${}8NnYO}Wh%Wt{oZEBZ1~m(I>zyzMd8E@ES#P10)vPsHevXDuY-z`>HQ28rBqEL zA#&2l>KmNpbG=?7?2g-eH&k~BM)v?EayE07E^K)eQX?tpxJHEW4HvL25 zAav`j5yR2cs!ztZ5E}^rJe{`xg7rl22O&!QwGT|IlUg`CGN0As*+6KCvf~dybPq3B z>`6OOmH1_M4P}O%S-R`Sk(I+-%&cHHOucU-b5mA}sUV111lDZLL8_Di&;3Mu_EVXpM=e3DU#T< za`O~Mj!mIYkyW^6WPHl>>8sUR$0x;4uEOn?^9)LIe36MlL}5Zoxun<)C6NyOa7zm8 zX$9PQo^X5O%pFnc`@%><4HRk4B@oR0@!MFlLeVZR$C4(JRMnPKANqY}#_VDzjmx#G z;hGkOW4F~dT4_zBturVUu4%7FSMrJ(-18Nvax(F_CNm=|-K9|(oyyCy_2DE@pDo`- z%m-%h$kea_%;j1vEjpe2HxbjF#Z@a#l4#KHUlf8Ap3`&CtD%d5zZHAj$Q4LCRjY>7 zC^IxENE|-Yxt)cpo*<1L#l3P%+%_#=Px-g~z=H`4KlR&$<*7iaG64)do#;oGKPjjL zE?fp(f0JX1=emA`mrY4aMRk>>*X)mw525`$zY{dWOct)gUy~n~POTPy*bZY&Z=Z6Q z__7%IV<2Av0?AxI9GO%qN7@}isZ7$~e|~(YUgLsXYpE7W3?KYUoswq+)(U(-Yn+{_KD#vE8A3yCZ}mnA0d$Q! zIhKJx9}$B_+6_y6`T5-xtr}S2veq`$7BQWz&UbGeHoG6JpbgCnB*f-1)NB=+KYr+I%;stb$bKa~vUj`C$~ZkLrSE6ogj2y}8f@8Uv7(~1 zhb%0GUQ-gAQf}VZfpfz1q1yV*8Mh0cg$$r0h=6;Igg_o8|D|N+@;=}RZ^HBb3zaIo z4!yZq3KNy8oHm@v@<e% z>ad5lj!Alue9Os$7{5#Ze?D^`aKM_2lM@#XJ?;zw5c3!>YC1|NRWGk4HpS+S;!KBY zd6ks8sbkp0yr>k?zZvcPHSdFiDCooX{v=5>je!NaJ*zzwVJC>9tyyn8hE=)m6Hh@# zdW!EYAu{RLzb+}lh*?=zgVi;KpCXz)M590YSMRRx(Acv7R7>8!@Qw;KwhexV>SLlQeo^1|LDpv-)u9El7*on%h4n8`?Qe2yw8d_zItDb9Z);Q z2LVUOdy$F=i@BgYF4eqJNK7X2r-{bbV%)4?&5n8&mG2zK1BlLh+9kbF)m||#m95Iq0Xk1 zlw05G3}*xj%BBbF{4f;H=yUPL@$$lXz@xrG_Sy(UZUMXrv> z?iZ#^MMZY%_{`Jp29xDH4ckS$YJCgQBQKeLW)X_{!JQkdZ8nr84K_=Jsemw3aHD#I%`2DDy?RQAXuy(?Gb!M#B@=#H#Wj6-P^8>e%rOOG^E3U z5|D(Qc#1g1%J!Cx;}I~IAt8DzORho%4YI@;in`1BSz#3?~nLGGE6 z@vYPDd@OzGXx`97m@0F~*DIrPFaH#XjkzKM}BH)2L)W|bb8YeQrs9gAr1 zf6!A@N$5#gmCu9)ux1>Rzc~zuYK6!AkV$0%PfJT%El>ioS&9NNfi3?n={LWD7m0PP zxb$1Y9MK;WsnMCBj}GlBSTe1@uU)!Qj_yiHx^@R&%$-;Gj)U?M1Ohu+s_RJJIM!0I z1E|W3N-P$&OIKi!v~WP}Q}u6G=jDL_%cXjA7DeH0xtQvUBDX^WV$yU&i}fS{I4r?T-e)v!i}_XGf;$tzR)JlJuo^1OKkCt zHH=fHE9S!=vjkrKNl4N9gc!!e5Cr$o+ZFRRrW1^i{e#&lmStNiw(DW!hqos)OB9r5 zzeHOHlZV;eN_|)%+qHy7rTF$1Wt{vCi`nGM7uNUA=Dy+Qm#Ni=drB;&0No{P$65^x z5GYG*%(T*qB7xSgR7tI^Q;1aLP1>)A!D=-QSl)siCO|~)l=9OHDzyqM5TktH@#e6( ztlDw+^c)2;?u_dI=&(YRRLQ=|B{cRJyAlw|YqSEe;Od-@RA6z~U@fDw0=x@Grc@OJ zg#=J+EIn8)=L6EyasJGeL8z*#0%R@UWMG43@_B~=WwLvgmQs(#BkpQ)nU}F8hQL#Ar zlEg`>|A)c%wlC#K$iq!+*ZsI?H3WqyzQBdgCMM6kagu&Nj~PwtB&WOesz&{xEUX%w zM;r;`IY$nY<%X{K@d-T4FZ=N`-!=lN{l$X&_6gRq^yi4XyXJZ7)XkaqXPp$KNGOtF#h}Zze|#`=#f9T5l$Z9 zKImSUs#H~rjBIFXyd8t>S&1|T(Bt&Y&Iktm8#$%lqL%|{7STpJ5}$s_CREiSJ**xQ zb5iX*Ym$SR8@+H(N=ArBc)pCJi4+R*{V^{!zQ-2;xP>1}ws|=i7-Fp7=uemI??f`q z;o;oVs9r|95tha(8+}K}Iu6Ey_W>w3ns(lpY~J1(ZInV|5Y8!28lCh%lR;mobb@sTE3hH9fa?@ z_!OIn!ms*4bL75$G65k?;sa2Si=6YD=r3_$d}9?6Im6Dc7M;ZE34;VMfbs$Zqk`US zA`b`%AA-V=>qt2tD@{ca$Z!^^m{(c0PoJz#NCr6NY{ktfy#e9OA49SWUJ%~~D4|>N>*-ptzbMMAJ{EWf-jj_t>4RYH~gw{mfy|T75Rr$>F17WzoCFT$w z-Py{2ebmMQ5BEZd##{%*I%#$M<;6IsnWYFLt;s4-b_b#NtPJ7$?2XEHnd&r51d#Y3 z@3ku*9|UNW@j;icv7Z=!wJQaJsvS{x_QS}C_wl}derL2jLE2~1arhYUqsuG>EkHSl zM8hIYNKF+ek~@tUU#EG%X2l~I27>FG7y3n;9vCinH0jFW?j|jwaPurg=|Bl*|POevd!{MZ3GBof6mG3jPYL6O}o{@u}zTk)q66S;-h2QvfRh<=% zTZc+a*i>x$%rBCe+;CW8(#n#(Fs!#oRqlL@S@;6lL~i>iihb*=8ajPcIRts=*8PSS z4aXTm*=0fYUlRoM6CZG-rki&UA|L^?XHhzLQIrLTs?e~rJ~YvfEPsi(%O~-Q2pe%G zF=nQUG37+0=i}=gkCt0(zOjel2B+mAWPDQ+XADF!7^9=#bS$Ln(&(->ALE#FD;N`7 zuWxGolm+@n?i{Yak?FK~qKpm23@+B0oBOo(x5w7|4^@jW>)pH#(JP`mp0V@vF2@)< z@c+oB`0MnO*lDt(tmy9k;25H9Il{+WmHQCJBC^>Ye(q_9|I#aIr#W1O7A;4AT4&7k zbz20Ks=hJ`lnNQFP5js70B;+dr0l5pmhUsVjuf1Hjs~Tdb7Afs@877C=N|LtP=X@>Cp5LkgqYDSY1m;h%o!(Sb6DM}k3qS<8o1Wh|}?q z3#BudS+$5FRm{yOb6ZH5#C5zXUfkFX2{&%<#0Vo3={Jh#0-w+R8@+Pqc1Cz_>(}(4 z3figXfnK{N7#J5>HJt{kN}BHZVGqa{HW%#G?S_D!72+wrTF91==~f@Pqm!l+#Q-iJ>7d$3s9d~(ORgL8|z850I&Yp}az``@aapeo3l3gx?aOT|;_n%7n^SPL!^AHvojhmUe?Ja!fn zcGqVc{MxX(_Tb$LXLe3oL$E8K3~yL%P*eJVm@D2uarEzXC^*>~4H}qUo3wF#lhv)> zGW~lNTC)S$YxuenyCL?j_T;nj#~W7N%6(EBhEkKut2bqN{l{g9ls1G@#^6D+9N!RE zI@CJqxx-4)1Rn#hrhBPi2MZ~Dn&^F=?1?vD^xn;s&`={;rc}btu_2;2t)uK-WN=ei z9MkppiWgvPsnU(Y(f4n7&A86vwH};BZ{FcraJ|`I?m6oCy(YH;d$n_Vx?cj1z`#t# zs}ILWaWmJDdww1YbiuT)}KV>DUh7g^F&BlsM6JfCo9|A;r2|Jt&u!@_$Z#G6Dc?&mQr=2MiX&?qDM% zZr*0tLEKSYwJ=#rbr^F7c-2NLY*wR$KTH}0x>?l2hBP7Uk7bBxEL{YX|DH@$YID!h z>wn11{xHI7J*uN3iZk2rOKd^}1gzR(3Z*x=!9N2Vpg;WWIz%F7@C$#5q&YwBGP=gD zgU~Ja)W1mzEjG!<1tn)F4=s^}nQG1Ee@S12`}K-N3?_|D)Z&cdxPS>)urkQTChxa1J79z@1DI!f}V@ zMDB0Fue`c&Za4Ju*{&?YQAn<>|r2jTs%Yv-YB<$DV&<*E)4PmzW_uDs9 zP=)xB7$eDOveN7|JSp-ClqmAizeQ(Sp&cszcn{9M{3i4~E}^9`~2{U=?B zKf9~{q8^GZ6CDq*38xP993+zW#Gy7FI3g>WW_`EsWKCI}f?O^vZRRPZcPWD(819Xt zpn~g8hSXL|ak!-n_uIhWq=7yn{=KA8f70?X&{V7cZ>56TMhnIdP-?^9zaz~oM{ozl z&%KPx#JNzUFG^S7ihNl6CB17+0U9t$3w)|c{JAcmkcHNB<@C6+=^ z9zM@|*MW0AFxYVi@;jXi+my=qgNq|d0tH|pUbY(AWe#b_6-N;I#9;-F`?Qr zr{d9MN7R@Z5HHNmKwAV4FzX(NXOWw6IW)In%Pr7S?I z$(*vJBnV*U%p%~_-Y#jWK1jg{9V35Drdt1i{R*YlbD~D)=|3s^v69o=hFZAOC`7Db z-}#-60k;#ZFq~eHRk6W{an_Z_|961#ly7-?Dsi;WhiCh+=hT%UCY7#p4ZX_SN2J}C zN`~WAJaiP7p3Na^v#g&W?J_tD)#~&qKdy~#w@Lzq97DH#2FJg@w7WIZ)4H$n9!9Hy z#X~*j5K8ii#`yUVN1hVTXU*4l8YLs=|h?HSWP zNrutf8B)eexuDL+PQEqb*qJ_!cG?8(syp$4s zoC#{s9o&3V)x??(xKjBHLGV)|7KQsY-q_hPK5r4O-|a$j*EZy5%j1yj#CSv%F&u|N z6NMvzuu(;%dJ_g@r>ji6&EF~_yJ_YlE8?M)kmjl}-lw(O6-Z2_UxE3wxtP>vu;RO3 zABveZ!Fp2*AzpJqE7uLb7FC6!Oc_jH#0cxO;Mxo=cX2!)o!onNwdO%g5i&o`0 zw;w`T$n!IX-(Ijg)-*fQ$S-B7Z0mNHtBv4P@Jkj^jBg7>UWsR|c=aBrE(GaOgXk49 z;UbsYXY|&9-b!~*C3e$ORkZ($t8;G5^NG53W81dvG`4NCv2FW_ZQDs>J87IWNn_ha z!}Ij_zUTaLzQR3o&+OTIt!q*6AW8~CeXnD}i!_MN8|)UQQ)8-%6C3fH-S&m?h+`Vv zN%58lX?VGlluFv#zc~=LI>TpS0mr-YuNCsfC*1IZ@IO7Pn0Ou<%6{G@V2BW(T;=xs z6s#fG`DTYMl)2`nR31lq_D6F6sCKR@l~sq65y{kx-GqI@_#`Hgy8a_pwD%t5Um=4; z5iq4Z4J>hq%u^K6Z0LXu6vT%(75qvw;^fI&0bnn<8urCfV=?JpgZ3}hII^!h4y8jc zV!kCyQjwpd-H;f)2jXGt7cHn3*GbSWnM8l1ta<}S@j@cW&OFS;W)EIh=86a5m9n(d zizgIR@|!zR|B?p^O|dx%yLhcX8MFVp)qS>dy`HJ)Xe+gZR)p=nPT4u4=o-MrRtN1ALX)P)1x9A( zrLKSUMx(zMXBU1Mz-seLf-_!5iNV>8#7kf$rR}5ejq2R4`3KL;mjnFNN$eMpNk?YI z9WQwn7ZA+j!be9W0r?fUsJ!eUp(EhkX6JSPnp1t)0W7Qn z!)=K6PYRd|HHi_o%=7Ya_=)S)g9Nj+GU&sE7Ta-CqTQn*sgug4xG3)5J|)mB8OOD_^Lt8D{|bH{37HrDq%TNN#b-yFP1i5 zcI06HJPFK0YG(2Zgd!Mzw1bfD3{n9lS9cLg+B+upy#=LJ3Dhk5z9&QH4)6uPglgRI z2N>o@^3L_*LY=6I$cdVo<@eDw6GjJtwjeL?i0V@alMHK`K1Bzw-RspfOne5K-Sl+g zhpcmmZ67LRuvtUcM^Bh6{t3}qDrRRroDRjbWHse=SFnSB^pI@WIHg>yMjZOn^VZ;C zxV93OIUOxjp#sXUc{$TyM>YCq!i(I!CL3C*nI4V@np~F&z-K74il^^$S+>QKfumFf-kk59}m!IDD_FU17u;gbF7<7 zqJ+^&qOgde(sYxh%5G!LDM{9fOyElXTNZT@EzCBtSB{=VWAz&5)~LFQhB`Kx;>ov^yvFEVlA&WedW^SOvxZnKUiIiR z*O>o?qg)+lniDC4i$!6KDZ)g)r1XBNo-9ke{zAq>kFktGdV;qKIui95q8i77l4us% zKO%D+7KGWc>Tma?Y5nDLS|oQ#aaHw2{NH(2`hhAe=LY;4(gAoU#jYTttpw;;@s4!@ z^AVAeDI|-v(O6*#ibPAmD#T&GSm}5iD8pc1xuf49INemfk^LqK?BQ0}&iR_qX_+;M-< zK+eG|tO3iS(NpRzWleBZk{90a2=wbPq`0eObqmupzBhvX6od20D!Hr=LE_RY zBqyNjh1r%Bq^l!Qste0KxD)`fHoG0x({)^!8U=gsvu%a5fdkkC4A&6#*c%dj@8Kf| zLQO&{{pc?IHzzh^h}IVEuML~6@x{!f4i2dKNgGmIC?JDPlkP$R{G#!yEfNu>R6yq` zELK|=cS{35;M0jMg~GGb4VlN|EL1_xl`y)P2o1%_1)`JU+3hUYs=OzKF7Gf4x%Go{ z?{`4-+#rve*R#XuJ{=l4PFWzGbpIqJ5k-6_(oiK;0VLQkV;*27Mz+<(xoewi*G70W z4xCl^;%Mlm&y#-g<)woT^iw?5?`4Avwd^J zCB*As#`|K}7nPS)w6TaaazZ0OCMeu6#xyS~Nbg+6O&v_6^S3hG?)Ehm^-4(6S=u}l zV>TZ}oM>;s$Y1wf(;m6cbBEmy1d9~ooo?SDHoUPa`38O=Xn#|fzKSR8q1?4pnALy4 zSBUGGbJ=m;h?D-im7NffmsR+k9^PMQ@Sq&I=&mB9-4ocPVR161^VYHHZ`hT~k-zG4 z1+n45P<$S-sJl^5?DfruN7+>dJevyK@qN>b(58?lA;u1laS#PQ(Y!5gqkv=b(LIW} z=Y8Sdrm#Rr;Q9NHB;eJY3g8&#lzaSbRKX9k%hMO^5ctBGijnW)G}rBmG)T8x0HRLr zZkg#nta`hL%vL|OuBn9d5q=LBghR_w7rO^(g{ou)>=A`}Bj6SY%Pfl1pdpN~#Dmz{ zlwLXnzPe48q4z0JF1Co2T5~8BOSq-5FR8q5xEW~|cJBQws<&prc1*tL4bwx7H_G>T zK`(s?Jw7cK+dR6kEX8cOD7Z(mko&Lh9q#s^>?8Om5WZ{kPQ%QE)mm%JD13%S}W?%Ko;%-U#u$NL)E3iqw2geK)Ay7<%%c zw+Mm+IU3!f(Dx}iaU*$eX#P7kt1tSfd`8UpeuVLfITtyMoY4SdU!D)&hY51#O@DH4 z@KXzMT{;QG*M`7}@Ayj|71-B>3U)qXM7Jj%xMEGJd@x_zn;(?F@Gyt5HF8v8qZ=GM z8-*MBPM3a!j##))-8#NgC!o5^AZVZkX$NMKv{n1nDu%DAGqQRZKfXI?D?A(dhwn|2 zP=&q^=x6>-W-6L72|JDeiYbq#&{5JJYIk1)OtiV;KsyT9e87?NOO&9A?(A8`W;a)k zK_s!}yFtyh9v<+OCt6F0cl$&nu=H=B@E4iQMqIr>u5IDpB5`qyY&%byh_}lEkjzK6 zJ%&heG)PDU<))JPu@_8nXC!~{;UMpSo{?a)eu3BJ8bpM_VHyc%X&tyCflENS@ z7rHIco~FZ@7wJgaI?uMt{eEEIS=rF{o(5saMVy5Y>W%2b-+`nF=5M|#45ydqj2`kshx zMp*e@eUk8B9(j>T<0LXlIeIQVt4U*fgC!c48{HT>TsA?K$L|GZy0{~um8+zQ*Btn6 zACcr%+`*t3gFHAeN`XXCP|Ni@dwIklfX5W~V)lN<*sAtx>XZsmN z=#LvWsch9ZLp`J4{@5`&aYE;N;+BMt!Se--)EfbUi)@sCIjV`vQb~|M(ASIzymuM?yT>h{o`0zR)MHvQ^Pif+d z-TtMlSZhS`=4ur^%FJRF>uC>HT_)TYZ2;GifK)=m6x!UCZ7_awkEyAFA?%~(nKB4* zJO5++IQ7)9m&dGFjXa^r6fdIlr2rdte0j+b@bSvwQwSWiD*=`66d#^?A7Q{rpb;=G z)ixXX&JTzn0T>U3H`)QQ_TZHjBsw|ZU>qE3^cnt z?yvYXPaoU~_KonWw{260VmM1H0d(bs>)j`Kk|XuHFa+=9#uaa*(W~e2hu8+WvXfo2 zM>}pJ3JHV3>J>dc#9#KOdy4An7><4HygtHU`oh5u*)!*(XIY z3UxyA`t)L*uu4x_=0Goy5n+Zl#X6}yWPEgx+9TZ^KV#~L4tt5Bo9?E|C(bIN0N?)7(hnp8SZw+?105~WnO z7A`L?j7&|3TAO~i+Ue%-`(OfR;^AFgyvuJcUY?$gwS7_822>U%zb=}fQ7VM04g;OC zHHrfoxmK7jnZa|hjMVrK_Ye}qcb8hKZ~oNABn3t6x^DiP307n|;WXp%A-ln{1S|p% zh%KODYk4=Zg#AqGZGT46#WkB7Bd(T?Sf9bniCu&gmDqSI8soa_%p1a}AagT24!bJ4 zbI`<)TWsHBFu>BOE+6d~k|()*=tM_L=1z2`P3Yq1_sBkO{IWXdVWzIYes_@yMSHj= zHmC2s+Y>r+8u-}`A_MhTTm-!~Spob5qo|^G=1>_#(c*g#A?U`%%Azl+(p}QIpl&!4UqVj6R*YkLdSe9_s-_T{eVi(Z% zkEZ@O@|lKFJhv7x==Si#p)%Q?Y~c97Q|+8R4VB56p0PW|DeQ z+Ytqi7fa3~;O^G~ww?{{7N>q#SGA5~kBoFZXOUn0y`|?4`)I1FSR!B@Ynzw02{=?40zoFJ)W1fH{Eo>i;v!yHjvO{(A*e zu~J5pE^?q`B|ys;A}=w({`?sKccXg;c>j#NrJ%&5;_qfIGSR;2m{q<*i?lcOj5^}h zG^H}ARTSd7H@5xxt8~b0(K!RD_>jrT%J#?`_ji1@(zZK2C{*rP^|XAdbfN99{rKW! zX^$M4-x9)TmvU{Xnj%g1D?UEo&nikOqxsHj2G;3JjnX2RxJMMR3RGIYzih0~WdE4= z)?#uSTZ(Gk8OsyRFUoZs`~hz-pdUV=SE%eY7XWR;NBbJ z5out@Dns}}2^00w_X!gD0n_>_pBNd2n2jd5f(46mQjl9z?z&1n9XhR(AA?y z*3V}=F(_3x!mxv$sOzCS(b{?Kz_G7^349GAn- zs@~6c=u66dzacGUvh&Qvi6R9&KZx`o_S4Dt%?@GM7E#L<#XAaIhQ}Pzq!c+>$@sWt zdtuj`x;*o{{|x2TbRyFW-<#Gc)Vq_`+_SMgn@5u+5-+~EAz1l6iwW9Gv*xf8Vx`V; z>nM)pnsp62vh*8yf2eTR^3W5T<%z7p=b3p0R$bZtMD3yZX0&r_$ngYDlU465nr!lU zP~RMCEcRZ>FdiyMT9;mX8Efqsv-tQ%6nf&$>nIi7u#Q1ul6IQ;tq5yT)_v#*@43MN zoB{wnjzhH9|4l5rrGJB?Sb3!Ndm8>m)G2`s@#ii}jV?DB+F0ypMIsrq?}`#U4j(kw z>$B*#fZu#TzY#pnStJUXr0K;A3=jrSYfjG%qA1MbiaUoEl2#K^976cgX~r^liaukT zAMGD|Q9y?97?VDm=-|)GI0boo6 zNMcE)HmMLXm;!;B&LZoz|NEX!ZIY1|T4Ag@+-4a%EF(y(JUkcJti~~7vjBP>GAcZu#4WZ0jz%D0CXhXW&Z;NWMw% zCEfo;T3IL1U7Vs$b%w#IiQiD}-~Z3Q|9cV%s0FV_^Kak+Jobdg`IPP7_e)3Sv_m3F zQtgUWa(FsfGs~gQ76eqHw38|%XC_b(^Krwf$ctj1mkDyKF>peU)z6xiN3j68KKhIt zXz|z6!Z26szgNKcjn}SRv{Gu_G;Gbh|g^#*C7f7z4SD>T{N=@S}PGVX1EH zB=KH=mQA34SX@xceIqm)3TlXRdXgBSJyKcT+jtU>h8A%nFy!24NJ0PBTgtDYOV>rt zR~2l1hAD7&$*&!)L=G{kC&y`V5>Y=}i_9!M^MuTP1FKj4s0z^oWhilbDlC!L*{W{p8iBt6gd;tM0FQDdL&V=AuN7U*wus9A;2b!qe z{7pC(8^!2-2)GRu8@QWUDlj;1Q z)gF~=Sz%_3#BJII>AXgQ|iY%=an8pq& zVf2kA9)9hb<$$vWAp(}p0b8+vL+;-%l0{aQp$WETa+oj(@VA_NNj-X--NfwbDs=S! zbleZXYxj2&WS7zTC$pLZL70`91gZ6zcn!l}bf#%YBgf5{ZCzTwN%rQ+(otgrI}bh2 zGI}EhIKK$qE8)4e>bRfg+P-)==h2dizQj!xfmI`q?FvH0YuxzS8@N+K&oK*g%^ODr z_Q_$Mc5}F08*zo2cd({Ei7&2I2tE`>3hsL?xEGs{$vbkxIeBmwCYHx|l!5sjO)yKK zGo~`>11mS&_~L8z1FIiH$~tT=&f<56q?FC4(Xq3nMkjMnvbkx+0i~cj(IF_vrqFCX zTp7xZoN1nw#g@h)-AfQjV*}f4-zY<7b{_wvZM!-$z8=JQ`9&BX${YT2l_{RB^3&q2 zAa(7yB>w`+qQ?V%2+dJJ?>pmCR(b|3u>=jjjFoFkeFEE}&?Fsc-MF|R|7dkmS1@w8 zMlYc=)DFWG8ZVHJC!ETe)JYJ#r-%349#?vu+yVOx84yqVf`X$X@aq}vJFmEjnQq{`oBXLdtFiJg4OEU)3`B}t={&BR}sIKYdH{f%NN6S`oGJ$Zvc=s z#)MPOW$3xj;VuDj8C$EQicr4NSLr(!nb7Cvc1kB}w(NCVIOjxTg%|z$?arQJX;Gx% zb43suhFyVXwb3L;6)olV8}J(*g|r^&b$swPtCo%U6eBpCoj{nQ5^I z;1@YLXeV$W$5G|svFtnLBoy?Y8lCQtFMuSeQs9e-&@)_XZ}K6o!MMKE$mJYlkpH@L zBXG$mD~zLE;50U8Fa_7oi%L8<6ruMHT2gf%R5ifnKZ=_%D_s78+Wp-H=eh(!sa{+j zy_s3VcT1~aGgz5=j}K)Dj>fR-VsNUCU(ecM=x^x|v-=RwL+ccuKlRnP^CJ7?>+5P* zVZK8a%T-w{!a6Jy7MEJ73}<;6=J7(;Xf=1t^Yh&+Z*SmK zx-zW5g*<|q_M3mcz^-ce4N{c&Xp0GJq6-WSq3TCSu@3Hj+mskVF1KDb7q}^v!tkqw za)TusUQ-s$h1Pc@Ra-6<|$m%@z+Z9EeqHT-{g-p?p@%k`=8acSa&mUAZ zCmQsVYa-SoYo_yQwHFU(dSji!)MQiduK-feCSL}&@caOBu_>GKb{J*PKW`-nf?v>) z?Qg`P2}BV{9cI#zlPkygVOASte?R`~xRugMecWS6_kFBu0B4OJ&`KqjYBlTaD0p0T zLb@(ENe8}K9J#+9Yey+Kv3z^1Px((OgxoSAw?}r&*`dbcL$^;eb{pQeI8lISO)GblHyB0xt$woEUyJ9g zKcuBhk_1Fj7u9=JDhlMT4_AP1>oUizO~j+`X4>cn(x^}5QB>&W)W{5C>@m4C+#%GJdP^AuB75fl}IjHpCbqZss1xBfuC#%(1-hK?;ixH`?M0_np zmUCp$mpDZ}5!Qfw@w8O|5vjOg&vtuaoi?2Cp6fo6LQ?;rq5Cr9;bzgB@eGutFrZ&O z2%FOxMB&xXrjr_R`M6m$RzlT@V{bh!zf|~`rB&JVLE8t4MQ^(;VZ?WU_gT~Pk@n3o z;0vd54$LBTgD?iujvjoAU|HXrO-5o`&eYAUjm!Kp@{i(jh7HTks17EEL2}Ef$)T=XehG;u!hh*Z8KC+bfk1^Ztp@7567NwiI zU`K5<8o%IgD0s|G%T~Ii{8hFn&;U`NJQaNyAaFRKQ0~Y95p#&&-MQp;ZIxka&zT2Dd8LUG+ z;ysMi@x>A*?HrZ`St|AMa`yas{~VQL@N+(;e1|Ait}mhDWXOpBd%rL?aW6c_rWV0B zhk=J_z9kSaj&jwCejS}yr?t|B<^6P#iTjaGw}nL!>3tGfX!W)r&Lp>XPpfZ2m3eTl zZ7Xm`7>*!e`~!?^tsy?Ho`-$Ks{3%# zw1T7a1a@{~foha(A2)JUr3i#{0=0oY(#&b&sxl1WE0f3CPL9BZ`;gAoYMks37f?2(0XK z<4}RURIvNQ<#5`LzMF?3U-#k^AL~ipy^!W9^;0_jP&rg<&8h7FrQ}qUgdLvX$iaT` z2Nd&Wt>g7!-gufS9(b7?d2^+uu3AST zxYv#cTJm)?z(W#1>ZHaE8ZwBSUY=>r8fiL+z6>WJhq?5A$nxxXbS8PCVHI+;*6Y~i z1;%CnBQhHarKAYSpx9*)*dh}^uU(dp^w?Y#EG%{zZL@y*{0 z-}@OoiO~I^`FR7;`KLmq9t4;D7xtsls!IMQ6`e1K`9m$b^o2~iHT9M~$WvW@BuEHP zL9>LrV2Gil2%1AF4}{EseTeFpdNh#5g5(s8j~8Aj(1==%rZikNtj z5rC+87s*k6Z(uY~*}juOJ0}Gn(!TP|_PY-X7E@X5{R&qTi&}^%35rg^<{E#-L4tn~ zj*HOO{BhIXM;;U!8{1WhXy|dZq|q6L;E6_7pgfz6sIC%t$e&qZG_damwzy$`^;jdy zi$F0(B*XZm;S=QmT-9BL<`GfcPXf_RoZ95?y<(!HszF(u-?H4+3dKn{e0yC%880j; z)VUYk0|`l;o+tRpX%%Q^2;9{5ScQu72$Yzs$>j1V0cusjkbY_VYm>*^tc5Talg*=p zlT15TS9#*y{h2tmMh@G^3y5~34XBme&pdV9=y&*YAA%%C` z<^DgdI0{)AVQ7RHX#Bnlv!2uHyc0R+cC5Uhj5=lx z+)#v7t-Ylw2KaKaPowcPOZ%}od*Uw)RU1u-5dcP0%vp&0Ub0$vyrg824(58QGR6?% zs}om7OnmUPq)}i9rNPB!AX|RlC&%RWe`ZV5W?vw^@<1G&aVjf%0mU7guZMT)<#Ykj z+FhT;#g-r2$)Og8u$3QMqZ?cuG zN!)s_XR%Gr`<=a63M+Du_SR-zYxcazNfvi(|2d2Z0y-hp4vtvtZr?*V7(=8#oD)Bk zho{3%xSm-}Ty2bApH3@EDt#3OO+Fgz4O~uSw}z|^@HZlpLdxr*>_(zu*ePLIoM}kG z_-=15q&JY{6fnncZ|`X3a``~wHrGvNT8u2yssIy^fOWO%81qL4Vz>^#f@bPCRjqaG ze2W_s5V0K-8hFpWhYDS&P4{4#>E7`v=Iay?`SyIh|5>b*g7fnm+}p~|qjE|kk5Dd&>RxF+tW6t*#Ml9tdI=9W< z*wi8$exy3}!0FlPUYOd`x>UI^2NzheuoTVGcBA7IZ&vAJYcgJbz!>q`elJ^Q5#L*{ z#b!IkHF1kIsur+7vv#?M{bSwx4f&KC!go{6Jzdjcat4)!1O6 zNXV{St}TxZJ?#1X&cTasXtWBJt$01K=Z^?2=#DS)=X&f{MEs=G zVUTQxNDoQg6$i&_w$3flR9+fvV12x$50Ewvg57$w)so)cg1}0DL zq};oNmife(@U8MPL?5$3ycw1S@cY9?LFF*@1x*z|>ToiTC*aQoCW;%QGwNZ7$+-f!E)9>to~-_LOG zZ+GuXH)361qyJ+%(Chz{`e&wt9?!(BLRydHk6xoj#P6uXKlzgp24x^{b34w%mqS(P z8lDVT(H(QTG6j98<`5SxrjN@2;v-qsVJ5rQL$F@4yD;eocP$3ysF$G5k!pmme|B!f zWr`boM>yUgw@j;y_da6b=JJ7zdzDtQ-ofO4Y?{^+Q?ravKBp|~jN>U8AljX65@C@= zkOCFl5%pvTW~`tegJL%PlJ#++mtJs zOAGGIk#LbyU)8JT^GIo$i5f+1WhF=wTh|h?L(yh-AqzN`AdM5ZR{X{oCNWT(0C+wT<_5y@P7Wvz_4}5cV~KwF0=LK@8yu=I z#_BC7S?mr`JQy;CC7I|3M5M+G0hr+$GK5UzB1KY=9WMxa^>mtlQY6Dev~vYNA7SV- z1PB%|SZoe)UdZG1P^*OWhrY?1=k;gqOFBKQ(B#PDXuHBLoMnoLigK)j&I<+)UxQqE zQ3%qQF`4e7sO(ft?~VSEaC!cl<(fDUc& z;0x{IV%nK|TFPMahwfyjflL)!RrK5PDpDropv4eM+BW68`Dh!({Ll4G-(ikI7Ukyw z3=T*_!{|Dz&|xkQ+1!wQdB7n7fLR9Z8KqOa#SERt?I!DCJ$9AXuaiYRj9wO>GgoSo zrBLf(f<1MKmRUrYZ}JoXIv`C9uO((#!;JMSab6X)VlKz(_M$vIt%c=}THEqO6Z8<& zc|3x0=f2r36NW8Zw5;@0F~l@|FH)uBMU5cj8|V=YQC4XJuTtlPKqWG{9z5iJa%Q(rjIjDED{t#)ZfOKzpP; z9K3cFTu)>J7O3R4G%MqQ39?S}-!L8o_xjh8Uw+XA=dqO)_)WCN%p+qkxb^Q_6*!JG zw-s(^RF|MUSFj=bg}<>OfsaxK0Q<1B=joMj?Cn6jTT|8Cim}{3brr^OhmtA5&H4RCg%29MF}HTJdZLdc9%NX zJgL&?1RP#Xo;k4EJ39v-_fXkYx+|+NngcV1{!*4GJ8Z0;sF^ z8X6mw#b_SZz2V}SX2m&{@}J96Uyca6DL}Q-Xj5PHxKY2LxLZBA+^@G{{mKbx1Ok3Q zd%0u!V5DdEf-U!;3-++rr|N}@HSYOo$Rk*<^8I+!Np`&3Q@+q@LPm?=5N9Dla+<&R z34>4Li#RPh`;@D4cKP8C*q;9V2U$vBZ?ns@?t1$_n?FldI56X-FP!fWj;W2BL@KV^ zT!FDEyOk0FhL8P#MF;1_1bRr-UMTs=qVH7K14Pw`kI4^S82c-|xXL(SZ^4-bI~!dx zx6ZWj?1rBl(;;ZST45*i*+!4Y7ZjM)+f+nPQ};_aU?Am8069nNu+lj5G>>OABlB+z&0H6!qw#&FShL zeDVnSg(K?Bq2)H|EhIc<)m^kF$KBry=VbYT~B=B-b*CdoBiyY zzXZ)QUn@6xi&e`NIsk;KpF`zx5ksIWkLy8Q@r1j|jC+NXe2c$;-Z3(`p%!trhRrct zQ7lB!Om;f=aq;;`02r)#RVb)|h0}#UJ(~sGsQD2y7gt2t7#UwQN|EovDD!N_d=iBS zeOgF)S)%`sBSk9lU+Ki;LeNcvn3)oHQUaYO=T+)jrIAaz^)y?er7tI)1l1kAu@)N( z%QuOVB$OJE82bDtN2HVesw|<20!WEKcA8;KWOtm=Rk&`#TC3Pw)oOjolMRfKi7_lf z9<=bMu-Axrx-^2X>!ha5qcPn)V#TdYe__qf>mf-jTE? zsGoLpP9gJ&1KPK_!>6Mujg^kgE38cccb7cH8q-^jwnia5`{qSG(nr@bF0xH) z==iIG7?)>@BKJYn4Sv@0?lL%v>UiO!lyeJm0qfx(Po&hOpeWnO22kE*=%<(xoyMEZ z5G!tM_!K;HLxnzyjz4~2Sh}n?6<1ZpgN{nfsf>TP998NRc`AchT2+WrW043LKq63T zFZ<9S)|`Csba0@qGUO%vZ62*1s6 z--DZYCJ2>bLE80w6})g>3zj7`kqQYxo~TjNF2>Tqh!45HO1_a5U{8nGd7saYT&#_9 zpN4Cs+POP9&sc9{076_sI4@s~BYfb84QEW8sp^=%^T>^K6y&?bFiE<~(cgnWF7=NakkPa8kKl`_B_!a|h%Za)ydkwg(XKoy-|>0yMPL94QtQ9vJn)E#0(QjocJjGz+|)oAM53=d*e z!(}scx&7AY$YrsTw4oPsoq-Kw9yQ{MA_7y|=PgW2L3usW2VVr?1G_3wrT1SJZuc*o z_h5m&E1*EXwp*DIQ6!97a0ut~(>_C+Mzfqyrb@l>U7Xoue-Al7A#!-wQw+ktf7%aC zd4c;`no30KPswLFbDcVztFAknTa~SSRKBr(eVQekwV(iNMBO{6WtSsHC)Q+64RXff zP(~?TgNgW+$?1wY^W*~+W7u(`lV$b@t(LaL@u6qN2s=A@$38GWF*0prXJB;qWX%Vf za!NPa&w`Llsc7Zm4JU}z4V_j8Y}5Ns@WFtBZC7QE1jh5pwo26==^H{?3&Iib9jz<#6>aqdc7_3 z@Re2O^@T9c36<+tQ-Dn-wcx7~*tnw2Mw$kB!LqJYYJCXqzt zjiJYrq>M35bg9|&f!dTU?d!Ksx=4@|WRhZV5Km~&fW zr`Kx1@rla|nf$Qu{Cgo_&wVZQrQ-Y2`!|rf7sJ+;-T06Dw@!@Mk<7ztC*A}8r~2<8 zg45O~Tm{|}ma=*72r*j+p$xjMMw73`j!IZUeP_hBMvP63Af{(J)Czxqq?6Ysg;lS) z-@n0uFR1pF99XXC-P;+-+b=Ob7u=x6^?b9${v?u}Ei})`2jW}qaIhJN1nPa^QX!I+ zHOz2h4bk2L&^Oi*;z&P0PjR@(OsP={PA4why6)k7sSM|$O!L?j1Y85L=#*Ox5}G+j z^BEX3SM;UJ>vG_gW|GkF6>f5Bg%Yn9yXZ%t4%aA(#&H~x+6Qy;;EIvY4z#p+O`c{wA@83*EV}-{6t4mTcb(baK z!I)+L_j+|17q3hH!TKwMgW(@G2{`$cJz2z#aM7&avvzl~M?5R5Zle^PE7jL(u;g^- zk1;M!xWwg0-}m4%v01KIc^pmBW+-^Ig_D5S`G(bMgh=e|h5}XJE1~oYim%&+P&Pt- zP)-e|WnR zUJQX2V>XX~im!W5hzF}^N@a$-Of$Zd6HH(yBx_32TOqs5>VOP&*mZW>ssbKbPfrXT zA?eDZUC`6MhKYiz--a$fC>m0C1V9@_EBlo6BnJ(#@3_#s4Y5I{=^K*lJF4!e;&3P2 zbt2~n8H^uvxp>5_0ck{k10((zr=0exDxn{OD&CT0JOa_2^P!dAsvzvsYAq~Pv&8cIuK*> zt;li_fgble(sIb~O=kn{K9s9Sv9-E#o1LJXUMrGEAGEP%F?ui((U?mV1dwJ5X+|py z`*oh(@S%X*)zE!#FJ7bV6I*MSBE7+OA$SMw91UrCl;HmRC${o@)Q*+4X!LZVWx;$f zTf2u1g3gJr*^C-tg_gQF1{{Z&fXC2&drhxv++eJq#SQ~?2qwb?F}ha4bnwWDLwYD4 z-SK`yIX3BRaYoAp>u~9t(GTAHDHupE@a|2<-J8DuP`)co8X=8oxVkwYg&ua+@KvPD z49Z8B1AB=unqwhbWO*9_YXh`^4rwg=8%Bz)0CdwbLBCm>PrJVf)lnN@uu+w$K)sNy zQ|*U}IE-h2EADHYU}$_+DzkqVT6l{3&}5J4yx$7X2As(#Cq?wY2g0P|L-IFu??<6h zcG+7t)jk0~=};LERcyGFkE*p-X6Cu0BueL2?ne(*hS(!{+kUtXgk#=`;0>!0!R_eS z^xpmwWu)dd4TYqAE6w4IB{&wBRa95?uDDFHw7tA$zVKP1nBU{Y0V{*6dTcD9yN%A+ z8CfDvhXU?$XK_zM8S1mN#urayD~;v?uW6`z14r79-Zj4PjRsrL38zDK&1T`^1P4;+ zEM`kyR?Eh3l|}Z3qqRynw%MGb;};TKAEp0u;Q_-lNs369|NFWJSkU}}gol?GM(KV~>0k`XW%FH>17m_zkTO64On{b26Yi>8<;$+-Rw2B}uw#CiXM&vys5c2IKE2G;C z4072j@RqmD*jf}WbYm+<*sjZTW_Juad33;UR!wg088!<8(%3fELyNaWm*YRIU^bPVcXudb%r0wDN2 zDEUVEt;gjZ&NjD=D9`$E&cm~5aY-0MttPvmJ*Zb_a*sb#1@oJ?8#kb@KH zG;Y3{)8r7?)v{hIHo;L3?QM@S-IzV=~!T)lSWKjEszT1FG{d3;ZBfu816&IV(t zJ)-Jjr;_o$t;P{xq*u$slI=B}%Sf@+i|6U2 zZI`$;t|}h1vEI@3QN~tV05v2{UsP>ZBcsU|dIJ4mfB)~tO0zQxCcR!P7?N1UoDE1C zJWX~|f#WtWK@U%>(8pP-Ql)~2tuH>JdUREV@x32LLs;0FiEF+yGXGY{bUK)-gkcol zrsI%o_=KoMfyM1ZazE*Hoy4_2oFjiTG0*?$?98K*%HlZgGEM`z9d)#QT+nhuQ`EG= z(jdrkn4A_U%TmKEPMMCT7+Fp(kcmrLHibGmlT9*WmXRgLEy*GgNgJ0?)9jQBTB)e% z-1kJS=FI#t{Q2IGyZtWrEqw3ygKlRoP7B+x*qw>FLpQ8 zG3b?YB{237lTyYQue?+EAq;rE>U{1$7{1hG+Ts7eAoYlKbT`M~8eq#uqviuE!8P;| zn=P4WEkHHqj`j}AuEtL-qsRQDJ_1s$DC0 zH$^Vvre8Y$?lC3gLrk&py>pjssU3msiZ$3j7}~dK)d{Qwd~~YKwiAi25j~!`$13BI zH&M%{I}Hqv>EJV6JxM$J9lb^5n6iuqt8-4%E(I!#1HPe$8@k|}8d}E9GtlM9!TKw6 zLpr+ylD`@XA(9%m+>%g`mXmu&J346JZK|m@Ta6IgTb{wUN9Am6=+;mOVqa@Hk1kFS zsb1vu373>${8wyLd+{Rr>#JE#?87FdW24X4QuT8w-&ZBL9a_BNm>PC{Sj|VBPF?GL zLFI9XIw_asV8JH@KkDij9MWbWG)Omo`8I5f$-SO@atNzyTCnUyIkIFMv+`++eLN>L zifC7w-7NpeF5RD`u7Bgzc$CDLy7kREM8uOgzvl9kqFoaYe8fj-G=5Wk#r;f&!Gh0t z9Ep9Yxbu4$Q+>ffuUfKJ9S$IX=THZTd^ZD%6(*Yv2@+{*_nM%awJy-SI({#F&aq4A z3~55dUt36`cGzQz#y;wc8JmG>$zKH+spkSXp()}SfZ0MI;lL6EW#kMae8cT2#gG)fdHz=azDT(|?1 zlEmB)Gm0CBjbYSC6nH%T`fi?SvllSKrKWAk@1xttW;_L&6%nIWqJY(NLQ8Tw8+GQ& zY2#M(f$Pa=QPy_Q4GaSZAz^2fszOYAW4 zbz!$%iZeOyYL$dH8&2?slVRGA!T74HE6G#P#LxoQ2!$8Q=(M!R*l3^p_HRf-gPV-02W{whJ4`gMm46<9E#HGG9Jlr_X&z2>8 zvHtC%w#s5_p8?a{K%^mhuka4SL`HqvRJh3X1gdpJxd3{<^*o8f)>L~~UCbqi9FZ38 zF#;Ieww)eRzW)Zr9&9f&f}z-;;lrhOXsumcHLO*>_b_ek?fe&<$5mCaM*!nylz01f z55;{!XtYfV!7niIoI_w;(J6N)2;=Np>}7~UF?FpC{VUq|&-skyZ=~F9AhQPL=5U$^ z-&v;t;A`rBIHjAyALRxp*XGs%nZSJlV0w#zv z*D5feCKLmLc&9kWbHknTngmX2SU+e!@Zr&4$o?%9H~=x5j7RMS_e% zO9yC7%f;kJ6BN!i^WQ-t*$<9H1n7Wl_V74eSGxY+}xOka`%3| VG}JuA$6Ez_h&b;}To1pqe*+9w3vmDd literal 0 HcmV?d00001 -- Gitee From 39b8e2ce6b016ae15baa442099b5ffc73ead2c9f Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 10:45:01 +0800 Subject: [PATCH 737/791] rev. level --- .../msprobe/core/common/const.py | 3 ++- .../accuracy_tools/msprobe/core/common/log.py | 22 +++++++++---------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 427ec2f9c..1f44a71b1 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -328,7 +328,8 @@ class MsgConst: """ MSPROBE_LOG_LEVEL = "MSPROBE_LOG_LEVEL" LEVEL_ENUM = ["0", "1", "2", "3", "4"] - LEVEL = ["SKIP_PRINT", "ERROR", "WARNING", "INFO", "DEBUG"] + DEFAULT_LEVEL = 1 + LEVEL = ["DEBUG", "INFO", "WARNING", "ERROR"] SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 2fadda511..699d3995b 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -16,38 +16,38 @@ class BaseLogger: if input_level in MsgConst.LEVEL_ENUM: return int(input_level) else: - return 3 + return MsgConst.DEFAULT_LEVEL def get_rank(self): return self.rank def filter_special_chars(func): @wraps(func) - def func_level(self, msg): + def func_level(self, msg, **kwargs): for char in MsgConst.SPECIAL_CHAR: msg = msg.replace(char, '_') - return func(self, msg) + return func(self, msg, **kwargs) return func_level - + @filter_special_chars def error(self, msg): - if self.level >= MsgConst.LEVEL.index(MsgConst.LEVEL[1]): - self._print_log(MsgConst.LEVEL[1], msg) + if self.level <= MsgConst.LEVEL.index(MsgConst.LEVEL[3]): + self._print_log(MsgConst.LEVEL[3], msg) @filter_special_chars def warning(self, msg): - if self.level >= MsgConst.LEVEL.index(MsgConst.LEVEL[2]): + if self.level <= MsgConst.LEVEL.index(MsgConst.LEVEL[2]): self._print_log(MsgConst.LEVEL[2], msg) @filter_special_chars def info(self, msg): - if self.level >= MsgConst.LEVEL.index(MsgConst.LEVEL[3]): - self._print_log(MsgConst.LEVEL[3], msg) + if self.level <= MsgConst.LEVEL.index(MsgConst.LEVEL[1]): + self._print_log(MsgConst.LEVEL[1], msg) @filter_special_chars def debug(self, msg): - if self.level >= MsgConst.LEVEL.index(MsgConst.LEVEL[4]): - self._print_log(MsgConst.LEVEL[4], msg) + if self.level <= MsgConst.LEVEL.index(MsgConst.LEVEL[0]): + self._print_log(MsgConst.LEVEL[0], msg) def on_rank_0(self, func): def func_rank_0(*args, **kwargs): -- Gitee From 135f006395dd5cfba6bf3fd5b2bc6ee43556986b Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 10 Sep 2024 10:57:09 +0800 Subject: [PATCH 738/791] grad_probe no file error + ut --- .../msprobe/mindspore/grad_probe/hook.py | 3 +- .../grad_probe/test_ms_grad_monitor.py | 70 ++++++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py index dabd6f710..ecd9c0fb4 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/hook.py @@ -11,7 +11,7 @@ from mindspore.common.initializer import initializer from msprobe.core.grad_probe.constant import GradConst from msprobe.mindspore.common.log import logger -from msprobe.core.common.file_utils import remove_path, write_csv +from msprobe.core.common.file_utils import remove_path, write_csv, create_directory from msprobe.mindspore.grad_probe.global_context import grad_context from msprobe.mindspore.grad_probe.grad_analyzer import grad_dump, get_rank_id from msprobe.mindspore.grad_probe.grad_analyzer import csv_generator @@ -64,6 +64,7 @@ def hook_pynative_optimizer(opt, hook_input): gradients, = input cur_step = grad_context.get_context(GradConst.CURRENT_STEP) if grad_context.step_need_dump(cur_step) and grad_context.rank_need_dump(hook_input.rank_id): + create_directory(hook_input.save_dir) output_lines = [] for index, grad_value in enumerate(gradients): param_name = hook_input.g_names[index] diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index 3af0d5bbc..c51f751f1 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -56,8 +56,12 @@ def get_hash(file_path): class TestMsGradientMonitor(TestCase): - def test_gradient_monitor(self): + def test_gradient_monitor_L2(self): gradient_output_path = os.path.join(directory, "gradient_output") + if os.path.isfile(config_json_path): + os.remove(config_json_path) + if os.path.isdir(gradient_output_path): + shutil.rmtree(gradient_output_path) config_dict = { "task": "grad_probe", "dump_path": gradient_output_path, @@ -90,5 +94,65 @@ class TestMsGradientMonitor(TestCase): target_md5_value = "874174395c56922f86118050e8c93e74" self.assertEqual(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") - os.remove(config_json_path) - shutil.rmtree(gradient_output_path) \ No newline at end of file + + def test_gradient_monitor_L1(self): + gradient_output_path = os.path.join(directory, "gradient_output") + if os.path.isfile(config_json_path): + os.remove(config_json_path) + if os.path.isdir(gradient_output_path): + shutil.rmtree(gradient_output_path) + config_dict = { + "task": "grad_probe", + "dump_path": gradient_output_path, + "rank": [], + "step": [1], + "grad_probe": { + "grad_level": "L1", + "param_list": [] + } + } + save_dict_as_json(config_dict, config_json_path) + + main() + + my_dense_bias_path = os.path.join(gradient_output_path, "rank0", "step1", "my_dense.bias.npy") + self.assertTrue(os.path.isfile(my_dense_bias_path), "bias npy file not found") + my_dense_bias_real = np.load(my_dense_bias_path) + my_dense_bias_target = np.arange(5).reshape(5) > 0 + + self.assertTrue((my_dense_bias_real == my_dense_bias_target).all(), "bias ndarray not same as target") + + my_dense_weight_path = os.path.join(gradient_output_path, "rank0", "step1", "my_dense.weight.npy") + self.assertTrue(os.path.isfile(my_dense_weight_path), "weight npy file not found") + my_dense_weight_real = np.load(my_dense_weight_path) + my_dense_weight_target = np.arange(5*16).reshape((5, 16)) > 0 + + self.assertTrue((my_dense_weight_real == my_dense_weight_target).all(), "weight ndarray not same as target") + + real_md5_value = get_hash(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv")) + target_md5_value = "874174395c56922f86118050e8c93e74" + self.assertEqual(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") + + def test_gradient_monitor_L0(self): + gradient_output_path = os.path.join(directory, "gradient_output") + if os.path.isfile(config_json_path): + os.remove(config_json_path) + if os.path.isdir(gradient_output_path): + shutil.rmtree(gradient_output_path) + config_dict = { + "task": "grad_probe", + "dump_path": gradient_output_path, + "rank": [], + "step": [1], + "grad_probe": { + "grad_level": "L0", + "param_list": [] + } + } + save_dict_as_json(config_dict, config_json_path) + + main() + + real_md5_value = get_hash(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv")) + target_md5_value = "874174395c56922f86118050e8c93e74" + self.assertEqual(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") \ No newline at end of file -- Gitee From 66b518a578acb6bd13d44b309f68c3282985afcb Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 11:33:45 +0800 Subject: [PATCH 739/791] deredun. step list --- debug/accuracy_tools/msprobe/core/common_config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index 0c8619bbc..cd85b0a1a 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -44,6 +44,7 @@ class CommonConfig: elif isinstance(step, str) and '-' in step: continual_step = self.get_step_from_string(step) real_step.extend(continual_step) + real_step = list(set(real_step)) real_step.sort() return real_step -- Gitee From 0619b042c64c268a0a4ec6209d094199347f7552 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Mon, 9 Sep 2024 15:35:42 +0800 Subject: [PATCH 740/791] continue to train when oc_nums reached --- .../msprobe/core/common/const.py | 12 ++- .../msprobe/core/common/file_utils.py | 2 +- .../msprobe/core/data_dump/data_collector.py | 32 +++++--- .../data_processor/mindspore_processor.py | 3 +- .../msprobe/docs/02.config_introduction.md | 6 +- .../msprobe/docs/03.config_examples.md | 2 +- .../msprobe/mindspore/dump/jit_dump.py | 18 +++- .../msprobe/mindspore/service.py | 82 ++++++++++++------- .../test_mindspore_processor.py | 4 +- .../core_ut/data_dump/test_data_collector.py | 24 ++++-- 10 files changed, 122 insertions(+), 63 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 4f686598a..1f508a239 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -8,6 +8,8 @@ class Const: """ Class for const """ + TOOL_NAME = "msprobe" + SEP = "." REGEX_PREFIX_MAX_LENGTH = 20 REGEX_PREFIX_PATTERN = r"^[a-zA-Z0-9_-]+$" @@ -157,7 +159,7 @@ class CompareConst: INPUT_STRUCT = "input_struct" OUTPUT_STRUCT = "output_struct" SUMMARY = "summary" - + COMPARE_RESULT_HEADER = [ NPU_NAME, BENCH_NAME, NPU_DTYPE, BENCH_DTYPE, NPU_SHAPE, BENCH_SHAPE, COSINE, MAX_ABS_ERR, MAX_RELATIVE_ERR, ONE_THOUSANDTH_ERR_RATIO, FIVE_THOUSANDTHS_ERR_RATIO, @@ -219,7 +221,7 @@ class CompareConst: RED = "FFFF0000" YELLOW = "FFFF00" BLUE = "0000FF" - + # run_ut const MAX_TOKENS = 65536 SPECIAL_SPARSE_MOED = 4 @@ -295,6 +297,7 @@ class OverflowConst: OVERFLOW_ORIGINAL_MODE = 0 OVERFLOW_DEBUG_MODE = 1 + class MsCompareConst: # api_info field MINT = "Mint" @@ -306,7 +309,7 @@ class MsCompareConst: DUMP_DATA_DIR_FIELD = "dump_data_dir" DATA_FIELD = "data" - #detail_csv + # detail_csv DETAIL_CSV_API_NAME = "API Name" DETAIL_CSV_BENCH_DTYPE = "Bench Dtype" DETAIL_CSV_TESTED_DTYPE = "Tested Dtype" @@ -315,13 +318,14 @@ class MsCompareConst: DETAIL_CSV_MESSAGE = "Message" DETAIL_CSV_FILE_NAME = "accuracy_checking_details" - #result_csv + # result_csv RESULT_CSV_FORWARD_TEST_SUCCESS = "Forward Test Success" RESULT_CSV_BACKWARD_TEST_SUCCESS = "Backward Test Success" RESULT_CSV_FILE_NAME = "accuracy_checking_result" EPSILON = 1e-8 + class MsgConst: """ Class for log messages const diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index 0976323b5..65163f3fa 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -241,7 +241,7 @@ def make_dir(dir_path): if os.path.isdir(dir_path): return try: - os.mkdir(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY) + os.makedirs(dir_path, mode=FileCheckConst.DATA_DIR_AUTHORITY, exist_ok=True) except OSError as ex: raise FileCheckException(FileCheckException.ILLEGAL_PATH_ERROR, f"Failed to create {dir_path}. " diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index ba7787489..0dde81412 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -28,6 +28,9 @@ class DataCollector: else: self.scope = build_scope(None, self.config.scope, self.config.list) + def __del__(self): + self.write_json() + @property def dump_data_dir(self): return self.data_writer.dump_tensor_data_dir @@ -89,10 +92,13 @@ class DataCollector: if self.config.level == "L2": return self.data_writer.update_stack(self.data_processor.analyze_api_call_stack(name)) - if self.data_processor.is_terminated: - self.handle_data(name, data_info, use_buffer=False) - raise Exception("[msprobe] exit") - self.handle_data(name, data_info) + if self.config.framework == Const.MS_FRAMEWORK: + self.handle_data(name, data_info, flush=self.data_processor.is_terminated) + else: + if self.data_processor.is_terminated: + self.handle_data(name, data_info, flush=True) + raise Exception(f"[{Const.TOOL_NAME}] exit") + self.handle_data(name, data_info) def backward_data_collect(self, name, module, pid, module_input_output): self.update_construct(name) @@ -100,10 +106,13 @@ class DataCollector: return data_info = self.data_processor.analyze_backward(name, module, module_input_output) - if self.data_processor.is_terminated: - self.handle_data(name, data_info, use_buffer=False) - raise Exception("[msprobe] exit") - self.handle_data(name, data_info) + if self.config.framework == Const.MS_FRAMEWORK: + self.handle_data(name, data_info, flush=self.data_processor.is_terminated) + else: + if self.data_processor.is_terminated: + self.handle_data(name, data_info, flush=True) + raise Exception(f"[{Const.TOOL_NAME}] exit") + self.handle_data(name, data_info) def backward_input_data_collect(self, name, module, pid, module_input_output): self.update_construct(name) @@ -122,16 +131,17 @@ class DataCollector: self.handle_data(name, data_info) def update_construct(self, name): - if self.config.framework == Const.PT_FRAMEWORK and self.config.level not in DataCollector.level_without_construct: + if self.config.framework == Const.PT_FRAMEWORK and \ + self.config.level not in DataCollector.level_without_construct: self.data_writer.update_construct({name: self.module_processor.api_parent_node}) self.data_writer.update_construct(self.module_processor.module_node) - def handle_data(self, name, data_info, use_buffer=True): + def handle_data(self, name, data_info, flush=False): if data_info: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) logger.info(MsgConst.CLEAR_SYMBOL + msg, end='\r') - if use_buffer: + if not flush: self.data_writer.flush_data_when_buffer_is_full() else: self.write_json() diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index a72096804..72bee39d1 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -139,7 +139,6 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): if self.overflow_nums == -1: return False if self.real_overflow_nums >= self.overflow_nums: - logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") return True return False @@ -160,6 +159,8 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): for file_path, tensor in self.cached_tensors_and_file_paths.items(): save_tensor_as_npy(tensor, file_path) self.real_overflow_nums += 1 + if self.overflow_nums != -1 and self.real_overflow_nums >= self.overflow_nums: + logger.info(f"[{Const.TOOL_NAME}] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") self.cached_tensors_and_file_paths = {} def _analyze_maybe_overflow_tensor(self, tensor_json): diff --git a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md index 0b6a4a137..b377e9126 100644 --- a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md +++ b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md @@ -116,10 +116,12 @@ num_workers = num_workers ### 1.5 task 配置为 overflow_check +PyTorch 与 MindSpore 动态图场景下，"level"须为"L1"；MindSpore 静态图场景下，"level"须为"L2"。 + | 参数名 | 解释 | 是否必选 | | ------------- | ---------------------- | -------- | -| overflow_nums | 控制溢出次数，int 类型，仅 PyTorch 场景支持，表示第 N 次溢出时，停止训练，过程中检测到溢出 API 对应的 kernel 数据均 dump。
**配置示例**："overflow_nums": 3。默认为 1，即检测到1次溢出，训练停止，配置为 -1 时，表示持续检测溢出直到训练结束。 | 否 | -| check_mode | MindSpore 场景 kernel 级别的溢出检测，str 类型，可取值：
"aicore"：开启 AI Core 的溢出检测；
"atomic"：开启 Atomic 的溢出检测；
"all"：开启 AI Core 和 Atomic 的溢出检测，默认值。
**配置示例**："check_mode": "aicore"。 | 否 | +| overflow_nums | 最大溢出次数，int 类型，默认为 1，仅 PyTorch 与 MindSpore 动态图场景支持。表示第 N 次溢出后，不再进行溢出检测。过程中检测到溢出 API 对应的 输入输出 数据均 dump。
**配置示例**："overflow_nums": 3。配置为 -1 时，表示持续检测溢出直到训练结束。 | 否 | +| check_mode | MindSpore 静态图场景 kernel 级别的溢出检测，str 类型，可取值：
"aicore"：开启 AI Core 的溢出检测；
"atomic"：开启 Atomic 的溢出检测；
"all"：开启 AI Core 和 Atomic 的溢出检测，默认值。
**配置示例**："check_mode": "aicore"。 | 否 | ### 1.6 task 配置为 free_benchmark diff --git a/debug/accuracy_tools/msprobe/docs/03.config_examples.md b/debug/accuracy_tools/msprobe/docs/03.config_examples.md index bfaae3f43..89e511009 100644 --- a/debug/accuracy_tools/msprobe/docs/03.config_examples.md +++ b/debug/accuracy_tools/msprobe/docs/03.config_examples.md @@ -204,7 +204,7 @@ "tensor": { "scope": [], "list":[], - "data_mode": ["all"], + "data_mode": ["all"] } } ``` diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py index 82faf70b6..dde585684 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py @@ -1,9 +1,12 @@ import os + from mindspore import Tensor from mindspore.common.api import _MindsporeFunctionExecutor from mindspore._c_expression import PyNativeExecutor_ + from msprobe.mindspore.dump.hook_cell.api_registry import api_register from msprobe.core.data_dump.data_processor.base import ModuleForwardInputsOutputs +from msprobe.core.common.const import Const def dump_jit(name, in_feat, out_feat, is_forward): @@ -18,8 +21,8 @@ def dump_jit(name, in_feat, out_feat, is_forward): name_template = "Jit." + result + ".forward" else: name_template = "Jit." + result + ".backward" - JitDump.data_collector.visit_and_clear_overflow_status(name_template) - if JitDump.data_collector: + if JitDump.need_dump(): + JitDump.data_collector.visit_and_clear_overflow_status(name_template) module_input_output = ModuleForwardInputsOutputs(args=in_feat, kwargs={}, output=out_feat) JitDump.data_collector.forward_data_collect(name_template, {}, pid, module_input_output) @@ -27,6 +30,7 @@ def dump_jit(name, in_feat, out_feat, is_forward): class JitDump(_MindsporeFunctionExecutor): dump_config = None jit_enable = False + def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self._executor = PyNativeExecutor_.get_instance() @@ -50,6 +54,14 @@ class JitDump(_MindsporeFunctionExecutor): def set_data_collector(cls, value): cls.data_collector = value + @classmethod + def need_dump(cls): + if cls.dump_config.task != Const.TENSOR and cls.dump_config.task != Const.STATISTICS: + return False + if not cls.data_collector or cls.data_collector.data_processor.is_terminated: + return False + return True + def grad(self, obj, grad, weights, grad_position, *args, **kwargs): if JitDump.jit_enable: api_register.api_set_ori_func() @@ -57,4 +69,4 @@ class JitDump(_MindsporeFunctionExecutor): if JitDump.jit_enable: dump_jit(obj, args, None, False) api_register.api_set_hook_func() - return output \ No newline at end of file + return output diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 3747bffa7..439a87a40 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -78,29 +78,31 @@ class Service: def build_hook(self, target_type, name): def forward_hook(api_or_cell_name, cell, input, output): + if not self.should_excute_hook(): + return None + if target_type == BaseScope.Module_Type_Module: api_or_cell_name = cell.mindstudio_reserved_name + module_input_output = ModuleForwardInputsOutputs(args=input, kwargs={}, output=output) + else: + module_input_output = ModuleForwardInputsOutputs(args=input, kwargs=cell.input_kwargs, + output=output) + self.data_collector.visit_and_clear_overflow_status(api_or_cell_name) - if not self.switch: - return None - if self.data_collector: - if target_type == BaseScope.Module_Type_Module: - module_input_output = ModuleForwardInputsOutputs(args=input, kwargs={}, output=output) - else: - module_input_output = ModuleForwardInputsOutputs(args=input, kwargs=cell.input_kwargs, output=output) - self.data_collector.forward_data_collect(api_or_cell_name, cell, pid, module_input_output) - if self.data_collector.if_return_forward_new_output(): - return self.data_collector.get_forward_new_output() - if target_type == BaseScope.Module_Type_API: - del cell.input_kwargs + self.data_collector.forward_data_collect(api_or_cell_name, cell, pid, module_input_output) + if self.data_collector.if_return_forward_new_output(): + return self.data_collector.get_forward_new_output() + if target_type == BaseScope.Module_Type_API: + del cell.input_kwargs return output def backward_hook(api_or_cell_name, cell, grad_input, grad_output): + if not self.should_excute_hook(): + return + if target_type == BaseScope.Module_Type_Module: api_or_cell_name = cell.mindstudio_reserved_name self.data_collector.visit_and_clear_overflow_status(api_or_cell_name) - if not self.switch: - return if self.data_collector: # 框架最新接口变更，grad_input和grad_output的含义发生了变化，与torch含义保持一致，因此此处调换顺序传入 module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input) @@ -256,16 +258,20 @@ class Service: self.start_call = True if self.should_stop_service: return - if self.config.step and self.current_iter > max(self.config.step): + if self.need_end_service(): api_register.api_set_ori_func() self.should_stop_service = True - logger.info("msprobe: dump has successfully ended") + self.switch = False + logger.info("************************************************") + logger.info(f"* {Const.TOOL_NAME} ends successfully. *") + logger.info("************************************************") return if self.config.step and self.current_iter not in self.config.step: return self.model = self.check_model_valid(model) - logger.info("msprobe: debugger.start() is set successfully") + logger.info(f"{Const.TOOL_NAME}: debugger.start() is set successfully") + if self.first_start: try: self.current_rank = get_rank_if_initialized() @@ -275,27 +281,28 @@ class Service: if self.config.rank and self.current_rank not in self.config.rank: return self.register_hook_new() + if self.config.level == "L1": + JitDump.set_config(self.config) + JitDump.set_data_collector(self.data_collector) + ms.common.api._MindsporeFunctionExecutor = JitDump + ms.common.api._PyNativeExecutor.grad = JitDump.grad + if pijit_label: + PIJitCaptureContext.__enter__ = self.empty + PIJitCaptureContext.__exit__ = self.empty self.first_start = False + self.switch = True logger.info(f"Dump switch is turned on at step {self.current_iter}. ") self.create_dirs() logger.info(f"Dump data will be saved in {self.dump_iter_dir}.") - if self.config.level == "L1": - JitDump.set_config(self.config) - JitDump.set_data_collector(self.data_collector) - ms.common.api._MindsporeFunctionExecutor = JitDump - ms.common.api._PyNativeExecutor.grad = JitDump.grad - if pijit_label: - PIJitCaptureContext.__enter__ = self.empty - PIJitCaptureContext.__exit__ = self.empty def stop(self): if self.should_stop_service: return - logger.info("msprobe: debugger.stop() is set successfully. " + logger.info(f"{Const.TOOL_NAME}: debugger.stop() is set successfully. " "Please set debugger.start() to turn on the dump switch again. ") if not self.start_call: - logger.error("msprobe: debugger.start() is not set in the current scope.") + logger.error(f"{Const.TOOL_NAME}: debugger.start() is not set in the current scope.") raise Exception("debugger.start() is not set in the current scope.") if self.config.step and self.current_iter not in self.config.step: return @@ -305,6 +312,20 @@ class Service: self.start_call = False self.data_collector.write_json() + def need_end_service(self): + if self.config.step and self.current_iter > max(self.config.step): + return True + if self.data_collector and self.data_collector.data_processor.is_terminated: + return True + return False + + def should_excute_hook(self): + if not self.switch: + return False + if not self.data_collector or self.data_collector.data_processor.is_terminated: + return False + return True + def create_dirs(self): create_directory(self.config.dump_path) self.dump_iter_dir = os.path.join(self.config.dump_path, f"step{self.current_iter}") @@ -322,7 +343,7 @@ class Service: construct_file_path = os.path.join(dump_dir, "construct.json") self.data_collector.update_dump_paths( dump_file_path, stack_file_path, construct_file_path, dump_data_dir, None) - + def empty(self, *args, **kwargs): pass @@ -336,7 +357,8 @@ class Service: if self.config.level == "L0": if not self.model: - raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "The current level is L0, the model cannot be None") + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, + "The current level is L0, the model cannot be None") for name, cell in self.model.cells_and_names(): if cell == self.model: continue @@ -353,4 +375,4 @@ class Service: cell.register_backward_pre_hook( self.cell_processor.node_hook(prefix + Const.BACKWARD, Const.START)) cell.register_backward_hook( - self.cell_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) \ No newline at end of file + self.cell_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py index e097f9c61..5ecd508ac 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -179,14 +179,12 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.assertEqual(mock_save.call_args_list[1][0], (tensor2, "tensor2")) - @patch("msprobe.core.data_dump.data_processor.mindspore_processor.logger.info") - def test_is_terminated(self, mock_info): + def test_is_terminated(self): self.data_processor.overflow_nums = -1 self.assertFalse(self.data_processor.is_terminated) self.data_processor.real_overflow_nums = 2 self.data_processor.overflow_nums = 2 self.assertTrue(self.data_processor.is_terminated) - mock_info.assert_called_with("[msprobe] 超过预设溢出次数 当前溢出次数: 2") self.data_processor.overflow_nums = 3 self.assertFalse(self.data_processor.is_terminated) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index 4282d8ffd..6ef99986f 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -75,7 +75,7 @@ class TestDataCollector(unittest.TestCase): self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - + mock_info.assert_called_with(MsgConst.CLEAR_SYMBOL + "msg", end='\r') mock_flush.assert_called() mock_write_json.assert_not_called() @@ -83,11 +83,11 @@ class TestDataCollector(unittest.TestCase): mock_update_data.reset_mock() mock_info.reset_mock() mock_flush.reset_mock() - self.data_collector.handle_data("Tensor.add", {}, use_buffer=False) + self.data_collector.handle_data("Tensor.add", {}, flush=True) mock_update_data.assert_not_called() mock_info.assert_not_called() mock_write_json.assert_called() - + @patch.object(DataCollector, "update_construct") @patch.object(DataWriter, "update_stack") @patch.object(BaseDataProcessor, "analyze_api_call_stack") @@ -98,10 +98,15 @@ class TestDataCollector(unittest.TestCase): patch.object(StatisticsDataProcessor, "analyze_forward", return_value={}): with patch.object(StatisticsDataProcessor, "is_terminated", return_value=True), \ self.assertRaises(Exception) as context: + self.data_collector.config.framework = Const.PT_FRAMEWORK self.data_collector.forward_data_collect("name", "module", "pid", "module_input_output") - mock_handle_data.assert_called_with("name", {}, use_buffer=False) - self.assertEqual(str(context.exception), "[msprobe] exit") + mock_handle_data.assert_called_with("name", {}, flush=True) + self.assertEqual(str(context.exception), "[msprobe] exit") + self.data_collector.config.framework = Const.MS_FRAMEWORK + self.data_collector.forward_data_collect("name", "module", "pid", "module_input_output") + mock_handle_data.assert_called_with("name", {}, flush=True) + self.data_collector.config.framework = Const.PT_FRAMEWORK self.data_collector.forward_data_collect("name", "module", "pid", "module_input_output") mock_handle_data.assert_called_with("name", {}) @@ -112,9 +117,14 @@ class TestDataCollector(unittest.TestCase): patch.object(StatisticsDataProcessor, "analyze_backward", return_value={}): with patch.object(StatisticsDataProcessor, "is_terminated", return_value=True), \ self.assertRaises(Exception) as context: + self.data_collector.config.framework = Const.PT_FRAMEWORK self.data_collector.backward_data_collect("name", "module", "pid", "module_input_output") - mock_handle_data.assert_called_with("name", {}, use_buffer=False) - self.assertEqual(str(context.exception), "[msprobe] exit") + mock_handle_data.assert_called_with("name", {}, flush=True) + self.assertEqual(str(context.exception), "[msprobe] exit") + self.data_collector.config.framework = Const.MS_FRAMEWORK + self.data_collector.forward_data_collect("name", "module", "pid", "module_input_output") + mock_handle_data.assert_called_with("name", {}, flush=True) + self.data_collector.config.framework = Const.PT_FRAMEWORK self.data_collector.backward_data_collect("name", "module", "pid", "module_input_output") mock_handle_data.assert_called_with("name", {}) -- Gitee From 467118854a40299b4df3a2cbacaa7ba499b8ebb9 Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Tue, 10 Sep 2024 15:16:18 +0800 Subject: [PATCH 741/791] =?UTF-8?q?grad=5Fprobe=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/docs/16.overflow_check_MindSpore.md | 0 debug/accuracy_tools/msprobe/docs/17.grad_probe.md | 2 +- debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py | 2 +- 3 files changed, 2 insertions(+), 2 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/docs/16.overflow_check_MindSpore.md diff --git a/debug/accuracy_tools/msprobe/docs/16.overflow_check_MindSpore.md b/debug/accuracy_tools/msprobe/docs/16.overflow_check_MindSpore.md new file mode 100644 index 000000000..e69de29bb diff --git a/debug/accuracy_tools/msprobe/docs/17.grad_probe.md b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md index 38eeb3dc6..95330b720 100644 --- a/debug/accuracy_tools/msprobe/docs/17.grad_probe.md +++ b/debug/accuracy_tools/msprobe/docs/17.grad_probe.md @@ -87,7 +87,7 @@ debugger.monitor(optimizer) 在训练结束之后，调用stop接口 ```python -gm.stop() +debugger.stop() ``` ### 输出结果 diff --git a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py index ca3c66706..5aeda41e1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py +++ b/debug/accuracy_tools/msprobe/pytorch/grad_probe/grad_monitor.py @@ -45,7 +45,7 @@ class GradientMonitor: param_grad = grad.clone().detach() is_positive = param_grad > 0 save_filepath = os.path.join(save_path, f"{param_name}.npy") - save_npy(is_positive.numpy(), save_filepath) + save_npy(is_positive.cpu().numpy(), save_filepath) def monitor(self, model): print_rank_0("> parameter names:") -- Gitee From 65c41bd0c3cb081d64c8ea4d9fb78602adc25f46 Mon Sep 17 00:00:00 2001 From: pxp1 <958876660@qq.com> Date: Tue, 10 Sep 2024 15:19:32 +0800 Subject: [PATCH 742/791] =?UTF-8?q?grad=5Fprobe=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/docs/16.overflow_check_MindSpore.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 debug/accuracy_tools/msprobe/docs/16.overflow_check_MindSpore.md diff --git a/debug/accuracy_tools/msprobe/docs/16.overflow_check_MindSpore.md b/debug/accuracy_tools/msprobe/docs/16.overflow_check_MindSpore.md deleted file mode 100644 index e69de29bb..000000000 -- Gitee From 77687ef194a137fdcaf3e1a9183fe4ee549a9d3c Mon Sep 17 00:00:00 2001 From: l30036321 Date: Tue, 10 Sep 2024 15:31:37 +0800 Subject: [PATCH 743/791] remove api --- .../msprobe/mindspore/debugger/precision_debugger.py | 4 ++-- .../msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py index 1bae781d9..ead554b34 100644 --- a/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py +++ b/debug/accuracy_tools/msprobe/mindspore/debugger/precision_debugger.py @@ -62,7 +62,7 @@ class PrecisionDebugger: return MsConst.PYNATIVE_MODE @classmethod - def start(cls, target=None): + def start(cls, model=None): instance = cls._instance if not instance: raise Exception("No instance of PrecisionDebugger found.") @@ -73,7 +73,7 @@ class PrecisionDebugger: if cls._need_service(): if not instance.service: instance.service = Service(instance.config) - instance.service.start(target) + instance.service.start(model) else: if not instance.first_start: handler = TaskHandlerFactory.create(instance.config) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml index 089f444b6..ca2a578f9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/support_wrap_ops.yaml @@ -347,7 +347,6 @@ ops: - linspace - logspace - one_hot - - arange - range - heaviside - bernoulli @@ -726,7 +725,6 @@ tensor: - trace - swapaxes - tile - - to - topk - tril - tensor_split @@ -760,7 +758,6 @@ mint.ops: - all - any - any_ex - - arange - argmax - avg_pool2d - baddbmm -- Gitee From 72fbeb51c5b7f3b66482a5e3c6274ba24f8b279f Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Sat, 7 Sep 2024 17:27:37 +0800 Subject: [PATCH 744/791] support stub_tensor hook --- .../data_processor/mindspore_processor.py | 16 +-- .../msprobe/mindspore/common/const.py | 13 ++ .../mindspore/dump/hook_cell/api_registry.py | 72 ++++++----- .../mindspore/dump/hook_cell/hook_cell.py | 6 +- .../mindspore/dump/hook_cell/wrap_api.py | 113 ++++++++++++++++++ .../dump/hook_cell/wrap_functional.py | 92 -------------- .../mindspore/dump/hook_cell/wrap_tensor.py | 64 ---------- 7 files changed, 182 insertions(+), 194 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py delete mode 100644 debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index a72096804..d7c99643d 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -16,14 +16,13 @@ import zlib import mindspore as ms -from mindspore import ops +from mindspore import mint, ops import numpy as np from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import (BaseDataProcessor, TensorStatInfo, ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs) from msprobe.core.common.file_utils import path_len_exceeds_limit -from msprobe.mindspore.dump.hook_cell.wrap_functional import load_ops_functions from msprobe.mindspore.common.utils import convert_bf16_to_fp32, save_tensor_as_npy from msprobe.mindspore.common.log import logger from msprobe.mindspore.dump.hook_cell.api_registry import api_register @@ -31,7 +30,6 @@ from msprobe.mindspore.dump.hook_cell.api_registry import api_register class MindsporeDataProcessor(BaseDataProcessor): mindspore_special_type = tuple([ms.Tensor]) - ops_func, mint_ops_func, _ = load_ops_functions() def __init__(self, config, data_writer): super().__init__(config, data_writer) @@ -74,10 +72,14 @@ class MindsporeDataProcessor(BaseDataProcessor): if data.dtype == ms.bfloat16 or not ops.is_floating_point(data): data = data.to(ms.float32) api_register.norm_inner_op_set_ori_func() - tensor_stat.max = self.mint_ops_func["max"](data).item() - tensor_stat.min = self.mint_ops_func["min"](data).item() - tensor_stat.mean = self.mint_ops_func["mean"](data).item() - tensor_stat.norm = self.ops_func["norm"](data).item() + get_max_value = api_register.mint_ops_ori_attr.get("max", mint.max) + get_min_value = api_register.mint_ops_ori_attr.get("min", mint.min) + get_mean_value = api_register.mint_ops_ori_attr.get("mean", mint.mean) + get_norm_value = api_register.functional_ori_attr.get("norm", ops.norm) + tensor_stat.max = get_max_value(data).item() + tensor_stat.min = get_min_value(data).item() + tensor_stat.mean = get_mean_value(data).item() + tensor_stat.norm = get_norm_value(data).item() api_register.norm_inner_op_set_hook_func() return tensor_stat diff --git a/debug/accuracy_tools/msprobe/mindspore/common/const.py b/debug/accuracy_tools/msprobe/mindspore/common/const.py index ec0ee5816..c642e399b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/const.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/const.py @@ -34,6 +34,19 @@ class Const: "mindspore.communication.comm_func.reduce", "mindspore.communication.comm_func.reduce_scatter_tensor" ] + TENSOR_DATA_PREFIX = "Tensor." + STUB_TENSOR_DATA_PREFIX = "Tensor." + OPS_DATA_PREFIX = "Functional." + MINT_DATA_PREFIX = "Mint." + MINT_NN_FUNC_DATA_PREFIX = "MintFunctional." + + SUPPORTED_API_LIST_FILE = "support_wrap_ops.yaml" + SUPPORTED_TENSOR_LIST_KEY = "tensor" + SUPPORTED_OPS_LIST_KEY = "ops" + SUPPORTED_MINT_LIST_KEY = "mint.ops" + SUPPORTED__MINT_NN_FUNC_LIST_KEY = "mint.nn.functional" + + DROPOUT_API_NAME_PREFIX = "dropout" class FreeBenchmarkConst: diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py index 584bf91c1..d2344c9d2 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/api_registry.py @@ -13,22 +13,27 @@ # limitations under the License. # ============================================================================ -import mindspore as ms -from msprobe.mindspore.dump.hook_cell.wrap_functional import get_functional_ops, setup_hooks, \ - HOOKFunctionalOP, HOOKMintOP, HOOKMintNNFunctionalOP -from msprobe.mindspore.dump.hook_cell.wrap_tensor import get_tensor_ops, wrap_tensor_ops_and_bind, HOOKTensor +from mindspore import Tensor, ops, mint +from mindspore.mint.nn import functional +from mindspore.common._stub_tensor import StubTensor + +from msprobe.mindspore.dump.hook_cell.wrap_api import (HOOKTensor, HOOKStubTensor, HOOKFunctionalOP, + HOOKMintOP, HOOKMintNNFunctionalOP, + get_wrap_api_list, setup_hooks) from msprobe.core.common.utils import Const class ApiRegistry: def __init__(self): self.tensor_ori_attr = {} + self.stub_tensor_ori_attr = {} self.functional_ori_attr = {} self.mint_ops_ori_attr = {} self.mint_func_ops_ori_attr = {} self.norm_inner_ops_ori_attr = {} self.tensor_hook_attr = {} + self.stub_tensor_hook_attr = {} self.functional_hook_attr = {} self.mint_ops_hook_attr = {} self.mint_func_ops_hook_attr = {} @@ -58,47 +63,56 @@ class ApiRegistry: setattr(api_group, api, api_attr) def norm_inner_op_set_hook_func(self): - self.set_api_attr(ms.ops, self.norm_inner_ops_hook_attr) + self.set_api_attr(ops, self.norm_inner_ops_hook_attr) def norm_inner_op_set_ori_func(self): - self.set_api_attr(ms.ops, self.norm_inner_ops_ori_attr) + self.set_api_attr(ops, self.norm_inner_ops_ori_attr) def api_set_hook_func(self): - self.set_api_attr(ms.Tensor, self.tensor_hook_attr) - self.set_api_attr(ms.ops, self.functional_hook_attr) - self.set_api_attr(ms.mint, self.mint_ops_hook_attr) - self.set_api_attr(ms.mint.nn.functional, self.mint_func_ops_hook_attr) + self.set_api_attr(Tensor, self.tensor_hook_attr) + self.set_api_attr(StubTensor, self.stub_tensor_hook_attr) + self.set_api_attr(ops, self.functional_hook_attr) + self.set_api_attr(mint, self.mint_ops_hook_attr) + self.set_api_attr(functional, self.mint_func_ops_hook_attr) def api_set_ori_func(self): - self.set_api_attr(ms.Tensor, self.tensor_ori_attr) - self.set_api_attr(ms.ops, self.functional_ori_attr) - self.set_api_attr(ms.mint, self.mint_ops_ori_attr) - self.set_api_attr(ms.mint.nn.functional, self.mint_func_ops_ori_attr) + self.set_api_attr(Tensor, self.tensor_ori_attr) + self.set_api_attr(StubTensor, self.stub_tensor_ori_attr) + self.set_api_attr(ops, self.functional_ori_attr) + self.set_api_attr(mint, self.mint_ops_ori_attr) + self.set_api_attr(functional, self.mint_func_ops_ori_attr) def initialize_hook(self, hook): - self.store_ori_attr(ms.Tensor, get_tensor_ops(), self.tensor_ori_attr) - wrap_tensor_ops_and_bind(hook) + wrap_api_name = get_wrap_api_list() + self.store_ori_attr(Tensor, wrap_api_name.tensor_api_names, self.tensor_ori_attr) + self.store_ori_attr(StubTensor, wrap_api_name.stub_tensor_api_names, self.stub_tensor_ori_attr) + self.store_ori_attr(ops, wrap_api_name.ops_api_names, self.functional_ori_attr) + self.store_ori_attr(mint, wrap_api_name.mint_api_names, self.mint_ops_ori_attr) + self.store_ori_attr(functional, wrap_api_name.mint_nn_func_api_names, self.mint_func_ops_ori_attr) + self.store_ori_attr(ops, self.norm_inner_ops, self.norm_inner_ops_ori_attr) + setup_hooks(hook) for attr_name in dir(HOOKTensor): if attr_name.startswith(Const.ATTR_NAME_PREFIX): - self.tensor_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKTensor, attr_name) - - functional_ops, mint_ops, mint_func_ops = get_functional_ops() - self.store_ori_attr(ms.ops, self.norm_inner_ops, self.norm_inner_ops_ori_attr) - self.store_ori_attr(ms.ops, functional_ops, self.functional_ori_attr) - self.store_ori_attr(ms.mint, mint_ops, self.mint_ops_ori_attr) - self.store_ori_attr(ms.mint.nn.functional, mint_func_ops, self.mint_func_ops_ori_attr) - setup_hooks(hook) + api_name = attr_name[Const.ATTR_NAME_PREFIX_LEN:] + self.tensor_hook_attr[api_name] = getattr(HOOKTensor, attr_name) + for attr_name in dir(HOOKStubTensor): + if attr_name.startswith(Const.ATTR_NAME_PREFIX): + api_name = attr_name[Const.ATTR_NAME_PREFIX_LEN:] + self.stub_tensor_hook_attr[api_name] = getattr(HOOKStubTensor, attr_name) for attr_name in dir(HOOKFunctionalOP): if attr_name.startswith(Const.ATTR_NAME_PREFIX): - self.functional_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKFunctionalOP, attr_name) - if attr_name[Const.ATTR_NAME_PREFIX_LEN:] in self.norm_inner_ops: - self.norm_inner_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKFunctionalOP, attr_name) + api_name = attr_name[Const.ATTR_NAME_PREFIX_LEN:] + self.functional_hook_attr[api_name] = getattr(HOOKFunctionalOP, attr_name) + if api_name in self.norm_inner_ops: + self.norm_inner_ops_hook_attr[api_name] = getattr(HOOKFunctionalOP, attr_name) for attr_name in dir(HOOKMintOP): if attr_name.startswith(Const.ATTR_NAME_PREFIX): - self.mint_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKMintOP, attr_name) + api_name = attr_name[Const.ATTR_NAME_PREFIX_LEN:] + self.mint_ops_hook_attr[api_name] = getattr(HOOKMintOP, attr_name) for attr_name in dir(HOOKMintNNFunctionalOP): if attr_name.startswith(Const.ATTR_NAME_PREFIX): - self.mint_func_ops_hook_attr[attr_name[Const.ATTR_NAME_PREFIX_LEN:]] = getattr(HOOKMintNNFunctionalOP, attr_name) + api_name = attr_name[Const.ATTR_NAME_PREFIX_LEN:] + self.mint_func_ops_hook_attr[api_name] = getattr(HOOKMintNNFunctionalOP, attr_name) api_register = ApiRegistry() diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py index 57ed44111..83e9d4d34 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/hook_cell.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================ + from collections import defaultdict from mindspore import nn + from msprobe.core.common.const import Const @@ -30,8 +32,8 @@ class HOOKCell(nn.Cell): if not HOOKCell.g_stop_hook: HOOKCell.g_stop_hook = True self.changed_status = True - if hasattr(self, "prefix_op_name_"): - self.prefix = self.prefix_op_name_ + if hasattr(self, "prefix_api_name"): + self.prefix = self.prefix_api_name HOOKCell.cell_count[self.prefix] += 1 self.prefix = self.prefix + str(HOOKCell.cell_count[self.prefix] - 1) + Const.SEP diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py new file mode 100644 index 000000000..f2eb6677c --- /dev/null +++ b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_api.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +""" + +import os + +from mindspore import Tensor, ops, mint +from mindspore.mint.nn import functional +from mindspore.common._stub_tensor import StubTensor + +from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell +from msprobe.core.common.const import Const +from msprobe.mindspore.common.const import Const as MsConst +from msprobe.core.common.file_utils import load_yaml + + +cur_path = os.path.dirname(os.path.realpath(__file__)) +yaml_path = os.path.join(cur_path, MsConst.SUPPORTED_API_LIST_FILE) + + +class HOOKTensor(object): + pass + + +class HOOKStubTensor(object): + pass + + +class HOOKFunctionalOP(object): + pass + + +class HOOKMintOP(object): + pass + + +class HOOKMintNNFunctionalOP(object): + pass + + +class ApiTemplate(HOOKCell): + def __init__(self, api_name, api_dict, prefix, hook): + self.api_name = api_name + self.api_func = api_dict[api_name] + self.prefix_api_name = prefix + str(api_name.split(Const.SEP)[-1]) + Const.SEP + super().__init__(hook) + + def construct(self, *args, **kwargs): + if self.api_name.startswith(MsConst.DROPOUT_API_NAME_PREFIX): + return args[0] if args else kwargs.get(Const.INPUT) + return self.api_func(*args, **kwargs) + + +class WrapApiName: + def __init__(self, tensor_api_names, stub_tensor_api_names, ops_api_names, mint_api_names, mint_nn_func_api_names): + self.tensor_api_names = tensor_api_names + self.stub_tensor_api_names = stub_tensor_api_names + self.ops_api_names = ops_api_names + self.mint_api_names = mint_api_names + self.mint_nn_func_api_names = mint_nn_func_api_names + + +def get_wrap_api_list(): + api_list = load_yaml(yaml_path) + tensor_api = api_list.get(MsConst.SUPPORTED_TENSOR_LIST_KEY) + ops_api = api_list.get(MsConst.SUPPORTED_OPS_LIST_KEY) + mint_api = api_list.get(MsConst.SUPPORTED_MINT_LIST_KEY) + mint_nn_func_api = api_list.get(MsConst.SUPPORTED__MINT_NN_FUNC_LIST_KEY) + wrap_api_name = WrapApiName(set(tensor_api) & set(dir(Tensor)), + set(tensor_api) & set(dir(StubTensor)), + set(ops_api) & set(dir(ops)), + set(mint_api) & set(dir(mint)), + set(mint_nn_func_api) & set(dir(functional))) + return wrap_api_name + + +def wrap_api_func(api_name, api_dict, prefix, hook): + def api_function(*args, **kwargs): + return ApiTemplate(api_name, api_dict, prefix, hook)(*args, **kwargs) + return api_function + + +def wrap_api_func_and_bind(api_list, api_dict, prefix, hook, hook_class): + for api_name in api_list: + if callable(api_dict[api_name]): + setattr(hook_class, Const.ATTR_NAME_PREFIX + api_name, wrap_api_func(api_name, api_dict, prefix, hook)) + + +def setup_hooks(hook): + wrap_api_name = get_wrap_api_list() + wrap_api_func_and_bind(wrap_api_name.tensor_api_names, {f: getattr(Tensor, f) for f in dir(Tensor)}, + MsConst.TENSOR_DATA_PREFIX, hook, HOOKTensor) + wrap_api_func_and_bind(wrap_api_name.stub_tensor_api_names, {f: getattr(StubTensor, f) for f in dir(StubTensor)}, + MsConst.STUB_TENSOR_DATA_PREFIX, hook, HOOKStubTensor) + wrap_api_func_and_bind(wrap_api_name.ops_api_names, {f: getattr(ops, f) for f in dir(ops)}, + MsConst.OPS_DATA_PREFIX, hook, HOOKFunctionalOP) + wrap_api_func_and_bind(wrap_api_name.mint_api_names, {f: getattr(mint, f) for f in dir(mint)}, + MsConst.MINT_DATA_PREFIX, hook, HOOKMintOP) + wrap_api_func_and_bind(wrap_api_name.mint_nn_func_api_names, {f: getattr(functional, f) for f in dir(functional)}, + MsConst.MINT_NN_FUNC_DATA_PREFIX, hook, HOOKMintNNFunctionalOP) diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py deleted file mode 100644 index 27c73c4ae..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_functional.py +++ /dev/null @@ -1,92 +0,0 @@ -# Copyright 2024 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -import os -import mindspore as ms -from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell -from msprobe.core.common.utils import Const -from msprobe.core.common.file_utils import load_yaml - - -cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") - - -def load_ops_functions(): - ops_func = {f: getattr(ms.ops, f) for f in dir(ms.ops)} - mint_ops_func = {f: getattr(ms.mint, f) for f in dir(ms.mint)} - mint_func_ops_func = {f: getattr(ms.mint.nn.functional, f) for f in dir(ms.mint.nn.functional)} - return ops_func, mint_ops_func, mint_func_ops_func - - -def get_functional_ops(): - ops_func, mint_ops_func, mint_func_ops_func = load_ops_functions() - config = load_yaml(yaml_path) - wrap_functional = config.get("ops") - wrap_mint = config.get("mint.ops") - wrap_mint_functional = config.get("mint.nn.functional") - return ( - set(wrap_functional) & set(ops_func.keys()), - set(wrap_mint) & set(mint_ops_func.keys()), - set(wrap_mint_functional) & set(mint_func_ops_func.keys()) - ) - - -class HOOKFunctionalOP(object): - pass - - -class HOOKMintOP(object): - pass - - -class HOOKMintNNFunctionalOP(object): - pass - - -class FunctionalOPTemplate(HOOKCell): - def __init__(self, op_name, op_dict, prefix, hook): - self.op_name = op_name - self.op_func = op_dict[op_name] - self.prefix_op_name_ = prefix + str(op_name.split(Const.SEP)[-1]) + Const.SEP - super().__init__(hook) - - def construct(self, *args, **kwargs): - if self.op_name.startswith('dropout'): - return args[0] if args else kwargs.get('input') - return self.op_func(*args, **kwargs) - - -def wrap_functional_op(op_name, op_dict, prefix, hook): - def op_template(*args, **kwargs): - return FunctionalOPTemplate(op_name, op_dict, prefix, hook)(*args, **kwargs) - return op_template - - -def wrap_functional_ops_and_bind(ops, op_dict, prefix, hook, hook_class): - for op_name in ops: - if callable(op_dict[op_name]): - setattr(hook_class, Const.ATTR_NAME_PREFIX + op_name, wrap_functional_op(op_name, op_dict, prefix, hook)) - - -def setup_hooks(hook): - functional_ops, mint_ops, mint_func_ops = get_functional_ops() - wrap_functional_ops_and_bind( - functional_ops, {f: getattr(ms.ops, f) for f in dir(ms.ops)}, "Functional.", hook, HOOKFunctionalOP) - wrap_functional_ops_and_bind( - mint_ops, {f: getattr(ms.mint, f) for f in dir(ms.mint)}, "Mint.", hook, HOOKMintOP) - wrap_functional_ops_and_bind( - mint_func_ops, {f: getattr(ms.mint.nn.functional, f) for f in dir(ms.mint.nn.functional)}, "MintFunctional.", hook, HOOKMintNNFunctionalOP) - diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py b/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py deleted file mode 100644 index eda473dc1..000000000 --- a/debug/accuracy_tools/msprobe/mindspore/dump/hook_cell/wrap_tensor.py +++ /dev/null @@ -1,64 +0,0 @@ -# Copyright 2024 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ - -import os -import mindspore as ms -from msprobe.mindspore.dump.hook_cell.hook_cell import HOOKCell -from msprobe.core.common.utils import Const -from msprobe.core.common.file_utils import load_yaml - - -cur_path = os.path.dirname(os.path.realpath(__file__)) -yaml_path = os.path.join(cur_path, "support_wrap_ops.yaml") - - -TensorFunc = {} -for f in dir(ms.Tensor): - TensorFunc[f] = getattr(ms.Tensor, f) - - -def get_tensor_ops(): - yaml_data = load_yaml(yaml_path) - wrap_tensor_ops = yaml_data.get('tensor') - _tensor_ops = dir(ms.Tensor) - return set(wrap_tensor_ops) & set(_tensor_ops) - - -class HOOKTensor(object): - pass - - -class TensorOPTemplate(HOOKCell): - - def __init__(self, op_name, hook): - self.op_name_ = op_name - self.prefix_op_name_ = "Tensor." + str(op_name) + Const.SEP - super().__init__(hook) - - def construct(self, *args, **kwargs): - return TensorFunc[str(self.op_name_)](*args, **kwargs) - - -def wrap_tensor_op(op_name, hook): - def tensor_op_template(*args, **kwargs): - return TensorOPTemplate(op_name, hook)(*args, **kwargs) - return tensor_op_template - - -def wrap_tensor_ops_and_bind(hook): - _tensor_ops = get_tensor_ops() - for op_name in _tensor_ops: - if callable(TensorFunc[op_name]): - setattr(HOOKTensor, Const.ATTR_NAME_PREFIX + str(op_name), wrap_tensor_op(op_name, hook)) -- Gitee From 836bac3149e4a3176c1ab6e62cd7daa4101da531 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Tue, 10 Sep 2024 16:02:41 +0800 Subject: [PATCH 745/791] fix --- .../test/mindspore_ut/grad_probe/test_ms_grad_monitor.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index c51f751f1..a087f809f 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -8,6 +8,8 @@ import mindspore from mindspore import nn, Tensor from mindspore.nn import SGD from msprobe.mindspore import PrecisionDebugger +from msprobe.mindspore.grad_probe.global_context import grad_context +from msprobe.core.grad_probe.constant import GradConst from msprobe.core.common.file_utils import FileOpen file_path = os.path.abspath(__file__) @@ -16,6 +18,8 @@ config_json_path = os.path.join(directory, "config.json") def main(): PrecisionDebugger._instance = None + PrecisionDebugger.initialized = False + grad_context._setting[GradConst.CURRENT_STEP] = 0 debugger = PrecisionDebugger(config_json_path) class SimpleNet(nn.Cell): @@ -130,7 +134,7 @@ class TestMsGradientMonitor(TestCase): self.assertTrue((my_dense_weight_real == my_dense_weight_target).all(), "weight ndarray not same as target") real_md5_value = get_hash(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv")) - target_md5_value = "874174395c56922f86118050e8c93e74" + target_md5_value = "a4ad300992cb10965fbc12c2ee19dd37" self.assertEqual(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") def test_gradient_monitor_L0(self): @@ -154,5 +158,5 @@ class TestMsGradientMonitor(TestCase): main() real_md5_value = get_hash(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv")) - target_md5_value = "874174395c56922f86118050e8c93e74" + target_md5_value = "62e137a119c0d1a44623f10049c3f80d" self.assertEqual(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") \ No newline at end of file -- Gitee From c75a9c4d51603f705caf2d446b7265cc6c4ca2cb Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 16:02:44 +0800 Subject: [PATCH 746/791] try call stack --- .../core/data_dump/data_processor/base.py | 28 +++++++++++-------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index da43b097c..8e0949724 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -94,19 +94,25 @@ class BaseDataProcessor: @staticmethod def analyze_api_call_stack(name): + try: + api_stack = inspect.stack()[5:] + except Exception as e: + logger.warning(f"The call stack of <{name}> failed to retrieve, {e}.") + api_stack = None stack_str = [] - for (_, path, line, func, code, _) in inspect.stack()[5:]: - if not code: - continue - stack_line = " ".join([ - "File", ", ".join([ - path, - " ".join(["line", str(line)]), - " ".join(["in", func]), - " ".join(["\n", code[0].strip()]) + if api_stack: + for (_, path, line, func, code, _) in api_stack: + if not code: + continue + stack_line = " ".join([ + "File", ", ".join([ + path, + " ".join(["line", str(line)]), + " ".join(["in", func]), + " ".join(["\n", code[0].strip()]) + ]) ]) - ]) - stack_str.append(stack_line) + stack_str.append(stack_line) stack_info_struct = {name: stack_str} return stack_info_struct -- Gitee From 2397bb3ce3882c1f839d3ccd139659edc762d4fa Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 10 Sep 2024 16:18:05 +0800 Subject: [PATCH 747/791] fix dispatch --- .../api_accuracy_checker/run_ut/run_ut.py | 100 +----------------- .../run_ut/run_ut_utils.py | 99 +++++++++++++++++ .../tensor_transport_layer/device_dispatch.py | 5 +- .../tensor_transport_layer/dump_dispatch.py | 93 ++++++++++++++++ .../torch_ops_config.yaml | 63 +++++++++++ .../accuracy_tools/msprobe/pytorch/service.py | 11 +- 6 files changed, 267 insertions(+), 104 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py create mode 100644 debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py index 0edbba21c..8aad03b31 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut.py @@ -33,6 +33,7 @@ from msprobe.pytorch.pt_config import parse_json_config from msprobe.core.common.const import Const, FileCheckConst, CompareConst from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import ATTL, ATTLConfig, move2device_exec from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.device_dispatch import ConsumerDispatcher +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import generate_cpu_params, generate_device_params current_time = time.strftime("%Y%m%d%H%M%S") @@ -46,14 +47,7 @@ RunUTConfig = namedtuple('RunUTConfig', ['forward_content', 'backward_content', OnlineConfig = namedtuple('OnlineConfig', ['is_online', 'nfs_path', 'host', 'port', 'rank_list', 'tls_path']) not_backward_list = ['repeat_interleave'] -not_detach_set = {'resize_', 'resize_as_', 'set_', 'transpose_', 't_', 'squeeze_', 'unsqueeze_'} -not_raise_dtype_set = {'type_as'} -RAISE_PRECISION = { - torch.float16: torch.float32, - torch.bfloat16: torch.float32, - torch.float32: torch.float64 -} tqdm_params = { 'smoothing': 0, # 平滑进度条的预计剩余时间，取值范围0到1 @@ -71,98 +65,6 @@ tqdm_params = { } -def deal_detach(arg, to_detach=True): - return arg.detach() if to_detach else arg - - -def raise_bench_data_dtype(api_name, arg, raise_dtype=None): - ''' - 将标杆数据的dtype转换为raise_dtype - 输入： - api_name：api名称 - arg：标杆输入 - raise_dtype：需要转换的dtype - 输出： - arg: 转换dtype的标杆输入 - ''' - if api_name in hf_32_standard_api and arg.dtype == torch.float32: - return arg - if raise_dtype is None or arg.dtype not in RAISE_PRECISION or raise_dtype == arg.dtype: - return arg - return arg.type(raise_dtype) - - -def generate_device_params(input_args, input_kwargs, need_backward, api_name): - def recursive_arg_to_device(arg_in, to_detach): - if isinstance(arg_in, (list, tuple)): - return type(arg_in)(recursive_arg_to_device(arg, to_detach) for arg in arg_in) - elif isinstance(arg_in, torch.Tensor): - if need_backward and arg_in.requires_grad: - arg_in = deal_detach(arg_in.clone(), to_detach).to(current_device).requires_grad_() - temp_arg_in = arg_in * 1 - arg_in = temp_arg_in.type_as(arg_in) - arg_in.retain_grad() - return arg_in - else: - return deal_detach(arg_in.clone(), to_detach).to(current_device) - else: - return arg_in - - is_detach = api_name not in not_detach_set - device_args = recursive_arg_to_device(input_args, is_detach) - device_kwargs = \ - {key: recursive_arg_to_device(value, key != "out" and is_detach) for key, value in input_kwargs.items()} - return device_args, device_kwargs - - -def generate_cpu_params(input_args, input_kwargs, need_backward, api_name): - def recursive_arg_to_cpu(arg_in, to_detach, raise_dtype=None): - if isinstance(arg_in, (list, tuple)): - return type(arg_in)(recursive_arg_to_cpu(arg, to_detach, raise_dtype=raise_dtype) for arg in arg_in) - elif isinstance(arg_in, torch.Tensor): - if need_backward and arg_in.requires_grad: - arg_in = deal_detach(raise_bench_data_dtype( - api_name, arg_in.clone(), raise_dtype=raise_dtype), to_detach).requires_grad_() - temp_arg_in = arg_in * 1 - arg_in = temp_arg_in.type_as(arg_in) - arg_in.retain_grad() - return arg_in - else: - return deal_detach(raise_bench_data_dtype(api_name, arg_in.clone(), raise_dtype=raise_dtype), to_detach) - else: - return arg_in - - def is_tensor_with_raise_precision(arg_in, check_kwargs=False): - if arg_in.dtype in RAISE_PRECISION: - return True - if check_kwargs and arg_in.dtype in [torch.half, torch.bfloat16]: - return True - return False - - def recursive_find_dtypes(arg_in, kwargs=None, check_kwargs=False): - if isinstance(arg_in, (list, tuple)): - return set().union(*tuple(recursive_find_dtypes(arg, kwargs, check_kwargs=check_kwargs) for arg in arg_in)) - elif isinstance(arg_in, torch.Tensor) and is_tensor_with_raise_precision(arg_in, check_kwargs): - return set([arg_in.dtype]) - elif isinstance(arg_in, dict) and check_kwargs: - return set().union(*tuple(recursive_find_dtypes(v, kwargs, check_kwargs=True) for v in arg_in.values())) - return set() - - raise_dtype = None - need_raise_dtypes = recursive_find_dtypes(input_args) - need_raise_dtypes.update(recursive_find_dtypes(input_kwargs, check_kwargs=True)) - if len(need_raise_dtypes) == 1: - raise_dtype = RAISE_PRECISION.get(need_raise_dtypes.pop(), torch.float32) - elif len(need_raise_dtypes) >= 2: - raise_dtype = torch.float32 - - raise_dtype = None if api_name in not_raise_dtype_set else raise_dtype - is_detach = api_name not in not_detach_set - cpu_args = recursive_arg_to_cpu(input_args, is_detach, raise_dtype=raise_dtype) - cpu_kwargs = {key: recursive_arg_to_cpu(value, key != "out" and is_detach, raise_dtype=raise_dtype) for key, value in input_kwargs.items()} - return cpu_args, cpu_kwargs - - def run_ut(config): logger.info("start UT test") if config.online_config.is_online: diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index 37fecac77..d3431c2c4 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -10,7 +10,14 @@ from msprobe.pytorch.hook_module.wrap_tensor import TensorOPTemplate from msprobe.pytorch.hook_module.wrap_torch import TorchOPTemplate hf_32_standard_api = ["conv1d", "conv2d"] +not_detach_set = {'resize_', 'resize_as_', 'set_', 'transpose_', 't_', 'squeeze_', 'unsqueeze_'} +not_raise_dtype_set = {'type_as'} +RAISE_PRECISION = { + torch.float16: torch.float32, + torch.bfloat16: torch.float32, + torch.float32: torch.float64 +} class Backward_Message: MULTIPLE_BACKWARD_MESSAGE = "Multiple backward is not supported." @@ -68,3 +75,95 @@ def exec_api(api_type, api_name, device, args, kwargs): torch_api = NpuOPTemplate(api_name, None, False, device) out = torch_api.forward(*args, **kwargs) return out + + +def deal_detach(arg, to_detach=True): + return arg.detach() if to_detach else arg + + +def raise_bench_data_dtype(api_name, arg, raise_dtype=None): + ''' + 将标杆数据的dtype转换为raise_dtype + 输入： + api_name：api名称 + arg：标杆输入 + raise_dtype：需要转换的dtype + 输出： + arg: 转换dtype的标杆输入 + ''' + if api_name in hf_32_standard_api and arg.dtype == torch.float32: + return arg + if raise_dtype is None or arg.dtype not in RAISE_PRECISION or raise_dtype == arg.dtype: + return arg + return arg.type(raise_dtype) + + +def generate_device_params(input_args, input_kwargs, need_backward, api_name): + def recursive_arg_to_device(arg_in, to_detach): + if isinstance(arg_in, (list, tuple)): + return type(arg_in)(recursive_arg_to_device(arg, to_detach) for arg in arg_in) + elif isinstance(arg_in, torch.Tensor): + if need_backward and arg_in.requires_grad: + arg_in = deal_detach(arg_in.clone(), to_detach).to(current_device).requires_grad_() + temp_arg_in = arg_in * 1 + arg_in = temp_arg_in.type_as(arg_in) + arg_in.retain_grad() + return arg_in + else: + return deal_detach(arg_in.clone(), to_detach).to(current_device) + else: + return arg_in + + is_detach = api_name not in not_detach_set + device_args = recursive_arg_to_device(input_args, is_detach) + device_kwargs = \ + {key: recursive_arg_to_device(value, key != "out" and is_detach) for key, value in input_kwargs.items()} + return device_args, device_kwargs + + +def generate_cpu_params(input_args, input_kwargs, need_backward, api_name): + def recursive_arg_to_cpu(arg_in, to_detach, raise_dtype=None): + if isinstance(arg_in, (list, tuple)): + return type(arg_in)(recursive_arg_to_cpu(arg, to_detach, raise_dtype=raise_dtype) for arg in arg_in) + elif isinstance(arg_in, torch.Tensor): + if need_backward and arg_in.requires_grad: + arg_in = deal_detach(raise_bench_data_dtype( + api_name, arg_in.clone(), raise_dtype=raise_dtype), to_detach).requires_grad_() + temp_arg_in = arg_in * 1 + arg_in = temp_arg_in.type_as(arg_in) + arg_in.retain_grad() + return arg_in + else: + return deal_detach(raise_bench_data_dtype(api_name, arg_in.clone(), raise_dtype=raise_dtype), to_detach) + else: + return arg_in + + def is_tensor_with_raise_precision(arg_in, check_kwargs=False): + if arg_in.dtype in RAISE_PRECISION: + return True + if check_kwargs and arg_in.dtype in [torch.half, torch.bfloat16]: + return True + return False + + def recursive_find_dtypes(arg_in, kwargs=None, check_kwargs=False): + if isinstance(arg_in, (list, tuple)): + return set().union(*tuple(recursive_find_dtypes(arg, kwargs, check_kwargs=check_kwargs) for arg in arg_in)) + elif isinstance(arg_in, torch.Tensor) and is_tensor_with_raise_precision(arg_in, check_kwargs): + return set([arg_in.dtype]) + elif isinstance(arg_in, dict) and check_kwargs: + return set().union(*tuple(recursive_find_dtypes(v, kwargs, check_kwargs=True) for v in arg_in.values())) + return set() + + raise_dtype = None + need_raise_dtypes = recursive_find_dtypes(input_args) + need_raise_dtypes.update(recursive_find_dtypes(input_kwargs, check_kwargs=True)) + if len(need_raise_dtypes) == 1: + raise_dtype = RAISE_PRECISION.get(need_raise_dtypes.pop(), torch.float32) + elif len(need_raise_dtypes) >= 2: + raise_dtype = torch.float32 + + raise_dtype = None if api_name in not_raise_dtype_set else raise_dtype + is_detach = api_name not in not_detach_set + cpu_args = recursive_arg_to_cpu(input_args, is_detach, raise_dtype=raise_dtype) + cpu_kwargs = {key: recursive_arg_to_cpu(value, key != "out" and is_detach, raise_dtype=raise_dtype) for key, value in input_kwargs.items()} + return cpu_args, cpu_kwargs \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py index 0865a8817..4199b25ac 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/device_dispatch.py @@ -12,6 +12,8 @@ from msprobe.pytorch.api_accuracy_checker.compare.compare_utils import DETAIL_TE from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import UtDataInfo, exec_api from msprobe.pytorch.common.log import logger from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import move2target_device +from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import generate_cpu_params + # NPU vs GPU api list CompareApi = set(absolute_standard_api) | set(binary_standard_api) | set(thousandth_standard_api) @@ -75,7 +77,8 @@ def online_precision_compare(api_data, device, common_config, api_precision_csv_ try: # NPU vs CPU - cpu_out = exec_api(api_type, api_name, Const.CPU_LOWERCASE, npu_args, npu_kwargs) + cpu_args, cpu_kwargs = generate_cpu_params(npu_args, npu_kwargs, False, api_name) + cpu_out = exec_api(api_type, api_name, Const.CPU_LOWERCASE, cpu_args, cpu_kwargs) npu_data_info = UtDataInfo(None, None, npu_out, cpu_out, None, [], None, rank=api_data.rank) npu_detail = compare.compare_output(api_full_name, npu_data_info, True) npu_data = pd.DataFrame(npu_detail, columns=DETAIL_TEST_ROWS[-1]) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py new file mode 100644 index 000000000..ba31fa76c --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py @@ -0,0 +1,93 @@ +import os +from functools import wraps + +import yaml +import torch +from torch.utils._python_dispatch import TorchDispatchMode +from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData +from msprobe.pytorch.common.utils import get_tensor_rank +from msprobe.core.common.const import Const +from msprobe.pytorch.common.log import logger +from msprobe.pytorch.api_accuracy_checker.common.config import msCheckerConfig +from msprobe.core.common.file_utils import load_yaml + + +def singleton(cls): + _instance = {} + + def inner(): + if cls not in _instance: + _instance[cls] = cls() + return _instance[cls] + return inner + + +@singleton +class Counter: + def __init__(self) -> None: + self.index_dict = {} + + +counter = Counter() +yaml_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "torch_ops_config.yaml") +yaml_file = load_yaml(yaml_path) + + +class AccuracyCheckerDispatch(TorchDispatchMode): + def __init__(self, attl): + super(AccuracyCheckerDispatch, self).__init__() + self.attl = attl + self.counter = counter + self.aten_ops_blacklist = [] + self.npu_adjust_autogard = [] + self.aten_ops_blacklist = yaml_file.get('aten_ops_blacklist') + self.npu_adjust_autogard = yaml_file.get('npu_adjust_autogard') + + def enable_autogard(self, aten_api): + if aten_api in self.npu_adjust_autogard: + torch._C._dispatch_tls_set_dispatch_key_excluded(torch._C.DispatchKey.AutogradFunctionality, False) + + def __torch_dispatch__(self, func, types, args=None, kwargs=None): + func_name_split_list = func.__name__.split(".") + aten_api = func_name_split_list[0] + self.enable_autogard(aten_api) + if aten_api in self.aten_ops_blacklist: + npu_out = func(*args, **kwargs) + return npu_out + + res = func(*args, **kwargs) + cur_rank = get_tensor_rank(args, res) + # if cur_rank not in DumpUtil.rank_list: + # return res + cur_api_number = self.counter.index_dict.setdefault(aten_api, 0) + api_name = f'Aten{Const.SEP}{aten_api}{Const.SEP}{cur_api_number}' + logger.info(f"tools is dumping api: {api_name}") + # api_data = ApiData(api_name, args, kwargs, res, DumpUtil.call_num, cur_rank) + api_data = ApiData(api_name, args, kwargs, res, 0, cur_rank) + if "device" in api_data.kwargs: + api_data.kwargs.pop("device") + if msCheckerConfig.nfs_path: + self.attl.upload(api_data) + else: + self.attl.send(api_data) + self.counter.index_dict[aten_api] += 1 + + return res + + +def dispatch4data(func, attl): + @wraps(func) + def wrapper(*args, **kwargs): + # if not DumpUtil.get_dump_switch() or DumpUtil.phase not in ("backward", "all") or \ + # not msCheckerConfig.is_online: + # return func(*args, **kwargs) + # DumpUtil.set_dump_switch("OFF") + with AccuracyCheckerDispatch(attl): + res = func(*args, **kwargs) + # DumpUtil.set_dump_switch("ON") + return res + + return wrapper + +def start_dispatch(attl): + torch.autograd.backward = dispatch4data(torch.autograd.backward, attl) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml new file mode 100644 index 000000000..373e6ed0f --- /dev/null +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/torch_ops_config.yaml @@ -0,0 +1,63 @@ +aten_ops_blacklist: + - npu_binary_cross_entropy_with_logits_backward + - npu_ciou_backward + - _cudnn_rnn + - _local_scalar_dense + - _pin_memory + - _to_copy + - _unsafe_view + - clone + - contiguous + - copy_ + - cudnn_batch_norm + - cudnn_batch_norm_backward + - detach + - empty + - index_put_ + - lift_fresh + - max_pool2d_with_indices_backward # shape unmatch + - native_batch_norm_backward + - new_empty + - new_empty_strided + - new_full + - new_ones + - new_zeros + - ones + - ones_like + - permute + - rand + - rand_like + - randint + - randint_like + - randn + - randn_like + - randperm + - scalar_tensor + - select + - to + - transpose + - unbind + - view + - zero + - zero_ + - zeros + - zeros_like + - _record_function_enter_new + - _record_function_exit + - broadcast_ + - allreduce_ + - npu_clear_float_status + - npu_format_cast + - npu_dtype_cast + - npu_dtype_cast_backward + - _allgather_base_ + - _reduce_scatter_base_ + - is_same_size + +npu_adjust_autogard: + - adaptive_avg_pool2d + - batch_norm + - log_softmax + - nll_loss + - to + \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 7da32d0f1..bf31188a5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -16,6 +16,7 @@ from msprobe.pytorch.hook_module.api_registry import api_register from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.module_processer import ModuleProcesser from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.dump_dispatch import start_dispatch torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' HookFn = namedtuple('hookFn', ['pre_hook', 'forward_hook', 'backward_hook', 'forward_hook_torch_version_below_2']) @@ -88,10 +89,10 @@ class Service: return if self.config.online_run_ut: - if self.data_collector.scope and not self.data_collector.scope.check(api_or_module_name): - return - api_data = ApiData(name[:-1], grad_input, {}, grad_output, self.current_iter, self.current_rank) - self.attl_send(api_data) + # if self.data_collector.scope and not self.data_collector.scope.check(api_or_module_name): + # return + # api_data = ApiData(name[:-1], grad_input, {}, grad_output, self.current_iter, self.current_rank) + # self.attl_send(api_data) return if self.data_collector: @@ -138,6 +139,8 @@ class Service: self.first_start = False if api_origin: api_register.api_modularity() + if self.config.online_run_ut: + start_dispatch(self.attl) self.switch = True logger.info_on_rank_0(f"Dump switch is turned on at step {self.current_iter}. ") if self.config.level != "L2" and not self.config.online_run_ut: -- Gitee From 6fac7cdfa51bddf2d62a4b810d27f2be895d053c Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 10 Sep 2024 16:31:40 +0800 Subject: [PATCH 748/791] fix dispatch --- .../msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index d3431c2c4..0441619e3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -1,5 +1,6 @@ import os import re +import torch from msprobe.core.common.const import FileCheckConst from msprobe.core.common.file_utils import FileChecker -- Gitee From 8746b18f866c7dc34956bb9c097083417e1c1015 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 10 Sep 2024 16:40:04 +0800 Subject: [PATCH 749/791] test --- .../msprobe/pytorch/hook_module/api_registry.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/api_registry.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/api_registry.py index f75201eaf..761bc3842 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/api_registry.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/api_registry.py @@ -137,12 +137,12 @@ class ApiRegistry: self.npu_distributed_hook_attr[attr_name[5:]] = getattr(wrap_distributed.HOOKDistributedOP, attr_name) - if torch_version_above_2: - self.store_ori_attr(torch.ops.aten, get_aten_ops(), self.aten_ori_attr) - wrap_aten.wrap_aten_ops_and_bind(hook) - for attr_name in dir(wrap_aten.HOOKAtenOP): - if attr_name.startswith(Const.ATTR_NAME_PREFIX): - self.aten_hook_attr[attr_name[5:]] = getattr(wrap_aten.HOOKAtenOP, attr_name) + # if torch_version_above_2: + # self.store_ori_attr(torch.ops.aten, get_aten_ops(), self.aten_ori_attr) + # wrap_aten.wrap_aten_ops_and_bind(hook) + # for attr_name in dir(wrap_aten.HOOKAtenOP): + # if attr_name.startswith(Const.ATTR_NAME_PREFIX): + # self.aten_hook_attr[attr_name[5:]] = getattr(wrap_aten.HOOKAtenOP, attr_name) self.store_ori_attr(torch._VF, get_vf_ops(), self.vf_ori_attr) wrap_vf.wrap_vf_ops_and_bind(hook) -- Gitee From 635bd0a2c331532b7e5bf912dee84ebffbbaea44 Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 10 Sep 2024 17:40:43 +0800 Subject: [PATCH 750/791] fix dispatch --- .../tensor_transport_layer/dump_dispatch.py | 25 ++++++++++--------- .../pytorch/hook_module/api_registry.py | 21 ++++++++++------ .../msprobe/pytorch/hook_module/wrap_aten.py | 2 ++ .../accuracy_tools/msprobe/pytorch/service.py | 11 +++----- 4 files changed, 33 insertions(+), 26 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py index ba31fa76c..8d7b30199 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py @@ -57,16 +57,13 @@ class AccuracyCheckerDispatch(TorchDispatchMode): res = func(*args, **kwargs) cur_rank = get_tensor_rank(args, res) - # if cur_rank not in DumpUtil.rank_list: - # return res cur_api_number = self.counter.index_dict.setdefault(aten_api, 0) api_name = f'Aten{Const.SEP}{aten_api}{Const.SEP}{cur_api_number}' logger.info(f"tools is dumping api: {api_name}") - # api_data = ApiData(api_name, args, kwargs, res, DumpUtil.call_num, cur_rank) api_data = ApiData(api_name, args, kwargs, res, 0, cur_rank) if "device" in api_data.kwargs: api_data.kwargs.pop("device") - if msCheckerConfig.nfs_path: + if self.attl.nfs_path: self.attl.upload(api_data) else: self.attl.send(api_data) @@ -75,19 +72,23 @@ class AccuracyCheckerDispatch(TorchDispatchMode): return res -def dispatch4data(func, attl): +def dispatch4data(func, attl, status): @wraps(func) def wrapper(*args, **kwargs): - # if not DumpUtil.get_dump_switch() or DumpUtil.phase not in ("backward", "all") or \ - # not msCheckerConfig.is_online: - # return func(*args, **kwargs) - # DumpUtil.set_dump_switch("OFF") + if not status: + return func(*args, **kwargs) with AccuracyCheckerDispatch(attl): res = func(*args, **kwargs) - # DumpUtil.set_dump_switch("ON") return res return wrapper -def start_dispatch(attl): - torch.autograd.backward = dispatch4data(torch.autograd.backward, attl) +def run_ut_dispatch(attl, status): + """ + This function called by online_run_ut. It is used to enable or disable dispatch for torch.autograd.backward function. + + Args: + attl (ATTL): online_run_ut class ATTL, which is used to upload or send api data to server. + status (bool): True means enable dispatch, False means disable dispatch. + """ + torch.autograd.backward = dispatch4data(torch.autograd.backward, attl, status) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/api_registry.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/api_registry.py index 761bc3842..263e9e70c 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/api_registry.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/api_registry.py @@ -107,7 +107,14 @@ class ApiRegistry: if not is_gpu: self.set_api_attr(torch_npu, self.torch_npu_ori_attr) - def initialize_hook(self, hook): + def initialize_hook(self, hook, online_run_ut=False): + """ + initialize_hook + Args: + hook (_type_): initialize_hook + online_run_ut (bool): default False, whether online run_ut or not. + If online_run_ut is True, the hook will not wrap the aten ops. + """ self.store_ori_attr(torch.Tensor, get_tensor_ops(), self.tensor_ori_attr) wrap_tensor.wrap_tensor_ops_and_bind(hook) for attr_name in dir(wrap_tensor.HOOKTensor): @@ -137,12 +144,12 @@ class ApiRegistry: self.npu_distributed_hook_attr[attr_name[5:]] = getattr(wrap_distributed.HOOKDistributedOP, attr_name) - # if torch_version_above_2: - # self.store_ori_attr(torch.ops.aten, get_aten_ops(), self.aten_ori_attr) - # wrap_aten.wrap_aten_ops_and_bind(hook) - # for attr_name in dir(wrap_aten.HOOKAtenOP): - # if attr_name.startswith(Const.ATTR_NAME_PREFIX): - # self.aten_hook_attr[attr_name[5:]] = getattr(wrap_aten.HOOKAtenOP, attr_name) + if torch_version_above_2 and not online_run_ut: + self.store_ori_attr(torch.ops.aten, get_aten_ops(), self.aten_ori_attr) + wrap_aten.wrap_aten_ops_and_bind(hook) + for attr_name in dir(wrap_aten.HOOKAtenOP): + if attr_name.startswith(Const.ATTR_NAME_PREFIX): + self.aten_hook_attr[attr_name[5:]] = getattr(wrap_aten.HOOKAtenOP, attr_name) self.store_ori_attr(torch._VF, get_vf_ops(), self.vf_ori_attr) wrap_vf.wrap_vf_ops_and_bind(hook) diff --git a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py index 651d63f89..e33dfa8f6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py +++ b/debug/accuracy_tools/msprobe/pytorch/hook_module/wrap_aten.py @@ -50,6 +50,8 @@ class AtenOPTemplate(HOOKModule): def __init__(self, op, hook, need_hook=True): if isinstance(op, torch._ops.OpOverloadPacket): op_name_ = op._qualified_op_name.split("::")[-1] + elif isinstance(op, str): + op_name_ = str(op) else: op_name_ = op.name().split("::")[-1] overload_name = op._overloadname diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index bf31188a5..d5ffd40c5 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -16,7 +16,7 @@ from msprobe.pytorch.hook_module.api_registry import api_register from msprobe.pytorch.hook_module.hook_module import HOOKModule from msprobe.pytorch.module_processer import ModuleProcesser from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData -from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.dump_dispatch import start_dispatch +from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.dump_dispatch import run_ut_dispatch torch_version_above_or_equal_2 = torch.__version__.split('+')[0] >= '2.0' HookFn = namedtuple('hookFn', ['pre_hook', 'forward_hook', 'backward_hook', 'forward_hook_torch_version_below_2']) @@ -89,10 +89,6 @@ class Service: return if self.config.online_run_ut: - # if self.data_collector.scope and not self.data_collector.scope.check(api_or_module_name): - # return - # api_data = ApiData(name[:-1], grad_input, {}, grad_output, self.current_iter, self.current_rank) - # self.attl_send(api_data) return if self.data_collector: @@ -140,7 +136,7 @@ class Service: if api_origin: api_register.api_modularity() if self.config.online_run_ut: - start_dispatch(self.attl) + run_ut_dispatch(self.attl, True) self.switch = True logger.info_on_rank_0(f"Dump switch is turned on at step {self.current_iter}. ") if self.config.level != "L2" and not self.config.online_run_ut: @@ -156,6 +152,7 @@ class Service: return self.switch = False if self.config.online_run_ut: + run_ut_dispatch(self.attl, False) return self.data_collector.write_json() @@ -211,7 +208,7 @@ class Service: self.module_processor.node_hook(prefix + Const.BACKWARD, Const.STOP)) if self.config.level in ["mix", "L1", "L2"]: - api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API)) + api_register.initialize_hook(functools.partial(self.build_hook, BaseScope.Module_Type_API), self.config.online_run_ut) api_register.api_modularity() if Const.STATISTICS == self.config.task or Const.TENSOR == self.config.task: -- Gitee From 54583909525af1926db49e6857b725d1b6ad3767 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 19:05:35 +0800 Subject: [PATCH 751/791] renew stack --- debug/accuracy_tools/msprobe/core/common/const.py | 1 + .../msprobe/core/data_dump/data_processor/base.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 4f686598a..af3d7287e 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -111,6 +111,7 @@ class Const: CONVERT_API = { "int32_to_int64": ["cross_entropy"] } + WITHOUT_CALL_STACK = "The call stack retrieval failed." class CompareConst: diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 8e0949724..5167acdac 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -113,6 +113,8 @@ class BaseDataProcessor: ]) ]) stack_str.append(stack_line) + else: + stack_str.append(Const.WITHOUT_CALL_STACK) stack_info_struct = {name: stack_str} return stack_info_struct -- Gitee From d285e1d56f3f62fd05c70c51c658a97e1804172f Mon Sep 17 00:00:00 2001 From: gitee Date: Tue, 10 Sep 2024 19:10:09 +0800 Subject: [PATCH 752/791] fix bug --- .../pytorch/api_accuracy_checker/tensor_transport_layer/attl.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py index b2a71e56f..ccb743ef2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/attl.py @@ -41,6 +41,7 @@ class ATTL: self.message_end = False self.kill_progress = False self.check_attl_config() + self.nfs_path = None if self.session_config.nfs_path: self.nfs_path = self.session_config.nfs_path elif self.session_config.is_benchmark_device: -- Gitee From 8a5efaf06d035d7c8dcfd72efd6b365d88accef4 Mon Sep 17 00:00:00 2001 From: wuyulong11 <2284273586@qq.com> Date: Tue, 10 Sep 2024 17:41:26 +0800 Subject: [PATCH 753/791] =?UTF-8?q?=E3=80=90=E4=BF=AE=E6=94=B9=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E3=80=91=E3=80=90tbplugin=E3=80=91=E9=99=90=E5=88=B6?= =?UTF-8?q?=E6=96=87=E4=BB=B6=E9=95=BF=E5=BA=A6=E5=92=8C=E4=B8=8D=E4=B8=BA?= =?UTF-8?q?=E8=BD=AF=E9=93=BE=E6=8E=A5=20=E3=80=90=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E4=BA=BA=E3=80=91=20wuyulong?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tensorboard-plugins/tb_plugin/README.md | 6 +-- .../tb_plugin/torch_tb_profiler/consts.py | 2 + .../torch_tb_profiler/io/__init__.py | 2 +- .../tb_plugin/torch_tb_profiler/io/file.py | 43 +++++++++++++++++++ .../torch_tb_profiler/profiler/data.py | 13 ++---- 5 files changed, 52 insertions(+), 14 deletions(-) diff --git a/plugins/tensorboard-plugins/tb_plugin/README.md b/plugins/tensorboard-plugins/tb_plugin/README.md index 152a91b95..3b3edbb9a 100644 --- a/plugins/tensorboard-plugins/tb_plugin/README.md +++ b/plugins/tensorboard-plugins/tb_plugin/README.md @@ -268,7 +268,7 @@ ###### 文件导入 界面分为左侧边栏和右侧展示界面。点击左侧的Import Files或在左侧未勾选文件时点击右侧界面中心的Import Files字体，将会弹出系统文件资源管理窗，可以上传需要比对的模型网络训练日志文件。 - 注：当前最多支持上传6个文件，单个文件大小不能超过50MB。 + **注：当前最多支持上传6个文件，单个文件大小不能超过50MB。** ![Alt text](./docs/images/accuracy.PNG) ###### 已上传文件操作 @@ -319,8 +319,8 @@ * 比对方式有三种，通过Comparison Setting进行设定。 * Comparison Normal：相同iteration，后选择文件的loss值减去先选择文件的loss值。 - * Comparison Normal：相同iteration，两个文件的loss的差值的绝对值。 - * Comparison Normal：相同iteration，两个文件的loss的差值的绝对值 / 先选择文件的loss值。 + * Comparison Absolute：相同iteration，两个文件的loss的差值的绝对值。 + * Comparison Relative：相同iteration，两个文件的loss的差值的绝对值 / 先选择文件的loss值。 ### 公网URL说明 diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/consts.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/consts.py index 533effb8b..b3e202af6 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/consts.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/consts.py @@ -35,6 +35,8 @@ NODE_PROCESS_PATTERN = re.compile(r"""^(.*)_(\d+)""") MONITOR_RUN_REFRESH_INTERNAL_IN_SECONDS = 10 MAX_GPU_PER_NODE = 64 MAX_FILE_SIZE = 500 * 1024 * 1024 +MAX_LINUX_PATH_LENGTH = 4096 +MAX_WINDOWS_PATH_LENGTH = 260 View = namedtuple('View', 'id, name, display_name') OVERALL_VIEW = View(1, 'overall', 'Overview') diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/__init__.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/__init__.py index 6bd764e88..7a62ac1d8 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/__init__.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/__init__.py @@ -1,4 +1,4 @@ from .cache import Cache from .file import (BaseFileSystem, StatData, abspath, basename, download_file, exists, get_filesystem, glob, isdir, join, listdir, - makedirs, read, register_filesystem, relpath, walk, stat) + makedirs, read, register_filesystem, relpath, walk, stat, check_file_valid) diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/file.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/file.py index a833902a2..104146af4 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/file.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/io/file.py @@ -15,11 +15,13 @@ The following functionalities are added after forking: """ import glob as py_glob import os +import platform import tempfile from .. import utils from .base import BaseFileSystem, LocalPath, RemotePath, StatData from .utils import as_bytes, as_text, parse_blob_url +from ..consts import MAX_FILE_SIZE, MAX_WINDOWS_PATH_LENGTH, MAX_LINUX_PATH_LENGTH logger = utils.get_logger() @@ -86,6 +88,10 @@ class LocalFileSystem(LocalPath, BaseFileSystem): def __init__(self): pass + @staticmethod + def islink(path): + return os.path.islink(path) + def exists(self, filename): return os.path.exists(filename) @@ -613,3 +619,40 @@ def stat(filename): def read(file): with File(file, 'rb') as f: return f.read() + + +def is_link(path): + return LocalFileSystem.islink(path) + + +def is_too_big_file(filepath): + return stat(filepath).length > MAX_FILE_SIZE + + +def has_too_long_path(filepath): + if platform.system() == 'Windows' and len(filepath) > MAX_WINDOWS_PATH_LENGTH: + logger.warning( + f'The path length of the file "{filepath}" exceeds the maximum limit of {MAX_WINDOWS_PATH_LENGTH} ' + f'and will be skipped.') + return True + elif len(filepath) > MAX_WINDOWS_PATH_LENGTH: + logger.warning( + f'The path length of the file "{filepath}" exceeds the maximum limit of {MAX_LINUX_PATH_LENGTH} ' + f'and will be skipped.') + return True + else: + return False + + +def check_file_valid(filepath): + if is_link(filepath): + logger.warning(f'File "{filepath}" is a soft link and will be skipped.') + return False + if is_too_big_file(filepath): + logger.warning( + f'File "{filepath}" exceeds the maximum limit size of 500MB and will be skipped.') + return False + if has_too_long_path(filepath): + return False + return True + diff --git a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py index 7cd6f0232..f368adbab 100644 --- a/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py +++ b/plugins/tensorboard-plugins/tb_plugin/torch_tb_profiler/profiler/data.py @@ -31,7 +31,7 @@ from configparser import ConfigParser from .op_tree import OpTreeBuilder from .. import io, utils -from ..consts import InputFilesType, MAX_FILE_SIZE, INPUT_FILE_LIST +from ..consts import InputFilesType, INPUT_FILE_LIST from ..utils import href from . import trace from .communication import analyze_communication_nodes @@ -169,15 +169,8 @@ class RunProfileData(object): has_communication_overlap = False has_communication_wait_ops = False - def _check_file_size_valid(filepath): - if io.stat(filepath).length > MAX_FILE_SIZE: - logger.warning( - f'File "{filepath}" exceeds the maximum limit size of 500MB and will be skipped.') - return False - return True - for file in io.listdir(path): - if utils.is_npu_trace_path(file) and _check_file_size_valid(io.join(path, file)): + if utils.is_npu_trace_path(file) and io.check_file_valid(io.join(path, file)): has_trace = True trace_file = io.join(path, file) trace_path, trace_json = RunProfileData._preprocess_file(trace_file, cache_dir, 'Ascend') @@ -199,7 +192,7 @@ class RunProfileData(object): profile.profiler_start_ts = 0 for file in io.listdir(path): - if str(file) in INPUT_FILE_LIST and _check_file_size_valid(io.join(path, file)): + if str(file) in INPUT_FILE_LIST and io.check_file_valid(io.join(path, file)): if InputFilesType(file) == InputFilesType.KERNEL_DETAILS_CSV: has_kernel = True profile.kernel_file_path = io.join(path, file) -- Gitee From 6597a2e257a2d48b506cee629e6a070470d96027 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Wed, 4 Sep 2024 20:18:53 +0800 Subject: [PATCH 754/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E6=BA=A2=E5=87=BA=E6=A3=80=E6=B5=8B=E5=9C=A8=E9=81=87?= =?UTF-8?q?=E5=88=B0ProcessGroup=E7=B1=BB=E5=9E=8B=E6=8A=A5=E9=94=99?= =?UTF-8?q?=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/core/common/const.py | 1 - .../msprobe/core/data_dump/data_collector.py | 4 +- .../core/data_dump/data_processor/base.py | 4 +- .../data_processor/mindspore_processor.py | 1 + .../data_processor/pytorch_processor.py | 69 ++++++++----------- .../msprobe/mindspore/dump/jit_dump.py | 2 +- .../msprobe/mindspore/service.py | 10 +-- .../accuracy_tools/msprobe/pytorch/service.py | 6 +- .../data_dump/data_processor/test_base.py | 6 +- .../data_processor/test_pytorch_processor.py | 48 ++++++------- 10 files changed, 68 insertions(+), 83 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 1f508a239..7e5a7022c 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -293,7 +293,6 @@ class OverflowConst: """ Class for Overflow """ - OVERFLOW_DEBUG_MODE_ENABLE = "OVERFLOW_DEBUG_MODE_ENABLE" OVERFLOW_ORIGINAL_MODE = 0 OVERFLOW_DEBUG_MODE = 1 diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index 0dde81412..29eae188b 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -53,8 +53,8 @@ class DataCollector: def get_forward_new_output(self): return self.data_processor.get_forward_new_output() - def visit_and_clear_overflow_status(self, api_or_module_name): - self.data_processor.visit_and_clear_overflow_status(api_or_module_name) + def update_api_or_module_name(self, api_or_module_name): + self.data_processor.update_api_or_module_name(api_or_module_name) def write_json(self): self.data_writer.write_json() diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index da43b097c..842b1c45a 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -79,7 +79,6 @@ class BaseDataProcessor: self.stack_info_struct = {} self.current_api_or_module_name = None self.api_data_category = None - self.has_overflow = False self.current_iter = 0 self._return_forward_new_output = False self._forward_new_output = None @@ -202,10 +201,9 @@ class BaseDataProcessor: def update_iter(self, current_iter): self.current_iter = current_iter - def visit_and_clear_overflow_status(self, api_or_module_name): + def update_api_or_module_name(self, api_or_module_name): if self.current_api_or_module_name != api_or_module_name: self.current_api_or_module_name = api_or_module_name - self.has_overflow = False def is_dump_for_data_mode(self, forward_backward, input_output): """ diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index 435d0e9f1..8e52d09dd 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -132,6 +132,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): def __init__(self, config, data_writer): super().__init__(config, data_writer) + self.has_overflow = False self.cached_tensors_and_file_paths = {} self.real_overflow_nums = 0 self.overflow_nums = config.overflow_nums diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index ba6611647..ffcd822ea 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -1,5 +1,3 @@ -import copy -import os import zlib from dataclasses import asdict from typing import List @@ -167,12 +165,13 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): def __init__(self, config, data_writer): super().__init__(config, data_writer) + self.has_overflow = False + self.support_inf_nan = None + self.cached_inplace_api_info = {} self.cached_tensors_and_file_paths = {} self.bits_for_overflow = 8 self.real_overflow_nums = 0 self.overflow_nums = config.overflow_nums - self.forward_inplace_inputs = None - self.is_support_inf_nan = self.is_support_inf_nan() @property def is_terminated(self): @@ -183,41 +182,38 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): return True return False - @staticmethod - def is_support_inf_nan(): - return is_gpu or (hasattr(torch_npu._C, '_npu_is_support_inf_nan') and torch_npu._C._npu_is_support_inf_nan()) - - @staticmethod - def overflow_debug_mode_enable(): - overflow_mode = os.getenv(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) - return overflow_mode == Const.ENV_ENABLE - def analyze_pre_forward_inplace(self, name, module_input_output: ModuleForwardInputsOutputs): - self.forward_inplace_inputs = copy.deepcopy(module_input_output) + self.has_overflow = False + self._is_support_inf_nan() + self.cached_inplace_api_info = super().analyze_pre_forward_inplace(name, module_input_output) return None def analyze_forward_inplace(self, name, module_input_output: ModuleForwardInputsOutputs): - module_input_output.output = module_input_output.concat_args_and_kwargs() - module_input_output.args = self.forward_inplace_inputs.args - module_input_output.kwargs = self.forward_inplace_inputs.kwargs - # release memory used by forward inputs - self.forward_inplace_inputs = None - return self.analyze_forward(name, None, module_input_output) + self._is_support_inf_nan() + api_info_struct = super().analyze_forward_inplace(name, module_input_output) + if name in self.cached_inplace_api_info and name in api_info_struct: + self.cached_inplace_api_info[name].update(api_info_struct[name]) + elif name in api_info_struct: + self.cached_inplace_api_info = api_info_struct + self.handle_overflow() + return self.cached_inplace_api_info if self.has_overflow else None def analyze_forward(self, name, module, module_input_output: ModuleForwardInputsOutputs): self.has_overflow = False + self._is_support_inf_nan() api_info_struct = super().analyze_forward(name, module, module_input_output) self.handle_overflow() return api_info_struct if self.has_overflow else None def analyze_backward(self, name, module, module_input_output: ModuleBackwardInputsOutputs): self.has_overflow = False + self._is_support_inf_nan() api_info_struct = super().analyze_backward(name, module, module_input_output) self.handle_overflow() return api_info_struct if self.has_overflow else None def handle_overflow(self): - if not self.is_support_inf_nan: + if not self.support_inf_nan: self._analyze_maybe_overflow_flag() if self.has_overflow: for file_path, tensor in self.cached_tensors_and_file_paths.items(): @@ -225,29 +221,20 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): self.real_overflow_nums += 1 self.cached_tensors_and_file_paths = {} - def check_overflow_npu(self): - if self.overflow_debug_mode_enable(): - float_status = torch.zeros(self.bits_for_overflow).npu() - result = torch_npu.npu_get_float_status(float_status, OverflowConst.OVERFLOW_DEBUG_MODE) - if result.cpu()[0] != 0: - return True - else: - return False - else: - return torch_npu._C._check_overflow_npu() - - def clear_overflow_npu(self): - if self.overflow_debug_mode_enable(): - float_status = torch.zeros(self.bits_for_overflow).npu() - torch_npu.npu_clear_float_status(float_status, OverflowConst.OVERFLOW_DEBUG_MODE) - else: - torch_npu._C._clear_overflow_npu() + def _is_support_inf_nan(self): + if self.support_inf_nan is not None: + return + try: + self.support_inf_nan = is_gpu or torch_npu.npu.utils.is_support_inf_nan() + except Exception: + logger.warning(f"Unable to determine if the current device supports inf/nan mode, default not supported.") + self.support_inf_nan = False def _analyze_maybe_overflow_flag(self): try: - self.has_overflow = self.check_overflow_npu() + self.has_overflow = torch_npu.npu.utils.get_npu_overflow_flag() if self.has_overflow: - self.clear_overflow_npu() + torch_npu.npu.utils.clear_npu_overflow_flag() except Exception as e: logger.error(f"Overflow check failed, the current environment may be abnormal.") raise RuntimeError(f"overflow check failed") from e @@ -266,7 +253,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): logger.warning(f'The file path {file_path} length exceeds limit.') single_arg = super()._analyze_tensor(tensor, suffix) single_arg.update({"data_name": dump_data_name}) - if not self.has_overflow and self.is_support_inf_nan: + if not self.has_overflow and self.support_inf_nan: self._analyze_maybe_overflow_tensor(single_arg) return single_arg diff --git a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py index dde585684..329ebf53a 100644 --- a/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py +++ b/debug/accuracy_tools/msprobe/mindspore/dump/jit_dump.py @@ -22,7 +22,7 @@ def dump_jit(name, in_feat, out_feat, is_forward): else: name_template = "Jit." + result + ".backward" if JitDump.need_dump(): - JitDump.data_collector.visit_and_clear_overflow_status(name_template) + JitDump.data_collector.update_api_or_module_name(name_template) module_input_output = ModuleForwardInputsOutputs(args=in_feat, kwargs={}, output=out_feat) JitDump.data_collector.forward_data_collect(name_template, {}, pid, module_input_output) diff --git a/debug/accuracy_tools/msprobe/mindspore/service.py b/debug/accuracy_tools/msprobe/mindspore/service.py index 439a87a40..1960573ee 100644 --- a/debug/accuracy_tools/msprobe/mindspore/service.py +++ b/debug/accuracy_tools/msprobe/mindspore/service.py @@ -88,7 +88,7 @@ class Service: module_input_output = ModuleForwardInputsOutputs(args=input, kwargs=cell.input_kwargs, output=output) - self.data_collector.visit_and_clear_overflow_status(api_or_cell_name) + self.data_collector.update_api_or_module_name(api_or_cell_name) self.data_collector.forward_data_collect(api_or_cell_name, cell, pid, module_input_output) if self.data_collector.if_return_forward_new_output(): return self.data_collector.get_forward_new_output() @@ -102,7 +102,7 @@ class Service: if target_type == BaseScope.Module_Type_Module: api_or_cell_name = cell.mindstudio_reserved_name - self.data_collector.visit_and_clear_overflow_status(api_or_cell_name) + self.data_collector.update_api_or_module_name(api_or_cell_name) if self.data_collector: # 框架最新接口变更，grad_input和grad_output的含义发生了变化，与torch含义保持一致，因此此处调换顺序传入 module_input_output = ModuleBackwardInputsOutputs(grad_input=grad_output, grad_output=grad_input) @@ -131,14 +131,14 @@ class Service: backward_primitive_name = f"{updated_primitive_name}.{Const.BACKWARD}" try: if len(captured_grads) == num_tensors and hook_type == Const.INPUT: - service_instance.data_collector.visit_and_clear_overflow_status(backward_primitive_name) + service_instance.data_collector.update_api_or_module_name(backward_primitive_name) new_module_input_output = ModuleBackwardOutputs(grad_output=tuple(captured_grads)) service_instance.data_collector.backward_output_data_collect( backward_primitive_name, service_instance, os.getpid(), new_module_input_output ) captured_grads.clear() elif len(captured_grads) == num_tensors and hook_type == Const.OUTPUT: - service_instance.data_collector.visit_and_clear_overflow_status(backward_primitive_name) + service_instance.data_collector.update_api_or_module_name(backward_primitive_name) new_module_input_output = ModuleBackwardInputs(grad_input=tuple(captured_grads)) service_instance.data_collector.backward_input_data_collect( backward_primitive_name, service_instance, os.getpid(), new_module_input_output @@ -207,7 +207,7 @@ class Service: " primitive_name: {}".format(exception, primitive_name)) from exception forward_primitive_name = f"{updated_primitive_name}.{Const.FORWARD}" - service_instance.data_collector.visit_and_clear_overflow_status(forward_primitive_name) + service_instance.data_collector.update_api_or_module_name(forward_primitive_name) if service_instance.data_collector: module_input_output = ModuleForwardInputsOutputs(args=hooked_inputs, kwargs=kwargs, output=out) try: diff --git a/debug/accuracy_tools/msprobe/pytorch/service.py b/debug/accuracy_tools/msprobe/pytorch/service.py index 7da32d0f1..3721f0cc0 100644 --- a/debug/accuracy_tools/msprobe/pytorch/service.py +++ b/debug/accuracy_tools/msprobe/pytorch/service.py @@ -43,7 +43,7 @@ class Service: def pre_hook(api_or_module_name, module, args, kwargs): if module_type == BaseScope.Module_Type_Module: api_or_module_name = module.mindstudio_reserved_name - self.data_collector.visit_and_clear_overflow_status(api_or_module_name) + self.data_collector.update_api_or_module_name(api_or_module_name) if not self.switch: return args, kwargs @@ -57,7 +57,7 @@ class Service: def forward_hook(api_or_module_name, module, args, kwargs, output): if module_type == BaseScope.Module_Type_Module: api_or_module_name = module.mindstudio_reserved_name - self.data_collector.visit_and_clear_overflow_status(api_or_module_name) + self.data_collector.update_api_or_module_name(api_or_module_name) if not self.switch: return None @@ -82,7 +82,7 @@ class Service: def backward_hook(api_or_module_name, module, grad_input, grad_output): if module_type == BaseScope.Module_Type_Module: api_or_module_name = module.mindstudio_reserved_name - self.data_collector.visit_and_clear_overflow_status(api_or_module_name) + self.data_collector.update_api_or_module_name(api_or_module_name) if not self.switch: return diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py index 76182b251..2cba5e890 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_base.py @@ -142,10 +142,8 @@ class TestBaseDataProcessor(unittest.TestCase): self.processor.update_iter(5) self.assertEqual(self.processor.current_iter, 5) - def test_visit_and_clear_overflow_status(self): - self.processor.has_overflow = True - self.processor.visit_and_clear_overflow_status("new_api") - self.assertFalse(self.processor.has_overflow) + def test_update_api_or_module_name(self): + self.processor.update_api_or_module_name("new_api") self.assertEqual(self.processor.current_api_or_module_name, "new_api") def test_is_dump_for_data_mode(self): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index 7d44c49e2..1004138ae 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -14,8 +14,6 @@ from msprobe.core.data_dump.data_processor.pytorch_processor import ( OverflowCheckDataProcessor ) -from msprobe.core.common.const import Const, OverflowConst - class TestPytorchDataProcessor(unittest.TestCase): @@ -207,11 +205,13 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.config.overflow_nums = 1 self.data_writer = MagicMock() self.processor = OverflowCheckDataProcessor(self.config, self.data_writer) + self.processor.support_inf_nan = True + self.processor.has_overflow = False self.current_api_or_module_name = "test_api_name" self.api_data_category = "input" sys.modules['torch_npu'] = Mock() sys.modules['torch_npu.npu'] = Mock() - sys.modules['torch_npu._C'] = Mock() + sys.modules['torch_npu.npu.utils'] = Mock() def test_is_terminated(self): self.processor.overflow_nums = -1 @@ -224,25 +224,21 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.processor.real_overflow_nums = 1 self.assertFalse(self.processor.is_terminated) - @patch('os.getenv', return_value=Const.ENV_ENABLE) - def test_overflow_debug_mode_enable(self, mock_getenv): - result = self.processor.overflow_debug_mode_enable() - self.assertTrue(result) - mock_getenv.assert_called_once_with(OverflowConst.OVERFLOW_DEBUG_MODE_ENABLE, Const.ENV_DISABLE) - def test_analyze_pre_forward_inplace(self): - module_input_output = ModuleForwardInputsOutputs(args=(1, 2), kwargs=None, output=None) - self.processor.analyze_pre_forward_inplace(None, module_input_output) - self.assertTrue(self.processor.forward_inplace_inputs == module_input_output) - self.assertFalse(self.processor.forward_inplace_inputs is module_input_output) + with patch.object(BaseDataProcessor, "analyze_pre_forward_inplace", return_value={"name": 1}): + api_info = self.processor.analyze_pre_forward_inplace("name", "module_input_output") + self.assertEqual(self.processor.cached_inplace_api_info, {"name": 1}) + self.assertIsNone(api_info) def test_analyze_forward_inplace(self): - with patch.object(OverflowCheckDataProcessor, "analyze_forward"): - module_input_output = ModuleForwardInputsOutputs(args=(1, 2), kwargs={"k1": 3, "k2": 4}, output=None) - self.processor.forward_inplace_inputs = module_input_output - self.processor.analyze_forward_inplace("name", module_input_output) - self.assertIsNone(self.processor.forward_inplace_inputs) - self.assertEqual(module_input_output.output, (1, 2, 3, 4)) + def func(_): + self.processor.has_overflow = True + + with patch.object(BaseDataProcessor, "analyze_forward_inplace", return_value={"name": {"output": 2}}), \ + patch.object(OverflowCheckDataProcessor, "handle_overflow", new=func): + self.processor.cached_inplace_api_info = {"name": {"intput": 1}} + api_info = self.processor.analyze_forward_inplace("name", "module_input_output") + self.assertEqual(api_info, {"name": {"intput": 1, "output": 2}}) def test_analyze_forward(self): def func(_): @@ -274,19 +270,25 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.assertTrue(self.processor.has_overflow) self.assertEqual(api_info, {"min", 0}) - @patch('torch.save') + @patch('msprobe.core.data_dump.data_processor.pytorch_processor.save_pt') def test_handle_overflow(self, mock_save): self.processor.has_overflow = True self.processor.real_overflow_nums = 0 - self.processor.is_support_inf_nan = True self.processor.cached_tensors_and_file_paths = {'dump_path': torch.tensor([1.0, 2.0, 3.0])} self.processor.handle_overflow() mock_save.assert_called_once() + self.processor.has_overflow = True self.processor.handle_overflow() self.assertEqual(self.processor.real_overflow_nums, 2) - @patch('msprobe.core.data_dump.data_processor.pytorch_processor.is_gpu', return_value=True) - def test_analyze_maybe_overflow_tensor(self, _): + @patch('msprobe.core.data_dump.data_processor.pytorch_processor.is_gpu') + def test_is_support_inf_nan_gpu(self, mock_gpu_device): + self.processor.support_inf_nan = None + mock_gpu_device.return_value = True + self.processor._is_support_inf_nan() + self.assertTrue(self.processor.support_inf_nan) + + def test_analyze_maybe_overflow_tensor(self): tensor_json = {'Max': None, 'Min': None} self.processor._analyze_maybe_overflow_tensor(tensor_json) self.assertFalse(self.processor.has_overflow) -- Gitee From 631fcf128986e2c72766b549bb18ebd8e039de11 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 20:29:37 +0800 Subject: [PATCH 755/791] renew --- .../msprobe/core/common/const.py | 16 ++++++++++++--- .../accuracy_tools/msprobe/core/common/log.py | 20 +++++++++---------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 1f44a71b1..2779e2d79 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -327,12 +327,22 @@ class MsgConst: Class for log messages const """ MSPROBE_LOG_LEVEL = "MSPROBE_LOG_LEVEL" - LEVEL_ENUM = ["0", "1", "2", "3", "4"] - DEFAULT_LEVEL = 1 - LEVEL = ["DEBUG", "INFO", "WARNING", "ERROR"] + LOG_LEVEL_ENUM = ["0", "1", "2", "3", "4"] + LOG_LEVEL = ["DEBUG", "INFO", "WARNING", "ERROR"] + class LogLevel: + class DEBUG: + value = 0 + class INFO: + value = 1 + class WARNING: + value = 2 + class ERROR: + value = 3 SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] +class + class GraphMode: NPY_MODE = "NPY_MODE" STATISTIC_MODE = "STATISTIC_MODE" diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 699d3995b..20cc84824 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -13,10 +13,10 @@ class BaseLogger: @staticmethod def get_level(): input_level = os.environ.get(MsgConst.MSPROBE_LOG_LEVEL) - if input_level in MsgConst.LEVEL_ENUM: + if input_level in MsgConst.LOG_LEVEL_ENUM: return int(input_level) else: - return MsgConst.DEFAULT_LEVEL + return MsgConst.LogLevel.INFO.value def get_rank(self): return self.rank @@ -31,23 +31,23 @@ class BaseLogger: @filter_special_chars def error(self, msg): - if self.level <= MsgConst.LEVEL.index(MsgConst.LEVEL[3]): - self._print_log(MsgConst.LEVEL[3], msg) + if self.level <= MsgConst.LogLevel.ERROR.value: + self._print_log(MsgConst.LOG_LEVEL[3], msg) @filter_special_chars def warning(self, msg): - if self.level <= MsgConst.LEVEL.index(MsgConst.LEVEL[2]): - self._print_log(MsgConst.LEVEL[2], msg) + if self.level <= MsgConst.LogLevel.WARNING.value: + self._print_log(MsgConst.LOG_LEVEL[2], msg) @filter_special_chars def info(self, msg): - if self.level <= MsgConst.LEVEL.index(MsgConst.LEVEL[1]): - self._print_log(MsgConst.LEVEL[1], msg) + if self.level <= MsgConst.LogLevel.INFO.value: + self._print_log(MsgConst.LOG_LEVEL[1], msg) @filter_special_chars def debug(self, msg): - if self.level <= MsgConst.LEVEL.index(MsgConst.LEVEL[0]): - self._print_log(MsgConst.LEVEL[0], msg) + if self.level <= MsgConst.LogLevel.DEBUG.value: + self._print_log(MsgConst.LOG_LEVEL[0], msg) def on_rank_0(self, func): def func_rank_0(*args, **kwargs): -- Gitee From 112bf8d080b6670dc0119dae68d287113fb1a1aa Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 20:35:52 +0800 Subject: [PATCH 756/791] renew --- debug/accuracy_tools/msprobe/core/common/const.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 2779e2d79..7c8347f13 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -341,8 +341,6 @@ class MsgConst: SPECIAL_CHAR = ["\n", "\r", "\u007F", "\b", "\f", "\t", "\u000B", "%08", "%0a", "%0b", "%0c", "%0d", "%7f"] -class - class GraphMode: NPY_MODE = "NPY_MODE" STATISTIC_MODE = "STATISTIC_MODE" -- Gitee From d093485cfd3034ec71287ef69ee1083fb9104465 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 20:47:15 +0800 Subject: [PATCH 757/791] renew --- debug/accuracy_tools/msprobe/core/data_dump/data_collector.py | 2 +- .../msprobe/test/core_ut/data_dump/test_data_collector.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index ea8cc1487..f9c85e366 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -141,7 +141,7 @@ class DataCollector: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) logger.debug(msg) - if use_buffer: + if flush: self.data_writer.flush_data_when_buffer_is_full() else: self.write_json() diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py index fa59d1a5e..83efe9056 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/test_data_collector.py @@ -74,7 +74,7 @@ class TestDataCollector(unittest.TestCase): self.data_collector.handle_data("Tensor.add", {"min": 0}) msg = "msprobe is collecting data on Tensor.add. " mock_update_data.assert_called_with({"min": 0}, msg) - + mock_debug.assert_called_with("msg") mock_flush.assert_called() mock_write_json.assert_not_called() -- Gitee From 3ff3e7e48bcf83d099a4d4f3d0340412d5c4e29d Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 20:48:25 +0800 Subject: [PATCH 758/791] renew --- debug/accuracy_tools/msprobe/core/data_dump/data_collector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py index f9c85e366..3df0e04cd 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_collector.py @@ -141,7 +141,7 @@ class DataCollector: msg = f"msprobe is collecting data on {name}. " msg = self.update_data(data_info, msg) logger.debug(msg) - if flush: + if not flush: self.data_writer.flush_data_when_buffer_is_full() else: self.write_json() -- Gitee From e61103fe79e4d326e08e019ae3aa77d5f661d3fa Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 20:53:09 +0800 Subject: [PATCH 759/791] renew --- debug/accuracy_tools/msprobe/core/common/log.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/log.py b/debug/accuracy_tools/msprobe/core/common/log.py index 20cc84824..e44c545b3 100644 --- a/debug/accuracy_tools/msprobe/core/common/log.py +++ b/debug/accuracy_tools/msprobe/core/common/log.py @@ -13,10 +13,10 @@ class BaseLogger: @staticmethod def get_level(): input_level = os.environ.get(MsgConst.MSPROBE_LOG_LEVEL) - if input_level in MsgConst.LOG_LEVEL_ENUM: - return int(input_level) - else: + if input_level not in MsgConst.LOG_LEVEL_ENUM: return MsgConst.LogLevel.INFO.value + else: + return int(input_level) def get_rank(self): return self.rank -- Gitee From da6934da88f9450884fd54f9fb3dd01ac598a264 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 22:21:17 +0800 Subject: [PATCH 760/791] minus limitation --- debug/accuracy_tools/msprobe/core/common_config.py | 13 +++++++++---- .../msprobe/test/core_ut/test_common_config.py | 11 +++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index cd85b0a1a..dcf0e9c24 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -18,11 +18,16 @@ class CommonConfig: @staticmethod def get_step_from_string(step): - try: - borderline = int(step.split('-')[0]), int(step.split('-')[-1]) - except (ValueError, IndexError) as e: + spilted_step = step.split('-') + if len(spilted_step) == 2: + try: + borderline = int(spilted_step[0]), int(spilted_step[1]) + except (ValueError, IndexError) as e: + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, + "The connector(-) must start and end with decimal numbers.") from e + else: raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, - "The connector(-) must start and end with decimal numbers.") from e + 'The string parameter for step only supports formats like "3-5" or "5-3".') if borderline[0] <= borderline[1]: continual_step = list(range(borderline[0], borderline[1] + 1)) else: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index ac8bb84d2..5a9e3f34d 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -64,14 +64,21 @@ class TestCommonConfig(TestCase): self.assertEqual(str(the_exception), f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}step is invalid, it should be a list") - json_config.update({"step": [0, 1, "4---6?"]}) + json_config.update({"step": [0, 1, "4-6?"]}) with self.assertRaises(MsprobeException) as cm: CommonConfig(json_config) the_exception = cm.exception self.assertEqual(str(the_exception), f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}The connector(-) must start and end with decimal numbers.") - json_config.update({"step": [0, 1, "4---6"]}) + json_config.update({"step": [0, 1, "4--6"]}) + with self.assertRaises(MsprobeException) as cm: + CommonConfig(json_config) + the_exception = cm.exception + self.assertEqual(str(the_exception), + f'{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}The string parameter for step only supports formats like "3-5" or "5-3".') + + json_config.update({"step": [0, 1, "4-6"]}) common_config = CommonConfig(json_config) self.assertEqual(common_config.step, [0, 1, 4, 5, 6]) -- Gitee From f4bfe0705e5814c1bfa591368893288254de1ec7 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 22:35:56 +0800 Subject: [PATCH 761/791] renew --- .../msprobe/core/data_dump/data_processor/base.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index 5167acdac..c1a0c1357 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -104,14 +104,7 @@ class BaseDataProcessor: for (_, path, line, func, code, _) in api_stack: if not code: continue - stack_line = " ".join([ - "File", ", ".join([ - path, - " ".join(["line", str(line)]), - " ".join(["in", func]), - " ".join(["\n", code[0].strip()]) - ]) - ]) + stack_line = f"File, {path}, line {str(line)}, in {func}, \n {code[0].strip()}" stack_str.append(stack_line) else: stack_str.append(Const.WITHOUT_CALL_STACK) -- Gitee From 56c6d28ec41f270fbac50b995e2ce682f5da2be8 Mon Sep 17 00:00:00 2001 From: makai Date: Tue, 10 Sep 2024 23:09:55 +0800 Subject: [PATCH 762/791] renew --- .../msprobe/core/data_dump/data_processor/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py index c1a0c1357..4d7a75185 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/base.py @@ -104,7 +104,7 @@ class BaseDataProcessor: for (_, path, line, func, code, _) in api_stack: if not code: continue - stack_line = f"File, {path}, line {str(line)}, in {func}, \n {code[0].strip()}" + stack_line = f"File {path}, line {str(line)}, in {func}, \n {code[0].strip()}" stack_str.append(stack_line) else: stack_str.append(Const.WITHOUT_CALL_STACK) -- Gitee From a8b3c80d15ce0ccc18d397443c6c9c24925474bc Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Mon, 9 Sep 2024 16:31:02 +0800 Subject: [PATCH 763/791] Check inputs for free benchmark. --- .../pytorch/debugger/debugger_config.py | 16 +- .../pytorch/free_benchmark/common/enums.py | 28 ++++ .../msprobe/pytorch/pt_config.py | 137 +++++++++++++++-- ...47\350\203\275\345\237\272\347\272\277.md" | 145 ------------------ 4 files changed, 158 insertions(+), 168 deletions(-) delete mode 100644 "\346\227\240\346\240\207\346\235\206\346\200\247\350\203\275\345\237\272\347\272\277.md" diff --git a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py index 7c32be7cc..dc909d6e3 100644 --- a/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/debugger/debugger_config.py @@ -25,15 +25,15 @@ class DebuggerConfig: self.framework = Const.PT_FRAMEWORK if self.task == Const.FREE_BENCHMARK: - self.fuzz_device = task_config.fuzz_device if task_config.fuzz_device else 'npu' - self.handler_type = task_config.handler_type if task_config.handler_type else 'check' - self.pert_mode = task_config.pert_mode if task_config.pert_mode else 'improve_precision' - self.fuzz_level = task_config.fuzz_level if task_config.fuzz_level else 'L1' - self.fuzz_stage = task_config.fuzz_stage if task_config.fuzz_stage else 'forward' + self.fuzz_device = task_config.fuzz_device + self.handler_type = task_config.handler_type + self.pert_mode = task_config.pert_mode + self.fuzz_level = task_config.fuzz_level + self.fuzz_stage = task_config.fuzz_stage self.preheat_config = { - "if_preheat": task_config.if_preheat if task_config.if_preheat is not None else True, - "preheat_step": task_config.preheat_step if task_config.preheat_step else 15, - "max_sample": task_config.max_sample if task_config.max_sample else 20, + "if_preheat": task_config.if_preheat, + "preheat_step": task_config.preheat_step, + "max_sample": task_config.max_sample } self.online_run_ut = False diff --git a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/enums.py b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/enums.py index bfb1bbaa4..181631c62 100644 --- a/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/enums.py +++ b/debug/accuracy_tools/msprobe/pytorch/free_benchmark/common/enums.py @@ -1,3 +1,6 @@ +from msprobe.core.common.const import Const + + class PerturbationMode: ADD_NOISE = "add_noise" CHANGE_VALUE = "change_value" @@ -35,3 +38,28 @@ class FuzzLevel: BASE_LEVEL = "L1" ADV_LEVEL = "L2" REAL_LEVEL = "L3" + + +class PytorchFreeBenchmarkConst: + PERTURBATION_MODE_LIST = [ + PerturbationMode.ADD_NOISE, + PerturbationMode.CHANGE_VALUE, + PerturbationMode.IMPROVE_PRECISION, + PerturbationMode.NO_CHANGE, + PerturbationMode.BIT_NOISE, + PerturbationMode.TO_CPU, + ] + DEFAULT_MODE = PerturbationMode.IMPROVE_PRECISION + DEVICE_LIST = [DeviceType.NPU, DeviceType.CPU] + DEFAULT_DEVICE = DeviceType.NPU + HANDLER_LIST = [HandlerType.CHECK, HandlerType.FIX] + DEFAULT_HANDLER = HandlerType.CHECK + FUZZ_LEVEL_LIST = [FuzzLevel.BASE_LEVEL] + DEFAULT_FUZZ_LEVEL = FuzzLevel.BASE_LEVEL + FUZZ_STAGE_LIST = [Const.FORWARD, Const.BACKWARD] + FIX_MODE_LIST = [PerturbationMode.IMPROVE_PRECISION, PerturbationMode.TO_CPU] + DEFAULT_FUZZ_STAGE = Const.FORWARD + DEFAULT_PREHEAT_STEP = 15 + DEFAULT_MAX_SAMPLE = 20 + CPU_MODE_LIST = [PerturbationMode.TO_CPU] + FIX_STAGE_LIST = [Const.FORWARD] diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index d1d95c3e5..47a2e62fa 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -1,12 +1,19 @@ import json import os -from msprobe.core.common_config import CommonConfig, BaseConfig -from msprobe.core.common.file_check import FileOpen from msprobe.core.common.const import Const -from msprobe.pytorch.hook_module.utils import get_ops +from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common.file_check import FileOpen +from msprobe.core.common.log import logger +from msprobe.core.common_config import BaseConfig, CommonConfig from msprobe.core.grad_probe.constant import level_adp from msprobe.core.grad_probe.utils import check_numeral_list_ascend +from msprobe.pytorch.free_benchmark.common.enums import ( + DeviceType, + HandlerType, + PytorchFreeBenchmarkConst, +) +from msprobe.pytorch.hook_module.utils import get_ops class TensorConfig(BaseConfig): @@ -61,24 +68,124 @@ class OverflowCheckConfig(BaseConfig): class FreeBenchmarkCheckConfig(BaseConfig): + def __init__(self, json_config): super().__init__(json_config) - self.fuzz_device = json_config.get("fuzz_device") - self.pert_mode = json_config.get("pert_mode") - self.handler_type = json_config.get("handler_type") - self.fuzz_level = json_config.get("fuzz_level") - self.fuzz_stage = json_config.get("fuzz_stage") - self.if_preheat = json_config.get("if_preheat") - self.preheat_step = json_config.get("preheat_step") - self.max_sample = json_config.get("max_sample") + self.fuzz_device = json_config.get("fuzz_device", PytorchFreeBenchmarkConst.DEFAULT_DEVICE) + self.pert_mode = json_config.get("pert_mode", PytorchFreeBenchmarkConst.DEFAULT_MODE) + self.handler_type = json_config.get("handler_type", PytorchFreeBenchmarkConst.DEFAULT_HANDLER) + self.fuzz_level = json_config.get("fuzz_level", PytorchFreeBenchmarkConst.DEFAULT_FUZZ_LEVEL) + self.fuzz_stage = json_config.get("fuzz_stage", PytorchFreeBenchmarkConst.DEFAULT_FUZZ_STAGE) + self.if_preheat = json_config.get("if_preheat", False) + self.preheat_step = json_config.get("preheat_step", PytorchFreeBenchmarkConst.DEFAULT_PREHEAT_STEP) + self.max_sample = json_config.get("max_sample", PytorchFreeBenchmarkConst.DEFAULT_PREHEAT_STEP) self.check_freebenchmark_config() + def check_freebenchmark_config(self): - if self.if_preheat and self.handler_type == "fix": - raise Exception("Preheating is not supported in fix handler type") - if self.preheat_step and self.preheat_step == 0: - raise Exception("preheat_step cannot be 0") + self._check_pert_mode() + self._check_fuzz_device() + self._check_handler_type() + self._check_fuzz_stage() + self._check_fuzz_level() + self._check_if_preheat() + if self.handler_type == HandlerType.FIX: + self._check_fix_config() + if self.if_preheat: + self._check_preheat_config() + + def _check_pert_mode(self): + if self.pert_mode not in PytorchFreeBenchmarkConst.PERTURBATION_MODE_LIST: + logger.error_log_with_exp( + f"pert_mode is invalid, it should be one of" + f" {PytorchFreeBenchmarkConst.PERTURBATION_MODE_LIST}", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + + def _check_fuzz_device(self): + if self.fuzz_device not in PytorchFreeBenchmarkConst.DEVICE_LIST: + logger.error_log_with_exp( + f"fuzz_device is invalid, it should be one of" + f" {PytorchFreeBenchmarkConst.DEVICE_LIST}", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + if (self.fuzz_device == DeviceType.CPU) ^ ( + self.pert_mode in PytorchFreeBenchmarkConst.CPU_MODE_LIST + ): + logger.error_log_with_exp( + f"You neet to and can only set fuzz_device as {DeviceType.CPU} " + f"when pert_mode in {PytorchFreeBenchmarkConst.CPU_MODE_LIST}", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + + def _check_handler_type(self): + if self.handler_type not in PytorchFreeBenchmarkConst.HANDLER_LIST: + logger.error_log_with_exp( + f"handler_type is invalid, it should be one of" + f" {PytorchFreeBenchmarkConst.HANDLER_LIST}", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + + def _check_fuzz_stage(self): + if self.fuzz_stage not in PytorchFreeBenchmarkConst.FUZZ_STAGE_LIST: + logger.error_log_with_exp( + f"fuzz_stage is invalid, it should be one of" + f" {PytorchFreeBenchmarkConst.FUZZ_STAGE_LIST}", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + + def _check_fuzz_level(self): + if self.fuzz_level not in PytorchFreeBenchmarkConst.FUZZ_LEVEL_LIST: + logger.error_log_with_exp( + f"fuzz_level is invalid, it should be one of" + f" {PytorchFreeBenchmarkConst.FUZZ_LEVEL_LIST}", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + + def _check_if_preheat(self): + if not isinstance(self.if_preheat, bool): + logger.error_log_with_exp( + f"if_preheat is invalid, it should be a boolean", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + + def _check_preheat_config(self): + if not isinstance(self.preheat_step, int): + logger.error_log_with_exp( + f"preheat_step is invalid, it should be an integer", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + if not isinstance(self.max_sample, int): + logger.error_log_with_exp( + f"max_sample is invalid, it should be an integer", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + if self.preheat_step < 1: + logger.error_log_with_exp( + "preheat_step must be greater than 0", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + if self.max_sample < 1: + logger.error_log_with_exp( + "max_sample must be greater than 0", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + ) + def _check_fix_config(self): + if self.if_preheat: + logger.error_log_with_exp( + f"Preheating is not supported for {HandlerType.FIX} handler type" + ) + if self.fuzz_stage not in PytorchFreeBenchmarkConst.FIX_STAGE_LIST: + logger.error_log_with_exp( + f"The fuzz_stage when opening {HandlerType.FIX} handler must be one of " + f"{PytorchFreeBenchmarkConst.FIX_STAGE_LIST}" + ) + if self.pert_mode not in PytorchFreeBenchmarkConst.FIX_MODE_LIST: + logger.error_log_with_exp( + f"The pert_mode when opening {HandlerType.FIX} handler must be one of " + f"{PytorchFreeBenchmarkConst.FIX_MODE_LIST}" + ) class RunUTConfig(BaseConfig): WrapApi = get_ops() diff --git "a/\346\227\240\346\240\207\346\235\206\346\200\247\350\203\275\345\237\272\347\272\277.md" "b/\346\227\240\346\240\207\346\235\206\346\200\247\350\203\275\345\237\272\347\272\277.md" deleted file mode 100644 index 981d00b34..000000000 --- "a/\346\227\240\346\240\207\346\235\206\346\200\247\350\203\275\345\237\272\347\272\277.md" +++ /dev/null @@ -1,145 +0,0 @@ -# 无标杆工具应用场景和性能基线报告 - -## 环境信息 - -NPU：Atlas A2 训练系列产品 - -CPU： - -![输入图片说明](img/cpu_info.png) - -Torch：2.1.0 - -CANN：8.0.T5 - -除上述环境信息影响性能外，API的数量、种类以及Shape都会对性能产生影响，因此本次选取不同场景网络和不同算子进行测试。 - -## 模型信息和性能基线 - -大模型在使用msprobe工具dump数据时，建议先简化模型层数，减少dump数据量。 - -以下场景的性能基线测试数据均为多次测试后取平均值，因此实际运行时性能数据可能会根据环境状态稍有浮动。 - - -### LLaMA2-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink -其中，softmax算子为FLOAT32，输入输出均为2G大小，为模型最大显存开销的API。 -在该模型下、对无标杆工具处理模式、插装范围、扰动方式组合下性能和显存基线进行覆盖。 - -性能基线报告 -其中耗时为训练10步，去除第一步耗时所得的平均每步耗时。 - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.24 | 13.69 | 1 | 1 | 混精模式基线 | -| check | 前 | ["softmax"] | improve_precision | 0.26 | 13.69 | 1.08 | 1.66 | softmax本身为高精度，跳过 | -| check | 前 | ["softmax"] | add_noise | 0.54 | 19.17 | 2.25 | 1.40 | | -| check | 前 | ["softmax"] | bit_noise | 0.56 | 19.17 | 2.33 | 1.40 | | -| check | 前 | ["softmax"] | change_value | 0.48 | 14.9 | 2 | 1.09 | | -| check | 前 | ["softmax"] | no_change | 0.47 | 14.9 | 1.96 | 1.09 | | -| check | 前 | ["softmax"] | to_cpu | 26.45 | 22.67 | 110.21 | 1.66 | 不建议整网 | -| check | 前 | ["matmul"] | improve_precision | 0.57 | 13.69 | 2.375 | 1 | | -| check | 前 | ["matmul"] | change_value | 0.48 | 13.69 | 2 | 1 | | -| check | 前 | ["matmul"] | to_cpu | 78.43 | 19.20 | 326.79 | 1.40 | 不建议整网 | -| check | 反 | ["softmax"] | improve_precision | 6.23 | 25.69 | 25.96 | 1.88 | 不建议整网 | -| check | 反 | ["softmax"] | add_noise | | | | | 不建议整网 | -| check | 反 | ["softmax"] | bit_noise | | | | | 不建议整网 | -| check | 反 | ["softmax"] | change_value | 22.76 | 25.69 | 94.83 | 1.88 | 不建议整网 | -| check | 反 | ["softmax"] | to_cpu | 141.71 | 26.19 | 590.46 | 1.91 | 不建议整网 | -| check | 前 | [] | improve_precision | 3.45 | 18.79 | 14.37 | 1.37 | | -| check | 前 | [] | add_noise | 4.67 | 19.17 | 19.46 | 1.40 | | -| check | 前 | [] | bit_noise | 16.99 | 19.17 | 70.79 | 1.40 | | -| check | 前 | [] | no_change | 3.22 | 14.90 | 13.42 | 1.09 | | -| fix | 前 | ["softmax"] | to_cpu | | | | | 不支持整网、不支持反向 | -| fix | 前 | ["softmax"] | improve_precision | | | | | 不支持整网、不支持反向 | -| 预热 | 前 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | -| 预热 | 反 | [] | improve_precision | | | | | 低精度模型基线、只测预热的迭代 | - -### Aquila2-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | | | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | | | | | | -| check | 反 | [] | add_noise | | | | | | -| fix | 前 | [] | improve_precision | | | | | | - -### Baichuan-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | | | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | | | | | | -| check | 反 | [] | add_noise | | | | | | -| fix | 前 | [] | improve_precision | | | | | | - -### Bloom-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | | | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | | | | | | -| check | 反 | [] | add_noise | | | | | | -| fix | 前 | [] | improve_precision | | | | | | - -### Interlm-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | | | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | | | | | | -| check | 反 | [] | add_noise | | | | | | -| fix | 前 | [] | improve_precision | | | | | | - -### Qwen-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | | | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | | | | | | -| check | 反 | [] | add_noise | | | | | | -| fix | 前 | [] | improve_precision | | | | | | - -### Gemma-7B - -NUM_LAYER：1，1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelLink - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | | | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | | | | | | -| check | 反 | [] | add_noise | | | | | | -| fix | 前 | [] | improve_precision | | | | | | - -### ResNet50-Cifar -1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch。 -主要算子为conv2d，每个step有51个, 因此对conv2d进行检测。 -CV模型、依赖mmcv实现（如果不修改mmcv代码、工具无法获取step信息和反向信息，且该框架下性能暂不保证）。 -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | 0.09 | 7.63 | 1 | 1 | 基线 | -| check | 前 | ["conv2d"] | improve_precision | 0.889 | 7.94 | 9.81 | 1.04 | | -| fix | 前 | ["conv2d"] | improve_precision | 0.328 | 7.47 | 3.64 | 0.91 | | -| fix | 前 | ["conv2d"] | to_cpu | 12.23 | 7.47 | 135.88 | 0.91 | | - -### OpenSora1.0 - -1卡，主要数据类型：FLOAT16，模型来源: ascend/ModelZoo-PyTorch - -| 处理模式 | 前/反向 | 算子范围 | 扰动方式 | 耗时（s） | 显存峰值（GB） | 耗时膨胀倍数 | 显存峰值膨胀倍数 | 备注 | -|--------------------------------|-----------------------------------|-----------------|----------|--------------------------------|--------------------------------|--------------------------------|--------------------------------|--------------------------------| -| / | / | / | / | | | 1 | 1 | 混精模式基线 | -| check | 前 | [] | improve_precision | | | | | | -| check | 反 | [] | add_noise | | | | | | -| fix | 前 | [] | improve_precision | | | | | | \ No newline at end of file -- Gitee From ace1435c6f0ae0fb89a0c7104e7da0929ff98d6a Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 11 Sep 2024 09:56:39 +0800 Subject: [PATCH 764/791] renew --- .../msprobe/core/common_config.py | 13 +++++++------ .../test/core_ut/test_common_config.py | 19 +++++++++++++------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common_config.py b/debug/accuracy_tools/msprobe/core/common_config.py index dcf0e9c24..766bc46a7 100644 --- a/debug/accuracy_tools/msprobe/core/common_config.py +++ b/debug/accuracy_tools/msprobe/core/common_config.py @@ -18,20 +18,21 @@ class CommonConfig: @staticmethod def get_step_from_string(step): - spilted_step = step.split('-') - if len(spilted_step) == 2: + splited_step = step.split('-') + if len(splited_step) == 2: try: - borderline = int(spilted_step[0]), int(spilted_step[1]) + borderline = int(splited_step[0]), int(splited_step[1]) except (ValueError, IndexError) as e: raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, - "The connector(-) must start and end with decimal numbers.") from e + "The hyphen(-) must start and end with decimal numbers.") from e else: raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, - 'The string parameter for step only supports formats like "3-5" or "5-3".') + f'The string parameter for step only supports formats like "3-5". Now string parameter for step is "{step}".') if borderline[0] <= borderline[1]: continual_step = list(range(borderline[0], borderline[1] + 1)) else: - continual_step = list(range(borderline[0], borderline[1] - 1, -1)) + raise MsprobeException(MsprobeException.INVALID_PARAM_ERROR, + f'For the hyphen(-) in step, the left boundary ({borderline[0]}) cannot be greater than the right boundary ({borderline[1]}).') return continual_step def get_real_step(self, step_input): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index 5a9e3f34d..22605e1f1 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -63,25 +63,32 @@ class TestCommonConfig(TestCase): self.assertEqual(the_exception.code, MsprobeException.INVALID_PARAM_ERROR) self.assertEqual(str(the_exception), f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}step is invalid, it should be a list") - + json_config.update({"step": [0, 1, "4-6?"]}) with self.assertRaises(MsprobeException) as cm: CommonConfig(json_config) the_exception = cm.exception self.assertEqual(str(the_exception), f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}The connector(-) must start and end with decimal numbers.") - - json_config.update({"step": [0, 1, "4--6"]}) + + json_config.update({"step": [0, 1, "3--5"]}) with self.assertRaises(MsprobeException) as cm: CommonConfig(json_config) the_exception = cm.exception self.assertEqual(str(the_exception), - f'{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}The string parameter for step only supports formats like "3-5" or "5-3".') - + f'{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}The string parameter for step only supports formats like "3-5". Now string parameter for step is "3--5".') + + json_config.update({"step": [0, 1, "5-3"]}) + with self.assertRaises(MsprobeException) as cm: + CommonConfig(json_config) + the_exception = cm.exception + self.assertEqual(str(the_exception), + f'{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}For the hyphen(-) in step, the left boundary (5) cannot be greater than the right boundary (3).') + json_config.update({"step": [0, 1, "4-6"]}) common_config = CommonConfig(json_config) self.assertEqual(common_config.step, [0, 1, 4, 5, 6]) - + json_config.update({"task": Const.TENSOR}) json_config.update({"rank": [0]}) json_config.update({"step": [0]}) -- Gitee From 6953c7dd73170cb81ad9b00ec75f5322eb38d416 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Wed, 11 Sep 2024 10:10:44 +0800 Subject: [PATCH 765/791] conflict. --- debug/accuracy_tools/msprobe/pytorch/pt_config.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index a1f6b5c12..802498f0a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -174,17 +174,20 @@ class FreeBenchmarkCheckConfig(BaseConfig): def _check_fix_config(self): if self.if_preheat: logger.error_log_with_exp( - f"Preheating is not supported for {HandlerType.FIX} handler type" + f"Preheating is not supported for {HandlerType.FIX} handler type", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), ) if self.fuzz_stage not in PytorchFreeBenchmarkConst.FIX_STAGE_LIST: logger.error_log_with_exp( f"The fuzz_stage when opening {HandlerType.FIX} handler must be one of " - f"{PytorchFreeBenchmarkConst.FIX_STAGE_LIST}" + f"{PytorchFreeBenchmarkConst.FIX_STAGE_LIST}", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), ) if self.pert_mode not in PytorchFreeBenchmarkConst.FIX_MODE_LIST: logger.error_log_with_exp( f"The pert_mode when opening {HandlerType.FIX} handler must be one of " - f"{PytorchFreeBenchmarkConst.FIX_MODE_LIST}" + f"{PytorchFreeBenchmarkConst.FIX_MODE_LIST}", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR), ) class RunUTConfig(BaseConfig): -- Gitee From 14a0ad17605465f853202fdca7bcebf075bf17c5 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Wed, 11 Sep 2024 10:19:53 +0800 Subject: [PATCH 766/791] Add msg. --- .../msprobe/pytorch/pt_config.py | 36 +++++++++---------- 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 802498f0a..3dc4e894a 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -99,7 +99,7 @@ class FreeBenchmarkCheckConfig(BaseConfig): logger.error_log_with_exp( f"pert_mode is invalid, it should be one of" f" {PytorchFreeBenchmarkConst.PERTURBATION_MODE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "pert_mode"), ) def _check_fuzz_device(self): @@ -107,7 +107,7 @@ class FreeBenchmarkCheckConfig(BaseConfig): logger.error_log_with_exp( f"fuzz_device is invalid, it should be one of" f" {PytorchFreeBenchmarkConst.DEVICE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_device"), ) if (self.fuzz_device == DeviceType.CPU) ^ ( self.pert_mode in PytorchFreeBenchmarkConst.CPU_MODE_LIST @@ -115,7 +115,7 @@ class FreeBenchmarkCheckConfig(BaseConfig): logger.error_log_with_exp( f"You neet to and can only set fuzz_device as {DeviceType.CPU} " f"when pert_mode in {PytorchFreeBenchmarkConst.CPU_MODE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_device"), ) def _check_handler_type(self): @@ -123,7 +123,7 @@ class FreeBenchmarkCheckConfig(BaseConfig): logger.error_log_with_exp( f"handler_type is invalid, it should be one of" f" {PytorchFreeBenchmarkConst.HANDLER_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "handler_type"), ) def _check_fuzz_stage(self): @@ -131,7 +131,7 @@ class FreeBenchmarkCheckConfig(BaseConfig): logger.error_log_with_exp( f"fuzz_stage is invalid, it should be one of" f" {PytorchFreeBenchmarkConst.FUZZ_STAGE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_stage"), ) def _check_fuzz_level(self): @@ -139,55 +139,55 @@ class FreeBenchmarkCheckConfig(BaseConfig): logger.error_log_with_exp( f"fuzz_level is invalid, it should be one of" f" {PytorchFreeBenchmarkConst.FUZZ_LEVEL_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_level"), ) def _check_if_preheat(self): if not isinstance(self.if_preheat, bool): logger.error_log_with_exp( f"if_preheat is invalid, it should be a boolean", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "if_preheat"), ) def _check_preheat_config(self): if not isinstance(self.preheat_step, int): logger.error_log_with_exp( f"preheat_step is invalid, it should be an integer", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), - ) - if not isinstance(self.max_sample, int): - logger.error_log_with_exp( - f"max_sample is invalid, it should be an integer", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "preheat_step"), ) if self.preheat_step < 1: logger.error_log_with_exp( "preheat_step must be greater than 0", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "preheat_step"), + ) + if not isinstance(self.max_sample, int): + logger.error_log_with_exp( + f"max_sample is invalid, it should be an integer", + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "max_sample"), ) if self.max_sample < 1: logger.error_log_with_exp( "max_sample must be greater than 0", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "max_sample"), ) def _check_fix_config(self): if self.if_preheat: logger.error_log_with_exp( f"Preheating is not supported for {HandlerType.FIX} handler type", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "if_preheat"), ) if self.fuzz_stage not in PytorchFreeBenchmarkConst.FIX_STAGE_LIST: logger.error_log_with_exp( f"The fuzz_stage when opening {HandlerType.FIX} handler must be one of " f"{PytorchFreeBenchmarkConst.FIX_STAGE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_stage"), ) if self.pert_mode not in PytorchFreeBenchmarkConst.FIX_MODE_LIST: logger.error_log_with_exp( f"The pert_mode when opening {HandlerType.FIX} handler must be one of " f"{PytorchFreeBenchmarkConst.FIX_MODE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR), + MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "pert_mode"), ) class RunUTConfig(BaseConfig): -- Gitee From 1cec6ed37eeac103ca6c37367607862008427f47 Mon Sep 17 00:00:00 2001 From: gitee Date: Wed, 11 Sep 2024 10:41:51 +0800 Subject: [PATCH 767/791] fix --- debug/accuracy_tools/msprobe/core/common/const.py | 1 + .../tensor_transport_layer/dump_dispatch.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/const.py b/debug/accuracy_tools/msprobe/core/common/const.py index 4f686598a..23248a110 100644 --- a/debug/accuracy_tools/msprobe/core/common/const.py +++ b/debug/accuracy_tools/msprobe/core/common/const.py @@ -97,6 +97,7 @@ class Const: CPU_LOWERCASE = 'cpu' CUDA_LOWERCASE = 'cuda' DISTRIBUTED = 'Distributed' + ATEN = "Aten" INPLACE_LIST = [ "broadcast", "all_reduce", "reduce", "all_gather", "gather", "scatter", "reduce_scatter", diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py index 8d7b30199..aa19e5ef1 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py @@ -40,15 +40,15 @@ class AccuracyCheckerDispatch(TorchDispatchMode): self.counter = counter self.aten_ops_blacklist = [] self.npu_adjust_autogard = [] - self.aten_ops_blacklist = yaml_file.get('aten_ops_blacklist') - self.npu_adjust_autogard = yaml_file.get('npu_adjust_autogard') + self.aten_ops_blacklist = yaml_file.get('aten_ops_blacklist', []) + self.npu_adjust_autogard = yaml_file.get('npu_adjust_autogard', []) def enable_autogard(self, aten_api): if aten_api in self.npu_adjust_autogard: torch._C._dispatch_tls_set_dispatch_key_excluded(torch._C.DispatchKey.AutogradFunctionality, False) def __torch_dispatch__(self, func, types, args=None, kwargs=None): - func_name_split_list = func.__name__.split(".") + func_name_split_list = func.__name__.split(Const.SEP) aten_api = func_name_split_list[0] self.enable_autogard(aten_api) if aten_api in self.aten_ops_blacklist: @@ -58,7 +58,7 @@ class AccuracyCheckerDispatch(TorchDispatchMode): res = func(*args, **kwargs) cur_rank = get_tensor_rank(args, res) cur_api_number = self.counter.index_dict.setdefault(aten_api, 0) - api_name = f'Aten{Const.SEP}{aten_api}{Const.SEP}{cur_api_number}' + api_name = f'{Const.ATEN}{Const.SEP}{aten_api}{Const.SEP}{cur_api_number}' logger.info(f"tools is dumping api: {api_name}") api_data = ApiData(api_name, args, kwargs, res, 0, cur_rank) if "device" in api_data.kwargs: -- Gitee From df9bc72e7c95571e4bb6b04a814d062389fc5367 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 11 Sep 2024 10:53:06 +0800 Subject: [PATCH 768/791] renew --- debug/accuracy_tools/msprobe/core/common/exceptions.py | 4 ++-- .../core/data_dump/data_processor/mindspore_processor.py | 3 ++- .../core/data_dump/data_processor/pytorch_processor.py | 5 +++-- .../data_dump/data_processor/test_mindspore_processor.py | 3 ++- .../msprobe/test/core_ut/test_common_config.py | 2 +- 5 files changed, 10 insertions(+), 7 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/exceptions.py b/debug/accuracy_tools/msprobe/core/common/exceptions.py index 8b8b9eb94..1ab5500a3 100644 --- a/debug/accuracy_tools/msprobe/core/common/exceptions.py +++ b/debug/accuracy_tools/msprobe/core/common/exceptions.py @@ -13,8 +13,8 @@ class MsprobeException(CodedException): OVERFLOW_NUMS_ERROR = 1 err_strs = { - INVALID_PARAM_ERROR: "[msprobe] 无效参数：", - OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数：" + INVALID_PARAM_ERROR: "[msprobe] Invalid parameter: ", + OVERFLOW_NUMS_ERROR: "[msprobe] Exceeded preset overflow number, current overflow number: " } diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index a72096804..dd0bfb500 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -23,6 +23,7 @@ from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import (BaseDataProcessor, TensorStatInfo, ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs) from msprobe.core.common.file_utils import path_len_exceeds_limit +from msprobe.core.common.exceptions import MsprobeException from msprobe.mindspore.dump.hook_cell.wrap_functional import load_ops_functions from msprobe.mindspore.common.utils import convert_bf16_to_fp32, save_tensor_as_npy from msprobe.mindspore.common.log import logger @@ -139,7 +140,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): if self.overflow_nums == -1: return False if self.real_overflow_nums >= self.overflow_nums: - logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") + logger.info(MsprobeException.OVERFLOW_NUMS_ERROR+str(self.real_overflow_nums)) return True return False diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index ba6611647..d63da645f 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -8,7 +8,8 @@ import numpy as np import torch from msprobe.core.common.file_utils import path_len_exceeds_limit, change_mode from msprobe.core.common.log import logger -from msprobe.core.common.const import Const, OverflowConst, FileCheckConst +from msprobe.core.common.exceptions import MsprobeException +from msprobe.core.common.const import Const, OverflowConst from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, ModuleBackwardInputsOutputs, \ ModuleForwardInputsOutputs, TensorStatInfo from msprobe.pytorch.free_benchmark import FreeBenchmarkCheck, UnequalRow @@ -179,7 +180,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): if self.overflow_nums == -1: return False if self.real_overflow_nums >= self.overflow_nums: - logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") + logger.info(MsprobeException.OVERFLOW_NUMS_ERROR+str(self.real_overflow_nums)) return True return False diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py index e097f9c61..d96eece32 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -22,6 +22,7 @@ import mindspore as ms from mindspore import Tensor import numpy as np +from msprobe.core.common.exceptions import MsprobeException from msprobe.core.data_dump.data_processor.base import BaseDataProcessor from msprobe.core.data_dump.data_processor.mindspore_processor import ( MindsporeDataProcessor, @@ -186,7 +187,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.data_processor.real_overflow_nums = 2 self.data_processor.overflow_nums = 2 self.assertTrue(self.data_processor.is_terminated) - mock_info.assert_called_with("[msprobe] 超过预设溢出次数 当前溢出次数: 2") + mock_info.assert_called_with(f"{MsprobeException.OVERFLOW_NUMS_ERROR}2") self.data_processor.overflow_nums = 3 self.assertFalse(self.data_processor.is_terminated) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py index 22605e1f1..244455980 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/test_common_config.py @@ -69,7 +69,7 @@ class TestCommonConfig(TestCase): CommonConfig(json_config) the_exception = cm.exception self.assertEqual(str(the_exception), - f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}The connector(-) must start and end with decimal numbers.") + f"{MsprobeException.err_strs.get(MsprobeException.INVALID_PARAM_ERROR)}The hyphen(-) must start and end with decimal numbers.") json_config.update({"step": [0, 1, "3--5"]}) with self.assertRaises(MsprobeException) as cm: -- Gitee From 464deacb2f93aa797d4f2004cc349f7be61fde08 Mon Sep 17 00:00:00 2001 From: jiandaobao Date: Wed, 11 Sep 2024 10:31:02 +0800 Subject: [PATCH 769/791] Add msg. --- .../msprobe/pytorch/pt_config.py | 92 +++++++++++-------- 1 file changed, 54 insertions(+), 38 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/pt_config.py b/debug/accuracy_tools/msprobe/pytorch/pt_config.py index 3dc4e894a..1c0572a3d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/pt_config.py +++ b/debug/accuracy_tools/msprobe/pytorch/pt_config.py @@ -96,98 +96,114 @@ class FreeBenchmarkCheckConfig(BaseConfig): def _check_pert_mode(self): if self.pert_mode not in PytorchFreeBenchmarkConst.PERTURBATION_MODE_LIST: - logger.error_log_with_exp( + msg = ( f"pert_mode is invalid, it should be one of" - f" {PytorchFreeBenchmarkConst.PERTURBATION_MODE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "pert_mode"), + f" {PytorchFreeBenchmarkConst.PERTURBATION_MODE_LIST}" + ) + logger.error_log_with_exp( + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) def _check_fuzz_device(self): if self.fuzz_device not in PytorchFreeBenchmarkConst.DEVICE_LIST: - logger.error_log_with_exp( + msg = ( f"fuzz_device is invalid, it should be one of" - f" {PytorchFreeBenchmarkConst.DEVICE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_device"), + f" {PytorchFreeBenchmarkConst.DEVICE_LIST}" + ) + logger.error_log_with_exp( + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) if (self.fuzz_device == DeviceType.CPU) ^ ( self.pert_mode in PytorchFreeBenchmarkConst.CPU_MODE_LIST ): - logger.error_log_with_exp( + msg = ( f"You neet to and can only set fuzz_device as {DeviceType.CPU} " - f"when pert_mode in {PytorchFreeBenchmarkConst.CPU_MODE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_device"), + f"when pert_mode in {PytorchFreeBenchmarkConst.CPU_MODE_LIST}" + ) + logger.error_log_with_exp( + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) def _check_handler_type(self): if self.handler_type not in PytorchFreeBenchmarkConst.HANDLER_LIST: - logger.error_log_with_exp( + msg = ( f"handler_type is invalid, it should be one of" - f" {PytorchFreeBenchmarkConst.HANDLER_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "handler_type"), + f" {PytorchFreeBenchmarkConst.HANDLER_LIST}" + ) + logger.error_log_with_exp( + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) def _check_fuzz_stage(self): if self.fuzz_stage not in PytorchFreeBenchmarkConst.FUZZ_STAGE_LIST: - logger.error_log_with_exp( + msg = ( f"fuzz_stage is invalid, it should be one of" - f" {PytorchFreeBenchmarkConst.FUZZ_STAGE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_stage"), + f" {PytorchFreeBenchmarkConst.FUZZ_STAGE_LIST}" + ) + logger.error_log_with_exp( + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) def _check_fuzz_level(self): if self.fuzz_level not in PytorchFreeBenchmarkConst.FUZZ_LEVEL_LIST: - logger.error_log_with_exp( + msg = ( f"fuzz_level is invalid, it should be one of" - f" {PytorchFreeBenchmarkConst.FUZZ_LEVEL_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_level"), + f" {PytorchFreeBenchmarkConst.FUZZ_LEVEL_LIST}" + ) + logger.error_log_with_exp( + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) def _check_if_preheat(self): if not isinstance(self.if_preheat, bool): + msg = "if_preheat is invalid, it should be a boolean" logger.error_log_with_exp( - f"if_preheat is invalid, it should be a boolean", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "if_preheat"), + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) def _check_preheat_config(self): if not isinstance(self.preheat_step, int): + msg = "preheat_step is invalid, it should be an integer" logger.error_log_with_exp( - f"preheat_step is invalid, it should be an integer", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "preheat_step"), + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) - if self.preheat_step < 1: + if self.preheat_step <= 0: + msg = "preheat_step must be greater than 0" logger.error_log_with_exp( - "preheat_step must be greater than 0", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "preheat_step"), + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) if not isinstance(self.max_sample, int): + msg = "max_sample is invalid, it should be an integer" logger.error_log_with_exp( - f"max_sample is invalid, it should be an integer", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "max_sample"), + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) - if self.max_sample < 1: + if self.max_sample <= 0: + msg = "max_sample must be greater than 0" logger.error_log_with_exp( - "max_sample must be greater than 0", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "max_sample"), + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) def _check_fix_config(self): if self.if_preheat: + msg = f"Preheating is not supported for {HandlerType.FIX} handler type" logger.error_log_with_exp( - f"Preheating is not supported for {HandlerType.FIX} handler type", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "if_preheat"), + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) if self.fuzz_stage not in PytorchFreeBenchmarkConst.FIX_STAGE_LIST: - logger.error_log_with_exp( + msg = ( f"The fuzz_stage when opening {HandlerType.FIX} handler must be one of " - f"{PytorchFreeBenchmarkConst.FIX_STAGE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "fuzz_stage"), + f"{PytorchFreeBenchmarkConst.FIX_STAGE_LIST}" ) - if self.pert_mode not in PytorchFreeBenchmarkConst.FIX_MODE_LIST: logger.error_log_with_exp( + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) + ) + if self.pert_mode not in PytorchFreeBenchmarkConst.FIX_MODE_LIST: + msg = ( f"The pert_mode when opening {HandlerType.FIX} handler must be one of " - f"{PytorchFreeBenchmarkConst.FIX_MODE_LIST}", - MsprobeException(MsprobeException.INVALID_PARAM_ERROR, "pert_mode"), + f"{PytorchFreeBenchmarkConst.FIX_MODE_LIST}" + ) + logger.error_log_with_exp( + msg, MsprobeException(MsprobeException.INVALID_PARAM_ERROR, msg) ) class RunUTConfig(BaseConfig): -- Gitee From 4a45fdaab292f7e474e836649c695a8e028d0647 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 11 Sep 2024 11:16:38 +0800 Subject: [PATCH 770/791] renew --- debug/accuracy_tools/msprobe/core/common/exceptions.py | 4 ++-- .../core/data_dump/data_processor/mindspore_processor.py | 3 +-- .../core/data_dump/data_processor/pytorch_processor.py | 5 ++--- .../data_dump/data_processor/test_mindspore_processor.py | 3 +-- 4 files changed, 6 insertions(+), 9 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/exceptions.py b/debug/accuracy_tools/msprobe/core/common/exceptions.py index 1ab5500a3..8b8b9eb94 100644 --- a/debug/accuracy_tools/msprobe/core/common/exceptions.py +++ b/debug/accuracy_tools/msprobe/core/common/exceptions.py @@ -13,8 +13,8 @@ class MsprobeException(CodedException): OVERFLOW_NUMS_ERROR = 1 err_strs = { - INVALID_PARAM_ERROR: "[msprobe] Invalid parameter: ", - OVERFLOW_NUMS_ERROR: "[msprobe] Exceeded preset overflow number, current overflow number: " + INVALID_PARAM_ERROR: "[msprobe] 无效参数：", + OVERFLOW_NUMS_ERROR: "[msprobe] 超过预设溢出次数 当前溢出次数：" } diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index dd0bfb500..a72096804 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -23,7 +23,6 @@ from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import (BaseDataProcessor, TensorStatInfo, ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs) from msprobe.core.common.file_utils import path_len_exceeds_limit -from msprobe.core.common.exceptions import MsprobeException from msprobe.mindspore.dump.hook_cell.wrap_functional import load_ops_functions from msprobe.mindspore.common.utils import convert_bf16_to_fp32, save_tensor_as_npy from msprobe.mindspore.common.log import logger @@ -140,7 +139,7 @@ class OverflowCheckDataProcessor(MindsporeDataProcessor): if self.overflow_nums == -1: return False if self.real_overflow_nums >= self.overflow_nums: - logger.info(MsprobeException.OVERFLOW_NUMS_ERROR+str(self.real_overflow_nums)) + logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") return True return False diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index d63da645f..ba6611647 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -8,8 +8,7 @@ import numpy as np import torch from msprobe.core.common.file_utils import path_len_exceeds_limit, change_mode from msprobe.core.common.log import logger -from msprobe.core.common.exceptions import MsprobeException -from msprobe.core.common.const import Const, OverflowConst +from msprobe.core.common.const import Const, OverflowConst, FileCheckConst from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, ModuleBackwardInputsOutputs, \ ModuleForwardInputsOutputs, TensorStatInfo from msprobe.pytorch.free_benchmark import FreeBenchmarkCheck, UnequalRow @@ -180,7 +179,7 @@ class OverflowCheckDataProcessor(PytorchDataProcessor): if self.overflow_nums == -1: return False if self.real_overflow_nums >= self.overflow_nums: - logger.info(MsprobeException.OVERFLOW_NUMS_ERROR+str(self.real_overflow_nums)) + logger.info(f"[msprobe] 超过预设溢出次数 当前溢出次数: {self.real_overflow_nums}") return True return False diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py index d96eece32..e097f9c61 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -22,7 +22,6 @@ import mindspore as ms from mindspore import Tensor import numpy as np -from msprobe.core.common.exceptions import MsprobeException from msprobe.core.data_dump.data_processor.base import BaseDataProcessor from msprobe.core.data_dump.data_processor.mindspore_processor import ( MindsporeDataProcessor, @@ -187,7 +186,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.data_processor.real_overflow_nums = 2 self.data_processor.overflow_nums = 2 self.assertTrue(self.data_processor.is_terminated) - mock_info.assert_called_with(f"{MsprobeException.OVERFLOW_NUMS_ERROR}2") + mock_info.assert_called_with("[msprobe] 超过预设溢出次数 当前溢出次数: 2") self.data_processor.overflow_nums = 3 self.assertFalse(self.data_processor.is_terminated) -- Gitee From b6673efdcf1df61bc2ece761229ff581f49d309b Mon Sep 17 00:00:00 2001 From: gitee Date: Wed, 11 Sep 2024 11:22:05 +0800 Subject: [PATCH 771/791] restrict precision range --- .../msprobe/pytorch/api_accuracy_checker/common/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index 61a163a2a..44a1b56f2 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -33,8 +33,8 @@ class Config: raise ValueError(f"{key} must be one of {validators.keys()}") if not isinstance(value, validators.get(key)): raise ValueError(f"{key} must be {validators[key].__name__} type") - if key == 'precision' and value < 0: - raise ValueError("precision must be greater than 0") + if key == 'precision' and (value < 0 or value > 20): + raise ValueError("precision must be greater than 0 and less than 21") if key == 'white_list': RunUTConfig.check_filter_list_config(key, value) if key == 'black_list': -- Gitee From 292b504c5b5a38a34d795a618c2373beef50f797 Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Wed, 11 Sep 2024 11:25:36 +0800 Subject: [PATCH 772/791] bug_fix_for_calculate_uncovered_comm_time --- .../compare_backend/profiling_parser/npu_profiling_parser.py | 1 + 1 file changed, 1 insertion(+) diff --git a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py index 9ad630daf..8ffb2dadf 100644 --- a/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py +++ b/profiler/compare_tools/compare_backend/profiling_parser/npu_profiling_parser.py @@ -222,6 +222,7 @@ class NPUProfilingParser(BaseProfilingParser): else: last_4_task_mode_dict[task_event.tid] = f"{last_4_task_mode[1:]}O" if last_4_task_mode else "OOOO" uncovered_communication_events = list(filter(lambda x: x.is_comm_not_overlap(), self._overlap_analysis)) + uncovered_communication_events.sort(key=lambda x: x.start_time) group_comm_time_dict = {} for comm_tid, tid_list in self._group_comm_tid_dict.items(): min_wait_time = float("inf") -- Gitee From 2f1b832c5f61f026af36c5cf8a3230c607398491 Mon Sep 17 00:00:00 2001 From: gitee Date: Wed, 11 Sep 2024 11:26:50 +0800 Subject: [PATCH 773/791] fix bug --- .../pytorch/api_accuracy_checker/run_ut/run_ut_utils.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index 0441619e3..58a329621 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -2,6 +2,13 @@ import os import re import torch +try: + import torch_npu +except ImportError: + current_device = "cuda" +else: + current_device = "npu" + from msprobe.core.common.const import FileCheckConst from msprobe.core.common.file_utils import FileChecker from msprobe.pytorch.hook_module.wrap_aten import AtenOPTemplate -- Gitee From b34eb2980f7966c468f8b4bc39a83de44c6fdb0a Mon Sep 17 00:00:00 2001 From: gitee Date: Wed, 11 Sep 2024 11:31:20 +0800 Subject: [PATCH 774/791] fix --- .../tensor_transport_layer/dump_dispatch.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py index aa19e5ef1..ab084f310 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py @@ -14,12 +14,12 @@ from msprobe.core.common.file_utils import load_yaml def singleton(cls): _instance = {} - - def inner(): + + def wrapper(*args, **kwargs): if cls not in _instance: - _instance[cls] = cls() + _instance[cls] = cls(*args, **kwargs) return _instance[cls] - return inner + return wrapper @singleton @@ -43,10 +43,6 @@ class AccuracyCheckerDispatch(TorchDispatchMode): self.aten_ops_blacklist = yaml_file.get('aten_ops_blacklist', []) self.npu_adjust_autogard = yaml_file.get('npu_adjust_autogard', []) - def enable_autogard(self, aten_api): - if aten_api in self.npu_adjust_autogard: - torch._C._dispatch_tls_set_dispatch_key_excluded(torch._C.DispatchKey.AutogradFunctionality, False) - def __torch_dispatch__(self, func, types, args=None, kwargs=None): func_name_split_list = func.__name__.split(Const.SEP) aten_api = func_name_split_list[0] @@ -70,6 +66,10 @@ class AccuracyCheckerDispatch(TorchDispatchMode): self.counter.index_dict[aten_api] += 1 return res + + def enable_autogard(self, aten_api): + if aten_api in self.npu_adjust_autogard: + torch._C._dispatch_tls_set_dispatch_key_excluded(torch._C.DispatchKey.AutogradFunctionality, False) def dispatch4data(func, attl, status): -- Gitee From e89a14259a3f834c24c0166c4d3c45eb481e31e8 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 11 Sep 2024 11:34:10 +0800 Subject: [PATCH 775/791] renew --- .../msprobe/core/data_dump/data_processor/mindspore_processor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py index c6484a3b7..483c1d424 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/mindspore_processor.py @@ -23,7 +23,6 @@ from msprobe.core.common.const import Const from msprobe.core.data_dump.data_processor.base import (BaseDataProcessor, TensorStatInfo, ModuleForwardInputsOutputs, ModuleBackwardInputsOutputs) from msprobe.core.common.file_utils import path_len_exceeds_limit -from msprobe.mindspore.dump.hook_cell.wrap_functional import load_ops_functions from msprobe.mindspore.common.utils import convert_bf16_to_fp32, save_tensor_as_npy from msprobe.mindspore.common.log import logger from msprobe.mindspore.dump.hook_cell.api_registry import api_register -- Gitee From 237ae34aade00fb1fb62a670849ced97429ca196 Mon Sep 17 00:00:00 2001 From: zhaolei Date: Wed, 11 Sep 2024 11:22:41 +0800 Subject: [PATCH 776/791] =?UTF-8?q?advisor=20=E4=BF=AE=E5=A4=8D=E5=BB=BA?= =?UTF-8?q?=E8=AE=AE=E8=BF=87=E9=95=BF=E5=AF=BC=E8=87=B4=E5=9C=A8python3.7?= =?UTF-8?q?=E4=B8=8B=E6=9F=90=E4=BA=9B=E5=88=86=E8=BE=A8=E7=8E=87=E8=BE=83?= =?UTF-8?q?=E5=B0=8F=E6=9C=BA=E5=99=A8=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- profiler/README.md | 1 + profiler/advisor/rules/bandwidth_contention.yaml | 6 +++--- profiler/version.txt | 2 +- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/profiler/README.md b/profiler/README.md index 0ab2dd4ba..50c82eada 100644 --- a/profiler/README.md +++ b/profiler/README.md @@ -107,6 +107,7 @@ Successfully installed msprof-analyze-{version} | profiler版本 | 发布日期 | 下载链接 | 校验码 | |------------|------------|-------------------------------------------------------------------------------------------------------------------------------------------| ----------------------------------------------------------- | + | 1.2.4 | 2024-09-11 | [msprof_analyze-1.2.4-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.4/msprof_analyze-1.2.4-py3-none-any.whl) | 5080d4d918df0fb152b390236f5e88687a39cd6904d8bf452faf86b8fe6ce571 | | 1.2.3 | 2024-08-29 | [msprof_analyze-1.2.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.3/msprof_analyze-1.2.3-py3-none-any.whl) | 354a55747f64ba1ec6ee6fe0f05a53e84e1b403ee0341ec40cc216dd25fda14c | | 1.2.2 | 2024-08-23 | [msprof_analyze-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.2/msprof_analyze-1.2.2-py3-none-any.whl) | ed92a8e4eaf5ada8a2b4079072ec0cc42501b1b1f2eb00c8fdcb077fecb4ae02 | | 1.2.1 | 2024-08-14 | [msprof_analyze-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/profiler/package/1.2.1/msprof_analyze-1.2.1-py3-none-any.whl) | 7acd477417bfb3ea29029dadf175d019ad3212403b7e11dc1f87e84c2412c078 | diff --git a/profiler/advisor/rules/bandwidth_contention.yaml b/profiler/advisor/rules/bandwidth_contention.yaml index 2ad30d8aa..0f25cc370 100644 --- a/profiler/advisor/rules/bandwidth_contention.yaml +++ b/profiler/advisor/rules/bandwidth_contention.yaml @@ -1,6 +1,6 @@ -problem: "The SDMA bandwidth is lower than {threshold} when computing and communication tasks are performed concurrently. -Generally, parallel computing and communication improves the running efficiency of the model. Concurrent computing -and communication tasks may affect the communication bandwidth." +problem: "The SDMA bandwidth is lower than {threshold} when computing and communication tasks are performed \n +concurrently. Generally, parallel computing and communication improves the running efficiency of the model. \n +Concurrent computing and communication tasks may affect the communication bandwidth." sdma_baseline: 18 #M threshold: 0.8 top_num: 10 diff --git a/profiler/version.txt b/profiler/version.txt index e2cac26c1..b966e81a4 100644 --- a/profiler/version.txt +++ b/profiler/version.txt @@ -1 +1 @@ -1.2.3 \ No newline at end of file +1.2.4 \ No newline at end of file -- Gitee From d6fa8833c64114c758045e2cbd4d436602974892 Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 11 Sep 2024 12:26:27 +0800 Subject: [PATCH 777/791] renew --- .../data_dump/data_processor/test_mindspore_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py index f43bc397f..6b3f1b74d 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -179,7 +179,7 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.assertEqual(mock_save.call_args_list[1][0], (tensor2, "tensor2")) - def test_is_terminated(self): + def test_is_terminated(self, mock_info): self.data_processor.overflow_nums = -1 self.assertFalse(self.data_processor.is_terminated) self.data_processor.real_overflow_nums = 2 -- Gitee From f9637c23a28f1358d4808488dbd4ab1efe3fa2be Mon Sep 17 00:00:00 2001 From: makai Date: Wed, 11 Sep 2024 12:43:31 +0800 Subject: [PATCH 778/791] renew --- .../data_dump/data_processor/test_mindspore_processor.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py index 6b3f1b74d..5ecd508ac 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_mindspore_processor.py @@ -179,13 +179,12 @@ class TestOverflowCheckDataProcessor(unittest.TestCase): self.assertEqual(mock_save.call_args_list[1][0], (tensor2, "tensor2")) - def test_is_terminated(self, mock_info): + def test_is_terminated(self): self.data_processor.overflow_nums = -1 self.assertFalse(self.data_processor.is_terminated) self.data_processor.real_overflow_nums = 2 self.data_processor.overflow_nums = 2 self.assertTrue(self.data_processor.is_terminated) - mock_info.assert_called_with("[msprobe] 超过预设溢出次数 当前溢出次数: 2") self.data_processor.overflow_nums = 3 self.assertFalse(self.data_processor.is_terminated) -- Gitee From 1fa8ffa8edd55a2c811d84a724d8d911e757947f Mon Sep 17 00:00:00 2001 From: gitee Date: Wed, 11 Sep 2024 14:23:46 +0800 Subject: [PATCH 779/791] fix --- .../tensor_transport_layer/dump_dispatch.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py index ab084f310..43dd7567d 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/tensor_transport_layer/dump_dispatch.py @@ -14,12 +14,13 @@ from msprobe.core.common.file_utils import load_yaml def singleton(cls): _instance = {} - - def wrapper(*args, **kwargs): + + @wraps(cls) + def inner(): if cls not in _instance: - _instance[cls] = cls(*args, **kwargs) + _instance[cls] = cls() return _instance[cls] - return wrapper + return inner @singleton -- Gitee From 74e3c6dfd286682e393e8de32d3e6add3cd39434 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Wed, 11 Sep 2024 16:33:16 +0800 Subject: [PATCH 780/791] fix write_json problem --- .../accuracy_tools/msprobe/core/data_dump/json_writer.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py index c970cec70..5a14b5b52 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/json_writer.py @@ -91,6 +91,9 @@ class DataWriter: save_json(file_path, self.cache_construct, indent=1) def write_json(self): - self.write_data_json(self.dump_file_path) - self.write_stack_info_json(self.stack_file_path) - self.write_construct_info_json(self.construct_file_path) + if self.cache_data.get(Const.DATA): + self.write_data_json(self.dump_file_path) + if self.cache_stack: + self.write_stack_info_json(self.stack_file_path) + if self.cache_construct: + self.write_construct_info_json(self.construct_file_path) -- Gitee From cc38328195672bb63264427306c0a3141aa47862 Mon Sep 17 00:00:00 2001 From: gitee Date: Wed, 11 Sep 2024 16:49:31 +0800 Subject: [PATCH 781/791] fix --- .../api_accuracy_checker/run_ut/run_ut_utils.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py index 58a329621..d83b31f7b 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/run_ut_utils.py @@ -21,7 +21,7 @@ hf_32_standard_api = ["conv1d", "conv2d"] not_detach_set = {'resize_', 'resize_as_', 'set_', 'transpose_', 't_', 'squeeze_', 'unsqueeze_'} not_raise_dtype_set = {'type_as'} -RAISE_PRECISION = { +PRECISION_MAPPING = { torch.float16: torch.float32, torch.bfloat16: torch.float32, torch.float32: torch.float64 @@ -101,7 +101,7 @@ def raise_bench_data_dtype(api_name, arg, raise_dtype=None): ''' if api_name in hf_32_standard_api and arg.dtype == torch.float32: return arg - if raise_dtype is None or arg.dtype not in RAISE_PRECISION or raise_dtype == arg.dtype: + if raise_dtype is None or arg.dtype not in PRECISION_MAPPING or raise_dtype == arg.dtype: return arg return arg.type(raise_dtype) @@ -147,7 +147,7 @@ def generate_cpu_params(input_args, input_kwargs, need_backward, api_name): return arg_in def is_tensor_with_raise_precision(arg_in, check_kwargs=False): - if arg_in.dtype in RAISE_PRECISION: + if arg_in.dtype in PRECISION_MAPPING: return True if check_kwargs and arg_in.dtype in [torch.half, torch.bfloat16]: return True @@ -166,7 +166,7 @@ def generate_cpu_params(input_args, input_kwargs, need_backward, api_name): need_raise_dtypes = recursive_find_dtypes(input_args) need_raise_dtypes.update(recursive_find_dtypes(input_kwargs, check_kwargs=True)) if len(need_raise_dtypes) == 1: - raise_dtype = RAISE_PRECISION.get(need_raise_dtypes.pop(), torch.float32) + raise_dtype = PRECISION_MAPPING.get(need_raise_dtypes.pop(), torch.float32) elif len(need_raise_dtypes) >= 2: raise_dtype = torch.float32 @@ -174,4 +174,4 @@ def generate_cpu_params(input_args, input_kwargs, need_backward, api_name): is_detach = api_name not in not_detach_set cpu_args = recursive_arg_to_cpu(input_args, is_detach, raise_dtype=raise_dtype) cpu_kwargs = {key: recursive_arg_to_cpu(value, key != "out" and is_detach, raise_dtype=raise_dtype) for key, value in input_kwargs.items()} - return cpu_args, cpu_kwargs \ No newline at end of file + return cpu_args, cpu_kwargs -- Gitee From eefd8300828fca19ce1f3a587cab886adeb2281b Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Wed, 11 Sep 2024 16:40:13 +0800 Subject: [PATCH 782/791] =?UTF-8?q?=E3=80=90=E4=BC=98=E5=8C=96=E3=80=91?= =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=8D=95=E5=85=83=E6=B5=8B=E8=AF=95=E8=A6=86?= =?UTF-8?q?=E7=9B=96=E7=8E=87=E6=9F=A5=E7=9C=8B=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/test/.coveragerc | 3 +++ .../{ => pytorch_ut}/test_module_processer.py | 19 ++++++++++--------- debug/accuracy_tools/msprobe/test/run_ut.py | 6 ++++-- 3 files changed, 17 insertions(+), 11 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/test/.coveragerc rename debug/accuracy_tools/msprobe/test/{ => pytorch_ut}/test_module_processer.py (89%) diff --git a/debug/accuracy_tools/msprobe/test/.coveragerc b/debug/accuracy_tools/msprobe/test/.coveragerc new file mode 100644 index 000000000..99f94acd0 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/.coveragerc @@ -0,0 +1,3 @@ +[run] +# 计算覆盖率时排除单元测试文件本身 +omit = */test_*.py diff --git a/debug/accuracy_tools/msprobe/test/test_module_processer.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_module_processer.py similarity index 89% rename from debug/accuracy_tools/msprobe/test/test_module_processer.py rename to debug/accuracy_tools/msprobe/test/pytorch_ut/test_module_processer.py index 448c35f05..61af37eef 100644 --- a/debug/accuracy_tools/msprobe/test/test_module_processer.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/test_module_processer.py @@ -1,14 +1,15 @@ import unittest -from msprobe.pytorch.module_processer import ModuleProcesser -from msprobe.pytorch.common.utils import Const import torch +from msprobe.pytorch.common.utils import Const +from msprobe.pytorch.module_processer import ModuleProcesser + class TestModuleProcesser(unittest.TestCase): def test_filter_tensor_and_tuple(self): def func(nope, x): return x * 2 - + result_1 = ModuleProcesser.filter_tensor_and_tuple(func)(None, torch.tensor([1])) self.assertEqual(result_1, torch.tensor([2])) @@ -18,11 +19,11 @@ class TestModuleProcesser(unittest.TestCase): def test_clone_return_value_and_test_clone_if_tensor(self): def func(x): return x - + input = torch.tensor([1]) input_tuple = (torch.tensor([1]), torch.tensor([2])) input_list = [torch.tensor([1]), torch.tensor([2])] - input_dict = {"A":torch.tensor([1]), "B":torch.tensor([2])} + input_dict = {"A": torch.tensor([1]), "B": torch.tensor([2])} result = ModuleProcesser.clone_return_value(func)(input) result[0] = 2 @@ -40,7 +41,6 @@ class TestModuleProcesser(unittest.TestCase): result_dict["A"][0] = 2 self.assertNotEqual(result_dict, input_dict) - def test_node_hook(self): empty_list = [] test = ModuleProcesser(None) @@ -49,11 +49,12 @@ class TestModuleProcesser(unittest.TestCase): end_hook = test.node_hook("test", "stop") self.assertIsNotNone(end_hook) - class A(): + class A: pass + pre_hook(A, None, None) self.assertIn("test", test.module_count) - self.assertFalse(test.module_stack==empty_list) + self.assertFalse(test.module_stack == empty_list) def test_module_count_func(self): test = ModuleProcesser(None) @@ -61,4 +62,4 @@ class TestModuleProcesser(unittest.TestCase): module_name = "nope" test.module_count_func(module_name) - self.assertEqual(test.module_count["nope"], 0) \ No newline at end of file + self.assertEqual(test.module_count["nope"], 0) diff --git a/debug/accuracy_tools/msprobe/test/run_ut.py b/debug/accuracy_tools/msprobe/test/run_ut.py index 8ea81ccca..df011e49c 100644 --- a/debug/accuracy_tools/msprobe/test/run_ut.py +++ b/debug/accuracy_tools/msprobe/test/run_ut.py @@ -11,8 +11,9 @@ def run_ut(): ut_path = cur_dir cov_dir = os.path.dirname(cur_dir) report_dir = os.path.join(cur_dir, "report") + cov_config_path = os.path.join(cur_dir, ".coveragerc") final_xml_path = os.path.join(report_dir, "final.xml") - cov_report_path = os.path.join(report_dir, "coverage.xml") + html_cov_report = os.path.join(report_dir, "htmlcov") if os.path.exists(report_dir): shutil.rmtree(report_dir) @@ -22,9 +23,10 @@ def run_ut(): "python3", "-m", "pytest", ut_path, f"--junitxml={final_xml_path}", + f"--cov-config={cov_config_path}", f"--cov={cov_dir}", "--cov-branch", - f"--cov-report=xml:{cov_report_path}", + f"--cov-report=html:{html_cov_report}", ] try: -- Gitee From cd1a98c183f870befa645b7fe4e75aa4ccd777fe Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Mon, 9 Sep 2024 17:20:49 +0800 Subject: [PATCH 783/791] =?UTF-8?q?[msprobe]=E8=BF=AD=E4=BB=A3=E5=9B=9B?= =?UTF-8?q?=E5=87=BA=E5=8C=85=E6=9B=B4=E6=96=B0=E8=B5=84=E6=96=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/docs/01.installation.md | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/docs/01.installation.md b/debug/accuracy_tools/msprobe/docs/01.installation.md index 9a84cc8b9..99b5bedd2 100644 --- a/debug/accuracy_tools/msprobe/docs/01.installation.md +++ b/debug/accuracy_tools/msprobe/docs/01.installation.md @@ -16,6 +16,7 @@ pip install mindstudio-probe[==版本号] |版本|发布日期|支持 PyTorch 版本|支持 MindSpore 版本|下载链接|校验码| |:--:|:--:|:--:|:--:|:--:|:--:| +|1.0.4|2024.09.09|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.0.4-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.4-py3-none-any.whl)|4e1909566a71a855b356597750c20ee43d964a22b2c2b02ac08312a5def75fd6| | 1.0.3 | 2024.08.23 | 1.11/2.0/2.1/2.2 | 2.4.0 | [mindstudio_probe-1.0.3-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.3-py3-none-any.whl) | 7060cc141a5b98ef770cd9220995d299393f32a61938261e632c7e8b5160bef2 | | 1.0.2 | 2024.08.09 | 1.11/2.0/2.1/2.2 | 2.4.0 | [mindstudio_probe-1.0.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.2-py3-none-any.whl) | e4a980e5d98c426ce5ce9842520d9bc031d3b3de621c74b3d59414cc6e238e0e | | 1.0.1 | 2024.07.25 | 2.0/2.1/2.2 | 2.4.0 | [mindstudio_probe-1.0.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.0/mindstudio_probe-1.0.1-py3-none-any.whl) | b699e224e4d4e3bcf9412c54fa858a1ee370f0d7a2bc69cb3f1273ac14a6dc82 | -- Gitee From fcb7f0060d5fd5f265b19cde30e739ddcde359ea Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 12 Sep 2024 09:03:21 +0800 Subject: [PATCH 784/791] fix --- .../msprobe/pytorch/api_accuracy_checker/common/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py index 44a1b56f2..d2970aa31 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/common/config.py @@ -34,7 +34,7 @@ class Config: if not isinstance(value, validators.get(key)): raise ValueError(f"{key} must be {validators[key].__name__} type") if key == 'precision' and (value < 0 or value > 20): - raise ValueError("precision must be greater than 0 and less than 21") + raise ValueError("precision must be greater than or equal to 0 and less than 21") if key == 'white_list': RunUTConfig.check_filter_list_config(key, value) if key == 'black_list': -- Gitee From 394129a18d7e3212a697bbfed4e6b90774fef5e2 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 12 Sep 2024 10:25:15 +0800 Subject: [PATCH 785/791] grad_probe config fix --- .../mindspore/grad_probe/global_context.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py index a081d1fad..85e4d239c 100644 --- a/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py +++ b/debug/accuracy_tools/msprobe/mindspore/grad_probe/global_context.py @@ -1,6 +1,6 @@ import os import threading -from typing import Dict, Union +from typing import Dict, Union, Tuple from msprobe.core.grad_probe.utils import check_str from msprobe.core.grad_probe.constant import GradConst @@ -18,7 +18,7 @@ class GlobalContext: GradConst.STEP: None, GradConst.RANK: None, GradConst.CURRENT_STEP: 0, - GradConst.BOUNDS: [-10, -1, -0.1, -0.01, -0.001, 0, 0.001, 0.01, 0.1, 1, 10], + GradConst.BOUNDS: [-1, 0, 1], GradConst.OUTPUT_PATH: None } @@ -38,7 +38,7 @@ class GlobalContext: raise ValueError("Invalid level set in config yaml file, level option: L0, L1, L2") self._set_input_list(config_dict, GradConst.PARAM_LIST, str) - self._set_input_list(config_dict, GradConst.BOUNDS, float) + self._set_input_list(config_dict, GradConst.BOUNDS, (float, int)) self._set_input_list(config_dict, GradConst.STEP, int) self._set_input_list(config_dict, GradConst.RANK, int) @@ -70,14 +70,21 @@ class GlobalContext: dump_rank_list = self.get_context(GradConst.RANK) return (not dump_rank_list) or (rank in dump_rank_list) - def _set_input_list(self, config_dict: Dict, name: str, dtype: Union[int, str, float]): - value = config_dict.get(name) + def _get_type_str(self, dtype: Union[int, str, float, Tuple[int, str, float]]): + if isinstance(dtype, tuple): + return "/".join([self._get_type_str(element) for element in dtype]) if dtype == int: type_str = "integer" elif dtype == float: type_str = "float" else: type_str = "string" + return type_str + + def _set_input_list(self, config_dict: Dict, name: str, + dtype: Union[int, str, float, Tuple[int, str, float]]): + value = config_dict.get(name) + type_str = self._get_type_str(dtype) if value and isinstance(value, list): for val in value: if not isinstance(val, dtype): -- Gitee From 6568c88dffcbaf37e3dc875d4c0f5b638f3a4828 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Thu, 12 Sep 2024 10:43:03 +0800 Subject: [PATCH 786/791] change hash val --- .../test/mindspore_ut/grad_probe/test_ms_grad_monitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py index a087f809f..955f77a19 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/grad_probe/test_ms_grad_monitor.py @@ -95,7 +95,7 @@ class TestMsGradientMonitor(TestCase): self.assertTrue((my_dense_weight_real == my_dense_weight_target).all(), "weight ndarray not same as target") real_md5_value = get_hash(os.path.join(gradient_output_path, "rank0", "grad_summary_1.csv")) - target_md5_value = "874174395c56922f86118050e8c93e74" + target_md5_value = "d5e71f1aa37d48ef0ca0a75932597a29" self.assertEqual(real_md5_value, target_md5_value, "hash value of grad_summary_1.csv is not same as target") -- Gitee From 45327e309fd81cce1390e3f319657251f93100ac Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Thu, 12 Sep 2024 11:13:45 +0800 Subject: [PATCH 787/791] bugfix: cluster db optimized --- profiler/cluster_analyse/analysis/base_analysis.py | 3 ++- profiler/cluster_analyse/analysis/communication_analysis.py | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/profiler/cluster_analyse/analysis/base_analysis.py b/profiler/cluster_analyse/analysis/base_analysis.py index a0e3aa4fd..908331f3e 100644 --- a/profiler/cluster_analyse/analysis/base_analysis.py +++ b/profiler/cluster_analyse/analysis/base_analysis.py @@ -36,6 +36,7 @@ class BaseAnalysis: self.collective_group_dict = param.get(Constant.COMM_DATA_DICT, {}).get(Constant.COLLECTIVE_GROUP) self.comm_ops_struct = {} self.adapter = DataTransferAdapter() + self.data_simplification = param.get(Constant.DATA_SIMPLIFICATION, False) @staticmethod def compute_ratio(dividend: float, divisor: float): @@ -68,7 +69,7 @@ class BaseAnalysis: if self.data_type == Constant.TEXT: self.dump_json() else: - if len(self.data_map) >= self.MAX_RANKS: + if len(self.data_map) >= self.MAX_RANKS and not self.data_simplification: print("[WARNING]The number of ranks is too large to dump to db, it will be dumped to json file.") self.dump_json() else: diff --git a/profiler/cluster_analyse/analysis/communication_analysis.py b/profiler/cluster_analyse/analysis/communication_analysis.py index 78e7d2306..44dce2bba 100644 --- a/profiler/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/cluster_analyse/analysis/communication_analysis.py @@ -142,7 +142,6 @@ class CommunicationAnalysisOptimized(BaseAnalysis): def run(self): if not self._communication_ops: - logger.error("[ERROR] communication data is null.") return self._aggregate_time = self._format_time_data(self._communication_ops[0]) self._aggregate_bandwidth = self._format_bandwidth_data(self._communication_ops[1]) -- Gitee From e3823eed968c468774bfced9a090c6978f38b4e8 Mon Sep 17 00:00:00 2001 From: stby <295887736@qq.com> Date: Thu, 12 Sep 2024 11:56:38 +0800 Subject: [PATCH 788/791] bugfix: cluster --- profiler/cluster_analyse/analysis/communication_analysis.py | 2 +- profiler/cluster_analyse/common_func/file_manager.py | 6 +----- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/profiler/cluster_analyse/analysis/communication_analysis.py b/profiler/cluster_analyse/analysis/communication_analysis.py index 44dce2bba..81215b0ce 100644 --- a/profiler/cluster_analyse/analysis/communication_analysis.py +++ b/profiler/cluster_analyse/analysis/communication_analysis.py @@ -141,7 +141,7 @@ class CommunicationAnalysisOptimized(BaseAnalysis): return data_dict def run(self): - if not self._communication_ops: + if not self._communication_ops[0] or not self._communication_ops[1]: return self._aggregate_time = self._format_time_data(self._communication_ops[0]) self._aggregate_bandwidth = self._format_bandwidth_data(self._communication_ops[1]) diff --git a/profiler/cluster_analyse/common_func/file_manager.py b/profiler/cluster_analyse/common_func/file_manager.py index 207581f38..121c4e2d3 100644 --- a/profiler/cluster_analyse/common_func/file_manager.py +++ b/profiler/cluster_analyse/common_func/file_manager.py @@ -121,13 +121,9 @@ class FileManager: raise RuntimeError(f"Can't create the file: {base_name}") from e @classmethod - def create_output_dir(cls, collection_path: str, is_overwrite: bool = False) -> None: + def create_output_dir(cls, collection_path: str) -> None: output_path = os.path.join( collection_path, Constant.CLUSTER_ANALYSIS_OUTPUT) - if is_overwrite: - if not os.path.exists(output_path): - PathManager.make_dir_safety(output_path) - return PathManager.remove_path_safety(output_path) PathManager.make_dir_safety(output_path) -- Gitee From 5039e14200d44f6e889c0566675b91cceee4b6dc Mon Sep 17 00:00:00 2001 From: gitee Date: Thu, 12 Sep 2024 15:43:56 +0800 Subject: [PATCH 789/791] add tensor clone --- .../msprobe/core/data_dump/data_processor/pytorch_processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index ffcd822ea..82f8cc05a 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -153,7 +153,7 @@ class StatisticsDataProcessor(PytorchDataProcessor): class TensorDataProcessor(PytorchDataProcessor): def _analyze_tensor(self, tensor, suffix): dump_data_name, file_path = self.get_save_file_path(suffix) - saved_tensor = tensor.contiguous().detach() + saved_tensor = tensor.clone().contiguous().detach() save_pt(saved_tensor, file_path) single_arg = super()._analyze_tensor(tensor, suffix) single_arg.update({"data_name": dump_data_name}) -- Gitee From d2d39f25ce0d2aeb89408ad9d1f471678071f36f Mon Sep 17 00:00:00 2001 From: lijiaojiao Date: Thu, 12 Sep 2024 09:58:58 +0800 Subject: [PATCH 790/791] =?UTF-8?q?=E3=80=90bugfix=E3=80=91online=5Fdispat?= =?UTF-8?q?ch=E5=A2=9E=E5=8A=A0=E7=AE=97=E5=AD=90=E9=BB=91=E5=90=8D?= =?UTF-8?q?=E5=8D=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pytorch/online_dispatch/single_compare.py | 14 +++++++------- .../pytorch/online_dispatch/torch_ops_config.yaml | 8 ++++++++ .../msprobe/pytorch/online_dispatch/utils.py | 5 +---- 3 files changed, 16 insertions(+), 11 deletions(-) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py index 83a6bafc6..229f7c44f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/single_compare.py @@ -329,20 +329,20 @@ def single_benchmark_compare(npu_out: torch.Tensor, bench_out: torch.Tensor, hig return result, details -def calc_status_details_list_tuple(npu_out, bench_out, high_precision, summary): +def calc_status_details_list_tuple(npu_out, bench_out, summary): status, details = [], [] if len(bench_out) != len(npu_out): summary.result = False summary.failed_info = "bench and npu output structure is different." return False, summary.to_column_value() for b_out_i, n_out_i in zip(bench_out, npu_out): - status_i, details_i = single_benchmark_compare_wrap(n_out_i, b_out_i, high_precision) + status_i, details_i = single_benchmark_compare_wrap(n_out_i, b_out_i) status.append(status_i) details.append(details_i) return status, details -def calc_status_details_dict(npu_out, bench_out, high_precision, summary): +def calc_status_details_dict(npu_out, bench_out, summary): b_keys, n_keys = set(bench_out.keys()), set(npu_out.keys()) if b_keys != n_keys: summary.result = False @@ -353,7 +353,7 @@ def calc_status_details_dict(npu_out, bench_out, high_precision, summary): return status, details -def calc_status_details_tensor(npu_out, bench_out, high_precision, summary): +def calc_status_details_tensor(npu_out, bench_out, summary): return single_benchmark_compare(npu_out, bench_out) @@ -365,13 +365,13 @@ def calc_status_details_builtin(npu_out, bench_out, summary): return status, summary.to_column_value() -def calc_status_details_none(npu_out, bench_out, high_precision, summary): +def calc_status_details_none(npu_out, bench_out, summary): summary.result = True summary.failed_info = "Output is None." return True, summary.to_column_value() -def single_benchmark_compare_wrap(npu_output: torch.Tensor, bench_output: torch.Tensor, high_precision=True): +def single_benchmark_compare_wrap(npu_output: torch.Tensor, bench_output: torch.Tensor): type_method_dict = { (list, tuple): calc_status_details_list_tuple, dict: calc_status_details_dict, @@ -384,7 +384,7 @@ def single_benchmark_compare_wrap(npu_output: torch.Tensor, bench_output: torch. bench_summary = SingleBenchSummary(result) for type1, func in type_method_dict.items(): if isinstance(bench_output, type1): - return func(npu_output, bench_output, high_precision, bench_summary) + return func(npu_output, bench_output, bench_summary) bench_summary.result = True bench_summary.failed_info = "Unexpected output type: {}".format(type(bench_output)) diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/torch_ops_config.yaml b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/torch_ops_config.yaml index 789ae2a7a..6be5a4a3f 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/torch_ops_config.yaml +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/torch_ops_config.yaml @@ -1,4 +1,12 @@ aten_ops_blacklist: + - max_pool2d_with_indices + - detach + - allreduce_ + - max + - npu_rotary_mul + - split_with_sizes + - npu_dtype_cast + - add_ - _cudnn_rnn - _local_scalar_dense - _pin_memory diff --git a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py index 52b6a637b..cf3605180 100644 --- a/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/online_dispatch/utils.py @@ -59,10 +59,7 @@ INT_TYPE = [np.int32, np.int64] def get_callstack(): callstack = [] for (_, path, line, func, code, _) in inspect.stack()[2:]: - if code: - stack_line = [path, str(line), func, code[0].strip() if code else code] - else: - stack_line = [path, str(line), func, code] + stack_line = [path, str(line), func, code[0].strip() if code else code] callstack.append(stack_line) return callstack -- Gitee From 363860eb10f0803097aa73b7226cbd8f8ae7644c Mon Sep 17 00:00:00 2001 From: wangchao Date: Thu, 12 Sep 2024 19:48:17 +0800 Subject: [PATCH 791/791] add root disclaimers --- debug/accuracy_tools/msprobe/README.md | 3 +++ debug/accuracy_tools/msprobe/core/common/file_utils.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index 6555417b8..899d5ac4a 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -94,6 +94,9 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. [无标杆工具场景验证和性能基线报告](./docs/S02.report_free_benchmarking_validation_performance_baseline.md) +## ❗ 免责声明 +本工具建议执行用户与安装用户保持一致，如果您要使用root执行，请自行关注root高权限触及的安全风险。 + ## ❓ FAQ [FAQ for PyTorch](./docs/FAQ_PyTorch.md) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index 65163f3fa..e5acf31e9 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -187,7 +187,7 @@ def check_other_user_writable(path): def check_path_owner_consistent(path): file_owner = os.stat(path).st_uid - if file_owner != os.getuid(): + if file_owner != os.getuid() and os.getuid() != 0: logger.error('The file path %s may be insecure because is does not belong to you.' % path) raise FileCheckException(FileCheckException.FILE_PERMISSION_ERROR) -- Gitee

&+fVx6DtB-QV(H|Pl>XC z^Bb3eAGS7nHndG48Wv8ql_Itu-%i$3YV_8^()07WYeXL}yQwHJOk=MW`hblnF4Ye+ zEA2P*VJhs%`$3Q#!=3x^@c47}RPLUp*E=yC+7S;NOe?#90r`>I))YVAG-3I!O*n_< zT8Fk~pzWg%jyX@^0o4cRCQe>2w{KRIs4ACHml~9wza8eN2)Ezaqrh@{g`K?%jT#Uw zso4<{8JPqdi0VaFjdmLQENLR%)1Dg&ku6Sru_&xlqAekIZ=EZbZcIHe(r-DrjFGpIEjl7 zza^>U?>kJXmsCw|y|(6AbT(d^e7YLIR=*1bv0ETZWoFQyN+Hq91_iT#8&#FH%~2nC z+iRPwPf5vcSi24@Dg{~gl6i!!KQ$9_rie500E^7u9xN`Q>o$;WbgstjXQ#_(BM2f_ zwp;^>pVLi3sacaQ{V1j!uL|`?dl^vDFS^l3#{zQKC;ipx#(I6rWS&FFTe7qdxHfJg zK<~H%`#WR+UCx$EeS#lcpnOiFBsXF_V#+5dT_!;?{WQyi)Uk0B68Kpy>Xval1h@Cb z5>fEZw0NlvdaD#oZ?Tnd6p&gKNPXr^dAdFOhm)eBOvgor zYygZ+G^A$BSaE2_4w%q=9VN4eT$3ewe0OE(p5X9-Qd>{#WOI6pR6I$(f?4|NFJ4?p@AU7Hu?AQGiIqXfunz>@wK9+v@XB z^Pryt5w4p`Nd)KhY^5s@_#*<7WXcKvZb)gbmwKE^_jaGtlPitK>(JHaHl&TCjn zUxBLXsICl+srOCL1J;0(-G0tBMlSizOgFt*k~Ynrn)FfAVeX@cJ|4|kwy$R4?@PO) z?lQzmOUwVr!P88w+cG>>hwt!cPa{RpPS5Gl18SC;MQtn}y`(=*sTWp5?ME5%{G~c?51QVyx$uxk;0w&ohbU3h!qA>wPsA2s4$wJKv6v z>?v*~2}xy90q$%PO(igCfJ>`!&PyBh45>~Ahi#Mtqgqbe0<#^r;(qQ6c|j3bWHn;L zH|hK4GsKANql3y!uFTN8o4{Xr93xe&9^_bOq0|jH=x3!Zo6b?9KX7}{*)n_iLrbq< zsXa6w0rD+M>_Om{WLWlxUN4dSvyOz@%q~!idVa8u6m)fBGM*m!urKij#cBxYjc~GZ zI9L5ba}H4VM|@Oc^fJkGRLkp^eG;bZTiRa;L1ps5hATbMZteUJm0??I!u<>{SObu- z=-$#1QmEib=-BSPNvA6x9JxxBtGKf;fvRg^owV20u`l$$1!Eefmcfq;$@y1 zq=va2$|1nO?4YUQM$*pOI{U)sxl-1+U*8O5J@WoKt&w;r! zZ)VhSMl9((*qM+l0_OcPdGd%u+ypd`cP*B|aYZ%9iRG>{T~#j{f}v0v1dj(BV(bHOyE z4DMjYCydU*f9VsV$oT~~Gr8T0o{pQ-Q*?N6Q=2Yj*PHr&T3=0{Rc1(ZW63zqCc}{Y zB?Pklb0EV|lJrv5xi(K=>W|dlWSIEH)B!a)Y6I)_+vX9WDNAL`gK=lX+q0KwV?G4X1lj!`c^Swx;)ZY(W*+1SS`JnDd41tJFy z#PM13rOJ~Go4`4K%^8g5JD-6H#pKA-q7dj8?KGF@IohuQ63~iSMFBt zbn1&qjSs$C<_xYER}538higS5X&^w}zB>SDg~ojA`%v^`+Rg=mI>vS6SE0GmAcDs+F}aYsL1pL2_V(5XKYjd~=yu9Wn}+-s-_8a%&41(LeOsURXhdWcbY*{@6njhX#VIui(w+(Y&Urp2c#kH&lvf$Z&^xC$2tG9XTY{gk^Na!BKxPhz%;L>jWtTw3aC0_0gI(%F~#VF`^PY!FXH8>;e89q_y5CfZKmQT)lr9%CJ zA-(#kvjR}`lPae67h+MJtDKb!lqc2eB~?{Q9e#ii=6&@$+pWyT$tTUWhGz zmCy|4^pGp8lRDKjRgS^HvNvX6qDvM7+VA(yTGcW{wO!L%dB~}EpD0t<4MJC&xw?7l z`yGiN`mX$d<$OD<#Iav%!Wy%e+S;Ru!M|)?%AdtlN|AXCe0SPMZ8H-=Ap=+_TLoU@ znP_XJl&Y63(m6l63lOgLI5{TR>Z#{1J3h5=$%!}>#;2V*v(fXbfPVO(58i^(5w$Z6 zO!B1}{aVHOXTdx4mC^L0%jpUAFUlc*{B0|UGn#8df>PxB zbt|B%2<3G2XJpH!&gRdbEB?6!t>f|VO>3X_A_p~|w0D$qJU@7I$+8u{xJF9bT|B*U zCRc!+vS&H;^hZ|DQ!@|6O{PmARWQvT^|5$*1*ow4gaw%`m2ZkkVKA#^{;ON>3AqU?0S8Xxg zOZH@uPN3v~-Hu(Zs$*aIzb-y~paJSw82Jy|FFJo z-L?K0!O+w9fALNb>X%0`i0lDW?tIB$h4*$#-u5aAk=7FpLgQ9C+~Ke9{qY{wqw19Y+`LT46F^O+ zuFU~tck@wD^zPW{&jW;?CPz15qc>%^En(u1_RpZ-e(5OXbEpVtz&PX6(8R2~Q)v6@ zgUpRs3%`)?!2b6aT{lNjbS-H{mrY#cCcx;RuQfwo4+6;1qQYpe{wqBqSFZ_^hh?K? z(l~lVOvtR`Y@L_I*pjRtiLl~N<{l4rd-^iQB7Lb3&?lbh2al0!U|iJ2$fvuKFO!z9 zNklCG|LAe(`t?6KI-bemkCvF;zNNF=JB^eFy6zz(!4=av+VmJ~7hUjP(JB=me3#O~ zYt563N&2*-iU!e(G?M!E8-v@)@r6g8i&tOn%F>O>C|qF7M&us{_x)_5*YG#nVH$<= z{mJ+JZ0;`xk6hU{o3d0OdKY4Q>1~%GZN*g+m;6@49Z^xtH>$U?gi{QN>L+@#&a`S2 z0(H#jymQ#Uj990AnU1&g>=HpwFTqX2=y%wBsQ7|HGEgo@K$lhF#H!;DDA!=kgIjJ+ zy(^EYPfu0_`GZx?66coH%A4=(9A3Qd*TFfzAQRV|=09TD7~8V^?DWJnaBfXfLx3@M zCxh+e_R1fX`cSdIM+ADP-{xMpXwlhc9F3E%U!nPZS2~>xN}d04?Q+XeQJ497~Z`mf1p#;s- zv)Q=~{qAxf%LcxC{G_Fdy8L zIyGvy`=Hd1OG9l73(V?g}y0*!v{p z6BKyV{$PMg>J@@eUp$n-z&cSNo9LxmNa_f9N!hgJ?lX#8|H)Qex>i>EEo+*J2xLLq5XYOA+g6S0@>)hmN5t@B-o1{I7dG z&k2@dyUj1%8S<)k=rZ~4(w_&+gxay*`;)pn z)Z!Hp%&`CSSdT?^Kby6V^uuVhKA@w|YEpmnbJK-C{j9+;!(VJg9MYc^LpV0dFnwk! z(o!cSP0DSnKP_hV|Kf*U`KO&y9CW`_CXll`l1{c59ZX=p@+X-O&r!&$f5v+krgJ94{J%(+zF+t+e9|+nzq|A2%>}>wvxds(VDG3|M|IXqKyhSDb2#L1?QvQ zYC`#`t_o`ZgkjgK`ZSQKDXP9fFJ7}`$>kr&R_L27_)_vO_~h=l+=vPw<(AwX=KS85 zGeaB%pIH>H^i23C1Yxjz`@5;?Z6}*-xj^(eY)xc>4+7}R?eu{Ovvxsi%h%9)A7|=>>$wGrf>NB)sD3^I44wU z_+fS0EF{pnx$(~+wq}ZQ4iCKxNb8YkVXC~GEnT%nC!1&2TUxO07iqD@FW60M3KZ{a z^R_ij$30W<3C9WV99469^Xsv$Yj=|ICl1Xo_ytVh3x&p=?k72(&JKZJcFy1r`+s;_ z?S_B8RABH$RdK;tf-EC>Q4;ex#-xwi`(}wQSH+^Hsbo(W?3IJCnYjVoDSJ~`2 z0OloyATbg90)eK)y}6XwIdTV~f=Y+KnxcB;rmZ>O#szeCy05%KkVH0JnfC z^1&mhocniNVC=pFjFh{-h1!0*-+B$SBGJLx%Ztx*nHxtJz6ysj`mjzA&WB6*<4WrS zNIITbUS7)0W)2UtJ}gdi-bWjCcHA31rtc&@*B1TU1P^Qs^sGM0IYepXpCr_8F;6Qg z8phrr<_Kvd4-mRmxmPsIe3?gO%`SXh^xnf{z-uQ39IFg0(2AeMSh1`Ojm$(>S5Wa9 zHG37&{+SbBj#YHg7G)=r>G)>ens^9Hs*mOo7O_ehwDt$;$nThz?USR{C?fp%+Kzs_ z3&^@}`IK2lzua>)&a?Ue3MVu&eFC;jeAlgF@$MJUU|G}a5lk&nyHUvxP-_3mB0vV! zs9s#d;uWm^;`8mwjj+a~7L6!j=wxpLMIxBO5zRhbnVCn^Nbwa7&{PUI*{nC!8#E_X zI5(cHGwp9tq2WqG!6;^Cq+DS&Rh@-l${Sib$tnD_S!r#1eWY0{yj9j}(`91>u%m<- zf&>E<;jkiU%tS|#F5)!nZBroR)#>!lQCt#HhaBKOV6kkywXkqo+8X2bKGCJObHBGU zcsWTCW2To^(|)*wE=a<3k0(iZwdHn)*GYi(U7P)UnCa7~TYVe#iL{5xGA#FK1a9lh z7;bj(?YwWzJ&}y^YI^-JXLMLwqO^zSmYyLQEZL?}%<2t&*O_}_zTq2_wsD00^c3Rx zv*PNvWmHE#FIq%w1QtBsE3-e4le8z_G3IFMd)gR)9+6vS-uan4Z@0Gia9*zyZ)8pU zH7uUK1Crt$1DoS!Frv5ei-*@-V(S8a%^+tMK^p=(K(hEGOuMWZ)>pCuFYU;_9nvJ= z+cyIbN5YI}T(&~k7Z|sYIR~+~7jch+Tk*E)ET?ZEooqOL)*xkm4drk^0JWzCq+XradB|;*cB!(Zd2^Az5}w-^d}B3^7pTE1;|Q}I-8ZCv5EJyh zALj0D_4>H+Tfkk;UpjXISFtst%k9|fq$GfxfPIv{vRQS1t?I}D-5-F&p(azxn?F0# zlsZ+<)Q-~+dQLpsYa1qxb~ae5%gKi&B=OaJ#PgC@}}r{iL#?9Ple5EIuK z|5q4qQ|%CuwZImCO+T?v`|V_Ib74wwenVOB#C3cla=+hcRN#6vg8 zfz-$If*_NfbP#EJ>*7WPJv>;erRcE$^^94Sh#h_rHxa`Pw4`--?@is;nP?dD(~TR; zHdiGr@Jd3oa4BKiRJ!yF6UKY}HMfSIGqONLk+Dyv%qpZ;xN~dk;jWr~wP$TwDYvOx z+KM-FP{Mu-ZaNnwMAqLZ9j@wdZxs%d(}HY523)=#)cCEme~wJMG{sfDaz=i@fpKeP zaEQ*AAWE4yZMmgcWhJqGDMI!sO>7dMac#N2RVWqF)rhH{r-KlDV}xNLp)rwu+ixh> zDp?KhaT8zFS#V8Y>&kjOAFVpl`A3W)x`5C4x@3qMgWJ?vWZKB)SkjS4kV=@|kolVv=`eiLki)K<(J-dmv;g^}}Wmy0L0TmHJ%A$7=3%yA11c=m7qyz|6MNyDqp?5)g2@n!m0F@FYK1KT26E3fFU(z}RfNHWQDC*Yj5ot~_Hg*<-Yhf)d#ktE{#X ziToQhQ*Q&}t_BosA{MAqa@JVuqE0Ev*01kU(Rb7oC+wEgA*p+K>mEE=sgiz z+n9Sm;L(T);`z2Gs&&B~_KlDG_dI6KV}&CrHTHHcPSAFECTwW2)Fwj_eouskmkzp7 zWYfEgSi>p{T@5xSyILYW6UM!mR}SCJDcf|$B;Obojq9x;SX*Kh}srHgc1D-}WJHb$}jt>VR>KAdK>PpYVSFgLQK zMqZ2~#hbAlF$xUEOg>6@PS-^=uwWm87B+=RvMD%sZV1OoLF=F_mVu8{KjJ8I#%Ge|D<>p z%qMQGhL0nXsk=9-PyaJlBqw?%o%L(V-fMJ&8J%{dR6r72HC$6D(SzwthFV@=iO4JA zA`Sbxny%mzg}1wVPt#30OL+Orw4MpWKXWbA9OEHO8b1_TB-%(e=$&|tD2?#UIh{|j z|E81zWIMEN#V&4R5#1 zN9ad{2fJu^6|!5kYL#$UmW- z4U-I7PDZ`yx%*z7QQzg`BeGd@HK4sMk@)!Bw(w-lf(;_96=vc7f7(I-z_6`va z@t5r3E0-9FFbA}y(Z1)^pi;km5BN;J@sZuD;2<%y$DK+dD@E~s-`7_`Sbu)Kx_Tp? zGgYx`dseef!U4_aA(7@Hc2S8X8Af~0?fxFM8w!AtqGBMfgd#4uZlW88E1huLw$J1h zI6T4{e%kJRWzIyAu>(QX0v>FO&w_iqtxN${ z*eJMbxhIWH!>_R_u0a$Y`!~){SsW?uCr<7_*8-WKjmo=;R@vyv5}l&0-iB@Wz37Ou z4YWl;TUy%biEDFEwY6PQrau8+P?Q^(M1+dOLnk4uV##*Ok}gBfpbU;dl=37tg9mc4%;3o^^JzQ7HHi6D66<>7q{7x5^a9VbO36@?CjnMYsCa0g%^ttA5<@On zsVJMmxCbqlFYfSIm(HJB{;s&T6z^H_%T(2^AE=|!tu#l^a?YpUhBVJT66?w5;gdK@ znf(wRLg2}78~$)a^Pfk)SNaQ|brGa$%_CN{w%i4Q@drCaP!lDyx}hKL?FA3xD(bRe zPyMCRA+i`{mNLg4XZ5FEbl{^nb~~SX?QLP0C3VTa71kx=8D0Pdj?_Yv(K(#P-|1mDxlv)& zd?VSdb^cw*#K-m?we?+}V3Q=4HB^^dVz6tHoP)kX`msvW7ivAESqw^HR12MwC@0fqMW>jQFg4T$$^s0hGeK=-tv^W8{NBC8&38s z|4hluo=A;qWRNpAnS4jAIz8d+TfRS3m{{hdpra8bZ=)7umM2avug?s?RDWCV^`T#J zGUPqIysI(K^gjMh2Hq6a;%u&<$=fgO!sX_{?S1P5&&PY`;|@D|v8S7DuD$n#5gNM< z3uCjd zpOW8PTDL~DDd|p$H+1le|RH@_F2stmDwJ-EH1{L91TajQW=lGny~)C?3lKTVBX zaWHcd_RY9mJ~=aX8gAEgTRY9_xBt;F5DXkLGq?K+VOYTKL(%L}5$x_B3T|&PvFQ!7d>Pq9axnG-G>7^TU>8#{mC} zJ%-=&@j!Me%{y2SeO52~M@hU=rT*mjBz(Di_NRmP@g` zKZ&dh+N?oq*wSL=v7UTS-jhv{A_g*sPI)6LdT7@BIz*w1_x zA1R_sJHEKl*2$6DFVa^x-nI;UFRv|WDlkzuUzGamgS9ivrf4Gi?0j<2qyHEPYyrK-%Rkc>XTc27v0Mm^7wzc_7B{*D$WN$Tj(pmR?S z-jVupI?T(?IQxz0opjIexQimrIl|j?Iaxbf^cT6B*@p}Td#dNt6vktAQao#q748_8FO-myy zLt|=J;|wMdT^}^$18gG0_LwyG&UPV|e7Y@Jn!y$m7FP@6cggprHo1q(aPc*3pQa0x z{JAku9`%;wq6$9I-U|>vlIV7ib~+iaPYW--e$=*NwHg9mrEdR#*X+;R1$QMChSQiy zi&GdxO7z#A7p6xG6;ul!?7tJ>W5-Hgrm7f)eI|Eb2; z%f~wk)OW^#E$!A%9Q3=Se}*_GG@Y6cTpL&ro1pJ>A#6EIWd{nqFwgG6-Y<2fo!-jv4qG$2`eita27@f z=E|V-en5WPb5cxsafgnxs*pnBE2NJvW@>W_>u~vvah*zRYH;wR*sRC$#~IY=ll&=ijGynr~j)qEIKdTjHlmkoHlm~sL7fxx8Ry!_Y71%&cHPUaS{yR_iz zipf;o4_i6sMOym)HH2`nt@pS>G-m`%H$y6aBZ0kwDkmp|WFaVYWZ0PmBBXZu(GV^d z0ZrmXczp6>)_8wcVf(KSq_~E|%4?4rxtWGU3G!}+ufu8OSD;;F?-XZ@G_>yY@@$Dd zg!hw_f>)SG%kD;fVTP>xr_L>hjq}4z#+*5m#UflW+#t-bED7ygqTc|<)M|kh$osxf z_q7#-ae3+ZV)VhbJjrze4rb(wH!wjgtLK;?ncG!zPs(8Vg{rAFE!sIpuEKV!u~#tYN&%@t zsZwnJ#%jvNOS9u?;y+E}R|2+*o8*HzbX!$2s)w^w%B*V(EaTDD;WV^JKxMC+ef=bY z=eF_uE^kBS<2ci^xIkcbC?Vm-Zt{X~dXyo6ea{BsT!$oi#9e?(D1p7RGQ*%*8VCP5 zlJ@zZK)4wN>_{zfFp4x_7aN9>UHfa*h4R1|QEdwvjIim`yzcgO3RvM7M zjmwizTp7;^$cgxVxHk_=gI!=P%47LS>>4apV&UDJ>3p?pWVbMI>0^${qOp0a%(_i1 z>H44=-0Akj{2}F~SD7u-4jkz$RP8LdH^a9v>(wr|Fa`Hf-sn!K<%}HY8o23TbFVX7 zm_%h6aW%FKr)Q;Lm>(E|{rI;<3jRH!S~$EewYt|KLG6Ax?E-PskiyZ496ddl7WrzN z#ua&Ec7lmG@^73C$P#&ONp_mxz(%j&zYgUKwG3RxTS8xS`d2}_Nt)^cGwkXZP1lrm zkJf$Xht9`~hZfIUndF0oXgy&(1b*Fox#ig=+8yvX8CWGrgpDMX>wpYH&$Aa~GZ@0+nyck*{zfH}WI|-=w$zUAxMv7@90%4KK-CLB ztiFB0U}=DbIzE;ntVirAG$!ube3nWuoZRL8T-gz8T8#??B70jMvH=k!HeT$7uVDcq zS?@?V&f+p>U{6R=l7JtvQ&=lr?cCY5$bZ8BE8swwJ25BNloYu0Bw8Pc280%kSoVbq zKfACF`?=rMAo^6*o8fQ6L=O-Yqj4?{&)6YUOm8mF;!*?Xzt}+>+|fPOwSmYzX@Hy> z<`gbC)iN#Z%3tiK>Uux2r;u{Vb1(N{7!M?!ewOrzc^^2cxa`AF?g@sM8iwv%TSBli zV@E#uCR&zbrnYR9@q&i}z7y^8JG8Cdy~w!?Ts>jLSEAGWh}fC33hFLCTt2QfAQ@6H z7|6yyIpR-c+rBNxg&1CPMT%hU-?6_7EH#c#b_uKO6))vH${#|ucDyrn4y15RHk)CuMNr);=!4f_z-{QjkF_oa_x?n}!vF%Wwi#9uUv z>K>{PdSQK2@XMic4fv}KH6$f!p)@sj;aGeP6lvPze^&PmuUDc9sP6znOc%Wn;{wH`D5UnlvCZ)=`u8Z(dJVV_Ns8im0X@aub@;^Dc{hs|p0M3kv2dlz<+nOf z?aunqULplE+UG;U)-%(m4rY~xroA9m6nztL%;A^$>QcAi+Q;NgQEEJN2~yzpZe04e zx4*KM_dB+}vDL9I2fRvp|M%&I?HSjSY^BY%xO9fS_UnYsTP%P$1ZGwo8%xS;*>}97 zCNGgJ_{@L|%0z*%g(@jxEVC+6v>L9?=auEeh`B~KX|n>KmNGxi)N6)KP$g=MxI(42 z(PytVHqUf!{E%&a3-qW)97(_Nj~S_5t9|09{${X{!p=IhjRrznPesMItV6x5#-V0I za8^YZC?GKLYND#qlXB7aLiXWFo2hTgs*Z{YzaI<7K^IJ`pmyD$9ThV$<}r?p-3{n_ z>~QhdHS)8ou&?23KJIxP?p>(v{uih1fv#O4t9Y2zO^ro=$^>?2x;#QOLsX-gQc1#-g zu;36I$HxC53ZEAJukFoXE+WV#2NjmC4j~%KeL^1I)?SFn~WQ zG24_4?Ox6z1u(n%Vt`7-maZce=a=OxAgZkqAU{$OpbxhFT~}0ayA6MvtXM&KU#Y{19E-pVKO0*kuiVg$cGcffTJVhehk& zD?@X8(p_sZo4JWml!(A&F%jVirf_U!FeFsm}KtWK?m9s9-O3E*ekgxUe&m#xJ zRc<<=s=@D~tgN9`i4R5w{@e-9>lgoNXAruQ2-*|FA?kgE^oW{7j%U#*WzxWRU6X-;k5mUN?F?UNbiUlNnqXyd(8$K64=U0m4SkWGC~5Ee=Qzg zwvOE}jXUZmS!G(V(nB*8w!1aUWCHi+P4gS?sDBx`NC@Mjd=%-7y$}{A$wMo6OH=!2 z30>ZuQf;AAa&A9#nhK-YDjywCZ=ZZ3`L7hd1tVnH#(iHJD1cXfx$fP+`p=u*PiU`Z zI0IT=*n|fC(51gwtbhM9SqfkWSqyaHb#w52lyE&+-a)%U{(mgw*s)|J6TpSiu>{te z$nO%{lE6Z%5&p*=9y?Zb86ZP_y=?xk%2V@q#>Aw^y zXg_gVP2u;b{U??B`L%x#rT=lszytTV`J4ao{C~yL`Y*lA|2=}(|Nqf|Sl53iBk%t^ z1AgNam9Jua0^mT*cd&-(K24Mb%C$NGL<7JzExRtwt$f$wBsoJ#AkqPU)T41uTL8&! zD(5&i-8bQ@{dNH6&2!W7Lt$}=P%S$pWoH2>ZwxV306H{3Wz~Z+2&Vl^de)5Hwliot z3=bIrZEgn^D2Z9)FUj5+l1Waaq|vbSv)0z3V<-%Srer9@yMfh&jdd-^zok`-B^!=E zI`%9A-1oqg)2Ub0DegKUQge@YjooSFq2sR;y371lzfL$+gZT&E$v<-KU1>8UdJppI zn$Fyu>NeLn(L9bR4^O{PG1{B5MRndF`snF=?A*E;M;kBIM%z%f>%PC~V;#2?Kk852 zNE!3dD=^fj=(~lE?2<+`EZe)1t`{|^7IuylR&?qp2hwHHBG-{UEfR#*p3X{X z9ZfY{typPp+46dxa->9p?-s@KZ=*BSm;x7Y7iMsSSlNprrO$wZz`4Oi-P2`lHRC3Q zFPy`+%ZK^c6Jutp;up&;w&I-cBv%OBQzb;$i5xDxMPlJM4RAa$E<*Q8xkOyBGkl{? zPu0`VcI86D#e~p`1#w8f9Qmz@nIKCPP`lM?eH^NENiX)17O60ePQf-X?jWy&cG|m# ziwJS8Qr?Exm8${ujgPI<<62RwlbV?=s<(P;bZC1=6j{Zy?%w5+jQi0^8@G_j?8Y$|nxz#N8NNj5_w)78<(wK8UEw0`YFH z{nZ^s&Xo1DE_+JV#U(7=0RdET+w|1FIRL;)u`Rq3DJht*BOQPIm^%evLVA*e2i+6L zyxQ}L>1(dSka8r8L2=pc6KGxTffsdt&7(CSzI@x8Ib;>LWJ4&{>yb>)-uAwJRU_8R zeX4kQi~~Hk6o36fwRU>#@8ZB>5A#SJhEOPa5q>ejzf@gV6Ck2CBhLM7&!mMS-E9N;rcekH z^CBt56R(ZzbPlxu)IpcpGpPfUA|9%^#~K(WmT!sii(<8F$$UkA+5pQ?$L z_m#`nr)KL=kY_OFnDY!$VG5ov(Bmx!3wCo{OkX3qBRlXS2AWKXJU)L%?Vv;{oM|Fw z5^tfErTLN4QqA8L_6@0*Si|FP_?i)}2g#R_HV>E5(;~l9$fvx5!m#X92tVZ5CN77r)`u6tD8^V>>qr0Aaw>{rKujxgE z7go!L!{Q0fF*!wAl(y=@n@Aa*WgV)XD!KnSH7s`;IAU}^(R{|6;o0(8KF6OojTYf` zrn&O2c*{ZduIZ44V}_GMIpRR!J5ftBjVn#8i780r;ib@sfI?dzYtmYi4kCuXDJnNW z`!%^mNX6iGNOH9@RiiP-Y}ez@8r8K$KLRwV<7qu$g=#&QjNiX%A3O#Zt1XiEsZ>O>-f!q+UQR41zkLO2T!Woe2!TSr2i*K>6Zvxvkse=WU zI^}*<%9@X|3;PtqHZ>*&brP2C?OR^!#(C#B@$cX3NL;b5@wKrlJm}R>;Z}FMD;7!R zQ(OF#THsl*A+cagcu_v;d0;A6BGA8!|moBa0RWf*#zyJc5Nl8O5l(;#1&^v2gD4-82<7r45O3!vL0RBw^kl_C3bIw~cGnik7-c zl#~EBBz+(#yZAdo_h8p_kFm0Xw-UFXsceeZPH*5;^nR<4tCXfe4JSL`3(hW5&4KN^VWjhy;PKx%^i-X_0vV|dQJ6uF-drc2D!(74I)6u*yffb)Z81yFPD%?z`Eu2o6_CIE$d>uEV1ggH z`mqvnebkVB{g{zfn9h^9hrPb5muqk?m?}rAZq%$OGdQxbF-c>j@o3uDsH}h zg&5Wjtelf@I9jmcrZ+ak`tU+{y)5+`Yl@84z|!&=vHQ+a6|-v>@79ec(@+vdpRvh+e$?rF>?S(3B;x-fuYPW?mZf;cRx@N{- zdS~nN)R^|T?Bc8Bj91mwHM^b=(kSWeHXW=G2CLpw_;7bJa&vI!Rl`HVZR#pESwdEy)}*bIFvD(#nn&P)Ed8Gh*iMtkExIZN?NPOb<%j0=1Yk&^)Q6{}N^K1+_ar$m8?P z%qw!rIzW!_t7Cf2UHh7W5-e%ZC^SAKLR4Ac+VarojIbmtC~`5R@+u)rP}$&iL~?aj z)8e6mb4d9Bs@U|%!)>RIfa+awWVRgozIZO8h-W!l9B!CEe<+ysa0>s0HQ^Q}HId-y zfa4Kg3mXnP`mq1o(L(`8FGJ@@%|7(Gd3`xhuJxWh7q_jDd4Yt5Ha=JX&sKOttk~_Y zt~_K!NJCtwHWv==p&gn1vNccJY+6VN^TMesT!k+_ey@{Tet4|#r<6j&iU+-`MBJxm z>!{{wqqljvV3u)QN_CgTL=f>4t;}y^EC0B;X%|xnQ?7JZsQUSc^Qx^ohdUwZsLUBOtcV&(=J$ zykmKw4URzV(_U|UarZB-D+_}wM}-!g@{3ADsur(j&s^amFS2d)I6U6C8ys8W*d+3; zscIa^M+N{h&2?n9(#(=Kg3Aesf4lk|z)8zbmj*_b^jAN6HGJ)CW@lFbf|h>amM@<; z`cmEUn_f8OyL0^8($OEfcc#vhw#tXCTHo-{I6)U^th5w|qo9D_u(-Wr;7 zc;2ox5Cu{MtKK%{5i2w9bS#9LJ_>nnAX4~+L*U`Ry`II-y?RnV%gC4Y$1d&Tmju`n z;w{Bw@|sPcrMG3w*jgv&(Li4mo2P$!#?D%Sa{2bR>gs3hyPKt9K{wV5yjEc52S!B} zMYjM6u9$v4U}NQtV_J{YO!Q-Q#P!N9zX9ZOztQrwDH%Fd&C7(&w|w!t$9_4CQ!njHD_YAtDJdHdMqY7K)FP`sQ zRK|@d-kR{6h?D|YB&f8VkzAASSAUzIYgziTQim;JPzIO8?-2=ZI9dI?S0-BIg3U}hNM>d@Othzy`MJ_m4*I{u>^$Uue4_ks| zczMjqPubf)4>rRC z)B!NW&5qd1fA_L2AhszUZg?su_z2)YEZ+_*-MF2*YD#6GUxFR?aT0DA_!+vSVqFn3 z_jQAGqln*Mhb&JILU&MwDt3LfA1~A-FqG2tm9)t!rv3DSssQUB2#tkp;j%#HW6z6I z-9E=$H#UZ<@4j5|QiBmLAUuMt6ba)tgNPXx;c$pd1D<;eL=1k1Lw1+hhnH-}E~dl# zPdf|^@Tm&ts7$w*frMb*8IBe3X_+3%UE)Fa!jn;3;x z2Ikd!PWL{=Ly~KQV54St@<5+Ld1C-P>eGYxie9&`F1B^#U$?U|g$8Ia#b9***;IH839)J#U4TL@!w^=D?XNHU& zTS-t}Ta5C|0x?)z%v9<(wbJoV@;=O?1lj5Fimlzv)_w1jdQtF7rIbtSKi>TCkst?K z#O-{&|7#!K-;1$K&?beCleE?)45Pc!MDzusJ48lO*mgo54*6b@l_`Yse(LNUrGQbr z`|Bx|K5&sS<>@AF%Y)6{&H&2T`cFmD`Jr7&=FuXOY?sN_s9Ad4-C6SnQ{6!;%;ze) zp+#m_WwRTINy!ea6Hp0Oa7{|Piz(}DmR8HsZeG0;=M+x9ngBN6U9|_HNV)g27B#v@ zFYhyR{TafS1&Srv4$LCw%LsrKsWy#fd|{XkI9s`UW@wKKqxiWR#T0UQ$s~9Nu3=|K zQ&9DgFl@%a#gl5~`RsquzxR=1E)&T(Z&YwZUG8B1rB6dU(*ACO30t$#U}btujd>59 zY7`&Ms$yWWC2X9r%L<7Y(dC!qRFwqkNoM{L9J>_b#l}thzHa^xV_j%vncf8db~FnN zlV41eo4z)<*23Nq)OxjHl5J=85;E4TDSLQ{`^2#uGR{`Tn{5&JXHA_7m~M72vE(?T z?R_J?nuJ@lY1$Mej#ECklG>Qoc z_?;{6zZjQhthy~9uDUXg8FrDHa=wkD>i%5GytqonZxwgD1<*ZV8OFT|Ox9MwIqr9Q z>N(#pM9!9bs)lGtjcvSL6elA>JOX6@vL$^o%ST%~wL^P4H78Eu=Uh`2X-y?#2bO>I z!nt%}w(oAAEedNb7v~%Wo|9nF8w%A`zSJTUVS>tfs9Vc zU|$&#A!YoszdzNua``b-Hi08+)o#x+`Jy@#kVWkE8@&p6xD>|sAeI(){_G}BoX^~S zqn!i)ZsX{=02@?+ z)6jcs`iZnbuBL&dQMR1JN2+y=+$VT2QasFMvbLVP8%Wx7z9XW|r{O1_0VAJY972qh zX*)B~e(eAJw!=Zf{51X7{sIGkENk}at&o)TB%WtXo}?(0)6oTH32 zMrRIZbZk{sFs&#C-6#mM9F`avvP>2ww;8(ltqaOJ=rF%lN@WS^>>D`GhYP+wqmbYE z(|x#YYpe^LM-?xW5XM2kX78mG&o$5b&suoku>BXog`WWV=5P<)&*89I= zZ7Y$CIBsE$1pwPNT~|g^>a_oe%(Hs|=@a_`0w9o5&$j)0kb%-&plmUR@ ze`0Bjjmf~H+kl#jMXv{>$qcIJ2{a|2nIl?Wp0k!^KXzt%m3~~SWtI(Cq?c~TiVpPzOw?jK0&P~k6AeWOM}D04n0CkHc3LZ|IIM`(^);N z$$Eafu`sMYS?7D)vOOskg9FZ4O!YgqNjeb#3wWZ0$PWckJ(4>{kzBg_dEW+=wjs2!+&q# z`PU22g@FST&;8Hc+$T_g?tT!{$a7lc%N$0PAs!7>!V!B|&|9%%|Na(fXFJx;{` zX+t2pk7$4{z-h|;Vo1`GIZlLEsD|og9)DjApB{vla)^~SM;y~>YW@^q;MO`bKPg@{ z2W$;IdSxiado($5*`4r8Rknc5+mRcwWm9*1;Cz}x-yh#2UmY5-a++ev&eRRA&gsg- zjZZ>fUc)5)PA<6fUb#OW*E*0>FE({Nm(x=QzVldQc2*$xsB{DWnS)2+#eDAWA6+Ee zT8g9eonHGieExua`qCO`f=;WU@LYND^ub=3t>mqDD%n&` z{5gUNgKl74!h8^3i7tl)O4SpRcGOraCFYG=fCH!BJRIIc92o5VO39U*OOvJ0I8?;j zY~#_S*0SpzLgyFOD*I@-oE~P5h=%y;^=(0z>2TiJVxBl!}{R?iJiY*}vG@QZXwyS0^dPNNvBg35`(M6cE;_h8*) zCDOGKQ{a}Mnx?|+9LWLh`$?gP+2O3?&Qu6!=xdw7@|b5Qs7XfV1L&)SY;D9@l-pZL zi=W#|?g=RM-IyM~ue}qP50pbQru4nlO-2r1Hetd#IP|D5rl;sl_+~kZ^TT7V^Z{>U z*qW*yAdmzj+BEfsmL9FkMY(81dfu=Q1|*4wSN{w?NHPF;Q(E4ui}LjWA0E?*%nHHa zh#*W&lkudmIq%Vi)hm~)=JLTr;#(0tW7^BwvUMmB&WVhm8|QOdlXEjk>CnGqL2 zO5N`4v(vsj7<&4w0zr+jT{)_?#>=}Vkkpxx#7i>dI4zR9`ppkwP*%_yB0hPx+q<2kJTZ7K^vIXIYeoV+$l+H~ zI2o6+ZjFRm_nl3)`|dr7f<}*Q#3~(RhSQge_E!f1g(d8>qiJ;D#1eDRc5H;l){?(q z?=eGRmT(kL#I#VEi=7CgcIf8Tu4q0QMB0lGjxap6vnAGXE=NscM{e$in44=v-+iWp zLslj}4?=0~WyhY|j-KW{9FLRJ3L|7U9e*_pF|j?}qsSR`Yf-JD#cAFlfJ$bQ#@}{qZiyI}FUnhCYID3_`Nd^|PzQU5nISQE zM_xVhBrd$3y$|=qK*>pWT)MBz<%^bWtX@Kd%bLruTB5g8BT2P*e?)7JvAZ~`oTBng zD(wCR(1m6Ak@Z0cpei<+-@`WAvfiYsNboI?+k_m>K+H2l{DoA_V>jFl(&&5lIL-;% zqQ;nCW?wKc)?62ykY#(JK1kvM8Dv(4m2YY69*C!AA zZm5&^LsKqZ7jHdCSl|8yD8YZly29fu+)>zXP4(GI#G<6lF2a0+16mfpwz;G{F;XCk(@&W- z-?EOx#=YB~uSX@1=L2$3C*Yh;R4LE{b5pmG__ur8+N1a1RICIKhy3|Tx_DmW!i)J9 zIj**mobA;48{aSMQ#j_j)0|_Vo&DKW*&sB@w}bp4mzHAaX!5m2v6p%#x+t5RC$|<4 z*>1tyTQC{t{qyp}Xz|*VTNtSlk||YXo@dU|gXyqXP0g?;J#J?B#D1%4nf-g0AN1*3 zf@W{vcb*dpUE{YX)T-@j3+U*!-4EzQA2_N#_-M;VS0PVjhWE|<=D1%b z=}BrMJ>LR_wvLo5K=OqKt$+_YvTF4;V4`-0uXT>`CNvddIR_XA@a=o}S_^z|yg6ID zQo@x3BL<*CMwXNg6(0@JorEGatnV$2QWY1F*)CyrE0Y6m=2maQt2eeoc$VoN%VoY| zM zc9yox6A^jWAQR`i9KUnXIqHuMnRE*}kXc?h(OmMw%v#`(mA!_tp5$T?$)agh!>W;oNd>P*ZfKW`XXrydxVkcgml##y{y6hvqp2%wMF zn}96ePslW`B!N#Qgq1q2a)@rrA74MJ3xu~CPSRR1s>)re_a8jsnmt)EK92d}TQ+<0 znNq>z?Mh9yrx_+O@18z+bXm=V!!2dTH@h8HCen^|N}44gy&<-ia#gTf80XZ}9e5}@JxybmU7oGT zi?A{nNnB6wZKzh*4fT>ZF|5dzYzZST^YS+tRZdpptZgpQ_R6O(r?k%rAeEY(!Wc|6 zR8$oxA!X=e(7U%Mi8s7Y)5T0Z6G9KwTvr>P#J-C*90Nto_ONnC_??Y46rBq)s7^H= z3BA8@O*{5;bD~tC8}?JRR<@U7F&O*l6rI(}v@~91J3P;##QM=yt|Qy$GDgp)C^7MC z7dlW-Qr4Hcd!0eZ2G%Imc(QrlPdrL->5XXJAp*(Y>j@ z?Jkv{Q)NO{=adVT28*}0c-sm>Dwkq1YgE+ta_X5U?&Tx(8y+3Oq=FxtPx!7*3BBr4 z7-LYc!4ZA0e`H=`PLkZr*!Wh{V|O~2;0lkh%n4h8|Jc#)ZnXj($jA9^O%daH@IRhz zJt&Hx|7GM}@&x7QG3G|}2FKYMR?xG8kS*PKt3QaD=49~~60-}ynOrC&3DP`mf7|Wm zP95|yW?EU0gMGy0t&eYv&min>Vm8X0PCc?v_ge~mVDuxlB+@JMr?t)wG>ECny z+#>jT5d;jbOE|{h)rB_d7(Glb2T$RMncb82KcAU5AM{xq-yI)&a<=bQSmwF?Ta1&B z-t;D0JV*nKcUw^?v;qWr+7aUapfBeUWz;BFordRKiWNU2R3_g#yO)CaFb!AbaPaVSP;u@KIB&|KBLwU2%&NbD zLTskIg5&Qk?38!z4NFeCml#aZd<7iN>Fty!r1|?`B2Y(!D^{R#f#3r-m-&3Q@v#hi$Ufl^}mk>Wkdr3$gykMg3?KeIH2|8j5&Pe2n_4u5`sd2bd^ z?t^zv7t@s!4xGD3e=(-Y@H`uL79LpdJR+ht6uNzP^3BoHp&!NPpKjE4%hhTUl^=uk z<*CQD9Pf%p?HCQD7Vf)#EW%#|bY8Z#_E~#<)I)V;0=NdfyCfObNSTlQo$t8YqT7S1 z=!xOl4J9w7dc}}}^);mpzdNzp>J#-wuHrY$ZhvHZa=LXWg#hza9Af-}>E4&-M1}9k zfuX+Do2qc-WhwQ;UjN*!MEswXv6fEXL$hGGD17c@C8bUMxozD;+(=VgpZ(%Dv1v2A zstZa*h7?@4aO}ln0j@bA)dnW+KYx^mc;+1KH($1aoou->-!d}SJxpEz>reCJ*s%9o zrbmt@^xg~F&B-=#)?@Tq#4X?P;wOjf5#9_}iv2ZqB`q-k6Y&u~k`Q_tcF%4=%*K`5 zg{teD;_3N?Q{&^))cX=20yD)g8YZ!B^m`hx6)O8`hrIKcS>}&QO90m@e~c9ro7#5y z+!=DYJ@ee+i5SciM`m)t?02nfJH;P#&#TMk?T1m?B{z&c4(eh&zl8dUGp^}NjWogA z7}ns7+%fp&!`@G-F3XG9gyn+~^Yg$=j&(3VBTQUnc9v=9_|vQItzVqIGskA*Ih8ar zqx>#>9U-AZyc*W)nD*K!;H*23R`|1B#p+zR^wCOyeMx)IgW+r1`<_X{jYpuo%@Q4% z&OhEF0x;Ga9IK}A03+*3h}YE4AJ6%}*>2BPqR%H?x2;#{WzE`0((k?&u#J|qxYU|| ziwp~VZ-FbvA@-H)gRAD$g20Zw2N%q1s%}Jpl+kOU`~-m^QWE0x*>C;Cal4spj45fc zobdVRihkn$_S7#%pHB

+

Communication Retransmission Analysis

+
+ {{ desc }} + + + + + + + {% for item in solutions %} + {% set rowloop = loop %} + {% for key, value in item.items() %} + + + + + {% endfor %} + {% endfor %} +
Suggestions
{{ rowloop.index }}. {{ value.desc }}
+

+ {{ desc }} + + + {% for header in headers %} + + {% endfor %} + + + {% for row in data %} + + {% for element in row %} + + {% endfor %} + + {% endfor %} +
{{ header }}
{{ element|safe }}
+ +
+