From fa26903179273a90016dd81b6c678c0f5650b10c Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 27 Feb 2025 11:03:26 +0800 Subject: [PATCH 01/37] compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve compare read data read improve --- .../msprobe/core/common/file_utils.py | 19 +++++- .../msprobe/core/compare/acc_compare.py | 25 ++++---- .../msprobe/core/compare/utils.py | 40 ++++++++++++- .../data_processor/pytorch_processor.py | 2 +- .../msprobe/mindspore/common/utils.py | 2 +- .../msprobe/mindspore/compare/ms_compare.py | 16 +---- .../run_ut/data_generate.py | 5 +- .../msprobe/pytorch/common/utils.py | 19 +----- .../pytorch/compare/distributed_compare.py | 10 +--- .../msprobe/pytorch/compare/pt_compare.py | 33 +---------- .../test/core_ut/common/test_file_utils.py | 39 +++++++++++- .../core_ut/compare/test_acc_compare_utils.py | 59 ++++++++++++++++++- .../test/mindspore_ut/common/test_ms_utils.py | 23 ++------ .../mindspore_ut/compare/test_ms_compare.py | 27 --------- .../tensor_transport_layer/test_attl.py | 1 + .../test/pytorch_ut/common/test_pt_utils.py | 37 +----------- .../pytorch_ut/compare/test_pt_compare.py | 35 +---------- 17 files changed, 183 insertions(+), 209 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index fdc626ca6a..ad59721b54 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,6 +23,8 @@ import shutil from datetime import datetime, timezone from dateutil import parser import yaml + +import torch import numpy as np import pandas as pd @@ -446,8 +448,6 @@ def save_excel(path, data): change_mode(path, FileCheckConst.DATA_FILE_AUTHORITY) - - def move_file(src_path, dst_path): check_file_or_directory_path(src_path) check_path_before_create(dst_path) @@ -671,3 +671,16 @@ def read_xlsx(file_path): logger.error(f"The xlsx file failed to load. Please check the path: {file_path}.") raise RuntimeError(f"Read xlsx file {file_path} failed.") from e return result_df + + +def load_pt(pt_path, to_cpu=False): + pt_path = os.path.realpath(pt_path) + check_file_or_directory_path(pt_path) + try: + if to_cpu: + pt = torch.load(pt_path, map_location=torch.device("cpu"), weights_only=True) + else: + pt = torch.load(pt_path, weights_only=True) + except Exception as e: + raise RuntimeError(f"load pt file {pt_path} failed") from e + return pt diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index f0ac97a029..a983776894 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -33,7 +33,7 @@ from msprobe.core.compare.highlight import find_compare_result_error_rows, highl from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _handle_multi_process, _save_cmp_result from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_flag_and_msg from msprobe.core.compare.utils import get_accuracy, get_rela_diff_summary_mode, get_un_match_accuracy, merge_tensor, \ - print_compare_ends_info, read_op, get_name_and_state, reorder_op_x_list + print_compare_ends_info, read_op, get_name_and_state, reorder_op_x_list, read_pt_data, read_npy_data class ModeConfig: @@ -363,27 +363,28 @@ class Comparator: npu_bench_name_list = op_name_mapping_dict[npu_op_name] data_name = safe_get_value(npu_bench_name_list, 1, "npu_bench_name_list") error_file, relative_err, error_flag = None, None, False - bench_data_name = get_bench_data_name(bench_op_name, bench_data) - if data_name == '-1' or data_name == -1: # 没有真实数据路径 + bench_file_name = get_bench_data_name(bench_op_name, bench_data) + if str(data_name) == "-1": # 没有真实数据路径 n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE error_flag = True - elif not bench_data_name: + elif not bench_file_name: n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True - error_file = 'no_bench_data' + error_file = "no_bench_data" else: + npu_dir = input_param.get("npu_dump_data_dir") + bench_dir = input_param.get("bench_dump_data_dir") + npu_file_name = npu_op_name + Const.NUMPY_SUFFIX try: - read_npy_data = getattr(self, "read_npy_data") frame_name = getattr(self, "frame_name") if frame_name == "MSComparator": - n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.NUMPY_SUFFIX) + n_value = read_npy_data(npu_dir, npu_file_name) if self.cross_frame: - b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_data_name, - load_pt_file=True) + b_value = read_pt_data(bench_dir, bench_file_name) else: - b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_data_name) + b_value = read_npy_data(bench_dir, bench_file_name) else: - n_value = read_npy_data(input_param.get("npu_dump_data_dir"), npu_op_name + Const.PT_SUFFIX) - b_value = read_npy_data(input_param.get("bench_dump_data_dir"), bench_data_name) + n_value = read_pt_data(npu_dir, npu_op_name + Const.PT_SUFFIX) + b_value = read_pt_data(bench_dir, bench_file_name) except IOError as error: error_file = error.filename n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 72b75ab254..e649d80e9a 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -19,11 +19,12 @@ import math import zlib from dataclasses import dataclass +import torch import numpy as np from msprobe.core.common.const import Const, CompareConst, FileCheckConst from msprobe.core.common.utils import CompareException, check_regex_prefix_format_valid, logger, safe_get_value -from msprobe.core.common.file_utils import check_file_or_directory_path +from msprobe.core.common.file_utils import check_file_or_directory_path, FileChecker, load_pt, load_npy def extract_json(dirname, stack_json=False): @@ -597,6 +598,43 @@ def reorder_op_x_list(op_name_list, summary_list, data_name_list): return op_name_reorder, summary_reorder, data_name_reorder +def read_pt_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.PT_SUFFIX, False) + data_path = path_checker.common_check() + try: + # detach because numpy can not process gradient information + data_value = load_pt(data_path, to_cpu=True).detach() + except RuntimeError as e: + # 这里捕获 load_pt 中抛出的异常 + logger.error(f"Failed to load the .pt file at {data_path}.") + raise CompareException(CompareException.INVALID_FILE_ERROR) from e + except AttributeError as e: + # 这里捕获 detach 方法抛出的异常 + logger.error(f"Failed to detach the loaded tensor.") + raise CompareException(CompareException.DETACH_ERROR) from e + if data_value.dtype == torch.bfloat16: + data_value = data_value.to(torch.float32) + data_value = data_value.numpy() + return data_value + + +def read_npy_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.NUMPY_SUFFIX, False) + data_path = path_checker.common_check() + data_value = load_npy(data_path) + return data_value + + def _compare_parser(parser): parser.add_argument("-i", "--input_path", dest="input_path", type=str, help=" The compare input path, a dict json.", required=True) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index 2cd98b1256..e93b86af20 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -29,7 +29,7 @@ from msprobe.core.common.log import logger from msprobe.core.common.utils import convert_tuple from msprobe.core.data_dump.data_processor.base import BaseDataProcessor, ModuleBackwardInputsOutputs, \ ModuleForwardInputsOutputs, TensorStatInfo -from msprobe.pytorch.common.utils import save_pt, load_pt +from msprobe.pytorch.common.utils import save_pt from msprobe.pytorch.free_benchmark import FreeBenchmarkCheck, UnequalRow from msprobe.core.common.utils import recursion_depth_decorator diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index ded3faaa22..b8ed5e143f 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -196,4 +196,4 @@ def check_save_param(variable, name, save_backward): logger.warning("PrecisionDebugger.save_backward name not valid, " "should be bool. " "Skip current save process.") - raise ValueError \ No newline at end of file + raise ValueError diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index de507e8766..755d624976 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -22,7 +22,7 @@ import pandas as pd from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import FileOpen, create_directory, load_json, load_npy, load_yaml +from msprobe.core.common.file_utils import FileOpen, create_directory, load_json, load_yaml from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, \ check_op_str_pattern_valid, get_dump_mode, set_dump_path @@ -203,20 +203,6 @@ class MSComparator(Comparator): npu_op_name = npu_op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) return npu_op_name - def read_npy_data(self, dir_path, file_name, load_pt_file=False): - if not file_name: - return None - data_path = os.path.join(dir_path, file_name) - if load_pt_file: - import torch - from msprobe.pytorch.common.utils import load_pt - data_value = load_pt(data_path, True).detach() - if data_value.dtype == torch.bfloat16: - data_value = data_value.to(torch.float32) - data_value = data_value.numpy() - else: - data_value = load_npy(data_path) - return data_value def process_internal_api_mapping(self, npu_op_name): # get api name & class name from op_name diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index 9d89b2de32..05da6954cd 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -23,9 +23,8 @@ import numpy from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import hf_32_standard_api from msprobe.pytorch.api_accuracy_checker.common.utils import check_object_type, get_full_data_path, \ CompareException, get_module_and_atttribute_name, get_attribute -from msprobe.core.common.file_utils import FileChecker, load_npy +from msprobe.core.common.file_utils import FileChecker, load_pt, load_npy from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import load_pt from msprobe.core.common.const import Const, FileCheckConst, CompareConst diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 16067f6d2b..1f938e5a38 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -25,8 +25,8 @@ import numpy as np import torch import torch.distributed as dist from msprobe.core.common.exceptions import DistributedNotInitializedError -from msprobe.core.common.file_utils import (FileCheckConst, change_mode, - check_file_or_directory_path, check_path_before_create, FileOpen) +from msprobe.core.common.file_utils import FileCheckConst, change_mode, check_file_or_directory_path, \ + check_path_before_create, FileOpen from msprobe.core.common.log import logger from msprobe.core.common.utils import check_seed_all from packaging import version @@ -309,19 +309,6 @@ def print_rank_0(message): logger.info(message) -def load_pt(pt_path, to_cpu=False): - pt_path = os.path.realpath(pt_path) - check_file_or_directory_path(pt_path) - try: - if to_cpu: - pt = torch.load(pt_path, map_location=torch.device("cpu"), weights_only=True) - else: - pt = torch.load(pt_path, weights_only=True) - except Exception as e: - raise RuntimeError(f"load pt file {pt_path} failed") from e - return pt - - def save_pt(tensor, filepath): check_path_before_create(filepath) filepath = os.path.realpath(filepath) diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py index de62af421b..08e2f897a9 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/distributed_compare.py @@ -15,14 +15,10 @@ import os -from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import create_directory -from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, get_dump_mode, \ - set_dump_path -from msprobe.core.compare.acc_compare import ModeConfig -from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json, set_stack_json_path +from msprobe.core.common.utils import CompareException +from msprobe.core.compare.utils import check_and_return_dir_contents, extract_json from msprobe.pytorch.common.log import logger -from msprobe.pytorch.compare.pt_compare import PTComparator, compare +from msprobe.pytorch.compare.pt_compare import compare def compare_distributed(npu_dump_dir, bench_dump_dir, output_path, **kwargs): diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 308a82b3d6..7595c866bf 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -1,4 +1,4 @@ -# Copyright (c) 2024-2024, Huawei Technologies Co., Ltd. +# Copyright (c) 2024-2025, Huawei Technologies Co., Ltd. # All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -13,19 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os.path - -import torch - -from msprobe.core.common.const import FileCheckConst from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import FileChecker, create_directory, load_yaml +from msprobe.core.common.file_utils import create_directory, load_yaml from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, get_dump_mode, \ set_dump_path from msprobe.core.compare.acc_compare import Comparator, ModeConfig from msprobe.core.compare.utils import set_stack_json_path from msprobe.pytorch.common.log import logger -from msprobe.pytorch.common.utils import load_pt class PTComparator(Comparator): @@ -55,29 +49,6 @@ class PTComparator(Comparator): mapping_dict = {} return mapping_dict - def read_npy_data(self, dir_path, file_name): - if not file_name: - return None - data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.PT_SUFFIX, False) - data_path = path_checker.common_check() - try: - # detach because numpy can not process gradient information - data_value = load_pt(data_path, to_cpu=True).detach() - except RuntimeError as e: - # 这里捕获 load_pt 中抛出的异常 - logger.error(f"Failed to load the .pt file at {data_path}.") - raise CompareException(CompareException.INVALID_FILE_ERROR) from e - except AttributeError as e: - # 这里捕获 detach 方法抛出的异常 - logger.error(f"Failed to detach the loaded tensor.") - raise CompareException(CompareException.DETACH_ERROR) from e - if data_value.dtype == torch.bfloat16: - data_value = data_value.to(torch.float32) - data_value = data_value.numpy() - return data_value - def compare(input_param, output_path, **kwargs): try: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_file_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_file_utils.py index 9ed13f78ae..303c083eaa 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_file_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_file_utils.py @@ -1,7 +1,7 @@ +import unittest from unittest.mock import patch, mock_open, MagicMock +import tempfile -import numpy as np -import pandas as pd import pytest from msprobe.core.common.file_utils import * @@ -533,4 +533,37 @@ class TestDirectoryChecks: # Test file path check_file_or_directory_path(self.test_file, isdir=False) # Test directory path - check_file_or_directory_path(self.test_dir, isdir=True) \ No newline at end of file + check_file_or_directory_path(self.test_dir, isdir=True) + + +class TestLoadPt(unittest.TestCase): + + def setUp(self): + self.temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.pt') + tensor = torch.tensor([1, 2, 3]) + torch.save(tensor, self.temp_file.name) + + @patch('torch.load') + def test_load_pt_cpu(self, mock_load): + mock_load.return_value = torch.tensor([1, 2, 3]) + result = load_pt(self.temp_file.name, to_cpu=True) + self.assertTrue(torch.equal(result, torch.tensor([1, 2, 3]))) + mock_load.assert_called_once_with(self.temp_file.name, map_location=torch.device("cpu"), weights_only=True) + + @patch('torch.load') + def test_load_pt_nogpu(self, mock_load): + mock_load.return_value = torch.tensor([1, 2, 3]) + result = load_pt(self.temp_file.name, to_cpu=False) + self.assertTrue(torch.equal(result, torch.tensor([1, 2, 3]))) + mock_load.assert_called_once_with(self.temp_file.name, weights_only=True) + + @patch('torch.load') + def test_load_pt_failure(self, mock_load): + mock_load.side_effect = RuntimeError("Load failed") + with self.assertRaises(RuntimeError) as context: + load_pt(self.temp_file.name) + self.assertIn("load pt file", str(context.exception)) + + def tearDown(self): + if os.path.isfile(self.temp_file.name): + os.remove(self.temp_file.name) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index 2e9a465726..69c7ceef7b 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -4,17 +4,19 @@ import json import os import shutil import unittest -from unittest.mock import patch +from unittest.mock import patch, MagicMock import zlib +import torch import numpy as np -from msprobe.core.common.const import CompareConst, Const +from msprobe.core.common.const import CompareConst, Const, FileCheckConst from msprobe.core.common.utils import CompareException from msprobe.core.compare.utils import ApiItemInfo, _compare_parser, check_and_return_dir_contents, extract_json, \ count_struct, get_accuracy, append_stack_info, get_rela_diff_summary_mode, get_un_match_accuracy, merge_tensor, \ op_item_parse, read_op, rename_api, resolve_api_special_parameters, result_item_init, stack_column_process, \ - table_value_is_valid, get_name_and_state, reorder_op_name_list, reorder_op_x_list, gen_op_item + table_value_is_valid, get_name_and_state, reorder_op_name_list, reorder_op_x_list, gen_op_item, read_pt_data, \ + read_npy_data # test_read_op_1 op_data = { @@ -854,3 +856,54 @@ class TestGenOpItem(unittest.TestCase): expected_md5 = f"{zlib.crc32(str(op_data['value']).encode()):08x}" self.assertEqual(result['md5'], expected_md5) + + +class TestReadPtData(unittest.TestCase): + + @patch('msprobe.core.compare.utils.load_pt') + @patch('msprobe.core.compare.utils.FileChecker') + @patch('os.path.join', return_value='/fake/path/to/file.pt') + def test_read_pt_data(self, mock_os, mock_file_checker, mock_load_pt): + mock_file_checker.return_value.common_check.return_value = '/fake/path/to/file.pt' + + mock_tensor = MagicMock() + mock_tensor.detach.return_value = mock_tensor + mock_tensor.to.return_value = mock_tensor + mock_tensor.dtype = torch.bfloat16 + mock_tensor.numpy.return_value = np.array([1.0, 2.0, 3.0]) + mock_load_pt.return_value = mock_tensor + + result = read_pt_data('/fake/dir', 'file_name.pt') + + mock_file_checker.assert_called_once_with('/fake/path/to/file.pt', FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.PT_SUFFIX, False) + mock_load_pt.assert_called_once_with('/fake/path/to/file.pt', to_cpu=True) + mock_tensor.to.assert_called_once_with(torch.float32) + self.assertTrue(np.array_equal(result, np.array([1.0, 2.0, 3.0]))) + + @patch('os.path.join', return_value='/fake/path/to/file.pt') + @patch('msprobe.core.compare.utils.FileChecker') + @patch('msprobe.core.compare.utils.load_pt') + def test_read_real_data_pt_exception(self, mock_load_pt, mock_file_checker, mock_os): + mock_file_checker.return_value.common_check.return_value = '/fake/path/to/file.pt' + + mock_load_pt.side_effect = RuntimeError("Test Error") + + with self.assertRaises(CompareException): + read_pt_data('/fake/dir', 'file_name.pt') + + +class TestReadNpyData(unittest.TestCase): + + @patch('msprobe.core.compare.utils.load_npy') + @patch('msprobe.core.compare.utils.FileChecker') + @patch('os.path.join', return_value='/fake/path/to/file.npy') + def test_read_real_data_ms(self, mock_os, mock_file_checker, mock_load_npy): + mock_file_checker.return_value.common_check.return_value = '/fake/path/to/file.npy' + + mock_load_npy.return_value = np.array([1.0, 2.0, 3.0]) + + result = read_npy_data('/fake/dir', 'file_name.npy') + + mock_file_checker.assert_called_once_with('/fake/path/to/file.npy', FileCheckConst.FILE, FileCheckConst.READ_ABLE, FileCheckConst.NUMPY_SUFFIX, False) + mock_load_npy.assert_called_once_with('/fake/path/to/file.npy') + self.assertTrue(np.array_equal(result, np.array([1.0, 2.0, 3.0]))) \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/common/test_ms_utils.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/common/test_ms_utils.py index 1ed3ca0161..80f91a53f7 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/common/test_ms_utils.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/common/test_ms_utils.py @@ -15,21 +15,13 @@ # limitations under the License. """ import unittest -from unittest.mock import MagicMock, patch, call +from unittest.mock import patch import numpy as np import mindspore as ms -import os -import random - -from msprobe.core.common.exceptions import DistributedNotInitializedError -from msprobe.mindspore.common.utils import (get_rank_if_initialized, - convert_bf16_to_fp32, - save_tensor_as_npy, - convert_to_int, - list_lowest_level_directories, - seed_all, - remove_dropout, - MsprobeStep) + +from msprobe.mindspore.common.utils import get_rank_if_initialized, convert_bf16_to_fp32, convert_to_int, \ + list_lowest_level_directories, seed_all, remove_dropout, MsprobeStep + class MockCell: def __init__(self): @@ -136,8 +128,3 @@ class TestMsprobeFunctions(unittest.TestCase): from mindspore.mint.nn.functional import dropout self.assertTrue((Dropout(0.5)(x1d).numpy() == x1d.numpy()).all()) self.assertTrue((dropout(x1d, p=0.5).numpy() == x1d.numpy()).all()) - - - -if __name__ == "__main__": - unittest.main() \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py index b5cbff9784..0edb55154e 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py @@ -7,7 +7,6 @@ import tempfile import unittest import numpy as np -import torch import yaml from msprobe.core.common.utils import CompareException @@ -466,32 +465,6 @@ class TestUtilsMethods(unittest.TestCase): npu_op_name = ms_comparator.process_cell_mapping(npu_cell_dict.get('op_name')[0]) self.assertEqual(npu_op_name, 'Module.fc1.Linear.forward.0.input.0') - def test_read_npy_data(self): - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - mapping_config = MappingConfig() - - ms_comparator = MSComparator(mode_config, mapping_config) - - self.temp_file = tempfile.NamedTemporaryFile(suffix='.pt') - tensor = torch.Tensor([1, 2, 3]) - filename = self.temp_file.name.split('/')[-1] - torch.save(tensor, self.temp_file.name) - result = ms_comparator.read_npy_data('/tmp', filename, load_pt_file=True) - self.assertTrue(np.array_equal(result, np.array([1, 2, 3]))) - self.temp_file.close() - - self.temp_file = tempfile.NamedTemporaryFile(suffix='.npy') - tensor = np.array([1, 2, 3]) - filename = self.temp_file.name.split('/')[-1] - np.save(self.temp_file.name, tensor) - result = ms_comparator.read_npy_data('/tmp', filename, load_pt_file=False) - self.assertTrue(np.array_equal(result, np.array([1, 2, 3]))) - self.temp_file.close() def test_process_internal_api_mapping(self): stack_mode = True diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_attl.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_attl.py index 7d4e6e950d..79df231a1a 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_attl.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_attl.py @@ -6,6 +6,7 @@ from multiprocessing import Queue from msprobe.pytorch.api_accuracy_checker.tensor_transport_layer.attl import * from msprobe.core.common.file_utils import create_directory + class TestATTL(unittest.TestCase): def setUp(self): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_pt_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_pt_utils.py index cdc922cc98..61f7d97b55 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_pt_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_pt_utils.py @@ -2,7 +2,6 @@ import os import io import unittest from unittest.mock import MagicMock, patch -import tempfile import torch import torch.distributed as dist @@ -11,7 +10,8 @@ from msprobe.core.common.file_utils import FileCheckConst from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData from msprobe.pytorch.common.utils import parameter_adapter, get_rank_if_initialized, \ - get_tensor_rank, get_rank_id, print_rank_0, load_pt, save_pt, save_api_data, load_api_data, save_pkl, load_pkl + get_tensor_rank, get_rank_id, print_rank_0, save_pt, save_api_data, \ + load_api_data, save_pkl, load_pkl class TestParameterAdapter(unittest.TestCase): @@ -148,38 +148,6 @@ class TestPrintRank0(unittest.TestCase): mock_logger_info.assert_called_once_with(message) -class TestLoadPt(unittest.TestCase): - - def setUp(self): - self.temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.pt') - tensor = torch.tensor([1, 2, 3]) - torch.save(tensor, self.temp_file.name) - - @patch('torch.load') - def test_load_pt_cpu(self, mock_load): - mock_load.return_value = torch.tensor([1, 2, 3]) - result = load_pt(self.temp_file.name, to_cpu=True) - self.assertTrue(torch.equal(result, torch.tensor([1, 2, 3]))) - mock_load.assert_called_once_with(self.temp_file.name, map_location=torch.device("cpu"), weights_only=True) - - @patch('torch.load') - def test_load_pt_nogpu(self, mock_load): - mock_load.return_value = torch.tensor([1, 2, 3]) - result = load_pt(self.temp_file.name, to_cpu=False) - self.assertTrue(torch.equal(result, torch.tensor([1, 2, 3]))) - mock_load.assert_called_once_with(self.temp_file.name, weights_only=True) - - @patch('torch.load') - def test_load_pt_failure(self, mock_load): - mock_load.side_effect = RuntimeError("Load failed") - with self.assertRaises(RuntimeError) as context: - load_pt(self.temp_file.name) - self.assertIn("load pt file", str(context.exception)) - - def tearDown(self): - if os.path.isfile(self.temp_file.name): - os.remove(self.temp_file.name) - class TestSavePT(unittest.TestCase): def setUp(self): @@ -195,6 +163,7 @@ class TestSavePT(unittest.TestCase): mock_torch_save.assert_called_once_with(self.tensor, self.filepath) mock_change_mode.assert_called_once_with(self.filepath, FileCheckConst.DATA_FILE_AUTHORITY) + class TestSavePT(unittest.TestCase): def setUp(self): diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py index b079e646c4..4eda1d6d97 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/compare/test_pt_compare.py @@ -3,13 +3,10 @@ import os import shutil import unittest -import numpy as np import torch -from msprobe.core.common.const import Const from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import ModeConfig -from msprobe.pytorch.compare.pt_compare import PTComparator, compare +from msprobe.pytorch.compare.pt_compare import compare from msprobe.test.core_ut.compare.test_acc_compare import generate_dump_json, generate_stack_json @@ -40,36 +37,6 @@ class TestUtilsMethods(unittest.TestCase): if os.path.exists(base_dir2): shutil.rmtree(base_dir2) - def test_read_npy_data_bf16(self): - generate_bf16_pt(base_dir1) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - pt_comparator = PTComparator(mode_config) - result = pt_comparator.read_npy_data(base_dir1, 'bf16.pt') - - target_result = torch.tensor([1, 2, 3, 4], dtype=torch.float32).numpy() - self.assertTrue(np.array_equal(result, target_result)) - - def test_read_npy_data_dict(self): - generate_dict_pt(base_dir1) - - stack_mode = True - auto_analyze = True - fuzzy_match = False - dump_mode = Const.ALL - mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) - - pt_comparator = PTComparator(mode_config) - - with self.assertRaises(CompareException) as context: - result = pt_comparator.read_npy_data(base_dir1, 'dict.pt') - self.assertEqual(context.exception.code, CompareException.DETACH_ERROR) - def test_compare(self): generate_dump_json(base_dir2) generate_stack_json(base_dir2) -- Gitee From 468043b23aaa41da99ab30eff9cf29c16d4fc883 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 28 Feb 2025 11:25:13 +0800 Subject: [PATCH 02/37] compare read data read improve --- .../api_accuracy_checker/tensor_transport_layer/test_attl.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_attl.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_attl.py index 79df231a1a..0320c43d0b 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_attl.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/api_accuracy_checker/tensor_transport_layer/test_attl.py @@ -49,7 +49,7 @@ class TestATTL(unittest.TestCase): self.assertIsNone(result) @patch('glob.glob') - @patch('msprobe.pytorch.common.utils.load_pt') + @patch('msprobe.core.common.file_utils.load_pt') def test_download_with_exception(self, mock_load_pt, mock_glob): mock_glob.return_value = ['/tmp/start_file.pt'] mock_load_pt.side_effect = Exception('Load error') -- Gitee From feec53a123c5dd55f5b5e4649be1a5dfc74fb5ef Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Fri, 28 Feb 2025 17:51:15 +0800 Subject: [PATCH 03/37] compare bench_data_name get improve --- .../msprobe/core/compare/acc_compare.py | 83 +++++-------------- .../core/compare/multiprocessing_compute.py | 10 +-- .../msprobe/core/compare/utils.py | 14 ++-- 3 files changed, 34 insertions(+), 73 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index a983776894..a4c5f23324 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -33,7 +33,7 @@ from msprobe.core.compare.highlight import find_compare_result_error_rows, highl from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _handle_multi_process, _save_cmp_result from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_flag_and_msg from msprobe.core.compare.utils import get_accuracy, get_rela_diff_summary_mode, get_un_match_accuracy, merge_tensor, \ - print_compare_ends_info, read_op, get_name_and_state, reorder_op_x_list, read_pt_data, read_npy_data + print_compare_ends_info, read_op, get_name_and_state, reorder_op_x_list class ModeConfig: @@ -349,48 +349,48 @@ class Comparator: result_df = self.make_result_table(result) return result_df - def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param, bench_data): + def compare_by_op(self, npu_op_name, bench_op_name, op_name_mapping_dict, input_param): """ :param npu_op_name: excel中的NPU_Name，例如：MintFunctional.conv2d.0.forward.input.3.0 :param bench_op_name: excel中的Bench_Name，例如：Functional.conv2d.0.forward.input.3.0 :param op_name_mapping_dict: op_name和npy或pt文件的映射关系 :param input_param: npu_json_path/bench_json_path/stack_json_path等参数 - :param bench_data: bench的dump数据中"data"字段 :return: result_list，包含余弦相似度、最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率和错误信息 用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、 最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率并生成错误信息 """ - npu_bench_name_list = op_name_mapping_dict[npu_op_name] - data_name = safe_get_value(npu_bench_name_list, 1, "npu_bench_name_list") + frame_name = getattr(self, "frame_name") error_file, relative_err, error_flag = None, None, False - bench_file_name = get_bench_data_name(bench_op_name, bench_data) - if str(data_name) == "-1": # 没有真实数据路径 - n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE - error_flag = True - elif not bench_file_name: + + data_name_pair = op_name_mapping_dict(npu_op_name) + npu_data_name = data_name_pair[0] + bench_data_name = data_name_pair[1] + + if str(npu_data_name) == '-1': # 没有npu真实数据路径 + n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True + elif str(bench_data_name) == '-1': # 没有bench真实数据路径 n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True - error_file = "no_bench_data" + error_file = 'no_bench_data' else: npu_dir = input_param.get("npu_dump_data_dir") bench_dir = input_param.get("bench_dump_data_dir") - npu_file_name = npu_op_name + Const.NUMPY_SUFFIX try: - frame_name = getattr(self, "frame_name") + read_npy_data = getattr(self, "read_npy_data") if frame_name == "MSComparator": - n_value = read_npy_data(npu_dir, npu_file_name) + n_value = read_npy_data(npu_dir, npu_data_name) if self.cross_frame: - b_value = read_pt_data(bench_dir, bench_file_name) + b_value = read_npy_data(bench_dir, bench_data_name, load_pt_file=True) else: - b_value = read_npy_data(bench_dir, bench_file_name) + b_value = read_npy_data(bench_dir, bench_data_name) else: - n_value = read_pt_data(npu_dir, npu_op_name + Const.PT_SUFFIX) - b_value = read_pt_data(bench_dir, bench_file_name) + n_value = read_npy_data(npu_dir, npu_data_name) + b_value = read_npy_data(bench_dir, bench_data_name) except IOError as error: error_file = error.filename n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE error_flag = True except (FileCheckException, CompareException): - error_file = data_name + error_file = npu_data_name n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE error_flag = True @@ -473,7 +473,7 @@ class Comparator: logger.info("start compare: {}".format(npu_op_name)) cos_sim, euc_dist, max_abs_err, max_relative_err, one_thousand_err_ratio, five_thousand_err_ratio, err_msg \ - = self.compare_by_op(npu_op_name, bench_op_name, dump_path_dict, input_param, bench_data) + = self.compare_by_op(npu_op_name, bench_op_name, dump_path_dict, input_param) if is_print_compare_log: logger.info( @@ -509,46 +509,3 @@ class Comparator: except ValueError as e: logger.error('result dataframe is not found.') raise CompareException(CompareException.INVALID_DATA_ERROR) from e - - -def get_bench_data_name(bench_op_name, bench_data): - bench_name_list = re.split(r'\.(input|output|kwargs|parameters|parameters_grad)\.', bench_op_name) - if len(bench_name_list) > 1 and bench_name_list[1] == Const.PARAMS_GRAD: - bench_data_bundle = bench_data.get(bench_name_list[0] + Const.SEP + bench_name_list[1], {}) - else: - bench_data_bundle = bench_data.get(bench_name_list[0], {}) - if not bench_data_bundle or len(bench_name_list) < 3: - return None - layers = bench_name_list[2].split(Const.SEP) - - def _get(key, container): - if isinstance(container, dict): - return container.get(key) - if isinstance(container, list): - try: - return container[int(key)] - except (ValueError, IndexError): - return None - return None - - def get_by_layer(container, params_grad=False): - data = container - # dump.json中parameters_grad的结构为key：[{}], 如果存在key，有且只有一个列表元素，而op_name中只命名到了key，因此加'0' - if params_grad: - layers.append('0') - for layer in layers: - data = _get(layer, data) - return _get(CompareConst.DATA_NAME.lower(), data) - - if Const.INPUT == bench_name_list[1]: - return get_by_layer(bench_data_bundle.get(Const.INPUT, bench_data_bundle.get(Const.INPUT_ARGS))) - elif Const.KWARGS == bench_name_list[1]: - return get_by_layer(bench_data_bundle.get(Const.INPUT_KWARGS)) - elif Const.OUTPUT == bench_name_list[1]: - return get_by_layer(bench_data_bundle.get(Const.OUTPUT)) - elif Const.PARAMS == bench_name_list[1]: - return get_by_layer(bench_data_bundle.get(Const.PARAMS)) - elif Const.PARAMS_GRAD == bench_name_list[1]: - return get_by_layer(bench_data_bundle, params_grad=True) - else: - return None diff --git a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py index f79671827c..71b0f29d64 100644 --- a/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/core/compare/multiprocessing_compute.py @@ -25,7 +25,7 @@ from msprobe.core.common.utils import CompareException from msprobe.core.common.const import CompareConst -def _handle_multi_process(func, input_parma, result_df, lock): +def _handle_multi_process(func, input_param, result_df, lock): process_num = max(int((multiprocessing.cpu_count() + 1) // 4), 1) op_name_mapping_dict = read_dump_data(result_df) @@ -55,7 +55,7 @@ def _handle_multi_process(func, input_parma, result_df, lock): idx = df_chunk_size * process_idx chunk_size = len(df_chunk) result = pool.apply_async(func, - args=(idx, op_name_mapping_dict, df_chunk, lock, input_parma), + args=(idx, op_name_mapping_dict, df_chunk, lock, input_param), error_callback=err_call, callback=partial(update_progress, chunk_size, lock) ) @@ -97,12 +97,12 @@ def _ms_graph_handle_multi_process(func, result_df, mode): def read_dump_data(result_df): try: npu_dump_name_list = result_df.iloc[0:, 0].tolist() - npu_dump_tensor_list = result_df.iloc[0:, -1].tolist() + dump_tensor_pair_list = result_df.iloc[0:, -1].tolist() op_name_mapping_dict = {} for index, _ in enumerate(npu_dump_name_list): npu_dump_name = npu_dump_name_list[index] - npu_dump_tensor = npu_dump_tensor_list[index] - op_name_mapping_dict[npu_dump_name] = [npu_dump_tensor, npu_dump_tensor] + dump_tensor_pair = dump_tensor_pair_list[index] + op_name_mapping_dict[npu_dump_name] = dump_tensor_pair return op_name_mapping_dict except ValueError as e: logger.error('result dataframe is not found.') diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index e649d80e9a..486daa4160 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -322,8 +322,8 @@ def get_accuracy(result, n_dict, b_dict, dump_mode): has_stack = npu_stack_info and bench_stack_info if dump_mode == Const.ALL: - npu_data_name = n_dict.get("data_name", None) - bench_data_name = b_dict.get("data_name", None) + npu_data_name_list = n_dict.get("data_name", None) + bench_data_name_list = b_dict.get("data_name", None) for index in range(min_len): n_name = safe_get_value(n_dict, n_start + index, "n_dict", key="op_name") @@ -354,7 +354,9 @@ def get_accuracy(result, n_dict, b_dict, dump_mode): result_item.append(err_msg) result_item = stack_column_process(result_item, has_stack, index, key, npu_stack_info) if dump_mode == Const.ALL: - result_item.append(safe_get_value(npu_data_name, n_start + index, "npu_data_name")) + npu_data_name = safe_get_value(npu_data_name_list, n_start + index, "npu_data_name_list") + bench_data_name = safe_get_value(bench_data_name_list, n_start + index, "bench_data_name_list") + result_item.append(npu_data_name, bench_data_name) result.append(result_item) @@ -389,7 +391,9 @@ def get_accuracy(result, n_dict, b_dict, dump_mode): result_item.append(err_msg) result_item = stack_column_process(result_item, has_stack, index, key, npu_stack_info) if dump_mode == Const.ALL: - result_item.append(safe_get_value(npu_data_name, n_start + index, "npu_data_name")) + npu_data_name = safe_get_value(npu_data_name_list, n_start + index, "npu_data_name_list") + bench_data_name = safe_get_value(bench_data_name_list, n_start + index, "bench_data_name_list") + result_item.append(npu_data_name, bench_data_name) result.append(result_item) @@ -468,7 +472,7 @@ def get_un_match_accuracy(result, n_dict, dump_mode): result_item.append(err_msg) append_stack_info(result_item, npu_stack_info, index) if dump_mode == Const.ALL and result_item[1] == CompareConst.N_A: - result_item.extend(["-1"]) + result_item.extend(["-1", "-1"]) result.append(result_item) -- Gitee From aca39e6e04c2da1f33f14a0edf53d6f09f1e54f7 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 11:42:49 +0800 Subject: [PATCH 04/37] compare bench_data_name get improve --- .../msprobe/core/compare/acc_compare.py | 4 +- .../test/core_ut/compare/test_acc_compare.py | 42 +++++-------------- .../test_cmp_multiprocessing_compute.py | 8 ++-- 3 files changed, 17 insertions(+), 37 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index a4c5f23324..06f5932879 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -359,10 +359,9 @@ class Comparator: 用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、 最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率并生成错误信息 """ - frame_name = getattr(self, "frame_name") error_file, relative_err, error_flag = None, None, False - data_name_pair = op_name_mapping_dict(npu_op_name) + data_name_pair = op_name_mapping_dict.get(npu_op_name) npu_data_name = data_name_pair[0] bench_data_name = data_name_pair[1] @@ -375,6 +374,7 @@ class Comparator: npu_dir = input_param.get("npu_dump_data_dir") bench_dir = input_param.get("bench_dump_data_dir") try: + frame_name = getattr(self, "frame_name") read_npy_data = getattr(self, "read_npy_data") if frame_name == "MSComparator": n_value = read_npy_data(npu_dir, npu_data_name) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py index c882e331f5..81e9ec30b6 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py @@ -11,7 +11,7 @@ import torch from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.utils import CompareException -from msprobe.core.compare.acc_compare import Comparator, ModeConfig, get_bench_data_name +from msprobe.core.compare.acc_compare import Comparator, ModeConfig from msprobe.core.compare.highlight import find_error_rows, find_compare_result_error_rows, ApiBatch from msprobe.core.compare.utils import get_accuracy from msprobe.pytorch.compare.pt_compare import PTComparator @@ -636,11 +636,11 @@ class TestUtilsMethods(unittest.TestCase): def test_do_multi_process(self): data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', [2, 2], [2, 2], - '', '', '', '', '', '', 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', '-1']] + '', '', '', '', '', '', 1, 1, 1, 1, 1, 1, 1, 1, 'Yes', '', ['-1', '-1']]] o_data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', [2, 2], [2, 2], 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 1, 1, 1, 1, 1, 1, 1, 1, 'None', 'No bench data matched.', '-1']] + 1, 1, 1, 1, 1, 1, 1, 1, 'None', 'No bench data matched.', ['-1', '-1']]] columns = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] result_df = pd.DataFrame(data, columns=columns) o_result = pd.DataFrame(o_data, columns=columns) @@ -670,7 +670,7 @@ class TestUtilsMethods(unittest.TestCase): mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) pt_comparator = PTComparator(mode_config) - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param, {}) + result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'No bench data matched.']) @@ -688,43 +688,23 @@ class TestUtilsMethods(unittest.TestCase): pt_comparator = PTComparator(mode_config) pt_name = '-1' - pt_path = os.path.join(base_dir, pt_name) - op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_path, pt_path]} + op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param, - {'Functional.linear.0.forward': {'input_args': [ - {'data_name': 'Functional.linear.0.forward.input.0.pt'}]}}) + result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', f'Dump file: {pt_path} not found.']) + 'unsupported', 'No bench data matched.']) pt_name = 'Functional.linear.0.forward.input.0.pt' - pt_path = os.path.join(base_dir, pt_name) - op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_path, pt_path]} + op_name_mapping_dict = {'Functional.linear.0.forward.input.0': [pt_name, pt_name]} input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param, {}) + result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', 'Bench does not have data file.']) + 'unsupported', 'Dump file: Functional.linear.0.forward.input.0.pt not found']) generate_pt(base_dir) - result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param, - {'Functional.linear.0.forward': {'input_args': [ - {'data_name': 'Functional.linear.0.forward.input.0.pt'}]}}) + result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) self.assertEqual(result, [1.0, 0.0, 0.0, 0.0, 1.0, 1.0, '']) - def test_get_bench_data_name_input(self): - bench_op_name = "Functional.linear.0.forward.input.0" - bench_data = {"Functional.linear.0.forward": {"input_args": [{"data_name": "Functional.linear.0.forward.input.0.pt"}], "input_kwargs": {}, "output": []}} - result = get_bench_data_name(bench_op_name, bench_data) - - self.assertEqual(result, "Functional.linear.0.forward.input.0.pt") - - def test_get_bench_data_name_output(self): - bench_op_name = "Functional.linear.0.forward.output.0" - bench_data = {"Functional.linear.0.forward": {"input_args": [], "input_kwargs": {}, "output": [{"data_name": "Functional.linear.0.forward.output.0.pt"}]}} - result = get_bench_data_name(bench_op_name, bench_data) - - self.assertEqual(result, "Functional.linear.0.forward.output.0.pt") - class TestComparator(unittest.TestCase): def setUp(self): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py index 3fa16b0d9d..49f084ce07 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_cmp_multiprocessing_compute.py @@ -18,12 +18,12 @@ data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.inp 'torch.float32', 'torch.float32', [2, 2], [2, 2], '', '', '', '', '', '', 1, 1, 1, 1, 1, 1, 1, 1, - 'Yes', '', '-1']] + 'Yes', '', ['-1', '-1']]] o_data = [['Functional.linear.0.forward.input.0', 'Functional.linear.0.forward.input.0', 'torch.float32', 'torch.float32', [2, 2], [2, 2], 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', 1, 1, 1, 1, 1, 1, 1, 1, - 'None', 'No bench data matched.', '-1']] + 'None', 'No bench data matched.', ['-1', '-1']]] columns = CompareConst.COMPARE_RESULT_HEADER + ['Data_name'] result_df = pd.DataFrame(data, columns=columns) o_result = pd.DataFrame(o_data, columns=columns) @@ -54,9 +54,9 @@ class TestUtilsMethods(unittest.TestCase): func = Comparator(mode_config).compare_ops generate_dump_json(base_dir) - input_parma = {'bench_json_path': os.path.join(base_dir, 'dump.json')} + input_param = {'bench_json_path': os.path.join(base_dir, 'dump.json')} lock = multiprocessing.Manager().RLock() - result = _handle_multi_process(func, input_parma, result_df, lock) + result = _handle_multi_process(func, input_param, result_df, lock) self.assertTrue(result.equals(o_result)) def test_read_dump_data(self): -- Gitee From 67bd1c8a4dd142c6a72126f456b55f2fa9add097 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 14:17:04 +0800 Subject: [PATCH 05/37] compare bench_data_name get improve --- .../accuracy_tools/msprobe/core/compare/utils.py | 2 +- .../test/core_ut/compare/test_acc_compare.py | 2 +- .../core_ut/compare/test_acc_compare_utils.py | 16 ++++++++-------- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 486daa4160..029a361216 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -472,7 +472,7 @@ def get_un_match_accuracy(result, n_dict, dump_mode): result_item.append(err_msg) append_stack_info(result_item, npu_stack_info, index) if dump_mode == Const.ALL and result_item[1] == CompareConst.N_A: - result_item.extend(["-1", "-1"]) + result_item.extend([["-1", "-1"]]) result.append(result_item) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py index 81e9ec30b6..1b2f6bb2fd 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare.py @@ -699,7 +699,7 @@ class TestUtilsMethods(unittest.TestCase): input_param = {'npu_dump_data_dir': base_dir, 'bench_dump_data_dir': base_dir} result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) self.assertEqual(result, ['unsupported', 'unsupported', 'unsupported', 'unsupported', 'unsupported', - 'unsupported', 'Dump file: Functional.linear.0.forward.input.0.pt not found']) + 'unsupported', 'Dump file: Functional.linear.0.forward.input.0.pt not found.']) generate_pt(base_dir) result = pt_comparator.compare_by_op(npu_op_name, bench_op_name, op_name_mapping_dict, input_param) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py index 69c7ceef7b..5327237066 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/compare/test_acc_compare_utils.py @@ -226,31 +226,31 @@ o_result_unmatch_3 = [ ['Functional.conv2d.0.forward.input.0', 'N/A', 'torch.float32', 'N/A', [1, 1, 28, 28], 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 3.029174327850342, -2.926689624786377, -0.06619918346405029, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', - 'No bench data matched.', 'None', '-1'], + 'No bench data matched.', 'None', ['-1', '-1']], ['Functional.conv2d.0.forward.input.1', 'N/A', 'torch.float32', 'N/A', [16, 1, 5, 5], 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 0.19919930398464203, -0.19974489510059357, 0.006269412115216255, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', - 'No bench data matched.', 'None', '-1'], + 'No bench data matched.', 'None', ['-1', '-1']], ['Functional.conv2d.0.forward.input.2', 'N/A', 'torch.float32', 'N/A', [16], 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 0.19734230637550354, -0.18177609145641327, 0.007903944700956345, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', - 'No bench data matched.', 'None', '-1'], + 'No bench data matched.', 'None', ['-1', '-1']], ['Functional.conv2d.0.forward.parameters.weight', 'N/A', 'torch.float32', 'N/A', [1, 16, 28, 28], 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', - 1.0, 1.0, 1.0, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'No bench data matched.', 'None', '-1'], + 1.0, 1.0, 1.0, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'No bench data matched.', 'None', ['-1', '-1']], ['Functional.conv2d.0.forward.parameters.bias', 'N/A', 'torch.float32', 'N/A', [1, 16, 28, 28], 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', - 1.0, 1.0, 1.0, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'No bench data matched.', 'None', '-1'], + 1.0, 1.0, 1.0, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'No bench data matched.', 'None', ['-1', '-1']], ['Functional.conv2d.0.forward.output.0', 'N/A', 'torch.float32', 'N/A', [1, 16, 28, 28], 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 2.1166646480560303, -2.190781354904175, -0.003579073818400502, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', - 'No bench data matched.', 'None', '-1'], + 'No bench data matched.', 'None', ['-1', '-1']], ['Functional.conv2d.0.parameters_grad.weight', 'N/A', 'torch.float32', 'N/A', [1, 16, 28, 28], 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', - 1.0, 1.0, 1.0, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'No bench data matched.', 'None', '-1'], + 1.0, 1.0, 1.0, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'No bench data matched.', 'None', ['-1', '-1']], ['Functional.conv2d.0.parameters_grad.bias', 'N/A', 'torch.float32', 'N/A', [1, 16, 28, 28], 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', - 1.0, 1.0, 1.0, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'No bench data matched.', 'None', '-1'] + 1.0, 1.0, 1.0, 1.0, 'N/A', 'N/A', 'N/A', 'N/A', 'N/A', 'No bench data matched.', 'None', ['-1', '-1']] ] # test_merge_tensor -- Gitee From 8248bafc1edb282576dd481e55ba200927f4a846 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 14:23:39 +0800 Subject: [PATCH 06/37] compare bench_data_name get improve --- debug/accuracy_tools/msprobe/core/compare/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index 029a361216..a2ba55fb46 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -356,7 +356,7 @@ def get_accuracy(result, n_dict, b_dict, dump_mode): if dump_mode == Const.ALL: npu_data_name = safe_get_value(npu_data_name_list, n_start + index, "npu_data_name_list") bench_data_name = safe_get_value(bench_data_name_list, n_start + index, "bench_data_name_list") - result_item.append(npu_data_name, bench_data_name) + result_item.append([npu_data_name, bench_data_name]) result.append(result_item) @@ -393,7 +393,7 @@ def get_accuracy(result, n_dict, b_dict, dump_mode): if dump_mode == Const.ALL: npu_data_name = safe_get_value(npu_data_name_list, n_start + index, "npu_data_name_list") bench_data_name = safe_get_value(bench_data_name_list, n_start + index, "bench_data_name_list") - result_item.append(npu_data_name, bench_data_name) + result_item.append([npu_data_name, bench_data_name]) result.append(result_item) -- Gitee From 1274b639165661fed606676f1f698d40a3523660 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 15:11:19 +0800 Subject: [PATCH 07/37] compare bench_data_name get improve --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index 06f5932879..cdc2e9fd84 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -356,7 +356,7 @@ class Comparator: :param op_name_mapping_dict: op_name和npy或pt文件的映射关系 :param input_param: npu_json_path/bench_json_path/stack_json_path等参数 :return: result_list，包含余弦相似度、最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率和错误信息 - 用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、 + 用于读取excel中的NPU_Name和Bench_Name，根据映射关系找到npy或pt文件，然后读取文件中的数据进行比较，计算余弦相似度、欧式距离 最大绝对误差、最大相对误差、千分之一误差率、千分之五误差率并生成错误信息 """ error_file, relative_err, error_flag = None, None, False @@ -365,9 +365,9 @@ class Comparator: npu_data_name = data_name_pair[0] bench_data_name = data_name_pair[1] - if str(npu_data_name) == '-1': # 没有npu真实数据路径 + if str(npu_data_name) == '-1': # 没有npu真实数据 n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True - elif str(bench_data_name) == '-1': # 没有bench真实数据路径 + elif str(bench_data_name) == '-1': # 没有bench真实数据 n_value, b_value, error_flag = CompareConst.READ_NONE, CompareConst.READ_NONE, True error_file = 'no_bench_data' else: @@ -465,7 +465,7 @@ class Comparator: err_mess = [] is_print_compare_log = input_param.get("is_print_compare_log") - bench_data = load_json(input_param.get("bench_json_path")).get('data') + for i in range(len(result_df)): npu_op_name = result_df.iloc[i, 0] bench_op_name = result_df.iloc[i, 1] -- Gitee From 1e311f2763258ab56008e70749c0b605b5096b4b Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 17:07:11 +0800 Subject: [PATCH 08/37] compare bench_data_name get improve --- .../msprobe/mindspore/compare/ms_compare.py | 5 ++++ .../mindspore_ut/compare/test_ms_compare.py | 28 ++++++++++++++++++- 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 755d624976..d061508c0e 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -78,6 +78,11 @@ class MSComparator(Comparator): raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " f"{type(self.data_mapping)}") + @staticmethod + def process_data_name(match_result): + match_result['data_name_x'] = match_result.apply(lambda row: [row['data_name_x'], row['data_name_y']], axis=1) + return match_result + def calc_accuracy(self, result_df, header): condition_no_bench = result_df[CompareConst.BENCH_NAME] == CompareConst.N_A result_df[condition_no_bench] = result_df[condition_no_bench].fillna(CompareConst.N_A) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py index 0edb55154e..62fcf5a0e7 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py @@ -7,6 +7,8 @@ import tempfile import unittest import numpy as np +import pandas as pd +import torch import yaml from msprobe.core.common.utils import CompareException @@ -506,4 +508,28 @@ class TestUtilsMethods(unittest.TestCase): api_list = ["Mint"] with self.assertRaises(CompareException): - ms_comparator.get_api_name(api_list) \ No newline at end of file + ms_comparator.get_api_name(api_list) + + def test_process_data_name(self): + stack_mode = True + auto_analyze = True + fuzzy_match = False + dump_mode = Const.ALL + + mode_config = ModeConfig(stack_mode, auto_analyze, fuzzy_match, dump_mode) + mapping_config = MappingConfig() + ms_comparator = MSComparator(mode_config, mapping_config) + + data = pd.DataFrame({ + 'data_name_x': ['A', 'B', 'C'], + 'data_name_y': ['X', 'Y', 'Z'] + }) + + result = ms_comparator.process_data_name(data.copy()) + + expected = pd.DataFrame({ + 'data_name_x': [['A', 'X'], ['B', 'Y'], ['C', 'Z']], + 'data_name_y': ['X', 'Y', 'Z'] + }) + + pd.testing.assert_frame_equal(result, expected) -- Gitee From e0f382effe454c433cb719bc6728b32d5cfbf240 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 17:29:26 +0800 Subject: [PATCH 09/37] compare bench_data_name get improve --- debug/accuracy_tools/msprobe/core/compare/acc_compare.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index cdc2e9fd84..f2aa8c479e 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -329,7 +329,9 @@ class Comparator: else: result_item.append(CompareConst.NONE) if self.dump_mode == Const.ALL: - result_item.append(npu_ops_all.get(ms_op_name).get("data_name", None)) + ms_data_name = npu_ops_all.get(ms_op_name).get("data_name", None) + pt_data_name = bench_ops_all.get(bench_op_name).get("data_name", None) + result_item.append([ms_data_name, pt_data_name]) result.append(result_item) elif ms_op_name not in npu_ops_all: logger.warning(f'Can not find npu op name : `{ms_op_name}` in npu dump json file.') -- Gitee From e6739597bf822954dff102dc96d0f04c5b318b73 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 17:38:55 +0800 Subject: [PATCH 10/37] compare bench_data_name get improve --- .../msprobe/docs/10.accuracy_compare_PyTorch.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md index a5f83d8dfc..6f886215b0 100644 --- a/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/10.accuracy_compare_PyTorch.md @@ -257,11 +257,11 @@ PyTorch 精度比对是以 CPU 或 GPU 的计算结果为标杆，通过计算 统计量有 4 种：最大值（max）、最小值（min）、平均值（mean）和 L2-范数（L2 norm）。 -|dump 数据模式|Cosine (tensor 余弦相似度)|EucDist (tensor 欧式距离)|MaxAbsErr (tensor 最大绝对误差)|MaxRelativeErr (tensor 最大相对误差)|One Thousandth Err Ratio (tensor 相对误差小于千分之一的比例)|Five Thousandth Err Ratio (tensor 相对误差小于千分之五的比例)|NPU 和 bench 的统计量绝对误差 (max, min, mean, L2 norm) diff| NPU 和 bench 的统计量相对误差 (max, min, mean, L2 norm) RelativeErr |NPU 和 bench 的统计量 (max, min, mean, L2 norm)|NPU MD5 (NPU 数据 CRC-32 值)|BENCH MD5 (bench 数据 CRC-32 值)|Result (比对结果)|Accuracy Reached or Not (计算精度是否达标)|Err_message (错误信息提示)|NPU_Stack_Info (堆栈信息)|Data_Name (NPU 真实数据名)| -|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -|真实数据模式|√|√|√|√|√|√|||√||||√|√|√|√| -|统计数据模式|||||||√|√|√|||√||√|√|| -|MD5 模式||||||||||√|√|√|||√|| +|dump 数据模式|Cosine (tensor 余弦相似度)|EucDist (tensor 欧式距离)|MaxAbsErr (tensor 最大绝对误差)|MaxRelativeErr (tensor 最大相对误差)|One Thousandth Err Ratio (tensor 相对误差小于千分之一的比例)|Five Thousandth Err Ratio (tensor 相对误差小于千分之五的比例)|NPU 和 bench 的统计量绝对误差 (max, min, mean, L2 norm) diff| NPU 和 bench 的统计量相对误差 (max, min, mean, L2 norm) RelativeErr |NPU 和 bench 的统计量 (max, min, mean, L2 norm)|NPU MD5 (NPU 数据 CRC-32 值)|BENCH MD5 (bench 数据 CRC-32 值)|Result (比对结果)|Accuracy Reached or Not (计算精度是否达标)|Err_message (错误信息提示)|NPU_Stack_Info (堆栈信息)| Data_Name ([NPU真实数据名，Bench真实数据名]) | +|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---------------------------------:| +|真实数据模式|√|√|√|√|√|√|||√||||√|√|√| √ | +|统计数据模式|||||||√|√|√|||√||√|√| | +|MD5 模式||||||||||√|√|√|||√| | 上表中NPU_Stack_Info字段需要配置-s参数生成。 -- Gitee From 6c140105fcec11721ada8462f1db46e44714402d Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 19:12:05 +0800 Subject: [PATCH 11/37] compare bench_data_name get improve --- debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 1 + 1 file changed, 1 insertion(+) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index d061508c0e..a1a96dbbf9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -272,6 +272,7 @@ class MSComparator(Comparator): ((npu_dtype == Const.TORCH_BFLOAT16) & (bench_dtype == Const.TORCH_FLOAT16))) match_result.loc[~gen_dtype_condition(), [i + '_y' for i in bench_df.columns]] = CompareConst.N_A + match_result = self.process_data_name(match_result) return self.make_result_df(match_result) def modify_compare_data_with_user_mapping(self, npu_df, bench_df): -- Gitee From 37dab86089b7f638c12ea0d7b349b8a52ac0d835 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 19:28:29 +0800 Subject: [PATCH 12/37] compare bench_data_name get improve --- .../msprobe/mindspore/compare/ms_compare.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index a1a96dbbf9..89639ab2ad 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -79,9 +79,9 @@ class MSComparator(Comparator): f"{type(self.data_mapping)}") @staticmethod - def process_data_name(match_result): - match_result['data_name_x'] = match_result.apply(lambda row: [row['data_name_x'], row['data_name_y']], axis=1) - return match_result + def process_data_name(result): + result['data_name_x'] = result.apply(lambda row: [row['data_name_x'], row['data_name_y']], axis=1) + return result def calc_accuracy(self, result_df, header): condition_no_bench = result_df[CompareConst.BENCH_NAME] == CompareConst.N_A @@ -175,6 +175,10 @@ class MSComparator(Comparator): result[npu_summary] = result['summary_x'].apply(set_summary).tolist() result[bench_summary] = result['summary_y'].apply(set_summary).tolist() + + if self.dump_mode == Const.ALL: + result = self.process_data_name(result) + result_df = pd.DataFrame(columns=header) for h in header: if h in result.columns: @@ -272,7 +276,6 @@ class MSComparator(Comparator): ((npu_dtype == Const.TORCH_BFLOAT16) & (bench_dtype == Const.TORCH_FLOAT16))) match_result.loc[~gen_dtype_condition(), [i + '_y' for i in bench_df.columns]] = CompareConst.N_A - match_result = self.process_data_name(match_result) return self.make_result_df(match_result) def modify_compare_data_with_user_mapping(self, npu_df, bench_df): -- Gitee From dfb9e76a4fec33f9e178866861d0e0d56abf3e97 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 19:33:18 +0800 Subject: [PATCH 13/37] compare bench_data_name get improve --- debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 89639ab2ad..9abe144659 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -145,6 +145,8 @@ class MSComparator(Comparator): header.append(CompareConst.STACK) if self.dump_mode == Const.ALL: header.append(CompareConst.DATA_NAME) + result = self.process_data_name(result) + result.rename(columns={'op_name_x': CompareConst.NPU_NAME, 'op_name_y': CompareConst.BENCH_NAME, 'dtype_x': CompareConst.NPU_DTYPE, @@ -176,9 +178,6 @@ class MSComparator(Comparator): result[npu_summary] = result['summary_x'].apply(set_summary).tolist() result[bench_summary] = result['summary_y'].apply(set_summary).tolist() - if self.dump_mode == Const.ALL: - result = self.process_data_name(result) - result_df = pd.DataFrame(columns=header) for h in header: if h in result.columns: -- Gitee From 5c854d7a37b671c4d991c49e6110aab130142ff5 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Mon, 3 Mar 2025 20:31:10 +0800 Subject: [PATCH 14/37] compare read data read improve --- .../msprobe/core/common/file_utils.py | 14 ------- .../msprobe/core/compare/acc_compare.py | 10 +++-- .../msprobe/core/compare/utils.py | 39 +------------------ .../msprobe/mindspore/compare/ms_compare.py | 15 ++++++- .../run_ut/data_generate.py | 3 +- .../msprobe/pytorch/common/utils.py | 38 ++++++++++++++++++ .../msprobe/pytorch/compare/pt_compare.py | 31 ++++++++++++++- .../test/core_ut/common/test_file_utils.py | 35 ----------------- .../test/pytorch_ut/common/test_pt_utils.py | 39 +++++++++++++++++-- 9 files changed, 127 insertions(+), 97 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/file_utils.py b/debug/accuracy_tools/msprobe/core/common/file_utils.py index ad59721b54..89d33a6a3e 100644 --- a/debug/accuracy_tools/msprobe/core/common/file_utils.py +++ b/debug/accuracy_tools/msprobe/core/common/file_utils.py @@ -24,7 +24,6 @@ from datetime import datetime, timezone from dateutil import parser import yaml -import torch import numpy as np import pandas as pd @@ -671,16 +670,3 @@ def read_xlsx(file_path): logger.error(f"The xlsx file failed to load. Please check the path: {file_path}.") raise RuntimeError(f"Read xlsx file {file_path} failed.") from e return result_df - - -def load_pt(pt_path, to_cpu=False): - pt_path = os.path.realpath(pt_path) - check_file_or_directory_path(pt_path) - try: - if to_cpu: - pt = torch.load(pt_path, map_location=torch.device("cpu"), weights_only=True) - else: - pt = torch.load(pt_path, weights_only=True) - except Exception as e: - raise RuntimeError(f"load pt file {pt_path} failed") from e - return pt diff --git a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py index f2aa8c479e..28a7b5f3a8 100644 --- a/debug/accuracy_tools/msprobe/core/compare/acc_compare.py +++ b/debug/accuracy_tools/msprobe/core/compare/acc_compare.py @@ -34,6 +34,8 @@ from msprobe.core.compare.multiprocessing_compute import ComparisonResult, _hand from msprobe.core.compare.npy_compare import compare_ops_apply, get_error_flag_and_msg from msprobe.core.compare.utils import get_accuracy, get_rela_diff_summary_mode, get_un_match_accuracy, merge_tensor, \ print_compare_ends_info, read_op, get_name_and_state, reorder_op_x_list +from msprobe.pytorch.compare.pt_compare import read_pt_data +from msprobe.mindspore.compare.ms_compare import read_npy_data class ModeConfig: @@ -377,16 +379,16 @@ class Comparator: bench_dir = input_param.get("bench_dump_data_dir") try: frame_name = getattr(self, "frame_name") - read_npy_data = getattr(self, "read_npy_data") + if frame_name == "MSComparator": n_value = read_npy_data(npu_dir, npu_data_name) if self.cross_frame: - b_value = read_npy_data(bench_dir, bench_data_name, load_pt_file=True) + b_value = read_pt_data(bench_dir, bench_data_name) else: b_value = read_npy_data(bench_dir, bench_data_name) else: - n_value = read_npy_data(npu_dir, npu_data_name) - b_value = read_npy_data(bench_dir, bench_data_name) + n_value = read_pt_data(npu_dir, npu_data_name) + b_value = read_pt_data(bench_dir, bench_data_name) except IOError as error: error_file = error.filename n_value, b_value = CompareConst.READ_NONE, CompareConst.READ_NONE diff --git a/debug/accuracy_tools/msprobe/core/compare/utils.py b/debug/accuracy_tools/msprobe/core/compare/utils.py index a2ba55fb46..471951ce4b 100644 --- a/debug/accuracy_tools/msprobe/core/compare/utils.py +++ b/debug/accuracy_tools/msprobe/core/compare/utils.py @@ -24,7 +24,7 @@ import numpy as np from msprobe.core.common.const import Const, CompareConst, FileCheckConst from msprobe.core.common.utils import CompareException, check_regex_prefix_format_valid, logger, safe_get_value -from msprobe.core.common.file_utils import check_file_or_directory_path, FileChecker, load_pt, load_npy +from msprobe.core.common.file_utils import check_file_or_directory_path, FileChecker, load_npy def extract_json(dirname, stack_json=False): @@ -602,43 +602,6 @@ def reorder_op_x_list(op_name_list, summary_list, data_name_list): return op_name_reorder, summary_reorder, data_name_reorder -def read_pt_data(dir_path, file_name): - if not file_name: - return None - - data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.PT_SUFFIX, False) - data_path = path_checker.common_check() - try: - # detach because numpy can not process gradient information - data_value = load_pt(data_path, to_cpu=True).detach() - except RuntimeError as e: - # 这里捕获 load_pt 中抛出的异常 - logger.error(f"Failed to load the .pt file at {data_path}.") - raise CompareException(CompareException.INVALID_FILE_ERROR) from e - except AttributeError as e: - # 这里捕获 detach 方法抛出的异常 - logger.error(f"Failed to detach the loaded tensor.") - raise CompareException(CompareException.DETACH_ERROR) from e - if data_value.dtype == torch.bfloat16: - data_value = data_value.to(torch.float32) - data_value = data_value.numpy() - return data_value - - -def read_npy_data(dir_path, file_name): - if not file_name: - return None - - data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.NUMPY_SUFFIX, False) - data_path = path_checker.common_check() - data_value = load_npy(data_path) - return data_value - - def _compare_parser(parser): parser.add_argument("-i", "--input_path", dest="input_path", type=str, help=" The compare input path, a dict json.", required=True) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 9abe144659..5344573ad9 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -22,7 +22,8 @@ import pandas as pd from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import FileOpen, create_directory, load_json, load_yaml +from msprobe.core.common.file_utils import FileOpen, create_directory, load_json, load_yaml, load_npy, FileChecker, \ + FileCheckConst from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, \ check_op_str_pattern_valid, get_dump_mode, set_dump_path @@ -385,6 +386,18 @@ def check_cross_framework(bench_json_path): return False +def read_npy_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.NUMPY_SUFFIX, False) + data_path = path_checker.common_check() + data_value = load_npy(data_path) + return data_value + + def ms_compare(input_param, output_path, **kwargs): try: auto_analyze = kwargs.get('auto_analyze', True) diff --git a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py index 05da6954cd..ec2a4b7165 100644 --- a/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py +++ b/debug/accuracy_tools/msprobe/pytorch/api_accuracy_checker/run_ut/data_generate.py @@ -23,8 +23,9 @@ import numpy from msprobe.pytorch.api_accuracy_checker.run_ut.run_ut_utils import hf_32_standard_api from msprobe.pytorch.api_accuracy_checker.common.utils import check_object_type, get_full_data_path, \ CompareException, get_module_and_atttribute_name, get_attribute -from msprobe.core.common.file_utils import FileChecker, load_pt, load_npy +from msprobe.core.common.file_utils import FileChecker, load_npy from msprobe.pytorch.common.log import logger +from msprobe.pytorch.common.utils import load_pt from msprobe.core.common.const import Const, FileCheckConst, CompareConst diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 1f938e5a38..4021430ed6 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -309,6 +309,19 @@ def print_rank_0(message): logger.info(message) +def load_pt(pt_path, to_cpu=False): + pt_path = os.path.realpath(pt_path) + check_file_or_directory_path(pt_path) + try: + if to_cpu: + pt = torch.load(pt_path, map_location=torch.device("cpu"), weights_only=True) + else: + pt = torch.load(pt_path, weights_only=True) + except Exception as e: + raise RuntimeError(f"load pt file {pt_path} failed") from e + return pt + + def save_pt(tensor, filepath): check_path_before_create(filepath) filepath = os.path.realpath(filepath) @@ -460,3 +473,28 @@ def replace_last_occurrence(text, old, new): if index != -1: return text[:index] + text[index:].replace(old, new, 1) return text + + +def read_pt_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.PT_SUFFIX, False) + data_path = path_checker.common_check() + try: + # detach because numpy can not process gradient information + data_value = load_pt(data_path, to_cpu=True).detach() + except RuntimeError as e: + # 这里捕获 load_pt 中抛出的异常 + logger.error(f"Failed to load the .pt file at {data_path}.") + raise CompareException(CompareException.INVALID_FILE_ERROR) from e + except AttributeError as e: + # 这里捕获 detach 方法抛出的异常 + logger.error(f"Failed to detach the loaded tensor.") + raise CompareException(CompareException.DETACH_ERROR) from e + if data_value.dtype == torch.bfloat16: + data_value = data_value.to(torch.float32) + data_value = data_value.numpy() + return data_value diff --git a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py index 7595c866bf..7c1670dac7 100644 --- a/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py +++ b/debug/accuracy_tools/msprobe/pytorch/compare/pt_compare.py @@ -12,14 +12,18 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import os + +import torch from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import create_directory, load_yaml +from msprobe.core.common.file_utils import create_directory, load_yaml, FileChecker, FileCheckConst from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, get_dump_mode, \ set_dump_path from msprobe.core.compare.acc_compare import Comparator, ModeConfig from msprobe.core.compare.utils import set_stack_json_path from msprobe.pytorch.common.log import logger +from msprobe.pytorch.common.utils import load_pt class PTComparator(Comparator): @@ -50,6 +54,31 @@ class PTComparator(Comparator): return mapping_dict +def read_pt_data(dir_path, file_name): + if not file_name: + return None + + data_path = os.path.join(dir_path, file_name) + path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, + FileCheckConst.PT_SUFFIX, False) + data_path = path_checker.common_check() + try: + # detach because numpy can not process gradient information + data_value = load_pt(data_path, to_cpu=True).detach() + except RuntimeError as e: + # 这里捕获 load_pt 中抛出的异常 + logger.error(f"Failed to load the .pt file at {data_path}.") + raise CompareException(CompareException.INVALID_FILE_ERROR) from e + except AttributeError as e: + # 这里捕获 detach 方法抛出的异常 + logger.error(f"Failed to detach the loaded tensor.") + raise CompareException(CompareException.DETACH_ERROR) from e + if data_value.dtype == torch.bfloat16: + data_value = data_value.to(torch.float32) + data_value = data_value.numpy() + return data_value + + def compare(input_param, output_path, **kwargs): try: auto_analyze = kwargs.get('auto_analyze', True) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_file_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_file_utils.py index 303c083eaa..ac3a859bf4 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_file_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_file_utils.py @@ -1,6 +1,4 @@ -import unittest from unittest.mock import patch, mock_open, MagicMock -import tempfile import pytest @@ -534,36 +532,3 @@ class TestDirectoryChecks: check_file_or_directory_path(self.test_file, isdir=False) # Test directory path check_file_or_directory_path(self.test_dir, isdir=True) - - -class TestLoadPt(unittest.TestCase): - - def setUp(self): - self.temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.pt') - tensor = torch.tensor([1, 2, 3]) - torch.save(tensor, self.temp_file.name) - - @patch('torch.load') - def test_load_pt_cpu(self, mock_load): - mock_load.return_value = torch.tensor([1, 2, 3]) - result = load_pt(self.temp_file.name, to_cpu=True) - self.assertTrue(torch.equal(result, torch.tensor([1, 2, 3]))) - mock_load.assert_called_once_with(self.temp_file.name, map_location=torch.device("cpu"), weights_only=True) - - @patch('torch.load') - def test_load_pt_nogpu(self, mock_load): - mock_load.return_value = torch.tensor([1, 2, 3]) - result = load_pt(self.temp_file.name, to_cpu=False) - self.assertTrue(torch.equal(result, torch.tensor([1, 2, 3]))) - mock_load.assert_called_once_with(self.temp_file.name, weights_only=True) - - @patch('torch.load') - def test_load_pt_failure(self, mock_load): - mock_load.side_effect = RuntimeError("Load failed") - with self.assertRaises(RuntimeError) as context: - load_pt(self.temp_file.name) - self.assertIn("load pt file", str(context.exception)) - - def tearDown(self): - if os.path.isfile(self.temp_file.name): - os.remove(self.temp_file.name) diff --git a/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_pt_utils.py b/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_pt_utils.py index 61f7d97b55..b1ac148ae7 100644 --- a/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_pt_utils.py +++ b/debug/accuracy_tools/msprobe/test/pytorch_ut/common/test_pt_utils.py @@ -2,6 +2,7 @@ import os import io import unittest from unittest.mock import MagicMock, patch +import tempfile import torch import torch.distributed as dist @@ -9,9 +10,8 @@ import torch.distributed as dist from msprobe.core.common.file_utils import FileCheckConst from msprobe.core.common.exceptions import DistributedNotInitializedError from msprobe.pytorch.api_accuracy_checker.common.utils import ApiData -from msprobe.pytorch.common.utils import parameter_adapter, get_rank_if_initialized, \ - get_tensor_rank, get_rank_id, print_rank_0, save_pt, save_api_data, \ - load_api_data, save_pkl, load_pkl +from msprobe.pytorch.common.utils import parameter_adapter, get_rank_if_initialized, get_tensor_rank, get_rank_id, \ + print_rank_0, load_pt, save_pt, save_api_data, load_api_data, save_pkl, load_pkl class TestParameterAdapter(unittest.TestCase): @@ -148,6 +148,39 @@ class TestPrintRank0(unittest.TestCase): mock_logger_info.assert_called_once_with(message) +class TestLoadPt(unittest.TestCase): + + def setUp(self): + self.temp_file = tempfile.NamedTemporaryFile(delete=False, suffix='.pt') + tensor = torch.tensor([1, 2, 3]) + torch.save(tensor, self.temp_file.name) + + @patch('torch.load') + def test_load_pt_cpu(self, mock_load): + mock_load.return_value = torch.tensor([1, 2, 3]) + result = load_pt(self.temp_file.name, to_cpu=True) + self.assertTrue(torch.equal(result, torch.tensor([1, 2, 3]))) + mock_load.assert_called_once_with(self.temp_file.name, map_location=torch.device("cpu"), weights_only=True) + + @patch('torch.load') + def test_load_pt_nogpu(self, mock_load): + mock_load.return_value = torch.tensor([1, 2, 3]) + result = load_pt(self.temp_file.name, to_cpu=False) + self.assertTrue(torch.equal(result, torch.tensor([1, 2, 3]))) + mock_load.assert_called_once_with(self.temp_file.name, weights_only=True) + + @patch('torch.load') + def test_load_pt_failure(self, mock_load): + mock_load.side_effect = RuntimeError("Load failed") + with self.assertRaises(RuntimeError) as context: + load_pt(self.temp_file.name) + self.assertIn("load pt file", str(context.exception)) + + def tearDown(self): + if os.path.isfile(self.temp_file.name): + os.remove(self.temp_file.name) + + class TestSavePT(unittest.TestCase): def setUp(self): -- Gitee From add8d615b26727f67c969709b957503a8167ade3 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Mon, 3 Mar 2025 11:01:05 +0800 Subject: [PATCH 15/37] =?UTF-8?q?=E3=80=90=E8=B5=84=E6=96=99=E4=BC=98?= =?UTF-8?q?=E5=8C=96=E3=80=91=E7=BA=A0=E6=AD=A3Atlas=E8=AE=AD=E7=BB=83?= =?UTF-8?q?=E7=B3=BB=E5=88=97=E4=BA=A7=E5=93=81=E4=B8=8D=E6=94=AF=E6=8C=81?= =?UTF-8?q?INF/NAN=E6=A8=A1=E5=BC=8F=E7=9A=84=E8=AF=B4=E6=98=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md | 2 +- .../accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md b/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md index 97b049000c..983477554e 100644 --- a/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/12.overflow_check_PyTorch.md @@ -28,7 +28,7 @@ msprobe 工具在 PyTorch 场景下提供溢出数据采集功能和溢出数据 溢出数据采集功能在昇腾 NPU 上支持饱和模式（仅支持 Atlas 训练系列产品）和 INF/NAN 模式。 -INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品，默认为饱和模式，且不建议使用 INF/NAN 模式；Atlas A2 训练系列产品，默认为 INF/NAN 模式，且不建议使用饱和模式。 +INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品，默认为饱和模式，且不支持使用 INF/NAN 模式；Atlas A2 训练系列产品，默认为 INF/NAN 模式，且不建议使用饱和模式。 INF/NAN 模式的使能方式如下： diff --git a/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md b/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md index 33ff4a0259..ef83aa1723 100644 --- a/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/13.overflow_check_MindSpore.md @@ -11,7 +11,7 @@ export INF_NAN_MODE_ENABLE=1 export MS_ASCEND_CHECK_OVERFLOW_MODE="INFNAN_MODE" ``` -**a**：在处理浮点数计算溢出问题时，NPU 当前支持两种溢出模式：INF/NAN 模式与饱和模式。INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品，默认为饱和模式，且不建议使用 INF/NAN 模式；Atlas A2训练系列产品，默认为 INF/NAN 模式，且不建议使用饱和模式。对于 MindSpore 框架侧配置，仅支持对 Atlas A2 训练系列产品进行设置，默认为 INF/NAN 模式。CANN 侧 与 MindSpore 框架侧配置须一致。 +**a**：在处理浮点数计算溢出问题时，NPU 当前支持两种溢出模式：INF/NAN 模式与饱和模式。INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品，默认为饱和模式，且不支持使用 INF/NAN 模式；Atlas A2训练系列产品，默认为 INF/NAN 模式，且不建议使用饱和模式。对于 MindSpore 框架侧配置，仅支持对 Atlas A2 训练系列产品进行设置，默认为 INF/NAN 模式。CANN 侧 与 MindSpore 框架侧配置须一致。 溢出检测任务的配置示例见[MindSpore 静态图场景下 task 配置为 overflow_check](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#23-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-overflow_check)、[MindSpore 动态图场景下 task 配置为 overflow_check](https://gitee.com/ascend/mstt/blob/master/debug/accuracy_tools/msprobe/docs/03.config_examples.md#33-task-%E9%85%8D%E7%BD%AE%E4%B8%BA-overflow_check)。 -- Gitee From f141ed422d33cee2ed3ff2799fa709e597be7a53 Mon Sep 17 00:00:00 2001 From: gitee Date: Mon, 3 Mar 2025 10:14:49 +0800 Subject: [PATCH 16/37] update 1.2.2 whl --- debug/accuracy_tools/msprobe/docs/01.installation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/docs/01.installation.md b/debug/accuracy_tools/msprobe/docs/01.installation.md index 1ab5f6419b..530783e87d 100644 --- a/debug/accuracy_tools/msprobe/docs/01.installation.md +++ b/debug/accuracy_tools/msprobe/docs/01.installation.md @@ -16,7 +16,7 @@ pip install mindstudio-probe |版本|发布日期|支持 PyTorch 版本|支持 MindSpore 版本|下载链接|校验码| |:--:|:--:|:--:|:--:|:--:|:--:| -|1.2.2|2025.2.26|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.2/mindstudio_probe-1.2.2-py3-none-any.whl)|1db0cf4572bc0305c68705b74775f652c6cb2c2bedb6c6e57f43e31ab273b288| +|1.2.2|2025.3.03|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.2.2-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.2/mindstudio_probe-1.2.2-py3-none-any.whl)|961411bb460d327ea51d6ca4d0c8e8c5565f07c0852d7b8592b781ca35b87212| |1.2.1|2025.2.07|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.2.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.2/mindstudio_probe-1.2.1-py3-none-any.whl)|b64b342118558e0339b39237f88a49b93fd24551b0cb202c872fbfef4260c86b| |1.2.0|2025.1.13|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.2.0-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.2/mindstudio_probe-1.2.0-py3-none-any.whl)|1e3aeea1706112f6ee52fd1165037936bb209138f0b9ec42ea21e2c1c8942cdc| |1.1.1|2024.12.09|1.11/2.0/2.1/2.2|2.4.0|[mindstudio_probe-1.1.1-py3-none-any.whl](https://ptdbg.obs.myhuaweicloud.com/msprobe/1.1/mindstudio_probe-1.1.1-py3-none-any.whl)|577b597555dc155b76ba1a62d575c3546004644e140a456c3ba0824d46283735| -- Gitee From 89e2db6f35a9d07fc5c36beba49c47b65ce33855 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 28 Feb 2025 10:38:47 +0800 Subject: [PATCH 17/37] save tuple fix --- debug/accuracy_tools/msprobe/mindspore/common/utils.py | 4 ++-- debug/accuracy_tools/msprobe/pytorch/common/utils.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/debug/accuracy_tools/msprobe/mindspore/common/utils.py b/debug/accuracy_tools/msprobe/mindspore/common/utils.py index b8ed5e143f..dc9da34490 100644 --- a/debug/accuracy_tools/msprobe/mindspore/common/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/common/utils.py @@ -182,9 +182,9 @@ def set_register_backward_hook_functions(): def check_save_param(variable, name, save_backward): # try catch this api to skip invalid call - if not isinstance(variable, (list, dict, ms.Tensor, int, float, str)): + if not isinstance(variable, (list, dict, tuple, ms.Tensor, int, float, str)): logger.warning("PrecisionDebugger.save variable type not valid, " - "should be one of list, dict, ms.Tensor, int, float or string. " + "should be one of list, dict, tuple, ms.Tensor, int, float or string. " "Skip current save process.") raise ValueError if not isinstance(name, str): diff --git a/debug/accuracy_tools/msprobe/pytorch/common/utils.py b/debug/accuracy_tools/msprobe/pytorch/common/utils.py index 4021430ed6..7a3735a529 100644 --- a/debug/accuracy_tools/msprobe/pytorch/common/utils.py +++ b/debug/accuracy_tools/msprobe/pytorch/common/utils.py @@ -449,9 +449,9 @@ def is_recomputation(): def check_save_param(variable, name, save_backward): # try catch this api to skip invalid call - if not isinstance(variable, (list, dict, torch.Tensor, int, float, str)): + if not isinstance(variable, (list, dict, tuple, torch.Tensor, int, float, str)): logger.warning("PrecisionDebugger.save variable type not valid, " - "should be one of list, dict, torch.Tensor, int, float or string. " + "should be one of list, dict, tuple, torch.Tensor, int, float or string. " "Skip current save process.") raise ValueError if not isinstance(name, str): -- Gitee From 612428c8b1a96b5391da956eb457570a27f020b8 Mon Sep 17 00:00:00 2001 From: qianzhengxin Date: Fri, 28 Feb 2025 10:48:09 +0800 Subject: [PATCH 18/37] doc fix --- debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md | 2 +- debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md index db9a989c9d..c2e33436e5 100644 --- a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md @@ -183,7 +183,7 @@ save(variable, name, save_backward=True) **参数说明**: | 参数名称 | 参数含义 | 支持数据类型 | 是否必选| | ---------- | ------------------| ------------------- | ------------------- | -| variable | 需要保存的变量 |dict, list, torch.tensor, int, float, str | 是 | +| variable | 需要保存的变量 |dict, list, tuple, torch.tensor, int, float, str | 是 | | name | 指定的名称 | str | 是 | | save_backward | 是否保存反向数据 | boolean | 否 | diff --git a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md index f7507facd2..96d37c170f 100644 --- a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md @@ -144,7 +144,7 @@ save(variable, name, save_backward=True) **参数说明**: | 参数名称 | 参数含义 | 支持数据类型 | 是否必选| | ---------- | ------------------| ------------------- | ------------------- | -| variable | 需要保存的变量 |dict, list, torch.tensor, int, float, str | 是 | +| variable | 需要保存的变量 |dict, list, tuple, torch.tensor, int, float, str | 是 | | name | 指定的名称 | str | 是 | | save_backward | 是否保存反向数据 | boolean | 否 | -- Gitee From d5a3cc6df1dddd0d9a1e163ebb7a379fee39c889 Mon Sep 17 00:00:00 2001 From: cai-weiwei1989 <734267852@qq.com> Date: Fri, 28 Feb 2025 16:44:47 +0800 Subject: [PATCH 19/37] =?UTF-8?q?[msprobe]10.accuracy=5Fcompare=5FPyTorch.?= =?UTF-8?q?md=E6=AF=94=E5=AF=B9=E7=BB=93=E6=9E=9C=E6=88=AA=E5=9B=BE?= =?UTF-8?q?=E6=9B=B4=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../msprobe/docs/img/compare_result.png | Bin 77942 -> 62634 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/img/compare_result.png b/debug/accuracy_tools/msprobe/docs/img/compare_result.png index 07cdb51707fe43d07723ed976275d99f55b50571..b6d7ec6dfcbc44b4b7056e1297a481f495ceb86e 100644 GIT binary patch literal 62634 zcmd43dstHG`akSUGu3o*S7W86%+6_QQpYhf#{;mNOxm3sM~&2!DG#VvmSQT1t6kGc zQ?|Rw(o&|VG*L)VOAM$f#WF-RKrNv%MLD8uRG@V)rGyLS4=r5*fg ze|*}3gs-v>9){kwvkOFKAKo8_OTd42AOV|v2)<*73;Ntw^*CG)M=LKf_LnRzc!0-Po`(Wy!4F7ES>bC{^L)SCOo~sdo>G6sF z-v&~v6Ce`pxuT?*7!=<2tTnHi?vMmOOQG^AP3*t!37X}+D^RL$RmHiu`& zj{#a&eA0wprqe z%}ABL6nH+fe_KghHfBP;ynIf1-bVB^N_~|%exnDO%5_2;_~#(vvspC*K>3jq>(CQe zqt)(vfPUY>a(0~J_9CF;q_sI8WeCgo7$|vZl&^={4%mWZf;aO(_wl+LJZy#oP*Om^ ze+*1dUWbzN!F^|;y5lF&Go8RAV;)e}WZN=M>Q8lSwM^>#=F`qs{<^mQ8O%_Hm z?Qenq9LrYZGXd{GCDf>vG!QKf*3yY7e0qAofl6njyrp9~%b?jeAG&FPu34?a{*GCjLIQ^WP zF=Old)jJG5*CWwJUklWV)#xdCioP@bufXWTd>VTWD<6se7#j80_ORJfh>YU&T@mBu zLkGZpi)R?gOfz-nWQZ?xTD<2zu~iss%iHflmCnr68#26!{?SZpC~oO%0Ec3tE250@yMx@Os}p;fcAQfa?U1muX;PTcq!qbZDJ@_M=-#>U|tdu{6{k$GME#Llj4)(RetajfC1h3bF?#rzS zKfW8oVRv9U<9pK1MTaQG)%!f#D zhS?1Z9CBILkmwKp4?Gl_4~t}L-eyq-G-K@%mgas^+|0~FYvFeuiJs!+MGkKNPx|XL z@z1p!{uls#4Yfbf0=+N~D03|eAyK)2J}PjyJ45J#2mVFOH3shko$p=@UCq|xt z`#$*dTHuYt%nvS+Mmnuam%^o;uc5agqMDg@;~=Gy@ypw3n&ioR@8u1go}pOcyFwoO z4Lv+c?1EP8n|Zzk`iDTwt6c&IOXQ3Bh6@mLdmS>f+_^#EzE&#DnBfDh>0dM$KL(&BZ~#hI7m{NB z;cXF`f(@of-Iq_I)GIBLYr(6`GM#`p0>8M>nA-*0O+Ui#s$B|GL+P`3KFw{9G^Wd&?vuznbRo zpkWy1UzlN>rn}u5ZzzA;gv&5$vi_Vb?qq%5tx26pUAv{t_Uch2+Yt2cq5z-B~-Ec`bqErTi* z&S=zBo_gi`JH`ILx%zsR{@otD2ncrhPl0C9pObDcL8Q+4bJE^VU`?1yyurFa7yE^? z7Cf-d@;+>{dw;w zA!)|6x~)$cLr?42n4};tFo^`P|lK2mvNqC3Vzh zw#l-B;J4m35w_LLsFgkG=MB+@KeRi>HLAF&ch;&$T5R=e$JPk|b=SQ{EvjeCrmP|p z%`{R2AjkY9=Vk?2nYi<<0(Ws@mE8Y~k#Igi~*uu`?w1)&u-#BuGjLAReX6rlRP z5Jb3;)4)-FD2W_Ql9|D?CKOvg;i(MRu_D|@5f!uk;ywisFK%UMcTMDKu`WC9W%No-Pi`I)q%2-^8u$Qsy2V~F= zF0to=LcH0AN3_D8OQI01l-k_mNGt+i2T%^%p8f!i^QX8a?D4i7Pt<0)6lEN(j>N>mKUtkNf;KUwon(D{ABQyK4j}gYRW(vngf(L z(?j;CZ*3F|^2aF&Ll)O6T~ViWg8fzLteuRyuHGy`c8l9KBEF5Fl~tf} zs>ilM3D@>Mic|y7?xddDj`rxd{OSpKE-6Opj@Ds6nfhs`rp_JvBh%veR{|rYrVqHD zT&P@3UTHOdD}R#W#_nG2k~SkF26pSd-kEmRXCX)2zPmoznrw;)g`Byx`+4?f%YN)C08)0O8+~;%b zUT!-Db{r+uS4)2wE3Cz0z{lOg6Ef}Na254RQHnYO=n$=n0rpU?Di3N@`o?VAYJ2h{ z-3{K+aW3v@ksCR@ph`#Aa%}A;;)wU&5O`CDjpQcfwm)|futZO2{afga_{lY;A(HQQY0(qgz~aSYZuttAYq5?OfE``Ylauc9ZFs~$ z3uRXvH5^y9EHnl~n5geHryCXdam_#8{AaubZU-tQ?_KX$XgHK3vk=bsK2Pj%`;cw#t_o@;Y=vx;7f zn7*Vym*-U~(BJ+}P2-K>3%zXS$J#W%Y%WUId7HR8>N&68+@@r@IO9^Ty(m_xwCjBH z9Z}}Ex^0&e;yF~m0~lPDSb{qsNeQW#rI%m0l?dDbu!e7@40UkYa>nbOK=3!jn{H-r z0^s~O!t&S!8Tz!V3_E2B(@qe{DFM||f7y~ZDr1E~+9yqwT>$CexvpBser|$@UjHT6^Evf7 z2G1Qa{_9O)SFOw|n)|ugOE+R~cX$(kNWHVXfLSHJv%F6E_vT-eckZoNFT6t3A@h?C z#ffD%6t?l2W?gYe)?J|Ji>aaYPxnErt)%}PTH8gsnM~p zgCQo;Pg#22nF`4QuFbvj0NGd+e-yx`Pg@o0f>9@FACk(l35Ms9DUb;FF?}>dZ^&)6 zW+nh7uban2L5NI=S~;2nu&GGmR1t{RFq*kiAtm9pu|r3bF&>qv%xA;(i~)7=rfZ%j zKopsV*1Ryx_i;7TZud8;oQO5ZFqzkLnR0Bt-AmB|>%}qbfsp!WO44%u>JhFkXl5VW zrkl`@2M(;{cp#p}X0fxHbj|CfvVd!n#4+t;CMx@9W^K7T6|d_vN(h!OCGwigfFd1m z3LXo?GR#%TXtUt#|w;& za5!SRKPG3-kc0jrA>&O{RuH@SGSa_=0+7u+h=OVV6Mb@qo0`Z_nJ*cP*Sx}rpA*!| zD+m{>xv^V6VcUk6HjdVDOZ)q@SXF>-M6G7@#oF0L-?f|`tLR(4DUN|~#F>!ieOF)) zi&?}CQ>TuOAtb|s`b0KbhC~^NzT#0vMJd? zw40#E<|34JtcsA3daLh5YRvPDHzN0F)4`dWfl$C7+BeRws9nh~7>*&@P)Pz;Asnw! zr4sT!+7#z=QSRXFh_kkMmol2L+7CZl-GZt&1T2MPY9=?sGD5lR($ciZNbcG`7HGuT z*JrM#zY);(GP!Q1^R<*#N&QVhAJ`j*iX#fg7(ExZtIniI=-fFdJ*8D3;rA;le45c; zJV$+s$p93CU%y@)7+YAw_}5UvB)gYpizV6F1sm(`u7#17G-Yew#Cy)x-GHxe0K!$~ z%k<2AvMLvsIIW7;M%a=%*WHAR#n_}n4Qdb2sjSi(L2lO}j;m!NlsUXQyXjO)MG}WD zRdJ+e4)n(zB%Kn7q)(OsnI*fS_c*6goX6CXGs^l)VuHoVMdqfe7M=Jqr5 zQFW>6>YNy2|HZXYJ0>$j$&1Pbf^W8O!hT9VfU{_lJZmH1-PYu38JTI-s1Zv)^`Ub5nXK1 zn^HpU2e=XvdvR+6uitdeeQk8Kq_)V~Oc~He9%pY=1>p>rbbF&I#Y0I*S(p!BpHxm_ zpGUHqFqljjMtYyp%BS-}iu$%tXWg%MLqY}2WKwR+<>jmh{jg>xi_ ztdd%D5y&2Y_>}Cn-zc$iraf^VVJO;m-Qw0vX=-pskNOlDSL;=uS8}nHYC4m2s4&9H z%*@@#?lFE~7yE!*X$fYy&|JLADuXgZ?I~f-xTh1Usy$aTBS|;OZS{sEch@I{%G}+V zHF&El{J_SL$xK(RtDmFDsq`jut6ymBr;G!Uxf@GfWuQ`m(*r{>@IBC5o>ZEi;7e(% zk6P2+xarB+scYE|9j~c%Mf5gRwAkr!$8_r0PQ73B0ihG>swz=MTQ-zj)E2&ye`{mH z=h%fDij9s-s*t!IoZc5#y_IkSdWcfG4ix#64CTPP>ofS0-ZM8@0%6>mZkBXMJ=t9f zjrH8fgV2N`T(0^EkAzcN;Zc;K#K|=ybtN^M@==h`HImW*+(4Zh{*u!MNDyVq`6J}U zJhxEf+ZKli*~w2SDBQWWrX||$IFW0x=2Ofd7YCDJd>B|B=ZpM=pk_i1;ZaiTkBmY> z)o{jiV1Y6FQ&r0Jj>pr&y~|*Q35Gyh49>?hN>BvPTV}fvZ%T46Ah&JEt*}*SX$^C0 zJJP*L8%tVIagGQ9Ac=zJp5YedTeiHFHj_QVV5!>eL!rv^m4w)#^%v-6ND1Ri)gA>Q zbu+dHbdoeigs~q=4C?;MC~9}`^qR;X9{VP*8tT#35laMlr+)G)9?q%9pMk4I5zoCz zm~>yex{Q{e$#e3~4H4Nvvt0{`&G)6@+%d6rw)1p2_?v?GqTJ19*ewe_Uhr$ zKPMf#Fk?x-(m6P3(0nX=0Q>c+A-{zWJkZp}eZIyA;-Ut4Aazh-`U{%juNraZbTkJBs{AW_+YE`VU!>^BH`ZTqM;xbkDti>J_{{P{@j0SYCew45t>_S>wk5P$T>YX~ z#2~3On9!}-v=|6TR25dCd@+@tZo?hv^@h25oT`vYa+fHRRXh~ItSybn(5`y9&0=lR zN{J!?(v_8&iHX+f$FJmAbqDy`0hXj#>Og*@Inh!l;B#i?d%)7ej|N**3{%vEJmVWv6n7Sw_kgCJu|&LqjA+c$NN+ub zvnRzWWDN(gLX#b+R&SQcF~%yqvKw<8z|n7%r7aAt?PU0s)BTDM1@*^GP2SKb7L-ec ze2JpMoiP<6N0WZ622gjv_eFT0qf>+;v?=Z-s8j|30o5SxOiGK4c z$&eN?`Ysnse=e0EUsrL^uNg?!w3P^R+uiksSrbw8iElF#pl-g5l8hj0$hjHWsJFJx zZ17g87OOThXtHSXFk~J8d??)_fs$E;Va1*vR>LXXS~Te>NB(9|l>{}5?K;IcP|FRq z`DnMJxp!Rnw(ZxnG~)$2r-#C~I*8CowxTLqnW+(Pji8h>H2T!vGuaV)2BZ*Q~KtcIf0wAB*ux3}ou5GY_A|F}%bQS74Zr|DTYtpiwWQWjQ4L6bE zT-P3|OELY(nkQ!oxX!AZyzSSMF>pruh24TPV%fKQ0{W5N_>dXKQ(m#SmV2_a-&N>a zL4NGp@T7v>`8PRlX5F0`k>bX!F0a?)iAubn)TO*xm;Ta(nhrdE$Tl#j>QPt}7E+c! z&^bJ+%6Rtv0a1z!(LXe&#&BaB5-Vq=CXmfa2fe&R8d^x+$}3BYCp>fQ_Mr#)g{JWp z4eSx|nTA7og$m;aL)mK;{uK4>F!{{Ja=(B(afOai7hAP~Pr_=c)hcl;6kTOIJYqh!yJ8|>qUtgh3YlulVzrG^)ljH(+p;We3rM`c>9urV7n z7SStl)r)Q~Bi3wvDn%0jr8Qu?_U<$;DBfH!MS5D4Y2Q!lw(b+23*9;Sw!vY^L}6`R zW?m0dy;!_MV3oK{;u2M8a4$zNNeF7R`B;5i6wVR58k#6a!u3f{85n;0M+26U3~?@F zm;q74p6V#lgy$|Ws(~B3vmhaRmui8t*8dr;k`vYr)xwtNZ#qTtB;@jK4 z7nV1H5i{KN814W=pu54ih*FiDFBaW%9f%FEE?||iRFKROoznDutu|1>PV_k+s$hbG z?pmPXke_2De{JX(@JQyW5RVS4ri(m$qHl=_dA%C4sBjoPDx8HVV7jCcg?vN{w|W!d zh0?g7UsmB3C?dr6u;5+Bk{|e`M#;~(fe*_4rg*P~M^48UoU66!e~xDKFgge1vVti= zJ1`F?Tb1Wzm__+kcL2FVXL$S?@_s;cN$2%gXXiW;5>8vAf01r63}#&=3O3MSf6pSl zZS{bO_)jBkQodQ8>6WNRs%-89CX~vV3339MU}tv>xmm;TVNA~%X$=ug+hlz?2 zQsZ)hnjV8?$j`wEFA75pmIDm)U{P-d$2M;KQDv|HK@0S8bo_Jhxq$>ao`o`_?C*w( zYcAI{o$ll=m>$8Cb-+m^BJC)t7PIGRouxD8Jhvo?v&hm%rphT}uoyvU9U8`os( z1Mi!E7WfF++Q&28`hF>1FKfpgi~t(w8g-#gq7498rpOq9$EIq*h7Gb&LWFI%dN`mg ziS3g&`bxGyiFtI6+mBJ9bQlq=R%KxXOC&~r8_NWl9!zK8W*q5`ZE-bNF6C!!6cTWj z-iKVrXWTPr0}W^Q;;fsDiNh9DES=t`fZ9yS*&WHG3*mBQZb1#zRAQ_}g)eW-bRnnU z^p&bit#gbGURv7I<-#<>_V2P4@6)zaE9fC;?t;!Yk(YtxdG7q-r>W)*RQ6+n*Txd< zgyG8Q0aIK^go=x*tJ(CVuw6HdtrcVPT`E#{$MEz%)loKhenR8+3m21JnVu(+{9p1K0bomHAbL-Y~6)lKWp+ZZ4{e~42==X7Ucgi?Q?8gzXs_I!W{jIl+Mz7$LD zc5XT@+NvcStp0O;T0G9<0U=bjeO7zbc)-JPH@J%kH7?6v&%Mm;x%dMjdWSwCsIKPp zT4HQhW-f+Wt(8&thSQ8tQwKn*W62LAa+g5U zN^K$y8F_FSnspDYh%AyYujJQ!Rp1sQ-4mm~k>i9Okn_LO7QTMco*Vk2cZVdf5Dj15 zuu184Eg`9PxcVkbI27K$;cF~v52f|OSLz5t$)MI9d#WaFrof#N(pJBYc|g;@MA#!Y zo^M&^tLyd&8H^y$tt}*+OLB~&HCNv{g!jA9wBYHjbGm}`^$QUNBl0*0SAC%rQ${OO zhIYo{;Q`Q0mrEq5y7l8{@iC}O*Y0a&>u>VP7*Rh_ibX4nmx+SB3Qz`1wV1~o%y;)J zSm}>0S(zMF$=W};UGGN->_%4i49*=>XQ1LD0=rrFzFJm1lyGD#_T0#H3)3WMpJg}7 zZ$E$Jk&ydICf*i0@tdCoQCf?HwfL)>V0J4W<{e4md;TTnY>R;FO9nvWX)FX(`r~oT zA+W-te1^#-kvxz9_*PQ>-=IezlP_Tc`{si`mThOZ=ab)_S*D#mkxWEnZn2BAs~@hbndx`!QmCkU*$_Uwv3jeY zL)1kP7E&*-hWF2P)iO>qcOPM3DA(ymcLTfA2fk)K;Eg+xYYqBzU2DkiG|6|{%OX5K5RPP}f2&>&x8d)>vX^SADHzz}GF3|^!%E~hJp;fQefT`H&ldSHh*Akb~{qs5cWfO8tFuT1< z37eJnjcEQ26;c^@gZ~|0HinUtbw8V&nx-)=r^G4NGF7EtHIBsFM)HxJaoKMq)?pbf zb*%uk1+2z(QgbX{@pEHlHVtPMM-25CQVFU(9%!}%hd-ZOaTnn%vbb7lgpivN;5h@b zW5lu;H(c-i z_6S9)+`M4T>YfYhVzuP-@aHG`K!0xKbBx?pe}J?7@_xK^!2p*K9|SqPu<>K_Y9%fM zrr?`}Vw9o3hL>d9?u}O;r5`yqmmP8!0Nc!=@SB*;jW_ups~mGWD-x#tfS=y!5qDzm z@a_|D!n0eQuo?-a_|8HK1QBqse30#-`!?;~^SHF5NQ1_+csVZiQHyDO*ZGwG5J4li zTS4rBhK{=h*Ju+7bFd776e6DDMGV&|K%kNd;d%OskiHgMDtifEF3jLlZnPkr{mG^t zEmTfYg?Jyr1NampesQ6SyX4b<9P z#w0=`!&?2q^^6!ZC~#+jBhABtYgVzzJ6d)G57>QEF* zFX1uot()_5K`^YKk3E|6DS2#r%d7zqrCUBUlJir>ocd;;n>po(jtcgAT?KI=D1ACE zs2#1V6dIedtBlcvE02ZD6x&luu_|MeaFpOObgpX*z{${GPZ`R}xhn3?8m|oiPuv*L zAQxwf#qc?Z`=yifrI%~%)}+~6v3mzGympJTXw=n;?4vAZ8FwN7#Z$|V zQ6+sJg7?-EoIQ>%C+jOj;|Fawtk%KL2<~`DlyGyIA59=OJ$t^7YG&S>TY!d>NTT^- zs-*KC!v*HOY5?$Hn#uTrt@^3Cp@IP67(lib?pwK)L9_K26r0!Pe;HX@tZ-E^_LB)W zRLL1`EKghGY#=RIFR#E83n4kRMx3ik>`*ekZ8)v0=pqISd_stqn}$q=#%Yx(+^ibW zA$xqy{pZIg8#D8$8f?iSu&B>k_=DtNhkvwcy~QU|wft{KZM~!@_;5s+by4qNPziqPctPd<8JDM0|O%Uj5s z2OltQU>G<1F({w3qq~B?3n`kYRQY&wy`nK*&xz1{-PVHs2M}Y;311oZ1&>qM>OPv6 zIdGq-n!DpXci9+MXUf^7()~>yecT5-j5$HioP!^3hZUfB&IFGQO*+|$L+ssL|11jN z`2^`|`!ev*1i8oi0qpLEqdH$Hs|?M2(P`8Yps+lKFC3dg9d zXY8is!rTGv*rnRj#IP6}6$V4;Vf)EeoHVMMKtwcUS|pHa3p+Tdp`&9kWBsyc29l7sQGjcEOj zqelh3>A=DAw2mmdwqiY*TS<c~lo0*y3VWG}JDy3`4EOQIkizF-c>=ci}diYnA2AmfK;vf@=k z&X!x>5CEKu4BItk8`os5VsmMNIfrKxFfwf(NA<$H*Ibw-HIERhv)&lZ7Z+Qy1Z(#J zcjwS-9lFM}_R-BRg&8+`;QdI#{lfahfpd*;Jecq^s*>1XbF)n5FjY8f{ehyPtno5S zV~{5ErQjQ;7Nv&jcZPNkW$PA~9Ruz$R(Drxfb~g`W>22mfY$GFu}TsuB9C&b)x%-V zu>~fE1gkR%y7|64B(@PN%E+N~X8HI7crc0U5C!<+^9BI9Y(Qv@dJ4-jOLE8VlpKPG zWLb;8hzgNMGASM-1pg3TDyel8q~@^lO6jEFE;BBzfjgch=0@T{xg1uoP$j#$UD@=E-XB#7F+Kbmtue}^5nIsifivjY1$lC9R68hUmcf2BZmbI>D zIr@`|jT<)DuRj<+Hnhuft6e7-`kh;swUE_BM?MGa@oh;rLne6wX2Mu?EPNVK^t~1) zy!b;A`l>1z7aiDxUC}IA&yKs!VIi1WSG*QC?aqtKA*9Md;-@qvvUuEt4~MigAMBf` zr_->jO60c@$PE-_zOKu(I9HuixS%#V3>OZbn}+pqb&gw!SSz+lC-d^hG}-8RdtjSY zk`1pZYG)Fqlp6(zVdc+@s8@C}E|sVC?YBjcPRleUR`N5I1>vH=>28ZD+b9WbwngPJ zlG~SDhz+QXR9)YBM*y#@L2cr;QP~QoP+M)1y2^G2$+8!0Bel5=Ym#=0)8cm-0u$A+ zTJix`@>$w7;@K{1pLFy^QQ>*ck!}&IFlY1kM1tcoM5NmU~X^{|L!M#%|dZSF%*0zE76lr|iAG zs^dSj!b?krf|Dwc64Sapt?**Hs_IZsRCrjcfAzg>8r)Xa+9!@@Z_!6gGul+}4abPx z*A=X42!~R9NtsV)%;;a?Wufp=vW}77lpdcJcij^_SIjSlWZn?Haaq-Nl8Zu{%)7tlGi zWy-cQdxr4wjgk9qO;sDi3(_}wt}a*^V_Q($)4vYV!jjeNu#PW)3rkzi(U02KOCw)+ zZ@^rDWcm?@C>ne^klImT!quokLq(m_lBinrF34ZBnx>6ELZvw?D9H}}(ZnDxvN(mF zfG48AR<@_Iqb9%2PqF~Cqo-m_)u1lPp3*mQ6 z*+=)yD3V*`Ptsj?slt8TV-*M#8D&Cc5>{QclE#y2@=;M2C9Q!FretO7>Uu&*rL0^u zgZBF&Ex6*`ox{^<+nonw_h*d4?~h~p`?lpnS#QQQYX0EkwSScvOs1$j(DGJa``_U8 z?d|TXj3!fe7tG-K{DYTEdMSUGnSHP3%GgBizo@#2ufSicp7~w%KtMtEFrMf3cFx)C zLs1H(<|5O8@kBdYUSt|m2o?kDtyRbed9zWYgXNarG}L3NI*EXM=WC5dRhvmXqBCU; zI$Qc@ygs)*k(Qj#=?g@SO{%MYghk(7)fZF%{-$ppT~)HAz`u4LLk@;(1`1v8J)(@A<9xTLKi~wC z@*r+%p?vMvroD#n8Lvk=iVG)R87GIF0v@;(2~sj8##K($+WEFW19x{FM% z;o1SqjPjSW{@^L0*45>_z#du{v0C+|T1=T4xDE{2wy}#_8#%vBj|Tkhq;%~IBrs@Q zk+T!__r~G&jMkPgqecSKg5Xa_`8F2b@7x;#vQFf3`a5Q>ibbdy}#GIFnz4L1y z#HrIDm(JZ^F5h%p3SU-~w={F@ADlWmus{hX&>>OL*l`)n|q= zJvP4_0GiLdP5zfneOHY*D}=kkvlsDYM9!jepR&qs?LH0pizuKF2(?zM8sNG@mq@{5 z`Dw`6BM8QZApwRBN!<<#FVjhv#eBWDFtvxne?UAg*&| z`R3NJ0I3$t|39lsp|xJu6m{Qg_s*U^lQdR?=x(^_PgY_!Zwv{#u6WVc3?UJ^SnhOr zXhn*1M9dwX6TDz6ylG;abY=KE#c8p{cpD!5dH)r1U!CKA4CF!axXCN7cm>HP(3)$U zYk6!{3`dt~sg9qP?=J+LT{T(ZO!BlX9ta*8_O*t+b#@4D5iW+JxDdJJNh|+tAx}1kG)ne2GqUQYx?gpfm?ZVMw|b3*2VKb zl$*xj8y>Sj_gQd%_(QQ$>1Lz4#7flt<|NBbe~MQNvX$qS)&e?_r;Dn;ho2zYS& zHKX`vtwB>>oks?UKhOcyHYO&|}j7bm3 z7ou%hPaFk4iotrIM7UfP*ht2{0jrdEUP_)drjmw300irpg?}z=L#Ef zWxfPhL>gxgY9{X3KSxd`-k$TIU5ZMV7W#m8_ncaMt?(ySljn?Y(as^uJZDq{xl&)L z1DVK%vRY(W3TGVjzg%prwz7{iC6u9dBZ@Ao^j6tYt-^3kc_uKzVpWhePV=wb; zw#7D@u_!T~Q<~F2Y1MrzN5`@`bcwN2o3CT`u7mVJp*9m)j>J+mQK>)C!4WIjfZ=C3 z;O*@k2jJY3Ssb}|e3vz5HetrQCD_OHQ8|&FDd$mUL*M{ciK40m2AI(JyjCX4!dvz6 ziMzKDpN<1R`dCrvB6V@uxnk39w0M|2@vv+Ue=M=Vvt_aM3sGQU0z440miV>tzw`-J zu6|BI$qk=R$i2I8?U|O{x_$GwGf^9lp*W;OdmvHGE!KbN7`0MF@srI(2_%XghhZzA zTS2GHs|TYW(l;#_Wzf`km|z;(3G)xx%BcMtzM)9!AvCV0@H@V5J=byCX~ZSAnL~&F z^3(!lg$$QFGOUTElPGz2A3lv0jFu@76$*${9R@&%!6h6%YymWfjG?&O^oDf~O`n<8Sb zZzbh&*Oe?D3a(4tRG_GnxYlO}H-U2p&g~Ea*PC1+ME6rumV`r%ddQ>g-L&O~BDdHc zH^IcIi&ER%*LGu#ldCMub!BovBO*WINXT`SfAx%OBiw7vlfte*fUXU#$tSLkk%U4h zjjEgZnc+Fb!;*abzqx&?uWakdjQBSX(L?0b^;D}55BXSHo&pm};gpwx7OLdy7BOX! z^sP|&Xe7hl#ac*UiYPUk3fu-vk-4Af{t}wJqX%%jlvYxc1-WyK;*qkj7NO5^a3ws8mK3JzK2u$8 zy^y-8ocQ_L1BDesPyeaqzdqIJFtmr2LWBywni<8AL+Xk6`T69i00>7{mtQUEu0bbH zNP{HsmN^>*yarBwGTF`BpE<0^^Y#pke6du)DpCet-|7>wlk9_OT!#_%ue%c^^S+h? z0mPu0fY53OTzx=dLo5oy?JX?#>?qt=8#Zt|`-XWH+(iKsDUST8(>GJ=*1IIf#4azG zz!`Gkf-@->c2oBEP>Sc+^T)AN6RIDWoG>WMD1 zO$v~<3B^p{`yY;D6B>EQLhGx^H^>A_n?!<6?yvk`!MrdSOa}nUkxsEC36T{xJ`eul z*Fd?Y4`*8X`*GJRw`-Bz;YGQ?c7X)yTqu)W{yJ^ps&jg`Cfqucz%iPdEC&7i95EK*w9NEcXBudc8 zY0LRY^X7nx5A0Swv?taveet@?wxwx}!;xvxOB= znQ+No{1p+3wYIw zB1zYYt9Rk1U+_kIsm)Fh_zBaPI<{_^GgezXes*K6FaySnBM+yz`-^i$^^Mg3VI|UU zpJgrNAe2G{8hccC$X1@0nTS>EWdFn4O@e>e`jKS=ylQI<$>tnwjodO6GE6~|@q(Jz zIuS)^WG|cff`lU!OdyC2x@JH3t?3`Rvtb6c1p3k}|7QH($>?%n+4AXoGH>+g?=!Bx z3>mY(zvr6yF>iFQ`iN~sD4!~KQkk+?uQD0N(t6%M$y>`_$AzYqZ}x!*pa#P2%myE^ z{(-DJ^fIJorD3V6Eficqej{7naX-~LiUC>UAQ9@{KRw`Qaj}dxVFsvz&|TWB&otOi zwMoyo@?$pQ2s0i$_moklL=XR!`}s?*o!v_xNE9E44Ql#c)L(D`CBSxp!h-#c6_SR| z`xH#lz+4vCT~k>T!&MC^qAw}VBJFtf!(jAz9Ny;woovHSSW+iPvhBsw?~;m;jlcbY z^MHz0u4iswRF+4d|GTi*KYrSv$$JV^4e0KjS`7FnT4~Bm&YE_K1`a8YK+!(i&WfoW zAA%5sHFk>``v!TErYR1!z{l$v;osnw31Ord#54)<>{{FUmh!$~iP{u1L^!ERDnjDcL589Yj zS(IVQxV1+I@t@~L-G1|2;pOIh?8NNv1Q&MelK`TIb<#(bSTlB2;|XaL15_({3MIMq znt<32L2Lrc!S$-`COGI0}LMmsyINPY#xEMp9hAB)2U}qqb{2hNu9V+irO}Ib! z?6*G?&3db9_-S?o-=TMdo|ZztU($?;$oRPFskOSy?RLUATD0#U9vR%~;d!a0<_I73 z4;?QU)qVx8QeYt2=8Q7*)U_eYq3{M|0iAmq$+!Zuvm58z6NOc=fupTYIHM<@yq8$W zf=L#A(Cicec`94$``7udetwzK7zKPu$gcOxbbc?(l!$`&(!U>LTt-G=GU}yNlRF3a z7|fLg++WZa$CIUjoQ?xgY2ntED;}mY9*0L**Vo_wy#3d--}(hzm?}U+|GA`t6y$~z ztt9&`dY7RqZ-dSMv?o*{`}P&1c6u}zX0?z`Hf`qqK%Kdrp_pCFo*e}#;P zDS_RjA{TQ;Az8_+8KVrv!yC9t$Vg{xZbNA!#nWWzsL{8Xvir>}Fc%mdfy>nakoN8( zqVD5|25Z-eA%H$z+utKecJI(sxeTQgZKy`o(Z%ifT1f?GcU)uzO1lo7@lz-~KeI9T zCXTI}&UEUS3amyomr>6tLu+V08)8(Am5iRfCGt@C6Va-KaPol|$O{<7fRoDG>I2Ej zQ2OvXDglDPFBD0hMHUUKOpp$KuK0@git@+uB+mFQVnmtOzcOZ1y^X8*5_U3UyLWJU z=MVTVC{!cMcPt{gZCj_~xz0Bk8!`Hu?DhN=kNr*vOLL)uqz&jBj;?;#ul>&pAg9u0 zUG>Je&*ck(u~U9!3}R30g~W4ju58{Iwuv%qyd*|@jU7q2R?0N`qC?!SNBP^dMkohh-|>qGopK63M>Dx$S}XDVhoH;#ES0diNdMZXkkrVHj00*1AxK^U3~H%b0~ z;28ho&!=d#+D3~U0I9DMB1?vhvfLLBt-ogWvN-1aq1t0BAI^b5HS+s^hSvC=wD11_ z@?oh|Zr0Zij~+YqKF0cCnr@0`O7euE$Ie52h8Eqs?*OYwIF(}>|0ey8voZH3Gc|m)G z%Amu(8T-4w2UD|QVl7X4ox74a_6bBZ&<1yCgxD*80&7QlPVIPr`p2J3h1fdfV{)Y3 zu{ekiw@l6bFNCAnGH;djJ_X6MBQ73b5V(r}ABsmaU&K0AXRprOlodZP!#n^WH z-W&Hpo)=6S_Fn%Iu}mv=4A+b_x3R$TU&WhWrJYqnGqSneqi{7hT(U|O4$RpA1c#D? zt;9tJ?rFr1VL`j4`c__o%dbMhLyGB&(d4 zr4<@_yi}M7hQ?zvr$T*}4l`!FF=3~E!YrAJ!pj%gj*&9X zPrH#$mJjZJ^}_atFO9slC{0wXGG=VBfs<7}MZ?yWF5Pe3pE9$AwP3GgQ;mAI`$4ZR6p0 zjJl@8gSJ)8TMmdLR+c-Lz__-#WKNhk%;V(lJ;Qs#U)*K~#k~9=$3k+S{#(-tMyA6X z=K_YRS@J1cn0Y7ifqrQ}fsVj_+_}-KpSv_0oejC-=e8pVGSq74$E1F$0JAb5N9wYB zd;&4m>YAr9={-1b_V4wz|74Qtb7!ouX#q8!MsJ39^n$e!!4TyVXg$oZIOm{>Q9tr2 zCllS71aFoX&C(r2i;-?buFmjpUL0g>5)y|8WUGO7&!4*A31iErZ^shCnF4|@zNm6J zmrguc>T<6baf=y5Z&$3X6E5Fdj0F^0mcE<*GeH~lU<#u9tMcTr_&Do@=Di#=RvM%kPcUM~p)X0-7fO}r3 zC?%nL>EPyv_U)K3i)YcMFESxl5*CC9=0OpU#d-|RZf%jz`zJ#jMC@BoR`Nl#0ogJOBA>{T6Zy*KEYxY^egvI=1f0QNs(+Tj(2JI_D!3C+6?1AHV)&_P&FxnQFDD z1$1=ux4_zACKU)6+z|_oh==;9YF>zEMduRBYkvNEz^Lfmi92v#O;^8iO4LXOk+VH% zy|LReAKhG{1HxRky86s*bk?0|FZ(bD)2pG1B#mD2p5+gmd zd0>BeMW|QKfH&Z0oBb_~mWApm5T&yJu%;HQ3aqv_*8@A5FOvp`W`B=%(R^$aJtFu{ z9Js9rJnc{1o@w24fH6+5)LIWv3oWk(9n?&F^8?+9w0blTQEg?_JT^HX6rA{z1Hz*A zQ(~g_kVsuDjbS1B;`K9{q6CKT(NN|T&0;2AO=$-$6^@KpQ7Qerlp5~~Fd+rNNkv+X zo#@Y;D!02;*b63f9$6Qii1M%lL%basz`Dfi=fBltcgJY!?Xf_uaOp84^viS5_R7+W z`t0O8j(;0>w6aQ?If0SLD)+C|g!b{2wl$C{bbC1%e@*HU4^^;N%RV6p6~sh+DHzFw z1;+U@?m97ZHwc@}s3b~VQvB6v1bz9$O|S1L>_Zbzg!bzl1H+fXc6#dDTm}WO+UnoZ z5u27UDbKiBGTPm(l&`Kqpp!=c{!?So$hhfo$kpk${9&)y5KUw_+3=04DH^P9nBSR& zuQfy`RxqUf&$xGuhGx3p^IUt=z?H)gMjv<71kO!PVChRz?G!aC$4wvfO^&{7piBxK zRnCo%>tM6Z#$!SClH#E%n3AR>tc~j{*74i~h!Y-z^bzb4kxID}67>dd(|G zjqYomHvT%<;uzm5d-=i86cy(6@r2iI=3RM6O`#tv=}}rXx#QhzAo#VpRfRy%dJ|qV9sMsXIRrm*R&4+W ztEStGa(@}{{~|B-`j)@(!BAV)A0$jTA%;9#S?fo2m^J*-VrA6Hd$fD#e8;q1)LOM| zZr-qm5+kFZX{+#%XAXZwonrMZlHxwiygapE7YDRs0k01+j_i<(cIt- z(8CE4`!2M&ZP0o~xR8EQw>{l(I`L#hx7WYbI;uy9LhfPO53Y<+L5t9%Ng+W99e*+J2&p zdU-di%kKlM)2na60nOSX(fT$D%xG3m{F#FrN4{%z|AagZdi85>_W);q(g*dRQ|uYd zymfiW$R&7)r6?BXqo1W~Q0pGXIzosOY6U1puGxBV4Z)8}Cm;7((P#r=FwtOZ9Do)jLE;uC&Z%8^Oy1EbUDTa83~!OK#_? zPtPR&C|J8BMn+RJb4-r6 z)VBGIAUw&-#m5sM_vP^MbJ zeFPsqy%ssZaOn*DVYjJPYT5}7b)|J>4^J;%l1lyTf%uN#?o<2pGCYpW(=V-%*4_tG zCzj?6njXUaT4WtnIH3^hFvU5=g6;ZOq(4}XZAN~(cVd%5bE=W6NHn>;LnJ{T>M-<1 zBe)UCK8^yvq3E{laX&;?EIqI~s;uEib^l|6sKU1>u~iTo!ANNI%+2wa{3B zFCt~$r*FS~(~c@w%BKhm2)ond9bu3iq2F&cq}dTxt)Gr#r^@wSrq$cZ8Ir)(ryG9> zFbV&zrs};b)7#h)-@+Tf6Ur#V^Z1Ft=1^*8iZ?BE;f4kuCxTyCiVfq#P8XSAn$nJu z>NCd1P2@S!m-BE)@qQdbd4l;St(Y;$B#KX4tT9s{Y1bA<$Q1 z^d*n032$g`8?*TaaQ>0TAHPyoKAqq_U$+oKJCz)FKBE9ztVa4XsTZ z1D5ExWciRVC^H1jq!2zQHu>5sSEWt$e-vk@weI?>=_e(iW#|}ZrpWh0mib}wTd_W5 z%T(@lD#4bb052zAF1%ftv589bV&2YPx@552g`SIW$ht^YcWvF(a&Ur8Pci2?zwF#{7AA!ajoy$@ zp5oq;w*MbKjHrYR6bC;|9i*&AV7;Tt`?)puXdWszI_L|1`lF@|j`yM@0iuMCG^a!t z$$gjajmg~7@hQX5_;n11fsoPgl74@DOoTi@PL9p@%E z1He5yxO8{E&+!;cC@&@Z)Ii|!!O>-fwF#!BQSX9vCuD@bDRjnyB{@~xthvBiCJH4k z8nA6*Yqo*4YuYthO8(xa3xVj|FL}0taUn26VVn%>&AtG291ov?FQt^Ac}>e4hq|C$ zrU%gobw?;pv@{Tam$ooQx-UU?`Rx3rc^FOiEcJmj^nv`v%lDEC0YjlQWWJ>IlRP*FLpE;7NKzQ5mSQ-?HX z_kB8*geC7W2NuG9yJJgOfho`6yTqXk+NE5{)=L#RF(rxHH~(-eVWtc{A*I9aIXMIm z^rx-nE3Vn+xQw+o`xpl%y;wuZrg)7zi1funBWUan;l z86*x@7I(EmT&v}c=fr!49e`4)*zN-sRD-?0VJi^pJyg>>Oba?4QUgrXi+SSxnllQ} zALWYiz?w9Cw~a1AMVlpxGetpMLQoXwP1o%GAxjO~iUiZD?_SFGh$TAtXJ($i0m4F6 zWbc)I9nW$qsBvk)Id&{G_!2QfX`5e=RbGF}4_F{%F!f?1l+H4BRL|SZX8SPK3D}L0 zt0^E{nHaBZ8$7~Ebn;<{{ouB5QQmkG@JS971ez++hdycCYhMsgytUqZi&L+Gk>`Se z`1r77;ZB*J4#(b#97NE9`|ES14uwJBNED<@*I|UMDfNV0ONX`!CcR&dyp?%PGf!lu z%h)m7a7*$GCz5X^6oX@l%HFmCYCEf#)9OhJs2x=K3eBd;azX_>~+ zeECa}8-8N|Lnb?#<%zJv=hL;$L{q&+F<&H}L24tt%{aq0p~h%8?32DZ3SCEzhY!K^ zr#ydU=n27+qr#tX zNLFU>aXnVcUG)q5=V&+r0|^Rck;;7Fjy=s`7*dwFLat=nk*ICxymI9(Ts-8_bs8;B z1HCg{#jhR8J%P}#=H}Z5f^AC5g5ZAS&PX`os6Mj-Dnfn*lzTgPY8k)+QB~rpxaO_H zP;8x?kqbElM}3a~tD4egOgEa=BX$E?LWQKv55YDYRSupH56iBrkAj&p^tAk2-y8NM zjjLq&PMYr{s`p}^og8Jx&mMJLe^0)Yi2%16;C>upngtI zZURc6xvx9jGXZ#jUO*`hDN~#8J@+wDz(cx;hx8bfn>gRd3q+1>AB^Osq}g_wr-C-Yl#3E}&E zS4Y*JDk?4SPm8e!E}4b-j*T8(thNWr=H%=ORJe;@%C5nH;8agkxS<8q zSEmPf@Z8hF0iQm_bWzFz=R#d!*IrLCMj1bofBO5gCFbei(7ysWr%r!Z57xv98PT3p z0XF?1VM|SuU}*rgufAfXZ~c}Fi3IL1teL^|+?Hw~eEP#BGBF3eVGVvRHa=5|Koy3F zT(d*e*ganXi?@2?{dGzeJnE&~g{Er2pxA&RWbkw+qw_J}psMN z`PjMDvP=7@F&(u6(mL7liwSe_Z_7&0Q1SU-`kz~5s2zX**@D2XcHGp-e_6^ftyhe^ zWcz}pC3{M z3~i3y)Z(_SeE}v9zb=L7&fumr{c>+8!#g}VuxoyfSV|0VFBnxWu)4NUr@ge-bmBqS zI!?jWQ1YwBtHTX0rDy-{@yj!;n!=FV#MxL5ILTOjM`)vzo^T;TZA%y zKmFRyt35Ys9eT_JT;CkzXE#WwG5leC4z|s5$JZ_+b5l%8>Jclei{m-~5Qj&S5T%i&9WYtHfVYSPPxp)_8FhygldWURucxf) z;5HXN5+&E0Kr<#EVCgR6L7{1~!+)lIw$;=@68o`-Hh0-QBFT%z1+Y7f+owyJAlF%q z#At{V@$*vCDrF6i+pJ1?m(LE?8g-KCKzhe52x*XKoo%3E@#wxGweY*sE)I1=J2%ce zC^-uKA|CkCybOr_8s?I&W~~C^pI^ZrXIJb~y)FNA$VJ-$FEU+ONZw6-1~lu6-Q?zc zqiw#^jhIq|ebA|DAI1E;K%C9?2Jf#j%v}5-#D*g3*L;pj2)>;sWGr7&SmNw_{>3$B1P)fYT@=V)b`Y0#i=(tK}&*s?t*bAo&y((@P(PGu9btFyCUpNmnWHa-F*k9pRn zhFUOn(edtv8Y+F+uK5ZU~m%NSR|V#3Xnp(Aw2l zoKUW^0XYk0o~!V;=*h6{xiPFnrdm4j^Xg{UwZxPtg=;D0fI!jo7F-^2z>M4MjIV|7 zz!}TB`I&#`SL8T`{t8N>4T}DSf$YLkg#f>9TahW-363Y8MBPM~YC|q*(1lo&_6$21 zI$2zo=bJEGxlMQ$qa2q?^`yda2Z7cldv0G@zQXoued<%I;7Zb(8{*gwk70G=N+uio zb$K!tQmj>9oD0^`t~ zao2Nmv4vD_?G7?@8BOGtkvWG+H`VO7#h$ydi0iS-^p;tH&qa*KVqEWrle5%nY9K6c$-9t; zZcjKx?{B$Z3yj6zIR9Kdceq}=fTYRISU0oqJvLi6tEg2l13k|t?Z|~B32_FivtRB1%WAhK}_lPUfd9Yu3Q9Cs> z9zNDavlX?Ogr{urKgv00a#&+gZ>#x`owtzW#c&@|xrLHaE6jI#?NgHh4=dhu`mTXZ z2Rq#gNwLo7Odv!M&0VZGm7xsYtjexDkl}IJsnig45+$KtHjOiuCw@}FKBw=5tG!J0 zN8SFU>hU|K6Y56`(OaFYugx7A{rgU>Xq#%ZMvbUI5GWbPc0+zmV16b4k`%zCIpnTP z{iLH83I|x2TigQNR2;;a(dm$8cU-)+hJB&IGaK#4@P0Zd&i%a`%jM8sN{b7j$!!#D z^1u@%L5*J{EM_bj=iAB|j9vNPquKYF;HU|VO=ap5jD+za0AU7E(F_3Dc~6PlTx(8L zFL{iey|sWN`*A@3dHoskF6z>{>=Nu68gIb1gGjJdYnRr^&CyIl^P|T!57KpwfuM2? zH#HiH+VS3NAdIu3Z11BKY?w-K*Qc1VW~-`pUOqY>2y+zo?i5tX);#^H2%E0Le=G+T zwc3sBlGJ1acGE%gw%r%yqYV+3VtR3jni|e<;+}!xIjxC}7dzC7c;FOV!qfW=P2Uai zOlT%t8p%(nYM4?tmfCE{im&F%872H?$%AJMZv%#Ej`0BHT!A#5cH@q^GAX`mCDHU6 zW5WDS4derFm3F5EDz&(-#mFNcn6trAb1iG3}nQVmHRraOr?dV*^hO-{xQ)-M6+I^LM>UwigWdv*!*^fU^tdQyx z^;!Pfqiu6=Y`Gq*UvKufpWx1iwUIKoL|0#xF!x|-FS5O>m(iReHnJ zf!YQ>?p*D045RV4A zM{Kv|A7j#n2LR51^!OGNSiSdlC&Jad>OLJXDa>==@v9x`T1w!`<}tTKV9+bnxsF7k zBPF!_*aIedPQGFbEG@H6UY<}L-E?clF!UDign5}XUA~goqxyv$QbfgNa|iL!mPAve zf{Oi`1I~(952K?V=-PFJ$_>w}wkgoENzhfXbMlOw{vp)J5w|Jj3JA zY|4i`qANEYErJ8^X%;gXvq_UDGEWD7VSSdDYD1q)Vvxa|&&(^ z^P+$Mqgz#ujbFoy2W~ywF1XnnSCx1DKi3Zhlw94kFo*mJsB5dYdpoZya$JS?>HqRV zMj7ksakMQCQty8^2SViEJhor>liJQ3Fv^EgS~wuqU?Y8;>>~>ZDX!ZF)M_)(!K=f5 ztWXSCaGF{A7&DGsl(;65@N1n%gg%FNMc z=^U7Py869RUawO;yiGr3#F-EVS+T}>EpU-Bp$is7AwXU3$@f)EK_8AOS?GEcL-JDF z1wzBDI<|)2ar)bt!}x)cOP*B}Z>+c+omx@-HhNvnfeQ*J=tqq>F9s@!a+w2mSY<|! zP=k_o7dkQ34lEfXPS@eXgKd~w=5hy7Op~{T{B?d{-w`g3!m#thp$?WyP4jr7_WTNJ zQLXm7YzxaAbkEB#7*-X6&3wY+$V+wiq*Lwo`f_5gWc%C1XmZWb%sYqu01TsC?iyWO_d$_=tM1sl;~0SK#h($;zS;eDvLdp zcWWljp#{mWlY%N5SpJKf9=Lj>eiFa}#6g$*kaM+uio2u!BH9^9_GtPDO1l zeP21Y@M@Wl((0szcgQmpNlj${#)0S?@)lON&*Zcxliv`z6M^i*TZaH~2&Jp}PR%sg zKJN)r+lv3h)OKgA>PjKXCmtLECq>%;b(s8X2m32^iF)*4hxoLXc}uPyM1886RsGvhIyH15`dq*5oY{+bOW7)^5jbY*kFP(zUi3HM`S z*Q{%z;t-@2oyfjt7~szMJ)JqQ^335Ca2>#%TeH zY4D=Xg`Ov*1!EHqHYJ!D{;Nu;f2Z7v@ueX`;bgzw`yL9fZL?7+Re2kOd z@(1#C`6q7Fn8NMnMYf`?2M~O(%Jq#XX3>y)$??#v15xq59m)Q)ZY>^elNMAdi$6~F z?zBsK&scjuIyMUnDBI`%ltOD__JsVn(?YuYPlVv49^uy`|DissBkWG|;?1L##r18! zcM{ z;P-&Htq_r+*De}?tjVVUCr}4)bobn!oqo$W4^R@00G-+6v^xN=O{yG|0*W=7xUx*3 zd_h_GQi#u0pRAQ7vfRk;qCCLz4~WE}KPeLDjvay%r@fgu8S2vwk*o0baD1yC3mBFg ze_}{Tg)$7hr~%G#?|bpQG!{NKRm>i~mG8-QTLaTYF76 zZ%)AaUnBE?553^ z8jFtX0C)?6#VPWD$6@480~4#$SfuCPogAhufh0Ayd_e`~HZ$GGODK71USp@;ISN1_ zbdX@u+CE^F7&>gaI^kSndcw$AT&}%;wbxJ<@|Jt{OIsnh0>?m+mo%a2&#pR5j90Ke zmL)rwd;#ws&US+Ay=p=)9}%vwcLAjnz&Xfo*eKW|QGZ@@o+*5}m;qL`2Ahs%w!`o{k37%-~f>2pu05dbS|au|^qx~D|mLAb#br4~20ok$-P_wD8h z-8CM9b$11VZl9O8y%eE>D_Tv|SDM`toO$#xS==@T#PoyYy~RShNtITI|3 zd^P`1^92lTfh^57(bu>*3~@)!?gOIJgGUB)-5&Jb3$KeVa_o$6V z9m>Sy934)%HA{($FSI?7Beim5P*OCcLw2joy!%|5qD+HOR7#QI@d8uE9m?9 z+3OdUm!G+$j<(LAp#jXD^Vr#=>1 zP}WvllNqyyUwAe%t8+u}+v@4KM%f8Jeyc_k&C;|1{&||}n3T9~SOHr2>dtU-)fL`= z&q);X!1;{J7ukb=L~UoEkW(RxDYzW~jKM%@=p*+fnvMxj+H}P{B#I@gyM4gRkYTWn3>GA3e?A7V%O~0V`^PSpMDvty6*8jE&Bje%|$b5mJWIR8XNoh68Vh;@c$rY|GU$y=CYS3>EsCY17%hLzni2>OYx$>{L zr8_RPqYBmapKUd~;|d{=uywQVNEFA&qo`9HvV|dOs?@!ne7Z;5Dp#ncu7I$mh+D|2 z1OGq<_>7FGNnBGZECD2EkW-n@elJ^yqpk;5pm@dn{W zb&P3)+s|3D>aw#6myxOPUXSq|b=_#sZK8I%)+jpmhhQ>PG~UugFFiI!f0X5ANISM# z6bC$H3s5zmrpb~|7IiN58(zOWm+@eg1xOa+T~k+q>`Pf}fjMkC*1a6K6+r?`<$o=K z<`-qN%C~Q;DMutcrEenu$n=6L9fFNlEZ%dJT8W>EMlE&rsApI%_kx z-lfn(r!Se8&cnq)cB-KC`)YiuK{+R0)+T3{)GM)`aOQ`3vEi{G_Vhc)Lef7{hmWh# zV{lFbTln|#eJVI)bUbe~7BteUSlm|Y<_99Ss5i==F`tF5(FRpH`3PwdEk;0<{R^}i zUx~Onufr5ZWemEkbK)xwdeD~EnV0{T&&=(K(#n$JO-`t2#_|InQha&vseV0h7O>_X zFNFwtg8Ku|yD9>4HJ4jL)xtsLDlWF#oH zUr5|<%uB)p_%cc^!-jH~09hAbeBmx=TjEKks}~bUg(%89_Cia*O|w+}xRV_g521!! z6t-Pt7<(VK)%os}<#BzdKs_fa4dRe+)5@zfv?xwgMp;8w+Kt}c{{zdiFNBEjWIs`; zCL7qNJRilC;u0bE4GAWpiYjN*15tRNTJt;QX&Us{xZ4RQrMDsYFUTHNafRYeFQbJ0 zK1L+2RwsU-WIUA`qNoyQ%M%8PU}!-X-j=RStbwZvJwXboTV3nv;W^*_V6Q-$&+dD; z;JZZ!L;}wx!++Is|9P)_#Qb?}vlaCylbo{4Vu2xjyJk8uAm*vnf+1|M*~{iIr+%UKB=r-1YsEL z)m&@FcIb!&wEUL-YL*9u(KaDYisg1ScDqHj?RP3*Ltov7v^$>Yr?dj26fluAJ*k?O zg_*!&n59$P?!AyGeUGr%N*E)}65th6x1_q95;qXWgzs~N-=hsH-ONKN)iNY!C?#v$hg-v*!SkQofwA3|od)z50JLm$0cbOm!y&NU z5C&3Wi41jPH!Flaos~(C#`%|O?-yvP8UalD#WW}XOCh6zrm$5DyZxxCbi+bs+k$0~ zE197a=iFo4L=IvqsQqR9(@){{&SFX=v{7mN3~)$A#_T3-N*!bdY)o ze29`kH4xoYuPUx_GPy;@oDQ|X)5Q5m?sYWb4z^E8 zUh24(pCdOUFZ8AdsM9s$khz|^8)LQdh0je0^HEMAjxy`&%XG4 zZNvEGm-XNCN9=jd0kOLBmPN=kFXfdVr{~07is#&0^0r2>Q0$SvtilxIhZ>eV$0tX` zK9yhVj>DKVDgaFTlq=bKb7Y}$%0XM<=3i~*Kj6k?v>C7(Nqd0lB$wj}j!u{IBQ&=AVfc~3S;W<_<^>zSt|H&=Lb0v8|vgAkI5M1gI zl-Fl>-9Fhg?xh*Sk)#Rygw++y_jl22=b&m4!{@38To>R=Em|s2k2S7_4=4i+TQ|pn z-{BrU+gk4zIoL>$_~8iE2*1`7U|b^b<6I0n*}n3d;9V`hAkGmI$*W3oJGv3&!tRa* zmXIJDTx_w)ZYO~il2_(C*V$EcY`>2%v&KfC!s2R|7KNSvv^@dGX}>a<+AnpvmL(-K z`+gV2DZh3MO^`Dxu(#JMQY{tRu2Luw@cU_;SDfZd_a3ShbQ>&w7Gjc)qaP>TIx!@D zci)9Q5ul<_+fSCWDSHI#5&-fYsDcSl{<%U>qyb>0O%1yZzYEAUx<{kswH@%r%OL(H zObC@DI2YCSWq0j>WGXRCGa43NOC*<)+U(#cJjo_mHajiHV`MR4%R^V9ZdbC1#PUnh0k3SyWCisCYuRbXM>S$07mr}HX5zGm_}f5cbk?I_t2A9H_z%L zfe5{t`ZiRWroB5$&Ja6jpm6hhxM0)$I5lP-O_K5wgxgQmXIGpa1oOxPAA&PLpo=)JXPQ(* zw8DJ0pOKjE85#NLfm_L_Wz6y$!%+U3!-`-vBc5M%^;q$zrR6mdUvTFqXKJhW1cn{C zLkCuwoZnbwTuYdHhn zwW0GHaq@8Odi=UI;$E!`cb-$#Yy@vV&z4Q#tq^5eA%=EkE*WIa%|qRIx&~9T1dmrA zTFC@QfWska7PHGfsOivNQHKeuJ^q68tO^L^ualF?NLq0#=6^5ef91fcB3bzhcVzFILy6)h_yK& z-s4T9)^*?HE5FCvFMfR+9GSy-5IJh7QgpI|@g(WKTX*09 z>>e-tVfBMJvwDLuw?p3nlmC-cQe~u!lM4;{2evm(C>ulmo(-I zxK&OdkOQ{vB3$Yxy&bzj^N--z6%$)g0D~+YeO|@(43O4`87WEd%$MhG>zs}Q!D47C zIOsj_nP1NC?2SDM!kO~$v)C8nW=G`7=ebw+UBqJ^#+eOQUFmWn4(m*4dTidLg_p8(cpoj%yG|5qH8Tuw) zGWE;{q#bRATsMO7QUOYE8AEN{*_~gjwLC?Vh&GX}^Xp``j&y>9NDzP5v03^&TZj4N zUsE>u3=B5v{KX4#+)%nSr4xiQiyAv2aKY{(JdR$H$Ezn{myJI_AUsy^5?G;Pf+t(Y&mz^KQi5ip77Nz`Ac1}eMQ`XLRJd8?y zfyCZZt%+Eu&htl27JaRoprK0^81#QImD+Y`Y}KA{&4m$66en~Ju#+_1NwLkDmc6D* zGxjkY9&h5etO$pbuU2}{;ZCsCj2=_1|BQ~<>pDssG@Qg=$;GQrZ;%oqGNP1QEAf=Vr8GzW^)+sx5N{6tTQi~78X zYx4R7{FMpmtu*yNOUkhf2+pi*Ij#z{uocRV|7o^@9RPRhY8Q)U^`-ynkBujD0yZbm zAEYl^@*p0^_wZ|5zbV06DBJ;(WR`I*FL9%4sCey<^ovMMee(hdk&;g0=rM5zA6-%su3XkmKTB=ErtxL!_BOilA9n7;4W|l zofCfCg5=rP?ETO&4WkFaHv#VLr+!I1h2OQYGE-xBH}J5)N1cthwYUUv5gVT11AIN} zhQ-qZ$)G~rLGwu9fco#jvYJFo`sK2MT?L;q9z^wTE%Xd5@1L?1xHpy+8hm*M9Eh0p zxYvw@9gfBV=eY7006!8x|N+(}Wi2ZJ1#C@{UNGvEN^||7C^P@cRl86AjVa z7L-2C67(fh9s`-J4z6KdMcm6TB}zMg^-L4gMnPBDv<76P~{)yjv_2~%!GA_{H?7hp_OzxG*N>>lJ|BleR;LX|Y8>3$E zZ7zes=tG;8$eE6S7{xXB_afvkC&5pCJHP9i?-@PK>hm`!5LZ5#KwmEh!@_}iz>~Au z9YO6IO^TjRsXmPsU`Vu3}akdDiAznHE&E&*vo_kN3f@-e(ZK283U8J0-6P*6V4 z{}(k!o;TNvr@=$r^ZvnPq)dRWuU1F^3j2SE%JkpRBGJ;Drcq{Gb%Z~B_^ySG^%sjA z8p?2qqU+T1TZDDLQBzfC27S2z%yF~ zmXignV_vZ*Q$T9wnB*|v#ZJUT-qZ@aLE>wS&G-pJ81YBVeG@jmw!{HsYYFE;nFH_s zU1V{-6RC#gV=;hVuwBS*pK0-JKZ|qsoVzQP>di`cn-3z^D}j60Ldt%DlC*%cF*@ZX zcZ!+x4m$J=WSd6QO!PRc#akY^ggq4?fr~M8*khVeypHyfGAf+H_h+xN)_pX^{8uY0 z#uH$w(=4vaE)j&3xf4APYp5YJO4VRG@HT8*EAzRpJ{{>1F|5Zawm?n*n6#WAA8=~$ z!A^jg-{x{bRHgRY&FsmWUQ&#}73_#al+-}xLtUsBQ$ET0SsTSsEsfQhM>Fw11 zI|*VyyF$JBNCyenSH$dzYjFY4KP6Mf78kw{4w=`(?}}~yN03n#`DstP%TrLAQ7qZW zzRe5F@l6pCrsYf+0bkuu0X;!y^pH25IC^=WgQ05PGPn9L`&3IBNwbWZsUY~*g3Q2{ z7<$*S-LVu9k$ge=1{R?AD%rO!I!04z;1rw~y9#8uU^-^)omeO@WTC z_S+OCL?kwA&No|GUADH!s^@zDX<4;R)zxCgZTnZqr%pIYjgeYYmzY0!<_zRD8`?cV6zS`4&Ofm-0KWoU zyFsuLZRU*tnBHlje(`7K|F*w;E#t}I16Bv)EC{DpmxFmXrU83!^O#zlO&Q(k z_QlQK(d`K8(IZcsNxCp{&1q_><}2`;U*e;#Nha^0R7$ui*&2}Z{^d8Ed{cvd+K=2Q zb0mm8Lf@rHl)7@-t<~^xus6KCXLvfiL%y^v#4MV4B1+v8^qofiQDW3DO+CKcAjot0 z@Y)Fn^g-W4&_@2tllP>UscaOZR2kd-(5jSEfJ*s!W0U@zgx1<=-8nh~oPhx@Hs8#! zr*S2p_+I7bMv;4tp=BUuj%oXsBs6f-=7HI2#c)|XRtIs0E*m;%g6>W!~6tbmNTC`K03TX73}M2fhxWp zu3gl=tS~ovGxVlQGMFM0y?a1NFDz9PPqq)B(pBUhuADrj&~VR^*YSTH+WH$c^J+Y^qa>nmg0b@l(^97)cx~O;i zNry@$t{xP}T{C;A0s(fqAG=xYp#13t;u)-8_Bt-MP+xA{o1nf(Z(V)`eR&H9?GkJi zREX7b6*%`<(lQD{{!Cg1<*!h;jd~9J<^sSGs_|991Zsl(-Z39lMOB!7~d#i31E4d>NoS-)js6fivLD_H8D-NR-&GyNR$snv3 zM-Fgc>3B+Iaozr6{%1VuCe6$TZr{ev_^&Jiu6HxmzPZ|%V;cSE+(?dvG7Jr(W3|Kf`p5gmEkRO$#g%)g?c4ZbpA@LZap8*+; zqvkV76vVmQk3y-&rTv%Ujh7N$e5kPoWt&%bKsU5t`~{?clFN-5kNefY=*X$~)dr3) z8}*e0Zv&bxnc@l8K>`TcB<0PJ)g|*l&QFyi>0DKu+Zv1QsTY8QUK0f0%ebTIk17}7 zQxOHbx4Z_Cv^iA@($oPNT6nfhTkLtb4gWn_tbOmZb(%+a<&DcPs(twEmX1P3@(`)) z=+EWltc;hpy^euY40DT~kZ)7p)PR?bHpd$(6piVFrqdvIL&eGmxw5>F#b6J6= zwJ#vQ4+>Ia1_(NU=s;vD26Iecm3ehjJnZEX*z{s4@c)7-ET^@9YcK`OxtFDcenxX8 z%3yTbF5@f!51nWXSRU!ld{aZ0AIQ3GoJ`12-h_TiR9matzL-qyO6i>T<0N~TcDqx1 zjrR77bNS?mol}g~qXJ%Gq#fG+6h@lb{89g-6ImV^dh`R?-t?he4*{&%N3FLSZ+~fK zQYx(1eemXyxg0#)2WI8AarsosK6Mby-md)8(Am7)%BrmQH|D11Yey>*!(Z5&QWp49 z2CCkA(=1{oe-%XWzUf~8MB9^*G^D2Qnu; zT!4u?kG4x3QLKmLmjT}A%EtifLG>&nfov@F`9%}Xnw3wvyx4d!Fk?h#Zb9eT3oWSF zwjX|*!e!ZT-_C?hbn^Aj|Ij1kjs8db1cyuPnbLLRx3WHrjxO_mc~*z!WLX0+BIfvD z;hCcLDe_-C4at+Ibg~isaRCie9C`eaJvudc*~D?x)mT9+c>bj zm)2HaRt9L8;+r&V8>mpU{r{)EFOO^DY}dB6imgjrDm$b~Egr>UM0S!?MdDi*q>6wG z0fR&Zr3lIrlBuF1vZU0?D#Q{6WM2f?BO(E1iL3z?BA^5a5FjKW+vIzKuh!b0bKduN zmTx=xBfn&r8D=KWJab?7ecjg;R(0~AvA*c|3A||g3GM}N+k)({dT!t`_7piPvkR#3 zxt#e`CuY2vbvTFdCEBLv)(05?10$zmL@QK+IafCjpwSlW*&NmxdU|%fc;agK&CdS^Hd7; z6nS;KegIA^Ix7Qy14(-!JNNczp5@Y@ zS$~z)&IhmW-l=X))3#Ior!4SsSzv2dwHmzjqui{WqLp@U^nI)HQQv=}Rax~MSH=KN zl&V57@1a>HTkRoj+521z1ZjSO`Y8DK0{`xF3GKn7fDc_a*%(itpRUJ&BP-D(6lsLx zLin}KLX3Zv!Fz|P_Q2`gaqB+X8%X~e2R3~Q3_Cy#{^sz5l(`K5KOmZ$Tq5tP&=||b z?R4S^*Qd7=HAiF*X}z*fZN)cLyrH~Aw(uz7zu#K(U-j<<0Ak)e3i%bg%DnBL^LzLw zx`~VY$y40;>Upi?H~-`H@c*aR!(0&be@o)tOD?tjXgceX4*JD$_%UliKnW@c?cxj1 z2cO!K!hz$+-Rbmcdv4?L=Y%ult3#e77U(BlpH}2Wtss!v2=nH3hpD*c5vhMRocwF9 zdG{h0*AAz_nBNoiXOu>Uyx$a#V$0w$mMJEaRhS0Nc(NEz6#i?$bSER=_?8gozr%6z zudzJnjitMi%aPZELjgALQ(<#+6*gb0lCk)GQ8L!S^$`94rG8u>jk!rFp(#%Wy2PFc zg3XcQAC>R=c$cW+kl$agm~$c^mXq$*zb~BkaPM4Rod9YA-!lM_{C59Tpb@oxN9|-% zmAnA+ys_R<_BH51o&z(g?OyrKZ7?X22#C`2Nr)GcukH=Cz(*CKkGLjK(e(AQKd15^ za(#v3FycqWyin|qUlKV^AIg*Xybe4d)_ZFd<;O*Q@t`ScEH@Y}nmYU`U!c z1%xB}^~9h3X#R)ieYm`=jbWVoQC$eo5aNMD!Ib?@KmcHE9qd+(=VKi1CQ#1Kv_sNgxYb_S!Lj?ADlg0Yj_XMK|iw&Gv^ zk9^_>E_UFU&Pp?I>GS~(?|^{X16Wg|Nh`uJU(KMbDROvoYc(JX$T->4^)V^^GB2M_z>Hrc zf1*HDZ6^)G8_N%Of?llojH{Qh#n^dA!Y2%Yhj>-Jf?)g3O#3RkYKZt$f5$&S7$DgE z0^yD37o9&@l`4%jjs7V6vk)4ZBuy5*pq!^-aHgG{i%m3#4!VOfWj@URjrEJfWDDbT zCO#od{>x9wPv`x_p5)eEAg$>6-qFkW(443s&iHC)uxq>3e2DVfd4pl;?xK{!3yR!; zbRYLLLl}gOi87zPxviZCff5F-Mo?5oA4EI1zg_eP@8gZ~Aa!N+|U`LGK)*rV+>%v^0Av`^_%9t-XJ;U zhV?({X4#xRD_GK38&%2R4a)`6vBS8P%~!C>fAg)Lt`a0+rS_2%p>nbP3o!q9b3rxE zz}9DILrRU#cW5V-r}lU=@4w2ED$!){oqu-Or$>e;|Gwq6QFNg4e7Yjwzn+dzGz-6{ zBj`a4iR{~mQ&kQ^{<)H2FMXLl?>7%GFCT;V%RXeOkfeiNCaJ|_xM7L97`>#I^sArx z$EfE39rp0sslRC(`OAuF*gCb!7O3+4AUPmO7o0$i7nERj_!U(V0XO2y_MYPUUcie* z&6_t?OXa?z?QFGya)l1r*GZsn#G&`v{fxBh|Kf1=@}G&7-j|JxN3*7O{T|GeK(RN3 z&*048D-!M4;36*PHN`3hF)L6M(8aeH{C~vA@}Hm^{1b-U2k{4p!2g=(^`C5{T{FA~ zgA<(h`D^#?);&ET?_4r(hI_UQu`3-qk}_~4D;tdAFL= zRHjk+<~>R1zD4j_!91v?R%!R6?T4nBANy6kx2d#y+dub5uPR_A{Cof0n*T`uTr-=^ zo~9`!)T?p8W}6EMd=cVz?RAeQRdkKtUuT!hnODyLqZs+iFXsq~v4C7T69E)YwDqa= z(}6ZymoR!7kvw`;$kAeZ)j7Z44@P5dgOqo= z=LZAm^lSSl`Mnsd>2$gse2s2F!Y8AZ>2$U6?TdUdShj!G zL#W7+aqFwg%RzL-hbRM%wtwT)9! z#p3V43V>Qsd*N#kU}4qu2$g_Z5w)w{_v`E7rwK~{wc^&veNf~S5TG_)t5+Yh{+rAw z)E+>s7@MEenc-IEN|QU;yz zXq0}j_ryt{8#k?T(2Nh|QktUCnh}In6q6p?tOv|DT_|@*YnS>>z67GhY6M>+D`sFv zVHq8GYc{G3Jl&V2fDZP)?FR!sqai!pKxj3doI?Xf-6a7f6yT%T)7K+S0LB<3B9Hj5 z{Lw6r$q-S4#e`02ZZ(AoBC0<)e7BE-%F4IkVtdGgx>Ly4Y!QpV8Rs#8uG?ZS$8u%! z21sQqgz2)=BkYg(8LKk*c3XNBTF|&t&t0lNYq?xl!4nXcx4$~ zAl;+#_vRe9fRGgI+97v7s6+a$8H@qus7de7*n+bSK-cXxU}H`jc16+UsIJ$1joozf zg+m4zJ_&;--;F2L;Gg^TO~UY@{NAnzuNhRvZ}8_Hu>Uq{Ib5avHo$5hL7H~tHGRSO z`V}4wioRMJpk}Tqyt#pm$+ii^%uNAN}nbEjH{~4z$QU8pe_@)eET)=cS;>e zdOi%FF(%J>cH&({X(o)S|6IVf@!k{ z(uhIySfm7mPp6^~?N>@y(LrTr_GHt6kw9hMDn3gu({-)r8kK2t^IPcjF}R}~5IDdu z!w0UPqb7sLES=nTbNEj|ge9dowXE)L6IYzeeHdHKx#_o{WJZyDYh(Uhe!!cf7dMd^ z8@Wc&v+((Jp-J>)7-|41daz`Obx8n-9o8}pFuelb@i04KzdQ1fs1Pwn_|$GZ9T*bH z_SMI;GtMb!t$L>(Xu!O0bCa2Pm{+sCOKK*x-A zYPIPSEzP7Hv_MTjM~&%uz0^ChN*|04kV48Ef+8;Y!ny)_!D(OtYJnKFyR*}>Q*&jV zo!c#Jz5Hahw%FFHB@}e?I_xv;S`!g5Isix>nl>V?MPDaT3vl~9O!G0Ln1xh@Wpseh z#cmXb7s=yZ0t@o5plXhFDOX-yTVT`@3dCwryPT*@=iHc)c&aH`Kl+5Peh1w-a$9-u zR_3{=J@p~kBZ41|A_TYf>y9sJ>!hP%y*#Xj2zzN^1eaTC!u9HepL6x23XU4IqmW)w z8{NhMn+#lmW~jgLWaJy|glLI#JGb9iox2Tvy-xE^7LH)V{*L@L>-c7UZvEk#p8m&M zZojxetqDA;)0MfEzRGW5Z%W7#=HT=ELwB{3<4uM|uL+y=*;k7klX^Fs+@)TQiLK$5p=a$e3kQAAIA^s*?rYMr zMW0Z(-$K^PPcQL0nvlWGh9Y^F`fN?NR9VMH1=Mdz?KVOT$~A|a2+o5OubyMhkilRx zOM-QRz%PHm5zWO;jF-QUmnq1W`)$;2E^_#yH9 zF}6&*bSX(j5Q;KPpCkPB|!?Tr@Jd4~5dK{&ve6jMYR>1^DlPK|9APQbI!m~A{ z17}bt4o+4(oq*Xb)_-1s2%$bXT@;jnY3x)?`IDL(rVBI-uGgY;sQKM4ienx;6HmDS z70*~+b%0UZ1t*;K5XRlM`?CG6U-AZ^IwUlJo@o%g6zhHREK^}?e{SBWOgn?`0G%3+ z%#o00AT>y{2Oj@H=p-rnOkbkeu_VI1e0Jw~vQ^XTK~wFr>fHpTQIz7vIIT1Mp{Yj| zsibAan|@t~$e0|TJSj0oNUhRQulF~|qI$rXP+`qO$pT>?Ax z2DLnF1XtVj;Ai10%S+NRiMnVEd;C6SYnS^v@!ED`&tb?IO7-QP`kVCnIlFsOqW?ga zl+In@NY;(;XYQdGR%c>^^&L9v3wF`m*U292=;RDrMxSY|V>C%kZy@Wa9Xrrg=7m;{={?dko{Ku zsG#`hv8mf3UVW=A37c>%%L5QqQ4^mdo3Az*x?3Irp3qg5uE0}%cHLw{*;-HT4J;HM zTr3`2W`rb#51^Q#+CzA~63kKi)h|9#;Nme)`xFKa3bc%29fQ?6QXNsCu-@IFOkoGy z$F#>}RELd3u7~bmnJ39<1`cFIJ27bQ1Sf-WTCoS0Z+^?Z<2IfSvxfLfTR>xDhcfw^ zl+zB}_>gK7Mtkisgi#`dFX73}l)h>D*O zmiRn24XQJGd`wjTAZ)y$oT4loQpL6AThN&+Ml7OjQ4VJ!-4E~lbyf2S71al_m3DUq zEmqdq7NgR#`cW%SIcu=cH%qnxLl$_GM_I2l=Bn2tKl{6zmZA+3p~WU;{GTL6`ts0= zZ^XJz0B|&p1s8?z{qDP_;V)(~=rRtSauaCDSA8n3O&nXXu#q4&DiS`BK2S4Z`yirI z7zdz3*kAT3P5eh$qX7%}BgLgGlbpdk3$;quR@0DrAwfv5w(>t0eH7V3%Q4U)y0vs! z;SNX|<;38yEa~*usa`*EiO+iQ4t&xNK#blemd-OUp5FaQ1aIVTuZml`@FN-Sx}*BH zbO_y=L&v=}Vj1U^9#v~e+5OjTdEquw^_o5&!3Pc6e09T*r#{!Ly|2}EULm@k;onhM zdTqUNW6u4%zX30_;ynJDMqLwI{hhYjqFc!?P|;fPaZ!UoUxpoyTmQgBUFdWwAth?3 zY2VJR<}JuN?|xRT$TY~{`h)@2v!2H{rimUMzIm1q_=m7sVw26d(u5TLSxZk^3w_*9y`JYYl}DQfB> z!b#>U(h{P%dZJPa5F~?sZ5$1d=grXF71ZT#3Z>wI!dT_P=RKCK5;)zmKTpQx;_sfw zml^WB7CYp`2+!P|HrE02E0Eckbc-^fr8Lucz`BV2h4|{YhVqfr8`^0}+CqW?H=OyK zzZrC;&IvwSBi`D^>P7M8v{K52s(d^DTgMEV>ZA_(T}*5Nc44m1;vTpl*dmO&ZK=@O zO}uucWTf5--rUQav}iLCRZ_ZTOGLMOl;McME{<}#%Qb_r)FwCI!YqvKz#BGj-ee=r z3mD)w4Y9iTgXBoI0WdV_sGr)!-=Bl1L17JBkirS>!^5pWYU-r3QoRTVLHZ>OZJ)r^ zj=5DmL6ad1vq3<=|AXLvxXEX*6*XkMjS&d%ONH$ohDZdIOnBlMu;S#!VP!QT9@5nr z!P)ogs>D5u2d$+>j#1wq(GHGdGZ~t8DS`2rMtz}6b@$u=nbHT^?dsw3`8{ZK^>V_Xzx%oUwE)F=3iCO_&p55xrmyx~`*MbBF zC7;oNQjF^MDtm+pE%D7n)dR<}yqbG0u?S`X??)UPn@E=dmz&-MA<+!4ySM@aun!u8 z?-~fp#pO{uX~&RR(0OO&9%7SpV|02smU?wOuSksG+1ns0@O{NRnw2` zYN7>6*}bF7p7KokTJ4F*xK2S!F+2*@NJRNnwZj_KPLn#s!064rp+aJHHuVp1hd)_g z-`ql#G=7I{6hLVDGJm&}7LjoxLgp@08!YI%xlQ@t zA($k+;b-QdhE~5$2Vl^W``p#C$wA}$fE2gz=g0gCbSbM+F_Pv*K}dat21LSq5@r8!|Nzn zx+R!c##YjFBop*BXEzqgM=~dsH1mH65eNo%FZc2o)Q@;F9l4(N=8$4%871WNtq7(g znvfk6MM}kFmDw0Q7&U$xl`*h3=Y?#$G?SScgw9q=)A3HZ^4Rupz{EP|fB>UJc5Sd* zd251Uo0YKW<%(w-pUJ*Mq~2QL3@}G^>}{n{z~wC2;1khdv*`TNY>gSWzn%7w?A||r zFrMhH9Y*nnFYt4``X4> z*M{6l25p$t3{Szj=}!hL^-sl2Y|F7`L=pDJvKr3)=&Dm|p@ZMWNk(BQ=TnPq-E?!= z%($<*8kp!YZA}MEV~$i%22r8#)=-lW=P*KyCqj}M7%2$86O?>W@a^BeZHe#*W+T6j z*oG(!iZS|D$2=8I?apY~m7BA!Jr^S~z8 zj`)c}UlwtRw|Y4}0)TGc1DzBmn)20%vUpRQf?c{@x2{*rv6iuGFMP0yYUQaFAE6~4 zZ@6A!{(7!of8-B&n7>GDzE9HrDNDJ?cN68+Pw$WB_v1RPLQfwZ#G1>CWm(C)@X$+A z!HK0u&OIhh)iPRb#`mVr2YKV%I(__|NTE0`p3M`oOrxtG_Fh5DFfV9ye#JypZ@89w zo;Wt)<+5Jb3T5CuMf}=qZDYM_X6d4)+7Ml2Cuc#D32(71TJ>wU6`WPCZLp$N6I5|GN`^)xqjkV8@aO0b6HGYXQ-ao&l-00iT5dgNj0}dJL;Li>q z2qhOf@x>GCCNtVLFRXXaV8jbge8VU=%P5GC{Qk%;W#>!1%#e=M_+^fLyTt(HzlUGS>?aX#GH1BOmExZ>}qB#T3vTya3Q!A;bo3K ziW%dArSXlD~-#Qtu<8#F}ZcakoA9@IgsT zG{|UDzD^cKNx3p{nFFR>;|E6V9;IGYCY2X)ygc6~ja67o8KO$Mg@)RA9hnC2Mdh(V zxZ>4I3&;^EMvOLkD5Al$uf7i_WrSi^dl8gdGU0e^1`d^B0N(7=$}xJL(+`f~A;o5- zpSA44D3XUmtxgWrd+OtN9~Hy~tM-XuqYz>Q(-hnG=q)B2L4LORPRk02LNZxkkgc9~lqg zT2`a1X>!T#4qii;1sFnLnwh$NRpJzugJ2)1yYQWIMH;m*K)LH6#De+zUfVNc`NzGL zEh~i#u23(B<^OdiHq|6m9s_IAg$dENLL5-py&*Fn%{z>3Uop0nyUKK!ouMp}kU|AT z3LPP2mQz8xKe3ORiWX#h|^# zk9M_K*U0>S&C0VsiV!8N!{~&tYWPc1=RlRV{7?kodi)VRT?Jf^eOuG9%s@c-)Q*WB z-1X(EPG)@W)%sm`+>hDh9bb_~v=#WQX&mpOZaumx#i?Ks(Bzgp*nC5_EFQ~yt1;Jx_16EeeWcT{?J3s1l%$-a*G#{r&st0$JeFov%Pd*JKmL>?k| z^5FAdhP~l0BJ$+Dm-wgdAlF!RyGG@cAFQsVl}z-Hx=#Dqdhj9!K+$DT8zdxU)v5Y) z)YlbCYlY);*L=$okx#IQc)9bDpEB5x(&7z2MMj8GuGEY!><`F#gJg#{wth1GigLAj zx!lnh_fYyEzDP*q`2Dd#8pl={zq=aUrhwJN4 z{-qro?li4I{o>z4IxZzM500LM>=b;7M5`0BB1t!v)J%ZV*g-zlIiOMDD^wiF0a-$m zrdjKi8y_c6^0l51^6Nve18Dw9x$JoH)$d7G0oCw;$*d?}5uVg~_EF1oYZQ%n4ZA%= zyF{!&->9ZL^CKXl}zG4bi zcwzJjQya`T74dNQfj3ety}-!)dQ?N#5GiK}e_cBz+x}KP+cfj2k_ZSWYaNfKoe#!; z0p+v3vo8rB&Mc5NHrQm`Z+Y1mHhZIHlpc4aK`wAZ+I=b8m!Fp|yV1M*nCEos{7UsD zGg>4!X{&!%7&=H$V}|Q7QIe4pu-I;m&psW!s`tV@`zTso37uBZ-zbv=)`lnN9bTp;{&WZ2f?-9xxtdZ|WNm>fIfs zjU&@^svpGYql}-{>D$S&y0DowA19~h!g0=1q)t6D@yp2rCHRzRl8QR0KQo{ zL-gw}k34CV)&FF|{r#miB{8v6y|?cnK(e|jV2{H;Nz zUT4!Na)gG=V;1UfSSpJM8QyZE;!CtADuxn3sZ3Pne9n~kHBp(FzV>$pZ8CIX)ZZK= z?D|sY;$Kw3=+`5*Vx8`gQsY}RM;EnF3o?E?doaxIeap5LvE%|e!O?csn@O- zGqGLRc8vj6TlSPkiS$ynr)lQY@D(fi^~Tcy!n<0lE(W=ry-R&f^gkaN&ir{FWcOv< z8@F4_cg+Y2uAfw{DW##*O*`oERc@vUXWeu~ypOAf`UIQP4k322KDLK{T@}?6c$u1a zP7!!P_u_e!nYO^~th&yr_^@*K!J89=$h%G@A-h*)y*`t-aAgpzY5*vNH-tjIn!y}qBXf!=E`*+gSz9( z(cYkR{5xEG`)STKDNes+@tSXI%5x#t0c+Jheg}e}Ql|iPC=+t;kb8yxmQI^aS`}Nb zvqG?=rAih(IzGbU2@kvGX)KH2e24|tJnrr0aqEwC3%KlnS53t!mSR_Lm)cm}DHr*q z&sKX<`^Zm#;^S4o4Xpbi6mg-I)A?x?8no+g zFGZSX7p+H8Kb_F^gyl~|)@>#uO1hYRsa8Fus8lBu?uX{yi*KBQ0~C82XO;+$|Ck_T1fRMIZE0J)a^@ z9~7c!BMwVO^9B_4_s8tlZ1|COQYnuR*D@wk$ZDOp#>m@W`}f@-4!_4pRYGljUxh&* zsy-n^=JWbY4bdkuYb|s+&{%ZhQJ}agKqQYWl39~rL*FaLXPmxy9zlcGo&9WXP=gaT z;jzZxh-oxk(J#-2dq99U8^)^%UX;c1_n47JUgBwuQ{R{*qY_#EmnGI#%JmM4>s4GL zSg#5Bh7_+{)WRC2?d{-PgG%+Y!}6tjaCUjD^`$ATAeYy!MVIWs#WUyv>dMZ%ZW3k9 zYnG~?35cNHII$njat>6maKRJouS$K%(v3W!1j;~7|>ksa_@*J&YJdWy%-T~`5V zI%B6JjOik*5*2z1)c1`mBEmOr<aEJ<6(Htefp=b?&m(;VCu45@} zY?R7&|5614{ui41pmjZuTI^7-*|GMu{RD#NvqpeJf7M-k4T{{SjLa18Qy9wHFA9C6 z4!6+Q!7;cc1(0km2U?941zukN-~gmhh}Ve{IN_*l&l#)5W&vpd3awAjWtXs8q#WgS zFM+Ms{en>#wIv6ftf!Fb=0T1Gn=z*A+yB8$tlb#StIn@k&cMn03KW+}8PD+RJ96AG=tty>tvdFxPv_8J?dagOUAj> zGBWj8ZzzERXT}-0DULL`D5`E?5if`uE^h{!89&cD7UJ>_sN!|5xYv9?J$uTkDOz;$U`9@W+E`)W)z0UM{2uz%S=}G;<7GblOTBqV?wxDpTzO2r zB*f+YQ>Y~Sb>o}%80^5)Zumap03=`5l0x&n01;Wn;8}4kksXcrTOsmRwuZ98wi77` z8%b%RrRSee7{Dnziy=*bMF%*YS}s@v0dYuL8JdY5LbRh3u=ehaQk)zeaay?#k?MYNy`L(OmYQ0|9p*-k+v8~n7$r_NifV-HYl3q1ZjG^i!z4I z9rc$>JW}#u1F^Y)I&!3jw&s;oKI?$K#=S&Bh&jhfy*MT|{gS#EzQn1%Pw=*i;!9T1 z#x_e&d?WqzGIvos+8J98V!?mEijvOqhR3A8U$cuOU%e1xg};hLxPtgdgp_4JmTx*J zV0G08o0VvEBU|d_y#1a0Vd54ZqLFAtzrmW(>XtVlBOjpV^Kb`&&Z;O(0A_MnJySyD z5zsm>7LOsg1o(W$Xuym~f-nqBpsi1c(k0%MAq4Y9{6(QbrX=rE(R$m36B4v&#*kmu zTKpShH+>;E31~&qF)rcZs^*g$ynjjN3+kPU=e!f>zE~9pr zVxN(;_sNst*Cu*eQH=RmZ->r-G0*(C-bH;Lm4Ozqos+Ygd4o39xeos!XmKDWhZRcJ zbQoLzZc2r|82(M{Ab-T)IiAB;rMz#@#k54UJfRorB>JXf%|NQrv_r$($4qIIt#ol# zd{2Y}?Q=&Fjd}hhl*ny4bf-XjMFBAlZtLa@f70U6OfA!_omti?*UMMK*D@!&Ylrei z=~ty~rI@08nCLFtm@FAYmt|r>Y|tY`Ei8_9?MDmiQxc@R2@qY)0%wXHI=-acB%8I5 z;n4j3D%#?Sq_O8jk~RoETi}WfrLl{S(sG(x#=EaE3$;}qK8f$H%6g5#9{G|II3sM` zu&EAVwhU#=TRONUq&Wjazk!$3p3`y|&+t5?+@+KUn#KzEpX$cH-l4`R!s@T#p(4U( zDg=X_avz7yEi?jU>!B#pT0QPI(1EZ%U3*7r_o7GvHy#<~IGD1lK4`!B+D%XIF*O~J z-jS365$8I8=6>9dl*pY+=R};pY5UQ&0}W&smz@8MTkqVlABZ?-^fgrov*NX!$LU;9 zz#QC$cteP+oFjT0F65A2w=!;cQ!LusMn4j z3@*OVv7Zx>eL+>X`ktSi*A{y<<)Rkh? z&CYIMQ;GE}oHYn^08u;(Vy*^d=$N^V0OFvvkZxlW{ZNW2u|5=q(~ckz^zqi$qnK+8 zLlEu!rZlCU>!LNHG=XR3l_KcC={(6>j{E!gc}i;*a-iTqawjEQsSwjr&iygCo(sqy zzGcAib}R2pzV%AA$NiZ_`6pWK+%NKuef*YdJJ)f5Rf@3|l}3$)m9P_)1o%ULrz)eQ rKCe~N;s??F*+~8Si`P81lw&_5t8H=nMoCk368vrFKHJ>wZkPTS0oMJr literal 77942 zcmc$`30#_2zBf+VnP$4?wrNbkOv>QcuVH6Th?ZIe`M6%iHbNR&iO3?eF`l9_hW z8U-dvHEv)NYFt3565Mb}Cqd!{H42DpA^}tof<#bpc^|WMzwiCe{lE9Vem;JlhjR|+ zoab`4?`8b^>ffIF@XNoY^%YbL+5&$6H`w16Pr9qWDOfjc zZS0IM*xImet*_~L6x7`pQk7rCSuU66*Co6DOSgqVv!usg<&^fU%XR`i{o&8$?>@jY zfODZGoWcr1fdwnXd0%zmWfDs&AM4YIBZVe4pl6Q_yWfD)|z3 zgHP^DVl*#fj`9TcUQAI;CavYXa*QqVBX8#}Ctjlt_BJ%W+LW21`a z1py@@{@Ba-f)voelrm&7lcJCqbRAuCDI9)Ao%VL*_6bf1rI2O$<0~n{8eT@Mj3Gu0rg=F4X*~_ zI^&?c)>@bkVYik)J??9m0EIkK=~sy*i}6CaUspx7SZY{AVVBI%^VKNbacZ247OQ~7u}a0O zN&!tCQd3$0bIYp6uQRN4*eYh0ULD|4iQ%N27dKarUz1m;nqy0PES+6ud?Rzl{9sa^ z6aEwkVnR6YaKC?7UyyV3jN(4!EfHzc$18DFIAERi&FksM^;Nqq$fV^F{$%fH7sopj zCdiI6idpL$-GALO@7n4d!CIujId+u%I(HbiQ@#Vl*xZ~Qn4137L)(Nu=|MX&!0F}v zc{~bJp=zr0#HwZVLelt7*DhD1*eqU|i?bCl>hC%E#zj0Tp!GtLuOW)D;DvDMk)R{5 z71{DdeB4n2^y+L|>+ygV%r$23K`|y0AsJ-5?#7tXHr&V%XT{Y}JPSW$j=kp4;~AWt zWjSQWqsI*?@F(HQs(nR@*Ak826ZwMK*v^mzeI5&UOn1EV!X>KnW!zkaOHqu`xx!vi^6i}MrCi?9gSb!!1(c4OFBBXM>&4Kv zJ0moFGSHy6Ub$V7gT@RWhZbQ;LCpf{<9>cn#rB50$yH!QM8;2&8j4S#G1y}H?J`Y{ z*FwL2AwiH#E2d}A92#qVhfJ}fC4?BOamvj@N6nZOjI>$(eTwX3JP(YsW9!uv%<%4z-SfIgW1mgAO7wEfmO-QXki%Wgf`f^Dj#S7tG@1;OxeTy z6?-_eRTEV0w?=wz(#KH35XT)OOI-un$I*(3AMdoXJX* z>mJ1|xv9;v5l5X3TXiVmi>eMy&+AEg$ZnM~#5WAt-FjZ;5E8I9KdVe3Gxu2gEtL#0 z*N?TrSh3&ezQ(i0)}I%pz+jB&bUjwERCP;LQbFPj--!@ zw>YD(Yv$U-RT^3>t;GwWXi8B?9uI(DsS$=jE9&)_CcXB{Aw{X04D>EYN(cK&Gu4B! z-CM;QZ8Xh$E>ekKQu}!GWE=Arxw1s8Gzl~A1zXB#VQR%yW!R1k?kw4HtDK=N?%x>7 z;8YK{N%0-bhAI(fNuO&We9>t1ZYB~>23NE1Mdx%&mJ6VjhDdws)hxnnVzwH~U?_|q z+6opM{OdHY4=CYEscPJcYb6+NDH0k54z0K?dzS{7otiSBSZr$h3w!sk|hHtdQXRO-p%6i$*sO8J4)>v z_nMQ!ziPZ#e{T{Q^I$LfxGfSo%Gx-S=)Ua#nhYjii)M&=uPy2$lW{f@XR)rzIG3L}X;rRranAfWHjc+B$m+7~ zd-{4HU`g($~#yZWv={wo>G;%cz|JR*frBp+fqAV=o=T#Be?~|M4yw{pK9-0 zuNBjYRqYsO6ldCH-sdn!S zgVmJAg`GUCiXr17_iLY}$Cu1#BN_hp{j(%56B)V^I$n|3aW(+-)=&f*m0w;^OjiD4S7!Cl#g5Rc}lQ zO1FQ84N6|5Fnxu!%TuU>XD+VN_^$X%goHjr)6(UXT04CBB<{hAaqTN@%_2dDS&Bx{ zKim(S&>d}rsY&+4YHW0Nk?^p@uo={WyC2T-!{vur<6qX!ccf%b1vyL=O%rNLk&@%n zsf{IW=4@-fSMv@W#<(Y8l3RDRz)=rx=u*opid0OCq7Xd^qo< ziJ1T$Igs9~nUC15=*xK11eIq=?&ZO@?1wC7@a9M|UMIGbI$Qe<1GAfcEwbDSO#a4v zbB+Q6XSZV|7YlW(aSaz(Di)M3?jCOIp9Lyt_Uk%VJw)l)=PDZ(Q>|>WHX{lDUEuL}|u_lole*OU=~=Xl4=vl|FS z;?%^;AuvmcSO7NS29KE{hp6Ht*lzPR5o9nm`SEl6Aq(VmY@zng({z}dGkr)wMETFf zVP6}lUiOC#jFc5BLJ1gPwUT0jXnlY#3 z(%vg&x(B&mT^yb~T=1%vT!~b}L#}b;f3CQw7U55pM~2(^m9){4ef`eSk9W6}!>1Q~ z?S~Y)$na2t=c-=`@nD#_xvUW1B`*cP=@Osv)kV?K{ag%kuy}S)NqYs0gdDu7JAN=? zC?oS|{K0{~q^qycF+4lzWtjh|Bn~8`Bg>Zy^XFzBtsLm>Y*&`C@3ob&e#k=(9xq7F zD`6?h%h;J|U$H_-*jc>(%V7r;zE;J&xQr=E!JOA{HG{oJiHwp*$t6xT3RYFICT59s zD0?)p0N6r#LaO{pmm{l!794xzn&vWlQBbj5{`g6@lY{8VnAd2@`8`>zmHy+*M0&=X z74dzpg}R%9W_&t6fl-K6c?>L<+N+QGMiuH}?Er}Q!Jz9h6>X#i4HbF&^>06(qRA^I z)O5ZK_PCjc^JSnoqq(g9Cef>?JTmW38v2?9qbw`Lu&c&?`lFzSNy_!K$#^G(X?!pP zVLGIV0V{KgPdFBWS$RVPy&s8PGsOAV9u8CV%~e*nIW~E)vcI?SeLUxa>dCLztA#7d z)uN`uiB5Erm-sld<){D8~*+^(_^Jdynl9%Jtpuzk>H1T&((L!ZMz*QK& zONM#sG2wv7S?H;JK%hG;G_Rd_m&fTH2Z5vyRIM`ws)*bjKd=%Y(EUTO%{vtQw~N*p z%9fPZv$>|&Bl-GYCqEt$AD}Y`zSU*3!R;q4fS|9S^M2aobvKh}?|)uk?7m7&r}*nx zr9YO<0={`iv+~Ecsts|QCb27}^@E;Uz7wq%M&7oz=lT#beNw?$DLg*{RYE`cC&wa! zpk7%v(PuiF&*RQ4;73f6G*Xh}Wt0vsN2=#m4v3>O5)k2{^Ms-3?S!_L;{lqEjE}>d z$(b-8UTpd|&5EfET3jPsNhoDunik`R8ugKVZ#xt9!yBEQNnPjr6P>{j?#W{TS)-|q z3#3pm#BRc#zse5I%=_wT&>`b)inx1eLsde)YrenU&O<-=u5fY61KT!$q*m|88DqdV zP!fl!ngT@C4aJbWJ4tF-&Pu?ed$&KNN?)Eh(sS!CCo{S5@Gllf9huXJWj%u|iI$G2 z2lE-_?93N&EU?6y|rI*mpycxjM%&@{!JNLI{_fzQnHxV$@um8X30Ir{eT_f0g!-P=aE#w+dx7DzBDMtr@EhUnc(z zt$*8h?cYhPGI8wCIyP-|NvZeMx2&n>Oc4Z8aPZ|)Q$j&X31hy6V!8E$6Gvryg{|)}VD9LJE(SWzB8Qp@iW+t{-w&80%su&avpvYB*Uz^rx9v z3mLauv5-5}mEn_YR#l{COO+MaMoCrSDq?i0auNDk&edVuH%k=@{osYcgk1>&dDah`s4mk_LrbYUc-PI%bh&p+zwr<2H*}T{C7t5V%Ic{J_mFDi$JnnuHrTZF zzpsAo&fj{o-7~e&BXveQCFq>Mtlrbw2_{vpZ6zqzjXTbOg-hf&^+QTKl!% z=}C*BQfv{r1}E)@lvEumh-tw__tn=;qTf$ucv{bOSe{C3w6)vLEq4&BS5JBP7poBR zjXm_&PygL8vxgUJG#Iip{iN9yCHYX$R`~MMrzw^oG4yWs7mV{W`0D>f=J9cq5T?j{ zZzg0>Kv*{K)YhQ$3iUe%TnmngQWVL0zMGk~R>y*``+ zu!7;@W_(m%`7jA9$CvnL=9R!4q_fDu`PhI{9$-I+aS+1AP+yLZn)$bxjTBd47VojX z@7)tt)53BpBHtj4VeDP`!x&CdcrwF{jGUaHxsmhFMfCKrSIyob>CyYKXT>|;YIW;D zNH$;6Wwn|?vE(IVl{+)v%4acbw-N>#Cv~WWzZp+kNcG!1%Qc6sXck$XBUi*892g!u zARg=;J+Mti`3fA2G0OCVT%c4Lz`2}cu5{O7V;Lt7GDR3K9TpQR_CE#iumpZngVJG} z6teG<^nPUL=gU=0D@=$6W=+Kx6q~Kchku1yFn@f~7BI6mK3lQ`WYG2N;!exP@FiO%qzehiAm(F)bTw zbvevoWah3RdWud2u5c6)IijEDv9Nt(4$~gm2F)TQm-$A^3^GhwupI0X9IPzh&+BKGo7~wG=V$)HtFxWo9$%3z>)18xg8!z(PpE&vrIInl==y@O$^zS2vouHD zy19YEXQoeDIBhs`L(VvmyKbHh!BS?r{~6?J zV=u6bzqb%tq^avs#3?>3yK}lco#WFTwkF!kd;l|}Tc#W8)Y2^6vw44{1m{&`Pwo}u zxdyQUy9&O@UVQ%R-hyh0Af#Tz#R&tNV%TmM2Xn6U;juld zJCLZv_8E33tMvNrRb$webA3LX^btHGZE-lKCB0WVw>sW)VbNFEidZ6BZ1V2rF_h$m zygzO+_D;C3joKhLDB^-8RmctUV3oq1ObM9TT;DNxa+B_p-RmrdPz zYY$6k(j3+5s^Sc?+nSqv16PoQOC1?|xn#V%W3#PR7`#j0zS*Ft)Q$6-!}q(>5Xs~d zBV{9aI&SMgWot*zxqb+W43D-th|Xls7cep#v8&?;1EQ3knplc<5Cz62lW*Eqf5)ra z|7VP}XJs5>?$pOgmjox$t~Mh|{mwbx)taw1c@nF-ie`Azt(K{UL58+9e;U5))6cd0 zxnFJ2HeOS1piQ|0iQ?*yZRwQF&i>jEN=RS5PxsB8t$!qcQq~_AW)&_xoAbbFPY|vE z=S>V(v`a=mL2eF%g`w7h&oxoNOVV|*u~ty5upgKY`C>pa(u7E`KBrXHtMMppCtyLN zz->FpRkP*{tuL1|QW7;Rki6VdFljDnwvn|Y3BVEwnGr#RGZDvle_yE>ha$vfxdAcoa+$QQGqt+hsq%gZ+sP5OCcL4S=%T^zE?Y;=4HfG2W6BA=#Bsji`S0s> zr1C~g^H`v=ealY(?LaG3uFk<<@Ja0UlC{j4OT0@%JWddV-mKd<0d!+-kWHrMIalxH z`3bdF0X(ZqxJgUfQg&4?tR9rE$w+=OViFC-GYn)_nJ#Y3{yuI`f3<#meaC&Ak?j={ ze?-(-rZSBo`2J9sq{t)#CsYN#qTp+AYXU3WuDY=%DRI!0qh~7}n66sU%7Rn5tV-Fo zZG|eDG`Lmh)_kbRen3WIwU=}%yRb2^dj-eu+*$A z4^6D_onNY^LKHk$aX6U)4|kGg45m6q0mvwGv^>M32H?>yT?R{_9|p<8umGJ25V8JUxn=WwwDa$lhOAP*cVAwIhP zZTH|WEDZ6rdbed-p7+*?Py!mYH;>2FCb!V~X7SFr>ibSs@`GwOzRkKll8KGUgGFE$ zdcJ@B4YR6qpKIe?l|+o$s%xwE4{o(*^oyFXM*xxm*$7vgF*C;#l1;;;1Sf@)jxCGZPEY_ld%EZ7 z5*E0G4QiZ`ybD_0iK6DRxS>E4xUNu;2MZ|=ynHy#)Z&byAAtC*~0IAeo zQstJSb3#&DXz>pXkgB2Krm@pbM7<0p-Yz^$3^k`<45z^Gj+mB|0E2f0wOuxrHc4W( z45m&-XMB5=_zhD_RNz~t&*x_{hC7vImFl_JwA6sBNz-20$%2?4zLP-W23(?&qWDea z!&So$th}#B=12j#(?d(yD!jg)Hc8jy78H{v;j{RofO((x6(&>_HtTCArULf3XC@W+ z(#mjKUs9;{rk$uMk|HAT*ZUz5N(rogI46KR8(ZqA=vy04Nt&*MsYv4qc79>1aB*;$ z%|p{3S&(w=?0j4p+}yjZ`b>e13S&I|qu~d-u3BIDY|ATu!q~I5S&gP_Eb~08-uAYAJPUjA_l=%2wh5t4tmSr0hb{S zE*{u-qph6cY!2aOT4O)S100swh(Ol5g^2-!7J#*|D;-+kl|M|YFQ&_Z33Np{M%y;> zpuV;73T6p0u>@fJsys6nkq^HV2EKZMaC48YaWKO}``FHHVd1*A7hh+E-B53Ia%XdR zEjQRx*ZHOxp5Z~$gQFfb*Pb>waVt&#Wq)GWwuv0w+eM8ARv9PIX^HzLv$0#8nWIiz zISS1-%X5QskM4IPr=Av(DV#s8zBa`8ly4A~){ z|0G!I@yM#pRPP%yX=WYdfchk#dD%LJ5;lkXyBD#6iF#@X1GqLkH1uty6fgVmu3Vgmr9++SSTz>Sg#5|bGVi504k;8ur5qlYeYdo!1K5;}<6 z!|5qLP75rdimFb@_&w(qdm!(s2W3@W-LvM*Q+7`$zD|BA#g}tpDv8|MZ*RzmA=L6S zo5^uq;`fAtF(s~7iqB#QA%yg7_7O2MF+U=^VP^Mwl20njCnNn;^FDU@Rxvd$q%!#I zlpgF$rsb@UB;X?eP?xrqy{2!iISKL5Ge|7FrIVBJG^1|npJAd1@!wC3UuV~ zAa7bBNqntR)x=mn^Agm@c6=%+$ZRt?lBPmFTn>J%<6Y-<{d95$pBK&1*pGA^{&749jAbSwZ)l>vC zAIo@y$7PX{y@dev@mjn#+(TQbj$VlFBYnXoOf7F3G=<9{0ICwQbeq7htd73v-%OlJ z(=$A@5VA!$9dt&T(nB8HU9dG~zZ$%1-)w~nEm{>48nH=Lyp-s@F?-~g&x-zio{7V2 zI$p|ZilmiO>mwifFr@k?`d9pRhT%8u;^wn7-clG)c@fwdAPV-@q0k+} z=HA}HFSIpISP}tTDZr&T&ybUg7@lN(l7%k*fY1qc*VkP=WSf*_L z76cGJ&UvD1V^(-74I%f821fiBvXBTY~if6k@Cbx2-BI2d?N7hCkx zyfEo2aJjh((X`Rol{CZ6&vQ=Ca_(ASRxN55_t?7EHVuQ-PxIHd6@J)a7aK+2pgcM?5q~5|F|z34sf|BN;6j*-4c%VK53{6E@Wi4PUlL6 zxP2)1mWZ#d7iE2_=?)|Ln(=V*v|(J>87gmvkR~TQr4|piLxTEf@Um~=;)nXqApbFj z^~5>}chWK@T_6&@d3nV`i9wgc*OhR)=;L6|(*mXWHus~!LQ+bL#Qj^F$xgYEa?r35 z>yE<@!D-@aMn&qY2t2Qij$~l{ryHBmgTr`X0GkL+e?YjYfb-3r71eLs3K~bVNQl^UGD1Y2;Gb|d}*G>7hGEGf((?CDa{M26nwfdH=+98 zp!szP`S1{*z?I}*4NtxHXo$N$TbzhAy6ILnQV6u~iAlEK<-(AR*?ihk)))l8QQI)u zXH>)^oOuBqmu@XZS0PZ^1MzN#{LhD+1=-)`)8+GNFLt9iD9gUii6|EpOVGL0okGUi zxRU2G=W-HztL7Y;Gg+`{N$ij`l~lqNZ^5522}%0wQQ zki4jeB44c5L>YBchbgjQ&uHePF(u+qdUmKcA&5wN$9v|laZIdqBU^sc(!I27?YFT? zrFV!+ErbYVCtJ7mo^Y8em_*gi_6vgG(s_Y7pwj#mx#xMFcgD_ag6)}4EqOScm z9C+wM)t|K^bfrR|MM#VjW|(cWsvZ4La}chg*JI z$-2gKahqwarPpfRqRB_wQ&iB+`m4>KUC zpW<>xbm_w+USbeXu`q4Z(?xYS+&$i=&)>1nY^9Hd4A)L8>^Flf7hSFP(-=)>2+O(( z*tsB0z?S$@km}M&*VJd%KZKoG!M$T+gt~qn@^|^Lu683lqP}K|4An`DGyM=Dnb=4W z2${HE$<>cAZ%jvpD(<(B&zzTWGfJ~fHknPnd223&KlW0*yJSd?uHUwCUsmM&oy$ka z=f?*CHFBIMCsKsEV_;Sp+aQS3IyYd^N|gJ6y(ne23ck4~3-@nuV2>(P@m`Vsg~kND zY#z}7GqDtS?O{n!CIgqv;5z8J7l(?h!zWt9hYb*<C z72^k6JOM|=>OHW}nD5&{#lhT-Q_fa+vMoUn@NLl(tDl!k?|WkNgAAj4(OCua$9kfV zCO5KdsO9JNb6}4w`MNxAp^8gj)Ymkxvj|yK!^R(mYD-}=HZC`Ayvk2oR;X>y;Q7_5 ziHZ3VT9=3P3$Au>+p^~m$gl`rU6gnYMQ_?R(Bm5Ugl;~>X!I!Q=wxLLPo*m1yW`!a z6|!XmN1e*xlIuBUT3pPS{cL+yqr86FC?PfA(0!6`2^PdT;zB`Uj3zF8(p)&(m8EFD z!Y>Wd9kVL%%(SVXZwbUn!1N<19HKMj^B&oC>FO7<*)@znru%f5x;yc5M$8>q2lJ8= z@>)^;MTQL)HNdhFgExnV35P0ME8^=Ttof_C%Ju#Zmid5?*5r)R6-zoh6t@&6k_o~m z7og*l6+)fymZf@1SkH_wbI^>yD3rTqYiL ztD6!Sfq))aIYmw>fjL@Z_1Ia6JkPsY3!6F+f@z1O1EO!B(CRE(&VyQlMEuwwGKKe^ zw>4VV^2uXRtOcxLfE!IFug2=b_?-)hyBWcJ+OrF=aE{|%=umBtv zpa@y*;BsFG+7{G6<^pK#)^@EgcZceY_gBZW=GTYg*j2JNq--h z%-8QK%x}hvkD1zNf`}ZNEqG0OUb$i0CSQ2+H-9$DboqU}R@Fp_o|INFCyFJ<^l_Aq z$UM;!rYKa~mw`>O!qo%}tYWdoe60s_jn11ljY&E3hNaaefcbA;nS;JT!_k&ysg2K7 zJKDnF(%X1V`cl!JTNCGg(=fh@c|$XJoU0X=iBbDq#G_te;!tqP!Gb@zU?2gnXy0Su zsaVzUfxx2jebS6SGW-T+`q;5*dapl|8pfIgU=MVlFjL}1ap%1=llErG@K13% zg-Za0%iwL{vP%YLi8^&;kVx*POpF~ zxMB@loD_se^^d&c;D|wX<)$?v@LmIO!24=63>O^jD%Q%5`V{GEMInF9z&< zua-K9UHm~uVSk#`4GWMH06DV#*$<#7+W+}D4Uz!}m>3W6o{sReHxi|D>%*Naup3}4 zx~j_nH?qgIH9jl_kQ+>OvTB+eFrm9;pQ39+(r6;UsI1a(GCUuX9lBe?fc9EWZEuT7 z4u36PTv?>>gHR5pm+&33`FQ7|j~aW!LRVR<|HCB3H)cKR`hUBKl88h6OU*Ox@@Uu%Qn-)EwG5U?u#$L0VC@P-+>X_19+7m z8b1Ac_k*WEZO{EwX+LfMFzsjR`{`#NraAny-1jt}E)Xd3ua5#09#wfJ?S{Vp@vE(AYZ^Cr+!uN+ftc#(O zpxsPvx#%Ag8@Hcd{yj*)rUsamJabcZlSz|NEE}ZshI-SRD&7a&W&Rxch!1)NbVfUy z-}&1Fk<=4MY87ktShL~Yrd@3B?K(f$YH-`S(7Ss&!La$-2S1^7Kp^h<0@MCxpY&r} z;#YoC2>p*0(q!~aq`GW4*I~qo{yy8}&U)lvL7JW`ILKV~2mIdyor^-j(^ed!FB0(b z2`M^(1{RtBxh%J7@efn$=Tur%qdKyL7z%#abrcwwgF8SuXM1{9-X!}80xiY8UFqgV zo;SC>-&FR{Mcc!$$Lh;Ab9&tz&34L3pa5E#{VNsF_$45FW}Wn~N#v_`e1|bXA>c(l z_#fVr-V|N}YU*i-3OF@%0-X%z=OQvDQ=M_)gp%G)SHe*{KL=|8^pbyY{Nn<6?Nia>B*S3w7^9C-rC2g`|q`N<3Hm)OLzqnD}q&bK< zF$jp0buz&Gd1#fjmdUO^vC#fo?&libPG%r!sLHb4=A(%4*0x#$R&h0htAs>gt#77+ zK$$CmLcNxf+nd_mRmhlPc}xs&RgUuu&@d@(b=#)dIvZD!=Bgla7a$RVLibeCpq^Q$ z_T`hWly=1s9hBvWdaH-_mdcKx>O2KNrokeDz9P;r^sCF-hXSWXOoKL#ak)b>GGr)> zbMgCmed592PORrY!zMP6MO6pl0UiqM5yTTNZKl{ls8a)VJ3!xGet>Z9`seQHN&VAj zTZ34oV-4K%vT=t^Z}xeA5a`>i$}QCZL)mjv zbK`x$6K33L^Us%eKHeS3&Emg{uRi${GLv|O8TNk&LVFf;y^FCnJ0PiCpS$t)Hq1{Z zD+MCypQMLiy<$yP$TaJg<&&Q+;e`sZ`krRT79W967;Qi%g6qfTAz z;UBC1cmM`Z{kH#dDWzfb(-Oj^xKNa~U+#<4M@9Z7%bqE_&5_SHSW&L6+_>rGO%N#O z9D4mD!m?&Qy<>cxIswo;%`0i%d(25RF$O&(zrTLSrx&3;ji3G?L0Z9^S4<<2c`c>q zNWNlK6E6jqQ&6>61^DD2k``wKNs@uUDULUu46p_0F8b?xn z_L?(a%nlG^nqaRe`fobnx@)&_IbN!`szooVlbGDQ5+D-C5;hxGUWOU6KG;^M46EZ0 z*4bSNgCqWvN+-H#^MeG>-2~1mvtTj5L==p!WKsl|ibx%2g8+(i%(W1goJ8}V8QFfO zHZ5+%%(G9mo9)~)0h1a z-!L=hv0v&RD=$q|7HV&Ira4~}VXap0^=^zS=u8P(|Ix^Q0kCS4XagBTUX(x=F*7w2 z#7_l6bWFshsdzvs6164bikk6@!m^9ti!4AZ zi|Ug(NsD~x!hH(d$;be1{$9eH*o=crD%PXDLPhe`;P8@>B`1;(Us%~4t}-*j|8 zf&ldabUAyyiCKRym8KN|;JR2RXgT1h=q>`6`+*&#y?*@3;n-4K z%Bs+G$P&%6*-y>W=W5p}f>etbO+&l%P13>Ipfka>kNg~Kw%+gkVE@k1=G~9p<0Fp| z)12v{yQS{Jj|d8QL8@FS_qU#%u-E_{ORE5MEZuY1C(!8U8}EOb8tc7f>)G1SEnq3@ zyC_~rk@-grV9`OO4Vc-9$-{qyd+i9LZ^ToV$=V7^RFUzw9>nkS8i`vL1 z&jXtN><63}_(1HM_iy?PxcA2M{>r_$6wHc}lgRgd{HWLSO*MN#*SFh)_A6m4nq88u zx4or&A6ZTm>rii~zoQ7y$_%|BuH+lwA6M=tO5Tn(KXmTh>_33+)S(}Ic0BEon`G*` z7xIy)B7Zq(!r)pkvoU?e&Dudo50lfgEaX}@>)JK))2CnH8J|=HKpGV%#Zo@v=sy|u z%l$woxCTvs;}P7{ofcJKGpogl5r6)^3?VxAayRGsrFiXz$1AdIoaE28S9+giRO}ytC zT76%WVGVpcm6T}N_0fv>yqFp%ajxJ0T z(W2#Ff~TVcPAxtcCU12D%*^Gr>+n!%{5FKGKV_=k|vES@QA|y zxvYCDvzfmNv!1W3X^dQ52d{A&9=Y*Rn)w1KAm$widlX4pdrk|ClJx$zc94FRwX4y= z+C9|T-O{G<5Muy2!2cY(4Ukzm621dHZ7}{AVH+JL&-{uP=R&j9w<_F9{F|L{04G`B z(Or$J`Q(d|eg`DYAKo{g`DCofl;yRAF2glmb5&cZA*v70CUU?-a0he2pfBL$K4~*0 z$RDIB+x%;R2#=eB%<99m_(sgo3GVGOD~1MW=&EVDw)rY{ptljc$o6vQ*4U!|ujALz z94g-Bm!@VyZjDl?~F1xsq2%rYvK#8tJxGCv8B<9wur+)?gSz2)NAp6 z1L(zgly*MFI*R;E29b3=4n1@#VJJ`}3O~QP*N{940?mEA#-=6SeK!4+Wyg&PM*d;% z#gHsqSNvzn_H`%=a7ZQXOnvt2**>0dY6gz@5yU0kWbPVm`vGq~S%;vPOS5{%PBQ?c z_#tn?Q};X2_1YsnEC1ZNUaj2u0=uZyBXyPUCb(uo%A3S|DDGdahvs@dupU+O47*A=x)dL&`^m!l=+=BLZhWnUKCTP| zIK8F_8T<%8u9iKWoj=);N>bRK2HknRz!W$ntE}yrPc;Xg`Y+P0zA6H%Ev1DEL?}@u zd&~#L6UTYT@V#i51+iYhirtB0*^2V_D(r%$!(?cFpbZ^mJdp0bNlna0O zadQWq5wDtdZaurv0)*adz<|&j9U5`yv;WfBBT8}V>de-L5siS^RFoA=p9k~O59?1; z+!D~$9~y3Z6%B*{B|JWmK;SSM%aM-%d>O8uxuRGSy@rE>Ixq|wnMMKd;+z7OHCsMNcT_EN3=B`u{ zGrzvK(m~`K(4ag35U8gEEvsXWk{?&+CnvBxd!4a*`}v-CL7=dWE3WTh9{bg(ut0q8 z!^jfam@;eYzRh5@b8c5K$U)5o*Z4C*zdSa0#ZpdnRKCT!1asTwk6K!E5Ja`UCi%p!EM^#KlID` z!BGQ;q&Z%WH^%-W&hgJ{p9fwN_B{K^fVr7+{GTuy^wemE;b#n_@zOQ{3G=f0oIdcc z5|FAApFcVw3Ewn_4w|k41Q_B_b2nh{H?b8vxo_r9r$TkOqau7d#EzQLX;g48utLGe zJiC!@n(*-dT$Y=7Z|$#W$cQlSlhl5#bQlY#o}h`Z$W|^g{tuj_bN+GJf7O$u-7`$? zRL?oDj*Tj*3YRT!BzTrUIMMQ38JzmV;k8f33UH57Xcvh8FQMLlPE~&0iz_z&@(3gC zxwy58Y5n+W;`Z4eO32I=_A@2K6mZtbq=#pGOwBSos!^5w?I}7a;}HRO*XYwdS2+*( zc357^%Az-<`3Pz0->}V2!8|oN2;;xym+hB|$xq2E@yXgiqY!+Hkv))%+Y#0Sqsb+Ou#2rA~mHT2(4Nz{WdT((fn2bpHTEFNOyglxzAaW_$0LbO$-) z4)m961PK7H#mF1}&2jNJdXoK{fzz<$r_7YRcE8QJRM3N%IH{GYwDS`I*K5%j#Fs8u*u@h=T@+1~R4v;qM3d2m^OZYe?IJE;ShASMtRnAdqLzSmo8$ zJwwau0D5BIRFzez=YR;g;$Heq%u#W|VEJ%@ln%hR!%$j1Am{?2fXCNAfc9plC4a@n zrGW9o5%2@(dr^GM|G{rzLr>P^@Z?W9!hYTo>2Z|&llbrCgXz}*9Uq`hyWj5I zO6TkVeJZVgs2_zdz$#_9fV1Q!pVx=FInjMP1p4=$sF9D*R_^-BV(ZMx#rJ-| zWWWtnLWiwH%(ZT8+KgMP%L?>X$3N|Vxg0*30KYA0{*W3ou(fHVY;~+DUAJzy9*Hq^ z{@@@_mtENI>XX0gH7mED2i@@iaNLt5(DmepX@`H7w*O&T>O;J@Edrx&kRD>5?Qfm2 zKI9D%cL1;V7dK2hU4JJ`fFId<=Oc@#Lb(3Uj58W z`FOyrKjJ?I-U;_Ktr`{}mM84d z#Fzb)*0h{CCJ-5EkJT`Q`-@#Y>@cq4eYpwKMZ!tH~wXJ`S#0I!W-#y~4`PhT~ zrf#(GFwH@I%!E8@g8<<*|6^G+2-}geGV&`6i6+0DaZ++lofGv$Uk{2OV0q|@Gqxj_ zi#jqIauS~c-G6~XF(q44AZw=^@WzQe9dPc|ue^y(w|YC@4+v!8jR*@$opTg$kPXG} zv7WP)$w)D(egcBV0e^yY8)&-f0cJGlZ0#`~#MmDYBZ9XpA^AZkfIksDsZ96ZCNhhU z`2sx@6ObePWR<+i0x#XIxSj6-J|&>}*R~(VOw+-y0SC&FVa$OFrnh6O=C^pu@5*E{ zM`F;g<`S1%!_y6M8OFVj0(!kWQa?swOEHSq9v$%JZK$^?1@%?97RiXRtJnbqat2NX zEb1<2HdUmjCttuUr7WW}zcv8xbz$Y_6pq0$|z37nn&)(T-~AZU$5s2 z_hG5uTKyJ04KvfGK>;m##|5T#TbgU*>tW(>Z2bd=>24k8%N2ZMZOu8b8pPc!|19}0XOSA zxxe-Ni}53~<0wf9MQCx(6FvZMV!WP42Q1QPeZRM&&kXuf?RB=Ef=+7f_EoGTr4e_tWXGm9nl{G>mPE;V2M%!;CZVfi6&b~ zl5<0e!OH?qxTzHf4Ve5!(WOJ(aw_TGOOrZDMW>~K@OYHW75~3(cj$VJDwxh`zW3Br|>ZHxgEXF;g8K){je+rre43QqzzV3i`Xw33okd__A5V-oPp*=Yn;jg8$5(-y z@L^+$YbWZ-Fm(Jf`HyET$=4%Zu1((mWV83A3i@0v&gfe0VgBkX1)qb%z0CZTPsqRT z`kz4YG{vo1$?G<`OuDw{x^}X;Q2Q0kxS{lG#>WF2Qa{-Jr@f8`ZrbI4u&?WWX;8=hd_+?px4a{@*=xAJ=OHgvl!AQn($CvA zX3_TlFXsx6{BXdvYW)K6L?qUm0O+;2B-O7F_&&!ANPfc?pHlUFFKTG6;js#F7&sNA(;-xD4LZg90j}%O?@2E>2=kk^{9|AK zYy6(dKJOTx^2yc3eZf8I?eY*$|8-^o3lQm!5y7sZbMlq^}6TWB#nC*--`ppBCSZRW9CSx;%6HzfoF>#>9AtD>sO{ zaV4P&MVt#X7IlU77`GeMZ3aqElwA(T^B)?#6jef;n2Jw~e-54`+;7UWxg8WwUCqX4 zhRHC7Fl(bqemowIhEGWOZ`a+7NG|aZSJXeq7UYudxv{j3K&hBZ;vn$tNG;mb4W6uaMFj!9#JOekE*9CNi{b}@!LFk2E2+v9A_ZS$Qo*Ai>a*k;dcvyzy z@dVaBh<`Njs92Uq+!`nhhk_D9?0-i_pfKd~B6?HB`euH@O#LV2zHS(yabwJwC*(5e zgiJtUQ28Vsw-A3FFTOQ?m^)@z-e?_GmlEwA#IOIg_F-K9R`&JQ221TTH$NV~YJk^P z!uANAdP9a~q~jv#z9$=_lJV4 z-*hMyHFxhAkgdZ)-!gii)W^}ER)zmZse>D1MKHDs$8P(wRkt^0CE%~mwpAtk8<=i; zmoS6KAQgT&GoJsW>#Ks1e|KmM7R-+^Gg`vg=gfyw)?caIvLhImdS(f>{<}{N?J@R( z@{YS?-jn}>&KJ1Cr&b7{>ry>actOy&QIE&9Po~4k+GF{T?*F5CcyP5@1T3~Fr{M5z zNyS8T-=K(5_esB@0%axrF!iOpOLY0?1|4S}(yCizH1%#2?+Koa50X%n+fz^J2fsci zmcUaB$ZxGb^|T&;Is`uw-!~i`NIa!0^RU!`MrDjHTV~@><6)l0Try~jbbGB;x96H4 z`NPky^~`;%WH94b9Mph(L)rHnibqOb`l|q~xsan54z>dvW6th5-@chiop`NqBI8xv zpU!5Jn~al=A)!-eD6NACUe$rleEs($f0~FNe#A!&JXRW#l#O54{QWojdjE6;{Ljn7 zPZJsdJ=FVC$1j`LPjp{ZKrk>H=~J=WI=G<{lvY_K-?aq4O;i}3Wx|v+J=0^tp#v&> z;*gJJj~nX2l6iI>BH4Lhu;WYmX5y}a-HpKD1LT7lKQ*rJMmso+U}iT!7rwTK!A}OD z_ZcfkB6wcz(5&Bj20p1YfAF{rI`8t>JMycaJt>Pe{@U=4Y+e4}?;S~>wlgo}U@QFO z^*G!M-slf*Tb%syx!1`npKSW%08DnL>#fVArWC)6%z;^7z^i=xF}`D?Rs>#^Pgqr} z7c$oFt+?Ku53;>o;_q9COpE57a010JWwChtnNVI^kD4?D@m&U0B^S3=tv! z5dq@=Pj!KduKzJ69r~U z-S_13{r`RnZ2vidHIwnN|AGQA)IOK(>_J);tJy4YH)gaLhejUv6*nyOLH7p+LjIH^0p8zCYRVUK_(Val;Am9Ct^Vsr^KP zPrO{+WDL_5K0RV}&+j_XaDhL#+TB@%UR-&4GP@<{cX`U#AvweMi8BM+`Y%Z4{&~Ih z-!O#y7#-(6i+lN-g_eH!ZEc(0hZ{>W$N=LVpcYCjd9QSZub;uC65dFrm$3*>;O|lI z4fGiXZeh+K?3gs3&9*&%qG^l6$Gu*b>Y1}|KA%wLy#tf-Em*VoU7Fd7*yB!#nrRTl zVvmNw(R_#{SukC<2}(f(0>Zdm+@nzRh31I2Ogs_Wn|S)A-9YmP=%u6A(h_V3rcOy7 zoDSM8jn48Fo0Iz02YMFb;;)o|)QzhCawlM8E|1_$S807$Y5iZ5;(b?X{o77pa0mac()zB_dfEf=U8VJ1rS)B< z<-hh_PwTs$)>Agu?|NFmMe$vw^${%TKbaotrLU~delbx4 zd(rfJAi&MW-sDjBH6hozz!mQU4h<{7Q}8v>7a)=B`?qQLWdE;+vaK$vsp~5ynCr)z z6+2#7Ti3n18G>$)bI<<^@(P;(3zdXhW*cNb;)A#dLF4dXrly^hxmC`tYS(pMX(Zqp zDLh@<{P5I-P)(rGeYYpx5y^h%rrrUNkV&2dsY=is$ zrW6)%q_l?k25-DK-yJ2_g%HCf(?o`N<$Bb>In6(@1FHYKa@xhXpNt1KPVuBXQW@KuNrI**J10bM3d92bHlg$vnE4k zdu?n;GeslGa#N>YY3&`%oLmS zj`ToO#dKi6+8jNoEhjl8-~NXRpqk2#!qV9&I55GJXM& zAU4K7GC)QZ;1Za-j1B8)D|O^Dm#)vYa(xg*EbIm!|Id;J3`~eMWLvx;*O1({9+{S$ zOfvL$F3uOpKgsX0tn2>v{l^B}r8LV<98Y^oA29jNcsO2e$LvppwHj-E&@e2>H7!5E zT&u*;mDYE281AXG%dJm5}X)Pmz9>6ua^x*jzH&Fg#ClqmH>_LXLGz9EH){h{1#U&N&9k z3a`_oi=-n;AIWWfl_=Mx2rBu9o3@Tqt|92B+s_|(0?S_}&W$Nt)5)W*=TPL+8S?GS z_6=r!3@ZOx;N4xd%Nj0oeUjU^Kj`XQF+OBJ|BY>YquCIrnrpf-@w&{%`8E*b5a{|m z+*#8e!Egb3T$B%+2FVbfU5zcUTrmdws4Q*;V5Hyy@W~JqmM&^OE=L@Qmv%3Dv>uVK ziq!hZN4)@~C+JX{4E4!!2pjBuN zk;Wsfy4EC%W2-2sN~@7l?jmE1`*ro77%-goeX>qP77>M`?!dh|k?xr|^O@Rjh{Rnh z>*fW2GRTW(3>!Z~PTFqet<=qT1kI1n)mv#)(`kA+YbiWid9`=;Y5;H3xw{JO^`1jw zWV-t~s9^v+on%z;XAiJCSZfZy6>hC&Q`cEg=W9fFNO`}dr&42J)cYgE} zwoeJT1jWg#Y5mh}Abd`mldNL3gx zJ(NL708v}2)k(8BF$z2`Kcke_EW9eF=--JGg{N^^mA(x9XQa^fFr}4Hz1>e**|-$| z^9>pY}acfx9Rf<{RsA`7Prz&yXe0P*!D;KF5zdO&2OP+&Is&J&1bOn;$NNg30Poey2 zHV{RlNXSq)=1XEFw7hi?C|ukqS??A7U4D=~R!b?>mBbkX-Zlz(n6cIVQvou^Xl?9x ze9LhfXX_Xon4$U62;OTw&3ty6p;q}xx#MC&2Fnp@e{13ru=>FHUQuzQajusnXmn}y zXc2@aMnUCu{p{v;7Ve!k4xDNj1Vqm!D5YEW+LrZ)APpufP2IX%`ujKGg*(# zmV_HbH{Gqd*CRz-ciLJ#3-%vWrKbXOdyIafZ~Qo%@m3#@go+ibREBXo8Ba2ty9hu) ziu#30%@XF;n9_v`qt8r-ApEZmPy=A~RqNBil#VQ~e)O5l-PV}8cInb~nV<}6I4(-O zp38_(dDO68R0Jhjf9+=N>@^WBetnI4>x|^?&ntBc%(@4~dc#tkA={`xs_Uth;*OGG zAD(HCF7LpGJ8xsQu_pkyhK!1)+bXfZ{w{GO~8Z-VLRJAlB z4n1YOMn1_T{o?nH8*u;l)sB)pB*_#Ez+w-p*2`BZ9|>~U*Gfaj8uvadSXnr?3Aa5X zclA|VOe!H#X-(HPx}eF__CCy~P)Uf{Q_6~^J<7c_5FhM?3J}^tj$J?Q zpps6!jxIxkHs45Wpf=Nwt6r;b=DXX(Rnxsb_*|?AUpz~G{|kF`6#Zf5Fhv>wOggX4yh#yl0?jxzmmcXmBw{P!s`? z31iEd^w~97vyXm!f+IuFD)MqY@y5N9vBn|8T6MG5(EiKHt@1ihxXES2x#6GJhwNjE znghz|J}@~*`}i}~lIqDSi;6|zxoibU0#MbxeOW1>*WooL_JdO(hfj>X)+-QBjtZ^V zTkaMI%4{wo#V!OshBK@m^`si)3yb05();=?BWhPcxWcu6cL{d=zUV&<{fi^tkm&nU z_?*7s7*MON&UTc~LcGNsm^B$9S&YVM$*|K<_R0Ny($pFq2!c)zBT}a-WhzeAMQBiu z9%Ar~^J2C4$csV5+Oi_9T(J~G6!z|lG415((!6}*CZHBPPn6WDjTvrjOLSG-JO#4i zcqAprkV6Dhi?ewNwH=%2pFzDg;Lol{{`cZODdWZPtYY16;JheUv8uU2K?BN*p1jANtaAbgXr4M0(MS% z8Yu?lA8wTDr(z}G-s$7&9WNET+4?0M;u|4=mz;v~%WUjz4+~JH5SLGCJW7SV?5Vav z!a=ajS_+JoAOenrZ%$I3o=_mF!R?71JTQpaHDET$2<}#(*l`Xv&!%nT1(O<@sT$6F zj1}dk1$!u-Xflq?!`F=`YCPapKwOT-xM?{Rb2CWVDn-Q{8mDJ!2{fmz<}g&q@8wR7 zt-Z1IJ~2vLf_{yDRa0WYut>fcI6vPCq$#b09(SQ;vpS!R{zzh9c(t}?+7gWg#U}2&sdwODsZzvokJu46kY!Xu6Op)MOD}#t| z`MVfo`nZ_O=Y|o>d*gUZ2RK{1fk4i+F*%ct;*9s5(uMQP(ZfwnyP1uwdID3wk(5qd zCmSaSR)=$c?YOkUWUIo{n1P~jc&5vS(9J#_{_RC_!B2`8cf1iY01``R`Xa9lVCZ6u zC7(6-qlx)9H-ij3s}1P&?O$#GSj=7{2Mn%{-QK@=*OvQ3kU#qPar>X=d;m~>EHT*O z<9=YTT7QtnTThn!x8vI1x@qhS`35T#NbbKTMgfwy|5{-CS5Nj^&hRAB-S-=ODDZv2 zHg^6PdBr52I9Bo)7_sRgByXt!usA0Pob0|h zUhCJCKl=B0+0@H?;iR`d&it99()tlg*9nzZ;A^*>*FAI$m84cD{+4yo1~7wPdc0oL z-y__8Ms18k&)7rVUN0Fw;OESRRFHev)(R63aKSbndf>Lu-h8peAnEFJH1QBv6+ZAu zu3K7(L@Vv3&t9@F+13{)BRfXI(_>r#U3=^7O(bydf6w8qtD=nDV(VRG?Ox6)=`R@J z&zQtet)+lUY#PxIihpmrSTTGGd#7UZFuQX2?+^)e)_ldWbNR?mjSsJgzqfb_w01tO zT-+UGH{^XTUH?n8!gqoZN_hGj2=$XAJ5=c^-~$e6+aZ?E00+Z!!+0xF2nI2UE#fP{x^VQgU5fw*ok)r-z* z4^P$(ipKRpz4l@}wFObd0;)%GTu}VpnoqSUL7k(0fJ3UQ1zrrNIn*KlIJy#K73Zb* zsM#WWfZ2TRNSC+;&Ax`cHM7x!;2e8Cx9D$lP$`oOu5kL;npmkxh+CM zUktE6WnAL3yA9St30QSqG}DBj{e_uK_G+2g?aLjU$Giv_1h)}2_*Li5PmMe8MMjb~PyB4-DTG|jS7|Zpov=0Ghx^xw zY&TI?WT&*8;+P!@6Re4l+VW1ByK50g9KZF?;Ud|0`w1TBrP%2rFWAFG4`c3-De)ZhJoxp1h-{=E8fn3n4!LorK{M@p8;Nmx za@`)qI$KdxCEKM7W=PIKgvi-U_xI+|w$MMua6xR+qnE4?f52Q``DB&WwfKJUACGOX ztb2eL{XFfscqvS)0C<8B>CY-Auw(@!MVl6`?YL~`8#9x&3p`sz-&A)QeB+(tOdR>x zzmV(vxc~ccaP}(kuW8OVQR`|%EKe8`vhf!ESq3cwf{ob-}y_I z`?QOIciHF~pZ9)`D0>{l86j^(CSY)kVkuSE*2dtswXSE!jxRTL^9Ce_rjZHn(fyo& zz{qD)mO7J`ATTXDp=8oD?!yCB-8X`pxAn=}A&KhLSE2goRG_dv%N^;>*rQJ%@zMER zYeUWj@ri246bNRjDGl+r=P1pEq?-bMN;oPGBX;v2b38%UWsni6vFbuuv77TKPhN7| zQ)&>WpiI4mMq^ z;|%d#8FPt=c}@HO()zOvkCPqDo*bqI;B_U49Rhj8Nh&+E{g4+3HUgOU^1-$qUZ(%< zs7#PPT-Ms$>-uz<^5sE7=JD-5thVx!H`$=44~XxluT*v)1*xIC=sUEhynPBQ6(>K; zxR&~=c%(LkbvVCT3Ma4iJP-R099sKRhfy`9tN?|hx4+KD-`yQLIE~;DsqL{7T;*#N z<3FgO(hHbx0(`)|6V!Gq8yF|jUk63b1!xaSIUrS0WDr0q^-0kc_cSLw_HRYL)$afN zm+eNNRh92qVpM8i!-Grgl@kD8pSexkJl=MESndak)zw|np{eGw07pE;5HJe!mX^%> zuzZioA@voPq|h}s#ld0ybN|m5_d%X9gndyf`C5!|sLjmVu49qJ;?RZ8bB*6|LjO4@ zG(0cN@Qh#o?knd(IMI%Ls)GY%xeKhyyFaV_g@66A2BzPGdVW)fGIS+>6nXAHMM71{ ziUkXHO5RtrgxX}LGHyADUfI2oPK0es8t@TF)w-FA*q^j3y+m&TY7?hxo$6)TbaU;} z&C$n|-qGtJmfvZQIx{VYzn}p;${j^L00}0~oIC{+)PuzIKL-qYj|=&9j=p-WrzUXz zcYl9|4ZmhA@?wQf4@_D%X;oM3Rg@crSQLf?A+pe!am7jmfHR{EY59A_BRQI){}8BB z|Ca-v>_J)yx7U=(8nbqqSO27Sep@m;U)u{|Ic?xSIaL5{3-%~CKUh~6i3Ue2JN!v% zK6)!A3E`LS88>$wbI4k&4-#Chq@%=X!5gM^07<$gLqNhY)LsdT*sdU4(Kbd~>0ff2 z{@*5j+SihQ&Ghy9QjwQs3Yl@kji@dOfI_%}s|D@hR_+6;gvZV$3yISpT5#W#i?Zwe zP>7V1CUV~Je;roKp2mApjKS=xL1I|Kf62LTC!K^Bhlnd?wJI5E{J(_qya^ILM<~ty zO+IE8$G`hY5tpCqxw?katNQeMS}s0xAG~(&`NI8O1{@g%f9FeNoISBEs4|~Hp42Ya zv^T4mkEDlT!!J3UkHmw@%JRZ5H_EmS!q{4FHSF`%8_w4;p-rXviQKmWgcRy598lv! zrS2mkahHn0KLN$>^P;b@Ho@s1CM**WyT6|Avr)DA8}O?|Nn_Tiki&VYo{5=}fhg`> z;rr$Y?%}>xP!7M?VfZo+?ez^-Tu5Ha#;OMDd_&_(_tnA557%xXhBan2PaIJ|U`79ha3A{{0-IePuLP{1k(${R)Vt5XIJuzU z-#tBDa(5z!zf{>OH373zd}zz!FMe*gt_HAjXBKI)f@eOJm!BZ+UTEkRt=9BS4pTiD zX8|d;dyO&}k~yFtq~sx9mG=!lvU~5#|6ljsR}2>l=+E4Lg#5b!2Wq^?CXIJ`@1`K3w`5{QOIx zx&!l*d?xYxoR2X-q1#J{2!HB5UxD$@;c^n2<7<4& z)j@vWH@`}DwCjA>X6xMWo%xRTC-ICUpO|aE^3*>eQRs*7XlKSfE;=^C(;x`!q4i>z z(N~QE1ci-`7k!TJ2PEg$xBmQq$SACpZV zMq8v@6Y~rg3@IIo;j6~x9Qp-vVxjxW#E!(Uu0J)m1>f)Z^|M_l8gPP0LH)HSVAS<% zGKSu?&K_SW@;Fccv|ipV{z~8Q)PZ}PFe&WYvw7~l&K$0y+&8_WvGv<<_Y{=908pN9vYI>? z-Syd^K7>i&(*ZE8O9{%fvu{~ZHkesb3671Y>E9iMu=vOXu84eU{FL`rb?zL=nS6aV zww{pF`8s%ih_8%P{@tJS_F`oe$UFpFtJD0eKso@7+|!7390R1zZdlK+mc$pu5NDhQ zg53hC22G0g#D>s-w=n56Py$lkCIK^hTZq5fBty6Wy+OEayiI#Ll+`{stFL}P>ORKD z1*Eun&)Iy>;aS>xJ6y^0)EC^#LT-Rh-&1isFE9>t#x!)xi()kZGzqQ_etYO8V}1;? z(d_AlonQPzOOFRR#kUphMaQQ&ZD>FOVK@NWASzN5SN)$+}Z+0dPe7)elNP){c=p_4c?$ z$XZCO=2TZCXc_dEn9oQ4CU>oJ7Q>MDFzqOBhjWoU>8y#ldaSf-xd8#!4mHkZ_t4Pc z+!3^8eA1b#jgV|r&NZFmi3Bc1d#tMaG@|GZ=$KIgSIBtTo_}IdJ7=qr1cGqSLUwN$%jlJQiw1-^>I7_Tb!LIKN30dUoPhLh?rK76eNp z2XE!5`7Xgz;CERIdDMN?ebK8FFPd5OYKsJH00+8iwezpmUKgTDXU!hYdQ|bCDd2Hi z+-W-mPMm<21& z{(sdmlPv@0Aj*%8g}1Z_QoVFA?Lf4=0ALsdUPGB-<9%J7i<4U(PXEQS;~#$H$wjC2 zndY_yWKrbDZC5EEi5#?o3eXbCWz$lo^A0(0yVdfzn9FHVD1x`!0>f;#d{Nrl4f7~_ zWp^||T*1|tS88^5!#onTvw0=#6G04ON{qYDHtYRAn9>oWgiL1kJ%mck1MNQ zqns?*lc_@|h^s)a?Cu?HPv?AFi&E?vvgw!9nUT}nw$dqplzqZQuUpAqMIB=Q<%|0D zWZzSlBxb%Ijomqi{hPF8^>oWs$&kHl?W`6fLwRxR)Ekr{bp?`y&2E&nO`S|^9Il@} zc{5r*GrGmPV6WJ|uRr_7w{*t*?csV@ewz=!s519rBuLY?u$X_t#h_ z(zp9~Gc>2N5mlE#VUwPM9CX9mzck2ElLFS;*yCZ^?ts|-;>i@h?K8VSe3F+!Va{OW z%&zCfD*~_mNVPHoB(4V%5@M!o`K zA``lsJp4JA7x)9WI!=(gG2u&(`(<8`gU|HMP;Tphqj{b@U5U)QQ1wi(4pc&gbuW8(zbx=~)U1VqV$W8N$^s|sMXp;`T@d2d<=OLuj{ zlh@p|wLIO-%E&cE84*gMpoMXA1RHBlkGE?N(oh{pI`nvQ9$#)h3H?MYie^7e{Var+ z31?Moa%g+83mN)jBmM9Ntu!=$FzJ>ho#r#HY3Px@|0G_##PD{PyX3wu1)bQ0vgQVI z+Pu{EfAF>M&RtJ@@R1)x?q(k}&AuuPhS=PVMj8>`s6*82PE-78CEZoM2$?xrvBE=_ z^|reOgkd`B(vD()#!gaI6O}BTZ9gQgoE>FXT<*q0Ee*8|n3}4?42=_ilvm$;+;@MG z>vh{nb6I_r_Fc|G_GYw86ip<^DxS)Tl3^j(xjMA{;Dcz%5N)UtE$sy{b>Mt)ja$XM zDpm}!BK7$8i4TYb@J(q6TJg%0Ge^ijn(~G!bTs?Zf!F17XfjAfif?1(Y8-cpXgAoKExw^&CI3~ zeJGyzqp{hm?}i*!{DndLR9@(h2tb$wX7$M@yLlESwaJo7gi+ z;M8g&lBFCByd74B^eFA_>Xt1aG(iQ*$cCGrmD^N}R_ADmiWW7jqUM$u==2MZvWCDQ@#%zTd!YugJ)|YGL-o1rNV^}Bwl)W zKM!M>C3wUHT&SEOw0b9Jzy@aHDz+UcMoly?nM z(N_1Mf4aj**GBOTOFKsTey+GuN3&WZ?#{PMhUc8A*sFU$@2c*7%$3G%iSe1MWkZe4 zIGAT*`woh1{x$XzO{dNmBzA4%$=l|(se`-#=G=vs8yXASjALSGRqyQ~#*L53Kuz1c zibF3&guAqs8;A>H560X}6ozvCMt-=<)`|U7>PVVa^PV;xCgSE5hwJ*C212jtI-_H{ z`pbdbM-t-oL=eDC&6Ykt!axZ&7&)aZQ!Hc9T{%55z`eY*o!+EC4Q%mHHFi$DYnwLEI`7j}XweBNduc3f7Jclaq$ zp}f)obBa!X$5uG^qn31yKKJFmqCA6#Sgq3Mp1RRPhhwLD1uUl`LZwXjsr+H-oYo{` z0E0!4ePPKI0uQQJ%Jd;R7Qe|_b{x1n9qU5x5K{Kv0O)Eg*pbKumSpun)D5~|b^TsXLzbM#g5 zDq>DtCIo?zm|d>&&cx8$p{{u|qYq~Dy~drJ`FaSkDFtUmn^cmM%Jpr)VWPZR&@?KY zKjr3)-$h@2Ep^GJkTyF#7G6^d+uez|uW*sc6L;sk77FK}lcO8LYcyR$8r312MPOlY z^M>NMtsa+*nC+OoiR6@5w~AwIT^P>?{QgdO z_D{KV5A+*uTgaz2{|P3>>_T4SKFk{<09Cpml&ygZ5bHvVE{13rja=YQ$7RrJmUGK@ zj`78?cC$Xv#N&q%J0teuTbKLCVY1OWq#fdQ`@DI z&|KBo71;veQDx3xy(f}cBscji|18K~Z{dOg`%ya&&Dhv0d3d`bjO zHCXeu*;Uv2_i`t$3z!k(9@Ri~O=4K#(UQ(WlMk`7J7(CHjY|Ey(yqF`RqO;eM+ML5 z6=-!mw~|WOt?}V4(Ss7-k>5(j7I+7$nj|oN`}<~2T(WM2U&6%tMdCLLBtN{DXkip? zCPBJ3Av9;Wt56%^l6=zHl0)P0U<@WV0rlue=;EiZTLOtuaqtf4iM|Qv4Y~E|!yl~u z_+4L7{*D${>e5jqsx^--A0_t$Y0A&hMAAH(w)X9s*#6^m3Fn<)THI)d|APjK>e2GM zcup&wz&A$pH@q0GTZRs#&bV=J*Hp^Rqc|5K3%cBO$^1jXCko!ZV_fP-wiv0#V`^Gp ztbTjH16LdRSZ~eoSs^Y&A*?lXo1~Lc@ihL|4A+iNk!w2=8T}JgAnI|_edp>wsLj^b z7gXtMeV6>5iF;EC&SQ}cCt}1jP7pNK(3YRA$;UYCfoVM6!t4!!xe)|O z{ioCRsd$AGP^B$mZ=^PEG+W=W$Cimp6fe~`kJA>AN3%7IKqcI`Cv0JYbc_R1z)~55 zCtKaK@~`#khT(m)y!i-4ei+gx^01t~fRL$g3&28$B%Ot{xQ`(#h9TY$RP++!0AHxr7Y!mT5>EkCs`M93PW50bllB*fy-Lg@JXQr2{o2Ldz zZ{i~RqcTr5vN#Cc>CM+>(amx0!u^<2QrS4HIk>@sXiR<;=|-S%u)brjkIUVmsaZGD zQL}Kus+)KDPNdvBGWs?#x}WqYA<%I0`y;Z2gcQP2KMTz%V7)xfz>zO3hYM}zaETg~ zGT5jOM=~u5i3_>O9vR{Q;e0k-XZV`!kT&Jd|?NEBsMSM2W@ zS(r_zMpEgWwu>>l&6kc;x{iRmXtOEyz&3SKF;(1Ip?SENljxKl`p(n)gtid)Y~VZ=h7V9Q+BkN6!i3m)$5pv;h>xDAL)6=UI`}wbQ3!Viy$&(I)j5f6F2NSBwQ= zW(Z>-oHdfxf6L^t?+!7=6h7$|V`34UrBkdEX?`h_Ior3+1oIa6WNup?2)>gL+j*Ef zaHi|^maq@Ky@9i~R--#*!>~;K9@N356UDIuC2@x;$F8?GR{^z=wP*XlRI!Ab)So8N zKZxzl9^@xs6jlDdl|Gq>%D$hha~Du2>MJy3u{Ez~mr|N|JLyH96$AE1 z6aH*JT?)betdF9O)+j1AUfU4xMts%*JVt~lUOVf_`HFjJD%Z-mfl65i`_>yQVpu?! z>9;k57{6QtxQ=4=xNtOp`uvNZsXE=Cmfxcd#=YvQ#V75d%gfTh8xUeGwc{pUQ+;#&) zvYFJI$Yv0I`Q^v?R`cABbuaCu)s-bPHFSN&Y;>=BX6EzFLEa2p@`zTsn2nI4&RLcNy<{8ZhoHt5mhe16Q{lyFpM1SPiE@TUD2qojYTKh?$I+{+WF(4zHm z4YvD4P#}4UlTkf_Xl%Yzqn9lrnk}ic;GksgVAL$WJ0vKh)K=&#G&^T@w-;*%vJ&}I z#f>$tEp5cr$lrT%hoEB-F7(zZWc~Z%p5{#GZ63=#ac|__Fm2-!h#&_GB>R=2f#I^o zt=2=MuwqjaJX6#bgALu+6MM5$hW@L-5Qo#mYrra49IMH6R3C%RN6TmL@~6I7Je!)* zg6;&VIME8?n|mW+#e+LD>MLsM-I>Z(LF+5X>`)aBYGt9LEq+5Zqa+C5V_$3EukqQp zn|g@a%5t;S*FKm@Kw~qK8zcHDoIKpbtSX`y>;a+_3B`4_hW+!lN2;`+v_)Bdl7c)r zBZkMw&f&fxi#c7tvh1^%7Ti(IjFbDIh;eX;CCggQi0;GKr1#6~vV|b5rv0yTdkmiSxAbLnB7gWYx?SC$ zFHC*-Yra!m6*untyNfatW%{L)vO2S`J*0LV+{k8dQ@*=y8%#|?>88Vsz7Wt%Vmw^6 z&~ua&2*$XL(Za-QyW}~xTh<-Ye)1B!r0c`=yYF@VO!dKAk>uai?X9mRF=6sMW%!|P`Z4axF2?!z-g@^wy};X%3V-ZoqNPz{sYN5_uFx)+fIbaU{V z?0U!I6cqvjcJZsV+p{-U*X<#>@hA6@WzPxMK%e>nw3axO~ohWHhF z^8$PehrCNn-uw!#qmm0e+2U)l#gr0oZ#Z5unSB(QceHVeF5ymwn-w{JQ`rY9{b^;p zM8=~ou-hM)*^7gDevFa;df(Ksy(wYFz{_IngSV7Rod;73;Kr5ygeH97^{%_9^%rQ} z9Yt-{^wNp=?$e<{g2UYs6@DAfnzC0LNb1tRqC2~KvTG;rW+JuB>^@SbXgIFq-i{hb z{anMon^moAU?O1Fn6 z@f6Q=xOBmO?}MZ_o6`CRrs`7yvJ=#K{nJ>#jArkx;)PKggj_R=sL$kDq?-~`DxFq0 zou?2d46U6lWteShU~)kFI>Is=+bFSfC|_4aRqJPlf$27@yN;8Tw-KMHcH?Z5d$U&? zrf5?11Q*(0w=@~-xEpGTDTp=MZW!gl$^0$ZY9`ba)6moxi7t!j2M zrb91qFvK-{g9d7w>m}}^cDck7e>B!Nn@21AnNT`*S)#mu6rRG#KgQ@~ZbkD@$v8{R z-EQlNfGy{)Dlli{`+9C2_UQvXj3Cu712e+J8+P!igzm*M(5wLR>jZZWGR77=fG^T@ z^WAKw%sCss{z3h%{*xU6&7~4s+0aQX$=RBppgW*Iti)mI%AC;MMY@w?@V@SZ3Y7S@ zE|W>o8SX7`r4~%KO|VyYVs<3raR=0DaIy9;X)k!eD-)E)cZ9FEl)_OUe9MsUsT$kG zwsrm96IO3K2uh_`#?ie~2Ut8;Ao>rr?TYPeCe>v zMmWKpy>!$>0khF})C_fN=dGUd>nwaqfg-voH1De3WYW9+7x3wLI|ctN2=tyFF=E zQL`_&R%1IUit*Ei?#_X0aXK|0t62383tnB=IR`fH4B0Cm`xSz*1eXb~bzdT*HcIs3cI`24*ZUI35t8$SRxlp~)#Rbj=N|dy z?Z<^>0_*-J z(V3wpSo1<)YV5`aeySl+Aw3O7R5LM3d4zTew8_|vjO*{$II@n3amm_~jTuQtX_~qR zWd9cOSqScjeZ)t;4xCRh1@6IfC}8Q)=WbG1X_2+T(k%$h-A##;l}*1SZmo>qC>J=( z7n_9C!aV_9sh$fA4vAiOy?Iedq&U(^mwKZ5CZze%Fwft4CJv-J>&VwBe1xe#TQ<`x z5(m-vg5l1a?}lR|vRmXO!wm~xG`Vg#1;UTuTT zdxx+%soL7D>mOVtrqkfA1LQYa*7W*nL1+mtf1$G+>}!#7Hl~~sj9WJBjyv1$df;sS zpSmoay*;S*=;9*p5D<%5dQ+_?W#;j>>@e;wSZW{~|CZy_{#k~igW8?HZ>njkeI;al zHcaiaooy{KG~ODJFk-uA%y%sL2d0+&p9*K*{`*y8x@CYo}&2ro?y-lJ8PiR=kS91aAG;Oqh+7$48a&vZa@~Dt^2$HuG z9)JWeG0Ur_gm|seKbUC;PM2xv`%D*zoHM5hK;#1x;coKdyGy2mj$e^77hm*9l%|x5 z50Cqla3XR<*23j1@Wo#_gA7h#hL1>$LUy0b$#3^AE;+f-ZZHpw`VFz`v|{z-9mm;O zaF*N@w|$3LB|`q4T>lw(moaL>BqC)R)iLxeXl;p@luBWA=t_w6arZ558nH092$*#y z>K_nLWO$ah3xR(JKlH`54Mp<{dnd=4yIfU$o*^o*^zRn(Jn%JoH) zS>G8PmZ%kcdB{yRl0M>a1{eD9V9l7W!pTDun}vYX!0Q{XRpJ?~iuSpI$aYE1>A$#e)V+zF zaeQjlBTm}|7%{sXaQGCQlDQ`yi(bQq+BtIS2D*XXB3p_{#cI{yJa<2)M3izDcD^iY zw|I)S)n}{EP7!l*1VfUqP;ux%R7jJob>PAbc;>1`4*OIXt)VqdvOZ%6yVcAaq>ft^_uSVUv@;QR|_sk?)Boq_q%x~ulWr&&2^8(jNr;Ff=Ovyw60zu?Tj`e~*+&_}FrW|r>%SDg9uR(I)snZ$Q*)a~B2 zbOT(R@HMtOj8^POl_rn1?{N;bE@t*O&zhO{>u#2JEV)O(?fi`bb@2&gfvQUD3Z0`s zb&kFJv=P;DB-f^4!G&SCfQomy*%l6=5_kpwL6N(vF9wG=gei?M?8R z4n3UP?h@H5@DLRx?>=?)$evXWQ{R~eq~A*qR)1$OuylyRe~uG3b<)mCRNu^QZn!`F zIbckpgg-C|nf6y!zPCnL?l_JbK+kt$3W+0N!0%$UTX*8^sG{4V7{7?nM=vnZCj`}A zC*iEr5ZY1U{)y7c63t8?a@(Z~9wl5HQdXXp5MyXaZp&(nyNyHJIWfFmi@kVGb<`&? zy<1#7(;)}UE1WW^za(i$X;L<0?;cn9mKS|+V0PmWBgXjPOI{_L&^qQBF(lBQNhFG1s_Ra=ah#DJ61;qrxwjY6 zck*iRsGITK{SDB2LAKqoB{B=$@E072@j#~!dV_A@yE*?B*kffyAJdMWFI&WG9nQ9%GU4Ib}+D-ReTwRC8 z&zt_{6}eAiX>Ti4tf@Ln9bkQq7?U-tI+(}uLqSO)9AF(`CZn2l;asi9b_qH;ymV6z zo?_ZxW-q?9^$%vKDglr%o}Xw%$dw-M*ep%0N5r+3+O(QPcfWx+^2Q-zafgzd*%)*C zys%JxQxa{IT?E-L<-TCGGAb`N$I3}|Mw~4z*5>mC9uB0yCn@vQqLS1V zo-2g*k15@KJV-T0YaV=9s!h5O7F_Zj@NE}2nLfSNWB>fm2S$e8#V*wAvLI}K#$WDd zRS1Fg4{@wrF~2r3w|FW^cO+OK1xad@VG4ALVX9>WM=%^Q3LBBhy%cdTQVBChY5PDEGv85n&!zw9m9KV!X2f#(3eCYkH%FvMS^ZWGfrX| zQC*RIfqMy++PMIF^NW95Hs9SZ-wrqFHZB#Wd2uC44HY9+MEf=0SQGhE1T)%{lH6cq zLo33%5<}3iGl*r9(YHB2&Z{ZJ>m!t!H+JkHZTid$SueI7)_sj9L?}oaXI3`eYrKLz z5BI_AT4%Jxxa0A?Lp>buk*lm3|2${3Copqj`b*8d-x1!6xEv)M;WYG}M)V)DKMK+} z^#*Kqt)5TkASQ0lxhA4Bm1)p;|ML^Ss!x^1&9^25eS)Ux@E@r3*`k@{hEn)+o>e!< z0_mVwm`r{F%Zs7S%%&u#$z72X61MK9wBJc%cBc$0NX<_Cw_;iKJ8FO~xhAmfM$e|P zgU=zJ+rCZJos~(4nU00CH%%U2iALX6DyNBtD^@lG|K-$yXqFNy!|zHfg#v=K3N2f^?_!gup{k zmYq9@CCf<4dz_q8K}i>9)fAP=L*9B`R*qO5yztfsB-NV2YaqXjO9u%nMYikLF3ExT z_63IpPu;@k?~PpXdJ`eki%ZtEp`@YEEM(0L)AXh7+9xpaAbFL}m=N5mjHpZDxF$hL zRH+k^gt$kxSq`O@H*#bUNY#3d>7>SG294|T-_)e+0@|vZ_O~Wb69Y>As`gUZ@T}FhT6yfa65e`9IhY zGb)H6mxQu%q=L7-P>Bt0ruC?<5yI7ZcdU}b!cb$|<}%qqH;MZaE%_=0V!0*t(mGrj zysvKTC=~z>@ z%+kJ-cpg&@8yq>swz!+L+FUfu$#O-tSTFZLBvH%}*yNZ*>@oKW{HF!fcLJiyq*;+8T@`(r@Y zL8MAXX0a}63Ogz{*nuKw7=yHxOY2se5>ki=O3@C*_r?7RutRuvl#IeUSG#V7$@cF8 z3^O!Z4xXmfjsbimDwOSW8Nc2=m$uOm-IksniGCFj-~(MBXu1%-*)^mhcLHcyyz*_b zw20(KSG^Hj=2QlFe@+)lv!0d+qmx}KsG}>bbBq@%p-`_MfU})l-E;-xbmw3lI269y zm{xD`;kND;EiQugcWCxpmvI)=#L83_EpA|2qs_b~H);?;cEm+GJ>FCkH&H)lL5)te zbcjMN85v2a23BmV`$JNCXte20lgfj5?bY1)o%8c*a4%w2r@q3Nb*&Gs0B~ zg%!`+6*3mob0kU3HXnJds;FWwP54fp)N}c-2|{<$Zq~RWfA^VgQMj`-`E?$8Bl(Za zw*;+dck}M;BIX?VoGgyL_>R+h6x^(jOO<{f`MBQ-AiN-O@@2Lk-ngrC9JH@Uuc1t@H)oaYg%S9A){8I8{%9 z=GojyuhGF>XK-<8T`S`FmPoQ_{7P$Dm%122H92Y+j;auSn69W>I9s7!4)@b;IA{fp z%Lg&j{zeABb+k#eemFw{0rtno8}Xs7&JTOtWhu^b1+DFs%xl1y-lbRn^~rww|4PX< z>F);|zWdmY`MfUb%HT@i_4~EMa5u^LAOct~f7+hYEdA*L zA6~n;3i;DbV5BZxxchqt=LedhYql#M!@qc44kuus-3oOk|7{esiv%R=Vr@!x@R75^ zhLz7$-L~S(Q13%Sxwa3A5ylI}PbDk_0lCx(*DUqQ3vpfMoy&Xide(xgC2m1+n*LC6 z&uktq(KY?+;6K&O1jKDE?YwKd&2Hw7h0Mr5`O$r@5YOC_>Rq8_kNy_%OwL2gMZs!P zMEgtH(x6Wij{R4TH}McDYa$ek0B#wW+kVB`z6E0IsiAIWP1p9%cPVJGWo$x|WnM9z z^CP!9{b`f&VK=Nu)+G>yc#T%^J{uYH|7M|ZVWGBlEL`_)r(!XmJn*(x)7-zApdjtc zRrl08c+bYGQ&-xIdp}b>vS|{kt)yvFG5lDGqw% zo^-g2Tt>gHd!TB)D#j1&P)cRw)LS?Uvc{M+Id4r|{I2Yl@_j{BxR}i^-p6)y0ksuH zwUcnVT(nS*=!O?*<fbaNVX3ofv%yDF@!P>J7O!npr~Ic5oTx1^$KNs8gp zI$yepdV4(Za`TGb;K9oZH|UTb`POe8Y%;y|JzSTT6Og`{Du2oOY|v0e5Bn0z zX&gi~FGm$g8lK%2PS#INTc1Tkjb1*w_s!-V5-`pt9wq6zSxQxyYcCNXEUI4XK&KmZpRK$D zlA|EF_(+0~GZvW2sgdPhYe`CAG`raqvWze$cyokRTd~vaASf%r@jKsS`wYViGY2+K+8} zTH9=Y?J4O1WgfM#QZd335+JB78yxr$vXI+F7+B}|x|_o(nMV5M2^|`?wX;mN4!iPk zyzXHp`{s=cLLL594ql#)&5GQUl(?gFRdhFvJ%ek_dr1l%)qVfZG@RPcYrY%m;X(BY zpkcR5**`j5-ds;!=)p>L^b73)E{O<1NS8-g8~{OmcWKQgQnA}g zNQJ{)Ked3{Y}@{;Y`ycnum}t8_2uCi*T-ELY0wz^lk;x*0q^~wK9J_#nl0zfY!Ust zB1=nTBInnSzMF_8|KBJsjZ| zQFuBx#JyGbaK!)0U^y&daOpZpL!=Z*Pu4kZLghn^GS5?Dv1M!l+OUahy;5}3N4WGIv5wM-siZPj`lxqx3Xwz-Fr;V`p%irBxGrO!wya2v}N%X|d+t5Xogh zTcJCGa%}Toml7MY`&g?uYE;(!AmvjC?ok}9r0-5LFGYTRq#6CEEjKFry{^I;>U(bz z{LrS8!Z^D!3Z;TAvVF(>tbJ{?fT7-*xnfpNuW4yS`A*1av{Ko0mvz|Bzyc0CQ!48N zf5P@u>E|WnN)O4y)tu@^sp%AeOCAw9(qFZ#*6Hm57Y+^)_XK4>M`fy3Li_PhYOCpG zdjUS)nyA=ux8k;*_t1M9mUUm4d+L;Uv^`?qld|=4=RV;s?iFi7rk+icWmrvmpd<|0tdH~decG>~YLwpd0qW<4z1 z%Yf~;V=odfCP)ftelER^ZUwOh-~dZ?kuC;q1bj8kP7^V&)!Km@#&7f(rxaWv1A_~E z1IgP{-_i9r2X8F0Yn<#Buro}-pmat46bA#!Ue@?_fE9<+n3&SH#eNcTVG`P~eT#2~ z-2-O8*K032nJ~8e2Ps!?_js2FYKh#@#ldF*p2IjW_rae&u4FZE&*(u$1;?p~(Gr^= z29?c9crXh9JfGT0yRIo(5x*@}r~^f0-5JYf%|7`L3r9-5mGq`apyt*#u49jx0GPrH zt!nMI`uhgQu1I_6E7;0qHS2}{i&folica;0;1?nzvhguu0mh0Cw7=2Y**1%HJx zR>#zI%6xMs&7@7)u3&D1R*K1`Fy%I;wM|Cj8l#9_ZT;~Rb^N65qHK*mMhmqZTuN)~ zp|Re(P1WpR+~bx4n=bA!ul6u;27w>`7Z-r+W@+wqZH5+7nv|Y!+#MHORdjME=*)eQUZ+w)73^JLPX;`1|1Jh0)eyYVQc*6hRLxnIfP+DL zs6puubiV!&Lh1%sm8oe!dK20YEgMxKM~>u+lIQt7JW7i~QF3__C^N)@%iFT`o)7Ez zwr@kcEIF^D+$1;I1sc}4j2sQdL5@JYX^%fVnV&MpErvzI#FLQ@eM%^rsdcrgiz43=WYi~T# zAQt<$;VJsPwLs~JZl|d0dAqjiX-A$_uS?zltCZ{0f}Mn^fUKf@&MJKi zVC&#tj}H4%@>fcCu-h{Yam3e)PB$@@7ChPL7c7@iZWzVevwsbeeqG92x$*NX^?>-! z|K%#$=I*tiitVegR$zubs)5hY;V$D&x*ATz2~b5lTE?sK4L_E!7N=}c*!~Y83v9qL zhHI&_LnUsog)M)!f6>M)C8P4Y0+}f<(bdde#FN z6Sk!qo7pmh7c7?vN*l&nAC;>Bwk?sJv)d@1Y}YR?{P25+yY5fB6|SeO+!yw<`?jsz zhXMEHe3d@Rx7t1f0>39mVSRTh3B)4Z{lxqFudItg?~fCi^%u!sZMVGNowiy~CBp(` z!=o){XYj)p9a?_xmj7LRfMwI@lF%%;Jdtbs+9@5gySOQ`J7oOvBR^CH8ou7Ny!?X0 zx3gU914qtdl*BJ!DVZ>jgrOVsuxZ;>*k;2`dgwg zM}6p@1;xF}`ye9r?eu~3vLCh>5Boj*ySrwZ+fq=6I%i6YvRYnOZ62}c^DBL~yse!& zm2}zL;qGPK@}}>NetSqQ>ZVGweBkm#KrDCZ^gtvY?FANw;S4?lbDA7?tPULe^M^Hob8C*a-&j*tYx|dPN>xZOpepK-F^2T?aIUVAkANi4oc336+4a}NHO#?-%GqPSvN84 zf^`@Pkv7&qt3!%+`Jqaz9)D~FJA&wO8s&lP_>y(*nLLNM{>goorH(S z**==tc1GcU^{+8{bd+G)h?{twS~C$)tuJ-!^1> zL-p5fp&L;LKr>7@KSukg)`{m0N>>U?Sk1i5a?s*lP{FHp_c=}xSN0ydh*FmL00an{ z{Vm?uwrgq1%3gj)a`MhZ(DI4RZ3rle37oV#qMAVrin7Gge2LIG*%+Wa!P@%K?=Jvwa^ir|8!i58sNY$4oGDxYRUL^)!2qw3>MiJR*SP3sf zC0AEWnBChj?*gAs6Lvkg4cev~<$WNCE!ybUY0#=Y=`T6_`4l;L;Q}mNdT1mac2mNP zqyQkNUp0+^aV?x2JN&Y}%t@H8|E|KBLf}d@$x7eZcROv9JdDd-N^!6}RNJYHV$^K* z*>2%~IpO28M=94FS>9?A64aFlSFX1bCGVRXAnIMYmrBLj0@J|!4%waRM4T$W22|+( z{<_$$4wpIIEdLx~?0=t9+ZJ1Ry4fxYV5SMQ9ft0HqJMU2*LhzbEi; zCMmi$?3E9b+I3H*x#}*?_kTu=b7}kl85@cnD3kR#8B?;ZJnJs(Vb0ahY#o@xuR1t1 zJjR4;n#>ay#AgwRB^mkbiA*E|=t{Ji;v{~U*GH@mO^H9PA@=L}52r>OS9&oZ7ZDAa z|E_E+?0%@gIr^lASq9$GJ*3A4xG;`0bw2#n)Z$4fSIcZkN+Dt@3T8)m!9Qqj`Q|@t zmD=A=VEJ=jc|Thb!*2`E62h~Q7heK06LVSEtFbZP#Id+$ zLq*yvwtZet+CfjQp>z1SSJi_e{JWieL}d((nFpKCnysIRlCE;N&Rx*LcEe#%>nY|Q z+jknd5&#>|vNJyu&a#a6#)w;eYW$?r&)M%YZVK^1m&(E5kRs!T@!g`KBaLz4Wj%cO z690|K&XbGwuvuDiwJ2vxqhG=Fu0j6?tH%rs>9;F(SXb78(KQTCLSe77HmwjO)&8}M za@waq|G z!EWOZlQf|jURXXJG)zrzF10Vq%(*Dog;Lv`VG*uU68^$7M zk(2`^mnQ`O{)6ePr2U|=>_^85osO&&3;Rwcb-GZXJ|W``EM(#yC8hRtt~aFWwB=5V zXUT05T4ixx8g5$SaJ>U>`Fi@;zGt>LeW*B})R5=wir*~6NIbVoSji6ur`z{Xy5qSY ztAg4jZzLH~d@@KI=<2SZcHq^ zU46O-fYGSaGRKkv(5N13Nx~L1=W4i)=L6P*?rHv|bqpWZP2&7vVFs^%V~SxA@3S4Z z9XM6Z;oD+S(4C+|TQHv2=+!aF(ItmZh8Gs24HaPxqMQWaQr;v6(QIFKV@HMMT@{YN zYV0c;h^z*Wa4+cP_m=N9{VoLEt4~Wx$r!{ttycwqHFntoFvD%H22yky2NKlwfCI*CtR>d^WN6nHN-lw3Z^zgb-R|uYFq1%$ z39W+1RU4_Io2_`iKy=klJvH>$OF@QBSTfab`zia5IHTy5MIr$I+Xi$TrDSnNur=xz{qJA@py@i5v=IEF!@ztZ z`Rk&`#l|AtHY_VvaU;qYm`s2ymW@Wh;;#^5VaWMHu307tjQAGZ7==*R{`pni?2i~X z{~}^APJ`c<9x6R|~`1>_6DRI9bLf0*_j~v)Cdt#pi82rc0yU}|-8}3=NGIAht zRI@3LE*QugKh^2Vs2~>=A!w+1cA)D7>jrl~!H{3d3&8|jjknj3u0OEY3EC4>_UOj| zM5o%-ZJeXP{X+=%L&0IemLZPK-+AVj+y)dw?ymm^lnm8n!|F|m&o0v;{o0Vv?)iFD zmDui46t@H3yS)|&n$wgB|L*uW@#N){gNCHm)h9mfG1S`|v(eS2OSqmlL>7|9-3Rm9 z`68+43d_LU1>~>cpimaLsVKQ!CugpshxjNK(!wo)I`{wbU$dW9O1)dlW-GJ)ao?Cq zZJc0+(IN|rj~15fOZ16>f;k24cBHk-@SD~ zlwyLL1JciqRP|)&;QfieFbjH=B(9v4v%*6_kg~Iqz$w*XFTQt*5?1Ufs;s}6*)t4h z{U&ou5Cec$nFTk`|KCKe|1pNPkF9UNGh<4!!_)r;xC*9X|1O_T*VU{5(!6=!t)j!D zWuQzW0@*As-Q@c~v6~pxiFyWHUH)dJq6Y~PfI$U19%D3vk*ge0hg1CyP1G?rpe}eu z0Ckwh5Q{sP&(0bxqNemYI84234=#GJ8U{Y}ucSv9-{f4uIXBMtICU8(fFb~B6g)s( zZG&!!4_V~Rxc_)bwSPD7@}_kL-2%bNU71lZ`gleGfT5qlUa?ZGAAgXgj|EQ6Yf<~F z1hZw_4t}O`j153x(sw4@s`${^FF@P{)!oM-PtTwIiNe~)O)1NkgITZ2Z()wTYNN<* zIXO5SB6QitI?pTIk`0~vO15&YEHG@Z-<#sc0>AC>bTKa<%zQ$=pyU2{p?bH|^d&F# zQ3K)cKwd;c%GDjt_GbhygQfuvyyB%V+^i?x1@Ssbz{Y!cS;ywc)-RC-KZQob?SHMasY`9H$UyR9g}_tGuG zz(ZT8-p0hPdnWPTBK;l&c4O~%IC1cUonsbBCEwJL3nE$U7!?110)y5xyf^f6BgH6K z_lt9xKX@aJ73X@sNDKuTG?u?D6)wx1>n4iBXpZ!ZemQIB7F|cw*^x48_f6u_ z6_m*~5I2yt1FIa~pWs@5czE?fldBs*XfdMkYT!;7enV6^i5HEROIjU$>Cf5KSeWdG z-nd=`_sP>_CRKGzHYVOv2htZ}BzpyCz?cIA7;})Wc{x{sXLOVTHh@}HWo}?RvJ#Hs#yK|aM3y+ z`a+NO|3Sn5WoUTa(mcAAM~$?br+|5%vErSQII=5svB15&O-l7i^&4$$V^P9r5L6e2 zAD@Nvpnzh0f6O$^1Hwgn;njtCD>uYMg4RrS ziN1g=e!Xet_YQk!xYl!+hh?$pOD7Z{RMy_|#x;;)#q_1DoJ|nI{9v^7uU_J8tbH<% zBj{=Tv|2s>F%(-dGLCBBL(n!%^d?n7B={IFy*I;a{x=TJXMy%``42_-g?n=k-9GM{ z1J3Ba>!5uDj+L~{lGw+z3Mzp3T3eX=DU%ojPO{4@j4|MS>+;CdFwUtY z#&uu}J%S1&-vv0`~lE&B*uJ$mA^O?Chc41wBoXg%#a(fE5Bz zS0~GtO_81rYguW*%b_u;JM1;bLj9%YT;f5w|3xGS3!0s>a#B_bv291RG6WxdPh4W3)NA+Qki3mQ9DCR>(_(__ zq>l=-r&}|rVTJJ%O_RH0r^`*K!rLIWl>pOrR%*Sfu1#KHHJ6v~m)lc|T(JjHl;T{N z2n1Jw+>vp9!?Y5de0z2GzvUpWfOGdCj4?_$Pi@D$>M3H{Z(s7b6rT_! zXJexja!)Cj%`OC}%LSaog5^I2p|S+le}QmAVgNc4yL|&f>9q1O9*D}cdGwvU$ao2T zGFqz%Hf^v^ZYk!Ii-$Mqn!r$kLdWw&WXD{&;}+d)_HNr*%UDqM0k4t~uosKTRX>Gg z7tgN<4AqWSS8TnYFuCv)*^->mzbo)EVKj5!6` z-c8GYdWbjv{xg{{egkJ|=@z6~V!S@ky^ri2#e zPudi_1||o&XldV*u$95_TY|@K#X2~=uko=Jct@m?KkxS56ja=IXNH^+FKj+*vF2vCo zr7GKnoZ67wtlJ+)V+ezyom=fe86e8<>r!%E-}@Wh_OiYA6VUC7!YIBox=24p{gPi# zwyhvrRr7-isa1-*Lswml|p-8gHvU3s?6rPk;0p@kf*`z0;4~!D*qaQL# zn(8XkS})d-R*Cy zzIEWq>L*%AP(NPjAof&=$4e#ECwPzU9E3Zg<0K3pbB0l}NWkWu(ORWZ@QA(lV1 z=0LiK)2!YOF;k$Dy0}6KNTE7HLPLwEs|!HM@7cSCJKl)JrT)B@Dq;GbK~5AJ7>;1v z7i*H(?R9&e9iR@trRx;ND-1Vf5 z@?%$(l3?KLN%RSX70m95Pe;44p<0lXd(UF8bsqW)}WknS|3msGZNLt)dkAAOd!%YKO2(H(~0rSAnavknxnIP;r zHQGJnS9QbJeCo?ox7hldA0o0ZJZkIPVT^qX;#(KT*DP=Usk-UldS6xIMm9M(_&(j0 zcc6V{+m`8YXwhug@pLg197DO@E&oUT0l0lVe(V!(`@x@V&rT@3{=(>6Hy#hGT6yI4 z$WfA2FSc1=4iPDCzrK55kj^e;B-nbc2IoUM7XSl0utQZbL+9) zl1l;6g&=yVs?ItU8k1AaAkIydhgTD0)8VJ< z?j*QRY6jq0jg)fL6e)b$rTW=?$y_(ER0U~O;jgxVX04%hgi{k+J432Z4#xjcH$p(h zX~faV_4OR}Q@{9l2m4G!tC$bB#%y(Jxl_`B`mTE0jA$>TijK z#xZvS>!xFo{*pF<@d)}qvJ&noEkeGj)l4l?pdf<59!nUV;zhldx-=W_)Wq~Y@ddsBgu~l2OLGtZ6trhn^jvYK#=mtfLZg&ZARlxr@&SIa~wTT4OQVC`+pQB>39)*3`7)u}K+ujYSWVa=A^ zJwnTo);dzT3?;~L;HI&tthlRqsSbOTJJtYi-wr=*X@BqHOBt{T=0y;I*!VUSbT1Qq zbl)_aa+Gwx&rOHjbD7zb`L5%pA^8Av?`Thu;H_a5#oh0+y0%T_DG}EPe0Urfwtx#N zNN>@cHzO0=vL=-t&|@&2R=Z|z`-{?7Qu#LPJZ$OMgViO)8vSj{H>H;^7>P-VowdAD zBk^)uEp-6$Zl}CM^1gfzpMFl&QI2103JPR(B)4Hs!~l&b<1;$?h6o=$mnpnyW;gql zTvqcyj0CoEXNI?8IqB%t8YTj~)}tJ5hXSYVvrw^ZYPMjLS|+6(P48_l$O)E}fH zl5cES`oz87@GFn}zy2ByLq(jo(;Yfm{2Z%Q4Bhf<8r0tVEVq?=Kqd*O$D|^*Pc2T- zUU#!=iinPM=}QZ_!xgH<(EMu%^E6D*)fZ67d+aF(1OR zDLDKKkzSF8KD72t%S_(Y;Cq(qE*6k;0fk7l#Y|Nlu;8*gPipvNRvsUX85d+rwN79##xs3rc(!1{SlaUsyS_Az6PqWB!N5E#HTOKQ$ui|ixfv2;dasCk z2TTrVN&y4-Y!80O&<$DpDgJyXD_?me|FxedPtbFVKYxy_cyl-I z9eeP^uF@OvDM(q%R;Wpx4`-4~HQn(kC2*P$2j(zLbAU`532Kh!7za|7d8T$t%JW;H}NuOsQ{MV->@B>*?$Fv zg!4q~c6I=);2QW=@f7ovOjy%C7fZ86z3jp!6iYPC33L$X#vQa^)&#l@?Hnpljz+>e z?`dw)3Uw)z+I3MMnDW>OXG?SqejOYXzMG+NZBIcPh|0D0=>`R9F6oMV3iGUHM2zq6 zI2mIYXt`EC{&`1bVk|*!E|iFkzjonVvABb4=Z0*Sf9D4CLK+BjjRP7#HL$Z~A4?7| z^kqY9uK&W#wz=qZ$WM8dTm>>@$TsuIwS}@?Ao~1=!x}wH64CaMMq-hE*#$dtTmB>& zV<%9S3rpvLVw&XfU1De#u(R3tugw0O5Vm=*M{#Np=E z{zojUdGHg7_$p>^J&^w-Th~V!cqNyv~zqW(rPEOXOme-x=?E4HeW_ip$-*?Y~^Y zWMB^mWXtFB-jPA~fF-aZA*;JWO^aMTYC7UV1?}!Mh$p{IRo`+dt;n9Y&M6$3_3Dg_ zW2|HE;)+GU)SR6KE%VvF__FjGRcueNWFtV1#k>G3*twi?B@w=Sl2m5fXAZMxK`KaK zOyY0&r<)Q9`4othzaAEky`oBlpmiry2z=hDN6K(D?2?IHCi$5%T+23;(C}NxSwz~m zr`$28At@}CdULhj(q2S6uc&6+%xuMZWrez^lREEtO?HTuXfAQC?{+eBiM*m274{lw8ZqKO7XSPw+4cTYsKpfF^sWXj$U3I(~5 zrBCHXt~Xom{VvU0S!f*nNJBx4P1EKKx@jq0lO98l>ynry{OvN1r=+SU*qA?$&{of@ z4LrKR?}5Azhas1Z;Kk;esUzLHmYs4AYCN_;;?t!XGsl+6C6H^;xa{KNcqZqDQKqj9 zn$NG9>8+{VIg-`P&Ye9G+Sq2ySNV!f>+c$-Os^lSnTnETeP$y)_)$2Dlix>yj)Ly@CDDU-jvNI9`PL{Z@`!lS5=3^4gH{83Kr}8R1@i9 z!%m7kNaUO$!zKrCHfwav7I(im5a$4*??dAb9t*K>)qEMqRRU2RcDqzJp;KpI`(a^GU@tb2xc#22#P=J?U}Crnddt9C7a{hI_2%iUW$ z`e-UuK=AzLEdh@dSB-@rtucW+n}Su$E6Psd-8O`tB*_pnL5Y^YsQoE=uF2Ui(6YLy zw$vu$U7hGfh-M3gnQBxfU$z)2CrQo_Sk)J?x|-O|)e(^Wa_oI~vpP@&CdarG_k+ce z<6){8L8E*(x}liEs2xK$X7X9ae+!Ih$9mO#t7>6EuobfVzwKOWa>l`vX|HQLnLwbJ zGn2~&p_T{n(6Z1uml1ej(`yvfr9ZASP#QSPz4`#q9fmy=gKQNEVGmsm6A|~?n zfl;@Ce^!0V=xU?+lYLZW`2kxB7C) zYm;tI@7`nCk7u7}^-|ql{PtN9maMrFkYkY+o10h~pU5elXPCO94L2RRNz~3vbyG*| zB07}OC_EOIe>J8XL<45{A_$p+xJHdT;ZTtn1k7TK=2CX|=Sco)D4avL?qXXT`?D!Q z4WPf2h{o1MlHhTp-Ti0uuJO0i$4NZ)XSYG(VRwI)J;*r=wo^n%+a@To%uAfM1C_gf zy!?<_qD)S^x%f+R_VlocUTiLm)42mNg7mVKxGL??M7Vk4uCXJb6NzfqW z0z__gRkYMAltkt^a;dm6*&SWr6Dr@e(YJDlGgu~t_ZPHwy-`!qy)RCUiC_Z=KY6@q zH9auKxCOUa;9Z4NR5IpMv56mWXi~WGPaTvZE*;%lO?Rf0*XMlZi)ALqV$Dl4;Vy*Y zfIf`%EJK2BCZwu-8CFm>?cLCOG%L6UsY#)TyCs`&m*di|f}HgjMt(3dPSvFxuMW%; z$?W_LlN~H;V#yG}g1}xXK~kt~(tC=}`hA#~D4nRtINvybJOtf+SXF|$#4dg6^do1H zo@nw%)!owNU4yn}1TICNK`JHeDASks5E=PB$7vlc1+!v;HLo9eIv{3SiBcD$r*pD# zHwmb;1Tl-l3c*%{F%{OGlc&e{spinuv;>_moSD+R)RP9IKsTZcY-qiceA+I-&$p2G zkFcE_^1mPpQfw3T>EB$Hoag@y*#uIB9&_kFk6jY%e(3~_{O9|AO_2Q#m+x6p+0?(*i_s?`i~g zTfpAn-~R9SUy2oNKv@F8jx>Q-XjT(jy0*KiBdS8MwM2r2xOb+2@t;3^HkwV{=!=4}eFxB$|J`0h(oaSntIXy(w?kIbKfS&IodNO~*r*o}*4g|7fAKiH@ zv{9Sb__cxVh(o`E0Wq;>BXC`F)kQrZ`cqL2bP!#|VSLANd!%ScDlUfMEF(4~J2<4H;bvukN%vNMo?Y1Bk`Htj5xb zjmd4pt+BN&x*nwtI!eV-dzz%>s4vB$STl<0VF2tuI|R=^3u+@Pk&4lBgq@MMw(P)$ znvR1xd-eBy?9&Pi(ODB*NQo#X3e*Hv5*rOi3jJeN5*=BWEOXhPVu08dR071{2$#Xg zhfqFvmfGgnZI{L*obWEc&Q0SDbuxZ-KcWdlHi_mN>RM{)mdHx!5VOPlR6XElzq>MB z0b!q-&{a&$_mNa~OeTkK8XBJ&)~ChgKerBG_=Z;wF`;Ii&6|D42m-7P55Zlnm0d)6 zm(mU9xZL-w{Bt_vm=m#yr@!#TXhC4h!63=1h9@T1zo-644@P5A)UMvQ#8~Y^58C^+@Xx8{z#83pq)-`85ExiZf z@4Oyo_t~aDz~@9Q_+PBO@D%M|-k$)xwl0@JmeU{*%0OtbwunCwb~5uuQn3LT`G-`q~(gUVdsNQ5!s9#?oNWm9Pf`6!nw9(c&oLIJw&*+dYX(hUE zr4iZl(my1P+7_>%X`?89@yY~*2a1B{%sF6$m}Y@zr0K{;Itq;HsfwcSwcBi?7`e62 z8g&qJa%u^rJk=DTQ3qAcxlyMpdMuLr87=`Kp0&&G6NTQd!fCpLeRG^-tA}TB|I87Z zJDE7v7Mi7k)X(OQ<{n7OFpl_n9W*Up0i#oEhdhjajPs0}(C&j?Qkt$#dLBaH?@Y{U zhUz`DsT1T5MMg;v?oVt3G-koD5m%|@?4sxMYg_mMZLF}a&HT|8f1Iq1?T73(exqWx zA5ZjIfQBZGGtm=MY?(EocuvuBE#ikr-_FQ7ajk5%H2Xg@Ho>KN$)4KM*l?2nF;iph zNeTgTAe=iF!38GQ#os;>0$f_y&VkH~;M9kKRsQxQrz0mvc!xbtUF+c2XN2Zx6=r6F&2y;gT8AqOTA(tf z%6llYf{y`@>cQnUN+wT-fRL#wVYTcm%P@>7#qUv}^`$~HnC}>jx^gf&u2;m1IT<_t z`sfXe*A`Q1hDWFg$4Ns{3G+^Ioiq2b-N{Pf<>T<`^eg=xJ(ko!30}zW^3{)PG@*I>JeIW!u2R;CkK*s|6v*jm! z$|e;)9qIY`S9OtvS~2RvUabiO6AcTW@O3A-Ju5U(h#b+ax%p_=<8RUb)Ue0q8u@Zp zoHZ+;DzqqRc5ikzPn%B-@w@zhz)L}&X${Sw)?)`-uE}Ui4WSD>HYZsdSq)*=cmM#O zt)yomgwgUmzo95-K^#^0c@KhhLTGNQt~)jjDG3OaLxP!+UFame_}z<_52WrPtj zi*L9XYfei@$&e9&3+fPwgOHlYB5fu9+n}6I+5lGPd^7|uaf_J zwA+keFP0qk3pf|bi>I;%bndFO;3_tDi=bnJ-w8IaCeCywFfs0wZ-}K{E1wg!RX||%iue^qF9RV8LN~nOHz9)9mkCyP)$uQ_gw&auptkF*a8z;RQ+u4I1 zNmNPlmz@E@1~gWwM_;Z{NE?ZC>*`Y7OrS<>F19*14VHBidS`E)%qeJI@J+??&QWn! zYPC*X{q@bx_F0c~Q%b%YEy_-NYlC(yq%HHNyKpLLJIpjlZpIIwuCUPsng!Baan_5> zAN1Dd`dRDHMT6zTiMdO1*5O%Lo$@-QM^`i$+*3=NeZ{tH{!*T3y6`GtDa*y0Pa2$E z-Od&-*)uo=4M0b<@N$UY`a25C*Lmswwu>EA#k_SKsHp#pN{YEMSMMZ&PIG`;{zHEk zTtBP*cum45XJ+C+uSQ}$M9WDp6(n=Kt9mb?Kok$|W(7wb9K;H!^yYd`Ce>*{)#z9t z7{Rh*w7^(2ud83LjUgUmw9UB?j~6$j=t2G_rMUm@=MZ^)M^fhwt#5OXmc;}zb(W5)*guN$~k~WPJRcucRQgAqs zOseqY%^>kGr5qC7U+9x)fg(O(7}+=a3kfBGCJw@5z5E)f>%2yP3xOWUWB9R&vCgf! zDFo|ZOc;5;_J^1qkJAoWapJJdO3b zR=k)2gA4DrsySy^$bCj9)?8+gN2t~(RJWz>{%lfjToM-FGV@BuoY*kjwDV8B&l&qT z+Cz|0q^PbQj*1CQWYwwkma(kKspX(+poBLGdp(|`k=G`b(mc9=_ON*~MXAFHG|0%gIz zz~KLHU7fFEcg~kxT^pLo2Rse#^rNufkjupk!TlGLc5~6|8>g)ez1e@Ry1S%CBzaY# z_SMg!m3=qLfftpp0AAwg=v%I9{pqO8V&L^5T877fw>Flp3l%I*1@;8gj>(=42Cf3$ zw;H&1e2r}B^PfI3`s+gPp4aMs*@dc4bk^b7U9TgocTEFFVDXWEeO`S#-shaRdcXG2 zW0Sg&$*1fZ_(;8bV*Rb1_U}&4ns*j>u)^-hK8f8z#Ygvs{Mfru{_M3&zqiyM!FyP! zRnd&BdF#Gzc)nZo7F}+3bk@dpv!m&zt#>nQF3BwZArahp`bCWxu)qXff2bVx)9tb3 zS>s*nfxXfxy{?;o11ERdpPl}6?%5qJwR73PR_u1;`A3gb=BA(d6!Piy-;J)j4MBG+ z&R7S!Ok!&3mwz{|YR3k&bwIQ_POWX>T^fMQj;&fsS5{3Zu-wvuQU20 z|A-ncz5yB&4#`{>xX-I6@+ipO3Rxouor?M~XAjV|w_kIo9snX~@L zi-SG;H<(uUhB-8GNSbS!tLgh=!agI)@r(iuPaqRR8b%|39@~cgO!f9q%3XF$TV0_;}Vy n^m9Xj^Rc4HE~!uk9t%~^STnQD)z4*}Q$iB}sO24` -- Gitee From 46c92c908ba9d45c1d54d71ef879535220c553d3 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 26 Feb 2025 11:30:10 +0800 Subject: [PATCH 20/37] framework get improve --- .../msprobe/core/common/utils.py | 15 +++--- .../msprobe/mindspore/compare/ms_compare.py | 53 ++++++++----------- .../msprobe/test/core_ut/common/test_utils.py | 38 ++++++++++++- .../mindspore_ut/compare/test_ms_compare.py | 18 +++++++ 4 files changed, 84 insertions(+), 40 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index c06b5b6492..340fa07905 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -247,14 +247,13 @@ def md5_find(data): def detect_framework_by_dump_json(file_path): - pattern_ms = r'"type":\s*"mindspore' - pattern_pt = r'"type":\s*"torch' - with FileOpen(file_path, 'r') as file: - for line in file: - if re.search(pattern_ms, line): - return Const.MS_FRAMEWORK - if re.search(pattern_pt, line): - return Const.PT_FRAMEWORK + bench_json_data = load_json(file_path) + framework = bench_json_data.get("framework", None) + if not framework: + logger.error("cannot find framework in dump.json") + raise CompareException(CompareException.INVALID_DUMP_FILE) + if framework in [Const.PT_FRAMEWORK, Const.MS_FRAMEWORK]: + return framework logger.error(f"{file_path} must be based on the MindSpore or PyTorch framework.") raise CompareException(CompareException.INVALID_PARAM_ERROR) diff --git a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py index 5344573ad9..e0915f8179 100644 --- a/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py +++ b/debug/accuracy_tools/msprobe/mindspore/compare/ms_compare.py @@ -22,11 +22,10 @@ import pandas as pd from msprobe.core.common.const import CompareConst, Const from msprobe.core.common.exceptions import FileCheckException -from msprobe.core.common.file_utils import FileOpen, create_directory, load_json, load_yaml, load_npy, FileChecker, \ - FileCheckConst +from msprobe.core.common.file_utils import create_directory, load_json, load_npy, load_yaml from msprobe.core.common.log import logger from msprobe.core.common.utils import CompareException, check_compare_param, check_configuration_param, \ - check_op_str_pattern_valid, get_dump_mode, set_dump_path + check_op_str_pattern_valid, get_dump_mode, set_dump_path, detect_framework_by_dump_json from msprobe.core.compare.acc_compare import Comparator, ModeConfig from msprobe.core.compare.check import dtype_mapping from msprobe.core.compare.layer_mapping import generate_data_mapping_by_layer_mapping @@ -79,11 +78,6 @@ class MSComparator(Comparator): raise TypeError(f"The type of parameter `data_mapping` must be dict, str or None, but got " f"{type(self.data_mapping)}") - @staticmethod - def process_data_name(result): - result['data_name_x'] = result.apply(lambda row: [row['data_name_x'], row['data_name_y']], axis=1) - return result - def calc_accuracy(self, result_df, header): condition_no_bench = result_df[CompareConst.BENCH_NAME] == CompareConst.N_A result_df[condition_no_bench] = result_df[condition_no_bench].fillna(CompareConst.N_A) @@ -131,8 +125,7 @@ class MSComparator(Comparator): result_df.loc[warning_flag, CompareConst.RESULT] = CompareConst.WARNING result_df.loc[warning_flag, CompareConst.ERROR_MESSAGE] = 'Need double check api accuracy.' else: - fill_cols = [CompareConst.COSINE, CompareConst.EUC_DIST, - CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR, + fill_cols = [CompareConst.COSINE, CompareConst.MAX_ABS_ERR, CompareConst.MAX_RELATIVE_ERR, CompareConst.ONE_THOUSANDTH_ERR_RATIO, CompareConst.FIVE_THOUSANDTHS_ERR_RATIO, CompareConst.ERROR_MESSAGE] result_df.loc[~condition_no_bench, fill_cols] = '' @@ -146,8 +139,6 @@ class MSComparator(Comparator): header.append(CompareConst.STACK) if self.dump_mode == Const.ALL: header.append(CompareConst.DATA_NAME) - result = self.process_data_name(result) - result.rename(columns={'op_name_x': CompareConst.NPU_NAME, 'op_name_y': CompareConst.BENCH_NAME, 'dtype_x': CompareConst.NPU_DTYPE, @@ -178,7 +169,6 @@ class MSComparator(Comparator): result[npu_summary] = result['summary_x'].apply(set_summary).tolist() result[bench_summary] = result['summary_y'].apply(set_summary).tolist() - result_df = pd.DataFrame(columns=header) for h in header: if h in result.columns: @@ -212,6 +202,20 @@ class MSComparator(Comparator): npu_op_name = npu_op_name.replace(cell_name, self.cell_mapping_dict[cell_name], 1) return npu_op_name + def read_npy_data(self, dir_path, file_name, load_pt_file=False): + if not file_name: + return None + data_path = os.path.join(dir_path, file_name) + if load_pt_file: + import torch + from msprobe.pytorch.common.utils import load_pt + data_value = load_pt(data_path, True).detach() + if data_value.dtype == torch.bfloat16: + data_value = data_value.to(torch.float32) + data_value = data_value.numpy() + else: + data_value = load_npy(data_path) + return data_value def process_internal_api_mapping(self, npu_op_name): # get api name & class name from op_name @@ -378,24 +382,11 @@ class MSComparator(Comparator): def check_cross_framework(bench_json_path): - pattern = r'"data_name":\s*"[^"]+\.pt"' - with FileOpen(bench_json_path, 'r') as file: - for line in file: - if re.search(pattern, line): - return True - return False - - -def read_npy_data(dir_path, file_name): - if not file_name: - return None - - data_path = os.path.join(dir_path, file_name) - path_checker = FileChecker(data_path, FileCheckConst.FILE, FileCheckConst.READ_ABLE, - FileCheckConst.NUMPY_SUFFIX, False) - data_path = path_checker.common_check() - data_value = load_npy(data_path) - return data_value + framework = detect_framework_by_dump_json(bench_json_path) + if framework == Const.PT_FRAMEWORK: + return True + else: + return False def ms_compare(input_param, output_path, **kwargs): diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 3472ca9018..cd8660bf16 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -18,6 +18,7 @@ import json import os import tempfile from datetime import datetime, timezone +import unittest from unittest import TestCase from unittest.mock import MagicMock, mock_open, patch @@ -53,7 +54,8 @@ from msprobe.core.common.utils import (CompareException, recursion_depth_decorator, MsprobeBaseException, check_str_param, - is_json_file) + is_json_file, + detect_framework_by_dump_json) class TestUtils(TestCase): @@ -488,3 +490,37 @@ class TestCheckCrtValid(TestCase): with self.assertRaises(RuntimeError) as context: check_crt_valid(self.cert_file_path) self.assertIn('The SSL certificate is invalid', str(context.exception)) + + +class TestDetectFrameworkByDumpJson(unittest.TestCase): + + @patch('msprobe.common.utils.load_json') + def test_valid_pytorch_framework(self, mock_load_json): + mock_load_json.return_value = {"framework": Const.PT_FRAMEWORK} + + result = detect_framework_by_dump_json("dummy_path") + + self.assertEqual(result, Const.PT_FRAMEWORK) + + @patch('msprobe.common.utils.load_json') + def test_valid_mindspore_framework(self, mock_load_json): + mock_load_json.return_value = {"framework": Const.MS_FRAMEWORK} + + result = detect_framework_by_dump_json("dummy_path") + + self.assertEqual(result, Const.MS_FRAMEWORK) + + @patch('msprobe.common.utils.load_json') + def test_invalid_framework(self, mock_load_json): + # 模拟 load_json 返回一个没有 "framework" 键的字典 + mock_load_json.return_value = {} + + with self.assertRaises(CompareException): + detect_framework_by_dump_json("dummy_path") + + # 模拟返回其他未知的框架 + mock_load_json.return_value = {"framework": "tensorflow"} + + with self.assertRaises(CompareException): + detect_framework_by_dump_json("dummy_path") + diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py index 62fcf5a0e7..50c7963c7b 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py @@ -5,6 +5,7 @@ import random import shutil import tempfile import unittest +from unittest.mock import patch import numpy as np import pandas as pd @@ -352,6 +353,7 @@ class TestUtilsMethods(unittest.TestCase): shutil.rmtree(data_path) def test_check_cross_framework(self): + # dffgf ms_data = { "data_name": "Cell.model.language_model.encoder.layers.5.input_norm.FusedRMSNorm.forward.0.input.0.npy", } @@ -367,6 +369,22 @@ class TestUtilsMethods(unittest.TestCase): self.assertFalse(check_data(ms_data)) self.assertTrue(check_data(pt_data)) + @patch('msprobe.mindspore.ms_compare.detect_framework_by_dump_json') + def test_check_cross_framework_valid_pytorch(self, mock_detect_framework): + mock_detect_framework.return_value = Const.PT_FRAMEWORK + + result = check_cross_framework("dummy_path") + + self.assertTrue(result) + + @patch('msprobe.mindspore.ms_compare.detect_framework_by_dump_json') + def test_check_cross_framework_invalid_framework(self, mock_detect_framework): + mock_detect_framework.return_value = Const.MS_FRAMEWORK + + result = check_cross_framework("dummy_path") + + self.assertFalse(result) + def test_comapre_process(self): data_path = tempfile.mkdtemp(prefix='dump_data', dir='/tmp') try: -- Gitee From 1ee71652ff7318605f8c14555c27be7add3217eb Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 26 Feb 2025 11:35:42 +0800 Subject: [PATCH 21/37] compare framework get improve --- debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index cd8660bf16..a0078061b9 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*- coding: utf-8 -*- """ -# Copyright (C) 2024-2024. Huawei Technologies Co., Ltd. All rights reserved. +# Copyright (C) 2024-2025. Huawei Technologies Co., Ltd. All rights reserved. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -- Gitee From 02cb21f79e9e0d69386830467d98ffa8da30c81a Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 26 Feb 2025 15:01:49 +0800 Subject: [PATCH 22/37] compare framework get improve --- .../mindspore_ut/compare/test_ms_compare.py | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py index 50c7963c7b..4bbfd919ec 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py @@ -352,23 +352,6 @@ class TestUtilsMethods(unittest.TestCase): finally: shutil.rmtree(data_path) - def test_check_cross_framework(self): - # dffgf - ms_data = { - "data_name": "Cell.model.language_model.encoder.layers.5.input_norm.FusedRMSNorm.forward.0.input.0.npy", - } - pt_data = { - "data_name": "Module.module.module.language_model.encoder.layers.0.input_norm.RMSNorm.forward.0.input.0.pt", - } - - def check_data(data): - with tempfile.NamedTemporaryFile(mode='w+', suffix='.json', encoding='utf-8', delete=True) as temp_file: - json.dump(data, temp_file, ensure_ascii=False, indent=4) - temp_file.flush() - return check_cross_framework(temp_file.name) - self.assertFalse(check_data(ms_data)) - self.assertTrue(check_data(pt_data)) - @patch('msprobe.mindspore.ms_compare.detect_framework_by_dump_json') def test_check_cross_framework_valid_pytorch(self, mock_detect_framework): mock_detect_framework.return_value = Const.PT_FRAMEWORK -- Gitee From 20a39e82d5bc872a262b6d72013f967d0ae18254 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 26 Feb 2025 15:04:59 +0800 Subject: [PATCH 23/37] compare framework get improve --- .../msprobe/test/core_ut/common/test_utils.py | 6 +++--- .../msprobe/test/mindspore_ut/compare/test_ms_compare.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index a0078061b9..79395cfc3d 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -494,7 +494,7 @@ class TestCheckCrtValid(TestCase): class TestDetectFrameworkByDumpJson(unittest.TestCase): - @patch('msprobe.common.utils.load_json') + @patch('msprobe.core.common.utils.load_json') def test_valid_pytorch_framework(self, mock_load_json): mock_load_json.return_value = {"framework": Const.PT_FRAMEWORK} @@ -502,7 +502,7 @@ class TestDetectFrameworkByDumpJson(unittest.TestCase): self.assertEqual(result, Const.PT_FRAMEWORK) - @patch('msprobe.common.utils.load_json') + @patch('msprobe.core.common.utils.load_json') def test_valid_mindspore_framework(self, mock_load_json): mock_load_json.return_value = {"framework": Const.MS_FRAMEWORK} @@ -510,7 +510,7 @@ class TestDetectFrameworkByDumpJson(unittest.TestCase): self.assertEqual(result, Const.MS_FRAMEWORK) - @patch('msprobe.common.utils.load_json') + @patch('msprobe.core.common.utils.load_json') def test_invalid_framework(self, mock_load_json): # 模拟 load_json 返回一个没有 "framework" 键的字典 mock_load_json.return_value = {} diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py index 4bbfd919ec..667fea2241 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/test_ms_compare.py @@ -352,7 +352,7 @@ class TestUtilsMethods(unittest.TestCase): finally: shutil.rmtree(data_path) - @patch('msprobe.mindspore.ms_compare.detect_framework_by_dump_json') + @patch('msprobe.mindspore.compare.ms_compare.detect_framework_by_dump_json') def test_check_cross_framework_valid_pytorch(self, mock_detect_framework): mock_detect_framework.return_value = Const.PT_FRAMEWORK @@ -360,7 +360,7 @@ class TestUtilsMethods(unittest.TestCase): self.assertTrue(result) - @patch('msprobe.mindspore.ms_compare.detect_framework_by_dump_json') + @patch('msprobe.mindspore.compare.ms_compare.detect_framework_by_dump_json') def test_check_cross_framework_invalid_framework(self, mock_detect_framework): mock_detect_framework.return_value = Const.MS_FRAMEWORK -- Gitee From 974870e887ab511a225af8a32da4eb5ccee79ecc Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 26 Feb 2025 15:57:46 +0800 Subject: [PATCH 24/37] compare framework get improve --- .../msprobe/test/resources/layer_mapping/mindspore/dump.json | 1 + .../msprobe/test/resources/layer_mapping/pytorch/dump.json | 1 + 2 files changed, 2 insertions(+) diff --git a/debug/accuracy_tools/msprobe/test/resources/layer_mapping/mindspore/dump.json b/debug/accuracy_tools/msprobe/test/resources/layer_mapping/mindspore/dump.json index b55f9e0699..153d84e7d1 100644 --- a/debug/accuracy_tools/msprobe/test/resources/layer_mapping/mindspore/dump.json +++ b/debug/accuracy_tools/msprobe/test/resources/layer_mapping/mindspore/dump.json @@ -1,6 +1,7 @@ { "task": "statistics", "level": "mix", + "framework": "mindspore", "dump_data_dir": null, "data": { "Cell.network_with_loss.module.language_model.embedding.word_embeddings.VocabParallelEmbedding.forward.0": { diff --git a/debug/accuracy_tools/msprobe/test/resources/layer_mapping/pytorch/dump.json b/debug/accuracy_tools/msprobe/test/resources/layer_mapping/pytorch/dump.json index d7dd1c0c38..02239176a9 100644 --- a/debug/accuracy_tools/msprobe/test/resources/layer_mapping/pytorch/dump.json +++ b/debug/accuracy_tools/msprobe/test/resources/layer_mapping/pytorch/dump.json @@ -1,6 +1,7 @@ { "task": "statistics", "level": "mix", + "framework": "pytorch", "dump_data_dir": null, "data": { "Module.module.module.language_model.embedding.word_embeddings.VocabParallelEmbedding.forward.0": { -- Gitee From 1762a7df7f372c64bdcb8937bfcabe470c0633f0 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Wed, 26 Feb 2025 16:18:37 +0800 Subject: [PATCH 25/37] compare framework get improve --- .../test/mindspore_ut/compare/dump_file/mindspore_data/dump.json | 1 + .../test/mindspore_ut/compare/dump_file/pytorch_data/dump.json | 1 + 2 files changed, 2 insertions(+) diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/dump_file/mindspore_data/dump.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/dump_file/mindspore_data/dump.json index 5b954f6d64..48800c0455 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/dump_file/mindspore_data/dump.json +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/dump_file/mindspore_data/dump.json @@ -1,6 +1,7 @@ { "task": "statistics", "level": "mix", + "framework": "mindspore", "dump_data_dir": null, "data": { "Tensor.__add__.0.forward": { diff --git a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/dump_file/pytorch_data/dump.json b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/dump_file/pytorch_data/dump.json index 150cbd43b1..b2704185ff 100644 --- a/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/dump_file/pytorch_data/dump.json +++ b/debug/accuracy_tools/msprobe/test/mindspore_ut/compare/dump_file/pytorch_data/dump.json @@ -1,6 +1,7 @@ { "task": "statistics", "level": "mix", + "framework": "pytorch", "dump_data_dir": null, "data": { "Tensor.__add__.0.forward": { -- Gitee From e7f12828dd1fec0752846cd3a8197450911a8ac5 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 27 Feb 2025 16:36:38 +0800 Subject: [PATCH 26/37] compare framework get improve --- .../msprobe/core/common/utils.py | 15 +++++++---- .../msprobe/test/core_ut/common/test_utils.py | 26 ++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/debug/accuracy_tools/msprobe/core/common/utils.py b/debug/accuracy_tools/msprobe/core/common/utils.py index 340fa07905..7ec0490168 100644 --- a/debug/accuracy_tools/msprobe/core/common/utils.py +++ b/debug/accuracy_tools/msprobe/core/common/utils.py @@ -247,13 +247,18 @@ def md5_find(data): def detect_framework_by_dump_json(file_path): - bench_json_data = load_json(file_path) - framework = bench_json_data.get("framework", None) - if not framework: - logger.error("cannot find framework in dump.json") - raise CompareException(CompareException.INVALID_DUMP_FILE) + json_data = load_json(file_path) + framework = json_data.get("framework", None) if framework in [Const.PT_FRAMEWORK, Const.MS_FRAMEWORK]: return framework + pattern_ms = r'"type":\s*"mindspore' + pattern_pt = r'"type":\s*"torch' + with FileOpen(file_path, 'r') as file: + for line in file: + if re.search(pattern_ms, line): + return Const.MS_FRAMEWORK + if re.search(pattern_pt, line): + return Const.PT_FRAMEWORK logger.error(f"{file_path} must be based on the MindSpore or PyTorch framework.") raise CompareException(CompareException.INVALID_PARAM_ERROR) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index 79395cfc3d..a8eef63f59 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -18,6 +18,7 @@ import json import os import tempfile from datetime import datetime, timezone +import re import unittest from unittest import TestCase from unittest.mock import MagicMock, mock_open, patch @@ -510,17 +511,18 @@ class TestDetectFrameworkByDumpJson(unittest.TestCase): self.assertEqual(result, Const.MS_FRAMEWORK) - @patch('msprobe.core.common.utils.load_json') - def test_invalid_framework(self, mock_load_json): - # 模拟 load_json 返回一个没有 "framework" 键的字典 - mock_load_json.return_value = {} - - with self.assertRaises(CompareException): - detect_framework_by_dump_json("dummy_path") + @patch("msprobe.core.common.utils.FileOpen", new_callable=mock_open) + @patch("re.search") # 模拟 re.search + def test_detect_framework_in_file(self, mock_search, mock_open): + # 测试框架是 MindSpore + fake_file_content = '{"type": "mindspore.float16"}\n' + mock_open.return_value.read.side_effect = fake_file_content - # 模拟返回其他未知的框架 - mock_load_json.return_value = {"framework": "tensorflow"} - - with self.assertRaises(CompareException): - detect_framework_by_dump_json("dummy_path") + result = detect_framework_by_dump_json("dummy_path") + self.assertEqual(result, Const.MS_FRAMEWORK) + # 测试框架是 PyTorch + fake_file_content = '{"type": "torch.float16"}\n' + mock_open.return_value.read.side_effect = fake_file_content + result = detect_framework_by_dump_json("dummy_path") + self.assertEqual(result, Const.PT_FRAMEWORK) -- Gitee From 5d468a5d30a76ec21d449a054efa40161168c1a7 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 27 Feb 2025 19:12:00 +0800 Subject: [PATCH 27/37] compare framework get improve --- .../test_dump_file/dump_no_pt_no_ms.json | 3 ++ .../test_dump_file/ms_dump_no_framework.json | 4 +++ .../test_dump_file/pt_dump_no_framework.json | 4 +++ .../msprobe/test/core_ut/common/test_utils.py | 30 +++++++++++-------- 4 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/dump_no_pt_no_ms.json create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/ms_dump_no_framework.json create mode 100644 debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/pt_dump_no_framework.json diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/dump_no_pt_no_ms.json b/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/dump_no_pt_no_ms.json new file mode 100644 index 0000000000..63a062d8ff --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/dump_no_pt_no_ms.json @@ -0,0 +1,3 @@ +{ + "task": "tensor" +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/ms_dump_no_framework.json b/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/ms_dump_no_framework.json new file mode 100644 index 0000000000..b223c74b23 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/ms_dump_no_framework.json @@ -0,0 +1,4 @@ +{ + "task": "tensor", + "type": "mindspore.float16" +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/pt_dump_no_framework.json b/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/pt_dump_no_framework.json new file mode 100644 index 0000000000..2444ae1fd4 --- /dev/null +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_dump_file/pt_dump_no_framework.json @@ -0,0 +1,4 @@ +{ + "task": "tensor", + "type": "torch.float16" +} \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index a8eef63f59..c0328008ad 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -18,13 +18,13 @@ import json import os import tempfile from datetime import datetime, timezone -import re import unittest from unittest import TestCase from unittest.mock import MagicMock, mock_open, patch import OpenSSL import numpy as np +from pathlib import Path from msprobe.core.common.const import Const from msprobe.core.common.file_utils import ( @@ -511,18 +511,22 @@ class TestDetectFrameworkByDumpJson(unittest.TestCase): self.assertEqual(result, Const.MS_FRAMEWORK) - @patch("msprobe.core.common.utils.FileOpen", new_callable=mock_open) - @patch("re.search") # 模拟 re.search - def test_detect_framework_in_file(self, mock_search, mock_open): - # 测试框架是 MindSpore - fake_file_content = '{"type": "mindspore.float16"}\n' - mock_open.return_value.read.side_effect = fake_file_content + def test_detect_framework_in_file(self): + self.current_dir = Path(__file__).parent + file_path = self.current_dir / "test_dump_file/pt_dump_no_framework.json" + result = detect_framework_by_dump_json(file_path) + self.assertEqual(result, Const.PT_FRAMEWORK) - result = detect_framework_by_dump_json("dummy_path") + self.current_dir = Path(__file__).parent + file_path = self.current_dir / "test_dump_file/ms_dump_no_framework.json" + result = detect_framework_by_dump_json(file_path) self.assertEqual(result, Const.MS_FRAMEWORK) - # 测试框架是 PyTorch - fake_file_content = '{"type": "torch.float16"}\n' - mock_open.return_value.read.side_effect = fake_file_content - result = detect_framework_by_dump_json("dummy_path") - self.assertEqual(result, Const.PT_FRAMEWORK) + @patch("msprobe.core.common.utils.logger") + def test_detect_framework_exception(self, mock_logger): + self.current_dir = Path(__file__).parent + file_path = self.current_dir / "test_dump_file/pt_dump_no_pt_no_ms.json" + with self.assertRaises(CompareException) as context: + result = detect_framework_by_dump_json(file_path) + self.assertEqual(context.exception.code, CompareException.INVALID_PARAM_ERROR) + mock_logger.error.assert_called_once_with(f"{file_path} must be based on the MindSpore or PyTorch framework.") -- Gitee From 90eafae2e2b43e9f5e9aa78abc2699b0b85c76d0 Mon Sep 17 00:00:00 2001 From: Linwei-Ying Date: Thu, 27 Feb 2025 19:13:34 +0800 Subject: [PATCH 28/37] compare framework get improve --- debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py index c0328008ad..61766ed27c 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/common/test_utils.py @@ -525,7 +525,7 @@ class TestDetectFrameworkByDumpJson(unittest.TestCase): @patch("msprobe.core.common.utils.logger") def test_detect_framework_exception(self, mock_logger): self.current_dir = Path(__file__).parent - file_path = self.current_dir / "test_dump_file/pt_dump_no_pt_no_ms.json" + file_path = self.current_dir / "test_dump_file/dump_no_pt_no_ms.json" with self.assertRaises(CompareException) as context: result = detect_framework_by_dump_json(file_path) self.assertEqual(context.exception.code, CompareException.INVALID_PARAM_ERROR) -- Gitee From fd11e64457ff0d54c0491edd78f51edc9de03bf4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AD=94=E7=82=B3=E7=BF=94?= <1120200577@qq.com> Date: Mon, 3 Mar 2025 17:33:53 +0800 Subject: [PATCH 29/37] update freq_analysis --- .../msprof_analyze/cluster_analyse/README.md | 1 + .../recipes/freq_analysis/__init__.py | 0 .../recipes/freq_analysis/freq_analysis.py | 114 ++++++++++++++++++ .../recipes/test_freq_analysis.py | 83 +++++++++++++ 4 files changed, 198 insertions(+) create mode 100644 profiler/msprof_analyze/cluster_analyse/recipes/freq_analysis/__init__.py create mode 100644 profiler/msprof_analyze/cluster_analyse/recipes/freq_analysis/freq_analysis.py create mode 100644 profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_freq_analysis.py diff --git a/profiler/msprof_analyze/cluster_analyse/README.md b/profiler/msprof_analyze/cluster_analyse/README.md index 325a098479..6612d0f198 100644 --- a/profiler/msprof_analyze/cluster_analyse/README.md +++ b/profiler/msprof_analyze/cluster_analyse/README.md @@ -79,6 +79,7 @@ experimental_config = torch_npu.profiler._ExperimentalConfig( | compute_op_sum | 集群场景性能数据的device运行算子信息汇总分析，输入性能数据需要基于ascend_pytorch_profiler_{rank_id}.db文件。--export_type为db时，输出交付件cluster_analysis.db；--export_type为notebook时，在cluster_analysis_output/ComputeOpSum目录下输出交付件stats.ipynb；可根据实际情况决定是否是否打开--exclude_op_name。 | 否 | | hccl_sum | 集合通信算子耗时分析，输入性能数据需要基于ascend_pytorch_profiler_{rank_id}.db文件。--export_type为db时，输出交付件cluster_analysis.db；--export_type为notebook时，在cluster_analysis_output/HcclSum目录下输出交付件stats.ipynb。 | 否 | | mstx_sum | 集群场景mstx打点信息汇总分析，输入性能数据需要基于ascend_pytorch_profiler_{rank_id}.db文件。--export_type为db时，输出交付件cluster_analysis.db；--export_type为notebook时，在cluster_analysis_output/MstxSum目录下输出交付件stats.ipynb。 | 否 | + | freq_analysis | 集群场景aicore frequency信息汇总分析，输入性能数据需要基于ascend_pytorch_profiler_{rank_id}.db文件。打屏输出是否存在aicore存在空闲（频率为800MHz）、异常（频率不为1800MHz或800MHz）的现象。如果有，则在输出交付件cluster_analysis.db增加对应的卡和频率信息。 | 否 | | 自定义分析参数 | 与cann_api_sum、compute_op_sum、hccl_sum等参数功能类似，用户可自定义一套性能数据的分析规则，需要详细了解性能分析的开发人员，具体开发指导请参见“[自定义分析规则开发指导](#自定义分析规则开发指导)”。 | 否 | --parallel_mode参数示例如下： diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/freq_analysis/__init__.py b/profiler/msprof_analyze/cluster_analyse/recipes/freq_analysis/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/profiler/msprof_analyze/cluster_analyse/recipes/freq_analysis/freq_analysis.py b/profiler/msprof_analyze/cluster_analyse/recipes/freq_analysis/freq_analysis.py new file mode 100644 index 0000000000..0bc7afa393 --- /dev/null +++ b/profiler/msprof_analyze/cluster_analyse/recipes/freq_analysis/freq_analysis.py @@ -0,0 +1,114 @@ +# Copyright (c) 2024, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from collections import defaultdict +import pandas as pd + +from msprof_analyze.cluster_analyse.recipes.base_recipe_analysis import BaseRecipeAnalysis +from msprof_analyze.prof_common.constant import Constant +from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.prof_common.database_service import DatabaseService + +logger = get_logger() + + +class FreqAnalysis(BaseRecipeAnalysis): + COMMON_FREQ = 1800 + FREE_FREQ = 800 + + def __init__(self, params): + super().__init__(params) + self.free_freq_ranks = [] + self.abnormal_freq_ranks = [] + self.abnormal_freq_ranks_map = {} + + @property + def base_dir(self): + return os.path.basename(os.path.dirname(__file__)) + + def reducer_func(self, mapper_res): + if self._is_msprof: + logger.warning("Freq analysis do not support msprof db now.") + return + + mapper_res = list(filter(lambda res: res is not None, mapper_res)) + if not mapper_res: + logger.error("Mapper data is None, load profiling data failed.") + return + + for freqs, rank_id in mapper_res: + if freqs == [self.COMMON_FREQ]: + continue + elif set(freqs) == {self.COMMON_FREQ, self.FREE_FREQ}: + self.free_freq_ranks.append(rank_id) + else: + self.abnormal_freq_ranks.append(rank_id) + self.abnormal_freq_ranks_map[rank_id] = str(freqs) + + self.free_freq_ranks.sort() + self.abnormal_freq_ranks.sort() + + def save_db(self): + if len(self.free_freq_ranks) > 0: + logger.info(f"Found {len(self.free_freq_ranks)} ranks with free time, " + f"aicore frequency in {[self.FREE_FREQ, self.COMMON_FREQ]}.") + free_ranks_df = pd.DataFrame() + free_ranks_df["rankId"] = self.free_freq_ranks + free_ranks_df["aicoreFrequency"] = str([self.FREE_FREQ, self.COMMON_FREQ]) + free_ranks_df.set_index(["rankId"], inplace=True) + self.dump_data(free_ranks_df, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, "FreeFrequencyRanks") + else: + logger.info("No rank found with free time.") + if len(self.abnormal_freq_ranks) > 0: + logger.info(f"Found {len(self.abnormal_freq_ranks)} ranks with abnormal aicore frequency.") + + abnormal_ranks_df = pd.DataFrame.from_dict(self.abnormal_freq_ranks_map, + orient="index", columns=["aicoreFrequency"]) + abnormal_ranks_df = abnormal_ranks_df.reset_index().rename(columns={"index": "rankId"}) + abnormal_ranks_df.set_index(["rankId"], inplace=True) + self.dump_data(abnormal_ranks_df, Constant.DB_CLUSTER_COMMUNICATION_ANALYZER, "AbnormalFrequencyRanks") + else: + logger.info("No rank found with abnormal aicore frequency.") + if len(self.free_freq_ranks) > 0 or len(self.abnormal_freq_ranks) > 0: + logger.info("Please verify result in output file.") + + def run(self, context): + mapper_res = self.mapper_func(context) + self.reducer_func(mapper_res) + self.save_db() + + def _mapper_func(self, data_map, analysis_class): + profiler_db_path = data_map.get(Constant.PROFILER_DB_PATH) + service = DatabaseService(profiler_db_path, None) + service.add_table_for_query("AICORE_FREQ", ["deviceId", "freq"]) + service.add_table_for_query("RANK_DEVICE_MAP", ["rankId"]) + service_res = service.query_data() + aic_freq = service_res.get("AICORE_FREQ", None) + rank_id = service_res.get("RANK_DEVICE_MAP", None) + + if aic_freq is None or aic_freq.empty: + logger.error(f"No aic freq data found in {profiler_db_path}.") + return None + + if rank_id is None or rank_id.empty: + logger.error(f"No rank_id data found in {profiler_db_path}.") + return None + + rank_id = rank_id["rankId"].values[0] + freq_arr = aic_freq["freq"].values + freqs = list(set(freq_arr)) + freqs.sort() + return freqs, rank_id diff --git a/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_freq_analysis.py b/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_freq_analysis.py new file mode 100644 index 0000000000..0a559b7917 --- /dev/null +++ b/profiler/msprof_analyze/test/ut/cluster_analyse/recipes/test_freq_analysis.py @@ -0,0 +1,83 @@ +# Copyright (c) 2025, Huawei Technologies Co., Ltd. +# All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import random +import unittest + +import pandas as pd + +from msprof_analyze.cluster_analyse.recipes.freq_analysis.freq_analysis import FreqAnalysis + + +class TestFreqAnalysis(unittest.TestCase): + + freq = [1800] + free_freq = [800, 1800] + abnormal_freq = [1200, 1300, 1800] + + def test_no_error_freq(self): + params = {} + recipe = FreqAnalysis(params) + mapper_res = [(self.freq, 0)] * 10 + recipe.reducer_func(mapper_res) + self.assertEqual(recipe.free_freq_ranks, []) + self.assertEqual(recipe.abnormal_freq_ranks, []) + self.assertEqual(recipe.abnormal_freq_ranks_map, {}) + + + def test_free_rank_map(self): + params = {} + recipe = FreqAnalysis(params) + mapper_res = [ + (self.freq, 0), + (self.free_freq, 1), + (self.free_freq, 2), + (self.freq, 3) + ] + recipe.reducer_func(mapper_res) + self.assertEqual(recipe.free_freq_ranks, [1, 2]) + self.assertEqual(recipe.abnormal_freq_ranks, []) + self.assertEqual(recipe.abnormal_freq_ranks_map, {}) + + def test_abnormal_rank_map(self): + params = {} + recipe = FreqAnalysis(params) + mapper_res = [ + (self.freq, 0), + (self.abnormal_freq, 1), + (self.abnormal_freq, 2), + (self.freq, 3) + ] + + recipe.reducer_func(mapper_res) + self.assertEqual(recipe.free_freq_ranks, []) + self.assertEqual(recipe.abnormal_freq_ranks, [1, 2]) + + def test_mix_freq_case(self): + params = {} + recipe = FreqAnalysis(params) + mapper_res = [] + rank_case = [[], [], []] + random_freq = {0: self.freq, 1: self.free_freq, 2: self.abnormal_freq} + + for i in range(1000): + random_num = random.choice([0, 1, 2]) + mapper_res.append((random_freq.get(random_num, self.freq), i)) + rank_case[random_num].append(i) + + recipe.reducer_func(mapper_res) + self.assertEqual(recipe.free_freq_ranks, rank_case[1]) + self.assertEqual(recipe.abnormal_freq_ranks, rank_case[2]) -- Gitee From 2e506307075ec547cf569b23ad424cb3a431bed6 Mon Sep 17 00:00:00 2001 From: jiangchao_j Date: Sat, 1 Mar 2025 10:20:42 +0800 Subject: [PATCH 30/37] add introduction of madater dump and overflow_check --- debug/accuracy_tools/msprobe/README.md | 15 +- .../msprobe/docs/02.config_introduction.md | 30 +- .../msprobe/docs/05.data_dump_PyTorch.md | 2 +- .../msprobe/docs/06.data_dump_MindSpore.md | 2 +- .../msprobe/docs/27.dump_json_instruction.md | 286 +++++++++++++++++- .../msprobe/docs/28.kernel_dump_MindSpore.md | 2 +- .../msprobe/docs/29.data_dump_MSAdapter.md | 229 ++++++++++++++ .../docs/30.overflow_check_MSAdapter.md | 31 ++ 8 files changed, 567 insertions(+), 30 deletions(-) create mode 100644 debug/accuracy_tools/msprobe/docs/29.data_dump_MSAdapter.md create mode 100644 debug/accuracy_tools/msprobe/docs/30.overflow_check_MSAdapter.md diff --git a/debug/accuracy_tools/msprobe/README.md b/debug/accuracy_tools/msprobe/README.md index e31490f01e..6b7d483078 100644 --- a/debug/accuracy_tools/msprobe/README.md +++ b/debug/accuracy_tools/msprobe/README.md @@ -44,6 +44,7 @@ export MSPROBE_LOG_LEVEL={x} - msprobe支持AscendPyTorch 1.11.0或更高版本，支持的PyTorch和CANN以及PyTorch和python软件版本配套关系请参见《[Ascend Extension for PyTorch插件](https://gitee.com/ascend/pytorch)》。 - msprobe支持MindSpore 2.4.0或更高版本，支持的MindSpore和CANN以及MindSpore和python软件版本配套关系请参见《[MindSpore版本发布列表](https://www.mindspore.cn/versions)》。 +- msprobe支持MSAdapter 2.1.0。 - msprobe支持的固件驱动版本与配套CANN软件支持的固件驱动版本相同，开发者可通过“[昇腾社区-固件与驱动](https://gitee.com/link?target=https%3A%2F%2Fwww.hiascend.com%2Fhardware%2Ffirmware-drivers%2Fcommunity%3Fproduct%3D2%26model%3D28%26cann%3D8.0.RC3.alpha003%26driver%3D1.0.25.alpha)”页面根据产品型号与CANN软件版本获取配套的固件与驱动。 @@ -69,15 +70,17 @@ export MSPROBE_LOG_LEVEL={x} ### 1 数据采集 -msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API 执行精度数据 dump 操作，对应 config.json 中的 task 为 statistics 或 tensor。 +msprobe 通过在训练脚本中添加 PrecisionDebugger 接口的方式对 API 执行精度数据 dump 操作。对应 config.json 中的 "statistics" 或 "tensor" task。 [PyTorch 场景的数据采集](./docs/05.data_dump_PyTorch.md) [MindSpore 场景的数据采集](./docs/06.data_dump_MindSpore.md) +[MSAdapter 场景的数据采集](./docs/29.data_dump_MSAdapter.md) + ### 2 精度预检 -精度预检旨在昇腾 NPU 上扫描训练模型中的所有 API 进行 API 复现，给出精度情况的诊断和分析。对应 config.json 中的 task 为 run_ut。 +精度预检旨在昇腾 NPU 上扫描训练模型中的所有 API 进行 API 复现，给出精度情况的诊断和分析。对应 config.json 中的 "run_ut" task。 PyTorch 场景的[离线预检](./docs/07.accuracy_checker_PyTorch.md)和[在线预检](./docs/08.accuracy_checker_online_PyTorch.md) @@ -143,12 +146,14 @@ MindSpore 动态图场景的[离线预检](./docs/09.accuracy_checker_MindSpore. ### 12 溢出检测与解析 -溢出检测与解析是在执行精度数据 dump 时，判断是否存在输入正常但输出存在溢出的 API，从而判断是否为正常溢出。对应 config.json 中的 overflow_check。 -推荐直接使用[数据采集](#1-数据采集)功能采集统计量信息检测溢出问题。 +溢出检测用于采集溢出 API 或 模块的精度数据，而溢出解析则是通过对溢出数据的分析，进一步判断是否为正常溢出。对应 config.json 中的 "overflow_check" task。 +推荐直接使用[数据采集](#1-数据采集)功能采集统计量信息,检测溢出问题。 [PyTorch 场景的溢出检测与解析](./docs/12.overflow_check_PyTorch.md) -[MindSpore 场景的溢出检测与解析](./docs/13.overflow_check_MindSpore.md) +[MindSpore 场景的溢出检测](./docs/13.overflow_check_MindSpore.md) + +[MSAdapter 场景的溢出检测](./docs/30.overflow_check_MSAdapter.md) ## 📑 补充材料 diff --git a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md index f134bd4536..a5f17637da 100644 --- a/debug/accuracy_tools/msprobe/docs/02.config_introduction.md +++ b/debug/accuracy_tools/msprobe/docs/02.config_introduction.md @@ -12,23 +12,23 @@ | 参数 | 解释 | 是否必选 | | ----------------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -------- | -| task | dump 的任务类型，str 类型。可选参数：
"statistics"：仅采集统计信息，默认值；
"tensor"：采集统计信息和完全复刻整网的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持，采集数据时勿选；
"overflow_check"：溢出检测；
"free_benchmark"：无标杆比对；
"grad_probe"：梯度监控；
"structure"：仅采集模型结构以及调用栈信息，不采集具体数据。
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)，
[1.7 task 配置为 grad_probe](#17-task-配置为-grad_probe)。
**配置示例**："task": "tensor"。 | 否 | +| task | dump 的任务类型，str 类型。可选参数：
"statistics"：仅采集统计信息，默认值；
"tensor"：采集统计信息和完全复刻整网的真实数据；
"run_ut"：精度预检，仅 PyTorch 场景支持，采集数据时勿选；
"overflow_check"：溢出检测；
"free_benchmark"：无标杆比对，不支持 MSAdapter 场景；
"grad_probe"：梯度监控， 不支持 MSAdapter 场景；
"structure"：仅采集模型结构以及调用栈信息，不采集具体数据。
根据 task 参数取值的不同，可以配置不同场景参数，详见：
[1.2 task 配置为 statistics](#12-task-配置为-statistics)，
[1.3 task 配置为 tensor](#13-task-配置为-tensor)，
[1.4 task 配置为 run_ut](#14-task-配置为-run_ut)，
[1.5 task 配置为 overflow_check](#15-task-配置为-overflow_check)，
[1.6 task 配置为 free_benchmark](#16-task-配置为-free_benchmark)，
[1.7 task 配置为 grad_probe](#17-task-配置为-grad_probe)。
**配置示例**："task": "tensor"。 | 否 | | dump_path | 设置 dump 数据目录路径，str 类型。
**配置示例**："dump_path": "./dump_path"。 | 是 | | rank | 指定对某张卡上的数据进行采集，list[Union[int, str]] 类型，默认未配置（表示采集所有卡的数据），应配置元素为 ≥0 的整数或类似"4-6"的字符串，且须配置实际可用的 Rank ID。
PyTorch 场景: Rank ID 从 0 开始计数，最大取值为所有节点可用卡总数-1，若所配置的值大于实际训练所运行的卡的 Rank ID，则 dump 数据为空，比如当前环境 Rank ID 为 0 到 7，实际训练运行 0 到 3 卡，此时若配置 Rank ID 为 4 或不存在的 10 等其他值，dump 数据为空。
MindSpore 场景：所有节点的 Rank ID 均从 0 开始计数，最大取值为每个节点可用卡总数-1，config.json 配置一次 rank 参数对所有节点同时生效。
注意，单卡训练时，rank必须为[]，即空列表，不能指定rank。
**配置示例**："rank": [1, "4-6"]。 | 否 | | step | 指定采集某个 step 的数据，list[Union[int, str]] 类型。默认未配置，表示采集所有 step 数据。采集特定 step 时，须指定为训练脚本中存在的 step，可逐个配置，也可以指定范围。
**配置示例**："step": [0, 1 , 2, "4-6"]。 | 否 | -| level | dump 级别，str 类型，根据不同级别采集不同数据。可选参数：
"L0"：dump 模块级精度数据，仅 PyTorch 与 MindSpore 动态图场景支持，使用背景详见 [1.1.1 模块级精度数据 dump 说明](#111-模块级精度数据-dump-说明)；
"L1"：dump API 级精度数据，默认值，仅 PyTorch 与 MindSpore 动态图场景支持；
"L2"：dump kernel 级精度数据，PyTorch场景详细介绍见 [PyTorch 场景的 kernel dump 说明](./04.kernel_dump_PyTorch.md)；MindSpore场景详细介绍见 [MindSpore 场景的 kernel dump 说明](./28.kernel_dump_MindSpore.md)；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch 与 MindSpore 动态图场景支持。
"debug"：单点保存功能，细节详见[单点保存工具 README](./28.debugger_save_instruction.md)
**配置示例**："level": "L1"。 | 否 | +| level | dump 级别，str 类型，根据不同级别采集不同数据。可选参数：
"L0"：dump 模块级精度数据，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持，使用背景详见 [1.1.1 模块级精度数据 dump 说明](#111-模块级精度数据-dump-说明)；
"L1"：dump API 级精度数据，默认值，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持；
"L2"：dump kernel 级精度数据，PyTorch 场景详细介绍见 [PyTorch 场景的 kernel dump 说明](./04.kernel_dump_PyTorch.md)；MindSpore 动态图场景详细介绍见 [MindSpore 动态图场景的 kernel dump 说明](./28.kernel_dump_MindSpore.md)；MindSpore 静态图场景详细介绍见《MindSpore 场景的数据采集》中的 ["**8.1 静态图场景**"](./06.data_dump_MindSpore.md#81-静态图场景)小节；
"mix"：dump module 模块级和 API 级精度数据，即"L0"+"L1"，仅 PyTorch、MSAdapter 以及 MindSpore 动态图场景支持。
"debug"：单点保存功能，细节详见[单点保存工具 README](./28.debugger_save_instruction.md)
**配置示例**："level": "L1"。 | 否 | | enable_dataloader | 自动控制开关，bool 类型，仅 PyTorch 场景支持。可选参数 true（开启）或 false（关闭），默认为 false。配置为 true 后自动识别 step 参数指定的迭代，并在该迭代执行完成后退出训练，此时 start、stop 和 step 函数可不配置，开启该开关要求训练脚本是通过 torch.utils.data.dataloader 方式加载数据。仅支持 PyTorch 单卡训练使用，分布式训练场景下存在数据 dump 不全问题。 **这个特性下个版本将被废弃** | 否 | | async_dump | 异步 dump 开关，bool 类型。可选参数 true（开启）或 false（关闭），默认为 false。配置为 true 后开启异步 dump，即采集的精度数据会在当前 step 训练结束后统一落盘，训练过程中工具不触发同步操作。由于使用该模式有**显存溢出**的风险，当 task 配置为 tensor 时，即真实数据的异步dump模式，必须配置 [list](#13-task-配置为-tensor) 参数，指定需要 dump 的 tensor 。该模式暂不支持复数类型 tensor
的统计量计算。 | 否 | #### 1.1.1 模块级精度数据 dump 说明 -仅 PyTorch 与 MindSpore 动态图场景支持。 +仅 PyTorch、MSAdapter以及 MindSpore 动态图场景支持。 大模型场景下，通常不是简单的利用自动迁移能力实现从 GPU 到 NPU 的训练脚本迁移，而是会对 NPU 网络进行一系列针对性的适配，因此，常常会造成迁移后的 NPU 模型存在部分子结构不能与 GPU 原始模型完全对应。模型结构不一致导致 API 调用类型及数量不一致，若直接按照 API 粒度进行精度数据 dump 和比对，则无法完全比对所有的 API。 本小节介绍的功能是对模型中的大粒度模块进行数据 dump，使其比对时，对于无法以 API 粒度比对的模块可以直接以模块粒度进行比对。 -模块指的是继承 nn.Module 类（PyTorch场景）或 nn.Cell 类（MindSpore场景）的子类，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump 数据时以模块为粒度进行 dump。 +模块指的是继承 nn.Module 类（PyTorch 与 MSAdapter 场景）或 nn.Cell 类（MindSpore 场景）的子类，通常情况下这类模块就是一个小模型，可以被视为一个整体，dump 数据时以模块为粒度进行 dump。 @@ -36,21 +36,23 @@ - - - - + + - + - + + +
参数解释是否必选
scopePyTorch 和 MindSpore 动态图场景 dump 范围，list[str] 类型，默认未配置（list 也未配置时表示 dump 所有 API 的数据）。该参数可以在 [ ] 内配置两个模块名或 API 名，要求列表长度必须为2，需要配置按照工具命名格式的完整模块名或API名称，用于锁定区间，dump 该范围内的数据。
配置示例： +
scopePyTorch、MSAdapter 以及 MindSpore 动态图场景 dump 范围，list[str] 类型，默认未配置（list 也未配置时表示 dump 所有 API 的数据）。该参数可以在 [ ] 内配置两个模块名或 API 名，要求列表长度必须为2，需要配置按照工具命名格式的完整模块名或API名称，用于锁定区间，dump 该范围内的数据。
配置示例： "scope": ["Module.conv1.Conv2d.forward.0", "Module.fc2.Linear.forward.0"], 或 "scope": ["Cell.conv1.Conv2d.forward.0", "Cell.fc2.Dense.backward.0"], 或"scope": ["Tensor.add.0.forward", "Functional.square.2.forward"]。与 level 参数取值相关，level 为 L0 级别时，可配置模块名；level 为 L1 级别时，可配置 API 名， level为 mix 级别时，可配置为模块名或API名。		否
list自定义采集的算子列表，list[str] 类型，默认未配置（scope 也未配置时表示 dump 所有 API 的数据），包含以下配置方法：否
PyTorch 和 MindSpore 动态图场景配置具体的 API 全称，dump 该 API 数据。在 PyTorch 场景，如果 level 配置成 L2，该配置为必填项。
配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch 和 MindSpore 动态图场景在level为 mix 级别时可以配置模块名称，dump该模块展开数据 （dump该模块从执行开始到执行结束期间的所有数据）。 +
PyTorch、MSAdapter 以及 MindSpore 动态图场景配置具体的 API 全称，dump 该 API 数据。在 PyTorch 场景，如果 level 配置成 L2，该配置为必填项。
配置示例："list": ["Tensor.permute.1.forward", "Tensor.transpose.2.forward", "Torch.relu.3.backward"]。
PyTorch 和 MindSpore 动态图场景在level为 mix 级别时可以配置模块名称，dump该模块展开数据 （dump该模块从执行开始到执行结束期间的所有数据）。
配置示例："list": ["Module.module.language_model.encoder.layers.0.mlp.ParallelMlp.forward.0"], 或 "list": ["Cell.network_with_loss.language_model.encoder.layers.0.mlp.ParallelMlp.forward.0"]
PyTorch 和 MindSpore 动态图场景指定某一类 API，dump 某一类的 API 级别输入输出数据。
配置示例："list": ["relu"]。
PyTorch 和 MindSpore 动态图场景在level为 mix 级别时, 会dump名称中包含list中配置的字符串的API数据，还会将名称中包含list中配置的字符串的模块进行展开dump （dump该模块从执行开始到执行结束期间的所有数据）。
MindSpore 静态图场景配置 kernel_name，可以是算子的名称列表，也可以指定算子类型（"level": "L2"时不支持），还可以配置算子名称的正则表达式（当字符串符合“name-regex(xxx)”格式时，后台则会将其作为正则表达式。
配置示例：list: ["name-regex(Default/.+)"]
可匹配算子名称以“Default/”开头的所有算子。
PyTorch、MSAdapter 以及 MindSpore 动态图场景指定某一类 API，dump 某一类的 API 级别输入输出数据。
配置示例："list": ["relu"]。
PyTorch、MSAdapter 以及 MindSpore 动态图场景在level为 mix 级别时, 会dump名称中包含list中配置的字符串的API数据，还会将名称中包含list中配置的字符串的模块进行展开dump （dump该模块从执行开始到执行结束期间的所有数据）。
MindSpore 静态图场景配置 kernel_name，可以是算子的名称列表，也可以指定算子类型（jit_level=O2 时不支持），还可以配置算子名称的正则表达式（当字符串符合“name-regex(xxx)”格式时，后台则会将其作为正则表达式。
配置示例：list: ["name-regex(Default/.+)"]
可匹配算子名称以“Default/”开头的所有算子。
data_modedump 数据过滤，str 类型。否
PyTorch 与 MindSpore 动态图场景：支持"all"、"forward"、"backward"、"input"和"output"，除"all"外，其余参数可以自由组合。默认为["all"]，即保存所有 dump 的数据。
配置示例："data_mode": ["backward"] （仅保存反向数据）或 "data_mode": ["forward", "input"]（仅保存前向的输入数据）。
PyTorch、MSAdapter 以及 MindSpore 动态图场景：支持"all"、"forward"、"backward"、"input"和"output"，除"all"外，其余参数可以自由组合。默认为["all"]，即保存所有 dump 的数据。
配置示例："data_mode": ["backward"] （仅保存反向数据）或 "data_mode": ["forward", "input"]（仅保存前向的输入数据）。
MindSpore 静态图场景：仅支持"all"、"input"和"output"参数，且各参数只能单独配置，不支持自由组合。
配置示例："data_mode": ["all"]。
summary_mode控制 dump 文件输出的模式，str 类型，仅 PyTorch 与 MindSpore 动态图场景支持，可选参数：
md5：dump 输出包含 CRC-32 值以及 API 统计信息的 dump.json 文件，用于验证数据的完整性；
statistics：dump 仅输出包含 API 统计信息的 dump.json 文件，默认值。
配置示例："summary_mode": "md5"。		否
MindSpore静态图jit_level=O2场景L2级dump，支持上述配置的同时额外支持配置统计项列表，可选统计项为max、min、mean、l2norm，可从中任意选取组合搭配。其中mean、l2norm的结果为float数据格式。
配置示例："summary_mode": ["max", "min"]。
summary_mode控制 dump 文件输出的模式，str 类型，支持 PyTorch、MSAdapter、MindSpore 动态图以及 MindSpore 静态图 jit_level=O2 场景。否
PyTorch、MSAdapter 以及 MindSpore 动态图场景：可选参数为
md5：dump 输出包含 CRC-32 值以及 API 统计信息的 dump.json 文件，用于验证数据的完整性；
statistics：dump 仅输出包含 API 统计信息的 dump.json 文件，默认值。
配置示例："summary_mode": "md5"。
MindSpore 静态图 jit_level=O2 场景：支持上述配置的同时额外支持配置统计项列表，可选统计项为max、min、mean、l2norm，可从中任意选取组合搭配。其中mean、l2norm的结果为float数据格式。
配置示例："summary_mode": ["max", "min"]。
-**说明**："summary_mode"配置为"md5"时，所使用的校验算法为CRC-32算法。 +**说明**："summary_mode" 配置为 "md5" 时，所使用的校验算法为 CRC-32 算法。 ### 1.3 task 配置为 tensor @@ -86,16 +88,16 @@ ### 1.5 task 配置为 overflow_check -PyTorch 与 MindSpore 动态图场景下，"level"须为"L0"或"L1"；MindSpore 静态图场景下，"level"须为"L2"，且模型编译优化等级（jit_level）须为"O2"。 +PyTorch、MSAdapter 以及 MindSpore 动态图场景下，"level"须为"L0"或"L1"；MindSpore 静态图场景下，"level"须为"L2"，且模型编译优化等级（jit_level）须为"O2"。 | 参数 | 解释 | 是否必选 | | ------------- | ---------------------- | -------- | | overflow_nums | 最大溢出次数，int 类型，默认为 1，仅 PyTorch 与 MindSpore 动态图场景支持。表示第 N 次溢出后，不再进行溢出检测。过程中检测到溢出 API 对应的 输入输出 数据均 dump。
**配置示例**："overflow_nums": 3。配置为 -1 时，表示持续检测溢出直到训练结束。 | 否 | -| check_mode | 溢出类型，str 类型，仅 MindSpore 场景支持，可选参数：
"aicore"：开启 AI Core 的溢出检测，不支持 MindSpore v2.3.0 以上版本；
"atomic"：开启 Atomic 的溢出检测，不支持 MindSpore v2.3.0 以上版本；
"all"：开启算子的溢出检测，默认值。
**配置示例**："check_mode": "all"。 | 否 | +| check_mode | 溢出类型，str 类型，仅 MindSpore v2.3.0 以下版本的静态图场景支持，可选参数：
"aicore"：开启 AI Core 的溢出检测；
"atomic"：开启 Atomic 的溢出检测；
"all"：开启算子的溢出检测，默认值。
**配置示例**："check_mode": "all"。 | 否 | ### 1.6 task 配置为 free_benchmark -仅 PyTorch 场景与 MindSpore 动态图场景支持，且"level"为"L1"。 +仅 PyTorch 与 MindSpore 动态图场景支持，且"level"为"L1"。 - task 配置为 free_benchmark 时，开启**无标杆比对**，在 NPU 环境下通过对当前模型 API 的输入添加扰动因子，二次执行，将得到的输出与未添加扰动因子前的输出进行比对，从而**得出该模型中可能存在因迁移等变化导致精度降低的 API**。 diff --git a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md index c2e33436e5..e45be7736b 100644 --- a/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md +++ b/debug/accuracy_tools/msprobe/docs/05.data_dump_PyTorch.md @@ -355,7 +355,7 @@ if __name__ == "__main__": ``` * `rank`：设备 ID，每张卡的数据保存在对应的 `rank{ID}` 目录下。非分布式场景下没有 rank ID，目录名称为 rank。 * `dump_tensor_data`：保存采集到的张量数据。 -* `dump.json`： 保存API或Module前反向数据的统计量信息。包含dump数据的API名称或Module名称，各数据的dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置summary_mode="md5"时的CRC-32数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#1-dumpjson文件介绍pytorch)。 +* `dump.json`： 保存API或Module前反向数据的统计量信息。包含dump数据的API名称或Module名称，各数据的dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置summary_mode="md5"时的CRC-32数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#1-PyTorch场景下的dump.json文件)。 * `stack.json`：API/Module的调用栈信息。 * `construct.json`：分层分级结构，level为L1时，construct.json内容为空。 diff --git a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md index 96d37c170f..158c5e3011 100644 --- a/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/06.data_dump_MindSpore.md @@ -372,7 +372,7 @@ dump 结果目录结构示例如下： * `rank`：设备 ID，每张卡的数据保存在对应的 `rank{ID}` 目录下。非分布式场景下没有 rank ID，目录名称为 rank。 * `dump_tensor_data`：保存采集到的张量数据。 -* `dump.json`： 保存API或Cell前反向数据的统计量信息。包含dump数据的API名称或Cell名称，各数据的dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置summary_mode="md5"时的CRC-32数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#2-dumpjson文件示例mindspore)。 +* `dump.json`： 保存API或Cell前反向数据的统计量信息。包含dump数据的API名称或Cell名称，各数据的dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置summary_mode="md5"时的CRC-32数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#2-MindSpore场景下的dump.json文件)。 * `stack.json`：API/Cell的调用栈信息。 * `construct.json`：分层分级结构，level为L1时，construct.json内容为空。 diff --git a/debug/accuracy_tools/msprobe/docs/27.dump_json_instruction.md b/debug/accuracy_tools/msprobe/docs/27.dump_json_instruction.md index f994dc2301..bf5998bce0 100644 --- a/debug/accuracy_tools/msprobe/docs/27.dump_json_instruction.md +++ b/debug/accuracy_tools/msprobe/docs/27.dump_json_instruction.md @@ -1,8 +1,8 @@ # dump.json文件说明及示例 -## 1. dump.json文件示例（PyTorch） +## 1. PyTorch 场景下的 dump.json 文件 -### 1.1 L0级别 +### 1.1 L0 级别 L0级别的dump.json文件包括模块的前反向的输入输出，以及模块的参数和参数梯度。以PyTorch的Conv2d模块为例，网络中模块调用代码为: `output = self.conv2(input) # self.conv2 = torch.nn.Conv2d(64, 128, 5, padding=2, bias=True)` @@ -168,7 +168,7 @@ dump.json文件中包含以下数据名称： } ``` -### 1.2 L1级别 +### 1.2 L1 级别 L1级别的dump.json文件包括API的前反向的输入输出。以PyTorch的relu函数为例，网络中API调用代码为: `output = torch.nn.functional.relu(input)` @@ -264,13 +264,13 @@ dump.json文件中包含以下数据名称： } ``` -### 1.3 mix级别 +### 1.3 mix 级别 mix级别的dump.json文件同时包括L0和L1级别的dump数据，文件格式与上述示例相同。 -## 2. dump.json文件示例（MindSpore） +## 2. MindSpore 场景下的 dump.json 文件 -### 2.1 L0级别 +### 2.1 L0 级别 L0级别的dump.json文件包括模块的前反向的输入输出，以及模块的参数和参数梯度。 以MindSpore的Conv2d模块为例，dump.json文件中使用的模块调用代码为: @@ -429,7 +429,7 @@ dump.json文件中包含以下数据名称： } ``` -### 2.2 L1级别 +### 2.2 L1 级别 L1级别的dump.json文件包括API的前反向的输入输出，以MindSpore的relu函数为例，网络中API调用代码为: `output = mindspore.ops.relu(input)` @@ -521,5 +521,275 @@ L1级别的dump.json文件包括API的前反向的输入输出，以MindSpore的 } ``` -### 2.3 mix级别 +### 2.3 mix 级别 + mix级别的dump.json文件同时包括L0和L1级别的dump数据，文件格式与上述示例相同。 + +## 3. MSAdapter 场景下的 dump.json 文件 + +### 3.1 L0 级别 + +L0 级别的 dump.json 文件包括模块的前反向的输入输出，以及模块的参数和参数梯度。以 Conv2d 模块为例，网络中模块调用代码为: +`output = self.conv2(input) # self.conv2 = torch.nn.Conv2d(64, 128, 5, padding=2, bias=True)` + +dump.json文件中包含以下数据名称： + +- `Module.conv2.Conv2d.forward.0`：模块的前向数据，其中input_args为模块的输入数据（位置参数），input_kwargs为模块的输入数据（关键字参数），output为模块的输出数据，parameters为模块的参数数据，包括权重（weight）和偏置（bias）。 +- `Module.conv2.Conv2d.parameters_grad`：模块的参数梯度数据，包括权重（weight）和偏置（bias）的梯度。 +- `Module.conv2.Conv2d.backward.0`：模块的反向数据，其中input为模块反向的输入梯度（对应前向输出的梯度），output为模块的反向输出梯度（对应前向输入的梯度）。 + +**说明**：当dump时传入的model参数为List[torch.nn.Module]或Tuple[torch.nn.Module]时，模块级数据的命名中包含该模块在列表中的索引index，命名格式为`{Module}.{index}.*`，*表示以上三种模块级数据的命名格式，例如：`Module.0.conv1.Conv2d.forward.0`。 + +```json +{ + "task": "tensor", + "level": "L0", + "framework": "mindtorch", + "dump_data_dir": "/dump/path", + "data": { + "Module.conv2.Conv2d.forward.0": { + "input_args": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 8, + 16, + 14, + 14 + ], + "Max": 1.638758659362793, + "Min": 0.0, + "Mean": 0.2544615864753723, + "Norm": 70.50277709960938, + "requires_grad": true, + "data_name": "Module.conv2.Conv2d.forward.0.input.0.npy" + } + ], + "input_kwargs": {}, + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 8, + 32, + 10, + 10 + ], + "Max": 1.6815717220306396, + "Min": -1.5120246410369873, + "Mean": -0.025344856083393097, + "Norm": 149.65576171875, + "requires_grad": true, + "data_name": "Module.conv2.Conv2d.forward.0.output.0.npy" + } + ], + "parameters": { + "weight": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 32, + 16, + 5, + 5 + ], + "Max": 0.05992485210299492, + "Min": -0.05999220535159111, + "Mean": -0.0006165213999338448, + "Norm": 3.421217441558838, + "requires_grad": true, + "data_name": "Module.conv2.Conv2d.forward.0.parameters.weight.npy" + }, + "bias": { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 32 + ], + "Max": 0.05744686722755432, + "Min": -0.04894155263900757, + "Mean": 0.006410328671336174, + "Norm": 0.17263513803482056, + "requires_grad": true, + "data_name": "Module.conv2.Conv2d.forward.0.parameters.bias.npy" + } + } + }, + "Module.conv2.Conv2d.parameters_grad": { + "weight": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 32, + 16, + 5, + 5 + ], + "Max": 0.018550323322415352, + "Min": -0.008627401664853096, + "Mean": 0.0006675920449197292, + "Norm": 0.26084786653518677, + "requires_grad": false, + "data_name": "Module.conv2.Conv2d.parameters_grad.weight.npy" + } + ], + "bias": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 32 + ], + "Max": 0.014914230443537235, + "Min": -0.006656786892563105, + "Mean": 0.002657240955159068, + "Norm": 0.029451673850417137, + "requires_grad": false, + "data_name": "Module.conv2.Conv2d.parameters_grad.bias.npy" + } + ] + }, + "Module.conv2.Conv2d.backward.0": { + "input": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 8, + 32, + 10, + 10 + ], + "Max": 0.0015069986693561077, + "Min": -0.001139344065450132, + "Mean": 3.3215508210560074e-06, + "Norm": 0.020567523315548897, + "requires_grad": false, + "data_name": "Module.conv2.Conv2d.backward.0.input.0.npy" + } + ], + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 8, + 16, + 14, + 14 + ], + "Max": 0.0007466732058674097, + "Min": -0.00044813455315306783, + "Mean": 6.814070275140693e-06, + "Norm": 0.01474067009985447, + "requires_grad": false, + "data_name": "Module.conv2.Conv2d.backward.0.output.0.npy" + } + ] + } + } +} +``` + +### 3.2 L1 级别 +L1级别的dump.json文件包括API的前反向的输入输出。以 relu API 为例，网络中 API 调用代码为: +`output = torch.nn.functional.relu(input)` + +dump.json文件中包含以下数据名称： +- `Functional.relu.0.forward`：API的前向数据，其中input_args为API的输入数据（位置参数），input_kwargs为API的输入数据（关键字参数），output为API的输出数据。 +- `Functional.relu.0.backward`：API的反向数据，其中input为API的反向输入梯度（对应前向输出的梯度），output为API的反向输出梯度（对应前向输入的梯度）。 + +```json +{ + "task": "tensor", + "level": "L1", + "framework": "mindtorch", + "dump_data_dir":"/dump/path", + "data": { + "Functional.relu.0.forward": { + "input_args": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 32, + 16, + 28, + 28 + ], + "Max": 1.3864083290100098, + "Min": -1.3364859819412231, + "Mean": 0.03711778670549393, + "Norm": 236.20692443847656, + "requires_grad": true, + "data_name": "Functional.relu.0.forward.input.0.npy" + } + ], + "input_kwargs": {}, + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 32, + 16, + 28, + 28 + ], + "Max": 1.3864083290100098, + "Min": 0.0, + "Mean": 0.16849493980407715, + "Norm": 175.23345947265625, + "requires_grad": true, + "data_name": "Functional.relu.0.forward.output.0.npy" + } + ] + }, + "Functional.relu.0.backward": { + "input": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 32, + 16, + 28, + 28 + ], + "Max": 0.0001815402356442064, + "Min": -0.00013352684618439525, + "Mean": 0.00011915402356442064, + "Norm": 0.007598237134516239, + "requires_grad": false, + "data_name": "Functional.relu.0.backward.input.0.npy" + } + ], + "output": [ + { + "type": "mindspore.Tensor", + "dtype": "Float32", + "shape": [ + 32, + 16, + 28, + 28 + ], + "Max": 0.0001815402356442064, + "Min": -0.00012117840378778055, + "Mean": 2.0098118724831693e-08, + "Norm": 0.006532244384288788, + "requires_grad": false, + "data_name": "Functional.relu.0.backward.output.0.npy" + } + ] + } + } +} +``` + +### 3.3 mix 级别 + +mix级别的dump.json文件同时包括L0和L1级别的dump数据，文件格式与上述示例相同。 \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/docs/28.kernel_dump_MindSpore.md b/debug/accuracy_tools/msprobe/docs/28.kernel_dump_MindSpore.md index 6b8cc558aa..4988586c05 100644 --- a/debug/accuracy_tools/msprobe/docs/28.kernel_dump_MindSpore.md +++ b/debug/accuracy_tools/msprobe/docs/28.kernel_dump_MindSpore.md @@ -1,4 +1,4 @@ -# MindSpore 场景的 kernel dump 说明 +# MindSpore 动态图场景的 kernel dump 说明 当使用 msprobe 数据采集功能时，level 配置为 "L2" 表示采集 kernel 层级的算子数据，仅支持昇腾 NPU 平台。 diff --git a/debug/accuracy_tools/msprobe/docs/29.data_dump_MSAdapter.md b/debug/accuracy_tools/msprobe/docs/29.data_dump_MSAdapter.md new file mode 100644 index 0000000000..cefcabafbc --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/29.data_dump_MSAdapter.md @@ -0,0 +1,229 @@ +# MSAdapter 场景的精度数据采集 + +MSAdapter 是一款 MindSpore 生态适配工具，可以将 PyTorch 训练脚本高效迁移至 MindSpore 框架执行，以实现在不改变原有 PyTorch 用户开发习惯的情况下，使得 PyTorch 代码能在昇腾上获得高效性能。 + +msprobe 工具主要通过在训练脚本内添加 dump 接口、启动训练的方式采集精度数据。 + +本工具提供固定的 API 支持列表，若需要删除或增加 dump 的 API，可以在 msprobe/pytorch/hook_module/support_wrap_ops.yaml 文件内手动修改，如下示例： + +```yaml +functional: # functional为算子类别，找到对应的类别，在该类别下按照下列格式删除或添加API + - conv1d + - conv2d + - conv3d +``` + +删除 API 的场景：部分模型代码逻辑会存在 API 原生类型校验，工具执行dump操作时，对封装后的模型 API 可能与模型的原生 API 类型不一致，此时可能引发校验失败，详见《[FAQ](FAQ.md)》中“异常情况”的第10和11条。 + +## 1. 工具安装 + +请参见[《msprobe 工具安装指南》](./01.installation.md)。 + +## 2 接口介绍 + +### 2.1 msprobe.mindspore.PrecisionDebugger + +**功能说明**：通过加载 dump 配置文件的方式来确定 dump 操作的详细配置。 + +**原型**： + +```Python +PrecisionDebugger(config_path=None, task=None, dump_path=None, level=None, step=None) +``` + +**参数说明**: + +1. config_path：指定 dump 配置文件路径，string 类型。参数示例："./config.json"。未配置该路径时，默认使用 [config.json](../config.json) 文件的默认配置，配置选项含义可见 [config.json 介绍](./02.config_introduction.md)。 + +2. 其他参数与 [config.json](../config.json) 文件中的同名配置字段含义相同，具体可见 [config.json 介绍](./02.config_introduction.md)。当参数值非None时，优先级高于 [config.json](../config.json) 文件中的同名配置。 + +#### 2.1.1 start + +**功能说明**：启动精度数据采集。需要与 [**stop**](#212-stop) 接口一起添加在训练迭代的 for 循环内。 + +**原型**： + +```Python +start(model=None) +``` + +**参数说明**: + +1. model：指定需要采集 Module 级数据的模型，支持传入 torch.nn.Module、list[torch.nn.Module]或Tuple[torch.nn.Module] 类型，默认未配置。level 配置为 "L0" 或 "mix" 时，必须在该接口中配置该参数。API级别（"L1" level）dump 时，传入 model 可以采集 model 内包含 primitive op 对象在内的所有 API 数据，若不传入 model 参数，则只采集非 primitive op 的 API 数据。 + +#### 2.1.2 stop + +**功能说明**：停止精度数据采集。在 **start** 接口调用之后的任意位置添加。若 **stop** 接口添加在反向计算代码之后，则会采集 **start** 和该接口之间的前反向数据。 +若 **stop** 接口添加在反向计算代码之前，则需要将 [**step**](#213-step) 接口添加到反向计算代码之后，才能采集 **start** 和该接口之间的前反向数据。 + +**注意**：**stop** 接口必须调用，否则可能导致精度数据落盘不全。 + +**原型**： + +```Python +stop() +``` + +#### 2.1.3 step + +**功能说明**：进行训练 step 数的自增，完成当前 step 所有数据的落盘并更新 dump 参数。在一个 step 训练结束的位置添加，且必须在 **stop** 接口之后的位置调用。该接口需要配合 **start** 和 **stop** 函数使用，尽量添加在反向计算代码之后，否则可能会导致反向数据丢失。 + +**原型**： + +```Python +step() +``` + +#### 2.1.4 forward_backward_dump_end + +**功能说明**：停止精度数据采集。与 **stop** 接口功能相同，该函数在将来会被移除，建议使用 **stop** 接口。 + +**原型**： + +```Python +forward_backward_dump_end() +``` + +#### 2.1.5 save + +**功能说明**：单点保存网络执行过程中正反向数值，并以统计值/张量文件落盘。 + +**原型**： +```python +save(variable, name, save_backward=True) +``` + +**参数说明**: +| 参数名称 | 参数含义 | 支持数据类型 | 是否必选| +| ---------- | ------------------| ------------------- | ------------------- | +| variable | 需要保存的变量 |dict, list, tuple, torch.tensor, int, float, str | 是 | +| name | 指定的名称 | str | 是 | +| save_backward | 是否保存反向数据 | boolean | 否 | + +### 2.2 msprobe.mindspore.seed_all + +**功能说明**：用于固定网络中的随机性和开启确定性计算。 + +**原型**： +```python +seed_all(seed=1234, mode=False, rm_dropout=True) +``` + +**参数说明**: + +1. seed: 随机性种子，默认值：1234，非必选。参数示例: seed=1000。该参数用于 random、numpy.random, mindspore.common.Initializer、mindspore.nn.probability.distribution的随机数生成以及 Python 中 str、bytes、datetime 对象的 hash 算法。 + +2. mode：确定性计算使能，可配置 True 或 False，默认值：False，非必选。参数示例：mode=True。该参数设置为 True 后，将会开启算子确定性运行模式与归约类通信算子（AllReduce、ReduceScatter、Reduce）的确定性计算。注意：确定性计算会导致 API 执行性能降低，建议在发现模型多次执行结果不同的情况下开启。 + +3. rm_dropout：控制 dropout 失效的开关。可配置 True 或 False，默认值：True，非必选。参数示例：rm_dropout=True。该参数设置为 True 后，将会使 mindspore.ops.Dropout，mindspore.ops.Dropout2D，mindspore.ops.Dropout3D，mindspore.mint.nn.Dropout和mindspore.mint.nn.functional.dropout 失效，以避免因随机 dropout 造成的网络随机性。建议在采集数据前调用。 + +**注意**：通过 rm_dropout 控制 dropout 失效或生效需要在初始化 Dropout 实例前调用才能生效。 + +## 3 示例代码 + +以下为添加了 msprobe 工具 dump 接口的示例训练脚本。 + +```python +import mindspore as ms +import torch +import torch.nn as nn +import torch.nn.functional as F + +# 导入工具的数据采集接口 +from msprobe.pytorch import PrecisionDebugger + +# 在模型训练开始前实例化PrecisionDebugger +debugger = PrecisionDebugger(config_path='./config.json') + + +# 定义网络 +class Net(nn.Module): + def __init__(self) -> None: + super().__init__() + self.linear1 = nn.Linear(in_features=8, out_features=4) + self.linear2 = nn.Linear(in_features=4, out_features=2) + + def forward(self, x): + x1 = self.linear1(x) + x2 = self.linear2(x1) + logits = F.relu(x2) + return logits + + +net = Net() + + +def train_step(inputs): + return net(inputs) + + +if __name__ == "__main__": + data = (torch.randn(10, 8), torch.randn(10, 8), torch.randn(10, 8)) + grad_fn = ms.value_and_grad(train_step, grad_position=0) + + for inputs in data: + # 开启数据 dump + debugger.start(model=net) + + out, grad = grad_fn(inputs) + + # 停止数据 dump + debugger.stop() + # 更新 step 信息 + debugger.step() +``` + +## 4 dump 结果文件介绍 + +训练结束后，工具将 dump 的数据保存在 dump_path 参数指定的目录下。目录结构示例如下： + +```lua +├── dump_path +│ ├── step0 +│ | ├── rank0 +│ | │ ├── dump_tensor_data +| | | | ├── Tensor.permute.1.forward.npy +| | | | ├── Functional.linear.5.backward.output.npy # 命名格式为{api_type}.{api_name}.{API调用次数}.{forward/backward}.{input/output}.{参数序号}, 其中，“参数序号”表示该API的第n个输入或输出，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该API的第1个参数的第1个元素。 +| | | | ... +| | | | ├── Module.conv1.Conv2d.forward.0.input.0.npy # 命名格式为{Module}.{module_name}.{class_name}.{forward/backward}.{调用次数}.{input/output}.{参数序号}, 其中，“参数序号”表示该Module的第n个参数，例如1，则为第一个参数，若该参数为list格式，则根据list继续排序，例如1.1，表示该Module的第1个参数的第1个元素。 +| | | | ├── Module.conv1.Conv2D.forward.0.parameters.bias.npy # 模块参数数据：命名格式为{Module}.{module_name}.{class_name}.forward.{调用次数}.parameters.{parameter_name}。 +| | | | └── Module.conv1.Conv2D.parameters_grad.weight.npy # 模块参数梯度数据：命名格式为{Module}.{module_name}.{class_name}.parameters_grad.{parameter_name}。因为同一模块的参数使用同一梯度进行更新，所以参数梯度文件名不包含调用次数。 +| | | | # 当dump时传入的model参数为List[torch.nn.Module]或Tuple[torch.nn.Module]时，模块级数据的命名中包含该模块在列表中的索引index，命名格式为{Module}.{index}.*，*表示以上三种模块级数据的命名格式，例如：Module.0.conv1.Conv2d.forward.0.input.0.npy。 +│ | | ├── dump.json +│ | | ├── stack.json +│ | | └── construct.json +│ | ├── rank1 +| | | ├── dump_tensor_data +| | | | └── ... +│ | | ├── dump.json +│ | | ├── stack.json +| | | └── construct.json +│ | ├── ... +│ | | +| | └── rank7 +│ ├── step1 +│ | ├── ... +│ ├── step2 +``` +* `rank`：设备 ID，每张卡的数据保存在对应的 `rank{ID}` 目录下。非分布式场景下没有 rank ID，目录名称为 rank。 +* `dump_tensor_data`：保存采集到的张量数据。 +* `dump.json`： 保存 API 或 Module 前反向数据的统计量信息。包含 dump 数据的 API 名称或 Module 名称，各数据的 dtype、 shape、max、min、mean、L2norm（L2范数，平方根）统计信息以及当配置 summary_mode="md5" 时的 CRC-32 数据。具体介绍可参考[dump.json文件说明](./27.dump_json_instruction.md#3-MSAdapter场景下的dump.json文件)。 +* `stack.json`：API/Module 的调用栈信息。 +* `construct.json`：分层分级结构，level 为 L1 时，construct.json 内容为空。 + + +当 task 为 tensor 时，dump 过程中，npy 文件在对应算子或者模块被执行后就会落盘，而 json 文件则需要在正常执行 PrecisionDebugger.stop() 后才会写入完整数据。因此如果程序异常终止，终止前被执行算子的相关 npy 文件得以保存，但 json 文件中的数据可能丢失。 + +其中 rank 为设备上各卡的 ID，每张卡上 dump 的数据会生成对应 dump 目录。非分布式场景下没有 rank ID，目录名称为 rank。 + +npy 文件名的前缀含义如下： + +| 前缀 | 含义 | +| ----------- | ---------------------------- | +| Tensor | torch.Tensor API数据 | +| Torch | torch API数据 | +| Functional | torch.nn.functional API数据 | +| NPU | NPU 亲和API数据 | +| Distributed | torch.distributed API数据 | +| Jit | 被 "jit" 装饰的模块或函数数据 | +| Module | torch.nn.Module 类（模块）数据 | \ No newline at end of file diff --git a/debug/accuracy_tools/msprobe/docs/30.overflow_check_MSAdapter.md b/debug/accuracy_tools/msprobe/docs/30.overflow_check_MSAdapter.md new file mode 100644 index 0000000000..01d64c808d --- /dev/null +++ b/debug/accuracy_tools/msprobe/docs/30.overflow_check_MSAdapter.md @@ -0,0 +1,31 @@ +# MSAdapter 场景的溢出检测 + +msprobe 工具提供 MSAdapter 场景下的溢出检测功能。其检测对象为 **API** 级别(除 Primitive 和 Jit 类 API)或**模块**级别，分别对应 config.json 配置中的 **"L1"** 、**"L0"** level。 + +需要注意，本工具仅支持在 INF/NAN 模式a下进行溢出检测。INF/NAN 模式的使能方式如下： + +```Shell +# 使能 CANN 侧 INF/NAN 模式 +export INF_NAN_MODE_ENABLE=1 +# 使能 MindSpore 框架侧 INF/NAN 模式 +export MS_ASCEND_CHECK_OVERFLOW_MODE="INFNAN_MODE" +``` + +**a**：在处理浮点数计算溢出问题时，NPU 当前支持两种溢出模式：INF/NAN 模式与饱和模式。INF/NAN 模式遵循 IEEE 754 标准，根据定义输出 INF/NAN 的计算结果。与之对应的饱和模式在计算出现溢出时，饱和为浮点数极值（+-MAX）。对于 CANN 侧配置，Atlas 训练系列产品，默认为饱和模式，且不建议使用 INF/NAN 模式；Atlas A2训练系列产品，默认为 INF/NAN 模式，且不建议使用饱和模式。对于 MindSpore 框架侧配置，仅支持对 Atlas A2 训练系列产品进行设置，默认为 INF/NAN 模式。CANN 侧 与 MindSpore 框架侧配置须一致。 + +溢出检测任务的配置示例见["**MindSpore 动态图场景 task 配置为 overflow_check**"](./03.config_examples.md#33-task配置为overflow_check)小节。 + + +## 1 接口介绍 + +溢出检测功能提供的接口与数据采集任务一致，详见 MSAdapter 场景的精度数据采集中的["**2 接口介绍**"](./29.data_dump_MSAdapter.md#2-接口介绍)小节。 + +需要注意，目前暂不支持 "L1" level 下 primitive op 的溢出检测。 + +## 2 示例代码 + +溢出检测功能使用方式与数据采集任务一致，详见 MSAdapter 场景的精度数据采集中的["**3 示例代码**"](./29.data_dump_MSAdapter.md#3-示例代码)小节。 + +## 3 溢出检测结果文件介绍 + +溢出检测结果文件目录结构与含义与数据采集任务一致，但仅保存溢出 API 或 模块 的真实数据或统计信息。详见 MSAdapter 场景的精度数据采集中的["**4 dump 结果文件介绍**"](./29.data_dump_MSAdapter.md#4-dump-结果文件介绍)小节。 \ No newline at end of file -- Gitee From 7c5583a76df1dc539f2d5e8971654aa82f9fea51 Mon Sep 17 00:00:00 2001 From: fanglanyue Date: Mon, 3 Mar 2025 18:03:09 +0800 Subject: [PATCH 31/37] dynolog_npu use glog --- .../plugin/ipc_monitor/DynoLogNpuMonitor.cpp | 12 ++------- .../plugin/ipc_monitor/NpuIpcClient.cpp | 21 +++++++-------- .../ipc_monitor/PyDynamicMonitorProxy.h | 26 ++++++++++++------- dynolog_npu/plugin/ipc_monitor/utils.cpp | 4 +-- dynolog_npu/plugin/ipc_monitor/utils.h | 2 +- dynolog_npu/plugin/setup.py | 15 ++++++----- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.cpp b/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.cpp index 940f5aae16..bba66d7297 100644 --- a/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.cpp +++ b/dynolog_npu/plugin/ipc_monitor/DynoLogNpuMonitor.cpp @@ -1,7 +1,4 @@ #include "DynoLogNpuMonitor.h" - -#include - #include "utils.h" namespace dynolog_npu { @@ -10,13 +7,13 @@ namespace ipc_monitor { bool DynoLogNpuMonitor::Init() { if (isInitialized_) { - std::cout << "[WRARNING] DynoLog npu monitor already initialized" << std::endl; + LOG(ERROR) << "DynoLog npu monitor already initialized"; return true; } bool res = ipcClient_.RegisterInstance(npuId_); if (res) { isInitialized_ = true; - std::cout << "[INFO] DynoLog npu monitor initialized success !" << std::endl; + LOG(INFO) << "DynoLog npu monitor initialized success!"; } return res; } @@ -24,11 +21,6 @@ bool DynoLogNpuMonitor::Init() std::string DynoLogNpuMonitor::Poll() { std::string res = ipcClient_.IpcClientNpuConfig(); - if (res.empty()) { - std::cout << "[INFO] Request for dynolog server is empty !" << std::endl; - return ""; - } - std::cout << "[INFO] Received NPU configuration successfully" << std::endl; return res; } diff --git a/dynolog_npu/plugin/ipc_monitor/NpuIpcClient.cpp b/dynolog_npu/plugin/ipc_monitor/NpuIpcClient.cpp index 97966e8eea..ca2429f1e3 100644 --- a/dynolog_npu/plugin/ipc_monitor/NpuIpcClient.cpp +++ b/dynolog_npu/plugin/ipc_monitor/NpuIpcClient.cpp @@ -1,6 +1,5 @@ #include "NpuIpcClient.h" -#include namespace dynolog_npu { namespace ipc_monitor { @@ -15,14 +14,14 @@ bool IpcClient::RegisterInstance(int32_t id) std::unique_ptr message = Message::ConstructMessage(context, "ctxt"); try { if (!SyncSendMessage(*message, std::string(DYNO_IPC_NAME))) { - std::cout << "[WARNING]Failed to send register ctxt for pid " << context.pid << " with dyno" << std::endl; + LOG(ERROR) << "Failed to send register ctxt for pid " << context.pid << " with dyno"; return false; } } catch (const std::exception &e) { - std::cout << "[WARNING] Error when SyncSendMessage: " << e.what() << std::endl; + LOG(ERROR) << " Error when SyncSendMessage: " << e.what(); return false; } - std::cout << "[INFO] Resigter pid " << context.pid << " for dynolog success !" << std::endl; + LOG(INFO) << "Resigter pid " << context.pid << " for dynolog success !"; return true; } std::string IpcClient::IpcClientNpuConfig() @@ -37,7 +36,7 @@ std::string IpcClient::IpcClientNpuConfig() } std::unique_ptr message = Message::ConstructMessage(*req, "req", size); if (!SyncSendMessage(*message, std::string(DYNO_IPC_NAME))) { - std::cout << "[WARNING] Failed to send config to dyno server fail !" << std::endl; + LOG(ERROR) << " Failed to send config to dyno server fail !"; free(req); req = nullptr; return ""; @@ -45,7 +44,7 @@ std::string IpcClient::IpcClientNpuConfig() free(req); message = PollRecvMessage(MAX_IPC_RETRIES, MAX_SLEEP_US); if (!message) { - std::cout << "[WARNING] Failed to receive on-demand config !" << std::endl; + LOG(ERROR) << " Failed to receive on-demand config !"; return ""; } std::string res = std::string(ReinterpretConvert(message->buf.get()), message->metadata.size); @@ -65,7 +64,7 @@ std::unique_ptr IpcClient::ReceiveMessage() bool IpcClient::SyncSendMessage(const Message &message, const std::string &destName, int numRetry, int seepTimeUs) { if (destName.empty()) { - std::cout << "[WARNING] Can not send to empty socket name !" << std::endl; + LOG(ERROR) << " Can not send to empty socket name !"; return false; } int i = 0; @@ -79,7 +78,7 @@ bool IpcClient::SyncSendMessage(const Message &message, const std::string &destN seepTimeUs *= 2; // 2: double sleep time } } catch (const std::exception &e) { - std::cout << "[ERROR] Error when SyncSendMessage: " << e.what() << std::endl; + LOG(ERROR) << " Error when SyncSendMessage: " << e.what(); return false; } return i < numRetry; @@ -94,7 +93,7 @@ bool IpcClient::Recv() try { successFlag = ep_.TryPeekMessage(*peekCtxt); } catch (std::exception &e) { - std::cout << "[ERROR] Error when TryPeekMessage: " << e.what() << std::endl; + LOG(ERROR) << " Error when TryPeekMessage: " << e.what(); return false; } if (successFlag) { @@ -108,7 +107,7 @@ bool IpcClient::Recv() try { successFlag = ep_.TryRcvMessage(*recvCtxt); } catch (std::exception &e) { - std::cout << "[ERROR] Error when TryRecvMsg: " << e.what() << std::endl; + LOG(ERROR) << " Error when TryRecvMsg: " << e.what(); return false; } if (successFlag) { @@ -118,7 +117,7 @@ bool IpcClient::Recv() } } } catch (std::exception &e) { - std::cout << "[ERROR] Error in Recv(): " << e.what() << std::endl; + LOG(ERROR) << " Error in Recv(): " << e.what(); return false; } return false; diff --git a/dynolog_npu/plugin/ipc_monitor/PyDynamicMonitorProxy.h b/dynolog_npu/plugin/ipc_monitor/PyDynamicMonitorProxy.h index 8b5f88abf9..0471a70a34 100644 --- a/dynolog_npu/plugin/ipc_monitor/PyDynamicMonitorProxy.h +++ b/dynolog_npu/plugin/ipc_monitor/PyDynamicMonitorProxy.h @@ -1,7 +1,7 @@ #ifndef PYDYNAMIC_MONITOR_PROXY_H #define PYDYNAMIC_MONITOR_PROXY_H -#include +#include #include #include "MonitorBase.h" #include "DynoLogNpuMonitor.h" @@ -14,15 +14,21 @@ public: PyDynamicMonitorProxy() = default; bool InitDyno(int npuId) { - try { - monitor_ = DynoLogNpuMonitor::GetInstance(); - monitor_->SetNpuId(npuId); - bool res = monitor_->Init(); - return res; - } catch (const std::exception &e) { - std::cout << "[ERROR] Error when init dyno " << e.what() << std::endl; - return false; - } + try { + if (!google::IsGoogleLoggingInitialized()) { + google::InitGoogleLogging("DynoLogNpuMonitor"); + google::SetLogDestination(google::GLOG_INFO, "/var/log/dynolog_npu_"); + google::SetLogFilenameExtension(".log"); + } + monitor_ = DynoLogNpuMonitor::GetInstance(); + monitor_->SetNpuId(npuId); + bool res = monitor_->Init(); + LOG(ERROR) << res; + return res; + } catch (const std::exception &e) { + LOG(ERROR) << "Error when init dyno " << e.what(); + return false; + } } std::string PollDyno() diff --git a/dynolog_npu/plugin/ipc_monitor/utils.cpp b/dynolog_npu/plugin/ipc_monitor/utils.cpp index 936821fd34..b57942082e 100644 --- a/dynolog_npu/plugin/ipc_monitor/utils.cpp +++ b/dynolog_npu/plugin/ipc_monitor/utils.cpp @@ -68,11 +68,11 @@ std::pair GetParentPidAndCommand(int32_t pid) if (std::getline(statFile, line)) { int ret = sscanf(line.c_str(), "%*d (%[^)]) %*c %d", command.data(), &parentPid); if (ret == 2) { // 2: 接收到2个字符 - std::cout << "[INFO] Success to get parent pid: " << parentPid << std::endl; + LOG(INFO) << "Success to get parent pid: " << parentPid; return std::make_pair(parentPid, command); } } - std::cout << "[WARNING] Failed to parse /proc/" << pid << "/stat" << std::endl; + LOG(ERROR) << " Failed to parse /proc/" << pid << "/stat"; return std::make_pair(0, ""); } diff --git a/dynolog_npu/plugin/ipc_monitor/utils.h b/dynolog_npu/plugin/ipc_monitor/utils.h index 0d8ceb8cfd..2374a27d41 100644 --- a/dynolog_npu/plugin/ipc_monitor/utils.h +++ b/dynolog_npu/plugin/ipc_monitor/utils.h @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include diff --git a/dynolog_npu/plugin/setup.py b/dynolog_npu/plugin/setup.py index 151b9b3fb3..55e924c6b6 100644 --- a/dynolog_npu/plugin/setup.py +++ b/dynolog_npu/plugin/setup.py @@ -13,25 +13,28 @@ # See the License for the specific language governing permissions and # limitations under the License. import os +from glob import glob from setuptools import setup from pybind11.setup_helpers import Pybind11Extension BASE_DIR = os.path.dirname(os.path.realpath(__file__)) +DYNOLOG_PATH = os.path.join(os.path.dirname(BASE_DIR), "third_party", "dynolog") +GLOG_INC_PATH = os.path.join(DYNOLOG_PATH, "third_party", "glog", "src") +GLOG_LIB_PATH = os.path.join(DYNOLOG_PATH, "build", "third_party", "glog") # Define the extension module ext_modules = [ Pybind11Extension( "IPCMonitor", # Name of the Python module - sources=["bindings.cpp", - "ipc_monitor/utils.cpp", - "ipc_monitor/DynoLogNpuMonitor.cpp", - "ipc_monitor/NpuIpcClient.cpp", - ], # Source files - include_dirs=[os.path.join(BASE_DIR, "ipc_monitor")], # Include Pybind11 headers + sources=["bindings.cpp"] + list(glob("ipc_monitor/*.cpp")), # Source files + include_dirs=[os.path.join(BASE_DIR, "ipc_monitor"), GLOG_INC_PATH, GLOG_LIB_PATH], # Include Pybind11 headers + library_dirs=[GLOG_LIB_PATH], + libraries=["glog"], language="c++", # Specify the language ), ] + # Set up the package setup( name="dynolog_npu_plugin", -- Gitee From c88f9641a3aaa2e743acae2e8f075b2c9fa396d6 Mon Sep 17 00:00:00 2001 From: curry3 <485078529@qq.com> Date: Mon, 3 Mar 2025 16:24:29 +0800 Subject: [PATCH 32/37] =?UTF-8?q?=E3=80=90bugfix=E3=80=91=E4=BF=AE?= =?UTF-8?q?=E5=A4=8D=E6=B2=A1=E6=9C=89=E5=AE=9E=E9=99=85=E5=85=83=E7=B4=A0?= =?UTF-8?q?=E7=9A=84tensor=E8=AE=A1=E7=AE=97=E7=BB=9F=E8=AE=A1=E9=87=8F?= =?UTF-8?q?=E6=8A=A5=E9=94=99=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../core/data_dump/data_processor/pytorch_processor.py | 2 +- .../data_dump/data_processor/test_pytorch_processor.py | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py index e93b86af20..c3c0c3aa73 100644 --- a/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/core/data_dump/data_processor/pytorch_processor.py @@ -145,7 +145,7 @@ class PytorchDataProcessor(BaseDataProcessor): if data.is_meta: return tensor_stat data_clone = data.detach() - if data_clone.numel() == 0: + if not data_clone.numel() or not data_clone.data_ptr(): return tensor_stat else: if data_clone.device.type == Const.CPU_LOWERCASE or not async_dump: diff --git a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py index 34064e7cc2..3d31a1bb51 100644 --- a/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py +++ b/debug/accuracy_tools/msprobe/test/core_ut/data_dump/data_processor/test_pytorch_processor.py @@ -19,6 +19,7 @@ from msprobe.core.data_dump.data_processor.pytorch_processor import ( KernelDumpDataProcessor ) from torch import distributed as dist +from torch._subclasses import FakeTensorMode class TestPytorchDataProcessor(unittest.TestCase): @@ -62,6 +63,15 @@ class TestPytorchDataProcessor(unittest.TestCase): result = PytorchDataProcessor.get_stat_info(mock_data) self.assertIsInstance(result, TensorStatInfo) + def test_get_stat_info_with_fake_tensor(self): + with FakeTensorMode() as fake_tensor_mode: + fake_tensor = fake_tensor_mode.from_tensor(torch.randn(1, 2, 3)) + result = PytorchDataProcessor.get_stat_info(fake_tensor) + self.assertIsNone(result.max) + self.assertIsNone(result.min) + self.assertIsNone(result.mean) + self.assertIsNone(result.norm) + def test_get_stat_info_float(self): tensor = torch.tensor([1.0, 2.0, 3.0]) result = self.processor.get_stat_info(tensor) -- Gitee From 28aebd021f5ac92dca38b0eeb72733604386b4e3 Mon Sep 17 00:00:00 2001 From: Mrtutu Date: Tue, 4 Mar 2025 14:58:11 +0800 Subject: [PATCH 33/37] add plugin docs --- dynolog_npu/README.md | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/dynolog_npu/README.md b/dynolog_npu/README.md index d6ebd6f7ff..86a23b7f82 100644 --- a/dynolog_npu/README.md +++ b/dynolog_npu/README.md @@ -51,6 +51,8 @@ sudo yum install -y cmake ninja ### 3. 编译 +- dynolog编译 + 默认编译生成dyno和dynolog二进制文件, -t参数可以支持将二进制文件打包成deb包或rpm包. ```bash @@ -64,6 +66,10 @@ bash scripts/build.sh -t deb bash scripts/build.sh -t rpm ``` +- dynolog_npu_plugin wheel包编译 + +dynolog_npu_plugin wheel包提供IPCMonitor，MsptiMonitor等公共能力，使用nputrace和npu-monitor功能前必须安装该wheel包，具体编译安装指导可参考dynolog_npu\plugin\README.md。 + ## 使用方式 ### Profiler trace dump功能 @@ -112,7 +118,9 @@ nputrace子命令支持的参数选项 - nputrace使用方法 -Step1： 拉起dynolog daemon进程 +Step0: 参考`3.编译`章节完成dynolog的编译，以及dynolog_npu_plugin wheel包的编译和安装。 + +Step1：拉起dynolog daemon进程 ```bash # 方法1：使用systemd拉起service # 修改配置文件/etc/dynolog.gflags, 使能ipc_monitor -- Gitee From 64bc545648bf8db8e3c1248e5716870c5ecc3f24 Mon Sep 17 00:00:00 2001 From: zhouxianqi <13165993773@163.com> Date: Mon, 3 Mar 2025 18:47:34 +0800 Subject: [PATCH 34/37] bug_fix_for_matrix --- .../analysis/comm_matrix_analysis.py | 69 ++++++++++++++----- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py index 2ad5797cc9..3839fe66aa 100644 --- a/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py +++ b/profiler/msprof_analyze/cluster_analyse/analysis/comm_matrix_analysis.py @@ -22,6 +22,8 @@ from msprof_analyze.prof_common.db_manager import DBManager from msprof_analyze.cluster_analyse.common_func.utils import increase_shared_value from msprof_analyze.prof_common.constant import Constant from msprof_analyze.prof_common.logger import get_logger +from msprof_analyze.cluster_analyse.common_func.utils import double_hash +from msprof_analyze.prof_common.file_manager import FileManager logger = get_logger() @@ -70,30 +72,46 @@ class CommMatrixAnalysis(BaseAnalysis): self.combine_link_info(step_dict) def merge_same_links(self, step_dict: dict): - def process_link_key(rank_id, rank_dict): + def update_rank_map(step_dict): + for op_name, op_dict in step_dict.items(): + group_name = op_name.split("@")[-1] + for rank_id, rank_dict in op_dict.items(): + for link_key in rank_dict: + if '-' not in link_key: + logger.warning("%s has an invalid link key %s!", str(op_name), str(link_key)) + break + src_rank = link_key.split('-')[0] + dst_rank = link_key.split('-')[1] + if src_rank == dst_rank: + if src_rank not in project_local_global_rank_map.get(group_name, {}): + project_local_global_rank_map.setdefault(group_name, {})[src_rank] = rank_id + elif project_local_global_rank_map.get(group_name, {}).get(src_rank) != rank_id: + logger.warning(f"In the same communication group {group_name}, global rank {rank_id} " + f"and {project_local_global_rank_map.get(group_name, {}).get(src_rank)} " + f"get the same local rank {src_rank}!") + + def process_link_key(rank_dict): for link_key in rank_dict: if '-' not in link_key: logger.warning("%s has an invalid link key %s!", str(op_name), str(link_key)) break - src_rank = link_key.split('-')[0] - dst_rank = link_key.split('-')[1] - if src_rank == dst_rank: - if src_rank not in project_local_global_rank_map: - project_local_global_rank_map[src_rank] = rank_id - elif project_local_global_rank_map.get(src_rank) != rank_id: - logger.warning("In the same communication group, local ranks projecting to global ranks " - "repeat!") self.combine_link(link_info[link_key], rank_dict[link_key]) - def convert_local_to_global_rank(): + def convert_local_to_global_rank(rank_map): tmp_link = {} for link_key, link_dict in link_info.items(): src_rank = link_key.split('-')[0] dst_rank = link_key.split('-')[1] - src_rank = project_local_global_rank_map[src_rank] \ - if src_rank in project_local_global_rank_map else src_rank - dst_rank = project_local_global_rank_map[dst_rank] \ - if dst_rank in project_local_global_rank_map else dst_rank + if src_rank not in rank_map: + logger.warning(f"The src local rank {src_rank} of the operator {op_name} " + f"cannot be mapped to the global rank.") + continue + if dst_rank not in rank_map: + logger.warning(f"The dst local rank {dst_rank} of the operator {op_name} " + f"cannot be mapped to the global rank.") + continue + src_rank = rank_map[src_rank] + dst_rank = rank_map[dst_rank] link_dict[Constant.BANDWIDTH_GB_S] = \ self.compute_ratio(link_dict.get(Constant.TRANSIT_SIZE_MB, 0), link_dict.get(Constant.TRANSIT_TIME_MS, 0)) @@ -106,12 +124,14 @@ class CommMatrixAnalysis(BaseAnalysis): Constant.TRANSIT_SIZE_MB: 0, Constant.OP_NAME: '' } + project_local_global_rank_map = self.get_parallel_group_info() + update_rank_map(step_dict) for op_name, op_dict in step_dict.items(): link_info = defaultdict(lambda: copy.deepcopy(default_value)) - project_local_global_rank_map = dict() - for rank_id, rank_dict in op_dict.items(): - process_link_key(rank_id, rank_dict) - step_dict[op_name] = convert_local_to_global_rank() + group_name = op_name.split("@")[-1] + for rank_dict in op_dict.values(): + process_link_key(rank_dict) + step_dict[op_name] = convert_local_to_global_rank(project_local_global_rank_map.get(group_name, {})) def combine_link_info(self, step_dict: dict): default_value = { @@ -131,6 +151,19 @@ class CommMatrixAnalysis(BaseAnalysis): link_dict.get(Constant.TRANSIT_TIME_MS, 0)) step_dict[Constant.TOTAL_OP_INFO] = total_op_info + def get_parallel_group_info(self): + parallel_group_info = {} + for profiler_path in self.data_map.values(): + meta_json = os.path.join(profiler_path, "profiler_metadata.json") + if os.path.exists(meta_json): + meta_data = FileManager.read_json_file(meta_json) + for group_name, group_info in meta_data.get("parallel_group_info", {}).items(): + global_ranks = group_info.get("global_ranks") + if isinstance(global_ranks, list) and global_ranks: + global_ranks.sort() + parallel_group_info[double_hash(group_name)] = dict(enumerate(global_ranks)) + return parallel_group_info + class CommMatrixAnalysisOptimized(CommMatrixAnalysis): SAVED_JSON = "cluster_communication_matrix.json" -- Gitee From 7dd59c45d265553f7cb5dcc8dc77dfc3de4719b5 Mon Sep 17 00:00:00 2001 From: RanZheng <364167184@qq.com> Date: Fri, 28 Feb 2025 17:50:07 +0800 Subject: [PATCH 35/37] =?UTF-8?q?=E5=85=BC=E5=AE=B9=E8=80=81=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3=EF=BC=8C=E8=B0=83=E6=95=B4=E5=87=BD=E6=95=B0=E8=B0=83?= =?UTF-8?q?=E7=94=A8=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/docs/19.monitor.md | 63 ++++- .../msprobe/pytorch/monitor/module_hook.py | 262 +++++++++--------- 2 files changed, 200 insertions(+), 125 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/19.monitor.md b/debug/accuracy_tools/msprobe/docs/19.monitor.md index 1c197ba549..fe22738d64 100644 --- a/debug/accuracy_tools/msprobe/docs/19.monitor.md +++ b/debug/accuracy_tools/msprobe/docs/19.monitor.md @@ -107,6 +107,37 @@ monitor.set_monitor( ) ``` +请注意以下两点： +- Mindspore功能在1.2.2版本后支持, <1.2.2版本不支持 +- 上述接口使用方式为1.2.2后更新的最新接口使用方式, <1.2.2版本的Pytorch旧接口使用方式为： +```Python +from msprobe.pytorch import TrainerMon +monitor = TrainerMon( + config_file_path="./monitor_config.json", + params_have_main_grad=True, # 权重是否使用main_grad，通常megatron为True，deepspeed为False。默认为True。 + opt_ty=None # 优化器类型，默认为None，具体取值参考公开接口 +) +monitor.set_wrapped_optimizer(optimizer) +# 挂载监控对象 +monitor.monitor_gnorm_with_ad( + model, + grad_acc_steps=args.global_batch_size//args.data_parallel_size//args.micro_batch_size, + optimizer=optimizer, + dp_group=None, + tp_group=None, + start_iteration=0 # 断点续训时提供当前iteration，默认从0开始 +) +``` + +具体接口变更说明如下： + +| 变更 | 说明 | +|-----------|-----------------------------------------------------------------------------------------------------------| +| 初始化接口统一精简 | TrainerMon.__init__(config_file_path, process_group=None, param_have_main_grad=True), 去除了需用户手动传入的opt_ty参数 | +| 主调接口修改 | 从monitor_gnorm_with_ad(...)改名为set_monitor(...)， 且此时optimizer从可选项改为必传项 | +| 优化器包装接口废除 | 接口废除， optimizer传入由set_monitor主调完成 | + +其中老版接口目前仍能使用，但预计将在2026年废弃，请及时更新到最新版使用方式 ### 权重监控 - 工具配置示例： @@ -451,7 +482,7 @@ TrainerMon.set_monitor(model, grad_acc_steps, optimizer, dp_group=None, tp_group | --------------- | ------------------------------------------------------------ | -------- | | model | 需要监控的模型，需要是一个torch.nn.Module或者mindspore.nn.Cell。 | 是 | | grad_acc_steps | 梯度累积步数。 | 是 | -| optimizer | 需要patch的优化器。 | 否 | +| optimizer | 需要patch的优化器。 | 是 | | dp_group | 数据并行的通信组。
dp域通信后，且没有使用分布式优化器时，group内所有rank的梯度相同，落盘数据冗余。
提供dp_group后，工具仅保留每个dp_group的第一个rank的梯度。 | 否 | | tp_group | 张量并行的通信组。
tp域通信后，group内部分参数所有rank的梯度相同，落盘数据冗余。
提供tp_group后，工具仅保留每个tp_group中冗余参数在第一个rank的梯度。
当前适配Megatron core_r0.6.0, 通过权重属性"tensor_model_parallel"判断是否冗余。 | 否 | | start_iteration | 训练的起始iteration，影响工具计数。**仅PyTorch场景支持此参数**。 | 否 | @@ -486,6 +517,36 @@ TrainerMon.generate_xy_metrics() -> tuple[dict, dict] actv, actv_grad = monitor.generate_xy_metrics() ``` +- 老版接口说明， 将在26年废弃： +```python +TrainerMon.__init__(config_file_path, process_group=None, params_have_main_grad=True, opt_ty=None) -> None +``` +| 参数 | 说明 | 是否必选 | +|-----------------------|--------------------------------------------------------------------------------------------------------------------------------| -------- | +| config_file_path | json配置文件路径。 | 是 | +| process_group | 传入ProcessGroup对象，用以确定pipeline并行不同rank异常间时序，megatron下通过core.parallel_state.get_pipeline_model_parallel_group()获得。仅在异常时序判断功能中使用。 | 否 | +| params_have_main_grad | 权重是否使用main_grad，通常megatron为True，deepspeed为False。默认为True。 | 否 | +| opt_ty | 优化器类型，默认为None。
-Megatron_DistributedOptimizer：megatron分布式优化器；
-Megatron_Float16OptimizerWithFloat16Params：megatron混合精度优化器；
-Megatron_ChainedDistributedOptimizer：megatron分布式优化器序列；
-Megatron_ChainedFloat16OptimizerWithFloat16Params：megatron混合精度优化器序列；
-DeepSpeedZeroOptimizer_Stage1_or_2：DeepSpeed Zero1和Zero2；
-DeepSpeedZeroOptimizer_Stage3：DeepSpeed Zero3。 || + +```python +TrainerMon.set_wrapped_optimizer(optimizer) -> None +``` +| 参数 | 说明 | 是否必选 | +|-----------|-------------------------------|------| +| optimizer | megatron、deepspeed创建好的混合精度优化器 | 是 | + +```python +TrainerMon.monitor_gnorm_with_ad(model, grad_acc_steps, optimizer, dp_group, tp_group, start_iteration) -> None +``` +| 参数 | 说明 | 是否必选 | +| --------------- | ------------------------------------------------------------ | -------- | +| model | 需要监控的模型，需要是一个torch.nn.Module或者mindspore.nn.Cell。 | 是 | +| grad_acc_steps | 梯度累积步数。 | 是 | +| optimizer | 需要patch的优化器。 | 否 | +| dp_group | 数据并行的通信组。
dp域通信后，且没有使用分布式优化器时，group内所有rank的梯度相同，落盘数据冗余。
提供dp_group后，工具仅保留每个dp_group的第一个rank的梯度。 | 否 | +| tp_group | 张量并行的通信组。
tp域通信后，group内部分参数所有rank的梯度相同，落盘数据冗余。
提供tp_group后，工具仅保留每个tp_group中冗余参数在第一个rank的梯度。
当前适配Megatron core_r0.6.0, 通过权重属性"tensor_model_parallel"判断是否冗余。 | 否 | +| start_iteration | 训练的起始iteration，影响工具计数。**仅PyTorch场景支持此参数**。 | 否 | + ## 详细配置 diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py b/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py index 0c9efaab99..6ab15216eb 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py @@ -176,7 +176,8 @@ class GradContext: class TrainerMon: tensor_metrics = TensorMetrics() - def __init__(self, config_file_path, process_group=None, params_have_main_grad=True) -> None: + # 保留原opt_ty参数，兼容msprobe1.2.2前旧版本 + def __init__(self, config_file_path, process_group=None, params_have_main_grad=True, opt_ty=None) -> None: # TYPE1: 只在这里初始化的变量, 不会随着训练中途config配置改变而重置 self.config_file_path = config_file_path self.process_group = get_process_group(process_group) @@ -379,45 +380,19 @@ class TrainerMon: if not self.cc_distribution.get('enable', False): logger.info_on_rank_0("> cc operator is not monitored.") - def hook_modules(self): - if self.module_rank_list and (self.rank not in self.module_rank_list): - return - - targets = self.config['targets'] - module_in_all_stage = [key for key in targets.keys() if MonitorConst.NAME_SEP not in key] - for key in module_in_all_stage: - struct = targets.pop(key) - targets.update({f'{vpp_stage}{MonitorConst.NAME_SEP}{key}': struct for vpp_stage in range(len(self.model))}) - - hooked_count = 0 - for vpp_stage, model_chunk in enumerate(self.model): - vpp_stage = f'{vpp_stage}{MonitorConst.NAME_SEP}' - targets = [x for x, _ in model_chunk.named_modules()] if self.print_struct else self.config[ - 'targets'].keys() - hooked_count += self._hook_module(targets, model_chunk, vpp_stage) - - logger.info_on_rank_0(f"> {hooked_count} modules are monitored.") - - def clone_if_tensor(args): - if isinstance(args, tuple): - return tuple([clone_if_tensor(arg) for arg in args]) - elif isinstance(args, torch.Tensor): - return args.clone() - else: - return args - - @torch.no_grad - def wrap_hook_setup(setup): - def wrapped_setup(*args, **kwargs): - args = setup(*args, **kwargs) - args = clone_if_tensor(args) - return args - - return wrapped_setup + # 保留原接口，兼容msprobe1.2.2前旧版本 + def monitor_gnorm_with_ad(self, model, optimizer=None, grad_acc_steps=1, tp_group=None, dp_group=None, + start_iteration=0): + if optimizer is None: + optimizer = getattr(self, 'optimizer_trans', None) # 兼容老版本可传None的情况, 从set_wrapped_optimizer获取 + if optimizer is None: + logger.error("monitor_gnorm_with_ad: please set_wrapped_optimizer before it or input optimizer!=None") + return + self.set_monitor(model, optimizer, grad_acc_steps, tp_group, dp_group, start_iteration) - BackwardHook.setup_input_hook = wrap_hook_setup(BackwardHook.setup_input_hook) - BackwardHook.setup_output_hook = wrap_hook_setup(BackwardHook.setup_output_hook) - return + # 保留原接口，兼容msprobe1.2.2前旧版本 + def set_wrapped_optimizer(self, optimizer): + self.optimizer_trans = optimizer def set_monitor( self, @@ -557,9 +532,9 @@ class TrainerMon: def write_mv_tb(self, opt_context): if not self.mv_distribution: return - self.summary_writer.write_metrics(self.ops, opt_context.exp_avg_metric, + self.summary_writer.write_metrics(self.ops, opt_context.exp_avg_metric, opt_context.step, MonitorConst.EXP_AVG) - self.summary_writer.write_metrics(self.ops, opt_context.exp_avg_sq_metric, + self.summary_writer.write_metrics(self.ops, opt_context.exp_avg_sq_metric, opt_context.step, MonitorConst.EXP_AVG_SQ) def write_grad_tb(self, step): @@ -572,6 +547,98 @@ class TrainerMon: self.summary_writer.write_metrics(self.ops, self.grad_context.acc_metric, step, 'grad_unreduced') self.summary_writer.write_metrics(self.ops, self.grad_context.post, step, 'grad_reduced') + def hook_step_final(self, optimizer): + def step_final_hook(optimizer, args, kwargs): + context = self.optimizer_context[optimizer] + rank = dist.get_rank() if dist.is_initialized() else None + # 静态在第0步就可以保存, 动态在第0步不可以, 因为动态设计的就是重置后下一步开启, 第0步的self.monitoring还是False + if self.monitoring: + module_rank_valid = not self.module_rank_list or ( + dist.is_initialized() and dist.get_rank() in self.module_rank_list) + step_condition = (context.step >= self.start_step and ( + context.step - self.start_step) % self.step_interval == 0) + if module_rank_valid and step_condition: + self.has_collect_times += 1 + + if self.anomaly_data_factory: + self.anomaly_data_factory.set_call_id(self.param_name_call_id) + self.write_xy_tb(context.step) + self.write_grad_tb(context.step) + self.write_mv_tb(context) + self.write_param_tb(context) + self.write_adhoc_check(context.step) + + if self.ur_distribution: + for param_name, _ in context.param_adam_update.items(): + self.update_heatmap_visualizer[param_name].visualize( + get_summary_writer_tag_name(param_name, 'adam_update', rank), context.step, + self.summary_writer) + for param_name, _ in context.param_adam_ratio.items(): + self.ratio_heatmap_visualizer[param_name].visualize( + get_summary_writer_tag_name(param_name, 'adam_ratio', rank), context.step, + self.summary_writer) + + if context.metric_dict: + self.summary_writer.write_metrics(self.ops, context.metric_dict, context.step, 'other') + context.metric_dict.clear() + + if self.anomaly_data_factory: + self.anomaly_data_writer.write_detected_json(self.summary_writer.get_anomalies()) + self.summary_writer.clear_anomalies() + self.call_id = 0 + self.param_name_call_id.clear() + + if self.has_collect_times >= self.collect_times: + self._remove_all_hooks_final(optimizer) + + context.step += 1 + self.dynamic_monitor(optimizer) + + def patch_step(func, optimizer): + def wrapper(*args, **kwargs): + out = func(*args, **kwargs) + step_final_hook(optimizer, args, kwargs) + return out + return wrapper + + optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) + self.origin_step_func = optimizer.__class__.step + return + + def dynamic_monitor(self, optimizer): + """ + If dynamic monitor enabled and config.json updated, + remove hooks and register new hooks according to new configuration. + """ + context = self.optimizer_context[optimizer] + if not self.dynamic_enable: + return + try: + # 如果文件时间戳没变, 可以不读取节省时间 + config_timestamp = os.path.getmtime(self.config_file_path) + if config_timestamp == self.config_timestamp: + return + # 更新config文件最新修改时间戳 + self.config_timestamp = config_timestamp + config = load_json(self.config_file_path) + except Exception as e: + logger.error(f"get config.json wrong because {e}, not updated, please check!!!") + return + + if config.get("dynamic_on", False): + try: + validate_config(config) + self.config = config + self.set_config() + logger.warning(f"config is updated at step{context.step - 1}, " + f"will start new hook at step{context.step}.") + except Exception as e: + logger.error(f"set config wrong because {e}, not updated, please check!!!") + return + + self._remove_all_hooks(optimizer) + self.register_hooks(optimizer) + def hook_optimizer(self, optimizer): # in DDP by default use params_have_main_grad def optimizer_pre_step_hook(optimizer, args, kwargs): @@ -648,97 +715,44 @@ class TrainerMon: self.optimizer_hooked = True return - def dynamic_monitor(self, optimizer): - """ - If dynamic monitor enabled and config.json updated, - remove hooks and register new hooks according to new configuration. - """ - context = self.optimizer_context[optimizer] - if not self.dynamic_enable: - return - try: - # 如果文件时间戳没变, 可以不读取节省时间 - config_timestamp = os.path.getmtime(self.config_file_path) - if config_timestamp == self.config_timestamp: - return - # 更新config文件最新修改时间戳 - self.config_timestamp = config_timestamp - config = load_json(self.config_file_path) - except Exception as e: - logger.error(f"get config.json wrong because {e}, not updated, please check!!!") + def hook_modules(self): + if self.module_rank_list and (self.rank not in self.module_rank_list): return - if config.get("dynamic_on", False): - try: - validate_config(config) - self.config = config - self.set_config() - logger.warning(f"config is updated at step{context.step - 1}, " - f"will start new hook at step{context.step}.") - except Exception as e: - logger.error(f"set config wrong because {e}, not updated, please check!!!") - return - - self._remove_all_hooks(optimizer) - self.register_hooks(optimizer) - - def hook_step_final(self, optimizer): - def step_final_hook(optimizer, args, kwargs): - context = self.optimizer_context[optimizer] - rank = dist.get_rank() if dist.is_initialized() else None - # 静态在第0步就可以保存, 动态在第0步不可以, 因为动态设计的就是重置后下一步开启, 第0步的self.monitoring还是False - if self.monitoring: - module_rank_valid = not self.module_rank_list or ( - dist.is_initialized() and dist.get_rank() in self.module_rank_list) - step_condition = (context.step >= self.start_step and ( - context.step - self.start_step) % self.step_interval == 0) - if module_rank_valid and step_condition: - self.has_collect_times += 1 - - if self.anomaly_data_factory: - self.anomaly_data_factory.set_call_id(self.param_name_call_id) - self.write_xy_tb(context.step) - self.write_grad_tb(context.step) - self.write_mv_tb(context) - self.write_param_tb(context) - self.write_adhoc_check(context.step) - - if self.ur_distribution: - for param_name, _ in context.param_adam_update.items(): - self.update_heatmap_visualizer[param_name].visualize( - get_summary_writer_tag_name(param_name, 'adam_update', rank), context.step, - self.summary_writer) - for param_name, _ in context.param_adam_ratio.items(): - self.ratio_heatmap_visualizer[param_name].visualize( - get_summary_writer_tag_name(param_name, 'adam_ratio', rank), context.step, - self.summary_writer) - - if context.metric_dict: - self.summary_writer.write_metrics(self.ops, context.metric_dict, context.step, 'other') - context.metric_dict.clear() + targets = self.config['targets'] + module_in_all_stage = [key for key in targets.keys() if MonitorConst.NAME_SEP not in key] + for key in module_in_all_stage: + struct = targets.pop(key) + targets.update({f'{vpp_stage}{MonitorConst.NAME_SEP}{key}': struct for vpp_stage in range(len(self.model))}) - if self.anomaly_data_factory: - self.anomaly_data_writer.write_detected_json(self.summary_writer.get_anomalies()) - self.summary_writer.clear_anomalies() - self.call_id = 0 - self.param_name_call_id.clear() + hooked_count = 0 + for vpp_stage, model_chunk in enumerate(self.model): + vpp_stage = f'{vpp_stage}{MonitorConst.NAME_SEP}' + targets = [x for x, _ in model_chunk.named_modules()] if self.print_struct else self.config[ + 'targets'].keys() + hooked_count += self._hook_module(targets, model_chunk, vpp_stage) - if self.has_collect_times >= self.collect_times: - self._remove_all_hooks_final(optimizer) + logger.info_on_rank_0(f"> {hooked_count} modules are monitored.") - context.step += 1 - self.dynamic_monitor(optimizer) + def clone_if_tensor(args): + if isinstance(args, tuple): + return tuple([clone_if_tensor(arg) for arg in args]) + elif isinstance(args, torch.Tensor): + return args.clone() + else: + return args - def patch_step(func, optimizer): - def wrapper(*args, **kwargs): - out = func(*args, **kwargs) - step_final_hook(optimizer, args, kwargs) - return out - return wrapper + @torch.no_grad + def wrap_hook_setup(setup): + def wrapped_setup(*args, **kwargs): + args = setup(*args, **kwargs) + args = clone_if_tensor(args) + return args - optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) - self.origin_step_func = optimizer.__class__.step + return wrapped_setup + BackwardHook.setup_input_hook = wrap_hook_setup(BackwardHook.setup_input_hook) + BackwardHook.setup_output_hook = wrap_hook_setup(BackwardHook.setup_output_hook) return def _remove_all_hooks(self, optimizer): -- Gitee From 33a495dc60385d3313c5b00a4dc78802bfd813f2 Mon Sep 17 00:00:00 2001 From: RanZheng <364167184@qq.com> Date: Tue, 4 Mar 2025 15:28:59 +0800 Subject: [PATCH 36/37] =?UTF-8?q?torch=E5=85=BC=E5=AE=B9=E8=80=81=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3+mindspore=E4=BF=AE=E5=A4=8Dmodule=5Franks=E4=B8=BAint?= =?UTF-8?q?=E7=B1=BB=E5=9E=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../accuracy_tools/msprobe/docs/19.monitor.md | 1 - .../monitor/distributed/wrap_distributed.py | 2 +- .../msprobe/mindspore/monitor/utils.py | 4 +- .../msprobe/pytorch/monitor/module_hook.py | 184 +++++++++--------- 4 files changed, 95 insertions(+), 96 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/19.monitor.md b/debug/accuracy_tools/msprobe/docs/19.monitor.md index fe22738d64..3e957c5052 100644 --- a/debug/accuracy_tools/msprobe/docs/19.monitor.md +++ b/debug/accuracy_tools/msprobe/docs/19.monitor.md @@ -548,7 +548,6 @@ TrainerMon.monitor_gnorm_with_ad(model, grad_acc_steps, optimizer, dp_group, tp_ | start_iteration | 训练的起始iteration，影响工具计数。**仅PyTorch场景支持此参数**。 | 否 | - ## 详细配置 ```json diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/distributed/wrap_distributed.py b/debug/accuracy_tools/msprobe/mindspore/monitor/distributed/wrap_distributed.py index 33fd58c727..e8a4739445 100644 --- a/debug/accuracy_tools/msprobe/mindspore/monitor/distributed/wrap_distributed.py +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/distributed/wrap_distributed.py @@ -281,7 +281,7 @@ def create_hooks(context, monitor): global RANK pre_hooks = [] hooks = [] - RANK = str(get_rank()) + RANK = get_rank() if communication.GlobalComm.INITED and RANK not in monitor.module_rank_list and monitor.module_rank_list != []: return [pre_hooks, hooks] diff --git a/debug/accuracy_tools/msprobe/mindspore/monitor/utils.py b/debug/accuracy_tools/msprobe/mindspore/monitor/utils.py index 506ad6c3f9..c85e66a65b 100644 --- a/debug/accuracy_tools/msprobe/mindspore/monitor/utils.py +++ b/debug/accuracy_tools/msprobe/mindspore/monitor/utils.py @@ -98,8 +98,8 @@ def validate_ranks(ranks): if not isinstance(ranks, list): raise TypeError("module_ranks should be a list") for rank in ranks: - if not isinstance(rank, str): - raise TypeError(f"element in module_ranks should be a str, get {type(rank)}") + if not isinstance(rank, int): + raise TypeError(f"element in module_ranks should be a int, get {type(rank)}") def validate_targets(targets): diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py b/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py index 6ab15216eb..777e52a617 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py @@ -547,98 +547,6 @@ class TrainerMon: self.summary_writer.write_metrics(self.ops, self.grad_context.acc_metric, step, 'grad_unreduced') self.summary_writer.write_metrics(self.ops, self.grad_context.post, step, 'grad_reduced') - def hook_step_final(self, optimizer): - def step_final_hook(optimizer, args, kwargs): - context = self.optimizer_context[optimizer] - rank = dist.get_rank() if dist.is_initialized() else None - # 静态在第0步就可以保存, 动态在第0步不可以, 因为动态设计的就是重置后下一步开启, 第0步的self.monitoring还是False - if self.monitoring: - module_rank_valid = not self.module_rank_list or ( - dist.is_initialized() and dist.get_rank() in self.module_rank_list) - step_condition = (context.step >= self.start_step and ( - context.step - self.start_step) % self.step_interval == 0) - if module_rank_valid and step_condition: - self.has_collect_times += 1 - - if self.anomaly_data_factory: - self.anomaly_data_factory.set_call_id(self.param_name_call_id) - self.write_xy_tb(context.step) - self.write_grad_tb(context.step) - self.write_mv_tb(context) - self.write_param_tb(context) - self.write_adhoc_check(context.step) - - if self.ur_distribution: - for param_name, _ in context.param_adam_update.items(): - self.update_heatmap_visualizer[param_name].visualize( - get_summary_writer_tag_name(param_name, 'adam_update', rank), context.step, - self.summary_writer) - for param_name, _ in context.param_adam_ratio.items(): - self.ratio_heatmap_visualizer[param_name].visualize( - get_summary_writer_tag_name(param_name, 'adam_ratio', rank), context.step, - self.summary_writer) - - if context.metric_dict: - self.summary_writer.write_metrics(self.ops, context.metric_dict, context.step, 'other') - context.metric_dict.clear() - - if self.anomaly_data_factory: - self.anomaly_data_writer.write_detected_json(self.summary_writer.get_anomalies()) - self.summary_writer.clear_anomalies() - self.call_id = 0 - self.param_name_call_id.clear() - - if self.has_collect_times >= self.collect_times: - self._remove_all_hooks_final(optimizer) - - context.step += 1 - self.dynamic_monitor(optimizer) - - def patch_step(func, optimizer): - def wrapper(*args, **kwargs): - out = func(*args, **kwargs) - step_final_hook(optimizer, args, kwargs) - return out - return wrapper - - optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) - self.origin_step_func = optimizer.__class__.step - return - - def dynamic_monitor(self, optimizer): - """ - If dynamic monitor enabled and config.json updated, - remove hooks and register new hooks according to new configuration. - """ - context = self.optimizer_context[optimizer] - if not self.dynamic_enable: - return - try: - # 如果文件时间戳没变, 可以不读取节省时间 - config_timestamp = os.path.getmtime(self.config_file_path) - if config_timestamp == self.config_timestamp: - return - # 更新config文件最新修改时间戳 - self.config_timestamp = config_timestamp - config = load_json(self.config_file_path) - except Exception as e: - logger.error(f"get config.json wrong because {e}, not updated, please check!!!") - return - - if config.get("dynamic_on", False): - try: - validate_config(config) - self.config = config - self.set_config() - logger.warning(f"config is updated at step{context.step - 1}, " - f"will start new hook at step{context.step}.") - except Exception as e: - logger.error(f"set config wrong because {e}, not updated, please check!!!") - return - - self._remove_all_hooks(optimizer) - self.register_hooks(optimizer) - def hook_optimizer(self, optimizer): # in DDP by default use params_have_main_grad def optimizer_pre_step_hook(optimizer, args, kwargs): @@ -715,6 +623,98 @@ class TrainerMon: self.optimizer_hooked = True return + def dynamic_monitor(self, optimizer): + """ + If dynamic monitor enabled and config.json updated, + remove hooks and register new hooks according to new configuration. + """ + context = self.optimizer_context[optimizer] + if not self.dynamic_enable: + return + try: + # 如果文件时间戳没变, 可以不读取节省时间 + config_timestamp = os.path.getmtime(self.config_file_path) + if config_timestamp == self.config_timestamp: + return + # 更新config文件最新修改时间戳 + self.config_timestamp = config_timestamp + config = load_json(self.config_file_path) + except Exception as e: + logger.error(f"get config.json wrong because {e}, not updated, please check!!!") + return + + if config.get("dynamic_on", False): + try: + validate_config(config) + self.config = config + self.set_config() + logger.warning(f"config is updated at step{context.step - 1}, " + f"will start new hook at step{context.step}.") + except Exception as e: + logger.error(f"set config wrong because {e}, not updated, please check!!!") + return + + self._remove_all_hooks(optimizer) + self.register_hooks(optimizer) + + def hook_step_final(self, optimizer): + def step_final_hook(optimizer, args, kwargs): + context = self.optimizer_context[optimizer] + rank = dist.get_rank() if dist.is_initialized() else None + # 静态在第0步就可以保存, 动态在第0步不可以, 因为动态设计的就是重置后下一步开启, 第0步的self.monitoring还是False + if self.monitoring: + module_rank_valid = not self.module_rank_list or ( + dist.is_initialized() and dist.get_rank() in self.module_rank_list) + step_condition = (context.step >= self.start_step and ( + context.step - self.start_step) % self.step_interval == 0) + if module_rank_valid and step_condition: + self.has_collect_times += 1 + + if self.anomaly_data_factory: + self.anomaly_data_factory.set_call_id(self.param_name_call_id) + self.write_xy_tb(context.step) + self.write_grad_tb(context.step) + self.write_mv_tb(context) + self.write_param_tb(context) + self.write_adhoc_check(context.step) + + if self.ur_distribution: + for param_name, _ in context.param_adam_update.items(): + self.update_heatmap_visualizer[param_name].visualize( + get_summary_writer_tag_name(param_name, 'adam_update', rank), context.step, + self.summary_writer) + for param_name, _ in context.param_adam_ratio.items(): + self.ratio_heatmap_visualizer[param_name].visualize( + get_summary_writer_tag_name(param_name, 'adam_ratio', rank), context.step, + self.summary_writer) + + if context.metric_dict: + self.summary_writer.write_metrics(self.ops, context.metric_dict, context.step, 'other') + context.metric_dict.clear() + + if self.anomaly_data_factory: + self.anomaly_data_writer.write_detected_json(self.summary_writer.get_anomalies()) + self.summary_writer.clear_anomalies() + self.call_id = 0 + self.param_name_call_id.clear() + + if self.has_collect_times >= self.collect_times: + self._remove_all_hooks_final(optimizer) + + context.step += 1 + self.dynamic_monitor(optimizer) + + def patch_step(func, optimizer): + def wrapper(*args, **kwargs): + out = func(*args, **kwargs) + step_final_hook(optimizer, args, kwargs) + return out + return wrapper + + optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) + self.origin_step_func = optimizer.__class__.step + return + def hook_modules(self): if self.module_rank_list and (self.rank not in self.module_rank_list): return -- Gitee From 5d8e1d8344ed4b79c1b399cf857b4bca49f03cd0 Mon Sep 17 00:00:00 2001 From: RanZheng <364167184@qq.com> Date: Mon, 3 Mar 2025 10:11:14 +0800 Subject: [PATCH 37/37] =?UTF-8?q?Revert=20"=E5=85=BC=E5=AE=B9=E8=80=81?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=EF=BC=8C=E8=B0=83=E6=95=B4=E5=87=BD=E6=95=B0?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E9=A1=BA=E5=BA=8F"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 8b1226f2709d7864f0c90fc3737ecb8e2961b970. --- .../accuracy_tools/msprobe/docs/19.monitor.md | 63 +------------ .../msprobe/pytorch/monitor/module_hook.py | 94 ++++++++----------- 2 files changed, 41 insertions(+), 116 deletions(-) diff --git a/debug/accuracy_tools/msprobe/docs/19.monitor.md b/debug/accuracy_tools/msprobe/docs/19.monitor.md index 3e957c5052..4bb82af8c7 100644 --- a/debug/accuracy_tools/msprobe/docs/19.monitor.md +++ b/debug/accuracy_tools/msprobe/docs/19.monitor.md @@ -107,37 +107,6 @@ monitor.set_monitor( ) ``` -请注意以下两点： -- Mindspore功能在1.2.2版本后支持, <1.2.2版本不支持 -- 上述接口使用方式为1.2.2后更新的最新接口使用方式, <1.2.2版本的Pytorch旧接口使用方式为： -```Python -from msprobe.pytorch import TrainerMon -monitor = TrainerMon( - config_file_path="./monitor_config.json", - params_have_main_grad=True, # 权重是否使用main_grad，通常megatron为True，deepspeed为False。默认为True。 - opt_ty=None # 优化器类型，默认为None，具体取值参考公开接口 -) -monitor.set_wrapped_optimizer(optimizer) -# 挂载监控对象 -monitor.monitor_gnorm_with_ad( - model, - grad_acc_steps=args.global_batch_size//args.data_parallel_size//args.micro_batch_size, - optimizer=optimizer, - dp_group=None, - tp_group=None, - start_iteration=0 # 断点续训时提供当前iteration，默认从0开始 -) -``` - -具体接口变更说明如下： - -| 变更 | 说明 | -|-----------|-----------------------------------------------------------------------------------------------------------| -| 初始化接口统一精简 | TrainerMon.__init__(config_file_path, process_group=None, param_have_main_grad=True), 去除了需用户手动传入的opt_ty参数 | -| 主调接口修改 | 从monitor_gnorm_with_ad(...)改名为set_monitor(...)， 且此时optimizer从可选项改为必传项 | -| 优化器包装接口废除 | 接口废除， optimizer传入由set_monitor主调完成 | - -其中老版接口目前仍能使用，但预计将在2026年废弃，请及时更新到最新版使用方式 ### 权重监控 - 工具配置示例： @@ -482,7 +451,7 @@ TrainerMon.set_monitor(model, grad_acc_steps, optimizer, dp_group=None, tp_group | --------------- | ------------------------------------------------------------ | -------- | | model | 需要监控的模型，需要是一个torch.nn.Module或者mindspore.nn.Cell。 | 是 | | grad_acc_steps | 梯度累积步数。 | 是 | -| optimizer | 需要patch的优化器。 | 是 | +| optimizer | 需要patch的优化器。 | 否 | | dp_group | 数据并行的通信组。
dp域通信后，且没有使用分布式优化器时，group内所有rank的梯度相同，落盘数据冗余。
提供dp_group后，工具仅保留每个dp_group的第一个rank的梯度。 | 否 | | tp_group | 张量并行的通信组。
tp域通信后，group内部分参数所有rank的梯度相同，落盘数据冗余。
提供tp_group后，工具仅保留每个tp_group中冗余参数在第一个rank的梯度。
当前适配Megatron core_r0.6.0, 通过权重属性"tensor_model_parallel"判断是否冗余。 | 否 | | start_iteration | 训练的起始iteration，影响工具计数。**仅PyTorch场景支持此参数**。 | 否 | @@ -517,36 +486,6 @@ TrainerMon.generate_xy_metrics() -> tuple[dict, dict] actv, actv_grad = monitor.generate_xy_metrics() ``` -- 老版接口说明， 将在26年废弃： -```python -TrainerMon.__init__(config_file_path, process_group=None, params_have_main_grad=True, opt_ty=None) -> None -``` -| 参数 | 说明 | 是否必选 | -|-----------------------|--------------------------------------------------------------------------------------------------------------------------------| -------- | -| config_file_path | json配置文件路径。 | 是 | -| process_group | 传入ProcessGroup对象，用以确定pipeline并行不同rank异常间时序，megatron下通过core.parallel_state.get_pipeline_model_parallel_group()获得。仅在异常时序判断功能中使用。 | 否 | -| params_have_main_grad | 权重是否使用main_grad，通常megatron为True，deepspeed为False。默认为True。 | 否 | -| opt_ty | 优化器类型，默认为None。
-Megatron_DistributedOptimizer：megatron分布式优化器；
-Megatron_Float16OptimizerWithFloat16Params：megatron混合精度优化器；
-Megatron_ChainedDistributedOptimizer：megatron分布式优化器序列；
-Megatron_ChainedFloat16OptimizerWithFloat16Params：megatron混合精度优化器序列；
-DeepSpeedZeroOptimizer_Stage1_or_2：DeepSpeed Zero1和Zero2；
-DeepSpeedZeroOptimizer_Stage3：DeepSpeed Zero3。 || - -```python -TrainerMon.set_wrapped_optimizer(optimizer) -> None -``` -| 参数 | 说明 | 是否必选 | -|-----------|-------------------------------|------| -| optimizer | megatron、deepspeed创建好的混合精度优化器 | 是 | - -```python -TrainerMon.monitor_gnorm_with_ad(model, grad_acc_steps, optimizer, dp_group, tp_group, start_iteration) -> None -``` -| 参数 | 说明 | 是否必选 | -| --------------- | ------------------------------------------------------------ | -------- | -| model | 需要监控的模型，需要是一个torch.nn.Module或者mindspore.nn.Cell。 | 是 | -| grad_acc_steps | 梯度累积步数。 | 是 | -| optimizer | 需要patch的优化器。 | 否 | -| dp_group | 数据并行的通信组。
dp域通信后，且没有使用分布式优化器时，group内所有rank的梯度相同，落盘数据冗余。
提供dp_group后，工具仅保留每个dp_group的第一个rank的梯度。 | 否 | -| tp_group | 张量并行的通信组。
tp域通信后，group内部分参数所有rank的梯度相同，落盘数据冗余。
提供tp_group后，工具仅保留每个tp_group中冗余参数在第一个rank的梯度。
当前适配Megatron core_r0.6.0, 通过权重属性"tensor_model_parallel"判断是否冗余。 | 否 | -| start_iteration | 训练的起始iteration，影响工具计数。**仅PyTorch场景支持此参数**。 | 否 | - ## 详细配置 diff --git a/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py b/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py index 777e52a617..ad0fba4639 100644 --- a/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py +++ b/debug/accuracy_tools/msprobe/pytorch/monitor/module_hook.py @@ -176,8 +176,7 @@ class GradContext: class TrainerMon: tensor_metrics = TensorMetrics() - # 保留原opt_ty参数，兼容msprobe1.2.2前旧版本 - def __init__(self, config_file_path, process_group=None, params_have_main_grad=True, opt_ty=None) -> None: + def __init__(self, config_file_path, process_group=None, params_have_main_grad=True) -> None: # TYPE1: 只在这里初始化的变量, 不会随着训练中途config配置改变而重置 self.config_file_path = config_file_path self.process_group = get_process_group(process_group) @@ -380,19 +379,45 @@ class TrainerMon: if not self.cc_distribution.get('enable', False): logger.info_on_rank_0("> cc operator is not monitored.") - # 保留原接口，兼容msprobe1.2.2前旧版本 - def monitor_gnorm_with_ad(self, model, optimizer=None, grad_acc_steps=1, tp_group=None, dp_group=None, - start_iteration=0): - if optimizer is None: - optimizer = getattr(self, 'optimizer_trans', None) # 兼容老版本可传None的情况, 从set_wrapped_optimizer获取 - if optimizer is None: - logger.error("monitor_gnorm_with_ad: please set_wrapped_optimizer before it or input optimizer!=None") - return - self.set_monitor(model, optimizer, grad_acc_steps, tp_group, dp_group, start_iteration) + def hook_modules(self): + if self.module_rank_list and (self.rank not in self.module_rank_list): + return + + targets = self.config['targets'] + module_in_all_stage = [key for key in targets.keys() if MonitorConst.NAME_SEP not in key] + for key in module_in_all_stage: + struct = targets.pop(key) + targets.update({f'{vpp_stage}{MonitorConst.NAME_SEP}{key}': struct for vpp_stage in range(len(self.model))}) + + hooked_count = 0 + for vpp_stage, model_chunk in enumerate(self.model): + vpp_stage = f'{vpp_stage}{MonitorConst.NAME_SEP}' + targets = [x for x, _ in model_chunk.named_modules()] if self.print_struct else self.config[ + 'targets'].keys() + hooked_count += self._hook_module(targets, model_chunk, vpp_stage) + + logger.info_on_rank_0(f"> {hooked_count} modules are monitored.") + + def clone_if_tensor(args): + if isinstance(args, tuple): + return tuple([clone_if_tensor(arg) for arg in args]) + elif isinstance(args, torch.Tensor): + return args.clone() + else: + return args + + @torch.no_grad + def wrap_hook_setup(setup): + def wrapped_setup(*args, **kwargs): + args = setup(*args, **kwargs) + args = clone_if_tensor(args) + return args + + return wrapped_setup - # 保留原接口，兼容msprobe1.2.2前旧版本 - def set_wrapped_optimizer(self, optimizer): - self.optimizer_trans = optimizer + BackwardHook.setup_input_hook = wrap_hook_setup(BackwardHook.setup_input_hook) + BackwardHook.setup_output_hook = wrap_hook_setup(BackwardHook.setup_output_hook) + return def set_monitor( self, @@ -713,46 +738,7 @@ class TrainerMon: optimizer.__class__.step = patch_step(optimizer.__class__.step, optimizer) self.origin_step_func = optimizer.__class__.step - return - def hook_modules(self): - if self.module_rank_list and (self.rank not in self.module_rank_list): - return - - targets = self.config['targets'] - module_in_all_stage = [key for key in targets.keys() if MonitorConst.NAME_SEP not in key] - for key in module_in_all_stage: - struct = targets.pop(key) - targets.update({f'{vpp_stage}{MonitorConst.NAME_SEP}{key}': struct for vpp_stage in range(len(self.model))}) - - hooked_count = 0 - for vpp_stage, model_chunk in enumerate(self.model): - vpp_stage = f'{vpp_stage}{MonitorConst.NAME_SEP}' - targets = [x for x, _ in model_chunk.named_modules()] if self.print_struct else self.config[ - 'targets'].keys() - hooked_count += self._hook_module(targets, model_chunk, vpp_stage) - - logger.info_on_rank_0(f"> {hooked_count} modules are monitored.") - - def clone_if_tensor(args): - if isinstance(args, tuple): - return tuple([clone_if_tensor(arg) for arg in args]) - elif isinstance(args, torch.Tensor): - return args.clone() - else: - return args - - @torch.no_grad - def wrap_hook_setup(setup): - def wrapped_setup(*args, **kwargs): - args = setup(*args, **kwargs) - args = clone_if_tensor(args) - return args - - return wrapped_setup - - BackwardHook.setup_input_hook = wrap_hook_setup(BackwardHook.setup_input_hook) - BackwardHook.setup_output_hook = wrap_hook_setup(BackwardHook.setup_output_hook) return def _remove_all_hooks(self, optimizer): @@ -1065,7 +1051,7 @@ class TrainerMon: self.enable_megatron = True logger.info("megatron version is > core_r0.8.0 <= core_r0.9.0") except ImportError: - self.enable_megatron = False | self.enable_megatron + self.enable_megatron = False if not self.enable_megatron: self._hook_weights() -- Gitee